Source code for trinity.algorithm.advantage_fn.on_policy_distill_advantage

# -*- coding: utf-8 -*-
"""On-Policy Distillation advantage computation.

Reference: Tinker library's on-policy distillation.

advantages = -(student_logprobs - teacher_logprobs)
           = teacher_logprobs - student_logprobs
"""

from typing import Dict, Tuple

from verl import DataProto

from trinity.algorithm.advantage_fn.advantage_fn import AdvantageFn


[docs] class OnPolicyDistillAdvantage(AdvantageFn): """Advantage function for on-policy distillation. Computes: advantages = kl_coef * (teacher_logprobs - student_logprobs) The teacher_logprobs should be stored in Experience.teacher_logprobs by the workflow during exploration. """
[docs] def __init__(self, kl_coef: float = 1.0) -> None: self.kl_coef = kl_coef
def __call__(self, exps: DataProto, **kwargs) -> Tuple[DataProto, Dict]: """Compute advantages from teacher and student logprobs. Args: exps: DataProto containing: - old_log_probs: student's sampling logprobs [batch, seq] - teacher_logprobs: teacher's logprobs [batch, seq] - response_mask: mask for response tokens [batch, seq] Returns: exps: DataProto with advantages and returns added metrics: Dict with kl and advantage statistics """ metrics = {} old_log_probs = exps.batch["old_log_probs"] # student sampling logprobs teacher_log_probs = exps.batch["teacher_logprobs"] response_mask = exps.batch["response_mask"] # advantages = -(student - teacher) = teacher - student advantages = self.kl_coef * (teacher_log_probs - old_log_probs) # Apply mask advantages = advantages * response_mask exps.batch["advantages"] = advantages exps.batch["returns"] = advantages.clone() # Metrics kl_per_token = old_log_probs - teacher_log_probs kl_sum = (kl_per_token * response_mask).sum(dim=-1) metrics["kl/mean"] = kl_sum.mean().item() metrics["kl/std"] = kl_sum.std().item() if kl_sum.numel() > 1 else 0.0 metrics["advantages/mean"] = advantages.sum(dim=-1).mean().item() return exps, metrics
[docs] @classmethod def default_args(cls) -> Dict: return {"kl_coef": 1.0}