Source code for trinity.algorithm.advantage_fn.clipv_advantage

"""GRPO advantage computation with Clip_V token filtering.
"""

from collections import defaultdict
from typing import TYPE_CHECKING, Dict, Tuple

import torch

if TYPE_CHECKING:
    from verl import DataProto

from trinity.algorithm.advantage_fn.advantage_fn import AdvantageFn


[docs] class ClipVAdvantageFn(AdvantageFn): """Clip_V advantage: one-side clip only negative-advantage tokens, and cap the global clipped-token ratio."""
[docs] def __init__( self, epsilon: float = 1e-6, mu: float = 2.0, max_frac: float = 1e-4, ) -> None: self.epsilon = epsilon self.mu = mu self.max_frac = max_frac
def __call__( self, exps: "DataProto", **kwargs, ) -> Tuple["DataProto", Dict]: token_level_rewards = exps.batch["token_level_rewards"] response_mask = exps.batch["response_mask"] index = exps.non_tensor_batch["uid"] new_log_probs = exps.batch["old_log_probs"] new_entropys = exps.batch["entropys"] necs = exps.batch["necs"] response_length = token_level_rewards.shape[-1] scores = token_level_rewards.sum(dim=-1) id2score = defaultdict(list) id2mean = {} id2std = {} with torch.no_grad(): bsz = scores.shape[0] for i in range(bsz): id2score[index[i]].append(scores[i]) for idx, grouped_scores in id2score.items(): if len(grouped_scores) == 1: id2mean[idx] = torch.tensor(0.0, dtype=scores.dtype, device=scores.device) id2std[idx] = torch.tensor(1.0, dtype=scores.dtype, device=scores.device) elif len(grouped_scores) > 1: group_scores = torch.stack(grouped_scores).to( dtype=scores.dtype, device=scores.device ) id2mean[idx] = torch.mean(group_scores) id2std[idx] = torch.std(group_scores) else: raise ValueError(f"no score in prompt index: {idx}") for i in range(bsz): scores[i] = (scores[i] - id2mean[index[i]]) / (id2std[index[i]] + self.epsilon) scores = scores.unsqueeze(-1).tile([1, response_length]) * response_mask exps.batch["advantages"] = scores exps.batch["returns"] = scores.clone() LP = new_log_probs H = new_entropys N = necs M = response_mask p = LP.exp() S = p * (H + LP) xD = (N - S).detach().to(torch.float32) A = exps.batch["advantages"].detach().to(torch.float32) m = M.to(torch.float32) n = m.sum().clamp_min(1.0) mean_d = (xD * m).sum() / n var_d = ((xD - mean_d) ** 2 * m).sum() / n std_d = var_d.sqrt() if std_d.item() < 1e-12: keep = torch.ones_like(M, dtype=M.dtype) else: pos_mask = A > 0 neg_mask = A < 0 keep_neg = xD <= (self.mu * std_d) keep_bool = torch.where(pos_mask, torch.ones_like(pos_mask), keep_neg) total_tokens = m.sum().clamp_min(1.0) clipped_mask = (M > 0) & (~keep_bool) frac_clipped = (clipped_mask.to(torch.float32).sum() / total_tokens).item() if frac_clipped <= self.max_frac: keep = keep_bool.to(M.dtype) else: max_clipped_tokens = max(int(self.max_frac * total_tokens.item()), 1) neg_to_clip = neg_mask & (M > 0) & (~keep_neg) neg_to_clip_count = int(neg_to_clip.to(torch.int32).sum().item()) if neg_to_clip_count <= max_clipped_tokens: keep = keep_bool.to(M.dtype) else: # Keep only top-K most violating negative-advantage tokens clipped. candidate_scores = xD.masked_fill(~neg_to_clip, float("-inf")).view(-1) k = min(max_clipped_tokens, neg_to_clip_count) _, indices = torch.topk(candidate_scores, k, largest=True) limited_clip_mask = torch.zeros_like(candidate_scores, dtype=torch.bool) limited_clip_mask[indices] = True limited_clip_mask = limited_clip_mask.view_as(xD) final_keep = keep_bool.clone() final_keep[neg_to_clip] = True final_keep[limited_clip_mask] = False keep = final_keep.to(M.dtype) M_clipped = M * keep exps.batch["response_mask"] = M_clipped total_tokens = M.to(torch.float32).sum().clamp_min(1.0) frac_clipped = 1.0 - (M_clipped.to(torch.float32).sum() / total_tokens).item() pos_mask = (A > 0).to(M.dtype) neg_mask = (A < 0).to(M.dtype) total_pos = (M * pos_mask).to(torch.float32).sum().clamp_min(1.0) total_neg = (M * neg_mask).to(torch.float32).sum().clamp_min(1.0) frac_clipped_pos = 1.0 - ((M_clipped * pos_mask).to(torch.float32).sum() / total_pos).item() frac_clipped_neg = 1.0 - ((M_clipped * neg_mask).to(torch.float32).sum() / total_neg).item() metrics = { "frac_clipped": frac_clipped, "frac_clipped_pos": frac_clipped_pos, "frac_clipped_neg": frac_clipped_neg, "stdD": std_d.item(), } return exps, metrics
[docs] @classmethod def default_args(cls) -> Dict: return { "epsilon": 1e-6, "mu": 8.5, "max_frac": 1e-4, }