Source code for trinity.algorithm.advantage_fn.rloo_advantage
"""RLOO advantage computation
Ref: https://github.com/volcengine/verl/blob/main/verl/trainer/ppo/core_algos.py
"""
from collections import defaultdict
from typing import Dict, Tuple
import torch
from verl import DataProto
from trinity.algorithm.advantage_fn import AdvantageFn
[docs]
class RLOOAdvantageFn(AdvantageFn):
[docs]
def __init__(self) -> None:
pass
def __call__(
self,
exps: DataProto,
**kwargs,
) -> Tuple[DataProto, Dict]:
"""
Compute advantage for RLOO based on https://arxiv.org/abs/2402.14740
token_level_rewards: `(torch.Tensor)`
shape: (bs, response_length)
eos_mask: `(torch.Tensor)`
shape: (bs, response_length)
scores: `(torch.Tensor)`
shape: (bs, response_length)
"""
token_level_rewards = exps.batch["token_level_rewards"]
eos_mask = exps.batch["response_mask"]
index = exps.non_tensor_batch["uid"]
response_length = token_level_rewards.shape[-1]
scores = token_level_rewards.sum(dim=-1)
id2score = defaultdict(list)
id2mean = {}
with torch.no_grad():
bsz = scores.shape[0]
for i in range(bsz):
id2score[index[i]].append(scores[i])
for idx in id2score:
if len(id2score[idx]) == 1:
id2mean[idx] = torch.tensor(0.0)
elif len(id2score[idx]) > 1:
id2mean[idx] = torch.mean(torch.tensor(id2score[idx], dtype=torch.float32))
else:
raise ValueError(f"no score in prompt index: {idx}")
for i in range(bsz):
response_num = len(id2score[index[i]])
if response_num > 1:
scores[i] = scores[i] * response_num / (response_num - 1) - id2mean[
index[i]
] * response_num / (response_num - 1)
scores = scores.unsqueeze(-1).tile([1, response_length]) * eos_mask
exps.batch["advantages"] = scores
exps.batch["returns"] = scores
metrics = {
# TODO: add meaningful metrics
}
return exps, metrics
[docs]
@classmethod
def default_args(cls) -> Dict:
return {}