cyy/Motrixlab
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: squeeze old_log_prob and advantages dims (match upstream)

Browse Source
This commit is contained in:
8x54zj-m
2026-06-30 15:39:03 +08:00
parent aa17ea853e
commit 6348cf72bc
1 changed files with 3 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
motrix_rl/src/motrix_rl/rslrl/torch/train/dreamwaq_ppo.py
View File

@@ -158,9 +158,9 @@ class DreamWaQPPO(PPO):
old_actions_log_prob_batch, advantages_batch old_actions_log_prob_batch, advantages_batch
) -> torch.Tensor: ) -> torch.Tensor:
"""计算 PPO 代理损失（从父类 PPO.update() 中提取）。""" """计算 PPO 代理损失（从父类 PPO.update() 中提取）。"""
ratio = torch.exp(actions_log_prob_batch - old_actions_log_prob_batch) ratio = torch.exp(actions_log_prob_batch - torch.squeeze(old_actions_log_prob_batch))
surrogate = -advantages_batch * ratio surrogate = -torch.squeeze(advantages_batch) * ratio
surrogate_clipped = -advantages_batch * torch.clamp( surrogate_clipped = -torch.squeeze(advantages_batch) * torch.clamp(
ratio, 1.0 - self.clip_param, 1.0 + self.clip_param) ratio, 1.0 - self.clip_param, 1.0 + self.clip_param)
return torch.max(surrogate, surrogate_clipped).mean() return torch.max(surrogate, surrogate_clipped).mean()