diff --git a/motrix_rl/src/motrix_rl/rslrl/torch/train/dreamwaq_ppo.py b/motrix_rl/src/motrix_rl/rslrl/torch/train/dreamwaq_ppo.py index 6f9050d..5a5afa9 100644 --- a/motrix_rl/src/motrix_rl/rslrl/torch/train/dreamwaq_ppo.py +++ b/motrix_rl/src/motrix_rl/rslrl/torch/train/dreamwaq_ppo.py @@ -158,9 +158,9 @@ class DreamWaQPPO(PPO): old_actions_log_prob_batch, advantages_batch ) -> torch.Tensor: """计算 PPO 代理损失(从父类 PPO.update() 中提取)。""" - ratio = torch.exp(actions_log_prob_batch - old_actions_log_prob_batch) - surrogate = -advantages_batch * ratio - surrogate_clipped = -advantages_batch * torch.clamp( + ratio = torch.exp(actions_log_prob_batch - torch.squeeze(old_actions_log_prob_batch)) + surrogate = -torch.squeeze(advantages_batch) * ratio + surrogate_clipped = -torch.squeeze(advantages_batch) * torch.clamp( ratio, 1.0 - self.clip_param, 1.0 + self.clip_param) return torch.max(surrogate, surrogate_clipped).mean()