diff --git a/motrix_rl/src/motrix_rl/rslrl/torch/models/cenet_actor.py b/motrix_rl/src/motrix_rl/rslrl/torch/models/cenet_actor.py index 7357e88..c3bb562 100644 --- a/motrix_rl/src/motrix_rl/rslrl/torch/models/cenet_actor.py +++ b/motrix_rl/src/motrix_rl/rslrl/torch/models/cenet_actor.py @@ -155,19 +155,10 @@ class CENetActorModel(MLPModel): self._last_cenet_output = out code, code_vel, decode, mean_vel, logvar_vel, mean_latent, logvar_latent = out - # ── AdaBoot: 自适应速度 Bootstrap(论文 Section II-C)── - if self.training and "privileged_obs" in obs: + # ── AdaBoot: 自适应 Bootstrap(论文:CV of episodic reward)── + # _adaboot_prob 由 DreamWaQPPO 根据 episodic reward CV 动态更新 + if self.training and "privileged_obs" in obs and self._adaboot_prob > 0.0: gt_vel = obs["privileged_obs"][:, 45:48] # ground truth body velocity - # 先计算 CENet 原始误差的 CV(替换前) - with torch.no_grad(): - raw_error = code_vel - gt_vel - cv = raw_error.std() / (raw_error.abs().mean() + 1e-6) - self._adaboot_cv_buffer.append(cv.item()) - if len(self._adaboot_cv_buffer) > 1000: - self._adaboot_cv_buffer = self._adaboot_cv_buffer[-1000:] - mean_cv = sum(self._adaboot_cv_buffer) / len(self._adaboot_cv_buffer) - self._adaboot_prob = max(0.0, min(1.0, mean_cv * 5.0)) - # 以概率 _adaboot_prob 用 GT 替换 CENet 的速度估计 mask = torch.rand(code_vel.shape[0], 1, device=code_vel.device) < self._adaboot_prob code_vel = torch.where(mask, gt_vel, code_vel) code = torch.cat([code_vel, code[:, 3:]], dim=-1) diff --git a/motrix_rl/src/motrix_rl/rslrl/torch/train/dreamwaq_ppo.py b/motrix_rl/src/motrix_rl/rslrl/torch/train/dreamwaq_ppo.py index 2c53f7f..b3b6b1b 100644 --- a/motrix_rl/src/motrix_rl/rslrl/torch/train/dreamwaq_ppo.py +++ b/motrix_rl/src/motrix_rl/rslrl/torch/train/dreamwaq_ppo.py @@ -161,6 +161,10 @@ class DreamWaQPPO(PPO): mean_surrogate_loss /= num_updates mean_autoenc_loss /= num_updates + # ── AdaBoot: 论文用 episodic reward 的 CV 控制 bootstrap 概率 ── + if hasattr(self.actor, '_adaboot_prob'): + self._update_adaboot(self.storage) + self.storage.clear() return { @@ -169,6 +173,55 @@ class DreamWaQPPO(PPO): "autoenc_loss": mean_autoenc_loss, } + def _update_adaboot(self, storage) -> None: + """论文 AdaBoost: episodic reward 的 CV(变异系数)控制 bootstrap 概率。 + + 精确遍历 episode 边界,计算每个完整 episode 的总奖励。 + CV = std(episode_rewards) / mean(episode_rewards)。 + """ + if not hasattr(self, '_adaboot_reward_buf'): + self._adaboot_reward_buf = [] + dones = storage.dones # (T, E) bool + rewards = storage.rewards # (T, E) 每步奖励 + if dones is None or rewards is None: + return + T, E = dones.shape + if T < 2 or E < 1: + return + + # 按 env 遍历,在 done 边界处累计 episode 总奖励 + ep_rewards = [] + for e in range(E): + start = 0 + for t in range(T): + if dones[t, e].item(): + # episode 结束:累计从 start 到 t 的奖励 + ep_sum = rewards[start:t + 1, e].sum().item() + ep_rewards.append(ep_sum) + start = t + 1 + + if len(ep_rewards) < 8: + return # 不够统计 + + # 维护缓冲区 + if not hasattr(self, '_adaboot_reward_buf'): + self._adaboot_reward_buf = [] + self._adaboot_reward_buf.extend(ep_rewards) + if len(self._adaboot_reward_buf) > 2000: + self._adaboot_reward_buf = self._adaboot_reward_buf[-2000:] + + if len(self._adaboot_reward_buf) < 50: + return + + # 论文公式: CV = σ / μ + buf = self._adaboot_reward_buf + mean_r = sum(buf) / len(buf) + var_r = sum((r - mean_r) ** 2 for r in buf) / len(buf) + std_r = var_r ** 0.5 + cv = std_r / (mean_r + 1e-6) + # CV → bootstrap 概率(CV 高→不稳定→多用 GT;映射系数 5.0 可调) + self.actor._adaboot_prob = max(0.0, min(1.0, cv * 5.0)) + def _compute_surrogate_loss( self, actions_batch, actions_log_prob_batch, old_actions_log_prob_batch, advantages_batch