feat: AdaBoot — paper CV of episodic reward (not velocity error)

motrix_rl/src/motrix_rl/rslrl/torch/models/cenet_actor.py

@@ -155,19 +155,10 @@ class CENetActorModel(MLPModel):
         self._last_cenet_output = out
         code, code_vel, decode, mean_vel, logvar_vel, mean_latent, logvar_latent = out
 
-        # ── AdaBoot: 自适应速度 Bootstrap（论文 Section II-C）──
+        # ── AdaBoot: 自适应 Bootstrap（论文：CV of episodic reward）──
-        if self.training and "privileged_obs" in obs:
+        # _adaboot_prob 由 DreamWaQPPO 根据 episodic reward CV 动态更新
+        if self.training and "privileged_obs" in obs and self._adaboot_prob > 0.0:
             gt_vel = obs["privileged_obs"][:, 45:48]  # ground truth body velocity
-            # 先计算 CENet 原始误差的 CV（替换前）
-            with torch.no_grad():
-                raw_error = code_vel - gt_vel
-                cv = raw_error.std() / (raw_error.abs().mean() + 1e-6)
-                self._adaboot_cv_buffer.append(cv.item())
-                if len(self._adaboot_cv_buffer) > 1000:
-                    self._adaboot_cv_buffer = self._adaboot_cv_buffer[-1000:]
-                mean_cv = sum(self._adaboot_cv_buffer) / len(self._adaboot_cv_buffer)
-                self._adaboot_prob = max(0.0, min(1.0, mean_cv * 5.0))
-            # 以概率 _adaboot_prob 用 GT 替换 CENet 的速度估计
             mask = torch.rand(code_vel.shape[0], 1, device=code_vel.device) < self._adaboot_prob
             code_vel = torch.where(mask, gt_vel, code_vel)
             code = torch.cat([code_vel, code[:, 3:]], dim=-1)

motrix_rl/src/motrix_rl/rslrl/torch/train/dreamwaq_ppo.py

@@ -161,6 +161,10 @@ class DreamWaQPPO(PPO):
         mean_surrogate_loss /= num_updates
         mean_autoenc_loss /= num_updates
 
+        # ── AdaBoot: 论文用 episodic reward 的 CV 控制 bootstrap 概率 ──
+        if hasattr(self.actor, '_adaboot_prob'):
+            self._update_adaboot(self.storage)
+
         self.storage.clear()
 
         return {
@@ -169,6 +173,55 @@ class DreamWaQPPO(PPO):
             "autoenc_loss": mean_autoenc_loss,
         }
 
+    def _update_adaboot(self, storage) -> None:
+        """论文 AdaBoost: episodic reward 的 CV（变异系数）控制 bootstrap 概率。
+
+        精确遍历 episode 边界，计算每个完整 episode 的总奖励。
+        CV = std(episode_rewards) / mean(episode_rewards)。
+        """
+        if not hasattr(self, '_adaboot_reward_buf'):
+            self._adaboot_reward_buf = []
+        dones = storage.dones                   # (T, E)  bool
+        rewards = storage.rewards               # (T, E)  每步奖励
+        if dones is None or rewards is None:
+            return
+        T, E = dones.shape
+        if T < 2 or E < 1:
+            return
+
+        # 按 env 遍历，在 done 边界处累计 episode 总奖励
+        ep_rewards = []
+        for e in range(E):
+            start = 0
+            for t in range(T):
+                if dones[t, e].item():
+                    # episode 结束：累计从 start 到 t 的奖励
+                    ep_sum = rewards[start:t + 1, e].sum().item()
+                    ep_rewards.append(ep_sum)
+                    start = t + 1
+
+        if len(ep_rewards) < 8:
+            return  # 不够统计
+
+        # 维护缓冲区
+        if not hasattr(self, '_adaboot_reward_buf'):
+            self._adaboot_reward_buf = []
+        self._adaboot_reward_buf.extend(ep_rewards)
+        if len(self._adaboot_reward_buf) > 2000:
+            self._adaboot_reward_buf = self._adaboot_reward_buf[-2000:]
+
+        if len(self._adaboot_reward_buf) < 50:
+            return
+
+        # 论文公式: CV = σ / μ
+        buf = self._adaboot_reward_buf
+        mean_r = sum(buf) / len(buf)
+        var_r = sum((r - mean_r) ** 2 for r in buf) / len(buf)
+        std_r = var_r ** 0.5
+        cv = std_r / (mean_r + 1e-6)
+        # CV → bootstrap 概率（CV 高→不稳定→多用 GT；映射系数 5.0 可调）
+        self.actor._adaboot_prob = max(0.0, min(1.0, cv * 5.0))
+
     def _compute_surrogate_loss(
         self, actions_batch, actions_log_prob_batch,
         old_actions_log_prob_batch, advantages_batch