v0.1.12 last gauge info

CMD.md

@@ -2,7 +2,6 @@
 ```bash
 python robogauge/scripts/run.py \
     --task go2_moe_flat \
-    --model-path /home/xfy/Coding/robot_gauge/mytest/models/kaiwu/kaiwu_script_v6-2_124004.pt \
     --experiment-name debug \
     --headless
 ```
@@ -11,17 +10,24 @@ python robogauge/scripts/run.py \
 ```bash
 python robogauge/scripts/run.py \
     --task go2_moe_flat \
-    --model-path /home/xfy/Coding/robot_gauge/mytest/models/kaiwu/kaiwu_script_v6-2_124004.pt \
     --experiment-name debug \
     --multi \
-    --num-processes 1 \
+    --num-processes 3 \
     --headless
 
 python robogauge/scripts/run.py \
     --task go2_moe_flat \
-    --model-path /home/xfy/Coding/robot_gauge/mytest/models/kaiwu/kaiwu_script_v6-2_102003.pt \
     --experiment-name debug \
     --multi \
-    --num-processes 1 \
+    --num-processes 3 \
     --headless
 ```
+
+# Radar Plot
+将Multi Run结果绘制在雷达图中
+```bash
+python robogauge/utils/radar_plot.py \
+    aggregated_results_1.yaml \
+    aggregated_results_2.yaml \
+    --out logs/go2_flat_vs_moe_flat.png
+```

UPDATE.md

@@ -1,5 +1,8 @@
 # UPDATE
 TODO: 在模型崩溃时也记录下最后的gauge信息
+## 20251221
+### v0.1.12
+1. 在模型崩溃时也记录下最后的gauge信息, 修改single/multi pipeline逻辑
 ## 20251220
 ### v0.1.11
 1. 加入`os.environ["OMP_NUM_THREADS"] = "2"; os.environ["MKL_NUM_THREADS"] = "2"`避免并行时cpu线程爆炸, `--multi`模式能稳定提高速度了

robogauge/tasks/gauge/base_gauge.py

@@ -40,8 +40,7 @@ class BaseGauge:
         self.goals: List[BaseGoal] = []
         self.metrics: List[function] = []
         self.info = {'goal': [], 'metric': []}
-        self.results = {
+        self.results = {}  # {'goal/sub_goal': {'metric': result}}
-        }  # {'goal/sub_goal': {'metric': result}}
 
         log_str = "Initialized Gauge with Goals and Metrics:\n"
         for name, kwargs in self.goals_cfg.items():
@@ -102,16 +101,8 @@ class BaseGauge:
         #     )
         # )
         goal_obj = self.goals[self.goal_idx]
-        if goal_obj.pre_get_goal(sim_data):
+        if goal_obj.pre_get_goal(sim_data):  # goal finished
-            metrics = goal_obj.goal_mean_metrics
+            self.switch_to_next_goal()
-            if hasattr(goal_obj, 'success'):  # target position goal
-                metrics['success'] = {'mean': float(goal_obj.success)}
-            
-            key = f"{goal_obj.name}"
-            self.results[key] = metrics
-
-            self.goal_idx += 1
-            self.create_new_goal_logger()
             return None
         goal = goal_obj.get_goal(sim_data)
 
@@ -121,6 +112,19 @@ class BaseGauge:
             logger.info(f"New Goal [{self.goal_idx+1}/{len(self.goals)}] [{goal_obj.count+1}/{goal_obj.total}]: {self.goal_str}")
         return goal
     
+    def switch_to_next_goal(self):
+        """ Switch to the next goal and log the metrics of the current goal. """
+        goal_obj = self.goals[self.goal_idx]
+        metrics = goal_obj.goal_mean_metrics
+        if hasattr(goal_obj, 'success'):  # target position goal
+            metrics['success'] = {'mean': float(goal_obj.success)}
+        
+        key = f"{goal_obj.name}"
+        self.results[key] = metrics
+
+        self.goal_idx += 1
+        self.create_new_goal_logger()
+    
     def update_metrics(self, sim_data: SimData, goal_data: GoalData):
         if sim_data.n_step % int(self.cfg.metrics.metric_dt / sim_data.sim_dt) != 0:
             return

robogauge/tasks/gauge/goals/base_goal.py

@@ -54,6 +54,7 @@ class BaseGoal:
             result[f'mean@{i}'] = 0
         if len(metrics) == 0:
             return result
+        metrics = [max(0.0, min(1.0, m)) for m in metrics]
         metrics.sort()
         result['mean'] = float(sum(metrics) / len(metrics))
         for i in [25, 50]:

robogauge/tasks/pipeline/base_pipeline.py

@@ -105,14 +105,16 @@ class BasePipeline:
                     sim_data = self.reset_sim(sim_data)
         except Exception as e:
             logger.error(f"❌ Pipeline execution failed with error: {e}")
-            raise e
+            self.gauge.switch_to_next_goal()  # save current goal metrics
+            self.gauge.save_results()
+            return logger.log_dir, e
         finally:
             self.sim.close_viewer()
             self.sim.close_video_writer()
             logger.info("✅ Pipeline execution finished.")
             logger.info(f"📁 Logging saved at: {logger.log_dir}")
 
-        return logger.log_dir
+        return logger.log_dir, None
     
     def reset_sim(self, sim_data: SimData):
         self.sim.reset()

robogauge/tasks/pipeline/multi_pipeline.py

@@ -36,19 +36,20 @@ def run_single_process(args, data):
         console_output=False
     )
     pipeline = task_register.make_pipeline(args=local_args, create_logger=False)
-    try:
+    log_dir, error = pipeline.run()
-        log_dir = pipeline.run()
+    if error is None:
         ret = {
             'status': 'success',
             'log_dir': log_dir,
             'model_path': pipeline.robot_cfg.control.model_path,
         }
-    except Exception as e:
+    else:
-        logger.error(f"❌ Process with seed={seed}, base_mass={base_mass}, friction={friction} failed with error: {e}")
+        logger.error(f"❌ Process with seed={seed}, base_mass={base_mass}, friction={friction} failed with error: {error}")
         ret = {
             'status': 'error',
+            'log_dir': log_dir,
             'data': data,
-            'error_msg': str(e),
+            'error_msg': str(error),
             'traceback': traceback.format_exc()
         }
     return ret
@@ -76,8 +77,9 @@ class MultiPipeline:
             iterator = pool.imap_unordered(worker_func, workers_data)
             for results in tqdm(iterator, total=len(workers_data), desc="Evaluation"):
                 success_flags.append(results['status'] == 'success')
+
+                result_log_dirs.append(results['log_dir'])
                 if results['status'] == 'success':
-                    result_log_dirs.append(results['log_dir'])
                     if self.model_path is None:
                         self.model_path = results['model_path']
                     else:

robogauge/utils/radar_plot.py

@@ -151,7 +151,7 @@ def plot_radar(data_list, labels, output_file=None):
     # loc='upper left' 意思是图例的左上角对齐这个锚点
     legend = plt.legend(
         loc='upper left', 
-        bbox_to_anchor=(1.1, 0.3),  # 调整这里的 0.3 可以上下移动图例
+        bbox_to_anchor=(1.0, 0.1),  # 调整这里的 0.3 可以上下移动图例
         title="Models", 
         title_fontsize=16,
         fontsize=12,