v0.1.18

CMD.md

@@ -1,9 +1,20 @@
 # Single Pipeline
 ```bash
+# Default goals: max_velocity, diagonal_velocity
 python robogauge/scripts/run.py \
     --task go2_moe.flat \
     --experiment-name debug \
     --headless
+
+# To evaluate metrics for level terrains, add `--goals` and `--spawn-type`
+# level terrains: wave, slope, stairs up, stairs down, obstacle (default is target_pos)
+python robogauge/scripts/run.py \
+    --task go2_moe.obstacle \
+    --experiment-name debug \
+    --level 10 \
+    --friction 2 \
+    --spawn-type level_eval \
+    --goals max_velocity diagonal_velocity
 ```
 
 # Multi Pipeline
@@ -20,13 +31,14 @@ python robogauge/scripts/run.py \
 
 # Terrain with Level, need specify goals (default is target_pos)
 python robogauge/scripts/run.py \
-    --task go2_moe.slope \
+    --task go2_moe.stairs_up \
     --experiment-name debug \
     --multi \
     --num-processes 5 \
     --seeds 0 1 2 3 4 \
-    --frictions 1.0 \
-    --level 3 \
+    --frictions 2.0 \
+    --level 10 \
+    --spawn-type level_eval \
     --goals max_velocity diagonal_velocity\
     --compress-logs \
     --headless
@@ -52,9 +64,9 @@ python robogauge/scripts/run.py \
     --experiment-name debug \
     --stress-benchmark \
     --stress-terrain-names flat slope stairs_up stairs_down wave obstacle \
-    --num-processes 6 \
+    --num-processes 50 \
     --seeds 0 1 2 3 4 \
-    --frictions 0.5 1.0 1.5 2.0 2.5 \
+    --frictions 0.5 0.75 1.0 1.25 1.5 1.75 2.0 2.25 2.5 \
     --compress-logs \
     --headless
 ```

UPDATE.md

@@ -4,7 +4,8 @@
 1. 将final_score改为merge_metrics, 并在multi_pipeline中也进行该评估
 2. [放弃, 好像也没这个必要] 上下台阶取二者的最小等级, 实现中都计算出来, 统计地形得分时仅计算二者种较低等级的一个
 3. 对有难度地形设置goals, 但是忘记设置到地形的正中心了!!!在每种地形上调出对应的中心初始位置
-4. 重新详细设计整个打分标准, 引入几个得分: quality_score, terrain_quality_score, robust_score, benchmark_score
+4. 添加`--spawn-type`配置, 根据评估和搜索修改初始生成点, 完成slope, wave, stairs up, stairs down, obstacle
+5. 重新详细设计整个打分标准, 引入几个阶段性得分: quality_score, terrain_quality_score, robust_score, benchmark_score
 ## 20251227
 ### v0.1.17
 1. args中添加`goals`配置, 在LevelPipeline和MultiPipeline中指定goals, 修正压力测试中评估的目标不对的问题

robogauge/tasks/__init__.py

@@ -1,6 +1,6 @@
 from robogauge.utils.task_register import task_register
 from robogauge.tasks.simulator.mujoco_config import MujocoConfig
-from robogauge.tasks.robots import RobotConfig, Go2Config, Go2MoEConfig, Go2TerrainConfig, Go2MoETerrainConfig
+from robogauge.tasks.robots import RobotConfig, Go2Config, Go2MoEConfig, Go2TerrainConfig, Go2MoETerrainConfig, Go2StairsConfig, Go2MoEStairsConfig
 from robogauge.tasks.pipeline import BasePipeline
 from robogauge.tasks.gauge import BaseGaugeConfig
 

robogauge/tasks/custom/go2/go2_flat_task.py

@@ -30,7 +30,7 @@ class Go2FlatGaugeConfig(FlatGaugeConfig):
             cmd_duration = 6.0
 
         class target_pos_velocity(FlatGaugeConfig.goals.target_pos_velocity):  # goal to reach a target position by velocity command, config target at assets.target_pos
-            enabled = True
+            enabled = False
             target_pos = [2, 2, 0]  # x y z [m], target position in the environment, used for target position goal
             lin_vel_x = 1.0  # +/- m/s
             lin_vel_y = 1.0  # +/- m/s

robogauge/tasks/custom/go2/go2_stairs_up_task.py

@@ -7,7 +7,6 @@
 @Blog    : https://wty-yy.github.io/
 @Desc    : Go2 Wave Task Configuration
 '''
-from robogauge.tasks.robots import Go2Config, Go2MoEConfig
 from robogauge.tasks.gauge import StairsUpGaugeConfig
 from robogauge.tasks.simulator.mujoco_config import MujocoConfig
 

robogauge/tasks/gauge/gauge_configs/terrain_levels_config.py

@@ -9,6 +9,8 @@
 '''
 from robogauge.utils.config import Config
 
+SEARCH_LEVELS_TERRAINS = ['slope', 'wave', 'stairs_up', 'stairs_down', 'obstacle']
+
 class TerrainSearchLevelsConfig(Config):
     class flat:
         levels = [0]
@@ -91,6 +93,9 @@ class TerrainSearchLevelsConfig(Config):
         levels = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
 
 class TerrainEvalLevelsConfig(Config):
+    class flat:
+        levels = [0]
+
     class slope:
         levels = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
         spawns = [
@@ -105,3 +110,63 @@ class TerrainEvalLevelsConfig(Config):
             [5.0, 0, 4.2 * (0.57 - 0.047)],
             [5.0, 0, 4.2 * 0.57],
         ]
+
+    class wave:
+        levels = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
+        spawns = [
+            [4.55, 0, 0.65 - 0.075 * 9],
+            [4.55, 0, 0.65 - 0.075 * 8],
+            [4.55, 0, 0.65 - 0.075 * 7],
+            [4.55, 0, 0.65 - 0.075 * 6],
+            [4.55, 0, 0.65 - 0.075 * 5],
+            [4.55, 0, 0.65 - 0.075 * 4],
+            [4.55, 0, 0.65 - 0.075 * 3],
+            [4.55, 0, 0.65 - 0.075 * 2],
+            [4.55, 0, 0.65 - 0.075 * 1],
+            [4.55, 0, 0.65],
+        ]
+
+    class stairs_up:
+        levels = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
+        spawns = [
+            [5.0, 0.0, 1.35],
+            [5.0, 0.0, 1.80],
+            [5.0, 0.0, 2.25 + 0.03],
+            [5.0, 0.0, 2.70 + 0.08],
+            [5.0, 0.0, 2.85 + 0.1],
+            [5.0, 0.0, 3.00 + 0.12],
+            [5.0, 0.0, 3.15 + 0.14],
+            [5.0, 0.0, 3.30 + 0.16],
+            [5.0, 0.0, 3.45 + 0.18],
+            [5.0, 0.0, 3.60 + 0.2],
+        ]
+
+    class stairs_down:
+        levels = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
+        spawns = [
+            [5.1, 0.0, 1.35],
+            [5.1, 0.0, 1.80 + 0.08],
+            [5.1, 0.0, 2.25 + 0.15],
+            [5.1, 0.0, 2.70 + 0.25],
+            [5.1, 0.0, 2.85 + 0.27],
+            [5.1, 0.0, 3.00 + 0.30],
+            [5.1, 0.0, 3.15 + 0.33],
+            [5.1, 0.0, 3.30 + 0.36],
+            [5.1, 0.0, 3.45 + 0.39],
+            [5.1, 0.0, 3.60 + 0.42],
+        ]
+
+    class obstacle:
+        levels = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
+        spawns = [
+            [5, 0.0, 0.20 - 0.023 * 9],
+            [5, 0.0, 0.20 - 0.023 * 8],
+            [5, 0.0, 0.20 - 0.023 * 7],
+            [5, 0.0, 0.20 - 0.023 * 6],
+            [5, 0.0, 0.20 - 0.023 * 5],
+            [5, 0.0, 0.20 - 0.023 * 4],
+            [5, 0.0, 0.20 - 0.023 * 3],
+            [5, 0.0, 0.20 - 0.023 * 2],
+            [5, 0.0, 0.20 - 0.023 * 1],
+            [5, 0.0, 0.20],
+        ]

robogauge/tasks/pipeline/multi_pipeline.py

@@ -25,6 +25,7 @@ from robogauge.utils.logger import Logger
 from robogauge.utils.process_utils import NoDaemonPool
 from robogauge.utils.progress_monitor import report_progress, ProgressTypes, ProgressData
 from robogauge.utils.file_utils import compress_directory
+from robogauge.tasks.gauge.gauge_configs.terrain_levels_config import SEARCH_LEVELS_TERRAINS
 
 multi_logger = Logger()  # MultiPipeline logger
 
@@ -132,7 +133,7 @@ class MultiPipeline:
         """ Process results from all processes and aggregate them. """
         multi_logger.info("📊 Aggregating Results from all runs...")
 
-        summary = {'success': {}, **self.static_info, 'summary': {}}
+        summary = {'success': {}, **self.static_info, 'summary': {}, 'quality_score': {}, 'terrain_quality_score': {}}
         finish_msg = (
             f"""\n{'='*20} Run Finish Summary {'='*20}\n"""
             f"""{'Seed':^10}{'Base Mass':^15}{'Friction':^15}{'Status':^10}\n"""
@@ -151,6 +152,7 @@ class MultiPipeline:
             multi_logger.error("No results to aggregate.")
             return
         
+        quality_score, terrain_quality_score = summary['quality_score'], summary['terrain_quality_score']
         value_collections = defaultdict(lambda: defaultdict(list))
         for result in all_results:
             for goal, metrics in result['results'].items():
@@ -159,11 +161,22 @@ class MultiPipeline:
                 for metric, means in metrics.items():
                     for mean_name, mean_value in means.items():
                         value_collections[metric][mean_name].append(float(mean_value.split(' ')[0]))
+                        quality_score[mean_name] = 1
         
         for metric, means in value_collections.items():
             summary['summary'][metric] = {}
             for mean_name, values in means.items():
-                summary['summary'][metric][mean_name] = f"{float(np.mean(values)):.4f} ± {float(np.std(values)):.4f}"
+                v = float(np.mean(values))
+                summary['summary'][metric][mean_name] = f"{v:.4f} ± {float(np.std(values)):.4f}"
+                weight = 1
+                if metric in ['ang_vel_err', 'lin_vel_err']:
+                    weight = 2
+                quality_score[mean_name] *= min(max(1e-9, v), 1.0) ** weight
+        for mean_name in quality_score:
+            quality_score[mean_name] = quality_score[mean_name] ** (1 / 8)  # 2 + 2 + 1 * 4
+            terrain_quality_score[mean_name] = quality_score[mean_name]
+            if summary['terrain_name'] in SEARCH_LEVELS_TERRAINS:
+                terrain_quality_score[mean_name] = 0.09 * (summary['terrain_level'] - 1) + 0.19 * quality_score[mean_name]
         
         save_path = multi_logger.log_dir / "aggregated_results.yaml"
         with open(save_path, 'w') as file:

robogauge/tasks/pipeline/stress_pipeline.py

@@ -27,7 +27,7 @@ from robogauge.utils.logger import Logger
 from robogauge.utils.process_utils import NoDaemonPool
 from robogauge.utils.progress_monitor import report_progress, ProgressTypes, start_progress_monitor_thread, ProgressData
 from robogauge.tasks.pipeline import MultiPipeline, LevelPipeline
-from robogauge.tasks.gauge.gauge_configs.terrain_levels_config import TerrainSearchLevelsConfig
+from robogauge.tasks.gauge.gauge_configs.terrain_levels_config import SEARCH_LEVELS_TERRAINS
 from robogauge.utils.file_utils import compress_directory
 
 stress_logger = Logger()  # StressPipeline logger
@@ -57,6 +57,7 @@ def run_pipeline(args, progress_queue, data):
 
     if search is True:
         args.goals = GOALS['level_pipeline']
+        args.spawn_type = "level_search"
         level, level_results = LevelPipeline(args, console_output=False, progress_data=progress_data).run()
         if level == 0:  # no valid level found
             report_progress(progress_data, ProgressTypes.FINISH, desc=f"❌ Failed (Lv 0)")
@@ -76,6 +77,7 @@ def run_pipeline(args, progress_queue, data):
         
     args.level = level
     args.goals = GOALS['multi_pipeline']
+    args.spawn_type = "level_eval"
     results = {
         'success': True,
         'results': MultiPipeline(args, console_output=False, progress_data=progress_data).run(),
@@ -113,12 +115,9 @@ class StressPipeline:
 
         ### Build worker data ###
         workers_data = []
-        terrain_search_levels_config = TerrainSearchLevelsConfig()
         for terrain_name in terrain_names:
             search_max_level = True
-            terrain_level_cfg = getattr(terrain_search_levels_config, terrain_name, None)
-            assert terrain_level_cfg is not None, f"Terrain '{terrain_name}' not found in TerrainSearchLevelsConfig."
-            if len(terrain_level_cfg.levels) == 1:  # Flattened terrain
+            if terrain_name not in SEARCH_LEVELS_TERRAINS:  # Flattened terrain
                 search_max_level = False
             data = {
                 'task_robot_model': self.task_robot_model,
@@ -179,8 +178,9 @@ class StressPipeline:
             stress_logger.error("No results to aggregate.")
             return
 
-        summary = {**self.static_info, 'summary': {}, 'final_score': {}}
-        value_collections = defaultdict(lambda: defaultdict(list))
+        summary = {**self.static_info, 'summary': {}, 'robust_score': {}, 'benchmark_score': 0.0}
+        metric_collections = defaultdict(lambda: defaultdict(list))
+        terrain_collections = defaultdict(lambda: defaultdict(list))
         zero_terrain_count = 0
         for result in all_results:
             terrain_name = result['data']['terrain_name']
@@ -194,21 +194,27 @@ class StressPipeline:
             summary[key] = result['results']
 
             for metric, means in result['results']['summary'].items():
-                for mean_name, mean_value in means.items():
-                    value = float(mean_value.split(' ')[0])
-                    if terrain_level is not None:  # level terrain
-                        value = (terrain_level - 1) * 0.1 + value * 0.1
-                    value_collections[metric][mean_name].append(value)
+                for mean_name, value_str in means.items():
+                    value = float(value_str.split(' ± ')[0])
+                    metric_collections[metric][mean_name].append(value)
+            for mean_name, value in result['results']['terrain_quality_score'].items():
+                terrain_collections[terrain_name][mean_name].append(value)
 
-        final_metrics = defaultdict(list)
-        for metric, means in value_collections.items():
+        for metric, means in metric_collections.items():
             summary['summary'][metric] = {}
             for mean_name, values in means.items():
                 values.extend([0.0] * zero_terrain_count)  # include zero terrains
                 summary['summary'][metric][mean_name] = f"{float(np.mean(values)):.4f} ± {float(np.std(values)):.4f}"
-                final_metrics[mean_name].append(np.mean(values))
-        for mean_name, values in final_metrics.items():
-            summary['final_score'][mean_name] = float(np.mean(values))
+        
+        robust_score = defaultdict(dict)
+        robust_scores = []
+        for terrain_name, means in terrain_collections.items():
+            for mean_name, values in means.items():
+                values.extend([0.0] * zero_terrain_count)  # include zero terrains
+                robust_score[terrain_name][mean_name] = float(np.mean(values))
+                robust_scores.append(robust_score[terrain_name][mean_name])
+        summary['robust_score'] = dict(robust_score)
+        summary['benchmark_score'] = float(np.mean(robust_scores))
 
         save_path = stress_logger.log_dir / "stress_benchmark_results.yaml"
         with open(save_path, 'w') as file:

robogauge/tasks/robots/__init__.py

@@ -1,6 +1,6 @@
 from .base_robot_config import RobotConfig
 from .base_robot import BaseRobot
-from .go2.go2_config import Go2Config, Go2TerrainConfig
+from .go2.go2_config import Go2Config, Go2TerrainConfig, Go2StairsConfig
 from .go2.go2 import Go2
-from .go2.go2_moe_config import Go2MoEConfig, Go2MoETerrainConfig
+from .go2.go2_moe_config import Go2MoEConfig, Go2MoETerrainConfig, Go2MoEStairsConfig
 from .go2.go2_moe import Go2MoE

robogauge/utils/helpers.py

@@ -76,6 +76,7 @@ def parse_args():
         {"name": "--base-mass", "type": float, "default": 0.0, "help": "Set the base mass of the robot."},
         {"name": "--friction", "type": float, "default": 1.0, "help": "Set the ground friction coefficient."},
         {"name": "--level", "type": int, "help": "Set the difficulty level of the environment, range 1-10 (flat is 0)."},
+        {"name": "--spawn-type", "type": str, "choices": ["level_eval", "level_search"], "help": "Spawn type for the robot."},
         {"name": "--goals", "type": str, "nargs": "+", "help": "List of goal names to evaluate."},
 
         # Multiprocessing parameters, with different seeds

robogauge/utils/task_register.py

@@ -10,7 +10,7 @@
 from robogauge import ROBOGAUGE_ROOT_DIR
 from robogauge.utils.logger import logger
 from robogauge.utils.helpers import parse_args, set_seed, class_to_dict
-from robogauge.tasks.gauge.gauge_configs.terrain_levels_config import TerrainSearchLevelsConfig
+from robogauge.tasks.gauge.gauge_configs.terrain_levels_config import TerrainSearchLevelsConfig, TerrainEvalLevelsConfig
 
 class TaskRegister():
     def __init__(self):
@@ -72,14 +72,18 @@ class TaskRegister():
             sim_cfg.domain_rand.base_mass = args.base_mass
         if args.level is not None:
             gauger_cfg.assets.terrain_level = args.level
-            levels_cfg = TerrainSearchLevelsConfig()
-            cfg = getattr(levels_cfg, gauger_cfg.assets.terrain_name, None)
-            assert cfg is not None, f"Level {args.level} configuration not found in TerrainLevelsConfig."
-            assert args.level in cfg.levels, f"Level must be in {cfg.levels}."
-            if hasattr(cfg, 'targets'):
-                gauger_cfg.goals.target_pos_velocity.target_pos = cfg.targets[cfg.levels.index(args.level)]
-            if hasattr(cfg, 'spawns'):
-                gauger_cfg.assets.terrain_spawn_pos = cfg.spawns[cfg.levels.index(args.level)]
+            search_level_cfg = TerrainSearchLevelsConfig()
+            eval_level_cfg = TerrainEvalLevelsConfig()
+            search_cfg = getattr(search_level_cfg, gauger_cfg.assets.terrain_name, None)
+            eval_cfg = getattr(eval_level_cfg, gauger_cfg.assets.terrain_name, None)
+            assert search_cfg is not None and eval_cfg is not None, f"Level {args.level} configuration not found in TerrainLevelsConfig or TerrainEvalLevelsConfig."
+            assert args.level in search_cfg.levels and args.level in eval_cfg.levels, f"Level must be in {search_cfg.levels=} and {eval_cfg.levels=}."
+            if hasattr(search_cfg, 'targets'):
+                gauger_cfg.goals.target_pos_velocity.target_pos = search_cfg.targets[search_cfg.levels.index(args.level)]
+            if hasattr(search_cfg, 'spawns') and args.spawn_type == "level_search":
+                gauger_cfg.assets.terrain_spawn_pos = search_cfg.spawns[search_cfg.levels.index(args.level)]
+            elif hasattr(eval_cfg, 'spawns') and args.spawn_type == "level_eval":
+                gauger_cfg.assets.terrain_spawn_pos = eval_cfg.spawns[eval_cfg.levels.index(args.level)]
             xml = gauger_cfg.assets.terrain_xmls[0]
             xml = xml.rsplit('/', 1)[0] + f"/{gauger_cfg.assets.terrain_name}_{args.level}.xml"
             gauger_cfg.assets.terrain_xmls[0] = xml