diff --git a/CMD.md b/CMD.md index dbe21aa..7612592 100644 --- a/CMD.md +++ b/CMD.md @@ -1,9 +1,20 @@ # Single Pipeline ```bash +# Default goals: max_velocity, diagonal_velocity python robogauge/scripts/run.py \ --task go2_moe.flat \ --experiment-name debug \ --headless + +# To evaluate metrics for level terrains, add `--goals` and `--spawn-type` +# level terrains: wave, slope, stairs up, stairs down, obstacle (default is target_pos) +python robogauge/scripts/run.py \ + --task go2_moe.obstacle \ + --experiment-name debug \ + --level 10 \ + --friction 2 \ + --spawn-type level_eval \ + --goals max_velocity diagonal_velocity ``` # Multi Pipeline @@ -20,13 +31,14 @@ python robogauge/scripts/run.py \ # Terrain with Level, need specify goals (default is target_pos) python robogauge/scripts/run.py \ - --task go2_moe.slope \ + --task go2_moe.stairs_up \ --experiment-name debug \ --multi \ --num-processes 5 \ --seeds 0 1 2 3 4 \ - --frictions 1.0 \ - --level 3 \ + --frictions 2.0 \ + --level 10 \ + --spawn-type level_eval \ --goals max_velocity diagonal_velocity\ --compress-logs \ --headless @@ -52,9 +64,9 @@ python robogauge/scripts/run.py \ --experiment-name debug \ --stress-benchmark \ --stress-terrain-names flat slope stairs_up stairs_down wave obstacle \ - --num-processes 6 \ + --num-processes 50 \ --seeds 0 1 2 3 4 \ - --frictions 0.5 1.0 1.5 2.0 2.5 \ + --frictions 0.5 0.75 1.0 1.25 1.5 1.75 2.0 2.25 2.5 \ --compress-logs \ --headless ``` diff --git a/UPDATE.md b/UPDATE.md index bea8086..253df25 100644 --- a/UPDATE.md +++ b/UPDATE.md @@ -4,7 +4,8 @@ 1. 将final_score改为merge_metrics, 并在multi_pipeline中也进行该评估 2. [放弃, 好像也没这个必要] 上下台阶取二者的最小等级, 实现中都计算出来, 统计地形得分时仅计算二者种较低等级的一个 3. 对有难度地形设置goals, 但是忘记设置到地形的正中心了!!!在每种地形上调出对应的中心初始位置 -4. 重新详细设计整个打分标准, 引入几个得分: quality_score, terrain_quality_score, robust_score, benchmark_score +4. 添加`--spawn-type`配置, 根据评估和搜索修改初始生成点, 完成slope, wave, stairs up, stairs down, obstacle +5. 重新详细设计整个打分标准, 引入几个阶段性得分: quality_score, terrain_quality_score, robust_score, benchmark_score ## 20251227 ### v0.1.17 1. args中添加`goals`配置, 在LevelPipeline和MultiPipeline中指定goals, 修正压力测试中评估的目标不对的问题 diff --git a/robogauge/tasks/__init__.py b/robogauge/tasks/__init__.py index d2f7242..61dce5d 100644 --- a/robogauge/tasks/__init__.py +++ b/robogauge/tasks/__init__.py @@ -1,6 +1,6 @@ from robogauge.utils.task_register import task_register from robogauge.tasks.simulator.mujoco_config import MujocoConfig -from robogauge.tasks.robots import RobotConfig, Go2Config, Go2MoEConfig, Go2TerrainConfig, Go2MoETerrainConfig +from robogauge.tasks.robots import RobotConfig, Go2Config, Go2MoEConfig, Go2TerrainConfig, Go2MoETerrainConfig, Go2StairsConfig, Go2MoEStairsConfig from robogauge.tasks.pipeline import BasePipeline from robogauge.tasks.gauge import BaseGaugeConfig diff --git a/robogauge/tasks/custom/go2/go2_flat_task.py b/robogauge/tasks/custom/go2/go2_flat_task.py index c07aaa1..1c598f4 100644 --- a/robogauge/tasks/custom/go2/go2_flat_task.py +++ b/robogauge/tasks/custom/go2/go2_flat_task.py @@ -30,7 +30,7 @@ class Go2FlatGaugeConfig(FlatGaugeConfig): cmd_duration = 6.0 class target_pos_velocity(FlatGaugeConfig.goals.target_pos_velocity): # goal to reach a target position by velocity command, config target at assets.target_pos - enabled = True + enabled = False target_pos = [2, 2, 0] # x y z [m], target position in the environment, used for target position goal lin_vel_x = 1.0 # +/- m/s lin_vel_y = 1.0 # +/- m/s diff --git a/robogauge/tasks/custom/go2/go2_stairs_up_task.py b/robogauge/tasks/custom/go2/go2_stairs_up_task.py index c108f91..6639e85 100644 --- a/robogauge/tasks/custom/go2/go2_stairs_up_task.py +++ b/robogauge/tasks/custom/go2/go2_stairs_up_task.py @@ -7,7 +7,6 @@ @Blog : https://wty-yy.github.io/ @Desc : Go2 Wave Task Configuration ''' -from robogauge.tasks.robots import Go2Config, Go2MoEConfig from robogauge.tasks.gauge import StairsUpGaugeConfig from robogauge.tasks.simulator.mujoco_config import MujocoConfig diff --git a/robogauge/tasks/gauge/gauge_configs/terrain_levels_config.py b/robogauge/tasks/gauge/gauge_configs/terrain_levels_config.py index 82a10d8..678728c 100644 --- a/robogauge/tasks/gauge/gauge_configs/terrain_levels_config.py +++ b/robogauge/tasks/gauge/gauge_configs/terrain_levels_config.py @@ -9,6 +9,8 @@ ''' from robogauge.utils.config import Config +SEARCH_LEVELS_TERRAINS = ['slope', 'wave', 'stairs_up', 'stairs_down', 'obstacle'] + class TerrainSearchLevelsConfig(Config): class flat: levels = [0] @@ -91,6 +93,9 @@ class TerrainSearchLevelsConfig(Config): levels = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] class TerrainEvalLevelsConfig(Config): + class flat: + levels = [0] + class slope: levels = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] spawns = [ @@ -105,3 +110,63 @@ class TerrainEvalLevelsConfig(Config): [5.0, 0, 4.2 * (0.57 - 0.047)], [5.0, 0, 4.2 * 0.57], ] + + class wave: + levels = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] + spawns = [ + [4.55, 0, 0.65 - 0.075 * 9], + [4.55, 0, 0.65 - 0.075 * 8], + [4.55, 0, 0.65 - 0.075 * 7], + [4.55, 0, 0.65 - 0.075 * 6], + [4.55, 0, 0.65 - 0.075 * 5], + [4.55, 0, 0.65 - 0.075 * 4], + [4.55, 0, 0.65 - 0.075 * 3], + [4.55, 0, 0.65 - 0.075 * 2], + [4.55, 0, 0.65 - 0.075 * 1], + [4.55, 0, 0.65], + ] + + class stairs_up: + levels = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] + spawns = [ + [5.0, 0.0, 1.35], + [5.0, 0.0, 1.80], + [5.0, 0.0, 2.25 + 0.03], + [5.0, 0.0, 2.70 + 0.08], + [5.0, 0.0, 2.85 + 0.1], + [5.0, 0.0, 3.00 + 0.12], + [5.0, 0.0, 3.15 + 0.14], + [5.0, 0.0, 3.30 + 0.16], + [5.0, 0.0, 3.45 + 0.18], + [5.0, 0.0, 3.60 + 0.2], + ] + + class stairs_down: + levels = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] + spawns = [ + [5.1, 0.0, 1.35], + [5.1, 0.0, 1.80 + 0.08], + [5.1, 0.0, 2.25 + 0.15], + [5.1, 0.0, 2.70 + 0.25], + [5.1, 0.0, 2.85 + 0.27], + [5.1, 0.0, 3.00 + 0.30], + [5.1, 0.0, 3.15 + 0.33], + [5.1, 0.0, 3.30 + 0.36], + [5.1, 0.0, 3.45 + 0.39], + [5.1, 0.0, 3.60 + 0.42], + ] + + class obstacle: + levels = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] + spawns = [ + [5, 0.0, 0.20 - 0.023 * 9], + [5, 0.0, 0.20 - 0.023 * 8], + [5, 0.0, 0.20 - 0.023 * 7], + [5, 0.0, 0.20 - 0.023 * 6], + [5, 0.0, 0.20 - 0.023 * 5], + [5, 0.0, 0.20 - 0.023 * 4], + [5, 0.0, 0.20 - 0.023 * 3], + [5, 0.0, 0.20 - 0.023 * 2], + [5, 0.0, 0.20 - 0.023 * 1], + [5, 0.0, 0.20], + ] diff --git a/robogauge/tasks/pipeline/multi_pipeline.py b/robogauge/tasks/pipeline/multi_pipeline.py index e82e286..e1d4a70 100644 --- a/robogauge/tasks/pipeline/multi_pipeline.py +++ b/robogauge/tasks/pipeline/multi_pipeline.py @@ -25,6 +25,7 @@ from robogauge.utils.logger import Logger from robogauge.utils.process_utils import NoDaemonPool from robogauge.utils.progress_monitor import report_progress, ProgressTypes, ProgressData from robogauge.utils.file_utils import compress_directory +from robogauge.tasks.gauge.gauge_configs.terrain_levels_config import SEARCH_LEVELS_TERRAINS multi_logger = Logger() # MultiPipeline logger @@ -132,7 +133,7 @@ class MultiPipeline: """ Process results from all processes and aggregate them. """ multi_logger.info("📊 Aggregating Results from all runs...") - summary = {'success': {}, **self.static_info, 'summary': {}} + summary = {'success': {}, **self.static_info, 'summary': {}, 'quality_score': {}, 'terrain_quality_score': {}} finish_msg = ( f"""\n{'='*20} Run Finish Summary {'='*20}\n""" f"""{'Seed':^10}{'Base Mass':^15}{'Friction':^15}{'Status':^10}\n""" @@ -151,6 +152,7 @@ class MultiPipeline: multi_logger.error("No results to aggregate.") return + quality_score, terrain_quality_score = summary['quality_score'], summary['terrain_quality_score'] value_collections = defaultdict(lambda: defaultdict(list)) for result in all_results: for goal, metrics in result['results'].items(): @@ -159,11 +161,22 @@ class MultiPipeline: for metric, means in metrics.items(): for mean_name, mean_value in means.items(): value_collections[metric][mean_name].append(float(mean_value.split(' ')[0])) + quality_score[mean_name] = 1 for metric, means in value_collections.items(): summary['summary'][metric] = {} for mean_name, values in means.items(): - summary['summary'][metric][mean_name] = f"{float(np.mean(values)):.4f} ± {float(np.std(values)):.4f}" + v = float(np.mean(values)) + summary['summary'][metric][mean_name] = f"{v:.4f} ± {float(np.std(values)):.4f}" + weight = 1 + if metric in ['ang_vel_err', 'lin_vel_err']: + weight = 2 + quality_score[mean_name] *= min(max(1e-9, v), 1.0) ** weight + for mean_name in quality_score: + quality_score[mean_name] = quality_score[mean_name] ** (1 / 8) # 2 + 2 + 1 * 4 + terrain_quality_score[mean_name] = quality_score[mean_name] + if summary['terrain_name'] in SEARCH_LEVELS_TERRAINS: + terrain_quality_score[mean_name] = 0.09 * (summary['terrain_level'] - 1) + 0.19 * quality_score[mean_name] save_path = multi_logger.log_dir / "aggregated_results.yaml" with open(save_path, 'w') as file: diff --git a/robogauge/tasks/pipeline/stress_pipeline.py b/robogauge/tasks/pipeline/stress_pipeline.py index add375d..4317ecb 100644 --- a/robogauge/tasks/pipeline/stress_pipeline.py +++ b/robogauge/tasks/pipeline/stress_pipeline.py @@ -27,7 +27,7 @@ from robogauge.utils.logger import Logger from robogauge.utils.process_utils import NoDaemonPool from robogauge.utils.progress_monitor import report_progress, ProgressTypes, start_progress_monitor_thread, ProgressData from robogauge.tasks.pipeline import MultiPipeline, LevelPipeline -from robogauge.tasks.gauge.gauge_configs.terrain_levels_config import TerrainSearchLevelsConfig +from robogauge.tasks.gauge.gauge_configs.terrain_levels_config import SEARCH_LEVELS_TERRAINS from robogauge.utils.file_utils import compress_directory stress_logger = Logger() # StressPipeline logger @@ -57,6 +57,7 @@ def run_pipeline(args, progress_queue, data): if search is True: args.goals = GOALS['level_pipeline'] + args.spawn_type = "level_search" level, level_results = LevelPipeline(args, console_output=False, progress_data=progress_data).run() if level == 0: # no valid level found report_progress(progress_data, ProgressTypes.FINISH, desc=f"❌ Failed (Lv 0)") @@ -76,6 +77,7 @@ def run_pipeline(args, progress_queue, data): args.level = level args.goals = GOALS['multi_pipeline'] + args.spawn_type = "level_eval" results = { 'success': True, 'results': MultiPipeline(args, console_output=False, progress_data=progress_data).run(), @@ -113,12 +115,9 @@ class StressPipeline: ### Build worker data ### workers_data = [] - terrain_search_levels_config = TerrainSearchLevelsConfig() for terrain_name in terrain_names: search_max_level = True - terrain_level_cfg = getattr(terrain_search_levels_config, terrain_name, None) - assert terrain_level_cfg is not None, f"Terrain '{terrain_name}' not found in TerrainSearchLevelsConfig." - if len(terrain_level_cfg.levels) == 1: # Flattened terrain + if terrain_name not in SEARCH_LEVELS_TERRAINS: # Flattened terrain search_max_level = False data = { 'task_robot_model': self.task_robot_model, @@ -179,8 +178,9 @@ class StressPipeline: stress_logger.error("No results to aggregate.") return - summary = {**self.static_info, 'summary': {}, 'final_score': {}} - value_collections = defaultdict(lambda: defaultdict(list)) + summary = {**self.static_info, 'summary': {}, 'robust_score': {}, 'benchmark_score': 0.0} + metric_collections = defaultdict(lambda: defaultdict(list)) + terrain_collections = defaultdict(lambda: defaultdict(list)) zero_terrain_count = 0 for result in all_results: terrain_name = result['data']['terrain_name'] @@ -194,21 +194,27 @@ class StressPipeline: summary[key] = result['results'] for metric, means in result['results']['summary'].items(): - for mean_name, mean_value in means.items(): - value = float(mean_value.split(' ')[0]) - if terrain_level is not None: # level terrain - value = (terrain_level - 1) * 0.1 + value * 0.1 - value_collections[metric][mean_name].append(value) + for mean_name, value_str in means.items(): + value = float(value_str.split(' ± ')[0]) + metric_collections[metric][mean_name].append(value) + for mean_name, value in result['results']['terrain_quality_score'].items(): + terrain_collections[terrain_name][mean_name].append(value) - final_metrics = defaultdict(list) - for metric, means in value_collections.items(): + for metric, means in metric_collections.items(): summary['summary'][metric] = {} for mean_name, values in means.items(): values.extend([0.0] * zero_terrain_count) # include zero terrains summary['summary'][metric][mean_name] = f"{float(np.mean(values)):.4f} ± {float(np.std(values)):.4f}" - final_metrics[mean_name].append(np.mean(values)) - for mean_name, values in final_metrics.items(): - summary['final_score'][mean_name] = float(np.mean(values)) + + robust_score = defaultdict(dict) + robust_scores = [] + for terrain_name, means in terrain_collections.items(): + for mean_name, values in means.items(): + values.extend([0.0] * zero_terrain_count) # include zero terrains + robust_score[terrain_name][mean_name] = float(np.mean(values)) + robust_scores.append(robust_score[terrain_name][mean_name]) + summary['robust_score'] = dict(robust_score) + summary['benchmark_score'] = float(np.mean(robust_scores)) save_path = stress_logger.log_dir / "stress_benchmark_results.yaml" with open(save_path, 'w') as file: diff --git a/robogauge/tasks/robots/__init__.py b/robogauge/tasks/robots/__init__.py index c8f7d39..d3fe888 100644 --- a/robogauge/tasks/robots/__init__.py +++ b/robogauge/tasks/robots/__init__.py @@ -1,6 +1,6 @@ from .base_robot_config import RobotConfig from .base_robot import BaseRobot -from .go2.go2_config import Go2Config, Go2TerrainConfig +from .go2.go2_config import Go2Config, Go2TerrainConfig, Go2StairsConfig from .go2.go2 import Go2 -from .go2.go2_moe_config import Go2MoEConfig, Go2MoETerrainConfig +from .go2.go2_moe_config import Go2MoEConfig, Go2MoETerrainConfig, Go2MoEStairsConfig from .go2.go2_moe import Go2MoE diff --git a/robogauge/utils/helpers.py b/robogauge/utils/helpers.py index 84d00d2..ff856ce 100644 --- a/robogauge/utils/helpers.py +++ b/robogauge/utils/helpers.py @@ -76,6 +76,7 @@ def parse_args(): {"name": "--base-mass", "type": float, "default": 0.0, "help": "Set the base mass of the robot."}, {"name": "--friction", "type": float, "default": 1.0, "help": "Set the ground friction coefficient."}, {"name": "--level", "type": int, "help": "Set the difficulty level of the environment, range 1-10 (flat is 0)."}, + {"name": "--spawn-type", "type": str, "choices": ["level_eval", "level_search"], "help": "Spawn type for the robot."}, {"name": "--goals", "type": str, "nargs": "+", "help": "List of goal names to evaluate."}, # Multiprocessing parameters, with different seeds diff --git a/robogauge/utils/task_register.py b/robogauge/utils/task_register.py index eb324cd..3e12851 100644 --- a/robogauge/utils/task_register.py +++ b/robogauge/utils/task_register.py @@ -10,7 +10,7 @@ from robogauge import ROBOGAUGE_ROOT_DIR from robogauge.utils.logger import logger from robogauge.utils.helpers import parse_args, set_seed, class_to_dict -from robogauge.tasks.gauge.gauge_configs.terrain_levels_config import TerrainSearchLevelsConfig +from robogauge.tasks.gauge.gauge_configs.terrain_levels_config import TerrainSearchLevelsConfig, TerrainEvalLevelsConfig class TaskRegister(): def __init__(self): @@ -72,14 +72,18 @@ class TaskRegister(): sim_cfg.domain_rand.base_mass = args.base_mass if args.level is not None: gauger_cfg.assets.terrain_level = args.level - levels_cfg = TerrainSearchLevelsConfig() - cfg = getattr(levels_cfg, gauger_cfg.assets.terrain_name, None) - assert cfg is not None, f"Level {args.level} configuration not found in TerrainLevelsConfig." - assert args.level in cfg.levels, f"Level must be in {cfg.levels}." - if hasattr(cfg, 'targets'): - gauger_cfg.goals.target_pos_velocity.target_pos = cfg.targets[cfg.levels.index(args.level)] - if hasattr(cfg, 'spawns'): - gauger_cfg.assets.terrain_spawn_pos = cfg.spawns[cfg.levels.index(args.level)] + search_level_cfg = TerrainSearchLevelsConfig() + eval_level_cfg = TerrainEvalLevelsConfig() + search_cfg = getattr(search_level_cfg, gauger_cfg.assets.terrain_name, None) + eval_cfg = getattr(eval_level_cfg, gauger_cfg.assets.terrain_name, None) + assert search_cfg is not None and eval_cfg is not None, f"Level {args.level} configuration not found in TerrainLevelsConfig or TerrainEvalLevelsConfig." + assert args.level in search_cfg.levels and args.level in eval_cfg.levels, f"Level must be in {search_cfg.levels=} and {eval_cfg.levels=}." + if hasattr(search_cfg, 'targets'): + gauger_cfg.goals.target_pos_velocity.target_pos = search_cfg.targets[search_cfg.levels.index(args.level)] + if hasattr(search_cfg, 'spawns') and args.spawn_type == "level_search": + gauger_cfg.assets.terrain_spawn_pos = search_cfg.spawns[search_cfg.levels.index(args.level)] + elif hasattr(eval_cfg, 'spawns') and args.spawn_type == "level_eval": + gauger_cfg.assets.terrain_spawn_pos = eval_cfg.spawns[eval_cfg.levels.index(args.level)] xml = gauger_cfg.assets.terrain_xmls[0] xml = xml.rsplit('/', 1)[0] + f"/{gauger_cfg.assets.terrain_name}_{args.level}.xml" gauger_cfg.assets.terrain_xmls[0] = xml