v0.1.16; Add StressPipeline
This commit is contained in:
wty-yy
@@ -4,8 +4,8 @@
|
||||
# specified in default ./evaluate_models.txt file.
|
||||
#
|
||||
# Usage: ./run_scripts.sh [-n EXP_NAME] [-t TASK_NAME] [-s]
|
||||
# -n EXP_NAME Experiment name (default: go2_moe_flat)
|
||||
# -t TASK_NAME Task name (default: go2_moe_flat)
|
||||
# -n EXP_NAME Experiment name (default: go2_moe.flat)
|
||||
# -t TASK_NAME Task name (default: go2_moe.flat)
|
||||
# -s Save video (default: false)
|
||||
# -h Show this help message
|
||||
|
||||
@@ -16,7 +16,7 @@ RUN_PY="$SCRIPT_DIR/run.py"
|
||||
|
||||
### Default Configure ###
|
||||
EXP_NAME="" # Experiment name [-n]
|
||||
TASK_NAME="go2_moe_flat" # Task name [-t]
|
||||
TASK_NAME="go2_moe.flat" # Task name [-t]
|
||||
SAVE_VIDEO=false # Whether to save video [-s]
|
||||
|
||||
### Parse Arguments ###
|
||||
|
||||
@@ -1,19 +1,24 @@
|
||||
from robogauge.utils.task_register import task_register
|
||||
from robogauge.tasks.simulator.mujoco_config import MujocoConfig
|
||||
from robogauge.tasks.robots import RobotConfig, Go2Config, Go2MoEConfig
|
||||
from robogauge.tasks.robots import RobotConfig, Go2Config, Go2MoEConfig, Go2TerrainConfig, Go2MoETerrainConfig
|
||||
from robogauge.tasks.pipeline import BasePipeline
|
||||
from robogauge.tasks.gauge import BaseGaugeConfig
|
||||
|
||||
from robogauge.tasks.custom.go2 import *
|
||||
|
||||
# Register tasks: Task name format '<robot_model>.<terrain>'
|
||||
task_register.register('base', BasePipeline, MujocoConfig, BaseGaugeConfig, RobotConfig)
|
||||
task_register.register('go2_flat', BasePipeline, MujocoConfig, Go2FlatGaugeConfig, Go2Config)
|
||||
task_register.register('go2_moe_flat', BasePipeline, MujocoConfig, Go2FlatGaugeConfig, Go2MoEConfig)
|
||||
task_register.register('go2_slope', BasePipeline, Go2SlopeMujocoConfig, Go2SlopeGaugeConfig, Go2Config)
|
||||
task_register.register('go2_moe_slope', BasePipeline, Go2SlopeMujocoConfig, Go2SlopeGaugeConfig, Go2MoEConfig)
|
||||
task_register.register('go2_wave', BasePipeline, MujocoConfig, Go2WaveGaugeConfig, Go2Config)
|
||||
task_register.register('go2_moe_wave', BasePipeline, MujocoConfig, Go2WaveGaugeConfig, Go2MoEConfig)
|
||||
task_register.register('go2_stairs_up', BasePipeline, MujocoConfig, Go2StairsUpGaugeConfig, Go2Config)
|
||||
task_register.register('go2_moe_stairs_up', BasePipeline, MujocoConfig, Go2StairsUpGaugeConfig, Go2MoEConfig)
|
||||
task_register.register('go2_stairs_down', BasePipeline, MujocoConfig, Go2StairsDownGaugeConfig, Go2Config)
|
||||
task_register.register('go2_moe_stairs_down', BasePipeline, MujocoConfig, Go2StairsDownGaugeConfig, Go2MoEConfig)
|
||||
|
||||
# Go2 MLP
|
||||
task_register.register('go2.flat', BasePipeline, MujocoConfig, Go2FlatGaugeConfig, Go2Config)
|
||||
task_register.register('go2.slope', BasePipeline, MujocoConfig, Go2SlopeGaugeConfig, Go2TerrainConfig)
|
||||
task_register.register('go2.wave', BasePipeline, MujocoConfig, Go2WaveGaugeConfig, Go2TerrainConfig)
|
||||
task_register.register('go2.stairs_up', BasePipeline, MujocoConfig, Go2StairsUpGaugeConfig, Go2TerrainConfig)
|
||||
task_register.register('go2.stairs_down', BasePipeline, MujocoConfig, Go2StairsDownGaugeConfig, Go2TerrainConfig)
|
||||
|
||||
# Go2 MoE
|
||||
task_register.register('go2_moe.flat', BasePipeline, MujocoConfig, Go2FlatGaugeConfig, Go2MoEConfig)
|
||||
task_register.register('go2_moe.slope', BasePipeline, MujocoConfig, Go2SlopeGaugeConfig, Go2MoETerrainConfig)
|
||||
task_register.register('go2_moe.wave', BasePipeline, MujocoConfig, Go2WaveGaugeConfig, Go2MoETerrainConfig)
|
||||
task_register.register('go2_moe.stairs_up', BasePipeline, MujocoConfig, Go2StairsUpGaugeConfig, Go2MoETerrainConfig)
|
||||
task_register.register('go2_moe.stairs_down', BasePipeline, MujocoConfig, Go2StairsDownGaugeConfig, Go2MoETerrainConfig)
|
||||
|
||||
@@ -7,7 +7,7 @@
|
||||
@Blog : https://wty-yy.github.io/
|
||||
'''
|
||||
from .go2_flat_task import Go2FlatGaugeConfig
|
||||
from .go2_slope_task import Go2SlopeGaugeConfig, Go2SlopeMujocoConfig
|
||||
from .go2_slope_task import Go2SlopeGaugeConfig
|
||||
from .go2_wave_task import Go2WaveGaugeConfig
|
||||
from .go2_stairs_up_task import Go2StairsUpGaugeConfig
|
||||
from .go2_stairs_down_task import Go2StairsDownGaugeConfig
|
||||
|
||||
@@ -17,8 +17,3 @@ class Go2SlopeGaugeConfig(SlopeGaugeConfig):
|
||||
enabled = True
|
||||
soft_dof_limit_ratio = 0.7
|
||||
dof_names = ['hip', 'thigh'] # List of DOF names to monitor, None for all
|
||||
|
||||
class Go2SlopeMujocoConfig(MujocoConfig):
|
||||
class domain_rand(MujocoConfig.domain_rand):
|
||||
action_delay = True
|
||||
friction = 2.4
|
||||
|
||||
@@ -22,7 +22,8 @@ class BaseGaugeConfig(Config):
|
||||
class goals:
|
||||
class max_velocity: # goal with maximum velocity
|
||||
enabled = False
|
||||
cmd_duration = 5.0 # [s] duration for each velocity command
|
||||
move_duration = 5.0 # [s] duration for each velocity command
|
||||
stance_duration = 2.0 # [s] duration for each stance (no movement)
|
||||
|
||||
class diagonal_velocity: # goal with diagonal velocity changes
|
||||
enabled = False
|
||||
|
||||
@@ -14,19 +14,20 @@ class SlopeGaugeConfig(BaseGaugeConfig):
|
||||
|
||||
class assets(BaseGaugeConfig.assets):
|
||||
terrain_name = "slope"
|
||||
terrain_level = 10 # 1-10
|
||||
terrain_level = 4 # 1-10
|
||||
terrain_xmls = [
|
||||
'{ROBOGAUGE_ROOT_DIR}/resources/terrains/slope/slope_10.xml',
|
||||
'{ROBOGAUGE_ROOT_DIR}/resources/terrains/slope/slope_4.xml',
|
||||
'{ROBOGAUGE_ROOT_DIR}/resources/terrains/wall/10x10_wall.xml',
|
||||
]
|
||||
terrain_spawn_pos = [0.8, 0, 1] # x y z [m], robot freejoint spawn position on the terrain
|
||||
terrain_spawn_pos = [0.5, 0, 0] # x y z [m], robot freejoint spawn position on the terrain
|
||||
|
||||
class goals(BaseGaugeConfig.goals):
|
||||
class target_pos_velocity: # goal to reach a target position by velocity command
|
||||
enabled = True
|
||||
target_pos = [4, 0, 2.28] # x y z [m], target position in the environment, used for target position goal
|
||||
target_pos = [4.8, 0, 1.152 + 0.1] # x y z [m], target position in the environment, used for target position goal
|
||||
lin_vel_x = 1.0 # +/- m/s
|
||||
lin_vel_y = 1.0 # +/- m/s
|
||||
ang_vel_yaw = 1.5 # +/- rad/s
|
||||
max_cmd_duration = 20.0 # [s] maximum duration to reach the target position
|
||||
reach_threshold = 0.1
|
||||
|
||||
@@ -9,7 +9,7 @@
|
||||
'''
|
||||
from robogauge.utils.config import Config
|
||||
|
||||
class TerrainLevelsConfig(Config):
|
||||
class TerrainSearchLevelsConfig(Config):
|
||||
class flat:
|
||||
levels = [0]
|
||||
targets = [[4, 0, 0]] # target positions for each level, if target goal is used
|
||||
@@ -17,28 +17,28 @@ class TerrainLevelsConfig(Config):
|
||||
class slope:
|
||||
levels = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
|
||||
targets = [
|
||||
[4, 0, 0.588 + 0.1],
|
||||
[4, 0, 0.776 + 0.1],
|
||||
[4, 0, 0.964 + 0.1],
|
||||
[4, 0, 1.152 + 0.1],
|
||||
[4, 0, 1.340 + 0.1],
|
||||
[4, 0, 1.528 + 0.1],
|
||||
[4, 0, 1.716 + 0.1],
|
||||
[4, 0, 1.904 + 0.1],
|
||||
[4, 0, 2.092 + 0.1],
|
||||
[4, 0, 2.280 + 0.1],
|
||||
[4.8, 0, 0.588 + 0.1],
|
||||
[4.8, 0, 0.776 + 0.1],
|
||||
[4.8, 0, 0.964 + 0.1],
|
||||
[4.8, 0, 1.152 + 0.1],
|
||||
[4.8, 0, 1.340 + 0.1],
|
||||
[4.8, 0, 1.528 + 0.1],
|
||||
[4.8, 0, 1.716 + 0.1],
|
||||
[4.8, 0, 1.904 + 0.1],
|
||||
[4.8, 0, 2.092 + 0.1],
|
||||
[4.8, 0, 2.280 + 0.1],
|
||||
]
|
||||
spawns = [
|
||||
[0.8, 0, 0.55 - 0.037 * 9],
|
||||
[0.8, 0, 0.55 - 0.037 * 8],
|
||||
[0.8, 0, 0.55 - 0.037 * 7],
|
||||
[0.8, 0, 0.55 - 0.037 * 6],
|
||||
[0.8, 0, 0.55 - 0.037 * 5],
|
||||
[0.8, 0, 0.55 - 0.037 * 4],
|
||||
[0.8, 0, 0.55 - 0.037 * 3],
|
||||
[0.8, 0, 0.55 - 0.037 * 2],
|
||||
[0.8, 0, 0.55 - 0.037],
|
||||
[0.8, 0, 0.55],
|
||||
[0.5, 0, 0],
|
||||
[0.5, 0, 0],
|
||||
[0.5, 0, 0],
|
||||
[0.5, 0, 0],
|
||||
[0.5, 0, 0],
|
||||
[0.5, 0, 0],
|
||||
[0.5, 0, 0],
|
||||
[0.5, 0, 0],
|
||||
[0.5, 0, 0],
|
||||
[0.5, 0, 0],
|
||||
]
|
||||
|
||||
class wave:
|
||||
@@ -98,3 +98,19 @@ class TerrainLevelsConfig(Config):
|
||||
[1.1, 3.0, 7 - 0.3],
|
||||
[1.1, 3.0, 7],
|
||||
]
|
||||
|
||||
class TerrainEvalLevelsConfig(Config):
|
||||
class slope:
|
||||
levels = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
|
||||
spawns = [
|
||||
[5.0, 0, 4.2 * (0.57 - 0.047 * 9)],
|
||||
[5.0, 0, 4.2 * (0.57 - 0.047 * 8)],
|
||||
[5.0, 0, 4.2 * (0.57 - 0.047 * 7)],
|
||||
[5.0, 0, 4.2 * (0.57 - 0.047 * 6)],
|
||||
[5.0, 0, 4.2 * (0.57 - 0.047 * 5)],
|
||||
[5.0, 0, 4.2 * (0.57 - 0.047 * 4)],
|
||||
[5.0, 0, 4.2 * (0.57 - 0.047 * 3)],
|
||||
[5.0, 0, 4.2 * (0.57 - 0.047 * 2)],
|
||||
[5.0, 0, 4.2 * (0.57 - 0.047)],
|
||||
[5.0, 0, 4.2 * 0.57],
|
||||
]
|
||||
|
||||
@@ -1 +1,4 @@
|
||||
from .base_pipeline import BasePipeline
|
||||
from .base_pipeline import BasePipeline
|
||||
from .multi_pipeline import MultiPipeline
|
||||
from .level_pipeline import LevelPipeline
|
||||
from .stress_pipeline import StressPipeline
|
||||
|
||||
@@ -10,17 +10,19 @@
|
||||
import yaml
|
||||
|
||||
from robogauge.tasks.pipeline.multi_pipeline import MultiPipeline
|
||||
from robogauge.utils.logger import logger
|
||||
from robogauge.utils.logger import Logger
|
||||
|
||||
level_logger = Logger() # LevelPipeline logger
|
||||
|
||||
class LevelPipeline:
|
||||
def __init__(self, args):
|
||||
self.args = args
|
||||
self.seeds = args.seeds
|
||||
logger.create(args.experiment_name+'_level', args.run_name)
|
||||
level_logger.create(args.experiment_name+'_level', args.run_name)
|
||||
|
||||
def run(self):
|
||||
logger.info(f"🚀 Starting Level Seacher for '{self.args.experiment_name}'.")
|
||||
logger.info(f"🔢 Seeds: {self.seeds}")
|
||||
level_logger.info(f"🚀 Starting Level Searcher for '{self.args.experiment_name}'.")
|
||||
level_logger.info(f"🔢 Seeds: {self.seeds}")
|
||||
|
||||
# binary search levels
|
||||
l, r = 0, 10
|
||||
@@ -40,22 +42,22 @@ class LevelPipeline:
|
||||
'terrain_level': 0,
|
||||
})
|
||||
if level >= 1:
|
||||
logger.info(f"🏆 Found maximum level: {level}")
|
||||
level_logger.info(f"🏆 Found maximum level: {level}")
|
||||
else:
|
||||
logger.info(f"❌ No valid level found [1-10].")
|
||||
with open(logger.log_dir / "level_search_results.yaml", 'w') as f:
|
||||
level_logger.info(f"❌ No valid level found [1-10].")
|
||||
with open(level_logger.log_dir / "level_search_results.yaml", 'w') as f:
|
||||
yaml.dump(level_results, f, allow_unicode=True, sort_keys=False)
|
||||
return level, level_results
|
||||
|
||||
def test_level(self, level: int) -> bool:
|
||||
logger.info(f"🔍 Testing level {level}...")
|
||||
def test_level(self, level: int):
|
||||
level_logger.info(f"🔍 Testing level {level}...")
|
||||
self.args.level = level
|
||||
multi_pipeline = MultiPipeline(self.args)
|
||||
aggregated_results = multi_pipeline.run()
|
||||
success_mean = float(aggregated_results['success']['mean'].split(' ')[0])
|
||||
all_success = success_mean >= 0.8
|
||||
if all_success:
|
||||
logger.info(f"✅ Level {level} passed all tests.")
|
||||
level_logger.info(f"✅ Level {level} passed all tests.")
|
||||
else:
|
||||
logger.info(f"❌ Level {level} failed some tests.")
|
||||
level_logger.info(f"❌ Level {level} failed some tests.")
|
||||
return all_success, aggregated_results
|
||||
|
||||
@@ -21,9 +21,12 @@ from collections import defaultdict
|
||||
from robogauge.tasks.pipeline.base_pipeline import BasePipeline
|
||||
|
||||
from robogauge.utils.task_register import task_register
|
||||
from robogauge.utils.logger import logger
|
||||
from robogauge.utils.logger import Logger
|
||||
|
||||
multi_logger = Logger() # MultiPipeline logger
|
||||
|
||||
def run_single_process(args, data):
|
||||
from robogauge.utils.logger import logger
|
||||
seed, base_mass, friction = data
|
||||
local_args = deepcopy(args)
|
||||
local_args.seed = seed
|
||||
@@ -41,6 +44,7 @@ def run_single_process(args, data):
|
||||
ret = {
|
||||
'status': 'success',
|
||||
'results': results,
|
||||
'data': data,
|
||||
'model_path': pipeline.robot_cfg.control.model_path,
|
||||
}
|
||||
if warning is not None:
|
||||
@@ -65,7 +69,7 @@ class MultiPipeline:
|
||||
self.base_masses = args.base_masses
|
||||
self.num_processes = args.num_processes
|
||||
self.static_info = {}
|
||||
logger.create(args.experiment_name+'_multi', args.run_name+'_multi')
|
||||
multi_logger.create(args.experiment_name+'_multi', args.run_name+'_multi')
|
||||
|
||||
def add_static_info(self, key: str, value):
|
||||
if key not in self.static_info:
|
||||
@@ -74,55 +78,54 @@ class MultiPipeline:
|
||||
assert self.static_info[key] == value, f"Static info key '{key}' has conflicting values: {self.static_info[key]} vs {value}"
|
||||
|
||||
def run(self):
|
||||
logger.info(f"🚀 Starting Multi-Process Evaluation with {self.num_processes} processes.")
|
||||
logger.info(f"🔢 Seeds: {self.seeds}, Frictions: {self.frictions}, Base masses: {self.base_masses}")
|
||||
multi_logger.info(f"🚀 Starting Multi-Process Evaluation with {self.num_processes} processes.")
|
||||
multi_logger.info(f"🔢 Seeds: {self.seeds}, Frictions: {self.frictions}, Base masses: {self.base_masses}")
|
||||
|
||||
workers_data = list(product(self.seeds, self.base_masses, self.frictions))
|
||||
ctx = multiprocessing.get_context('spawn')
|
||||
worker_func = functools.partial(run_single_process, self.args)
|
||||
results_list = []
|
||||
success_flags = []
|
||||
with ctx.Pool(processes=self.num_processes) as pool:
|
||||
iterator = pool.imap_unordered(worker_func, workers_data)
|
||||
for results in tqdm(iterator, total=len(workers_data), desc="Evaluation"):
|
||||
success_flags.append(results['status'] == 'success')
|
||||
|
||||
results_list.append(results['results'])
|
||||
results_list.append(results)
|
||||
self.add_static_info('model_path', results['model_path'])
|
||||
self.add_static_info('terrain_name', results['results']['terrain_name'])
|
||||
self.add_static_info('terrain_level', results['results']['terrain_level'])
|
||||
if results['status'] != 'success':
|
||||
data = results['data']
|
||||
logger.error(f"❌ Process with seed={data[0]}, base_mass={data[1]}, friction={data[2]} failed with error: {results['error_msg']}")
|
||||
multi_logger.error(f"❌ Process with seed={data[0]}, base_mass={data[1]}, friction={data[2]} failed with error: {results['error_msg']}")
|
||||
|
||||
logger.info("✅ Multi-Process Evaluation Completed.")
|
||||
aggregated_results = self.aggregate_results(results_list, success_flags, workers_data)
|
||||
multi_logger.info("✅ Multi-Process Evaluation Completed.")
|
||||
aggregated_results = self.aggregate_results(results_list)
|
||||
return aggregated_results
|
||||
|
||||
def aggregate_results(self, all_results, success_flags, workers_data):
|
||||
def aggregate_results(self, all_results):
|
||||
""" Process results from all processes and aggregate them. """
|
||||
logger.info("📊 Aggregating Results from all runs...")
|
||||
multi_logger.info("📊 Aggregating Results from all runs...")
|
||||
|
||||
summary = {'success': {}, **self.static_info}
|
||||
finish_msg = (
|
||||
f"""\n{'='*20} Run Finish Summary {'='*20}\n"""
|
||||
f"""{'Seed':^10}{'Base Mass':^15}{'Friction':^15}{'Status':^10}\n"""
|
||||
)
|
||||
for success, data in zip(success_flags, workers_data):
|
||||
seed, base_mass, friction = data
|
||||
all_results = sorted(all_results, key=lambda x: x['data'])
|
||||
for result in all_results:
|
||||
seed, base_mass, friction = result['data']
|
||||
success = result['status'] == 'success'
|
||||
status_str = "✅" if success else "❌"
|
||||
finish_msg += f"{seed:^10}{base_mass:^15}{friction:^15}{status_str:^10}\n"
|
||||
summary['success'][f"Seed_{seed}_BaseMass_{base_mass}_Friction_{friction}"] = True if success else False
|
||||
finish_msg += f"""{'='*88}"""
|
||||
logger.info(finish_msg)
|
||||
multi_logger.info(finish_msg)
|
||||
|
||||
if not all_results:
|
||||
logger.error("No results to aggregate.")
|
||||
multi_logger.error("No results to aggregate.")
|
||||
return
|
||||
|
||||
value_collections = defaultdict(lambda: defaultdict(list))
|
||||
for result in all_results:
|
||||
for goal, metrics in result.items():
|
||||
for goal, metrics in result['results'].items():
|
||||
if goal != 'summary':
|
||||
continue
|
||||
for metric, means in metrics.items():
|
||||
@@ -134,13 +137,13 @@ class MultiPipeline:
|
||||
for mean_name, values in means.items():
|
||||
summary[metric][mean_name] = f"{float(np.mean(values)):.4f} ± {float(np.std(values)):.4f}"
|
||||
|
||||
save_path = logger.log_dir / "aggregated_results.yaml"
|
||||
save_path = multi_logger.log_dir / "aggregated_results.yaml"
|
||||
with open(save_path, 'w') as file:
|
||||
yaml.dump(summary, file, allow_unicode=True, sort_keys=False)
|
||||
logger.info("✅ Aggregated execution finished.")
|
||||
logger.info(f"📁 Aggregated results saved to: {save_path}")
|
||||
multi_logger.info("✅ Aggregated execution finished.")
|
||||
multi_logger.info(f"📁 Aggregated results saved to: {save_path}")
|
||||
|
||||
logger.info(
|
||||
multi_logger.info(
|
||||
f"""\n{'='*20} Multi-Run Summary {'='*20}\n"""
|
||||
f"""{yaml.dump(summary, allow_unicode=True)}"""
|
||||
f"""{'='*60}"""
|
||||
|
||||
131
robogauge/tasks/pipeline/stress_pipeline.py
Normal file
131
robogauge/tasks/pipeline/stress_pipeline.py
Normal file
@@ -0,0 +1,131 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
'''
|
||||
@File : stress_pipeline.py
|
||||
@Time : 2025/12/25 21:20:43
|
||||
@Author : wty-yy
|
||||
@Version : 1.0
|
||||
@Blog : https://wty-yy.github.io/
|
||||
@Desc : Stress Pipeline for Robogauge
|
||||
'''
|
||||
import yaml
|
||||
import functools
|
||||
from tqdm import tqdm
|
||||
import multiprocessing
|
||||
from copy import deepcopy
|
||||
from itertools import product
|
||||
|
||||
from robogauge.utils.logger import Logger
|
||||
from robogauge.tasks.pipeline import MultiPipeline, LevelPipeline
|
||||
from robogauge.tasks.gauge.gauge_configs.terrain_levels_config import TerrainSearchLevelsConfig
|
||||
|
||||
stress_logger = Logger() # StressPipeline logger
|
||||
|
||||
def run_pipeline(args, data):
|
||||
args = deepcopy(args)
|
||||
search = data['search_max_level']
|
||||
if search is True:
|
||||
args.friction = data['friction']
|
||||
args.frictions = [data['friction']]
|
||||
args.base_mass = data['base_mass']
|
||||
args.base_masses = [data['base_mass']]
|
||||
args.task_name = f"{data['task_robot_model']}.{data['terrain_name']}"
|
||||
args.experiment_name = f"{args.experiment_name}_{data['terrain_name']}_baseMass{data['base_mass']}_friction{data['friction']}"
|
||||
else:
|
||||
args.task_name = f"{data['task_robot_model']}.{data['terrain_name']}"
|
||||
args.experiment_name = f"{args.experiment_name}_{data['terrain_name']}"
|
||||
|
||||
level = None # flat terrain
|
||||
if search:
|
||||
level, results = LevelPipeline(args).run()
|
||||
|
||||
if level == 0: # no valid level found
|
||||
results = {
|
||||
'success': False,
|
||||
'results': results,
|
||||
'data': data,
|
||||
}
|
||||
else:
|
||||
args.level = level
|
||||
results = {
|
||||
'success': True,
|
||||
'results': MultiPipeline(args).run(),
|
||||
'data': data,
|
||||
}
|
||||
|
||||
return results
|
||||
|
||||
class StressPipeline:
|
||||
def __init__(self, args):
|
||||
self.args = args
|
||||
self.seeds = args.seeds
|
||||
self.task_robot_model = args.task_name.split('.')[0]
|
||||
self.num_processes = args.stress_num_processes
|
||||
args.experiment_name = self.task_robot_model + '_stress' + ('' if args.cli_experiment_name is None else '_' + args.cli_experiment_name)
|
||||
self.static_info = {}
|
||||
stress_logger.create(args.experiment_name, args.run_name)
|
||||
|
||||
def add_static_info(self, key: str, value):
|
||||
if key not in self.static_info:
|
||||
self.static_info[key] = value
|
||||
else:
|
||||
assert self.static_info[key] == value, f"Static info key '{key}' has conflicting values: {self.static_info[key]} vs {value}"
|
||||
|
||||
def run(self):
|
||||
stress_logger.info(f"🚀 Starting Stress Benchmark for '{self.args.experiment_name}'.")
|
||||
stress_logger.info(f"🔢 Seeds: {self.seeds}")
|
||||
terrain_names = self.args.stress_terrain_names
|
||||
stress_logger.info(f"🌄 Stress Test Terrain Names: {terrain_names}")
|
||||
|
||||
ctx = multiprocessing.get_context('spawn')
|
||||
worker_func = functools.partial(run_pipeline, self.args)
|
||||
|
||||
### Build worker data ###
|
||||
workers_data = []
|
||||
terrain_search_levels_config = TerrainSearchLevelsConfig()
|
||||
for terrain_name in terrain_names:
|
||||
search_max_level = True
|
||||
terrain_level_cfg = getattr(terrain_search_levels_config, terrain_name, None)
|
||||
assert terrain_level_cfg is not None, f"Terrain '{terrain_name}' not found in TerrainSearchLevelsConfig."
|
||||
if len(terrain_level_cfg.levels) == 1: # Flattened terrain
|
||||
search_max_level = False
|
||||
data = {
|
||||
'task_robot_model': self.task_robot_model,
|
||||
'terrain_name': terrain_name,
|
||||
'search_max_level': search_max_level,
|
||||
}
|
||||
if search_max_level:
|
||||
for friction, base_mass in product(self.args.frictions, self.args.base_masses):
|
||||
data.update({
|
||||
'friction': friction,
|
||||
'base_mass': base_mass,
|
||||
})
|
||||
workers_data.append(data)
|
||||
|
||||
### Run and collect results ###
|
||||
results_list = []
|
||||
with ctx.Pool(processes=self.num_processes) as pool:
|
||||
iterator = pool.imap_unordered(worker_func, workers_data)
|
||||
for results in tqdm(iterator, total=len(workers_data), desc="Stress Benchmark"):
|
||||
results_list.append(results)
|
||||
self.add_static_info('model_path', results['results']['model_path'])
|
||||
|
||||
stress_logger.info("✅ Stress Benchmark Completed.")
|
||||
stress_results = self.aggregate_results(results_list)
|
||||
return stress_results
|
||||
|
||||
def aggregate_results(self, all_results):
|
||||
stress_logger.info("📊 Aggregating Stress Benchmark Results...")
|
||||
summary = {}
|
||||
finish_msg = (
|
||||
f"""\n{'='*20} Stress Benchmark Summary {'='*20}\n"""
|
||||
f"""{'Terrain Name':^20}{'Base Mass':^15}{'Friction':^15}{'Status':^10}\n"""
|
||||
)
|
||||
all_results = sorted(all_results, key=lambda x: (x['data']['terrain_name'], x['data'].get('base_mass', 0), x['data'].get('friction', 0)))
|
||||
for result in all_results:
|
||||
terrain_name = result['data']['terrain_name']
|
||||
base_mass = result['data'].get('base_mass', self.args.base_masses)
|
||||
friction = result['data'].get('friction', self.args.frictions)
|
||||
status = "✅" if result['success'] else "❌"
|
||||
finish_msg += f"{terrain_name:^20}{str(base_mass):^15}{str(friction):^15}{status:^10}\n"
|
||||
finish_msg += f"""{'='*66}"""
|
||||
stress_logger.info(finish_msg)
|
||||
@@ -1,6 +1,6 @@
|
||||
from .base_robot_config import RobotConfig
|
||||
from .base_robot import BaseRobot
|
||||
from .go2.go2_config import Go2Config
|
||||
from .go2.go2_config import Go2Config, Go2TerrainConfig
|
||||
from .go2.go2 import Go2
|
||||
from .go2.go2_moe_config import Go2MoEConfig
|
||||
from .go2.go2_moe_config import Go2MoEConfig, Go2MoETerrainConfig
|
||||
from .go2.go2_moe import Go2MoE
|
||||
|
||||
@@ -48,9 +48,15 @@ class Go2Config(RobotConfig):
|
||||
|
||||
class commands(RobotConfig.commands):
|
||||
lin_vel_x = [-2.0, 2.0] # min max [m/s]
|
||||
lin_vel_y = [-2.0, 2.0] # min max [m/s]
|
||||
lin_vel_y = [-1.0, 1.0] # min max [m/s]
|
||||
lin_vel_z = None # min max [m/s]
|
||||
ang_vel_roll = None # min max [rad/s]
|
||||
ang_vel_pitch = None # min max [rad/s]
|
||||
ang_vel_yaw = [-2, 2] # min max [rad/s]
|
||||
|
||||
class Go2TerrainConfig(Go2Config):
|
||||
""" Go2 Robot Configuration for Terrain Tasks (wave, stairs up/down, slope, obstacles) """
|
||||
class commands(Go2Config.commands):
|
||||
lin_vel_x = [-1.0, 1.0] # min max [m/s]
|
||||
lin_vel_y = [-1.0, 1.0] # min max [m/s]
|
||||
ang_vel_yaw = [-1.5, 1.5] # min max [rad/s]
|
||||
|
||||
@@ -14,3 +14,10 @@ class Go2MoEConfig(Go2Config):
|
||||
|
||||
class control(Go2Config.control):
|
||||
model_path = "{ROBOGAUGE_ROOT_DIR}/resources/models/go2/go2_moe_cts_124k.pt"
|
||||
|
||||
class Go2MoETerrainConfig(Go2MoEConfig):
|
||||
""" Go2 MoE Robot Configuration for Terrain Tasks (wave, stairs up/down, slope, obstacles) """
|
||||
class commands(Go2MoEConfig.commands):
|
||||
lin_vel_x = [-1.0, 1.0] # min max [m/s]
|
||||
lin_vel_y = [-1.0, 1.0] # min max [m/s]
|
||||
ang_vel_yaw = [-1.5, 1.5] # min max [rad/s]
|
||||
|
||||
@@ -49,4 +49,6 @@ class MujocoConfig(Config):
|
||||
class truncation:
|
||||
enabled = True
|
||||
projected_gravity_rad = 2.5 # [rad], if gravity projection angle exceeds this value, truncate episode
|
||||
penetration_threshold = -0.05 # [m], if any contact penetration depth is below this threshold, truncate episode
|
||||
penetration_threshold = -0.035 # [m], if any contact penetration depth is below this threshold, truncate episode
|
||||
skip_penetration_geoms = ['wall', 'floor'] # Geometries to skip penetration check
|
||||
skip_self_penetration = True # Whether to check self-penetration
|
||||
|
||||
@@ -283,9 +283,19 @@ class MujocoSimulator:
|
||||
if -projected_gravity[2] < np.cos(self.cfg.truncation.projected_gravity_rad):
|
||||
raise RuntimeError(f"[Roll Error] Episode truncated due to excessive projected gravity, angle: {np.arccos(-projected_gravity[2]):.3f} rad, projected: {projected_gravity}")
|
||||
|
||||
# is_penetrated, geom1, geom2, dist = self.check_penetration(self.cfg.truncation.penetration_threshold)
|
||||
# if is_penetrated:
|
||||
# raise RuntimeError(f"[Penetration Error] Episode truncated: Penetration ({geom1} <-> {geom2}), distance: {dist}")
|
||||
is_penetrated, geom1, geom2, dist = self.check_penetration(self.cfg.truncation.penetration_threshold)
|
||||
if is_penetrated:
|
||||
flat = True
|
||||
if self.cfg.truncation.skip_penetration_geoms is not None and (
|
||||
any(skip_geom in geom1.lower() for skip_geom in self.cfg.truncation.skip_penetration_geoms) or
|
||||
any(skip_geom in geom2.lower() for skip_geom in self.cfg.truncation.skip_penetration_geoms)
|
||||
):
|
||||
flat = False
|
||||
if self.cfg.truncation.skip_self_penetration:
|
||||
if geom1.split('/')[0] == geom2.split('/')[0]:
|
||||
flat = False
|
||||
if flat:
|
||||
raise RuntimeError(f"[Penetration Error] Episode truncated: Penetration ({geom1} <-> {geom2}), distance: {dist}")
|
||||
|
||||
def reset(self):
|
||||
""" Reset the simulator to initial state. """
|
||||
|
||||
@@ -80,20 +80,27 @@ def parse_args():
|
||||
# Multiprocessing parameters, with different seeds
|
||||
{"name": "--multi", "action": "store_true", "default": False, "help": "Enable multiprocessing."},
|
||||
{"name": "--num-processes", "type": int, "default": 2, "help": "Number of parallel processes."},
|
||||
{"name": "--seeds", "type": int, "nargs": "+", "default": [0], "help": "List of random seeds for multiple runs."},
|
||||
{"name": "--seeds", "type": int, "nargs": "+", "default": [0, 1, 2, 3, 4], "help": "List of random seeds for multiple runs."},
|
||||
{"name": "--base-masses", "type": float, "nargs": "+", "default": [0], "help": "List of base masses for the model."},
|
||||
{"name": "--frictions", "type": float, "nargs": "+", "default": [0.4, 0.7, 1.0, 1.3, 1.6], "help": "List of friction coefficients for the model."},
|
||||
{"name": "--frictions", "type": float, "nargs": "+", "default": [0.5, 0.75, 1.0, 1.25, 1.5, 1.75, 2.0, 2.25, 2.5], "help": "List of friction coefficients for the model."},
|
||||
|
||||
# Level pipeline parameters
|
||||
{"name": "--search-max-level", "action": "store_true", "default": False, "help": "Use level pipeline to search maximum level."},
|
||||
|
||||
# Stress pipeline parameters
|
||||
{"name": "--stress-benchmark", "action": "store_true", "default": False, "help": "Use stress pipeline to benchmark model robustness."},
|
||||
{"name": "--stress-terrain-names", "type": str, "nargs": "+", "default": ["flat", "slope", "wave", "stairs_up", "stairs_down"], "help": "List of terrain names for stress benchmark."},
|
||||
{"name": "--stress-num-processes", "type": int, "default": 2, "help": "Number of parallel processes for stress benchmark."},
|
||||
]
|
||||
for param in parameters:
|
||||
parser.add_argument(param['name'], **{k: v for k, v in param.items() if k != 'name'})
|
||||
args = parser.parse_args()
|
||||
flatten_task_name = args.task_name.replace('.', '_')
|
||||
args.cli_experiment_name = args.experiment_name
|
||||
if args.experiment_name is not None:
|
||||
args.experiment_name = f"{args.task_name}_{args.experiment_name}"
|
||||
args.experiment_name = f"{flatten_task_name}_{args.experiment_name}"
|
||||
else:
|
||||
args.experiment_name = args.task_name
|
||||
args.experiment_name = flatten_task_name
|
||||
return args
|
||||
|
||||
def snake_to_pascal(s: str) -> str:
|
||||
|
||||
@@ -10,7 +10,7 @@
|
||||
from robogauge import ROBOGAUGE_ROOT_DIR
|
||||
from robogauge.utils.logger import logger
|
||||
from robogauge.utils.helpers import parse_args, set_seed
|
||||
from robogauge.tasks.gauge.gauge_configs.terrain_levels_config import TerrainLevelsConfig
|
||||
from robogauge.tasks.gauge.gauge_configs.terrain_levels_config import TerrainSearchLevelsConfig
|
||||
|
||||
class TaskRegister():
|
||||
def __init__(self):
|
||||
@@ -72,7 +72,7 @@ class TaskRegister():
|
||||
sim_cfg.domain_rand.base_mass = args.base_mass
|
||||
if args.level is not None:
|
||||
gauger_cfg.assets.terrain_level = args.level
|
||||
levels_cfg = TerrainLevelsConfig()
|
||||
levels_cfg = TerrainSearchLevelsConfig()
|
||||
cfg = getattr(levels_cfg, gauger_cfg.assets.terrain_name)
|
||||
assert cfg is not None, f"Level {args.level} configuration not found in TerrainLevelsConfig."
|
||||
assert args.level in cfg.levels, f"Level must be in {cfg.levels}."
|
||||
|
||||
Reference in New Issue
Block a user