cyy/RoboGauge
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

v0.1.16; Add StressPipeline

Browse Source
This commit is contained in:
wty-yy
2025-12-26 14:47:39 +08:00
parent 9d9d83ed9f
commit 65490fe591
31 changed files with 318 additions and 122 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
robogauge/tasks/pipeline/__init__.py
View File

@@ -1 +1,4 @@
from .base_pipeline import BasePipeline
from .base_pipeline import BasePipeline
from .multi_pipeline import MultiPipeline
from .level_pipeline import LevelPipeline
from .stress_pipeline import StressPipeline

24
robogauge/tasks/pipeline/level_pipeline.py
View File

@@ -10,17 +10,19 @@
import yaml
from robogauge.tasks.pipeline.multi_pipeline import MultiPipeline
from robogauge.utils.logger import logger
from robogauge.utils.logger import Logger
level_logger = Logger() # LevelPipeline logger
class LevelPipeline:
def __init__(self, args):
self.args = args
self.seeds = args.seeds
logger.create(args.experiment_name+'_level', args.run_name)
level_logger.create(args.experiment_name+'_level', args.run_name)
def run(self):
logger.info(f"🚀 Starting Level Seacher for '{self.args.experiment_name}'.")
logger.info(f"🔢 Seeds: {self.seeds}")
level_logger.info(f"🚀 Starting Level Searcher for '{self.args.experiment_name}'.")
level_logger.info(f"🔢 Seeds: {self.seeds}")
# binary search levels
l, r = 0, 10
@@ -40,22 +42,22 @@ class LevelPipeline:
'terrain_level': 0,
})
if level >= 1:
logger.info(f"🏆 Found maximum level: {level}")
level_logger.info(f"🏆 Found maximum level: {level}")
else:
logger.info(f"❌ No valid level found [1-10].")
with open(logger.log_dir / "level_search_results.yaml", 'w') as f:
level_logger.info(f"❌ No valid level found [1-10].")
with open(level_logger.log_dir / "level_search_results.yaml", 'w') as f:
yaml.dump(level_results, f, allow_unicode=True, sort_keys=False)
return level, level_results
def test_level(self, level: int) -> bool:
logger.info(f"🔍 Testing level {level}...")
def test_level(self, level: int):
level_logger.info(f"🔍 Testing level {level}...")
self.args.level = level
multi_pipeline = MultiPipeline(self.args)
aggregated_results = multi_pipeline.run()
success_mean = float(aggregated_results['success']['mean'].split(' ')[0])
all_success = success_mean >= 0.8
if all_success:
logger.info(f"✅ Level {level} passed all tests.")
level_logger.info(f"✅ Level {level} passed all tests.")
else:
logger.info(f"❌ Level {level} failed some tests.")
level_logger.info(f"❌ Level {level} failed some tests.")
return all_success, aggregated_results

47
robogauge/tasks/pipeline/multi_pipeline.py
View File

@@ -21,9 +21,12 @@ from collections import defaultdict
from robogauge.tasks.pipeline.base_pipeline import BasePipeline
from robogauge.utils.task_register import task_register
from robogauge.utils.logger import logger
from robogauge.utils.logger import Logger
multi_logger = Logger() # MultiPipeline logger
def run_single_process(args, data):
from robogauge.utils.logger import logger
seed, base_mass, friction = data
local_args = deepcopy(args)
local_args.seed = seed
@@ -41,6 +44,7 @@ def run_single_process(args, data):
ret = {
'status': 'success',
'results': results,
'data': data,
'model_path': pipeline.robot_cfg.control.model_path,
}
if warning is not None:
@@ -65,7 +69,7 @@ class MultiPipeline:
self.base_masses = args.base_masses
self.num_processes = args.num_processes
self.static_info = {}
logger.create(args.experiment_name+'_multi', args.run_name+'_multi')
multi_logger.create(args.experiment_name+'_multi', args.run_name+'_multi')
def add_static_info(self, key: str, value):
if key not in self.static_info:
@@ -74,55 +78,54 @@ class MultiPipeline:
assert self.static_info[key] == value, f"Static info key '{key}' has conflicting values: {self.static_info[key]} vs {value}"
def run(self):
logger.info(f"🚀 Starting Multi-Process Evaluation with {self.num_processes} processes.")
logger.info(f"🔢 Seeds: {self.seeds}, Frictions: {self.frictions}, Base masses: {self.base_masses}")
multi_logger.info(f"🚀 Starting Multi-Process Evaluation with {self.num_processes} processes.")
multi_logger.info(f"🔢 Seeds: {self.seeds}, Frictions: {self.frictions}, Base masses: {self.base_masses}")
workers_data = list(product(self.seeds, self.base_masses, self.frictions))
ctx = multiprocessing.get_context('spawn')
worker_func = functools.partial(run_single_process, self.args)
results_list = []
success_flags = []
with ctx.Pool(processes=self.num_processes) as pool:
iterator = pool.imap_unordered(worker_func, workers_data)
for results in tqdm(iterator, total=len(workers_data), desc="Evaluation"):
success_flags.append(results['status'] == 'success')
results_list.append(results['results'])
results_list.append(results)
self.add_static_info('model_path', results['model_path'])
self.add_static_info('terrain_name', results['results']['terrain_name'])
self.add_static_info('terrain_level', results['results']['terrain_level'])
if results['status'] != 'success':
data = results['data']
logger.error(f"❌ Process with seed={data[0]}, base_mass={data[1]}, friction={data[2]} failed with error: {results['error_msg']}")
multi_logger.error(f"❌ Process with seed={data[0]}, base_mass={data[1]}, friction={data[2]} failed with error: {results['error_msg']}")
logger.info("✅ Multi-Process Evaluation Completed.")
aggregated_results = self.aggregate_results(results_list, success_flags, workers_data)
multi_logger.info("✅ Multi-Process Evaluation Completed.")
aggregated_results = self.aggregate_results(results_list)
return aggregated_results
def aggregate_results(self, all_results, success_flags, workers_data):
def aggregate_results(self, all_results):
""" Process results from all processes and aggregate them. """
logger.info("📊 Aggregating Results from all runs...")
multi_logger.info("📊 Aggregating Results from all runs...")
summary = {'success': {}, **self.static_info}
finish_msg = (
f"""\n{'='*20} Run Finish Summary {'='*20}\n"""
f"""{'Seed':^10}{'Base Mass':^15}{'Friction':^15}{'Status':^10}\n"""
)
for success, data in zip(success_flags, workers_data):
seed, base_mass, friction = data
all_results = sorted(all_results, key=lambda x: x['data'])
for result in all_results:
seed, base_mass, friction = result['data']
success = result['status'] == 'success'
status_str = "✅" if success else "❌"
finish_msg += f"{seed:^10}{base_mass:^15}{friction:^15}{status_str:^10}\n"
summary['success'][f"Seed_{seed}_BaseMass_{base_mass}_Friction_{friction}"] = True if success else False
finish_msg += f"""{'='*88}"""
logger.info(finish_msg)
multi_logger.info(finish_msg)
if not all_results:
logger.error("No results to aggregate.")
multi_logger.error("No results to aggregate.")
return
value_collections = defaultdict(lambda: defaultdict(list))
for result in all_results:
for goal, metrics in result.items():
for goal, metrics in result['results'].items():
if goal != 'summary':
continue
for metric, means in metrics.items():
@@ -134,13 +137,13 @@ class MultiPipeline:
for mean_name, values in means.items():
summary[metric][mean_name] = f"{float(np.mean(values)):.4f} ± {float(np.std(values)):.4f}"
save_path = logger.log_dir / "aggregated_results.yaml"
save_path = multi_logger.log_dir / "aggregated_results.yaml"
with open(save_path, 'w') as file:
yaml.dump(summary, file, allow_unicode=True, sort_keys=False)
logger.info("✅ Aggregated execution finished.")
logger.info(f"📁 Aggregated results saved to: {save_path}")
multi_logger.info("✅ Aggregated execution finished.")
multi_logger.info(f"📁 Aggregated results saved to: {save_path}")
logger.info(
multi_logger.info(
f"""\n{'='*20} Multi-Run Summary {'='*20}\n"""
f"""{yaml.dump(summary, allow_unicode=True)}"""
f"""{'='*60}"""

131
robogauge/tasks/pipeline/stress_pipeline.py Normal file
View File

@@ -0,0 +1,131 @@
# -*- coding: utf-8 -*-
'''
@File : stress_pipeline.py
@Time : 2025/12/25 21:20:43
@Author : wty-yy
@Version : 1.0
@Blog : https://wty-yy.github.io/
@Desc : Stress Pipeline for Robogauge
'''
import yaml
import functools
from tqdm import tqdm
import multiprocessing
from copy import deepcopy
from itertools import product
from robogauge.utils.logger import Logger
from robogauge.tasks.pipeline import MultiPipeline, LevelPipeline
from robogauge.tasks.gauge.gauge_configs.terrain_levels_config import TerrainSearchLevelsConfig
stress_logger = Logger() # StressPipeline logger
def run_pipeline(args, data):
args = deepcopy(args)
search = data['search_max_level']
if search is True:
args.friction = data['friction']
args.frictions = [data['friction']]
args.base_mass = data['base_mass']
args.base_masses = [data['base_mass']]
args.task_name = f"{data['task_robot_model']}.{data['terrain_name']}"
args.experiment_name = f"{args.experiment_name}_{data['terrain_name']}_baseMass{data['base_mass']}_friction{data['friction']}"
else:
args.task_name = f"{data['task_robot_model']}.{data['terrain_name']}"
args.experiment_name = f"{args.experiment_name}_{data['terrain_name']}"
level = None # flat terrain
if search:
level, results = LevelPipeline(args).run()
if level == 0: # no valid level found
results = {
'success': False,
'results': results,
'data': data,
}
else:
args.level = level
results = {
'success': True,
'results': MultiPipeline(args).run(),
'data': data,
}
return results
class StressPipeline:
def __init__(self, args):
self.args = args
self.seeds = args.seeds
self.task_robot_model = args.task_name.split('.')[0]
self.num_processes = args.stress_num_processes
args.experiment_name = self.task_robot_model + '_stress' + ('' if args.cli_experiment_name is None else '_' + args.cli_experiment_name)
self.static_info = {}
stress_logger.create(args.experiment_name, args.run_name)
def add_static_info(self, key: str, value):
if key not in self.static_info:
self.static_info[key] = value
else:
assert self.static_info[key] == value, f"Static info key '{key}' has conflicting values: {self.static_info[key]} vs {value}"
def run(self):
stress_logger.info(f"🚀 Starting Stress Benchmark for '{self.args.experiment_name}'.")
stress_logger.info(f"🔢 Seeds: {self.seeds}")
terrain_names = self.args.stress_terrain_names
stress_logger.info(f"🌄 Stress Test Terrain Names: {terrain_names}")
ctx = multiprocessing.get_context('spawn')
worker_func = functools.partial(run_pipeline, self.args)
### Build worker data ###
workers_data = []
terrain_search_levels_config = TerrainSearchLevelsConfig()
for terrain_name in terrain_names:
search_max_level = True
terrain_level_cfg = getattr(terrain_search_levels_config, terrain_name, None)
assert terrain_level_cfg is not None, f"Terrain '{terrain_name}' not found in TerrainSearchLevelsConfig."
if len(terrain_level_cfg.levels) == 1: # Flattened terrain
search_max_level = False
data = {
'task_robot_model': self.task_robot_model,
'terrain_name': terrain_name,
'search_max_level': search_max_level,
}
if search_max_level:
for friction, base_mass in product(self.args.frictions, self.args.base_masses):
data.update({
'friction': friction,
'base_mass': base_mass,
})
workers_data.append(data)
### Run and collect results ###
results_list = []
with ctx.Pool(processes=self.num_processes) as pool:
iterator = pool.imap_unordered(worker_func, workers_data)
for results in tqdm(iterator, total=len(workers_data), desc="Stress Benchmark"):
results_list.append(results)
self.add_static_info('model_path', results['results']['model_path'])
stress_logger.info("✅ Stress Benchmark Completed.")
stress_results = self.aggregate_results(results_list)
return stress_results
def aggregate_results(self, all_results):
stress_logger.info("📊 Aggregating Stress Benchmark Results...")
summary = {}
finish_msg = (
f"""\n{'='*20} Stress Benchmark Summary {'='*20}\n"""
f"""{'Terrain Name':^20}{'Base Mass':^15}{'Friction':^15}{'Status':^10}\n"""
)
all_results = sorted(all_results, key=lambda x: (x['data']['terrain_name'], x['data'].get('base_mass', 0), x['data'].get('friction', 0)))
for result in all_results:
terrain_name = result['data']['terrain_name']
base_mass = result['data'].get('base_mass', self.args.base_masses)
friction = result['data'].get('friction', self.args.frictions)
status = "✅" if result['success'] else "❌"
finish_msg += f"{terrain_name:^20}{str(base_mass):^15}{str(friction):^15}{status:^10}\n"
finish_msg += f"""{'='*66}"""
stress_logger.info(finish_msg)