cyy/RoboGauge
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

v1.0.1

Browse Source
This commit is contained in:
wty-yy
2025-12-30 15:59:07 +08:00
parent 5a5500668b
commit de7d740012
7 changed files with 46 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
UPDATE.md
View File

@@ -1,4 +1,7 @@
# UPDATE
## 20251230
### v1.0.1
1. quality_score到每个step时计算, 结果更加准确, 可以有效避免站立也能获得高score得分的问题 (因为平均后的score可能相对比较高, 而每一步的score都非常低)
## 20251229
### v1.0.0
1. 完成stress pipeline的客户端, 服务端代码, 支持异步推理

12
robogauge/tasks/gauge/base_gauge.py
View File

@@ -21,6 +21,7 @@ from robogauge.tasks.robots import RobotConfig
from robogauge.tasks.gauge.base_gauge_config import BaseGaugeConfig
from robogauge.tasks.gauge.goal_data import GoalData, VelocityGoal, PositionGoal
from robogauge.tasks.simulator.sim_data import SimData
from robogauge.tasks.gauge.gauge_configs.terrain_levels_config import SEARCH_LEVELS_TERRAINS
from robogauge.tasks.gauge.goals import *
from robogauge.tasks.gauge.metrics import *
@@ -149,7 +150,7 @@ class BaseGauge:
for metric_name, quantiles in self.results[goal].items():
for quantile, val in quantiles.items():
metrics[metric_name][quantile].append(val)
self.results['summary'] = {}
self.results['summary'] = {'quality_score': {}, 'terrain_quality_score': {}}
for metric_name, quantiles in metrics.items():
if metric_name not in self.results['summary']:
self.results['summary'][metric_name] = {}
@@ -157,14 +158,19 @@ class BaseGauge:
mean = float(np.mean(vals))
std = float(np.std(vals))
self.results['summary'][metric_name][quantile] = f"{mean:.4f} ± {std:.4f}"
if metric_name == 'quality_score':
tqs = mean
if self.cfg.assets.terrain_name in SEARCH_LEVELS_TERRAINS:
tqs = 0.09 * (self.cfg.assets.terrain_level - 1) + 0.19 * mean
self.results['summary']['terrain_quality_score'][quantile] = f"{tqs:.4f} ± {std:.4f}"
save_path = Path(logger.log_dir) / "results.yaml"
self.results["terrain_name"] = self.cfg.assets.terrain_name
self.results["terrain_level"] = self.cfg.assets.terrain_level
with open(save_path, 'w') as file:
yaml.dump(self.results, file, allow_unicode=True, sort_keys=False)
with open(save_path, 'w', encoding='utf-8') as file:
yaml_str = yaml.dump(self.results, allow_unicode=True, sort_keys=False)
file.write(yaml_str)
logger.info(
f"""\n{'='*20} Goals and Metrics results {'='*20}\n"""
f"""{yaml_str}"""

9
robogauge/tasks/gauge/base_gauge_config.py
View File

@@ -9,6 +9,15 @@
'''
from robogauge.utils.config import Config
QUALITY_WEIGHTS = { # Weights for geometric average, to calculate quality score
'lin_vel_err': 2,
'ang_vel_err': 2,
'dof_limits': 1,
'dof_power': 1,
'orientation_stability': 1,
'torque_smoothness': 1,
}
class BaseGaugeConfig(Config):
gauge_class = 'BaseGauge'
write_tensorboard = False # Whether to write tensorboard logs

14
robogauge/tasks/gauge/goals/base_goal.py
View File

@@ -12,6 +12,7 @@ from collections import defaultdict
from robogauge.utils.measure import Average
from robogauge.tasks.gauge.goal_data import GoalData
from robogauge.tasks.simulator.sim_data import SimData
from robogauge.tasks.gauge.base_gauge_config import QUALITY_WEIGHTS
class BaseGoal:
name = 'base_goal'
@@ -21,7 +22,8 @@ class BaseGoal:
self.total = 0 # total tasks
self.sub_name = None
self._goal_mean_metrics = defaultdict(list)
self.goal_metrics = defaultdict(list)
self.goal_quality_scores = []
def pre_get_goal(self) -> bool:
raise NotImplementedError
@@ -42,13 +44,19 @@ class BaseGoal:
def update_metrics(self, metrics: dict):
""" Update step metrics for the current goal."""
quality_score = 1.0
for metric_name, value in metrics.items():
self._goal_mean_metrics[metric_name].append(value)
self.goal_metrics[metric_name].append(value)
quality_score *= min(max(1e-9, value), 1.0) ** QUALITY_WEIGHTS[metric_name]
quality_score = quality_score ** (1.0 / sum(QUALITY_WEIGHTS.values()))
self.goal_quality_scores.append(quality_score)
@property
def goal_mean_metrics(self):
""" Get the mean metrics for the current goal. """
return {k: self._analysis_metrics(v) for k, v in self._goal_mean_metrics.items()}
result = {k: self._analysis_metrics(v) for k, v in self.goal_metrics.items()}
result['quality_score'] = self._analysis_metrics(self.goal_quality_scores)
return result
@staticmethod
def _analysis_metrics(metrics: list):

20
robogauge/tasks/pipeline/multi_pipeline.py
View File

@@ -133,7 +133,7 @@ class MultiPipeline:
""" Process results from all processes and aggregate them. """
multi_logger.info("📊 Aggregating Results from all runs...")
summary = {'success': {}, **self.static_info, 'summary': {}, 'terrain_weighted_summary': {}, 'quality_score': {}, 'terrain_quality_score': {}}
summary = {'success': {}, **self.static_info, 'summary': {}, 'terrain_weighted_summary': {}}
finish_msg = (
f"""\n{'='*20} Run Finish Summary {'='*20}\n"""
f"""{'Seed':^10}{'Base Mass':^15}{'Friction':^15}{'Status':^10}\n"""
@@ -152,7 +152,6 @@ class MultiPipeline:
multi_logger.error("No results to aggregate.")
return
quality_score, terrain_quality_score = summary['quality_score'], summary['terrain_quality_score']
value_collections = defaultdict(lambda: defaultdict(list))
for result in all_results:
for goal, metrics in result['results'].items():
@@ -161,27 +160,20 @@ class MultiPipeline:
for metric, means in metrics.items():
for mean_name, mean_value in means.items():
value_collections[metric][mean_name].append(float(mean_value.split(' ')[0]))
quality_score[mean_name] = 1
for metric, means in value_collections.items():
summary['summary'][metric] = {}
summary['terrain_weighted_summary'][metric] = {}
for mean_name, values in means.items():
v = float(np.mean(values))
summary['summary'][metric][mean_name] = f"{v:.4f} ± {float(np.std(values)):.4f}"
twv = v
if 'quality_score' in metric: continue
summary['terrain_weighted_summary'][metric] = {}
for mean_name, values in means.items():
twv = float(np.mean(values))
if summary['terrain_name'] in SEARCH_LEVELS_TERRAINS:
twv = 0.09 * (summary['terrain_level'] - 1) + 0.19 * v
summary['terrain_weighted_summary'][metric][mean_name] = f"{twv:.4f} ± {float(np.std(values)):.4f}"
weight = 1
if metric in ['ang_vel_err', 'lin_vel_err']:
weight = 2
quality_score[mean_name] *= min(max(1e-9, v), 1.0) ** weight
for mean_name in quality_score:
quality_score[mean_name] = quality_score[mean_name] ** (1 / 8) # 2 + 2 + 1 * 4
terrain_quality_score[mean_name] = quality_score[mean_name]
if summary['terrain_name'] in SEARCH_LEVELS_TERRAINS:
terrain_quality_score[mean_name] = 0.09 * (summary['terrain_level'] - 1) + 0.19 * quality_score[mean_name]
save_path = multi_logger.log_dir / "aggregated_results.yaml"
with open(save_path, 'w') as file:

12
robogauge/tasks/pipeline/stress_pipeline.py
View File

@@ -182,10 +182,12 @@ class StressPipeline:
metric_collections = defaultdict(lambda: defaultdict(list))
terrain_collections = defaultdict(lambda: defaultdict(list))
zero_terrain_count = defaultdict(lambda: 0)
robust_score = summary['robust_score']
for result in all_results:
terrain_name = result['data']['terrain_name']
terrain_level = result['level'] # None, 0, 1, ..., 10
scores[terrain_name] = 0.0
robust_score[terrain_name] = {}
key = f'{terrain_name}_{terrain_level}'
key += f'_baseMass{result["data"]["base_mass"]}_friction{result["data"]["friction"]}'
if terrain_level == 0:
@@ -198,7 +200,8 @@ class StressPipeline:
for mean_name, value_str in means.items():
value = float(value_str.split(' ± ')[0])
metric_collections[metric][mean_name].append(value)
for mean_name, value in result['results']['terrain_quality_score'].items():
for mean_name, value_str in result['results']['summary']['terrain_quality_score'].items():
value = float(value_str.split(' ± ')[0])
terrain_collections[terrain_name][mean_name].append(value)
for metric, means in metric_collections.items():
@@ -207,15 +210,14 @@ class StressPipeline:
values.extend([0.0] * sum(zero_terrain_count.values())) # include zero terrains
summary['summary'][metric][mean_name] = f"{float(np.mean(values)):.4f} ± {float(np.std(values)):.4f}"
robust_score = defaultdict(dict)
robust_scores = []
for terrain_name, means in terrain_collections.items():
for mean_name, values in means.items():
values.extend([0.0] * zero_terrain_count[terrain_name]) # include zero terrains
robust_score[terrain_name][mean_name] = float(np.mean(values))
scores[terrain_name] = robust_score[terrain_name]['mean@50']
robust_scores.append(robust_score[terrain_name]['mean@50'])
summary['robust_score'] = dict(robust_score)
for terrain_name in robust_score:
if len(robust_score[terrain_name]) == 0:
robust_score[terrain_name] = None
summary['benchmark_score'] = float(np.mean(list(scores.values())))
scores['benchmark'] = summary['benchmark_score']

2
setup.py
View File

@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
setup(
name="robogauge", # 包名
version="1.0.0", # 版本号
version="1.0.1", # 版本号
author="Wu Tianyang", # 你的名字
author_email="993660140@qq.com",
description="A generic robot RL model evaluation library based on MuJoCo",