diff --git a/UPDATE.md b/UPDATE.md index 8a4a604..5d18d63 100644 --- a/UPDATE.md +++ b/UPDATE.md @@ -1,4 +1,7 @@ # UPDATE +## 20251230 +### v1.0.1 +1. quality_score到每个step时计算, 结果更加准确, 可以有效避免站立也能获得高score得分的问题 (因为平均后的score可能相对比较高, 而每一步的score都非常低) ## 20251229 ### v1.0.0 1. 完成stress pipeline的客户端, 服务端代码, 支持异步推理 diff --git a/robogauge/tasks/gauge/base_gauge.py b/robogauge/tasks/gauge/base_gauge.py index 0128ef4..7ab4067 100644 --- a/robogauge/tasks/gauge/base_gauge.py +++ b/robogauge/tasks/gauge/base_gauge.py @@ -21,6 +21,7 @@ from robogauge.tasks.robots import RobotConfig from robogauge.tasks.gauge.base_gauge_config import BaseGaugeConfig from robogauge.tasks.gauge.goal_data import GoalData, VelocityGoal, PositionGoal from robogauge.tasks.simulator.sim_data import SimData +from robogauge.tasks.gauge.gauge_configs.terrain_levels_config import SEARCH_LEVELS_TERRAINS from robogauge.tasks.gauge.goals import * from robogauge.tasks.gauge.metrics import * @@ -149,7 +150,7 @@ class BaseGauge: for metric_name, quantiles in self.results[goal].items(): for quantile, val in quantiles.items(): metrics[metric_name][quantile].append(val) - self.results['summary'] = {} + self.results['summary'] = {'quality_score': {}, 'terrain_quality_score': {}} for metric_name, quantiles in metrics.items(): if metric_name not in self.results['summary']: self.results['summary'][metric_name] = {} @@ -157,14 +158,19 @@ class BaseGauge: mean = float(np.mean(vals)) std = float(np.std(vals)) self.results['summary'][metric_name][quantile] = f"{mean:.4f} ± {std:.4f}" + if metric_name == 'quality_score': + tqs = mean + if self.cfg.assets.terrain_name in SEARCH_LEVELS_TERRAINS: + tqs = 0.09 * (self.cfg.assets.terrain_level - 1) + 0.19 * mean + self.results['summary']['terrain_quality_score'][quantile] = f"{tqs:.4f} ± {std:.4f}" save_path = Path(logger.log_dir) / "results.yaml" self.results["terrain_name"] = self.cfg.assets.terrain_name self.results["terrain_level"] = self.cfg.assets.terrain_level - with open(save_path, 'w') as file: - yaml.dump(self.results, file, allow_unicode=True, sort_keys=False) + with open(save_path, 'w', encoding='utf-8') as file: yaml_str = yaml.dump(self.results, allow_unicode=True, sort_keys=False) + file.write(yaml_str) logger.info( f"""\n{'='*20} Goals and Metrics results {'='*20}\n""" f"""{yaml_str}""" diff --git a/robogauge/tasks/gauge/base_gauge_config.py b/robogauge/tasks/gauge/base_gauge_config.py index 32612a9..1af60d1 100644 --- a/robogauge/tasks/gauge/base_gauge_config.py +++ b/robogauge/tasks/gauge/base_gauge_config.py @@ -9,6 +9,15 @@ ''' from robogauge.utils.config import Config +QUALITY_WEIGHTS = { # Weights for geometric average, to calculate quality score + 'lin_vel_err': 2, + 'ang_vel_err': 2, + 'dof_limits': 1, + 'dof_power': 1, + 'orientation_stability': 1, + 'torque_smoothness': 1, +} + class BaseGaugeConfig(Config): gauge_class = 'BaseGauge' write_tensorboard = False # Whether to write tensorboard logs diff --git a/robogauge/tasks/gauge/goals/base_goal.py b/robogauge/tasks/gauge/goals/base_goal.py index 25040c5..4dda693 100644 --- a/robogauge/tasks/gauge/goals/base_goal.py +++ b/robogauge/tasks/gauge/goals/base_goal.py @@ -12,6 +12,7 @@ from collections import defaultdict from robogauge.utils.measure import Average from robogauge.tasks.gauge.goal_data import GoalData from robogauge.tasks.simulator.sim_data import SimData +from robogauge.tasks.gauge.base_gauge_config import QUALITY_WEIGHTS class BaseGoal: name = 'base_goal' @@ -21,7 +22,8 @@ class BaseGoal: self.total = 0 # total tasks self.sub_name = None - self._goal_mean_metrics = defaultdict(list) + self.goal_metrics = defaultdict(list) + self.goal_quality_scores = [] def pre_get_goal(self) -> bool: raise NotImplementedError @@ -42,13 +44,19 @@ class BaseGoal: def update_metrics(self, metrics: dict): """ Update step metrics for the current goal.""" + quality_score = 1.0 for metric_name, value in metrics.items(): - self._goal_mean_metrics[metric_name].append(value) + self.goal_metrics[metric_name].append(value) + quality_score *= min(max(1e-9, value), 1.0) ** QUALITY_WEIGHTS[metric_name] + quality_score = quality_score ** (1.0 / sum(QUALITY_WEIGHTS.values())) + self.goal_quality_scores.append(quality_score) @property def goal_mean_metrics(self): """ Get the mean metrics for the current goal. """ - return {k: self._analysis_metrics(v) for k, v in self._goal_mean_metrics.items()} + result = {k: self._analysis_metrics(v) for k, v in self.goal_metrics.items()} + result['quality_score'] = self._analysis_metrics(self.goal_quality_scores) + return result @staticmethod def _analysis_metrics(metrics: list): diff --git a/robogauge/tasks/pipeline/multi_pipeline.py b/robogauge/tasks/pipeline/multi_pipeline.py index b32ab0c..6114549 100644 --- a/robogauge/tasks/pipeline/multi_pipeline.py +++ b/robogauge/tasks/pipeline/multi_pipeline.py @@ -133,7 +133,7 @@ class MultiPipeline: """ Process results from all processes and aggregate them. """ multi_logger.info("📊 Aggregating Results from all runs...") - summary = {'success': {}, **self.static_info, 'summary': {}, 'terrain_weighted_summary': {}, 'quality_score': {}, 'terrain_quality_score': {}} + summary = {'success': {}, **self.static_info, 'summary': {}, 'terrain_weighted_summary': {}} finish_msg = ( f"""\n{'='*20} Run Finish Summary {'='*20}\n""" f"""{'Seed':^10}{'Base Mass':^15}{'Friction':^15}{'Status':^10}\n""" @@ -152,7 +152,6 @@ class MultiPipeline: multi_logger.error("No results to aggregate.") return - quality_score, terrain_quality_score = summary['quality_score'], summary['terrain_quality_score'] value_collections = defaultdict(lambda: defaultdict(list)) for result in all_results: for goal, metrics in result['results'].items(): @@ -161,27 +160,20 @@ class MultiPipeline: for metric, means in metrics.items(): for mean_name, mean_value in means.items(): value_collections[metric][mean_name].append(float(mean_value.split(' ')[0])) - quality_score[mean_name] = 1 for metric, means in value_collections.items(): summary['summary'][metric] = {} - summary['terrain_weighted_summary'][metric] = {} for mean_name, values in means.items(): v = float(np.mean(values)) summary['summary'][metric][mean_name] = f"{v:.4f} ± {float(np.std(values)):.4f}" - twv = v + + if 'quality_score' in metric: continue + summary['terrain_weighted_summary'][metric] = {} + for mean_name, values in means.items(): + twv = float(np.mean(values)) if summary['terrain_name'] in SEARCH_LEVELS_TERRAINS: twv = 0.09 * (summary['terrain_level'] - 1) + 0.19 * v summary['terrain_weighted_summary'][metric][mean_name] = f"{twv:.4f} ± {float(np.std(values)):.4f}" - weight = 1 - if metric in ['ang_vel_err', 'lin_vel_err']: - weight = 2 - quality_score[mean_name] *= min(max(1e-9, v), 1.0) ** weight - for mean_name in quality_score: - quality_score[mean_name] = quality_score[mean_name] ** (1 / 8) # 2 + 2 + 1 * 4 - terrain_quality_score[mean_name] = quality_score[mean_name] - if summary['terrain_name'] in SEARCH_LEVELS_TERRAINS: - terrain_quality_score[mean_name] = 0.09 * (summary['terrain_level'] - 1) + 0.19 * quality_score[mean_name] save_path = multi_logger.log_dir / "aggregated_results.yaml" with open(save_path, 'w') as file: diff --git a/robogauge/tasks/pipeline/stress_pipeline.py b/robogauge/tasks/pipeline/stress_pipeline.py index a25c7c0..87cb802 100644 --- a/robogauge/tasks/pipeline/stress_pipeline.py +++ b/robogauge/tasks/pipeline/stress_pipeline.py @@ -182,10 +182,12 @@ class StressPipeline: metric_collections = defaultdict(lambda: defaultdict(list)) terrain_collections = defaultdict(lambda: defaultdict(list)) zero_terrain_count = defaultdict(lambda: 0) + robust_score = summary['robust_score'] for result in all_results: terrain_name = result['data']['terrain_name'] terrain_level = result['level'] # None, 0, 1, ..., 10 scores[terrain_name] = 0.0 + robust_score[terrain_name] = {} key = f'{terrain_name}_{terrain_level}' key += f'_baseMass{result["data"]["base_mass"]}_friction{result["data"]["friction"]}' if terrain_level == 0: @@ -198,7 +200,8 @@ class StressPipeline: for mean_name, value_str in means.items(): value = float(value_str.split(' ± ')[0]) metric_collections[metric][mean_name].append(value) - for mean_name, value in result['results']['terrain_quality_score'].items(): + for mean_name, value_str in result['results']['summary']['terrain_quality_score'].items(): + value = float(value_str.split(' ± ')[0]) terrain_collections[terrain_name][mean_name].append(value) for metric, means in metric_collections.items(): @@ -207,15 +210,14 @@ class StressPipeline: values.extend([0.0] * sum(zero_terrain_count.values())) # include zero terrains summary['summary'][metric][mean_name] = f"{float(np.mean(values)):.4f} ± {float(np.std(values)):.4f}" - robust_score = defaultdict(dict) - robust_scores = [] for terrain_name, means in terrain_collections.items(): for mean_name, values in means.items(): values.extend([0.0] * zero_terrain_count[terrain_name]) # include zero terrains robust_score[terrain_name][mean_name] = float(np.mean(values)) scores[terrain_name] = robust_score[terrain_name]['mean@50'] - robust_scores.append(robust_score[terrain_name]['mean@50']) - summary['robust_score'] = dict(robust_score) + for terrain_name in robust_score: + if len(robust_score[terrain_name]) == 0: + robust_score[terrain_name] = None summary['benchmark_score'] = float(np.mean(list(scores.values()))) scores['benchmark'] = summary['benchmark_score'] diff --git a/setup.py b/setup.py index 96b1305..6759725 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages setup( name="robogauge", # 包名 - version="1.0.0", # 版本号 + version="1.0.1", # 版本号 author="Wu Tianyang", # 你的名字 author_email="993660140@qq.com", description="A generic robot RL model evaluation library based on MuJoCo",