diff --git a/UPDATE.md b/UPDATE.md index 253df25..48b3ad5 100644 --- a/UPDATE.md +++ b/UPDATE.md @@ -6,6 +6,8 @@ 3. 对有难度地形设置goals, 但是忘记设置到地形的正中心了!!!在每种地形上调出对应的中心初始位置 4. 添加`--spawn-type`配置, 根据评估和搜索修改初始生成点, 完成slope, wave, stairs up, stairs down, obstacle 5. 重新详细设计整个打分标准, 引入几个阶段性得分: quality_score, terrain_quality_score, robust_score, benchmark_score +6. 对于楼梯环境, 计算benchmark_score和统计每个metrics平均分时, 不计算下楼梯, 因为下楼梯几乎对模型没有仍和区分度 (都是最高难度) +Fix bug: 修复零难度的地形个数计算问题 ## 20251227 ### v0.1.17 1. args中添加`goals`配置, 在LevelPipeline和MultiPipeline中指定goals, 修正压力测试中评估的目标不对的问题 diff --git a/robogauge/tasks/pipeline/stress_pipeline.py b/robogauge/tasks/pipeline/stress_pipeline.py index aac3761..58c0e4f 100644 --- a/robogauge/tasks/pipeline/stress_pipeline.py +++ b/robogauge/tasks/pipeline/stress_pipeline.py @@ -181,7 +181,7 @@ class StressPipeline: summary = {**self.static_info, 'summary': {}, 'robust_score': {}, 'benchmark_score': 0.0} metric_collections = defaultdict(lambda: defaultdict(list)) terrain_collections = defaultdict(lambda: defaultdict(list)) - zero_terrain_count = 0 + zero_terrain_count = defaultdict(lambda: 0) for result in all_results: terrain_name = result['data']['terrain_name'] terrain_level = result['level'] # None, 0, 1, ..., 10 @@ -189,30 +189,32 @@ class StressPipeline: key += f'_baseMass{result["data"]["base_mass"]}_friction{result["data"]["friction"]}' if terrain_level == 0: summary[key] = None - zero_terrain_count += 1 + zero_terrain_count[terrain_name] += 1 continue summary[key] = result['results'] - for metric, means in result['results']['terrain_weighted_summary'].items(): - for mean_name, value_str in means.items(): - value = float(value_str.split(' ± ')[0]) - metric_collections[metric][mean_name].append(value) + if terrain_name != 'stairs_down': # skip stairs_down for metrics means calculation + for metric, means in result['results']['terrain_weighted_summary'].items(): + for mean_name, value_str in means.items(): + value = float(value_str.split(' ± ')[0]) + metric_collections[metric][mean_name].append(value) for mean_name, value in result['results']['terrain_quality_score'].items(): terrain_collections[terrain_name][mean_name].append(value) for metric, means in metric_collections.items(): summary['summary'][metric] = {} for mean_name, values in means.items(): - values.extend([0.0] * zero_terrain_count) # include zero terrains + values.extend([0.0] * sum(zero_terrain_count.values())) # include zero terrains summary['summary'][metric][mean_name] = f"{float(np.mean(values)):.4f} ± {float(np.std(values)):.4f}" robust_score = defaultdict(dict) robust_scores = [] for terrain_name, means in terrain_collections.items(): for mean_name, values in means.items(): - values.extend([0.0] * zero_terrain_count) # include zero terrains + values.extend([0.0] * zero_terrain_count[terrain_name]) # include zero terrains robust_score[terrain_name][mean_name] = float(np.mean(values)) - robust_scores.append(robust_score[terrain_name][mean_name]) + if terrain_name != 'stairs_down': # skip stairs_down for benchmark score calculation + robust_scores.append(robust_score[terrain_name]['mean@50']) summary['robust_score'] = dict(robust_score) summary['benchmark_score'] = float(np.mean(robust_scores)) diff --git a/run.bash b/run.bash index 9d047e0..17dfc41 100755 --- a/run.bash +++ b/run.bash @@ -9,7 +9,7 @@ python robogauge/scripts/run.py \ --stress-terrain-names flat slope stairs_up stairs_down wave obstacle \ --num-processes 62 \ --seeds 0 1 2 3 4 \ - --frictions 0.75 1.0 1.25 1.5 1.75 2.0 2.25 2.5 \ + --frictions 0.5 0.75 1.0 1.25 1.5 1.75 2.0 2.25 2.5 \ --compress-logs \ --headless @@ -21,7 +21,7 @@ python robogauge/scripts/run.py \ --stress-terrain-names flat slope stairs_up stairs_down wave obstacle \ --num-processes 62 \ --seeds 0 1 2 3 4 \ - --frictions 0.75 1.0 1.25 1.5 1.75 2.0 2.25 2.5 \ + --frictions 0.5 0.75 1.0 1.25 1.5 1.75 2.0 2.25 2.5 \ --compress-logs \ --headless @@ -33,7 +33,7 @@ python robogauge/scripts/run.py \ --stress-terrain-names flat slope stairs_up stairs_down wave obstacle \ --num-processes 62 \ --seeds 0 1 2 3 4 \ - --frictions 0.75 1.0 1.25 1.5 1.75 2.0 2.25 2.5 \ + --frictions 0.5 0.75 1.0 1.25 1.5 1.75 2.0 2.25 2.5 \ --compress-logs \ --headless @@ -44,6 +44,30 @@ python robogauge/scripts/run.py \ --stress-terrain-names flat slope stairs_up stairs_down wave obstacle \ --num-processes 62 \ --seeds 0 1 2 3 4 \ - --frictions 0.75 1.0 1.25 1.5 1.75 2.0 2.25 2.5 \ + --frictions 0.5 0.75 1.0 1.25 1.5 1.75 2.0 2.25 2.5 \ + --compress-logs \ + --headless + +python robogauge/scripts/run.py \ + --task go2_moe \ + --model-path /root/Coding/RoboGauge/mytest/go2_moe_cts_hard_terrain_100k.pt \ + --experiment-name debug \ + --stress-benchmark \ + --stress-terrain-names flat slope stairs_up stairs_down wave obstacle \ + --num-processes 62 \ + --seeds 0 1 2 3 4 \ + --frictions 0.5 0.75 1.0 1.25 1.5 1.75 2.0 2.25 2.5 \ + --compress-logs \ + --headless + +python robogauge/scripts/run.py \ + --task go2 \ + --model-path /root/Coding/RoboGauge/mytest/go2_cts_hard_terrain_100k.pt \ + --experiment-name debug \ + --stress-benchmark \ + --stress-terrain-names flat slope stairs_up stairs_down wave obstacle \ + --num-processes 62 \ + --seeds 0 1 2 3 4 \ + --frictions 0.5 0.75 1.0 1.25 1.5 1.75 2.0 2.25 2.5 \ --compress-logs \ --headless