v1.0.0; fix bugs

CMD.md

@@ -24,7 +24,7 @@ python robogauge/scripts/run.py \
     --experiment-name debug \
     --multi \
     --num-processes 5 \
-    --seeds 0 1 2 3 4 \
+    --seeds 0 1 2 \
     --frictions 0.5 1.0 1.5 2.0 2.5 \
     --compress-logs \
     --headless
@@ -35,7 +35,7 @@ python robogauge/scripts/run.py \
     --experiment-name debug \
     --multi \
     --num-processes 5 \
-    --seeds 0 1 2 3 4 \
+    --seeds 0 1 2 \
     --frictions 2.0 \
     --level 10 \
     --spawn-type level_eval \
@@ -51,7 +51,8 @@ python robogauge/scripts/run.py \
     --task go2_moe.slope \
     --experiment-name debug \
     --search-max-level \
-    --seeds 0 1 2 3 4 \
+    --seeds 0 1 2 \
+    --search-seeds 0 1 2 3 4 \
     --frictions 1.0 \
     --compress-logs \
     --headless
@@ -63,9 +64,10 @@ python robogauge/scripts/run.py \
     --task go2_moe \
     --experiment-name debug \
     --stress-benchmark \
-    --stress-terrain-names flat slope_fb slope_bd stairs_fb stairs_bd wave obstacle \
+    --stress-terrain-names flat slope_fd slope_bd stairs_fd stairs_bd wave obstacle \
     --num-processes 50 \
-    --seeds 0 1 2 3 4 \
+    --seeds 0 1 2 \
+    --search-seeds 0 1 2 3 4 \
     --frictions 0.5 0.75 1.0 1.25 1.5 1.75 2.0 2.25 2.5 \
     --compress-logs \
     --headless

UPDATE.md

@@ -5,6 +5,7 @@
 2. 加入MultiPipeline和LevelPipeline分开的种子数量`seeds, search_seeds`, Level 5个, Multi 3个, 这样可以显著提高速度, 并且保证评估精度不掉 (两种评估总分前三位都是相同的), 速度提升`3m 32s -> 2m 47s`
 3. 删除stairs_down任务, 将stairs_up重命名为stairs
 4. 对于slope, stairs再分化为两个任务, slope_fd, slope_bd, stairs_fd, stairs_bd, 分别表示正向/背向朝更高处的初始化, target_pos_velocity中指令同时根据是否backward反转, 分别为slope, stairs写初始化位置, 原来6个地形换成现在7种, 测试用时变为`2m 59s`
+5. 修复当地形都是0分时, benchmark得分计算错误的问题
 ## 20251228
 ### v0.1.18
 1. 将final_score改为merge_metrics, 并在multi_pipeline中也进行该评估

robogauge/__init__.py

@@ -1,5 +1,5 @@
 from pathlib import Path
 
-__version__ = "0.1.0"
+__version__ = "1.0.0"
 ROBOGAUGE_ROOT_DIR = str(Path(__file__).parents[1])
 ROBOGAUGE_LOGS_DIR = str(Path(ROBOGAUGE_ROOT_DIR) / "logs")

robogauge/scripts/server.py

@@ -28,9 +28,9 @@ from pprint import pprint
 
 default_args_list = [
     '--stress-benchmark',
-    # '--stress-terrain-names', 'flat', 'wave', 'slope_fd',  'slope_bd', 'stairs_fd', 'stairs_bd', 'obstacle',
-    '--stress-terrain-names', 'flat', 'wave',
-    '--num-processes', '50',
+    '--stress-terrain-names', 'flat', 'wave', 'slope_fd',  'slope_bd', 'stairs_fd', 'stairs_bd', 'obstacle',
+    # '--stress-terrain-names', 'flat', 'wave',
+    '--num-processes', '30',
     '--seeds', '0', '1', '2',
     '--search-seeds', '0', '1', '2', '3', '4',
     '--frictions', '0.5', '0.75', '1.0', '1.25', '1.5', '1.75', '2.0', '2.25', '2.5',
@@ -58,7 +58,7 @@ class ResponseStatus:
     ERROR = "error"
     NOT_FOUND = "not_found"
 
-def run_api_server(input_queue, result_dict, port=9973):
+def run_api_server(input_queue: multiprocessing.Queue, result_dict: dict, port=9973):
     """
     Running in a separate subprocess.
     I/O Process: submit requests -> put into queue -> return ID.
@@ -83,7 +83,8 @@ def run_api_server(input_queue, result_dict, port=9973):
     def get_result(task_id: str):
         if task_id not in result_dict:
             return {"status": ResponseStatus.NOT_FOUND}
-        return result_dict[task_id]
+        result = result_dict.pop(task_id)
+        return result
 
     print(f"📡 API Server listening on port {port}...")
     uvicorn.run(app, host="127.0.0.1", port=port, log_level="error")

robogauge/tasks/pipeline/stress_pipeline.py

@@ -178,12 +178,14 @@ class StressPipeline:
             return
 
         summary = {**self.static_info, 'summary': {}, 'robust_score': {}, 'benchmark_score': 0.0, 'scores': {}}
+        scores = summary['scores']
         metric_collections = defaultdict(lambda: defaultdict(list))
         terrain_collections = defaultdict(lambda: defaultdict(list))
         zero_terrain_count = defaultdict(lambda: 0)
         for result in all_results:
             terrain_name = result['data']['terrain_name']
             terrain_level = result['level']  # None, 0, 1, ..., 10
+            scores[terrain_name] = 0.0
             key = f'{terrain_name}_{terrain_level}'
             key += f'_baseMass{result["data"]["base_mass"]}_friction{result["data"]["friction"]}'
             if terrain_level == 0:
@@ -207,7 +209,6 @@ class StressPipeline:
         
         robust_score = defaultdict(dict)
         robust_scores = []
-        scores = summary['scores']
         for terrain_name, means in terrain_collections.items():
             for mean_name, values in means.items():
                 values.extend([0.0] * zero_terrain_count[terrain_name])  # include zero terrains
@@ -215,7 +216,7 @@ class StressPipeline:
             scores[terrain_name] = robust_score[terrain_name]['mean@50']
             robust_scores.append(robust_score[terrain_name]['mean@50'])
         summary['robust_score'] = dict(robust_score)
-        summary['benchmark_score'] = float(np.mean(robust_scores))
+        summary['benchmark_score'] = float(np.mean(list(scores.values())))
         scores['benchmark'] = summary['benchmark_score']
 
         save_path = stress_logger.log_dir / "stress_benchmark_results.yaml"

setup.py

@@ -10,7 +10,8 @@ setup(
     packages=find_packages(),
     install_requires=[
         "torch",  # Refer: https://pytorch.org/get-started/locally/
-        "numpy>=1.20.0",
+        "numpy==1.20.0",
+        "pillow==9.0.0",
         "mujoco>=3.0.0",
         "dm_control>=1.0.14",
         "scipy",