diff --git a/UPDATE.md b/UPDATE.md index d55525b..f74166b 100644 --- a/UPDATE.md +++ b/UPDATE.md @@ -1,3 +1,6 @@ +# 20260117 +## v1.0.1 +1. 加入日志数据读取脚本`logs_merge.py`，压缩日志`logs_compress.py`工具 # 20260113 ## v1.0.0 1. 确定最大训练步数为120k diff --git a/setup.py b/setup.py index 089bd04..328c560 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import find_packages from distutils.core import setup setup(name='go2_rl_gym', - version='1.0.0', + version='1.0.1', author='Wu Tianyang', license="MIT", packages=find_packages(), diff --git a/tools/logs_compress.py b/tools/logs_compress.py new file mode 100644 index 0000000..feffd6d --- /dev/null +++ b/tools/logs_compress.py @@ -0,0 +1,79 @@ +# -*- coding: utf-8 -*- +''' +@File : batch_compress.py +@Time : 2026/01/09 17:49:54 +@Author : wty-yy, Gemini 3 +@Version : 1.0 +@Blog : https://wty-yy.github.io/ +@Desc : None +''' +import os +import subprocess +from pathlib import Path + +def smart_compress(logs_root): + logs_path = Path(logs_root).resolve() + if not logs_path.exists(): + print(f"❌ 找不到目录: {logs_root}") + return + + # 遍历 logs 下的第一层子目录 (cts_vanilla, go2_moe_cts 等) + projects = [d for d in logs_path.iterdir() if d.is_dir()] + + for project in projects: + project_name = project.name + print(f"\n🚀 正在处理项目: {project_name}") + + # 1. 搜寻需要包含的内容 + include_items = [] + + # 检查是否存在 exported 文件夹 + if (project / "exported").exists(): + include_items.append("exported") + + # 搜寻所有包含 tfevents 的文件夹 (如 Jan04_15-55-59_) + # 我们寻找 events 文件，然后取其父目录名（相对于项目根目录） + event_folders = set() + for event_file in project.rglob("events.out.tfevents*"): + # 计算相对于项目根目录的路径 + relative_folder = event_file.parent.relative_to(project) + event_folders.add(str(relative_folder)) + + include_items.extend(list(event_folders)) + + if not include_items: + print(f"⚠️ 跳过 {project_name}: 未发现符合条件的训练数据或 exported 文件夹") + continue + + # 2. 构造压缩命令 + output_zst = logs_path / f"{project_name}.tar.zst" + + # 命令解释: + # -C: 切换到项目所在目录，这样压缩包内的路径不会带一堆无用的父级前缀 + # --exclude='*.pt': 显式排除所有模型权重文件 + # -T0: zstd 开启全核并行 + tar_cmd = [ + "tar", + "-I", "zstd -T0 -3", + "-C", str(project), + "--exclude=*.pt", + "--exclude=*.pth", # 预防万一有 .pth + "-cf", str(output_zst) + ] + include_items + + print(f"📦 正在打包 (已排除 .pt 文件)...") + + try: + # 执行压缩 + subprocess.run(tar_cmd, check=True) + + # 统计结果 + final_size = output_zst.stat().st_size / (1024 * 1024) + print(f"✅ 完成! 压缩包: {output_zst.name} ({final_size:.2f} MB)") + except subprocess.CalledProcessError as e: + print(f"❌ {project_name} 压缩失败: {e}") + +if __name__ == "__main__": + # 执行目录 + TARGET_LOGS_DIR = "./logs" + smart_compress(TARGET_LOGS_DIR) diff --git a/tools/logs_merge.py b/tools/logs_merge.py new file mode 100644 index 0000000..3977cc8 --- /dev/null +++ b/tools/logs_merge.py @@ -0,0 +1,121 @@ +import time +import yaml +import argparse +import pandas as pd +from tqdm import tqdm +from pathlib import Path +from collections import defaultdict + +from tensorboard.compat.proto import event_pb2 +from tensorboard.backend.event_processing import event_file_loader + +PATH_PARENT = Path(__file__).parent.resolve() +BASE_COLUMNS = [ + 'it', 'benchmark', + 'lin_vel_err', + 'ang_vel_err', + 'dof_limits', + 'dof_power', + 'orientation_stability', + 'torque_smoothness', + 'flat', 'wave', 'obstacle', + 'slope_fd', 'slope_bd', + 'stairs_fd', 'stairs_bd', + 'terrain_level' +] + +def fast_read(event_file_path, tag_names): + loader = event_file_loader.RawEventFileLoader(event_file_path) + steps = [] + values = [] + + for raw_event in loader.Load(): + event = event_pb2.Event.FromString(raw_event) + + if event.HasField('summary'): + for value in event.summary.value: + if value.tag in tag_names: + steps.append(event.step) + values.append(value.simple_value) + + return pd.DataFrame({'step': steps, 'value': values}) + +class Collector: + def __init__(self, log_dirs): + self.log_dirs = Path(log_dirs) + assert self.log_dirs.exists(), f"Log directory {log_dirs} does not exist." + assert self.log_dirs.is_dir(), f"{log_dirs} is not a directory." + alg_name = self.log_dirs.parent.name + date_str = self.log_dirs.name + self.output_dir = PATH_PARENT / f"{alg_name}_{date_str}" + if self.output_dir.exists(): + s = input(f"[Warning] Output directory {self.output_dir} already exists, press Enter to continue and overwrite or type 'q' to quit...") + if s.lower() == 'q': + exit(0) + self.output_dir.mkdir(parents=True, exist_ok=True) + self.output_csv = self.output_dir / f"{alg_name}_{date_str}.csv" + self.datas = defaultdict(list) + + + self.output_tb = self.output_dir / "tb.csv" + if self.output_tb.exists(): + print(f"Loading existing tensorboard data from {self.output_tb}") + self.tb_df = pd.read_csv(self.output_tb) + else: + start_time = time.time() + print(f"Start reading tensorboard events at {time.ctime(start_time)}") + self.tb_df = fast_read(str(self.log_dirs.glob("events.out.tfevents.*").__next__()), ['Terrain/terrain_level_all', 'Episode/terrain_level_all']) + print(f"Finished reading tensorboard events in {time.time() - start_time:.2f} seconds.") + self.tb_df.to_csv(self.output_tb, index=False) + print(f"Saved tensorboard data to {self.output_tb}") + + def collect(self): + robogauge_results_path = self.log_dirs / "robogauge_results" + results = list(robogauge_results_path.glob("*.yaml")) + results = sorted(results, key=lambda x: int(x.stem.split("_")[-1])) + for result in tqdm(results): + it = int(result.stem.split("_")[-1]) + with open(result, 'r', encoding='utf-8') as f: + data = yaml.safe_load(f) + self.datas['it'].append(it) + self.datas['benchmark'].append(float(data['benchmark_score'])) + for metric_name in [ + 'lin_vel_err', + 'ang_vel_err', + 'dof_limits', + 'dof_power', + 'orientation_stability', + 'torque_smoothness' + ]: + self.datas[f'{metric_name}_mean'].append(float(data['summary'][metric_name]['mean'].split(' ')[0])) + self.datas[f'{metric_name}_mean@25'].append(float(data['summary'][metric_name]['mean@25'].split(' ')[0])) + self.datas[f'{metric_name}_mean@50'].append(float(data['summary'][metric_name]['mean@50'].split(' ')[0])) + for terrain_name in [ + 'flat', + 'wave', + 'obstacle', + 'slope_fd', + 'slope_bd', + 'stairs_fd', + 'stairs_bd', + ]: + if data['robust_score'][terrain_name] is None: + self.datas[f'{terrain_name}_mean'].append(0.0) + self.datas[f'{terrain_name}_mean@25'].append(0.0) + self.datas[f'{terrain_name}_mean@50'].append(0.0) + continue + self.datas[f'{terrain_name}_mean'].append(float(data['robust_score'][terrain_name]['mean'])) + self.datas[f'{terrain_name}_mean@25'].append(float(data['robust_score'][terrain_name]['mean@25'])) + self.datas[f'{terrain_name}_mean@50'].append(float(data['robust_score'][terrain_name]['mean@50'])) + + self.datas['terrain_level'].append(float(self.tb_df[self.tb_df['step'] == it]['value'].values[0])) + df = pd.DataFrame(self.datas) + df.to_csv(self.output_csv, index=False) + print(f"Saved merged results to {self.output_csv}") + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument("--log-dirs") + args = parser.parse_args() + collector = Collector(args.log_dirs) + collector.collect()