v1.0.1; add tools
This commit is contained in:
wty-yy
@@ -1,3 +1,6 @@
|
||||
# 20260117
|
||||
## v1.0.1
|
||||
1. 加入日志数据读取脚本`logs_merge.py`，压缩日志`logs_compress.py`工具
|
||||
# 20260113
|
||||
## v1.0.0
|
||||
1. 确定最大训练步数为120k
|
||||
|
||||
2
setup.py
2
setup.py
@@ -2,7 +2,7 @@ from setuptools import find_packages
|
||||
from distutils.core import setup
|
||||
|
||||
setup(name='go2_rl_gym',
|
||||
version='1.0.0',
|
||||
version='1.0.1',
|
||||
author='Wu Tianyang',
|
||||
license="MIT",
|
||||
packages=find_packages(),
|
||||
|
||||
79
tools/logs_compress.py
Normal file
79
tools/logs_compress.py
Normal file
@@ -0,0 +1,79 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
'''
|
||||
@File : batch_compress.py
|
||||
@Time : 2026/01/09 17:49:54
|
||||
@Author : wty-yy, Gemini 3
|
||||
@Version : 1.0
|
||||
@Blog : https://wty-yy.github.io/
|
||||
@Desc : None
|
||||
'''
|
||||
import os
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
def smart_compress(logs_root):
|
||||
logs_path = Path(logs_root).resolve()
|
||||
if not logs_path.exists():
|
||||
print(f"❌ 找不到目录: {logs_root}")
|
||||
return
|
||||
|
||||
# 遍历 logs 下的第一层子目录 (cts_vanilla, go2_moe_cts 等)
|
||||
projects = [d for d in logs_path.iterdir() if d.is_dir()]
|
||||
|
||||
for project in projects:
|
||||
project_name = project.name
|
||||
print(f"\n🚀 正在处理项目: {project_name}")
|
||||
|
||||
# 1. 搜寻需要包含的内容
|
||||
include_items = []
|
||||
|
||||
# 检查是否存在 exported 文件夹
|
||||
if (project / "exported").exists():
|
||||
include_items.append("exported")
|
||||
|
||||
# 搜寻所有包含 tfevents 的文件夹 (如 Jan04_15-55-59_)
|
||||
# 我们寻找 events 文件，然后取其父目录名（相对于项目根目录）
|
||||
event_folders = set()
|
||||
for event_file in project.rglob("events.out.tfevents*"):
|
||||
# 计算相对于项目根目录的路径
|
||||
relative_folder = event_file.parent.relative_to(project)
|
||||
event_folders.add(str(relative_folder))
|
||||
|
||||
include_items.extend(list(event_folders))
|
||||
|
||||
if not include_items:
|
||||
print(f"⚠️ 跳过 {project_name}: 未发现符合条件的训练数据或 exported 文件夹")
|
||||
continue
|
||||
|
||||
# 2. 构造压缩命令
|
||||
output_zst = logs_path / f"{project_name}.tar.zst"
|
||||
|
||||
# 命令解释:
|
||||
# -C: 切换到项目所在目录，这样压缩包内的路径不会带一堆无用的父级前缀
|
||||
# --exclude='*.pt': 显式排除所有模型权重文件
|
||||
# -T0: zstd 开启全核并行
|
||||
tar_cmd = [
|
||||
"tar",
|
||||
"-I", "zstd -T0 -3",
|
||||
"-C", str(project),
|
||||
"--exclude=*.pt",
|
||||
"--exclude=*.pth", # 预防万一有 .pth
|
||||
"-cf", str(output_zst)
|
||||
] + include_items
|
||||
|
||||
print(f"📦 正在打包 (已排除 .pt 文件)...")
|
||||
|
||||
try:
|
||||
# 执行压缩
|
||||
subprocess.run(tar_cmd, check=True)
|
||||
|
||||
# 统计结果
|
||||
final_size = output_zst.stat().st_size / (1024 * 1024)
|
||||
print(f"✅ 完成! 压缩包: {output_zst.name} ({final_size:.2f} MB)")
|
||||
except subprocess.CalledProcessError as e:
|
||||
print(f"❌ {project_name} 压缩失败: {e}")
|
||||
|
||||
if __name__ == "__main__":
|
||||
# 执行目录
|
||||
TARGET_LOGS_DIR = "./logs"
|
||||
smart_compress(TARGET_LOGS_DIR)
|
||||
121
tools/logs_merge.py
Normal file
121
tools/logs_merge.py
Normal file
@@ -0,0 +1,121 @@
|
||||
import time
|
||||
import yaml
|
||||
import argparse
|
||||
import pandas as pd
|
||||
from tqdm import tqdm
|
||||
from pathlib import Path
|
||||
from collections import defaultdict
|
||||
|
||||
from tensorboard.compat.proto import event_pb2
|
||||
from tensorboard.backend.event_processing import event_file_loader
|
||||
|
||||
PATH_PARENT = Path(__file__).parent.resolve()
|
||||
BASE_COLUMNS = [
|
||||
'it', 'benchmark',
|
||||
'lin_vel_err',
|
||||
'ang_vel_err',
|
||||
'dof_limits',
|
||||
'dof_power',
|
||||
'orientation_stability',
|
||||
'torque_smoothness',
|
||||
'flat', 'wave', 'obstacle',
|
||||
'slope_fd', 'slope_bd',
|
||||
'stairs_fd', 'stairs_bd',
|
||||
'terrain_level'
|
||||
]
|
||||
|
||||
def fast_read(event_file_path, tag_names):
|
||||
loader = event_file_loader.RawEventFileLoader(event_file_path)
|
||||
steps = []
|
||||
values = []
|
||||
|
||||
for raw_event in loader.Load():
|
||||
event = event_pb2.Event.FromString(raw_event)
|
||||
|
||||
if event.HasField('summary'):
|
||||
for value in event.summary.value:
|
||||
if value.tag in tag_names:
|
||||
steps.append(event.step)
|
||||
values.append(value.simple_value)
|
||||
|
||||
return pd.DataFrame({'step': steps, 'value': values})
|
||||
|
||||
class Collector:
|
||||
def __init__(self, log_dirs):
|
||||
self.log_dirs = Path(log_dirs)
|
||||
assert self.log_dirs.exists(), f"Log directory {log_dirs} does not exist."
|
||||
assert self.log_dirs.is_dir(), f"{log_dirs} is not a directory."
|
||||
alg_name = self.log_dirs.parent.name
|
||||
date_str = self.log_dirs.name
|
||||
self.output_dir = PATH_PARENT / f"{alg_name}_{date_str}"
|
||||
if self.output_dir.exists():
|
||||
s = input(f"[Warning] Output directory {self.output_dir} already exists, press Enter to continue and overwrite or type 'q' to quit...")
|
||||
if s.lower() == 'q':
|
||||
exit(0)
|
||||
self.output_dir.mkdir(parents=True, exist_ok=True)
|
||||
self.output_csv = self.output_dir / f"{alg_name}_{date_str}.csv"
|
||||
self.datas = defaultdict(list)
|
||||
|
||||
|
||||
self.output_tb = self.output_dir / "tb.csv"
|
||||
if self.output_tb.exists():
|
||||
print(f"Loading existing tensorboard data from {self.output_tb}")
|
||||
self.tb_df = pd.read_csv(self.output_tb)
|
||||
else:
|
||||
start_time = time.time()
|
||||
print(f"Start reading tensorboard events at {time.ctime(start_time)}")
|
||||
self.tb_df = fast_read(str(self.log_dirs.glob("events.out.tfevents.*").__next__()), ['Terrain/terrain_level_all', 'Episode/terrain_level_all'])
|
||||
print(f"Finished reading tensorboard events in {time.time() - start_time:.2f} seconds.")
|
||||
self.tb_df.to_csv(self.output_tb, index=False)
|
||||
print(f"Saved tensorboard data to {self.output_tb}")
|
||||
|
||||
def collect(self):
|
||||
robogauge_results_path = self.log_dirs / "robogauge_results"
|
||||
results = list(robogauge_results_path.glob("*.yaml"))
|
||||
results = sorted(results, key=lambda x: int(x.stem.split("_")[-1]))
|
||||
for result in tqdm(results):
|
||||
it = int(result.stem.split("_")[-1])
|
||||
with open(result, 'r', encoding='utf-8') as f:
|
||||
data = yaml.safe_load(f)
|
||||
self.datas['it'].append(it)
|
||||
self.datas['benchmark'].append(float(data['benchmark_score']))
|
||||
for metric_name in [
|
||||
'lin_vel_err',
|
||||
'ang_vel_err',
|
||||
'dof_limits',
|
||||
'dof_power',
|
||||
'orientation_stability',
|
||||
'torque_smoothness'
|
||||
]:
|
||||
self.datas[f'{metric_name}_mean'].append(float(data['summary'][metric_name]['mean'].split(' ')[0]))
|
||||
self.datas[f'{metric_name}_mean@25'].append(float(data['summary'][metric_name]['mean@25'].split(' ')[0]))
|
||||
self.datas[f'{metric_name}_mean@50'].append(float(data['summary'][metric_name]['mean@50'].split(' ')[0]))
|
||||
for terrain_name in [
|
||||
'flat',
|
||||
'wave',
|
||||
'obstacle',
|
||||
'slope_fd',
|
||||
'slope_bd',
|
||||
'stairs_fd',
|
||||
'stairs_bd',
|
||||
]:
|
||||
if data['robust_score'][terrain_name] is None:
|
||||
self.datas[f'{terrain_name}_mean'].append(0.0)
|
||||
self.datas[f'{terrain_name}_mean@25'].append(0.0)
|
||||
self.datas[f'{terrain_name}_mean@50'].append(0.0)
|
||||
continue
|
||||
self.datas[f'{terrain_name}_mean'].append(float(data['robust_score'][terrain_name]['mean']))
|
||||
self.datas[f'{terrain_name}_mean@25'].append(float(data['robust_score'][terrain_name]['mean@25']))
|
||||
self.datas[f'{terrain_name}_mean@50'].append(float(data['robust_score'][terrain_name]['mean@50']))
|
||||
|
||||
self.datas['terrain_level'].append(float(self.tb_df[self.tb_df['step'] == it]['value'].values[0]))
|
||||
df = pd.DataFrame(self.datas)
|
||||
df.to_csv(self.output_csv, index=False)
|
||||
print(f"Saved merged results to {self.output_csv}")
|
||||
|
||||
if __name__ == '__main__':
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--log-dirs")
|
||||
args = parser.parse_args()
|
||||
collector = Collector(args.log_dirs)
|
||||
collector.collect()
|
||||
Reference in New Issue
Block a user