diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..e4f05d4 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +deploy/deploy_mujoco/deploy_go2.py whitespace=cr-at-eol diff --git a/README.md b/README.md index 3ee1144..1baaaef 100644 --- a/README.md +++ b/README.md @@ -134,6 +134,10 @@ python scripts/rsl_rl/train.py --task=RobotLab-Go2-v0 --headless python scripts/rsl_rl/play.py --task=RobotLab-Go2-v0 ``` +The Go1 adaptation is available on the `go1` branch as +`RobotLab-Go1-v0`; see [docs/go1.md](docs/go1.md) for its training, export, +observation, and MuJoCo deployment details. + --- ### Training with RoboGauge Evaluation diff --git a/deploy/deploy_mujoco/configs/go1.yaml b/deploy/deploy_mujoco/configs/go1.yaml new file mode 100644 index 0000000..1c594cb --- /dev/null +++ b/deploy/deploy_mujoco/configs/go1.yaml @@ -0,0 +1,43 @@ +policy_path: "{ROOT_DIR}/deploy/pre_train/go1/policy.pt" +xml_path: "{ROOT_DIR}/resources/go1/stairs_and_slope.xml" + +render_fps: 300 +video_fps: 60 +save_video: false + +simulation_duration: 60000000.0 +simulation_dt: 0.002 +control_decimation: 10 +base_init_pos: [0.0, 0.0, 0.34] +base_init_quat: [1.0, 0.0, 0.0, 0.0] + +# Deployment defaults to no extra delay. Training randomizes 0-4 physics steps; +# this script's delay buffer is measured in 20 ms policy steps instead. +actuator_delay_min: 0 +actuator_delay_max: 0 +actuator_delay_seed: 0 + +kps: [28.0, 28.0, 28.0, 28.0, 28.0, 28.0, 28.0, 28.0, 28.0, 28.0, 28.0, 28.0] +kds: [0.7, 0.7, 0.7, 0.7, 0.7, 0.7, 0.7, 0.7, 0.7, 0.7, 0.7, 0.7] + +default_angles: [-0.1, 0.8, -1.5, 0.1, 0.8, -1.5, + -0.1, 1.0, -1.5, 0.1, 1.0, -1.5] + +lin_vel_scale: 2.0 +ang_vel_scale: 0.25 +dof_pos_scale: 1.0 +dof_vel_scale: 0.05 +action_pos_scale: 0.25 +cmd_scale: [1.0, 1.0, 1.0] +num_actions: 12 +num_obs: 45 +max_cmd: [1.0, 1.0, 1.0] +cmd_init: [0.5, 0.0, 0.0] +history_len: 10 + +mujoco_joint_names: &joint_names + [FR_hip_joint, FR_thigh_joint, FR_calf_joint, + FL_hip_joint, FL_thigh_joint, FL_calf_joint, + RR_hip_joint, RR_thigh_joint, RR_calf_joint, + RL_hip_joint, RL_thigh_joint, RL_calf_joint] +model_joint_names: *joint_names diff --git a/deploy/deploy_mujoco/deploy_go1.py b/deploy/deploy_mujoco/deploy_go1.py new file mode 100644 index 0000000..a5fd997 --- /dev/null +++ b/deploy/deploy_mujoco/deploy_go1.py @@ -0,0 +1,12 @@ +"""Run an exported Go1 CTS policy in MuJoCo. + +The model, observations, actions, and MuJoCo actuators all use the Unitree SDK +joint order: FR, FL, RR, RL. The exported policy maintains its own history and +therefore consumes one 45-dimensional observation frame per call. +""" + +from deploy_go2 import main + + +if __name__ == "__main__": + main("go1.yaml") diff --git a/deploy/deploy_mujoco/deploy_go2.py b/deploy/deploy_mujoco/deploy_go2.py index 1f682bc..627c9d6 100644 --- a/deploy/deploy_mujoco/deploy_go2.py +++ b/deploy/deploy_mujoco/deploy_go2.py @@ -12,6 +12,8 @@ Notes: This script currently uses CONFIG_NAME = "go2.yaml" and does not expose CLI flags. """ +from __future__ import annotations + import time from pathlib import Path @@ -95,9 +97,9 @@ def build_single_obs(features: dict[str, np.ndarray], layout: list[tuple[str, in return np.concatenate([features[name] for name, _ in layout], axis=0).astype(np.float32, copy=False) -def main() -> None: +def main(config_name: str = CONFIG_NAME) -> None: """Run MuJoCo simulation and deploy the CTS policy in closed-loop control.""" - cfg = load_config(CONFIG_NAME) + cfg = load_config(config_name) layout = [ ("ang_vel", 3), ("gravity", 3), diff --git a/deploy/deploy_mujoco/utils.py b/deploy/deploy_mujoco/utils.py index c136f94..f3be708 100644 --- a/deploy/deploy_mujoco/utils.py +++ b/deploy/deploy_mujoco/utils.py @@ -1,5 +1,7 @@ """Shared helpers for MuJoCo deployment scripts.""" +from __future__ import annotations + from collections import deque from pathlib import Path from types import SimpleNamespace diff --git a/deploy/pre_train/go1/.gitkeep b/deploy/pre_train/go1/.gitkeep new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/deploy/pre_train/go1/.gitkeep @@ -0,0 +1 @@ + diff --git a/docs/go1.md b/docs/go1.md new file mode 100644 index 0000000..99a5349 --- /dev/null +++ b/docs/go1.md @@ -0,0 +1,63 @@ +# Go1 Adaptation + +The `go1` branch adds the `RobotLab-Go1-v0` IsaacLab task and a matching +MuJoCo deployment. It uses the complete Unitree Go1 URDF/MuJoCo model. + +## Interface + +- Policy joint order: `FR, FL, RR, RL`, with hip, thigh, calf for each leg. +- Single-frame observation: 45 values in this order: + body angular velocity (3), projected gravity (3), velocity command (3), + relative joint position (12), joint velocity (12), previous action (12). +- Actor history: 10 frames, or 450 values. +- Action: 12 normalized joint-position offsets, scaled by `0.25 rad` and added + to the default joint angles. +- Policy frequency: 50 Hz (`0.005 s` physics step and decimation 4). +- PD gains: `Kp=28`, `Kd=0.7`; torque is limited to `33.5 Nm`. +- Training randomizes actuator delay over 0-4 physics steps (0-20 ms). The + MuJoCo config defaults to zero extra delay because its delay setting is in + 20 ms policy steps. + +## Train + +Install the editable packages as described in the main README, then run: + +```bash +cd go2_rl_robotlab +python scripts/rsl_rl/train.py \ + --task=RobotLab-Go1-v0 \ + --num_envs=4096 \ + --max_iterations=5000 \ + --headless +``` + +## Play And Export + +`play.py` exports both `policy.pt` and `policy.onnx` into the checkpoint run's +`exported/` directory before starting the rollout. + +```bash +python scripts/rsl_rl/play.py \ + --task=RobotLab-Go1-v0 \ + --num_envs=64 \ + --checkpoint=/absolute/path/to/model_5000.pt +``` + +## MuJoCo + +Place the exported TorchScript file at +`deploy/pre_train/go1/policy.pt`, or edit `policy_path` in +`deploy/deploy_mujoco/configs/go1.yaml`. Then run: + +```bash +MUJOCO_GL=glfw python deploy/deploy_mujoco/deploy_go1.py +``` + +Set `xml_path` in `go1.yaml` to select `flat.xml`, `stairs.xml`, `boxes.xml`, +or `stairs_and_slope.xml`. + +## Actuator Assumption + +The Go1 asset publishes a `33.5 Nm` effort limit but does not include measured +torque-speed knee points. The training actuator therefore uses the known effort +limit with delayed PD control instead of reusing the Go2-HV torque-speed curve. diff --git a/resources/go1/boxes.xml b/resources/go1/boxes.xml new file mode 100644 index 0000000..271958c --- /dev/null +++ b/resources/go1/boxes.xml @@ -0,0 +1,696 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/resources/go1/flat.xml b/resources/go1/flat.xml new file mode 100644 index 0000000..cb94b63 --- /dev/null +++ b/resources/go1/flat.xml @@ -0,0 +1,22 @@ + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/resources/go1/go1.xml b/resources/go1/go1.xml new file mode 100644 index 0000000..d98ddf6 --- /dev/null +++ b/resources/go1/go1.xml @@ -0,0 +1,198 @@ + + + + diff --git a/resources/go1/meshes/calf.stl b/resources/go1/meshes/calf.stl new file mode 100644 index 0000000..3e8ac94 Binary files /dev/null and b/resources/go1/meshes/calf.stl differ diff --git a/resources/go1/meshes/hip.stl b/resources/go1/meshes/hip.stl new file mode 100644 index 0000000..ae590bf Binary files /dev/null and b/resources/go1/meshes/hip.stl differ diff --git a/resources/go1/meshes/thigh.stl b/resources/go1/meshes/thigh.stl new file mode 100644 index 0000000..353801b Binary files /dev/null and b/resources/go1/meshes/thigh.stl differ diff --git a/resources/go1/meshes/thigh_mirror.stl b/resources/go1/meshes/thigh_mirror.stl new file mode 100644 index 0000000..11ef770 Binary files /dev/null and b/resources/go1/meshes/thigh_mirror.stl differ diff --git a/resources/go1/meshes/trunk.stl b/resources/go1/meshes/trunk.stl new file mode 100644 index 0000000..2659eb1 Binary files /dev/null and b/resources/go1/meshes/trunk.stl differ diff --git a/resources/go1/stairs.xml b/resources/go1/stairs.xml new file mode 100644 index 0000000..3599442 --- /dev/null +++ b/resources/go1/stairs.xml @@ -0,0 +1,28 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/resources/go1/stairs_and_slope.xml b/resources/go1/stairs_and_slope.xml new file mode 100644 index 0000000..613d94b --- /dev/null +++ b/resources/go1/stairs_and_slope.xml @@ -0,0 +1,39 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/resources/go1/urdf/go1.urdf b/resources/go1/urdf/go1.urdf new file mode 100644 index 0000000..2d4a9d5 --- /dev/null +++ b/resources/go1/urdf/go1.urdf @@ -0,0 +1,581 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/source/robot_lab/robot_lab/assets/unitree.py b/source/robot_lab/robot_lab/assets/unitree.py index 83d7d3f..7d3f5fa 100644 --- a/source/robot_lab/robot_lab/assets/unitree.py +++ b/source/robot_lab/robot_lab/assets/unitree.py @@ -12,7 +12,7 @@ from isaaclab.assets.articulation import ArticulationCfg from isaaclab.utils import configclass from robot_lab.assets import ISAACLAB_ASSETS_DATA_DIR -from robot_lab.assets.unitree_actuator import UnitreeActuatorCfg_Go2HV +from robot_lab.assets.unitree_actuator import UnitreeActuatorCfg_Go1, UnitreeActuatorCfg_Go2HV ## # Configuration @@ -158,3 +158,40 @@ GO2_CFG_UNITREE = UnitreeArticulationCfg( ], # fmt: on ) + + +# Go1 uses the complete Unitree URDF and the joint order used by the Unitree SDK. +GO1_CFG_UNITREE = UnitreeArticulationCfg( + spawn=UnitreeUrdfFileCfg( + asset_path=f"{ISAACLAB_ASSETS_DATA_DIR}/go1/urdf/go1.urdf", + ), + init_state=ArticulationCfg.InitialStateCfg( + pos=(0.0, 0.0, 0.34), + joint_pos={ + ".*R_hip_joint": -0.1, + ".*L_hip_joint": 0.1, + "F[L,R]_thigh_joint": 0.8, + "R[L,R]_thigh_joint": 1.0, + ".*_calf_joint": -1.5, + }, + joint_vel={".*": 0.0}, + ), + actuators={ + "GO1": UnitreeActuatorCfg_Go1( + joint_names_expr=[".*"], + stiffness=28.0, + damping=0.7, + friction=0.01, + min_delay=0, + max_delay=4, + ), + }, + # fmt: off + joint_sdk_names=[ + "FR_hip_joint", "FR_thigh_joint", "FR_calf_joint", + "FL_hip_joint", "FL_thigh_joint", "FL_calf_joint", + "RR_hip_joint", "RR_thigh_joint", "RR_calf_joint", + "RL_hip_joint", "RL_thigh_joint", "RL_calf_joint", + ], + # fmt: on +) diff --git a/source/robot_lab/robot_lab/assets/unitree_actuator.py b/source/robot_lab/robot_lab/assets/unitree_actuator.py index bb894ec..11d0201 100644 --- a/source/robot_lab/robot_lab/assets/unitree_actuator.py +++ b/source/robot_lab/robot_lab/assets/unitree_actuator.py @@ -128,3 +128,17 @@ class UnitreeActuatorCfg_Go2HV(UnitreeActuatorCfg): Y1 = 20.2 Y2 = 23.4 + +@configclass +class UnitreeActuatorCfg_Go1(UnitreeActuatorCfg): + """Go1 actuator using the URDF effort limit. + + Public Go1 torque-speed knee measurements are not provided with the asset, + so this configuration intentionally applies a constant 33.5 Nm limit + instead of borrowing the materially different Go2-HV motor curve. + """ + + X1 = 1.0e9 + X2 = 1.0e9 + Y1 = 33.5 + Y2 = 33.5 diff --git a/source/robot_lab/robot_lab/tasks/go1/__init__.py b/source/robot_lab/robot_lab/tasks/go1/__init__.py new file mode 100644 index 0000000..95a49e7 --- /dev/null +++ b/source/robot_lab/robot_lab/tasks/go1/__init__.py @@ -0,0 +1,22 @@ +# Copyright (c) 2024-2025 Ziqi Fan +# SPDX-License-Identifier: Apache-2.0 + +"""Go1 locomotion task registration.""" + +import gymnasium as gym +from isaaclab_tasks.utils import import_packages + + +gym.register( + id="RobotLab-Go1-v0", + entry_point="robot_lab.tasks.go1.env.go1_env:Go1Env", + disable_env_checker=True, + kwargs={ + "env_cfg_entry_point": f"{__name__}.env_cfg:Go1EnvCfg", + "rsl_rl_cfg_entry_point": f"{__name__}.rsl_rl_cfg:MoECTSRunnerCfg", + }, +) + + +_BLACKLIST_PKGS = ["utils"] +import_packages(__name__, _BLACKLIST_PKGS) diff --git a/source/robot_lab/robot_lab/tasks/go1/env/__init__.py b/source/robot_lab/robot_lab/tasks/go1/env/__init__.py new file mode 100644 index 0000000..36cce54 --- /dev/null +++ b/source/robot_lab/robot_lab/tasks/go1/env/__init__.py @@ -0,0 +1 @@ +"""Go1 environment implementation.""" diff --git a/source/robot_lab/robot_lab/tasks/go1/env/go1_env.py b/source/robot_lab/robot_lab/tasks/go1/env/go1_env.py new file mode 100644 index 0000000..2c602b7 --- /dev/null +++ b/source/robot_lab/robot_lab/tasks/go1/env/go1_env.py @@ -0,0 +1,14 @@ +from isaaclab.envs import ManagerBasedRLEnv, ManagerBasedRLEnvCfg + +from robot_lab.tasks.go2.manager.action_manager import ActionManagerGo2 + + +class Go1Env(ManagerBasedRLEnv): + """Go1 environment using the shared ordered action manager.""" + + cfg: ManagerBasedRLEnvCfg + + def load_managers(self): + super().load_managers() + self.action_manager = ActionManagerGo2(self.cfg.actions, self) + print("[Go1Env-INFO] Using ordered action manager:", self.action_manager) diff --git a/source/robot_lab/robot_lab/tasks/go1/env_cfg.py b/source/robot_lab/robot_lab/tasks/go1/env_cfg.py new file mode 100644 index 0000000..7b67ecf --- /dev/null +++ b/source/robot_lab/robot_lab/tasks/go1/env_cfg.py @@ -0,0 +1,509 @@ +import math +import isaaclab.sim as sim_utils +from isaaclab.assets import ArticulationCfg, AssetBaseCfg +from isaaclab.envs import ManagerBasedRLEnvCfg +from isaaclab.managers import CurriculumTermCfg as CurrTerm +from isaaclab.managers import EventTermCfg as EventTerm +from isaaclab.managers import ObservationGroupCfg as ObsGroup +from isaaclab.managers import ObservationTermCfg as ObsTerm +from isaaclab.managers import RewardTermCfg as RewTerm +from isaaclab.managers import SceneEntityCfg +from isaaclab.managers import TerminationTermCfg as DoneTerm +from isaaclab.scene import InteractiveSceneCfg +from isaaclab.sensors import ContactSensorCfg, RayCasterCfg, patterns +from isaaclab.terrains import TerrainImporterCfg +from isaaclab.utils import configclass +from isaaclab.utils.assets import ISAAC_NUCLEUS_DIR, ISAACLAB_NUCLEUS_DIR +from isaaclab.utils.noise import AdditiveUniformNoiseCfg as Unoise + +import robot_lab.tasks.go2.mdp as mdp +from robot_lab.assets.unitree import GO1_CFG_UNITREE +from robot_lab.tasks.go2.mdp.terrains import TERRAIN_CFG + +JOINT_NAMES = [ + "FR_hip_joint", "FR_thigh_joint", "FR_calf_joint", + "FL_hip_joint", "FL_thigh_joint", "FL_calf_joint", + "RR_hip_joint", "RR_thigh_joint", "RR_calf_joint", + "RL_hip_joint", "RL_thigh_joint", "RL_calf_joint", +] + +BASE_LINK_NAME = "trunk" +FOOT_LINK_NAME = ".*_foot" +BASE_HEIGHT_TARGET = 0.30 # Go1 nominal trunk height used by the original Gym task. + +## +# Scene definition +## + +@configclass +class Go1SceneCfg(InteractiveSceneCfg): + """Configuration for the terrain scene with the Go1 robot.""" + + terrain = TerrainImporterCfg( + prim_path="/World/ground", + terrain_type="generator", + terrain_generator=TERRAIN_CFG, + max_init_terrain_level=5, + collision_group=-1, + physics_material=sim_utils.RigidBodyMaterialCfg( + friction_combine_mode="average", + restitution_combine_mode="average", + static_friction=1.0, + dynamic_friction=1.0, + restitution=0.0, + ), + visual_material=sim_utils.MdlFileCfg( + mdl_path=f"{ISAACLAB_NUCLEUS_DIR}/Materials/TilesMarbleSpiderWhiteBrickBondHoned/TilesMarbleSpiderWhiteBrickBondHoned.mdl", + project_uvw=True, + texture_scale=(0.25, 0.25), + ), + debug_vis=False + ) + + robot: ArticulationCfg = GO1_CFG_UNITREE.replace(prim_path="{ENV_REGEX_NS}/Robot") + + height_scanner = RayCasterCfg( + prim_path="{ENV_REGEX_NS}/Robot/trunk", + offset=RayCasterCfg.OffsetCfg(pos=(0.0, 0.0, 20.0)), + ray_alignment="yaw", + pattern_cfg=patterns.GridPatternCfg(resolution=0.1, size=[1.6, 1.0]), + debug_vis=False, + mesh_prim_paths=["/World/ground"], + ) + height_scanner_small = RayCasterCfg( + prim_path="{ENV_REGEX_NS}/Robot/trunk", + offset=RayCasterCfg.OffsetCfg(pos=(0.0, 0.0, 20.0)), + ray_alignment="yaw", + pattern_cfg=patterns.GridPatternCfg(resolution=0.1, size=[0.4, 0.3]), + debug_vis=False, + mesh_prim_paths=["/World/ground"], + ) + contact_forces = ContactSensorCfg( + prim_path="{ENV_REGEX_NS}/Robot/.*", + history_length=3, + track_air_time=True, + ) + + # 灯光 + sky_light = AssetBaseCfg( + prim_path="/World/skyLight", + spawn=sim_utils.DomeLightCfg( + intensity=750.0, + texture_file=f"{ISAAC_NUCLEUS_DIR}/Materials/Textures/Skies/PolyHaven/kloofendal_43d_clear_puresky_4k.hdr", + ), + ) + +## +# MDP settings +## + +@configclass +class CommandsCfg: + """Command specifications for the MDP.""" + base_velocity = mdp.Go2RLGymCommandCfg() + +@configclass +class ActionsCfg: + """Action specifications for the MDP.""" + + # 腿部关节:位置控制 + joint_pos = mdp.JointPositionActionCfg( + asset_name="robot", + joint_names=JOINT_NAMES, + scale={".*_hip_joint": 0.25, "^(?!.*_hip_joint).*": 0.25}, + use_default_offset=True, + clip={".*": (-100.0, 100.0)}, + preserve_order=True + ) + +@configclass +class ObservationsCfg: + @configclass + class PolicyCfg(ObsGroup): + """Observations for policy group.""" + base_ang_vel = ObsTerm( + func=mdp.base_ang_vel, + noise=Unoise(n_min=-0.2, n_max=0.2), + clip=(-100.0, 100.0), + scale=0.25, + ) + projected_gravity = ObsTerm( + func=mdp.projected_gravity, + noise=Unoise(n_min=-0.05, n_max=0.05), + clip=(-100.0, 100.0), + scale=1.0, + ) + velocity_commands = ObsTerm( + func=mdp.generated_commands, + params={"command_name": "base_velocity"}, + clip=(-100.0, 100.0), + scale=1.0, + ) + joint_pos = ObsTerm( + func=mdp.joint_pos_rel, + params={"asset_cfg": SceneEntityCfg("robot", joint_names=JOINT_NAMES, preserve_order=True)}, + noise=Unoise(n_min=-0.03, n_max=0.03), + clip=(-100.0, 100.0), + scale=1.0, + ) + joint_vel = ObsTerm( + func=mdp.joint_vel_rel, + params={"asset_cfg": SceneEntityCfg("robot", joint_names=JOINT_NAMES, preserve_order=True)}, + noise=Unoise(n_min=-2.0, n_max=2.0), + clip=(-100.0, 100.0), + scale=0.05, + ) + actions = ObsTerm( + func=mdp.last_action, + clip=(-100.0, 100.0), + scale=1.0, + ) + def __post_init__(self): + self.history_length = 10 + self.enable_corruption = True + self.concatenate_terms = True + self.flatten_history_dim = True + + @configclass + class CriticCfg(ObsGroup): + base_lin_vel = ObsTerm( + func=mdp.base_lin_vel, + clip=(-100.0, 100.0), + scale=2.0, + ) + base_ang_vel = ObsTerm( + func=mdp.base_ang_vel, + clip=(-100.0, 100.0), + scale=0.25, + ) + projected_gravity = ObsTerm( + func=mdp.projected_gravity, + clip=(-100.0, 100.0), + scale=1.0, + ) + velocity_commands = ObsTerm( + func=mdp.generated_commands, + params={"command_name": "base_velocity"}, + clip=(-100.0, 100.0), + scale=1.0, + ) + joint_pos = ObsTerm( + func=mdp.joint_pos_rel, + params={"asset_cfg": SceneEntityCfg("robot", joint_names=JOINT_NAMES, preserve_order=True)}, + clip=(-100.0, 100.0), + scale=1.0, + ) + joint_vel = ObsTerm( + func=mdp.joint_vel_rel, + params={"asset_cfg": SceneEntityCfg("robot", joint_names=JOINT_NAMES, preserve_order=True)}, + clip=(-100.0, 100.0), + scale=0.05, + ) + actions = ObsTerm( + func=mdp.last_action, + clip=(-100.0, 100.0), + scale=1.0, + ) + joint_acc = ObsTerm( + func=mdp.joint_acc, + params={"asset_cfg": SceneEntityCfg("robot", joint_names=JOINT_NAMES, preserve_order=True)}, + clip=(-100.0, 100.0), + scale=1e-4, + ) + joint_torque = ObsTerm( + func=mdp.joint_effort, + params={"asset_cfg": SceneEntityCfg("robot", joint_names=JOINT_NAMES, preserve_order=True)}, + clip=(-100.0, 100.0), + scale=0.01, + ) + contact_force = ObsTerm( + func=mdp.foot_contact_force_norm, + params={"sensor_cfg": SceneEntityCfg("contact_forces", body_names=FOOT_LINK_NAME)}, + clip=(-100.0, 100.0), + scale=1e-3, + ) + height_scan = ObsTerm( + func=mdp.height_scan, + params={"sensor_cfg": SceneEntityCfg("height_scanner")}, + clip=(-1.0, 1.0), + scale=2.5, + ) + def __post_init__(self): + self.enable_corruption = False + self.concatenate_terms = True + + @configclass + class SingleObsCfg(PolicyCfg): + def __post_init__(self): + super().__post_init__() + self.history_length = 1 + + policy: PolicyCfg = PolicyCfg() + critic: CriticCfg = CriticCfg() + single_obs: SingleObsCfg = SingleObsCfg() # Used to obtain the current-timestep observation for the MoE CTS model + +@configclass +class EventCfg: + """Configuration for events.""" + + randomize_rigid_body_mass_base = EventTerm( + func=mdp.randomize_rigid_body_mass, + mode="startup", + params={ + "asset_cfg": SceneEntityCfg("robot", body_names=BASE_LINK_NAME), + "mass_distribution_params": (-1.0, 1.0), + "operation": "add", + "recompute_inertia": True, + }, + ) + randomize_rigid_body_mass_others = EventTerm( + func=mdp.randomize_rigid_body_mass, + mode="startup", + params={ + "asset_cfg": SceneEntityCfg("robot", body_names="^(?!.*trunk).*"), + "mass_distribution_params": (0.9, 1.1), + "operation": "scale", + "recompute_inertia": True, + }, + ) + randomize_com_positions = EventTerm( + func=mdp.randomize_rigid_body_com, + mode="startup", + params={ + "asset_cfg": SceneEntityCfg("robot", body_names=BASE_LINK_NAME), + "com_range": {"x": (-0.03, 0.03), "y": (-0.03, 0.03), "z": (-0.03, 0.03)}, + }, + ) + reset_robot_joints = EventTerm( + func=mdp.reset_joints_by_scale, + mode="reset", + params={ + "position_range": (0.5, 1.5), + "velocity_range": (0.0, 0.0), + }, + ) + randomize_actuator_gains = EventTerm( + func=mdp.randomize_actuator_gains, + mode="reset", + params={ + "asset_cfg": SceneEntityCfg("robot", joint_names=".*"), + "stiffness_distribution_params": (0.9, 1.1), + "damping_distribution_params": (0.9, 1.1), + "operation": "scale", + "distribution": "uniform", + }, + ) + randomize_motor_zero_offset = EventTerm( + func=mdp.randomize_action_joint_pos_offset, + mode="reset", + params={ + "action_term_name": "joint_pos", + "offset_range": (-0.035, 0.035), + }, + ) + randomize_push_robot = EventTerm( + func=mdp.push_by_setting_velocity, + mode="interval", + interval_range_s=(4.0, 4.0), + params={ + "velocity_range": { + "x": (-0.4, 0.4), + "y": (-0.4, 0.4), + "roll": (-0.6, 0.6), + "pitch": (-0.6, 0.6), + "yaw": (-0.6, 0.6) + } + } + ) + randomize_rigid_body_material = EventTerm( + func=mdp.randomize_rigid_body_material, + mode="startup", + params={ + "asset_cfg": SceneEntityCfg("robot", body_names=".*"), + "static_friction_range": (0.0, 2.0), + "dynamic_friction_range": (0.0, 2.0), + "restitution_range": (0.0, 0.5), + "num_buckets": 64, + "make_consistent": True + }, + ) + reset_base = EventTerm( + func=mdp.reset_root_state_uniform, + mode="reset", + params={ + "pose_range": {"x": (-0.5, 0.5), "y": (-0.5, 0.5), "z": (0.0, 0.2), "yaw": (-3.14, 3.14)}, + "velocity_range": { + "x": (-0.5, 0.5), + "y": (-0.5, 0.5), + "z": (-0.5, 0.5), + "roll": (-0.5, 0.5), + "pitch": (-0.5, 0.5), + "yaw": (-0.5, 0.5), + }, + }, + ) + + +@configclass +class RewardsCfg: + """Reward terms for the MDP.""" + + track_lin_vel_xy_exp = RewTerm( + func=mdp.track_lin_vel_xy_exp, + weight=2.0, + params={"command_name": "base_velocity", "std": 0.5} + ) + track_ang_vel_z_exp = RewTerm( + func=mdp.track_ang_vel_z_exp, + weight=1.0, + params={"command_name": "base_velocity", "std": 0.5} + ) + lin_vel_z_l2 = RewTerm(func=mdp.lin_vel_z_l2, weight=-2.0) + ang_vel_xy_l2 = RewTerm(func=mdp.ang_vel_xy_l2, weight=-0.05) + + # The joint_acc reward is not computed on the same scale in Gym and Lab. + # In Gym, it is computed at the policy-step level, + # while in Lab, it is computed at the physics-step level. + # In Lab, the reward calculation is more precise, and because the L2 term is more sensitive to outliers. + # Thus, the reward value is overall higher, so we need to decrease the weights to be suitable for Lab. + joint_acc_l2 = RewTerm( + func=mdp.joint_acc_l2, + weight=-1.0e-7, + params={"asset_cfg": SceneEntityCfg("robot", joint_names=JOINT_NAMES)} + ) + + joint_power = RewTerm( + func=mdp.joint_power, + weight=-2e-5, + params={"asset_cfg": SceneEntityCfg("robot", joint_names=JOINT_NAMES)} + ) + joint_torques_l2 = RewTerm( + func=mdp.joint_torques_l2, + weight=-1e-4, + params={"asset_cfg": SceneEntityCfg("robot", joint_names=JOINT_NAMES)} + ) + base_height_l2 = RewTerm( + func=mdp.base_height_l2, + weight=-1.0, + params={ + "asset_cfg": SceneEntityCfg("robot", body_names=BASE_LINK_NAME), + "target_height": BASE_HEIGHT_TARGET, + "sensor_cfg": SceneEntityCfg("height_scanner_small"), + } + ) + action_rate_l2 = RewTerm(func=mdp.action_rate_l2, weight=-0.01) + action_smoothness_l2 = RewTerm(func=mdp.action_smoothness_l2, weight=-0.01) + undesired_contacts = RewTerm( + func=mdp.undesired_contacts, + weight=-1.0, + params={"sensor_cfg": SceneEntityCfg("contact_forces", body_names=".*_thigh|.*_calf"), "threshold": 5.0}, + ) + joint_pos_limits = RewTerm( + func=mdp.joint_pos_limits, + weight=-2.0, + params={"asset_cfg": SceneEntityCfg("robot", joint_names=JOINT_NAMES)}, + ) + feet_regulation = RewTerm( + func=mdp.feet_regulation, + weight=-0.05, + params={ + "base_height_target": BASE_HEIGHT_TARGET, + "asset_cfg": SceneEntityCfg("robot", body_names=FOOT_LINK_NAME), + "sensor_cfg": SceneEntityCfg("height_scanner_small"), + }, + ) + hip_pos_penalty_l1 = RewTerm( + func=mdp.hip_pos_penalty_l1, + weight=-0.05, + params={ + "command_name": "base_velocity", + "asset_cfg": SceneEntityCfg("robot", joint_names=".*_hip_joint"), + "stand_still_scale": 1.0, + "command_threshold": 0.1, + }, + ) + joint_pos_penalty_l1 = RewTerm( + func=mdp.joint_pos_penalty_l1, + weight=-0.01, + params={ + "command_name": "base_velocity", + "asset_cfg": SceneEntityCfg("robot", joint_names=".*_(thigh|calf)_joint"), + "stand_still_scale": 1.0, + "velocity_threshold": 0.1, + "command_threshold": 0.1, + }, + ) + +@configclass +class TerminationsCfg: + """Termination terms for the MDP.""" + time_out = DoneTerm(func=mdp.time_out, time_out=True) + illegal_contact = DoneTerm( + func=mdp.illegal_contact, + params={ + "sensor_cfg": SceneEntityCfg("contact_forces", body_names=BASE_LINK_NAME), + "threshold": 1.0 + }, + ) + +@configclass +class CurriculumCfg: + """Curriculum terms for the MDP.""" + terrain_levels = CurrTerm(func=mdp.terrain_levels_vel_gym) + base_linear_velocity = CurrTerm(mdp.gradual_reward_weight_modification, params={ + "term_name": "lin_vel_z_l2", "initial_weight": -2.0, "final_weight": -0.0, "start_it": 0, "end_it": 1500 + }) + base_height_l2 = CurrTerm(mdp.gradual_reward_weight_modification, params={ + "term_name": "base_height_l2", "initial_weight": -1.0, "final_weight": -10.0, "start_it": 0, "end_it": 5000 + }) + +## +# Environment configuration +## + +@configclass +class Go1EnvCfg(ManagerBasedRLEnvCfg): + """Merged configuration for the Go1 robot on rough terrain.""" + + # Scene settings + scene: Go1SceneCfg = Go1SceneCfg(num_envs=16384, env_spacing=0.5) + # Basic settings + observations: ObservationsCfg = ObservationsCfg() + actions: ActionsCfg = ActionsCfg() + commands: CommandsCfg = CommandsCfg() + # MDP settings + rewards: RewardsCfg = RewardsCfg() + terminations: TerminationsCfg = TerminationsCfg() + events: EventCfg = EventCfg() + curriculum: CurriculumCfg = CurriculumCfg() + + def __post_init__(self): + """Post initialization.""" + # General settings + self.decimation = 4 + self.episode_length_s = 25.0 + # Simulation settings + self.sim.dt = 0.005 + self.sim.render_interval = self.decimation + + # Physics material settings from subclass + self.sim.physics_material = self.scene.terrain.physics_material + self.sim.physx.gpu_max_rigid_patch_count = int(1 * 1024 * 1024) # 1 million + self.sim.physx.gpu_collision_stack_size = int(512 * 1024 * 1024) # 128 MB + self.sim.physx.enable_external_forces_every_iteration = True + + # Update sensor periods + if self.scene.height_scanner is not None: + self.scene.height_scanner.update_period = self.decimation * self.sim.dt + if self.scene.height_scanner_small is not None: + self.scene.height_scanner_small.update_period = self.decimation * self.sim.dt + if self.scene.contact_forces is not None: + self.scene.contact_forces.update_period = self.sim.dt + + # Handle curriculum for terrain generator + if getattr(self.curriculum, "terrain_levels", None) is not None: + if self.scene.terrain.terrain_generator is not None: + self.scene.terrain.terrain_generator.curriculum = True + else: + if self.scene.terrain.terrain_generator is not None: + self.scene.terrain.terrain_generator.curriculum = False diff --git a/source/robot_lab/robot_lab/tasks/go1/rsl_rl_cfg.py b/source/robot_lab/robot_lab/tasks/go1/rsl_rl_cfg.py new file mode 100644 index 0000000..35547be --- /dev/null +++ b/source/robot_lab/robot_lab/tasks/go1/rsl_rl_cfg.py @@ -0,0 +1,86 @@ +from isaaclab.utils import configclass +from isaaclab_rl.rsl_rl import RslRlOnPolicyRunnerCfg, RslRlPpoActorCriticCfg, RslRlPpoAlgorithmCfg + +@configclass +class PPORunnerCfg(RslRlOnPolicyRunnerCfg): + num_steps_per_env = 24 + max_iterations = 300000 + save_interval = 500 + experiment_name = "go1_rough" + + policy = RslRlPpoActorCriticCfg( + init_noise_std=1.0, + actor_obs_normalization=False, + critic_obs_normalization=False, + actor_hidden_dims=[512, 256, 128], + critic_hidden_dims=[512, 256, 128], + activation="elu", + ) + algorithm = RslRlPpoAlgorithmCfg( + value_loss_coef=1.0, + use_clipped_value_loss=True, + clip_param=0.2, + entropy_coef=0.01, + num_learning_epochs=5, + num_mini_batches=4, + learning_rate=1.0e-3, + schedule="adaptive", + gamma=0.99, + lam=0.95, + desired_kl=0.01, + max_grad_norm=1.0, + ) + + +@configclass +class RslRlMoeCtsActorCriticCfg(RslRlPpoActorCriticCfg): + class_name = "ActorCriticMoECTS" + init_noise_std = 1.0 + expert_num = 8 # number of experts in the student model + latent_dim = 32 + norm_type = 'l2norm' # normalization type for encoders: l2norm, simnorm + teacher_encoder_hidden_dims = [512, 256] + student_encoder_hidden_dims = [512, 256, 256] + actor_hidden_dims=[512, 256, 128] + critic_hidden_dims=[512, 256, 128] + activation="elu" + actor_obs_normalization = False + critic_obs_normalization = False + +@configclass +class RslRlMoeCtsAlgorithmCfg(RslRlPpoAlgorithmCfg): + class_name = "MoECTS" + value_loss_coef = 1.0 + load_balance_coef = 0.01 # coefficient for load balance loss + use_clipped_value_loss = True + clip_param = 0.2 + entropy_coef = 0.01 + num_learning_epochs = 5 + num_mini_batches = 4 + learning_rate = 1e-3 + student_encoder_learning_rate = 1e-3 + schedule = "adaptive" + gamma = 0.99 + lam = 0.95 + betas = (0.9, 0.999) + weight_decay = 0.0 + desired_kl = 0.01 + max_grad_norm = 1.0 + teacher_env_ratio = 0.75 # percentage of envs assigned to teacher + +@configclass +class MoECTSRunnerCfg(RslRlOnPolicyRunnerCfg): + experiment_name = "go1_moe_cts" + class_name = "OnPolicyRunnerCTS" + num_steps_per_env = 24 + max_iterations = 300000 + save_interval = 500 + policy = RslRlMoeCtsActorCriticCfg() + algorithm = RslRlMoeCtsAlgorithmCfg() + +# concat elu inspired by concat relu from https://arxiv.org/pdf/2303.07507 +@configclass +class MoECTSCatELURunnerCfg(MoECTSRunnerCfg): + def __post_init__(self): + super().__post_init__() + self.policy.activation = 'cat_elu'