cyy/go2_rl_robotlab
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add Go1 training and MuJoCo deployment

Browse Source
This commit is contained in:
youyuan.chen
2026-07-26 00:40:06 +08:00
parent 28b4516d22
commit 0d2afbf2bb
26 changed files with 2378 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

39
source/robot_lab/robot_lab/assets/unitree.py
View File

@@ -12,7 +12,7 @@ from isaaclab.assets.articulation import ArticulationCfg
from isaaclab.utils import configclass
from robot_lab.assets import ISAACLAB_ASSETS_DATA_DIR
from robot_lab.assets.unitree_actuator import UnitreeActuatorCfg_Go2HV
from robot_lab.assets.unitree_actuator import UnitreeActuatorCfg_Go1, UnitreeActuatorCfg_Go2HV
##
# Configuration
@@ -158,3 +158,40 @@ GO2_CFG_UNITREE = UnitreeArticulationCfg(
],
# fmt: on
)
# Go1 uses the complete Unitree URDF and the joint order used by the Unitree SDK.
GO1_CFG_UNITREE = UnitreeArticulationCfg(
spawn=UnitreeUrdfFileCfg(
asset_path=f"{ISAACLAB_ASSETS_DATA_DIR}/go1/urdf/go1.urdf",
),
init_state=ArticulationCfg.InitialStateCfg(
pos=(0.0, 0.0, 0.34),
joint_pos={
".*R_hip_joint": -0.1,
".*L_hip_joint": 0.1,
"F[L,R]_thigh_joint": 0.8,
"R[L,R]_thigh_joint": 1.0,
".*_calf_joint": -1.5,
},
joint_vel={".*": 0.0},
),
actuators={
"GO1": UnitreeActuatorCfg_Go1(
joint_names_expr=[".*"],
stiffness=28.0,
damping=0.7,
friction=0.01,
min_delay=0,
max_delay=4,
),
},
# fmt: off
joint_sdk_names=[
"FR_hip_joint", "FR_thigh_joint", "FR_calf_joint",
"FL_hip_joint", "FL_thigh_joint", "FL_calf_joint",
"RR_hip_joint", "RR_thigh_joint", "RR_calf_joint",
"RL_hip_joint", "RL_thigh_joint", "RL_calf_joint",
],
# fmt: on
)

14
source/robot_lab/robot_lab/assets/unitree_actuator.py
View File

@@ -128,3 +128,17 @@ class UnitreeActuatorCfg_Go2HV(UnitreeActuatorCfg):
Y1 = 20.2
Y2 = 23.4
@configclass
class UnitreeActuatorCfg_Go1(UnitreeActuatorCfg):
"""Go1 actuator using the URDF effort limit.
Public Go1 torque-speed knee measurements are not provided with the asset,
so this configuration intentionally applies a constant 33.5 Nm limit
instead of borrowing the materially different Go2-HV motor curve.
"""
X1 = 1.0e9
X2 = 1.0e9
Y1 = 33.5
Y2 = 33.5

22
source/robot_lab/robot_lab/tasks/go1/__init__.py Normal file
View File

@@ -0,0 +1,22 @@
# Copyright (c) 2024-2025 Ziqi Fan
# SPDX-License-Identifier: Apache-2.0
"""Go1 locomotion task registration."""
import gymnasium as gym
from isaaclab_tasks.utils import import_packages
gym.register(
id="RobotLab-Go1-v0",
entry_point="robot_lab.tasks.go1.env.go1_env:Go1Env",
disable_env_checker=True,
kwargs={
"env_cfg_entry_point": f"{__name__}.env_cfg:Go1EnvCfg",
"rsl_rl_cfg_entry_point": f"{__name__}.rsl_rl_cfg:MoECTSRunnerCfg",
},
)
_BLACKLIST_PKGS = ["utils"]
import_packages(__name__, _BLACKLIST_PKGS)

1
source/robot_lab/robot_lab/tasks/go1/env/__init__.py vendored Normal file
View File

@@ -0,0 +1 @@
"""Go1 environment implementation."""

14
source/robot_lab/robot_lab/tasks/go1/env/go1_env.py vendored Normal file
View File

@@ -0,0 +1,14 @@
from isaaclab.envs import ManagerBasedRLEnv, ManagerBasedRLEnvCfg
from robot_lab.tasks.go2.manager.action_manager import ActionManagerGo2
class Go1Env(ManagerBasedRLEnv):
"""Go1 environment using the shared ordered action manager."""
cfg: ManagerBasedRLEnvCfg
def load_managers(self):
super().load_managers()
self.action_manager = ActionManagerGo2(self.cfg.actions, self)
print("[Go1Env-INFO] Using ordered action manager:", self.action_manager)

509
source/robot_lab/robot_lab/tasks/go1/env_cfg.py Normal file
View File

@@ -0,0 +1,509 @@
import math
import isaaclab.sim as sim_utils
from isaaclab.assets import ArticulationCfg, AssetBaseCfg
from isaaclab.envs import ManagerBasedRLEnvCfg
from isaaclab.managers import CurriculumTermCfg as CurrTerm
from isaaclab.managers import EventTermCfg as EventTerm
from isaaclab.managers import ObservationGroupCfg as ObsGroup
from isaaclab.managers import ObservationTermCfg as ObsTerm
from isaaclab.managers import RewardTermCfg as RewTerm
from isaaclab.managers import SceneEntityCfg
from isaaclab.managers import TerminationTermCfg as DoneTerm
from isaaclab.scene import InteractiveSceneCfg
from isaaclab.sensors import ContactSensorCfg, RayCasterCfg, patterns
from isaaclab.terrains import TerrainImporterCfg
from isaaclab.utils import configclass
from isaaclab.utils.assets import ISAAC_NUCLEUS_DIR, ISAACLAB_NUCLEUS_DIR
from isaaclab.utils.noise import AdditiveUniformNoiseCfg as Unoise
import robot_lab.tasks.go2.mdp as mdp
from robot_lab.assets.unitree import GO1_CFG_UNITREE
from robot_lab.tasks.go2.mdp.terrains import TERRAIN_CFG
JOINT_NAMES = [
"FR_hip_joint", "FR_thigh_joint", "FR_calf_joint",
"FL_hip_joint", "FL_thigh_joint", "FL_calf_joint",
"RR_hip_joint", "RR_thigh_joint", "RR_calf_joint",
"RL_hip_joint", "RL_thigh_joint", "RL_calf_joint",
]
BASE_LINK_NAME = "trunk"
FOOT_LINK_NAME = ".*_foot"
BASE_HEIGHT_TARGET = 0.30 # Go1 nominal trunk height used by the original Gym task.
##
# Scene definition
##
@configclass
class Go1SceneCfg(InteractiveSceneCfg):
"""Configuration for the terrain scene with the Go1 robot."""
terrain = TerrainImporterCfg(
prim_path="/World/ground",
terrain_type="generator",
terrain_generator=TERRAIN_CFG,
max_init_terrain_level=5,
collision_group=-1,
physics_material=sim_utils.RigidBodyMaterialCfg(
friction_combine_mode="average",
restitution_combine_mode="average",
static_friction=1.0,
dynamic_friction=1.0,
restitution=0.0,
),
visual_material=sim_utils.MdlFileCfg(
mdl_path=f"{ISAACLAB_NUCLEUS_DIR}/Materials/TilesMarbleSpiderWhiteBrickBondHoned/TilesMarbleSpiderWhiteBrickBondHoned.mdl",
project_uvw=True,
texture_scale=(0.25, 0.25),
),
debug_vis=False
)
robot: ArticulationCfg = GO1_CFG_UNITREE.replace(prim_path="{ENV_REGEX_NS}/Robot")
height_scanner = RayCasterCfg(
prim_path="{ENV_REGEX_NS}/Robot/trunk",
offset=RayCasterCfg.OffsetCfg(pos=(0.0, 0.0, 20.0)),
ray_alignment="yaw",
pattern_cfg=patterns.GridPatternCfg(resolution=0.1, size=[1.6, 1.0]),
debug_vis=False,
mesh_prim_paths=["/World/ground"],
)
height_scanner_small = RayCasterCfg(
prim_path="{ENV_REGEX_NS}/Robot/trunk",
offset=RayCasterCfg.OffsetCfg(pos=(0.0, 0.0, 20.0)),
ray_alignment="yaw",
pattern_cfg=patterns.GridPatternCfg(resolution=0.1, size=[0.4, 0.3]),
debug_vis=False,
mesh_prim_paths=["/World/ground"],
)
contact_forces = ContactSensorCfg(
prim_path="{ENV_REGEX_NS}/Robot/.*",
history_length=3,
track_air_time=True,
)
# 灯光
sky_light = AssetBaseCfg(
prim_path="/World/skyLight",
spawn=sim_utils.DomeLightCfg(
intensity=750.0,
texture_file=f"{ISAAC_NUCLEUS_DIR}/Materials/Textures/Skies/PolyHaven/kloofendal_43d_clear_puresky_4k.hdr",
),
)
##
# MDP settings
##
@configclass
class CommandsCfg:
"""Command specifications for the MDP."""
base_velocity = mdp.Go2RLGymCommandCfg()
@configclass
class ActionsCfg:
"""Action specifications for the MDP."""
# 腿部关节：位置控制
joint_pos = mdp.JointPositionActionCfg(
asset_name="robot",
joint_names=JOINT_NAMES,
scale={".*_hip_joint": 0.25, "^(?!.*_hip_joint).*": 0.25},
use_default_offset=True,
clip={".*": (-100.0, 100.0)},
preserve_order=True
)
@configclass
class ObservationsCfg:
@configclass
class PolicyCfg(ObsGroup):
"""Observations for policy group."""
base_ang_vel = ObsTerm(
func=mdp.base_ang_vel,
noise=Unoise(n_min=-0.2, n_max=0.2),
clip=(-100.0, 100.0),
scale=0.25,
)
projected_gravity = ObsTerm(
func=mdp.projected_gravity,
noise=Unoise(n_min=-0.05, n_max=0.05),
clip=(-100.0, 100.0),
scale=1.0,
)
velocity_commands = ObsTerm(
func=mdp.generated_commands,
params={"command_name": "base_velocity"},
clip=(-100.0, 100.0),
scale=1.0,
)
joint_pos = ObsTerm(
func=mdp.joint_pos_rel,
params={"asset_cfg": SceneEntityCfg("robot", joint_names=JOINT_NAMES, preserve_order=True)},
noise=Unoise(n_min=-0.03, n_max=0.03),
clip=(-100.0, 100.0),
scale=1.0,
)
joint_vel = ObsTerm(
func=mdp.joint_vel_rel,
params={"asset_cfg": SceneEntityCfg("robot", joint_names=JOINT_NAMES, preserve_order=True)},
noise=Unoise(n_min=-2.0, n_max=2.0),
clip=(-100.0, 100.0),
scale=0.05,
)
actions = ObsTerm(
func=mdp.last_action,
clip=(-100.0, 100.0),
scale=1.0,
)
def __post_init__(self):
self.history_length = 10
self.enable_corruption = True
self.concatenate_terms = True
self.flatten_history_dim = True
@configclass
class CriticCfg(ObsGroup):
base_lin_vel = ObsTerm(
func=mdp.base_lin_vel,
clip=(-100.0, 100.0),
scale=2.0,
)
base_ang_vel = ObsTerm(
func=mdp.base_ang_vel,
clip=(-100.0, 100.0),
scale=0.25,
)
projected_gravity = ObsTerm(
func=mdp.projected_gravity,
clip=(-100.0, 100.0),
scale=1.0,
)
velocity_commands = ObsTerm(
func=mdp.generated_commands,
params={"command_name": "base_velocity"},
clip=(-100.0, 100.0),
scale=1.0,
)
joint_pos = ObsTerm(
func=mdp.joint_pos_rel,
params={"asset_cfg": SceneEntityCfg("robot", joint_names=JOINT_NAMES, preserve_order=True)},
clip=(-100.0, 100.0),
scale=1.0,
)
joint_vel = ObsTerm(
func=mdp.joint_vel_rel,
params={"asset_cfg": SceneEntityCfg("robot", joint_names=JOINT_NAMES, preserve_order=True)},
clip=(-100.0, 100.0),
scale=0.05,
)
actions = ObsTerm(
func=mdp.last_action,
clip=(-100.0, 100.0),
scale=1.0,
)
joint_acc = ObsTerm(
func=mdp.joint_acc,
params={"asset_cfg": SceneEntityCfg("robot", joint_names=JOINT_NAMES, preserve_order=True)},
clip=(-100.0, 100.0),
scale=1e-4,
)
joint_torque = ObsTerm(
func=mdp.joint_effort,
params={"asset_cfg": SceneEntityCfg("robot", joint_names=JOINT_NAMES, preserve_order=True)},
clip=(-100.0, 100.0),
scale=0.01,
)
contact_force = ObsTerm(
func=mdp.foot_contact_force_norm,
params={"sensor_cfg": SceneEntityCfg("contact_forces", body_names=FOOT_LINK_NAME)},
clip=(-100.0, 100.0),
scale=1e-3,
)
height_scan = ObsTerm(
func=mdp.height_scan,
params={"sensor_cfg": SceneEntityCfg("height_scanner")},
clip=(-1.0, 1.0),
scale=2.5,
)
def __post_init__(self):
self.enable_corruption = False
self.concatenate_terms = True
@configclass
class SingleObsCfg(PolicyCfg):
def __post_init__(self):
super().__post_init__()
self.history_length = 1
policy: PolicyCfg = PolicyCfg()
critic: CriticCfg = CriticCfg()
single_obs: SingleObsCfg = SingleObsCfg() # Used to obtain the current-timestep observation for the MoE CTS model
@configclass
class EventCfg:
"""Configuration for events."""
randomize_rigid_body_mass_base = EventTerm(
func=mdp.randomize_rigid_body_mass,
mode="startup",
params={
"asset_cfg": SceneEntityCfg("robot", body_names=BASE_LINK_NAME),
"mass_distribution_params": (-1.0, 1.0),
"operation": "add",
"recompute_inertia": True,
},
)
randomize_rigid_body_mass_others = EventTerm(
func=mdp.randomize_rigid_body_mass,
mode="startup",
params={
"asset_cfg": SceneEntityCfg("robot", body_names="^(?!.*trunk).*"),
"mass_distribution_params": (0.9, 1.1),
"operation": "scale",
"recompute_inertia": True,
},
)
randomize_com_positions = EventTerm(
func=mdp.randomize_rigid_body_com,
mode="startup",
params={
"asset_cfg": SceneEntityCfg("robot", body_names=BASE_LINK_NAME),
"com_range": {"x": (-0.03, 0.03), "y": (-0.03, 0.03), "z": (-0.03, 0.03)},
},
)
reset_robot_joints = EventTerm(
func=mdp.reset_joints_by_scale,
mode="reset",
params={
"position_range": (0.5, 1.5),
"velocity_range": (0.0, 0.0),
},
)
randomize_actuator_gains = EventTerm(
func=mdp.randomize_actuator_gains,
mode="reset",
params={
"asset_cfg": SceneEntityCfg("robot", joint_names=".*"),
"stiffness_distribution_params": (0.9, 1.1),
"damping_distribution_params": (0.9, 1.1),
"operation": "scale",
"distribution": "uniform",
},
)
randomize_motor_zero_offset = EventTerm(
func=mdp.randomize_action_joint_pos_offset,
mode="reset",
params={
"action_term_name": "joint_pos",
"offset_range": (-0.035, 0.035),
},
)
randomize_push_robot = EventTerm(
func=mdp.push_by_setting_velocity,
mode="interval",
interval_range_s=(4.0, 4.0),
params={
"velocity_range": {
"x": (-0.4, 0.4),
"y": (-0.4, 0.4),
"roll": (-0.6, 0.6),
"pitch": (-0.6, 0.6),
"yaw": (-0.6, 0.6)
}
}
)
randomize_rigid_body_material = EventTerm(
func=mdp.randomize_rigid_body_material,
mode="startup",
params={
"asset_cfg": SceneEntityCfg("robot", body_names=".*"),
"static_friction_range": (0.0, 2.0),
"dynamic_friction_range": (0.0, 2.0),
"restitution_range": (0.0, 0.5),
"num_buckets": 64,
"make_consistent": True
},
)
reset_base = EventTerm(
func=mdp.reset_root_state_uniform,
mode="reset",
params={
"pose_range": {"x": (-0.5, 0.5), "y": (-0.5, 0.5), "z": (0.0, 0.2), "yaw": (-3.14, 3.14)},
"velocity_range": {
"x": (-0.5, 0.5),
"y": (-0.5, 0.5),
"z": (-0.5, 0.5),
"roll": (-0.5, 0.5),
"pitch": (-0.5, 0.5),
"yaw": (-0.5, 0.5),
},
},
)
@configclass
class RewardsCfg:
"""Reward terms for the MDP."""
track_lin_vel_xy_exp = RewTerm(
func=mdp.track_lin_vel_xy_exp,
weight=2.0,
params={"command_name": "base_velocity", "std": 0.5}
)
track_ang_vel_z_exp = RewTerm(
func=mdp.track_ang_vel_z_exp,
weight=1.0,
params={"command_name": "base_velocity", "std": 0.5}
)
lin_vel_z_l2 = RewTerm(func=mdp.lin_vel_z_l2, weight=-2.0)
ang_vel_xy_l2 = RewTerm(func=mdp.ang_vel_xy_l2, weight=-0.05)
# The joint_acc reward is not computed on the same scale in Gym and Lab.
# In Gym, it is computed at the policy-step level,
# while in Lab, it is computed at the physics-step level.
# In Lab, the reward calculation is more precise, and because the L2 term is more sensitive to outliers.
# Thus, the reward value is overall higher, so we need to decrease the weights to be suitable for Lab.
joint_acc_l2 = RewTerm(
func=mdp.joint_acc_l2,
weight=-1.0e-7,
params={"asset_cfg": SceneEntityCfg("robot", joint_names=JOINT_NAMES)}
)
joint_power = RewTerm(
func=mdp.joint_power,
weight=-2e-5,
params={"asset_cfg": SceneEntityCfg("robot", joint_names=JOINT_NAMES)}
)
joint_torques_l2 = RewTerm(
func=mdp.joint_torques_l2,
weight=-1e-4,
params={"asset_cfg": SceneEntityCfg("robot", joint_names=JOINT_NAMES)}
)
base_height_l2 = RewTerm(
func=mdp.base_height_l2,
weight=-1.0,
params={
"asset_cfg": SceneEntityCfg("robot", body_names=BASE_LINK_NAME),
"target_height": BASE_HEIGHT_TARGET,
"sensor_cfg": SceneEntityCfg("height_scanner_small"),
}
)
action_rate_l2 = RewTerm(func=mdp.action_rate_l2, weight=-0.01)
action_smoothness_l2 = RewTerm(func=mdp.action_smoothness_l2, weight=-0.01)
undesired_contacts = RewTerm(
func=mdp.undesired_contacts,
weight=-1.0,
params={"sensor_cfg": SceneEntityCfg("contact_forces", body_names=".*_thigh|.*_calf"), "threshold": 5.0},
)
joint_pos_limits = RewTerm(
func=mdp.joint_pos_limits,
weight=-2.0,
params={"asset_cfg": SceneEntityCfg("robot", joint_names=JOINT_NAMES)},
)
feet_regulation = RewTerm(
func=mdp.feet_regulation,
weight=-0.05,
params={
"base_height_target": BASE_HEIGHT_TARGET,
"asset_cfg": SceneEntityCfg("robot", body_names=FOOT_LINK_NAME),
"sensor_cfg": SceneEntityCfg("height_scanner_small"),
},
)
hip_pos_penalty_l1 = RewTerm(
func=mdp.hip_pos_penalty_l1,
weight=-0.05,
params={
"command_name": "base_velocity",
"asset_cfg": SceneEntityCfg("robot", joint_names=".*_hip_joint"),
"stand_still_scale": 1.0,
"command_threshold": 0.1,
},
)
joint_pos_penalty_l1 = RewTerm(
func=mdp.joint_pos_penalty_l1,
weight=-0.01,
params={
"command_name": "base_velocity",
"asset_cfg": SceneEntityCfg("robot", joint_names=".*_(thigh|calf)_joint"),
"stand_still_scale": 1.0,
"velocity_threshold": 0.1,
"command_threshold": 0.1,
},
)
@configclass
class TerminationsCfg:
"""Termination terms for the MDP."""
time_out = DoneTerm(func=mdp.time_out, time_out=True)
illegal_contact = DoneTerm(
func=mdp.illegal_contact,
params={
"sensor_cfg": SceneEntityCfg("contact_forces", body_names=BASE_LINK_NAME),
"threshold": 1.0
},
)
@configclass
class CurriculumCfg:
"""Curriculum terms for the MDP."""
terrain_levels = CurrTerm(func=mdp.terrain_levels_vel_gym)
base_linear_velocity = CurrTerm(mdp.gradual_reward_weight_modification, params={
"term_name": "lin_vel_z_l2", "initial_weight": -2.0, "final_weight": -0.0, "start_it": 0, "end_it": 1500
})
base_height_l2 = CurrTerm(mdp.gradual_reward_weight_modification, params={
"term_name": "base_height_l2", "initial_weight": -1.0, "final_weight": -10.0, "start_it": 0, "end_it": 5000
})
##
# Environment configuration
##
@configclass
class Go1EnvCfg(ManagerBasedRLEnvCfg):
"""Merged configuration for the Go1 robot on rough terrain."""
# Scene settings
scene: Go1SceneCfg = Go1SceneCfg(num_envs=16384, env_spacing=0.5)
# Basic settings
observations: ObservationsCfg = ObservationsCfg()
actions: ActionsCfg = ActionsCfg()
commands: CommandsCfg = CommandsCfg()
# MDP settings
rewards: RewardsCfg = RewardsCfg()
terminations: TerminationsCfg = TerminationsCfg()
events: EventCfg = EventCfg()
curriculum: CurriculumCfg = CurriculumCfg()
def __post_init__(self):
"""Post initialization."""
# General settings
self.decimation = 4
self.episode_length_s = 25.0
# Simulation settings
self.sim.dt = 0.005
self.sim.render_interval = self.decimation
# Physics material settings from subclass
self.sim.physics_material = self.scene.terrain.physics_material
self.sim.physx.gpu_max_rigid_patch_count = int(1 * 1024 * 1024) # 1 million
self.sim.physx.gpu_collision_stack_size = int(512 * 1024 * 1024) # 128 MB
self.sim.physx.enable_external_forces_every_iteration = True
# Update sensor periods
if self.scene.height_scanner is not None:
self.scene.height_scanner.update_period = self.decimation * self.sim.dt
if self.scene.height_scanner_small is not None:
self.scene.height_scanner_small.update_period = self.decimation * self.sim.dt
if self.scene.contact_forces is not None:
self.scene.contact_forces.update_period = self.sim.dt
# Handle curriculum for terrain generator
if getattr(self.curriculum, "terrain_levels", None) is not None:
if self.scene.terrain.terrain_generator is not None:
self.scene.terrain.terrain_generator.curriculum = True
else:
if self.scene.terrain.terrain_generator is not None:
self.scene.terrain.terrain_generator.curriculum = False

86
source/robot_lab/robot_lab/tasks/go1/rsl_rl_cfg.py Normal file
View File

@@ -0,0 +1,86 @@
from isaaclab.utils import configclass
from isaaclab_rl.rsl_rl import RslRlOnPolicyRunnerCfg, RslRlPpoActorCriticCfg, RslRlPpoAlgorithmCfg
@configclass
class PPORunnerCfg(RslRlOnPolicyRunnerCfg):
num_steps_per_env = 24
max_iterations = 300000
save_interval = 500
experiment_name = "go1_rough"
policy = RslRlPpoActorCriticCfg(
init_noise_std=1.0,
actor_obs_normalization=False,
critic_obs_normalization=False,
actor_hidden_dims=[512, 256, 128],
critic_hidden_dims=[512, 256, 128],
activation="elu",
)
algorithm = RslRlPpoAlgorithmCfg(
value_loss_coef=1.0,
use_clipped_value_loss=True,
clip_param=0.2,
entropy_coef=0.01,
num_learning_epochs=5,
num_mini_batches=4,
learning_rate=1.0e-3,
schedule="adaptive",
gamma=0.99,
lam=0.95,
desired_kl=0.01,
max_grad_norm=1.0,
)
@configclass
class RslRlMoeCtsActorCriticCfg(RslRlPpoActorCriticCfg):
class_name = "ActorCriticMoECTS"
init_noise_std = 1.0
expert_num = 8 # number of experts in the student model
latent_dim = 32
norm_type = 'l2norm' # normalization type for encoders: l2norm, simnorm
teacher_encoder_hidden_dims = [512, 256]
student_encoder_hidden_dims = [512, 256, 256]
actor_hidden_dims=[512, 256, 128]
critic_hidden_dims=[512, 256, 128]
activation="elu"
actor_obs_normalization = False
critic_obs_normalization = False
@configclass
class RslRlMoeCtsAlgorithmCfg(RslRlPpoAlgorithmCfg):
class_name = "MoECTS"
value_loss_coef = 1.0
load_balance_coef = 0.01 # coefficient for load balance loss
use_clipped_value_loss = True
clip_param = 0.2
entropy_coef = 0.01
num_learning_epochs = 5
num_mini_batches = 4
learning_rate = 1e-3
student_encoder_learning_rate = 1e-3
schedule = "adaptive"
gamma = 0.99
lam = 0.95
betas = (0.9, 0.999)
weight_decay = 0.0
desired_kl = 0.01
max_grad_norm = 1.0
teacher_env_ratio = 0.75 # percentage of envs assigned to teacher
@configclass
class MoECTSRunnerCfg(RslRlOnPolicyRunnerCfg):
experiment_name = "go1_moe_cts"
class_name = "OnPolicyRunnerCTS"
num_steps_per_env = 24
max_iterations = 300000
save_interval = 500
policy = RslRlMoeCtsActorCriticCfg()
algorithm = RslRlMoeCtsAlgorithmCfg()
# concat elu inspired by concat relu from https://arxiv.org/pdf/2303.07507
@configclass
class MoECTSCatELURunnerCfg(MoECTSRunnerCfg):
def __post_init__(self):
super().__post_init__()
self.policy.activation = 'cat_elu'