chore: release v0.3.0

motrix_rl/template/rslrl_config.yaml

@@ -0,0 +1,85 @@
+runner:
+    class_name: OnPolicyRunner
+    # General
+    num_steps_per_env: 24 # Number of steps per environment per iteration
+    max_iterations: 1500 # Number of policy updates
+    seed: 1
+    # Observations
+    obs_groups: { "actor": ["policy"], "critic": ["policy", "privileged"] } # Maps from observation sets to groups. See `vec_env.py` for more information
+    # Logging parameters
+    save_interval: 50 # Check for potential saves every `save_interval` iterations
+    experiment_name: walking_experiment
+    run_name: ""
+    # Logging writer
+    logger: tensorboard # tensorboard, neptune, wandb
+    neptune_project: rsl_rl
+    wandb_project: rsl_rl
+
+    # Actor
+    actor:
+        class_name: MLPModel
+        hidden_dims: [256, 256, 256]
+        activation: elu
+        obs_normalization: false
+        stochastic: true
+        init_noise_std: 1.0
+        noise_std_type: "scalar" # 'scalar' or 'log'
+        state_dependent_std: false
+
+    # Critic
+    critic:
+        class_name: MLPModel
+        hidden_dims: [256, 256, 256]
+        activation: elu
+        obs_normalization: false
+        stochastic: false
+
+    # Algorithm
+    algorithm:
+        class_name: PPO
+        # Training
+        optimizer: adam # adam, adamw, sgd, rmsprop
+        learning_rate: 0.001
+        num_learning_epochs: 5
+        num_mini_batches: 4 # mini batch size = num_envs * num_steps / num_mini_batches
+        schedule: adaptive # adaptive, fixed
+        # Value function
+        value_loss_coef: 1.0
+        clip_param: 0.2
+        use_clipped_value_loss: true
+        # Surrogate loss
+        desired_kl: 0.01
+        entropy_coef: 0.01
+        gamma: 0.99
+        lam: 0.95
+        max_grad_norm: 1.0
+        # Miscellaneous
+        normalize_advantage_per_mini_batch: false
+
+        # Random network distillation
+        rnd_cfg:
+            weight: 0.0 # Initial weight of the RND reward
+            weight_schedule: null # This is a dictionary with a required key called "mode". Please check the RND module for more information
+            reward_normalization: false # Whether to normalize RND reward
+            # Learning parameters
+            learning_rate: 0.001 # Learning rate for RND
+            # Network parameters
+            num_outputs: 1 # Number of outputs of RND network. Note: if -1, then the network will use dimensions of the observation
+            predictor_hidden_dims: [-1] # Hidden dimensions of predictor network
+            target_hidden_dims: [-1] # Hidden dimensions of target network
+
+        # Symmetry augmentation
+        symmetry_cfg:
+            use_data_augmentation: true # This adds symmetric trajectories to the batch
+            use_mirror_loss: false # This adds symmetry loss term to the loss function
+            data_augmentation_func: null # String containing the module and function name to import
+            # Example: "legged_gym.envs.locomotion.anymal_c.symmetry:get_symmetric_states"
+            #
+            # .. code-block:: python
+            #
+            #     @torch.no_grad()
+            #     def get_symmetric_states(
+            #        env: VecEnv, obs: Optional[torch.Tensor] = None, actions: Optional[torch.Tensor] = None,
+            #     ) -> Tuple[torch.Tensor, torch.Tensor]:
+            #
+            mirror_loss_coeff: 0.0 # Coefficient for symmetry loss term. If 0, no symmetry loss is used

motrix_rl/template/skrl_config.yaml

@@ -0,0 +1,75 @@
+seed: 42
+
+# Models are instantiated using skrl's model instantiator utility
+# https://skrl.readthedocs.io/en/latest/api/utils/model_instantiators.html
+models:
+    separate: False
+    policy: # gaussian model
+        class: "GaussianMixin"
+        clip_actions: True
+        clip_log_std: True
+        initial_log_std: 0.0
+        min_log_std: -20.0
+        max_log_std: 2.0
+        input: "STATES"
+        hiddens: [32, 32]
+        hidden_activation: ["elu", "elu"]
+        output: "ACTIONS"
+        output_activation: "tanh"
+        output_scale: 1.0
+    value: # deterministic model
+        class: "DeterministicMixin"
+        clip_actions: False
+        input: "STATES"
+        hiddens: [32, 32]
+        hidden_activation: ["elu", "elu"]
+        output: "ONE"
+        output_activation: ""
+        output_scale: 1.0
+
+# Memory
+# https://skrl.readthedocs.io/en/latest/api/memories/random.html
+memory:
+    class: "RandomMemory"
+    memory_size: -1 # -1: automatically determined value
+
+# PPO agent configuration (field names are from PPO_DEFAULT_CONFIG)
+# https://skrl.readthedocs.io/en/latest/api/agents/ppo.html
+agent:
+    class: "PPO"
+    rollouts: 16
+    learning_epochs: 8
+    mini_batches: 1
+    discount_factor: 0.99
+    lambda: 0.95
+    learning_rate: 3.e-4
+    learning_rate_scheduler: "KLAdaptiveLR"
+    learning_rate_scheduler_kwargs:
+        kl_threshold: 0.008
+    random_timesteps: 0 # random exploration steps
+    learning_starts: 0 # learning starts after this many steps
+    grad_norm_clip: 1.0
+    ratio_clip: 0.2
+    value_clip: 0.2
+    clip_predicted_values: True
+    entropy_loss_scale: 0.0
+    value_loss_scale: 2.0
+    kl_threshold: 0
+    rewards_shaper_scale: 1.0
+    time_limit_bootstrap: False
+    # logging and checkpoint
+    experiment:
+        directory: "runs"
+        experiment_name: ""
+        write_interval: 16
+        checkpoint_interval: 80
+        store_separately: False
+        wandb: False
+        wandb_kwargs: null
+
+# Sequential trainer
+# https://skrl.readthedocs.io/en/latest/api/trainers/sequential.html
+trainer:
+    class: "SequentialTrainer"
+    timesteps: 1600
+    environment_info: "log"