seed: 42

# Models are instantiated using skrl's model instantiator utility
# https://skrl.readthedocs.io/en/latest/api/utils/model_instantiators.html
models:
    separate: False
    policy: # gaussian model
        class: "GaussianMixin"
        clip_actions: True
        clip_log_std: True
        initial_log_std: 0.0
        min_log_std: -20.0
        max_log_std: 2.0
        input: "STATES"
        hiddens: [32, 32]
        hidden_activation: ["elu", "elu"]
        output: "ACTIONS"
        output_activation: "tanh"
        output_scale: 1.0
    value: # deterministic model
        class: "DeterministicMixin"
        clip_actions: False
        input: "STATES"
        hiddens: [32, 32]
        hidden_activation: ["elu", "elu"]
        output: "ONE"
        output_activation: ""
        output_scale: 1.0

# Memory
# https://skrl.readthedocs.io/en/latest/api/memories/random.html
memory:
    class: "RandomMemory"
    memory_size: -1 # -1: automatically determined value

# PPO agent configuration (field names are from PPO_DEFAULT_CONFIG)
# https://skrl.readthedocs.io/en/latest/api/agents/ppo.html
agent:
    class: "PPO"
    rollouts: 16
    learning_epochs: 8
    mini_batches: 1
    discount_factor: 0.99
    lambda: 0.95
    learning_rate: 3.e-4
    learning_rate_scheduler: "KLAdaptiveLR"
    learning_rate_scheduler_kwargs:
        kl_threshold: 0.008
    random_timesteps: 0 # random exploration steps
    learning_starts: 0 # learning starts after this many steps
    grad_norm_clip: 1.0
    ratio_clip: 0.2
    value_clip: 0.2
    clip_predicted_values: True
    entropy_loss_scale: 0.0
    value_loss_scale: 2.0
    kl_threshold: 0
    rewards_shaper_scale: 1.0
    time_limit_bootstrap: False
    # logging and checkpoint
    experiment:
        directory: "runs"
        experiment_name: ""
        write_interval: 16
        checkpoint_interval: 80
        store_separately: False
        wandb: False
        wandb_kwargs: null

# Sequential trainer
# https://skrl.readthedocs.io/en/latest/api/trainers/sequential.html
trainer:
    class: "SequentialTrainer"
    timesteps: 1600
    environment_info: "log"