chore: release v0.0.1
This commit is contained in:
motphys-developers
@@ -0,0 +1,6 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<svg id="Layer_1" data-name="Layer 1" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 100 100">
|
||||
<polygon points="44.76 37.22 26.34 37.22 22.32 42.33 48.75 42.33 44.76 37.22"/>
|
||||
<path d="m37.49,27.94c-.98-1.25-2.87-1.25-3.86,0l-3.28,4.18h10.4l-3.26-4.17Z"/>
|
||||
<path d="m98.68,69.04l-32.16-41.1c-.98-1.25-2.87-1.25-3.86,0l-12.65,16.01-2.77,3.5h-28.95l-4.02,5.11h29.03l-4.02,5.11H10.26l-4.02,5.11h29.03l-4.02,5.11H2.22l-.9,1.15c-1.26,1.61-.12,3.96,1.93,3.96h93.5c2.04,0,3.19-2.35,1.93-3.96Z"/>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 539 B |
13
docs/source/_static/Motphys_Logo_only_White_100x100px.svg
Normal file
13
docs/source/_static/Motphys_Logo_only_White_100x100px.svg
Normal file
@@ -0,0 +1,13 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<svg id="Layer_1" data-name="Layer 1" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 100 100">
|
||||
<defs>
|
||||
<style>
|
||||
.cls-1 {
|
||||
fill: #fff;
|
||||
}
|
||||
</style>
|
||||
</defs>
|
||||
<polygon class="cls-1" points="44.76 37.22 26.34 37.22 22.32 42.33 48.75 42.33 44.76 37.22"/>
|
||||
<path class="cls-1" d="m37.49,27.94c-.98-1.25-2.87-1.25-3.86,0l-3.28,4.18h10.4l-3.26-4.17Z"/>
|
||||
<path class="cls-1" d="m98.68,69.04l-32.16-41.1c-.98-1.25-2.87-1.25-3.86,0l-12.65,16.01-2.77,3.5h-28.95l-4.02,5.11h29.03l-4.02,5.11H10.26l-4.02,5.11h29.03l-4.02,5.11H2.22l-.9,1.15c-1.26,1.61-.12,3.96,1.93,3.96h93.5c2.04,0,3.19-2.35,1.93-3.96Z"/>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 668 B |
17
docs/source/_static/Motphys_logo_Black.svg
Normal file
17
docs/source/_static/Motphys_logo_Black.svg
Normal file
@@ -0,0 +1,17 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<svg id="Layer_1" data-name="Layer 1" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 800 450">
|
||||
<g>
|
||||
<path d="m363.5,253.29h-13.56s0-45.66,0-45.66l-17.49,31.46h-9.11l-17.8-31.77v45.99h-13.56v-71.86h13.56l22.36,40.88,22.05-40.88h13.56v71.85Z"/>
|
||||
<path d="m372.51,229.05v-6.42c0-8,2.17-14.09,6.52-18.27,4.35-4.17,10.35-6.26,18.01-6.26s13.54,2.1,17.85,6.31,6.47,10.28,6.47,18.22v6.42c0,8.07-2.16,14.21-6.47,18.42-4.31,4.21-10.26,6.31-17.85,6.31s-13.66-2.1-18.01-6.31c-4.35-4.21-6.52-10.35-6.52-18.42Zm36.12-6.21c0-4.55-1-7.92-3-10.09-2-2.17-4.86-3.26-8.59-3.26-7.87,0-11.8,4.45-11.8,13.35v6.11c0,8.9,3.93,13.35,11.8,13.35s11.59-4.45,11.59-13.35v-6.11Z"/>
|
||||
<path d="m445.89,235.47c0,2.35.5,4.02,1.5,5.02,1,1,2.67,1.5,5.02,1.5h4.55v10.87h-6.93c-5.45,0-9.63-1.43-12.52-4.29-2.9-2.86-4.35-6.95-4.35-12.27v-28.25h-9.42v-9h9.42v-10.56l12.73-4.45v15.01h12.21v9h-12.21v27.43Z"/>
|
||||
<path d="m479.63,199.04v6.11c4-4.69,9.11-7.04,15.32-7.04s11.52,2.07,15.32,6.21,5.69,10.11,5.69,17.91v6.73c0,5.18-.91,9.64-2.74,13.4-1.83,3.76-4.33,6.61-7.5,8.54-3.17,1.93-6.8,2.9-10.87,2.9-6.42,0-11.38-2.14-14.9-6.42v21.63h-12.83v-69.97h12.52Zm23.49,29.39v-5.59c0-8.9-3.83-13.35-11.49-13.35-3.66,0-6.52,1.16-8.59,3.47-2.07,2.31-3.1,5.54-3.1,9.68v6.83c0,3.93,1.07,7.06,3.21,9.37,2.14,2.31,5.04,3.47,8.69,3.47s6.45-1.16,8.38-3.47c1.93-2.31,2.9-5.78,2.9-10.4Z"/>
|
||||
<path d="m537.48,181.44v23.29c1.86-2.07,4.14-3.64,6.83-4.71,2.69-1.07,5.59-1.6,8.69-1.6,6.21,0,11.04,1.83,14.49,5.49s5.17,8.8,5.17,15.42v33.53h-12.83v-30.95c0-8.07-3.55-12.11-10.66-12.11-3.66,0-6.52,1.05-8.59,3.16-2.07,2.1-3.11,5.26-3.11,9.47v30.43h-12.83v-71.42h12.83Z"/>
|
||||
<path d="m606.52,257.52c-1.52,3.93-3.76,6.8-6.73,8.59-2.97,1.79-6.83,2.69-11.59,2.69h-4.97v-11.49h5.59c2.48,0,4.29-.33,5.43-.98,1.14-.66,2.05-1.88,2.74-3.67l.31-.93-21.94-52.68h13.87l13.97,36.02,12.83-36.02h13.35l-22.87,58.48Z"/>
|
||||
<path d="m631.56,236.3v-.31h12.01v.21c0,2.14,1,3.83,3,5.07s4.97,1.86,8.9,1.86c3.66,0,6.43-.52,8.33-1.55s2.85-2.55,2.85-4.55c0-1.66-.67-2.93-2.02-3.83-1.34-.9-3.54-1.52-6.57-1.86l-7.97-.93c-6.07-.76-10.54-2.36-13.4-4.81-2.86-2.45-4.3-5.95-4.3-10.51,0-5.31,2-9.42,6-12.32,4-2.9,9.49-4.35,16.46-4.35s12.8,1.45,16.87,4.35c4.07,2.9,6.11,6.9,6.11,12.01v.41h-11.9v-.31c0-2.07-.93-3.66-2.79-4.76-1.86-1.1-4.66-1.66-8.38-1.66-3.52,0-6.12.5-7.81,1.5-1.69,1-2.54,2.43-2.54,4.3,0,1.52.59,2.73,1.76,3.62,1.17.9,3.14,1.48,5.9,1.76l8.59,1.04c6.21.69,10.8,2.33,13.77,4.92,2.97,2.59,4.45,6.12,4.45,10.61,0,5.59-2.16,9.87-6.47,12.83-4.31,2.97-10.19,4.45-17.65,4.45s-13.01-1.5-17.08-4.5c-4.07-3-6.11-7.23-6.11-12.68Z"/>
|
||||
</g>
|
||||
<g>
|
||||
<polygon points="189.56 197.29 160.73 197.29 154.44 205.29 195.82 205.29 189.56 197.29"/>
|
||||
<path d="m178.19,182.76c-1.53-1.96-4.5-1.96-6.04,0l-5.14,6.54h16.28l-5.11-6.53Z"/>
|
||||
<path d="m273.97,247.09l-50.33-64.33c-1.53-1.96-4.5-1.96-6.04,0l-19.8,25.06-4.34,5.47h-45.31l-6.29,8h45.44l-6.29,8h-45.43l-6.29,8h45.44l-6.29,8h-45.44l-1.41,1.79c-1.98,2.52-.19,6.21,3.01,6.21h146.35c3.2,0,4.99-3.68,3.02-6.2Z"/>
|
||||
</g>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 3.0 KiB |
24
docs/source/_static/Motphys_logo_White.svg
Normal file
24
docs/source/_static/Motphys_logo_White.svg
Normal file
@@ -0,0 +1,24 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<svg id="Layer_1" data-name="Layer 1" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 800 450">
|
||||
<defs>
|
||||
<style>
|
||||
.cls-1 {
|
||||
fill: #fff;
|
||||
}
|
||||
</style>
|
||||
</defs>
|
||||
<g>
|
||||
<path class="cls-1" d="m363.5,253.29h-13.56s0-45.66,0-45.66l-17.49,31.46h-9.11l-17.8-31.77v45.99h-13.56v-71.86h13.56l22.36,40.88,22.05-40.88h13.56v71.85Z"/>
|
||||
<path class="cls-1" d="m372.51,229.05v-6.42c0-8,2.17-14.09,6.52-18.27,4.35-4.17,10.35-6.26,18.01-6.26s13.54,2.1,17.85,6.31,6.47,10.28,6.47,18.22v6.42c0,8.07-2.16,14.21-6.47,18.42-4.31,4.21-10.26,6.31-17.85,6.31s-13.66-2.1-18.01-6.31c-4.35-4.21-6.52-10.35-6.52-18.42Zm36.12-6.21c0-4.55-1-7.92-3-10.09-2-2.17-4.86-3.26-8.59-3.26-7.87,0-11.8,4.45-11.8,13.35v6.11c0,8.9,3.93,13.35,11.8,13.35s11.59-4.45,11.59-13.35v-6.11Z"/>
|
||||
<path class="cls-1" d="m445.89,235.47c0,2.35.5,4.02,1.5,5.02,1,1,2.67,1.5,5.02,1.5h4.55v10.87h-6.93c-5.45,0-9.63-1.43-12.52-4.29-2.9-2.86-4.35-6.95-4.35-12.27v-28.25h-9.42v-9h9.42v-10.56l12.73-4.45v15.01h12.21v9h-12.21v27.43Z"/>
|
||||
<path class="cls-1" d="m479.63,199.04v6.11c4-4.69,9.11-7.04,15.32-7.04s11.52,2.07,15.32,6.21,5.69,10.11,5.69,17.91v6.73c0,5.18-.91,9.64-2.74,13.4-1.83,3.76-4.33,6.61-7.5,8.54-3.17,1.93-6.8,2.9-10.87,2.9-6.42,0-11.38-2.14-14.9-6.42v21.63h-12.83v-69.97h12.52Zm23.49,29.39v-5.59c0-8.9-3.83-13.35-11.49-13.35-3.66,0-6.52,1.16-8.59,3.47-2.07,2.31-3.1,5.54-3.1,9.68v6.83c0,3.93,1.07,7.06,3.21,9.37,2.14,2.31,5.04,3.47,8.69,3.47s6.45-1.16,8.38-3.47c1.93-2.31,2.9-5.78,2.9-10.4Z"/>
|
||||
<path class="cls-1" d="m537.48,181.44v23.29c1.86-2.07,4.14-3.64,6.83-4.71,2.69-1.07,5.59-1.6,8.69-1.6,6.21,0,11.04,1.83,14.49,5.49s5.17,8.8,5.17,15.42v33.53h-12.83v-30.95c0-8.07-3.55-12.11-10.66-12.11-3.66,0-6.52,1.05-8.59,3.16-2.07,2.1-3.11,5.26-3.11,9.47v30.43h-12.83v-71.42h12.83Z"/>
|
||||
<path class="cls-1" d="m606.52,257.52c-1.52,3.93-3.76,6.8-6.73,8.59-2.97,1.79-6.83,2.69-11.59,2.69h-4.97v-11.49h5.59c2.48,0,4.29-.33,5.43-.98,1.14-.66,2.05-1.88,2.74-3.67l.31-.93-21.94-52.68h13.87l13.97,36.02,12.83-36.02h13.35l-22.87,58.48Z"/>
|
||||
<path class="cls-1" d="m631.56,236.3v-.31h12.01v.21c0,2.14,1,3.83,3,5.07s4.97,1.86,8.9,1.86c3.66,0,6.43-.52,8.33-1.55s2.85-2.55,2.85-4.55c0-1.66-.67-2.93-2.02-3.83-1.34-.9-3.54-1.52-6.57-1.86l-7.97-.93c-6.07-.76-10.54-2.36-13.4-4.81-2.86-2.45-4.3-5.95-4.3-10.51,0-5.31,2-9.42,6-12.32,4-2.9,9.49-4.35,16.46-4.35s12.8,1.45,16.87,4.35c4.07,2.9,6.11,6.9,6.11,12.01v.41h-11.9v-.31c0-2.07-.93-3.66-2.79-4.76-1.86-1.1-4.66-1.66-8.38-1.66-3.52,0-6.12.5-7.81,1.5-1.69,1-2.54,2.43-2.54,4.3,0,1.52.59,2.73,1.76,3.62,1.17.9,3.14,1.48,5.9,1.76l8.59,1.04c6.21.69,10.8,2.33,13.77,4.92,2.97,2.59,4.45,6.12,4.45,10.61,0,5.59-2.16,9.87-6.47,12.83-4.31,2.97-10.19,4.45-17.65,4.45s-13.01-1.5-17.08-4.5c-4.07-3-6.11-7.23-6.11-12.68Z"/>
|
||||
</g>
|
||||
<g>
|
||||
<polygon class="cls-1" points="189.56 197.29 160.73 197.29 154.44 205.29 195.82 205.29 189.56 197.29"/>
|
||||
<path class="cls-1" d="m178.19,182.76c-1.53-1.96-4.5-1.96-6.04,0l-5.14,6.54h16.28l-5.11-6.53Z"/>
|
||||
<path class="cls-1" d="m273.97,247.09l-50.33-64.33c-1.53-1.96-4.5-1.96-6.04,0l-19.8,25.06-4.34,5.47h-45.31l-6.29,8h45.44l-6.29,8h-45.43l-6.29,8h45.44l-6.29,8h-45.44l-1.41,1.79c-1.98,2.52-.19,6.21,3.01,6.21h146.35c3.2,0,4.99-3.68,3.02-6.2Z"/>
|
||||
</g>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 3.2 KiB |
16
docs/source/_static/css/custom.css
Normal file
16
docs/source/_static/css/custom.css
Normal file
@@ -0,0 +1,16 @@
|
||||
/* Custom CSS for MotrixSim Documentation */
|
||||
|
||||
/* Logo adjustments for PyData Sphinx Theme */
|
||||
.navbar-brand img {
|
||||
max-height: 130px !important;
|
||||
height: 130px !important;
|
||||
width: auto !important;
|
||||
}
|
||||
|
||||
.bd-main .bd-content .bd-article-container {
|
||||
max-width: 80em; /* default is 60em */
|
||||
}
|
||||
|
||||
.bd-page-width {
|
||||
max-width: 100rem; /* default is 88rem */
|
||||
}
|
||||
3
docs/source/_static/images/poster/cartpole.jpg
Normal file
3
docs/source/_static/images/poster/cartpole.jpg
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:10b78648c8a4ded8c56292377d5a0823a1c8e9cc5f8764fa8082f6a3341a41ad
|
||||
size 176434
|
||||
3
docs/source/_static/images/poster/dm_walker.jpg
Normal file
3
docs/source/_static/images/poster/dm_walker.jpg
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:987c2ac4e86e1fe6634a16b3ea5fb1a633f07b41d39918c6a23df4da88d5b45c
|
||||
size 140437
|
||||
3
docs/source/_static/images/poster/go1_walk.jpg
Normal file
3
docs/source/_static/images/poster/go1_walk.jpg
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:194d9a0d40b04d9d41aff936503a1fc8a4d95e1bb3ba28e968996cc29793060a
|
||||
size 161837
|
||||
14
docs/source/_static/js/custom.js
Normal file
14
docs/source/_static/js/custom.js
Normal file
@@ -0,0 +1,14 @@
|
||||
function openVideoControls(video) {
|
||||
video.controls = true;
|
||||
}
|
||||
|
||||
function isWeChatMobile() {
|
||||
const ua = navigator.userAgent.toLowerCase();
|
||||
return /micromessenger/.test(ua) && /android|iphone|ipad|ipod/.test(ua);
|
||||
}
|
||||
|
||||
if (isWeChatMobile()) {
|
||||
document.addEventListener("DOMContentLoaded", function () {
|
||||
document.querySelectorAll("video").forEach(openVideoControls);
|
||||
});
|
||||
}
|
||||
3
docs/source/_static/videos/cartpole.mp4
Normal file
3
docs/source/_static/videos/cartpole.mp4
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:723e272be6aff6e0c10f72f871b5674c509a3fa9b6b0ca26c0671b4b67d343ee
|
||||
size 1816175
|
||||
3
docs/source/_static/videos/dm_walker.mp4
Normal file
3
docs/source/_static/videos/dm_walker.mp4
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:e877e592a3fcf323ca37aeced5e0d5383a128d82958dc3044816020c64e25d01
|
||||
size 1628617
|
||||
3
docs/source/_static/videos/go1_walk.mp4
Normal file
3
docs/source/_static/videos/go1_walk.mp4
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:540a9fa3591f456c509f8c3383dc2dbe2106cdf75a7415d49307d74a9388105f
|
||||
size 4770163
|
||||
183
docs/source/conf.py
Normal file
183
docs/source/conf.py
Normal file
@@ -0,0 +1,183 @@
|
||||
# Copyright (C) 2020-2025 Motphys Technology Co., Ltd. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ==============================================================================
|
||||
|
||||
# Configuration file for the Sphinx documentation builder.
|
||||
#
|
||||
# For the full list of built-in configuration values, see the documentation:
|
||||
# https://www.sphinx-doc.org/en/master/usage/configuration.html
|
||||
|
||||
import os
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
|
||||
__version__ = "0.1.0"
|
||||
|
||||
# -- Project information -----------------------------------------------------
|
||||
# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information
|
||||
|
||||
project = "Motrixlab"
|
||||
copyright = "2025, Motphys"
|
||||
author = "Motphys"
|
||||
release = __version__
|
||||
version = __version__
|
||||
language = "zh_CN"
|
||||
|
||||
|
||||
def setup(app):
|
||||
app.connect("builder-inited", copy_file_by_language)
|
||||
|
||||
|
||||
def copy_file_by_language(app):
|
||||
"""根据语言配置复制语言目录"""
|
||||
lang = app.config.language
|
||||
srcdir = Path(app.srcdir)
|
||||
|
||||
lang_dir = srcdir / lang
|
||||
if lang_dir.exists() and lang_dir.is_dir():
|
||||
shutil.copytree(lang_dir, srcdir, dirs_exist_ok=True, copy_function=shutil.copy2)
|
||||
print(f"Copied language directory: {lang_dir} -> {srcdir}")
|
||||
else:
|
||||
print(f"[WARNING] Language directory not found: {lang_dir}")
|
||||
|
||||
|
||||
# -- General configuration ---------------------------------------------------
|
||||
# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration
|
||||
|
||||
extensions = [
|
||||
# Include documentation from docstrings
|
||||
"sphinx.ext.autodoc",
|
||||
# Support for NumPy and Google style docstrings
|
||||
"sphinx.ext.napoleon",
|
||||
# Generate autodoc summaries
|
||||
"sphinx.ext.autosummary",
|
||||
# Extending your autodoc API docs with a summary
|
||||
"autodocsumm",
|
||||
"sphinx.ext.mathjax",
|
||||
"sphinx.ext.viewcode",
|
||||
"sphinx.ext.intersphinx",
|
||||
"sphinx_copybutton",
|
||||
"myst_parser",
|
||||
"sphinx_subfigure",
|
||||
"sphinxcontrib.video",
|
||||
"sphinx_togglebutton",
|
||||
"sphinx_design",
|
||||
]
|
||||
|
||||
# https://sphinx-design.readthedocs.io/en/pydata-theme/get_started.html#usage
|
||||
myst_enable_extensions = ["colon_fence", "deflist"]
|
||||
|
||||
### Autodoc configurations ###
|
||||
# put type hints inside the signature instead of the description (easier to maintain)
|
||||
autodoc_typehints = "both"
|
||||
# Define the order in which automodule and autoclass members are listed
|
||||
autodoc_member_order = "groupwise"
|
||||
# default autodoc settings
|
||||
autodoc_default_options = {
|
||||
"autosummary": True,
|
||||
}
|
||||
|
||||
autodoc_typehints_description_target = "all"
|
||||
autodoc_default_flags = ["members", "show-inheritance", "undoc-members"]
|
||||
|
||||
|
||||
# generate autosummary even if no references
|
||||
autosummary_generate = True
|
||||
autosummary_generate_overwrite = False
|
||||
|
||||
# Add any paths that contain templates here, relative to this directory.
|
||||
templates_path = ["_templates"]
|
||||
# List of patterns, relative to source directory, that match files and
|
||||
# directories to ignore when looking for source files.
|
||||
# This pattern also affects html_static_path and html_extra_path.
|
||||
exclude_patterns = ["index_*.md", "api_reference/low**", "zh_CN/**", "en/**"]
|
||||
|
||||
# https://myst-parser.readthedocs.io/en/latest/syntax/optional.html
|
||||
myst_enable_extensions = ["colon_fence", "dollarmath"]
|
||||
# https://github.com/executablebooks/MyST-Parser/issues/519#issuecomment-1037239655
|
||||
myst_heading_anchors = 4
|
||||
|
||||
# -- Options for HTML output -------------------------------------------------
|
||||
# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output
|
||||
html_title = "Motrixlab Documentation"
|
||||
html_theme = "pydata_sphinx_theme"
|
||||
html_static_path = ["_static"]
|
||||
html_logo = "_static/Motphys_logo_Black.svg"
|
||||
html_favicon = "_static/Motphys_Logo_only_Black_100x100px.svg"
|
||||
html_show_copyright = True
|
||||
html_show_sphinx = False
|
||||
html_last_updated_fmt = "" # to reveal the build date in the pages meta
|
||||
|
||||
html_theme_options = {
|
||||
# https://pydata-sphinx-theme.readthedocs.io/en/stable/user_guide/navigation.html
|
||||
"show_nav_level": 2,
|
||||
# https://pydata-sphinx-theme.readthedocs.io/en/stable/user_guide/source-buttons.html#add-an-edit-button
|
||||
"use_edit_page_button": False,
|
||||
# https://pydata-sphinx-theme.readthedocs.io/en/stable/user_guide/indices.html#add-indices-links
|
||||
"primary_sidebar_end": ["indices.html"],
|
||||
"logo": {
|
||||
"image_light": "_static/Motphys_logo_Black.svg",
|
||||
"image_dark": "_static/Motphys_logo_White.svg",
|
||||
},
|
||||
# https://pydata-sphinx-theme.readthedocs.io/en/stable/user_guide/layout.html#configure-the-navbar-center-alignment
|
||||
"navbar_start": ["navbar-logo"],
|
||||
"navbar_center": ["navbar-nav"], # "version-switcher"
|
||||
"navbar_persistent": ["search-field.html"],
|
||||
"navbar_end": ["navbar-icon-links", "theme-switcher"],
|
||||
# https://pydata-sphinx-theme.readthedocs.io/en/stable/user_guide/layout.html#configure-the-navbar-center-alignment
|
||||
"navbar_align": "content",
|
||||
# https://pydata-sphinx-theme.readthedocs.io/en/stable/user_guide/page-toc.html#per-page-secondary-sidebar-content
|
||||
"secondary_sidebar_items": ["page-toc", "sidebar-ethical-ads"],
|
||||
# https://pydata-sphinx-theme.readthedocs.io/en/stable/user_guide/layout.html#footer-content
|
||||
"show_prev_next": True,
|
||||
# https://pydata-sphinx-theme.readthedocs.io/en/stable/user_guide/announcements.html#version-warning-banners
|
||||
"show_version_warning_banner": False,
|
||||
# https://pydata-sphinx-theme.readthedocs.io/en/stable/user_guide/layout.html#footer-content
|
||||
"footer_start": ["copyright", "sphinx-version"],
|
||||
"footer_end": ["theme-version"],
|
||||
# https://pydata-sphinx-theme.readthedocs.io/en/stable/user_guide/header-links.html#navigation-bar-external-links
|
||||
"external_links": [
|
||||
{"name": "Issues", "url": "https://github.com/Motphys/MotrixLab/issues"},
|
||||
{"name": "Discussions", "url": "https://github.com/Motphys/MotrixLab/discussions"},
|
||||
],
|
||||
"header_links_before_dropdown": 2,
|
||||
# https://pydata-sphinx-theme.readthedocs.io/en/stable/user_guide/header-links.html#icon-links
|
||||
"icon_links": [
|
||||
{
|
||||
# Label for this link
|
||||
"name": "GitHub",
|
||||
# URL where the link will redirect
|
||||
"url": "https://github.com/Motphys/MotrixLab", # required
|
||||
# Icon class (if "type": "fontawesome"), or path to local image (if "type": "local")
|
||||
"icon": "fa-brands fa-github",
|
||||
"type": "fontawesome",
|
||||
},
|
||||
{
|
||||
"name": "About Motphys",
|
||||
"url": "https://www.motphys.com",
|
||||
"icon": "fa-solid fa-building",
|
||||
"type": "fontawesome",
|
||||
},
|
||||
],
|
||||
}
|
||||
project_version = os.environ.get("READTHEDOCS_VERSION")
|
||||
if project_version is None:
|
||||
project_version = __version__
|
||||
html_context = {}
|
||||
html_css_files = [
|
||||
"css/custom.css",
|
||||
]
|
||||
html_js_files = [
|
||||
"js/custom.js",
|
||||
]
|
||||
66
docs/source/en/index.md
Normal file
66
docs/source/en/index.md
Normal file
@@ -0,0 +1,66 @@
|
||||
# MotrixLab
|
||||
|
||||
MotrixLab is a general-purpose machine learning architecture designed for robot training. Built on the MotrixSim simulation platform, it enables policy training on both CPU and GPU, providing flexible and easy-to-use training capabilities for users across various operating systems and hardware devices.
|
||||
|
||||
::::{grid} 1 2 3 3
|
||||
:gutter: 2 2 2 2
|
||||
|
||||
:::{grid-item-card}
|
||||
|
||||
```{video} _static/videos/cartpole.mp4
|
||||
:poster: _static/images/poster/cartpole.jpg
|
||||
:nocontrols:
|
||||
:autoplay:
|
||||
:playsinline:
|
||||
:muted:
|
||||
:loop:
|
||||
:width: 100%
|
||||
```
|
||||
|
||||
:::
|
||||
|
||||
:::{grid-item-card}
|
||||
|
||||
```{video} _static/videos/go1_walk.mp4
|
||||
:poster: _static/images/poster/go1_walk.jpg
|
||||
:nocontrols:
|
||||
:autoplay:
|
||||
:playsinline:
|
||||
:muted:
|
||||
:loop:
|
||||
:width: 100%
|
||||
```
|
||||
|
||||
:::
|
||||
|
||||
:::{grid-item-card}
|
||||
|
||||
```{video} _static/videos/dm_walker.mp4
|
||||
:poster: _static/images/poster/dm_walker.jpg
|
||||
:nocontrols:
|
||||
:autoplay:
|
||||
:playsinline:
|
||||
:muted:
|
||||
:loop:
|
||||
:width: 100%
|
||||
```
|
||||
|
||||
:::
|
||||
::::
|
||||
|
||||
## Key Features
|
||||
|
||||
- **Cross-Platform Software Support**: Compatible with Windows and Linux operating systems
|
||||
- **CPU Simulation Support**: Supports CPU-based simulation, significantly reducing GPU requirements
|
||||
|
||||
## Use Cases
|
||||
|
||||
- Robot control algorithm development and testing
|
||||
- Building reinforcement learning environments
|
||||
- Education and research
|
||||
|
||||
```{toctree}
|
||||
:maxdepth: 1
|
||||
|
||||
user_guide/index
|
||||
```
|
||||
112
docs/source/en/user_guide/demo/cartpole.md
Normal file
112
docs/source/en/user_guide/demo/cartpole.md
Normal file
@@ -0,0 +1,112 @@
|
||||
# CartPole Training Example
|
||||
|
||||
CartPole is a classic control task in reinforcement learning. The goal is to keep the pole balanced by controlling the cart's left-right movement.
|
||||
|
||||
|
||||
## Task Description
|
||||
|
||||
- **State Space**: Cart position, cart velocity, pole angle, pole angular velocity
|
||||
- **Action Space**: Apply force left or right
|
||||
- **Reward Function**: +1 reward for each step the pole stays upright
|
||||
- **Termination Conditions**: Pole angle exceeds ±15 degrees or episode length exceeds 10 seconds
|
||||
|
||||
## Quick Start
|
||||
|
||||
### 1. Environment Preview
|
||||
|
||||
```bash
|
||||
uv run scripts/view.py --env cartpole
|
||||
```
|
||||
|
||||
### 2. Start Training
|
||||
|
||||
```bash
|
||||
# Train with default parameters
|
||||
uv run scripts/train.py --env cartpole
|
||||
|
||||
# Customize number of environments
|
||||
uv run scripts/train.py --env cartpole --num-envs 1024
|
||||
|
||||
# Enable rendering (visualize during training)
|
||||
uv run scripts/train.py --env cartpole --render
|
||||
```
|
||||
|
||||
### 3. View Training Progress
|
||||
|
||||
```bash
|
||||
uv run tensorboard --logdir runs/cartpole
|
||||
```
|
||||
|
||||
### 4. Test Training Results
|
||||
|
||||
```bash
|
||||
# Automatically find best policy for testing (recommended)
|
||||
uv run scripts/play.py --env cartpole
|
||||
|
||||
# Manually specify policy file for testing
|
||||
uv run scripts/play.py --env cartpole --policy runs/cartpole/nn/best_policy.pickle
|
||||
```
|
||||
|
||||
> **Tip**: The system will automatically find the latest and best policy files in the `runs/cartpole/` directory for testing. You can also manually specify specific policy files using the `--policy` parameter.
|
||||
|
||||
## Configuration Parameters
|
||||
|
||||
Main configuration parameters for the CartPole environment:
|
||||
|
||||
```python
|
||||
@dataclass
|
||||
class CartPoleEnvCfg(EnvCfg):
|
||||
model_file: str = "path/to/inverted_pendulum.xml" # MJCF model file
|
||||
reset_noise_scale: float = 0.01 # Reset noise
|
||||
max_episode_seconds: float = 10.0 # Maximum episode length
|
||||
```
|
||||
|
||||
Training configuration parameters:
|
||||
|
||||
```python
|
||||
from dataclasses import dataclass
|
||||
from motrix_rl.skrl.cfg import PPOCfg
|
||||
from motrix_rl import registry
|
||||
|
||||
@registry.rlcfg("cartpole")
|
||||
@dataclass
|
||||
class CartPolePPO(PPOCfg):
|
||||
max_env_steps: int = 10_000_000 # Maximum environment steps
|
||||
check_point_interval: int = 500 # Checkpoint interval
|
||||
|
||||
# Network structure (small network suitable for simple tasks)
|
||||
policy_hidden_layer_sizes: tuple[int, ...] = (32, 32)
|
||||
value_hidden_layer_sizes: tuple[int, ...] = (32, 32)
|
||||
|
||||
# PPO parameters
|
||||
rollouts: int = 32 # Experience replay rounds
|
||||
learning_epochs: int = 5 # Training rounds
|
||||
mini_batches: int = 4 # Number of mini-batches
|
||||
```
|
||||
|
||||
**Note**: CartPole is a simple task and currently uses universal configuration. If you need to create specialized configurations for different training backends (JAX/Torch), refer to the environment configuration documentation examples.
|
||||
|
||||
## Custom Training
|
||||
|
||||
You can override default configurations through command line arguments:
|
||||
|
||||
```bash
|
||||
uv run scripts/train.py --env cartpole \
|
||||
--num-envs 1024 \
|
||||
--train-backend jax \
|
||||
--sim-backend np
|
||||
```
|
||||
|
||||
## Expected Results
|
||||
|
||||
- Pole angle stays within ±5 degrees most of the time
|
||||
- Cart displacement range is reasonable
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
If training performance is poor, you can try:
|
||||
|
||||
1. Adjust learning rate (try 1e-4 to 1e-3)
|
||||
2. Increase number of environments (more parallel training)
|
||||
3. Adjust reward function weights
|
||||
4. Check if physical parameters are reasonable
|
||||
153
docs/source/en/user_guide/demo/dm_walker.md
Normal file
153
docs/source/en/user_guide/demo/dm_walker.md
Normal file
@@ -0,0 +1,153 @@
|
||||
# 2D Walker Robot Training Example
|
||||
|
||||
The 2D Walker Robot (Walker2D) is a classic robot control task from DeepMind Control Suite. The goal is to achieve standing, walking, and running by controlling the robot's joints.
|
||||
|
||||
```{video} /_static/videos/dm_walker.mp4
|
||||
:poster: _static/images/poster/dm_walker.jpg
|
||||
:nocontrols:
|
||||
:autoplay:
|
||||
:playsinline:
|
||||
:muted:
|
||||
:loop:
|
||||
:width: 100%
|
||||
```
|
||||
|
||||
## Task Description
|
||||
|
||||
Walker2D is a 2D planar bipedal robot with multiple joints and actuators:
|
||||
|
||||
- **State Space**: Includes rotation angles and angular velocities of various robot parts, torso height and velocity, etc.
|
||||
- **Action Space**: Control torques for each joint
|
||||
- **Reward Function**: Mainly composed of maintaining standing balance and forward speed
|
||||
- **Termination Conditions**: Robot falls or joints reach limit positions
|
||||
|
||||
### Three Task Modes
|
||||
|
||||
1. **dm-stander**: Static standing task (move_speed = 0.0)
|
||||
|
||||
```bash
|
||||
uv run scripts/train.py --env dm-stander
|
||||
```
|
||||
|
||||
2. **dm-walker**: Walking task (move_speed = 1.0)
|
||||
|
||||
```bash
|
||||
uv run scripts/train.py --env dm-walker
|
||||
```
|
||||
|
||||
3. **dm-runner**: Running task (move_speed = 5.0)
|
||||
|
||||
```bash
|
||||
uv run scripts/train.py --env dm-runner
|
||||
```
|
||||
|
||||
## Quick Start
|
||||
|
||||
### 1. Environment Preview
|
||||
|
||||
```bash
|
||||
# View standing task
|
||||
uv run scripts/view.py --env dm-stander
|
||||
|
||||
# View walking task
|
||||
uv run scripts/view.py --env dm-walker
|
||||
|
||||
# View running task
|
||||
uv run scripts/view.py --env dm-runner
|
||||
```
|
||||
|
||||
### 2. Start Training
|
||||
|
||||
```bash
|
||||
# Train standing task
|
||||
uv run scripts/train.py --env dm-stander
|
||||
|
||||
# Train walking task (default)
|
||||
uv run scripts/train.py --env dm-walker
|
||||
|
||||
# Train running task
|
||||
uv run scripts/train.py --env dm-runner
|
||||
|
||||
# Customize number of environments
|
||||
uv run scripts/train.py --env dm-walker --num-envs 512
|
||||
|
||||
# Enable rendering (visualize during training)
|
||||
uv run scripts/train.py --env dm-walker --render
|
||||
```
|
||||
|
||||
### 3. View Training Progress
|
||||
|
||||
```bash
|
||||
uv run tensorboard --logdir runs/dm-walker
|
||||
```
|
||||
|
||||
### 4. Test Training Results
|
||||
|
||||
```bash
|
||||
# Automatically find best policy for testing (recommended)
|
||||
uv run scripts/play.py --env dm-walker
|
||||
|
||||
# Manually specify policy file for testing
|
||||
uv run scripts/play.py --env dm-walker --policy runs/dm-walker/nn/best_policy.pickle
|
||||
```
|
||||
|
||||
> **Tip**: The system will automatically find the latest and best policy files in the `runs/dm-walker/` directory for testing. Supports dm-stander, dm-walker, dm-runner three task modes.
|
||||
|
||||
## Configuration Parameters
|
||||
|
||||
### Environment Configuration
|
||||
|
||||
```python
|
||||
@dataclass
|
||||
class WalkerEnvCfg(EnvCfg):
|
||||
model_file: str = "walker.xml" # MJCF model file
|
||||
max_episode_seconds: float = 25.0 # Maximum episode length
|
||||
sim_dt: float = 0.0125 # Simulation time step
|
||||
ctrl_dt: float = 0.025 # Control time step
|
||||
move_speed: float = 1.0 # Target movement speed
|
||||
stand_height: float = 1.2 # Target standing height
|
||||
```
|
||||
|
||||
### Training Configuration
|
||||
|
||||
```python
|
||||
@dataclass
|
||||
class WalkerRLCfg(BaseRLCfg):
|
||||
num_envs: int = 512 # Number of parallel environments
|
||||
learning_rate: float = 3e-4 # Learning rate
|
||||
batch_size: int = 512 # Batch size
|
||||
max_epochs: int = 1000 # Maximum training epochs
|
||||
```
|
||||
|
||||
## Reward Function Design
|
||||
|
||||
Walker2D's reward function consists of the following components:
|
||||
|
||||
### Basic Standing Reward
|
||||
|
||||
```python
|
||||
# Height reward: keep torso at target height
|
||||
|
||||
# Upright reward: keep torso upright
|
||||
```
|
||||
|
||||
### Movement Reward (walking and running tasks)
|
||||
|
||||
```python
|
||||
# Speed reward: track target speed
|
||||
|
||||
# Total reward = standing reward * movement weight
|
||||
```
|
||||
|
||||
## Expected Results
|
||||
|
||||
1. **dm-stander**:
|
||||
|
||||
- Torso height maintained in 1.0-1.4m range
|
||||
|
||||
2. **dm-walker**:
|
||||
|
||||
- Actual walking speed close to 1.0 m/s
|
||||
|
||||
3. **dm-runner**:
|
||||
- Running speed reaches 4.0-5.0 m/s
|
||||
169
docs/source/en/user_guide/demo/locomotion_unitree_go1.md
Normal file
169
docs/source/en/user_guide/demo/locomotion_unitree_go1.md
Normal file
@@ -0,0 +1,169 @@
|
||||
# Unitree GO1 Robot Walking Training Example
|
||||
|
||||
Unitree GO1 is a quadruped robot platform. This example demonstrates how to train GO1 to achieve stable gait walking on flat terrain.
|
||||
|
||||
```{video} /_static/videos/go1_walk.mp4
|
||||
:poster: _static/images/poster/go1_walk.jpg
|
||||
:nocontrols:
|
||||
:autoplay:
|
||||
:playsinline:
|
||||
:muted:
|
||||
:loop:
|
||||
:width: 100%
|
||||
```
|
||||
|
||||
## Task Description
|
||||
|
||||
The GO1 quadruped robot has 12 degrees of freedom (3 joints per leg) and needs to learn coordinated gait control through deep reinforcement learning:
|
||||
|
||||
- **State Space**: 48-dimensional, including robot linear velocity, angular velocity, posture, joint angles, joint velocities, actions, and commands
|
||||
- **Action Space**: 12-dimensional, controlling target positions of each joint (converted to torques through PD controller)
|
||||
- **Reward Function**: Composite reward including speed tracking, posture stability, energy efficiency, and other components
|
||||
- **Termination Conditions**: Robot trunk contacts ground or other unstable states
|
||||
|
||||
### Training Task
|
||||
|
||||
```bash
|
||||
uv run scripts/train.py --env go1-flat-terrain-walk
|
||||
```
|
||||
|
||||
## Configuration Parameters
|
||||
|
||||
### Environment Configuration
|
||||
|
||||
```python
|
||||
@dataclass
|
||||
class Go1WalkNpEnvCfg(EnvCfg):
|
||||
max_episode_seconds: float = 20.0 # Maximum episode length
|
||||
model_file: str = "scene_motor_actuator.xml"
|
||||
sim_dt: float = 0.01 # Simulation time step
|
||||
ctrl_dt: float = 0.01 # Control time step
|
||||
```
|
||||
|
||||
### Training Configuration
|
||||
|
||||
```python
|
||||
from dataclasses import dataclass
|
||||
from motrix_rl.skrl.cfg import PPOCfg
|
||||
from motrix_rl import registry
|
||||
|
||||
@registry.rlcfg("go1-flat-terrain-walk")
|
||||
@dataclass
|
||||
class Go1WalkPPO(PPOCfg):
|
||||
"""
|
||||
GO1 quadruped robot walking training configuration
|
||||
"""
|
||||
|
||||
seed = 42
|
||||
max_env_steps: int = 40960000 # Maximum training steps
|
||||
num_envs: int = 2048 # Number of parallel environments
|
||||
|
||||
# Large network structure (suitable for complex robot control tasks)
|
||||
policy_hidden_layer_sizes: tuple[int, ...] = (512, 256, 128)
|
||||
value_hidden_layer_sizes: tuple[int, ...] = (512, 256, 128)
|
||||
|
||||
# PPO parameters (optimized for robot tasks)
|
||||
learning_epochs: int = 2 # Training rounds
|
||||
mini_batches: int = 32 # Number of mini-batches
|
||||
learning_rate: float = 1e-3 # Learning rate
|
||||
```
|
||||
|
||||
**Note**: GO1 is a complex task that uses large network structures. If you need to create specialized configurations for different training backends (JAX/Torch), refer to the environment configuration documentation examples.
|
||||
|
||||
### Control Configuration
|
||||
|
||||
```python
|
||||
@dataclass
|
||||
class ControlConfig:
|
||||
stiffness = 80 # PD controller stiffness [N*m/rad]
|
||||
damping = 1 # PD controller damping [N*m*s/rad]
|
||||
action_scale = 0.1 # Action scaling factor
|
||||
```
|
||||
|
||||
### Initial Joint Angles
|
||||
|
||||
```python
|
||||
default_joint_angles = {
|
||||
"FL_hip": 0.0, # Front left hip joint
|
||||
"RL_hip": 0.0, # Rear left hip joint
|
||||
"FR_hip": -0.0, # Front right hip joint
|
||||
"RR_hip": -0.0, # Rear right hip joint
|
||||
"FL_thigh": 0.9, # Front left thigh
|
||||
"RL_thigh": 0.9, # Rear left thigh
|
||||
"FR_thigh": 0.9, # Front right thigh
|
||||
"RR_thigh": 0.9, # Rear right thigh
|
||||
"FL_calf": -1.8, # Front left calf
|
||||
"RL_calf": -1.8, # Rear left calf
|
||||
"FR_calf": -1.8, # Front right calf
|
||||
"RR_calf": -1.8, # Rear right calf
|
||||
}
|
||||
```
|
||||
|
||||
## Reward Function Design
|
||||
|
||||
GO1's reward function is a complex composite function containing multiple components:
|
||||
|
||||
### Main Reward Components
|
||||
|
||||
```python
|
||||
reward_config.scales = {
|
||||
"tracking_lin_vel": 1.0, # Linear velocity tracking reward
|
||||
"tracking_ang_vel": 0.5, # Angular velocity tracking reward
|
||||
"feet_air_time": 1.0, # Foot air time reward
|
||||
"lin_vel_z": -2.0, # Z-axis linear velocity penalty
|
||||
"ang_vel_xy": -0.05, # XY-axis angular velocity penalty
|
||||
"orientation": -0.0, # Posture deviation penalty
|
||||
"torques": -0.00001, # Torque consumption penalty
|
||||
"dof_acc": -2.5e-7, # Joint acceleration penalty
|
||||
"action_rate": -0.001, # Action change rate penalty
|
||||
"hip_pos": -1, # Hip joint position penalty
|
||||
"calf_pos": -0.3, # Calf joint position penalty
|
||||
}
|
||||
```
|
||||
|
||||
### Key Reward Functions
|
||||
|
||||
#### Velocity Tracking Reward
|
||||
|
||||
```python
|
||||
# Track linear velocity commands (xy plane)
|
||||
def _reward_tracking_lin_vel(self, data, commands):
|
||||
|
||||
# Track angular velocity commands (yaw)
|
||||
def _reward_tracking_ang_vel(self, data, commands):
|
||||
```
|
||||
|
||||
#### Foot Air Time Reward
|
||||
|
||||
```python
|
||||
def _reward_feet_air_time(self, commands, info):
|
||||
```
|
||||
|
||||
## Observation Space Composition
|
||||
|
||||
GO1's observation space is 48-dimensional, containing the following information:
|
||||
|
||||
```python
|
||||
obs = np.hstack([
|
||||
noisy_linvel, # 3D: Local coordinate system linear velocity
|
||||
noisy_gyro, # 3D: Gyroscope data
|
||||
local_gravity, # 3D: Local gravity direction
|
||||
noisy_joint_angle, # 12D: Joint angles (relative to default values)
|
||||
noisy_joint_vel, # 12D: Joint velocities
|
||||
last_actions, # 12D: Previous frame actions
|
||||
command, # 3D: Velocity commands [vx, vy, vyaw]
|
||||
])
|
||||
```
|
||||
|
||||
## Motion Velocity Command Generation
|
||||
|
||||
Random velocity commands are generated during training to ensure the agent can track different movement speeds:
|
||||
|
||||
```python
|
||||
def resample_commands(self, num_envs: int):
|
||||
```
|
||||
|
||||
## Expected Training Results
|
||||
|
||||
1. Stable quadruped gait
|
||||
2. Good speed tracking
|
||||
84
docs/source/en/user_guide/getting_started/hello_motrixlab.md
Normal file
84
docs/source/en/user_guide/getting_started/hello_motrixlab.md
Normal file
@@ -0,0 +1,84 @@
|
||||
# Quick Start: Hello MotrixLab
|
||||
|
||||
This tutorial demonstrates the MotrixLab workflow through a simple example - loading and training a cartpole environment:
|
||||
|
||||
## Environment Preview
|
||||
|
||||
We provide a simple script to visualize an environment without executing any training. This helps you verify that system dependencies are correctly configured:
|
||||
|
||||
```bash
|
||||
uv run scripts/view.py --env cartpole
|
||||
```
|
||||
|
||||
This will open a visualization window showing the cartpole physics simulation environment with random actions for demonstration.
|
||||
|
||||
## Train Model
|
||||
|
||||
Start training the cartpole balancing task:
|
||||
|
||||
```bash
|
||||
uv run scripts/train.py --env cartpole
|
||||
```
|
||||
|
||||
The training process will automatically:
|
||||
|
||||
1. Automatically select training backend (JAX or PyTorch) based on hardware environment
|
||||
2. Create training environments
|
||||
3. Start PPO algorithm training
|
||||
|
||||
Training results will be saved in the `runs/cartpole/` directory, including:
|
||||
|
||||
- Training checkpoints
|
||||
- TensorBoard log files
|
||||
|
||||
## Visualize Training Process
|
||||
|
||||
If you want to observe the model's learning process during training, you can enable visualization rendering:
|
||||
|
||||
```bash
|
||||
uv run scripts/train.py --env cartpole --render
|
||||
```
|
||||
|
||||
### 🎮 Interactive Rendering Control
|
||||
|
||||
> **Important Note**: Visualization significantly reduces training speed and is recommended mainly for debugging and demonstration purposes.
|
||||
|
||||
During visualized training, you can use the **spacebar** to dynamically control rendering:
|
||||
|
||||
- **Enable Rendering**: Press spacebar to enable visualization and observe robot behavior
|
||||
- **Disable Rendering**: Press spacebar again to disable rendering and improve training speed
|
||||
- **Switch Anytime**: No need to restart the program; you can switch at any time during training
|
||||
|
||||
This interactive control allows you to observe training effects when needed and enjoy fast training when not needed. This feature also works during inference.
|
||||
|
||||
## View Training Results
|
||||
|
||||
Use TensorBoard to view training progress:
|
||||
|
||||
```bash
|
||||
uv run tensorboard --logdir runs/cartpole
|
||||
```
|
||||
|
||||
## Test Trained Model
|
||||
|
||||
After training is complete, test the trained policy:
|
||||
|
||||
```bash
|
||||
# Automatically find best policy for testing (recommended)
|
||||
uv run scripts/play.py --env cartpole
|
||||
|
||||
# Manually specify policy file for testing (if you need a specific version)
|
||||
uv run scripts/play.py --env cartpole --policy runs/cartpole/YOUR_RESULT_NUMBER/best_agent.pickle
|
||||
```
|
||||
|
||||
> **Tip**: The system will automatically find the latest and best policy files in the `runs/cartpole/` directory. Usually, using the auto-discovery feature is sufficient.
|
||||
|
||||
## That Completes Our Example
|
||||
|
||||
Next, you can try modifying parameters to observe physical effects under different settings, or try other environments.
|
||||
|
||||
## Next Steps
|
||||
|
||||
- Learn about the [Basic Framework](../tutorial/basic_frame.md)
|
||||
- Study [Physics Environment Configuration](../tutorial/physics_environment.md)
|
||||
- View more [Training Examples](../demo/cartpole.md)
|
||||
55
docs/source/en/user_guide/getting_started/installation.md
Normal file
55
docs/source/en/user_guide/getting_started/installation.md
Normal file
@@ -0,0 +1,55 @@
|
||||
# Installation Environment
|
||||
|
||||
## Installation Requirements
|
||||
|
||||
- **Python Version**: {bdg-danger-line}`3.10.*`
|
||||
|
||||
| Python Version | Support Status |
|
||||
| :------------: | :------------: |
|
||||
| ≤ 3.9 | ❌ |
|
||||
| 3.10 | ✅ |
|
||||
| ≥ 3.11 | ❌ |
|
||||
|
||||
- **Package Manager**: {bdg-danger-line}`UV`
|
||||
|
||||
- **System and Architecture**:
|
||||
|
||||
- {bdg-danger-line}`Windows(x86_64)`
|
||||
- {bdg-danger-line}`Linux(x86_64)`
|
||||
|
||||
```{note}
|
||||
Features supported on each platform:
|
||||
|
||||
| Operating System | CPU Simulation | Interactive Viewer | GPU Simulation |
|
||||
| :--------------: | :------------: | :----------------: | :------------: |
|
||||
| Linux | ✅ | ✅ | 🛠️ In Development |
|
||||
| Windows | ✅ | ✅ | 🛠️ In Development |
|
||||
```
|
||||
|
||||
## Installation Method
|
||||
|
||||
### Clone Project
|
||||
|
||||
```bash
|
||||
git clone https://github.com/Motphys/MotrixLab.git
|
||||
cd MotrixLab
|
||||
```
|
||||
|
||||
### Install Dependencies
|
||||
|
||||
Use UV to install project dependencies:
|
||||
|
||||
```bash
|
||||
# Install all dependencies
|
||||
uv sync --all-packages --all-extras
|
||||
```
|
||||
|
||||
If you only need to install one training backend, you can choose to install a specific backend type:
|
||||
|
||||
```bash
|
||||
# Install SKRL JAX (support Linux only)
|
||||
uv sync --all-packages --extra skrl-jax
|
||||
|
||||
# Install SKRL PyTorch
|
||||
uv sync --all-packages --extra skrl-torch
|
||||
```
|
||||
32
docs/source/en/user_guide/index.md
Normal file
32
docs/source/en/user_guide/index.md
Normal file
@@ -0,0 +1,32 @@
|
||||
# User Guide
|
||||
|
||||
```{toctree}
|
||||
:caption: Getting Started
|
||||
:maxdepth: 1
|
||||
|
||||
getting_started/installation
|
||||
getting_started/hello_motrixlab
|
||||
|
||||
```
|
||||
|
||||
```{toctree}
|
||||
:caption: Tutorials
|
||||
:maxdepth: 1
|
||||
|
||||
tutorial/basic_frame
|
||||
tutorial/physics_environment
|
||||
tutorial/training_environment_config
|
||||
tutorial/rewards
|
||||
tutorial/training_and_result
|
||||
|
||||
```
|
||||
|
||||
```{toctree}
|
||||
:caption: Training Examples
|
||||
:maxdepth: 1
|
||||
|
||||
demo/cartpole
|
||||
demo/dm_walker
|
||||
demo/locomotion_unitree_go1
|
||||
|
||||
```
|
||||
149
docs/source/en/user_guide/tutorial/basic_frame.md
Normal file
149
docs/source/en/user_guide/tutorial/basic_frame.md
Normal file
@@ -0,0 +1,149 @@
|
||||
# Basic Framework
|
||||
|
||||
MotrixLab is a robot reinforcement learning platform. This section introduces MotrixLab's framework design and the relationships between various components. If you are already familiar with reinforcement learning content, you can skip directly to the next section to learn how to develop your own training environments.
|
||||
|
||||
## MotrixLab's Framework Design
|
||||
|
||||
MotrixLab adopts a layered architecture design, clearly separating training environments from training logic:
|
||||
|
||||
```
|
||||
MotrixLab/
|
||||
├── motrix_envs/ # Environment layer: Physics simulation and task definition
|
||||
│ ├── basic/ # Basic environments (cartpole, walker, etc.)
|
||||
│ ├── locomotion/ # Locomotion environments (GO1 robot, etc.)
|
||||
│ ├── np/ # NumPy simulation backend framework
|
||||
│ ├── base.py # Environment base class
|
||||
│ └── registry.py # Environment registry system
|
||||
├── motrix_rl/ # Training layer: RL algorithms and configuration
|
||||
│ ├── skrl/ # SKRL framework integration (JAX/PyTorch)
|
||||
│ ├── base.py # RL configuration base class
|
||||
│ └── registry.py # RL configuration registry system
|
||||
└── scripts
|
||||
├── train.py # Training entry script
|
||||
├── play.py # Testing entry script
|
||||
└── view.py # Visualization script
|
||||
```
|
||||
|
||||
## Core Component Architecture
|
||||
|
||||
```
|
||||
┌─────────────────────────────────────────────────────────────────┐
|
||||
│ User Interface Layer │
|
||||
│ train.py │ play.py │ view.py │
|
||||
└─────────────────────────────────────────────────────────────────┘
|
||||
│
|
||||
▼
|
||||
┌─────────────────────────────────────────────────────────────────┐
|
||||
│ Training Algorithm Layer (SKRL) │
|
||||
│ PPO Trainer │ Network Architecture │ Optimizer │
|
||||
└─────────────────────────────────────────────────────────────────┘
|
||||
│
|
||||
▼
|
||||
┌─────────────────────────────────────────────────────────────────┐
|
||||
│ Environment Implementation Layer │
|
||||
│ Environment Config(EnvCfg) │ Environment Impl(Env) │ Reward │
|
||||
└─────────────────────────────────────────────────────────────────┘
|
||||
│
|
||||
▼
|
||||
┌─────────────────────────────────────────────────────────────────┐
|
||||
│ Physics Simulation Layer (MotrixSim) │
|
||||
│ MJCF Model │ Physics Engine │ Collision │
|
||||
└─────────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
## Detailed Core Components
|
||||
|
||||
### 1. Training Environment
|
||||
|
||||
**Location**: Environment Implementation Layer
|
||||
|
||||
The training environment is the core component of MotrixLab, containing three key parts:
|
||||
|
||||
- **Environment Configuration (EnvCfg)**: Defines physics simulation parameters (model files, time steps, episode length, etc.) and task-specific parameters
|
||||
- **Environment Implementation (Env)**: Inherits from base environment class, implements specific task logic, physics simulation interaction, and termination condition checking
|
||||
- **Reward Function (Reward)**: Implemented in the environment's step method, calculates reward values based on current state and actions
|
||||
|
||||
Environments are registered to the system through decorators.
|
||||
|
||||
### 2. Reward Function
|
||||
|
||||
**Location**: Configuration Management Layer + Environment Implementation Layer
|
||||
|
||||
The reward function in MotrixLab adopts a dual-structure design:
|
||||
|
||||
- **Configuration Level**: Define reward weights, reward component types, and scaling parameters in configuration classes
|
||||
- **Implementation Level**: Calculate specific reward values in the environment's `_compute_reward` method based on configuration parameters
|
||||
|
||||
This design allows reward functions to be flexibly adjusted through configuration files while implementing complex computational logic in code.
|
||||
|
||||
### 3. Configuration Parameters
|
||||
|
||||
**Location**: Configuration Management Layer
|
||||
|
||||
Configuration parameters adopt a hierarchical management structure:
|
||||
|
||||
- **Environment Configuration (EnvCfg)**: Controls physics simulation and task behavior, including simulation parameters, reset noise, time limits, etc.
|
||||
- **Training Configuration (RLCfg)**: Controls reinforcement learning algorithms, including network structure, learning rate, batch size, training steps, etc.
|
||||
|
||||
Configuration classes support inheritance, parameter validation, and runtime overriding, ensuring parameter reasonableness and flexibility.
|
||||
|
||||
### 4. Registry System
|
||||
|
||||
**Location**: Hub connecting various components
|
||||
|
||||
The registry system implements automatic component registration through the decorator pattern:
|
||||
|
||||
- Environment configuration classes are registered through `@registry.envcfg()`
|
||||
- Environment implementation classes are registered through `@registry.env()`, supporting multiple backends
|
||||
- RL configuration classes are registered through `@registry.rlcfg()`
|
||||
|
||||
The registry system achieves component decoupling, making it simple and fast to add new environments or modify configurations.
|
||||
|
||||
## Data Flow and Workflow
|
||||
|
||||
### Training Process Overview
|
||||
|
||||
```
|
||||
User Command → Configuration Parsing → Environment Creation → Training Loop → Model Save
|
||||
↓
|
||||
train.py --env cartpole
|
||||
↓
|
||||
Find Configuration Classes → Create Environment → Start PPO Training → Save Model
|
||||
```
|
||||
|
||||
### Core Workflow
|
||||
|
||||
1. **Environment Definition**: Create environment configuration classes and implementation classes in `src/motrix_envs/`
|
||||
2. **Automatic Registration**: Register components to the system through decorators
|
||||
3. **Configuration Loading**: When starting from command line, the system automatically finds and loads corresponding configurations
|
||||
4. **Environment Creation**: Factory pattern creates environment instances, supporting parameter override
|
||||
5. **Training Execution**: PPO algorithm interacts with the environment, collects data and updates policy
|
||||
6. **Result Saving**: Periodically save checkpoints and final models
|
||||
|
||||
### Role of Configuration Parameters
|
||||
|
||||
Configuration parameters play a key connecting role throughout the process:
|
||||
|
||||
- **Environment Configuration** determines physics simulation behavior (time steps, model files, noise, etc.)
|
||||
- **Reward Configuration** affects learning signals (reward weights, calculation methods, etc.)
|
||||
- **Training Configuration** controls algorithm behavior (network structure, learning rate, batch size, etc.)
|
||||
|
||||
## Multi-Backend Support
|
||||
|
||||
MotrixLab's layered design naturally supports multiple backends:
|
||||
|
||||
- **Simulation Backends**: MotrixSim (CPU)
|
||||
- **Training Backends**: JAX and PyTorch, supporting GPU acceleration
|
||||
- **Algorithm Framework**: Mainly integrates SKRL, easy to extend to other algorithms
|
||||
|
||||
## Design Advantages
|
||||
|
||||
This architecture design brings the following core advantages:
|
||||
|
||||
1. **Module Decoupling**: Environment development and training logic are completely separated
|
||||
2. **Flexible Configuration**: Supports hierarchical configuration and runtime parameter override
|
||||
3. **Strong Extensibility**: Easily add new components through the registry system
|
||||
4. **Multi-Backend Compatibility**: Same environment can use different simulation and training backends
|
||||
5. **Experiment-Friendly**: Configurations can be saved and compared, ensuring experimental reproducibility
|
||||
|
||||
Through this framework design, MotrixLab provides a clear, flexible, and easy-to-use development platform for robot reinforcement learning.
|
||||
66
docs/source/en/user_guide/tutorial/physics_environment.md
Normal file
66
docs/source/en/user_guide/tutorial/physics_environment.md
Normal file
@@ -0,0 +1,66 @@
|
||||
# Physics Environment Configuration
|
||||
|
||||
Physics environment configuration defines simulation parameters and model file settings in reinforcement learning training.
|
||||
MotrixLab uses [MotrixSim](https://motrixsim.readthedocs.io/en/latest/user_guide/index.html) as the physics simulation backend.
|
||||
|
||||
## Supported File Formats
|
||||
|
||||
- [**MJCF**](https://mujoco.readthedocs.io/en/stable/XMLreference.html) (MuJoCo XML format) - Provides rich physics features and simulation configuration
|
||||
|
||||
## Model File Configuration
|
||||
|
||||
You need to specify model file paths in environment configuration classes:
|
||||
|
||||
```python
|
||||
|
||||
@registry.envcfg("my-task")
|
||||
@dataclass
|
||||
class MyTaskEnvCfg(EnvCfg):
|
||||
# Model file path (required)
|
||||
model_file: str = "my_model.xml"
|
||||
|
||||
# Simulation time parameters
|
||||
sim_dt: float = 0.002 # Simulation time step
|
||||
ctrl_dt: float = 0.02 # Control update frequency
|
||||
|
||||
# Episode parameters
|
||||
max_episode_seconds: float = 20.0
|
||||
reset_noise_scale: float = 0.01
|
||||
```
|
||||
|
||||
### Recommended Directory Structure
|
||||
|
||||
```
|
||||
motrix_envs/my_task/
|
||||
├── __init__.py # Module initialization
|
||||
├── cfg.py # Environment configuration
|
||||
├── my_model.xml # Physics model file
|
||||
└── my_env.py # Environment implementation
|
||||
```
|
||||
|
||||
For complex models with many referenced files, it's recommended to use folder management.
|
||||
|
||||
## Common Configuration Issues
|
||||
|
||||
### File Path Issues
|
||||
|
||||
- When using relative paths, ensure paths are relative to the configuration file location
|
||||
- Avoid using hardcoded absolute paths
|
||||
- Check file permissions and accessibility
|
||||
- Ensure all referenced sub-files exist
|
||||
|
||||
### Time Step Settings
|
||||
|
||||
- `ctrl_dt` should be an integer multiple of `sim_dt`
|
||||
- `sim_dt` that is too small will affect simulation performance
|
||||
- `ctrl_dt` that is too large will affect control precision
|
||||
- Recommend `sim_dt` between 0.001-0.02 seconds
|
||||
|
||||
### Simulation Stability
|
||||
|
||||
- Avoid excessively large time steps
|
||||
- Set contact parameters reasonably to avoid penetration
|
||||
- Mass and inertia distribution should be reasonable
|
||||
- Joint limits should match actual conditions
|
||||
|
||||
Through proper physics environment configuration, you can create accurate and efficient simulation environments for reinforcement learning training.
|
||||
50
docs/source/en/user_guide/tutorial/rewards.md
Normal file
50
docs/source/en/user_guide/tutorial/rewards.md
Normal file
@@ -0,0 +1,50 @@
|
||||
# Reward Function Design
|
||||
|
||||
The reward function tells the agent what behaviors are desired and is a core part of reinforcement learning environment design.
|
||||
|
||||
## Position of Reward Function in Training Loop
|
||||
|
||||
In MotrixLab's NpEnv, reward calculation occurs in the `update_state` phase of the `step` function:
|
||||
|
||||
```python
|
||||
# Execution flow of NpEnv.step()
|
||||
def step(self, actions: np.ndarray) -> NpEnvState:
|
||||
# 1. Preparation phase: Clear rewards and state
|
||||
self._prev_physics_step() # reward = 0.0, terminated = False, truncated = False
|
||||
|
||||
# 2. Apply actions
|
||||
self._state = self.apply_action(actions, self._state)
|
||||
|
||||
# 3. Physics simulation
|
||||
self.physics_step() # Execute physics simulation
|
||||
|
||||
# 4. Update state ← Reward function is calculated here
|
||||
self._state = self.update_state(self._state) # Calculate rewards and observations
|
||||
|
||||
# 5. Post-processing
|
||||
self._update_truncate() # Check time truncation
|
||||
self._reset_done_envs() # Reset completed environments
|
||||
|
||||
return self._state
|
||||
```
|
||||
|
||||
You need to implement reward calculation logic in the `update_state` method of subclasses. For specific reward function design ideas, please refer to the training examples.
|
||||
|
||||
### Reward Component Design Principles
|
||||
|
||||
1. **Separation of Concerns**: Each reward function should handle a specific goal
|
||||
2. **Weight Configuration**: Manage weights of different components through configuration files
|
||||
3. **Normalization**: Keep reward values within reasonable ranges
|
||||
4. **Smoothness**: Avoid hard thresholds, use exponential functions for smooth transitions
|
||||
|
||||
This approach makes reward functions modular, facilitating debugging and adjustment of individual component weights.
|
||||
|
||||
## Design Principles
|
||||
|
||||
1. **Clear Goal Orientation**: Reward functions should directly reflect task goals
|
||||
2. **Reasonable Reward Range**: Avoid overly large or small reward values to maintain training stability
|
||||
3. **Balance Exploration and Exploitation**: Appropriately reward behaviors close to goals, avoiding sparse rewards
|
||||
4. **Avoid Reward Hacking**: Check if agents can obtain high rewards through unintended means
|
||||
5. **Debug-Friendly**: Output reward decomposition information during development for optimization
|
||||
|
||||
By correctly implementing reward calculation in the `update_state` method, you can design effective learning signals for various robot tasks.
|
||||
92
docs/source/en/user_guide/tutorial/training_and_result.md
Normal file
92
docs/source/en/user_guide/tutorial/training_and_result.md
Normal file
@@ -0,0 +1,92 @@
|
||||
# Training Execution and Result Analysis
|
||||
|
||||
This section introduces how to execute reinforcement learning training and how to analyze and use training results.
|
||||
|
||||
## Start Training
|
||||
|
||||
### Basic Training Commands
|
||||
|
||||
```bash
|
||||
# Train with default parameters
|
||||
uv run scripts/train.py --env cartpole
|
||||
|
||||
# Specify simulation backend
|
||||
uv run scripts/train.py --env cartpole --sim-backend np
|
||||
|
||||
# Specify training backend
|
||||
uv run scripts/train.py --env cartpole --train-backend jax
|
||||
uv run scripts/train.py --env cartpole --train-backend torch
|
||||
```
|
||||
|
||||
### Advanced Training Configuration
|
||||
|
||||
```bash
|
||||
# Customize training parameters
|
||||
uv run scripts/train.py --env cartpole \
|
||||
--num-envs 1024 \
|
||||
--train-backend jax \
|
||||
--sim-backend np
|
||||
|
||||
# Enable rendering to monitor training process
|
||||
uv run scripts/train.py --env cartpole --render
|
||||
```
|
||||
|
||||
### Different Backend Configuration Differences
|
||||
|
||||
The system supports configuring different reinforcement learning parameters for different training backends (JAX/Torch). For example:
|
||||
|
||||
- **dm-walker environment**:
|
||||
|
||||
- JAX backend: `mini_batches: 4`
|
||||
- Torch backend: `mini_batches: 32`
|
||||
|
||||
- **dm-runner environment**:
|
||||
- JAX backend: `learning_epochs: 4`
|
||||
- Torch backend: `learning_epochs: 2`
|
||||
|
||||
These differences are implemented through the `@rlcfg(env_name, backend="jax/torch")` decorator in configuration classes. The system automatically applies the corresponding configuration based on the selected training backend.
|
||||
|
||||
### Supported Command Line Parameters
|
||||
|
||||
| Parameter | Description | Default Value |
|
||||
| ----------------- | ------------------------------- | ------------- |
|
||||
| `--env` | Environment name | `cartpole` |
|
||||
| `--sim-backend` | Simulation backend (np) | Auto select |
|
||||
| `--train-backend` | Training backend (jax/torch) | Auto select |
|
||||
| `--num-envs` | Number of parallel environments | 2048 |
|
||||
| `--render` | Enable rendering | False |
|
||||
|
||||
> **Note**: Other parameters such as learning rate, network structure, etc., can be set in configuration files. Some environments support configuring different parameters for different training backends.
|
||||
|
||||
## Training Process Monitoring
|
||||
|
||||
### TensorBoard Monitoring
|
||||
|
||||
Start TensorBoard to view training progress:
|
||||
|
||||
```bash
|
||||
uv run tensorboard --logdir runs/{env-name}
|
||||
```
|
||||
|
||||
For example:
|
||||
|
||||
```bash
|
||||
uv run tensorboard --logdir runs/cartpole
|
||||
```
|
||||
|
||||
## Model Evaluation and Testing
|
||||
|
||||
### Using Trained Policies
|
||||
|
||||
```bash
|
||||
# Automatically find best policy for testing (recommended)
|
||||
uv run scripts/play.py --env cartpole
|
||||
|
||||
# Manually specify policy file for testing
|
||||
uv run scripts/play.py --env cartpole --policy runs/cartpole/nn/best_policy.pickle
|
||||
|
||||
# Specify number of test environments
|
||||
uv run scripts/play.py --env cartpole --num-envs 100
|
||||
```
|
||||
|
||||
> **Note**: The system will automatically find the latest and best policy files in the `runs/cartpole/` directory for testing.
|
||||
@@ -0,0 +1,231 @@
|
||||
# Training Environment Configuration
|
||||
|
||||
MotrixLab provides a flexible configuration system that allows users to customize reinforcement learning training parameters. This section introduces how to configure training environments and reinforcement learning algorithm parameters.
|
||||
|
||||
## RL Training Configuration (PPOCfg)
|
||||
|
||||
Training configuration defines parameters for reinforcement learning algorithms. MotrixLab now supports configuring different parameters for different training backends.
|
||||
|
||||
### Basic Training Configuration
|
||||
|
||||
```python
|
||||
from dataclasses import dataclass
|
||||
from motrix_rl.skrl.cfg import PPOCfg
|
||||
from motrix_rl import registry
|
||||
|
||||
# Universal configuration (applies to all backends)
|
||||
@registry.rlcfg("my-task")
|
||||
@dataclass
|
||||
class MyTaskRLCfg(PPOCfg):
|
||||
# Environment parameters
|
||||
num_envs: int = 2048 # Number of parallel environments during training
|
||||
play_num_envs: int = 16 # Number of parallel environments during evaluation
|
||||
|
||||
# PPO algorithm parameters
|
||||
learning_rate: float = 3e-4 # Learning rate
|
||||
rollouts: int = 32 # Experience replay rounds
|
||||
learning_epochs: int = 10 # Number of epochs per update
|
||||
mini_batches: int = 32 # Number of mini-batches
|
||||
discount_factor: float = 0.99 # Discount factor
|
||||
grad_norm_clip: float = 1.0 # Gradient clipping
|
||||
|
||||
# Network structure parameters
|
||||
policy_hidden_layer_sizes: tuple[int, ...] = (256, 128, 64) # Policy network hidden layers
|
||||
value_hidden_layer_sizes: tuple[int, ...] = (256, 128, 64) # Value network hidden layers
|
||||
|
||||
# Training control parameters
|
||||
max_env_steps: int = 1_000_000 # Maximum environment steps
|
||||
check_point_interval: int = 10_000 # Checkpoint interval
|
||||
```
|
||||
|
||||
### Different Backend Configurations
|
||||
|
||||
```python
|
||||
# JAX backend specific configuration
|
||||
@registry.rlcfg("my-task", backend="jax")
|
||||
@dataclass
|
||||
class MyTaskJAXCfg(PPOCfg):
|
||||
# JAX optimized configuration
|
||||
learning_rate: float = 2e-4 # JAX backend uses smaller learning rate
|
||||
mini_batches: int = 4 # JAX supports large batches, fewer mini-batches
|
||||
learning_epochs: int = 4 # Training rounds
|
||||
num_envs: int = 2048 # More parallel environments
|
||||
|
||||
# Torch backend specific configuration
|
||||
@registry.rlcfg("my-task", backend="torch")
|
||||
@dataclass
|
||||
class MyTaskTorchCfg(PPOCfg):
|
||||
# Torch optimized configuration
|
||||
learning_rate: float = 1e-4 # Torch backend uses even smaller learning rate
|
||||
mini_batches: int = 32 # Torch needs more mini-batches
|
||||
learning_epochs: int = 2 # Fewer training rounds
|
||||
num_envs: int = 1024 # Fewer parallel environments
|
||||
```
|
||||
|
||||
### Complete Configuration Example
|
||||
|
||||
```python
|
||||
@dataclass
|
||||
class CompletePPOConfig(PPOCfg):
|
||||
"""
|
||||
Complete reinforcement learning training configuration example
|
||||
Contains all configuration parameters from basic to advanced
|
||||
"""
|
||||
|
||||
# ===== Basic Training Parameters =====
|
||||
seed: Optional[int] = None # Random seed
|
||||
num_envs: int = 2048 # Number of parallel environments during training
|
||||
play_num_envs: int = 16 # Number of parallel environments during evaluation
|
||||
max_env_steps: int = 2_048_000 # Maximum training steps
|
||||
check_point_interval: int = 1000 # Checkpoint save interval
|
||||
|
||||
# ===== PPO Algorithm Core Parameters =====
|
||||
learning_rate: float = 3e-4 # Learning rate
|
||||
rollouts: int = 32 # Experience replay rounds
|
||||
learning_epochs: int = 2 # Number of training rounds per update
|
||||
mini_batches: int = 32 # Number of mini-batches
|
||||
discount_factor: float = 0.99 # Discount factor
|
||||
lambda_param: float = 0.95 # GAE parameter
|
||||
grad_norm_clip: float = 1.0 # Gradient clipping
|
||||
|
||||
# ===== PPO Clipping Parameters =====
|
||||
ratio_clip: float = 0.2 # PPO clipping ratio
|
||||
value_clip: float = 0.2 # Value clipping
|
||||
clip_predicted_values: bool = True # Clip predicted values
|
||||
|
||||
# ===== Loss Function Parameters =====
|
||||
entropy_loss_scale: float = 0.0 # Entropy loss coefficient
|
||||
value_loss_scale: float = 2.0 # Value loss coefficient
|
||||
kl_threshold: float = 0 # KL divergence threshold
|
||||
|
||||
# ===== Learning Rate Scheduler =====
|
||||
learning_rate_scheduler_kl_threshold: float = 0.008 # Adaptive learning rate KL threshold
|
||||
|
||||
# ===== Network Architecture Configuration =====
|
||||
# Small network (suitable for simple tasks like CartPole)
|
||||
# policy_hidden_layer_sizes: tuple[int, ...] = (128, 64)
|
||||
# value_hidden_layer_sizes: tuple[int, ...] = (128, 64)
|
||||
|
||||
# Medium network (default configuration, suitable for most tasks)
|
||||
policy_hidden_layer_sizes: tuple[int, ...] = (256, 128, 64)
|
||||
value_hidden_layer_sizes: tuple[int, ...] = (256, 128, 64)
|
||||
|
||||
# Large network (suitable for complex tasks like robot control)
|
||||
# policy_hidden_layer_sizes: tuple[int, ...] = (512, 256, 128)
|
||||
# value_hidden_layer_sizes: tuple[int, ...] = (512, 256, 128)
|
||||
|
||||
# ===== Network Sharing Configuration =====
|
||||
share_policy_value_features: bool = True # Policy and value networks share feature extraction layers
|
||||
|
||||
# ===== Training Control Parameters =====
|
||||
random_timesteps: int = 0 # Random timesteps
|
||||
learning_starts: int = 0 # Timesteps to start learning
|
||||
time_limit_bootstrap: bool = True # Time limit bootstrap
|
||||
|
||||
# ===== Reward Shaping =====
|
||||
rewards_shaper_scale: float = 1.0 # Reward scaling factor
|
||||
```
|
||||
|
||||
## Configuration Usage Methods
|
||||
|
||||
### 1. Default Configuration Usage
|
||||
|
||||
```bash
|
||||
# Use configuration given in code
|
||||
uv run scripts/train.py --env my-task
|
||||
|
||||
# Specify training backend, system will automatically select corresponding backend configuration
|
||||
uv run scripts/train.py --env my-task --train-backend jax
|
||||
uv run scripts/train.py --env my-task --train-backend torch
|
||||
```
|
||||
|
||||
### 2. Command Line Parameter Override
|
||||
|
||||
```bash
|
||||
# Override supported command line parameters
|
||||
uv run scripts/train.py --env my-task \
|
||||
--num-envs 1024 \
|
||||
--train-backend jax \
|
||||
--sim-backend np
|
||||
|
||||
# System will automatically select JAX backend configuration
|
||||
```
|
||||
|
||||
### 3. Configuration Priority
|
||||
|
||||
System selects configuration in the following priority:
|
||||
|
||||
1. **Backend-specific configuration**: If there is a configuration decorated with `@rlcfg(env_name, backend="jax/torch")`
|
||||
2. **Universal configuration**: If there is a configuration decorated with `@rlcfg(env_name)` (no backend parameter)
|
||||
3. **Configuration override**: Command line `cfg_override` parameter
|
||||
|
||||
For example:
|
||||
|
||||
```python
|
||||
# Highest priority - backend specific configuration
|
||||
@rlcfg("my-task", backend="jax")
|
||||
@dataclass
|
||||
class MyTaskJAXCfg(PPOCfg):
|
||||
mini_batches: int = 4
|
||||
|
||||
# Second priority - universal configuration
|
||||
@rlcfg("my-task")
|
||||
@dataclass
|
||||
class MyTaskRLCfg(PPOCfg):
|
||||
mini_batches: int = 32
|
||||
|
||||
# When using --train-backend jax, system will select MyTaskJAXCfg
|
||||
# When using --train-backend torch, system will select MyTaskRLCfg
|
||||
```
|
||||
|
||||
#### User Configurable Parameters
|
||||
|
||||
| MotrixLab Configuration Class | SKRL Framework Parameter | Description |
|
||||
| -------------------------------------- | --------------------------------------------- | ----------------------------------- |
|
||||
| `learning_rate` | `learning_rate` | Learning rate |
|
||||
| `rollouts` | `rollouts` | Experience replay rounds |
|
||||
| `learning_epochs` | `learning_epochs` | Training rounds |
|
||||
| `mini_batches` | `mini_batches` | Number of mini-batches |
|
||||
| `discount_factor` | `discount_factor` | Discount factor |
|
||||
| `grad_norm_clip` | `grad_norm_clip` | Gradient clipping |
|
||||
| `lambda_param` | `lambda` | GAE parameter |
|
||||
| `ratio_clip` | `ratio_clip` | PPO clipping ratio |
|
||||
| `value_clip` | `value_clip` | Value clipping |
|
||||
| `clip_predicted_values` | `clip_predicted_values` | Clip predicted values |
|
||||
| `entropy_loss_scale` | `entropy_loss_scale` | Entropy loss coefficient |
|
||||
| `value_loss_scale` | `value_loss_scale` | Value loss coefficient |
|
||||
| `kl_threshold` | `kl_threshold` | KL divergence threshold |
|
||||
| `random_timesteps` | `random_timesteps` | Random timesteps |
|
||||
| `learning_starts` | `learning_starts` | Learning start timesteps |
|
||||
| `time_limit_bootstrap` | `time_limit_bootstrap` | Time limit bootstrap |
|
||||
| `learning_rate_scheduler_kl_threshold` | `learning_rate_scheduler_kwargs.kl_threshold` | Adaptive learning rate KL threshold |
|
||||
| `check_point_interval` | `experiment.write_interval` | Log write interval |
|
||||
| `check_point_interval` | `experiment.checkpoint_interval` | Checkpoint save interval |
|
||||
| `rewards_shaper_scale` | `rewards_shaper` | Reward scaling function |
|
||||
|
||||
#### Preprocessor Parameters
|
||||
|
||||
| SKRL Framework Parameter | Type | Description |
|
||||
| ------------------------ | --------------------- | ------------------- |
|
||||
| `state_preprocessor` | RunningStandardScaler | State normalization |
|
||||
| `value_preprocessor` | RunningStandardScaler | Value normalization |
|
||||
|
||||
### Configuration Hierarchy Summary
|
||||
|
||||
```
|
||||
User Configuration Class (PPOCfg)
|
||||
↓ Backend specific selection
|
||||
Backend Configuration (JAX/Torch)
|
||||
↓ Parameter mapping
|
||||
SKRL Framework Configuration Dictionary
|
||||
↓ Pass to
|
||||
PPO Agent
|
||||
↓ Execute
|
||||
Reinforcement Learning Training
|
||||
```
|
||||
|
||||
This design allows users to:
|
||||
|
||||
1. Control complex training parameters through simple configuration classes
|
||||
2. Configure different parameters for different training backends to achieve optimal performance
|
||||
3. Maintain full compatibility with the SKRL framework
|
||||
66
docs/source/zh_CN/index.md
Normal file
66
docs/source/zh_CN/index.md
Normal file
@@ -0,0 +1,66 @@
|
||||
# MotrixLab
|
||||
|
||||
MotrixLab 是一个为机器人训练设计的机器学习通用架构。它基于 MotrixSim 仿真平台搭建，可以实现在 CPU 或 GPU 上的策略训练，为各类操作系统和硬件设备使用者，提供灵活易用的训练能力。
|
||||
|
||||
::::{grid} 1 2 3 3
|
||||
:gutter: 2 2 2 2
|
||||
|
||||
:::{grid-item-card}
|
||||
|
||||
```{video} _static/videos/cartpole.mp4
|
||||
:poster: _static/images/poster/cartpole.jpg
|
||||
:nocontrols:
|
||||
:autoplay:
|
||||
:playsinline:
|
||||
:muted:
|
||||
:loop:
|
||||
:width: 100%
|
||||
```
|
||||
|
||||
:::
|
||||
|
||||
:::{grid-item-card}
|
||||
|
||||
```{video} _static/videos/go1_walk.mp4
|
||||
:poster: _static/images/poster/go1_walk.jpg
|
||||
:nocontrols:
|
||||
:autoplay:
|
||||
:playsinline:
|
||||
:muted:
|
||||
:loop:
|
||||
:width: 100%
|
||||
```
|
||||
|
||||
:::
|
||||
|
||||
:::{grid-item-card}
|
||||
|
||||
```{video} _static/videos/dm_walker.mp4
|
||||
:poster: _static/images/poster/dm_walker.jpg
|
||||
:nocontrols:
|
||||
:autoplay:
|
||||
:playsinline:
|
||||
:muted:
|
||||
:loop:
|
||||
:width: 100%
|
||||
```
|
||||
|
||||
:::
|
||||
::::
|
||||
|
||||
## 主要特性
|
||||
|
||||
- **跨软件平台能力**: 支持 Windows 和 Linux 操作系统环境
|
||||
- **支持 CPU 仿真**: 使用 MotrixSim 在 CPU 上进行物理仿真，对 GPU 的要求大幅降低
|
||||
|
||||
## 适用场景
|
||||
|
||||
- 机器人控制算法开发和测试
|
||||
- 强化学习环境构建
|
||||
- 教育和研究
|
||||
|
||||
```{toctree}
|
||||
:maxdepth: 1
|
||||
|
||||
user_guide/index
|
||||
```
|
||||
99
docs/source/zh_CN/user_guide/demo/cartpole.md
Normal file
99
docs/source/zh_CN/user_guide/demo/cartpole.md
Normal file
@@ -0,0 +1,99 @@
|
||||
# 倒立摆训练示例
|
||||
|
||||
倒立摆（CartPole）是强化学习中的经典控制任务，目标是通过控制小车左右移动来保持杆子平衡。
|
||||
|
||||
|
||||
## 任务描述
|
||||
|
||||
- **状态空间**：小车位置、小车速度、杆子角度、杆子角速度
|
||||
- **动作空间**：向左或向右施加力
|
||||
- **奖励函数**：每一步保持杆子不倒下获得+1 奖励
|
||||
- **终止条件**：杆子角度超过 ±15 度或 episode 长度超过 10 秒
|
||||
|
||||
## 快速开始
|
||||
|
||||
### 1. 环境预览
|
||||
|
||||
```bash
|
||||
uv run scripts/view.py --env cartpole
|
||||
```
|
||||
|
||||
### 2. 开始训练
|
||||
|
||||
```bash
|
||||
# 使用默认参数训练
|
||||
uv run scripts/train.py --env cartpole
|
||||
|
||||
# 自定义环境数量
|
||||
uv run scripts/train.py --env cartpole --num-envs 1024
|
||||
|
||||
# 启用渲染（训练时可视化）
|
||||
uv run scripts/train.py --env cartpole --render
|
||||
```
|
||||
|
||||
### 3. 查看训练进度
|
||||
|
||||
```bash
|
||||
uv run tensorboard --logdir runs/cartpole
|
||||
```
|
||||
|
||||
### 4. 测试训练结果
|
||||
|
||||
```bash
|
||||
# 自动寻找最佳策略测试（推荐）
|
||||
uv run scripts/play.py --env cartpole
|
||||
|
||||
# 手动指定策略文件测试
|
||||
uv run scripts/play.py --env cartpole --policy runs/cartpole/nn/best_agent.pickle
|
||||
|
||||
```
|
||||
|
||||
> **提示**：系统会自动在 `runs/cartpole/` 目录下寻找最新、最佳的策略文件进行测试。您也可以通过 `--policy` 参数手动指定特定的策略文件。
|
||||
|
||||
## 配置参数
|
||||
|
||||
倒立摆环境的主要配置参数：
|
||||
|
||||
```python
|
||||
@dataclass
|
||||
class CartPoleEnvCfg(EnvCfg):
|
||||
model_file: str = "path/to/inverted_pendulum.xml" # MJCF模型文件
|
||||
reset_noise_scale: float = 0.01 # 重置噪声
|
||||
max_episode_seconds: float = 10.0 # 最大episode长度
|
||||
```
|
||||
|
||||
训练配置参数：
|
||||
|
||||
```python
|
||||
@dataclass
|
||||
class CartPoleRLCfg(BaseRLCfg):
|
||||
num_envs: int = 2048 # 并行环境数量
|
||||
learning_rate: float = 3e-4 # 学习率
|
||||
batch_size: int = 2048 # 批大小
|
||||
max_epochs: int = 500 # 最大训练轮数
|
||||
```
|
||||
|
||||
## 自定义训练
|
||||
|
||||
您可以通过命令行参数覆盖默认配置：
|
||||
|
||||
```bash
|
||||
uv run scripts/train.py --env cartpole \
|
||||
--num-envs 1024 \
|
||||
--train-backend jax \
|
||||
--sim-backend np
|
||||
```
|
||||
|
||||
## 预期结果
|
||||
|
||||
- 杆子角度大部分时间保持在 ±5 度以内
|
||||
- 小车位移范围适中
|
||||
|
||||
## 故障排除
|
||||
|
||||
如果训练效果不佳，可以尝试：
|
||||
|
||||
1. 调整学习率（尝试 1e-4 到 1e-3）
|
||||
2. 增加环境数量（更多并行训练）
|
||||
3. 调整奖励函数权重
|
||||
4. 检查物理参数设置是否合理
|
||||
102
docs/source/zh_CN/user_guide/demo/dm_walker.md
Normal file
102
docs/source/zh_CN/user_guide/demo/dm_walker.md
Normal file
@@ -0,0 +1,102 @@
|
||||
# 二维步行机器人训练示例
|
||||
|
||||
二维步行机器人（Walker2D）是基于 DeepMind Control Suite 的经典机器人控制任务，目标是通过控制机器人关节来实现站立、行走和奔跑。
|
||||
|
||||
```{video} /_static/videos/dm_walker.mp4
|
||||
:poster: _static/images/poster/dm_walker.jpg
|
||||
:nocontrols:
|
||||
:autoplay:
|
||||
:playsinline:
|
||||
:muted:
|
||||
:loop:
|
||||
:width: 100%
|
||||
```
|
||||
|
||||
## 任务描述
|
||||
|
||||
Walker2D 是一个二维平面的双足机器人，具有多个关节和执行器：
|
||||
|
||||
- **状态空间**：包括机器人各部位的旋转角度、角速度、躯干高度和速度等
|
||||
- **动作空间**：控制各个关节的力矩
|
||||
- **奖励函数**：主要由保持站立、前进速度等组成
|
||||
- **终止条件**：机器人摔倒或关节达到极限位置
|
||||
|
||||
### 三种任务模式
|
||||
|
||||
1. **dm-stander**: 静止站立任务 (move_speed = 0.0)
|
||||
|
||||
```bash
|
||||
uv run scripts/train.py --env dm-stander
|
||||
```
|
||||
|
||||
2. **dm-walker**: 行走任务 (move_speed = 1.0)
|
||||
|
||||
```bash
|
||||
uv run scripts/train.py --env dm-walker
|
||||
```
|
||||
|
||||
3. **dm-runner**: 奔跑任务 (move_speed = 5.0)
|
||||
|
||||
```bash
|
||||
uv run scripts/train.py --env dm-runner
|
||||
```
|
||||
|
||||
## 配置参数
|
||||
|
||||
### 环境配置
|
||||
|
||||
```python
|
||||
@dataclass
|
||||
class WalkerEnvCfg(EnvCfg):
|
||||
model_file: str = "walker.xml" # MJCF模型文件
|
||||
max_episode_seconds: float = 25.0 # 最大episode长度
|
||||
sim_dt: float = 0.0125 # 仿真时间步
|
||||
ctrl_dt: float = 0.025 # 控制时间步
|
||||
move_speed: float = 1.0 # 目标移动速度
|
||||
stand_height: float = 1.2 # 目标站立高度
|
||||
```
|
||||
|
||||
### 训练配置
|
||||
|
||||
```python
|
||||
@dataclass
|
||||
class WalkerRLCfg(BaseRLCfg):
|
||||
num_envs: int = 512 # 并行环境数量
|
||||
learning_rate: float = 3e-4 # 学习率
|
||||
batch_size: int = 512 # 批大小
|
||||
max_epochs: int = 1000 # 最大训练轮数
|
||||
```
|
||||
|
||||
## 奖励函数设计
|
||||
|
||||
Walker2D 的奖励函数由以下几个部分组成：
|
||||
|
||||
### 基础站立奖励
|
||||
|
||||
```python
|
||||
# 高度奖励：保持躯干在目标高度
|
||||
# 直立奖励：保持躯干直立
|
||||
```
|
||||
|
||||
### 移动奖励（行走和奔跑任务）
|
||||
|
||||
```python
|
||||
# 速度奖励：追踪目标速度
|
||||
# 总奖励 = 站立奖励 * 移动权重
|
||||
```
|
||||
|
||||
## 预期结果
|
||||
|
||||
1. **dm-stander**：
|
||||
|
||||
- 躯干高度保持在 1.0-1.4m 范围
|
||||
- 躯干直立角度偏差小于 15 度
|
||||
|
||||
2. **dm-walker**：
|
||||
|
||||
- 实际行走速度接近 1.0 m/s
|
||||
- 步态协调，无明显摔倒
|
||||
|
||||
3. **dm-runner**：
|
||||
- 奔跑速度达到 4.0-5.0 m/s
|
||||
- 出现飞行相（双脚同时离地）
|
||||
139
docs/source/zh_CN/user_guide/demo/locomotion_unitree_go1.md
Normal file
139
docs/source/zh_CN/user_guide/demo/locomotion_unitree_go1.md
Normal file
@@ -0,0 +1,139 @@
|
||||
# Unitree GO1 机器人行走训练示例
|
||||
|
||||
Unitree GO1 是一个四足机器人平台，本示例展示了如何训练 GO1 在平坦地形上实现稳定的步态行走。
|
||||
|
||||
```{video} /_static/videos/go1_walk.mp4
|
||||
:poster: _static/images/poster/go1_walk.jpg
|
||||
:nocontrols:
|
||||
:autoplay:
|
||||
:playsinline:
|
||||
:muted:
|
||||
:loop:
|
||||
:width: 100%
|
||||
```
|
||||
|
||||
## 任务描述
|
||||
|
||||
GO1 四足机器人具有 12 个自由度（每条腿 3 个关节），需要通过深度强化学习学习协调的步态控制：
|
||||
|
||||
- **状态空间**：48 维，包含机器人线速度、角速度、姿态、关节角度、关节速度、动作和命令等
|
||||
- **动作空间**：12 维，控制各个关节的目标位置（通过 PD 控制器转换为力矩）
|
||||
- **奖励函数**：复合奖励，包含速度跟踪、姿态稳定、能量效率等多个组件
|
||||
- **终止条件**：机器人躯干接触地面或其他不稳定状态
|
||||
|
||||
### 训练任务
|
||||
|
||||
```bash
|
||||
uv run scripts/train.py --env go1-flat-terrain-walk
|
||||
```
|
||||
|
||||
## 配置参数
|
||||
|
||||
### 环境配置
|
||||
|
||||
```python
|
||||
@dataclass
|
||||
class Go1WalkNpEnvCfg(EnvCfg):
|
||||
max_episode_seconds: float = 20.0 # 最大episode长度
|
||||
model_file: str = "scene_motor_actuator.xml"
|
||||
sim_dt: float = 0.01 # 仿真时间步
|
||||
ctrl_dt: float = 0.01 # 控制时间步
|
||||
```
|
||||
|
||||
### 控制配置
|
||||
|
||||
```python
|
||||
@dataclass
|
||||
class ControlConfig:
|
||||
stiffness = 80 # PD 控制器刚度 [N*m/rad]
|
||||
damping = 1 # PD 控制器阻尼 [N*m*s/rad]
|
||||
action_scale = 0.05 # 动作缩放因子
|
||||
```
|
||||
|
||||
### 初始关节角度
|
||||
|
||||
```python
|
||||
default_joint_angles = {
|
||||
"FL_hip": 0.0, # 前左髋关节
|
||||
"RL_hip": 0.0, # 后左髋关节
|
||||
"FR_hip": -0.0, # 前右髋关节
|
||||
"RR_hip": -0.0, # 后右髋关节
|
||||
"FL_thigh": 0.9, # 前左大腿
|
||||
"RL_thigh": 0.9, # 后左大腿
|
||||
"FR_thigh": 0.9, # 前右大腿
|
||||
"RR_thigh": 0.9, # 后右大腿
|
||||
"FL_calf": -1.8, # 前左小腿
|
||||
"RL_calf": -1.8, # 后左小腿
|
||||
"FR_calf": -1.8, # 前右小腿
|
||||
"RR_calf": -1.8, # 后右小腿
|
||||
}
|
||||
```
|
||||
|
||||
## 奖励函数设计
|
||||
|
||||
GO1 的奖励函数是一个复杂的复合函数，包含多个组件：
|
||||
|
||||
### 主要奖励组件
|
||||
|
||||
```python
|
||||
reward_config.scales = {
|
||||
"tracking_lin_vel": 1.0, # 线速度跟踪奖励
|
||||
"tracking_ang_vel": 0.5, # 角速度跟踪奖励
|
||||
"feet_air_time": 1.0, # 足部空中时间奖励
|
||||
"lin_vel_z": -2.0, # Z轴线速度惩罚
|
||||
"ang_vel_xy": -0.05, # XY轴角速度惩罚
|
||||
"orientation": -0.0, # 姿态偏离惩罚
|
||||
"torques": -0.00001, # 力矩消耗惩罚
|
||||
"dof_acc": -2.5e-7, # 关节加速度惩罚
|
||||
"action_rate": -0.001, # 动作变化率惩罚
|
||||
"hip_pos": -1, # 髋关节位置惩罚
|
||||
"calf_pos": -0.3, # 腿关节位置惩罚
|
||||
}
|
||||
```
|
||||
|
||||
### 关键奖励函数
|
||||
|
||||
#### 速度跟踪奖励
|
||||
|
||||
```python
|
||||
# 跟踪线速度命令（xy平面）
|
||||
def _reward_tracking_lin_vel(self, data, commands):
|
||||
|
||||
# 跟踪角速度命令（偏航）
|
||||
def _reward_tracking_ang_vel(self, data, commands):
|
||||
```
|
||||
|
||||
#### 足部空中时间奖励
|
||||
|
||||
```python
|
||||
def _reward_feet_air_time(self, commands, info):
|
||||
```
|
||||
|
||||
## 观察空间构成
|
||||
|
||||
GO1 的观察空间为 48 维，包含以下信息：
|
||||
|
||||
```python
|
||||
obs = np.hstack([
|
||||
noisy_linvel, # 3维：局部坐标系线速度
|
||||
noisy_gyro, # 3维：陀螺仪数据
|
||||
local_gravity, # 3维：局部重力方向
|
||||
noisy_joint_angle, # 12维：关节角度（相对于默认值）
|
||||
noisy_joint_vel, # 12维：关节速度
|
||||
last_actions, # 12维：上一帧动作
|
||||
command, # 3维：速度命令 [vx, vy, vyaw]
|
||||
])
|
||||
```
|
||||
|
||||
## 运动速度命令生成
|
||||
|
||||
训练过程中随机生成速度命令，确保智能体能够跟踪不同的移动速度：
|
||||
|
||||
```python
|
||||
def resample_commands(self, num_envs: int):
|
||||
```
|
||||
|
||||
## 预期训练结果
|
||||
|
||||
1. 稳定的四足步态
|
||||
2. 良好的速度跟踪
|
||||
@@ -0,0 +1,84 @@
|
||||
# 快速入门：Hello MotrixLab
|
||||
|
||||
本教程通过演示一个简单例子 - 加载倒立摆并进行训练，以此来展示 MotrixLab 工作流程：
|
||||
|
||||
## 环境预览
|
||||
|
||||
我们提供了一个简单的脚本，用于可视化一个环境，而不执行任何训练，这可以帮助您检测系统的环境依赖是否配置正确：
|
||||
|
||||
```bash
|
||||
uv run scripts/view.py --env cartpole
|
||||
```
|
||||
|
||||
这将打开一个可视化窗口，显示倒立摆的物理仿真环境，使用随机动作进行演示。
|
||||
|
||||
## 训练模型
|
||||
|
||||
开始训练倒立摆平衡任务：
|
||||
|
||||
```bash
|
||||
uv run scripts/train.py --env cartpole
|
||||
```
|
||||
|
||||
训练过程会自动：
|
||||
|
||||
1. 根据硬件环境自动选择训练后端（JAX 或 PyTorch）
|
||||
2. 创建训练环境
|
||||
3. 开始 PPO 算法训练
|
||||
|
||||
训练结果会保存在 `runs/cartpole/` 目录下，包含：
|
||||
|
||||
- 训练检查点（checkpoint）
|
||||
- TensorBoard 日志文件
|
||||
|
||||
## 可视化训练过程
|
||||
|
||||
如果您想要在训练过程中观察模型的学习过程，可以启用可视化渲染：
|
||||
|
||||
```bash
|
||||
uv run scripts/train.py --env cartpole --render
|
||||
```
|
||||
|
||||
### 🎮 交互式渲染控制
|
||||
|
||||
> **重要提示**：可视化会显著降低训练速度，建议主要用于调试和演示。
|
||||
|
||||
在可视化训练过程中，您可以使用**空格键**来动态控制渲染：
|
||||
|
||||
- **开启渲染**：按下空格键开启可视化，观察机器人行为
|
||||
- **关闭渲染**：再次按下空格键关闭渲染，提升训练速度
|
||||
- **随时切换**：无需重新启动程序，可以在训练过程中随时切换
|
||||
|
||||
这种交互式控制让您可以在需要时观察训练效果，在不需要时享受快速训练。这项功能在运行推断时也能生效。
|
||||
|
||||
## 查看训练结果
|
||||
|
||||
使用 TensorBoard 查看训练进度：
|
||||
|
||||
```bash
|
||||
uv run tensorboard --logdir runs/cartpole
|
||||
```
|
||||
|
||||
## 测试训练好的模型
|
||||
|
||||
训练完成后，使用训练好的策略进行测试：
|
||||
|
||||
```bash
|
||||
# 自动寻找最佳策略测试（推荐）
|
||||
uv run scripts/play.py --env cartpole
|
||||
|
||||
# 手动指定策略文件测试（如果需要特定版本）
|
||||
uv run scripts/play.py --env cartpole --policy runs/cartpole/YOUR_RESULT_NUMBER/best_agent.pickle
|
||||
```
|
||||
|
||||
> **提示**：系统会自动在 `runs/cartpole/` 目录下寻找最新、最佳的策略文件。通常情况下，使用自动发现功能即可。
|
||||
|
||||
## 至此我们完成了整个示例
|
||||
|
||||
接下来可以尝试修改参数，观察不同设置下的物理效果，或者尝试其他环境。
|
||||
|
||||
## 下一步
|
||||
|
||||
- 了解 [基础框架](../tutorial/basic_frame.md)
|
||||
- 学习 [物理环境配置](../tutorial/physics_environment.md)
|
||||
- 查看更多 [训练示例](../demo/cartpole.md)
|
||||
57
docs/source/zh_CN/user_guide/getting_started/installation.md
Normal file
57
docs/source/zh_CN/user_guide/getting_started/installation.md
Normal file
@@ -0,0 +1,57 @@
|
||||
# 安装环境
|
||||
|
||||
## 安装要求
|
||||
|
||||
- **Python 版本**：{bdg-danger-line}`3.10.*`
|
||||
|
||||
| Python 版本 | 支持状态 |
|
||||
| :---------: | :------: |
|
||||
| ≤ 3.9 | ❌ |
|
||||
| 3.10 | ✅ |
|
||||
| ≥ 3.11 | ❌ |
|
||||
|
||||
- **包管理器**：{bdg-danger-line}`UV`
|
||||
[UV 安装参考](https://docs.astral.sh/uv/getting-started/installation/)
|
||||
|
||||
- **系统及架构**：
|
||||
|
||||
- {bdg-danger-line}`Windows(x86_64)`
|
||||
- {bdg-danger-line}`Linux(x86_64)`
|
||||
|
||||
```{note}
|
||||
各平台支持的功能如下：
|
||||
|
||||
| 操作系统 | CPU 仿真 | 交互式查看器 | GPU 仿真 |
|
||||
| :------: | :------: | :----------: | :------: |
|
||||
| Linux | ✅ | ✅ | 🛠️ 开发中 |
|
||||
| Windows | ✅ | ✅ | 🛠️ 开发中 |
|
||||
```
|
||||
|
||||
## 安装方法
|
||||
|
||||
### 克隆项目
|
||||
|
||||
```bash
|
||||
git clone https://github.com/Motphys/MotrixLab.git
|
||||
cd MotrixLab
|
||||
```
|
||||
|
||||
### 安装依赖
|
||||
|
||||
使用 UV 安装项目依赖：
|
||||
|
||||
```bash
|
||||
# 安装所有依赖
|
||||
uv sync --all-packages --all-extras
|
||||
```
|
||||
|
||||
如果只需要安装一种训练后端，可以选择单独安装指定的后端类型：
|
||||
|
||||
```bash
|
||||
|
||||
# 安装 SKRL JAX （仅支持 Linux 平台）
|
||||
uv sync --all-packages --extra skrl-jax
|
||||
|
||||
# 安装 SKRL PyTorch
|
||||
uv sync --all-packages --extra skrl-torch
|
||||
```
|
||||
32
docs/source/zh_CN/user_guide/index.md
Normal file
32
docs/source/zh_CN/user_guide/index.md
Normal file
@@ -0,0 +1,32 @@
|
||||
# 用户指南
|
||||
|
||||
```{toctree}
|
||||
:caption: 入门指南
|
||||
:maxdepth: 1
|
||||
|
||||
getting_started/installation
|
||||
getting_started/hello_motrixlab
|
||||
|
||||
```
|
||||
|
||||
```{toctree}
|
||||
:caption: 使用教程
|
||||
:maxdepth: 1
|
||||
|
||||
tutorial/basic_frame
|
||||
tutorial/physics_environment
|
||||
tutorial/training_environment_config
|
||||
tutorial/rewards
|
||||
tutorial/training_and_result
|
||||
|
||||
```
|
||||
|
||||
```{toctree}
|
||||
:caption: 训练示例
|
||||
:maxdepth: 1
|
||||
|
||||
demo/cartpole
|
||||
demo/dm_walker
|
||||
demo/locomotion_unitree_go1
|
||||
|
||||
```
|
||||
149
docs/source/zh_CN/user_guide/tutorial/basic_frame.md
Normal file
149
docs/source/zh_CN/user_guide/tutorial/basic_frame.md
Normal file
@@ -0,0 +1,149 @@
|
||||
# 基础框架
|
||||
|
||||
MotrixLab 是一个机器人强化学习平台，这一节我们会介绍 MotrixLab 的框架设计以及各个组成部分之间的关系。如果您已经熟悉强化学习的内容，可以直接跳转至下一节，了解如何开发自己的训练环境。
|
||||
|
||||
## MotrixLab 的框架设计
|
||||
|
||||
MotrixLab 采用分层架构设计，将训练环境与训练逻辑进行了清晰拆分：
|
||||
|
||||
```
|
||||
MotrixLab/
|
||||
├── motrix_envs/ # 环境层：物理仿真和任务定义
|
||||
│ ├── basic/ # 基础环境（cartpole、walker等）
|
||||
│ ├── locomotion/ # 运动环境（GO1机器人等）
|
||||
│ ├── np/ # NumPy仿真后端框架
|
||||
│ ├── base.py # 环境基类
|
||||
│ └── registry.py # 环境注册系统
|
||||
├── motrix_rl/ # 训练层：RL算法和配置
|
||||
│ ├── skrl/ # SKRL框架集成（JAX/PyTorch）
|
||||
│ ├── base.py # RL配置基类
|
||||
│ └── registry.py # RL配置注册系统
|
||||
└── scripts
|
||||
├── train.py # 训练入口脚本
|
||||
├── play.py # 测试入口脚本
|
||||
└── view.py # 可视化脚本
|
||||
```
|
||||
|
||||
## 核心组件架构
|
||||
|
||||
```
|
||||
┌─────────────────────────────────────────────────────────────────┐
|
||||
│ 用户接口层 │
|
||||
│ train.py │ play.py │ view.py │
|
||||
└─────────────────────────────────────────────────────────────────┘
|
||||
│
|
||||
▼
|
||||
┌─────────────────────────────────────────────────────────────────┐
|
||||
│ 训练算法层 (SKRL) │
|
||||
│ PPO训练器 │ 网络架构 │ 优化器 │
|
||||
└─────────────────────────────────────────────────────────────────┘
|
||||
│
|
||||
▼
|
||||
┌─────────────────────────────────────────────────────────────────┐
|
||||
│ 环境实现层 │
|
||||
│ 环境配置(EnvCfg) │ 环境实现(Env) │ 奖励函数(Reward) │
|
||||
└─────────────────────────────────────────────────────────────────┘
|
||||
│
|
||||
▼
|
||||
┌─────────────────────────────────────────────────────────────────┐
|
||||
│ 物理仿真层 (MotrixSim) │
|
||||
│ MJCF模型 │ 物理引擎 │ 碰撞检测 │
|
||||
└─────────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
## 核心组件详解
|
||||
|
||||
### 1. 训练环境 (Training Environment)
|
||||
|
||||
**位置**：环境实现层
|
||||
|
||||
训练环境是 MotrixLab 的核心组件，包含三个关键部分：
|
||||
|
||||
- **环境配置 (EnvCfg)**：定义物理仿真参数（模型文件、时间步长、episode 长度等）和任务特定参数
|
||||
- **环境实现 (Env)**：继承基础环境类，实现具体的任务逻辑、物理仿真交互和终止条件检查
|
||||
- **奖励函数 (Reward)**：在环境的 step 方法中实现，根据当前状态和动作计算奖励值
|
||||
|
||||
环境通过装饰器注册到系统中。
|
||||
|
||||
### 2. 奖励函数 (Reward Function)
|
||||
|
||||
**位置**：配置管理层 + 环境实现层
|
||||
|
||||
奖励函数在 MotrixLab 中采用双重结构设计：
|
||||
|
||||
- **配置层面**：在配置类中定义奖励权重、奖励组件类型和缩放参数
|
||||
- **实现层面**：在环境的 `_compute_reward` 方法中根据配置参数计算具体奖励值
|
||||
|
||||
这种设计使得奖励函数既可以通过配置文件灵活调整，又能在代码中实现复杂的计算逻辑。
|
||||
|
||||
### 3. 配置参数 (Configuration Parameters)
|
||||
|
||||
**位置**：配置管理层
|
||||
|
||||
配置参数采用分层管理结构：
|
||||
|
||||
- **环境配置 (EnvCfg)**：控制物理仿真和任务行为，包括仿真参数、重置噪声、时间限制等
|
||||
- **训练配置 (RLCfg)**：控制强化学习算法，包括网络结构、学习率、批次大小、训练步数等
|
||||
|
||||
配置类支持继承、参数验证和运行时覆盖，确保参数的合理性和灵活性。
|
||||
|
||||
### 4. 注册系统 (Registry System)
|
||||
|
||||
**位置**：连接各组件的枢纽
|
||||
|
||||
注册系统通过装饰器模式实现组件的自动注册：
|
||||
|
||||
- 环境配置类通过 `@registry.envcfg()` 注册
|
||||
- 环境实现类通过 `@registry.env()` 注册，支持多后端
|
||||
- RL 配置类通过 `@rlcfg()` 注册
|
||||
|
||||
注册系统实现了组件的解耦，使得新增环境或修改配置变得简单快捷。
|
||||
|
||||
## 数据流和工作流程
|
||||
|
||||
### 训练流程概览
|
||||
|
||||
```
|
||||
用户命令 → 配置解析 → 环境创建 → 训练循环 → 模型保存
|
||||
↓
|
||||
train.py --env cartpole
|
||||
↓
|
||||
查找配置类 → 创建环境 → 启动PPO训练 → 保存模型
|
||||
```
|
||||
|
||||
### 核心工作流程
|
||||
|
||||
1. **环境定义**：在 `/motrix_envs/` 中创建环境配置类和实现类
|
||||
2. **自动注册**：通过装饰器将组件注册到系统中
|
||||
3. **配置加载**：命令行启动时，系统自动查找并加载对应的配置
|
||||
4. **环境创建**：工厂模式创建环境实例，支持参数覆盖
|
||||
5. **训练执行**：PPO 算法与环境交互，收集数据并更新策略
|
||||
6. **结果保存**：定期保存检查点和最终模型
|
||||
|
||||
### 配置参数的作用
|
||||
|
||||
配置参数在整个流程中起到关键的连接作用：
|
||||
|
||||
- **环境配置**决定物理仿真行为（时间步长、模型文件、噪声等）
|
||||
- **奖励配置**影响学习信号（奖励权重、计算方式等）
|
||||
- **训练配置**控制算法行为（网络结构、学习率、批次大小等）
|
||||
|
||||
## 多后端支持
|
||||
|
||||
MotrixLab 的分层设计天然支持多种后端：
|
||||
|
||||
- **仿真后端**：MotrixSim
|
||||
- **训练后端**：JAX 和 PyTorch，支持 GPU 加速
|
||||
- **算法框架**：主要集成 SKRL，易于扩展其他算法
|
||||
|
||||
## 设计优势
|
||||
|
||||
这种架构设计带来了以下核心优势：
|
||||
|
||||
1. **模块解耦**：环境开发与训练逻辑完全分离
|
||||
2. **配置灵活**：支持分层配置和运行时参数覆盖
|
||||
3. **扩展性强**：通过注册系统轻松添加新组件
|
||||
4. **多后端兼容**：同一环境可使用不同仿真和训练后端
|
||||
5. **实验友好**：配置可保存、比较，确保实验可重现
|
||||
|
||||
通过这个框架设计，MotrixLab 为机器人强化学习提供了一个清晰、灵活且易用的开发平台。
|
||||
61
docs/source/zh_CN/user_guide/tutorial/physics_environment.md
Normal file
61
docs/source/zh_CN/user_guide/tutorial/physics_environment.md
Normal file
@@ -0,0 +1,61 @@
|
||||
# 物理环境配置
|
||||
|
||||
物理环境配置定义了强化学习训练中的仿真参数和模型文件设置。
|
||||
MotrixLab 使用了[MotrixSim](https://motrixsim.readthedocs.io/zh-cn/latest/user_guide/index.html)作为物理仿真后端。
|
||||
|
||||
## 支持的文件格式
|
||||
|
||||
- [**MJCF**](https://mujoco.readthedocs.io/en/stable/XMLreference.html)(MuJoCo XML 格式) - 提供丰富的物理特性和仿真配置
|
||||
|
||||
## 模型文件配置
|
||||
|
||||
需要在环境配置类中指定模型文件路径：
|
||||
|
||||
```python
|
||||
@registry.envcfg("my-task")
|
||||
@dataclass
|
||||
class MyTaskEnvCfg(EnvCfg):
|
||||
# 模型文件路径（必需）
|
||||
model_file: str = "my_model.xml"
|
||||
|
||||
# 仿真时间参数
|
||||
sim_dt: float = 0.002 # 仿真时间步
|
||||
ctrl_dt: float = 0.02 # 控制更新频率
|
||||
```
|
||||
|
||||
### 推荐目录结构
|
||||
|
||||
```
|
||||
motrix_envs/my_task/
|
||||
├── __init__.py # 模块初始化
|
||||
├── cfg.py # 环境配置
|
||||
├── my_model.xml # 物理模型文件
|
||||
└── my_env.py # 环境实现
|
||||
```
|
||||
|
||||
对于结构复杂，引用文件较多的模型，推荐使用文件夹管理。
|
||||
|
||||
## 常见配置问题
|
||||
|
||||
### 文件路径问题
|
||||
|
||||
- 使用相对路径时，确保路径相对于配置文件位置
|
||||
- 避免使用硬编码的绝对路径
|
||||
- 检查文件权限和可访问性
|
||||
- 确保所有引用的子文件都存在
|
||||
|
||||
### 时间步设置
|
||||
|
||||
- `ctrl_dt` 应该是 `sim_dt` 的整数倍
|
||||
- `sim_dt` 过小会影响仿真性能
|
||||
- `ctrl_dt` 过大会影响控制精度
|
||||
- 推荐 `sim_dt` 在 0.001-0.02 秒之间
|
||||
|
||||
### 仿真稳定性
|
||||
|
||||
- 避免过大的时间步长
|
||||
- 合理设置接触参数避免穿透
|
||||
- 质量和惯性分布要合理
|
||||
- 关节限制要符合实际情况
|
||||
|
||||
通过合理的物理环境配置，您可以为强化学习训练创建准确且高效的仿真环境。
|
||||
50
docs/source/zh_CN/user_guide/tutorial/rewards.md
Normal file
50
docs/source/zh_CN/user_guide/tutorial/rewards.md
Normal file
@@ -0,0 +1,50 @@
|
||||
# 奖励函数设计
|
||||
|
||||
奖励函数告诉智能体什么样的行为是期望的，是强化学习环境设计中的核心部分。
|
||||
|
||||
## 奖励函数在训练循环中的位置
|
||||
|
||||
在 MotrixLab 的 NpEnv 中，奖励计算发生在 `step` 函数的 `update_state` 阶段：
|
||||
|
||||
```python
|
||||
# NpEnv.step() 的执行流程
|
||||
def step(self, actions: np.ndarray) -> NpEnvState:
|
||||
# 1. 准备阶段：清空奖励和状态
|
||||
self._prev_physics_step() # reward = 0.0, terminated = False, truncated = False
|
||||
|
||||
# 2. 应用动作
|
||||
self._state = self.apply_action(actions, self._state)
|
||||
|
||||
# 3. 物理仿真
|
||||
self.physics_step() # 执行物理仿真
|
||||
|
||||
# 4. 更新状态 ← 奖励函数在这里计算
|
||||
self._state = self.update_state(self._state) # 计算奖励和观察值
|
||||
|
||||
# 5. 后续处理
|
||||
self._update_truncate() # 检查时间截断
|
||||
self._reset_done_envs() # 重置完成的环境
|
||||
|
||||
return self._state
|
||||
```
|
||||
|
||||
您需要在子类的 `update_state` 方法中实现奖励计算逻辑，具体奖励函数设计思路请参考训练示例。
|
||||
|
||||
### 奖励组件设计原则
|
||||
|
||||
1. **分离关注点**：每个奖励函数负责一个特定的目标
|
||||
2. **权重配置**：通过配置文件管理不同组件的权重
|
||||
3. **归一化**：保持奖励值在合理的范围内
|
||||
4. **平滑性**：避免硬性阈值，使用指数函数等平滑过渡
|
||||
|
||||
这种方法使得奖励函数模块化，便于调试和调整各个组件的权重。
|
||||
|
||||
## 设计原则
|
||||
|
||||
1. **明确的目标导向**：奖励函数应该直接反映任务目标
|
||||
2. **合理的奖励范围**：避免过大或过小的奖励值，保持训练稳定
|
||||
3. **平衡探索与利用**：适当奖励接近目标的行为，避免稀疏奖励
|
||||
4. **避免奖励漏洞**：检查智能体是否可能通过不期望的方式获得高奖励
|
||||
5. **调试友好**：在开发阶段输出奖励分解信息，便于调优
|
||||
|
||||
通过在 `update_state` 方法中正确实现奖励计算，您可以为各种机器人任务设计有效的学习信号。
|
||||
79
docs/source/zh_CN/user_guide/tutorial/training_and_result.md
Normal file
79
docs/source/zh_CN/user_guide/tutorial/training_and_result.md
Normal file
@@ -0,0 +1,79 @@
|
||||
# 训练执行和结果分析
|
||||
|
||||
本节介绍如何执行强化学习训练，以及如何分析和使用训练结果。
|
||||
|
||||
## 启动训练
|
||||
|
||||
### 基本训练命令
|
||||
|
||||
```bash
|
||||
# 使用默认参数训练
|
||||
uv run scripts/train.py --env cartpole
|
||||
|
||||
# 指定仿真后端
|
||||
uv run scripts/train.py --env cartpole --sim-backend np
|
||||
|
||||
# 指定训练后端
|
||||
uv run scripts/train.py --env cartpole --train-backend jax
|
||||
uv run scripts/train.py --env cartpole --train-backend torch
|
||||
```
|
||||
|
||||
### 高级训练配置
|
||||
|
||||
```bash
|
||||
# 自定义训练参数
|
||||
uv run scripts/train.py --env cartpole \
|
||||
--num-envs 1024 \
|
||||
--train-backend jax \
|
||||
--sim-backend np
|
||||
|
||||
# 注意：学习率等参数需要通过配置文件或代码覆盖设置
|
||||
|
||||
# 启用渲染监控训练过程
|
||||
uv run scripts/train.py --env cartpole --render
|
||||
```
|
||||
|
||||
### 支持的命令行参数
|
||||
|
||||
| 参数 | 说明 | 默认值 |
|
||||
| ----------------- | -------------------- | ---------- |
|
||||
| `--env` | 环境名称 | `cartpole` |
|
||||
| `--sim-backend` | 仿真后端 (np) | 自动选择 |
|
||||
| `--train-backend` | 训练后端 (jax/torch) | 自动选择 |
|
||||
| `--num-envs` | 并行环境数量 | 2048 |
|
||||
| `--render` | 启用渲染 | False |
|
||||
|
||||
> **注意**: 其他参数如学习率、网络结构等需要通过单独文件设置。
|
||||
|
||||
## 训练过程监控
|
||||
|
||||
### TensorBoard 监控
|
||||
|
||||
启动 TensorBoard 查看训练进度：
|
||||
|
||||
```bash
|
||||
uv run tensorboard --logdir runs/{env-name}
|
||||
```
|
||||
|
||||
例如：
|
||||
|
||||
```bash
|
||||
uv run tensorboard --logdir runs/cartpole
|
||||
```
|
||||
|
||||
## 模型评估和测试
|
||||
|
||||
### 使用训练好的策略
|
||||
|
||||
```bash
|
||||
# 自动寻找最佳策略测试（推荐）
|
||||
uv run scripts/play.py --env cartpole
|
||||
|
||||
# 手动指定策略文件测试
|
||||
uv run scripts/play.py --env cartpole --policy runs/cartpole/nn/best_agent.pickle
|
||||
|
||||
# 指定测试环境数量
|
||||
uv run scripts/play.py --env cartpole --num-envs 100
|
||||
```
|
||||
|
||||
> **说明**：系统会自动在 `runs/cartpole/` 目录下寻找最新、最佳的策略文件进行测试。
|
||||
@@ -0,0 +1,178 @@
|
||||
# 训练环境配置
|
||||
|
||||
MotrixLab 提供了灵活的配置系统，允许用户自定义强化学习训练参数。本节介绍如何配置训练环境和强化学习算法参数。
|
||||
|
||||
## RL 训练配置 (PPOCfg)
|
||||
|
||||
训练配置定义了基于 PPO 算法的强化学习算法的参数。MotrixLab 现在支持为不同训练后端配置不同的参数。
|
||||
|
||||
### 完整配置示例
|
||||
|
||||
```python
|
||||
@dataclass
|
||||
class CompletePPOConfig(PPOCfg):
|
||||
"""
|
||||
完整的强化学习训练配置示例
|
||||
包含了从基础到高级的所有配置参数
|
||||
"""
|
||||
|
||||
# ===== 基础训练参数 =====
|
||||
seed: Optional[int] = None # 随机种子
|
||||
num_envs: int = 2048 # 训练时并行环境数量
|
||||
play_num_envs: int = 16 # 评估时并行环境数量
|
||||
max_env_steps: int = 2_048_000 # 最大训练步数
|
||||
check_point_interval: int = 1000 # 检查点保存间隔
|
||||
|
||||
# ===== PPO算法核心参数 =====
|
||||
learning_rate: float = 3e-4 # 学习率
|
||||
rollouts: int = 32 # 经验回放轮数
|
||||
learning_epochs: int = 2 # 每次更新的训练轮数
|
||||
mini_batches: int = 32 # 小批量数量
|
||||
discount_factor: float = 0.99 # 折扣因子
|
||||
lambda_param: float = 0.95 # GAE参数
|
||||
grad_norm_clip: float = 1.0 # 梯度裁剪
|
||||
|
||||
# ===== PPO裁剪参数 =====
|
||||
ratio_clip: float = 0.2 # PPO裁剪比率
|
||||
value_clip: float = 0.2 # 价值裁剪
|
||||
clip_predicted_values: bool = True # 裁剪预测值
|
||||
|
||||
# ===== 损失函数参数 =====
|
||||
entropy_loss_scale: float = 0.0 # 熵损失系数
|
||||
value_loss_scale: float = 2.0 # 价值损失系数
|
||||
kl_threshold: float = 0 # KL散度阈值
|
||||
|
||||
# ===== 学习率调度器 =====
|
||||
learning_rate_scheduler_kl_threshold: float = 0.008 # 自适应学习率KL阈值
|
||||
|
||||
# ===== 网络架构配置 =====
|
||||
# 小型网络（适合简单任务如 CartPole）
|
||||
# policy_hidden_layer_sizes: tuple[int, ...] = (128, 64)
|
||||
# value_hidden_layer_sizes: tuple[int, ...] = (128, 64)
|
||||
|
||||
# 中型网络（默认配置，适合大部分任务）
|
||||
policy_hidden_layer_sizes: tuple[int, ...] = (256, 128, 64)
|
||||
value_hidden_layer_sizes: tuple[int, ...] = (256, 128, 64)
|
||||
|
||||
# 大型网络（适合复杂任务如机器人控制）
|
||||
# policy_hidden_layer_sizes: tuple[int, ...] = (512, 256, 128)
|
||||
# value_hidden_layer_sizes: tuple[int, ...] = (512, 256, 128)
|
||||
|
||||
# ===== 网络共享配置 =====
|
||||
share_policy_value_features: bool = True # 策略和价值网络共享特征提取层
|
||||
|
||||
# ===== 训练控制参数 =====
|
||||
random_timesteps: int = 0 # 随机步数
|
||||
learning_starts: int = 0 # 开始学习的步数
|
||||
time_limit_bootstrap: bool = True # 时间限制引导
|
||||
|
||||
# ===== 奖励整形 =====
|
||||
rewards_shaper_scale: float = 1.0 # 奖励缩放因子
|
||||
```
|
||||
|
||||
## 配置使用方法
|
||||
|
||||
### 1. 默认配置使用
|
||||
|
||||
```bash
|
||||
# 使用代码中给定的配置
|
||||
uv run scripts/train.py --env my-task
|
||||
|
||||
# 指定训练后端，系统会自动选择对应的后端配置
|
||||
uv run scripts/train.py --env my-task --train-backend jax
|
||||
uv run scripts/train.py --env my-task --train-backend torch
|
||||
```
|
||||
|
||||
### 2. 命令行参数覆盖
|
||||
|
||||
```bash
|
||||
# 覆盖支持的命令行参数
|
||||
uv run scripts/train.py --env my-task \
|
||||
--num-envs 1024 \
|
||||
--train-backend jax \
|
||||
--sim-backend np
|
||||
|
||||
# 系统会自动选择JAX后端对应的配置
|
||||
```
|
||||
|
||||
### 3. 配置优先级
|
||||
|
||||
系统按以下优先级选择配置：
|
||||
|
||||
1. **后端特定配置**: 如果存在 `@rlcfg(env_name, backend="jax/torch")` 装饰的配置
|
||||
2. **通用配置**: 如果存在 `@rlcfg(env_name)` 装饰的配置（无 backend 参数）
|
||||
|
||||
例如：
|
||||
|
||||
```python
|
||||
# 最高优先级 - 后端特定配置
|
||||
@rlcfg("my-task", backend="jax")
|
||||
@dataclass
|
||||
class MyTaskJAXCfg(PPOCfg):
|
||||
mini_batches: int = 4
|
||||
|
||||
# 次优先级 - 通用配置
|
||||
@rlcfg("my-task")
|
||||
@dataclass
|
||||
class MyTaskRLCfg(PPOCfg):
|
||||
mini_batches: int = 32
|
||||
|
||||
# 当使用 --train-backend jax 时，系统会选择 MyTaskJAXCfg
|
||||
# 当使用 --train-backend torch 时，系统会选择 MyTaskRLCfg
|
||||
```
|
||||
|
||||
## SKRL 框架配置映射
|
||||
|
||||
在 MotrixLab 中，用户通过 `PPOCfg` 配置类设置参数，这些参数会被映射到 SKRL 框架的配置字典中。
|
||||
|
||||
### 用户可配置参数
|
||||
|
||||
| MotrixLab 配置类 | SKRL 框架参数 | 说明 |
|
||||
| -------------------------------------- | --------------------------------------------- | -------------------- |
|
||||
| `learning_rate` | `learning_rate` | 学习率 |
|
||||
| `rollouts` | `rollouts` | 经验回放轮数 |
|
||||
| `learning_epochs` | `learning_epochs` | 训练轮数 |
|
||||
| `mini_batches` | `mini_batches` | 小批量数量 |
|
||||
| `discount_factor` | `discount_factor` | 折扣因子 |
|
||||
| `grad_norm_clip` | `grad_norm_clip` | 梯度裁剪 |
|
||||
| `lambda_param` | `lambda` | GAE 参数 |
|
||||
| `ratio_clip` | `ratio_clip` | PPO 裁剪比率 |
|
||||
| `value_clip` | `value_clip` | 价值裁剪 |
|
||||
| `clip_predicted_values` | `clip_predicted_values` | 裁剪预测值 |
|
||||
| `entropy_loss_scale` | `entropy_loss_scale` | 熵损失系数 |
|
||||
| `value_loss_scale` | `value_loss_scale` | 价值损失系数 |
|
||||
| `kl_threshold` | `kl_threshold` | KL 散度阈值 |
|
||||
| `random_timesteps` | `random_timesteps` | 随机步数 |
|
||||
| `learning_starts` | `learning_starts` | 开始学习的步数 |
|
||||
| `time_limit_bootstrap` | `time_limit_bootstrap` | 时间限制引导 |
|
||||
| `learning_rate_scheduler_kl_threshold` | `learning_rate_scheduler_kwargs.kl_threshold` | 自适应学习率 KL 阈值 |
|
||||
| `check_point_interval` | `experiment.write_interval` | 日志写入间隔 |
|
||||
| `check_point_interval` | `experiment.checkpoint_interval` | 检查点保存间隔 |
|
||||
| `rewards_shaper_scale` | `rewards_shaper` | 奖励缩放函数 |
|
||||
|
||||
### 预处理器参数
|
||||
|
||||
| SKRL 框架参数 | 类型 | 说明 |
|
||||
| -------------------- | --------------------- | ---------- |
|
||||
| `state_preprocessor` | RunningStandardScaler | 状态标准化 |
|
||||
| `value_preprocessor` | RunningStandardScaler | 价值标准化 |
|
||||
|
||||
### 配置层次总结
|
||||
|
||||
```
|
||||
用户配置类 (PPOCfg)
|
||||
↓ 后端特定选择
|
||||
后端配置 (JAX/Torch)
|
||||
↓ 参数映射
|
||||
SKRL 框架配置字典
|
||||
↓ 传递给
|
||||
PPO Agent
|
||||
↓ 执行
|
||||
强化学习训练
|
||||
```
|
||||
|
||||
这种设计允许用户：
|
||||
|
||||
1. 通过简单的配置类来控制复杂的训练参数
|
||||
2. 为不同训练后端配置不同的参数以获得最佳性能
|
||||
3. 保持与 SKRL 框架的完全兼容性
|
||||
Reference in New Issue
Block a user