init commit

drdh · Oct 27, 2022 · d432681 · d432681
commit d432681
Show file tree

Hide file tree

Showing 113 changed files with 9,726 additions and 0 deletions.
diff --git a/.idea/.gitignore b/.idea/.gitignore
diff --git a/.idea/Synergy-RL.iml b/.idea/Synergy-RL.iml
diff --git a/.idea/inspectionProfiles/Project_Default.xml b/.idea/inspectionProfiles/Project_Default.xml
diff --git a/.idea/inspectionProfiles/profiles_settings.xml b/.idea/inspectionProfiles/profiles_settings.xml
diff --git a/.idea/misc.xml b/.idea/misc.xml
diff --git a/.idea/modules.xml b/.idea/modules.xml
diff --git a/.idea/vcs.xml b/.idea/vcs.xml
diff --git a/LICENSE b/LICENSE
@@ -0,0 +1,127 @@
+MIT License
+
+Copyright (c) 2022 Heng Dong
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+
+--------------------------------------------------------------------------------
+Original Amorpheus License:
+--------------------------------------------------------------------------------
+
+
+MIT License
+
+Copyright (c) 2021 Vitaly Kurin
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+--------------------------------------------------------------------------------
+Original SMP License:
+--------------------------------------------------------------------------------
+
+MIT License
+
+Copyright (c) 2020 Wenlong Huang, Deepak Pathak
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+--------------------------------------------------------------------------------
+Original TD3 License:
+--------------------------------------------------------------------------------
+MIT License
+
+Copyright (c) 2020 Scott Fujimoto
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+--------------------------------------------------------------------------------
+Original DGN License
+--------------------------------------------------------------------------------
+Copyright (c) 2019 Deepak Pathak, Chris Lu
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+* Redistributions of source code must retain the above copyright notice, this
+  list of conditions and the following disclaimer.
+
+* Redistributions in binary form must reproduce the above copyright notice,
+  this list of conditions and the following disclaimer in the documentation
+  and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
diff --git a/README.md b/README.md
@@ -0,0 +1,44 @@
+## Low-Rank Modular Reinforcement Learning via Muscle Synergy
+
+This repository is the official implementation of [Low-Rank Modular Reinforcement Learning via Muscle Synergy](https://openreview.net/pdf?id=zYc5FSxL6ar).
+
+## Setup
+
+```bash
+pip install -r requirements.txt
+```
+
+## Training
+
+```
+python main.py 
+--custom_xml environments/humanoids 
+--actor_type transformer 
+--critic_type transformer 
+--grad_clipping_value 0.1 
+--attention_layers 3 
+--attention_heads 2 
+--lr 0.0001 
+--transformer_norm 1 
+--attention_hidden_size 256 
+--condition_decoder 1
+--label SOLAR
+--save_buffer 1
+--save_freq 100000
+--start_timesteps 2000
+--enable_synergy_obs 1\
+--enable_synergy_act 1\
+--synergy_action_dim 1\
+--d_model 1\
+```
+
+We were using [Sacred](https://github.com/IDSIA/sacred) and [WandB](https://wandb.ai) for experiment management. 
+You can disable WandB by setting `--enable_wandb 0`.  
+
+To train other environments, change the `--custom_xml` to `hoppers`, `walkers`, 
+`walker_humanoids_hopper`.
+
+## Acknowledgement
+
+- The code is built on top of [Amorpheus](https://github.com/yobibyte/amorpheus) and [SMP](https://github.com/huangwl18/modular-rl) repository.
+- Initial implementation of the transformers was taken from the [official Pytorch tutorial](https://pytorch.org/tutorials/beginner/transformer_tutorial.html) and modified thereafter. 
diff --git a/__init__.py b/__init__.py
diff --git a/modular-rl/.gitignore b/modular-rl/.gitignore
@@ -0,0 +1,20 @@
+mrEnv
+**/__pycache__
+**pyc
+**npy
+**DS_Store
+**MUJOCO_LOG.TXT
+**/.ipynb_checkpoints
+src/baselines/
+src/gym/
+src/trained_models/
+src/SaveData/
+src/temp
+src/results
+src/runs
+src/pytorch_models
+src/temp.txt
+.idea
+wandb/
+analysis/data/
+analysis/figure/
diff --git a/modular-rl/__init__.py b/modular-rl/__init__.py
diff --git a/modular-rl/requirements.txt b/modular-rl/requirements.txt
@@ -0,0 +1,54 @@
+absl-py==0.9.0
+asn1crypto==0.24.0
+astor==0.8.1
+cffi==1.14.0
+cloudpickle==1.4.1
+cryptography==2.1.4
+future==0.18.2
+gast==0.2.2
+glfw==2.5.0
+google-pasta==0.2.0
+grpcio==1.28.1
+gym==0.18.3
+h5py==2.10.0
+horovod==0.19.1
+idna==2.6
+imageio==2.5.0
+Keras-Applications==1.0.8
+Keras-Preprocessing==1.1.0
+keyring==10.6.0
+keyrings.alt==3.0
+llvmlite==0.36.0
+lockfile==0.12.2
+lxml==4.8.0
+Markdown==3.2.1
+Cython==0.29.26
+numba==0.53.0
+numpy==1.18.3
+Pillow==8.1.2
+protobuf==3.17.0
+psutil==5.7.0
+pycparser==2.20
+pycrypto==2.6.1
+pyglet==1.5.15
+PyGObject==3.26.1
+pyxdg==0.25
+PyYAML==5.3.1
+pymongo==4.1.1
+sacred==0.8.2
+scipy==1.5.4
+SecretStorage==2.3.1
+six==1.16.0
+tensorboard==1.15.0
+tensorboardX==1.8
+tensorflow-estimator==1.15.1
+termcolor==1.1.0
+torch==1.2.0
+torchvision==0.4.0
+torchfold==0.1.0
+Werkzeug==1.0.1
+wrapt==1.12.1
+wandb==0.12.11
+xmltodict==0.12.0
+Cython==0.29.26
+sklearn
diff --git a/modular-rl/src/__init__.py b/modular-rl/src/__init__.py