[Algorithm] GAIL

.github/unittest/linux/scripts/environment.yml

@@ -24,7 +24,8 @@ dependencies:
     - tensorboard
     - imageio==2.26.0
     - wandb
-    - dm_control
+    - dm_control<1.0.21
+    - mujoco<3.2.1
     - mlflow
     - av
     - coverage

.github/unittest/linux/scripts/run_all.sh

@@ -91,7 +91,7 @@ echo "installing gymnasium"
 pip3 install "gymnasium"
 pip3 install ale_py
 pip3 install mo-gymnasium[mujoco]  # requires here bc needs mujoco-py
-pip3 install mujoco -U
+pip3 install "mujoco<3.2.1" -U
 
 # sanity check: remove?
 python3 -c """

.github/unittest/linux_distributed/scripts/environment.yml

@@ -23,7 +23,8 @@ dependencies:
     - tensorboard
     - imageio==2.26.0
     - wandb
-    - dm_control
+    - dm_control<1.0.21
+    - mujoco<3.2.1
     - mlflow
     - av
     - coverage

.github/unittest/linux_examples/scripts/environment.yml

@@ -21,7 +21,8 @@ dependencies:
     - scipy
     - hydra-core
     - imageio==2.26.0
-    - dm_control
+    - dm_control<1.0.21
+    - mujoco<3.2.1
     - mlflow
     - av
     - coverage

.github/unittest/linux_examples/scripts/run_test.sh

@@ -205,6 +205,13 @@ python .github/unittest/helpers/coverage_run_parallel.py sota-implementations/iq
   env.train_num_envs=2 \
   logger.mode=offline \
   logger.backend=
+  python .github/unittest/helpers/coverage_run_parallel.py sota-implementations/gail/gail.py \
+  ppo.collector.total_frames=48 \
+  replay_buffer.batch_size=16 \
+  ppo.loss.mini_batch_size=10 \
+  ppo.collector.frames_per_batch=16 \
+  logger.mode=offline \
+  logger.backend=
 
 # With single envs
 python .github/unittest/helpers/coverage_run_parallel.py sota-implementations/dreamer/dreamer.py \

.github/unittest/linux_libs/scripts_envpool/environment.yml

@@ -18,5 +18,6 @@ dependencies:
     - expecttest
     - pyyaml
     - scipy
-    - dm_control
+    - dm_control<1.0.21
+    - mujoco<3.2.1
     - coverage

.github/unittest/linux_olddeps/scripts_gym_0_13/environment.yml

@@ -22,6 +22,7 @@ dependencies:
     - scipy
     - hydra-core
     - dm_control -e git+https://github.com/deepmind/dm_control.git@c053360edea6170acfd9c8f65446703307d9d352#egg={dm_control}
+    - mujoco<3.2.1
     - patchelf
     - pyopengl==3.1.4
     - ray

.github/workflows/benchmarks.yml

@@ -35,7 +35,7 @@ jobs:
           python3 setup.py develop
           python3 -m pip install pytest pytest-benchmark
           python3 -m pip install "gym[accept-rom-license,atari]"
-          python3 -m pip install dm_control
+          python3 -m pip install "dm_control<1.0.21" "mujoco<3.2.1"
           export TD_GET_DEFAULTS_TO_NONE=1
       - name: Run benchmarks
         run: |
@@ -97,7 +97,7 @@ jobs:
           python3 setup.py develop
           python3 -m pip install pytest pytest-benchmark
           python3 -m pip install "gym[accept-rom-license,atari]"
-          python3 -m pip install dm_control
+          python3 -m pip install "dm_control<1.0.21" "mujoco<3.2.1"
           export TD_GET_DEFAULTS_TO_NONE=1
       - name: check GPU presence
         run: |

.github/workflows/benchmarks_pr.yml

@@ -34,7 +34,7 @@ jobs:
           python3 setup.py develop
           python3 -m pip install pytest pytest-benchmark
           python3 -m pip install "gym[accept-rom-license,atari]"
-          python3 -m pip install dm_control
+          python3 -m pip install "dm_control<1.0.21" "mujoco<3.2.1"
           export TD_GET_DEFAULTS_TO_NONE=1
       - name: Setup benchmarks
         run: |
@@ -108,7 +108,7 @@ jobs:
           python3 setup.py develop
           python3 -m pip install pytest pytest-benchmark
           python3 -m pip install "gym[accept-rom-license,atari]"
-          python3 -m pip install dm_control
+          python3 -m pip install "dm_control<1.0.21" "mujoco<3.2.1"
           export TD_GET_DEFAULTS_TO_NONE=1
       - name: check GPU presence
         run: |

docs/requirements.txt

@@ -14,7 +14,8 @@ docutils
 sphinx_design
 
 torchvision
-dm_control
+dm_control<1.0.21
+mujoco<3.2.1
 atari-py
 ale-py
 gym[classic_control,accept-rom-license]

docs/source/reference/objectives.rst

@@ -179,6 +179,15 @@ CQL
     CQLLoss
     DiscreteCQLLoss
 
+GAIL
+----
+
+.. autosummary::
+    :toctree: generated/
+    :template: rl_template_noinherit.rst
+
+    GAILLoss
+
 DT
 ----
 

sota-implementations/gail/config.yaml

@@ -0,0 +1,46 @@
+env:
+  env_name: HalfCheetah-v4
+  seed: 42
+  backend: gymnasium
+
+logger:
+  backend: wandb
+  project_name: gail
+  group_name: null
+  exp_name: gail_ppo
+  test_interval: 5000
+  num_test_episodes: 5
+  video: False
+  mode: online
+
+ppo:
+  collector:
+    frames_per_batch: 2048
+    total_frames: 1_000_000
+
+  optim:
+    lr: 3e-4
+    weight_decay: 0.0
+    anneal_lr: True
+
+  loss:
+    gamma: 0.99
+    mini_batch_size: 64
+    ppo_epochs: 10
+    gae_lambda: 0.95
+    clip_epsilon: 0.2
+    anneal_clip_epsilon: False
+    critic_coef: 0.25
+    entropy_coef: 0.0
+    loss_critic_type: l2
+
+gail:
+  hidden_dim: 128
+  lr: 3e-4
+  use_grad_penalty: False
+  gp_lambda: 10.0
+  device: null
+
+replay_buffer:
+  dataset: halfcheetah-expert-v2
+  batch_size: 256