diff --git a/.readthedocs.yml b/.readthedocs.yml
index 6753d8744..dbb2fad03 100644
--- a/.readthedocs.yml
+++ b/.readthedocs.yml
@@ -14,3 +14,8 @@ formats: all
 # Set requirements using conda env
 conda:
   environment: docs/conda_env.yml
+
+build:
+  os: ubuntu-22.04
+  tools:
+    python: "mambaforge-22.9"
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 2c06fa5c6..4c9d601e4 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,7 +1,9 @@
-## Release 2.2.0a4 (WIP)
+## Release 2.2.0a8 (WIP)
 
 ### Breaking Changes
 - Removed `gym` dependency, the package is still required for some pretrained agents.
+- Upgraded to SB3 >= 2.2.0
+- Upgraded to Huggingface-SB3 >= 3.0
 
 ### New Features
 - Add `--eval-env-kwargs` to `train.py` (@Quentin18)
diff --git a/docs/conda_env.yml b/docs/conda_env.yml
index 9b0c845bf..0e01034c1 100644
--- a/docs/conda_env.yml
+++ b/docs/conda_env.yml
@@ -13,13 +13,12 @@ dependencies:
     - pandas
     - numpy
     - matplotlib
-    - sphinx_autodoc_typehints
-    - sphinx>=5.3,<7.0
-    - sphinx_rtd_theme>=1.0
+    - sphinx>=5,<8
+    - sphinx_rtd_theme>=1.3.0
     - sphinx_copybutton
     - sb3-contrib>=2.0.0
     - optuna
-    - huggingface_sb3>=2.2.1
+    - huggingface_sb3>=3.0
     - rich
     - tqdm
     - pyyaml>=5.1
diff --git a/docs/conf.py b/docs/conf.py
index 804269d0d..aa23918a7 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -64,7 +64,6 @@
 # ones.
 extensions = [
     "sphinx.ext.autodoc",
-    "sphinx_autodoc_typehints",
     "sphinx.ext.autosummary",
     "sphinx.ext.mathjax",
     "sphinx.ext.ifconfig",
@@ -73,6 +72,8 @@
     # 'sphinx.ext.doctest'
 ]
 
+autodoc_typehints = "description"
+
 if enable_spell_check:
     extensions.append("sphinxcontrib.spelling")
 
diff --git a/requirements.txt b/requirements.txt
index 50f20bbad..00bb83af8 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,6 +1,6 @@
 gym==0.26.2
-stable-baselines3[extra_no_roms,tests,docs]>=2.1.0
-sb3-contrib>=2.1.0
+stable-baselines3[extra_no_roms,tests,docs]>=2.2.0a8,<3.0
+sb3-contrib>=2.2.0a8,<3.0
 box2d-py==2.3.8
 pybullet
 pybullet_envs_gymnasium
@@ -15,7 +15,7 @@ plotly
 # panda-gym~=3.0.1
 rliable>=1.0.5
 wandb
-huggingface_sb3>=2.3
+huggingface_sb3>=3.0,<4.0
 seaborn
 tqdm
 rich
diff --git a/rl_zoo3/version.txt b/rl_zoo3/version.txt
index ddcf0926b..f1f23b30a 100644
--- a/rl_zoo3/version.txt
+++ b/rl_zoo3/version.txt
@@ -1 +1 @@
-2.2.0a4
+2.2.0a8
diff --git a/setup.py b/setup.py
index be0052093..e776e929c 100644
--- a/setup.py
+++ b/setup.py
@@ -27,9 +27,9 @@
     },
     entry_points={"console_scripts": ["rl_zoo3=rl_zoo3.cli:main"]},
     install_requires=[
-        "sb3_contrib>=2.1.0",
+        "sb3_contrib>=2.2.0a8,<3.0",
         "gymnasium~=0.29.1",
-        "huggingface_sb3>=2.3",
+        "huggingface_sb3>=3.0,<4.0",
         "tqdm",
         "rich",
         "optuna>=3.0",
@@ -38,7 +38,7 @@
         # TODO: add test dependencies
     ],
     extras_require={
-        "plots": ["seaborn", "rliable>=1.0.5", "scipy~=1.7.3"],
+        "plots": ["seaborn", "rliable>=1.0.5", "scipy~=1.10"],
     },
     description="A Training Framework for Stable Baselines3 Reinforcement Learning Agents",
     author="Antonin Raffin",