Add Gym 0.26 support

.github/ISSUE_TEMPLATE/custom_env.yml

@@ -49,15 +49,16 @@ body:
                 self.observation_space = spaces.Box(low=-np.inf, high=np.inf, shape=(14,))
                 self.action_space = spaces.Box(low=-1, high=1, shape=(6,))
 
-            def reset(self):
-                return self.observation_space.sample()
+            def reset(self, seed=None):
+                return self.observation_space.sample(), {}
 
             def step(self, action):
                 obs = self.observation_space.sample()
                 reward = 1.0
-                done = False
+                terminated = False
+                truncated = False
                 info = {}
-                return obs, reward, done, info
+                return obs, reward, terminated, truncated, info
 
         env = CustomEnv()
         check_env(env)

.gitlab-ci.yml

@@ -1,4 +1,4 @@
-image: stablebaselines/stable-baselines3-cpu:1.4.1a0
+image: stablebaselines/stable-baselines3-cpu:1.5.1a6
 
 type-check:
   script:

Dockerfile

@@ -3,6 +3,9 @@ FROM $PARENT_IMAGE
 ARG PYTORCH_DEPS=cpuonly
 ARG PYTHON_VERSION=3.7
 
+# for tzdata
+ENV DEBIAN_FRONTEND="noninteractive" TZ="Europe/Paris"
+
 RUN apt-get update && apt-get install -y --no-install-recommends \
          build-essential \
          cmake \
@@ -20,7 +23,7 @@ RUN curl -o ~/miniconda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest
      ~/miniconda.sh -b -p /opt/conda && \
      rm ~/miniconda.sh && \
      /opt/conda/bin/conda install -y python=$PYTHON_VERSION numpy pyyaml scipy ipython mkl mkl-include && \
-     /opt/conda/bin/conda install -y pytorch $PYTORCH_DEPS -c pytorch && \
+     /opt/conda/bin/conda install -y pytorch=1.11 $PYTORCH_DEPS -c pytorch && \
      /opt/conda/bin/conda clean -ya
 ENV PATH /opt/conda/bin:$PATH
 

Makefile

@@ -10,6 +10,12 @@ pytype:
 mypy:
 	mypy ${LINT_PATHS}
 
+missing-annotations:
+	mypy --disallow-untyped-calls --disallow-untyped-defs --ignore-missing-imports stable_baselines3
+
+# missing docstrings
+# pylint -d R,C,W,E -e C0116 stable_baselines3 -j 4
+
 type: pytype mypy
 
 lint:

docs/conda_env.yml

@@ -4,11 +4,11 @@ channels:
   - defaults
 dependencies:
   - cpuonly=1.0=0
-  - pip=21.1
+  - pip=22.1.1
   - python=3.7
-  - pytorch=1.11=py3.7_cpu_0
+  - pytorch=1.11.0=py3.7_cpu_0
   - pip:
-    - gym==0.21
+    - gym==0.26
     - cloudpickle
     - opencv-python-headless
     - pandas

docs/guide/checking_nan.rst

@@ -129,7 +129,7 @@ It will monitor the actions, observations, and rewards, indicating what action o
       def reset(self):
           return [0.0]
 
-      def render(self, mode="human", close=False):
+      def render(self, close=False):
           pass
 
   # Create environment

docs/guide/custom_env.rst

@@ -54,7 +54,7 @@ That is to say, your environment must implement the following methods (and inher
           ...
           return observation  # reward, done, info can't be included
 
-      def render(self, mode="human"):
+      def render(self):
           ...
 
       def close(self):

docs/guide/examples.rst

@@ -471,19 +471,19 @@ The parking env is a goal-conditioned continuous control task, in which the vehi
   # HER must be loaded with the env
   model = SAC.load("her_sac_highway", env=env)
 
-  obs = env.reset()
+  obs, info = env.reset()
 
   # Evaluate the agent
   episode_reward = 0
   for _ in range(100):
       action, _ = model.predict(obs, deterministic=True)
-      obs, reward, done, info = env.step(action)
+      obs, reward, done, truncated, info = env.step(action)
       env.render()
       episode_reward += reward
-      if done or info.get("is_success", False):
+      if done or truncated or info.get("is_success", False):
           print("Reward:", episode_reward, "Success?", info.get("is_success", False))
           episode_reward = 0.0
-          obs = env.reset()
+          obs, info = env.reset()
 
 
 Learning Rate Schedule

docs/guide/quickstart.rst

@@ -4,6 +4,12 @@
 Getting Started
 ===============
 
+.. note::
+
+  Stable-Baselines3 (SB3) uses :ref:`vectorized environments (VecEnv) <vec_env>` internally.
+  Please read the associated section to learn more about its features and differences compared to a single Gym environment.
+
+
 Most of the library tries to follow a sklearn-like syntax for the Reinforcement Learning algorithms.
 
 Here is a quick example of how to train and run A2C on a CartPole environment:

docs/guide/vec_envs.rst

@@ -44,6 +44,38 @@ SubprocVecEnv ✔️       ✔️           ✔️        ✔️         ✔️
     For more information, see Python's `multiprocessing guidelines <https://docs.python.org/3/library/multiprocessing.html#the-spawn-and-forkserver-start-methods>`_.
 
 
+VecEnv API vs Gym API
+---------------------
+
+For consistency across Stable-Baselines3 (SB3) versions and because of its special requirements and features,
+SB3 VecEnv API is not the same as Gym API.
+SB3 VecEnv API is actually close to Gym 0.21 API but differs to Gym 0.26+ API:
+
+- the ``reset()`` method only returns the observation (``obs = vec_env.reset()``) and not a tuple, the info at reset are stored in ``vec_env.reset_infos``.
+- only the initial call to ``vec_env.reset()`` is required, environments are reset automatically afterward (and ``reset_infos`` is updated automatically).
+- the ``vec_env.step(actions)`` method expects an array as input
+  (with a batch size corresponding to the number of environments) and returns a 4-tuple (and not a 5-tuple): ``obs, rewards, dones, infos`` instead of ``obs, reward, terminated, truncated, info``
+  where ``dones = terminated or truncated`` (for each env).
+  ``obs, rewards, dones`` are numpy arrays with shape ``(n_envs, shape_for_single_env)`` (so with a batch dimension).
+  Additional information is passed via the ``infos`` value which is a list of dictionaries.
+- at the end of an episode, ``infos[env_idx]["TimeLimit.truncated"] = truncated and not terminated``
+  tells the user if an episode was truncated or not:
+  you should bootstrap when ``infos[env_idx]["TimeLimit.truncated"] is True`` or ``dones[env_idx] is False``.
+  Note: compared to Gym 0.26+ ``infos[env_idx]["TimeLimit.truncated"]`` and ``terminated`` `are mutually exclusive <https://github.com/openai/gym/issues/3102>`_.
+- at the end of an episode, because the environment resets automatically,
+  we provide ``infos[env_idx]["terminal_observation"]`` which contains the last observation
+  of an episode (and can be used when bootstrapping, see note in the previous section)
+- if you pass ``render_mode="rgb_array"`` to your Gym env, a corresponding VecEnv can automatically show the rendered image
+  by calling ``vec_env.render(mode="human")``. This is different from Gym which currently `doesn't allow multiple render modes <https://github.com/Farama-Foundation/Gymnasium/issues/100>`_
+  and doesn't allow passing a ``mode`` parameter. Note that if ``render_mode != "rgb_array"``, you can only call ``vec_env.render()`` (without argument or with ``mode=env.render_mode``).
+- the ``reset()`` method doesn't take any parameter. If you want to seed the pseudo-random generator,
+  you should call ``vec_env.seed(seed=seed)`` and ``obs = vec_env.reset()`` afterward.
+  If your Gym env implements a ``seed()`` method then it will be called,
+  otherwise ``env.reset(seed=seed)`` will be called (in that case, you will need two resets to set the seed).
+- methods and attributes of the underlying Gym envs can be accessed, called and set using ``vec_env.get_attr("attribute_name")``,
+  ``vec_env.env_method("method_name", args1, args2, kwargs1=kwargs1)`` and ``vec_env.set_attr("attribute_name", new_value)``.
+
+
 Vectorized Environments Wrappers
 --------------------------------
 

docs/misc/changelog.rst

@@ -71,6 +71,7 @@ Breaking Changes:
   please use an ``EvalCallback`` instead
 - Removed deprecated ``sde_net_arch`` parameter
 - Removed ``ret`` attributes in ``VecNormalize``, please use ``returns`` instead
+- Switched minimum Gym version to 0.26 (@carlosluis, @arjun-kg, @tlpss)
 - ``VecNormalize`` now updates the observation space when normalizing images
 
 New Features:
@@ -254,6 +255,7 @@ Breaking Changes:
 
 New Features:
 ^^^^^^^^^^^^^
+- ``noop_max`` and ``frame_skip`` are now allowed to be equal to zero when using ``AtariWrapper``
 
 `SB3-Contrib`_
 ^^^^^^^^^^^^^^
@@ -279,6 +281,7 @@ Deprecations:
 Others:
 ^^^^^^^
 - Upgraded to Python 3.7+ syntax using ``pyupgrade``
+- Updated docker base image to Ubuntu 20.04 and cuda 11.3
 - Removed redundant double-check for nested observations from ``BaseAlgorithm._wrap_env`` (@TibiGG)
 
 Documentation:
@@ -300,7 +303,7 @@ Release 1.5.0 (2022-03-25)
 
 Breaking Changes:
 ^^^^^^^^^^^^^^^^^
-- Switched minimum Gym version to 0.21.0.
+- Switched minimum Gym version to 0.21.0
 
 New Features:
 ^^^^^^^^^^^^^
@@ -1225,6 +1228,7 @@ And all the contributors:
 @eleurent @ac-93 @cove9988 @theDebugger811 @hsuehch @Demetrio92 @thomasgubler @IperGiove @ScheiklP
 @simoninithomas @armandpl @manuel-delverme @Gautam-J @gianlucadecola @buoyancy99 @caburu @xy9485
 @Gregwar @ycheng517 @quantitative-technologies @bcollazo @git-thor @TibiGG @cool-RR @MWeltevrede
-@Melanol @qgallouedec @francescoluciano @jlp-ue @burakdmb @timothe-chaumont @honglu2875 @yuanmingqi
+@carlosluis @arjun-kg @tlpss
+@Melanol @qgallouedec @francescoluciano @jlp-ue @burakdmb @timothe-chaumont @honglu2875
 @anand-bala @hughperkins @sidney-tio @AlexPasqua @dominicgkerr @Akhilez @Rocamonde @tobirohrer @ZikangXiong
 @DavyMorgan @luizapozzobon @Bonifatius94 @theSquaredError

scripts/build_docker.sh

@@ -1,14 +1,14 @@
 #!/bin/bash
 
-CPU_PARENT=ubuntu:18.04
-GPU_PARENT=nvidia/cuda:10.1-cudnn7-runtime-ubuntu18.04
+CPU_PARENT=ubuntu:20.04
+GPU_PARENT=nvidia/cuda:11.3.1-base-ubuntu20.04
 
 TAG=stablebaselines/stable-baselines3
 VERSION=$(cat ./stable_baselines3/version.txt)
 
 if [[ ${USE_GPU} == "True" ]]; then
   PARENT=${GPU_PARENT}
-  PYTORCH_DEPS="cudatoolkit=10.1"
+  PYTORCH_DEPS="cudatoolkit=11.3"
 else
   PARENT=${CPU_PARENT}
   PYTORCH_DEPS="cpuonly"

setup.cfg

@@ -13,11 +13,8 @@ filterwarnings =
 		# Tensorboard warnings
 		ignore::DeprecationWarning:tensorboard
 		# Gym warnings
-		ignore:Parameters to load are deprecated.:DeprecationWarning
-		ignore:the imp module is deprecated in favour of importlib:PendingDeprecationWarning
 		ignore::UserWarning:gym
-		ignore:SelectableGroups dict interface is deprecated.:DeprecationWarning
-		ignore:`np.bool` is a deprecated alias for the builtin `bool`:DeprecationWarning
+    ignore::DeprecationWarning:.*passive_env_checker.*
 markers =
     expensive: marks tests as expensive (deselect with '-m "not expensive"')
 

setup.py

@@ -76,7 +76,7 @@
     packages=[package for package in find_packages() if package.startswith("stable_baselines3")],
     package_data={"stable_baselines3": ["py.typed", "version.txt"]},
     install_requires=[
-        "gym==0.21",  # Fixed version due to breaking changes in 0.22
+        "gym==0.26.2",
         "numpy",
         "torch>=1.11",
         'typing_extensions>=4.0,<5; python_version < "3.8.0"',
@@ -107,8 +107,6 @@
             "isort>=5.0",
             # Reformat
             "black",
-            # For toy text Gym envs
-            "scipy>=1.4.1",
         ],
         "docs": [
             "sphinx",
@@ -124,8 +122,9 @@
         "extra": [
             # For render
             "opencv-python",
+            "pygame",
             # For atari games,
-            "ale-py==0.7.4",
+            "ale-py~=0.8.0",
             "autorom[accept-rom-license]~=0.4.2",
             "pillow",
             # Tensorboard support

stable_baselines3/__init__.py

@@ -1,7 +1,5 @@
 import os
 
-import numpy as np
-
 from stable_baselines3.a2c import A2C
 from stable_baselines3.common.utils import get_system_info
 from stable_baselines3.ddpg import DDPG
@@ -11,10 +9,6 @@
 from stable_baselines3.sac import SAC
 from stable_baselines3.td3 import TD3
 
-# Small monkey patch so gym 0.21 is compatible with numpy >= 1.24
-# TODO: remove when upgrading to gym 0.26
-np.bool = bool  # type: ignore[attr-defined]
-
 # Read version from file
 version_file = os.path.join(os.path.dirname(__file__), "version.txt")
 with open(version_file) as file_handler: