From 6802e04a70b3dfc9d18bdaafd678cc1b51cb41b1 Mon Sep 17 00:00:00 2001 From: Carolin Benjamins Date: Thu, 20 Apr 2023 12:04:20 +0200 Subject: [PATCH 01/37] Add gymnax folder --- carl/envs/gymnax/__init__.py | 1 + carl/envs/gymnax/carl_pendulum.py | 92 +++++++++++++++++++++++++++++++ 2 files changed, 93 insertions(+) create mode 100644 carl/envs/gymnax/__init__.py create mode 100644 carl/envs/gymnax/carl_pendulum.py diff --git a/carl/envs/gymnax/__init__.py b/carl/envs/gymnax/__init__.py new file mode 100644 index 00000000..ca032fdf --- /dev/null +++ b/carl/envs/gymnax/__init__.py @@ -0,0 +1 @@ +# TODO add imports \ No newline at end of file diff --git a/carl/envs/gymnax/carl_pendulum.py b/carl/envs/gymnax/carl_pendulum.py new file mode 100644 index 00000000..7911e120 --- /dev/null +++ b/carl/envs/gymnax/carl_pendulum.py @@ -0,0 +1,92 @@ +from typing import Dict, List, Optional, Union + +import gymnasium.envs.classic_control as gccenvs # TODO import from gymnax +import numpy as np + +from carl.context.selection import AbstractSelector +from carl.envs.carl_env import CARLEnv +from carl.utils.trial_logger import TrialLogger +from carl.utils.types import Context, Contexts + +DEFAULT_CONTEXT = { + "max_speed": 8.0, + "dt": 0.05, + "g": 10.0, + "m": 1.0, + "l": 1.0, + "initial_angle_max": np.pi, # Upper bound for uniform distribution to sample from + "initial_velocity_max": 1, # Upper bound for uniform distribution to sample from + # The lower bound will be the negative value. +} + +CONTEXT_BOUNDS = { + "max_speed": (-np.inf, np.inf, float), + "dt": (0, np.inf, float), + "g": (0, np.inf, float), + "m": (1e-6, np.inf, float), + "l": (1e-6, np.inf, float), + "initial_angle_max": (0, np.inf, float), + "initial_velocity_max": (0, np.inf, float), +} + + +class CARLJaxPendulumEnv(CARLEnv): # TODO think of a good name + def __init__( + self, + env: PROPERCLASS = GymnaxWrapper(GymnaxPendulum), + contexts: Contexts = {}, + hide_context: bool = True, + add_gaussian_noise_to_context: bool = False, + gaussian_noise_std_percentage: float = 0.01, + logger: Optional[TrialLogger] = None, + scale_context_features: str = "no", + default_context: Optional[Context] = DEFAULT_CONTEXT, + max_episode_length: int = 200, # from https://github.com/openai/gym/blob/master/gym/envs/__init__.py + state_context_features: Optional[List[str]] = None, + context_mask: Optional[List[str]] = None, + dict_observation_space: bool = False, + context_selector: Optional[ + Union[AbstractSelector, type[AbstractSelector]] + ] = None, + context_selector_kwargs: Optional[Dict] = None, + ): + """ + Max torque is not a context feature because it changes the action space. + + Parameters + ---------- + env + contexts + instance_mode + hide_context + add_gaussian_noise_to_context + gaussian_noise_std_percentage + """ + if not contexts: + contexts = {0: DEFAULT_CONTEXT} + super().__init__( + env=env, + contexts=contexts, + hide_context=hide_context, + add_gaussian_noise_to_context=add_gaussian_noise_to_context, + gaussian_noise_std_percentage=gaussian_noise_std_percentage, + logger=logger, + scale_context_features=scale_context_features, + default_context=default_context, + max_episode_length=max_episode_length, + state_context_features=state_context_features, + dict_observation_space=dict_observation_space, + context_selector=context_selector, + context_selector_kwargs=context_selector_kwargs, + context_mask=context_mask, + ) + self.whitelist_gaussian_noise = list( + DEFAULT_CONTEXT.keys() + ) # allow to augment all values + + def _update_context(self) -> None: + self.env: GymnaxWrapper # TODO declare proper env + self.env.env_params.update(self.context) + + high = np.array([1.0, 1.0, self.max_speed], dtype=np.float32) + self.build_observation_space(-high, high, CONTEXT_BOUNDS) From 10b1e85f06005c209d844a62d63826ea421c310f Mon Sep 17 00:00:00 2001 From: Carolin Benjamins Date: Thu, 20 Apr 2023 12:12:33 +0200 Subject: [PATCH 02/37] Update requirements --- setup.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/setup.py b/setup.py index ded774a4..e59e6a3d 100644 --- a/setup.py +++ b/setup.py @@ -31,6 +31,9 @@ def read_file(filepath: str) -> str: "dm_control": [ "dm_control>=1.0.3", ], + "gymnax": [ + "GYMNAX>=???", + ], "mario": [ "torch>=1.9.0", "Pillow>=8.3.1", From ae2caf46d083a212671595f507ff001cd1b36063 Mon Sep 17 00:00:00 2001 From: Abbas Dehghan Date: Mon, 22 May 2023 18:33:10 +0200 Subject: [PATCH 03/37] add CARLJaxPendulumEnv --- carl/envs/gymnax/carl_pendulum.py | 30 ++++++++++++++---------------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/carl/envs/gymnax/carl_pendulum.py b/carl/envs/gymnax/carl_pendulum.py index 7911e120..38441237 100644 --- a/carl/envs/gymnax/carl_pendulum.py +++ b/carl/envs/gymnax/carl_pendulum.py @@ -1,7 +1,7 @@ from typing import Dict, List, Optional, Union -import gymnasium.envs.classic_control as gccenvs # TODO import from gymnax -import numpy as np +from gymnax.environments.classic_control.pendulum import EnvParams, Pendulum +import jax.numpy as jnp from carl.context.selection import AbstractSelector from carl.envs.carl_env import CARLEnv @@ -10,30 +10,29 @@ DEFAULT_CONTEXT = { "max_speed": 8.0, + "max_torque": 2.0, "dt": 0.05, "g": 10.0, "m": 1.0, "l": 1.0, - "initial_angle_max": np.pi, # Upper bound for uniform distribution to sample from - "initial_velocity_max": 1, # Upper bound for uniform distribution to sample from - # The lower bound will be the negative value. + "max_steps_in_episode": 200, } CONTEXT_BOUNDS = { - "max_speed": (-np.inf, np.inf, float), - "dt": (0, np.inf, float), - "g": (0, np.inf, float), - "m": (1e-6, np.inf, float), - "l": (1e-6, np.inf, float), - "initial_angle_max": (0, np.inf, float), - "initial_velocity_max": (0, np.inf, float), + "max_speed": (-jnp.inf, jnp.inf, float), + "max_torque": (-jnp.inf, jnp.inf, float), + "dt": (0, jnp.inf, float), + "g": (0, jnp.inf, float), + "m": (1e-6, jnp.inf, float), + "l": (1e-6, jnp.inf, float), + "max_steps_in_episode": (1, jnp.inf, int), } class CARLJaxPendulumEnv(CARLEnv): # TODO think of a good name def __init__( self, - env: PROPERCLASS = GymnaxWrapper(GymnaxPendulum), + env: Pendulum = Pendulum(), contexts: Contexts = {}, hide_context: bool = True, add_gaussian_noise_to_context: bool = False, @@ -85,8 +84,7 @@ def __init__( ) # allow to augment all values def _update_context(self) -> None: - self.env: GymnaxWrapper # TODO declare proper env - self.env.env_params.update(self.context) + self.env_params = EnvParams(**self.context) - high = np.array([1.0, 1.0, self.max_speed], dtype=np.float32) + high = jnp.array([1.0, 1.0, self.max_speed], dtype=jnp.float32) self.build_observation_space(-high, high, CONTEXT_BOUNDS) From 694241e3d1f6ca67f9d3f8a318860688e79a3b86 Mon Sep 17 00:00:00 2001 From: Abbas Dehghan Date: Mon, 22 May 2023 18:33:55 +0200 Subject: [PATCH 04/37] add init --- carl/envs/gymnax/__init__.py | 38 +++++++++++++++++++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/carl/envs/gymnax/__init__.py b/carl/envs/gymnax/__init__.py index ca032fdf..7d38c4bc 100644 --- a/carl/envs/gymnax/__init__.py +++ b/carl/envs/gymnax/__init__.py @@ -1 +1,37 @@ -# TODO add imports \ No newline at end of file +from carl.envs.gymnax.carl_jax_acrobot import ( # noqa: F401 + CONTEXT_BOUNDS as CARLJaxAcrobotEnv_bounds, +) +from carl.envs.gymnax.carl_jax_acrobot import ( # noqa: F401 + DEFAULT_CONTEXT as CARLJaxAcrobotEnv_defaults, +) +from carl.envs.gymnax.carl_jax_acrobot import CARLJaxAcrobotEnv # noqa: F401 +from carl.envs.gymnax.carl_jax_cartpole import ( # noqa: F401 + CONTEXT_BOUNDS as CARLJaxCartPoleEnv_bounds, +) +from carl.envs.gymnax.carl_jax_cartpole import ( # noqa: F401 + DEFAULT_CONTEXT as CARLJaxCartPoleEnv_defaults, +) +from carl.envs.gymnax.carl_jax_cartpole import CARLJaxCartPoleEnv # noqa: F401 +from carl.envs.gymnax.carl_jax_mountaincar import ( # noqa: F401 + CONTEXT_BOUNDS as CARLJaxMountainCarEnv_bounds, +) +from carl.envs.gymnax.carl_jax_mountaincar import ( # noqa: F401 + DEFAULT_CONTEXT as CARLJaxMountainCarEnv_defaults, +) +from carl.envs.gymnax.carl_jax_mountaincar import CARLJaxMountainCarEnv # noqa: F401 +from carl.envs.gymnax.carl_jax_mountaincarcontinuous import ( # noqa: F401 + CONTEXT_BOUNDS as CARLJaxMountainCarContinuousEnv_bounds, +) +from carl.envs.gymnax.carl_jax_mountaincarcontinuous import ( # noqa: F401 + DEFAULT_CONTEXT as CARLJaxMountainCarContinuousEnv_defaults, +) +from carl.envs.gymnax.carl_jax_mountaincarcontinuous import ( # noqa: F401 + CARLJaxMountainCarContinuousEnv, +) +from carl.envs.gymnax.carl_jax_pendulum import ( # noqa: F401 + CONTEXT_BOUNDS as CARLJaxPendulumEnv_bounds, +) +from carl.envs.gymnax.carl_jax_pendulum import ( # noqa: F401 + DEFAULT_CONTEXT as CARLJaxPendulumEnv_defaults, +) +from carl.envs.gymnax.carl_jax_pendulum import CARLJaxPendulumEnv # noqa: F401 From f1ec7b30672abe0c6fc1645e7a4a68b9501faea8 Mon Sep 17 00:00:00 2001 From: Abbas Dehghan Date: Mon, 22 May 2023 18:47:42 +0200 Subject: [PATCH 05/37] add CARLJaxAcrobotEnv --- carl/envs/gymnax/carl_jax_acrobot.py | 121 +++++++++++++++++++++++++++ 1 file changed, 121 insertions(+) create mode 100644 carl/envs/gymnax/carl_jax_acrobot.py diff --git a/carl/envs/gymnax/carl_jax_acrobot.py b/carl/envs/gymnax/carl_jax_acrobot.py new file mode 100644 index 00000000..6d49e8b9 --- /dev/null +++ b/carl/envs/gymnax/carl_jax_acrobot.py @@ -0,0 +1,121 @@ +from typing import Dict, List, Optional, Union + +import jax.numpy as jnp +import numpy as np +from gymnax.environments.classic_control.acrobot import Acrobot + +from carl.context.selection import AbstractSelector +from carl.envs.carl_env import CARLEnv +from carl.utils.trial_logger import TrialLogger +from carl.utils.types import Context, Contexts + +DEFAULT_CONTEXT = { + "link_length_1": 1, + "link_length_2": 1, + "link_mass_1": 1, + "link_mass_2": 1, + "link_com_1": 0.5, + "link_com_2": 0.5, + "link_moi": 1, + "max_velocity_1": 4 * jnp.pi, + "max_velocity_2": 9 * jnp.pi, +} + +CONTEXT_BOUNDS = { + "link_length_1": ( + 0.1, + 10, + float, + ), # Links can be shrunken and grown by a factor of 10 + "link_length_2": (0.1, 10, float), + "link_mass_1": ( + 0.1, + 10, + float, + ), # Link mass can be shrunken and grown by a factor of 10 + "link_mass_2": (0.1, 10, float), + "link_com_1": (0, 1, float), # Center of mass can move from one end to the other + "link_com_2": (0, 1, float), + "link_moi": ( + 0.1, + 10, + float, + ), # Moments on inertia can be shrunken and grown by a factor of 10 + "max_velocity_1": ( + 0.4 * np.pi, + 40 * np.pi, + float, + ), # Velocity can vary by a factor of 10 in either direction + "max_velocity_2": (0.9 * np.pi, 90 * np.pi, float), + "torque_noise_max": ( + -1.0, + 1.0, + float, + ), # torque is either {-1., 0., 1}. Applying noise of 1. would be quite extreme + "initial_angle_lower": (-jnp.inf, jnp.inf, float), + "initial_angle_upper": (-jnp.inf, jnp.inf, float), + "initial_velocity_lower": (-jnp.inf, jnp.inf, float), + "initial_velocity_upper": (-jnp.inf, jnp.inf, float), +} + + +class CARLJaxAcrobotEnv(CARLEnv): + def __init__( + self, + env: Acrobot = Acrobot(), + contexts: Contexts = {}, + hide_context: bool = True, + add_gaussian_noise_to_context: bool = False, + gaussian_noise_std_percentage: float = 0.01, + logger: Optional[TrialLogger] = None, + scale_context_features: str = "no", + default_context: Optional[Context] = DEFAULT_CONTEXT, + max_episode_length: int = 500, # from https://github.com/openai/gym/blob/master/gym/envs/__init__.py + state_context_features: Optional[List[str]] = None, + context_mask: Optional[List[str]] = None, + dict_observation_space: bool = False, + context_selector: Optional[ + Union[AbstractSelector, type[AbstractSelector]] + ] = None, + context_selector_kwargs: Optional[Dict] = None, + ): + if not contexts: + contexts = {0: DEFAULT_CONTEXT} + super().__init__( + env=env, + contexts=contexts, + hide_context=hide_context, + add_gaussian_noise_to_context=add_gaussian_noise_to_context, + gaussian_noise_std_percentage=gaussian_noise_std_percentage, + logger=logger, + scale_context_features=scale_context_features, + default_context=default_context, + max_episode_length=max_episode_length, + state_context_features=state_context_features, + dict_observation_space=dict_observation_space, + context_selector=context_selector, + context_selector_kwargs=context_selector_kwargs, + context_mask=context_mask, + ) + self.whitelist_gaussian_noise = list( + DEFAULT_CONTEXT.keys() + ) # allow to augment all values + + def _update_context(self) -> None: + self.env: Acrobot + self.env.LINK_LENGTH_1 = self.context["link_length_1"] + self.env.LINK_LENGTH_2 = self.context["link_length_2"] + self.env.LINK_MASS_1 = self.context["link_mass_1"] + self.env.LINK_MASS_2 = self.context["link_mass_2"] + self.env.LINK_COM_POS_1 = self.context["link_com_1"] + self.env.LINK_COM_POS_2 = self.context["link_com_2"] + self.env.LINK_MOI = self.context["link_moi"] + self.env.MAX_VEL_1 = self.context["max_velocity_1"] + self.env.MAX_VEL_2 = self.context["max_velocity_2"] + + high = jnp.array( + [1.0, 1.0, 1.0, 1.0, self.env.MAX_VEL_1, self.env.MAX_VEL_2], + dtype=jnp.float32, + ) + low = -high + self.build_observation_space(low, high, CONTEXT_BOUNDS) From 3ed010d395d6649b207eeb05880701f41946d623 Mon Sep 17 00:00:00 2001 From: Abbas Dehghan Date: Mon, 22 May 2023 18:49:03 +0200 Subject: [PATCH 06/37] remove comment --- carl/envs/gymnax/carl_pendulum.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/carl/envs/gymnax/carl_pendulum.py b/carl/envs/gymnax/carl_pendulum.py index 38441237..19e21a6f 100644 --- a/carl/envs/gymnax/carl_pendulum.py +++ b/carl/envs/gymnax/carl_pendulum.py @@ -29,7 +29,7 @@ } -class CARLJaxPendulumEnv(CARLEnv): # TODO think of a good name +class CARLJaxPendulumEnv(CARLEnv): def __init__( self, env: Pendulum = Pendulum(), From 9fd4ea6cbaa10ec3cf64980a7e2783e797464ecd Mon Sep 17 00:00:00 2001 From: Abbas Dehghan Date: Mon, 22 May 2023 18:50:22 +0200 Subject: [PATCH 07/37] add CARLJaxCartPoleEnv --- carl/envs/gymnax/carl_jax_cartpole.py | 91 +++++++++++++++++++++++++++ 1 file changed, 91 insertions(+) create mode 100644 carl/envs/gymnax/carl_jax_cartpole.py diff --git a/carl/envs/gymnax/carl_jax_cartpole.py b/carl/envs/gymnax/carl_jax_cartpole.py new file mode 100644 index 00000000..eb4be74c --- /dev/null +++ b/carl/envs/gymnax/carl_jax_cartpole.py @@ -0,0 +1,91 @@ +from typing import Dict, List, Optional, Union + +import jax.numpy as jnp +from gymnax.environments.classic_control.cartpole import CartPole + +from carl.context.selection import AbstractSelector +from carl.envs.carl_env import CARLEnv +from carl.utils.trial_logger import TrialLogger +from carl.utils.types import Context, Contexts + +DEFAULT_CONTEXT = { + "gravity": 9.8, + "masscart": 1.0, + "masspole": 0.1, + "length": 0.5, + "force_mag": 10.0, + "tau": 0.02, +} + +CONTEXT_BOUNDS = { + "gravity": (5.0, 15.0, float), + "masscart": (0.5, 2.0, float), + "masspole": (0.05, 0.2, float), + "length": (0.25, 1.0, float), + "force_mag": (5.0, 15.0, float), + "tau": (0.01, 0.05, float), +} + + +class CARLJaxCartPoleEnv(CARLEnv): + def __init__( + self, + env: CartPole = CartPole(), + contexts: Contexts = {}, + hide_context: bool = True, + add_gaussian_noise_to_context: bool = False, + gaussian_noise_std_percentage: float = 0.01, + logger: Optional[TrialLogger] = None, + scale_context_features: str = "no", + default_context: Optional[Context] = DEFAULT_CONTEXT, + max_episode_length: int = 500, # from https://github.com/openai/gym/blob/master/gym/envs/__init__.py + state_context_features: Optional[List[str]] = None, + context_mask: Optional[List[str]] = None, + dict_observation_space: bool = False, + context_selector: Optional[ + Union[AbstractSelector, type[AbstractSelector]] + ] = None, + context_selector_kwargs: Optional[Dict] = None, + ): + if not contexts: + contexts = {0: DEFAULT_CONTEXT} + super().__init__( + env=env, + contexts=contexts, + hide_context=hide_context, + add_gaussian_noise_to_context=add_gaussian_noise_to_context, + gaussian_noise_std_percentage=gaussian_noise_std_percentage, + logger=logger, + scale_context_features=scale_context_features, + default_context=default_context, + max_episode_length=max_episode_length, + state_context_features=state_context_features, + dict_observation_space=dict_observation_space, + context_selector=context_selector, + context_selector_kwargs=context_selector_kwargs, + context_mask=context_mask, + ) + self.whitelist_gaussian_noise = list( + DEFAULT_CONTEXT.keys() + ) # allow to augment all values + + def _update_context(self) -> None: + self.env: CartPole + self.env.gravity = self.context["gravity"] + self.env.masscart = self.context["masscart"] + self.env.masspole = self.context["masspole"] + self.env.length = self.context["length"] + self.env.force_mag = self.context["force_mag"] + self.env.tau = self.context["tau"] + + high = jnp.array( + [ + self.env.x_threshold * 2, + jnp.finfo(jnp.float32).max, + self.env.theta_threshold_radians * 2, + jnp.finfo(jnp.float32).max, + ], + dtype=jnp.float32, + ) + low = -high + self.build_observation_space(low, high, CONTEXT_BOUNDS) From 5b6eee5780dba038d62309e8f34b53f7e7fc3563 Mon Sep 17 00:00:00 2001 From: Abbas Dehghan Date: Mon, 22 May 2023 18:52:48 +0200 Subject: [PATCH 08/37] add CARLJaxMountainCarEnv --- carl/envs/gymnax/carl_jax_mountaincar.py | 105 +++++++++++++++++++++++ 1 file changed, 105 insertions(+) create mode 100644 carl/envs/gymnax/carl_jax_mountaincar.py diff --git a/carl/envs/gymnax/carl_jax_mountaincar.py b/carl/envs/gymnax/carl_jax_mountaincar.py new file mode 100644 index 00000000..c239f068 --- /dev/null +++ b/carl/envs/gymnax/carl_jax_mountaincar.py @@ -0,0 +1,105 @@ +from typing import Dict, List, Optional, Union + +import jax.numpy as jnp +from gymnax.environments.classic_control.mountain_car import MountainCar + +from carl.context.selection import AbstractSelector +from carl.envs.carl_env import CARLEnv +from carl.utils.trial_logger import TrialLogger +from carl.utils.types import Context, Contexts + +DEFAULT_CONTEXT = { + "min_position": -1.2, # unit? + "max_position": 0.6, # unit? + "max_speed": 0.07, # unit? + "goal_position": 0.5, # unit? + "goal_velocity": 0, # unit? + "force": 0.001, # unit? + "gravity": 0.0025, # unit? +} + +CONTEXT_BOUNDS = { + "min_position": (-jnp.inf, jnp.inf, float), + "max_position": (-jnp.inf, jnp.inf, float), + "max_speed": (0, jnp.inf, float), + "goal_position": (-jnp.inf, jnp.inf, float), + "goal_velocity": (-jnp.inf, jnp.inf, float), + "force": (-jnp.inf, jnp.inf, float), + "gravity": (0, jnp.inf, float), +} + + +class CARLJaxMountainCarEnv(CARLEnv): + def __init__( + self, + env: MountainCar = MountainCar(), + contexts: Contexts = {}, + hide_context: bool = True, + add_gaussian_noise_to_context: bool = False, + gaussian_noise_std_percentage: float = 0.01, + logger: Optional[TrialLogger] = None, + scale_context_features: str = "no", + default_context: Optional[Context] = DEFAULT_CONTEXT, + max_episode_length: int = 200, # from https://github.com/openai/gym/blob/master/gym/envs/__init__.py + state_context_features: Optional[List[str]] = None, + context_mask: Optional[List[str]] = None, + dict_observation_space: bool = False, + context_selector: Optional[ + Union[AbstractSelector, type[AbstractSelector]] + ] = None, + context_selector_kwargs: Optional[Dict] = None, + ): + """ + + Parameters + ---------- + env: gym.Env, optional + Defaults to classic control environment mountain car from gym (MountainCarEnv). + contexts: List[Dict], optional + Different contexts / different environment parameter settings. + instance_mode: str, optional + """ + if not contexts: + contexts = {0: DEFAULT_CONTEXT} + super().__init__( + env=env, + contexts=contexts, + hide_context=hide_context, + add_gaussian_noise_to_context=add_gaussian_noise_to_context, + gaussian_noise_std_percentage=gaussian_noise_std_percentage, + logger=logger, + scale_context_features=scale_context_features, + default_context=default_context, + max_episode_length=max_episode_length, + state_context_features=state_context_features, + dict_observation_space=dict_observation_space, + context_selector=context_selector, + context_selector_kwargs=context_selector_kwargs, + context_mask=context_mask, + ) + self.whitelist_gaussian_noise = list( + DEFAULT_CONTEXT.keys() + ) # allow to augment all values + + def _update_context(self) -> None: + self.env: CARLJaxMountainCarEnv + self.env.min_position = self.context["min_position"] + self.env.max_position = self.context["max_position"] + self.env.max_speed = self.context["max_speed"] + self.env.goal_position = self.context["goal_position"] + self.env.goal_velocity = self.context["goal_velocity"] + self.env.min_position_start = self.context["min_position_start"] + self.env.max_position_start = self.context["max_position_start"] + self.env.min_velocity_start = self.context["min_velocity_start"] + self.env.max_velocity_start = self.context["max_velocity_start"] + self.env.force = self.context["force"] + self.env.gravity = self.context["gravity"] + + self.low = jnp.array( + [self.env.min_position, -self.env.max_speed], dtype=jnp.float32 + ).squeeze() + self.high = jnp.array( + [self.env.max_position, self.env.max_speed], dtype=jnp.float32 + ).squeeze() + + self.build_observation_space(self.low, self.high, CONTEXT_BOUNDS) From 4fa6f98a58053ebd47ead8bb129c849f15c77c0a Mon Sep 17 00:00:00 2001 From: Abbas Dehghan Date: Mon, 22 May 2023 18:54:58 +0200 Subject: [PATCH 09/37] add CARLJaxMountainCarContinuousEnv --- .../gymnax/carl_jax_mountaincarcontinuous.py | 113 ++++++++++++++++++ 1 file changed, 113 insertions(+) create mode 100644 carl/envs/gymnax/carl_jax_mountaincarcontinuous.py diff --git a/carl/envs/gymnax/carl_jax_mountaincarcontinuous.py b/carl/envs/gymnax/carl_jax_mountaincarcontinuous.py new file mode 100644 index 00000000..cf791624 --- /dev/null +++ b/carl/envs/gymnax/carl_jax_mountaincarcontinuous.py @@ -0,0 +1,113 @@ +from typing import Dict, List, Optional, Union + +import numpy as jnp +from gymnax.environments.classic_control.continuous_mountain_car import ( + ContinuousMountainCar, +) + +from carl.context.selection import AbstractSelector +from carl.envs.carl_env import CARLEnv +from carl.utils.trial_logger import TrialLogger +from carl.utils.types import Context, Contexts + +DEFAULT_CONTEXT = { + "min_action": -1.0, + "max_action": 1.0, + "min_position": -1.2, + "max_position": 0.6, + "max_speed": 0.07, + "goal_position": 0.45, + "goal_velocity": 0.0, + "power": 0.0015, + "gravity": 0.0025, +} + + +CONTEXT_BOUNDS = { + "min_action": (-jnp.inf, jnp.inf, float), + "max_action:": (-jnp.inf, jnp.inf, float), + "min_position": (-jnp.inf, jnp.inf, float), + "max_position": (-jnp.inf, jnp.inf, float), + "max_speed": (0, jnp.inf, float), + "goal_position": (-jnp.inf, jnp.inf, float), + "goal_velocity": (-jnp.inf, jnp.inf, float), + "power": (-jnp.inf, jnp.inf, float), + "gravity": (0, jnp.inf, float), +} + + +class CARLJaxMountainCarContinuousEnv(CARLEnv): + def __init__( + self, + env: ContinuousMountainCar = ContinuousMountainCar(), + contexts: Contexts = {}, + hide_context: bool = True, + add_gaussian_noise_to_context: bool = True, + gaussian_noise_std_percentage: float = 0.01, + logger: Optional[TrialLogger] = None, + scale_context_features: str = "no", + default_context: Optional[Context] = DEFAULT_CONTEXT, + max_episode_length: int = 999, # from https://github.com/openai/gym/blob/master/gym/envs/__init__.py + state_context_features: Optional[List[str]] = None, + context_mask: Optional[List[str]] = None, + dict_observation_space: bool = False, + context_selector: Optional[ + Union[AbstractSelector, type[AbstractSelector]] + ] = None, + context_selector_kwargs: Optional[Dict] = None, + ): + """ + + Parameters + ---------- + env: gym.Env, optional + Defaults to classic control environment mountain car from gym (MountainCarEnv). + contexts: List[Dict], optional + Different contexts / different environment parameter settings. + instance_mode: str, optional + """ + if not contexts: + contexts = {0: DEFAULT_CONTEXT} + super().__init__( + env=env, + contexts=contexts, + hide_context=hide_context, + add_gaussian_noise_to_context=add_gaussian_noise_to_context, + gaussian_noise_std_percentage=gaussian_noise_std_percentage, + logger=logger, + scale_context_features=scale_context_features, + default_context=default_context, + max_episode_length=max_episode_length, + state_context_features=state_context_features, + dict_observation_space=dict_observation_space, + context_selector=context_selector, + context_selector_kwargs=context_selector_kwargs, + context_mask=context_mask, + ) + self.whitelist_gaussian_noise = list( + DEFAULT_CONTEXT.keys() + ) # allow to augment all values + + def _update_context(self) -> None: + self.env: ContinuousMountainCar + self.env.min_position = self.context["min_position"] + self.env.max_position = self.context["max_position"] + self.env.max_speed = self.context["max_speed"] + self.env.goal_position = self.context["goal_position"] + self.env.goal_velocity = self.context["goal_velocity"] + self.env.min_position_start = self.context["min_position_start"] + self.env.max_position_start = self.context["max_position_start"] + self.env.min_velocity_start = self.context["min_velocity_start"] + self.env.max_velocity_start = self.context["max_velocity_start"] + self.env.power = self.context["power"] + # self.env.force = self.context["force"] + # self.env.gravity = self.context["gravity"] + + self.low = jnp.array( + [self.env.min_position, -self.env.max_speed], dtype=jnp.float32 + ).squeeze() + self.high = jnp.array( + [self.env.max_position, self.env.max_speed], dtype=jnp.float32 + ).squeeze() + + self.build_observation_space(self.low, self.high, CONTEXT_BOUNDS) From 68048f5d75421ae71703584b1238dcbdebc1587c Mon Sep 17 00:00:00 2001 From: Abbas Dehghan Date: Mon, 22 May 2023 19:13:24 +0200 Subject: [PATCH 10/37] fix setup --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index e59e6a3d..24857f5a 100644 --- a/setup.py +++ b/setup.py @@ -32,7 +32,7 @@ def read_file(filepath: str) -> str: "dm_control>=1.0.3", ], "gymnax": [ - "GYMNAX>=???", + "gymnax>=0.0.6", ], "mario": [ "torch>=1.9.0", From 666a3239b458d1932e1d6dcb36ad54e5d33a14fd Mon Sep 17 00:00:00 2001 From: Carolin Benjamins Date: Tue, 23 May 2023 16:44:28 +0200 Subject: [PATCH 11/37] Rename file --- carl/envs/gymnax/{carl_pendulum.py => carl_jax_pendulum.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename carl/envs/gymnax/{carl_pendulum.py => carl_jax_pendulum.py} (100%) diff --git a/carl/envs/gymnax/carl_pendulum.py b/carl/envs/gymnax/carl_jax_pendulum.py similarity index 100% rename from carl/envs/gymnax/carl_pendulum.py rename to carl/envs/gymnax/carl_jax_pendulum.py From ffc2bb9485aee08aec83e5fabc8da6f0f783b8ec Mon Sep 17 00:00:00 2001 From: Carolin Benjamins Date: Tue, 23 May 2023 16:44:35 +0200 Subject: [PATCH 12/37] Add imports --- carl/envs/__init__.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/carl/envs/__init__.py b/carl/envs/__init__.py index 829cb87c..1f9ecbcd 100644 --- a/carl/envs/__init__.py +++ b/carl/envs/__init__.py @@ -42,3 +42,13 @@ warnings.warn( "Module 'dm_control' not found. If you want to use these environments, please follow the installation guide." ) + + +gymnax_spec = iutil.find_spec("gymnax") +found = gymnax_spec is not None +if found: + from carl.envs.gymnax import * +else: + warnings.warn( + "Module 'gymnax' not found. If you want to use these environments, please follow the installation guide." + ) From 9a3d4164c4097091466dac0b098daf3fe2b511bc Mon Sep 17 00:00:00 2001 From: Abbas Dehghan Date: Sat, 27 May 2023 21:02:08 +0200 Subject: [PATCH 13/37] add modified version for GymnaxToGymWrapper --- carl/envs/gymnax/carl_jax_acrobot.py | 14 +++++++++++++- carl/envs/gymnax/carl_jax_cartpole.py | 14 +++++++++++++- carl/envs/gymnax/carl_jax_mountaincar.py | 14 +++++++++++++- .../gymnax/carl_jax_mountaincarcontinuous.py | 18 +++++++++++++++--- carl/envs/gymnax/carl_jax_pendulum.py | 14 +++++++++++++- 5 files changed, 67 insertions(+), 7 deletions(-) diff --git a/carl/envs/gymnax/carl_jax_acrobot.py b/carl/envs/gymnax/carl_jax_acrobot.py index 6d49e8b9..2e31d87e 100644 --- a/carl/envs/gymnax/carl_jax_acrobot.py +++ b/carl/envs/gymnax/carl_jax_acrobot.py @@ -3,6 +3,8 @@ import jax.numpy as jnp import numpy as np from gymnax.environments.classic_control.acrobot import Acrobot +from gymnax.wrappers.gym import GymnaxToGymWrapper +from gymnax.environments.spaces import gymnax_space_to_gym_space from carl.context.selection import AbstractSelector from carl.envs.carl_env import CARLEnv @@ -58,11 +60,21 @@ "initial_velocity_upper": (-jnp.inf, jnp.inf, float), } +class CustomGymnaxToGymWrapper(GymnaxToGymWrapper): + @property + def observation_space(self): + return gymnax_space_to_gym_space( + self._env.observation_space(self.env_params) + ) + + @observation_space.setter + def observation_space(self, value): + self._observation_space = value class CARLJaxAcrobotEnv(CARLEnv): def __init__( self, - env: Acrobot = Acrobot(), + env: Acrobot = CustomGymnaxToGymWrapper(Acrobot()), contexts: Contexts = {}, hide_context: bool = True, add_gaussian_noise_to_context: bool = False, diff --git a/carl/envs/gymnax/carl_jax_cartpole.py b/carl/envs/gymnax/carl_jax_cartpole.py index eb4be74c..0253517d 100644 --- a/carl/envs/gymnax/carl_jax_cartpole.py +++ b/carl/envs/gymnax/carl_jax_cartpole.py @@ -2,6 +2,8 @@ import jax.numpy as jnp from gymnax.environments.classic_control.cartpole import CartPole +from gymnax.wrappers.gym import GymnaxToGymWrapper +from gymnax.environments.spaces import gymnax_space_to_gym_space from carl.context.selection import AbstractSelector from carl.envs.carl_env import CARLEnv @@ -26,11 +28,21 @@ "tau": (0.01, 0.05, float), } +class CustomGymnaxToGymWrapper(GymnaxToGymWrapper): + @property + def observation_space(self): + return gymnax_space_to_gym_space( + self._env.observation_space(self.env_params) + ) + + @observation_space.setter + def observation_space(self, value): + self._observation_space = value class CARLJaxCartPoleEnv(CARLEnv): def __init__( self, - env: CartPole = CartPole(), + env: CartPole = CustomGymnaxToGymWrapper(CartPole()), contexts: Contexts = {}, hide_context: bool = True, add_gaussian_noise_to_context: bool = False, diff --git a/carl/envs/gymnax/carl_jax_mountaincar.py b/carl/envs/gymnax/carl_jax_mountaincar.py index c239f068..97d8653b 100644 --- a/carl/envs/gymnax/carl_jax_mountaincar.py +++ b/carl/envs/gymnax/carl_jax_mountaincar.py @@ -2,6 +2,8 @@ import jax.numpy as jnp from gymnax.environments.classic_control.mountain_car import MountainCar +from gymnax.wrappers.gym import GymnaxToGymWrapper +from gymnax.environments.spaces import gymnax_space_to_gym_space from carl.context.selection import AbstractSelector from carl.envs.carl_env import CARLEnv @@ -28,11 +30,21 @@ "gravity": (0, jnp.inf, float), } +class CustomGymnaxToGymWrapper(GymnaxToGymWrapper): + @property + def observation_space(self): + return gymnax_space_to_gym_space( + self._env.observation_space(self.env_params) + ) + + @observation_space.setter + def observation_space(self, value): + self._observation_space = value class CARLJaxMountainCarEnv(CARLEnv): def __init__( self, - env: MountainCar = MountainCar(), + env: MountainCar = CustomGymnaxToGymWrapper(MountainCar()), contexts: Contexts = {}, hide_context: bool = True, add_gaussian_noise_to_context: bool = False, diff --git a/carl/envs/gymnax/carl_jax_mountaincarcontinuous.py b/carl/envs/gymnax/carl_jax_mountaincarcontinuous.py index cf791624..dc4f5616 100644 --- a/carl/envs/gymnax/carl_jax_mountaincarcontinuous.py +++ b/carl/envs/gymnax/carl_jax_mountaincarcontinuous.py @@ -1,10 +1,12 @@ from typing import Dict, List, Optional, Union -import numpy as jnp +import jax.numpy as jnp + from gymnax.environments.classic_control.continuous_mountain_car import ( ContinuousMountainCar, ) - +from gymnax.environments.spaces import gymnax_space_to_gym_space +from gymnax.wrappers.gym import GymnaxToGymWrapper from carl.context.selection import AbstractSelector from carl.envs.carl_env import CARLEnv from carl.utils.trial_logger import TrialLogger @@ -35,11 +37,21 @@ "gravity": (0, jnp.inf, float), } +class CustomGymnaxToGymWrapper(GymnaxToGymWrapper): + @property + def observation_space(self): + return gymnax_space_to_gym_space( + self._env.observation_space(self.env_params) + ) + + @observation_space.setter + def observation_space(self, value): + self._observation_space = value class CARLJaxMountainCarContinuousEnv(CARLEnv): def __init__( self, - env: ContinuousMountainCar = ContinuousMountainCar(), + env: ContinuousMountainCar = CustomGymnaxToGymWrapper(ContinuousMountainCar()), contexts: Contexts = {}, hide_context: bool = True, add_gaussian_noise_to_context: bool = True, diff --git a/carl/envs/gymnax/carl_jax_pendulum.py b/carl/envs/gymnax/carl_jax_pendulum.py index 19e21a6f..d2ab45b9 100644 --- a/carl/envs/gymnax/carl_jax_pendulum.py +++ b/carl/envs/gymnax/carl_jax_pendulum.py @@ -1,6 +1,8 @@ from typing import Dict, List, Optional, Union from gymnax.environments.classic_control.pendulum import EnvParams, Pendulum +from gymnax.wrappers.gym import GymnaxToGymWrapper +from gymnax.environments.spaces import gymnax_space_to_gym_space import jax.numpy as jnp from carl.context.selection import AbstractSelector @@ -28,11 +30,21 @@ "max_steps_in_episode": (1, jnp.inf, int), } +class CustomGymnaxToGymWrapper(GymnaxToGymWrapper): + @property + def observation_space(self): + return gymnax_space_to_gym_space( + self._env.observation_space(self.env_params) + ) + + @observation_space.setter + def observation_space(self, value): + self._observation_space = value class CARLJaxPendulumEnv(CARLEnv): def __init__( self, - env: Pendulum = Pendulum(), + env: Pendulum = CustomGymnaxToGymWrapper(Pendulum()), contexts: Contexts = {}, hide_context: bool = True, add_gaussian_noise_to_context: bool = False, From 5874515a5e929956f0c4f05194f597693e97b2e4 Mon Sep 17 00:00:00 2001 From: Abbas Dehghan Date: Sat, 27 May 2023 21:24:16 +0200 Subject: [PATCH 14/37] make precommit --- carl/envs/gymnax/carl_jax_acrobot.py | 18 ++++++++--------- carl/envs/gymnax/carl_jax_cartpole.py | 18 ++++++++--------- carl/envs/gymnax/carl_jax_mountaincar.py | 18 ++++++++--------- .../gymnax/carl_jax_mountaincarcontinuous.py | 18 ++++++++--------- carl/envs/gymnax/carl_jax_pendulum.py | 20 +++++++++---------- 5 files changed, 46 insertions(+), 46 deletions(-) diff --git a/carl/envs/gymnax/carl_jax_acrobot.py b/carl/envs/gymnax/carl_jax_acrobot.py index 2e31d87e..fb3831fd 100644 --- a/carl/envs/gymnax/carl_jax_acrobot.py +++ b/carl/envs/gymnax/carl_jax_acrobot.py @@ -3,8 +3,8 @@ import jax.numpy as jnp import numpy as np from gymnax.environments.classic_control.acrobot import Acrobot -from gymnax.wrappers.gym import GymnaxToGymWrapper from gymnax.environments.spaces import gymnax_space_to_gym_space +from gymnax.wrappers.gym import GymnaxToGymWrapper from carl.context.selection import AbstractSelector from carl.envs.carl_env import CARLEnv @@ -60,16 +60,16 @@ "initial_velocity_upper": (-jnp.inf, jnp.inf, float), } + class CustomGymnaxToGymWrapper(GymnaxToGymWrapper): - @property - def observation_space(self): - return gymnax_space_to_gym_space( - self._env.observation_space(self.env_params) - ) + @property + def observation_space(self): + return gymnax_space_to_gym_space(self._env.observation_space(self.env_params)) + + @observation_space.setter + def observation_space(self, value): + self._observation_space = value - @observation_space.setter - def observation_space(self, value): - self._observation_space = value class CARLJaxAcrobotEnv(CARLEnv): def __init__( diff --git a/carl/envs/gymnax/carl_jax_cartpole.py b/carl/envs/gymnax/carl_jax_cartpole.py index 0253517d..94f6809b 100644 --- a/carl/envs/gymnax/carl_jax_cartpole.py +++ b/carl/envs/gymnax/carl_jax_cartpole.py @@ -2,8 +2,8 @@ import jax.numpy as jnp from gymnax.environments.classic_control.cartpole import CartPole -from gymnax.wrappers.gym import GymnaxToGymWrapper from gymnax.environments.spaces import gymnax_space_to_gym_space +from gymnax.wrappers.gym import GymnaxToGymWrapper from carl.context.selection import AbstractSelector from carl.envs.carl_env import CARLEnv @@ -28,16 +28,16 @@ "tau": (0.01, 0.05, float), } + class CustomGymnaxToGymWrapper(GymnaxToGymWrapper): - @property - def observation_space(self): - return gymnax_space_to_gym_space( - self._env.observation_space(self.env_params) - ) + @property + def observation_space(self): + return gymnax_space_to_gym_space(self._env.observation_space(self.env_params)) + + @observation_space.setter + def observation_space(self, value): + self._observation_space = value - @observation_space.setter - def observation_space(self, value): - self._observation_space = value class CARLJaxCartPoleEnv(CARLEnv): def __init__( diff --git a/carl/envs/gymnax/carl_jax_mountaincar.py b/carl/envs/gymnax/carl_jax_mountaincar.py index 97d8653b..0dbac5b7 100644 --- a/carl/envs/gymnax/carl_jax_mountaincar.py +++ b/carl/envs/gymnax/carl_jax_mountaincar.py @@ -2,8 +2,8 @@ import jax.numpy as jnp from gymnax.environments.classic_control.mountain_car import MountainCar -from gymnax.wrappers.gym import GymnaxToGymWrapper from gymnax.environments.spaces import gymnax_space_to_gym_space +from gymnax.wrappers.gym import GymnaxToGymWrapper from carl.context.selection import AbstractSelector from carl.envs.carl_env import CARLEnv @@ -30,16 +30,16 @@ "gravity": (0, jnp.inf, float), } + class CustomGymnaxToGymWrapper(GymnaxToGymWrapper): - @property - def observation_space(self): - return gymnax_space_to_gym_space( - self._env.observation_space(self.env_params) - ) + @property + def observation_space(self): + return gymnax_space_to_gym_space(self._env.observation_space(self.env_params)) + + @observation_space.setter + def observation_space(self, value): + self._observation_space = value - @observation_space.setter - def observation_space(self, value): - self._observation_space = value class CARLJaxMountainCarEnv(CARLEnv): def __init__( diff --git a/carl/envs/gymnax/carl_jax_mountaincarcontinuous.py b/carl/envs/gymnax/carl_jax_mountaincarcontinuous.py index dc4f5616..69c5aa02 100644 --- a/carl/envs/gymnax/carl_jax_mountaincarcontinuous.py +++ b/carl/envs/gymnax/carl_jax_mountaincarcontinuous.py @@ -1,12 +1,12 @@ from typing import Dict, List, Optional, Union import jax.numpy as jnp - from gymnax.environments.classic_control.continuous_mountain_car import ( ContinuousMountainCar, ) from gymnax.environments.spaces import gymnax_space_to_gym_space from gymnax.wrappers.gym import GymnaxToGymWrapper + from carl.context.selection import AbstractSelector from carl.envs.carl_env import CARLEnv from carl.utils.trial_logger import TrialLogger @@ -37,16 +37,16 @@ "gravity": (0, jnp.inf, float), } + class CustomGymnaxToGymWrapper(GymnaxToGymWrapper): - @property - def observation_space(self): - return gymnax_space_to_gym_space( - self._env.observation_space(self.env_params) - ) + @property + def observation_space(self): + return gymnax_space_to_gym_space(self._env.observation_space(self.env_params)) + + @observation_space.setter + def observation_space(self, value): + self._observation_space = value - @observation_space.setter - def observation_space(self, value): - self._observation_space = value class CARLJaxMountainCarContinuousEnv(CARLEnv): def __init__( diff --git a/carl/envs/gymnax/carl_jax_pendulum.py b/carl/envs/gymnax/carl_jax_pendulum.py index d2ab45b9..bc23972e 100644 --- a/carl/envs/gymnax/carl_jax_pendulum.py +++ b/carl/envs/gymnax/carl_jax_pendulum.py @@ -1,9 +1,9 @@ from typing import Dict, List, Optional, Union +import jax.numpy as jnp from gymnax.environments.classic_control.pendulum import EnvParams, Pendulum -from gymnax.wrappers.gym import GymnaxToGymWrapper from gymnax.environments.spaces import gymnax_space_to_gym_space -import jax.numpy as jnp +from gymnax.wrappers.gym import GymnaxToGymWrapper from carl.context.selection import AbstractSelector from carl.envs.carl_env import CARLEnv @@ -30,16 +30,16 @@ "max_steps_in_episode": (1, jnp.inf, int), } + class CustomGymnaxToGymWrapper(GymnaxToGymWrapper): - @property - def observation_space(self): - return gymnax_space_to_gym_space( - self._env.observation_space(self.env_params) - ) + @property + def observation_space(self): + return gymnax_space_to_gym_space(self._env.observation_space(self.env_params)) + + @observation_space.setter + def observation_space(self, value): + self._observation_space = value - @observation_space.setter - def observation_space(self, value): - self._observation_space = value class CARLJaxPendulumEnv(CARLEnv): def __init__( From 92458e0aefae9900293e92c57239bb085ba2ce0c Mon Sep 17 00:00:00 2001 From: Abbas Dehghan Date: Sat, 27 May 2023 21:52:40 +0200 Subject: [PATCH 15/37] make precommit --- carl/envs/gymnax/carl_jax_acrobot.py | 6 +++--- carl/envs/gymnax/carl_jax_cartpole.py | 6 +++--- carl/envs/gymnax/carl_jax_mountaincar.py | 6 +++--- carl/envs/gymnax/carl_jax_mountaincarcontinuous.py | 6 +++--- carl/envs/gymnax/carl_jax_pendulum.py | 6 +++--- 5 files changed, 15 insertions(+), 15 deletions(-) diff --git a/carl/envs/gymnax/carl_jax_acrobot.py b/carl/envs/gymnax/carl_jax_acrobot.py index fb3831fd..8b59dcbb 100644 --- a/carl/envs/gymnax/carl_jax_acrobot.py +++ b/carl/envs/gymnax/carl_jax_acrobot.py @@ -3,7 +3,7 @@ import jax.numpy as jnp import numpy as np from gymnax.environments.classic_control.acrobot import Acrobot -from gymnax.environments.spaces import gymnax_space_to_gym_space +from gymnax.environments.spaces import Space, gymnax_space_to_gym_space from gymnax.wrappers.gym import GymnaxToGymWrapper from carl.context.selection import AbstractSelector @@ -63,11 +63,11 @@ class CustomGymnaxToGymWrapper(GymnaxToGymWrapper): @property - def observation_space(self): + def observation_space(self) -> Dict: return gymnax_space_to_gym_space(self._env.observation_space(self.env_params)) @observation_space.setter - def observation_space(self, value): + def observation_space(self, value: Space) -> None: self._observation_space = value diff --git a/carl/envs/gymnax/carl_jax_cartpole.py b/carl/envs/gymnax/carl_jax_cartpole.py index 94f6809b..2066b1c8 100644 --- a/carl/envs/gymnax/carl_jax_cartpole.py +++ b/carl/envs/gymnax/carl_jax_cartpole.py @@ -2,7 +2,7 @@ import jax.numpy as jnp from gymnax.environments.classic_control.cartpole import CartPole -from gymnax.environments.spaces import gymnax_space_to_gym_space +from gymnax.environments.spaces import Space, gymnax_space_to_gym_space from gymnax.wrappers.gym import GymnaxToGymWrapper from carl.context.selection import AbstractSelector @@ -31,11 +31,11 @@ class CustomGymnaxToGymWrapper(GymnaxToGymWrapper): @property - def observation_space(self): + def observation_space(self) -> Dict: return gymnax_space_to_gym_space(self._env.observation_space(self.env_params)) @observation_space.setter - def observation_space(self, value): + def observation_space(self, value: Space) -> None: self._observation_space = value diff --git a/carl/envs/gymnax/carl_jax_mountaincar.py b/carl/envs/gymnax/carl_jax_mountaincar.py index 0dbac5b7..b4c4c7d3 100644 --- a/carl/envs/gymnax/carl_jax_mountaincar.py +++ b/carl/envs/gymnax/carl_jax_mountaincar.py @@ -2,7 +2,7 @@ import jax.numpy as jnp from gymnax.environments.classic_control.mountain_car import MountainCar -from gymnax.environments.spaces import gymnax_space_to_gym_space +from gymnax.environments.spaces import Space, gymnax_space_to_gym_space from gymnax.wrappers.gym import GymnaxToGymWrapper from carl.context.selection import AbstractSelector @@ -33,11 +33,11 @@ class CustomGymnaxToGymWrapper(GymnaxToGymWrapper): @property - def observation_space(self): + def observation_space(self) -> Dict: return gymnax_space_to_gym_space(self._env.observation_space(self.env_params)) @observation_space.setter - def observation_space(self, value): + def observation_space(self, value: Space) -> None: self._observation_space = value diff --git a/carl/envs/gymnax/carl_jax_mountaincarcontinuous.py b/carl/envs/gymnax/carl_jax_mountaincarcontinuous.py index 69c5aa02..8d6574ee 100644 --- a/carl/envs/gymnax/carl_jax_mountaincarcontinuous.py +++ b/carl/envs/gymnax/carl_jax_mountaincarcontinuous.py @@ -4,7 +4,7 @@ from gymnax.environments.classic_control.continuous_mountain_car import ( ContinuousMountainCar, ) -from gymnax.environments.spaces import gymnax_space_to_gym_space +from gymnax.environments.spaces import Space, gymnax_space_to_gym_space from gymnax.wrappers.gym import GymnaxToGymWrapper from carl.context.selection import AbstractSelector @@ -40,11 +40,11 @@ class CustomGymnaxToGymWrapper(GymnaxToGymWrapper): @property - def observation_space(self): + def observation_space(self) -> Dict: return gymnax_space_to_gym_space(self._env.observation_space(self.env_params)) @observation_space.setter - def observation_space(self, value): + def observation_space(self, value: Space) -> None: self._observation_space = value diff --git a/carl/envs/gymnax/carl_jax_pendulum.py b/carl/envs/gymnax/carl_jax_pendulum.py index bc23972e..a4b485e3 100644 --- a/carl/envs/gymnax/carl_jax_pendulum.py +++ b/carl/envs/gymnax/carl_jax_pendulum.py @@ -2,7 +2,7 @@ import jax.numpy as jnp from gymnax.environments.classic_control.pendulum import EnvParams, Pendulum -from gymnax.environments.spaces import gymnax_space_to_gym_space +from gymnax.environments.spaces import Space, gymnax_space_to_gym_space from gymnax.wrappers.gym import GymnaxToGymWrapper from carl.context.selection import AbstractSelector @@ -33,11 +33,11 @@ class CustomGymnaxToGymWrapper(GymnaxToGymWrapper): @property - def observation_space(self): + def observation_space(self) -> Dict: return gymnax_space_to_gym_space(self._env.observation_space(self.env_params)) @observation_space.setter - def observation_space(self, value): + def observation_space(self, value: Space) -> None: self._observation_space = value From 9de0e29ffe0d31a3185ab50665ab7bd8044a701c Mon Sep 17 00:00:00 2001 From: Carolin Benjamins Date: Wed, 7 Jun 2023 14:52:59 +0200 Subject: [PATCH 16/37] Fix gymnax --- carl/envs/gymnax/__init__.py | 14 +- carl/envs/gymnax/carl_gymnax_env.py | 83 ++++++++++++ carl/envs/gymnax/carl_jax_acrobot.py | 114 ++++++---------- carl/envs/gymnax/carl_jax_cartpole.py | 87 +++--------- carl/envs/gymnax/carl_jax_mountaincar.py | 115 +++++----------- .../gymnax/carl_jax_mountaincarcontinuous.py | 125 ------------------ carl/envs/gymnax/carl_jax_pendulum.py | 81 ++---------- carl/envs/gymnax/wrappers.py | 31 +++++ test/test_all_envs.py | 1 + 9 files changed, 224 insertions(+), 427 deletions(-) create mode 100644 carl/envs/gymnax/carl_gymnax_env.py delete mode 100644 carl/envs/gymnax/carl_jax_mountaincarcontinuous.py create mode 100644 carl/envs/gymnax/wrappers.py diff --git a/carl/envs/gymnax/__init__.py b/carl/envs/gymnax/__init__.py index 7d38c4bc..a983dfa9 100644 --- a/carl/envs/gymnax/__init__.py +++ b/carl/envs/gymnax/__init__.py @@ -13,19 +13,19 @@ ) from carl.envs.gymnax.carl_jax_cartpole import CARLJaxCartPoleEnv # noqa: F401 from carl.envs.gymnax.carl_jax_mountaincar import ( # noqa: F401 + CONTEXT_BOUNDS as CARLJaxMountainCarContinuousEnv_bounds, +) +from carl.envs.gymnax.carl_jax_mountaincar import ( CONTEXT_BOUNDS as CARLJaxMountainCarEnv_bounds, ) from carl.envs.gymnax.carl_jax_mountaincar import ( # noqa: F401 + DEFAULT_CONTEXT as CARLJaxMountainCarContinuousEnv_defaults, +) +from carl.envs.gymnax.carl_jax_mountaincar import ( DEFAULT_CONTEXT as CARLJaxMountainCarEnv_defaults, ) from carl.envs.gymnax.carl_jax_mountaincar import CARLJaxMountainCarEnv # noqa: F401 -from carl.envs.gymnax.carl_jax_mountaincarcontinuous import ( # noqa: F401 - CONTEXT_BOUNDS as CARLJaxMountainCarContinuousEnv_bounds, -) -from carl.envs.gymnax.carl_jax_mountaincarcontinuous import ( # noqa: F401 - DEFAULT_CONTEXT as CARLJaxMountainCarContinuousEnv_defaults, -) -from carl.envs.gymnax.carl_jax_mountaincarcontinuous import ( # noqa: F401 +from carl.envs.gymnax.carl_jax_mountaincar import ( # noqa: F401 CARLJaxMountainCarContinuousEnv, ) from carl.envs.gymnax.carl_jax_pendulum import ( # noqa: F401 diff --git a/carl/envs/gymnax/carl_gymnax_env.py b/carl/envs/gymnax/carl_gymnax_env.py new file mode 100644 index 00000000..0bb5c9b0 --- /dev/null +++ b/carl/envs/gymnax/carl_gymnax_env.py @@ -0,0 +1,83 @@ +from typing import Any, Dict, List, Optional, Union + +import gymnasium + +from carl.context.selection import AbstractSelector +from carl.envs.carl_env import CARLEnv +from carl.envs.gymnax.wrappers import make_gymnax_env +from carl.utils.trial_logger import TrialLogger +from carl.utils.types import Context, Contexts + + +class CARLGymnaxEnv(CARLEnv): + env_name: str + DEFAULT_CONTEXT: Context + max_episode_steps: int + + def __init__( + self, + env: gymnasium.Env | None = None, + contexts: Contexts = {}, + hide_context: bool = True, + add_gaussian_noise_to_context: bool = False, + gaussian_noise_std_percentage: float = 0.01, + logger: Optional[TrialLogger] = None, + scale_context_features: str = "no", + default_context: Optional[Context] = None, + state_context_features: Optional[List[str]] = None, + context_mask: Optional[List[str]] = None, + dict_observation_space: bool = False, + context_selector: Optional[ + Union[AbstractSelector, type[AbstractSelector]] + ] = None, + context_selector_kwargs: Optional[Dict] = None, + ): + """ + Max torque is not a context feature because it changes the action space. + + Parameters + ---------- + env + contexts + instance_mode + hide_context + add_gaussian_noise_to_context + gaussian_noise_std_percentage + """ + if env is None: + env = make_gymnax_env(env_name=self.env_name) + + if not contexts: + contexts = {0: self.DEFAULT_CONTEXT} + + if not default_context: + default_context = self.DEFAULT_CONTEXT + + super().__init__( + env=env, + contexts=contexts, + hide_context=hide_context, + add_gaussian_noise_to_context=add_gaussian_noise_to_context, + gaussian_noise_std_percentage=gaussian_noise_std_percentage, + logger=logger, + scale_context_features=scale_context_features, + default_context=default_context, + max_episode_length=self.max_episode_steps, + state_context_features=state_context_features, + dict_observation_space=dict_observation_space, + context_selector=context_selector, + context_selector_kwargs=context_selector_kwargs, + context_mask=context_mask, + ) + self.whitelist_gaussian_noise = list( + self.DEFAULT_CONTEXT.keys() + ) # allow to augment all values + + def _update_context(self) -> None: + raise NotImplementedError + + def __getattr__(self, name: str) -> Any: + if name in ["sys", "__getstate__"]: + return getattr(self.env._environment, name) + else: + return getattr(self, name) diff --git a/carl/envs/gymnax/carl_jax_acrobot.py b/carl/envs/gymnax/carl_jax_acrobot.py index 8b59dcbb..2e76620b 100644 --- a/carl/envs/gymnax/carl_jax_acrobot.py +++ b/carl/envs/gymnax/carl_jax_acrobot.py @@ -1,13 +1,16 @@ +from __future__ import annotations + from typing import Dict, List, Optional, Union +import gymnasium +import gymnax import jax.numpy as jnp import numpy as np from gymnax.environments.classic_control.acrobot import Acrobot -from gymnax.environments.spaces import Space, gymnax_space_to_gym_space -from gymnax.wrappers.gym import GymnaxToGymWrapper from carl.context.selection import AbstractSelector from carl.envs.carl_env import CARLEnv +from carl.envs.gymnax.carl_gymnax_env import CARLGymnaxEnv from carl.utils.trial_logger import TrialLogger from carl.utils.types import Context, Contexts @@ -16,11 +19,13 @@ "link_length_2": 1, "link_mass_1": 1, "link_mass_2": 1, - "link_com_1": 0.5, - "link_com_2": 0.5, + "link_com_pos_1": 0.5, + "link_com_pos_2": 0.5, "link_moi": 1, - "max_velocity_1": 4 * jnp.pi, - "max_velocity_2": 9 * jnp.pi, + "max_vel_1": 4 * jnp.pi, + "max_vel_2": 9 * jnp.pi, + "torque_noise_max": 0.0, + "max_steps_in_episode": 500, } CONTEXT_BOUNDS = { @@ -36,97 +41,54 @@ float, ), # Link mass can be shrunken and grown by a factor of 10 "link_mass_2": (0.1, 10, float), - "link_com_1": (0, 1, float), # Center of mass can move from one end to the other - "link_com_2": (0, 1, float), + "link_com_pos_1": ( + 0, + 1, + float, + ), # Center of mass can move from one end to the other + "link_com_pos_2": (0, 1, float), "link_moi": ( 0.1, 10, float, ), # Moments on inertia can be shrunken and grown by a factor of 10 - "max_velocity_1": ( + "max_vel_1": ( 0.4 * np.pi, 40 * np.pi, float, ), # Velocity can vary by a factor of 10 in either direction - "max_velocity_2": (0.9 * np.pi, 90 * np.pi, float), + "max_vel_2": (0.9 * np.pi, 90 * np.pi, float), "torque_noise_max": ( -1.0, 1.0, float, ), # torque is either {-1., 0., 1}. Applying noise of 1. would be quite extreme - "initial_angle_lower": (-jnp.inf, jnp.inf, float), - "initial_angle_upper": (-jnp.inf, jnp.inf, float), - "initial_velocity_lower": (-jnp.inf, jnp.inf, float), - "initial_velocity_upper": (-jnp.inf, jnp.inf, float), + "max_steps_in_episode": (1, jnp.inf, int), } -class CustomGymnaxToGymWrapper(GymnaxToGymWrapper): - @property - def observation_space(self) -> Dict: - return gymnax_space_to_gym_space(self._env.observation_space(self.env_params)) - - @observation_space.setter - def observation_space(self, value: Space) -> None: - self._observation_space = value - - -class CARLJaxAcrobotEnv(CARLEnv): - def __init__( - self, - env: Acrobot = CustomGymnaxToGymWrapper(Acrobot()), - contexts: Contexts = {}, - hide_context: bool = True, - add_gaussian_noise_to_context: bool = False, - gaussian_noise_std_percentage: float = 0.01, - logger: Optional[TrialLogger] = None, - scale_context_features: str = "no", - default_context: Optional[Context] = DEFAULT_CONTEXT, - max_episode_length: int = 500, # from https://github.com/openai/gym/blob/master/gym/envs/__init__.py - state_context_features: Optional[List[str]] = None, - context_mask: Optional[List[str]] = None, - dict_observation_space: bool = False, - context_selector: Optional[ - Union[AbstractSelector, type[AbstractSelector]] - ] = None, - context_selector_kwargs: Optional[Dict] = None, - ): - if not contexts: - contexts = {0: DEFAULT_CONTEXT} - super().__init__( - env=env, - contexts=contexts, - hide_context=hide_context, - add_gaussian_noise_to_context=add_gaussian_noise_to_context, - gaussian_noise_std_percentage=gaussian_noise_std_percentage, - logger=logger, - scale_context_features=scale_context_features, - default_context=default_context, - max_episode_length=max_episode_length, - state_context_features=state_context_features, - dict_observation_space=dict_observation_space, - context_selector=context_selector, - context_selector_kwargs=context_selector_kwargs, - context_mask=context_mask, - ) - self.whitelist_gaussian_noise = list( - DEFAULT_CONTEXT.keys() - ) # allow to augment all values +class CARLJaxAcrobotEnv(CARLGymnaxEnv): + env_name: str = "Acrobot-v1" + max_episode_steps: int = DEFAULT_CONTEXT["max_steps_in_episode"] + DEFAULT_CONTEXT: Context = DEFAULT_CONTEXT def _update_context(self) -> None: - self.env: Acrobot - self.env.LINK_LENGTH_1 = self.context["link_length_1"] - self.env.LINK_LENGTH_2 = self.context["link_length_2"] - self.env.LINK_MASS_1 = self.context["link_mass_1"] - self.env.LINK_MASS_2 = self.context["link_mass_2"] - self.env.LINK_COM_POS_1 = self.context["link_com_1"] - self.env.LINK_COM_POS_2 = self.context["link_com_2"] - self.env.LINK_MOI = self.context["link_moi"] - self.env.MAX_VEL_1 = self.context["max_velocity_1"] - self.env.MAX_VEL_2 = self.context["max_velocity_2"] + content = self.env.env.env_params.__dict__ + content.update(self.context) + # We cannot directly set attributes of env_params because it is a frozen dataclass + self.env.env.env_params = gymnax.environments.classic_control.acrobot.EnvParams( + **content + ) high = jnp.array( - [1.0, 1.0, 1.0, 1.0, self.env.MAX_VEL_1, self.env.MAX_VEL_2], + [ + 1.0, + 1.0, + 1.0, + 1.0, + self.env.env.env_params.max_vel_1, + self.env.env.env_params.max_vel_2, + ], dtype=jnp.float32, ) low = -high diff --git a/carl/envs/gymnax/carl_jax_cartpole.py b/carl/envs/gymnax/carl_jax_cartpole.py index 2066b1c8..5e5bb755 100644 --- a/carl/envs/gymnax/carl_jax_cartpole.py +++ b/carl/envs/gymnax/carl_jax_cartpole.py @@ -1,13 +1,10 @@ -from typing import Dict, List, Optional, Union +from __future__ import annotations +import gymnax import jax.numpy as jnp from gymnax.environments.classic_control.cartpole import CartPole -from gymnax.environments.spaces import Space, gymnax_space_to_gym_space -from gymnax.wrappers.gym import GymnaxToGymWrapper -from carl.context.selection import AbstractSelector -from carl.envs.carl_env import CARLEnv -from carl.utils.trial_logger import TrialLogger +from carl.envs.gymnax.carl_gymnax_env import CARLGymnaxEnv from carl.utils.types import Context, Contexts DEFAULT_CONTEXT = { @@ -17,6 +14,9 @@ "length": 0.5, "force_mag": 10.0, "tau": 0.02, + "polemass_length": None, + "total_mass": None, + "max_steps_in_episode": 500, } CONTEXT_BOUNDS = { @@ -26,75 +26,32 @@ "length": (0.25, 1.0, float), "force_mag": (5.0, 15.0, float), "tau": (0.01, 0.05, float), + "polemass_length": (0, jnp.inf, float), + "total_mass": (0, jnp.inf, float), + "max_steps_in_episode": (1, jnp.inf, int), } -class CustomGymnaxToGymWrapper(GymnaxToGymWrapper): - @property - def observation_space(self) -> Dict: - return gymnax_space_to_gym_space(self._env.observation_space(self.env_params)) +class CARLJaxCartPoleEnv(CARLGymnaxEnv): + env_name: str = "CartPole-v1" + max_episode_steps: int = DEFAULT_CONTEXT["max_steps_in_episode"] + DEFAULT_CONTEXT: Context = DEFAULT_CONTEXT - @observation_space.setter - def observation_space(self, value: Space) -> None: - self._observation_space = value - - -class CARLJaxCartPoleEnv(CARLEnv): - def __init__( - self, - env: CartPole = CustomGymnaxToGymWrapper(CartPole()), - contexts: Contexts = {}, - hide_context: bool = True, - add_gaussian_noise_to_context: bool = False, - gaussian_noise_std_percentage: float = 0.01, - logger: Optional[TrialLogger] = None, - scale_context_features: str = "no", - default_context: Optional[Context] = DEFAULT_CONTEXT, - max_episode_length: int = 500, # from https://github.com/openai/gym/blob/master/gym/envs/__init__.py - state_context_features: Optional[List[str]] = None, - context_mask: Optional[List[str]] = None, - dict_observation_space: bool = False, - context_selector: Optional[ - Union[AbstractSelector, type[AbstractSelector]] - ] = None, - context_selector_kwargs: Optional[Dict] = None, - ): - if not contexts: - contexts = {0: DEFAULT_CONTEXT} - super().__init__( - env=env, - contexts=contexts, - hide_context=hide_context, - add_gaussian_noise_to_context=add_gaussian_noise_to_context, - gaussian_noise_std_percentage=gaussian_noise_std_percentage, - logger=logger, - scale_context_features=scale_context_features, - default_context=default_context, - max_episode_length=max_episode_length, - state_context_features=state_context_features, - dict_observation_space=dict_observation_space, - context_selector=context_selector, - context_selector_kwargs=context_selector_kwargs, - context_mask=context_mask, + def _update_context(self) -> None: + self.context["polemass_length"] = ( + self.context["masspole"] * self.context["length"] ) - self.whitelist_gaussian_noise = list( - DEFAULT_CONTEXT.keys() - ) # allow to augment all values + self.context["total_mass"] = self.context["masscart"] + self.context["masspole"] - def _update_context(self) -> None: - self.env: CartPole - self.env.gravity = self.context["gravity"] - self.env.masscart = self.context["masscart"] - self.env.masspole = self.context["masspole"] - self.env.length = self.context["length"] - self.env.force_mag = self.context["force_mag"] - self.env.tau = self.context["tau"] + self.env.env.env_params = ( + gymnax.environments.classic_control.cartpole.EnvParams(**self.context) + ) high = jnp.array( [ - self.env.x_threshold * 2, + self.env.env.env_params.x_threshold * 2, jnp.finfo(jnp.float32).max, - self.env.theta_threshold_radians * 2, + self.env.env.env_params.theta_threshold_radians * 2, jnp.finfo(jnp.float32).max, ], dtype=jnp.float32, diff --git a/carl/envs/gymnax/carl_jax_mountaincar.py b/carl/envs/gymnax/carl_jax_mountaincar.py index b4c4c7d3..18c010e4 100644 --- a/carl/envs/gymnax/carl_jax_mountaincar.py +++ b/carl/envs/gymnax/carl_jax_mountaincar.py @@ -1,23 +1,27 @@ +from __future__ import annotations + from typing import Dict, List, Optional, Union +import gymnax import jax.numpy as jnp from gymnax.environments.classic_control.mountain_car import MountainCar -from gymnax.environments.spaces import Space, gymnax_space_to_gym_space -from gymnax.wrappers.gym import GymnaxToGymWrapper from carl.context.selection import AbstractSelector from carl.envs.carl_env import CARLEnv +from carl.envs.gymnax.carl_gymnax_env import CARLGymnaxEnv +from carl.envs.gymnax.wrappers import CustomGymnaxToGymWrapper from carl.utils.trial_logger import TrialLogger from carl.utils.types import Context, Contexts DEFAULT_CONTEXT = { - "min_position": -1.2, # unit? - "max_position": 0.6, # unit? - "max_speed": 0.07, # unit? - "goal_position": 0.5, # unit? - "goal_velocity": 0, # unit? - "force": 0.001, # unit? - "gravity": 0.0025, # unit? + "min_position": -1.2, + "max_position": 0.6, + "max_speed": 0.07, + "goal_position": 0.5, + "goal_velocity": 0, + "force": 0.001, + "gravity": 0.0025, + "max_steps_in_episode": 200, } CONTEXT_BOUNDS = { @@ -28,90 +32,33 @@ "goal_velocity": (-jnp.inf, jnp.inf, float), "force": (-jnp.inf, jnp.inf, float), "gravity": (0, jnp.inf, float), + "max_steps_in_episode": (1, jnp.inf, int), } -class CustomGymnaxToGymWrapper(GymnaxToGymWrapper): - @property - def observation_space(self) -> Dict: - return gymnax_space_to_gym_space(self._env.observation_space(self.env_params)) - - @observation_space.setter - def observation_space(self, value: Space) -> None: - self._observation_space = value - - -class CARLJaxMountainCarEnv(CARLEnv): - def __init__( - self, - env: MountainCar = CustomGymnaxToGymWrapper(MountainCar()), - contexts: Contexts = {}, - hide_context: bool = True, - add_gaussian_noise_to_context: bool = False, - gaussian_noise_std_percentage: float = 0.01, - logger: Optional[TrialLogger] = None, - scale_context_features: str = "no", - default_context: Optional[Context] = DEFAULT_CONTEXT, - max_episode_length: int = 200, # from https://github.com/openai/gym/blob/master/gym/envs/__init__.py - state_context_features: Optional[List[str]] = None, - context_mask: Optional[List[str]] = None, - dict_observation_space: bool = False, - context_selector: Optional[ - Union[AbstractSelector, type[AbstractSelector]] - ] = None, - context_selector_kwargs: Optional[Dict] = None, - ): - """ - - Parameters - ---------- - env: gym.Env, optional - Defaults to classic control environment mountain car from gym (MountainCarEnv). - contexts: List[Dict], optional - Different contexts / different environment parameter settings. - instance_mode: str, optional - """ - if not contexts: - contexts = {0: DEFAULT_CONTEXT} - super().__init__( - env=env, - contexts=contexts, - hide_context=hide_context, - add_gaussian_noise_to_context=add_gaussian_noise_to_context, - gaussian_noise_std_percentage=gaussian_noise_std_percentage, - logger=logger, - scale_context_features=scale_context_features, - default_context=default_context, - max_episode_length=max_episode_length, - state_context_features=state_context_features, - dict_observation_space=dict_observation_space, - context_selector=context_selector, - context_selector_kwargs=context_selector_kwargs, - context_mask=context_mask, - ) - self.whitelist_gaussian_noise = list( - DEFAULT_CONTEXT.keys() - ) # allow to augment all values +class CARLJaxMountainCarEnv(CARLGymnaxEnv): + env_name: str = "MountainCar-v0" + max_episode_steps: int = DEFAULT_CONTEXT["max_steps_in_episode"] + DEFAULT_CONTEXT: Context = DEFAULT_CONTEXT def _update_context(self) -> None: - self.env: CARLJaxMountainCarEnv - self.env.min_position = self.context["min_position"] - self.env.max_position = self.context["max_position"] - self.env.max_speed = self.context["max_speed"] - self.env.goal_position = self.context["goal_position"] - self.env.goal_velocity = self.context["goal_velocity"] - self.env.min_position_start = self.context["min_position_start"] - self.env.max_position_start = self.context["max_position_start"] - self.env.min_velocity_start = self.context["min_velocity_start"] - self.env.max_velocity_start = self.context["max_velocity_start"] - self.env.force = self.context["force"] - self.env.gravity = self.context["gravity"] + self.env.env.env_params = ( + gymnax.environments.classic_control.mountain_car.EnvParams(**self.context) + ) self.low = jnp.array( - [self.env.min_position, -self.env.max_speed], dtype=jnp.float32 + [self.env.env.env_params.min_position, -self.env.env.env_params.max_speed], + dtype=jnp.float32, ).squeeze() self.high = jnp.array( - [self.env.max_position, self.env.max_speed], dtype=jnp.float32 + [self.env.env.env_params.max_position, self.env.env.env_params.max_speed], + dtype=jnp.float32, ).squeeze() self.build_observation_space(self.low, self.high, CONTEXT_BOUNDS) + + +class CARLJaxMountainCarContinuousEnv(CARLJaxMountainCarEnv): + env_name: str = "MountainCarContinuous-v0" + max_episode_steps: int = 999 + DEFAULT_CONTEXT: Context = DEFAULT_CONTEXT diff --git a/carl/envs/gymnax/carl_jax_mountaincarcontinuous.py b/carl/envs/gymnax/carl_jax_mountaincarcontinuous.py deleted file mode 100644 index 8d6574ee..00000000 --- a/carl/envs/gymnax/carl_jax_mountaincarcontinuous.py +++ /dev/null @@ -1,125 +0,0 @@ -from typing import Dict, List, Optional, Union - -import jax.numpy as jnp -from gymnax.environments.classic_control.continuous_mountain_car import ( - ContinuousMountainCar, -) -from gymnax.environments.spaces import Space, gymnax_space_to_gym_space -from gymnax.wrappers.gym import GymnaxToGymWrapper - -from carl.context.selection import AbstractSelector -from carl.envs.carl_env import CARLEnv -from carl.utils.trial_logger import TrialLogger -from carl.utils.types import Context, Contexts - -DEFAULT_CONTEXT = { - "min_action": -1.0, - "max_action": 1.0, - "min_position": -1.2, - "max_position": 0.6, - "max_speed": 0.07, - "goal_position": 0.45, - "goal_velocity": 0.0, - "power": 0.0015, - "gravity": 0.0025, -} - - -CONTEXT_BOUNDS = { - "min_action": (-jnp.inf, jnp.inf, float), - "max_action:": (-jnp.inf, jnp.inf, float), - "min_position": (-jnp.inf, jnp.inf, float), - "max_position": (-jnp.inf, jnp.inf, float), - "max_speed": (0, jnp.inf, float), - "goal_position": (-jnp.inf, jnp.inf, float), - "goal_velocity": (-jnp.inf, jnp.inf, float), - "power": (-jnp.inf, jnp.inf, float), - "gravity": (0, jnp.inf, float), -} - - -class CustomGymnaxToGymWrapper(GymnaxToGymWrapper): - @property - def observation_space(self) -> Dict: - return gymnax_space_to_gym_space(self._env.observation_space(self.env_params)) - - @observation_space.setter - def observation_space(self, value: Space) -> None: - self._observation_space = value - - -class CARLJaxMountainCarContinuousEnv(CARLEnv): - def __init__( - self, - env: ContinuousMountainCar = CustomGymnaxToGymWrapper(ContinuousMountainCar()), - contexts: Contexts = {}, - hide_context: bool = True, - add_gaussian_noise_to_context: bool = True, - gaussian_noise_std_percentage: float = 0.01, - logger: Optional[TrialLogger] = None, - scale_context_features: str = "no", - default_context: Optional[Context] = DEFAULT_CONTEXT, - max_episode_length: int = 999, # from https://github.com/openai/gym/blob/master/gym/envs/__init__.py - state_context_features: Optional[List[str]] = None, - context_mask: Optional[List[str]] = None, - dict_observation_space: bool = False, - context_selector: Optional[ - Union[AbstractSelector, type[AbstractSelector]] - ] = None, - context_selector_kwargs: Optional[Dict] = None, - ): - """ - - Parameters - ---------- - env: gym.Env, optional - Defaults to classic control environment mountain car from gym (MountainCarEnv). - contexts: List[Dict], optional - Different contexts / different environment parameter settings. - instance_mode: str, optional - """ - if not contexts: - contexts = {0: DEFAULT_CONTEXT} - super().__init__( - env=env, - contexts=contexts, - hide_context=hide_context, - add_gaussian_noise_to_context=add_gaussian_noise_to_context, - gaussian_noise_std_percentage=gaussian_noise_std_percentage, - logger=logger, - scale_context_features=scale_context_features, - default_context=default_context, - max_episode_length=max_episode_length, - state_context_features=state_context_features, - dict_observation_space=dict_observation_space, - context_selector=context_selector, - context_selector_kwargs=context_selector_kwargs, - context_mask=context_mask, - ) - self.whitelist_gaussian_noise = list( - DEFAULT_CONTEXT.keys() - ) # allow to augment all values - - def _update_context(self) -> None: - self.env: ContinuousMountainCar - self.env.min_position = self.context["min_position"] - self.env.max_position = self.context["max_position"] - self.env.max_speed = self.context["max_speed"] - self.env.goal_position = self.context["goal_position"] - self.env.goal_velocity = self.context["goal_velocity"] - self.env.min_position_start = self.context["min_position_start"] - self.env.max_position_start = self.context["max_position_start"] - self.env.min_velocity_start = self.context["min_velocity_start"] - self.env.max_velocity_start = self.context["max_velocity_start"] - self.env.power = self.context["power"] - # self.env.force = self.context["force"] - # self.env.gravity = self.context["gravity"] - - self.low = jnp.array( - [self.env.min_position, -self.env.max_speed], dtype=jnp.float32 - ).squeeze() - self.high = jnp.array( - [self.env.max_position, self.env.max_speed], dtype=jnp.float32 - ).squeeze() - - self.build_observation_space(self.low, self.high, CONTEXT_BOUNDS) diff --git a/carl/envs/gymnax/carl_jax_pendulum.py b/carl/envs/gymnax/carl_jax_pendulum.py index a4b485e3..ff3574f2 100644 --- a/carl/envs/gymnax/carl_jax_pendulum.py +++ b/carl/envs/gymnax/carl_jax_pendulum.py @@ -1,13 +1,11 @@ +from __future__ import annotations + from typing import Dict, List, Optional, Union import jax.numpy as jnp from gymnax.environments.classic_control.pendulum import EnvParams, Pendulum -from gymnax.environments.spaces import Space, gymnax_space_to_gym_space -from gymnax.wrappers.gym import GymnaxToGymWrapper -from carl.context.selection import AbstractSelector -from carl.envs.carl_env import CARLEnv -from carl.utils.trial_logger import TrialLogger +from carl.envs.gymnax.carl_gymnax_env import CARLGymnaxEnv from carl.utils.types import Context, Contexts DEFAULT_CONTEXT = { @@ -31,72 +29,15 @@ } -class CustomGymnaxToGymWrapper(GymnaxToGymWrapper): - @property - def observation_space(self) -> Dict: - return gymnax_space_to_gym_space(self._env.observation_space(self.env_params)) - - @observation_space.setter - def observation_space(self, value: Space) -> None: - self._observation_space = value - - -class CARLJaxPendulumEnv(CARLEnv): - def __init__( - self, - env: Pendulum = CustomGymnaxToGymWrapper(Pendulum()), - contexts: Contexts = {}, - hide_context: bool = True, - add_gaussian_noise_to_context: bool = False, - gaussian_noise_std_percentage: float = 0.01, - logger: Optional[TrialLogger] = None, - scale_context_features: str = "no", - default_context: Optional[Context] = DEFAULT_CONTEXT, - max_episode_length: int = 200, # from https://github.com/openai/gym/blob/master/gym/envs/__init__.py - state_context_features: Optional[List[str]] = None, - context_mask: Optional[List[str]] = None, - dict_observation_space: bool = False, - context_selector: Optional[ - Union[AbstractSelector, type[AbstractSelector]] - ] = None, - context_selector_kwargs: Optional[Dict] = None, - ): - """ - Max torque is not a context feature because it changes the action space. - - Parameters - ---------- - env - contexts - instance_mode - hide_context - add_gaussian_noise_to_context - gaussian_noise_std_percentage - """ - if not contexts: - contexts = {0: DEFAULT_CONTEXT} - super().__init__( - env=env, - contexts=contexts, - hide_context=hide_context, - add_gaussian_noise_to_context=add_gaussian_noise_to_context, - gaussian_noise_std_percentage=gaussian_noise_std_percentage, - logger=logger, - scale_context_features=scale_context_features, - default_context=default_context, - max_episode_length=max_episode_length, - state_context_features=state_context_features, - dict_observation_space=dict_observation_space, - context_selector=context_selector, - context_selector_kwargs=context_selector_kwargs, - context_mask=context_mask, - ) - self.whitelist_gaussian_noise = list( - DEFAULT_CONTEXT.keys() - ) # allow to augment all values +class CARLJaxPendulumEnv(CARLGymnaxEnv): + env_name: str = "Pendulum-v1" + max_episode_steps: int = DEFAULT_CONTEXT["max_steps_in_episode"] + DEFAULT_CONTEXT: Context = DEFAULT_CONTEXT def _update_context(self) -> None: - self.env_params = EnvParams(**self.context) + self.env.env.env_params = EnvParams(**self.context) - high = jnp.array([1.0, 1.0, self.max_speed], dtype=jnp.float32) + high = jnp.array( + [1.0, 1.0, self.env.env.env_params.max_speed], dtype=jnp.float32 + ) self.build_observation_space(-high, high, CONTEXT_BOUNDS) diff --git a/carl/envs/gymnax/wrappers.py b/carl/envs/gymnax/wrappers.py new file mode 100644 index 00000000..1c08e905 --- /dev/null +++ b/carl/envs/gymnax/wrappers.py @@ -0,0 +1,31 @@ +from __future__ import annotations + +import gym +import gymnasium +import gymnax +from gymnasium.wrappers import EnvCompatibility +from gymnax.environments.spaces import Space, gymnax_space_to_gym_space +from gymnax.wrappers.gym import GymnaxToGymWrapper + + +class CustomGymnaxToGymWrapper(GymnaxToGymWrapper): + @property + def observation_space(self) -> dict: + return gymnax_space_to_gym_space(self._env.observation_space(self.env_params)) + + @observation_space.setter + def observation_space(self, value: Space) -> None: + self._observation_space = value + + +def make_gymnax_env(env_name: str) -> gymnasium.Env: + # Make gymnax env + env, env_params = gymnax.make(env_id=env_name) + + # Convert gymnax to gym API + env = CustomGymnaxToGymWrapper(env=env, params=env_params) + + # Convert gym to gymnasium API + env = EnvCompatibility(old_env=env) + + return env diff --git a/test/test_all_envs.py b/test/test_all_envs.py index 141b06b4..1d1e1240 100644 --- a/test/test_all_envs.py +++ b/test/test_all_envs.py @@ -16,6 +16,7 @@ def test_init_all_envs(self): env = ( # noqa: F841 local variable is assigned to but never used var() ) + _ = env.reset() except Exception as e: print(f"Cannot instantiate {var} environment.") raise e From 0bba185a8668403b995e55966f781593b830b65f Mon Sep 17 00:00:00 2001 From: Carolin Benjamins Date: Wed, 7 Jun 2023 15:04:06 +0200 Subject: [PATCH 17/37] Fix pre-commit --- carl/envs/gymnax/__init__.py | 29 +++++++++++----------- carl/envs/gymnax/carl_gymnax_env.py | 4 ++- carl/envs/gymnax/carl_jax_acrobot.py | 11 ++------ carl/envs/gymnax/carl_jax_cartpole.py | 5 ++-- carl/envs/gymnax/carl_jax_mountaincar.py | 11 ++------ carl/envs/gymnax/carl_jax_pendulum.py | 8 +++--- carl/envs/gymnax/{wrappers.py => utils.py} | 1 - 7 files changed, 26 insertions(+), 43 deletions(-) rename carl/envs/gymnax/{wrappers.py => utils.py} (98%) diff --git a/carl/envs/gymnax/__init__.py b/carl/envs/gymnax/__init__.py index a983dfa9..39bccb14 100644 --- a/carl/envs/gymnax/__init__.py +++ b/carl/envs/gymnax/__init__.py @@ -1,37 +1,36 @@ -from carl.envs.gymnax.carl_jax_acrobot import ( # noqa: F401 - CONTEXT_BOUNDS as CARLJaxAcrobotEnv_bounds, -) -from carl.envs.gymnax.carl_jax_acrobot import ( # noqa: F401 +# flake8: noqa: F401 +from carl.envs.gymnax.carl_jax_acrobot import CONTEXT_BOUNDS as CARLJaxAcrobotEnv_bounds +from carl.envs.gymnax.carl_jax_acrobot import ( DEFAULT_CONTEXT as CARLJaxAcrobotEnv_defaults, ) -from carl.envs.gymnax.carl_jax_acrobot import CARLJaxAcrobotEnv # noqa: F401 -from carl.envs.gymnax.carl_jax_cartpole import ( # noqa: F401 +from carl.envs.gymnax.carl_jax_acrobot import CARLJaxAcrobotEnv +from carl.envs.gymnax.carl_jax_cartpole import ( CONTEXT_BOUNDS as CARLJaxCartPoleEnv_bounds, ) -from carl.envs.gymnax.carl_jax_cartpole import ( # noqa: F401 +from carl.envs.gymnax.carl_jax_cartpole import ( DEFAULT_CONTEXT as CARLJaxCartPoleEnv_defaults, ) -from carl.envs.gymnax.carl_jax_cartpole import CARLJaxCartPoleEnv # noqa: F401 -from carl.envs.gymnax.carl_jax_mountaincar import ( # noqa: F401 +from carl.envs.gymnax.carl_jax_cartpole import CARLJaxCartPoleEnv +from carl.envs.gymnax.carl_jax_mountaincar import ( CONTEXT_BOUNDS as CARLJaxMountainCarContinuousEnv_bounds, ) from carl.envs.gymnax.carl_jax_mountaincar import ( CONTEXT_BOUNDS as CARLJaxMountainCarEnv_bounds, ) -from carl.envs.gymnax.carl_jax_mountaincar import ( # noqa: F401 +from carl.envs.gymnax.carl_jax_mountaincar import ( DEFAULT_CONTEXT as CARLJaxMountainCarContinuousEnv_defaults, ) from carl.envs.gymnax.carl_jax_mountaincar import ( DEFAULT_CONTEXT as CARLJaxMountainCarEnv_defaults, ) -from carl.envs.gymnax.carl_jax_mountaincar import CARLJaxMountainCarEnv # noqa: F401 -from carl.envs.gymnax.carl_jax_mountaincar import ( # noqa: F401 +from carl.envs.gymnax.carl_jax_mountaincar import ( CARLJaxMountainCarContinuousEnv, + CARLJaxMountainCarEnv, ) -from carl.envs.gymnax.carl_jax_pendulum import ( # noqa: F401 +from carl.envs.gymnax.carl_jax_pendulum import ( CONTEXT_BOUNDS as CARLJaxPendulumEnv_bounds, ) -from carl.envs.gymnax.carl_jax_pendulum import ( # noqa: F401 +from carl.envs.gymnax.carl_jax_pendulum import ( DEFAULT_CONTEXT as CARLJaxPendulumEnv_defaults, ) -from carl.envs.gymnax.carl_jax_pendulum import CARLJaxPendulumEnv # noqa: F401 +from carl.envs.gymnax.carl_jax_pendulum import CARLJaxPendulumEnv diff --git a/carl/envs/gymnax/carl_gymnax_env.py b/carl/envs/gymnax/carl_gymnax_env.py index 0bb5c9b0..ec768e43 100644 --- a/carl/envs/gymnax/carl_gymnax_env.py +++ b/carl/envs/gymnax/carl_gymnax_env.py @@ -1,10 +1,12 @@ +from __future__ import annotations + from typing import Any, Dict, List, Optional, Union import gymnasium +from CARL.carl.envs.gymnax.utils import make_gymnax_env from carl.context.selection import AbstractSelector from carl.envs.carl_env import CARLEnv -from carl.envs.gymnax.wrappers import make_gymnax_env from carl.utils.trial_logger import TrialLogger from carl.utils.types import Context, Contexts diff --git a/carl/envs/gymnax/carl_jax_acrobot.py b/carl/envs/gymnax/carl_jax_acrobot.py index 2e76620b..d07c1e8b 100644 --- a/carl/envs/gymnax/carl_jax_acrobot.py +++ b/carl/envs/gymnax/carl_jax_acrobot.py @@ -1,18 +1,11 @@ from __future__ import annotations -from typing import Dict, List, Optional, Union - -import gymnasium import gymnax import jax.numpy as jnp import numpy as np -from gymnax.environments.classic_control.acrobot import Acrobot -from carl.context.selection import AbstractSelector -from carl.envs.carl_env import CARLEnv from carl.envs.gymnax.carl_gymnax_env import CARLGymnaxEnv -from carl.utils.trial_logger import TrialLogger -from carl.utils.types import Context, Contexts +from carl.utils.types import Context DEFAULT_CONTEXT = { "link_length_1": 1, @@ -69,7 +62,7 @@ class CARLJaxAcrobotEnv(CARLGymnaxEnv): env_name: str = "Acrobot-v1" - max_episode_steps: int = DEFAULT_CONTEXT["max_steps_in_episode"] + max_episode_steps: int = int(DEFAULT_CONTEXT["max_steps_in_episode"]) DEFAULT_CONTEXT: Context = DEFAULT_CONTEXT def _update_context(self) -> None: diff --git a/carl/envs/gymnax/carl_jax_cartpole.py b/carl/envs/gymnax/carl_jax_cartpole.py index 5e5bb755..3110f850 100644 --- a/carl/envs/gymnax/carl_jax_cartpole.py +++ b/carl/envs/gymnax/carl_jax_cartpole.py @@ -2,10 +2,9 @@ import gymnax import jax.numpy as jnp -from gymnax.environments.classic_control.cartpole import CartPole from carl.envs.gymnax.carl_gymnax_env import CARLGymnaxEnv -from carl.utils.types import Context, Contexts +from carl.utils.types import Context DEFAULT_CONTEXT = { "gravity": 9.8, @@ -34,7 +33,7 @@ class CARLJaxCartPoleEnv(CARLGymnaxEnv): env_name: str = "CartPole-v1" - max_episode_steps: int = DEFAULT_CONTEXT["max_steps_in_episode"] + max_episode_steps: int = int(DEFAULT_CONTEXT["max_steps_in_episode"]) # type: ignore[arg-type] DEFAULT_CONTEXT: Context = DEFAULT_CONTEXT def _update_context(self) -> None: diff --git a/carl/envs/gymnax/carl_jax_mountaincar.py b/carl/envs/gymnax/carl_jax_mountaincar.py index 18c010e4..75cc344b 100644 --- a/carl/envs/gymnax/carl_jax_mountaincar.py +++ b/carl/envs/gymnax/carl_jax_mountaincar.py @@ -1,17 +1,10 @@ from __future__ import annotations -from typing import Dict, List, Optional, Union - import gymnax import jax.numpy as jnp -from gymnax.environments.classic_control.mountain_car import MountainCar -from carl.context.selection import AbstractSelector -from carl.envs.carl_env import CARLEnv from carl.envs.gymnax.carl_gymnax_env import CARLGymnaxEnv -from carl.envs.gymnax.wrappers import CustomGymnaxToGymWrapper -from carl.utils.trial_logger import TrialLogger -from carl.utils.types import Context, Contexts +from carl.utils.types import Context DEFAULT_CONTEXT = { "min_position": -1.2, @@ -38,7 +31,7 @@ class CARLJaxMountainCarEnv(CARLGymnaxEnv): env_name: str = "MountainCar-v0" - max_episode_steps: int = DEFAULT_CONTEXT["max_steps_in_episode"] + max_episode_steps: int = int(DEFAULT_CONTEXT["max_steps_in_episode"]) DEFAULT_CONTEXT: Context = DEFAULT_CONTEXT def _update_context(self) -> None: diff --git a/carl/envs/gymnax/carl_jax_pendulum.py b/carl/envs/gymnax/carl_jax_pendulum.py index ff3574f2..85faa27b 100644 --- a/carl/envs/gymnax/carl_jax_pendulum.py +++ b/carl/envs/gymnax/carl_jax_pendulum.py @@ -1,12 +1,10 @@ from __future__ import annotations -from typing import Dict, List, Optional, Union - import jax.numpy as jnp -from gymnax.environments.classic_control.pendulum import EnvParams, Pendulum +from gymnax.environments.classic_control.pendulum import EnvParams from carl.envs.gymnax.carl_gymnax_env import CARLGymnaxEnv -from carl.utils.types import Context, Contexts +from carl.utils.types import Context DEFAULT_CONTEXT = { "max_speed": 8.0, @@ -31,7 +29,7 @@ class CARLJaxPendulumEnv(CARLGymnaxEnv): env_name: str = "Pendulum-v1" - max_episode_steps: int = DEFAULT_CONTEXT["max_steps_in_episode"] + max_episode_steps: int = int(DEFAULT_CONTEXT["max_steps_in_episode"]) DEFAULT_CONTEXT: Context = DEFAULT_CONTEXT def _update_context(self) -> None: diff --git a/carl/envs/gymnax/wrappers.py b/carl/envs/gymnax/utils.py similarity index 98% rename from carl/envs/gymnax/wrappers.py rename to carl/envs/gymnax/utils.py index 1c08e905..ca400ee5 100644 --- a/carl/envs/gymnax/wrappers.py +++ b/carl/envs/gymnax/utils.py @@ -1,6 +1,5 @@ from __future__ import annotations -import gym import gymnasium import gymnax from gymnasium.wrappers import EnvCompatibility From 97ae965f2723c77eeb94c71d6f5488c4f4a5940f Mon Sep 17 00:00:00 2001 From: benjamc Date: Mon, 11 Dec 2023 10:48:33 +0100 Subject: [PATCH 18/37] Move files --- carl/envs/gymnax/carl_gymnax_env.py | 85 +++++++++---------- .../{ => classic_control}/carl_jax_acrobot.py | 0 .../carl_jax_cartpole.py | 0 .../carl_jax_mountaincar.py | 0 .../carl_jax_pendulum.py | 0 5 files changed, 38 insertions(+), 47 deletions(-) rename carl/envs/gymnax/{ => classic_control}/carl_jax_acrobot.py (100%) rename carl/envs/gymnax/{ => classic_control}/carl_jax_cartpole.py (100%) rename carl/envs/gymnax/{ => classic_control}/carl_jax_mountaincar.py (100%) rename carl/envs/gymnax/{ => classic_control}/carl_jax_pendulum.py (100%) diff --git a/carl/envs/gymnax/carl_gymnax_env.py b/carl/envs/gymnax/carl_gymnax_env.py index ec768e43..7bdfc128 100644 --- a/carl/envs/gymnax/carl_gymnax_env.py +++ b/carl/envs/gymnax/carl_gymnax_env.py @@ -1,79 +1,70 @@ from __future__ import annotations -from typing import Any, Dict, List, Optional, Union +from typing import Any import gymnasium +from gymnasium.core import Env -from CARL.carl.envs.gymnax.utils import make_gymnax_env +from carl.envs.gymnax.utils import make_gymnax_env from carl.context.selection import AbstractSelector from carl.envs.carl_env import CARLEnv -from carl.utils.trial_logger import TrialLogger from carl.utils.types import Context, Contexts class CARLGymnaxEnv(CARLEnv): env_name: str - DEFAULT_CONTEXT: Context - max_episode_steps: int def __init__( self, - env: gymnasium.Env | None = None, - contexts: Contexts = {}, - hide_context: bool = True, - add_gaussian_noise_to_context: bool = False, - gaussian_noise_std_percentage: float = 0.01, - logger: Optional[TrialLogger] = None, - scale_context_features: str = "no", - default_context: Optional[Context] = None, - state_context_features: Optional[List[str]] = None, - context_mask: Optional[List[str]] = None, - dict_observation_space: bool = False, - context_selector: Optional[ - Union[AbstractSelector, type[AbstractSelector]] - ] = None, - context_selector_kwargs: Optional[Dict] = None, - ): + env: Env | None = None, + contexts: Contexts | None = None, + obs_context_features: list[str] + | None = None, # list the context features which should be added to the state + obs_context_as_dict: bool = True, + context_selector: AbstractSelector | type[AbstractSelector] | None = None, + context_selector_kwargs: dict = None, + **kwargs, + ) -> None: """ - Max torque is not a context feature because it changes the action space. + CARL Gymnax Environment. Parameters ---------- - env - contexts - instance_mode - hide_context - add_gaussian_noise_to_context - gaussian_noise_std_percentage + + env : Env | None + Gymnasium environment, the default is None. + If None, instantiate the env with gymnasium's make function and + `self.env_name` which is defined in each child class. + contexts : Contexts | None, optional + Context set, by default None. If it is None, we build the + context set with the default context. + obs_context_features : list[str] | None, optional + Context features which should be included in the observation, by default None. + If they are None, add all context features. + context_selector: AbstractSelector | type[AbstractSelector] | None, optional + The context selector (class), after each reset selects a new context to use. + If None, use a round robin selector. + context_selector_kwargs : dict, optional + Optional keyword arguments for the context selector, by default None. + Only used when `context_selector` is not None. + + Attributes + ---------- + env_name: str + The registered gymnax environment name. """ if env is None: env = make_gymnax_env(env_name=self.env_name) - if not contexts: - contexts = {0: self.DEFAULT_CONTEXT} - - if not default_context: - default_context = self.DEFAULT_CONTEXT - super().__init__( env=env, contexts=contexts, - hide_context=hide_context, - add_gaussian_noise_to_context=add_gaussian_noise_to_context, - gaussian_noise_std_percentage=gaussian_noise_std_percentage, - logger=logger, - scale_context_features=scale_context_features, - default_context=default_context, - max_episode_length=self.max_episode_steps, - state_context_features=state_context_features, - dict_observation_space=dict_observation_space, + obs_context_features=obs_context_features, + obs_context_as_dict=obs_context_as_dict, context_selector=context_selector, context_selector_kwargs=context_selector_kwargs, - context_mask=context_mask, + **kwargs, ) - self.whitelist_gaussian_noise = list( - self.DEFAULT_CONTEXT.keys() - ) # allow to augment all values def _update_context(self) -> None: raise NotImplementedError diff --git a/carl/envs/gymnax/carl_jax_acrobot.py b/carl/envs/gymnax/classic_control/carl_jax_acrobot.py similarity index 100% rename from carl/envs/gymnax/carl_jax_acrobot.py rename to carl/envs/gymnax/classic_control/carl_jax_acrobot.py diff --git a/carl/envs/gymnax/carl_jax_cartpole.py b/carl/envs/gymnax/classic_control/carl_jax_cartpole.py similarity index 100% rename from carl/envs/gymnax/carl_jax_cartpole.py rename to carl/envs/gymnax/classic_control/carl_jax_cartpole.py diff --git a/carl/envs/gymnax/carl_jax_mountaincar.py b/carl/envs/gymnax/classic_control/carl_jax_mountaincar.py similarity index 100% rename from carl/envs/gymnax/carl_jax_mountaincar.py rename to carl/envs/gymnax/classic_control/carl_jax_mountaincar.py diff --git a/carl/envs/gymnax/carl_jax_pendulum.py b/carl/envs/gymnax/classic_control/carl_jax_pendulum.py similarity index 100% rename from carl/envs/gymnax/carl_jax_pendulum.py rename to carl/envs/gymnax/classic_control/carl_jax_pendulum.py From c0c9620db472bd11d820ffa1ac8d1ee97400a93d Mon Sep 17 00:00:00 2001 From: benjamc Date: Mon, 11 Dec 2023 11:12:22 +0100 Subject: [PATCH 19/37] Rename envs and imports --- carl/envs/gymnax/__init__.py | 51 +++----- carl/envs/gymnax/classic_control/__init__.py | 13 ++ .../classic_control/carl_jax_acrobot.py | 113 +++++++----------- .../classic_control/carl_jax_cartpole.py | 2 +- .../classic_control/carl_jax_mountaincar.py | 4 +- .../classic_control/carl_jax_pendulum.py | 2 +- 6 files changed, 75 insertions(+), 110 deletions(-) create mode 100644 carl/envs/gymnax/classic_control/__init__.py diff --git a/carl/envs/gymnax/__init__.py b/carl/envs/gymnax/__init__.py index 39bccb14..8a574589 100644 --- a/carl/envs/gymnax/__init__.py +++ b/carl/envs/gymnax/__init__.py @@ -1,36 +1,15 @@ -# flake8: noqa: F401 -from carl.envs.gymnax.carl_jax_acrobot import CONTEXT_BOUNDS as CARLJaxAcrobotEnv_bounds -from carl.envs.gymnax.carl_jax_acrobot import ( - DEFAULT_CONTEXT as CARLJaxAcrobotEnv_defaults, -) -from carl.envs.gymnax.carl_jax_acrobot import CARLJaxAcrobotEnv -from carl.envs.gymnax.carl_jax_cartpole import ( - CONTEXT_BOUNDS as CARLJaxCartPoleEnv_bounds, -) -from carl.envs.gymnax.carl_jax_cartpole import ( - DEFAULT_CONTEXT as CARLJaxCartPoleEnv_defaults, -) -from carl.envs.gymnax.carl_jax_cartpole import CARLJaxCartPoleEnv -from carl.envs.gymnax.carl_jax_mountaincar import ( - CONTEXT_BOUNDS as CARLJaxMountainCarContinuousEnv_bounds, -) -from carl.envs.gymnax.carl_jax_mountaincar import ( - CONTEXT_BOUNDS as CARLJaxMountainCarEnv_bounds, -) -from carl.envs.gymnax.carl_jax_mountaincar import ( - DEFAULT_CONTEXT as CARLJaxMountainCarContinuousEnv_defaults, -) -from carl.envs.gymnax.carl_jax_mountaincar import ( - DEFAULT_CONTEXT as CARLJaxMountainCarEnv_defaults, -) -from carl.envs.gymnax.carl_jax_mountaincar import ( - CARLJaxMountainCarContinuousEnv, - CARLJaxMountainCarEnv, -) -from carl.envs.gymnax.carl_jax_pendulum import ( - CONTEXT_BOUNDS as CARLJaxPendulumEnv_bounds, -) -from carl.envs.gymnax.carl_jax_pendulum import ( - DEFAULT_CONTEXT as CARLJaxPendulumEnv_defaults, -) -from carl.envs.gymnax.carl_jax_pendulum import CARLJaxPendulumEnv +from carl.envs.gymnax.classic_control import ( + CARLGymnaxAcrobot, + CARLGymnaxCartPole, + CARLGymnaxMountainCar, + CARLGymnaxMountainCarContinuous, + CARLGymnaxPendulum, +) + +__all__ = [ + "CARLGymnaxAcrobot", + "CARLGymnaxCartPole", + "CARLGymnaxMountainCar", + "CARLGymnaxMountainCarContinuous", + "CARLGymnaxPendulum", +] diff --git a/carl/envs/gymnax/classic_control/__init__.py b/carl/envs/gymnax/classic_control/__init__.py new file mode 100644 index 00000000..534ecb63 --- /dev/null +++ b/carl/envs/gymnax/classic_control/__init__.py @@ -0,0 +1,13 @@ +from carl.envs.gymnax.classic_control.carl_jax_acrobot import CARLGymnaxAcrobot +from carl.envs.gymnax.classic_control.carl_jax_cartpole import CARLGymnaxCartPole +from carl.envs.gymnax.classic_control.carl_jax_mountaincar import CARLGymnaxMountainCar +from carl.envs.gymnax.classic_control.carl_jax_mountaincar import CARLGymnaxMountainCarContinuous +from carl.envs.gymnax.classic_control.carl_jax_pendulum import CARLGymnaxPendulum + +__all__ = [ + "CARLGymnaxAcrobot", + "CARLGymnaxCartPole", + "CARLGymnaxMountainCar", + "CARLGymnaxMountainCarContinuous", + "CARLGymnaxPendulum", +] diff --git a/carl/envs/gymnax/classic_control/carl_jax_acrobot.py b/carl/envs/gymnax/classic_control/carl_jax_acrobot.py index d07c1e8b..173881ce 100644 --- a/carl/envs/gymnax/classic_control/carl_jax_acrobot.py +++ b/carl/envs/gymnax/classic_control/carl_jax_acrobot.py @@ -5,65 +5,52 @@ import numpy as np from carl.envs.gymnax.carl_gymnax_env import CARLGymnaxEnv -from carl.utils.types import Context +from carl.context.context_space import ContextFeature, UniformFloatContextFeature -DEFAULT_CONTEXT = { - "link_length_1": 1, - "link_length_2": 1, - "link_mass_1": 1, - "link_mass_2": 1, - "link_com_pos_1": 0.5, - "link_com_pos_2": 0.5, - "link_moi": 1, - "max_vel_1": 4 * jnp.pi, - "max_vel_2": 9 * jnp.pi, - "torque_noise_max": 0.0, - "max_steps_in_episode": 500, -} -CONTEXT_BOUNDS = { - "link_length_1": ( - 0.1, - 10, - float, - ), # Links can be shrunken and grown by a factor of 10 - "link_length_2": (0.1, 10, float), - "link_mass_1": ( - 0.1, - 10, - float, - ), # Link mass can be shrunken and grown by a factor of 10 - "link_mass_2": (0.1, 10, float), - "link_com_pos_1": ( - 0, - 1, - float, - ), # Center of mass can move from one end to the other - "link_com_pos_2": (0, 1, float), - "link_moi": ( - 0.1, - 10, - float, - ), # Moments on inertia can be shrunken and grown by a factor of 10 - "max_vel_1": ( - 0.4 * np.pi, - 40 * np.pi, - float, - ), # Velocity can vary by a factor of 10 in either direction - "max_vel_2": (0.9 * np.pi, 90 * np.pi, float), - "torque_noise_max": ( - -1.0, - 1.0, - float, - ), # torque is either {-1., 0., 1}. Applying noise of 1. would be quite extreme - "max_steps_in_episode": (1, jnp.inf, int), -} - - -class CARLJaxAcrobotEnv(CARLGymnaxEnv): +class CARLGymnaxAcrobot(CARLGymnaxEnv): env_name: str = "Acrobot-v1" - max_episode_steps: int = int(DEFAULT_CONTEXT["max_steps_in_episode"]) - DEFAULT_CONTEXT: Context = DEFAULT_CONTEXT + + @staticmethod + def get_context_features() -> dict[str, ContextFeature]: + return { + "link_length_1": UniformFloatContextFeature( + "link_length_1", lower=0.1, upper=10, default_value=1 + ), # Links can be shrunken and grown by a factor of 10 + "link_length_2": UniformFloatContextFeature( + "link_length_2", lower=0.1, upper=10, default_value=1 + ), # Links can be shrunken and grown by a factor of 10 + "link_mass_1": UniformFloatContextFeature( + "link_mass_1", lower=0.1, upper=10, default_value=1 + ), # Link mass can be shrunken and grown by a factor of 10 + "link_mass_2": UniformFloatContextFeature( + "link_mass_2", lower=0.1, upper=10, default_value=1 + ), # Link mass can be shrunken and grown by a factor of 10 + "link_com_pos_1": UniformFloatContextFeature( + "link_com_pos_1", lower=0, upper=1, default_value=0.5 + ), # Center of mass can move from one end to the other + "link_com_pos_2": UniformFloatContextFeature( + "link_com_pos_2", lower=0, upper=1, default_value=0.5 + ), # Center of mass can move from one end to the other + "link_moi": UniformFloatContextFeature( + "link_moi", lower=0.1, upper=10, default_value=1 + ), # Moments on inertia can be shrunken and grown by a factor of 10 + "max_vel_1": UniformFloatContextFeature( + "max_vel_1", + lower= 0.4 * jnp.pi, + upper= 40 * jnp.pi, + default_value= 4 * jnp.pi, + ), # Velocity can vary by a factor of 10 in either direction + "max_vel_2": UniformFloatContextFeature( + "max_vel_2", + lower=0.9 * np.pi, + upper=90 * np.pi, + default_value=9 * np.pi, + ), # Velocity can vary by a factor of 10 in either direction + "torque_noise_max": UniformFloatContextFeature( + "torque_noise_max", lower=-1, upper=1, default_value=0 + ), # torque is either {-1., 0., 1}. Applying noise of 1. would be quite extreme + } def _update_context(self) -> None: content = self.env.env.env_params.__dict__ @@ -72,17 +59,3 @@ def _update_context(self) -> None: self.env.env.env_params = gymnax.environments.classic_control.acrobot.EnvParams( **content ) - - high = jnp.array( - [ - 1.0, - 1.0, - 1.0, - 1.0, - self.env.env.env_params.max_vel_1, - self.env.env.env_params.max_vel_2, - ], - dtype=jnp.float32, - ) - low = -high - self.build_observation_space(low, high, CONTEXT_BOUNDS) diff --git a/carl/envs/gymnax/classic_control/carl_jax_cartpole.py b/carl/envs/gymnax/classic_control/carl_jax_cartpole.py index 3110f850..10acbaa9 100644 --- a/carl/envs/gymnax/classic_control/carl_jax_cartpole.py +++ b/carl/envs/gymnax/classic_control/carl_jax_cartpole.py @@ -31,7 +31,7 @@ } -class CARLJaxCartPoleEnv(CARLGymnaxEnv): +class CARLGymnaxCartPole(CARLGymnaxEnv): env_name: str = "CartPole-v1" max_episode_steps: int = int(DEFAULT_CONTEXT["max_steps_in_episode"]) # type: ignore[arg-type] DEFAULT_CONTEXT: Context = DEFAULT_CONTEXT diff --git a/carl/envs/gymnax/classic_control/carl_jax_mountaincar.py b/carl/envs/gymnax/classic_control/carl_jax_mountaincar.py index 75cc344b..e2987580 100644 --- a/carl/envs/gymnax/classic_control/carl_jax_mountaincar.py +++ b/carl/envs/gymnax/classic_control/carl_jax_mountaincar.py @@ -29,7 +29,7 @@ } -class CARLJaxMountainCarEnv(CARLGymnaxEnv): +class CARLGymnaxMountainCar(CARLGymnaxEnv): env_name: str = "MountainCar-v0" max_episode_steps: int = int(DEFAULT_CONTEXT["max_steps_in_episode"]) DEFAULT_CONTEXT: Context = DEFAULT_CONTEXT @@ -51,7 +51,7 @@ def _update_context(self) -> None: self.build_observation_space(self.low, self.high, CONTEXT_BOUNDS) -class CARLJaxMountainCarContinuousEnv(CARLJaxMountainCarEnv): +class CARLGymnaxMountainCarContinuous(CARLGymnaxMountainCar): env_name: str = "MountainCarContinuous-v0" max_episode_steps: int = 999 DEFAULT_CONTEXT: Context = DEFAULT_CONTEXT diff --git a/carl/envs/gymnax/classic_control/carl_jax_pendulum.py b/carl/envs/gymnax/classic_control/carl_jax_pendulum.py index 85faa27b..983bd4ac 100644 --- a/carl/envs/gymnax/classic_control/carl_jax_pendulum.py +++ b/carl/envs/gymnax/classic_control/carl_jax_pendulum.py @@ -27,7 +27,7 @@ } -class CARLJaxPendulumEnv(CARLGymnaxEnv): +class CARLGymnaxPendulum(CARLGymnaxEnv): env_name: str = "Pendulum-v1" max_episode_steps: int = int(DEFAULT_CONTEXT["max_steps_in_episode"]) DEFAULT_CONTEXT: Context = DEFAULT_CONTEXT From 24066169d0db0bc4b8fbc31faff5bb28827dc2ee Mon Sep 17 00:00:00 2001 From: benjamc Date: Mon, 11 Dec 2023 11:18:45 +0100 Subject: [PATCH 20/37] Rename files --- carl/envs/gymnax/classic_control/__init__.py | 10 +++++----- .../{carl_jax_acrobot.py => carl_gymnax_acrobot.py} | 0 .../{carl_jax_cartpole.py => carl_gymnax_cartpole.py} | 0 ...l_jax_mountaincar.py => carl_gymnax_mountaincar.py} | 0 .../{carl_jax_pendulum.py => carl_gymnax_pendulum.py} | 0 5 files changed, 5 insertions(+), 5 deletions(-) rename carl/envs/gymnax/classic_control/{carl_jax_acrobot.py => carl_gymnax_acrobot.py} (100%) rename carl/envs/gymnax/classic_control/{carl_jax_cartpole.py => carl_gymnax_cartpole.py} (100%) rename carl/envs/gymnax/classic_control/{carl_jax_mountaincar.py => carl_gymnax_mountaincar.py} (100%) rename carl/envs/gymnax/classic_control/{carl_jax_pendulum.py => carl_gymnax_pendulum.py} (100%) diff --git a/carl/envs/gymnax/classic_control/__init__.py b/carl/envs/gymnax/classic_control/__init__.py index 534ecb63..0a2bb1a1 100644 --- a/carl/envs/gymnax/classic_control/__init__.py +++ b/carl/envs/gymnax/classic_control/__init__.py @@ -1,8 +1,8 @@ -from carl.envs.gymnax.classic_control.carl_jax_acrobot import CARLGymnaxAcrobot -from carl.envs.gymnax.classic_control.carl_jax_cartpole import CARLGymnaxCartPole -from carl.envs.gymnax.classic_control.carl_jax_mountaincar import CARLGymnaxMountainCar -from carl.envs.gymnax.classic_control.carl_jax_mountaincar import CARLGymnaxMountainCarContinuous -from carl.envs.gymnax.classic_control.carl_jax_pendulum import CARLGymnaxPendulum +from carl.envs.gymnax.classic_control.carl_gymnax_acrobot import CARLGymnaxAcrobot +from carl.envs.gymnax.classic_control.carl_gymnax_cartpole import CARLGymnaxCartPole +from carl.envs.gymnax.classic_control.carl_gymnax_mountaincar import CARLGymnaxMountainCar +from carl.envs.gymnax.classic_control.carl_gymnax_mountaincar import CARLGymnaxMountainCarContinuous +from carl.envs.gymnax.classic_control.carl_gymnax_pendulum import CARLGymnaxPendulum __all__ = [ "CARLGymnaxAcrobot", diff --git a/carl/envs/gymnax/classic_control/carl_jax_acrobot.py b/carl/envs/gymnax/classic_control/carl_gymnax_acrobot.py similarity index 100% rename from carl/envs/gymnax/classic_control/carl_jax_acrobot.py rename to carl/envs/gymnax/classic_control/carl_gymnax_acrobot.py diff --git a/carl/envs/gymnax/classic_control/carl_jax_cartpole.py b/carl/envs/gymnax/classic_control/carl_gymnax_cartpole.py similarity index 100% rename from carl/envs/gymnax/classic_control/carl_jax_cartpole.py rename to carl/envs/gymnax/classic_control/carl_gymnax_cartpole.py diff --git a/carl/envs/gymnax/classic_control/carl_jax_mountaincar.py b/carl/envs/gymnax/classic_control/carl_gymnax_mountaincar.py similarity index 100% rename from carl/envs/gymnax/classic_control/carl_jax_mountaincar.py rename to carl/envs/gymnax/classic_control/carl_gymnax_mountaincar.py diff --git a/carl/envs/gymnax/classic_control/carl_jax_pendulum.py b/carl/envs/gymnax/classic_control/carl_gymnax_pendulum.py similarity index 100% rename from carl/envs/gymnax/classic_control/carl_jax_pendulum.py rename to carl/envs/gymnax/classic_control/carl_gymnax_pendulum.py From 63cba379c181333f6eccc8871cde5fa96b612a65 Mon Sep 17 00:00:00 2001 From: benjamc Date: Mon, 11 Dec 2023 12:00:52 +0100 Subject: [PATCH 21/37] Fix env comapts --- carl/envs/carl_env.py | 1 - .../classic_control/carl_gymnax_acrobot.py | 2 +- carl/envs/gymnax/utils.py | 40 +++++++++++++++---- 3 files changed, 33 insertions(+), 10 deletions(-) diff --git a/carl/envs/carl_env.py b/carl/envs/carl_env.py index 00105c88..af20a924 100644 --- a/carl/envs/carl_env.py +++ b/carl/envs/carl_env.py @@ -178,7 +178,6 @@ def get_observation_space( context_feature_names=obs_context_feature_names, as_dict=self.obs_context_as_dict, ) - obs_space = spaces.Dict( { "obs": self.base_observation_space, diff --git a/carl/envs/gymnax/classic_control/carl_gymnax_acrobot.py b/carl/envs/gymnax/classic_control/carl_gymnax_acrobot.py index 173881ce..35a3ecf9 100644 --- a/carl/envs/gymnax/classic_control/carl_gymnax_acrobot.py +++ b/carl/envs/gymnax/classic_control/carl_gymnax_acrobot.py @@ -53,7 +53,7 @@ def get_context_features() -> dict[str, ContextFeature]: } def _update_context(self) -> None: - content = self.env.env.env_params.__dict__ + content = self.env.env_params.__dict__ content.update(self.context) # We cannot directly set attributes of env_params because it is a frozen dataclass self.env.env.env_params = gymnax.environments.classic_control.acrobot.EnvParams( diff --git a/carl/envs/gymnax/utils.py b/carl/envs/gymnax/utils.py index ca400ee5..d30620fe 100644 --- a/carl/envs/gymnax/utils.py +++ b/carl/envs/gymnax/utils.py @@ -1,30 +1,54 @@ from __future__ import annotations +from typing import Any, Optional, Sequence import gymnasium +from gymnasium.wrappers.compatibility import LegacyEnv +import gymnasium.spaces import gymnax from gymnasium.wrappers import EnvCompatibility +from gymnax.environments.environment import EnvParams, Environment from gymnax.environments.spaces import Space, gymnax_space_to_gym_space from gymnax.wrappers.gym import GymnaxToGymWrapper +from numpy._typing import DTypeLike +from numpy.random._generator import Generator as Generator -class CustomGymnaxToGymWrapper(GymnaxToGymWrapper): + +# Although this converts to gym, the step API already is for gymnasium +class CustomGymnaxToGymnasiumWrapper(GymnaxToGymWrapper): + def __init__(self, env: Environment, params: EnvParams | None = None, seed: int | None = None): + super().__init__(env, params, seed) + + self._observation_space = SpaceWrapper(gymnax_space_to_gym_space(self._env.observation_space(self.env_params))) + @property - def observation_space(self) -> dict: - return gymnax_space_to_gym_space(self._env.observation_space(self.env_params)) + def env(self) -> Environment: + return self._env + + @env.setter + def env(self, value: Environment) -> None: + self._env = value + + @property + def observation_space(self) -> gymnasium.Space: + return self._observation_space @observation_space.setter def observation_space(self, value: Space) -> None: self._observation_space = value +class SpaceWrapper(gymnasium.Space): + def __init__(self, space): + self.space = space + + def __getattr__(self, __name: str) -> Any: + return self.space.__getattr__(__name=__name) def make_gymnax_env(env_name: str) -> gymnasium.Env: # Make gymnax env env, env_params = gymnax.make(env_id=env_name) - # Convert gymnax to gym API - env = CustomGymnaxToGymWrapper(env=env, params=env_params) - - # Convert gym to gymnasium API - env = EnvCompatibility(old_env=env) + # Convert gymnax to gymnasium API + env = CustomGymnaxToGymnasiumWrapper(env=env, params=env_params) return env From 689a2c893838c068cad99ab35cc0aec3b20962c7 Mon Sep 17 00:00:00 2001 From: benjamc Date: Mon, 11 Dec 2023 12:01:25 +0100 Subject: [PATCH 22/37] Reformat --- carl/envs/gymnax/carl_gymnax_env.py | 2 +- carl/envs/gymnax/classic_control/__init__.py | 6 +++-- .../classic_control/carl_gymnax_acrobot.py | 26 +++++++++---------- carl/envs/gymnax/utils.py | 18 ++++++++----- 4 files changed, 30 insertions(+), 22 deletions(-) diff --git a/carl/envs/gymnax/carl_gymnax_env.py b/carl/envs/gymnax/carl_gymnax_env.py index 7bdfc128..c6633671 100644 --- a/carl/envs/gymnax/carl_gymnax_env.py +++ b/carl/envs/gymnax/carl_gymnax_env.py @@ -5,9 +5,9 @@ import gymnasium from gymnasium.core import Env -from carl.envs.gymnax.utils import make_gymnax_env from carl.context.selection import AbstractSelector from carl.envs.carl_env import CARLEnv +from carl.envs.gymnax.utils import make_gymnax_env from carl.utils.types import Context, Contexts diff --git a/carl/envs/gymnax/classic_control/__init__.py b/carl/envs/gymnax/classic_control/__init__.py index 0a2bb1a1..5450542d 100644 --- a/carl/envs/gymnax/classic_control/__init__.py +++ b/carl/envs/gymnax/classic_control/__init__.py @@ -1,7 +1,9 @@ from carl.envs.gymnax.classic_control.carl_gymnax_acrobot import CARLGymnaxAcrobot from carl.envs.gymnax.classic_control.carl_gymnax_cartpole import CARLGymnaxCartPole -from carl.envs.gymnax.classic_control.carl_gymnax_mountaincar import CARLGymnaxMountainCar -from carl.envs.gymnax.classic_control.carl_gymnax_mountaincar import CARLGymnaxMountainCarContinuous +from carl.envs.gymnax.classic_control.carl_gymnax_mountaincar import ( + CARLGymnaxMountainCar, + CARLGymnaxMountainCarContinuous, +) from carl.envs.gymnax.classic_control.carl_gymnax_pendulum import CARLGymnaxPendulum __all__ = [ diff --git a/carl/envs/gymnax/classic_control/carl_gymnax_acrobot.py b/carl/envs/gymnax/classic_control/carl_gymnax_acrobot.py index 35a3ecf9..04f3257b 100644 --- a/carl/envs/gymnax/classic_control/carl_gymnax_acrobot.py +++ b/carl/envs/gymnax/classic_control/carl_gymnax_acrobot.py @@ -4,8 +4,8 @@ import jax.numpy as jnp import numpy as np -from carl.envs.gymnax.carl_gymnax_env import CARLGymnaxEnv from carl.context.context_space import ContextFeature, UniformFloatContextFeature +from carl.envs.gymnax.carl_gymnax_env import CARLGymnaxEnv class CARLGymnaxAcrobot(CARLGymnaxEnv): @@ -16,37 +16,37 @@ def get_context_features() -> dict[str, ContextFeature]: return { "link_length_1": UniformFloatContextFeature( "link_length_1", lower=0.1, upper=10, default_value=1 - ), # Links can be shrunken and grown by a factor of 10 + ), # Links can be shrunken and grown by a factor of 10 "link_length_2": UniformFloatContextFeature( "link_length_2", lower=0.1, upper=10, default_value=1 - ), # Links can be shrunken and grown by a factor of 10 + ), # Links can be shrunken and grown by a factor of 10 "link_mass_1": UniformFloatContextFeature( "link_mass_1", lower=0.1, upper=10, default_value=1 - ), # Link mass can be shrunken and grown by a factor of 10 + ), # Link mass can be shrunken and grown by a factor of 10 "link_mass_2": UniformFloatContextFeature( "link_mass_2", lower=0.1, upper=10, default_value=1 - ), # Link mass can be shrunken and grown by a factor of 10 + ), # Link mass can be shrunken and grown by a factor of 10 "link_com_pos_1": UniformFloatContextFeature( "link_com_pos_1", lower=0, upper=1, default_value=0.5 - ), # Center of mass can move from one end to the other + ), # Center of mass can move from one end to the other "link_com_pos_2": UniformFloatContextFeature( "link_com_pos_2", lower=0, upper=1, default_value=0.5 - ), # Center of mass can move from one end to the other + ), # Center of mass can move from one end to the other "link_moi": UniformFloatContextFeature( "link_moi", lower=0.1, upper=10, default_value=1 - ), # Moments on inertia can be shrunken and grown by a factor of 10 + ), # Moments on inertia can be shrunken and grown by a factor of 10 "max_vel_1": UniformFloatContextFeature( "max_vel_1", - lower= 0.4 * jnp.pi, - upper= 40 * jnp.pi, - default_value= 4 * jnp.pi, - ), # Velocity can vary by a factor of 10 in either direction + lower=0.4 * jnp.pi, + upper=40 * jnp.pi, + default_value=4 * jnp.pi, + ), # Velocity can vary by a factor of 10 in either direction "max_vel_2": UniformFloatContextFeature( "max_vel_2", lower=0.9 * np.pi, upper=90 * np.pi, default_value=9 * np.pi, - ), # Velocity can vary by a factor of 10 in either direction + ), # Velocity can vary by a factor of 10 in either direction "torque_noise_max": UniformFloatContextFeature( "torque_noise_max", lower=-1, upper=1, default_value=0 ), # torque is either {-1., 0., 1}. Applying noise of 1. would be quite extreme diff --git a/carl/envs/gymnax/utils.py b/carl/envs/gymnax/utils.py index d30620fe..dbf70a95 100644 --- a/carl/envs/gymnax/utils.py +++ b/carl/envs/gymnax/utils.py @@ -1,30 +1,34 @@ from __future__ import annotations + from typing import Any, Optional, Sequence import gymnasium -from gymnasium.wrappers.compatibility import LegacyEnv import gymnasium.spaces import gymnax from gymnasium.wrappers import EnvCompatibility -from gymnax.environments.environment import EnvParams, Environment +from gymnasium.wrappers.compatibility import LegacyEnv +from gymnax.environments.environment import Environment, EnvParams from gymnax.environments.spaces import Space, gymnax_space_to_gym_space from gymnax.wrappers.gym import GymnaxToGymWrapper from numpy._typing import DTypeLike from numpy.random._generator import Generator as Generator - # Although this converts to gym, the step API already is for gymnasium class CustomGymnaxToGymnasiumWrapper(GymnaxToGymWrapper): - def __init__(self, env: Environment, params: EnvParams | None = None, seed: int | None = None): + def __init__( + self, env: Environment, params: EnvParams | None = None, seed: int | None = None + ): super().__init__(env, params, seed) - self._observation_space = SpaceWrapper(gymnax_space_to_gym_space(self._env.observation_space(self.env_params))) + self._observation_space = SpaceWrapper( + gymnax_space_to_gym_space(self._env.observation_space(self.env_params)) + ) @property def env(self) -> Environment: return self._env - + @env.setter def env(self, value: Environment) -> None: self._env = value @@ -37,6 +41,7 @@ def observation_space(self) -> gymnasium.Space: def observation_space(self, value: Space) -> None: self._observation_space = value + class SpaceWrapper(gymnasium.Space): def __init__(self, space): self.space = space @@ -44,6 +49,7 @@ def __init__(self, space): def __getattr__(self, __name: str) -> Any: return self.space.__getattr__(__name=__name) + def make_gymnax_env(env_name: str) -> gymnasium.Env: # Make gymnax env env, env_params = gymnax.make(env_id=env_name) From 7ec0dfc3782d6c68cfb912ed393cca6b0b037b0f Mon Sep 17 00:00:00 2001 From: benjamc Date: Mon, 11 Dec 2023 12:18:41 +0100 Subject: [PATCH 23/37] Smarter update context --- carl/envs/gymnax/carl_gymnax_env.py | 16 ++++++++++--- .../classic_control/carl_gymnax_acrobot.py | 9 +------ .../classic_control/carl_gymnax_cartpole.py | 24 ++++--------------- 3 files changed, 18 insertions(+), 31 deletions(-) diff --git a/carl/envs/gymnax/carl_gymnax_env.py b/carl/envs/gymnax/carl_gymnax_env.py index c6633671..d8947290 100644 --- a/carl/envs/gymnax/carl_gymnax_env.py +++ b/carl/envs/gymnax/carl_gymnax_env.py @@ -2,7 +2,10 @@ from typing import Any +import importlib + import gymnasium +import gymnax.environments from gymnasium.core import Env from carl.context.selection import AbstractSelector @@ -66,11 +69,18 @@ def __init__( **kwargs, ) - def _update_context(self) -> None: - raise NotImplementedError - def __getattr__(self, name: str) -> Any: if name in ["sys", "__getstate__"]: return getattr(self.env._environment, name) else: return getattr(self, name) + + def _update_context(self) -> None: + content = self.env.env_params.__dict__ + content.update(self.context) + # We cannot directly set attributes of env_params because it is a frozen dataclass + + # TODO Make this faster by preloading module? + self.env.env.env_params = getattr( + importlib.import_module(f"gymnax.environments.{self.module}"), "EnvParams" + )(**content) diff --git a/carl/envs/gymnax/classic_control/carl_gymnax_acrobot.py b/carl/envs/gymnax/classic_control/carl_gymnax_acrobot.py index 04f3257b..cf16034e 100644 --- a/carl/envs/gymnax/classic_control/carl_gymnax_acrobot.py +++ b/carl/envs/gymnax/classic_control/carl_gymnax_acrobot.py @@ -10,6 +10,7 @@ class CARLGymnaxAcrobot(CARLGymnaxEnv): env_name: str = "Acrobot-v1" + module: str = "classic_control.acrobot" @staticmethod def get_context_features() -> dict[str, ContextFeature]: @@ -51,11 +52,3 @@ def get_context_features() -> dict[str, ContextFeature]: "torque_noise_max", lower=-1, upper=1, default_value=0 ), # torque is either {-1., 0., 1}. Applying noise of 1. would be quite extreme } - - def _update_context(self) -> None: - content = self.env.env_params.__dict__ - content.update(self.context) - # We cannot directly set attributes of env_params because it is a frozen dataclass - self.env.env.env_params = gymnax.environments.classic_control.acrobot.EnvParams( - **content - ) diff --git a/carl/envs/gymnax/classic_control/carl_gymnax_cartpole.py b/carl/envs/gymnax/classic_control/carl_gymnax_cartpole.py index 10acbaa9..d1537b06 100644 --- a/carl/envs/gymnax/classic_control/carl_gymnax_cartpole.py +++ b/carl/envs/gymnax/classic_control/carl_gymnax_cartpole.py @@ -33,27 +33,11 @@ class CARLGymnaxCartPole(CARLGymnaxEnv): env_name: str = "CartPole-v1" - max_episode_steps: int = int(DEFAULT_CONTEXT["max_steps_in_episode"]) # type: ignore[arg-type] - DEFAULT_CONTEXT: Context = DEFAULT_CONTEXT def _update_context(self) -> None: - self.context["polemass_length"] = ( - self.context["masspole"] * self.context["length"] - ) - self.context["total_mass"] = self.context["masscart"] + self.context["masspole"] - + content = self.env.env_params.__dict__ + content.update(self.context) + # We cannot directly set attributes of env_params because it is a frozen dataclass self.env.env.env_params = ( - gymnax.environments.classic_control.cartpole.EnvParams(**self.context) - ) - - high = jnp.array( - [ - self.env.env.env_params.x_threshold * 2, - jnp.finfo(jnp.float32).max, - self.env.env.env_params.theta_threshold_radians * 2, - jnp.finfo(jnp.float32).max, - ], - dtype=jnp.float32, + gymnax.environments.classic_control.cartpole.EnvParams(**content) ) - low = -high - self.build_observation_space(low, high, CONTEXT_BOUNDS) From adde74099de176b5faab1254982fdb802433921c Mon Sep 17 00:00:00 2001 From: benjamc Date: Mon, 11 Dec 2023 12:24:42 +0100 Subject: [PATCH 24/37] Update cartpole --- .../classic_control/carl_gymnax_cartpole.py | 64 ++++++++++--------- 1 file changed, 34 insertions(+), 30 deletions(-) diff --git a/carl/envs/gymnax/classic_control/carl_gymnax_cartpole.py b/carl/envs/gymnax/classic_control/carl_gymnax_cartpole.py index d1537b06..fb61066d 100644 --- a/carl/envs/gymnax/classic_control/carl_gymnax_cartpole.py +++ b/carl/envs/gymnax/classic_control/carl_gymnax_cartpole.py @@ -1,43 +1,47 @@ from __future__ import annotations import gymnax -import jax.numpy as jnp +import importlib +import jax.numpy as np from carl.envs.gymnax.carl_gymnax_env import CARLGymnaxEnv -from carl.utils.types import Context - -DEFAULT_CONTEXT = { - "gravity": 9.8, - "masscart": 1.0, - "masspole": 0.1, - "length": 0.5, - "force_mag": 10.0, - "tau": 0.02, - "polemass_length": None, - "total_mass": None, - "max_steps_in_episode": 500, -} - -CONTEXT_BOUNDS = { - "gravity": (5.0, 15.0, float), - "masscart": (0.5, 2.0, float), - "masspole": (0.05, 0.2, float), - "length": (0.25, 1.0, float), - "force_mag": (5.0, 15.0, float), - "tau": (0.01, 0.05, float), - "polemass_length": (0, jnp.inf, float), - "total_mass": (0, jnp.inf, float), - "max_steps_in_episode": (1, jnp.inf, int), -} +from carl.context.context_space import ContextFeature, UniformFloatContextFeature class CARLGymnaxCartPole(CARLGymnaxEnv): env_name: str = "CartPole-v1" + module: str = "classic_control.cartpole" + + @staticmethod + def get_context_features() -> dict[str, ContextFeature]: + return { + "gravity": UniformFloatContextFeature( + "gravity", lower=0.1, upper=np.inf, default_value=9.8 + ), + "masscart": UniformFloatContextFeature( + "masscart", lower=0.1, upper=10, default_value=1.0 + ), + "masspole": UniformFloatContextFeature( + "masspole", lower=0.01, upper=1, default_value=0.1 + ), + "length": UniformFloatContextFeature( + "length", lower=0.05, upper=5, default_value=0.5 + ), + "force_mag": UniformFloatContextFeature( + "force_mag", lower=1, upper=100, default_value=10.0 + ), + "tau": UniformFloatContextFeature( + "tau", lower=0.002, upper=0.2, default_value=0.02 + ), + } def _update_context(self) -> None: content = self.env.env_params.__dict__ content.update(self.context) - # We cannot directly set attributes of env_params because it is a frozen dataclass - self.env.env.env_params = ( - gymnax.environments.classic_control.cartpole.EnvParams(**content) - ) + content["total_mass"] = content["masspole"] + content["masscart"] + content["polemass_length"] = content["masspole"] * content["length"] + + # TODO Make this faster by preloading module? + self.env.env.env_params = getattr( + importlib.import_module(f"gymnax.environments.{self.module}"), "EnvParams" + )(**content) From 52bd50103dc46fd8f09fee988e040a7228de07d1 Mon Sep 17 00:00:00 2001 From: benjamc Date: Mon, 11 Dec 2023 13:14:57 +0100 Subject: [PATCH 25/37] Update mountaincar --- .../carl_gymnax_mountaincar.py | 88 +++++++++---------- 1 file changed, 43 insertions(+), 45 deletions(-) diff --git a/carl/envs/gymnax/classic_control/carl_gymnax_mountaincar.py b/carl/envs/gymnax/classic_control/carl_gymnax_mountaincar.py index e2987580..2a99a236 100644 --- a/carl/envs/gymnax/classic_control/carl_gymnax_mountaincar.py +++ b/carl/envs/gymnax/classic_control/carl_gymnax_mountaincar.py @@ -1,57 +1,55 @@ from __future__ import annotations import gymnax -import jax.numpy as jnp +import jax.numpy as np from carl.envs.gymnax.carl_gymnax_env import CARLGymnaxEnv -from carl.utils.types import Context - -DEFAULT_CONTEXT = { - "min_position": -1.2, - "max_position": 0.6, - "max_speed": 0.07, - "goal_position": 0.5, - "goal_velocity": 0, - "force": 0.001, - "gravity": 0.0025, - "max_steps_in_episode": 200, -} - -CONTEXT_BOUNDS = { - "min_position": (-jnp.inf, jnp.inf, float), - "max_position": (-jnp.inf, jnp.inf, float), - "max_speed": (0, jnp.inf, float), - "goal_position": (-jnp.inf, jnp.inf, float), - "goal_velocity": (-jnp.inf, jnp.inf, float), - "force": (-jnp.inf, jnp.inf, float), - "gravity": (0, jnp.inf, float), - "max_steps_in_episode": (1, jnp.inf, int), -} +from carl.context.context_space import ContextFeature, UniformFloatContextFeature class CARLGymnaxMountainCar(CARLGymnaxEnv): env_name: str = "MountainCar-v0" - max_episode_steps: int = int(DEFAULT_CONTEXT["max_steps_in_episode"]) - DEFAULT_CONTEXT: Context = DEFAULT_CONTEXT - - def _update_context(self) -> None: - self.env.env.env_params = ( - gymnax.environments.classic_control.mountain_car.EnvParams(**self.context) - ) - - self.low = jnp.array( - [self.env.env.env_params.min_position, -self.env.env.env_params.max_speed], - dtype=jnp.float32, - ).squeeze() - self.high = jnp.array( - [self.env.env.env_params.max_position, self.env.env.env_params.max_speed], - dtype=jnp.float32, - ).squeeze() - - self.build_observation_space(self.low, self.high, CONTEXT_BOUNDS) - + module: str = "classic_control.mountain_car" + + @staticmethod + def get_context_features() -> dict[str, ContextFeature]: + return { + "min_position": UniformFloatContextFeature( + "min_position", lower=-np.inf, upper=np.inf, default_value=-1.2 + ), + "max_position": UniformFloatContextFeature( + "max_position", lower=-np.inf, upper=np.inf, default_value=0.6 + ), + "max_speed": UniformFloatContextFeature( + "max_speed", lower=0, upper=np.inf, default_value=0.07 + ), + "goal_position": UniformFloatContextFeature( + "goal_position", lower=-np.inf, upper=np.inf, default_value=0.45 + ), + "goal_velocity": UniformFloatContextFeature( + "goal_velocity", lower=-np.inf, upper=np.inf, default_value=0 + ), + "force": UniformFloatContextFeature( + "force", lower=-np.inf, upper=np.inf, default_value=0.001 + ), + "gravity": UniformFloatContextFeature( + "gravity", lower=0, upper=np.inf, default_value=0.0025 + ), + "min_position_start": UniformFloatContextFeature( + "min_position_start", lower=-np.inf, upper=np.inf, default_value=-0.6 + ), + "max_position_start": UniformFloatContextFeature( + "max_position_start", lower=-np.inf, upper=np.inf, default_value=-0.4 + ), + "min_velocity_start": UniformFloatContextFeature( + "min_velocity_start", lower=-np.inf, upper=np.inf, default_value=0 + ), + "max_velocity_start": UniformFloatContextFeature( + "max_velocity_start", lower=-np.inf, upper=np.inf, default_value=0 + ), + } + class CARLGymnaxMountainCarContinuous(CARLGymnaxMountainCar): env_name: str = "MountainCarContinuous-v0" - max_episode_steps: int = 999 - DEFAULT_CONTEXT: Context = DEFAULT_CONTEXT + module: str = "classic_control.continuous_mountain_car" From 9874e52542c3fe8e882ff6e10f7020f1af6d1c4a Mon Sep 17 00:00:00 2001 From: benjamc Date: Mon, 11 Dec 2023 13:22:42 +0100 Subject: [PATCH 26/37] Fix context space --- carl/envs/gymnasium/classic_control/carl_pendulum.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/carl/envs/gymnasium/classic_control/carl_pendulum.py b/carl/envs/gymnasium/classic_control/carl_pendulum.py index 4148886a..812f3c90 100644 --- a/carl/envs/gymnasium/classic_control/carl_pendulum.py +++ b/carl/envs/gymnasium/classic_control/carl_pendulum.py @@ -14,9 +14,6 @@ class CARLPendulum(CARLGymnasiumEnv): @staticmethod def get_context_features() -> dict[str, ContextFeature]: return { - "gravity": UniformFloatContextFeature( - "gravity", lower=-np.inf, upper=np.inf, default_value=8.0 - ), "dt": UniformFloatContextFeature( "dt", lower=0, upper=np.inf, default_value=0.05 ), From dd0c4302a45a84a109aa46e502e5ce5f22ac47d2 Mon Sep 17 00:00:00 2001 From: benjamc Date: Mon, 11 Dec 2023 13:23:18 +0100 Subject: [PATCH 27/37] Update pendulum --- .../classic_control/carl_gymnax_pendulum.py | 59 ++++++++----------- 1 file changed, 25 insertions(+), 34 deletions(-) diff --git a/carl/envs/gymnax/classic_control/carl_gymnax_pendulum.py b/carl/envs/gymnax/classic_control/carl_gymnax_pendulum.py index 983bd4ac..745390bf 100644 --- a/carl/envs/gymnax/classic_control/carl_gymnax_pendulum.py +++ b/carl/envs/gymnax/classic_control/carl_gymnax_pendulum.py @@ -1,41 +1,32 @@ from __future__ import annotations -import jax.numpy as jnp -from gymnax.environments.classic_control.pendulum import EnvParams - from carl.envs.gymnax.carl_gymnax_env import CARLGymnaxEnv -from carl.utils.types import Context - -DEFAULT_CONTEXT = { - "max_speed": 8.0, - "max_torque": 2.0, - "dt": 0.05, - "g": 10.0, - "m": 1.0, - "l": 1.0, - "max_steps_in_episode": 200, -} - -CONTEXT_BOUNDS = { - "max_speed": (-jnp.inf, jnp.inf, float), - "max_torque": (-jnp.inf, jnp.inf, float), - "dt": (0, jnp.inf, float), - "g": (0, jnp.inf, float), - "m": (1e-6, jnp.inf, float), - "l": (1e-6, jnp.inf, float), - "max_steps_in_episode": (1, jnp.inf, int), -} +from carl.context.context_space import ContextFeature, UniformFloatContextFeature class CARLGymnaxPendulum(CARLGymnaxEnv): env_name: str = "Pendulum-v1" - max_episode_steps: int = int(DEFAULT_CONTEXT["max_steps_in_episode"]) - DEFAULT_CONTEXT: Context = DEFAULT_CONTEXT - - def _update_context(self) -> None: - self.env.env.env_params = EnvParams(**self.context) - - high = jnp.array( - [1.0, 1.0, self.env.env.env_params.max_speed], dtype=jnp.float32 - ) - self.build_observation_space(-high, high, CONTEXT_BOUNDS) + module: str = "classic_control.pendulum" + + @staticmethod + def get_context_features() -> dict[str, ContextFeature]: + return { + "dt": UniformFloatContextFeature( + "dt", lower=0.001, upper=10, default_value=0.05 + ), + "g": UniformFloatContextFeature( + "g", lower=-100, upper=100, default_value=10 + ), + "m": UniformFloatContextFeature( + "m", lower=1e-6, upper=100, default_value=1 + ), + "l": UniformFloatContextFeature( + "l", lower=1e-6, upper=100, default_value=1 + ), + "max_speed": UniformFloatContextFeature( + "max_speed", lower=0.08, upper=80, default_value=8 + ), + "max_torque": UniformFloatContextFeature( + "max_torque", lower=0.02, upper=40, default_value=2 + ), + } From 86e17373f93ceb16797671e9ae6502f8c1fa387c Mon Sep 17 00:00:00 2001 From: benjamc Date: Mon, 11 Dec 2023 13:23:32 +0100 Subject: [PATCH 28/37] Format --- carl/envs/gymnax/classic_control/carl_gymnax_cartpole.py | 5 +++-- carl/envs/gymnax/classic_control/carl_gymnax_mountaincar.py | 4 ++-- carl/envs/gymnax/classic_control/carl_gymnax_pendulum.py | 2 +- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/carl/envs/gymnax/classic_control/carl_gymnax_cartpole.py b/carl/envs/gymnax/classic_control/carl_gymnax_cartpole.py index fb61066d..90037d06 100644 --- a/carl/envs/gymnax/classic_control/carl_gymnax_cartpole.py +++ b/carl/envs/gymnax/classic_control/carl_gymnax_cartpole.py @@ -1,11 +1,12 @@ from __future__ import annotations -import gymnax import importlib + +import gymnax import jax.numpy as np -from carl.envs.gymnax.carl_gymnax_env import CARLGymnaxEnv from carl.context.context_space import ContextFeature, UniformFloatContextFeature +from carl.envs.gymnax.carl_gymnax_env import CARLGymnaxEnv class CARLGymnaxCartPole(CARLGymnaxEnv): diff --git a/carl/envs/gymnax/classic_control/carl_gymnax_mountaincar.py b/carl/envs/gymnax/classic_control/carl_gymnax_mountaincar.py index 2a99a236..7ac1af02 100644 --- a/carl/envs/gymnax/classic_control/carl_gymnax_mountaincar.py +++ b/carl/envs/gymnax/classic_control/carl_gymnax_mountaincar.py @@ -3,8 +3,8 @@ import gymnax import jax.numpy as np -from carl.envs.gymnax.carl_gymnax_env import CARLGymnaxEnv from carl.context.context_space import ContextFeature, UniformFloatContextFeature +from carl.envs.gymnax.carl_gymnax_env import CARLGymnaxEnv class CARLGymnaxMountainCar(CARLGymnaxEnv): @@ -48,7 +48,7 @@ def get_context_features() -> dict[str, ContextFeature]: "max_velocity_start", lower=-np.inf, upper=np.inf, default_value=0 ), } - + class CARLGymnaxMountainCarContinuous(CARLGymnaxMountainCar): env_name: str = "MountainCarContinuous-v0" diff --git a/carl/envs/gymnax/classic_control/carl_gymnax_pendulum.py b/carl/envs/gymnax/classic_control/carl_gymnax_pendulum.py index 745390bf..676477af 100644 --- a/carl/envs/gymnax/classic_control/carl_gymnax_pendulum.py +++ b/carl/envs/gymnax/classic_control/carl_gymnax_pendulum.py @@ -1,7 +1,7 @@ from __future__ import annotations -from carl.envs.gymnax.carl_gymnax_env import CARLGymnaxEnv from carl.context.context_space import ContextFeature, UniformFloatContextFeature +from carl.envs.gymnax.carl_gymnax_env import CARLGymnaxEnv class CARLGymnaxPendulum(CARLGymnaxEnv): From 72b82e163d572b3a0631137b1de2b8b0dca35e8d Mon Sep 17 00:00:00 2001 From: benjamc Date: Mon, 11 Dec 2023 13:25:24 +0100 Subject: [PATCH 29/37] Update changelog.md --- changelog.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/changelog.md b/changelog.md index 76d89dae..bc4be264 100644 --- a/changelog.md +++ b/changelog.md @@ -1,3 +1,6 @@ +# 1.1.0 +- Add gymnax classic control environments (#90) + # 1.0.0 Major overhaul of the CARL environment - Contexts are stored in each environment's class From 1a0f98596f197f2aba52a2b09d36789d47effe64 Mon Sep 17 00:00:00 2001 From: benjamc Date: Mon, 11 Dec 2023 13:26:07 +0100 Subject: [PATCH 30/37] Fix context space --- .../classic_control/carl_gymnax_mountaincar.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/carl/envs/gymnax/classic_control/carl_gymnax_mountaincar.py b/carl/envs/gymnax/classic_control/carl_gymnax_mountaincar.py index 7ac1af02..5aa5add8 100644 --- a/carl/envs/gymnax/classic_control/carl_gymnax_mountaincar.py +++ b/carl/envs/gymnax/classic_control/carl_gymnax_mountaincar.py @@ -35,18 +35,6 @@ def get_context_features() -> dict[str, ContextFeature]: "gravity": UniformFloatContextFeature( "gravity", lower=0, upper=np.inf, default_value=0.0025 ), - "min_position_start": UniformFloatContextFeature( - "min_position_start", lower=-np.inf, upper=np.inf, default_value=-0.6 - ), - "max_position_start": UniformFloatContextFeature( - "max_position_start", lower=-np.inf, upper=np.inf, default_value=-0.4 - ), - "min_velocity_start": UniformFloatContextFeature( - "min_velocity_start", lower=-np.inf, upper=np.inf, default_value=0 - ), - "max_velocity_start": UniformFloatContextFeature( - "max_velocity_start", lower=-np.inf, upper=np.inf, default_value=0 - ), } From a35e087e702f0c138eaa2461be863a66fbc3d469 Mon Sep 17 00:00:00 2001 From: benjamc Date: Mon, 11 Dec 2023 13:40:03 +0100 Subject: [PATCH 31/37] Fix context space --- .../carl_gymnax_mountaincar.py | 36 +++++++++++++------ 1 file changed, 25 insertions(+), 11 deletions(-) diff --git a/carl/envs/gymnax/classic_control/carl_gymnax_mountaincar.py b/carl/envs/gymnax/classic_control/carl_gymnax_mountaincar.py index 5aa5add8..2d59e691 100644 --- a/carl/envs/gymnax/classic_control/carl_gymnax_mountaincar.py +++ b/carl/envs/gymnax/classic_control/carl_gymnax_mountaincar.py @@ -14,26 +14,20 @@ class CARLGymnaxMountainCar(CARLGymnaxEnv): @staticmethod def get_context_features() -> dict[str, ContextFeature]: return { - "min_position": UniformFloatContextFeature( - "min_position", lower=-np.inf, upper=np.inf, default_value=-1.2 - ), - "max_position": UniformFloatContextFeature( - "max_position", lower=-np.inf, upper=np.inf, default_value=0.6 - ), "max_speed": UniformFloatContextFeature( - "max_speed", lower=0, upper=np.inf, default_value=0.07 + "max_speed", lower=1e-3, upper=10, default_value=0.07 ), "goal_position": UniformFloatContextFeature( - "goal_position", lower=-np.inf, upper=np.inf, default_value=0.45 + "goal_position", lower=-2, upper=2, default_value=0.45 ), "goal_velocity": UniformFloatContextFeature( - "goal_velocity", lower=-np.inf, upper=np.inf, default_value=0 + "goal_velocity", lower=-10, upper=10, default_value=0 ), "force": UniformFloatContextFeature( - "force", lower=-np.inf, upper=np.inf, default_value=0.001 + "force", lower=-10, upper=10, default_value=0.001 ), "gravity": UniformFloatContextFeature( - "gravity", lower=0, upper=np.inf, default_value=0.0025 + "gravity", lower=10, upper=-10, default_value=0.0025 ), } @@ -41,3 +35,23 @@ def get_context_features() -> dict[str, ContextFeature]: class CARLGymnaxMountainCarContinuous(CARLGymnaxMountainCar): env_name: str = "MountainCarContinuous-v0" module: str = "classic_control.continuous_mountain_car" + + @staticmethod + def get_context_features() -> dict[str, ContextFeature]: + return { + "max_speed": UniformFloatContextFeature( + "max_speed", lower=0, upper=np.inf, default_value=0.07 + ), + "goal_position": UniformFloatContextFeature( + "goal_position", lower=-np.inf, upper=np.inf, default_value=0.45 + ), + "goal_velocity": UniformFloatContextFeature( + "goal_velocity", lower=-np.inf, upper=np.inf, default_value=0 + ), + "power": UniformFloatContextFeature( + "power", lower=1e-6, upper=10, default_value=0.001 + ), + "gravity": UniformFloatContextFeature( + "gravity", lower=-10, upper=10, default_value=0.0025 + ), + } From 39782de87668e1c7c0e21367c8ebeac1d061a3e2 Mon Sep 17 00:00:00 2001 From: benjamc Date: Mon, 11 Dec 2023 13:40:10 +0100 Subject: [PATCH 32/37] Fix context space --- test/test_context_selector.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_context_selector.py b/test/test_context_selector.py index 3e135ba8..784e5939 100644 --- a/test/test_context_selector.py +++ b/test/test_context_selector.py @@ -12,7 +12,7 @@ class TestContextSelection(unittest.TestCase): @staticmethod def generate_contexts() -> Dict[Any, Context]: keys = "abc" - context = {"dt": 0.03, "gravity": 10.0, "m": 1.0, "l": 1.8} + context = {"dt": 0.03, "g": 10.0, "m": 1.0, "l": 1.8} contexts = {k: context for k in keys} return contexts From 10aa4063f0a24095a8457c7e5ae15aad19676cf5 Mon Sep 17 00:00:00 2001 From: benjamc Date: Mon, 11 Dec 2023 13:47:07 +0100 Subject: [PATCH 33/37] Fix context space --- carl/envs/gymnax/classic_control/carl_gymnax_cartpole.py | 2 +- carl/envs/gymnax/classic_control/carl_gymnax_mountaincar.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/carl/envs/gymnax/classic_control/carl_gymnax_cartpole.py b/carl/envs/gymnax/classic_control/carl_gymnax_cartpole.py index 90037d06..7d86b0ee 100644 --- a/carl/envs/gymnax/classic_control/carl_gymnax_cartpole.py +++ b/carl/envs/gymnax/classic_control/carl_gymnax_cartpole.py @@ -17,7 +17,7 @@ class CARLGymnaxCartPole(CARLGymnaxEnv): def get_context_features() -> dict[str, ContextFeature]: return { "gravity": UniformFloatContextFeature( - "gravity", lower=0.1, upper=np.inf, default_value=9.8 + "gravity", lower=0.01, upper=100, default_value=9.8 ), "masscart": UniformFloatContextFeature( "masscart", lower=0.1, upper=10, default_value=1.0 diff --git a/carl/envs/gymnax/classic_control/carl_gymnax_mountaincar.py b/carl/envs/gymnax/classic_control/carl_gymnax_mountaincar.py index 2d59e691..371d8554 100644 --- a/carl/envs/gymnax/classic_control/carl_gymnax_mountaincar.py +++ b/carl/envs/gymnax/classic_control/carl_gymnax_mountaincar.py @@ -27,7 +27,7 @@ def get_context_features() -> dict[str, ContextFeature]: "force", lower=-10, upper=10, default_value=0.001 ), "gravity": UniformFloatContextFeature( - "gravity", lower=10, upper=-10, default_value=0.0025 + "gravity", lower=-10, upper=10, default_value=0.0025 ), } From 7df9d72cb5e83b3b96540e57145303995dd1378b Mon Sep 17 00:00:00 2001 From: benjamc Date: Mon, 11 Dec 2023 14:07:36 +0100 Subject: [PATCH 34/37] Remove warnings --- carl/envs/__init__.py | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/carl/envs/__init__.py b/carl/envs/__init__.py index 5751ac5a..a51850d7 100644 --- a/carl/envs/__init__.py +++ b/carl/envs/__init__.py @@ -56,10 +56,7 @@ def check_spec(spec_name: str) -> bool: found = check_spec("dm_control") if found: from carl.envs.dmc import * -else: - warnings.warn( - "Module 'dm_control' not found. If you want to use these environments, please follow the installation guide." - ) + __all__ += envs.dmc.__all__ found = check_spec("distance") @@ -72,9 +69,5 @@ def check_spec(spec_name: str) -> bool: found = gymnax_spec is not None if found: from carl.envs.gymnax import * + __all__ += envs.gymnax.__all__ -else: - warnings.warn( - "Module 'gymnax' not found. If you want to use these environments, please follow the installation guide." - ) - \ No newline at end of file From 134f1617bc9a74e7383debd815a6f5a66819ffb7 Mon Sep 17 00:00:00 2001 From: benjamc Date: Mon, 11 Dec 2023 14:10:56 +0100 Subject: [PATCH 35/37] Fix pre-commit --- carl/envs/gymnax/carl_gymnax_env.py | 4 +--- carl/envs/gymnax/classic_control/carl_gymnax_acrobot.py | 1 - carl/envs/gymnax/classic_control/carl_gymnax_cartpole.py | 3 --- carl/envs/gymnax/classic_control/carl_gymnax_mountaincar.py | 1 - carl/envs/gymnax/utils.py | 6 +----- 5 files changed, 2 insertions(+), 13 deletions(-) diff --git a/carl/envs/gymnax/carl_gymnax_env.py b/carl/envs/gymnax/carl_gymnax_env.py index d8947290..84db31f3 100644 --- a/carl/envs/gymnax/carl_gymnax_env.py +++ b/carl/envs/gymnax/carl_gymnax_env.py @@ -4,14 +4,12 @@ import importlib -import gymnasium -import gymnax.environments from gymnasium.core import Env from carl.context.selection import AbstractSelector from carl.envs.carl_env import CARLEnv from carl.envs.gymnax.utils import make_gymnax_env -from carl.utils.types import Context, Contexts +from carl.utils.types import Contexts class CARLGymnaxEnv(CARLEnv): diff --git a/carl/envs/gymnax/classic_control/carl_gymnax_acrobot.py b/carl/envs/gymnax/classic_control/carl_gymnax_acrobot.py index cf16034e..e28b4a7e 100644 --- a/carl/envs/gymnax/classic_control/carl_gymnax_acrobot.py +++ b/carl/envs/gymnax/classic_control/carl_gymnax_acrobot.py @@ -1,6 +1,5 @@ from __future__ import annotations -import gymnax import jax.numpy as jnp import numpy as np diff --git a/carl/envs/gymnax/classic_control/carl_gymnax_cartpole.py b/carl/envs/gymnax/classic_control/carl_gymnax_cartpole.py index 7d86b0ee..5e3df165 100644 --- a/carl/envs/gymnax/classic_control/carl_gymnax_cartpole.py +++ b/carl/envs/gymnax/classic_control/carl_gymnax_cartpole.py @@ -2,9 +2,6 @@ import importlib -import gymnax -import jax.numpy as np - from carl.context.context_space import ContextFeature, UniformFloatContextFeature from carl.envs.gymnax.carl_gymnax_env import CARLGymnaxEnv diff --git a/carl/envs/gymnax/classic_control/carl_gymnax_mountaincar.py b/carl/envs/gymnax/classic_control/carl_gymnax_mountaincar.py index 371d8554..746ce67e 100644 --- a/carl/envs/gymnax/classic_control/carl_gymnax_mountaincar.py +++ b/carl/envs/gymnax/classic_control/carl_gymnax_mountaincar.py @@ -1,6 +1,5 @@ from __future__ import annotations -import gymnax import jax.numpy as np from carl.context.context_space import ContextFeature, UniformFloatContextFeature diff --git a/carl/envs/gymnax/utils.py b/carl/envs/gymnax/utils.py index dbf70a95..454335d1 100644 --- a/carl/envs/gymnax/utils.py +++ b/carl/envs/gymnax/utils.py @@ -1,17 +1,13 @@ from __future__ import annotations -from typing import Any, Optional, Sequence +from typing import Any import gymnasium import gymnasium.spaces import gymnax -from gymnasium.wrappers import EnvCompatibility -from gymnasium.wrappers.compatibility import LegacyEnv from gymnax.environments.environment import Environment, EnvParams from gymnax.environments.spaces import Space, gymnax_space_to_gym_space from gymnax.wrappers.gym import GymnaxToGymWrapper -from numpy._typing import DTypeLike -from numpy.random._generator import Generator as Generator # Although this converts to gym, the step API already is for gymnasium From 778e383442739144415fec82093984b68e693980 Mon Sep 17 00:00:00 2001 From: benjamc Date: Mon, 11 Dec 2023 14:11:46 +0100 Subject: [PATCH 36/37] Fix context space --- carl/envs/gymnax/classic_control/carl_gymnax_mountaincar.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/carl/envs/gymnax/classic_control/carl_gymnax_mountaincar.py b/carl/envs/gymnax/classic_control/carl_gymnax_mountaincar.py index 746ce67e..a51f1c4f 100644 --- a/carl/envs/gymnax/classic_control/carl_gymnax_mountaincar.py +++ b/carl/envs/gymnax/classic_control/carl_gymnax_mountaincar.py @@ -39,13 +39,13 @@ class CARLGymnaxMountainCarContinuous(CARLGymnaxMountainCar): def get_context_features() -> dict[str, ContextFeature]: return { "max_speed": UniformFloatContextFeature( - "max_speed", lower=0, upper=np.inf, default_value=0.07 + "max_speed", lower=1e-3, upper=10, default_value=0.07 ), "goal_position": UniformFloatContextFeature( - "goal_position", lower=-np.inf, upper=np.inf, default_value=0.45 + "goal_position", lower=-2, upper=2, default_value=0.45 ), "goal_velocity": UniformFloatContextFeature( - "goal_velocity", lower=-np.inf, upper=np.inf, default_value=0 + "goal_velocity", lower=-10, upper=10, default_value=0 ), "power": UniformFloatContextFeature( "power", lower=1e-6, upper=10, default_value=0.001 From 7c2e21e1157f4f1be37b9728601b0d28283dbfb0 Mon Sep 17 00:00:00 2001 From: benjamc Date: Mon, 11 Dec 2023 14:11:58 +0100 Subject: [PATCH 37/37] Fix pre-commit --- carl/envs/gymnax/classic_control/carl_gymnax_mountaincar.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/carl/envs/gymnax/classic_control/carl_gymnax_mountaincar.py b/carl/envs/gymnax/classic_control/carl_gymnax_mountaincar.py index a51f1c4f..f0421ac4 100644 --- a/carl/envs/gymnax/classic_control/carl_gymnax_mountaincar.py +++ b/carl/envs/gymnax/classic_control/carl_gymnax_mountaincar.py @@ -1,7 +1,5 @@ from __future__ import annotations -import jax.numpy as np - from carl.context.context_space import ContextFeature, UniformFloatContextFeature from carl.envs.gymnax.carl_gymnax_env import CARLGymnaxEnv