diff --git a/.gitignore b/.gitignore index a25976e..0ad959e 100644 --- a/.gitignore +++ b/.gitignore @@ -138,6 +138,4 @@ cython_debug/ .vscode .idea -/cmrl.egg-info/ /exp/ -/stable-baselines3/ diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1d44313..c4370e3 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -43,7 +43,7 @@ Emoji | Description :art: `:art:` | When you improved / added assets like themes. :rocket: `:rocket:` | When you improved performance. :memo: `:memo:` | When you wrote documentation. -:beetle: `:beetle:` | When you fixed a bug. +:bug: `:bug:` | When you fixed a bug. :twisted_rightwards_arrows: `:twisted_rightwards_arrows:` | When you merged a branch. :fire: `:fire:` | When you removed something. :truck: `:truck:` | When you moved / renamed something. diff --git a/README.md b/README.md index 3219478..4091dd8 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -![](/img/cmrl_logo.png) +![](/docs/cmrl_logo.png) # Causal-MBRL @@ -10,7 +10,7 @@ `cmrl`(short for `Causal-MBRL`) is a toolbox for facilitating the development of Causal Model-based Reinforcement -learning algorithms. It use [Stable-Baselines3](https://github.com/DLR-RM/stable-baselines3) as model-free engine and +learning algorithms. It uses [Stable-Baselines3](https://github.com/DLR-RM/stable-baselines3) as model-free engine and allows flexible use of causal models. `cmrl` is inspired by [MBRL-Lib](https://github.com/facebookresearch/mbrl-lib). Unlike MBRL-Lib, `cmrl` focuses on the @@ -111,18 +111,44 @@ cd causal-mbrl # create conda env conda create -n cmrl python=3.8 conda activate cmrl +# install torch +conda install pytorch -c pytorch # install cmrl and its dependent packages pip install -e . ``` -If there is no `cuda` in your device, it's convenient to install `cuda` and `pytorch` from conda directly (refer -to [pytorch](https://pytorch.org/get-started/locally/)): +for pytorch -````shell -# for example, in the case of cuda=11.3 -conda install pytorch cudatoolkit=11.3 -c pytorch -```` +```shell +# for MacOS +conda install pytorch -c pytorch +# for Linux +conda install pytorch pytorch-cuda=11.6 -c pytorch -c nvidia +``` + +for KCIT and RCIT + +```shell +conda install -c conda-forge r-base +conda install -c conda-forge r-devtools +R +``` +```shell +# Install the RCIT from Github. +install.packages("devtools") +library(devtools) +install_github("ericstrobl/RCIT") +library(RCIT) + +# Install R libraries for RCIT +install.packages("MASS") +install.packages("momentchi2") +install.packages("devtools") + +# test RCIT +RCIT(rnorm(1000),rnorm(1000),rnorm(1000)) +``` ## install using pip coming soon. diff --git a/cmrl/agent/__init__.py b/cmrl/agent/__init__.py deleted file mode 100644 index 012afb4..0000000 --- a/cmrl/agent/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from cmrl.agent.core import Agent, RandomAgent, complete_agent_cfg, load_agent diff --git a/cmrl/agent/core.py b/cmrl/agent/core.py deleted file mode 100644 index b6a074e..0000000 --- a/cmrl/agent/core.py +++ /dev/null @@ -1,137 +0,0 @@ -import abc -import pathlib -from typing import Any, Optional, Union - -import gym -import hydra -import numpy as np -import omegaconf - - -class Agent: - """Abstract class for all agents.""" - - @abc.abstractmethod - def act(self, obs: np.ndarray, **kwargs) -> np.ndarray: - pass - - def reset(self): - pass - - -class RandomAgent(Agent): - """An agent that samples action from the environments action space. - - Args: - env (gym.Env): the environment on which the agent will act. - """ - - def __init__(self, env: gym.Env): - self.env = env - - def act(self, obs: np.ndarray, **kwargs) -> np.ndarray: - return self.env.action_space.sample() - - -def complete_agent_cfg(env: gym.Env, agent_cfg: omegaconf.DictConfig): - obs_shape = env.observation_space.shape - act_shape = env.action_space.shape - - def _check_and_replace(key: str, value: Any, cfg: omegaconf.DictConfig): - if key in cfg.keys() and key not in cfg: - setattr(cfg, key, value) - - # create numpy object by existed object - def _create_numpy_config(array): - return { - "_target_": "numpy.array", - "object": array.tolist(), - "dtype": str(array.dtype), - } - - _check_and_replace("num_inputs", obs_shape[0], agent_cfg) - if "action_space" in agent_cfg.keys() and isinstance(agent_cfg.action_space, omegaconf.DictConfig): - _check_and_replace("low", _create_numpy_config(env.action_space.low), agent_cfg.action_space) - _check_and_replace("high", _create_numpy_config(env.action_space.high), agent_cfg.action_space) - _check_and_replace("shape", env.action_space.shape, agent_cfg.action_space) - - if "obs_dim" in agent_cfg.keys() and "obs_dim" not in agent_cfg: - agent_cfg.obs_dim = obs_shape[0] - if "action_dim" in agent_cfg.keys() and "action_dim" not in agent_cfg: - agent_cfg.action_dim = act_shape[0] - if "action_range" in agent_cfg.keys() and "action_range" not in agent_cfg: - agent_cfg.action_range = [ - float(env.action_space.low.min()), - float(env.action_space.high.max()), - ] - if "action_lb" in agent_cfg.keys() and "action_lb" not in agent_cfg: - agent_cfg.action_lb = _create_numpy_config(env.action_space.low) - if "action_ub" in agent_cfg.keys() and "action_ub" not in agent_cfg: - agent_cfg.action_ub = _create_numpy_config(env.action_space.high) - - if "env" in agent_cfg.keys(): - _check_and_replace( - "low", - _create_numpy_config(env.action_space.low), - agent_cfg.env.action_space, - ) - _check_and_replace( - "high", - _create_numpy_config(env.action_space.high), - agent_cfg.env.action_space, - ) - _check_and_replace("shape", env.action_space.shape, agent_cfg.env.action_space) - - _check_and_replace( - "low", - _create_numpy_config(env.observation_space.low), - agent_cfg.env.observation_space, - ) - _check_and_replace( - "high", - _create_numpy_config(env.observation_space.high), - agent_cfg.env.observation_space, - ) - _check_and_replace("shape", env.observation_space.shape, agent_cfg.env.observation_space) - - return agent_cfg - - -def load_agent( - agent_path: Union[str, pathlib.Path], - env: gym.Env, - type: Optional[str] = "best", - device: Optional[str] = None, -) -> Agent: - """Loads an agent from a Hydra config file at the given path. - - For agent of type "pytorch_sac.agent.sac.SACAgent", the directory - must contain the following files: - - - ".hydra/config.yaml": the Hydra configuration for the agent. - - "critic.pth": the saved checkpoint for the critic. - - "actor.pth": the saved checkpoint for the actor. - - Args: - agent_path (str or pathlib.Path): a path to the directory where the agent is saved. - env (gym.Env): the environment on which the agent will operate (only used to complete - the agent's configuration). - - Returns: - (Agent): the new agent. - """ - agent_path = pathlib.Path(agent_path) - cfg = omegaconf.OmegaConf.load(agent_path / ".hydra" / "config.yaml") - cfg.device = device - - if cfg.algorithm.agent._target_ == "cmrl.third_party.pytorch_sac.sac.SAC": - import cmrl.third_party.pytorch_sac as pytorch_sac - - from .sac_wrapper import SACAgent - - complete_agent_cfg(env, cfg.algorithm.agent) - agent: pytorch_sac.SAC = hydra.utils.instantiate(cfg.algorithm.agent) - agent.load_checkpoint(ckpt_path=agent_path / "sac_{}.pth".format(type), device=device) - return SACAgent(agent) - else: - raise ValueError("Invalid agent configuration.") diff --git a/cmrl/agent/sac_wrapper.py b/cmrl/agent/sac_wrapper.py deleted file mode 100644 index b83c5a5..0000000 --- a/cmrl/agent/sac_wrapper.py +++ /dev/null @@ -1,28 +0,0 @@ -import numpy as np -import torch - -import cmrl.third_party.pytorch_sac as pytorch_sac - -from .core import Agent - - -class SACAgent(Agent): - def __init__(self, sac_agent: pytorch_sac.SAC): - self.sac_agent = sac_agent - - def act(self, obs: np.ndarray, sample: bool = False, batched: bool = False, **kwargs) -> np.ndarray: - """Issues an action given an observation. - - Args: - obs (np.ndarray): the observation (or batch of observations) for which the action - is needed. - sample (bool): if ``True`` the agent samples actions from its policy, otherwise it - returns the mean policy value. Defaults to ``False``. - batched (bool): if ``True`` signals to the agent that the obs should be interpreted - as a batch. - - Returns: - (np.ndarray): the action. - """ - with torch.no_grad(): - return self.sac_agent.select_action(obs, batched=batched, evaluate=not sample) diff --git a/cmrl/algorithms/__init__.py b/cmrl/algorithms/__init__.py index 0f9ea59..5b1eb8c 100644 --- a/cmrl/algorithms/__init__.py +++ b/cmrl/algorithms/__init__.py @@ -1,4 +1,4 @@ -from cmrl.algorithms.offline import mopo -from cmrl.algorithms.offline import off_dyna -from cmrl.algorithms.online import mbpo -from cmrl.algorithms.online import on_dyna +from cmrl.algorithms.off_dyna import OfflineDyna +from cmrl.algorithms.mopo import MOPO +from cmrl.algorithms.on_dyna import OnlineDyna +from cmrl.algorithms.mbpo import MBPO diff --git a/cmrl/algorithms/base_algorithm.py b/cmrl/algorithms/base_algorithm.py new file mode 100644 index 0000000..532ea13 --- /dev/null +++ b/cmrl/algorithms/base_algorithm.py @@ -0,0 +1,116 @@ +import os +from typing import Optional +from functools import partial + +import numpy as np +import torch +from omegaconf import DictConfig, OmegaConf +from stable_baselines3.common.buffers import ReplayBuffer +from stable_baselines3.common.callbacks import BaseCallback +import wandb + +from cmrl.models.fake_env import VecFakeEnv +from cmrl.sb3_extension.logger import configure as logger_configure +from cmrl.sb3_extension.eval_callback import EvalCallback +from cmrl.utils.creator import create_dynamics, create_agent +from cmrl.utils.env import make_env + + +class BaseAlgorithm: + def __init__( + self, + cfg: DictConfig, + work_dir: Optional[str] = None, + ): + self.cfg = cfg + self.work_dir = work_dir or os.getcwd() + + self.env, fns = make_env(self.cfg) + self.reward_fn, self.termination_fn, self.get_init_obs_fn, self.obs2state_fn, self.state2obs_fn = fns + + self.eval_env, *_ = make_env(self.cfg) + np.random.seed(self.cfg.seed) + torch.manual_seed(self.cfg.seed) + + format_strings = ["tensorboard", "multi_csv"] + if self.cfg.verbose: + format_strings += ["stdout"] + self.logger = logger_configure("log", format_strings) + + if cfg.wandb: + wandb.init( + project="causal-mbrl", + group=cfg.exp_name, + config=OmegaConf.to_container(cfg, resolve=True), + sync_tensorboard=True, + ) + + # create ``cmrl`` dynamics + self.dynamics = create_dynamics( + self.cfg, self.env.state_space, self.env.action_space, self.obs2state_fn, self.state2obs_fn, logger=self.logger + ) + + if self.cfg.transition.name == "oracle_transition": + graph = self.env.get_transition_graph() if self.cfg.transition.oracle == "truth" else None + self.dynamics.transition.set_oracle_graph(graph) + if self.cfg.reward_mech.learn and not self.cfg.reward_mech.name == "oracle_reward_mech": + graph = self.env.get_reward_mech_graph() if self.cfg.transition.oracle == "truth" else None + self.dynamics.reward_mech.set_oracle_graph(graph) + if self.cfg.termination_mech.learn and not self.cfg.termination_mech.name == "oracle_termination_mech": + graph = self.env.get_termination_mech_graph() if self.cfg.transition.oracle == "truth" else None + self.dynamics.termination_mech.set_oracle_graph(graph) + + # create sb3's replay buffer for real offline data + self.real_replay_buffer = ReplayBuffer( + cfg.task.num_steps, + self.env.observation_space, + self.env.action_space, + self.cfg.device, + handle_timeout_termination=False, + ) + + self.partial_fake_env = partial( + VecFakeEnv, + self.cfg.algorithm.num_envs, + self.env.state_space, + self.env.action_space, + self.dynamics, + self.reward_fn, + self.termination_fn, + self.get_init_obs_fn, + self.real_replay_buffer, + penalty_coeff=self.cfg.task.penalty_coeff, + logger=self.logger, + ) + self.agent = create_agent(self.cfg, self.fake_env, self.logger) + + @property + def fake_env(self) -> VecFakeEnv: + return self.partial_fake_env( + deterministic=self.cfg.algorithm.deterministic, + max_episode_steps=self.env.spec.max_episode_steps, + branch_rollout=False, + ) + + @property + def callback(self) -> BaseCallback: + fake_eval_env = self.partial_fake_env( + deterministic=True, max_episode_steps=self.env.spec.max_episode_steps, branch_rollout=False + ) + return EvalCallback( + self.eval_env, + fake_eval_env, + n_eval_episodes=self.cfg.task.n_eval_episodes, + best_model_save_path="./", + eval_freq=self.cfg.task.eval_freq, + deterministic=True, + render=False, + ) + + def learn(self): + self._setup_learn() + + self.agent.learn(total_timesteps=self.cfg.task.num_steps, callback=self.callback) + + def _setup_learn(self): + pass diff --git a/cmrl/algorithms/mbpo.py b/cmrl/algorithms/mbpo.py new file mode 100644 index 0000000..e34f2f3 --- /dev/null +++ b/cmrl/algorithms/mbpo.py @@ -0,0 +1,40 @@ +from typing import Optional + +from omegaconf import DictConfig +from stable_baselines3.common.callbacks import BaseCallback, CallbackList + +from cmrl.models.fake_env import VecFakeEnv +from cmrl.algorithms.base_algorithm import BaseAlgorithm +from cmrl.sb3_extension.online_mb_callback import OnlineModelBasedCallback + + +class MBPO(BaseAlgorithm): + def __init__( + self, + cfg: DictConfig, + work_dir: Optional[str] = None, + ): + super(MBPO, self).__init__(cfg, work_dir) + + @property + def fake_env(self) -> VecFakeEnv: + return self.partial_fake_env( + deterministic=self.cfg.algorithm.deterministic, + max_episode_steps=self.cfg.algorithm.branch_rollout_length, + branch_rollout=True, + ) + + @property + def callback(self) -> BaseCallback: + eval_callback = super(MBPO, self).callback + omb_callback = OnlineModelBasedCallback( + self.env, + self.dynamics, + self.real_replay_buffer, + total_online_timesteps=self.cfg.task.online_num_steps, + initial_exploration_steps=self.cfg.algorithm.initial_exploration_steps, + freq_train_model=self.cfg.task.freq_train_model, + device=self.cfg.device, + ) + + return CallbackList([eval_callback, omb_callback]) diff --git a/cmrl/algorithms/mopo.py b/cmrl/algorithms/mopo.py new file mode 100644 index 0000000..4a98517 --- /dev/null +++ b/cmrl/algorithms/mopo.py @@ -0,0 +1,38 @@ +from typing import Optional + +from omegaconf import DictConfig + +from cmrl.models.fake_env import VecFakeEnv +from cmrl.algorithms.base_algorithm import BaseAlgorithm +from cmrl.utils.env import load_offline_data +from cmrl.algorithms.util import maybe_load_offline_model + + +class MOPO(BaseAlgorithm): + def __init__( + self, + cfg: DictConfig, + work_dir: Optional[str] = None, + ): + super(MOPO, self).__init__(cfg, work_dir) + + @property + def fake_env(self) -> VecFakeEnv: + return self.partial_fake_env( + deterministic=self.cfg.algorithm.deterministic, + max_episode_steps=self.cfg.algorithm.branch_rollout_length, + branch_rollout=True, + ) + + def _setup_learn(self): + load_offline_data(self.env, self.real_replay_buffer, self.cfg.task.dataset, self.cfg.task.use_ratio) + + if self.cfg.task.get("auto_load_offline_model", False): + existed_trained_model = maybe_load_offline_model(self.dynamics, self.cfg, work_dir=self.work_dir) + else: + existed_trained_model = None + if not existed_trained_model: + self.dynamics.learn( + real_replay_buffer=self.real_replay_buffer, + work_dir=self.work_dir, + ) diff --git a/cmrl/algorithms/off_dyna.py b/cmrl/algorithms/off_dyna.py new file mode 100644 index 0000000..3911c05 --- /dev/null +++ b/cmrl/algorithms/off_dyna.py @@ -0,0 +1,29 @@ +from typing import Optional + +from omegaconf import DictConfig + +from cmrl.algorithms.base_algorithm import BaseAlgorithm +from cmrl.utils.env import load_offline_data +from cmrl.algorithms.util import maybe_load_offline_model + + +class OfflineDyna(BaseAlgorithm): + def __init__( + self, + cfg: DictConfig, + work_dir: Optional[str] = None, + ): + super(OfflineDyna, self).__init__(cfg, work_dir) + + def _setup_learn(self): + load_offline_data(self.env, self.real_replay_buffer, self.cfg.task.dataset, self.cfg.task.use_ratio) + + if self.cfg.task.get("auto_load_offline_model", False): + existed_trained_model = maybe_load_offline_model(self.dynamics, self.cfg, work_dir=self.work_dir) + else: + existed_trained_model = None + if not existed_trained_model: + self.dynamics.learn( + real_replay_buffer=self.real_replay_buffer, + work_dir=self.work_dir, + ) diff --git a/cmrl/algorithms/offline/mopo.py b/cmrl/algorithms/offline/mopo.py deleted file mode 100644 index 860518e..0000000 --- a/cmrl/algorithms/offline/mopo.py +++ /dev/null @@ -1,84 +0,0 @@ -import os -from typing import Optional, cast - -import emei -import hydra.utils -import numpy as np -from omegaconf import DictConfig -from stable_baselines3.common.base_class import BaseAlgorithm -from stable_baselines3.common.buffers import ReplayBuffer - -from cmrl.agent import complete_agent_cfg -from cmrl.algorithms.util import maybe_load_trained_offline_model, setup_fake_env, load_offline_data -from cmrl.models.dynamics import ConstraintBasedDynamics -from cmrl.sb3_extension.eval_callback import EvalCallback -from cmrl.sb3_extension.logger import configure as logger_configure -from cmrl.types import InitObsFnType, RewardFnType, TermFnType -from cmrl.util.creator import create_dynamics - - -def train( - env: emei.EmeiEnv, - eval_env: emei.EmeiEnv, - termination_fn: Optional[TermFnType], - reward_fn: Optional[RewardFnType], - get_init_obs_fn: Optional[InitObsFnType], - cfg: DictConfig, - work_dir: Optional[str] = None, -): - obs_shape = env.observation_space.shape - act_shape = env.action_space.shape - - # build model-free agent, which is a stable-baselines3's agent - complete_agent_cfg(env, cfg.algorithm.agent) - agent = cast(BaseAlgorithm, hydra.utils.instantiate(cfg.algorithm.agent)) - - work_dir = work_dir or os.getcwd() - logger = logger_configure("log", ["tensorboard", "multi_csv", "stdout"]) - - numpy_generator = np.random.default_rng(seed=cfg.seed) - - # create initial dataset and add it to replay buffer - dynamics = create_dynamics(cfg.dynamics, obs_shape, act_shape, logger=logger) - real_replay_buffer = ReplayBuffer( - cfg.task.num_steps, env.observation_space, env.action_space, cfg.device, handle_timeout_termination=False - ) - load_offline_data(cfg, env, real_replay_buffer) - - fake_eval_env = setup_fake_env( - cfg=cfg, - agent=agent, - dynamics=dynamics, - reward_fn=reward_fn, - termination_fn=termination_fn, - get_init_obs_fn=get_init_obs_fn, - real_replay_buffer=real_replay_buffer, - logger=logger, - max_episode_steps=env.spec.max_episode_steps, - penalty_coeff=cfg.algorithm.penalty_coeff, - ) - - if hasattr(env, "get_causal_graph"): - oracle_causal_graph = env.get_causal_graph() - else: - oracle_causal_graph = None - - if isinstance(dynamics, ConstraintBasedDynamics): - dynamics.set_oracle_mask("transition", oracle_causal_graph.T) - - existed_trained_model = maybe_load_trained_offline_model(dynamics, cfg, obs_shape, act_shape, work_dir=work_dir) - if not existed_trained_model: - dynamics.learn(real_replay_buffer, **cfg.dynamics, work_dir=work_dir) - - eval_callback = EvalCallback( - eval_env, - fake_eval_env, - n_eval_episodes=cfg.task.n_eval_episodes, - best_model_save_path="./", - eval_freq=1000, - deterministic=True, - render=False, - ) - - agent.set_logger(logger) - agent.learn(total_timesteps=cfg.task.num_steps, callback=eval_callback) diff --git a/cmrl/algorithms/offline/off_dyna.py b/cmrl/algorithms/offline/off_dyna.py deleted file mode 100644 index 9993c55..0000000 --- a/cmrl/algorithms/offline/off_dyna.py +++ /dev/null @@ -1,81 +0,0 @@ -import os -from typing import Optional, cast - -import emei -import hydra.utils -import numpy as np -from omegaconf import DictConfig -from stable_baselines3.common.base_class import BaseAlgorithm -from stable_baselines3.common.buffers import ReplayBuffer - -from cmrl.agent import complete_agent_cfg -from cmrl.algorithms.util import maybe_load_trained_offline_model, setup_fake_env, load_offline_data -from cmrl.models.dynamics import ConstraintBasedDynamics -from cmrl.sb3_extension.eval_callback import EvalCallback -from cmrl.sb3_extension.logger import configure as logger_configure -from cmrl.types import InitObsFnType, RewardFnType, TermFnType -from cmrl.util.creator import create_dynamics - - -def train( - env: emei.EmeiEnv, - eval_env: emei.EmeiEnv, - termination_fn: Optional[TermFnType], - reward_fn: Optional[RewardFnType], - get_init_obs_fn: Optional[InitObsFnType], - cfg: DictConfig, - work_dir: Optional[str] = None, -): - obs_shape = env.observation_space.shape - act_shape = env.action_space.shape - - # build model-free agent, which is a stable-baselines3's agent - complete_agent_cfg(env, cfg.algorithm.agent) - agent = cast(BaseAlgorithm, hydra.utils.instantiate(cfg.algorithm.agent)) - - work_dir = work_dir or os.getcwd() - logger = logger_configure("log", ["tensorboard", "multi_csv", "stdout"]) - - # create initial dataset and add it to replay buffer - dynamics = create_dynamics(cfg.dynamics, obs_shape, act_shape, logger=logger) - real_replay_buffer = ReplayBuffer( - cfg.task.num_steps, env.observation_space, env.action_space, cfg.device, handle_timeout_termination=False - ) - load_offline_data(cfg, env, real_replay_buffer) - - fake_eval_env = setup_fake_env( - cfg=cfg, - agent=agent, - dynamics=dynamics, - reward_fn=reward_fn, - termination_fn=termination_fn, - get_init_obs_fn=get_init_obs_fn, - logger=logger, - max_episode_steps=env.spec.max_episode_steps, - penalty_coeff=cfg.algorithm.penalty_coeff, - ) - - if hasattr(env, "get_causal_graph"): - oracle_causal_graph = env.get_causal_graph() - else: - oracle_causal_graph = None - - if isinstance(dynamics, ConstraintBasedDynamics): - dynamics.set_oracle_mask("transition", oracle_causal_graph.T) - - existed_trained_model = maybe_load_trained_offline_model(dynamics, cfg, obs_shape, act_shape, work_dir=work_dir) - if not existed_trained_model: - dynamics.learn(real_replay_buffer, **cfg.dynamics, work_dir=work_dir) - - eval_callback = EvalCallback( - eval_env, - fake_eval_env, - n_eval_episodes=cfg.task.n_eval_episodes, - best_model_save_path="./", - eval_freq=1000, - deterministic=True, - render=False, - ) - - agent.set_logger(logger) - agent.learn(total_timesteps=cfg.task.num_steps, callback=eval_callback) diff --git a/cmrl/algorithms/on_dyna.py b/cmrl/algorithms/on_dyna.py new file mode 100644 index 0000000..32bbdb4 --- /dev/null +++ b/cmrl/algorithms/on_dyna.py @@ -0,0 +1,32 @@ +from typing import Optional + +from omegaconf import DictConfig +from stable_baselines3.common.callbacks import BaseCallback, CallbackList + +from cmrl.models.fake_env import VecFakeEnv +from cmrl.algorithms.base_algorithm import BaseAlgorithm +from cmrl.sb3_extension.online_mb_callback import OnlineModelBasedCallback + + +class OnlineDyna(BaseAlgorithm): + def __init__( + self, + cfg: DictConfig, + work_dir: Optional[str] = None, + ): + super(OnlineDyna, self).__init__(cfg, work_dir) + + @property + def callback(self) -> BaseCallback: + eval_callback = super(OnlineDyna, self).callback + omb_callback = OnlineModelBasedCallback( + self.env, + self.dynamics, + self.real_replay_buffer, + total_online_timesteps=self.cfg.task.online_num_steps, + initial_exploration_steps=self.cfg.algorithm.initial_exploration_steps, + freq_train_model=self.cfg.task.freq_train_model, + device=self.cfg.device, + ) + + return CallbackList([eval_callback, omb_callback]) diff --git a/cmrl/algorithms/online/mbpo.py b/cmrl/algorithms/online/mbpo.py deleted file mode 100644 index d858141..0000000 --- a/cmrl/algorithms/online/mbpo.py +++ /dev/null @@ -1,94 +0,0 @@ -import os -from typing import Optional, cast - -import emei -import hydra.utils -import numpy as np -from omegaconf import DictConfig -from stable_baselines3.common.base_class import BaseAlgorithm -from stable_baselines3.common.callbacks import CallbackList -from stable_baselines3.common.buffers import ReplayBuffer - -from cmrl.agent import complete_agent_cfg -from cmrl.algorithms.util import setup_fake_env -from cmrl.models.dynamics import ConstraintBasedDynamics -from cmrl.sb3_extension.eval_callback import EvalCallback -from cmrl.sb3_extension.online_mb_callback import OnlineModelBasedCallback -from cmrl.sb3_extension.logger import configure as logger_configure -from cmrl.types import InitObsFnType, RewardFnType, TermFnType -from cmrl.util.creator import create_dynamics - - -def train( - env: emei.EmeiEnv, - eval_env: emei.EmeiEnv, - termination_fn: Optional[TermFnType], - reward_fn: Optional[RewardFnType], - get_init_obs_fn: Optional[InitObsFnType], - cfg: DictConfig, - work_dir: Optional[str] = None, -): - obs_shape = env.observation_space.shape - act_shape = env.action_space.shape - - # build model-free agent, which is a stable-baselines3's agent - complete_agent_cfg(env, cfg.algorithm.agent) - agent = cast(BaseAlgorithm, hydra.utils.instantiate(cfg.algorithm.agent)) - - work_dir = work_dir or os.getcwd() - logger = logger_configure("log", ["tensorboard", "multi_csv", "stdout"]) - - numpy_generator = np.random.default_rng(seed=cfg.seed) - - dynamics = create_dynamics(cfg.dynamics, obs_shape, act_shape, logger=logger) - real_replay_buffer = ReplayBuffer( - cfg.task.online_num_steps, - env.observation_space, - env.action_space, - device=cfg.device, - n_envs=1, - optimize_memory_usage=False, - ) - - fake_eval_env = setup_fake_env( - cfg=cfg, - agent=agent, - dynamics=dynamics, - reward_fn=reward_fn, - termination_fn=termination_fn, - get_init_obs_fn=get_init_obs_fn, - real_replay_buffer=real_replay_buffer, - logger=logger, - max_episode_steps=env.spec.max_episode_steps, - ) - - if hasattr(env, "causal_graph"): - oracle_causal_graph = env.causal_graph - else: - oracle_causal_graph = None - - if isinstance(dynamics, ConstraintBasedDynamics): - dynamics.set_oracle_mask("transition", oracle_causal_graph.T) - - eval_callback = EvalCallback( - eval_env, - fake_eval_env, - n_eval_episodes=cfg.task.n_eval_episodes, - best_model_save_path="./", - eval_freq=cfg.task.eval_freq, - deterministic=True, - render=False, - ) - - omb_callback = OnlineModelBasedCallback( - env, - dynamics, - real_replay_buffer, - total_num_steps=cfg.task.online_num_steps, - initial_exploration_steps=cfg.algorithm.initial_exploration_steps, - freq_train_model=cfg.task.freq_train_model, - device=cfg.device, - ) - - agent.set_logger(logger) - agent.learn(total_timesteps=int(1e10), callback=CallbackList([eval_callback, omb_callback])) diff --git a/cmrl/algorithms/online/on_dyna.py b/cmrl/algorithms/online/on_dyna.py deleted file mode 100644 index a1056ca..0000000 --- a/cmrl/algorithms/online/on_dyna.py +++ /dev/null @@ -1,93 +0,0 @@ -import os -from typing import Optional, cast - -import emei -import hydra.utils -import numpy as np -from omegaconf import DictConfig -from stable_baselines3.common.base_class import BaseAlgorithm -from stable_baselines3.common.callbacks import CallbackList -from stable_baselines3.common.buffers import ReplayBuffer - -from cmrl.agent import complete_agent_cfg -from cmrl.algorithms.util import setup_fake_env -from cmrl.models.dynamics import ConstraintBasedDynamics -from cmrl.sb3_extension.eval_callback import EvalCallback -from cmrl.sb3_extension.online_mb_callback import OnlineModelBasedCallback -from cmrl.sb3_extension.logger import configure as logger_configure -from cmrl.types import InitObsFnType, RewardFnType, TermFnType -from cmrl.util.creator import create_dynamics - - -def train( - env: emei.EmeiEnv, - eval_env: emei.EmeiEnv, - termination_fn: Optional[TermFnType], - reward_fn: Optional[RewardFnType], - get_init_obs_fn: Optional[InitObsFnType], - cfg: DictConfig, - work_dir: Optional[str] = None, -): - obs_shape = env.observation_space.shape - act_shape = env.action_space.shape - - # build model-free agent, which is a stable-baselines3's agent - complete_agent_cfg(env, cfg.algorithm.agent) - agent = cast(BaseAlgorithm, hydra.utils.instantiate(cfg.algorithm.agent)) - - work_dir = work_dir or os.getcwd() - logger = logger_configure("log", ["tensorboard", "multi_csv", "stdout"]) - - numpy_generator = np.random.default_rng(seed=cfg.seed) - - dynamics = create_dynamics(cfg.dynamics, obs_shape, act_shape, logger=logger) - real_replay_buffer = ReplayBuffer( - cfg.task.online_num_steps, - env.observation_space, - env.action_space, - device=cfg.device, - n_envs=1, - optimize_memory_usage=False, - ) - - fake_eval_env = setup_fake_env( - cfg=cfg, - agent=agent, - dynamics=dynamics, - reward_fn=reward_fn, - termination_fn=termination_fn, - get_init_obs_fn=get_init_obs_fn, - logger=logger, - max_episode_steps=env.spec.max_episode_steps, - ) - - if hasattr(env, "causal_graph"): - oracle_causal_graph = env.causal_graph - else: - oracle_causal_graph = None - - if isinstance(dynamics, ConstraintBasedDynamics): - dynamics.set_oracle_mask("transition", oracle_causal_graph.T) - - eval_callback = EvalCallback( - eval_env, - fake_eval_env, - n_eval_episodes=cfg.task.n_eval_episodes, - best_model_save_path="./", - eval_freq=cfg.task.eval_freq, - deterministic=True, - render=False, - ) - - omb_callback = OnlineModelBasedCallback( - env, - dynamics, - real_replay_buffer, - total_num_steps=cfg.task.online_num_steps, - initial_exploration_steps=cfg.algorithm.initial_exploration_steps, - freq_train_model=cfg.task.freq_train_model, - device=cfg.device, - ) - - agent.set_logger(logger) - agent.learn(total_timesteps=int(1e10), callback=CallbackList([eval_callback, omb_callback])) diff --git a/cmrl/algorithms/util.py b/cmrl/algorithms/util.py index 6e901c4..fd594f1 100644 --- a/cmrl/algorithms/util.py +++ b/cmrl/algorithms/util.py @@ -1,29 +1,29 @@ -import pathlib from typing import Optional, cast from copy import deepcopy +import pathlib -import emei import hydra -import numpy as np -from omegaconf import DictConfig +from omegaconf import DictConfig, OmegaConf from stable_baselines3.common.vec_env.vec_monitor import VecMonitor from stable_baselines3.common.base_class import BaseAlgorithm from stable_baselines3.common.buffers import ReplayBuffer from cmrl.types import InitObsFnType, RewardFnType, TermFnType -from cmrl.models.dynamics import BaseDynamics -from cmrl.util.config import get_complete_dynamics_cfg, load_hydra_cfg -from cmrl.models.fake_env import VecFakeEnv + +from cmrl.models.dynamics import Dynamics +from cmrl.utils.config import load_hydra_cfg -def is_same_dict(dict1, dict2): +def compare_dict(dict1, dict2): + if len(list(dict1)) != len(list(dict2)): + return False for key in dict1: if key not in dict2: return False else: - if isinstance(dict1[key], DictConfig) and isinstance(dict2[key], DictConfig): - if not is_same_dict(dict1[key], dict2[key]): + if isinstance(dict1[key], dict) and isinstance(dict2[key], dict): + if not compare_dict(dict1[key], dict2[key]): return False else: if dict1[key] != dict2[key]: @@ -31,94 +31,38 @@ def is_same_dict(dict1, dict2): return True -def maybe_load_trained_offline_model(dynamics: BaseDynamics, cfg, obs_shape, act_shape, work_dir): +def maybe_load_offline_model( + dynamics: Dynamics, + cfg: DictConfig, + work_dir, +): work_dir = pathlib.Path(work_dir) if "." not in work_dir.name: # exp by hydra's MULTIRUN mode - task_exp_dir = work_dir.parent.parent.parent - else: task_exp_dir = work_dir.parent.parent - dynamics_cfg = cfg.dynamics - - for date_dir in task_exp_dir.glob(r"*"): - for time_dir in date_dir.glob(r"*"): - if (time_dir / "multirun.yaml").exists(): # exp by hydra's MULTIRUN mode, multi exp in this time - this_time_exp_dir_list = list(time_dir.glob(r"*")) - else: # only one exp in this time - this_time_exp_dir_list = [time_dir] - - for exp_dir in this_time_exp_dir_list: - if not (exp_dir / ".hydra").exists(): - continue - exp_cfg = load_hydra_cfg(exp_dir) - exp_dynamics_cfg = get_complete_dynamics_cfg(exp_cfg.dynamics, obs_shape, act_shape) - - if exp_cfg.seed == cfg.seed and is_same_dict(dynamics_cfg, exp_dynamics_cfg): - exist_model_file = True - for mech in dynamics.learn_mech: - mech_file_name = getattr(dynamics, mech).model_file_name - if not (exp_dir / mech_file_name).exists(): - exist_model_file = False - if exist_model_file: - dynamics.load(exp_dir) - print("loaded dynamics from {}".format(exp_dir)) - return True + else: + task_exp_dir = work_dir.parent + + transition_cfg = OmegaConf.to_container(cfg.transition, resolve=True) + + for time_dir in task_exp_dir.glob(r"*"): + if (time_dir / "multirun.yaml").exists(): # exp by hydra's MULTIRUN mode, multi exp in this time + this_time_exp_dir_list = list(time_dir.glob(r"*")) + else: # only one exp in this time + this_time_exp_dir_list = [time_dir] + + for exp_dir in this_time_exp_dir_list: + if not (exp_dir / ".hydra").exists(): + continue + exp_cfg = load_hydra_cfg(exp_dir) + + exp_transition_dir = OmegaConf.to_container(exp_cfg.transition, resolve=True) + if ( + cfg.seed == exp_cfg.seed + and cfg.task.use_ratio == exp_cfg.task.use_ratio + and compare_dict(exp_transition_dir, transition_cfg) + and (exp_dir / "transition").exists() + ): + dynamics.transition.load(exp_dir / "transition") + print("loaded dynamics from {}".format(exp_dir)) + return True return False - - -def setup_fake_env( - cfg: DictConfig, - agent: BaseAlgorithm, - dynamics, - reward_fn: Optional[RewardFnType], - termination_fn: Optional[TermFnType], - get_init_obs_fn: Optional[InitObsFnType], - real_replay_buffer: Optional[ReplayBuffer] = None, - logger=None, - max_episode_steps: int = 1000, - penalty_coeff: Optional[float] = 0, -): - fake_env = cast(VecFakeEnv, agent.env) - fake_env.set_up( - dynamics, - reward_fn, - termination_fn, - get_init_obs_fn=get_init_obs_fn, - real_replay_buffer=real_replay_buffer, - logger=logger, - max_episode_steps=max_episode_steps, - penalty_coeff=penalty_coeff, - ) - agent.env = VecMonitor(fake_env) - - fake_eval_env_cfg = deepcopy(cfg.algorithm.agent.env) - fake_eval_env_cfg.num_envs = cfg.task.n_eval_episodes - fake_eval_env = cast(VecFakeEnv, hydra.utils.instantiate(fake_eval_env_cfg)) - fake_eval_env.set_up( - dynamics, - reward_fn, - termination_fn, - get_init_obs_fn=get_init_obs_fn, - max_episode_steps=max_episode_steps, - penalty_coeff=penalty_coeff, - ) - fake_eval_env.seed(seed=cfg.seed) - return fake_eval_env - - -def load_offline_data(cfg: DictConfig, env, replay_buffer: ReplayBuffer): - assert hasattr(env, "get_dataset"), "env must have `get_dataset` method" - - params, dataset_type = cfg.task.env.split("___")[-2:] - data_dict = env.get_dataset("{}-{}".format(params, dataset_type)) - all_data_num = len(data_dict["observations"]) - sample_data_num = int(cfg.task.use_ratio * all_data_num) - sample_idx = np.random.permutation(all_data_num)[:sample_data_num] - - replay_buffer.extend( - data_dict["observations"][sample_idx], - data_dict["next_observations"][sample_idx], - data_dict["actions"][sample_idx], - data_dict["rewards"][sample_idx], - data_dict["terminals"][sample_idx].astype(bool) | data_dict["timeouts"][sample_idx].astype(bool), - [{}] * sample_data_num, - ) diff --git a/cmrl/diagnostics/base_diagnostic.py b/cmrl/diagnostics/base_diagnostic.py new file mode 100644 index 0000000..3039cd4 --- /dev/null +++ b/cmrl/diagnostics/base_diagnostic.py @@ -0,0 +1,11 @@ +from typing import Union +import pathlib + + +class BaseDiagnostic: + def __init__(self, exp_dir: Union[str, pathlib.Path]): + if isinstance(exp_dir, str): + self.exp_dir = pathlib.Path(exp_dir) + else: + self.exp_dir = exp_dir + pass diff --git a/cmrl/diagnostics/eval_model_on_dataset.py b/cmrl/diagnostics/eval_model_on_dataset.py index a0166c2..faf379a 100644 --- a/cmrl/diagnostics/eval_model_on_dataset.py +++ b/cmrl/diagnostics/eval_model_on_dataset.py @@ -8,9 +8,10 @@ import matplotlib.pylab as plt import numpy as np -import cmrl.util.creator -import cmrl.util.env -from cmrl.util.config import load_hydra_cfg +import cmrl.utils.creator +import cmrl.utils.env +from cmrl.utils.config import load_hydra_cfg +from cmrl.utils.transition_iterator import TransitionIterator class DatasetEvaluator: @@ -24,7 +25,7 @@ def __init__(self, model_dir: str, dataset: str, batch_size: int = 4096, device= self.dynamics = cmrl.util.creator.create_dynamics( self.cfg.dynamics, - self.env.observation_space.shape, + self.env.state_space.shape, self.env.action_space.shape, load_dir=self.model_path, load_device=device, @@ -32,7 +33,7 @@ def __init__(self, model_dir: str, dataset: str, batch_size: int = 4096, device= self.replay_buffer = cmrl.util.creator.create_replay_buffer( self.cfg, - self.env.observation_space.shape, + self.env.state_space.shape, self.env.action_space.shape, ) @@ -62,7 +63,7 @@ def __init__(self, model_dir: str, dataset: str, batch_size: int = 4096, device= def plot_dataset_results( self, - dataset: cmrl.util.TransitionIterator, + dataset: TransitionIterator, hist_bins: int = 20, hist_log: bool = True, ): diff --git a/cmrl/diagnostics/eval_model_on_space.py b/cmrl/diagnostics/eval_model_on_space.py index 645b10e..4e17e53 100644 --- a/cmrl/diagnostics/eval_model_on_space.py +++ b/cmrl/diagnostics/eval_model_on_space.py @@ -11,22 +11,16 @@ import numpy as np from matplotlib.widgets import Button, RadioButtons, Slider -import cmrl.util.creator -import cmrl.util.env -from cmrl.util.config import load_hydra_cfg +import cmrl.utils.creator +import cmrl.utils.env +from cmrl.utils.config import load_hydra_cfg +from cmrl.utils.creator import create_dynamics, create_agent +from cmrl.models.fake_env import get_penalty -mpl.use("Qt5Agg") +mpl.use("TKAgg") SIN_COS_BINDINGS = {"BoundaryInvertedPendulumSwingUp-v0": [1]} -def calculate_penalty(ensemble_mean): - avg_ensemble_mean = np.mean(ensemble_mean, axis=0) # average predictions over models - diffs = ensemble_mean - avg_ensemble_mean - dists = np.linalg.norm(diffs, axis=2) # distance in obs space - penalty = np.max(dists, axis=0) # max distances over models - return penalty - - def set_ylim(y_min, y_max, ax): if y_max - y_min > 0.1: obs_y_lim = [y_min - 0.05, y_max + 0.05] @@ -73,25 +67,30 @@ def __init__( self.cfg = load_hydra_cfg(self.model_path) self.cfg.device = device - self.env, *_ = cmrl.util.env.make_env(self.cfg) + self.env, *_ = cmrl.utils.env.make_env(self.cfg) if penalty_coeff is None: self.penalty_coeff = self.cfg.task.penalty_coeff else: self.penalty_coeff = penalty_coeff - self.dynamics = cmrl.util.creator.create_dynamics( - self.cfg.dynamics, - self.env.observation_space.shape, - self.env.action_space.shape, - load_dir=self.model_path, - load_device=device, + self.dynamics = create_dynamics( + self.cfg, + self.env.state_space, + self.env.action_space, ) + if not self.cfg.transition.discovery: + self.dynamics.transition.set_oracle_graph(self.env.get_transition_graph()) + if self.cfg.reward_mech.learn and not self.cfg.reward_mech.discovery: + self.dynamics.reward_mech.set_oracle_graph(self.env.get_reward_mech_graph()) + if self.cfg.termination_mech.learn and not self.cfg.termination_mech.discovery: + self.dynamics.termination_mech.set_oracle_graph(self.env.get_termination_mech_graph()) + self.dynamics.transition.load(self.model_path / "transition") self.bindings = [] self.obs_range, self.action_range = self.get_range() self.range = np.concatenate([self.obs_range, self.action_range], axis=0) - self.real_obs_dim_num = self.env.observation_space.shape[0] + self.real_obs_dim_num = self.env.state_space.shape[0] self.compact_obs_dim_num, self.action_dim_num = ( self.obs_range.shape[0], self.action_range.shape[0], @@ -213,20 +212,19 @@ def slider_changed(value, dim=dim): self.draw_button.on_clicked(self.draw) def get_range(self, dataset_type="SAC-expert-replay"): - universe, basic_env_name, params, origin_dataset_type = self.cfg.task.env.split("___") - data_dict = self.env.get_dataset("{}-{}".format(params, dataset_type)) + data_dict = self.env.get_dataset(dataset_type) obs_min = np.percentile(data_dict["observations"], self.range_quantile, axis=0) obs_max = np.percentile(data_dict["observations"], 100 - self.range_quantile, axis=0) action_min = np.percentile(data_dict["actions"], self.range_quantile, axis=0) action_max = np.percentile(data_dict["actions"], 100 - self.range_quantile, axis=0) obs_range, action_range = np.array(list(zip(obs_min, obs_max))), np.array(list(zip(action_min, action_max))) - if basic_env_name in SIN_COS_BINDINGS: - self.bindings = SIN_COS_BINDINGS[basic_env_name] - for idx, binding_idx in enumerate(self.bindings): - theta_idx = binding_idx - idx - obs_range = np.delete(obs_range, [binding_idx, binding_idx + 1], axis=0) - obs_range = np.insert(obs_range, theta_idx, np.array([0, 2 * np.pi]), axis=0) + # if basic_env_name in SIN_COS_BINDINGS: + # self.bindings = SIN_COS_BINDINGS[basic_env_name] + # for idx, binding_idx in enumerate(self.bindings): + # theta_idx = binding_idx - idx + # obs_range = np.delete(obs_range, [binding_idx, binding_idx + 1], axis=0) + # obs_range = np.insert(obs_range, theta_idx, np.array([0, 2 * np.pi]), axis=0) return obs_range, action_range def build_model_in(self): @@ -250,7 +248,7 @@ def build_model_in(self): real_model_in[:, dim] = np.cos(compact_model_in[:, compact_dim].copy()) else: # is an action compact_dim = dim - (self.real_obs_dim_num - self.compact_obs_dim_num) - real_model_in[:, dim] = np.cos(compact_model_in[:, compact_dim].copy()) + real_model_in[:, dim] = compact_model_in[:, compact_dim].copy() return x, real_model_in def draw(self, event): @@ -285,32 +283,29 @@ def get_model_out(self, model_in): penalized_reward = np.empty(self.plot_dot_num) for batch_idx in range(batch_num): + f, t = self.batch_size * batch_idx, self.batch_size * (batch_idx + 1) + batch_input = model_in[self.batch_size * batch_idx : self.batch_size * (batch_idx + 1)] batch_obs, batch_action = ( batch_input[:, : self.real_obs_dim_num], batch_input[:, self.real_obs_dim_num :], ) - dynamics_result = self.dynamics.query(batch_obs, batch_action, return_as_np=True) - gt_next_obs, gt_reward, gt_terminal, gt_truncated, _ = self.env.query(batch_obs, batch_action) - # predict and ground truth - batch_predict_obs = dynamics_result["batch_next_obs"]["mean"].mean(0) - batch_gt_obs = gt_next_obs + + predict_next_obs, predict_reward, terminal, info = self.dynamics.step(batch_obs, batch_action) + gt_next_obs = self.env.get_batch_next_obs(batch_obs, batch_action) + gt_reward = self.env.get_batch_reward(gt_next_obs) + if self.draw_diff: - batch_predict_obs -= batch_obs - batch_gt_obs -= batch_obs - batch_predict = batch_predict_obs[:, self.current_out_dim] - batch_ground_truth = batch_gt_obs[:, self.current_out_dim] - # reward - # batch_reward = dynamics_result["batch_reward"]["mean"].mean(0)[:, 0] - batch_reward = gt_reward - # penalized_reward - batch_penalty = calculate_penalty(dynamics_result["batch_next_obs"]["mean"]) - batch_penalized_reward = batch_reward - batch_penalty * self.penalty_coeff - - predict[self.batch_size * batch_idx : self.batch_size * (batch_idx + 1)] = batch_predict - ground_truth[self.batch_size * batch_idx : self.batch_size * (batch_idx + 1)] = batch_ground_truth - reward[self.batch_size * batch_idx : self.batch_size * (batch_idx + 1)] = batch_reward - penalized_reward[self.batch_size * batch_idx : self.batch_size * (batch_idx + 1)] = batch_penalized_reward + predict_next_obs -= batch_obs + gt_next_obs -= batch_obs + + batch_penalty = get_penalty(info["origin-next_obs"]) + + predict[f:t] = predict_next_obs[:, self.current_out_dim] + ground_truth[f:t] = gt_next_obs[:, self.current_out_dim] + reward[f:t] = gt_reward[:, 0] + penalized_reward[f:t] = gt_reward[:, 0] - batch_penalty * self.penalty_coeff + return predict, ground_truth, reward, penalized_reward def run(self): @@ -326,7 +321,7 @@ def run(self): np.linspace(0, 1, 100), color="black", lw=2, - label="gt", + label="ground truth", ) (self.reward_line,) = self.reward_ax.plot( np.linspace(0, 1, 100), @@ -358,10 +353,10 @@ def run(self): parser = argparse.ArgumentParser() parser.add_argument("model_dir", type=str, default=None) parser.add_argument("--penalty_coeff", type=float, default=None) - parser.add_argument("--draw_diff", action="store_true") + parser.add_argument("--not_draw_diff", action="store_true", default=False) args = parser.parse_args() - evaluator = DatasetEvaluator(args.model_dir, penalty_coeff=args.penalty_coeff, draw_diff=args.draw_diff) + evaluator = DatasetEvaluator(args.model_dir, penalty_coeff=args.penalty_coeff, draw_diff=not args.not_draw_diff) mpl.rcParams["figure.facecolor"] = "white" mpl.rcParams["font.size"] = 14 diff --git a/cmrl/diagnostics/run_trained_model.py b/cmrl/diagnostics/run_trained_model.py index 379c842..0871d43 100644 --- a/cmrl/diagnostics/run_trained_model.py +++ b/cmrl/diagnostics/run_trained_model.py @@ -17,9 +17,9 @@ import cmrl import cmrl.agent import cmrl.models -import cmrl.util.creator -from cmrl.util.config import load_hydra_cfg -from cmrl.util.env import make_env +import cmrl.utils.creator +from cmrl.utils.config import load_hydra_cfg +from cmrl.utils.env import make_env class Runner: @@ -34,7 +34,7 @@ def __init__(self, model_dir: str, device: str = "cuda:0", render: bool = False) self.dynamics = cmrl.util.creator.create_dynamics( self.cfg.dynamics, - self.env.observation_space.shape, + self.env.state_space.shape, self.env.action_space.shape, load_dir=self.model_path, load_device=device, @@ -61,7 +61,7 @@ def __init__(self, model_dir: str, device: str = "cuda:0", render: bool = False) self.agent = agent_class.load(self.model_path / "best_model") def run(self): - # from emei.util import random_policy_test + # from emei.utils import random_policy_test obs = self.fake_eval_env.reset() if self.render: self.fake_eval_env.render() diff --git a/cmrl/diagnostics/run_trained_policy.py b/cmrl/diagnostics/run_trained_policy.py index 2e8f8a8..94f2008 100644 --- a/cmrl/diagnostics/run_trained_policy.py +++ b/cmrl/diagnostics/run_trained_policy.py @@ -10,8 +10,8 @@ import cmrl import cmrl.agent import cmrl.models -from cmrl.util.config import load_hydra_cfg -from cmrl.util.env import make_env +from cmrl.utils.config import load_hydra_cfg +from cmrl.utils.env import make_env class Runner: @@ -26,7 +26,7 @@ def __init__(self, agent_dir: str, type: str = "best", device="cuda:0"): self.agent = agent_class.load(self.agent_dir / "best_model") def run(self): - # from emei.util import random_policy_test + # from emei.utils import random_policy_test obs = self.env.reset() self.env.render() total_reward = 0 diff --git a/cmrl/examples/conf/algorithm/mbpo.yaml b/cmrl/examples/conf/algorithm/mbpo.yaml index 55fab6d..c53e292 100644 --- a/cmrl/examples/conf/algorithm/mbpo.yaml +++ b/cmrl/examples/conf/algorithm/mbpo.yaml @@ -1,32 +1,22 @@ name: "mbpo" +algo: + _partial_: true + _target_: cmrl.algorithms.MBPO + freq_train_model: ${task.freq_train_model} -real_data_ratio: 0.0 -sac_samples_action: true -initial_exploration_steps: 5000 -random_initial_explore: false num_eval_episodes: 5 -# -------------------------------------------- -# SAC Agent configuration -# -------------------------------------------- +initial_exploration_steps: 1000 + +num_envs: 1000 +deterministic: false agent: + _partial_: true _target_: stable_baselines3.sac.SAC policy: "MlpPolicy" - env: - _target_: cmrl.models.fake_env.VecFakeEnv - num_envs: 1000 - action_space: - _target_: gym.spaces.Box - low: ??? - high: ??? - shape: ??? - observation_space: - _target_: gym.spaces.Box - low: ??? - high: ??? - shape: ??? + env: ??? learning_starts: 0 batch_size: 256 tau: 0.005 diff --git a/cmrl/examples/conf/algorithm/mopo.yaml b/cmrl/examples/conf/algorithm/mopo.yaml index 9daa227..9ff83d8 100644 --- a/cmrl/examples/conf/algorithm/mopo.yaml +++ b/cmrl/examples/conf/algorithm/mopo.yaml @@ -1,33 +1,21 @@ name: "mopo" -freq_train_model: ${task.freq_train_model} -real_data_ratio: 0.0 - -sac_samples_action: true -num_eval_episodes: 5 +algo: + _partial_: true + _target_: cmrl.algorithms.MOPO dataset_size: 1000000 penalty_coeff: ${task.penalty_coeff} -# -------------------------------------------- -# SAC Agent configuration -# -------------------------------------------- +branch_rollout_length: 5 + +num_envs: 100 +deterministic: false agent: + _partial_: true _target_: stable_baselines3.sac.SAC policy: "MlpPolicy" - env: - _target_: cmrl.models.fake_env.VecFakeEnv - num_envs: 1000 - action_space: - _target_: gym.spaces.Box - low: ??? - high: ??? - shape: ??? - observation_space: - _target_: gym.spaces.Box - low: ??? - high: ??? - shape: ??? + env: ??? learning_starts: 0 batch_size: 256 tau: 0.005 diff --git a/cmrl/examples/conf/algorithm/off_dyna.yaml b/cmrl/examples/conf/algorithm/off_dyna.yaml index d22c034..c4d0bec 100644 --- a/cmrl/examples/conf/algorithm/off_dyna.yaml +++ b/cmrl/examples/conf/algorithm/off_dyna.yaml @@ -1,33 +1,19 @@ name: "off_dyna" -freq_train_model: ${task.freq_train_model} -real_data_ratio: 0.0 - -sac_samples_action: true -num_eval_episodes: 5 +algo: + _partial_: true + _target_: cmrl.algorithms.OfflineDyna dataset_size: 1000000 penalty_coeff: ${task.penalty_coeff} -# -------------------------------------------- -# SAC Agent configuration -# -------------------------------------------- +num_envs: 8 +deterministic: false agent: + _partial_: true _target_: stable_baselines3.sac.SAC policy: "MlpPolicy" - env: - _target_: cmrl.models.fake_env.VecFakeEnv - num_envs: 16 - action_space: - _target_: gym.spaces.Box - low: ??? - high: ??? - shape: ??? - observation_space: - _target_: gym.spaces.Box - low: ??? - high: ??? - shape: ??? + env: ??? learning_starts: 0 batch_size: 256 tau: 0.005 diff --git a/cmrl/examples/conf/algorithm/on_dyna.yaml b/cmrl/examples/conf/algorithm/on_dyna.yaml index d3ae6e6..e66c5b6 100644 --- a/cmrl/examples/conf/algorithm/on_dyna.yaml +++ b/cmrl/examples/conf/algorithm/on_dyna.yaml @@ -1,32 +1,22 @@ name: "on_dyna" +algo: + _partial_: true + _target_: cmrl.algorithms.OnlineDyna + freq_train_model: ${task.freq_train_model} -real_data_ratio: 0.0 -sac_samples_action: true num_eval_episodes: 5 initial_exploration_steps: 1000 -# -------------------------------------------- -# SAC Agent configuration -# -------------------------------------------- +num_envs: 16 +deterministic: false agent: + _partial_: true _target_: stable_baselines3.sac.SAC policy: "MlpPolicy" - env: - _target_: cmrl.models.fake_env.VecFakeEnv - num_envs: 16 - action_space: - _target_: gym.spaces.Box - low: ??? - high: ??? - shape: ??? - observation_space: - _target_: gym.spaces.Box - low: ??? - high: ??? - shape: ??? + env: ??? learning_starts: 0 batch_size: 256 tau: 0.005 diff --git a/cmrl/examples/conf/dynamics/constraint_based_dynamics.yaml b/cmrl/examples/conf/dynamics/constraint_based_dynamics.yaml deleted file mode 100644 index 2fefed1..0000000 --- a/cmrl/examples/conf/dynamics/constraint_based_dynamics.yaml +++ /dev/null @@ -1,63 +0,0 @@ -name: constraint_based_dynamics - -multi_step: ${task.multi_step} - -transition: - _target_: cmrl.models.transition.ExternalMaskEnsembleGaussianTransition - # transition info - obs_size: ??? - action_size: ??? - deterministic: false - # algorithm parameters - ensemble_num: ${task.ensemble_num} - elite_num: ${task.elite_num} - residual: true - learn_logvar_bounds: false # so far this works better - # network parameters - num_layers: 4 - hid_size: 200 - activation_fn_cfg: - _target_: torch.nn.SiLU - # others - device: ${device} - -learned_reward: ${task.learning_reward} -reward_mech: - _target_: cmrl.models.BaseRewardMech - # transition info - obs_size: ??? - action_size: ??? - deterministic: false - # algorithm parameters - learn_logvar_bounds: false # so far this works better - ensemble_num: ${task.ensemble_num} - elite_num: ${task.elite_num} - # network parameters - num_layers: 4 - hid_size: 200 - activation_fn_cfg: - _target_: torch.nn.SiLU - # others - device: ${device} - -learned_termination: ${task.learning_terminal} -termination_mech: - _target_: cmrl.models.BaseTerminationMech - # transition info - obs_size: ??? - action_size: ??? - deterministic: false - -optim_lr: ${task.optim_lr} -weight_decay: ${task.weight_decay} -patience: ${task.patience} -batch_size: ${task.batch_size} -use_ratio: ${task.use_ratio} -validation_ratio: ${task.validation_ratio} -shuffle_each_epoch: ${task.shuffle_each_epoch} -bootstrap_permutes: ${task.bootstrap_permutes} -longest_epoch: ${task.longest_epoch} -improvement_threshold: ${task.improvement_threshold} - -normalize: true -normalize_double_precision: true diff --git a/cmrl/examples/conf/dynamics/plain_dynamics.yaml b/cmrl/examples/conf/dynamics/plain_dynamics.yaml deleted file mode 100644 index 74914c2..0000000 --- a/cmrl/examples/conf/dynamics/plain_dynamics.yaml +++ /dev/null @@ -1,63 +0,0 @@ -name: plain_dynamics - -multi_step: ${task.multi_step} - -transition: - _target_: cmrl.models.transition.PlainEnsembleGaussianTransition - # transition info - obs_size: ??? - action_size: ??? - deterministic: false - # algorithm parameters - ensemble_num: ${task.ensemble_num} - elite_num: ${task.elite_num} - residual: true - learn_logvar_bounds: false # so far this works better - # network parameters - num_layers: 4 - hid_size: 200 - activation_fn_cfg: - _target_: torch.nn.SiLU - # others - device: ${device} - -learned_reward: ${task.learning_reward} -reward_mech: - _target_: cmrl.models.BaseRewardMech - # transition info - obs_size: ??? - action_size: ??? - deterministic: fase - # algorithm parameters - learn_logvar_bounds: false # so far this works better - ensemble_num: ${task.ensemble_num} - elite_num: ${task.elite_num} - # network parameters - num_layers: 4 - hid_size: 200 - activation_fn_cfg: - _target_: torch.nn.SiLU - # others - device: ${device} - -learned_termination: ${task.learning_terminal} -termination_mech: - _target_: cmrl.models.BaseTerminationMech - # transition info - obs_size: ??? - action_size: ??? - deterministic: false - -optim_lr: ${task.optim_lr} -weight_decay: ${task.weight_decay} -patience: ${task.patience} -batch_size: ${task.batch_size} -use_ratio: ${task.use_ratio} -validation_ratio: ${task.validation_ratio} -shuffle_each_epoch: ${task.shuffle_each_epoch} -bootstrap_permutes: ${task.bootstrap_permutes} -longest_epoch: ${task.longest_epoch} -improvement_threshold: ${task.improvement_threshold} - -normalize: true -normalize_double_precision: true diff --git a/cmrl/examples/conf/main.yaml b/cmrl/examples/conf/main.yaml index 65eda8e..39abbe5 100644 --- a/cmrl/examples/conf/main.yaml +++ b/cmrl/examples/conf/main.yaml @@ -1,20 +1,23 @@ defaults: - algorithm: off_dyna - - dynamics: constraint_based_dynamics - - task: BIPS + - task: continuous_cart_pole_swingup + - transition: oracle + - reward_mech: oracle + - termination_mech: oracle - _self_ seed: 0 -device: "cuda:0" +device: "cpu" exp_name: default wandb: false +verbose: false root_dir: "./exp" hydra: run: - dir: ${root_dir}/${exp_name}/${task.env}/${dynamics.name}/${now:%Y.%m.%d}/${now:%H.%M.%S} + dir: ${root_dir}/${exp_name}/${task.env_id}/${to_str:${task.params}}/${task.dataset}/${now:%Y.%m.%d.%H%M%S} sweep: - dir: ${root_dir}/${exp_name}/${task.env}/${dynamics.name}/${now:%Y.%m.%d}/${now:%H.%M.%S} + dir: ${root_dir}/${exp_name}/${task.env_id}/${to_str:${task.params}}/${task.dataset}/${now:%Y.%m.%d.%H%M%S} job: chdir: true diff --git a/cmrl/examples/conf/reward_mech/oracle.yaml b/cmrl/examples/conf/reward_mech/oracle.yaml new file mode 100644 index 0000000..ffe3f12 --- /dev/null +++ b/cmrl/examples/conf/reward_mech/oracle.yaml @@ -0,0 +1,62 @@ +name: "oracle_reward_mech" +learn: false +discovery: false + +encoder_cfg: + _partial_: true + _recursive_: false + _target_: cmrl.models.networks.VariableEncoder + output_dim: 100 + hidden_dims: [ 100 ] + bias: true + activation_fn_cfg: + _target_: torch.nn.SiLU + +decoder_cfg: + _partial_: true + _recursive_: false + _target_: cmrl.models.networks.VariableDecoder + input_dim: 100 + hidden_dims: [ 100 ] + bias: true + activation_fn_cfg: + _target_: torch.nn.SiLU + +network_cfg: + _partial_: true + _recursive_: false + _target_: cmrl.models.networks.ParallelMLP + hidden_dims: [ 200, 200 ] + bias: true + activation_fn_cfg: + _target_: torch.nn.SiLU + +optimizer_cfg: + _partial_: true + _target_: torch.optim.Adam + lr: 1e-4 + weight_decay: 1e-5 + eps: 1e-8 + +mech: + _partial_: true + _recursive_: false + _target_: cmrl.models.causal_mech.OracleMech + # base causal-mech params + name: reward_mech + input_variables: ??? + output_variables: ??? + ensemble_num: 7 + elite_num: 5 + # cfgs + network_cfg: ${transition.network_cfg} + encoder_cfg: ${transition.encoder_cfg} + decoder_cfg: ${transition.decoder_cfg} + optimizer_cfg: ${transition.optimizer_cfg} + # forward method + residual: true + multi_step: "none" + # logger + logger: ??? + # others + device: ${device} diff --git a/cmrl/examples/conf/task/BI2PB.yaml b/cmrl/examples/conf/task/BI2PB.yaml index 8d13511..8684ca1 100644 --- a/cmrl/examples/conf/task/BI2PB.yaml +++ b/cmrl/examples/conf/task/BI2PB.yaml @@ -1,7 +1,12 @@ -env: "emei___BoundaryInvertedDoublePendulumBalancing-v0___freq_rate=${task.freq_rate}&time_step=${task.time_step}___${task.dataset}" +# env parameters +env_id: "BoundaryInvertedDoublePendulumBalancing-v0" + +params: + freq_rate: 1 + real_time_scale: 0.02 + integrator: "euler" + dataset: "SAC-expert-replay" -freq_rate: 1 -time_step: 0.02 # basic RL params num_steps: 3000000 diff --git a/cmrl/examples/conf/task/BI2PS.yaml b/cmrl/examples/conf/task/BI2PS.yaml index 06918ab..c5feda6 100644 --- a/cmrl/examples/conf/task/BI2PS.yaml +++ b/cmrl/examples/conf/task/BI2PS.yaml @@ -1,5 +1,12 @@ -env: "emei___BoundaryInvertedDoublePendulumSwingUp-v0___freq_rate=1&time_step=0.02___${task.dataset}" -dataset: "expert-replay" +# env parameters +env_id: "BoundaryInvertedDoublePendulumSwingUp-v0" + +params: + freq_rate: 1 + real_time_scale: 0.02 + integrator: "euler" + +dataset: "SAC-expert-replay" # basic RL params num_steps: 1000000 diff --git a/cmrl/examples/conf/task/BIPB.yaml b/cmrl/examples/conf/task/BIPB.yaml index a0536e4..321c5d2 100644 --- a/cmrl/examples/conf/task/BIPB.yaml +++ b/cmrl/examples/conf/task/BIPB.yaml @@ -1,7 +1,12 @@ -env: "emei___BoundaryInvertedPendulumBalancing-v0___freq_rate=${task.freq_rate}&time_step=${task.time_step}___${task.dataset}" +# env parameters +env_id: "BoundaryInvertedPendulumBalancing-v0" + +params: + freq_rate: 1 + real_time_scale: 0.02 + integrator: "euler" + dataset: "SAC-expert-replay" -freq_rate: 1 -time_step: 0.02 # basic RL params num_steps: 2000000 diff --git a/cmrl/examples/conf/task/BIPS.yaml b/cmrl/examples/conf/task/BIPS.yaml index c9c0ea2..13de2af 100644 --- a/cmrl/examples/conf/task/BIPS.yaml +++ b/cmrl/examples/conf/task/BIPS.yaml @@ -1,10 +1,15 @@ -env: "emei___BoundaryInvertedPendulumSwingUp-v0___freq_rate=${task.freq_rate}&time_step=${task.time_step}___${task.dataset}" +# env parameters +env_id: "BoundaryInvertedPendulumSwingUp-v0" + +params: + freq_rate: 1 + real_time_scale: 0.02 + integrator: "euler" + dataset: "SAC-expert-replay" -freq_rate: 1 -time_step: 0.02 # basic RL params -num_steps: 300000 +num_steps: 10000000 online_num_steps: 10000 epoch_length: 10000 n_eval_episodes: 8 @@ -15,7 +20,7 @@ learning_reward: false learning_terminal: false ensemble_num: 7 elite_num: 5 -multi_step: "none" +multi_step: "forward_euler_5" # conditional mutual information test(causal discovery) oracle: true @@ -26,36 +31,13 @@ update_causal_mask_ratio: 0.25 discovery_schedule: [ 1, 30, 250, 250 ] # offline -penalty_coeff: 0.5 +penalty_coeff: 0.2 use_ratio: 1 # dyna freq_train_model: 100 + # model learning patience: 20 -optim_lr: 0.0001 -weight_decay: 0.00001 -batch_size: 256 -validation_ratio: 0.2 -shuffle_each_epoch: true -bootstrap_permutes: false longest_epoch: -1 improvement_threshold: 0.01 -# model using -effective_model_rollouts_per_step: 50 -rollout_schedule: [ 1, 15, 1, 1 ] -num_sac_updates_per_step: 1 -sac_updates_every_steps: 1 -num_epochs_to_retain_sac_buffer: 1 - -# SAC -sac_gamma: 0.99 -sac_tau: 0.005 -sac_alpha: 0.2 -sac_policy: "Gaussian" -sac_target_update_interval: 1 -sac_automatic_entropy_tuning: true -sac_hidden_size: 256 -sac_lr: 0.0003 -sac_batch_size: 256 -sac_target_entropy: -1 diff --git a/cmrl/examples/conf/task/continuous_cart_pole_swingup.yaml b/cmrl/examples/conf/task/continuous_cart_pole_swingup.yaml new file mode 100644 index 0000000..bdf1483 --- /dev/null +++ b/cmrl/examples/conf/task/continuous_cart_pole_swingup.yaml @@ -0,0 +1,29 @@ +# env parameters +env_id: "ContinuousCartPoleSwingUp-v0" + +params: + freq_rate: 1 + real_time_scale: 0.02 + integrator: "euler" + gravity: 9.8 + length: 0.5 + force_mag: 10.0 + +dataset: "SAC-expert-replay" + +extra_variable_info: + Radian: + - "obs_1" + +# basic RL params +num_steps: 3000000 +online_num_steps: 10000 +n_eval_episodes: 5 +eval_freq: 10000 + +# offline +penalty_coeff: 1 +use_ratio: 1 + +# dyna +freq_train_model: 100 diff --git a/cmrl/examples/conf/task/hopper.yaml b/cmrl/examples/conf/task/hopper.yaml new file mode 100644 index 0000000..9a851ce --- /dev/null +++ b/cmrl/examples/conf/task/hopper.yaml @@ -0,0 +1,43 @@ +# env parameters +env_id: "HopperRunning-v0" + +params: + freq_rate: 1 + real_time_scale: 0.01 + integrator: "euler" + +dataset: "SAC-medium" + +# basic RL params +num_steps: 10000000 +online_num_steps: 10000 +epoch_length: 10000 +n_eval_episodes: 8 +eval_freq: 100 + +# dynamics +learning_reward: false +learning_terminal: false +ensemble_num: 7 +elite_num: 5 +multi_step: "none" + +# conditional mutual information test(causal discovery) +oracle: true +cit_threshold: 0.02 +test_freq: 100 +# causal +update_causal_mask_ratio: 0.25 +discovery_schedule: [ 1, 30, 250, 250 ] + +# offline +penalty_coeff: 1.0 +use_ratio: 1 + +# dyna +freq_train_model: 100 + +# model learning +patience: 10 +longest_epoch: -1 +improvement_threshold: 0.01 \ No newline at end of file diff --git a/cmrl/examples/conf/task/mbpo_ant.yaml b/cmrl/examples/conf/task/mbpo_ant.yaml deleted file mode 100644 index 813cd89..0000000 --- a/cmrl/examples/conf/task/mbpo_ant.yaml +++ /dev/null @@ -1,28 +0,0 @@ -env: "ant_truncated_obs" -# term_fn is set automatically by cmrl.util.env.EnvHandler.make_env - -num_steps: 300000 -epoch_length: 1000 -num_elites: 5 -patience: 10 -model_lr: 0.0003 -model_wd: 5e-5 -model_batch_size: 256 -validation_ratio: 0.2 -freq_train_model: 250 -effective_model_rollouts_per_step: 400 -rollout_schedule: [20, 100, 1, 25] -num_sac_updates_per_step: 20 -sac_updates_every_steps: 1 -num_epochs_to_retain_sac_buffer: 1 - -sac_gamma: 0.99 -sac_tau: 0.005 -sac_alpha: 0.2 -sac_policy: "Gaussian" -sac_target_update_interval: 4 -sac_automatic_entropy_tuning: false -sac_target_entropy: -1 # ignored, since entropy tuning is false -sac_hidden_size: 1024 -sac_lr: 0.0001 -sac_batch_size: 256 diff --git a/cmrl/examples/conf/task/mbpo_boundary_inverted_double_pendulum_swing_up.yaml b/cmrl/examples/conf/task/mbpo_boundary_inverted_double_pendulum_swing_up.yaml deleted file mode 100644 index e31a548..0000000 --- a/cmrl/examples/conf/task/mbpo_boundary_inverted_double_pendulum_swing_up.yaml +++ /dev/null @@ -1,33 +0,0 @@ -env: "emei___BoundaryInvertedDoublePendulumSwingUp-v0___freq_rate=1&time_step=0.02" - -oracle: true -cit_threshold: 0.02 -test_freq: 500 - -num_steps: 800000 -epoch_length: 1000 -num_elites: 5 -patience: 5 -model_lr: 0.001 -model_wd: 0.00001 -model_batch_size: 256 -validation_ratio: 0.2 -freq_train_model: 250 -update_causal_mask_ratio: 0.25 -discovery_schedule: [1, 30, 250, 250] -effective_model_rollouts_per_step: 400 -rollout_schedule: [1, 15, 100, 100] -num_sac_updates_per_step: 10 -sac_updates_every_steps: 1 -num_epochs_to_retain_sac_buffer: 1 - -sac_gamma: 0.99 -sac_tau: 0.005 -sac_alpha: 0.2 -sac_policy: "Gaussian" -sac_target_update_interval: 1 -sac_automatic_entropy_tuning: true -sac_hidden_size: 256 -sac_lr: 0.0003 -sac_batch_size: 256 -sac_target_entropy: -1 diff --git a/cmrl/examples/conf/task/mbpo_boundary_inverted_pendulum_holding.yaml b/cmrl/examples/conf/task/mbpo_boundary_inverted_pendulum_holding.yaml deleted file mode 100644 index a8f7da6..0000000 --- a/cmrl/examples/conf/task/mbpo_boundary_inverted_pendulum_holding.yaml +++ /dev/null @@ -1,33 +0,0 @@ -env: "emei___BoundaryInvertedPendulumHolding-v0___freq_rate=1&time_step=0.02" - -oracle: true -cit_threshold: 0.02 -test_freq: 1000 - -num_steps: 20000 -epoch_length: 1000 -num_elites: 5 -patience: 5 -model_lr: 0.001 -model_wd: 0.00001 -model_batch_size: 256 -validation_ratio: 0.2 -freq_train_model: 250 -update_causal_mask_ratio: 0.25 -discovery_schedule: [1, 30, 250, 250] -effective_model_rollouts_per_step: 400 -rollout_schedule: [1, 15, 100, 100] -num_sac_updates_per_step: 10 -sac_updates_every_steps: 1 -num_epochs_to_retain_sac_buffer: 1 - -sac_gamma: 0.99 -sac_tau: 0.005 -sac_alpha: 0.2 -sac_policy: "Gaussian" -sac_target_update_interval: 1 -sac_automatic_entropy_tuning: true -sac_hidden_size: 256 -sac_lr: 0.0003 -sac_batch_size: 256 -sac_target_entropy: -1 diff --git a/cmrl/examples/conf/task/mbpo_boundary_inverted_pendulum_swing_up.yaml b/cmrl/examples/conf/task/mbpo_boundary_inverted_pendulum_swing_up.yaml deleted file mode 100644 index 6dbbc1a..0000000 --- a/cmrl/examples/conf/task/mbpo_boundary_inverted_pendulum_swing_up.yaml +++ /dev/null @@ -1,33 +0,0 @@ -env: "emei___BoundaryInvertedPendulumSwingUp-v0___freq_rate=1&time_step=0.02" - -oracle: true -cit_threshold: 0.02 -test_freq: 500 - -num_steps: 8000 -epoch_length: 1000 -num_elites: 5 -patience: 5 -model_lr: 0.001 -model_wd: 0.00001 -model_batch_size: 256 -validation_ratio: 0.2 -freq_train_model: 250 -update_causal_mask_ratio: 0.25 -discovery_schedule: [1, 30, 250, 250] -effective_model_rollouts_per_step: 400 -rollout_schedule: [1, 15, 1, 1] -num_sac_updates_per_step: 10 -sac_updates_every_steps: 1 -num_epochs_to_retain_sac_buffer: 1 - -sac_gamma: 0.99 -sac_tau: 0.005 -sac_alpha: 0.2 -sac_policy: "Gaussian" -sac_target_update_interval: 1 -sac_automatic_entropy_tuning: true -sac_hidden_size: 256 -sac_lr: 0.0003 -sac_batch_size: 256 -sac_target_entropy: -1 diff --git a/cmrl/examples/conf/task/mbpo_cartpole.yaml b/cmrl/examples/conf/task/mbpo_cartpole.yaml deleted file mode 100644 index a6016cf..0000000 --- a/cmrl/examples/conf/task/mbpo_cartpole.yaml +++ /dev/null @@ -1,28 +0,0 @@ -env: "cartpole_continuous" -trial_length: 200 - -num_steps: 5000 -epoch_length: 200 -num_elites: 5 -patience: 5 -model_lr: 0.001 -model_wd: 0.00005 -model_batch_size: 256 -validation_ratio: 0.2 -freq_train_model: 200 -effective_model_rollouts_per_step: 400 -rollout_schedule: [1, 15, 1, 1] -num_sac_updates_per_step: 20 -sac_updates_every_steps: 1 -num_epochs_to_retain_sac_buffer: 1 - -sac_gamma: 0.99 -sac_tau: 0.005 -sac_alpha: 0.2 -sac_policy: "Gaussian" -sac_target_update_interval: 4 -sac_automatic_entropy_tuning: true -sac_target_entropy: -0.05 -sac_hidden_size: 256 -sac_lr: 0.0003 -sac_batch_size: 256 diff --git a/cmrl/examples/conf/task/mbpo_halfcheetah.yaml b/cmrl/examples/conf/task/mbpo_halfcheetah.yaml deleted file mode 100644 index 3b5e3f9..0000000 --- a/cmrl/examples/conf/task/mbpo_halfcheetah.yaml +++ /dev/null @@ -1,28 +0,0 @@ -env: "gym___HalfCheetah-v2" -term_fn: "no_termination" - -num_steps: 400000 -epoch_length: 1000 -num_elites: 5 -patience: 5 -model_lr: 0.001 -model_wd: 0.00001 -model_batch_size: 256 -validation_ratio: 0.2 -freq_train_model: 250 -effective_model_rollouts_per_step: 400 -rollout_schedule: [20, 150, 1, 1] -num_sac_updates_per_step: 10 -sac_updates_every_steps: 1 -num_epochs_to_retain_sac_buffer: 1 - -sac_gamma: 0.99 -sac_tau: 0.005 -sac_alpha: 0.2 -sac_policy: "Gaussian" -sac_target_update_interval: 1 -sac_automatic_entropy_tuning: true -sac_target_entropy: -1 -sac_hidden_size: 512 -sac_lr: 0.0003 -sac_batch_size: 256 diff --git a/cmrl/examples/conf/task/mbpo_hopper.yaml b/cmrl/examples/conf/task/mbpo_hopper.yaml deleted file mode 100644 index 5ee267c..0000000 --- a/cmrl/examples/conf/task/mbpo_hopper.yaml +++ /dev/null @@ -1,28 +0,0 @@ -env: "gym___Hopper-v2" -term_fn: "hopper" - -num_steps: 500000 -epoch_length: 1000 -num_elites: 5 -patience: 5 -model_lr: 0.001 -model_wd: 0.00001 -model_batch_size: 256 -validation_ratio: 0.2 -freq_train_model: 250 -effective_model_rollouts_per_step: 400 -rollout_schedule: [20, 150, 200, 200] -num_sac_updates_per_step: 100 -sac_updates_every_steps: 1 -num_epochs_to_retain_sac_buffer: 1 - -sac_gamma: 0.99 -sac_tau: 0.005 -sac_alpha: 0.2 -sac_policy: "Gaussian" -sac_target_update_interval: 4 -sac_automatic_entropy_tuning: false -sac_target_entropy: 1 # ignored, since entropy tuning is false -sac_hidden_size: 512 -sac_lr: 0.0003 -sac_batch_size: 256 diff --git a/cmrl/examples/conf/task/mbpo_humanoid.yaml b/cmrl/examples/conf/task/mbpo_humanoid.yaml deleted file mode 100644 index 4cf0d8a..0000000 --- a/cmrl/examples/conf/task/mbpo_humanoid.yaml +++ /dev/null @@ -1,28 +0,0 @@ -env: "humanoid_truncated_obs" -# term_fn is set automatically by cmrl.util.env.EnvHandler.make_env - -num_steps: 300000 -epoch_length: 1000 -num_elites: 5 -patience: 10 -model_lr: 0.0003 -model_wd: 5e-5 -model_batch_size: 256 -validation_ratio: 0.2 -freq_train_model: 250 -effective_model_rollouts_per_step: 400 -rollout_schedule: [20, 300, 1, 25] -num_sac_updates_per_step: 20 -sac_updates_every_steps: 1 -num_epochs_to_retain_sac_buffer: 5 - -sac_gamma: 0.99 -sac_tau: 0.005 -sac_alpha: 0.2 -sac_policy: "Gaussian" -sac_target_update_interval: 4 -sac_automatic_entropy_tuning: false -sac_target_entropy: -1 # ignored, since entropy tuning is false -sac_hidden_size: 1024 -sac_lr: 0.0001 -sac_batch_size: 256 diff --git a/cmrl/examples/conf/task/mbpo_inv_pendulum.yaml b/cmrl/examples/conf/task/mbpo_inv_pendulum.yaml deleted file mode 100644 index 912d4f5..0000000 --- a/cmrl/examples/conf/task/mbpo_inv_pendulum.yaml +++ /dev/null @@ -1,29 +0,0 @@ -env: "inv_pendulum___0.25___1" - -test_freq: 500 - -num_steps: 500000 -epoch_length: 1000 -num_elites: 5 -patience: 5 -model_lr: 0.001 -model_wd: 0.00001 -model_batch_size: 256 -validation_ratio: 0.2 -freq_train_model: 250 -effective_model_rollouts_per_step: 400 -rollout_schedule: [1, 15, 10, 10] -num_sac_updates_per_step: 10 -sac_updates_every_steps: 1 -num_epochs_to_retain_sac_buffer: 1 - -sac_gamma: 0.99 -sac_tau: 0.005 -sac_alpha: 0.2 -sac_policy: "Gaussian" -sac_target_update_interval: 1 -sac_automatic_entropy_tuning: true -sac_hidden_size: 256 -sac_lr: 0.0003 -sac_batch_size: 256 -sac_target_entropy: -1 diff --git a/cmrl/examples/conf/task/mbpo_pusher.yaml b/cmrl/examples/conf/task/mbpo_pusher.yaml deleted file mode 100644 index c90c0b1..0000000 --- a/cmrl/examples/conf/task/mbpo_pusher.yaml +++ /dev/null @@ -1,29 +0,0 @@ -env: "pets_pusher" -term_fn: "no_termination" -trial_length: 150 - -num_steps: 20000 -epoch_length: 150 -num_elites: 5 -patience: 5 -model_lr: 0.001 -model_wd: 0.00005 -model_batch_size: 256 -validation_ratio: 0.2 -freq_train_model: 250 -effective_model_rollouts_per_step: 400 -rollout_schedule: [1, 15, 1, 1] -num_sac_updates_per_step: 20 -sac_updates_every_steps: 1 -num_epochs_to_retain_sac_buffer: 1 - -sac_gamma: 0.99 -sac_tau: 0.005 -sac_alpha: 0.2 -sac_policy: "Gaussian" -sac_target_update_interval: 4 -sac_automatic_entropy_tuning: true -sac_target_entropy: -0.05 -sac_hidden_size: 256 -sac_lr: 0.0003 -sac_batch_size: 256 diff --git a/cmrl/examples/conf/task/mbpo_walker.yaml b/cmrl/examples/conf/task/mbpo_walker.yaml deleted file mode 100644 index 5ef2095..0000000 --- a/cmrl/examples/conf/task/mbpo_walker.yaml +++ /dev/null @@ -1,28 +0,0 @@ -env: "gym___Walker2d-v2" -term_fn: "walker2d" - -num_steps: 300000 -epoch_length: 1000 -num_elites: 5 -patience: 10 -model_lr: 0.001 -model_wd: 0.00001 -model_batch_size: 256 -validation_ratio: 0.2 -freq_train_model: 250 -effective_model_rollouts_per_step: 400 -rollout_schedule: [20, 150, 1, 1] -num_sac_updates_per_step: 20 -sac_updates_every_steps: 1 -num_epochs_to_retain_sac_buffer: 1 - -sac_gamma: 0.99 -sac_tau: 0.005 -sac_alpha: 0.2 -sac_policy: "Gaussian" -sac_target_update_interval: 4 -sac_automatic_entropy_tuning: false -sac_target_entropy: -1 # ignored, since entropy tuning is false -sac_hidden_size: 1024 -sac_lr: 0.0001 -sac_batch_size: 256 diff --git a/cmrl/examples/conf/task/parallel_cart_pole.yaml b/cmrl/examples/conf/task/parallel_cart_pole.yaml new file mode 100644 index 0000000..35145cd --- /dev/null +++ b/cmrl/examples/conf/task/parallel_cart_pole.yaml @@ -0,0 +1,29 @@ +# env parameters +env_id: "ParallelContinuousCartPoleSwingUp-v0" + +params: + freq_rate: 1 + real_time_scale: 0.02 + integrator: "euler" + parallel_num: 3 + +dataset: "SAC-expert-replay" + +extra_variable_info: + Radian: + - "obs_1" + - "obs_5" + - "obs_9" + +# basic RL params +num_steps: 10000000 +online_num_steps: 10000 +n_eval_episodes: 5 +eval_freq: 10000 + +# offline +penalty_coeff: 1 +use_ratio: 1 + +# dyna +freq_train_model: 100 diff --git a/cmrl/examples/conf/termination_mech/oracle.yaml b/cmrl/examples/conf/termination_mech/oracle.yaml new file mode 100644 index 0000000..c6077ce --- /dev/null +++ b/cmrl/examples/conf/termination_mech/oracle.yaml @@ -0,0 +1,62 @@ +name: "oracle_termination_mech" +learn: false +discovery: false + +encoder_cfg: + _partial_: true + _recursive_: false + _target_: cmrl.models.networks.VariableEncoder + output_dim: 100 + hidden_dims: [ 100 ] + bias: true + activation_fn_cfg: + _target_: torch.nn.SiLU + +decoder_cfg: + _partial_: true + _recursive_: false + _target_: cmrl.models.networks.VariableDecoder + input_dim: 100 + hidden_dims: [ 100 ] + bias: true + activation_fn_cfg: + _target_: torch.nn.SiLU + +network_cfg: + _partial_: true + _recursive_: false + _target_: cmrl.models.networks.ParallelMLP + hidden_dims: [ 200, 200 ] + bias: true + activation_fn_cfg: + _target_: torch.nn.SiLU + +optimizer_cfg: + _partial_: true + _target_: torch.optim.Adam + lr: 1e-4 + weight_decay: 1e-5 + eps: 1e-8 + +mech: + _partial_: true + _recursive_: false + _target_: cmrl.models.causal_mech.OracleMech + # base causal-mech params + name: termination_mech + input_variables: ??? + output_variables: ??? + ensemble_num: 7 + elite_num: 5 + # cfgs + network_cfg: ${transition.network_cfg} + encoder_cfg: ${transition.encoder_cfg} + decoder_cfg: ${transition.decoder_cfg} + optimizer_cfg: ${transition.optimizer_cfg} + # forward method + residual: true + multi_step: "none" + # logger + logger: ??? + # others + device: ${device} diff --git a/cmrl/examples/conf/transition/CMI_test.yaml b/cmrl/examples/conf/transition/CMI_test.yaml new file mode 100644 index 0000000..4461b49 --- /dev/null +++ b/cmrl/examples/conf/transition/CMI_test.yaml @@ -0,0 +1,74 @@ +name: "CMI_test_transition" +learn: true + +encoder_cfg: + _partial_: true + _recursive_: false + _target_: cmrl.models.networks.VariableEncoder + output_dim: 200 + hidden_dims: [ 100 ] + bias: true + activation_fn_cfg: + _target_: torch.nn.SiLU + +decoder_cfg: + _partial_: true + _recursive_: false + _target_: cmrl.models.networks.VariableDecoder + input_dim: 100 + hidden_dims: [ 100 ] + bias: true + activation_fn_cfg: + _target_: torch.nn.SiLU + +network_cfg: + _partial_: true + _recursive_: false + _target_: cmrl.models.networks.ParallelMLP + hidden_dims: [ 100, 100 ] + bias: true + activation_fn_cfg: + _target_: torch.nn.SiLU + +optimizer_cfg: + _partial_: true + _target_: torch.optim.Adam + lr: 1e-4 + weight_decay: 1e-5 + eps: 1e-8 + +scheduler_cfg: + _partial_: true + _target_: torch.optim.lr_scheduler.StepLR + step_size: 1 + gamma: 1 + +mech: + _partial_: true + _recursive_: false + _target_: cmrl.models.causal_mech.CMITestMEch + # base causal-mech params + name: transition + input_variables: ??? + output_variables: ??? + # model learning + patience: 5 + longest_epoch: -1 + improvement_threshold: 0.01 + batch_size: 256 + # ensemble + ensemble_num: 7 + elite_num: 5 + # cfgs + network_cfg: ${transition.network_cfg} + encoder_cfg: ${transition.encoder_cfg} + decoder_cfg: ${transition.decoder_cfg} + optimizer_cfg: ${transition.optimizer_cfg} + scheduler_cfg: ${transition.scheduler_cfg} + # forward method + residual: true + encoder_reduction: "sum" + # logger + logger: ??? + # others + device: ${device} diff --git a/cmrl/examples/conf/transition/kernel_test.yaml b/cmrl/examples/conf/transition/kernel_test.yaml new file mode 100644 index 0000000..f7d750b --- /dev/null +++ b/cmrl/examples/conf/transition/kernel_test.yaml @@ -0,0 +1,77 @@ +name: "kernal_test_transition" +learn: true + +encoder_cfg: + _partial_: true + _recursive_: false + _target_: cmrl.models.networks.VariableEncoder + output_dim: 200 + hidden_dims: [ 100 ] + bias: true + activation_fn_cfg: + _target_: torch.nn.SiLU + +decoder_cfg: + _partial_: true + _recursive_: false + _target_: cmrl.models.networks.VariableDecoder + input_dim: 100 + hidden_dims: [ 100 ] + bias: true + activation_fn_cfg: + _target_: torch.nn.SiLU + +network_cfg: + _partial_: true + _recursive_: false + _target_: cmrl.models.networks.ParallelMLP + hidden_dims: [ 100, 100 ] + bias: true + activation_fn_cfg: + _target_: torch.nn.SiLU + +optimizer_cfg: + _partial_: true + _target_: torch.optim.Adam + lr: 1e-4 + weight_decay: 1e-5 + eps: 1e-8 + +scheduler_cfg: + _partial_: true + _target_: torch.optim.lr_scheduler.StepLR + step_size: 1 + gamma: 1 + +mech: + _partial_: true + _recursive_: false + _target_: cmrl.models.causal_mech.KernelTestMech + # base causal-mech params + name: transition + input_variables: ??? + output_variables: ??? + logger: ??? + # model learning + patience: 5 + longest_epoch: -1 + improvement_threshold: 0.01 + batch_size: 256 + # ensemble + ensemble_num: 7 + elite_num: 5 + # cfgs + network_cfg: ${transition.network_cfg} + encoder_cfg: ${transition.encoder_cfg} + decoder_cfg: ${transition.decoder_cfg} + optimizer_cfg: ${transition.optimizer_cfg} + scheduler_cfg: ${transition.scheduler_cfg} + # forward method + residual: true + encoder_reduction: "sum" + # others + device: ${device} + # KCI + sample_num: 256 + kci_times: 16 + not_confident_bound: 0.2 diff --git a/cmrl/examples/conf/transition/oracle.yaml b/cmrl/examples/conf/transition/oracle.yaml new file mode 100644 index 0000000..45fae3b --- /dev/null +++ b/cmrl/examples/conf/transition/oracle.yaml @@ -0,0 +1,74 @@ +name: "oracle_transition" +learn: true +oracle: "truth" + +encoder_cfg: + _partial_: true + _recursive_: false + _target_: cmrl.models.networks.VariableEncoder + output_dim: 200 + hidden_dims: [ 100 ] + bias: true + activation_fn_cfg: + _target_: torch.nn.SiLU + +decoder_cfg: + _partial_: true + _recursive_: false + _target_: cmrl.models.networks.VariableDecoder + input_dim: 100 + hidden_dims: [ 100 ] + bias: true + activation_fn_cfg: + _target_: torch.nn.SiLU + +network_cfg: + _partial_: true + _recursive_: false + _target_: cmrl.models.networks.ParallelMLP + hidden_dims: [ 100, 100] + bias: true + activation_fn_cfg: + _target_: torch.nn.SiLU + +optimizer_cfg: + _partial_: true + _target_: torch.optim.Adam + lr: 1e-4 + weight_decay: 1e-5 + eps: 1e-8 + +scheduler_cfg: + _partial_: true + _target_: torch.optim.lr_scheduler.StepLR + step_size: 1 + gamma: 0.8 + +mech: + _partial_: true + _recursive_: false + _target_: cmrl.models.causal_mech.OracleMech + # base causal-mech params + name: transition + input_variables: ??? + output_variables: ??? + # model learning + patience: 5 + longest_epoch: -1 + improvement_threshold: 0.01 + batch_size: 1024 + # ensemble + ensemble_num: 7 + elite_num: 5 + # cfgs + network_cfg: ${transition.network_cfg} + encoder_cfg: ${transition.encoder_cfg} + decoder_cfg: ${transition.decoder_cfg} + optimizer_cfg: ${transition.optimizer_cfg} + scheduler_cfg: ${transition.scheduler_cfg} + # forward method + residual: true + # logger + logger: ??? + # others + device: ${device} diff --git a/cmrl/examples/conf/transition/reinforce.yaml b/cmrl/examples/conf/transition/reinforce.yaml new file mode 100644 index 0000000..44f9102 --- /dev/null +++ b/cmrl/examples/conf/transition/reinforce.yaml @@ -0,0 +1,81 @@ +name: "reinforce_transition" +learn: true +discovery: true + +encoder_cfg: + _partial_: true + _recursive_: false + _target_: cmrl.models.networks.VariableEncoder + output_dim: 100 + hidden_dims: [ 100 ] + bias: true + activation_fn_cfg: + _target_: torch.nn.SiLU + +decoder_cfg: + _partial_: true + _recursive_: false + _target_: cmrl.models.networks.VariableDecoder + input_dim: 100 + hidden_dims: [ 100 ] + bias: true + activation_fn_cfg: + _target_: torch.nn.SiLU + +network_cfg: + _partial_: true + _recursive_: false + _target_: cmrl.models.networks.ParallelMLP + hidden_dims: [ 200, 200 ] + bias: true + activation_fn_cfg: + _target_: torch.nn.SiLU + +optimizer_cfg: + _partial_: true + _target_: torch.optim.Adam + lr: 1e-4 + weight_decay: 1e-5 + eps: 1e-8 + +graph_optimizer_cfg: + _partial_: true + _target_: torch.optim.Adam + lr: 1e-3 + weight_decay: 0.0 + eps: 1e-8 + +mech: + _partial_: true + _recursive_: false + _target_: cmrl.models.causal_mech.ReinforceCausalMech + # base causal-mech params + name: transition + input_variables: ??? + output_variables: ??? + # model learning + patience: 5 + longest_epoch: -1 + improvement_threshold: 0.01 + # ensemble + ensemble_num: 7 + elite_num: 5 + # cfgs + network_cfg: ${transition.network_cfg} + encoder_cfg: ${transition.encoder_cfg} + decoder_cfg: ${transition.decoder_cfg} + optimizer_cfg: ${transition.optimizer_cfg} + graph_optimizer_cfg: ${transition.graph_optimizer_cfg} + # graph params + concat_mask: true + graph_MC_samples: 20 + graph_max_stack: 20 + lambda_sparse: 5e-2 + # forward method + residual: true + encoder_reduction: "sum" + multi_step: "forward-euler 1" + # logger + logger: ??? + # others + device: ${device} diff --git a/cmrl/examples/main.py b/cmrl/examples/main.py index 59282b0..e64eb57 100644 --- a/cmrl/examples/main.py +++ b/cmrl/examples/main.py @@ -1,43 +1,15 @@ import hydra -import numpy as np -import torch -import wandb +from hydra.utils import instantiate from omegaconf import DictConfig, OmegaConf - -from cmrl.algorithms import mopo, mbpo, off_dyna, on_dyna -from cmrl.util.env import make_env +from emei.core import get_params_str @hydra.main(version_base=None, config_path="conf", config_name="main") def run(cfg: DictConfig): - if cfg.wandb: - wandb.init( - project="causal-mbrl", - group=cfg.exp_name, - config=OmegaConf.to_container(cfg, resolve=True), - sync_tensorboard=True, - ) - - env, term_fn, reward_fn, init_obs_fn = make_env(cfg) - test_env, *_ = make_env(cfg) - np.random.seed(cfg.seed) - torch.manual_seed(cfg.seed) - - if cfg.algorithm.name == "on_dyna": - test_env, *_ = make_env(cfg) - return on_dyna.train(env, test_env, term_fn, reward_fn, init_obs_fn, cfg) - elif cfg.algorithm.name == "mopo": - test_env, *_ = make_env(cfg) - return mopo.train(env, test_env, term_fn, reward_fn, init_obs_fn, cfg) - elif cfg.algorithm.name == "off_dyna": - test_env, *_ = make_env(cfg) - return off_dyna.train(env, test_env, term_fn, reward_fn, init_obs_fn, cfg) - elif cfg.algorithm.name == "mbpo": - test_env, *_ = make_env(cfg) - return mbpo.train(env, test_env, term_fn, reward_fn, init_obs_fn, cfg) - else: - raise NotImplementedError + algo = instantiate(cfg.algorithm.algo)(cfg=cfg) + algo.learn() if __name__ == "__main__": + OmegaConf.register_new_resolver("to_str", get_params_str) run() diff --git a/cmrl/models/causal_discovery/CMI_test.py b/cmrl/models/causal_discovery/CMI_test.py deleted file mode 100644 index 34a6668..0000000 --- a/cmrl/models/causal_discovery/CMI_test.py +++ /dev/null @@ -1,130 +0,0 @@ -from typing import Dict, Optional, Sequence, Tuple, Union - -import hydra -import omegaconf -import torch -from torch import nn as nn -from torch.nn import functional as F - -from cmrl.models.util import gaussian_nll -from cmrl.models.layers import ParallelEnsembleLinearLayer, truncated_normal_init -from cmrl.models.networks.mlp import EnsembleMLP -from cmrl.models.util import to_tensor - - -class TransitionConditionalMutualInformationTest(EnsembleMLP): - _MODEL_FILENAME = "conditional_mutual_information_test.pth" - - def __init__( - self, - # transition info - obs_size: int, - action_size: int, - # algorithm parameters - ensemble_num: int = 7, - elite_num: int = 5, - residual: bool = True, - learn_logvar_bounds: bool = False, - # network parameters - num_layers: int = 4, - hid_size: int = 200, - activation_fn_cfg: Optional[Union[Dict, omegaconf.DictConfig]] = None, - # others - device: Union[str, torch.device] = "cpu", - ): - super().__init__( - ensemble_num=ensemble_num, - elite_num=elite_num, - device=device, - ) - self.obs_size = obs_size - self.action_size = action_size - self.residual = residual - self.learn_logvar_bounds = learn_logvar_bounds - - self.num_layers = num_layers - self.hid_size = hid_size - - self.parallel_num = self.obs_size + self.action_size + 1 - - self._input_mask = 1 - torch.eye(self.parallel_num, self.obs_size + self.action_size).to(self.device) - - def create_activation(): - if activation_fn_cfg is None: - return nn.ReLU() - else: - return hydra.utils.instantiate(activation_fn_cfg) - - hidden_layers = [ - nn.Sequential( - self.create_linear_layer(obs_size + action_size, hid_size), - create_activation(), - ) - ] - for i in range(num_layers - 1): - hidden_layers.append( - nn.Sequential( - self.create_linear_layer(hid_size, hid_size), - create_activation(), - ) - ) - self.hidden_layers = nn.Sequential(*hidden_layers) - - self.mean_and_logvar = self.create_linear_layer(hid_size, 2 * self.obs_size) - self.min_logvar = nn.Parameter( - -10 * torch.ones(self.parallel_num, 1, 1, self.obs_size), requires_grad=learn_logvar_bounds - ) - self.max_logvar = nn.Parameter( - 0.5 * torch.ones(self.parallel_num, 1, 1, self.obs_size), requires_grad=learn_logvar_bounds - ) - - self.apply(truncated_normal_init) - self.to(self.device) - - def create_linear_layer(self, l_in, l_out): - return ParallelEnsembleLinearLayer(l_in, l_out, parallel_num=self.parallel_num, ensemble_num=self.ensemble_num) - - @property - def input_mask(self): - return self._input_mask - - def mask_input(self, x: torch.Tensor) -> torch.Tensor: - assert x.ndim == 4 - assert self._input_mask.ndim == 2 - input_mask = self._input_mask[:, None, None, :] - return x * input_mask - - def forward( - self, - batch_obs: torch.Tensor, # shape: (parallel_num, )ensemble_num, batch_size, obs_size - batch_action: torch.Tensor, # shape: ensemble_num, batch_size, action_size - ) -> Tuple[torch.Tensor, Optional[torch.Tensor]]: - assert len(batch_action.shape) == 3 and batch_action.shape[-1] == self.action_size - - batch_action = batch_action.repeat((self.parallel_num, 1, 1, 1)) - if len(batch_obs.shape) == 3: # non-repeat or first repeat - batch_obs = batch_obs.repeat((self.parallel_num, 1, 1, 1)) - - batch_input = torch.concat([batch_obs, batch_action], dim=-1) - - masked_input = self.mask_input(batch_input) - hidden = self.hidden_layers(masked_input) - mean_and_logvar = self.mean_and_logvar(hidden) - - mean = mean_and_logvar[..., : self.obs_size] - logvar = mean_and_logvar[..., self.obs_size :] - logvar = self.max_logvar - F.softplus(self.max_logvar - logvar) - logvar = self.min_logvar + F.softplus(logvar - self.min_logvar) - - if self.residual: - mean += batch_obs - - return mean, logvar - - def get_nll_loss(self, model_in: Dict[(str, torch.Tensor)], target: torch.Tensor) -> torch.Tensor: - pred_mean, pred_logvar = self.forward(**model_in) - target = target.repeat((self.parallel_num, 1, 1, 1)) - - nll_loss = gaussian_nll(pred_mean, pred_logvar, target, reduce=False) - nll_loss += 0.01 * (self.max_logvar.sum() - self.min_logvar.sum()) - return nll_loss diff --git a/cmrl/models/causal_mech/CMI_test.py b/cmrl/models/causal_mech/CMI_test.py new file mode 100644 index 0000000..4cadde4 --- /dev/null +++ b/cmrl/models/causal_mech/CMI_test.py @@ -0,0 +1,274 @@ +from typing import Optional, List, Dict, Union, MutableMapping +import pathlib +from functools import partial +from itertools import count + +import torch +import numpy as np +from torch.utils.data import DataLoader +from omegaconf import DictConfig +from hydra.utils import instantiate +from stable_baselines3.common.logger import Logger + +from cmrl.utils.variables import Variable +from cmrl.models.causal_mech.base import EnsembleNeuralMech +from cmrl.models.graphs.binary_graph import BinaryGraph +from cmrl.models.causal_mech.util import variable_loss_func, train_func, eval_func + + +class CMITestMech(EnsembleNeuralMech): + def __init__( + self, + name: str, + input_variables: List[Variable], + output_variables: List[Variable], + logger: Optional[Logger] = None, + # model learning + longest_epoch: int = -1, + improvement_threshold: float = 0.01, + patience: int = 5, + batch_size: int = 256, + # ensemble + ensemble_num: int = 7, + elite_num: int = 5, + # cfgs + network_cfg: Optional[DictConfig] = None, + encoder_cfg: Optional[DictConfig] = None, + decoder_cfg: Optional[DictConfig] = None, + optimizer_cfg: Optional[DictConfig] = None, + # forward method + residual: bool = True, + encoder_reduction: str = "sum", + # others + device: Union[str, torch.device] = "cpu", + ): + EnsembleNeuralMech.__init__( + self, + name=name, + input_variables=input_variables, + output_variables=output_variables, + logger=logger, + longest_epoch=longest_epoch, + improvement_threshold=improvement_threshold, + patience=patience, + batch_size=batch_size, + ensemble_num=ensemble_num, + elite_num=elite_num, + network_cfg=network_cfg, + encoder_cfg=encoder_cfg, + decoder_cfg=decoder_cfg, + optimizer_cfg=optimizer_cfg, + residual=residual, + encoder_reduction=encoder_reduction, + device=device, + ) + + self.total_CMI_epoch = 0 + + def build_network(self): + self.network = instantiate(self.network_cfg)( + input_dim=self.encoder_output_dim, + output_dim=self.decoder_input_dim, + extra_dims=[self.output_var_num, self.ensemble_num], + ).to(self.device) + + def build_graph(self): + self.graph = BinaryGraph(self.input_var_num, self.output_var_num, device=self.device) + + @property + def CMI_mask(self) -> torch.Tensor: + mask = torch.zeros(self.input_var_num + 1, self.output_var_num, self.input_var_num, dtype=torch.long) + for i in range(self.input_var_num + 1): + m = torch.ones(self.output_var_num, self.input_var_num) + if i != self.input_var_num: + m[:, i] = 0 + mask[i] = m + return mask.to(self.device) + + def multi_graph_forward(self, inputs: MutableMapping[str, torch.Tensor]) -> Dict[str, torch.Tensor]: + """when first step, inputs should be dict of str and Tensor with (ensemble-num, batch-size, specific-dim) shape, + since twice step, the shape of Tensor becomes (input-var-num + 1, ensemble-num, batch-size, specific-dim) + + Args: + inputs: + + Returns: + + """ + batch_size, extra_dim = self.get_inputs_info(inputs) + + inputs_tensor = torch.empty(*extra_dim, self.ensemble_num, batch_size, self.input_var_num, self.encoder_output_dim).to( + self.device + ) + for i, var in enumerate(self.input_variables): + out = self.variable_encoders[var.name](inputs[var.name].to(self.device)) + inputs_tensor[..., i, :] = out + + # if len(extra_dim) == 0: + # # [..., output-var-num, input-var-num] + # mask = self.CMI_mask + # # [..., output-var-num, ensemble-num, batch-size, input-var-num] + # mask = mask.unsqueeze(-2).unsqueeze(-2) + # mask = mask.repeat((1,) * len(mask.shape[:-3]) + (self.ensemble_num, batch_size, 1)) + # reduced_inputs_tensor = self.reduce_encoder_output(inputs_tensor, mask) + # assert ( + # not torch.isinf(reduced_inputs_tensor).any() and not torch.isnan(reduced_inputs_tensor).any() + # ), "tensor must not be inf or nan" + # output_tensor = self.network(reduced_inputs_tensor) + # else: + # output_tensor = torch.empty( + # *extra_dim, self.output_var_num, self.ensemble_num, batch_size, self.decoder_input_dim + # ).to(self.device) + # + # CMI_mask = self.CMI_mask + # for i in range(self.input_var_num + 1): + # # [..., output-var-num, input-var-num] + # mask = CMI_mask[i] + # # [..., output-var-num, ensemble-num, batch-size, input-var-num] + # mask = mask.unsqueeze(-2).unsqueeze(-2) + # mask = mask.repeat((1,) * len(mask.shape[:-3]) + (self.ensemble_num, batch_size, 1)) + # if i == len(inputs_tensor) - 1: + # reduced_inputs_tensor = self.reduce_encoder_output(inputs_tensor[i], mask) + # outs = self.network(reduced_inputs_tensor) + # output_tensor[i] = outs + # else: + # for j in range(self.output_var_num): + # ins = inputs_tensor[-1] + # ins[:, :, j] = inputs_tensor[i, :, :, j, :] + # reduced_inputs_tensor = self.reduce_encoder_output(inputs_tensor[i], mask) + # outs = self.network(reduced_inputs_tensor) + # output_tensor[i, j] = outs[j] + + mask = self.CMI_mask + # [..., output-var-num, ensemble-num, batch-size, input-var-num] + mask = mask.unsqueeze(-2).unsqueeze(-2) + mask = mask.repeat((1,) * len(mask.shape[:-3]) + (self.ensemble_num, batch_size, 1)) + reduced_inputs_tensor = self.reduce_encoder_output(inputs_tensor, mask) + assert ( + not torch.isinf(reduced_inputs_tensor).any() and not torch.isnan(reduced_inputs_tensor).any() + ), "tensor must not be inf or nan" + output_tensor = self.network(reduced_inputs_tensor) + + outputs = {} + for i, var in enumerate(self.output_variables): + hid = output_tensor[:, i] + outputs[var.name] = self.variable_decoders[var.name](hid) + + if self.residual: + outputs = self.residual_outputs(inputs, outputs) + return outputs + + def calculate_CMI(self, nll_loss: torch.Tensor, threshold=1): + nll_loss_diff = nll_loss[:-1] - nll_loss[-1] + graph_data = (nll_loss_diff.mean(dim=(1, 2)) > threshold).to(torch.long) + return graph_data, nll_loss_diff.mean(dim=(1, 2)) + + def learn( + self, + inputs: MutableMapping[str, np.ndarray], + outputs: MutableMapping[str, np.ndarray], + work_dir: Optional[pathlib.Path] = None, + **kwargs + ): + work_dir = pathlib.Path(".") if work_dir is None else work_dir + + open(work_dir / "history_mask.txt", "w") + open(work_dir / "history_cmi.txt", "w") + train_loader, valid_loader = self.get_data_loaders(inputs, outputs) + + final_graph_data = None + + epoch_iter = range(self.longest_epoch) if self.longest_epoch >= 0 else count() + epochs_since_update = 0 + + loss_func = partial(variable_loss_func, output_variables=self.output_variables, device=self.device) + train = partial(train_func, forward=self.multi_graph_forward, optimizer=self.optimizer, loss_func=loss_func) + eval = partial(eval_func, forward=self.multi_graph_forward, loss_func=loss_func) + + best_eval_loss = eval(valid_loader).mean(dim=(0, 2, 3)) + + for epoch in epoch_iter: + train_loss = train(train_loader) + eval_loss = eval(valid_loader) + + improvement = (best_eval_loss - eval_loss.mean(dim=(0, 2, 3))) / torch.abs(best_eval_loss) + if (improvement > self.improvement_threshold).any().item(): + best_eval_loss = torch.minimum(best_eval_loss, eval_loss.mean(dim=(0, 2, 3))) + epochs_since_update = 0 + + final_graph_data, mean_nll_loss_diff = self.calculate_CMI(eval_loss) + with open(work_dir / "history_mask.txt", "a") as f: + f.write(str(final_graph_data) + "\n") + with open(work_dir / "history_cmi.txt", "a") as f: + f.write(str(mean_nll_loss_diff) + "\n") + print( + "new best valid, CMI test result:\n{}\nwith mean nll loss diff:\n{}".format( + final_graph_data, mean_nll_loss_diff + ) + ) + else: + epochs_since_update += 1 + + # log + self.total_CMI_epoch += 1 + if self.logger is not None: + self.logger.record("{}-CMI-test/epoch".format(self.name), epoch) + self.logger.record("{}-CMI-test/epochs_since_update".format(self.name), epochs_since_update) + self.logger.record("{}-CMI-test/train_dataset_size".format(self.name), len(train_loader.dataset)) + self.logger.record("{}-CMI-test/valid_dataset_size".format(self.name), len(valid_loader.dataset)) + self.logger.record("{}-CMI-test/train_loss".format(self.name), train_loss.mean().item()) + self.logger.record("{}-CMI-test/val_loss".format(self.name), eval_loss.mean().item()) + self.logger.record("{}-CMI-test/best_val_loss".format(self.name), best_eval_loss.mean().item()) + self.logger.record("{}-CMI-test/lr".format(self.name), self.optimizer.param_groups[0]["lr"]) + + self.logger.dump(self.total_CMI_epoch) + + if self.patience and epochs_since_update >= self.patience: + break + + self.scheduler.step() + print(self.optimizer) + + assert final_graph_data is not None + self.graph.set_data(final_graph_data) + self.build_optimizer() + + super(CMITestMech, self).learn(inputs, outputs, work_dir=work_dir, **kwargs) + + +if __name__ == "__main__": + import gym + from stable_baselines3.common.buffers import ReplayBuffer + from torch.utils.data import DataLoader + + from cmrl.models.data_loader import EnsembleBufferDataset, collate_fn, buffer_to_dict + from cmrl.utils.creator import parse_space + from cmrl.sb3_extension.logger import configure as logger_configure + + from cmrl.utils.env import load_offline_data + from cmrl.models.causal_mech.util import variable_loss_func + + def unwrap_env(env): + while isinstance(env, gym.Wrapper): + env = env.env + return env + + env = unwrap_env(gym.make("ParallelContinuousCartPoleSwingUp-v0")) + real_replay_buffer = ReplayBuffer( + int(1e6), env.observation_space, env.action_space, "cpu", handle_timeout_termination=False + ) + load_offline_data(env, real_replay_buffer, "SAC-expert", use_ratio=0.01) + + extra_info = {"Radian": ["obs_1", "obs_5", "obs_9"]} + # extra_info = {"Radian": ["obs_1"]} + + input_variables = parse_space(env.state_space, "obs", extra_info=extra_info) + parse_space(env.action_space, "act") + output_variables = parse_space(env.state_space, "next_obs", extra_info=extra_info) + + logger = logger_configure("cmi-log", ["tensorboard", "stdout"]) + + mech = CMITestMech("kernel_test_mech", input_variables, output_variables) + + inputs, outputs = buffer_to_dict(env.state_space, env.action_space, env.obs2state, real_replay_buffer, "transition") + + mech.learn(inputs, outputs) diff --git a/cmrl/models/causal_mech/__init__.py b/cmrl/models/causal_mech/__init__.py new file mode 100644 index 0000000..dd309df --- /dev/null +++ b/cmrl/models/causal_mech/__init__.py @@ -0,0 +1,4 @@ +from cmrl.models.causal_mech.oracle_mech import OracleMech +from cmrl.models.causal_mech.CMI_test import CMITestMech +# from cmrl.models.causal_mech.reinforce import ReinforceCausalMech +from cmrl.models.causal_mech.kernel_test import KernelTestMech \ No newline at end of file diff --git a/cmrl/models/causal_mech/base.py b/cmrl/models/causal_mech/base.py new file mode 100644 index 0000000..4a46b5a --- /dev/null +++ b/cmrl/models/causal_mech/base.py @@ -0,0 +1,419 @@ +from typing import Optional, List, Dict, Union, MutableMapping +from abc import abstractmethod, ABC +from itertools import chain, count +import pathlib +from functools import partial +import copy +from multiprocessing import cpu_count + +import numpy as np +import torch +from torch.utils.data import DataLoader +from torch.optim import Optimizer +from omegaconf import DictConfig +from stable_baselines3.common.logger import Logger +from hydra.utils import instantiate + +from cmrl.models.graphs.base_graph import BaseGraph +from cmrl.models.graphs.binary_graph import BinaryGraph +from cmrl.utils.variables import Variable +from cmrl.models.constant import NETWORK_CFG, ENCODER_CFG, DECODER_CFG, OPTIMIZER_CFG, SCHEDULER_CFG +from cmrl.models.networks.base_network import BaseNetwork +from cmrl.models.graphs.base_graph import BaseGraph +from cmrl.models.networks.coder import VariableEncoder, VariableDecoder +from cmrl.utils.variables import Variable, ContinuousVariable, DiscreteVariable, BinaryVariable +from cmrl.models.causal_mech.util import variable_loss_func, train_func, eval_func +from cmrl.models.data_loader import EnsembleBufferDataset, collate_fn + + +class BaseCausalMech(ABC): + """The base class of causal-mech learned by neural networks. + Pay attention that the causal discovery maybe not realized through a neural way. + """ + + def __init__( + self, + name: str, + input_variables: List[Variable], + output_variables: List[Variable], + logger: Optional[Logger] = None, + ): + self.name = name + self.input_variables = input_variables + self.output_variables = output_variables + self.logger = logger + + self.input_variables_dict = dict([(v.name, v) for v in self.input_variables]) + self.output_variables_dict = dict([(v.name, v) for v in self.output_variables]) + + self.input_var_num = len(self.input_variables) + self.output_var_num = len(self.output_variables) + self.graph: Optional[BaseGraph] = None + + @abstractmethod + def learn( + self, + inputs: MutableMapping[str, np.ndarray], + outputs: MutableMapping[str, np.ndarray], + work_dir: Optional[Union[str, pathlib.Path]] = None, + **kwargs + ): + raise NotImplementedError + + @abstractmethod + def forward(self, inputs: MutableMapping[str, np.ndarray]) -> Dict[str, torch.Tensor]: + raise NotImplementedError + + @property + def causal_graph(self) -> torch.Tensor: + """property causal graph""" + if self.graph is None: + return torch.ones(len(self.input_variables), len(self.output_variables), dtype=torch.int, device=self.device) + else: + return self.graph.get_binary_adj_matrix() + + def save(self, save_dir: Union[str, pathlib.Path]): + pass + + def load(self, load_dir: Union[str, pathlib.Path]): + pass + + +class EnsembleNeuralMech(BaseCausalMech): + def __init__( + self, + # base + name: str, + input_variables: List[Variable], + output_variables: List[Variable], + logger: Optional[Logger] = None, + # model learning + longest_epoch: int = -1, + improvement_threshold: float = 0.01, + patience: int = 5, + batch_size: int = 256, + # ensemble + ensemble_num: int = 7, + elite_num: int = 5, + # cfgs + network_cfg: Optional[DictConfig] = None, + encoder_cfg: Optional[DictConfig] = None, + decoder_cfg: Optional[DictConfig] = None, + optimizer_cfg: Optional[DictConfig] = None, + scheduler_cfg: Optional[DictConfig] = None, + # forward method + residual: bool = True, + encoder_reduction: str = "sum", + # others + device: Union[str, torch.device] = "cpu", + ): + BaseCausalMech.__init__( + self, name=name, input_variables=input_variables, output_variables=output_variables, logger=logger + ) + # model learning + self.longest_epoch = longest_epoch + self.improvement_threshold = improvement_threshold + self.patience = patience + self.batch_size = batch_size + # ensemble + self.ensemble_num = ensemble_num + self.elite_num = elite_num + # cfgs + self.network_cfg = NETWORK_CFG if network_cfg is None else network_cfg + self.encoder_cfg = ENCODER_CFG if encoder_cfg is None else encoder_cfg + self.decoder_cfg = DECODER_CFG if decoder_cfg is None else decoder_cfg + self.optimizer_cfg = OPTIMIZER_CFG if optimizer_cfg is None else optimizer_cfg + self.scheduler_cfg = SCHEDULER_CFG if scheduler_cfg is None else scheduler_cfg + # forward method + self.residual = residual + self.encoder_reduction = encoder_reduction + # others + self.device = device + + # build member object + self.variable_encoders: Optional[Dict[str, VariableEncoder]] = None + self.variable_decoders: Optional[Dict[str, VariableEncoder]] = None + self.network: Optional[BaseNetwork] = None + self.graph: Optional[BaseGraph] = None + self.optimizer: Optional[Optimizer] = None + self.scheduler: Optional[object] = None + self.build_coders() + self.build_network() + self.build_graph() + self.build_optimizer() + + self.total_epoch = 0 + self.elite_indices: List[int] = [] + + @property + def encoder_output_dim(self): + return self.encoder_cfg.output_dim + + @property + def decoder_input_dim(self): + return self.decoder_cfg.input_dim + + def build_network(self): + self.network = instantiate(self.network_cfg)( + input_dim=self.encoder_output_dim, + output_dim=self.decoder_input_dim, + extra_dims=[self.output_var_num, self.ensemble_num], + ).to(self.device) + + def build_optimizer(self): + assert self.network, "you must build network first" + assert self.variable_encoders and self.variable_decoders, "you must build coders first" + params = ( + [self.network.parameters()] + + [encoder.parameters() for encoder in self.variable_encoders.values()] + + [decoder.parameters() for decoder in self.variable_decoders.values()] + ) + + self.optimizer = instantiate(self.optimizer_cfg)(params=chain(*params)) + self.scheduler = instantiate(self.scheduler_cfg)(optimizer=self.optimizer) + + def forward(self, inputs: MutableMapping[str, torch.Tensor]) -> Dict[str, torch.Tensor]: + batch_size, _ = self.get_inputs_batch_size(inputs) + + inputs_tensor = torch.zeros(self.ensemble_num, batch_size, self.input_var_num, self.encoder_output_dim).to(self.device) + for i, var in enumerate(self.input_variables): + out = self.variable_encoders[var.name](inputs[var.name].to(self.device)) + inputs_tensor[:, :, i] = out + + output_tensor = self.network(self.reduce_encoder_output(inputs_tensor)) + + outputs = {} + for i, var in enumerate(self.output_variables): + hid = output_tensor[i] + outputs[var.name] = self.variable_decoders[var.name](hid) + + if self.residual: + outputs = self.residual_outputs(inputs, outputs) + return outputs + + def build_graph(self): + pass + + def build_coders(self): + self.variable_encoders = {} + for var in self.input_variables: + assert var.name not in self.variable_encoders, "duplicate name in encoders: {}".format(var.name) + self.variable_encoders[var.name] = instantiate(self.encoder_cfg)(variable=var).to(self.device) + + self.variable_decoders = {} + for var in self.output_variables: + assert var.name not in self.variable_decoders, "duplicate name in decoders: {}".format(var.name) + self.variable_decoders[var.name] = instantiate(self.decoder_cfg)(variable=var).to(self.device) + + def save(self, save_dir: Union[str, pathlib.Path]): + if isinstance(save_dir, str): + save_dir = pathlib.Path(save_dir) + save_dir = save_dir / pathlib.Path(self.name) + save_dir.mkdir(exist_ok=True) + + self.network.save(save_dir) + if self.graph is not None: + self.graph.save(save_dir) + for coder in self.variable_encoders.values(): + coder.save(save_dir) + for coder in self.variable_decoders.values(): + coder.save(save_dir) + + def load(self, load_dir: Union[str, pathlib.Path]): + if isinstance(load_dir, str): + load_dir = pathlib.Path(load_dir) + assert load_dir.exists() + + self.network.load(load_dir) + if self.graph is not None: + self.graph.load(load_dir) + for coder in self.variable_encoders.values(): + coder.load(load_dir) + for coder in self.variable_decoders.values(): + coder.load(load_dir) + + def get_inputs_info(self, inputs: MutableMapping[str, torch.Tensor]): + assert len(set(inputs.keys()) & set(self.input_variables_dict.keys())) == len(inputs) + data_shape = next(iter(inputs.values())).shape + # assert len(data_shape) == 3, "{}".format(data_shape) # ensemble-num, batch-size, specific-dim + ensemble, batch_size, specific_dim = data_shape[-3:] + assert ensemble == self.ensemble_num + + return batch_size, data_shape[:-3] + + def residual_outputs( + self, + inputs: MutableMapping[str, torch.Tensor], + outputs: MutableMapping[str, torch.Tensor], + ) -> MutableMapping[str, torch.Tensor]: + for name in filter(lambda s: s.startswith("obs"), inputs.keys()): + # assert inputs[name].shape[:2] == outputs["next_{}".format(name)].shape[:2] + # assert inputs[name].shape[2] * 2 == outputs["next_{}".format(name)].shape[2] + var_dim = inputs[name].shape[-1] + outputs["next_{}".format(name)][..., :var_dim] += inputs[name].to(self.device) + return outputs + + def reduce_encoder_output( + self, + encoder_output: torch.Tensor, + mask: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + assert len(encoder_output.shape) == 4, ( + "shape of `encoder_output` should be (ensemble-num, batch-size, input-var-num, encoder-output-dim), " + "rather than {}".format(encoder_output.shape) + ) + + if mask is None: + # [..., input-var-num] + mask = self.forward_mask + # [..., ensemble-num, batch-size, input-var-num] + mask = mask.unsqueeze(-2).unsqueeze(-2) + mask = mask.repeat((1,) * len(mask.shape[:-3]) + (*encoder_output.shape[:2], 1)) + + # mask shape [..., ensemble-num, batch-size, input-var-num] + assert ( + mask.shape[-3:] == encoder_output.shape[:-1] + ), "mask shape should be (..., ensemble-num, batch-size, input-var-num)" + + # [*mask-extra-dims, ensemble-num, batch-size, input-var-num, encoder-output-dim] + mask = mask[..., None].repeat([1] * len(mask.shape) + [encoder_output.shape[-1]]) + masked_encoder_output = encoder_output.repeat(tuple(mask.shape[:-4]) + (1,) * 4) + + # choose mask value + mask_value = 0 + if self.encoder_reduction == "max": + mask_value = -float("inf") + masked_encoder_output[mask == 0] = mask_value + + if self.encoder_reduction == "sum": + return masked_encoder_output.sum(-2) + elif self.encoder_reduction == "mean": + return masked_encoder_output.mean(-2) + elif self.encoder_reduction == "max": + values, indices = masked_encoder_output.max(-2) + return values + else: + raise NotImplementedError("not implemented encoder reduction method: {}".format(self.encoder_reduction)) + + @property + def forward_mask(self) -> torch.Tensor: + """property input masks""" + return self.causal_graph.T + + def get_data_loaders( + self, + inputs: MutableMapping[str, np.ndarray], + outputs: MutableMapping[str, np.ndarray], + ): + train_set = EnsembleBufferDataset( + inputs=inputs, outputs=outputs, training=True, train_ratio=0.8, ensemble_num=self.ensemble_num, seed=1 + ) + valid_set = EnsembleBufferDataset( + inputs=inputs, outputs=outputs, training=False, train_ratio=0.8, ensemble_num=self.ensemble_num, seed=1 + ) + + train_loader = DataLoader(train_set, batch_size=self.batch_size, collate_fn=collate_fn, num_workers=cpu_count()) + valid_loader = DataLoader(valid_set, batch_size=self.batch_size, collate_fn=collate_fn, num_workers=cpu_count()) + + return train_loader, valid_loader + + def learn( + self, + inputs: MutableMapping[str, np.ndarray], + outputs: MutableMapping[str, np.ndarray], + work_dir: Optional[Union[str, pathlib.Path]] = None, + **kwargs + ): + train_loader, valid_loader = self.get_data_loaders(inputs, outputs) + + best_weights: Optional[Dict] = None + epoch_iter = range(self.longest_epoch) if self.longest_epoch >= 0 else count() + epochs_since_update = 0 + + loss_func = partial(variable_loss_func, output_variables=self.output_variables, device=self.device) + train = partial(train_func, forward=self.forward, optimizer=self.optimizer, loss_func=loss_func) + eval = partial(eval_func, forward=self.forward, loss_func=loss_func) + + best_eval_loss = eval(valid_loader).mean(dim=(-2, -1)) + + for epoch in epoch_iter: + train_loss = train(train_loader) + eval_loss = eval(valid_loader) + + maybe_best_weights = self._maybe_get_best_weights( + best_eval_loss, eval_loss.mean(dim=(-2, -1)), self.improvement_threshold + ) + if maybe_best_weights: + # best loss + best_eval_loss = torch.minimum(best_eval_loss, eval_loss.mean(dim=(-2, -1))) + best_weights = maybe_best_weights + epochs_since_update = 0 + else: + epochs_since_update += 1 + + # log + self.total_epoch += 1 + if self.logger is not None: + self.logger.record("{}/epoch".format(self.name), epoch) + self.logger.record("{}/epochs_since_update".format(self.name), epochs_since_update) + self.logger.record("{}/train_dataset_size".format(self.name), len(train_loader.dataset)) + self.logger.record("{}/valid_dataset_size".format(self.name), len(valid_loader.dataset)) + self.logger.record("{}/train_loss".format(self.name), train_loss.mean().item()) + self.logger.record("{}/val_loss".format(self.name), eval_loss.mean().item()) + self.logger.record("{}/best_val_loss".format(self.name), best_eval_loss.mean().item()) + self.logger.record("{}/lr".format(self.name), self.optimizer.param_groups[0]["lr"]) + + self.logger.dump(self.total_epoch) + + if self.patience and epochs_since_update >= self.patience: + break + + self.scheduler.step() + + # saving the best models: + self._maybe_set_best_weights_and_elite(best_weights, best_eval_loss) + + self.save(save_dir=work_dir) + + def _maybe_get_best_weights( + self, + best_val_loss: torch.Tensor, + val_loss: torch.Tensor, + threshold: float = 0.01, + ) -> Optional[Dict]: + """Return the current model state dict if the validation score improves. + For ensembles, this checks the validation for each ensemble member separately. + Copy from https://github.com/facebookresearch/mbrl-lib/blob/main/mbrl/models/model_trainer.py + + Args: + best_val_score (tensor): the current best validation losses per model. + val_score (tensor): the new validation loss per model. + threshold (float): the threshold for relative improvement. + Returns: + (dict, optional): if the validation score's relative improvement over the + best validation score is higher than the threshold, returns the state dictionary + of the stored model, otherwise returns ``None``. + """ + improvement = (best_val_loss - val_loss) / torch.abs(best_val_loss) + if (improvement > threshold).any().item(): + best_weights = copy.deepcopy(self.network.state_dict()) + else: + best_weights = None + + return best_weights + + def _maybe_set_best_weights_and_elite(self, best_weights: Optional[Dict], best_val_score: torch.Tensor): + if best_weights is not None: + self.network.load_state_dict(best_weights) + + sorted_indices = np.argsort(best_val_score.tolist()) + self.elite_indices = sorted_indices[: self.elite_num] + + def get_inputs_batch_size(self, inputs: MutableMapping[str, torch.Tensor]) -> int: + assert len(set(inputs.keys()) & set(self.variable_encoders.keys())) == len(inputs) + data_shape = list(inputs.values())[0].shape + # assert len(data_shape) == 3, "{}".format(data_shape) # ensemble-num, batch-size, specific-dim + ensemble, batch_size, specific_dim = data_shape[-3:] + assert ensemble == self.ensemble_num + + return batch_size, data_shape[:-3] diff --git a/cmrl/models/causal_mech/kernel_test.py b/cmrl/models/causal_mech/kernel_test.py new file mode 100644 index 0000000..f4389c3 --- /dev/null +++ b/cmrl/models/causal_mech/kernel_test.py @@ -0,0 +1,256 @@ +from typing import Optional, List, Dict, Union, MutableMapping +from functools import partial +from collections import defaultdict + +import pathlib +import numpy +import numpy as np +import torch +from omegaconf import DictConfig +from stable_baselines3.common.logger import Logger +from hydra.utils import instantiate + +# from cmrl.utils.RCIT import KCI_CInd +from causallearn.utils.KCI.KCI import KCI_CInd +from tqdm import tqdm + +from cmrl.models.causal_mech.base import EnsembleNeuralMech +from cmrl.utils.variables import Variable, ContinuousVariable, DiscreteVariable, BinaryVariable, RadianVariable +from cmrl.models.graphs.binary_graph import BinaryGraph + + +class KernelTestMech(EnsembleNeuralMech): + def __init__( + self, + # base + name: str, + input_variables: List[Variable], + output_variables: List[Variable], + logger: Optional[Logger] = None, + # model learning + longest_epoch: int = -1, + improvement_threshold: float = 0.01, + patience: int = 5, + batch_size: int = 256, + # ensemble + ensemble_num: int = 7, + elite_num: int = 5, + # cfgs + network_cfg: Optional[DictConfig] = None, + encoder_cfg: Optional[DictConfig] = None, + decoder_cfg: Optional[DictConfig] = None, + optimizer_cfg: Optional[DictConfig] = None, + scheduler_cfg: Optional[DictConfig] = None, + # forward method + residual: bool = True, + encoder_reduction: str = "sum", + # others + device: Union[str, torch.device] = "cpu", + # KCI + sample_num: int = 2000, + kci_times: int = 10, + not_confident_bound: float = 0.25, + longest_sample: int = 5000, + ): + EnsembleNeuralMech.__init__( + self, + name=name, + input_variables=input_variables, + output_variables=output_variables, + logger=logger, + longest_epoch=longest_epoch, + improvement_threshold=improvement_threshold, + patience=patience, + batch_size=batch_size, + ensemble_num=ensemble_num, + elite_num=elite_num, + network_cfg=network_cfg, + encoder_cfg=encoder_cfg, + decoder_cfg=decoder_cfg, + optimizer_cfg=optimizer_cfg, + scheduler_cfg=scheduler_cfg, + residual=residual, + encoder_reduction=encoder_reduction, + device=device, + ) + self.sample_num = sample_num + self.kci_times = kci_times + self.not_confident_bound = not_confident_bound + self.longest_sample = longest_sample + + def kci( + self, + input_idx: int, + output_idx: int, + inputs: MutableMapping[str, numpy.ndarray], + outputs: MutableMapping[str, numpy.ndarray], + sample_indices: np.ndarray, + ): + in_name, out_name = list(inputs.keys())[input_idx], list(outputs.keys())[output_idx] + + if self.residual: + data_x = outputs[out_name][sample_indices] - inputs[out_name.replace("next_", "")][sample_indices] + else: + data_x = outputs[out_name][sample_indices] + + def deal_with_radian_input(name, data): + if isinstance(self.input_variables_dict[name], RadianVariable): + return (data + np.pi) % (2 * np.pi) - np.pi + else: + return data + + data_y = deal_with_radian_input(in_name, inputs[in_name])[sample_indices] + data_z = [ + deal_with_radian_input(other_in_name, in_data)[sample_indices] + for other_in_name, in_data in inputs.items() + if other_in_name != in_name + ] + data_z = np.concatenate(data_z, axis=1) + + kci = KCI_CInd() + p_value, test_stat = kci.compute_pvalue(data_x, data_y, data_z) + return p_value + + def kci_compute_graph( + self, + inputs: MutableMapping[str, numpy.ndarray], + outputs: MutableMapping[str, numpy.ndarray], + work_dir: Optional[pathlib.Path] = None, + **kwargs + ): + + open(work_dir / "history_vote.txt", "w") + + length = next(iter(inputs.values())).shape[0] + sample_length = min(length, self.sample_num) if self.sample_num > 0 else length + + init_pvalues_array = np.empty((self.kci_times, self.input_var_num, self.output_var_num)) + with tqdm( + total=self.kci_times * self.input_var_num * self.output_var_num, + desc="init kci of {} samples".format(sample_length), + ) as pbar: + for time in range(self.kci_times): + sample_indices = np.random.permutation(length)[:sample_length] + kci = partial(self.kci, inputs=inputs, outputs=outputs, sample_indices=sample_indices) + for out_idx in range(len(outputs)): + for in_idx in range(len(inputs)): + init_pvalues_array[time][in_idx][out_idx] = kci(in_idx, out_idx) + pbar.update(1) + + votes = (init_pvalues_array < 0.05).mean(axis=0) + is_not_confident = np.logical_and(votes > self.not_confident_bound, votes < 1 - self.not_confident_bound) + not_confident_list = np.array(np.where(is_not_confident)).T + + recompute_times = 1 + while len(not_confident_list) != 0: + with open(work_dir / "history_vote.txt", "a") as f: + f.write(str(votes) + "\n") + print(votes) + + new_sample_length = int(sample_length * 1.5**recompute_times) + if new_sample_length > min(self.longest_sample, length): + break + + pvalues_dict = defaultdict(list) + with tqdm( + total=self.kci_times * len(not_confident_list), + desc="{}th re-compute kci of {} samples".format(recompute_times, new_sample_length), + ) as pbar: + for time in range(self.kci_times): + sample_indices = np.random.permutation(length)[:new_sample_length] + kci = partial(self.kci, inputs=inputs, outputs=outputs, sample_indices=sample_indices) + for in_idx, out_idx in not_confident_list: + pvalues_dict[(in_idx, out_idx)].append(kci(in_idx, out_idx)) + pbar.update(1) + + not_confident_list = [] + for key, value in pvalues_dict.items(): + vote = (np.array(value) < 0.05).mean() + if self.not_confident_bound < vote < 1 - self.not_confident_bound: + not_confident_list.append(key) + else: + votes[key] = vote + recompute_times += 1 + + return votes > 0.5 + + def build_network(self): + self.network = instantiate(self.network_cfg)( + input_dim=self.encoder_output_dim, + output_dim=self.decoder_input_dim, + extra_dims=[self.ensemble_num], + ).to(self.device) + + def forward(self, inputs: MutableMapping[str, torch.Tensor]) -> Dict[str, torch.Tensor]: + batch_size, _ = self.get_inputs_batch_size(inputs) + + inputs_tensor = torch.zeros(self.ensemble_num, batch_size, self.input_var_num, self.encoder_output_dim).to(self.device) + for i, var in enumerate(self.input_variables): + out = self.variable_encoders[var.name](inputs[var.name].to(self.device)) + inputs_tensor[:, :, i] = out + + output_tensor = self.network(self.reduce_encoder_output(inputs_tensor)) + + outputs = {} + for i, var in enumerate(self.output_variables): + hid = output_tensor[i] + outputs[var.name] = self.variable_decoders[var.name](hid) + + if self.residual: + outputs = self.residual_outputs(inputs, outputs) + return outputs + + def build_graph(self): + self.graph = BinaryGraph(self.input_var_num, self.output_var_num, device=self.device) + + def learn( + self, + inputs: MutableMapping[str, np.ndarray], + outputs: MutableMapping[str, np.ndarray], + work_dir: Optional[pathlib.Path] = None, + **kwargs + ): + work_dir = pathlib.Path(".") if work_dir is None else work_dir + graph = self.kci_compute_graph(inputs, outputs, work_dir) + self.graph.set_data(graph) + + super(KernelTestMech, self).learn(inputs, outputs, work_dir=work_dir, **kwargs) + + +if __name__ == "__main__": + import gym + from emei import EmeiEnv + from stable_baselines3.common.buffers import ReplayBuffer + from torch.utils.data import DataLoader + from typing import cast + + from cmrl.models.data_loader import EnsembleBufferDataset, collate_fn, buffer_to_dict + from cmrl.utils.creator import parse_space + from cmrl.utils.env import load_offline_data + from cmrl.sb3_extension.logger import configure as logger_configure + from cmrl.models.causal_mech.util import variable_loss_func + + def unwrap_env(env): + while isinstance(env, gym.Wrapper): + env = env.env + return env + + env = unwrap_env(gym.make("ParallelContinuousCartPoleSwingUp-v0")) + real_replay_buffer = ReplayBuffer( + int(1e6), env.observation_space, env.action_space, "cpu", handle_timeout_termination=False + ) + load_offline_data(env, real_replay_buffer, "SAC-expert", use_ratio=1) + + extra_info = {"Radian": ["obs_1", "obs_5", "obs_9"]} + # extra_info = {"Radian": ["obs_1"]} + + input_variables = parse_space(env.state_space, "obs", extra_info=extra_info) + parse_space(env.action_space, "act") + output_variables = parse_space(env.state_space, "next_obs", extra_info=extra_info) + + logger = logger_configure("kci-log", ["tensorboard", "stdout"]) + + mech = KernelTestMech("kernel_test_mech", input_variables, output_variables, sample_num=100, kci_times=20, logger=logger) + + inputs, outputs = buffer_to_dict(env.state_space, env.action_space, env.obs2state, real_replay_buffer, "transition") + + mech.learn(inputs, outputs) diff --git a/cmrl/models/causal_mech/oracle_mech.py b/cmrl/models/causal_mech/oracle_mech.py new file mode 100644 index 0000000..dda20c1 --- /dev/null +++ b/cmrl/models/causal_mech/oracle_mech.py @@ -0,0 +1,103 @@ +from typing import Optional, List, Dict, Union, MutableMapping + +import numpy +import torch +from torch.utils.data import DataLoader +import numpy as np +from omegaconf import DictConfig +from hydra.utils import instantiate +from stable_baselines3.common.logger import Logger + +from cmrl.utils.variables import Variable +from cmrl.models.causal_mech.base import EnsembleNeuralMech +from cmrl.models.graphs.binary_graph import BinaryGraph +from cmrl.models.data_loader import EnsembleBufferDataset, collate_fn + + +class OracleMech(EnsembleNeuralMech): + def __init__( + self, + name: str, + input_variables: List[Variable], + output_variables: List[Variable], + logger: Optional[Logger] = None, + # model learning + longest_epoch: int = -1, + improvement_threshold: float = 0.01, + patience: int = 5, + batch_size: int = 256, + # ensemble + ensemble_num: int = 7, + elite_num: int = 5, + # cfgs + network_cfg: Optional[DictConfig] = None, + encoder_cfg: Optional[DictConfig] = None, + decoder_cfg: Optional[DictConfig] = None, + optimizer_cfg: Optional[DictConfig] = None, + scheduler_cfg: Optional[DictConfig] = None, + # forward method + residual: bool = True, + encoder_reduction: str = "sum", + # others + device: Union[str, torch.device] = "cpu", + ): + EnsembleNeuralMech.__init__( + self, + name=name, + input_variables=input_variables, + output_variables=output_variables, + logger=logger, + longest_epoch=longest_epoch, + improvement_threshold=improvement_threshold, + patience=patience, + batch_size=batch_size, + ensemble_num=ensemble_num, + elite_num=elite_num, + network_cfg=network_cfg, + encoder_cfg=encoder_cfg, + decoder_cfg=decoder_cfg, + optimizer_cfg=optimizer_cfg, + scheduler_cfg=scheduler_cfg, + residual=residual, + encoder_reduction=encoder_reduction, + device=device, + ) + + def set_oracle_graph(self, graph_data: Optional[numpy.ndarray]): + self.graph = BinaryGraph(self.input_var_num, self.output_var_num, device=self.device) + if graph_data is None: + graph_data = np.ones([self.input_var_num, self.output_var_num]) + self.graph.set_data(graph_data=graph_data) + print("set oracle causal graph successfully: \n{}".format(graph_data)) + + +if __name__ == "__main__": + from typing import cast + + import gym + from stable_baselines3.common.buffers import ReplayBuffer + from torch.utils.data import DataLoader + from emei import EmeiEnv + + from cmrl.models.data_loader import EnsembleBufferDataset, collate_fn, buffer_to_dict + from cmrl.utils.creator import parse_space + from cmrl.utils.env import load_offline_data + from cmrl.models.causal_mech.util import variable_loss_func + from cmrl.sb3_extension.logger import configure as logger_configure + + env = cast(EmeiEnv, gym.make("ParallelContinuousCartPoleSwingUp-v0")) + real_replay_buffer = ReplayBuffer( + int(1e6), env.observation_space, env.action_space, "cpu", handle_timeout_termination=False + ) + load_offline_data(env, real_replay_buffer, "SAC-expert", use_ratio=1) + + input_variables = parse_space(env.state_space, "obs") + parse_space(env.action_space, "act") + output_variables = parse_space(env.state_space, "next_obs") + + logger = logger_configure("kci-log", ["tensorboard", "stdout"]) + + mech = OracleMech("plain_mech", input_variables, output_variables, logger=logger, device="cuda:1") + + inputs, outputs = buffer_to_dict(env.observation_space, env.action_space, env.obs2state, real_replay_buffer, "transition") + + mech.learn(inputs, outputs) diff --git a/cmrl/models/causal_mech/reinforce.py b/cmrl/models/causal_mech/reinforce.py new file mode 100644 index 0000000..cc089a8 --- /dev/null +++ b/cmrl/models/causal_mech/reinforce.py @@ -0,0 +1,389 @@ +from typing import List, Optional, Dict, Union, MutableMapping, Tuple +import math +import pathlib +from itertools import count +from functools import partial +import copy + +import torch +import numpy as np +from torch.utils.data import DataLoader +from stable_baselines3.common.logger import Logger +from omegaconf import DictConfig +from hydra.utils import instantiate + +from cmrl.utils.variables import Variable +from cmrl.models.causal_mech.neural_causal_mech import NeuralCausalMech +from cmrl.models.graphs.prob_graph import BernoulliGraph +from cmrl.models.causal_mech.util import variable_loss_func, train_func, eval_func + +default_graph_optimizer_cfg = DictConfig( + dict( + _target_="torch.optim.Adam", + _partial_=True, + lr=1e-3, + weight_decay=0.0, + eps=1e-8, + ) +) + + +class ReinforceCausalMech(NeuralCausalMech): + def __init__( + self, + name: str, + input_variables: List[Variable], + output_variables: List[Variable], + # model learning + longest_epoch: int = -1, + improvement_threshold: float = 0.01, + patience: int = 5, + # ensemble + ensemble_num: int = 7, + elite_num: int = 5, + # cfgs + network_cfg: Optional[DictConfig] = None, + encoder_cfg: Optional[DictConfig] = None, + decoder_cfg: Optional[DictConfig] = None, + optimizer_cfg: Optional[DictConfig] = None, + graph_optimizer_cfg: Optional[DictConfig] = default_graph_optimizer_cfg, + # graph params + concat_mask: bool = True, + graph_MC_samples: int = 100, + graph_max_stack: int = 200, + lambda_sparse: float = 1e-3, + # forward method + residual: bool = True, + encoder_reduction: str = "sum", + multi_step: str = "none", + # logger + logger: Optional[Logger] = None, + # others + device: Union[str, torch.device] = "cpu", + **kwargs + ): + if multi_step == "none": + multi_step = "forward-euler 1" + + # cfgs + self.graph_optimizer_cfg = graph_optimizer_cfg + + # graph params + self._concat_mask = concat_mask + self._graph_MC_samples = graph_MC_samples + self._graph_max_stack = graph_max_stack + self._lambda_sparse = lambda_sparse + + self.graph_optimizer = None + + super(ReinforceCausalMech, self).__init__( + name=name, + input_variables=input_variables, + output_variables=output_variables, + longest_epoch=longest_epoch, + improvement_threshold=improvement_threshold, + patience=patience, + ensemble_num=ensemble_num, + elite_num=elite_num, + network_cfg=network_cfg, + encoder_cfg=encoder_cfg, + decoder_cfg=decoder_cfg, + optimizer_cfg=optimizer_cfg, + residual=residual, + encoder_reduction=encoder_reduction, + multi_step=multi_step, + logger=logger, + device=device, + **kwargs + ) + + def build_network(self): + input_dim = self.encoder_output_dim + if self._concat_mask: + input_dim += self.input_var_num + + self.network = instantiate(self.network_cfg)( + input_dim=input_dim, + output_dim=self.decoder_input_dim, + extra_dims=[self.output_var_num, self.ensemble_num], + ).to(self.device) + + def build_graph(self): + self.graph = BernoulliGraph( + in_dim=self.input_var_num, + out_dim=self.output_var_num, + include_input=False, + init_param=1e-6, + requires_grad=True, + device=self.device, + ) + + def build_optimizer(self): + assert ( + self.network is not None and self.graph is not None + ), "network and graph are both required when building optimizer" + super().build_optimizer() + + # graph optimizer + self.graph_optimizer = instantiate(self.graph_optimizer_cfg)(self.graph.parameters) + + @property + def causal_graph(self) -> torch.Tensor: + """property causal graph""" + assert self.graph is not None, "graph incorrectly initialized" + + return self.graph.get_binary_adj_matrix(threshold=0.5) + + def single_step_forward( + self, + inputs: MutableMapping[str, torch.Tensor], + train: bool = False, + mask: Optional[torch.Tensor] = None, + ) -> Dict[str, torch.Tensor]: + batch_size, extra_dim = self.get_inputs_batch_size(inputs) + assert len(extra_dim) == 0, "unexpected dimension in the inputs" + + inputs_tensor = torch.zeros(self.ensemble_num, batch_size, self.input_var_num, self.encoder_output_dim).to(self.device) + for i, var in enumerate(self.input_variables): + out = self.variable_encoders[var.name](inputs[var.name].to(self.device)) + inputs_tensor[..., i, :] = out + + if train and self.discovery: + # [ensemble-num, batch-size, input-var-num, output-var-num] + adj_matrix = self.graph.sample(None, sample_size=(self.ensemble_num, batch_size)) + # [ensemble-num, batch-size, output-var-num, input-var-num] + mask = adj_matrix.transpose(-1, -2) + # [output-var-num, ensemble-num, batch-size, input-var-num] + mask = mask.permute(2, 0, 1, 3) + else: + if mask is None: + mask = self.forward_mask + mask = mask.unsqueeze(-2).unsqueeze(-2) + mask = mask.repeat(1, self.ensemble_num, batch_size, 1) + + # [output-var-num, ensemble-num, batch-size, encoder-output-dim] + reduced_inputs_tensor = self.reduce_encoder_output(inputs_tensor, mask=mask) + if self._concat_mask: + # [output-var-num, ensemble-num, batch-size, encoder-output-dim + input-var-num] + reduced_inputs_tensor = torch.cat([reduced_inputs_tensor, mask], dim=-1) + output_tensor = self.network(reduced_inputs_tensor) + + outputs = {} + for i, var in enumerate(self.output_variables): + hid = output_tensor[i] + outputs[var.name] = self.variable_decoders[var.name](hid) + + if self.residual: + outputs = self.residual_outputs(inputs, outputs) + return outputs + + def forward( + self, + inputs: MutableMapping[str, torch.Tensor], + train: bool = False, + mask: Optional[torch.Tensor] = None, + ) -> Dict[str, torch.Tensor]: + if self.multi_step.startswith("forward-euler"): + step_num = int(self.multi_step.split()[-1]) + + outputs = {} + for step in range(step_num): + outputs = self.single_step_forward(inputs, train=train, mask=mask) + if step < step_num - 1: + for name in filter(lambda s: s.startswith("obs"), inputs.keys()): + inputs[name] = outputs["next_{}".format(name)][..., : inputs[name].shape[-1]] + else: + raise NotImplementedError("multi-step method {} is not supported".format(self.multi_step)) + + return outputs + + def train_graph(self, loader: DataLoader, data_ratio: float): + num_batches = len(loader) + train_num = int(num_batches * data_ratio) + + grads = torch.tensor([0], dtype=torch.float32) + for i, (inputs, targets) in enumerate(loader): + if train_num <= i: + break + + grads = grads + self._update_graph(inputs, targets) + + return grads + + def _update_graph( + self, + inputs: MutableMapping[str, torch.Tensor], + targets: MutableMapping[str, torch.Tensor], + ) -> torch.Tensor: + # do Monte-Carlo sampling to obtain adjacent matrices and corresponding model losses + adj_matrices, losses = self._MC_sample(inputs, targets) + + # calculate graph gradients + graph_grads = self._estimate_graph_grads(adj_matrices, losses) + + # update graph + graph_params = self.graph.parameters[0] # only one tensor parameter + self.graph_optimizer.zero_grad() + graph_params.grad = graph_grads + self.graph_optimizer.step() + + return graph_grads.detach().cpu() + + def _MC_sample( + self, + inputs: MutableMapping[str, torch.Tensor], + targets: MutableMapping[str, torch.Tensor], + ) -> Tuple[torch.Tensor]: + num_graph_list = [ + min(self._graph_max_stack, self._graph_MC_samples - i * self._graph_max_stack) + for i in range(math.ceil(self._graph_MC_samples / self._graph_max_stack)) + ] + num_graph_list = [(num_graph_list[i], sum(num_graph_list[:i])) for i in range(len(num_graph_list))] + + # sample graphs + adj_mats = self.graph.sample(None, sample_size=self._graph_MC_samples) + + # evaluate scores using the sampled adjacency matrices and data + batch_size, extra_dim = self.get_inputs_batch_size(inputs) + assert len(extra_dim) == 0, "unexpected dimension in the inputs" + + losses = [] + for graph_count, start_idx in num_graph_list: + # [ensemble-num, samples*batch_size, input-var-num, output-var-num] + expanded_adj_mats = ( + adj_mats[None, start_idx : start_idx + graph_count, None] + .expand(self.ensemble_num, -1, batch_size, -1, -1) + .flatten(1, 2) + ) + expanded_masks = expanded_adj_mats.transpose(-1, -2).permute(2, 0, 1, 3) + + expanded_inputs = {} + expanded_targets = {} + # expand inputs and targets + for in_key in inputs: + expanded_inputs[in_key] = inputs[in_key].repeat(1, graph_count, 1) + for tar_key in targets: + expanded_targets[tar_key] = targets[tar_key].repeat(1, graph_count, 1) + + with torch.no_grad(): + outputs = self.forward(expanded_inputs, train=False, mask=expanded_masks) + loss = variable_loss_func(outputs, expanded_targets, self.output_variables, device=self.device) + loss = loss.reshape(loss.shape[0], graph_count, batch_size, -1) + losses.append(loss.mean(dim=(0, 2))) + losses = sum(losses) + + return adj_mats, losses + + def _estimate_graph_grads( + self, + adj_matrices: torch.Tensor, + losses: torch.Tensor, + ) -> torch.Tensor: + """Use MC samples and corresponding losses to estimate gradients via REINFORCE. + + Args: + adj_matrices (tensor): MC sampled adjacent matrices from current graph, + shaped [num-samples, input-var-num, output-var-num]. + losses (tensor): the model losses corresponding to the adjacent matrices, + shaped [num-samples, output-var-num] + + """ + num_graphs = adj_matrices.shape[0] + losses = losses.unsqueeze(dim=1) + + # calculate graph gradients + edge_prob = self.graph.get_adj_matrix() + num_pos = adj_matrices.sum(dim=0) + num_neg = num_graphs - num_pos + mask = ((num_pos > 0) * (num_neg > 0)).float() + pos_grads = (losses * adj_matrices).sum(dim=0) / num_pos.clamp_(min=1e-5) + neg_grads = (losses * (1 - adj_matrices)).sum(dim=0) / num_neg.clamp_(min=1e-5) + graph_grads = mask * edge_prob * (1 - edge_prob) * (pos_grads - neg_grads + self._lambda_sparse) + + return graph_grads + + def learn( + self, + train_loader: DataLoader, + valid_loader: DataLoader, + graph_data_ratio: float = 0.5, + train_graph_freq: int = 2, + work_dir: Optional[Union[str, pathlib.Path]] = None, + **kwargs + ): + assert 0 <= graph_data_ratio <= 1, "graph data ratio should be in [0, 1]" + + best_weights: Optional[Dict] = None + epoch_iter = range(self.longest_epoch) if self.longest_epoch >= 0 else count() + epochs_since_update = 0 + + loss_fn = partial(variable_loss_func, output_variables=self.output_variables, device=self.device) + train_fn = partial(train_func, forward=partial(self.forward, train=True), optimizer=self.optimizer, loss_func=loss_fn) + eval_fn = partial(eval_func, forward=partial(self.forward, train=False), loss_func=loss_fn) + + best_eval_loss = eval_fn(valid_loader).mean(dim=(-2, -1)) + for epoch in epoch_iter: + if self.discovery and epoch % train_graph_freq == 0: + grads = self.train_graph(train_loader, data_ratio=graph_data_ratio) + print(self.graph.parameters[0]) + print(self.graph.get_binary_adj_matrix()) + + train_loss = train_fn(train_loader) + eval_loss = eval_fn(valid_loader) + + maybe_best_weights = self._maybe_get_best_weights( + best_eval_loss, eval_loss.mean(dim=(-2, -1)), self.improvement_threshold + ) + if maybe_best_weights: + # best loss + best_eval_loss = torch.minimum(best_eval_loss, eval_loss.mean(dim=(-2, -1))) + best_weights = maybe_best_weights + epochs_since_update = 0 + else: + epochs_since_update += 1 + + # log + self.total_epoch += 1 + if self.logger is not None: + self.logger.record("{}/epoch".format(self.name), epoch) + self.logger.record("{}/epoch_since_update".format(self.name), epochs_since_update) + self.logger.record("{}/train_dataset_size".format(self.name), len(train_loader.dataset)) + self.logger.record("{}/valid_dataset_size".format(self.name), len(valid_loader.dataset)) + self.logger.record("{}/train_loss".format(self.name), train_loss.mean().item()) + self.logger.record("{}/val_loss".format(self.name), eval_loss.mean().item()) + self.logger.record("{}/best_val_loss".format(self.name), best_eval_loss.mean().item()) + + if self.discovery and epoch % train_graph_freq == 0: + self.logger.record("{}/graph_update_grads".format(self.name), grads.abs().mean().item()) + + self.logger.dump(self.total_epoch) + + if self.patience and epochs_since_update >= self.patience: + break + + # saving the best models + self._maybe_set_best_weights_and_elite(best_weights, best_eval_loss) + + self.save(save_dir=work_dir) + + def _maybe_get_best_weights( + self, best_val_loss: torch.Tensor, val_loss: torch.Tensor, threshold: float = 0.01 + ) -> Optional[Dict]: + improvement = (best_val_loss - val_loss) / torch.abs(best_val_loss) + if (improvement > threshold).any().item(): + best_weights = { + "graph": copy.deepcopy(self.graph.parameters[0].detach().clone()), + "model": copy.deepcopy(self.network.state_dict()), + } + else: + best_weights = None + + return best_weights + + def _maybe_set_best_weights_and_elite(self, best_weights: Optional[Dict], best_val_score: torch.Tensor): + if best_weights is not None: + self.network.load_state_dict(best_weights["model"]) + self.graph.set_data(best_weights["graph"]) + + sorted_indices = np.argsort(best_val_score.tolist()) + self.elite_indices = sorted_indices[: self.elite_num] diff --git a/cmrl/models/causal_mech/util.py b/cmrl/models/causal_mech/util.py new file mode 100644 index 0000000..3ec9d6c --- /dev/null +++ b/cmrl/models/causal_mech/util.py @@ -0,0 +1,193 @@ +from typing import Callable, Dict, List, Union, MutableMapping +from collections import defaultdict +import math +import time + +import torch +from torch import Tensor +import torch.nn.functional as F +from torch.utils.data import DataLoader +from torch.optim import Optimizer +from torch.distributions.von_mises import _log_modified_bessel_fn +from tqdm import tqdm + +from cmrl.utils.variables import Variable, ContinuousVariable, DiscreteVariable, BinaryVariable, RadianVariable + + +def von_mises_nll_loss( + input: Tensor, + target: Tensor, + var: Tensor, + full: bool = False, + eps: float = 1e-6, + reduction: str = "mean", +) -> Tensor: + r"""Von Mises negative log likelihood loss. + + Args: + input: loc of the Von Mises distribution. + target: sample from the Von Mises distribution. + var: tensor of positive var(s), one for each of the expectations + in the input (heteroscedastic), or a single one (homoscedastic). + full (bool, optional): include the constant term in the loss calculation. Default: ``False``. + eps (float, optional): value added to var, for stability. Default: 1e-6. + reduction (string, optional): specifies the reduction to apply to the output: + ``'none'`` | ``'mean'`` | ``'sum'``. ``'none'``: no reduction will be applied, + ``'mean'``: the output is the average of all batch member losses, + ``'sum'``: the output is the sum of all batch member losses. + Default: ``'mean'``. + """ + # Entries of var must be non-negative + if torch.any(var < 0): + raise ValueError("var has negative entry/entries") + + # Clamp for stability + var = var.clone() + with torch.no_grad(): + var.clamp_(min=eps) + + concentration = 1 / var + loss = -concentration * torch.cos(input - target) + _log_modified_bessel_fn(concentration, order=0) + if full: + loss += math.log(2 * math.pi) + + if reduction == "mean": + return loss.mean() + elif reduction == "sum": + return loss.sum() + else: + return loss + + +def circular_gaussian_nll_loss( + input: Tensor, + target: Tensor, + var: Tensor, + full: bool = False, + eps: float = 1e-6, + reduction: str = "mean", +) -> Tensor: + # Entries of var must be non-negative + if torch.any(var < 0): + raise ValueError("var has negative entry/entries") + + # Clamp for stability + var = var.clone() + with torch.no_grad(): + var.clamp_(min=eps) + + diff = torch.remainder(input - target, 2 * torch.pi) + diff[diff > torch.pi] = 2 * torch.pi - diff[diff > torch.pi] + loss = 0.5 * (torch.log(var) + diff**2 / var) + if full: + loss += 0.5 * math.log(2 * math.pi) + + if reduction == "mean": + return loss.mean() + elif reduction == "sum": + return loss.sum() + else: + return loss + + +def variable_loss_func( + outputs: Dict[str, torch.Tensor], + targets: Dict[str, torch.Tensor], + output_variables: List[Variable], + device: Union[str, torch.device] = "cpu", +): + dims = list(outputs.values())[0].shape[:-1] + total_loss = torch.zeros(*dims, len(outputs)).to(device) + + for i, var in enumerate(output_variables): + output = outputs[var.name] + target = targets[var.name].to(device) + if isinstance(var, ContinuousVariable): + dim = target.shape[-1] # (xxx, ensemble-num, batch-size, dim) + assert output.shape[-1] == 2 * dim + mean, log_var = output[..., :dim], output[..., dim:] + # clip log_var to avoid nan loss + log_var = torch.clamp(log_var, min=-10, max=10) + loss = F.gaussian_nll_loss(mean, target, log_var.exp(), reduction="none", full=True, eps=1e-4).mean(dim=-1) + total_loss[..., i] = loss + elif isinstance(var, RadianVariable): + dim = target.shape[-1] # (xxx, ensemble-num, batch-size, dim) + assert output.shape[-1] == 2 * dim + mean, log_var = output[..., :dim], output[..., dim:] + loss = circular_gaussian_nll_loss(mean, target, log_var.exp(), reduction="none").mean(dim=-1) + total_loss[..., i] = loss + elif isinstance(var, DiscreteVariable): + # TODO: onehot to int? + raise NotImplementedError + elif isinstance(var, BinaryVariable): + total_loss[..., i] = F.binary_cross_entropy(output, target, reduction="none") + else: + raise NotImplementedError + + if torch.isnan(total_loss[..., i]).any(): + raise ValueError(f"nan loss for {var.name} ({type(var)})") + elif torch.isinf(total_loss[..., i]).any(): + raise ValueError(f"inf loss for {var.name} ({type(var)})") + return total_loss + + +def train_func( + loader: DataLoader, + forward: Callable[[MutableMapping[str, torch.Tensor]], Dict[str, torch.Tensor]], + optimizer: Optimizer, + loss_func: Callable[[MutableMapping[str, torch.Tensor], MutableMapping[str, torch.Tensor]], torch.Tensor], +): + """train for data + + Args: + forward: forward function. + loader: train data-loader. + optimizer: Optimizer + loss_func: loss function + + Returns: tensor of train loss, with shape (xxx, ensemble-num, batch-size). + + """ + batch_loss_list = [] + with tqdm(loader) as pbar: + for inputs, targets in loader: + outputs = forward(inputs) + loss = loss_func(outputs, targets) # ensemble-num, batch-size, output-var-num + + optimizer.zero_grad() + loss.mean().backward() + optimizer.step() + batch_loss_list.append(loss) + + pbar.set_description(f"train loss: {loss.mean().item():.4f}") + pbar.update() + + return torch.cat(batch_loss_list, dim=-2).detach().cpu() + + +def eval_func( + loader: DataLoader, + forward: Callable[[MutableMapping[str, torch.Tensor]], Dict[str, torch.Tensor]], + loss_func: Callable[[MutableMapping[str, torch.Tensor], MutableMapping[str, torch.Tensor]], torch.Tensor], +): + """evaluate for data + + Args: + forward: forward function. + loader: train data-loader. + loss_func: loss function + + Returns: tensor of train loss, with shape (xxx, ensemble-num, batch-size). + + """ + batch_loss_list = [] + with torch.no_grad(): + with tqdm(loader) as pbar: + for inputs, targets in loader: + outputs = forward(inputs) + loss = loss_func(outputs, targets) # ensemble-num, batch-size, output-var-num + batch_loss_list.append(loss) + + pbar.set_description(f"eval loss: {loss.mean().item():.4f}") + pbar.update() + return torch.cat(batch_loss_list, dim=-2).detach().cpu() diff --git a/cmrl/models/constant.py b/cmrl/models/constant.py new file mode 100644 index 0000000..7003ccd --- /dev/null +++ b/cmrl/models/constant.py @@ -0,0 +1,55 @@ +from omegaconf import DictConfig + +NETWORK_CFG = DictConfig( + dict( + _target_="cmrl.models.networks.ParallelMLP", + _partial_=True, + _recursive_=False, + hidden_dims=[200, 200], + bias=True, + activation_fn_cfg=dict(_target_="torch.nn.SiLU"), + ) +) + +ENCODER_CFG = DictConfig( + dict( + _target_="cmrl.models.networks.VariableEncoder", + _partial_=True, + _recursive_=False, + output_dim=100, + hidden_dims=[100], + bias=True, + activation_fn_cfg=dict(_target_="torch.nn.SiLU"), + ) +) + +DECODER_CFG = DictConfig( + dict( + _target_="cmrl.models.networks.VariableDecoder", + _partial_=True, + _recursive_=False, + input_dim=100, + hidden_dims=[100], + bias=True, + activation_fn_cfg=dict(_target_="torch.nn.SiLU"), + ) +) + +OPTIMIZER_CFG = DictConfig( + dict( + _target_="torch.optim.Adam", + _partial_=True, + lr=1e-4, + weight_decay=1e-5, + eps=1e-8, + ) +) + +SCHEDULER_CFG = DictConfig( + dict( + _target_="torch.optim.lr_scheduler.StepLR", + _partial_=True, + step_size=1, + gamma=1, + ) +) diff --git a/cmrl/models/data_loader.py b/cmrl/models/data_loader.py new file mode 100644 index 0000000..e9f5843 --- /dev/null +++ b/cmrl/models/data_loader.py @@ -0,0 +1,103 @@ +from typing import Optional, MutableMapping + +from gym import spaces, Env +import torch +from torch.utils.data import Dataset, default_collate +import numpy as np +from stable_baselines3.common.buffers import ReplayBuffer, DictReplayBuffer + +from cmrl.utils.variables import to_dict_by_space + + +def buffer_to_dict(state_space, action_space, obs2state_fn, replay_buffer: ReplayBuffer, mech: str, device: str = "cpu"): + assert mech in ["transition", "reward_mech", "termination_mech"] + # dict action is not supported by SB3(so not done by cmrl) + assert not isinstance(action_space, spaces.Dict) + assert hasattr(replay_buffer, "extra_obs") + assert hasattr(replay_buffer, "next_extra_obs") + + real_buffer_size = replay_buffer.buffer_size if replay_buffer.full else replay_buffer.pos + + if hasattr(replay_buffer, "extra_obs"): + states = obs2state_fn(replay_buffer.observations[:real_buffer_size, 0], replay_buffer.extra_obs[:real_buffer_size, 0]) + else: + states = replay_buffer.observations[:real_buffer_size, 0] + state_dict = to_dict_by_space(states, state_space, prefix="obs", to_tensor=True) + act_dict = to_dict_by_space(replay_buffer.actions[:real_buffer_size, 0], action_space, prefix="act", to_tensor=True) + + if hasattr(replay_buffer, "next_extra_obs"): + next_states = obs2state_fn( + replay_buffer.next_observations[:real_buffer_size, 0], replay_buffer.next_extra_obs[:real_buffer_size, 0] + ) + else: + next_states = replay_buffer.next_observations[:real_buffer_size, 0] + next_state_dict = to_dict_by_space(next_states, state_space, prefix="next_obs", to_tensor=True) + + inputs = {} + inputs.update(state_dict) + inputs.update(act_dict) + + if mech == "transition": + outputs = next_state_dict + elif mech == "reward_mech": + rewards = replay_buffer.rewards[:real_buffer_size, 0] + rewards_dict = {"reward": torch.from_numpy(rewards[:, None])} + inputs.update(next_state_dict) + outputs = rewards_dict + elif mech == "termination_mech": + terminals = replay_buffer.dones[:real_buffer_size, 0] * (1 - replay_buffer.timeouts[:real_buffer_size, 0]) + terminals_dict = {"terminal": torch.from_numpy(terminals[:, None])} + inputs.update(next_state_dict) + outputs = terminals_dict + else: + raise NotImplementedError("support mechs in [transition, reward_mech, termination_mech] only") + + return inputs, outputs + + +class EnsembleBufferDataset(Dataset): + def __init__( + self, + inputs: MutableMapping, + outputs: MutableMapping, + training: bool = False, + train_ratio: float = 0.8, + ensemble_num: int = 7, + seed: int = 10086, + ): + self.inputs = inputs + self.outputs = outputs + self.training = training + self.train_ratio = train_ratio + self.ensemble_num = ensemble_num + self.seed = seed + self.indexes = None + + size = next(iter(inputs.values())).shape[0] + + np.random.seed(self.seed) + permutation = np.random.permutation(size) + if self.training: + train_indexes = permutation[: int(size * self.train_ratio)] + indexes = [np.random.permutation(train_indexes) for _ in range(self.ensemble_num)] + else: + valid_indexes = permutation[int(size * self.train_ratio) :] + indexes = [valid_indexes for _ in range(self.ensemble_num)] + self.indexes = np.array(indexes).T + + def __getitem__(self, item): + index = self.indexes[item] + + inputs = dict([(key, self.inputs[key][index]) for key in self.inputs]) + outputs = dict([(key, self.outputs[key][index]) for key in self.outputs]) + return inputs, outputs + + def __len__(self): + return len(self.indexes) + + +def collate_fn(data): + inputs, outputs = default_collate(data) + inputs = dict([(key, value.transpose(0, 1)) for key, value in inputs.items()]) + outputs = dict([(key, value.transpose(0, 1)) for key, value in outputs.items()]) + return [inputs, outputs] diff --git a/cmrl/models/dynamics.py b/cmrl/models/dynamics.py new file mode 100644 index 0000000..10367dd --- /dev/null +++ b/cmrl/models/dynamics.py @@ -0,0 +1,84 @@ +import abc +from collections import ChainMap +import pathlib +from typing import Dict, List, Optional, Tuple, Union +from functools import partial + +import numpy as np +import torch +from gym import spaces +from torch.utils.data import DataLoader +from stable_baselines3.common.logger import Logger +from stable_baselines3.common.buffers import ReplayBuffer + +from cmrl.utils.variables import to_dict_by_space +from cmrl.models.causal_mech.base import BaseCausalMech +from cmrl.models.data_loader import buffer_to_dict +from cmrl.types import Obs2StateFnType, State2ObsFnType + + +class Dynamics: + def __init__( + self, + transition: BaseCausalMech, + state_space: spaces.Space, + action_space: spaces.Space, + obs2state_fn: Obs2StateFnType, + state2obs_fn: State2ObsFnType, + reward_mech: Optional[BaseCausalMech] = None, + termination_mech: Optional[BaseCausalMech] = None, + seed: int = 7, + logger: Optional[Logger] = None, + ): + self.transition = transition + self.state_space = state_space + self.action_space = action_space + self.obs2state_fn = obs2state_fn + self.state2obs_fn = state2obs_fn + self.reward_mech = reward_mech + self.termination_mech = termination_mech + self.seed = seed + self.logger = logger + + self.learn_reward = reward_mech is not None + self.learn_termination = termination_mech is not None + + self.device = self.transition.device + pass + + def learn(self, real_replay_buffer: ReplayBuffer, work_dir: Optional[Union[str, pathlib.Path]] = None, **kwargs): + get_dataset = partial( + buffer_to_dict, + state_space=self.state_space, + action_space=self.action_space, + obs2state_fn=self.obs2state_fn, + replay_buffer=real_replay_buffer, + device=self.device + ) + + # transition + self.transition.learn(*get_dataset(mech="transition"), work_dir=work_dir) + # reward-mech + if self.learn_reward: + self.reward_mech.learn(*get_dataset(mech="reward_mech"), work_dir=work_dir) + # termination-mech + if self.learn_termination: + self.termination_mech.learn(*get_dataset(mech="termination_mech"), work_dir=work_dir) + + def step(self, batch_obs, batch_action): + with torch.no_grad(): + obs_dict = to_dict_by_space(batch_obs, self.state_space, "obs", + repeat=7, to_tensor=True, device=self.device) + act_dict = to_dict_by_space(batch_action, self.action_space, "act", + repeat=7, to_tensor=True, device=self.device) + + inputs = ChainMap(obs_dict, act_dict) + outputs = self.transition.forward(inputs) + + batch_next_state = torch.concat([tensor.mean(dim=0)[:, :1] for tensor in outputs.values()], + dim=-1).cpu().numpy() + batch_next_obs = self.state2obs_fn(batch_next_state) + info = { + "origin-next_obs": torch.concat([tensor[:, :, :1] for tensor in outputs.values()], dim=-1).cpu().numpy()} + + return batch_next_obs, None, None, info diff --git a/cmrl/models/dynamics/__init__.py b/cmrl/models/dynamics/__init__.py deleted file mode 100644 index 3b6c6f1..0000000 --- a/cmrl/models/dynamics/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from .base_dynamics import BaseDynamics -from .constraint_based_dynamics import ConstraintBasedDynamics -from .plain_dynamics import PlainEnsembleDynamics diff --git a/cmrl/models/dynamics/base_dynamics.py b/cmrl/models/dynamics/base_dynamics.py deleted file mode 100644 index 8bc05cd..0000000 --- a/cmrl/models/dynamics/base_dynamics.py +++ /dev/null @@ -1,221 +0,0 @@ -import abc -import collections -import pathlib -from typing import Dict, List, Optional, Tuple, Union - -import numpy as np -import torch -from stable_baselines3.common.logger import Logger -from stable_baselines3.common.buffers import ReplayBuffer - -from cmrl.models.reward_mech.base_reward_mech import BaseRewardMech -from cmrl.models.termination_mech.base_termination_mech import BaseTerminationMech -from cmrl.models.transition.base_transition import BaseTransition -from cmrl.types import InteractionBatch -from cmrl.util.transition_iterator import BootstrapIterator, TransitionIterator - - -def split_dict(old_dict: Dict, need_keys: List[str]): - return dict([(key, old_dict[key]) for key in need_keys]) - - -class BaseDynamics: - _MECH_TO_VARIABLE = { - "transition": "batch_next_obs", - "reward_mech": "batch_reward", - "termination_mech": "batch_terminal", - } - _VARIABLE_TO_MECH = dict([(value, key) for key, value in _MECH_TO_VARIABLE.items()]) - - def __init__( - self, - transition: BaseTransition, - learned_reward: bool = True, - reward_mech: Optional[BaseRewardMech] = None, - learned_termination: bool = False, - termination_mech: Optional[BaseTerminationMech] = None, - optim_lr: float = 1e-4, - weight_decay: float = 1e-5, - optim_eps: float = 1e-8, - logger: Optional[Logger] = None, - ): - super(BaseDynamics, self).__init__() - self.transition = transition - self.learned_reward = learned_reward - self.reward_mech = reward_mech - self.learned_termination = learned_termination - self.termination_mech = termination_mech - - self.optim_lr = optim_lr - self.weight_decay = weight_decay - self.optim_eps = optim_eps - self.logger = logger - - self.device = self.transition.device - self.ensemble_num = self.transition.ensemble_num - - self.learn_mech = ["transition"] - self.transition_optimizer = torch.optim.Adam( - self.transition.parameters(), - lr=optim_lr, - weight_decay=weight_decay, - eps=optim_eps, - ) - if self.learned_reward: - self.reward_mech_optimizer = torch.optim.Adam( - self.reward_mech.parameters(), - lr=optim_lr, - weight_decay=weight_decay, - eps=optim_eps, - ) - self.learn_mech.append("reward_mech") - if self.learned_termination: - self.termination_mech_optimizer = torch.optim.Adam( - self.termination_mech.parameters(), - lr=optim_lr, - weight_decay=weight_decay, - eps=optim_eps, - ) - self.learn_mech.append("termination_mech") - - self.total_epoch = {} - for mech in self.learn_mech: - self.total_epoch[mech] = 0 - - @abc.abstractmethod - def learn(self, replay_buffer: ReplayBuffer, **kwargs): - pass - - # auxiliary method for "single batch data" - def get_3d_tensor(self, data: Union[np.ndarray, torch.Tensor], is_ensemble: bool): - if isinstance(data, np.ndarray): - data = torch.from_numpy(data) - if is_ensemble: - if data.ndim == 2: # reward or terminal - data = data.unsqueeze(data.ndim) - return data.to(self.device) - else: - if data.ndim == 1: # reward or terminal - data = data.unsqueeze(data.ndim) - return data.repeat([self.ensemble_num, 1, 1]).to(self.device) - - # auxiliary method for "interaction batch data" - def get_mech_loss( - self, - batch: InteractionBatch, - mech: str = "transition", - loss_type: str = "default", - is_ensemble: bool = False, - ): - data = {} - for attr in batch.attrs: - data[attr] = self.get_3d_tensor(getattr(batch, attr).copy(), is_ensemble=is_ensemble) - model_in = split_dict(data, ["batch_obs", "batch_action"]) - - if loss_type == "default": - loss_type = "mse" if getattr(self, mech).deterministic else "nll" - - variable = self._MECH_TO_VARIABLE[mech] - get_loss = getattr(getattr(self, mech), "get_{}_loss".format(loss_type)) - return get_loss(model_in, data[variable]) - - # auxiliary method for "replay buffer" - def dataset_split( - self, - replay_buffer: ReplayBuffer, - validation_ratio: float = 0.2, - batch_size: int = 256, - shuffle_each_epoch: bool = True, - bootstrap_permutes: bool = False, - ) -> Tuple[TransitionIterator, Optional[TransitionIterator]]: - size = replay_buffer.buffer_size if replay_buffer.full else replay_buffer.pos - data = InteractionBatch( - replay_buffer.observations[:size, 0].astype(np.float32), - replay_buffer.actions[:size, 0], - replay_buffer.next_observations[:size, 0].astype(np.float32), - replay_buffer.rewards[:size, 0], - replay_buffer.dones[:size, 0], - ) - - val_size = int(len(data) * validation_ratio) - train_size = len(data) - val_size - train_data = data[:train_size] - train_iter = BootstrapIterator( - train_data, - batch_size, - self.ensemble_num, - shuffle_each_epoch=shuffle_each_epoch, - permute_indices=bootstrap_permutes, - ) - - val_iter = None - if val_size > 0: - val_data = data[train_size:] - val_iter = TransitionIterator(val_data, batch_size, shuffle_each_epoch=False) - - return train_iter, val_iter - - # auxiliary method for "dataset" - def evaluate( - self, - dataset: TransitionIterator, - mech: str = "transition", - ): - assert not isinstance(dataset, BootstrapIterator) - - batch_loss_list = [] - with torch.no_grad(): - for batch in dataset: - val_loss = self.get_mech_loss(batch, mech=mech, loss_type="mse", is_ensemble=False) - batch_loss_list.append(val_loss) - return torch.cat(batch_loss_list, dim=batch_loss_list[0].ndim - 2).cpu() - - def train( - self, - dataset: TransitionIterator, - mech: str = "transition", - ): - assert isinstance(dataset, BootstrapIterator) - - batch_loss_list = [] - for batch in dataset: - train_loss = self.get_mech_loss(batch, mech=mech, is_ensemble=True) - optim = getattr(self, "{}_optimizer".format(mech)) - optim.zero_grad() - train_loss.mean().backward() - optim.step() - batch_loss_list.append(train_loss) - return torch.cat(batch_loss_list, dim=batch_loss_list[0].ndim - 2).detach().cpu() - - def query(self, obs, action, return_as_np=True): - result = collections.defaultdict(dict) - obs = self.get_3d_tensor(obs, is_ensemble=False) - action = self.get_3d_tensor(action, is_ensemble=False) - for mech in self.learn_mech: - with torch.no_grad(): - mean, logvar = getattr(self, "{}".format(mech)).forward(obs, action) - variable = self.get_variable_by_mech(mech) - if return_as_np: - result[variable]["mean"] = mean.cpu().numpy() - result[variable]["logvar"] = logvar.cpu().numpy() - else: - result[variable]["mean"] = mean.cpu() - result[variable]["logvar"] = logvar.cpu() - return result - - # other auxiliary method - def save(self, save_dir: Union[str, pathlib.Path]): - for mech in self.learn_mech: - getattr(self, mech).save(save_dir=save_dir) - - def load(self, load_dir: Union[str, pathlib.Path], load_device: Optional[str] = None): - for mech in self.learn_mech: - getattr(self, mech).load(load_dir=load_dir, load_device=load_device) - - def get_variable_by_mech(self, mech: str) -> str: - assert mech in self._MECH_TO_VARIABLE - return self._MECH_TO_VARIABLE[mech] - - def get_mach_by_variable(self, variable: str) -> str: - assert variable in self._VARIABLE_TO_MECH - return self._VARIABLE_TO_MECH[variable] diff --git a/cmrl/models/dynamics/constraint_based_dynamics.py b/cmrl/models/dynamics/constraint_based_dynamics.py deleted file mode 100644 index 5463566..0000000 --- a/cmrl/models/dynamics/constraint_based_dynamics.py +++ /dev/null @@ -1,183 +0,0 @@ -import copy -import itertools -import pathlib -from typing import Callable, Dict, List, Optional, Tuple, Union, cast - -import numpy as np -import torch -from stable_baselines3.common.logger import Logger -from stable_baselines3.common.buffers import ReplayBuffer - -from cmrl.models.dynamics.base_dynamics import BaseDynamics -from cmrl.models.networks.mlp import EnsembleMLP -from cmrl.models.reward_mech.base_reward_mech import BaseRewardMech -from cmrl.models.termination_mech.base_termination_mech import BaseTerminationMech -from cmrl.models.transition.base_transition import BaseTransition -from cmrl.models.causal_discovery.CMI_test import TransitionConditionalMutualInformationTest -from cmrl.util.transition_iterator import BootstrapIterator, TransitionIterator -from cmrl.models.util import to_tensor -from cmrl.types import TensorType - - -class ConstraintBasedDynamics(BaseDynamics): - def __init__( - self, - transition: BaseTransition, - learned_reward: bool = True, - reward_mech: Optional[BaseRewardMech] = None, - learned_termination: bool = False, - termination_mech: Optional[BaseTerminationMech] = None, - # trainer - optim_lr: float = 1e-4, - weight_decay: float = 1e-5, - optim_eps: float = 1e-8, - logger: Optional[Logger] = None, - ): - super(ConstraintBasedDynamics, self).__init__( - transition=transition, - learned_reward=learned_reward, - reward_mech=reward_mech, - learned_termination=learned_termination, - termination_mech=termination_mech, - optim_lr=optim_lr, - weight_decay=weight_decay, - optim_eps=optim_eps, - logger=logger, - ) - # self.cmi_test: Optional[EnsembleMLP] = None - # self.build_cmi_test() - # - # self.cmi_test_optimizer = torch.optim.Adam( - # self.cmi_test.parameters(), - # lr=optim_lr, - # weight_decay=weight_decay, - # eps=optim_eps, - # ) - # self.learn_mech.append("cmi_test") - # self.total_epoch["cmi_test"] = 0 - # self._MECH_TO_VARIABLE["cmi_test"] = self._MECH_TO_VARIABLE["transition"] - - for mech in self.learn_mech: - if hasattr(getattr(self, mech), "input_mask"): - setattr(self, "{}_oracle_mask".format(mech), None) - setattr(self, "{}_history_mask".format(mech), torch.ones(getattr(self, mech).input_mask.shape).to(self.device)) - - def build_cmi_test(self): - self.cmi_test = TransitionConditionalMutualInformationTest( - obs_size=self.transition.obs_size, - action_size=self.transition.action_size, - ensemble_num=1, - elite_num=1, - residual=self.transition.residual, - learn_logvar_bounds=self.transition.learn_logvar_bounds, - num_layers=4, - hid_size=200, - activation_fn_cfg=self.transition.activation_fn_cfg, - device=self.transition.device, - ) - - def set_oracle_mask(self, mech: str, mask: TensorType): - assert hasattr(self, "{}_oracle_mask".format(mech)) - setattr(self, "{}_oracle_mask".format(mech), to_tensor(mask)) - - def learn( - self, - # data - replay_buffer: ReplayBuffer, - # dataset split - validation_ratio: float = 0.2, - batch_size: int = 256, - shuffle_each_epoch: bool = True, - bootstrap_permutes: bool = False, - # model learning - longest_epoch: Optional[int] = None, - improvement_threshold: float = 0.1, - patience: int = 5, - work_dir: Optional[Union[str, pathlib.Path]] = None, - # other - **kwargs - ): - train_dataset, val_dataset = self.dataset_split( - replay_buffer, - validation_ratio, - batch_size, - shuffle_each_epoch, - bootstrap_permutes, - ) - - for mech in self.learn_mech: - - if hasattr(self, "{}_oracle_mask".format(mech)): - getattr(self, mech).set_input_mask(getattr(self, "{}_oracle_mask".format(mech))) - - best_weights: Optional[Dict] = None - epoch_iter = range(longest_epoch) if longest_epoch > 0 else itertools.count() - epochs_since_update = 0 - - best_val_loss = self.evaluate(val_dataset, mech=mech).mean(dim=(1, 2)) - - for epoch in epoch_iter: - train_loss = self.train(train_dataset, mech=mech) - val_loss = self.evaluate(val_dataset, mech=mech).mean(dim=(1, 2)) - - maybe_best_weights = self.maybe_get_best_weights( - best_val_loss, - val_loss, - mech, - improvement_threshold, - ) - if maybe_best_weights: - # best loss - best_val_loss = torch.minimum(best_val_loss, val_loss) - best_weights = maybe_best_weights - epochs_since_update = 0 - else: - epochs_since_update += 1 - - # log - self.total_epoch[mech] += 1 - if self.logger is not None: - self.logger.record("{}/epoch".format(mech), epoch) - self.logger.record("{}/train_dataset_size".format(mech), train_dataset.num_stored) - self.logger.record("{}/val_dataset_size".format(mech), val_dataset.num_stored) - self.logger.record("{}/train_loss".format(mech), train_loss.mean().item()) - self.logger.record("{}/val_loss".format(mech), val_loss.mean().item()) - self.logger.record("{}/best_val_loss".format(mech), best_val_loss.mean().item()) - self.logger.dump(self.total_epoch[mech]) - if patience and epochs_since_update >= patience: - break - - # saving the best models: - self.maybe_set_best_weights_and_elite(best_weights, best_val_loss, mech=mech) - self.save(work_dir) - - def maybe_get_best_weights( - self, - best_val_loss: torch.Tensor, - val_loss: torch.Tensor, - mech: str = "transition", - threshold: float = 0.01, - ): - improvement = (best_val_loss - val_loss) / torch.abs(best_val_loss) - if (improvement > threshold).any().item(): - model = getattr(self, mech) - best_weights = copy.deepcopy(model.state_dict()) - else: - best_weights = None - - return best_weights - - def maybe_set_best_weights_and_elite( - self, - best_weights: Optional[Dict], - best_val_loss: torch.Tensor, - mech: str = "transition", - ): - model = getattr(self, mech) - assert isinstance(model, EnsembleMLP) - - if best_weights is not None: - model.load_state_dict(best_weights) - sorted_indices = np.argsort(best_val_loss.tolist()) - elite_models = sorted_indices[: model.elite_num] - model.set_elite_members(elite_models) diff --git a/cmrl/models/dynamics/ncd_dynamics.py b/cmrl/models/dynamics/ncd_dynamics.py deleted file mode 100644 index fa1e071..0000000 --- a/cmrl/models/dynamics/ncd_dynamics.py +++ /dev/null @@ -1,182 +0,0 @@ -import copy -import itertools -import pathlib -from typing import Callable, Dict, List, Optional, Tuple, Union, cast - -import numpy as np -import torch -from stable_baselines3.common.logger import Logger -from stable_baselines3.common.buffers import ReplayBuffer - -from cmrl.models.dynamics.base_dynamics import BaseDynamics -from cmrl.models.networks.mlp import EnsembleMLP -from cmrl.models.reward_mech.base_reward_mech import BaseRewardMech -from cmrl.models.termination_mech.base_termination_mech import BaseTerminationMech -from cmrl.models.transition.base_transition import BaseTransition -from cmrl.models.causal_discovery.CMI_test import TransitionConditionalMutualInformationTest -from cmrl.util.transition_iterator import BootstrapIterator, TransitionIterator -from cmrl.models.util import to_tensor -from cmrl.types import TensorType - - -class ConstraintBasedDynamics(BaseDynamics): - def __init__( - self, - transition: BaseTransition, - learned_reward: bool = True, - reward_mech: Optional[BaseRewardMech] = None, - learned_termination: bool = False, - termination_mech: Optional[BaseTerminationMech] = None, - # trainer - optim_lr: float = 1e-4, - weight_decay: float = 1e-5, - optim_eps: float = 1e-8, - logger: Optional[Logger] = None, - ): - super(ConstraintBasedDynamics, self).__init__( - transition=transition, - learned_reward=learned_reward, - reward_mech=reward_mech, - learned_termination=learned_termination, - termination_mech=termination_mech, - optim_lr=optim_lr, - weight_decay=weight_decay, - optim_eps=optim_eps, - logger=logger, - ) - # self.cmi_test: Optional[EnsembleMLP] = None - # self.build_cmi_test() - # - # self.cmi_test_optimizer = torch.optim.Adam( - # self.cmi_test.parameters(), - # lr=optim_lr, - # weight_decay=weight_decay, - # eps=optim_eps, - # ) - # self.learn_mech.append("cmi_test") - # self.total_epoch["cmi_test"] = 0 - # self._MECH_TO_VARIABLE["cmi_test"] = self._MECH_TO_VARIABLE["transition"] - - for mech in self.learn_mech: - if hasattr(getattr(self, mech), "input_mask"): - setattr(self, "{}_oracle_mask".format(mech), None) - setattr(self, "{}_history_mask".format(mech), torch.ones(getattr(self, mech).input_mask.shape).to(self.device)) - - def build_cmi_test(self): - self.cmi_test = TransitionConditionalMutualInformationTest( - obs_size=self.transition.obs_size, - action_size=self.transition.action_size, - ensemble_num=1, - elite_num=1, - residual=self.transition.residual, - learn_logvar_bounds=self.transition.learn_logvar_bounds, - num_layers=4, - hid_size=200, - activation_fn_cfg=self.transition.activation_fn_cfg, - device=self.transition.device, - ) - - def set_oracle_mask(self, mech: str, mask: TensorType): - assert hasattr(self, "{}_oracle_mask".format(mech)) - setattr(self, "{}_oracle_mask".format(mech), to_tensor(mask)) - - def learn( - self, - # data - replay_buffer: ReplayBuffer, - # dataset split - validation_ratio: float = 0.2, - batch_size: int = 256, - shuffle_each_epoch: bool = True, - bootstrap_permutes: bool = False, - # model learning - longest_epoch: Optional[int] = None, - improvement_threshold: float = 0.1, - patience: int = 5, - work_dir: Optional[Union[str, pathlib.Path]] = None, - # other - **kwargs - ): - train_dataset, val_dataset = self.dataset_split( - replay_buffer, - validation_ratio, - batch_size, - shuffle_each_epoch, - bootstrap_permutes, - ) - - for mech in self.learn_mech: - if hasattr(self, "{}_oracle_mask".format(mech)): - getattr(self, mech).set_input_mask(getattr(self, "{}_oracle_mask".format(mech))) - - best_weights: Optional[Dict] = None - epoch_iter = range(longest_epoch) if longest_epoch > 0 else itertools.count() - epochs_since_update = 0 - - best_val_loss = self.evaluate(val_dataset, mech=mech).mean(dim=(1, 2)) - - for epoch in epoch_iter: - train_loss = self.train(train_dataset, mech=mech) - val_loss = self.evaluate(val_dataset, mech=mech).mean(dim=(1, 2)) - - maybe_best_weights = self.maybe_get_best_weights( - best_val_loss, - val_loss, - mech, - improvement_threshold, - ) - if maybe_best_weights: - # best loss - best_val_loss = torch.minimum(best_val_loss, val_loss) - best_weights = maybe_best_weights - epochs_since_update = 0 - else: - epochs_since_update += 1 - - # log - self.total_epoch[mech] += 1 - if self.logger is not None: - self.logger.record("{}/epoch".format(mech), epoch) - self.logger.record("{}/train_dataset_size".format(mech), train_dataset.num_stored) - self.logger.record("{}/val_dataset_size".format(mech), val_dataset.num_stored) - self.logger.record("{}/train_loss".format(mech), train_loss.mean().item()) - self.logger.record("{}/val_loss".format(mech), val_loss.mean().item()) - self.logger.record("{}/best_val_loss".format(mech), best_val_loss.mean().item()) - self.logger.dump(self.total_epoch[mech]) - if patience and epochs_since_update >= patience: - break - - # saving the best models: - self.maybe_set_best_weights_and_elite(best_weights, best_val_loss, mech=mech) - self.save(work_dir) - - def maybe_get_best_weights( - self, - best_val_loss: torch.Tensor, - val_loss: torch.Tensor, - mech: str = "transition", - threshold: float = 0.01, - ): - improvement = (best_val_loss - val_loss) / torch.abs(best_val_loss) - if (improvement > threshold).any().item(): - model = getattr(self, mech) - best_weights = copy.deepcopy(model.state_dict()) - else: - best_weights = None - - return best_weights - - def maybe_set_best_weights_and_elite( - self, - best_weights: Optional[Dict], - best_val_loss: torch.Tensor, - mech: str = "transition", - ): - model = getattr(self, mech) - assert isinstance(model, EnsembleMLP) - - if best_weights is not None: - model.load_state_dict(best_weights) - sorted_indices = np.argsort(best_val_loss.tolist()) - elite_models = sorted_indices[: model.elite_num] - model.set_elite_members(elite_models) diff --git a/cmrl/models/dynamics/plain_dynamics.py b/cmrl/models/dynamics/plain_dynamics.py deleted file mode 100644 index 317ac46..0000000 --- a/cmrl/models/dynamics/plain_dynamics.py +++ /dev/null @@ -1,142 +0,0 @@ -import copy -import itertools -import pathlib -from typing import Callable, Dict, List, Optional, Tuple, Union, cast - -import numpy as np -import torch -from stable_baselines3.common.logger import Logger -from stable_baselines3.common.buffers import ReplayBuffer - -from cmrl.models.dynamics import BaseDynamics -from cmrl.models.networks.mlp import EnsembleMLP -from cmrl.models.reward_mech.base_reward_mech import BaseRewardMech -from cmrl.models.termination_mech.base_termination_mech import BaseTerminationMech -from cmrl.models.transition.base_transition import BaseTransition - - -class PlainEnsembleDynamics(BaseDynamics): - def __init__( - self, - transition: BaseTransition, - learned_reward: bool = True, - reward_mech: Optional[BaseRewardMech] = None, - learned_termination: bool = False, - termination_mech: Optional[BaseTerminationMech] = None, - # trainer - optim_lr: float = 1e-4, - weight_decay: float = 1e-5, - optim_eps: float = 1e-8, - logger: Optional[Logger] = None, - ): - super(PlainEnsembleDynamics, self).__init__( - transition=transition, - learned_reward=learned_reward, - reward_mech=reward_mech, - learned_termination=learned_termination, - termination_mech=termination_mech, - optim_lr=optim_lr, - weight_decay=weight_decay, - optim_eps=optim_eps, - logger=logger, - ) - - def learn( - self, - # data - replay_buffer: ReplayBuffer, - # dataset split - validation_ratio: float = 0.2, - batch_size: int = 256, - shuffle_each_epoch: bool = True, - bootstrap_permutes: bool = False, - # model learning - longest_epoch: int = -1, - improvement_threshold: float = 0.1, - patience: int = 5, - work_dir: Optional[Union[str, pathlib.Path]] = None, - # other - **kwargs - ): - train_dataset, val_dataset = self.dataset_split( - replay_buffer, - validation_ratio, - batch_size, - shuffle_each_epoch, - bootstrap_permutes, - ) - - for mech in self.learn_mech: - best_weights: Optional[Dict] = None - epoch_iter = range(longest_epoch) if longest_epoch > 0 else itertools.count() - epochs_since_update = 0 - - best_val_loss = self.evaluate(val_dataset, mech=mech).mean(dim=(1, 2)) - - for epoch in epoch_iter: - train_loss = self.train(train_dataset, mech=mech) - val_loss = self.evaluate(val_dataset, mech=mech).mean(dim=(1, 2)) - - maybe_best_weights = self.maybe_get_best_weights( - best_val_loss, - val_loss, - mech, - improvement_threshold, - ) - if maybe_best_weights: - # best loss - best_val_loss = torch.minimum(best_val_loss, val_loss) - best_weights = maybe_best_weights - epochs_since_update = 0 - else: - epochs_since_update += 1 - - # log - self.total_epoch[mech] += 1 - if self.logger is not None: - self.logger.record("{}/epoch".format(mech), epoch) - self.logger.record("{}/train_dataset_size".format(mech), train_dataset.num_stored) - self.logger.record("{}/val_dataset_size".format(mech), val_dataset.num_stored) - self.logger.record("{}/train_loss".format(mech), train_loss.mean().item()) - self.logger.record("{}/val_loss".format(mech), val_loss.mean().item()) - self.logger.record("{}/best_val_loss".format(mech), best_val_loss.mean().item()) - self.logger.dump(self.total_epoch[mech]) - - if patience and epochs_since_update >= patience: - break - - # saving the best models: - self.maybe_set_best_weights_and_elite(best_weights, best_val_loss, mech=mech) - if work_dir is not None: - self.save(work_dir) - - def maybe_get_best_weights( - self, - best_val_loss: torch.Tensor, - val_loss: torch.Tensor, - mech: str = "transition", - threshold: float = 0.01, - ): - improvement = (best_val_loss - val_loss) / torch.abs(best_val_loss) - if (improvement > threshold).any().item(): - model = getattr(self, mech) - best_weights = copy.deepcopy(model.state_dict()) - else: - best_weights = None - - return best_weights - - def maybe_set_best_weights_and_elite( - self, - best_weights: Optional[Dict], - best_val_loss: torch.Tensor, - mech: str = "transition", - ): - model = getattr(self, mech) - assert isinstance(model, EnsembleMLP) - - if best_weights is not None: - model.load_state_dict(best_weights) - sorted_indices = np.argsort(best_val_loss.tolist()) - elite_models = sorted_indices[: model.elite_num] - model.set_elite_members(elite_models) diff --git a/cmrl/models/fake_env.py b/cmrl/models/fake_env.py index 68d195a..0e6aefe 100644 --- a/cmrl/models/fake_env.py +++ b/cmrl/models/fake_env.py @@ -1,124 +1,103 @@ -# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. -from typing import Any, Callable, Dict, List, Optional, Sequence, Type, Union +from typing import Any, Dict, List, Optional, Type import gym import numpy as np import torch -from gym.core import ActType, ObsType -from stable_baselines3.common.vec_env.base_vec_env import ( - VecEnv, - VecEnvIndices, - VecEnvObs, - VecEnvStepReturn, -) +from stable_baselines3.common.vec_env.base_vec_env import VecEnv, VecEnvIndices +from stable_baselines3.common.logger import Logger from stable_baselines3.common.buffers import ReplayBuffer -import cmrl.types -from cmrl.models.dynamics import BaseDynamics +from cmrl.types import RewardFnType, TermFnType, InitObsFnType +from cmrl.models.dynamics import Dynamics + + +def get_penalty(ensemble_batch_next_obs): + avg = np.mean(ensemble_batch_next_obs, axis=0) # average predictions over models + diffs = ensemble_batch_next_obs - avg + dists = np.linalg.norm(diffs, axis=2) # distance in obs space + penalty = np.max(dists, axis=0) # max distances over models + return penalty class VecFakeEnv(VecEnv): def __init__( - self, - num_envs: int, - observation_space: gym.spaces.Space, - action_space: gym.spaces.Space, + self, + # for need of sb3's agent + num_envs: int, + observation_space: gym.spaces.Space, + action_space: gym.spaces.Space, + # for dynamics + dynamics: Dynamics, + reward_fn: Optional[RewardFnType] = None, + termination_fn: Optional[TermFnType] = None, + get_init_obs_fn: Optional[InitObsFnType] = None, + real_replay_buffer: Optional[ReplayBuffer] = None, + # for offline + penalty_coeff: float = 0.0, + # for behaviour + deterministic: bool = False, + max_episode_steps: int = 1000, + branch_rollout: bool = False, + # others + logger: Optional[Logger] = None, + **kwargs, ): super(VecFakeEnv, self).__init__( num_envs=num_envs, observation_space=observation_space, action_space=action_space, ) - - self.has_set_up = False - - self.penalty_coeff = None - self.deterministic = None - self.max_episode_steps = None - - self.dynamics = None - self.reward_fn = None - self.termination_fn = None - self.learned_reward = None - self.learned_termination = None - self.get_init_obs_fn = None - self.replay_buffer = None - self.generator = np.random.default_rng() - self.device = None - self.logger = None - - self._current_batch_obs = None - self._current_batch_action = None - - self._reset_by_buffer = True - - self._envs_length = np.zeros(self.num_envs, dtype=int) - - def set_up( - self, - dynamics: BaseDynamics, - reward_fn: Optional[cmrl.types.RewardFnType] = None, - termination_fn: Optional[cmrl.types.TermFnType] = None, - get_init_obs_fn: Optional[cmrl.types.InitObsFnType] = None, - real_replay_buffer: Optional[ReplayBuffer] = None, - penalty_coeff: float = 0.0, - deterministic=False, - max_episode_steps=1000, - logger=None, - ): self.dynamics = dynamics + self.reward_fn = reward_fn + self.termination_fn = termination_fn + assert self.dynamics.learn_reward or reward_fn, "you must learn a reward-mech or give one" + assert self.dynamics.learn_termination or termination_fn, "you must learn a termination-mech or give one" + self.learn_reward = self.dynamics.learn_reward + self.learn_termination = self.dynamics.learn_termination + self.get_init_obs_fn = get_init_obs_fn + self.replay_buffer = real_replay_buffer self.penalty_coeff = penalty_coeff self.deterministic = deterministic self.max_episode_steps = max_episode_steps + self.branch_rollout = branch_rollout + if self.branch_rollout: + assert self.replay_buffer, "you must provide a replay buffer if using branch-rollout" + else: + assert self.get_init_obs_fn, "you must provide a get-init-obs function if using fully-virtual" - self.reward_fn = reward_fn - self.termination_fn = termination_fn - assert self.dynamics.learned_reward or reward_fn - assert self.dynamics.learned_termination or termination_fn - self.learned_reward = self.dynamics.learned_reward - self.learned_termination = self.dynamics.learned_termination - self.get_init_obs_fn = get_init_obs_fn - self.replay_buffer = real_replay_buffer self.logger = logger - assert self.get_init_obs_fn or self.replay_buffer - self._reset_by_buffer = self.replay_buffer is not None - self.device = dynamics.device - self.has_set_up = True + + self._current_batch_obs = None + self._current_batch_action = None + self._envs_length = np.zeros(self.num_envs, dtype=int) def step_async(self, actions: np.ndarray) -> None: + assert len(actions.shape) == 2 # batch, action_dim self._current_batch_action = actions def step_wait(self): - assert self.has_set_up, "fake-env has not set up" - assert len(self._current_batch_action.shape) == 2 # batch, action_dim - with torch.no_grad(): - batch_obs_tensor = torch.from_numpy(self._current_batch_obs).to(torch.float32).to(self.device) - batch_action_tensor = torch.from_numpy(self._current_batch_action).to(torch.float32).to(self.device) - dynamics_pred = self.dynamics.query(batch_obs_tensor, batch_action_tensor, return_as_np=True) - - # transition - batch_next_obs = self.get_dynamics_predict(dynamics_pred, "transition", deterministic=self.deterministic) - if self.learned_reward: - batch_reward = self.get_dynamics_predict(dynamics_pred, "reward_mech", deterministic=self.deterministic) - else: - batch_reward = self.reward_fn(batch_next_obs, self._current_batch_obs, self._current_batch_action) - if self.learned_termination: - batch_terminal = self.get_dynamics_predict(dynamics_pred, "termination_mech", deterministic=self.deterministic) - else: - batch_terminal = self.termination_fn(batch_next_obs, self._current_batch_obs, self._current_batch_action) - - if self.penalty_coeff != 0: - penalty = self.get_penalty(dynamics_pred["batch_next_obs"]["mean"]).reshape(batch_reward.shape) - batch_reward -= penalty * self.penalty_coeff - - if self.logger is not None: - self.logger.record_mean("rollout/penalty", penalty.mean().item()) + batch_next_obs, batch_reward, batch_terminal, info = self.dynamics.step( + self._current_batch_obs, self._current_batch_action + ) + + if not self.learn_reward: + batch_reward = self.reward_fn(batch_next_obs, self._current_batch_obs, self._current_batch_action) + if not self.learn_termination: + batch_terminal = self.termination_fn(batch_next_obs, self._current_batch_obs, self._current_batch_action) + + if self.penalty_coeff != 0: + penalty = get_penalty(info["origin-next_obs"]).reshape(batch_reward.shape) * self.penalty_coeff + batch_reward -= penalty + + if self.logger is not None: + self.logger.record_mean("rollout/penalty", penalty.mean().item()) + + assert not np.isnan(batch_next_obs).any(), "next obs of fake env should not be nan." + assert not np.isnan(batch_reward).any(), "reward of fake env should not be nan." + assert not np.isnan(batch_terminal).any(), "terminal of fake env should not be nan." self._current_batch_obs = batch_next_obs.copy() batch_reward = batch_reward.reshape(self.num_envs) @@ -142,21 +121,23 @@ def step_wait(self): ) def reset( - self, - *, - seed: Optional[int] = None, - return_info: bool = False, - options: Optional[dict] = None, + self, + *, + seed: Optional[int] = None, + return_info: bool = False, + options: Optional[dict] = None, ): - if self.has_set_up: - if self._reset_by_buffer: - upper_bound = self.replay_buffer.buffer_size if self.replay_buffer.full else self.replay_buffer.pos - batch_inds = np.random.randint(0, upper_bound, size=self.num_envs) - self._current_batch_obs = self.replay_buffer.observations[batch_inds, 0] - else: - self._current_batch_obs = self.get_init_obs_fn(self.num_envs) - self._envs_length = np.zeros(self.num_envs, dtype=int) + if self.branch_rollout: + upper_bound = self.replay_buffer.buffer_size if self.replay_buffer.full else self.replay_buffer.pos + batch_inds = np.random.randint(0, upper_bound, size=self.num_envs) + self._current_batch_obs = self.replay_buffer.observations[batch_inds, 0] + else: + self._current_batch_obs = self.get_init_obs_fn(self.num_envs) + self._envs_length = np.zeros(self.num_envs, dtype=int) + if return_info: + return self._current_batch_obs.copy(), {} + else: return self._current_batch_obs.copy() def seed(self, seed: Optional[int] = None): @@ -169,13 +150,11 @@ def env_is_wrapped(self, wrapper_class: Type[gym.Wrapper], indices: VecEnvIndice return [False for _ in range(self.num_envs)] def single_reset(self, idx): - assert self.has_set_up, "fake-env has not set up" - self._envs_length[idx] = 0 - if self._reset_by_buffer: + if self.branch_rollout: upper_bound = self.replay_buffer.buffer_size if self.replay_buffer.full else self.replay_buffer.pos - batch_inds = np.random.randint(0, upper_bound) - self._current_batch_obs[idx] = self.replay_buffer.observations[batch_inds, 0] + batch_idxs = np.random.randint(0, upper_bound) + self._current_batch_obs[idx] = self.replay_buffer.observations[batch_idxs, 0] else: assert self.get_init_obs_fn is not None self._current_batch_obs[idx] = self.get_init_obs_fn(1) @@ -183,43 +162,12 @@ def single_reset(self, idx): def render(self, mode="human"): raise NotImplementedError - @staticmethod - def get_penalty(ensemble_batch_next_obs): - avg = np.mean(ensemble_batch_next_obs, axis=0) # average predictions over models - diffs = ensemble_batch_next_obs - avg - dists = np.linalg.norm(diffs, axis=2) # distance in obs space - penalty = np.max(dists, axis=0) # max distances over models - - return penalty - - def get_dynamics_predict( - self, - origin_predict: Dict, - mech: str, - deterministic: bool = False, - ): - variable = self.dynamics.get_variable_by_mech(mech) - ensemble_mean, ensemble_logvar = ( - origin_predict[variable]["mean"], - origin_predict[variable]["logvar"], - ) - batch_size = ensemble_mean.shape[1] - random_index = getattr(self.dynamics, mech).get_random_index(batch_size, self.generator) - if deterministic: - pred = ensemble_mean[random_index, np.arange(batch_size)] - else: - ensemble_std = np.sqrt(np.exp(ensemble_logvar)) - pred = ensemble_mean[random_index, np.arange(batch_size)] + ensemble_std[ - random_index, np.arange(batch_size) - ] * self.generator.normal(size=ensemble_mean.shape[1:]).astype(np.float32) - return pred - def env_method( - self, - method_name: str, - *method_args, - indices: VecEnvIndices = None, - **method_kwargs, + self, + method_name: str, + *method_args, + indices: VecEnvIndices = None, + **method_kwargs, ) -> List[Any]: pass diff --git a/cmrl/algorithms/offline/__init__.py b/cmrl/models/graphs/__init__.py similarity index 100% rename from cmrl/algorithms/offline/__init__.py rename to cmrl/models/graphs/__init__.py diff --git a/cmrl/models/graphs/base_graph.py b/cmrl/models/graphs/base_graph.py index df531c8..a75eec2 100644 --- a/cmrl/models/graphs/base_graph.py +++ b/cmrl/models/graphs/base_graph.py @@ -1,123 +1,73 @@ import abc import pathlib -from typing import Any, Dict, Optional, Sequence, Tuple, Union +from typing import Optional, Tuple, Union import torch -import torch.nn as nn -class BaseGraph(nn.Module, abc.ABC): +class BaseGraph(abc.ABC): """Base abstract class for all graph models. All classes derived from `BaseGraph` must implement the following methods: - - ``forward``: computes the graph (parameters). - - ``update``: updates the structural parameters. - - ``get_binary_graph``: gets the binary graph. + - ``parameters``: the graph parameters property. + - ``get_adj_matrix``: get the (raw) adjacency matrix. + - ``get_binary_adj_matrix``: get the binary format of the adjacency matrix. + - ``save``: save the graph data + - ``load``: load the graph data Args: in_dim (int): input dimension. out_dim (int): output dimension. - device (str or torch.device): device to use for the structural parameters. + extra_dim (int | tuple(int) | None): extra dimensions (multi-graph). + include_input (bool): whether include input variables in the output variables. """ - _GRAPH_FNAME = "graph.pth" + def __init__( + self, + in_dim: int, + out_dim: int, + extra_dim: Optional[Union[int, Tuple[int]]] = None, + include_input: bool = False, + *args, + **kwargs + ) -> None: + self._in_dim = in_dim + self._out_dim = out_dim + self._extra_dim = extra_dim + self._include_input = include_input - def __init__(self, in_dim: int, out_dim: int, device: Union[str, torch.device] = "cpu", *args, **kwargs): - super().__init__() - self.in_dim = in_dim - self.out_dim = out_dim - self.device = device + assert not (include_input and out_dim < in_dim), "Once include input, the out dimension must >= in dimension" + @property @abc.abstractmethod - def forward(self, *args, **kwargs) -> Tuple[torch.Tensor, ...]: - """Computes the graph parameters. + def parameters(self) -> Tuple[torch.Tensor]: + """Get the graph parameters (raw graph). - Returns: - (tuple of tensors): all tensors representing the output - graph (e.g. existence and orientation) + Returns: (tuple of tensor) the true graph parameters """ @abc.abstractmethod - def get_binary_graph(self, *args, **kwargs) -> torch.Tensor: - """Gets the binary graph. + def get_adj_matrix(self, *args, **kwargs) -> torch.Tensor: + """Get the raw adjacency matrix. Returns: - (tensor): the binary graph tensor, shape [in_dim, out_dim]; - graph[i, j] == 1 represents i causes j + (tensor): the raw adjacency matrix tensor, shape [in_dim, out_dim]; """ - def get_mask(self, *args, **kwargs) -> torch.Tensor: - # [..., in_dim, out_dim] - binary_mat = self.get_binary_graph(*args, **kwargs) - # [..., out_dim, in_dim], mask apply on the input for each output variable - return binary_mat.transpose(-1, -2) - - def save(self, save_dir: Union[str, pathlib.Path]): - """Saves the model to the given directory.""" - torch.save(self.state_dict(), pathlib.Path(save_dir) / self._GRAPH_FNAME) - - def load(self, load_dir: Union[str, pathlib.Path]): - """Loads the model from the given path.""" - self.load_state_dict(torch.load(pathlib.Path(load_dir) / self._GRAPH_FNAME, map_location=self.device)) - - -class BaseEnsembleGraph(BaseGraph, abc.ABC): - """Base abstract class for all ensemble of bootstrapped 1-D graph models. - - Valid propagation options are: - - - "random_model": for each output in the batch a model will be chosen at random. - - "fixed_model": for output j-th in the batch, the model will be chosen according to - the model index in `propagation_indices[j]`. - - "expectation": the output for each element in the batch will be the mean across - models. - - "majority": the output for each element in the batch will be determined by the - majority voting with the models (only for binary edge). - - The default value of ``None`` indicates that no uncertainty propagation, and the forward - method returns all outpus of all models. - - Args: - num_members (int): number of models in the ensemble. - in_dim (int): input dimension. - out_dim (int): output dimension. - device (str or torch.device): device to use for the model. - propagation_method (str, optional): the uncertainty method to use. Defaults to ``None``. - """ - - def __init__( - self, - num_members: int, - in_dim: int, - out_dim: int, - device: Union[str, torch.device], - propagation_method: str, - *args, - **kwargs - ): - super().__init__(in_dim, out_dim, device, *args, **kwargs) - self.num_members = num_members - self.propagation_method = propagation_method - self.device = torch.device(device) - - def __len__(self): - return self.num_members - - def set_elite(self, elite_grpahs: Sequence[int]): - """For ensemble graphs, indicates if some graphs should be considered elite.""" - pass - @abc.abstractmethod - def sample_propagation_indices(self, batch_size: int, rng: torch.Generator) -> torch.Tensor: - """Samples uncertainty propagation indices. + def get_binary_adj_matrix(self, *args, **kwargs) -> torch.Tensor: + """Get the binary adjacency matrix. - Args: - batch_size (int): the desired batch size. - rng (`torch.Generator`): a random number generator to use for sampling. Returns: - (tensor) with ``batch_size`` integers from [0, ``self.num_members``). + (tensor): the binary adjacency matrix tensor, shape [in_dim, out_dim]; + graph[i, j] == 1 represents i causes j """ - def set_propagation_method(self, propagation_method: Optional[str] = None): - self.propagation_method = propagation_method + @abc.abstractmethod + def save(self, save_dir: Union[str, pathlib.Path]): + """Save the model to the given directory.""" + + @abc.abstractmethod + def load(self, load_dir: Union[str, pathlib.Path]): + """Load the model from the given path.""" diff --git a/cmrl/models/graphs/binary_graph.py b/cmrl/models/graphs/binary_graph.py new file mode 100644 index 0000000..bcdce56 --- /dev/null +++ b/cmrl/models/graphs/binary_graph.py @@ -0,0 +1,81 @@ +import copy +import pathlib +from typing import Optional, Union, Tuple + +import torch +import numpy as np + +from cmrl.models.graphs.base_graph import BaseGraph + + +class BinaryGraph(BaseGraph): + """Binary graph models (binary graph data) + + Args: + in_dim (int): input dimension. + out_dim (int): output dimension. + extra_dim (int | tuple(int) | None): extra dimensions (multi-graph). + include_input (bool): whether inlcude input variables in the output variables. + init_param (int | Tensor | ndarray): initial parameter of the binary graph + device (str or torch.device): device to use for the graph parameters. + """ + + def __init__( + self, + in_dim: int, + out_dim: int, + extra_dim: Optional[Union[int, Tuple[int]]] = None, + include_input: bool = False, + init_param: Union[int, torch.Tensor, np.ndarray] = 1, + device: Union[str, torch.device] = "cpu", + *args, + **kwargs, + ) -> None: + super().__init__(in_dim, out_dim, extra_dim, include_input, *args, **kwargs) + + graph_size = (in_dim, out_dim) + if extra_dim is not None: + if isinstance(extra_dim, int): + extra_dim = (extra_dim,) + graph_size = extra_dim + graph_size + + if isinstance(init_param, int): + self.graph = torch.ones(graph_size, dtype=torch.int, device=device) * int(bool(init_param)) + else: + assert ( + init_param.shape == graph_size + ), f"initial parameters shape mismatch (given {init_param.shape}, while {graph_size} required)" + self.graph = torch.as_tensor(init_param, dtype=torch.bool, device=device).int() + + # remove self loop + if self._include_input: + self.graph[..., torch.arange(self._in_dim), torch.arange(self._in_dim)] = 0 + + self.device = device + + @property + def parameters(self) -> Tuple[torch.Tensor]: + return (self.graph,) + + def get_adj_matrix(self, *args, **kwargs) -> torch.Tensor: + return self.graph + + def get_binary_adj_matrix(self, *args, **kwargs) -> torch.Tensor: + return self.get_adj_matrix() + + def set_data(self, graph_data: Union[torch.Tensor, np.ndarray]): + assert ( + self.graph.shape == graph_data.shape + ), f"graph data shape mismatch (given {graph_data.shape}, while {self.graph.shape} required)" + self.graph.data = torch.as_tensor(graph_data, dtype=torch.bool, device=self.device).int() + + # remove self loop + if self._include_input: + self.graph[..., torch.arange(self._in_dim), torch.arange(self._in_dim)] = 0 + + def save(self, save_dir: Union[str, pathlib.Path]): + torch.save({"graph_data": self.graph}, pathlib.Path(save_dir) / "graph.pth") + + def load(self, load_dir: Union[str, pathlib.Path]): + data_dict = torch.load(pathlib.Path(load_dir) / "graph.pth", map_location=self.device) + self.graph = data_dict["graph_data"] diff --git a/cmrl/models/graphs/neural_graph.py b/cmrl/models/graphs/neural_graph.py new file mode 100644 index 0000000..84935cc --- /dev/null +++ b/cmrl/models/graphs/neural_graph.py @@ -0,0 +1,186 @@ +import pathlib +from typing import Optional, Union, Tuple + +import torch +import torch.nn as nn +import torch.nn.functional as F +from omegaconf import DictConfig +from hydra.utils import instantiate + +from cmrl.models.graphs.base_graph import BaseGraph +from cmrl.models.graphs.prob_graph import BaseProbGraph + +default_network_cfg = DictConfig( + dict( + _target_="cmrl.models.networks.ParallelMLP", + _partial_=True, + _recursive_=False, + hidden_dims=[200, 200], + bias=True, + activation_fn_cfg=dict(_target_="torch.nn.ReLU"), + ) +) + + +class NeuralGraph(BaseGraph): + + _MASK_VALUE = 0 + + def __init__( + self, + in_dim: int, + out_dim: int, + extra_dim: Optional[Union[int, Tuple[int]]] = None, + include_input: bool = False, + network_cfg: Optional[DictConfig] = default_network_cfg, + device: Union[str, torch.device] = "cpu", + *args, + **kwargs + ) -> None: + super().__init__(in_dim=in_dim, out_dim=out_dim, extra_dim=extra_dim, include_input=include_input, *args, **kwargs) + + self._network_cfg = network_cfg + self.device = device + + self._build_graph_network() + + def _build_graph_network(self): + """called at the last of ``NeuralGraph.__init__``""" + network_extra_dims = self._extra_dim + if isinstance(network_extra_dims, int): + network_extra_dims = [network_extra_dims] + + self.graph = instantiate(self._network_cfg)( + input_dim=self._in_dim, + output_dim=self._in_dim * self._out_dim, + extra_dims=network_extra_dims, + ).to(self.device) + + @property + def parameters(self) -> Tuple[torch.Tensor]: + return tuple(self.graph.parameters()) + + def get_adj_matrix(self, inputs: torch.Tensor, *args, **kwargs) -> torch.Tensor: + adj_mat = self.graph(inputs) + adj_mat = adj_mat.reshape(*adj_mat.shape[:-1], self._in_dim, self._out_dim) + + if self._include_input: + adj_mat[..., torch.arange(self._in_dim), torch.arange(self._in_dim)] = self._MASK_VALUE + + return adj_mat + + def get_binary_adj_matrix(self, inputs: torch.Tensor, threshold: float, *args, **kwargs) -> torch.Tensor: + return (self.get_adj_matrix(inputs) > threshold).int() + + def save(self, save_dir: Union[str, pathlib.Path]): + torch.save({"graph_network": self.graph.state_dict()}, pathlib.Path(save_dir) / "graph.pth") + + def load(self, load_dir: Union[str, pathlib.Path]): + data_dict = torch.load(pathlib.Path(load_dir) / "graph.pth", map_location=self.device) + self.graph.load_state_dict(data_dict["graph_network"]) + + +class NeuralBernoulliGraph(NeuralGraph, BaseProbGraph): + + _MASK_VALUE = -9e15 + + def __init__( + self, + in_dim: int, + out_dim: int, + extra_dim: Optional[Union[int, Tuple[int]]] = None, + include_input: bool = False, + network_cfg: Optional[DictConfig] = default_network_cfg, + device: Union[str, torch.device] = "cpu", + *args, + **kwargs + ) -> None: + super().__init__( + in_dim=in_dim, + out_dim=out_dim, + extra_dim=extra_dim, + include_input=include_input, + network_cfg=network_cfg, + device=device, + *args, + **kwargs + ) + + def _build_graph_network(self): + super()._build_graph_network() + + def init_weights_zero(layer): + for pname, params in layer.named_parameters(): + if "weight" in pname: + nn.init.zeros_(params) + + self.graph.apply(init_weights_zero) + + def get_adj_matrix(self, inputs: torch.Tensor, *args, **kwargs) -> torch.Tensor: + return torch.sigmoid(super().get_adj_matrix(inputs, *args, **kwargs)) + + def get_binary_adj_matrix(self, inputs: torch.Tensor, threshold: float = 0.5, *args, **kwargs) -> torch.Tensor: + """return the binary adjacency matrices corresponding to the inputs (w/o grad.)""" + return super().get_binary_adj_matrix(inputs, threshold, *args, **kwargs) + + def sample( + self, + prob_matrix: Optional[torch.Tensor], + sample_size: Union[Tuple[int], int], + reparameterization: Optional[str] = None, + *args, + **kwargs + ) -> torch.Tensor: + """sample from given or current graph probability (Bernoulli distribution). + + Args: + prob_matrix (tensor), graph probability, can not be empty here. + sample_size (tuple(int) or int), extra size of sampled graphs. + + Return: + (tensor): [*sample_size, *extra_dim, in_dim, out_dim] shaped multiple graphs. + """ + if prob_matrix is None: + raise ValueError("Porb. matrix can not be empty") + + if isinstance(sample_size, int): + sample_size = (sample_size,) + + sample_prob = prob_matrix[None].expand(*sample_size, -1, -1) + + if reparameterization is None: + return torch.bernoulli(sample_prob) + elif reparameterization == "gumbel-softmax": + return F.gumbel_softmax(torch.stack((sample_prob, 1 - sample_prob)), hard=True, dim=0)[0] + else: + raise NotImplementedError + + def sample_from_inputs( + self, + inputs: torch.Tensor, + sample_size: Union[Tuple[int], int], + reparameterization: Optional[str] = "gumbel-softmax", + *args, + **kwargs + ) -> torch.Tensor: + """sample adjacency matrix from inputs (genereated Bernoulli distribution given the inputs). + + Args: + inputs (tensor), input samples. + sample_size (tuple(int) or int), extra size of sampled graphs. + + Return: + (tensor): [*sample_size, *extra_dim, in_dim, out_dim] shaped multiple graphs. + """ + if isinstance(sample_size, int): + sample_size = (sample_size,) + + inputs = inputs[None].expand(*sample_size, *((-1,) * len(inputs.shape))) + sample_prob = self.get_adj_matrix(inputs) + + if reparameterization is None: + return torch.bernoulli(sample_prob) + elif reparameterization == "gumbel-softmax": + return F.gumbel_softmax(torch.stack((sample_prob, 1 - sample_prob)), hard=True, dim=0)[0] + else: + raise NotImplementedError diff --git a/cmrl/models/graphs/prob_graph.py b/cmrl/models/graphs/prob_graph.py index ebf7517..6bfc65e 100644 --- a/cmrl/models/graphs/prob_graph.py +++ b/cmrl/models/graphs/prob_graph.py @@ -1,11 +1,12 @@ from abc import abstractmethod -import math from typing import Union, Tuple, Optional import torch -import torch.nn as nn +import numpy as np +import torch.nn.functional as F from cmrl.models.graphs.base_graph import BaseGraph +from cmrl.models.graphs.weight_graph import WeightGraph class BaseProbGraph(BaseGraph): @@ -13,12 +14,14 @@ class BaseProbGraph(BaseGraph): All classes derived from `BaseProbGraph` must implement the following additional methods: - - ``sample``: sample graphs from given (or current) graph probability. + - ``sample``: sample graphs from current (or given) graph probability. """ @abstractmethod - def sample(self, graph: Optional[torch.Tensor], sample_size: Union[Tuple[int], int], *args, **kwargs) -> torch.Tensor: - """sample from given or current graph probability. + def sample( + self, prob_matrix: Optional[torch.Tensor], sample_size: Union[Tuple[int], int], *args, **kwargs + ) -> torch.Tensor: + """sample from given or current probability adjacency matrix. Args: graph (tensor), graph probability, use current graph parameter when given `None`. @@ -30,57 +33,62 @@ def sample(self, graph: Optional[torch.Tensor], sample_size: Union[Tuple[int], i pass -class BernoulliGraph(BaseProbGraph): - """Probability (Bernoulli dist.) modeled graphs, store the graph with the - probability parameter of the existence/orientation of edges. +class BernoulliGraph(WeightGraph, BaseProbGraph): + """Probability (Bernoulli dist.) graph models, store the graph with the + probability parameter of the existence of edges. Args: in_dim (int): input dimension. out_dim (int): output dimension. - init_param (float or torch.Tensor): initial parameter of the graph - (sigmoid(init_param) representing the initial edge probabilities). - device (str or torch.device): device to use for the structural parameters. + extra_dim (int | tuple(int) | None): extra dimensions (multi-graph). + include_input (bool): whether inlcude input variables in the output variables. + init_param (int | Tensor | ndarray): initial parameter of the bernoulli graph。 + requires_grad (bool): whether the graph parameters require gradient computation. + device (str or torch.device): device to use for the graph parameters. """ + _MASK_VALUE = -9e15 + def __init__( self, in_dim: int, out_dim: int, - init_param: Union[float, torch.Tensor] = 1e-6, + extra_dim: Optional[Union[int, Tuple[int]]] = None, + include_input: bool = False, + init_param: Union[float, torch.Tensor, np.ndarray] = 1e-6, + requires_grad: bool = False, device: Union[str, torch.device] = "cpu", *args, **kwargs - ): - super().__init__(in_dim, out_dim, device, *args, **kwargs) - - if isinstance(init_param, float): - init_param = torch.ones(in_dim, out_dim) * init_param - self.graph = nn.Parameter(init_param, requires_grad=True) - - self.to(device) - - def forward(self, *args, **kwargs) -> Tuple[torch.Tensor, ...]: - """Computes the graph parameters. - - Returns: - (tuple of tensors): all tensors representing the output - graph (e.g. existence and orientation) - """ + ) -> None: + super().__init__( + in_dim=in_dim, + out_dim=out_dim, + extra_dim=extra_dim, + include_input=include_input, + init_param=init_param, + requires_grad=requires_grad, + device=device, + *args, + **kwargs + ) + + def get_adj_matrix(self, *args, **kwargs) -> torch.Tensor: return torch.sigmoid(self.graph) - def get_binary_graph(self, thresh: float = 0.5) -> torch.Tensor: - """Gets the binary graph. + def get_binary_adj_matrix(self, threshold: float = 0.5, *args, **kwargs) -> torch.Tensor: + assert 0 <= threshold <= 1, "threshold of bernoulli graph should be in [0, 1]" - Returns: - (tensor): the binary graph tensor, shape [in_dim, out_dim]; - graph[i, j] == 1 represents i causes j - """ - assert 0 <= thresh <= 1 - - prob_graph = self() - return prob_graph > thresh + return super().get_binary_adj_matrix(threshold, *args, **kwargs) - def sample(self, graph: Optional[torch.Tensor], sample_size: Union[Tuple[int], int], *args, **kwargs): + def sample( + self, + prob_matrix: Optional[torch.Tensor], + sample_size: Union[Tuple[int], int], + reparameterization: Optional[str] = None, + *args, + **kwargs + ): """sample from given or current graph probability (Bernoulli distribution). Args: @@ -88,14 +96,19 @@ def sample(self, graph: Optional[torch.Tensor], sample_size: Union[Tuple[int], i sample_size (tuple(int) or int), extra size of sampled graphs. Return: - (tensor): [*sample_size, in_dim, out_dim] shaped multiple graphs. + (tensor): [*sample_size, *extra_dim, in_dim, out_dim] shaped multiple graphs. """ - if graph is None: - graph = self() + if prob_matrix is None: + prob_matrix = self.get_adj_matrix() if isinstance(sample_size, int): sample_size = (sample_size,) - sample_prob = graph[None].expand(*sample_size, -1, -1) + sample_prob = prob_matrix[None].expand(*sample_size, *((-1,) * len(prob_matrix.shape))) - return torch.bernoulli(sample_prob) + if reparameterization is None: + return torch.bernoulli(sample_prob) + elif reparameterization == "gumbel-softmax": + return F.gumbel_softmax(torch.stack((sample_prob, 1 - sample_prob)), hard=True, dim=0)[0] + else: + raise NotImplementedError diff --git a/cmrl/models/graphs/weight_graph.py b/cmrl/models/graphs/weight_graph.py new file mode 100644 index 0000000..a752551 --- /dev/null +++ b/cmrl/models/graphs/weight_graph.py @@ -0,0 +1,94 @@ +import pathlib +from typing import Optional, Union, Tuple + +import torch +import numpy as np + +from cmrl.models.graphs.base_graph import BaseGraph + + +class WeightGraph(BaseGraph): + """Weight graph models (real graph data) + + Args: + in_dim (int): input dimension. + out_dim (int): output dimension. + extra_dim (int | tuple(int) | None): extra dimensions (multi-graph). + include_input (bool): whether inlcude input variables in the output variables. + init_param (int | Tensor | ndarray): initial parameter of the weight graph。 + requires_grad (bool): whether the graph parameters require gradient computation. + device (str or torch.device): device to use for the graph parameters. + """ + + _MASK_VALUE = 0 + + def __init__( + self, + in_dim: int, + out_dim: int, + extra_dim: Optional[Union[int, Tuple[int]]] = None, + include_input: bool = False, + init_param: Union[float, torch.Tensor, np.ndarray] = 1.0, + requires_grad: bool = False, + device: Union[str, torch.device] = "cpu", + *args, + **kwargs, + ) -> None: + super().__init__(in_dim, out_dim, extra_dim, include_input, *args, **kwargs) + self._requires_grad = requires_grad + + graph_size = (in_dim, out_dim) + if extra_dim is not None: + if isinstance(extra_dim, int): + extra_dim = (extra_dim,) + graph_size = extra_dim + graph_size + + if isinstance(init_param, float): + self.graph = torch.ones(graph_size, dtype=torch.float32, device=device) * init_param + else: + assert ( + init_param.shape == graph_size + ), f"initial parameters shape mismatch (given {init_param.shape}, while {graph_size} required)" + self.graph = torch.as_tensor(init_param, dtype=torch.float32, device=device) + + if requires_grad: + self.graph.requires_grad_() + + # remove self loop + if self._include_input: + with torch.no_grad(): + self.graph[..., torch.arange(self._in_dim), torch.arange(self._in_dim)] = self._MASK_VALUE + + self.device = device + + @property + def parameters(self) -> Tuple[torch.Tensor]: + return (self.graph,) + + @property + def requries_grad(self) -> bool: + return self._requires_grad + + def get_adj_matrix(self, *args, **kwargs) -> torch.Tensor: + return self.graph + + def get_binary_adj_matrix(self, threshold: float, *args, **kwargs) -> torch.Tensor: + return (self.get_adj_matrix() > threshold).int() + + @torch.no_grad() + def set_data(self, graph_data: Union[torch.Tensor, np.ndarray]): + assert ( + self.graph.shape == graph_data.shape + ), f"graph data shape mismatch (given {graph_data.shape}, while {self.graph.shape} required)" + self.graph.data = torch.as_tensor(graph_data, dtype=torch.float32, device=self.device) + + # remove self loop + if self._include_input: + self.graph[..., torch.arange(self._in_dim), torch.arange(self._in_dim)] = self._MASK_VALUE + + def save(self, save_dir: Union[str, pathlib.Path]): + torch.save({"graph_data": self.graph}, pathlib.Path(save_dir) / "graph.pth") + + def load(self, load_dir: Union[str, pathlib.Path]): + data_dict = torch.load(pathlib.Path(load_dir) / "graph.pth", map_location=self.device) + self.graph = data_dict["graph_data"] diff --git a/cmrl/models/layers.py b/cmrl/models/layers.py index 5222dc6..b92dca7 100644 --- a/cmrl/models/layers.py +++ b/cmrl/models/layers.py @@ -1,117 +1,92 @@ +from typing import Optional, List + import numpy as np import torch from torch import nn as nn +from torch import Tensor +from itertools import product + +from cmrl.models.util import truncated_normal_ -import cmrl.models.util as model_util - - -def truncated_normal_init(m: nn.Module): - """Initializes the weights of the given module using a truncated normal distribution.""" - - if isinstance(m, nn.Linear): - input_dim = m.weight.data.shape[0] - stddev = 1 / (2 * np.sqrt(input_dim)) - model_util.truncated_normal_(m.weight.data, std=stddev) - m.bias.data.fill_(0.0) - elif isinstance(m, EnsembleLinearLayer): - num_members, input_dim, _ = m.weight.data.shape - stddev = 1 / (2 * np.sqrt(input_dim)) - for i in range(num_members): - model_util.truncated_normal_(m.weight.data[i], std=stddev) - m.bias.data.fill_(0.0) - elif isinstance(m, ParallelEnsembleLinearLayer): - num_parallel, num_members, input_dim, _ = m.weight.data.shape - stddev = 1 / (2 * np.sqrt(input_dim)) - for i in range(num_parallel): - for j in range(num_members): - model_util.truncated_normal_(m.weight.data[i, j], std=stddev) - m.bias.data.fill_(0.0) - - -class EnsembleLinearLayer(nn.Module): - """Implements an ensemble of layers. - - Args: - in_size (int): the input size of this layer. - out_size (int): the output size of this layer. - use_bias (bool): use bias in this layer or not. - ensemble_num (int): the ensemble dimension of this layer, - the corresponding part of each dimension is called a "member". - """ +# partial from https://github.com/phlippe/ENCO/blob/main/causal_discovery/multivariable_mlp.py +class ParallelLinear(nn.Module): def __init__( self, - in_size: int, - out_size: int, - use_bias: bool = True, - ensemble_num: int = 1, + input_dim: int, + output_dim: int, + extra_dims: Optional[List[int]] = None, + bias: bool = True, + init_type: str = "truncated_normal", ): + """Linear layer with the same properties as Parallel MLP. It effectively applies N independent linear layers + in parallel. + + Args: + input_dim: Number of input dimensions per layer. + output_dim: Number of output dimensions per layer. + extra_dims: Number of neural networks to have in parallel (e.g. number of variables). Can have multiple + dimensions if needed. + bias: Weather using bias in this layer. + init_type: How to initialize weights and biases. + """ super().__init__() - self.ensemble_num = ensemble_num - self.in_size = in_size - self.out_size = out_size - self.weight = nn.Parameter(torch.rand(self.ensemble_num, self.in_size, self.out_size)) - if use_bias: - self.bias = nn.Parameter(torch.rand(self.ensemble_num, 1, self.out_size)) + self.input_dim = input_dim + self.output_dim = output_dim + self.extra_dims = [] if extra_dims is None else extra_dims + self.init_type = init_type + + self.weight = nn.Parameter(torch.zeros(*self.extra_dims, self.input_dim, self.output_dim)) + if bias: + self.bias = nn.Parameter(torch.zeros(*self.extra_dims, 1, self.output_dim)) self.use_bias = True else: self.use_bias = False - def forward(self, x): - xw = x.matmul(self.weight) - if self.use_bias: - return xw + self.bias - else: - return xw - - def __repr__(self) -> str: - return ( - f"in_size={self.in_size}, out_size={self.out_size}, use_bias={self.use_bias}, " f"ensemble_num={self.ensemble_num}" - ) + self.init_params() + def init_params(self): + """Initialize weights and biases. Currently, only `kaiming_uniform` and `truncated_normal` are supported. -class ParallelEnsembleLinearLayer(nn.Module): - """Implements an ensemble of parallel layers. + Returns: None - Args: - in_size (int): the input size of this layer. - out_size (int): the output size of this layer. - use_bias (bool): use bias in this layer or not. - parallel_num (int): the parallel dimension of this layer, - the corresponding part of each dimension is called a "sub-network". - ensemble_num (int): the ensemble dimension of this layer, - the corresponding part of each dimension is called a "member". - """ - - def __init__( - self, - in_size: int, - out_size: int, - use_bias: bool = True, - parallel_num: int = 1, - ensemble_num: int = 1, - ): - super().__init__() - self.parallel_num = parallel_num - self.ensemble_num = ensemble_num - self.in_size = in_size - self.out_size = out_size - self.weight = nn.Parameter(torch.rand(self.parallel_num, self.ensemble_num, self.in_size, self.out_size)) - if use_bias: - self.bias = nn.Parameter(torch.rand(self.parallel_num, self.ensemble_num, 1, self.out_size)) - self.use_bias = True + """ + if self.init_type == "kaiming_uniform": + nn.init.kaiming_uniform_(self.weight, nonlinearity="relu") + elif self.init_type == "truncated_normal": + stddev = 1 / (2 * np.sqrt(self.input_dim)) + for dims in product(*map(range, self.extra_dims)): + truncated_normal_(self.weight.data[dims], std=stddev) else: - self.use_bias = False + raise NotImplementedError - def forward(self, x): + def forward(self, x: Tensor) -> Tensor: xw = x.matmul(self.weight) if self.use_bias: return xw + self.bias else: return xw - def __repr__(self) -> str: - return ( - f"in_size={self.in_size}, out_size={self.out_size}, use_bias={self.use_bias}, " - f"parallel_num={self.parallel_num}, ensemble_num={self.ensemble_num}" + @property + def device(self) -> torch.device: + """Infer which device this policy lives on by inspecting its parameters. + If it has no parameters, the 'cpu' device is used as a fallback. + + Returns: device + """ + for param in self.parameters(): + return param.device + return torch.device("cpu") + + def extra_repr(self): + return 'input_dims={}, output_dims={}, extra_dims={}, bias={}, init_type="{}"'.format( + self.input_dim, self.output_dim, str(self.extra_dims), self.use_bias, self.init_type ) + + +class RadianLayer(nn.Module): + def __init__(self) -> None: + super(RadianLayer, self).__init__() + + def forward(self, input: Tensor) -> Tensor: + return torch.remainder(input + torch.pi, 2 * torch.pi) - torch.pi diff --git a/cmrl/models/networks/__init__.py b/cmrl/models/networks/__init__.py new file mode 100644 index 0000000..8563cbc --- /dev/null +++ b/cmrl/models/networks/__init__.py @@ -0,0 +1,2 @@ +from cmrl.models.networks.coder import VariableEncoder, VariableDecoder +from cmrl.models.networks.parallel_mlp import ParallelMLP diff --git a/cmrl/models/networks/base_network.py b/cmrl/models/networks/base_network.py new file mode 100644 index 0000000..ab32a5e --- /dev/null +++ b/cmrl/models/networks/base_network.py @@ -0,0 +1,71 @@ +import pathlib +from typing import List, Optional, Sequence, Union +from abc import abstractmethod + +import torch +import torch.nn as nn +import hydra +from omegaconf import DictConfig + + +class BaseNetwork(nn.Module): + def __init__(self, **kwargs): + """Base class of all neural network. + + Args: + network_cfg: + """ + super(BaseNetwork, self).__init__() + + self._model_filename = "base_network.pth" + self._save_attrs: List[str] = ["state_dict"] + self._layers: Optional[nn.ModuleList] = None + + self.build() + + def save(self, save_dir: Union[str, pathlib.Path]): + """Saves the model to the given directory.""" + model_dict = {} + for attr in self._save_attrs: + if attr == "state_dict": + model_dict["state_dict"] = self.state_dict() + else: + model_dict[attr] = getattr(self, attr) + torch.save(model_dict, pathlib.Path(save_dir) / self._model_filename) + + def load(self, load_dir: Union[str, pathlib.Path]): + """Loads the model from the given path.""" + model_dict = torch.load(pathlib.Path(load_dir) / self._model_filename, map_location=self.device) + for attr in model_dict: + if attr == "state_dict": + self.load_state_dict(model_dict["state_dict"]) + else: + getattr(self, attr)(model_dict[attr]) + + def forward(self, x) -> torch.Tensor: + for layer in self._layers: + x = layer(x) + return x + + @abstractmethod + def build(self): + raise NotImplementedError + + @property + def save_attrs(self): + return self._save_attrs + + @property + def model_filename(self): + return self._model_filename + + @property + def device(self): + return next(iter(self.parameters())).device + + +def create_activation(activation_fn_cfg: DictConfig): + if activation_fn_cfg is None: + return nn.ReLU() + else: + return hydra.utils.instantiate(activation_fn_cfg) diff --git a/cmrl/models/networks/coder.py b/cmrl/models/networks/coder.py new file mode 100644 index 0000000..ca0a133 --- /dev/null +++ b/cmrl/models/networks/coder.py @@ -0,0 +1,104 @@ +from typing import List, Optional + +import torch.nn as nn +from omegaconf import DictConfig + +from cmrl.utils.variables import Variable, DiscreteVariable, ContinuousVariable, BinaryVariable, RadianVariable +from cmrl.models.networks.base_network import BaseNetwork, create_activation +from cmrl.models.layers import RadianLayer + + +class VariableEncoder(BaseNetwork): + def __init__( + self, + variable: Variable, + output_dim: int = 100, + hidden_dims: Optional[List[int]] = None, + bias: bool = True, + activation_fn_cfg: Optional[DictConfig] = None, + ): + self.variable = variable + self.output_dim = output_dim + self.hidden_dims = hidden_dims if hidden_dims is not None else [] + self.bias = bias + self.activation_fn_cfg = activation_fn_cfg + + self.name = "{}_encoder".format(variable.name) + + super(VariableEncoder, self).__init__() + self._model_filename = "{}.pth".format(self.name) + + def build(self): + layers = [] + if len(self.hidden_dims) == 0: + hidden_dim = self.output_dim + else: + hidden_dim = self.hidden_dims[0] + + if isinstance(self.variable, ContinuousVariable): + layers.append(nn.Linear(self.variable.dim, hidden_dim)) + elif isinstance(self.variable, RadianVariable): + layers.append(RadianLayer()) + layers.append(nn.Linear(self.variable.dim, hidden_dim)) + elif isinstance(self.variable, DiscreteVariable): + layers.append(nn.Linear(self.variable.n, hidden_dim)) + elif isinstance(self.variable, BinaryVariable): + layers.append(nn.Linear(1, hidden_dim)) + else: + raise NotImplementedError("Type {} is not supported by VariableEncoder".format(type(self.variable))) + + hidden_dims = self.hidden_dims + [self.output_dim] + for i in range(len(hidden_dims) - 1): + layers += [nn.Linear(hidden_dims[i], hidden_dims[i + 1], bias=self.bias)] + layers += [create_activation(self.activation_fn_cfg)] + + self._layers = nn.ModuleList(layers) + + +class VariableDecoder(BaseNetwork): + def __init__( + self, + variable: Variable, + input_dim: int = 100, + hidden_dims: Optional[List[int]] = None, + bias: bool = True, + activation_fn_cfg: Optional[DictConfig] = None, + ): + self.variable = variable + self.input_dim = input_dim + self.hidden_dims = hidden_dims if hidden_dims is not None else [] + self.bias = bias + self.activation_fn_cfg = activation_fn_cfg + + self.name = "{}_decoder".format(variable.name) + + super(VariableDecoder, self).__init__() + self._model_filename = "{}.pth".format(self.name) + + def build(self): + layers = [create_activation(self.activation_fn_cfg)] + + hidden_dims = [self.input_dim] + self.hidden_dims + for i in range(len(hidden_dims) - 1): + layers += [nn.Linear(hidden_dims[i], hidden_dims[i + 1], bias=self.bias)] + layers += [create_activation(self.activation_fn_cfg)] + + if len(self.hidden_dims) == 0: + hidden_dim = self.input_dim + else: + hidden_dim = self.hidden_dims[-1] + + if isinstance(self.variable, ContinuousVariable): + layers.append(nn.Linear(hidden_dim, self.variable.dim * 2)) + elif isinstance(self.variable, RadianVariable): + layers.append(nn.Linear(hidden_dim, self.variable.dim * 2)) + elif isinstance(self.variable, DiscreteVariable): + layers.append(nn.Linear(hidden_dim, self.variable.n)) + layers.append(nn.Softmax()) + elif isinstance(self.variable, BinaryVariable): + layers.append(nn.Linear(hidden_dim, 1)) + layers.append(nn.Sigmoid()) + else: + raise NotImplementedError("Type {} is not supported by VariableDecoder".format(type(self.variable))) + + self._layers = nn.ModuleList(layers) diff --git a/cmrl/models/networks/mlp.py b/cmrl/models/networks/mlp.py deleted file mode 100644 index 9b9e8c3..0000000 --- a/cmrl/models/networks/mlp.py +++ /dev/null @@ -1,100 +0,0 @@ -import pathlib -from typing import Dict, Optional, Sequence, Union - -import numpy as np -import torch -import torch.nn as nn -import torch.nn.functional as F - -from cmrl.models.util import gaussian_nll -from cmrl.models.layers import EnsembleLinearLayer - - -class EnsembleMLP(nn.Module): - _MODEL_FILENAME = "ensemble_mlp.pth" - - def __init__( - self, - ensemble_num: int = 7, - elite_num: int = 5, - device: Union[str, torch.device] = "cpu", - ): - super(EnsembleMLP, self).__init__() - self.ensemble_num = ensemble_num - self.elite_num = elite_num - self.device = device - - self._elite_members: Optional[Sequence[int]] = np.random.permutation(ensemble_num)[:elite_num] - - self._model_save_attrs = ["elite_members", "state_dict"] - - def set_elite_members(self, elite_indices: Sequence[int]): - if len(elite_indices) != self.ensemble_num: - assert len(elite_indices) == self.elite_num - self._elite_members = list(elite_indices) - - @property - def elite_members(self): - return self._elite_members - - def get_random_index(self, batch_size: int, numpy_generator: Optional[np.random.Generator] = None): - if numpy_generator: - return numpy_generator.choice(self._elite_members, size=batch_size) - else: - return np.random.choice(self._elite_members, size=batch_size) - - def save(self, save_dir: Union[str, pathlib.Path]): - """Saves the model to the given directory.""" - model_dict = {} - for attr in self._model_save_attrs: - if attr == "state_dict": - model_dict["state_dict"] = self.state_dict() - else: - model_dict[attr] = getattr(self, attr) - torch.save(model_dict, pathlib.Path(save_dir) / self._MODEL_FILENAME) - - def load(self, load_dir: Union[str, pathlib.Path], load_device: Optional[str] = None): - """Loads the model from the given path.""" - model_dict = torch.load(pathlib.Path(load_dir) / self._MODEL_FILENAME, map_location=load_device) - for attr in model_dict: - if attr == "state_dict": - self.load_state_dict(model_dict["state_dict"]) - else: - getattr(self, "set_" + attr)(model_dict[attr]) - - def create_linear_layer(self, l_in, l_out): - return EnsembleLinearLayer(l_in, l_out, ensemble_num=self.ensemble_num) - - def get_mse_loss(self, model_in: Dict[(str, torch.Tensor)], target: torch.Tensor) -> torch.Tensor: - pred_mean, pred_logvar = self.forward(**model_in) - return F.mse_loss(pred_mean, target, reduction="none") - - def get_nll_loss(self, model_in: Dict[(str, torch.Tensor)], target: torch.Tensor) -> torch.Tensor: - pred_mean, pred_logvar = self.forward(**model_in) - nll_loss = gaussian_nll(pred_mean, pred_logvar, target, reduce=False) - nll_loss += 0.01 * (self.max_logvar.sum() - self.min_logvar.sum()) - return nll_loss - - def add_save_attr(self, attr: str): - assert hasattr(self, attr), "Class must has attribute {}".format(attr) - assert attr not in self._model_save_attrs, "Attribute {} has been in model-save-list".format(attr) - self._model_save_attrs.append(attr) - - @property - def save_attr(self): - return self._model_save_attrs - - @property - def model_file_name(self): - return self._MODEL_FILENAME - - -class ExternalMaskEnsembleMLP(EnsembleMLP): - """Ensemble of multi-layer perceptrons with input mask inside - - Args: - TODO - """ - - def __init__(self, ensemble_num: int = 7, elite_num: int = 5, device: Union[str, torch.device] = "cpu"): - super().__init__(ensemble_num, elite_num, device) diff --git a/cmrl/models/networks/parallel_mlp.py b/cmrl/models/networks/parallel_mlp.py new file mode 100644 index 0000000..e322f19 --- /dev/null +++ b/cmrl/models/networks/parallel_mlp.py @@ -0,0 +1,52 @@ +import pathlib +from typing import List, Optional, Sequence, Union +from abc import abstractmethod + +import torch +import torch.nn as nn +import hydra +from omegaconf import DictConfig + +from cmrl.models.layers import ParallelLinear +from cmrl.models.networks.base_network import BaseNetwork, create_activation + + +# partial from https://github.com/phlippe/ENCO/blob/main/causal_discovery/multivariable_mlp.py +class ParallelMLP(BaseNetwork): + def __init__( + self, + input_dim: int, + output_dim: int, + extra_dims: Optional[List[int]] = None, + hidden_dims: Optional[List[int]] = None, + bias: bool = True, + init_type: str = "truncated_normal", + activation_fn_cfg: Optional[DictConfig] = None, + **kwargs + ): + self.input_dim = input_dim + self.output_dim = output_dim + self.extra_dims = extra_dims if extra_dims is not None else [] + self.hidden_dims = hidden_dims if hidden_dims is not None else [200, 200, 200, 200] + self.bias = bias + self.init_type = init_type + self.activation_fn_cfg = activation_fn_cfg + + super().__init__(**kwargs) + self._model_filename = "parallel_mlp.pth" + + def build(self): + layers = [] + hidden_dims = [self.input_dim] + self.hidden_dims + for i in range(len(hidden_dims) - 1): + layers += [ + ParallelLinear( + input_dim=hidden_dims[i], output_dim=hidden_dims[i + 1], extra_dims=self.extra_dims, bias=self.bias + ) + ] + layers += [create_activation(self.activation_fn_cfg)] + layers += [ + ParallelLinear(input_dim=hidden_dims[-1], output_dim=self.output_dim, extra_dims=self.extra_dims, bias=self.bias) + ] + + self._layers = nn.ModuleList(layers) diff --git a/cmrl/algorithms/online/__init__.py b/cmrl/models/networks/util.py similarity index 100% rename from cmrl/algorithms/online/__init__.py rename to cmrl/models/networks/util.py diff --git a/cmrl/models/reward_mech/__init__.py b/cmrl/models/reward_mech/__init__.py deleted file mode 100644 index 77ea159..0000000 --- a/cmrl/models/reward_mech/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from cmrl.models.reward_mech.base_reward_mech import BaseRewardMech - -from cmrl.models.reward_mech.plain_reward_mech import PlainRewardMech diff --git a/cmrl/models/reward_mech/base_reward_mech.py b/cmrl/models/reward_mech/base_reward_mech.py deleted file mode 100644 index f2addef..0000000 --- a/cmrl/models/reward_mech/base_reward_mech.py +++ /dev/null @@ -1,26 +0,0 @@ -from typing import Union - -import torch - -from cmrl.models.networks.mlp import EnsembleMLP - - -class BaseRewardMech(EnsembleMLP): - _MODEL_FILENAME = "base_reward_mech.pth" - - def __init__( - self, - obs_size: int, - action_size: int, - deterministic: bool = False, - ensemble_num: int = 7, - elite_num: int = 5, - device: Union[str, torch.device] = "cpu", - ): - super(BaseRewardMech, self).__init__(ensemble_num=ensemble_num, elite_num=elite_num, device=device) - self.obs_size = obs_size - self.action_size = action_size - self.deterministic = deterministic - - def forward(self, state: torch.Tensor, action: torch.Tensor): - pass diff --git a/cmrl/models/reward_mech/plain_reward_mech.py b/cmrl/models/reward_mech/plain_reward_mech.py deleted file mode 100644 index a111471..0000000 --- a/cmrl/models/reward_mech/plain_reward_mech.py +++ /dev/null @@ -1,94 +0,0 @@ -from typing import Dict, Optional, Tuple, Union - -import hydra -import omegaconf -import torch -from torch import nn as nn -from torch.nn import functional as F - -from cmrl.models.layers import truncated_normal_init -from cmrl.models.reward_mech.base_reward_mech import BaseRewardMech - - -class PlainRewardMech(BaseRewardMech): - _MODEL_FILENAME = "plain_reward_mech.pth" - - def __init__( - self, - # transition info - obs_size: int, - action_size: int, - deterministic: bool = False, - # algorithm parameters - ensemble_num: int = 7, - elite_num: int = 5, - learn_logvar_bounds: bool = False, - # network parameters - num_layers: int = 4, - hid_size: int = 200, - activation_fn_cfg: Optional[Union[Dict, omegaconf.DictConfig]] = None, - # others - device: Union[str, torch.device] = "cpu", - ): - super(PlainRewardMech, self).__init__( - obs_size=obs_size, - action_size=action_size, - deterministic=deterministic, - ensemble_num=ensemble_num, - elite_num=elite_num, - device=device, - ) - self.num_layers = num_layers - self.hid_size = hid_size - - def create_activation(): - if activation_fn_cfg is None: - return nn.ReLU() - else: - return hydra.utils.instantiate(activation_fn_cfg) - - hidden_layers = [ - nn.Sequential( - self.create_linear_layer(obs_size + action_size, hid_size), - create_activation(), - ) - ] - for i in range(num_layers - 1): - hidden_layers.append( - nn.Sequential( - self.create_linear_layer(hid_size, hid_size), - create_activation(), - ) - ) - self.hidden_layers = nn.Sequential(*hidden_layers) - - if deterministic: - self.mean_and_logvar = self.create_linear_layer(hid_size, 1) - else: - self.mean_and_logvar = self.create_linear_layer(hid_size, 2) - self.min_logvar = nn.Parameter(-10 * torch.ones(1), requires_grad=learn_logvar_bounds) - self.max_logvar = nn.Parameter(0.5 * torch.ones(1), requires_grad=learn_logvar_bounds) - - self.apply(truncated_normal_init) - self.to(self.device) - - def forward( - self, - batch_obs: torch.Tensor, # shape: ensemble_num, batch_size, state_size - batch_action: torch.Tensor, # shape: ensemble_num, batch_size, action_size - ) -> Tuple[torch.Tensor, Optional[torch.Tensor]]: - assert len(batch_obs.shape) == 3 and batch_obs.shape[-1] == self.obs_size - assert len(batch_action.shape) == 3 and batch_action.shape[-1] == self.action_size - - hidden = self.hidden_layers(torch.concat([batch_obs, batch_action], dim=-1)) - mean_and_logvar = self.mean_and_logvar(hidden) - - if self.deterministic: - mean, logvar = mean_and_logvar, None - else: - mean = mean_and_logvar[..., :1] - logvar = mean_and_logvar[..., 1:] - logvar = self.max_logvar - F.softplus(self.max_logvar - logvar) - logvar = self.min_logvar + F.softplus(logvar - self.min_logvar) - - return mean, logvar diff --git a/cmrl/models/termination_mech/__init__.py b/cmrl/models/termination_mech/__init__.py deleted file mode 100644 index 43a4ca3..0000000 --- a/cmrl/models/termination_mech/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from cmrl.models.termination_mech.base_termination_mech import BaseTerminationMech - -from cmrl.models.termination_mech.plain_termination_mech import PlainTerminationMech diff --git a/cmrl/models/termination_mech/base_termination_mech.py b/cmrl/models/termination_mech/base_termination_mech.py deleted file mode 100644 index cd80284..0000000 --- a/cmrl/models/termination_mech/base_termination_mech.py +++ /dev/null @@ -1,26 +0,0 @@ -from typing import Union - -import torch - -from cmrl.models.networks.mlp import EnsembleMLP - - -class BaseTerminationMech(EnsembleMLP): - _MODEL_FILENAME = "base_reward_mech.pth" - - def __init__( - self, - obs_size: int, - action_size: int, - deterministic: bool = False, - ensemble_num: int = 7, - elite_num: int = 5, - device: Union[str, torch.device] = "cpu", - ): - super(BaseTerminationMech, self).__init__(ensemble_num=ensemble_num, elite_num=elite_num, device=device) - self.obs_size = obs_size - self.action_size = action_size - self.deterministic = deterministic - - def forward(self, state: torch.Tensor, action: torch.Tensor): - pass diff --git a/cmrl/models/termination_mech/plain_termination_mech.py b/cmrl/models/termination_mech/plain_termination_mech.py deleted file mode 100644 index 1143690..0000000 --- a/cmrl/models/termination_mech/plain_termination_mech.py +++ /dev/null @@ -1,94 +0,0 @@ -from typing import Dict, Optional, Tuple, Union - -import hydra -import omegaconf -import torch -from torch import nn as nn -from torch.nn import functional as F - -from cmrl.models.layers import truncated_normal_init -from cmrl.models.termination_mech.base_termination_mech import BaseTerminationMech - - -class PlainTerminationMech(BaseTerminationMech): - _MODEL_FILENAME = "plain_termination_mech.pth" - - def __init__( - self, - # transition info - obs_size: int, - action_size: int, - deterministic: bool = False, - # algorithm parameters - ensemble_num: int = 7, - elite_num: int = 5, - learn_logvar_bounds: bool = False, - # network parameters - num_layers: int = 4, - hid_size: int = 200, - activation_fn_cfg: Optional[Union[Dict, omegaconf.DictConfig]] = None, - # others - device: Union[str, torch.device] = "cpu", - ): - super(PlainTerminationMech, self).__init__( - obs_size=obs_size, - action_size=action_size, - deterministic=deterministic, - ensemble_num=ensemble_num, - elite_num=elite_num, - device=device, - ) - self.num_layers = num_layers - self.hid_size = hid_size - - def create_activation(): - if activation_fn_cfg is None: - return nn.ReLU() - else: - return hydra.utils.instantiate(activation_fn_cfg) - - hidden_layers = [ - nn.Sequential( - self.create_linear_layer(obs_size + action_size, hid_size), - create_activation(), - ) - ] - for i in range(num_layers - 1): - hidden_layers.append( - nn.Sequential( - self.create_linear_layer(hid_size, hid_size), - create_activation(), - ) - ) - self.hidden_layers = nn.Sequential(*hidden_layers) - - if deterministic: - self.mean_and_logvar = self.create_linear_layer(hid_size, 1) - else: - self.mean_and_logvar = self.create_linear_layer(hid_size, 2) - self.min_logvar = nn.Parameter(-10 * torch.ones(1), requires_grad=learn_logvar_bounds) - self.max_logvar = nn.Parameter(0.5 * torch.ones(1), requires_grad=learn_logvar_bounds) - - self.apply(truncated_normal_init) - self.to(self.device) - - def forward( - self, - batch_obs: torch.Tensor, # shape: ensemble_num, batch_size, state_size - batch_action: torch.Tensor, # shape: ensemble_num, batch_size, action_size - ) -> Tuple[torch.Tensor, Optional[torch.Tensor]]: - assert len(batch_obs.shape) == 3 and batch_obs.shape[-1] == self.obs_size - assert len(batch_action.shape) == 3 and batch_action.shape[-1] == self.action_size - - hidden = self.hidden_layers(torch.concat([batch_obs, batch_action], dim=-1)) - mean_and_logvar = self.mean_and_logvar(hidden) - - if self.deterministic: - mean, logvar = mean_and_logvar, None - else: - mean = mean_and_logvar[..., :1] - logvar = mean_and_logvar[..., 1:] - logvar = self.max_logvar - F.softplus(self.max_logvar - logvar) - logvar = self.min_logvar + F.softplus(logvar - self.min_logvar) - - return mean, logvar diff --git a/cmrl/models/transition/__init__.py b/cmrl/models/transition/__init__.py deleted file mode 100644 index 23295bd..0000000 --- a/cmrl/models/transition/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -from cmrl.models.transition.base_transition import BaseTransition - -from cmrl.models.transition.one_step.external_mask_transition import ExternalMaskTransition -from cmrl.models.transition.one_step.plain_transition import PlainTransition - -from cmrl.models.transition.multi_step.forward_euler import ForwardEulerTransition diff --git a/cmrl/models/transition/base_transition.py b/cmrl/models/transition/base_transition.py deleted file mode 100644 index 7f8f5f7..0000000 --- a/cmrl/models/transition/base_transition.py +++ /dev/null @@ -1,26 +0,0 @@ -from typing import Union - -import torch - -from cmrl.models.networks.mlp import EnsembleMLP - - -class BaseTransition(EnsembleMLP): - _MODEL_FILENAME = "base_ensemble_transition.pth" - - def __init__( - self, - obs_size: int, - action_size: int, - deterministic: bool, - ensemble_num: int = 7, - elite_num: int = 5, - device: Union[str, torch.device] = "cpu", - ): - super(BaseTransition, self).__init__(ensemble_num=ensemble_num, elite_num=elite_num, device=device) - self.obs_size = obs_size - self.action_size = action_size - self.deterministic = deterministic - - def forward(self, state: torch.Tensor, action: torch.Tensor): - pass diff --git a/cmrl/models/transition/multi_step/forward_euler.py b/cmrl/models/transition/multi_step/forward_euler.py deleted file mode 100644 index 7b59388..0000000 --- a/cmrl/models/transition/multi_step/forward_euler.py +++ /dev/null @@ -1,41 +0,0 @@ -from typing import Dict, Optional, Sequence, Tuple, Union - -import torch - -from cmrl.models.transition.base_transition import BaseTransition - - -class ForwardEulerTransition(BaseTransition): - def __init__(self, one_step_transition: BaseTransition, repeat_times: int = 2): - super().__init__( - obs_size=one_step_transition.obs_size, - action_size=one_step_transition.action_size, - ensemble_num=one_step_transition.ensemble_num, - deterministic=one_step_transition.deterministic, - device=one_step_transition.device, - ) - - self.one_step_transition = one_step_transition - self.repeat_times = repeat_times - - if hasattr(self.one_step_transition, "max_logvar"): - self.max_logvar = one_step_transition.max_logvar - self.min_logvar = one_step_transition.min_logvar - - if hasattr(self.one_step_transition, "input_mask"): - self.input_mask = self.one_step_transition.input_mask - self.set_input_mask = self.one_step_transition.set_input_mask - - def set_elite_members(self, elite_indices: Sequence[int]): - self.one_step_transition.set_elite_members(elite_indices) - - def forward( - self, - batch_obs: torch.Tensor, - batch_action: torch.Tensor, - ) -> Tuple[torch.Tensor, Optional[torch.Tensor]]: - logvar = torch.zeros(batch_obs.shape, device=self.device) - mean = batch_obs - for t in range(self.repeat_times): - mean, logvar = self.one_step_transition.forward(mean, batch_action.clone()) - return mean, logvar diff --git a/cmrl/models/transition/one_step/external_mask_transition.py b/cmrl/models/transition/one_step/external_mask_transition.py deleted file mode 100644 index 783236c..0000000 --- a/cmrl/models/transition/one_step/external_mask_transition.py +++ /dev/null @@ -1,169 +0,0 @@ -from typing import Dict, Optional, Sequence, Tuple, Union - -import hydra -import omegaconf -import torch -from torch import nn as nn -from torch.nn import functional as F - -import cmrl.types -from cmrl.models.layers import ParallelEnsembleLinearLayer, truncated_normal_init -from cmrl.models.transition.base_transition import BaseTransition -from cmrl.models.util import to_tensor - - -class ExternalMaskTransition(BaseTransition): - """Implements an ensemble of multi-layer perceptrons each modeling a Gaussian distribution - corresponding to each independent dimension. - - Args: - obs_size (int): size of state. - action_size (int): size of action. - device (str or torch.device): the device to use for the model. - num_layers (int): the number of layers in the model - (e.g., if ``num_layers == 3``, then model graph looks like - input -h1-> -h2-> -l3-> output). - ensemble_num (int): the number of members in the ensemble. Defaults to 1. - hid_size (int): the size of the hidden layers (e.g., size of h1 and h2 in the graph above). - deterministic (bool): if ``True``, the model predicts the mean and logvar of the conditional - gaussian distribution, otherwise only predicts the mean. Defaults to ``False``. - residual (bool): if ``True``, the model predicts the residual of output and input. Defaults to ``True``. - learn_logvar_bounds (bool): if ``True``, the log-var bounds will be learned, otherwise - they will be constant. Defaults to ``False``. - activation_fn_cfg (dict or omegaconf.DictConfig, optional): configuration of the - desired activation function. Defaults to torch.nn.ReLU when ``None``. - """ - - _MODEL_FILENAME = "external_mask_transition.pth" - - def __init__( - self, - # transition info - obs_size: int, - action_size: int, - deterministic: bool = False, - # algorithm parameters - ensemble_num: int = 7, - elite_num: int = 5, - residual: bool = True, - learn_logvar_bounds: bool = False, - # network parameters - num_layers: int = 4, - hid_size: int = 200, - activation_fn_cfg: Optional[Union[Dict, omegaconf.DictConfig]] = None, - # others - device: Union[str, torch.device] = "cpu", - ): - super().__init__( - obs_size=obs_size, - action_size=action_size, - deterministic=deterministic, - ensemble_num=ensemble_num, - elite_num=elite_num, - device=device, - ) - self.residual = residual - self.learn_logvar_bounds = learn_logvar_bounds - - self.num_layers = num_layers - self.hid_size = hid_size - self.activation_fn_cfg = activation_fn_cfg - - self._input_mask: Optional[torch.Tensor] = torch.ones((obs_size, obs_size + action_size)).to(device) - self.add_save_attr("input_mask") - - def create_activation(): - if activation_fn_cfg is None: - return nn.ReLU() - else: - return hydra.utils.instantiate(activation_fn_cfg) - - hidden_layers = [ - nn.Sequential( - self.create_linear_layer(obs_size + action_size, hid_size), - create_activation(), - ) - ] - for i in range(num_layers - 1): - hidden_layers.append( - nn.Sequential( - self.create_linear_layer(hid_size, hid_size), - create_activation(), - ) - ) - self.hidden_layers = nn.Sequential(*hidden_layers) - - if deterministic: - self.mean_and_logvar = self.create_linear_layer(hid_size, 1) - else: - self.mean_and_logvar = self.create_linear_layer(hid_size, 2) - self.min_logvar = nn.Parameter(-10 * torch.ones(obs_size, 1, 1, 1), requires_grad=learn_logvar_bounds) - self.max_logvar = nn.Parameter(0.5 * torch.ones(obs_size, 1, 1, 1), requires_grad=learn_logvar_bounds) - - self.apply(truncated_normal_init) - self.to(self.device) - - def create_linear_layer(self, l_in, l_out): - return ParallelEnsembleLinearLayer(l_in, l_out, parallel_num=self.obs_size, ensemble_num=self.ensemble_num) - - def set_input_mask(self, mask: cmrl.types.TensorType): - self._input_mask = to_tensor(mask).to(self.device) - - @property - def input_mask(self): - return self._input_mask - - def mask_input(self, x: torch.Tensor) -> torch.Tensor: - assert x.ndim == 4 - assert self._input_mask is not None - assert 2 <= self._input_mask.ndim <= 4 - assert x.shape[0] == self._input_mask.shape[0] and x.shape[-1] == self._input_mask.shape[-1] - - if self._input_mask.ndim == 2: - # [parallel_size x in_dim] - input_mask = self._input_mask[:, None, None, :] - elif self._input_mask.ndim == 3: - if self._input_mask.shape[1] == x.shape[1]: - # [parallel_size x ensemble_size x in_dim] - input_mask = self._input_mask[:, :, None, :] - elif self._input_mask.shape[1] == x.shape[2]: - # [parallel_size x batch_size x in_dim] - input_mask = self._input_mask[:, None, :, :] - else: - raise RuntimeError("input mask shape %a does not match x shape %a" % (self._input_mask.shape, x.shape)) - else: - assert self._input_mask.shape == x.shape - input_mask = self._input_mask - - x *= input_mask - return x - - def forward( - self, - batch_obs: torch.Tensor, # shape: ensemble_num, batch_size, obs_size - batch_action: torch.Tensor, # shape: ensemble_num, batch_size, action_size - ) -> Tuple[torch.Tensor, Optional[torch.Tensor]]: - assert len(batch_obs.shape) == 3 and batch_obs.shape[-1] == self.obs_size - assert len(batch_action.shape) == 3 and batch_action.shape[-1] == self.action_size - - repeated_input = torch.concat([batch_obs, batch_action], dim=-1).repeat((self.obs_size, 1, 1, 1)) - masked_input = self.mask_input(repeated_input) - hidden = self.hidden_layers(masked_input) - mean_and_logvar = self.mean_and_logvar(hidden) - - if self.deterministic: - mean, logvar = mean_and_logvar, None - else: - mean = mean_and_logvar[..., :1] - logvar = mean_and_logvar[..., 1:] - logvar = self.max_logvar - F.softplus(self.max_logvar - logvar) - logvar = self.min_logvar + F.softplus(logvar - self.min_logvar) - - mean = torch.transpose(mean, 0, -1)[0] - if logvar is not None: - logvar = torch.transpose(logvar, 0, -1)[0] - - if self.residual: - mean += batch_obs - - return mean, logvar diff --git a/cmrl/models/transition/one_step/internal_mask_transition.py b/cmrl/models/transition/one_step/internal_mask_transition.py deleted file mode 100644 index e69de29..0000000 diff --git a/cmrl/models/transition/one_step/plain_transition.py b/cmrl/models/transition/one_step/plain_transition.py deleted file mode 100644 index 3efa810..0000000 --- a/cmrl/models/transition/one_step/plain_transition.py +++ /dev/null @@ -1,122 +0,0 @@ -from typing import Dict, Optional, Sequence, Tuple, Union - -import hydra -import omegaconf -import torch -from torch import nn as nn -from torch.nn import functional as F - -from cmrl.models.layers import EnsembleLinearLayer, truncated_normal_init -from cmrl.models.transition.base_transition import BaseTransition - - -class PlainTransition(BaseTransition): - """Implements an ensemble of multi-layer perceptrons each modeling a Gaussian distribution. - - Args: - obs_size (int): size of state. - action_size (int): size of action. - device (str or torch.device): the device to use for the model. - num_layers (int): the number of layers in the model - (e.g., if ``num_layers == 3``, then model graph looks like - input -h1-> -h2-> -l3-> output). - ensemble_num (int): the number of members in the ensemble. Defaults to 1. - hid_size (int): the size of the hidden layers (e.g., size of h1 and h2 in the graph above). - deterministic (bool): if ``True``, the model predicts the mean and logvar of the conditional - gaussian distribution, otherwise only predicts the mean. Defaults to ``False``. - residual (bool): if ``True``, the model predicts the residual of output and input. Defaults to ``True``. - learn_logvar_bounds (bool): if ``True``, the log-var bounds will be learned, otherwise - they will be constant. Defaults to ``False``. - activation_fn_cfg (dict or omegaconf.DictConfig, optional): configuration of the - desired activation function. Defaults to torch.nn.ReLU when ``None``. - """ - - _MODEL_FILENAME = "plain_transition.pth" - - def __init__( - self, - # transition info - obs_size: int, - action_size: int, - deterministic: bool = False, - # algorithm parameters - ensemble_num: int = 7, - elite_num: int = 5, - residual: bool = True, - learn_logvar_bounds: bool = False, - # network parameters - num_layers: int = 4, - hid_size: int = 200, - activation_fn_cfg: Optional[Union[Dict, omegaconf.DictConfig]] = None, - # others - device: Union[str, torch.device] = "cpu", - ): - super().__init__( - obs_size=obs_size, - action_size=action_size, - deterministic=deterministic, - ensemble_num=ensemble_num, - elite_num=elite_num, - device=device, - ) - self.residual = residual - self.learn_logvar_bounds = learn_logvar_bounds - - self.num_layers = num_layers - self.hid_size = hid_size - self.activation_fn_cfg = activation_fn_cfg - - def create_activation(): - if activation_fn_cfg is None: - return nn.ReLU() - else: - return hydra.utils.instantiate(activation_fn_cfg) - - hidden_layers = [ - nn.Sequential( - self.create_linear_layer(obs_size + action_size, hid_size), - create_activation(), - ) - ] - for i in range(num_layers - 1): - hidden_layers.append( - nn.Sequential( - self.create_linear_layer(hid_size, hid_size), - create_activation(), - ) - ) - self.hidden_layers = nn.Sequential(*hidden_layers) - - if deterministic: - self.mean_and_logvar = self.create_linear_layer(hid_size, obs_size) - else: - self.mean_and_logvar = self.create_linear_layer(hid_size, 2 * obs_size) - self.min_logvar = nn.Parameter(-10 * torch.ones(1, obs_size), requires_grad=learn_logvar_bounds) - self.max_logvar = nn.Parameter(0.5 * torch.ones(1, obs_size), requires_grad=learn_logvar_bounds) - - self.apply(truncated_normal_init) - self.to(self.device) - - def forward( - self, - batch_obs: torch.Tensor, # shape: ensemble_num, batch_size, obs_size - batch_action: torch.Tensor, # shape: ensemble_num, batch_size, action_size - ) -> Tuple[torch.Tensor, Optional[torch.Tensor]]: - assert len(batch_obs.shape) == 3 and batch_obs.shape[-1] == self.obs_size - assert len(batch_action.shape) == 3 and batch_action.shape[-1] == self.action_size - - hidden = self.hidden_layers(torch.concat([batch_obs, batch_action], dim=-1)) - mean_and_logvar = self.mean_and_logvar(hidden) - - if self.deterministic: - mean, logvar = mean_and_logvar, None - else: - mean = mean_and_logvar[..., : self.obs_size] - logvar = mean_and_logvar[..., self.obs_size :] - logvar = self.max_logvar - F.softplus(self.max_logvar - logvar) - logvar = self.min_logvar + F.softplus(logvar - self.min_logvar) - - if self.residual: - mean += batch_obs - - return mean, logvar diff --git a/cmrl/models/util.py b/cmrl/models/util.py index efc6849..a2c63ef 100644 --- a/cmrl/models/util.py +++ b/cmrl/models/util.py @@ -1,40 +1,10 @@ -# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. -from typing import List, Sequence, Tuple +from typing import List, Optional, Union, Dict import numpy as np import torch -import torch.nn.functional as F +from gym import spaces -import cmrl.types - - -def gaussian_nll( - pred_mean: torch.Tensor, - pred_logvar: torch.Tensor, - target: torch.Tensor, - reduce: bool = True, -) -> torch.Tensor: - """Negative log-likelihood for Gaussian distribution - - Args: - pred_mean (tensor): the predicted mean. - pred_logvar (tensor): the predicted log variance. - target (tensor): the target value. - reduce (bool): if ``False`` the loss is returned w/o reducing. - Defaults to ``True``. - - Returns: - (tensor): the negative log-likelihood. - """ - l2 = F.mse_loss(pred_mean, target, reduction="none") - inv_var = (-pred_logvar).exp() - losses = l2 * inv_var + pred_logvar - if reduce: - return losses.sum(dim=1).mean() - return losses +from cmrl.utils.variables import Variable, ContinuousVariable, DiscreteVariable, BinaryVariable # inplace truncated normal function for pytorch. @@ -59,11 +29,3 @@ def truncated_normal_(tensor: torch.Tensor, mean: float = 0, std: float = 1) -> break tensor[cond] = torch.normal(mean, std, size=(bound_violations,), device=tensor.device) return tensor - - -def to_tensor(x: cmrl.types.TensorType): - if isinstance(x, torch.Tensor): - return x - if isinstance(x, np.ndarray): - return torch.from_numpy(x) - raise ValueError("Input must be torch.Tensor or np.ndarray.") diff --git a/cmrl/sb3_extension/online_mb_callback.py b/cmrl/sb3_extension/online_mb_callback.py index 49cb918..06f2655 100644 --- a/cmrl/sb3_extension/online_mb_callback.py +++ b/cmrl/sb3_extension/online_mb_callback.py @@ -1,5 +1,5 @@ import os -import warnings +import pathlib from typing import Any, Callable, Dict, List, Optional, Union from copy import deepcopy @@ -8,53 +8,67 @@ from stable_baselines3.common.callbacks import BaseCallback, EventCallback from stable_baselines3.common.evaluation import evaluate_policy from stable_baselines3.common.buffers import ReplayBuffer -from stable_baselines3.common.vec_env import ( - DummyVecEnv, - VecEnv, - sync_envs_normalization, -) +from stable_baselines3.common.vec_env import DummyVecEnv from cmrl.models.fake_env import VecFakeEnv -from cmrl.models.dynamics.base_dynamics import BaseDynamics +from cmrl.models.dynamics import Dynamics class OnlineModelBasedCallback(BaseCallback): def __init__( self, env: gym.Env, - dynamics: BaseDynamics, + dynamics: Dynamics, real_replay_buffer: ReplayBuffer, - total_num_steps: int = int(1e5), + # online RL + total_online_timesteps: int = int(1e5), initial_exploration_steps: int = 1000, freq_train_model: int = 250, + # dynamics learning + longest_epoch: int = -1, + improvement_threshold: float = 0.01, + patience: int = 5, + work_dir: Optional[Union[str, pathlib.Path]] = None, device: str = "cpu", ): super(OnlineModelBasedCallback, self).__init__(verbose=2) self.env = DummyVecEnv([lambda: env]) self.dynamics = dynamics - self.total_num_steps = total_num_steps + self.real_replay_buffer = real_replay_buffer + # online RL + self.total_online_timesteps = total_online_timesteps self.initial_exploration_steps = initial_exploration_steps self.freq_train_model = freq_train_model + # dynamics learning + self.longest_epoch = longest_epoch + self.improvement_threshold = improvement_threshold + self.patience = patience + self.work_dir = work_dir self.device = device self.action_space = env.action_space self.observation_space = env.observation_space - self.real_replay_buffer = real_replay_buffer - - self.now_num_steps = 0 - self.step_times = 0 + self.now_online_timesteps = 0 self._last_obs = None def _on_step(self) -> bool: - if self.step_times % self.freq_train_model == 0: - self.dynamics.learn(self.real_replay_buffer) + if self.n_calls % self.freq_train_model == 0: + # dump some residual log before dynamics learn + self.model.logger.dump(step=self.num_timesteps) + + self.dynamics.learn( + self.real_replay_buffer, + longest_epoch=self.longest_epoch, + improvement_threshold=self.improvement_threshold, + patience=self.patience, + work_dir=self.work_dir, + ) - self.step_and_add(explore=False) - self.step_times += 1 + self.step_and_add(explore=False) - if self.now_num_steps >= self.total_num_steps: + if self.now_online_timesteps >= self.total_online_timesteps: return False return True @@ -62,7 +76,7 @@ def _on_training_start(self): assert self.env.num_envs == 1 self._last_obs = self.env.reset() - while self.now_num_steps < self.initial_exploration_steps: + while self.now_online_timesteps < self.initial_exploration_steps: self.step_and_add(explore=True) def step_and_add(self, explore=True): @@ -73,7 +87,7 @@ def step_and_add(self, explore=True): buffer_actions = self.model.policy.scale_action(actions) new_obs, rewards, dones, infos = self.env.step(actions) - self.now_num_steps += 1 + self.now_online_timesteps += 1 next_obs = deepcopy(new_obs) if dones[0] and infos[0].get("terminal_observation") is not None: diff --git a/cmrl/types.py b/cmrl/types.py index 1bb28b4..07a1c4b 100644 --- a/cmrl/types.py +++ b/cmrl/types.py @@ -1,7 +1,5 @@ -from dataclasses import dataclass from typing import Callable, Optional, Tuple, Union -import numpy as np import torch # (next_obs, pre_obs, action) -> reward @@ -9,72 +7,5 @@ # (next_obs, pre_obs, action) -> terminal TermFnType = Callable[[torch.Tensor, torch.Tensor, torch.Tensor], torch.Tensor] InitObsFnType = Callable[[int], torch.Tensor] -ObsProcessFnType = Callable[[np.ndarray], np.ndarray] - -TensorType = Union[torch.Tensor, np.ndarray] -TrajectoryEvalFnType = Callable[[TensorType, torch.Tensor], torch.Tensor] -# obs, action, next_obs, reward, done -InteractionData = Tuple[TensorType, TensorType, TensorType, TensorType, TensorType] - - -@dataclass -class InteractionBatch: - """Represents a batch of transitions""" - - batch_obs: Optional[TensorType] - batch_action: Optional[TensorType] - batch_next_obs: Optional[TensorType] - batch_reward: Optional[TensorType] - batch_done: Optional[TensorType] - - @property - def attrs(self): - return [ - "batch_obs", - "batch_action", - "batch_next_obs", - "batch_reward", - "batch_done", - ] - - def __len__(self): - return self.batch_obs.shape[0] - - def as_tuple(self) -> InteractionData: - return ( - self.batch_obs, - self.batch_action, - self.batch_next_obs, - self.batch_reward, - self.batch_done, - ) - - def __getitem__(self, item): - return InteractionBatch( - self.batch_obs[item], - self.batch_action[item], - self.batch_next_obs[item], - self.batch_reward[item], - self.batch_done[item], - ) - - @staticmethod - def _get_new_shape(old_shape: Tuple[int, ...], batch_size: int): - new_shape = list((1,) + old_shape) - new_shape[0] = batch_size - new_shape[1] = old_shape[0] // batch_size - return tuple(new_shape) - - def add_new_batch_dim(self, batch_size: int): - if not len(self) % batch_size == 0: - raise ValueError("Current batch of transitions size is not a " "multiple of the new batch size. ") - return InteractionBatch( - self.batch_obs.reshape(self._get_new_shape(self.batch_obs.shape, batch_size)), - self.batch_action.reshape(self._get_new_shape(self.batch_action.shape, batch_size)), - self.batch_next_obs.reshape(self._get_new_shape(self.batch_obs.shape, batch_size)), - self.batch_reward.reshape(self._get_new_shape(self.batch_reward.shape, batch_size)), - self.batch_done.reshape(self._get_new_shape(self.batch_done.shape, batch_size)), - ) - - -ModelInput = Union[torch.Tensor, InteractionBatch] +Obs2StateFnType = Callable[[torch.Tensor, torch.Tensor], torch.Tensor] +State2ObsFnType = Callable[[torch.Tensor], torch.Tensor] diff --git a/cmrl/util/config.py b/cmrl/util/config.py deleted file mode 100644 index c07dba2..0000000 --- a/cmrl/util/config.py +++ /dev/null @@ -1,43 +0,0 @@ -import pathlib -from typing import Tuple, Union - -import omegaconf - - -def load_hydra_cfg(results_dir: Union[str, pathlib.Path]) -> omegaconf.DictConfig: - """Loads a Hydra configuration from the given directory path. - - Tries to load the configuration from "results_dir/.hydra/config.yaml". - - Args: - results_dir (str or pathlib.Path): the path to the directory containing the config. - - Returns: - (omegaconf.DictConfig): the loaded configuration. - - """ - results_dir = pathlib.Path(results_dir) - cfg_file = results_dir / ".hydra" / "config.yaml" - cfg = omegaconf.OmegaConf.load(cfg_file) - if not isinstance(cfg, omegaconf.DictConfig): - raise RuntimeError("Configuration format not a omegaconf.DictConf") - return cfg - - -def get_complete_dynamics_cfg( - dynamics_cfg: omegaconf.DictConfig, - obs_shape: Tuple[int, ...], - act_shape: Tuple[int, ...], -): - transition_cfg = dynamics_cfg.transition - transition_cfg.obs_size = obs_shape[0] - transition_cfg.action_size = act_shape[0] - - reward_cfg = dynamics_cfg.reward_mech - reward_cfg.obs_size = obs_shape[0] - reward_cfg.action_size = act_shape[0] - - termination_cfg = dynamics_cfg.termination_mech - termination_cfg.obs_size = obs_shape[0] - termination_cfg.action_size = act_shape[0] - return dynamics_cfg diff --git a/cmrl/util/creator.py b/cmrl/util/creator.py deleted file mode 100644 index f9fc723..0000000 --- a/cmrl/util/creator.py +++ /dev/null @@ -1,64 +0,0 @@ -import pathlib -from typing import Callable, Dict, List, Optional, Sequence, Tuple, Type, Union - -import gym.wrappers -import hydra -import numpy as np -import omegaconf -from stable_baselines3.common.logger import Logger - -from cmrl.models.dynamics import ConstraintBasedDynamics, PlainEnsembleDynamics -from cmrl.models.transition import ForwardEulerTransition -from cmrl.util.config import get_complete_dynamics_cfg - - -def create_dynamics( - dynamics_cfg: omegaconf.DictConfig, - obs_shape: Tuple[int, ...], - act_shape: Tuple[int, ...], - logger: Optional[Logger] = None, - load_dir: Optional[Union[str, pathlib.Path]] = None, - load_device: Optional[str] = None, -): - if dynamics_cfg.name == "plain_dynamics": - dynamics_class = PlainEnsembleDynamics - elif dynamics_cfg.name == "constraint_based_dynamics": - dynamics_class = ConstraintBasedDynamics - else: - raise NotImplementedError - - dynamics_cfg = get_complete_dynamics_cfg(dynamics_cfg, obs_shape, act_shape) - transition = hydra.utils.instantiate(dynamics_cfg.transition, _recursive_=False) - if dynamics_cfg.multi_step == "none": - pass - elif dynamics_cfg.multi_step.startswith("forward_euler"): - repeat_times = int(dynamics_cfg.multi_step[len("forward_euler") + 1 :]) - transition = ForwardEulerTransition(transition, repeat_times) - else: - raise NotImplementedError - - if dynamics_cfg.learned_reward: - reward_mech = hydra.utils.instantiate(dynamics_cfg.reward_mech, _recursive_=False) - else: - reward_mech = None - - if dynamics_cfg.learned_termination: - termination_mech = hydra.utils.instantiate(dynamics_cfg.termination_mech, _recursive_=False) - raise NotImplementedError - else: - termination_mech = None - - dynamics_model = dynamics_class( - transition=transition, - learned_reward=dynamics_cfg.learned_reward, - reward_mech=reward_mech, - learned_termination=dynamics_cfg.learned_termination, - termination_mech=termination_mech, - optim_lr=dynamics_cfg.optim_lr, - weight_decay=dynamics_cfg.weight_decay, - logger=logger, - ) - if load_dir: - dynamics_model.load(load_dir, load_device) - - return dynamics_model diff --git a/cmrl/util/env.py b/cmrl/util/env.py deleted file mode 100644 index a7628f6..0000000 --- a/cmrl/util/env.py +++ /dev/null @@ -1,47 +0,0 @@ -from typing import Dict, Optional, Tuple, Union, cast - -import emei -import gym -import omegaconf -import torch - -import cmrl.types - - -def to_num(s): - try: - return int(s) - except ValueError: - return float(s) - - -def get_term_and_reward_fn( - cfg: omegaconf.DictConfig, -) -> Tuple[cmrl.types.TermFnType, Optional[cmrl.types.RewardFnType]]: - return None, None - - -def make_env( - cfg: omegaconf.DictConfig, -) -> Tuple[emei.EmeiEnv, cmrl.types.TermFnType, Optional[cmrl.types.RewardFnType], Optional[cmrl.types.InitObsFnType],]: - if "gym___" in cfg.task.env: - env = gym.make(cfg.task.env.split("___")[1]) - term_fn, reward_fn = get_term_and_reward_fn(cfg) - init_obs_fn = None - elif "emei___" in cfg.task.env: - env_name, params, = cfg.task.env.split( - "___" - )[1:3] - kwargs = dict([(item.split("=")[0], to_num(item.split("=")[1])) for item in params.split("&")]) - env = cast(emei.EmeiEnv, gym.make(env_name, **kwargs)) - term_fn = env.get_terminal - reward_fn = env.get_reward - init_obs_fn = env.get_batch_init_obs - else: - raise NotImplementedError - - # set seed - env.reset(seed=cfg.seed) - env.observation_space.seed(cfg.seed + 1) - env.action_space.seed(cfg.seed + 2) - return env, term_fn, reward_fn, init_obs_fn diff --git a/cmrl/util/transition_iterator.py b/cmrl/util/transition_iterator.py deleted file mode 100644 index b5e1928..0000000 --- a/cmrl/util/transition_iterator.py +++ /dev/null @@ -1,161 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. -import pathlib -import warnings -from typing import Any, List, Optional, Sequence, Sized, Tuple, Type, Union - -import numpy as np - -from cmrl.types import InteractionBatch - - -def _consolidate_batches(batches: Sequence[InteractionBatch]) -> InteractionBatch: - len_batches = len(batches) - b0 = batches[0] - obs = np.empty((len_batches,) + b0.batch_obs.shape, dtype=b0.batch_obs.dtype) - act = np.empty((len_batches,) + b0.batch_action.shape, dtype=b0.batch_action.dtype) - next_obs = np.empty((len_batches,) + b0.batch_obs.shape, dtype=b0.batch_obs.dtype) - rewards = np.empty((len_batches,) + b0.batch_reward.shape, dtype=np.float32) - dones = np.empty((len_batches,) + b0.batch_done.shape, dtype=bool) - for i, b in enumerate(batches): - obs[i] = b.batch_obs - act[i] = b.batch_action - next_obs[i] = b.batch_next_obs - rewards[i] = b.batch_reward - dones[i] = b.batch_done - return InteractionBatch(obs, act, next_obs, rewards, dones) - - -class TransitionIterator: - """An iterator for batches of transitions. - - The iterator can be used doing: - - .. code-block:: python - - for batch in batch_iterator: - do_something_with_batch() - - Rather than be constructed directly, the preferred way to use objects of this class - is for the user to obtain them from :class:`ReplayBuffer`. - - Args: - transitions (:class:`InteractionBatch`): the transition data used to built - the iterator. - batch_size (int): the batch size to use when iterating over the stored data. - shuffle_each_epoch (bool): if ``True`` the iteration order is shuffled everytime a - loop over the data is completed. Defaults to ``False``. - rng (np.random.Generator, optional): a random number generator when sampling - batches. If None (default value), a new default generator will be used. - """ - - def __init__( - self, - transitions: InteractionBatch, - batch_size: int, - shuffle_each_epoch: bool = False, - rng: Optional[np.random.Generator] = None, - ): - self.transitions = transitions - self.num_stored = len(transitions) - self._order: np.ndarray = np.arange(self.num_stored) - self.batch_size = batch_size - self._current_batch = 0 - self._shuffle_each_epoch = shuffle_each_epoch - self._rng = rng if rng is not None else np.random.default_rng() - - def _get_indices_next_batch(self) -> Sized: - start_idx = self._current_batch * self.batch_size - if start_idx >= self.num_stored: - raise StopIteration - end_idx = min((self._current_batch + 1) * self.batch_size, self.num_stored) - order_indices = range(start_idx, end_idx) - indices = self._order[order_indices] - self._current_batch += 1 - return indices - - def __iter__(self): - self._current_batch = 0 - if self._shuffle_each_epoch: - self._order = self._rng.permutation(self.num_stored) - return self - - def __next__(self): - return self[self._get_indices_next_batch()] - - def ensemble_size(self): - return 0 - - def __len__(self): - return (self.num_stored - 1) // self.batch_size + 1 - - def __getitem__(self, item): - return self.transitions[item] - - -class BootstrapIterator(TransitionIterator): - def __init__( - self, - transitions: InteractionBatch, - batch_size: int, - ensemble_size: int, - shuffle_each_epoch: bool = False, - permute_indices: bool = True, - rng: Optional[np.random.Generator] = None, - ): - super().__init__(transitions, batch_size, shuffle_each_epoch=shuffle_each_epoch, rng=rng) - self._ensemble_size = ensemble_size - self._permute_indices = permute_indices - self._bootstrap_iter = ensemble_size > 1 - self.member_indices = self._sample_member_indices() - - def _sample_member_indices(self) -> np.ndarray: - member_indices = np.empty((self.ensemble_size, self.num_stored), dtype=int) - if self._permute_indices: - for i in range(self.ensemble_size): - member_indices[i] = self._rng.permutation(self.num_stored) - else: - member_indices = self._rng.choice( - self.num_stored, - size=(self.ensemble_size, self.num_stored), - replace=True, - ) - return member_indices - - def __iter__(self): - super().__iter__() - return self - - def __next__(self): - if not self._bootstrap_iter: - return super().__next__() - indices = self._get_indices_next_batch() - batches = [] - for member_idx in self.member_indices: - content_indices = member_idx[indices] - batches.append(self[content_indices]) - return _consolidate_batches(batches) - - def toggle_bootstrap(self): - """Toggles whether the iterator returns a batch per model or a single batch.""" - if self.ensemble_size > 1: - self._bootstrap_iter = not self._bootstrap_iter - - @property - def ensemble_size(self): - return self._ensemble_size - - -def _sequence_getitem_impl( - transitions: InteractionBatch, - batch_size: int, - sequence_length: int, - valid_starts: np.ndarray, - item: Any, -): - start_indices = valid_starts[item].repeat(sequence_length) - increment_array = np.tile(np.arange(sequence_length), len(item)) - full_trajectory_indices = start_indices + increment_array - return transitions[full_trajectory_indices].add_new_batch_dim(min(batch_size, len(item))) diff --git a/cmrl/utils/RCIT.py b/cmrl/utils/RCIT.py new file mode 100644 index 0000000..76c6c23 --- /dev/null +++ b/cmrl/utils/RCIT.py @@ -0,0 +1,643 @@ +import numpy as np +from numpy import sqrt +from numpy.linalg import eigh, eigvalsh +from scipy import stats +from sklearn.gaussian_process import GaussianProcessRegressor +from sklearn.gaussian_process.kernels import RBF +from sklearn.gaussian_process.kernels import ConstantKernel as C +from sklearn.gaussian_process.kernels import WhiteKernel + +from causallearn.utils.KCI.GaussianKernel import GaussianKernel +from causallearn.utils.KCI.Kernel import Kernel +from causallearn.utils.KCI.LinearKernel import LinearKernel +from causallearn.utils.KCI.PolynomialKernel import PolynomialKernel +import random +import math +import time +from numpy.linalg import inv + +##################### For Random Feature ##################### +try: + import rpy2 + import rpy2.robjects + + rpy2.robjects.r['options'](warn=-1) + from rpy2.robjects.packages import importr + import rpy2.robjects.numpy2ri + + rpy2.robjects.numpy2ri.activate() +except: + print("Could not import rpy package") + +try: + importr('RCIT') +except: + print("Could not import r-package RCIT") +import random + + +def set_random_seed(seed): + random.seed(seed) + np.random.seed(seed) + + +############################################################### + + +# Cannot find reference 'xxx' in '__init__.pyi | __init__.pyi | __init__.pxd' is a bug in pycharm, please ignore +class KCI_UInd(object): + """ + Python implementation of Kernel-based Conditional Independence (KCI) test. Unconditional version. + The original Matlab implementation can be found in http://people.tuebingen.mpg.de/kzhang/KCI-test.zip + + References + ---------- + [1] K. Zhang, J. Peters, D. Janzing, and B. Schölkopf, + "A kernel-based conditional independence test and application in causal discovery," In UAI 2011. + [2] A. Gretton, K. Fukumizu, C.-H. Teo, L. Song, B. Schölkopf, and A. Smola, "A kernel + Statistical test of independence." In NIPS 21, 2007. + """ + + def __init__(self, kernelX='Gaussian', kernelY='Gaussian', null_ss=1000, approx=True, est_width='empirical', + polyd=2, kwidthx=None, kwidthy=None): + """ + Construct the KCI_UInd model. + + Parameters + ---------- + kernelX: kernel function for input data x + 'Gaussian': Gaussian kernel + 'Polynomial': Polynomial kernel + 'Linear': Linear kernel + kernelY: kernel function for input data y + est_width: set kernel width for Gaussian kernels + 'empirical': set kernel width using empirical rules + 'median': set kernel width using the median trick + 'manual': set by users + null_ss: sample size in simulating the null distribution + approx: whether to use gamma approximation (default=True) + polyd: polynomial kernel degrees (default=1) + kwidthx: kernel width for data x (standard deviation sigma) + kwidthy: kernel width for data y (standard deviation sigma) + """ + + self.kernelX = kernelX + self.kernelY = kernelY + self.est_width = est_width + self.polyd = polyd + self.kwidthx = kwidthx + self.kwidthy = kwidthy + self.nullss = null_ss + self.thresh = 1e-6 + self.approx = approx + + def compute_pvalue(self, data_x=None, data_y=None): + """ + Main function: compute the p value and return it together with the test statistic + + Parameters + ---------- + data_x: input data for x (nxd1 array) + data_y: input data for y (nxd2 array) + + Returns + _________ + pvalue: p value (scalar) + test_stat: test statistic (scalar) + + [Notes for speedup optimization] + Kx, Ky are both symmetric with diagonals equal to 1 (no matter what the kernel is) + Kxc, Kyc are both symmetric + """ + + Kx, Ky = self.kernel_matrix(data_x, data_y) + test_stat, Kxc, Kyc = self.HSIC_V_statistic(Kx, Ky) + + if self.approx: + k_appr, theta_appr = self.get_kappa(Kxc, Kyc) + pvalue = 1 - stats.gamma.cdf(test_stat, k_appr, 0, theta_appr) + else: + null_dstr = self.null_sample_spectral(Kxc, Kyc) + pvalue = sum(null_dstr.squeeze() > test_stat) / float(self.nullss) + return pvalue, test_stat + + def compute_pvalue_rf(self, data_x=None, data_y=None): + rit = rpy2.robjects.r['RIT'](data_x, data_y, approx="lpd4", seed=42) + sta = float(rit.rx2('Sta')[0]) + pval = float(rit.rx2('p')[0]) + return pval, sta + + def kernel_matrix(self, data_x, data_y): + """ + Compute kernel matrix for data x and data y + + Parameters + ---------- + data_x: input data for x (nxd1 array) + data_y: input data for y (nxd2 array) + + Returns + _________ + Kx: kernel matrix for data_x (nxn) + Ky: kernel matrix for data_y (nxn) + """ + if self.kernelX == 'Gaussian': + if self.est_width == 'manual': + if self.kwidthx is not None: + kernelX = GaussianKernel(self.kwidthx) + else: + raise Exception('specify kwidthx') + else: + kernelX = GaussianKernel() + if self.est_width == 'median': + kernelX.set_width_median(data_x) + elif self.est_width == 'empirical': + kernelX.set_width_empirical_hsic(data_x) + else: + raise Exception('Undefined kernel width estimation method') + elif self.kernelX == 'Polynomial': + kernelX = PolynomialKernel(self.polyd) + elif self.kernelX == 'Linear': + kernelX = LinearKernel() + else: + raise Exception('Undefined kernel function') + + if self.kernelY == 'Gaussian': + if self.est_width == 'manual': + if self.kwidthy is not None: + kernelY = GaussianKernel(self.kwidthy) + else: + raise Exception('specify kwidthy') + else: + kernelY = GaussianKernel() + if self.est_width == 'median': + kernelY.set_width_median(data_y) + elif self.est_width == 'empirical': + kernelY.set_width_empirical_hsic(data_y) + else: + raise Exception('Undefined kernel width estimation method') + elif self.kernelY == 'Polynomial': + kernelY = PolynomialKernel(self.polyd) + elif self.kernelY == 'Linear': + kernelY = LinearKernel() + else: + raise Exception('Undefined kernel function') + + data_x = stats.zscore(data_x, ddof=1, axis=0) + data_x[np.isnan(data_x)] = 0. # in case some dim of data_x is constant + data_y = stats.zscore(data_y, ddof=1, axis=0) + data_y[np.isnan(data_y)] = 0. + # We set 'ddof=1' to conform to the normalization way in the original Matlab implementation in + # http://people.tuebingen.mpg.de/kzhang/KCI-test.zip + + Kx = kernelX.kernel(data_x) + Ky = kernelY.kernel(data_y) + return Kx, Ky + + def HSIC_V_statistic(self, Kx, Ky): + """ + Compute V test statistic from kernel matrices Kx and Ky + Parameters + ---------- + Kx: kernel matrix for data_x (nxn) + Ky: kernel matrix for data_y (nxn) + + Returns + _________ + Vstat: HSIC v statistics + Kxc: centralized kernel matrix for data_x (nxn) + Kyc: centralized kernel matrix for data_y (nxn) + """ + Kxc = Kernel.center_kernel_matrix(Kx) + Kyc = Kernel.center_kernel_matrix(Ky) + V_stat = np.sum(Kxc * Kyc) + return V_stat, Kxc, Kyc + + def null_sample_spectral(self, Kxc, Kyc): + """ + Simulate data from null distribution + + Parameters + ---------- + Kxc: centralized kernel matrix for data_x (nxn) + Kyc: centralized kernel matrix for data_y (nxn) + + Returns + _________ + null_dstr: samples from the null distribution + + """ + T = Kxc.shape[0] + if T > 1000: + num_eig = np.int(np.floor(T / 2)) + else: + num_eig = T + lambdax = eigvalsh(Kxc) + lambday = eigvalsh(Kyc) + lambdax = -np.sort(-lambdax) + lambday = -np.sort(-lambday) + lambdax = lambdax[0:num_eig] + lambday = lambday[0:num_eig] + lambda_prod = np.dot(lambdax.reshape(num_eig, 1), lambday.reshape(1, num_eig)).reshape( + (num_eig ** 2, 1)) + lambda_prod = lambda_prod[lambda_prod > lambda_prod.max() * self.thresh] + f_rand = np.random.chisquare(1, (lambda_prod.shape[0], self.nullss)) + null_dstr = lambda_prod.T.dot(f_rand) / T + return null_dstr + + def get_kappa(self, Kx, Ky): + """ + Get parameters for the approximated gamma distribution + Parameters + ---------- + Kx: kernel matrix for data_x (nxn) + Ky: kernel matrix for data_y (nxn) + + Returns + _________ + k_appr, theta_appr: approximated parameters of the gamma distribution + + [Updated @Haoyue 06/24/2022] + equivalent to: + var_appr = 2 * np.trace(Kx.dot(Kx)) * np.trace(Ky.dot(Ky)) / T / T + based on the fact that: + np.trace(K.dot(K)) == np.sum(K * K.T), where here K is symmetric + we can save time on the dot product by only considering the diagonal entries of K.dot(K) + time complexity is reduced from O(n^3) (matrix dot) to O(n^2) (traverse each element), + where n is usually big (sample size). + """ + T = Kx.shape[0] + mean_appr = np.trace(Kx) * np.trace(Ky) / T + var_appr = 2 * np.sum(Kx ** 2) * np.sum(Ky ** 2) / T / T # same as np.sum(Kx * Kx.T) ..., here Kx is symmetric + k_appr = mean_appr ** 2 / var_appr + theta_appr = var_appr / mean_appr + return k_appr, theta_appr + + +class KCI_CInd(object): + """ + Python implementation of Kernel-based Conditional Independence (KCI) test. Conditional version. + The original Matlab implementation can be found in http://people.tuebingen.mpg.de/kzhang/KCI-test.zip + + References + ---------- + [1] K. Zhang, J. Peters, D. Janzing, and B. Schölkopf, "A kernel-based conditional independence test and application in causal discovery," In UAI 2011. + """ + + def __init__(self, kernelX='Gaussian', kernelY='Gaussian', kernelZ='Gaussian', nullss=5000, est_width='empirical', + use_gp=False, approx=True, polyd=2, kwidthx=None, kwidthy=None, kwidthz=None): + """ + Construct the KCI_CInd model. + Parameters + ---------- + kernelX: kernel function for input data x + 'Gaussian': Gaussian kernel + 'Polynomial': Polynomial kernel + 'Linear': Linear kernel + kernelY: kernel function for input data y + kernelZ: kernel function for input data z (conditional variable) + est_width: set kernel width for Gaussian kernels + 'empirical': set kernel width using empirical rules + 'median': set kernel width using the median trick + 'manual': set by users + null_ss: sample size in simulating the null distribution + use_gp: whether use gaussian process to determine kernel width for z + approx: whether to use gamma approximation (default=True) + polyd: polynomial kernel degrees (default=1) + kwidthx: kernel width for data x (standard deviation sigma, default None) + kwidthy: kernel width for data y (standard deviation sigma) + kwidthz: kernel width for data z (standard deviation sigma) + """ + self.kernelX = kernelX + self.kernelY = kernelY + self.kernelZ = kernelZ + self.est_width = est_width + self.polyd = polyd + self.kwidthx = kwidthx + self.kwidthy = kwidthy + self.kwidthz = kwidthz + self.nullss = nullss + self.epsilon_x = 1e-3 # To conform to the original Matlab implementation. + self.epsilon_y = 1e-3 + self.use_gp = use_gp + self.thresh = 1e-5 + self.approx = approx + + def compute_pvalue_rf(self, data_x=None, data_y=None, data_z=None): + rit = rpy2.robjects.r['RCIT'](data_x, data_y, data_z, num_f=10000, num_f2=200, approx="lpd4", seed=42) + sta = float(rit.rx2('Sta')[0]) + pval = float(rit.rx2('p')[0]) + print(pval) + return pval, sta + + def compute_pvalue(self, data_x=None, data_y=None, data_z=None): + """ + Main function: compute the p value and return it together with the test statistic + Parameters + ---------- + data_x: input data for x (nxd1 array) + data_y: input data for y (nxd2 array) + data_z: input data for z (nxd3 array) + + Returns + _________ + pvalue: p value + test_stat: test statistic + """ + Kx, Ky, Kzx, Kzy = self.kernel_matrix(data_x, data_y, data_z) + test_stat, KxR, KyR = self.KCI_V_statistic(Kx, Ky, Kzx, Kzy) + uu_prod, size_u = self.get_uuprod(KxR, KyR) + if self.approx: + k_appr, theta_appr = self.get_kappa(uu_prod) + pvalue = 1 - stats.gamma.cdf(test_stat, k_appr, 0, theta_appr) + else: + null_samples = self.null_sample_spectral(uu_prod, size_u, Kx.shape[0]) + pvalue = sum(null_samples > test_stat) / float(self.nullss) + return pvalue, test_stat + + def kernel_matrix(self, data_x, data_y, data_z): + """ + Compute kernel matrix for data x, data y, and data_z + Parameters + ---------- + data_x: input data for x (nxd1 array) + data_y: input data for y (nxd2 array) + data_z: input data for z (nxd3 array) + + Returns + _________ + Kx: kernel matrix for data_x (nxn) + Ky: kernel matrix for data_y (nxn) + Kzx: centering kernel matrix for data_x (nxn) + kzy: centering kernel matrix for data_y (nxn) + """ + # normalize the data + data_x = stats.zscore(data_x, ddof=1, axis=0) + data_x[np.isnan(data_x)] = 0. + + data_y = stats.zscore(data_y, ddof=1, axis=0) + data_y[np.isnan(data_y)] = 0. + + data_z = stats.zscore(data_z, ddof=1, axis=0) + data_z[np.isnan(data_z)] = 0. + # We set 'ddof=1' to conform to the normalization way in the original Matlab implementation in + # http://people.tuebingen.mpg.de/kzhang/KCI-test.zip + + # concatenate x and z + data_x = np.concatenate((data_x, 0.5 * data_z), axis=1) + if self.kernelX == 'Gaussian': + if self.est_width == 'manual': + if self.kwidthx is not None: + kernelX = GaussianKernel(self.kwidthx) + else: + raise Exception('specify kwidthx') + else: + kernelX = GaussianKernel() + if self.est_width == 'median': + kernelX.set_width_median(data_x) + elif self.est_width == 'empirical': + # kernelX's empirical width is determined by data_z's shape, please refer to the original code + # (http://people.tuebingen.mpg.de/kzhang/KCI-test.zip) in the file + # 'algorithms/CInd_test_new_withGP.m', Line 37 to 52. + kernelX.set_width_empirical_kci(data_z) + else: + raise Exception('Undefined kernel width estimation method') + elif self.kernelX == 'Polynomial': + kernelX = PolynomialKernel(self.polyd) + elif self.kernelX == 'Linear': + kernelX = LinearKernel() + else: + raise Exception('Undefined kernel function') + + if self.kernelY == 'Gaussian': + if self.est_width == 'manual': + if self.kwidthy is not None: + kernelY = GaussianKernel(self.kwidthy) + else: + raise Exception('specify kwidthy') + else: + kernelY = GaussianKernel() + if self.est_width == 'median': + kernelY.set_width_median(data_y) + elif self.est_width == 'empirical': + # kernelY's empirical width is determined by data_z's shape, please refer to the original code + # (http://people.tuebingen.mpg.de/kzhang/KCI-test.zip) in the file + # 'algorithms/CInd_test_new_withGP.m', Line 37 to 52. + kernelY.set_width_empirical_kci(data_z) + else: + raise Exception('Undefined kernel width estimation method') + elif self.kernelY == 'Polynomial': + kernelY = PolynomialKernel(self.polyd) + elif self.kernelY == 'Linear': + kernelY = LinearKernel() + else: + raise Exception('Undefined kernel function') + + Kx = kernelX.kernel(data_x) + Ky = kernelY.kernel(data_y) + + # centering kernel matrix + Kx = Kernel.center_kernel_matrix(Kx) + Ky = Kernel.center_kernel_matrix(Ky) + + if self.kernelZ == 'Gaussian': + if not self.use_gp: + if self.est_width == 'manual': + if self.kwidthz is not None: + kernelZ = GaussianKernel(self.kwidthz) + else: + raise Exception('specify kwidthz') + else: + kernelZ = GaussianKernel() + if self.est_width == 'median': + kernelZ.set_width_median(data_z) + elif self.est_width == 'empirical': + kernelZ.set_width_empirical_kci(data_z) + Kzx = kernelZ.kernel(data_z) + Kzx = Kernel.center_kernel_matrix(Kzx) + # centering kernel matrix to conform with the original Matlab implementation, + # specifically, Line 100 in the file 'algorithms/CInd_test_new_withGP.m' + Kzy = Kzx + else: + # learning the kernel width of Kz using Gaussian process + n, Dz = data_z.shape + if self.kernelX == 'Gaussian': + widthz = sqrt(1.0 / (kernelX.width * data_x.shape[1])) + else: + widthz = 1.0 + # Instantiate a Gaussian Process model for x + wx, vx = eigh(0.5 * (Kx + Kx.T)) + topkx = int(np.min((400, np.floor(n / 4)))) + idx = np.argsort(-wx) + wx = wx[idx] + vx = vx[:, idx] + wx = wx[0:topkx] + vx = vx[:, 0:topkx] + vx = vx[:, wx > wx.max() * self.thresh] + wx = wx[wx > wx.max() * self.thresh] + vx = 2 * sqrt(n) * vx.dot(np.diag(np.sqrt(wx))) / sqrt(wx[0]) + kernelx = C(1.0, (1e-3, 1e3)) * RBF(widthz * np.ones(Dz), (1e-2, 1e2)) + WhiteKernel(0.1, (1e-10, 1e+1)) + gpx = GaussianProcessRegressor(kernel=kernelx) + # fit Gaussian process, including hyperparameter optimization + gpx.fit(data_z, vx) + + # construct Gaussian kernels according to learned hyperparameters + Kzx = gpx.kernel_.k1(data_z, data_z) + self.epsilon_x = np.exp(gpx.kernel_.theta[-1]) + + # Instantiate a Gaussian Process model for y + wy, vy = eigh(0.5 * (Ky + Ky.T)) + topky = int(np.min((400, np.floor(n / 4)))) + idy = np.argsort(-wy) + wy = wy[idy] + vy = vy[:, idy] + wy = wy[0:topky] + vy = vy[:, 0:topky] + vy = vy[:, wy > wy.max() * self.thresh] + wy = wy[wy > wy.max() * self.thresh] + vy = 2 * sqrt(n) * vy.dot(np.diag(np.sqrt(wy))) / sqrt(wy[0]) + kernely = C(1.0, (1e-3, 1e3)) * RBF(widthz * np.ones(Dz), (1e-2, 1e2)) + WhiteKernel(0.1, (1e-10, 1e+1)) + gpy = GaussianProcessRegressor(kernel=kernely) + # fit Gaussian process, including hyperparameter optimization + gpy.fit(data_z, vy) + + # construct Gaussian kernels according to learned hyperparameters + Kzy = gpy.kernel_.k1(data_z, data_z) + self.epsilon_y = np.exp(gpy.kernel_.theta[-1]) + elif self.kernelZ == 'Polynomial': + kernelZ = PolynomialKernel(self.polyd) + Kzx = kernelZ.kernel(data_z) + Kzx = Kernel.center_kernel_matrix(Kzx) + Kzy = Kzx + elif self.kernelZ == 'Linear': + kernelZ = LinearKernel() + Kzx = kernelZ.kernel(data_z) + Kzx = Kernel.center_kernel_matrix(Kzx) + Kzy = Kzx + else: + raise Exception('Undefined kernel function') + return Kx, Ky, Kzx, Kzy + + def KCI_V_statistic(self, Kx, Ky, Kzx, Kzy): + """ + Compute V test statistic from kernel matrices Kx and Ky + Parameters + ---------- + Kx: kernel matrix for data_x (nxn) + Ky: kernel matrix for data_y (nxn) + Kzx: centering kernel matrix for data_x (nxn) + kzy: centering kernel matrix for data_y (nxn) + + Returns + _________ + Vstat: KCI v statistics + KxR: centralized kernel matrix for data_x (nxn) + KyR: centralized kernel matrix for data_y (nxn) + + [Updated @Haoyue 06/24/2022] + 1. Kx, Ky, Kzx, Kzy are all symmetric matrices. + - * Kx's diagonal elements are not the same, because the kernel Kx is centered. + * Before centering, Kx's all diagonal elements are 1 (because of exp(-0.5 * sq_dists * self.width)). + * The same applies to Ky. + - * If (self.kernelZ == 'Gaussian' and self.use_gp), then Kzx has all the same diagonal elements (not necessarily 1). + * The same applies to Kzy. + 2. If not (self.kernelZ == 'Gaussian' and self.use_gp): assert (Kzx == Kzy).all() + With this we could save one repeated calculation of pinv(Kzy+\epsilonI), which consumes most time. + """ + KxR, Rzx = Kernel.center_kernel_matrix_regression(Kx, Kzx, self.epsilon_x) + if self.epsilon_x != self.epsilon_y or (self.kernelZ == 'Gaussian' and self.use_gp): + KyR, _ = Kernel.center_kernel_matrix_regression(Ky, Kzy, self.epsilon_y) + else: + # assert np.all(Kzx == Kzy), 'Kzx and Kzy are the same' + KyR = Rzx.dot(Ky.dot(Rzx)) + Vstat = np.sum(KxR * KyR) + return Vstat, KxR, KyR + + def get_uuprod(self, Kx, Ky): + """ + Compute eigenvalues for null distribution estimation + + Parameters + ---------- + Kx: centralized kernel matrix for data_x (nxn) + Ky: centralized kernel matrix for data_y (nxn) + + Returns + _________ + uu_prod: product of the eigenvectors of Kx and Ky + size_u: number of producted eigenvectors + + """ + wx, vx = eigh(0.5 * (Kx + Kx.T)) + wy, vy = eigh(0.5 * (Ky + Ky.T)) + idx = np.argsort(-wx) + idy = np.argsort(-wy) + wx = wx[idx] + vx = vx[:, idx] + wy = wy[idy] + vy = vy[:, idy] + vx = vx[:, wx > np.max(wx) * self.thresh] + wx = wx[wx > np.max(wx) * self.thresh] + vy = vy[:, wy > np.max(wy) * self.thresh] + wy = wy[wy > np.max(wy) * self.thresh] + vx = vx.dot(np.diag(np.sqrt(wx))) + vy = vy.dot(np.diag(np.sqrt(wy))) + + # calculate their product + T = Kx.shape[0] + num_eigx = vx.shape[1] + num_eigy = vy.shape[1] + size_u = num_eigx * num_eigy + uu = np.zeros((T, size_u)) + for i in range(0, num_eigx): + for j in range(0, num_eigy): + uu[:, i * num_eigy + j] = vx[:, i] * vy[:, j] + + if size_u > T: + uu_prod = uu.dot(uu.T) + else: + uu_prod = uu.T.dot(uu) + + return uu_prod, size_u + + def null_sample_spectral(self, uu_prod, size_u, T): + """ + Simulate data from null distribution + + Parameters + ---------- + uu_prod: product of the eigenvectors of Kx and Ky + size_u: number of producted eigenvectors + T: sample size + + Returns + _________ + null_dstr: samples from the null distribution + + """ + eig_uu = eigvalsh(uu_prod) + eig_uu = -np.sort(-eig_uu) + eig_uu = eig_uu[0:np.min((T, size_u))] + eig_uu = eig_uu[eig_uu > np.max(eig_uu) * self.thresh] + + f_rand = np.random.chisquare(1, (eig_uu.shape[0], self.nullss)) + null_dstr = eig_uu.T.dot(f_rand) + return null_dstr + + def get_kappa(self, uu_prod): + """ + Get parameters for the approximated gamma distribution + Parameters + ---------- + uu_prod: product of the eigenvectors of Kx and Ky + + Returns + ---------- + k_appr, theta_appr: approximated parameters of the gamma distribution + + """ + mean_appr = np.trace(uu_prod) + var_appr = 2 * np.trace(uu_prod.dot(uu_prod)) + k_appr = mean_appr ** 2 / var_appr + theta_appr = var_appr / mean_appr + return k_appr, theta_appr diff --git a/cmrl/util/__init__.py b/cmrl/utils/__init__.py similarity index 92% rename from cmrl/util/__init__.py rename to cmrl/utils/__init__.py index d4085c1..c59acbe 100644 --- a/cmrl/util/__init__.py +++ b/cmrl/utils/__init__.py @@ -40,11 +40,11 @@ def create_handler(cfg: Union[Dict, omegaconf.ListConfig, omegaconf.DictConfig]) target = cfg.overrides.env_cfg.get_dynamics_predict("_target_") if "pybulletgym" in target: - from cmrl.util.pybullet import PybulletEnvHandler + from cmrl.utils.pybullet import PybulletEnvHandler return PybulletEnvHandler() elif "mujoco" in target: - from cmrl.util.mujoco import MujocoEnvHandler + from cmrl.utils.mujoco import MujocoEnvHandler return MujocoEnvHandler() else: @@ -74,15 +74,15 @@ def create_handler_from_str(env_name: str): (EnvHandler): A handler for the associated gym environment """ if "dmcontrol___" in env_name: - from cmrl.util.dmcontrol import DmcontrolEnvHandler + from cmrl.utils.dmcontrol import DmcontrolEnvHandler return DmcontrolEnvHandler() elif "pybulletgym___" in env_name: - from cmrl.util.pybullet import PybulletEnvHandler + from cmrl.utils.pybullet import PybulletEnvHandler return PybulletEnvHandler() elif "gym___" in env_name or env_name == "ideal_inv_pendulum": - from cmrl.util.mujoco import MujocoEnvHandler + from cmrl.utils.mujoco import MujocoEnvHandler return MujocoEnvHandler() else: diff --git a/cmrl/utils/config.py b/cmrl/utils/config.py new file mode 100644 index 0000000..befe862 --- /dev/null +++ b/cmrl/utils/config.py @@ -0,0 +1,64 @@ +import pathlib +from typing import Dict, Union, Optional +from collections import defaultdict + +import omegaconf +from omegaconf import DictConfig +import pandas as pd +import numpy as np + +PACKAGE_PATH = pathlib.Path(__file__).parent.parent.parent + + +def load_hydra_cfg(results_dir: Union[str, pathlib.Path]) -> omegaconf.DictConfig: + """Loads a Hydra configuration from the given directory path. + + Tries to load the configuration from "results_dir/.hydra/config.yaml". + + Args: + results_dir (str or pathlib.Path): the path to the directory containing the config. + + Returns: + (omegaconf.DictConfig): the loaded configuration. + + """ + results_dir = pathlib.Path(results_dir) + cfg_file = results_dir / ".hydra" / "config.yaml" + cfg = omegaconf.OmegaConf.load(cfg_file) + if not isinstance(cfg, omegaconf.DictConfig): + raise RuntimeError("Configuration format not a omegaconf.DictConf") + return cfg + + +def exp_collect(cfg_extractor, + csv_extractor, + env_name="ContinuousCartPoleSwingUp-v0", + exp_name="default", + exp_path=None): + data = defaultdict(list) + + if exp_path is None: + exp_path = PACKAGE_PATH / "exp" + exp_dir = exp_path / exp_name + env_dir = exp_dir / env_name + + for params_dir in env_dir.glob("*"): + for dataset_dir in params_dir.glob("*"): + for time_dir in dataset_dir.glob("*"): + if not (time_dir / ".hydra").exists(): # exp by hydra's MULTIRUN mode, multi exp in this time + time_dir_list = list(time_dir.glob("*")) + else: # only one exp in this time + time_dir_list = [time_dir] + + for single_dir in time_dir_list: + if single_dir.name == "multirun.yaml": + continue + + cfg = load_hydra_cfg(single_dir) + + key = cfg_extractor(cfg, params_dir.name, dataset_dir.name, time_dir.name) + if not key: + continue + + data[key] = csv_extractor(single_dir / "log") + return data diff --git a/cmrl/utils/creator.py b/cmrl/utils/creator.py new file mode 100644 index 0000000..8b71dd8 --- /dev/null +++ b/cmrl/utils/creator.py @@ -0,0 +1,83 @@ +from typing import Optional, cast, List + +from gym import spaces +from hydra.utils import instantiate +from omegaconf import DictConfig +import numpy as np +from stable_baselines3.common.vec_env import VecMonitor +from stable_baselines3.common.logger import Logger +from stable_baselines3.common.base_class import BaseAlgorithm + +from cmrl.types import Obs2StateFnType, State2ObsFnType +from cmrl.models.dynamics import Dynamics +from cmrl.models.fake_env import VecFakeEnv +from cmrl.models.causal_mech.base import BaseCausalMech +from cmrl.utils.variables import ContinuousVariable, BinaryVariable, DiscreteVariable, Variable, parse_space + + +def create_agent(cfg: DictConfig, fake_env: VecFakeEnv, logger: Optional[Logger] = None): + agent = instantiate(cfg.algorithm.agent)(env=VecMonitor(fake_env)) + agent = cast(BaseAlgorithm, agent) + agent.set_logger(logger) + + return agent + + +def create_dynamics( + cfg: DictConfig, + state_space: spaces.Space, + action_space: spaces.Space, + obs2state_fn: Obs2StateFnType, + state2obs_fn: State2ObsFnType, + logger: Optional[Logger] = None, +): + extra_info = cfg.task.get("extra_variable_info", {}) + obs_variables = parse_space(state_space, "obs", extra_info=extra_info) + act_variables = parse_space(action_space, "act", extra_info=extra_info) + next_obs_variables = parse_space(state_space, "next_obs", extra_info=extra_info) + + # transition + assert cfg.transition.learn, "transition must be learned, or you should try model-free RL:)" + transition = instantiate(cfg.transition.mech)( + input_variables=obs_variables + act_variables, + output_variables=next_obs_variables, + logger=logger, + ) + transition = cast(BaseCausalMech, transition) + + # reward mech + assert cfg.reward_mech.mech.multi_step == "none", "reward-mech must be one-step" + if cfg.reward_mech.learn: + reward_mech = instantiate(cfg.reward_mech.mech)( + input_variables=obs_variables + act_variables + next_obs_variables, + output_variables=[ContinuousVariable("reward", dim=1, low=-np.inf, high=np.inf)], + logger=logger, + ) + reward_mech = cast(BaseCausalMech, reward_mech) + else: + reward_mech = None + + # termination mech + assert cfg.termination_mech.mech.multi_step == "none", "termination-mech must be one-step" + if cfg.termination_mech.learn: + termination_mech = instantiate(cfg.termination_mech.mech)( + input_variables=obs_variables + act_variables + next_obs_variables, + output_variables=[BinaryVariable("terminal")], + logger=logger, + ) + termination_mech = cast(BaseCausalMech, termination_mech) + else: + termination_mech = None + + dynamics = Dynamics( + transition=transition, + reward_mech=reward_mech, + termination_mech=termination_mech, + state_space=state_space, + action_space=action_space, + obs2state_fn=obs2state_fn, + state2obs_fn=state2obs_fn, + logger=logger, + ) + + return dynamics diff --git a/cmrl/utils/env.py b/cmrl/utils/env.py new file mode 100644 index 0000000..f5d68a0 --- /dev/null +++ b/cmrl/utils/env.py @@ -0,0 +1,62 @@ +from typing import Dict, Optional, Tuple, cast + +import numpy as np +import emei +import gym +import omegaconf +from stable_baselines3.common.buffers import ReplayBuffer + +import cmrl.utils.variables +from cmrl.types import TermFnType, RewardFnType, InitObsFnType, Obs2StateFnType + + +def make_env( + cfg: omegaconf.DictConfig, +) -> Tuple[emei.EmeiEnv, tuple]: + env = cast(emei.EmeiEnv, gym.make(cfg.task.env_id, **cfg.task.params)) + fns = ( + env.get_batch_reward, + env.get_batch_terminal, + env.get_batch_init_obs, + env.obs2state, + env.state2obs + ) + + # set seed + env.reset(seed=cfg.seed) + env.state_space.seed(cfg.seed + 1) + env.action_space.seed(cfg.seed + 2) + return env, fns + + +def load_offline_data(env, replay_buffer: ReplayBuffer, dataset_name: str, use_ratio: float = 1): + assert hasattr(env, "get_dataset"), "env must have `get_dataset` method" + + data_dict = env.get_dataset(dataset_name) + all_data_num = len(data_dict["observations"]) + sample_data_num = int(use_ratio * all_data_num) + sample_idx = np.random.permutation(all_data_num)[:sample_data_num] + + assert replay_buffer.n_envs == 1 + assert replay_buffer.buffer_size >= sample_data_num + + if sample_data_num == replay_buffer.buffer_size: + replay_buffer.full = True + replay_buffer.pos = 0 + else: + replay_buffer.pos = sample_data_num + + # set all data + for attr in ["observations", "next_observations", "actions", "rewards", "dones", "timeouts"]: + # if attr == "dones" and attr not in data_dict and "terminals" in data_dict: + # replay_buffer.dones[:sample_data_num, 0] = data_dict["terminals"][sample_idx] + # continue + getattr(replay_buffer, attr)[:sample_data_num, 0] = data_dict[attr][sample_idx] + + for attr in ["extra_obs", "next_extra_obs"]: + setattr( + replay_buffer, + attr, + np.zeros((replay_buffer.buffer_size, replay_buffer.n_envs) + data_dict[attr].shape[1:], dtype=np.float32) + ) + getattr(replay_buffer, attr)[:sample_data_num, 0] = data_dict[attr][sample_idx] diff --git a/cmrl/utils/variables.py b/cmrl/utils/variables.py new file mode 100644 index 0000000..534c316 --- /dev/null +++ b/cmrl/utils/variables.py @@ -0,0 +1,116 @@ +from dataclasses import dataclass +from typing import Optional, Dict, Union, List + +from gym import spaces +import numpy as np +import torch + + +@dataclass +class Variable: + name: str + pass + + +@dataclass +class ContinuousVariable(Variable): + dim: int + low: np.ndarray = None + high: np.ndarray = None + + +@dataclass +class RadianVariable(Variable): + dim: int + + +@dataclass +class BinaryVariable(Variable): + pass + + +@dataclass +class DiscreteVariable(Variable): + n: int + + +def parse_space( + space: spaces.Space, + prefix="obs", + extra_info=None +) -> List[Variable]: + extra_info = extra_info if extra_info is not None else {} + + variables = [] + if isinstance(space, spaces.Box): + for i, (low, high) in enumerate(zip(space.low, space.high)): + name = "{}_{}".format(prefix, i) + if "Radian" in extra_info and name in extra_info["Radian"]: + variables.append(RadianVariable(dim=1, name=name)) + else: + variables.append(ContinuousVariable(dim=1, low=low, high=high, name=name)) + elif isinstance(space, spaces.Discrete): + variables.append(DiscreteVariable(n=space.n, name="{}_0".format(prefix))) + elif isinstance(space, spaces.MultiDiscrete): + for i, n in enumerate(space.nvec): + variables.append(DiscreteVariable(n=n, name="{}_{}".format(prefix, i))) + elif isinstance(space, spaces.MultiBinary): + for i in range(space.n): + variables.append(BinaryVariable(name="{}_{}".format(prefix, i))) + elif isinstance(space, spaces.Dict): + # TODO + raise NotImplementedError + + return variables + + +def to_dict_by_space( + data: np.ndarray, + space: spaces.Space, + prefix="obs", + repeat: Optional[int] = None, + to_tensor: bool = False, + device: str = "cpu" +) -> Dict[str, Union[np.ndarray, torch.Tensor]]: + """Transform the interaction data from its own type to python's dict, by the signature of space. + + Args: + data: interaction data from replay buffer + space: space of gym + prefix: prefix of the key in dict + repeat: copy data in a new dimension + to_tensor: transform the data from numpy's ndarray to torch's tensor + device: device + + + Returns: interaction data organized in dictionary form + + """ + if repeat: + assert repeat > 1, "repeat must be a int greater than 1" + + dict_data = {} + if isinstance(space, spaces.Box): + # shape of data: (batch-size, node-num), every node has exactly one dim + for i, (low, high) in enumerate(zip(space.low, space.high)): + # shape of dict_data['xxx']: (batch-size, 1) + dict_data["{}_{}".format(prefix, i)] = data[:, i, None].astype(np.float32) + else: + # TODO + raise NotImplementedError + + for name in dict_data: + if repeat: + # shape of dict_data['xxx']: (repeat-dim, batch-size, specific-dim) + # specific-dim is 1 for the case of spaces.Box + dict_data[name] = np.tile(dict_data[name][None, :, :], [repeat, 1, 1]) + if to_tensor: + dict_data[name] = torch.from_numpy(dict_data[name]).to(device) + + return dict_data + + +def dict2space( + data: Dict[str, Union[np.ndarray, torch.Tensor]], space: spaces.Space +) -> Dict[str, Union[np.ndarray, torch.Tensor]]: + pass diff --git a/cmrl/util/video.py b/cmrl/utils/video.py similarity index 100% rename from cmrl/util/video.py rename to cmrl/utils/video.py diff --git a/docs/about.md b/docs/about.md new file mode 100644 index 0000000..8e22f96 --- /dev/null +++ b/docs/about.md @@ -0,0 +1,43 @@ +# Iphitiden Phoebo caede retiaque solvit genis abdiderat + +## Humo utinam + +Lorem markdownum illos non, somni et evocet Messeniaque diva *agitatis* +nocentius. Templum Erymanthidas prius, duris mihi, iuvenum, nec quod acceptus +una, secuit. + +1. Foret sanguine puniceo +2. Erubuit mittit ipso lenta adspexit arbiter nondum +3. Insanis sum est oves domus nam pars +4. Distinxit verba + +## Iacet venit + +Languore manus est ad prima et caelum sit aristas, ante Styphelumque moris ad +pulsant: vertitur novat. Latrare **minimam coniunx imbribus**: acceptior, ipso +verum demit laudibus non peperi operiri, iussae arva. Ferit fibris gradieris +Dianae, et **dabant dependent** adfixa versa flectit: signumque. + +Indigenae talia, ora rari est in inter coetus *protinus summaque mittitur* +fuerant gravisque agitur et sedibus attulit. Multa capessamus Bacchi: de ut +saeva funera, certamine Chimaera auxiliumque teloque! + +## Imponit requirit armigerae quoque sitimque + +Inplevit nimium. Sub loqui innectens in cincta ripis plangens est. Annua et +stabat Panopeusque naidas audentia quantum videoque quam ipsum. + +## Ingreditur totidem illi + +Est **corpore referam** est rates leti vertitur ab dictu rex quoque sceptra +flamma. Ut quod? Tota nil, horruit hoc. Ubi colla sopore vides dixit qua non +sanguineaque dixerat Iove. Cernis est viribus, **indue** genae verbis solio et +tantum bibulas *et surgere Caesaris* damno Iolaus umquam; scelerate animam? + +1. Testudine tener herosmaxime loco tonitruque +2. Graium pectora pavet sit cum ianua +3. Quod inque +4. De patens ictus spectantia ereptaque constitit falsa +5. Minis modo minor gravet numerusque duorum mediam + +Celsior quid fores, tremore, est quo culpatque terret pati. Anno pietas poples! diff --git a/img/cmrl_logo.png b/docs/cmrl_logo.png similarity index 100% rename from img/cmrl_logo.png rename to docs/cmrl_logo.png diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 0000000..d7703e7 --- /dev/null +++ b/docs/index.md @@ -0,0 +1,19 @@ +``# Welcome to MkDocs + +For full documentation visit [mkdocs.org](https://www.mkdocs.org). + +## Commands + +* `mkdocs new [dir-name]` - Create a new project. +* `mkdocs serve` - Start the live-reloading docs server. +* `mkdocs build` - Build the documentation site. +* `mkdocs -h` - Print help message and exit. + +## Project layout + + mkdocs.yml # The configuration file. + docs/ + index.md # The documentation homepage. + ... # Other markdown pages, images and other files. + +::: cmrl.models.layers.ParallelLinear diff --git a/exp_reader.ipynb b/exp_reader.ipynb new file mode 100644 index 0000000..cb2b37d --- /dev/null +++ b/exp_reader.ipynb @@ -0,0 +1,263 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "import pandas as pd\n", + "import numpy as np\n", + "import cmrl\n", + "from emei.core import get_params_str\n", + "from pathlib import Path\n", + "import matplotlib.pyplot as plt\n", + "import yaml\n", + "from collections import defaultdict" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "# 递归判断a字典中存在的值是否与b字典相等\n", + "def dict_equal(a, b):\n", + " for k, v in a.items():\n", + " if isinstance(v, dict):\n", + " if not dict_equal(v, b[k]):\n", + " return False\n", + " elif v != b[k]:\n", + " return False\n", + " return True\n", + "\n", + "\n", + "def get_value(d, key):\n", + " if isinstance(key, str):\n", + " if key not in d:\n", + " raise ValueError(f\"{key} not in dict\")\n", + " return d[key]\n", + " elif isinstance(key, tuple):\n", + " if key[0] not in d:\n", + " raise ValueError(f\"{key[0]} not in dict\")\n", + " if len(key) <= 1:\n", + " raise ValueError(\"length of tuple-key must be 2\")\n", + " return get_value(d[key[0]], key[1])\n", + " else:\n", + " raise ValueError(\"key must be str or tuple\")\n", + "\n", + "\n", + "# 返回多个字典中不同的value对应的key, 通过递归的方法\n", + "def get_diff_key(dicts):\n", + " if len(dicts) <= 1:\n", + " return []\n", + " keys = set(dicts[0].keys())\n", + " for d in dicts[1:]:\n", + " keys = keys & set(d.keys())\n", + "\n", + " diff_keys = []\n", + " for k in keys:\n", + " if isinstance(dicts[0][k], dict):\n", + " diff_keys += [(k, dk) for dk in get_diff_key([d[k] for d in dicts])]\n", + " elif not all([dicts[0][k] == d[k] for d in dicts[1:]]):\n", + " diff_keys.append(k)\n", + " return diff_keys\n", + "\n", + "\n", + "def argmax(l):\n", + " return max(l), l.index(max(l))" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "default_params = dict(freq_rate=1,\n", + " real_time_scale=0.02,\n", + " integrator=\"euler\",\n", + " parallel_num=3)\n", + "default_custom_cfg = {}\n", + "default_result_key = [\"seed\"]\n", + "\n", + "\n", + "def load_log(exp_name=\"default\",\n", + " task_name=\"ParallelContinuousCartPoleSwingUp-v0\",\n", + " params=default_params,\n", + " dataset=\"SAC-expert-replay\",\n", + " custom_cfg=default_custom_cfg,\n", + " log_file=\"rollout.csv\",\n", + " log_key=\"ep_rew_mean\"):\n", + " path = Path(\"./exp\") / exp_name / task_name / get_params_str(params) / dataset\n", + "\n", + " result_list = []\n", + " cfg_list = []\n", + " for time_dir in path.glob(r\"*\"):\n", + " if not time_dir.is_dir() or not (time_dir / \".hydra\").exists():\n", + " continue\n", + "\n", + " config_path = time_dir / \".hydra\" / \"config.yaml\"\n", + " with open(config_path, \"r\") as f:\n", + " cfg = yaml.load(f, Loader=yaml.FullLoader)\n", + "\n", + " if not dict_equal(custom_cfg, cfg):\n", + " print(\"{} is passed cause its inconsistent cfg\".format(time_dir))\n", + " continue\n", + "\n", + " log_path = time_dir / \"log\" / log_file\n", + " if not log_path.exists():\n", + " continue\n", + "\n", + " df = pd.read_csv(log_path)\n", + " result_list.append(df[log_key].to_numpy())\n", + " cfg_list.append(cfg)\n", + "\n", + " diff_key = get_diff_key(cfg_list)\n", + " result_dict = {}\n", + " for i, cfg in enumerate(cfg_list):\n", + " result_dict[tuple([get_value(cfg, key) for key in diff_key])] = result_list[i]\n", + " return diff_key, result_dict" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "outputs": [], + "source": [ + "def draw_result(exp_name=\"default\",\n", + " task_name=\"ParallelContinuousCartPoleSwingUp-v0\",\n", + " params=default_params,\n", + " dataset=\"SAC-expert-replay\",\n", + " custom_cfg=default_custom_cfg,\n", + " log_file=\"rollout.csv\",\n", + " log_key=\"ep_rew_mean\",\n", + " group_key=('transition', 'oracle')):\n", + " diff_key, result_dict = load_log(exp_name=exp_name,\n", + " task_name=task_name,\n", + " params=params,\n", + " dataset=dataset,\n", + " custom_cfg=custom_cfg,\n", + " log_file=log_file,\n", + " log_key=log_key)\n", + "\n", + " idx = diff_key.index(group_key)\n", + "\n", + " for name in set([key[idx] for key in result_dict.keys()]):\n", + " values = [value for key, value in result_dict.items() if key[idx] == name]\n", + " longest, longest_idx = argmax([value.shape[0] for value in values])\n", + " values_array = np.empty((len(values), longest))\n", + " for i, value in enumerate(values):\n", + " values_array[i, :len(value)] = value\n", + " values_array[i, len(value):] = values[longest_idx][len(value):]\n", + " plt.plot(values_array.mean(axis=0), label=name)\n", + " plt.fill_between(np.arange(len(values_array.mean(axis=0))),\n", + " values_array.mean(axis=0) - values_array.std(axis=0),\n", + " values_array.mean(axis=0) + values_array.std(axis=0), alpha=0.5)\n", + " plt.legend()" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": 35, + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [ + { + "data": { + "text/plain": "
", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjAAAAGdCAYAAAAMm0nCAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8qNh9FAAAACXBIWXMAAA9hAAAPYQGoP6dpAACb+0lEQVR4nOzdd5ycVb348c/0srOzvZdkN703QhIIIUBICF1QRClSBLk3XEUEvSjyk6JcUVRQ1OtVAQUELIC0QCAkkJCEkEJ632Q32+vMzuz0eX5/PLuzmWyd3dnK9/165TUzz3OeZ85sAvPdc77nezSKoigIIYQQQowg2qHugBBCCCFErCSAEUIIIcSIIwGMEEIIIUYcCWCEEEIIMeJIACOEEEKIEUcCGCGEEEKMOBLACCGEEGLEkQBGCCGEECOOfqg7MFDC4TAVFRUkJiai0WiGujtCCCGE6AVFUWhubiY3NxettutxllEbwFRUVFBQUDDU3RBCCCFEH5SVlZGfn9/l+VEbwCQmJgLqD8Butw9xb4QQQgjRG06nk4KCgsj3eFdGbQDTNm1kt9slgBFCCCFGmJ7SP2JK4n300UeZP38+iYmJZGZmcuWVV3Lw4MGoNkuXLkWj0UT9ueOOO6LalJaWcskll2C1WsnMzOTee+8lGAxGtVm3bh1z587FZDIxfvx4nnnmmVi6KoQQQohRLKYAZv369axatYrNmzezZs0aAoEAy5cvx+12R7W77bbbqKysjPx57LHHIudCoRCXXHIJfr+fjz/+mGeffZZnnnmGBx54INKmpKSESy65hPPOO4+dO3dy11138fWvf5133nmnnx9XCCGEEKOBRlEUpa8X19bWkpmZyfr161myZAmgjsDMnj2bX/3qV51e8/bbb3PppZdSUVFBVlYWAL///e/53ve+R21tLUajke9973u8+eab7NmzJ3LdtddeS1NTE6tXr+5V35xOJ0lJSTgcDplCEkIIIUaI3n5/9ysHxuFwAJCamhp1/Pnnn+e5554jOzubyy67jB/+8IdYrVYANm3axIwZMyLBC8CKFSv4j//4D/bu3cucOXPYtGkTy5Yti7rnihUruOuuu/rT3Q4URSEYDBIKheJ6X9FOp9Oh1+tlKbsQQoi46nMAEw6Hueuuuzj77LOZPn165PhXv/pVxowZQ25uLrt27eJ73/seBw8e5F//+hcAVVVVUcELEHldVVXVbRun04nH48FisXToj8/nw+fzRV47nc5u++/3+6msrKSlpSWGTy36wmq1kpOTg9FoHOquCCGEGCX6HMCsWrWKPXv2sGHDhqjjt99+e+T5jBkzyMnJ4YILLuDo0aOMGzeu7z3twaOPPsqDDz7Yq7bhcJiSkhJ0Oh25ubkYjUYZIRgAiqLg9/upra2lpKSECRMmdFuUSAghhOitPgUwd955J2+88QYffvhht0VmABYsWADAkSNHGDduHNnZ2XzyySdRbaqrqwHIzs6OPLYdO7WN3W7vdPQF4L777uPuu++OvG5bR94Zv99POBymoKAgMrUlBobFYsFgMHDixAn8fj9ms3mouySEEGIUiOnXYUVRuPPOO3nllVdYu3YtRUVFPV6zc+dOAHJycgBYtGgRu3fvpqamJtJmzZo12O12pk6dGmnz/vvvR91nzZo1LFq0qMv3MZlMkZovva39IqMBg0N+zkIIIeItpm+WVatW8dxzz/HCCy+QmJhIVVUVVVVVeDweAI4ePcrDDz/Mtm3bOH78OP/+97+58cYbWbJkCTNnzgRg+fLlTJ06lRtuuIHPPvuMd955h/vvv59Vq1ZhMpkAuOOOOzh27Bjf/e53OXDgAL/97W95+eWX+fa3vx3njy+EEEKIkSimAOZ3v/sdDoeDpUuXkpOTE/nz0ksvAWA0GnnvvfdYvnw5kydP5jvf+Q5XX301r7/+euQeOp2ON954A51Ox6JFi7j++uu58cYbeeihhyJtioqKePPNN1mzZg2zZs3i8ccf549//CMrVqyI08cWQgghxEjWrzoww1l368i9Xi8lJSUUFRVJTsZp1q1bx3nnnUdjYyPJyclxuaf8vIUQQvRWb+vASHLCCLN06dK41cOJ572EEEKIwSQBzCjTVpxPCCGEGM0kgEH90m/xB4fkTywzeDfddBPr16/niSeeiGyU+cwzz6DRaHj77beZN28eJpOJDRs2cNNNN3HllVdGXX/XXXexdOnSLu91/PjxSNtt27ZxxhlnYLVaOeusszps2imEEKL3Tv9/fXmTJ6b//4uO+rWVwGjhCYSY+sDQbBS576EVWI29+2t44oknOHToENOnT48kPe/duxeA//7v/+bnP/85xcXFpKSk9OleGRkZkSDmBz/4AY8//jgZGRnccccd3HLLLWzcuLEPn1AIIcSGI3VMyk4EYG+Fkz0nHVw+O5cxaQlD3LORSwKYESQpKQmj0YjVao0U/Ttw4AAADz30EBdeeGG/7nWqH//4x5x77rmAGhxdcskleL1eScIVQogY1TR72VfhZEJmIiFF4Ui1i2BYISwDMP0iAQxgMejY99DQLNG2GHRxuc8ZZ5wRl/u0aavbA+1FCGtqaigsLIzr+wghxGi3v7KZQY1VDr8HZZshazpMvQJ8zbD777D7H2AwQ9ESSCqAvLmQWjyYPYsrCWAAjUbT62mc4SohIXoYUqvVdphfDQQCvb6fwWCIPG/bJyocDvejh0II8fl0ot49eG92dC08/0U4JWRStAY04UB0mzbX/AWmXK4+H2F7Ao7sb+3PIaPRSCgU6rFdRkYGe/bsiTq2c+fOqMCkt/cSQgjRN7tONlHv8mMxdj7afqzWxbv7qrlgciapCUbeP1DD7IJk6l1+mr0BxmXaGJuWgE7bi+Ai4EX5x61oThvv0YQDkD5RHW059A6nBje8fCMAoVvWoCuYDwffgrTxkDGp07doavHz543HybKbmD82lfEZNrS96dsAkABmhBk7dixbtmzh+PHj2Gy2LkdFzj//fH72s5/xl7/8hUWLFvHcc8+xZ88e5syZ0+W9UlNTB+tjCCHEqBcMhfnocF23bVy+IB5/iDd2VUaOTcm2U9HkobShhb0VTi6dmUOi2UBGoqnTQMblC/LkX17mjuofkRpsoF5J5FzfL0nRNONXDExI0zN34lx8wTCGaXfgV7RkBSu54fB/YVT8ADQ/cw3JigOUMKQUwTd3dDoi89zmEzz5/uHI6/svmcLXzxmaaShZRj3C3HPPPeh0OqZOnUpGRgalpaWdtluxYgU//OEP+e53v8v8+fNpbm7mxhtv7NO9hBBCxK6kzo0/GJ+p9/cPVPPh4VpCp2T+KorC3z8t4/wf/5s7T95NarAagF9qbuDaxdO48KyFVJPKhno7T649wv9+eIzfbGvhD9tdPLwrkfO9j/FxSN1EOTncqAYvAI0lvL9hA6v3VLGvwhnVj6O16nRYXrIFs0HL3DE9r3odKDICM8JMnDiRTZs2RR276aabOm374IMP8uCDD8Z0r7Fjx3bInZk9e7bUKxBCiBiVN3nier+dpU2U1rdw/uRMAP7z+e3sLnewTLsHu1F9r63zH+d7F3yNRLOaLjAt186WknrMBh0mvRazQYc3EEKr1ZCbNI3q0GyOrbmWYm0VZQnTcTU7maIt5YPV/+S5UPTKVr1WQ3aSuhL1+xdPYfm0LLRDmDcjAYwQQggxAAbi174Gt5+fvLWft/dUAWDWa7ktvxKqgHk3Mf+Sr0e1v3pePlfPy+/yfuHwGCav/hUp/jpqvMnM1ByjKDuFWst48hq8UUFYMKxwslF9XZhqxaAb2kkcCWCEEEKIOAqGwgT7WOSl2unlWK2ry/NVTm8keAFYc/e5FLz0iPpizNkxv59Wq6E4I4EDVer00THTZF76z2WYDToUReHjo/V4/CFe3FpKIKRg1GvJTDQxLbfrTRYHiwQwQgghRBztKncwLsMW83Un6t28urOCF7eWAVCUnoBeq2FfhYMkq4GNR+qpdHgBWFicyl9vXYDhzW9B9R5AA2MX96m/j31xJu/tqybZamTRuDTMrfXJNBoNZ49PB2DZ1Kw+3XsgSQAjhBBCxIk/GGbzsfo+BTDH61uiXpfUqQmzh2uiR2TMBi0//sIMDFoN7H1NPbhoFdhz+9TnmfnJzMxP7tO1Q0kCGCGEECJO9lc68QX6tvLIG1Drcs3IS+KMsSmcqG+hxRdkc0lDpM3ls3K5dGaOGiC568HnUE+cf3+/+z7SSAAjhBBCxEE4rLDtRGOfr28LYJZOymBsWgL21pVED10xjfcO1NDiC6HTashINKkXNBxVH+15YLD0q+8jkQQwQgghRBz4Q2Ecnp63bGnxB/n3ZxV8dLiWK2bnRY57W0du2pZAt9FoNCRZDPgCYdJsRsa27WBdsUN9zJgcnw8wwkgAI4QQQgyCQCjMttJGnt9SSk2zjxKg3uUnGFYoTk+gyaNWxbWbO/9qNuq1LJuSRUqCEVy1sO0Z9cTY2FcfjQYSwIwCS5cuZfbs2fzqV78a6q4IIcTnkssXROlh6fQfPzrGs5tORB070aAm7p5abyXRbIhMJ53q3IkZ5Ca3ThWt+wnU7ANzMkz/Yv86P0LJVgJCCCFEP5U1tBDupmL5K9vLo4KXs8en8b83zCPZaqAoPSFyvCDFwtzC5Mhro14b2ZIokvsCcHyD+njFU5AyJi6fYaSRERghhBCin7oLXkrq3BysbgZgbmEyK6fn4PIFmZiVyB9umMfGI/WUN3qwmnSsnJ5DTrKFxRPSaWzxMyEzsfUuGtJtrQHMq6ug7pD6vHDRAH6q4U1GYEYYt9vNjTfeiM1mIycnh8cffzzqvM/n45577iEvL4+EhAQWLFjAunXrIuefeeYZkpOTeeedd5gyZQo2m42LLrqIysr2nVDXrVvHmWeeSUJCAsnJyZx99tmcONH+m8Nrr73G3LlzMZvNFBcX8+CDDxIMBgf8swshxEjj8gZ5b7+6yeKKadn8/Euzos63FY3LS7GQYjVi0KnDLVl2M5Oz7ei0GnRaDeMzbepO1OEQ7HxOvThvHiSkDd6HGWZkBAZAUSDQ0nO7gWCwdrpleVfuvfde1q9fz2uvvUZmZibf//732b59O7NnzwbgzjvvZN++fbz44ovk5ubyyiuvcNFFF7F7924mTJgAQEtLCz//+c/561//ilar5frrr+eee+7h+eefJxgMcuWVV3Lbbbfxt7/9Db/fzyeffIKmtY8fffQRN954I08++STnnHMOR48e5fbbbwfg//2//xffn40QQowADW5/l+d2lDXS4g+RlmDkvpWTMei7HzfQ97S/UO0B9dFog1vXxNrVUUUCGFCDl5/0rYJhv32/AowJPbcDXC4Xf/rTn3juuee44IILAHj22WfJz1c36iotLeXpp5+mtLSU3Fz189xzzz2sXr2ap59+mp/85CcABAIBfv/73zNu3DhADXoeeughAJxOJw6Hg0svvTRyfsqUKZE+PPjgg/z3f/83X/va1wAoLi7m4Ycf5rvf/a4EMEKIz6WTjS3q6MhpgqEwB6vUqaMlEzMwG3SEuplqAnXH526VbVEf8+aBVten/o4WEsCMIEePHsXv97NgwYLIsdTUVCZNmgTA7t27CYVCTJw4Meo6n89HWlr7MKPVao0EJwA5OTnU1NRE7nfTTTexYsUKLrzwQpYtW8Y111xDTk4OAJ999hkbN27kxz/+ceT6UCiE1+ulpaUFq9Ua/w8uhBDDlD8Y5litmwlZHbcO+OR4A25/iASjjvGZvdtaoNsdnsMhWNP6i2LBmX3p7qgiAQyo0zjfrxi6944Tl8uFTqdj27Zt6HTRkbnN1v4fj8HQsUiScspvBU8//TTf/OY3Wb16NS+99BL3338/a9asYeHChbhcLh588EGuuuqqDu9vNpvj9lmEEGIkqGjy4A913DpAURTe3avmvswsSO50hKYzel037Q68CT6n+nzsOTH3dbSRAAbUHJReTuMMpXHjxmEwGNiyZQuFhYUANDY2cujQIc4991zmzJlDKBSipqaGc87p3z/uOXPmMGfOHO677z4WLVrECy+8wMKFC5k7dy4HDx5k/Pjx8fhIQggxolWcUr/lVNtONHKivgW9VsOMvKRe36/bKaT9r6uPxedB0ZJYujkqSQAzgthsNm699Vbuvfde0tLSyMzM5Ac/+AFarTrkOHHiRK677jpuvPFGHn/8cebMmUNtbS3vv/8+M2fO5JJLLunxPUpKSvjDH/7A5ZdfTm5uLgcPHuTw4cPceOONADzwwANceumlFBYW8sUvfhGtVstnn33Gnj17eOSRRwb08wshxHASCivsrXCSZDV0OPfXzerKzSk5diyG3ueqdJvE21SqPs77WkyLP0YrCWBGmJ/97Ge4XC4uu+wyEhMT+c53voPD4Yicf/rpp3nkkUf4zne+Q3l5Oenp6SxcuJBLL720V/e3Wq0cOHCAZ599lvr6enJycli1ahXf+MY3AFixYgVvvPEGDz30ED/96U8xGAxMnjyZr3/96wPyeYUQYrgqbWjB5Qt2CGC2lzZi0GkZl5HAnILkmO5p6G4Eprk11SFxiBadDDMaRekhJXqEcjqdJCUl4XA4sNvtUee8Xi8lJSUUFRVJ3sYgkJ+3EGI0en9/NbtOOshLsTAt105+soX//fAYf/yoBH8ozD0rJhIIql+xFqOOK2fnEVIU3tpVicsX5Mo5eTR7A7y/vyZyz6+dNZbUBGPHN1MUeCQTQn64azckFw7Wxxx03X1/n0oK2QkhhBAxOtnYQkmdu8PxQ9XN+ENh8lMsTMnp+su3jdWo44yxKSSY1GmmLpN4T2xUgxeNFmzZ/er7aCFTSEIIIUQvNLX42V7ayPmTs9h8rIFmb3QF8lBYYWdZEwBXz81H24s8lfGZiYzPTGRchg29ToO1q3yZfa+pj5MvAX0nIzSfQzICI4QQQpyipM7N1uMNUeUlgqEw7+6tZn9lc5fXvb6rgjqXH6Neyxfm5nX7HlajjpRTcmdyky1kJpq7TuJtKlMfi8/r/QcZ5SSAEUIIISASsKzZV8WGw3W8ubsSfzDMW7srqW72Ud7FkmmAepeP36w9AsCZY1NJsXY/SjI5x05yD22iOE6qj6M49yVWEsAIIYQQQLXTF/X6cLULly9IpcPb47U3/nkrtS51T6RZBd3XfdHGWBsGAEfrEuqk/NiuG8U+1wHMKF2ANezIz1kIMdwFQ2F2nWzq07Un6tuTea+YlYte2/1X65fnF3S+0qgrXid4W8tlSAAT8bkMYNpK6be0DNEO1J8zbT/n07cwEEKI4eJYnZv6bnaV7s6ecrW8/+Wzchmb3nVVd40GEow6bKYY1884y9VHczKYEvvUx9Hoc7kKSafTkZycHNnA0Gq1opGqhnGnKAotLS3U1NSQnJzcYX8mIYQYLg5Vd52c2x1/MEyD28+FU7O4+awxrDtU12XbZKsBk74P/x9sS+BNKuhTH0erz2UAA5Cdra6jbwtixMBJTk6O/LyFEGK4CYTCnKhviW1ap1VJnZvd5Q4a3H6K0hO6DWD6FLwAOFoDmGQJYE71uQ1gNBoNOTk5ZGZmEggEhro7o5bBYJCRFyHEsHa8zo0/2HFH6d5oG7k5e3xatyP5CbFOG52qLYCR/Jcon9sApo1Op5MvWCGE+BzzBvoWvDS2+DnWWo33vEmZXbZLMOlYPjWrT+8BtC+hlimkKJ/LJF4hhBCiv/ZWqMm7k7ITKUyzdtnuwqnZZNn7sQ9cJICREZhTSQAjhBBCxCgUVthfqQYwC4pSu2xnNugYk9p1cNMrDSXqY/KY/t1nlJEARgghxOfW8U42ZOyNfZVOWvwhrEZdt5s2js+0odX2Y5Wr1wGuKvV5+vi+32cUkgBGCCHE51Jts48dZY0xX7e/0smavWpQMSXHjq6bAGVOYXJfu6eqOaA+2rLBHGP13lHuc5/EK4QQ4vPpRL2bcIz5u2FF4aanP6Ha6UOn1TA9t+vRl3HpNtJtpv51cvfL6mPBmf27zygkAYwQQoiRzdMEQR+01EHWtF5fdqzOjS7GIqZajYYvzStg/aFapuQkdrshY3eJvb1WvU99nHpF/+81ykgAI4QQYmQ68THs/jt8+uf2Y1f8Vv2yN9m6vTQYClPR5KEgpfdBRpXDy/pDtRj0Gr51wUTKGrrejqY/aS9RWurVR1vXy7Q/rySAEUIIMfJ4nfDCl8HnjD7+2n+qf278NxSf2+mlLl+QUChMLPvMNrX4eWNXBW5/CFCr93Yl2Wog0Rynvd88DeqjpeuVTp9XksQrhBBi5Pn0z+3BS8FCsGVBYk77+be/2+WlpfUtxBC70OwNcN+/duP2h7Cb9bx+5+JuE3fHZ3Y/+tNrigItrQGMNS0+9xxFJIARQggxsrQ0wMdPqs+v/B3c+g7ctQcu/VV7G2clnWXoOloCbDpW3+u3qnR4+NV7h9l10oFGA1fOyWNGfhJajQaNEmLlwR9wzRvTSTz2FgA6rYZJWXHaMdrrAEUd8cEqIzCnkwBGCCHEyNHSAI8VqbkhSQUw40vqcb0RJl0E/7lFfe1zwO8WtY9gtFp3qAanp3f73wVCYR56fV/k9dKJGe0VdZUwE+reZ3LduwBkb/kxAGeMTSGzP1V3T9WW/2JIAH0/VzONQhLACCGEGDnWPtz+/Nzvge60XJPMyXDWN9XntQdg/U8jp8JhhZONnm5v//yWE2w8UseGI3X8dt1RtpSoAdCLty9kZn5ypF1OzYdccugHkdfG5jISdAoLi+I41eOqVh8lgbdTEsAIIYQYGdY/1r7i6Pz7Ye4Nnbe78CE47371+f7XI1NJ/lC4212nK5o8/PTtA6w/VMe2E+0F7mbmJ1Fw2nYAmbUfAxDStG8GrG8u61/V3dM5K9RHe1787jmKSAAjhBBi+At4YMOv1OfzbobFd3fdVqOBs+4Ekx2c5fD0Sijd0u3tHZ4Av157BKc3SEaiiUSzHqtRx23nFHPF7NMCCFcNVZnn8MLMZ3lu9gs0JU5Ujzce7/PH61QkgMnpvt3nlAQwQgghhr/D70LAjZJUQN3S/wGtrvv2BgvMaR2hKdsML3ypy6ZhReH9/ep0zZKJGdx2ThG3nF3EbecU8x9Lx0U3VhR47mrO2XIHZ5f+Foc5n1BirnrOebKvn65zzZXqY6IEMJ2RAEYIIcTwVncEVn8fgKMZF/Li1jJ8wRDeQKj765bc0/7c64BQx+TdsKLw2DsHKWvNjbnl7LHd3jLddRCqdgHg0SeTbLdhz2q9xhHnACYyApMb3/uOEhLACCGEGBb8wTBKa3W5YGuhuJaPfgO/mQfOk4RNSaxLuhIAlzfI9tY8FaWrinTWVPhuSeSlpuFIhyZv7qpk45E6AApTrSyZkNFtH1Na1Pv5jCm8O+GHXDg1C0NqoXoy3gGMjMB0K6YA5tFHH2X+/PkkJiaSmZnJlVdeycGDB6PaeL1eVq1aRVpaGjabjauvvprq6uqoNqWlpVxyySVYrVYyMzO59957CQaDUW3WrVvH3LlzMZlMjB8/nmeeeaZvn1AIIcSwtP5QLW/trqSm2Uuj28+GI7XUNvvYfdLB6r1VUPIR1vfbV/o4ljxIs7n9y3xXuQN/MMzucgf1Lh8t/iBHalw0ewOEw61BjTUVpWgpANpDq6PePxgKR1YZrZyezR3nFveYhJvkVYOUiqylJNpsZNvNkFKknqw73I+fRiecrQGMjMB0KqYAZv369axatYrNmzezZs0aAoEAy5cvx+12R9p8+9vf5vXXX+fvf/8769evp6KigquuuipyPhQKcckll+D3+/n444959tlneeaZZ3jggQcibUpKSrjkkks477zz2LlzJ3fddRdf//rXeeedd+LwkYUQQgwHh6ubOVjVTLM3yLv7qjhU7SKswHv7qzlW64Ytv4+03THmZjxTr4m63uMP4fYFCYYVdpY18YcPj/Huvio8gRBHal2RdiU5FwFgWPcwYxs3oigKobDCe/tr8ARCmA1aitIT0PRiY8ckTxkAbmsBi8alqde0bSBZs7/T4nl9Eg7LCEwPYtoLafXq6Oj1mWeeITMzk23btrFkyRIcDgd/+tOfeOGFFzj//PMBePrpp5kyZQqbN29m4cKFvPvuu+zbt4/33nuPrKwsZs+ezcMPP8z3vvc9fvSjH2E0Gvn9739PUVERjz/+OABTpkxhw4YN/PKXv2TFihVx+uhCCCGGi9OXNxtCLXDgDQBemPkMrvSZXKrp+nduTyCEohDZ3+jt3VXkJJnRaDS8Fz6T21vbfWHfXfzReisL1p5LED0a4HsXTaappefidjnOz5hc/SYA2UVTSc+2qydSx4HeAgE3vHUPXPqLmD57p1rqIRwANJCY3f/7jUL9yoFxOBwApKaqJY63bdtGIBBg2bJlkTaTJ0+msLCQTZs2AbBp0yZmzJhBVlZWpM2KFStwOp3s3bs30ubUe7S1abtHZ3w+H06nM+qPEEKIoXei3s3W4+pUTSAUxuUL9nAFLD72SwBCljRqbJNjfs+wohAOwwtbTuDWJXJs8u2Rc19v+RP36P8OwDVnFHDGmJ7L9Gt8Tq7d/fXI6/SCSe0ndXqYfIn6/NM/wYmuv6t6rbk1gTcho2OxPgH0I4AJh8PcddddnH322UyfPh2AqqoqjEYjycnJUW2zsrKoqqqKtDk1eGk733auuzZOpxOPp/Mqio8++ihJSUmRPwUFBX39aEIIIeLEHwzz6o4KSurUVINPjzfy9u7Kbq8xhFqYVKuW6G8464comh6WTHfD7VNXKh2e8R2+l/8CO8PqsuiV2i1MyU7kzKKegxddyEvWX0/b2Tq1KPr1ih+3Pz/+UZ/7GxHJf5Hpo670OYBZtWoVe/bs4cUXX4xnf/rsvvvuw+FwRP6UlZUNdZeEEOJzrbzJQ4s/iHLa3s++bqrhAsys+ifGkBtSx+Ge/MW49MXhCfCPYxpu8N9HWNEwRlvDFeN7ERiFAtywbjE61ylB14wvgSUlul1itlodGOJT0K5tBCZREni70qcA5s477+SNN97ggw8+ID8/P3I8Ozsbv99PU1NTVPvq6mqys7MjbU5fldT2uqc2drsdi8XSaZ9MJhN2uz3qjxBCiKGhKAprD9TEfmGghXnlz6nPF38busl7iYXbF+SMMSnYklJpsqgj9Omekh6uAn3pBnSKOuVVkryIPy/bCVf/sfPGbauRGo/Dgbfg+Wug/mjfOiwjMD2K6V+GoijceeedvPLKK6xdu5aioughtHnz5mEwGHj//fcjxw4ePEhpaSmLFi0CYNGiRezevZuamvZ/2GvWrMFutzN16tRIm1Pv0dam7R5CCCGGt+2ljdQ1+2K+zrrrLyQEGnCY82DWtXHrT5bdzLO3nMlVc/NotI4FIK2lBBQFa9Un6EOdpycYyjdHnh9OX8aC4m6mnFLU+1K+HV78Chx+B3Y817cOywhMj2JahbRq1SpeeOEFXnvtNRITEyM5K0lJSVgsFpKSkrj11lu5++67SU1NxW6381//9V8sWrSIhQsXArB8+XKmTp3KDTfcwGOPPUZVVRX3338/q1atwmRStwu/4447+M1vfsN3v/tdbrnlFtauXcvLL7/Mm2++GeePL4QQIt7qXD4+OlwX83WGoBvbp08CsLXgZpbpDIC/3/2pafby7MfHuWVxEXqtlgbLWMbxIeeV/Jzw8V+hVYKcnXMtB1N/0OFafelGALwFi6kuvorluUldv1HbCEzwlGCo7lDfOi0jMD2KaQTmd7/7HQ6Hg6VLl5KTkxP589JLL0Xa/PKXv+TSSy/l6quvZsmSJWRnZ/Ovf/0rcl6n0/HGG2+g0+lYtGgR119/PTfeeCMPPfRQpE1RURFvvvkma9asYdasWTz++OP88Y9/lCXUQggxApTUuemqOG535p74I1pPPY3mQvZnXRq3/uw66aDJE+BAlbo6dVd2e20ybev00JzKlzpcl+w5gb5yGwDNy37OBVN7CCasnYzONPQ8TdUpqQHTo5hGYLos13wKs9nMU089xVNPPdVlmzFjxvDWW291e5+lS5eyY8eOWLonhBBiiNW5fO1VcHtJ621kYunLzCpTp1s+HPtNFE1MX09d8vhDHKhqBuD6BWPYUtKA05yH5+zvYtn4WKRdSGNAo7TvraQPeVl56IdowkFOpi4if8yUnt9MowGNDk65DzX71C0GkvK7vq4zsg9Sj2QvJCGEEHERDIU5UuPqueEpDEE3+X+/lAX7f4xWCRFMncCx1HP63RdFUVAUhTX7qwmFFQpTrcwpTI6c9531HV6e/gdcVjWhV6/4ufDj68FVC8C8iufIdu0nbLJTdtZPev/Gt38AF/8cHmiE/DMBBUo+jK3z/hbwNqnPZQSmSxLACCGEiIs1+6ppdPc+Z0UTDrKw7I8YnMcBKEk/l8bL/9LvlUehsMKrOyuY/fCaSP2Zry4ojN4qQKOlPGkO71ywmpb0GQCkOfbAP24myX2cBWXqSiPPWfeSXxxDIb2cWXDmbaDVQkZrsbtYN3lsmz4yWMHcTc7N51x8xuiEEEJ8ru2vdBJWoIe9EAGwuUvhl5dwq6O9XtfWSfeyLfcrXJmSB5T2uR8ub5BFj66NOrZ4fDpF6QldXuNLnoC1brf64vhHfLm1EF2TOZ/w7JvITey8fEePkloLqjbF+Hnapo8Sc9RpKdEpGYERQgjRZ4qi4A2E8ARCPTduddbuH8IpwUvAlsfR/Cvj0p8KR/Ry6OsWFDJvTEoXrVV1M26nJLljmY4Piu9BazCj601U1pnk1gDGEWNh1WbZhbo3JIARQgjRZ7XNPlr8vQxeQkGSKjeQ2aQu0Gi0FvH8rL9y8svvENDb4tKf8Zk2nv/6mayYlsV73z6XdJupx2u8aVN5ddqTvHp+e/2xNyY9Sk3WEpIs/diHqG0KqXwHhHsf4EWNwIguyRSSEEKIPtt2opEFxWndtknw1bJw+5Owej1T22qkTL6Uf+Y9TLM3SNicAsReN6YzGmBCViKTs+1kJEYHL1qNhgVFqWwpaej0Wo85E774Zz6uMaBPm8PiDHt03kyscmaDKQl8DqjYCfnzendds9SA6Q0JYIQQQvRJg9vPoWpXjwHMsiOPUNj4ceR1UGtCv2hVf1JdOqUoCi9uPUmlw0uiqePXm1GvZdG4NPJSLNiMerStwUlUjDL9as6KV4e0Oph2hbqqKJYdpZ1Shbc3JIARQgjRJx8driXcQ30wfclaitqCly8/xxbtHI7VNvOVMVOg9Fhc+7OnwsnJRg//2l7OdQsKO22j0WgYk6Ym9F57ZgGKAtlJZsqbPJj0A5BVcfmvIRSAxhO9v0ZGYHpFAhghhBAxC4cVTtS3dNtmavXr2DaqVdabE8aQOPlSwscaCOljyAfpJacnwPpDag2XG88ag1kfvdN0us3Y4ZosuznyfFJWIsnWfuS7dMVdD7+YDOEgfL8CDL1Y0dS2jYCMwHRLkniFEELELKwohDqpuGv01mN0lTO2YSMrjqjBi9uQygeLnhmwJcHN3gAvbS0jFFbItpu5+eyxUee1Gg2Tsu3d3qMg1UqytWOQ02/WVDAlghKG2gM9tw+HwaXuMygjMN2TERghhBAx8QZCBEPhDsfTaj5m4YZbAJjTesw/6TKeTbmbdHPGgPXnLx+f4LOTDgDOm5wRyW1pMz7Thq2TnJhT9XmpdE80Gsiaplbjrd4LuXO6b++uVUdr0IAta2D6NErICIwQQoiYHKpuJnj66IuiMHP7D6PbpV1Ay8VP4Y/TEunOHKt18f6BGgDyki3kp1ijzuelWDhnYvqAvX+vZE5VH2v299y2uTWB15YZW+Lv55CMwAghhOi1GqeXHaVNFKa2Bwq6mj2kH/4Ya0s5AEcW/4KSyjr2ZH+Bm3uT89EPf954HIDMRBOXzuw45fKFOXkYdEP8u3pKkfrY1EMir6LAc19Un+fOHdg+jQISwAghhOiVBrefbScaaThlv6N8xzbsf1lFUusOzDXZS6kvvpI9Sv2A96epxc/be9SE15vOHosvED2tpdNphj54AUgZoz72tBKp6QS01IHWANf8ZeD7NcJJACOEEKJHe8odnVbcnVPxIprW4MWRPJU9cx+i612H4svpDXLepEw8gRCFqVYOV7fvhG026HrMexk0ya0BTEOJmqSr7SKoqtqjPmZOBv0AJBSPMsMgNBVCCDGYqp1e3t1bhdMbwBcMEegkIfd0x+vVXZ3Tqz7EEmgEn4vL9t/L+IZ1ABy49BU2nPdPfJbMgex6xL+2n+SVHeW8f6CGSzqZOspO6nkLgUGTPgFMdvA3Q/m2rttVtwYwWTMGp18j3DAJT4UQQgwGly/IqzvKsZr01Dh9ZNhMPLflBF86I5/MRHOH9v5gGAWFRrefsf7DTN94O/MBPoHk1ja+KVfTkjkHmryD8hn2Vjh5e09V5HWW3Yz3lM0ktRoN500anECqV3QGGHM2HHobKnZAwfzO21W17oidLQFMb8gIjBBCfE64fEGe23yiw1SQPxjG6QlS6fAQDitUOjwEQ2E8/hAt/iDljR4yGrYx/Y3LO9xza96NuC/+TUz9CIUVvv3STl7bWUFZQwtKD9V8T6UoCvsqnQDMzEti6w8uIDUherol1WocmJou/ZHYuiTa0/k+TMApAcz0ge/PKCAjMEII8Tnx8ZE6PN3sHP3e/hpm5CVR5fBytNZFMKTwtbPGQDjImQd+Gmm3fuxdLEj38pxyMc3mHMZp9YCv2/c+XufG4QlQ2+zlp6sPRIKo3eUOqpu9rJjac80TRVFYvacqUgF4el4S+k7ySbQDVdOlPyyp6mNLFwFMKABNrZtDZUwenD6NcBLACCHE54AvGMLfi1yXtqkYfzAcKQiXtOtPpDYfBGDT0hfYHpjArLPH0ty6hLk7iqLwWVkTHx6q47frjkadm5qTyL7KZjYeqWfjkXoyEk2MTbN2cSf49EQjh2rURN2Hr5hGncvfZdthx9oawHQ1AtNcCSigM4J1iOvWjBAyhSSEEJ8Dx+u637eoM5mu/Vjevou01v2MShb9hKa03tcnqW32Mf/H7/PBwVpCp0wTWQw6/n7HIq6am8/cwuSo9nc8tz2yNPp0z358HIA5hclcPGOEldnvaQTGodbQwZ7b9SolEUVGYIQQYhTzBkIoCrj9wZium1r9OsuPPIwGNfBotI2ndvw10NK7+9S5fLyxqyJSsXfJxHT+78YzeGbjcRJMeuaPTeWzsibOLEoly26OSsota/B02GfpX9vLOVDVjFYD88emxPRZhgVLa5+7GoFxtgUw+YPTn1FAwjwhhBjF1h2sjVqh0xuakJ/FJ36DBoVQ6gTqz/oh7837HWh1PV8M7DrZxIuflOH0qsHORdOzuXRmLia9jpn5yVFtTXod96yYxPdWTOKquXkkmvV4AiHe219NIBRmR2kj//fRMQ5UNQOQm2zBbh6BJfbbknidnY8uRQKYpLzB6c8oICMwQggxSm05Vs/+SicLilJ7fY3JeZwVr54PgEefhO/m9Ti8Cp4jdT1e6w2E+NOGksgqofwUC3+4YR7v7qvu8VqbWc/ErEQmZibyP6sPsLfCyYQfvB3V5pIZORSkDuzWBAMmtVh9dFWBzwWm0/aHOnUKSfSKjMAIIcQotPukg2ZvbNNGAHm7fxt5vjX/5l5vKLj+YC3/3H4yErzkJpm5fFYuucmxBRxXzsljxdQsjPror6dzJqTz3YsmYdL3bhRo2LGkgDVNfV5/pOP5yBSSjMD0lozACCHEKOLwBAiEwgTCPa84Op3J30j6sVcBeG3yzzmevpSZPVyz+Vg9T7x/uMPxC6dm9Xkfosk5dqbk2Ll0Vg7/2l5OaUMLM/OTenU/k34Y/16eMwuOroWDb0Pu7OhzjpPqY5LkwPTWMP6bFkIIEYtGt581+6p7tTVAZ3LqN6NVgjQnTeRY2rk9tnd6A2w9Hp2Uuqg4jRe+vqDfheS0Wg3jMxO55owCxmXYer6g1dQce7/ed0DN+JL6eGSN+hgKwP7X4aNfQOVO9ZhMIfWajMAIIcQI4wuGqGjykmDUkZJgjAQsb+6upLbZB6TFfE9NwM05n30PgLqsxT229/hDvLazgrbFQr+/YS6hEGg1oBuiQnJGvZaFxbF/9kGTf6b6WL4NPn0awkF4657oNrIKqdckgBFCiBHicHUzY9MTaGoJ8OqOcnRaDTcsHEOTJ8CHh2ppcPe9sJt9398izyvzV3ZZWDcQChNWFB5+cx8Nbj8JRh0v37EIXzBM5SDthdSVzETT8KzC2ya1WK0H42mAN+7qvI219wnXn3cSwAghxDAVCiuEw2HWHqwlLcHI5mP13LakOO7vk+Q5SdquRwEoOfNBnKkzoTUZ91S/X3+UtQdqoo6tnJ5DTpIlslv1UCpI7bqK77Cg1cLyh+G1VV230QzjAGyYkQBGCCGGIUVRKKlzk24zsq+iYzART7MrX0Yb9NJkG0fN+C/BaQM5jS1+/rShJLIHUeS6gmTyUobPsubi9ISh7kLPZn0VSj6EXS+1H7vop2oOzPhlQ9atkUgCGCGEGIb2VjgxGwZ+nYWm6QSzK18GYMeEb2LRm8EfiJz/8FAtr+wojwpezh6fxkOXT+PVnRUD3r/eshh0ZCSahrobPdNq4ao/qEFMc2tRu5SxsPCOIe3WSCQBjBBCDDP7KpzsLGtiYfEA50MoCpY196IlhCt7AWUZ5zKx9VQwFKas0cMv3zsUaX7fyskUplo5UNVMms0U2exxOBiXYUMzjPrTI1tWewBjyxzavoxQEsAIIcQwUunw8N7+alIT+rcMuTsNbj8Hq5txlOzkzpK1APxX6RI+OL6PnCQz4bBCdXN0Fm+23czsguR+JQoPpDHd7GI9LJ1xM7z+LciYDNk9VdsRnZEARgghholD1c0EQuEOGxnGy/bSRv65rRyHJ4AJPz8z/QdoYGNoGh+E5wBQ6WhfSaTTahiXkcDSSZnkp1iG9QjHcO5bp+Z+TR2FyZkFOvkq7gv5qQkhxBCrbfbh8ARw+YIDUkn2eJ2bq373ceS1jhBPGJ4iUeMBYLXlEs4rzkCj0bD2QA1zC1OYW5iMzaTnprPH0uD2s6EXeyGJGGg0MGnlUPdiRJMARgghhoDbFySkKLy2s4K6Zh9LJmYMyArag1XNUaX+p+fZeUD7NGfWbgXAN+0a0pKu4muzC9h0rJ5bFxfh8AQ4WNU8YkY1jH3cskCMbPK3LoQQg+RkYwst/iAn6t0crG7GHwxT19xFxbg4cHmDrN5bFXn9pXn5XDzewry6VwGoW/RDvJf8FkXTvkGiTqsZVsm5vVE40vJfRFzICIwQQgygtpwWs0HHv7aXEworTM5OJCvJPKDv6w+G+dPGEgDMBi3fvnAiBYkaFr1zBTolRFNCMU2zv0HGgPZicNjNvdsxW4wuMgIjhBADxBsI8eLWMqocg1tiv6TOzbV/2MzucgdaDfz++nmcazvJ2eu+Soq3FIDtk74tVV/FiCYjMEIIMQD8wTBv7qoc0CmizuytcETlvFwyM4dZmXqSn/wCGiUEwGuTf44z81wKB7VnQsSXBDBCCNFPiqLg8gXx+EM0+4IEQmE0aChtaOn54jg6WNXM/v17WaytolTJ5Mlx27HVlpL0f3sjwcuOnC9zLO1c0ge1ZwNnWG/eKAaUBDBCCNFH/mAYo17LR4frqG32MTErEU8ghF6nIcE4uP973Xikjnf3VbHW+CjF2tbE3ZPt5wOWDA5M/AbrbFcMar8GUn6KhSSL5L98XkkAI4QQp/AHwzi9AYIhhcxEE2FFQa/TEm4tLqfVanB4AryztwqrUcelM3MHrPBcb/iCIb7x1085VO3iC9oN7cELENbo0CohfFO/xP55D3LSrYFq15D1Nd5ykobPRpJi8EkAI4QQrZpa1IJth6tdLChORafV8O/PKvjy/AKe/fg4K6ZlA1DT7KW80cP4TNuQ9rekzs3CR9WtABZo9vOY8Y+Rcwcv+QdlCTPZfbKRm84uJlzvBjxD1NOBkWYbuO0WxPAnAYwQ4nPNGwixv9LJuEwbx+rcHc47PQFCIQV/MKy+9gY6tNGEA9CaYxJPiqLw+LsH8QbCWAw6jta6uPiJCswGHXsrnABY8fKI4c9crNuKgQCH087HdsNzuJ1+cPlBMzoXm+p12pG3/5GIKwlghBAjSjisoNVq2F7aSGaiifyUvn2J1bt8bD3ewPyxqaw7WMvYtISY72F1nYAX72XpgTfwGtNomnEzXn0i/rTJKIVnx3w/fUsNGmsyAO/sreLB1/cSCHU9PaUnxPf0f+Mq3QYAKhJn8PaEB/nSMA5aTAYtM/KS+n0fyX0REsAIIYa9epePDw7WMi3XzrFaNwuLU9l8rB672cBZ49LISDThD4ZJs5kIhsLoTyktHw4r+ENhvIEQpQ0tePwhFhSnUevycaTGxfyxqTH3x9BSzZd2ryLfuSNyzOyvJ3vbzyOvG6d9DZJW9brWSkLNNia+/RW0YT8fT/gtHx1O7tAm2WLg+oWFGHyN4Chj1bH/RK+oI0InMi/g38X/j5BuYAvk9ZXNpH7dzMxLivz9TMpO5NPjjUPZLTGCSQAjhBj2mjwByhpaKEpPIKwoUUmzba+f31LK5bNyaWjxq8m3YUi1GXlrdyUXTsmiyRNgR2kTvqAawPTH+M3fJ8W5AwUNmry5OD0Bmiz5JGu92MvUnJSUvc+yMr2Styc+0uk9NEEvhE0ke04wvn49kzf+OnLu64f/k63ab7MhPJ13v3cxf95wHKMmyFTnBi6u+yPakg/RBNvzWRwzb2Vd9jcJuv39+lzdOWNMKikJBsamWwnGmLRs0GlZUKQGiqcGl4vHp2Md5NVaYvSQfzlCiGHrRL2bMb2c2gmFFUKK+sVaWt9CSoKRTbvqqHbGt5CcueEAKeXrCKPlw/NfYemSpWw7UEOD28/ErET8rgYmv38TCbU7mVz3Li5jJsx9DGoPkdxSiltnJ/vkRuZsuQuAm7t4n/81/lJ98gT8EPDqbJhDHVcQBcYupeGcB+FwfVw/p1ajwWrUkWg2YNBpSTC1f11cODWLYEjNCUqzGdFpNOhPq8eSZjPi9ql5QRdNzybNZurwHhqNhtkFydS5Yvs7GimbTIqBJQGMEGJYanD7qXf7ex3ADJasPX8A4Eja+TQnTexwPmRK4uBlr1Kw+ykytv6MMyqeI/CXD8FZyo093PvYxFtZuesc/mT4GYt1e6POnRq8uOd+gzWeyQQMiSxbfhl445NAbNKroyNpNiNfP6cInVaDTquJCl4AzAYdil7LxTNymJSdCMBMazIub5Bkq5HUBCNTcu1YDOomkcnWrlcL6bQasuxmdFot0LvPUZAqybtCAhghxCCqaPKQm9xeu8PjD2E2aNFoNPiDYTz+EC5/kI8O1ZJkMQz4hoe9pXHXsvj4ryk4sBVT/X4APs27nsRurmmY9y2ajm5lQsM6DM7SDufd6TNR5tzAmnIDJ5IXsGBMMl95Ws2puS/hYe5amMgiWxWZ259AX/Ep1QmTeXPSo1x33kxcio2SLaXotJrWVUbxCWCuOaOgQw5RVzQaTSR4aXPW+L7X9/3ivHz+9knHn1PH94VF/ZwCFKODBDBCiLhoavFHftMOhRU8gVAkcbPe5UOv1fLS1jJuObuIRLOez042YTXq2XikjuwkM1Ny7ByrdWHUa6l0eIfNKhObrxrbc1cz39H+5epNyKM6cVq3AQzA25Meoe7kX5jNASxnfYPflhVi8VYz1ebGMP5cClOtHG85gS8YigQvADMKkvGY02kpnIJr6kU8vUHdVRqNBiwp0NJxKXd/TMpOJD9FDSx7E7wMBKNeG+lDdwpTrWQkdpyOEp8/EsAIIWIWDit8dKSOBUWpKAqsP1TD/spmlk3JIjfZzO5yB0a9liM1Ls6fnMm7e6u5am4eAIFwGG8wxIGqZuYWpuDwBNBpNUzJGfzPkeY+gvHgTpI9KcDYqHOakB/rxv/htk9/AYBPl4Dj7B/gsI6hIWkanOw+iFAUhZDWxObC28iZk8fY9ASCFYdpsoyhISORrNZ2/mCYl7aWRa578PJpNJyejNvPnA+NBtISjHj9Ic4Ym8L2E00ApCYYMRnUqaDh4Ozx6Tg83f9cizOGtnigGD4kgBFCxKSt8Nv2E43Mzk+mrLEFV2uy5ifHG1g+NQtf6x5B9S71i3joCu1HM/vqsbvLQSnGUrGJa7Zfh25biK+hw2e5F92YLwNmcp2fMffV+zG1qGX5FY2WtcXfY8L0r9HsDRAKhICGyH0rmjz8qLVmS7bdjMWoY9mULHyBEC3+EGGl65/A5pJ6GlsCWI06blg0hjPGpvDu3uq4feacJDNJFgPJVmMkn+iscenUNvtITRhelWzNBh2K0nW8ptVoKM4YXjlRYuhIACOE6LXjdW7q3T4qHd6h7krvKQr60g3kvfMdvtx0TD2208KYU5Yhawlh2fg/zNr4P0wwpmP110XONZgL0d25lQObSplwym09gRAn6t3sOtnEo28fiBScq3KqP5v/++hYpK03GOLXX5nToWv/3HaSHaVNANy1bAJn9KEmTXesJh3nF2R2WLWj02rIHib5RaezGHVkJJoiwe+pMhNNsvu0iJAARgjRK4qi0NTD8P5wNLZxI4kb7oo+2Bq8OE057L78LaxvrmJOy8cAkeDlZMI0bOffwz8qsyg64aC2WV3qG1YUqp1entt8ghZ/iJ+/ewgAs0HL+ZOzeGt3ZYc+vLGrskPBvLKGFh59+wAA88emMG9MfIMXrUbD3MIUzK0rgUaSWfnJrD1QE3VMg0aCFxFFAhghRI8OVjVT3tRCasIIS55UFM4oezrycse4/6TJmM154S2U+Sz8Z8kidv91P3Zu5iZdLkZNgE/Dk6hSUjngLYC/a7CZXLg2fArA9tJGDlQ1R73F1Bw7douey2bmMjM/ma8tGoMvGKbFH2L3ySZ2ljVR2tBCldNLlcNLYaqV2mYfj69RA5+xadYBWVUzJScRk37kBS8A0/OS2F/pjDo2Ndc+RL0Rw5UEMEKIbm070ciHh2opSk8YMQGMNugh54PvMeXQPwBQdEbKbtjMrgodTR4//647l9cOltNWUNZJAk+GruKBy6ay0qjjL5tOQOtmiS5fMHLfU4OXjEQTF0zO5IHLprLlWEMk8Vaj0WAz68mym9lf6WTemBRuWDSGb764U93uIMFI/SlJunMLU+JWmM1s0DE5O5EMm2nIVhPFy6m5LsUZCZzdjyXaYnSK+V/4hx9+yGWXXUZubi4ajYZXX3016vxNN92ERqOJ+nPRRRdFtWloaOC6667DbreTnJzMrbfeissVXWFy165dnHPOOZjNZgoKCnjsscdi/3RCiD4LhRVK61v48FDtUHclJpqgl8nrvkFya/AC4FnyQ0K2bLaXNvLr94/wyg41eMlNMvP76+fy9cVF7PjhhcwuSCbBpOeCyZl8+8IJ3LdyMudMSOfl2xdx2Ux1pU5esoXrFxbylfkFnN2LUvgajYaitASuXzAGICp4+cstZ8atKFuSxcB1Cws5b3LmiA9eAGYXpABqHs/ls3KHuDdiOIp5BMbtdjNr1ixuueUWrrrqqk7bXHTRRTz9dPuwrckU/VvbddddR2VlJWvWrCEQCHDzzTdz++2388ILLwDgdDpZvnw5y5Yt4/e//z27d+/mlltuITk5mdtvvz3WLgshutHo9pPSyWoUtU7LyJqC0AQ8THz7WhJqdwJwMus8DiYuInvcjXzjT1sob/SgABOzbNx09lgaXH7mj01lf2X0tJBGo2H+2FQmZCbS4g8xKTuR4gwb7357CS5fEG8gxJZjDR070AWTQccDl00lEAqz/lAtKVYD31w2gam5dradiM9mhudNzsRuHh61c+JB15rvkmEzydYBolMxBzArV65k5cqV3bYxmUxkZ2d3em7//v2sXr2arVu3csYZZwDw61//mosvvpif//zn5Obm8vzzz+P3+/nzn/+M0Whk2rRp7Ny5k1/84hcSwAgRR4FQmL9tLeX8yZkUplqxGvWUNbQMdbe6lVXxPgkVRzEYrgTU39K1TSdYdOL3jNn2L3TeRoIGGycvepot4clUNHl54tU9nGxUE3fnFibzz/84i4PVzby3L7blytp+fpFmJJr4wpw8JmcnktJNef2+GC6F/+JNghfRlQHJgVm3bh2ZmZmkpKRw/vnn88gjj5CWpiapbdq0ieTk5EjwArBs2TK0Wi1btmzhC1/4Aps2bWLJkiUYje3/ga9YsYKf/vSnNDY2kpKS0uE9fT4fPl/7hmBOp7NDGyGEyuMPsaWknga3H18gTI3TR4s/xNzCFNYeqGFqrj3yG/BwYi9by4QtdwJwG7/kaO7l5Ox1YC1bz8LWNgFzKgcXP4EudyGcbOIPHx7DE1Dr1KyYlsXcwuRR9aVoMmjJMoyM3CQh4inuAcxFF13EVVddRVFREUePHuX73/8+K1euZNOmTeh0OqqqqsjMzIzuhF5PamoqVVVq0aiqqiqKioqi2mRlZUXOdRbAPProozz44IPx/jhCjEpv76nkRP3wHmk5XUrLccZt/s+oY+Mq/h312l20goPn/gZvUEsKsPFIXSR4uemssRSlJ+ALxmffoOFAp9X0mIMjxGgV93/51157beT5jBkzmDlzJuPGjWPdunVccMEF8X67iPvuu4+777478trpdFJQUDBg7yfEYDlY1YzVqMOk15Ju618hr0AoTKPbj/uUlTUjgdbn4PL930Eb8uJImkJ4+U/Q//sOEn3qFFAodQK/nvxXLptTAJ4An5TU8qv3D0eu/6/zx3PV3PwRl5CcZTdjNuiYNyYFu8VAtdOLVqPBrNexoCiVCVk97cYkxOg14KF7cXEx6enpHDlyhAsuuIDs7GxqaqILFAWDQRoaGiJ5M9nZ2VRXR89Nt73uKrfGZDJ1SBYWYqTaWdbE9Fw7dS4/b+2u5IyxKfiDYdz+UJ9WZJyodxMKK6w/VMuSiRkD0OMBFAqQ/datWLylKGjYccb/MLVgIX9b9BZ2s4GFxamk20woG48DUFLnjgpezixK5eazxuLwjqygrTg9gcTWpNzT/86SrNp+7fwsxGgw4GvtTp48SX19PTk56hLERYsW0dTUxLZt2yJt1q5dSzgcZsGCBZE2H374IYFAe9XPNWvWMGnSpE6nj4QYTbadaOSDAzWRGiWnOlrj4khNc8cTXdhT7gDUpbuNLQGa4ryL8aD44CdYKjYR1JooOfdJmu0Tu2za4g/ynZc/A8Bq1HHR9CwWj08fcTkvGg3MLEge6m4IMazFHMC4XC527tzJzp07ASgpKWHnzp2Ulpbicrm499572bx5M8ePH+f999/niiuuYPz48axYsQKAKVOmcNFFF3HbbbfxySefsHHjRu68806uvfZacnPV3yy/+tWvYjQaufXWW9m7dy8vvfQSTzzxRNQUkRCjkdsXZOORum7bvP5ZJav3VOFoDUYURSEYClPj9OIPhgmFFRRFYevxBj463P29RoQ9/wRgzbjv01h8WZfNqp1eLvv1xsjrBy6dynmTMrtsP1xp0HDpzBxsJsltEaI7Mf8X8umnn3LeeedFXrcFFV/72tf43e9+x65du3j22WdpamoiNzeX5cuX8/DDD0dN7zz//PPceeedXHDBBWi1Wq6++mqefPLJyPmkpCTeffddVq1axbx580hPT+eBBx6QJdRiVAuFFQIhNQDpyf5KJ8UZCSRZDeytcJKdZOb5LaXcfPZYXt9ViV6rocrhxWQY2QXNtH4nNJ0AoCTlbDoLRw5WNbNq7Q4OnlIl95ErpzOrIJldJ5sGp6OtkiwG/KH+JQmnJhiH7UaLQgwnMQcwS5cuRelma/h33nmnx3ukpqZGitZ1ZebMmXz00Uexdk+IEeuzk00Upyf03PA0/lA46nW9y0c3/4mOGHM2f5Ps8ncBCCQW4DMkdWjzyo7yqEJwU3IS+eK8/D79HEGddjL0pYqtBhZPSCfLbiY90cixWnef3h/UHZeFED2TMUohhoEjNS62ljT0+Yt3tDEGXZHgBaBu8Y/gtNJOnx5vjApeLpySxX9fPBmHJ4DX37tRELvFgM2sZ+mkDOwWAykJRiZkJuLoxa7bGg2ktVYwnpGXRLpNDTysRj3Tcu0cqnZ1d3nn9wTZcVmIXhrZ48tCjBLbSxsj9UoETKl9q/3Fl56hpTh6P7X/++gYaw+oqxkNOg1vfnNxzLsVF6UncOOisZj0OuYUpkQK9+m0Gq6cnUfqadsrnF7Yb3J2ImcWpQJEgpc2Go2GwlQrU3JiW+YsOy4L0XsyAiPEEFIUJbKLsWg3t+JvAFTOupOcaV+AWnU04/8+Osb7+6sJhNQ5siUT07n/kilkJvY+ZySjdYrGpO96nyejXsu4DBtOT4BJ2YnYzHpWTs/GpNdiNxsoTLViM+m73TTRYtSxdFJmr1ZAmQxaClIt3fZJCBFNAhgxKtS5fLi8QcaOsCmYQ9WuXiXtfp7oAs0ke08CUDPtFnKAA1VOXt1ZHlU9OMli4MIpWTHtT6TVarhgSlav2hp0WtJsJi6eoZaA6EvROLNBx5ljU3uckpqSY48pCBNCyBSSGCXqXX5e2VGO0zs0dU5cfahs2+IP8sHBmp4b9lN4OARIShhNSx26cM+jTXn71Z3svZYsQuZUnt5Ywp3P74gEL0a9lre+uZhvnFscc32X6bn2Qd+xOSXBGBn16cokqagrRMxkBEaMeDVOb+R5aX0L0/M6rlYZKOGwQr3bz98+KeXWxUW4/UEyE800ewMY9dpupwRK6tx4epls2hfBcJgP9tTw3KYTjMu0kWQxxJQkHI7jUqYZJ54l6f0n+CbgzP4rTLm4y7bppW8DUJN9LvsrnVgMOhTUHJSvLRrL7IKkDjknvTUrP7lP1/VXQjc1XfJTLOQmWwaxN0KMDhLAiBFte2kjZQ0tTM5Wkx/rBymfZP2hWuaPSWHDkToK06yEwgrv7Ve3u7hidh57yp24fUGWTVWnK+pdPnRaDRqNhiSLOgIw0EudNx9rYNdJtRJvhUMN8qxGHU+uPUJ+soXpeUlMzLJ1GMVQFIXXdpTzwcFaHvvSzMjxvRUObGY9oXCYT0oaIiM7obBaSK87Zx55IvLc/soNeFLWAh23NMg68ToJDnUbgFfsN/DUv/dhNen46dUzqGjysmRiBhVNnph/FhoN2M36YbnCR7YEEKJvJIARI5aiKGw+Vk+KtX21SFPLwAcwR2qa2X6ikVn5STS4/RSmWQE4VuumqHWEIxAKs7vcwZzCZMIKvLCllLHpVgpSrRyvc3PV3PwB7WNNs5fPypoAGJtm5Xjr9EtL64jPySYPJ5s8rN4LOUlmDDoNi8al4fQGWH+olg8OqpseXvd/WzDoNJgNOpq9Qf7vo5Ko93luS2nkeZLFQLLVwC/ePcgXz8hnTFrraE84hIIGDe0Rm/Ht78DsZzv0O//wcwB4tVZ+uUUtTHdOYTrT85KoaPJ2aN9bSRbDsNxOwG42YNTLTL4QfSEBjBixmloC+ALRv/nXueIfwLRNE2UkmgiEwry3v/d5K8GwwvE6d9R0TOMA7kfUFjit3V+DAoxJs7L2O0vZebIJR0uAJ9ceJhxWMOi0bDvRiAJUOrz8/N1D6LUagp3kywRCCoFQzzk+Dk8gkqz67Mdq9dxzxqfzNcsGNCgoJjv/N+Nv3Lb9CnSVO0iYVAGkRq43VWwhuX4HAMs8P4kc/+31cznZGPuoy6mGY/ACSPAiRD9IACNGrHq3r8MxpydAszcQ2cU3Hk40tLC3wsGlM3NxeYMx560MVgrt3goH83/8fuR1aoKRK2bnRqZN7BYDSyZkcPnsXD4+UkdxRgJ7yp3sr3KiKESCF51W3YunMNXK6r1VBIJh8lOszC1MZkJWIkkWPcfq3FQ2eXlxaxnZdjMmgxZvIEQ4DEdq2wu4fXSkjl0nnmGpDtYkX0ODLo1Q6gT0dfuxu44SVqYB4AuGUFZ/H4AXgudRrmRy5thUVp0/rt9Li8e0jpANBya9NjIKJoToHwlgxIjV4O58JONEnBN5Gwd4Wmp/pZOfrj6ASa8lEAzj9ocieTK9pW7e2F6VdkyalfMnZWI2dP3ln5NkISfJwvmTM3lvfzVVTi+/vW4uh6tdTM21o9NquHhGNqv3qLk9RekJFKRaCYXDpNtMNLoDrP7WOdS6fByqdlHW0ILdYmD51Cz2VDhYs6+a1duPski7D4CflU7k8IljXJ6bx1T2o6new8onUkg067F5q7ndvJ+QouHx4DXcsHAMi8al9TsQtZsNLB1GGzpePS+ff25Tl4gPVUKxEKOFBDBixOoq3+VAVXPcAhhFUahr7jjS01+VDg8/ffsAbn+Q7aVNkeNPrTsKwJ3njY/pfvsqnZHpm28sKe42cDmdTqvh+xdPwajXsrA4jcOnlMDvaerFatSj1XT8e9BqNHxxXj4LAlsxHQ5wUknnsJIHwAvVBTxigHPLfseD+iP8r/cybtavBuBkwjR+cd0ytp9o6nX/u3P2hHQsxuFTHC7dZuKCKZm0+EMdKvsKIWIjE7BixGrqojhYtdMbt9on7++vYW+Fs+eGMfrbJ2V8dKQuKng51c6yzo935X/XHwNg5fTsPn9hD0Q+Rp5fTfr1Zs9nxw8vZFqunX+Fzomc/5p+Da9aHuTW1gCmac5/xFSYrit6nZYki2HQa770xvjMRGbK6IsQ/SYBjBiRwmGFOlfnIyP+YDguBe3KGlo4UBX/4AXg4hnZrJyeTZLFwLIpmdy+pJi7lk1g1XnjANhwpI6/bDqOu4cCeX/dfILfrTvKx0fr0WrgjnOLB6S/fZXkUadLnNZCNBoNSydlsO4Hl/C3yb9hl30pAJlKPVrChHUmGnKWxOV9c5PNUltFiFFOAhgxItW5fB1WIJ2qvwMwHn+Id/ZWRfbciSenJ8CkrERWnTeem84ay90XTmJ8pg2Aq+fmY20dQXnkzf3d1rUJKwr/89Z+1uxTc1Rm5CW1L10eJlK86jJrp7UQAL1Wi1aj4YqrvsLexb+m9Ip/0mTOQ0GDY96dhHV9K1B3umy7lOUXYrSTHBgx4jS1+PF2E7y0qXJ4yU7q2xfZh4drafbGvj1AT1zeIH/eWMLeCge3nF3U4XxqgpEbF41ha0kjRr0WfTd5Er5AmCvn5PHJ8Qa+ODcf1wD0tz+0gRaym/cCUJ84qdM2npwF/HPuK6Ql6DlnUnZk08b+6m6TRSHE6CABjBhx9lU4yU/peWnsm7sruXVxxyChN7yB+C91DYUVHn17P83eIEdr3V3mnJj0OhZPSOfSmTlc+MsPybKbONHgpqzBQ4Pbz+TsRHKSzGw8Use3LpjAySYPhalW/rrpRId7xXM5eazG7PgZesVPMDGfpoRxXTfUaFA0wyfRVggxMkgAI0YUbyDEkVpXrwKYvvIHex7d6Yv1h2rZX9mMUa/liWtn95is2pY8XO308eqOisjxkjp35HleioWC1K5/FguK1EJxKVYjDYO0zQKAxVVGzsG/AOBYfD8EBm/FTUYf90kSQowsMs4qRpSyhhZCMSS4ePwhlBg2HfrwUC37K+OfuBsKK7yztwqAFVOzerUK5eIZOfzxxjMwdTFSk59i6Xa5tM2kZ2xrTkxRegIpVsOgLd21N6m1X2oSJuCZeMWgvGeb7gI6IcToISMwYkQ5XBNbjkRjix+dVkNWD0mdDk+AKoeXbScaWTqp4yaD/bW73IE3EMao0zKjNXjpzWrh6XlJ3Lq4iEnZidhMerYca2BPhQOLUcc9yycxJcfO9tLGTq9dPi0rapqqOMPGrILkOHya7mndNRQd+D8AahKmMPDvqDIbdH3epVoIMfLICIwYMRrdfo7GGMCAuutzYxfTJ83eAJ+UNOALhiK7SQ+Ez042AXDT2WMjoyA9BVVtDDotqQnqhpUWo475Y1OZnpvU5WiKTqNhUnZipyuS5o9NwWYawN9bFIWMf19PUuNuwloj+zIvHbj3Os3SSRkxVzAWQoxcEsCIEeNYnavTzQZ7Y1e5o9PjZQ0egmE152Wgcl9qnF6aWgJoNXDOhPTI8XSbaUCmOxYUp3HRtOxOz1mNerLsZoozbHF/X4BMx2cYa3cDsGf585QnzRmQ9zmdRqNhSo59UN5LCDE8SAAjRoza5r4noXY1AqMMwlaLHx2uA2Bcho2E00Y/LpuVQ0qCgQSjPqby/92ZPzYlsoFjV84YmxL3yrv6kIfLPr0JgIaMM3FlzI3r/bumwSjLpoX43JH/6sWI0dnu073V4PbHbXuBWO2pUEd/5o1J6XDOpNeRk2Rh8fh0Vk7vfNQkVj3tX9T2vovHp/fYLhaXHLwPgJA5lT3zfxLXe3cnP8UyrPY7EkIMDglgxIgQCis0uPo+AuMPhdl0rD6y4eFgURSF43UtABSmdT1dlGQ1kBeH0vexjOLMyEviounZXSYTx7InUWLzUYobNwLQPPcOPAkFvb62P4x6bdxGroQQI4sEMGJEaHD7+5z/0uaTkgb+9kkpx2pdKIoyYDkvp6p0ePEEQug0mkh5+64STftbPTbJYojpy1yrVfNGMhKjV+4kmPSMSbWSZY8+rg80o3d3nug8tuwVADyGFJrn3BFjz/tu7DDbOkEIMXgkgBEjQrXTG5f7ePwhXttZQb3bz6Hq5rjcsztt75FmM0ZWDc0uSOqyfX/qtPR2VdPpzhiTyrjM9qTeL87LJ9NuJivxlPspYRZv/BpT/nEuBndV1PV6dxXFx18CYMPU/we6wVkJZNRpmZYribtCfF5JACNGhCpHfAKYNusO1rJvAArWne5Qtbrsu60+iVGnpSi96xVAiWYDiea+LXMe080UVXcmZSeSl2xh3Gn90mo1ZNhMFKUnUOD4lGTHAXRBD7aqLVHt0vb8CUOohUrbdErT47ObdG+cMTa1Q1K0EOLzQwIYMex5AyEOVMU32ChraInr/bqSbDEwNq19OiY32dLjKEtOUt9yYbrLsemNJGvHkZO0xl2c7d/IzOpXIseyd/8ebUgNKDXNlaTtfQaAT/JvAs3g/S9lZl7XI1lCiNFPfn0Rw16D208gNDQriPqjxull2dQs5hfNZNPReoBIQbruJHcSSPQkyWLAEO+lxDX74U/LyABOrU1sbTzA+IN/5GDaTehL3kcb8tGYNJVjqUsYjAmd8Zk20mymHpeKCyFGNxmBEYOjfDv8aTkcXhPzpY0tg7cJYbyEwgrv7a/h689+yoeHamO6ti/VZOMevAA8f02HQ41z/gOArMq1jGnchHX1XQBUZSzu3d4IcTAlxz5oezoJIYYvGYERg+O9H0HZFnj+i1B8HlzzLJh7NwXQ6B7cpc/xsPZADbUuH3azntkFyZGdpXsjN9mCQdf7L+hxA1FVN+ABR2nrG1wAYxeDNY2U8RfAjt+R5NjPVY5vRppXZQ187otBp4nLUnMhxOggAYwYeOEQHN/Q/vrYB/A/hWC0wQ2vQsH8bi+vitMKpMH0/gF1ufHtS4qxt46o9LauSmqCsdN9jLoyIPv/VKu7SZOQAdf/s310JRwGNHBKBWN/YiH1KbOh0cNAjYsUplr56oIxvZqCE0J8PsgUkhh4TSdACYFWD1Muaz/ud6kjM91QFCVuS6gHy9FaFxVNXrQaWDk9J3I8llVCE7JsgzUj07nKnepj9szoqSGtllODl9Alv8R/5w5sZgN5KRa+MCcPo14b90Dm1A0thRACJIARg6H+qPqYPhG+/BzctQfO+4F6rGwzhIJdXtrUEhiUgnPx9O5edfSlIMWK7ZQl0bGsLpqcbR+YvJbeqvxMfcyZ2fHc5NYdpheuQjf/FmwmPQuKUrl0Zg4pCUay7GZm5MdnhVCqVYIWIUTnJIARA6/hmPqYWqw+JhfAOfeAzgjhIDjLu7y0ztX3/Y+Gyt7WvY+KM6KngQZkqmcgVH4Ge/6lPi9c1PH8ysfg6j/B8kcih1ISjFiN0cGa3dz/zzs2XSrtCiE6JzkwYuA1nlAfU8a2H9NqIbkQ6o+oU0wpYzq9tKGLXaSHs998dS5njy/leJ07csxq1MV99+e4CwXg1f+A3X9XXyfmwPhlHdsl5cGML/Z4u+l5SX2qt6NBQ1F6AmeMTY35WiHE54cEMGLgNbUGMMmnBSkpRWoAU3sQijpfxeL0dj29NBy1bRaZl2yJqh6cMgD5G7Fsttgrb3+vPXgB9e9E27+NEi+anh1zDZ/ZhcnYpMKuEKIHw/xXQjEqNKnLcY8EUlGUU77M8uapj2WfdHnpYO8e3R/1Lh8/en0v//HcNsKnbTyZYTN1cVXfFKRaGZcRx+mVcAh2/6P9tVYP87/e79tqNBqMei29XRVuM+kleBFC9Ir8n0IMvNYRmI/rrCS7/ZF9gShcqD7ufhkmXwzTvtDh0pEUwOwoa0JRQFHoUCU2vY8BTLrNSLqtffRmZn4Sh2tcLChKRRPPEZh9r4HPASY73L1PDWAM8au58sUzCtDQfa27LLuJwtT+bYcghPj8kABGDCxPE3jVpFanOZfKJm/7l3n+fDUPpqkUdv29QwATDiu4RsgUkqIoHKtVc15uPGsMwdOmTfq6QaO+dfNHBYXJOXYybCbOHp+O2dC/qZ0onib4d2tRujNvA1Ni/O7dqm1U5bxJmR2WWGckmphdkByVBCyEED2RKSQxsBxlAHiNKQR01uiaLiYbXPaE+rz+cIdLm71BwsrI2AOpptmHJxDCqNNyxpiOyae2PgYwABOzbKRYjeQlWzDqtfENXgD2/xv8zZA2Ac797/je+zSzCpJJTTCiQa2qOyHTxnULCpkuGzMKIWIkv/KIgdW6AsltyQWguvm0onTpE9XH+qMQ9IO+fbpkpOyB5A+GWd+639H0PHunq436k9eRaTeTaTf3+foelW9TH6deHvXzHygajQaLUceSiRk9NxZCiC7ICIwYWK0JvG5LHgANLn90gmtiLlhS1Eq9256JurSmeWTUgHlu8wkqHWrl3eVTszucH5BRk3iqax39Sp80tP0QQogYSAAjBlZbAGNVA5hgWIkeWdFqYen31ecbfqlmwALv7q2KFIQbzhRF4aPDdQCcNS6d7KSOIyWJw3lVjaJA3SH1efr4oe2LEELEQAIYMbCaoqeQAHaXnxaYzPsait4MzRVQf5QjNS72VTppahn+K5AqHV4qHV50Wg0zuyifnxiHirQDpv4ouGvVqsiZU4e6N0II0WsSwIgBoyhKhykkoENgEtYacSS3fnme3MqGw7WMhNxdly/Ik2uPADAm1drl3kV9XYE0KI5/qD7mnxnXZdNCCDHQJIARA6be5eswhdSmoskTqfFyoKqZEnNrAPPqHYTDoUHtZ199erwh8nxOYXKX7azGYZj/oijgaYQ3vq2+LjpnaPsjhBAxkgBGDJjammrwOQFwW3KiztU0+/j3ZxXUNvvYcKSWavuMyLmxNe8Naj/7QlEU3thVCcAlM7LJT+m6AFtcC871hqsWDrwJJz8Ft5qfg9cB/lP2Jdr/b/jp2PbXEy8a1C4KIUR/DeOxbTGShcIKzbWteyBZUgnpLED0Euq6Zh9/+6SUUFihNGVh5Hhh7Xo+s58/iL2NTWOLn0+PN7KvUg3OFk9Ip8oxTFZMBbzw9Mr2ujrmZPjyc/DXL6h1d+7cBglpsOP59muWfh9yZw9Fb4UQos9kBEYMiHqXD31L62//tqwu24Val1QH9Db46ssApLiODnj/+qq80cPLn5ZFgpeFRanMKUwZ4l612vsK/DgruiigtwnWPgzhgDpldPgdaK6GI62jXMt+BEvuGYreCiFEv8gIjBgQx+rc2HxtAUwvC5a17lZt81YMUK/675G39uENhAFIthi4eEZOD1cMkqAPXvuvzs+VbWl/fngN+N1q3Z38+bD424PTPyGEiDMZgRED4litG6O3NYBJyOzdRckFAJiCzZiCzQPUs77zBkJsO9EIwFVz8vjfG+Z1WnV3QCkKvPcjePf+SM0cvE54JFPdDgBg4X/CvUfhqj92vH7vv2Dvq+rzccN3mk4IIXoiIzAi7vzBMLXNPozeevWArZcBjDEBrOnQUkeirxKfvvebCgZD4T70tPe8gRDbSxsJhBQSTDryUyxMyk6kpM49oO8bxd8CvzsLGkvU12fcCqlF8OFj7W1mfQUuelR9njun8/uc2KA+5swauL4KIcQAkxEYEXflTR7CitI+AtPbAAbU3akBu7ey15fUNHv56h+38MYudVVTvDW2+Hngtb2sO1jLg5dPY2FRWq9XFhm7qA3TK9V7IdS6G3coAP97TnvwAlCzD5qr4JNTRlpmfaX9edq46PutPCXQ0eigYEHf+yaEEENMAhgRd2UN6nJdo691BKa3U0gQCWCSfD3nwYQVhdpmH79ccxiHJ8DRWjev7izHF4hfHZkWf5C/f3qSFn+I3ScdXDozp9c7J1uNOlIS+rg5oqcRnr4YfrcI6o7AgTeg/kh0m4NvweOTIOiBlCK46U0oPrf9vEYD535PDVa++jJkTWs/N/UKSEjvW9+EEGIYkCkkEXfBsDqdY/KqOzRjy4TGXl7cyxEYfzDM3z89SZUzeml2iz/E3tYVQvFwstGDpzUg+tmXZqKNoaZLTnIfK9sqCrx5j7qCyNsEv5kXff7Mb8An/ws7nms/dsVTMPbsjvc6979h0Z1gtqu7fU9YDkn5sPzHfeubEEIMExLAiAETyYFJyIg9gPF1HsCohe/qeOSN/YRO2W9g1Xnj2VnWyMYj9VQ0efrT7Sh1LnVK6qxxaeSnWGl0+3u4ol26rY+jL+/9P9jzj47HNTr4zgEwJ8Gnf1aXRoM6bdRZ8ALqZplmu/pcb4Tr/t63PgkhxDAT8xTShx9+yGWXXUZubi4ajYZXX3016ryiKDzwwAPk5ORgsVhYtmwZhw8fjmrT0NDAddddh91uJzk5mVtvvRWXyxXVZteuXZxzzjmYzWYKCgp47LHHEMNfW10XlDAGX2up/ZhyYNSl1PZOppAcngAvfVrGthONkeBlYpaNa+cXcOnMHPJaRzxO1LdQ3uihxR/E4w9R1tBCpcPT3rcYVDrUEZ68PoympNtMMV9DSwNsfKL9tb11C4aChfBf29Sfpd4EGZPa21z6y9jfRwghRriYR2DcbjezZs3illtu4aqrrupw/rHHHuPJJ5/k2WefpaioiB/+8IesWLGCffv2YTabAbjuuuuorKxkzZo1BAIBbr75Zm6//XZeeOEFAJxOJ8uXL2fZsmX8/ve/Z/fu3dxyyy0kJydz++239/Mji4FU71ZHLCxBB1qlNRclIQPoZVJuF1NIYUXhg4M1UUFIgknHuRMzKM6wodVoyLabyUu2UN7k4Vsv7qDJE+Dxdw9F2i8qTuPMotRefxaXN0h5ozqaMy3P3uvrIh/FEuMu1DX71VVGAGjgtrWQN7fztisfg3WPqrkssgmjEOJzKOYAZuXKlaxcubLTc4qi8Ktf/Yr777+fK664AoC//OUvZGVl8eqrr3Lttdeyf/9+Vq9ezdatWznjjDMA+PWvf83FF1/Mz3/+c3Jzc3n++efx+/38+c9/xmg0Mm3aNHbu3MkvfvELCWCGuRqnGsBY/a3TR5ZU0MXwRd5aC8YcasYYdOHX2wDYX+nkRH0LWg3cdk4x0/OSOFTVjFbbnpOi0WhYNiWTZzed4Fgny5s3HaunscXPkgkZWHqxweLWEw0oQG6yuU+jKUnWGAOYt+4FpXU5+Pxbuw5eQJ0yuumNmPskhBCjRVxXIZWUlFBVVcWyZcsix5KSkliwYAGbNm0CYNOmTSQnJ0eCF4Bly5ah1WrZsmVLpM2SJUswGttzCFasWMHBgwdpbOw8mcLn8+F0OqP+iMEVDiuRHaatgT5MHwEYE/DrElrvUR853Fa6/8yxqWQnmZmVnxwVvLRJtho5Z7y6uibJYmB8ho2Lp2dHzh+oaubtverojscf4qWtZby2s5ywEj29tL/Sya6TDgDm9mGrAItRh0kfwy7UtYfg+Eftr09dDi2EEKKDuCbxVlVVAZCVFb33TVZWVuRcVVUVmZnRX2p6vZ7U1NSoNkVFRR3u0XYuJaXjF8qjjz7Kgw8+GJ8PIvrE4QlEpngiAUxCL7cROIXHlIaxxU2Cv54myxj8wTBVrbkoU3J6nsqZOyaF718yhQ8P1XLpzBx0Wi05yRbe2l1JpcNLWYOHPRUOvIEQG4+oQVJGohld6wojRVF4b181ADlJZorTE2L+DNl2c2wX7HtNfRy/TF1RlJjdfXshhPicGzV1YO677z4cDkfkT1lZ2VB36XOnsaV9hU6CP8YqvKfwGNURlITWERi1MJ46omLvZV5Jlt2MXqeNFJzLSDRxzRkFzMxXa7h8cKCG7KT2IOM3a4+ws6wJgDd2VXKk1o1Oo+Giadm9Llp3+vvHpOm4+liwQIIXIYTohbiOwGRnq//jra6uJienfZO76upqZs+eHWlTU1MTdV0wGKShoSFyfXZ2NtXV1VFt2l63tTmdyWTCZOrDqg8RN/WnLDFuH4GJPYBpMaYBkOQ9CcCJejWfpSCl/8mqSydmUNrQQlNLgH9uKyc/xcK0XDvv7K1m/aFadpY1RabBzhqX1uuA6XQx75HkKFcf21YdCSGE6FZcR2CKiorIzs7m/fffjxxzOp1s2bKFRYsWAbBo0SKamprYtm1bpM3atWsJh8MsWLAg0ubDDz8kEAhE2qxZs4ZJkyZ1On0khoe6U8r4R/JXersT9Snq7VMAmF79Gr5giL0Vav5LUUbsUzmn02g05J+yJHpMqpUbFo1hRmt13bbgBWBOYXK/36/XHGqwRlL+4L2nEEKMYDEHMC6Xi507d7Jz505ATdzduXMnpaWlaDQa7rrrLh555BH+/e9/s3v3bm688UZyc3O58sorAZgyZQoXXXQRt912G5988gkbN27kzjvv5NprryU3NxeAr371qxiNRm699Vb27t3LSy+9xBNPPMHdd98dtw8u4q/m1ADG35bEm9VF667tz7+GMDqSveX46ssIhhVsJj1Faf0PYAAWFKeRmWhCp9Fw+exctBoNT35lNhdMzsSg02A16rhv5eQ+TR31SSgITaXq89ZVWEIIIboX8xTSp59+ynnnnRd53RZUfO1rX+OZZ57hu9/9Lm63m9tvv52mpiYWL17M6tWrIzVgAJ5//nnuvPNOLrjgArRaLVdffTVPPvlk5HxSUhLvvvsuq1atYt68eaSnp/PAAw/IEuphzBsI0djip4i2FUR9n0IK6G3UJYwj032IpMbdwGSy7ea4BRQ2k55r5xeQZjMxPS8JhydAgknP9LwkxmXYOHtCGk0tAY7VDvBO04qi7ld04A0I+cCQAMljB/Y9hRBilIg5gFm6dCmK0nVFU41Gw0MPPcRDDz3UZZvU1NRI0bquzJw5k48++qjbNmL4qG32ceo/i4TIMurYp5AAmsyFZLoPoWuuACZHJdzGg0aj6bQWjMWow2420NQS6OSqOCr5EF68Hs6/H9b/VD2WVqyW/hdCCNEj2QtJxEX9aXsEbSy8g/HmZsa1bg0QK3drIq/Bo24ImR+HBN5ho2IHPHuZ+vzte9uPn3PP0PRHCCFGIAlgRFycvsnhvqzL0eUlMc6qlu5XFIVwWOm0+Fxn3AY1gElTGjHptWQkjpIVZo0n4PlrOh5PGQvTrhzs3gghxIglAYyIiyqnN/Jc72tCF/IB6sqeaqeXX71/GJc3yNJJmRT1ojBc21LqDI2D/GQL2sFKqB1o6x4Fdw0YrBBoaT++WBLUhRAiFjLhLjrVXZ7T6QKhMLWnrECatuk7fHPzOeQcfwWAx989SLXTh9sf4q3dlTS1+Lu6VYQ7EsA09W1X5yGi626EafX34bO/qc9XPgZffVl9nj4J5tww8J0TQohRREZgRAfHal04PAHm9HIPoMomb9Qu0Qaful+V35hMiz/I2gPthQsV4JOSBmb3cG+3Qa3Gm6FxkJZg7LbtcNLlVJenETY/pT7X6NTpIlMifLcEtHpJ3hVCiBjJ/zVFFEVRWLOvOqZVOKUNLVGvjT51BVLAlIrVqOeDe5by5TPyWT5VrQnz1p4qqk+ZcuqM26DmzqThID1h5MTZXW4hcHxD+/Pb16nBC4A1Fcw97+8khBAimgQwIkqFw0uLPxTTNYdrmqNet43ABEzqKEui2cCcwhTGZdgibb7x122RLQI6Ux2yEVY06DQK2foBrscSR+m2LkaLmqsADZx5O+TMHNQ+CSHEaCQBjIiy+2RTTO0dLYGo0Rpt0IMu5AHAb0qNamvUa5me2z7a8OrOCupcPjpT7QpRjzpKYQ/Wx9SnoWI26Eg0n7Z3UjgET18Mb90DObPgop8OTeeEEGKUkQBGRKlydD+1c7odZY1Rr9tGX0IaPc3hjvkg503K5M7zxkdev7GrEl8gesTH4w9xoMpJuaIWwUvxlsbUp6GS2Vn+S91hOLFRfa7VgRIe3E4JIcQoJQGMiAiGwjR5YqtAW9boiXqtbw1gmrVJfOvlz/j2Szujzmu1GsZn2njqq3OwGHQ4PAH+uKGEV3aU8+Dre3H7gvzvh0c5WutmX1gtgpfhPtT3DzWIcpM7KbbXWKI+ZkyB29aCbuTk8wghxHAm/zcVgJq8Gwwp9Hb19MYjdUzIsnU43jYC00giigIp1s5zQvJSrKycns17+6txeoMcqXFRUucm0aSn2RsEwGGfCC1rSXMf7duHGmSdrkBqPK4+pk8Y1L4IIcRoJyMwAoADVc14g50n7566RBrUkZo95Y5IoHEqQ+sKpNqQWqxu0bi0Lt+zINXK9QvHMCkrkbxkCw9ePo3ntqjTRedMSCdr7FQAknzlsX+gIdDpcu/aA60nxw1uZ4QQYpSTERgBQHmjh5wuNkw8WNVMeZMHo16LQashrNDlSqW2EZiaUCIaDZxZlNppu0h7nZaLpmdj1Gu5bGYueyucvL2nkpl5STj8eQAkecvbd24epkwGLUkWQ8cTFTvUx5zZg9ofIYQY7SSAEQBUdlOXRUFhT7kj8tqo73rgri2AaVASKUy1dv6l3oUkq4FHr5rBmDQrHn8IpykHBQ2GsA9roCGyvcBwNLcwpeM+TwEvVO9Tn+fOGfxOCSHEKCZTSII6l4+65s6XM3fGH+x6JU3bFFIjiUzOSuxTfww69Z9lWGugpbWgXYK/rk/3GiwdtjsIBWHfqxAOgCUVkguHpF9CCDFayQiM4JOShrjd69QRmEnZfQtgTuUyZpAQqMfmr6WWSf2+30CJyn9pKIEnZ7e/zp0zrKe/hBBiJJIRmM85hyfAkRpX3O6nuNWic40kMrGPIzCnchvVPZES/LX9vtdAsRh0JFtbp8pCAfjziugGk1YOfqeEEGKUkwBmlCo7bX+izngD6u7Qp68y6o+gSw00tAnpmA26ft+vLYBJ9Nf00HLo5CRZ0LSNsFTuAld1+8nZ18Oc64emY0IIMYrJFNIoVOXw8q/t5cwuTCYv2cL4zI71WkrrW1izvxpnjIXremIJNAGQnpkbl/s1WMYCkOk6EJf7DYSopOYj76mPlhS45R3IGL7TXkIIMZJJADMKbT3eQFhR2H6ikX0VTuwWPWkJJnStq2T2Vzp5d2814d5WreulYDBEkqJu7JiWkR2Xe5bbZwOQ27xr2C+lJhSE7c+qzy/6HwlehBBiAEkAM8r4g2FK6tp3b/YGQpTUutlR2oTDE8Bs0HE0jjkvpyqpqESrqMXt2nai7q+6BHXfJHPQiSXYhMcQn/vGS9Toy7F14CwHazpM+8KQ9UkIIT4PJAdmlClv8nSa0xIOK5Q3egYseAEIO6sAaNHaCOk6L4oXq5DWRLMxE2gtaDfMpJ66+qhyp/o4/gLQd7KtgBBCiLiRAGaUKT9tc8XBZGhRk1fdpoy43tdhbqvIezKu942HtIRTApXag+pjxuSh6YwQQnyOSAAzylQ0DU0AoygKJo+6UihozYrrvRst6q7UaS3H4nrf/tJqNBRnJLQfqN6rPkoAI4QQA05yYEaRQChMVTdbAgwkhydAhqIWxFNs8UngbVOToCbDDreVSBajLlI1GL8baverz2XbACGEGHAyAjOKVHSR/zIY6lx+CjTqCIwvIT5LqNvU2NQRjaKmTVyx79towvFd+h0rjaaT/aAqd4EShsQcsOcMTceEEOJzREZgRpET9T0XrxsoVU4vYzVqEq8ncUxc711rHR95Xty4gX3O/UB+XN8jFiun55CaYER/6uaN5dvUx9y5Q9MpIYT4nJERmFGkrHHoApjKJg9jtGoSb0vi2LjeO6QzU5MwIfI6wVsZ1/vHqjDVSkaiqb36LsDJT9THPJk+EkKIwSABzCjhDYSojWFH6XgKhRVqmj1koW7k6LXGfwrlvXH3R54neirifv/eshh1WIynbZEQ8Ko1YACKzh30PgkhxOeRBDCjRGlDC3EurNtrtc0+EsJu9JowAAFTatzfozpxKpvzbwEg0VMW8/VWo46r5+YzIy+pX/3odH+nV/8DvA5IzIW8ef26vxBCiN6RAGaUODmE00cVDg/pGgcAQUMiis7YwxV902AtBiDFdSTma5dPzaYwzcriCekdE3D7o+YA7P2X+vzyX4O2/xtYCiGE6JkEMKNEjXNopo9AXf2UiroH0kCMvoC68qdo2gIAUt1H1BU/MciyqwXnzAYds/KT49exQ2+rjxMvggnL4ndfIYQQ3ZIApg+UoZqr6UIorFDnGpoAJqwoVDR5SdM4AfCb0wbkfYrSE5gyfS7ojOiDLeidpb26TqOBJIshKuF2YXEqJkMc/um3NMB7P1Kfj13c//sJIYToNQlg+sAXjO23/4FW0eQhEBqaoKq22YcnECJDqwYwAfPAjMBMy7WDTh+pcmuo29er69ISTB2SbvU6Ldn22PZqyk+xqk+CPvjgUTi2Hp69vL1BwcKY7ieEEKJ/pA5MH/iC4c6TOYdIpWNoqu8ClNSqO1+PsXggMDBTSEkWA+MybOqLrOlQtQtz40HQzuzx2q7yXcakWTla6+703OnyUizMLkhWX/zzVtj/eieNpP6LEEIMJhmB6QNfMDTUXYjS4B66/JcdperS6UKjmkQc6GIK6dSSKbEqykhonwJKGweAzV3aYxBpPX258ymm5iRhNeoYk5aAqYek3vwUi/qkclfnwcu1f5PkXSGEGGQSwPSBPxjG4x8+QUydyz8k71vl8FLh8KLRQL5JHc3oagTm4hk5ZNrbd242G7RoexnVjG8bfQFIVVciaRqOcWZRSrfXdbdk2mLUsWRiBjaTnqJ0W5ftAHW6qWoP/OPmjidXbYXJF3d7vRBCiPiTKaQ+UBS1dH5RekLPjQdYIBSmfogCmK3H1c0bs+1mbKEmtT9dBDCJZgMrp+fwWVkTep2Gydl2jte52V/Z3O17JFsN7SMgEBmBoeEYM/KS+fR4Y5fXTsmxd3tvu9kAQGqCkZrmrqfh0mwm+ORtqG9dvr38ETAnq8FUxsRu30MIIcTAkBGYPggrCtVDtOvz6epcPsJDtCrqnb3q3kfFGQkk+OuB7lchpSYYOW9yJmeNSwdgUXF6j6MwE7MSo0v2pxSpjy11GIPNnFmUik6jYUyalStm55JmM6LRgF6nQaftx7xVK6Nei92sb99pOmcWTPsCzL0Bxp7d7/sLIYToGxmBiVEgFOaHr+0hM9HEwuKBWTIci+AQrT4KhNSVWFoNTMlKJLFCDWZ8CXk9XtsWWCRZDSwsTuXjo/Vdts1NtkQfMNshIQPctdBwjIlZM8i0mzEbdBRn2Eg0GwiEwpHRlf7KTTarAVTtQfXA0u9D0tBtJCmEEEIlAUyM3txVyYeH6gB45MoZw2o10mDyBcN86YwCWnxBdN5GjGEPAN6E3Jjuc2ZRKp+dbOp0GbhGAzlJnSx3Th2nBjD1R0nInUOCqf2fcUaiqWP7XuhqJGhchg0c5VC9Rz2QNbVP9xdCCBFfMoUUI09ATd7Npp7GI58MaV+aWoYm90VRFN7ZU8VDr+9jV7kDu0/dHdptSEPRxRZAaDQairtIok23mToPENNbd6auOxTTe3Un3WZiTmFy1LEZeUlMy02CP7Ru0JiQAUkFcXtPIYQQfScBTIyMOvVHttn8X+S8dBHhmvh9icaqvMkzJO9b1ujhZJMHs17L1Bw7Nn8tAM2mzKh2hl7uOTS2i2ToSdmJnV+QMUl9bJvWiZNF49JINLeP5kzJtaPzNqqjPQCTL+nfenAhhBBxIwFMjE7/Uq7Y//EQ9YQhW310tMYFwHmTM0mzmbD61dVILYbonKB0W+82dcxPsUTVYjHoNBj1Woq7WuWV1joC01gSY8+7Z9LrIhV385It5BncsOM59WT6JLjsibi+nxBCiL6THJgYqV+07fkaJxu9DFVKZ2OLnyIGdym3oigcrVUDmHMmphMKgzWgJuG6TwtgMhN7V67fbNBx/cIxkddZdjOTs43q8uXOJGarj83VMfa+Z0XpCQRCYebUvgLPfKv9xMI74v5eQggh+k5GYGJk0GkwEYi8bvQECIUHfyWQoig0tQR6bhhnVU4vbn8Io07LnAK1kJw10DoCY2yvAZNiNcSUUHtqrktxegJnFnWzJUFbAOOugXB8CwpOyLSp9WPW/U/7waRCmPXVuL6PEEKI/pERmBiFwjBeUx55HVY01Lt9oEBmjBsE9ke9209wkAMnRVHYfEwNVorSEyL7DCVEppBSaQtDijO6r27bnXE9XZuQARotKGFw10FiVp/f63RarQaaq6BZTUxm5WMw5XIwDN7frRBCiJ7JCEyM9O5K3jT9IPJaGw7w1q5KVrcWdRssQ1FI7/0DNZQ2qHsejctsn7pqm0JqMbZPIY1N6/vUlranAnRanRrEALji/HNvKoXHW5OE0yfCgm+APSe+7yGEEKLfJICJkdl5PPpAwENjS4BGdwBvYPD2R6oa5B2oj9W62FvhBNSdnItOCVCs/rYcGHXax6zXRZf/Hwi21lGXeObBuGrgt4vaX0+5LH73FkIIEVcSwMQoSHRdkmaX+qUeVhQ2Heu6omy8VQ3gCIwvGOLvn5bR4FZXObX4gry3vwaAeYUpXDk7D72u/Z9OJAemNYl3dmFyz6Mo/dWWBxOvEZhwGP68AvxqgjLWdDjrv+JzbyGEEHEnAUyMwqcljZ7T8K/I80NVzYQHIS8lEApT1zxwS6g/Pd7IW3uquOWZrSiKwsufnsQTCJFsUUv/n0ob8mEOqV/6bUm8VuMgpFbFewSmdBM0HFOf6y3wrZ1g6X63ayGEEENHApgYhUPRAUyOUo0x2PoF7g9RUu8e8D7UNg/cBo4uX5CdZU0A/OfScfz4zf38fdtJAOYXpUaNvABYWqePghoDPl0XhecGQjxHYI5vhHdb85qyZ8Kt74JpED+LEEKImEkAE6tOlu0ags2R56X1LQPehdpm34Dde0+5g2BYYXymjQunZvGlMwqwGnUsmZDO1Bx7h/ZmX2sCryEVNJoed5eOm9Rx6uOR96HkI+hrQNd4HJ65GCp2qK+XPww5M+PSRSGEEANHApgYnT6FBBBscUSeH693owzQ6EibttyUeAsrCvsr1Zye8ydlotFomJSdyDM3z2dOYefTKW0jMG3TR+mJvau+22+TLgKdUa3G++ylcODNvt1n37/bn+tMMPac+PRPCCHEgJIAJkbhcLjDsYCrMfK8qSXAoWrXgBa3q3UNzAjM9hONOL1BjDotc8ckR46futvz6cw+dWfutgTenKQBXn3UxpICE5a3vz6ypm/32f96+/Ob3lSXaAshhBj2JICJkRIOAnBEP4ESrVr+3t/SFNVm9Z4qtpTUEwiF475jtMsXpGKANnH892dq8bYJWTZM+t59kVtai9i5DWmYDFqSLIYB6Vunlj/c/nzPK7Dvtdiu/+xFONm6o/jdB6Bgfvz6JoQQYkBJABMjpTWJV6PV4TGq0ypNjdHLp8OKwtaSRl7+tIz6HqZ7SurcMdWPqXJ4+5zu0Z3yRg///qwCoNNcl65EcmCMqaQn9H7rgLhILYab31af+xzw8o3tK4m6Eg7Dlj/AH86DV76hHsufL8XqhBBihJEAJkZtU0garRaDNVl97nN2mDIKKwo1TnWqp6sARVEU1h+sYdPRevzBjlNTnakfgOmjQCjMP7afjLzOSep92XyLrxYAtzGd3ORBmj46VcECSCpof92WjNuVj5+Et++Fiu2g1as5Lxf9dGD7KIQQIu4kgIlV6xQSGh1JKekALNLupdnb9caKeyscnR4/WN1MY0uAnWVNvLqznGCo5yCmrDH+00cbj6h5LJmJJq5fUIgmhpVEllNyYIozBndnbEDNWbn2+fbX/7gFyj7pvK2iwJbfq89zZsOdW+GmNyB/3oB3UwghRHzFPYD50Y9+hEajifozefLkyHmv18uqVatIS0vDZrNx9dVXU10dXYystLSUSy65BKvVSmZmJvfeey/BYDDeXe2TsNIaZGh1BMcuBeBc7S6aPF0HMIerXZ0GOJ8eb0/+LW/08Mnxhm7fu8rhpawhvsu0a5t9fHZSDbAevHwaabbYpoHaAhifJYPsQdzMMkrOLPj62vbXe/7ZsU3tQXhqgbpJo84It7yjTkEJIYQYkQZkBGbatGlUVlZG/mzYsCFy7tvf/javv/46f//731m/fj0VFRVcddVVkfOhUIhLLrkEv9/Pxx9/zLPPPsszzzzDAw88MBBdjV3bMmqNltD4FQDYNF6OV9R0eUkgrPDS1jJqnF4cLQEURcHREuhQz+Vwtavbt95T3vlITl8pisL6Q+oU0MRMG4vGpfVwRYcbRAIYe0bewG8f0J38eXDhQ+rz8m0dz7/+Lag7qD4vWCC7SwshxAg3IAGMXq8nOzs78ic9XZ1qcTgc/OlPf+IXv/gF559/PvPmzePpp5/m448/ZvPmzQC8++677Nu3j+eee47Zs2ezcuVKHn74YZ566in8/oErn99bkWXUGh2mhET8GvWL0FlXgS/YdTJuszfIG7sqeXbTcY7WuthS0nHfpMYWP+WnrTDadqKR/ZVO9lc6OVDljN8HAY7UuChv8qDTajh7QnrM12sCLvRhdU+mgoKiuPatT9o2Xzy5Fcq3tx/f9Xd1qwCAaVfB5b8e/L4JIYSIqwEJYA4fPkxubi7FxcVcd911lJaWArBt2zYCgQDLli2LtJ08eTKFhYVs2qR+wWzatIkZM2aQlZUVabNixQqcTid79+7t8j19Ph9OpzPqz4BoHYFRNFosRn2kgFuK0sSO0qZuL3V4AoTCCh8cqOVAVXOH84oC/9x2kmO17SMxbl+Q1XuqWL2nikAofsuPgqEwH7XmvpwxJgW7Ofblz3q3OvXn0yWQmxnj6M1ASC2GiRepz49/pD4qCqz7ifp80Z3wpachdRgEW0IIIfol7gHMggULeOaZZ1i9ejW/+93vKCkp4ZxzzqG5uZmqqiqMRiPJyclR12RlZVFVpe5pU1VVFRW8tJ1vO9eVRx99lKSkpMifgoKCLtv2h3LKFJLVqKfFqH5xZ2iaON7LfZBcvmCXhe5CYQWXb+DzfbadaKTZG8Rm0jNvTN82LdS1qNNPHlN6nwKgAZHfWsulcpf66DipLq3W6mHpfUPXLyGEEHEV922DV65cGXk+c+ZMFixYwJgxY3j55ZexWAZume19993H3XffHXntdDoHJIhRwu1JvDqthhZzJjRDrqae95r9BEJhDLrhvbjLFwixrVRNID5nQnqf+6tvac37SciMV9f6ry2AOfIeBLzqdBJA1jQw2YauX0IIIeJqwL9pk5OTmThxIkeOHCE7Oxu/309TU1NUm+rqarKz1d2Fs7OzO6xKanvd1qYzJpMJu90e9WcgKEr7CAyAN0ENksYbagkpCrtOxjfRdiDsLncQCCmkJRiZkNn3L3Vd6xSSNS03Xl3rv7GLITEXvE3w9nfhHzerx/Olyq4QQowmAx7AuFwujh49Sk5ODvPmzcNgMPD+++9Hzh88eJDS0lIWLVoEwKJFi9i9ezc1Ne2retasWYPdbmfq1KkD3d0eKZEcGLXUvt+ubicw3aKOaOwudwz4Zo794fQE2Nq6fHtuYUpMNV9Op3epWw8YUwZmuq5PtDoYf4H6fPuz7cclgBFCiFEl7gHMPffcw/r16zl+/Dgff/wxX/jCF9DpdHzlK18hKSmJW2+9lbvvvpsPPviAbdu2cfPNN7No0SIWLlwIwPLly5k6dSo33HADn332Ge+88w73338/q1atwmQa5FL1nYmsQlJ/dKEUtZbImHApWo2aqOvopibMUPvwcC3+UJicJDOTcxL7dS+duzUnyT6MRmAAipd2PDbu/EHvhhBCiIET9wDm5MmTfOUrX2HSpElcc801pKWlsXnzZjIyMgD45S9/yaWXXsrVV1/NkiVLyM7O5l//+lfkep1OxxtvvIFOp2PRokVcf/313HjjjTz00EPx7mrfKG17Iak/uoQxcwFI9lUyxa4u8x6IarnxUOP0crRWTTS+YHIm2n6MvgDo3eoIzLALYCZdHJ2we8dGsA2jPB0hhBD9Fvck3hdffLHb82azmaeeeoqnnnqqyzZjxozhrbfeinfX4uL0KaSi/DwazQWkeMtYkFDFXkchJxtamJGXNJTd7MDpCfDWHnXEZFJ2YswVdztjcJapT+z5/b5XXBmtULNffX7xzyF7+tD2RwghRNzFPYAZ9dpGYFqnkCxGHTWWfFK8ZUw2NwKFHK9vwR8MY9QP/WqksKKwp8LB/7x9gEBIQa/VsKAotd/3NQea0LdNIWVM6vf94u5Lz6gVedPGD3VPhBBCDAAJYGJ05tgUqIXMJGvkmMeaB41QqK0j2WKgyRNgf6WTWQXJQ9ZPhyfA659VUO9ur15sM+m5eEY2KVZjv++f4T6sPkkZC+aBWfHVLxoN5J8x1L0QQggxQIZ+iGCEyU9Sv/wTLe1TMH6bOoVi91UyI1+dOtp4tK7bHaoH0tbjDTzz8fGo4AXg+oWF5CTFpxZPhvuQ+iRLpmeEEEIMPhmBiVU4ug4MAMmFgBrAzByXxMGqZmqafby1u4r5cZiuiUV5o4ePj6r7LJn0Wi6dmcOVs/Nw+YKdbl/QV5ERmOwZcbunEEII0VsyAhOrthovWl3kUEreOADs3gr0Oi1LJqgrrqqcXr75tx0crel+l+l48QfDvLtPzUuZkGnj9nOKyU+xDkguTqanNYCRERghhBBDQAKYWCkdR2CSstVEUZu/Fm04QF6KhS/MySMnyUworNDi73qX6rh1S1F4Z28VTm+QRLOeC6ZkotV2vkzaEmikuH49xmDfAittOECq+5j6QkZghBBCDAGZQoqV0lbIrn0ExpaWS0BrwhD2YfdV0GQZQ2GqldxkM+Mzbbh8IeqafQParU+ON3Cszo1Oq2Hl9GxMel2n7fQhL1/a/Q3SPCVU2qbx9+m/J6Qzx/Reme6DaJUgmJIi02dCCCHEYJIRmFh1kgOj1Wlx2tSKvGktJZHjOo0Gu8XA7pMOgm0VfAfAsVoXm481ALB0Uka3ibpLjv+KNI/axxzXXm7ddgW6cGzB1cS6NeqTCcvU1T5CCCHEIJMAJlZtU0ja6BEOb/IEANJajkUd/9bfdvLyp2U0ugdmRZLLF2TdoVoAZuUnMT236wJ6xfXrmVX1TwCOppwDQEKggezmfb1/QyXMxLr31OfTrupbp4UQQoh+kgAmVkr0XkhtNK3VXjNd+9uPaTQUZyQAUOeK/xRSKKzws9UHafYGsZv1nD0+vcu21uYSlh95GIBPc6/j31N/wbGUxQDkNO/q9Xumeo6T6K8BvQXGL+vfBxBCCCH6SAKYWHW2jBqwFC0A1GmZUxWnD0wAEwyH+clb+1l7sAYNsHJ6DgZd53+dttrtnPvuSixBBzUJE/l4zH8CUJakFnrL7UUAow0HsPmq+OKe/1AP5J8BhthyZ4QQQoh4kSTeWHWyjBogedx8wuiw+Wux+apxmbIAmJxjh50VHK11c/b49H5voNhma0lj5PnM/CSykzoPJoxBF+M2fCfy+oPi7xLSqsX4KuwzAchx7mr/XJ2w1OzgW5uuiD444cK+dl0IIYToNxmBiVUny6gBNMYEWlLUPYFynZ9Fji+ekI7FoMPhCXC8zh11TTAUZmdZE2/trmRLST1KN0HEqXyBELvLHQB8/+LJnDsxo9N2ma4D3Prp5ZibT+A3JvGXOS9SYZ8VOV+bMJEwOqzBJmz+6i7erJnC1TdHHQpnzYAzbu1VX4UQQoiBIAFMrCJTSB2XKZsmXQDArKp/RI5ZDDrmjUkBYEdZUyRICYbCvLWnivWHajlco64i+uvmExyvcxMKq20a3H7+suk4gVD7CqZwWGHN/mo8gRApVgMXTslC08WozvTqVzGH1Oq72xf+hnrruKjzIa2Jequ6eirLdaDjDZqr4dP/396dB0dV5XsA//aS3tLpJemkO0tng0hYAgYCMYA6DhkjMi7gOMpknAwqPphQwuBDQAv9YwZhtB7l8lDUKnDeU0GpAnQoxMcEBDOGBEISCGjYDVsSEJIOS7bu8/7I5EpLEpOQXm74fqq6qvve07d/9wd0/zj3nnPWQN3UPsJpX/R0VORWQDm7ENAau88TERGRD7GA6a2Om3iVNxYwIZkzAQBxrjJEX9cLk5kcDqUCOH3pGirPuAAAhUcv4MRPemQuXW3FZxVn8YfVJfjgXycw5+N9+Oe3dfh873Ekn1yHp/ZMgbF0JU6cb4RSAUxNj4O6i/tewq8ex5Dz/wcAOPyLd3Epcmyn7TouI42vXgXV2VIAgLG5BtHH1gNvjAK2LQEAFDmfQfnwhUgblNCjNBEREfkSC5je6uISEgDAmoBruigAwOMHnoah5QIAwGLQ4PZ/r0y9vaoOa0uqUXG6/RLQr0dGY+6kFDw1MQlWQwgAoLGpDaFaNZ679zYM19ZiQ+tsPHTmv2BqqcN/tPwdazV/xX2DQxFr7Xq+l18d/St07kacDUtDfdwvumx3NOIeAIDt6jGEfXgfhpzfit9U5iN1zwtA2zWp3WlTOhIiDF3O7ktERORPLGB6q4th1B1qx8yXnqfVbpKej4qzoOO3v+7fs/LGWfTSKCWjVo3fjYvHrLuT8crUEfjVMDvGOC34e/j/IEpR7/UZmcrv8Grt01CIzpcosF49iZjGA3Ar1Nic+iqEMqTL06m2ZOKfgxZDoD24+w8vgbWp2qtNQ/IDqAkbAWe4ocvjEBER+RNHIfWWp/sCxjrxaXxxvgWTD7+EtJqNqPO8AAAw6UMw557BONfQhOMXrsCkU2NotMnr/hW1Sgk1gBGxZlgMGuDQ57BdKkOL0oC/DfoAp9wReKx1EyadegvGlvO4t2wODJ6HYb52Gxr0cdJxYl3lAICzYSNxRdP13DAdDjimocY4HI//sBJtNQeha3Phh+i7EDH9PcAUjdNnG6A+fAGDInnfCxERBQcWML3VxUy8Hcz6EDSnPICrx1cgrKUOl08WAGifb0WhUCDGokeMpetLPz9+jgD2fwIAKI9+FMaoJAwFUOn5HYY0FCLOVYa4i0XAjiJMCU3Fx7f/r/TW6MYDAIBzYT1faPG8cQgu53yODwqPwdx0BkmDhuAXpmhpf4xFBxUvHxERUZDgJaTe6mQxx59KdITjoP1BAIDjy2egbbnY+8/ZsgD4bjOAH+9TAQCPUo31ae/hG+cz0jb7le8Qdd0oor4UMB2EQoV6fTyESuu1vbv1lYiIiPyNBUxvdTET7/VusxtxMOaR9maeNsTWfd2rjwi5fAbY8770us6YekObYufT2DL6fbREthcpd514HQCgb7koLdbYlwKmK4kRvP+FiIiCBwuY3upmGHUHg0aNSOcQfG9pX17A0FzXq4+wHtnQ/iRhIoqmFkF01tujUOBc+Fhc+PVquBVqOF2liGsoxaCLOwEAtaFDcU0T3qvP7UqsRY8oE5cNICKi4MECprce+m/g+RPA6D902ywzORyXw9t7QPTNF3p+fOGB5ejG9ucjf4tWfeez7HZwh8XhgP1hAEBW9bvSDbwnwif0/DN/hsWg6bdjERER9QcWML2lCQUM4UBI9/eE2IxaDB7UPvOtvvl8jw6tcjfhN5WzoWs4BmhNwPCpPXpfSdwMtCk0iHOVYdj5LQCAGuPwHr2XiIhIjljA+JDWGgsAMDT17BLSL4+/CqdrX/uLO58DdKYeve+KNgr7HdOk19fUZq81j4iIiAYaDqP2JWsSAMB09fv2YdHdrEStdl/DsLr23pPmsHhos/J79VHfJMyGSrSiVWXAfvs0NKvD+h43ERFRkGMB40u2FAAKaFsboG+91O1NtUmXvoESbjRoY/D9b3dhpKrr2XM706oyYPugRTcZMBERkTzwEpIvhegBayIAIPLK4S6badsacfeJFQCAw7bsbntq/EUX0vUoKyIiokBjAeNrznEAfpzevzNZ1e8irKUOl3RO7HY+7afAupcWZw50CERERF1iAeNrse3LCNiuHu2ySdKlfwEAdiXORZsq8DPeRhg1MGp5dZGIiIIXCxhfM7cvsmhs6XwkUuTlKliaTgMAzphH+y2s7jitnHWXiIiCG/+b7Wv/XhDR+JO5YExNZ3HP8deQfKkQAHAs/K6gGTnkDGcBQ0REwY09ML4WFgMAMLZegK61Xto8+fASqXhpUeqxPfn5QER3A7M+BMm20ECHQURE1C0WML4WaoNH1b6O0NjTfwcAxNcXI6ZxPwBgR9J/4sP0j3FZaw9YiNdLjgyFUhn4UVBERETd4SUkX1Oq0JT+JAx730bG2Q9Rr3fizpNvAgD226eiPOaxAAcIhKgUcHsAtVKBsYn9swAkERGRL7GA8QPDhFnA3rcBANnHlgFoXy36q+TnAhmW5J7UKIRq1AjVqqEIgjloiIiIfg4vIfmDNQG1MZO8Nn2dOAdupTZAAf3IFqbF8BgzEm2hLF6IiEg2WMD4yeWc13HekIImtQm7Ep/FKfPYTttp1Epo1f6bBTctlhPWERGR/PASkp8kOp1Yk/UpGpvaumyjUirwyOg4OMw6aVv0dc/7m16jwoiYnq14TUREFEzYA+MnKqUCQ6M7LxaGRodhTIIV00bHehUvAJBiD8PU9FiofDAyKNURBrWKfwWIiEh+2APjRyl2Iw6caYDbI+DxCIQbNVApFMhKtsFs6Hr16URbKDKTwvHNsR/6NZ5h7H0hIiKZYgHjR1FhOvxqmB1atRJuj0CsRd/jHpDRCVZUnK7HlWZ3v8Ri1ocgKsx3l6eIiIh8idcP/GxQpBFxVgMSIkJ7dfkmRKVEVrKt3+KI53IBREQkYyxgZCQtzow4a/+sVh1tYe8LERHJFwsYmemP+1ZCtSqkRAXHwpFERER9wQJGZgZFGhGiurkRSWMSrNCo+UdPRETyxV8xmdGFqDD4JntPBkey94WIiOSNBYwMDb+Jy0gRRk23Q7aJiIjkgAWMDMVZ9TDr+1aEDI409nM0RERE/scCRoYUCgUyk8P79N4UOy8fERGR/LGAkalUhwnhob3rhYmz6hEZFvgVsImIiG4WCxiZUikVuPu2qF69Z2JK/02ER0REFEgsYGQsPFTT47bDY8yINvfPJHhERESBxgJG5qLNOoTpfn5Jq/gILh1AREQDBwsYmVOrlEh1dD2sWqNWYnAURx4REdHAwgJmAMhItHbZC/PL1ChEGHnjLhERDSwsYAYAXYgKk4baofjJCgO3x1swNPrm104iIiIKNixgBogkWyjuSI6QXisUwNjEvs0VQ0REFOxYwAwgdyRHINbaPtJoXGI4jNqfv7mXiIhIjoK6gFm5ciUSExOh0+mQmZmJkpKSQIcU9O4ZEoWc4Q6MH8w5X4iIaOAK2gLmk08+wfz58/Hyyy9j3759GDVqFHJyclBXVxfo0IJaZJgWw25isUciIiI5CNoCZsWKFZg5cyZmzJiBYcOGYdWqVTAYDFi9enWgQyMiIqIAC8oCpqWlBaWlpcjOzpa2KZVKZGdno6ioqNP3NDc3w+VyeT2IiIhoYArKAubChQtwu92w2+1e2+12O2pqajp9z7Jly2A2m6WH0+n0R6hEREQUAEFZwPTF4sWL0dDQID1OnToV6JCIiIjIR4JynK3NZoNKpUJtba3X9traWjgcjk7fo9VqodVyxlkiIqJbQVD2wGg0GowZMwYFBQXSNo/Hg4KCAmRlZQUwMiIiIgoGQdkDAwDz589HXl4eMjIyMG7cOLz++uu4cuUKZsyYEejQiIiIKMCCtoB57LHHcP78ebz00kuoqanB7bffjq1bt95wYy8RERHdehRCCBHoIHzB5XLBbDajoaEBJhMndiMiIpKDnv5+B+U9MERERETdYQFDREREssMChoiIiGSHBQwRERHJDgsYIiIikp2gHUZ9szoGV3FRRyIiIvno+N3+uUHSA7aAaWxsBAAu6khERCRDjY2NMJvNXe4fsPPAeDwenD17FmFhYVAoFP12XJfLBafTiVOnTnF+GR9gfn2HufUd5ta3mF/fCcbcCiHQ2NiImJgYKJVd3+kyYHtglEol4uLifHZ8k8kUNH/YAxHz6zvMre8wt77F/PpOsOW2u56XDryJl4iIiGSHBQwRERHJDguYXtJqtXj55Zeh1WoDHcqAxPz6DnPrO8ytbzG/viPn3A7Ym3iJiIho4GIPDBEREckOCxgiIiKSHRYwREREJDssYIiIiEh2WMD00sqVK5GYmAidTofMzEyUlJQEOqSgt2zZMowdOxZhYWGIiorCww8/jKqqKq82TU1NyM/PR0REBIxGIx555BHU1tZ6tamursaUKVNgMBgQFRWFBQsWoK2tzZ+nEvSWL18OhUKBefPmSduY2747c+YMfv/73yMiIgJ6vR5paWnYu3evtF8IgZdeegnR0dHQ6/XIzs7GkSNHvI5x8eJF5ObmwmQywWKx4KmnnsLly5f9fSpBxe12Y8mSJUhKSoJer8egQYPwl7/8xWvtG+a253bt2oUHHngAMTExUCgU2LRpk9f+/srl/v37ceedd0Kn08HpdOLVV1/19al1T1CPrVu3Tmg0GrF69Wpx8OBBMXPmTGGxWERtbW2gQwtqOTk5Ys2aNaKyslKUl5eL+++/X8THx4vLly9LbWbNmiWcTqcoKCgQe/fuFXfccYcYP368tL+trU2MGDFCZGdni7KyMrFlyxZhs9nE4sWLA3FKQamkpEQkJiaKkSNHirlz50rbmdu+uXjxokhISBB//OMfRXFxsTh+/Lj48ssvxdGjR6U2y5cvF2azWWzatElUVFSIBx98UCQlJYlr165Jbe677z4xatQosXv3bvH111+LwYMHi+nTpwfilILG0qVLRUREhNi8ebM4ceKEWL9+vTAajeKNN96Q2jC3Pbdlyxbx4osvig0bNggAYuPGjV77+yOXDQ0Nwm63i9zcXFFZWSnWrl0r9Hq9ePfdd/11mjdgAdML48aNE/n5+dJrt9stYmJixLJlywIYlfzU1dUJAGLnzp1CCCHq6+tFSEiIWL9+vdTm22+/FQBEUVGREKL9H6hSqRQ1NTVSm3feeUeYTCbR3Nzs3xMIQo2NjSIlJUVs27ZN3H333VIBw9z23cKFC8XEiRO73O/xeITD4RCvvfaatK2+vl5otVqxdu1aIYQQhw4dEgDEnj17pDZffPGFUCgU4syZM74LPshNmTJFPPnkk17bpk2bJnJzc4UQzO3N+GkB01+5fPvtt4XVavX6Tli4cKEYMmSIj8+oa7yE1EMtLS0oLS1Fdna2tE2pVCI7OxtFRUUBjEx+GhoaAADh4eEAgNLSUrS2tnrlNjU1FfHx8VJui4qKkJaWBrvdLrXJycmBy+XCwYMH/Rh9cMrPz8eUKVO8cggwtzfj888/R0ZGBh599FFERUUhPT0d77//vrT/xIkTqKmp8cqt2WxGZmamV24tFgsyMjKkNtnZ2VAqlSguLvbfyQSZ8ePHo6CgAIcPHwYAVFRUoLCwEJMnTwbA3Pan/splUVER7rrrLmg0GqlNTk4OqqqqcOnSJT+djbcBu5hjf7tw4QLcbrfXlzwA2O12fPfddwGKSn48Hg/mzZuHCRMmYMSIEQCAmpoaaDQaWCwWr7Z2ux01NTVSm85y37HvVrZu3Trs27cPe/bsuWEfc9t3x48fxzvvvIP58+fjhRdewJ49e/Dss89Co9EgLy9Pyk1nubs+t1FRUV771Wo1wsPDb+ncLlq0CC6XC6mpqVCpVHC73Vi6dClyc3MBgLntR/2Vy5qaGiQlJd1wjI59VqvVJ/F3hwUM+VV+fj4qKytRWFgY6FAGhFOnTmHu3LnYtm0bdDpdoMMZUDweDzIyMvDKK68AANLT01FZWYlVq1YhLy8vwNHJ26effoqPPvoIH3/8MYYPH47y8nLMmzcPMTExzC31GC8h9ZDNZoNKpbph9EZtbS0cDkeAopKXOXPmYPPmzdixYwfi4uKk7Q6HAy0tLaivr/dqf31uHQ5Hp7nv2HerKi0tRV1dHUaPHg21Wg21Wo2dO3fizTffhFqtht1uZ277KDo6GsOGDfPaNnToUFRXVwP4MTfdfSc4HA7U1dV57W9ra8PFixdv6dwuWLAAixYtwuOPP460tDQ88cQT+POf/4xly5YBYG77U3/lMhi/J1jA9JBGo8GYMWNQUFAgbfN4PCgoKEBWVlYAIwt+QgjMmTMHGzduxPbt22/ohhwzZgxCQkK8cltVVYXq6mopt1lZWThw4IDXP7Jt27bBZDLd8CNzK5k0aRIOHDiA8vJy6ZGRkYHc3FzpOXPbNxMmTLhhuP/hw4eRkJAAAEhKSoLD4fDKrcvlQnFxsVdu6+vrUVpaKrXZvn07PB4PMjMz/XAWwenq1atQKr1/flQqFTweDwDmtj/1Vy6zsrKwa9cutLa2Sm22bduGIUOGBOTyEQAOo+6NdevWCa1WKz744ANx6NAh8cwzzwiLxeI1eoNuNHv2bGE2m8VXX30lzp07Jz2uXr0qtZk1a5aIj48X27dvF3v37hVZWVkiKytL2t8x1Pfee+8V5eXlYuvWrSIyMvKWH+rbmetHIQnB3PZVSUmJUKvVYunSpeLIkSPio48+EgaDQXz44YdSm+XLlwuLxSI+++wzsX//fvHQQw91Ojw1PT1dFBcXi8LCQpGSknJLDvW9Xl5enoiNjZWGUW/YsEHYbDbx/PPPS22Y255rbGwUZWVloqysTAAQK1asEGVlZeL7778XQvRPLuvr64XdbhdPPPGEqKysFOvWrRMGg4HDqOXkrbfeEvHx8UKj0Yhx48aJ3bt3BzqkoAeg08eaNWukNteuXRN/+tOfhNVqFQaDQUydOlWcO3fO6zgnT54UkydPFnq9XthsNvHcc8+J1tZWP59N8PtpAcPc9t0//vEPMWLECKHVakVqaqp47733vPZ7PB6xZMkSYbfbhVarFZMmTRJVVVVebX744Qcxffp0YTQahclkEjNmzBCNjY3+PI2g43K5xNy5c0V8fLzQ6XQiOTlZvPjii15DdJnbntuxY0en37F5eXlCiP7LZUVFhZg4caLQarUiNjZWLF++3F+n2CmFENdNfUhEREQkA7wHhoiIiGSHBQwRERHJDgsYIiIikh0WMERERCQ7LGCIiIhIdljAEBERkeywgCEiIiLZYQFDREREssMChoiIiGSHBQwRERHJDgsYIiIikh0WMERERCQ7/w+TluAa1PQ+SAAAAABJRU5ErkJggg==\n" + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "draw_result(\n", + " task_name=\"ParallelContinuousCartPoleSwingUp-v0\",\n", + " log_file=\"rollout.csv\",\n", + " log_key=\"ep_rew_mean\",\n", + " params=dict(freq_rate=1,\n", + " real_time_scale=0.02,\n", + " integrator=\"euler\",\n", + " parallel_num=3),\n", + " group_key=('transition', 'oracle'),\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [ + { + "data": { + "text/plain": "
", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjAAAAGdCAYAAAAMm0nCAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8qNh9FAAAACXBIWXMAAA9hAAAPYQGoP6dpAACp+klEQVR4nOzdd3hb13n48e/FJAESAPemRG1qT9vyijxieduJncRZtjOb1E6bnTo7TlI3idPEadO4+bWx49ROnOW9lyxbwxrWHpREcS+QBAEQe9z7+wMEREocWCRA8nyeh49I4OLeA4okXpzznveVFEVREARBEARBmEZUmR6AIAiCIAhCokQAIwiCIAjCtCMCGEEQBEEQph0RwAiCIAiCMO2IAEYQBEEQhGlHBDCCIAiCIEw7IoARBEEQBGHaEQGMIAiCIAjTjibTA5gssizT2dlJfn4+kiRlejiCIAiCIMRBURQGBweprKxEpRp7nmXGBjCdnZ3U1NRkehiCIAiCICShra2N6urqMe+fsQFMfn4+EPkGmEymDI9GEARBEIR4OJ1OampqYq/jY5mxAUx02chkMokARhAEQRCmmYnSP0QSryAIgiAI044IYARBEARBmHZEACMIgiAIwrQzY3Ng4qEoCqFQiHA4nOmhzFhqtRqNRiO2sguCIAhpNWsDmEAgQFdXFx6PJ9NDmfEMBgMVFRXodLpMD0UQBEGYIWZlACPLMk1NTajVaiorK9HpdGKGYBIoikIgEKC3t5empiYWLlw4blEiQRAEQYjXrAxgAoEAsixTU1ODwWDI9HBmtNzcXLRaLS0tLQQCAXJycjI9JEEQBGEGmNVvh8VswNQQ32dBEAQh3cQriyAIgiAI044IYARBEARBmHZEACOMsGXLFiRJwm63Z3oogiAIgjAmEcBMM5s2beKLX/xi1p1LEARBEKaSCGBmmGhxPkEQBEGYyUQAQ+RF3xMIZeRDUZS4x3nnnXfy5ptv8sADDyBJEpIk8fDDDyNJEi+88ALr1q1Dr9fz9ttvc+edd3LzzTePePwXv/hFNm3aNOa5mpubY8fu3buX9evXYzAYuPDCC2loaEjDd1oQBEHIRq39Hg61O+hyeAmE5EwPJy6zsg7M2bzBMEu/+1JGrn303s0YdPH9NzzwwAOcOHGC5cuXc++99wJw5MgRAP7lX/6F+++/n3nz5lFQUJDUuUpKSmJBzLe+9S1+/vOfU1JSwuc+9zk++clPsm3btiSeoSAIgpDNDrbbeeN4L/LQG2pJAlOOlqI8HSV5eory9BTn6Sgw6FCpsqfoqwhgphGz2YxOp8NgMFBeXg7A8ePHAbj33nt573vfm9K5hvvxj3/Me97zHiASHF133XX4fD5RiE4QBGGGUBSFbaf62d1sO+t2cHiDOLxBTve6Y7drVBIFRh3FeTqK8/QsqTCRp89cGCECGCBXq+bovZszdu10WL9+fVrOE7Vy5crY5xUVFQBYrVZqa2vTeh1BEARh6oVlhVeOdnOsazDux4Rkhd5BP72DfmCQAqOOvJK8yRvkBEQAA0iSFPcyTrYyGo0jvlapVOfk1wSDwbjPp9VqY59H+0TJ8vRYFxUEQRDG5guGeeZAJ+0D3kwPJSUiiXea0el0hMPhCY8rKSmhq6trxG379+9P6lyCIAjCzOD0BfnLnrZpH7yACGCmnblz5/LOO+/Q3NxMX1/fmLMil19+OXv27OGRRx7h5MmTfO973+Pw4cNJnUsQBEHInEBITmjH6lisgz4e39VGnyuQhlFlnghgppmvfvWrqNVqli5dSklJCa2traMet3nzZr7zne/w9a9/nQ0bNjA4OMjtt9+e1LkEQRCEzOi0e/n99mZ+t62Z7af6sHuSCz6a+9z8ZU87Lv/MqRMmKekI67KQ0+nEbDbjcDgwmUwj7vP5fDQ1NVFXVyd21UwB8f0WBEFI3IE2O2+e6CUsj3yZrrTksLTCzMKyPHLi2AhyuMPBa8essW3S6XLj6krmT0IS73iv38NN78xVQRAEQZhhQmGZ149bOdLpHPX+TruPTruPLQ1W5pfmUV9hYk6hYdQaLdsb+3jntG2Us0x/IoARBEEQhCzh9AV59kAXPU7fhMeGZIWG7kEaugcx6tUsKTdRX2GiJF9PWFZ49VgPR8cIgmYCEcAIgiAIQhZo7ffw/OEuvIHEd4e6/WH2tgywt2WAUpMerUpFh3367zQajwhgBEEQBCHD9jTb2HaqPy15KlanPw0jyn4igBEEQRCEDAmEZF452sOJnvgr4goRIoARBEEQhAwYcAd49mDnjKnLMtVEACMIgiAIU+x0r4sXj3TjD4oCoslKqJDdfffdx4YNG8jPz6e0tJSbb76ZhoaGEcds2rQJSZJGfHzuc58bcUxrayvXXXcdBoOB0tJSvva1rxEKjSyus2XLFtauXYter2fBggU8/PDDyT1DQRAEQZgkwbCM2x9i0Bfp3mz3BLC5A/S5Ik0PrU4f3Q4fnXYv7QMe2mwetp/q4+kDnSJ4SVFCMzBvvvkmd911Fxs2bCAUCvHNb36Tq666iqNHj45oJviZz3yGe++9N/a1wWCIfR4Oh7nuuusoLy9n+/btdHV1cfvtt6PVavnXf/1XAJqamrjuuuv43Oc+x6OPPsprr73Gpz/9aSoqKti8OTNdo7PZpk2bWL16Nb/85S8zPRRBEIQZzxcM09jr4pTVRWu/h5A8I+vBZr2EApgXX3xxxNcPP/wwpaWl7N27l0svvTR2u8FgoLy8fNRzvPzyyxw9epRXX32VsrIyVq9ezQ9/+EO+8Y1v8P3vfx+dTseDDz5IXV0dP//5zwGor6/n7bff5he/+IUIYARBEIQp5wmEaLS6OWkdpM3mTXtVWyFxKfVCcjgcABQWFo64/dFHH6W4uJjly5dzzz334PF4Yvft2LGDFStWUFZWFrtt8+bNOJ1Ojhw5EjvmyiuvHHHOzZs3s2PHjjHH4vf7cTqdIz4EQRAEIVkuf4h9rQP8ZU8b/29rE68e66Gl3yOClyyRdAAjyzJf/OIXueiii1i+fHns9o985CP83//9H2+88Qb33HMPf/jDH/jYxz4Wu7+7u3tE8ALEvu7u7h73GKfTidc7emGe++67D7PZHPuoqalJ9qllNbfbze23305eXh4VFRWxWaoov9/PV7/6VaqqqjAajZx//vls2bIldv/DDz+MxWLhpZdeor6+nry8PK6++mq6urpix2zZsoXzzjsPo9GIxWLhoosuoqWlJXb/U089xdq1a8nJyWHevHn84Ac/OCeHSRAEYTpyeIPsbbHx+O5W/uet02xp6KV9QMy4ZKOkdyHdddddHD58mLfffnvE7Z/97Gdjn69YsYKKigquuOIKGhsbmT9/fvIjncA999zDl7/85djXTqcz/iBGUSDomfi4yaA1gHRu/4qxfO1rX+PNN9/kqaeeorS0lG9+85u8++67rF69GoC7776bo0eP8qc//YnKykqeeOIJrr76ag4dOsTChQsB8Hg83H///fzhD39ApVLxsY99jK9+9as8+uijhEIhbr75Zj7zmc/wxz/+kUAgwK5du5CGxvjWW29x++2386tf/YpLLrmExsbG2P/59773vfR+bwRBEKZQICTzyPZmkdMyTSQVwNx99908++yzbN26lerq6nGPPf/88wE4deoU8+fPp7y8nF27do04pqenByCWN1NeXh67bfgxJpOJ3NzcUa+j1+vR6/XJPJ1I8PKvlck9NlXf7ASdceLjAJfLxf/+7//yf//3f1xxxRUA/P73v4/9H7S2tvLQQw/R2tpKZWXk+Xz1q1/lxRdf5KGHHoolSQeDQR588MFYQHn33XfHkq6dTicOh4Prr78+dn99fX1sDD/4wQ/4l3/5F+644w4A5s2bxw9/+EO+/vWviwBGEIRprdXmFsHLNJJQAKMoCl/4whd44okn2LJlC3V1dRM+Zv/+/QBUVFQAsHHjRn784x9jtVopLS0F4JVXXsFkMrF06dLYMc8///yI87zyyits3LgxkeHOOI2NjQQCgVhQCJH8o8WLFwNw6NAhwuEwixYtGvE4v99PUVFR7GuDwTBiNqyiogKr1Ro735133snmzZt573vfy5VXXskHP/jB2P/fgQMH2LZtGz/+8Y9jjw+Hw/h8Pjwez4gdZ4IgCNPJKas700MQEpBQAHPXXXfx2GOP8dRTT5Gfnx/LWTGbzeTm5tLY2Mhjjz3GtddeS1FREQcPHuRLX/oSl156KStXrgTgqquuYunSpXz84x/npz/9Kd3d3Xz729/mrrvuis2gfO5zn+M///M/+frXv84nP/lJXn/9df785z/z3HPPpfnpD9EaIjMhmaBN3wu+y+VCrVazd+9e1Gr1iPvy8vLOXFKrHXGfJEkow9Z3H3roIf7pn/6JF198kccff5xvf/vbvPLKK1xwwQW4XC5+8IMf8P73v/+c6+fk5KTtuQiCIEwlWVZo6hMBzHSSUADzm9/8BojUHRnuoYce4s4770Sn0/Hqq6/yy1/+ErfbTU1NDbfccgvf/va3Y8eq1WqeffZZPv/5z7Nx40aMRiN33HHHiLoxdXV1PPfcc3zpS1/igQceoLq6mv/5n/+ZvC3UkhT3Mk4mzZ8/H61WyzvvvENtbS0AAwMDnDhxgve85z2sWbOGcDiM1WrlkksuSelaa9asYc2aNdxzzz1s3LiRxx57jAsuuIC1a9fS0NDAggUL0vGUBEEQskKH3YsvmHgXaCFzEl5CGk9NTQ1vvvnmhOeZM2fOOUtEZ9u0aRP79u1LZHgzXl5eHp/61Kf42te+RlFREaWlpXzrW99CpYpsJlu0aBEf/ehHuf322/n5z3/OmjVr6O3t5bXXXmPlypVcd911E16jqamJ3/72t9x4441UVlbS0NDAyZMnuf322wH47ne/y/XXX09tbS233norKpWKAwcOcPjwYX70ox9N6vMXBEGYLI29rkwPQUiQ6IU0zfzsZz/D5XJxww03kJ+fz1e+8pVYPR6IzIb96Ec/4itf+QodHR0UFxdzwQUXcP3118d1foPBwPHjx/n9739Pf38/FRUV3HXXXfzDP/wDEKnH8+yzz3Lvvffyk5/8BK1Wy5IlS/j0pz89Kc9XEARhKjT2zvLlI0UGKaXScFNOUiaaVpmmnE4nZrMZh8OByWQacZ/P56OpqYm6ujqRtzEFxPdbEIRsZh308ejO1kwPY0powl4Kvc0UeZoo9DZR6GmiyNOEyd/BkdIbeW3BN+M+142rK5lfkjfxgQka7/V7ODEDIwiCIMxqp7N49kUXcnFp0y/Rhd0ENEYCaiMBdR5+TeTfgNqIX20koBn2udqICpnC4UHK0L9mf9eY11rR8yS7qj/BYE7FFD7D5IkARhAEQZjVsjn/5YK2/8cK61NpPadHW0B/bh02Qx223Dpshrmc3/a/VDv3saLnSbbP+XxarzdZRAAjCIIgzFpOXxCr05/pYYzK7G1nddefAdhVfSdBVQ76sBtdyI0u7EIXdg99PfJzFTIAg7pSbIa6EcFKv6EOn9ZyzrX0oUGqnftY3vMkO2s+jazSnnNMthEBjCAIgjBrZfPy0cUtv0athGi2XMC2OXfF9yBFQSP7kFAIquOvM9ZYuAm3tghjsJ8Fti2cKH5vkqOeOtMr5VgQBEEQ0qjRmp3LRxXOgyzqfxUZFVvn/nP8D5QkQurchIIXAFml4VDZTQCs7PpbQo/NlFkdwMzQDVhZR3yfBUHIRr5gmPYBb6aHcS5F4dLmXwJwpPQG+o1TUzj0cPn7kFFR49xLgad5Sq6ZilkZwERL6Xs8GepAPctEv89ntzAQBEHIpKY+N3IWvsFa2P86lYOHCKpy2FH7D1N23UF9OU0FFwGwsjv7Z2FmZQ6MWq3GYrHEGhgaDAYkScrwqGYeRVHweDxYrVYsFss5/ZkEQRAyKRvzX1RykItb/gOAPVUfx60vmdLrH6y4hfkDb7HU+hzb5txFSJ29tbtmZQADUF5eDhALYoTJY7FYYt9vQRCEbBCWFZr7sy+AWdX1Fyy+DlzaYvZUfXzKr99s2YhDX4nZ38mivpc5WnbjlI8hXrM2gJEkiYqKCkpLSwkGg5kezoyl1WrFzIsgCFmn1eYhEJIzPYwR9EEHF7T/LwDb53yOkDp36gchqThU/j4ubvk1q7r/JgKYbKZWq8ULrCAIwixzOguL153f/jtyQk76DPM5Whpf/7rJcLj0Rja2/jflrqOUuo5hzavP2FjGMyuTeAVBEITZS1GUrMt/MfvOFK3bOvefUaTMvbH26go5WXQFkN1bqkUAIwiCIMwq3U4fLn8o08MY4aLmM0XrWgo2Zno4HCy/BYAlfS+hDw1meDSjEwGMIAiCMKs0WrNr9qXCeZDF/a+iIPHW3H/K9HAA6DCtps8wD63so976fKaHMyoRwAiCIAizyum+LMp/URQubX4AiBSt6zMuzPCAhkhSbBZmZfffIAvr5YgARhAEQZg1BtwB+l2BTA8jZkH/G1QOHiSoymH7FBati8exkmsJqnIo8jZR5Xw308M5hwhgBEEQhFmjMYt2H6nkIJcMFa3bW/Ux3PrSDI9opIAmj+MlVwOwsvvvGR7NuUQAIwiCIMwa2bT7aFX3X7H42nFrCzNStC4e0WWkhf2vYwj0Z3g0I4kARhAEQch6Dk+Q/W32lM7hCYTodGRH80Z9yMn5bUNF62o/l3D36KlizVtCV94y1EqIZdanMz2cEUQAIwiCIGS9t0/1saXByilr8lt6T/e6syYX9by235EbctBnmMeRshsyPZxxRWdhVnQ/gaSEMzyaM0QAIwiCIGS1LoeXEz2DKAq8cKibTntysyjZkv8yvGjdW3P/CUXK7qL4DcXvxafOx+zvYs7AzkwPJ0YEMIIgCEJK/KEwVqdv0s7/1sm+2OchWeHpA50MuBPbSRQIybT2e9I9tKRc1PJfaJQgLZbzabZcmOnhTCiszom1NljV/dcMj+YMEcAIgiAIKWm0unnlWA+ynP71mVNWFx0DI2dcvIEwT+zrwBOIv5puq81NaBLGl6jywUMs7nsFBYmtc/8JJCnTQ4rLwfL3A1A3sI18X1eGRxMhAhhBEAQhJSetg1idft5tHUjreWVZYdupvlHvc3iDPLW/k2A4vo7Sp7Kk+u7G1t8CcKT0evqMizI8mvgNGObSal6PhMKKnicyPRxABDCCIAhCCnzBMC1DSzM7T/dj96SvSNyhDge2cZaKuh0+nj/UNeHMjywrNPdnRwBT4j4BwIGKD2R4JIk7WH4rAMt7nkIlBzM8GhHACIIgCCk4ZXURHgoggmGF145Z03JefyjMztMT1x053evmjYbxr9lh9+INZH73jEoOYQzaABjUl2d4NIlrLHwPbm0RxqCNBbYtmR6OCGAEQRCE5J3oGbmtudXm4XCHI+Xz7m0ewBNn0HGw3cGuJtuY92fL7iNDMLIcFpY0eDXmDI8mcbJKw6GymwBY2fW3DI9GBDCCIAhCkjyBEG22c7c0v3WyD7d/nARbWxM8dhu0jr4l1+UPJZxPs72xj2NdzlHvy5bqu3mBXgDcumKQpufL76Hy9yGjosa5F+3AqYyOZXp+BwVBEISMO2V1IY9SGc4XDPPmid6xH7jtl3DiBXjy8xA+N9DZfqqPYDixHUOKAq8c7aHNNnKrdO+gH4c38/kacCaAcelKMjyS5Ln05TQVXgyA6fAjGR2LCGAEQRCEpDR0j10Vt6F7kNOjLd3IMjS8EPncdhoO/mnE3X0uP0fHmEmZSFhWeOZgJ30uf+y2bFk+AjAGIktI7mkcwAAcGKrMm9/wVwhkrraOCGAEQRCEhLn8ITomqIj7+nEr/tBZeSwde8HVc+brN38CoTM7jd4+2ZdSuX9/UObJfR0M+iKzLtkUwMyEGRiAFssFOPSVqP0OOJy5XBgRwAiCIAgJOzlU2n88g74Q20+dtZPo+LORfxdfC3llYG+F/f8HQGu/h6a+1PNVBn0hntofmYmxOv0TP2CKGIfnwExnkop9FR9icOFNULYsY8MQAYwgCIKQsLN3H43lQLt9ZO+ihucj/664FS7+cuTzrfejBH28dWqcvJkE9Q76+eve9rSdLx3y/DNjBgZgX9VHsF71X1C1NmNjEAGMIAiCkBCnL0iXI77eR4oCrx3ridSK6T0BfSdApYUF74V1d0J+JTg76H7jt2mfLcmG2i/DzZQlpGwhAhhBEAQhIfEsHw3X5wqwu9kGDc9Fbqi7FHJMoM2BS78CgGnPA6jDk9cQMhvMlCTebCECGEEQBCEhDd2JJ8buarIRPDqU/7Lk2jN3rLkdv7ESY6CPld1/T9MIs48m7CMnHFl2EzMw6SECGEEQBCFudk+AHmfiMyU5vl40nXsjXyw+E8B4ZTXbqz4JwIaO36MJj7+zabqKJvAGVTkE1MYMj2ZmEAGMIAiCELcTPcltS55n24qEgrtkNZgqY7fvbOrnYNF12HOqMAZtrO76c5pGml3O5L+UgiRleDQzgwhgBEEQhLg1xLn76GzzbFsBOGS8KFajxe4JcKjdgazSsLPmMwCs7/gDulD21G5Jl2j+i2u6b6HOIiKAEQRBEOJicwfoG0x8p5A25KbWvguAhoJLeaMhMhux7VR/rJP18ZLN2HJqyQ05WN31ePoGnSXO9EES+S/pIgIYQRAEIS7jtQ4Yz1z7DjRKkIGcWmy5dTRaXbx1sndELRlF0rCz9rMArOt4FH0ouWtlq7yAFRAzMOkkAhhBEAQhLvEWrzvbfNubADQWXhrL/9jTfG636RPFV9JnmEdOeJC1nY8lP9AsJLZQp58IYARBEIQJ9Q76sbkDEx94FpUcos72NgCNRZvGPVaR1OysiczCrOn8IzlBe8LXy1aiiF36iQBGEARBmFCysy/Vzr3khF24tYV05S+f8PiTRZdhNS5EH3azrvPRpK6ZjcQMTPqJAEYQBEGYULIBTHT30emCS1Ak9cQPkFTsqPkHAFZ3Pk5u8NylpmlHUcQMzCQQAYwgCIIwrh6nD7snmPgDFYUF/VsAaCx6T9wPO114Kd159ehkL+vbf5/4dbOMPuxCK0eK/4kk3vQRAYwgCMIMFpYVPIFQSudIdvdRqfs4+QErAVUureYN8T9QkthROzQL0/3X2PLLdBWtwuvTmAirczI8mplDBDCCIAgzWKvNw593t+HwJjGDAiiKkvzuo/7I7qOWggsSfuFutlxIZ/4KNLKfDe0PJ3X9bHFm+UjMvqSTCGAEQRBmsEariwFPkD/vbqM3iSJ0XQ4fg77kZnDObJ/elPiDh83CrOj+O3n+nqTGkA1EAu/kEAGMIAjCDKUoCo29kbL8Ln+Iv+xto8OeWLPEZFsHmH3tlHhOIaPmdMFFSZ2j1Xwe7aY1aJQg57U/lNQ5soEoYjc5RAAjCIIwQ3U6fHgC4djX/qDME++2x4KaiSiKwqlkmzf2R3YftZvX4NeakzoHksT2oVmY5T1Pke/rSu48GXamD1Jphkcys4gARhAEYYY6ZT03+AiGFZ490MWRTseEj28f8OLyJ7d8tMC2BYDGwvh3H42mw7yOVvMG1EqI89v+N6VzZcqZPkhiBiadRAAjCIIwQzWOEsAAyIrCy0d62NNsG/fxySbv5gTtVDoPRMaQYgADsL32cwAssz6LydeR8vmm2pkZGJEDk04JBTD33XcfGzZsID8/n9LSUm6++WYaGhpGHOPz+bjrrrsoKioiLy+PW265hZ6ekclXra2tXHfddRgMBkpLS/na175GKDQyyt+yZQtr165Fr9ezYMECHn744eSeoSAIwixkHfRNuPPorZN9bD3Ri6Io59wny8qoMzjxmGd7CxUyVuMiBnMqkjrHcF2mlTRbLkBFmHUd0686b54/kgMjknjTK6EA5s033+Suu+5i586dvPLKKwSDQa666ircbnfsmC996Us888wz/OUvf+HNN9+ks7OT97///bH7w+Ew1113HYFAgO3bt/P73/+ehx9+mO9+97uxY5qamrjuuuu47LLL2L9/P1/84hf59Kc/zUsvvZSGpywIgjDzxRt87G0Z4KUjPcjyyCCmbcAzIn8mESntPhrDnqrbAVhmfXp6VedVZIxBMQMzGSRltNA7Tr29vZSWlvLmm29y6aWX4nA4KCkp4bHHHuPWW28F4Pjx49TX17Njxw4uuOACXnjhBa6//no6OzspKysD4MEHH+Qb3/gGvb296HQ6vvGNb/Dcc89x+PDh2LVuu+027HY7L774YlxjczqdmM1mHA4HJpMp2acoCIIwLf1hZwt9CWybnldi5NoVFWhb34YDf2Jn0c3s8M1N+LqasI/P7boSreznD6sfpc+4KOFzjEpR+PDBOyh3HWNn9afYMedz6TnvJMsN2Pjc7s0oSPxq43ZklSbTQ0qbG1dXMr8kL+3njff1O6UcGIcjkgRWWFgIwN69ewkGg1x55ZWxY5YsWUJtbS07duwAYMeOHaxYsSIWvABs3rwZp9PJkSNHYscMP0f0mOg5RuP3+3E6nSM+BEEQZiO7J5BQ8ALQ2dlB+0N3wu9vgP2PsnbLHVQ53k342rX2d9DKfhz6CvoMCxN+/Jgkid1VdwKwuvsvaMOe9J17EkUTeD3aghkVvGSDpAMYWZb54he/yEUXXcTy5ZEOo93d3eh0OiwWy4hjy8rK6O7ujh0zPHiJ3h+9b7xjnE4nXu/oNQzuu+8+zGZz7KOmpibZpyYIgjCtxbtNGgBFYWnP09z57q3UtT+FgoTfNBdd2MP7jv4zNfZdCV17xO4jSUrosRNpLHoPAzm15IScLO9+Mq3nniwigXfyJB3A3HXXXRw+fJg//elP6RxP0u655x4cDkfso62tLdNDEgRByIh4818KPM3cevhzbD71Q3JDDnoNC/nTyt/x30v/QJNlI1rZx83Hvsycge1xnU9SQsyzvQVAY9GmZIc/JkVSs6fqYwCs63wUlZxce4SpFC1iJxJ40y+pAObuu+/m2Wef5Y033qC6ujp2e3l5OYFAALvdPuL4np4eysvLY8ecvSsp+vVEx5hMJnJzc0cdk16vx2QyjfgQBEHIRoqi0OXwsqOxH18wuUTZsbj9IbocvnGPUct+NrY8yMf3f5ga57sEVTlsnftPPLbqEbrzlxNW5/BM/f00FlyCRvZz47GvMs+2dcJrVzoPkhty4NWY6TCtStdTGuFY6bW4tUXkB6ws6cv+jR15YgZm0iQUwCiKwt13380TTzzB66+/Tl1d3Yj7161bh1ar5bXXXovd1tDQQGtrKxs3bgRg48aNHDp0CKvVGjvmlVdewWQysXTp0tgxw88RPSZ6DkEQhOnGFwxzvNvJi4e7+O3W0/xpVxs7T/dzsH3ignKJaOx1Md7WjBr7Lj6+7yNc0P6/qJUQpwsu4pE1j7O36uMjcjTCKh3PLvkJJ4suR6MEuf7411nQ9/q4147uPmoquBhFmpx8j7BKz7uVHwFgffsjoMiTcp10McaK2IkAJt0S+gm76667eOyxx3jqqafIz8+P5ayYzWZyc3Mxm8186lOf4stf/jKFhYWYTCa+8IUvsHHjRi644AIArrrqKpYuXcrHP/5xfvrTn9Ld3c23v/1t7rrrLvR6PQCf+9zn+M///E++/vWv88lPfpLXX3+dP//5zzz33HNpfvqCIAiTx+r00dTnprnfTbfDjzxKZLGvdYC1tRY06vTUFR0r/yU3OMClTb9kae/zALi0xWyZ91VOFl0+Zq6KrNLy3OIfc/WJ77Ok7yWua/gmLyrfp6Hk6nMPVpRYAHOqKPXideM5WP5+zmv/HUXeJubZ3uL0JF8vFaIT9eRJKID5zW9+A8CmTZtG3P7QQw9x5513AvCLX/wClUrFLbfcgt/vZ/PmzfzXf/1X7Fi1Ws2zzz7L5z//eTZu3IjRaOSOO+7g3nvvjR1TV1fHc889x5e+9CUeeOABqqur+Z//+R82b96c5NMUBEGYfP5QmNZ+D019blr6PXGV4fcEwhzpdLKqxpLy9X3BMG22szY6KDLLrM9wSfN/kBtyoCBxoPxWts35RwKaibfAKpKGFxf9gLBKyzLrs1xz4ruo5SBHy24YcVyRpxGLr4OQSk+L5YKUn8t4Apo8DpbfwoaOR9jQ8UhWBzBiBmbyJBTAxFMyJicnh1//+tf8+te/HvOYOXPm8Pzzz497nk2bNrFv375EhicIgpAx1kEff3+3A28Sxd/2tgywosqMSpXarp3mfjfhYQXptGEPNxz7GnMckZ1EVuNCXp3/LXrylyV0XkVS8/KC7xCWtKzseYLNp+5FrQQ5VH6mSGl091GL+TxC6tFzFdNpX+WHWdP5RyoHD1Lp3E+nafWkXzMZZ2ZgRACTbqIXkiAIQoq6HT7+tje54AXA4Q1ywppc36Hhhu8+Uod93HT0y8xx7CKoyuHNuf/MY6seSTh4iZFUvDb/HvZVfAiAKxvvY3Xn47G75w91n26cotkQt66Yo6XXA7Ch/fdTcs1EqeQQhqGqwSKAST8RwAiCIKSgw+7lb++2p7ybaE9zauXxQ2GZlv5IcTe1HOCG41+nxrkXv9rIX5Y/yLtVH0s9sVaS2FL3FfZURrYyX9Z0P+s6/kCev5sy9zEUJE4XXJLaNRKwt+pjKEjMG3ibIvepKbtuvAzBfiQUwpIar9aS6eHMOCKAEQRBSFKbzcOT+zoIhFLfCdM76Ke5zz3xgWNosXkIhGRUcohrG75FnX0HQZWep+p/kfysy2gkibfm/hM7qz8JwKXNv+K6hm8C0GlahVdXmL5rTcCeWxtJQgbWd/xhyq4br+jykVtbDJJ4uU038R0VBEFIQnOfm6f2pyd4idrdbEv6saesLiQlzOaT32eBbQshScfT9T+nw7wmbeOLkSR2zPk822oj/YgqBw8B0Fh4afqvNYFok8clvS+R7+ua8uuPJ5rA69KXZngkM5MIYARBEBLU2Ovi6QOdBMNJ98IdVfuAly7H6O1SxiPLCk29g1x56scs6XuJsKTm2SX/Rqvl/LSO72y7aj7F1jn/FPv6VBq7T8erJ38preb1qAizrvPRKb/+eGIzMGIL9aQQAYwgCEICTvQM8tzBrhG7fdJpdxK5MB0DHi44/hOWW59BRsULi35MU+HU5KLsrf44Ty/5Gc8t/lccuZnpQbe7+k4Alvc8RU7QnpExjEb0QZpcIoARBEGI07EuJy8c6p604AXgdK8LmzsQ/wMUBeWV77C6+y8oSLy08HucLL5i0sY3msaiTZwofu+UXnO4VvN59BgXo5V9rO76S8bGcTYxAzO5RAAjCIIQh8MdDl460j1qNd10UpQEc2G2/Bu1x/8XgNfm/wvHS6+dpJFlMUmK5cKs7nocTTjxZbjJcKYGjMiBmQwigBEEQZjA/jY7rx7rGbfHUDo1dA8y6Iuj0/Lbv4A3/w2ALXVfGlFYbrY5WXw59pwqckMOlvc8lenhAMOr8IoZmMkgAhhBEIRx7G2x8cZx65QFLwBhWeHdVvv4B73z3/Dq9wF4e84/sm+oweFspUga9lZ+HIB1nY+ikidu4zDZRCfqySUCGEEQhDG8c7qfrSf6MnLtwx2OsYvjvfsIvPB1AA7O+wy7qz8xhSPLXkdKr8OtLcTk72ZR38sZHYs67CMn5AREH6TJIgIYQRCEs7TZPDyxr53tjf0ZG0MgJLO/zX7uHQf/Ak9Hti5713+e18o/M7UDy2JhdQ77Km4DYEPHI0zptNlZorMvQZUev3ripplC4kQAIwiCQKSWSkP3II+908pf97bT3OfJ9JDY32YnGB4qlBdww9b74Yl/ABRY/0kOLf0aSKk1gJxpDlbcQkBloNjTSN3AtoyNY0QCr/g/mhQpNsYQBEGY3oJhmSOdTt5tGcDhjSNxdgp5A2GOtPay2voUbP0ZuK2RO1Z9BK79OY172jM7wCzk15g4WP5+1nf+H+s7HqGp8OKMjEMk8E4+EcAIgjAreQNh9rUNcLDdkXQX6ckkKWHqe19g/t7/B77OyI2WOXDZN2HFBxkMhOlx+jI7yCz1buWHWdP1J6qd+6hwHqDLtGrKxyCK2E0+EcAIgjCr2D0B3m0d4GinM+2tANJCUVhge4MLWx6kyNsEQNBQivayb8Ca20GjA6Cx15nJFI+s5taXcqzkWpZbn+aaE9/lyaW/wGaYN6VjyAtEZsvEDMzkEQGMIAizgtXpY3fzAKesrkkvRpcURaHWsYuLWn5NuesYAF6Nmd1Vd9C+8CN8eP0SpGG5FKesrkyNdFrYUftZqp17sfg6uO3gJ3l28b/RWnDBlF3/zBZqUcRusogARhCEGS0Yltl2qo/9bfasnbGocB7kopb/osa5F4CAKpd3Kz/C3qqPEdDkgRea+tzMK4nsZvEFw3QMZEe12Wzl0pfxx5UPc+Pxr1Hl3M/7jn6RN+Z9lYMVt07J9WOdqMUS0qQRAYwgCDNWp93Ly0e6GfBkV3JuVJGnkYuaf838gbcACElaDpbfyq7qO/HqCkccu6d5IBbANPZm6SxSlvFpLfxt2a+58tS/srT3Oa44/RMKvc28WfdFFGlyX/6iMzDZsoSkKAqyEimSGFaUyL/DPxQFjUpCr1Gh16jRqqURM37ZSAQwgiDMOKGwzI7T/extGcjaWZd8Xxe3HfgEOtmLjJojZdfzTs2nGdSXj3p8h91Lh91LlSWXxl73FI92+gqrdLy08HvYDHO4uOW/WNP1OBZfG88v+nFkdmsyKErGZmD6XH7ebRmg3e4lFB4ZrCRCgkgwo1Wj16jQaVSx4EY/9HlIlnn/2mrml2Smzo0IYARBmFG6HT5ePtpNvyuBjs4ZsLHtv9HJXnqM9byw6F4GDHMnfMyeZhulKypo7RcBTEIkid3Vn8CeU8vmk9+jbmA7Hzr0aZ6q/3ecOZVpv5wu7EYnR5b4pqIKr6IodNp97Gmx0dwfX/0iSQK1JKFWnfkIhRX8oTCyAgrgC8n4QvKY59jZZGNRWb4IYARBEFIRlhV2nu5nT/NA1i+vFLlPsdT6PACvzf9GXMELRPJg9rYMZOfuqWngZPEVOPXl3HTsKxR7Grnt4Cd4ZsnP6DKtTOt1okXsfOp8QuqctJ57OEVRON3nZk/zAN3DttQvKM1jRZWZXK0ajWpkkBL9UI2xPKQMzdj4Q/LQRxh/cNjnw24vztMzp8g4ac9vIiKAEQQhrbY0WCk357Ck3DRl17Q6fbx0tIe+Qf+UXTMVF7X+BgmFk0WX05O/LO7HKQrsPJ259gYzQU/+Mh5b9TA3HfsKpe4T3Hr487y88Ls0lGxO2zUmu4hdSJZp6B5kb8tALL9LrZKor8hnbW0BBQZd0ueWJAmNWkKjVmHUj3/sjasrMzb7AiKAEQQhjZr73OxrtaOSJBQF6ismN4gJywrvNPWzuyn7Z12iKpwHmG/bioyKbbWfT/jx0+RpZjWXvpw/r/h/XHPiO8y3beXaE9+mwNvCzprPpKXs/2R1ofaHwhzucLKvbQC3P1J8UadRsbLKzOoaC0b97HpJn13PVhCESeMNhHnlaA8AsqLw0pFuFAWWVk5OENM76Oflo91YndNj1gUAReHill8DcKTshriXjoT0C6oNPLPkp1zc/J+s7/w/Nrb9Pwq8Lby84DuEU1z2iRaxc+nTE8C4/SH2t9k52OEgMJSTYtSrWVNTwPIqE3qNOi3XmW5EACMIQlq8eqwHlz8U+1pR4OWj3SgoLKs0p+06iqKwt2WA7Y39Ce+syLS5A9updu4jJOki7/aFjFIkNW/V/TMDuXO4/PS/saTvZfL9Pfx1+YPIquRfHo2xLdSpBzAnegZ55WgPoaGf9QKDlnVzClhcno9GNbv7MYsARhBmkaOdTqoKcjHnatN63sMdjlErwyoKvHK0B0WB5VWpBzEuf4iXDnfTapvkTtGKzOK+l3HpSukwr03bOaOzL/srPohLX5ae8wopO1x+M/acam48/lWqBg9Q7diTUtXevDRsoY4G6tsaIzlP5aYc1s8tYF6xMevrs0wVEcAIwiwx6AvyRoMVo07NBzfUYNCl59ff4Qny5oneMe9XlMjsjKLAiurkg5jTvS5ePtoz6Y0X1WEfm0/dy+K+VwhLav627Dd0mNekfN4lvS9R4jmJT53Hruo7Ux+okFbtlvWcLryE+t4XqRw8NGEAIysKVqef3kE/VQW5FBrPJM6mmsQrywpvnLByuMMJwJoaCxcvLB5z59BsJQIYQYjD6V5XrArqdPXmiV4CIZlASOaJfR3cuq465bVzWVZ48UhXbF1+LIoCrx3vQUFhZbUloWuEwjJvnepjf6s9+YHGKTc4wI3HvkLl4CEA1EqY6xr+hUdX/QG3PvmeNio5yIWtDwKwp+p2/Nr0LakJ6dOVv4L63hepGDx4zn2KomD3Bmm1eWizeWgf8OIf+rlXSXB+XRHr5hSgVknDZmAS/5kJhGSeP9xFy1A9l/csKmF1jSX5JzWDiQBGyCr9Lj9dDl9alhvSQVEU3miwcrTTyT+8Zz5a9fRcc27qc3Oy58wSj9Xp55kDXdy8uhJNCs9pd7ONTrtv4gOJBDGvH7eiKLAqzj/I/S4/zx/unpLt0YWeJm4++kXM/k586nxeXHQvF7X8mhLPKW44/g3+suK/CauS2566oucJzP5O3Noi9lXeluaRC+nSlR+pB1M+eBgUGXdApm3AMxS0eEfkeEFkB5A5V0vvoJ8dp/s51eviqiUlsRwYV4IzMC5fiKcOdNDnCqBRSVy9vDyj25SznQhghKwhywovH+2hx+lDVhJ/pz454+nmWNcgAK02z6T+MWnoHqS6IDftWyGDYZk3jlvPub3N5uGFw91ct6IClSrxqekep4+dp20JPSYaxMiKwpragnGPPdTu4M0T1ikp2lZj3831x79OTtiFPaeKJ+t/yYBhLrbcuXzkwO1UuA6z6fT9vLbgmwmfWxv2cH7b/wKws+bThNS56R7+rLCr2cagL8hli0snbSmlK2cefimHnLCL7Tu3s9szMk9JLUlUWHKoLTRQU2igNF+PBDT0DPJmQy+9g35e2nOUH+sjgY5HG38A0zvo5+kDnbj8IQw6NTesqqTcNHlF8GYCEcAIWWNv6wDdjsi7+deOWQnLE7/ITZZQWOa5Q12cHtZz5nSve1IDmAPtdg6227llbXVSAcVYdjfZcHhHb2Z4yuriteNW3rs0sYTSYFjmxcPdSdde2dLQiwKsHeX/1xcM8+qxnhEzRpNpac/TXNn4r6iVMJ35K3m6/n682si4HLnVPL/4R7zv6BdZ2fMEPXn1HC5/X0LnX9v5GMagDXtONYfLbp6EZzDz9Q762TGUzLqk3ESVZXKCwL8f6OGi0Dw2qo8yz3+U3ZRRmq+nptBATUEulZbcUWdhl5SbqCkw8EaDFV1fCwD9mOlxhynJn/hltqXfzfOHugmEZQoNOm5aXYkpzYn2M5EIYISs0O/ys7NxZIXRLQ29yIrCujmFYzxqcvhDYZ7e30n7gHfE7U19LhSldFJ2AHgCITrtXhQFdpzu56IF6angaXMH2NMyMO4xhzscGHTqhK751slebO7Ueg292dCLctb/b/uAhxcPdzPoC43zyDRRZC5q/Q3ntT8MwPHizby88DuEVSPLj7YUXMi2OZ/n4pb/4rLTP6PPuIDu/BVxXSInaGddx/8BsK328yltzZ3N9rScmemLNrRMt8jvoI93NQvYyFFuKmzDvGQeubr48sSMeg3XrahAe/oodEO3bOFPu1tZP7eQ8+YWoh7jTcnhTkdsabXakst1KyvI0c7Oui6Jmp4L+sKMIsvKiDoHw2090ceupsSWKVLhDYT5+7sd5wQvAG5/eES/kXQ63euOVVjd3WzjdG96Zh9eO9YTV62UXU029k4Q6EQ19bk50OZIdWhA5P93T7MNWVbY3tjH3/Z2TEnwog77uLbhW7HgZWf1p3hh0Q/PCV6idlfdycmiy9AoQW44/g0MQzkOEzmv/SH0YTc9xsWcKL4yXcOfVeyewIjZuM5RfjfTocMeOe9J3VIAFoeOxx28REmSxBJjZKyDuhJkJfK79cfdrfSc9bdDUSI/868diwQvS8rzuXlNlQheEiACGCHj3m0doMsxdmCw7VRfbPp4Mrn8If6yty22jDWa4UtK6TS8hoqiwEtHesZc9onX0U7nqIHYWN462cvRTue4x3gCIV452p3SuM69bh+P7GjmndO2KWkHkBuwceuRf2Rx/6uEJQ0vLfweO+Z8bvwS8pLESwu+R39uHXmBXq5vuAeVPP7/T76/m1VdfwFg25y7QJq5f24PtTvY02xDmYT/v70tAyiAZWhJpdPhRZ6EAoYdQ78rvZZIIm+Rtxl9MPFAPbqF2lhUzbXLy8nVqul3BXh8TxvbG/sIyTIhWealIz3sbo68aTi/rpCrlpaNOUsjjG7m/kYJ00K/yx9XcLLzdD/bTsX3rjcZdk+AP+9uo981/rJIumZGhvOHwrSdVZjNFwzz/KGupCvN+oJh3jo5dm2W0USLzo33HF89Zo31YEmnaEO6yVboaeLDBz9B5eAhfBoTf1/2nxwtvT6uxwY1Rp6u/xl+tZEq534ubf7luMdvbP1vNEqQNtM6WizJF0XLdr2Dfl5vsLKtsZ/GNAf4Ln8olkR/RX0peo2KYFjB6kr/rrToDIypsBxbTi0AFYOHEz7P8D5IC8vy+dgFtSwqzUNRYHfzAH/a1cbf3+2goWcQlQTvXVrGBfOKRHG6JIgFWSFjxls6Gs2uJhthWeHSReltkNbn8vPEux3nbJEc/dgADk8QsyF9CXbNfZ5RvwfdDh9bT/Zy2eLEa0m8fbIPTxIF32RF4flDXdy8porqAsOI+w53OGgcpdruVMsJ2rmk+VfkB6x4NRa8WgtejRmv1oJPaxl2mwWf1hzb+jxyp1E1T9b/IuFeRPbcOby46F5uOvYV1nT9GWte/agBUJGnkXrr8wC8PffutDQIzFa7m88s8b59qo+6YmPaZhL2tQ4QVhQqzDlUFxiotOTS1Oemc8Cb1h06vmCYvqE3L1WWXLqcKyj0tVIxeJDmwosSOtfZRewMOg3XrKhgodXF68et9A/ljunUKq5bWUFtoWHMcwnjEwGMkDETLR2NZm9L5A9aMi/qo+lyeHlyXye+YPwv9o19rlF3zyRrtBL8Uftb7VRZcllUlh/3+bocXg53Jp+jEgwrPH2gkw+sq6EkP5ITYvcExq22O1Xy/D28/8gXKPI2xf0Yv9qIT2MmL9CDWgnTkb+Kp+vvx6e1JDWG04WXsqPms2xs+y1XnLqPPsN8rHn1I465sOVBVMicLNxEd/7ypK4zHdjcAU4O/fzqNSoc3iAH2+1p2T3oC4Y51BH5Od4wN5LoXTUUwHTYvaydk77fwejsS4FBi1GvoSt/Jcusz1E5SkG7iYxVxG5BaR5VBbm8fbIPmzvAFfWlFOeNnnMlxEcEMEJG2NyBpPNa9rfaUYaCmFSmXdtsHp4+0DlhFdmzne51py2ACYVlmvvHn3Z/5WgPJXl6CowTF1GTZSWWFJgKf1DmiX3tfGh9Lfk5Gl483J3w9yndCjzNvP/oFzD5uxnUlbKz5jNowx5yQ3Zyg/Yz/wbt5IQc5AYdqAijD7vRhyPf42PFm3lllJ1GidpZ8ylKXceYP/AWNxz7Go+t/kNs63WF8yALbFuQUbF9zj+m/Lyz2Z6h2Zd5xUbqio28dtzKO0026itMKSejHmx3EAwrFOfpmFsUmaWI7j7qsHtRFCVtyy7R/Jfo+TtjBe2OIikhFCn+l8rx+iDlatUJlywQxiYCGGHKybLCy0e64146Gs2BNgdhGa6sTy6IOWV18cKhrqTG0DHgxRcMp2W3QIvNM2FgEAjJPHuoi9s21ExYCXhfm53eNFWtdfvD/H1fO3OLjQnPlKVbqesY7zvyTxhCdmy5c/j7sv9kUF8+/oMUGX3YFQtqwiotVuOS9CznSCpeXHQvHz5wB4W+Vq5t+CZ/X/YfKKi5uOU/AThaej02Q13q18pSDm+Q4z2R/JTz6gopydezv91OvyvAriZbSku9wbDM/jY7AOvnFMZ+x0vy9WjVEv6QTL87kLYZjOgMTFVBJICxGerwq43ow26K3Y305i2O6zySEsIQjAR1yfZBEuInAhhhyiWzdDSawx0OwrLCVUvLRhR+k2UFdyCEyx/C5QsxOPTv8K8HfcGkZylkRaG5382SclPKzyHenJK+QT9vHLdy1bKxX7QHfUF2nk7vbi27JzglPYjGU23fw03HvoJO9tBjrOeJZQ/EZjvGJanwa0z4NSbsubVpH1dAk8cz9T/jwwc+Qa1jD5c0/wetlvOodu4jJOnYUfuZtF8zm0R2HcGcIgNlQ/kolywo5sn9nRxot7Oy2ozFkFzrhSOdTrzBMOZcLQtLzxSPVKskKsy5tNo8dAx40xLA+EPhWNBfbYnM9CiSmu785cyxv0PF4KG4AxhDwIaEQlhS44nnZ1RIiQhghCmVytLRaI51OXH5Q+g1Klz+EG5/CLc/POnbcU/3ph7AyLLC6b74d20c6XRSackds0/UlobejC/zpNuC/te5puHbaJQgreb1PL3kfoIaY6aHFWMzzOOlRd/jhuPfYF3nYyzpfRGA/RUfwDXRDNE0NugLcrQrsuX+vLlnChHOKTIyp8hAS7+Hbaf6uW5lRcLnDstKrCbR2lrLOVWpqyxDAYzdG3dPrfF02n0ogDlXS17OmZfErvwVsQDmYMWtcZ0runzk1hbP6G3z2UJ8h4Upk46lo9G02TycsrrodvgY9IWmpJZIc7876S3OUR12L94EdwptabCOukR0utc1bjLwdLSs5ymuO34PGiXIycJNPLn0l1kVvESdKrqcd6o/AYAxaMOvNrK7+s7MDmqSvdtiRx6qHFt5VlXcSxYUIwGnel2xpZlENHQPxvoBLa04903C2XkwqYotH531PDqHqi0nksgbC2BGyX8R0k8EMMKUSdfSUTbwB+VY4l+ykgk4gmGF5w524g+FoXkbbPk3gu4BtjRMzg4hbdhDkfvUpJx7POvbH+GqUz9Chcyh0pt4bsl9KSfeTqYdtf9AU8GFAOyuvjPpHU7Tgdsf4tDQLrcNdee2+SjK07OsMhJ4vHWyN6EgQ1GUWNuANbWWUTull5n0qCUJTyCMPcVij3Amgbe6YGQAE20XYfG1kxuIrxq4MZbAK/JfpoIIYIQpke6lo2zQ2Jf8jIeiKDQmWRRvwBPk9YMt8PhHYct9BH97JQzEv604XiWuBm7f9yFu3/9h3nP650hK+gvYnUNRuKT5V1zS8h8A7K66nVcXfCuhXSCZoEhqnl5yP4+v+B92V92R6eFMqn1tdsKyQrkph5qC0XsSXTCvCK1aosfp50QCTTkbe90MeILoNSpWjLFUqlGrKDNHgtlU30QEQjI9g5E3VWfPwPg1+fTnRpKwKwYPxXW+aBE7MQMzNUQAI0y6yVo6yrRU2gr0OP0p9fzRHPkLeCN5AgbHKW478AkqnfuTPt/ZFvW9wocOfQqTP9I2YG3Xn7jh2NfQhj0TPDJ5khLivad+xPqOPwCwde4/8fbcL0ybInCySkunadW0GW8yvMEwB9vtQGTn0Vg7AI16DeuHmnRua+wjFJ44N2v47MvKajN6zdi7/IYvI6WiyxFpoJqfoxm1+3OnKbKdOt5lpNgMjF4EMFNBBDDCpJtJS0fDOb3BpLcsp5Svoiis6foTAEfq7qTHuARDyM4th/8xVv01+XPLXNjyG65r+CZa2U+z5QJeXvAdQpKO+QNv8YFDn8Xot6Z2jVGoZT/XH7+H5dankVHx0oLvsLfq42m/jpCa/W12gmGFkjx9rDbLWNbUWsjTaxj0hWJbosfTNuClx+lHo5JYPUFybroCmLHyX6K6hurBxD8DI3JgppIIYIRJNROXjoZLtjfSKetg0tesceyh2HOagCqXN0tv588rfhvrlHz1ye9xYctvQEl8N5Iu5OLGY1/l/PbfAbCn8mM8ufSXHCm7kb+seBCPtoAydwMfPvgJSlwNSY9/tOu+78g/s8C2hZCk49klP+Fo2Y1pO7+QHv5QmANDgciGuoIJ6y9p1SounF8ERHoAeQLjzzhGi+ItqzRh0I2/ZFhhzkWSYNAXwplCHky02WnVGEthXUN5MGWuo6jkiWdMxytiJ6SfCGCESXWowzHjlo6GS2QbdFS/y59S88Lo7MvR0uvwa/IJqXN5dvG/sWto58v57b/juoZvognHP+tl9rZx28FPMn/gLUKSjhcW/oC36v4ZRYpM43fnr+CPKx+iP7eO/ICVDx76DHW2t5J+DgAqOcSK7r9zx7sfoMa5F7/ayBPLfkVj0aaUzitMjoPtDvwhmUKDjgUleRM/AFhSnk9pvp5AWGbn6bETYbudPtoGvKgk4qpyrdOoKB1qc9GZ5CxMMCzT44z8jlSPMQNjy52DT2NCK/spcZ+Y8JzGWA6MSOKdCiKAESZV9A/ETNXj9MXVBHK4VJaPzN525g0FDvsrPnTmDknFtjl38dLC7xGWNCzqf40PHP5s7A/qeGoHdvKRA3dQ5G3CpSvhzyt+y/HSa885zplTxeMr/5dW8wZ0spcbj32V1Z1/SvxJKDKLel/m9n0f5MrG+8gL9uHQV/LX5Q/Sbl6X+PmESRcMy+wbKmgYz+xLlCRJXLIw8mJ+uNOBzT16t/fo7MvisvxRc1FGk+oyUrfDh6yAUa/GPNY1JRVdQ72sJlpGUst+ckOR3VliBmZqiABGmDSKoqStrH22UhRoSjCZ91SSy04Aq7r/goRCk2XjqJ2Uj5Zez9+W/RdejZly1zE+fOCOsZd7FIU1HY/xvqP/TE54kM78FTy26hF68peNeX2/Jp8nlv6KQ2U3oULmsqafs+n0/fHtUFKUWLB03YlvUeBrw60t5PV5X+PhtX/Fmrckzu+CMNUOdThilXEXlcbfWBSgusDAvGIjihLZVn02mztA49Dv0LoEGjSmGsAMz38ZLyDrjOXBjJ/IG10+Cqr0+NWJfY+E5IgARpg0A57gjKsMO5rTCWyndniDWJ3JBXXakJvlPU8BsL/yQ2Me12Fewx9XPkx/7tzYcs+8/jdHHKOW/Ww++QM2Nf8CFTKHS2/gr8sfjGvqW1ZpeHX+t9g6558AWNP1ODce+yra0NiBXPngYW458o/ccvQLlLmP41cb2V77Dzy07gkOVHwQWRXfu25h6oXCMu8OVcZdP7fgnMq48bh4YTEqCZr7PbTaRu5ki+48ml9ipCiB1gDRAnoDniDuBGdBYVj9F8v4ycjRPJiJZmCMw7dQz+CdaNlEBDDCpLEOzuzlo6jWfg/BOLaJAknXfgFY2vsc+rAbW04tzZaN4x7ryK3m8ZW/o8VyfmS55/jXWNfxB1AUjP5ePnDoH1ja+xwyat6o+wqvLPgOYVUCfWskib3VH+eZxT8hpNIzb+BtPnj4M+T5e0YcVuhp4vpjX+PDByP9gkKSlr2VH+F3657knZpPE1SP/+IhZN7RLifuQJg8vYb6JNtnFBh0rKyyAJFZmGi1bKcvSEN3JKE9uu06XjlaNcV5kZ/ZRPNgQrJM19Dy9lgJvFHdecuQUWH2d2H0j10wUiTwTj0RwAiTpifJmYbpJiQrtPTHVx8l6fwXRWZ15+PA0OxLHH1W/Jp8nqz/JQfKb0FC4dLmX3Ftwzf5yIHbqXAdwacx8fdlv2J/5W1Jv2M8VXw5f17+37i1hZS6T/LhA3dS6jpOnr+b9578IR/fdxsLbVuQUXG49AYeXvd3ttZ9aUZXqp1JwrLCnujsy5wC1EnMvkSdN68QvUZFnyvAsaE+SvuiLQkKcik35yR8zmSXkXocfsKygkGnpsAw/uxfUGOkzzgfGH8WxhjbQi0SeKdKwgHM1q1bueGGG6isrESSJJ588skR9995551IkjTi4+qrrx5xjM1m46Mf/SgmkwmLxcKnPvUpXK6Rf9gPHjzIJZdcQk5ODjU1Nfz0pz9N/NkJGWWd4Qm8w8WzndoTCCW9Y2KufQeFvlb8aiNHS66L+3GySsPr877BG3VfQUbF4v5XyQv20WeYx2Mrf0+b5bykxjNcT/4y/rjyYfoM88gL9vHBQ5/mE3tvYbn1aVTInCzcxB/W/JFXFn6XwRnc4HAmOt7tZNAX6UsUbQ+QrFytOtb4cUdjP05vkMNDLQnWJ5D7MlyyAUy8+S9RXXHkweT5xQzMVEs4gHG73axatYpf//rXYx5z9dVX09XVFfv44x//OOL+j370oxw5coRXXnmFZ599lq1bt/LZz342dr/T6eSqq65izpw57N27l5/97Gd8//vf57e//W2iwxUyRFEUrDM8gXe4pj73hD1fGq1uku09t2Zot8/hspsSb2goSeyvvI2nlv47g7pSjhdfxZ9W/A5HbnVygxnFYE4Fj6/4X5otF6CV/WiUAG2mdfxx5e94tv5n2Azz0nYtYWrIssLu5sjsy7raglH7EiVqZY0Zc64WdyDMX99tJyQrlObrqS1MbikxmgfT5wrgC8bf6qLdHpkxHauA3dm64mjsaBRLSFMu4QYj11xzDddcc824x+j1esrLR3+ndezYMV588UV2797N+vXrAfiP//gPrr32Wu6//34qKyt59NFHCQQC/O53v0On07Fs2TL279/Pv//7v48IdITsZZ8lCbxRnkCYLofvnM68wyWb/1LgaWaufScKEgcqPpDsEGkuuIj/Wf/spCUYBjR5PLn0FyzveRpHThWt5vNEMuM0dsI6iMMbJEerYvkYfYkSpVGpuGh+Ec8f7o610lg/N/5t2Wcz6jUUGLQMeIJ02r3Mi6M+TVhW6LLHl/8SFd2JVOo6jloOjJovJvogTb1JyYHZsmULpaWlLF68mM9//vP095+pxLpjxw4sFksseAG48sorUalUvPPOO7FjLr30UnS6Mz8kmzdvpqGhgYGBgckYspBms2n2JWq83kj+UPic3RfxWt0VyX05XXgJjpwUZ00mOaBQJA2Hyt9Pq+V8EbxMY4pyZvZlTU0BOk36XioWlOZRMZTvUmDQxl0UbyyJLiNZB32EZIUcjYoiY3yJ646cajwaCxolSKnr+KjHiE7UUy/tAczVV1/NI488wmuvvcZPfvIT3nzzTa655hrC4cj0Xnd3N6WlpSMeo9FoKCwspLu7O3ZMWVnZiGOiX0ePOZvf78fpdI74EDJnphewG81426mb+tyEk6hIrA8NstT6HAD7Km5LemyCkIjGXjc2dwCdRsWqmvTMvkRJksQVS0qpLTRw+ZLSpGdfohINYDqGtQ+I+9qSRJdp/L5I0RkYl6501PuF9Et7j/rbbjvzR3bFihWsXLmS+fPns2XLFq644op0Xy7mvvvu4wc/+MGknV9IzGycgel3BbB7AlgM576rS3b30bKep9HJXvoM82kzr5/4AYKQIkVR2DVUGXd1tWXcrtDJKsrT8741VWk5V+XQMpB10E8gJE84W9Q+QQPHsXTlr2C+beuoAYwu5EInR2ZYxS6kqTPp26jnzZtHcXExp06dAqC8vByrdWQ321AohM1mi+XNlJeX09Mzsp5E9OuxcmvuueceHA5H7KOtrS3dT0WIUySBN3tnYOb3v8G69khNlHSLVhTlqbvhlyuh6wChsBz3NuvhJCXM6q4/A7Cv4kNiSUaYdIO+IG+f6qN30I9WPXFX6GxgytGSn6NBUaDLMf4sjJxE/ktU5/BE3rP+dkSL2PnUeYTUiZ1XSN6kBzDt7e309/dTUVEBwMaNG7Hb7ezduzd2zOuvv44sy5x//vmxY7Zu3UoweKbh3SuvvMLixYspKBh9u51er8dkMo34EDLD4Q3iD2ZnAq8m7OOaE9/h0pZfMce+M+3nP93rgoEW2PcHsLfA72+k6/jOpBKa59newuzvxKsxc7xk/MR5QUiWLCuc7nXx9IFOHtrWzLtDPY9W11jI1aV/9mUyxLuM1OvyEwhHZmmKE6j6C9CTtwwZNXmBXvIDI99g58VqwIgE3qmUcADjcrnYv38/+/fvB6CpqYn9+/fT2tqKy+Xia1/7Gjt37qS5uZnXXnuNm266iQULFrB582YA6uvrufrqq/nMZz7Drl272LZtG3fffTe33XYblZWVAHzkIx9Bp9PxqU99iiNHjvD444/zwAMP8OUvfzl9z1yYNNm8fFTj2INWjoxvqfXZtJ+/0+4juG9Y2QCfnfInP0jZ4JGEzxVN3j1UdjMhdeJFvgRhPIO+IDtP9/PQ9maeOdgVKQVAJBjYvKyMjfOKMj3EuMUbwMTyXyy5qBKc0Qypc+g1LgKgwjlyO7XYQp0ZCefA7Nmzh8suuyz2dTSouOOOO/jNb37DwYMH+f3vf4/dbqeyspKrrrqKH/7wh+j1Z6LdRx99lLvvvpsrrrgClUrFLbfcwq9+9avY/WazmZdffpm77rqLdevWUVxczHe/+12xhXqayOYE3rqBt2OfL+jfgj7kxK9J32ydLMsoBx+NfHHt/SiH/oa2bQe3HLmLvy/7D7qHpqEnUuw+Sa1jDzJqDlTcmrbxCbObLCs097s51OGgpd9DdCEkR6tiaYWJ5ZVmCuLcmZNNostBPQ4/obA8Zs2aZPNfojpNKylzH6Ny8CAnSq6K3S5mYDIj4QBm06ZN4xbseumllyY8R2FhIY899ti4x6xcuZK33nor0eEJWSDZZoWTTlGoG9gGQEilRyP7Wdz7CgcrbknbJaqc+9E5W0GXB6s/QkftzfDYB6h27uP9R77Ak0sfoNO0asLzRNsGnCq6DJeoXiukyOkLcqTTydFOJ65hjQ+rLbksrzIzv9SIRjV9O8tYcrUYdGo8gTA9Tv+o+S2yosQqYSea/xLVlb+CNV2Pn5PImye2UGfE9P2JFbJWti4hFXkaMfm7Can0vFP9SQCWWp9J6zXqh7Y8y0tvAp2Rk3aFJ5Y+QKt5Pfqwm/cd+QJVjnfHPUdO0E5934sA7KsUW6eF5CmKwuvHrTy8rZldTTZc/hC5WjVray3cfsEcbllXzeLy/GkdvEBka/ZEy0j9rgD+kIxWLVGaYP5LVLQib4m7AXX4zEyzURSxy4jp/VMrZB2HJ5hQSe+pVDewHYA28zoOld1MWFJT4TpCoed0Ws6vCftY1P8qAD11t6AoCo29LkLqXJ6q/wUt5vPQyV7ed/SfqXbsHfM8K3qeQCP76TEuiVUAFYRkdDl8HOpwoBBpmHj1snI+efFcLllYMi2XisYzUQATvb3SnIsqyaaUTn0FLm0xaiVMuetY7HbRiTozRAAjpFU2b5+O5r80FVyEV1dIU8HFACxLUzLvgv430IfdOPSVHNMtpdvpi5VLD6lzeKr+50O9gnzcfPSfqbHvOuccKjnEqq6/AkOzL2LrtJCCo0Ndn5eU53PL2pkx2zKW6LJQl8M7atHI9gHPiOOSIkl0mSKzMMOXkWJJvHoRwEylmfmTPAOc7BlETqJya6b1ZGn+iz7kpHJo50A0cDlaej0A9dbnkZTQmI+NV3RX09HS6znd5z2neF1YncPT9fdzuuAitLKfm499mdqBkVu5F9jeID9gxa0t5ETxe1MekzB7BcMyJ3siP4OpdpKeDoqMOvQaFcGwQu9Zy9iKotAZrf+SZAJvVOfZnakVRSTxZogIYLLUoQ4HbzRYJz4wy2TrDMycgZ2oCNOfW4czJ7Jdv6ngYjzaAozBfuYOpFYTJs/fTa1jNwBHS69l0BfiYLvjnOPCKj3PLvkpjQWXoJH93HTsK8wdSiyGM12nD5W/f9SGcYIQr0ari0BYxpSjSflFezoYLw/G5g7gDYbRqCTKTKmVJIjmwVQMHgJFISfkQD30BsitnT5bz2cCEcBkKYc3yMF2B3uGSnpPF9k6AxPdfdRUcFHsNlml4dhQgbhlKSbz1ltfQEKhzbQWZ06kRPpYxevCKh3PLvkJpwo3oVEC3HDsa9TZ3qJs8CiVgwcJSxoOlKdvZ5QwOx0ZWj6qrzCl3G9ouhgrgIlun64w56BOMv8lypq3hLCkwRi0YfZ1xGZfPNoCZJU2pXMLiREBTBZSFCWWO/H2qT5O9Aym/RqhsExT39jdk5Ph8GZpAq8iM3cogbep8OIRd0WXkebZtpITtCd5fiWWRxM930RklZbnFt/HyaLL0ShBbjj+da5ovA+AE8XvxSO2YwopcHqDtA8VbVtaMfOXj6KifZE67F7kYeU+hjdwTFVYpacnrx6ILCOJInaZIwKYLOTyh2JJaIoCLx3ujrvTajz8oTBP7u9kS5qXqKxZWsCu3HUUQ8iOT51HZ/7IGix9xoX0GJegVkIs6Z24htFoKgYPUeBrJajK4WTR5XE/TlZpeH7Rj2kofi9qJUSZ+zgw1PdIEFIQTd6tLsjFlDt7ZgVK8/Ro1RKBkEy/KwBE3hBG/35WWwxpuc7wZSSxAylzRACThRze4IivQ7LCMwc6GXAHUj63yx/iL3vaabN5sHuC9LnSt+STrfVf6myR3UetlguQVefWbozOmiRbEyaavHuy6AqCGmNCj5VVGl5YdC/HiiOtNjryV9GTvyypcQgCRF6wjw0FMMtm0ewLgEolUWkeuYxk9wTxBMKoVRJlpuTqv5ytayiRt3Lw4LAEXjFrOtVEAJOFnN5zd8R4A2Ge3N+BN5D8Eo3NHeDx3W0jMvQbz9opk4psTeCN5b8UXjTq/cdLNhOStJS5Gyh2n0jo3Oqwj8V9LwNwJM7lo7MpkoaXFv2Ap5bcz3NL/i2pcwjTkz8UJhROb+PT9gEvTl8InVrF/NK8tJ57Ohi+jDT833JTzpgtBhIV7Uxd7D5FgbcVEDuQMkEEMFnI6QuOervdE+TpAx1J/cHrcnj58542nGfN7pzqTV8Ak40JvMZAX2xppsly4ajH+LQWThdeCsCynsRqwsy3bR2q/VJBu3lt0uNUJDWni94j3sXNAk5fkH2tA/x1bzv//eZpHt3VmtYgJrp8tKgsD22aXrCnk1gi74AXRVFS7n80Gre+FKeuDBVy7A2SWEKaerPvp3saODvIGK7T7uPFI93j9qM62+leF3/b2z7q7I3V6T9nySoZTl8wpdmhyRLdotydtxSvrnDM46LLSEt6X0Alx//9iO5eOlZyLUji10kYnc0dYFeTjT/uauWhbc1sPdlHh92LQuSNyYFRttwnwx8Kx+oPLZ0FtV9GU2bSo1ZJeINhBjzBtCbwDtdliiwj5YQiAaMIYKae+IubhSYKKE72uHjrZF9c5zrc4eCZA10Ew2MHPI1pmIXJ1gTeM9unLx73uOaCC3BrizCE7LHHTMTot1I7VE33aOl1qQ1UmFEURaHb4WPbqT4e2dHMH3a2sON0fyxPrMqSy6ULi7loQaRuyO5mG/407OA7aXURkhUKDFrKU6x3Ml1pVKrYcz/WFWleqZIiW6jTqeuszvJiCWnqJdyNWph8Tt/EVWH3tgxgztWyqsYy5jHvnO5ne2P/hOdqtLpYW1uQyBDPkY0dqFVyMBZgDK//MhpF0nCs9FrWd/yBZdZnaCzaNOH563tfQIVMh2k1jtyadAxZmMYURaF9wEtjr4vGXveIrs9qSaKmMJf5JXnMKzFi0EX+9MqKwvGuQfrdAfa0DHDRgtSWEI92RmYDls6i2i+jqbLk0mH3cqDdDkCZKSfty2ln9ykTnainnghgsowsK7jiCGAAtjT0kp+jYV7JyEQ9RVF4o8HKgbb4pqU77T68gTC5OnXC443qycIE3irnPvRhN25tIT15SyY8/kjp9azv+AN1tm0YAv14dONU1VQUlg51nk42eVeYOVptHt462Uuf68xOQa1aYm6RkfklecwtNqDXnPv7pZIkLpxfxDMHu9jfZmdVjYU8fXJ/lgc8AbocPiRgySzbfXS2qoJcaCY28zwZlYh7jYsIqfRoZD8yajzasZeohckhlpCyzKA/RJnjADcd/RIFnuZxj5UVhRcOd9MzbPkmFJZ59mBX3MFL9DypLiNl4wzMiOq7ceSn2Azz6MpbhoowS3pfHPfYMtdRirxNBFV6ThZdkZbxCtNPv8vPU/s7eGJfB32uADq1iqUVJm5YVcFnL5nHtSsqWFyeP2rwElVXbKTCnENIVnjn9MQzpmOJzr7UFhmSDoJmigpzDsML7lanOf8FIsUoowXt3LoikQOXAeI7nmWc3iAru//GvIG34ypvHwjJPLW/A6cvUgX37/s6zmkiGI9UAphBX6TOQraJ1n+ZKP9luKOlNwBDybnjJEpH/29OFV1GQDP7tqrOdp5AiNePW3l0VyvN/R5UEqyqNnPnhXN579Iy5hXnxb1lV5Kk2NLRkS5nUvWeZEXheHekYvdsq/0yGq1aRWl+JOdFkqDCPDm9oKLLSCKBNzNEAJNlHN4geYFIhdxCb3Ncj3H7wzy5r4O/7GmLZdwnqrXfM2bvnolk4/Zps7eNQl8rYUlNq+W8uB/XUHIVIUlHsaeR0qHt12dTy34W970CxN86QJgZQmGZ3c02fr+9hUMdDhQF5pcY+dgFc9i0uDTpZdgqSy51xUYUBbYnMQvTavPg8ofI0aioK0msmOJMFd11VJqvR6eZnJe6aPmFsxN6hakxu+cZs5DTF6TSHwlgJlpCGq7flVqV3pCs0NLvZmFZfsKPzcYCdtHlow7TmoRmSPyafE4VbWJJ38ss63kG69AU8XDzbG+RE3IyqCulzbw+bWMWspeiKDR0D7KtsT+WnFuar+eShcVUF6SnPP2F84to6nNzyuqi2+lLaBdRdPlocXk+GpV4XwqwvNJE+4CH9XMmLzel07SKB897Ga/GMmnXEMYmftKzjNMbJC8Q2SJt8XUkVJMkVckuI2V9/kuCjgwtIy3pewm1fG5gGE3ePVp6HYqUfOKzMD10DHj50+42Xjrag8sfIk+vYfOyMm7bUJO24AWgOE/PkvLIG4jtp+IrkwDgC4Y53RtpzDqbGjdOxGLQcduGWhZMcjVir7Ygsk4lTDkxA5NlPE47OtkDgIowFl87NkPdlFz7dJ+bsKwk3G4+22ZgtGEP1Y69QGL5L1Ftlg0M6krJD1iZZ9vKyeIrY/cZA33MHdgBDBWvE87h9ofQaVRZUwXW7Q+xu9lGU58btUpCp1Gh06jQq9WxzyNfD/tco0IlSRxot9M4FBzo1CrWzy1gTY0lbSXpz7ZxXhEne1y0DXhp6Xczp2ji5aCGnkHCikJxno6S/PT0+hGE6UAEMFlGdnaM+LrA2zxlAYw/KNM+4Inrj2bUoC+I259dCbw19t1olCD2nCoGcuck/HhFUnO09DrOb3+IpdZnRwQwS3pfQEWYzvwVDBjmpnHUkUabuVp1wgFkNmnud/PMgU5K83P4wLpqVBl8Lv5gmL2tA+xrtROS469cfTZJguWVZi6YVxir3zJZTLlaVlSb2d9mZ3tjP7WFhgnruYjaL8JsJQKYLBKWFSRX14jbCr3NNE7hGBp7XQkFMNnYgXreQHT30UVJT+0eLb2e89sfYu7ADoz+Xtz6khG1X9KdvNsx4OVv77azsCyPa5ZXpPXcU8XuCfDi4W5kBbqdPg6021mTYoHEZATDMgfa7expHsA/lJhebsphw9wCdBoVgZBMICTjD8kEwkP/Dn0evS/6dXGejgvnF1No1E3Z+DfMLeBopxProJ+TVheLxslL63P5sQ76UUmR/BdBmE1EAJNFnN4gef7eEbcVelqmdAyNVjeXLVbifieXdfkvisLcge1AcvkvUfbcWjryV1E1eID63ufZU30Hpe7jFHsaCUk6ThS/N10jBuBgux0FONHjYl2tj9JpVgY+EIrUH/KHZAw6NZ5AmB2n+1lQmkd+jnZKxhCWFY52OnmnqR/30Lb+QqOOC+cXMa/YOG1mJww6DWtrLexssrG9sZ/5JXljzspFGzfWFRsnfXZIELJNdixSC0BkB5JxKIDxqSPvpgri3EqdLi5/iO4E+hplW/5Lseck+QErQVUO7eZ1KZ3rSFkkmXep9dkRsy+nijbh16Tv3a4/FKaxzx37emeTLW3nngqKovDqsR763QEMOjUf3lBLhTmHYFhhS0PvxCdIw/Ubugf5w84WXm+w4g6EMeVouGppGR89v5b5JXnTJniJWlNbQK5WjcMb5Ejn6EUpw3KkDQGI5F1hdhIBTBZxekOxGjDRF99Cb/O4BdUmQ6PVPfFBQ7JtBqbOFtl91GreQFiVWkLjyaIrCKr0FHmbqXLui1XnTXfjxpNWF2FZIU+vQQKa+twjqitnu72tA5y0ulBJcN2KCvJyNFyxpBSVFEkMT0ez0NEoikJTn5vHdrXy4pFuHN4guVo171lUwsc3zqG+woRqmgUuUTqNivPrItt/32myEQyfW6Opud+NNxgmV6tOaNlXEGYKEcBkEacvSF4g8o613bwGGRX6sBtjMPny4sk4ZR2M6ziXPzSiYV02iOW/FCa+++hsAU1erE3AVad+SG7IgUtXQqvl/JTPPVz0XfSqanMsj+GdaTIL09LvZvupyM/nexaVUDnUc6YoTx9rELqloTfpIolj6XJ4+eu77Tx9oDNWwn/jvCLuvHAuq2ssM6IWyvIqM6YcDZ5AmH2t9nPujybv1lfkT+vEb0FI1vT/LZ9BhlfhdeircORUAYkVtEuHAU+QftfEMyvWLJslyAnaKR88DEBTwYVpOWe0JozF1w7AsZJr0lr7xekN0mGPVE9eXJ7PeXWF02YWxuEN8sLhbhRgWaWJFVXmEfefV1eIOVeLyx9iRwo9fs52yuriz3va6bT7UKsk1tZauPOiuZxXVzhpFVczQa2S2Dg/0lB0b8sA3uCZ3X6eQIjmflH7RZjdZs5v+wwwPInXpS+NbQGOt6VAOsXTTynbWgjMse9EhUyvYQEufXlaztluXotDXxn7Ot27j6L9a2oKcsnP0VJg0MWKme1M44t+ugXDMs8e7MQfkikz6dm0qOScPBOtWsVliyM9Yg602dMSkHXYvbx4pBuAhaV53LFxDpcsLCFXOzMLCi4uy6c4T0dgqIVB1PHuQWQFykx6ivJE7RdhdhIBTBYZ9HgxBCN/pNy6Emy5c4HMBDDR4l3jybYE3jPNG5PffXQOSRXLeenOW5rWmjyKonCsK7oMcOZd9Hl1hUgSNPd76HZk1/cYziTt9rkiSbvXr6gcs7DbnCIji8ryUIDXj1uRU6jHYnMHeOZAJ2FZYV6xkauXl0/ZDqdMkSSJi+ZHGj0ebHPg9AVRFGVE7RdBmK1EAJMlgmEZxdWLChkZNR5tAbahQmmZCGB6nD6cvvHbGPROQg0YXzCMO4m8GkkJM9e+E0hP/stwL5s/yO9Ut/JgwVfSet4epx+7N4hGJTG/5Ey5c8uwWZh3mrJvFmZfq50TPZGk3WuXR5J2x3PpwhL0GhXWQT8H2u1JXdPtD/Hk/g78IZlyUw5XLy+ftgm6iZpTZKDKkktYUXjntA3roJ9+dwC1Shq3RowgzHQigMkSkR5IkeUjt64IRVIzMDQDM9U5MFGN4ywjuf0hBn3pTeD1BcM8+k4rv9/RjM2dWHPKisHD5IYc+DQmuvKXp21MYVnh2QYn93rez5+ajaPuBklWdPZlQWneObkb583NzlmYVpuHt4f69Fy6sCTW8Xc8Rr2GixZEZhF2nO5ncILA+Gz+UJin9ncy6AthydVy46rKrGlTMBUkSeKiBZFcmGNdTrY3RoLa+SVGcmbo0pkgxGP2/BXIck7fmS3ULl0pALahHBhToAdt2DPlYxpvGWkyKvBua+zD5Q8RDCu8dKSbcALLDXVDu4+aLRtRpPQV9Hq3dSA20+QLyhzqGL0mR6LCssKJnkj+y5JRKqhaDDrqyyPLAzuzZBbG6Q3ywuEuFCI7X1ZWmyd8TNTySlOsNsybJ+KvDROWFZ4/1E2vy0+uVs3Na6rI1c2+F+0Kcy7zS4woRIJIEMtHgiACmCwxfAbGpY8kPvq0FjzayFbUAm/rlI+pY8CLLzh6n6N075DptHs53BGZkdCqJayD/hFJixOJ1n9JZ/6LzR2IbWeeWxTpOry3ZYBQGmZhmvvd+EIyRr2amsLROxpvmFuAJEFLv4cuhzfla6YikrTbhS8oU5qv5/LFpQkVh5MkicuHasM09sZXG0ZRFF471kOrzYNWLXHT6krMuTM752U8F84vJvodz9Nrxvy5EYTZQgQwWcLhDZLnHzkDA8QSeae6Ii+ArChjvtCkcwYmLCu8fjzy3JdWmLhiSRkAu5ptcVUFzvP3UOI5iYJEc8HGtIxJHkpUDcsKc4oMXL+ykvyhmhzR8u2piC4fLSkbu9ja8FmYd05nri6Moii8dtwamwW5fmVFUt2YixOsDbPjdD/HugeRhnJtyqZZe4V0KzTqWFYZ+XlYXjV9i/QJQrqIACZLDC9i59KVxG6PbaXOVB7MGMtI6awBs691gH53gBytiosXFLO4PD+yc0WBl490TzjjUTcQmX3pyl+OT2tJy5gOtjvocvjQqiMzB2qVxLqhF989LQMJLW+dzRsM0zTUOmBJxfhJmOfVFaKSoMWW+iyM0xfkxcPdPLW/g9eO9bDzdD+HOhyc7nNhdfpw+0PIo1R93t9mpyEaSKxIbedPvLVhDrbb2d08AMAVS0qZWywqzQJsWlzK+9ZUsWFOYaaHIggZJ7p/ZYnhbQSGBzBntlJPbVPHqNZ+N8GwPCJp0hNIXwKvwxuMLdNcsrAklt9w2eJSOga8DHiCbDvVz3sWl4x5jmgA01SQnt1HDm+QbUOJqhcvKMY09IK9rNLErmYbg74Qx7udLKuMPwdkuJM9kRoeJXl6iieo4WHO1VJfYeJIp5Odp228b01VUte0uQM8sa9jwsrJkgRGnQajXo1RpyFHq+ZYd2S26NKFJVQXpLZsEa0N8+T+Tg602VlSnn/OzEpjryvWQ+mCusKkv88zkVolUSuWjgQBEDMwWWPkDEx2LCEBBMMKLf0jZ2HS1f9IURTeaLASkhWqLbnUD0tmzdGquXJpZClpf7s9lrh4NrXsp9a+C0hP/ktkuaSHkKxQZckdUV1Wo1admYVpHhh1tiIex4ZaB0w0+xK1YW5kFqbV5qHTnvgsTLfDx1/2tOHyhyg06Lh8cSnn1xWyvNJEXbGR0nw9Rp0aiUjbLZc/RI/Tz+k+N0e7nChKJNF4VQJJu+MZrzZMl8Mbq+67vNLEeXVipkEQhNGJGZgs4A+F8QbCw6rwDltCMkSWkAq8rUhKOK1l7ON1yupmQemZF9t0JfCesrpo6fegHkrwPDspdG6RkRVVZg51OHjlaA8fO78W/VnbRqsd76KVfbh0JfQaF6U8piNdTtpsXtQqiSvqzx3T8iozu5tt2L1BTva4Yr2L4jXgCdDt9CFJkSqr8Rg+C/NOU2KzMC39bp471EUwrFBm0nPT6qoxq9bKsoJnqA5P5COMKxBCBaybU5DWjs6XLiyhpd8Tqw2zpraAAU+Ap4cK1c0tMnBZgonCgiDMLmIGJgs4vSF0IRc6OTLL4B62hOTUVxCSdGiUACZ/V0bG19TnHvEuOR0JvP5QmC1D22nXzy2gwKgb9bhLFhbHcia2jLL9dqn1WQBOF1wSWf9Igcsf4q2TkaWjjfOKKDCcOyadRsWaoVmY3c02lARnYaKNG2sLDRj18b9/OC+JWZgTPYM8faCTYFihttDA+9dUj1tyX6WSyNNrKDPlMK8kjxXVZjbOK+L8eUVJJe2O5+zaMD1OH0/u68AXjLQmuHZFBSrRoFAQhHGIACYLOH1BjIHIC6dfbSSoPrPGHSloVwtAgSczeTC+YJj2gTMvmumYgdl+qh9PIIzFoGX9nIIxj9OqVWxeVoZEpP/LyWGdso1+Kwv7XwPgYPn7UxqPoii8cdxKYKi3z5oay5jHrqo2o1Or6HcHON03ccuF4dc4PpRPEt1dFC9TrjZW9yOeujAH2u28cLgbWYFFpXncuKoy6xodDq8N8/ieNpy+EOZZWKhOEITkiL8SWcA5rAv18PyXqIEM9kSKOtUbCRy8gXDKCbzdDh8HhwrCXb64dMJ39xXmXNbPjQQ5rx+3xloNrOr+G2olTLtpDb15i1Ma04keF6f73KgkuLK+bNx3/3qtmlU1kXyQXU3xz8J02n04fSF0ahXzSxLfVRPNhWmzeWMdrM+mKAo7T/fHkmBXVpnZvLwcdRbOZgyvDaMoRArVra7EoBMr24IgTEwEMFnA4R19C3WULYNdqaNO97pRFCXl2RdZjiTJAtSX58ddjOv8uiKK83T4gjKvHbeiCvtY0f13APZV3pbSmDyBUKw67Ia5hRPuDAJYXWNBo4oU3Bsrwfhs0d08C0rzklqSGTELM8oWZEWJVLmN7uo6v66QTYtLsrpeSHGenk2LSikz6blxdSWWUZbtBEEQRiMCmCwwoo2AfpQAZqipY0GGtlIDDPoiO1NSzX/Z32anzxUgR6Pi4oXFcT9OrZLYvKwctSTR1OfGdOpJDCE7Tn05jYWXpjSmN0/04g2GKcrTsWFufLteDDoNK4Z25bwTxyxMKCxzsidSFLA+zt1Ho9kwVBemfcBLx7BlvbCs8OKRbg60R2a2Ni0q4YJ5RdMiCXZFtZnbNtRSPssL1Qnj06ikWdlGQhibCGCygNMbPLMDKUuXkCCya8g6mPwMjNMbjBUvu3hhccJLBcV5ejbOLwIULuj9KwD7yz+YUu+j070uTvS4kIgsHSWy1LK2tgC1JNHl8I25pBO7Tp+bQFgmP0dDlWXiBohjMeVoY3VRorkwwbDMMwc6Yx2ir15WzqpxcngEYTpaWJYf1+yoMHuIACYLjFWFNyqaxGsIDpATtE/l0EZo7HXRk2QNGEVR2HKiN1ZfJdlGdGtqLVxrOk291IIPHYdKb0jqPAD+YJjXGyIzX2trCxKeAcjTa1g6VNo9WjV2LLHWAeX5Kc+KrJ9bEJuFOWV18fd3O2ixedCoJG5cVZnw1m5BmA5W1ZixzOJeWMK5RACTYb5gGH9QHjeJN6g24NRFirplchnJ5g7g9AaTemxjr5umoSTZ0Wq+xEslSdxtiOw8+lvoEnZ0JV/S/61Tfbj9YSy5Wi6Yl1zBtPVzCmLbm7sdo89OeQIhWobyZBLdfTSa4bMwzx3qotvpI0ej4v1rq5hTJEruCzNPmSmHCnMuFoMIYIQzRACTYdGAIDoD4x5lBgZgYCgPJlMtBVIRqfkSCdDWzymkcIyaL/Ew+TpZYt8KwMPhzexo7KfPlfisUKvNw5HOyKzIlfVlSdc5MeVqYzMeu8bont3QPYiiQJlJP2a9m0RtmBtZvoLITNCt66qpMCe/NCUI2WzlUL6ZCGCE4UQAk2FOXxBJCWEIRPIZXPpzZ2BgeE+k5ikaWfrsbLTh9ocx52rZMHfsmi/xWNX1F1TItJjPI1S0mLCi8NKRboJhOe7tzMGwzGvHIjuhVlaZqSpI7YU/mvjb1Oemd5Qk5+PdkS3o6Zh9icrP0XL5klIWlebxgXXVFIncACGD8nM05OdMzvZ3vVYVe5NgzhW71IQzRMGFDHN4gxgCNlTIyKjxaEd/gY9upS7IUFfqZPU4fexvtwORpaNUKrpqwl6W9zwFRLZOX2Es5dF3WulzBfivLY1AZLeSWiWhGfr33M9V+IJhnL4Q+TlnqsGmosCgY1FZHid6XOxutnHtiorYff2uyM4tlQSL4mwdEK+llaZYDo4gZIpWHcm96nT4eOO4Ne3nX1phihU2tBi0SEN1gwRBzMBkWKQLdXT5qGjMXkcDGe5KnQxZVnh96A/a4vL8lLvoLrU+T054EHtONU0FF2HUa7iyvjS2lAKR7cSBkIxnqOCe3ROkzxWgx+mn0+6j1eaJbQW/Yklp2qrTRmdhTlpd2NyB2O3HhmZf5hYZxRZQYcaRJNi8rJxSUw7LK01pn4WRJFhVbYl9rVWrMIpCh8IQ8ZOQYRPtQIqKLiGZfR2o5QBhVfZPpR7tcmId9KPXqLgk1ZkORWF11+MA7K/4IEiRwGNeSR6f3zSfkCwTCiuEFYWwrEQ+l4c+l+XY55GvFUy52pQDquGK8/TMKzZyus/NnmYbVy0rR1YUGqLLR0nuuhKEbLZxXhELh2YWNWoV6+YUxKpAp0NNgeGcvDGzIdIbTRBEAJNhTm+QqtgOpLEDGLeuGL/aiD7sxuxrx2aYN1VDTNrRoa3DG+YWJtS4cDS1jl0UeZsIqAwcOWvrdGR5SE2Kl0jZhrpCTve5Od4zyPnzinB4g7j8IfQaFXOL0xcsCUI2qK/I5/x5RSNuW1FlZk/zQNoCjGjLjuEsudoRRRyF2UssIWVYpArv0AzMGAm8AEgSA1nQUiBeTm+QrqFtxemoS7Km808AHCm7gYAmL+XzTYZyUw61hQYUBfa02Dg+FMAtKstHoxK/asLMUWHO4cr6snNu16hVrEsxUT8qP0fDvOJzf9dFuwkhSvxVzSBPIEQgJMeq8I61hToquoyUqa7UiTjRE1k6qS7IJS/FqRGLt5V5A28DsL/iAymPbTKdN5QLc6xzkJPW1FsHCEK2yc/RcMOqyjET8ldUmTHqU8/3Wl5lHrWpqthKLUQlHMBs3bqVG264gcrKSiRJ4sknnxxxv6IofPe736WiooLc3FyuvPJKTp48OeIYm83GRz/6UUwmExaLhU996lO4XK4Rxxw8eJBLLrmEnJwcampq+OlPf5r4s8tyTm9kmnW8InbDTaet1CeG+v6kY+fN6q4/A3C64CLsQ7NQ2aqqIJcqSy5hJZJrY87Vih4/woyh06i4aXXVuEvCWrWKdXOSKwwZpVZJLK86d/kIENV4hZiEAxi3282qVav49a9/Per9P/3pT/nVr37Fgw8+yDvvvIPRaGTz5s34fGeqlH70ox/lyJEjvPLKKzz77LNs3bqVz372s7H7nU4nV111FXPmzGHv3r387Gc/4/vf/z6//e1vk3iK2csxVMTOGEcSL8CAYXosIdncAXpdka3DC0pTW+7RhVwstT4LwL6K1LpOT5XhtW7q09A6QBCygSTB1cvLKcmfuObQyurUZmHml+SNOXNrFjMwccnmLvTpkvDc/jXXXMM111wz6n2KovDLX/6Sb3/729x0000APPLII5SVlfHkk09y2223cezYMV588UV2797N+vXrAfiP//gPrr32Wu6//34qKyt59NFHCQQC/O53v0On07Fs2TL279/Pv//7v48IdKY7p29kFd6JApjYEpK3JVIIIUt/QBuGlo9qCw3kalObSl5qfRZ92E1/7lxaLeenY3iTrrbQQG2hgR6nj3pRp0WYIS5eUMz8kvjekGiHdiRtPdGX1LWilXdHo9eoMejUeALhpM49G1QX5HJFfRnPHuyk3xWY+AHTVFpzYJqamuju7ubKK6+M3WY2mzn//PPZsWMHADt27MBiscSCF4Arr7wSlUrFO++8Ezvm0ksvRac7k6y1efNmGhoaGBgYvWme3+/H6XSO+Mh2Tm8QbciNPuwGJkjiBRw51cio0YfdGAPJ/WGYbIqixPJfFqe6fKTIseWj/RUfytqA7WySJHHTqko+fXEdphzxblGY/pZVmlg/N7FloZXVFgxJ1D4qytNRM0GJA5EHM7Y8vYZrV1RQaNTxoQ011BXP3P5oaQ1guru7ASgrG5mdXlZWFruvu7ub0tKRL9QajYbCwsIRx4x2juHXONt9992H2WyOfdTU1KT+hCbZ8BowfrWRoHr8X9qwSocjpwqAQm/TpI8vGb2DfuyeIGqVxLw4362NpW5gOwW+NnzqPI6VXpumEU4NlUpKqeqwIGSLqqF384nSqlWsT2JH0ooxcl+GEy0FRqeSJK5dWRHLUdJr1Ny0ujKp/4fpYMb8hb3nnntwOByxj7a2tkwPaUIOz/AiduPPvkTZsnwrdXT5aF6xMeUqt2u6IlunD5fdNGFwJwhC+plztdywshL1KLuB4pHoLIxOo4qrPYaYgRndxQuLqLKM7O0mSRKXLCxh87JyNEn+P2artAYw5eXlAPT09Iy4vaenJ3ZfeXk5VuvIfhmhUAibzTbimNHOMfwaZ9Pr9ZhMphEf2UxRFAZ9w9sIxFepNtqVuiALWwpElo/Ss/uo0HOaOfZ3kFFxIMu3TgvCTBTZcVSZUguMaC5MvBaX5aPXTHw9EcCca0Fp3ri7v5ZWmrh1fXXKZS2ySVoDmLq6OsrLy3nttdditzmdTt555x02btwIwMaNG7Hb7ezduzd2zOuvv44sy5x//vmxY7Zu3UowGIwd88orr7B48WIKCmbGVJg7ECYkK3FvoY6KbaXOwqaOnQ4fLn8InVrF3KLUZkxiW6cLL8U5tGwmCMLUUEkS162oSEuX85XVlriDoJWjVN4djUUsIY1QYNBy1bKJl/kqzLncdl4NZTOktEPCAYzL5WL//v3s378fiCTu7t+/n9bWViRJ4otf/CI/+tGPePrppzl06BC33347lZWV3HzzzQDU19dz9dVX85nPfIZdu3axbds27r77bm677TYqKysB+MhHPoJOp+NTn/oUR44c4fHHH+eBBx7gy1/+ctqeeKY5z95CrR9/B1JUNi8hnRjq+zO/1JhS/oc+5GSp9Tkg0nVaEISpdcG8QuamKflTp4lvFqbSkkNpfnwvrGIG5gytWuK6lZVxzVwB5Odo+cD6apakoUJ6piU8l7Rnzx4uu+yy2NfRoOKOO+7g4Ycf5utf/zput5vPfvaz2O12Lr74Yl588UVycs78YD766KPcfffdXHHFFahUKm655RZ+9atfxe43m828/PLL3HXXXaxbt47i4mK++93vzqgt1NEaMHn+ifsgDRdtJ5AfsKINuQlqsiPDXJaVWOXZVJePlvc8hVb20WtYSLtpbTqGJwhCnAoM2oR3HE1kVbWFvS0DeMfZ+rxyWNfpieRo1eRo1fiCYiv1ZUtK46rNM5xWreKaoRm27Y19KMokDW6SJRzAbNq0CWWcZytJEvfeey/33nvvmMcUFhby2GOPjXudlStX8tZbbyU6vGkjOgOTaBKvT2vBoy3AEBygwNeKNa9+0saYiLYBD95gmFytmpqC5JePJCXEqq6/ALCvcvpsnRaEmeLSRSVJJ+2ORadRsba2gG2nRi//YNCpWZhg0UuLQUu3Y3YHMCuqzCyrjG/ZbTTn1RVSlKfjxcPdBEJyGkc2NWbMLqTpxumLthGIr4jdcNmYBxPdfbSgNC+lP37zbVsx+7vwaswcL96cruEJghCHumJjyuUPxrKqxkzOGIUtl1WaE152nu0tBUpNejYtjv91YyzzS/L40IYazGd9P9UqiRytGlOuluI8HZWWHOYUGVhYlsfSShOray0Z/z+YOenI04zTG0RSwhgD/UD8MzAQWUaqdu6jIEvyYEKyTKM1UowvleJ1khJiXcf/AXCo/H2E1TMj0UwQpgO1SuI9i1J/QRyLXqNmba2F7Y39I26XJFgxTuXdsczmlgI5WjXXrxi7oWaiivP0fPSCWnwBGZ1GhVY9PepYiQAmQxzeIIaADRVhZNR4dPGvOffn1gFQmCVbqVv6PQTCMnl6DZWW5IIOlRzimhPfoXLwECFJx4HyW9I8SkEQxrOm1kKBcXJ396yutfBuq31E7kpdsfGcd//xyNROJEmKzP6Um3MoNeVgytHy0pGpW4KRJNi8rCztAZxeo447EThbiAAmA2RZweUPUTy0hdqtK0KR4v/BObupo6IoDHiCGHTqMadoJ1PD0O6jRWV5STUuVMkhrj3xLRb2v05Y0vDckvtw6Uev9yMIQvoZ9WrOq0tv4u5o9Bo1a2ot7Bg2C5NI8u5wU7UTyZSrpcykp9yUQ5kph5J8/Tl/Z0NyKS8cGr1KfLptmFs4act8040IYDLAFQgRlpWk8l8A+vSRAMbsaeW5/W20OwL4QjKaoSngZZWmKeuAHAjJNPVFlo+S2X2kkoNc2/BNFtq2EJK0PLvkJzQVXpLuYQqCMI6LFhRP2bvv1TUW3m0dwB+UMedqk64ZNVkBzJwiA5WWXMpMOZSZ9Bh0E79MLik30WbzcrjDMSljiqopNLBxXtGkXmM6EQFMBjg8I2vAuCcIYEJhmR6nnw67lw67lx6Hh49rtORIQQK2FnxKGSoJQrLCa8ettNg8XLGkdEpmY073uQjJCuZcLaUJbuVTyUGua7iHBbY3CUk6nlnyU5oLL5qkkQqCMJoKcw5LK6aucnmOVs2amgJ2nu5nRbU56TdbBp0GvVaFP5i+pRu1SuKm1VVJbUTYtLiEbqePvkF/2sYzXKRJYzmqGdYOIBUigMkAp+/sLdQjAxhFUWi1eYYFLH7CI7auSzRrKlhCKzdWuuitWE9xvo4DbQ62N/Zxyuqi2+Hj6mXlVBWM7IuRbtHWAYvL8hP6Q6SWA1x3/F+YP/AWIUnH0/U/o6XgwskapiAIo5Ak2LS4dMpmbKPW1Fo41GFneQpbgCHSq8kaTF/AUJynT3oXpVat4roVFfxxV2va82GiTRrjmQ2aTcR3IwOc3ugW6qEidmdV4d3ZZGNXk23EbQadmipLbuSjIBepfRH0t7Ihv493zZHE2XVzCqguyOWFw904vEH+9m47G+oKOX9u4aRE7b5gmJb+6PJR/GuyajnA9ce/wbyBtwmp9Dy15H5aCy5I+/gEQRhffYWJcvPU7/bL0aq5eXVVSn2WIJLIa3WmL4ApM6XWOqHQqOOyxaW8dCR9+TAqSeKqZWXnNGkURACTEbEZGP/oRexO90ZmNeqKjcwvMVJpycWSqx3xLmnAMBf6z20pUGbK4SPn1bLlhJVjXYPsarLRZvNw9bJyTGnes3/K6kJWoDhPF3fPFLXsHwpetkWCl/qf02o5P63jEgRhYjqNiosXxNdEdjKUpqEfT7rzYNLRI2hppYm2AQ9HO50pn0ujkrhmRQULEizyN1tk/0bvGchxThXeMzMw3mCYPlcAgCuWlLKs0kyBQXfOFG+smN0oW6l1GhVXLS3n6mXl6NQquhw+Ht3VyomhYnPpEi1eF2/yrlr2c8OxrzFvYBtBlZ4n6/9dBC+CkCEXzCvEOM07Eyez/Xo8pSnOwERdvqSUorzUtnlr1RI3rq4Uwcs4RACTAWfaCJzbibp9wANAkVE37h+XaABTME413sXl+Xzk/FrKTTkEQjIvHO7mlaM9aVmfdftDtA94gfgCGHXYx43HvkqdfQdBVQ5P1v+SNst5KY9DEITEFRp1rK6ZuMFitkvnDIxWLVFsTE8Ao1WruHZFBVp1ckv3eq2K962tZk5RdvS6y1YigJliYVnB7Q+jDXvQhyP5I8NnYKJBQfUEybcDubUAGEJ2coL2MY8z52q5dV01G+ZG/lgd7XLyx92tWJ2+VJ5GrHFjuSlnwndBmrCPm45/lbn2nQRVOTyx9Je0W9andH1BEJL3nknod5QJFkP6itkV5+nTmitYnKdn0+L4K6xHGXRqbl1bLXJe4jC95w+nIZcvhKwomIe6UPvVxhEdpc8EMOPXRgipc3HqyzH5uyn0ttCptYx5rFolceH8YmoLDbx0pAe7J8jje9q4aH4xa2otce9AWNX1Z0pdDbh1xYS61OhVeVQVzCPPr8WjLUZWnfvjpAn7uOnYl6l17CagyuXJpQ/QYV4T1/UEQUi/eSVG5hbPjHf2eXoNOo0qLbPK6ch/OdvyKjPtAx6OdcW3fJ+fo+H9a6spnOSKyDOFCGCm2Hj5L25/CJs7kv8y0QwMRJaRTP5uCrzNdJpWTXh8dYGBj55fy6vHemjsdfPWqT5O97m5sr50wncyNfbdXH76Z7GvzwfQAT2RDwUJj7YAt64Yl64Et64Yt7aYaue7VDv3EVAZeGLZA3SaVk84TkEQJsdk9zvKBFOuNi21V9KV/3K2y5eU0eP0x/62j8Wcq+WWtdWzusdTokQAM8XOrgHjHmX5qCTv3FLVoxnIncNc+86EulLnaNVct6KCwx1Otp7spcPu5dF3Wtk4v4jVNRZUo8zGSEqITU0/B6DZcgEngiXIzm5qNA5qtA6MwT7UShhj0IYxaKPUfWLE4/1qI08sfYCuOIKs2SxHqx7RI0YQ0m1tbUFal12ygSVNAcxkzMBAZFPFtSsq+NOuVkKyMuoxRXk63r+2mrxpnlQ91cR3a4pFE3iNgXO3UEcTeKsL41v7PLMTqTmhMUiSxIpqM7VFBl491kP7gJe3TkYK4F1ZX3bO9OXK7r9T7GnEqzHz/KIf8bt37fQFA1wxv5TlVWZQZHJDDoyBXvICvRgDfbF/NbKf/RUfxJpXn9AYZ6Mr60t59ZhVBDHCpMjTa6ak39FUS0cir06jomgSl21K8iP5MK8e6znnvlKTnvevqU65Js5sJAKYKXbuEtKZOgxtcSbwRsV2IiXZldqcq+X9a6o40unkrZN9dDl8PLarlfPrCllXW4BKJZETtHNhy4MAbK/9PJ3+HPpcAVQSZ7b3SSq82gK82gL6jIuSGstsZzFoWVCaR9uAhwNtk9tPRZidLlpQjE4z8/ZtpKMrdUmeftKrEa+oNtM24Ik1vwWosuRy05rKadcFOlvMvJ/mLHdmCSlahTcyAzPoC+LwBpEk4s4+HzDMBcDs60Atj7++OhZJklheZeZjF9Qyp8hAWFbY3tjP43va6B30c2Hrg+SEB7EaF3Ko/OZY64A5RcaMdL6eqVZWR5Kpl1akVlpdEEZTacmhviLxZqvTQTpmYCYr/+VsV9SXUjA03rnFBt63tkoELykQAcwUi7UR8I9M4o3mv5Tm6+P+gXZri/CrjaiQsfjaUhpXfo6Wm1ZVctXSMvQaFdZBPwf2bGVF998B2FL3VWRUw4rXieJK6aLTqFhWGWmmV27OSbkAliAMl6l+R1MlHUmvk5X/cja9Rs21KytYUp7Pjauq0KrFS3AqxHdvCoXCMu7AWX2QhnJg2obyX2om2D49giQNK2iX3DLSyNNJ1FeY+PgFc5hfbOC7mkdQofCSdCHvSkuxDvpxeINoVBLzikUAky71FfkjZrPqp7AzsDDzzSvJm7IX6EzI12vQpFi/ZSq/P6X5OVyzomJG1OHJNBHATCGnL4SigKSEMQb6gcgMjKIocRewO9tAkom84zHqNXyp8gjnq47jVXR83/thHt/dxitHIwlodcXGGbmWngmSxDkVUesrTKPuBpsu9Frxs5FNokUsZypJklKahdFpVLFlHWF6EX9pplB0B5IhOICKMDIqPLpCnL4Qg74QKgkqE6y+aDPMAdIbwGjCXt7T/AAAu6rvIL9sDgrQP1THYHH5zFxLz4TaQsM5u77y9Bpqi6ZnFU6jXs2HN9SKZbAsUV2QS4V5ev4sJSKVnkil+ZOfwCtMDhHATKEzXagjy0ceXRGKpKHNFlk+KjflJLwmemYnUnPaxrmh/ffkB6w49JUcqLmda5ZXcP3KCvL0GorydMwpTGCZSxjX6hrLqLdPx2RevVbFzWuqKDDqWFs7s9/1Txcb5s68bdOjSaW2zUxeXpvpxDbqKRRN4DUGRk/grU4iMIgtIXlaiKxPpfZOwuTrYH3HHwDYWvfPhNWRX+75JXnML8lDVpRpvbyRTSwGLXVjlHSfX2JEr1XhD6ZeIn0qaNUSN62uojQ/8vOypDyf7Y19uP2ipk2mlJr0M6ZlwEQsKczAiABm+hIzMFPIMUoXakVRhiXwJj7Va8+pRkaNTvbEzpuKS5sfQKMEaDWv51ThZefcL4KX9FlVM3YfKo1axeI4unxnA5Ukce2KihHb/zVqFSurLZkb1FmqCnJZU2tJujvwdDRbZl8gta3UZVO0hVpIPzEDM4XO1IDpAyIzMAOeIJ5AGLVKojyJdwKySos9p4pCXyuF3mZc+rKkx1dj38XC/jeQUbOl7ispz+YIYxu+dXosSytNHGzP7qJ2kgRXLStjXsm5u9JWVVvY02wjGB69fPpky9NrqK8wsazSRMFQntG6OQVsO9XH8e5BlMwMa8T4Fpblcbx7EG8gvTNVBQYtC0tnz07BZJeQcrTqGddaYTYRAcwUcp4zA1MSm32pMOegSbImwIBhLoW+Vgq8LbRazk/qHMP7HR2ouJV+44KkziPEZ2mFacJ6PxXmXIrydPS7kitSOBXes6hkzG3fuTo1SytNU1pZWCVJ1JUYWVZpoq7IiOqsrar5OVquXl7BqhoLbzb00uXwTdnYIPI9WVCSx+LyfKoLcpEkiZJ8PS8fObfEfCrWzSmcVYmpphwNapVEeIxeQ2MRsy/TmwhgpkggJOMZepcVa+SoL6G9J5L/klD9l7PYcucyn60JNXU826quv1HsOY1XY2ZH7WeTPo8wscjWaUtcx9ZXmHj7ZN/kDihJF8wrYs0Eybpraws42O6Y9NmOQqOOZZUm6itMGONoiFdhzuVDG2o43j3ItlN9DPpCkzY2nUbF/BIji8tN1BYazqn/sazSzLGuwVgyf6ry9BqWTjC7N9NIkoQ5Vzthx+ezifyX6U0EMFMkunwEZ3YhDWpLzjRwTCL/JcqWm9pW6tzgABtb/xuAbXP+Eb9mdv3xm2pzigyxJY2J1FeY2H6qHznT6x1nWV1rYeP8ogmPsxh0zC/J45TVlfYx6DQqFpbmsbzKnHD5AThTuHFBaR57mgfY25K+5S6NKjITtLgsn7nFxgl3F16+pJRHd7aM2a04EWvnWGZlkTSLIZkARszATGcigJki0eUjOLMLqS1kwReU0aqllN4JRHciFbtPUj54mO68ZQnlr1zYEu13tIjDZTclPQ4hPmcXrhtPtCZMc1963p2nQ31FPpsWlcR9/Lo5BWkPYOaVGLlmeUVaCipq1So2zi9iWZWJbSf7aOhJPD9GkiLLU8V5OhaW5jO/1JhQj5tCo471cwvZebo/wdGPlKNVRzrEz0LJ1IIpFTMw05oIYKaIc2iKWhP2khOO/DE/5jYCXiotuSm9Y+o3zCMkaTGE7Hz44CcYyKnheMk1HCu5GkduzbiPLXE1sKLnCSDS70iRRGOxyVRg0DK3KLHlwqUV5qwJYOaVGLlqaXlC+RWVllwqLTl02tOTb5KjVXNFfVnaq0GbcrRcs6KC1bUWtjT00j1KfoxRH0n6LDDosBi0FBi0WAw6LLnapHPYojbMLeBEz2DCswjDrao2z9rmgIkm4xp0akw5ogLvdCYCmCly9hbqgMrAKUfkD14qy0cAAU0ef13+IKu6/8qC/jco8LWxse23bGz7LZ35KzhecjUNxVfh01pGPlBRuKzpfiQUjhdfRYd5TUrjECY23tbpsWRLTZiqglyuXVFxTmJsPNbWFtBp70rLOC5dVExeHHkuyaow53LbUH6M3ROkwKiNBSyTGRxo1CouX1LKX/e2J/V4rVqaMCdpJku0FozIf5n+RAAzRWI7kKJdqPUltNtTT+CN6jKtpMu0Em3Yw/z+N6nvfZ5a+y4qBw9ROXiI9zT9O82WCzleeg2NBZcQVuewqO8Vqpz7Car0vDX3CymPQRifTqNKKrkyWhMmk1uqS/L13LiqMunuuQtK87AYtNg9wYkPHsfcYgPLKid/iSSaHzPVagoN1FeYONblTPixyyrN5Opm5+wLJF4LplTkv0x7IoCZImdqwERmYGyqIgIhGZ1GRUl++n6RgmoDx0uv4XjpNRgCfSzue5l664uUuY8xf+At5g+8hV9t5GTR5cyxvwPArupP4NKXp20MqdBrVeg16hE5Q9ng0kXFtNm8NPW5kz7H0sqJt06P99hMBTAWg5b3raka0TE7UZIUmR1443jyxRZ1GhVX1Cdf52i6eM+iEpr73QnVhlFJEmvnzN7ZF4gsAaokKe6EdzEDM/2JSrxTJNpGILqFukuJVMmstuROWnVbj66YfZUf4bHVj/D7NX/mnepP4tBXoA+7WW59JtbvaG/lRyfl+sm4YF4RG+dNvLtlKhn1alZVW7hxVSWrapJ79y9JsDqFyrTRmjBTrdCo45Z11XFtTZ7IskpTSkHQxQuKZ0XOQq5OzcULihN6zOLy/JQaGs4EKpWEKTf+n1MRwEx/IoCZAt5AGF9wZA2YlmDkhTDV/Jd42Qx1bJ/zeX637kkeX/H/OFj2PnoNC3l5wXdi/Y4yrThPx+pqC/UV+VnVzXhNbQEatQqVSuLyJWW8Z3FJwkWK5xYZ4946PZapXtKoskRqpaQraNCqVayqTi4ArCrIZWWSj52OlleZ4/7bIEmRBGAh/mWkPL1mUvOohKkhApgpsK9tIPZ5NIBp9EX63FSnIf8lIZKKTtNqXlvwTf5vzWO0W9ZP7fXH8Z5FpahUEpIkcUGWzMLotapzXjjX1hZww6rKhHbBxFu4bjz1FaYp60W1oDSP969NbdloNKtqLGgSTALWqiXeW182qyrLAlxRXxbX7sS6YiNFeSKfA8CSG9+bBJH/MjOIAGaSeQNh9rXaY19Ha8B0hi3kaFUUZ9FMQyYtKM2jdtj24oWleWnNDUrWqmrLqHkr80vyuHVddVzv4gqNOuYkuHV6NNGaMJNtda2F61dWpLwteDRGvYbF5Yk1qdw4vyjl2avpKFIbZuKZlfPq/n97dx7c5HXvDfz7aLeszZItyfKGF2xjwAZMMM5CcoMvy+TmksBNKelNaZIhEwqdNjSZhE4bIJ0pGTqTbkObe9+2Ye5MGxI6pXmbpn0TSKBvExOKgbIlFBwSm+AFDHi3ZUvn/iEsIrxbj/Ro+X5mNBg9j6RzfDzST+c55/dLnqKN47FOcAaGl48SAwOYCDv86VV4B29ufx3Kwtsi7Mi2GZPuW+VItGoJi25JjCZJEm6fQKbXSNKoJMzNtY163GUxYPWCHKSPE2hNZev0aMoyI3cZRZKAu6an419KnBH9u6zMS5vwJTi31YC5k0j8l2gWTLMjbYwP5ey0FGRao3MZOh5MdCs1A5jEwAAmgjr7BnCi8frNO4QfqQOBujbNIg3Zdr7xAMC8vLQRFyAWZJjgsSn3RjMzywKjbuwZFotBiy/Nz8a09JFnWHQaFcpkXLsylBNGbmqVhGWz3Jg/LfLf5h0mPaY5UifUpn8tc00p70yi0KjH3nl1WxTGK55MNJkdSwgkBgYwEXT4wtWQ2ibGgatQCx98QsJl2GTJ/xLvzAbNmG/CtxdObjeGXFSShMrciX046DVqrKjIGnGR6UyPRdaMsUM5YeSk16rw4NwslLqjt0i4cgJbfhfk25HOtR3B3DC3clr0mJY+fiCYTKwp2nFn98wGzbhfTCg+MICJkOs9Xpy+FJqMamgB7xVYodfpxpwaThaLijPGTI6WYzcixx79QK/YZZrw9XQgsIVz8QwXFhWnB99AJ1N1ejLkrDRsNmjwUGVO1H/HOXbjmAsp0816zi58waLi9GELqufn8fdzK7VKgnmcXXO8fJQ4GMBEyKFP2uC7pbLs0PqXZmFHdlpK0q9/ybEbUTyB2YQ7iqK/Fmaql1Iq8+z4t3IPtGoJ+empk67PMhFy5YRJN+nwpdtyFFssPW+UtPcqScKSsontwEkWRp0Gd02/ORtpM2ox3WlSsEWxa7x1MAxgEgcDmAi40tWPj5s7h90/NAPTItKS/vKRSpJwT8nEKhpnWlNQkBG9qfL89NSwPtSLnCY8ND8noutJws0Jk52Wgofmy5fjZSpKXGaYDcOn8ivz0vghM4KZHguybuSGmZ9nT+q1QWMZLxcM178kDgYwEfBBfRtGymadcssMTDIrz7ZOan1DdaFj0snjpmoiW1fH47IYkGWL3BiHkxOmxG0OuzSAHFQj7PKyp+qwsICXRkYiSRIWlzphSdFiRqa866ASyfgBDIPjRMEARmbN7X2ob+0a8Zi6K1CN95rGkdRpv1N0alRPcou002zAdGfk37SzbCnRTy44BZPJCaOSJGSY9ZidZcXSmW4sn+WOSI6XqZiVZQ0ucpYkoKbMFTNti0UOkx7/UZnN39EYrGMks7OmaBUP3Ek+XIots/fPXxn1mL43MAMzaHRHLaNqLLq90DGlN5HqQgfOt3ZNuFjbVFTGUUr2skwrPr3SM+x+S4oWbosBbqsebmsKnGb9lKtIR5peo8bsLCvqPruGihxbRGetEkUyf/mZiLFmYDj7klgYwMio8WoPGq4O/0AZYh4IrIFRWTzRalLMcVoCMwFTYU/VoTTTjDO37O6SS7pJh4I42pZamJEKa4oWNmMgYHFZDXBbDLIUXoymObk2fHK5C3cotGWeEsvQVuqRvudw/Utiia93uhj3Qf3osy/9gz6k+9sACdDbs9AfxXbFknvCzPK6MN+Bs82dw3Z4yaEyzx5XO8M0ahUeuzNf6WaEzWLQ4qH5ObLmy6HkpVWrYNJr0Nk3OOwYZ2ASC98xZPLJ5S5cut436vHLV6/DKgVmZ4Q5OWdgSt3msC8RWI1azMqSP+GaJUWL0knW6CH5xNusEcW2kS6zSRKLOCYaBjAyEELgg/q2Mc/pa2sEAPTCAK86fi5TyEWnUeGu4oltmx7PgnwHtGp5Z0oq89K4LZUoQYyUf8mWoh2xMCvFLwYwMvhnSxcud459UWiw/RIAoF2bjqjtB44ht02zT6hy80SY9BqUZ9tkeS4AMOrUmCljdlsiUtZIC3l5+SjxMIAJk98vUDvG2hcA6BvwwdDbEvjZMHphtkRlM2oxb4yqzlNx2zS7bGsm5uTYYnaXDhFN3kjZeJ0MYBIO37XDdKapA9d6BsY85+K1XrilawCAXoMzGs2KKXcXZ8ietyJFp8ZcGeoM6TQqVESgXhERKWekOmbcgZR4GMCEYdDnx6FPxl77AgAXr/XALV0FAHTp5FkHEi8KMlJRkBGZmi3z8tLCTko1O8vKxFZECcZ2SzI7SQokw6TEInsAs3XrVkiSFHIrLS0NHu/r68OGDRvgcDhgMpmwatUqtLS0hDxHQ0MD7rvvPhiNRjidTjzzzDMYHBy+JU5pJz9vH3Gr3q0uXuuF68YMTJcueWZg5uba8G/lkdtxZdCqUZk39cRzapWEeWE8nohik06jQqr+5hcTe6qO2/QTUET2Ls6cORP79u27+SKamy/z1FNP4U9/+hP27NkDq9WKjRs3YuXKlXj//fcBAD6fD/fddx/cbjc++OADNDU14atf/Sq0Wi1+8IMfRKK5U+Id9OPvn14d97zOvgG0dXvh1t2YgdEn/gyMQavGkpkuFEZo5uWL5uTYcLzxGrr7fZN+7IxMi2wLi4kotthSdOju7wXA2ZdEFZF3b41GA7fbPez+9vZ2/OpXv8Jvf/tb3HvvvQCAV155BTNmzMChQ4ewcOFCvP322zhz5gz27dsHl8uFOXPm4Pvf/z6effZZbN26FTrd6HUuoul44/UJfWie/LwdAOBRBf5N9BmYLFsKls12R63KsU6jwr2lTpy+1IErXV509g2MmIHzVpIEzOfsC1HCshq1+Px6IIDh+pfEFJEA5ty5c/B4PDAYDKiursb27duRm5uLuro6DAwMoKamJnhuaWkpcnNzUVtbi4ULF6K2thazZ8+Gy3Vzt87SpUuxfv16nD59GnPnzh3xNfv7+9Hff3Mrc0dHZNLNA4FdRUc+G3/2ZdDnx6nPOyDBjwwMrYGJTLr0ErcZbd1eXBlnO3ekSFJgZ1B1gSPq+VSKnGYU3Sj06B3042q3F1e6+tHW7cXV7n60dXmHXeorcpqQlhobwTARye+LO5G4hToxyR7AVFVVYdeuXSgpKUFTUxO2bduGu+66C6dOnUJzczN0Oh1sNlvIY1wuF5qbmwEAzc3NIcHL0PGhY6PZvn07tm3bJm9nRnG88Tr6B/zjnvfP1i70DviQp++BGj4ISOjRyh/A6DQqLCrOwKdXuvHOmZbxHyCzVL0aS2e6kedQPkGfTqOC22qA2xr6htU34MPVbi/aurxo6+5HGfO+ECW0oWR2Q9XYKfHIHsAsX748+HN5eTmqqqqQl5eH119/HSkpkas0u3nzZmzatCn4/46ODuTk5ETktSaycFcIgX80XgcA3On0ApeBHq0dfpX8k163Fzpg0mtQ4jbj/5+7gr6Bya8HmapcuxHLZrljPhW8QauGx5YCD6sdEyWFoWR2dpOOeZ4SVMRH1Wazobi4GOfPn4fb7YbX68X169dDzmlpaQmumXG73cN2JQ39f6R1NUP0ej0sFkvITUlN7X1o7eyHWiWhwha4DhuJ9S9Oix4VN7LSatWqiNQJGolKknB7oQMr52XFfPBCRMlnqB6Si7MvCSviAUxXVxfq6+uRmZmJyspKaLVa7N+/P3j87NmzaGhoQHV1NQCguroaJ0+eRGtra/Ccd955BxaLBWVlZZFurmyGZl9K3WY4fIFcMXLvQJIk4N5SZ8iak/JsG1QRLlVgNmiwqjILVQWOuKreTETJw6BVI0Wn5vqXBCb7V+enn34a999/P/Ly8nDp0iVs2bIFarUaa9asgdVqxeOPP45NmzbBbrfDYrHgG9/4Bqqrq7Fw4UIAwJIlS1BWVoZHHnkEO3bsQHNzM7773e9iw4YN0OvjI5Lu6hvEuctdAICKbBtSrwaCMbmT2M3OsiLTGnpJxJqiRX5GKupbu2R9rSEFGalYUuZGio7J34gottlStAxgEpjsAczFixexZs0atLW1ISMjA3feeScOHTqEjIzAh/ePfvQjqFQqrFq1Cv39/Vi6dCl+/vOfBx+vVqvx5ptvYv369aiurkZqairWrl2LF154Qe6mRszJz9shRGBLcYZZD1PzZQDyXkIy6tS4o2jkBcFzc2wRCWAyrQb8e4WHsy5EFBfsqTou4E1gsgcwu3fvHvO4wWDAzp07sXPnzlHPycvLw1tvvSV306Ji0OcP5n6pyLYCAEzeoQBGvhmYO6enj5oCP8duRLpJhytdXtleb+g1GbwQUbwodJqgjnJaB4oeLs2W2bkbW6dNek0wE63Je+MSkl6eGZistBTM9FjHPEfuAoUFGanITjPK+pxERJE0LQZSO1DkMICRkRACx28s3i3PtgYX15q8VwDIMwOjVklYXDp+IDQj0yJbkUJJAm4vjEwCPiKiSOHsS2JjACOj5o6bW6dn3ZghUfv6YBgMZAWWYw3MvNw0OEzjX9PVqlWYKVOytlK3hdeRiYgopjCAkdHQ7EuJyxzcpZPb/ncAQJ/aDK86vOlMS4oWVQX2CZ9fkW1DuEtWNCoJtxc5wnsSIiIimTGAkUlX/yDO39j5U5FzY32KEKhu+D8AgFPuBxBuNHF3ccakMkpajVrkp4cXNJXn2KJWmJGIiGiiGMDI5OTFdvgF4LEagqXbC67+Fa7uj+BVpeBI1iNhPX9BRiqKnKZJP25uztQrLuu1KlTlT3zGh4iIKFoYwMhg0H9z6/Scod0/QqC68b8BAMczv4Re7dQDCa1awj0lU1s/k+swwmGaWtXl+Xl22RYCExERyYkBjAzOtdzcOl1wY+t04dUDcHb/E16VEXVZ/xnW81cVOIJ1PaZiqFbSZJj0GszNnfzjiIiIooEBTJi+uHV6drY1sG1P+INrX455vow+rW3Kz+8w6TAvd+qzN0BgS7VeO7mhXljgYAVXIiKKWfyEClPo1unAtuWitgPI6DmHfnUq6jwPh/X8/1LiDDuXgU6jQlnmxLdU21N1sm3BJiIiigQGMGEamn0pdplg1GkCsy831r4c86xBv3bsjLljmZFpQY5dnuy3c3ImvqX6jiJHSIVrIiKiWMMAJgxf3Do958Y6k+K2/UjvqUef2oSjN2ZfJClQ2n0yMykGrRqLiuXLfmsz6ia0pTrTakCR0yzb6xIREUWC7MUck8nJzwNbpzOtBjgtBkjCh4XBtS8Po18TCATy01OxYk4WAMDvF/D6/Bjw+THoExjw+THgFxgY9GPQ74d3UGDQ70eaUReY0ZFRRbYNn1zuHvOcO6ezZAAREcU+BjBTNOj34+TF0K3TxVf2wdF7AX1qM4561gTPLXXfXE+iUkkwqNSKbE/OcxhhT9XhavfIVapZsJGIiOIFLyFN0fmW0KrTkvChqvGXAIC6rK/Aqwlsp9ZpVCjMiI2KqJIkjVqlmgUbiYgonjCAmaLjF68DAGZnBbZOl1x+G47eT9GrseJ45urgedOdJmhiaDvyjEwzdJrh7WHBRiIiiiex88kaRz653IWWjn6oJQmzsiyQxCAWjjD7AgR2EsUSvUaNslu2SLNgIxERxRsGMFOw/+NWAECxO7B1uvTy/0NaX0Ng9sX9peB5ZoMG2WkpSjVzVHNuqVLNgo1ERBRvGMBMUmtHH+o+uwYgsKvni7MvR7IewYDm5nqXUrcFUpgVqCMhLVWHPEdgsS4LNhIRUTxiADNJv/mwAT6/QKbVAJfFgBmtb8HWdxE92jT8I/OhkHNnZMZuPpU5N6pUs2AjERHFIwYwk3S0ITD7MifHBpV/EAsbfwXgxuyL+uYWZKdFD4cpdhfFTnMYkZ2WwoKNREQUl5gHZpL+57EF2PlePfoGfJhx+f/C2n8J3Vo7/uH+j5Dzvpj7JRZJkoR/n+NhwUYiIopL/PSaJEmSUOI2Q4tBVN2Yffl71loMqm8u1lVJEkrdsXv5aIhew0tHREQUnxjATNHM1j/C2t+Ebq0DJ9wrQ47lOlKQqufkFhERUaQwgJkCyefFgouvAAD+nr0WPrUh5His5X4hIiJKNAxgpiDrwu9g6W9Gly4DJ1wPhhwLlA4wjfJIIiIikgMDmMka7Ef+Ry8DAA5nf23Y7EthhokLY4mIiCKMn7STdfR/YOhtRqfOiVOuFcMOl/HyERERUcQxgJkMIYDjvwUAHM5+FD5VaJ4Xs0GDHHvslQ4gIiJKNNwqMxmSBDz6Fj7683/htL5m2OFilzkmSwcQERElGs7ATJY2BReL1sCn0g07xN1HRERE0cEARibpZj0yzLFbOoCIiCiRMICRSVkMF24kIiJKNAxgZCBJQEmM1z4iIiJKJAxgZJCTZoSJpQOIiIiihgGMDLh4l4iIKLoYwIRJq5ZQ5GTpACIiomhiABOmwgwTdBr+GomIiKKJn7xh4uUjIiKi6GMAE4ZUvRq5dqPSzSAiIko6DGDCUOK2QKVi6QAiIqJoYwAThhluJq8jIiJSAgOYKXKYdHBaDEo3g4iIKCkxgJkiLt4lIiJSDgOYKZAAlPDyERERkWIYwExBtj0FFoNW6WYQERElLQYwU1CYwcy7RERESmIAMwVaNX9tRERESuInMREREcUdBjBEREQUdxjAEBERUdxhAENERERxhwEMERERxR0GMERERBR3GMAQERFR3InpAGbnzp2YNm0aDAYDqqqqcPjwYaWbRERERDEgZgOY1157DZs2bcKWLVtw9OhRVFRUYOnSpWhtbVW6aURERKSwmA1gXnrpJaxbtw6PPvooysrK8PLLL8NoNOLXv/610k0jIiIihcVkAOP1elFXV4eamprgfSqVCjU1NaitrR3xMf39/ejo6Ai5ERERUWKKyQDmypUr8Pl8cLlcIfe7XC40NzeP+Jjt27fDarUGbzk5OdFoKhERESkgJgOYqdi8eTPa29uDt8bGRqWbRERERBGiUboBI0lPT4darUZLS0vI/S0tLXC73SM+Rq/XQ6/XR6N5REREpLCYDGB0Oh0qKyuxf/9+PPDAAwAAv9+P/fv3Y+PGjRN6DiEEAHAtDBERURwZ+twe+hwfTUwGMACwadMmrF27FvPnz8eCBQvw4x//GN3d3Xj00Ucn9PjOzk4A4FoYIiKiONTZ2Qmr1Trq8ZgNYFavXo3Lly/j+eefR3NzM+bMmYO//OUvwxb2jsbj8aCxsRFmsxmSJMnWro6ODuTk5KCxsREWi0W2540H7Hvy9T1Z+w2w78nY92TtNxBbfRdCoLOzEx6PZ8zzJDHeHA2F6OjogNVqRXt7u+KDHG3se/L1PVn7DbDvydj3ZO03EJ99T5hdSERERJQ8GMAQERFR3GEAM0l6vR5btmxJyi3b7Hvy9T1Z+w2w78nY92TtNxCffecaGCIiIoo7nIEhIiKiuMMAhoiIiOIOAxgiIiKKOwxgiIiIKO4wgJmknTt3Ytq0aTAYDKiqqsLhw4eVblLEbd26FZIkhdxKS0uVbpbs/vrXv+L++++Hx+OBJEn4wx/+EHJcCIHnn38emZmZSElJQU1NDc6dO6dMY2U2Xt+/9rWvDfsbWLZsmTKNldH27dtx2223wWw2w+l04oEHHsDZs2dDzunr68OGDRvgcDhgMpmwatWqYYVm49FE+n7PPfcMG/cnn3xSoRbL4xe/+AXKy8thsVhgsVhQXV2NP//5z8HjiTrewPh9j7fxZgAzCa+99ho2bdqELVu24OjRo6ioqMDSpUvR2tqqdNMibubMmWhqagre/va3vyndJNl1d3ejoqICO3fuHPH4jh078NOf/hQvv/wyPvzwQ6SmpmLp0qXo6+uLckvlN17fAWDZsmUhfwOvvvpqFFsYGQcPHsSGDRtw6NAhvPPOOxgYGMCSJUvQ3d0dPOepp57CH//4R+zZswcHDx7EpUuXsHLlSgVbLY+J9B0A1q1bFzLuO3bsUKjF8sjOzsaLL76Iuro6HDlyBPfeey9WrFiB06dPA0jc8QbG7zsQZ+MtaMIWLFggNmzYEPy/z+cTHo9HbN++XcFWRd6WLVtERUWF0s2IKgBi7969wf/7/X7hdrvFD3/4w+B9169fF3q9Xrz66qsKtDBybu27EEKsXbtWrFixQpH2RFNra6sAIA4ePCiECIyxVqsVe/bsCZ7z0UcfCQCitrZWqWZGxK19F0KIu+++W3zzm99UrlFRkpaWJn75y18m1XgPGeq7EPE33pyBmSCv14u6ujrU1NQE71OpVKipqUFtba2CLYuOc+fOwePxoKCgAF/5ylfQ0NCgdJOi6sKFC2hubg4Zf6vViqqqqqQYfwA4cOAAnE4nSkpKsH79erS1tSndJNm1t7cDAOx2OwCgrq4OAwMDIeNeWlqK3NzchBv3W/s+5De/+Q3S09Mxa9YsbN68GT09PUo0LyJ8Ph92796N7u5uVFdXJ9V439r3IfE03jFbjTrWXLlyBT6fb1g1bJfLhY8//lihVkVHVVUVdu3ahZKSEjQ1NWHbtm246667cOrUKZjNZqWbFxXNzc0AMOL4Dx1LZMuWLcPKlSuRn5+P+vp6fOc738Hy5ctRW1sLtVqtdPNk4ff78a1vfQt33HEHZs2aBSAw7jqdDjabLeTcRBv3kfoOAA8//DDy8vLg8Xhw4sQJPPvsszh79ix+//vfK9ja8J08eRLV1dXo6+uDyWTC3r17UVZWhuPHjyf8eI/WdyD+xpsBDI1r+fLlwZ/Ly8tRVVWFvLw8vP7663j88ccVbBlFy5e//OXgz7Nnz0Z5eTkKCwtx4MABLF68WMGWyWfDhg04depUQq7vGs9ofX/iiSeCP8+ePRuZmZlYvHgx6uvrUVhYGO1myqakpATHjx9He3s7fve732Ht2rU4ePCg0s2KitH6XlZWFnfjzUtIE5Seng61Wj1sNXpLSwvcbrdCrVKGzWZDcXExzp8/r3RTomZojDn+AQUFBUhPT0+Yv4GNGzfizTffxHvvvYfs7Ozg/W63G16vF9evXw85P5HGfbS+j6SqqgoA4n7cdTodioqKUFlZie3bt6OiogI/+clPkmK8R+v7SGJ9vBnATJBOp0NlZSX2798fvM/v92P//v0h1w+TQVdXF+rr65GZmal0U6ImPz8fbrc7ZPw7Ojrw4YcfJt34A8DFixfR1tYW938DQghs3LgRe/fuxbvvvov8/PyQ45WVldBqtSHjfvbsWTQ0NMT9uI/X95EcP34cAOJ+3G/l9/vR39+f0OM9mqG+jyTmx1vpVcTxZPfu3UKv14tdu3aJM2fOiCeeeELYbDbR3NysdNMi6tvf/rY4cOCAuHDhgnj//fdFTU2NSE9PF62trUo3TVadnZ3i2LFj4tixYwKAeOmll8SxY8fEZ599JoQQ4sUXXxQ2m0288cYb4sSJE2LFihUiPz9f9Pb2Ktzy8I3V987OTvH000+L2tpaceHCBbFv3z4xb948MX36dNHX16d008Oyfv16YbVaxYEDB0RTU1Pw1tPTEzznySefFLm5ueLdd98VR44cEdXV1aK6ulrBVstjvL6fP39evPDCC+LIkSPiwoUL4o033hAFBQVi0aJFCrc8PM8995w4ePCguHDhgjhx4oR47rnnhCRJ4u233xZCJO54CzF23+NxvBnATNLPfvYzkZubK3Q6nViwYIE4dOiQ0k2KuNWrV4vMzEyh0+lEVlaWWL16tTh//rzSzZLde++9JwAMu61du1YIEdhK/b3vfU+4XC6h1+vF4sWLxdmzZ5VttEzG6ntPT49YsmSJyMjIEFqtVuTl5Yl169YlROA+Up8BiFdeeSV4Tm9vr/j6178u0tLShNFoFA8++KBoampSrtEyGa/vDQ0NYtGiRcJutwu9Xi+KiorEM888I9rb25VteJgee+wxkZeXJ3Q6ncjIyBCLFy8OBi9CJO54CzF23+NxvCUhhIjefA8RERFR+LgGhoiIiOIOAxgiIiKKOwxgiIiIKO4wgCEiIqK4wwCGiIiI4g4DGCIiIoo7DGCIiIgo7jCAISIiorjDAIaIiIjiDgMYIiIiijsMYIiIiCjuMIAhIiKiuPO/YMZDeA55PwAAAAAASUVORK5CYII=\n" + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "draw_result(\n", + " task_name=\"ParallelContinuousCartPoleSwingUp-v0\",\n", + " log_file=\"eval.csv\",\n", + " log_key=\"mean_reward\",\n", + " params=dict(freq_rate=1,\n", + " real_time_scale=0.02,\n", + " integrator=\"euler\",\n", + " parallel_num=3),\n", + " group_key=('transition', 'oracle'),\n", + ")" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.13" + } + }, + "nbformat": 4, + "nbformat_minor": 1 +} \ No newline at end of file diff --git a/mkdocs.yml b/mkdocs.yml new file mode 100644 index 0000000..716cbbc --- /dev/null +++ b/mkdocs.yml @@ -0,0 +1,11 @@ +site_name: Causal MBRL +theme: readthedocs + +plugins: + - search # necessary for search to work + - mkdocstrings + + +nav: + - Home: index.md + - About: about.md diff --git a/requirements/dev.txt b/requirements/dev.txt index 2bf5b67..ae90ea9 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -2,3 +2,7 @@ pre-commit>=2.20.0 pytest>=7.1.3 pytest-cov>=4.0.0 flake8>=5.0.4 +mkdocs>=1.4.1 +Pygments>=2.13.0 +# mkdocstrings>=0.19.0 +# mkdocstrings-python>=1.0.14 diff --git a/requirements/main.txt b/requirements/main.txt index 4d7f2d4..4fbd491 100644 --- a/requirements/main.txt +++ b/requirements/main.txt @@ -8,6 +8,5 @@ imageio>=2.19.0 tensorboard>=2.9.0 mujoco >= 2.2.0 wandb >= 0.13 -emei @ git+https://github.com/FrankTianTT/emei@dev -stable-baselines3 @ git+https://github.com/carlosluis/stable-baselines3@fix_tests -PyQt5>=5.15.7 +stable-baselines3 @ git+https://gitee.com/franktian424/stable-baselines3 +causal-learn >= 0.1.3.3 \ No newline at end of file diff --git a/setup.py b/setup.py index de81688..b895d12 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,6 @@ from setuptools import find_packages, setup - def parse_requirements_file(path): return [line.rstrip() for line in open(path, "r")] @@ -25,7 +24,7 @@ def parse_requirements_file(path): long_description=long_description, long_description_content_type="text/markdown", url="https://github.com/FrankTianTT/causal-mbrl", - packages=find_packages(), + packages=[package for package in find_packages() if package.startswith("cmrl")], classifiers=[ "License :: OSI Approved :: MIT License", "Programming Language :: Python :: 3", diff --git a/tests/__init__.py b/tests/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/tests/constants.py b/tests/constants.py deleted file mode 100644 index 4ae3190..0000000 --- a/tests/constants.py +++ /dev/null @@ -1,125 +0,0 @@ -from omegaconf import DictConfig - -cfg = DictConfig( - { - "algorithm": { - "name": "off_dyna", - "freq_train_model": "${task.freq_train_model}", - "real_data_ratio": 0.0, - "sac_samples_action": True, - "num_eval_episodes": 5, - "dataset_size": 1000000, - "penalty_coeff": "${task.penalty_coeff}", - "agent": { - "_target_": "stable_baselines3.sac.SAC", - "policy": "MlpPolicy", - "env": { - "_target_": "cmrl.models.fake_env.VecFakeEnv", - "num_envs": 16, - "action_space": {"_target_": "gym.spaces.Box", "low": "???", "high": "???", "shape": "???"}, - "observation_space": {"_target_": "gym.spaces.Box", "low": "???", "high": "???", "shape": "???"}, - }, - "learning_starts": 0, - "batch_size": 256, - "tau": 0.005, - "gamma": 0.99, - "ent_coef": "auto", - "target_entropy": "auto", - "verbose": 0, - "seed": "${seed}", - "device": "${device}", - }, - }, - "dynamics": { - "name": "constraint_based_dynamics", - "multi_step": "${task.multi_step}", - "transition": { - "_target_": "cmrl.models.transition.ExternalMaskTransition", - "obs_size": "???", - "action_size": "???", - "deterministic": False, - "ensemble_num": "${task.ensemble_num}", - "elite_num": "${task.elite_num}", - "residual": True, - "learn_logvar_bounds": False, - "num_layers": 4, - "hid_size": 200, - "activation_fn_cfg": {"_target_": "torch.nn.SiLU"}, - "device": "${device}", - }, - "learned_reward": "${task.learning_reward}", - "reward_mech": {"_target_": "cmrl.models.reward_mech.BaseRewardMech", "obs_size": "???", "action_size": "???"}, - "learned_termination": "${task.learning_terminal}", - "termination_mech": { - "_target_": "cmrl.models.termination_mech.BaseTerminationMech", - "obs_size": "???", - "action_size": "???", - }, - "optim_lr": "${task.optim_lr}", - "weight_decay": "${task.weight_decay}", - "patience": "${task.patience}", - "batch_size": "${task.batch_size}", - "use_ratio": "${task.use_ratio}", - "validation_ratio": "${task.validation_ratio}", - "shuffle_each_epoch": "${task.shuffle_each_epoch}", - "bootstrap_permutes": "${task.bootstrap_permutes}", - "longest_epoch": "${task.longest_epoch}", - "improvement_threshold": "${task.improvement_threshold}", - "normalize": True, - "normalize_double_precision": True, - }, - "task": { - "env": "emei___BoundaryInvertedPendulumSwingUp-v0___" - "freq_rate=${task.freq_rate}&time_step=${task.time_step}___${task.dataset}", - "dataset": "SAC-expert-replay", - "freq_rate": 1, - "time_step": 0.02, - "num_steps": 1000, - "online_num_steps": 10000, - "epoch_length": 10000, - "n_eval_episodes": 8, - "eval_freq": 100, - "learning_reward": False, - "learning_terminal": False, - "ensemble_num": 7, - "elite_num": 5, - "multi_step": "none", - "oracle": True, - "cit_threshold": 0.02, - "test_freq": 100, - "update_causal_mask_ratio": 0.25, - "discovery_schedule": [1, 30, 250, 250], - "penalty_coeff": 0.5, - "use_ratio": 0.01, - "freq_train_model": 100, - "patience": 20, - "optim_lr": 0.0001, - "weight_decay": 1e-05, - "batch_size": 256, - "validation_ratio": 0.2, - "shuffle_each_epoch": True, - "bootstrap_permutes": False, - "longest_epoch": 10, - "improvement_threshold": 0.01, - "effective_model_rollouts_per_step": 50, - "rollout_schedule": [1, 15, 1, 1], - "num_sac_updates_per_step": 1, - "sac_updates_every_steps": 1, - "num_epochs_to_retain_sac_buffer": 1, - "sac_gamma": 0.99, - "sac_tau": 0.005, - "sac_alpha": 0.2, - "sac_policy": "Gaussian", - "sac_target_update_interval": 1, - "sac_automatic_entropy_tuning": True, - "sac_hidden_size": 256, - "sac_lr": 0.0003, - "sac_batch_size": 256, - "sac_target_entropy": -1, - }, - "seed": 0, - "device": "cpu", - "exp_name": "default", - "wandb": False, - } -) diff --git a/tests/test_algorithms/__init__.py b/tests/test_algorithms/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/tests/test_algorithms/test_offline/__init__.py b/tests/test_algorithms/test_offline/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/tests/test_algorithms/test_offline/test_off_dyna.py b/tests/test_algorithms/test_offline/test_off_dyna.py deleted file mode 100644 index 48dc7de..0000000 --- a/tests/test_algorithms/test_offline/test_off_dyna.py +++ /dev/null @@ -1,16 +0,0 @@ -import numpy as np -import torch - -from cmrl.algorithms.offline.off_dyna import train -from cmrl.util.env import make_env - -from tests.constants import cfg - - -def test_off_dyna(): - env, term_fn, reward_fn, init_obs_fn = make_env(cfg) - test_env, *_ = make_env(cfg) - np.random.seed(cfg.seed) - torch.manual_seed(cfg.seed) - - train(env, test_env, term_fn, reward_fn, init_obs_fn, cfg) diff --git a/tests/test_algorithms/test_util.py b/tests/test_algorithms/test_util.py deleted file mode 100644 index 0a4da4e..0000000 --- a/tests/test_algorithms/test_util.py +++ /dev/null @@ -1,15 +0,0 @@ -from stable_baselines3.common.buffers import ReplayBuffer - -from cmrl.util.env import make_env -from cmrl.algorithms.util import load_offline_data - -from tests.constants import cfg - - -def test_load_offline_data(): - env, term_fn, reward_fn, init_obs_fn = make_env(cfg) - replay_buffer = ReplayBuffer( - cfg.task.num_steps, env.observation_space, env.action_space, cfg.device, handle_timeout_termination=False - ) - - load_offline_data(cfg, env, replay_buffer) diff --git a/tests/test_diagnostics.py b/tests/test_diagnostics.py deleted file mode 100644 index e69de29..0000000 diff --git a/cmrl/models/causal_discovery/__init__.py b/tests/test_diagnostics/__init__.py similarity index 100% rename from cmrl/models/causal_discovery/__init__.py rename to tests/test_diagnostics/__init__.py diff --git a/tests/test_diagnostics/test_base_diagnostics.py b/tests/test_diagnostics/test_base_diagnostics.py new file mode 100644 index 0000000..92f8b3f --- /dev/null +++ b/tests/test_diagnostics/test_base_diagnostics.py @@ -0,0 +1,5 @@ +from cmrl.diagnostics.base_diagnostic import BaseDiagnostic + + +def test_base_diagnostics(): + pass diff --git a/tests/test_examples/__init__.py b/tests/test_examples/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/tests/test_examples/test_main.py b/tests/test_examples/test_main.py deleted file mode 100644 index e69de29..0000000 diff --git a/tests/test_models/test_causal_discovery/__init__.py b/tests/test_models/test_causal_discovery/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/tests/test_models/test_causal_discovery/test_CMI_test.py b/tests/test_models/test_causal_discovery/test_CMI_test.py deleted file mode 100644 index 40064eb..0000000 --- a/tests/test_models/test_causal_discovery/test_CMI_test.py +++ /dev/null @@ -1,73 +0,0 @@ -import tempfile -from pathlib import Path -from unittest import TestCase - -import torch - -from cmrl.models.causal_discovery.CMI_test import TransitionConditionalMutualInformationTest - - -class TestTransitionConditionalMutualInformationTest(TestCase): - def setUp(self) -> None: - self.obs_size = 11 - self.action_size = 3 - self.num_layers = 4 - self.ensemble_num = 7 - self.hid_size = 200 - self.batch_size = 128 - self.device = "cuda" if torch.cuda.is_available() else "cpu" - - self.cmi_test = TransitionConditionalMutualInformationTest( - obs_size=self.obs_size, - action_size=self.action_size, - device=self.device, - num_layers=self.num_layers, - ensemble_num=self.ensemble_num, - hid_size=self.hid_size, - ) - self.batch_obs = torch.rand([self.ensemble_num, self.batch_size, self.obs_size]).to(self.device) - self.batch_action = torch.rand([self.ensemble_num, self.batch_size, self.action_size]).to(self.device) - - def test_parallel_deterministic_forward(self): - parallel_batch_obs = torch.rand( - [self.obs_size + self.action_size + 1, self.ensemble_num, self.batch_size, self.obs_size] - ).to(self.device) - mean, logvar = self.cmi_test.forward(parallel_batch_obs, self.batch_action) - assert ( - mean.shape - == logvar.shape - == (self.obs_size + self.action_size + 1, self.ensemble_num, self.batch_size, self.obs_size) - ) - - def test_gaussian_forward(self): - mean, logvar = self.cmi_test.forward(self.batch_obs, self.batch_action) - assert ( - mean.shape - == logvar.shape - == (self.obs_size + self.action_size + 1, self.ensemble_num, self.batch_size, self.obs_size) - ) - - def test_load(self): - tempdir = Path(tempfile.gettempdir()) - model_dir = tempdir / "temp_model" - if not model_dir.exists(): - model_dir.mkdir() - - mean, logvar = self.cmi_test.forward(self.batch_obs, self.batch_action) - self.cmi_test.save(model_dir) - - new_cmi_test = TransitionConditionalMutualInformationTest( - obs_size=self.obs_size, - action_size=self.action_size, - device=self.device, - num_layers=self.num_layers, - ensemble_num=self.ensemble_num, - hid_size=self.hid_size, - ) - - new_mean, new_logvar = new_cmi_test.forward(self.batch_obs, self.batch_action) - assert not (mean == new_mean).all() - - new_cmi_test.load(model_dir) - new_mean, new_logvar = new_cmi_test.forward(self.batch_obs, self.batch_action) - assert (mean == new_mean).all() diff --git a/cmrl/models/transition/multi_step/__init__.py b/tests/test_models/test_causal_mech/__init__.py similarity index 100% rename from cmrl/models/transition/multi_step/__init__.py rename to tests/test_models/test_causal_mech/__init__.py diff --git a/tests/test_models/test_causal_mech/test_CMI_test.py b/tests/test_models/test_causal_mech/test_CMI_test.py new file mode 100644 index 0000000..1322f62 --- /dev/null +++ b/tests/test_models/test_causal_mech/test_CMI_test.py @@ -0,0 +1,111 @@ +import gym +from stable_baselines3.common.buffers import ReplayBuffer +import torch +from torch.utils.data import DataLoader + +from cmrl.models.causal_mech.CMI_test import CMITestMech +from cmrl.models.data_loader import EnsembleBufferDataset, EnsembleBufferDataset, collate_fn +from cmrl.utils.creator import parse_space +from cmrl.utils.env import load_offline_data +from cmrl.models.causal_mech.util import variable_loss_func + + +def prepare(freq_rate): + env = gym.make("BoundaryInvertedPendulumSwingUp-v0", freq_rate=freq_rate, time_step=0.02) + + real_replay_buffer = ReplayBuffer(1000, env.observation_space, env.action_space, "cpu", handle_timeout_termination=False) + load_offline_data(env, real_replay_buffer, "SAC-expert-replay", use_ratio=0.001) + + ensemble_num = 7 + # test for transition + train_dataset = EnsembleBufferDataset( + real_replay_buffer, + env.observation_space, + env.action_space, + training=False, + mech="transition", + train_ensemble=True, + ensemble_num=ensemble_num, + ) + train_loader = DataLoader(train_dataset, batch_size=8, collate_fn=collate_fn) + valid_dataset = EnsembleBufferDataset( + real_replay_buffer, env.observation_space, env.action_space, training=True, mech="transition", repeat=ensemble_num + ) + valid_loader = DataLoader(valid_dataset, batch_size=8, collate_fn=collate_fn) + + input_variables = parse_space(env.observation_space, "obs") + parse_space(env.action_space, "act") + output_variables = parse_space(env.observation_space, "next_obs") + + return input_variables, output_variables, train_loader, valid_loader + + +def test_mask(): + input_variables, output_variables, train_loader, valid_loader = prepare(freq_rate=1) + + mech = CMITestMech( + name="test", + input_variables=input_variables, + output_variables=output_variables, + ) + + for inputs, targets in train_loader: + batch_size, extra_dim = mech.get_inputs_batch_size(inputs) + + inputs_tensor = torch.zeros(mech.ensemble_num, batch_size, mech.input_var_num, mech.encoder_output_dim).to(mech.device) + for i, var in enumerate(mech.input_variables): + out = mech.variable_encoders[var.name](inputs[var.name].to(mech.device)) + inputs_tensor[:, :, i] = out + + mask = None + masked_inputs_tensor = mech.reduce_encoder_output(inputs_tensor, mask) + assert masked_inputs_tensor.shape == (mech.output_var_num, mech.ensemble_num, batch_size, mech.encoder_output_dim) + + mask = torch.ones(mech.ensemble_num, batch_size, mech.input_var_num).to(mech.device) + masked_inputs_tensor = mech.reduce_encoder_output(inputs_tensor, mask) + assert masked_inputs_tensor.shape == (mech.ensemble_num, batch_size, mech.encoder_output_dim) + + mask = torch.ones(mech.output_var_num, mech.ensemble_num, batch_size, mech.input_var_num).to(mech.device) + masked_inputs_tensor = mech.reduce_encoder_output(inputs_tensor, mask) + assert masked_inputs_tensor.shape == (mech.output_var_num, mech.ensemble_num, batch_size, mech.encoder_output_dim) + + mask = torch.ones(mech.input_var_num + 1, mech.output_var_num, mech.ensemble_num, batch_size, mech.input_var_num).to( + mech.device + ) + masked_inputs_tensor = mech.reduce_encoder_output(inputs_tensor, mask) + assert masked_inputs_tensor.shape == ( + mech.input_var_num + 1, + mech.output_var_num, + mech.ensemble_num, + batch_size, + mech.encoder_output_dim, + ) + + break + + +def test_CMI_forward(): + input_variables, output_variables, train_loader, valid_loader = prepare(freq_rate=1) + + mech = CMITestMech( + name="test", + input_variables=input_variables, + output_variables=output_variables, + ) + + for inputs, targets in train_loader: + outputs = mech.CMI_single_step_forward(inputs) + variable_loss_func(outputs, targets, output_variables) + + break + + +def test_forward(): + input_variables, output_variables, train_loader, valid_loader = prepare(freq_rate=1) + + mech = CMITestMech( + name="test", + input_variables=input_variables, + output_variables=output_variables, + ) + + mech.learn(train_loader, valid_loader, longest_epoch=10) diff --git a/tests/test_models/test_causal_mech/test_plain_mech.py b/tests/test_models/test_causal_mech/test_plain_mech.py new file mode 100644 index 0000000..dad59c3 --- /dev/null +++ b/tests/test_models/test_causal_mech/test_plain_mech.py @@ -0,0 +1,66 @@ +import gym +from stable_baselines3.common.buffers import ReplayBuffer +from torch.utils.data import DataLoader + +from cmrl.models.causal_mech.oracle_mech import OracleMech +from cmrl.models.data_loader import EnsembleBufferDataset, EnsembleBufferDataset, collate_fn +from cmrl.utils.creator import parse_space +from cmrl.utils.env import load_offline_data + + +def prepare(freq_rate): + env = gym.make("BoundaryInvertedPendulumSwingUp-v0", freq_rate=freq_rate, time_step=0.02) + + real_replay_buffer = ReplayBuffer(1000, env.observation_space, env.action_space, "cpu", handle_timeout_termination=False) + load_offline_data(env, real_replay_buffer, "SAC-expert-replay", use_ratio=0.001) + + ensemble_num = 7 + # test for transition + train_dataset = EnsembleBufferDataset( + real_replay_buffer, + env.observation_space, + env.action_space, + training=False, + mech="transition", + train_ensemble=True, + ensemble_num=ensemble_num, + ) + train_loader = DataLoader(train_dataset, batch_size=8, collate_fn=collate_fn) + valid_dataset = EnsembleBufferDataset( + real_replay_buffer, env.observation_space, env.action_space, training=True, mech="transition", repeat=ensemble_num + ) + valid_loader = DataLoader(valid_dataset, batch_size=8, collate_fn=collate_fn) + + input_variables = parse_space(env.observation_space, "obs") + parse_space(env.action_space, "act") + output_variables = parse_space(env.observation_space, "next_obs") + + return input_variables, output_variables, train_loader, valid_loader + + +def test_inv_pendulum_single_step(): + input_variables, output_variables, train_loader, valid_loader = prepare(freq_rate=1) + + mech = OracleMech( + name="test", + input_variables=input_variables, + output_variables=output_variables, + ) + + mech.learn(train_loader, valid_loader, longest_epoch=1) + + +def test_inv_pendulum_multi_step(): + input_variables, output_variables, train_loader, valid_loader = prepare(freq_rate=2) + + mech = OracleMech( + name="test", + input_variables=input_variables, + output_variables=output_variables, + multi_step="forward-euler 2", + ) + + mech.learn(train_loader, valid_loader, longest_epoch=1) + + +if __name__ == "__main__": + test_inv_pendulum_multi_step() diff --git a/tests/test_models/test_causal_mech/test_reinforce.py b/tests/test_models/test_causal_mech/test_reinforce.py new file mode 100644 index 0000000..414100e --- /dev/null +++ b/tests/test_models/test_causal_mech/test_reinforce.py @@ -0,0 +1,200 @@ +import gym +from stable_baselines3.common.buffers import ReplayBuffer +import torch +from torch.utils.data import DataLoader + +from cmrl.models.causal_mech.reinforce import ReinforceCausalMech +from cmrl.models.data_loader import EnsembleBufferDataset, EnsembleBufferDataset, collate_fn +from cmrl.utils.creator import parse_space +from cmrl.utils.env import load_offline_data +from cmrl.models.causal_mech.util import variable_loss_func + + +def prepare(freq_rate): + env = gym.make("BoundaryInvertedPendulumSwingUp-v0", freq_rate=freq_rate, time_step=0.02) + + real_replay_buffer = ReplayBuffer( + 1000, env.observation_space, env.action_space, device="cpu", handle_timeout_termination=False + ) + load_offline_data(env, real_replay_buffer, "SAC-expert-replay", use_ratio=0.001) + + ensemble_num = 7 + # test for transition + train_dataset = EnsembleBufferDataset( + real_replay_buffer, + env.observation_space, + env.action_space, + training=False, + mech="transition", + train_ensemble=True, + ensemble_num=ensemble_num, + ) + train_loader = DataLoader(train_dataset, batch_size=8, collate_fn=collate_fn) + valid_dataset = EnsembleBufferDataset( + real_replay_buffer, env.observation_space, env.action_space, training=True, mech="transition", repeat=ensemble_num + ) + valid_loader = DataLoader(valid_dataset, batch_size=8, collate_fn=collate_fn) + + input_variables = parse_space(env.observation_space, "obs") + parse_space(env.action_space, "act") + output_variables = parse_space(env.observation_space, "next_obs") + + return input_variables, output_variables, train_loader, valid_loader + + +def test_init(): + input_variables, output_variables, _, _ = prepare(freq_rate=1) + + mech = ReinforceCausalMech( + name="test", + input_variables=input_variables, + output_variables=output_variables, + ) + + assert mech.network, "network incorrectly initialized" + assert mech.graph, "graph incorrectly initialized" + assert mech.optimizer, "network optimizer incorrectly initialized" + assert mech.graph_optimizer, "graph optimizer incorrectly initialized" + + +def test_causal_graph(): + input_variables, output_variables, _, _ = prepare(freq_rate=1) + + mech = ReinforceCausalMech( + name="test", + input_variables=input_variables, + output_variables=output_variables, + ) + + assert mech.causal_graph is not None, "cannot obtain causal graph correctly" + assert mech.causal_graph.all(), "incorrect initial causal graph" + + +def test_single_step_forward(): + input_variables, output_variables, train_loader, _ = prepare(freq_rate=1) + + mech = ReinforceCausalMech( + name="test", + input_variables=input_variables, + output_variables=output_variables, + ) + train_iter = iter(train_loader) + inputs, targets = next(train_iter) + + # test training + outputs = mech.single_step_forward(inputs, train=True) + assert len(outputs.keys() ^ targets.keys()) == 0, "single forward output keys mismatch when training" + loss = variable_loss_func(outputs, targets, mech.output_variables) + assert loss is not None, "single forward output is incorrect when training" + + # test evaluating + outputs = mech.single_step_forward(inputs, train=False) + assert len(outputs.keys() ^ targets.keys()) == 0, "single forward output keys mismatch when evaluating" + loss = variable_loss_func(outputs, targets, mech.output_variables) + assert loss is not None, "single forward outupt is incorrect when evaluating" + + # test evaluating with fixed mask + extra_dims = next(iter(inputs.values())).shape[:-1] + mask = torch.randint(0, 2, size=(len(targets), *extra_dims, len(inputs))) + outputs = mech.single_step_forward(inputs, train=False, mask=mask) + assert len(outputs.keys() ^ targets.keys()) == 0, "single forward output keys mismatch when evaluating with given mask" + loss = variable_loss_func(outputs, targets, mech.output_variables) + assert loss is not None, "single forward output is incorrect when evaluating with given mask" + + +def test_forward(): + input_variables, output_variables, train_loader, _ = prepare(freq_rate=1) + + # test single step + mech = ReinforceCausalMech( + name="test", + input_variables=input_variables, + output_variables=output_variables, + ) + train_iter = iter(train_loader) + inputs, targets = next(train_iter) + + outputs = mech.forward(inputs, train=True) + assert len(outputs.keys() ^ targets.keys()) == 0, "forward output keys mismatch when evaluating (single step)" + loss = variable_loss_func(outputs, targets, mech.output_variables) + assert loss is not None, "forward outupt is incorrect when evaluating (single step)" + + # test multi-step + mech = ReinforceCausalMech( + name="test", input_variables=input_variables, output_variables=output_variables, multi_step="forward-euler 2" + ) + train_iter = iter(train_loader) + inputs, targets = next(train_iter) + + outputs = mech.forward(inputs, train=True) + assert len(outputs.keys() ^ targets.keys()) == 0, "forward output keys mismatch when evaluating (2 steps)" + loss = variable_loss_func(outputs, targets, mech.output_variables) + assert loss is not None, "forward outupt is incorrect when evaluating (2 steps)" + + +def test_train_graph(): + input_variables, output_variables, train_loader, _ = prepare(freq_rate=1) + + # test single step + mech = ReinforceCausalMech( + name="test", + input_variables=input_variables, + output_variables=output_variables, + ) + + grads = mech.train_graph(train_loader, data_ratio=1.0) + assert grads is not None, "train graph single step failed" + + # test multi-step + mech = ReinforceCausalMech( + name="test", + input_variables=input_variables, + output_variables=output_variables, + multi_step="forward-euler 2", + ) + + grads = mech.train_graph(train_loader, data_ratio=0.5) + assert grads is not None, "train graph multi-step (2) failed" + + +def test_learn(): + input_variables, output_variables, train_loader, valid_loader = prepare(freq_rate=1) + + # test single step on cpu + mech = ReinforceCausalMech( + name="test single on cpu", + input_variables=input_variables, + output_variables=output_variables, + ) + + mech.learn(train_loader, valid_loader) + + # test multi-step on cpu + mech = ReinforceCausalMech( + name="test multi on cpu", + input_variables=input_variables, + output_variables=output_variables, + multi_step="forward-euler 2", + ) + + mech.learn(train_loader, valid_loader) + + # test single step on cuda + mech = ReinforceCausalMech( + name="test single on cuda", + input_variables=input_variables, + output_variables=output_variables, + device="cuda:0", + ) + + mech.learn(train_loader, valid_loader) + + # test multi-step on cuda + mech = ReinforceCausalMech( + name="test multi on cuda", + input_variables=input_variables, + output_variables=output_variables, + multi_step="forward-euler 2", + device="cuda:0", + ) + + mech.learn(train_loader, valid_loader) diff --git a/tests/test_models/test_data_loader.py b/tests/test_models/test_data_loader.py new file mode 100644 index 0000000..bf436cb --- /dev/null +++ b/tests/test_models/test_data_loader.py @@ -0,0 +1,231 @@ +import gym +import emei +import numpy as np +from stable_baselines3.common.buffers import ReplayBuffer +from torch.utils.data import DataLoader + +from cmrl.models.data_loader import EnsembleBufferDataset, EnsembleBufferDataset +from cmrl.utils.env import load_offline_data + + +def test_buffer_dataset(): + env = gym.make("BoundaryInvertedPendulumSwingUp-v0", freq_rate=1, time_step=0.02) + + real_replay_buffer = ReplayBuffer( + int(1e5), env.observation_space, env.action_space, "cpu", handle_timeout_termination=False + ) + load_offline_data(env, real_replay_buffer, "SAC-expert-replay", use_ratio=0.1) + + # test for transition + dataset = EnsembleBufferDataset(real_replay_buffer, env.observation_space, env.action_space, mech="transition") + loader = DataLoader(dataset, batch_size=128, drop_last=True) + + for inputs, outputs in loader: + assert list(inputs.keys()) == ["obs_0", "obs_1", "obs_2", "obs_3", "obs_4", "act_0"] + assert list(outputs.keys()) == ["next_obs_0", "next_obs_1", "next_obs_2", "next_obs_3", "next_obs_4"] + for key in inputs: + assert inputs[key].shape == (128, 1) + for key in outputs: + assert outputs[key].shape == (128, 1) + + # test for reward + dataset = EnsembleBufferDataset(real_replay_buffer, env.observation_space, env.action_space, mech="reward_mech") + loader = DataLoader(dataset, batch_size=128, drop_last=True) + + for inputs, outputs in loader: + assert list(inputs.keys()) == [ + "obs_0", + "obs_1", + "obs_2", + "obs_3", + "obs_4", + "act_0", + "next_obs_0", + "next_obs_1", + "next_obs_2", + "next_obs_3", + "next_obs_4", + ] + assert list(outputs.keys()) == ["reward"] + for key in inputs: + assert inputs[key].shape == (128, 1) + for key in outputs: + assert outputs[key].shape == (128, 1) + + # test for termination + dataset = EnsembleBufferDataset(real_replay_buffer, env.observation_space, env.action_space, mech="termination_mech") + loader = DataLoader(dataset, batch_size=128, drop_last=True) + + for inputs, outputs in loader: + assert list(inputs.keys()) == [ + "obs_0", + "obs_1", + "obs_2", + "obs_3", + "obs_4", + "act_0", + "next_obs_0", + "next_obs_1", + "next_obs_2", + "next_obs_3", + "next_obs_4", + ] + assert list(outputs.keys()) == ["terminal"] + for key in inputs: + assert inputs[key].shape == (128, 1) + for key in outputs: + assert outputs[key].shape == (128, 1) + + +def test_ensemble_buffer_dataset(): + env = gym.make("BoundaryInvertedPendulumSwingUp-v0", freq_rate=1, time_step=0.02) + # assert isinstance(env, emei.EmeiEnv) + + real_replay_buffer = ReplayBuffer( + int(1e5), env.observation_space, env.action_space, "cpu", handle_timeout_termination=False + ) + load_offline_data(env, real_replay_buffer, "SAC-expert-replay", use_ratio=0.1) + + # test for transition + dataset = EnsembleBufferDataset(real_replay_buffer, env.observation_space, env.action_space, mech="transition") + loader = DataLoader(dataset, batch_size=128, drop_last=True) + + for inputs, outputs in loader: + assert list(inputs.keys()) == ["obs_0", "obs_1", "obs_2", "obs_3", "obs_4", "act_0"] + assert list(outputs.keys()) == ["next_obs_0", "next_obs_1", "next_obs_2", "next_obs_3", "next_obs_4"] + for key in inputs: + assert inputs[key].shape == (128, 7, 1) + for key in outputs: + assert outputs[key].shape == (128, 7, 1) + + # test for reward + dataset = EnsembleBufferDataset(real_replay_buffer, env.observation_space, env.action_space, mech="reward_mech") + print(dataset[0]) + loader = DataLoader(dataset, batch_size=128, drop_last=True) + + for inputs, outputs in loader: + assert list(inputs.keys()) == [ + "obs_0", + "obs_1", + "obs_2", + "obs_3", + "obs_4", + "act_0", + "next_obs_0", + "next_obs_1", + "next_obs_2", + "next_obs_3", + "next_obs_4", + ] + assert list(outputs.keys()) == ["reward"] + for key in inputs: + assert inputs[key].shape == (128, 7, 1) + for key in outputs: + assert outputs[key].shape == (128, 7, 1) + + # test for termination + dataset = EnsembleBufferDataset(real_replay_buffer, env.observation_space, env.action_space, mech="termination_mech") + loader = DataLoader(dataset, batch_size=128, drop_last=True) + + for inputs, outputs in loader: + assert list(inputs.keys()) == [ + "obs_0", + "obs_1", + "obs_2", + "obs_3", + "obs_4", + "act_0", + "next_obs_0", + "next_obs_1", + "next_obs_2", + "next_obs_3", + "next_obs_4", + ] + assert list(outputs.keys()) == ["terminal"] + for key in inputs: + assert inputs[key].shape == (128, 7, 1) + for key in outputs: + assert outputs[key].shape == (128, 7, 1) + + +def test_train_valid(): + env = gym.make("BoundaryInvertedPendulumSwingUp-v0", freq_rate=1, time_step=0.02) + + real_replay_buffer = ReplayBuffer( + int(1e5), env.observation_space, env.action_space, "cpu", handle_timeout_termination=False + ) + load_offline_data(env, real_replay_buffer, "SAC-expert-replay", use_ratio=0.1) + + train_dataset = EnsembleBufferDataset( + real_replay_buffer, env.observation_space, env.action_space, mech="transition", training=False + ) + valid_dataset = EnsembleBufferDataset( + real_replay_buffer, env.observation_space, env.action_space, mech="transition", training=True + ) + + buffer_size = real_replay_buffer.buffer_size if real_replay_buffer.full else real_replay_buffer.pos + assert len(set(train_dataset.indexes).intersection(set(valid_dataset.indexes))) == 0 + assert len(train_dataset.indexes) + len(valid_dataset.indexes) == buffer_size + + +def test_ensemble_train_valid(): + env = gym.make("BoundaryInvertedPendulumSwingUp-v0", freq_rate=1, time_step=0.02) + + real_replay_buffer = ReplayBuffer( + int(1e5), env.observation_space, env.action_space, "cpu", handle_timeout_termination=False + ) + load_offline_data(env, real_replay_buffer, "SAC-expert-replay", use_ratio=0.1) + + ensemble_num = 7 + train_dataset = EnsembleBufferDataset( + real_replay_buffer, + env.observation_space, + env.action_space, + mech="transition", + training=False, + ensemble_num=ensemble_num, + train_ensemble=False, + ) + valid_dataset = EnsembleBufferDataset( + real_replay_buffer, + env.observation_space, + env.action_space, + mech="transition", + training=True, + ensemble_num=ensemble_num, + train_ensemble=False, + ) + + buffer_size = real_replay_buffer.buffer_size if real_replay_buffer.full else real_replay_buffer.pos + for i in range(ensemble_num): + assert len(set(train_dataset.indexes[:, i]).intersection(set(valid_dataset.indexes[:, i]))) == 0 + assert len(train_dataset.indexes[:, i]) + len(valid_dataset.indexes[:, i]) == buffer_size + + +def test_mixed(): + env = gym.make("BoundaryInvertedPendulumSwingUp-v0", freq_rate=1, time_step=0.02) + # assert isinstance(env, emei.EmeiEnv) + + real_replay_buffer = ReplayBuffer( + int(1e5), env.observation_space, env.action_space, "cpu", handle_timeout_termination=False + ) + load_offline_data(env, real_replay_buffer, "SAC-expert-replay", use_ratio=0.1) + + ensemble_num = 7 + train_dataset = EnsembleBufferDataset( + real_replay_buffer, + env.observation_space, + env.action_space, + mech="transition", + training=False, + ensemble_num=ensemble_num, + train_ensemble=True, + ) + valid_dataset = EnsembleBufferDataset( + real_replay_buffer, env.observation_space, env.action_space, mech="transition", training=True + ) + + buffer_size = real_replay_buffer.buffer_size if real_replay_buffer.full else real_replay_buffer.pos + for i in range(ensemble_num): + assert len(set(train_dataset.indexes[:, i]).intersection(set(valid_dataset.indexes))) == 0 + assert len(train_dataset.indexes[:, i]) + len(valid_dataset.indexes) == buffer_size diff --git a/tests/test_models/test_graphs/test_binary_graph.py b/tests/test_models/test_graphs/test_binary_graph.py new file mode 100644 index 0000000..ff224d1 --- /dev/null +++ b/tests/test_models/test_graphs/test_binary_graph.py @@ -0,0 +1,83 @@ +import os +import time +import shutil + +import torch + +from cmrl.models.graphs.binary_graph import BinaryGraph + + +def test_init(): + g = BinaryGraph(5, 5, include_input=True, init_param=1) + + assert g.graph.size() == (5, 5) + assert g.graph.sum().item() == 20 + assert g.graph[torch.arange(5), torch.arange(5)].any().item() == False + + g = BinaryGraph(5, 5, extra_dim=3, init_param=torch.zeros(3, 5, 5)) + + assert g.graph.size() == (3, 5, 5) + assert g.graph.any().item() == False + + +def test_parameters(): + g = BinaryGraph(5, 5, include_input=True, init_param=1) + params = g.parameters + + assert isinstance(params, tuple) + assert len(params) == 1 + assert params[0].sum().item() == 20 + assert params[0][torch.arange(5), torch.arange(5)].any().item() == False + + g = BinaryGraph(5, 5, include_input=False, init_param=1) + params = g.parameters + + assert params[0].all().item() == True + + +def test_adj_matrix(): + g = BinaryGraph(5, 5, include_input=True, init_param=1) + adj_mat = g.get_adj_matrix() + + expected = torch.ones(5, 5, dtype=torch.int) + expected[torch.arange(5), torch.arange(5)] = 0 + + assert adj_mat.equal(expected), "get_adj_matrix failed" + + binary_adj_matrix = g.get_binary_adj_matrix() + + assert binary_adj_matrix.equal(expected), "get_binary_adj_matrix failed" + + +def test_set_data(): + g = BinaryGraph(5, 5, include_input=True, init_param=1) + + test_data = torch.ones(5, 5, dtype=torch.int) + test_data[:2, :2] = 0 + + expected = test_data.clone() + expected[torch.arange(2, 5), torch.arange(2, 5)] = 0 + + g.set_data(test_data) + + assert (g.graph == expected).all().item() == True + + +def test_save_load(): + # create a temp folder + while True: + save_dir = "./tmp" + str(time.time()) + if not os.path.exists(save_dir): + os.mkdir(save_dir) + break + + g = BinaryGraph(5, 5, include_input=True, init_param=1) + g.save(save_dir) + old_graph = g.graph + + g.load(save_dir) + assert g.graph is not old_graph + assert g.graph.equal(old_graph) + + # clear the temp folder + shutil.rmtree(save_dir) diff --git a/tests/test_models/test_graphs/test_neural_graph.py b/tests/test_models/test_graphs/test_neural_graph.py new file mode 100644 index 0000000..d8709ca --- /dev/null +++ b/tests/test_models/test_graphs/test_neural_graph.py @@ -0,0 +1,88 @@ +import os +import time +import shutil + +import torch +import torch.nn as nn + +from cmrl.models.graphs.neural_graph import NeuralGraph, NeuralBernoulliGraph + + +def test_init(): + g = NeuralGraph(5, 5, include_input=True) + + assert isinstance(g.graph, nn.Module) + + +def test_parameters(): + g = NeuralGraph(5, 5, include_input=True) + p = g.parameters + + assert len(p) == len(list(g.graph.parameters())) + + +def test_adj_matrix(): + g = NeuralGraph(5, 5, include_input=True) + assert next(g.graph.parameters()).grad is None + + inputs = torch.ones(2, 5) + adj_mat = g.get_adj_matrix(inputs) + + assert adj_mat.size() == (2, 5, 5), "get_adj_matrix failed" + b = adj_mat.sum() + b.backward() + assert next(g.graph.parameters()).grad is not None + + binary_adj_matrix = g.get_binary_adj_matrix(inputs, 0.5) + + assert binary_adj_matrix.size() == (2, 5, 5), "get_binary_adj_matrix failed" + + +def test_save_load(): + # create a temp folder + while True: + save_dir = "./tmp" + str(time.time()) + if not os.path.exists(save_dir): + os.mkdir(save_dir) + break + + g = NeuralGraph(5, 5, include_input=True) + g.save(save_dir) + old_graph = g.graph + state_dict = old_graph.state_dict() + + g.load(save_dir) + assert g.graph is old_graph + assert g.graph.state_dict() is not state_dict + + # clear the temp folder + shutil.rmtree(save_dir) + + +def test_bernoulli(): + g = NeuralBernoulliGraph(5, 5, include_input=True) + assert next(g.parameters).grad is None + + inputs = torch.ones(2, 5) + adj_mat = g.get_adj_matrix(inputs) + + assert adj_mat.size() == (2, 5, 5), "get_adj_matrix failed" + assert (adj_mat[:, torch.arange(5), torch.arange(5)] == 0).all() + assert ((adj_mat >= 0) & (adj_mat <= 1)).all() + b = adj_mat.sum() + b.backward() + assert next(g.graph.parameters()).grad is not None + + binary_adj_matrix = g.get_binary_adj_matrix(inputs, 0.5) + + assert binary_adj_matrix.size() == (2, 5, 5), "get_binary_adj_matrix failed" + + +if __name__ == "__main__": + test_init() + + test_parameters() + + test_adj_matrix() + + test_save_load() diff --git a/tests/test_models/test_graphs/test_prob_graph.py b/tests/test_models/test_graphs/test_prob_graph.py new file mode 100644 index 0000000..ae568ca --- /dev/null +++ b/tests/test_models/test_graphs/test_prob_graph.py @@ -0,0 +1,35 @@ +import torch + +from cmrl.models.graphs.prob_graph import BernoulliGraph + + +def test_init(): + g = BernoulliGraph(5, 5, include_input=True, init_param=0.1) + expected = torch.ones(5, 5) * 0.1 + expected[torch.arange(5), torch.arange(5)] = BernoulliGraph._MASK_VALUE + + assert g.graph.size() == (5, 5) + assert g.graph.equal(expected) + + +def test_adj_matrix(): + g = BernoulliGraph(5, 5, include_input=True, init_param=0.1) + adj_mat = g.get_adj_matrix() + expected = torch.sigmoid(torch.ones(5, 5) * 0.1) + expected[torch.arange(5), torch.arange(5)] = 0 + + assert adj_mat.equal(expected), "get_adj_matrix failed" + + binary_adj_matrix = g.get_binary_adj_matrix(0.5) + expected = torch.ones(5, 5, dtype=torch.int) + expected[torch.arange(5), torch.arange(5)] = 0 + + assert binary_adj_matrix.equal(expected), "get_binary_adj_matrix failed" + + +def test_sample(): + g = BernoulliGraph(5, 5, include_input=True, init_param=0.1) + samples = g.sample(None, 10, None) + + assert samples.size() == (10, 5, 5) + assert samples[:, torch.arange(5), torch.arange(5)].any() == False diff --git a/tests/test_models/test_graphs/test_weight_graph.py b/tests/test_models/test_graphs/test_weight_graph.py new file mode 100644 index 0000000..171f46b --- /dev/null +++ b/tests/test_models/test_graphs/test_weight_graph.py @@ -0,0 +1,122 @@ +import os +import time +import shutil + +import torch + +from cmrl.models.graphs.weight_graph import WeightGraph + + +def test_init(): + g = WeightGraph(5, 5, include_input=True, init_param=0.5) + expected = torch.ones(5, 5) * 0.5 + expected[torch.arange(5), torch.arange(5)] = 0 + + assert g.graph.size() == (5, 5) + assert g.graph.equal(expected) + + g = WeightGraph(5, 5, extra_dim=3, init_param=torch.ones(3, 5, 5) * 0.1) + expected = torch.ones(3, 5, 5) * 0.1 + + assert g.graph.size() == (3, 5, 5) + assert g.graph.equal(expected) + + +def test_parameters(): + g = WeightGraph(5, 5, include_input=True, init_param=0.5) + params = g.parameters + expected = torch.ones(5, 5) * 0.5 + expected[torch.arange(5), torch.arange(5)] = 0 + + assert isinstance(params, tuple) + assert len(params) == 1 + assert params[0].equal(expected) + + g = WeightGraph(5, 5, include_input=False, init_param=0.1) + params = g.parameters + expected = torch.ones(5, 5) * 0.1 + + assert params[0].equal(expected) + + +def test_adj_matrix(): + g = WeightGraph(5, 5, include_input=True, init_param=0.5) + adj_mat = g.get_adj_matrix() + expected = torch.ones(5, 5) * 0.5 + expected[torch.arange(5), torch.arange(5)] = 0 + + assert adj_mat.equal(expected), "get_adj_matrix failed" + + binary_adj_matrix = g.get_binary_adj_matrix(0.4) + expected = torch.ones(5, 5, dtype=torch.int) + expected[torch.arange(5), torch.arange(5)] = 0 + + assert binary_adj_matrix.equal(expected), "get_binary_adj_matrix failed" + + +def test_set_data(): + g = WeightGraph(5, 5, include_input=True, init_param=0.5) + + test_data = torch.ones(5, 5) + test_data[:2, :2] = 0 + + expected = test_data.clone() + expected[torch.arange(2, 5), torch.arange(2, 5)] = 0 + + g.set_data(test_data) + + assert (g.graph == expected).all().item() == True + + +def test_grad(): + g = WeightGraph(5, 5, init_param=0.5, requires_grad=False) + assert g.requries_grad == False, "no grad test failed" + + g = WeightGraph(5, 5, init_param=0.5, requires_grad=True) + assert g.requries_grad, "grad test, requires_grad failed" + assert g.graph.grad is None, "grad test, grad is None" + + c = g.parameters[0].abs().sum() + c.backward() + expected = torch.ones(5, 5) + assert g.graph.grad is not None and g.graph.grad.equal(expected), "grad test, incorrect grad" + + g = WeightGraph(5, 5, init_param=0.5, include_input=True, requires_grad=True) + assert g.graph.grad is None, "grad test, include input, requires_grad failed" + + c = g.parameters[0].abs().sum() + c.backward() + expected = torch.ones(5, 5) + expected[torch.arange(5), torch.arange(5)] = 0 + assert g.graph.grad is not None and g.graph.grad.equal(expected), "grad test, include input, incorrect grad" + + g = WeightGraph(5, 5, init_param=0.5, include_input=True, requires_grad=True) + p = g.parameters[0] + test_data = torch.ones(5, 5) + g.set_data(test_data) + + c = p.abs().sum() + c.backward() + expected = torch.ones(5, 5) + expected[torch.arange(5), torch.arange(5)] = 0 + assert g.graph.grad is not None and g.graph.grad.equal(expected), "grad test, include input, set_data, incorrect grad" + + +def test_save_load(): + # create a temp folder + while True: + save_dir = "./tmp" + str(time.time()) + if not os.path.exists(save_dir): + os.mkdir(save_dir) + break + + g = WeightGraph(5, 5, include_input=True, init_param=0.5) + g.save(save_dir) + old_graph = g.graph + + g.load(save_dir) + assert g.graph is not old_graph + assert g.graph.equal(old_graph) + + # clear the temp folder + shutil.rmtree(save_dir) diff --git a/tests/test_models/test_layers.py b/tests/test_models/test_layers.py index dc98bd1..3baf89f 100644 --- a/tests/test_models/test_layers.py +++ b/tests/test_models/test_layers.py @@ -1,54 +1,135 @@ from unittest import TestCase import torch +from torch.nn import Linear -from cmrl.models.layers import EnsembleLinearLayer, ParallelEnsembleLinearLayer - - -class TestParallelEnsembleLinearLayer(TestCase): - def setUp(self) -> None: - self.in_size = 5 - self.out_size = 6 - self.use_bias = True - self.parallel_num = 3 - self.ensemble_num = 4 - self.batch_size = 128 - self.device = "cuda" if torch.cuda.is_available() else "cpu" - self.layer = ParallelEnsembleLinearLayer( - in_size=self.in_size, - out_size=self.out_size, - use_bias=self.use_bias, - parallel_num=self.parallel_num, - ensemble_num=self.ensemble_num, - ).to(self.device) - - def test_forward(self): - model_in = torch.rand((self.parallel_num, self.ensemble_num, self.batch_size, self.in_size)).to(self.device) - model_out = self.layer(model_in) - assert model_out.shape == ( - self.parallel_num, - self.ensemble_num, - self.batch_size, - self.out_size, - ) - - -class TestEnsembleLinearLayer(TestCase): - def setUp(self) -> None: - self.in_size = 5 - self.out_size = 6 - self.use_bias = True - self.ensemble_num = 4 - self.batch_size = 128 - self.device = "cuda" if torch.cuda.is_available() else "cpu" - self.layer = EnsembleLinearLayer( - in_size=self.in_size, - out_size=self.out_size, - use_bias=self.use_bias, - ensemble_num=self.ensemble_num, - ).to(self.device) - - def test_forward(self): - model_in = torch.rand((self.ensemble_num, self.batch_size, self.in_size)).to(self.device) - model_out = self.layer(model_in) - assert model_out.shape == (self.ensemble_num, self.batch_size, self.out_size) +from cmrl.models.layers import ParallelLinear + + +def test_origin_layer(): + input_dim = 5 + output_dim = 6 + bias = True + batch_size = 128 + device = "cuda" if torch.cuda.is_available() else "cpu" + + layer = ParallelLinear( + input_dim=input_dim, + output_dim=output_dim, + bias=bias, + ).to(device) + + model_in = torch.rand((batch_size, input_dim)).to(device) + model_out = layer(model_in) + assert model_out.shape == ( + batch_size, + output_dim, + ) + + +def test_one_extra_dims_linear(): + input_dim = 5 + output_dim = 6 + bias = True + extra_dims = [7] + batch_size = 128 + device = "cuda" if torch.cuda.is_available() else "cpu" + + layer = ParallelLinear( + input_dim=input_dim, + output_dim=output_dim, + bias=bias, + extra_dims=extra_dims, + ).to(device) + + model_in = torch.rand((*extra_dims, batch_size, input_dim)).to(device) + model_out = layer(model_in) + assert model_out.shape == ( + extra_dims[0], + batch_size, + output_dim, + ) + + +def test_two_extra_dims_linear(): + input_dim = 5 + output_dim = 1 + bias = True + extra_dims = [6, 7] + batch_size = 128 + device = "cuda" if torch.cuda.is_available() else "cpu" + + layer = ParallelLinear( + input_dim=input_dim, + output_dim=output_dim, + bias=bias, + extra_dims=extra_dims, + ).to(device) + + model_in = torch.rand((*extra_dims, batch_size, input_dim)).to(device) + model_out = layer(model_in) + assert model_out.shape == ( + extra_dims[0], + extra_dims[1], + batch_size, + output_dim, + ) + + +def test_broadcast_two_extra_dims_linear(): + input_dim = 5 + output_dim = 1 + bias = True + extra_dims = [6, 7] + batch_size = 128 + device = "cuda" if torch.cuda.is_available() else "cpu" + + layer = ParallelLinear( + input_dim=input_dim, + output_dim=output_dim, + bias=bias, + extra_dims=extra_dims, + ).to(device) + + broadcast_dim = 10 + + model_in = torch.rand((broadcast_dim, *extra_dims, batch_size, input_dim)).to(device) + model_out = layer(model_in) + assert model_out.shape == ( + broadcast_dim, + extra_dims[0], + extra_dims[1], + batch_size, + output_dim, + ) + + +def test_broadcast_linear(): + input_dim = 5 + output_dim = 1 + bias = True + batch_size = 128 + device = "cuda" if torch.cuda.is_available() else "cpu" + + layer = Linear(input_dim, output_dim, bias=bias).to(device) + + broadcast_dim = 10 + + model_in = torch.rand((broadcast_dim, batch_size, input_dim)).to(device) + model_out = layer(model_in) + assert model_out.shape == ( + broadcast_dim, + batch_size, + output_dim, + ) + + +def test_repr(): + layer = ParallelLinear(3, 5) + print(repr(layer)) + assert True + + +def test_device(): + layer = ParallelLinear(3, 5).to("cpu") + assert str(layer.device) == "cpu" diff --git a/cmrl/models/transition/one_step/__init__.py b/tests/test_models/test_network/__init__.py similarity index 100% rename from cmrl/models/transition/one_step/__init__.py rename to tests/test_models/test_network/__init__.py diff --git a/tests/test_models/test_network/test_base_network.py b/tests/test_models/test_network/test_base_network.py new file mode 100644 index 0000000..81772d7 --- /dev/null +++ b/tests/test_models/test_network/test_base_network.py @@ -0,0 +1,11 @@ +from omegaconf import DictConfig + +from cmrl.models.networks.base_network import BaseNetwork + + +def test_base_network(): + try: + base_network = BaseNetwork(device="cpu") + assert False + except NotImplementedError: + pass diff --git a/tests/test_models/test_network/test_coder.py b/tests/test_models/test_network/test_coder.py new file mode 100644 index 0000000..88f3518 --- /dev/null +++ b/tests/test_models/test_network/test_coder.py @@ -0,0 +1,102 @@ +import torch +from torch.nn.functional import one_hot + +from cmrl.models.networks.coder import VariableEncoder, VariableDecoder +from cmrl.utils.variables import ContinuousVariable, DiscreteVariable, BinaryVariable + + +def test_continuous_encoder(): + var_dim = 3 + output_dim = 5 + batch_size = 128 + + var = ContinuousVariable(name="obs_0", dim=var_dim) + + encoder = VariableEncoder(var, output_dim, hidden_dims=[20]) + inputs = torch.rand(batch_size, var_dim) + outputs = encoder(inputs) + + assert outputs.shape == (batch_size, output_dim) + + +def test_discrete_encoder(): + var_n = 3 + output_dim = 5 + batch_size = 128 + + var = DiscreteVariable(name="obs_0", n=var_n) + + encoder = VariableEncoder(var, output_dim, hidden_dims=[20]) + inputs = one_hot(torch.randint(3, (batch_size,))).to(torch.float32) + outputs = encoder(inputs) + + assert outputs.shape == (batch_size, output_dim) + + +def test_binary_encoder(): + output_dim = 5 + batch_size = 128 + + var = BinaryVariable(name="obs_0") + + encoder = VariableEncoder(var, output_dim, hidden_dims=[20]) + inputs = one_hot(torch.randint(1, (batch_size,))).to(torch.float32) + outputs = encoder(inputs) + + assert outputs.shape == (batch_size, output_dim) + + +def test_identity_decoder(): + var_dim = 3 + batch_size = 128 + + var = ContinuousVariable(name="obs_0", dim=var_dim) + + decoder = VariableDecoder(var, identity=True) + inputs = torch.rand(batch_size, var_dim * 2) + outputs = decoder(inputs) + + assert outputs.shape == (batch_size, var_dim * 2) + + +def test_continuous_decoder(): + var_dim = 3 + input_dim = 5 + batch_size = 128 + + var = ContinuousVariable(name="obs_0", dim=var_dim) + + decoder = VariableDecoder(var, input_dim, hidden_dims=[200]) + inputs = torch.rand(batch_size, input_dim) + outputs = decoder(inputs) + + assert outputs.shape == (batch_size, var_dim * 2) + + +def test_discrete_decoder(): + var_n = 3 + input_dim = 5 + batch_size = 128 + + var = DiscreteVariable(name="obs_0", n=var_n) + + decoder = VariableDecoder(var, input_dim, hidden_dims=[200]) + inputs = torch.rand(batch_size, input_dim) + outputs = decoder(inputs) + + assert outputs.shape == (batch_size, var_n) + assert torch.allclose(outputs.sum(dim=1), torch.tensor(1.0)) + + +def test_binary_decoder(): + input_dim = 5 + batch_size = 128 + + var = BinaryVariable(name="obs_0") + + decoder = VariableDecoder(var, input_dim, hidden_dims=[200]) + inputs = torch.rand(batch_size, input_dim) + outputs = decoder(inputs) + + assert outputs.shape == (batch_size, 1) + assert (outputs >= 0).all() and (outputs <= 1).all() diff --git a/tests/test_models/test_network/test_parallel_mlp.py b/tests/test_models/test_network/test_parallel_mlp.py new file mode 100644 index 0000000..50931b9 --- /dev/null +++ b/tests/test_models/test_network/test_parallel_mlp.py @@ -0,0 +1,36 @@ +from omegaconf import DictConfig +import torch + +from cmrl.models.networks.parallel_mlp import ParallelMLP + + +def test_parallel_mlp(): + input_dim = 5 + output_dim = 6 + use_bias = True + extra_dims = [7] + batch_size = 128 + device = "cuda" if torch.cuda.is_available() else "cpu" + + network_cfg = dict( + { + "input_dim": input_dim, + "output_dim": output_dim, + "hidden_dims": [32, 32], + "bias": use_bias, + "extra_dims": extra_dims, + "activation_fn_cfg": DictConfig({"_target_": "torch.nn.SiLU"}), + } + ) + + mlp = ParallelMLP(**network_cfg).to(device) + + model_in = torch.rand((batch_size, input_dim)).to(device) + model_out = mlp(model_in) + assert model_out.shape == ( + *extra_dims, + batch_size, + output_dim, + ) + + assert str(mlp.device).startswith(device) diff --git a/tests/test_models/test_reward_mech/__init__.py b/tests/test_models/test_reward_mech/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/tests/test_models/test_reward_mech/test_plain_reward_mech.py b/tests/test_models/test_reward_mech/test_plain_reward_mech.py deleted file mode 100644 index 749cfe5..0000000 --- a/tests/test_models/test_reward_mech/test_plain_reward_mech.py +++ /dev/null @@ -1,30 +0,0 @@ -from unittest import TestCase - -import torch - -from cmrl.models.reward_mech.plain_reward_mech import PlainRewardMech - - -class TestPlainRewardMech(TestCase): - def setUp(self) -> None: - self.obs_size = 11 - self.action_size = 3 - self.num_layers = 4 - self.ensemble_num = 7 - self.hid_size = 200 - self.batch_size = 128 - self.device = "cuda" if torch.cuda.is_available() else "cpu" - self.reward_mech = PlainRewardMech( - obs_size=self.obs_size, - action_size=self.action_size, - device=self.device, - num_layers=self.num_layers, - hid_size=self.hid_size, - deterministic=True, - ) - self.batch_obs = torch.rand([self.ensemble_num, self.batch_size, self.obs_size]).to(self.device) - self.batch_action = torch.rand([self.ensemble_num, self.batch_size, self.action_size]).to(self.device) - - def test_forward(self): - mean, logvar = self.reward_mech.forward(self.batch_obs, self.batch_action) - assert mean.shape == (self.ensemble_num, self.batch_size, 1) diff --git a/tests/test_models/test_termination_mech/__init__.py b/tests/test_models/test_termination_mech/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/tests/test_models/test_termination_mech/test_plain_termination_mech.py b/tests/test_models/test_termination_mech/test_plain_termination_mech.py deleted file mode 100644 index e69de29..0000000 diff --git a/tests/test_models/test_transition/__init__.py b/tests/test_models/test_transition/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/tests/test_models/test_transition/test_base_transition.py b/tests/test_models/test_transition/test_base_transition.py deleted file mode 100644 index 87c913c..0000000 --- a/tests/test_models/test_transition/test_base_transition.py +++ /dev/null @@ -1,25 +0,0 @@ -import tempfile -from pathlib import Path -from unittest import TestCase - -import torch - -from cmrl.models.transition.base_transition import BaseTransition - - -class TestBasicEnsembleGaussianMLP(TestCase): - def setUp(self) -> None: - self.obs_size = 11 - self.action_size = 3 - self.ensemble_num = 7 - self.device = "cuda" if torch.cuda.is_available() else "cpu" - self.deterministic_transition = BaseTransition( - obs_size=self.obs_size, - action_size=self.action_size, - device=self.device, - ensemble_num=self.ensemble_num, - deterministic=True, - ) - - def test_build(self): - assert True diff --git a/tests/test_models/test_transition/test_multi_step/__init__.py b/tests/test_models/test_transition/test_multi_step/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/tests/test_models/test_transition/test_multi_step/test_forward_euler.py b/tests/test_models/test_transition/test_multi_step/test_forward_euler.py deleted file mode 100644 index af8f1de..0000000 --- a/tests/test_models/test_transition/test_multi_step/test_forward_euler.py +++ /dev/null @@ -1,51 +0,0 @@ -import tempfile -from pathlib import Path -from unittest import TestCase - -import torch - -from cmrl.models.transition.multi_step.forward_euler import ForwardEulerTransition -from cmrl.models.transition.one_step.plain_transition import PlainTransition - - -class TestForwardEulerTransition(TestCase): - def setUp(self) -> None: - self.obs_size = 11 - self.action_size = 3 - self.num_layers = 4 - self.ensemble_num = 7 - self.hid_size = 200 - self.batch_size = 128 - self.repeat_times = 3 - self.device = "cuda" if torch.cuda.is_available() else "cpu" - self.deterministic_one_step = PlainTransition( - obs_size=self.obs_size, - action_size=self.action_size, - device=self.device, - num_layers=self.num_layers, - ensemble_num=self.ensemble_num, - hid_size=self.hid_size, - deterministic=True, - ) - self.deterministic_transition = ForwardEulerTransition( - one_step_transition=self.deterministic_one_step, - repeat_times=self.repeat_times, - ) - self.gaussian_one_step = PlainTransition( - obs_size=self.obs_size, - action_size=self.action_size, - device=self.device, - num_layers=self.num_layers, - ensemble_num=self.ensemble_num, - hid_size=self.hid_size, - deterministic=False, - ) - self.gaussian_transition = ForwardEulerTransition( - one_step_transition=self.gaussian_one_step, repeat_times=self.repeat_times - ) - self.batch_obs = torch.rand([self.ensemble_num, self.batch_size, self.obs_size]).to(self.device) - self.batch_action = torch.rand([self.ensemble_num, self.batch_size, self.action_size]).to(self.device) - - def test_deterministic_forward(self): - mean, logvar = self.deterministic_transition.forward(self.batch_obs, self.batch_action) - assert mean.shape == (self.ensemble_num, self.batch_size, self.obs_size) diff --git a/tests/test_models/test_transition/test_one_step/__init__.py b/tests/test_models/test_transition/test_one_step/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/tests/test_models/test_transition/test_one_step/test_basic_ensemble.py b/tests/test_models/test_transition/test_one_step/test_basic_ensemble.py deleted file mode 100644 index 2ea754d..0000000 --- a/tests/test_models/test_transition/test_one_step/test_basic_ensemble.py +++ /dev/null @@ -1,72 +0,0 @@ -import tempfile -from pathlib import Path -from unittest import TestCase - -import torch - -from cmrl.models.transition.one_step.plain_transition import PlainTransition - - -class TestBasicEnsembleGaussianMLP(TestCase): - def setUp(self) -> None: - self.obs_size = 11 - self.action_size = 3 - self.num_layers = 4 - self.ensemble_num = 7 - self.hid_size = 200 - self.batch_size = 128 - self.device = "cuda" if torch.cuda.is_available() else "cpu" - self.deterministic_transition = PlainTransition( - obs_size=self.obs_size, - action_size=self.action_size, - device=self.device, - num_layers=self.num_layers, - ensemble_num=self.ensemble_num, - hid_size=self.hid_size, - deterministic=True, - ) - self.gaussian_transition = PlainTransition( - obs_size=self.obs_size, - action_size=self.action_size, - device=self.device, - num_layers=self.num_layers, - ensemble_num=self.ensemble_num, - hid_size=self.hid_size, - deterministic=False, - ) - self.batch_obs = torch.rand([self.ensemble_num, self.batch_size, self.obs_size]).to(self.device) - self.batch_action = torch.rand([self.ensemble_num, self.batch_size, self.action_size]).to(self.device) - - def test_deterministic_forward(self): - mean, logvar = self.deterministic_transition.forward(self.batch_obs, self.batch_action) - assert mean.shape == (self.ensemble_num, self.batch_size, self.obs_size) and logvar is None - - def test_gaussian_forward(self): - mean, logvar = self.gaussian_transition.forward(self.batch_obs, self.batch_action) - assert mean.shape == logvar.shape == (self.ensemble_num, self.batch_size, self.obs_size) - - def test_load(self): - tempdir = Path(tempfile.gettempdir()) - model_dir = tempdir / "temp_model" - if not model_dir.exists(): - model_dir.mkdir() - - mean, logvar = self.gaussian_transition.forward(self.batch_obs, self.batch_action) - self.gaussian_transition.save(model_dir) - - new_gaussian_transition = PlainTransition( - obs_size=self.obs_size, - action_size=self.action_size, - device=self.device, - num_layers=self.num_layers, - ensemble_num=self.ensemble_num, - hid_size=self.hid_size, - deterministic=False, - ) - - new_mean, new_logvar = new_gaussian_transition.forward(self.batch_obs, self.batch_action) - assert not (mean == new_mean).all() - - new_gaussian_transition.load(model_dir) - new_mean, new_logvar = new_gaussian_transition.forward(self.batch_obs, self.batch_action) - assert (mean == new_mean).all() diff --git a/tests/test_models/test_transition/test_one_step/test_external_mask_ensemble.py b/tests/test_models/test_transition/test_one_step/test_external_mask_ensemble.py deleted file mode 100644 index b4ad599..0000000 --- a/tests/test_models/test_transition/test_one_step/test_external_mask_ensemble.py +++ /dev/null @@ -1,93 +0,0 @@ -import tempfile -from pathlib import Path -from unittest import TestCase - -import torch - -from cmrl.models.transition.one_step.external_mask_transition import ( - ExternalMaskTransition, -) - - -class TestExternalMaskEnsembleGaussianTransition(TestCase): - def setUp(self) -> None: - self.obs_size = 11 - self.action_size = 3 - self.num_layers = 4 - self.ensemble_num = 7 - self.hid_size = 200 - self.batch_size = 128 - self.device = "cuda" if torch.cuda.is_available() else "cpu" - self.deterministic_transition = ExternalMaskTransition( - obs_size=self.obs_size, - action_size=self.action_size, - device=self.device, - num_layers=self.num_layers, - ensemble_num=self.ensemble_num, - hid_size=self.hid_size, - deterministic=True, - ) - self.gaussian_transition = ExternalMaskTransition( - obs_size=self.obs_size, - action_size=self.action_size, - device=self.device, - num_layers=self.num_layers, - ensemble_num=self.ensemble_num, - hid_size=self.hid_size, - deterministic=False, - ) - self.batch_obs = torch.rand([self.ensemble_num, self.batch_size, self.obs_size]).to(self.device) - self.batch_action = torch.rand([self.ensemble_num, self.batch_size, self.action_size]).to(self.device) - - def test_deterministic_forward(self): - input_mask = torch.ones(self.obs_size, self.obs_size + self.action_size) - self.deterministic_transition.set_input_mask(input_mask) - mean, logvar = self.deterministic_transition.forward(self.batch_obs, self.batch_action) - assert mean.shape == (self.ensemble_num, self.batch_size, self.obs_size) and logvar is None - - def test_gaussian_forward(self): - input_mask = torch.ones(self.obs_size, self.obs_size + self.action_size) - self.gaussian_transition.set_input_mask(input_mask) - mean, logvar = self.gaussian_transition.forward(self.batch_obs, self.batch_action) - assert mean.shape == logvar.shape == (self.ensemble_num, self.batch_size, self.obs_size) - - def test_mask_input(self): - input_mask = torch.ones(self.obs_size, self.obs_size + self.action_size) - self.gaussian_transition.set_input_mask(input_mask) - mean, logvar = self.gaussian_transition.forward(self.batch_obs, self.batch_action) - - new_input_mask = input_mask.clone() - new_input_mask[0] = torch.zeros(self.obs_size + self.action_size) - self.gaussian_transition.set_input_mask(new_input_mask) - new_mean, new_logvar = self.gaussian_transition.forward(self.batch_obs, self.batch_action) - assert not (mean == new_mean).all() - assert (mean[..., 1:] == new_mean[..., 1:]).all() - - def test_load(self): - tempdir = Path(tempfile.gettempdir()) - model_dir = tempdir / "temp_model" - if not model_dir.exists(): - model_dir.mkdir() - - input_mask = torch.ones(self.obs_size, self.obs_size + self.action_size) - self.gaussian_transition.set_input_mask(input_mask) - mean, logvar = self.gaussian_transition.forward(self.batch_obs, self.batch_action) - self.gaussian_transition.save(model_dir) - - new_gaussian_transition = ExternalMaskTransition( - obs_size=self.obs_size, - action_size=self.action_size, - device=self.device, - num_layers=self.num_layers, - ensemble_num=self.ensemble_num, - hid_size=self.hid_size, - deterministic=False, - ) - - new_gaussian_transition.set_input_mask(input_mask) - new_mean, new_logvar = new_gaussian_transition.forward(self.batch_obs, self.batch_action) - assert not (mean == new_mean).all() - - new_gaussian_transition.load(model_dir) - new_mean, new_logvar = new_gaussian_transition.forward(self.batch_obs, self.batch_action) - assert (mean == new_mean).all() diff --git a/tests/test_sb3_extension/test_online_mb_callback.py b/tests/test_sb3_extension/test_online_mb_callback.py index 61fb283..a2ac0ce 100644 --- a/tests/test_sb3_extension/test_online_mb_callback.py +++ b/tests/test_sb3_extension/test_online_mb_callback.py @@ -7,34 +7,44 @@ from stable_baselines3.common.buffers import ReplayBuffer from cmrl.sb3_extension.online_mb_callback import OnlineModelBasedCallback -from cmrl.models.dynamics import PlainEnsembleDynamics -from cmrl.models.transition.one_step.plain_transition import PlainTransition +from cmrl.utils.creator import parse_space +from cmrl.models.causal_mech.oracle_mech import OracleMech +from cmrl.models.dynamics import Dynamics from cmrl.models.fake_env import VecFakeEnv def test_callback(): env = cast(emei.EmeiEnv, gym.make("BoundaryInvertedPendulumSwingUp-v0", freq_rate=1, time_step=0.02)) - term_fn = env.get_terminal reward_fn = env.get_reward - init_obs_fn = env.get_batch_init_obs + termination_fn = env.get_terminal + get_init_obs_fn = env.get_batch_init_obs - transition = PlainTransition(obs_size=5, action_size=1) + obs_variables = parse_space(env.state_space, "obs") + act_variables = parse_space(env.action_space, "act") + next_obs_variables = parse_space(env.state_space, "next_obs") - dynamics = PlainEnsembleDynamics( - transition=transition, - learned_reward=False, - reward_mech=reward_fn, - learned_termination=False, - termination_mech=term_fn, + transition = OracleMech( + name="transition", + input_variables=obs_variables + act_variables, + output_variables=next_obs_variables, ) + + dynamics = Dynamics(transition, env.state_space, env.action_space) real_replay_buffer = ReplayBuffer( - 100, env.observation_space, env.action_space, device="cpu", handle_timeout_termination=False + 100, env.state_space, env.action_space, device="cpu", handle_timeout_termination=False ) - fake_env = VecFakeEnv(1, env.observation_space, env.action_space) - fake_env.set_up(dynamics, reward_fn, term_fn, init_obs_fn) + fake_env = VecFakeEnv( + num_envs=1, + observation_space=env.state_space, + action_space=env.action_space, + dynamics=dynamics, + reward_fn=reward_fn, + termination_fn=termination_fn, + get_init_obs_fn=get_init_obs_fn, + ) - callback = OnlineModelBasedCallback(env, dynamics, real_replay_buffer=real_replay_buffer, freq_train_model=5) + callback = OnlineModelBasedCallback(env, dynamics, real_replay_buffer, freq_train_model=20, longest_epoch=1) model = SAC("MlpPolicy", fake_env, verbose=1) model.learn(total_timesteps=100, log_interval=4, callback=callback) diff --git a/tests/test_types.py b/tests/test_types.py index 930a31d..34ade49 100644 --- a/tests/test_types.py +++ b/tests/test_types.py @@ -2,46 +2,3 @@ from unittest import TestCase import torch - -from cmrl.types import InteractionBatch - - -class TestTransitionBatch(TestCase): - def setUp(self) -> None: - self.batch_size = random.randint(0, 1000) - self.obs_size = random.randint(0, 1000) - self.action_size = random.randint(0, 1000) - self.transition_batch = InteractionBatch( - torch.rand(self.batch_size, self.obs_size), - torch.rand(self.batch_size, self.action_size), - torch.rand(self.batch_size, self.obs_size), - torch.rand(self.batch_size, 1), - torch.rand(self.batch_size, 1), - ) - - def test_as_tuple(self): - ( - batch_obs, - batch_action, - batch_next_obs, - batch_reward, - batch_done, - ) = self.transition_batch.as_tuple() - assert batch_obs.shape == (self.batch_size, self.obs_size) - assert batch_action.shape == (self.batch_size, self.action_size) - assert batch_next_obs.shape == (self.batch_size, self.obs_size) - assert batch_reward.shape == (self.batch_size, 1) - assert batch_done.shape == (self.batch_size, 1) - - def test___getitem__(self): - slice_transition = self.transition_batch[0] - assert len(slice_transition) == self.obs_size - - new_transition = slice_transition.add_new_batch_dim(1) - assert new_transition.batch_obs.shape == (1, self.obs_size) - - def test__get_new_shape(self): - new_batch_size = 1 - old_shape = self.transition_batch.batch_obs.shape - new_shape = self.transition_batch._get_new_shape(old_shape, new_batch_size) - assert new_shape == (new_batch_size, self.batch_size, self.obs_size)