From 022251dc52cd68b8942889015fe858b4d4132c3d Mon Sep 17 00:00:00 2001 From: frank Date: Tue, 25 Oct 2022 02:18:51 +0800 Subject: [PATCH 01/68] :hammer: try to refactor the models module --- cmrl/agent/sac_wrapper.py | 28 ---- cmrl/diagnostics/eval_model_on_dataset.py | 3 +- cmrl/models/causal_discovery/CMI_test.py | 9 +- cmrl/models/layers.py | 143 +++++++----------- cmrl/models/networks/__init__.py | 0 cmrl/models/networks/base_network.py | 59 ++++++++ cmrl/models/networks/mlp.py | 10 +- cmrl/models/networks/parallel_mlp.py | 44 ++++++ cmrl/models/networks/util.py | 0 cmrl/models/reward_mech/plain_reward_mech.py | 4 +- .../plain_termination_mech.py | 4 +- .../one_step/external_mask_transition.py | 9 +- .../transition/one_step/plain_transition.py | 4 +- tests/test_models/test_layers.py | 108 +++++++------ tests/test_models/test_network/__init__.py | 0 .../test_network/test_base_network.py | 13 ++ .../test_network/test_parallel_mlp.py | 35 +++++ 17 files changed, 286 insertions(+), 187 deletions(-) delete mode 100644 cmrl/agent/sac_wrapper.py create mode 100644 cmrl/models/networks/__init__.py create mode 100644 cmrl/models/networks/base_network.py create mode 100644 cmrl/models/networks/parallel_mlp.py create mode 100644 cmrl/models/networks/util.py create mode 100644 tests/test_models/test_network/__init__.py create mode 100644 tests/test_models/test_network/test_base_network.py create mode 100644 tests/test_models/test_network/test_parallel_mlp.py diff --git a/cmrl/agent/sac_wrapper.py b/cmrl/agent/sac_wrapper.py deleted file mode 100644 index b83c5a5..0000000 --- a/cmrl/agent/sac_wrapper.py +++ /dev/null @@ -1,28 +0,0 @@ -import numpy as np -import torch - -import cmrl.third_party.pytorch_sac as pytorch_sac - -from .core import Agent - - -class SACAgent(Agent): - def __init__(self, sac_agent: pytorch_sac.SAC): - self.sac_agent = sac_agent - - def act(self, obs: np.ndarray, sample: bool = False, batched: bool = False, **kwargs) -> np.ndarray: - """Issues an action given an observation. - - Args: - obs (np.ndarray): the observation (or batch of observations) for which the action - is needed. - sample (bool): if ``True`` the agent samples actions from its policy, otherwise it - returns the mean policy value. Defaults to ``False``. - batched (bool): if ``True`` signals to the agent that the obs should be interpreted - as a batch. - - Returns: - (np.ndarray): the action. - """ - with torch.no_grad(): - return self.sac_agent.select_action(obs, batched=batched, evaluate=not sample) diff --git a/cmrl/diagnostics/eval_model_on_dataset.py b/cmrl/diagnostics/eval_model_on_dataset.py index a0166c2..689511e 100644 --- a/cmrl/diagnostics/eval_model_on_dataset.py +++ b/cmrl/diagnostics/eval_model_on_dataset.py @@ -11,6 +11,7 @@ import cmrl.util.creator import cmrl.util.env from cmrl.util.config import load_hydra_cfg +from cmrl.util.transition_iterator import TransitionIterator class DatasetEvaluator: @@ -62,7 +63,7 @@ def __init__(self, model_dir: str, dataset: str, batch_size: int = 4096, device= def plot_dataset_results( self, - dataset: cmrl.util.TransitionIterator, + dataset: TransitionIterator, hist_bins: int = 20, hist_log: bool = True, ): diff --git a/cmrl/models/causal_discovery/CMI_test.py b/cmrl/models/causal_discovery/CMI_test.py index 34a6668..623b11f 100644 --- a/cmrl/models/causal_discovery/CMI_test.py +++ b/cmrl/models/causal_discovery/CMI_test.py @@ -7,7 +7,8 @@ from torch.nn import functional as F from cmrl.models.util import gaussian_nll -from cmrl.models.layers import ParallelEnsembleLinearLayer, truncated_normal_init + +# from cmrl.models.layers import ParallelEnsembleLinearLayer, truncated_normal_init from cmrl.models.networks.mlp import EnsembleMLP from cmrl.models.util import to_tensor @@ -78,11 +79,11 @@ def create_activation(): 0.5 * torch.ones(self.parallel_num, 1, 1, self.obs_size), requires_grad=learn_logvar_bounds ) - self.apply(truncated_normal_init) + # self.apply(truncated_normal_init) self.to(self.device) - def create_linear_layer(self, l_in, l_out): - return ParallelEnsembleLinearLayer(l_in, l_out, parallel_num=self.parallel_num, ensemble_num=self.ensemble_num) + # def create_linear_layer(self, l_in, l_out): + # return ParallelEnsembleLinearLayer(l_in, l_out, parallel_num=self.parallel_num, ensemble_num=self.ensemble_num) @property def input_mask(self): diff --git a/cmrl/models/layers.py b/cmrl/models/layers.py index 5222dc6..1db758e 100644 --- a/cmrl/models/layers.py +++ b/cmrl/models/layers.py @@ -1,117 +1,80 @@ +from typing import Optional, List + import numpy as np import torch from torch import nn as nn +from itertools import product -import cmrl.models.util as model_util - - -def truncated_normal_init(m: nn.Module): - """Initializes the weights of the given module using a truncated normal distribution.""" - - if isinstance(m, nn.Linear): - input_dim = m.weight.data.shape[0] - stddev = 1 / (2 * np.sqrt(input_dim)) - model_util.truncated_normal_(m.weight.data, std=stddev) - m.bias.data.fill_(0.0) - elif isinstance(m, EnsembleLinearLayer): - num_members, input_dim, _ = m.weight.data.shape - stddev = 1 / (2 * np.sqrt(input_dim)) - for i in range(num_members): - model_util.truncated_normal_(m.weight.data[i], std=stddev) - m.bias.data.fill_(0.0) - elif isinstance(m, ParallelEnsembleLinearLayer): - num_parallel, num_members, input_dim, _ = m.weight.data.shape - stddev = 1 / (2 * np.sqrt(input_dim)) - for i in range(num_parallel): - for j in range(num_members): - model_util.truncated_normal_(m.weight.data[i, j], std=stddev) - m.bias.data.fill_(0.0) +from cmrl.models.util import truncated_normal_ -class EnsembleLinearLayer(nn.Module): - """Implements an ensemble of layers. - - Args: - in_size (int): the input size of this layer. - out_size (int): the output size of this layer. - use_bias (bool): use bias in this layer or not. - ensemble_num (int): the ensemble dimension of this layer, - the corresponding part of each dimension is called a "member". - """ - +# partial from https://github.com/phlippe/ENCO/blob/main/causal_discovery/multivariable_mlp.py +class ParallelLinear(nn.Module): def __init__( self, - in_size: int, - out_size: int, + input_dim: int, + output_dim: int, + extra_dims: Optional[List[int]] = None, use_bias: bool = True, - ensemble_num: int = 1, + init_type: str = "truncated_normal", ): + """Linear layer with the same properties as Parallel MLP. It effectively applies N independent + linear layers in parallel. + + Args: + input_dim: Number of input dimensions per network. + output_dim: s + extra_dims: s + use_bias: s + init_type: s + """ super().__init__() - self.ensemble_num = ensemble_num - self.in_size = in_size - self.out_size = out_size - self.weight = nn.Parameter(torch.rand(self.ensemble_num, self.in_size, self.out_size)) + self.input_dim = input_dim + self.output_dim = output_dim + self.extra_dims = [] if extra_dims is None else extra_dims + self.init_type = init_type + + self.weight = nn.Parameter(torch.zeros(*self.extra_dims, self.input_dim, self.output_dim)) if use_bias: - self.bias = nn.Parameter(torch.rand(self.ensemble_num, 1, self.out_size)) + self.bias = nn.Parameter(torch.zeros(*self.extra_dims, 1, self.output_dim)) self.use_bias = True else: self.use_bias = False - def forward(self, x): - xw = x.matmul(self.weight) - if self.use_bias: - return xw + self.bias - else: - return xw - - def __repr__(self) -> str: - return ( - f"in_size={self.in_size}, out_size={self.out_size}, use_bias={self.use_bias}, " f"ensemble_num={self.ensemble_num}" - ) - - -class ParallelEnsembleLinearLayer(nn.Module): - """Implements an ensemble of parallel layers. - - Args: - in_size (int): the input size of this layer. - out_size (int): the output size of this layer. - use_bias (bool): use bias in this layer or not. - parallel_num (int): the parallel dimension of this layer, - the corresponding part of each dimension is called a "sub-network". - ensemble_num (int): the ensemble dimension of this layer, - the corresponding part of each dimension is called a "member". - """ + self.init() - def __init__( - self, - in_size: int, - out_size: int, - use_bias: bool = True, - parallel_num: int = 1, - ensemble_num: int = 1, - ): - super().__init__() - self.parallel_num = parallel_num - self.ensemble_num = ensemble_num - self.in_size = in_size - self.out_size = out_size - self.weight = nn.Parameter(torch.rand(self.parallel_num, self.ensemble_num, self.in_size, self.out_size)) - if use_bias: - self.bias = nn.Parameter(torch.rand(self.parallel_num, self.ensemble_num, 1, self.out_size)) - self.use_bias = True + def init(self): + if self.init_type == "kaiming_uniform": + nn.init.kaiming_uniform_(self.weight, nonlinearity="relu") + elif self.init_type == "truncated_normal": + stddev = 1 / (2 * np.sqrt(self.input_dim)) + for dims in product(*map(range, self.extra_dims)): + truncated_normal_(self.weight.data[dims], std=stddev) else: - self.use_bias = False + raise NotImplementedError def forward(self, x): + # Shape preparation + x_extra_dims = x.shape[:-2] + if len(x_extra_dims) > 0: + for i in range(len(x_extra_dims)): + assert x_extra_dims[-(i + 1)] == self.extra_dims[-(i + 1)], "Shape mismatch: X=%s, Layer=%s" % ( + str(x.shape), + str(self.extra_dims), + ) + xw = x.matmul(self.weight) if self.use_bias: return xw + self.bias else: return xw - def __repr__(self) -> str: - return ( - f"in_size={self.in_size}, out_size={self.out_size}, use_bias={self.use_bias}, " - f"parallel_num={self.parallel_num}, ensemble_num={self.ensemble_num}" + @property + def device(self): + return next(iter(self.parameters())).device + + def __repr__(self): + # For printing + return 'ParallelLinear(input_dims={}, output_dims={}, extra_dims={}, use_bias={}, init_type="{}")'.format( + self.input_dim, self.output_dim, str(self.extra_dims), self.use_bias, self.init_type ) diff --git a/cmrl/models/networks/__init__.py b/cmrl/models/networks/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/cmrl/models/networks/base_network.py b/cmrl/models/networks/base_network.py new file mode 100644 index 0000000..7e1b9f6 --- /dev/null +++ b/cmrl/models/networks/base_network.py @@ -0,0 +1,59 @@ +import pathlib +from typing import List, Optional, Sequence, Union +from abc import abstractmethod + +import torch +import torch.nn as nn +from omegaconf import DictConfig + +from cmrl.models.util import gaussian_nll +from cmrl.models.layers import ParallelLinear + + +class BaseNetwork(nn.Module): + _MODEL_FILENAME = "base_network.pth" + + def __init__(self, network_cfg: DictConfig): + super(BaseNetwork, self).__init__() + + self.network_cfg = network_cfg + + self._model_save_attrs: List[str] = ["state_dict"] + self._layers: Optional[nn.Module] = None + + self.build_network() + + def save(self, save_dir: Union[str, pathlib.Path]): + """Saves the model to the given directory.""" + model_dict = {} + for attr in self._model_save_attrs: + if attr == "state_dict": + model_dict["state_dict"] = self.state_dict() + else: + model_dict[attr] = getattr(self, attr) + torch.save(model_dict, pathlib.Path(save_dir) / self._MODEL_FILENAME) + + def load(self, load_dir: Union[str, pathlib.Path]): + """Loads the model from the given path.""" + model_dict = torch.load(pathlib.Path(load_dir) / self._MODEL_FILENAME, map_location=self.device) + for attr in model_dict: + if attr == "state_dict": + self.load_state_dict(model_dict["state_dict"]) + else: + getattr(self, "set_" + attr)(model_dict[attr]) + + @abstractmethod + def build_network(self): + raise NotImplementedError + + @property + def save_attrs(self): + return self._model_save_attrs + + @property + def model_file_name(self): + return self._MODEL_FILENAME + + @property + def device(self): + return next(iter(self.parameters())).device diff --git a/cmrl/models/networks/mlp.py b/cmrl/models/networks/mlp.py index 9b9e8c3..c4f15b5 100644 --- a/cmrl/models/networks/mlp.py +++ b/cmrl/models/networks/mlp.py @@ -1,13 +1,15 @@ import pathlib from typing import Dict, Optional, Sequence, Union +from abc import abstractmethod -import numpy as np import torch import torch.nn as nn -import torch.nn.functional as F +import numpy as np +from omegaconf import DictConfig from cmrl.models.util import gaussian_nll -from cmrl.models.layers import EnsembleLinearLayer +from cmrl.models.layers import ParallelLinear +from cmrl.models.networks.base_network import BaseNetwork class EnsembleMLP(nn.Module): @@ -63,7 +65,7 @@ def load(self, load_dir: Union[str, pathlib.Path], load_device: Optional[str] = getattr(self, "set_" + attr)(model_dict[attr]) def create_linear_layer(self, l_in, l_out): - return EnsembleLinearLayer(l_in, l_out, ensemble_num=self.ensemble_num) + return ParallelLinear(l_in, l_out) def get_mse_loss(self, model_in: Dict[(str, torch.Tensor)], target: torch.Tensor) -> torch.Tensor: pred_mean, pred_logvar = self.forward(**model_in) diff --git a/cmrl/models/networks/parallel_mlp.py b/cmrl/models/networks/parallel_mlp.py new file mode 100644 index 0000000..69342f8 --- /dev/null +++ b/cmrl/models/networks/parallel_mlp.py @@ -0,0 +1,44 @@ +import pathlib +from typing import List, Optional, Sequence, Union +from abc import abstractmethod + +import torch +import torch.nn as nn +import hydra +from omegaconf import DictConfig + +from cmrl.models.util import gaussian_nll +from cmrl.models.layers import ParallelLinear +from cmrl.models.networks.base_network import BaseNetwork + + +# partial from https://github.com/phlippe/ENCO/blob/main/causal_discovery/multivariable_mlp.py +class ParallelMLP(BaseNetwork): + _MODEL_FILENAME = "parallel_mlp.pth" + + def __init__(self, network_cfg: DictConfig): + super().__init__(network_cfg) + + def build_network(self): + activation_fn_cfg = self.network_cfg.get("activation_fn_cfg", None) + extra_dims = self.network_cfg.get("extra_dims", None) + + def create_activation(): + if activation_fn_cfg is None: + return nn.ReLU() + else: + return hydra.utils.instantiate(activation_fn_cfg) + + layers = [] + hidden_dims = [self.network_cfg.input_dim] + self.network_cfg.hidden_dims + for i in range(len(hidden_dims) - 1): + layers += [ParallelLinear(input_dim=hidden_dims[i], output_dim=hidden_dims[i + 1], extra_dims=extra_dims)] + layers += [create_activation()] + layers += [ParallelLinear(input_dim=hidden_dims[-1], output_dim=self.network_cfg.output_dim, extra_dims=extra_dims)] + + self._layers = nn.ModuleList(layers) + + def forward(self, x): + for layer in self._layers: + x = layer(x) + return x diff --git a/cmrl/models/networks/util.py b/cmrl/models/networks/util.py new file mode 100644 index 0000000..e69de29 diff --git a/cmrl/models/reward_mech/plain_reward_mech.py b/cmrl/models/reward_mech/plain_reward_mech.py index a111471..0cebead 100644 --- a/cmrl/models/reward_mech/plain_reward_mech.py +++ b/cmrl/models/reward_mech/plain_reward_mech.py @@ -6,7 +6,7 @@ from torch import nn as nn from torch.nn import functional as F -from cmrl.models.layers import truncated_normal_init +# from cmrl.models.layers import truncated_normal_init from cmrl.models.reward_mech.base_reward_mech import BaseRewardMech @@ -69,7 +69,7 @@ def create_activation(): self.min_logvar = nn.Parameter(-10 * torch.ones(1), requires_grad=learn_logvar_bounds) self.max_logvar = nn.Parameter(0.5 * torch.ones(1), requires_grad=learn_logvar_bounds) - self.apply(truncated_normal_init) + # self.apply(truncated_normal_init) self.to(self.device) def forward( diff --git a/cmrl/models/termination_mech/plain_termination_mech.py b/cmrl/models/termination_mech/plain_termination_mech.py index 1143690..59b8478 100644 --- a/cmrl/models/termination_mech/plain_termination_mech.py +++ b/cmrl/models/termination_mech/plain_termination_mech.py @@ -6,7 +6,7 @@ from torch import nn as nn from torch.nn import functional as F -from cmrl.models.layers import truncated_normal_init +# from cmrl.models.layers import truncated_normal_init from cmrl.models.termination_mech.base_termination_mech import BaseTerminationMech @@ -69,7 +69,7 @@ def create_activation(): self.min_logvar = nn.Parameter(-10 * torch.ones(1), requires_grad=learn_logvar_bounds) self.max_logvar = nn.Parameter(0.5 * torch.ones(1), requires_grad=learn_logvar_bounds) - self.apply(truncated_normal_init) + # self.apply(truncated_normal_init) self.to(self.device) def forward( diff --git a/cmrl/models/transition/one_step/external_mask_transition.py b/cmrl/models/transition/one_step/external_mask_transition.py index 783236c..d0fa815 100644 --- a/cmrl/models/transition/one_step/external_mask_transition.py +++ b/cmrl/models/transition/one_step/external_mask_transition.py @@ -7,7 +7,8 @@ from torch.nn import functional as F import cmrl.types -from cmrl.models.layers import ParallelEnsembleLinearLayer, truncated_normal_init + +# from cmrl.models.layers import ParallelEnsembleLinearLayer, truncated_normal_init from cmrl.models.transition.base_transition import BaseTransition from cmrl.models.util import to_tensor @@ -100,11 +101,11 @@ def create_activation(): self.min_logvar = nn.Parameter(-10 * torch.ones(obs_size, 1, 1, 1), requires_grad=learn_logvar_bounds) self.max_logvar = nn.Parameter(0.5 * torch.ones(obs_size, 1, 1, 1), requires_grad=learn_logvar_bounds) - self.apply(truncated_normal_init) + # self.apply(truncated_normal_init) self.to(self.device) - def create_linear_layer(self, l_in, l_out): - return ParallelEnsembleLinearLayer(l_in, l_out, parallel_num=self.obs_size, ensemble_num=self.ensemble_num) + # def create_linear_layer(self, l_in, l_out): + # return ParallelEnsembleLinearLayer(l_in, l_out, parallel_num=self.obs_size, ensemble_num=self.ensemble_num) def set_input_mask(self, mask: cmrl.types.TensorType): self._input_mask = to_tensor(mask).to(self.device) diff --git a/cmrl/models/transition/one_step/plain_transition.py b/cmrl/models/transition/one_step/plain_transition.py index 3efa810..94e99bb 100644 --- a/cmrl/models/transition/one_step/plain_transition.py +++ b/cmrl/models/transition/one_step/plain_transition.py @@ -6,7 +6,7 @@ from torch import nn as nn from torch.nn import functional as F -from cmrl.models.layers import EnsembleLinearLayer, truncated_normal_init +# from cmrl.models.layers import EnsembleLinearLayer, truncated_normal_init from cmrl.models.transition.base_transition import BaseTransition @@ -94,7 +94,7 @@ def create_activation(): self.min_logvar = nn.Parameter(-10 * torch.ones(1, obs_size), requires_grad=learn_logvar_bounds) self.max_logvar = nn.Parameter(0.5 * torch.ones(1, obs_size), requires_grad=learn_logvar_bounds) - self.apply(truncated_normal_init) + # self.apply(truncated_normal_init) self.to(self.device) def forward( diff --git a/tests/test_models/test_layers.py b/tests/test_models/test_layers.py index dc98bd1..6e76530 100644 --- a/tests/test_models/test_layers.py +++ b/tests/test_models/test_layers.py @@ -2,53 +2,61 @@ import torch -from cmrl.models.layers import EnsembleLinearLayer, ParallelEnsembleLinearLayer - - -class TestParallelEnsembleLinearLayer(TestCase): - def setUp(self) -> None: - self.in_size = 5 - self.out_size = 6 - self.use_bias = True - self.parallel_num = 3 - self.ensemble_num = 4 - self.batch_size = 128 - self.device = "cuda" if torch.cuda.is_available() else "cpu" - self.layer = ParallelEnsembleLinearLayer( - in_size=self.in_size, - out_size=self.out_size, - use_bias=self.use_bias, - parallel_num=self.parallel_num, - ensemble_num=self.ensemble_num, - ).to(self.device) - - def test_forward(self): - model_in = torch.rand((self.parallel_num, self.ensemble_num, self.batch_size, self.in_size)).to(self.device) - model_out = self.layer(model_in) - assert model_out.shape == ( - self.parallel_num, - self.ensemble_num, - self.batch_size, - self.out_size, - ) - - -class TestEnsembleLinearLayer(TestCase): - def setUp(self) -> None: - self.in_size = 5 - self.out_size = 6 - self.use_bias = True - self.ensemble_num = 4 - self.batch_size = 128 - self.device = "cuda" if torch.cuda.is_available() else "cpu" - self.layer = EnsembleLinearLayer( - in_size=self.in_size, - out_size=self.out_size, - use_bias=self.use_bias, - ensemble_num=self.ensemble_num, - ).to(self.device) - - def test_forward(self): - model_in = torch.rand((self.ensemble_num, self.batch_size, self.in_size)).to(self.device) - model_out = self.layer(model_in) - assert model_out.shape == (self.ensemble_num, self.batch_size, self.out_size) +from cmrl.models.layers import ParallelLinear + + +def test_origin_layer(): + input_dim = 5 + output_dim = 6 + use_bias = True + batch_size = 128 + device = "cuda" if torch.cuda.is_available() else "cpu" + + layer = ParallelLinear( + input_dim=input_dim, + output_dim=output_dim, + use_bias=use_bias, + ).to(device) + + model_in = torch.rand((batch_size, input_dim)).to(device) + model_out = layer(model_in) + assert model_out.shape == ( + batch_size, + output_dim, + ) + + +def test_two_extra_dims_layer(): + input_dim = 5 + output_dim = 6 + use_bias = True + extra_dims = [3, 4] + batch_size = 128 + device = "cuda" if torch.cuda.is_available() else "cpu" + + layer = ParallelLinear( + input_dim=input_dim, + output_dim=output_dim, + use_bias=use_bias, + extra_dims=extra_dims, + ).to(device) + + model_in = torch.rand((*extra_dims, batch_size, input_dim)).to(device) + model_out = layer(model_in) + assert model_out.shape == ( + extra_dims[0], + extra_dims[1], + batch_size, + output_dim, + ) + + +def test_repr(): + layer = ParallelLinear(3, 5) + print(repr(layer)) + assert True + + +def test_device(): + layer = ParallelLinear(3, 5).to("cpu") + assert str(layer.device) == "cpu" diff --git a/tests/test_models/test_network/__init__.py b/tests/test_models/test_network/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_models/test_network/test_base_network.py b/tests/test_models/test_network/test_base_network.py new file mode 100644 index 0000000..f8d409a --- /dev/null +++ b/tests/test_models/test_network/test_base_network.py @@ -0,0 +1,13 @@ +from omegaconf import DictConfig + +from cmrl.models.networks.base_network import BaseNetwork + + +def test_base_network(): + network_cfg = DictConfig({"input_dim": 5, "output_dim": 10, "hidden_dims": [32, 32], "extra_dims": [7]}) + + try: + base_network = BaseNetwork(network_cfg, device="cpu") + assert False + except NotImplementedError: + pass diff --git a/tests/test_models/test_network/test_parallel_mlp.py b/tests/test_models/test_network/test_parallel_mlp.py new file mode 100644 index 0000000..dda431d --- /dev/null +++ b/tests/test_models/test_network/test_parallel_mlp.py @@ -0,0 +1,35 @@ +from omegaconf import DictConfig +import torch + +from cmrl.models.networks.parallel_mlp import ParallelMLP + + +def test_parallel_mlp(): + input_dim = 5 + output_dim = 6 + use_bias = True + extra_dims = [7] + batch_size = 128 + device = "cuda" if torch.cuda.is_available() else "cpu" + + network_cfg = DictConfig( + { + "input_dim": input_dim, + "output_dim": output_dim, + "hidden_dims": [32, 32], + "use_bias": use_bias, + "extra_dims": extra_dims, + } + ) + + mlp = ParallelMLP(network_cfg).to(device) + + model_in = torch.rand((batch_size, input_dim)).to(device) + model_out = mlp(model_in) + assert model_out.shape == ( + *extra_dims, + batch_size, + output_dim, + ) + + assert str(mlp.device).startswith(device) From 3a5a808373bf58da789bd4501045d5189017c7fc Mon Sep 17 00:00:00 2001 From: frank Date: Tue, 25 Oct 2022 02:19:50 +0800 Subject: [PATCH 02/68] :memo: add docs --- docs/about.md | 43 +++++++++++++++++++++++++++++++++++++++++++ docs/index.md | 21 +++++++++++++++++++++ mkdocs.yml | 11 +++++++++++ requirements/dev.txt | 2 ++ 4 files changed, 77 insertions(+) create mode 100644 docs/about.md create mode 100644 docs/index.md create mode 100644 mkdocs.yml diff --git a/docs/about.md b/docs/about.md new file mode 100644 index 0000000..8e22f96 --- /dev/null +++ b/docs/about.md @@ -0,0 +1,43 @@ +# Iphitiden Phoebo caede retiaque solvit genis abdiderat + +## Humo utinam + +Lorem markdownum illos non, somni et evocet Messeniaque diva *agitatis* +nocentius. Templum Erymanthidas prius, duris mihi, iuvenum, nec quod acceptus +una, secuit. + +1. Foret sanguine puniceo +2. Erubuit mittit ipso lenta adspexit arbiter nondum +3. Insanis sum est oves domus nam pars +4. Distinxit verba + +## Iacet venit + +Languore manus est ad prima et caelum sit aristas, ante Styphelumque moris ad +pulsant: vertitur novat. Latrare **minimam coniunx imbribus**: acceptior, ipso +verum demit laudibus non peperi operiri, iussae arva. Ferit fibris gradieris +Dianae, et **dabant dependent** adfixa versa flectit: signumque. + +Indigenae talia, ora rari est in inter coetus *protinus summaque mittitur* +fuerant gravisque agitur et sedibus attulit. Multa capessamus Bacchi: de ut +saeva funera, certamine Chimaera auxiliumque teloque! + +## Imponit requirit armigerae quoque sitimque + +Inplevit nimium. Sub loqui innectens in cincta ripis plangens est. Annua et +stabat Panopeusque naidas audentia quantum videoque quam ipsum. + +## Ingreditur totidem illi + +Est **corpore referam** est rates leti vertitur ab dictu rex quoque sceptra +flamma. Ut quod? Tota nil, horruit hoc. Ubi colla sopore vides dixit qua non +sanguineaque dixerat Iove. Cernis est viribus, **indue** genae verbis solio et +tantum bibulas *et surgere Caesaris* damno Iolaus umquam; scelerate animam? + +1. Testudine tener herosmaxime loco tonitruque +2. Graium pectora pavet sit cum ianua +3. Quod inque +4. De patens ictus spectantia ereptaque constitit falsa +5. Minis modo minor gravet numerusque duorum mediam + +Celsior quid fores, tremore, est quo culpatque terret pati. Anno pietas poples! diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 0000000..04f8734 --- /dev/null +++ b/docs/index.md @@ -0,0 +1,21 @@ +``# Welcome to MkDocs + +For full documentation visit [mkdocs.org](https://www.mkdocs.org). + +## Commands + +* `mkdocs new [dir-name]` - Create a new project. +* `mkdocs serve` - Start the live-reloading docs server. +* `mkdocs build` - Build the documentation site. +* `mkdocs -h` - Print help message and exit. + +## Project layout + + mkdocs.yml # The configuration file. + docs/ + index.md # The documentation homepage. + ... # Other markdown pages, images and other files. + +[//]: # (![mkapi](cmrl.models.layers)) + +::: cmrl.models.layers.ParallelLinear diff --git a/mkdocs.yml b/mkdocs.yml new file mode 100644 index 0000000..716cbbc --- /dev/null +++ b/mkdocs.yml @@ -0,0 +1,11 @@ +site_name: Causal MBRL +theme: readthedocs + +plugins: + - search # necessary for search to work + - mkdocstrings + + +nav: + - Home: index.md + - About: about.md diff --git a/requirements/dev.txt b/requirements/dev.txt index 2bf5b67..ce08726 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -2,3 +2,5 @@ pre-commit>=2.20.0 pytest>=7.1.3 pytest-cov>=4.0.0 flake8>=5.0.4 +mkdocs>=1.4.1 +mkapi>=1.0.14 From 52afe3a4aa58430a04170e222c05987145dad9f0 Mon Sep 17 00:00:00 2001 From: frank Date: Tue, 25 Oct 2022 10:56:19 +0800 Subject: [PATCH 03/68] :beetle: update requirements/dev.txt --- requirements/dev.txt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/requirements/dev.txt b/requirements/dev.txt index ce08726..e47eb23 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -3,4 +3,6 @@ pytest>=7.1.3 pytest-cov>=4.0.0 flake8>=5.0.4 mkdocs>=1.4.1 -mkapi>=1.0.14 +Pygments>2.13.0 +mkdocstrings>=0.19.0 +mkdocstrings-python>=1.0.14 From 158ae90c0c44d557b09795f982ef8a47e6add596 Mon Sep 17 00:00:00 2001 From: frank Date: Tue, 25 Oct 2022 21:39:03 +0800 Subject: [PATCH 04/68] :tada: try to import BaseCausalMechanism --- cmrl/models/base_cuasal_mech.py | 20 +++++++++++++++ cmrl/models/layers.py | 12 ++++++--- cmrl/models/reward_mech/base_reward_mech.py | 24 ++++++------------ .../termination_mech/base_termination_mech.py | 25 ++++++------------- cmrl/models/transition/base_transition.py | 22 +++++----------- 5 files changed, 49 insertions(+), 54 deletions(-) create mode 100644 cmrl/models/base_cuasal_mech.py diff --git a/cmrl/models/base_cuasal_mech.py b/cmrl/models/base_cuasal_mech.py new file mode 100644 index 0000000..eb60e80 --- /dev/null +++ b/cmrl/models/base_cuasal_mech.py @@ -0,0 +1,20 @@ +from typing import Union +from abc import abstractmethod + +import torch +from gym import Space + + +class BaseCausalMechanism: + def __init__(self, obs_space: Space, action_space: Space, deterministic: bool): + self.obs_space = obs_space + self.action_space = action_space + self.deterministic = deterministic + + self.network = None + self.graph = None + pass + + @abstractmethod + def predict(self, obs, action, next_obs=None): + pass diff --git a/cmrl/models/layers.py b/cmrl/models/layers.py index 1db758e..1d84b87 100644 --- a/cmrl/models/layers.py +++ b/cmrl/models/layers.py @@ -70,11 +70,17 @@ def forward(self, x): return xw @property - def device(self): - return next(iter(self.parameters())).device + def device(self) -> torch.device: + """Infer which device this policy lives on by inspecting its parameters. + If it has no parameters, the 'cpu' device is used as a fallback. + + Returns: device + """ + for param in self.parameters(): + return param.device + return torch.device("cpu") def __repr__(self): - # For printing return 'ParallelLinear(input_dims={}, output_dims={}, extra_dims={}, use_bias={}, init_type="{}")'.format( self.input_dim, self.output_dim, str(self.extra_dims), self.use_bias, self.init_type ) diff --git a/cmrl/models/reward_mech/base_reward_mech.py b/cmrl/models/reward_mech/base_reward_mech.py index f2addef..a7a9716 100644 --- a/cmrl/models/reward_mech/base_reward_mech.py +++ b/cmrl/models/reward_mech/base_reward_mech.py @@ -1,26 +1,16 @@ from typing import Union import torch +from gym import Space -from cmrl.models.networks.mlp import EnsembleMLP +from cmrl.models.base_cuasal_mech import BaseCausalMechanism -class BaseRewardMech(EnsembleMLP): - _MODEL_FILENAME = "base_reward_mech.pth" - +class BaseRewardMech(BaseCausalMechanism): def __init__( self, - obs_size: int, - action_size: int, - deterministic: bool = False, - ensemble_num: int = 7, - elite_num: int = 5, - device: Union[str, torch.device] = "cpu", + obs_space: Space, + action_space: Space, + deterministic: bool, ): - super(BaseRewardMech, self).__init__(ensemble_num=ensemble_num, elite_num=elite_num, device=device) - self.obs_size = obs_size - self.action_size = action_size - self.deterministic = deterministic - - def forward(self, state: torch.Tensor, action: torch.Tensor): - pass + super(BaseRewardMech, self).__init__(obs_space, action_space, deterministic) diff --git a/cmrl/models/termination_mech/base_termination_mech.py b/cmrl/models/termination_mech/base_termination_mech.py index cd80284..e78150d 100644 --- a/cmrl/models/termination_mech/base_termination_mech.py +++ b/cmrl/models/termination_mech/base_termination_mech.py @@ -1,26 +1,15 @@ from typing import Union import torch +from gym import Space +from cmrl.models.base_cuasal_mech import BaseCausalMechanism -from cmrl.models.networks.mlp import EnsembleMLP - - -class BaseTerminationMech(EnsembleMLP): - _MODEL_FILENAME = "base_reward_mech.pth" +class BaseTerminationMech(BaseCausalMechanism): def __init__( self, - obs_size: int, - action_size: int, - deterministic: bool = False, - ensemble_num: int = 7, - elite_num: int = 5, - device: Union[str, torch.device] = "cpu", + obs_space: Space, + action_space: Space, + deterministic: bool, ): - super(BaseTerminationMech, self).__init__(ensemble_num=ensemble_num, elite_num=elite_num, device=device) - self.obs_size = obs_size - self.action_size = action_size - self.deterministic = deterministic - - def forward(self, state: torch.Tensor, action: torch.Tensor): - pass + super(BaseTerminationMech, self).__init__(obs_space, action_space, deterministic) diff --git a/cmrl/models/transition/base_transition.py b/cmrl/models/transition/base_transition.py index 7f8f5f7..8515f0a 100644 --- a/cmrl/models/transition/base_transition.py +++ b/cmrl/models/transition/base_transition.py @@ -1,26 +1,16 @@ from typing import Union import torch +from gym import Space -from cmrl.models.networks.mlp import EnsembleMLP +from cmrl.models.base_cuasal_mech import BaseCausalMechanism -class BaseTransition(EnsembleMLP): - _MODEL_FILENAME = "base_ensemble_transition.pth" - +class BaseTransition(BaseCausalMechanism): def __init__( self, - obs_size: int, - action_size: int, + obs_space: Space, + action_space: Space, deterministic: bool, - ensemble_num: int = 7, - elite_num: int = 5, - device: Union[str, torch.device] = "cpu", ): - super(BaseTransition, self).__init__(ensemble_num=ensemble_num, elite_num=elite_num, device=device) - self.obs_size = obs_size - self.action_size = action_size - self.deterministic = deterministic - - def forward(self, state: torch.Tensor, action: torch.Tensor): - pass + super(BaseTransition, self).__init__(obs_space, action_space, deterministic) From d0eaa3531f3c78185c201fb5740306843f0738ba Mon Sep 17 00:00:00 2001 From: frank Date: Fri, 4 Nov 2022 17:37:02 +0800 Subject: [PATCH 05/68] :hammer: introduce causal-mech class --- .../examples/conf/termination_mech/plain.yaml | 0 cmrl/examples/conf/transition/plain.yaml | 24 ++++ cmrl/models/causal_mech/CMI_test.py | 7 ++ cmrl/models/causal_mech/__init__.py | 1 + cmrl/models/causal_mech/base_causal_mech.py | 109 ++++++++++++++++++ cmrl/models/causal_mech/plain_mech.py | 64 ++++++++++ cmrl/models/dynamics.py | 27 +++++ cmrl/models/networks/coder.py | 69 +++++++++++ .../test_models/test_causal_mech/__init__.py | 0 .../test_causal_mech/test_plain_mech.py | 24 ++++ tests/test_models/test_network/test_coder.py | 64 ++++++++++ 11 files changed, 389 insertions(+) create mode 100644 cmrl/examples/conf/termination_mech/plain.yaml create mode 100644 cmrl/examples/conf/transition/plain.yaml create mode 100644 cmrl/models/causal_mech/CMI_test.py create mode 100644 cmrl/models/causal_mech/__init__.py create mode 100644 cmrl/models/causal_mech/base_causal_mech.py create mode 100644 cmrl/models/causal_mech/plain_mech.py create mode 100644 cmrl/models/dynamics.py create mode 100644 cmrl/models/networks/coder.py create mode 100644 tests/test_models/test_causal_mech/__init__.py create mode 100644 tests/test_models/test_causal_mech/test_plain_mech.py create mode 100644 tests/test_models/test_network/test_coder.py diff --git a/cmrl/examples/conf/termination_mech/plain.yaml b/cmrl/examples/conf/termination_mech/plain.yaml new file mode 100644 index 0000000..e69de29 diff --git a/cmrl/examples/conf/transition/plain.yaml b/cmrl/examples/conf/transition/plain.yaml new file mode 100644 index 0000000..febdecc --- /dev/null +++ b/cmrl/examples/conf/transition/plain.yaml @@ -0,0 +1,24 @@ +name: "plain_transition" +#multi_step: "forward_euler_5" +multi_step: "none" +enable_coder: false + +transition: + _target_: cmrl.models.causal_mech.PlainMech + # base causal-mech params + input_variables: ??? + output_variables: ??? + node_dim: 1 + variable_encoders: ??? + variable_decoders: ??? + # network params + deterministic: false + hidden_dims: [200, 200, 200, 200] + ensemble_num: 7 + use_bias: true + activation_fn_cfg: + _target_: torch.nn.SiLU + # forward method + residual: true + # others + device: ${device} diff --git a/cmrl/models/causal_mech/CMI_test.py b/cmrl/models/causal_mech/CMI_test.py new file mode 100644 index 0000000..c67ae18 --- /dev/null +++ b/cmrl/models/causal_mech/CMI_test.py @@ -0,0 +1,7 @@ +from cmrl.models.causal_mech.base_causal_mech import BaseCausalMech + + +class CMIMech(BaseCausalMech): + def __init__(self): + super(CMIMech, self).__init__() + pass diff --git a/cmrl/models/causal_mech/__init__.py b/cmrl/models/causal_mech/__init__.py new file mode 100644 index 0000000..c99657e --- /dev/null +++ b/cmrl/models/causal_mech/__init__.py @@ -0,0 +1 @@ +from cmrl.models.causal_mech.plain_mech import PlainMech diff --git a/cmrl/models/causal_mech/base_causal_mech.py b/cmrl/models/causal_mech/base_causal_mech.py new file mode 100644 index 0000000..f5eb126 --- /dev/null +++ b/cmrl/models/causal_mech/base_causal_mech.py @@ -0,0 +1,109 @@ +from typing import Optional, List, Dict, Union, TypeVar, Type +from abc import abstractmethod + +import torch + +from cmrl.types import Variable, ContinuousVariable, DiscreteVariable +from cmrl.models.networks.base_network import BaseNetwork +from cmrl.models.graphs.base_graph import BaseGraph +from cmrl.models.networks.coder import VariableEncoder, VariableDecoder + + +class BaseCausalMech: + def __init__( + self, + input_variables: List[Variable], + output_variables: List[Variable], + node_dim: int, + variable_encoders: Dict[str, VariableEncoder], + variable_decoders: Dict[str, VariableDecoder], + # forward method + residual: bool = True, + # others + device: Union[str, torch.device] = "cpu", + **kwargs + ): + self.input_variables = input_variables + self.output_variables = output_variables + self.node_dim = node_dim + self.variable_encoders = variable_encoders + self.variable_decoders = variable_decoders + self.residual = residual + self.device = device + + self.input_var_num = len(self.input_variables) + self.output_var_num = len(self.output_variables) + + self.check_coder() + + self.network: Optional[BaseNetwork] = None + self.graph: Optional[BaseGraph] = None + + self.build_network() + self.build_graph() + + def check_coder(self): + assert len(self.input_variables) == len(self.variable_encoders) + assert len(self.output_variables) == len(self.variable_decoders) + + for var in self.input_variables: + assert var.name in self.variable_encoders + encoder = self.variable_encoders[var.name] + assert encoder.node_dim == self.node_dim + + for var in self.output_variables: + assert var.name in self.variable_decoders + decoder = self.variable_decoders[var.name] + assert decoder.node_dim == self.node_dim + + @abstractmethod + def learn(self): + raise NotImplementedError + + @abstractmethod + def build_network(self): + raise NotImplementedError + + @abstractmethod + def build_graph(self): + raise NotImplementedError + + def encode(self, inputs): + pass + + def decode(self, hidden): + pass + + +Causal = TypeVar("Causal", bound=BaseCausalMech) + + +class BaseMultiStepCausalMech(BaseCausalMech): + def __init__( + self, + single_step_mech_class: Type[Causal], + input_variables: List[Variable], + output_variables: List[Variable], + node_dim: int, + variable_encoders: Dict[str, VariableEncoder], + variable_decoders: Dict[str, VariableDecoder], + **kwargs + ): + super(BaseMultiStepCausalMech, self).__init__( + input_variables=input_variables, + output_variables=output_variables, + node_dim=node_dim, + variable_encoders=variable_encoders, + variable_decoders=variable_decoders, + ) + + self.single_step_mech = single_step_mech_class(**kwargs) + pass + + @abstractmethod + def build_network(self): + raise NotImplementedError + + @abstractmethod + def build_graph(self): + raise NotImplementedError diff --git a/cmrl/models/causal_mech/plain_mech.py b/cmrl/models/causal_mech/plain_mech.py new file mode 100644 index 0000000..a15d52c --- /dev/null +++ b/cmrl/models/causal_mech/plain_mech.py @@ -0,0 +1,64 @@ +from typing import Optional, List, Dict, Union +import torch +from omegaconf import DictConfig + +from cmrl.types import Variable +from cmrl.models.networks.parallel_mlp import ParallelMLP +from cmrl.models.graphs.base_graph import BaseGraph +from cmrl.models.causal_mech.base_causal_mech import BaseCausalMech +from cmrl.models.networks.coder import VariableEncoder, VariableDecoder + + +class PlainMech(BaseCausalMech): + def __init__( + self, + # base causal-mech params + input_variables: List[Variable], + output_variables: List[Variable], + node_dim: int, + variable_encoders: Dict[str, VariableEncoder], + variable_decoders: Dict[str, VariableDecoder], + # network params + deterministic: bool = False, + hidden_dims: Optional[List[int]] = None, + ensemble_num: int = 7, + use_bias: bool = True, + activation_fn_cfg: Optional[DictConfig] = None, + # forward method + residual: bool = True, + # others + device: Union[str, torch.device] = "cpu", + **kwargs + ): + self.deterministic = deterministic + self.hidden_dims = hidden_dims if hidden_dims is not None else [200] * 4 + self.ensemble_num = ensemble_num + self.use_bias = use_bias + self.activation_fn_cfg = activation_fn_cfg + + super(PlainMech, self).__init__( + input_variables=input_variables, + output_variables=output_variables, + node_dim=node_dim, + variable_encoders=variable_encoders, + variable_decoders=variable_decoders, + residual=residual, + device=device, + **kwargs + ) + + def build_network(self): + self.network = ParallelMLP( + input_dim=self.input_var_num * self.node_dim, + output_dim=self.output_var_num * self.node_dim, + hidden_dims=self.hidden_dims, + use_bias=self.use_bias, + extra_dims=[self.ensemble_num], + activation_fn_cfg=self.activation_fn_cfg, + ) + + def build_graph(self): + self.graph = None + + def learn(self): + pass diff --git a/cmrl/models/dynamics.py b/cmrl/models/dynamics.py new file mode 100644 index 0000000..4b47f18 --- /dev/null +++ b/cmrl/models/dynamics.py @@ -0,0 +1,27 @@ +import abc +import collections +import pathlib +from typing import Dict, List, Optional, Tuple, Union + +import numpy as np +import torch +from stable_baselines3.common.logger import Logger +from stable_baselines3.common.buffers import ReplayBuffer + +from cmrl.models.reward_mech.base_reward_mech import BaseRewardMech +from cmrl.models.termination_mech.base_termination_mech import BaseTerminationMech +from cmrl.models.transition.base_transition import BaseTransition +from cmrl.types import InteractionBatch +from cmrl.util.transition_iterator import BootstrapIterator, TransitionIterator +from cmrl.models.causal_mech.base_causal_mech import BaseCausalMech + + +def split_dict(old_dict: Dict, need_keys: List[str]): + return dict([(key, old_dict[key]) for key in need_keys]) + + +class Dynamics: + def __init__( + self, transition: BaseCausalMech, reward_mech: Optional[BaseCausalMech], terminal_mech: Optional[BaseCausalMech] + ): + pass diff --git a/cmrl/models/networks/coder.py b/cmrl/models/networks/coder.py new file mode 100644 index 0000000..597394c --- /dev/null +++ b/cmrl/models/networks/coder.py @@ -0,0 +1,69 @@ +from typing import List, Optional, Sequence, Union + +import torch +import torch.nn as nn +from omegaconf import DictConfig + +from cmrl.types import Variable, DiscreteVariable, ContinuousVariable +from cmrl.models.networks.base_network import BaseNetwork, create_activation + + +class VariableEncoder(BaseNetwork): + def __init__( + self, + variable: Variable, + node_dim: int, + activation_fn_cfg: Optional[DictConfig] = None, + ): + self.variable = variable + self.node_dim = node_dim + self.name = "{}_encoder".format(variable.name) + self.activation_fn_cfg = activation_fn_cfg + + super(VariableEncoder, self).__init__() + self._model_filename = "{}.pth".format(self.name) + + def build(self): + layers = [] + if isinstance(self.variable, ContinuousVariable): + layers.append(nn.Linear(self.variable.dim, self.node_dim)) + elif isinstance(self.variable, DiscreteVariable): + layers.append(nn.Linear(self.variable.n, self.node_dim)) + else: + raise NotImplementedError + + layers.append(create_activation(self.activation_fn_cfg)) + self._layers = nn.ModuleList(layers) + + +class VariableDecoder(BaseNetwork): + def __init__( + self, + variable: Variable, + node_dim: int, + normal_distribution: bool = False, + activation_fn_cfg: Optional[DictConfig] = None, + ): + self.variable = variable + self.node_dim = node_dim + self.normal_distribution = normal_distribution + self.name = "{}_decoder".format(variable.name) + self.activation_fn_cfg = activation_fn_cfg + + super(VariableDecoder, self).__init__() + self._model_filename = "{}.pth".format(self.name) + + def build(self): + layers = [] + if isinstance(self.variable, ContinuousVariable): + if self.normal_distribution: + layers.append(nn.Linear(self.node_dim, self.variable.dim * 2)) + else: + layers.append(nn.Linear(self.node_dim, self.variable.dim)) + elif isinstance(self.variable, DiscreteVariable): + layers.append(nn.Linear(self.node_dim, self.variable.n)) + layers.append(nn.Softmax()) + else: + raise NotImplementedError + + self._layers = nn.ModuleList(layers) diff --git a/tests/test_models/test_causal_mech/__init__.py b/tests/test_models/test_causal_mech/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_models/test_causal_mech/test_plain_mech.py b/tests/test_models/test_causal_mech/test_plain_mech.py new file mode 100644 index 0000000..3cc90c4 --- /dev/null +++ b/tests/test_models/test_causal_mech/test_plain_mech.py @@ -0,0 +1,24 @@ +from cmrl.models.causal_mech.plain_mech import PlainMech +from cmrl.types import Variable, ContinuousVariable, DiscreteVariable + + +def test_without_coder(): + node_dim = 1 + + input_variables = [ContinuousVariable(name="state0", dim=node_dim), ContinuousVariable(name="action0", dim=node_dim)] + output_variables = [ContinuousVariable(name="state0", dim=node_dim)] + + mech = PlainMech( + input_variables=input_variables, + output_variables=output_variables, + node_dim=node_dim, + variable_encoders={"state0": None, "action0": None}, + variable_decoders={"state0": None}, + ) + + +def test_single_dim_continuous(): + input_variables = [ContinuousVariable(1), ContinuousVariable(1)] + output_variables = [ContinuousVariable(1)] + + mech = PlainMech(input_variables=input_variables, output_variables=output_variables) diff --git a/tests/test_models/test_network/test_coder.py b/tests/test_models/test_network/test_coder.py new file mode 100644 index 0000000..02041e9 --- /dev/null +++ b/tests/test_models/test_network/test_coder.py @@ -0,0 +1,64 @@ +import torch +import numpy as np +from torch.nn.functional import one_hot + +from cmrl.models.networks.coder import VariableEncoder, VariableDecoder +from cmrl.types import Variable, ContinuousVariable, DiscreteVariable + + +def test_continuous_encoder(): + var_dim = 3 + node_dim = 5 + batch_size = 128 + + var = ContinuousVariable(name="state0", dim=var_dim) + + encoder = VariableEncoder(var, node_dim) + inputs = torch.rand(batch_size, var_dim) + outputs = encoder(inputs) + + assert outputs.shape == (batch_size, node_dim) + + +def test_discrete_encoder(): + var_n = 3 + node_dim = 5 + batch_size = 128 + + var = DiscreteVariable(name="state0", n=var_n) + + encoder = VariableEncoder(var, node_dim) + inputs = one_hot(torch.randint(3, (batch_size,))).to(torch.float32) + outputs = encoder(inputs) + + assert outputs.shape == (batch_size, node_dim) + + +def test_continuous_decoder(): + var_dim = 3 + node_dim = 5 + batch_size = 128 + + var = ContinuousVariable(name="state0", dim=var_dim) + + decoder = VariableDecoder(var, node_dim) + inputs = torch.rand(batch_size, node_dim) + outputs = decoder(inputs) + + assert outputs.shape == (batch_size, var_dim) + + +def test_discrete_decoder(): + var_n = 3 + node_dim = 5 + batch_size = 128 + + var = DiscreteVariable(name="state0", n=var_n) + + decoder = VariableDecoder(var, node_dim) + inputs = torch.rand(batch_size, node_dim) + outputs = decoder(inputs) + + assert outputs.shape == (batch_size, var_n) + batch_sum = outputs.detach().numpy().sum(axis=1) + assert np.allclose(batch_sum, 1) From b65c71ea058c47d3cddeb598eab276519ae15582 Mon Sep 17 00:00:00 2001 From: frank Date: Fri, 4 Nov 2022 17:38:18 +0800 Subject: [PATCH 06/68] :beetle: fix parallel-linear bug --- cmrl/algorithms/util.py | 27 ++++---- .../dynamics/constraint_based_dynamics.yaml | 2 +- .../conf/dynamics/plain_dynamics.yaml | 2 +- cmrl/examples/conf/main.yaml | 2 +- cmrl/examples/conf/task/BIPS.yaml | 8 +-- cmrl/models/layers.py | 26 ++++---- cmrl/models/networks/base_network.py | 45 +++++++++----- cmrl/models/networks/mlp.py | 3 +- cmrl/models/networks/parallel_mlp.py | 61 +++++++++++-------- .../one_step/external_mask_transition.py | 6 +- cmrl/types.py | 16 +++++ docs/index.md | 2 - .../test_network/test_base_network.py | 4 +- .../test_network/test_parallel_mlp.py | 5 +- 14 files changed, 128 insertions(+), 81 deletions(-) diff --git a/cmrl/algorithms/util.py b/cmrl/algorithms/util.py index 6e901c4..7f14781 100644 --- a/cmrl/algorithms/util.py +++ b/cmrl/algorithms/util.py @@ -105,20 +105,23 @@ def setup_fake_env( return fake_eval_env -def load_offline_data(cfg: DictConfig, env, replay_buffer: ReplayBuffer): +def load_offline_data(env, replay_buffer: ReplayBuffer, dataset_name: str, use_ratio: float = 1): assert hasattr(env, "get_dataset"), "env must have `get_dataset` method" - params, dataset_type = cfg.task.env.split("___")[-2:] - data_dict = env.get_dataset("{}-{}".format(params, dataset_type)) + data_dict = env.get_dataset(dataset_name) all_data_num = len(data_dict["observations"]) - sample_data_num = int(cfg.task.use_ratio * all_data_num) + sample_data_num = int(use_ratio * all_data_num) sample_idx = np.random.permutation(all_data_num)[:sample_data_num] - replay_buffer.extend( - data_dict["observations"][sample_idx], - data_dict["next_observations"][sample_idx], - data_dict["actions"][sample_idx], - data_dict["rewards"][sample_idx], - data_dict["terminals"][sample_idx].astype(bool) | data_dict["timeouts"][sample_idx].astype(bool), - [{}] * sample_data_num, - ) + assert replay_buffer.n_envs == 1 + assert replay_buffer.buffer_size >= sample_data_num + + if sample_data_num == replay_buffer.buffer_size: + replay_buffer.full = True + replay_buffer.pos = 0 + else: + replay_buffer.pos = sample_data_num + + # set all data + for attr in ["observations", "next_observations", "actions", "rewards", "dones", "timeouts"]: + getattr(replay_buffer, attr)[:sample_data_num, 0] = data_dict[attr][sample_idx] diff --git a/cmrl/examples/conf/dynamics/constraint_based_dynamics.yaml b/cmrl/examples/conf/dynamics/constraint_based_dynamics.yaml index 2fefed1..8c7fc9a 100644 --- a/cmrl/examples/conf/dynamics/constraint_based_dynamics.yaml +++ b/cmrl/examples/conf/dynamics/constraint_based_dynamics.yaml @@ -3,7 +3,7 @@ name: constraint_based_dynamics multi_step: ${task.multi_step} transition: - _target_: cmrl.models.transition.ExternalMaskEnsembleGaussianTransition + _target_: cmrl.models.transition.ExternalMaskTransition # transition info obs_size: ??? action_size: ??? diff --git a/cmrl/examples/conf/dynamics/plain_dynamics.yaml b/cmrl/examples/conf/dynamics/plain_dynamics.yaml index 74914c2..933ea15 100644 --- a/cmrl/examples/conf/dynamics/plain_dynamics.yaml +++ b/cmrl/examples/conf/dynamics/plain_dynamics.yaml @@ -3,7 +3,7 @@ name: plain_dynamics multi_step: ${task.multi_step} transition: - _target_: cmrl.models.transition.PlainEnsembleGaussianTransition + _target_: cmrl.models.transition.PlainTransition # transition info obs_size: ??? action_size: ??? diff --git a/cmrl/examples/conf/main.yaml b/cmrl/examples/conf/main.yaml index 65eda8e..4a580ac 100644 --- a/cmrl/examples/conf/main.yaml +++ b/cmrl/examples/conf/main.yaml @@ -7,7 +7,7 @@ defaults: seed: 0 device: "cuda:0" -exp_name: default +exp_name: causal_mstep wandb: false root_dir: "./exp" diff --git a/cmrl/examples/conf/task/BIPS.yaml b/cmrl/examples/conf/task/BIPS.yaml index c9c0ea2..39fa95c 100644 --- a/cmrl/examples/conf/task/BIPS.yaml +++ b/cmrl/examples/conf/task/BIPS.yaml @@ -1,7 +1,7 @@ env: "emei___BoundaryInvertedPendulumSwingUp-v0___freq_rate=${task.freq_rate}&time_step=${task.time_step}___${task.dataset}" dataset: "SAC-expert-replay" -freq_rate: 1 -time_step: 0.02 +freq_rate: 5 +time_step: 0.05 # basic RL params num_steps: 300000 @@ -15,7 +15,7 @@ learning_reward: false learning_terminal: false ensemble_num: 7 elite_num: 5 -multi_step: "none" +multi_step: "forward_euler_5" # conditional mutual information test(causal discovery) oracle: true @@ -26,7 +26,7 @@ update_causal_mask_ratio: 0.25 discovery_schedule: [ 1, 30, 250, 250 ] # offline -penalty_coeff: 0.5 +penalty_coeff: 0.2 use_ratio: 1 # dyna diff --git a/cmrl/models/layers.py b/cmrl/models/layers.py index 1db758e..f93b4b3 100644 --- a/cmrl/models/layers.py +++ b/cmrl/models/layers.py @@ -18,15 +18,16 @@ def __init__( use_bias: bool = True, init_type: str = "truncated_normal", ): - """Linear layer with the same properties as Parallel MLP. It effectively applies N independent - linear layers in parallel. + """Linear layer with the same properties as Parallel MLP. It effectively applies N independent linear layers + in parallel. Args: - input_dim: Number of input dimensions per network. - output_dim: s - extra_dims: s - use_bias: s - init_type: s + input_dim: Number of input dimensions per layer. + output_dim: Number of output dimensions per layer. + extra_dims: Number of neural networks to have in parallel (e.g. number of variables). Can have multiple + dimensions if needed. + use_bias: Weather using bias in this layer. + init_type: How to initialize weights and biases. """ super().__init__() self.input_dim = input_dim @@ -41,9 +42,14 @@ def __init__( else: self.use_bias = False - self.init() + self.init_params() - def init(self): + def init_params(self): + """Initialize weights and biases. Currently, only `kaiming_uniform` and `truncated_normal` are supported. + + Returns: None + + """ if self.init_type == "kaiming_uniform": nn.init.kaiming_uniform_(self.weight, nonlinearity="relu") elif self.init_type == "truncated_normal": @@ -54,7 +60,6 @@ def init(self): raise NotImplementedError def forward(self, x): - # Shape preparation x_extra_dims = x.shape[:-2] if len(x_extra_dims) > 0: for i in range(len(x_extra_dims)): @@ -74,7 +79,6 @@ def device(self): return next(iter(self.parameters())).device def __repr__(self): - # For printing return 'ParallelLinear(input_dims={}, output_dims={}, extra_dims={}, use_bias={}, init_type="{}")'.format( self.input_dim, self.output_dim, str(self.extra_dims), self.use_bias, self.init_type ) diff --git a/cmrl/models/networks/base_network.py b/cmrl/models/networks/base_network.py index 7e1b9f6..5af379a 100644 --- a/cmrl/models/networks/base_network.py +++ b/cmrl/models/networks/base_network.py @@ -4,6 +4,7 @@ import torch import torch.nn as nn +import hydra from omegaconf import DictConfig from cmrl.models.util import gaussian_nll @@ -11,49 +12,63 @@ class BaseNetwork(nn.Module): - _MODEL_FILENAME = "base_network.pth" + def __init__(self, **kwargs): + """Base class of all neural network. - def __init__(self, network_cfg: DictConfig): + Args: + network_cfg: + """ super(BaseNetwork, self).__init__() - self.network_cfg = network_cfg + self._model_filename = "base_network.pth" + self._save_attrs: List[str] = ["state_dict"] + self._layers: Optional[nn.ModuleList] = None - self._model_save_attrs: List[str] = ["state_dict"] - self._layers: Optional[nn.Module] = None - - self.build_network() + self.build() def save(self, save_dir: Union[str, pathlib.Path]): """Saves the model to the given directory.""" model_dict = {} - for attr in self._model_save_attrs: + for attr in self._save_attrs: if attr == "state_dict": model_dict["state_dict"] = self.state_dict() else: model_dict[attr] = getattr(self, attr) - torch.save(model_dict, pathlib.Path(save_dir) / self._MODEL_FILENAME) + torch.save(model_dict, pathlib.Path(save_dir) / self._model_filename) def load(self, load_dir: Union[str, pathlib.Path]): """Loads the model from the given path.""" - model_dict = torch.load(pathlib.Path(load_dir) / self._MODEL_FILENAME, map_location=self.device) + model_dict = torch.load(pathlib.Path(load_dir) / self._model_filename, map_location=self.device) for attr in model_dict: if attr == "state_dict": self.load_state_dict(model_dict["state_dict"]) else: - getattr(self, "set_" + attr)(model_dict[attr]) + getattr(self, attr)(model_dict[attr]) + + def forward(self, x): + for layer in self._layers: + x = layer(x) + return x @abstractmethod - def build_network(self): + def build(self): raise NotImplementedError @property def save_attrs(self): - return self._model_save_attrs + return self._save_attrs @property - def model_file_name(self): - return self._MODEL_FILENAME + def model_filename(self): + return self._model_filename @property def device(self): return next(iter(self.parameters())).device + + +def create_activation(activation_fn_cfg: DictConfig): + if activation_fn_cfg is None: + return nn.ReLU() + else: + return hydra.utils.instantiate(activation_fn_cfg) diff --git a/cmrl/models/networks/mlp.py b/cmrl/models/networks/mlp.py index c4f15b5..7390ebe 100644 --- a/cmrl/models/networks/mlp.py +++ b/cmrl/models/networks/mlp.py @@ -4,6 +4,7 @@ import torch import torch.nn as nn +import torch.nn.functional as F import numpy as np from omegaconf import DictConfig @@ -65,7 +66,7 @@ def load(self, load_dir: Union[str, pathlib.Path], load_device: Optional[str] = getattr(self, "set_" + attr)(model_dict[attr]) def create_linear_layer(self, l_in, l_out): - return ParallelLinear(l_in, l_out) + return ParallelLinear(l_in, l_out, extra_dims=[self.ensemble_num]) def get_mse_loss(self, model_in: Dict[(str, torch.Tensor)], target: torch.Tensor) -> torch.Tensor: pred_mean, pred_logvar = self.forward(**model_in) diff --git a/cmrl/models/networks/parallel_mlp.py b/cmrl/models/networks/parallel_mlp.py index 69342f8..3a97ffa 100644 --- a/cmrl/models/networks/parallel_mlp.py +++ b/cmrl/models/networks/parallel_mlp.py @@ -9,36 +9,47 @@ from cmrl.models.util import gaussian_nll from cmrl.models.layers import ParallelLinear -from cmrl.models.networks.base_network import BaseNetwork +from cmrl.models.networks.base_network import BaseNetwork, create_activation # partial from https://github.com/phlippe/ENCO/blob/main/causal_discovery/multivariable_mlp.py class ParallelMLP(BaseNetwork): - _MODEL_FILENAME = "parallel_mlp.pth" - - def __init__(self, network_cfg: DictConfig): - super().__init__(network_cfg) - - def build_network(self): - activation_fn_cfg = self.network_cfg.get("activation_fn_cfg", None) - extra_dims = self.network_cfg.get("extra_dims", None) - - def create_activation(): - if activation_fn_cfg is None: - return nn.ReLU() - else: - return hydra.utils.instantiate(activation_fn_cfg) - + def __init__( + self, + input_dim: int, + output_dim: int, + extra_dims: Optional[List[int]] = None, + hidden_dims: Optional[List[int]] = None, + use_bias: bool = True, + init_type: str = "truncated_normal", + activation_fn_cfg: Optional[DictConfig] = None, + **kwargs + ): + self.input_dim = input_dim + self.output_dim = output_dim + self.extra_dims = extra_dims + self.hidden_dims = hidden_dims + self.use_bias = use_bias + self.init_type = init_type + self.activation_fn_cfg = activation_fn_cfg + + super().__init__(**kwargs) + self._model_filename = "parallel_mlp.pth" + + def build(self): layers = [] - hidden_dims = [self.network_cfg.input_dim] + self.network_cfg.hidden_dims + hidden_dims = [self.input_dim] + self.hidden_dims for i in range(len(hidden_dims) - 1): - layers += [ParallelLinear(input_dim=hidden_dims[i], output_dim=hidden_dims[i + 1], extra_dims=extra_dims)] - layers += [create_activation()] - layers += [ParallelLinear(input_dim=hidden_dims[-1], output_dim=self.network_cfg.output_dim, extra_dims=extra_dims)] + layers += [ + ParallelLinear( + input_dim=hidden_dims[i], output_dim=hidden_dims[i + 1], extra_dims=self.extra_dims, use_bias=self.use_bias + ) + ] + layers += [create_activation(self.activation_fn_cfg)] + layers += [ + ParallelLinear( + input_dim=hidden_dims[-1], output_dim=self.output_dim, extra_dims=self.extra_dims, use_bias=self.use_bias + ) + ] self._layers = nn.ModuleList(layers) - - def forward(self, x): - for layer in self._layers: - x = layer(x) - return x diff --git a/cmrl/models/transition/one_step/external_mask_transition.py b/cmrl/models/transition/one_step/external_mask_transition.py index d0fa815..fea816c 100644 --- a/cmrl/models/transition/one_step/external_mask_transition.py +++ b/cmrl/models/transition/one_step/external_mask_transition.py @@ -8,7 +8,7 @@ import cmrl.types -# from cmrl.models.layers import ParallelEnsembleLinearLayer, truncated_normal_init +from cmrl.models.layers import ParallelLinear from cmrl.models.transition.base_transition import BaseTransition from cmrl.models.util import to_tensor @@ -104,8 +104,8 @@ def create_activation(): # self.apply(truncated_normal_init) self.to(self.device) - # def create_linear_layer(self, l_in, l_out): - # return ParallelEnsembleLinearLayer(l_in, l_out, parallel_num=self.obs_size, ensemble_num=self.ensemble_num) + def create_linear_layer(self, l_in, l_out): + return ParallelLinear(l_in, l_out, extra_dims=[self.obs_size, self.ensemble_num]) def set_input_mask(self, mask: cmrl.types.TensorType): self._input_mask = to_tensor(mask).to(self.device) diff --git a/cmrl/types.py b/cmrl/types.py index 1bb28b4..de6e8b9 100644 --- a/cmrl/types.py +++ b/cmrl/types.py @@ -78,3 +78,19 @@ def add_new_batch_dim(self, batch_size: int): ModelInput = Union[torch.Tensor, InteractionBatch] + + +@dataclass +class Variable: + name: str + pass + + +@dataclass +class ContinuousVariable(Variable): + dim: int + + +@dataclass +class DiscreteVariable(Variable): + n: int diff --git a/docs/index.md b/docs/index.md index 04f8734..d7703e7 100644 --- a/docs/index.md +++ b/docs/index.md @@ -16,6 +16,4 @@ For full documentation visit [mkdocs.org](https://www.mkdocs.org). index.md # The documentation homepage. ... # Other markdown pages, images and other files. -[//]: # (![mkapi](cmrl.models.layers)) - ::: cmrl.models.layers.ParallelLinear diff --git a/tests/test_models/test_network/test_base_network.py b/tests/test_models/test_network/test_base_network.py index f8d409a..81772d7 100644 --- a/tests/test_models/test_network/test_base_network.py +++ b/tests/test_models/test_network/test_base_network.py @@ -4,10 +4,8 @@ def test_base_network(): - network_cfg = DictConfig({"input_dim": 5, "output_dim": 10, "hidden_dims": [32, 32], "extra_dims": [7]}) - try: - base_network = BaseNetwork(network_cfg, device="cpu") + base_network = BaseNetwork(device="cpu") assert False except NotImplementedError: pass diff --git a/tests/test_models/test_network/test_parallel_mlp.py b/tests/test_models/test_network/test_parallel_mlp.py index dda431d..2624cae 100644 --- a/tests/test_models/test_network/test_parallel_mlp.py +++ b/tests/test_models/test_network/test_parallel_mlp.py @@ -12,17 +12,18 @@ def test_parallel_mlp(): batch_size = 128 device = "cuda" if torch.cuda.is_available() else "cpu" - network_cfg = DictConfig( + network_cfg = dict( { "input_dim": input_dim, "output_dim": output_dim, "hidden_dims": [32, 32], "use_bias": use_bias, "extra_dims": extra_dims, + "activation_fn_cfg": DictConfig({"_target_": "torch.nn.SiLU"}), } ) - mlp = ParallelMLP(network_cfg).to(device) + mlp = ParallelMLP(**network_cfg).to(device) model_in = torch.rand((batch_size, input_dim)).to(device) model_out = mlp(model_in) From c16136240ee0ad1cb96a2b8fae95993e491a74a8 Mon Sep 17 00:00:00 2001 From: frank Date: Fri, 4 Nov 2022 17:39:06 +0800 Subject: [PATCH 07/68] :tada: create OfflineDataset --- cmrl/models/data_loader.py | 122 ++++++++++++++++++++++++++ tests/test_models/test_data_loader.py | 18 ++++ 2 files changed, 140 insertions(+) create mode 100644 cmrl/models/data_loader.py create mode 100644 tests/test_models/test_data_loader.py diff --git a/cmrl/models/data_loader.py b/cmrl/models/data_loader.py new file mode 100644 index 0000000..f776831 --- /dev/null +++ b/cmrl/models/data_loader.py @@ -0,0 +1,122 @@ +import gym +from gym import spaces +import torch +from torch.utils.data import Dataset, DataLoader, Sampler +import numpy as np +from stable_baselines3.common.buffers import ReplayBuffer, DictReplayBuffer + + +class OfflineDataset(Dataset): + def __init__( + self, + replay_buffer: ReplayBuffer, + observation_space: spaces.Space, + action_space: spaces.Space, + mech: str, + is_valid: bool = False, + train_ratio: float = 0.8, + seed: int = 10086, + ): + assert mech in ["transition", "reward_mech", "termination_mech"] + # dict action is not supported by SB3(so not done by cmrl) + assert not isinstance(action_space, spaces.Dict) + + self.replay_buffer = replay_buffer + self.observation_space = observation_space + self.action_space = action_space + self.mech = mech + self.is_valid = is_valid + self.train_ratio = train_ratio + self.seed = seed + + self.inputs = None + self.outputs = None + + size = self.load_from_buffer() + + np.random.seed(seed) + self.permutation = np.random.permutation(size) + if is_valid: # for valid set + self.indexes = self.permutation[int(size * train_ratio) :] + else: # for train set + self.indexes = self.permutation[: int(size * train_ratio)] + self.length = len(self.indexes) + + def load_from_buffer(self): + size = self.replay_buffer.buffer_size if self.replay_buffer.full else self.replay_buffer.pos + if isinstance(self.replay_buffer, DictReplayBuffer): + # TODO: DictReplayBuffer case + raise NotImplementedError + else: + observations = self.replay_buffer.observations[:size, 0].astype(np.float32) + assert len(observations.shape) == 2 + next_observations = self.replay_buffer.next_observations[:size, 0].astype(np.float32) + + observations_dict = dict([("obs_{}".format(i), obs[:, None]) for i, obs in enumerate(observations.T)]) + next_observations_dict = dict([("obs_{}".format(i), obs[:, None]) for i, obs in enumerate(next_observations.T)]) + + assert isinstance(self.observation_space, spaces.Box) + # TODO: other spaces for observation and action(e.g. one-hot for spaces.Discrete) + # see: https://github.com/DLR-RM/stable-baselines3/blob/master/stable_baselines3/common/preprocessing.py#L85 + + actions = self.replay_buffer.actions[:size, 0] + rewards = self.replay_buffer.rewards[:size, 0] + dones = self.replay_buffer.dones[:size, 0] + timeouts = self.replay_buffer.timeouts[:size, 0] + + actions_dict = dict([("act_{}".format(i), obs[:, None]) for i, obs in enumerate(actions.T)]) + rewards_dict = {"reward": rewards[:, None]} + terminals_dict = {"terminal": (dones * (1 - timeouts))[:, None]} + + self.inputs = observations_dict.update(actions_dict) + if self.mech == "transition": + self.outputs = next_observations_dict + elif self.mech == "reward_mech": + self.outputs = rewards_dict + else: + self.outputs = terminals_dict + + return size + + def __getitem__(self, item): + index = self.indexes[item] + + inputs = [self.inputs[key][index] for key in self.inputs] + outputs = [self.outputs[key][index] for key in self.outputs] + return inputs, outputs + + def __len__(self): + return self.length + + +class EnsembleOfflineDataset(OfflineDataset): + def __init__( + self, + replay_buffer: ReplayBuffer, + observation_space: spaces.Space, + action_space: spaces.Space, + mech: str, + is_valid: bool = False, + train_ratio: float = 0.8, + ensemble_num: int = 7, + seed: int = 10086, + ): + super(EnsembleOfflineDataset, self).__init__( + replay_buffer=replay_buffer, + observation_space=observation_space, + action_space=action_space, + mech=mech, + is_valid=is_valid, + train_ratio=train_ratio, + seed=seed, + ) + + def __getitem__(self, item): + inputs = [torch.from_numpy(self.inputs[index[item]]).float() for index in self.indexes_list] + inputs = torch.stack(inputs) + targets = [torch.from_numpy(self.targets[index[item]]).float() for index in self.indexes_list] + targets = torch.stack(targets) + return inputs, targets + + def __len__(self): + return self.length diff --git a/tests/test_models/test_data_loader.py b/tests/test_models/test_data_loader.py new file mode 100644 index 0000000..fd58e7f --- /dev/null +++ b/tests/test_models/test_data_loader.py @@ -0,0 +1,18 @@ +import gym +import emei +from stable_baselines3.common.buffers import ReplayBuffer + +from cmrl.models.data_loader import OfflineDataset +from cmrl.algorithms.util import load_offline_data + + +def test_offline_dataset(): + env = gym.make("BoundaryInvertedPendulumSwingUp-v0", freq_rate=1, time_step=0.02) + # assert isinstance(env, emei.EmeiEnv) + + real_replay_buffer = ReplayBuffer( + int(1e5), env.observation_space, env.action_space, "cpu", handle_timeout_termination=False + ) + load_offline_data(env, real_replay_buffer, "SAC-expert-replay") + + OfflineDataset(real_replay_buffer, env.observation_space, env.action_space, mech="transition") From b0ffd7a6b4f713e30e4ad938fde5b6f917d38fcb Mon Sep 17 00:00:00 2001 From: frank Date: Fri, 4 Nov 2022 18:47:03 +0800 Subject: [PATCH 08/68] :tada: add OfflineDataset and EnsembleOfflineDataset --- cmrl/models/data_loader.py | 65 +++++++++++--------- tests/test_models/test_data_loader.py | 87 ++++++++++++++++++++++++++- 2 files changed, 120 insertions(+), 32 deletions(-) diff --git a/cmrl/models/data_loader.py b/cmrl/models/data_loader.py index f776831..050b826 100644 --- a/cmrl/models/data_loader.py +++ b/cmrl/models/data_loader.py @@ -29,28 +29,31 @@ def __init__( self.train_ratio = train_ratio self.seed = seed + self.size = self.replay_buffer.buffer_size if self.replay_buffer.full else self.replay_buffer.pos + self.inputs = None self.outputs = None + self.load_from_buffer() - size = self.load_from_buffer() + self.indexes = None + self.build_indexes() - np.random.seed(seed) - self.permutation = np.random.permutation(size) - if is_valid: # for valid set - self.indexes = self.permutation[int(size * train_ratio) :] + def build_indexes(self): + np.random.seed(self.seed) + permutation = np.random.permutation(self.size) + if self.is_valid: # for valid set + self.indexes = permutation[int(self.size * self.train_ratio) :] else: # for train set - self.indexes = self.permutation[: int(size * train_ratio)] - self.length = len(self.indexes) + self.indexes = permutation[: int(self.size * self.train_ratio)] def load_from_buffer(self): - size = self.replay_buffer.buffer_size if self.replay_buffer.full else self.replay_buffer.pos if isinstance(self.replay_buffer, DictReplayBuffer): # TODO: DictReplayBuffer case raise NotImplementedError else: - observations = self.replay_buffer.observations[:size, 0].astype(np.float32) + observations = self.replay_buffer.observations[: self.size, 0].astype(np.float32) assert len(observations.shape) == 2 - next_observations = self.replay_buffer.next_observations[:size, 0].astype(np.float32) + next_observations = self.replay_buffer.next_observations[: self.size, 0].astype(np.float32) observations_dict = dict([("obs_{}".format(i), obs[:, None]) for i, obs in enumerate(observations.T)]) next_observations_dict = dict([("obs_{}".format(i), obs[:, None]) for i, obs in enumerate(next_observations.T)]) @@ -59,16 +62,19 @@ def load_from_buffer(self): # TODO: other spaces for observation and action(e.g. one-hot for spaces.Discrete) # see: https://github.com/DLR-RM/stable-baselines3/blob/master/stable_baselines3/common/preprocessing.py#L85 - actions = self.replay_buffer.actions[:size, 0] - rewards = self.replay_buffer.rewards[:size, 0] - dones = self.replay_buffer.dones[:size, 0] - timeouts = self.replay_buffer.timeouts[:size, 0] + actions = self.replay_buffer.actions[: self.size, 0] + rewards = self.replay_buffer.rewards[: self.size, 0] + dones = self.replay_buffer.dones[: self.size, 0] + timeouts = self.replay_buffer.timeouts[: self.size, 0] actions_dict = dict([("act_{}".format(i), obs[:, None]) for i, obs in enumerate(actions.T)]) rewards_dict = {"reward": rewards[:, None]} terminals_dict = {"terminal": (dones * (1 - timeouts))[:, None]} - self.inputs = observations_dict.update(actions_dict) + self.inputs = {} + self.inputs.update(observations_dict) + self.inputs.update(actions_dict) + if self.mech == "transition": self.outputs = next_observations_dict elif self.mech == "reward_mech": @@ -76,17 +82,15 @@ def load_from_buffer(self): else: self.outputs = terminals_dict - return size - def __getitem__(self, item): index = self.indexes[item] - inputs = [self.inputs[key][index] for key in self.inputs] - outputs = [self.outputs[key][index] for key in self.outputs] + inputs = dict([(key, self.inputs[key][index]) for key in self.inputs]) + outputs = dict([(key, self.outputs[key][index]) for key in self.outputs]) return inputs, outputs def __len__(self): - return self.length + return len(self.indexes) class EnsembleOfflineDataset(OfflineDataset): @@ -101,6 +105,8 @@ def __init__( ensemble_num: int = 7, seed: int = 10086, ): + self.ensemble_num = ensemble_num + super(EnsembleOfflineDataset, self).__init__( replay_buffer=replay_buffer, observation_space=observation_space, @@ -111,12 +117,13 @@ def __init__( seed=seed, ) - def __getitem__(self, item): - inputs = [torch.from_numpy(self.inputs[index[item]]).float() for index in self.indexes_list] - inputs = torch.stack(inputs) - targets = [torch.from_numpy(self.targets[index[item]]).float() for index in self.indexes_list] - targets = torch.stack(targets) - return inputs, targets - - def __len__(self): - return self.length + def build_indexes(self): + np.random.seed(self.seed) + if self.is_valid: # for valid set + self.indexes = np.array( + [np.random.permutation(self.size)[int(self.size * self.train_ratio) :] for _ in range(self.ensemble_num)] + ).T + else: # for train set + self.indexes = np.array( + [np.random.permutation(self.size)[: int(self.size * self.train_ratio)] for _ in range(self.ensemble_num)] + ).T diff --git a/tests/test_models/test_data_loader.py b/tests/test_models/test_data_loader.py index fd58e7f..f5b4afa 100644 --- a/tests/test_models/test_data_loader.py +++ b/tests/test_models/test_data_loader.py @@ -1,8 +1,9 @@ import gym import emei from stable_baselines3.common.buffers import ReplayBuffer +from torch.utils.data import DataLoader -from cmrl.models.data_loader import OfflineDataset +from cmrl.models.data_loader import OfflineDataset, EnsembleOfflineDataset from cmrl.algorithms.util import load_offline_data @@ -13,6 +14,86 @@ def test_offline_dataset(): real_replay_buffer = ReplayBuffer( int(1e5), env.observation_space, env.action_space, "cpu", handle_timeout_termination=False ) - load_offline_data(env, real_replay_buffer, "SAC-expert-replay") + load_offline_data(env, real_replay_buffer, "SAC-expert-replay", use_ratio=0.1) - OfflineDataset(real_replay_buffer, env.observation_space, env.action_space, mech="transition") + # test for transition + dataset = OfflineDataset(real_replay_buffer, env.observation_space, env.action_space, mech="transition") + loader = DataLoader(dataset, batch_size=128, drop_last=True) + + for inputs, outputs in loader: + assert list(inputs.keys()) == ["obs_0", "obs_1", "obs_2", "obs_3", "obs_4", "act_0"] + assert list(outputs.keys()) == ["obs_0", "obs_1", "obs_2", "obs_3", "obs_4"] + for key in inputs: + assert inputs[key].shape == (128, 1) + for key in outputs: + assert outputs[key].shape == (128, 1) + + # test for reward + dataset = OfflineDataset(real_replay_buffer, env.observation_space, env.action_space, mech="reward_mech") + loader = DataLoader(dataset, batch_size=128, drop_last=True) + + for inputs, outputs in loader: + assert list(inputs.keys()) == ["obs_0", "obs_1", "obs_2", "obs_3", "obs_4", "act_0"] + assert list(outputs.keys()) == ["reward"] + for key in inputs: + assert inputs[key].shape == (128, 1) + for key in outputs: + assert outputs[key].shape == (128, 1) + + # test for termination + dataset = OfflineDataset(real_replay_buffer, env.observation_space, env.action_space, mech="termination_mech") + loader = DataLoader(dataset, batch_size=128, drop_last=True) + + for inputs, outputs in loader: + assert list(inputs.keys()) == ["obs_0", "obs_1", "obs_2", "obs_3", "obs_4", "act_0"] + assert list(outputs.keys()) == ["terminal"] + for key in inputs: + assert inputs[key].shape == (128, 1) + for key in outputs: + assert outputs[key].shape == (128, 1) + + +def test_ensemble_offline_dataset(): + env = gym.make("BoundaryInvertedPendulumSwingUp-v0", freq_rate=1, time_step=0.02) + # assert isinstance(env, emei.EmeiEnv) + + real_replay_buffer = ReplayBuffer( + int(1e5), env.observation_space, env.action_space, "cpu", handle_timeout_termination=False + ) + load_offline_data(env, real_replay_buffer, "SAC-expert-replay", use_ratio=0.1) + + # test for transition + dataset = EnsembleOfflineDataset(real_replay_buffer, env.observation_space, env.action_space, mech="transition") + loader = DataLoader(dataset, batch_size=128, drop_last=True) + + for inputs, outputs in loader: + assert list(inputs.keys()) == ["obs_0", "obs_1", "obs_2", "obs_3", "obs_4", "act_0"] + assert list(outputs.keys()) == ["obs_0", "obs_1", "obs_2", "obs_3", "obs_4"] + for key in inputs: + assert inputs[key].shape == (128, 7, 1) + for key in outputs: + assert outputs[key].shape == (128, 7, 1) + + # test for reward + dataset = EnsembleOfflineDataset(real_replay_buffer, env.observation_space, env.action_space, mech="reward_mech") + loader = DataLoader(dataset, batch_size=128, drop_last=True) + + for inputs, outputs in loader: + assert list(inputs.keys()) == ["obs_0", "obs_1", "obs_2", "obs_3", "obs_4", "act_0"] + assert list(outputs.keys()) == ["reward"] + for key in inputs: + assert inputs[key].shape == (128, 7, 1) + for key in outputs: + assert outputs[key].shape == (128, 7, 1) + + # test for termination + dataset = EnsembleOfflineDataset(real_replay_buffer, env.observation_space, env.action_space, mech="termination_mech") + loader = DataLoader(dataset, batch_size=128, drop_last=True) + + for inputs, outputs in loader: + assert list(inputs.keys()) == ["obs_0", "obs_1", "obs_2", "obs_3", "obs_4", "act_0"] + assert list(outputs.keys()) == ["terminal"] + for key in inputs: + assert inputs[key].shape == (128, 7, 1) + for key in outputs: + assert outputs[key].shape == (128, 7, 1) From 51d3101104642caf58cc36630292cce25b8fd03f Mon Sep 17 00:00:00 2001 From: frank Date: Fri, 4 Nov 2022 23:08:03 +0800 Subject: [PATCH 09/68] :tada: add binary-variable and update encoder and decoder --- cmrl/models/causal_mech/base_causal_mech.py | 64 +++++++++---------- cmrl/models/causal_mech/plain_mech.py | 2 +- cmrl/models/data_loader.py | 23 ++++--- cmrl/models/networks/coder.py | 54 +++++++++++++--- cmrl/models/networks/parallel_mlp.py | 4 +- cmrl/models/util.py | 6 ++ cmrl/types.py | 5 ++ .../test_causal_mech/test_plain_mech.py | 32 +++++++--- tests/test_models/test_data_loader.py | 14 ++-- tests/test_models/test_network/test_coder.py | 48 ++++++++++---- 10 files changed, 169 insertions(+), 83 deletions(-) diff --git a/cmrl/models/causal_mech/base_causal_mech.py b/cmrl/models/causal_mech/base_causal_mech.py index f5eb126..974aac1 100644 --- a/cmrl/models/causal_mech/base_causal_mech.py +++ b/cmrl/models/causal_mech/base_causal_mech.py @@ -75,35 +75,35 @@ def decode(self, hidden): pass -Causal = TypeVar("Causal", bound=BaseCausalMech) - - -class BaseMultiStepCausalMech(BaseCausalMech): - def __init__( - self, - single_step_mech_class: Type[Causal], - input_variables: List[Variable], - output_variables: List[Variable], - node_dim: int, - variable_encoders: Dict[str, VariableEncoder], - variable_decoders: Dict[str, VariableDecoder], - **kwargs - ): - super(BaseMultiStepCausalMech, self).__init__( - input_variables=input_variables, - output_variables=output_variables, - node_dim=node_dim, - variable_encoders=variable_encoders, - variable_decoders=variable_decoders, - ) - - self.single_step_mech = single_step_mech_class(**kwargs) - pass - - @abstractmethod - def build_network(self): - raise NotImplementedError - - @abstractmethod - def build_graph(self): - raise NotImplementedError +# Causal = TypeVar("Causal", bound=BaseCausalMech) +# +# +# class BaseMultiStepCausalMech(BaseCausalMech): +# def __init__( +# self, +# single_step_mech_class: Type[Causal], +# input_variables: List[Variable], +# output_variables: List[Variable], +# node_dim: int, +# variable_encoders: Dict[str, VariableEncoder], +# variable_decoders: Dict[str, VariableDecoder], +# **kwargs +# ): +# super(BaseMultiStepCausalMech, self).__init__( +# input_variables=input_variables, +# output_variables=output_variables, +# node_dim=node_dim, +# variable_encoders=variable_encoders, +# variable_decoders=variable_decoders, +# ) +# +# self.single_step_mech = single_step_mech_class(**kwargs) +# pass +# +# @abstractmethod +# def build_network(self): +# raise NotImplementedError +# +# @abstractmethod +# def build_graph(self): +# raise NotImplementedError diff --git a/cmrl/models/causal_mech/plain_mech.py b/cmrl/models/causal_mech/plain_mech.py index a15d52c..0cb7b8a 100644 --- a/cmrl/models/causal_mech/plain_mech.py +++ b/cmrl/models/causal_mech/plain_mech.py @@ -60,5 +60,5 @@ def build_network(self): def build_graph(self): self.graph = None - def learn(self): + def learn(self, train_loader, valid_loader): pass diff --git a/cmrl/models/data_loader.py b/cmrl/models/data_loader.py index 050b826..d79399c 100644 --- a/cmrl/models/data_loader.py +++ b/cmrl/models/data_loader.py @@ -6,7 +6,7 @@ from stable_baselines3.common.buffers import ReplayBuffer, DictReplayBuffer -class OfflineDataset(Dataset): +class BufferDataset(Dataset): def __init__( self, replay_buffer: ReplayBuffer, @@ -56,20 +56,16 @@ def load_from_buffer(self): next_observations = self.replay_buffer.next_observations[: self.size, 0].astype(np.float32) observations_dict = dict([("obs_{}".format(i), obs[:, None]) for i, obs in enumerate(observations.T)]) - next_observations_dict = dict([("obs_{}".format(i), obs[:, None]) for i, obs in enumerate(next_observations.T)]) + next_observations_dict = dict( + [("next_obs_{}".format(i), obs[:, None]) for i, obs in enumerate(next_observations.T)] + ) assert isinstance(self.observation_space, spaces.Box) # TODO: other spaces for observation and action(e.g. one-hot for spaces.Discrete) # see: https://github.com/DLR-RM/stable-baselines3/blob/master/stable_baselines3/common/preprocessing.py#L85 actions = self.replay_buffer.actions[: self.size, 0] - rewards = self.replay_buffer.rewards[: self.size, 0] - dones = self.replay_buffer.dones[: self.size, 0] - timeouts = self.replay_buffer.timeouts[: self.size, 0] - actions_dict = dict([("act_{}".format(i), obs[:, None]) for i, obs in enumerate(actions.T)]) - rewards_dict = {"reward": rewards[:, None]} - terminals_dict = {"terminal": (dones * (1 - timeouts))[:, None]} self.inputs = {} self.inputs.update(observations_dict) @@ -78,8 +74,15 @@ def load_from_buffer(self): if self.mech == "transition": self.outputs = next_observations_dict elif self.mech == "reward_mech": + rewards = self.replay_buffer.rewards[: self.size, 0] + rewards_dict = {"reward": rewards[:, None]} + self.inputs.update(next_observations_dict) self.outputs = rewards_dict else: + dones = self.replay_buffer.dones[: self.size, 0] + timeouts = self.replay_buffer.timeouts[: self.size, 0] + terminals_dict = {"terminal": (dones * (1 - timeouts))[:, None]} + self.inputs.update(next_observations_dict) self.outputs = terminals_dict def __getitem__(self, item): @@ -93,7 +96,7 @@ def __len__(self): return len(self.indexes) -class EnsembleOfflineDataset(OfflineDataset): +class EnsembleBufferDataset(BufferDataset): def __init__( self, replay_buffer: ReplayBuffer, @@ -107,7 +110,7 @@ def __init__( ): self.ensemble_num = ensemble_num - super(EnsembleOfflineDataset, self).__init__( + super(EnsembleBufferDataset, self).__init__( replay_buffer=replay_buffer, observation_space=observation_space, action_space=action_space, diff --git a/cmrl/models/networks/coder.py b/cmrl/models/networks/coder.py index 597394c..4097a7e 100644 --- a/cmrl/models/networks/coder.py +++ b/cmrl/models/networks/coder.py @@ -4,7 +4,7 @@ import torch.nn as nn from omegaconf import DictConfig -from cmrl.types import Variable, DiscreteVariable, ContinuousVariable +from cmrl.types import Variable, DiscreteVariable, ContinuousVariable, BinaryVariable from cmrl.models.networks.base_network import BaseNetwork, create_activation @@ -13,10 +13,14 @@ def __init__( self, variable: Variable, node_dim: int, + hidden_dims: Optional[List[int]] = None, + bias: bool = True, activation_fn_cfg: Optional[DictConfig] = None, ): self.variable = variable self.node_dim = node_dim + self.hidden_dims = hidden_dims if hidden_dims is not None else [] + self.bias = bias self.name = "{}_encoder".format(variable.name) self.activation_fn_cfg = activation_fn_cfg @@ -25,14 +29,25 @@ def __init__( def build(self): layers = [] + if len(self.hidden_dims) == 0: + hidden_dim = self.node_dim + else: + hidden_dim = self.hidden_dims[0] + if isinstance(self.variable, ContinuousVariable): - layers.append(nn.Linear(self.variable.dim, self.node_dim)) + layers.append(nn.Linear(self.variable.dim, hidden_dim)) elif isinstance(self.variable, DiscreteVariable): - layers.append(nn.Linear(self.variable.n, self.node_dim)) + layers.append(nn.Linear(self.variable.n, hidden_dim)) + elif isinstance(self.variable, BinaryVariable): + layers.append(nn.Linear(1, hidden_dim)) else: raise NotImplementedError - layers.append(create_activation(self.activation_fn_cfg)) + hidden_dims = self.hidden_dims + [self.node_dim] + for i in range(len(hidden_dims) - 1): + layers += [nn.Linear(hidden_dims[i], hidden_dims[i + 1], bias=self.bias)] + layers += [create_activation(self.activation_fn_cfg)] + self._layers = nn.ModuleList(layers) @@ -41,28 +56,47 @@ def __init__( self, variable: Variable, node_dim: int, - normal_distribution: bool = False, + hidden_dims: Optional[List[int]] = None, + bias: bool = True, activation_fn_cfg: Optional[DictConfig] = None, + normal_distribution: bool = False, ): self.variable = variable self.node_dim = node_dim - self.normal_distribution = normal_distribution + self.hidden_dims = hidden_dims if hidden_dims is not None else [] + self.bias = bias self.name = "{}_decoder".format(variable.name) self.activation_fn_cfg = activation_fn_cfg + self.normal_distribution = normal_distribution + super(VariableDecoder, self).__init__() self._model_filename = "{}.pth".format(self.name) def build(self): - layers = [] + layers = [create_activation(self.activation_fn_cfg)] + + hidden_dims = [self.node_dim] + self.hidden_dims + for i in range(len(hidden_dims) - 1): + layers += [nn.Linear(hidden_dims[i], hidden_dims[i + 1], bias=self.bias)] + layers += [create_activation(self.activation_fn_cfg)] + + if len(self.hidden_dims) == 0: + hidden_dim = self.node_dim + else: + hidden_dim = self.hidden_dims[-1] + if isinstance(self.variable, ContinuousVariable): if self.normal_distribution: - layers.append(nn.Linear(self.node_dim, self.variable.dim * 2)) + layers.append(nn.Linear(hidden_dim, self.variable.dim * 2)) else: - layers.append(nn.Linear(self.node_dim, self.variable.dim)) + layers.append(nn.Linear(hidden_dim, self.variable.dim)) elif isinstance(self.variable, DiscreteVariable): - layers.append(nn.Linear(self.node_dim, self.variable.n)) + layers.append(nn.Linear(hidden_dim, self.variable.n)) layers.append(nn.Softmax()) + elif isinstance(self.variable, BinaryVariable): + layers.append(nn.Linear(hidden_dim, 1)) + layers.append(nn.Sigmoid()) else: raise NotImplementedError diff --git a/cmrl/models/networks/parallel_mlp.py b/cmrl/models/networks/parallel_mlp.py index 3a97ffa..c977a85 100644 --- a/cmrl/models/networks/parallel_mlp.py +++ b/cmrl/models/networks/parallel_mlp.py @@ -27,8 +27,8 @@ def __init__( ): self.input_dim = input_dim self.output_dim = output_dim - self.extra_dims = extra_dims - self.hidden_dims = hidden_dims + self.extra_dims = extra_dims if extra_dims is not None else [] + self.hidden_dims = hidden_dims if hidden_dims is not None else [200, 200, 200, 200] self.use_bias = use_bias self.init_type = init_type self.activation_fn_cfg = activation_fn_cfg diff --git a/cmrl/models/util.py b/cmrl/models/util.py index efc6849..ed37a84 100644 --- a/cmrl/models/util.py +++ b/cmrl/models/util.py @@ -7,8 +7,10 @@ import numpy as np import torch import torch.nn.functional as F +from gym import spaces import cmrl.types +from cmrl.types import Variable, ContinuousVariable, DiscreteVariable def gaussian_nll( @@ -67,3 +69,7 @@ def to_tensor(x: cmrl.types.TensorType): if isinstance(x, np.ndarray): return torch.from_numpy(x) raise ValueError("Input must be torch.Tensor or np.ndarray.") + + +# def parse_space(space: spaces.Space): +# if isinstance(space, ) diff --git a/cmrl/types.py b/cmrl/types.py index de6e8b9..21dc1c0 100644 --- a/cmrl/types.py +++ b/cmrl/types.py @@ -91,6 +91,11 @@ class ContinuousVariable(Variable): dim: int +@dataclass +class BinaryVariable(Variable): + pass + + @dataclass class DiscreteVariable(Variable): n: int diff --git a/tests/test_models/test_causal_mech/test_plain_mech.py b/tests/test_models/test_causal_mech/test_plain_mech.py index 3cc90c4..cc7879b 100644 --- a/tests/test_models/test_causal_mech/test_plain_mech.py +++ b/tests/test_models/test_causal_mech/test_plain_mech.py @@ -1,12 +1,31 @@ +import gym +import emei +from stable_baselines3.common.buffers import ReplayBuffer +from torch.utils.data import DataLoader + from cmrl.models.causal_mech.plain_mech import PlainMech from cmrl.types import Variable, ContinuousVariable, DiscreteVariable +from cmrl.models.data_loader import BufferDataset, EnsembleBufferDataset +from cmrl.algorithms.util import load_offline_data + + +def test_single_dim_continuous(): + env = gym.make("BoundaryInvertedPendulumSwingUp-v0", freq_rate=1, time_step=0.02) + # assert isinstance(env, emei.EmeiEnv) + + real_replay_buffer = ReplayBuffer( + int(1e5), env.observation_space, env.action_space, "cpu", handle_timeout_termination=False + ) + load_offline_data(env, real_replay_buffer, "SAC-expert-replay", use_ratio=0.01) + # test for transition + dataset = EnsembleBufferDataset(real_replay_buffer, env.observation_space, env.action_space, mech="transition") + loader = DataLoader(dataset, batch_size=8) -def test_without_coder(): node_dim = 1 - input_variables = [ContinuousVariable(name="state0", dim=node_dim), ContinuousVariable(name="action0", dim=node_dim)] - output_variables = [ContinuousVariable(name="state0", dim=node_dim)] + input_variables = [ContinuousVariable(name="obs_0", dim=node_dim), ContinuousVariable(name="act_0", dim=node_dim)] + output_variables = [ContinuousVariable(name="obs_0", dim=node_dim)] mech = PlainMech( input_variables=input_variables, @@ -15,10 +34,3 @@ def test_without_coder(): variable_encoders={"state0": None, "action0": None}, variable_decoders={"state0": None}, ) - - -def test_single_dim_continuous(): - input_variables = [ContinuousVariable(1), ContinuousVariable(1)] - output_variables = [ContinuousVariable(1)] - - mech = PlainMech(input_variables=input_variables, output_variables=output_variables) diff --git a/tests/test_models/test_data_loader.py b/tests/test_models/test_data_loader.py index f5b4afa..6ed2bdd 100644 --- a/tests/test_models/test_data_loader.py +++ b/tests/test_models/test_data_loader.py @@ -3,7 +3,7 @@ from stable_baselines3.common.buffers import ReplayBuffer from torch.utils.data import DataLoader -from cmrl.models.data_loader import OfflineDataset, EnsembleOfflineDataset +from cmrl.models.data_loader import BufferDataset, EnsembleBufferDataset from cmrl.algorithms.util import load_offline_data @@ -17,7 +17,7 @@ def test_offline_dataset(): load_offline_data(env, real_replay_buffer, "SAC-expert-replay", use_ratio=0.1) # test for transition - dataset = OfflineDataset(real_replay_buffer, env.observation_space, env.action_space, mech="transition") + dataset = BufferDataset(real_replay_buffer, env.observation_space, env.action_space, mech="transition") loader = DataLoader(dataset, batch_size=128, drop_last=True) for inputs, outputs in loader: @@ -29,7 +29,7 @@ def test_offline_dataset(): assert outputs[key].shape == (128, 1) # test for reward - dataset = OfflineDataset(real_replay_buffer, env.observation_space, env.action_space, mech="reward_mech") + dataset = BufferDataset(real_replay_buffer, env.observation_space, env.action_space, mech="reward_mech") loader = DataLoader(dataset, batch_size=128, drop_last=True) for inputs, outputs in loader: @@ -41,7 +41,7 @@ def test_offline_dataset(): assert outputs[key].shape == (128, 1) # test for termination - dataset = OfflineDataset(real_replay_buffer, env.observation_space, env.action_space, mech="termination_mech") + dataset = BufferDataset(real_replay_buffer, env.observation_space, env.action_space, mech="termination_mech") loader = DataLoader(dataset, batch_size=128, drop_last=True) for inputs, outputs in loader: @@ -63,7 +63,7 @@ def test_ensemble_offline_dataset(): load_offline_data(env, real_replay_buffer, "SAC-expert-replay", use_ratio=0.1) # test for transition - dataset = EnsembleOfflineDataset(real_replay_buffer, env.observation_space, env.action_space, mech="transition") + dataset = EnsembleBufferDataset(real_replay_buffer, env.observation_space, env.action_space, mech="transition") loader = DataLoader(dataset, batch_size=128, drop_last=True) for inputs, outputs in loader: @@ -75,7 +75,7 @@ def test_ensemble_offline_dataset(): assert outputs[key].shape == (128, 7, 1) # test for reward - dataset = EnsembleOfflineDataset(real_replay_buffer, env.observation_space, env.action_space, mech="reward_mech") + dataset = EnsembleBufferDataset(real_replay_buffer, env.observation_space, env.action_space, mech="reward_mech") loader = DataLoader(dataset, batch_size=128, drop_last=True) for inputs, outputs in loader: @@ -87,7 +87,7 @@ def test_ensemble_offline_dataset(): assert outputs[key].shape == (128, 7, 1) # test for termination - dataset = EnsembleOfflineDataset(real_replay_buffer, env.observation_space, env.action_space, mech="termination_mech") + dataset = EnsembleBufferDataset(real_replay_buffer, env.observation_space, env.action_space, mech="termination_mech") loader = DataLoader(dataset, batch_size=128, drop_last=True) for inputs, outputs in loader: diff --git a/tests/test_models/test_network/test_coder.py b/tests/test_models/test_network/test_coder.py index 02041e9..e3812f9 100644 --- a/tests/test_models/test_network/test_coder.py +++ b/tests/test_models/test_network/test_coder.py @@ -3,7 +3,7 @@ from torch.nn.functional import one_hot from cmrl.models.networks.coder import VariableEncoder, VariableDecoder -from cmrl.types import Variable, ContinuousVariable, DiscreteVariable +from cmrl.types import Variable, ContinuousVariable, DiscreteVariable, BinaryVariable def test_continuous_encoder(): @@ -11,9 +11,9 @@ def test_continuous_encoder(): node_dim = 5 batch_size = 128 - var = ContinuousVariable(name="state0", dim=var_dim) + var = ContinuousVariable(name="obs_0", dim=var_dim) - encoder = VariableEncoder(var, node_dim) + encoder = VariableEncoder(var, node_dim, hidden_dims=[200]) inputs = torch.rand(batch_size, var_dim) outputs = encoder(inputs) @@ -25,23 +25,36 @@ def test_discrete_encoder(): node_dim = 5 batch_size = 128 - var = DiscreteVariable(name="state0", n=var_n) + var = DiscreteVariable(name="obs_0", n=var_n) - encoder = VariableEncoder(var, node_dim) + encoder = VariableEncoder(var, node_dim, hidden_dims=[200]) inputs = one_hot(torch.randint(3, (batch_size,))).to(torch.float32) outputs = encoder(inputs) assert outputs.shape == (batch_size, node_dim) +def test_binary_encoder(): + node_dim = 5 + batch_size = 128 + + var = BinaryVariable(name="obs_0") + + encoder = VariableEncoder(var, node_dim, hidden_dims=[200]) + inputs = one_hot(torch.randint(1, (batch_size,))).to(torch.float32) + outputs = encoder(inputs) + + assert outputs.shape == (batch_size, node_dim) + + def test_continuous_decoder(): var_dim = 3 node_dim = 5 batch_size = 128 - var = ContinuousVariable(name="state0", dim=var_dim) + var = ContinuousVariable(name="obs_0", dim=var_dim) - decoder = VariableDecoder(var, node_dim) + decoder = VariableDecoder(var, node_dim, hidden_dims=[200]) inputs = torch.rand(batch_size, node_dim) outputs = decoder(inputs) @@ -53,12 +66,25 @@ def test_discrete_decoder(): node_dim = 5 batch_size = 128 - var = DiscreteVariable(name="state0", n=var_n) + var = DiscreteVariable(name="obs_0", n=var_n) - decoder = VariableDecoder(var, node_dim) + decoder = VariableDecoder(var, node_dim, hidden_dims=[200]) inputs = torch.rand(batch_size, node_dim) outputs = decoder(inputs) assert outputs.shape == (batch_size, var_n) - batch_sum = outputs.detach().numpy().sum(axis=1) - assert np.allclose(batch_sum, 1) + assert torch.allclose(outputs.sum(dim=1), torch.tensor(1.0)) + + +def test_binary_decoder(): + node_dim = 5 + batch_size = 128 + + var = BinaryVariable(name="obs_0") + + decoder = VariableDecoder(var, node_dim, hidden_dims=[200]) + inputs = torch.rand(batch_size, node_dim) + outputs = decoder(inputs) + + assert outputs.shape == (batch_size, 1) + assert (outputs >= 0).all() and (outputs <= 1).all() From d450eae993cc801909f8103f633677c636c47372 Mon Sep 17 00:00:00 2001 From: frank Date: Sat, 5 Nov 2022 01:06:44 +0800 Subject: [PATCH 10/68] :tada: add forward and loss for PlainMech --- cmrl/models/causal_mech/base_causal_mech.py | 26 +++++- cmrl/models/causal_mech/plain_mech.py | 86 ++++++++++++++++++- cmrl/models/data_loader.py | 9 +- cmrl/models/dynamics/plain_dynamics.py | 1 + cmrl/models/networks/base_network.py | 5 +- cmrl/models/util.py | 64 +++++++++++++- cmrl/types.py | 2 + .../test_causal_mech/test_plain_mech.py | 28 ++++-- tests/test_models/test_data_loader.py | 60 +++++++++++-- 9 files changed, 254 insertions(+), 27 deletions(-) diff --git a/cmrl/models/causal_mech/base_causal_mech.py b/cmrl/models/causal_mech/base_causal_mech.py index 974aac1..9832dac 100644 --- a/cmrl/models/causal_mech/base_causal_mech.py +++ b/cmrl/models/causal_mech/base_causal_mech.py @@ -2,6 +2,8 @@ from abc import abstractmethod import torch +from torch.utils.data import DataLoader +from stable_baselines3.common.logger import Logger from cmrl.types import Variable, ContinuousVariable, DiscreteVariable from cmrl.models.networks.base_network import BaseNetwork @@ -19,6 +21,13 @@ def __init__( variable_decoders: Dict[str, VariableDecoder], # forward method residual: bool = True, + # trainer + optim_lr: float = 1e-4, + optim_weight_decay: float = 1e-5, + optim_eps: float = 1e-8, + optim_coder: bool = True, + # logger + logger: Optional[Logger] = None, # others device: Union[str, torch.device] = "cpu", **kwargs @@ -28,7 +37,16 @@ def __init__( self.node_dim = node_dim self.variable_encoders = variable_encoders self.variable_decoders = variable_decoders + # forward method self.residual = residual + # trainer + self.optim_lr = optim_lr + self.optim_weight_decay = optim_weight_decay + self.optim_eps = optim_eps + self.optim_coder = optim_coder + # logger + self.logger = logger + # others self.device = device self.input_var_num = len(self.input_variables) @@ -57,7 +75,13 @@ def check_coder(self): assert decoder.node_dim == self.node_dim @abstractmethod - def learn(self): + def learn( + self, + # loader + train_loader: DataLoader, + valid_loader: DataLoader, + **kwargs + ): raise NotImplementedError @abstractmethod diff --git a/cmrl/models/causal_mech/plain_mech.py b/cmrl/models/causal_mech/plain_mech.py index 0cb7b8a..dfc596d 100644 --- a/cmrl/models/causal_mech/plain_mech.py +++ b/cmrl/models/causal_mech/plain_mech.py @@ -1,8 +1,15 @@ from typing import Optional, List, Dict, Union +import pathlib +import itertools + import torch +from torch.utils.data import DataLoader +import torch.nn.functional as F +from torch.optim import Adam from omegaconf import DictConfig +from stable_baselines3.common.logger import Logger -from cmrl.types import Variable +from cmrl.types import Variable, ContinuousVariable from cmrl.models.networks.parallel_mlp import ParallelMLP from cmrl.models.graphs.base_graph import BaseGraph from cmrl.models.causal_mech.base_causal_mech import BaseCausalMech @@ -26,6 +33,13 @@ def __init__( activation_fn_cfg: Optional[DictConfig] = None, # forward method residual: bool = True, + # trainer + optim_lr: float = 1e-4, + optim_weight_decay: float = 1e-5, + optim_eps: float = 1e-8, + optim_coder: bool = True, + # logger + logger: Optional[Logger] = None, # others device: Union[str, torch.device] = "cpu", **kwargs @@ -43,6 +57,11 @@ def __init__( variable_encoders=variable_encoders, variable_decoders=variable_decoders, residual=residual, + optim_lr=optim_lr, + optim_weight_decay=optim_weight_decay, + optim_eps=optim_eps, + optim_coder=optim_coder, + logger=logger, device=device, **kwargs ) @@ -57,8 +76,69 @@ def build_network(self): activation_fn_cfg=self.activation_fn_cfg, ) + if self.optim_coder: + parmas = itertools.chain( + self.network.parameters(), + *[encoder.parameters() for encoder in self.variable_encoders.values()], + *[decoder.parameters() for decoder in self.variable_decoders.values()] + ) + self.optim = Adam(parmas, lr=self.optim_lr, weight_decay=self.optim_weight_decay, eps=self.optim_eps) + def build_graph(self): self.graph = None - def learn(self, train_loader, valid_loader): - pass + def forward(self, inputs: Dict[str, torch.Tensor]) -> Dict[str, torch.Tensor]: + assert len(inputs) > 0, "inputs should not be null" + ensemble, batch_size = list(inputs.values())[0].shape[:2] + assert ensemble == self.ensemble_num + assert list(inputs.keys()) == list(self.variable_encoders.keys()) + + inputs_tensor = torch.zeros(ensemble, batch_size, self.input_var_num * self.node_dim) + for i, (name, encoder) in enumerate(self.variable_encoders.items()): + out = encoder(inputs[name]) # ensemble-num, batch-size, node-dim + inputs_tensor[:, :, i * self.node_dim : (i + 1) * self.node_dim] = out + + hidden_tensor = self.network(inputs_tensor) + + outputs = {} + for i, (name, decoder) in enumerate(self.variable_decoders.items()): + hid = hidden_tensor[:, :, i * self.node_dim : (i + 1) * self.node_dim] + out = decoder(hid) + outputs[name] = out + + return outputs + + def loss(self, outputs, targets): + loss = torch.tensor(0.0) + for var in self.output_variables: + output = outputs[var.name] + target = targets[var.name] + if isinstance(var, ContinuousVariable): + loss += F.mse_loss(output, target) + return loss + + def learn( + self, + # loader + train_loader: DataLoader, + valid_loader: DataLoader, + # model learning + longest_epoch: int = -1, + improvement_threshold: float = 0.1, + patience: int = 5, + work_dir: Optional[Union[str, pathlib.Path]] = None, + **kwargs + ): + best_weights: Optional[Dict] = None + epoch_iter = range(longest_epoch) if longest_epoch > 0 else itertools.count() + epochs_since_update = 0 + + for inputs, targets in train_loader: + outputs = self.forward(inputs) + loss = self.loss(outputs, targets) + + self.optim.zero_grad() + loss.backward() + self.optim.step() + + break diff --git a/cmrl/models/data_loader.py b/cmrl/models/data_loader.py index d79399c..9e15a55 100644 --- a/cmrl/models/data_loader.py +++ b/cmrl/models/data_loader.py @@ -1,7 +1,7 @@ import gym from gym import spaces import torch -from torch.utils.data import Dataset, DataLoader, Sampler +from torch.utils.data import Dataset, default_collate import numpy as np from stable_baselines3.common.buffers import ReplayBuffer, DictReplayBuffer @@ -130,3 +130,10 @@ def build_indexes(self): self.indexes = np.array( [np.random.permutation(self.size)[: int(self.size * self.train_ratio)] for _ in range(self.ensemble_num)] ).T + + +def collate_fn(data): + inputs, outputs = default_collate(data) + inputs = dict([(key, value.transpose(0, 1)) for key, value in inputs.items()]) + outputs = dict([(key, value.transpose(0, 1)) for key, value in outputs.items()]) + return [inputs, outputs] diff --git a/cmrl/models/dynamics/plain_dynamics.py b/cmrl/models/dynamics/plain_dynamics.py index 317ac46..d600cb1 100644 --- a/cmrl/models/dynamics/plain_dynamics.py +++ b/cmrl/models/dynamics/plain_dynamics.py @@ -27,6 +27,7 @@ def __init__( optim_lr: float = 1e-4, weight_decay: float = 1e-5, optim_eps: float = 1e-8, + # logger logger: Optional[Logger] = None, ): super(PlainEnsembleDynamics, self).__init__( diff --git a/cmrl/models/networks/base_network.py b/cmrl/models/networks/base_network.py index 5af379a..ab32a5e 100644 --- a/cmrl/models/networks/base_network.py +++ b/cmrl/models/networks/base_network.py @@ -7,9 +7,6 @@ import hydra from omegaconf import DictConfig -from cmrl.models.util import gaussian_nll -from cmrl.models.layers import ParallelLinear - class BaseNetwork(nn.Module): def __init__(self, **kwargs): @@ -45,7 +42,7 @@ def load(self, load_dir: Union[str, pathlib.Path]): else: getattr(self, attr)(model_dict[attr]) - def forward(self, x): + def forward(self, x) -> torch.Tensor: for layer in self._layers: x = layer(x) return x diff --git a/cmrl/models/util.py b/cmrl/models/util.py index ed37a84..34a202c 100644 --- a/cmrl/models/util.py +++ b/cmrl/models/util.py @@ -2,15 +2,17 @@ # # This source code is licensed under the MIT license found in the # LICENSE file in the root directory of this source tree. -from typing import List, Sequence, Tuple +from typing import List, Optional import numpy as np import torch import torch.nn.functional as F from gym import spaces +from omegaconf import DictConfig import cmrl.types -from cmrl.types import Variable, ContinuousVariable, DiscreteVariable +from cmrl.types import Variable, ContinuousVariable, DiscreteVariable, BinaryVariable +from cmrl.models.networks.coder import VariableEncoder, VariableDecoder def gaussian_nll( @@ -71,5 +73,59 @@ def to_tensor(x: cmrl.types.TensorType): raise ValueError("Input must be torch.Tensor or np.ndarray.") -# def parse_space(space: spaces.Space): -# if isinstance(space, ) +def parse_space(space: spaces.Space, prefix="obs") -> List[Variable]: + variables = [] + if isinstance(space, spaces.Box): + for i, (low, high) in enumerate(zip(space.low, space.high)): + variables.append(ContinuousVariable(dim=1, low=low, high=high, name="{}_{}".format(prefix, i))) + elif isinstance(space, spaces.Discrete): + variables.append(DiscreteVariable(n=space.n, name="{}_0".format(prefix))) + elif isinstance(space, spaces.MultiDiscrete): + for i, n in enumerate(space.nvec): + variables.append(DiscreteVariable(n=n, name="{}_{}".format(prefix, i))) + elif isinstance(space, spaces.MultiBinary): + for i in range(space.n): + variables.append(BinaryVariable(name="{}_{}".format(prefix, i))) + elif isinstance(space, spaces.Dict): + # TODO + raise NotImplementedError + + return variables + + +def create_encoders( + input_variables: List[Variable], + node_dim: int, + hidden_dims: Optional[List[int]] = None, + bias: bool = True, + activation_fn_cfg: Optional[DictConfig] = None, +): + encoders = {} + for var in input_variables: + assert var.name not in encoders, "Duplicate name in decoders: {}".format(var.name) + encoders[var.name] = VariableEncoder( + variable=var, node_dim=node_dim, hidden_dims=hidden_dims, bias=bias, activation_fn_cfg=activation_fn_cfg + ) + return encoders + + +def create_decoders( + input_variables: List[Variable], + node_dim: int, + hidden_dims: Optional[List[int]] = None, + bias: bool = True, + activation_fn_cfg: Optional[DictConfig] = None, + normal_distribution: bool = False, +): + decoders = {} + for var in input_variables: + assert var.name not in decoders, "Duplicate name in decoders: {}".format(var.name) + decoders[var.name] = VariableDecoder( + variable=var, + node_dim=node_dim, + hidden_dims=hidden_dims, + bias=bias, + activation_fn_cfg=activation_fn_cfg, + normal_distribution=normal_distribution, + ) + return decoders diff --git a/cmrl/types.py b/cmrl/types.py index 21dc1c0..59f86ab 100644 --- a/cmrl/types.py +++ b/cmrl/types.py @@ -89,6 +89,8 @@ class Variable: @dataclass class ContinuousVariable(Variable): dim: int + low: np.ndarray + high: np.ndarray @dataclass diff --git a/tests/test_models/test_causal_mech/test_plain_mech.py b/tests/test_models/test_causal_mech/test_plain_mech.py index cc7879b..1c687d5 100644 --- a/tests/test_models/test_causal_mech/test_plain_mech.py +++ b/tests/test_models/test_causal_mech/test_plain_mech.py @@ -2,11 +2,13 @@ import emei from stable_baselines3.common.buffers import ReplayBuffer from torch.utils.data import DataLoader +from torch.utils.data import default_collate from cmrl.models.causal_mech.plain_mech import PlainMech from cmrl.types import Variable, ContinuousVariable, DiscreteVariable -from cmrl.models.data_loader import BufferDataset, EnsembleBufferDataset +from cmrl.models.data_loader import BufferDataset, EnsembleBufferDataset, collate_fn from cmrl.algorithms.util import load_offline_data +from cmrl.models.util import parse_space, create_decoders, create_encoders def test_single_dim_continuous(): @@ -19,18 +21,28 @@ def test_single_dim_continuous(): load_offline_data(env, real_replay_buffer, "SAC-expert-replay", use_ratio=0.01) # test for transition - dataset = EnsembleBufferDataset(real_replay_buffer, env.observation_space, env.action_space, mech="transition") - loader = DataLoader(dataset, batch_size=8) + tran_dataset = EnsembleBufferDataset( + real_replay_buffer, env.observation_space, env.action_space, is_valid=False, mech="transition" + ) + train_loader = DataLoader(tran_dataset, batch_size=8, collate_fn=collate_fn) + valid_dataset = EnsembleBufferDataset( + real_replay_buffer, env.observation_space, env.action_space, is_valid=True, mech="transition" + ) + valid_loader = DataLoader(valid_dataset, batch_size=8, collate_fn=collate_fn) - node_dim = 1 + node_dim = 16 - input_variables = [ContinuousVariable(name="obs_0", dim=node_dim), ContinuousVariable(name="act_0", dim=node_dim)] - output_variables = [ContinuousVariable(name="obs_0", dim=node_dim)] + input_variables = parse_space(env.observation_space, "obs") + parse_space(env.action_space, "act") + variable_encoders = create_encoders(input_variables, node_dim=node_dim) + output_variables = parse_space(env.observation_space, "next_obs") + variable_decoders = create_decoders(output_variables, node_dim=node_dim) mech = PlainMech( input_variables=input_variables, output_variables=output_variables, node_dim=node_dim, - variable_encoders={"state0": None, "action0": None}, - variable_decoders={"state0": None}, + variable_encoders=variable_encoders, + variable_decoders=variable_decoders, ) + + mech.learn(train_loader, valid_loader) diff --git a/tests/test_models/test_data_loader.py b/tests/test_models/test_data_loader.py index 6ed2bdd..dde5729 100644 --- a/tests/test_models/test_data_loader.py +++ b/tests/test_models/test_data_loader.py @@ -22,7 +22,7 @@ def test_offline_dataset(): for inputs, outputs in loader: assert list(inputs.keys()) == ["obs_0", "obs_1", "obs_2", "obs_3", "obs_4", "act_0"] - assert list(outputs.keys()) == ["obs_0", "obs_1", "obs_2", "obs_3", "obs_4"] + assert list(outputs.keys()) == ["next_obs_0", "next_obs_1", "next_obs_2", "next_obs_3", "next_obs_4"] for key in inputs: assert inputs[key].shape == (128, 1) for key in outputs: @@ -33,7 +33,19 @@ def test_offline_dataset(): loader = DataLoader(dataset, batch_size=128, drop_last=True) for inputs, outputs in loader: - assert list(inputs.keys()) == ["obs_0", "obs_1", "obs_2", "obs_3", "obs_4", "act_0"] + assert list(inputs.keys()) == [ + "obs_0", + "obs_1", + "obs_2", + "obs_3", + "obs_4", + "act_0", + "next_obs_0", + "next_obs_1", + "next_obs_2", + "next_obs_3", + "next_obs_4", + ] assert list(outputs.keys()) == ["reward"] for key in inputs: assert inputs[key].shape == (128, 1) @@ -45,7 +57,19 @@ def test_offline_dataset(): loader = DataLoader(dataset, batch_size=128, drop_last=True) for inputs, outputs in loader: - assert list(inputs.keys()) == ["obs_0", "obs_1", "obs_2", "obs_3", "obs_4", "act_0"] + assert list(inputs.keys()) == [ + "obs_0", + "obs_1", + "obs_2", + "obs_3", + "obs_4", + "act_0", + "next_obs_0", + "next_obs_1", + "next_obs_2", + "next_obs_3", + "next_obs_4", + ] assert list(outputs.keys()) == ["terminal"] for key in inputs: assert inputs[key].shape == (128, 1) @@ -68,7 +92,7 @@ def test_ensemble_offline_dataset(): for inputs, outputs in loader: assert list(inputs.keys()) == ["obs_0", "obs_1", "obs_2", "obs_3", "obs_4", "act_0"] - assert list(outputs.keys()) == ["obs_0", "obs_1", "obs_2", "obs_3", "obs_4"] + assert list(outputs.keys()) == ["next_obs_0", "next_obs_1", "next_obs_2", "next_obs_3", "next_obs_4"] for key in inputs: assert inputs[key].shape == (128, 7, 1) for key in outputs: @@ -79,7 +103,19 @@ def test_ensemble_offline_dataset(): loader = DataLoader(dataset, batch_size=128, drop_last=True) for inputs, outputs in loader: - assert list(inputs.keys()) == ["obs_0", "obs_1", "obs_2", "obs_3", "obs_4", "act_0"] + assert list(inputs.keys()) == [ + "obs_0", + "obs_1", + "obs_2", + "obs_3", + "obs_4", + "act_0", + "next_obs_0", + "next_obs_1", + "next_obs_2", + "next_obs_3", + "next_obs_4", + ] assert list(outputs.keys()) == ["reward"] for key in inputs: assert inputs[key].shape == (128, 7, 1) @@ -91,7 +127,19 @@ def test_ensemble_offline_dataset(): loader = DataLoader(dataset, batch_size=128, drop_last=True) for inputs, outputs in loader: - assert list(inputs.keys()) == ["obs_0", "obs_1", "obs_2", "obs_3", "obs_4", "act_0"] + assert list(inputs.keys()) == [ + "obs_0", + "obs_1", + "obs_2", + "obs_3", + "obs_4", + "act_0", + "next_obs_0", + "next_obs_1", + "next_obs_2", + "next_obs_3", + "next_obs_4", + ] assert list(outputs.keys()) == ["terminal"] for key in inputs: assert inputs[key].shape == (128, 7, 1) From d0a909afe5d433cbdd466157cd7bd469638a2a63 Mon Sep 17 00:00:00 2001 From: frank Date: Sat, 5 Nov 2022 15:27:52 +0800 Subject: [PATCH 11/68] :tada: mixed use of BufferDataset and EnsembleBufferDataset is supported --- cmrl/models/data_loader.py | 24 +++++--- tests/test_models/test_data_loader.py | 89 ++++++++++++++++++++++++++- 2 files changed, 102 insertions(+), 11 deletions(-) diff --git a/cmrl/models/data_loader.py b/cmrl/models/data_loader.py index 9e15a55..3939cc7 100644 --- a/cmrl/models/data_loader.py +++ b/cmrl/models/data_loader.py @@ -106,9 +106,11 @@ def __init__( is_valid: bool = False, train_ratio: float = 0.8, ensemble_num: int = 7, + only_for_training: bool = True, seed: int = 10086, ): self.ensemble_num = ensemble_num + self.only_for_training = only_for_training super(EnsembleBufferDataset, self).__init__( replay_buffer=replay_buffer, @@ -121,15 +123,21 @@ def __init__( ) def build_indexes(self): + indexes = [] np.random.seed(self.seed) - if self.is_valid: # for valid set - self.indexes = np.array( - [np.random.permutation(self.size)[int(self.size * self.train_ratio) :] for _ in range(self.ensemble_num)] - ).T - else: # for train set - self.indexes = np.array( - [np.random.permutation(self.size)[: int(self.size * self.train_ratio)] for _ in range(self.ensemble_num)] - ).T + + if self.only_for_training: # call ``np.random`` ensemble-num + 1 times + assert not self.is_valid + train_indexes = np.random.permutation(self.size)[: int(self.size * self.train_ratio)] + indexes = [np.random.permutation(train_indexes) for _ in range(self.ensemble_num)] + else: + for i in range(self.ensemble_num): + permutation = np.random.permutation(self.size) + if self.is_valid: + indexes.append(permutation[int(self.size * self.train_ratio) :]) + else: + indexes.append(permutation[: int(self.size * self.train_ratio)]) + self.indexes = np.array(indexes).T def collate_fn(data): diff --git a/tests/test_models/test_data_loader.py b/tests/test_models/test_data_loader.py index dde5729..e176ec8 100644 --- a/tests/test_models/test_data_loader.py +++ b/tests/test_models/test_data_loader.py @@ -1,5 +1,6 @@ import gym import emei +import numpy as np from stable_baselines3.common.buffers import ReplayBuffer from torch.utils.data import DataLoader @@ -7,9 +8,8 @@ from cmrl.algorithms.util import load_offline_data -def test_offline_dataset(): +def test_buffer_dataset(): env = gym.make("BoundaryInvertedPendulumSwingUp-v0", freq_rate=1, time_step=0.02) - # assert isinstance(env, emei.EmeiEnv) real_replay_buffer = ReplayBuffer( int(1e5), env.observation_space, env.action_space, "cpu", handle_timeout_termination=False @@ -77,7 +77,7 @@ def test_offline_dataset(): assert outputs[key].shape == (128, 1) -def test_ensemble_offline_dataset(): +def test_ensemble_buffer_dataset(): env = gym.make("BoundaryInvertedPendulumSwingUp-v0", freq_rate=1, time_step=0.02) # assert isinstance(env, emei.EmeiEnv) @@ -145,3 +145,86 @@ def test_ensemble_offline_dataset(): assert inputs[key].shape == (128, 7, 1) for key in outputs: assert outputs[key].shape == (128, 7, 1) + + +def test_train_valid(): + env = gym.make("BoundaryInvertedPendulumSwingUp-v0", freq_rate=1, time_step=0.02) + + real_replay_buffer = ReplayBuffer( + int(1e5), env.observation_space, env.action_space, "cpu", handle_timeout_termination=False + ) + load_offline_data(env, real_replay_buffer, "SAC-expert-replay", use_ratio=0.1) + + train_dataset = BufferDataset( + real_replay_buffer, env.observation_space, env.action_space, mech="transition", is_valid=False + ) + valid_dataset = BufferDataset( + real_replay_buffer, env.observation_space, env.action_space, mech="transition", is_valid=True + ) + + buffer_size = real_replay_buffer.buffer_size if real_replay_buffer.full else real_replay_buffer.pos + assert len(set(train_dataset.indexes).intersection(set(valid_dataset.indexes))) == 0 + assert len(train_dataset.indexes) + len(valid_dataset.indexes) == buffer_size + + +def test_ensemble_train_valid(): + env = gym.make("BoundaryInvertedPendulumSwingUp-v0", freq_rate=1, time_step=0.02) + + real_replay_buffer = ReplayBuffer( + int(1e5), env.observation_space, env.action_space, "cpu", handle_timeout_termination=False + ) + load_offline_data(env, real_replay_buffer, "SAC-expert-replay", use_ratio=0.1) + + ensemble_num = 7 + train_dataset = EnsembleBufferDataset( + real_replay_buffer, + env.observation_space, + env.action_space, + mech="transition", + is_valid=False, + ensemble_num=ensemble_num, + only_for_training=False, + ) + valid_dataset = EnsembleBufferDataset( + real_replay_buffer, + env.observation_space, + env.action_space, + mech="transition", + is_valid=True, + ensemble_num=ensemble_num, + only_for_training=False, + ) + + buffer_size = real_replay_buffer.buffer_size if real_replay_buffer.full else real_replay_buffer.pos + for i in range(ensemble_num): + assert len(set(train_dataset.indexes[:, i]).intersection(set(valid_dataset.indexes[:, i]))) == 0 + assert len(train_dataset.indexes[:, i]) + len(valid_dataset.indexes[:, i]) == buffer_size + + +def test_only_for_training(): + env = gym.make("BoundaryInvertedPendulumSwingUp-v0", freq_rate=1, time_step=0.02) + # assert isinstance(env, emei.EmeiEnv) + + real_replay_buffer = ReplayBuffer( + int(1e5), env.observation_space, env.action_space, "cpu", handle_timeout_termination=False + ) + load_offline_data(env, real_replay_buffer, "SAC-expert-replay", use_ratio=0.1) + + ensemble_num = 7 + train_dataset = EnsembleBufferDataset( + real_replay_buffer, + env.observation_space, + env.action_space, + mech="transition", + is_valid=False, + ensemble_num=ensemble_num, + only_for_training=True, + ) + valid_dataset = BufferDataset( + real_replay_buffer, env.observation_space, env.action_space, mech="transition", is_valid=True + ) + + buffer_size = real_replay_buffer.buffer_size if real_replay_buffer.full else real_replay_buffer.pos + for i in range(ensemble_num): + assert len(set(train_dataset.indexes[:, i]).intersection(set(valid_dataset.indexes))) == 0 + assert len(train_dataset.indexes[:, i]) + len(valid_dataset.indexes) == buffer_size From db4e4fb374618c4d976a55edfdfabf9efd7cd631 Mon Sep 17 00:00:00 2001 From: frank Date: Sat, 5 Nov 2022 19:34:24 +0800 Subject: [PATCH 12/68] :tada: finish plain_mech.py --- cmrl/models/causal_discovery/CMI_test.py | 2 - cmrl/models/causal_mech/base_causal_mech.py | 39 +++- cmrl/models/causal_mech/plain_mech.py | 188 ++++++++++++++---- cmrl/models/data_loader.py | 13 +- cmrl/models/networks/coder.py | 2 +- cmrl/models/networks/mlp.py | 1 - cmrl/models/networks/parallel_mlp.py | 1 - cmrl/models/util.py | 28 +-- .../test_causal_mech/test_plain_mech.py | 50 ++++- tests/test_models/test_data_loader.py | 9 +- 10 files changed, 239 insertions(+), 94 deletions(-) diff --git a/cmrl/models/causal_discovery/CMI_test.py b/cmrl/models/causal_discovery/CMI_test.py index 623b11f..7ec688b 100644 --- a/cmrl/models/causal_discovery/CMI_test.py +++ b/cmrl/models/causal_discovery/CMI_test.py @@ -6,8 +6,6 @@ from torch import nn as nn from torch.nn import functional as F -from cmrl.models.util import gaussian_nll - # from cmrl.models.layers import ParallelEnsembleLinearLayer, truncated_normal_init from cmrl.models.networks.mlp import EnsembleMLP from cmrl.models.util import to_tensor diff --git a/cmrl/models/causal_mech/base_causal_mech.py b/cmrl/models/causal_mech/base_causal_mech.py index 9832dac..2a5f0e5 100644 --- a/cmrl/models/causal_mech/base_causal_mech.py +++ b/cmrl/models/causal_mech/base_causal_mech.py @@ -3,9 +3,10 @@ import torch from torch.utils.data import DataLoader +import torch.nn.functional as F from stable_baselines3.common.logger import Logger -from cmrl.types import Variable, ContinuousVariable, DiscreteVariable +from cmrl.types import Variable, ContinuousVariable, DiscreteVariable, BinaryVariable from cmrl.models.networks.base_network import BaseNetwork from cmrl.models.graphs.base_graph import BaseGraph from cmrl.models.networks.coder import VariableEncoder, VariableDecoder @@ -14,6 +15,7 @@ class BaseCausalMech: def __init__( self, + name: str, input_variables: List[Variable], output_variables: List[Variable], node_dim: int, @@ -21,17 +23,19 @@ def __init__( variable_decoders: Dict[str, VariableDecoder], # forward method residual: bool = True, + multi_step: str = "none", # trainer optim_lr: float = 1e-4, optim_weight_decay: float = 1e-5, optim_eps: float = 1e-8, - optim_coder: bool = True, + optim_encoder: bool = True, # logger logger: Optional[Logger] = None, # others device: Union[str, torch.device] = "cpu", **kwargs ): + self.name = name self.input_variables = input_variables self.output_variables = output_variables self.node_dim = node_dim @@ -39,11 +43,12 @@ def __init__( self.variable_decoders = variable_decoders # forward method self.residual = residual + self.multi_step = multi_step # trainer self.optim_lr = optim_lr self.optim_weight_decay = optim_weight_decay self.optim_eps = optim_eps - self.optim_coder = optim_coder + self.optim_encoder = optim_encoder # logger self.logger = logger # others @@ -60,6 +65,8 @@ def __init__( self.build_network() self.build_graph() + self.total_epoch = 0 + def check_coder(self): assert len(self.input_variables) == len(self.variable_encoders) assert len(self.output_variables) == len(self.variable_decoders) @@ -92,11 +99,27 @@ def build_network(self): def build_graph(self): raise NotImplementedError - def encode(self, inputs): - pass - - def decode(self, hidden): - pass + def loss(self, outputs, targets): + ensemble_num, batch_size = list(targets.values())[0].shape[:2] + total_loss = torch.zeros(ensemble_num, batch_size, self.output_var_num) + for i, var in enumerate(self.output_variables): + output = outputs[var.name] + target = targets[var.name] + if isinstance(var, ContinuousVariable): + dim = target.shape[-1] # ensemble-num, batch-size, dim + assert output.shape[-1] == 2 * dim + mean, log_var = output[:, :, :dim], output[:, :, dim:] + loss = F.gaussian_nll_loss(mean, target, log_var.exp(), reduction="none").mean(dim=-1) + total_loss[..., i] = loss + elif isinstance(var, DiscreteVariable): + # TODO: onehot to int? + raise NotImplementedError + total_loss[..., i] = F.cross_entropy(output, target, reduction="none") + elif isinstance(var, BinaryVariable): + total_loss[..., i] = F.binary_cross_entropy(output, target, reduction="none") + else: + raise NotImplementedError + return total_loss # Causal = TypeVar("Causal", bound=BaseCausalMech) diff --git a/cmrl/models/causal_mech/plain_mech.py b/cmrl/models/causal_mech/plain_mech.py index dfc596d..256ceb4 100644 --- a/cmrl/models/causal_mech/plain_mech.py +++ b/cmrl/models/causal_mech/plain_mech.py @@ -1,7 +1,9 @@ from typing import Optional, List, Dict, Union import pathlib import itertools +import copy +import numpy as np import torch from torch.utils.data import DataLoader import torch.nn.functional as F @@ -9,7 +11,7 @@ from omegaconf import DictConfig from stable_baselines3.common.logger import Logger -from cmrl.types import Variable, ContinuousVariable +from cmrl.types import Variable, ContinuousVariable, DiscreteVariable, BinaryVariable from cmrl.models.networks.parallel_mlp import ParallelMLP from cmrl.models.graphs.base_graph import BaseGraph from cmrl.models.causal_mech.base_causal_mech import BaseCausalMech @@ -19,6 +21,7 @@ class PlainMech(BaseCausalMech): def __init__( self, + name: str, # base causal-mech params input_variables: List[Variable], output_variables: List[Variable], @@ -29,15 +32,17 @@ def __init__( deterministic: bool = False, hidden_dims: Optional[List[int]] = None, ensemble_num: int = 7, + elite_num: int = 5, use_bias: bool = True, activation_fn_cfg: Optional[DictConfig] = None, # forward method residual: bool = True, + multi_step: str = "none", # trainer optim_lr: float = 1e-4, optim_weight_decay: float = 1e-5, optim_eps: float = 1e-8, - optim_coder: bool = True, + optim_encoder: bool = True, # logger logger: Optional[Logger] = None, # others @@ -47,20 +52,28 @@ def __init__( self.deterministic = deterministic self.hidden_dims = hidden_dims if hidden_dims is not None else [200] * 4 self.ensemble_num = ensemble_num + self.elite_num = elite_num self.use_bias = use_bias self.activation_fn_cfg = activation_fn_cfg + if multi_step == "none": + multi_step = "forward-euler 1" + + self.elite_indices: List[int] = [] + super(PlainMech, self).__init__( + name=name, input_variables=input_variables, output_variables=output_variables, node_dim=node_dim, variable_encoders=variable_encoders, variable_decoders=variable_decoders, residual=residual, + multi_step=multi_step, optim_lr=optim_lr, optim_weight_decay=optim_weight_decay, optim_eps=optim_eps, - optim_coder=optim_coder, + optim_encoder=optim_encoder, logger=logger, device=device, **kwargs @@ -76,46 +89,87 @@ def build_network(self): activation_fn_cfg=self.activation_fn_cfg, ) - if self.optim_coder: - parmas = itertools.chain( - self.network.parameters(), - *[encoder.parameters() for encoder in self.variable_encoders.values()], - *[decoder.parameters() for decoder in self.variable_decoders.values()] - ) - self.optim = Adam(parmas, lr=self.optim_lr, weight_decay=self.optim_weight_decay, eps=self.optim_eps) + parmas = [self.network.parameters()] + [decoder.parameters() for decoder in self.variable_decoders.values()] + if self.optim_encoder: + parmas.extend([encoder.parameters() for encoder in self.variable_encoders.values()]) + self.optim = Adam(itertools.chain(*parmas), lr=self.optim_lr, weight_decay=self.optim_weight_decay, eps=self.optim_eps) def build_graph(self): self.graph = None def forward(self, inputs: Dict[str, torch.Tensor]) -> Dict[str, torch.Tensor]: - assert len(inputs) > 0, "inputs should not be null" - ensemble, batch_size = list(inputs.values())[0].shape[:2] - assert ensemble == self.ensemble_num + torch.autograd.set_detect_anomaly(True) assert list(inputs.keys()) == list(self.variable_encoders.keys()) + data_shape = list(inputs.values())[0].shape + assert len(data_shape) == 3 # ensemble-num, batch-size, specific-dim + ensemble, batch_size, specific_dim = data_shape + assert ensemble == self.ensemble_num inputs_tensor = torch.zeros(ensemble, batch_size, self.input_var_num * self.node_dim) - for i, (name, encoder) in enumerate(self.variable_encoders.items()): - out = encoder(inputs[name]) # ensemble-num, batch-size, node-dim + for i, var in enumerate(self.input_variables): + out = self.variable_encoders[var.name](inputs[var.name]) # ensemble-num, batch-size, node-dim inputs_tensor[:, :, i * self.node_dim : (i + 1) * self.node_dim] = out - hidden_tensor = self.network(inputs_tensor) + if self.multi_step.startswith("forward-euler"): + step_num = int(self.multi_step.split()[-1]) + output_tensor = None + for step in range(step_num): + if step > 0: + inputs_tensor = torch.concat( + [output_tensor, inputs_tensor[:, :, self.output_var_num * self.node_dim :]], dim=-1 + ) + output_tensor = self.network(inputs_tensor) + if self.residual: + output_tensor += inputs_tensor[:, :, : self.output_var_num * self.node_dim] + else: + raise NotImplementedError outputs = {} - for i, (name, decoder) in enumerate(self.variable_decoders.items()): - hid = hidden_tensor[:, :, i * self.node_dim : (i + 1) * self.node_dim] - out = decoder(hid) - outputs[name] = out + for i, var in enumerate(self.output_variables): + hid = output_tensor[:, :, i * self.node_dim : (i + 1) * self.node_dim] + out = self.variable_decoders[var.name](hid) + outputs[var.name] = out return outputs - def loss(self, outputs, targets): - loss = torch.tensor(0.0) - for var in self.output_variables: - output = outputs[var.name] - target = targets[var.name] - if isinstance(var, ContinuousVariable): - loss += F.mse_loss(output, target) - return loss + def train(self, loader: DataLoader): + """train for ensemble data + + Args: + loader: train data-loader. + + Returns: tensor of train loss, with shape (ensemble-num, batch-size). + + """ + batch_loss_list = [] + for inputs, targets in loader: + outputs = self.forward(inputs) + loss = self.loss(outputs, targets) # ensemble-num, batch-size, output-var-num + + self.optim.zero_grad() + loss.mean().backward() + self.optim.step() + + batch_loss_list.append(loss) + return torch.cat(batch_loss_list, dim=-2).detach().cpu() + + def eval(self, loader: DataLoader): + """evaluate for non-ensemble data + + Args: + loader: valid data-loader. + + Returns: tensor of eval loss, with shape (batch-size). + + """ + batch_loss_list = [] + with torch.no_grad(): + for inputs, targets in loader: + outputs = self.forward(inputs) + loss = self.loss(outputs, targets) # ensemble-num, batch-size, output-var-num + + batch_loss_list.append(loss) + return torch.cat(batch_loss_list, dim=-2).detach().cpu() def learn( self, @@ -124,7 +178,7 @@ def learn( valid_loader: DataLoader, # model learning longest_epoch: int = -1, - improvement_threshold: float = 0.1, + improvement_threshold: float = 0.01, patience: int = 5, work_dir: Optional[Union[str, pathlib.Path]] = None, **kwargs @@ -132,13 +186,69 @@ def learn( best_weights: Optional[Dict] = None epoch_iter = range(longest_epoch) if longest_epoch > 0 else itertools.count() epochs_since_update = 0 - - for inputs, targets in train_loader: - outputs = self.forward(inputs) - loss = self.loss(outputs, targets) - - self.optim.zero_grad() - loss.backward() - self.optim.step() - - break + best_eval_loss = self.eval(valid_loader).mean(dim=(1, 2)) + + for epoch in epoch_iter: + train_loss = self.train(train_loader) + eval_loss = self.eval(valid_loader).mean(dim=(1, 2)) + + maybe_best_weights = self._maybe_get_best_weights(best_eval_loss, eval_loss, improvement_threshold) + if maybe_best_weights: + # best loss + best_eval_loss = torch.minimum(best_eval_loss, eval_loss) + best_weights = maybe_best_weights + epochs_since_update = 0 + else: + epochs_since_update += 1 + + # log + self.total_epoch += 1 + if self.logger is not None: + self.logger.record("{}/epoch".format(self.name), epoch) + self.logger.record("{}/train_dataset_size".format(self.name), len(train_loader.dataset)) + self.logger.record("{}/valid_dataset_size".format(self.name), len(valid_loader.dataset)) + self.logger.record("{}/train_loss".format(self.name), train_loss.mean().item()) + self.logger.record("{}/val_loss".format(self.name), eval_loss.mean().item()) + self.logger.record("{}/best_val_loss".format(self.name), best_eval_loss.mean().item()) + + self.logger.dump(self.total_epoch) + + if patience and epochs_since_update >= patience: + break + + # saving the best models: + self._maybe_set_best_weights_and_elite(best_weights, best_eval_loss) + + def _maybe_get_best_weights( + self, + best_val_loss: torch.Tensor, + val_loss: torch.Tensor, + threshold: float = 0.01, + ) -> Optional[Dict]: + """Return the current model state dict if the validation score improves. + For ensembles, this checks the validation for each ensemble member separately. + Copy from https://github.com/facebookresearch/mbrl-lib/blob/main/mbrl/models/model_trainer.py + + Args: + best_val_score (tensor): the current best validation losses per model. + val_score (tensor): the new validation loss per model. + threshold (float): the threshold for relative improvement. + Returns: + (dict, optional): if the validation score's relative improvement over the + best validation score is higher than the threshold, returns the state dictionary + of the stored model, otherwise returns ``None``. + """ + improvement = (best_val_loss - val_loss) / torch.abs(best_val_loss) + if (improvement > threshold).any().item(): + best_weights = copy.deepcopy(self.network.state_dict()) + else: + best_weights = None + + return best_weights + + def _maybe_set_best_weights_and_elite(self, best_weights: Optional[Dict], best_val_score: torch.Tensor): + if best_weights is not None: + self.network.load_state_dict(best_weights) + + sorted_indices = np.argsort(best_val_score.tolist()) + self.elite_indices = sorted_indices[: self.elite_num] diff --git a/cmrl/models/data_loader.py b/cmrl/models/data_loader.py index 3939cc7..16d2202 100644 --- a/cmrl/models/data_loader.py +++ b/cmrl/models/data_loader.py @@ -1,3 +1,5 @@ +from typing import Optional + import gym from gym import spaces import torch @@ -16,6 +18,7 @@ def __init__( is_valid: bool = False, train_ratio: float = 0.8, seed: int = 10086, + repeat: Optional[int] = None, ): assert mech in ["transition", "reward_mech", "termination_mech"] # dict action is not supported by SB3(so not done by cmrl) @@ -28,6 +31,7 @@ def __init__( self.is_valid = is_valid self.train_ratio = train_ratio self.seed = seed + self.repeat = repeat self.size = self.replay_buffer.buffer_size if self.replay_buffer.full else self.replay_buffer.pos @@ -87,6 +91,9 @@ def load_from_buffer(self): def __getitem__(self, item): index = self.indexes[item] + if self.repeat: + assert len(self.indexes.shape) == 1 + index = np.tile(index, self.repeat) inputs = dict([(key, self.inputs[key][index]) for key in self.inputs]) outputs = dict([(key, self.outputs[key][index]) for key in self.outputs]) @@ -106,11 +113,11 @@ def __init__( is_valid: bool = False, train_ratio: float = 0.8, ensemble_num: int = 7, - only_for_training: bool = True, + train_ensemble: bool = True, seed: int = 10086, ): self.ensemble_num = ensemble_num - self.only_for_training = only_for_training + self.train_ensemble = train_ensemble super(EnsembleBufferDataset, self).__init__( replay_buffer=replay_buffer, @@ -126,7 +133,7 @@ def build_indexes(self): indexes = [] np.random.seed(self.seed) - if self.only_for_training: # call ``np.random`` ensemble-num + 1 times + if self.train_ensemble: # call ``np.random`` ensemble-num + 1 times assert not self.is_valid train_indexes = np.random.permutation(self.size)[: int(self.size * self.train_ratio)] indexes = [np.random.permutation(train_indexes) for _ in range(self.ensemble_num)] diff --git a/cmrl/models/networks/coder.py b/cmrl/models/networks/coder.py index 4097a7e..894484c 100644 --- a/cmrl/models/networks/coder.py +++ b/cmrl/models/networks/coder.py @@ -59,7 +59,7 @@ def __init__( hidden_dims: Optional[List[int]] = None, bias: bool = True, activation_fn_cfg: Optional[DictConfig] = None, - normal_distribution: bool = False, + normal_distribution: bool = True, ): self.variable = variable self.node_dim = node_dim diff --git a/cmrl/models/networks/mlp.py b/cmrl/models/networks/mlp.py index 7390ebe..ddce2f9 100644 --- a/cmrl/models/networks/mlp.py +++ b/cmrl/models/networks/mlp.py @@ -8,7 +8,6 @@ import numpy as np from omegaconf import DictConfig -from cmrl.models.util import gaussian_nll from cmrl.models.layers import ParallelLinear from cmrl.models.networks.base_network import BaseNetwork diff --git a/cmrl/models/networks/parallel_mlp.py b/cmrl/models/networks/parallel_mlp.py index c977a85..b842fc9 100644 --- a/cmrl/models/networks/parallel_mlp.py +++ b/cmrl/models/networks/parallel_mlp.py @@ -7,7 +7,6 @@ import hydra from omegaconf import DictConfig -from cmrl.models.util import gaussian_nll from cmrl.models.layers import ParallelLinear from cmrl.models.networks.base_network import BaseNetwork, create_activation diff --git a/cmrl/models/util.py b/cmrl/models/util.py index 34a202c..e96268f 100644 --- a/cmrl/models/util.py +++ b/cmrl/models/util.py @@ -15,32 +15,6 @@ from cmrl.models.networks.coder import VariableEncoder, VariableDecoder -def gaussian_nll( - pred_mean: torch.Tensor, - pred_logvar: torch.Tensor, - target: torch.Tensor, - reduce: bool = True, -) -> torch.Tensor: - """Negative log-likelihood for Gaussian distribution - - Args: - pred_mean (tensor): the predicted mean. - pred_logvar (tensor): the predicted log variance. - target (tensor): the target value. - reduce (bool): if ``False`` the loss is returned w/o reducing. - Defaults to ``True``. - - Returns: - (tensor): the negative log-likelihood. - """ - l2 = F.mse_loss(pred_mean, target, reduction="none") - inv_var = (-pred_logvar).exp() - losses = l2 * inv_var + pred_logvar - if reduce: - return losses.sum(dim=1).mean() - return losses - - # inplace truncated normal function for pytorch. # credit to https://github.com/Xingyu-Lin/mbpo_pytorch/blob/main/model.py#L64 def truncated_normal_(tensor: torch.Tensor, mean: float = 0, std: float = 1) -> torch.Tensor: @@ -115,7 +89,7 @@ def create_decoders( hidden_dims: Optional[List[int]] = None, bias: bool = True, activation_fn_cfg: Optional[DictConfig] = None, - normal_distribution: bool = False, + normal_distribution: bool = True, ): decoders = {} for var in input_variables: diff --git a/tests/test_models/test_causal_mech/test_plain_mech.py b/tests/test_models/test_causal_mech/test_plain_mech.py index 1c687d5..61494ba 100644 --- a/tests/test_models/test_causal_mech/test_plain_mech.py +++ b/tests/test_models/test_causal_mech/test_plain_mech.py @@ -11,22 +11,28 @@ from cmrl.models.util import parse_space, create_decoders, create_encoders -def test_single_dim_continuous(): - env = gym.make("BoundaryInvertedPendulumSwingUp-v0", freq_rate=1, time_step=0.02) - # assert isinstance(env, emei.EmeiEnv) +def prepare(freq_rate): + env = gym.make("BoundaryInvertedPendulumSwingUp-v0", freq_rate=freq_rate, time_step=0.02) real_replay_buffer = ReplayBuffer( int(1e5), env.observation_space, env.action_space, "cpu", handle_timeout_termination=False ) - load_offline_data(env, real_replay_buffer, "SAC-expert-replay", use_ratio=0.01) + load_offline_data(env, real_replay_buffer, "SAC-expert-replay", use_ratio=0.001) + ensemble_num = 7 # test for transition tran_dataset = EnsembleBufferDataset( - real_replay_buffer, env.observation_space, env.action_space, is_valid=False, mech="transition" + real_replay_buffer, + env.observation_space, + env.action_space, + is_valid=False, + mech="transition", + train_ensemble=True, + ensemble_num=ensemble_num, ) train_loader = DataLoader(tran_dataset, batch_size=8, collate_fn=collate_fn) - valid_dataset = EnsembleBufferDataset( - real_replay_buffer, env.observation_space, env.action_space, is_valid=True, mech="transition" + valid_dataset = BufferDataset( + real_replay_buffer, env.observation_space, env.action_space, is_valid=True, mech="transition", repeat=ensemble_num ) valid_loader = DataLoader(valid_dataset, batch_size=8, collate_fn=collate_fn) @@ -37,12 +43,40 @@ def test_single_dim_continuous(): output_variables = parse_space(env.observation_space, "next_obs") variable_decoders = create_decoders(output_variables, node_dim=node_dim) + return input_variables, output_variables, node_dim, variable_encoders, variable_decoders, train_loader, valid_loader + + +def test_inv_pendulum_single_step(): + input_variables, output_variables, node_dim, variable_encoders, variable_decoders, train_loader, valid_loader = prepare( + freq_rate=1 + ) + + mech = PlainMech( + name="test", + input_variables=input_variables, + output_variables=output_variables, + node_dim=node_dim, + variable_encoders=variable_encoders, + variable_decoders=variable_decoders, + multi_step="none", + ) + + mech.learn(train_loader, valid_loader, longest_epoch=1) + + +def test_inv_pendulum_multi_step(): + input_variables, output_variables, node_dim, variable_encoders, variable_decoders, train_loader, valid_loader = prepare( + freq_rate=2 + ) + mech = PlainMech( + name="test", input_variables=input_variables, output_variables=output_variables, node_dim=node_dim, variable_encoders=variable_encoders, variable_decoders=variable_decoders, + multi_step="forward-euler 2", ) - mech.learn(train_loader, valid_loader) + mech.learn(train_loader, valid_loader, longest_epoch=1) diff --git a/tests/test_models/test_data_loader.py b/tests/test_models/test_data_loader.py index e176ec8..bb8f455 100644 --- a/tests/test_models/test_data_loader.py +++ b/tests/test_models/test_data_loader.py @@ -100,6 +100,7 @@ def test_ensemble_buffer_dataset(): # test for reward dataset = EnsembleBufferDataset(real_replay_buffer, env.observation_space, env.action_space, mech="reward_mech") + print(dataset[0]) loader = DataLoader(dataset, batch_size=128, drop_last=True) for inputs, outputs in loader: @@ -183,7 +184,7 @@ def test_ensemble_train_valid(): mech="transition", is_valid=False, ensemble_num=ensemble_num, - only_for_training=False, + train_ensemble=False, ) valid_dataset = EnsembleBufferDataset( real_replay_buffer, @@ -192,7 +193,7 @@ def test_ensemble_train_valid(): mech="transition", is_valid=True, ensemble_num=ensemble_num, - only_for_training=False, + train_ensemble=False, ) buffer_size = real_replay_buffer.buffer_size if real_replay_buffer.full else real_replay_buffer.pos @@ -201,7 +202,7 @@ def test_ensemble_train_valid(): assert len(train_dataset.indexes[:, i]) + len(valid_dataset.indexes[:, i]) == buffer_size -def test_only_for_training(): +def test_mixed(): env = gym.make("BoundaryInvertedPendulumSwingUp-v0", freq_rate=1, time_step=0.02) # assert isinstance(env, emei.EmeiEnv) @@ -218,7 +219,7 @@ def test_only_for_training(): mech="transition", is_valid=False, ensemble_num=ensemble_num, - only_for_training=True, + train_ensemble=True, ) valid_dataset = BufferDataset( real_replay_buffer, env.observation_space, env.action_space, mech="transition", is_valid=True From 9cba1977a3885e277c22f3465de1aab8ad1ce705 Mon Sep 17 00:00:00 2001 From: frank Date: Sun, 6 Nov 2022 01:04:29 +0800 Subject: [PATCH 13/68] :tada: add dynamics --- cmrl/algorithms/offline/mopo.py | 19 +-- cmrl/algorithms/offline/off_dyna.py | 77 +++++----- cmrl/algorithms/online/mbpo.py | 3 +- cmrl/algorithms/online/on_dyna.py | 3 +- cmrl/algorithms/util.py | 67 ++++----- cmrl/examples/conf/main.yaml | 6 +- cmrl/examples/conf/reward_mech/plain.yaml | 34 +++++ cmrl/examples/conf/task/BIPS.yaml | 12 -- .../examples/conf/termination_mech/plain.yaml | 34 +++++ cmrl/examples/conf/transition/plain.yaml | 22 ++- cmrl/models/causal_mech/base_causal_mech.py | 17 ++- cmrl/models/causal_mech/plain_mech.py | 18 +-- cmrl/models/dynamics.py | 66 ++++++++- cmrl/models/fake_env.py | 20 +-- cmrl/models/networks/coder.py | 4 +- .../{dynamics => old_dynamics}/__init__.py | 0 .../base_dynamics.py | 0 .../constraint_based_dynamics.py | 0 .../ncd_dynamics.py | 0 .../plain_dynamics.py | 0 cmrl/models/util.py | 8 +- cmrl/sb3_extension/online_mb_callback.py | 4 +- cmrl/util/creator.py | 131 +++++++++++++----- .../test_causal_mech/test_plain_mech.py | 17 ++- 24 files changed, 382 insertions(+), 180 deletions(-) create mode 100644 cmrl/examples/conf/reward_mech/plain.yaml rename cmrl/models/{dynamics => old_dynamics}/__init__.py (100%) rename cmrl/models/{dynamics => old_dynamics}/base_dynamics.py (100%) rename cmrl/models/{dynamics => old_dynamics}/constraint_based_dynamics.py (100%) rename cmrl/models/{dynamics => old_dynamics}/ncd_dynamics.py (100%) rename cmrl/models/{dynamics => old_dynamics}/plain_dynamics.py (100%) diff --git a/cmrl/algorithms/offline/mopo.py b/cmrl/algorithms/offline/mopo.py index 860518e..f043323 100644 --- a/cmrl/algorithms/offline/mopo.py +++ b/cmrl/algorithms/offline/mopo.py @@ -9,8 +9,9 @@ from stable_baselines3.common.buffers import ReplayBuffer from cmrl.agent import complete_agent_cfg -from cmrl.algorithms.util import maybe_load_trained_offline_model, setup_fake_env, load_offline_data -from cmrl.models.dynamics import ConstraintBasedDynamics + +# from cmrl.algorithms.util import maybe_load_trained_offline_model, setup_fake_env, load_offline_data +# from cmrl.models.dynamics import ConstraintBasedDynamics from cmrl.sb3_extension.eval_callback import EvalCallback from cmrl.sb3_extension.logger import configure as logger_configure from cmrl.types import InitObsFnType, RewardFnType, TermFnType @@ -18,13 +19,13 @@ def train( - env: emei.EmeiEnv, - eval_env: emei.EmeiEnv, - termination_fn: Optional[TermFnType], - reward_fn: Optional[RewardFnType], - get_init_obs_fn: Optional[InitObsFnType], - cfg: DictConfig, - work_dir: Optional[str] = None, + env: emei.EmeiEnv, + eval_env: emei.EmeiEnv, + termination_fn: Optional[TermFnType], + reward_fn: Optional[RewardFnType], + get_init_obs_fn: Optional[InitObsFnType], + cfg: DictConfig, + work_dir: Optional[str] = None, ): obs_shape = env.observation_space.shape act_shape = env.action_space.shape diff --git a/cmrl/algorithms/offline/off_dyna.py b/cmrl/algorithms/offline/off_dyna.py index 9993c55..3c4920d 100644 --- a/cmrl/algorithms/offline/off_dyna.py +++ b/cmrl/algorithms/offline/off_dyna.py @@ -9,8 +9,9 @@ from stable_baselines3.common.buffers import ReplayBuffer from cmrl.agent import complete_agent_cfg -from cmrl.algorithms.util import maybe_load_trained_offline_model, setup_fake_env, load_offline_data -from cmrl.models.dynamics import ConstraintBasedDynamics +from cmrl.algorithms.util import setup_fake_env, load_offline_data + +# from cmrl.models.dynamics import ConstraintBasedDynamics from cmrl.sb3_extension.eval_callback import EvalCallback from cmrl.sb3_extension.logger import configure as logger_configure from cmrl.types import InitObsFnType, RewardFnType, TermFnType @@ -18,17 +19,14 @@ def train( - env: emei.EmeiEnv, - eval_env: emei.EmeiEnv, - termination_fn: Optional[TermFnType], - reward_fn: Optional[RewardFnType], - get_init_obs_fn: Optional[InitObsFnType], - cfg: DictConfig, - work_dir: Optional[str] = None, + env: emei.EmeiEnv, + eval_env: emei.EmeiEnv, + termination_fn: Optional[TermFnType], + reward_fn: Optional[RewardFnType], + get_init_obs_fn: Optional[InitObsFnType], + cfg: DictConfig, + work_dir: Optional[str] = None, ): - obs_shape = env.observation_space.shape - act_shape = env.action_space.shape - # build model-free agent, which is a stable-baselines3's agent complete_agent_cfg(env, cfg.algorithm.agent) agent = cast(BaseAlgorithm, hydra.utils.instantiate(cfg.algorithm.agent)) @@ -37,11 +35,12 @@ def train( logger = logger_configure("log", ["tensorboard", "multi_csv", "stdout"]) # create initial dataset and add it to replay buffer - dynamics = create_dynamics(cfg.dynamics, obs_shape, act_shape, logger=logger) + dynamics = create_dynamics(cfg, env.observation_space, env.action_space, logger=logger) + real_replay_buffer = ReplayBuffer( cfg.task.num_steps, env.observation_space, env.action_space, cfg.device, handle_timeout_termination=False ) - load_offline_data(cfg, env, real_replay_buffer) + load_offline_data(env, real_replay_buffer, cfg.task.dataset, cfg.task.use_ratio) fake_eval_env = setup_fake_env( cfg=cfg, @@ -54,28 +53,28 @@ def train( max_episode_steps=env.spec.max_episode_steps, penalty_coeff=cfg.algorithm.penalty_coeff, ) - - if hasattr(env, "get_causal_graph"): - oracle_causal_graph = env.get_causal_graph() - else: - oracle_causal_graph = None - - if isinstance(dynamics, ConstraintBasedDynamics): - dynamics.set_oracle_mask("transition", oracle_causal_graph.T) - - existed_trained_model = maybe_load_trained_offline_model(dynamics, cfg, obs_shape, act_shape, work_dir=work_dir) - if not existed_trained_model: - dynamics.learn(real_replay_buffer, **cfg.dynamics, work_dir=work_dir) - - eval_callback = EvalCallback( - eval_env, - fake_eval_env, - n_eval_episodes=cfg.task.n_eval_episodes, - best_model_save_path="./", - eval_freq=1000, - deterministic=True, - render=False, - ) - - agent.set_logger(logger) - agent.learn(total_timesteps=cfg.task.num_steps, callback=eval_callback) + # + # if hasattr(env, "get_causal_graph"): + # oracle_causal_graph = env.get_causal_graph() + # else: + # oracle_causal_graph = None + # + # if isinstance(dynamics, ConstraintBasedDynamics): + # dynamics.set_oracle_mask("transition", oracle_causal_graph.T) + # + # existed_trained_model = maybe_load_trained_offline_model(dynamics, cfg, obs_shape, act_shape, work_dir=work_dir) + # if not existed_trained_model: + dynamics.learn(real_replay_buffer, **cfg.dynamics, work_dir=work_dir) + # + # eval_callback = EvalCallback( + # eval_env, + # fake_eval_env, + # n_eval_episodes=cfg.task.n_eval_episodes, + # best_model_save_path="./", + # eval_freq=1000, + # deterministic=True, + # render=False, + # ) + # + # agent.set_logger(logger) + # agent.learn(total_timesteps=cfg.task.num_steps, callback=eval_callback) diff --git a/cmrl/algorithms/online/mbpo.py b/cmrl/algorithms/online/mbpo.py index d858141..c3479e7 100644 --- a/cmrl/algorithms/online/mbpo.py +++ b/cmrl/algorithms/online/mbpo.py @@ -11,7 +11,8 @@ from cmrl.agent import complete_agent_cfg from cmrl.algorithms.util import setup_fake_env -from cmrl.models.dynamics import ConstraintBasedDynamics + +# from cmrl.models.dynamics import ConstraintBasedDynamics from cmrl.sb3_extension.eval_callback import EvalCallback from cmrl.sb3_extension.online_mb_callback import OnlineModelBasedCallback from cmrl.sb3_extension.logger import configure as logger_configure diff --git a/cmrl/algorithms/online/on_dyna.py b/cmrl/algorithms/online/on_dyna.py index a1056ca..912e0d8 100644 --- a/cmrl/algorithms/online/on_dyna.py +++ b/cmrl/algorithms/online/on_dyna.py @@ -11,7 +11,8 @@ from cmrl.agent import complete_agent_cfg from cmrl.algorithms.util import setup_fake_env -from cmrl.models.dynamics import ConstraintBasedDynamics + +# from cmrl.models.dynamics import ConstraintBasedDynamics from cmrl.sb3_extension.eval_callback import EvalCallback from cmrl.sb3_extension.online_mb_callback import OnlineModelBasedCallback from cmrl.sb3_extension.logger import configure as logger_configure diff --git a/cmrl/algorithms/util.py b/cmrl/algorithms/util.py index 7f14781..e59816f 100644 --- a/cmrl/algorithms/util.py +++ b/cmrl/algorithms/util.py @@ -12,7 +12,8 @@ from stable_baselines3.common.buffers import ReplayBuffer from cmrl.types import InitObsFnType, RewardFnType, TermFnType -from cmrl.models.dynamics import BaseDynamics + +# from cmrl.models.dynamics import BaseDynamics from cmrl.util.config import get_complete_dynamics_cfg, load_hydra_cfg from cmrl.models.fake_env import VecFakeEnv @@ -31,38 +32,38 @@ def is_same_dict(dict1, dict2): return True -def maybe_load_trained_offline_model(dynamics: BaseDynamics, cfg, obs_shape, act_shape, work_dir): - work_dir = pathlib.Path(work_dir) - if "." not in work_dir.name: # exp by hydra's MULTIRUN mode - task_exp_dir = work_dir.parent.parent.parent - else: - task_exp_dir = work_dir.parent.parent - dynamics_cfg = cfg.dynamics - - for date_dir in task_exp_dir.glob(r"*"): - for time_dir in date_dir.glob(r"*"): - if (time_dir / "multirun.yaml").exists(): # exp by hydra's MULTIRUN mode, multi exp in this time - this_time_exp_dir_list = list(time_dir.glob(r"*")) - else: # only one exp in this time - this_time_exp_dir_list = [time_dir] - - for exp_dir in this_time_exp_dir_list: - if not (exp_dir / ".hydra").exists(): - continue - exp_cfg = load_hydra_cfg(exp_dir) - exp_dynamics_cfg = get_complete_dynamics_cfg(exp_cfg.dynamics, obs_shape, act_shape) - - if exp_cfg.seed == cfg.seed and is_same_dict(dynamics_cfg, exp_dynamics_cfg): - exist_model_file = True - for mech in dynamics.learn_mech: - mech_file_name = getattr(dynamics, mech).model_file_name - if not (exp_dir / mech_file_name).exists(): - exist_model_file = False - if exist_model_file: - dynamics.load(exp_dir) - print("loaded dynamics from {}".format(exp_dir)) - return True - return False +# def maybe_load_trained_offline_model(dynamics: BaseDynamics, cfg, obs_shape, act_shape, work_dir): +# work_dir = pathlib.Path(work_dir) +# if "." not in work_dir.name: # exp by hydra's MULTIRUN mode +# task_exp_dir = work_dir.parent.parent.parent +# else: +# task_exp_dir = work_dir.parent.parent +# dynamics_cfg = cfg.dynamics +# +# for date_dir in task_exp_dir.glob(r"*"): +# for time_dir in date_dir.glob(r"*"): +# if (time_dir / "multirun.yaml").exists(): # exp by hydra's MULTIRUN mode, multi exp in this time +# this_time_exp_dir_list = list(time_dir.glob(r"*")) +# else: # only one exp in this time +# this_time_exp_dir_list = [time_dir] +# +# for exp_dir in this_time_exp_dir_list: +# if not (exp_dir / ".hydra").exists(): +# continue +# exp_cfg = load_hydra_cfg(exp_dir) +# exp_dynamics_cfg = get_complete_dynamics_cfg(exp_cfg.dynamics, obs_shape, act_shape) +# +# if exp_cfg.seed == cfg.seed and is_same_dict(dynamics_cfg, exp_dynamics_cfg): +# exist_model_file = True +# for mech in dynamics.learn_mech: +# mech_file_name = getattr(dynamics, mech).model_file_name +# if not (exp_dir / mech_file_name).exists(): +# exist_model_file = False +# if exist_model_file: +# dynamics.load(exp_dir) +# print("loaded dynamics from {}".format(exp_dir)) +# return True +# return False def setup_fake_env( diff --git a/cmrl/examples/conf/main.yaml b/cmrl/examples/conf/main.yaml index 4a580ac..83fc84f 100644 --- a/cmrl/examples/conf/main.yaml +++ b/cmrl/examples/conf/main.yaml @@ -2,12 +2,16 @@ defaults: - algorithm: off_dyna - dynamics: constraint_based_dynamics - task: BIPS + + - transition: plain + - reward_mech: plain + - termination_mech: plain - _self_ seed: 0 device: "cuda:0" -exp_name: causal_mstep +exp_name: refactor wandb: false root_dir: "./exp" diff --git a/cmrl/examples/conf/reward_mech/plain.yaml b/cmrl/examples/conf/reward_mech/plain.yaml new file mode 100644 index 0000000..8d06712 --- /dev/null +++ b/cmrl/examples/conf/reward_mech/plain.yaml @@ -0,0 +1,34 @@ +name: "plain_reward" +learn: false + +mech: + _partial_: true + _recursive_: false + _target_: cmrl.models.causal_mech.PlainMech + name: ${transition.name} + # base causal-mech params + input_variables: ??? + output_variables: ??? + node_dim: 20 + variable_encoders: ??? + variable_decoders: ??? + # network params + deterministic: false + hidden_dims: [ 200, 200, 200, 200 ] + ensemble_num: 7 + elite_num: 5 + use_bias: true + activation_fn_cfg: + _target_: torch.nn.SiLU + # forward method + residual: true + multi_step: "none" + # trainer + optim_lr: 1e-4 + optim_weight_decay: 1e-5 + optim_eps: 1e-8 + optim_encoder: true + # logger + logger: ??? + # others + device: ${device} diff --git a/cmrl/examples/conf/task/BIPS.yaml b/cmrl/examples/conf/task/BIPS.yaml index 39fa95c..d8560ca 100644 --- a/cmrl/examples/conf/task/BIPS.yaml +++ b/cmrl/examples/conf/task/BIPS.yaml @@ -47,15 +47,3 @@ rollout_schedule: [ 1, 15, 1, 1 ] num_sac_updates_per_step: 1 sac_updates_every_steps: 1 num_epochs_to_retain_sac_buffer: 1 - -# SAC -sac_gamma: 0.99 -sac_tau: 0.005 -sac_alpha: 0.2 -sac_policy: "Gaussian" -sac_target_update_interval: 1 -sac_automatic_entropy_tuning: true -sac_hidden_size: 256 -sac_lr: 0.0003 -sac_batch_size: 256 -sac_target_entropy: -1 diff --git a/cmrl/examples/conf/termination_mech/plain.yaml b/cmrl/examples/conf/termination_mech/plain.yaml index e69de29..37ff0be 100644 --- a/cmrl/examples/conf/termination_mech/plain.yaml +++ b/cmrl/examples/conf/termination_mech/plain.yaml @@ -0,0 +1,34 @@ +name: "plain_termination" +learn: false + +mech: + _partial_: true + _recursive_: false + _target_: cmrl.models.causal_mech.PlainMech + name: ${transition.name} + # base causal-mech params + input_variables: ??? + output_variables: ??? + node_dim: 20 + variable_encoders: ??? + variable_decoders: ??? + # network params + deterministic: false + hidden_dims: [ 200, 200, 200, 200 ] + ensemble_num: 7 + elite_num: 5 + use_bias: true + activation_fn_cfg: + _target_: torch.nn.SiLU + # forward method + residual: true + multi_step: "none" + # trainer + optim_lr: 1e-4 + optim_weight_decay: 1e-5 + optim_eps: 1e-8 + optim_encoder: true + # logger + logger: ??? + # others + device: ${device} diff --git a/cmrl/examples/conf/transition/plain.yaml b/cmrl/examples/conf/transition/plain.yaml index febdecc..1411538 100644 --- a/cmrl/examples/conf/transition/plain.yaml +++ b/cmrl/examples/conf/transition/plain.yaml @@ -1,24 +1,34 @@ name: "plain_transition" -#multi_step: "forward_euler_5" -multi_step: "none" -enable_coder: false +learn: true -transition: +mech: + _partial_: true + _recursive_: false _target_: cmrl.models.causal_mech.PlainMech + name: ${transition.name} # base causal-mech params input_variables: ??? output_variables: ??? - node_dim: 1 + node_dim: 20 variable_encoders: ??? variable_decoders: ??? # network params deterministic: false - hidden_dims: [200, 200, 200, 200] + hidden_dims: [ 200, 200, 200, 200 ] ensemble_num: 7 + elite_num: 5 use_bias: true activation_fn_cfg: _target_: torch.nn.SiLU # forward method residual: true + multi_step: "none" + # trainer + optim_lr: 1e-4 + optim_weight_decay: 1e-5 + optim_eps: 1e-8 + optim_encoder: true + # logger + logger: ??? # others device: ${device} diff --git a/cmrl/models/causal_mech/base_causal_mech.py b/cmrl/models/causal_mech/base_causal_mech.py index 2a5f0e5..1024dec 100644 --- a/cmrl/models/causal_mech/base_causal_mech.py +++ b/cmrl/models/causal_mech/base_causal_mech.py @@ -10,6 +10,7 @@ from cmrl.models.networks.base_network import BaseNetwork from cmrl.models.graphs.base_graph import BaseGraph from cmrl.models.networks.coder import VariableEncoder, VariableDecoder +from cmrl.models.util import parse_space, create_decoders, create_encoders class BaseCausalMech: @@ -19,8 +20,10 @@ def __init__( input_variables: List[Variable], output_variables: List[Variable], node_dim: int, - variable_encoders: Dict[str, VariableEncoder], - variable_decoders: Dict[str, VariableDecoder], + variable_encoders: Optional[Dict[str, VariableEncoder]], + variable_decoders: Optional[Dict[str, VariableDecoder]], + ensemble_num: int = 7, + elite_num: int = 5, # forward method residual: bool = True, multi_step: str = "none", @@ -41,6 +44,8 @@ def __init__( self.node_dim = node_dim self.variable_encoders = variable_encoders self.variable_decoders = variable_decoders + self.ensemble_num = ensemble_num + self.elite_num = elite_num # forward method self.residual = residual self.multi_step = multi_step @@ -57,6 +62,11 @@ def __init__( self.input_var_num = len(self.input_variables) self.output_var_num = len(self.output_variables) + if self.variable_encoders is None: + assert self.optim_encoder + self.variable_encoders = create_encoders(input_variables, node_dim=self.node_dim, device=self.device) + if self.variable_decoders is None: + self.variable_decoders = create_decoders(output_variables, node_dim=self.node_dim, device=self.device) self.check_coder() self.network: Optional[BaseNetwork] = None @@ -66,6 +76,7 @@ def __init__( self.build_graph() self.total_epoch = 0 + self.elite_indices: List[int] = [] def check_coder(self): assert len(self.input_variables) == len(self.variable_encoders) @@ -104,7 +115,7 @@ def loss(self, outputs, targets): total_loss = torch.zeros(ensemble_num, batch_size, self.output_var_num) for i, var in enumerate(self.output_variables): output = outputs[var.name] - target = targets[var.name] + target = targets[var.name].to(self.device) if isinstance(var, ContinuousVariable): dim = target.shape[-1] # ensemble-num, batch-size, dim assert output.shape[-1] == 2 * dim diff --git a/cmrl/models/causal_mech/plain_mech.py b/cmrl/models/causal_mech/plain_mech.py index 256ceb4..a26c147 100644 --- a/cmrl/models/causal_mech/plain_mech.py +++ b/cmrl/models/causal_mech/plain_mech.py @@ -26,13 +26,13 @@ def __init__( input_variables: List[Variable], output_variables: List[Variable], node_dim: int, - variable_encoders: Dict[str, VariableEncoder], - variable_decoders: Dict[str, VariableDecoder], + variable_encoders: Optional[Dict[str, VariableEncoder]], + variable_decoders: Optional[Dict[str, VariableDecoder]], + ensemble_num: int = 7, + elite_num: int = 5, # network params deterministic: bool = False, hidden_dims: Optional[List[int]] = None, - ensemble_num: int = 7, - elite_num: int = 5, use_bias: bool = True, activation_fn_cfg: Optional[DictConfig] = None, # forward method @@ -51,16 +51,12 @@ def __init__( ): self.deterministic = deterministic self.hidden_dims = hidden_dims if hidden_dims is not None else [200] * 4 - self.ensemble_num = ensemble_num - self.elite_num = elite_num self.use_bias = use_bias self.activation_fn_cfg = activation_fn_cfg if multi_step == "none": multi_step = "forward-euler 1" - self.elite_indices: List[int] = [] - super(PlainMech, self).__init__( name=name, input_variables=input_variables, @@ -87,7 +83,7 @@ def build_network(self): use_bias=self.use_bias, extra_dims=[self.ensemble_num], activation_fn_cfg=self.activation_fn_cfg, - ) + ).to(self.device) parmas = [self.network.parameters()] + [decoder.parameters() for decoder in self.variable_decoders.values()] if self.optim_encoder: @@ -105,9 +101,9 @@ def forward(self, inputs: Dict[str, torch.Tensor]) -> Dict[str, torch.Tensor]: ensemble, batch_size, specific_dim = data_shape assert ensemble == self.ensemble_num - inputs_tensor = torch.zeros(ensemble, batch_size, self.input_var_num * self.node_dim) + inputs_tensor = torch.zeros(ensemble, batch_size, self.input_var_num * self.node_dim).to(self.device) for i, var in enumerate(self.input_variables): - out = self.variable_encoders[var.name](inputs[var.name]) # ensemble-num, batch-size, node-dim + out = self.variable_encoders[var.name](inputs[var.name].to(self.device)) # ensemble-num, batch-size, node-dim inputs_tensor[:, :, i * self.node_dim : (i + 1) * self.node_dim] = out if self.multi_step.startswith("forward-euler"): diff --git a/cmrl/models/dynamics.py b/cmrl/models/dynamics.py index 4b47f18..9c2b592 100644 --- a/cmrl/models/dynamics.py +++ b/cmrl/models/dynamics.py @@ -5,6 +5,8 @@ import numpy as np import torch +from gym import spaces +from torch.utils.data import DataLoader from stable_baselines3.common.logger import Logger from stable_baselines3.common.buffers import ReplayBuffer @@ -14,14 +16,68 @@ from cmrl.types import InteractionBatch from cmrl.util.transition_iterator import BootstrapIterator, TransitionIterator from cmrl.models.causal_mech.base_causal_mech import BaseCausalMech - - -def split_dict(old_dict: Dict, need_keys: List[str]): - return dict([(key, old_dict[key]) for key in need_keys]) +from cmrl.models.data_loader import BufferDataset, EnsembleBufferDataset, collate_fn class Dynamics: def __init__( - self, transition: BaseCausalMech, reward_mech: Optional[BaseCausalMech], terminal_mech: Optional[BaseCausalMech] + self, + transition: BaseCausalMech, + reward_mech: Optional[BaseCausalMech], + termination_mech: Optional[BaseCausalMech], + observation_space: spaces.Space, + action_space: spaces.Space, + logger: Optional[Logger] = None, ): + self.transition = transition + self.reward_mech = reward_mech + self.termination_mech = termination_mech + self.observation_space = observation_space + self.action_space = action_space + self.logger = logger + + self.learn_reward = reward_mech is None + self.learn_termination = termination_mech is None + + self.device = self.transition.device pass + + def get_loader(self, real_replay_buffer, mech: str, batch_size: int = 128): + train_dataset = EnsembleBufferDataset( + real_replay_buffer, + self.observation_space, + self.action_space, + is_valid=False, + mech=mech, + train_ensemble=True, + ensemble_num=self.transition.ensemble_num, + ) + train_loader = DataLoader(train_dataset, batch_size=8, collate_fn=collate_fn) + valid_dataset = BufferDataset( + real_replay_buffer, + self.observation_space, + self.action_space, + is_valid=True, + mech=mech, + repeat=self.transition.ensemble_num, + ) + valid_loader = DataLoader(valid_dataset, batch_size=8, collate_fn=collate_fn) + + return train_loader, valid_loader + + def learn( + self, + real_replay_buffer: ReplayBuffer, + # model learning + longest_epoch: int = -1, + improvement_threshold: float = 0.01, + patience: int = 5, + work_dir: Optional[Union[str, pathlib.Path]] = None, + **kwargs + ): + # transition + self.transition.learn(*self.get_loader(real_replay_buffer, "transition")) + # reward-mech + self.reward_mech.learn(*self.get_loader(real_replay_buffer, "reward_mech")) + # termination-mech + self.termination_mech.learn(*self.get_loader(real_replay_buffer, "termination_mech")) diff --git a/cmrl/models/fake_env.py b/cmrl/models/fake_env.py index 68d195a..9bebe18 100644 --- a/cmrl/models/fake_env.py +++ b/cmrl/models/fake_env.py @@ -17,7 +17,7 @@ from stable_baselines3.common.buffers import ReplayBuffer import cmrl.types -from cmrl.models.dynamics import BaseDynamics +from cmrl.models.dynamics import Dynamics class VecFakeEnv(VecEnv): @@ -42,8 +42,8 @@ def __init__( self.dynamics = None self.reward_fn = None self.termination_fn = None - self.learned_reward = None - self.learned_termination = None + self.learn_reward = None + self.learn_termination = None self.get_init_obs_fn = None self.replay_buffer = None self.generator = np.random.default_rng() @@ -59,7 +59,7 @@ def __init__( def set_up( self, - dynamics: BaseDynamics, + dynamics: Dynamics, reward_fn: Optional[cmrl.types.RewardFnType] = None, termination_fn: Optional[cmrl.types.TermFnType] = None, get_init_obs_fn: Optional[cmrl.types.InitObsFnType] = None, @@ -77,10 +77,10 @@ def set_up( self.reward_fn = reward_fn self.termination_fn = termination_fn - assert self.dynamics.learned_reward or reward_fn - assert self.dynamics.learned_termination or termination_fn - self.learned_reward = self.dynamics.learned_reward - self.learned_termination = self.dynamics.learned_termination + assert self.dynamics.learn_reward or reward_fn + assert self.dynamics.learn_termination or termination_fn + self.learn_reward = self.dynamics.learn_reward + self.learn_termination = self.dynamics.learn_termination self.get_init_obs_fn = get_init_obs_fn self.replay_buffer = real_replay_buffer self.logger = logger @@ -104,11 +104,11 @@ def step_wait(self): # transition batch_next_obs = self.get_dynamics_predict(dynamics_pred, "transition", deterministic=self.deterministic) - if self.learned_reward: + if self.learn_reward: batch_reward = self.get_dynamics_predict(dynamics_pred, "reward_mech", deterministic=self.deterministic) else: batch_reward = self.reward_fn(batch_next_obs, self._current_batch_obs, self._current_batch_action) - if self.learned_termination: + if self.learn_termination: batch_terminal = self.get_dynamics_predict(dynamics_pred, "termination_mech", deterministic=self.deterministic) else: batch_terminal = self.termination_fn(batch_next_obs, self._current_batch_obs, self._current_batch_action) diff --git a/cmrl/models/networks/coder.py b/cmrl/models/networks/coder.py index 894484c..d689105 100644 --- a/cmrl/models/networks/coder.py +++ b/cmrl/models/networks/coder.py @@ -41,7 +41,7 @@ def build(self): elif isinstance(self.variable, BinaryVariable): layers.append(nn.Linear(1, hidden_dim)) else: - raise NotImplementedError + raise NotImplementedError("Type {} is not supported by VariableEncoder".format(type(self.variable))) hidden_dims = self.hidden_dims + [self.node_dim] for i in range(len(hidden_dims) - 1): @@ -98,6 +98,6 @@ def build(self): layers.append(nn.Linear(hidden_dim, 1)) layers.append(nn.Sigmoid()) else: - raise NotImplementedError + raise NotImplementedError("Type {} is not supported by VariableDecoder".format(type(self.variable))) self._layers = nn.ModuleList(layers) diff --git a/cmrl/models/dynamics/__init__.py b/cmrl/models/old_dynamics/__init__.py similarity index 100% rename from cmrl/models/dynamics/__init__.py rename to cmrl/models/old_dynamics/__init__.py diff --git a/cmrl/models/dynamics/base_dynamics.py b/cmrl/models/old_dynamics/base_dynamics.py similarity index 100% rename from cmrl/models/dynamics/base_dynamics.py rename to cmrl/models/old_dynamics/base_dynamics.py diff --git a/cmrl/models/dynamics/constraint_based_dynamics.py b/cmrl/models/old_dynamics/constraint_based_dynamics.py similarity index 100% rename from cmrl/models/dynamics/constraint_based_dynamics.py rename to cmrl/models/old_dynamics/constraint_based_dynamics.py diff --git a/cmrl/models/dynamics/ncd_dynamics.py b/cmrl/models/old_dynamics/ncd_dynamics.py similarity index 100% rename from cmrl/models/dynamics/ncd_dynamics.py rename to cmrl/models/old_dynamics/ncd_dynamics.py diff --git a/cmrl/models/dynamics/plain_dynamics.py b/cmrl/models/old_dynamics/plain_dynamics.py similarity index 100% rename from cmrl/models/dynamics/plain_dynamics.py rename to cmrl/models/old_dynamics/plain_dynamics.py diff --git a/cmrl/models/util.py b/cmrl/models/util.py index e96268f..7651681 100644 --- a/cmrl/models/util.py +++ b/cmrl/models/util.py @@ -2,7 +2,7 @@ # # This source code is licensed under the MIT license found in the # LICENSE file in the root directory of this source tree. -from typing import List, Optional +from typing import List, Optional, Union import numpy as np import torch @@ -73,13 +73,14 @@ def create_encoders( hidden_dims: Optional[List[int]] = None, bias: bool = True, activation_fn_cfg: Optional[DictConfig] = None, + device: Union[str, torch.device] = "cpu", ): encoders = {} for var in input_variables: assert var.name not in encoders, "Duplicate name in decoders: {}".format(var.name) encoders[var.name] = VariableEncoder( variable=var, node_dim=node_dim, hidden_dims=hidden_dims, bias=bias, activation_fn_cfg=activation_fn_cfg - ) + ).to(device) return encoders @@ -90,6 +91,7 @@ def create_decoders( bias: bool = True, activation_fn_cfg: Optional[DictConfig] = None, normal_distribution: bool = True, + device: Union[str, torch.device] = "cpu", ): decoders = {} for var in input_variables: @@ -101,5 +103,5 @@ def create_decoders( bias=bias, activation_fn_cfg=activation_fn_cfg, normal_distribution=normal_distribution, - ) + ).to(device) return decoders diff --git a/cmrl/sb3_extension/online_mb_callback.py b/cmrl/sb3_extension/online_mb_callback.py index 49cb918..582574f 100644 --- a/cmrl/sb3_extension/online_mb_callback.py +++ b/cmrl/sb3_extension/online_mb_callback.py @@ -15,14 +15,14 @@ ) from cmrl.models.fake_env import VecFakeEnv -from cmrl.models.dynamics.base_dynamics import BaseDynamics +from cmrl.models.dynamics import Dynamics class OnlineModelBasedCallback(BaseCallback): def __init__( self, env: gym.Env, - dynamics: BaseDynamics, + dynamics: Dynamics, real_replay_buffer: ReplayBuffer, total_num_steps: int = int(1e5), initial_exploration_steps: int = 1000, diff --git a/cmrl/util/creator.py b/cmrl/util/creator.py index f9fc723..97c640b 100644 --- a/cmrl/util/creator.py +++ b/cmrl/util/creator.py @@ -2,63 +2,122 @@ from typing import Callable, Dict, List, Optional, Sequence, Tuple, Type, Union import gym.wrappers +from gym import spaces import hydra +from hydra.utils import instantiate import numpy as np import omegaconf from stable_baselines3.common.logger import Logger -from cmrl.models.dynamics import ConstraintBasedDynamics, PlainEnsembleDynamics +from cmrl.models.dynamics import Dynamics from cmrl.models.transition import ForwardEulerTransition from cmrl.util.config import get_complete_dynamics_cfg +from cmrl.models.util import parse_space, create_decoders, create_encoders +from cmrl.types import DiscreteVariable, ContinuousVariable, BinaryVariable def create_dynamics( - dynamics_cfg: omegaconf.DictConfig, - obs_shape: Tuple[int, ...], - act_shape: Tuple[int, ...], + cfg, + observation_space: spaces.Space, + action_space: spaces.Space, logger: Optional[Logger] = None, - load_dir: Optional[Union[str, pathlib.Path]] = None, - load_device: Optional[str] = None, ): - if dynamics_cfg.name == "plain_dynamics": - dynamics_class = PlainEnsembleDynamics - elif dynamics_cfg.name == "constraint_based_dynamics": - dynamics_class = ConstraintBasedDynamics - else: - raise NotImplementedError - - dynamics_cfg = get_complete_dynamics_cfg(dynamics_cfg, obs_shape, act_shape) - transition = hydra.utils.instantiate(dynamics_cfg.transition, _recursive_=False) - if dynamics_cfg.multi_step == "none": - pass - elif dynamics_cfg.multi_step.startswith("forward_euler"): - repeat_times = int(dynamics_cfg.multi_step[len("forward_euler") + 1 :]) - transition = ForwardEulerTransition(transition, repeat_times) - else: - raise NotImplementedError + obs_variables = parse_space(observation_space, "obs") + act_variables = parse_space(action_space, "act") + next_obs_variables = parse_space(observation_space, "next_obs") - if dynamics_cfg.learned_reward: - reward_mech = hydra.utils.instantiate(dynamics_cfg.reward_mech, _recursive_=False) + # transition + assert cfg.transition.learn + # TODO: share encoders + transition = instantiate(cfg.transition.mech)( + input_variables=obs_variables + act_variables, + output_variables=next_obs_variables, + variable_encoders=None, + variable_decoders=None, + logger=logger, + ) + # reward mech + if cfg.reward_mech.learn: + reward_mech = instantiate(cfg.reward_mech.mech)( + input_variables=obs_variables + act_variables + next_obs_variables, + output_variables=[ContinuousVariable("reward", dim=1, low=-np.inf, high=np.inf)], + variable_encoders=None, + variable_decoders=None, + logger=logger, + ) else: reward_mech = None - - if dynamics_cfg.learned_termination: - termination_mech = hydra.utils.instantiate(dynamics_cfg.termination_mech, _recursive_=False) - raise NotImplementedError + # termination mech + if cfg.reward_mech.learn: + termination_mech = instantiate(cfg.termination_mech.mech)( + input_variables=obs_variables + act_variables + next_obs_variables, + output_variables=[BinaryVariable("terminal")], + variable_encoders=None, + variable_decoders=None, + logger=logger, + ) else: termination_mech = None - dynamics_model = dynamics_class( + dynamics = Dynamics( transition=transition, - learned_reward=dynamics_cfg.learned_reward, reward_mech=reward_mech, - learned_termination=dynamics_cfg.learned_termination, termination_mech=termination_mech, - optim_lr=dynamics_cfg.optim_lr, - weight_decay=dynamics_cfg.weight_decay, + observation_space=observation_space, + action_space=action_space, logger=logger, ) - if load_dir: - dynamics_model.load(load_dir, load_device) - return dynamics_model + return dynamics + + +# def create_dynamics( +# dynamics_cfg: omegaconf.DictConfig, +# obs_shape: Tuple[int, ...], +# act_shape: Tuple[int, ...], +# logger: Optional[Logger] = None, +# load_dir: Optional[Union[str, pathlib.Path]] = None, +# load_device: Optional[str] = None, +# ): +# if dynamics_cfg.name == "plain_dynamics": +# dynamics_class = PlainEnsembleDynamics +# elif dynamics_cfg.name == "constraint_based_dynamics": +# dynamics_class = ConstraintBasedDynamics +# else: +# raise NotImplementedError +# +# dynamics_cfg = get_complete_dynamics_cfg(dynamics_cfg, obs_shape, act_shape) +# transition = hydra.utils.instantiate(dynamics_cfg.transition, _recursive_=False) +# if dynamics_cfg.multi_step == "none": +# pass +# elif dynamics_cfg.multi_step.startswith("forward_euler"): +# repeat_times = int(dynamics_cfg.multi_step[len("forward_euler") + 1 :]) +# transition = ForwardEulerTransition(transition, repeat_times) +# else: +# raise NotImplementedError +# +# if dynamics_cfg.learned_reward: +# reward_mech = hydra.utils.instantiate(dynamics_cfg.reward_mech, _recursive_=False) +# else: +# reward_mech = None +# +# if dynamics_cfg.learned_termination: +# termination_mech = hydra.utils.instantiate(dynamics_cfg.termination_mech, _recursive_=False) +# raise NotImplementedError +# else: +# termination_mech = None +# +# dynamics_model = dynamics_class( +# transition=transition, +# learned_reward=dynamics_cfg.learned_reward, +# reward_mech=reward_mech, +# learned_termination=dynamics_cfg.learned_termination, +# termination_mech=termination_mech, +# optim_lr=dynamics_cfg.optim_lr, +# weight_decay=dynamics_cfg.weight_decay, +# logger=logger, +# ) +# if load_dir: +# dynamics_model.load(load_dir, load_device) +# +# return dynamics_model diff --git a/tests/test_models/test_causal_mech/test_plain_mech.py b/tests/test_models/test_causal_mech/test_plain_mech.py index 61494ba..b4fb4d8 100644 --- a/tests/test_models/test_causal_mech/test_plain_mech.py +++ b/tests/test_models/test_causal_mech/test_plain_mech.py @@ -21,7 +21,7 @@ def prepare(freq_rate): ensemble_num = 7 # test for transition - tran_dataset = EnsembleBufferDataset( + train_dataset = EnsembleBufferDataset( real_replay_buffer, env.observation_space, env.action_space, @@ -30,7 +30,7 @@ def prepare(freq_rate): train_ensemble=True, ensemble_num=ensemble_num, ) - train_loader = DataLoader(tran_dataset, batch_size=8, collate_fn=collate_fn) + train_loader = DataLoader(train_dataset, batch_size=8, collate_fn=collate_fn) valid_dataset = BufferDataset( real_replay_buffer, env.observation_space, env.action_space, is_valid=True, mech="transition", repeat=ensemble_num ) @@ -56,9 +56,12 @@ def test_inv_pendulum_single_step(): input_variables=input_variables, output_variables=output_variables, node_dim=node_dim, - variable_encoders=variable_encoders, - variable_decoders=variable_decoders, + # variable_encoders=variable_encoders, + # variable_decoders=variable_decoders, + variable_encoders=None, + variable_decoders=None, multi_step="none", + # device="cuda" ) mech.learn(train_loader, valid_loader, longest_epoch=1) @@ -74,8 +77,10 @@ def test_inv_pendulum_multi_step(): input_variables=input_variables, output_variables=output_variables, node_dim=node_dim, - variable_encoders=variable_encoders, - variable_decoders=variable_decoders, + # variable_encoders=variable_encoders, + # variable_decoders=variable_decoders, + variable_encoders=None, + variable_decoders=None, multi_step="forward-euler 2", ) From c79a0b044428b9cccdd728170cdab8555014616b Mon Sep 17 00:00:00 2001 From: frank Date: Sun, 6 Nov 2022 20:04:34 +0800 Subject: [PATCH 14/68] :hammer: refactor dynamics and fake-env --- cmrl/algorithms/offline/off_dyna.py | 77 +++++----- cmrl/examples/conf/algorithm/mbpo.yaml | 1 + cmrl/examples/conf/algorithm/mopo.yaml | 1 + cmrl/examples/conf/algorithm/off_dyna.yaml | 17 +-- cmrl/examples/conf/algorithm/on_dyna.yaml | 1 + cmrl/examples/main.py | 8 +- cmrl/models/causal_mech/base_causal_mech.py | 4 + cmrl/models/causal_mech/plain_mech.py | 6 +- cmrl/models/dynamics.py | 69 ++++++++- cmrl/models/fake_env.py | 149 ++++++++------------ cmrl/util/creator.py | 24 +++- 11 files changed, 206 insertions(+), 151 deletions(-) diff --git a/cmrl/algorithms/offline/off_dyna.py b/cmrl/algorithms/offline/off_dyna.py index 3c4920d..f84c972 100644 --- a/cmrl/algorithms/offline/off_dyna.py +++ b/cmrl/algorithms/offline/off_dyna.py @@ -1,11 +1,11 @@ import os from typing import Optional, cast +from functools import partial import emei import hydra.utils import numpy as np from omegaconf import DictConfig -from stable_baselines3.common.base_class import BaseAlgorithm from stable_baselines3.common.buffers import ReplayBuffer from cmrl.agent import complete_agent_cfg @@ -13,47 +13,61 @@ # from cmrl.models.dynamics import ConstraintBasedDynamics from cmrl.sb3_extension.eval_callback import EvalCallback +from cmrl.models.fake_env import VecFakeEnv from cmrl.sb3_extension.logger import configure as logger_configure from cmrl.types import InitObsFnType, RewardFnType, TermFnType -from cmrl.util.creator import create_dynamics +from cmrl.util.creator import create_dynamics, create_agent def train( env: emei.EmeiEnv, eval_env: emei.EmeiEnv, - termination_fn: Optional[TermFnType], reward_fn: Optional[RewardFnType], + termination_fn: Optional[TermFnType], get_init_obs_fn: Optional[InitObsFnType], cfg: DictConfig, work_dir: Optional[str] = None, ): - # build model-free agent, which is a stable-baselines3's agent - complete_agent_cfg(env, cfg.algorithm.agent) - agent = cast(BaseAlgorithm, hydra.utils.instantiate(cfg.algorithm.agent)) - + ######################################### + # create class + ######################################### work_dir = work_dir or os.getcwd() logger = logger_configure("log", ["tensorboard", "multi_csv", "stdout"]) - # create initial dataset and add it to replay buffer + # create ``cmrl`` dynamics dynamics = create_dynamics(cfg, env.observation_space, env.action_space, logger=logger) + # create sb3's replay buffer for real offline data real_replay_buffer = ReplayBuffer( cfg.task.num_steps, env.observation_space, env.action_space, cfg.device, handle_timeout_termination=False ) - load_offline_data(env, real_replay_buffer, cfg.task.dataset, cfg.task.use_ratio) - fake_eval_env = setup_fake_env( - cfg=cfg, - agent=agent, - dynamics=dynamics, - reward_fn=reward_fn, - termination_fn=termination_fn, - get_init_obs_fn=get_init_obs_fn, + partial_fake_env = partial( + VecFakeEnv, + cfg.algorithm.num_envs, + env.observation_space, + env.action_space, + dynamics, + reward_fn, + termination_fn, + get_init_obs_fn, + real_replay_buffer, + penalty_coeff=cfg.task.penalty_coeff, logger=logger, - max_episode_steps=env.spec.max_episode_steps, - penalty_coeff=cfg.algorithm.penalty_coeff, ) - # + fake_env = partial_fake_env( + deterministic=cfg.algorithm.deterministic, max_episode_steps=env.spec.max_episode_steps, branch_rollout=False + ) + fake_eval_env = partial_fake_env(deterministic=True, max_episode_steps=env.spec.max_episode_steps, branch_rollout=False) + + # create sb3's agent + agent = create_agent(cfg, fake_env, logger) + + ######################################### + # learn + ######################################### + load_offline_data(env, real_replay_buffer, cfg.task.dataset, cfg.task.use_ratio) + # if hasattr(env, "get_causal_graph"): # oracle_causal_graph = env.get_causal_graph() # else: @@ -65,16 +79,15 @@ def train( # existed_trained_model = maybe_load_trained_offline_model(dynamics, cfg, obs_shape, act_shape, work_dir=work_dir) # if not existed_trained_model: dynamics.learn(real_replay_buffer, **cfg.dynamics, work_dir=work_dir) - # - # eval_callback = EvalCallback( - # eval_env, - # fake_eval_env, - # n_eval_episodes=cfg.task.n_eval_episodes, - # best_model_save_path="./", - # eval_freq=1000, - # deterministic=True, - # render=False, - # ) - # - # agent.set_logger(logger) - # agent.learn(total_timesteps=cfg.task.num_steps, callback=eval_callback) + + eval_callback = EvalCallback( + eval_env, + fake_eval_env, + n_eval_episodes=cfg.task.n_eval_episodes, + best_model_save_path="./", + eval_freq=1000, + deterministic=True, + render=False, + ) + + agent.learn(total_timesteps=cfg.task.num_steps, callback=eval_callback) diff --git a/cmrl/examples/conf/algorithm/mbpo.yaml b/cmrl/examples/conf/algorithm/mbpo.yaml index 55fab6d..e974c8a 100644 --- a/cmrl/examples/conf/algorithm/mbpo.yaml +++ b/cmrl/examples/conf/algorithm/mbpo.yaml @@ -12,6 +12,7 @@ num_eval_episodes: 5 # SAC Agent configuration # -------------------------------------------- agent: + _partial_: true _target_: stable_baselines3.sac.SAC policy: "MlpPolicy" env: diff --git a/cmrl/examples/conf/algorithm/mopo.yaml b/cmrl/examples/conf/algorithm/mopo.yaml index 9daa227..7c249b6 100644 --- a/cmrl/examples/conf/algorithm/mopo.yaml +++ b/cmrl/examples/conf/algorithm/mopo.yaml @@ -13,6 +13,7 @@ penalty_coeff: ${task.penalty_coeff} # SAC Agent configuration # -------------------------------------------- agent: + _partial_: true _target_: stable_baselines3.sac.SAC policy: "MlpPolicy" env: diff --git a/cmrl/examples/conf/algorithm/off_dyna.yaml b/cmrl/examples/conf/algorithm/off_dyna.yaml index d22c034..3724f94 100644 --- a/cmrl/examples/conf/algorithm/off_dyna.yaml +++ b/cmrl/examples/conf/algorithm/off_dyna.yaml @@ -12,22 +12,13 @@ penalty_coeff: ${task.penalty_coeff} # -------------------------------------------- # SAC Agent configuration # -------------------------------------------- +num_envs: 16 +deterministic: true agent: + _partial_: true _target_: stable_baselines3.sac.SAC policy: "MlpPolicy" - env: - _target_: cmrl.models.fake_env.VecFakeEnv - num_envs: 16 - action_space: - _target_: gym.spaces.Box - low: ??? - high: ??? - shape: ??? - observation_space: - _target_: gym.spaces.Box - low: ??? - high: ??? - shape: ??? + env: ??? learning_starts: 0 batch_size: 256 tau: 0.005 diff --git a/cmrl/examples/conf/algorithm/on_dyna.yaml b/cmrl/examples/conf/algorithm/on_dyna.yaml index d3ae6e6..f849d1e 100644 --- a/cmrl/examples/conf/algorithm/on_dyna.yaml +++ b/cmrl/examples/conf/algorithm/on_dyna.yaml @@ -12,6 +12,7 @@ initial_exploration_steps: 1000 # SAC Agent configuration # -------------------------------------------- agent: + _partial_: true _target_: stable_baselines3.sac.SAC policy: "MlpPolicy" env: diff --git a/cmrl/examples/main.py b/cmrl/examples/main.py index 59282b0..a82e5c5 100644 --- a/cmrl/examples/main.py +++ b/cmrl/examples/main.py @@ -25,16 +25,16 @@ def run(cfg: DictConfig): if cfg.algorithm.name == "on_dyna": test_env, *_ = make_env(cfg) - return on_dyna.train(env, test_env, term_fn, reward_fn, init_obs_fn, cfg) + return on_dyna.train(env, test_env, reward_fn, term_fn, init_obs_fn, cfg) elif cfg.algorithm.name == "mopo": test_env, *_ = make_env(cfg) - return mopo.train(env, test_env, term_fn, reward_fn, init_obs_fn, cfg) + return mopo.train(env, test_env, reward_fn, term_fn, init_obs_fn, cfg) elif cfg.algorithm.name == "off_dyna": test_env, *_ = make_env(cfg) - return off_dyna.train(env, test_env, term_fn, reward_fn, init_obs_fn, cfg) + return off_dyna.train(env, test_env, reward_fn, term_fn, init_obs_fn, cfg) elif cfg.algorithm.name == "mbpo": test_env, *_ = make_env(cfg) - return mbpo.train(env, test_env, term_fn, reward_fn, init_obs_fn, cfg) + return mbpo.train(env, test_env, reward_fn, term_fn, init_obs_fn, cfg) else: raise NotImplementedError diff --git a/cmrl/models/causal_mech/base_causal_mech.py b/cmrl/models/causal_mech/base_causal_mech.py index 1024dec..401fedb 100644 --- a/cmrl/models/causal_mech/base_causal_mech.py +++ b/cmrl/models/causal_mech/base_causal_mech.py @@ -92,6 +92,10 @@ def check_coder(self): decoder = self.variable_decoders[var.name] assert decoder.node_dim == self.node_dim + @abstractmethod + def forward(self, inputs: Dict[str, torch.Tensor]) -> Dict[str, torch.Tensor]: + raise NotImplementedError + @abstractmethod def learn( self, diff --git a/cmrl/models/causal_mech/plain_mech.py b/cmrl/models/causal_mech/plain_mech.py index a26c147..d6d64f1 100644 --- a/cmrl/models/causal_mech/plain_mech.py +++ b/cmrl/models/causal_mech/plain_mech.py @@ -17,6 +17,8 @@ from cmrl.models.causal_mech.base_causal_mech import BaseCausalMech from cmrl.models.networks.coder import VariableEncoder, VariableDecoder +from time import time + class PlainMech(BaseCausalMech): def __init__( @@ -94,7 +96,6 @@ def build_graph(self): self.graph = None def forward(self, inputs: Dict[str, torch.Tensor]) -> Dict[str, torch.Tensor]: - torch.autograd.set_detect_anomaly(True) assert list(inputs.keys()) == list(self.variable_encoders.keys()) data_shape = list(inputs.values())[0].shape assert len(data_shape) == 3 # ensemble-num, batch-size, specific-dim @@ -180,14 +181,13 @@ def learn( **kwargs ): best_weights: Optional[Dict] = None - epoch_iter = range(longest_epoch) if longest_epoch > 0 else itertools.count() + epoch_iter = range(longest_epoch) if longest_epoch >= 0 else itertools.count() epochs_since_update = 0 best_eval_loss = self.eval(valid_loader).mean(dim=(1, 2)) for epoch in epoch_iter: train_loss = self.train(train_loader) eval_loss = self.eval(valid_loader).mean(dim=(1, 2)) - maybe_best_weights = self._maybe_get_best_weights(best_eval_loss, eval_loss, improvement_threshold) if maybe_best_weights: # best loss diff --git a/cmrl/models/dynamics.py b/cmrl/models/dynamics.py index 9c2b592..37ab2d6 100644 --- a/cmrl/models/dynamics.py +++ b/cmrl/models/dynamics.py @@ -36,8 +36,8 @@ def __init__( self.action_space = action_space self.logger = logger - self.learn_reward = reward_mech is None - self.learn_termination = termination_mech is None + self.learn_reward = reward_mech is not None + self.learn_termination = termination_mech is not None self.device = self.transition.device pass @@ -75,9 +75,68 @@ def learn( work_dir: Optional[Union[str, pathlib.Path]] = None, **kwargs ): + longest_epoch = 0 + # transition - self.transition.learn(*self.get_loader(real_replay_buffer, "transition")) + self.transition.learn( + *self.get_loader(real_replay_buffer, "transition"), + longest_epoch=longest_epoch, + improvement_threshold=improvement_threshold, + patience=patience, + work_dir=work_dir + ) # reward-mech - self.reward_mech.learn(*self.get_loader(real_replay_buffer, "reward_mech")) + if self.learn_reward: + self.reward_mech.learn( + *self.get_loader(real_replay_buffer, "reward_mech"), + longest_epoch=longest_epoch, + improvement_threshold=improvement_threshold, + patience=patience, + work_dir=work_dir + ) # termination-mech - self.termination_mech.learn(*self.get_loader(real_replay_buffer, "termination_mech")) + if self.learn_termination: + self.termination_mech.learn( + *self.get_loader(real_replay_buffer, "termination_mech"), + longest_epoch=longest_epoch, + improvement_threshold=improvement_threshold, + patience=patience, + work_dir=work_dir + ) + + def step(self, batch_obs, batch_action): + with torch.no_grad(): + if isinstance(self.observation_space, spaces.Box): + observations_dict = dict( + [ + ( + "obs_{}".format(i), + torch.from_numpy(np.tile(batch_obs.T[0][None, :, None], [7, 1, 1])).to(torch.float32), + ) + for i, obs in enumerate(batch_obs.T) + ] + ) + else: + raise NotImplementedError + + if isinstance(self.action_space, spaces.Box): + actions_dict = dict( + [ + ( + "act_{}".format(i), + torch.from_numpy(np.tile(batch_obs.T[0][None, :, None], [7, 1, 1])).to(torch.float32), + ) + for i, obs in enumerate(batch_action.T) + ] + ) + else: + raise NotImplementedError + + inputs = {} + inputs.update(observations_dict) + inputs.update(actions_dict) + outputs = self.transition.forward(inputs) + + info = {"origin-next_obs": torch.concat([tensor[:, :, :1] for tensor in outputs.values()], dim=-1).cpu().numpy()} + + return torch.concat([tensor.mean(dim=0)[:, :1] for tensor in outputs.values()], dim=-1).cpu().numpy(), None, None, info diff --git a/cmrl/models/fake_env.py b/cmrl/models/fake_env.py index 9bebe18..a6aa7d3 100644 --- a/cmrl/models/fake_env.py +++ b/cmrl/models/fake_env.py @@ -8,12 +8,8 @@ import numpy as np import torch from gym.core import ActType, ObsType -from stable_baselines3.common.vec_env.base_vec_env import ( - VecEnv, - VecEnvIndices, - VecEnvObs, - VecEnvStepReturn, -) +from stable_baselines3.common.vec_env.base_vec_env import VecEnv, VecEnvIndices +from stable_baselines3.common.logger import Logger from stable_baselines3.common.buffers import ReplayBuffer import cmrl.types @@ -23,102 +19,78 @@ class VecFakeEnv(VecEnv): def __init__( self, + # for need of sb3's agent num_envs: int, observation_space: gym.spaces.Space, action_space: gym.spaces.Space, - ): - super(VecFakeEnv, self).__init__( - num_envs=num_envs, - observation_space=observation_space, - action_space=action_space, - ) - - self.has_set_up = False - - self.penalty_coeff = None - self.deterministic = None - self.max_episode_steps = None - - self.dynamics = None - self.reward_fn = None - self.termination_fn = None - self.learn_reward = None - self.learn_termination = None - self.get_init_obs_fn = None - self.replay_buffer = None - self.generator = np.random.default_rng() - self.device = None - self.logger = None - - self._current_batch_obs = None - self._current_batch_action = None - - self._reset_by_buffer = True - - self._envs_length = np.zeros(self.num_envs, dtype=int) - - def set_up( - self, + # for dynamics dynamics: Dynamics, reward_fn: Optional[cmrl.types.RewardFnType] = None, termination_fn: Optional[cmrl.types.TermFnType] = None, get_init_obs_fn: Optional[cmrl.types.InitObsFnType] = None, real_replay_buffer: Optional[ReplayBuffer] = None, + # for offline penalty_coeff: float = 0.0, - deterministic=False, - max_episode_steps=1000, - logger=None, + # for behaviour + deterministic: bool = False, + max_episode_steps: int = 1000, + branch_rollout: bool = False, + # others + logger: Optional[Logger] = None, + **kwargs, ): + super(VecFakeEnv, self).__init__( + num_envs=num_envs, + observation_space=observation_space, + action_space=action_space, + ) self.dynamics = dynamics - - self.penalty_coeff = penalty_coeff - self.deterministic = deterministic - self.max_episode_steps = max_episode_steps - self.reward_fn = reward_fn self.termination_fn = termination_fn - assert self.dynamics.learn_reward or reward_fn - assert self.dynamics.learn_termination or termination_fn + assert self.dynamics.learn_reward or reward_fn, "you must learn a reward-mech or give one" + assert self.dynamics.learn_termination or termination_fn, "you must learn a termination-mech or give one" self.learn_reward = self.dynamics.learn_reward self.learn_termination = self.dynamics.learn_termination self.get_init_obs_fn = get_init_obs_fn self.replay_buffer = real_replay_buffer - self.logger = logger - assert self.get_init_obs_fn or self.replay_buffer - self._reset_by_buffer = self.replay_buffer is not None + self.penalty_coeff = penalty_coeff + self.deterministic = deterministic + self.max_episode_steps = max_episode_steps + self.branch_rollout = branch_rollout + if self.branch_rollout: + assert self.replay_buffer, "you must provide a replay buffer if using branch-rollout" + else: + assert self.get_init_obs_fn, "you must provide a get-init-obs function if using fully-virtual" + + self.logger = logger self.device = dynamics.device - self.has_set_up = True + + self._current_batch_obs = None + self._current_batch_action = None + self._envs_length = np.zeros(self.num_envs, dtype=int) def step_async(self, actions: np.ndarray) -> None: + assert len(actions.shape) == 2 # batch, action_dim self._current_batch_action = actions def step_wait(self): - assert self.has_set_up, "fake-env has not set up" - assert len(self._current_batch_action.shape) == 2 # batch, action_dim - with torch.no_grad(): - batch_obs_tensor = torch.from_numpy(self._current_batch_obs).to(torch.float32).to(self.device) - batch_action_tensor = torch.from_numpy(self._current_batch_action).to(torch.float32).to(self.device) - dynamics_pred = self.dynamics.query(batch_obs_tensor, batch_action_tensor, return_as_np=True) - - # transition - batch_next_obs = self.get_dynamics_predict(dynamics_pred, "transition", deterministic=self.deterministic) - if self.learn_reward: - batch_reward = self.get_dynamics_predict(dynamics_pred, "reward_mech", deterministic=self.deterministic) - else: - batch_reward = self.reward_fn(batch_next_obs, self._current_batch_obs, self._current_batch_action) - if self.learn_termination: - batch_terminal = self.get_dynamics_predict(dynamics_pred, "termination_mech", deterministic=self.deterministic) - else: - batch_terminal = self.termination_fn(batch_next_obs, self._current_batch_obs, self._current_batch_action) - - if self.penalty_coeff != 0: - penalty = self.get_penalty(dynamics_pred["batch_next_obs"]["mean"]).reshape(batch_reward.shape) - batch_reward -= penalty * self.penalty_coeff - - if self.logger is not None: - self.logger.record_mean("rollout/penalty", penalty.mean().item()) + batch_next_obs, batch_reward, batch_terminal, info = self.dynamics.step( + self._current_batch_obs, self._current_batch_action + ) + + if not self.learn_reward: + batch_reward = self.reward_fn(batch_next_obs, self._current_batch_obs, self._current_batch_action) + if not self.learn_termination: + batch_terminal = self.termination_fn(batch_next_obs, self._current_batch_obs, self._current_batch_action) + + if self.penalty_coeff != 0: + penalty = self.get_penalty(info["origin-next_obs"]).reshape(batch_reward.shape) + batch_reward -= penalty * self.penalty_coeff + + if self.logger is not None: + self.logger.record_mean("rollout/penalty", penalty.mean().item()) self._current_batch_obs = batch_next_obs.copy() batch_reward = batch_reward.reshape(self.num_envs) @@ -148,15 +120,17 @@ def reset( return_info: bool = False, options: Optional[dict] = None, ): - if self.has_set_up: - if self._reset_by_buffer: - upper_bound = self.replay_buffer.buffer_size if self.replay_buffer.full else self.replay_buffer.pos - batch_inds = np.random.randint(0, upper_bound, size=self.num_envs) - self._current_batch_obs = self.replay_buffer.observations[batch_inds, 0] - else: - self._current_batch_obs = self.get_init_obs_fn(self.num_envs) - self._envs_length = np.zeros(self.num_envs, dtype=int) + if self.branch_rollout: + upper_bound = self.replay_buffer.buffer_size if self.replay_buffer.full else self.replay_buffer.pos + batch_inds = np.random.randint(0, upper_bound, size=self.num_envs) + self._current_batch_obs = self.replay_buffer.observations[batch_inds, 0] + else: + self._current_batch_obs = self.get_init_obs_fn(self.num_envs) + self._envs_length = np.zeros(self.num_envs, dtype=int) + if return_info: + return self._current_batch_obs.copy(), {} + else: return self._current_batch_obs.copy() def seed(self, seed: Optional[int] = None): @@ -169,10 +143,8 @@ def env_is_wrapped(self, wrapper_class: Type[gym.Wrapper], indices: VecEnvIndice return [False for _ in range(self.num_envs)] def single_reset(self, idx): - assert self.has_set_up, "fake-env has not set up" - self._envs_length[idx] = 0 - if self._reset_by_buffer: + if self.branch_rollout: upper_bound = self.replay_buffer.buffer_size if self.replay_buffer.full else self.replay_buffer.pos batch_inds = np.random.randint(0, upper_bound) self._current_batch_obs[idx] = self.replay_buffer.observations[batch_inds, 0] @@ -189,7 +161,6 @@ def get_penalty(ensemble_batch_next_obs): diffs = ensemble_batch_next_obs - avg dists = np.linalg.norm(diffs, axis=2) # distance in obs space penalty = np.max(dists, axis=0) # max distances over models - return penalty def get_dynamics_predict( diff --git a/cmrl/util/creator.py b/cmrl/util/creator.py index 97c640b..055db8b 100644 --- a/cmrl/util/creator.py +++ b/cmrl/util/creator.py @@ -1,5 +1,6 @@ import pathlib -from typing import Callable, Dict, List, Optional, Sequence, Tuple, Type, Union +from typing import Callable, Dict, List, Optional, Sequence, Tuple, Type, Union, cast +from functools import partial import gym.wrappers from gym import spaces @@ -8,12 +9,22 @@ import numpy as np import omegaconf from stable_baselines3.common.logger import Logger +from stable_baselines3.common.base_class import BaseAlgorithm +from stable_baselines3.common.buffers import ReplayBuffer from cmrl.models.dynamics import Dynamics -from cmrl.models.transition import ForwardEulerTransition -from cmrl.util.config import get_complete_dynamics_cfg -from cmrl.models.util import parse_space, create_decoders, create_encoders -from cmrl.types import DiscreteVariable, ContinuousVariable, BinaryVariable +from cmrl.models.fake_env import VecFakeEnv +from cmrl.models.causal_mech.base_causal_mech import BaseCausalMech +from cmrl.models.util import parse_space +from cmrl.types import DiscreteVariable, ContinuousVariable, BinaryVariable, InitObsFnType, RewardFnType, TermFnType + + +def create_agent(cfg, fake_env: VecFakeEnv, logger: Optional[Logger] = None): + agent = instantiate(cfg.algorithm.agent)(env=fake_env) + agent = cast(BaseAlgorithm, agent) + agent.set_logger(logger) + + return agent def create_dynamics( @@ -36,6 +47,7 @@ def create_dynamics( variable_decoders=None, logger=logger, ) + transition = cast(BaseCausalMech, transition) # reward mech if cfg.reward_mech.learn: reward_mech = instantiate(cfg.reward_mech.mech)( @@ -45,6 +57,7 @@ def create_dynamics( variable_decoders=None, logger=logger, ) + reward_mech = cast(BaseCausalMech, reward_mech) else: reward_mech = None # termination mech @@ -56,6 +69,7 @@ def create_dynamics( variable_decoders=None, logger=logger, ) + termination_mech = cast(BaseCausalMech, termination_mech) else: termination_mech = None From 13913991d446a505177d6f7c315f017bbd36620e Mon Sep 17 00:00:00 2001 From: frank Date: Sun, 6 Nov 2022 22:27:08 +0800 Subject: [PATCH 15/68] :hammer: add space2dict --- cmrl/algorithms/__init__.py | 4 -- cmrl/algorithms/offline/mopo.py | 6 +- cmrl/algorithms/offline/off_dyna.py | 11 ++- cmrl/algorithms/online/mbpo.py | 4 +- cmrl/algorithms/online/on_dyna.py | 4 +- cmrl/algorithms/util.py | 28 +------- cmrl/diagnostics/eval_model_on_dataset.py | 8 +-- cmrl/diagnostics/eval_model_on_space.py | 6 +- cmrl/diagnostics/run_trained_model.py | 8 +-- cmrl/diagnostics/run_trained_policy.py | 6 +- cmrl/examples/conf/task/mbpo_ant.yaml | 2 +- cmrl/examples/conf/task/mbpo_humanoid.yaml | 2 +- cmrl/examples/main.py | 7 +- cmrl/models/causal_mech/base_causal_mech.py | 9 +-- cmrl/models/causal_mech/plain_mech.py | 14 ++-- cmrl/models/data_loader.py | 59 +++++++++------- cmrl/models/dynamics.py | 39 ++--------- cmrl/models/fake_env.py | 12 ++-- cmrl/models/networks/coder.py | 5 +- cmrl/models/old_dynamics/base_dynamics.py | 4 +- .../old_dynamics/constraint_based_dynamics.py | 5 +- cmrl/models/old_dynamics/ncd_dynamics.py | 5 +- .../one_step/external_mask_transition.py | 6 +- cmrl/models/util.py | 69 +++++++++++++++---- cmrl/{util => utils}/__init__.py | 10 +-- cmrl/{util => utils}/config.py | 0 cmrl/{util => utils}/creator.py | 62 +---------------- cmrl/{util => utils}/env.py | 14 ++-- cmrl/{util => utils}/transition_iterator.py | 6 +- cmrl/{ => utils}/types.py | 0 cmrl/{util => utils}/video.py | 0 .../test_offline/test_off_dyna.py | 2 +- tests/test_algorithms/test_util.py | 2 +- .../test_causal_mech/test_plain_mech.py | 3 - tests/test_models/test_data_loader.py | 2 +- tests/test_models/test_network/test_coder.py | 3 +- tests/test_types.py | 2 +- 37 files changed, 178 insertions(+), 251 deletions(-) rename cmrl/{util => utils}/__init__.py (92%) rename cmrl/{util => utils}/config.py (100%) rename cmrl/{util => utils}/creator.py (52%) rename cmrl/{util => utils}/env.py (77%) rename cmrl/{util => utils}/transition_iterator.py (97%) rename cmrl/{ => utils}/types.py (100%) rename cmrl/{util => utils}/video.py (100%) diff --git a/cmrl/algorithms/__init__.py b/cmrl/algorithms/__init__.py index 0f9ea59..e69de29 100644 --- a/cmrl/algorithms/__init__.py +++ b/cmrl/algorithms/__init__.py @@ -1,4 +0,0 @@ -from cmrl.algorithms.offline import mopo -from cmrl.algorithms.offline import off_dyna -from cmrl.algorithms.online import mbpo -from cmrl.algorithms.online import on_dyna diff --git a/cmrl/algorithms/offline/mopo.py b/cmrl/algorithms/offline/mopo.py index f043323..6531d9f 100644 --- a/cmrl/algorithms/offline/mopo.py +++ b/cmrl/algorithms/offline/mopo.py @@ -10,12 +10,12 @@ from cmrl.agent import complete_agent_cfg -# from cmrl.algorithms.util import maybe_load_trained_offline_model, setup_fake_env, load_offline_data +# from cmrl.algorithms.utils import maybe_load_trained_offline_model, setup_fake_env, load_offline_data # from cmrl.models.dynamics import ConstraintBasedDynamics from cmrl.sb3_extension.eval_callback import EvalCallback from cmrl.sb3_extension.logger import configure as logger_configure -from cmrl.types import InitObsFnType, RewardFnType, TermFnType -from cmrl.util.creator import create_dynamics +from cmrl.utils.types import InitObsFnType, RewardFnType, TermFnType +from cmrl.utils.creator import create_dynamics def train( diff --git a/cmrl/algorithms/offline/off_dyna.py b/cmrl/algorithms/offline/off_dyna.py index f84c972..5329c13 100644 --- a/cmrl/algorithms/offline/off_dyna.py +++ b/cmrl/algorithms/offline/off_dyna.py @@ -1,22 +1,19 @@ import os -from typing import Optional, cast +from typing import Optional from functools import partial import emei -import hydra.utils -import numpy as np from omegaconf import DictConfig from stable_baselines3.common.buffers import ReplayBuffer -from cmrl.agent import complete_agent_cfg -from cmrl.algorithms.util import setup_fake_env, load_offline_data +from cmrl.models.util import load_offline_data # from cmrl.models.dynamics import ConstraintBasedDynamics from cmrl.sb3_extension.eval_callback import EvalCallback from cmrl.models.fake_env import VecFakeEnv from cmrl.sb3_extension.logger import configure as logger_configure -from cmrl.types import InitObsFnType, RewardFnType, TermFnType -from cmrl.util.creator import create_dynamics, create_agent +from cmrl.utils.types import InitObsFnType, RewardFnType, TermFnType +from cmrl.utils.creator import create_dynamics, create_agent def train( diff --git a/cmrl/algorithms/online/mbpo.py b/cmrl/algorithms/online/mbpo.py index c3479e7..a9d8abc 100644 --- a/cmrl/algorithms/online/mbpo.py +++ b/cmrl/algorithms/online/mbpo.py @@ -16,8 +16,8 @@ from cmrl.sb3_extension.eval_callback import EvalCallback from cmrl.sb3_extension.online_mb_callback import OnlineModelBasedCallback from cmrl.sb3_extension.logger import configure as logger_configure -from cmrl.types import InitObsFnType, RewardFnType, TermFnType -from cmrl.util.creator import create_dynamics +from cmrl.utils.types import InitObsFnType, RewardFnType, TermFnType +from cmrl.utils.creator import create_dynamics def train( diff --git a/cmrl/algorithms/online/on_dyna.py b/cmrl/algorithms/online/on_dyna.py index 912e0d8..aa5539a 100644 --- a/cmrl/algorithms/online/on_dyna.py +++ b/cmrl/algorithms/online/on_dyna.py @@ -16,8 +16,8 @@ from cmrl.sb3_extension.eval_callback import EvalCallback from cmrl.sb3_extension.online_mb_callback import OnlineModelBasedCallback from cmrl.sb3_extension.logger import configure as logger_configure -from cmrl.types import InitObsFnType, RewardFnType, TermFnType -from cmrl.util.creator import create_dynamics +from cmrl.utils.types import InitObsFnType, RewardFnType, TermFnType +from cmrl.utils.creator import create_dynamics def train( diff --git a/cmrl/algorithms/util.py b/cmrl/algorithms/util.py index e59816f..e71d6f2 100644 --- a/cmrl/algorithms/util.py +++ b/cmrl/algorithms/util.py @@ -1,20 +1,16 @@ -import pathlib from typing import Optional, cast from copy import deepcopy -import emei import hydra -import numpy as np from omegaconf import DictConfig from stable_baselines3.common.vec_env.vec_monitor import VecMonitor from stable_baselines3.common.base_class import BaseAlgorithm from stable_baselines3.common.buffers import ReplayBuffer -from cmrl.types import InitObsFnType, RewardFnType, TermFnType +from cmrl.utils.types import InitObsFnType, RewardFnType, TermFnType # from cmrl.models.dynamics import BaseDynamics -from cmrl.util.config import get_complete_dynamics_cfg, load_hydra_cfg from cmrl.models.fake_env import VecFakeEnv @@ -104,25 +100,3 @@ def setup_fake_env( ) fake_eval_env.seed(seed=cfg.seed) return fake_eval_env - - -def load_offline_data(env, replay_buffer: ReplayBuffer, dataset_name: str, use_ratio: float = 1): - assert hasattr(env, "get_dataset"), "env must have `get_dataset` method" - - data_dict = env.get_dataset(dataset_name) - all_data_num = len(data_dict["observations"]) - sample_data_num = int(use_ratio * all_data_num) - sample_idx = np.random.permutation(all_data_num)[:sample_data_num] - - assert replay_buffer.n_envs == 1 - assert replay_buffer.buffer_size >= sample_data_num - - if sample_data_num == replay_buffer.buffer_size: - replay_buffer.full = True - replay_buffer.pos = 0 - else: - replay_buffer.pos = sample_data_num - - # set all data - for attr in ["observations", "next_observations", "actions", "rewards", "dones", "timeouts"]: - getattr(replay_buffer, attr)[:sample_data_num, 0] = data_dict[attr][sample_idx] diff --git a/cmrl/diagnostics/eval_model_on_dataset.py b/cmrl/diagnostics/eval_model_on_dataset.py index 689511e..d8531bc 100644 --- a/cmrl/diagnostics/eval_model_on_dataset.py +++ b/cmrl/diagnostics/eval_model_on_dataset.py @@ -8,10 +8,10 @@ import matplotlib.pylab as plt import numpy as np -import cmrl.util.creator -import cmrl.util.env -from cmrl.util.config import load_hydra_cfg -from cmrl.util.transition_iterator import TransitionIterator +import cmrl.utils.creator +import cmrl.utils.env +from cmrl.utils.config import load_hydra_cfg +from cmrl.utils.transition_iterator import TransitionIterator class DatasetEvaluator: diff --git a/cmrl/diagnostics/eval_model_on_space.py b/cmrl/diagnostics/eval_model_on_space.py index 645b10e..c883d22 100644 --- a/cmrl/diagnostics/eval_model_on_space.py +++ b/cmrl/diagnostics/eval_model_on_space.py @@ -11,9 +11,9 @@ import numpy as np from matplotlib.widgets import Button, RadioButtons, Slider -import cmrl.util.creator -import cmrl.util.env -from cmrl.util.config import load_hydra_cfg +import cmrl.utils.creator +import cmrl.utils.env +from cmrl.utils.config import load_hydra_cfg mpl.use("Qt5Agg") SIN_COS_BINDINGS = {"BoundaryInvertedPendulumSwingUp-v0": [1]} diff --git a/cmrl/diagnostics/run_trained_model.py b/cmrl/diagnostics/run_trained_model.py index 379c842..ebdc9c0 100644 --- a/cmrl/diagnostics/run_trained_model.py +++ b/cmrl/diagnostics/run_trained_model.py @@ -17,9 +17,9 @@ import cmrl import cmrl.agent import cmrl.models -import cmrl.util.creator -from cmrl.util.config import load_hydra_cfg -from cmrl.util.env import make_env +import cmrl.utils.creator +from cmrl.utils.config import load_hydra_cfg +from cmrl.utils.env import make_env class Runner: @@ -61,7 +61,7 @@ def __init__(self, model_dir: str, device: str = "cuda:0", render: bool = False) self.agent = agent_class.load(self.model_path / "best_model") def run(self): - # from emei.util import random_policy_test + # from emei.utils import random_policy_test obs = self.fake_eval_env.reset() if self.render: self.fake_eval_env.render() diff --git a/cmrl/diagnostics/run_trained_policy.py b/cmrl/diagnostics/run_trained_policy.py index 2e8f8a8..94f2008 100644 --- a/cmrl/diagnostics/run_trained_policy.py +++ b/cmrl/diagnostics/run_trained_policy.py @@ -10,8 +10,8 @@ import cmrl import cmrl.agent import cmrl.models -from cmrl.util.config import load_hydra_cfg -from cmrl.util.env import make_env +from cmrl.utils.config import load_hydra_cfg +from cmrl.utils.env import make_env class Runner: @@ -26,7 +26,7 @@ def __init__(self, agent_dir: str, type: str = "best", device="cuda:0"): self.agent = agent_class.load(self.agent_dir / "best_model") def run(self): - # from emei.util import random_policy_test + # from emei.utils import random_policy_test obs = self.env.reset() self.env.render() total_reward = 0 diff --git a/cmrl/examples/conf/task/mbpo_ant.yaml b/cmrl/examples/conf/task/mbpo_ant.yaml index 813cd89..86772f3 100644 --- a/cmrl/examples/conf/task/mbpo_ant.yaml +++ b/cmrl/examples/conf/task/mbpo_ant.yaml @@ -1,5 +1,5 @@ env: "ant_truncated_obs" -# term_fn is set automatically by cmrl.util.env.EnvHandler.make_env +# term_fn is set automatically by cmrl.utils.env.EnvHandler.make_env num_steps: 300000 epoch_length: 1000 diff --git a/cmrl/examples/conf/task/mbpo_humanoid.yaml b/cmrl/examples/conf/task/mbpo_humanoid.yaml index 4cf0d8a..72c34ef 100644 --- a/cmrl/examples/conf/task/mbpo_humanoid.yaml +++ b/cmrl/examples/conf/task/mbpo_humanoid.yaml @@ -1,5 +1,5 @@ env: "humanoid_truncated_obs" -# term_fn is set automatically by cmrl.util.env.EnvHandler.make_env +# term_fn is set automatically by cmrl.utils.env.EnvHandler.make_env num_steps: 300000 epoch_length: 1000 diff --git a/cmrl/examples/main.py b/cmrl/examples/main.py index a82e5c5..6bfd22b 100644 --- a/cmrl/examples/main.py +++ b/cmrl/examples/main.py @@ -4,8 +4,11 @@ import wandb from omegaconf import DictConfig, OmegaConf -from cmrl.algorithms import mopo, mbpo, off_dyna, on_dyna -from cmrl.util.env import make_env +import cmrl.algorithms.offline.off_dyna as off_dyna +import cmrl.algorithms.offline.mopo as mopo +import cmrl.algorithms.online.on_dyna as on_dyna +import cmrl.algorithms.online.mbpo as mbpo +from cmrl.utils.env import make_env @hydra.main(version_base=None, config_path="conf", config_name="main") diff --git a/cmrl/models/causal_mech/base_causal_mech.py b/cmrl/models/causal_mech/base_causal_mech.py index 401fedb..6c2b7f1 100644 --- a/cmrl/models/causal_mech/base_causal_mech.py +++ b/cmrl/models/causal_mech/base_causal_mech.py @@ -1,16 +1,17 @@ -from typing import Optional, List, Dict, Union, TypeVar, Type +from typing import Optional, List, Dict, Union, MutableMapping from abc import abstractmethod import torch +import numpy as np from torch.utils.data import DataLoader import torch.nn.functional as F from stable_baselines3.common.logger import Logger -from cmrl.types import Variable, ContinuousVariable, DiscreteVariable, BinaryVariable +from cmrl.utils.types import Variable, ContinuousVariable, DiscreteVariable, BinaryVariable from cmrl.models.networks.base_network import BaseNetwork from cmrl.models.graphs.base_graph import BaseGraph from cmrl.models.networks.coder import VariableEncoder, VariableDecoder -from cmrl.models.util import parse_space, create_decoders, create_encoders +from cmrl.models.util import create_decoders, create_encoders class BaseCausalMech: @@ -93,7 +94,7 @@ def check_coder(self): assert decoder.node_dim == self.node_dim @abstractmethod - def forward(self, inputs: Dict[str, torch.Tensor]) -> Dict[str, torch.Tensor]: + def forward(self, inputs: MutableMapping[str, Union[torch.Tensor]]) -> Dict[str, torch.Tensor]: raise NotImplementedError @abstractmethod diff --git a/cmrl/models/causal_mech/plain_mech.py b/cmrl/models/causal_mech/plain_mech.py index d6d64f1..29fafc8 100644 --- a/cmrl/models/causal_mech/plain_mech.py +++ b/cmrl/models/causal_mech/plain_mech.py @@ -1,4 +1,4 @@ -from typing import Optional, List, Dict, Union +from typing import Optional, List, Dict, Union, MutableMapping import pathlib import itertools import copy @@ -6,19 +6,15 @@ import numpy as np import torch from torch.utils.data import DataLoader -import torch.nn.functional as F from torch.optim import Adam from omegaconf import DictConfig from stable_baselines3.common.logger import Logger -from cmrl.types import Variable, ContinuousVariable, DiscreteVariable, BinaryVariable +from cmrl.utils.types import Variable from cmrl.models.networks.parallel_mlp import ParallelMLP -from cmrl.models.graphs.base_graph import BaseGraph from cmrl.models.causal_mech.base_causal_mech import BaseCausalMech from cmrl.models.networks.coder import VariableEncoder, VariableDecoder -from time import time - class PlainMech(BaseCausalMech): def __init__( @@ -95,10 +91,10 @@ def build_network(self): def build_graph(self): self.graph = None - def forward(self, inputs: Dict[str, torch.Tensor]) -> Dict[str, torch.Tensor]: - assert list(inputs.keys()) == list(self.variable_encoders.keys()) + def forward(self, inputs: MutableMapping[str, Union[torch.Tensor]]) -> Dict[str, torch.Tensor]: + assert len(set(inputs.keys()) & set(self.variable_encoders.keys())) == len(inputs) data_shape = list(inputs.values())[0].shape - assert len(data_shape) == 3 # ensemble-num, batch-size, specific-dim + assert len(data_shape) == 3, "{}".format(data_shape) # ensemble-num, batch-size, specific-dim ensemble, batch_size, specific_dim = data_shape assert ensemble == self.ensemble_num diff --git a/cmrl/models/data_loader.py b/cmrl/models/data_loader.py index 16d2202..552e7e0 100644 --- a/cmrl/models/data_loader.py +++ b/cmrl/models/data_loader.py @@ -7,6 +7,8 @@ import numpy as np from stable_baselines3.common.buffers import ReplayBuffer, DictReplayBuffer +from cmrl.models.util import space2dict + class BufferDataset(Dataset): def __init__( @@ -33,6 +35,9 @@ def __init__( self.seed = seed self.repeat = repeat + if self.repeat: + assert self.repeat > 1, "repeat must be a int greater than 1" + self.size = self.replay_buffer.buffer_size if self.replay_buffer.full else self.replay_buffer.pos self.inputs = None @@ -51,48 +56,52 @@ def build_indexes(self): self.indexes = permutation[: int(self.size * self.train_ratio)] def load_from_buffer(self): - if isinstance(self.replay_buffer, DictReplayBuffer): - # TODO: DictReplayBuffer case - raise NotImplementedError - else: - observations = self.replay_buffer.observations[: self.size, 0].astype(np.float32) - assert len(observations.shape) == 2 - next_observations = self.replay_buffer.next_observations[: self.size, 0].astype(np.float32) - - observations_dict = dict([("obs_{}".format(i), obs[:, None]) for i, obs in enumerate(observations.T)]) - next_observations_dict = dict( - [("next_obs_{}".format(i), obs[:, None]) for i, obs in enumerate(next_observations.T)] - ) - - assert isinstance(self.observation_space, spaces.Box) - # TODO: other spaces for observation and action(e.g. one-hot for spaces.Discrete) - # see: https://github.com/DLR-RM/stable-baselines3/blob/master/stable_baselines3/common/preprocessing.py#L85 - - actions = self.replay_buffer.actions[: self.size, 0] - actions_dict = dict([("act_{}".format(i), obs[:, None]) for i, obs in enumerate(actions.T)]) + # if isinstance(self.replay_buffer, DictReplayBuffer): + # # TODO: DictReplayBuffer case + # raise NotImplementedError + # else: + # observations = self.replay_buffer.observations[: self.size, 0].astype(np.float32) + # assert len(observations.shape) == 2 + # next_observations = self.replay_buffer.next_observations[: self.size, 0].astype(np.float32) + # + # observations_dict = dict([("obs_{}".format(i), obs[:, None]) for i, obs in enumerate(observations.T)]) + # next_observations_dict = dict( + # [("next_obs_{}".format(i), obs[:, None]) for i, obs in enumerate(next_observations.T)] + # ) + + # assert isinstance(self.observation_space, spaces.Box) + # # TODO: other spaces for observation and action(e.g. one-hot for spaces.Discrete) + # # see: https://github.com/DLR-RM/stable-baselines3/blob/master/stable_baselines3/common/preprocessing.py#L85 + # + # actions = self.replay_buffer.actions[: self.size, 0] + # actions_dict = dict([("act_{}".format(i), obs[:, None]) for i, obs in enumerate(actions.T)]) + + obs_dict = space2dict(self.replay_buffer.observations[: self.size, 0], self.observation_space, "obs") + act_dict = space2dict(self.replay_buffer.actions[: self.size, 0], self.action_space, "act") + next_obs_dict = space2dict(self.replay_buffer.next_observations[: self.size, 0], self.observation_space, "next_obs") self.inputs = {} - self.inputs.update(observations_dict) - self.inputs.update(actions_dict) + self.inputs.update(obs_dict) + self.inputs.update(act_dict) if self.mech == "transition": - self.outputs = next_observations_dict + self.outputs = next_obs_dict elif self.mech == "reward_mech": rewards = self.replay_buffer.rewards[: self.size, 0] rewards_dict = {"reward": rewards[:, None]} - self.inputs.update(next_observations_dict) + self.inputs.update(next_obs_dict) self.outputs = rewards_dict else: dones = self.replay_buffer.dones[: self.size, 0] timeouts = self.replay_buffer.timeouts[: self.size, 0] terminals_dict = {"terminal": (dones * (1 - timeouts))[:, None]} - self.inputs.update(next_observations_dict) + self.inputs.update(next_obs_dict) self.outputs = terminals_dict def __getitem__(self, item): index = self.indexes[item] if self.repeat: - assert len(self.indexes.shape) == 1 + assert len(self.indexes.shape) == 1, "repeating conflicts with ensemble" index = np.tile(index, self.repeat) inputs = dict([(key, self.inputs[key][index]) for key in self.inputs]) diff --git a/cmrl/models/dynamics.py b/cmrl/models/dynamics.py index 37ab2d6..66e854b 100644 --- a/cmrl/models/dynamics.py +++ b/cmrl/models/dynamics.py @@ -1,5 +1,5 @@ import abc -import collections +from collections import ChainMap import pathlib from typing import Dict, List, Optional, Tuple, Union @@ -10,11 +10,7 @@ from stable_baselines3.common.logger import Logger from stable_baselines3.common.buffers import ReplayBuffer -from cmrl.models.reward_mech.base_reward_mech import BaseRewardMech -from cmrl.models.termination_mech.base_termination_mech import BaseTerminationMech -from cmrl.models.transition.base_transition import BaseTransition -from cmrl.types import InteractionBatch -from cmrl.util.transition_iterator import BootstrapIterator, TransitionIterator +from cmrl.models.util import space2dict from cmrl.models.causal_mech.base_causal_mech import BaseCausalMech from cmrl.models.data_loader import BufferDataset, EnsembleBufferDataset, collate_fn @@ -106,35 +102,10 @@ def learn( def step(self, batch_obs, batch_action): with torch.no_grad(): - if isinstance(self.observation_space, spaces.Box): - observations_dict = dict( - [ - ( - "obs_{}".format(i), - torch.from_numpy(np.tile(batch_obs.T[0][None, :, None], [7, 1, 1])).to(torch.float32), - ) - for i, obs in enumerate(batch_obs.T) - ] - ) - else: - raise NotImplementedError + obs_dict = space2dict(batch_obs, self.observation_space, "obs", repeat=7, to_tensor=True) + act_dict = space2dict(batch_action, self.action_space, "act", repeat=7, to_tensor=True) - if isinstance(self.action_space, spaces.Box): - actions_dict = dict( - [ - ( - "act_{}".format(i), - torch.from_numpy(np.tile(batch_obs.T[0][None, :, None], [7, 1, 1])).to(torch.float32), - ) - for i, obs in enumerate(batch_action.T) - ] - ) - else: - raise NotImplementedError - - inputs = {} - inputs.update(observations_dict) - inputs.update(actions_dict) + inputs = ChainMap(obs_dict, act_dict) outputs = self.transition.forward(inputs) info = {"origin-next_obs": torch.concat([tensor[:, :, :1] for tensor in outputs.values()], dim=-1).cpu().numpy()} diff --git a/cmrl/models/fake_env.py b/cmrl/models/fake_env.py index a6aa7d3..f18c0fb 100644 --- a/cmrl/models/fake_env.py +++ b/cmrl/models/fake_env.py @@ -2,17 +2,15 @@ # # This source code is licensed under the MIT license found in the # LICENSE file in the root directory of this source tree. -from typing import Any, Callable, Dict, List, Optional, Sequence, Type, Union +from typing import Any, Dict, List, Optional, Type import gym import numpy as np -import torch -from gym.core import ActType, ObsType from stable_baselines3.common.vec_env.base_vec_env import VecEnv, VecEnvIndices from stable_baselines3.common.logger import Logger from stable_baselines3.common.buffers import ReplayBuffer -import cmrl.types +import cmrl.utils.types from cmrl.models.dynamics import Dynamics @@ -25,9 +23,9 @@ def __init__( action_space: gym.spaces.Space, # for dynamics dynamics: Dynamics, - reward_fn: Optional[cmrl.types.RewardFnType] = None, - termination_fn: Optional[cmrl.types.TermFnType] = None, - get_init_obs_fn: Optional[cmrl.types.InitObsFnType] = None, + reward_fn: Optional[cmrl.utils.types.RewardFnType] = None, + termination_fn: Optional[cmrl.utils.types.TermFnType] = None, + get_init_obs_fn: Optional[cmrl.utils.types.InitObsFnType] = None, real_replay_buffer: Optional[ReplayBuffer] = None, # for offline penalty_coeff: float = 0.0, diff --git a/cmrl/models/networks/coder.py b/cmrl/models/networks/coder.py index d689105..64c8e87 100644 --- a/cmrl/models/networks/coder.py +++ b/cmrl/models/networks/coder.py @@ -1,10 +1,9 @@ -from typing import List, Optional, Sequence, Union +from typing import List, Optional -import torch import torch.nn as nn from omegaconf import DictConfig -from cmrl.types import Variable, DiscreteVariable, ContinuousVariable, BinaryVariable +from cmrl.utils.types import Variable, DiscreteVariable, ContinuousVariable, BinaryVariable from cmrl.models.networks.base_network import BaseNetwork, create_activation diff --git a/cmrl/models/old_dynamics/base_dynamics.py b/cmrl/models/old_dynamics/base_dynamics.py index 8bc05cd..f9bfb35 100644 --- a/cmrl/models/old_dynamics/base_dynamics.py +++ b/cmrl/models/old_dynamics/base_dynamics.py @@ -11,8 +11,8 @@ from cmrl.models.reward_mech.base_reward_mech import BaseRewardMech from cmrl.models.termination_mech.base_termination_mech import BaseTerminationMech from cmrl.models.transition.base_transition import BaseTransition -from cmrl.types import InteractionBatch -from cmrl.util.transition_iterator import BootstrapIterator, TransitionIterator +from cmrl.utils.types import InteractionBatch +from cmrl.utils.transition_iterator import BootstrapIterator, TransitionIterator def split_dict(old_dict: Dict, need_keys: List[str]): diff --git a/cmrl/models/old_dynamics/constraint_based_dynamics.py b/cmrl/models/old_dynamics/constraint_based_dynamics.py index 5463566..41d4c8d 100644 --- a/cmrl/models/old_dynamics/constraint_based_dynamics.py +++ b/cmrl/models/old_dynamics/constraint_based_dynamics.py @@ -1,7 +1,7 @@ import copy import itertools import pathlib -from typing import Callable, Dict, List, Optional, Tuple, Union, cast +from typing import Dict, Optional, Union import numpy as np import torch @@ -14,9 +14,8 @@ from cmrl.models.termination_mech.base_termination_mech import BaseTerminationMech from cmrl.models.transition.base_transition import BaseTransition from cmrl.models.causal_discovery.CMI_test import TransitionConditionalMutualInformationTest -from cmrl.util.transition_iterator import BootstrapIterator, TransitionIterator from cmrl.models.util import to_tensor -from cmrl.types import TensorType +from cmrl.utils.types import TensorType class ConstraintBasedDynamics(BaseDynamics): diff --git a/cmrl/models/old_dynamics/ncd_dynamics.py b/cmrl/models/old_dynamics/ncd_dynamics.py index fa1e071..cc2f773 100644 --- a/cmrl/models/old_dynamics/ncd_dynamics.py +++ b/cmrl/models/old_dynamics/ncd_dynamics.py @@ -1,7 +1,7 @@ import copy import itertools import pathlib -from typing import Callable, Dict, List, Optional, Tuple, Union, cast +from typing import Dict, Optional, Union import numpy as np import torch @@ -14,9 +14,8 @@ from cmrl.models.termination_mech.base_termination_mech import BaseTerminationMech from cmrl.models.transition.base_transition import BaseTransition from cmrl.models.causal_discovery.CMI_test import TransitionConditionalMutualInformationTest -from cmrl.util.transition_iterator import BootstrapIterator, TransitionIterator from cmrl.models.util import to_tensor -from cmrl.types import TensorType +from cmrl.utils.types import TensorType class ConstraintBasedDynamics(BaseDynamics): diff --git a/cmrl/models/transition/one_step/external_mask_transition.py b/cmrl/models/transition/one_step/external_mask_transition.py index fea816c..e8e6b9d 100644 --- a/cmrl/models/transition/one_step/external_mask_transition.py +++ b/cmrl/models/transition/one_step/external_mask_transition.py @@ -1,4 +1,4 @@ -from typing import Dict, Optional, Sequence, Tuple, Union +from typing import Dict, Optional, Tuple, Union import hydra import omegaconf @@ -6,7 +6,7 @@ from torch import nn as nn from torch.nn import functional as F -import cmrl.types +import cmrl.utils.types from cmrl.models.layers import ParallelLinear from cmrl.models.transition.base_transition import BaseTransition @@ -107,7 +107,7 @@ def create_activation(): def create_linear_layer(self, l_in, l_out): return ParallelLinear(l_in, l_out, extra_dims=[self.obs_size, self.ensemble_num]) - def set_input_mask(self, mask: cmrl.types.TensorType): + def set_input_mask(self, mask: cmrl.utils.types.TensorType): self._input_mask = to_tensor(mask).to(self.device) @property diff --git a/cmrl/models/util.py b/cmrl/models/util.py index 7651681..0d38380 100644 --- a/cmrl/models/util.py +++ b/cmrl/models/util.py @@ -2,16 +2,16 @@ # # This source code is licensed under the MIT license found in the # LICENSE file in the root directory of this source tree. -from typing import List, Optional, Union +from typing import List, Optional, Union, Dict import numpy as np import torch -import torch.nn.functional as F from gym import spaces from omegaconf import DictConfig -import cmrl.types -from cmrl.types import Variable, ContinuousVariable, DiscreteVariable, BinaryVariable +from stable_baselines3.common.buffers import ReplayBuffer + +from cmrl.utils.types import Variable, ContinuousVariable, DiscreteVariable, BinaryVariable from cmrl.models.networks.coder import VariableEncoder, VariableDecoder @@ -39,14 +39,6 @@ def truncated_normal_(tensor: torch.Tensor, mean: float = 0, std: float = 1) -> return tensor -def to_tensor(x: cmrl.types.TensorType): - if isinstance(x, torch.Tensor): - return x - if isinstance(x, np.ndarray): - return torch.from_numpy(x) - raise ValueError("Input must be torch.Tensor or np.ndarray.") - - def parse_space(space: spaces.Space, prefix="obs") -> List[Variable]: variables = [] if isinstance(space, spaces.Box): @@ -67,6 +59,37 @@ def parse_space(space: spaces.Space, prefix="obs") -> List[Variable]: return variables +def space2dict( + data: np.ndarray, space: spaces.Space, prefix="obs", repeat: Optional[int] = None, to_tensor: bool = False +) -> Dict[str, Union[np.ndarray, torch.Tensor]]: + if repeat: + assert repeat > 1, "repeat must be a int greater than 1" + + dict_data = {} + if isinstance(space, spaces.Box): # shape: (batch-size, node-num), every node has exactly one dim + for i, (low, high) in enumerate(zip(space.low, space.high)): + # shape: (batch-size, specific-dim) + dict_data["{}_{}".format(prefix, i)] = data[:, i, None].astype(np.float32) + else: + # TODO + raise NotImplementedError + + for name in dict_data: + if repeat: + # shape: (repeat-dim, batch-size, specific-dim) + dict_data[name] = np.tile(dict_data[name][None, :, :], [repeat, 1, 1]) + if to_tensor: + dict_data[name] = torch.from_numpy(dict_data[name]) + + return dict_data + + +def dict2space( + data: Dict[str, Union[np.ndarray, torch.Tensor]], space: spaces.Space +) -> Dict[str, Union[np.ndarray, torch.Tensor]]: + pass + + def create_encoders( input_variables: List[Variable], node_dim: int, @@ -105,3 +128,25 @@ def create_decoders( normal_distribution=normal_distribution, ).to(device) return decoders + + +def load_offline_data(env, replay_buffer: ReplayBuffer, dataset_name: str, use_ratio: float = 1): + assert hasattr(env, "get_dataset"), "env must have `get_dataset` method" + + data_dict = env.get_dataset(dataset_name) + all_data_num = len(data_dict["observations"]) + sample_data_num = int(use_ratio * all_data_num) + sample_idx = np.random.permutation(all_data_num)[:sample_data_num] + + assert replay_buffer.n_envs == 1 + assert replay_buffer.buffer_size >= sample_data_num + + if sample_data_num == replay_buffer.buffer_size: + replay_buffer.full = True + replay_buffer.pos = 0 + else: + replay_buffer.pos = sample_data_num + + # set all data + for attr in ["observations", "next_observations", "actions", "rewards", "dones", "timeouts"]: + getattr(replay_buffer, attr)[:sample_data_num, 0] = data_dict[attr][sample_idx] diff --git a/cmrl/util/__init__.py b/cmrl/utils/__init__.py similarity index 92% rename from cmrl/util/__init__.py rename to cmrl/utils/__init__.py index d4085c1..c59acbe 100644 --- a/cmrl/util/__init__.py +++ b/cmrl/utils/__init__.py @@ -40,11 +40,11 @@ def create_handler(cfg: Union[Dict, omegaconf.ListConfig, omegaconf.DictConfig]) target = cfg.overrides.env_cfg.get_dynamics_predict("_target_") if "pybulletgym" in target: - from cmrl.util.pybullet import PybulletEnvHandler + from cmrl.utils.pybullet import PybulletEnvHandler return PybulletEnvHandler() elif "mujoco" in target: - from cmrl.util.mujoco import MujocoEnvHandler + from cmrl.utils.mujoco import MujocoEnvHandler return MujocoEnvHandler() else: @@ -74,15 +74,15 @@ def create_handler_from_str(env_name: str): (EnvHandler): A handler for the associated gym environment """ if "dmcontrol___" in env_name: - from cmrl.util.dmcontrol import DmcontrolEnvHandler + from cmrl.utils.dmcontrol import DmcontrolEnvHandler return DmcontrolEnvHandler() elif "pybulletgym___" in env_name: - from cmrl.util.pybullet import PybulletEnvHandler + from cmrl.utils.pybullet import PybulletEnvHandler return PybulletEnvHandler() elif "gym___" in env_name or env_name == "ideal_inv_pendulum": - from cmrl.util.mujoco import MujocoEnvHandler + from cmrl.utils.mujoco import MujocoEnvHandler return MujocoEnvHandler() else: diff --git a/cmrl/util/config.py b/cmrl/utils/config.py similarity index 100% rename from cmrl/util/config.py rename to cmrl/utils/config.py diff --git a/cmrl/util/creator.py b/cmrl/utils/creator.py similarity index 52% rename from cmrl/util/creator.py rename to cmrl/utils/creator.py index 055db8b..012ae3a 100644 --- a/cmrl/util/creator.py +++ b/cmrl/utils/creator.py @@ -1,22 +1,16 @@ -import pathlib -from typing import Callable, Dict, List, Optional, Sequence, Tuple, Type, Union, cast -from functools import partial +from typing import Optional, cast -import gym.wrappers from gym import spaces -import hydra from hydra.utils import instantiate import numpy as np -import omegaconf from stable_baselines3.common.logger import Logger from stable_baselines3.common.base_class import BaseAlgorithm -from stable_baselines3.common.buffers import ReplayBuffer from cmrl.models.dynamics import Dynamics from cmrl.models.fake_env import VecFakeEnv from cmrl.models.causal_mech.base_causal_mech import BaseCausalMech from cmrl.models.util import parse_space -from cmrl.types import DiscreteVariable, ContinuousVariable, BinaryVariable, InitObsFnType, RewardFnType, TermFnType +from cmrl.utils.types import ContinuousVariable, BinaryVariable def create_agent(cfg, fake_env: VecFakeEnv, logger: Optional[Logger] = None): @@ -83,55 +77,3 @@ def create_dynamics( ) return dynamics - - -# def create_dynamics( -# dynamics_cfg: omegaconf.DictConfig, -# obs_shape: Tuple[int, ...], -# act_shape: Tuple[int, ...], -# logger: Optional[Logger] = None, -# load_dir: Optional[Union[str, pathlib.Path]] = None, -# load_device: Optional[str] = None, -# ): -# if dynamics_cfg.name == "plain_dynamics": -# dynamics_class = PlainEnsembleDynamics -# elif dynamics_cfg.name == "constraint_based_dynamics": -# dynamics_class = ConstraintBasedDynamics -# else: -# raise NotImplementedError -# -# dynamics_cfg = get_complete_dynamics_cfg(dynamics_cfg, obs_shape, act_shape) -# transition = hydra.utils.instantiate(dynamics_cfg.transition, _recursive_=False) -# if dynamics_cfg.multi_step == "none": -# pass -# elif dynamics_cfg.multi_step.startswith("forward_euler"): -# repeat_times = int(dynamics_cfg.multi_step[len("forward_euler") + 1 :]) -# transition = ForwardEulerTransition(transition, repeat_times) -# else: -# raise NotImplementedError -# -# if dynamics_cfg.learned_reward: -# reward_mech = hydra.utils.instantiate(dynamics_cfg.reward_mech, _recursive_=False) -# else: -# reward_mech = None -# -# if dynamics_cfg.learned_termination: -# termination_mech = hydra.utils.instantiate(dynamics_cfg.termination_mech, _recursive_=False) -# raise NotImplementedError -# else: -# termination_mech = None -# -# dynamics_model = dynamics_class( -# transition=transition, -# learned_reward=dynamics_cfg.learned_reward, -# reward_mech=reward_mech, -# learned_termination=dynamics_cfg.learned_termination, -# termination_mech=termination_mech, -# optim_lr=dynamics_cfg.optim_lr, -# weight_decay=dynamics_cfg.weight_decay, -# logger=logger, -# ) -# if load_dir: -# dynamics_model.load(load_dir, load_device) -# -# return dynamics_model diff --git a/cmrl/util/env.py b/cmrl/utils/env.py similarity index 77% rename from cmrl/util/env.py rename to cmrl/utils/env.py index a7628f6..63f4ec4 100644 --- a/cmrl/util/env.py +++ b/cmrl/utils/env.py @@ -1,11 +1,10 @@ -from typing import Dict, Optional, Tuple, Union, cast +from typing import Dict, Optional, Tuple, cast import emei import gym import omegaconf -import torch -import cmrl.types +import cmrl.utils.types def to_num(s): @@ -17,13 +16,18 @@ def to_num(s): def get_term_and_reward_fn( cfg: omegaconf.DictConfig, -) -> Tuple[cmrl.types.TermFnType, Optional[cmrl.types.RewardFnType]]: +) -> Tuple[cmrl.utils.types.TermFnType, Optional[cmrl.utils.types.RewardFnType]]: return None, None def make_env( cfg: omegaconf.DictConfig, -) -> Tuple[emei.EmeiEnv, cmrl.types.TermFnType, Optional[cmrl.types.RewardFnType], Optional[cmrl.types.InitObsFnType],]: +) -> Tuple[ + emei.EmeiEnv, + cmrl.utils.types.TermFnType, + Optional[cmrl.utils.types.RewardFnType], + Optional[cmrl.utils.types.InitObsFnType], +]: if "gym___" in cfg.task.env: env = gym.make(cfg.task.env.split("___")[1]) term_fn, reward_fn = get_term_and_reward_fn(cfg) diff --git a/cmrl/util/transition_iterator.py b/cmrl/utils/transition_iterator.py similarity index 97% rename from cmrl/util/transition_iterator.py rename to cmrl/utils/transition_iterator.py index b5e1928..f4915ab 100644 --- a/cmrl/util/transition_iterator.py +++ b/cmrl/utils/transition_iterator.py @@ -2,13 +2,11 @@ # # This source code is licensed under the MIT license found in the # LICENSE file in the root directory of this source tree. -import pathlib -import warnings -from typing import Any, List, Optional, Sequence, Sized, Tuple, Type, Union +from typing import Any, Optional, Sequence, Sized import numpy as np -from cmrl.types import InteractionBatch +from cmrl.utils.types import InteractionBatch def _consolidate_batches(batches: Sequence[InteractionBatch]) -> InteractionBatch: diff --git a/cmrl/types.py b/cmrl/utils/types.py similarity index 100% rename from cmrl/types.py rename to cmrl/utils/types.py diff --git a/cmrl/util/video.py b/cmrl/utils/video.py similarity index 100% rename from cmrl/util/video.py rename to cmrl/utils/video.py diff --git a/tests/test_algorithms/test_offline/test_off_dyna.py b/tests/test_algorithms/test_offline/test_off_dyna.py index 48dc7de..fabe1b6 100644 --- a/tests/test_algorithms/test_offline/test_off_dyna.py +++ b/tests/test_algorithms/test_offline/test_off_dyna.py @@ -2,7 +2,7 @@ import torch from cmrl.algorithms.offline.off_dyna import train -from cmrl.util.env import make_env +from cmrl.utils.env import make_env from tests.constants import cfg diff --git a/tests/test_algorithms/test_util.py b/tests/test_algorithms/test_util.py index 0a4da4e..03d3ae4 100644 --- a/tests/test_algorithms/test_util.py +++ b/tests/test_algorithms/test_util.py @@ -1,6 +1,6 @@ from stable_baselines3.common.buffers import ReplayBuffer -from cmrl.util.env import make_env +from cmrl.utils.env import make_env from cmrl.algorithms.util import load_offline_data from tests.constants import cfg diff --git a/tests/test_models/test_causal_mech/test_plain_mech.py b/tests/test_models/test_causal_mech/test_plain_mech.py index b4fb4d8..fb73c21 100644 --- a/tests/test_models/test_causal_mech/test_plain_mech.py +++ b/tests/test_models/test_causal_mech/test_plain_mech.py @@ -1,11 +1,8 @@ import gym -import emei from stable_baselines3.common.buffers import ReplayBuffer from torch.utils.data import DataLoader -from torch.utils.data import default_collate from cmrl.models.causal_mech.plain_mech import PlainMech -from cmrl.types import Variable, ContinuousVariable, DiscreteVariable from cmrl.models.data_loader import BufferDataset, EnsembleBufferDataset, collate_fn from cmrl.algorithms.util import load_offline_data from cmrl.models.util import parse_space, create_decoders, create_encoders diff --git a/tests/test_models/test_data_loader.py b/tests/test_models/test_data_loader.py index bb8f455..ab7ef1c 100644 --- a/tests/test_models/test_data_loader.py +++ b/tests/test_models/test_data_loader.py @@ -5,7 +5,7 @@ from torch.utils.data import DataLoader from cmrl.models.data_loader import BufferDataset, EnsembleBufferDataset -from cmrl.algorithms.util import load_offline_data +from cmrl.models.util import load_offline_data def test_buffer_dataset(): diff --git a/tests/test_models/test_network/test_coder.py b/tests/test_models/test_network/test_coder.py index e3812f9..7bb7e9d 100644 --- a/tests/test_models/test_network/test_coder.py +++ b/tests/test_models/test_network/test_coder.py @@ -1,9 +1,8 @@ import torch -import numpy as np from torch.nn.functional import one_hot from cmrl.models.networks.coder import VariableEncoder, VariableDecoder -from cmrl.types import Variable, ContinuousVariable, DiscreteVariable, BinaryVariable +from cmrl.utils.types import ContinuousVariable, DiscreteVariable, BinaryVariable def test_continuous_encoder(): diff --git a/tests/test_types.py b/tests/test_types.py index 930a31d..e32a5bf 100644 --- a/tests/test_types.py +++ b/tests/test_types.py @@ -3,7 +3,7 @@ import torch -from cmrl.types import InteractionBatch +from cmrl.utils.types import InteractionBatch class TestTransitionBatch(TestCase): From 38aa16241207096996f4542fa94c4afa2133d019 Mon Sep 17 00:00:00 2001 From: frank Date: Sun, 6 Nov 2022 23:28:30 +0800 Subject: [PATCH 16/68] :hammer: refactor algorithm(from function to class) --- cmrl/agent/__init__.py | 1 - cmrl/agent/core.py | 137 ---------- cmrl/algorithms/__init__.py | 6 + cmrl/algorithms/offline/mopo.py | 85 ------ cmrl/algorithms/online/__init__.py | 0 cmrl/algorithms/util.py | 40 --- cmrl/examples/conf/algorithm/mbpo.yaml | 4 + cmrl/examples/conf/algorithm/mopo.yaml | 22 +- cmrl/examples/conf/algorithm/off_dyna.yaml | 4 + cmrl/examples/conf/algorithm/on_dyna.yaml | 4 + .../dynamics/constraint_based_dynamics.yaml | 63 ----- .../conf/dynamics/plain_dynamics.yaml | 63 ----- cmrl/examples/conf/main.yaml | 6 +- cmrl/examples/conf/task/BIPS.yaml | 13 +- cmrl/examples/conf/task/mbpo_ant.yaml | 28 -- ...ary_inverted_double_pendulum_swing_up.yaml | 33 --- ...po_boundary_inverted_pendulum_holding.yaml | 33 --- ...o_boundary_inverted_pendulum_swing_up.yaml | 33 --- cmrl/examples/conf/task/mbpo_cartpole.yaml | 28 -- cmrl/examples/conf/task/mbpo_halfcheetah.yaml | 28 -- cmrl/examples/conf/task/mbpo_hopper.yaml | 28 -- cmrl/examples/conf/task/mbpo_humanoid.yaml | 28 -- .../examples/conf/task/mbpo_inv_pendulum.yaml | 29 -- cmrl/examples/conf/task/mbpo_pusher.yaml | 29 -- cmrl/examples/conf/task/mbpo_walker.yaml | 28 -- cmrl/examples/main.py | 30 +-- cmrl/models/causal_discovery/CMI_test.py | 1 - cmrl/models/causal_mech/CMI_test.py | 251 +++++++++++++++++- cmrl/models/causal_mech/base_causal_mech.py | 4 +- cmrl/models/causal_mech/plain_mech.py | 10 +- cmrl/models/dynamics.py | 2 - .../old_dynamics/constraint_based_dynamics.py | 2 +- cmrl/models/old_dynamics/ncd_dynamics.py | 2 +- cmrl/models/util.py | 4 - cmrl/utils/env.py | 4 +- 35 files changed, 294 insertions(+), 789 deletions(-) delete mode 100644 cmrl/agent/__init__.py delete mode 100644 cmrl/agent/core.py delete mode 100644 cmrl/algorithms/offline/mopo.py delete mode 100644 cmrl/algorithms/online/__init__.py delete mode 100644 cmrl/examples/conf/dynamics/constraint_based_dynamics.yaml delete mode 100644 cmrl/examples/conf/dynamics/plain_dynamics.yaml delete mode 100644 cmrl/examples/conf/task/mbpo_ant.yaml delete mode 100644 cmrl/examples/conf/task/mbpo_boundary_inverted_double_pendulum_swing_up.yaml delete mode 100644 cmrl/examples/conf/task/mbpo_boundary_inverted_pendulum_holding.yaml delete mode 100644 cmrl/examples/conf/task/mbpo_boundary_inverted_pendulum_swing_up.yaml delete mode 100644 cmrl/examples/conf/task/mbpo_cartpole.yaml delete mode 100644 cmrl/examples/conf/task/mbpo_halfcheetah.yaml delete mode 100644 cmrl/examples/conf/task/mbpo_hopper.yaml delete mode 100644 cmrl/examples/conf/task/mbpo_humanoid.yaml delete mode 100644 cmrl/examples/conf/task/mbpo_inv_pendulum.yaml delete mode 100644 cmrl/examples/conf/task/mbpo_pusher.yaml delete mode 100644 cmrl/examples/conf/task/mbpo_walker.yaml diff --git a/cmrl/agent/__init__.py b/cmrl/agent/__init__.py deleted file mode 100644 index 012afb4..0000000 --- a/cmrl/agent/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from cmrl.agent.core import Agent, RandomAgent, complete_agent_cfg, load_agent diff --git a/cmrl/agent/core.py b/cmrl/agent/core.py deleted file mode 100644 index b6a074e..0000000 --- a/cmrl/agent/core.py +++ /dev/null @@ -1,137 +0,0 @@ -import abc -import pathlib -from typing import Any, Optional, Union - -import gym -import hydra -import numpy as np -import omegaconf - - -class Agent: - """Abstract class for all agents.""" - - @abc.abstractmethod - def act(self, obs: np.ndarray, **kwargs) -> np.ndarray: - pass - - def reset(self): - pass - - -class RandomAgent(Agent): - """An agent that samples action from the environments action space. - - Args: - env (gym.Env): the environment on which the agent will act. - """ - - def __init__(self, env: gym.Env): - self.env = env - - def act(self, obs: np.ndarray, **kwargs) -> np.ndarray: - return self.env.action_space.sample() - - -def complete_agent_cfg(env: gym.Env, agent_cfg: omegaconf.DictConfig): - obs_shape = env.observation_space.shape - act_shape = env.action_space.shape - - def _check_and_replace(key: str, value: Any, cfg: omegaconf.DictConfig): - if key in cfg.keys() and key not in cfg: - setattr(cfg, key, value) - - # create numpy object by existed object - def _create_numpy_config(array): - return { - "_target_": "numpy.array", - "object": array.tolist(), - "dtype": str(array.dtype), - } - - _check_and_replace("num_inputs", obs_shape[0], agent_cfg) - if "action_space" in agent_cfg.keys() and isinstance(agent_cfg.action_space, omegaconf.DictConfig): - _check_and_replace("low", _create_numpy_config(env.action_space.low), agent_cfg.action_space) - _check_and_replace("high", _create_numpy_config(env.action_space.high), agent_cfg.action_space) - _check_and_replace("shape", env.action_space.shape, agent_cfg.action_space) - - if "obs_dim" in agent_cfg.keys() and "obs_dim" not in agent_cfg: - agent_cfg.obs_dim = obs_shape[0] - if "action_dim" in agent_cfg.keys() and "action_dim" not in agent_cfg: - agent_cfg.action_dim = act_shape[0] - if "action_range" in agent_cfg.keys() and "action_range" not in agent_cfg: - agent_cfg.action_range = [ - float(env.action_space.low.min()), - float(env.action_space.high.max()), - ] - if "action_lb" in agent_cfg.keys() and "action_lb" not in agent_cfg: - agent_cfg.action_lb = _create_numpy_config(env.action_space.low) - if "action_ub" in agent_cfg.keys() and "action_ub" not in agent_cfg: - agent_cfg.action_ub = _create_numpy_config(env.action_space.high) - - if "env" in agent_cfg.keys(): - _check_and_replace( - "low", - _create_numpy_config(env.action_space.low), - agent_cfg.env.action_space, - ) - _check_and_replace( - "high", - _create_numpy_config(env.action_space.high), - agent_cfg.env.action_space, - ) - _check_and_replace("shape", env.action_space.shape, agent_cfg.env.action_space) - - _check_and_replace( - "low", - _create_numpy_config(env.observation_space.low), - agent_cfg.env.observation_space, - ) - _check_and_replace( - "high", - _create_numpy_config(env.observation_space.high), - agent_cfg.env.observation_space, - ) - _check_and_replace("shape", env.observation_space.shape, agent_cfg.env.observation_space) - - return agent_cfg - - -def load_agent( - agent_path: Union[str, pathlib.Path], - env: gym.Env, - type: Optional[str] = "best", - device: Optional[str] = None, -) -> Agent: - """Loads an agent from a Hydra config file at the given path. - - For agent of type "pytorch_sac.agent.sac.SACAgent", the directory - must contain the following files: - - - ".hydra/config.yaml": the Hydra configuration for the agent. - - "critic.pth": the saved checkpoint for the critic. - - "actor.pth": the saved checkpoint for the actor. - - Args: - agent_path (str or pathlib.Path): a path to the directory where the agent is saved. - env (gym.Env): the environment on which the agent will operate (only used to complete - the agent's configuration). - - Returns: - (Agent): the new agent. - """ - agent_path = pathlib.Path(agent_path) - cfg = omegaconf.OmegaConf.load(agent_path / ".hydra" / "config.yaml") - cfg.device = device - - if cfg.algorithm.agent._target_ == "cmrl.third_party.pytorch_sac.sac.SAC": - import cmrl.third_party.pytorch_sac as pytorch_sac - - from .sac_wrapper import SACAgent - - complete_agent_cfg(env, cfg.algorithm.agent) - agent: pytorch_sac.SAC = hydra.utils.instantiate(cfg.algorithm.agent) - agent.load_checkpoint(ckpt_path=agent_path / "sac_{}.pth".format(type), device=device) - return SACAgent(agent) - else: - raise ValueError("Invalid agent configuration.") diff --git a/cmrl/algorithms/__init__.py b/cmrl/algorithms/__init__.py index e69de29..2e4e217 100644 --- a/cmrl/algorithms/__init__.py +++ b/cmrl/algorithms/__init__.py @@ -0,0 +1,6 @@ +from cmrl.algorithms.base_algorithm import BaseAlgorithm +from cmrl.algorithms.mopo import MOPO +from cmrl.algorithms.on_dyna import OnlineDyna +from cmrl.algorithms.mbpo import MBPO + +OfflineDyna = BaseAlgorithm diff --git a/cmrl/algorithms/offline/mopo.py b/cmrl/algorithms/offline/mopo.py deleted file mode 100644 index 6531d9f..0000000 --- a/cmrl/algorithms/offline/mopo.py +++ /dev/null @@ -1,85 +0,0 @@ -import os -from typing import Optional, cast - -import emei -import hydra.utils -import numpy as np -from omegaconf import DictConfig -from stable_baselines3.common.base_class import BaseAlgorithm -from stable_baselines3.common.buffers import ReplayBuffer - -from cmrl.agent import complete_agent_cfg - -# from cmrl.algorithms.utils import maybe_load_trained_offline_model, setup_fake_env, load_offline_data -# from cmrl.models.dynamics import ConstraintBasedDynamics -from cmrl.sb3_extension.eval_callback import EvalCallback -from cmrl.sb3_extension.logger import configure as logger_configure -from cmrl.utils.types import InitObsFnType, RewardFnType, TermFnType -from cmrl.utils.creator import create_dynamics - - -def train( - env: emei.EmeiEnv, - eval_env: emei.EmeiEnv, - termination_fn: Optional[TermFnType], - reward_fn: Optional[RewardFnType], - get_init_obs_fn: Optional[InitObsFnType], - cfg: DictConfig, - work_dir: Optional[str] = None, -): - obs_shape = env.observation_space.shape - act_shape = env.action_space.shape - - # build model-free agent, which is a stable-baselines3's agent - complete_agent_cfg(env, cfg.algorithm.agent) - agent = cast(BaseAlgorithm, hydra.utils.instantiate(cfg.algorithm.agent)) - - work_dir = work_dir or os.getcwd() - logger = logger_configure("log", ["tensorboard", "multi_csv", "stdout"]) - - numpy_generator = np.random.default_rng(seed=cfg.seed) - - # create initial dataset and add it to replay buffer - dynamics = create_dynamics(cfg.dynamics, obs_shape, act_shape, logger=logger) - real_replay_buffer = ReplayBuffer( - cfg.task.num_steps, env.observation_space, env.action_space, cfg.device, handle_timeout_termination=False - ) - load_offline_data(cfg, env, real_replay_buffer) - - fake_eval_env = setup_fake_env( - cfg=cfg, - agent=agent, - dynamics=dynamics, - reward_fn=reward_fn, - termination_fn=termination_fn, - get_init_obs_fn=get_init_obs_fn, - real_replay_buffer=real_replay_buffer, - logger=logger, - max_episode_steps=env.spec.max_episode_steps, - penalty_coeff=cfg.algorithm.penalty_coeff, - ) - - if hasattr(env, "get_causal_graph"): - oracle_causal_graph = env.get_causal_graph() - else: - oracle_causal_graph = None - - if isinstance(dynamics, ConstraintBasedDynamics): - dynamics.set_oracle_mask("transition", oracle_causal_graph.T) - - existed_trained_model = maybe_load_trained_offline_model(dynamics, cfg, obs_shape, act_shape, work_dir=work_dir) - if not existed_trained_model: - dynamics.learn(real_replay_buffer, **cfg.dynamics, work_dir=work_dir) - - eval_callback = EvalCallback( - eval_env, - fake_eval_env, - n_eval_episodes=cfg.task.n_eval_episodes, - best_model_save_path="./", - eval_freq=1000, - deterministic=True, - render=False, - ) - - agent.set_logger(logger) - agent.learn(total_timesteps=cfg.task.num_steps, callback=eval_callback) diff --git a/cmrl/algorithms/online/__init__.py b/cmrl/algorithms/online/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/cmrl/algorithms/util.py b/cmrl/algorithms/util.py index e71d6f2..364954e 100644 --- a/cmrl/algorithms/util.py +++ b/cmrl/algorithms/util.py @@ -60,43 +60,3 @@ def is_same_dict(dict1, dict2): # print("loaded dynamics from {}".format(exp_dir)) # return True # return False - - -def setup_fake_env( - cfg: DictConfig, - agent: BaseAlgorithm, - dynamics, - reward_fn: Optional[RewardFnType], - termination_fn: Optional[TermFnType], - get_init_obs_fn: Optional[InitObsFnType], - real_replay_buffer: Optional[ReplayBuffer] = None, - logger=None, - max_episode_steps: int = 1000, - penalty_coeff: Optional[float] = 0, -): - fake_env = cast(VecFakeEnv, agent.env) - fake_env.set_up( - dynamics, - reward_fn, - termination_fn, - get_init_obs_fn=get_init_obs_fn, - real_replay_buffer=real_replay_buffer, - logger=logger, - max_episode_steps=max_episode_steps, - penalty_coeff=penalty_coeff, - ) - agent.env = VecMonitor(fake_env) - - fake_eval_env_cfg = deepcopy(cfg.algorithm.agent.env) - fake_eval_env_cfg.num_envs = cfg.task.n_eval_episodes - fake_eval_env = cast(VecFakeEnv, hydra.utils.instantiate(fake_eval_env_cfg)) - fake_eval_env.set_up( - dynamics, - reward_fn, - termination_fn, - get_init_obs_fn=get_init_obs_fn, - max_episode_steps=max_episode_steps, - penalty_coeff=penalty_coeff, - ) - fake_eval_env.seed(seed=cfg.seed) - return fake_eval_env diff --git a/cmrl/examples/conf/algorithm/mbpo.yaml b/cmrl/examples/conf/algorithm/mbpo.yaml index e974c8a..6280510 100644 --- a/cmrl/examples/conf/algorithm/mbpo.yaml +++ b/cmrl/examples/conf/algorithm/mbpo.yaml @@ -8,6 +8,10 @@ initial_exploration_steps: 5000 random_initial_explore: false num_eval_episodes: 5 +algo: + _partial_: true + _target_: cmrl.algorithms.MBPO + # -------------------------------------------- # SAC Agent configuration # -------------------------------------------- diff --git a/cmrl/examples/conf/algorithm/mopo.yaml b/cmrl/examples/conf/algorithm/mopo.yaml index 7c249b6..0990596 100644 --- a/cmrl/examples/conf/algorithm/mopo.yaml +++ b/cmrl/examples/conf/algorithm/mopo.yaml @@ -9,26 +9,22 @@ num_eval_episodes: 5 dataset_size: 1000000 penalty_coeff: ${task.penalty_coeff} +branch_rollout_length: 5 + +algo: + _partial_: true + _target_: cmrl.algorithms.MOPO + # -------------------------------------------- # SAC Agent configuration # -------------------------------------------- +num_envs: 1000 +deterministic: true agent: _partial_: true _target_: stable_baselines3.sac.SAC policy: "MlpPolicy" - env: - _target_: cmrl.models.fake_env.VecFakeEnv - num_envs: 1000 - action_space: - _target_: gym.spaces.Box - low: ??? - high: ??? - shape: ??? - observation_space: - _target_: gym.spaces.Box - low: ??? - high: ??? - shape: ??? + env: ??? learning_starts: 0 batch_size: 256 tau: 0.005 diff --git a/cmrl/examples/conf/algorithm/off_dyna.yaml b/cmrl/examples/conf/algorithm/off_dyna.yaml index 3724f94..6fc2abf 100644 --- a/cmrl/examples/conf/algorithm/off_dyna.yaml +++ b/cmrl/examples/conf/algorithm/off_dyna.yaml @@ -9,6 +9,10 @@ num_eval_episodes: 5 dataset_size: 1000000 penalty_coeff: ${task.penalty_coeff} +algo: + _partial_: true + _target_: cmrl.algorithms.OfflineDyna + # -------------------------------------------- # SAC Agent configuration # -------------------------------------------- diff --git a/cmrl/examples/conf/algorithm/on_dyna.yaml b/cmrl/examples/conf/algorithm/on_dyna.yaml index f849d1e..70cdcdd 100644 --- a/cmrl/examples/conf/algorithm/on_dyna.yaml +++ b/cmrl/examples/conf/algorithm/on_dyna.yaml @@ -8,6 +8,10 @@ num_eval_episodes: 5 initial_exploration_steps: 1000 +algo: + _partial_: true + _target_: cmrl.algorithms.OnlineDyna + # -------------------------------------------- # SAC Agent configuration # -------------------------------------------- diff --git a/cmrl/examples/conf/dynamics/constraint_based_dynamics.yaml b/cmrl/examples/conf/dynamics/constraint_based_dynamics.yaml deleted file mode 100644 index 8c7fc9a..0000000 --- a/cmrl/examples/conf/dynamics/constraint_based_dynamics.yaml +++ /dev/null @@ -1,63 +0,0 @@ -name: constraint_based_dynamics - -multi_step: ${task.multi_step} - -transition: - _target_: cmrl.models.transition.ExternalMaskTransition - # transition info - obs_size: ??? - action_size: ??? - deterministic: false - # algorithm parameters - ensemble_num: ${task.ensemble_num} - elite_num: ${task.elite_num} - residual: true - learn_logvar_bounds: false # so far this works better - # network parameters - num_layers: 4 - hid_size: 200 - activation_fn_cfg: - _target_: torch.nn.SiLU - # others - device: ${device} - -learned_reward: ${task.learning_reward} -reward_mech: - _target_: cmrl.models.BaseRewardMech - # transition info - obs_size: ??? - action_size: ??? - deterministic: false - # algorithm parameters - learn_logvar_bounds: false # so far this works better - ensemble_num: ${task.ensemble_num} - elite_num: ${task.elite_num} - # network parameters - num_layers: 4 - hid_size: 200 - activation_fn_cfg: - _target_: torch.nn.SiLU - # others - device: ${device} - -learned_termination: ${task.learning_terminal} -termination_mech: - _target_: cmrl.models.BaseTerminationMech - # transition info - obs_size: ??? - action_size: ??? - deterministic: false - -optim_lr: ${task.optim_lr} -weight_decay: ${task.weight_decay} -patience: ${task.patience} -batch_size: ${task.batch_size} -use_ratio: ${task.use_ratio} -validation_ratio: ${task.validation_ratio} -shuffle_each_epoch: ${task.shuffle_each_epoch} -bootstrap_permutes: ${task.bootstrap_permutes} -longest_epoch: ${task.longest_epoch} -improvement_threshold: ${task.improvement_threshold} - -normalize: true -normalize_double_precision: true diff --git a/cmrl/examples/conf/dynamics/plain_dynamics.yaml b/cmrl/examples/conf/dynamics/plain_dynamics.yaml deleted file mode 100644 index 933ea15..0000000 --- a/cmrl/examples/conf/dynamics/plain_dynamics.yaml +++ /dev/null @@ -1,63 +0,0 @@ -name: plain_dynamics - -multi_step: ${task.multi_step} - -transition: - _target_: cmrl.models.transition.PlainTransition - # transition info - obs_size: ??? - action_size: ??? - deterministic: false - # algorithm parameters - ensemble_num: ${task.ensemble_num} - elite_num: ${task.elite_num} - residual: true - learn_logvar_bounds: false # so far this works better - # network parameters - num_layers: 4 - hid_size: 200 - activation_fn_cfg: - _target_: torch.nn.SiLU - # others - device: ${device} - -learned_reward: ${task.learning_reward} -reward_mech: - _target_: cmrl.models.BaseRewardMech - # transition info - obs_size: ??? - action_size: ??? - deterministic: fase - # algorithm parameters - learn_logvar_bounds: false # so far this works better - ensemble_num: ${task.ensemble_num} - elite_num: ${task.elite_num} - # network parameters - num_layers: 4 - hid_size: 200 - activation_fn_cfg: - _target_: torch.nn.SiLU - # others - device: ${device} - -learned_termination: ${task.learning_terminal} -termination_mech: - _target_: cmrl.models.BaseTerminationMech - # transition info - obs_size: ??? - action_size: ??? - deterministic: false - -optim_lr: ${task.optim_lr} -weight_decay: ${task.weight_decay} -patience: ${task.patience} -batch_size: ${task.batch_size} -use_ratio: ${task.use_ratio} -validation_ratio: ${task.validation_ratio} -shuffle_each_epoch: ${task.shuffle_each_epoch} -bootstrap_permutes: ${task.bootstrap_permutes} -longest_epoch: ${task.longest_epoch} -improvement_threshold: ${task.improvement_threshold} - -normalize: true -normalize_double_precision: true diff --git a/cmrl/examples/conf/main.yaml b/cmrl/examples/conf/main.yaml index 83fc84f..b943805 100644 --- a/cmrl/examples/conf/main.yaml +++ b/cmrl/examples/conf/main.yaml @@ -1,8 +1,6 @@ defaults: - algorithm: off_dyna - - dynamics: constraint_based_dynamics - task: BIPS - - transition: plain - reward_mech: plain - termination_mech: plain @@ -17,8 +15,8 @@ wandb: false root_dir: "./exp" hydra: run: - dir: ${root_dir}/${exp_name}/${task.env}/${dynamics.name}/${now:%Y.%m.%d}/${now:%H.%M.%S} + dir: ${root_dir}/${exp_name}/${task.env}/${now:%Y.%m.%d}/${now:%H.%M.%S} sweep: - dir: ${root_dir}/${exp_name}/${task.env}/${dynamics.name}/${now:%Y.%m.%d}/${now:%H.%M.%S} + dir: ${root_dir}/${exp_name}/${task.env}/${now:%Y.%m.%d}/${now:%H.%M.%S} job: chdir: true diff --git a/cmrl/examples/conf/task/BIPS.yaml b/cmrl/examples/conf/task/BIPS.yaml index d8560ca..9a54676 100644 --- a/cmrl/examples/conf/task/BIPS.yaml +++ b/cmrl/examples/conf/task/BIPS.yaml @@ -31,19 +31,8 @@ use_ratio: 1 # dyna freq_train_model: 100 + # model learning patience: 20 -optim_lr: 0.0001 -weight_decay: 0.00001 -batch_size: 256 -validation_ratio: 0.2 -shuffle_each_epoch: true -bootstrap_permutes: false longest_epoch: -1 improvement_threshold: 0.01 -# model using -effective_model_rollouts_per_step: 50 -rollout_schedule: [ 1, 15, 1, 1 ] -num_sac_updates_per_step: 1 -sac_updates_every_steps: 1 -num_epochs_to_retain_sac_buffer: 1 diff --git a/cmrl/examples/conf/task/mbpo_ant.yaml b/cmrl/examples/conf/task/mbpo_ant.yaml deleted file mode 100644 index 86772f3..0000000 --- a/cmrl/examples/conf/task/mbpo_ant.yaml +++ /dev/null @@ -1,28 +0,0 @@ -env: "ant_truncated_obs" -# term_fn is set automatically by cmrl.utils.env.EnvHandler.make_env - -num_steps: 300000 -epoch_length: 1000 -num_elites: 5 -patience: 10 -model_lr: 0.0003 -model_wd: 5e-5 -model_batch_size: 256 -validation_ratio: 0.2 -freq_train_model: 250 -effective_model_rollouts_per_step: 400 -rollout_schedule: [20, 100, 1, 25] -num_sac_updates_per_step: 20 -sac_updates_every_steps: 1 -num_epochs_to_retain_sac_buffer: 1 - -sac_gamma: 0.99 -sac_tau: 0.005 -sac_alpha: 0.2 -sac_policy: "Gaussian" -sac_target_update_interval: 4 -sac_automatic_entropy_tuning: false -sac_target_entropy: -1 # ignored, since entropy tuning is false -sac_hidden_size: 1024 -sac_lr: 0.0001 -sac_batch_size: 256 diff --git a/cmrl/examples/conf/task/mbpo_boundary_inverted_double_pendulum_swing_up.yaml b/cmrl/examples/conf/task/mbpo_boundary_inverted_double_pendulum_swing_up.yaml deleted file mode 100644 index e31a548..0000000 --- a/cmrl/examples/conf/task/mbpo_boundary_inverted_double_pendulum_swing_up.yaml +++ /dev/null @@ -1,33 +0,0 @@ -env: "emei___BoundaryInvertedDoublePendulumSwingUp-v0___freq_rate=1&time_step=0.02" - -oracle: true -cit_threshold: 0.02 -test_freq: 500 - -num_steps: 800000 -epoch_length: 1000 -num_elites: 5 -patience: 5 -model_lr: 0.001 -model_wd: 0.00001 -model_batch_size: 256 -validation_ratio: 0.2 -freq_train_model: 250 -update_causal_mask_ratio: 0.25 -discovery_schedule: [1, 30, 250, 250] -effective_model_rollouts_per_step: 400 -rollout_schedule: [1, 15, 100, 100] -num_sac_updates_per_step: 10 -sac_updates_every_steps: 1 -num_epochs_to_retain_sac_buffer: 1 - -sac_gamma: 0.99 -sac_tau: 0.005 -sac_alpha: 0.2 -sac_policy: "Gaussian" -sac_target_update_interval: 1 -sac_automatic_entropy_tuning: true -sac_hidden_size: 256 -sac_lr: 0.0003 -sac_batch_size: 256 -sac_target_entropy: -1 diff --git a/cmrl/examples/conf/task/mbpo_boundary_inverted_pendulum_holding.yaml b/cmrl/examples/conf/task/mbpo_boundary_inverted_pendulum_holding.yaml deleted file mode 100644 index a8f7da6..0000000 --- a/cmrl/examples/conf/task/mbpo_boundary_inverted_pendulum_holding.yaml +++ /dev/null @@ -1,33 +0,0 @@ -env: "emei___BoundaryInvertedPendulumHolding-v0___freq_rate=1&time_step=0.02" - -oracle: true -cit_threshold: 0.02 -test_freq: 1000 - -num_steps: 20000 -epoch_length: 1000 -num_elites: 5 -patience: 5 -model_lr: 0.001 -model_wd: 0.00001 -model_batch_size: 256 -validation_ratio: 0.2 -freq_train_model: 250 -update_causal_mask_ratio: 0.25 -discovery_schedule: [1, 30, 250, 250] -effective_model_rollouts_per_step: 400 -rollout_schedule: [1, 15, 100, 100] -num_sac_updates_per_step: 10 -sac_updates_every_steps: 1 -num_epochs_to_retain_sac_buffer: 1 - -sac_gamma: 0.99 -sac_tau: 0.005 -sac_alpha: 0.2 -sac_policy: "Gaussian" -sac_target_update_interval: 1 -sac_automatic_entropy_tuning: true -sac_hidden_size: 256 -sac_lr: 0.0003 -sac_batch_size: 256 -sac_target_entropy: -1 diff --git a/cmrl/examples/conf/task/mbpo_boundary_inverted_pendulum_swing_up.yaml b/cmrl/examples/conf/task/mbpo_boundary_inverted_pendulum_swing_up.yaml deleted file mode 100644 index 6dbbc1a..0000000 --- a/cmrl/examples/conf/task/mbpo_boundary_inverted_pendulum_swing_up.yaml +++ /dev/null @@ -1,33 +0,0 @@ -env: "emei___BoundaryInvertedPendulumSwingUp-v0___freq_rate=1&time_step=0.02" - -oracle: true -cit_threshold: 0.02 -test_freq: 500 - -num_steps: 8000 -epoch_length: 1000 -num_elites: 5 -patience: 5 -model_lr: 0.001 -model_wd: 0.00001 -model_batch_size: 256 -validation_ratio: 0.2 -freq_train_model: 250 -update_causal_mask_ratio: 0.25 -discovery_schedule: [1, 30, 250, 250] -effective_model_rollouts_per_step: 400 -rollout_schedule: [1, 15, 1, 1] -num_sac_updates_per_step: 10 -sac_updates_every_steps: 1 -num_epochs_to_retain_sac_buffer: 1 - -sac_gamma: 0.99 -sac_tau: 0.005 -sac_alpha: 0.2 -sac_policy: "Gaussian" -sac_target_update_interval: 1 -sac_automatic_entropy_tuning: true -sac_hidden_size: 256 -sac_lr: 0.0003 -sac_batch_size: 256 -sac_target_entropy: -1 diff --git a/cmrl/examples/conf/task/mbpo_cartpole.yaml b/cmrl/examples/conf/task/mbpo_cartpole.yaml deleted file mode 100644 index a6016cf..0000000 --- a/cmrl/examples/conf/task/mbpo_cartpole.yaml +++ /dev/null @@ -1,28 +0,0 @@ -env: "cartpole_continuous" -trial_length: 200 - -num_steps: 5000 -epoch_length: 200 -num_elites: 5 -patience: 5 -model_lr: 0.001 -model_wd: 0.00005 -model_batch_size: 256 -validation_ratio: 0.2 -freq_train_model: 200 -effective_model_rollouts_per_step: 400 -rollout_schedule: [1, 15, 1, 1] -num_sac_updates_per_step: 20 -sac_updates_every_steps: 1 -num_epochs_to_retain_sac_buffer: 1 - -sac_gamma: 0.99 -sac_tau: 0.005 -sac_alpha: 0.2 -sac_policy: "Gaussian" -sac_target_update_interval: 4 -sac_automatic_entropy_tuning: true -sac_target_entropy: -0.05 -sac_hidden_size: 256 -sac_lr: 0.0003 -sac_batch_size: 256 diff --git a/cmrl/examples/conf/task/mbpo_halfcheetah.yaml b/cmrl/examples/conf/task/mbpo_halfcheetah.yaml deleted file mode 100644 index 3b5e3f9..0000000 --- a/cmrl/examples/conf/task/mbpo_halfcheetah.yaml +++ /dev/null @@ -1,28 +0,0 @@ -env: "gym___HalfCheetah-v2" -term_fn: "no_termination" - -num_steps: 400000 -epoch_length: 1000 -num_elites: 5 -patience: 5 -model_lr: 0.001 -model_wd: 0.00001 -model_batch_size: 256 -validation_ratio: 0.2 -freq_train_model: 250 -effective_model_rollouts_per_step: 400 -rollout_schedule: [20, 150, 1, 1] -num_sac_updates_per_step: 10 -sac_updates_every_steps: 1 -num_epochs_to_retain_sac_buffer: 1 - -sac_gamma: 0.99 -sac_tau: 0.005 -sac_alpha: 0.2 -sac_policy: "Gaussian" -sac_target_update_interval: 1 -sac_automatic_entropy_tuning: true -sac_target_entropy: -1 -sac_hidden_size: 512 -sac_lr: 0.0003 -sac_batch_size: 256 diff --git a/cmrl/examples/conf/task/mbpo_hopper.yaml b/cmrl/examples/conf/task/mbpo_hopper.yaml deleted file mode 100644 index 5ee267c..0000000 --- a/cmrl/examples/conf/task/mbpo_hopper.yaml +++ /dev/null @@ -1,28 +0,0 @@ -env: "gym___Hopper-v2" -term_fn: "hopper" - -num_steps: 500000 -epoch_length: 1000 -num_elites: 5 -patience: 5 -model_lr: 0.001 -model_wd: 0.00001 -model_batch_size: 256 -validation_ratio: 0.2 -freq_train_model: 250 -effective_model_rollouts_per_step: 400 -rollout_schedule: [20, 150, 200, 200] -num_sac_updates_per_step: 100 -sac_updates_every_steps: 1 -num_epochs_to_retain_sac_buffer: 1 - -sac_gamma: 0.99 -sac_tau: 0.005 -sac_alpha: 0.2 -sac_policy: "Gaussian" -sac_target_update_interval: 4 -sac_automatic_entropy_tuning: false -sac_target_entropy: 1 # ignored, since entropy tuning is false -sac_hidden_size: 512 -sac_lr: 0.0003 -sac_batch_size: 256 diff --git a/cmrl/examples/conf/task/mbpo_humanoid.yaml b/cmrl/examples/conf/task/mbpo_humanoid.yaml deleted file mode 100644 index 72c34ef..0000000 --- a/cmrl/examples/conf/task/mbpo_humanoid.yaml +++ /dev/null @@ -1,28 +0,0 @@ -env: "humanoid_truncated_obs" -# term_fn is set automatically by cmrl.utils.env.EnvHandler.make_env - -num_steps: 300000 -epoch_length: 1000 -num_elites: 5 -patience: 10 -model_lr: 0.0003 -model_wd: 5e-5 -model_batch_size: 256 -validation_ratio: 0.2 -freq_train_model: 250 -effective_model_rollouts_per_step: 400 -rollout_schedule: [20, 300, 1, 25] -num_sac_updates_per_step: 20 -sac_updates_every_steps: 1 -num_epochs_to_retain_sac_buffer: 5 - -sac_gamma: 0.99 -sac_tau: 0.005 -sac_alpha: 0.2 -sac_policy: "Gaussian" -sac_target_update_interval: 4 -sac_automatic_entropy_tuning: false -sac_target_entropy: -1 # ignored, since entropy tuning is false -sac_hidden_size: 1024 -sac_lr: 0.0001 -sac_batch_size: 256 diff --git a/cmrl/examples/conf/task/mbpo_inv_pendulum.yaml b/cmrl/examples/conf/task/mbpo_inv_pendulum.yaml deleted file mode 100644 index 912d4f5..0000000 --- a/cmrl/examples/conf/task/mbpo_inv_pendulum.yaml +++ /dev/null @@ -1,29 +0,0 @@ -env: "inv_pendulum___0.25___1" - -test_freq: 500 - -num_steps: 500000 -epoch_length: 1000 -num_elites: 5 -patience: 5 -model_lr: 0.001 -model_wd: 0.00001 -model_batch_size: 256 -validation_ratio: 0.2 -freq_train_model: 250 -effective_model_rollouts_per_step: 400 -rollout_schedule: [1, 15, 10, 10] -num_sac_updates_per_step: 10 -sac_updates_every_steps: 1 -num_epochs_to_retain_sac_buffer: 1 - -sac_gamma: 0.99 -sac_tau: 0.005 -sac_alpha: 0.2 -sac_policy: "Gaussian" -sac_target_update_interval: 1 -sac_automatic_entropy_tuning: true -sac_hidden_size: 256 -sac_lr: 0.0003 -sac_batch_size: 256 -sac_target_entropy: -1 diff --git a/cmrl/examples/conf/task/mbpo_pusher.yaml b/cmrl/examples/conf/task/mbpo_pusher.yaml deleted file mode 100644 index c90c0b1..0000000 --- a/cmrl/examples/conf/task/mbpo_pusher.yaml +++ /dev/null @@ -1,29 +0,0 @@ -env: "pets_pusher" -term_fn: "no_termination" -trial_length: 150 - -num_steps: 20000 -epoch_length: 150 -num_elites: 5 -patience: 5 -model_lr: 0.001 -model_wd: 0.00005 -model_batch_size: 256 -validation_ratio: 0.2 -freq_train_model: 250 -effective_model_rollouts_per_step: 400 -rollout_schedule: [1, 15, 1, 1] -num_sac_updates_per_step: 20 -sac_updates_every_steps: 1 -num_epochs_to_retain_sac_buffer: 1 - -sac_gamma: 0.99 -sac_tau: 0.005 -sac_alpha: 0.2 -sac_policy: "Gaussian" -sac_target_update_interval: 4 -sac_automatic_entropy_tuning: true -sac_target_entropy: -0.05 -sac_hidden_size: 256 -sac_lr: 0.0003 -sac_batch_size: 256 diff --git a/cmrl/examples/conf/task/mbpo_walker.yaml b/cmrl/examples/conf/task/mbpo_walker.yaml deleted file mode 100644 index 5ef2095..0000000 --- a/cmrl/examples/conf/task/mbpo_walker.yaml +++ /dev/null @@ -1,28 +0,0 @@ -env: "gym___Walker2d-v2" -term_fn: "walker2d" - -num_steps: 300000 -epoch_length: 1000 -num_elites: 5 -patience: 10 -model_lr: 0.001 -model_wd: 0.00001 -model_batch_size: 256 -validation_ratio: 0.2 -freq_train_model: 250 -effective_model_rollouts_per_step: 400 -rollout_schedule: [20, 150, 1, 1] -num_sac_updates_per_step: 20 -sac_updates_every_steps: 1 -num_epochs_to_retain_sac_buffer: 1 - -sac_gamma: 0.99 -sac_tau: 0.005 -sac_alpha: 0.2 -sac_policy: "Gaussian" -sac_target_update_interval: 4 -sac_automatic_entropy_tuning: false -sac_target_entropy: -1 # ignored, since entropy tuning is false -sac_hidden_size: 1024 -sac_lr: 0.0001 -sac_batch_size: 256 diff --git a/cmrl/examples/main.py b/cmrl/examples/main.py index 6bfd22b..fac471d 100644 --- a/cmrl/examples/main.py +++ b/cmrl/examples/main.py @@ -1,15 +1,8 @@ import hydra -import numpy as np -import torch +from hydra.utils import instantiate import wandb from omegaconf import DictConfig, OmegaConf -import cmrl.algorithms.offline.off_dyna as off_dyna -import cmrl.algorithms.offline.mopo as mopo -import cmrl.algorithms.online.on_dyna as on_dyna -import cmrl.algorithms.online.mbpo as mbpo -from cmrl.utils.env import make_env - @hydra.main(version_base=None, config_path="conf", config_name="main") def run(cfg: DictConfig): @@ -21,25 +14,8 @@ def run(cfg: DictConfig): sync_tensorboard=True, ) - env, term_fn, reward_fn, init_obs_fn = make_env(cfg) - test_env, *_ = make_env(cfg) - np.random.seed(cfg.seed) - torch.manual_seed(cfg.seed) - - if cfg.algorithm.name == "on_dyna": - test_env, *_ = make_env(cfg) - return on_dyna.train(env, test_env, reward_fn, term_fn, init_obs_fn, cfg) - elif cfg.algorithm.name == "mopo": - test_env, *_ = make_env(cfg) - return mopo.train(env, test_env, reward_fn, term_fn, init_obs_fn, cfg) - elif cfg.algorithm.name == "off_dyna": - test_env, *_ = make_env(cfg) - return off_dyna.train(env, test_env, reward_fn, term_fn, init_obs_fn, cfg) - elif cfg.algorithm.name == "mbpo": - test_env, *_ = make_env(cfg) - return mbpo.train(env, test_env, reward_fn, term_fn, init_obs_fn, cfg) - else: - raise NotImplementedError + algo = instantiate(cfg.algorithm.algo)(cfg=cfg) + algo.learn() if __name__ == "__main__": diff --git a/cmrl/models/causal_discovery/CMI_test.py b/cmrl/models/causal_discovery/CMI_test.py index 7ec688b..03bc3a1 100644 --- a/cmrl/models/causal_discovery/CMI_test.py +++ b/cmrl/models/causal_discovery/CMI_test.py @@ -8,7 +8,6 @@ # from cmrl.models.layers import ParallelEnsembleLinearLayer, truncated_normal_init from cmrl.models.networks.mlp import EnsembleMLP -from cmrl.models.util import to_tensor class TransitionConditionalMutualInformationTest(EnsembleMLP): diff --git a/cmrl/models/causal_mech/CMI_test.py b/cmrl/models/causal_mech/CMI_test.py index c67ae18..75e2f2f 100644 --- a/cmrl/models/causal_mech/CMI_test.py +++ b/cmrl/models/causal_mech/CMI_test.py @@ -1,7 +1,250 @@ +from typing import Optional, List, Dict, Union, MutableMapping +import pathlib +import itertools +import copy + +import numpy as np +import torch +from torch.utils.data import DataLoader +from torch.optim import Adam +from omegaconf import DictConfig +from stable_baselines3.common.logger import Logger + +from cmrl.utils.types import Variable +from cmrl.models.networks.parallel_mlp import ParallelMLP from cmrl.models.causal_mech.base_causal_mech import BaseCausalMech +from cmrl.models.networks.coder import VariableEncoder, VariableDecoder + + +class CMItest(BaseCausalMech): + def __init__( + self, + name: str, + # base causal-mech params + input_variables: List[Variable], + output_variables: List[Variable], + node_dim: int, + variable_encoders: Optional[Dict[str, VariableEncoder]], + variable_decoders: Optional[Dict[str, VariableDecoder]], + ensemble_num: int = 7, + elite_num: int = 5, + # network params + deterministic: bool = False, + hidden_dims: Optional[List[int]] = None, + use_bias: bool = True, + activation_fn_cfg: Optional[DictConfig] = None, + # forward method + residual: bool = True, + multi_step: str = "none", + # trainer + optim_lr: float = 1e-4, + optim_weight_decay: float = 1e-5, + optim_eps: float = 1e-8, + optim_encoder: bool = True, + # logger + logger: Optional[Logger] = None, + # others + device: Union[str, torch.device] = "cpu", + **kwargs + ): + self.deterministic = deterministic + self.hidden_dims = hidden_dims if hidden_dims is not None else [200] * 4 + self.use_bias = use_bias + self.activation_fn_cfg = activation_fn_cfg + + if multi_step == "none": + multi_step = "forward-euler 1" + + super(CMItest, self).__init__( + name=name, + input_variables=input_variables, + output_variables=output_variables, + node_dim=node_dim, + variable_encoders=variable_encoders, + variable_decoders=variable_decoders, + ensemble_num=ensemble_num, + elite_num=elite_num, + residual=residual, + multi_step=multi_step, + optim_lr=optim_lr, + optim_weight_decay=optim_weight_decay, + optim_eps=optim_eps, + optim_encoder=optim_encoder, + logger=logger, + device=device, + **kwargs + ) + + def build_network(self): + self.network = ParallelMLP( + input_dim=self.node_dim, + output_dim=self.node_dim, + hidden_dims=self.hidden_dims, + use_bias=self.use_bias, + extra_dims=[self.output_var_num, self.ensemble_num], + activation_fn_cfg=self.activation_fn_cfg, + ).to(self.device) + + parmas = [self.network.parameters()] + [decoder.parameters() for decoder in self.variable_decoders.values()] + if self.optim_encoder: + parmas.extend([encoder.parameters() for encoder in self.variable_encoders.values()]) + self.optimizer = Adam( + itertools.chain(*parmas), lr=self.optim_lr, weight_decay=self.optim_weight_decay, eps=self.optim_eps + ) + + def build_graph(self): + self.graph = None + + def forward(self, inputs: MutableMapping[str, Union[torch.Tensor]]) -> Dict[str, torch.Tensor]: + assert len(set(inputs.keys()) & set(self.variable_encoders.keys())) == len(inputs) + data_shape = list(inputs.values())[0].shape + assert len(data_shape) == 3, "{}".format(data_shape) # ensemble-num, batch-size, specific-dim + ensemble, batch_size, specific_dim = data_shape + assert ensemble == self.ensemble_num + + inputs_tensor = torch.zeros(ensemble, batch_size, self.input_var_num * self.node_dim).to(self.device) + for i, var in enumerate(self.input_variables): + out = self.variable_encoders[var.name](inputs[var.name].to(self.device)) # ensemble-num, batch-size, node-dim + inputs_tensor[:, :, i * self.node_dim : (i + 1) * self.node_dim] = out + + if self.multi_step.startswith("forward-euler"): + step_num = int(self.multi_step.split()[-1]) + output_tensor = None + for step in range(step_num): + if step > 0: + inputs_tensor = torch.concat( + [output_tensor, inputs_tensor[:, :, self.output_var_num * self.node_dim :]], dim=-1 + ) + output_tensor = self.network(inputs_tensor) + if self.residual: + output_tensor += inputs_tensor[:, :, : self.output_var_num * self.node_dim] + else: + raise NotImplementedError + + outputs = {} + for i, var in enumerate(self.output_variables): + hid = output_tensor[:, :, i * self.node_dim : (i + 1) * self.node_dim] + out = self.variable_decoders[var.name](hid) + outputs[var.name] = out + + return outputs + + def train(self, loader: DataLoader): + """train for ensemble data + + Args: + loader: train data-loader. + + Returns: tensor of train loss, with shape (ensemble-num, batch-size). + + """ + batch_loss_list = [] + for inputs, targets in loader: + outputs = self.forward(inputs) + loss = self.loss(outputs, targets) # ensemble-num, batch-size, output-var-num + + self.optimizer.zero_grad() + loss.mean().backward() + self.optimizer.step() + + batch_loss_list.append(loss) + return torch.cat(batch_loss_list, dim=-2).detach().cpu() + + def eval(self, loader: DataLoader): + """evaluate for non-ensemble data + + Args: + loader: valid data-loader. + + Returns: tensor of eval loss, with shape (batch-size). + + """ + batch_loss_list = [] + with torch.no_grad(): + for inputs, targets in loader: + outputs = self.forward(inputs) + loss = self.loss(outputs, targets) # ensemble-num, batch-size, output-var-num + + batch_loss_list.append(loss) + return torch.cat(batch_loss_list, dim=-2).detach().cpu() + + def learn( + self, + # loader + train_loader: DataLoader, + valid_loader: DataLoader, + # model learning + longest_epoch: int = -1, + improvement_threshold: float = 0.01, + patience: int = 5, + work_dir: Optional[Union[str, pathlib.Path]] = None, + **kwargs + ): + best_weights: Optional[Dict] = None + epoch_iter = range(longest_epoch) if longest_epoch >= 0 else itertools.count() + epochs_since_update = 0 + best_eval_loss = self.eval(valid_loader).mean(dim=(1, 2)) + + for epoch in epoch_iter: + train_loss = self.train(train_loader) + eval_loss = self.eval(valid_loader).mean(dim=(1, 2)) + maybe_best_weights = self._maybe_get_best_weights(best_eval_loss, eval_loss, improvement_threshold) + if maybe_best_weights: + # best loss + best_eval_loss = torch.minimum(best_eval_loss, eval_loss) + best_weights = maybe_best_weights + epochs_since_update = 0 + else: + epochs_since_update += 1 + + # log + self.total_epoch += 1 + if self.logger is not None: + self.logger.record("{}/epoch".format(self.name), epoch) + self.logger.record("{}/train_dataset_size".format(self.name), len(train_loader.dataset)) + self.logger.record("{}/valid_dataset_size".format(self.name), len(valid_loader.dataset)) + self.logger.record("{}/train_loss".format(self.name), train_loss.mean().item()) + self.logger.record("{}/val_loss".format(self.name), eval_loss.mean().item()) + self.logger.record("{}/best_val_loss".format(self.name), best_eval_loss.mean().item()) + + self.logger.dump(self.total_epoch) + + if patience and epochs_since_update >= patience: + break + + # saving the best models: + self._maybe_set_best_weights_and_elite(best_weights, best_eval_loss) + + def _maybe_get_best_weights( + self, + best_val_loss: torch.Tensor, + val_loss: torch.Tensor, + threshold: float = 0.01, + ) -> Optional[Dict]: + """Return the current model state dict if the validation score improves. + For ensembles, this checks the validation for each ensemble member separately. + Copy from https://github.com/facebookresearch/mbrl-lib/blob/main/mbrl/models/model_trainer.py + + Args: + best_val_score (tensor): the current best validation losses per model. + val_score (tensor): the new validation loss per model. + threshold (float): the threshold for relative improvement. + Returns: + (dict, optional): if the validation score's relative improvement over the + best validation score is higher than the threshold, returns the state dictionary + of the stored model, otherwise returns ``None``. + """ + improvement = (best_val_loss - val_loss) / torch.abs(best_val_loss) + if (improvement > threshold).any().item(): + best_weights = copy.deepcopy(self.network.state_dict()) + else: + best_weights = None + + return best_weights + def _maybe_set_best_weights_and_elite(self, best_weights: Optional[Dict], best_val_score: torch.Tensor): + if best_weights is not None: + self.network.load_state_dict(best_weights) -class CMIMech(BaseCausalMech): - def __init__(self): - super(CMIMech, self).__init__() - pass + sorted_indices = np.argsort(best_val_score.tolist()) + self.elite_indices = sorted_indices[: self.elite_num] diff --git a/cmrl/models/causal_mech/base_causal_mech.py b/cmrl/models/causal_mech/base_causal_mech.py index 6c2b7f1..e3704fd 100644 --- a/cmrl/models/causal_mech/base_causal_mech.py +++ b/cmrl/models/causal_mech/base_causal_mech.py @@ -5,13 +5,14 @@ import numpy as np from torch.utils.data import DataLoader import torch.nn.functional as F +from torch.optim import Optimizer from stable_baselines3.common.logger import Logger -from cmrl.utils.types import Variable, ContinuousVariable, DiscreteVariable, BinaryVariable from cmrl.models.networks.base_network import BaseNetwork from cmrl.models.graphs.base_graph import BaseGraph from cmrl.models.networks.coder import VariableEncoder, VariableDecoder from cmrl.models.util import create_decoders, create_encoders +from cmrl.utils.types import Variable, ContinuousVariable, DiscreteVariable, BinaryVariable class BaseCausalMech: @@ -71,6 +72,7 @@ def __init__( self.check_coder() self.network: Optional[BaseNetwork] = None + self.optimizer: Optional[Optimizer] = None self.graph: Optional[BaseGraph] = None self.build_network() diff --git a/cmrl/models/causal_mech/plain_mech.py b/cmrl/models/causal_mech/plain_mech.py index 29fafc8..4c21747 100644 --- a/cmrl/models/causal_mech/plain_mech.py +++ b/cmrl/models/causal_mech/plain_mech.py @@ -62,6 +62,8 @@ def __init__( node_dim=node_dim, variable_encoders=variable_encoders, variable_decoders=variable_decoders, + ensemble_num=ensemble_num, + elite_num=elite_num, residual=residual, multi_step=multi_step, optim_lr=optim_lr, @@ -86,7 +88,9 @@ def build_network(self): parmas = [self.network.parameters()] + [decoder.parameters() for decoder in self.variable_decoders.values()] if self.optim_encoder: parmas.extend([encoder.parameters() for encoder in self.variable_encoders.values()]) - self.optim = Adam(itertools.chain(*parmas), lr=self.optim_lr, weight_decay=self.optim_weight_decay, eps=self.optim_eps) + self.optimizer = Adam( + itertools.chain(*parmas), lr=self.optim_lr, weight_decay=self.optim_weight_decay, eps=self.optim_eps + ) def build_graph(self): self.graph = None @@ -139,9 +143,9 @@ def train(self, loader: DataLoader): outputs = self.forward(inputs) loss = self.loss(outputs, targets) # ensemble-num, batch-size, output-var-num - self.optim.zero_grad() + self.optimizer.zero_grad() loss.mean().backward() - self.optim.step() + self.optimizer.step() batch_loss_list.append(loss) return torch.cat(batch_loss_list, dim=-2).detach().cpu() diff --git a/cmrl/models/dynamics.py b/cmrl/models/dynamics.py index 66e854b..d0e51e1 100644 --- a/cmrl/models/dynamics.py +++ b/cmrl/models/dynamics.py @@ -71,8 +71,6 @@ def learn( work_dir: Optional[Union[str, pathlib.Path]] = None, **kwargs ): - longest_epoch = 0 - # transition self.transition.learn( *self.get_loader(real_replay_buffer, "transition"), diff --git a/cmrl/models/old_dynamics/constraint_based_dynamics.py b/cmrl/models/old_dynamics/constraint_based_dynamics.py index 41d4c8d..c506b0c 100644 --- a/cmrl/models/old_dynamics/constraint_based_dynamics.py +++ b/cmrl/models/old_dynamics/constraint_based_dynamics.py @@ -46,7 +46,7 @@ def __init__( # self.cmi_test: Optional[EnsembleMLP] = None # self.build_cmi_test() # - # self.cmi_test_optimizer = torch.optim.Adam( + # self.cmi_test_optimizer = torch.optimizer.Adam( # self.cmi_test.parameters(), # lr=optim_lr, # weight_decay=weight_decay, diff --git a/cmrl/models/old_dynamics/ncd_dynamics.py b/cmrl/models/old_dynamics/ncd_dynamics.py index cc2f773..6f7b7c9 100644 --- a/cmrl/models/old_dynamics/ncd_dynamics.py +++ b/cmrl/models/old_dynamics/ncd_dynamics.py @@ -46,7 +46,7 @@ def __init__( # self.cmi_test: Optional[EnsembleMLP] = None # self.build_cmi_test() # - # self.cmi_test_optimizer = torch.optim.Adam( + # self.cmi_test_optimizer = torch.optimizer.Adam( # self.cmi_test.parameters(), # lr=optim_lr, # weight_decay=weight_decay, diff --git a/cmrl/models/util.py b/cmrl/models/util.py index 0d38380..5d8f83f 100644 --- a/cmrl/models/util.py +++ b/cmrl/models/util.py @@ -1,7 +1,3 @@ -# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. from typing import List, Optional, Union, Dict import numpy as np diff --git a/cmrl/utils/env.py b/cmrl/utils/env.py index 63f4ec4..4ae7f4c 100644 --- a/cmrl/utils/env.py +++ b/cmrl/utils/env.py @@ -38,8 +38,8 @@ def make_env( )[1:3] kwargs = dict([(item.split("=")[0], to_num(item.split("=")[1])) for item in params.split("&")]) env = cast(emei.EmeiEnv, gym.make(env_name, **kwargs)) - term_fn = env.get_terminal reward_fn = env.get_reward + term_fn = env.get_terminal init_obs_fn = env.get_batch_init_obs else: raise NotImplementedError @@ -48,4 +48,4 @@ def make_env( env.reset(seed=cfg.seed) env.observation_space.seed(cfg.seed + 1) env.action_space.seed(cfg.seed + 2) - return env, term_fn, reward_fn, init_obs_fn + return env, reward_fn, term_fn, init_obs_fn From 4a9f9898ff4500eab2d3d731d309ddddcee04e72 Mon Sep 17 00:00:00 2001 From: wz139704646 <632291793@qq.com> Date: Mon, 7 Nov 2022 15:24:17 +0800 Subject: [PATCH 17/68] :beetle: fix type check in causal mechs --- cmrl/models/causal_mech/CMI_test.py | 2 +- cmrl/models/causal_mech/base_causal_mech.py | 2 +- cmrl/models/causal_mech/plain_mech.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cmrl/models/causal_mech/CMI_test.py b/cmrl/models/causal_mech/CMI_test.py index 75e2f2f..043bbcc 100644 --- a/cmrl/models/causal_mech/CMI_test.py +++ b/cmrl/models/causal_mech/CMI_test.py @@ -95,7 +95,7 @@ def build_network(self): def build_graph(self): self.graph = None - def forward(self, inputs: MutableMapping[str, Union[torch.Tensor]]) -> Dict[str, torch.Tensor]: + def forward(self, inputs: MutableMapping[str, torch.Tensor]) -> Dict[str, torch.Tensor]: assert len(set(inputs.keys()) & set(self.variable_encoders.keys())) == len(inputs) data_shape = list(inputs.values())[0].shape assert len(data_shape) == 3, "{}".format(data_shape) # ensemble-num, batch-size, specific-dim diff --git a/cmrl/models/causal_mech/base_causal_mech.py b/cmrl/models/causal_mech/base_causal_mech.py index e3704fd..cd26827 100644 --- a/cmrl/models/causal_mech/base_causal_mech.py +++ b/cmrl/models/causal_mech/base_causal_mech.py @@ -96,7 +96,7 @@ def check_coder(self): assert decoder.node_dim == self.node_dim @abstractmethod - def forward(self, inputs: MutableMapping[str, Union[torch.Tensor]]) -> Dict[str, torch.Tensor]: + def forward(self, inputs: MutableMapping[str, torch.Tensor]) -> Dict[str, torch.Tensor]: raise NotImplementedError @abstractmethod diff --git a/cmrl/models/causal_mech/plain_mech.py b/cmrl/models/causal_mech/plain_mech.py index 4c21747..0f51b2e 100644 --- a/cmrl/models/causal_mech/plain_mech.py +++ b/cmrl/models/causal_mech/plain_mech.py @@ -95,7 +95,7 @@ def build_network(self): def build_graph(self): self.graph = None - def forward(self, inputs: MutableMapping[str, Union[torch.Tensor]]) -> Dict[str, torch.Tensor]: + def forward(self, inputs: MutableMapping[str, torch.Tensor]) -> Dict[str, torch.Tensor]: assert len(set(inputs.keys()) & set(self.variable_encoders.keys())) == len(inputs) data_shape = list(inputs.values())[0].shape assert len(data_shape) == 3, "{}".format(data_shape) # ensemble-num, batch-size, specific-dim From 6a1e90ff8fe15c6ace1e65fcec0a58d549b996b4 Mon Sep 17 00:00:00 2001 From: frank Date: Mon, 7 Nov 2022 19:14:17 +0800 Subject: [PATCH 18/68] :hammer: refactor CausalMech --- cmrl/algorithms/base_algorithm.py | 98 +++++++++++ cmrl/algorithms/mbpo.py | 41 +++++ cmrl/algorithms/mopo.py | 22 +++ cmrl/algorithms/offline/__init__.py | 0 cmrl/algorithms/offline/off_dyna.py | 90 ---------- cmrl/algorithms/on_dyna.py | 34 ++++ cmrl/algorithms/online/mbpo.py | 95 ---------- cmrl/algorithms/online/on_dyna.py | 94 ---------- cmrl/examples/conf/transition/plain.yaml | 53 ++++-- cmrl/models/causal_mech/base_causal_mech.py | 165 ++++++++++-------- cmrl/models/causal_mech/plain_mech.py | 89 +++------- cmrl/models/data_loader.py | 20 --- cmrl/models/dynamics.py | 4 +- cmrl/models/networks/coder.py | 35 ++-- cmrl/models/reward_mech/__init__.py | 3 - cmrl/models/reward_mech/base_reward_mech.py | 16 -- cmrl/models/reward_mech/plain_reward_mech.py | 94 ---------- cmrl/models/termination_mech/__init__.py | 3 - .../termination_mech/base_termination_mech.py | 15 -- .../plain_termination_mech.py | 94 ---------- cmrl/models/util.py | 8 +- cmrl/utils/creator.py | 20 +-- cmrl/utils/types.py | 4 +- .../test_causal_mech/test_plain_mech.py | 5 +- tests/test_models/test_network/test_coder.py | 51 ++++-- .../test_models/test_reward_mech/__init__.py | 0 .../test_plain_reward_mech.py | 30 ---- .../test_termination_mech/__init__.py | 0 .../test_plain_termination_mech.py | 0 tests/test_models/test_transition/__init__.py | 0 .../test_transition/test_base_transition.py | 25 --- .../test_multi_step/__init__.py | 0 .../test_multi_step/test_forward_euler.py | 51 ------ .../test_transition/test_one_step/__init__.py | 0 .../test_one_step/test_basic_ensemble.py | 72 -------- .../test_external_mask_ensemble.py | 93 ---------- 36 files changed, 425 insertions(+), 999 deletions(-) create mode 100644 cmrl/algorithms/base_algorithm.py create mode 100644 cmrl/algorithms/mbpo.py create mode 100644 cmrl/algorithms/mopo.py delete mode 100644 cmrl/algorithms/offline/__init__.py delete mode 100644 cmrl/algorithms/offline/off_dyna.py create mode 100644 cmrl/algorithms/on_dyna.py delete mode 100644 cmrl/algorithms/online/mbpo.py delete mode 100644 cmrl/algorithms/online/on_dyna.py delete mode 100644 cmrl/models/reward_mech/__init__.py delete mode 100644 cmrl/models/reward_mech/base_reward_mech.py delete mode 100644 cmrl/models/reward_mech/plain_reward_mech.py delete mode 100644 cmrl/models/termination_mech/__init__.py delete mode 100644 cmrl/models/termination_mech/base_termination_mech.py delete mode 100644 cmrl/models/termination_mech/plain_termination_mech.py delete mode 100644 tests/test_models/test_reward_mech/__init__.py delete mode 100644 tests/test_models/test_reward_mech/test_plain_reward_mech.py delete mode 100644 tests/test_models/test_termination_mech/__init__.py delete mode 100644 tests/test_models/test_termination_mech/test_plain_termination_mech.py delete mode 100644 tests/test_models/test_transition/__init__.py delete mode 100644 tests/test_models/test_transition/test_base_transition.py delete mode 100644 tests/test_models/test_transition/test_multi_step/__init__.py delete mode 100644 tests/test_models/test_transition/test_multi_step/test_forward_euler.py delete mode 100644 tests/test_models/test_transition/test_one_step/__init__.py delete mode 100644 tests/test_models/test_transition/test_one_step/test_basic_ensemble.py delete mode 100644 tests/test_models/test_transition/test_one_step/test_external_mask_ensemble.py diff --git a/cmrl/algorithms/base_algorithm.py b/cmrl/algorithms/base_algorithm.py new file mode 100644 index 0000000..b1fc537 --- /dev/null +++ b/cmrl/algorithms/base_algorithm.py @@ -0,0 +1,98 @@ +import os +from typing import Optional +from functools import partial + +import numpy as np +import torch +from omegaconf import DictConfig +from stable_baselines3.common.buffers import ReplayBuffer +from stable_baselines3.common.callbacks import BaseCallback + +from cmrl.models.util import load_offline_data +from cmrl.models.fake_env import VecFakeEnv +from cmrl.sb3_extension.logger import configure as logger_configure +from cmrl.sb3_extension.eval_callback import EvalCallback +from cmrl.utils.creator import create_dynamics, create_agent +from cmrl.utils.env import make_env + + +class BaseAlgorithm: + def __init__( + self, + cfg: DictConfig, + work_dir: Optional[str] = None, + ): + self.cfg = cfg + self.work_dir = work_dir or os.getcwd() + + self.env, self.reward_fn, self.termination_fn, self.get_init_obs_fn = make_env(cfg) + self.eval_env, *_ = make_env(cfg) + np.random.seed(cfg.seed) + torch.manual_seed(cfg.seed) + + self.logger = logger_configure("log", ["tensorboard", "multi_csv", "stdout"]) + + # create ``cmrl`` dynamics + self.dynamics = create_dynamics(cfg, self.env.observation_space, self.env.action_space, logger=self.logger) + + # create sb3's replay buffer for real offline data + self.real_replay_buffer = ReplayBuffer( + cfg.task.num_steps, self.env.observation_space, self.env.action_space, cfg.device, handle_timeout_termination=False + ) + + self.partial_fake_env = partial( + VecFakeEnv, + cfg.algorithm.num_envs, + self.env.observation_space, + self.env.action_space, + self.dynamics, + self.reward_fn, + self.termination_fn, + self.get_init_obs_fn, + self.real_replay_buffer, + penalty_coeff=cfg.task.penalty_coeff, + logger=self.logger, + ) + + self.fake_env = self.get_fake_env() + + self.agent = create_agent(cfg, self.fake_env, self.logger) + + self.callback = self.get_callback() + + def get_fake_env(self) -> VecFakeEnv: + return self.partial_fake_env( + deterministic=self.cfg.algorithm.deterministic, + max_episode_steps=self.env.spec.max_episode_steps, + branch_rollout=False, + ) + + def get_callback(self) -> BaseCallback: + fake_eval_env = self.partial_fake_env( + deterministic=True, max_episode_steps=self.env.spec.max_episode_steps, branch_rollout=False + ) + return EvalCallback( + self.eval_env, + fake_eval_env, + n_eval_episodes=self.cfg.task.n_eval_episodes, + best_model_save_path="./", + eval_freq=1000, + deterministic=True, + render=False, + ) + + def learn(self): + self._setup_learn() + + self.dynamics.learn( + real_replay_buffer=self.real_replay_buffer, + longest_epoch=self.cfg.task.longest_epoch, + improvement_threshold=self.cfg.task.improvement_threshold, + patience=self.cfg.task.patience, + work_dir=self.work_dir, + ) + + self.agent.learn(total_timesteps=self.cfg.task.num_steps, callback=self.callback) + + def _setup_learn(self): + load_offline_data(self.env, self.real_replay_buffer, self.cfg.task.dataset, self.cfg.task.use_ratio) diff --git a/cmrl/algorithms/mbpo.py b/cmrl/algorithms/mbpo.py new file mode 100644 index 0000000..8c8189f --- /dev/null +++ b/cmrl/algorithms/mbpo.py @@ -0,0 +1,41 @@ +from typing import Optional + +from omegaconf import DictConfig +from stable_baselines3.common.callbacks import BaseCallback, CallbackList + +from cmrl.models.fake_env import VecFakeEnv +from cmrl.algorithms.base_algorithm import BaseAlgorithm +from cmrl.sb3_extension.online_mb_callback import OnlineModelBasedCallback + + +class MBPO(BaseAlgorithm): + def __init__( + self, + cfg: DictConfig, + work_dir: Optional[str] = None, + ): + super(MBPO, self).__init__(cfg, work_dir) + + def get_fake_env(self) -> VecFakeEnv: + return self.partial_fake_env( + deterministic=self.cfg.algorithm.deterministic, + max_episode_steps=self.cfg.algorithm.branch_rollout_length, + branch_rollout=True, + ) + + def get_callback(self) -> BaseCallback: + eval_callback = super(MBPO, self).get_callback() + omb_callback = OnlineModelBasedCallback( + self.env, + self.dynamics, + self.real_replay_buffer, + total_num_steps=self.cfg.task.online_num_steps, + initial_exploration_steps=self.cfg.algorithm.initial_exploration_steps, + freq_train_model=self.cfg.task.freq_train_model, + device=self.cfg.device, + ) + + return CallbackList([eval_callback, omb_callback]) + + def _setup_learn(self): + pass diff --git a/cmrl/algorithms/mopo.py b/cmrl/algorithms/mopo.py new file mode 100644 index 0000000..70b02d7 --- /dev/null +++ b/cmrl/algorithms/mopo.py @@ -0,0 +1,22 @@ +from typing import Optional + +from omegaconf import DictConfig + +from cmrl.models.fake_env import VecFakeEnv +from cmrl.algorithms.base_algorithm import BaseAlgorithm + + +class MOPO(BaseAlgorithm): + def __init__( + self, + cfg: DictConfig, + work_dir: Optional[str] = None, + ): + super(MOPO, self).__init__(cfg, work_dir) + + def get_fake_env(self) -> VecFakeEnv: + return self.partial_fake_env( + deterministic=self.cfg.algorithm.deterministic, + max_episode_steps=self.cfg.algorithm.branch_rollout_length, + branch_rollout=True, + ) diff --git a/cmrl/algorithms/offline/__init__.py b/cmrl/algorithms/offline/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/cmrl/algorithms/offline/off_dyna.py b/cmrl/algorithms/offline/off_dyna.py deleted file mode 100644 index 5329c13..0000000 --- a/cmrl/algorithms/offline/off_dyna.py +++ /dev/null @@ -1,90 +0,0 @@ -import os -from typing import Optional -from functools import partial - -import emei -from omegaconf import DictConfig -from stable_baselines3.common.buffers import ReplayBuffer - -from cmrl.models.util import load_offline_data - -# from cmrl.models.dynamics import ConstraintBasedDynamics -from cmrl.sb3_extension.eval_callback import EvalCallback -from cmrl.models.fake_env import VecFakeEnv -from cmrl.sb3_extension.logger import configure as logger_configure -from cmrl.utils.types import InitObsFnType, RewardFnType, TermFnType -from cmrl.utils.creator import create_dynamics, create_agent - - -def train( - env: emei.EmeiEnv, - eval_env: emei.EmeiEnv, - reward_fn: Optional[RewardFnType], - termination_fn: Optional[TermFnType], - get_init_obs_fn: Optional[InitObsFnType], - cfg: DictConfig, - work_dir: Optional[str] = None, -): - ######################################### - # create class - ######################################### - work_dir = work_dir or os.getcwd() - logger = logger_configure("log", ["tensorboard", "multi_csv", "stdout"]) - - # create ``cmrl`` dynamics - dynamics = create_dynamics(cfg, env.observation_space, env.action_space, logger=logger) - - # create sb3's replay buffer for real offline data - real_replay_buffer = ReplayBuffer( - cfg.task.num_steps, env.observation_space, env.action_space, cfg.device, handle_timeout_termination=False - ) - - partial_fake_env = partial( - VecFakeEnv, - cfg.algorithm.num_envs, - env.observation_space, - env.action_space, - dynamics, - reward_fn, - termination_fn, - get_init_obs_fn, - real_replay_buffer, - penalty_coeff=cfg.task.penalty_coeff, - logger=logger, - ) - fake_env = partial_fake_env( - deterministic=cfg.algorithm.deterministic, max_episode_steps=env.spec.max_episode_steps, branch_rollout=False - ) - fake_eval_env = partial_fake_env(deterministic=True, max_episode_steps=env.spec.max_episode_steps, branch_rollout=False) - - # create sb3's agent - agent = create_agent(cfg, fake_env, logger) - - ######################################### - # learn - ######################################### - load_offline_data(env, real_replay_buffer, cfg.task.dataset, cfg.task.use_ratio) - - # if hasattr(env, "get_causal_graph"): - # oracle_causal_graph = env.get_causal_graph() - # else: - # oracle_causal_graph = None - # - # if isinstance(dynamics, ConstraintBasedDynamics): - # dynamics.set_oracle_mask("transition", oracle_causal_graph.T) - # - # existed_trained_model = maybe_load_trained_offline_model(dynamics, cfg, obs_shape, act_shape, work_dir=work_dir) - # if not existed_trained_model: - dynamics.learn(real_replay_buffer, **cfg.dynamics, work_dir=work_dir) - - eval_callback = EvalCallback( - eval_env, - fake_eval_env, - n_eval_episodes=cfg.task.n_eval_episodes, - best_model_save_path="./", - eval_freq=1000, - deterministic=True, - render=False, - ) - - agent.learn(total_timesteps=cfg.task.num_steps, callback=eval_callback) diff --git a/cmrl/algorithms/on_dyna.py b/cmrl/algorithms/on_dyna.py new file mode 100644 index 0000000..380ebec --- /dev/null +++ b/cmrl/algorithms/on_dyna.py @@ -0,0 +1,34 @@ +from typing import Optional + +from omegaconf import DictConfig +from stable_baselines3.common.callbacks import BaseCallback, CallbackList + +from cmrl.models.fake_env import VecFakeEnv +from cmrl.algorithms.base_algorithm import BaseAlgorithm +from cmrl.sb3_extension.online_mb_callback import OnlineModelBasedCallback + + +class OnlineDyna(BaseAlgorithm): + def __init__( + self, + cfg: DictConfig, + work_dir: Optional[str] = None, + ): + super(OnlineDyna, self).__init__(cfg, work_dir) + + def get_callback(self) -> BaseCallback: + eval_callback = super(OnlineDyna, self).get_callback() + omb_callback = OnlineModelBasedCallback( + self.env, + self.dynamics, + self.real_replay_buffer, + total_num_steps=self.cfg.task.online_num_steps, + initial_exploration_steps=self.cfg.algorithm.initial_exploration_steps, + freq_train_model=self.cfg.task.freq_train_model, + device=self.cfg.device, + ) + + return CallbackList([eval_callback, omb_callback]) + + def _setup_learn(self): + pass diff --git a/cmrl/algorithms/online/mbpo.py b/cmrl/algorithms/online/mbpo.py deleted file mode 100644 index a9d8abc..0000000 --- a/cmrl/algorithms/online/mbpo.py +++ /dev/null @@ -1,95 +0,0 @@ -import os -from typing import Optional, cast - -import emei -import hydra.utils -import numpy as np -from omegaconf import DictConfig -from stable_baselines3.common.base_class import BaseAlgorithm -from stable_baselines3.common.callbacks import CallbackList -from stable_baselines3.common.buffers import ReplayBuffer - -from cmrl.agent import complete_agent_cfg -from cmrl.algorithms.util import setup_fake_env - -# from cmrl.models.dynamics import ConstraintBasedDynamics -from cmrl.sb3_extension.eval_callback import EvalCallback -from cmrl.sb3_extension.online_mb_callback import OnlineModelBasedCallback -from cmrl.sb3_extension.logger import configure as logger_configure -from cmrl.utils.types import InitObsFnType, RewardFnType, TermFnType -from cmrl.utils.creator import create_dynamics - - -def train( - env: emei.EmeiEnv, - eval_env: emei.EmeiEnv, - termination_fn: Optional[TermFnType], - reward_fn: Optional[RewardFnType], - get_init_obs_fn: Optional[InitObsFnType], - cfg: DictConfig, - work_dir: Optional[str] = None, -): - obs_shape = env.observation_space.shape - act_shape = env.action_space.shape - - # build model-free agent, which is a stable-baselines3's agent - complete_agent_cfg(env, cfg.algorithm.agent) - agent = cast(BaseAlgorithm, hydra.utils.instantiate(cfg.algorithm.agent)) - - work_dir = work_dir or os.getcwd() - logger = logger_configure("log", ["tensorboard", "multi_csv", "stdout"]) - - numpy_generator = np.random.default_rng(seed=cfg.seed) - - dynamics = create_dynamics(cfg.dynamics, obs_shape, act_shape, logger=logger) - real_replay_buffer = ReplayBuffer( - cfg.task.online_num_steps, - env.observation_space, - env.action_space, - device=cfg.device, - n_envs=1, - optimize_memory_usage=False, - ) - - fake_eval_env = setup_fake_env( - cfg=cfg, - agent=agent, - dynamics=dynamics, - reward_fn=reward_fn, - termination_fn=termination_fn, - get_init_obs_fn=get_init_obs_fn, - real_replay_buffer=real_replay_buffer, - logger=logger, - max_episode_steps=env.spec.max_episode_steps, - ) - - if hasattr(env, "causal_graph"): - oracle_causal_graph = env.causal_graph - else: - oracle_causal_graph = None - - if isinstance(dynamics, ConstraintBasedDynamics): - dynamics.set_oracle_mask("transition", oracle_causal_graph.T) - - eval_callback = EvalCallback( - eval_env, - fake_eval_env, - n_eval_episodes=cfg.task.n_eval_episodes, - best_model_save_path="./", - eval_freq=cfg.task.eval_freq, - deterministic=True, - render=False, - ) - - omb_callback = OnlineModelBasedCallback( - env, - dynamics, - real_replay_buffer, - total_num_steps=cfg.task.online_num_steps, - initial_exploration_steps=cfg.algorithm.initial_exploration_steps, - freq_train_model=cfg.task.freq_train_model, - device=cfg.device, - ) - - agent.set_logger(logger) - agent.learn(total_timesteps=int(1e10), callback=CallbackList([eval_callback, omb_callback])) diff --git a/cmrl/algorithms/online/on_dyna.py b/cmrl/algorithms/online/on_dyna.py deleted file mode 100644 index aa5539a..0000000 --- a/cmrl/algorithms/online/on_dyna.py +++ /dev/null @@ -1,94 +0,0 @@ -import os -from typing import Optional, cast - -import emei -import hydra.utils -import numpy as np -from omegaconf import DictConfig -from stable_baselines3.common.base_class import BaseAlgorithm -from stable_baselines3.common.callbacks import CallbackList -from stable_baselines3.common.buffers import ReplayBuffer - -from cmrl.agent import complete_agent_cfg -from cmrl.algorithms.util import setup_fake_env - -# from cmrl.models.dynamics import ConstraintBasedDynamics -from cmrl.sb3_extension.eval_callback import EvalCallback -from cmrl.sb3_extension.online_mb_callback import OnlineModelBasedCallback -from cmrl.sb3_extension.logger import configure as logger_configure -from cmrl.utils.types import InitObsFnType, RewardFnType, TermFnType -from cmrl.utils.creator import create_dynamics - - -def train( - env: emei.EmeiEnv, - eval_env: emei.EmeiEnv, - termination_fn: Optional[TermFnType], - reward_fn: Optional[RewardFnType], - get_init_obs_fn: Optional[InitObsFnType], - cfg: DictConfig, - work_dir: Optional[str] = None, -): - obs_shape = env.observation_space.shape - act_shape = env.action_space.shape - - # build model-free agent, which is a stable-baselines3's agent - complete_agent_cfg(env, cfg.algorithm.agent) - agent = cast(BaseAlgorithm, hydra.utils.instantiate(cfg.algorithm.agent)) - - work_dir = work_dir or os.getcwd() - logger = logger_configure("log", ["tensorboard", "multi_csv", "stdout"]) - - numpy_generator = np.random.default_rng(seed=cfg.seed) - - dynamics = create_dynamics(cfg.dynamics, obs_shape, act_shape, logger=logger) - real_replay_buffer = ReplayBuffer( - cfg.task.online_num_steps, - env.observation_space, - env.action_space, - device=cfg.device, - n_envs=1, - optimize_memory_usage=False, - ) - - fake_eval_env = setup_fake_env( - cfg=cfg, - agent=agent, - dynamics=dynamics, - reward_fn=reward_fn, - termination_fn=termination_fn, - get_init_obs_fn=get_init_obs_fn, - logger=logger, - max_episode_steps=env.spec.max_episode_steps, - ) - - if hasattr(env, "causal_graph"): - oracle_causal_graph = env.causal_graph - else: - oracle_causal_graph = None - - if isinstance(dynamics, ConstraintBasedDynamics): - dynamics.set_oracle_mask("transition", oracle_causal_graph.T) - - eval_callback = EvalCallback( - eval_env, - fake_eval_env, - n_eval_episodes=cfg.task.n_eval_episodes, - best_model_save_path="./", - eval_freq=cfg.task.eval_freq, - deterministic=True, - render=False, - ) - - omb_callback = OnlineModelBasedCallback( - env, - dynamics, - real_replay_buffer, - total_num_steps=cfg.task.online_num_steps, - initial_exploration_steps=cfg.algorithm.initial_exploration_steps, - freq_train_model=cfg.task.freq_train_model, - device=cfg.device, - ) - - agent.set_logger(logger) - agent.learn(total_timesteps=int(1e10), callback=CallbackList([eval_callback, omb_callback])) diff --git a/cmrl/examples/conf/transition/plain.yaml b/cmrl/examples/conf/transition/plain.yaml index 1411538..7cbfd9c 100644 --- a/cmrl/examples/conf/transition/plain.yaml +++ b/cmrl/examples/conf/transition/plain.yaml @@ -1,33 +1,56 @@ name: "plain_transition" learn: true +encoder_cfg: + _partial_: true + _recursive_: false + _target_: cmrl.models.networks.coder.VariableEncoder + output_dim: 100 + hidden_dims: [ 100 ] + bias: true + activation_fn_cfg: + _target_: torch.nn.SiLU + +decoder_cfg: + _partial_: true + _recursive_: false + _target_: cmrl.models.networks.coder.VariableDecoder + identity: true + +network_cfg: + _partial_: true + _recursive_: false + _target_: cmrl.models.networks.parallel_mlp.ParallelMLP + hidden_dims: [ 200, 200 ] + use_bias: true + activation_fn_cfg: + _target_: torch.nn.SiLU + +optimizer_cfg: + _partial_: true + _target_: torch.optim.Adam + lr: 1e-4 + weight_decay: 1e-5 + eps: 1e-8 + mech: _partial_: true _recursive_: false _target_: cmrl.models.causal_mech.PlainMech - name: ${transition.name} # base causal-mech params + name: ${transition.name} input_variables: ??? output_variables: ??? - node_dim: 20 - variable_encoders: ??? - variable_decoders: ??? - # network params - deterministic: false - hidden_dims: [ 200, 200, 200, 200 ] ensemble_num: 7 elite_num: 5 - use_bias: true - activation_fn_cfg: - _target_: torch.nn.SiLU + # cfgs + network_cfg: ${transition.network_cfg} + encoder_cfg: ${transition.encoder_cfg} + decoder_cfg: ${transition.decoder_cfg} + optimizer_cfg: ${transition.optimizer_cfg} # forward method residual: true multi_step: "none" - # trainer - optim_lr: 1e-4 - optim_weight_decay: 1e-5 - optim_eps: 1e-8 - optim_encoder: true # logger logger: ??? # others diff --git a/cmrl/models/causal_mech/base_causal_mech.py b/cmrl/models/causal_mech/base_causal_mech.py index e3704fd..9fb6dae 100644 --- a/cmrl/models/causal_mech/base_causal_mech.py +++ b/cmrl/models/causal_mech/base_causal_mech.py @@ -1,12 +1,15 @@ from typing import Optional, List, Dict, Union, MutableMapping from abc import abstractmethod +from itertools import chain import torch import numpy as np from torch.utils.data import DataLoader import torch.nn.functional as F -from torch.optim import Optimizer +from torch.optim import Optimizer, Adam from stable_baselines3.common.logger import Logger +from omegaconf import DictConfig +from hydra.utils import instantiate from cmrl.models.networks.base_network import BaseNetwork from cmrl.models.graphs.base_graph import BaseGraph @@ -21,19 +24,17 @@ def __init__( name: str, input_variables: List[Variable], output_variables: List[Variable], - node_dim: int, - variable_encoders: Optional[Dict[str, VariableEncoder]], - variable_decoders: Optional[Dict[str, VariableDecoder]], ensemble_num: int = 7, elite_num: int = 5, + # cfgs + network_cfg: Optional[DictConfig] = None, + encoder_cfg: Optional[DictConfig] = None, + decoder_cfg: Optional[DictConfig] = None, + optimizer_cfg: Optional[DictConfig] = None, # forward method residual: bool = True, + encoder_reduction: str = "sum", multi_step: str = "none", - # trainer - optim_lr: float = 1e-4, - optim_weight_decay: float = 1e-5, - optim_eps: float = 1e-8, - optim_encoder: bool = True, # logger logger: Optional[Logger] = None, # others @@ -43,19 +44,17 @@ def __init__( self.name = name self.input_variables = input_variables self.output_variables = output_variables - self.node_dim = node_dim - self.variable_encoders = variable_encoders - self.variable_decoders = variable_decoders self.ensemble_num = ensemble_num self.elite_num = elite_num + # cfgs + self.network_cfg = network_cfg + self.encoder_cfg = encoder_cfg + self.decoder_cfg = decoder_cfg + self.optimizer_cfg = optimizer_cfg # forward method self.residual = residual + self.encoder_reduction = encoder_reduction self.multi_step = multi_step - # trainer - self.optim_lr = optim_lr - self.optim_weight_decay = optim_weight_decay - self.optim_eps = optim_eps - self.optim_encoder = optim_encoder # logger self.logger = logger # others @@ -64,40 +63,40 @@ def __init__( self.input_var_num = len(self.input_variables) self.output_var_num = len(self.output_variables) - if self.variable_encoders is None: - assert self.optim_encoder - self.variable_encoders = create_encoders(input_variables, node_dim=self.node_dim, device=self.device) - if self.variable_decoders is None: - self.variable_decoders = create_decoders(output_variables, node_dim=self.node_dim, device=self.device) - self.check_coder() - + self.variable_encoders: Optional[Dict[str, VariableEncoder]] = None + self.variable_decoders: Optional[Dict[str, VariableEncoder]] = None self.network: Optional[BaseNetwork] = None - self.optimizer: Optional[Optimizer] = None self.graph: Optional[BaseGraph] = None + self.optimizer: Optional[Optimizer] = None + self.build_coder() self.build_network() self.build_graph() + self.build_optimizer() self.total_epoch = 0 self.elite_indices: List[int] = [] - def check_coder(self): - assert len(self.input_variables) == len(self.variable_encoders) - assert len(self.output_variables) == len(self.variable_decoders) + @abstractmethod + def single_step_forward(self, inputs: MutableMapping[str, torch.Tensor]) -> Dict[str, torch.Tensor]: + raise NotImplementedError - for var in self.input_variables: - assert var.name in self.variable_encoders - encoder = self.variable_encoders[var.name] - assert encoder.node_dim == self.node_dim + def forward(self, inputs: MutableMapping[str, torch.Tensor]) -> Dict[str, torch.Tensor]: + if self.multi_step.startswith("forward-euler"): + step_num = int(self.multi_step.split()[-1]) - for var in self.output_variables: - assert var.name in self.variable_decoders - decoder = self.variable_decoders[var.name] - assert decoder.node_dim == self.node_dim + outputs = {} + for step in range(step_num): + outputs = self.single_step_forward(inputs) + if step < step_num - 1: + for name in filter(lambda s: s.startswith("obs"), inputs.keys()): + assert inputs[name].shape[:2] == outputs["next_{}".format(name)].shape[:2] + assert inputs[name].shape[2] * 2 == outputs["next_{}".format(name)].shape[2] + inputs[name] = outputs["next_{}".format(name)][:, :, : inputs[name].shape[2]] + else: + raise NotImplementedError("multi-step method {} is not supported".format(self.multi_step)) - @abstractmethod - def forward(self, inputs: MutableMapping[str, Union[torch.Tensor]]) -> Dict[str, torch.Tensor]: - raise NotImplementedError + return outputs @abstractmethod def learn( @@ -113,10 +112,33 @@ def learn( def build_network(self): raise NotImplementedError + def build_optimizer(self): + assert self.network is not None, "you must build network first" + params = ( + [self.network.parameters()] + + [encoder.parameters() for encoder in self.variable_encoders.values()] + + [decoder.parameters() for decoder in self.variable_decoders.values()] + ) + + self.optimizer = instantiate(self.optimizer_cfg)(params=chain(*params)) + @abstractmethod def build_graph(self): raise NotImplementedError + def build_coder(self): + self.variable_encoders = {} + for var in self.input_variables: + assert var.name not in self.variable_encoders, "duplicate name in encoders: {}".format(var.name) + self.variable_encoders[var.name] = instantiate(self.encoder_cfg)(variable=var).to(self.device) + + assert self.decoder_input_dim + + self.variable_decoders = {} + for var in self.output_variables: + assert var.name not in self.variable_decoders, "duplicate name in decoders: {}".format(var.name) + self.variable_decoders[var.name] = instantiate(self.decoder_cfg)(variable=var).to(self.device) + def loss(self, outputs, targets): ensemble_num, batch_size = list(targets.values())[0].shape[:2] total_loss = torch.zeros(ensemble_num, batch_size, self.output_var_num) @@ -139,36 +161,37 @@ def loss(self, outputs, targets): raise NotImplementedError return total_loss + @property + def encoder_output_dim(self): + return self.encoder_cfg.output_dim -# Causal = TypeVar("Causal", bound=BaseCausalMech) -# -# -# class BaseMultiStepCausalMech(BaseCausalMech): -# def __init__( -# self, -# single_step_mech_class: Type[Causal], -# input_variables: List[Variable], -# output_variables: List[Variable], -# node_dim: int, -# variable_encoders: Dict[str, VariableEncoder], -# variable_decoders: Dict[str, VariableDecoder], -# **kwargs -# ): -# super(BaseMultiStepCausalMech, self).__init__( -# input_variables=input_variables, -# output_variables=output_variables, -# node_dim=node_dim, -# variable_encoders=variable_encoders, -# variable_decoders=variable_decoders, -# ) -# -# self.single_step_mech = single_step_mech_class(**kwargs) -# pass -# -# @abstractmethod -# def build_network(self): -# raise NotImplementedError -# -# @abstractmethod -# def build_graph(self): -# raise NotImplementedError + @property + def union_output_var_dim(self): + # all output variables should be ContinuousVariable and have same variable.dim + output_dim = [] + for var in self.output_variables: + assert isinstance(var, ContinuousVariable), "all output variables should be ContinuousVariable" + output_dim.append(var.dim) + assert len(set(output_dim)) == 1, "all output variables should have same variable.dim" + return output_dim[0] + + @property + def decoder_input_dim(self): + if self.decoder_cfg.identity: + return self.union_output_var_dim * 2 + else: + return self.decoder_cfg.input_dim + + def reduce_encoder_output(self, encoder_output: torch.Tensor) -> torch.Tensor: + assert len(encoder_output.shape) == 4, ( + "shape of encoder_output should be (ensemble-num, batch-size, input-cvar-num, encoder-output-dim), " + "rather than {}".format(encoder_output.shape) + ) + if self.encoder_reduction == "sum": + return encoder_output.sum(-2) + elif self.encoder_reduction == "mean": + return encoder_output.mean(-2) + elif self.encoder_reduction == "sum": + return encoder_output.sum(-2) + else: + raise NotImplementedError("not implemented encoder reduction method: {}".format(self.encoder_reduction)) diff --git a/cmrl/models/causal_mech/plain_mech.py b/cmrl/models/causal_mech/plain_mech.py index 4c21747..df88c32 100644 --- a/cmrl/models/causal_mech/plain_mech.py +++ b/cmrl/models/causal_mech/plain_mech.py @@ -8,6 +8,7 @@ from torch.utils.data import DataLoader from torch.optim import Adam from omegaconf import DictConfig +from hydra.utils import instantiate from stable_baselines3.common.logger import Logger from cmrl.utils.types import Variable @@ -20,38 +21,25 @@ class PlainMech(BaseCausalMech): def __init__( self, name: str, - # base causal-mech params input_variables: List[Variable], output_variables: List[Variable], - node_dim: int, - variable_encoders: Optional[Dict[str, VariableEncoder]], - variable_decoders: Optional[Dict[str, VariableDecoder]], ensemble_num: int = 7, elite_num: int = 5, - # network params - deterministic: bool = False, - hidden_dims: Optional[List[int]] = None, - use_bias: bool = True, - activation_fn_cfg: Optional[DictConfig] = None, + # cfgs + network_cfg: Optional[DictConfig] = None, + encoder_cfg: Optional[DictConfig] = None, + decoder_cfg: Optional[DictConfig] = None, + optimizer_cfg: Optional[DictConfig] = None, # forward method residual: bool = True, + encoder_reduction: str = "sum", multi_step: str = "none", - # trainer - optim_lr: float = 1e-4, - optim_weight_decay: float = 1e-5, - optim_eps: float = 1e-8, - optim_encoder: bool = True, # logger logger: Optional[Logger] = None, # others device: Union[str, torch.device] = "cpu", **kwargs ): - self.deterministic = deterministic - self.hidden_dims = hidden_dims if hidden_dims is not None else [200] * 4 - self.use_bias = use_bias - self.activation_fn_cfg = activation_fn_cfg - if multi_step == "none": multi_step = "forward-euler 1" @@ -59,74 +47,53 @@ def __init__( name=name, input_variables=input_variables, output_variables=output_variables, - node_dim=node_dim, - variable_encoders=variable_encoders, - variable_decoders=variable_decoders, ensemble_num=ensemble_num, elite_num=elite_num, + network_cfg=network_cfg, + encoder_cfg=encoder_cfg, + decoder_cfg=decoder_cfg, + optimizer_cfg=optimizer_cfg, residual=residual, + encoder_reduction=encoder_reduction, multi_step=multi_step, - optim_lr=optim_lr, - optim_weight_decay=optim_weight_decay, - optim_eps=optim_eps, - optim_encoder=optim_encoder, logger=logger, device=device, **kwargs ) def build_network(self): - self.network = ParallelMLP( - input_dim=self.input_var_num * self.node_dim, - output_dim=self.output_var_num * self.node_dim, - hidden_dims=self.hidden_dims, - use_bias=self.use_bias, + self.network = instantiate(self.network_cfg)( + input_dim=self.encoder_output_dim, + output_dim=self.output_var_num * self.decoder_input_dim, extra_dims=[self.ensemble_num], - activation_fn_cfg=self.activation_fn_cfg, ).to(self.device) - parmas = [self.network.parameters()] + [decoder.parameters() for decoder in self.variable_decoders.values()] - if self.optim_encoder: - parmas.extend([encoder.parameters() for encoder in self.variable_encoders.values()]) - self.optimizer = Adam( - itertools.chain(*parmas), lr=self.optim_lr, weight_decay=self.optim_weight_decay, eps=self.optim_eps - ) - def build_graph(self): self.graph = None - def forward(self, inputs: MutableMapping[str, Union[torch.Tensor]]) -> Dict[str, torch.Tensor]: + def single_step_forward(self, inputs: MutableMapping[str, torch.Tensor]) -> Dict[str, torch.Tensor]: assert len(set(inputs.keys()) & set(self.variable_encoders.keys())) == len(inputs) data_shape = list(inputs.values())[0].shape assert len(data_shape) == 3, "{}".format(data_shape) # ensemble-num, batch-size, specific-dim ensemble, batch_size, specific_dim = data_shape assert ensemble == self.ensemble_num - inputs_tensor = torch.zeros(ensemble, batch_size, self.input_var_num * self.node_dim).to(self.device) + inputs_tensor = torch.zeros(ensemble, batch_size, self.input_var_num, self.encoder_output_dim).to(self.device) for i, var in enumerate(self.input_variables): - out = self.variable_encoders[var.name](inputs[var.name].to(self.device)) # ensemble-num, batch-size, node-dim - inputs_tensor[:, :, i * self.node_dim : (i + 1) * self.node_dim] = out - - if self.multi_step.startswith("forward-euler"): - step_num = int(self.multi_step.split()[-1]) - output_tensor = None - for step in range(step_num): - if step > 0: - inputs_tensor = torch.concat( - [output_tensor, inputs_tensor[:, :, self.output_var_num * self.node_dim :]], dim=-1 - ) - output_tensor = self.network(inputs_tensor) - if self.residual: - output_tensor += inputs_tensor[:, :, : self.output_var_num * self.node_dim] - else: - raise NotImplementedError + out = self.variable_encoders[var.name](inputs[var.name].to(self.device)) + inputs_tensor[:, :, i] = out + output_tensor = self.network(self.reduce_encoder_output(inputs_tensor)) outputs = {} for i, var in enumerate(self.output_variables): - hid = output_tensor[:, :, i * self.node_dim : (i + 1) * self.node_dim] - out = self.variable_decoders[var.name](hid) - outputs[var.name] = out - + hid = output_tensor[:, :, i * self.decoder_input_dim : (i + 1) * self.decoder_input_dim] + outputs[var.name] = self.variable_decoders[var.name](hid) + + if self.residual: + for name in filter(lambda s: s.startswith("obs"), inputs.keys()): + assert inputs[name].shape[:2] == outputs["next_{}".format(name)].shape[:2] + assert inputs[name].shape[2] * 2 == outputs["next_{}".format(name)].shape[2] + outputs["next_{}".format(name)][:, :, : inputs[name].shape[2]] += inputs[name].to(self.device) return outputs def train(self, loader: DataLoader): diff --git a/cmrl/models/data_loader.py b/cmrl/models/data_loader.py index 552e7e0..be45548 100644 --- a/cmrl/models/data_loader.py +++ b/cmrl/models/data_loader.py @@ -56,26 +56,6 @@ def build_indexes(self): self.indexes = permutation[: int(self.size * self.train_ratio)] def load_from_buffer(self): - # if isinstance(self.replay_buffer, DictReplayBuffer): - # # TODO: DictReplayBuffer case - # raise NotImplementedError - # else: - # observations = self.replay_buffer.observations[: self.size, 0].astype(np.float32) - # assert len(observations.shape) == 2 - # next_observations = self.replay_buffer.next_observations[: self.size, 0].astype(np.float32) - # - # observations_dict = dict([("obs_{}".format(i), obs[:, None]) for i, obs in enumerate(observations.T)]) - # next_observations_dict = dict( - # [("next_obs_{}".format(i), obs[:, None]) for i, obs in enumerate(next_observations.T)] - # ) - - # assert isinstance(self.observation_space, spaces.Box) - # # TODO: other spaces for observation and action(e.g. one-hot for spaces.Discrete) - # # see: https://github.com/DLR-RM/stable-baselines3/blob/master/stable_baselines3/common/preprocessing.py#L85 - # - # actions = self.replay_buffer.actions[: self.size, 0] - # actions_dict = dict([("act_{}".format(i), obs[:, None]) for i, obs in enumerate(actions.T)]) - obs_dict = space2dict(self.replay_buffer.observations[: self.size, 0], self.observation_space, "obs") act_dict = space2dict(self.replay_buffer.actions[: self.size, 0], self.action_space, "act") next_obs_dict = space2dict(self.replay_buffer.next_observations[: self.size, 0], self.observation_space, "next_obs") diff --git a/cmrl/models/dynamics.py b/cmrl/models/dynamics.py index d0e51e1..ec7b656 100644 --- a/cmrl/models/dynamics.py +++ b/cmrl/models/dynamics.py @@ -48,7 +48,7 @@ def get_loader(self, real_replay_buffer, mech: str, batch_size: int = 128): train_ensemble=True, ensemble_num=self.transition.ensemble_num, ) - train_loader = DataLoader(train_dataset, batch_size=8, collate_fn=collate_fn) + train_loader = DataLoader(train_dataset, batch_size=batch_size, collate_fn=collate_fn) valid_dataset = BufferDataset( real_replay_buffer, self.observation_space, @@ -57,7 +57,7 @@ def get_loader(self, real_replay_buffer, mech: str, batch_size: int = 128): mech=mech, repeat=self.transition.ensemble_num, ) - valid_loader = DataLoader(valid_dataset, batch_size=8, collate_fn=collate_fn) + valid_loader = DataLoader(valid_dataset, batch_size=batch_size, collate_fn=collate_fn) return train_loader, valid_loader diff --git a/cmrl/models/networks/coder.py b/cmrl/models/networks/coder.py index 64c8e87..f9635a9 100644 --- a/cmrl/models/networks/coder.py +++ b/cmrl/models/networks/coder.py @@ -11,25 +11,26 @@ class VariableEncoder(BaseNetwork): def __init__( self, variable: Variable, - node_dim: int, + output_dim: int = 100, hidden_dims: Optional[List[int]] = None, bias: bool = True, activation_fn_cfg: Optional[DictConfig] = None, ): self.variable = variable - self.node_dim = node_dim + self.output_dim = output_dim self.hidden_dims = hidden_dims if hidden_dims is not None else [] self.bias = bias - self.name = "{}_encoder".format(variable.name) self.activation_fn_cfg = activation_fn_cfg + self.name = "{}_encoder".format(variable.name) + super(VariableEncoder, self).__init__() self._model_filename = "{}.pth".format(self.name) def build(self): layers = [] if len(self.hidden_dims) == 0: - hidden_dim = self.node_dim + hidden_dim = self.output_dim else: hidden_dim = self.hidden_dims[0] @@ -42,7 +43,7 @@ def build(self): else: raise NotImplementedError("Type {} is not supported by VariableEncoder".format(type(self.variable))) - hidden_dims = self.hidden_dims + [self.node_dim] + hidden_dims = self.hidden_dims + [self.output_dim] for i in range(len(hidden_dims) - 1): layers += [nn.Linear(hidden_dims[i], hidden_dims[i + 1], bias=self.bias)] layers += [create_activation(self.activation_fn_cfg)] @@ -54,42 +55,44 @@ class VariableDecoder(BaseNetwork): def __init__( self, variable: Variable, - node_dim: int, + input_dim: int = 100, hidden_dims: Optional[List[int]] = None, bias: bool = True, + identity: bool = False, activation_fn_cfg: Optional[DictConfig] = None, - normal_distribution: bool = True, ): self.variable = variable - self.node_dim = node_dim + self.input_dim = input_dim self.hidden_dims = hidden_dims if hidden_dims is not None else [] self.bias = bias - self.name = "{}_decoder".format(variable.name) + self.identity = identity self.activation_fn_cfg = activation_fn_cfg - self.normal_distribution = normal_distribution + self.name = "{}_decoder".format(variable.name) super(VariableDecoder, self).__init__() self._model_filename = "{}.pth".format(self.name) def build(self): + if self.identity: + assert isinstance(self.variable, ContinuousVariable), "only ContinuousVariable could use identity" + self._layers = nn.ModuleList([nn.Identity()]) + return + layers = [create_activation(self.activation_fn_cfg)] - hidden_dims = [self.node_dim] + self.hidden_dims + hidden_dims = [self.input_dim] + self.hidden_dims for i in range(len(hidden_dims) - 1): layers += [nn.Linear(hidden_dims[i], hidden_dims[i + 1], bias=self.bias)] layers += [create_activation(self.activation_fn_cfg)] if len(self.hidden_dims) == 0: - hidden_dim = self.node_dim + hidden_dim = self.input_dim else: hidden_dim = self.hidden_dims[-1] if isinstance(self.variable, ContinuousVariable): - if self.normal_distribution: - layers.append(nn.Linear(hidden_dim, self.variable.dim * 2)) - else: - layers.append(nn.Linear(hidden_dim, self.variable.dim)) + layers.append(nn.Linear(hidden_dim, self.variable.dim * 2)) elif isinstance(self.variable, DiscreteVariable): layers.append(nn.Linear(hidden_dim, self.variable.n)) layers.append(nn.Softmax()) diff --git a/cmrl/models/reward_mech/__init__.py b/cmrl/models/reward_mech/__init__.py deleted file mode 100644 index 77ea159..0000000 --- a/cmrl/models/reward_mech/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from cmrl.models.reward_mech.base_reward_mech import BaseRewardMech - -from cmrl.models.reward_mech.plain_reward_mech import PlainRewardMech diff --git a/cmrl/models/reward_mech/base_reward_mech.py b/cmrl/models/reward_mech/base_reward_mech.py deleted file mode 100644 index a7a9716..0000000 --- a/cmrl/models/reward_mech/base_reward_mech.py +++ /dev/null @@ -1,16 +0,0 @@ -from typing import Union - -import torch -from gym import Space - -from cmrl.models.base_cuasal_mech import BaseCausalMechanism - - -class BaseRewardMech(BaseCausalMechanism): - def __init__( - self, - obs_space: Space, - action_space: Space, - deterministic: bool, - ): - super(BaseRewardMech, self).__init__(obs_space, action_space, deterministic) diff --git a/cmrl/models/reward_mech/plain_reward_mech.py b/cmrl/models/reward_mech/plain_reward_mech.py deleted file mode 100644 index 0cebead..0000000 --- a/cmrl/models/reward_mech/plain_reward_mech.py +++ /dev/null @@ -1,94 +0,0 @@ -from typing import Dict, Optional, Tuple, Union - -import hydra -import omegaconf -import torch -from torch import nn as nn -from torch.nn import functional as F - -# from cmrl.models.layers import truncated_normal_init -from cmrl.models.reward_mech.base_reward_mech import BaseRewardMech - - -class PlainRewardMech(BaseRewardMech): - _MODEL_FILENAME = "plain_reward_mech.pth" - - def __init__( - self, - # transition info - obs_size: int, - action_size: int, - deterministic: bool = False, - # algorithm parameters - ensemble_num: int = 7, - elite_num: int = 5, - learn_logvar_bounds: bool = False, - # network parameters - num_layers: int = 4, - hid_size: int = 200, - activation_fn_cfg: Optional[Union[Dict, omegaconf.DictConfig]] = None, - # others - device: Union[str, torch.device] = "cpu", - ): - super(PlainRewardMech, self).__init__( - obs_size=obs_size, - action_size=action_size, - deterministic=deterministic, - ensemble_num=ensemble_num, - elite_num=elite_num, - device=device, - ) - self.num_layers = num_layers - self.hid_size = hid_size - - def create_activation(): - if activation_fn_cfg is None: - return nn.ReLU() - else: - return hydra.utils.instantiate(activation_fn_cfg) - - hidden_layers = [ - nn.Sequential( - self.create_linear_layer(obs_size + action_size, hid_size), - create_activation(), - ) - ] - for i in range(num_layers - 1): - hidden_layers.append( - nn.Sequential( - self.create_linear_layer(hid_size, hid_size), - create_activation(), - ) - ) - self.hidden_layers = nn.Sequential(*hidden_layers) - - if deterministic: - self.mean_and_logvar = self.create_linear_layer(hid_size, 1) - else: - self.mean_and_logvar = self.create_linear_layer(hid_size, 2) - self.min_logvar = nn.Parameter(-10 * torch.ones(1), requires_grad=learn_logvar_bounds) - self.max_logvar = nn.Parameter(0.5 * torch.ones(1), requires_grad=learn_logvar_bounds) - - # self.apply(truncated_normal_init) - self.to(self.device) - - def forward( - self, - batch_obs: torch.Tensor, # shape: ensemble_num, batch_size, state_size - batch_action: torch.Tensor, # shape: ensemble_num, batch_size, action_size - ) -> Tuple[torch.Tensor, Optional[torch.Tensor]]: - assert len(batch_obs.shape) == 3 and batch_obs.shape[-1] == self.obs_size - assert len(batch_action.shape) == 3 and batch_action.shape[-1] == self.action_size - - hidden = self.hidden_layers(torch.concat([batch_obs, batch_action], dim=-1)) - mean_and_logvar = self.mean_and_logvar(hidden) - - if self.deterministic: - mean, logvar = mean_and_logvar, None - else: - mean = mean_and_logvar[..., :1] - logvar = mean_and_logvar[..., 1:] - logvar = self.max_logvar - F.softplus(self.max_logvar - logvar) - logvar = self.min_logvar + F.softplus(logvar - self.min_logvar) - - return mean, logvar diff --git a/cmrl/models/termination_mech/__init__.py b/cmrl/models/termination_mech/__init__.py deleted file mode 100644 index 43a4ca3..0000000 --- a/cmrl/models/termination_mech/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from cmrl.models.termination_mech.base_termination_mech import BaseTerminationMech - -from cmrl.models.termination_mech.plain_termination_mech import PlainTerminationMech diff --git a/cmrl/models/termination_mech/base_termination_mech.py b/cmrl/models/termination_mech/base_termination_mech.py deleted file mode 100644 index e78150d..0000000 --- a/cmrl/models/termination_mech/base_termination_mech.py +++ /dev/null @@ -1,15 +0,0 @@ -from typing import Union - -import torch -from gym import Space -from cmrl.models.base_cuasal_mech import BaseCausalMechanism - - -class BaseTerminationMech(BaseCausalMechanism): - def __init__( - self, - obs_space: Space, - action_space: Space, - deterministic: bool, - ): - super(BaseTerminationMech, self).__init__(obs_space, action_space, deterministic) diff --git a/cmrl/models/termination_mech/plain_termination_mech.py b/cmrl/models/termination_mech/plain_termination_mech.py deleted file mode 100644 index 59b8478..0000000 --- a/cmrl/models/termination_mech/plain_termination_mech.py +++ /dev/null @@ -1,94 +0,0 @@ -from typing import Dict, Optional, Tuple, Union - -import hydra -import omegaconf -import torch -from torch import nn as nn -from torch.nn import functional as F - -# from cmrl.models.layers import truncated_normal_init -from cmrl.models.termination_mech.base_termination_mech import BaseTerminationMech - - -class PlainTerminationMech(BaseTerminationMech): - _MODEL_FILENAME = "plain_termination_mech.pth" - - def __init__( - self, - # transition info - obs_size: int, - action_size: int, - deterministic: bool = False, - # algorithm parameters - ensemble_num: int = 7, - elite_num: int = 5, - learn_logvar_bounds: bool = False, - # network parameters - num_layers: int = 4, - hid_size: int = 200, - activation_fn_cfg: Optional[Union[Dict, omegaconf.DictConfig]] = None, - # others - device: Union[str, torch.device] = "cpu", - ): - super(PlainTerminationMech, self).__init__( - obs_size=obs_size, - action_size=action_size, - deterministic=deterministic, - ensemble_num=ensemble_num, - elite_num=elite_num, - device=device, - ) - self.num_layers = num_layers - self.hid_size = hid_size - - def create_activation(): - if activation_fn_cfg is None: - return nn.ReLU() - else: - return hydra.utils.instantiate(activation_fn_cfg) - - hidden_layers = [ - nn.Sequential( - self.create_linear_layer(obs_size + action_size, hid_size), - create_activation(), - ) - ] - for i in range(num_layers - 1): - hidden_layers.append( - nn.Sequential( - self.create_linear_layer(hid_size, hid_size), - create_activation(), - ) - ) - self.hidden_layers = nn.Sequential(*hidden_layers) - - if deterministic: - self.mean_and_logvar = self.create_linear_layer(hid_size, 1) - else: - self.mean_and_logvar = self.create_linear_layer(hid_size, 2) - self.min_logvar = nn.Parameter(-10 * torch.ones(1), requires_grad=learn_logvar_bounds) - self.max_logvar = nn.Parameter(0.5 * torch.ones(1), requires_grad=learn_logvar_bounds) - - # self.apply(truncated_normal_init) - self.to(self.device) - - def forward( - self, - batch_obs: torch.Tensor, # shape: ensemble_num, batch_size, state_size - batch_action: torch.Tensor, # shape: ensemble_num, batch_size, action_size - ) -> Tuple[torch.Tensor, Optional[torch.Tensor]]: - assert len(batch_obs.shape) == 3 and batch_obs.shape[-1] == self.obs_size - assert len(batch_action.shape) == 3 and batch_action.shape[-1] == self.action_size - - hidden = self.hidden_layers(torch.concat([batch_obs, batch_action], dim=-1)) - mean_and_logvar = self.mean_and_logvar(hidden) - - if self.deterministic: - mean, logvar = mean_and_logvar, None - else: - mean = mean_and_logvar[..., :1] - logvar = mean_and_logvar[..., 1:] - logvar = self.max_logvar - F.softplus(self.max_logvar - logvar) - logvar = self.min_logvar + F.softplus(logvar - self.min_logvar) - - return mean, logvar diff --git a/cmrl/models/util.py b/cmrl/models/util.py index 5d8f83f..a00532f 100644 --- a/cmrl/models/util.py +++ b/cmrl/models/util.py @@ -96,9 +96,9 @@ def create_encoders( ): encoders = {} for var in input_variables: - assert var.name not in encoders, "Duplicate name in decoders: {}".format(var.name) + assert var.name not in encoders, "duplicate name in decoders: {}".format(var.name) encoders[var.name] = VariableEncoder( - variable=var, node_dim=node_dim, hidden_dims=hidden_dims, bias=bias, activation_fn_cfg=activation_fn_cfg + variable=var, output_dim=node_dim, hidden_dims=hidden_dims, bias=bias, activation_fn_cfg=activation_fn_cfg ).to(device) return encoders @@ -114,10 +114,10 @@ def create_decoders( ): decoders = {} for var in input_variables: - assert var.name not in decoders, "Duplicate name in decoders: {}".format(var.name) + assert var.name not in decoders, "duplicate name in decoders: {}".format(var.name) decoders[var.name] = VariableDecoder( variable=var, - node_dim=node_dim, + input_dim=node_dim, hidden_dims=hidden_dims, bias=bias, activation_fn_cfg=activation_fn_cfg, diff --git a/cmrl/utils/creator.py b/cmrl/utils/creator.py index 012ae3a..5d9ccd6 100644 --- a/cmrl/utils/creator.py +++ b/cmrl/utils/creator.py @@ -2,6 +2,7 @@ from gym import spaces from hydra.utils import instantiate +from omegaconf import DictConfig import numpy as np from stable_baselines3.common.logger import Logger from stable_baselines3.common.base_class import BaseAlgorithm @@ -13,7 +14,7 @@ from cmrl.utils.types import ContinuousVariable, BinaryVariable -def create_agent(cfg, fake_env: VecFakeEnv, logger: Optional[Logger] = None): +def create_agent(cfg: DictConfig, fake_env: VecFakeEnv, logger: Optional[Logger] = None): agent = instantiate(cfg.algorithm.agent)(env=fake_env) agent = cast(BaseAlgorithm, agent) agent.set_logger(logger) @@ -22,7 +23,7 @@ def create_agent(cfg, fake_env: VecFakeEnv, logger: Optional[Logger] = None): def create_dynamics( - cfg, + cfg: DictConfig, observation_space: spaces.Space, action_space: spaces.Space, logger: Optional[Logger] = None, @@ -32,35 +33,32 @@ def create_dynamics( next_obs_variables = parse_space(observation_space, "next_obs") # transition - assert cfg.transition.learn - # TODO: share encoders + assert cfg.transition.learn, "transition must be learned, or you should try model-free RL:)" transition = instantiate(cfg.transition.mech)( input_variables=obs_variables + act_variables, output_variables=next_obs_variables, - variable_encoders=None, - variable_decoders=None, logger=logger, ) transition = cast(BaseCausalMech, transition) + # reward mech + assert cfg.reward_mech.mech.multi_step == "none", "reward-mech must be one-step" if cfg.reward_mech.learn: reward_mech = instantiate(cfg.reward_mech.mech)( input_variables=obs_variables + act_variables + next_obs_variables, output_variables=[ContinuousVariable("reward", dim=1, low=-np.inf, high=np.inf)], - variable_encoders=None, - variable_decoders=None, logger=logger, ) reward_mech = cast(BaseCausalMech, reward_mech) else: reward_mech = None + # termination mech - if cfg.reward_mech.learn: + assert cfg.termination_mech.mech.multi_step == "none", "termination-mech must be one-step" + if cfg.termination_mech.learn: termination_mech = instantiate(cfg.termination_mech.mech)( input_variables=obs_variables + act_variables + next_obs_variables, output_variables=[BinaryVariable("terminal")], - variable_encoders=None, - variable_decoders=None, logger=logger, ) termination_mech = cast(BaseCausalMech, termination_mech) diff --git a/cmrl/utils/types.py b/cmrl/utils/types.py index 59f86ab..1582228 100644 --- a/cmrl/utils/types.py +++ b/cmrl/utils/types.py @@ -89,8 +89,8 @@ class Variable: @dataclass class ContinuousVariable(Variable): dim: int - low: np.ndarray - high: np.ndarray + low: np.ndarray = None + high: np.ndarray = None @dataclass diff --git a/tests/test_models/test_causal_mech/test_plain_mech.py b/tests/test_models/test_causal_mech/test_plain_mech.py index fb73c21..2359092 100644 --- a/tests/test_models/test_causal_mech/test_plain_mech.py +++ b/tests/test_models/test_causal_mech/test_plain_mech.py @@ -4,8 +4,7 @@ from cmrl.models.causal_mech.plain_mech import PlainMech from cmrl.models.data_loader import BufferDataset, EnsembleBufferDataset, collate_fn -from cmrl.algorithms.util import load_offline_data -from cmrl.models.util import parse_space, create_decoders, create_encoders +from cmrl.models.util import parse_space, load_offline_data def prepare(freq_rate): @@ -40,7 +39,7 @@ def prepare(freq_rate): output_variables = parse_space(env.observation_space, "next_obs") variable_decoders = create_decoders(output_variables, node_dim=node_dim) - return input_variables, output_variables, node_dim, variable_encoders, variable_decoders, train_loader, valid_loader + return input_variables, output_variables, train_loader, valid_loader def test_inv_pendulum_single_step(): diff --git a/tests/test_models/test_network/test_coder.py b/tests/test_models/test_network/test_coder.py index 7bb7e9d..8ee0e1a 100644 --- a/tests/test_models/test_network/test_coder.py +++ b/tests/test_models/test_network/test_coder.py @@ -7,68 +7,81 @@ def test_continuous_encoder(): var_dim = 3 - node_dim = 5 + output_dim = 5 batch_size = 128 var = ContinuousVariable(name="obs_0", dim=var_dim) - encoder = VariableEncoder(var, node_dim, hidden_dims=[200]) + encoder = VariableEncoder(var, output_dim, hidden_dims=[20]) inputs = torch.rand(batch_size, var_dim) outputs = encoder(inputs) - assert outputs.shape == (batch_size, node_dim) + assert outputs.shape == (batch_size, output_dim) def test_discrete_encoder(): var_n = 3 - node_dim = 5 + output_dim = 5 batch_size = 128 var = DiscreteVariable(name="obs_0", n=var_n) - encoder = VariableEncoder(var, node_dim, hidden_dims=[200]) + encoder = VariableEncoder(var, output_dim, hidden_dims=[20]) inputs = one_hot(torch.randint(3, (batch_size,))).to(torch.float32) outputs = encoder(inputs) - assert outputs.shape == (batch_size, node_dim) + assert outputs.shape == (batch_size, output_dim) def test_binary_encoder(): - node_dim = 5 + output_dim = 5 batch_size = 128 var = BinaryVariable(name="obs_0") - encoder = VariableEncoder(var, node_dim, hidden_dims=[200]) + encoder = VariableEncoder(var, output_dim, hidden_dims=[20]) inputs = one_hot(torch.randint(1, (batch_size,))).to(torch.float32) outputs = encoder(inputs) - assert outputs.shape == (batch_size, node_dim) + assert outputs.shape == (batch_size, output_dim) + + +def test_identity_decoder(): + var_dim = 3 + batch_size = 128 + + var = ContinuousVariable(name="obs_0", dim=var_dim) + + decoder = VariableDecoder(var, identity=True) + inputs = torch.rand(batch_size, var_dim * 2) + outputs = decoder(inputs) + + assert outputs.shape == (batch_size, var_dim * 2) def test_continuous_decoder(): var_dim = 3 - node_dim = 5 + input_dim = 5 batch_size = 128 var = ContinuousVariable(name="obs_0", dim=var_dim) - decoder = VariableDecoder(var, node_dim, hidden_dims=[200]) - inputs = torch.rand(batch_size, node_dim) + decoder = VariableDecoder(var, input_dim, hidden_dims=[200]) + inputs = torch.rand(batch_size, input_dim) outputs = decoder(inputs) - assert outputs.shape == (batch_size, var_dim) + assert outputs.shape == (batch_size, var_dim * 2) def test_discrete_decoder(): var_n = 3 - node_dim = 5 + input_dim = 5 batch_size = 128 var = DiscreteVariable(name="obs_0", n=var_n) - decoder = VariableDecoder(var, node_dim, hidden_dims=[200]) - inputs = torch.rand(batch_size, node_dim) + decoder = VariableDecoder(var, input_dim, hidden_dims=[200]) + inputs = torch.rand(batch_size, input_dim) outputs = decoder(inputs) assert outputs.shape == (batch_size, var_n) @@ -76,13 +89,13 @@ def test_discrete_decoder(): def test_binary_decoder(): - node_dim = 5 + input_dim = 5 batch_size = 128 var = BinaryVariable(name="obs_0") - decoder = VariableDecoder(var, node_dim, hidden_dims=[200]) - inputs = torch.rand(batch_size, node_dim) + decoder = VariableDecoder(var, input_dim, hidden_dims=[200]) + inputs = torch.rand(batch_size, input_dim) outputs = decoder(inputs) assert outputs.shape == (batch_size, 1) diff --git a/tests/test_models/test_reward_mech/__init__.py b/tests/test_models/test_reward_mech/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/tests/test_models/test_reward_mech/test_plain_reward_mech.py b/tests/test_models/test_reward_mech/test_plain_reward_mech.py deleted file mode 100644 index 749cfe5..0000000 --- a/tests/test_models/test_reward_mech/test_plain_reward_mech.py +++ /dev/null @@ -1,30 +0,0 @@ -from unittest import TestCase - -import torch - -from cmrl.models.reward_mech.plain_reward_mech import PlainRewardMech - - -class TestPlainRewardMech(TestCase): - def setUp(self) -> None: - self.obs_size = 11 - self.action_size = 3 - self.num_layers = 4 - self.ensemble_num = 7 - self.hid_size = 200 - self.batch_size = 128 - self.device = "cuda" if torch.cuda.is_available() else "cpu" - self.reward_mech = PlainRewardMech( - obs_size=self.obs_size, - action_size=self.action_size, - device=self.device, - num_layers=self.num_layers, - hid_size=self.hid_size, - deterministic=True, - ) - self.batch_obs = torch.rand([self.ensemble_num, self.batch_size, self.obs_size]).to(self.device) - self.batch_action = torch.rand([self.ensemble_num, self.batch_size, self.action_size]).to(self.device) - - def test_forward(self): - mean, logvar = self.reward_mech.forward(self.batch_obs, self.batch_action) - assert mean.shape == (self.ensemble_num, self.batch_size, 1) diff --git a/tests/test_models/test_termination_mech/__init__.py b/tests/test_models/test_termination_mech/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/tests/test_models/test_termination_mech/test_plain_termination_mech.py b/tests/test_models/test_termination_mech/test_plain_termination_mech.py deleted file mode 100644 index e69de29..0000000 diff --git a/tests/test_models/test_transition/__init__.py b/tests/test_models/test_transition/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/tests/test_models/test_transition/test_base_transition.py b/tests/test_models/test_transition/test_base_transition.py deleted file mode 100644 index 87c913c..0000000 --- a/tests/test_models/test_transition/test_base_transition.py +++ /dev/null @@ -1,25 +0,0 @@ -import tempfile -from pathlib import Path -from unittest import TestCase - -import torch - -from cmrl.models.transition.base_transition import BaseTransition - - -class TestBasicEnsembleGaussianMLP(TestCase): - def setUp(self) -> None: - self.obs_size = 11 - self.action_size = 3 - self.ensemble_num = 7 - self.device = "cuda" if torch.cuda.is_available() else "cpu" - self.deterministic_transition = BaseTransition( - obs_size=self.obs_size, - action_size=self.action_size, - device=self.device, - ensemble_num=self.ensemble_num, - deterministic=True, - ) - - def test_build(self): - assert True diff --git a/tests/test_models/test_transition/test_multi_step/__init__.py b/tests/test_models/test_transition/test_multi_step/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/tests/test_models/test_transition/test_multi_step/test_forward_euler.py b/tests/test_models/test_transition/test_multi_step/test_forward_euler.py deleted file mode 100644 index af8f1de..0000000 --- a/tests/test_models/test_transition/test_multi_step/test_forward_euler.py +++ /dev/null @@ -1,51 +0,0 @@ -import tempfile -from pathlib import Path -from unittest import TestCase - -import torch - -from cmrl.models.transition.multi_step.forward_euler import ForwardEulerTransition -from cmrl.models.transition.one_step.plain_transition import PlainTransition - - -class TestForwardEulerTransition(TestCase): - def setUp(self) -> None: - self.obs_size = 11 - self.action_size = 3 - self.num_layers = 4 - self.ensemble_num = 7 - self.hid_size = 200 - self.batch_size = 128 - self.repeat_times = 3 - self.device = "cuda" if torch.cuda.is_available() else "cpu" - self.deterministic_one_step = PlainTransition( - obs_size=self.obs_size, - action_size=self.action_size, - device=self.device, - num_layers=self.num_layers, - ensemble_num=self.ensemble_num, - hid_size=self.hid_size, - deterministic=True, - ) - self.deterministic_transition = ForwardEulerTransition( - one_step_transition=self.deterministic_one_step, - repeat_times=self.repeat_times, - ) - self.gaussian_one_step = PlainTransition( - obs_size=self.obs_size, - action_size=self.action_size, - device=self.device, - num_layers=self.num_layers, - ensemble_num=self.ensemble_num, - hid_size=self.hid_size, - deterministic=False, - ) - self.gaussian_transition = ForwardEulerTransition( - one_step_transition=self.gaussian_one_step, repeat_times=self.repeat_times - ) - self.batch_obs = torch.rand([self.ensemble_num, self.batch_size, self.obs_size]).to(self.device) - self.batch_action = torch.rand([self.ensemble_num, self.batch_size, self.action_size]).to(self.device) - - def test_deterministic_forward(self): - mean, logvar = self.deterministic_transition.forward(self.batch_obs, self.batch_action) - assert mean.shape == (self.ensemble_num, self.batch_size, self.obs_size) diff --git a/tests/test_models/test_transition/test_one_step/__init__.py b/tests/test_models/test_transition/test_one_step/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/tests/test_models/test_transition/test_one_step/test_basic_ensemble.py b/tests/test_models/test_transition/test_one_step/test_basic_ensemble.py deleted file mode 100644 index 2ea754d..0000000 --- a/tests/test_models/test_transition/test_one_step/test_basic_ensemble.py +++ /dev/null @@ -1,72 +0,0 @@ -import tempfile -from pathlib import Path -from unittest import TestCase - -import torch - -from cmrl.models.transition.one_step.plain_transition import PlainTransition - - -class TestBasicEnsembleGaussianMLP(TestCase): - def setUp(self) -> None: - self.obs_size = 11 - self.action_size = 3 - self.num_layers = 4 - self.ensemble_num = 7 - self.hid_size = 200 - self.batch_size = 128 - self.device = "cuda" if torch.cuda.is_available() else "cpu" - self.deterministic_transition = PlainTransition( - obs_size=self.obs_size, - action_size=self.action_size, - device=self.device, - num_layers=self.num_layers, - ensemble_num=self.ensemble_num, - hid_size=self.hid_size, - deterministic=True, - ) - self.gaussian_transition = PlainTransition( - obs_size=self.obs_size, - action_size=self.action_size, - device=self.device, - num_layers=self.num_layers, - ensemble_num=self.ensemble_num, - hid_size=self.hid_size, - deterministic=False, - ) - self.batch_obs = torch.rand([self.ensemble_num, self.batch_size, self.obs_size]).to(self.device) - self.batch_action = torch.rand([self.ensemble_num, self.batch_size, self.action_size]).to(self.device) - - def test_deterministic_forward(self): - mean, logvar = self.deterministic_transition.forward(self.batch_obs, self.batch_action) - assert mean.shape == (self.ensemble_num, self.batch_size, self.obs_size) and logvar is None - - def test_gaussian_forward(self): - mean, logvar = self.gaussian_transition.forward(self.batch_obs, self.batch_action) - assert mean.shape == logvar.shape == (self.ensemble_num, self.batch_size, self.obs_size) - - def test_load(self): - tempdir = Path(tempfile.gettempdir()) - model_dir = tempdir / "temp_model" - if not model_dir.exists(): - model_dir.mkdir() - - mean, logvar = self.gaussian_transition.forward(self.batch_obs, self.batch_action) - self.gaussian_transition.save(model_dir) - - new_gaussian_transition = PlainTransition( - obs_size=self.obs_size, - action_size=self.action_size, - device=self.device, - num_layers=self.num_layers, - ensemble_num=self.ensemble_num, - hid_size=self.hid_size, - deterministic=False, - ) - - new_mean, new_logvar = new_gaussian_transition.forward(self.batch_obs, self.batch_action) - assert not (mean == new_mean).all() - - new_gaussian_transition.load(model_dir) - new_mean, new_logvar = new_gaussian_transition.forward(self.batch_obs, self.batch_action) - assert (mean == new_mean).all() diff --git a/tests/test_models/test_transition/test_one_step/test_external_mask_ensemble.py b/tests/test_models/test_transition/test_one_step/test_external_mask_ensemble.py deleted file mode 100644 index b4ad599..0000000 --- a/tests/test_models/test_transition/test_one_step/test_external_mask_ensemble.py +++ /dev/null @@ -1,93 +0,0 @@ -import tempfile -from pathlib import Path -from unittest import TestCase - -import torch - -from cmrl.models.transition.one_step.external_mask_transition import ( - ExternalMaskTransition, -) - - -class TestExternalMaskEnsembleGaussianTransition(TestCase): - def setUp(self) -> None: - self.obs_size = 11 - self.action_size = 3 - self.num_layers = 4 - self.ensemble_num = 7 - self.hid_size = 200 - self.batch_size = 128 - self.device = "cuda" if torch.cuda.is_available() else "cpu" - self.deterministic_transition = ExternalMaskTransition( - obs_size=self.obs_size, - action_size=self.action_size, - device=self.device, - num_layers=self.num_layers, - ensemble_num=self.ensemble_num, - hid_size=self.hid_size, - deterministic=True, - ) - self.gaussian_transition = ExternalMaskTransition( - obs_size=self.obs_size, - action_size=self.action_size, - device=self.device, - num_layers=self.num_layers, - ensemble_num=self.ensemble_num, - hid_size=self.hid_size, - deterministic=False, - ) - self.batch_obs = torch.rand([self.ensemble_num, self.batch_size, self.obs_size]).to(self.device) - self.batch_action = torch.rand([self.ensemble_num, self.batch_size, self.action_size]).to(self.device) - - def test_deterministic_forward(self): - input_mask = torch.ones(self.obs_size, self.obs_size + self.action_size) - self.deterministic_transition.set_input_mask(input_mask) - mean, logvar = self.deterministic_transition.forward(self.batch_obs, self.batch_action) - assert mean.shape == (self.ensemble_num, self.batch_size, self.obs_size) and logvar is None - - def test_gaussian_forward(self): - input_mask = torch.ones(self.obs_size, self.obs_size + self.action_size) - self.gaussian_transition.set_input_mask(input_mask) - mean, logvar = self.gaussian_transition.forward(self.batch_obs, self.batch_action) - assert mean.shape == logvar.shape == (self.ensemble_num, self.batch_size, self.obs_size) - - def test_mask_input(self): - input_mask = torch.ones(self.obs_size, self.obs_size + self.action_size) - self.gaussian_transition.set_input_mask(input_mask) - mean, logvar = self.gaussian_transition.forward(self.batch_obs, self.batch_action) - - new_input_mask = input_mask.clone() - new_input_mask[0] = torch.zeros(self.obs_size + self.action_size) - self.gaussian_transition.set_input_mask(new_input_mask) - new_mean, new_logvar = self.gaussian_transition.forward(self.batch_obs, self.batch_action) - assert not (mean == new_mean).all() - assert (mean[..., 1:] == new_mean[..., 1:]).all() - - def test_load(self): - tempdir = Path(tempfile.gettempdir()) - model_dir = tempdir / "temp_model" - if not model_dir.exists(): - model_dir.mkdir() - - input_mask = torch.ones(self.obs_size, self.obs_size + self.action_size) - self.gaussian_transition.set_input_mask(input_mask) - mean, logvar = self.gaussian_transition.forward(self.batch_obs, self.batch_action) - self.gaussian_transition.save(model_dir) - - new_gaussian_transition = ExternalMaskTransition( - obs_size=self.obs_size, - action_size=self.action_size, - device=self.device, - num_layers=self.num_layers, - ensemble_num=self.ensemble_num, - hid_size=self.hid_size, - deterministic=False, - ) - - new_gaussian_transition.set_input_mask(input_mask) - new_mean, new_logvar = new_gaussian_transition.forward(self.batch_obs, self.batch_action) - assert not (mean == new_mean).all() - - new_gaussian_transition.load(model_dir) - new_mean, new_logvar = new_gaussian_transition.forward(self.batch_obs, self.batch_action) - assert (mean == new_mean).all() From 0492f30cd446887db789c77453cb633a47d687c7 Mon Sep 17 00:00:00 2001 From: frank Date: Mon, 7 Nov 2022 22:23:52 +0800 Subject: [PATCH 19/68] :hammer: refactor BaseCausalMech --- README.md | 2 +- cmrl/algorithms/base_algorithm.py | 45 +++- cmrl/examples/conf/algorithm/off_dyna.yaml | 13 +- cmrl/examples/conf/main.yaml | 2 +- cmrl/examples/conf/reward_mech/plain.yaml | 55 +++-- .../examples/conf/termination_mech/plain.yaml | 55 +++-- cmrl/examples/conf/transition/CMI_test.yaml | 57 +++++ cmrl/examples/conf/transition/plain.yaml | 8 +- cmrl/models/base_cuasal_mech.py | 20 -- cmrl/models/causal_mech/CMI_test.py | 221 +++--------------- cmrl/models/causal_mech/__init__.py | 1 + cmrl/models/causal_mech/base_causal_mech.py | 215 +++++++++++++++-- cmrl/models/causal_mech/plain_mech.py | 146 +----------- cmrl/models/data_loader.py | 30 ++- cmrl/models/dynamics.py | 12 +- cmrl/models/fake_env.py | 4 - cmrl/models/networks/__init__.py | 2 + cmrl/models/transition/__init__.py | 6 - cmrl/models/transition/base_transition.py | 16 -- cmrl/models/transition/multi_step/__init__.py | 0 .../transition/multi_step/forward_euler.py | 41 ---- cmrl/models/transition/one_step/__init__.py | 0 .../one_step/external_mask_transition.py | 170 -------------- .../one_step/internal_mask_transition.py | 0 .../transition/one_step/plain_transition.py | 122 ---------- cmrl/models/util.py | 75 +----- cmrl/utils/creator.py | 8 +- {img => docs}/cmrl_logo.png | Bin 28 files changed, 449 insertions(+), 877 deletions(-) create mode 100644 cmrl/examples/conf/transition/CMI_test.yaml delete mode 100644 cmrl/models/base_cuasal_mech.py delete mode 100644 cmrl/models/transition/__init__.py delete mode 100644 cmrl/models/transition/base_transition.py delete mode 100644 cmrl/models/transition/multi_step/__init__.py delete mode 100644 cmrl/models/transition/multi_step/forward_euler.py delete mode 100644 cmrl/models/transition/one_step/__init__.py delete mode 100644 cmrl/models/transition/one_step/external_mask_transition.py delete mode 100644 cmrl/models/transition/one_step/internal_mask_transition.py delete mode 100644 cmrl/models/transition/one_step/plain_transition.py rename {img => docs}/cmrl_logo.png (100%) diff --git a/README.md b/README.md index 3219478..6925e6e 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -![](/img/cmrl_logo.png) +![](/docs/cmrl_logo.png) # Causal-MBRL diff --git a/cmrl/algorithms/base_algorithm.py b/cmrl/algorithms/base_algorithm.py index b1fc537..4b128d1 100644 --- a/cmrl/algorithms/base_algorithm.py +++ b/cmrl/algorithms/base_algorithm.py @@ -8,7 +8,6 @@ from stable_baselines3.common.buffers import ReplayBuffer from stable_baselines3.common.callbacks import BaseCallback -from cmrl.models.util import load_offline_data from cmrl.models.fake_env import VecFakeEnv from cmrl.sb3_extension.logger import configure as logger_configure from cmrl.sb3_extension.eval_callback import EvalCallback @@ -16,6 +15,28 @@ from cmrl.utils.env import make_env +def load_offline_data(env, replay_buffer: ReplayBuffer, dataset_name: str, use_ratio: float = 1): + assert hasattr(env, "get_dataset"), "env must have `get_dataset` method" + + data_dict = env.get_dataset(dataset_name) + all_data_num = len(data_dict["observations"]) + sample_data_num = int(use_ratio * all_data_num) + sample_idx = np.random.permutation(all_data_num)[:sample_data_num] + + assert replay_buffer.n_envs == 1 + assert replay_buffer.buffer_size >= sample_data_num + + if sample_data_num == replay_buffer.buffer_size: + replay_buffer.full = True + replay_buffer.pos = 0 + else: + replay_buffer.pos = sample_data_num + + # set all data + for attr in ["observations", "next_observations", "actions", "rewards", "dones", "timeouts"]: + getattr(replay_buffer, attr)[:sample_data_num, 0] = data_dict[attr][sample_idx] + + class BaseAlgorithm: def __init__( self, @@ -25,24 +46,28 @@ def __init__( self.cfg = cfg self.work_dir = work_dir or os.getcwd() - self.env, self.reward_fn, self.termination_fn, self.get_init_obs_fn = make_env(cfg) - self.eval_env, *_ = make_env(cfg) - np.random.seed(cfg.seed) - torch.manual_seed(cfg.seed) + self.env, self.reward_fn, self.termination_fn, self.get_init_obs_fn = make_env(self.cfg) + self.eval_env, *_ = make_env(self.cfg) + np.random.seed(self.cfg.seed) + torch.manual_seed(self.cfg.seed) self.logger = logger_configure("log", ["tensorboard", "multi_csv", "stdout"]) # create ``cmrl`` dynamics - self.dynamics = create_dynamics(cfg, self.env.observation_space, self.env.action_space, logger=self.logger) + self.dynamics = create_dynamics(self.cfg, self.env.observation_space, self.env.action_space, logger=self.logger) # create sb3's replay buffer for real offline data self.real_replay_buffer = ReplayBuffer( - cfg.task.num_steps, self.env.observation_space, self.env.action_space, cfg.device, handle_timeout_termination=False + cfg.task.num_steps, + self.env.observation_space, + self.env.action_space, + self.cfg.device, + handle_timeout_termination=False, ) self.partial_fake_env = partial( VecFakeEnv, - cfg.algorithm.num_envs, + self.cfg.algorithm.num_envs, self.env.observation_space, self.env.action_space, self.dynamics, @@ -50,13 +75,13 @@ def __init__( self.termination_fn, self.get_init_obs_fn, self.real_replay_buffer, - penalty_coeff=cfg.task.penalty_coeff, + penalty_coeff=self.cfg.task.penalty_coeff, logger=self.logger, ) self.fake_env = self.get_fake_env() - self.agent = create_agent(cfg, self.fake_env, self.logger) + self.agent = create_agent(self.cfg, self.fake_env, self.logger) self.callback = self.get_callback() diff --git a/cmrl/examples/conf/algorithm/off_dyna.yaml b/cmrl/examples/conf/algorithm/off_dyna.yaml index 6fc2abf..35ec7be 100644 --- a/cmrl/examples/conf/algorithm/off_dyna.yaml +++ b/cmrl/examples/conf/algorithm/off_dyna.yaml @@ -1,21 +1,16 @@ name: "off_dyna" -freq_train_model: ${task.freq_train_model} -real_data_ratio: 0.0 +algo: + _partial_: true + _target_: cmrl.algorithms.OfflineDyna -sac_samples_action: true num_eval_episodes: 5 dataset_size: 1000000 penalty_coeff: ${task.penalty_coeff} -algo: - _partial_: true - _target_: cmrl.algorithms.OfflineDyna -# -------------------------------------------- -# SAC Agent configuration -# -------------------------------------------- + num_envs: 16 deterministic: true agent: diff --git a/cmrl/examples/conf/main.yaml b/cmrl/examples/conf/main.yaml index b943805..91ce51f 100644 --- a/cmrl/examples/conf/main.yaml +++ b/cmrl/examples/conf/main.yaml @@ -1,7 +1,7 @@ defaults: - algorithm: off_dyna - task: BIPS - - transition: plain + - transition: CMI_test - reward_mech: plain - termination_mech: plain - _self_ diff --git a/cmrl/examples/conf/reward_mech/plain.yaml b/cmrl/examples/conf/reward_mech/plain.yaml index 8d06712..46a3b9f 100644 --- a/cmrl/examples/conf/reward_mech/plain.yaml +++ b/cmrl/examples/conf/reward_mech/plain.yaml @@ -1,33 +1,56 @@ -name: "plain_reward" +name: "plain_reward_mech" learn: false +encoder_cfg: + _partial_: true + _recursive_: false + _target_: cmrl.models.networks.VariableEncoder + output_dim: 100 + hidden_dims: [ 100 ] + bias: true + activation_fn_cfg: + _target_: torch.nn.SiLU + +decoder_cfg: + _partial_: true + _recursive_: false + _target_: cmrl.models.networks.VariableDecoder + identity: true + +network_cfg: + _partial_: true + _recursive_: false + _target_: cmrl.models.networks.ParallelMLP + hidden_dims: [ 200, 200 ] + use_bias: true + activation_fn_cfg: + _target_: torch.nn.SiLU + +optimizer_cfg: + _partial_: true + _target_: torch.optim.Adam + lr: 1e-4 + weight_decay: 1e-5 + eps: 1e-8 + mech: _partial_: true _recursive_: false _target_: cmrl.models.causal_mech.PlainMech - name: ${transition.name} # base causal-mech params + name: reward_mech input_variables: ??? output_variables: ??? - node_dim: 20 - variable_encoders: ??? - variable_decoders: ??? - # network params - deterministic: false - hidden_dims: [ 200, 200, 200, 200 ] ensemble_num: 7 elite_num: 5 - use_bias: true - activation_fn_cfg: - _target_: torch.nn.SiLU + # cfgs + network_cfg: ${transition.network_cfg} + encoder_cfg: ${transition.encoder_cfg} + decoder_cfg: ${transition.decoder_cfg} + optimizer_cfg: ${transition.optimizer_cfg} # forward method residual: true multi_step: "none" - # trainer - optim_lr: 1e-4 - optim_weight_decay: 1e-5 - optim_eps: 1e-8 - optim_encoder: true # logger logger: ??? # others diff --git a/cmrl/examples/conf/termination_mech/plain.yaml b/cmrl/examples/conf/termination_mech/plain.yaml index 37ff0be..2755da7 100644 --- a/cmrl/examples/conf/termination_mech/plain.yaml +++ b/cmrl/examples/conf/termination_mech/plain.yaml @@ -1,33 +1,56 @@ -name: "plain_termination" +name: "plain_termination_mech" learn: false +encoder_cfg: + _partial_: true + _recursive_: false + _target_: cmrl.models.networks.VariableEncoder + output_dim: 100 + hidden_dims: [ 100 ] + bias: true + activation_fn_cfg: + _target_: torch.nn.SiLU + +decoder_cfg: + _partial_: true + _recursive_: false + _target_: cmrl.models.networks.VariableDecoder + identity: true + +network_cfg: + _partial_: true + _recursive_: false + _target_: cmrl.models.networks.ParallelMLP + hidden_dims: [ 200, 200 ] + use_bias: true + activation_fn_cfg: + _target_: torch.nn.SiLU + +optimizer_cfg: + _partial_: true + _target_: torch.optim.Adam + lr: 1e-4 + weight_decay: 1e-5 + eps: 1e-8 + mech: _partial_: true _recursive_: false _target_: cmrl.models.causal_mech.PlainMech - name: ${transition.name} # base causal-mech params + name: termination_mech input_variables: ??? output_variables: ??? - node_dim: 20 - variable_encoders: ??? - variable_decoders: ??? - # network params - deterministic: false - hidden_dims: [ 200, 200, 200, 200 ] ensemble_num: 7 elite_num: 5 - use_bias: true - activation_fn_cfg: - _target_: torch.nn.SiLU + # cfgs + network_cfg: ${transition.network_cfg} + encoder_cfg: ${transition.encoder_cfg} + decoder_cfg: ${transition.decoder_cfg} + optimizer_cfg: ${transition.optimizer_cfg} # forward method residual: true multi_step: "none" - # trainer - optim_lr: 1e-4 - optim_weight_decay: 1e-5 - optim_eps: 1e-8 - optim_encoder: true # logger logger: ??? # others diff --git a/cmrl/examples/conf/transition/CMI_test.yaml b/cmrl/examples/conf/transition/CMI_test.yaml new file mode 100644 index 0000000..95b491a --- /dev/null +++ b/cmrl/examples/conf/transition/CMI_test.yaml @@ -0,0 +1,57 @@ +name: "CMI_test_transition" +learn: true + +encoder_cfg: + _partial_: true + _recursive_: false + _target_: cmrl.models.networks.VariableEncoder + output_dim: 100 + hidden_dims: [ 100 ] + bias: true + activation_fn_cfg: + _target_: torch.nn.SiLU + +decoder_cfg: + _partial_: true + _recursive_: false + _target_: cmrl.models.networks.VariableDecoder + identity: true + +network_cfg: + _partial_: true + _recursive_: false + _target_: cmrl.models.networks.ParallelMLP + hidden_dims: [ 200, 200 ] + use_bias: true + activation_fn_cfg: + _target_: torch.nn.SiLU + +optimizer_cfg: + _partial_: true + _target_: torch.optim.Adam + lr: 1e-4 + weight_decay: 1e-5 + eps: 1e-8 + +mech: + _partial_: true + _recursive_: false + _target_: cmrl.models.causal_mech.CMItest + # base causal-mech params + name: transition + input_variables: ??? + output_variables: ??? + ensemble_num: 7 + elite_num: 5 + # cfgs + network_cfg: ${transition.network_cfg} + encoder_cfg: ${transition.encoder_cfg} + decoder_cfg: ${transition.decoder_cfg} + optimizer_cfg: ${transition.optimizer_cfg} + # forward method + residual: true + multi_step: "none" + # logger + logger: ??? + # others + device: ${device} diff --git a/cmrl/examples/conf/transition/plain.yaml b/cmrl/examples/conf/transition/plain.yaml index 7cbfd9c..cbb0d80 100644 --- a/cmrl/examples/conf/transition/plain.yaml +++ b/cmrl/examples/conf/transition/plain.yaml @@ -4,7 +4,7 @@ learn: true encoder_cfg: _partial_: true _recursive_: false - _target_: cmrl.models.networks.coder.VariableEncoder + _target_: cmrl.models.networks.VariableEncoder output_dim: 100 hidden_dims: [ 100 ] bias: true @@ -14,13 +14,13 @@ encoder_cfg: decoder_cfg: _partial_: true _recursive_: false - _target_: cmrl.models.networks.coder.VariableDecoder + _target_: cmrl.models.networks.VariableDecoder identity: true network_cfg: _partial_: true _recursive_: false - _target_: cmrl.models.networks.parallel_mlp.ParallelMLP + _target_: cmrl.models.networks.ParallelMLP hidden_dims: [ 200, 200 ] use_bias: true activation_fn_cfg: @@ -38,7 +38,7 @@ mech: _recursive_: false _target_: cmrl.models.causal_mech.PlainMech # base causal-mech params - name: ${transition.name} + name: transition input_variables: ??? output_variables: ??? ensemble_num: 7 diff --git a/cmrl/models/base_cuasal_mech.py b/cmrl/models/base_cuasal_mech.py deleted file mode 100644 index eb60e80..0000000 --- a/cmrl/models/base_cuasal_mech.py +++ /dev/null @@ -1,20 +0,0 @@ -from typing import Union -from abc import abstractmethod - -import torch -from gym import Space - - -class BaseCausalMechanism: - def __init__(self, obs_space: Space, action_space: Space, deterministic: bool): - self.obs_space = obs_space - self.action_space = action_space - self.deterministic = deterministic - - self.network = None - self.graph = None - pass - - @abstractmethod - def predict(self, obs, action, next_obs=None): - pass diff --git a/cmrl/models/causal_mech/CMI_test.py b/cmrl/models/causal_mech/CMI_test.py index 043bbcc..48f0e55 100644 --- a/cmrl/models/causal_mech/CMI_test.py +++ b/cmrl/models/causal_mech/CMI_test.py @@ -1,57 +1,37 @@ from typing import Optional, List, Dict, Union, MutableMapping -import pathlib -import itertools -import copy -import numpy as np import torch -from torch.utils.data import DataLoader -from torch.optim import Adam from omegaconf import DictConfig +from hydra.utils import instantiate from stable_baselines3.common.logger import Logger from cmrl.utils.types import Variable -from cmrl.models.networks.parallel_mlp import ParallelMLP -from cmrl.models.causal_mech.base_causal_mech import BaseCausalMech -from cmrl.models.networks.coder import VariableEncoder, VariableDecoder +from cmrl.models.causal_mech.base_causal_mech import NeuralCausalMech -class CMItest(BaseCausalMech): +class CMItest(NeuralCausalMech): def __init__( self, name: str, - # base causal-mech params input_variables: List[Variable], output_variables: List[Variable], - node_dim: int, - variable_encoders: Optional[Dict[str, VariableEncoder]], - variable_decoders: Optional[Dict[str, VariableDecoder]], ensemble_num: int = 7, elite_num: int = 5, - # network params - deterministic: bool = False, - hidden_dims: Optional[List[int]] = None, - use_bias: bool = True, - activation_fn_cfg: Optional[DictConfig] = None, + # cfgs + network_cfg: Optional[DictConfig] = None, + encoder_cfg: Optional[DictConfig] = None, + decoder_cfg: Optional[DictConfig] = None, + optimizer_cfg: Optional[DictConfig] = None, # forward method residual: bool = True, + encoder_reduction: str = "sum", multi_step: str = "none", - # trainer - optim_lr: float = 1e-4, - optim_weight_decay: float = 1e-5, - optim_eps: float = 1e-8, - optim_encoder: bool = True, # logger logger: Optional[Logger] = None, # others device: Union[str, torch.device] = "cpu", **kwargs ): - self.deterministic = deterministic - self.hidden_dims = hidden_dims if hidden_dims is not None else [200] * 4 - self.use_bias = use_bias - self.activation_fn_cfg = activation_fn_cfg - if multi_step == "none": multi_step = "forward-euler 1" @@ -59,192 +39,45 @@ def __init__( name=name, input_variables=input_variables, output_variables=output_variables, - node_dim=node_dim, - variable_encoders=variable_encoders, - variable_decoders=variable_decoders, ensemble_num=ensemble_num, elite_num=elite_num, + network_cfg=network_cfg, + encoder_cfg=encoder_cfg, + decoder_cfg=decoder_cfg, + optimizer_cfg=optimizer_cfg, residual=residual, + encoder_reduction=encoder_reduction, multi_step=multi_step, - optim_lr=optim_lr, - optim_weight_decay=optim_weight_decay, - optim_eps=optim_eps, - optim_encoder=optim_encoder, logger=logger, device=device, **kwargs ) def build_network(self): - self.network = ParallelMLP( - input_dim=self.node_dim, - output_dim=self.node_dim, - hidden_dims=self.hidden_dims, - use_bias=self.use_bias, + self.network = instantiate(self.network_cfg)( + input_dim=self.encoder_output_dim, + output_dim=self.decoder_input_dim, extra_dims=[self.output_var_num, self.ensemble_num], - activation_fn_cfg=self.activation_fn_cfg, ).to(self.device) - parmas = [self.network.parameters()] + [decoder.parameters() for decoder in self.variable_decoders.values()] - if self.optim_encoder: - parmas.extend([encoder.parameters() for encoder in self.variable_encoders.values()]) - self.optimizer = Adam( - itertools.chain(*parmas), lr=self.optim_lr, weight_decay=self.optim_weight_decay, eps=self.optim_eps - ) - def build_graph(self): self.graph = None - def forward(self, inputs: MutableMapping[str, torch.Tensor]) -> Dict[str, torch.Tensor]: - assert len(set(inputs.keys()) & set(self.variable_encoders.keys())) == len(inputs) - data_shape = list(inputs.values())[0].shape - assert len(data_shape) == 3, "{}".format(data_shape) # ensemble-num, batch-size, specific-dim - ensemble, batch_size, specific_dim = data_shape - assert ensemble == self.ensemble_num + def single_step_forward(self, inputs: MutableMapping[str, torch.Tensor]) -> Dict[str, torch.Tensor]: + batch_size = self.get_inputs_batch_size(inputs) - inputs_tensor = torch.zeros(ensemble, batch_size, self.input_var_num * self.node_dim).to(self.device) + inputs_tensor = torch.zeros(self.ensemble_num, batch_size, self.input_var_num, self.encoder_output_dim).to(self.device) for i, var in enumerate(self.input_variables): - out = self.variable_encoders[var.name](inputs[var.name].to(self.device)) # ensemble-num, batch-size, node-dim - inputs_tensor[:, :, i * self.node_dim : (i + 1) * self.node_dim] = out + out = self.variable_encoders[var.name](inputs[var.name].to(self.device)) + inputs_tensor[:, :, i] = out - if self.multi_step.startswith("forward-euler"): - step_num = int(self.multi_step.split()[-1]) - output_tensor = None - for step in range(step_num): - if step > 0: - inputs_tensor = torch.concat( - [output_tensor, inputs_tensor[:, :, self.output_var_num * self.node_dim :]], dim=-1 - ) - output_tensor = self.network(inputs_tensor) - if self.residual: - output_tensor += inputs_tensor[:, :, : self.output_var_num * self.node_dim] - else: - raise NotImplementedError + output_tensor = self.network(self.reduce_encoder_output(inputs_tensor)) outputs = {} for i, var in enumerate(self.output_variables): - hid = output_tensor[:, :, i * self.node_dim : (i + 1) * self.node_dim] - out = self.variable_decoders[var.name](hid) - outputs[var.name] = out + hid = output_tensor[i] + outputs[var.name] = self.variable_decoders[var.name](hid) + if self.residual: + outputs = self.residual_outputs(inputs, outputs) return outputs - - def train(self, loader: DataLoader): - """train for ensemble data - - Args: - loader: train data-loader. - - Returns: tensor of train loss, with shape (ensemble-num, batch-size). - - """ - batch_loss_list = [] - for inputs, targets in loader: - outputs = self.forward(inputs) - loss = self.loss(outputs, targets) # ensemble-num, batch-size, output-var-num - - self.optimizer.zero_grad() - loss.mean().backward() - self.optimizer.step() - - batch_loss_list.append(loss) - return torch.cat(batch_loss_list, dim=-2).detach().cpu() - - def eval(self, loader: DataLoader): - """evaluate for non-ensemble data - - Args: - loader: valid data-loader. - - Returns: tensor of eval loss, with shape (batch-size). - - """ - batch_loss_list = [] - with torch.no_grad(): - for inputs, targets in loader: - outputs = self.forward(inputs) - loss = self.loss(outputs, targets) # ensemble-num, batch-size, output-var-num - - batch_loss_list.append(loss) - return torch.cat(batch_loss_list, dim=-2).detach().cpu() - - def learn( - self, - # loader - train_loader: DataLoader, - valid_loader: DataLoader, - # model learning - longest_epoch: int = -1, - improvement_threshold: float = 0.01, - patience: int = 5, - work_dir: Optional[Union[str, pathlib.Path]] = None, - **kwargs - ): - best_weights: Optional[Dict] = None - epoch_iter = range(longest_epoch) if longest_epoch >= 0 else itertools.count() - epochs_since_update = 0 - best_eval_loss = self.eval(valid_loader).mean(dim=(1, 2)) - - for epoch in epoch_iter: - train_loss = self.train(train_loader) - eval_loss = self.eval(valid_loader).mean(dim=(1, 2)) - maybe_best_weights = self._maybe_get_best_weights(best_eval_loss, eval_loss, improvement_threshold) - if maybe_best_weights: - # best loss - best_eval_loss = torch.minimum(best_eval_loss, eval_loss) - best_weights = maybe_best_weights - epochs_since_update = 0 - else: - epochs_since_update += 1 - - # log - self.total_epoch += 1 - if self.logger is not None: - self.logger.record("{}/epoch".format(self.name), epoch) - self.logger.record("{}/train_dataset_size".format(self.name), len(train_loader.dataset)) - self.logger.record("{}/valid_dataset_size".format(self.name), len(valid_loader.dataset)) - self.logger.record("{}/train_loss".format(self.name), train_loss.mean().item()) - self.logger.record("{}/val_loss".format(self.name), eval_loss.mean().item()) - self.logger.record("{}/best_val_loss".format(self.name), best_eval_loss.mean().item()) - - self.logger.dump(self.total_epoch) - - if patience and epochs_since_update >= patience: - break - - # saving the best models: - self._maybe_set_best_weights_and_elite(best_weights, best_eval_loss) - - def _maybe_get_best_weights( - self, - best_val_loss: torch.Tensor, - val_loss: torch.Tensor, - threshold: float = 0.01, - ) -> Optional[Dict]: - """Return the current model state dict if the validation score improves. - For ensembles, this checks the validation for each ensemble member separately. - Copy from https://github.com/facebookresearch/mbrl-lib/blob/main/mbrl/models/model_trainer.py - - Args: - best_val_score (tensor): the current best validation losses per model. - val_score (tensor): the new validation loss per model. - threshold (float): the threshold for relative improvement. - Returns: - (dict, optional): if the validation score's relative improvement over the - best validation score is higher than the threshold, returns the state dictionary - of the stored model, otherwise returns ``None``. - """ - improvement = (best_val_loss - val_loss) / torch.abs(best_val_loss) - if (improvement > threshold).any().item(): - best_weights = copy.deepcopy(self.network.state_dict()) - else: - best_weights = None - - return best_weights - - def _maybe_set_best_weights_and_elite(self, best_weights: Optional[Dict], best_val_score: torch.Tensor): - if best_weights is not None: - self.network.load_state_dict(best_weights) - - sorted_indices = np.argsort(best_val_score.tolist()) - self.elite_indices = sorted_indices[: self.elite_num] diff --git a/cmrl/models/causal_mech/__init__.py b/cmrl/models/causal_mech/__init__.py index c99657e..d55c608 100644 --- a/cmrl/models/causal_mech/__init__.py +++ b/cmrl/models/causal_mech/__init__.py @@ -1 +1,2 @@ from cmrl.models.causal_mech.plain_mech import PlainMech +from cmrl.models.causal_mech.CMI_test import CMItest diff --git a/cmrl/models/causal_mech/base_causal_mech.py b/cmrl/models/causal_mech/base_causal_mech.py index 9fb6dae..77024a0 100644 --- a/cmrl/models/causal_mech/base_causal_mech.py +++ b/cmrl/models/causal_mech/base_causal_mech.py @@ -1,12 +1,15 @@ from typing import Optional, List, Dict, Union, MutableMapping -from abc import abstractmethod +from abc import abstractmethod, ABC from itertools import chain +import pathlib +import itertools +import copy import torch import numpy as np from torch.utils.data import DataLoader import torch.nn.functional as F -from torch.optim import Optimizer, Adam +from torch.optim import Optimizer from stable_baselines3.common.logger import Logger from omegaconf import DictConfig from hydra.utils import instantiate @@ -14,11 +17,49 @@ from cmrl.models.networks.base_network import BaseNetwork from cmrl.models.graphs.base_graph import BaseGraph from cmrl.models.networks.coder import VariableEncoder, VariableDecoder -from cmrl.models.util import create_decoders, create_encoders from cmrl.utils.types import Variable, ContinuousVariable, DiscreteVariable, BinaryVariable -class BaseCausalMech: +class BaseCausalMech(ABC): + def __init__( + self, + name: str, + input_variables: List[Variable], + output_variables: List[Variable], + device: Union[str, torch.device] = "cpu", + ): + self.name = name + self.input_variables = input_variables + self.output_variables = output_variables + self.device = device + + self.input_var_num = len(self.input_variables) + self.output_var_num = len(self.output_variables) + + @abstractmethod + def learn( + self, + # loader + train_loader: DataLoader, + valid_loader: DataLoader, + **kwargs + ): + raise NotImplementedError + + @abstractmethod + def train(self, loader: DataLoader): + raise NotImplementedError + + @abstractmethod + def eval(self, loader: DataLoader): + raise NotImplementedError + + @abstractmethod + def forward(self, inputs: MutableMapping[str, torch.Tensor]) -> Dict[str, torch.Tensor]: + raise NotImplementedError + + +class NeuralCausalMech(BaseCausalMech): def __init__( self, name: str, @@ -41,9 +82,12 @@ def __init__( device: Union[str, torch.device] = "cpu", **kwargs ): - self.name = name - self.input_variables = input_variables - self.output_variables = output_variables + super(NeuralCausalMech, self).__init__( + name=name, + input_variables=input_variables, + output_variables=output_variables, + device=device, + ) self.ensemble_num = ensemble_num self.elite_num = elite_num # cfgs @@ -57,18 +101,13 @@ def __init__( self.multi_step = multi_step # logger self.logger = logger - # others - self.device = device - - self.input_var_num = len(self.input_variables) - self.output_var_num = len(self.output_variables) + # build member object self.variable_encoders: Optional[Dict[str, VariableEncoder]] = None self.variable_decoders: Optional[Dict[str, VariableEncoder]] = None self.network: Optional[BaseNetwork] = None self.graph: Optional[BaseGraph] = None self.optimizer: Optional[Optimizer] = None - self.build_coder() self.build_network() self.build_graph() @@ -98,16 +137,6 @@ def forward(self, inputs: MutableMapping[str, torch.Tensor]) -> Dict[str, torch. return outputs - @abstractmethod - def learn( - self, - # loader - train_loader: DataLoader, - valid_loader: DataLoader, - **kwargs - ): - raise NotImplementedError - @abstractmethod def build_network(self): raise NotImplementedError @@ -161,6 +190,146 @@ def loss(self, outputs, targets): raise NotImplementedError return total_loss + def get_inputs_batch_size(self, inputs: MutableMapping[str, torch.Tensor]) -> int: + assert len(set(inputs.keys()) & set(self.variable_encoders.keys())) == len(inputs) + data_shape = list(inputs.values())[0].shape + assert len(data_shape) == 3, "{}".format(data_shape) # ensemble-num, batch-size, specific-dim + ensemble, batch_size, specific_dim = data_shape + assert ensemble == self.ensemble_num + + return batch_size + + def residual_outputs( + self, + inputs: MutableMapping[str, torch.Tensor], + outputs: MutableMapping[str, torch.Tensor], + ) -> MutableMapping[str, torch.Tensor]: + for name in filter(lambda s: s.startswith("obs"), inputs.keys()): + assert inputs[name].shape[:2] == outputs["next_{}".format(name)].shape[:2] + assert inputs[name].shape[2] * 2 == outputs["next_{}".format(name)].shape[2] + outputs["next_{}".format(name)][:, :, : inputs[name].shape[2]] += inputs[name].to(self.device) + return outputs + + def train(self, loader: DataLoader): + """train for ensemble data + + Args: + loader: train data-loader. + + Returns: tensor of train loss, with shape (ensemble-num, batch-size). + + """ + batch_loss_list = [] + for inputs, targets in loader: + outputs = self.forward(inputs) + loss = self.loss(outputs, targets) # ensemble-num, batch-size, output-var-num + + self.optimizer.zero_grad() + loss.mean().backward() + self.optimizer.step() + + batch_loss_list.append(loss) + return torch.cat(batch_loss_list, dim=-2).detach().cpu() + + def eval(self, loader: DataLoader): + """evaluate for non-ensemble data + + Args: + loader: valid data-loader. + + Returns: tensor of eval loss, with shape (batch-size). + + """ + batch_loss_list = [] + with torch.no_grad(): + for inputs, targets in loader: + outputs = self.forward(inputs) + loss = self.loss(outputs, targets) # ensemble-num, batch-size, output-var-num + + batch_loss_list.append(loss) + return torch.cat(batch_loss_list, dim=-2).detach().cpu() + + def learn( + self, + # loader + train_loader: DataLoader, + valid_loader: DataLoader, + # model learning + longest_epoch: int = -1, + improvement_threshold: float = 0.01, + patience: int = 5, + work_dir: Optional[Union[str, pathlib.Path]] = None, + **kwargs + ): + best_weights: Optional[Dict] = None + epoch_iter = range(longest_epoch) if longest_epoch >= 0 else itertools.count() + epochs_since_update = 0 + best_eval_loss = self.eval(valid_loader).mean(dim=(1, 2)) + + for epoch in epoch_iter: + train_loss = self.train(train_loader) + eval_loss = self.eval(valid_loader).mean(dim=(1, 2)) + maybe_best_weights = self._maybe_get_best_weights(best_eval_loss, eval_loss, improvement_threshold) + if maybe_best_weights: + # best loss + best_eval_loss = torch.minimum(best_eval_loss, eval_loss) + best_weights = maybe_best_weights + epochs_since_update = 0 + else: + epochs_since_update += 1 + + # log + self.total_epoch += 1 + if self.logger is not None: + self.logger.record("{}/epoch".format(self.name), epoch) + self.logger.record("{}/train_dataset_size".format(self.name), len(train_loader.dataset)) + self.logger.record("{}/valid_dataset_size".format(self.name), len(valid_loader.dataset)) + self.logger.record("{}/train_loss".format(self.name), train_loss.mean().item()) + self.logger.record("{}/val_loss".format(self.name), eval_loss.mean().item()) + self.logger.record("{}/best_val_loss".format(self.name), best_eval_loss.mean().item()) + + self.logger.dump(self.total_epoch) + + if patience and epochs_since_update >= patience: + break + + # saving the best models: + self._maybe_set_best_weights_and_elite(best_weights, best_eval_loss) + + def _maybe_get_best_weights( + self, + best_val_loss: torch.Tensor, + val_loss: torch.Tensor, + threshold: float = 0.01, + ) -> Optional[Dict]: + """Return the current model state dict if the validation score improves. + For ensembles, this checks the validation for each ensemble member separately. + Copy from https://github.com/facebookresearch/mbrl-lib/blob/main/mbrl/models/model_trainer.py + + Args: + best_val_score (tensor): the current best validation losses per model. + val_score (tensor): the new validation loss per model. + threshold (float): the threshold for relative improvement. + Returns: + (dict, optional): if the validation score's relative improvement over the + best validation score is higher than the threshold, returns the state dictionary + of the stored model, otherwise returns ``None``. + """ + improvement = (best_val_loss - val_loss) / torch.abs(best_val_loss) + if (improvement > threshold).any().item(): + best_weights = copy.deepcopy(self.network.state_dict()) + else: + best_weights = None + + return best_weights + + def _maybe_set_best_weights_and_elite(self, best_weights: Optional[Dict], best_val_score: torch.Tensor): + if best_weights is not None: + self.network.load_state_dict(best_weights) + + sorted_indices = np.argsort(best_val_score.tolist()) + self.elite_indices = sorted_indices[: self.elite_num] + @property def encoder_output_dim(self): return self.encoder_cfg.output_dim diff --git a/cmrl/models/causal_mech/plain_mech.py b/cmrl/models/causal_mech/plain_mech.py index df88c32..89a04eb 100644 --- a/cmrl/models/causal_mech/plain_mech.py +++ b/cmrl/models/causal_mech/plain_mech.py @@ -1,23 +1,15 @@ from typing import Optional, List, Dict, Union, MutableMapping -import pathlib -import itertools -import copy -import numpy as np import torch -from torch.utils.data import DataLoader -from torch.optim import Adam from omegaconf import DictConfig from hydra.utils import instantiate from stable_baselines3.common.logger import Logger from cmrl.utils.types import Variable -from cmrl.models.networks.parallel_mlp import ParallelMLP -from cmrl.models.causal_mech.base_causal_mech import BaseCausalMech -from cmrl.models.networks.coder import VariableEncoder, VariableDecoder +from cmrl.models.causal_mech.base_causal_mech import NeuralCausalMech -class PlainMech(BaseCausalMech): +class PlainMech(NeuralCausalMech): def __init__( self, name: str, @@ -72,16 +64,13 @@ def build_graph(self): self.graph = None def single_step_forward(self, inputs: MutableMapping[str, torch.Tensor]) -> Dict[str, torch.Tensor]: - assert len(set(inputs.keys()) & set(self.variable_encoders.keys())) == len(inputs) - data_shape = list(inputs.values())[0].shape - assert len(data_shape) == 3, "{}".format(data_shape) # ensemble-num, batch-size, specific-dim - ensemble, batch_size, specific_dim = data_shape - assert ensemble == self.ensemble_num + batch_size = self.get_inputs_batch_size(inputs) - inputs_tensor = torch.zeros(ensemble, batch_size, self.input_var_num, self.encoder_output_dim).to(self.device) + inputs_tensor = torch.zeros(self.ensemble_num, batch_size, self.input_var_num, self.encoder_output_dim).to(self.device) for i, var in enumerate(self.input_variables): out = self.variable_encoders[var.name](inputs[var.name].to(self.device)) inputs_tensor[:, :, i] = out + output_tensor = self.network(self.reduce_encoder_output(inputs_tensor)) outputs = {} @@ -90,128 +79,5 @@ def single_step_forward(self, inputs: MutableMapping[str, torch.Tensor]) -> Dict outputs[var.name] = self.variable_decoders[var.name](hid) if self.residual: - for name in filter(lambda s: s.startswith("obs"), inputs.keys()): - assert inputs[name].shape[:2] == outputs["next_{}".format(name)].shape[:2] - assert inputs[name].shape[2] * 2 == outputs["next_{}".format(name)].shape[2] - outputs["next_{}".format(name)][:, :, : inputs[name].shape[2]] += inputs[name].to(self.device) + outputs = self.residual_outputs(inputs, outputs) return outputs - - def train(self, loader: DataLoader): - """train for ensemble data - - Args: - loader: train data-loader. - - Returns: tensor of train loss, with shape (ensemble-num, batch-size). - - """ - batch_loss_list = [] - for inputs, targets in loader: - outputs = self.forward(inputs) - loss = self.loss(outputs, targets) # ensemble-num, batch-size, output-var-num - - self.optimizer.zero_grad() - loss.mean().backward() - self.optimizer.step() - - batch_loss_list.append(loss) - return torch.cat(batch_loss_list, dim=-2).detach().cpu() - - def eval(self, loader: DataLoader): - """evaluate for non-ensemble data - - Args: - loader: valid data-loader. - - Returns: tensor of eval loss, with shape (batch-size). - - """ - batch_loss_list = [] - with torch.no_grad(): - for inputs, targets in loader: - outputs = self.forward(inputs) - loss = self.loss(outputs, targets) # ensemble-num, batch-size, output-var-num - - batch_loss_list.append(loss) - return torch.cat(batch_loss_list, dim=-2).detach().cpu() - - def learn( - self, - # loader - train_loader: DataLoader, - valid_loader: DataLoader, - # model learning - longest_epoch: int = -1, - improvement_threshold: float = 0.01, - patience: int = 5, - work_dir: Optional[Union[str, pathlib.Path]] = None, - **kwargs - ): - best_weights: Optional[Dict] = None - epoch_iter = range(longest_epoch) if longest_epoch >= 0 else itertools.count() - epochs_since_update = 0 - best_eval_loss = self.eval(valid_loader).mean(dim=(1, 2)) - - for epoch in epoch_iter: - train_loss = self.train(train_loader) - eval_loss = self.eval(valid_loader).mean(dim=(1, 2)) - maybe_best_weights = self._maybe_get_best_weights(best_eval_loss, eval_loss, improvement_threshold) - if maybe_best_weights: - # best loss - best_eval_loss = torch.minimum(best_eval_loss, eval_loss) - best_weights = maybe_best_weights - epochs_since_update = 0 - else: - epochs_since_update += 1 - - # log - self.total_epoch += 1 - if self.logger is not None: - self.logger.record("{}/epoch".format(self.name), epoch) - self.logger.record("{}/train_dataset_size".format(self.name), len(train_loader.dataset)) - self.logger.record("{}/valid_dataset_size".format(self.name), len(valid_loader.dataset)) - self.logger.record("{}/train_loss".format(self.name), train_loss.mean().item()) - self.logger.record("{}/val_loss".format(self.name), eval_loss.mean().item()) - self.logger.record("{}/best_val_loss".format(self.name), best_eval_loss.mean().item()) - - self.logger.dump(self.total_epoch) - - if patience and epochs_since_update >= patience: - break - - # saving the best models: - self._maybe_set_best_weights_and_elite(best_weights, best_eval_loss) - - def _maybe_get_best_weights( - self, - best_val_loss: torch.Tensor, - val_loss: torch.Tensor, - threshold: float = 0.01, - ) -> Optional[Dict]: - """Return the current model state dict if the validation score improves. - For ensembles, this checks the validation for each ensemble member separately. - Copy from https://github.com/facebookresearch/mbrl-lib/blob/main/mbrl/models/model_trainer.py - - Args: - best_val_score (tensor): the current best validation losses per model. - val_score (tensor): the new validation loss per model. - threshold (float): the threshold for relative improvement. - Returns: - (dict, optional): if the validation score's relative improvement over the - best validation score is higher than the threshold, returns the state dictionary - of the stored model, otherwise returns ``None``. - """ - improvement = (best_val_loss - val_loss) / torch.abs(best_val_loss) - if (improvement > threshold).any().item(): - best_weights = copy.deepcopy(self.network.state_dict()) - else: - best_weights = None - - return best_weights - - def _maybe_set_best_weights_and_elite(self, best_weights: Optional[Dict], best_val_score: torch.Tensor): - if best_weights is not None: - self.network.load_state_dict(best_weights) - - sorted_indices = np.argsort(best_val_score.tolist()) - self.elite_indices = sorted_indices[: self.elite_num] diff --git a/cmrl/models/data_loader.py b/cmrl/models/data_loader.py index be45548..c5fcc0f 100644 --- a/cmrl/models/data_loader.py +++ b/cmrl/models/data_loader.py @@ -1,6 +1,5 @@ from typing import Optional -import gym from gym import spaces import torch from torch.utils.data import Dataset, default_collate @@ -56,9 +55,25 @@ def build_indexes(self): self.indexes = permutation[: int(self.size * self.train_ratio)] def load_from_buffer(self): - obs_dict = space2dict(self.replay_buffer.observations[: self.size, 0], self.observation_space, "obs") - act_dict = space2dict(self.replay_buffer.actions[: self.size, 0], self.action_space, "act") - next_obs_dict = space2dict(self.replay_buffer.next_observations[: self.size, 0], self.observation_space, "next_obs") + obs_dict = space2dict( + self.replay_buffer.observations[: self.size, 0], + self.observation_space, + prefix="obs", + to_tensor=True, + ) + act_dict = space2dict( + self.replay_buffer.actions[: self.size, 0], + self.action_space, + prefix="act", + to_tensor=True, + ) + next_obs_dict = space2dict( + self.replay_buffer.next_observations[: self.size, 0], + self.observation_space, + prefix="next_obs", + to_tensor=True, + # device=self.device + ) self.inputs = {} self.inputs.update(obs_dict) @@ -68,13 +83,12 @@ def load_from_buffer(self): self.outputs = next_obs_dict elif self.mech == "reward_mech": rewards = self.replay_buffer.rewards[: self.size, 0] - rewards_dict = {"reward": rewards[:, None]} + rewards_dict = {"reward": torch.from_numpy(rewards[:, None])} self.inputs.update(next_obs_dict) self.outputs = rewards_dict else: - dones = self.replay_buffer.dones[: self.size, 0] - timeouts = self.replay_buffer.timeouts[: self.size, 0] - terminals_dict = {"terminal": (dones * (1 - timeouts))[:, None]} + terminals = self.replay_buffer.dones[: self.size, 0] * (1 - self.replay_buffer.timeouts[: self.size, 0]) + terminals_dict = {"terminal": torch.from_numpy(terminals[:, None])} self.inputs.update(next_obs_dict) self.outputs = terminals_dict diff --git a/cmrl/models/dynamics.py b/cmrl/models/dynamics.py index ec7b656..3def165 100644 --- a/cmrl/models/dynamics.py +++ b/cmrl/models/dynamics.py @@ -11,18 +11,19 @@ from stable_baselines3.common.buffers import ReplayBuffer from cmrl.models.util import space2dict -from cmrl.models.causal_mech.base_causal_mech import BaseCausalMech +from cmrl.models.causal_mech.base_causal_mech import NeuralCausalMech from cmrl.models.data_loader import BufferDataset, EnsembleBufferDataset, collate_fn class Dynamics: def __init__( self, - transition: BaseCausalMech, - reward_mech: Optional[BaseCausalMech], - termination_mech: Optional[BaseCausalMech], + transition: NeuralCausalMech, + reward_mech: Optional[NeuralCausalMech], + termination_mech: Optional[NeuralCausalMech], observation_space: spaces.Space, action_space: spaces.Space, + seed: int = 7, logger: Optional[Logger] = None, ): self.transition = transition @@ -30,6 +31,7 @@ def __init__( self.termination_mech = termination_mech self.observation_space = observation_space self.action_space = action_space + self.seed = seed self.logger = logger self.learn_reward = reward_mech is not None @@ -47,6 +49,7 @@ def get_loader(self, real_replay_buffer, mech: str, batch_size: int = 128): mech=mech, train_ensemble=True, ensemble_num=self.transition.ensemble_num, + seed=self.seed, ) train_loader = DataLoader(train_dataset, batch_size=batch_size, collate_fn=collate_fn) valid_dataset = BufferDataset( @@ -56,6 +59,7 @@ def get_loader(self, real_replay_buffer, mech: str, batch_size: int = 128): is_valid=True, mech=mech, repeat=self.transition.ensemble_num, + seed=self.seed, ) valid_loader = DataLoader(valid_dataset, batch_size=batch_size, collate_fn=collate_fn) diff --git a/cmrl/models/fake_env.py b/cmrl/models/fake_env.py index f18c0fb..e576dfd 100644 --- a/cmrl/models/fake_env.py +++ b/cmrl/models/fake_env.py @@ -1,7 +1,3 @@ -# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. from typing import Any, Dict, List, Optional, Type import gym diff --git a/cmrl/models/networks/__init__.py b/cmrl/models/networks/__init__.py index e69de29..8563cbc 100644 --- a/cmrl/models/networks/__init__.py +++ b/cmrl/models/networks/__init__.py @@ -0,0 +1,2 @@ +from cmrl.models.networks.coder import VariableEncoder, VariableDecoder +from cmrl.models.networks.parallel_mlp import ParallelMLP diff --git a/cmrl/models/transition/__init__.py b/cmrl/models/transition/__init__.py deleted file mode 100644 index 23295bd..0000000 --- a/cmrl/models/transition/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -from cmrl.models.transition.base_transition import BaseTransition - -from cmrl.models.transition.one_step.external_mask_transition import ExternalMaskTransition -from cmrl.models.transition.one_step.plain_transition import PlainTransition - -from cmrl.models.transition.multi_step.forward_euler import ForwardEulerTransition diff --git a/cmrl/models/transition/base_transition.py b/cmrl/models/transition/base_transition.py deleted file mode 100644 index 8515f0a..0000000 --- a/cmrl/models/transition/base_transition.py +++ /dev/null @@ -1,16 +0,0 @@ -from typing import Union - -import torch -from gym import Space - -from cmrl.models.base_cuasal_mech import BaseCausalMechanism - - -class BaseTransition(BaseCausalMechanism): - def __init__( - self, - obs_space: Space, - action_space: Space, - deterministic: bool, - ): - super(BaseTransition, self).__init__(obs_space, action_space, deterministic) diff --git a/cmrl/models/transition/multi_step/__init__.py b/cmrl/models/transition/multi_step/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/cmrl/models/transition/multi_step/forward_euler.py b/cmrl/models/transition/multi_step/forward_euler.py deleted file mode 100644 index 7b59388..0000000 --- a/cmrl/models/transition/multi_step/forward_euler.py +++ /dev/null @@ -1,41 +0,0 @@ -from typing import Dict, Optional, Sequence, Tuple, Union - -import torch - -from cmrl.models.transition.base_transition import BaseTransition - - -class ForwardEulerTransition(BaseTransition): - def __init__(self, one_step_transition: BaseTransition, repeat_times: int = 2): - super().__init__( - obs_size=one_step_transition.obs_size, - action_size=one_step_transition.action_size, - ensemble_num=one_step_transition.ensemble_num, - deterministic=one_step_transition.deterministic, - device=one_step_transition.device, - ) - - self.one_step_transition = one_step_transition - self.repeat_times = repeat_times - - if hasattr(self.one_step_transition, "max_logvar"): - self.max_logvar = one_step_transition.max_logvar - self.min_logvar = one_step_transition.min_logvar - - if hasattr(self.one_step_transition, "input_mask"): - self.input_mask = self.one_step_transition.input_mask - self.set_input_mask = self.one_step_transition.set_input_mask - - def set_elite_members(self, elite_indices: Sequence[int]): - self.one_step_transition.set_elite_members(elite_indices) - - def forward( - self, - batch_obs: torch.Tensor, - batch_action: torch.Tensor, - ) -> Tuple[torch.Tensor, Optional[torch.Tensor]]: - logvar = torch.zeros(batch_obs.shape, device=self.device) - mean = batch_obs - for t in range(self.repeat_times): - mean, logvar = self.one_step_transition.forward(mean, batch_action.clone()) - return mean, logvar diff --git a/cmrl/models/transition/one_step/__init__.py b/cmrl/models/transition/one_step/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/cmrl/models/transition/one_step/external_mask_transition.py b/cmrl/models/transition/one_step/external_mask_transition.py deleted file mode 100644 index e8e6b9d..0000000 --- a/cmrl/models/transition/one_step/external_mask_transition.py +++ /dev/null @@ -1,170 +0,0 @@ -from typing import Dict, Optional, Tuple, Union - -import hydra -import omegaconf -import torch -from torch import nn as nn -from torch.nn import functional as F - -import cmrl.utils.types - -from cmrl.models.layers import ParallelLinear -from cmrl.models.transition.base_transition import BaseTransition -from cmrl.models.util import to_tensor - - -class ExternalMaskTransition(BaseTransition): - """Implements an ensemble of multi-layer perceptrons each modeling a Gaussian distribution - corresponding to each independent dimension. - - Args: - obs_size (int): size of state. - action_size (int): size of action. - device (str or torch.device): the device to use for the model. - num_layers (int): the number of layers in the model - (e.g., if ``num_layers == 3``, then model graph looks like - input -h1-> -h2-> -l3-> output). - ensemble_num (int): the number of members in the ensemble. Defaults to 1. - hid_size (int): the size of the hidden layers (e.g., size of h1 and h2 in the graph above). - deterministic (bool): if ``True``, the model predicts the mean and logvar of the conditional - gaussian distribution, otherwise only predicts the mean. Defaults to ``False``. - residual (bool): if ``True``, the model predicts the residual of output and input. Defaults to ``True``. - learn_logvar_bounds (bool): if ``True``, the log-var bounds will be learned, otherwise - they will be constant. Defaults to ``False``. - activation_fn_cfg (dict or omegaconf.DictConfig, optional): configuration of the - desired activation function. Defaults to torch.nn.ReLU when ``None``. - """ - - _MODEL_FILENAME = "external_mask_transition.pth" - - def __init__( - self, - # transition info - obs_size: int, - action_size: int, - deterministic: bool = False, - # algorithm parameters - ensemble_num: int = 7, - elite_num: int = 5, - residual: bool = True, - learn_logvar_bounds: bool = False, - # network parameters - num_layers: int = 4, - hid_size: int = 200, - activation_fn_cfg: Optional[Union[Dict, omegaconf.DictConfig]] = None, - # others - device: Union[str, torch.device] = "cpu", - ): - super().__init__( - obs_size=obs_size, - action_size=action_size, - deterministic=deterministic, - ensemble_num=ensemble_num, - elite_num=elite_num, - device=device, - ) - self.residual = residual - self.learn_logvar_bounds = learn_logvar_bounds - - self.num_layers = num_layers - self.hid_size = hid_size - self.activation_fn_cfg = activation_fn_cfg - - self._input_mask: Optional[torch.Tensor] = torch.ones((obs_size, obs_size + action_size)).to(device) - self.add_save_attr("input_mask") - - def create_activation(): - if activation_fn_cfg is None: - return nn.ReLU() - else: - return hydra.utils.instantiate(activation_fn_cfg) - - hidden_layers = [ - nn.Sequential( - self.create_linear_layer(obs_size + action_size, hid_size), - create_activation(), - ) - ] - for i in range(num_layers - 1): - hidden_layers.append( - nn.Sequential( - self.create_linear_layer(hid_size, hid_size), - create_activation(), - ) - ) - self.hidden_layers = nn.Sequential(*hidden_layers) - - if deterministic: - self.mean_and_logvar = self.create_linear_layer(hid_size, 1) - else: - self.mean_and_logvar = self.create_linear_layer(hid_size, 2) - self.min_logvar = nn.Parameter(-10 * torch.ones(obs_size, 1, 1, 1), requires_grad=learn_logvar_bounds) - self.max_logvar = nn.Parameter(0.5 * torch.ones(obs_size, 1, 1, 1), requires_grad=learn_logvar_bounds) - - # self.apply(truncated_normal_init) - self.to(self.device) - - def create_linear_layer(self, l_in, l_out): - return ParallelLinear(l_in, l_out, extra_dims=[self.obs_size, self.ensemble_num]) - - def set_input_mask(self, mask: cmrl.utils.types.TensorType): - self._input_mask = to_tensor(mask).to(self.device) - - @property - def input_mask(self): - return self._input_mask - - def mask_input(self, x: torch.Tensor) -> torch.Tensor: - assert x.ndim == 4 - assert self._input_mask is not None - assert 2 <= self._input_mask.ndim <= 4 - assert x.shape[0] == self._input_mask.shape[0] and x.shape[-1] == self._input_mask.shape[-1] - - if self._input_mask.ndim == 2: - # [parallel_size x in_dim] - input_mask = self._input_mask[:, None, None, :] - elif self._input_mask.ndim == 3: - if self._input_mask.shape[1] == x.shape[1]: - # [parallel_size x ensemble_size x in_dim] - input_mask = self._input_mask[:, :, None, :] - elif self._input_mask.shape[1] == x.shape[2]: - # [parallel_size x batch_size x in_dim] - input_mask = self._input_mask[:, None, :, :] - else: - raise RuntimeError("input mask shape %a does not match x shape %a" % (self._input_mask.shape, x.shape)) - else: - assert self._input_mask.shape == x.shape - input_mask = self._input_mask - - x *= input_mask - return x - - def forward( - self, - batch_obs: torch.Tensor, # shape: ensemble_num, batch_size, obs_size - batch_action: torch.Tensor, # shape: ensemble_num, batch_size, action_size - ) -> Tuple[torch.Tensor, Optional[torch.Tensor]]: - assert len(batch_obs.shape) == 3 and batch_obs.shape[-1] == self.obs_size - assert len(batch_action.shape) == 3 and batch_action.shape[-1] == self.action_size - - repeated_input = torch.concat([batch_obs, batch_action], dim=-1).repeat((self.obs_size, 1, 1, 1)) - masked_input = self.mask_input(repeated_input) - hidden = self.hidden_layers(masked_input) - mean_and_logvar = self.mean_and_logvar(hidden) - - if self.deterministic: - mean, logvar = mean_and_logvar, None - else: - mean = mean_and_logvar[..., :1] - logvar = mean_and_logvar[..., 1:] - logvar = self.max_logvar - F.softplus(self.max_logvar - logvar) - logvar = self.min_logvar + F.softplus(logvar - self.min_logvar) - - mean = torch.transpose(mean, 0, -1)[0] - if logvar is not None: - logvar = torch.transpose(logvar, 0, -1)[0] - - if self.residual: - mean += batch_obs - - return mean, logvar diff --git a/cmrl/models/transition/one_step/internal_mask_transition.py b/cmrl/models/transition/one_step/internal_mask_transition.py deleted file mode 100644 index e69de29..0000000 diff --git a/cmrl/models/transition/one_step/plain_transition.py b/cmrl/models/transition/one_step/plain_transition.py deleted file mode 100644 index 94e99bb..0000000 --- a/cmrl/models/transition/one_step/plain_transition.py +++ /dev/null @@ -1,122 +0,0 @@ -from typing import Dict, Optional, Sequence, Tuple, Union - -import hydra -import omegaconf -import torch -from torch import nn as nn -from torch.nn import functional as F - -# from cmrl.models.layers import EnsembleLinearLayer, truncated_normal_init -from cmrl.models.transition.base_transition import BaseTransition - - -class PlainTransition(BaseTransition): - """Implements an ensemble of multi-layer perceptrons each modeling a Gaussian distribution. - - Args: - obs_size (int): size of state. - action_size (int): size of action. - device (str or torch.device): the device to use for the model. - num_layers (int): the number of layers in the model - (e.g., if ``num_layers == 3``, then model graph looks like - input -h1-> -h2-> -l3-> output). - ensemble_num (int): the number of members in the ensemble. Defaults to 1. - hid_size (int): the size of the hidden layers (e.g., size of h1 and h2 in the graph above). - deterministic (bool): if ``True``, the model predicts the mean and logvar of the conditional - gaussian distribution, otherwise only predicts the mean. Defaults to ``False``. - residual (bool): if ``True``, the model predicts the residual of output and input. Defaults to ``True``. - learn_logvar_bounds (bool): if ``True``, the log-var bounds will be learned, otherwise - they will be constant. Defaults to ``False``. - activation_fn_cfg (dict or omegaconf.DictConfig, optional): configuration of the - desired activation function. Defaults to torch.nn.ReLU when ``None``. - """ - - _MODEL_FILENAME = "plain_transition.pth" - - def __init__( - self, - # transition info - obs_size: int, - action_size: int, - deterministic: bool = False, - # algorithm parameters - ensemble_num: int = 7, - elite_num: int = 5, - residual: bool = True, - learn_logvar_bounds: bool = False, - # network parameters - num_layers: int = 4, - hid_size: int = 200, - activation_fn_cfg: Optional[Union[Dict, omegaconf.DictConfig]] = None, - # others - device: Union[str, torch.device] = "cpu", - ): - super().__init__( - obs_size=obs_size, - action_size=action_size, - deterministic=deterministic, - ensemble_num=ensemble_num, - elite_num=elite_num, - device=device, - ) - self.residual = residual - self.learn_logvar_bounds = learn_logvar_bounds - - self.num_layers = num_layers - self.hid_size = hid_size - self.activation_fn_cfg = activation_fn_cfg - - def create_activation(): - if activation_fn_cfg is None: - return nn.ReLU() - else: - return hydra.utils.instantiate(activation_fn_cfg) - - hidden_layers = [ - nn.Sequential( - self.create_linear_layer(obs_size + action_size, hid_size), - create_activation(), - ) - ] - for i in range(num_layers - 1): - hidden_layers.append( - nn.Sequential( - self.create_linear_layer(hid_size, hid_size), - create_activation(), - ) - ) - self.hidden_layers = nn.Sequential(*hidden_layers) - - if deterministic: - self.mean_and_logvar = self.create_linear_layer(hid_size, obs_size) - else: - self.mean_and_logvar = self.create_linear_layer(hid_size, 2 * obs_size) - self.min_logvar = nn.Parameter(-10 * torch.ones(1, obs_size), requires_grad=learn_logvar_bounds) - self.max_logvar = nn.Parameter(0.5 * torch.ones(1, obs_size), requires_grad=learn_logvar_bounds) - - # self.apply(truncated_normal_init) - self.to(self.device) - - def forward( - self, - batch_obs: torch.Tensor, # shape: ensemble_num, batch_size, obs_size - batch_action: torch.Tensor, # shape: ensemble_num, batch_size, action_size - ) -> Tuple[torch.Tensor, Optional[torch.Tensor]]: - assert len(batch_obs.shape) == 3 and batch_obs.shape[-1] == self.obs_size - assert len(batch_action.shape) == 3 and batch_action.shape[-1] == self.action_size - - hidden = self.hidden_layers(torch.concat([batch_obs, batch_action], dim=-1)) - mean_and_logvar = self.mean_and_logvar(hidden) - - if self.deterministic: - mean, logvar = mean_and_logvar, None - else: - mean = mean_and_logvar[..., : self.obs_size] - logvar = mean_and_logvar[..., self.obs_size :] - logvar = self.max_logvar - F.softplus(self.max_logvar - logvar) - logvar = self.min_logvar + F.softplus(logvar - self.min_logvar) - - if self.residual: - mean += batch_obs - - return mean, logvar diff --git a/cmrl/models/util.py b/cmrl/models/util.py index a00532f..8951ee4 100644 --- a/cmrl/models/util.py +++ b/cmrl/models/util.py @@ -3,12 +3,8 @@ import numpy as np import torch from gym import spaces -from omegaconf import DictConfig - -from stable_baselines3.common.buffers import ReplayBuffer from cmrl.utils.types import Variable, ContinuousVariable, DiscreteVariable, BinaryVariable -from cmrl.models.networks.coder import VariableEncoder, VariableDecoder # inplace truncated normal function for pytorch. @@ -56,7 +52,12 @@ def parse_space(space: spaces.Space, prefix="obs") -> List[Variable]: def space2dict( - data: np.ndarray, space: spaces.Space, prefix="obs", repeat: Optional[int] = None, to_tensor: bool = False + data: np.ndarray, + space: spaces.Space, + prefix="obs", + repeat: Optional[int] = None, + to_tensor: bool = False, + device: Union[str, torch.device] = "cpu", ) -> Dict[str, Union[np.ndarray, torch.Tensor]]: if repeat: assert repeat > 1, "repeat must be a int greater than 1" @@ -75,7 +76,7 @@ def space2dict( # shape: (repeat-dim, batch-size, specific-dim) dict_data[name] = np.tile(dict_data[name][None, :, :], [repeat, 1, 1]) if to_tensor: - dict_data[name] = torch.from_numpy(dict_data[name]) + dict_data[name] = torch.from_numpy(dict_data[name]).to(device) return dict_data @@ -84,65 +85,3 @@ def dict2space( data: Dict[str, Union[np.ndarray, torch.Tensor]], space: spaces.Space ) -> Dict[str, Union[np.ndarray, torch.Tensor]]: pass - - -def create_encoders( - input_variables: List[Variable], - node_dim: int, - hidden_dims: Optional[List[int]] = None, - bias: bool = True, - activation_fn_cfg: Optional[DictConfig] = None, - device: Union[str, torch.device] = "cpu", -): - encoders = {} - for var in input_variables: - assert var.name not in encoders, "duplicate name in decoders: {}".format(var.name) - encoders[var.name] = VariableEncoder( - variable=var, output_dim=node_dim, hidden_dims=hidden_dims, bias=bias, activation_fn_cfg=activation_fn_cfg - ).to(device) - return encoders - - -def create_decoders( - input_variables: List[Variable], - node_dim: int, - hidden_dims: Optional[List[int]] = None, - bias: bool = True, - activation_fn_cfg: Optional[DictConfig] = None, - normal_distribution: bool = True, - device: Union[str, torch.device] = "cpu", -): - decoders = {} - for var in input_variables: - assert var.name not in decoders, "duplicate name in decoders: {}".format(var.name) - decoders[var.name] = VariableDecoder( - variable=var, - input_dim=node_dim, - hidden_dims=hidden_dims, - bias=bias, - activation_fn_cfg=activation_fn_cfg, - normal_distribution=normal_distribution, - ).to(device) - return decoders - - -def load_offline_data(env, replay_buffer: ReplayBuffer, dataset_name: str, use_ratio: float = 1): - assert hasattr(env, "get_dataset"), "env must have `get_dataset` method" - - data_dict = env.get_dataset(dataset_name) - all_data_num = len(data_dict["observations"]) - sample_data_num = int(use_ratio * all_data_num) - sample_idx = np.random.permutation(all_data_num)[:sample_data_num] - - assert replay_buffer.n_envs == 1 - assert replay_buffer.buffer_size >= sample_data_num - - if sample_data_num == replay_buffer.buffer_size: - replay_buffer.full = True - replay_buffer.pos = 0 - else: - replay_buffer.pos = sample_data_num - - # set all data - for attr in ["observations", "next_observations", "actions", "rewards", "dones", "timeouts"]: - getattr(replay_buffer, attr)[:sample_data_num, 0] = data_dict[attr][sample_idx] diff --git a/cmrl/utils/creator.py b/cmrl/utils/creator.py index 5d9ccd6..96ac099 100644 --- a/cmrl/utils/creator.py +++ b/cmrl/utils/creator.py @@ -9,7 +9,7 @@ from cmrl.models.dynamics import Dynamics from cmrl.models.fake_env import VecFakeEnv -from cmrl.models.causal_mech.base_causal_mech import BaseCausalMech +from cmrl.models.causal_mech.base_causal_mech import NeuralCausalMech from cmrl.models.util import parse_space from cmrl.utils.types import ContinuousVariable, BinaryVariable @@ -39,7 +39,7 @@ def create_dynamics( output_variables=next_obs_variables, logger=logger, ) - transition = cast(BaseCausalMech, transition) + transition = cast(NeuralCausalMech, transition) # reward mech assert cfg.reward_mech.mech.multi_step == "none", "reward-mech must be one-step" @@ -49,7 +49,7 @@ def create_dynamics( output_variables=[ContinuousVariable("reward", dim=1, low=-np.inf, high=np.inf)], logger=logger, ) - reward_mech = cast(BaseCausalMech, reward_mech) + reward_mech = cast(NeuralCausalMech, reward_mech) else: reward_mech = None @@ -61,7 +61,7 @@ def create_dynamics( output_variables=[BinaryVariable("terminal")], logger=logger, ) - termination_mech = cast(BaseCausalMech, termination_mech) + termination_mech = cast(NeuralCausalMech, termination_mech) else: termination_mech = None diff --git a/img/cmrl_logo.png b/docs/cmrl_logo.png similarity index 100% rename from img/cmrl_logo.png rename to docs/cmrl_logo.png From 119bfb086bf995bb750e2dcd7d187560f699af1d Mon Sep 17 00:00:00 2001 From: frank Date: Mon, 7 Nov 2022 22:45:28 +0800 Subject: [PATCH 20/68] :wrench: make offline-dyna independent --- CONTRIBUTING.md | 2 +- cmrl/algorithms/__init__.py | 4 +- cmrl/algorithms/base_algorithm.py | 21 +- cmrl/algorithms/mbpo.py | 11 +- cmrl/algorithms/mopo.py | 16 +- cmrl/algorithms/off_dyna.py | 26 +++ cmrl/algorithms/on_dyna.py | 8 +- cmrl/examples/conf/algorithm/mbpo.yaml | 31 +-- cmrl/examples/conf/algorithm/mopo.yaml | 17 +- cmrl/examples/conf/algorithm/off_dyna.yaml | 4 +- cmrl/examples/conf/algorithm/on_dyna.yaml | 29 +-- cmrl/examples/conf/main.yaml | 2 +- cmrl/models/causal_discovery/CMI_test.py | 128 ---------- .../{causal_discovery => graphs}/__init__.py | 0 cmrl/models/old_dynamics/__init__.py | 3 - cmrl/models/old_dynamics/base_dynamics.py | 221 ------------------ .../old_dynamics/constraint_based_dynamics.py | 182 --------------- cmrl/models/old_dynamics/ncd_dynamics.py | 181 -------------- cmrl/models/old_dynamics/plain_dynamics.py | 143 ------------ .../test_online_mb_callback.py | 3 +- 20 files changed, 77 insertions(+), 955 deletions(-) create mode 100644 cmrl/algorithms/off_dyna.py delete mode 100644 cmrl/models/causal_discovery/CMI_test.py rename cmrl/models/{causal_discovery => graphs}/__init__.py (100%) delete mode 100644 cmrl/models/old_dynamics/__init__.py delete mode 100644 cmrl/models/old_dynamics/base_dynamics.py delete mode 100644 cmrl/models/old_dynamics/constraint_based_dynamics.py delete mode 100644 cmrl/models/old_dynamics/ncd_dynamics.py delete mode 100644 cmrl/models/old_dynamics/plain_dynamics.py diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1d44313..c4370e3 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -43,7 +43,7 @@ Emoji | Description :art: `:art:` | When you improved / added assets like themes. :rocket: `:rocket:` | When you improved performance. :memo: `:memo:` | When you wrote documentation. -:beetle: `:beetle:` | When you fixed a bug. +:bug: `:bug:` | When you fixed a bug. :twisted_rightwards_arrows: `:twisted_rightwards_arrows:` | When you merged a branch. :fire: `:fire:` | When you removed something. :truck: `:truck:` | When you moved / renamed something. diff --git a/cmrl/algorithms/__init__.py b/cmrl/algorithms/__init__.py index 2e4e217..5b1eb8c 100644 --- a/cmrl/algorithms/__init__.py +++ b/cmrl/algorithms/__init__.py @@ -1,6 +1,4 @@ -from cmrl.algorithms.base_algorithm import BaseAlgorithm +from cmrl.algorithms.off_dyna import OfflineDyna from cmrl.algorithms.mopo import MOPO from cmrl.algorithms.on_dyna import OnlineDyna from cmrl.algorithms.mbpo import MBPO - -OfflineDyna = BaseAlgorithm diff --git a/cmrl/algorithms/base_algorithm.py b/cmrl/algorithms/base_algorithm.py index 4b128d1..4eb30e0 100644 --- a/cmrl/algorithms/base_algorithm.py +++ b/cmrl/algorithms/base_algorithm.py @@ -78,21 +78,18 @@ def __init__( penalty_coeff=self.cfg.task.penalty_coeff, logger=self.logger, ) - - self.fake_env = self.get_fake_env() - self.agent = create_agent(self.cfg, self.fake_env, self.logger) - self.callback = self.get_callback() - - def get_fake_env(self) -> VecFakeEnv: + @property + def fake_env(self) -> VecFakeEnv: return self.partial_fake_env( deterministic=self.cfg.algorithm.deterministic, max_episode_steps=self.env.spec.max_episode_steps, branch_rollout=False, ) - def get_callback(self) -> BaseCallback: + @property + def callback(self) -> BaseCallback: fake_eval_env = self.partial_fake_env( deterministic=True, max_episode_steps=self.env.spec.max_episode_steps, branch_rollout=False ) @@ -109,15 +106,7 @@ def get_callback(self) -> BaseCallback: def learn(self): self._setup_learn() - self.dynamics.learn( - real_replay_buffer=self.real_replay_buffer, - longest_epoch=self.cfg.task.longest_epoch, - improvement_threshold=self.cfg.task.improvement_threshold, - patience=self.cfg.task.patience, - work_dir=self.work_dir, - ) - self.agent.learn(total_timesteps=self.cfg.task.num_steps, callback=self.callback) def _setup_learn(self): - load_offline_data(self.env, self.real_replay_buffer, self.cfg.task.dataset, self.cfg.task.use_ratio) + pass diff --git a/cmrl/algorithms/mbpo.py b/cmrl/algorithms/mbpo.py index 8c8189f..5e1f027 100644 --- a/cmrl/algorithms/mbpo.py +++ b/cmrl/algorithms/mbpo.py @@ -16,15 +16,17 @@ def __init__( ): super(MBPO, self).__init__(cfg, work_dir) - def get_fake_env(self) -> VecFakeEnv: + @property + def fake_env(self) -> VecFakeEnv: return self.partial_fake_env( deterministic=self.cfg.algorithm.deterministic, max_episode_steps=self.cfg.algorithm.branch_rollout_length, branch_rollout=True, ) - def get_callback(self) -> BaseCallback: - eval_callback = super(MBPO, self).get_callback() + @property + def callback(self) -> BaseCallback: + eval_callback = super(MBPO, self).callback omb_callback = OnlineModelBasedCallback( self.env, self.dynamics, @@ -36,6 +38,3 @@ def get_callback(self) -> BaseCallback: ) return CallbackList([eval_callback, omb_callback]) - - def _setup_learn(self): - pass diff --git a/cmrl/algorithms/mopo.py b/cmrl/algorithms/mopo.py index 70b02d7..76774a0 100644 --- a/cmrl/algorithms/mopo.py +++ b/cmrl/algorithms/mopo.py @@ -3,7 +3,7 @@ from omegaconf import DictConfig from cmrl.models.fake_env import VecFakeEnv -from cmrl.algorithms.base_algorithm import BaseAlgorithm +from cmrl.algorithms.base_algorithm import BaseAlgorithm, load_offline_data class MOPO(BaseAlgorithm): @@ -14,9 +14,21 @@ def __init__( ): super(MOPO, self).__init__(cfg, work_dir) - def get_fake_env(self) -> VecFakeEnv: + @property + def fake_env(self) -> VecFakeEnv: return self.partial_fake_env( deterministic=self.cfg.algorithm.deterministic, max_episode_steps=self.cfg.algorithm.branch_rollout_length, branch_rollout=True, ) + + def _setup_learn(self): + load_offline_data(self.env, self.real_replay_buffer, self.cfg.task.dataset, self.cfg.task.use_ratio) + + self.dynamics.learn( + real_replay_buffer=self.real_replay_buffer, + longest_epoch=self.cfg.task.longest_epoch, + improvement_threshold=self.cfg.task.improvement_threshold, + patience=self.cfg.task.patience, + work_dir=self.work_dir, + ) diff --git a/cmrl/algorithms/off_dyna.py b/cmrl/algorithms/off_dyna.py new file mode 100644 index 0000000..6fe89f6 --- /dev/null +++ b/cmrl/algorithms/off_dyna.py @@ -0,0 +1,26 @@ +from typing import Optional + +from omegaconf import DictConfig + +from cmrl.models.fake_env import VecFakeEnv +from cmrl.algorithms.base_algorithm import BaseAlgorithm, load_offline_data + + +class OfflineDyna(BaseAlgorithm): + def __init__( + self, + cfg: DictConfig, + work_dir: Optional[str] = None, + ): + super(OfflineDyna, self).__init__(cfg, work_dir) + + def _setup_learn(self): + load_offline_data(self.env, self.real_replay_buffer, self.cfg.task.dataset, self.cfg.task.use_ratio) + + self.dynamics.learn( + real_replay_buffer=self.real_replay_buffer, + longest_epoch=self.cfg.task.longest_epoch, + improvement_threshold=self.cfg.task.improvement_threshold, + patience=self.cfg.task.patience, + work_dir=self.work_dir, + ) diff --git a/cmrl/algorithms/on_dyna.py b/cmrl/algorithms/on_dyna.py index 380ebec..4c8d622 100644 --- a/cmrl/algorithms/on_dyna.py +++ b/cmrl/algorithms/on_dyna.py @@ -16,8 +16,9 @@ def __init__( ): super(OnlineDyna, self).__init__(cfg, work_dir) - def get_callback(self) -> BaseCallback: - eval_callback = super(OnlineDyna, self).get_callback() + @property + def callback(self) -> BaseCallback: + eval_callback = super(OnlineDyna, self).callback omb_callback = OnlineModelBasedCallback( self.env, self.dynamics, @@ -29,6 +30,3 @@ def get_callback(self) -> BaseCallback: ) return CallbackList([eval_callback, omb_callback]) - - def _setup_learn(self): - pass diff --git a/cmrl/examples/conf/algorithm/mbpo.yaml b/cmrl/examples/conf/algorithm/mbpo.yaml index 6280510..c53e292 100644 --- a/cmrl/examples/conf/algorithm/mbpo.yaml +++ b/cmrl/examples/conf/algorithm/mbpo.yaml @@ -1,37 +1,22 @@ name: "mbpo" +algo: + _partial_: true + _target_: cmrl.algorithms.MBPO + freq_train_model: ${task.freq_train_model} -real_data_ratio: 0.0 -sac_samples_action: true -initial_exploration_steps: 5000 -random_initial_explore: false num_eval_episodes: 5 -algo: - _partial_: true - _target_: cmrl.algorithms.MBPO +initial_exploration_steps: 1000 -# -------------------------------------------- -# SAC Agent configuration -# -------------------------------------------- +num_envs: 1000 +deterministic: false agent: _partial_: true _target_: stable_baselines3.sac.SAC policy: "MlpPolicy" - env: - _target_: cmrl.models.fake_env.VecFakeEnv - num_envs: 1000 - action_space: - _target_: gym.spaces.Box - low: ??? - high: ??? - shape: ??? - observation_space: - _target_: gym.spaces.Box - low: ??? - high: ??? - shape: ??? + env: ??? learning_starts: 0 batch_size: 256 tau: 0.005 diff --git a/cmrl/examples/conf/algorithm/mopo.yaml b/cmrl/examples/conf/algorithm/mopo.yaml index 0990596..270e1ac 100644 --- a/cmrl/examples/conf/algorithm/mopo.yaml +++ b/cmrl/examples/conf/algorithm/mopo.yaml @@ -1,25 +1,16 @@ name: "mopo" -freq_train_model: ${task.freq_train_model} -real_data_ratio: 0.0 +algo: + _partial_: true + _target_: cmrl.algorithms.MOPO -sac_samples_action: true num_eval_episodes: 5 dataset_size: 1000000 penalty_coeff: ${task.penalty_coeff} -branch_rollout_length: 5 - -algo: - _partial_: true - _target_: cmrl.algorithms.MOPO - -# -------------------------------------------- -# SAC Agent configuration -# -------------------------------------------- num_envs: 1000 -deterministic: true +deterministic: false agent: _partial_: true _target_: stable_baselines3.sac.SAC diff --git a/cmrl/examples/conf/algorithm/off_dyna.yaml b/cmrl/examples/conf/algorithm/off_dyna.yaml index 35ec7be..445ff78 100644 --- a/cmrl/examples/conf/algorithm/off_dyna.yaml +++ b/cmrl/examples/conf/algorithm/off_dyna.yaml @@ -9,10 +9,8 @@ num_eval_episodes: 5 dataset_size: 1000000 penalty_coeff: ${task.penalty_coeff} - - num_envs: 16 -deterministic: true +deterministic: false agent: _partial_: true _target_: stable_baselines3.sac.SAC diff --git a/cmrl/examples/conf/algorithm/on_dyna.yaml b/cmrl/examples/conf/algorithm/on_dyna.yaml index 70cdcdd..e66c5b6 100644 --- a/cmrl/examples/conf/algorithm/on_dyna.yaml +++ b/cmrl/examples/conf/algorithm/on_dyna.yaml @@ -1,37 +1,22 @@ name: "on_dyna" +algo: + _partial_: true + _target_: cmrl.algorithms.OnlineDyna + freq_train_model: ${task.freq_train_model} -real_data_ratio: 0.0 -sac_samples_action: true num_eval_episodes: 5 initial_exploration_steps: 1000 -algo: - _partial_: true - _target_: cmrl.algorithms.OnlineDyna - -# -------------------------------------------- -# SAC Agent configuration -# -------------------------------------------- +num_envs: 16 +deterministic: false agent: _partial_: true _target_: stable_baselines3.sac.SAC policy: "MlpPolicy" - env: - _target_: cmrl.models.fake_env.VecFakeEnv - num_envs: 16 - action_space: - _target_: gym.spaces.Box - low: ??? - high: ??? - shape: ??? - observation_space: - _target_: gym.spaces.Box - low: ??? - high: ??? - shape: ??? + env: ??? learning_starts: 0 batch_size: 256 tau: 0.005 diff --git a/cmrl/examples/conf/main.yaml b/cmrl/examples/conf/main.yaml index 91ce51f..15ffe4e 100644 --- a/cmrl/examples/conf/main.yaml +++ b/cmrl/examples/conf/main.yaml @@ -1,5 +1,5 @@ defaults: - - algorithm: off_dyna + - algorithm: on_dyna - task: BIPS - transition: CMI_test - reward_mech: plain diff --git a/cmrl/models/causal_discovery/CMI_test.py b/cmrl/models/causal_discovery/CMI_test.py deleted file mode 100644 index 03bc3a1..0000000 --- a/cmrl/models/causal_discovery/CMI_test.py +++ /dev/null @@ -1,128 +0,0 @@ -from typing import Dict, Optional, Sequence, Tuple, Union - -import hydra -import omegaconf -import torch -from torch import nn as nn -from torch.nn import functional as F - -# from cmrl.models.layers import ParallelEnsembleLinearLayer, truncated_normal_init -from cmrl.models.networks.mlp import EnsembleMLP - - -class TransitionConditionalMutualInformationTest(EnsembleMLP): - _MODEL_FILENAME = "conditional_mutual_information_test.pth" - - def __init__( - self, - # transition info - obs_size: int, - action_size: int, - # algorithm parameters - ensemble_num: int = 7, - elite_num: int = 5, - residual: bool = True, - learn_logvar_bounds: bool = False, - # network parameters - num_layers: int = 4, - hid_size: int = 200, - activation_fn_cfg: Optional[Union[Dict, omegaconf.DictConfig]] = None, - # others - device: Union[str, torch.device] = "cpu", - ): - super().__init__( - ensemble_num=ensemble_num, - elite_num=elite_num, - device=device, - ) - self.obs_size = obs_size - self.action_size = action_size - self.residual = residual - self.learn_logvar_bounds = learn_logvar_bounds - - self.num_layers = num_layers - self.hid_size = hid_size - - self.parallel_num = self.obs_size + self.action_size + 1 - - self._input_mask = 1 - torch.eye(self.parallel_num, self.obs_size + self.action_size).to(self.device) - - def create_activation(): - if activation_fn_cfg is None: - return nn.ReLU() - else: - return hydra.utils.instantiate(activation_fn_cfg) - - hidden_layers = [ - nn.Sequential( - self.create_linear_layer(obs_size + action_size, hid_size), - create_activation(), - ) - ] - for i in range(num_layers - 1): - hidden_layers.append( - nn.Sequential( - self.create_linear_layer(hid_size, hid_size), - create_activation(), - ) - ) - self.hidden_layers = nn.Sequential(*hidden_layers) - - self.mean_and_logvar = self.create_linear_layer(hid_size, 2 * self.obs_size) - self.min_logvar = nn.Parameter( - -10 * torch.ones(self.parallel_num, 1, 1, self.obs_size), requires_grad=learn_logvar_bounds - ) - self.max_logvar = nn.Parameter( - 0.5 * torch.ones(self.parallel_num, 1, 1, self.obs_size), requires_grad=learn_logvar_bounds - ) - - # self.apply(truncated_normal_init) - self.to(self.device) - - # def create_linear_layer(self, l_in, l_out): - # return ParallelEnsembleLinearLayer(l_in, l_out, parallel_num=self.parallel_num, ensemble_num=self.ensemble_num) - - @property - def input_mask(self): - return self._input_mask - - def mask_input(self, x: torch.Tensor) -> torch.Tensor: - assert x.ndim == 4 - assert self._input_mask.ndim == 2 - input_mask = self._input_mask[:, None, None, :] - return x * input_mask - - def forward( - self, - batch_obs: torch.Tensor, # shape: (parallel_num, )ensemble_num, batch_size, obs_size - batch_action: torch.Tensor, # shape: ensemble_num, batch_size, action_size - ) -> Tuple[torch.Tensor, Optional[torch.Tensor]]: - assert len(batch_action.shape) == 3 and batch_action.shape[-1] == self.action_size - - batch_action = batch_action.repeat((self.parallel_num, 1, 1, 1)) - if len(batch_obs.shape) == 3: # non-repeat or first repeat - batch_obs = batch_obs.repeat((self.parallel_num, 1, 1, 1)) - - batch_input = torch.concat([batch_obs, batch_action], dim=-1) - - masked_input = self.mask_input(batch_input) - hidden = self.hidden_layers(masked_input) - mean_and_logvar = self.mean_and_logvar(hidden) - - mean = mean_and_logvar[..., : self.obs_size] - logvar = mean_and_logvar[..., self.obs_size :] - logvar = self.max_logvar - F.softplus(self.max_logvar - logvar) - logvar = self.min_logvar + F.softplus(logvar - self.min_logvar) - - if self.residual: - mean += batch_obs - - return mean, logvar - - def get_nll_loss(self, model_in: Dict[(str, torch.Tensor)], target: torch.Tensor) -> torch.Tensor: - pred_mean, pred_logvar = self.forward(**model_in) - target = target.repeat((self.parallel_num, 1, 1, 1)) - - nll_loss = gaussian_nll(pred_mean, pred_logvar, target, reduce=False) - nll_loss += 0.01 * (self.max_logvar.sum() - self.min_logvar.sum()) - return nll_loss diff --git a/cmrl/models/causal_discovery/__init__.py b/cmrl/models/graphs/__init__.py similarity index 100% rename from cmrl/models/causal_discovery/__init__.py rename to cmrl/models/graphs/__init__.py diff --git a/cmrl/models/old_dynamics/__init__.py b/cmrl/models/old_dynamics/__init__.py deleted file mode 100644 index 3b6c6f1..0000000 --- a/cmrl/models/old_dynamics/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from .base_dynamics import BaseDynamics -from .constraint_based_dynamics import ConstraintBasedDynamics -from .plain_dynamics import PlainEnsembleDynamics diff --git a/cmrl/models/old_dynamics/base_dynamics.py b/cmrl/models/old_dynamics/base_dynamics.py deleted file mode 100644 index f9bfb35..0000000 --- a/cmrl/models/old_dynamics/base_dynamics.py +++ /dev/null @@ -1,221 +0,0 @@ -import abc -import collections -import pathlib -from typing import Dict, List, Optional, Tuple, Union - -import numpy as np -import torch -from stable_baselines3.common.logger import Logger -from stable_baselines3.common.buffers import ReplayBuffer - -from cmrl.models.reward_mech.base_reward_mech import BaseRewardMech -from cmrl.models.termination_mech.base_termination_mech import BaseTerminationMech -from cmrl.models.transition.base_transition import BaseTransition -from cmrl.utils.types import InteractionBatch -from cmrl.utils.transition_iterator import BootstrapIterator, TransitionIterator - - -def split_dict(old_dict: Dict, need_keys: List[str]): - return dict([(key, old_dict[key]) for key in need_keys]) - - -class BaseDynamics: - _MECH_TO_VARIABLE = { - "transition": "batch_next_obs", - "reward_mech": "batch_reward", - "termination_mech": "batch_terminal", - } - _VARIABLE_TO_MECH = dict([(value, key) for key, value in _MECH_TO_VARIABLE.items()]) - - def __init__( - self, - transition: BaseTransition, - learned_reward: bool = True, - reward_mech: Optional[BaseRewardMech] = None, - learned_termination: bool = False, - termination_mech: Optional[BaseTerminationMech] = None, - optim_lr: float = 1e-4, - weight_decay: float = 1e-5, - optim_eps: float = 1e-8, - logger: Optional[Logger] = None, - ): - super(BaseDynamics, self).__init__() - self.transition = transition - self.learned_reward = learned_reward - self.reward_mech = reward_mech - self.learned_termination = learned_termination - self.termination_mech = termination_mech - - self.optim_lr = optim_lr - self.weight_decay = weight_decay - self.optim_eps = optim_eps - self.logger = logger - - self.device = self.transition.device - self.ensemble_num = self.transition.ensemble_num - - self.learn_mech = ["transition"] - self.transition_optimizer = torch.optim.Adam( - self.transition.parameters(), - lr=optim_lr, - weight_decay=weight_decay, - eps=optim_eps, - ) - if self.learned_reward: - self.reward_mech_optimizer = torch.optim.Adam( - self.reward_mech.parameters(), - lr=optim_lr, - weight_decay=weight_decay, - eps=optim_eps, - ) - self.learn_mech.append("reward_mech") - if self.learned_termination: - self.termination_mech_optimizer = torch.optim.Adam( - self.termination_mech.parameters(), - lr=optim_lr, - weight_decay=weight_decay, - eps=optim_eps, - ) - self.learn_mech.append("termination_mech") - - self.total_epoch = {} - for mech in self.learn_mech: - self.total_epoch[mech] = 0 - - @abc.abstractmethod - def learn(self, replay_buffer: ReplayBuffer, **kwargs): - pass - - # auxiliary method for "single batch data" - def get_3d_tensor(self, data: Union[np.ndarray, torch.Tensor], is_ensemble: bool): - if isinstance(data, np.ndarray): - data = torch.from_numpy(data) - if is_ensemble: - if data.ndim == 2: # reward or terminal - data = data.unsqueeze(data.ndim) - return data.to(self.device) - else: - if data.ndim == 1: # reward or terminal - data = data.unsqueeze(data.ndim) - return data.repeat([self.ensemble_num, 1, 1]).to(self.device) - - # auxiliary method for "interaction batch data" - def get_mech_loss( - self, - batch: InteractionBatch, - mech: str = "transition", - loss_type: str = "default", - is_ensemble: bool = False, - ): - data = {} - for attr in batch.attrs: - data[attr] = self.get_3d_tensor(getattr(batch, attr).copy(), is_ensemble=is_ensemble) - model_in = split_dict(data, ["batch_obs", "batch_action"]) - - if loss_type == "default": - loss_type = "mse" if getattr(self, mech).deterministic else "nll" - - variable = self._MECH_TO_VARIABLE[mech] - get_loss = getattr(getattr(self, mech), "get_{}_loss".format(loss_type)) - return get_loss(model_in, data[variable]) - - # auxiliary method for "replay buffer" - def dataset_split( - self, - replay_buffer: ReplayBuffer, - validation_ratio: float = 0.2, - batch_size: int = 256, - shuffle_each_epoch: bool = True, - bootstrap_permutes: bool = False, - ) -> Tuple[TransitionIterator, Optional[TransitionIterator]]: - size = replay_buffer.buffer_size if replay_buffer.full else replay_buffer.pos - data = InteractionBatch( - replay_buffer.observations[:size, 0].astype(np.float32), - replay_buffer.actions[:size, 0], - replay_buffer.next_observations[:size, 0].astype(np.float32), - replay_buffer.rewards[:size, 0], - replay_buffer.dones[:size, 0], - ) - - val_size = int(len(data) * validation_ratio) - train_size = len(data) - val_size - train_data = data[:train_size] - train_iter = BootstrapIterator( - train_data, - batch_size, - self.ensemble_num, - shuffle_each_epoch=shuffle_each_epoch, - permute_indices=bootstrap_permutes, - ) - - val_iter = None - if val_size > 0: - val_data = data[train_size:] - val_iter = TransitionIterator(val_data, batch_size, shuffle_each_epoch=False) - - return train_iter, val_iter - - # auxiliary method for "dataset" - def evaluate( - self, - dataset: TransitionIterator, - mech: str = "transition", - ): - assert not isinstance(dataset, BootstrapIterator) - - batch_loss_list = [] - with torch.no_grad(): - for batch in dataset: - val_loss = self.get_mech_loss(batch, mech=mech, loss_type="mse", is_ensemble=False) - batch_loss_list.append(val_loss) - return torch.cat(batch_loss_list, dim=batch_loss_list[0].ndim - 2).cpu() - - def train( - self, - dataset: TransitionIterator, - mech: str = "transition", - ): - assert isinstance(dataset, BootstrapIterator) - - batch_loss_list = [] - for batch in dataset: - train_loss = self.get_mech_loss(batch, mech=mech, is_ensemble=True) - optim = getattr(self, "{}_optimizer".format(mech)) - optim.zero_grad() - train_loss.mean().backward() - optim.step() - batch_loss_list.append(train_loss) - return torch.cat(batch_loss_list, dim=batch_loss_list[0].ndim - 2).detach().cpu() - - def query(self, obs, action, return_as_np=True): - result = collections.defaultdict(dict) - obs = self.get_3d_tensor(obs, is_ensemble=False) - action = self.get_3d_tensor(action, is_ensemble=False) - for mech in self.learn_mech: - with torch.no_grad(): - mean, logvar = getattr(self, "{}".format(mech)).forward(obs, action) - variable = self.get_variable_by_mech(mech) - if return_as_np: - result[variable]["mean"] = mean.cpu().numpy() - result[variable]["logvar"] = logvar.cpu().numpy() - else: - result[variable]["mean"] = mean.cpu() - result[variable]["logvar"] = logvar.cpu() - return result - - # other auxiliary method - def save(self, save_dir: Union[str, pathlib.Path]): - for mech in self.learn_mech: - getattr(self, mech).save(save_dir=save_dir) - - def load(self, load_dir: Union[str, pathlib.Path], load_device: Optional[str] = None): - for mech in self.learn_mech: - getattr(self, mech).load(load_dir=load_dir, load_device=load_device) - - def get_variable_by_mech(self, mech: str) -> str: - assert mech in self._MECH_TO_VARIABLE - return self._MECH_TO_VARIABLE[mech] - - def get_mach_by_variable(self, variable: str) -> str: - assert variable in self._VARIABLE_TO_MECH - return self._VARIABLE_TO_MECH[variable] diff --git a/cmrl/models/old_dynamics/constraint_based_dynamics.py b/cmrl/models/old_dynamics/constraint_based_dynamics.py deleted file mode 100644 index c506b0c..0000000 --- a/cmrl/models/old_dynamics/constraint_based_dynamics.py +++ /dev/null @@ -1,182 +0,0 @@ -import copy -import itertools -import pathlib -from typing import Dict, Optional, Union - -import numpy as np -import torch -from stable_baselines3.common.logger import Logger -from stable_baselines3.common.buffers import ReplayBuffer - -from cmrl.models.dynamics.base_dynamics import BaseDynamics -from cmrl.models.networks.mlp import EnsembleMLP -from cmrl.models.reward_mech.base_reward_mech import BaseRewardMech -from cmrl.models.termination_mech.base_termination_mech import BaseTerminationMech -from cmrl.models.transition.base_transition import BaseTransition -from cmrl.models.causal_discovery.CMI_test import TransitionConditionalMutualInformationTest -from cmrl.models.util import to_tensor -from cmrl.utils.types import TensorType - - -class ConstraintBasedDynamics(BaseDynamics): - def __init__( - self, - transition: BaseTransition, - learned_reward: bool = True, - reward_mech: Optional[BaseRewardMech] = None, - learned_termination: bool = False, - termination_mech: Optional[BaseTerminationMech] = None, - # trainer - optim_lr: float = 1e-4, - weight_decay: float = 1e-5, - optim_eps: float = 1e-8, - logger: Optional[Logger] = None, - ): - super(ConstraintBasedDynamics, self).__init__( - transition=transition, - learned_reward=learned_reward, - reward_mech=reward_mech, - learned_termination=learned_termination, - termination_mech=termination_mech, - optim_lr=optim_lr, - weight_decay=weight_decay, - optim_eps=optim_eps, - logger=logger, - ) - # self.cmi_test: Optional[EnsembleMLP] = None - # self.build_cmi_test() - # - # self.cmi_test_optimizer = torch.optimizer.Adam( - # self.cmi_test.parameters(), - # lr=optim_lr, - # weight_decay=weight_decay, - # eps=optim_eps, - # ) - # self.learn_mech.append("cmi_test") - # self.total_epoch["cmi_test"] = 0 - # self._MECH_TO_VARIABLE["cmi_test"] = self._MECH_TO_VARIABLE["transition"] - - for mech in self.learn_mech: - if hasattr(getattr(self, mech), "input_mask"): - setattr(self, "{}_oracle_mask".format(mech), None) - setattr(self, "{}_history_mask".format(mech), torch.ones(getattr(self, mech).input_mask.shape).to(self.device)) - - def build_cmi_test(self): - self.cmi_test = TransitionConditionalMutualInformationTest( - obs_size=self.transition.obs_size, - action_size=self.transition.action_size, - ensemble_num=1, - elite_num=1, - residual=self.transition.residual, - learn_logvar_bounds=self.transition.learn_logvar_bounds, - num_layers=4, - hid_size=200, - activation_fn_cfg=self.transition.activation_fn_cfg, - device=self.transition.device, - ) - - def set_oracle_mask(self, mech: str, mask: TensorType): - assert hasattr(self, "{}_oracle_mask".format(mech)) - setattr(self, "{}_oracle_mask".format(mech), to_tensor(mask)) - - def learn( - self, - # data - replay_buffer: ReplayBuffer, - # dataset split - validation_ratio: float = 0.2, - batch_size: int = 256, - shuffle_each_epoch: bool = True, - bootstrap_permutes: bool = False, - # model learning - longest_epoch: Optional[int] = None, - improvement_threshold: float = 0.1, - patience: int = 5, - work_dir: Optional[Union[str, pathlib.Path]] = None, - # other - **kwargs - ): - train_dataset, val_dataset = self.dataset_split( - replay_buffer, - validation_ratio, - batch_size, - shuffle_each_epoch, - bootstrap_permutes, - ) - - for mech in self.learn_mech: - - if hasattr(self, "{}_oracle_mask".format(mech)): - getattr(self, mech).set_input_mask(getattr(self, "{}_oracle_mask".format(mech))) - - best_weights: Optional[Dict] = None - epoch_iter = range(longest_epoch) if longest_epoch > 0 else itertools.count() - epochs_since_update = 0 - - best_val_loss = self.evaluate(val_dataset, mech=mech).mean(dim=(1, 2)) - - for epoch in epoch_iter: - train_loss = self.train(train_dataset, mech=mech) - val_loss = self.evaluate(val_dataset, mech=mech).mean(dim=(1, 2)) - - maybe_best_weights = self.maybe_get_best_weights( - best_val_loss, - val_loss, - mech, - improvement_threshold, - ) - if maybe_best_weights: - # best loss - best_val_loss = torch.minimum(best_val_loss, val_loss) - best_weights = maybe_best_weights - epochs_since_update = 0 - else: - epochs_since_update += 1 - - # log - self.total_epoch[mech] += 1 - if self.logger is not None: - self.logger.record("{}/epoch".format(mech), epoch) - self.logger.record("{}/train_dataset_size".format(mech), train_dataset.num_stored) - self.logger.record("{}/val_dataset_size".format(mech), val_dataset.num_stored) - self.logger.record("{}/train_loss".format(mech), train_loss.mean().item()) - self.logger.record("{}/val_loss".format(mech), val_loss.mean().item()) - self.logger.record("{}/best_val_loss".format(mech), best_val_loss.mean().item()) - self.logger.dump(self.total_epoch[mech]) - if patience and epochs_since_update >= patience: - break - - # saving the best models: - self.maybe_set_best_weights_and_elite(best_weights, best_val_loss, mech=mech) - self.save(work_dir) - - def maybe_get_best_weights( - self, - best_val_loss: torch.Tensor, - val_loss: torch.Tensor, - mech: str = "transition", - threshold: float = 0.01, - ): - improvement = (best_val_loss - val_loss) / torch.abs(best_val_loss) - if (improvement > threshold).any().item(): - model = getattr(self, mech) - best_weights = copy.deepcopy(model.state_dict()) - else: - best_weights = None - - return best_weights - - def maybe_set_best_weights_and_elite( - self, - best_weights: Optional[Dict], - best_val_loss: torch.Tensor, - mech: str = "transition", - ): - model = getattr(self, mech) - assert isinstance(model, EnsembleMLP) - - if best_weights is not None: - model.load_state_dict(best_weights) - sorted_indices = np.argsort(best_val_loss.tolist()) - elite_models = sorted_indices[: model.elite_num] - model.set_elite_members(elite_models) diff --git a/cmrl/models/old_dynamics/ncd_dynamics.py b/cmrl/models/old_dynamics/ncd_dynamics.py deleted file mode 100644 index 6f7b7c9..0000000 --- a/cmrl/models/old_dynamics/ncd_dynamics.py +++ /dev/null @@ -1,181 +0,0 @@ -import copy -import itertools -import pathlib -from typing import Dict, Optional, Union - -import numpy as np -import torch -from stable_baselines3.common.logger import Logger -from stable_baselines3.common.buffers import ReplayBuffer - -from cmrl.models.dynamics.base_dynamics import BaseDynamics -from cmrl.models.networks.mlp import EnsembleMLP -from cmrl.models.reward_mech.base_reward_mech import BaseRewardMech -from cmrl.models.termination_mech.base_termination_mech import BaseTerminationMech -from cmrl.models.transition.base_transition import BaseTransition -from cmrl.models.causal_discovery.CMI_test import TransitionConditionalMutualInformationTest -from cmrl.models.util import to_tensor -from cmrl.utils.types import TensorType - - -class ConstraintBasedDynamics(BaseDynamics): - def __init__( - self, - transition: BaseTransition, - learned_reward: bool = True, - reward_mech: Optional[BaseRewardMech] = None, - learned_termination: bool = False, - termination_mech: Optional[BaseTerminationMech] = None, - # trainer - optim_lr: float = 1e-4, - weight_decay: float = 1e-5, - optim_eps: float = 1e-8, - logger: Optional[Logger] = None, - ): - super(ConstraintBasedDynamics, self).__init__( - transition=transition, - learned_reward=learned_reward, - reward_mech=reward_mech, - learned_termination=learned_termination, - termination_mech=termination_mech, - optim_lr=optim_lr, - weight_decay=weight_decay, - optim_eps=optim_eps, - logger=logger, - ) - # self.cmi_test: Optional[EnsembleMLP] = None - # self.build_cmi_test() - # - # self.cmi_test_optimizer = torch.optimizer.Adam( - # self.cmi_test.parameters(), - # lr=optim_lr, - # weight_decay=weight_decay, - # eps=optim_eps, - # ) - # self.learn_mech.append("cmi_test") - # self.total_epoch["cmi_test"] = 0 - # self._MECH_TO_VARIABLE["cmi_test"] = self._MECH_TO_VARIABLE["transition"] - - for mech in self.learn_mech: - if hasattr(getattr(self, mech), "input_mask"): - setattr(self, "{}_oracle_mask".format(mech), None) - setattr(self, "{}_history_mask".format(mech), torch.ones(getattr(self, mech).input_mask.shape).to(self.device)) - - def build_cmi_test(self): - self.cmi_test = TransitionConditionalMutualInformationTest( - obs_size=self.transition.obs_size, - action_size=self.transition.action_size, - ensemble_num=1, - elite_num=1, - residual=self.transition.residual, - learn_logvar_bounds=self.transition.learn_logvar_bounds, - num_layers=4, - hid_size=200, - activation_fn_cfg=self.transition.activation_fn_cfg, - device=self.transition.device, - ) - - def set_oracle_mask(self, mech: str, mask: TensorType): - assert hasattr(self, "{}_oracle_mask".format(mech)) - setattr(self, "{}_oracle_mask".format(mech), to_tensor(mask)) - - def learn( - self, - # data - replay_buffer: ReplayBuffer, - # dataset split - validation_ratio: float = 0.2, - batch_size: int = 256, - shuffle_each_epoch: bool = True, - bootstrap_permutes: bool = False, - # model learning - longest_epoch: Optional[int] = None, - improvement_threshold: float = 0.1, - patience: int = 5, - work_dir: Optional[Union[str, pathlib.Path]] = None, - # other - **kwargs - ): - train_dataset, val_dataset = self.dataset_split( - replay_buffer, - validation_ratio, - batch_size, - shuffle_each_epoch, - bootstrap_permutes, - ) - - for mech in self.learn_mech: - if hasattr(self, "{}_oracle_mask".format(mech)): - getattr(self, mech).set_input_mask(getattr(self, "{}_oracle_mask".format(mech))) - - best_weights: Optional[Dict] = None - epoch_iter = range(longest_epoch) if longest_epoch > 0 else itertools.count() - epochs_since_update = 0 - - best_val_loss = self.evaluate(val_dataset, mech=mech).mean(dim=(1, 2)) - - for epoch in epoch_iter: - train_loss = self.train(train_dataset, mech=mech) - val_loss = self.evaluate(val_dataset, mech=mech).mean(dim=(1, 2)) - - maybe_best_weights = self.maybe_get_best_weights( - best_val_loss, - val_loss, - mech, - improvement_threshold, - ) - if maybe_best_weights: - # best loss - best_val_loss = torch.minimum(best_val_loss, val_loss) - best_weights = maybe_best_weights - epochs_since_update = 0 - else: - epochs_since_update += 1 - - # log - self.total_epoch[mech] += 1 - if self.logger is not None: - self.logger.record("{}/epoch".format(mech), epoch) - self.logger.record("{}/train_dataset_size".format(mech), train_dataset.num_stored) - self.logger.record("{}/val_dataset_size".format(mech), val_dataset.num_stored) - self.logger.record("{}/train_loss".format(mech), train_loss.mean().item()) - self.logger.record("{}/val_loss".format(mech), val_loss.mean().item()) - self.logger.record("{}/best_val_loss".format(mech), best_val_loss.mean().item()) - self.logger.dump(self.total_epoch[mech]) - if patience and epochs_since_update >= patience: - break - - # saving the best models: - self.maybe_set_best_weights_and_elite(best_weights, best_val_loss, mech=mech) - self.save(work_dir) - - def maybe_get_best_weights( - self, - best_val_loss: torch.Tensor, - val_loss: torch.Tensor, - mech: str = "transition", - threshold: float = 0.01, - ): - improvement = (best_val_loss - val_loss) / torch.abs(best_val_loss) - if (improvement > threshold).any().item(): - model = getattr(self, mech) - best_weights = copy.deepcopy(model.state_dict()) - else: - best_weights = None - - return best_weights - - def maybe_set_best_weights_and_elite( - self, - best_weights: Optional[Dict], - best_val_loss: torch.Tensor, - mech: str = "transition", - ): - model = getattr(self, mech) - assert isinstance(model, EnsembleMLP) - - if best_weights is not None: - model.load_state_dict(best_weights) - sorted_indices = np.argsort(best_val_loss.tolist()) - elite_models = sorted_indices[: model.elite_num] - model.set_elite_members(elite_models) diff --git a/cmrl/models/old_dynamics/plain_dynamics.py b/cmrl/models/old_dynamics/plain_dynamics.py deleted file mode 100644 index d600cb1..0000000 --- a/cmrl/models/old_dynamics/plain_dynamics.py +++ /dev/null @@ -1,143 +0,0 @@ -import copy -import itertools -import pathlib -from typing import Callable, Dict, List, Optional, Tuple, Union, cast - -import numpy as np -import torch -from stable_baselines3.common.logger import Logger -from stable_baselines3.common.buffers import ReplayBuffer - -from cmrl.models.dynamics import BaseDynamics -from cmrl.models.networks.mlp import EnsembleMLP -from cmrl.models.reward_mech.base_reward_mech import BaseRewardMech -from cmrl.models.termination_mech.base_termination_mech import BaseTerminationMech -from cmrl.models.transition.base_transition import BaseTransition - - -class PlainEnsembleDynamics(BaseDynamics): - def __init__( - self, - transition: BaseTransition, - learned_reward: bool = True, - reward_mech: Optional[BaseRewardMech] = None, - learned_termination: bool = False, - termination_mech: Optional[BaseTerminationMech] = None, - # trainer - optim_lr: float = 1e-4, - weight_decay: float = 1e-5, - optim_eps: float = 1e-8, - # logger - logger: Optional[Logger] = None, - ): - super(PlainEnsembleDynamics, self).__init__( - transition=transition, - learned_reward=learned_reward, - reward_mech=reward_mech, - learned_termination=learned_termination, - termination_mech=termination_mech, - optim_lr=optim_lr, - weight_decay=weight_decay, - optim_eps=optim_eps, - logger=logger, - ) - - def learn( - self, - # data - replay_buffer: ReplayBuffer, - # dataset split - validation_ratio: float = 0.2, - batch_size: int = 256, - shuffle_each_epoch: bool = True, - bootstrap_permutes: bool = False, - # model learning - longest_epoch: int = -1, - improvement_threshold: float = 0.1, - patience: int = 5, - work_dir: Optional[Union[str, pathlib.Path]] = None, - # other - **kwargs - ): - train_dataset, val_dataset = self.dataset_split( - replay_buffer, - validation_ratio, - batch_size, - shuffle_each_epoch, - bootstrap_permutes, - ) - - for mech in self.learn_mech: - best_weights: Optional[Dict] = None - epoch_iter = range(longest_epoch) if longest_epoch > 0 else itertools.count() - epochs_since_update = 0 - - best_val_loss = self.evaluate(val_dataset, mech=mech).mean(dim=(1, 2)) - - for epoch in epoch_iter: - train_loss = self.train(train_dataset, mech=mech) - val_loss = self.evaluate(val_dataset, mech=mech).mean(dim=(1, 2)) - - maybe_best_weights = self.maybe_get_best_weights( - best_val_loss, - val_loss, - mech, - improvement_threshold, - ) - if maybe_best_weights: - # best loss - best_val_loss = torch.minimum(best_val_loss, val_loss) - best_weights = maybe_best_weights - epochs_since_update = 0 - else: - epochs_since_update += 1 - - # log - self.total_epoch[mech] += 1 - if self.logger is not None: - self.logger.record("{}/epoch".format(mech), epoch) - self.logger.record("{}/train_dataset_size".format(mech), train_dataset.num_stored) - self.logger.record("{}/val_dataset_size".format(mech), val_dataset.num_stored) - self.logger.record("{}/train_loss".format(mech), train_loss.mean().item()) - self.logger.record("{}/val_loss".format(mech), val_loss.mean().item()) - self.logger.record("{}/best_val_loss".format(mech), best_val_loss.mean().item()) - self.logger.dump(self.total_epoch[mech]) - - if patience and epochs_since_update >= patience: - break - - # saving the best models: - self.maybe_set_best_weights_and_elite(best_weights, best_val_loss, mech=mech) - if work_dir is not None: - self.save(work_dir) - - def maybe_get_best_weights( - self, - best_val_loss: torch.Tensor, - val_loss: torch.Tensor, - mech: str = "transition", - threshold: float = 0.01, - ): - improvement = (best_val_loss - val_loss) / torch.abs(best_val_loss) - if (improvement > threshold).any().item(): - model = getattr(self, mech) - best_weights = copy.deepcopy(model.state_dict()) - else: - best_weights = None - - return best_weights - - def maybe_set_best_weights_and_elite( - self, - best_weights: Optional[Dict], - best_val_loss: torch.Tensor, - mech: str = "transition", - ): - model = getattr(self, mech) - assert isinstance(model, EnsembleMLP) - - if best_weights is not None: - model.load_state_dict(best_weights) - sorted_indices = np.argsort(best_val_loss.tolist()) - elite_models = sorted_indices[: model.elite_num] - model.set_elite_members(elite_models) diff --git a/tests/test_sb3_extension/test_online_mb_callback.py b/tests/test_sb3_extension/test_online_mb_callback.py index 61fb283..441d772 100644 --- a/tests/test_sb3_extension/test_online_mb_callback.py +++ b/tests/test_sb3_extension/test_online_mb_callback.py @@ -7,8 +7,7 @@ from stable_baselines3.common.buffers import ReplayBuffer from cmrl.sb3_extension.online_mb_callback import OnlineModelBasedCallback -from cmrl.models.dynamics import PlainEnsembleDynamics -from cmrl.models.transition.one_step.plain_transition import PlainTransition +from cmrl.models.dynamics import Dynamics from cmrl.models.fake_env import VecFakeEnv From 190f4d657d69a9810316bc5f3fb3d1dbee0ac4f6 Mon Sep 17 00:00:00 2001 From: frank Date: Mon, 7 Nov 2022 23:29:27 +0800 Subject: [PATCH 21/68] :bug: fix online-RL wrong log --- cmrl/algorithms/mbpo.py | 2 +- cmrl/algorithms/on_dyna.py | 2 +- cmrl/models/causal_mech/base_causal_mech.py | 16 +------------ cmrl/sb3_extension/online_mb_callback.py | 25 +++++++++------------ 4 files changed, 14 insertions(+), 31 deletions(-) diff --git a/cmrl/algorithms/mbpo.py b/cmrl/algorithms/mbpo.py index 5e1f027..e34f2f3 100644 --- a/cmrl/algorithms/mbpo.py +++ b/cmrl/algorithms/mbpo.py @@ -31,7 +31,7 @@ def callback(self) -> BaseCallback: self.env, self.dynamics, self.real_replay_buffer, - total_num_steps=self.cfg.task.online_num_steps, + total_online_timesteps=self.cfg.task.online_num_steps, initial_exploration_steps=self.cfg.algorithm.initial_exploration_steps, freq_train_model=self.cfg.task.freq_train_model, device=self.cfg.device, diff --git a/cmrl/algorithms/on_dyna.py b/cmrl/algorithms/on_dyna.py index 4c8d622..32bbdb4 100644 --- a/cmrl/algorithms/on_dyna.py +++ b/cmrl/algorithms/on_dyna.py @@ -23,7 +23,7 @@ def callback(self) -> BaseCallback: self.env, self.dynamics, self.real_replay_buffer, - total_num_steps=self.cfg.task.online_num_steps, + total_online_timesteps=self.cfg.task.online_num_steps, initial_exploration_steps=self.cfg.algorithm.initial_exploration_steps, freq_train_model=self.cfg.task.freq_train_model, device=self.cfg.device, diff --git a/cmrl/models/causal_mech/base_causal_mech.py b/cmrl/models/causal_mech/base_causal_mech.py index 77024a0..9b2489e 100644 --- a/cmrl/models/causal_mech/base_causal_mech.py +++ b/cmrl/models/causal_mech/base_causal_mech.py @@ -37,21 +37,7 @@ def __init__( self.output_var_num = len(self.output_variables) @abstractmethod - def learn( - self, - # loader - train_loader: DataLoader, - valid_loader: DataLoader, - **kwargs - ): - raise NotImplementedError - - @abstractmethod - def train(self, loader: DataLoader): - raise NotImplementedError - - @abstractmethod - def eval(self, loader: DataLoader): + def learn(self, train_loader: DataLoader, valid_loader: DataLoader, **kwargs): raise NotImplementedError @abstractmethod diff --git a/cmrl/sb3_extension/online_mb_callback.py b/cmrl/sb3_extension/online_mb_callback.py index 582574f..8161dbd 100644 --- a/cmrl/sb3_extension/online_mb_callback.py +++ b/cmrl/sb3_extension/online_mb_callback.py @@ -8,11 +8,7 @@ from stable_baselines3.common.callbacks import BaseCallback, EventCallback from stable_baselines3.common.evaluation import evaluate_policy from stable_baselines3.common.buffers import ReplayBuffer -from stable_baselines3.common.vec_env import ( - DummyVecEnv, - VecEnv, - sync_envs_normalization, -) +from stable_baselines3.common.vec_env import DummyVecEnv from cmrl.models.fake_env import VecFakeEnv from cmrl.models.dynamics import Dynamics @@ -24,7 +20,7 @@ def __init__( env: gym.Env, dynamics: Dynamics, real_replay_buffer: ReplayBuffer, - total_num_steps: int = int(1e5), + total_online_timesteps: int = int(1e5), initial_exploration_steps: int = 1000, freq_train_model: int = 250, device: str = "cpu", @@ -33,7 +29,7 @@ def __init__( self.env = DummyVecEnv([lambda: env]) self.dynamics = dynamics - self.total_num_steps = total_num_steps + self.total_online_timesteps = total_online_timesteps self.initial_exploration_steps = initial_exploration_steps self.freq_train_model = freq_train_model self.device = device @@ -43,18 +39,19 @@ def __init__( self.real_replay_buffer = real_replay_buffer - self.now_num_steps = 0 - self.step_times = 0 + self.now_online_timesteps = 0 self._last_obs = None def _on_step(self) -> bool: - if self.step_times % self.freq_train_model == 0: + if self.n_calls % self.freq_train_model == 0: + # dump some residual log before dynamics learn + self.model.logger.dump(step=self.num_timesteps) + self.dynamics.learn(self.real_replay_buffer) self.step_and_add(explore=False) - self.step_times += 1 - if self.now_num_steps >= self.total_num_steps: + if self.now_online_timesteps >= self.total_online_timesteps: return False return True @@ -62,7 +59,7 @@ def _on_training_start(self): assert self.env.num_envs == 1 self._last_obs = self.env.reset() - while self.now_num_steps < self.initial_exploration_steps: + while self.now_online_timesteps < self.initial_exploration_steps: self.step_and_add(explore=True) def step_and_add(self, explore=True): @@ -73,7 +70,7 @@ def step_and_add(self, explore=True): buffer_actions = self.model.policy.scale_action(actions) new_obs, rewards, dones, infos = self.env.step(actions) - self.now_num_steps += 1 + self.now_online_timesteps += 1 next_obs = deepcopy(new_obs) if dones[0] and infos[0].get("terminal_observation") is not None: From 09c651cf9e399798eb9d6e8fd09bebe0dd7ddb0e Mon Sep 17 00:00:00 2001 From: frank Date: Tue, 8 Nov 2022 00:42:38 +0800 Subject: [PATCH 22/68] :bug: fix tests --- cmrl/algorithms/base_algorithm.py | 22 -- cmrl/algorithms/util.py | 2 +- cmrl/examples/conf/reward_mech/plain.yaml | 9 +- .../examples/conf/termination_mech/plain.yaml | 9 +- cmrl/examples/conf/transition/CMI_test.yaml | 9 +- cmrl/examples/conf/transition/plain.yaml | 9 +- cmrl/models/causal_mech/CMI_test.py | 4 +- cmrl/models/causal_mech/base_causal_mech.py | 309 +-------------- cmrl/models/causal_mech/neural_causal_mech.py | 374 ++++++++++++++++++ cmrl/models/causal_mech/plain_mech.py | 4 +- cmrl/models/data_loader.py | 2 +- cmrl/models/dynamics.py | 14 +- cmrl/models/fake_env.py | 8 +- cmrl/models/networks/coder.py | 2 +- cmrl/models/networks/parallel_mlp.py | 8 +- cmrl/models/util.py | 58 +-- cmrl/sb3_extension/online_mb_callback.py | 27 +- cmrl/types.py | 9 + cmrl/utils/config.py | 22 +- cmrl/utils/creator.py | 13 +- cmrl/utils/env.py | 36 +- cmrl/utils/transition_iterator.py | 159 -------- cmrl/utils/types.py | 103 ----- cmrl/utils/variables.py | 85 ++++ tests/constants.py | 125 ------ tests/test_algorithms/__init__.py | 0 .../test_algorithms/test_offline/__init__.py | 0 .../test_offline/test_off_dyna.py | 16 - tests/test_algorithms/test_util.py | 15 - tests/test_examples/__init__.py | 0 tests/test_examples/test_main.py | 0 .../test_causal_discovery/__init__.py | 0 .../test_causal_discovery/test_CMI_test.py | 73 ---- .../test_causal_mech/test_plain_mech.py | 31 +- tests/test_models/test_data_loader.py | 2 +- tests/test_models/test_network/test_coder.py | 2 +- .../test_online_mb_callback.py | 35 +- tests/test_types.py | 43 -- 38 files changed, 608 insertions(+), 1031 deletions(-) create mode 100644 cmrl/models/causal_mech/neural_causal_mech.py create mode 100644 cmrl/types.py delete mode 100644 cmrl/utils/transition_iterator.py delete mode 100644 cmrl/utils/types.py create mode 100644 cmrl/utils/variables.py delete mode 100644 tests/constants.py delete mode 100644 tests/test_algorithms/__init__.py delete mode 100644 tests/test_algorithms/test_offline/__init__.py delete mode 100644 tests/test_algorithms/test_offline/test_off_dyna.py delete mode 100644 tests/test_algorithms/test_util.py delete mode 100644 tests/test_examples/__init__.py delete mode 100644 tests/test_examples/test_main.py delete mode 100644 tests/test_models/test_causal_discovery/__init__.py delete mode 100644 tests/test_models/test_causal_discovery/test_CMI_test.py diff --git a/cmrl/algorithms/base_algorithm.py b/cmrl/algorithms/base_algorithm.py index 4eb30e0..3441eae 100644 --- a/cmrl/algorithms/base_algorithm.py +++ b/cmrl/algorithms/base_algorithm.py @@ -15,28 +15,6 @@ from cmrl.utils.env import make_env -def load_offline_data(env, replay_buffer: ReplayBuffer, dataset_name: str, use_ratio: float = 1): - assert hasattr(env, "get_dataset"), "env must have `get_dataset` method" - - data_dict = env.get_dataset(dataset_name) - all_data_num = len(data_dict["observations"]) - sample_data_num = int(use_ratio * all_data_num) - sample_idx = np.random.permutation(all_data_num)[:sample_data_num] - - assert replay_buffer.n_envs == 1 - assert replay_buffer.buffer_size >= sample_data_num - - if sample_data_num == replay_buffer.buffer_size: - replay_buffer.full = True - replay_buffer.pos = 0 - else: - replay_buffer.pos = sample_data_num - - # set all data - for attr in ["observations", "next_observations", "actions", "rewards", "dones", "timeouts"]: - getattr(replay_buffer, attr)[:sample_data_num, 0] = data_dict[attr][sample_idx] - - class BaseAlgorithm: def __init__( self, diff --git a/cmrl/algorithms/util.py b/cmrl/algorithms/util.py index 364954e..de7d6de 100644 --- a/cmrl/algorithms/util.py +++ b/cmrl/algorithms/util.py @@ -8,7 +8,7 @@ from stable_baselines3.common.base_class import BaseAlgorithm from stable_baselines3.common.buffers import ReplayBuffer -from cmrl.utils.types import InitObsFnType, RewardFnType, TermFnType +from cmrl.utils.variables import InitObsFnType, RewardFnType, TermFnType # from cmrl.models.dynamics import BaseDynamics from cmrl.models.fake_env import VecFakeEnv diff --git a/cmrl/examples/conf/reward_mech/plain.yaml b/cmrl/examples/conf/reward_mech/plain.yaml index 46a3b9f..1e447d2 100644 --- a/cmrl/examples/conf/reward_mech/plain.yaml +++ b/cmrl/examples/conf/reward_mech/plain.yaml @@ -15,14 +15,19 @@ decoder_cfg: _partial_: true _recursive_: false _target_: cmrl.models.networks.VariableDecoder - identity: true + identity: false + input_dim: 100 + hidden_dims: [ 100 ] + bias: true + activation_fn_cfg: + _target_: torch.nn.SiLU network_cfg: _partial_: true _recursive_: false _target_: cmrl.models.networks.ParallelMLP hidden_dims: [ 200, 200 ] - use_bias: true + bias: true activation_fn_cfg: _target_: torch.nn.SiLU diff --git a/cmrl/examples/conf/termination_mech/plain.yaml b/cmrl/examples/conf/termination_mech/plain.yaml index 2755da7..44b97e5 100644 --- a/cmrl/examples/conf/termination_mech/plain.yaml +++ b/cmrl/examples/conf/termination_mech/plain.yaml @@ -15,14 +15,19 @@ decoder_cfg: _partial_: true _recursive_: false _target_: cmrl.models.networks.VariableDecoder - identity: true + identity: false + input_dim: 100 + hidden_dims: [ 100 ] + bias: true + activation_fn_cfg: + _target_: torch.nn.SiLU network_cfg: _partial_: true _recursive_: false _target_: cmrl.models.networks.ParallelMLP hidden_dims: [ 200, 200 ] - use_bias: true + bias: true activation_fn_cfg: _target_: torch.nn.SiLU diff --git a/cmrl/examples/conf/transition/CMI_test.yaml b/cmrl/examples/conf/transition/CMI_test.yaml index 95b491a..77eaeb4 100644 --- a/cmrl/examples/conf/transition/CMI_test.yaml +++ b/cmrl/examples/conf/transition/CMI_test.yaml @@ -15,14 +15,19 @@ decoder_cfg: _partial_: true _recursive_: false _target_: cmrl.models.networks.VariableDecoder - identity: true + identity: false + input_dim: 100 + hidden_dims: [ 100 ] + bias: true + activation_fn_cfg: + _target_: torch.nn.SiLU network_cfg: _partial_: true _recursive_: false _target_: cmrl.models.networks.ParallelMLP hidden_dims: [ 200, 200 ] - use_bias: true + bias: true activation_fn_cfg: _target_: torch.nn.SiLU diff --git a/cmrl/examples/conf/transition/plain.yaml b/cmrl/examples/conf/transition/plain.yaml index cbb0d80..6104811 100644 --- a/cmrl/examples/conf/transition/plain.yaml +++ b/cmrl/examples/conf/transition/plain.yaml @@ -15,14 +15,19 @@ decoder_cfg: _partial_: true _recursive_: false _target_: cmrl.models.networks.VariableDecoder - identity: true + identity: false + input_dim: 100 + hidden_dims: [ 100 ] + bias: true + activation_fn_cfg: + _target_: torch.nn.SiLU network_cfg: _partial_: true _recursive_: false _target_: cmrl.models.networks.ParallelMLP hidden_dims: [ 200, 200 ] - use_bias: true + bias: true activation_fn_cfg: _target_: torch.nn.SiLU diff --git a/cmrl/models/causal_mech/CMI_test.py b/cmrl/models/causal_mech/CMI_test.py index 48f0e55..c7a5cda 100644 --- a/cmrl/models/causal_mech/CMI_test.py +++ b/cmrl/models/causal_mech/CMI_test.py @@ -5,8 +5,8 @@ from hydra.utils import instantiate from stable_baselines3.common.logger import Logger -from cmrl.utils.types import Variable -from cmrl.models.causal_mech.base_causal_mech import NeuralCausalMech +from cmrl.utils.variables import Variable +from cmrl.models.causal_mech.neural_causal_mech import NeuralCausalMech class CMItest(NeuralCausalMech): diff --git a/cmrl/models/causal_mech/base_causal_mech.py b/cmrl/models/causal_mech/base_causal_mech.py index 9b2489e..978706c 100644 --- a/cmrl/models/causal_mech/base_causal_mech.py +++ b/cmrl/models/causal_mech/base_causal_mech.py @@ -17,7 +17,7 @@ from cmrl.models.networks.base_network import BaseNetwork from cmrl.models.graphs.base_graph import BaseGraph from cmrl.models.networks.coder import VariableEncoder, VariableDecoder -from cmrl.utils.types import Variable, ContinuousVariable, DiscreteVariable, BinaryVariable +from cmrl.utils.variables import Variable, ContinuousVariable, DiscreteVariable, BinaryVariable class BaseCausalMech(ABC): @@ -43,310 +43,3 @@ def learn(self, train_loader: DataLoader, valid_loader: DataLoader, **kwargs): @abstractmethod def forward(self, inputs: MutableMapping[str, torch.Tensor]) -> Dict[str, torch.Tensor]: raise NotImplementedError - - -class NeuralCausalMech(BaseCausalMech): - def __init__( - self, - name: str, - input_variables: List[Variable], - output_variables: List[Variable], - ensemble_num: int = 7, - elite_num: int = 5, - # cfgs - network_cfg: Optional[DictConfig] = None, - encoder_cfg: Optional[DictConfig] = None, - decoder_cfg: Optional[DictConfig] = None, - optimizer_cfg: Optional[DictConfig] = None, - # forward method - residual: bool = True, - encoder_reduction: str = "sum", - multi_step: str = "none", - # logger - logger: Optional[Logger] = None, - # others - device: Union[str, torch.device] = "cpu", - **kwargs - ): - super(NeuralCausalMech, self).__init__( - name=name, - input_variables=input_variables, - output_variables=output_variables, - device=device, - ) - self.ensemble_num = ensemble_num - self.elite_num = elite_num - # cfgs - self.network_cfg = network_cfg - self.encoder_cfg = encoder_cfg - self.decoder_cfg = decoder_cfg - self.optimizer_cfg = optimizer_cfg - # forward method - self.residual = residual - self.encoder_reduction = encoder_reduction - self.multi_step = multi_step - # logger - self.logger = logger - - # build member object - self.variable_encoders: Optional[Dict[str, VariableEncoder]] = None - self.variable_decoders: Optional[Dict[str, VariableEncoder]] = None - self.network: Optional[BaseNetwork] = None - self.graph: Optional[BaseGraph] = None - self.optimizer: Optional[Optimizer] = None - self.build_coder() - self.build_network() - self.build_graph() - self.build_optimizer() - - self.total_epoch = 0 - self.elite_indices: List[int] = [] - - @abstractmethod - def single_step_forward(self, inputs: MutableMapping[str, torch.Tensor]) -> Dict[str, torch.Tensor]: - raise NotImplementedError - - def forward(self, inputs: MutableMapping[str, torch.Tensor]) -> Dict[str, torch.Tensor]: - if self.multi_step.startswith("forward-euler"): - step_num = int(self.multi_step.split()[-1]) - - outputs = {} - for step in range(step_num): - outputs = self.single_step_forward(inputs) - if step < step_num - 1: - for name in filter(lambda s: s.startswith("obs"), inputs.keys()): - assert inputs[name].shape[:2] == outputs["next_{}".format(name)].shape[:2] - assert inputs[name].shape[2] * 2 == outputs["next_{}".format(name)].shape[2] - inputs[name] = outputs["next_{}".format(name)][:, :, : inputs[name].shape[2]] - else: - raise NotImplementedError("multi-step method {} is not supported".format(self.multi_step)) - - return outputs - - @abstractmethod - def build_network(self): - raise NotImplementedError - - def build_optimizer(self): - assert self.network is not None, "you must build network first" - params = ( - [self.network.parameters()] - + [encoder.parameters() for encoder in self.variable_encoders.values()] - + [decoder.parameters() for decoder in self.variable_decoders.values()] - ) - - self.optimizer = instantiate(self.optimizer_cfg)(params=chain(*params)) - - @abstractmethod - def build_graph(self): - raise NotImplementedError - - def build_coder(self): - self.variable_encoders = {} - for var in self.input_variables: - assert var.name not in self.variable_encoders, "duplicate name in encoders: {}".format(var.name) - self.variable_encoders[var.name] = instantiate(self.encoder_cfg)(variable=var).to(self.device) - - assert self.decoder_input_dim - - self.variable_decoders = {} - for var in self.output_variables: - assert var.name not in self.variable_decoders, "duplicate name in decoders: {}".format(var.name) - self.variable_decoders[var.name] = instantiate(self.decoder_cfg)(variable=var).to(self.device) - - def loss(self, outputs, targets): - ensemble_num, batch_size = list(targets.values())[0].shape[:2] - total_loss = torch.zeros(ensemble_num, batch_size, self.output_var_num) - for i, var in enumerate(self.output_variables): - output = outputs[var.name] - target = targets[var.name].to(self.device) - if isinstance(var, ContinuousVariable): - dim = target.shape[-1] # ensemble-num, batch-size, dim - assert output.shape[-1] == 2 * dim - mean, log_var = output[:, :, :dim], output[:, :, dim:] - loss = F.gaussian_nll_loss(mean, target, log_var.exp(), reduction="none").mean(dim=-1) - total_loss[..., i] = loss - elif isinstance(var, DiscreteVariable): - # TODO: onehot to int? - raise NotImplementedError - total_loss[..., i] = F.cross_entropy(output, target, reduction="none") - elif isinstance(var, BinaryVariable): - total_loss[..., i] = F.binary_cross_entropy(output, target, reduction="none") - else: - raise NotImplementedError - return total_loss - - def get_inputs_batch_size(self, inputs: MutableMapping[str, torch.Tensor]) -> int: - assert len(set(inputs.keys()) & set(self.variable_encoders.keys())) == len(inputs) - data_shape = list(inputs.values())[0].shape - assert len(data_shape) == 3, "{}".format(data_shape) # ensemble-num, batch-size, specific-dim - ensemble, batch_size, specific_dim = data_shape - assert ensemble == self.ensemble_num - - return batch_size - - def residual_outputs( - self, - inputs: MutableMapping[str, torch.Tensor], - outputs: MutableMapping[str, torch.Tensor], - ) -> MutableMapping[str, torch.Tensor]: - for name in filter(lambda s: s.startswith("obs"), inputs.keys()): - assert inputs[name].shape[:2] == outputs["next_{}".format(name)].shape[:2] - assert inputs[name].shape[2] * 2 == outputs["next_{}".format(name)].shape[2] - outputs["next_{}".format(name)][:, :, : inputs[name].shape[2]] += inputs[name].to(self.device) - return outputs - - def train(self, loader: DataLoader): - """train for ensemble data - - Args: - loader: train data-loader. - - Returns: tensor of train loss, with shape (ensemble-num, batch-size). - - """ - batch_loss_list = [] - for inputs, targets in loader: - outputs = self.forward(inputs) - loss = self.loss(outputs, targets) # ensemble-num, batch-size, output-var-num - - self.optimizer.zero_grad() - loss.mean().backward() - self.optimizer.step() - - batch_loss_list.append(loss) - return torch.cat(batch_loss_list, dim=-2).detach().cpu() - - def eval(self, loader: DataLoader): - """evaluate for non-ensemble data - - Args: - loader: valid data-loader. - - Returns: tensor of eval loss, with shape (batch-size). - - """ - batch_loss_list = [] - with torch.no_grad(): - for inputs, targets in loader: - outputs = self.forward(inputs) - loss = self.loss(outputs, targets) # ensemble-num, batch-size, output-var-num - - batch_loss_list.append(loss) - return torch.cat(batch_loss_list, dim=-2).detach().cpu() - - def learn( - self, - # loader - train_loader: DataLoader, - valid_loader: DataLoader, - # model learning - longest_epoch: int = -1, - improvement_threshold: float = 0.01, - patience: int = 5, - work_dir: Optional[Union[str, pathlib.Path]] = None, - **kwargs - ): - best_weights: Optional[Dict] = None - epoch_iter = range(longest_epoch) if longest_epoch >= 0 else itertools.count() - epochs_since_update = 0 - best_eval_loss = self.eval(valid_loader).mean(dim=(1, 2)) - - for epoch in epoch_iter: - train_loss = self.train(train_loader) - eval_loss = self.eval(valid_loader).mean(dim=(1, 2)) - maybe_best_weights = self._maybe_get_best_weights(best_eval_loss, eval_loss, improvement_threshold) - if maybe_best_weights: - # best loss - best_eval_loss = torch.minimum(best_eval_loss, eval_loss) - best_weights = maybe_best_weights - epochs_since_update = 0 - else: - epochs_since_update += 1 - - # log - self.total_epoch += 1 - if self.logger is not None: - self.logger.record("{}/epoch".format(self.name), epoch) - self.logger.record("{}/train_dataset_size".format(self.name), len(train_loader.dataset)) - self.logger.record("{}/valid_dataset_size".format(self.name), len(valid_loader.dataset)) - self.logger.record("{}/train_loss".format(self.name), train_loss.mean().item()) - self.logger.record("{}/val_loss".format(self.name), eval_loss.mean().item()) - self.logger.record("{}/best_val_loss".format(self.name), best_eval_loss.mean().item()) - - self.logger.dump(self.total_epoch) - - if patience and epochs_since_update >= patience: - break - - # saving the best models: - self._maybe_set_best_weights_and_elite(best_weights, best_eval_loss) - - def _maybe_get_best_weights( - self, - best_val_loss: torch.Tensor, - val_loss: torch.Tensor, - threshold: float = 0.01, - ) -> Optional[Dict]: - """Return the current model state dict if the validation score improves. - For ensembles, this checks the validation for each ensemble member separately. - Copy from https://github.com/facebookresearch/mbrl-lib/blob/main/mbrl/models/model_trainer.py - - Args: - best_val_score (tensor): the current best validation losses per model. - val_score (tensor): the new validation loss per model. - threshold (float): the threshold for relative improvement. - Returns: - (dict, optional): if the validation score's relative improvement over the - best validation score is higher than the threshold, returns the state dictionary - of the stored model, otherwise returns ``None``. - """ - improvement = (best_val_loss - val_loss) / torch.abs(best_val_loss) - if (improvement > threshold).any().item(): - best_weights = copy.deepcopy(self.network.state_dict()) - else: - best_weights = None - - return best_weights - - def _maybe_set_best_weights_and_elite(self, best_weights: Optional[Dict], best_val_score: torch.Tensor): - if best_weights is not None: - self.network.load_state_dict(best_weights) - - sorted_indices = np.argsort(best_val_score.tolist()) - self.elite_indices = sorted_indices[: self.elite_num] - - @property - def encoder_output_dim(self): - return self.encoder_cfg.output_dim - - @property - def union_output_var_dim(self): - # all output variables should be ContinuousVariable and have same variable.dim - output_dim = [] - for var in self.output_variables: - assert isinstance(var, ContinuousVariable), "all output variables should be ContinuousVariable" - output_dim.append(var.dim) - assert len(set(output_dim)) == 1, "all output variables should have same variable.dim" - return output_dim[0] - - @property - def decoder_input_dim(self): - if self.decoder_cfg.identity: - return self.union_output_var_dim * 2 - else: - return self.decoder_cfg.input_dim - - def reduce_encoder_output(self, encoder_output: torch.Tensor) -> torch.Tensor: - assert len(encoder_output.shape) == 4, ( - "shape of encoder_output should be (ensemble-num, batch-size, input-cvar-num, encoder-output-dim), " - "rather than {}".format(encoder_output.shape) - ) - if self.encoder_reduction == "sum": - return encoder_output.sum(-2) - elif self.encoder_reduction == "mean": - return encoder_output.mean(-2) - elif self.encoder_reduction == "sum": - return encoder_output.sum(-2) - else: - raise NotImplementedError("not implemented encoder reduction method: {}".format(self.encoder_reduction)) diff --git a/cmrl/models/causal_mech/neural_causal_mech.py b/cmrl/models/causal_mech/neural_causal_mech.py new file mode 100644 index 0000000..352b58f --- /dev/null +++ b/cmrl/models/causal_mech/neural_causal_mech.py @@ -0,0 +1,374 @@ +from typing import Optional, List, Dict, Union, MutableMapping +from abc import abstractmethod, ABC +from itertools import chain +import pathlib +import itertools +import copy + +import torch +import numpy as np +from torch.utils.data import DataLoader +import torch.nn.functional as F +from torch.optim import Optimizer +from stable_baselines3.common.logger import Logger +from omegaconf import DictConfig +from hydra.utils import instantiate + +from cmrl.models.causal_mech.base_causal_mech import BaseCausalMech +from cmrl.models.networks.base_network import BaseNetwork +from cmrl.models.graphs.base_graph import BaseGraph +from cmrl.models.networks.coder import VariableEncoder, VariableDecoder +from cmrl.utils.variables import Variable, ContinuousVariable, DiscreteVariable, BinaryVariable + +default_network_cfg = DictConfig( + dict( + _target_="cmrl.models.networks.ParallelMLP", + _partial_=True, + _recursive_=False, + hidden_dims=[200, 200], + bias=True, + activation_fn_cfg=dict(_target_="torch.nn.SiLU"), + ) +) + +default_encoder_cfg = DictConfig( + dict( + _target_="cmrl.models.networks.VariableEncoder", + _partial_=True, + _recursive_=False, + output_dim=100, + hidden_dims=[100], + bias=True, + activation_fn_cfg=dict(_target_="torch.nn.SiLU"), + ) +) + +default_decoder_cfg = DictConfig( + dict( + _target_="cmrl.models.networks.VariableDecoder", + _partial_=True, + _recursive_=False, + input_dim=100, + hidden_dims=[100], + bias=True, + identity=False, + activation_fn_cfg=dict(_target_="torch.nn.SiLU"), + ) +) + +default_optimizer_cfg = DictConfig( + dict( + _target_="torch.optim.Adam", + _partial_=True, + lr=1e-4, + weight_decay=1e-5, + eps=1e-8, + ) +) + + +class NeuralCausalMech(BaseCausalMech): + def __init__( + self, + name: str, + input_variables: List[Variable], + output_variables: List[Variable], + ensemble_num: int = 7, + elite_num: int = 5, + # cfgs + network_cfg: Optional[DictConfig] = None, + encoder_cfg: Optional[DictConfig] = None, + decoder_cfg: Optional[DictConfig] = None, + optimizer_cfg: Optional[DictConfig] = None, + # forward method + residual: bool = True, + encoder_reduction: str = "sum", + multi_step: str = "none", + # logger + logger: Optional[Logger] = None, + # others + device: Union[str, torch.device] = "cpu", + **kwargs + ): + super(NeuralCausalMech, self).__init__( + name=name, + input_variables=input_variables, + output_variables=output_variables, + device=device, + ) + self.ensemble_num = ensemble_num + self.elite_num = elite_num + # cfgs + self.network_cfg = default_network_cfg if network_cfg is None else network_cfg + self.encoder_cfg = default_encoder_cfg if encoder_cfg is None else encoder_cfg + self.decoder_cfg = default_decoder_cfg if decoder_cfg is None else decoder_cfg + self.optimizer_cfg = default_optimizer_cfg if optimizer_cfg is None else optimizer_cfg + # forward method + self.residual = residual + self.encoder_reduction = encoder_reduction + self.multi_step = multi_step + # logger + self.logger = logger + + # build member object + self.variable_encoders: Optional[Dict[str, VariableEncoder]] = None + self.variable_decoders: Optional[Dict[str, VariableEncoder]] = None + self.network: Optional[BaseNetwork] = None + self.graph: Optional[BaseGraph] = None + self.optimizer: Optional[Optimizer] = None + self.build_coder() + self.build_network() + self.build_graph() + self.build_optimizer() + + self.total_epoch = 0 + self.elite_indices: List[int] = [] + + @abstractmethod + def single_step_forward(self, inputs: MutableMapping[str, torch.Tensor]) -> Dict[str, torch.Tensor]: + raise NotImplementedError + + def forward(self, inputs: MutableMapping[str, torch.Tensor]) -> Dict[str, torch.Tensor]: + if self.multi_step.startswith("forward-euler"): + step_num = int(self.multi_step.split()[-1]) + + outputs = {} + for step in range(step_num): + outputs = self.single_step_forward(inputs) + if step < step_num - 1: + for name in filter(lambda s: s.startswith("obs"), inputs.keys()): + assert inputs[name].shape[:2] == outputs["next_{}".format(name)].shape[:2] + assert inputs[name].shape[2] * 2 == outputs["next_{}".format(name)].shape[2] + inputs[name] = outputs["next_{}".format(name)][:, :, : inputs[name].shape[2]] + else: + raise NotImplementedError("multi-step method {} is not supported".format(self.multi_step)) + + return outputs + + @abstractmethod + def build_network(self): + raise NotImplementedError + + def build_optimizer(self): + assert self.network is not None, "you must build network first" + params = ( + [self.network.parameters()] + + [encoder.parameters() for encoder in self.variable_encoders.values()] + + [decoder.parameters() for decoder in self.variable_decoders.values()] + ) + + self.optimizer = instantiate(self.optimizer_cfg)(params=chain(*params)) + + @abstractmethod + def build_graph(self): + raise NotImplementedError + + def build_coder(self): + self.variable_encoders = {} + for var in self.input_variables: + assert var.name not in self.variable_encoders, "duplicate name in encoders: {}".format(var.name) + self.variable_encoders[var.name] = instantiate(self.encoder_cfg)(variable=var).to(self.device) + + assert self.decoder_input_dim + + self.variable_decoders = {} + for var in self.output_variables: + assert var.name not in self.variable_decoders, "duplicate name in decoders: {}".format(var.name) + self.variable_decoders[var.name] = instantiate(self.decoder_cfg)(variable=var).to(self.device) + + def loss(self, outputs, targets): + ensemble_num, batch_size = list(targets.values())[0].shape[:2] + total_loss = torch.zeros(ensemble_num, batch_size, self.output_var_num) + for i, var in enumerate(self.output_variables): + output = outputs[var.name] + target = targets[var.name].to(self.device) + if isinstance(var, ContinuousVariable): + dim = target.shape[-1] # ensemble-num, batch-size, dim + assert output.shape[-1] == 2 * dim + mean, log_var = output[:, :, :dim], output[:, :, dim:] + loss = F.gaussian_nll_loss(mean, target, log_var.exp(), reduction="none").mean(dim=-1) + total_loss[..., i] = loss + elif isinstance(var, DiscreteVariable): + # TODO: onehot to int? + raise NotImplementedError + total_loss[..., i] = F.cross_entropy(output, target, reduction="none") + elif isinstance(var, BinaryVariable): + total_loss[..., i] = F.binary_cross_entropy(output, target, reduction="none") + else: + raise NotImplementedError + return total_loss + + def get_inputs_batch_size(self, inputs: MutableMapping[str, torch.Tensor]) -> int: + assert len(set(inputs.keys()) & set(self.variable_encoders.keys())) == len(inputs) + data_shape = list(inputs.values())[0].shape + assert len(data_shape) == 3, "{}".format(data_shape) # ensemble-num, batch-size, specific-dim + ensemble, batch_size, specific_dim = data_shape + assert ensemble == self.ensemble_num + + return batch_size + + def residual_outputs( + self, + inputs: MutableMapping[str, torch.Tensor], + outputs: MutableMapping[str, torch.Tensor], + ) -> MutableMapping[str, torch.Tensor]: + for name in filter(lambda s: s.startswith("obs"), inputs.keys()): + assert inputs[name].shape[:2] == outputs["next_{}".format(name)].shape[:2] + assert inputs[name].shape[2] * 2 == outputs["next_{}".format(name)].shape[2] + outputs["next_{}".format(name)][:, :, : inputs[name].shape[2]] += inputs[name].to(self.device) + return outputs + + def train(self, loader: DataLoader): + """train for ensemble data + + Args: + loader: train data-loader. + + Returns: tensor of train loss, with shape (ensemble-num, batch-size). + + """ + batch_loss_list = [] + for inputs, targets in loader: + outputs = self.forward(inputs) + loss = self.loss(outputs, targets) # ensemble-num, batch-size, output-var-num + + self.optimizer.zero_grad() + loss.mean().backward() + self.optimizer.step() + + batch_loss_list.append(loss) + return torch.cat(batch_loss_list, dim=-2).detach().cpu() + + def eval(self, loader: DataLoader): + """evaluate for non-ensemble data + + Args: + loader: valid data-loader. + + Returns: tensor of eval loss, with shape (batch-size). + + """ + batch_loss_list = [] + with torch.no_grad(): + for inputs, targets in loader: + outputs = self.forward(inputs) + loss = self.loss(outputs, targets) # ensemble-num, batch-size, output-var-num + + batch_loss_list.append(loss) + return torch.cat(batch_loss_list, dim=-2).detach().cpu() + + def learn( + self, + # loader + train_loader: DataLoader, + valid_loader: DataLoader, + # model learning + longest_epoch: int = -1, + improvement_threshold: float = 0.01, + patience: int = 5, + work_dir: Optional[Union[str, pathlib.Path]] = None, + **kwargs + ): + best_weights: Optional[Dict] = None + epoch_iter = range(longest_epoch) if longest_epoch >= 0 else itertools.count() + epochs_since_update = 0 + best_eval_loss = self.eval(valid_loader).mean(dim=(1, 2)) + + for epoch in epoch_iter: + train_loss = self.train(train_loader) + eval_loss = self.eval(valid_loader).mean(dim=(1, 2)) + maybe_best_weights = self._maybe_get_best_weights(best_eval_loss, eval_loss, improvement_threshold) + if maybe_best_weights: + # best loss + best_eval_loss = torch.minimum(best_eval_loss, eval_loss) + best_weights = maybe_best_weights + epochs_since_update = 0 + else: + epochs_since_update += 1 + + # log + self.total_epoch += 1 + if self.logger is not None: + self.logger.record("{}/epoch".format(self.name), epoch) + self.logger.record("{}/train_dataset_size".format(self.name), len(train_loader.dataset)) + self.logger.record("{}/valid_dataset_size".format(self.name), len(valid_loader.dataset)) + self.logger.record("{}/train_loss".format(self.name), train_loss.mean().item()) + self.logger.record("{}/val_loss".format(self.name), eval_loss.mean().item()) + self.logger.record("{}/best_val_loss".format(self.name), best_eval_loss.mean().item()) + + self.logger.dump(self.total_epoch) + + if patience and epochs_since_update >= patience: + break + + # saving the best models: + self._maybe_set_best_weights_and_elite(best_weights, best_eval_loss) + + def _maybe_get_best_weights( + self, + best_val_loss: torch.Tensor, + val_loss: torch.Tensor, + threshold: float = 0.01, + ) -> Optional[Dict]: + """Return the current model state dict if the validation score improves. + For ensembles, this checks the validation for each ensemble member separately. + Copy from https://github.com/facebookresearch/mbrl-lib/blob/main/mbrl/models/model_trainer.py + + Args: + best_val_score (tensor): the current best validation losses per model. + val_score (tensor): the new validation loss per model. + threshold (float): the threshold for relative improvement. + Returns: + (dict, optional): if the validation score's relative improvement over the + best validation score is higher than the threshold, returns the state dictionary + of the stored model, otherwise returns ``None``. + """ + improvement = (best_val_loss - val_loss) / torch.abs(best_val_loss) + if (improvement > threshold).any().item(): + best_weights = copy.deepcopy(self.network.state_dict()) + else: + best_weights = None + + return best_weights + + def _maybe_set_best_weights_and_elite(self, best_weights: Optional[Dict], best_val_score: torch.Tensor): + if best_weights is not None: + self.network.load_state_dict(best_weights) + + sorted_indices = np.argsort(best_val_score.tolist()) + self.elite_indices = sorted_indices[: self.elite_num] + + @property + def encoder_output_dim(self): + return self.encoder_cfg.output_dim + + @property + def union_output_var_dim(self): + # all output variables should be ContinuousVariable and have same variable.dim + output_dim = [] + for var in self.output_variables: + assert isinstance(var, ContinuousVariable), "all output variables should be ContinuousVariable" + output_dim.append(var.dim) + assert len(set(output_dim)) == 1, "all output variables should have same variable.dim" + return output_dim[0] + + @property + def decoder_input_dim(self): + if self.decoder_cfg.identity: + return self.union_output_var_dim * 2 + else: + return self.decoder_cfg.input_dim + + def reduce_encoder_output(self, encoder_output: torch.Tensor) -> torch.Tensor: + assert len(encoder_output.shape) == 4, ( + "shape of encoder_output should be (ensemble-num, batch-size, input-cvar-num, encoder-output-dim), " + "rather than {}".format(encoder_output.shape) + ) + if self.encoder_reduction == "sum": + return encoder_output.sum(-2) + elif self.encoder_reduction == "mean": + return encoder_output.mean(-2) + elif self.encoder_reduction == "sum": + return encoder_output.sum(-2) + else: + raise NotImplementedError("not implemented encoder reduction method: {}".format(self.encoder_reduction)) diff --git a/cmrl/models/causal_mech/plain_mech.py b/cmrl/models/causal_mech/plain_mech.py index 89a04eb..560776a 100644 --- a/cmrl/models/causal_mech/plain_mech.py +++ b/cmrl/models/causal_mech/plain_mech.py @@ -5,8 +5,8 @@ from hydra.utils import instantiate from stable_baselines3.common.logger import Logger -from cmrl.utils.types import Variable -from cmrl.models.causal_mech.base_causal_mech import NeuralCausalMech +from cmrl.utils.variables import Variable +from cmrl.models.causal_mech.neural_causal_mech import NeuralCausalMech class PlainMech(NeuralCausalMech): diff --git a/cmrl/models/data_loader.py b/cmrl/models/data_loader.py index c5fcc0f..062b905 100644 --- a/cmrl/models/data_loader.py +++ b/cmrl/models/data_loader.py @@ -6,7 +6,7 @@ import numpy as np from stable_baselines3.common.buffers import ReplayBuffer, DictReplayBuffer -from cmrl.models.util import space2dict +from cmrl.utils.variables import space2dict class BufferDataset(Dataset): diff --git a/cmrl/models/dynamics.py b/cmrl/models/dynamics.py index 3def165..f8acb5a 100644 --- a/cmrl/models/dynamics.py +++ b/cmrl/models/dynamics.py @@ -10,27 +10,27 @@ from stable_baselines3.common.logger import Logger from stable_baselines3.common.buffers import ReplayBuffer -from cmrl.models.util import space2dict -from cmrl.models.causal_mech.base_causal_mech import NeuralCausalMech +from cmrl.utils.variables import space2dict +from cmrl.models.causal_mech.base_causal_mech import BaseCausalMech from cmrl.models.data_loader import BufferDataset, EnsembleBufferDataset, collate_fn class Dynamics: def __init__( self, - transition: NeuralCausalMech, - reward_mech: Optional[NeuralCausalMech], - termination_mech: Optional[NeuralCausalMech], + transition: BaseCausalMech, observation_space: spaces.Space, action_space: spaces.Space, + reward_mech: Optional[BaseCausalMech] = None, + termination_mech: Optional[BaseCausalMech] = None, seed: int = 7, logger: Optional[Logger] = None, ): self.transition = transition - self.reward_mech = reward_mech - self.termination_mech = termination_mech self.observation_space = observation_space self.action_space = action_space + self.reward_mech = reward_mech + self.termination_mech = termination_mech self.seed = seed self.logger = logger diff --git a/cmrl/models/fake_env.py b/cmrl/models/fake_env.py index e576dfd..8cb0b3a 100644 --- a/cmrl/models/fake_env.py +++ b/cmrl/models/fake_env.py @@ -6,7 +6,7 @@ from stable_baselines3.common.logger import Logger from stable_baselines3.common.buffers import ReplayBuffer -import cmrl.utils.types +from cmrl.types import RewardFnType, TermFnType, InitObsFnType from cmrl.models.dynamics import Dynamics @@ -19,9 +19,9 @@ def __init__( action_space: gym.spaces.Space, # for dynamics dynamics: Dynamics, - reward_fn: Optional[cmrl.utils.types.RewardFnType] = None, - termination_fn: Optional[cmrl.utils.types.TermFnType] = None, - get_init_obs_fn: Optional[cmrl.utils.types.InitObsFnType] = None, + reward_fn: Optional[RewardFnType] = None, + termination_fn: Optional[TermFnType] = None, + get_init_obs_fn: Optional[InitObsFnType] = None, real_replay_buffer: Optional[ReplayBuffer] = None, # for offline penalty_coeff: float = 0.0, diff --git a/cmrl/models/networks/coder.py b/cmrl/models/networks/coder.py index f9635a9..2f8935a 100644 --- a/cmrl/models/networks/coder.py +++ b/cmrl/models/networks/coder.py @@ -3,7 +3,7 @@ import torch.nn as nn from omegaconf import DictConfig -from cmrl.utils.types import Variable, DiscreteVariable, ContinuousVariable, BinaryVariable +from cmrl.utils.variables import Variable, DiscreteVariable, ContinuousVariable, BinaryVariable from cmrl.models.networks.base_network import BaseNetwork, create_activation diff --git a/cmrl/models/networks/parallel_mlp.py b/cmrl/models/networks/parallel_mlp.py index b842fc9..06d7c88 100644 --- a/cmrl/models/networks/parallel_mlp.py +++ b/cmrl/models/networks/parallel_mlp.py @@ -19,7 +19,7 @@ def __init__( output_dim: int, extra_dims: Optional[List[int]] = None, hidden_dims: Optional[List[int]] = None, - use_bias: bool = True, + bias: bool = True, init_type: str = "truncated_normal", activation_fn_cfg: Optional[DictConfig] = None, **kwargs @@ -28,7 +28,7 @@ def __init__( self.output_dim = output_dim self.extra_dims = extra_dims if extra_dims is not None else [] self.hidden_dims = hidden_dims if hidden_dims is not None else [200, 200, 200, 200] - self.use_bias = use_bias + self.bias = bias self.init_type = init_type self.activation_fn_cfg = activation_fn_cfg @@ -41,13 +41,13 @@ def build(self): for i in range(len(hidden_dims) - 1): layers += [ ParallelLinear( - input_dim=hidden_dims[i], output_dim=hidden_dims[i + 1], extra_dims=self.extra_dims, use_bias=self.use_bias + input_dim=hidden_dims[i], output_dim=hidden_dims[i + 1], extra_dims=self.extra_dims, use_bias=self.bias ) ] layers += [create_activation(self.activation_fn_cfg)] layers += [ ParallelLinear( - input_dim=hidden_dims[-1], output_dim=self.output_dim, extra_dims=self.extra_dims, use_bias=self.use_bias + input_dim=hidden_dims[-1], output_dim=self.output_dim, extra_dims=self.extra_dims, use_bias=self.bias ) ] diff --git a/cmrl/models/util.py b/cmrl/models/util.py index 8951ee4..a2c63ef 100644 --- a/cmrl/models/util.py +++ b/cmrl/models/util.py @@ -4,7 +4,7 @@ import torch from gym import spaces -from cmrl.utils.types import Variable, ContinuousVariable, DiscreteVariable, BinaryVariable +from cmrl.utils.variables import Variable, ContinuousVariable, DiscreteVariable, BinaryVariable # inplace truncated normal function for pytorch. @@ -29,59 +29,3 @@ def truncated_normal_(tensor: torch.Tensor, mean: float = 0, std: float = 1) -> break tensor[cond] = torch.normal(mean, std, size=(bound_violations,), device=tensor.device) return tensor - - -def parse_space(space: spaces.Space, prefix="obs") -> List[Variable]: - variables = [] - if isinstance(space, spaces.Box): - for i, (low, high) in enumerate(zip(space.low, space.high)): - variables.append(ContinuousVariable(dim=1, low=low, high=high, name="{}_{}".format(prefix, i))) - elif isinstance(space, spaces.Discrete): - variables.append(DiscreteVariable(n=space.n, name="{}_0".format(prefix))) - elif isinstance(space, spaces.MultiDiscrete): - for i, n in enumerate(space.nvec): - variables.append(DiscreteVariable(n=n, name="{}_{}".format(prefix, i))) - elif isinstance(space, spaces.MultiBinary): - for i in range(space.n): - variables.append(BinaryVariable(name="{}_{}".format(prefix, i))) - elif isinstance(space, spaces.Dict): - # TODO - raise NotImplementedError - - return variables - - -def space2dict( - data: np.ndarray, - space: spaces.Space, - prefix="obs", - repeat: Optional[int] = None, - to_tensor: bool = False, - device: Union[str, torch.device] = "cpu", -) -> Dict[str, Union[np.ndarray, torch.Tensor]]: - if repeat: - assert repeat > 1, "repeat must be a int greater than 1" - - dict_data = {} - if isinstance(space, spaces.Box): # shape: (batch-size, node-num), every node has exactly one dim - for i, (low, high) in enumerate(zip(space.low, space.high)): - # shape: (batch-size, specific-dim) - dict_data["{}_{}".format(prefix, i)] = data[:, i, None].astype(np.float32) - else: - # TODO - raise NotImplementedError - - for name in dict_data: - if repeat: - # shape: (repeat-dim, batch-size, specific-dim) - dict_data[name] = np.tile(dict_data[name][None, :, :], [repeat, 1, 1]) - if to_tensor: - dict_data[name] = torch.from_numpy(dict_data[name]).to(device) - - return dict_data - - -def dict2space( - data: Dict[str, Union[np.ndarray, torch.Tensor]], space: spaces.Space -) -> Dict[str, Union[np.ndarray, torch.Tensor]]: - pass diff --git a/cmrl/sb3_extension/online_mb_callback.py b/cmrl/sb3_extension/online_mb_callback.py index 8161dbd..06f2655 100644 --- a/cmrl/sb3_extension/online_mb_callback.py +++ b/cmrl/sb3_extension/online_mb_callback.py @@ -1,5 +1,5 @@ import os -import warnings +import pathlib from typing import Any, Callable, Dict, List, Optional, Union from copy import deepcopy @@ -20,25 +20,36 @@ def __init__( env: gym.Env, dynamics: Dynamics, real_replay_buffer: ReplayBuffer, + # online RL total_online_timesteps: int = int(1e5), initial_exploration_steps: int = 1000, freq_train_model: int = 250, + # dynamics learning + longest_epoch: int = -1, + improvement_threshold: float = 0.01, + patience: int = 5, + work_dir: Optional[Union[str, pathlib.Path]] = None, device: str = "cpu", ): super(OnlineModelBasedCallback, self).__init__(verbose=2) self.env = DummyVecEnv([lambda: env]) self.dynamics = dynamics + self.real_replay_buffer = real_replay_buffer + # online RL self.total_online_timesteps = total_online_timesteps self.initial_exploration_steps = initial_exploration_steps self.freq_train_model = freq_train_model + # dynamics learning + self.longest_epoch = longest_epoch + self.improvement_threshold = improvement_threshold + self.patience = patience + self.work_dir = work_dir self.device = device self.action_space = env.action_space self.observation_space = env.observation_space - self.real_replay_buffer = real_replay_buffer - self.now_online_timesteps = 0 self._last_obs = None @@ -47,9 +58,15 @@ def _on_step(self) -> bool: # dump some residual log before dynamics learn self.model.logger.dump(step=self.num_timesteps) - self.dynamics.learn(self.real_replay_buffer) + self.dynamics.learn( + self.real_replay_buffer, + longest_epoch=self.longest_epoch, + improvement_threshold=self.improvement_threshold, + patience=self.patience, + work_dir=self.work_dir, + ) - self.step_and_add(explore=False) + self.step_and_add(explore=False) if self.now_online_timesteps >= self.total_online_timesteps: return False diff --git a/cmrl/types.py b/cmrl/types.py new file mode 100644 index 0000000..f65d340 --- /dev/null +++ b/cmrl/types.py @@ -0,0 +1,9 @@ +from typing import Callable, Optional, Tuple, Union + +import torch + +# (next_obs, pre_obs, action) -> reward +RewardFnType = Callable[[torch.Tensor, torch.Tensor, torch.Tensor], torch.Tensor] +# (next_obs, pre_obs, action) -> terminal +TermFnType = Callable[[torch.Tensor, torch.Tensor, torch.Tensor], torch.Tensor] +InitObsFnType = Callable[[int], torch.Tensor] diff --git a/cmrl/utils/config.py b/cmrl/utils/config.py index c07dba2..b62d9a4 100644 --- a/cmrl/utils/config.py +++ b/cmrl/utils/config.py @@ -1,7 +1,8 @@ import pathlib -from typing import Tuple, Union +from typing import Dict, Union, Optional import omegaconf +from omegaconf import DictConfig def load_hydra_cfg(results_dir: Union[str, pathlib.Path]) -> omegaconf.DictConfig: @@ -22,22 +23,3 @@ def load_hydra_cfg(results_dir: Union[str, pathlib.Path]) -> omegaconf.DictConfi if not isinstance(cfg, omegaconf.DictConfig): raise RuntimeError("Configuration format not a omegaconf.DictConf") return cfg - - -def get_complete_dynamics_cfg( - dynamics_cfg: omegaconf.DictConfig, - obs_shape: Tuple[int, ...], - act_shape: Tuple[int, ...], -): - transition_cfg = dynamics_cfg.transition - transition_cfg.obs_size = obs_shape[0] - transition_cfg.action_size = act_shape[0] - - reward_cfg = dynamics_cfg.reward_mech - reward_cfg.obs_size = obs_shape[0] - reward_cfg.action_size = act_shape[0] - - termination_cfg = dynamics_cfg.termination_mech - termination_cfg.obs_size = obs_shape[0] - termination_cfg.action_size = act_shape[0] - return dynamics_cfg diff --git a/cmrl/utils/creator.py b/cmrl/utils/creator.py index 96ac099..a6fffa6 100644 --- a/cmrl/utils/creator.py +++ b/cmrl/utils/creator.py @@ -1,4 +1,4 @@ -from typing import Optional, cast +from typing import Optional, cast, List from gym import spaces from hydra.utils import instantiate @@ -9,9 +9,8 @@ from cmrl.models.dynamics import Dynamics from cmrl.models.fake_env import VecFakeEnv -from cmrl.models.causal_mech.base_causal_mech import NeuralCausalMech -from cmrl.models.util import parse_space -from cmrl.utils.types import ContinuousVariable, BinaryVariable +from cmrl.models.causal_mech.base_causal_mech import BaseCausalMech +from cmrl.utils.variables import ContinuousVariable, BinaryVariable, DiscreteVariable, Variable, parse_space def create_agent(cfg: DictConfig, fake_env: VecFakeEnv, logger: Optional[Logger] = None): @@ -39,7 +38,7 @@ def create_dynamics( output_variables=next_obs_variables, logger=logger, ) - transition = cast(NeuralCausalMech, transition) + transition = cast(BaseCausalMech, transition) # reward mech assert cfg.reward_mech.mech.multi_step == "none", "reward-mech must be one-step" @@ -49,7 +48,7 @@ def create_dynamics( output_variables=[ContinuousVariable("reward", dim=1, low=-np.inf, high=np.inf)], logger=logger, ) - reward_mech = cast(NeuralCausalMech, reward_mech) + reward_mech = cast(BaseCausalMech, reward_mech) else: reward_mech = None @@ -61,7 +60,7 @@ def create_dynamics( output_variables=[BinaryVariable("terminal")], logger=logger, ) - termination_mech = cast(NeuralCausalMech, termination_mech) + termination_mech = cast(BaseCausalMech, termination_mech) else: termination_mech = None diff --git a/cmrl/utils/env.py b/cmrl/utils/env.py index 4ae7f4c..48bf4f1 100644 --- a/cmrl/utils/env.py +++ b/cmrl/utils/env.py @@ -1,10 +1,13 @@ from typing import Dict, Optional, Tuple, cast +import numpy as np import emei import gym import omegaconf +from stable_baselines3.common.buffers import ReplayBuffer -import cmrl.utils.types +import cmrl.utils.variables +from cmrl.types import TermFnType, RewardFnType, InitObsFnType def to_num(s): @@ -16,18 +19,13 @@ def to_num(s): def get_term_and_reward_fn( cfg: omegaconf.DictConfig, -) -> Tuple[cmrl.utils.types.TermFnType, Optional[cmrl.utils.types.RewardFnType]]: +) -> Tuple[Optional[TermFnType], Optional[RewardFnType]]: return None, None def make_env( cfg: omegaconf.DictConfig, -) -> Tuple[ - emei.EmeiEnv, - cmrl.utils.types.TermFnType, - Optional[cmrl.utils.types.RewardFnType], - Optional[cmrl.utils.types.InitObsFnType], -]: +) -> Tuple[emei.EmeiEnv, TermFnType, Optional[RewardFnType], Optional[InitObsFnType],]: if "gym___" in cfg.task.env: env = gym.make(cfg.task.env.split("___")[1]) term_fn, reward_fn = get_term_and_reward_fn(cfg) @@ -49,3 +47,25 @@ def make_env( env.observation_space.seed(cfg.seed + 1) env.action_space.seed(cfg.seed + 2) return env, reward_fn, term_fn, init_obs_fn + + +def load_offline_data(env, replay_buffer: ReplayBuffer, dataset_name: str, use_ratio: float = 1): + assert hasattr(env, "get_dataset"), "env must have `get_dataset` method" + + data_dict = env.get_dataset(dataset_name) + all_data_num = len(data_dict["observations"]) + sample_data_num = int(use_ratio * all_data_num) + sample_idx = np.random.permutation(all_data_num)[:sample_data_num] + + assert replay_buffer.n_envs == 1 + assert replay_buffer.buffer_size >= sample_data_num + + if sample_data_num == replay_buffer.buffer_size: + replay_buffer.full = True + replay_buffer.pos = 0 + else: + replay_buffer.pos = sample_data_num + + # set all data + for attr in ["observations", "next_observations", "actions", "rewards", "dones", "timeouts"]: + getattr(replay_buffer, attr)[:sample_data_num, 0] = data_dict[attr][sample_idx] diff --git a/cmrl/utils/transition_iterator.py b/cmrl/utils/transition_iterator.py deleted file mode 100644 index f4915ab..0000000 --- a/cmrl/utils/transition_iterator.py +++ /dev/null @@ -1,159 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. -from typing import Any, Optional, Sequence, Sized - -import numpy as np - -from cmrl.utils.types import InteractionBatch - - -def _consolidate_batches(batches: Sequence[InteractionBatch]) -> InteractionBatch: - len_batches = len(batches) - b0 = batches[0] - obs = np.empty((len_batches,) + b0.batch_obs.shape, dtype=b0.batch_obs.dtype) - act = np.empty((len_batches,) + b0.batch_action.shape, dtype=b0.batch_action.dtype) - next_obs = np.empty((len_batches,) + b0.batch_obs.shape, dtype=b0.batch_obs.dtype) - rewards = np.empty((len_batches,) + b0.batch_reward.shape, dtype=np.float32) - dones = np.empty((len_batches,) + b0.batch_done.shape, dtype=bool) - for i, b in enumerate(batches): - obs[i] = b.batch_obs - act[i] = b.batch_action - next_obs[i] = b.batch_next_obs - rewards[i] = b.batch_reward - dones[i] = b.batch_done - return InteractionBatch(obs, act, next_obs, rewards, dones) - - -class TransitionIterator: - """An iterator for batches of transitions. - - The iterator can be used doing: - - .. code-block:: python - - for batch in batch_iterator: - do_something_with_batch() - - Rather than be constructed directly, the preferred way to use objects of this class - is for the user to obtain them from :class:`ReplayBuffer`. - - Args: - transitions (:class:`InteractionBatch`): the transition data used to built - the iterator. - batch_size (int): the batch size to use when iterating over the stored data. - shuffle_each_epoch (bool): if ``True`` the iteration order is shuffled everytime a - loop over the data is completed. Defaults to ``False``. - rng (np.random.Generator, optional): a random number generator when sampling - batches. If None (default value), a new default generator will be used. - """ - - def __init__( - self, - transitions: InteractionBatch, - batch_size: int, - shuffle_each_epoch: bool = False, - rng: Optional[np.random.Generator] = None, - ): - self.transitions = transitions - self.num_stored = len(transitions) - self._order: np.ndarray = np.arange(self.num_stored) - self.batch_size = batch_size - self._current_batch = 0 - self._shuffle_each_epoch = shuffle_each_epoch - self._rng = rng if rng is not None else np.random.default_rng() - - def _get_indices_next_batch(self) -> Sized: - start_idx = self._current_batch * self.batch_size - if start_idx >= self.num_stored: - raise StopIteration - end_idx = min((self._current_batch + 1) * self.batch_size, self.num_stored) - order_indices = range(start_idx, end_idx) - indices = self._order[order_indices] - self._current_batch += 1 - return indices - - def __iter__(self): - self._current_batch = 0 - if self._shuffle_each_epoch: - self._order = self._rng.permutation(self.num_stored) - return self - - def __next__(self): - return self[self._get_indices_next_batch()] - - def ensemble_size(self): - return 0 - - def __len__(self): - return (self.num_stored - 1) // self.batch_size + 1 - - def __getitem__(self, item): - return self.transitions[item] - - -class BootstrapIterator(TransitionIterator): - def __init__( - self, - transitions: InteractionBatch, - batch_size: int, - ensemble_size: int, - shuffle_each_epoch: bool = False, - permute_indices: bool = True, - rng: Optional[np.random.Generator] = None, - ): - super().__init__(transitions, batch_size, shuffle_each_epoch=shuffle_each_epoch, rng=rng) - self._ensemble_size = ensemble_size - self._permute_indices = permute_indices - self._bootstrap_iter = ensemble_size > 1 - self.member_indices = self._sample_member_indices() - - def _sample_member_indices(self) -> np.ndarray: - member_indices = np.empty((self.ensemble_size, self.num_stored), dtype=int) - if self._permute_indices: - for i in range(self.ensemble_size): - member_indices[i] = self._rng.permutation(self.num_stored) - else: - member_indices = self._rng.choice( - self.num_stored, - size=(self.ensemble_size, self.num_stored), - replace=True, - ) - return member_indices - - def __iter__(self): - super().__iter__() - return self - - def __next__(self): - if not self._bootstrap_iter: - return super().__next__() - indices = self._get_indices_next_batch() - batches = [] - for member_idx in self.member_indices: - content_indices = member_idx[indices] - batches.append(self[content_indices]) - return _consolidate_batches(batches) - - def toggle_bootstrap(self): - """Toggles whether the iterator returns a batch per model or a single batch.""" - if self.ensemble_size > 1: - self._bootstrap_iter = not self._bootstrap_iter - - @property - def ensemble_size(self): - return self._ensemble_size - - -def _sequence_getitem_impl( - transitions: InteractionBatch, - batch_size: int, - sequence_length: int, - valid_starts: np.ndarray, - item: Any, -): - start_indices = valid_starts[item].repeat(sequence_length) - increment_array = np.tile(np.arange(sequence_length), len(item)) - full_trajectory_indices = start_indices + increment_array - return transitions[full_trajectory_indices].add_new_batch_dim(min(batch_size, len(item))) diff --git a/cmrl/utils/types.py b/cmrl/utils/types.py deleted file mode 100644 index 1582228..0000000 --- a/cmrl/utils/types.py +++ /dev/null @@ -1,103 +0,0 @@ -from dataclasses import dataclass -from typing import Callable, Optional, Tuple, Union - -import numpy as np -import torch - -# (next_obs, pre_obs, action) -> reward -RewardFnType = Callable[[torch.Tensor, torch.Tensor, torch.Tensor], torch.Tensor] -# (next_obs, pre_obs, action) -> terminal -TermFnType = Callable[[torch.Tensor, torch.Tensor, torch.Tensor], torch.Tensor] -InitObsFnType = Callable[[int], torch.Tensor] -ObsProcessFnType = Callable[[np.ndarray], np.ndarray] - -TensorType = Union[torch.Tensor, np.ndarray] -TrajectoryEvalFnType = Callable[[TensorType, torch.Tensor], torch.Tensor] -# obs, action, next_obs, reward, done -InteractionData = Tuple[TensorType, TensorType, TensorType, TensorType, TensorType] - - -@dataclass -class InteractionBatch: - """Represents a batch of transitions""" - - batch_obs: Optional[TensorType] - batch_action: Optional[TensorType] - batch_next_obs: Optional[TensorType] - batch_reward: Optional[TensorType] - batch_done: Optional[TensorType] - - @property - def attrs(self): - return [ - "batch_obs", - "batch_action", - "batch_next_obs", - "batch_reward", - "batch_done", - ] - - def __len__(self): - return self.batch_obs.shape[0] - - def as_tuple(self) -> InteractionData: - return ( - self.batch_obs, - self.batch_action, - self.batch_next_obs, - self.batch_reward, - self.batch_done, - ) - - def __getitem__(self, item): - return InteractionBatch( - self.batch_obs[item], - self.batch_action[item], - self.batch_next_obs[item], - self.batch_reward[item], - self.batch_done[item], - ) - - @staticmethod - def _get_new_shape(old_shape: Tuple[int, ...], batch_size: int): - new_shape = list((1,) + old_shape) - new_shape[0] = batch_size - new_shape[1] = old_shape[0] // batch_size - return tuple(new_shape) - - def add_new_batch_dim(self, batch_size: int): - if not len(self) % batch_size == 0: - raise ValueError("Current batch of transitions size is not a " "multiple of the new batch size. ") - return InteractionBatch( - self.batch_obs.reshape(self._get_new_shape(self.batch_obs.shape, batch_size)), - self.batch_action.reshape(self._get_new_shape(self.batch_action.shape, batch_size)), - self.batch_next_obs.reshape(self._get_new_shape(self.batch_obs.shape, batch_size)), - self.batch_reward.reshape(self._get_new_shape(self.batch_reward.shape, batch_size)), - self.batch_done.reshape(self._get_new_shape(self.batch_done.shape, batch_size)), - ) - - -ModelInput = Union[torch.Tensor, InteractionBatch] - - -@dataclass -class Variable: - name: str - pass - - -@dataclass -class ContinuousVariable(Variable): - dim: int - low: np.ndarray = None - high: np.ndarray = None - - -@dataclass -class BinaryVariable(Variable): - pass - - -@dataclass -class DiscreteVariable(Variable): - n: int diff --git a/cmrl/utils/variables.py b/cmrl/utils/variables.py new file mode 100644 index 0000000..1ecdc3d --- /dev/null +++ b/cmrl/utils/variables.py @@ -0,0 +1,85 @@ +from dataclasses import dataclass +from typing import Optional, Dict, Union, List + +from gym import spaces +import numpy as np +import torch + + +@dataclass +class Variable: + name: str + pass + + +@dataclass +class ContinuousVariable(Variable): + dim: int + low: np.ndarray = None + high: np.ndarray = None + + +@dataclass +class BinaryVariable(Variable): + pass + + +@dataclass +class DiscreteVariable(Variable): + n: int + + +def parse_space(space: spaces.Space, prefix="obs") -> List[Variable]: + variables = [] + if isinstance(space, spaces.Box): + for i, (low, high) in enumerate(zip(space.low, space.high)): + variables.append(ContinuousVariable(dim=1, low=low, high=high, name="{}_{}".format(prefix, i))) + elif isinstance(space, spaces.Discrete): + variables.append(DiscreteVariable(n=space.n, name="{}_0".format(prefix))) + elif isinstance(space, spaces.MultiDiscrete): + for i, n in enumerate(space.nvec): + variables.append(DiscreteVariable(n=n, name="{}_{}".format(prefix, i))) + elif isinstance(space, spaces.MultiBinary): + for i in range(space.n): + variables.append(BinaryVariable(name="{}_{}".format(prefix, i))) + elif isinstance(space, spaces.Dict): + # TODO + raise NotImplementedError + + return variables + + +def space2dict( + data: np.ndarray, + space: spaces.Space, + prefix="obs", + repeat: Optional[int] = None, + to_tensor: bool = False, + device: Union[str, torch.device] = "cpu", +) -> Dict[str, Union[np.ndarray, torch.Tensor]]: + if repeat: + assert repeat > 1, "repeat must be a int greater than 1" + + dict_data = {} + if isinstance(space, spaces.Box): # shape: (batch-size, node-num), every node has exactly one dim + for i, (low, high) in enumerate(zip(space.low, space.high)): + # shape: (batch-size, specific-dim) + dict_data["{}_{}".format(prefix, i)] = data[:, i, None].astype(np.float32) + else: + # TODO + raise NotImplementedError + + for name in dict_data: + if repeat: + # shape: (repeat-dim, batch-size, specific-dim) + dict_data[name] = np.tile(dict_data[name][None, :, :], [repeat, 1, 1]) + if to_tensor: + dict_data[name] = torch.from_numpy(dict_data[name]).to(device) + + return dict_data + + +def dict2space( + data: Dict[str, Union[np.ndarray, torch.Tensor]], space: spaces.Space +) -> Dict[str, Union[np.ndarray, torch.Tensor]]: + pass diff --git a/tests/constants.py b/tests/constants.py deleted file mode 100644 index 4ae3190..0000000 --- a/tests/constants.py +++ /dev/null @@ -1,125 +0,0 @@ -from omegaconf import DictConfig - -cfg = DictConfig( - { - "algorithm": { - "name": "off_dyna", - "freq_train_model": "${task.freq_train_model}", - "real_data_ratio": 0.0, - "sac_samples_action": True, - "num_eval_episodes": 5, - "dataset_size": 1000000, - "penalty_coeff": "${task.penalty_coeff}", - "agent": { - "_target_": "stable_baselines3.sac.SAC", - "policy": "MlpPolicy", - "env": { - "_target_": "cmrl.models.fake_env.VecFakeEnv", - "num_envs": 16, - "action_space": {"_target_": "gym.spaces.Box", "low": "???", "high": "???", "shape": "???"}, - "observation_space": {"_target_": "gym.spaces.Box", "low": "???", "high": "???", "shape": "???"}, - }, - "learning_starts": 0, - "batch_size": 256, - "tau": 0.005, - "gamma": 0.99, - "ent_coef": "auto", - "target_entropy": "auto", - "verbose": 0, - "seed": "${seed}", - "device": "${device}", - }, - }, - "dynamics": { - "name": "constraint_based_dynamics", - "multi_step": "${task.multi_step}", - "transition": { - "_target_": "cmrl.models.transition.ExternalMaskTransition", - "obs_size": "???", - "action_size": "???", - "deterministic": False, - "ensemble_num": "${task.ensemble_num}", - "elite_num": "${task.elite_num}", - "residual": True, - "learn_logvar_bounds": False, - "num_layers": 4, - "hid_size": 200, - "activation_fn_cfg": {"_target_": "torch.nn.SiLU"}, - "device": "${device}", - }, - "learned_reward": "${task.learning_reward}", - "reward_mech": {"_target_": "cmrl.models.reward_mech.BaseRewardMech", "obs_size": "???", "action_size": "???"}, - "learned_termination": "${task.learning_terminal}", - "termination_mech": { - "_target_": "cmrl.models.termination_mech.BaseTerminationMech", - "obs_size": "???", - "action_size": "???", - }, - "optim_lr": "${task.optim_lr}", - "weight_decay": "${task.weight_decay}", - "patience": "${task.patience}", - "batch_size": "${task.batch_size}", - "use_ratio": "${task.use_ratio}", - "validation_ratio": "${task.validation_ratio}", - "shuffle_each_epoch": "${task.shuffle_each_epoch}", - "bootstrap_permutes": "${task.bootstrap_permutes}", - "longest_epoch": "${task.longest_epoch}", - "improvement_threshold": "${task.improvement_threshold}", - "normalize": True, - "normalize_double_precision": True, - }, - "task": { - "env": "emei___BoundaryInvertedPendulumSwingUp-v0___" - "freq_rate=${task.freq_rate}&time_step=${task.time_step}___${task.dataset}", - "dataset": "SAC-expert-replay", - "freq_rate": 1, - "time_step": 0.02, - "num_steps": 1000, - "online_num_steps": 10000, - "epoch_length": 10000, - "n_eval_episodes": 8, - "eval_freq": 100, - "learning_reward": False, - "learning_terminal": False, - "ensemble_num": 7, - "elite_num": 5, - "multi_step": "none", - "oracle": True, - "cit_threshold": 0.02, - "test_freq": 100, - "update_causal_mask_ratio": 0.25, - "discovery_schedule": [1, 30, 250, 250], - "penalty_coeff": 0.5, - "use_ratio": 0.01, - "freq_train_model": 100, - "patience": 20, - "optim_lr": 0.0001, - "weight_decay": 1e-05, - "batch_size": 256, - "validation_ratio": 0.2, - "shuffle_each_epoch": True, - "bootstrap_permutes": False, - "longest_epoch": 10, - "improvement_threshold": 0.01, - "effective_model_rollouts_per_step": 50, - "rollout_schedule": [1, 15, 1, 1], - "num_sac_updates_per_step": 1, - "sac_updates_every_steps": 1, - "num_epochs_to_retain_sac_buffer": 1, - "sac_gamma": 0.99, - "sac_tau": 0.005, - "sac_alpha": 0.2, - "sac_policy": "Gaussian", - "sac_target_update_interval": 1, - "sac_automatic_entropy_tuning": True, - "sac_hidden_size": 256, - "sac_lr": 0.0003, - "sac_batch_size": 256, - "sac_target_entropy": -1, - }, - "seed": 0, - "device": "cpu", - "exp_name": "default", - "wandb": False, - } -) diff --git a/tests/test_algorithms/__init__.py b/tests/test_algorithms/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/tests/test_algorithms/test_offline/__init__.py b/tests/test_algorithms/test_offline/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/tests/test_algorithms/test_offline/test_off_dyna.py b/tests/test_algorithms/test_offline/test_off_dyna.py deleted file mode 100644 index fabe1b6..0000000 --- a/tests/test_algorithms/test_offline/test_off_dyna.py +++ /dev/null @@ -1,16 +0,0 @@ -import numpy as np -import torch - -from cmrl.algorithms.offline.off_dyna import train -from cmrl.utils.env import make_env - -from tests.constants import cfg - - -def test_off_dyna(): - env, term_fn, reward_fn, init_obs_fn = make_env(cfg) - test_env, *_ = make_env(cfg) - np.random.seed(cfg.seed) - torch.manual_seed(cfg.seed) - - train(env, test_env, term_fn, reward_fn, init_obs_fn, cfg) diff --git a/tests/test_algorithms/test_util.py b/tests/test_algorithms/test_util.py deleted file mode 100644 index 03d3ae4..0000000 --- a/tests/test_algorithms/test_util.py +++ /dev/null @@ -1,15 +0,0 @@ -from stable_baselines3.common.buffers import ReplayBuffer - -from cmrl.utils.env import make_env -from cmrl.algorithms.util import load_offline_data - -from tests.constants import cfg - - -def test_load_offline_data(): - env, term_fn, reward_fn, init_obs_fn = make_env(cfg) - replay_buffer = ReplayBuffer( - cfg.task.num_steps, env.observation_space, env.action_space, cfg.device, handle_timeout_termination=False - ) - - load_offline_data(cfg, env, replay_buffer) diff --git a/tests/test_examples/__init__.py b/tests/test_examples/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/tests/test_examples/test_main.py b/tests/test_examples/test_main.py deleted file mode 100644 index e69de29..0000000 diff --git a/tests/test_models/test_causal_discovery/__init__.py b/tests/test_models/test_causal_discovery/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/tests/test_models/test_causal_discovery/test_CMI_test.py b/tests/test_models/test_causal_discovery/test_CMI_test.py deleted file mode 100644 index 40064eb..0000000 --- a/tests/test_models/test_causal_discovery/test_CMI_test.py +++ /dev/null @@ -1,73 +0,0 @@ -import tempfile -from pathlib import Path -from unittest import TestCase - -import torch - -from cmrl.models.causal_discovery.CMI_test import TransitionConditionalMutualInformationTest - - -class TestTransitionConditionalMutualInformationTest(TestCase): - def setUp(self) -> None: - self.obs_size = 11 - self.action_size = 3 - self.num_layers = 4 - self.ensemble_num = 7 - self.hid_size = 200 - self.batch_size = 128 - self.device = "cuda" if torch.cuda.is_available() else "cpu" - - self.cmi_test = TransitionConditionalMutualInformationTest( - obs_size=self.obs_size, - action_size=self.action_size, - device=self.device, - num_layers=self.num_layers, - ensemble_num=self.ensemble_num, - hid_size=self.hid_size, - ) - self.batch_obs = torch.rand([self.ensemble_num, self.batch_size, self.obs_size]).to(self.device) - self.batch_action = torch.rand([self.ensemble_num, self.batch_size, self.action_size]).to(self.device) - - def test_parallel_deterministic_forward(self): - parallel_batch_obs = torch.rand( - [self.obs_size + self.action_size + 1, self.ensemble_num, self.batch_size, self.obs_size] - ).to(self.device) - mean, logvar = self.cmi_test.forward(parallel_batch_obs, self.batch_action) - assert ( - mean.shape - == logvar.shape - == (self.obs_size + self.action_size + 1, self.ensemble_num, self.batch_size, self.obs_size) - ) - - def test_gaussian_forward(self): - mean, logvar = self.cmi_test.forward(self.batch_obs, self.batch_action) - assert ( - mean.shape - == logvar.shape - == (self.obs_size + self.action_size + 1, self.ensemble_num, self.batch_size, self.obs_size) - ) - - def test_load(self): - tempdir = Path(tempfile.gettempdir()) - model_dir = tempdir / "temp_model" - if not model_dir.exists(): - model_dir.mkdir() - - mean, logvar = self.cmi_test.forward(self.batch_obs, self.batch_action) - self.cmi_test.save(model_dir) - - new_cmi_test = TransitionConditionalMutualInformationTest( - obs_size=self.obs_size, - action_size=self.action_size, - device=self.device, - num_layers=self.num_layers, - ensemble_num=self.ensemble_num, - hid_size=self.hid_size, - ) - - new_mean, new_logvar = new_cmi_test.forward(self.batch_obs, self.batch_action) - assert not (mean == new_mean).all() - - new_cmi_test.load(model_dir) - new_mean, new_logvar = new_cmi_test.forward(self.batch_obs, self.batch_action) - assert (mean == new_mean).all() diff --git a/tests/test_models/test_causal_mech/test_plain_mech.py b/tests/test_models/test_causal_mech/test_plain_mech.py index 2359092..58257b5 100644 --- a/tests/test_models/test_causal_mech/test_plain_mech.py +++ b/tests/test_models/test_causal_mech/test_plain_mech.py @@ -4,15 +4,14 @@ from cmrl.models.causal_mech.plain_mech import PlainMech from cmrl.models.data_loader import BufferDataset, EnsembleBufferDataset, collate_fn -from cmrl.models.util import parse_space, load_offline_data +from cmrl.utils.creator import parse_space +from cmrl.utils.env import load_offline_data def prepare(freq_rate): env = gym.make("BoundaryInvertedPendulumSwingUp-v0", freq_rate=freq_rate, time_step=0.02) - real_replay_buffer = ReplayBuffer( - int(1e5), env.observation_space, env.action_space, "cpu", handle_timeout_termination=False - ) + real_replay_buffer = ReplayBuffer(1000, env.observation_space, env.action_space, "cpu", handle_timeout_termination=False) load_offline_data(env, real_replay_buffer, "SAC-expert-replay", use_ratio=0.001) ensemble_num = 7 @@ -32,51 +31,31 @@ def prepare(freq_rate): ) valid_loader = DataLoader(valid_dataset, batch_size=8, collate_fn=collate_fn) - node_dim = 16 - input_variables = parse_space(env.observation_space, "obs") + parse_space(env.action_space, "act") - variable_encoders = create_encoders(input_variables, node_dim=node_dim) output_variables = parse_space(env.observation_space, "next_obs") - variable_decoders = create_decoders(output_variables, node_dim=node_dim) return input_variables, output_variables, train_loader, valid_loader def test_inv_pendulum_single_step(): - input_variables, output_variables, node_dim, variable_encoders, variable_decoders, train_loader, valid_loader = prepare( - freq_rate=1 - ) + input_variables, output_variables, train_loader, valid_loader = prepare(freq_rate=1) mech = PlainMech( name="test", input_variables=input_variables, output_variables=output_variables, - node_dim=node_dim, - # variable_encoders=variable_encoders, - # variable_decoders=variable_decoders, - variable_encoders=None, - variable_decoders=None, - multi_step="none", - # device="cuda" ) mech.learn(train_loader, valid_loader, longest_epoch=1) def test_inv_pendulum_multi_step(): - input_variables, output_variables, node_dim, variable_encoders, variable_decoders, train_loader, valid_loader = prepare( - freq_rate=2 - ) + input_variables, output_variables, train_loader, valid_loader = prepare(freq_rate=2) mech = PlainMech( name="test", input_variables=input_variables, output_variables=output_variables, - node_dim=node_dim, - # variable_encoders=variable_encoders, - # variable_decoders=variable_decoders, - variable_encoders=None, - variable_decoders=None, multi_step="forward-euler 2", ) diff --git a/tests/test_models/test_data_loader.py b/tests/test_models/test_data_loader.py index ab7ef1c..f9c35f3 100644 --- a/tests/test_models/test_data_loader.py +++ b/tests/test_models/test_data_loader.py @@ -5,7 +5,7 @@ from torch.utils.data import DataLoader from cmrl.models.data_loader import BufferDataset, EnsembleBufferDataset -from cmrl.models.util import load_offline_data +from cmrl.utils.env import load_offline_data def test_buffer_dataset(): diff --git a/tests/test_models/test_network/test_coder.py b/tests/test_models/test_network/test_coder.py index 8ee0e1a..88f3518 100644 --- a/tests/test_models/test_network/test_coder.py +++ b/tests/test_models/test_network/test_coder.py @@ -2,7 +2,7 @@ from torch.nn.functional import one_hot from cmrl.models.networks.coder import VariableEncoder, VariableDecoder -from cmrl.utils.types import ContinuousVariable, DiscreteVariable, BinaryVariable +from cmrl.utils.variables import ContinuousVariable, DiscreteVariable, BinaryVariable def test_continuous_encoder(): diff --git a/tests/test_sb3_extension/test_online_mb_callback.py b/tests/test_sb3_extension/test_online_mb_callback.py index 441d772..95a0a7f 100644 --- a/tests/test_sb3_extension/test_online_mb_callback.py +++ b/tests/test_sb3_extension/test_online_mb_callback.py @@ -7,33 +7,44 @@ from stable_baselines3.common.buffers import ReplayBuffer from cmrl.sb3_extension.online_mb_callback import OnlineModelBasedCallback +from cmrl.utils.creator import parse_space +from cmrl.models.causal_mech.plain_mech import PlainMech from cmrl.models.dynamics import Dynamics from cmrl.models.fake_env import VecFakeEnv def test_callback(): env = cast(emei.EmeiEnv, gym.make("BoundaryInvertedPendulumSwingUp-v0", freq_rate=1, time_step=0.02)) - term_fn = env.get_terminal reward_fn = env.get_reward - init_obs_fn = env.get_batch_init_obs + termination_fn = env.get_terminal + get_init_obs_fn = env.get_batch_init_obs - transition = PlainTransition(obs_size=5, action_size=1) + obs_variables = parse_space(env.observation_space, "obs") + act_variables = parse_space(env.action_space, "act") + next_obs_variables = parse_space(env.observation_space, "next_obs") - dynamics = PlainEnsembleDynamics( - transition=transition, - learned_reward=False, - reward_mech=reward_fn, - learned_termination=False, - termination_mech=term_fn, + transition = PlainMech( + name="transition", + input_variables=obs_variables + act_variables, + output_variables=next_obs_variables, ) + + dynamics = Dynamics(transition, env.observation_space, env.action_space) real_replay_buffer = ReplayBuffer( 100, env.observation_space, env.action_space, device="cpu", handle_timeout_termination=False ) - fake_env = VecFakeEnv(1, env.observation_space, env.action_space) - fake_env.set_up(dynamics, reward_fn, term_fn, init_obs_fn) + fake_env = VecFakeEnv( + num_envs=1, + observation_space=env.observation_space, + action_space=env.action_space, + dynamics=dynamics, + reward_fn=reward_fn, + termination_fn=termination_fn, + get_init_obs_fn=get_init_obs_fn, + ) - callback = OnlineModelBasedCallback(env, dynamics, real_replay_buffer=real_replay_buffer, freq_train_model=5) + callback = OnlineModelBasedCallback(env, dynamics, real_replay_buffer, freq_train_model=20, longest_epoch=1) model = SAC("MlpPolicy", fake_env, verbose=1) model.learn(total_timesteps=100, log_interval=4, callback=callback) diff --git a/tests/test_types.py b/tests/test_types.py index e32a5bf..34ade49 100644 --- a/tests/test_types.py +++ b/tests/test_types.py @@ -2,46 +2,3 @@ from unittest import TestCase import torch - -from cmrl.utils.types import InteractionBatch - - -class TestTransitionBatch(TestCase): - def setUp(self) -> None: - self.batch_size = random.randint(0, 1000) - self.obs_size = random.randint(0, 1000) - self.action_size = random.randint(0, 1000) - self.transition_batch = InteractionBatch( - torch.rand(self.batch_size, self.obs_size), - torch.rand(self.batch_size, self.action_size), - torch.rand(self.batch_size, self.obs_size), - torch.rand(self.batch_size, 1), - torch.rand(self.batch_size, 1), - ) - - def test_as_tuple(self): - ( - batch_obs, - batch_action, - batch_next_obs, - batch_reward, - batch_done, - ) = self.transition_batch.as_tuple() - assert batch_obs.shape == (self.batch_size, self.obs_size) - assert batch_action.shape == (self.batch_size, self.action_size) - assert batch_next_obs.shape == (self.batch_size, self.obs_size) - assert batch_reward.shape == (self.batch_size, 1) - assert batch_done.shape == (self.batch_size, 1) - - def test___getitem__(self): - slice_transition = self.transition_batch[0] - assert len(slice_transition) == self.obs_size - - new_transition = slice_transition.add_new_batch_dim(1) - assert new_transition.batch_obs.shape == (1, self.obs_size) - - def test__get_new_shape(self): - new_batch_size = 1 - old_shape = self.transition_batch.batch_obs.shape - new_shape = self.transition_batch._get_new_shape(old_shape, new_batch_size) - assert new_shape == (new_batch_size, self.batch_size, self.obs_size) From 2e34bc920164fc1960a1775c437bf081654e8232 Mon Sep 17 00:00:00 2001 From: frank Date: Tue, 8 Nov 2022 16:32:58 +0800 Subject: [PATCH 23/68] :tada: add mask in reduce_encoder_output --- cmrl/algorithms/util.py | 2 +- cmrl/examples/conf/transition/CMI_test.yaml | 2 +- cmrl/models/causal_mech/CMI_test.py | 17 +++- cmrl/models/causal_mech/__init__.py | 2 +- cmrl/models/causal_mech/neural_causal_mech.py | 25 ++++-- .../test_causal_mech/test_CMI_test.py | 80 +++++++++++++++++++ 6 files changed, 116 insertions(+), 12 deletions(-) create mode 100644 tests/test_models/test_causal_mech/test_CMI_test.py diff --git a/cmrl/algorithms/util.py b/cmrl/algorithms/util.py index de7d6de..1f9a43f 100644 --- a/cmrl/algorithms/util.py +++ b/cmrl/algorithms/util.py @@ -8,7 +8,7 @@ from stable_baselines3.common.base_class import BaseAlgorithm from stable_baselines3.common.buffers import ReplayBuffer -from cmrl.utils.variables import InitObsFnType, RewardFnType, TermFnType +from cmrl.types import InitObsFnType, RewardFnType, TermFnType # from cmrl.models.dynamics import BaseDynamics from cmrl.models.fake_env import VecFakeEnv diff --git a/cmrl/examples/conf/transition/CMI_test.yaml b/cmrl/examples/conf/transition/CMI_test.yaml index 77eaeb4..010cc23 100644 --- a/cmrl/examples/conf/transition/CMI_test.yaml +++ b/cmrl/examples/conf/transition/CMI_test.yaml @@ -41,7 +41,7 @@ optimizer_cfg: mech: _partial_: true _recursive_: false - _target_: cmrl.models.causal_mech.CMItest + _target_: cmrl.models.causal_mech.CMITest # base causal-mech params name: transition input_variables: ??? diff --git a/cmrl/models/causal_mech/CMI_test.py b/cmrl/models/causal_mech/CMI_test.py index c7a5cda..a34ce67 100644 --- a/cmrl/models/causal_mech/CMI_test.py +++ b/cmrl/models/causal_mech/CMI_test.py @@ -9,12 +9,15 @@ from cmrl.models.causal_mech.neural_causal_mech import NeuralCausalMech -class CMItest(NeuralCausalMech): +class CMITest(NeuralCausalMech): def __init__( self, name: str, input_variables: List[Variable], output_variables: List[Variable], + # mask + mask_method: str = "zero", + # ensemble ensemble_num: int = 7, elite_num: int = 5, # cfgs @@ -35,7 +38,9 @@ def __init__( if multi_step == "none": multi_step = "forward-euler 1" - super(CMItest, self).__init__( + self.mask_method = mask_method + + super(CMITest, self).__init__( name=name, input_variables=input_variables, output_variables=output_variables, @@ -71,7 +76,10 @@ def single_step_forward(self, inputs: MutableMapping[str, torch.Tensor]) -> Dict out = self.variable_encoders[var.name](inputs[var.name].to(self.device)) inputs_tensor[:, :, i] = out - output_tensor = self.network(self.reduce_encoder_output(inputs_tensor)) + reduced_inputs_tensor = self.reduce_encoder_output( + inputs_tensor, + ) + output_tensor = self.network(reduced_inputs_tensor) outputs = {} for i, var in enumerate(self.output_variables): @@ -81,3 +89,6 @@ def single_step_forward(self, inputs: MutableMapping[str, torch.Tensor]) -> Dict if self.residual: outputs = self.residual_outputs(inputs, outputs) return outputs + + def CMI_forward(self, inputs: MutableMapping[str, torch.Tensor]) -> Dict[str, torch.Tensor]: + pass diff --git a/cmrl/models/causal_mech/__init__.py b/cmrl/models/causal_mech/__init__.py index d55c608..5f93938 100644 --- a/cmrl/models/causal_mech/__init__.py +++ b/cmrl/models/causal_mech/__init__.py @@ -1,2 +1,2 @@ from cmrl.models.causal_mech.plain_mech import PlainMech -from cmrl.models.causal_mech.CMI_test import CMItest +from cmrl.models.causal_mech.CMI_test import CMITest diff --git a/cmrl/models/causal_mech/neural_causal_mech.py b/cmrl/models/causal_mech/neural_causal_mech.py index 352b58f..54ae5d8 100644 --- a/cmrl/models/causal_mech/neural_causal_mech.py +++ b/cmrl/models/causal_mech/neural_causal_mech.py @@ -73,6 +73,7 @@ def __init__( name: str, input_variables: List[Variable], output_variables: List[Variable], + # ensemble ensemble_num: int = 7, elite_num: int = 5, # cfgs @@ -96,6 +97,7 @@ def __init__( output_variables=output_variables, device=device, ) + # ensemble self.ensemble_num = ensemble_num self.elite_num = elite_num # cfgs @@ -359,16 +361,27 @@ def decoder_input_dim(self): else: return self.decoder_cfg.input_dim - def reduce_encoder_output(self, encoder_output: torch.Tensor) -> torch.Tensor: + def reduce_encoder_output( + self, + encoder_output: torch.Tensor, + mask: Optional[torch.Tensor] = None, + ) -> torch.Tensor: assert len(encoder_output.shape) == 4, ( - "shape of encoder_output should be (ensemble-num, batch-size, input-cvar-num, encoder-output-dim), " + "shape of encoder_output should be (ensemble-num, batch-size, input-var-num, encoder-output-dim), " "rather than {}".format(encoder_output.shape) ) + + if mask is not None: + mask = mask.unsqueeze(-1).unsqueeze(-3).unsqueeze(-4) + masked_encoder_output = encoder_output * mask + else: + masked_encoder_output = encoder_output + if self.encoder_reduction == "sum": - return encoder_output.sum(-2) + return masked_encoder_output.sum(-2) elif self.encoder_reduction == "mean": - return encoder_output.mean(-2) - elif self.encoder_reduction == "sum": - return encoder_output.sum(-2) + return masked_encoder_output.mean(-2) + elif self.encoder_reduction == "max": + return masked_encoder_output.max(-2) else: raise NotImplementedError("not implemented encoder reduction method: {}".format(self.encoder_reduction)) diff --git a/tests/test_models/test_causal_mech/test_CMI_test.py b/tests/test_models/test_causal_mech/test_CMI_test.py new file mode 100644 index 0000000..8395c40 --- /dev/null +++ b/tests/test_models/test_causal_mech/test_CMI_test.py @@ -0,0 +1,80 @@ +import gym +from stable_baselines3.common.buffers import ReplayBuffer +import torch +from torch.utils.data import DataLoader + +from cmrl.models.causal_mech.CMI_test import CMITest +from cmrl.models.data_loader import BufferDataset, EnsembleBufferDataset, collate_fn +from cmrl.utils.creator import parse_space +from cmrl.utils.env import load_offline_data + + +def prepare(freq_rate): + env = gym.make("BoundaryInvertedPendulumSwingUp-v0", freq_rate=freq_rate, time_step=0.02) + + real_replay_buffer = ReplayBuffer(1000, env.observation_space, env.action_space, "cpu", handle_timeout_termination=False) + load_offline_data(env, real_replay_buffer, "SAC-expert-replay", use_ratio=0.001) + + ensemble_num = 7 + # test for transition + train_dataset = EnsembleBufferDataset( + real_replay_buffer, + env.observation_space, + env.action_space, + is_valid=False, + mech="transition", + train_ensemble=True, + ensemble_num=ensemble_num, + ) + train_loader = DataLoader(train_dataset, batch_size=8, collate_fn=collate_fn) + valid_dataset = BufferDataset( + real_replay_buffer, env.observation_space, env.action_space, is_valid=True, mech="transition", repeat=ensemble_num + ) + valid_loader = DataLoader(valid_dataset, batch_size=8, collate_fn=collate_fn) + + input_variables = parse_space(env.observation_space, "obs") + parse_space(env.action_space, "act") + output_variables = parse_space(env.observation_space, "next_obs") + + return input_variables, output_variables, train_loader, valid_loader + + +def test_mask(): + input_variables, output_variables, train_loader, valid_loader = prepare(freq_rate=1) + + mech = CMITest( + name="test", + input_variables=input_variables, + output_variables=output_variables, + ) + + for inputs, targets in train_loader: + batch_size = mech.get_inputs_batch_size(inputs) + + inputs_tensor = torch.zeros(mech.ensemble_num, batch_size, mech.input_var_num, mech.encoder_output_dim).to(mech.device) + for i, var in enumerate(mech.input_variables): + out = mech.variable_encoders[var.name](inputs[var.name].to(mech.device)) + inputs_tensor[:, :, i] = out + + mask = None + masked_inputs_tensor = mech.reduce_encoder_output(inputs_tensor, mask) + assert masked_inputs_tensor.shape == (mech.ensemble_num, batch_size, mech.encoder_output_dim) + + mask = torch.ones(mech.input_var_num).to(mech.device) + masked_inputs_tensor = mech.reduce_encoder_output(inputs_tensor, mask) + assert masked_inputs_tensor.shape == (mech.ensemble_num, batch_size, mech.encoder_output_dim) + + mask = torch.ones(mech.output_var_num, mech.input_var_num).to(mech.device) + masked_inputs_tensor = mech.reduce_encoder_output(inputs_tensor, mask) + assert masked_inputs_tensor.shape == (mech.output_var_num, mech.ensemble_num, batch_size, mech.encoder_output_dim) + + mask = torch.ones(mech.input_var_num + 1, mech.output_var_num, mech.input_var_num).to(mech.device) + masked_inputs_tensor = mech.reduce_encoder_output(inputs_tensor, mask) + assert masked_inputs_tensor.shape == ( + mech.input_var_num + 1, + mech.output_var_num, + mech.ensemble_num, + batch_size, + mech.encoder_output_dim, + ) + + break From e7c685091a0032ca58a6b5c9bc08d7fd733c67f9 Mon Sep 17 00:00:00 2001 From: frank Date: Wed, 9 Nov 2022 17:05:22 +0800 Subject: [PATCH 24/68] :tada: finish CMI test! --- cmrl/algorithms/mopo.py | 3 +- cmrl/algorithms/off_dyna.py | 3 +- cmrl/examples/conf/main.yaml | 2 +- cmrl/examples/conf/task/BIPS.yaml | 4 +- cmrl/examples/conf/transition/CMI_test.yaml | 6 +- cmrl/models/causal_mech/CMI_test.py | 169 +++++++++++++++++- cmrl/models/causal_mech/neural_causal_mech.py | 134 ++++++-------- cmrl/models/causal_mech/plain_mech.py | 21 +-- cmrl/models/causal_mech/util.py | 93 ++++++++++ cmrl/models/layers.py | 16 +- cmrl/models/networks/parallel_mlp.py | 6 +- .../test_causal_mech/test_CMI_test.py | 29 +++ tests/test_models/test_layers.py | 85 ++++++++- .../test_network/test_parallel_mlp.py | 2 +- 14 files changed, 440 insertions(+), 133 deletions(-) create mode 100644 cmrl/models/causal_mech/util.py diff --git a/cmrl/algorithms/mopo.py b/cmrl/algorithms/mopo.py index 76774a0..c7ecd58 100644 --- a/cmrl/algorithms/mopo.py +++ b/cmrl/algorithms/mopo.py @@ -3,7 +3,8 @@ from omegaconf import DictConfig from cmrl.models.fake_env import VecFakeEnv -from cmrl.algorithms.base_algorithm import BaseAlgorithm, load_offline_data +from cmrl.algorithms.base_algorithm import BaseAlgorithm +from cmrl.utils.env import load_offline_data class MOPO(BaseAlgorithm): diff --git a/cmrl/algorithms/off_dyna.py b/cmrl/algorithms/off_dyna.py index 6fe89f6..e9de279 100644 --- a/cmrl/algorithms/off_dyna.py +++ b/cmrl/algorithms/off_dyna.py @@ -3,7 +3,8 @@ from omegaconf import DictConfig from cmrl.models.fake_env import VecFakeEnv -from cmrl.algorithms.base_algorithm import BaseAlgorithm, load_offline_data +from cmrl.algorithms.base_algorithm import BaseAlgorithm +from cmrl.utils.env import load_offline_data class OfflineDyna(BaseAlgorithm): diff --git a/cmrl/examples/conf/main.yaml b/cmrl/examples/conf/main.yaml index 15ffe4e..91ce51f 100644 --- a/cmrl/examples/conf/main.yaml +++ b/cmrl/examples/conf/main.yaml @@ -1,5 +1,5 @@ defaults: - - algorithm: on_dyna + - algorithm: off_dyna - task: BIPS - transition: CMI_test - reward_mech: plain diff --git a/cmrl/examples/conf/task/BIPS.yaml b/cmrl/examples/conf/task/BIPS.yaml index 9a54676..3f1216a 100644 --- a/cmrl/examples/conf/task/BIPS.yaml +++ b/cmrl/examples/conf/task/BIPS.yaml @@ -1,7 +1,7 @@ env: "emei___BoundaryInvertedPendulumSwingUp-v0___freq_rate=${task.freq_rate}&time_step=${task.time_step}___${task.dataset}" dataset: "SAC-expert-replay" -freq_rate: 5 -time_step: 0.05 +freq_rate: 2 +time_step: 0.02 # basic RL params num_steps: 300000 diff --git a/cmrl/examples/conf/transition/CMI_test.yaml b/cmrl/examples/conf/transition/CMI_test.yaml index 010cc23..cb964d0 100644 --- a/cmrl/examples/conf/transition/CMI_test.yaml +++ b/cmrl/examples/conf/transition/CMI_test.yaml @@ -46,6 +46,9 @@ mech: name: transition input_variables: ??? output_variables: ??? + # mask + mask_method: "zero" + # ensemble ensemble_num: 7 elite_num: 5 # cfgs @@ -55,7 +58,8 @@ mech: optimizer_cfg: ${transition.optimizer_cfg} # forward method residual: true - multi_step: "none" + encoder_reduction: "sum" + multi_step: "forward-euler 2" # logger logger: ??? # others diff --git a/cmrl/models/causal_mech/CMI_test.py b/cmrl/models/causal_mech/CMI_test.py index a34ce67..3d2e245 100644 --- a/cmrl/models/causal_mech/CMI_test.py +++ b/cmrl/models/causal_mech/CMI_test.py @@ -1,12 +1,17 @@ from typing import Optional, List, Dict, Union, MutableMapping +import pathlib +from functools import partial +from itertools import count import torch +from torch.utils.data import DataLoader from omegaconf import DictConfig from hydra.utils import instantiate from stable_baselines3.common.logger import Logger from cmrl.utils.variables import Variable from cmrl.models.causal_mech.neural_causal_mech import NeuralCausalMech +from cmrl.models.causal_mech.util import variable_loss_func, train_func, eval_func class CMITest(NeuralCausalMech): @@ -39,6 +44,7 @@ def __init__( multi_step = "forward-euler 1" self.mask_method = mask_method + self.total_CMI_epoch = 0 super(CMITest, self).__init__( name=name, @@ -76,9 +82,7 @@ def single_step_forward(self, inputs: MutableMapping[str, torch.Tensor]) -> Dict out = self.variable_encoders[var.name](inputs[var.name].to(self.device)) inputs_tensor[:, :, i] = out - reduced_inputs_tensor = self.reduce_encoder_output( - inputs_tensor, - ) + reduced_inputs_tensor = self.reduce_encoder_output(inputs_tensor) output_tensor = self.network(reduced_inputs_tensor) outputs = {} @@ -90,5 +94,164 @@ def single_step_forward(self, inputs: MutableMapping[str, torch.Tensor]) -> Dict outputs = self.residual_outputs(inputs, outputs) return outputs + @property + def CMI_mask(self) -> torch.Tensor: + mask = torch.zeros(self.input_var_num + 1, self.output_var_num, self.input_var_num) + for i in range(self.input_var_num + 1): + m = torch.ones(self.output_var_num, self.input_var_num) + if i != self.input_var_num: + m[:, i] = 0 + mask[i] = m + + if self.mask_method == "zero": + pass + elif self.mask_method == "-inf": + mask[mask == 0] = -torch.inf + + return mask.to(self.device) + + def CMI_single_step_forward(self, inputs: MutableMapping[str, torch.Tensor]) -> Dict[str, torch.Tensor]: + """when first step, inputs should be dict of str and Tensor with (ensemble-num, batch-size, specific-dim) shape, + since twice step, the shape of Tensor becomes (input-var-num + 1, ensemble-num, batch-size, specific-dim) + + Args: + inputs: + + Returns: + + """ + batch_size, extra_dim = self.get_inputs_batch_size(inputs) + + inputs_tensor = torch.empty(*extra_dim, self.ensemble_num, batch_size, self.input_var_num, self.encoder_output_dim).to( + self.device + ) + for i, var in enumerate(self.input_variables): + out = self.variable_encoders[var.name](inputs[var.name].to(self.device)) + inputs_tensor[..., i, :] = out + + if len(extra_dim) == 0: + reduced_inputs_tensor = self.reduce_encoder_output(inputs_tensor, self.CMI_mask) + assert ( + not torch.isinf(reduced_inputs_tensor).any() and not torch.isnan(reduced_inputs_tensor).any() + ), "tensor must not be inf or nan" + output_tensor = self.network(reduced_inputs_tensor) + else: + output_tensor = torch.empty( + *extra_dim, self.output_var_num, self.ensemble_num, batch_size, self.decoder_input_dim + ).to(self.device) + for i in range(self.input_var_num + 1): + if i == len(inputs_tensor) - 1: + reduced_inputs_tensor = self.reduce_encoder_output(inputs_tensor[i], self.CMI_mask[i]) + outs = self.network(reduced_inputs_tensor) + output_tensor[i] = outs + else: + for j in range(self.output_var_num): + ins = inputs_tensor[-1] + ins[:, :, j] = inputs_tensor[i, :, :, j, :] + reduced_inputs_tensor = self.reduce_encoder_output(inputs_tensor[i], self.CMI_mask[i]) + outs = self.network(reduced_inputs_tensor) + output_tensor[i, j] = outs[j] + + outputs = {} + for i, var in enumerate(self.output_variables): + hid = output_tensor[:, i] + outputs[var.name] = self.variable_decoders[var.name](hid) + + if self.residual: + outputs = self.residual_outputs(inputs, outputs) + return outputs + def CMI_forward(self, inputs: MutableMapping[str, torch.Tensor]) -> Dict[str, torch.Tensor]: + """inputs should be dict of str and Tensor with (ensemble-num, batch-size, specific-dim) shape + + Args: + inputs: + + Returns: + + """ + if self.multi_step.startswith("forward-euler"): + step_num = int(self.multi_step.split()[-1]) + + outputs = {} + for step in range(step_num): + outputs = self.CMI_single_step_forward(inputs) + # outputs shape: (input-var-num + 1, ensemble-num, batch-size, specific-dim * 2) + # new inputs shape: (input-var-num + 1, ensemble-num, batch-size, specific-dim) + if step == 0: + for name in filter(lambda s: s.startswith("act"), inputs.keys()): + inputs[name] = inputs[name][None, ...].repeat([self.input_var_num + 1, 1, 1, 1]) + if step < step_num - 1: + for name in filter(lambda s: s.startswith("obs"), inputs.keys()): + inputs[name] = outputs["next_{}".format(name)][..., : inputs[name].shape[-1]] + else: + raise NotImplementedError("multi-step method {} is not supported".format(self.multi_step)) + + return outputs + + def calculate_CMI(self, nll_loss: torch.Tensor): + print("fc loss", nll_loss[-1].mean(dim=(0, 1))) + print("mask", nll_loss[:-1].mean(dim=(1, 2))) + nll_loss_diff = nll_loss[:-1] - nll_loss[-1] + print("cmi", nll_loss_diff.mean(dim=(1, 2))) pass + + def learn( + self, + # loader + train_loader: DataLoader, + valid_loader: DataLoader, + # model learning + longest_epoch: int = -1, + improvement_threshold: float = 0.01, + patience: int = 5, + work_dir: Optional[Union[str, pathlib.Path]] = None, + **kwargs + ): + epoch_iter = range(longest_epoch) if longest_epoch >= 0 else count() + epochs_since_update = 0 + + loss_func = partial(variable_loss_func, output_variables=self.output_variables, device=self.device) + train = partial(train_func, forward=self.CMI_forward, optimizer=self.optimizer, loss_func=loss_func) + eval = partial(eval_func, forward=self.CMI_forward, loss_func=loss_func) + + best_eval_loss = eval(valid_loader).mean(dim=(0, 2, 3)) + + for epoch in epoch_iter: + train_loss = train(train_loader) + eval_loss = eval(valid_loader) + self.calculate_CMI(eval_loss) + + improvement = (best_eval_loss - eval_loss.mean(dim=(0, 2, 3))) / torch.abs(best_eval_loss) + if (improvement > improvement_threshold).any().item(): + best_eval_loss = torch.minimum(best_eval_loss, eval_loss.mean(dim=(0, 2, 3))) + epochs_since_update = 0 + else: + epochs_since_update += 1 + + # log + self.total_CMI_epoch += 1 + if self.logger is not None: + self.logger.record("{}-CMI-test/epoch".format(self.name), epoch) + self.logger.record("{}-CMI-test/epochs_since_update".format(self.name), epochs_since_update) + self.logger.record("{}-CMI-test/train_dataset_size".format(self.name), len(train_loader.dataset)) + self.logger.record("{}-CMI-test/valid_dataset_size".format(self.name), len(valid_loader.dataset)) + self.logger.record("{}-CMI-test/train_loss".format(self.name), train_loss.mean().item()) + self.logger.record("{}-CMI-test/val_loss".format(self.name), eval_loss.mean().item()) + self.logger.record("{}-CMI-test/best_val_loss".format(self.name), best_eval_loss.mean().item()) + + self.logger.dump(self.total_CMI_epoch) + + if patience and epochs_since_update >= patience: + break + + # super(CMITest, self).learn( + # train_loader=train_loader, + # valid_loader=valid_loader, + # # model learning + # longest_epoch=longest_epoch, + # improvement_threshold=improvement_threshold, + # patience=patience, + # work_dir=work_dir, + # **kwargs + # ) diff --git a/cmrl/models/causal_mech/neural_causal_mech.py b/cmrl/models/causal_mech/neural_causal_mech.py index 54ae5d8..4a436c9 100644 --- a/cmrl/models/causal_mech/neural_causal_mech.py +++ b/cmrl/models/causal_mech/neural_causal_mech.py @@ -1,8 +1,8 @@ from typing import Optional, List, Dict, Union, MutableMapping from abc import abstractmethod, ABC -from itertools import chain +from itertools import chain, count +from functools import partial import pathlib -import itertools import copy import torch @@ -19,6 +19,7 @@ from cmrl.models.graphs.base_graph import BaseGraph from cmrl.models.networks.coder import VariableEncoder, VariableDecoder from cmrl.utils.variables import Variable, ContinuousVariable, DiscreteVariable, BinaryVariable +from cmrl.models.causal_mech.util import variable_loss_func, train_func, eval_func default_network_cfg = DictConfig( dict( @@ -126,9 +127,24 @@ def __init__( self.total_epoch = 0 self.elite_indices: List[int] = [] - @abstractmethod def single_step_forward(self, inputs: MutableMapping[str, torch.Tensor]) -> Dict[str, torch.Tensor]: - raise NotImplementedError + batch_size = self.get_inputs_batch_size(inputs) + + inputs_tensor = torch.zeros(self.ensemble_num, batch_size, self.input_var_num, self.encoder_output_dim).to(self.device) + for i, var in enumerate(self.input_variables): + out = self.variable_encoders[var.name](inputs[var.name].to(self.device)) + inputs_tensor[:, :, i] = out + + output_tensor = self.network(self.reduce_encoder_output(inputs_tensor)) + + outputs = {} + for i, var in enumerate(self.output_variables): + hid = output_tensor[:, :, i * self.decoder_input_dim : (i + 1) * self.decoder_input_dim] + outputs[var.name] = self.variable_decoders[var.name](hid) + + if self.residual: + outputs = self.residual_outputs(inputs, outputs) + return outputs def forward(self, inputs: MutableMapping[str, torch.Tensor]) -> Dict[str, torch.Tensor]: if self.multi_step.startswith("forward-euler"): @@ -137,6 +153,7 @@ def forward(self, inputs: MutableMapping[str, torch.Tensor]) -> Dict[str, torch. outputs = {} for step in range(step_num): outputs = self.single_step_forward(inputs) + inputs = {} if step < step_num - 1: for name in filter(lambda s: s.startswith("obs"), inputs.keys()): assert inputs[name].shape[:2] == outputs["next_{}".format(name)].shape[:2] @@ -178,36 +195,14 @@ def build_coder(self): assert var.name not in self.variable_decoders, "duplicate name in decoders: {}".format(var.name) self.variable_decoders[var.name] = instantiate(self.decoder_cfg)(variable=var).to(self.device) - def loss(self, outputs, targets): - ensemble_num, batch_size = list(targets.values())[0].shape[:2] - total_loss = torch.zeros(ensemble_num, batch_size, self.output_var_num) - for i, var in enumerate(self.output_variables): - output = outputs[var.name] - target = targets[var.name].to(self.device) - if isinstance(var, ContinuousVariable): - dim = target.shape[-1] # ensemble-num, batch-size, dim - assert output.shape[-1] == 2 * dim - mean, log_var = output[:, :, :dim], output[:, :, dim:] - loss = F.gaussian_nll_loss(mean, target, log_var.exp(), reduction="none").mean(dim=-1) - total_loss[..., i] = loss - elif isinstance(var, DiscreteVariable): - # TODO: onehot to int? - raise NotImplementedError - total_loss[..., i] = F.cross_entropy(output, target, reduction="none") - elif isinstance(var, BinaryVariable): - total_loss[..., i] = F.binary_cross_entropy(output, target, reduction="none") - else: - raise NotImplementedError - return total_loss - def get_inputs_batch_size(self, inputs: MutableMapping[str, torch.Tensor]) -> int: assert len(set(inputs.keys()) & set(self.variable_encoders.keys())) == len(inputs) data_shape = list(inputs.values())[0].shape - assert len(data_shape) == 3, "{}".format(data_shape) # ensemble-num, batch-size, specific-dim - ensemble, batch_size, specific_dim = data_shape + # assert len(data_shape) == 3, "{}".format(data_shape) # ensemble-num, batch-size, specific-dim + ensemble, batch_size, specific_dim = data_shape[-3:] assert ensemble == self.ensemble_num - return batch_size + return batch_size, data_shape[:-3] def residual_outputs( self, @@ -215,50 +210,12 @@ def residual_outputs( outputs: MutableMapping[str, torch.Tensor], ) -> MutableMapping[str, torch.Tensor]: for name in filter(lambda s: s.startswith("obs"), inputs.keys()): - assert inputs[name].shape[:2] == outputs["next_{}".format(name)].shape[:2] - assert inputs[name].shape[2] * 2 == outputs["next_{}".format(name)].shape[2] - outputs["next_{}".format(name)][:, :, : inputs[name].shape[2]] += inputs[name].to(self.device) + # assert inputs[name].shape[:2] == outputs["next_{}".format(name)].shape[:2] + # assert inputs[name].shape[2] * 2 == outputs["next_{}".format(name)].shape[2] + var_dim = inputs[name].shape[-1] + outputs["next_{}".format(name)][..., :var_dim] += inputs[name].to(self.device) return outputs - def train(self, loader: DataLoader): - """train for ensemble data - - Args: - loader: train data-loader. - - Returns: tensor of train loss, with shape (ensemble-num, batch-size). - - """ - batch_loss_list = [] - for inputs, targets in loader: - outputs = self.forward(inputs) - loss = self.loss(outputs, targets) # ensemble-num, batch-size, output-var-num - - self.optimizer.zero_grad() - loss.mean().backward() - self.optimizer.step() - - batch_loss_list.append(loss) - return torch.cat(batch_loss_list, dim=-2).detach().cpu() - - def eval(self, loader: DataLoader): - """evaluate for non-ensemble data - - Args: - loader: valid data-loader. - - Returns: tensor of eval loss, with shape (batch-size). - - """ - batch_loss_list = [] - with torch.no_grad(): - for inputs, targets in loader: - outputs = self.forward(inputs) - loss = self.loss(outputs, targets) # ensemble-num, batch-size, output-var-num - - batch_loss_list.append(loss) - return torch.cat(batch_loss_list, dim=-2).detach().cpu() - def learn( self, # loader @@ -272,17 +229,24 @@ def learn( **kwargs ): best_weights: Optional[Dict] = None - epoch_iter = range(longest_epoch) if longest_epoch >= 0 else itertools.count() + epoch_iter = range(longest_epoch) if longest_epoch >= 0 else count() epochs_since_update = 0 - best_eval_loss = self.eval(valid_loader).mean(dim=(1, 2)) + + loss_func = partial(variable_loss_func, output_variables=self.output_variables, device=self.device) + train = partial(train_func, forward=self.forward, optimizer=self.optimizer, loss_func=loss_func) + eval = partial(eval_func, forward=self.forward, loss_func=loss_func) + + best_eval_loss = eval(valid_loader).mean(dim=(-2, -1)) for epoch in epoch_iter: - train_loss = self.train(train_loader) - eval_loss = self.eval(valid_loader).mean(dim=(1, 2)) - maybe_best_weights = self._maybe_get_best_weights(best_eval_loss, eval_loss, improvement_threshold) + train_loss = train(train_loader) + eval_loss = eval(valid_loader) + maybe_best_weights = self._maybe_get_best_weights( + best_eval_loss, eval_loss.mean(dim=(-2, -1)), improvement_threshold + ) if maybe_best_weights: # best loss - best_eval_loss = torch.minimum(best_eval_loss, eval_loss) + best_eval_loss = torch.minimum(best_eval_loss, eval_loss.mean(dim=(-2, -1))) best_weights = maybe_best_weights epochs_since_update = 0 else: @@ -292,6 +256,7 @@ def learn( self.total_epoch += 1 if self.logger is not None: self.logger.record("{}/epoch".format(self.name), epoch) + self.logger.record("{}/epochs_since_update".format(self.name), epochs_since_update) self.logger.record("{}/train_dataset_size".format(self.name), len(train_loader.dataset)) self.logger.record("{}/valid_dataset_size".format(self.name), len(valid_loader.dataset)) self.logger.record("{}/train_loss".format(self.name), train_loss.mean().item()) @@ -371,17 +336,20 @@ def reduce_encoder_output( "rather than {}".format(encoder_output.shape) ) - if mask is not None: - mask = mask.unsqueeze(-1).unsqueeze(-3).unsqueeze(-4) - masked_encoder_output = encoder_output * mask - else: - masked_encoder_output = encoder_output + if mask is None: + mask = self.forward_mask + + mask = mask.unsqueeze(-1).unsqueeze(-3).unsqueeze(-4) + masked_encoder_output = encoder_output * mask + if torch.isinf(masked_encoder_output).any(): + masked_encoder_output[torch.isinf(masked_encoder_output)] = -torch.inf if self.encoder_reduction == "sum": return masked_encoder_output.sum(-2) elif self.encoder_reduction == "mean": return masked_encoder_output.mean(-2) elif self.encoder_reduction == "max": - return masked_encoder_output.max(-2) + values, indices = masked_encoder_output.max(-2) + return values else: raise NotImplementedError("not implemented encoder reduction method: {}".format(self.encoder_reduction)) diff --git a/cmrl/models/causal_mech/plain_mech.py b/cmrl/models/causal_mech/plain_mech.py index 560776a..5393ba6 100644 --- a/cmrl/models/causal_mech/plain_mech.py +++ b/cmrl/models/causal_mech/plain_mech.py @@ -63,21 +63,6 @@ def build_network(self): def build_graph(self): self.graph = None - def single_step_forward(self, inputs: MutableMapping[str, torch.Tensor]) -> Dict[str, torch.Tensor]: - batch_size = self.get_inputs_batch_size(inputs) - - inputs_tensor = torch.zeros(self.ensemble_num, batch_size, self.input_var_num, self.encoder_output_dim).to(self.device) - for i, var in enumerate(self.input_variables): - out = self.variable_encoders[var.name](inputs[var.name].to(self.device)) - inputs_tensor[:, :, i] = out - - output_tensor = self.network(self.reduce_encoder_output(inputs_tensor)) - - outputs = {} - for i, var in enumerate(self.output_variables): - hid = output_tensor[:, :, i * self.decoder_input_dim : (i + 1) * self.decoder_input_dim] - outputs[var.name] = self.variable_decoders[var.name](hid) - - if self.residual: - outputs = self.residual_outputs(inputs, outputs) - return outputs + @property + def forward_mask(self): + return torch.ones(self.input_var_num).to(self.device) diff --git a/cmrl/models/causal_mech/util.py b/cmrl/models/causal_mech/util.py new file mode 100644 index 0000000..392e282 --- /dev/null +++ b/cmrl/models/causal_mech/util.py @@ -0,0 +1,93 @@ +from typing import Callable, Dict, List, Union, MutableMapping +from collections import defaultdict + +import torch +import torch.nn.functional as F +from torch.utils.data import DataLoader +from torch.optim import Optimizer + +from cmrl.utils.variables import Variable, ContinuousVariable, DiscreteVariable, BinaryVariable + + +def variable_loss_func( + outputs: Dict[str, torch.Tensor], + targets: Dict[str, torch.Tensor], + output_variables: List[Variable], + device: Union[str, torch.device] = "cpu", +): + dims = list(outputs.values())[0].shape[:-1] + total_loss = torch.zeros(*dims, len(outputs)) + + for i, var in enumerate(output_variables): + output = outputs[var.name] + target = targets[var.name].to(device) + if isinstance(var, ContinuousVariable): + dim = target.shape[-1] # (xxx, ensemble-num, batch-size, dim) + assert output.shape[-1] == 2 * dim + mean, log_var = output[..., :dim], output[..., dim:] + loss = F.gaussian_nll_loss(mean, target, log_var.exp(), reduction="none").mean(dim=-1) + total_loss[..., i] = loss + elif isinstance(var, DiscreteVariable): + # TODO: onehot to int? + raise NotImplementedError + total_loss[..., i] = F.cross_entropy(output, target, reduction="none") + elif isinstance(var, BinaryVariable): + total_loss[..., i] = F.binary_cross_entropy(output, target, reduction="none") + else: + raise NotImplementedError + return total_loss + + +def train_func( + loader: DataLoader, + forward: Callable[[MutableMapping[str, torch.Tensor]], Dict[str, torch.Tensor]], + optimizer: Optimizer, + loss_func: Callable[[MutableMapping[str, torch.Tensor], MutableMapping[str, torch.Tensor]], torch.Tensor], +): + """train for data + + Args: + forward: forward function. + loader: train data-loader. + optimizer: Optimizer + loss_func: loss function + + Returns: tensor of train loss, with shape (xxx, ensemble-num, batch-size). + + """ + batch_loss_list = [] + for inputs, targets in loader: + outputs = forward(inputs) + loss = loss_func(outputs, targets) # ensemble-num, batch-size, output-var-num + + optimizer.zero_grad() + loss.mean().backward() + optimizer.step() + + batch_loss_list.append(loss) + return torch.cat(batch_loss_list, dim=-2).detach().cpu() + + +def eval_func( + loader: DataLoader, + forward: Callable[[MutableMapping[str, torch.Tensor]], Dict[str, torch.Tensor]], + loss_func: Callable[[MutableMapping[str, torch.Tensor], MutableMapping[str, torch.Tensor]], torch.Tensor], +): + """evaluate for data + + Args: + forward: forward function. + loader: train data-loader. + loss_func: loss function + + Returns: tensor of train loss, with shape (xxx, ensemble-num, batch-size). + + """ + batch_loss_list = [] + with torch.no_grad(): + for inputs, targets in loader: + outputs = forward(inputs) + loss = loss_func(outputs, targets) # ensemble-num, batch-size, output-var-num + + batch_loss_list.append(loss) + return torch.cat(batch_loss_list, dim=-2).detach().cpu() diff --git a/cmrl/models/layers.py b/cmrl/models/layers.py index b583cd5..f93858e 100644 --- a/cmrl/models/layers.py +++ b/cmrl/models/layers.py @@ -15,7 +15,7 @@ def __init__( input_dim: int, output_dim: int, extra_dims: Optional[List[int]] = None, - use_bias: bool = True, + bias: bool = True, init_type: str = "truncated_normal", ): """Linear layer with the same properties as Parallel MLP. It effectively applies N independent linear layers @@ -26,7 +26,7 @@ def __init__( output_dim: Number of output dimensions per layer. extra_dims: Number of neural networks to have in parallel (e.g. number of variables). Can have multiple dimensions if needed. - use_bias: Weather using bias in this layer. + bias: Weather using bias in this layer. init_type: How to initialize weights and biases. """ super().__init__() @@ -36,7 +36,7 @@ def __init__( self.init_type = init_type self.weight = nn.Parameter(torch.zeros(*self.extra_dims, self.input_dim, self.output_dim)) - if use_bias: + if bias: self.bias = nn.Parameter(torch.zeros(*self.extra_dims, 1, self.output_dim)) self.use_bias = True else: @@ -60,14 +60,6 @@ def init_params(self): raise NotImplementedError def forward(self, x): - x_extra_dims = x.shape[:-2] - if len(x_extra_dims) > 0: - for i in range(len(x_extra_dims)): - assert x_extra_dims[-(i + 1)] == self.extra_dims[-(i + 1)], "Shape mismatch: X=%s, Layer=%s" % ( - str(x.shape), - str(self.extra_dims), - ) - xw = x.matmul(self.weight) if self.use_bias: return xw + self.bias @@ -86,6 +78,6 @@ def device(self) -> torch.device: return torch.device("cpu") def __repr__(self): - return 'ParallelLinear(input_dims={}, output_dims={}, extra_dims={}, use_bias={}, init_type="{}")'.format( + return 'ParallelLinear(input_dims={}, output_dims={}, extra_dims={}, bias={}, init_type="{}")'.format( self.input_dim, self.output_dim, str(self.extra_dims), self.use_bias, self.init_type ) diff --git a/cmrl/models/networks/parallel_mlp.py b/cmrl/models/networks/parallel_mlp.py index 06d7c88..e322f19 100644 --- a/cmrl/models/networks/parallel_mlp.py +++ b/cmrl/models/networks/parallel_mlp.py @@ -41,14 +41,12 @@ def build(self): for i in range(len(hidden_dims) - 1): layers += [ ParallelLinear( - input_dim=hidden_dims[i], output_dim=hidden_dims[i + 1], extra_dims=self.extra_dims, use_bias=self.bias + input_dim=hidden_dims[i], output_dim=hidden_dims[i + 1], extra_dims=self.extra_dims, bias=self.bias ) ] layers += [create_activation(self.activation_fn_cfg)] layers += [ - ParallelLinear( - input_dim=hidden_dims[-1], output_dim=self.output_dim, extra_dims=self.extra_dims, use_bias=self.bias - ) + ParallelLinear(input_dim=hidden_dims[-1], output_dim=self.output_dim, extra_dims=self.extra_dims, bias=self.bias) ] self._layers = nn.ModuleList(layers) diff --git a/tests/test_models/test_causal_mech/test_CMI_test.py b/tests/test_models/test_causal_mech/test_CMI_test.py index 8395c40..73be7cd 100644 --- a/tests/test_models/test_causal_mech/test_CMI_test.py +++ b/tests/test_models/test_causal_mech/test_CMI_test.py @@ -7,6 +7,7 @@ from cmrl.models.data_loader import BufferDataset, EnsembleBufferDataset, collate_fn from cmrl.utils.creator import parse_space from cmrl.utils.env import load_offline_data +from cmrl.models.causal_mech.util import variable_loss_func def prepare(freq_rate): @@ -78,3 +79,31 @@ def test_mask(): ) break + + +def test_CMI_forward(): + input_variables, output_variables, train_loader, valid_loader = prepare(freq_rate=1) + + mech = CMITest( + name="test", + input_variables=input_variables, + output_variables=output_variables, + ) + + for inputs, targets in train_loader: + outputs = mech.CMI_single_step_forward(inputs) + variable_loss_func(outputs, targets, output_variables) + + break + + +def test_forward(): + input_variables, output_variables, train_loader, valid_loader = prepare(freq_rate=1) + + mech = CMITest( + name="test", + input_variables=input_variables, + output_variables=output_variables, + ) + + mech.learn(train_loader, valid_loader, longest_epoch=10) diff --git a/tests/test_models/test_layers.py b/tests/test_models/test_layers.py index 6e76530..3baf89f 100644 --- a/tests/test_models/test_layers.py +++ b/tests/test_models/test_layers.py @@ -1,6 +1,7 @@ from unittest import TestCase import torch +from torch.nn import Linear from cmrl.models.layers import ParallelLinear @@ -8,14 +9,14 @@ def test_origin_layer(): input_dim = 5 output_dim = 6 - use_bias = True + bias = True batch_size = 128 device = "cuda" if torch.cuda.is_available() else "cpu" layer = ParallelLinear( input_dim=input_dim, output_dim=output_dim, - use_bias=use_bias, + bias=bias, ).to(device) model_in = torch.rand((batch_size, input_dim)).to(device) @@ -26,24 +27,76 @@ def test_origin_layer(): ) -def test_two_extra_dims_layer(): +def test_one_extra_dims_linear(): input_dim = 5 output_dim = 6 - use_bias = True - extra_dims = [3, 4] + bias = True + extra_dims = [7] batch_size = 128 device = "cuda" if torch.cuda.is_available() else "cpu" layer = ParallelLinear( input_dim=input_dim, output_dim=output_dim, - use_bias=use_bias, + bias=bias, extra_dims=extra_dims, ).to(device) model_in = torch.rand((*extra_dims, batch_size, input_dim)).to(device) model_out = layer(model_in) assert model_out.shape == ( + extra_dims[0], + batch_size, + output_dim, + ) + + +def test_two_extra_dims_linear(): + input_dim = 5 + output_dim = 1 + bias = True + extra_dims = [6, 7] + batch_size = 128 + device = "cuda" if torch.cuda.is_available() else "cpu" + + layer = ParallelLinear( + input_dim=input_dim, + output_dim=output_dim, + bias=bias, + extra_dims=extra_dims, + ).to(device) + + model_in = torch.rand((*extra_dims, batch_size, input_dim)).to(device) + model_out = layer(model_in) + assert model_out.shape == ( + extra_dims[0], + extra_dims[1], + batch_size, + output_dim, + ) + + +def test_broadcast_two_extra_dims_linear(): + input_dim = 5 + output_dim = 1 + bias = True + extra_dims = [6, 7] + batch_size = 128 + device = "cuda" if torch.cuda.is_available() else "cpu" + + layer = ParallelLinear( + input_dim=input_dim, + output_dim=output_dim, + bias=bias, + extra_dims=extra_dims, + ).to(device) + + broadcast_dim = 10 + + model_in = torch.rand((broadcast_dim, *extra_dims, batch_size, input_dim)).to(device) + model_out = layer(model_in) + assert model_out.shape == ( + broadcast_dim, extra_dims[0], extra_dims[1], batch_size, @@ -51,6 +104,26 @@ def test_two_extra_dims_layer(): ) +def test_broadcast_linear(): + input_dim = 5 + output_dim = 1 + bias = True + batch_size = 128 + device = "cuda" if torch.cuda.is_available() else "cpu" + + layer = Linear(input_dim, output_dim, bias=bias).to(device) + + broadcast_dim = 10 + + model_in = torch.rand((broadcast_dim, batch_size, input_dim)).to(device) + model_out = layer(model_in) + assert model_out.shape == ( + broadcast_dim, + batch_size, + output_dim, + ) + + def test_repr(): layer = ParallelLinear(3, 5) print(repr(layer)) diff --git a/tests/test_models/test_network/test_parallel_mlp.py b/tests/test_models/test_network/test_parallel_mlp.py index 2624cae..50931b9 100644 --- a/tests/test_models/test_network/test_parallel_mlp.py +++ b/tests/test_models/test_network/test_parallel_mlp.py @@ -17,7 +17,7 @@ def test_parallel_mlp(): "input_dim": input_dim, "output_dim": output_dim, "hidden_dims": [32, 32], - "use_bias": use_bias, + "bias": use_bias, "extra_dims": extra_dims, "activation_fn_cfg": DictConfig({"_target_": "torch.nn.SiLU"}), } From c244d8c175eb6a9dcea75c747828a728b55ed867 Mon Sep 17 00:00:00 2001 From: acez Date: Wed, 9 Nov 2022 21:27:14 +0800 Subject: [PATCH 25/68] :wrench: use index to mask the encoder output --- cmrl/models/causal_mech/neural_causal_mech.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/cmrl/models/causal_mech/neural_causal_mech.py b/cmrl/models/causal_mech/neural_causal_mech.py index 4a436c9..29c414f 100644 --- a/cmrl/models/causal_mech/neural_causal_mech.py +++ b/cmrl/models/causal_mech/neural_causal_mech.py @@ -339,10 +339,20 @@ def reduce_encoder_output( if mask is None: mask = self.forward_mask + assert mask.shape[-1] == self.input_var_num + + # [..., ensemble_num, batch_size, input_var_num, encoder_output_dim] mask = mask.unsqueeze(-1).unsqueeze(-3).unsqueeze(-4) - masked_encoder_output = encoder_output * mask - if torch.isinf(masked_encoder_output).any(): - masked_encoder_output[torch.isinf(masked_encoder_output)] = -torch.inf + mask = mask.repeat(tuple(*mask.shape[:-4], *encoder_output.shape[:-2], 1, encoder_output.shape[-1])) + + # [*mask_extra_dims, ensemble_num, batch_size, input_var_num, encoder_output_dim] + masked_encoder_output = encoder_output.repeat(tuple(mask.shape[:-4]) + (1,) * 4) + + # choose mask value + mask_value = 0 + if self.encoder_reduction == "max": + mask_value = -float("inf") + masked_encoder_output[1 - mask] == mask_value if self.encoder_reduction == "sum": return masked_encoder_output.sum(-2) From 04e5a03b95cd98fa822ffe498195fe6606745f82 Mon Sep 17 00:00:00 2001 From: frank Date: Thu, 10 Nov 2022 00:09:54 +0800 Subject: [PATCH 26/68] :bug: fix mask bug --- cmrl/examples/conf/task/BIPS.yaml | 2 +- cmrl/examples/conf/transition/CMI_test.yaml | 2 +- cmrl/models/causal_mech/CMI_test.py | 11 +---------- cmrl/models/causal_mech/neural_causal_mech.py | 10 ++++++++-- 4 files changed, 11 insertions(+), 14 deletions(-) diff --git a/cmrl/examples/conf/task/BIPS.yaml b/cmrl/examples/conf/task/BIPS.yaml index 3f1216a..1b28ead 100644 --- a/cmrl/examples/conf/task/BIPS.yaml +++ b/cmrl/examples/conf/task/BIPS.yaml @@ -1,6 +1,6 @@ env: "emei___BoundaryInvertedPendulumSwingUp-v0___freq_rate=${task.freq_rate}&time_step=${task.time_step}___${task.dataset}" dataset: "SAC-expert-replay" -freq_rate: 2 +freq_rate: 1 time_step: 0.02 # basic RL params diff --git a/cmrl/examples/conf/transition/CMI_test.yaml b/cmrl/examples/conf/transition/CMI_test.yaml index cb964d0..945d5c4 100644 --- a/cmrl/examples/conf/transition/CMI_test.yaml +++ b/cmrl/examples/conf/transition/CMI_test.yaml @@ -59,7 +59,7 @@ mech: # forward method residual: true encoder_reduction: "sum" - multi_step: "forward-euler 2" + multi_step: "forward-euler 1" # logger logger: ??? # others diff --git a/cmrl/models/causal_mech/CMI_test.py b/cmrl/models/causal_mech/CMI_test.py index 3d2e245..66c1568 100644 --- a/cmrl/models/causal_mech/CMI_test.py +++ b/cmrl/models/causal_mech/CMI_test.py @@ -20,8 +20,6 @@ def __init__( name: str, input_variables: List[Variable], output_variables: List[Variable], - # mask - mask_method: str = "zero", # ensemble ensemble_num: int = 7, elite_num: int = 5, @@ -43,7 +41,6 @@ def __init__( if multi_step == "none": multi_step = "forward-euler 1" - self.mask_method = mask_method self.total_CMI_epoch = 0 super(CMITest, self).__init__( @@ -96,18 +93,12 @@ def single_step_forward(self, inputs: MutableMapping[str, torch.Tensor]) -> Dict @property def CMI_mask(self) -> torch.Tensor: - mask = torch.zeros(self.input_var_num + 1, self.output_var_num, self.input_var_num) + mask = torch.zeros(self.input_var_num + 1, self.output_var_num, self.input_var_num, dtype=torch.long) for i in range(self.input_var_num + 1): m = torch.ones(self.output_var_num, self.input_var_num) if i != self.input_var_num: m[:, i] = 0 mask[i] = m - - if self.mask_method == "zero": - pass - elif self.mask_method == "-inf": - mask[mask == 0] = -torch.inf - return mask.to(self.device) def CMI_single_step_forward(self, inputs: MutableMapping[str, torch.Tensor]) -> Dict[str, torch.Tensor]: diff --git a/cmrl/models/causal_mech/neural_causal_mech.py b/cmrl/models/causal_mech/neural_causal_mech.py index 29c414f..c5fc713 100644 --- a/cmrl/models/causal_mech/neural_causal_mech.py +++ b/cmrl/models/causal_mech/neural_causal_mech.py @@ -343,7 +343,10 @@ def reduce_encoder_output( # [..., ensemble_num, batch_size, input_var_num, encoder_output_dim] mask = mask.unsqueeze(-1).unsqueeze(-3).unsqueeze(-4) - mask = mask.repeat(tuple(*mask.shape[:-4], *encoder_output.shape[:-2], 1, encoder_output.shape[-1])) + repeat_shape = (*mask.shape[:-4], *encoder_output.shape[:-2], 1, encoder_output.shape[-1]) + # mask = mask.repeat((*mask.shape[:-4], *encoder_output.shape[:-2], 1, encoder_output.shape[-1])) + + mask = mask.repeat((1,) * len(mask.shape[:-4]) + (*encoder_output.shape[:-2], 1, encoder_output.shape[-1])) # [*mask_extra_dims, ensemble_num, batch_size, input_var_num, encoder_output_dim] masked_encoder_output = encoder_output.repeat(tuple(mask.shape[:-4]) + (1,) * 4) @@ -352,7 +355,10 @@ def reduce_encoder_output( mask_value = 0 if self.encoder_reduction == "max": mask_value = -float("inf") - masked_encoder_output[1 - mask] == mask_value + assert masked_encoder_output.shape == mask.shape + # print(torch.unique(mask, return_counts=True)) + + masked_encoder_output[mask == 0] = mask_value if self.encoder_reduction == "sum": return masked_encoder_output.sum(-2) From 0f16f2fee16aaa41db5c7ceaa8b4adf4da543ebe Mon Sep 17 00:00:00 2001 From: wz139704646 <632291793@qq.com> Date: Fri, 11 Nov 2022 17:13:29 +0800 Subject: [PATCH 27/68] :tada: add binary, weight, neural and prob graphs --- cmrl/models/causal_mech/base_causal_mech.py | 28 ++-- cmrl/models/causal_mech/neural_causal_mech.py | 6 - cmrl/models/graphs/base_graph.py | 132 ++++++------------ cmrl/models/graphs/binary_graph.py | 81 +++++++++++ cmrl/models/graphs/neural_graph.py | 77 ++++++++++ cmrl/models/graphs/prob_graph.py | 99 +++++++------ cmrl/models/graphs/weight_graph.py | 94 +++++++++++++ graph.pth | Bin 0 -> 187799 bytes .../test_graphs/test_binary_graph.py | 69 +++++++++ .../test_graphs/test_neural_graph.py | 55 ++++++++ .../test_graphs/test_prob_graph.py | 35 +++++ .../test_graphs/test_weight_graph.py | 110 +++++++++++++++ 12 files changed, 632 insertions(+), 154 deletions(-) create mode 100644 cmrl/models/graphs/binary_graph.py create mode 100644 cmrl/models/graphs/neural_graph.py create mode 100644 cmrl/models/graphs/weight_graph.py create mode 100644 graph.pth create mode 100644 tests/test_models/test_graphs/test_binary_graph.py create mode 100644 tests/test_models/test_graphs/test_neural_graph.py create mode 100644 tests/test_models/test_graphs/test_prob_graph.py create mode 100644 tests/test_models/test_graphs/test_weight_graph.py diff --git a/cmrl/models/causal_mech/base_causal_mech.py b/cmrl/models/causal_mech/base_causal_mech.py index 978706c..19077dd 100644 --- a/cmrl/models/causal_mech/base_causal_mech.py +++ b/cmrl/models/causal_mech/base_causal_mech.py @@ -1,23 +1,11 @@ from typing import Optional, List, Dict, Union, MutableMapping from abc import abstractmethod, ABC -from itertools import chain -import pathlib -import itertools -import copy import torch -import numpy as np from torch.utils.data import DataLoader -import torch.nn.functional as F -from torch.optim import Optimizer -from stable_baselines3.common.logger import Logger -from omegaconf import DictConfig -from hydra.utils import instantiate -from cmrl.models.networks.base_network import BaseNetwork from cmrl.models.graphs.base_graph import BaseGraph -from cmrl.models.networks.coder import VariableEncoder, VariableDecoder -from cmrl.utils.variables import Variable, ContinuousVariable, DiscreteVariable, BinaryVariable +from cmrl.utils.variables import Variable class BaseCausalMech(ABC): @@ -35,6 +23,7 @@ def __init__( self.input_var_num = len(self.input_variables) self.output_var_num = len(self.output_variables) + self.graph: Optional[BaseGraph] = None @abstractmethod def learn(self, train_loader: DataLoader, valid_loader: DataLoader, **kwargs): @@ -43,3 +32,16 @@ def learn(self, train_loader: DataLoader, valid_loader: DataLoader, **kwargs): @abstractmethod def forward(self, inputs: MutableMapping[str, torch.Tensor]) -> Dict[str, torch.Tensor]: raise NotImplementedError + + @property + def causal_graph(self) -> torch.Tensor: + """property causal graph""" + if self.graph is None: + return torch.ones(len(self.input_variables), len(self.output_variables), dtype=torch.int, device=self.device) + else: + return self.graph.get_binary_adj_matrix() + + @property + def forward_mask(self) -> torch.Tensor: + """property input masks""" + return self.causal_graph.T diff --git a/cmrl/models/causal_mech/neural_causal_mech.py b/cmrl/models/causal_mech/neural_causal_mech.py index c5fc713..eca665f 100644 --- a/cmrl/models/causal_mech/neural_causal_mech.py +++ b/cmrl/models/causal_mech/neural_causal_mech.py @@ -343,9 +343,6 @@ def reduce_encoder_output( # [..., ensemble_num, batch_size, input_var_num, encoder_output_dim] mask = mask.unsqueeze(-1).unsqueeze(-3).unsqueeze(-4) - repeat_shape = (*mask.shape[:-4], *encoder_output.shape[:-2], 1, encoder_output.shape[-1]) - # mask = mask.repeat((*mask.shape[:-4], *encoder_output.shape[:-2], 1, encoder_output.shape[-1])) - mask = mask.repeat((1,) * len(mask.shape[:-4]) + (*encoder_output.shape[:-2], 1, encoder_output.shape[-1])) # [*mask_extra_dims, ensemble_num, batch_size, input_var_num, encoder_output_dim] @@ -355,9 +352,6 @@ def reduce_encoder_output( mask_value = 0 if self.encoder_reduction == "max": mask_value = -float("inf") - assert masked_encoder_output.shape == mask.shape - # print(torch.unique(mask, return_counts=True)) - masked_encoder_output[mask == 0] = mask_value if self.encoder_reduction == "sum": diff --git a/cmrl/models/graphs/base_graph.py b/cmrl/models/graphs/base_graph.py index df531c8..2f27eb8 100644 --- a/cmrl/models/graphs/base_graph.py +++ b/cmrl/models/graphs/base_graph.py @@ -1,123 +1,73 @@ import abc import pathlib -from typing import Any, Dict, Optional, Sequence, Tuple, Union +from typing import Optional, Tuple, Union import torch -import torch.nn as nn -class BaseGraph(nn.Module, abc.ABC): +class BaseGraph(abc.ABC): """Base abstract class for all graph models. All classes derived from `BaseGraph` must implement the following methods: - - ``forward``: computes the graph (parameters). - - ``update``: updates the structural parameters. - - ``get_binary_graph``: gets the binary graph. + - ``parameters``: the graph parameters property. + - ``get_adj_matrix``: get the (raw) adjacency matrix. + - ``get_binary_adj_matrix``: get the binary format of the adjacency matrix. + - ``save``: save the graph data + - ``load``: load the graph data Args: in_dim (int): input dimension. out_dim (int): output dimension. - device (str or torch.device): device to use for the structural parameters. + extra_dim (int | tuple(int) | None): extra dimensions (multi-graph). + include_input (bool): whether inlcude input variables in the output variables. """ - _GRAPH_FNAME = "graph.pth" + def __init__( + self, + in_dim: int, + out_dim: int, + extra_dim: Optional[Union[int, Tuple[int]]] = None, + include_input: bool = False, + *args, + **kwargs + ) -> None: + self._in_dim = in_dim + self._out_dim = out_dim + self._extra_dim = extra_dim + self._include_input = include_input - def __init__(self, in_dim: int, out_dim: int, device: Union[str, torch.device] = "cpu", *args, **kwargs): - super().__init__() - self.in_dim = in_dim - self.out_dim = out_dim - self.device = device + assert not (include_input and out_dim < in_dim), "Once include input, the out dimesnion must >= in dimensino" + @property @abc.abstractmethod - def forward(self, *args, **kwargs) -> Tuple[torch.Tensor, ...]: - """Computes the graph parameters. + def parameters(self) -> Tuple[torch.Tensor]: + """Get the graph parameters (raw graph). - Returns: - (tuple of tensors): all tensors representing the output - graph (e.g. existence and orientation) + Returns: (tuple of tensor) the true graph parameters """ @abc.abstractmethod - def get_binary_graph(self, *args, **kwargs) -> torch.Tensor: - """Gets the binary graph. + def get_adj_matrix(self, *args, **kwargs) -> torch.Tensor: + """Get the raw adjacency matrix. Returns: - (tensor): the binary graph tensor, shape [in_dim, out_dim]; - graph[i, j] == 1 represents i causes j + (tensor): the raw adjacency matrix tensor, shape [in_dim, out_dim]; """ - def get_mask(self, *args, **kwargs) -> torch.Tensor: - # [..., in_dim, out_dim] - binary_mat = self.get_binary_graph(*args, **kwargs) - # [..., out_dim, in_dim], mask apply on the input for each output variable - return binary_mat.transpose(-1, -2) - - def save(self, save_dir: Union[str, pathlib.Path]): - """Saves the model to the given directory.""" - torch.save(self.state_dict(), pathlib.Path(save_dir) / self._GRAPH_FNAME) - - def load(self, load_dir: Union[str, pathlib.Path]): - """Loads the model from the given path.""" - self.load_state_dict(torch.load(pathlib.Path(load_dir) / self._GRAPH_FNAME, map_location=self.device)) - - -class BaseEnsembleGraph(BaseGraph, abc.ABC): - """Base abstract class for all ensemble of bootstrapped 1-D graph models. - - Valid propagation options are: - - - "random_model": for each output in the batch a model will be chosen at random. - - "fixed_model": for output j-th in the batch, the model will be chosen according to - the model index in `propagation_indices[j]`. - - "expectation": the output for each element in the batch will be the mean across - models. - - "majority": the output for each element in the batch will be determined by the - majority voting with the models (only for binary edge). - - The default value of ``None`` indicates that no uncertainty propagation, and the forward - method returns all outpus of all models. - - Args: - num_members (int): number of models in the ensemble. - in_dim (int): input dimension. - out_dim (int): output dimension. - device (str or torch.device): device to use for the model. - propagation_method (str, optional): the uncertainty method to use. Defaults to ``None``. - """ - - def __init__( - self, - num_members: int, - in_dim: int, - out_dim: int, - device: Union[str, torch.device], - propagation_method: str, - *args, - **kwargs - ): - super().__init__(in_dim, out_dim, device, *args, **kwargs) - self.num_members = num_members - self.propagation_method = propagation_method - self.device = torch.device(device) - - def __len__(self): - return self.num_members - - def set_elite(self, elite_grpahs: Sequence[int]): - """For ensemble graphs, indicates if some graphs should be considered elite.""" - pass - @abc.abstractmethod - def sample_propagation_indices(self, batch_size: int, rng: torch.Generator) -> torch.Tensor: - """Samples uncertainty propagation indices. + def get_binary_adj_matrix(self, *args, **kwargs) -> torch.Tensor: + """Get the binary adjacency matrix. - Args: - batch_size (int): the desired batch size. - rng (`torch.Generator`): a random number generator to use for sampling. Returns: - (tensor) with ``batch_size`` integers from [0, ``self.num_members``). + (tensor): the binary adjacency matrix tensor, shape [in_dim, out_dim]; + graph[i, j] == 1 represents i causes j """ - def set_propagation_method(self, propagation_method: Optional[str] = None): - self.propagation_method = propagation_method + @abc.abstractmethod + def save(self, save_dir: Union[str, pathlib.Path]): + """Save the model to the given directory.""" + + @abc.abstractmethod + def load(self, load_dir: Union[str, pathlib.Path]): + """Load the model from the given path.""" diff --git a/cmrl/models/graphs/binary_graph.py b/cmrl/models/graphs/binary_graph.py new file mode 100644 index 0000000..bcdce56 --- /dev/null +++ b/cmrl/models/graphs/binary_graph.py @@ -0,0 +1,81 @@ +import copy +import pathlib +from typing import Optional, Union, Tuple + +import torch +import numpy as np + +from cmrl.models.graphs.base_graph import BaseGraph + + +class BinaryGraph(BaseGraph): + """Binary graph models (binary graph data) + + Args: + in_dim (int): input dimension. + out_dim (int): output dimension. + extra_dim (int | tuple(int) | None): extra dimensions (multi-graph). + include_input (bool): whether inlcude input variables in the output variables. + init_param (int | Tensor | ndarray): initial parameter of the binary graph + device (str or torch.device): device to use for the graph parameters. + """ + + def __init__( + self, + in_dim: int, + out_dim: int, + extra_dim: Optional[Union[int, Tuple[int]]] = None, + include_input: bool = False, + init_param: Union[int, torch.Tensor, np.ndarray] = 1, + device: Union[str, torch.device] = "cpu", + *args, + **kwargs, + ) -> None: + super().__init__(in_dim, out_dim, extra_dim, include_input, *args, **kwargs) + + graph_size = (in_dim, out_dim) + if extra_dim is not None: + if isinstance(extra_dim, int): + extra_dim = (extra_dim,) + graph_size = extra_dim + graph_size + + if isinstance(init_param, int): + self.graph = torch.ones(graph_size, dtype=torch.int, device=device) * int(bool(init_param)) + else: + assert ( + init_param.shape == graph_size + ), f"initial parameters shape mismatch (given {init_param.shape}, while {graph_size} required)" + self.graph = torch.as_tensor(init_param, dtype=torch.bool, device=device).int() + + # remove self loop + if self._include_input: + self.graph[..., torch.arange(self._in_dim), torch.arange(self._in_dim)] = 0 + + self.device = device + + @property + def parameters(self) -> Tuple[torch.Tensor]: + return (self.graph,) + + def get_adj_matrix(self, *args, **kwargs) -> torch.Tensor: + return self.graph + + def get_binary_adj_matrix(self, *args, **kwargs) -> torch.Tensor: + return self.get_adj_matrix() + + def set_data(self, graph_data: Union[torch.Tensor, np.ndarray]): + assert ( + self.graph.shape == graph_data.shape + ), f"graph data shape mismatch (given {graph_data.shape}, while {self.graph.shape} required)" + self.graph.data = torch.as_tensor(graph_data, dtype=torch.bool, device=self.device).int() + + # remove self loop + if self._include_input: + self.graph[..., torch.arange(self._in_dim), torch.arange(self._in_dim)] = 0 + + def save(self, save_dir: Union[str, pathlib.Path]): + torch.save({"graph_data": self.graph}, pathlib.Path(save_dir) / "graph.pth") + + def load(self, load_dir: Union[str, pathlib.Path]): + data_dict = torch.load(pathlib.Path(load_dir) / "graph.pth", map_location=self.device) + self.graph = data_dict["graph_data"] diff --git a/cmrl/models/graphs/neural_graph.py b/cmrl/models/graphs/neural_graph.py new file mode 100644 index 0000000..5ea56a9 --- /dev/null +++ b/cmrl/models/graphs/neural_graph.py @@ -0,0 +1,77 @@ +import pathlib +from typing import Optional, Union, Tuple + +import torch +from omegaconf import DictConfig +from hydra.utils import instantiate + +from cmrl.models.graphs.base_graph import BaseGraph + +default_network_cfg = DictConfig( + dict( + _target_="cmrl.models.networks.ParallelMLP", + _partial_=True, + _recursive_=False, + hidden_dims=[200, 200], + bias=True, + activation_fn_cfg=dict(_target_="torch.nn.ReLU"), + ) +) + + +class NeuralGraph(BaseGraph): + + _MASK_VALUE = 0 + + def __init__( + self, + in_dim: int, + out_dim: int, + extra_dim: Optional[Union[int, Tuple[int]]] = None, + include_input: bool = False, + network_cfg: Optional[DictConfig] = default_network_cfg, + device: Union[str, torch.device] = "cpu", + *args, + **kwargs + ) -> None: + super().__init__(in_dim=in_dim, out_dim=out_dim, extra_dim=extra_dim, include_input=include_input, *args, **kwargs) + + self._network_cfg = network_cfg + self.device = device + + self._build_graph_network() + + def _build_graph_network(self): + """called at the last of ``NeuralGraph.__init__``""" + network_extra_dims = self._extra_dim + if isinstance(network_extra_dims, int): + network_extra_dims = [network_extra_dims] + + self.graph = instantiate(self._network_cfg)( + input_dim=self._in_dim, + output_dim=self._in_dim * self._out_dim, + extra_dims=network_extra_dims, + ).to(self.device) + + @property + def parameters(self) -> Tuple[torch.Tensor]: + return tuple(self.graph.parameters()) + + def get_adj_matrix(self, inputs: torch.Tensor, *args, **kwargs) -> torch.Tensor: + adj_mat = self.graph(inputs) + adj_mat = adj_mat.reshape(*adj_mat.shape[:-1], self._in_dim, self._out_dim) + + if self._include_input: + adj_mat[..., torch.arange(self._in_dim), torch.arange(self._in_dim)] = self._MASK_VALUE + + return adj_mat + + def get_binary_adj_matrix(self, inputs: torch.Tensor, threshold: float, *args, **kwargs) -> torch.Tensor: + return (self.get_adj_matrix(inputs) > threshold).int() + + def save(self, save_dir: Union[str, pathlib.Path]): + torch.save({"graph_network": self.graph.state_dict()}, pathlib.Path(save_dir) / "graph.pth") + + def load(self, load_dir: Union[str, pathlib.Path]): + data_dict = torch.load(pathlib.Path(load_dir) / "graph.pth", map_location=self.device) + self.graph.load_state_dict(data_dict["graph_network"]) diff --git a/cmrl/models/graphs/prob_graph.py b/cmrl/models/graphs/prob_graph.py index ebf7517..90ea326 100644 --- a/cmrl/models/graphs/prob_graph.py +++ b/cmrl/models/graphs/prob_graph.py @@ -1,11 +1,11 @@ from abc import abstractmethod -import math from typing import Union, Tuple, Optional import torch -import torch.nn as nn +import numpy as np from cmrl.models.graphs.base_graph import BaseGraph +from cmrl.models.graphs.weight_graph import WeightGraph class BaseProbGraph(BaseGraph): @@ -13,12 +13,14 @@ class BaseProbGraph(BaseGraph): All classes derived from `BaseProbGraph` must implement the following additional methods: - - ``sample``: sample graphs from given (or current) graph probability. + - ``sample``: sample graphs from current (or given) graph probability. """ @abstractmethod - def sample(self, graph: Optional[torch.Tensor], sample_size: Union[Tuple[int], int], *args, **kwargs) -> torch.Tensor: - """sample from given or current graph probability. + def sample( + self, prob_matrix: Optional[torch.Tensor], sample_size: Union[Tuple[int], int], *args, **kwargs + ) -> torch.Tensor: + """sample from given or current probability adjacency matrix. Args: graph (tensor), graph probability, use current graph parameter when given `None`. @@ -30,57 +32,62 @@ def sample(self, graph: Optional[torch.Tensor], sample_size: Union[Tuple[int], i pass -class BernoulliGraph(BaseProbGraph): - """Probability (Bernoulli dist.) modeled graphs, store the graph with the - probability parameter of the existence/orientation of edges. +class BernoulliGraph(WeightGraph, BaseProbGraph): + """Probability (Bernoulli dist.) graph models, store the graph with the + probability parameter of the existence of edges. Args: in_dim (int): input dimension. out_dim (int): output dimension. - init_param (float or torch.Tensor): initial parameter of the graph - (sigmoid(init_param) representing the initial edge probabilities). - device (str or torch.device): device to use for the structural parameters. + extra_dim (int | tuple(int) | None): extra dimensions (multi-graph). + include_input (bool): whether inlcude input variables in the output variables. + init_param (int | Tensor | ndarray): initial parameter of the bernoulli graph。 + requires_grad (bool): whether the graph parameters require gradient computation. + device (str or torch.device): device to use for the graph parameters. """ + _MASK_VALUE = -9e15 + def __init__( self, in_dim: int, out_dim: int, - init_param: Union[float, torch.Tensor] = 1e-6, + extra_dim: Optional[Union[int, Tuple[int]]] = None, + include_input: bool = False, + init_param: Union[float, torch.Tensor, np.ndarray] = 1e-6, + requires_grad: bool = False, device: Union[str, torch.device] = "cpu", *args, **kwargs - ): - super().__init__(in_dim, out_dim, device, *args, **kwargs) - - if isinstance(init_param, float): - init_param = torch.ones(in_dim, out_dim) * init_param - self.graph = nn.Parameter(init_param, requires_grad=True) - - self.to(device) - - def forward(self, *args, **kwargs) -> Tuple[torch.Tensor, ...]: - """Computes the graph parameters. - - Returns: - (tuple of tensors): all tensors representing the output - graph (e.g. existence and orientation) - """ + ) -> None: + super().__init__( + in_dim=in_dim, + out_dim=out_dim, + extra_dim=extra_dim, + include_input=include_input, + init_param=init_param, + requires_grad=requires_grad, + device=device, + *args, + **kwargs + ) + + def get_adj_matrix(self, *args, **kwargs) -> torch.Tensor: return torch.sigmoid(self.graph) - def get_binary_graph(self, thresh: float = 0.5) -> torch.Tensor: - """Gets the binary graph. + def get_binary_adj_matrix(self, threshold: float, *args, **kwargs) -> torch.Tensor: + assert 0 <= threshold <= 1, "threshold of bernoulli graph should be in [0, 1]" - Returns: - (tensor): the binary graph tensor, shape [in_dim, out_dim]; - graph[i, j] == 1 represents i causes j - """ - assert 0 <= thresh <= 1 - - prob_graph = self() - return prob_graph > thresh + return super().get_binary_adj_matrix(threshold, *args, **kwargs) - def sample(self, graph: Optional[torch.Tensor], sample_size: Union[Tuple[int], int], *args, **kwargs): + def sample( + self, + prob_matrix: Optional[torch.Tensor], + sample_size: Union[Tuple[int], int], + reparameterization: Optional[str] = None, + *args, + **kwargs + ): """sample from given or current graph probability (Bernoulli distribution). Args: @@ -88,14 +95,18 @@ def sample(self, graph: Optional[torch.Tensor], sample_size: Union[Tuple[int], i sample_size (tuple(int) or int), extra size of sampled graphs. Return: - (tensor): [*sample_size, in_dim, out_dim] shaped multiple graphs. + (tensor): [*sample_size, *extra_dim, in_dim, out_dim] shaped multiple graphs. """ - if graph is None: - graph = self() + if prob_matrix is None: + prob_matrix = self.get_adj_matrix() if isinstance(sample_size, int): sample_size = (sample_size,) - sample_prob = graph[None].expand(*sample_size, -1, -1) + sample_prob = prob_matrix[None].expand(*sample_size, -1, -1) - return torch.bernoulli(sample_prob) + if reparameterization is None: + return torch.bernoulli(sample_prob) + else: + # TODO: reparameterization for bernoulli distribution + raise NotImplementedError diff --git a/cmrl/models/graphs/weight_graph.py b/cmrl/models/graphs/weight_graph.py new file mode 100644 index 0000000..a752551 --- /dev/null +++ b/cmrl/models/graphs/weight_graph.py @@ -0,0 +1,94 @@ +import pathlib +from typing import Optional, Union, Tuple + +import torch +import numpy as np + +from cmrl.models.graphs.base_graph import BaseGraph + + +class WeightGraph(BaseGraph): + """Weight graph models (real graph data) + + Args: + in_dim (int): input dimension. + out_dim (int): output dimension. + extra_dim (int | tuple(int) | None): extra dimensions (multi-graph). + include_input (bool): whether inlcude input variables in the output variables. + init_param (int | Tensor | ndarray): initial parameter of the weight graph。 + requires_grad (bool): whether the graph parameters require gradient computation. + device (str or torch.device): device to use for the graph parameters. + """ + + _MASK_VALUE = 0 + + def __init__( + self, + in_dim: int, + out_dim: int, + extra_dim: Optional[Union[int, Tuple[int]]] = None, + include_input: bool = False, + init_param: Union[float, torch.Tensor, np.ndarray] = 1.0, + requires_grad: bool = False, + device: Union[str, torch.device] = "cpu", + *args, + **kwargs, + ) -> None: + super().__init__(in_dim, out_dim, extra_dim, include_input, *args, **kwargs) + self._requires_grad = requires_grad + + graph_size = (in_dim, out_dim) + if extra_dim is not None: + if isinstance(extra_dim, int): + extra_dim = (extra_dim,) + graph_size = extra_dim + graph_size + + if isinstance(init_param, float): + self.graph = torch.ones(graph_size, dtype=torch.float32, device=device) * init_param + else: + assert ( + init_param.shape == graph_size + ), f"initial parameters shape mismatch (given {init_param.shape}, while {graph_size} required)" + self.graph = torch.as_tensor(init_param, dtype=torch.float32, device=device) + + if requires_grad: + self.graph.requires_grad_() + + # remove self loop + if self._include_input: + with torch.no_grad(): + self.graph[..., torch.arange(self._in_dim), torch.arange(self._in_dim)] = self._MASK_VALUE + + self.device = device + + @property + def parameters(self) -> Tuple[torch.Tensor]: + return (self.graph,) + + @property + def requries_grad(self) -> bool: + return self._requires_grad + + def get_adj_matrix(self, *args, **kwargs) -> torch.Tensor: + return self.graph + + def get_binary_adj_matrix(self, threshold: float, *args, **kwargs) -> torch.Tensor: + return (self.get_adj_matrix() > threshold).int() + + @torch.no_grad() + def set_data(self, graph_data: Union[torch.Tensor, np.ndarray]): + assert ( + self.graph.shape == graph_data.shape + ), f"graph data shape mismatch (given {graph_data.shape}, while {self.graph.shape} required)" + self.graph.data = torch.as_tensor(graph_data, dtype=torch.float32, device=self.device) + + # remove self loop + if self._include_input: + self.graph[..., torch.arange(self._in_dim), torch.arange(self._in_dim)] = self._MASK_VALUE + + def save(self, save_dir: Union[str, pathlib.Path]): + torch.save({"graph_data": self.graph}, pathlib.Path(save_dir) / "graph.pth") + + def load(self, load_dir: Union[str, pathlib.Path]): + data_dict = torch.load(pathlib.Path(load_dir) / "graph.pth", map_location=self.device) + self.graph = data_dict["graph_data"] diff --git a/graph.pth b/graph.pth new file mode 100644 index 0000000000000000000000000000000000000000..c060948c2c3c225a65796b291d6a59bd858f98b7 GIT binary patch literal 187799 zcmeFYc{Env`!{YDqRg`-ndjMku6>FULWWeNN#@dUn@TD3tVonfNTpIlBlp?+R5TD# zi9(}PG@wCqdc42y-}77R^ZP!(>Hp`fb=JM^Yn}Z%ue0}c?X$1fzK)lhpb#J5lqr1w z?U>Fd!xtD86dn;5Vj3J66KJw|edGd1DZb_ZaU==s;PUxNPPPt<3fvsNdSgh;mQ7LX zx%@#vn<67af?^^zZHyN8j0z5k3JIPc5fmfN70~zL3hMbypKP-_GB7?QD%!-%WJ^dy zSa=LqC@5yr#H3AYwE z{tWKa?C^gGNyc!ce7Mv6r2d1|+K9kt?)31*u-T$Cv^*?Fp{Rfs6SAUXLmTU~y z;9pwG8C*j*<$rq1=wC477_NyA*YrPVS^Z1P>_4Eax#p9!lw-IS|I(U7viujx>R%-5 z7_N;E*EV(s*UnGupT6G^5)=53yK(LRYg{;bev?){F13RT|26D!lm0U4Q(WZ6AN^m3 z|HFs4^bYQv@c**@7ta6mEbiPLT!;T|H|IL;;LiK+b_?$O9bBjXZnxwv*ui!F?{+J$ z%MPyVf7xTB*G8Loxrt73G<&~p`lQ?ab9(*X#+MYI)qlj7so4TYv;RwU{YR+%FaHc* zC_>++RFg}ubP%aGrcY-QB0jbhEv*t@iz^>vzt7WHA5T59?q&a^PNOYlN6|)MBbNWpVOllfKD|u+7&6+e&)$q{W>dE^ zAh}AFR{Q=LlF|!Um(PW)TjeN<9hgBc_MT3N*l|)YlusY36=cWvYS4`beAzcE+)&MT z#MUU=kmQyg38i&C;tS!@BIVv$SsYPw>d})jxAsX_r;K#!Ou{N z)Fpa_i2>XH^cU%_DL~eC9jH!nG10i;iZWm7v1U{2Xi={_Bv-_Z4hx?_x6c83PADH$ zx_>uo?NLB>rp2%x^X}8tU#rQ8L>an-fIjsoi{=yep)bgB=r@|_?1c{-5x=-MX)ldn zTbHk3qpnud_ij@_phzFxOUAg`QtPuR+KHVsja1HPeKIt_$AoQ0xerjp?q zt4WEXA8PvZfS&PqDl2ww9$VH_$kt0+QQnD%pbquGl=}Ydg{(cth`-T&=3%>{2 zK0O*;?o*&8GqqXWC;lM&q=r^hd%+GcR&31>ADZ{KlpN@7MEb_t$zVq_y>^}t8$VZ( z7OTjo^OV(Cfj64qyyPz_TOG-c=l{gVdgR%i*TrZhjpt;A!ArFAvLTT-j$-G&;%9Z8 zJ?Uxu=h&yhfQ9)g&sS95gS&f>?h*B6nDEL~b5cOTkdAB*<%+Od_wkCDxne0KdWNow$~9_>BFhi$%gl-0|K zpjF&yS})s=1g~l8D5w^wyl_MP1K-Gip}XY4;HieRv2B{T!3s&^V3tk?$GNN z??R{A9^rgTbN2USo4R!Zv~P|I=}@_VG@OUfB083hc+rRye{Ml(iNOod-RW7?{ ziwv8)Q32j9TY>sxjOc6LTI{E-8)*X*LGonJ8nWZ-JeG|vVOK5Er#sdjVt;G?fY)t9 zWZRql^w`2;T0Pf*cAI;c^(p!ggziT6|0u~h3g_U?30;mh>_e*g8f`*$&SBdG**aO zdZeCh)6PKIV@9CqzJPtVVh`QqKa7@NyTd-K$-*C`I?0;pCrG`oBKx+>j!qZ2Oi{{* z>Bnw7bn@j83bwe)7BB1|-%j3ztc?+9L$)xx^QRj= z1a__>kLva*VeP1MU?D9)AHF8Z_6qyb4{w#wWe?=oy`tmDF+Y<%Yg0+ThfCzR&^4lJ zdY65>F@)85SVboEH_?A`a_EZ&*U0q}BUZwwk-P~_qt(+dvf@uQ*}ILW*%Log*uGUB zyjs0P+E+M@?){pDHe^0#dx!U-We;tLPFoTy9#~Hty2I$z3X51PiD7j6Y69x?3S!$V zPos|mHZ(J4MXIcJ(EKh3QTF#;v|5i79d4zDcnf!9{da#zHwOCVKmn4>`GdTts?i~P z=OA^r4Eknb1|6kpO~Oy)AQ$_6^gX3Y68C!<3N=e(i$m|zC;a5-vgDe= zBR14|9!dN#m0oBTz;-Gr5MG8o8G0U1t7!PLkf=+yxCN4pYYI_?Lppx7Y#%b`#L<;E zZqoh9OKFYAf6?)R?{Mhn5@dW{0dEu4CD+1+NpQg()c$N1Yq6-0cvSha$1bO_7mAhG zvC}4a>91{crR{W9Dd9Hk8qPp@cTdu*HoDMLOe^S!CtY+uTf;fw9?7EPr)byfIqV(d zeRNt$2A${DNq#EMVGH%uS>22cAQ5K5{=T0^AL}_x*U#HUk}Go9BVYR1d8b`zug|)4 zo4yTlnf4O9gcKr%vSroXXQR=q8sv7ym=(uaq$ltP5uTIHrsT)d!JB8$yR<6ULE9tr zS))L-*F%j(nNQfd+RJR1_b2k&J-tUt_ z){64ex0@`f=Ke=v3frBw7y&m zJ4!0ii)DJW8MeW3GL2{A#oy5$vV+-k!*$2QUaxt}5Ic|AM#&U@7S;SyVvc$W?3 z(&U&%^lf<`*~!8L>3*k6Sl6TcIm=_}cl zrQJkirvqzyD4X^QjX`IQ1k)li#z?VLyZ7jj)o*sO7G}>O zeSHNR`NM@&KK~8RHf7QBj`qZFN+Gz(Zl-Nk`m#@LW{_{|r?IqYKEAHFjvf4Ukx1BI zB(HNRyZ@jDy*?nFUFbPX?)N;WU+zp|S(zk4UOAGW_ZsXIuZ`>$ zyCv2jLeH}TPQqR(;>QH?*vw)aP{A*J)_HDh6uZ4BAEM;TKhMgHG+^rC1yZI}i*}VSK#_kv z=&?k9cKHUg|4_cdsBpQqd2JG19_U(sc; zM`#zXoFh?e0cCZ2NcolzD62;nyJl6hH`j-;4@;9-zbt~Dx`ogoe6?so)f@MScEEe^ zU=Qb)&^i!8&sbkgA{L)VC6QW$s}an~dzjN&H>KzSOKs%3C7E?$8`*-R%Iu&=2peaA zjO|?XluCK2&z`m}C+1eW*(6sz=KMTNKbpUS{?M?K9+s+RUx`K1ZBh#COVJ~=>2NtI z$oPUD36`^lx|KMR5k`u&n(QU#V$vYo%68~VvkF<`h@JBVwv;>23A|1eB6Nw|5iMo^ z_Uh90LI&)H$~WXhc{9szjKX4zOresQ+bmg$!1FdMbhY3w@m!wzBw`Zrm3Zqw)dXYU1 zUr4{vY3w1DjVvlVXs@TfteVL(mRk^xT6GNB^F7aK^H(LD=Y}F|%^^*)S0a|)ZQjUk zHoC^Xoqvuj{#=EuPFTm_NOQ-J*JW!SO9 zI*4xEi=$UuVRzBpn6LL1v2%!K7aj>D*;mI=i+m%xUcw`8BRk1vvl#XtJ)qX_j%UR* zKE8j>|0g{_`#<#n^97EY|F3$$KjO&$KK^$P{O=z4U-!U2@%OSaKU{NCi~HyN{}F!{ z|6k(o{8nqqR{RJx-;u#<7Ttk8*FDLp@-uKcP#%?w@t7-&GwS=TMjo8gCBsu>$w2&J zaQ87sIz7&~!SS!%%3>EPfISQMx&jcNY$RwH7$Yy+U>HttB>Dk4#Lf0I+?XRuLK9|E ziEpGyUf){Yl}bA}Gmy&kI!!K4ruIWmK>$9|x*t3)WuwE}_G6)vjo@VNhJ+ndN!1=5 zBz;Q{V!I4T=88Ee>~bnRC{V!x#lI-U#(2n%Pl4hxDU!Bu%-+~ZiTdTf0MC(GMtBD= zfm6RXzNXp=g&mRb@vknrR&)>+?$Mx@KjJ4zUYe-YTLp2mzJs!X|K!{~1^7I+g1f67 zEEBqdXT>zY-$&cg2Xq79I%-VRZhgi&!HQ(%YHiB)atI01(u1}`+lZk3D)M<%4Le6}7og$nZp5$*eB3+{W)cN9e{9VqK=gAd93M*WD#t*RWayO&#zarvxlj$U^xQFm&4y<({mWiJExKgcCEqqVLc^!WeLfwIuCEZUxi(BrxA^4XL5^=L&~3j#{xkD=y+~3)-Onb z^i!)q=iF+X+S`gthgFEJqbkYSHiOxlaFOXvcSEP&mqEmv6#Tg;8eQqJhBI$zbXg!5 zhKGgp#w!^wz@!*mvfl?%uiL=%Zyur8`&b-scQYr zuWyBaa#-7=+K|7y4IfzQip$pZQ)xDCIA!o1Hr@Xm%N^>0X{TqS^0E}t8o|IrzTL=W zY#E4&nv<)mEs0V0H7xM-6+Tei1s!V%wd-sjlh|npzlST~KwlbhF*wI`JwD2rVJHM= zuSKDO^$l29G#gZt($P?Q7R0CT#NL%^gb`?Fm|ht?V{INH>l@*_z+%V@kO4u_C`Kbl z1w}uUV0LumLaC%M85o^P6ke*5CBJDD^;HRt)Tf|!Wf5HS_Xnf6=^!OR*TA5HG+4G* z;{t^aygKVH-tRd86EQXTr`Sy>@te&w*B*mHv-w2s?&O$sZpU}_q?6V2($UC-AZpqE ziZh*m0?IDl#z&hgaIWFo#(wpD>~a4CPa>let6xupi20##IHD9XD%UW%Vka3PqY^yO za|dSh8>74pd%^Nz2w4Spn4%|JvD5_ta=q^*Tv)dXO7pV7PpF8QnkP#_4kmN9wQIs| z-Dc`T$}%*w*OL^ciIX;yX~=G68jQTVgiq-Qla&SmsCH5>e)lpJ+)k$A2d3+(E8z^5 zFRQ}V7fLYuNF2?JRL4Hs;)qCuIu%4;vC~5NJjrAC;ry$G@XI?8Jy|D&{NqfCQRrs; zIaUmvE6bqJ7#I7W6(ygNt>JyJKUo>Pfl1iAiKP2$!`uTOa83e0lG@t9tF9P^;XUfe zG;MCR-;9$R$SLk@ABvVoczgl~&{ z0JrXGXveYy5?A-FF{g7En)ON_8OK!P2iXRsSf++qH6;uc__|QQ*7UyuMJs zx+1BRm8r-n)(Uo5l);8KJE>o#rX({_5#P`&U|MXx;#i80QOKQ-b~NuGBEO^(JeiG5c7}tZ$OWAHbstV>I6}!P^uyrSlC%N-d0iT8LJP^ut7TDsx)&EO=;pW7GS+*skjq z9x342`}FUC)XYUN`|WZVlMF)Ihi8ySZ_ZKol2h>Vb@otmRurCXe+yf}PUGEKoy^9W zUD*GcBN2`)$7!RfSYhUOyylH2Iy}>YJge6yI>zS!9S(!E#zx2rFQKdpA2COr^Kkw2 z?^L-lg{1$;W9_f$sNdri&bYh_A24L7z&HiwuX#5l_)jORs=M$4w?p_yWeEP98;(l1 zWbr~{C3)*7((#7%MbsXtC}b&J2upI7QQlLgk>f9eK(2ikHd{Oe?XZ~&w>ND@VcUxt z8@>Y&=OS7peNgRVJPYu`#V#*srlK`La{ z2SqYsunyI~5+Wfz_Pp7{Ih;R@YDjdu3z1HrihL7)W6gjK#A|yoHcFfd?ep*8%{wRG z`3t3x!O3@!k)Vs390Qmi#}n}8*?;kOha;51oK-O7!N*h@@1+bDtVQyB5Aa@}vV*H( zo_J(w7V$9|#pF{4)Sex}D>bUHU4sqj*#8qmk88rK_lD&2%$Jm-r-l-&?q@)!#i zWfO_$eYnNPnfLn3N+NQY!1KxY(b}tnO!oz#H8-TtkK+4y`Nju~x`jHK?{pR(Ih5kN zEz6gaiTV#r}@4_MTUft7?4F>-wg2OF+n*&sW#Q8oehhL3R`I;TSD z>nky&eOkpIVU2=lT-f1)zcfxantyLA#de>sIW zFIAs({`mwkYEH!IxfI&E=PW2F>Y?!YGniYnFq0S4PrWiT!)vc?f|9Xkuv6td-ne-J zKaaFUsb@vepB24WVv8mj86ARkoac=2_buq^?^2li{TJT#HG{z1T6{m7;Vh&O0Yt~WdDyO|rQL!7GgXZYgx zEHJ&&4Ucm7Fvs?DNw@Gg?9nzvb2c1AkoG$sX|_Cxg7v&fDEMGG3WfNX|#tXet_>jj-sP6vQ7? z1#|wsfH?P?oXlIj@Hl=mF&W?^dM;Hs%+H(*>F~hfj5j`$@(80A71(@i6*3alfD@Me zpg-Y@5^XOt2(KWvFC4IM#uAkG(;ej7?1|3iuh=8*2&~Th1c&c;Vaq4Y*f3fMh?grl zdd?hm`8rX({dLeBNK%h0+f=W(~m05-h)nQAtD(D=G#0t{x)BH@#Bx7gl4*lBXT z@b{!7+Ly1#>>n$l&Lql^EBC7@s!EdZ=XK!LQE_N7U5>10UjeU3FSwqb4^RDrh)%FG z@UBPNsqCggC?d~7;6pSWUGqNG=h-x++tyII zJ{4Jg*^h6h0+V5BMcjQBAwKWc}#1RH`H+>^<^8xL`fXDAU_ zb^@=AR-xuB9A}>SZN!l=eE556DmKZ!3Im5TN#KqMxc170V&bh)=f)hoY0FfU*DA;a zjvB$A%fYa)y%1Uw7Qt1OJkC(f6k?*#i{tc?sFUYs!V*y%ve(Oaa^FA-riJp8D64OH zX=o;H*H%G79tr3`z%IO9%z-5NO+}k?zQBp*TfnoDC-L)V5cf0+emp9|z9HdcWsNs} zO5K1{Yb}V;nlc>EI}2`Rf{0FPgWH$9km+kdwD03RXjhU)FZIS5w<25gt!E*Ll)TO? zBsTb~WHlr(CWy#h#cyRiQE_P`)^gxSLXvaQwr}4-%wPx>hwnstB}-tx=vj*PsKT9E zUU<2R29u#(2##i+C~etASRAyKv?LASiXd^+rWh=%zG_K0Jj*FNBcoA*PDzdu3&y=%z|>m z4aly;pLlRJ(K@R>CVHbMD%iM?Tv~VycR1EJKA{ZB;O{edrPy{9niNh|2fU`fKK_m8 z#QlMd9u<_maxGlBtB6e6Qh4g$`eBvK46?OnGXG4@gxNVsjH$aZ`nh3%w|`ujJW4)} zKXuPQqJLZPY!y%9KX*3nGu1&_S)X}J(Nh@qPXPRO23~159r4SG6Qfz0=%}#}F*zxN zl=nTs?ea^|>*8Fz$D*DozTrxy$lt^_z0TlgrFY?^#x8Va-WeQz%mVAr{*IkxjspKJ zQ8a9M#%@`18&&n|IbKQ4CciYoaPZ$n_NEIr-SnLrs23ljZFRhX}nXK6R;uwI`iA4vw;kYqq$1caEXHu$!IMF>(b?LZE_#q z@G3t(v|Ru>JbD3dO5QTroBF_bPzO&5-w6E$ft*(t9T?NfYFKj75)Z4oF)iZBm_43@ z&4u+j184M5%H~@z)05z2uM^;QZ7Qmr=M1eOW~9zro7C@l1ke8{lJZaubZfYf^Y+1E zxcp`}YznKUk`{b{rWMDjzRX~v@gNaGmst=IJ!`c0s28a)7RG09>Y{bq4N&0fa$p;O zVCl7bNU~iAIXycJ=9kP#o|!&~aSrwYe%&HWDvW8T-@S{5UxqWhHdrOz$TF_ z8}??BKkb2y0{+Bymk!BVlZL`pu7djwtB6Nb9NJ6`z!#i=N7d7D%iG1|TxK%t4_E^C zM^EEeei1?jEU_dt0=j;=D75%6{#z&lJr9mEGHE9m!QVPaDJz30UTVauBNGgNk1w9H z^DotLeJTEJwgGXjOh=DZC72n}Hb_}J0|Y)~@-9eEh03X77*17Eh0Q16HS}}DJxz)I z+RgB$>ZZLPKLMuhA`WZKB-d#jQrp^sFI~NYJ?n2nTRH1CGyK(zaQ6t2P(2`r(#vt*r4kjlC6u-y{+>FJXioHNKf3*)ezW_Ww_N$jjUfSJS1yn7F}!7I%KYG}GYzWP%d zyD50%{u~QN#8{MkBlXx_bS9aw5d->91ynvsd7w6}4dRqyU=~4m0ERhe55jmO6aI0)468O{U1+#V`7{ zGr=4oGOCw@q|Y3Lo4ecz-?}`!C}##$I-!i#QKsbQ?vptB)>qD<1?kiyx(SxtNJYEb zzA(XF>DXZ3EEKR^0slR*9l18{qV65nK=5@YQX5wzPILwtJ=67ysa{lnbMn%6I`Vrv0njp zd)L6F%_4N3b-kk1McxaG<{T-S9MCroYR70=Z~uWf;B z%~VBerSlkN?O-_ltsUFH(!zqKb=d00JY4@~I^Jic49d=8#Jy-czHI%A;k7%!f&3v# zAkmoktd~NibA!aQwB-J!&qd+V^~?z$-5bsOFEqV@!HZW@Mp_BlviqiMxCm$ zld1-izG_WsLla?++!-oC%7hpR`{R#kYLKVx1P<3s(V)t9+`dH>9dX!89f}QQ>JN3} z9S1*C8Oq*x)!Q@7{!##>w}LNJY^o zvvw0ANq4Q#?T=#Q)W|)aD^rMveDv^w);G+@k#zi4e+n~UluvCvDg)1U*I>(CCSbQ< zG3S}|3d%Vf5y6Lf@b&KjeCcHY^|o#vGx3WiW8FT)XuCD2$@0TSI~iEN^$b`%c*?qK#}A9l>icspQHSAA5PXbMfKzPc;=`%vb9+TuSev`gY~O` zr=W^6OwM43&kSUK4F!9%VXDdZG5&S75|Uz7$dI@qsvB}5p)2R0Te4fwkJeJk(<>VM zd0QZOs}eMydW0XY(kFr0);PMH1M#0F$zZPs8jY*KvyYWi*$qj^cnd!gh?_=?3bOG~ zOB-c^msMZ`07%P*(rG4P3 zrjOR=JCY^F!62bFlQiUA0%2zjawg**Osq>nPR)DqZH9xrH>;wbF)PWdv3|-T<0>Ba z&qj9=x*72Y$C&q#(_yek4K1;@MlR!hpfIo+289an3rSOywAPA@iA-QGv>y*loPui) zg0R!3&rqM*fJe8zhu2%eh@rnHN=|)9^+>Ho({CoAZ0b3z)EfYu9mCX|S&2wFYCl}h zsKh=$Tw&=RO=O_uP3(=-;E2`~^udvW1>@>uRZ|sc4>f?z?cKzC{S&;mUXB?ke*@bO zN0W`}Ca6|>1piU909m0v@CsW6txap;){O^{`Q;hjwN3)8^8{e2lpIOfFN3P8czC-^ z7_%#LKDxj?OC3-NL09&s!I+mCb?=HN=Ih=H>Sv7cnW%9}A?hpUkQNXQQ$$T7hA?A? zB;i&sAjw)&nI8@OcoUm~Biof>$?8+E^Fue5I5GbrK#CTZQ9Mx&HVXQJ*ABRn$BZ43tza3M zkDKwIos*ir{CQAG=3$fbVmP`Z4i8=`#ypXaSbXbdnD~>2^a7@#%`=Wd(39g(^-u&? z`zzwXKT-JRjb_}{D2!s)4`Ry;MNFr{I4=JNXyI!g9O*R&t(%=hmTxsjC%cAm*9}3E zd%J=&rZWQ{Hfd(M)=Vb>%~I4VErQ>!T8uT0x4<7^6Jq>|#8tAaM}c!fJ#h8kb}cHkuDNt1)T9OycO$VaJl*t7g0zVUcHS#sW4`_ zI?0#CEaO#yUac$PoLWtWlI4(}=Qr4Pp2kW$vcdLp8~%GP3PNXckVWKT6l9kTL3@g^ zz3e)o6J$v4Yvsa($T~7#oeygrGr$!_D%5W8g*bC~6c;ZZW%>{8X3lf7?Ce|uV0*(F zG(FCXbZD%{I}@{b$z%rPy?MhSCbz-aP?nU{m=lFNiHLJP8vi=b%`AVo8l60#K(_rz zVEl`W@v|+$~t_1fkrJjDRlOnt*BH?uJKBFJW_ME>2Bfg}iGPF-k(I z$jdO5So-iF_u&xkh1y09q)M)&2U2z32Z33(2!Gxph4#A6K&Dnb%%+4iwE6UXYPgKY zQ?0YX(@vG*cD^a-?CCmuvsE15kJtvI>j8BnNFj+gH-Y?-V+wLl;dcHMe8f~9FZ8I{1V z?pdSx^JilxS50*3+I%!DEQt1gbSK6yenQEkwRpmQ9r+U{1Dm&*L3&C&)jA+WrZ12p z73+iGz-ek# zLJK6?V0%#$wbSODebC8B=uIoZOP^|!<$)Bqe<@|Egr<|SDN;mwS2bQPYmc91su8|@ zF0i6A%*%x5r#kz_UiD9z20d2lwNOi}Bd+{RGy&b`p2qzJWEKO(pjpXrPxBdT=2o3CGK< zCZ;o_bJ9wo}dC<$Zf6! zC&@41d9V_9X^k^Z-)%^=-(7ga`~j!*@36vP5!|TZC-HJy;YWZrzSd%e&~ZH&Ke7&m zxZJZFKq`b!Z-n94U13xPREV$j0^X3@4D21%#DKgpiki5FpKK6A#Y?85H-c|?pQYRJ zm_0=d8bi^?U%L>jyNrWJLc!vYKbiY>8d5x?gq&0bN%rcSl+}iF%{*#DI*`r*79H9{(Sq_>Gu@2Z;= z*A^lhdwp^%Di%AHg(GppMJVm)Ro-1sTf{iD;h{VOqLHWxL#JK9a>9<$n*5eP=Upup z64Jw`gZJ>506VyTDv`|6O~$XjG~;gtf=KuLQ~c_d8ku!V7zJl4psrM5(ibR6rt3Fh zQ^P0BYobW9=j)Ss`+J#O9WDwvQU^`43MgyEEx4ykV5@viFe`{Zi zA9sro6RTW(i>8gb^F<{dl8p$}xC_ND;6=l`8`JkI?1I&t{&cu z2BOx~d-$z@FDG%uFMR7^HO{D;OSIJT$+7)uNbK1%D0q9EN*Ja%J68?iUsi9JPw$)| z=)f4R_6jHa=c}OjfRk|DClEz{Q9~MMZ&L5(yo3Vj3m`FY0Xnt{<2Cc^DLuL4n3Gom zXOKL)pT7f7EZ+cC$-&6Odn3d~*$`8&Ja~Vw723vQ@r+O#65m=2KgF+57wa|2r}{K# zefkFi*Sg{Llkd>m-%4@!+62szsRYqa6bc`lOZruZa6*g*dRur72QJFTO%1vr^|zN1 z+Z%xM)p{G>i)N6+3Z9f^jx%RfTN&=V<4l?cJ8<~qVnHEtD#(V0Ldd>cJZ0`?v`@$h z=U)^@yO&jihps7lk+GJ@pKQcUCNjiER{~2YrJ#t}ll$*IfOe{lW6{YvsT*5+nOh6w z(QNU1@N~fj_;7C;u`%5O%6-~MENC?vwoyW=g7cU|KMjfPzB@R%+ip@zdtY;Ezu1d4UZ- zM!==E0(kvxnA4&92D;zB!95?&fNpXR^~7Eg_a4PI$YwN51q=3KzW zzZQ*P!1L-T!XD)=#f18|x z(~iQYf(*?Ck#e@O9m#FC};KcK}`6nSecBQ0Bv(Wntc+_k4rs(zDmOyd#I6K=($Mhd9?s|-Y3 zG$g4*^Rd$0CU|}~43*4k#dBRO;Httw%6Hmp*f*Ak+S7MIzLhMQS(uI%zyFI>?GvGK z`4;4rzX}#d^+VGqOKM_{8(uT|2i$qX)FFR+@(^8yuoD5`krPIyA3-ST03VWBoB^|^ z31Wvx1r(HV14{0$qkOt2YbY8;(Tee()RcimMCLPQ2Kws2B10W5Y8OR%x&!dmD+j$$ zJPvUJZph}ybfURBlT1g_q{3w@ygTX+JGwNH_A5Q&ojweXTXpc7vUEKEQWB}H+Cx3z z=2Ngu0yw7=u;7+uaO98$B>R0~CUra{6<$eY8Mu=m13)dE*5uFI1T@;P9D6=6z(tOp zu+Om%@?~Gcz+{f4#P<3G#3q_E=*cgkgH?@5iz^VTuW6#N*gCNg1RWOCe{Hb3~TZZ zIK)rZEkuyavJ{-vy$>#kPRF9xZ-cz&BRuqEA4gi>6}kT|0`;4cBYH$&v~M&iR{WMwdAUfD3Lsog^Prwn3_RZY|u|GL3bm3y3q^g z-UuLTt8i@jp$*iDE}1P5iq>Xok`v=w;qi{$F#P8@r2qQtxcw0vnmbjpd3KD#f58V$h=9?e` zM;R=1>IQRP{vn>1XbBk}8u+BL7S7Z;1&SOQa_Qzv9H8$&dXmM+rJq`)H~Jl}pVTZC zCcnTx4ox8?9ScFCeFo}Qt%Rq(@#GQHgj*z6;v_#7ryjF`?_a0G{fy{_-R=rVZh9qb zJg#R#O{w@1~shRCk-DHA%GBTZC`qOe{3q&zv97%vmWpWn_VVl__Ks;-FX z-~Ak{6Zhgyn+;^R#sPV+HAYiIQ>nAN)?sbiNxr`n(C`DbNe#pa9oS!kPZS=U+G##1EW?Xrg;tkyQF;;)O-$!0PpGN-c^5Zk1EXm2H`fX_4$?U9K=P->*%reRY>} zb@d|LqoIbfPx>(U3I#eczbIKjX*4h#2s{H(bk?bbS;f&IdI92S=%h3A89U;`CC2z7 z|01HU7LNXYT7X2Z*TKVQA0R@x84i9HAxk>@!9TH*3f(G+F4!30p2=9BH>qK8k9`5@ zkWi-Pn>d;ppHEIrVx$hOVv1xj935TzHJb@YA%C_ zS%PT&qI#zGXc4yGP)IF2j3DmYK6quOLAvFg$jAIH;QcYf;X-;y<-80@a!`R96Gt-m zU4Rts5<>O$?J)jEgB;C0O#MDMz|5&uK$**un9H}GGOT+wV|QPjWOR;D{k^(qyJ;|7 zO^rnJWhKeJzWs1wPA)e3T^*<;& z?|81hKaP{V_pU^UvQml9eV?p`D56MdC?S=K_8??SWM!mjq{TN0pZh+clnNy)rD#Y( zq@7Z~`}^C&!8aP zubzayI|gv9v5Hx@s{}ho{()8HK8)IM1cHx$K-V9ERMBvdy>FC)`k^;qerF``W~kG9 zTO~>Txh&X!_dbYSyud$q$&TDmNn&m$YSH}lMXcVGLNvSo#%fU3mTuL#1y=i zm>hKzS{OW=EbqPzT4htorPCSM@`>RO>%j(|p{E{Xecm=?ZU z%k}dvfX%&dvbFjsJf86gjXD%@)W?N>Fd4>$V`hxw%0&A8VI*o)o&j&kW^B=}f-18D zM#a4y#?3B6)PfEO=iV_%%N2NVnkOE7*vYnT%Gt<3D_oO|ZhD7pA{RXTw)dVEdQK!@j%%Zhw@ajZ=qVEZ%{aB&0`_UXH?y5NVp> zS_oZi1Uc)AY_F^h*`xo0P161YPbYoAW$LZC;HEqMTE85kOf0F?LS17+o3;fgOIL~`#{Xv~Ucr=E49(=#LKg0~l-T-lfozE&l4V-osrYQ~Ob z!5~;24@#HkVa>U}=+ZfX!K!XhePT-a>Q>-2=uZCq6(+%3k3zdu62+mP?B8`$uvKLZ z{q;&o6|=XDzU;L<3H&_9U&ego)$kuhv8jT5{+lxc!v{ zjmgsFN*;p;YY?D6*Vr5Gq zbZi98Ot`aeq6LWGj>YRJ41-EXAp4XZtXNrykEWQB%)`-4_xvm-Yjz}rSMSHD)a|sd zZ5Jx&NnrDmJ5cc1hh16_iGPyqNUU}o)&4LC|E^lf_(2YDR??cPx(|Bn8n4M@TZ}gi z=W9^M0s*!jkC9(Nt7!T~2ddY=5aVqZShJ-Yk?8tiU#$Y^wAf85Osc_7T!6-W+d>j( z1)I`iLf&-)%^ojgjH?eZ(_XztVblL$`}r%l>qa$bNAi)e2xdiWxbt%IBC>oLLqh#S z34ib>-c4EvLdwTL=!qe@+a*lirEi8j)5l=+<^>uaID?Dc$C2R4xX?KAE*{|pzZ4nmQ&K8^dKNXKmkS%bkh zY?8!Z0C8u!#$Yp>>Q{~`Z)U*5fip0Atc9%|ab=9vIv_P?7oN6%jR_8Yp!Zw{Vg_>< zj}wAKdbK%uYZe7}^aw~!x&$wqp2OFWT7IU;R3hLM&&E*&tjV=x#cDQDb&oh$JzR}j z|9g+~N7u6h5*;Y(brzc6G{Mk8K(9JZuWsBA&FNOGnS~@-GBcK0dc&Od?>AyU@9D>B zsjuMEYejNpEDf?|MT2CI8qNRL0nI~&sB%G!m`$5ZU+xN=Zm-Y5M=b;RjU_z!;AT)bb{A- zH+)H$C(kic39XXNuBnof!_RB#&!KPbbWma?$rKe1}f%~o;#IbT`>boH< zDV82#tk9=L7Z$kuLa!qlWK47p`RJuerq9%d-_FOdQy;PO<92X)Gly!dQ)4Bk^s@02 zOlaPzMPyUj7w~%|LWl27h4xA{P>7jApYGKJoz;5uefBwEjy=ZQ@{5phRS6Q;iZYQ0 zB}wpzGCd>}MRmg^@Tay4)gI_)zkN2NVK>F8s>gKPdg4BlV!8?swEu>H<-)|RP>mjs znt)qW4Cua+eU*>S!~trG^Nuq!kq*9O<&57lwqnbujja=ju6LlcO&t18x1hP28nvKh zAgWQwjO>sSrwS*|!2bMk9A3GBJU()S zv~k&*>RMg$D|8qAK6e*Dm=o>kP@}8PNt4zI9^jbfLSLP+qxH|)SUW>OGEYy4o@SGY zQH3Xb&Zz)bA0g(%k()^GE~Ehs7vamz-%PLVK0NgB2flGS%@dQ@MH?p;vEvKAWASVT zHRd)$!%276A~BjV`q0RCUX+Du;wKrS2X8?xHh^sARk0Qi*OCDCDK3mWja5(2;2LK| z^1a)EwH%&ABXZraUR|B&xeT10?g9PB&f;dycRBmdj<)v@nphJ7 z0eu1V$hjrt<+?uLKU&5`_MITovKBqAUt*F54-=Z-14=55SX91(XddmyvM~v|yrmc< z*Bqek&U4A3i0ztk89WHFnb>gQ}qc+yy&8Stcg(|k%j3% zeI&>Podm|TEsLFns-!{7mMX7*2nrr4*y4YLJ>HuR*2nc|>#Kw0kjG2x-*S<^y@OD> zN;A^>ijNIylVHV?zo;-}HM&?`Mm_O~B*le0kIp%dA!%{M(6}5oJktfA-%4~ztSbF; zw*!CQQY0(Fd>NPNrJ(&OvudFK3RIh$usWMKos!#!Ox7n-#wn8x9y|lSU)+i0>3HTw zdMLErOrWNfs-!*WJ_H=H#%;5svGVi->{l;k4U%rLE4P-wlE1fMuHZCUZtH=1Bjr%W zWq3pM&ogc3RO!8o+}<)Wiu1(7Kxy%8(hzb6@(uIAGM&?xM_!`FyHwZ|(+#B-W@J-A zI@(Hw(yb0anxo~&9BE0y?kt zI$2>oz_f9Btd?CG{9UXj$KEP~`J2t?R22gZ?7D;JDTSToo4B)?9-gu8LSLhuWc*br zh^&f$tw}40RNYZ@c5@+r6bd1!aTP1)WJ~xXq13bM0)9C1n)%+XO8WZa@$Rwt(CGP% zJ?0!sH3L`Bn7mqCb#NoGrU`_R)n>JB7K6gO0e05uIV7O-J6m-#gnhI#8bjI~NT}Zv z+$P|Jt+vW^pT`&kdW_)&v%S<)P?OY)&1I+O5KR5O11{iB7*GImaX1xrz8Yj#)J{QR z<#puiv!yV9dMi^~`xy?!+razyYW%;pG5^4Hs>+B=lNAZ zwtJY`=k{dn909U#EP>8l*aS|W|Ki>mD~QhlUGBV^2}Y}rfXv1vIC>x)c8rW5u@;2H zd*8q!7PL3h!)XnJBhNvueRhVE(PkJ&tG8S?_G=j0;4 zMGIYgBQWugJb_Fj(0?UD*B4gfSj{L;+HDYK=~}?j$jvlNISphsR71A19tisRz?{8F zSbjN*JX297!&_oVjw}JY3Q^iMrwEUfTx8Sb@?qs&IkNaxEC~{iBh1qiurD*3d`NaB zISMKCWSkNPCxt`V?_)T)!kq59R)*n%^WkFNdwl2i3ssixCCaPcGRIYX$dnb1Fe%28 z400Wy$B+Kt#28O_*Eoqt#qJ@EhVPl*%Fc9ACBWPAXxy|Uj!4KF!-OXj=^M<3rDdy_ zbs5oUyeOAxR!n0W8iC$TD(aQnnT$THNo*#I@I+bZpb1n`{ZY3SRt7Ngw`!$DakY}|ScpU=Kt z71=$&XuV5>bpLl&ei_D~nC1nGT%MtHR|8B3EqeOGYTEf%mLAd+qH6bc;@y$c_&K-_ zq-Gz7vX`mw%OQeoY`hOsh66w#s{yTUOd%6_Q^|S!0zdc?%HMaaajqwefw4Tt!c$aa&cAPd6=TPlzh9F zhl$C#{BP3h*_80}`0!L0RwN{oa#e!?cDD@m_VFY?e`Uk%C#$Kde0tUMH|J5h@iIC# z#{sXU6jT0w<~KzCghfN9G^xrN{kKduI{F{wF)SMwLdS%A>;abiBc> zy~7J7;7O+#YV%@Q`<71hv~~c$)CR`0(~^n?7|_7GbBM_cD{|Swgua;`&ilD85LQ}j zfrBj*$f<|wNUwOu@G}|(x5-%IaLCIV-(y>&A zZhaijK6YS8#qbJ(J^~P1AV!Zhw(|_Hy~B4Qq3Ds2%d-)8B(pO_dA*%K(QuIkol5~PyYnkqQd1!wy$R~ z4REB;Y~V>G8$aSi#Q^3##4@vwOF__&#js5x1k6>%arzWF;^X7aWpgsvG?x~1jh)LR ztFET^tWBxMM|*l@o(_$2iKczwxmfnx2fuI`mj<6zc%`fxVlouzkq!FPq3;2d|F{6R zB=#}sPru+mb{s6kL^RBPhytw=Xcno$rpj}Sgx_r8ou z3(+~M4>v6C=8r!d;9D3(@Z2ZJlH;o_$v|X0KeUs}A}p_BeoXIVOegd)rct{gZr^Hp z@Jbyh&5WQ!b}_Vb>@MDs&;e#KH@*eC&^gVAz{p0O9y8Ua&x?!jR>Ukkw8e(3+)*WKz|1YV!O!6t-l-=w_~uY-vW~k}5DT@(`R|tVHu) zD$^I7&!H0V1@GI*Fd5mp%!y65Y^|IjI2cP~#al!EK%fwD4a$UP7GH2;&nMJf$58V= zf4V=8M?JJe$jHifgzGF$ZpV>jsy z`_RmfJdl)RG}O%L)oyvZROuo^e~7Y92dkN5ZQkVfVM7`wr$TR;>60n5mhj?&`a$4E zGun-BBBg7-u%E}-JxOWSkX#WMIHIwKc2CV>^*tu2aj=@#75|J+(HAj zxtyiaFVJ3o3XV&j#T$nv)8Qptj{Ua^QI2?u_BHmbylDgTa^f!XL|_%Il1U*CNHu%v zbp<2&<0Ae$y@i&Z5yCC`%joTP3nD5lL6XYyae(u@CR~=KxeLZYhRe?U$3(I}M^eD2 zX-P1zR$?S|3sA^$@?H zpce$RlQ2Xv72NW7Vv1q|T)ET2>7x=5IKBqLV4y zSp;b{3#s$HJa}buj%n}8geS+N=}QqgD!4?5M6arb;1#{F*FTW4Hi(9)H)UvPd<=V7 zUJcw`t1wlr5Z0D=g6vE#3paiS<6r%U72o$m@pg6ay^zbx;kpamd*s1&cO08|<_}wE zc@keJbV8YV5^iVY18rzd^Kt6Y zY)Td9bukgo_VRab6Q>unA97Z$4DmL&g};8?#lbm3AmhqldqpjKy1?yKOoPo|ToN(L9xm3w!qfAJ$re9a z>kl;J_VX(K@KKmyKan*KaR$k9F|uX59PQrYO;_CV$FHp?;8&O)H8*vq>8*8e!bj+aaJMc9SM~6`$PQoIncw~4rLu1$#UaASjzEG_8mD7q23$tz_t-=o%IG~ zXSlNu?iXW;_-?Yna~}CI=?ap4TX0$1bC~0L8gbozCgNfzqiFhypBAByZ{!@QP~cG- zYAQ(|?y{uk_8f)#u#HUFdy98s=qpNB8`6TiK-D>qy64ALw$#K9c{1tDkosEk@yug}Z^qG~8w_HfNiVghlySRIEsM`!ht>v>7|E zFD6})U6^Bf2-}sm!Lhy$tSF~!qGKR_|HjQT`v%!awcC)UE=z*`*^?(bfHVe8A+NlQ z=-*vm@RcR^^B3|oQo0lW+rWb>M{l4}m@X8HNfFO2No*yrgVmT<$i^D2qH+5*h;-F= z5Oen=d4mh+fA_>N{*ETm8*9W{Gaae%{umq`--Z)Ll`vk<2?HA6;7N}ebl^!C3Avtu z(E>qa^}kWJ?RGLzcJqLhsuNlM`j0sM!3TU~+78Vd_mOPdlW3sy8`#Z5n9_40af#a*usI}2v)cLKq87t+ z6>J7BGeeuZ?a1L7Ui8I{iS)XZG8Kq7Ciu>dseU^$QYU$em@@ex8iN{?AY+A_M*A z`I7D%f+RGm7~-z`vi2Pk(5VpwZ|#ThY?~c@I{J=9?)#i8-oqrWbY)fWJ2fdQXGep- zPr*mQ+vqdbDKw#D18Gs4O!N5vp!>Elz53T1^tXTI8Q(CW^?^%K+1-|?O6J{sJ9YsD zrwX&{BL`vbm4kGXkq@ztyozGiMevJB7la$1;Frj3W=*5#k)hiXR8C8V>=Mqxo=aR` z=Xy83kXI(AIt_TtB^1v6mVvO`GmQ52HmDs`$79uYZ0Al{;wV@ME~_TcvCuGj_M<(l z%A8J5)YuZ4+SMc>M3Tw|*F)f#A3AMDRJ{8NN>pbPwSynAlxL0mu8C36@h}YIc!ZAS zj;QH*lr9xhpcz>*)TLCL+;LZ+joTja&y{CGa&IZ@nnE!z{VJHW=7FeR2rRstgN14$ z?BtGF^wH@vSZZYm{d?lDgVWl)eb>PcO+DP-aE6Wh5yNFMf!2-OfU6fde$Pz}vZS^a z1bu$u%k9P>1_*Im%=oZcg3k0bB^&TX)sAL)`Zs72-Dvk6E;Jhj>H`y*wX^hLx9AjNbM++43mFHc>w0kO zzv$!UB5R|#cQu?9>s6#fS6r~=7RSYEEko_f z5QrU7#+6Tbc>mET_s4$6!>@RJn~uZqLb(QO{-stHtV@S&_U%|t%yG`VJCJ)chV=hr z;kd#X@ORI}W|1BAudArc)>t*@xFy38$NIb6YtC`Bq^0_G55SIF5F@ z8Ix<0z?^zzNk3f7 zt`P43O|}kg{>@ym`3o!67F(CE2%~hz0_-_7m)69{;a;8t30Qp(^^e#w!Y-$k}0&H__H#i>h$4i!A&K&O*Ds2bxzYvC|&=y(&PE9epx+ne0n{2G=wT*d2g zm3VD@DKmY00p49@%o;{~;?L{;51ch6aL*)hy3QgF(!Lg9>zB7oS@t|A$*260zBq=M z@1WUj$1s=u!qP2E=oW=()KFFsy!WMI`bJF}bZsViDH%?mFMh!*`qWVsXOKvx!&S-i zGHvRqB@Es=6R7ltAMkKYgLaReV-2EH@S=hgvG?OLXFpysJ}&N@wjoU8+ACmo{89Fa zQ3S5qS&XGS3USvVci7WAk(|1ygI-IvF;C8}B-3ibiSSP$+LhFg#S^&P*VQyQyqoJ< zeT=1Fl(yrOEL%u6*h0%|fK1WeM(4gMfGmsI)J^yUd#k{TzQ4E@=NuYm7GIDe$JI>8 z1dgfIb6b<}&6^on@0YyzygsgjcLSc?&&QkZUbE}@#qh2{kA`fVLT2>+;6L~_7rr;k z(xnHu?q|CyE}dGy8o8L!&qXGL-CqmIQ~xm;m4ETMWe(5lfDqlvv4T&&t^)D9nV9+| z8z1esi*@$`=!M!!6c^B-Str!Vn_D{Mg~WbTKIuh8&JUnty*d;`OG5tV*^pLw9NtV4 zCcpiour|k;|M+hLyjAJs)?aB7UM~ud=EM`8wnyVFt~mX@o#OP$B}7?U zhO%cSlM3Y%?5TIoj1*Z0(uqgur(DiM&X$I^yu)fgG)Gw!v)E4Dj3GMXT+m z5Z7Jb;9E}|cFo=nYor-C_skQY8t9@`$SZCyCxpo~#g+xTQMG0f&D^jXZ?s7fyUADa zZrlvi$rU9|Yo9^LG&zpfewf~N{g3mcfJnVsMlxq?K`FS+jCLr}UBq^%2R{ageX||;Qc_Lst>}+*Wr)s?2C&?`^YM~!si9#!!Ks?Y*+I7 zJjYd(y1)kSGNo((_MoDc8P1;24<7^ca;K_=kb zI|56#7?XY(G5F_9thBw}?Or|~LL z&C)`ACQ^ZBuWrJgSj-iGBkzN3oO z9ymgRxSna+wjtI%a0ab&F9p%V!gRXJD-76{hdnx?^hSy`mDRor0a@BqPC$aJr@h!- zm`L=j6iBJadQzcVf=$jZ@ZrCB6sYzf-)A@xHT5Or-JTr$^2(6byqriDDCFS5EkbCs zel|!m@$6-brR1b&HhSKj%9vm8WeD#sC}lM8wsT%tAg#n{o)wi>xXkwPFR#&K^#a;4 z%7dG48?f$h5kAz3r2S3Pz`QmU0+h_5bL}BGnsW>V27&I~d;}9a>}lx+agq=o${Gir z1YYB7Ff|ZBzkN%f*0_sR%vU1^CL2+4AtSQ%RT4^njAm3Kx3NRy4BlSdi5Y#*;ebvF zZVnhh!9o?>EnbMtYq{rc;!Z#8n?_DmIkQQMyBWQLU+`Q}fZlwoi6!6laQ<&a^4dO` zv9ss=UPm?Rz8rwxwupYnjv!7xv%ti4Xm;Z% zh&VU{LuaC?*>w8n?@v}MN`TD0EdicyT<~y5G76?tV%nrC)-ki6FXO$0Uim9QhZN(; z7vG0i|6&dFKli1l1|x`&)N1mHtR@vk~N zXdi-ECjIbBFOIcOyp5MeXHm7SCJ^Y?487BL&`TZhtX0rjfO$)))~Q(}2Oi+c$`>%Q za|eA_(!lXvW>G86i|^g$6hb6~glQ^YD0q8%cSZzz9n;VpsAlsERX2!`l}@G*t`L z=byy3M^)gP(1Y35!Zheg8P;vmBTxPMaBtBCC?bNmxa=F~W!Ay3;`8wDjw2jYdI@We zZNej0O=y{}8< z)UE|z{}5zpUlK1 zg3S2bhgG(bJl{#mWK*UvGLF(D+wHCOs(%Y;1h@!r{WUJP+&h>6d9qPFYlWGl?ov5pdS(zqtkb|CZ7tcSuS`nLyOH^R z4$PCq51EmgmoQuW3S%>%MpH+v$VTa6^ z9gM|OXryTsdppB`Jm`6Z3RjmvmB?~@l;{D~k=vjxs{z_ReMbM5CXQ1eMEia;@$=TJ z6Qgzy;%8QXvLf8M;revaU)7HatSB`<{)Qd+n}G@Ey&>%m$NC!>VPBL>Ljsq9`J$VE z^_LxJmgYOqdc7X>Q{{>NX&?GUd=M5(-DM_}?;wfaKEWk}r|gU!=IqO9Wvu!#YY13< ziS4ZOBW+n5Xxk1E=5fMCs1X?CZ(P*M?6Nl^mt!rcZS*91cGDC(UMI$}3e@PId=tFb z!m&FS3evr4c0?s}59wizxXj~fnD8KhMlCsmqUV>BJhx@^-X&Ai`o0xJLT*C!?iJ+G zKoBJUcN7}lOd@W%f}l26oPLVZ#4WS;5oRvO9@}b0Oyu3^(APBhb_$vH!>O2~uL*^h z?4jDdo?r9v09=V(3KGkofRKO{IcO<{Cc2YgE0<-l+llooGL~P9sc=a_4 z>%M%d%xpPXC@u>{f1Nt`-Jka?LunM8O;1%xQtA_q))#DjfYzPKW)fjyQjSkHWvy=)r(- zc6R&>T%Q_&O=qt&%kJ)HYmIz~kC!MtDxMBE!uzl$Y%$GCUQGNBZ=$U?M5y!(VOpWU zveyjPk>v)?XyP;q+wH~B?3z3kKa>xpye%+=mABq1qff4ec~Q#;&(T7%9GIRr@G-lB zH*Qq{QDyOXZ(B8H+-1l+6DhWql|d8fxwyBb48Jq6l-Kx&efI;ILl5-G3a>mA>X%_= zPw}AZR1*MgAHdYGDvZ^OWr|~Zm~H2OTCef;;gDY;%wJankQfbxOT$K_Bj1`>b$y1} zi=IREQW;Y2=R}V$o(#)0`K;rdp}$F z$r@p77TZF5zYDn;5kwzvyvyvmL2wG^QJm47jr$CQAvWC|Uqx&rU->$0;0$TD_IeY? zY`ujmA3VVg^Y1dJ4;{lcuWW#jpKN586jAei!!}%7L|%sVF^{UY!NT|x@Hj1iXuk=g z*{d8G$ygy2wJ(9m|EYkgPzxq!m%w22Cg$hjTBc^L0hwU4n&~!JK`VYuq>sQHKfkyD z&#rG}D#x1ese}xDQoobz6AYt0HAz?#?uCEmY=nnfuS3|uA{gtGp*6Y{tc!FXrWiV* zq_QPZEWN~m?^LPB32mA-Rhb-`Sc`$LOz}>?GA_S1k^d@j7W%rXz>)+za{K2TTysu^ z{Q9|tIE_a`|NRvheryhn;qnnlw}oid14pWJ`5jNne<35jtq_$tR;Z1O5pC)*qaV7D zlX*K;>DruY*n80#ebti5e^L^-O8yoIR(122CO3m^439}!&m-T4r;-gDRB7V3R^0Vd zi=O1TVZvMAq4+TyqWz0wOg2Z5-t;S2|KbgLl5JKAzooE_t0)KQeI9Ah4 z+TkFJvQzhS-TCF@p}P{to_+=&9<;O0-HU0WRw}PiC5lw%EFumf(}|>jFMX&#ge%0~ zK!th~{@tL-`NM-4Uj7A&R;|Q%zlC&VzXu8qe&qjAZoxDwOO%M#f)_SsRQDa%F)lLX zt0yYcwR)Y_V*3%rCbsccK$2atw5CNJ$P zQu}T++s*mWh0~!sR0nppZKl0y!iU{Qrvk22%R66s_YX(5P&#e=%JL8Ry9) zFs+x-*KUV82@?|e(-1aJbR{QU#p!aNlZ>>78ePY+D<151AlHnFv7uQ3GX@37QFwt* z6K7MejJnDh+WT4FvP3HDp-)me-?2ZX*C0*WgROFdtXlUel=$pOJzW1WX`fu_y%lb> zhvO!>9iBv{y1LQQ(vNT_*A-P)UT4mE3X|&WlW;O~EiE`(%0H+-mt6a7PLrl?q}pT? ztLAhb59!&$+ObW{yl>Cgvpsrr?;>4#PJcFxKkC7yKlcIOO%cm8*HL-KgPc zskmr2zW%uk&6*})K=miAs~_RrTKEw~ha+LWxEn0d{KI!Rb$}kcF3EX!eW2Yeg!im! z@bMB`=IfJAyy*U(=eghoqJ}DrM9+fsZ&$D~Ly3GkVa6W+u1MY&+tb8SU3&GZFcET) zrMK;4n7Q2CwzZ@h`bQ_C<_A+^$1&~alKrsLM1y(uNDORaV<@kDF5FI&BC#*ml5Cd& zn6ds3MzlP{&C6GjrvbLil+*_Jz22BKM?3-3HMY?6UV$BY(9HaE+`&xEI08+lIbZLZ z2x@CdF&?MgJJhH0R=%o6Ut$c5Yq>Jfx4P$ZWwn^6AGbT;eNDC6Hb z4fC((pt5TwzAd~DaQO^QZ-2>*w@Z=cE;-tlTF$S}(3uyQAJ+r5u@3aqo;E0*oR7t^!sN8^4JiEF!n!{Ah|Q;G(6%Tya=`iq zww?kaaU~SySnj0~Y4`an+M*awvodIOx`V6H9IUrbB+Z`hxbF50I%>F>RDH@+RsqErw{WcOjz57x8QEBxc#r4KO^P zL_c>s@e{W`!T4Nd_-JiQ8U~)R@{2}b^9%!8XQ|5USo(|kHYtQYoS(v&f4;=8IPegw zJ$F=D3l}pFRs__Tr{FeWdiJx(iwU7sR9JvDx z_9u`ru~1T`@)~|~xz;4-Mtt4R;|(A5r4d=Wa8oy$QNC*id*0uL4b5$=_+S=2$wQ@Q(B;Y zu`e_A!2qtfd=+2TIB}W(RlKWB@x15p3<>%C4W8XihiMx=fUMUWe0rdj7cr?Cr)*o# zI5%lS#Tq_i(QQeHPcT{WEt}2h;BJbP6(b^%bBWgx4`xVDl$16(;6d@PutWSme$@U% zBBbMnhSpJ3WvMU*M_$ArPBJshyp4I&meCm|7U(q17P{&*$TCKPNnW#r`Rw0|ix_>b zAM$}+Vz7?tP5qDE+s(aC?|<{6iwTD8Jb_yS?OA(|#Y9wa500hYWXC^b08gU|R=gFj zTG6NuZ&%Ntol^GnbAU{PA!rif28J(Ut49Xi*b) zzrxK3q8eAs+rD`!88wZE<^L=Rf5tC7+{JZ|4}HX8-6inI_6l<}&51k@Si z>SL~94VRN$jx}Iovj%hLdqL%;crF`rg+1tTlXqyQ7g7HwPUSC+u_B{!@O+XZNqW_d zb%D!y@*Ll3?w%S}wU}c|drFW8A#bpsb!3(dNRlnbWJuJD{;K@dTZrDj2rx4gf(xl@ zv78Abj>Cz>OXLdt&_Bjy4;P`I=WgclkP9QaPn!(?k|s@y#ptIu!o>fl0NuRCAC^_9 z(YG9f_mM_3dYnFru8JCvd;1V*-kZkxjZttaNfK_%jHE9GH*(*60c^D_WZk3h@Fty1 z2h}}NsBU417S&EnnS?Dpa8nwdjaI>hTUFR|`ny%4w>Ec9b|Xj51~Mithspk#d(cdA zG3I`$g}vh0@XWj(Bflt6l_)>R4j}Zr;eJ-m&j6S}5pwRE9(mkpkCnsHn9sanYZ5*| zWAScw;7=lHl@v#zuhZxZpGoxZiF07eUI7E<2`)_XgM$Agh?Y$$*6-o^*^6wc_v2C6 z^0Nm2j-AKQbQvOFbQLaMxCqK~T;Qy+8o3k{M`YuE@k=c9=*2hh+1*znU|E?M@!$R* zzLY(L=Hs7G#B>(zyA=!mZUFh}TSWojFp!@5upjCJd-sbi(LZNG@ z_bWki;KFhe3k~q|*GkG{x`QvLzm6v!0D<{k*7lRR?2y_7nyIk_$yINn&=E;RzdnVu zIf7VqvKRlkOkj_Nnvy3q4)m-_H%6HYlk$01bo&**D!Sz#?vnY=Zhjd9(k@YWCFKGX zT$Q0Gn|x?aHAAQRmSOFs!(iMdOqdaCyp%A&8*NTw2HN5nLrrH~WOR!i^(cnEp3U4n z4YxS{Wewa)MZBMoj>2^u&ye$;HvT>fRxu-(OOnAyP?9V=MR50>Hy{q7aCN;FtnG5c zF|R$eNg)ot58JXmy7CzOXcx`QZf2xaKEa)m=dki%45r5ql zd5dEqz(fG{%5gc}>Iif}E+SNS_FZMh5GyQyvtP`wgSqiv>C4$EbE~j^h zftoL?S-GQ8z<|OT8J}}>S*S#%~I;LS-j5F%IRUn(znWNuf6~-kZj@%91gO+9D zH0Y=(^-^j>KPe58D|3;{Or+qJr+TEVN}Xhs+-HBCn2HVkPf^83kpDF`7=Ek^XE)Gw zWGKs)SRTE}WU7YKche_>VAwJ)ceR2yc`nzT3Ok7&pRa?u(I{_+eG0$w@&s}ZI6hTE zF20P2qQo~89^2+X@pwGbvO^C?qMNb&^(fq%b_+hmSdr$}-2e591}@Wepq8zh$kp$X zjMV|PANXVfM$s%xl=li?wV&v%Pf!KldEshxJ4A+aNl_Y>w)%k2$UBVF&u6H z!wkN%cg(`^O?(Pu@%eauo+c^0phSnItMOEb6*)OE5qISqkm=NzY&LI!mTd>n$?h&{ zy=iBA4XP@CCd~oeumtAmN^4f2P>||BoDZ)KL~|LnDO7|NW2zcVa6w2o>!z1Ni|m#W zIUj41xnd`^TmA;7WVsT)z6`0^wuD$4YLOEjYd}Qx89dHl!SJ;nl@80rV{ZGXbZi$M zS-6y%^(qqqSwq@Cxfj!_vY50d0d)I;T_iJIjGm1NBD%7t@Z96~s4>%sci8?c-v2wm zmK}?MdR`bb>;1w@n~NFo-UuL-R(SC0Fq*rrC1Wwkj8dxz(aM}enle6PPfIvy=^MwF zLo@L5qzZUh9aeF#t{dKm*kD$@8tLOYTT-$IXvTKVw?FTTx9|OiSIJhyW4$m~aJqiu z#A1$Vxq|d093_5N&$H9GTqj&EX8zbY&^eckXxBw~cEZC9e9v`A+g<1L=MRU_$zpP( zHL01&2|zftwuTA4;!h8c|HJHM=IncsWGbk&i~XRzlFaleW1j1&lc`%~k-#}6SSULQ zqXNg!XRZ-$ZMcUi`O`SfU5N(1oJl0Oe&95hKsY8j01k!Wq(1c;z8jAt58G#v0v$)* z)A|AcO?~F>Omn=ayq3%^v?0f`-8e?HF>Q8AM*V1p*)z;z{WC1c-`r5VEnfr=>W)F6 zX+Opfd-8mWBG_r0mJp+t>)^wuj2+{kN7#`=9_zeW(!?Q8Od%YNWx?l+;Xo%36T^BJwU@-{JoJA+8$I-H~ zdNjFWf!?P%=929V(7E2sgnYV*e`V|N%eXBKF8zQeLisGuX#q}JR>?F9$k5u4w>TX| zk0{3+w$7y=U|aBX(x%}}VkIAf;FqIREp!CpFE_&fC^`?np8hY6w`po>s6<=ZMt$yi zH?$Bcl{7>Yg+gYD_Lc?>MM9F5mDJ~+H$r42qm;hcB2lSS%KF{kpYZ9?=X38l=knlTovpG(b&_`(Hyem&^k3i0yU9^G!99G14vMqTtXxQUQMnq;d zy_91>CBw@w;o2?Wt!ThyuX1tUGmfhf6OI2#%8;QhGottB2yS{_%akt8fX9W~iB98d zc$4=8*asKUh{Jhn=a%7`PaJ!G`Wg0z(Mr1dvJ0G`;gG1JgW)@3>2T>x`byV`jgS3= z*L9W1nzC0`KeeCZ-*`WIB;1tdjHuDZMK-uNA`V9%7VE3S|na{K*u(olJgN zErwbD!j*59P=nzFl>3ADbKoU*e)$YWn?8V_{auu7<8Ji#mf(u8$7od}NMemoz*8M1 z5+h{`FQdP)so63#{5>JJ1|LGj8c8xYCWd~wv7gJ0C6iObn@C~hIUettH#xGb4~9bo z$c5|QS&Pta#$fq)+TF(;8G!2H3ez zhy)pKASG7suq1&(s`Ldo(_IIv?g^1|k5kz1m$=^Rmf;%rMy_wJ+=LDOQ^>p7n&{Ns z3F^ki>=pVH9QgL+(E(|?^|vpv(>?;qS{lS#it9CgG$JvN{==r&CB$hYm|R#SK(mBi zVXlKLd9iL0n_l=EmQ{rklkg+V_OnZAuK8R3^I58N^NKTQ^``^t6aq-|#7*Re@(0$Y zW(!%N_z9M#w_*NLcediP9Oa!jgsTIF@KyRG5bYS`4F}JsevV)GtAZ4%*8M3Eo>#*^ zcfFhA8x*j|0t`UP)QM>hdkX0mvLt-z}ic;zimj zg5j`RA{&;G1>vQ8>CMVsaMa*9W9h|=l9@hzF)f(vx8eF8dY(A0piTk~OeF0ic_@{< z4rUq0Vnpg)9Iac!%=i>cd(B^gdG=K}BXAzBs2zd$)75w=e;$924gBEGwC%+PVy?;MsMI%MK<+J=GrA0XUw_5BS=wav6m=3G)MLFuOPqKq9R!n!RjAUe zO67j#vE_0iI6H0<9<}gcdMzF4CQq)TEm(`@lH1_sqY#|v7K(vmL+m0c33@C0pgV^u@TQQ=3@b8C_!|uEKE(P&moclYI1cE}6O6U(Iyz7g z%GSLyr-Ql9^x-l|P|2Q&PFwgeCVqrH`7s}^y|tmCA-^Gbc|Xja_!)V)f>h*tP`R~t zVN9(8Q+HcXvcjMJP&A*rd8{Gc-$LQ!aV`Vuv6^o@uEAc)cB7fXH+YK!o6x7qpX&C$ z8aqmcAj8fWgE}| z;c2*9?=<^$?>(kn@hWaH)ur1OM1tJCK=2lGhiF4FuwYU^;y**YJZMY9V|Z}FtBGlP zeT4Du{syOo)cJbyvQ%|J1YUnz$O?U&K~L?E#Ahd6h@xd96yM3fl4)AR`=Sdtyp<+L z12$9Mt-DOQ;A-N%Y6kuD+LD-7O(0WEkD=f_ak6jCJa+mH0g7%I2d?1hN3$Y$byv-L`;bxdhieIzFYvI zU&ipXFpi$Ns6vz+UvpWY8b(yPoB1#G0rNq#5szOor?qPHaE68e`7kw~k(<-Yn!0Gx zPa&@$In$5du_y(Z!hV#))imeXB@A3Ul^*Bzm@bz&&ZbEsblIuF=i^JD<9I)4rG7`% z{tDdGJ_$V+N>R!0bI9X>rL=y@1#U(*ViNqg4(gsau)BaF`-97%c*@{p*$?dcpr>qM zEBCx|2_TsPfu!vKWpcdlXMY@)HMa_Tg)T zetcp*0B-lp=zY^D5`K^mj*>p;A}Gi_7g@!2>iq@3{spu)%?}#4P9+05id5(RRQhvd z3^x1@!oDCruDfgsJs*_FF8!C7x8i zpiWiMaG%H$Qx8cJwk&`>c=$OSJKqTBjxWTV-)r#fdX7_kTY+?xIng;@-2O?{otzqZ zz&NJrP=EV))UmjO4-{`OtL4Yx(Z#3uB$!9;|NaGP*DKNZ#Y9|UEX)2k{2j*<_rkg* zwGi^Gl+{hX3(jjqXu?S`8e8f_%F{e?nSR8atqvi*e77 zLyJliE^MXPxb-2v8MY&9YiD8l-zntiqoqVkzXX52e}V1MdN`Y7@T7*8G09r#%&ae- zn6XHldatl$Thfk$=j+EPI^zPgyv!i+t|s(>ogI`+qv$5HlDa9SLq@hd8yvlhYFB@R zv!55TU!7ONx=UVInSH3vv#+23ytJFq`=rS8>S+L{08NrUEK8?N{)IO0xU2$Sg6>$Q z0^+!y%3M8$zC;%<=48UnF%_DUW&_V|W-wt*`gBERAXP5h%w9kC5$DL=Mwf-b?Eb^A z;bqiB`i`AVmzsB@p|cv@W>*KxVrp1o6@q&>uWjQWXQ+&sP8N&51;gKMydQd+)Nw5X zBT5u<%T?&}iS_7KrcY-+9b^VRXMpsYue{INHLQ*PVenOvBKCZ5vg%nPeeXUS#D)~f zC9;lKN|xh~x_1oUWg3|>cG{IU*X$HvjmFbc*$euXzp-I%C0j>m+TF!hyMr0RzWDXFglPwun$>hEOSpGw1> z>9fGTOoZUSE4a(2f`^Ow?9jg`cC%MIwq|Z5`;Wvhm)s>t^zu#^GpWF_>E2i@K7-i( z{s^iT18~~KnN$zuutBfyGbL|AQC3TyR-D(xLwXWKiMNcLqv~X4x;*nvU;}OB3y}{8 zuHpD;JND;NEjpX!@{v_X;Hbg@6z{$QB+-hjnp?+sicQ8AuD4k5Y!`E7f-6l=kOF~{ zH@H^lFD`33!#QI@amAfiF!Xv6xzaj;%ov;jpVjx!v?dXH_}@Wb+eL})s2VvoU6iKZ zw}+{Bw4m=U=UaKNLHBnZXRZ3*F@IGQP-Dg;RH@wz_a9&5=JO38CCkSH9IxngV?G9t zXfv-q9mC!qnUMOn6g^bs7=frraKEvWb!w4;it|@^&nq?PmoxRinXvGT$}Lnq_>D=c zpH5jF13KOQIn=qmgqw2Sctcl(!Xz1JFE|Yr8=f)WyN;s#atC@^@dF-JZiLqL4BR#> zV_T+MV(?~5>RzV?h0-n@GlRv>JqaMJBL=Y-L#WztF8EFprWGz$bi7cSNd1Y!e?AnN zTv~8ywgM>-T0^AEPebEPQ&|6B90Z+?!Qm&;jPghoP85tMzn{ub8z8u$psKFcP#o5d ztI?PrtKsF$U)H|j3*h@CMG~;A7%vGvf_CRfx_FHyz3+1vx1ZF)_hJL=)NU1epyECR z-~SBLBJ}AKlP>JGzYWz-65)&MTdb-cVYRNMpr7VfINKmbCw;wufp-49vL!n7L;5vn&yil0swUyKGwdaZEe?7F_0=)7ZC&Y;D_ZuHyZNDgP$W1xx`yn3L9DfAjjtXBYUKYywz za}9RBc#R{K_wnX4MG!8UP37a`NtcHhS-$Q!E|y*Ztv~0HFH&lB|5Hu+e$)q5ldhn_ z?sg_m+L$~}7scfoq0}fAVfOwW@V(K79R_XeYb{Y4bk>P}((A;}pZ3AZ?$vaYs|8JF zIp5f)e*9Us4lSaiAn3j=%EdRL-MKd~;t)zFM^%8*)?A3Yt3#^4$3gG{PpW=R2Hh7< zWb8Lh!Yzk1iOtId0MOo+)yN&HoZoMCu>#8D&RU}M%Glu+|N0>h zK=TGq5->WE9JIG0l}~cuUyLyQZ0$tq=c-aEF+v&|SCeB7P;7g}r7iGmrnG_cN#Ib-b2 z&1;Xre_(v-!7v{^h|(>CkR~to$N_6o9^%C9{Vp+z z9J^<$#cWb!5`j&V>$vaT6$M0UA8R`nRtTLqhtZ*4(6%F}S6 zw;lX()WXRZKH=*n9NVng4c4mN{=XlJn9+CE|8BkKGMZL6XPyDI5fdZvrux)4WhL%4 z+Cpq!E~Al~-l5&+8~n)e%e<+X%H+Z6OYCrOK2&Uy!@d{hV0-Zn>y&H)J8>cL+hT!5 z2c_s62abUf8A$GD%%D0y3c#T!5bmhQQ$hI;OjJTJM7&xLJ{#4cw|6-m+G|Ku4u+Ad znHyQFcXz>ZycsW;{DhbX8id~4kMV~JU{0-o7^yXb`kWU6N zj(6duQv%Jji-kuLG{(b$$aUG$1qmV~!JW%wTkFuc2cne8_=y_!F4QNn7_WVa=iL&I zB*%(7q4nV))SL~2f=5NnHUU+7c$9PNyfL9m|C&H}zXd(xn2F^QlOU|=4r(iyki3Vo*DjUL^J;<}p0g?> zjiyVhl$h`y5jwHnfm}YmmCmwU&E@j;P$TY{H6u-mevMVn z&nLAKs ztvglEok@4@RUt7Z5AjFC6Vz0%!@xt?P_=Li`*YD^;w#b0T)jUXkL+s#`$Q}3dtAf@ z*qvf7d`rgbXR_&WAyFnF5< zf{{4qC_V)1-))B4t~SP6rI>y1kwr^?MKhLXWKo^lalCR6BQ@-FnjC2df>*9Giw33X zacd=$BfI@c9L};0qd~9UVo=I%NGc76dhcg=&9$4k z?8*%I@1_Z<@3drpD=47H^*|`*7&v986o~BFVZ2>-3KuLRG@y^$yWE>c zF1*zPm(gZMFy5GawS0{p_m?w&6CKl@^ETta}$bvmJsE=Z!xYT7B+9&$90w6 z!TII|Y*{Nr|C(vig{SmL=zmGXc#Q(u3B5tFTa%&K>OJaro6*0&w1~Wnz4h{AYINR( zN?hk2jdX$rdU^23)eVCv#hQ^|pL~4x$PLfD+=xeZ-oW*|t7vy;Cp|3VNEMZL<0tQM zG|Te_J-Zy3FB1zQ;-2*D>mOJj_6WRC* zJ9-ojFPMhM{O`dj(`T^RZ8cg~H^Z?WZ94s)ACp-TB?)%x~aIoNf^8R<*0<{c`TcJ3lA6AQtxwzP^U9U2MM@$C!RX+ZsUv?8G6E)q1$UO^KO)QP{00ZWbgNL zST5dy2UbVnTc=H!x1|w_4{NgHH*ILvuski9dY{euA&3*xYN11X6iiAUUOTbzGX$9W z5}hgXaH#AsciYt>X=2j2LunFGjFqFua(l4)p&-qv=*EorRaDA3m_4(R`yVOpf_1hMIFGaTzE_|YSFXUVT18lyM97rJHOzmrD^b~&bEE81A?BIZH09VFTJrV~Ju@Y`Eixhd}0rn3$lHa4=Q%8T*-iUvkp#|F;t9Kr3bT<6t%4dw(n zGP6&7M630xfbT{y>nFGWoH384SG~dqXA?p1+ct9SZyn~$%&q+sd>SQsjfn}7g9lw) zS9YZk>0Zh4J+A6v^;0X7zG@cway#nvHWGN^`!M))@o|#936^O1v$vcjh~fN5(lmJl zY?M5#RZ1B6I;ohc-uf5jMq1;5*cg1ak)W@&+VINoS~hDyiBt!fl19_NP#5_P&pTve zk8c*5ig)v`=E-CGmpG_i{{au#{pIdevuNq9IGnOhoEW|fhlelD;sW&(%&ph*jD8*; zPeL^m<>`|hkzwTP*D>D9S7KD@uo7Kw{ReNU6{F@cVY;wk1O)$8@&u;6~ z${Xq1gf+&-)U)Ca;}c)Yc-=h1NE~cN0SQl-%j0HKof{aTFU4hkj^oT*!L(mYj!u|& z9#i7t@z7g$CjUhiyKId)9p|#8>o1+e==U*nL`8yHal43;wR_2`v5Tk{+0A;HiBJa@ zZ+as88e`KK#=^=9_JnON#+;y7m^lXmvTwlaCEwUbU$$9GKVsQ;l2UZ*;bPpnDUxj3 zoysb^uI2q(6h}W_Q=_{MJcnu0uC!+)j*gFAf!aAD)WYEIbiy@*eW+3-DvuH*k>|W6$1PLRLRw=sgofa(+R0UB^%(=Khxj8^Hm8 zlxOpH{YYS!d=4b%FZ#kR_`weMXX6~_OFZ7JEPCg;Gcn<2qbKj;NcvY<{P8#tdon*l z-r0|+_i_%$OXIc%<(#vlRSTliIIbd%1;y<15Yqn@Ps~HstY3#%?>mn@t)(z2d{GWW%1T^j-tJZ z7`^#Ph&=K50-IALN$&TL@Y&UsoVwcpJT7w)x_%##m2bg2`;h4kyUXz#6zMO~UyvNS ziPn9yVvpu25HqJlxE@vn*{3I>MT!)i6Dh*ofU4L{dK`1*RtmiFr7&MolRn9v%tnpI z!(-t#kpCVJJWESlKX?mma>Yrft~-eJbwZ?8E3A@U%_vVwqWhmL!||16AX0e**G-r} z7Us**x2h>5Nj?qqa3+(JRSz>b{-*J<=_FU|5YuAhMmu~CLxawEos~fc#OzIFp9$*H zwp|sVJUWG}oMJ(v)u+<#zNgS|+zrlusKIuHbSkm+H7<_dO+Jr)1^%JcRLkN7JDr^j zW4SA+OoKR^bB^QqZ>+<$EeYT)-Hs((FY7$lr7Ej`2!WFyvO)`Q!$yfjv>umdWpwoE zEKgJVpNuZ8$TJ}2_7{NMSU^wjnUD2x4E0pbhMpZpxGjGVeq8PYjW3eeHHW156IM#$ ztyB4opWtPTlc{IV)c$2gRbwIPZV%otY2y5Q24u2LI-M{#6v|Vxndg3mpfo{~&zJcJ zbCktNB+Mrrn?v}vOS>5hMKwlkVi+x6yo0sQm`fJh^s{H^J?5p*B|MzF1pCsh*@{KO z?B!~X!*gZ=4T|-p3uN_CQ`m*B`~94~C<}DzveVWdR~!Y^7beWhnGDQdd>UUq%BGhe zWYU()LAb+t9KBawsS9$Q#jz8b_`5bV;+h?0P@|W}Ugh}h%L4W?Dze|$nKMSXjKC2l z*Ot#r7}KGm3v}q;o&sE1AOm*O3Yco?NO;)5IWmuxGX2?EV7QqNx>_n^hto3hAwq=s z)=Q#$D`ijGkAq3fY4{$#oH%$X(C2I``*iC@_&ZI2B%c_<)7~>Ma9R;`)9uimew=x{ z+yqw_*2Bx4&rsl#IeF#GG5N~wGb@|xAoR+94D6F4x77Zj&FXL5UD6zWR&4|6?s&#Y zAO)14Pb2ZqCX>MuJA8UP3~mqRLG8V-C?_dTPfq&>`R*&3LPswm^mQ7oFF*vjB_!A5Z5?xJ-v_y3wnY`oFe+IDu=rhroeka zXY&4iA|z$>GIlZr%nYMbcq)tAd5xvvk`x6Z?K6p)A;`yrq8wx9fhKsI9)}|~k8$n! z)0o6xh~!fW9d!K#G9Nm5N<03+Sw@6Z487(UB8m{&PO$s4FxDD;WIB~4h}DK4@H}@m zIn{TO$yAy`TcaX)cNVlT{skx5^`Dl|ujR8)BR7f8Q?Y@!oJaM~tz2|FvIuqFsFG-7 zSC}29Lmu4rB9|m);IQO5bZ(tZ{RG>Y4)PuIx7@~`l49(wNCP^t(u~+tpWz8e*RWI8 zy=R(Yor%NVgZS7&jqLd=K?2pU;NRp527kzr)Ph);|BIlfgd^;ZRAy$qG^9ftxeRE{ zTCDB;gGcTx!nCqv9PJ^j+T3P%*Wg6V)i_S^3LoN-w~N|Fe8ud9d0@R$j5g2ONN=bq z5C@F}I=QX~zRH-AT)jE;UFtn_+WFp!62KTx>g-BTi ztmCWG?s9JuAE3eX9GFWRB5vYR&iO6ZBaBB1)F99&igzM)m?>C6I7Xm6*1QmAw}ywK zBT1r1SMagL;3^Ys1@r|B!r~N9G87|49JYrO*&zq~c~*v*dtRI9iN9dtOYfm??`%@L zK@LqX%5pvrA6Rnb5_3J`5UMjh7+LENg}vwDS<7d}`-3sr?P7(uH_l=IwfHe2MLbeB zn8~>#CeV#a%JkZX3uxsa!812dr+Jw%&~vi{-2WwD5yzmhF<41{K9^<;LOQ^~X$jd- z*@oT1N5N@8g&cc@9K%(ZWai3}hyE4#QCyXnT@)d$b3}2}S~!EQHO|JJ&N5(WI*~Xm^r9o`EzolL z3f|WiWG-wTM>(VW80@i@%kwMJnTH+8pADY)>Fsp-L39giVs;sS#P`9Pz(QO!-;Eg0 z<$P~r^HFA_3;TAzG2MFVA+wMvfc36?_Hg<)=GpF02yE5GCH5hB>1Yr5RUL1+${aHL|sz7z*P4Vd7 zSMax2k$f4^Clg!_vf|4*R?vfRYQM&p^vfDhBW{15X)2Frm&_s>CJX86eeu>u3zC?c zo<|`2a3gN_48m2_;k!o0wgAXALm<u z8+O~fodi7Bq%Ch>qKu;e?~8W;6|>t+#gb;@1xF{kNPdu+Wh+EgB!4rx*8*ryfi_if zDFErJ*BHofEYsNvR65n2e3*EZrOU1}yh?R))8G>Je3=UOR~XYR+8pco;5#_*KA4ss z(}rJ@7DBV_UA!6O%^H3T!5ec5;PVejNKeRuzmpH*I6sa|d|?E8Unjw_z6uO(e9Gw8 zl)<6P3yIURWa_%F53emzq4MvqqJT{hd(nI+?BezxGJZ|4T&Dxi&$dG%Re+X)rX(S3 z5!sVtK{{6dfJKFEpmSB5Ua8+s$8r?N`EagxJ@5`t_9Yx#F_YX3e`}pM8VX|^3o2@C zF`f$_XQQGP)83kR)LxMYmY=lAQ_CitXeLR=9~MAQ?muSFoG9MQP06IiHl1nfT8yfb zJjo3%^YAH2h=z|(qD*ryo^(thmsF=f{--$7E8opBoC9E$^-^fycAv!?gK4dQI5O#X zd6BQS!>>wJG#~zkHY?nLH-80b_T^#SdO5l*Jd@52dc(NsZX~@ARN+Wh8EPpmC;3X9 z;I8rwE%mc-X`u~PFss=T{spM^h`{r=JmJbgUmEz|1hVqF$u&z=K@vS5;7ac`G?=GN zV*T8SYdDbiqv}L^&kmegtVt(4d<=TG#~`th#S7QC9?^_E*0)WX4sf4m!j6ZSXDCf| z=dC7N|4Sxr#dqQ6^%Hn|of*~skq4?vy&(9|JFE-Z2>dnr^ySl1z%Me;o7n?!ne&W} z7w#NB4K2ey z)P?laqa;$mGXbTvpZwA@%gE>IU!Y;yfU<&oxKU+DefHEs$)qFrH*qoqW4=NP!<~}9r`1C#OR5B#y3)*3MYd-#!Jqdl) zqSir6a-dE0Ge5LWns(inBto-{NcK=VM$h50Q#~3~esdrk+ti7c9OhXc@^E8xCVp|< z#ulDp$c2U_aAQRTZOUkRP8|Zd>96psy)d+f3P5*?AYbX97{0Gd!!7WW>DlE* zS5El?&xaEsmX_e7_V=)Bnkx}5T!@hl`>5WmcC@{-ms&a&vy(an;F@q3C@RE!*ZY6E@zNM4UQGYDL}(bBy)h%s0o4 zblcx0;P~u6#&PpfA|uMe)R*FP_dQn<_azDCCS(wj^ctVFL{q(LVWv310^WS!k?mW~ zqhED;-O#iK%zWI0;4q7=5c~$>GtV(cf0_}sRiQLPjC0VBzGrkx7Q@_X9TKunlVfDK zQgfln=4j5v^IC@Ph1`6mQ-?NS*VUn5*R#}q=3M#9QTanzwq zh^!ekMSE^f*JE!<6Zc+bYu>ptjxmpM!d^u-v*9s*F0X=T%U6-ay&53!a2mdEMBJRl zW9wd*;|{kHrozRT@`^jLSHg^Z&z?!l$wESWL}<`H7F8-t(Bek|d7-r9s0y8v@QfV_vw}r|hcROQ4&0m{P24^?(h{+?MtLToYrqiL*5qJNjtW%> zNM)CvF{A1MufS0DiuLi2JR+bc2TKfJTALiqAhVN=*(ujH)5XYv#G+Hi*jKRv?v9$^5ATU|%wcIPY2}eETLuzaC#h zv@$kg(A#2Z(7Jya29(_# z-$I$blK{GFb_h*Uc)+*Muz<&0zUzAGPmI_q$-HP?M-HoSyvKe8a2kt^xiO3*>6bgwbQ6- zKj%El-N}wLJ_2+7cvkdk7CzaK#2+|0#%##QWfqEhFiEn~5aLk-UtT4Ufc9N98U3SIQplW7u%q2j}3oIBGVV?T&c`{!w}3NPc4Wz%rZxC4HB^c_qi zbMSMpFS(Z~LhI9J<0JtK;wXIur0*Q$?>My;2g`oIo3{+)hH{=&w4l}DquABt#9UG} zrY0NC;L!9uIO#DN=M5g>*BJYwN6U6#h3CK}c_Y}Wx|CyY8^W#3bnAbE8)%bc4u+}N zk)Bfzu=`Lju1Jl?(rzwSzOWn)+MdAg!k2OBmWeP)LWT?oS~71=36idJ*WiMeH4HD_ zM$)q+Nk<9SLmm+)9bD%A`fWW(D0_{3Wp}brPlu!@_>;+2`$7C*15ZG29?>~Ji8=<( zgssvuVcoh^=7>#uUC6qvWdGa$Gd)DWa{~{p;VLu>=;3i`j#&~cPg;YW!FedHws`U~ zI(T$3wcezT8=stp_P^oexK1HX6%{7)go@yKtu?FuVhio&G9n5|4b0!9>0H;M8DnZ@ zLY~HM+L2>TKE}7RJ|kRC=I&w4xi*Ky_soOhrzas}Zx_3{LXMl?Rd9p#atLkFBOlB3 zNT0tPaj&l7x7{Vg?1cyN4#|MrbFN>^+c2`(kbKhAKy-@O zgd6eDl$?QTnTyC57-!>q=ED2$)_CF@kIHsb;z5-$e7$`GiFoS;bz$$A{rxNP`jul$ zqJaWEH=axuuldDv&66S}DrI=IN{z^m-Nq+pH<7#19OF`N1AGqAW~I}$$^A)xc`sEH z*s*(ccrJB_HJrGNY&YZ4qmNBUV|WP5aYCU#ND6~owTbhf7hSol3uHK!`pGI5z7{Ew zW*h;=IFv5EqeX)aIPOu~CHUSskBqKqh4#6pn3-GS@OSDYx_xvysHf-R>DxCzslFV) zXK7*ZlLOG(CqRd%y0C(g-8iMuj%fOZQnw#f_pw%;=1B5GF%?e; zD^Xiz9q`(*2lrhZ#eE8)C(SN-BkQYpy&QUx)CkSR9 z+YNzD_T>7aN%T>QGqXwVBgbwqBQmPFY}V*;mTXGF{q9ZdlBY+pQdyU17)+(o^S^WZ z^-%V-c+!k`<}x zsYl1XNi_d_6x$nWOC#2O$7k*YX)2tvQJq%@ttt;x9Y%ya99ewhkFc z{EPvIkFyK5Ex|7n1;~QrZ@i{C$?$Td1XdNwVPYTR?HSpywJ(J}5@7I^2GEmxr&4+J zhv5g}ConQnBxik&lM z2|p+j^c9nE%2}>67AZy67D&Uv(%H;}3-6iNx*a&D=pyQtPN&Z^y-B#T2@|Y$3{H;8 zQcd$oWJj|(X)fG_irk)I!n11_-RlN=H9Q)sXh>oObh-VS3EAc@L&qkIut{BiAw#eq z8{$%-`L`(~#wX*7=nZ7j)J;$`N0q#7_=Qeh6`=mLh;iDXN0;v4LB*r3Xcse=OkHJw z{#oy#Q1~R9oe@o?WD0R*S2p=mElK`yx9@u&{xI6gaWsTuz3sVXM*E_b*tZW#AwE%x z`k2gSy-(_qyOKKGT`w8e2U(KaS@NW~>?8aqa1s|4sMFW^O`irqe$9s??wAFnngae+2RKO@+IK?z{l~UMN+NpeLl}(i#<2x{fFlCoabwDkV&B zRH;+($cbca%X|njp2%8o9+`5zZ*{@JOG)|xE%eQuMsuZKkA2Z^bqH6+&hh$=GP#v(g1CHk3#6cB^;Pj3lhUa@Njb?IX`t64%G>gf~VJc zvH!W^hjIb3&uRdc?+t~>J=UnFbQ|7P7GRiaAFPavVU@q%Wj_t@i2al|`1OMW%@GNt z-;U?7N|g!B7YA{ATVIEcUP`h)d{2_JY*K(HDSrIkYhh&9nMknCJPit`IaiJIDhv;t z$y3o$hgr#gVeffo-jj55R-xuUyqU?uq6bTv^;c)XWM__PQm~2s*qsIvQ-aa&+y>Sp zp%D8V9>SqJ0z|8a!NlqJK`0U7P4OSbB=`Fm6Da+|-bSk($Tg^temw^1@C~{8bB5rOYY3jF@fB+h#q z@!Y&rVrA5Yai{mv)b9YNmQJJt{ZDYP$`AU4g;}rO$zT$k$(rt;%(2AJ1Mh1Htx9L; zx|UcJHB6xE0=JUFQ{3HP;WhpYHUdRGd~j*M4ox3Xr;!eceBBu(Fn#$8{*>Z2XyvEk zO8as&>Ix+zdA_)(Bo41Vd5bM~JD~fr5-z;1N1HA=fc*MVw*5ji>}tOaTZ*jcnj>yB zV2LcvG|D6&xXy)ItSpUuDoFoJ&7%Fcy*bC^SKQsEON;emnfUX=aGBYS6<&niLFlvTjQJl^mOW`hM1S3cfZTM-n)AWZ;w-xL zodvtQJwO|-u-5E2+#8c4TRnqFTuv0J+Q@w;9_it-^=+(E?p=&t7)L8Nw)0P4^CL%+ z_2{bH3}`sKpH3*%qL*x5z$bkRa$PqZJ047c&?vzZp<#r3l+XvuV;5cSf`x2LsNUo13 zLBE@;P)$2u`htH6B0B@9z}9QbK=VcLE!M%vIjw9bm-SqtydL2h;Xhz!um_g)Ph^iznF9~5Pv#4sI?Xm7s)9-R#zeJ5 zl8q1TgcA|&tn8_|Y}Q}|Y zIj7TR>?@c7OFaZYXv7Q0dXjK0*Tvo|=s}k=!MHi`11qujFSML{k3R!MS)sqfaNf4a zx~fK){w%Vky{r9Tp5hvteXs0)haS7LAg=7%inaUhYbwtPRRFIm$BkSt@>1`uz5Z%U*WaT4p7-x|^4z29! zAHSf@;x*j5?LlH@RYUC8T3Axg^?FCHLiG2~c%teWYCbH1xD0m^rJMs#O~vWxHf^SL z<^`N@qC*{zxZ~s3Zj9g;X`VBeHL&~F#KiZ1!@p_bIQln~&|))wr;& z5vkPZ%^cR?@fsMm6CyXaM__1<21Y*~W_+BLVaxmS+5+o1OlgoMg_BQXsfjtASF)R* zx_T}CjTnNT&v;}PP)Cjfm6xKyUjB^j1+I7hdmrCZdkC{Dn^0BH7W7W$ zGh*-kv7~Q^jrngD-R?cg7%2%LUq_K%AFPG`DW_4_q5^8ak6=L+mn-4k7wSKUQ8!y- z>`C5CZd;|m} zS#;C-gX#*rBI`!BYZ}o|nf+w$^|z>eSd6i;UP>-McH)#^TEyM37`}!0k(gT+uqSIV z*=gF1=UTs`_QA98&N&|6!sVt^%!>x7f2Gga|rY&@`=CWRk%}aM^bN zSM3v`eI$ekWTn7n{#kr^^dRc)O2)-|JaJXZ791(sh;OZq!1~@4NGp|Qb_n>Aiq4~$ zcd&!qzRnktMOc_V*oLu>r{KZ*V*EYq2>mNduyjlrpM9G~jgDGVmu4RAJJ&Xgox_8G(}bP+l4y@&WrDqCY>zVKf?-g+GVsV%@x_hst4B#atRbU!BNUd82QL z8Z)%h4b!E$`)x}!C^Qy>O+!9dFZLo47L!O~>{`O=P347k1!Lv(e^5}q5!M%a(cSHF zbaT*8{BCQ?PY+ZeRy!Bdlw42h$?azHM?|b84sqRed3AD@R$!Uy2T=thi z=1Ef#G>5!rOC&w=Xqvqt(}DEDtNJ=v9y zi9Zd9@yjp3^Et`H%$Fzs+MYr%$9UCkisxzH{RDkM`(R>40H`N9&^w2mF+MSod3bj| z{0!mTY-vx}Wn3@8?QI+}&+;LLc_-lTHUol}4`Jr~bf_N_qzB*Zz>=G1@IlBf`nF$% ze?hSfa6Y$_()`ARyw|4vMG{~x>_w_=R?@Z^=Wyrq61ZZS|yOui<1)4)ygPW=(boUjE z%~2FAR z5^PuxQ;!17W*`j&2_0sAO;3IDD0ldpn|%RABK&o@{QX9!~JayYiVAUX3b zi3C5sf>1t#c4}tQ(>G+G@ca-uwJ)tRUptqyzFWoac_vJ&Zym>L&+X~I4r$zeI)i#; zW`T`uH`Xt_2~jp*nLX`^@M$Q5`E6JQVN{gL&2c4KT%Mw&kz<9Z?;_^y2O%I@ocw1a zO-^o^z%+aBBZgdOW*?W8`>8NVA2BRe^g2opUz(7-V7 z9*B}vKaAN0woM>YmxnjFo!W1=7f8*L!RWmUf6#0Zk0bDAKIl<+DKGNilQkc{m$=yF0RkT zd7g9M@AvEFhaFkf;P6z1yWcr5V)st7PsHzms_tP>iVCHVl>KeJChsx(-(D~vH?@`etdcOHDm?k-r_|3wo$Jk;C3zFDG|tI z+ic7mZGz=j7GrLa3%xnN8ow^oAlfqjz+%~RJR7D=?0AlF-DDxD&J;iKXggPx6^1=B# zFT-0)1Ny+Cl5aHh0bX?<28DM{m=)c?G;FjX4q975Z8!l8)MnA9eJ7!=qnYvErbi04 zo}4_^oz2vP zQ1t}(ebIuu18}|TdSQmsK;z2esb)rCkURLk z=Mc`jy9uSvx#1JB0%G!eHEbRiriAN-ItV0Et~o(-o3BDukRx64zMkzJokPYy3DfkT zad@=Ef!KR@Fe}`iqJCgH{CccNh2-UlPYqk;YDnSm2?LrJ982G}sKRqUMUG!_ z0{_+=gt(a(!BtnAY8GuK4{98!w%Q|9S$`5mR=ffEAZ2($9w zJI2NCIj-n#V0%nCM>uy+jpsab?_?ZcrI#2iRR4oN7f)x5N?eKZ+YZbe%Vs+7?cyKn z2*y@dIhsB7Ff=^Z=2$99Ah_i^$TPaMQEL`1^&Z8ror882+!IrFP<6+dP?4nL+l5qbSi=8kMGIY)D8()o`}$Hp3n-nj^Cx>d=? zU=}J@aBKv-__K__sJywG+l<&wCL%N$&)%`K?BWoVMeu z1P2n-mjXW*>yp^s2TVwE9Bq#M#W)N)LXqoJG}~N_v3_B2Y19ghrfg@nIelQYYk%Up zkXo3;WuhEr^+40DZvnk~UId*7 zIA^>R=jNuG=yZ`~wZ=GZ?gurRwdyN4{HF>IRxD25qCx!%y1^t(2+J3CKubCB59!=R zN6Tg8YQ;T>?9ikd^{dfJU=%#kMTq*1cUXBrmws-l1n0`H_~QFr-iVY5ybWJYqFvR= zv2z}@?(9FNV!=W(VZ%h)WI2ZX2~sqDe-zDA35Lqi38ebtMYxgABS{5m>`q%bGFd2s zPPM7Vjgku>^u8&TEgxi_83aLZToKbX@)?wzU9slhPI~C-1oFFbJ(r7>CjoIRUv%UM znp&@;zwfb3|2IqY_&3Td*+Nw|$U$^-F`O+h zCW6TpWJ^ONF8=FIOl}C%z?qwvsuzn;aKx5exwH=*3mu5n{de$oNCC`DZbH`eSrDFl zo0-zK9W;ASVRzYYcpW^Ky8BGUb>+r%=bm8DNZUwy^>^ZknE;ugvxvDKvzRC}>(PCy zl9|0)XE8~o1D|Otk~r5CCdbf>1pWKQUYe*!i^PK9&V?p;Ul~B-Ql}G*JHDjf$Qr%{ zxX}%Ti}`QWgo()61QdFtMm)Rc(n?WlvSdgVm(BAcWyd(5nK5bF4?0`CP>HEmP;O>I$p~iGwT?!m8@5Mp?i}?IV0kQen#ap@7jocklV~f_F z#~-G%sn8V}rcu0siTB;lK0LjSRqV-vy@RWmAx%4)SZ9Py;~BW<(+K+iFeh=H7s2-H zFZA1{Naxym(~(EpVR~f{y*(~MuW&za*xZgjVKS+oM;gvL(+JHglc`qKB&xe`G42^W z24Q0x*yt%4xXz%2G4?CP)!#&kd)+noyIY4H;l9T=K8g-n1)^Q&B(nciBst{j1XU3) zU_j+3Y)$*X3Ub^Hi#9|1qg%}=4XuS&(v^59*q=%&IN|yF3*c0D4zfobLHc_tIa}NT zX~m1k!r)S-+{>FBHm}4&t^4eXd9kD~b|KT7yiwmo9ORyyP^oCNg)2BedB#}nGLzEi&0ZNP*dC;ju(>&*r-`;2WJ}D(= z=%|iivl@9%g=@jd&57u!854fQA}aJppM6&Nf#pxpg#HP+?4j`|81crCyPsV}spZ<3 zn`en-BI49lmOB%2CCMyFDYmUwoa8JGWM9b2Q{f&JNLccaJtmxkdj){4*B79far0?G zEDzY#d2nO&C)eR~B$exHc(YD#r&|wBqtT96@rdU}=AOqQTv%Ypru!DLa^4F`N5X7k z(#+ifT_53mp%!iyAYS!v-22PNk1CL}5pqJ9Jzx!WelNoM$i03ff#X*(ZKs-1!(pY#SY@-IFsQ+nG;{_wjJ6=yVu+XODX~u7KC8$MLD} z3UoWr!Z}~2!g?5H1C-`7>nbDpy~y+bo3c~>#lt=_V-RvZ(}tBY0JbB1-9JeeMz zCP|l7t59Q&1?0e>7lk9Z;HCppd{!$bnmL;W@CBu zh{1iPVIf1yi4}QyLWp@>q(C~%df2vW`7pLlkhRHf<7XHNliL#ec_!CiVY~7he#VVh zSgW`n>6xpnmyiU>o19H1DRR%1lu6{>zN`3ZOAzx-%^Ox9KL%4*oM7`B3h3=F&igLk z26_Me;kT|RWu=1cVBsCkg}O2phi*65m=%w}-1h$>v}EZ}-TtC^yeRg9&fC~Y%PBYW8I(A)on8R^`^ z8m9KZV&6izzHtm!d}~7ihRf&fDTFD-1I(&@#q0r1YvNfwk8+*~ZWrZ4Oz&7D?XJP5 z1X)_i^#Oi(bub;5Hc`PV)Rz(X$TwINcFGx;aDq={IK3Z%bnt>r@A@FMXM7Lt;zIF=3Kbld>$;o zi&{L8A)*n+=tAr|b-s=cSD|*<~#X3xNWGd|NQYWj^ z|AAP=2pCF+qqM*Uru#z}&QZC@HruGv+mqMRl`85?w^TBI72Cmje^JI+o1fs0>prZc z@k5;DWk>c^58|7&KJ5Hi2*+zeajm5g6B~;u{d8jFX|h?iuf1 z5#2%JWIwRKA55acoO`)_T{?{S|Kc1jh1}C* zHLYI!9d>TWMD~#e`O=(D%)b5Qoi?)M^1nN9=vX=~@?Jxp+HmgDEmb%!-pcGxX#~A( zD_}x&3v5Vez(-4J8O|XIix*kZ=Vg?6c0ZqE-DiS>;TV%+QOWb0y`4z><`@;Bli7hh zKxIZ9P3H-M;`l#Mve1WNtK;nTsghuNa5p^RXHvuQZ|MC-g^UX-(ESn}V05#A?Je8O z@wa5D^yxvk!tltP(^jZik;)`F9>5Dtei(X6lO8+DZf;$7C%|(L3TMtnle}n5xnxgG{z;INBZ=hGUsp1<>LIgy*+tNkvBk5K zpJJeoDE)JH79s0)pvMa(sN?bgxlVUMb&(Ls^9-U2FK0rCm=%2({DBqWoXCe=&Y*f| zD;uIUnVCEN34HE6#=crCNUPM|GfQ5qWv1OwftV)+Y+>XGq*Wcj1DAHN>Rk6;WAQb1 zvXdgH$6MfV=N$61yM;AvJIc$mSi!kWm04fZz*FmViF>L8=)T=TWw^dTdu{{vw?ATU zN%n)#tpl(sgxmLU`NpR)nJ`j5!qzU|N(!2$(A3sKd!0N9S`g|^#c(?@2Gio8R?c`um^i3r@)#rd+p_gBaG;ddp;m5h5(J3NB_sw3$4YCD_>7{{Z= zoxIQG5opw_MR(qGC%a42Q0K=)y3)1+ucS-jz)4qX_)mfMXMKace=MnRaUdqeMq)}x zKCC&pk-oj)OUfnl6c0<6Sn=s|WI@VA5H2Wag zmWZueN^^3Dz{IK_W*!n^zN8D1n4G2LsJb^1Fv|op(Ot}9$5o)WtRLb|Mss_nKOC<> zga)a{;`z(oFqUkLtDUC6gpp8?IW&#_$@QsIJy1N#G&X6aAzSGH+2easwYh7&HYb}zuMU?nnd)=c9HnuruGEsATgk?Xk zvC6G79Iw5bd+zvhyOJ2(v%UeV?wgT!e;c95X)W{cN(r8qH728drbr*Cl3z<}F#YRv zVpeJnxANm5XF!%rjn9M@za^B~r^9Z00yF=7XIgKZ>kUUX|of|%Z z7++d~FMkTs*8$q(WO)G0Kl%&>ceI1uyhR{VJb^!!G{U@Z52fAmQrOyW1L;=cwBoKT ztvyo*=BDq#Db#@)xaCs2)AyKKu>yP%-HVqT>sc4dxm#{l!T$2=kQ4YCH+zWEV-n|a z)}~g}=FdQ>WN-Ru0ZWkwK@dK-r$Ayo6$Et{m4oR5lP27U4`+>5F=^J<*`OW^+8Q&!_IStOnorg=($j}KH@N<| z-3-(dP@qc#7r^pPOWMbtM5{a=)oF-kV@N2dciGUFi-`b+I?#esnWt_s79=u3r{=KaJd*=#NJ%r_i)0M|>#v z5>0*ylTP!0c+=FLw`3@g_#{2TZ*P-PFzF_4+ib@E%Cut7_2om$`aO)K`g|zTtc4#d zl;|9m+cT!=kf`Yc7s@@T=}&*# z?!fGi-!XT#DqYI2WF%R85cJO?=S2jm$+JjCd4?-J6CKQE_07bKCUtOp?iy6S??#XK zed8(gedZP0&LswZqSW<47w+3z%67z@!-1s*#QT6d{r&3~XgM@9zYM1EyDo=(M_T7QlU`E4_&P!KH+FK17qFVw@Wwht5%Y0@pGm=?Y5`8WZ7w8jdo=Hf{#J zvLgz-wg{4lC;^%+E=c>7&ZufqO?N#_PKxSxbQn?8%+!*k*Sa*O=$xi+N+X zS34bKB~#%0(tq$`Z7|nQmZlX^lW;IQ4_^gOBp(82va_^2h_#pj^CWyX+)ni&&ks)} zo9D?hf4(folUe_PJ=2AHcKujd5=?Jd&1K$PUq^lJ$-(;iHH0k>C1JDH;qzS?kbU(R z)Sb{G8Qecq__aWcuJa~R)w*=4N)*f~e$VFnFUL0z#$f9<1tJ*u94hLm{k~0qF?QN9 zfQ!cV?^E>1*E73Ohs{U)D{1$7&kSy+=0Sao9I2gR4=c2$mi_!{7WHs}O zqh0Y%PU{?%~@7vxxZ-Luwln##T=Okoh!$ zS(=f;J7dpzcpJHyl1v!ntdNHBn=`1;t~}UedJx(*vvGXyHn`S$m8Umx7U|N9z~|c- zFe`JS%ZFsqH18BfWJQvWT^^8|F_lTmlB80zccQ9)1#WpJO-_kSAOqj#(D7kD{E~`< zGAWL;mgCykFI9sX5-m;I{d>lhpa5tWFJSq0`0YG827>eXKdcf z+?rIvah04Q>isEjYwlqMmHzQBh3^8>t(tU~hyz+J-w#6ST6EH?e0XlG207pMgVUct z{B~4{XmG5FX{I`~(yxXE{Z{6izYtBForp_orlXWoCH{B27QPipkgpC527k(7RvSOY zIJf`sqTvFV|KCQq;**F%8f#%r`(@zII>ugh^daxB&PM5UWPk7_QGCBIshACq)~4(NpReY&zuI(v-$--x;?^t>OK$0NCb@)OTz85 zZh`8i+hDq|9EKQU)ZdXxcbt$Wyxeziucrc={xe~uLk~fi$t!-%!7*%W*+${t9;nWq zg)X!iu9YeiHT!gYmZ8tAyrc*w|2g6DAB*u_;T*apRg(n#ZDUn?KG|>5)Le?|DcCC4<|kR*<+J8V42|#4CXjf>)q8c zRx}5%=AFgnjJfDECQOzm_+ib2B)nav%&wT$%f9^inU(sogoN!`M-FG)W1_vdIZ~V+ zjT*3{_Z5$_S(ZQHW9DmIXswS)UrsVl8eCw#$|S-oG$#{%SJ4;UGicK*3u?V=*xp1x zovyv6f@j?8(b%_=J@jKX`RJ`dn(x1YjlWV+V-|M?ZRYa9N7G1*DjyX5H?iZFbYP}L z8-%4Tplxl|q$qJVE&95OdSChij_Y!}X{(=cjwm zhq?+S`buMJP84*IX*^(O^9KU|+4*lbvOZEfnBKK4o_}OUCGl}guKgmGY#L^ERVl!#gjVPtbECII*KvHE6nJzq7WaKmfb@+XM8fZR9XL}9h zjnCrf?w_nlupnJMC6>NwSx>L3^1DiR*S%+*^k*s+Jr<#(Xa z_FZtpa4k3AP$ZKr^GH*rB{{Uan2q`8Ldr_Jz&s&~CpO!b>h_Lf%kD0A@xx9W3C?AH zhVDRzU+dtg9ceq8>lwww;rQ{f22^8*&a)Go0}L;8N(` zq>3fsi|`ZM%cxlflM3^>?8Bc1n)JrrRjiHj5ypC-EPch@0WBmx!`Z=FxYMObRF81I z!dR~JF58ag1CsO$zYBlQ@x_#qG@@CZh2LtE$i)-0iC6wGigP~IGO6X{WBo*OW=aG- zqT@_f6z+%oVoQ`W-_L9RvlK!vR^lBaUHFx70k;;J;?7l$oO5Xh$X!z;4Qa!m`6rj_ zg2<43i5&FnTu$#DEC!w2kL=RG_h=pb4PVCIL}#bV?AP7_+~rupScD6L@Y^>2ymQ>{ zF5Zj$lQ;}tn(HxrF~`&?mBTTYldx_=CF}bA6m!?~JGgeQg;(D4RAhB99hj1X9&PDl z*y|G*TXTGiz}=AA+rntG$Dp8U1#>9QpMS*P0G8HI!0*joSd(r_ov)km-Z(9wjOH$k zyC6W`TmM79fk*h`MJVA+xv<(rm{)&$1@S2KXYs#wRN%6i9||HtDc_W6HhAMrVNa_3 zK^PtjJFxb;CUEn_7{1#lL(M+?WOn^nj=W?0p(*wjtW%pu)E8>dGP4e7zbZp!HI{?t za~tgZV?}J1=@Fp>A=+Yd5gtYk;Elb$4HlIjQDAN;-5M7FRjWT@``3dwBh?S)d#xsW zaSfb#@ev<%-ef+!SxAR3y+?cA893;a#<8D&qsQhBeB)h&T?3~;4z^&>K|#9I!i`Mp zT}AuGbK%=^VdAvKneG#`g$1{r$>NSy4B)u)scYI$IA=4Jy{1Mo_f2M2uT5nZKHFq( zbKx1kQsM}dP1=haj7#yLu^#$V%wlNv7-&SQ;~Kh;Ij~8R+&lS-jf;K9H88ds==Fc!;MJ~ZIPMUNo<|QuT*Ive6B&JiFu)RH&#vI5685c)bH8Pz}vfYVmnWSL6pKDE=`>YBlku%pGCc z1)Ry>+=(>s??STRjw(4~znq@Am`)9pOTgz{YUAYSEyR5L1~_%x3&ov|R(YYNKTsVtuCQo_QYx+Rr zW_{D6OULDj4S5P8wHib@OP`7yO~InjLAGvt0$SZ1h78-HnbQsu(2O_*M>Q8x`?PBC zIkB1gjZJ3-y3^^O@BZwT3VG^~G0aSE`T*6OYwy6za5~s1M*|hAm`2+@tjw$c@_Okl zyfp7FsPFVAkzB@9{@E_nEkxY3MxM^!s}0iM9x;~`77~$7{qUk^J1IJP0%L=2GQJxQ zGr5)NDE#pnb39=OIc1_lmrb|f{}3L4Ki|5bOL!$1wsG0lQ!?~pggYjUWz!=bEK7sR`|+BwHt7F$WlwSU zzDX)#bh-Hq!f^S3jU~z)6W5WQd2$l{&uJsRtP>;URsVP^y-caq6$)n7k)#4ONo}b= z`dFl4##}MFA$K~BDPYa+1KsXxG}_#iC@7ZUl;yiml=jbseCGI zYplduDk5O!^qdu|%)|{=Ze;wW13t18WF5TIICnFl2QHbziy~vl_!LXE#LZ~>%Y{t1 zehviQNWg^=5p>A48Cy6%fkUhT&+NSv-S;&Ro+Z2D{QaD}8cgV5CwC{{_Oy2C0VMkC zUFd2VW4^5yB;EJde zRR5_uu~tC5w5J-JCuq>AwR|+ca}N|dB>4OwOPbbGg$J(NkOFH}R!5_dtm<|p%TCr| zG7%!vJh=>4WfexdEaK-1UV`qzxkONXHv9X-Yv#!PU}%)zN&@;-P}@%#LMOLi1#5!~ zx=*7;Rui0)T8d)b*KqB+sif=yGWM(0*}&m>B)L+H6zrY>k%rmS>b@%3*OY+lpN-7# z=u5cgLK@y4O@WK?nI!Bwch;zDk!&3?%IhzN4Gk$wu7egMbkL0kbc9pIXeILFMlxB~ zU5AgYo#?*dm3*JNc6OVH4bgtvi2dtt!;wqg^w^IzMAtO{Q#}hAqnUc_MZGyt_8^`y zwAIInS&iJjb1B=hgAW}Gji`&zQ=DXUn@zB&hJtyM=@(fbwPrH7B7TH@7kdg-w=W~N ztm|Q)S`V(aT?K2LbBUO&FmYJ*89TEdLU*(cz0ci`BtK8Zz$jZdt?UU0i$+nvB9fUt zEfv>$wcyx{P4rBZA3o#%f`2Y9IAL`adJ0xzqwiJb*b{M__OlEXPjWk3Je1dvGRZv0ZTLddbYowXb2iCD-d)hD?;58ksbk+e-C2$XQx7`hD$6XHsz{xCbLsfTCDiUfE%YYF;@!B9=h$1rSq97*Jz;-ujVM_-Nru+S2GTwg0jf4%3?=^Y z$mG`hjE_Y&QA>`e(Ho3u!FzG?eO#8-$2YRZ1tN5!UMXY%;B z3a;694zffWIcMiQv~rz9(;r9Dw;gK4!&;B-4qJ(JK6{w2&3~ANoa6YP!A^|qnnVu$ ztYs|guV7WsR}i=Uf(K6|ft!6OsW-636ANFmf0L$=C+j@vkKV(LbE}hCp%ETkD7AvD zG>#|wzZP(_E;Tw`CkJ!AJD~X8adz2-Im|(h3-2D#)UbJ3D)sI6L7!6#xz1q$WW1@w zp}vpo;ZOx~@%>iHIKG6p7aE{nbUU$*ddWZB^p5@Da}&IKgV-Z#p^*1b2r^x`9F5sJ zJh3jF?w)ZJm(L8Qu3jxeD1oHm0~hxew%qL+GineDb_% zJx{@*2@}NEQ1#7|Kp=b~jSER3D)T#-+CB9+_4_R*`;Z0BS?R+(l9>kgPvx{c}=0jW2bIZM4ruT3si2>xVViAB)T zwU;1nIXtI+wD7EFo6TdY4o_HQ+?H-4z=GCzNga_4F zn9sH>YQ{6EX=KfvapvAGSNkvD_+Xf`g`QJfO~(@kn0b?)aqfuMa3<>?Yq{bh(_*p& zIvS=yV1*>Hk#B@KRG+wCaKIgB@`y#SK8Z`-$xmplLBlmQ=zBpBy@q1QlT)LtS3)n_ z-N|uOe+hxH9Oup8-fjN-q{kQ=&V;MZ1w>yupU&T0j`2sw@Mmp5j7nUDAIh)!wwJ2m z>Yh&+Ges26HwKWuo8yUZx-nh8W);_k-G&WOZ7`T)PR^-4VUHc&%L;EdfT6W!#O~xr zh!F!6U(*ER+hs_Xw<2}Sbz|ma_HwRA9a1I8^A5dVEGlDYm%BaZmH7m}8)eCl zq7CGjgC_i*r9w1If-pYtFn{@x4BqUORu~ed3(2ZS*lmAb<9K}{S*r1vzj?MTI2OKO zHz`iZFFr6;LTVqn$ed{j%6j&6(iH=#lG{n zP+W*;My6xVpgL7G7oY=W#W2D707hMugZsBfVV=oZX1QYuXeva4+AlriS2#gU*r$fQ zT{^UU@DC3B$-+aS)+FL}3&hS7CgT=7Na?)+X&is5ylNDlw#MLwo>V+GAcWo{Ul==A zZ5%zm7oESF@wDH{vq$8;*(YlXiCDfEUQujkY`MMJ)~O?mli60{^8Ga{F}9bb-oK$* zuBP#UX+8e^!(~g3+n_|2B-y=n2RYkTNV9!hiHUIyDo82O^p1(xm_LU0YsHDms`ZR@ z=_l5zbt$=0>O*Fv?1BZ>=h?Bin~Wv<4e-)dv(xa_mQ2U zlSQKr7GUFyHlFDIId*4`1j6F%Q5-fYLajD+@<^SbQ+mwVc^5MIjl1kf;>i+7jQs|^ zH|^m`>jan<_?sURn2TBASt$2x0b~r8uxaDNSk#z;vtMg4<#S%K{{>Eg`SKK#7k8q3 z(h@52=^}fD>v7Fnz~E;tA1p91i5BrysnBIr;=}R#|0K;u=M6miSojwR)MP=LkU4uu za1$y2;7_wuBFHDDC^U?e1@AMIbN{6gm6ws^;pYrCd?Xr@aU+>lrA6MwKLr_SJ`wS(HG>&aRL_rNQMGTu+L07&aUa8r>%TyY`&>FNVl7m(g_4iPliBm9?!f1T z8N^cRKPdgA4*hMnq}6Xl_TG10X{z^PTWLe&~m{}^t~%b>niSm zz^zdjQSm0~m2a>`5N5L_wfsNB{#wx2z_|?l8MpF{0PL~4wIr^N5 z`K?Uz6GCa*mq<7(okv^VSyEUhMbv-XN5Ai8{Nz4Kx%Vj=407SR%O@EV z`K{c{a`OxjQ;+6Vcf3KjOgXaOb0X&i)+4!z^N8)>BG{Xyjnz|K>5B0-cyuC)8Jov> z#Vt9X`6NT?d$^IWDrZb0bZ;@AHY!k`e(mrW4qkqoyQw3tfEQPk;CBvrpY;XrpXwi-%Mjmrg0 zldA+{d@%->hrNgIhqs_gstrjhuYSQ0@zt>nMBtzmHmYoZ5$>Jhu&X%P{YabqT6vSTTKp7w^EaU;^Bz2&?!%pC``ALC z*ZeMrrSMsWK=*Az{#`F)6zv`8RN2jZ(U~%s9}>($VKNrUxZrH@OKj@uVg6rlY5ep~ zmAcq-jDb`;I_rG|lkM+|)5mXc+;%&%0 z+!k(o$dQ7l+u>)^K2|th6}LMY&^CSq@lu~=KX3d8Dgu8iw$$kVT%xGcXy z0^ej<1Ei!bL#}ZMRvaxPYoc2iqqIO0uzNMH_u@?YdWt!#WLE{=9?E1)S%Un1s0=*W zdYJ2T77LfHrr`pf*mBE<>*I3It$SSF%=i&3bykrN@znB@~U5xI;GG0Kl0A09K zj@lj&XsmnCjLZJJ!8l)iz}oJeM~3`F*hMEFK+gp!GC#i%tyJwu+JaD0nx9RzR^NqC zE_V}n*PgWGIzd9Z9JtR8pzC{=Fxxn1y-|=iKYxw^zhG`1RyEEg?IKlh>7z6~H?WCg zfJAYtS5=ZeTbq7ev>Yeeh%xt>dhGjd0pEr~=%@Fobh*t+(*18I5zRV)Hp(Tq?#OKoUaHy4#x~z&o$dzF zN^ak;sO&^@V?SZ*%Q_zav>|Z9BQzdZOdBSRpaJc~Q|?mCJk{gO@C{pHJ3kjzrdVKo zw<;AF$wN7}&CHkiZlr8sF}5F4XI$;_@z=Ig^k+nf=@}KeLE4$_;<6MA>Q=)3=~j%W z4+|Tgu0Y*s9@NBfF3D`zhYzn9)B85_!Ou{Es(Gw~OGgX{Z?Yg+_wy$2&TCKZ{4T*| z9hn&PJeK^I$mPc_++w#jR`5hyFS5ja8vgkA3>G-1kP!JNVA=W?Q-T%fn`hB*B&!+B zmkx2f*10rl@hv=XWfa^rwc(Y)F1UR!5(=VMFcpX20_m7cPxfyo@+m2J^Y(q#bWy=ElfFfjg8uEK~~Mv zr;bjlbdFvM`MBaHJd%A1>tFn24~ZJ$yZJQ5fN*|zVY81aNRDy_>Jcq126Iw0# z6rL>F%=1nW!kwQ@(YB8Mma zPVrL2g0f*0`TPKP+vh-N=2Yr;@*udGWK#aSpYSdB2uKGlrdK;T2jVVaKI#r9s zmtV8#pvfN4O-{sK$2$;Z5KV$?w`0Pw0p7G3eGnCW5*{yb!{f24G+9oDgtTYUnb9Md z5ZQ(|H!86g7u;pXy)x;I17#c=g|N4ZZt-KbuYq9$O}IZ`OQx(=Aj@a0=3Jce#OIbf zYR~%%;SI~_^pZ0emo$N<9`I&->l@i;GKnNcq>?=CQ{fkSN8gagd3x_s)I_m*{$Hs!CKuErn!nML@+4B7c5 zgp8LwfwiuAOj0?=atd@{>odOcEs}4-y-i`Te#22P-_-zx#jWTqHJ22~3laQy5&|t# z>78o};i=LQ=5%8*?{sJj`pA(zwVD(Tt7PeL_sW88J8hIPJ3`7KV ziISQgSWG{~ez~WP-WBtS^vMp+moy)*C!R*Ph}&p++z1q-D;c9fEi8DbKtj8Y%zX7! z2HRJg(u2qQKz8&3t17abmevRmp}E#1YK;s19{YifF_57dnm{J({mQ<%V@tFq=)+C8N9}c#WncdY8Pfvdxmzug+oj~+FT?g-^L2>rX`?1x7#oO zV1>3>Zp{8XQ{q?BfT?!*oTu6k|E`IkyH*8*X88tiI6ax%&YeldJafzC^}bX$GyNK0hVuh}wq*+P!SSnfp2^AY$rUx^gtePEJh z&tTE~In2O|A+YOrBt5ZSL}y?!`*)f-*;;!NZ+ke?PiKr#>h)vZ;AWuv?;7DsC2KIw zD2FS-u5j+70QqJlKq6;^lNBmgQRLxd(px{7Ru0c$TU^ew6Z1XkP--!1-uMCYeh6XP zhCjG-!Z>bilAt&GJm`gG*HO8A5tU|L$;!Y$D)=Cc%lKU4UoQE8?D%6gLDYaPPFzfW zItkMak1}`y2b5{wzFf4oJ&Z|Rp14jR6})%dY7`Q6Aljvt!2djv1|R6gb3bQ;RAvIK zEq3&{Si!75smIPt7)C|0Q}|}{Gf4G`rJMD~nTt75bl--_)aNSeteM@-z>{(Q1rwT63FKLcx8c%UsHPE|b# zJbXBWo!dM3%fzp--&Um&o1#p3YQK&ayv%}{Wue%ARGlWZDA7IBckwnJP@*C8xPFuT z96YgqAuBDMYELLL^2 zkl{c@dRD=cyld>@RlaTc=Ptp<3K8&f+EefAbd(90sz*WwS7G;| z1jdbRgQmhj>Z+zlH`-o;zQmu6UJ6&>&|zE7DXoK+s>bxlpb-&_o`wf9j`N`2aso z_T=^1o&uft{o_4io@cDy~L0XE8KX*o7wN8q!PG(=g7ep3T>oL`-}Y zQC!HD#y9Q9_t|2!uyG3gd|H(*+S7-J7Ty77<9l|(l|;sJnFkRV;(Q5-)94I-HOyA~ zkN4&3c8;g+KtuhrITnKqZH>B#GZ&}f(f&M8FL{P{xnAF%Gn`vvt}!wiZ{Yc+Oum3+ z4D-H#!K-0snY}`M*zWU@{qQxP%Gp0=liymAE8YQAb(bADzB$V7aB-(MXB-Du=0enE zZ9yPN6BYSuP~e;+27fdqeKwsqG(8*+&-#mtR6QU+ayIQRILJgl=Q^X0cY>~68I!na z6{yt?u%^z&aCogR*>9E(rmlqUP)#7`s&hzpuPRM!D8`t`hof4413@AX`Eb?Os- zsr+O*tm#fhBZQbXsz=f$JV0eoqFn(2AnpKO!d1>sHh}~C!-2!-lqRibl!njy>A#N zB9SdSWY44&@3|j)HDpAgk`ig4Bt<2A%P6TNQX(TF%6sl7v^0p)(2%03J$#$$cYc5R z_jTUqJqCx_O^f%jq_sIKrG zt=vukh;^ZT(q1Sm6^BcgD2gu3CVM{zK|q`##-IPlYRF#Yt9;xCb8>7+Mriy*&S7b83E8oU|={58A&ZXr<v4B6&b?26A07(=CV&&En1?U5AM@e(>8?@$lrY$r6C`p zlAl4{#p5v7*Ow*+?7#!+d8}0LI&$5?6~9MUvBn=al0|K0Y`bJMt$CJ6#e9+(u@9?h z%q|`Y%eui_yfGKnKdFJRyJ5tBdmS8mco#LNs?x^TN6?ZrgqIl)lFHUV-@O0O`9cTV zdi)9;*{wz^yI$eb51Z+A)dpCR+T7MZG=v__V>rNNH`A6`AX%V7cAWZwu{))(F-M%L zRm~*d^2f2nsh2%ExeVq0EJBErp@L7Bk-JmZQPP`C(NB#!v@6lStG~kx&WU_M^DtKC zH1T5AXF=8*jt#=`E*6)~;kAz|frvv6Tr5*3>vaF3x~2lTqJ9lN-z~%|kbvgB2OvRe zE%~@KnFR2@vy-$rU)Q84sIS=p)y7Amb#EUI&6`f=dy3-8hr_I|sR*6DD+vE4ts?Ig zp5hT;KR&{d7#qrXl|b&bTK>h!F&vj9 zfLL@(klPQ3SZ9r5C?7Pz>S1~MUHd)8ehZ`{wL8d)r}NOozZ)O@tp|evjxVjW8>z}d zHoVP;9JA|&eILBgrs67SoNfXAukYaj*T(g|I@$+`Z zz4s6Hl*?lxx91Khx{8Xj)u?n)fCx&;lXxQyQaRxgqsB2l+H2-ouP-f#gG{D_te!fXcI8;_KQkZ2rO}bc?zHdpoBUoF`{NE`J8dQZd78cp#>ayy@2|M{$jW_ z5pAz0WUp)*aT=e3$?6is+dTwd{5#LI>&$@gpBq3Z!iK)EYQ-{*GUnuv3)t_u1rygO z(q@S{IEg7v4qQGIg64qn~gNEKa$-v*_!m5)nmTu7aWOGATQJT zATg9cC*&N1JK+sH>G;jKQqB!M_UMw6*HVe4(N^e~kwV@~?PEVaya^rW-U3sd#|sfk zgqn%Zt(@gOX=-*HYa1Da~*2%};@nnbh}| z1sMwpN6X(0xIJ$!J%059=x)iwh}Vk5vYXI!v(;pEL;)6@-^~8&Im9Z8 zcd#WNOW-Z^Lf#Z$_ZHsahs`*ND<;*zDi>86IT3*|JP6CfZt@R_XVaohy?8kI8#0$w zu+G88o{u5Ui6L$`y!kJ0-$dC^j%O)mKqfX_0F(1+#CEsQ*}*Uw zp3|b0L|r$WTwY;IFa6-Mi})D*YIAAToXI5M#X(?O1gO$f6}bJyf+NF=5aoR;Y}=oY zAS+%Ef~yV4fW9|*x51jcd~=Gw`%^C?T%L|Kx)$X2%rN%b(KG6Vzt)NFJ)%dvx)(5xcEU7v;Y1Si>ltGTskF#Wjh)z?0tO$0X?IW;gv>Ak zt?>qC=FV(JdEy4gUZyRRfm?iWwJiJmE@16GZjCRh3AtC@RZFV^xg0g zMG9uqY2!VRA(01Ug)~IZPsg`|xp3Xdo?hRU$lM*QhR~fN*u$M4F`Mg|&9mR2jFmne z5UgfGOlOiUqh3VP&X+ztodEBaSTSnpS3u({L+Tbg(gElF{LelonRA2XDB!pbIy+{f zaKUjnxNI#Wk`fBSYyjdXV(`Kd6&i4I1ZV4Pf~-AH@M@wb**m0%{U0;vxjP|b z=~4>_cDlwm)>Ol32TR&m7svPuC6IF%NWDOtj=C6HqG%)B-L+iFxfa74bGJ@V@7w`oHG+?lD9HX zFSH6(X4sKo`v;J~f54aWv?ekeYgoyJRiIl~&v8KausSZ1M6dn=S{;+*oa-id?qn+U z+G3AW&n;zFhouq_NxoSFIUyE@WTwcnq5=)K2fS8QnRn+>4mFrSLK>(Drl zUiKH*lTfaU(C*{S`shy}$`&GIRQ?8pAKeI#>k8m&LMi+>>djb``_k1i{W!l)h_~tj z=cGJ*hd*5Q8)m9(1?BMuEbX3%=907MCm&0)B>fv5sA7tfIqDXVY zO4`hdfYFCByg%+tlg2}6Fe6POMGwI3UC!3tKc|rV=zXBDa6Ro!xq@a_bJ_RCwT#*h zJF@me9jqUkLL2_*Q&pks?2$c>K<-=_uBeoN<1RP(kB%;a|-c6_X3-h7`)R|Uz?Sq_5a&t**-GOY!F zwr_*Jxm6&Ss7yU?w&BkS2F$S>OS&NN8S{h7?I+(U2AK_iv9CXt$WCj8!Q(SYZnZV} zbxMLw4ne!I!nP-Ga3pSR#_ z^hEkT(43|eE@u0joM?XEF^C+!f$j%VsIKk~x@P1Ae&#Y@3nT={*v0GUkgyN8r9h8Ww+(@Y;J`dGt;;GRg1?c@M+1g~I9couVcu_z5^ zmC1#Jaj5mE3WF_6;b!qXdfGReeh_@ap6&>sDl$Gi52J7_S!GTN!N^`@uvT1jONpQ0QnJ}EsJexRxa;`d% z{fl$x_ROW~&;4k5VFsqmxd4lYlc>SIX-PVAH4vAFnziT*c zUo^TlPXgWG43c%8_gy&X%4@K=SwLRXE#!i{-37wfc$pQTf zXz#X){5(5|E@58u;68V{EBzfvQ(JOZxf2tr5^<(pDM*;dz_UMh@ulK&5-p*O>5C{M z&^HKM?ep=0^(QQMUq{O~8qg_oCPH(h2W^;`iTOuQvwEQ|ypbph4}&Jb&5Ueve7-c8 ztk_DcZ<&#wVge+gR)cO&b=A zeYarAsyMpko)%60I*gs|A=J5UA>=P~A_w0n;J25-5O9F&>HJN=C9ihikm+u+!FU{0 z9AxQ=!L2CQ8bJDd)8LJ~4B3^j4I~V#X=j23{VdJ#(#~z+dq1hb2*F0C>)LMW!Oc~s zNF7F132p{4-v`SQLa;P`3kfO-hifK%tZivE4)iX-^k08*@4myJYbZwC*J+T0Y!{>b zRe?4+^zwDL9O8@rDgv2Bsmz0nFp&Omh3Deri;Z;*ksJ%4^BP<5LVh%(^YjJlyt)+U zh62PcpGAhRTxH&^T0?%GFTt~ix5BErQW*Zej6C9Pr+-G=soaHQ?9RQypnI>bb=c1i z)wNW}{>u4GbQ<;z^qS@fUtjZL?dG* zgldR_v0oL-=&m9I+ryZ^sulcIrONbV<1F$p{Q&!YI?J=vmZsv$wp4xXAmnaCXi#dmPLmT+EUl)Lx*I!_~LSSvS1v77!Dy_e92;Z+UrK>)l0;R_s@2rfw z!-WUo=eq)A&nx7cOp<1|ON+viE7gq8(om`;_a1%*)j{?@Y20y8l%`pJcb)u&*V-)Fd2WyeO@ZlVT1KeN5xBcb=g5ZZpsVh4Sb zVDRr1%&^J8e^&l9tkIj8iUE6CO@O33-b4d4b)sT+3GK^L;fP5)URfB+A1PFT#v6pS zX)wSq@{?&`nGm^f`ysA)<^Ts9j^Sg~oA5bMj#@75ocxI4Kmuk#bv_ zP;!F(*EE5D!dHp1|4K0;JQb%hmL&VE2*-^MBeoBoz`;dQP&YmZy6&bpqpus@%w*9% zSct|xUC7_IsRdrllgG1Z-|&xsDX#K)$`CO!4!%#nbcYRHu z8(ULgDD5;X{Ur%M0}bevW%W=#@h|S)6;196tC7j;ikP}NiR8>t1#a}Vg`9hqN-p`e zV|Pm+Ya*Bj|7ID{uO9-i>Vz@V3bL zowX;H{#rDRR!lp@2xhp-VMyyMlt#Z4>=(}%YUMLGN`UMdfX(mcVuBFg5it429$25*<`2+jobZB5<2`KH) zVK!7BfCJ_$V40f{%?a^CiBFoeZ|irMw{j)7!*rybA{~rr#zUCVl}5DF`D}2PFL=C- zqK%xt!s)gt`Ix+b7c1m}%}L@=zF&v_o->&)o>YKIt#VLObP)b>tgF}!N1=C{2+}$B(~$4?n+c0|HRI{pF`Jh?kLB>Jmej{h928=>H60Q z{MY4h?GM5K0<_6Czb+gfwkHe5m%y%(SvVec2P4b^=)Wy9$nhsfVaPLzrVsSM!VCv` z;NEe3dwr07C%G5owq;SNgMT1?-5|CWA`~n&qbXPFAo6=8O^FmGHv{xZys9Y<7g`b% zF?~jDyco~*?E(4lM{vseGy6HH5d%11+|lTXq@uxu6fe)Gm)2?0aN#`c?X{(LY68?P zWG>i=bz@abDIT_%2W=t0P_g_7V{0Z&_S8OQrC8v6NNZ_-mwt(x2Z%Fqw z!)Prxk6rYE*_-_if2EwnK5qALJ==&bJSa+NVHy=Ve;ht99m9{k_4xAAPjvBKO6Dj` zCKDIA(&bxTzHXfiGFNqd)mkvwLyL^B+(>S&o>R zSHSG+Jd&hVgyL%PGM`t%7B%p`2qU@I!9i;o zypcHwj&{?*hdWcgm`c;Ti~r!~*LOLt?HA13u#)|x*$eGOV$k+D8=Z&RVegsUs3XOl zb0d3bujC}kzbOsw9Q(pgQIdM8RI^d6BTf74LW{+kIj@i`QQlDwJ(UTpPL35GTs8s& zlYFrC#Ziu{zKkmW5Fup}CgjY$Fx0aVz`H^$+T2p4zaIPJnj@zKpexa*olTCQwh=dC&c z!7h%dx^p+QM3@rhx;BmSTZx9(S{WM@AnLn1z}U4N_Z?Rv_FPwEfa&{>viUtAmNWQ6KYi9``Km z31E&4KSJGQ+eqB$cbHU>%#IrG04*?rV|^R(`wCZJS8@4#mH(hX|1dC*yg=kcDAtMv zvLSQ(*<4vy@`5i)&9@km!H(I?aqhkKsdzj3bIjJ2H$7nU>KcfYvY~%ZAbUS34;Q#P zll8`$#79zq#@(2Uf8V!a(~?jko;tu-PS}H;M?$Fg^8mJTe;g4C6(jnLI2N|r^UAMg zQKwDIP{j2D^K|2O+9H+4yl=>-!?z0gF)N(N;Bh^?Z*(5MoYJ7`5{5*&Cy(9ubprc4 zOptnriPQNaIaENf81<*>Qmq4L5xw6qZv6uExwZtbA#Eteaj3642C#<$oS3D{@CB$0bBH*Y(`_A6IlftE4ICf1;4p(GqkzXz8VKH)m4i-1mcqggjM;HLAd zp)>9#>&@@v9iK9XCn-I~9{76}kIN|F;ix=h=EY-m(-c&=ZbL7>m`dIy-i9h;Va{VA zPQ>a@qwJZlaJX|P1my(5;+Yz>^6(_8Sd|KXfeJMH^$7^IK8|Z4mHnP@hW9e&qi`}qh(6*K!IBmYGAw=ua+l7c zjLB|tmt%Vfa6I)_FE(NN)3Ua}Pb)~z++^mP*(2B`_8+e%!JBOQT>)dy)2Z`I6SSDU z0ctb(Xke|54K>|NQD+dC+|x$yL}wN2Sr9-SpyjhqA@g) zoZo#NygRDdN|kV^^}7ul7Q8qv&H z&e_jCu3QW6p3eoB@4jGO;g3RW3LfJ)o$I89nUP=DSm{s^#^>n-vNpns6q|%ndsRd7 za9#(7SH;7>43_^O;wN(UL%MB#Cj94c3tSE_!sv*%%$v=&^e#6eY6>m6lM zcSC`88i^Cle{s~+@ebz*4WUI*e5{Z6M(O8w@apkY-tvlGBv;hfI2?M}DsTt!oV2EdMoO7us;85n3v$7wHANzoGSJ)trii^_UycrU^8V{}31E zFQ=XrOL5hw9NKbIj!5<-QASmcCdTsUSZFT2w>p8@InA7gItY;56S{Ed!wGoA|IFUH z)Cikj+mYEO$?WfSIcRe19B5Q*hhJrV&?{2JMxAM8p2_Ajw&QZ_9Cid;0^-P`b;_)& zyD~lOphQCj=0Kc!8$?e$#yc=GhTcCnkNp}khzajcFsC1J&r+Q}Q8ARJZ;cM&wvs~5 zHS`w#Q6C-+6ab6aKG+zXOfIVmlJ1cpq9LsTyB3^cM`Uk9R{-buW3Q#)7^xs7Oi zpF*FsN6!#FIVcF4HMDBGMC}quGp3OOIlj6(1Gf zM1Y}O5q5iVKGN1C`f8mX$9}iM=Q%dS=+P(`^s(r;!IOmlTZ$tR9?V#5ALNdjqN~<0 zPUmK)ue;0UEX0f%*hbLH_m4?Ur(kBcX3Y229Bxfb)Q#% z-vZX>>M%Byi^#(dTM^!Lu)i(zal-~ru2W-72HH$%!^{Y@TjImWSzObiPt~n+iM-N62vyM{Sq~;sT?Ie1t=)t1FS5w!)rWX&^I0HQey}@d zaZZLYH9l8qWA=1!A&+XlGTAz_sq5Zdyb$Y$@M3QhgwOD&zW%_{sM^nnza_QHb5ueRe0@E)*pQiR~}O)+9z zWWjbnc*sgCcH(lm|3L5E7myHo#AOXQuBZEM*yWYPM(K~DQ*RU$aD9ZMGIh-JJUu$E zN06{NskrfD4s4b;r=Kt8VzqQGw#@U#rn+2M+?E84f?%jhWuIWJtIYbhaqKwt5%Zoa9DkuMLEPwn#ju(95(uFbBI@GkS8dHJuw2 zi@^@?^s{R^!kFyxNcpE#bN__~92kShI}Q3`wEdNI04-jH9pEDm0J3+9p3D zOtuUJ5chy-^ydk2V(4N58zovf*2ZF5J2K4JmjuGm4eRK?<{;ADZ$`@c*Tc@>Ag1N2 zH{$m#Ae6Y7&#PKP>UU-muQ}>u?(BJZXO}I`{b5G--0tS@;3r^*Q3mr!TNST!+1Q@v z&gig5sEv2$6FmD82~Tf)VwL9h@jeB#ENFp!=C`2Xw-+4PN@4mO zTROF4AyIv)M@JuN;NrkPQ1VBM4p2M#Z-pP>-w)${lkPzD><}tcyB{m{OzFn7$NXo1 z4}Fi)u{RBmbT^v1B^^pEyvdy z1XT~9?;d9||Mj{9UHgGKBm03-x++JcKB;jI^p&vrvN{e)TQIBTjL78f42UVa4C{?F zaGI$Y4ZAl1#w$9pr*D7>_?SiZEV>5WqQI_`kD||CaV#&TdXTHXi|cG3;Q13*(7wcz zHo0q4(_fX`|7;3T-z6oP&Af> zd$w!PU5V4l%`gdiJpV2ldkn+s<^>RQJg?2@?;RBN)qz`@4|6p3`f>sXYlei|gvn;`TdoIUKXg{xmw!@BB|T*YMQJ8km=L4&G1e0oRm7 z(xq_{zfCp6c+W%NeYqW%WO!58OnLHq?E<_jDoERY?}xw3BEd6H2YKmlv4_iZmv>ac z^!`GaTp@;mgMKLbyBMD<#(xSbjQ+O8zRMG0r8aiGxomN~GpkLN~!4r3$!`_T~Xfzg} z77Kbn=z|zJx$h^(6!N8hE6u5dSOVGmb1{DYZcpP53q$SeMm+Fn0%=+_8B+vHu|ltt zk^B|~QVMERs8ltIwQKcTAVw^}PbdkjV&^2obq@im#<(aQ3vzcx+iGuYc=KdfL$u7VTFD zk-&>=#|lS6AB{2ZJGoBXk4~QSc~d&DN1pTwPNjvv^YFk&hFqRKmAtCrSaaVyU{1wm z{>142P-O2}cyq54<9MO8%Xombc3()pNBjo)`AYP?WExBw_<_~O3B8nX2~T9HL-cJf z!`G9B`$j3F%(=8ZZAY2RuPdkz)6SnPHV0PNE0a^V1ni7jm` zRi~wM+R;J4ghmuv5XblwVk|KZCqDm&4(%c^YknXKq+e%e-HQZ~s8>*T=`F_&NC$kJ zNF^M8f$%Q;>}2(Zyz zczMD`sGK$n$=(B8$PRqwF93-e%cX-v+=5*Dzhw$LC z4*et`2Y17gXhgR(sh7Nj1+6Ch3DxaPj{X!n*_R=~>(kg9)dg(Nbry9wFH7l@#V}l` zL8hPOb~$<(xQJ=T8vk5Ib7=gd4VQNuu-)z=q;qaOjaMJ#RRP z2v;kU=p(=QYbEZpDtE-r$>rU|-M2jWhNiXb^@_uAcl&IVTFzyu6y3d3X}(muf>hT}9fa zsLtH@n~9#Q))E6%8&YbOOsA~efCm2g9QP=NH@wY-PMO$f{pj}xcK1`R$FkXlKJmEC z@8q~VhUT6$I7^kji4~_Gxt$$PR)*A!NVfUwJwR8b&yeXXL^6Zlf@EDYyqo<8T6n{( z)+2oeo=U;09z8m8;y!qBtb^K!RF=klM(-XAqI&Q!u3R|>#x{=d|90MC{`&+tG*-aa zsrgc6gHe8Sl@~d?N`Z#!l)xEFYEGdRJ0@Y(;e%-T@CZ}0 zcQfk09YA}B^)xkLKMwzU1qzMPkoamkZ5r>yymd*~w{i`Q@Xvr6Sy{a9ydUfGbx}cc z6MkByPaXPX=sPZV|7p-2Z_Q1Ikb(E8(*Dl+=J6ghdqp7cVGn<$?R~HlT>-g_6I2^C zV0u<{XY`megs)Ct1erC# z#CD1xkv$_sPP=*%qZ;VTYJK4?MGnlAvFJMie z8!4YlV1lJ7$%#%ve-#aKyv>3PTiKGr&Pi0kGZ{w&Z?gKQN8!E5W_I$Sf zA?9s$+u|%aB4eEhvja?uhyASVq!e<*&H%ekR>7O447pz>NmoU7v7Za4 z(5UG)WNWz_n<#pmm(+X*Z=D~&|8_9MMLLLH8j?Vh;&x`oE)UQMl_VR;YFhJ9RSJcoWfBC1X46+xPCBNlczhS9hQ>9l&)bN0XFCN{>& zpPJ2V1ey0WQ0CSSj%k@pzUEpw(&PdaHzv?lX;a!KR{`1Wj=bXBdbE91hQ$iaXjLyn zC1+}p$kV=bTER_F3)_VkG#uHnbVF)gPv~wB8KQ1AodyaoBH^8j@$Lg#TI}mV1q-z4 z7x|+w+EmNNJyIu1-1~^jL@iw2^M^SXwv)}8{{aK1B1(2--~-~wQh2$`lPEkp2lH3o4&GK)nnF>bX?}N|KJU$)|6FWwkqf z)W_iZ^WjwcjW5;o+=lInsr1E}OYGHG{jjNa3iGw5gzu0v3MVuy$bTD^V77)ad-9<+ z$x1uOTpVKP7;^z;CNE@8HSNTf|DLzyG{3A0i10)pM2Vu|K@So!J=M74FH zag+ldOZ~;@AIb&0mqRe;jw}<~SHo%@8OPuAs`y4fMxgb!2J6sj%mj%#kdEoYaO>d# zkh3$Txlso(R#1k_=WCEi89}&Z8b^J4IN!j15uV0NaWb-$(~Av6p!xmvgtOT>sXF?EW#tSc>+ov^PnBZ`Vcv$h;dQ3(Dy?zJTDd|8&B;du7dLH&yNeJk3l`u zr*>oa^dh#RESsr#Z$M`6T>%Rws?+z$8kkzaW%RlI^v}5u@VnAl7<{Hgm;RfJoAs74 zY{pg^X8Yzc?)!BEN^@%1y!*oR&GtI%%3~qbDixOAUQVZ* zx)bfk%i!0RXN>h@j;nji90V>8foZlW&9;7ovBCMgMVW^5oBdq+xYU&tK}ny56sq+iW!UHeOnlfkE^zBD9^{m ztrO^hMjjcy(F96P=d7QHe88orI2ZN~FT8%|B|9m^9G94{XCre~z~6B(ta~FyDuqOG zry%Ej^m_#*3Op=g+{=m$-3Dn&4KB8DCHCW}3hJ%i5OHp zQ=K`BD9XNpYngv>cux?WEZ|BF_76b!!9&d653VHP>~{K3O%j#Ld-*er^H8V8h~8ie z!D%p)`@Ou3Tjn=$S-@acGR+ervIALBrS0tY5Lx`|)QAc{L}|`~n{ZyI04;mg0DpEd zoXFaBF6xU7Mtzl~Qrf!oky98x zc2bWzTW;w}GC8)*9@)*HtLco#>dvFO@_(TCE|@aSmNc&ZB{=Kn^6qhdD%t1eq$!ch zd(E9iwqE}S8E@_BqFwvYa`_~BLi;ltyrzxa_xcaKFH@z)0m_75+RH>I2IF&+ejU^=S72_u+fDoTTF}{Ct|+T3g??Qu zh(T2CfvNM{E9}tn3=DYtskkti|c|NR9#X?I&EnAWi2erBRE1YaE$*5&bqiLzmcS zHpyR;j48|~IXo@gFXaatmaeANrxeI)vpLWdHwzPIsKb8q8a95r3i;Swi0nFhGQZ0;aXt#lh=Z^w}w(xO)-(wE`ueQl1=wF(R$D!FE6ga z4<`BWy-JNnsN2Kudd>;<_6Yy&G(}v#T>}fBsX4|(*Ct` zD4qQh7fZ4{`AADj%3aCm^gA#&jdN%{odf;BD@p6z9{5)zOEzoGAv#*2II=$y7{xv3 zy3N#hB9E`JMly~3(V5=7{AKyXJ!mCtS^6LR6%!`;p(X71!5EZ#y@iG@Ph^v{&tkf@ zG)+q{0w1Lp%p?gBq9q?qZF}#dOMe1JPqL!5LDO+pJa-q|Qi!->7MpisF0VsmFVy@Q>e_z0`w4bfUyJF zc-(Uu9c=G~XxB29|Gf!ztBVjtFHijSNC>??4@3NbHuGNf1=gw_O*+PY|GpG~$xsH0!-&|OrAAq9M{6Ob{JMP`!P6wo$G2$S1cPCHb%ZDrQdTKnZ z-FT6o^d}YkHAV6JA476ostkM27z5ud6AHDzumf=m$iTe`STT7VE^S&$z4nBl_2E+Z zu;vO*$vwh$NuI>lPVzJ&mj+@u^v#*^W!$NB%tTw1+XPvWUkENc%H#`HL z)iOnL_=OQk{Q3>mT-Nc+P3JQw{yKw$`b;ckXVEV)O?Wbob7H%CVEBQlFtElBr*65< zu6DGbKScFOhRtN&g?S$t%Z}f8NI{*}6#LURiHg*ya044RB28Y6-DiKzTmZB^h&3EC zA=Wdvz0N>_wUfUd)+kMZk6Zskd-gkT<%<@M+rEqJKl_p?kdUC7<=f%A&TU*)+=s_L ze1@5;Z{hFP6_DjP5u(EfKvsS=vt(NhL=1H?mR90)qF5I!*;5bNj3k~>I}d3~(&+rs za`c*g1S4H1lW8GI%(JI@*phLV&Ajy$`$Y(mx8l*CpiBE+Dbd||d#DQsu>5^Q98?w$ zu*#X$*k-qc&h0+~6Z7?OUA7yewCN=8c()sl)z6@fn|IUCf0O88gBNIEYRI(9o1<@> z8SVFqAytDhCwX5=^(OpN_!H?SH@}hSfmz)y_K5b<;`As1%OZ{W8SX@oUT8ZQbl(*K3x(JO#&Z z-baPce;8|_jm+DS6(I7d*E*>|gxC#;z(R>%Y~e}wHp4}tbjGv>+;V;ry_+(QKQFyR zyHYc>QC~(CA0jGyN|WqMJ7HJe8;I2oBV=L;Ce7Tye#tea9+h8l^Qm{xa;pqZrPFg) zU$&#;%{$D-B?sq_A!F?+aY*N?kzO2cBEpB0G%*sOTLJi(ErN)Aol8Z+?S(8>h8#peLnk{HY;`7 zVc(AIe0g*}m)Dj}6);#QkvvrT19$)JhVd8!qNNuKE5xli&$S_4x?msNvF98{93SFQ zvKk%amf1r!9n2$#^+fpSbn;v61T1W2+q8rF`OEE(urnT1gW{|pCg?*X-l>zMw`%sY zv(0nbZo~@_;ZRLf^5vKnmj!5XKqXquXu#Z~#W32L4qv8k=HH9x2RyAn+Zvx+^`#~9 zuX{Ygi(^ihr5J{HHfhuUL}f|vQVS}oGNY|_#yEPt8^GIA_K^1EChCc+@_o5^Y^A$A z>6hz=iw|?a=W8j79&ZKXZHqWQju*1E0=VSeZt$0VzzCajInU8+u-pu&%)dp{XUc0( z(9;4Lw-mNj{60jkG9#W-=Mk@cn?c4flZL(a=8x4J!}>BC>poXSkj>4<@2@+V?Gb}O ze;49koeB6iVm7@iWlaipoPe(_-tab33(N$2nYO@qsbg_*hU3s@W! zXM_e0~w)YiB~U>mj@sbO4)EuCnee z+Ax!zfH}d!bd7g9Rt-v!L)BF{-?|$j9%nGF)e0ozxDD<*qsz=YKO0r^bpSOVi?P$j+ z$YXh5M(OWPekyl==oEawCe>YmTEPf3{Om{lvc0MB5)ZIyl%?r$6R3y$T6R-q0(tp- z2CSJNjbR(3>50g*aQ1}-74V+Hwe28R5HZ zb!VN|jliPDn^n%+N-Ye;)(4+=#?6X~AMBhpf7N~mav zgvu&1N;^@gXoznpDY6RP^Ld0+Dx+wTl!$~fQ$~L0_a}5a_nznZeBSTZ>qNq2F!w@6 zW96U}EWAoV&6UMG`kmkrF7x( zm1wf=(KVdCNfM@2?PlgiaZFbi2T(q-jO_M)#cqsHr1Awmq(JExW<85!|1Qxd{h5Ab z;Ls1oUg09f$w;IArCs>9DxZJ-dlwXVG~&7~3u(9GLDbtEi%a$AgSA^JXhi>lnK9QH z0+z(eR2j!5nwjLS%Yk3C3coqEW4XmTTD_HHM_pmSJ30ivemn}f2ffIWrb1RGEP))U z5uuqa)8Xue(`>WYS1{5VWYczUr{*5B;Khsibc1~{^lnV#wcOqhUmwNM&W{c3Sa&pZ zn`)5UA4()UR~F8yjq_fb=AgmzVD{cIZCdO*0~`y_@oRrB#nf;Auq*v8*k4wsKg@v& zn@u7cQpFdQz8F;STK?63-QL*H!yHM1G;7v!+OnpJSJShNVq$aDW~tV zl}T%Gr;#d6Ql3V6dsdKb^Mk3*#rsfR)`K0_OBlA)5^i2Q!1>boIDRFCD5knXO^X%l zXQ)Nq?{9~Fv7-2-s{*fbnKt*|JDKKPa`fG|5IS^w2`u+6MUPYwSWx`|I>S2{vqd7f z_Z%PFoY$d5L_BMLitCX+qwx8BB5$777c{>g1TQTNh}LssSd%Wsw)a?J#y}dKws;9W zMz5m7HF^3c*^YRG9fL1_e!yvgeEgHK8o3D(jV)P7rG>+pDk%c9Vgj&tW*xS4?;w-z zyu^7jPQ2&n>iBNF4S{FNE`z z8%fl{1YnhSp`o53xp;AaUHkMs*v*U~r{=2i#ZnK`{T)J_v#*WoOZ~zlnX+`N$vVjU zVhB+Im)Vf#ocqG56(>9&XGa|;(zAWsY8oYfk9NXDvUV;_GReY6T)v&z!u9u-sv~;wvN}_$d%AXM!e&_(~`FW7}vHKqeWG;Z? zg_~G~uOAtSAtSiCHG#fsx`rlBhp}l$oXp610UgEtRp3)kNMTnAyWy7%op<#xakIB0 z)eeUd_j3KO(XS}ixgJs#ZiCs3Jsew=&$M;g6X(i_q^UNS7xXa+EF-MRJo`NMTeTqW z`OlJ;>Y9V+ttQKnFmrZN=mhL^ae>FKhp|tg7@WF(!nF2(ps5)HOZOWR*Xwbt{f7-C zV>BB+cgDh`-QD=ZPK6n~l+3i{ zQ%dnw*bJiZy9ds28O`X1*X$0j9MEg6g#s?C`@2+vvFZEEd~~19Thnw0-eijrlafjJ z_+vFb>HovX8Sf_wL17@3p+ff`D1i*q)i@)=id0P7LDw}uK!vx0G;Ovi{aUZg8=E|b zr2OQZA=y^+b*KVS`}_!c_Q>Nd@l}lM$5Pv0}8V{;c@>G zw$VO}ZaBIRjNDd|x0jq~j$9O+=}BYmek=iRXAQdczCBcTS)i5DL%98Z7K!3`UprQ+ zkvQ!)V7Jx>i$^4?T!hw=)9-a@wyHH(fOdsFI66m$+@JE3-RAB_1PZ0-=T=b~SE`D&i>I(Q5qjiz_(~`+ znM@KoUNL=tXR)T8w(PXeYe}Sl1!E8}2MT&$FmE?15{FbLMnSU=;n-hhg_$uv*?11K zcKSiX)3wyzKLx%#&;-5v^C0!<6*g^^A9?#X5a#VX0~=qTXIj)Yvu+*Lc<=oi*xytO z#TLu)faFc=S*=U8Q;qRa%nF`Fs}M>1qEDY5-hsKd)To)L5NM@w-nKyr&{pBxNvEIS zt(FE9W=9ySe|?xA)eMmv#pzS8EEwzM<`)Kvv}k21|Jt2C@YYVpn=}KjK0gVfOUm(| zVLEE(<$|lGIt+8&DOJZLs$T6y?wIPbyUr{Eb8bI3HnalYeVa(;tk$AiJAQyL*Y8mM zHp*0(wo;`6k8Ds+@2`FUZCo4-Bsy+jcE;+Fi*Y)D-`?}n2 zU4ax5Kbkw)9}3g&ICg~ou`lbWu{+U5iI41B91%{WN(#-Un4`;3xts+l1 zN>GuC8T9o9mRbKx0V`^^(hfYpx(g0M(29Lz&B`*kBD|5b99_!}&mLiF-wfisE*Y|X zttYM%+DQHC$GI7)5*Zux#M`%n==qgKv}wHy`pRfg+pkuP(qIk9?9are>t}I=b0!*` z(k99$>L7iB6urJBgs9b=!Iq;Zv6)DKpZ+^WuU3Q1=t9m(lFn4INU)V)EMd?=10p$m-2v!XI)iSTv2xXgmK)JpL`{LX;WY0gCdQ5!6C z$>Ql<=X@|jm+;J09nzLyNqsnfn(A5+X!@>B78EX`M{@q4y)}0|U`$ZqeFkGAdj(oL z1Ht{7He3w6iofpY(tOSba_hlzjI`Uw)O=PXH96*F`nd^gq_R12@8Zt>?V}i_If@#& z+sXSq6Pco`FYr#50hA5bf^h3HGQH^@dpB?^4G_-6-ViY=V6Dp}>kKo}YHm39f;P`k zE(<@#t)u}}u{6fA2lelth3LbDIL5?La|<){u+syP-}mwH3PQGXXAKkk3fw<-kS+dU zN*Cb)OsY%)5tB69t$GcXIX&T@ingcfAsmao)|uKU41==s5&WR8MfI=i(HHgi@u(cf z#Le|biM842_-8lA?v`O*iHWf-0nWI+U7TjQzrpW2&oW**ml!d+0*)`9PJ-Wk$H2zL z=aimo`e9-v!8KOO!ne6+0 zOu=>rYA2boYX|=0=nik0#LW4m?d)Wty5|PW(N(94j63;jW5GUnIKZOZFIM<=99?!| zB`>S7gm-@ED|p1Z;tm6IYHMVSQ$pfMXWw^FwoHJAb=*vL=sBEO#dVr?c#^QRQp;q$ zI9j*Y5GHt^hmEe5imUcxhp{X5<#?VK9~JQBT=mF**H_}H&bh>P zLq8NW?gDF>XE-zQKKyh*_ES2+^P?-6+LwrzsV>C4KgX1+hZ37?B|2f-PU4r9b`2Fcr?$8=iYpX)G zu8KyXWJlOHFoz0We+jBHrm+wrLau-V75rBU;-W7h_JbCRS2~02b_KGEyR-E+?!`cv zYWS`{nH<#n3QM9La6H%-dNxm{-Al*eaf3Q+nYWtAH!mPXpKrr2VG|-K8cmn$hVzG? z3oz=r+vv%`cKA|mL;8Y4@cwOA^89Qo9_QFbiL2G=;)4P7-TJ*mf8sg3KCVy8_vf)R z;{+?ZcRe{8nFd3qL+l~B9eB2UF?CU_2FtbyRP)*@IQpXm7hD=*{^_^j#VT3Wl-QA> z@i1bX^psJ2DaY|CS76AUz3A6-0VR??urAyVVwI2>xt4hdQ_n_`O_g$_PsbJF!c5_k zR1B#QP#`MEqxa&jqGoFfNR3&LA^9j;|0f1K0zElD+E!ezM&K^zz;bf`%v5fdfy%)D z@b&2-xE0Oygcgl~zpEM9U33qdUi7o)-mimGpWninqa0JV`6ARTb0T+l&jpX88<>TU zPhd~47Zdhl1W)a$VOy@4V7v7MG{37$ZF5ygv!56vnehltXjfqWj57z_dnCec}s9jK}C7YI`2Bh*ZU z#MTs|AN>y#WW>2Mxfzw|`2w%QMxptuJ5?Op2|12eVA9YY=9zpfJmfM(>N}2N7V6XK z6218RGY>xP&VvcI((rc87_WH>_gTaQ(B-xbD6#4evwe9H+x;LLhSCkF+F&lu-rR|< z?P;up(JSmw(WV=AI%09%0BjUtsMo_b&|Nl}*q8l8i5M5!0yQeO z^n2-BhQInEIC*lc{kOU#ExZgm@*L@WuG8)wa}>*d>5}C-bvOGjf9JCHP27(70juR9 zKzb_+fUhjc${lc`Usm`My#)hMHCYOljtGO-_Y$bSH5(0<{ls)pFZO#*2kK|9Am9E4 z;OEL-yx(vRB3*N#cYi!SyH>?MY{-J>Td&cY`~9+;t4XSzITlvZ3fgM+375wN(RIh$ z(LJ=3nX!Bq?-Hzq!nWBskiU@;l7ER;_GZJ)@D0=?oSP5F<-oeJRGi#Cf=>rmQZuay zR9NN>G|6p3?<7$=Zpt~5GTZPQ_k9&j{f)BZKE@6npvM&@F|h*3nSn2id&qQhKB^JS zy3A>=p%}i3ktf6dy<%n8x3a!Zf_V>X)7ZxA{SXoHm?a#i=Dpor*!@PIwp`F8Tie!< zw^bTAQr*ogQQJr8G&>ak7XI@LqvH#+T#ZTmc|($x$#L1; zbxE>%JPd?C!ko^>Y|hhH49~L-XO-;76ka+eE%ky=AJwT_vMFJo`{MJ)x4fz>QJSZ^ z9d%27G2KDs@Muc~Yp$h_t+DOMNEgBr4Q^VU@;daJw5pzq$`&tk>V4N`ObPAxw+P?yWFf>ro~5w(8K%f-_8bwh;}iSVV7{RxmQ&dgP_T+{!%ZQEkrEeK!tc z=S^j*BRiRmZIWbbSu8zzUys^-3(1rhC$Xc7;TvW_2 z+I0gqTV2Pp1Mws&lFLu7dWq*1*CFqbHN>ijQfJR=cw)Lg<9c}#wqL77<%(NuYGxKb zQ;#RZc!)em-$0UAZQxjUHaK@Lmn%IaK|VqnUwZR1CRJS**QLe4vA$q7?Oqh{^PaHB zk6eO}^ZsIC>MmMouf%_sW=_Rk_CkE4{>>eA8T7{mV>0!D4R874Y<%+k2rP8Y!y^U5 z%p{{jASoD02AHp?AF~*O>%7R}y@jmKKOWWn@ElbiYiP}DyK<~Ya z;c`yBcg}yXt z$B~9l7{&D>PRO`$9h}XW@yw8%)m0+TN>iD%-wp7}Ya1LjItl(0#AzlsgVRe;r_L|* z=xK#cypp#Bat55q>rH>zfuJMcAK1hCaJ$mAZ}MQ??^_spEEasEjxf7^-NWAPlH^QB z3QW1I&iN5#souvx81enX9{jvp|sc$F+(X%JOwK|cv>Q*Ma(lnrJ;*jxE`4T7PCgMyM*K7xR=6){8ZT7`G8q+_&S82 z-%9$q`ChYnFS9H=7AsJhdo#Ir==PWV znDP7UmAMZ1-SR$`KkH=$-IEx%NmrO(uase#59cbostg#o2^S?E#1&0XVO;+|wk0GR z6pdx&+T0z}xhfgjg-8R$N)4`xY-)8jMTJ2{sb{vkr|8z_>|LnDm+UK3*B zQi6jA@1Rz7I>e1>Gxv;&SZOZ1RK?AKr9Q?ou0A{u6#=cB`kJ<*s-nR$}%{L)w0v0)K}METQ|(TAah5Ec>}Ikav%1D8sYZH)pSvAI{s8#12S!hlMb(^e?Q8T zRj$c6;Ta#am!)#|&LcRS_ltk3uaVLGu7|CP`tXIykXKXF@B~kt^c!8p>8jfxs^$;8 z_%Zhqza513|K<~g zm=5R*Qe=K6tc2(70U$0FfwgW1q`K`7t0OZ97X?exx@n?B);5T_-4RVn3XDkY!=$_0q3!?UW`;;<+?-} zThVKAEzG&|9m-a_Vd2qUzT4ajU>uyrG~CL77WoF|PU`{~?iz(&i#q;UdjYaT{xzKA zUaOz~69ok4he(vfa@S`PuPVx;BfeeAsR79yj|k&9*0%fq3>R7HpQ)c?nJ zUzkSLt1p3XHBVt;oHe|S{g1blIfh{h`QSTY2B{Y*<69pw!>*ha7@Dvbb0gECaQS{n z3u*-)b1^(xt4Vz=Wr%B>7(M+chE~jafn7_clAuj%iT;8pTHE3UKED@IiN=Yf;nQx` zxzUS!RE(orWX#wT!=0emB!fW`A)t}#1qJO=5VoX^p-}=Po!c)MMcIP%$P_$kJr{SZ z5TXBc789kS{p8smJxJJGz}Tr|aE`0180O%CM>eX_pT{}Y_+uTK^GJhQJ#E3rv+wbe ztrP_2rjbfDL$b<1l-rfd!@^f1@SrUhd)<;y$9EpFSd&JIee_9ecRwCA9cK%Zm*SRt z2Z+JKd!V$n5~;>wUb#dl4X^j4MTTcE`_gNE)v60v&Q~5=-G^}5(32ch64HKk_^-Kq~WO3D4Z$d`fFbf;j~Am z*!3D!5Y*8F-$W{4&gzFCmf}e_7ffQSWOjD2Sk)c}bAZWx{VfJ&%5?9&=ep0c6j1@tuqAV#AJ=bkU_?YJR+)w?k|K4V`s> zF3abhrSo!BHE@XIP1_N#!z-aWUYREE7(?HiSGnIhEzXl5K?;^;!#B05c-8h2TeifA z8JMaC;{gY7%{m9F|L6{el-`5X3(lleUW$4@(FPeERjSW?;(B}s*-{ln8d@WdU(K#T zA=mRPHPACv>cEg zvc;0U|1t9?9w0T%(&TQHIZ6Gk0l6zII1jiwqhgxG%70!+J%_&Xwtmp1 zbc!22QYlOot4d*Xp&gC7CIJe460~N#0_^8DvXODKam%tW;@NtMca9f9Bxan&zBzWZ zS?v+(aemU07yU4e*8m;tRh+~5DzDDF!1i24++kddTYviz|H)gpzV2M&e@crMHzeVL zHGPo2HijOvIuDw<$JmYa$@I3~Y8q^Fm%XB*NsP{IgmUk|e`NptYgTp^C(T?cfUBbD2$fkTS<{0ZjeaGVlXRN~ZtJ+r0h z$h+la-}*JEUR8mK`zY*=w^*wAX8^ZfHK9=khq2|eBCb1YMc+>@z{OXRK>2t#Z|Z3;xRBSvNaaL=Q{*;w zLW&`r;^|2nk~J;Hloc*`id zvX=DG(hjuoJ51X~JvnY+GX8ffhOV;S!R`7_viq;^g#l4M1YMX&9LJBd($Yz!!uuxl zdj7z=4#1!4GURS1<@=|I(w;yIQk~Jky1kwP>+d^~z8RIsjIAI~IB%oYL=%#+ryeff zInOw8`=OoR0;rpbC7Jop2irsQv8YUpo0s2DpDhVjGy9*JPEv~k%}_5 zmvPsznM9~O7;UD`pxyhnvp%dVIUyJip)FFB&DwnP)IS#3d@I_2^e*O)y@1^<8yL-m zYj|%%Bd+Lv4;iO@@YviYCRBkt$0-c4kpogBaOE{<3=AcgJyIdr>K9Bqr-tA5FQcvh zR-?iE5mff)x;bFjcI%MxfeL;RF}Ucokqiso!gq}v zbGy!%9`s+27Lz}*(p}&1(zNq%c{l`JE+#=w;%zw8?#>Ro%TZ&=0DN%Ro*pRp2v1|B ziLZ$<+bX}By0aG|Pn-JYMMV>ySC3w3_m8z8J;`l@z33p|`6g^_%sOd1kk& z>X&jnM11~%nPO!ST*4zKZvMg92J6xEX&f>9umXRkilLpNEw0hK2^F7T8(bFuY<*S0{bIK4wg#zpuZMh&Q>E}G8=omI zqUF}3%n^?<@XJ;w(~gJ3+QX6@uThZ65>zJfPBvtBsvm26VFf&2-i*@YjmS?=#EVIa zM5S8{56|g?U20L($=9A^@ZV$j+NQM7+>O8w57a*4MNPc2;dVg^yZ56hy=QEX_2X0N z^XYC!ZE^NT8d$-RU8RRpO_Vl^0`IrUn8m#OmB!f>$$Hf?1Gh}XZy$nhs2w#f>4 z+UHGZ#{Zs^?{Nn(Z7XuN(c2xKh4$W^Uqvtbndw+y2HA5WS+?=nsf znGI(e2H?k!WGdmQO^*49&_BoCGC!4`aO$Qe@OU|!_-F;tlO-xdjN8xAU0l|O(T4k7 z86@-B5iAM20^5fIz;>Mj=1(}t1z9TC&TtKsO;}8|yL{O6!-r5H`!{Uyy3HSy+DL3| zU13qGFcJLj#XYzAkXu&8f12(rxV~~y)tvf`yd``mb)nN?kS9gcmD~%jC?K+5~fW?tnPwAdj=MC$VLUC;9qQ zfebkrk_S8^9IlXNt9owZ=hs}O*r$uVHQvIMw9Fx~oi^ZZpiZs_Po&J?4Jh6xNd;3B zVQc1bHeM+b{xN?rO+%b^m0V#I-!3NxrAZ7>aBW^~LU(PfP{od%TfN!&Zmd&T7{a-T5WqVq`1xHQq8VL;>6 z-eLFd#jyQ-7u?^cMJJhXGuB&)c;xg8NNMYbO~Nrei$$N=!^@?}{t<#ruT}B%iWGXe zB7vk_(B*5(}uSI4hE$1zT>BnB6obNNZo)Cl9E<*I(c^z7?SrEoQYe8qIIkTo9 zm^3`AfaJm!khA5wOO7(WyjSRFvtATywfv0__dp;wp21UEb=8@_hN9?ztvTP166o>{5ROD`_k1P zTfsk^^Xs0S0&jMmVqN+t)1Kd9B#WCPPs#IvhK(CY0?A~}P42Vf-)7R#u@pvNv=R+X z7n7UIv{)I>N0#9_6Y#dB9t3$OlZ!PD%)D_6Vy-`l9C?&Zex`ddG3iAZYOan8Y_!?) z^3_c7lMoi z=Y!yzf&@=^sw6Qt$bblrT|F*P2c}=^(RS)|qO@)vig(-toz=hbxZ`Bf-%y5w;t^!9 zOM!G47|r0p7@R3W2{EL z=pSL&pUU*d`}st4&3Q1tV95F>s*$VbR`J)ww&179EI)66R)cL{h`k8~;X$wfEPX;8N2Ab!xPh?ZEbZRdD#^#sI4WUFTyL=9PFSG?85A^ZQFqFAE z+%tc=`b^rrrUDP{cV^6ulIf<*k8E%dw~IGFivMLl;&?@FL{Hg?AGQ29X!=#)y{g0P zeUYW~!(2@`lj%f72I6q;VHMJEu$(rhJjIAhD|jW`JT4^BoBCDhQl(jnH1o(x(k9`- z>^hhQx$D=XoPZ|Tbxnvw@fG>=GFsUsZKjyqeE?U9n-D4G+vpIulm2O#iLcc^Vc%&< zl-(JD!Ed6tv)eo>mX`rTY&vdH79?JdNtjqvkIzS%P~=cKYv$JoT58_tm^A}O_Fl!9 z^)pGB4?{iA{lJ})4>|VmUiwnjnA8aOdf|CO&^f}I}n%xL5G}pr|n>=I> zFW_Z~55rT55Tbb}1C|ao;}ou&5+vjRZ}%vW=l?B6t^O3&XrUXq+_iyu?&LssmM1Y zxsDk!Y{yqGt~1s93Q<7zCoJ7IfesdZWp_RDpf^Nh$%5?2 z=t~WA>}4V<>`TTQMVxE!zb(vbF)d#E-H({6Fb7;ybm8Ue7*<`GWx|rY$fB!vnB;3p zFjZ;^o`189>^i%HoxMq$rQJj7u4&OschMIbnUtfTW9)0k-izW>SQrt zkuG>}a2T#+HNayRMfThUakB2TH~E)Z05?uAAiyuc@0aJ$cfpA?zCV~qS1cyy^-57s z?jirEG-c?2%NSAJSS(p0LCqB6;RAP8QH;3+rso96t{=|mDG^Uq9nORErDQzC%_EDy zp1_8)TVbR4cP7kSopp0AK|hYK(w}3;=FZOJU9As8+r1+6bZs#^GE^`#Fk}}A>J-^}7RvuB>{0#;tPo^pNZejU@F=pz)1Ufp;f-FwnNOxIW!55l_9QVb5 zS}j-#p6KZ#%9eBTBt6Cwot`%NHP(T+^&5utTxrm0^Y=pj(IrzHGfX>(<4W)mzd9F?NWbUDf zbobY-*xFupeT~r?a^PketXbR)aXK+nWZrW6aUdVhcKcAlhpObWcPr!b`Y66BwILFF zW|H4rk85AD2&paiz`v$DaenGz61S1#qu;57A6xQPCNS=hF)<|kqPhNX;KN69yO#dv`@l9&V!=< zd<|X|(5BOV*$^e|ZX6JtjvHQ7u?^oGnOP=}nEAJYv8TrkqfC3)M*%0X{8U_({e@+8 zaAO=5T1`mUfDO*B@kY~V1-7y}d49Pd(AG`6VEV{iSg}_VR9B>c^^ciM-4Bj|nY9HY zZm*-(xiVz<;$z$sS_@dkBMm^=^zpf%ByCDJZrzP3^5*V3Kmc3lUdQsD*bG3yI|Qk@F%-oCKv*|l!AvoHV? zJFA(%6&8HK*DJ}32R^8KJQ)i3`Rw6sGVF`XvyeS?5;lEFB239ENRNLDZIUK*`GaOw zag7w|FixZAA`jEOUxnyNe>FU1I+fh1Nu?>;U0C@;oAkaFV7BCEK)cjja?sb6yt5Ie zYICw6{r6`4KC;X*pKnFws}*RX5itDeAJq1rBBB;eq01Op(th zE)twZ4j4J1_?6?}zt@AJh9(?%U_~Z!v(_!`GIV_yLqq?qp+{t95Vb-RvQ16^i$t7w zsty8lk6s(R-?tFNxBmpC8$oRQ=5d~#D4}UFkr?4C4~h{>>6sO_gc$yG4onp7of&l7~x73*c9DD-4}G!Mi@~B2+Ke zB69;ypjg`mMtW09)x6>;P@X%P5$tw=$VGjy_iZ|M?$N-z+trBo6bD*gp$z`ATn6uC zD)qfs4oNlNAS*VSUVQQgt#+Q|6={^>=3xt1v1cQBuxcCz=P%?R4ZRHHZ!WWS(Gz5| zet~)2C|Z5)fGK(M{k?LlvidJS>k)=-}xi|E3)?lkDz6dJSq6<#M?FWSMGm~aeRYn@Rz7ncI} zzVTp*%@k-b%f%;iJaM&xI$11jNM8T$!=T;;#8g2XhjU`dwy)8=dj*q7S<*!6t9%wi zEEW*2imU7{y-wVwnt{6?h|tO+AGR`fI?c)70FL8U9OqSq{B4|%@~gA3s5=A(Jj?NQ zj0j!y=pU1J%Z)xsX-3E5Wa{tRS+!HB0fOCv=&93|OmWgHrmobKBm`OD(^d{IJ0eVu zzrTmB?8OM%`ynJCe}gy(l4gmwx#k)?_H zOd97=FJ-pSh`1n13iOyTxkWVXw-Gt$B1oEwlSssMZnxtqfge((==;cdT;BK|JC}q~ z#mQSB-|$bB@h>6Lo~!}8_nhU~%ITBBJzS?`?PHGlWJ*?hCDGGklUUCu3poCYG0d;z zc!`faD5Q4qp3D+PH-pLKzwO@e=QE3&W+hLCS2nLm=n*OM(nfYdR{J&x;WmKpG}NoANG$Y+nk=e_n}~0&TI#%ZGJw)S$Bj zxbJzvZI-vuo$K+vX8!v-#>~7ZLO*^sLH5~iND)eB8>_B^ueT{JwU(q(c{6F(W^w-f zyX*M<9@D{=^OscR#X(1dD%X)7#*@~WO!i7;-0YhOQ~9fDjj89|S zM+uT)c@HY5>JXiJO;&rZKhDnwx?s0DIUIV47xGq`1ngK$7uS9SwcKi)&X+gw3C3m(2(o(Hby6o`S{N{o2p3R$yNV6%Te`YF2uf3F@By-AC_ zxTj7&lvvS^vvdgG*_*Na&AE4b1;}oZNWAZ-MGl1c!9rs{Mlh!pA0?b&;^x#><)yA6 zQxBep2`TlUJ0%wtg{siRiwDKI87w}EBPp*Aa?Z3C5Y8FE5FhUR!~JfH2ApSZ{NI07 zw~&~Ai^kc}OWBM2V`26rVfu1a-_1{Bk9jT;3_bsQA)Hys|rnNo}fl2Z8!!>&YO6lTpwbCpfNQQ9)LOGi>PiyC2VYzp+B;gfcPlG zY<{@_V@#^C-hDaL+NG0;ca!iy;aR*}!y_#kNpyaQ3u%~1$Z?-jpqwz5O5BU3@1x|X z$dc9UXif+3{^44@)8EZhezk$1b@uelo=~hwzla&Htf0ly3fJq$Q@vPavg6%$dRqB3 z_{ToRD>C!x#+fzv_rEkoj;m;h+kR%AwEo9zU*ibPH}cs!W+lCLaw_@MsZRg1ehiD9 z&B)#&AzUHsL}o|35n**b`a@b3B;)zGb5$&^J$D9_l8144uQpSjSq9H=5aJFD;Dx>j z@>|%CY|G)H(6dsW$}G-9y`15uUQcO?7 z8SWka=9n988Q#haZjY~;^sX1rrHYXNW(COAjqU zUoMiCcL-pRu{v3*G=n*B^bLe%-|=#v05ueGB(tZELWFrC=M31!zYwTKq#M71RpVsx zE^QlbKj=jE&FhA^?Z0@-*GOTXqz49U;ykbkRx~v(6$UTv$Ky9#$?rifBguOVy0+87 zM|cORGz-Rr#jmP#@8q$Q8oQYAd)sj+Lz6B)l?h?iJ{a?p>xD-fQ0rUb$Xr`N4W2u} zOi5Yl!0m?wT_@nQ?N;>gxCoDVE zz+S^Hthy;f#=2LLfhFPK_yMoA7p~lTiI#F*J0sQQL3+~ zjIq-$W0X)ToqV8|;a_)SeK+}YoS&KCRgSDSw|l+uX$h^*p2{ZtDMID?1@wu8Bk`VU zfdZCgkbS|3lyu)_iWgMDo2xJmGt$EX>t&zMN{pCf$Y_(LY9F1SRGM=!gdt2*WI8(^rO;|a*n zB;ID4`FJ<6wHgQMn-8(u$D6F1qE1Bb=P?z=zu2}t8L;?FGs=38gHN9>$4j`6Y5{$W z!lzrz%o7WU(m!Du$lc>@E61T)LkHz-bnvIg1sJuFqc#rF@aXFS{<)qwl5}x4XvXnT zm#f(iKYYvGgSmch-CCGxbd+;9b6!LHrMUG{6Hm193)nRaP??zZWS&MIS0RUnB+2*jN(=}ehl!7rc{_q(e*iSFB<1BbgafPdK*XK1Nmh}uECRGf%L++1|-J2(1c zsVOP9m`@JX=dmp}yqV$$SGeBRF;;6pf%w%~(DSvgA--Y@$qSxNn>WUhYAdeSzkdpv z^mETwRupXQbs@2C6?kPb=jxm;P6N|-B5x>)b{zbKC;c2~;)kP5i_s+dt<{3;R=$fK zgVnfy&Lf=Bkpz0}SK(BD90=&HCt=+hjPFuMDqIu?bDDbbXZ0X1keG?{b}FKP(=8}@ zSpX8dCXx7dzGcjnjr3}b7aJYhi-RZ1z-yiW@t&(lv>d(h6_@Lo-qnL!d!=Za%_h3g z|0H%e&Bcu6qinIM4sQRcMndcK$-;>ibWxTrRB{eHxKjxUEOdRY`)%6%D-7(avR#q^a;HF3W#5 z0w)fS;p213FeCHyjm)S%j_;&{>mrxZ;X^l=XcrZdw01kOVcPNZYHQpvqD20BKjHGx zAFxnq8h$H1gclxi-oDyE`rRo4^m1EaHg`8lT6&gA-(YJwYmWc~oGNBQByvFFYAg+n zT1;&(b2;#CNj$vHj+AADva|dnK}c;A#Pg1`UR*D#YPJwLTW3azoFP=Ny$x+oW{^{B zTUfDk)~MuXNSuDg(u2hbSQ=?c{?(=7>TyTVRm`BWN$W^C#|sJIyymnyn24O>xO&r! zV4=uG%l4~ECZgM4aYqVe0(k!Mb$1Ekc!)9VE5g>;1T0YEbFF|7ia#U-Mav6PEE%klmU^8 z;-rM@F3w@zU{26h7*Cu<=M-`LRsU!FO)sUXahMtGJST&*B74BaEuP$ayNy0?90Kv@ z_QZ1(@lz7=wWnEv)&_Z+etH%mF_Z9n*GX2XdkWP)vI_dTlZd^YI$hfMjxTU6ntdl- z!)!Xk!irg%MAY{bWIshn3{9a5_zemb3`pWXRbJhM*I;F8PAhLb#(ks5VCTyi&^DHW z^}l8T%JQME3Fu^*Pnasd0Gg_d$j|0GvCFfaAvmj$Ey{F*!Mn4`jFH2P;ophWEdDZx=$LUc z#T!g-j~%XyFk?2}dc{62Urux%O{3?cZb8K@QR4K-67=sJz{ul{}NcT&B!>%O|P6*kIruOXjnKA z_vm!;68-uy`-%l=_fH@?m(Q>c|D)(k+-iEeFx;R)lhTMvDHW*{O=qt+B_TAD3Kc4( zA`~Jt&!Z_Cg-l5bsk7H>3MnZxk)%Q9p+vus`u6u1bX{k6-m~Aep69-Ix8f4}k6^*Q zi=Dp|feTOCkc|mWC}PW^@Ao! zW5<{Mf;1U7=E)3m{)$!OxaFxOEt1m5{!s$D_qRdyoIA|VCx*;z9ELZKCnM`24rhbb z!+!$5AU0?(iOl@V>>f7%{2cTkuX-1_Yj6OvXi%;O>^Cv{LsT!{A&-TVETPkxO3|_p8L0h z*oSN(-c2pcy;w^!P17ItPF)2f_AOBBB*&VDj)Bw45_YNF1akDO3%Q_U4r_NTw+yWt zWS+Y&C1FMqWKI*8hf|2=RqH8HKO>HdyR!^E-v0;2^TU`7-yhK2YfsvjNYS?y<*Y*O zHfF&SH(*4>$fW=UA{aNDw&|U&I(9gN>6w}f-?@FviFw9ENQC>{yWv2ppJ-80mo_jy zdl^q>_>yG(Zt_&zh?re@28%j+A^FNWaJ!cT0aNaOcc9HzrHEbTV(k3ag57`hpy<9K z4E>ZL+h0$m$t4nG-n8qmWs3&0YWOa6Tcu+T*LT)x45Cg?^Dto7dNMI%3A%hzVb)&| zC%rPwQ6vor4XV)fz(R&BI+krODU#- zzKt);d36hC^p?Yq>LS!$@62&&SK_rTvuMX<73@^nkI(DH$-hU_NH}jgtyHoA{fkS< z*BxGT+td9tbKWa3A>1zd9R-bBt?Yvq9x;5go_c4;(371m{4G+;f#>E8x$4Pq)o~HN z_2n|&WDi-??en+q#_E?YP3&Z_T9IA{l62Z-xQq zH1Obc?sG6&h>7c6!7buFZUb3jw7d%>-$qj-b0r{|K>Z&j;mB7_j_=@xwZ>(ff4q;) zIJOElPg_Au0)N1SAIcZ+2% z^RD<1gl=^orr(GDhc>9}p(3dkC@Bkd8oG=Fy+zE_I@olaF4Rp^7bv;M^K{1xEvFq26jL@E)b`CRP+fQqOPbHxs4XGUQOfvk%`J3DWg{ zrqiviB2*{z2^7kd!-oCw{Ivli>_C?mnR#I^{I#%vyj7lbRe3IspRDCvKcX~#wj@7w@E1NDb}W+%Xbmi7^Uj^XD4yj zIJYm_cLP>Unoe(8q>))VQ<>)uzQkE%GOglsovA?jMHR79EUMDQTxo;y|! zOef1x20rJrXtYy=R;WDzfnsSkRi_KBP54-=TY}OEp%80CGf>@C~h%%qa{ zTo(2t=B!+g3Sal}6XmR_?BPf3L*36HyKpXX@_Y>EC(TBmlj0=G$Phk``{I5*Zq{~h zKT+ens5bGweM2NHl-~kE zKhA?gsS_*>QG-s-qiZBJ7ry>_4CANsnAJfGi0&+9;=3k_>eaU}Dn<9%J)iVwRRj0; z66#@bXbJa!olF{vC&8I)VauvdBCPINQ}kOD1}-ZWf+`%Yf#5!*7#+@#;p$u*Nt8bh5R{-DkgG@tObN+S_D2P;#2DN@f#dQE%SGxEc(; zwGLf98({1ZpPSk2#GFmNjI*dd)Ezy5dd8e%UT!Bn7jKQ;ztV71z!o~Ci0dg#$%O&c z38XryiMe@8k*du$28Fytni5{atn(w#WoSoi6?ru0q#{P#R!9DZDOA6}1lN5X2Ehz*5@oF9oznMrHmASBOKefY!B_+*p6As zf@IE-w@CM@l2>5?)ZA@7oVBz?J&$D=nwW&;$G2nbpEo%3x**5*b%AfW4rEPH0%Y7S zfH{pZ#JOY&&4_u*kLnOY+1$Ixe`QLQh1BUKt#o>WmBg+WOQ0m~A}0UQhQjql#{)1x^PQszXh45OJ4}ZigX_hlXpFh)v zL&MSZQt=C{{?*2QyP!*_9LvPfyWiQgToZbBv>c?&FEU04uCm7iLsYKCO^dss$O{bQaeN+SP4IDIX=42KcFh<_~(2C)%Oshws9r!k8VKc$0qdV zqDr=Tras5`)x+#1$Kl{6RoukRAb$?2fM$+31e{Ca-=FGB4JL0v&A)LZLGv$9bA2l$ zYAcdjGkZ)KHUWM|DDI5lL)#2H=BCD1CRXY_il#S1ZYswSxo<-s-~>8kUJQFL-hug* z`)ESlUEpp97<_gK)Q}o>yGiyUn7FP2LPIcn?~v^~I>v?|gyxrO0jn;cNX{#)CH<4G+cO`~X?xJo? z0b?R_PM*^!DMCF*MNkM9$=j9#&xjS;&7Nq>ePyI+%=fgYGZT-Yjh z)1A2_;T$rKPgAJZl34m?i5GO47V{ni#9&ZD0#tj+fcvN8+%>@&^Qe!|P+4sD{@X+Wtg}o{x(s zdmA^?0eN@&-)$MP-+3lI?5#p=!gX*c@&T(bGR90@QH|@|szC>UDmEz)rTe0!OZE*c z3qK5P8dI4STPD+YN88y~m1h}&_HZ^jLj-dlJp{uq+m7HbN$iM@a|6?NsjLv~Sc=G!Z?I@echL|Ga(^LLOZ8^vgS{WaL%Rsi=mM{&F1 z6qIZkXX+INX_WqR-ktC_?1r1Bv~#Bx2J0SW=bAdv&$HgM;iF!}h6a#7zunPe%_=BL z$m2cUSBelppdpr<&sRuMtzcoY;h#D&Hi@IUtFJPFCL<^{Z4?g!`2^b5n1RQKg`{Kj zI_PHifTtP#PKhdUJHEatT!7$+U5r=e7Ksl ze-~nZ{3i<+zoo!|^J&P}utCG4h1l;6RN=`k80|`gT#va#Y%)UwdQ{1(mHE7H7bR#_ zUo5SC??~?Ky33f)=XwrL0%3Q03%@9K3$FFjrswY(QR#|XsHeIR_B42t_KWLTmFOR^ zen9~B85#uJTe-~XFhO#HV_h|+?It;L5Ag`!jT&%$!a%ogu*YAXx?Z@9x{+MIJ^vd! z7;8@$omdc?p4&frAxw!dl|(I8T4zm5qm{NnAWu_Qmb|973EPZc@FsAK^1u2#5aIePay9 z!g1VjdL8+8*9g@vIHHw%J!+YbF-J>|GbJB=z<-x7N*dqbZTMh~=Z`(Z)1}Kvv%_la znD0e4>rSK_Gv*Qf{o7zulQ84_?g9MPxzEN6E~Gnq(#T9lL3&8Aj+>L7V#Kf9fS+6j z_tzRroXy)nnvZ_LjnU_rm6~I)`%VL*^KE#1Xf>q#dW3eZubKRu5=^gLi9IjZ!isIF zeDU5kR>)YL+;MfMg{&HfCf`jG6SCOz-b)xYf9{Ogas>3B8&M0(op{yD3%7q0A)5?+ z;9|5r{cE=csAdPE? zrFk;wCC-KNI1}&^(4~5(Q;C@VxmEZq~s=_x+LRvdoJ2ws9eO zsWk_7$4wwq{t#`t?8!M#bm5o98QfYSK>J_*#LsKhse!3EJz1E;HdGeD9=%nl_RE7h z>^KF(%cZID{8w;2P=T&W_r>?Rj-)KN7w7T5qV(qpRP(SO33lJeG;B~H^%E?pjqiWF zV6RlnzIqm6 z)`R%05Ab-%9pyiAp0;dDIK7ec>5iPj57XlC1jl^!U!Q`5n*zzdz1HN*#2{96*D0p- zbOm-=TS9B2KNakZMu8cMRI}BPDqN2u6lG|%+9PHx)|li8bVKLzPDri4Z7I2L8a4VI z39Dl*suTANG3T>#(aUTV-k+ci=bisS>g9>#^>$y-`230OHL}E|Lm%MNvVG*_@fyg! zGlN9Uw4$1oR(SQq4~{1tj!DjnvVxRdL9h3$}_ zhnyXtxygo9=bg4Zx>lN+$?PK^W{Eh6DaRSLq0q{}7t=l%-bm0BJlrd%Ir{d2VczL~b({>I4Nwk7d9(us|TAGl94 zL+4fza#25$?tUmqcdfCa2@URa*`p@~xT)bv@7xY1Aro^Utxknq@mUmoj66xm zwk2e)oDk>h`Y-bcKyU&szGcf z*j9bx<{I;Q+xWhsDkRIR2dakbKvd)jTpKuoS5HL|PqPPDAreKaXgR7B2w`B*77WV@ zB5S0yXvrKYGJC}wYE?Fl@>z|Tc2)&;-)@3qL!m_Q<2009EyT8ne`K<*v|-JzI<}VM zjhq=i#lA53iqnJ$vr2UVF}th6(jEFl;pSfmw&8LdXBts>Bot+u7?RFqYbJbX#Oz3W za;rOooI9|FUiS0FsUEM{c|OD7G_(i|b(N^nAr<2DK$&bhY(P(Cucq~!Yv9vrLSLw? zB{AhUV6~S3RSjJY)(+a#@7`*-(4Ne+M2OL^J!zQbevv&;@dIe_b!JJ80mgoCrV~Aa zfTY~R!L1v}`wQ;mY=i}9f-mY6b;9mQXVT6wVi!c0;C@R!#%t)2w{^)lc&wN8x}1Y< zFv`DVA%{zT)WTw}{+oQP5tKA1V1S7`qaYVar?wo!A?IsYtSZS z8-%ZIdZ7R89X7wG9j<7J(GDS1%4KKBxOX|vKE;&e#m}XAlBQT8K9$Cu)SzL@TtL28 z9vx?mVe~%}dI&?vy0{N`sU&3bFkkjpUZNLbL@Oe$TdyHifoQGx5tpTd7}z(mcD_ZN9Aef?_B1i^$A=*aXquA zUz+ZlKEgb%{f^&u-@)Uea@0LE475a>SoEzyJYLKXP~&*{!w9)n4^a1&Cmw56Ay?C9 zGRI^>*;iWT_LmGR4s z2e&U0^nA@W5|bHC1D89&3(v{awB-wHm2vvBA* zcs+wYJbff69Td>*g0k^L_26mSjT;R+3 z&fjTs8Ifb~af>K>iI+}h42+=jnI3p0t&Z0+;vintntl-a0|E=`nA`T#VZ`ShXvFA{ zYi{vGRY?GwvyPzrDK4x1ekc9+n1zwkru2E+0DSWv1cehiD50oNUS&!` z-i|Wvl2>rsKs@-rSqKA($*g#MB(W{BA(!v`f%ADuJfZJn?0Bg(NuArvL_KN4u#YLU zwD&N3=wd2QLXXFIe#(TwyuEyZ!`Co*nhD*U<3cx2{)7r=l<>aVdz>D86u&vng_15^ z=%0HM9|m(g%dm8^VU__*bpFfSD~-n}4F!^tIg`p=4}q8I^D$699wb(jKaFI&M58>f++=b7-6%N~n$N#fC5N4`5NOj8}#pt9yB zsF=SF*Eb8(X&%P()=N2Z;`baVKUct*Zxo?)T^&3-83eNSJ3;jEYcQAVW7e)6!|So8|t?fJmY9;<^V+&MSsp$7l98B)`KyK%R|B+fl$MqheL;-4Wo+#w#wa+2CgnBbxHHV=&Vg}COc0VVM4KlyC7-HBSjCwY#=s|cYjMzm(pdsv-O$-dhy!M^x32!VHlsmil9 zet+;o-0Ack4h#1|{Gw#~%Y7>xs`3Q6ib9Oc79$0pFW^ZZF^F=Rh?V9uh-^?HFEZ;i zqwjVVq&eq+j)gfDE%n2NYuE9b9b@S2<$#fJ5;J|xC&nVS3ey}raKVp7 z#Op~L)MlPX8P{m;?)wNA%kQA}vnCNQXJ2w*&JA2T_Zicn{Fb#ba%3`WXOZ~kr9}Nv z5lSzzq_3x5MUy?c&b{i+IYG+c)O-Z}u zD^TFLs;}CYQ~Ox1w+0I26=Op|ift zgAyT#x*yM` zR(A2MOC`rZ;+V|QiZgky{wroAPUV7P%tV-Z^)Gz3^<Z z8sW2&87~Udi8qILF;Eivu+j7pz(&`++* zQAv9?@p$VE>jUpHX+H9V-~9m~WG0c@szrY~WJ7!;!&KbbL2F;nV_R1-v?1{tSa94= z0oz`P>S|yQMBZgD)o0=kH7>8%gW5~I1hk3op zg@!*irIR!!kW}~eFt}TZdW+1a{*%_?X*LtDf2zfn!~jNe>v`~8pGNxUDw0DB-LUCt zH)`&l&w5lQqxas4%;j79G-PNp&InFr_S)GK$680oGS;C26{<8iN|u(~X$BwBTVVW% z;{vB^(63g$3uT^6p^^m% zrsEe-wLO_E|0+xO$$Y_^{l)Cw;#{J0(n|lFji@lO(-ll1{3I#~~uJh7mK;q1p-*ZH}}-=Fjcq@6jy$ zRP>AU*~!txOe`DQF_Y@sZYA~$Ey=;MdZuV3jde4xfZ?UTvFoBVed;s^6^diXEN~*! zcLKR}TmWXS^?`Q%=lJpp*YPtiLDl(hz#=^r1&ZcFc|6C*ELWrcAHAvgf34tjumbx_ zZ!`TNYe+G7M@Y%Pf*bnOS*_nfOzChA?=Jr(*iMP(n_eHpIWLrn+CT-Amz!Wmn+=t< zGN3>Hdf|(UCwR9k-=o-=H??C^@X5oAkiEVhtwTAE_NxQrNy2hkTONf198=qrV`a)3 zwJ=xXjoH_;gox_&V0`vpB9?3zh50k*@?R@_25ax1_=(F-v|1nxwkgs4rOchxrRixi zS=J_I2dR53h%E|dAz`$Noz$gCBl5q1xU3Y|<;D|7m&kC;t1jVa=n z<-6(UtMf=+!7bJ<_cZH&e1CY)wWJ$R%sH3}U^KX9|Ld|WMf zlVdwx#jEr6nQ#L+mY?iS^k)x4&J}gIsvtx?jd)b>kpelV{fl2cvk=Pa^+=mfCYO6! zLiHcoqEYsI8o@k)pM!0zxv&In^U23;!U9mdMG)>9d}0CLGcsV>B=_BG7Vf@2_H_Lg1tLWLf=)xl-Bl;{f0&5T3003A(@B$qk<>Hg^R zmNoOXL586={239Y623Rs-0d90CTbj(e13)A-F3`0L0SCVo5*AR1<<-I2M*QSkr&0^ zL38Ll(;q*DY_H_rF7D|}?Wmn|f_9;NXGo0D*IZ5z1NO(54~@*rH`CF5y45!S{RF>3_w zFn<5plY$~MtX=sAuC3&_OFrJvw_q)`^bjD;dv?Ok)?u8uma-4k2&@cy2;Y4=QMceT z3je1{LoVsV&{hNbII9>%{O^O7-Yd3AGK1=z2%wKwY7!mJZx)-*vAC5tu-R|s5>Vd& z)79sY?BZgao1ui?p6IedCqLuh-3AE#yNnpmT?8SP^EpnP2bZR5 z@y&fPc&z41%8n)y5uP@F6*x##Ck~+wjU^uw#L2?c3;gF@eQa*h6inX|#7rAeBkNuY zk=KRuNwbzBRJ5;ThwcHbD>A`y;RSH;?{`$NYk|=@9!$a(Ycj0*n_qDtng4odG5r;# zM}?Q!@s*zm)5&wxiI=Sk$;=sG+(V7jW%F5LlGo9+F?S^8IEHS2%|o- zbi$MARNf+qTr&Q{*gyl`zG_c~LzmL3>MbOn+r=0j-bK?lEGAt+mh|!^fbWh|ka6NV z4wK&Sp1+PG{d)xYB2O1!_qaQ2mXgJ6?bo0m$D`m}(?0ULi;t1*x-@0332p5PW7Qip zp-1WrGxy5^eDcU2x0|NI714NRdfz3`F*yp2^D=Rpsw@o+xB=5*73sqF8Xz+*h1RUg zpxvQkSb4>V@}mWa%IrsYC?FkT#ow`Ela`?v*U3nIVMxphd~mae2A0QNV$L}?ad*AM zRvR0m$~_C3qp^s19tZ)QDG9{7(}Lfs$n_>Rrm!EK%&BU}6No$T6ff`SVBb$JM&IMo z#8^mxarZilu^msLGn7XHer=}hokfgrYXB@+xq!<1dt#c_cecOVkiDrp4NSQ^f#25x zC|aXORP#)U`j{T=dO4ew`V@n=Z^uG|+%EXM>J+-}T1>;1ZYHMgX>8uP0^IL-1;#em zlR68)8zQ+?L0$ng*xH-U-+Ti2Zf&6Jun`RR*+Bm6sh~467sa_QL6zDmoVBBt>tke6 zr{-$T(^kbKJ~%|f4>0J+`AjALY@nBf{FqRiIDS^x3Dmn8hW%$0Y0N88uv6Pk+Tzq` z!E;l*Hg_`oc|Qfawk~COhKE>(J?St@MS<-3&m8U=+j5<^LabW#7!UIQ;gKh{xGPbO z(oQ>i;CwDS>+=QnfR7b<@7@N@Up}C8+ZMWQ<~D51ibko#7}y&jNq(=~PUc=rK#$f; z@PGZ7ZQS?>w)FU+{qE{S?R}`* zhwCUDu1A(ho=3eesm#ylos8MWE)aLYETreSzVwALH4@L5LRfe=Yt4w&ic%b9IL4{cLK5lc z$b0R!h`wAUM=lQCg2U1wkbm(XmSvc;PYN43mg!Br|85ew7WNy&pOrEhf0HoU-~j4Z z3J@O&FX&-qkco*W0_)C$MXD{!B&WWve!y{Bn*Dun@nUzPy#G2IQS}nKhjY+xa1yy}_zw)a z9dGfAazAvFDohYo{L{FdoTI`+(C8#kZ8D-p`zU?67sX8ICh0WvW0Ru=ZX zUdq1l7N?)0x6@oDKRo}b4cQSZ=xPq49U3>_uu>8K-;8r?X6kmbpy&Wjf6&9_oNmC? zq_wm>s~Qe-Jer)ScnGadqd&MT$-7-F)Em6u<$X7yrtu-Pv1lhQcpFLO1NTAKS{Jg% zZ9f^;)udJfGcDzA<}e1Q>fk}vTX0fKXOcn+neadsO2fM#E`dkptrH>*|Av@Kcdp;L zR*dXSR>F5e8*%faM2s@*hm&7*=!Huh5AXY2s%D;oL1OoDVV^o(B9{$ImK$N@(s20j zCz6(2-9beQs-bWw8pi|osCXC_C&m*wwQ+gSw)N%%%^`2 z&w>Q5N4w{^DM)fYQpeRZ$=<`qaY>;*=fnJgPfPNc7dy{j*J5CHuJ^`E*Ef>K?z`Et z-Ww=0@j9j)Jp-=q4QONGAMCzXgLUDf)eWaO_Vj5Uoj0)sjqS6*NOjQS%|kwv`;~)) z{R(nD+mi;K-2sOZ&%#GNdD=b0n*JE@CmZ_jay^kG8s;TIE;XOS$|?hV*82m#-sp!b zcCVO;qJJU5Z3-2eq=ZR<|FJ8*#?gP>cCfVI^M;<4L0G&fJuF@ce$PsvkMl2UiN40S z`D;kc0X2O6xCpMBtc1f2Bk*&{6MS6D?H@D~$OF!;E#DnM%NJZ>Ob>14?zjTV+h z0lhH@Ni{{0$PPTkGVmN7j%L6}AbuPCOEm6&Ttp5!j2b*mRs|9#VO z*M$$Tt#A-;o4DamKQG*=^@VwsH;q~UW(!u>{=z3>k@QMXHqM$a0pFiHkZD7NN^oaS z<&=-GE3gs9Te$bc?c2!LoF)izRK)zHMnpTp7UoSY;Y;1`K>P0dO!7-pT(NE@I&RSg z%A3M8EH{FwwjB3)_ay9GWy46ui?Q$Iim>dh0DaV*Lgp;!<1Jco9yL6yS+m$&ut*j` z9VI>3)w!9b7s!yb4;-^$Y%|{bnZ+?B@LE<<003V?=)L zSc7Y`^yu{DGpLh42$Hvx;3xTw|N5NR3lnVUH#1Y_U$y{Qr*e;3e{U6;*cpN6)0?Vi zw$);?JLKTIZz5*=2>~yqe(=ueM1hcMC>=TnbMzaq^0^D0u~Ho5SO10`GGQFsax*e@ z&p||SGgV)iiQ3#Vy}U=6<9IiKRbwH_x)s61@t-JQase~a*AcZP;$+gMziek(D#t_B zB1yWXc!aEAT&WrN{XK==YXZP^o*4O6BM7~79i=9wy4R|s@B}INAnS?4$NW(0w+tS>p~*3AvLV?r42qnxA-wu3 zS~!gIv-}2d#;P$exc`e)dBKOx`#2{C(*jc~#4*X}J*rO*0(f^CJ9=a23E>8exvc}z zf($jLRv6K(4+*|*WU^j2b0$?6EIB9kko9D;p;rJVUzR40UpNQPj5l~|k^psj{Dgo0 z29I2`4M&&OaM)_b!@D=iVM?4nv!Z?gKXU$n;u9BOchxA*{mxlFd)OP6ByIp=Jef$8 zo8#UAVf4+sgyJ{L*|HbQ!N6*W`4#dXY)XhnLtiJn@W_>>e0|G&o}5X4@eQ$0E{fv{ zoJLRKjnHsM4*%o{(XiMOI5o+K+%f^q5n(`16}XUy$sE&osx(Rjjxwuxd(m-%6h@w? zsV;ASf}8xr$@W4?3a!?3bSAey)?JJ8hJsXi`&Z24KG&yk3_2ThNCs;S!RPJa?3Al~ zs2Re&9f!!q*n;Uz?CqQORHA1- zah%=&+udDYS?B=onbTU_x;GO`SRo>6x)SrArsHU~IIWm*gt^=w317K+(H6@CjJ233 z9h1$5&u@J3%`{V3ytx3hF1}~#r1glsqYXLGU&ZTOnt?ipOF?fSn5IgY;ll};_>^*t zk)g-fkdn$|TiH{u#6a4;-~%o$oC1FyjKI87S?VUb4RmCVfsdOibnl%_bhmIG<*Z`R z+3<>aJUJQqc($~(YX{k=wG?*Vj3&EOPB3F(5tx>r=~v>#Dt+oYJKuVI7USEO2ZGEiEP!zF!kpU^6oXOHLeQleht6}As!fV9h&Q6 zGNjRDn9H;kg3}aJdhUK5PauT>=TAE^#YUKla6Qqb*#&&t*a-w*$x(%S1(3uCz8+weQ%05?lp;QO*nCUWHfySzvj zuWM&u_g4XCP;v#Cp3S*eR^9>BSln3gV2QJ0c_ms4G*10M8yZsr*Gjzc`*8&*x9EMSy z2&B#3up_~U(NCL(`?^FSZs-WOHQvH!{Mpp<+A^3NR>o|#Fr(8VE<%00AUbfrBbK_+ zw8ABc25w(OUOls>LpNLDw3awsRj)<2UzB6Ik8|vXZ;@o#0Y~ngN(l5znwU=`M)cFk zU|KaX3m@*ThwVES(~j*@WWm#VR@5^YuJK0E>E>tjcASWNt&+LmOC%vbuUVGgs^HE_ zG0L<12var8upnKY^mIH!k^E^mJ{ZC6T!o>bSfAQ%i-#YQI^^CRuEXAOg)y5eM~;-v zAR;AMSj-8rZGM*`6U-3V>gRZ4Fb>)uo@Fb?)A{4hC&0phM>Q2zlZfnduoUXCp!+{e zs-8g1O(vn0@(6ThaCxPta%A6*rBs@mvEa$31JkvVZtfGnr| za9epJGi&N}QZUaA!q01y@I{&Mz(bEFSJlEcZufKfK{X1#@TPn5BwC7Iha1|D1U^L=?Faz!EZ@DgdU{S?q?kRv{?Kk{NmkMaa( z9-zJTChVLO#>DXSDV+2vA1=KUBCf_M0FmB|N%KvR{@_nHCE0?Fwgc&In?@IX)}RN9 zHskzfuhsI)kp{3wxq%ROU3`L(o1 zjG_4nL3n0!F`6FNfp(QeME}7FXkPOgKHm0c^DhFk;7A?b>3fOQuV-Sp+E(6t7lQE> zCHTZ!m9~D=BPphwGcsd44a}cTQgNiCLOf#-f)10`kqS8(dNn1I3?4rOj)OPR!)zI7P2=-yg!3RRHpPPfp@NcH6>=yoyqVw>l@_plY zRz~*DN+Oan3+K6RLPXhF4O&PV@>NulkxfRT&``8wL`lwb-H~W$mzGi~MJh@qmGpal z|A5zVUgsJ2b$veXw}jL(@@KUWiRzyOYfF`g@gr;MHuM+5uPM>hUsu9G(QmlKfpb|J z$G7_#S~G_4gjL5N=gdlS}^69qhs>dWPizOTc!W*<;EV1c$>?Ra|I8(!{?U3A~+ zMXb~#6S}xGh-`}%Aid9C@S`_!yhw2m+TA*bgiR43*B%Sffm_dD-&|+XGiJklQ98}U z-dat(Gv8W&wkZVuvus?l>jRTn^a3WH@+DbL|M)fA(^0Li7DlX9>AL>o%xaE9p~Pn~ zQR@aqr(J+fw-{o*!3okikN7gKn2ARuram_hJ2sFjAp#9%9TgXjH=){PDOJcdOX4sRs^1%1Z^P zwRr-!RG+Z1(1wGXFEZLKGwAC1-RwW#0aV*|iG65W$l8pTLyV*XX&=@jCyH9FlV(o> zo|hh3VZD=XEoefGS7LN%3X6RS$8o{=6vlPF3_JeCh<2XdicdceLqo<<=D3Y7j*V}p zS4yoI(rb zV&4W?vMFABA^+Y~_SA}`DICOh1Bo{pYGdZ{{yt~R0_Pg8M`WgA~+O&-w{j6{<&sx))*XGY?)72L8(rop*qL1EMo z?Pg^op6SIydj>f+eL9)jSj&`T3)5vw#+cSh2MQ%c7@oO`-23+lzRYRHdKCqjleCoN zug<_ZJ$KNj&l+=AR57-%xHGL`Ihy95NB^;ZxaEl_xgy&FrzfQm|An8JQz4GDCFdAZ zq%)EDpE?X-ay{VgVnf||pCRMd3wC>nAh4P~n01Jt-VZ+<=X^VQ+)O7ib~5!`dk_tK zI$-~FuAjY(%kOdw&%WO}f&5gcJ{|LM0 zWWlTZKXJtlPr7~|iy{jPQHDXhcE1dA6vI$HMwMs^tI+%%( z;Ok?!N3$1a^ErOV>-jW8QJWsT62i=jy3ST<>rtU|l5|G;Z%Ap_f(B+R`$5W#SO4`Y z)Noy!)H8%kmb7NKT~y-yS9-jOLrOS-?}5%|_32s7h4l2HOd9X#j`@Kvp{L`Kb+qAI z{?Ep26t;>Y1$()Uc&ZtVd2#=oaH9o2Rk1h<~McwUuNK=Fi zEZ@TM`qfV1@%|$?+eMighreXJRdlGjNDjJ2y~o8xl4QG20GHq9kw=0+B2^xsRrx*k zvEo-|&AO@N(w~etpCGYaMfos9ddwCF(fJ@o9SXXE-%mRj(5rVvzu)5sMdmX zx`w;WD#zu`T$6{zxu_c!2?&#S4f=WXxWRKI5J6m6=Axzz{z|Q%HsoR zcjs6lE!K33{S1WPJh~_36uur9Hroti(0&i}(>7pv^1E1` zj~idc~9BinO|9>Hf|p)Ho|uQ-UofBzT&f!uTfq*m-Ran$`<&F0oWD64c~H>>2QI* zYCpL6Uoh?cw*!~S_)}RXfFC(2g!Vj2AvP~`Nql`1KmPf0*yh~;e}2uNAY}@xgaFa(35Mn8Rk`RtatDz%S=I zuiQ)>tThYyZyK$Mb7I9t3otrWZv4l;;f+vSLX7N)fEUpBb z>?yQgHjJ&ivJIR#zu=LQtGM-&5p+5l(%S-Q%%w%TRBX#f@GprZb_oIy)wPo57_Y#D z_x6cs*SeOxTV(<4qG^2Z z$->0{t~1ja*8p=D*s()mOSyh3*Ke%(g`a0U#QMAcV6N$HEcts3;tVQqxO;N31msvCHOiBa$+UotmQt#(*mWQ%%sU`5%4_o3+o!n<$jxo*aN&^D4cc;r=79JEteFz%+yyb ztauDEE<)gw_nX;Y@C)}Hyu!RK)gUXQBxt+hFtQ#JbmYu1Z{tZd^sEym0jI^u7Hb3I zWOW%urtF9GZ--HMM+^_t;?S4#jbyLbZ*xqN|XwJJo)bOv-j-GavL<|LkqQ%`Xb zk|gIzHtl3#;$A{aco9_KTP~QlrNYi6W7=G`gxvGf!RH@t@=G>#VNCT@5dH5g^KSMh z=(RqH#=kbx;J{&4jC?a1P0OOH1a1LpL<|O(N>-OSWWgD2cluOdl`# zf=!n*xZc5HdV?l&d((9K^S@XqYIfdx{7-Me@7W38YRq$zQ0{5H! z2jM5b!F9O})KOKA+}NN(>nC&0qOCcgW=()L2=Hsu87B#KZ`MNv{aIDny|5$IQyhg-r|;`qz$R5x@QSZ&`nq} zlBy4w(Yq0}*w=#|eLu{5wr&{yUYg7(SN&#cL?6N4rKK=F&5N{KEhAZSKE&^}78*)3Uh=MtrvDtO=Dk;IBO zGIj6&Bl9%#sNe$=xMH6SbIj~HUiWznKuhM+e0z@lFTuzho6kJ-nF*15+=z3`J$8wg zB@B0OBa2HPaNR2jVv_Gl1}{jF<1=QFwy~wqIg$n)AEIdHh7_LqI%!5+hTG|L+3MFv z5SH6M;@e$)$~v6wgn|t=WTeUh%vUWYSD90&sI#7&{3Ae@3PqEHnZ|hg)0H|Afs_6U4oz}mVbEAZkvz^VDq0cefJKc%4v;}v)KV^2A%pg-HZKQuo z08=(hBQcBbLE=pbvNFaPj&QrzyvycpGl*6sKjY zrMP_ORd&A8GdNJFKz&cik+);hNtr-6xju3ZXMg_%HB-#2TITGeO0Ows-gak%mSy7E zm|ZY$*>R5ZI+>?)Fpa(~UrxohbF4kP8r(egfb}u#h5lT9M!tUmEGuq=Bw2Bu?Zw5k z+-DZ$uM(yW*DQ#e-AeN7g(1CAkqo|lb6`)f0CAZ6j7_!R!7tBhOnUc{S;O&BH?;-R zXRXpuqi2Be+`e&|UOG8G+Zub*?Wk7E)lBD(Hf%NQ( z4BRQbAF`I0G9BO4Q7d3Ro{jJ&uN7t4wJm%$hMN~U#_7=>;|z50I>FvhQY6Rp{osa2 zBHd59o=MFf+MJvP^~-d~`uJ{`%uIuubG%4EtRfCvILn?Z+)T?`cCs3tPw>onTL|kY zVa+(7^48&pxNTR3)z^4kyi&(`PsbF--Qb@rqe0L&UEHp zLF)N9i+HJS;8;iLnQuvhN`?;;BVz=yv!i>`ksM*^dwOE19*2=JEr5 zjxe=cPk7RTRA>?6Iz1aRN!^Ps#(F|EO3dtm^se*xThWIs4BiT}g+=J(kOyo{q&oEe znoq5M^!6tnF%MAkwF54gDn`yOKM$)dTVc0e zAo1CCgneq6KzdIKQIm~8AD_v=n9?$M`C1Cpk;{KQ<$N@6FES4zrKm5*(Qp`Y!)=y1 zP}w`iZu{2^b=x;#l8_2Hl3fhaYvt%8o+4Wp?dO5Ya^;)fJDn(|AK=6%v4hfB+u zn8n|*L$C$nxs2z?{zGu!Q73yy>7&(7?MJXAr4^IrSkN-Ft2n1~9}aI{ONXQNiTms_ zj4P~wO6e?Ox<-p?4v11C-?J#&z@l)@b(kUX6je+K3(!YeiDf? zN~6OiTPV42LQbojpuLMW37EEtw5@-I_v8fVTva)C(dGS&zWz$0kP9@n;WNDT%|L65 zbUN0z2y13~U{~cWuxpUvu)l?jWS1kTFH{0{jtJe%v38vHWRhz!(u{FS5-vE1uvI#l z+E(p{3jKSqPV)zdG~VGGXi}_pJPJY9PdOfR3Yl7aj_Ee8LJQ##Cg7qEPExR-^J?$m zo%C{uUCuUQU19XPbq4JG2?bP5Jqmct`6KJI*`&)XFm8tMJvl?B+)dp8K(WxkUd>UCk_8A~es>>sO^n#6Hc zmy(&U22jZ25_l%WLztKey_PeROt%vR<#Vc(dBNbWikrCD_6Rz;O(c;aJLpT-r!bi7 z$L?;dVq9-drPeaD$@%!zV02^%G?cvPWvhPPE_GGfHjoC%e{8ABjW9;K;R~D%z0aOB zKfoM2JB?!j>av$3Twn#4zZ?BwK??axsi@FVFdeZ6@qO*=tHO<7q2WmPY^a9WlI9>8 zQG=!fXCb-o6&OcNgyDEW=A@A#-k>&Ir@|Hki{*%T{|Q!9Y_Ij^DSzSig%hCY+K7oG zm#{v;46Y_Uhox0NAtgD0xjMQE&M@of@69gY_@D+SE0se|(QN3?Zh-UUZ{XWwC%*a0 zFxaJ>fenUT(A%2?;T!IOX!b(Xk7@(|-dQMkXEhP(DuWapE{0;v1&x6|T=m(**^O&2DLrA~ZLzJJE zj8@a#NU=>CbL+%CIQ6Rsme^XL{f1kxIQ%E5zLIN-_G$#Bff2B_JIlzPx{R|I>(lDm z|H#tvMv(J&rT;un!Flxw%*~-BVxnD%rK_~)dnEzd6>}9lXDq-oFBIq@4Gms3mwh(K zlcrlnc4M3O7~AKq3{91HVcM>L{A$BsDzEScdR14k8EVSxPmZCuQp=7ODIJ6OhF4gp zuK^niTTsSLjlO>!3n8-a;I69+u?hBL`)e4Qv;vvGan0<{BR^oyL=DnDOPw6(K8nZ_ z$0vhEq+c(HCa9Oe?|5r67)3~9XevGAQi_reMR+gj3FCd_8B^_%Ni97!NRf0fq275k zW85CY{>ucT;LV`fAr1z!I&jX4r5rmt8M-T{&<}%t%&I+vsBYll(;XQ^_f;S@_L&Rj z>HwFfuA!dXZ&Y`QH?fg?4##&QPtoK(vO_ZDakVn|tGwcz*EeDIiz~QftPPy)v|(?7 zIeu7egnSEWnq$lLXzz%S=rM{D-f?G!@g638dpjgGU&5|`0;FE1h3!7Okox+ppmF(} zJN3~LB9wClcxRW9N|*)Era(V@X@xX}7$(<;yI!AYlA)|ByfZnNO6%-~E@lV0IZAQv z?oQzQe? z|KR4;J#_p<1+@Q|OCM-$hGU<&O7#X6SoJasmq^)>p#^WjSu~L@FdfE{9$)yCe*nyy z-+(Rm`GO5rfKIV7QA?b~eB5P6_w66W2@_*U*Ty_LtlkdIAGqgl-AsZ%ZE56JU+5on zhwDZrBE#*c*TD4dy?CX30nFWZ3rsb~Kv+y;&`v5BW=gdX7^jlcY}N|jhMzW`4kiJ+^0>Vju%HGY2d zh}mA8M}Gf4&MIw}q_Y&Sv1Lb2faY0!_~w$s=IN#L{*bO}9{tU-TSjDn8)Y3!4Tq7l_w@W(?ly6mtpiO=7P?`5UPS=BeVf-i~# zo4&Ej3~oaDxeR5=kJoV*9oe>?UTU2RJG{;4u3JLrGIkVhkQPQi`~d!p-G>=I zv#H*H8%gdXFV;HdDBm>tH+yfq0H&1^T(Nc$O`moZn`gDcdcHR)O!Q)pily_VYJQ`W zGRK68D1eU-g7}K5R#bSSFmX9Lg`ejrOE<~ggu84t2K2^5!5i)!>0bhEOv(VgH9I(e z!E;D`aT=b-JO%IP2SAC-a;_^@q$gZ&Ldo^*pc?c77AdFWjY0az$>AcsEx zsZ8$_eZpO?_n7;O{=mG&n~7|J490YNgJ+35uWG9*l#Bnt`lky?-qbAeUsWHZ${z%~ zE3veF|0>43BoBk%o#UA4DL5;<8fr}ap;*g=*mFSYjra|-43985Ws0PJUkG*T%jaJ= z6hu6F9`q(#!}pQhaDL`>+`M0nF!7>fc?jnPj@!pvm$s%FZVY>_Sr(&aDAKbgiS&5W z7p8B%Gn@I=i`17JU|szP{yjefMtWFQ(q0`R3^GbYf+s3}y=ufwC&aE2D zYgX33W>XpdJeKFev3D(cF!#O~-74_`(?+A1BZ}Niqq~qv{Sd;O`<}z>i&wy@yA?^H z@ou{6VGe#w(WWgg#^KZ54EX1uOB@T;s8JP<_U69<<5kkCupyY-5hLZ|Txoo8dC+GD6N4kn76bN9m0 z8E}5-7GmY}iy3Tnz~6mB%;_s{`F%q^tof%xgyS*P@WOmDO}q%xW0#P5)Da|~2{SMA z^P5I4D1uUE0I&97G$TALO|L3mg;NqDRH?cV%s!^zHjW{2b;dB}trfurUo}qLb_qND zlu7%Bhpeq;7h}|MhdDPT8B_+M*@q|iU{$Tc9532VvFjJU8Vn#6(;YegvmA(i-wo0s zs-(Tl0MaVo;hJPuob9@W{1+O-7|!{R_0s%+LBc}h)JJnX_GBx*xO))$dgtQ`g_W?y zPMf?`ABJtU(v0N5G)xRW!uI)gqhq=$`sxT%+m1lafqaIYd(4RFI=ADRfY(g3=Mvmj z*3McCFJ;aPXpyweBX~wygt^RdvO2PQp?mrq`e}zV{WZRd1g)3D?2>){iO$)iHK!)3zotLQ)6 z`{XU2lOKeH4Vy^ER%ue7y#@qti;;wQ&Oy3Zh?~pZhVoP&CT{f<^qc$;;vLRI`)~|> zJz9YaHw;4EdL3N;RF2p817Knc*GcTSfoESdh0WVR);vZvC|A-O2-vVa>WZ75B3FMMCm&?_ZrUg0@aDCNMvi{sU zOpCW7*R>tc+$WR0r>0JwGIle!#h-w3zykXIMIJlco&$#xB}q-EK2oE#bl%oUv~YPR zSn0gx{070)+5HY$+07z+cXMU|W{^*Fb12LSMKz;rkh=FAM>lK)Uh-Xhn0gZKP3~fR zZtcP4GnA?0y!&8g%H=r*-RK=rj@kWc0hNr_gz5Iz;iQ8kHD}s^=l2Z?BI}sc&|~=B zHV}UKeP<_m7lFg`ugvLiQ>|53hD_+?7`kr%dU|hUC4DtuN4kc0gXug|unnH{YNzzjX4DkooJK6EEl_z&khwi*`gJ+g*LbNV7LED^M=8$X~oILK2 zualoJLfW>tR6~;{j)qawWJ$W=N;H{?*nNoe3&|0Q>z-t#f*BnM&Vt-Ov#{_d7Zehcg1&~=IK1^YCO3Febo7IC zjyv{Wu{C-4q6VA0rs905L90LW_A)n&8}PdIS}LV(LpM+Q4cD!8xLsr?xI+a~uaHPO zYM(I6jU(yzccS#h%-v-0lmaa36u@t8T-T{fg8u9Xr%@rVL5$HP>8_U;vk7n6;ZGBx zCA9^9y*Z7pdC?$Mp$D3o&lv^tShC`y75qL`#w?6(Mv?iU@UAMK;{@u0UF>??@aiV( zd3O$pRcwNu&}yD|!89^|l=CsG4a3k%PyYAsoTte<4W6!4Wd?i>AYO#xkDh?{?v+Qvm0RNyg*i5C6;^=^T1~-wP16_6w?3p0G6fhfp^U@ z)Z|P$xnN_(9C!34m$NS8gUhEtJTHVEEVRMO_f@Etv4@MWZ0uu9H{H)AYAVl2MZZXVi~j-P5CEykC(tO(Wk6O6GKaiHF4nbj>^qzej~x5p{l~!He7L8QI<$a3@Bc`XnaMKJ8Cnf0%RH``Qv$ zu5X&DegzCRWzwlH^FX6l4H_1HfnWXc@FwLrNdGOtPc|Bu$jx$p+uP8o3)8`u;}FdG z=tUY|DN#W|SrYa6Dvk$Bki!M-%s<<$nw4ZI5+khf(>Tv2H zg@%8paJWmBbjAOKYOz*E@rpk!TW(Fu40P$TdmEu~SRcNA*+c@@D=}*(q+n{{6f6~y zgQx|zc-T*xR-Svp<~!X(iRrOSPNP4Ir2}};!H^jCiqb<}u4K1VzO`(^8ah&xfmS9q z^zG_mXbXCQWjT{@qStJC*EALDd&KDzeN9@cWlp~|88G*a=aU2e_Tys`2WoFQM2DT zV!`cEc0Yo?ITeuc@e*{7Ct#D78@azW3qx-kljr~6p?tc|?X3h+=#CCCnI1~?;wO@y z=l8MBT@z_oR5k4Tq68WwjFukw#k-Q00}7_gsQN1}YI<82u6Uloi%X{xjqQu++NZmi z;Qkb9#17)v=p16Znd_YtU*K5{6rojlrQSKP* zCXRIdDq&PRag?zk=7hbcObrh#1g*c>*jKg-+q^ix_f2)$*jd-K!euI5bF7kiJ6{wv zzt_T@0t@nT{ch-JtzpCas$oLX5r`~MA$vDIVXq$GQTyF%Nyi;^+SxY1RvcSJ@27^6 z(sgIqEA{gDbsxvpJ1I!FIBq7l=9s|K(IuqyNERJe@`dgr+ljjT4yJyF567SKC!J1{ zY1oM|c6h!NJv=Nz{Wm4^szT-I+Rz!K;YcJ&`D{naZo3c;4#7Ngmm(^^IF@QSmmy?Q zuva$)V((=`6bCU-^Z$l*l^QsybrRCsIf(9_KR9Vz(UGgW30N*b~No*&P8>iO$=J^uX?wR5MJ5^lx4d zb;AcB$03=->K+1|$Fi&5C6Te-@8Ih2hqo(E8P73Q@LpM;B!2E;bd`p{(ReEHF;zri zpCP6#ZZ`cQy@A&DFD7-fby0NWXc^+_GEo6Y85lw|164+B$SyLuKRX8 zb~dRgCp2+hAr2%Ju|?kuiN$~*DZHydj?B|1Z5Ku8q4hZ!tssowOLQQv@d6kwI*pQR z7hw8z9lX*_!RPf=Eb9BtNdB3@zgR0sz6x^A^SK^y+TkJm&N$674JFJ*D{Jy<<1Ai- zi#pwYW)E$bO~?5shB#kfE3;+;$7lHO3hQ-!IqT^Cku6@~OEUuJ!vGZ^qBDd*^RzBW zE!QL9r9qyy$%79s7mGvd*^Y<*;i%(N5V&~=m2@Onjc9H5y3ZUkmO)^iqaQZ3x#IM? zlkBlC%Jfr}CE3n77hVMz(h~iMug#? z;HSK=fNLLja#@X5)ZFGtVrIX=oyF5=O78+Q#o$Wlb&_D`HW*x`;0YiG4^RD`8t)&-BhOl~;38|Of4d(=G;L|G> z9&|2Z48A+k1hGU$Y^gJIetHC94AbG{DRw9Xex99*YEfV+kBHr&u)7FzBD}PDAO5aO#?<2)7m}SbV|f=^u6>EGP|>w zd9R8Yl~WXFEDNJ&=W_4T&ATz_#tO8ZSP6c^nrQi@o9Q?cz_$Kju`9j|Bigq%t>%AZ zy?VR2-nlzT@0B1z(nrwZuQN00^K9~WlM&A)`w;Z{3NxK+^Z-SH_I$A>i{{;7Voqce zohP?>@ykkCtK=GNebbMjF=|Om~Y8F z2TowT$V<+1>p^+T>zI$x)?mtcoF8rU!5GK2)W3ZrtbZIxVjcH_c)?!`$rdKzbt!c6 z!l5Rwy>H=M%x6$Lq{1YaiIWLmw!&LWRmNXnCW%X*&yP8_o|?MJP?HCe6tAjLFW1LB z;iCgkG+GCpnX{S0jsV|MB{AlSKb`Wyfm!PI3hMeSaY}Xtek<=rnRD;Z%RL8LL=n<% zbi%}@UVLIF!9Ef?jk~dh+oAVjW_uFd^jUxmcK<;0;x3GhoP@5EoN@4i3G;FM6|_HJ zOkQ89hehin$ttZoAPmRw+}t8CiCc#2#%0K~GF?y`o~)HFTZI!_f8rV?*qb|t05)9IZZpZI*8Uc8fLMNd{3fluZKkozM@RhER2fay6r*}Ob1r;|x^Cj{dX z%Ofz&Ze<}SVI;S& z1%2-?#+a|cMDu4AbKpc26@GsS(>b6D5`r;~#}r*=O!7Un$C-f4GudNGoESn*L&IY69(;Fp0jN z5>HyrwXhQ|#6wNPVT{R&BzwExLsm;E6CQL1{w==CqS!MM`5q9CI4~HJktHZk|dE$ux*19d{dcCafIW1&i=yopWyaM+`Ukz^&)O6 zGUONcsxXtDEv9+@6mW6&V=Q!9Ot%;az}F!LJ&zrR%R}d_bDtS9x1%i>jprWtw>S%n~e&yV|^7L0+kzM-bpEw=)Qe$ zxk?R(pq4F&%Oq~rKG3T22rkU|1e@N=k|o0X`O2H+h)7{3@irth;zT>tl*Hn8gQvV+ zQ)428zT|-3M*35|443VxXP>v9;adj0=gnJ_#GTjTspU!q7$9{p#ZZw*w!1(#y@X8? z&V=_;nrxjh46YY$VF}kc>+U(kOSq%Uxq0t{_M3-zAVY)8{5@gr3P_MT`$cr?h#CHV z9R^35R>S$PKcV5%Tw>j@hIJ9!3p#%+N&Y96r@BeeYAEFr8^7KO{u_IOBL6@|t{zP|#W|48DNdp_V*>Bsd&H2E^!-0W7GVAV1{ z93DCb9Y)JZ)2o}XV1*TPcFH3R*Ihz%=DEVkJLj0eTEJ259rWAn$!Kj=jvISanK<)n z(9Sv8LN|56S2Un57ZwxuyRvY7Xd#KI;Y@(9W$DhzYMA#kid{9pF}8(Y;<27=R1sOl z`6{@6f5|%3{e^H%GMsiO4&dy{8*o`?Ia6$BidS@`$O?rmQ2s~^p6YyutdL0fk@F60 zR@%`^N)w4(>_O(TX#xh#;9M};dtgq@O#A^c^svnWw9m_64L&@B%g!et?VuLjJK{nP zxrZ@pCb?kjz$p8)`X(+{P$pZBt|u3d^`d==58l)@W6%C_q+5Pw!3wh)fPLmv;A|;$ zP;KmV%ZC%*o6$^>+cTG^)13dL;E3`JvcFb@Bp>DYDbeESSsIJ`8WKs+%6Dw|oUd@U zPK%Pvet29QhDJAd;I9eHIg=zsQ!mDg1J?BUc_0N*W+c#a z2VMU1AiM{a3QnW`p}EwtAq(X$wlR@&w7H(<7IJ*+ z5B7|QCH|MLiBiRssq0+>kLy>{hsF2tafu#LdpZSH>8R4nA%z&D^ByO&X7p}y1H72+ z1LqcUF8H{a^zPTUjGLVf$CekMdB!WKcytArEW5{U-qniXDq=W`tf1*b^~ia_*_^=w zUhphIdZTDPJ#+UO{7^Nc$M_rrNhORdj}NA0b2%48qz3-LT2xHu_?+Wa?5*J*>z8s1 z_?hAh!Ej41erZ}vUs+W!n^XeHmJ607p;wHU9@M2fy{)--A0e{8%#ZcDD}z5q>}mHr z1#&l?J8QS-(5x6I=FpZ_w6ikeRVh1>Zu###+X)kiOIZNXo%x6{J12rvyAW@YN6=L! ziQm6WClUKx@es%C3l%paKd#N9A}=d3-7}KvZ{eH@V+f%|Q>n{JYktjT4@Otao2L9z zp?|Z(!QxaM$OT%`HjV8}@6!QD(bcER?UOJM?P<}P0gxEE#)L^5GR=1uk+`k%sED!( zsrpgRhCJF2HSx!BTBs)(_bCU9%4XcOE(#PpC(vv|0qXOa;SayPh{+FU(t^A0TxEL3Y{L~rM%t`=U3T~ zsp`fwb%G2$81sZ%>pZx9K?Ejb#juTnbBWVPAt>mYq1G8eNFENM%P-!;-x^b>)Z+w{ zthfm$zI^8xPTZ_UeKKAVQzg0y^Jp$JfDMB@zJ+%>v!<;G^SOJ(@7Y=8eGx^ya7{cu z@e)%!xebmz6CzuTI9J$YSMoW=io{BXqUE?A_1F?Y?yEH8)4(!3)h|h`y6&?N7AVsE zOmSH7fx!U3^Dz2S5Km}Ngz%VDa&nO#?)zbmN;6Nv`b`=PTM$Tl_BxON?yNI6UYYKh z!SP=rbwQ^8E_*}&1N<_bMwZ_fp`2}iO6DZf_4gI1;r4QVXr~~ZmN~?6*$%^PSyOUQ znQ+XVqj>p3CvL8=!Th(IaQ9m`!c$40=VY^St!a8wo5?Kldqp1>nr$LC+h)^GmrL0n?YGe&r`9Xx_ZnA{q17_rVvk?iH zl0_QFCgY-woRdu$sKts+?8WXyxG}2(PlT-j<(a`~bL=^+lX}A>FZjlY@7~BPb1Ox+ zvS_~7XeAUmAICS@fsj7b1NU!6pq^7Uemrm)ZhjZT#xIg@R~yZ z%p@)xYj5O+D(SPUW%J$Q!A3q2JnTftX{RL8lAla3x=)2jojt_(q7#{Ew*rPkKg0U{ zqQvT6Ghej3nw9-qiMCe@__B9Z>3_T&;vshl@`aV?IR_K^uEmQgmm(H#cV*_^?_zGr zCV<)rV@!R^dBo2taQ)&m=>8N%V>&Z%*u)3y6`x>ofFI3tSVI-U?xDeZ?t5soBe#q0 zz&q&;=#Wzl^UadT)paGjfaG;37%7a6(NP%RwFE|ndH66!jZBS+fLmslaA&UtSrqC? zqxViGjv+yeTjPnSd1x#3Ju;4=d-ec*uTG=qS%CevjFX@qT_kAF zXh=R{*HnoU?F1@dex)3nCRiICzR=7xtVw3OiL<9PGXfd!sW+>|t55O~ito z@MtH@Tjx)#xjBga_i5Cz^gVpZ_`|%|#yNau8Peab5=3OaFHP{TU=NkZk&-hUw^Do( z5w?E?*R{LYQ(=v8BP@Yk9^r{4uH9H(E64=&=)iwjL3H%Oap=;wAi+zW>Ag?gXzi*> z4|4v*u=sA~#|AYz_pS!neqWKu9zTzHe^hz{^0(s|jdlB_<_Iv7~71MX2)X2H2Kw>!YQw0Gzs z<^UJT9Z31GfL!cqvkqIi8vMMp@X(KSq)#v!R)#&mM^i>2_Pq?v5jG{Oe+m%~lU1zT zvlcei(T+G>zYY7`%fS5a96CkZm^{JY$1c;rAO% z{`(1&eplc>k4Sv0KaYIix)y~~=Qn-O)}iI!mojGrYEsYo+8R{OJgZ0JhmaiFIPZ*TsQwygAX$%!u1ek;@Kbe%gF}2ax5L=Vk2Q* zRJ$gIPEcqBzEw4>PR_-hEplvSXBodNX(qX5E`@$~DwxZT-=Il(3cdTE0w!{~*L8lL zY?ic3dd&HKtadWt-uE#=?h{DF>5_X!TYTt*Ug*`W+>7!Q5FlzG|2p__iUK`7x1GuVbX0|ntf;% z2D^LEpe8pu#EdJ2v6rPzCE9;C8ifVq1Wh@A3CW>`?3 z%xM%OL!T7rEZha}O~t6b-D0jYoK7~Gm4m>tmmKBo1{C{B2)V!$yY%%_`^Ga{#; zO-q>cmUXbLMdI{Yhzkwcx*JCu9BD-MHnO+XfG{;n2r_qL#UVW7aqNnApY8(kW`fj+stp{w|Z~fX&i^` zPHRc5m=))ZjG_kLo-y&y7jtvs8DyEwPY9`4M*6nwXO?&mLQuLM@fz8{PBJ6#x;~8E ze{m|@*|Z<>rmE8%^^?qpK5gQ;shV}LUPL@Z70LJCH*rW;4wjqp&}CIU9$Cq<6ZcfH zNri57dhb^7b)QL}OqfloE_cJ%f1-2;cfCQ@Aa-!9GOsgdFeP1-E;v?-{a$X^@@WAn zX^Em@GiKpQ!2pmnYh*8Ei4y8xZej53`G_kQn&lk+ZYmyei{c~~gKNoO`&mnbz?>6iFVH*Dd+0%(Ul<&O2H5W@V&uhZdUb>V^899wq z@2TLF5M#VCUW&8-r_3ImyG9$nH^I1<*J;hW5$tezJa6z|C+C}40#j3mV5^1~JpFWn z5})lQ&u@-6qU;4oh05WDySqX4KYy`uy)VDc5PmR;$*rYqt zpfgH??U%a8O9gL*mshB=uy4(Ef{L3`(+jSV#?7@cKz0DQb3ElQgbvUFe zbP4>|!Q)6B8lR8YA1UlR=W6h&AJg!FoC@V!Ov5P~2VqvQG<)?!8~>!`ldr1{?~bPU zZpUNTx5*B-m5rcQtvdcvL=+@NoPwt>jB)(8XVfty3VTKkV_tUMe4)z``0SpE`OOjH zcZKexFk~z$lt$5uz$;X{_5hR{OR>pKu40wEIT#b&%PT)nXI%0hlGyZyAJHg@byMr; z?)72x%hZouAEb-syUbyYv(R~JNu=M6+d%e20So;xjZ?9HPE!^RL#NEqtid`PEz@|u z>t_-FW7{w~qHah20_V_Y&~ut`{x<)rE{#g&TEO0rAy8X>9W?s4(-g4+OO6T!r-%0_ zrrZJMZ~6@LY!p%d=M4&~(r3ejUQ)EnHEz1oB5uYe7tqMCW|wgx$Ze2^#hLEt zVyp?{Kdj_W#8-plp=V@}wVCprC*xk9ULN@5$zD3Ttb$q@jl{(U15G zWOru(HVfy4MYd9G$-HNrIX*O}$C{rKHIL-3cfjtGL-FM#8|b7lgB~4&#|@%1Q9bc4l*aFJY?&h0vGJ#ll%+FlSem$S%yfK;EVU&|3Qf z3_hU)%UU6UjY#4yQI}5_>iAJ$-&8>Q`xk6MwmN21@@od_0(o}*=m@%%lSx(n>ge=q5DU=khb3PY@Q=#F zm{Fu6bl27JThN4EG%Ud4SHkZ6eJfx7cmS)_l&9D6@9Dz8x7236gt-p1$Ch_9nE%av zyp-&9SiZ^?zy9eWd&_;|ak3$tXVpjg(>)9q+_!gro~giAW!@Dn8WO^G8CtQ3F(zQ_ zy%^3*KLGVPZfGV_Wyb0^X^vzJ9u2a^AEx>A{p(_3etSr5IF0siJ1sW2r_P_Py;d{l zNI7jEGK)3U@CNiIo?Siv3R#EjI-XgIrqnTtL$u`wQBKAHZDS4D9+Yfl=XyV9dJRbU(z3 zN?ddK+O!B~gCPdYP^zC4e!nBbsn+;tj6S-Z3}wgqGPs7$0QOAC<*v&-3PYCG(3MnK zVNIsUA~nmoukB_ySLqluyGvl*i-pQ?OBBByZaMnUZS<;GLHbX_B-p z$doCf?VRDbeNBdFRJ|gLlYB*Y<`r=fB6X3YLOob-N@BMbKPR0FR(yisv>Di7D!%1% z8D>c9kbcWgr)lS!xau9&f-h1BRc0sR*pMvn37yKCG^Oc6j-TM-(SX2t$Rg~e+32@2 zI9BNRsP9}R&Zv7qhOavLhH*uZv%(AtbTXkrZ317BJ`YA}?1QVDmqTN%1M5wkzykcr z#KmvUP;rkE{rzGt`l_42X0%9Z@(9d+OuclTf-{){t4 z#pl+HUG~Bm99$%7mnl}r~bB$Xf9!n=g-V%%BowqkdoUp zGuao1zT6H~`-|w8ga->xHpUmNS!|i}efSgqi2gjT6ED7%O+$W9XO}fOh>Ez!&)<8O zuXA5SH(e*9dY&)aHPDBD5WN%9ze=FD!5{KGSVu~40cD>B(x{~Ce23i_ez@Os+Et*% z9)DTFOOIIhVF8^f5dBdc1z+4+4%+`01Ludu2mhQ;)`Vu->4 z(dLdwEI+M}^L!_=2Zt>%LNpPE=B>t_3`csma0)wk(wb=)BA#7kDDZSlxVwir*3zX$ z3L^(o%F8^sGh_iIeyN76JEi=KzzO_S$N4mSXD5FyvJtv>)^N*$2BCR~2{RXVLd!hO z*~YJ`V5A#B%MHiVc&!KgUBwo1Y!RN_Ll1jo-w zQuo6s=KWt6|9tl;zP?ppU8hav$1l<+E%8gf(Krkn=cRy;+B?u$6TyC7J6ViFqTA=fUP`AW~G)OsiIF1KLcI+xLY|4P26sjg!X5W(oe!M$zX4D>?--1- z$)n*XW>GoUBkW3s^5X)A;bS2m`hMzKw(|2zRvqF_V^571?#~Z(V^hPEPQ)D3s+# zO=r#ehv2GO46}Z8j}N?P0S1S%X|l2j8#wYR*nHBW{VmsEzj~(V>q$>gc|Vd8I$hzw zgA_U>^i#Ud1I}EaO0uFsY|^Ab?8%$aFld4yGi#A&+w2Cyg{3x(qR+q`$4XdnPzk0x zCX-C(X&COFhYG@ToSG&D(pLn2_`+Vexl#_pZ9K8*!(Xr#@_TujMb5?LW;k46twy}t zM(2YTqoTqrkeAn_wCpR85VwkJv{{S)y7rM%c?@g~{Y^20GT^i@cbzjIj7sAKw^r7C zY<5p&$%n@XJJ?9vQk^HR{J25XYWW-T8^fmNQ&ZUGw86qvEL^wEC z=)0PY#Ww-FxzOKk26uKI6tgQoAgiC#Nfs`j546DNKQV>Nx{r!X??1R|(8@sy;4;vAFGeJ}us3 z#C9eR!c|7n_&Ll6?Y=yR%@4oQ*W)4_BM}N_l5e2lK{^jb&ZnRiBS}e}9vj;H^0%9?ksYW!T>xGfCBZ zp1@TW4FT)Ai65Z_HZb`l;pF z6#Ug|%wjG$fK9~|wzRo{|JFDadY&#L$lvH3Js_3~UmS?j_xs{y;qK%^NGfY3u`uI} zBQL*L=*#be@oXiisUxlSGh-f`MeM3-F7_sGhPHK+A^Fh)Ui0q(I{(2L7b!b`O(!BwL zcDP{AA3gkU(k|}ZQ4{*)^qj7puxG1fQb8km4S&8gmEF9~bH5jM!*Iv74!zFn}O zxG9M9&!iHcu!|};_@j!?0q6;T17(J3Y<6k{dfgj|e#0|C>vJ}xa&6EobOpXFkEKU- zlhJy+DrE@FV~wy>>PS>$2{)$E`^qF(UEN8~lP^PYuzz=|Xlys>FF$@jYhGjL<_6H`zjeKv|lOJGKAw&>Nka&QEFoMokfY2!*^%bz%2 zX5&l&g)9f}OJ&e*%opd39sfAjq)2qyIS#Lts8nHC3Xrsr@k1JR%F0vM-@m^iiDaI2Dt6+ z1u%471Z}rI!kw8BaD4v_UScia)$(63jh_OtiPMDV`+@4D<{$O z)#F&Bo(iUm7SQ%D!uN{GPq18i6Q+6B@Yd5;VbGOVurI!ddgDttpYCJu{Xh{vtFIp( zXiQQ&VwQJ)RPEt~kfVLbUBF`*yYHqdZtH&qTf zL-k9_Xpnm!WM}8mV{;x16b=%9dk}sYWlM(J$FqlX{rF!CH_>R5ZqR$G%mSlHEdB2` zS?b2(+f(}F)7Zz)i_xOGH&J+Ym=e{wDWl%|-&_Qx^FvoDGQaDWAxK|`g($Rf7g~;z z%p-Zo_Wucsj0WS(AJ@n}X9pdC!AxOKHZ@-E74;3M=MU;AGqK+(zH{yvA(trhc|zmG z^OT+07(FFiG+2wDxjT;6x}bt`a&cnUuhrb0?%UjfnmN=|Cx^ds^=Mx}AZk1_LixE} z&>N>nvls56nyC_`_b-`pM_q)C^9)lQ3<56^7KnjBk4mETR+FH0BUp8N!SUSbQE&v>xq`$C!7Noo8TrOR6W3?j$*2l-F; zhvBa>Wpv&eM&oY|z}n+N$6o#|w0Ki# zxX1?@EBDZ^QBusz>^$|gZU^NnJ7HApbiof2!AhoH;xB0~hsph!FvrgtkKUM$iAg)a zZ+S2Guwn;%(@6)XQ3F^{TOIdK=bor}OE5oBeG-L@vSfV2K>Q%4DjFde57T$mN#XE`W)5&(mZObxQ%5OaH1FoUo5-3@(aem;#@#7*8IG z&R8$%;qyxtQ~mG-6eS%^Ycwh;!odOOSKG2FyFA!#lpvoQd89sPFk3fmE^D&@ko-7> z)}=k7C7svlMKiQsQ+ieY<&T-iP+)|2ePg35)y zoBbmQ=~ZV7=FMXJ9NNKS{z?oRv6Tx*E~FU|K=TI0Q_vS3Oz+TSN`r^Oj2C+BTWK`* zCtV0-zA8h9j%Wu0rY>B+P%=frKRW&uBvDgrxjsBl7VBH!Iym$qF4kvdFq_0q7Rli zDzODmZSa2FO}Mng0SCH=3jVoSTtbr?u8?Y{$Qc2k6E+%CbAwpgsl`-$bqQB=x0Z{l z%@vtQjAU7>^kK>$3BE_I5axa)@Gd$<=ZlkY@P@C>eA8h3@nc6qTYc7j`KZ##MGDl^0P^+vqfGqCEvas(PAX?~& z&7P-3VJcZnQnlT2sn+H4JJlCA;WPH1x_3x)wS|@LPA%nX(Py zJz5$tDSSU@H=GCM>yJs^Ish()yo0tg%w)@J?xSrg zi+*Q}TL0EC^K2XVIzAh!*Xy&l3kC?j*RiMuZfw9veVV*&1om!?Vty4>v;`_?;?0Q| z8C^mTqo#?nY=Tf>W)}PT>L-{DKfryjmLqkcugXSVsb+WhR`d&ukp{N(oLmrBRCQBN$?#BV@*fbBoGAnChcI&54@a zhd*MdOx#a4$4>IkrD{oUlpK?pe2~|5`~X+~e4<7vV`hG!h)VoyaA8m?%TSsICvRrM z?t6pS-kme>#U3@Ndfq`>PN(9Zf*QTK1q9d8At?^m;m(eEH^v?AlisnaYeeV9{M z03(;CqUo2)w{Kbz) zp)*?(9~z06mB0y57KBJkwn6 zitsL3#sqfG*|nHzdqbE9??KhGFQDNq0jmQS({# zq*e?O=|vPg>^Nlj|ACb{1x)_%8j55(Y*<@5-OTX8_T3tEE!P^_qxVBWfHRkor^ez> zd%~~_=`iE1lW>RF2yd=A(&F##T;CdfQO7(J;?lx#zzs_{+mS+X!aPvzZb=8f1~KJt z`TP{p#%}jEzHhrG-sl(#rsD#+5z;qkWX>37a^oGu80mrbEuIo%m2gVUVNQGeb}}(I zB2K!YLoY+^5XU~Gm$CJ*VVU5svCrfWT+WA$n~u|x_oHZ0Rx|7^&*V!I=hNT6B5rR* zKHvIrHc4g=gVsB z^FGo&pkDn5l9OMF-^c=H>?DAPAXYg?GGEAZajzXCq{3so=M8 zH~YxxJ>hvT*A^~Y$nqtLqPYu)RLRrSj%?)-lAkYu&=MmyR(3Jxx$_p?ktw3`_Sc;8 zx2;^m(mwL!TA(H50zBU_Q0!%pChY2@aZ~q9(P*{1+^AYNF3qWtu6@}9nm#(zcG3@8 z-v_}v*FX%>?&H0pZJ1)qcz)*ANH%h020!4WJ{Ik8WzoTfyv4tJQ2g{Eefw`Ash_ZA zJy#jjywJpqWPdh(wJW=`YXu)%_JIx>h45$gFXvj0Te0a0f-kpVXf6BD4fj+I!NL`D znWn!gjq7#B>bqqG>iftJvq-=4E!VKS4idM{VWUjrm}0^}hB2wIt~7^c4SfgegzjzH zK{@V%<766AWQ+wLMzU=Ka>;Pl1}vODOyE!|k@W9qT5wneKb+4ME&I3;LT(F;oXy8- zB`4?d)oaUOj;lOQNE32LEJNJvJ%D}bHfD?cG^xyQDoYrcO0|)mP`IYnd7l4S@cGh0 zm&b)bmt6rHljOllKW*pc3eJM~dFt#%gy4ZmoeQQx3uvP0ZPL1Um>lcYveC~hczYp# zYWw>D9W9jS=Z+8rMiScW^s_3;i&tl5AIGx6tH-l~Ps(igWk+_neK*+c)S-)S8~K|a zmiT?cZ{AJyKL6o#EPLoNla|=3;^IAj#XGLIlc`r8%Y4-g0j~`4cCjC>uJwaB_aT^J zlF8e8J>tbNYOEvc1Eh9^Q0IzN@y+IUpc?PbSyv{aSagtkS*3=Ne%r;L7Yrr8Pl2pA zc!|)--ozFxTtv512Q&A)0Bm^L4!TthqWi(EWcT$K`8esYb}PY0xcvs~R_%m)W+mLg z?s_VfHNw43Rxrb;fL+uw5lIAhbB%*GbE^{k8FwNH)>^&cV?QY{jct{p)O$76xC+iLk#zU>TUeB*j3utYApI>0rCi+j)DKxC{ZbLc zmo-5qtdks;r}95#FLOt02Xd3Rn>4TIJ+0fDK_&+aM4F={sp6fGy-47M{l5`e={T_S ziHfwdV>c9+C8NR58u}hK0f*R%p|@c$-rGJJZ~O^ma_X;0KJ5&5K$x$kbgkI}uZ>td zLl!GP4aM8~^H68D6I*oPIXCl>BA;m#M)SoDu#*4I=?%-L`7`&E?!x)3ZcR7+I{2LX zJ2(Wgr&=?IA5Uq*Tfs*(_5x(YNMOCR8m^JetCbMC)zvrS*tn8w5WRjKjgYj&H$twx zPv;K(mpK{U{W(oqu9`G(=t5Sxrx$G9?vnYtP~^|7z_!vHycyt#{b!xn$#?3QG&z*L zfA^lxI(rpL53LvSeeUdI{bVfPn$3GWj^bv9HuGE3o59fZ4}Cdij%fkreEZb1wAWV~ z+@nuYYx`U_vg|GCtg^+m*7wBr$2AyJ8p)P;E`$FrzU0FA8s4fkooU?Ya=IXJBL=9b zh}$btM9ONzL`SmBdE<@C@y*W|`t&aXu6!$pmYbQi6QjhmB>fJJknV%cIo`Zx<6!35 zV2OJTRx;@~SExiUNBBDhA@@F--KAihe9MAP1nr{0;j#Gb?l%5KY%YJwc_|)nS7Wad z(%Ig#JE7T8h7>yI(dQ;D8hh+Ax9e!DNMf@z`}Dz@jvfr=G}j!X@MB`GY>ff>SE-;) zrkLDUMbaa3#^eA2NiYX_M&C9@S{CgVC^2FJvSxJ>w6rykja zyIaDbpqayp8unLc6)zB7;%^;Wd1UfF> zhZljfz$-_AZJAR~#w%;6Y`Zf1+#R0;O5BbW_U!A5QZ72?8C2~QJT$LT(Di}PY4kh|osPpqFE*Ca)!BzB@OU<9ms+Fx zHbZ8!Usm{RNa2&9JLG47g$yRfF?)f#u{y*W{MR^wdS4HBea;d1KEo2PbuMMAer3WX zqZqg*=Pw@nU_aOQ=mdY~Y7rMzJs7@yYz4MY9^;$i@Y9BueClQ|T;&$SZ(X$%Cd`bX z^0@)1t5(NJ#Rs8j;sL=!mO!P(#@=z`f*PRFZ<*sp= zOApYGM@3u(b71b4(L&!T2rlFng8Z7ly&4Uo={J?wnHfgx`O*Z+IIf09k|S}*j8aG!ynT1Fdf@s7O>u;WDNEs} z(Pm%9ZMPhR&1dhEjlyH<5D#YOclxuLf8=oZMmu(nuROOTd=B2t&EpR*;o+vwOLG0z z&OP2S8_l|gvVPmmure_K6$GENcB>_7IE@j`n$zgoN?$g5>u3y~p^AM$mSSm(CAh4p z1HRs^qyg{8fULX)$FI#5?|;9Zg04mJy3swfYmYEDEIdzI`#*xR;BIu)pTWPF7R+=z zk3ib2E^xhb7l!l}ItTxJ2Q*5OJAPE2b+zo`#{V>Pu9$d)Yf^c~8O3@#*ECILMsB=# z@ytrPQgI3H*Azj=87J83p}^kjh47OKWzl?-8YZ6<(?hdmpzBmid(Y>v>#c{tcXK+r zygdPuW|@>(lok#1afuyP{-c}UGH|-I#884d=Qb#iaMwnVGO=R{2KCyuC_|D#uuivxaliRme@l#Qi8DKrKQ-@<0O}E zd>t-1yP(6$o78f$9-2lr!*-Q)au_Iy9tRNH)D78?XZJ*B%lDJgLkZe8Sf0%(TgJ9p zyoLGS4uT*yJP#n0jQAzz;&z^Wz16p*Ie0_(}st=Zhl+N6n9!ifn+! zPwE}>0`i`>aMQLu;*^uz(7z~6)d*9;K_EHZsHPzexriV0xlpdfjKHQ39P3O=>KH|Nk{T= zn8_m=7^uiXUk_%!>Vuilm`)mZKay6HyZGl0C0td&IMvgUY)9qor`{8pqlEQ{)tdUFRjiEuS5IfC3{G(0H{p`AYub zV^e?gul8+V|DI^F`-x7>Z7RWmx$3E}V zfD(l?K5Mlp>!_M5-s=!ZM_z}Z!&52dd(W5El;=T#5#t@ZD#rIaor4>@ufT5-mzOHJtmsDhU4d&tSLmkJDdCGv;N{ zPWStcQs+q-*4!hU9io~!&HZYyV}~7gml?6t@$z8ozl0=~tz*FwvUu%+BU2Q-mVfS8 zFoU_W;9I7F3*9Bykd$#$o-vJiTp7bTZkkR%g#79pmDeQgzl1$;GD6>sGQMYf8)UUh z;FEuo*l!sjBY!!GIrn|7jg3nOdHw@M4lgEG^J(m^^%77ns;5A+{S>)R3OD(_f#VXE zWRtE55qZ or3JXeO`M_Dt)?Z&<1)f$iHAvHQ+H2u+Cqz0TF3x;_;~ zE%*->3Nyg7%ld4^wlNsk>x&N-3}Uf)x%fjhoc4Hap^g+|b}FKiwyqqFDo=Ut*}6kC zAfo~rG6vvs$#b0B#|U;x^hmtv;8w6|-T=n~Ghp27FgT;<#g?oq6!&W$r_oj;g!2kd zE~}havdVR?El~$MZXc$CpIfLlXcG(Cslhh>%fZl?I9fX4C=C3Y2-Bx&Vd=cD(0uVL zWk)&TB-w4;{Gk%~?d&jScus-Ul!nsAT~kTr-@tR1Bb#XaIS2L$J1Ny;C~mpt0=|nb zLE*T&T=^jj_V(kk?n|~=F!wf6`n9)TKAB;LXgP*@?0(&xT;j2F>6Y54f-Qt=$AHO$cU)Y8D_3pU1G}ncVC7V0 z(2N;GlHs{9d;1IcHp+`+mbb#Ir`yE6R~_)^fk7-L`XVsJRZu?oAL*@Dz!mY@FjC40 zPxXYbU?n5Gd9sLB%cQeOF}0MImLbC+PQ3ymKUGbHwg{8C!pU57eTIdlcWZCp=F5- zg*fxT51C8-5kk;^mozp6%w@e9ZtR0?4<90QbY56qAT_UGvi$Oy|JvYzu@eN}#==I@ zD%&f1)#-xe6NJvfC22T0vXf6e>j+H}1F&sZxRCu*fnh!VIAX3W^Jo5WywMzm1Q_(( zH(%?icuBhH)ZBQa+^Z*#gcRGFj!mv252}HFO(@ zg8R7&&Kh=;dAcReeLey9HV2TD_e&73b;XU(zCzdrXN+FdMC(pIgA+p@^4D?>lBOvm zqhu+z;dB|ujO(MYoZ)=xgiD;+e=685_&sk~Rq_9v+)=0FD`fco;7@$upl|6hegMvZ z8A$`!t+Bo=!YGT6s4SpASsCw3e{2y?8Y=-Z`zEoH$T4EMqw_&=qqK0(GLV%lNaY~t2zkmVQR3>E0%KRm zq3&M+HxF+oQ~S@{zqtjNf6$n@4ebT{`W71fd?CN4dns#C$buaoJK&PZ6nb=cJd0O3 z18-IqLhUXYY_6;Ti-P&wa*KzY-@734f+X6-+~J%4Ua{!RIcj4xs3>hpBGOK@-; z&4RjzFJQ~PjnJO&!CYh~v(jx7$mES2E10Lqe5O{2QXaVAn$ziYr%;Z44@dzG8+ZD_ zuL3ReYW{>)K8bFHvzj?SVbC5`c4m(qjSQ*+iLGB@K5q^@i^DPEd78tIo2=4MPrYPR(Y*~0J{;)Y{h+uw4aY33XYMt&1$-u zuFjTvPDRmw8`$7IgPB^_cCKR1d_Gz82Ka53rEwP{ndHw)H19*V$i}Cg#6iC~cBP(F zPq^W?E6SMmAdczEmDBY7uLQ@$M!fp!5|`s7xap7op>J(roNmTd@$AS1G9B&-Bg!M7 z{he&q+QoXd8bZx`)eI*cvoJ|dp|%$vK}=fWzzj>BHX0pjvg znylUQ9arQTD)?gyp<$qg*n0OHTBiCK?wooCNw?$(Um4)@>#FC<1gC0cVF|tdo6kNi z$N`s~gYn?+ZlU95!&V7%<1IaF>ap2Ei7LB5TQ3YhUfV+wS0uSJ=hl&9cM99_I*08F zZl>G+wehCCVPLN*u&4uKFyCPV6~A=CS664SO$oj*t6%szu@mTE^A6x_|Iy37%XGx0 zhu4kd!QK8bcd{TCm_{u2k3CPn^-Un8qlK;q$ARR#Ji+_c4Hj<(vKXy_+=6K>6dpVW z4!K?j8P8U6nEIbuw|6nz#&cI-oq+@A=FmaKv#Y3I=MCA`0sA%DiA9C!p>6jUYKYeY zr|>>_9yuD9pZkyAq*lW1r=uu)(|C}1V~)Rq2e3j@HKk_mpn_z zrD$CvlYDi4%HMoSzSv0q)>3FF@K-iT+QQ6lcj&~}Lv;H4AU3rY1b0vbjPzJ6%#_bT zwC*$~`)n~P4WC9vi%!y4&*!u)vW&VSIau_05z8~q6~Bq?7d=})8P<&b!CS`1v+b3B zWWN8WD94cp+m!-$<*`0ntk)o((C&mOuO0BOiasa_pS_Qk2Ou;#l@*1juvnG-+@tp) z@bu16n%W_}fS*3%EqXTKrH{gQ;j0wbmo}5iU3xiH^nf6btze|{x7O>_8a5UFabbVw zz>_m)snn}d9Hnhf?HiGPj=4>BRk=8E+)$?Ll#jYOBbm%$qOj#r_^XL%ifjpa_(O@yGi`G8K3WMCmvD2Cmr+l!2czV52% zRzwCD)~*RQgFCo82_HcGM3r?-5L_~0Wi->Q0QQ`k$P$;tfY&!8G_r^l`~`*8AITW^SdA(G7QX)G3Xoaa^j zHJO6#Bud;B3R^QJF-7eW=UTB90)12Q{FiZTq+dR5I(mib`(;p4H3RH!*rK+?TMn3Uj^Sp%10>Glc(@&mGFMv2#g$IN}j^G>f&#~ zu{%;*@Iu^l&WgUvmtJ~F3iSeG$y65KKAsMXUG~wKOTD5J?@C&Gfzw;!2WGk)v?*N(p zIm~Osr}6P$x*<%+dm0wp;IgG3)2|7DUmTy&q0h@`u(bhm99RX-@7~da(SF#acZPdb zumyZ4tpHcewV*m@18jX;PTy(-UctNB0%v-+JkZZGD_-M0W+8O(D#97sdt(clL!zv0v*Zta?E=;`Jfu&_P5i~Atj_z zyBgGcL~wWPddL;HI%bw*P{pE$6Hi*kDn1NDVfV%-o*7CPk{9yw(`T}__Zvy#V5c~; z@h7+9pbH+mT1i@ig*%qo>HL%Y7yMfLa@aN}l3ev<5gtrN<=qPCel-|Pa+ZOCqY?fZ zAaE)|LivaVfoKqa3id~~@gF^Nx#vyC;90XT>&*5dz2#Qq@#+Kx++WE;jvR-oBoDk2 z@{tZE8R3du$~0EGkE(5*;YE!Y9(oSs7X9vr0|Mwsrbf)O}hD<&@fmOW95^LmUaqiy* zUi>v*ypS(TgGS{uEB$JI=QRy__BnzkFWE(3(*5xaeWhdR`lxy-j(Pj&fd0Sjn%7_RYkjA57VtoJ-JpNf8%$kA<=!=fziB)z~71Z30iI8rrqfAjZm` zHr#WdCS!T7akUvQD==>_Zc(JQ8HxC3&lDyyR%OF-RN1~s!W~FuGKMYhp}ILADCOo( z;tT9p`h;$(Z(q--Se@bmmGExj09-3J!O!0{S?U#iJf0_Db83l8IQ2!9oTx0c@J_^A&;-?f}G^wMD4&JSn*j-H0E&o&9Hm3{D} zOh&kKAH}B6NM#8UZ|Lo-;VjCp31seZOuFwm4Be5!pXm~MOkhFI4kKW@#vmMHn@lg{ zW`oLsvF!FJeSDiW2puIYx!1w_!RE6Reztr_%5w#4qly9BJI4b~)-3>Kt-Y{XBNhLC zZ6(>@XK?7ECRAypqr}r*?nc8=2wEj%A5xQCfx9fVkQ0-mcHkZy3!RSp4Fg8y(0!@BvsYh+^D*bymDm6-9AP;s@ItSYq6M zF16|?mmvKT6auHg;y!g|Q&q|@$-B-wWGHceb0?tPRDogiMICleSHLjAU+3rd1r*Ir z!KkE8nsIRuX;viQ?SfyTzty{F(Yt!kSGS>--ww=bSrKKicKC9J6peg) zm>)msG#AK*P?3oPT{9@6%VU>ORLdrM-L{4E-x9)BS?%GO`qkQ>cD6Y0(J=l$3tcuh z*9zR`#tPlo-Js505?{M}By%%z-mtJOa{@dQsq8N9qY|dZ@YEr?l z?kKpF9iY|IrlDlRAS?{XVc$zX^Zb`X@Y{0;=7eanQSWlG-W<{U65_VVcJjHH!Jj%b zlPTRvhxf*>;ZE!`7`baS|L?vE^A4TKJTsKJGBX)?aCQnZMQ^I!v5g*AFQtp8h3s6Q z8QZfk9A_tmvgB`SOn-GA=t-1ASgjYN{XPdnO9#`6pPdx<ueVJ`xyB*sj*2Ck?g^L0eD^A6BFv< z*!*yBH2V7ld@kFP{hn6tQ{+PY>$HJ9gHtfusE_EqNN^trKJnVaoZi1n&>x%$9m9iI zxZ^)~deEp3lC$&4lT1i)h6PT|A#jl(I?ay45V=Tl4mbZ%&Y6uiN#Y zK01#EEcM}DELg?o-aG;APl4vQEMsc-$77GAOeN)N0S_yd7lM_m`S zRPH9;C7C4+@I>X!3G`6-+*e;*Bo29(3y!-b*#WgC8Y1KX?|(6bvvPv_>~I(@h^~h{ z52GOG?oXPwekuF#Ba7_ZF7XzYcBr@hFsOaLKuK##Xm4s4Z~E#D9jtPM%JsJ78I;P0 zHaX!J{YF~$bRFFv^$Dau=CX;mM$(#6CQ2AOfj5-g3p=};`2U2A+;kmHwpB6+=lu=D z;pKC1^D0Zck~E#kjuCc-Z-!$^mJXgcq|e&?tGKaEmIC7dsiKs_pV8Mq{i7ahd+I>- zcEWrj|AD_*9L}l>E4iQB&kD|*?V_P$N3cKV&1tGmGiP|ejN*iE)0YCrGR7(ne2?7V zdT;x%ae`khAzuN{2lL`XT{E!M-iIBCRif$b+IZPCm7kz_lv{LK4&|e-@weA%quMl% zk1#U97LPFW_k0HdU&|?P^(nf&@FZ=Y{Fc@<6@&VewW#$=l0_uRGB#ZW%X0mBuC|um z+qlE5gJq&G3c)1zIFFC<3x$rZcz#IHIeI-pcwcy^vbr#9*yB8l1-b6zUY!5Riz_`z z#<2oEDShMRFZ(&ms)fVlr?;T*f$%PWDCS(-p78drM4CAQkvHQUHOIFNONB> zbF8GdiN$~w?X=9mhWl1BkStX+ng1+loOwK&yDSw*Yc45c)tffR)#KtOYdo|v z9bUvXLB=fZzMET(x1JWm#FVwPHP;<(ifZ$FKmK8unsX3(Y!w|?o=JwvGU?*hX1ti4 zLY~Eqpw{U&804j}17k@rvwb!>@rTPT_x(hXt>xG^r5U~X26#5*7k|#F2fIb(Q|1*<1~J2n_#%ehKZFm} zxcr;x5~$Mrj5@_3MC`%+>6U>C6MX z$>+Ts$AVY3OuY0Uyem0{3jHqNq!CE2DCR)1QU+wzn33{PMY_T|gX8@Uv5Qvh#0_PZ zRJL9WRW)5{Ti!9&dLhS`4U~YeC8M}CDjv*DFT<{*y|^L#F(ecU;?QAfh<5H{dt8>F zYHAmz*JRP-Jcf20uEh{>adK7TG<-d<2pon2NO)}yh#!-sn>fCY_zVM5u*QvP=5}sO zs5ag^Do87MUQh;m89nhEkjt^N5`-vvz4s&QOjppVa}QzH0tfP-XDPZT+=4Ep0W?1{ zm39cokf+fb=#@!%WK>`X1vbwl!!keF%|CU3K4}7t$7vXSyAzg<4&#+kcak-jiZ09q ze3}^n`ELkeBlckj9>B^KBD5f78r}8XldMo!&2*hxL9G1@N%rAlJUO?NZ5;j8^npGG zN6$Xkk|0WRUOC{}EsA{mICc86^8@>^N}bV`aU-t<^zo*?3%kc;KJTpHSDvf%L)hW; z1HuIJ;DcxoX$|{;i^F(y`RxT%c&0L$Cu~A^9$FYF%=IZxUFNc0+%76ilxcD2yyrWN zkdYCCe;-7Njg|~OSENEi_Ku>mt2U$FevQfdP{g0xya!*S9%-z90OnSiO!rJ}2-!9Y zDOE+FFUe8xU-)K_(U$(vO?I;#0*E)X204aXYS8 z>AZ@G^yX0#okVBroMa$9l`amCfs#kpFr&#G2Mq1uVwoj5-?^5mN#|k2@fUbxsUFE! zOylNd)^wKKeOS9jl|0F-hyME)nD+l>z=b;+1a)e#HB6g)TQHB#Q4xZLvsCfnp1@4N{CH}X5aCIiR<(Pa%Di0N#mG5t`EF$nolX#MhtUViW*+{^*iW2 z!eybSR>QZ~hqyh^EU>83hv&6*TvAwuJbY|Sx(uaYR%sGsuWN$gO-eM#KahNa99XWo zow%&zQAc|fYU5st3$_!3GDat)V6z}jY-`X)tW+@QI`D55$u1k`?YLG=o_Ve_O zQke3Su~4nQ1_%8f;iM~aWWV+}7Pi@vLfPFcTwbK3GdjUcer{$f#ihDA?n;cB5BeDoTFm`H&pGI*zi>tRpdc^OYY*pm^%&R z+}Xtj_t>e!CU7cH5L-G%*v%p>m^^a`E%E}{j)-XHatC*2Ty+AB{v=R`!;3i|!bN`d z^eVivV~p{B`Hi)nZ2>-e4BE zxEbW6@<)hu{)f*Tgvj&YVaEEc02$}zaXy7obVF%AfhzUtXg-w0c}Ia`NnmD-c| z1Ml$TJ59L!+mcwD&Y_2oTGFnt{p`?=dT2j^_^#OzR4y0eR^KH0yj+BKzZ1f9oXd?N z3+bB$582#n=NTVi9Xj!p8cEum1t$6<_-o=STJtWA_MSSxZ2oIUXGJ=YhLb0GDQ1)5 zsag~K3O&eNmz&ACG*eiMH8NDUTbY!1CXvQ6FILMjj4G^+pub~F8Btpu+W7AnBrSc6 z0UD=Sm#|{YV~xr3pF2s?RzXHO|016EoP(F{E~O`ZbfEc%B=Jmth7XiXsl3cp>^?h< zp8J-IQ`^75y?Y$9MR^&VaJtFU2VS5t!fMT_iF*S0^!l`UJh6GJ! z8yDMh`N?E>=iUn4@^R$)co&`x^=8(E9cDMxS=g}!EUxY@WGuBUsC<_#Rkr`l3r}9e zGzeRP*7}8{p{^fWcD!NZy<-615?n2=1Fw&cuzlA*qx7#5w(H;>4396tf^1a~SoaDZ zUc1h$6ugYHe=Z`1JEpR$D(mbb&npupV>z(a?F7eDYrt!h1S}Z6jhj=%!EMrAe6nO0 z8>A@%k9$*y%IFcubrQt%2T$QcyEy)Pt^^-qQc1NnWfngbrF93Qsrkz9Y|FQJx;)2; zc{0m`eC5Bum)@_DQEbGnM`zgCQ*z;)pbg4Sln1pM&TCxh0pm*_Fwr*a@%q6{kjTw` zEY{0{64xzrjX8@My9nJ0e_-W}=V&5*muVaAL|)=oHd8H%s&(+#d9~#*ZOMCBaH9hQ zYliKrmc{Y4#xn4WZY5-x_kiGGC910?Ov2wUgMvC?QeVClPK9>B#*z%O*2IgrCg))G zwq)3PF$j-!jnpBjUVX43>osl~NTB;> zh|_)Z_wzbGxS)Qm2MwiB`X4mk6E_|)?ig9>5J21Mm;CCnzS zZ|d+zkUm<@hs5JkAuZgIj<()Lso8(Ar)vrvh-<u!5Yd3Ge{af7VXvIc$dTJtF9coYRKKKojxcN`-1!p3ESb{En7Yxdx3XDP3I`TOB7x-*_1l#xZVneU zI+gl!4Bia-!GqUHAa}ABR^1dOL6?re=4aCQ`PFp%a^n$`qsF-?Ke%D%*}25KuLKp( zZXo_bZEX90ci1IcEosE8v-oM)6BC<$;{$<3@o^o@> z;Af{m*Z3PAdL=^cb^T%QG-%?XH~mmNdp_N(E=zM9&tS-fqju$!;$iFc=jnqvWH#}(MOF7zA zzn+xnhEtnUawN*I7gkzj5ev$|sqyz{+G0T``lmqk+Usz7Q5~q%Ol8(9b>g=fgUpQI zv2?G)Dn{LJ3mMUiW#1a8GjpEGQ!!>y`hUHAg(Z?8a{b9S>@ z6NRXd%_sggx3kP4nOBg@&G#i9rqNHU!|8)RmZYy%iM~*mq@~Zc6R&AMV0ec*&ROe< z>Npi2+_{U^8*ecF4>a)$o`u!x7ohKfJR)1@K-NFI%evSpFs-dSsez~w(LO55D=Fjd z>4!q;j&>(f@-3A*sWGs1VHj@8eZhH5+8FVYE(}sH0J+U;!A8`SoKA2cT7Gr#pYaOf zweu-Ve)@rxcJZY5bR>z__E+#Ox(VN?{{VvsEu0wgjF+&>kM?spf;Xn0_=-_tr02N> z9R@QRWTQeO?hmnzwz0I;V*wT2>PHU-rP1rH1%%h4ijEv3G5G=?HdYB@y#7n}sInF# z+V+L5v0Xqu|N4qUoqcw^#)%}+tOrJt{=npf1DG`M9B!9}5Up1_)GqE0gzn^+4h4_# z+Ky}(FCPP!)tcyDwe+<&`X4F07AdHz-VDPjI80WZy&!%wQ*$4SV z*C-dQx#!ry+q2lpr%*@IX@wZqPf(fbKtB>^z_J zv|Q&Wz7`>l9&PNN^FP5p$dxvq;qK-d*6h#{0!4qPQG@Jh^4gDW84Qk8TqyU6&=bq8LyC#V@;fH|9jG3ucd9N*alS~Do-CH+9X4Q^zI ze!p#+za`_5Cr6${Poal0GNESgKRC2Z7M6LL(3(B@G*yhj+cM(xOxsRSR7|4_#iC$M z{uEMKp~$x&)ohO+L)tT+fO@F~RFq$X@(n{6mLiOQ*KoO%if?Fp+7>mByvD>pD;zO@ zje@oWFZbe}dE!quM+?ly4pcYOptAfe1r15F?sfMeAEtr?3f!?O! zjKMN4-z>t-Jj!jEz%S*n_C*2H?!J{B@4d{l-_?SHJ9`)o?~W>K+A-oIp(%Owcue&g zV|b?*ozu<1cPxP&v(;p0O|q=sPW2|pix>kUIK}+S}=RAE4y$F$NtltPo^E! z1nV{#_O|V5n7U~qt-9C^?NWp6UdeNf$=}w{RniA>+*688*2=`O07B-xIgah6>6GzT zBMp6BF!}pJ>TS6fC@&5|w#y|fc?D1M+FZ-Qs{AGmc9~i7V?oT{#rclBo zbody;qv<4GAV2SI;5eHG(i|(kn)?F`8ru1G&a|IJzrX zXqA#9x`KP5{l8i8FesC$SS$jzjfK!PC4-i=iZVt$%iw|kIY{n8{BN%rHSvsPZpw0= z57#h~dPA4pc_W-2iK<84?f(wGOTq_yoddKERDh3PkMFdfrOy3d|W(AvA{19x~D-C3lb6 zt!Tag0dLx|Z)_6l-!uqW@8!tL$Z7PF{zqPsXeT@w*JBUNnL_j;5980l3XIvOhTnBw z@?)1v61{IFs3mqAg$Kv+=DsTCg^wiFsk(&u%{eH))S9N-O(4eESMk?A0StPQ4G-tG zV`X?WoBczK6n{Mnw?E6VX;WJmRXuy!$5W;A#We8T%JYm+)B!l1F@e~cv+&3HGxOmy zmtkv}Mxs>A$ejLkVrn=5jTMhzQPeqBan@=Y&z;3?wRb}1SwS#LFd~aMp3UvRTvW>v zBlgALz+5&5eSB}>9sf0;=q^Sgb~mtI`AW>S$a_rDD^FV85ka2wk72Ti06qFwnRary zOqt;*m}TKj>a^sD^8sCIal8UNM&7X9RY_o`SA@T=ck*HP2+OQBqVCCjJW zEH#a6Y}<^-ijQLUfmBlWMuM0VZbx&;mHG1NApHD$5oNBg-j1*scwVVwQczRvoXDLLJA6YO{1E6N4V!t5Iw#$pKOhOg1*O;iOt+4sPOhY zW03fb-*a^;Jw0Uafx0BkA%lAQ9DtQ<7km5PCU*Zb zIgXj`MlDY%k=lN1`e2_9srfJgZ^|gpubGd*YAp|%Vni{@+k^T%I?tM}k%P_)TExAu zmdq4y^Ee(Oh|=m z72e9}WsQ&I((3}*xP9j>T(1vsSolBiB+1}?ZZ*evzsi&480KpfHDN?ph#okkN`}o8 zseBKzt(BQXB6ADLuT3S>GR5h?x{1V7HjExGDFmr!>5K$?z?pHP3x*(Igv zS~ZTFt|vsY9E;&&i3)Z4na(=>e29K4UtnPE3O+yW0B>q^Gizq%O`Y4dSm(Yv)Dm!D z#QzP#RJFZKOSdvAaPz;tiW($s@pKXtwVHhyyM^{`QX+y+FXE5pGk9Jo8um7&0-sZ2 z-7>w#hGeay!)|&M^xK*iJ#Jy*=I9d%!3i{l>n;zh52u>#5pZkQUOZaKF$F!2k!$y&iJOxg@fZQx!wpWi&wIjzj1;kdxHG2qs?WCNq!vssYLMs2$1y(c z77EoIWwf|`>ilk1D!pnsuh};rHhWAYx}Tq6ikbuIZ8xNUvMuQYZme9PyN#`q-a^;O zIFL1q=ac`+l8MiYG-~*ZM{E+MX`1i@*4IOwZ1{4RnGyCB`;MAYmnBa@V?`HedX_PF zrg+e8$4)|CmlIX$dj&Gk~%_cwC4APmgP~h!EYnla!c6=F^0aPPj z_T`a-@D7LHaoznr@v!pB4w(P&IQu((3)DPVK(9V9=6G#uxgMJgDVgcY2Aj`d&PvIE z-eMJcq5L&!UJZup;{s&ii)a%5708XEcq=5ts6n)j>}Q`R_M-f3hHjqx8{|$E@UKoT!hWY~IN$IctG=U%S+GQt z*d-~@?^(h4JwFkq?`Vb%e-}~difyET>qROCo`=?5Pe33gjS;y#m3e(gjW~F@7)T}XFy2cv|c6&CN&V&xLsziP&L%Ov#eF~{*<2eK{H4l;X1fc0OC@25{7GZz)% z3g$mJ{@0&oC@dq+!@)GN^&X@oXETQPl0faUF;Vyw1{cHIno_ykd}X^Gs-h>o!mt*$1hOmh_P1TsB1KI3s^7kj9}d(b(0? zge3{mX<`AS*j)^t2+W3TQx*EH(}*edZ@`A`Kk%mR2*$4BK0_88(vE}9*!$<3@TA>cd_2>G%6+O!HQ_qAKO>|5-V9SUS$&~#{h9?o1y5hg~^1sR<-P9qN(lRAGXYy!bViCMHbHk((OQuTy3aWZ7rG9c&Fww9A zN1cVicFP6Kkud{k%)~fu=GP^tO{|YCp&wlT@MBKj<5~NO(r?^7&~1q@tqM4SORXBf z>+@V*^}PvX@Tm?&h3F95!}nqJluR@&m!O6FtjVgOVz$~!g5FMxC7V_rvnESZouI5cgU!^{Ag%G7 zyI_Ff&)pQ8`@p+QIukf(QsrO>b>i5UGO z?6nVCJVO6xdw_e5||GI zGbo%~PYn}AXt<=KBhr`Auvv>(A;?&-AoRwG0lR3lzTe(}{s>lv+TPog6mz$WLp z5LX&Vr;94ml&59<=1mV#a!`&fxo1v?{TEhHXBIc;R3c~N*d=)Ri!Z% zv$3uxn6~5;!orgQcy5C^sug_Z+yI`Wkz=mhO!6awS=uCo*-r8+>v3?+23!*zKqG_l zL8*E_R4>>8f8zIZF3mCgVy}tbT+Xed?=-5nM1$w0O4h<=B^wc`%QlAH0{N_DcJamU z&_4MeY%7%^zHT+>y3-y1*q?^D>-TZ{_KkSaXbuft5(-+o1W0~{9las1h3b)te2E?% zx=(o`3Q6@cn@nE89K!^9$}bNOY>=dnj0ET%&oVq}{sg>zq~O&7t{d{;5J)O=OlQ*$ z=DDp7Gn6b2U-!Jj`EOUyhOw8deJa84)mx~?zUidE-++wS<|2vX{Jwhn}uz1`Dki*gLU7mPVKLW z!@i&Yz*Sp{N-4L)EeknfSXPS}+u9*m=_-H9%9&1nJ#Rv3cm z@n>kWGRjV{IgOq@J)Q5^A4TquucM9iVR*`04OKZGhfw=0rc7}cKE5yy8?9U6-lE;G ztZbZp=dDYnJEzkGnOKr1;KXJ`BvQBM+`cCDC^K3sgD$^5FbyXMSfNm^lQYSIoc$?C z?D=;v&galw(K<&MLvR&(O5#^nn~RH@PL9P;OhGo0-xvFn|CANIGJ5YTg{J_R@M z#q67`!M;N<#N8=EANIndPl@O~v=3l<4Dnp=0_xto>9=c^^rwa~(dy&+C=Juen$kn? zDCQv?N&Co{JDK8`bPXG-zJjLiNh0|@pD<;G1Zb|aLu=!9_F-l!P0F^X2i#V%M$W(R zvi)SJ&T3&V9g^&qSLj%5ooA#1TSDpsbFuF8v8c{z&C zbo#_oPd*M)4-;x1JQL>bvcyI9pJD%uaTqj{#ye*&W5IlB5}5DLKODakrRFX}QSq6q z$nC9MM%{{3|MTQ8;%4YROSaMC`Ic0S?P0E+bHLQNB%(gJoXWDlnfF@}q^y%LPX85_ z+#f@~VtxL?P!pOXCQds3NYEP>tm$Ac$K>uVXT<_bQSQkoih8fX+@v8eC>4bW`yBG{ z*;U@dc|{;NKL94bF2_%&d9b8OoV3j>$HH}%b{3z1LE@+$2zW^_NuDWq+De6-j-A3E zj=aK;bpl%4{frR|;angx^Y{|CR+3vIMNPqW&CuE2#V*#DB0Uaw67gNiknLSRVP$VRh6Oj`O4b`hyNfYY=?!e0HW~O_7x{?l zGOWIHnK5X#BZnkZNz6W5$T*)wr`f&4w{Nx*FONuyBEA^=_$t$!XTdo|+(AChk#(7M z6eD(aqW=*VJ9dk+r#EwVmxO1`jm;wT$-6n^w8wm6I@AVrp~j@%umv7}G=->0d8+rO z14Sj?p}~1Rnn~(#+1^ezFEa}r_i}T>ayNRfa|V}b+>MLxRWi$-+LO`E(ezWx6Qn-k zaI9wy$5q&Y&P^XdV$vG$4*H0F--Ss1FDu-YHHAFAHI>ecnMBjyI+I7hRX#k zm>0_?k#74@E)!D@abtG$pu;JcWs*wr`?5*B|0IaIdx~+YJcfQJ)o|DIY-m;MV@oyF z>A)msFfHze>k76c?$SnlZOM5FJsfGqcmOKh-VU~Z7SbQ<-oat#^Sld(ZD7j-1?ui3 zOyW`m$coluSV&BX{>gl1;mKA^S~-N51SN_6?lbsG#D;KsY-%mEkhMH?3g4fZ4sG(Y z>9%F+#CQ5H{Pn&G`{rk`w>Zy5e|IEl;kX^T=c?FhGcP#$gWJ7$tI!)?Wmp5lI#8(C zNxt~nf;|5j_@i7W%Vx>ajJrwf8U6t@Y7wUivw;g*mB z?sj~OPG_vhyS`io5s-iyK)RXB!D8jU}1491)05pkJMa7gSl*U9dMao<5G{c#Q@ zUvdoA^mOK5aTqo6;v6)CuVL@vZ)h3ThBo8ZIL0c1q)Q@no;w9|YzpYT$C03kharNS zJuRJ_#1!~`W8+(NSyP2v)=&(&|KlcP>*pNyS!f0=d29^VbA9-YMS^z0=l;VxtsBtQ zd_R|65+&|~#W0ZN$I$xA_%AbzR?V@*os!>i!ATP`y67xEk9i4Y#_t)6(7W*K`v16I z-m=sB?8xxh!!S^7!W1l;izoX!(E7MKZG4qR!-j0wgom%0JMZd2f8U$!_@#Xb zV?N6XQT{H+hL}&E%n@X-sON&@=ZSE`r4hz5(n-KBb^b1)>2&s;Mm9EK9mKQ3l=Eqz zMU4r$_(dK@%9W|@^>+NTb06BS=)|+_LPY(l1vxaQtEuHQmJtzE5F)unT%0)T7a9gti`g&&~|LgL27=RNc>s97q+S%^O~T zjzKk}{W*geyyUv&XWaO@Ph!}E*1PG|vI7`wp-7kLDv@xReu&w5ot^8<^<(~JK=G1b znAPe=?vKsGFbGAB7s~+R#X!-E;xC`&cuKSdI|fC_wPz}H^v@(B{&bVS7WLxRL^AcI?n`}m1JkkhjeG3P<9(ih)AhR<11#0&QP}3n*&K+q( zb4_e$2j2^|Z5!eI41hglX{h5a4rQ18Xm(Hp@mnWOf;D&Jsvq0v`gJqF`#D1%MmjPt zFO`AqZ55{P{xp8&8AV2TXcKfzFs2LAxxRFcHl4nx4in5K(0$LRprfWU$PVs8Bav<_ zyW0TALcFN;$OKF->tY0Q4|5*9WO_&R2PC%*;7dVaHq0rX7>HNFkgNm^e`?M)*sce? z#uKnh%?#d(N^xB^ePYNQF%^Bqny`Dj74^27 zK#tZ3!6VLh;i+MUJD((j@$gSHlQO35v+c;`@`IqgS%Ns|_Q5~Siy(VYrf>`DPS|x znu>EwI)l5mEAn)|D0e_^@140M*n0SnvmmUB=i&Tg|u7ZiST*lXAl+@r1kAr_$FU6 zNJM}QE0~J7T_YdrUiyOHw?uYm$`;n+VLI92UCX409)_rUal}g_lU$tI&0G6P8=|bU z+2Q&)_E9Nw#}Z4gBIyr;j$>ht$aJbXrgsJG7vYz35tp z%8ysj&iY++&%0N^TR0QO;!d%7E&mve1UbefErZdK45NE0``~A4B3w1R3$w4qplU-W zQ^GMdYB>+<0wPHdoZE{^>1(j*7sr!q9z-LDSEv$q2)fd^`KV|jk#;{0<(|7(iR=Y% z%&H!ihEpipYYg*VtS6h_EP}^FNz`SnDqTGyM;pAKLudXZC_MTHa<j?`8{LlCQ*Za%x|NZ{5Trf{hVA=oA*AdsJY~jQoXz#O=UJ>;tL!J4QC&pz)_s*uP zBn@$3X)j8t_e1!Q7zAhAlJp)U{FD0`5;M;u+kXV^7JXxz6X($u@#`oZu?F{9o&!eJ zlCL*oB02M%JNHVNfbQ~s^!^=%N&`pH-C6|hIRwx~VLLXo_BOLK?<#01M8P$K3()`m zJ1o{{!CY=8{Q8jx-EjOLsvLaH#MdmR{Ju`o+gBx%rn9<86)!n&y31Ci6KiNQm6y77%R>2U6VVdtOx zI_D62uSuHX+!%U)N<7(^seq1>Ol3ZHk zL{ytUv2o6GKvjGRKTR&5ac-JMBND^O;7%nx+-^*sRSD376~*W%-3N+Sa;eh#NMgA6 zHrVgk%l_AKn^(CZ32YV}##1*^+0FKbc<1d+JRPGB;c=31n{{BGZqH(V*UFPeP5q3| zM+4&RdxyFEgxe8L@4@&36Bj;n}|UozI;U5CYA&ax?XsvHxan>UD@6#4HrQz4m0 zO6N|4^WVp?J;WJiOyP3e|L(%H9vc#qu^nYsp210PGV%6iOQb6w!>y&IV591W^3AVhZCDq($tFWnOM9c)H?=$7Ox z{&=z%Y`#AN@`_7g>6FiW3)eGE%c>^BchA{81FlO_Fl0$`mki=aA(z=fAA07k0cyMI zkbM%uyvx&cX?2VbnR{Xqbr^ofN_4L#@}!M@V4;nFOT*|V-Xd6+=t@rn{zR`J2N>FC z4L2s_qHWnWBG{@!?5{1y1$AdwO~C=ww+^MYn+14J^Cn_Y%SSxAR*LOedJEHsft3C| z4h0X)$?j>ntn$Yr@a0Jq`E0q8tTh$`KK6{xh}f3udhPhE*F6qD@&91Z!~_m%UMB)=3>^>Wy?5sA*uN zQZ(q@Q;t+Dj)lVChV+K>9VlcpaGXfcBHkRjRTR;E`B!Y=stn6^XRwuy@`V3TlAOqt zq<#yov8!emVnF8;_FCFxFjl#Mwo@*{@8=aTZ{Y~*oup4^@3@H_Z=>L8e-kg`!!>NV zHU~2#HsUrxds5@DlP)-IK;JGc!TC3m7&^rh+#~C`E`1^`tsKOyU0+czZ!hP%+X1_5 z!pIDD9r{nli=J=W2aP9PY1rioc<(Jv9#*a;B3DO2=GI5t;ClmR?|H?$@x2duw|La> z=OP*#6~!O?E=n)YmLmyWOQiayq)5*+=HO#c*#UNmo(j;bj5Ci{-lS_`G#Ds+7m2oMu zejS%deJIP?|CmH8YCD-ffz9Yp9)z;LHj)k(H!Aiwmi)UPTfq4%F|hPWfo|%H?^qdz}VZoO2HR7j!Uo4tMa9Z!q~0>q!h^ zXOp(_XN+vA3wKA&;;$R-XH^8+G5)LvewgErbN_JpE3q;-9#al-Aun;od>`yt_6tVe zE}+4or{I3fI4(2aKtz@oGg;x$@M>8a!*AJ5QjHVnZfZjw2UtR7HVby6Iat`YknR=~ zhbzj*u_yi@n}6JmwuB5}95v?q$!O3k*8`}Bga?W4H>W{&%GvY}SJ6!3Hv}piV4gNd zP%0Wtmhf!IXrTgsd|)*tMeVHE=82@&RFdxZOJ{9wWw8coUi9ax-*B$m4h-#XGD4Tk zXxC3qA{X+MJ^Z8s?oWD$ahoqd%Eb93-Mp6BE-OZ>Prktp2SHM`MVzX4*W<(oom^IU zDQGjhy4|Hwbiqz^5bCYFXDnbuh+8wHf6w>AUB$J zDi{B}T1T_F-t)I>Pmpa`MM6YE(a^_@SrG9ES!z!V9Dkzbsne(=sZToJ?Pf0+D&W!A zGBi>mk4-$a59J3c!0dKD8U?*!x;L0WXb9{##Rqc}GI>gJd;?qVH&*Hql0;SA@Vn=e;(;w=qR6vj4&IFuj zhlN|u;qjlh@jxx-eRho}XP!DT7jPafY2M3?%GzK@izL`dq=4neo47x#5B#27Vb2>H z(39uSV_D&PSaT|YRpPqtbA_GBHN9EXkXG|7lG*AITb)bK&kL`JH0 zEhhKL5OKv=ycU{AJ}j^%-gbnB3QZuUIqu|TkP&!F$r2x0f`|UB!s;hWsS3Lal&_Dn z1(y2A8;+x%redVMD3=_oQz94K9Pmb)5P5SPKAV2}1vDeo9`oAj+1Il&*n(08w!V2jX(@V-@|IkWzO?2}=>`HL8(c5@Y%C;x?5F`P^UN7sY&5! zEw)%m(nIT{sCFZFexDP{hW}T}#9eO0FGanuCs>I)$FD}~Ev}?#S1!mbaHi>+U*R)% zE`8nTPKJ+~)80IB`shnB1lXuiWzxyE)Np;6HHToi)9`{CO;iW>Wo70y&wfzS>E_Tfe&w|o?_yh@3Q6U zt|ar`R$j7TGn$yDL8-__xFIizLuSvQ{(LLs8f!rvw`02&%!<70$qio$Y8 zk{ti2icS+lX~CSg@GDP*Y{}?n7mcgXFue)<+mc4?{q=U(<|j?mSDyhvC4G`HX&o9~ zEd$ul36b}%!hdsD;mti7WZw@FdiBI=>b3JKdre0e;oEEo37kil8{C2@32xUFREeAP zFEa0ZOqs~6`zWeEjpj_`TviJ9C@|ZBiSFn_li~(^wIm5W)Le*C&*QKjlHOLd4cWMkPXb$F%77--asCRT9fag$8g}> zcaYKJG8SElL{~o)^OIK4Qz@c2c6|lID-xhrRHo6ErLHus!XDe)aBFRqxFVN z;5+KGDYZqwYn#emdzXOXiSFdlS7~~#S%J7ObH-0iN1^z@I%exVH(Kj;l|Stj(8;Gw z*fMS(IK<^Quk3DNMI9_jSmSkI`>L?^vNEpYSZK!#T)?>~LAc3}eDzI#6}`4U4~lyVrS52?IUf4M?!ohaEf$(T`B_=M}kzw*;l z99b=EM=DXKK`U=cQ}gyX)C&j=~T$848p}H`Ji*pfGAA6&u->85}%qc zaUBUcvRq{hKgAl;PQNm?P3AevR|_M`8@q6Ztu1}@SC_HSJ_{e_z2m>~vuf(~f5O}_ z9c30+JmQ!_pIIkG5xC#2jgR!aLEwW5$vgRgt@#@Pavct!cUKI|z8Uj>dS$@&q$K9> zxnC^rl_9wpwvnhU@#Oi6RDtnAHSQjBlSzCpMn2uxgVzI)|6*h@xuU{{jC<>dSTsXE z3jM%2xu0>Y$&kbr*Yc)VB@!oLHM->QFdO#Q()LA3EPkn72Tr!0R5H^6M)va2;>2c1 z?8qS+x{)NfMVNZs$bbuPpTYFDTv!mHOe5nD;Klf*C^$F{nQ6u(_p>tfjC>8-7E6## z%j@tTSD0A;dl#1p_zA&*jTm_C1y;zaLvEHA)=XPXzc^IkXA`c+-<}9x^K}TSU4*=X zR$MkY4ks4bk-STZ?6JGP_z1*ktfde;9_GT<>gS`P-#+{#nF&?9RH#dlG<%`%A69rs z(ci)StbM*2Xr{cux3;CsVz)$+J>QJRJx{?0dMlvg_8j~c6^lJ>4s=RU1|<7Uq<7Q;cs5#mPx40Ww!T6`ox+p(=+ph~bN3#&GOE$P-f} zx36cwP=FoWUvd?`N@f!&zr)a(YCso1ONFhEEy%o^XF#TS7R=xNoxNP;O}y^zfaCer zsK{}RUXAU79o23q)7He2N`LA%{}mXtcQ7jwG~l120qJ~yho7;djD7Sx4g}K8z)ouw zm*)-TIyHqz3XJKCyYe{8ca(82$YY3t3pq3BgdUNJcrq%DzHq9?s4pKuGKGO@l|VR| zCFJJWljr%pAZ5xq=l@S_Zyrw7`~H2K$(#%sD)SVQA?$Tt4OGS=$q|USW@4kP>_xnAL`#GNHk7qm9v9Z=^*?V7muj@L` z_xXD3h|`rqx1jrYAull4mx^7GhDJ>b(s#|4zUh&mvIXbicexDNcv6VoQ>|kHTX^)3 z77MAOi@2WL98`7s!HA0IF&|>vVW+`qNS)vSqG{h@SA8wq(h?v{++Tjx0ZIJf5Cm2q z-r6nZSS7zrx-fBrJ;l@@9Q?GDcKzA{FJH_ePZthB=$vNkC{-l27AM)J4Q}Lk;X-iE z<@$x(rW1#-DB4vc!C%jhLm9axM49!VA6E3S$=0bvaHTr+O>AdHIH+~kq&ylG*9#A)712+2``DBc+T~rzE`0ZHy+}n8=CaJ@;7vf zF(u~@04iCgfuzW7z5P zml@z4!a3W%@*Q6pp@HWn%-Pt@XeM4_+$|JQ%`6+|kC!ETcYQ>o1Sxnnp`DdEvIph7 zzanK{VysUP#J!3oWZ7-ZIWr0GO%TCV&As^Jks7`4ZH4M`ZEVZKFtoVw8rR)6pmtC8 z;fLZ77?zn#jQ^B?;+GrXUFU{_ug6j2&)ebi;e*WdcMXi@cLIVg;m|oNh@1^6$Li~; zXy6qEdJ#vNvfX<0w^}&y+4BJoU*{Mu%G)4qk`4W)6G`fJPo~p%<)HCTTO#8*9$!Ud z@{5X;1iP^O%ci*d z(k>XceiSak4YaMxBXYXU@I(CqoaYd`=p@G0{o+5_ z?ZjLP5+TXY3F*A}9DVXa>8ceAiQmagh#HWirPd|543hSV&Qpp|}xWZS|m5TwP2_UCb!d(VKB z3`CRJGGo}<`9idM&SJWvJ%u)YTMhj73S7W3(A~_Fh>XJ;QgBd`D0hb7UH4e5^E4&V ztQqySwSn@mPQ2q9K}Wgy-0oQ=u)_Zu7}rjrV{6pdJrWtP!1x^APl~6F=S{%&gD-h? zZUjYYwgL>_!_;nJcvbWh2FoW<+Yhbicpwu)b0R@^qXLx))28%3#RJyk(ZflCuK8w4 z_q-m#p2yDM+{fud#Kc*v;q`R>Qie(tB!T#;XJBUm|GlEBWqhTd?`=|!B!Dk zBe<4m53L02U{2TB)x!GcsuQuP3Ai!(CHn5)j{BZZWmEGbxCPro>+mwCEBS1ngU$AycB)3NSzEOVc==t*Zu6cc z)Ff;zushYru{Q_!M$XQ7Ov;oR45`tPT|fB#GqY%1!wX~;w!?upT+Yo-hZr4AgG<%2 zfM+zCT-AKQWSTr_>&gY$w<(PJ^9(X=-%IA+dA&x~>oPmK-4iJfH97; zRBf6f`fPV3Z#h5smtF$&{S0B^l2*fWCtMGBUkYijHpTjtzd-y>6x?!kAPai_;`xuW z$*~wAkV}q)0nI_CF)X|hEtu{^nlSZFbKEf^YN8rk%VrKK@X(;V`494CM zC$GD`LHcDm>|ACHl~=Oph7)Z(`5{lp|Mi!jF?=8HAIU_6sjnH0VSRF`B!nia*03!v zG|28v?^)^DOUdA4h8{jXz-S8^!3H6qvs&*llTRN9Z?(C2(T9Pm`f(&$mD4`G^`*Y? zk5RoQk(hZ$KuE+S+GD)~PrY%ark&O_b6`GRdpZR=jwN&bAZ3h5Tq;o_2O-QzmN`*e z2z!h6BTwkBT|d#M$NcZ(#P%pEGn zIz537dqvP$p|bR_^d#J^*Z|7G9M1K^KDg4u`6th6!4r$d@qBIcc$GnFSbV7gpB0|L ztp_?F%gPZ9PXAiKGj$5>ZD5@E-$x4QJgB(M>z-)ZyV-M?H zrlOVQ1ft^Z#FWv~pwzkoT4raXWOZcZNXY+FG8ToptgR;5cas<5_{#q-KNyaS(X zkdD_R0cT$@ewy}F^W;G=O<}Z8Z>Q6kzMDM1_Z{q?V~fe6^)txGKs6p5ybphF z?#C6MqNv2oHlE|G_v~&tTNLfjgVw})$OyNfcHPSK(y1!e+FXv2+ayY^J3BO6)6vFWUsZ~dNh$ocI$Mn5dL=%cGk~!wVZ`p21NYo-#M#T|kUe$gsO0;S zZSXf^_gbZ6#yx^^(~`)C=VdU>?-LZISwY_?NvK*9Lf5*BP|;Q`@>P92{QP_aN!UV| zctM6edge49C>5jgKJ>7|2I^@0?G}6I+d-V5JcQr3|6)_rn&Hk1GrUp~2X5Qly(SR>j5X4gKWd9oe8 zS6EGspISoY-q=zXuQd8>&J*}v{e&GidI1D4g#d4T7rQ14tLTT4E|4>0C_59G*#ky)&W7ruVVsa~{?o^`cc4THt-X3m5x_GFg$5 zr2dCIjaFX>-PvWZ<)SsT2DwsgR}bc9&31b8tu;LC9?S6o#*pPbR`lYUC=y-ZKr1bD ziCyL-VyiUD7Jx0`4fUdX*DE}-xC+FydvV@3Wo(dI0AjDKaaCgyZobur(gE2_Rc09| zPq8IJ%7JuT&qZ9kGZ!B{%cNUbKJV{KMWQK-AbmoKL`m}Kvz>=1lIqAGQNqC{ut&Y~Sp!pXT= zmXPZ76J+nq#Dse$?5wdDm=JR(vKh3ff#fkZ+OCv$RY2L!p#LAr9YV1@DR>- zPKHRt0kMmeqvk4hH1*UfdQm7A4y>%gK8b5kwUq1I4Hcj*ylfQOWJ70#Tt~m{qA2@m zKO8FQfM(-4)bOVbEfM{Sv$H3W!2KWC3o2ZWXJZU`Rpd{k#u%gGU=if_3X*jle_;-r z%?gV@=Gfsa@bq~r@6jET z4te0QohN^&l)YmrKv!;j4c!w{XwzmZ+9xaku`T&@B3}yEx_NQBOd;&rkPL6)vZzw) zHKuK4Hh9N3!HRoQ92(*`$S;b7eRKS%Vo)cDb&sNXvh8So+v( zgkd>77+-1<8<%?#zS*^9O)F1vyl014@7S1gnQnp zgh_9PvL75naLy$1r|~5}!9auAWe`V3riXxCO$wXG`C_W17%>Wg?_lzcS$2lI0*LbO zG|=62lE1=n3QRbjhN3T=>Esvrpt}DC^jCL5hKLe*-4{R~s#=n~az1(l1vjna9#3_a z4slr`&7-H15ufqMxG@Led00vl>nH{v7Wz_+(FNpE#a%ek6i0d{PNCjL=H%J9W;U8DcP|0ec2m#=p#xVbu3BI*unHIb|NL8d^nX+1b-e<4-Vo zOC+h{epC8!P?i2D)&SQZD^NXg1!_K!M2#p1TwN0l_HVDVcDFqmuYL4}F;XJ%c1|!m zym0{yojH+O*5yIKSwUKRgU1F8UWeu$Lwc}w0JOIf+ROEc+D&{7OZJ?_q9ZM^=KL(e zGr7(_ZrB0Rrk3QZGRx!}CepKKl*rw*#qemd2|G@ZEiq}M*pIR#3(pGHQKBHV)8O4alIq5L|8|hs5xdfB`^L1 zhsLUqC-ow5RY{vH=(b>GmuL~!l`gPpY6T;`*%$Wye8fETT1Q02-)HTPU17ehs$<5q zjOB}WE&?rcNn-Fa9gi6&ko)s)!<(7}oFL9Z=BqrA(SC$OKLNr6UqG*_7DU9hpy*va zP^t~(ci&K^*Gn|%=7T%oy;%SUXFrSvObXvt_Gc3f$aAmP1xsq zja_|GmHBM0KumR3(u}EbAjtKOuh1%EN=(*Lk$77gIKhJ0EqIBae2kgm_zeCF-)UI5 z(v;rJImSjwEMzr$i!l02KO}g@6E}VvEIi6F4nohv%z_25(|I|Y%9}~($7VPaMyR}M z6~4~VqV@M?F$q!i;IUYaeX>B23>Le?o)^x?$1XOCgfqOM~ z*#*}UDHA`Q$YtJ!)Z^E2oaG_r%ZyWYGq%V;oR=(FGtddMon@He_ASKhg*av;+Q4yF zEt>R7i#Fd5CK|cBAx~9|!6_M4DM?K;>bGlnMAzh=Ac7J#L}cU-c47I_nK z6n>4h=GazYXgk9lR&nc)v0g@;wx=FboW@b9G7%V~Sc$TCCt$)~0SL6%4@Yjc;DU}o zx}(ShX?ZD|qjHtWyrqN-n$1Du&?0)5-p38vWf1#g57Vx4lv%s?5X|)uL61)yi)=)j znbWot2bC7nluKn8eJ6>&tW2bd<^j-k&L8cJzQXUA40wBn2lGG5Q-0?z{$nQOA zWS-C>vn#&Bl;Ui>^n3zw&EQyjCmo6Y^$WWfgnN*HF`^txna3m*-Dae>+o0Hb&WB0a8gHzVAt|9-ux*+i zp08DA0y~Rv<(#X``4}FAzlb0~-EBD4UX88He8-cr*P)F$Luhr{m3~*%g^-dx(5`7j zY@#Zep@3S5#atsRrWyf1u;x)R)(64Y<4-%pXawAllFOD2jj6;)IU^^Y`T&|zB+7Yugb}QbxAGW zh$4*1Pe(eRt0V1yrb$;H*CSR=TiKc9GklT}Ax=+QS*tr7)5A3fZ+BnED|TFMwoe;F zf_z}cy(S1KNW!$gCiKIG7^qrxno;EPPzmEjNP2D{ysCC(2doa^_v9Ab_?oL*t9cNW zNuqS(WJ!8`^K-T{eFmvt6if{|7m>csATY=jg`LCBtmm{rX8XH%D!AtaCR}qStL_?* zom`)de!VPR^T(M!eQOMzvIF2apYrPtnKepsYY9JbGjel41Lv0ZH%(IL2Q3TEZwUJF z#fK(b{KA9GU+GHk6zynCByp(2>0*VKi&3TH-lQz`9?bunjkkj1u#}a6=}uqK+_x8f z|70V7k~vpL<~%S(16g~0OIr4*kWKVR#wxB?@Jb=)yJ@xzkBk@)TaI=2{@xOh*yv4p ze)I4{>=w>L-HhXzT(r{Q^-&V ztvR<6Vt<9;Gi5!dc>m+fq? zo0Fv-?|433uCi*$XUNI9g(a^C;DF>|xN>KSQp-sq?z*FSk0Y%dmpMjYH^%=Jrt5Nla{Vy!^u75BNO5(Ck}0(~r)4SIHBbk$!#9y54V=D| zbmCCiLDu-5Ai3DP1>Gty!x&)|GW@34%TnyK7oi7M zs*sr?9LH$98WeKAHwE=lO{y0tD%A=R)0s*bYhpv6tiA%Gl{z%1{XO(2$f5mEIlkO0 zK(=;qUck*S*)JvIXwNh;dM9H(9poG0yo(HZuwygvHZvsWr})A0-LCZ9oi=8tPy>^( zDwl{)_=fHN+vsHDI)2l5H{$m-m|kCfk*o2XV)Q%>NK%m`=YuFtwwxPFCS^78?)nYj z>#lqp`gj`6>mAAI51vH$#S~PVIfXnl^up>0X;M=;hD57tQ%7#jE@Q#zm5AbHkEE2Xnl8g&5#d|huiB+vW@hgA8?tryO z1G%0XM{Zu<&ogl6;7%*``Q`7O2t=ZhhP8 zeHcDI*$M&%UCi0VXQ5T$3LX?RBzp^PL;mYXa{GEDmAl=MSbMsF>do+@&*sLH5IIYneQ7nko{Ok{Y!Y_d zIE&K+3IUoE=)*HU)TlI=C~~#q0C{0tarQYlhzgOU<=Kqbr4-bZmnPNfQ~2K|_|VIn zG;x)B21@Pu1obBug6>W^GBJ7vHCZKux9d~EQiWrkx=%!z?Q5uhzcBF9Qt7dAet3iC zz>P;*9JAb?xcJ@1pG6LAM3pCATfs@`59sq|kDH9|svW_wdWcaSFu|wk57@zMW0<_g zk$&@6rw-zy?1@-O_Ctg*R;O;l_C^udc=`s!shoh=`(li1xe@I%ai=%Nhq4FbGU+4L z(39pFvN+vF9^nAIU>^rNu-EDc%jwgyYMCG4zlj8JDd@E#62G z9mwF7TnwTIET1uKb|5KiAncj)7`je#JMP|_hmTEuV5!g!@?2&LuX3L#b@*D!t{?lG zd3*8_tCHCOi32Lc+TVm@`I*zjK^i1G%Y{m9lqZEZ_G3acLk2jO^JG?ow57P(^=PV!3jD1<4J#H;Aw5}tFw-UwvuBU+Gd;dB zcj_+j3$A7ml|pM`^;nANi{FHJ`KPcXz89`bB;(*UMfUEOb1;66DgE7%gDa9pczS`W z7@7TIWXa_`*kyJWi#K}^$p$+-W-3H&#k<*%mS(7rwV{WyKk-^-Z=f?~iNL(85X<f~vP2Mfp2CSMu) zEFzVP{}@L7V?}sXb_jmZWmqu%BYN7crP*rYw8kg|#;mj^5|NW(nrJ5omoKD$3PWl3 z?MB4w<5AN~#CEq!0c>gI*8UD2^x693Y`peKOfauud-P8-QmfXJf=`a*Mo1PKEe;}| zUMN7dybu+bQ;KSP_QA_%PUMKTICV1AVW%GuL#y?Z>Arwbtlsw*R<%ixchwW1T{j&q zl9Z^zMqSz#QO?iL<#PMy!|Wm_?L|?|JiMMb%KX-rBAuQ7RKc_y)xKu2?Q68rv$P35 z&*;Fkrck=Ou?dq1j|!76_+>B?64&>zE(v#0^Mo3>?{LSHL-!F+KY-OA2XMl;18mlZfT}>CDw^FR(u~j+&2*p~3$5_{*S zcYmTI|0C?!8%RediIcWT(OA>o20_7`NABd#gISDXBBk)ck$eNY)BK#PCclf$RC!;u;xa(pT0d+L7*rK>;Vtr2mWpAiL1 zTNO#mVnW6jCNSlko^@KV4>a991UK%#fR|g`n1L&g*+Zju`14~7Nd2P>0{aogxw>BG zh$1YqJB0W8s(I(DuQS8zxV-+vvup;B!mBDv>cZ8puU;MphHf3`GVvO-YLw8@1Qxp2 z3sW&^4ff{rJ7{103kNnC;@glC2-zkGE!i2^`_`KX=PMEQXXz9cF*vXLp@sZ2H)H#0}Ne}Rwc941si9IXEaqFqBXxE`59F02#d73!DZ zv@u3ZrqW~*P-Vr4tXKp0r{;miY|0PUkERixHngZho#eaBAd6FXkln(W?EFoqF#gv* z?<%IFdpKx&EBsb<=2Mhdq6sFG%!_b1}BG2rLzIdC_54K-qsB zfis5Wy;dmGp{WAZ9?oR`q`BB(T+HsgcO4en*>D{43!u@Fj4vL|r>CR@iBwt@Zt|BS z@)8`&%gTq8lN_+r$a0nX_yXP>D>J9pf{X&_<(VLAIx}b%z$BywUlStj5W3u=&qKHd&Aa}1xrd` znvW>em~b23hszNY>#yigVhYnQePExcPN44I3e@{I!2S&rFmtmAWIdIm_XEzNUbPK; z^-Y7NvI9M*i%i(&i7BzC@w@P5bblK~bw52ohYoeBwssUP%btKv z@jYBMm`N-+50!Uojo@pgJ9KZE%Py6gPYlPbWs>5=;kx-UVifTUcrCpke9(m4F_U6W z1)T+n-63qBq|s((=w|c~fLFH{}*zk`htn*8|RNn@6{L=S$Co-Pc z*-fRNt~p`6Y#)@*4kDKFJJ^M+9^Uw zq%**ptWROcy~3~Tm@}G?_$i)Qm)yy|v9iTkoodY6PjksC`|H5JW2ztyg}+6$zrl3;9SL0h zFbC)L>hP_;O(lV0vUJOoQOtWTLME#}1D`o3;X?guJY`zNCjHVNWn9m}tIL;(<14~bIGv#wh#^bvWKzEaRYX9AuQh}Nu)>YN$ApRP-J)# z>E-eC#r$8;FlQy3d~pG-7VTxO%r>BH8LL_6EIX2sr-bbz8(?MAHym)wXXHlph<)f{ zCLwMPt(EV_%<(ov$I6y!{H}!`A5Mdz^9*`rawsq9xh(O^e1eYQk0IhgAes1TDm=6q z;q701zG-oAB5c_iOGG>&K>3#gVDF@M8+<{JG73F28^^QkUVy z+i0vR&*P7D%%b|OoQI&#K70_DK_6vCP>cI&^kQ)<4&9thPV2^#ULruIzlowM4{eFo zPC??Ha~@`d=aa8%G;pcSb94|3g6>n(Xh(w$?P*b@_pDjie|sERl3j*-61mS54t(>x3ez6#!9wY-NF2{`j|D1BSDAN!&s9U!d^`MkG01L>sXB=_u;usi z+^MiD&FDc0w?sa6XZ0Ijcii?Zs_pe!;tx5_{yje zmqdNUgC#st>(~y7f4#|}@MN~mX9pCti<5PYj&$eJcsR4J5&RQ#@t_0eo405hst(Q~ zP7>Q;)OI=%n6ne-FUVwT=Oi+AqyaKlyRxNXm*6lg#hMQ*F}MCcYAT%s?@&KXS~7)5 z?h~Nlrjf)_#hl#rt3>&9KN@4gtvzERm=48ewrl7Hv)M<2$m~kPvp&Yq>taIgy)Yn0 z4{@xQ9z*s$F{G2blsF&2Tevqwm`+?%%M`6V27Z}_So9&8=~b$L#WLAM)2I_Tn`64{ zq6-;_%Obt{sl0ZPhp1|{0IpDBcz0zR|8nP4*kL9MmcWTGq^-kpJjTZ|p+?cc^+(m>JvtksWtff=NHK14ZXP zXUBO4(^U@RxVqQ_95`PAGxl=+qs5QGPDh5U%t@k!r3=Yf17Yw@T0+Kc-HHMJBVhbM zgp_;>BG)%b(@R{vE_p^fx~%_z729&5Z8(Qk50#+$U>?!rnC&^Gr(xqeLF$?~g)UeI zRLUd^&De3IZTBgr!?GI{+Z0H9|15G;>?%%pHJ|*N$A^4l0v^-+sqAr85Zs#z;w{>A zkCQc#Zg!=?_vEPOCl&TUpC&DH--kN#+nHy_JuyJ>4f;ILVYV+^PM*#jPaHzNqSx)c zyarEY+8E$K4n?|?VwbVRI!Blu^ZJHb>Vt?+S92UARboA&fjshq-H8WG7rJZFJDhP#mqrFnqC3vK1Hlm~ zYI;c;Y;Wh&y5L`!c2%3n{Az*K%2s5}LVY;Z(+AtkxSX{hN6qVsqQ^@UG0x-?fA^|I z@LQ-2`2B@&sv!sB9!zB8!#tUc^cA=&ECRZQj5rPY1QPaH9()>&sE3mx{a7N+HYlf| zY44`qJh#bKsN=_--|b^cZEx^&{GBs)#411AMq1Hi<^64o{yMn4s^WA%J!N>cQWb4GyYSku`Y!XHe(?-xp4N%3|8ed*Hd@y z6KtEWMsl@x54&a)W-%=h<(V_2S3ib^a4p^qLIak=qAp=#R(hcVMYaCcDV867uyl z(Y7NQHg`$Uq(Lik_3BSnQqK&Wk7+NBCoe^;?@PO6crb8jKm&49#ej$nm%9s5_e0X7PZr{{UL zxJfhy#kqconI_vc<$FIm_E%x!+=jHzUvyC(7wRs z)jr@yj)h;Bv4d1cJM%wV6{4F`KfEv0N3%ZyRO56o?f+wM8(fx!k*D0qYM)GMdoc`! zy8B_5b~A4D>S1;&Cb4sj!|BJ6N|fy0i0*4_uyI=|F0Jgu^65Qzx;6vry-tIhy(x_@ z6(idP|FUA`zA#-shvq!J%8M);OZW{%a3eUl#>t3Gdn4 zRfkv|u{y|^y%|1oK1r!(%1~B#GwF=aViZFsvAY(W;Fs$m92|`%Ssde2?r1K=r|J`D zuTl1pD-T946k(TjK0jpm0LI3tkZ;uvWJ`r0@mku$iblqv;i}0*f9)jtyS9tD}nsA2bdf?fA&wGEZMDk5$7(w0>0gD9ASGFDy(?Se0?HHMA=l# z5ZFwvI%kokj^oMx_AiWLnhw2r#u3{_Cy*T|OT*r9*SAPB`gF`2$a^gW3Qt>^FY9WV z&-FJ!^qdk|-q6n5$N8o-)U>0Dk|^E0@DZHS=P_=nRk*t=6pee3b(yC#`^r z*gUBJyq$yvSisSx2rKs3P+qYe73288J8Qb(yxbJx&Hn+OCy$}V<2?`z45x*%g`yFAAE%NBu$Wh#l*yN1s1*WygW0x(fb zptny8vVr|Na4q!)Z)u4!^*ED8NAj0K#!WstYE1;07Y*pj_0_ec{A4#>HOJzyhhW+A ztx(Z?nOV@$idK6vah%sx+$I$+gH zQLs!NWp`vtQokcnREcAC9+3$p83*gIUP6J~(Kn;zMIz)-9H&dSQ6s@N&9HC@L2;MA zOyWWu`)8oJNM@%;^*t9=|K}0h9Pggg8W~(A@<# z^!A(e%)I7_q@Jte9UqV+;mciU+#Ve`5p{>sgxIh|m5xd8;H&Z9$9IJ~p9IZ-$KfHlKc&|m!< zli-tz(cf<{+nuuMK=BtS>V3>E&=Y`g-97MqoG$0lX-CfmYv9>yZQ%Fp8azAG#_l?9 zOLF6ln5kpjsVHw5Z?>>LY+G`N(GLOi=TnxLCE@D=Z zKNWvGg-T8k<`{?}=-9RwcIXws&7OX4Y@b8h$-|s>rV>BL$nkSE3mffy{uB{}Ah_NEdl84vn-!wOK+Q6Piwk1(F+YvFTvBP&s`hbhg~rH3B< zWXyM6<|PiTX6~)r0Rb7@eqKJ4Yc+>w^6ow6D0#pq!HM(Ovrg6M? zF=92O1bYQ%l8E6`On8Y6Xzt4(BUYkVlTwK=@R-xim=foRetxgicw%aP8h$UBPF)wh zK!MYS{Jc%ZP&#=r*?w#)9&T)ffKCHOzD))HEP9XZ;(5$0m2${(H~<&VUgXvAN{&0|y4^?cu5uJD>OF_`<9i`FlHyF+@7Vpqk?y)#4Vt}@G&Y4} z)K1K$@-dzm{K|$T1pUUD{hV)f#d+k(v_Mtc5_lE$o4NnxA+DQ{#X8n)rSm>V(B9h; z^!U9%yOSjbFzExgX4RL6BMvLk&wc{^n=Q=wk!a8`ip=?|f)Et(8Wt{J&r~LeGL^40 zz~iSFIcISMb*GA=c$xxvl*ILPb?GCBSdratxjxm+`=H6ifkf7MvoY^uNO^_;wac;t zb*?sk`A;BIjODnvt_Rn<+{a5RwOFUgcQL=>B3@cy&8}FWLL@U4$laCY%$T)SL~eF3 z$3EP`OpG36XRn@4r?l6wMZr15$%oTLOU<-XHT0r)bu;0u=Oh-j+TcN4Egpz@h+89C zpus;4Q2?m>%q6(Lpo~4>faoZ*70bq^(YKz$R9iKjx+pk6w}}ETsdNQt-1r5&#rNY> zll|!R>jpCq#K_Ny?;zG#fE%tOaFyluKVByYG5 z&c9F~#oT+XZ(kEya-Pp!)<0qE=o2>Xa4+UQ;@;a8p2XqyE-0u-f||9%P{?_N7;%1S zz3DQjy{Q2redeLggkvy!=>t5S$!Q?3JCd27r$gr4Uf2LzIp0|&w3Y-sd)=9QJrc}m zxs~uA{@gORFj>v{9Dk16{y*^NtpDHe=l%tC_y60s|2&3kZCNGDxikE;{ofyhoWP|2 zVdy1 z-){d(#59Xt8TFsh$9+58_sR(f{IkjaTjWyiZ~kla6!ZU#{a0?_zsJUo`;XYE{~7zQ zOwWIh{h<6Gu@?Us`+q;LN0!jP|Ie2H8T+qe?cdLPGUq>Ht^PCiUti;Yk2UZ6k67#f zjJ@?QuORnS|I6*mzr_CUG2!7VCT8-#51A-z`acK%AL{A Date: Sun, 13 Nov 2022 20:15:55 +0800 Subject: [PATCH 28/68] :fire: remove a test temp file --- graph.pth | Bin 187799 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 graph.pth diff --git a/graph.pth b/graph.pth deleted file mode 100644 index c060948c2c3c225a65796b291d6a59bd858f98b7..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 187799 zcmeFYc{Env`!{YDqRg`-ndjMku6>FULWWeNN#@dUn@TD3tVonfNTpIlBlp?+R5TD# zi9(}PG@wCqdc42y-}77R^ZP!(>Hp`fb=JM^Yn}Z%ue0}c?X$1fzK)lhpb#J5lqr1w z?U>Fd!xtD86dn;5Vj3J66KJw|edGd1DZb_ZaU==s;PUxNPPPt<3fvsNdSgh;mQ7LX zx%@#vn<67af?^^zZHyN8j0z5k3JIPc5fmfN70~zL3hMbypKP-_GB7?QD%!-%WJ^dy zSa=LqC@5yr#H3AYwE z{tWKa?C^gGNyc!ce7Mv6r2d1|+K9kt?)31*u-T$Cv^*?Fp{Rfs6SAUXLmTU~y z;9pwG8C*j*<$rq1=wC477_NyA*YrPVS^Z1P>_4Eax#p9!lw-IS|I(U7viujx>R%-5 z7_N;E*EV(s*UnGupT6G^5)=53yK(LRYg{;bev?){F13RT|26D!lm0U4Q(WZ6AN^m3 z|HFs4^bYQv@c**@7ta6mEbiPLT!;T|H|IL;;LiK+b_?$O9bBjXZnxwv*ui!F?{+J$ z%MPyVf7xTB*G8Loxrt73G<&~p`lQ?ab9(*X#+MYI)qlj7so4TYv;RwU{YR+%FaHc* zC_>++RFg}ubP%aGrcY-QB0jbhEv*t@iz^>vzt7WHA5T59?q&a^PNOYlN6|)MBbNWpVOllfKD|u+7&6+e&)$q{W>dE^ zAh}AFR{Q=LlF|!Um(PW)TjeN<9hgBc_MT3N*l|)YlusY36=cWvYS4`beAzcE+)&MT z#MUU=kmQyg38i&C;tS!@BIVv$SsYPw>d})jxAsX_r;K#!Ou{N z)Fpa_i2>XH^cU%_DL~eC9jH!nG10i;iZWm7v1U{2Xi={_Bv-_Z4hx?_x6c83PADH$ zx_>uo?NLB>rp2%x^X}8tU#rQ8L>an-fIjsoi{=yep)bgB=r@|_?1c{-5x=-MX)ldn zTbHk3qpnud_ij@_phzFxOUAg`QtPuR+KHVsja1HPeKIt_$AoQ0xerjp?q zt4WEXA8PvZfS&PqDl2ww9$VH_$kt0+QQnD%pbquGl=}Ydg{(cth`-T&=3%>{2 zK0O*;?o*&8GqqXWC;lM&q=r^hd%+GcR&31>ADZ{KlpN@7MEb_t$zVq_y>^}t8$VZ( z7OTjo^OV(Cfj64qyyPz_TOG-c=l{gVdgR%i*TrZhjpt;A!ArFAvLTT-j$-G&;%9Z8 zJ?Uxu=h&yhfQ9)g&sS95gS&f>?h*B6nDEL~b5cOTkdAB*<%+Od_wkCDxne0KdWNow$~9_>BFhi$%gl-0|K zpjF&yS})s=1g~l8D5w^wyl_MP1K-Gip}XY4;HieRv2B{T!3s&^V3tk?$GNN z??R{A9^rgTbN2USo4R!Zv~P|I=}@_VG@OUfB083hc+rRye{Ml(iNOod-RW7?{ ziwv8)Q32j9TY>sxjOc6LTI{E-8)*X*LGonJ8nWZ-JeG|vVOK5Er#sdjVt;G?fY)t9 zWZRql^w`2;T0Pf*cAI;c^(p!ggziT6|0u~h3g_U?30;mh>_e*g8f`*$&SBdG**aO zdZeCh)6PKIV@9CqzJPtVVh`QqKa7@NyTd-K$-*C`I?0;pCrG`oBKx+>j!qZ2Oi{{* z>Bnw7bn@j83bwe)7BB1|-%j3ztc?+9L$)xx^QRj= z1a__>kLva*VeP1MU?D9)AHF8Z_6qyb4{w#wWe?=oy`tmDF+Y<%Yg0+ThfCzR&^4lJ zdY65>F@)85SVboEH_?A`a_EZ&*U0q}BUZwwk-P~_qt(+dvf@uQ*}ILW*%Log*uGUB zyjs0P+E+M@?){pDHe^0#dx!U-We;tLPFoTy9#~Hty2I$z3X51PiD7j6Y69x?3S!$V zPos|mHZ(J4MXIcJ(EKh3QTF#;v|5i79d4zDcnf!9{da#zHwOCVKmn4>`GdTts?i~P z=OA^r4Eknb1|6kpO~Oy)AQ$_6^gX3Y68C!<3N=e(i$m|zC;a5-vgDe= zBR14|9!dN#m0oBTz;-Gr5MG8o8G0U1t7!PLkf=+yxCN4pYYI_?Lppx7Y#%b`#L<;E zZqoh9OKFYAf6?)R?{Mhn5@dW{0dEu4CD+1+NpQg()c$N1Yq6-0cvSha$1bO_7mAhG zvC}4a>91{crR{W9Dd9Hk8qPp@cTdu*HoDMLOe^S!CtY+uTf;fw9?7EPr)byfIqV(d zeRNt$2A${DNq#EMVGH%uS>22cAQ5K5{=T0^AL}_x*U#HUk}Go9BVYR1d8b`zug|)4 zo4yTlnf4O9gcKr%vSroXXQR=q8sv7ym=(uaq$ltP5uTIHrsT)d!JB8$yR<6ULE9tr zS))L-*F%j(nNQfd+RJR1_b2k&J-tUt_ z){64ex0@`f=Ke=v3frBw7y&m zJ4!0ii)DJW8MeW3GL2{A#oy5$vV+-k!*$2QUaxt}5Ic|AM#&U@7S;SyVvc$W?3 z(&U&%^lf<`*~!8L>3*k6Sl6TcIm=_}cl zrQJkirvqzyD4X^QjX`IQ1k)li#z?VLyZ7jj)o*sO7G}>O zeSHNR`NM@&KK~8RHf7QBj`qZFN+Gz(Zl-Nk`m#@LW{_{|r?IqYKEAHFjvf4Ukx1BI zB(HNRyZ@jDy*?nFUFbPX?)N;WU+zp|S(zk4UOAGW_ZsXIuZ`>$ zyCv2jLeH}TPQqR(;>QH?*vw)aP{A*J)_HDh6uZ4BAEM;TKhMgHG+^rC1yZI}i*}VSK#_kv z=&?k9cKHUg|4_cdsBpQqd2JG19_U(sc; zM`#zXoFh?e0cCZ2NcolzD62;nyJl6hH`j-;4@;9-zbt~Dx`ogoe6?so)f@MScEEe^ zU=Qb)&^i!8&sbkgA{L)VC6QW$s}an~dzjN&H>KzSOKs%3C7E?$8`*-R%Iu&=2peaA zjO|?XluCK2&z`m}C+1eW*(6sz=KMTNKbpUS{?M?K9+s+RUx`K1ZBh#COVJ~=>2NtI z$oPUD36`^lx|KMR5k`u&n(QU#V$vYo%68~VvkF<`h@JBVwv;>23A|1eB6Nw|5iMo^ z_Uh90LI&)H$~WXhc{9szjKX4zOresQ+bmg$!1FdMbhY3w@m!wzBw`Zrm3Zqw)dXYU1 zUr4{vY3w1DjVvlVXs@TfteVL(mRk^xT6GNB^F7aK^H(LD=Y}F|%^^*)S0a|)ZQjUk zHoC^Xoqvuj{#=EuPFTm_NOQ-J*JW!SO9 zI*4xEi=$UuVRzBpn6LL1v2%!K7aj>D*;mI=i+m%xUcw`8BRk1vvl#XtJ)qX_j%UR* zKE8j>|0g{_`#<#n^97EY|F3$$KjO&$KK^$P{O=z4U-!U2@%OSaKU{NCi~HyN{}F!{ z|6k(o{8nqqR{RJx-;u#<7Ttk8*FDLp@-uKcP#%?w@t7-&GwS=TMjo8gCBsu>$w2&J zaQ87sIz7&~!SS!%%3>EPfISQMx&jcNY$RwH7$Yy+U>HttB>Dk4#Lf0I+?XRuLK9|E ziEpGyUf){Yl}bA}Gmy&kI!!K4ruIWmK>$9|x*t3)WuwE}_G6)vjo@VNhJ+ndN!1=5 zBz;Q{V!I4T=88Ee>~bnRC{V!x#lI-U#(2n%Pl4hxDU!Bu%-+~ZiTdTf0MC(GMtBD= zfm6RXzNXp=g&mRb@vknrR&)>+?$Mx@KjJ4zUYe-YTLp2mzJs!X|K!{~1^7I+g1f67 zEEBqdXT>zY-$&cg2Xq79I%-VRZhgi&!HQ(%YHiB)atI01(u1}`+lZk3D)M<%4Le6}7og$nZp5$*eB3+{W)cN9e{9VqK=gAd93M*WD#t*RWayO&#zarvxlj$U^xQFm&4y<({mWiJExKgcCEqqVLc^!WeLfwIuCEZUxi(BrxA^4XL5^=L&~3j#{xkD=y+~3)-Onb z^i!)q=iF+X+S`gthgFEJqbkYSHiOxlaFOXvcSEP&mqEmv6#Tg;8eQqJhBI$zbXg!5 zhKGgp#w!^wz@!*mvfl?%uiL=%Zyur8`&b-scQYr zuWyBaa#-7=+K|7y4IfzQip$pZQ)xDCIA!o1Hr@Xm%N^>0X{TqS^0E}t8o|IrzTL=W zY#E4&nv<)mEs0V0H7xM-6+Tei1s!V%wd-sjlh|npzlST~KwlbhF*wI`JwD2rVJHM= zuSKDO^$l29G#gZt($P?Q7R0CT#NL%^gb`?Fm|ht?V{INH>l@*_z+%V@kO4u_C`Kbl z1w}uUV0LumLaC%M85o^P6ke*5CBJDD^;HRt)Tf|!Wf5HS_Xnf6=^!OR*TA5HG+4G* z;{t^aygKVH-tRd86EQXTr`Sy>@te&w*B*mHv-w2s?&O$sZpU}_q?6V2($UC-AZpqE ziZh*m0?IDl#z&hgaIWFo#(wpD>~a4CPa>let6xupi20##IHD9XD%UW%Vka3PqY^yO za|dSh8>74pd%^Nz2w4Spn4%|JvD5_ta=q^*Tv)dXO7pV7PpF8QnkP#_4kmN9wQIs| z-Dc`T$}%*w*OL^ciIX;yX~=G68jQTVgiq-Qla&SmsCH5>e)lpJ+)k$A2d3+(E8z^5 zFRQ}V7fLYuNF2?JRL4Hs;)qCuIu%4;vC~5NJjrAC;ry$G@XI?8Jy|D&{NqfCQRrs; zIaUmvE6bqJ7#I7W6(ygNt>JyJKUo>Pfl1iAiKP2$!`uTOa83e0lG@t9tF9P^;XUfe zG;MCR-;9$R$SLk@ABvVoczgl~&{ z0JrXGXveYy5?A-FF{g7En)ON_8OK!P2iXRsSf++qH6;uc__|QQ*7UyuMJs zx+1BRm8r-n)(Uo5l);8KJE>o#rX({_5#P`&U|MXx;#i80QOKQ-b~NuGBEO^(JeiG5c7}tZ$OWAHbstV>I6}!P^uyrSlC%N-d0iT8LJP^ut7TDsx)&EO=;pW7GS+*skjq z9x342`}FUC)XYUN`|WZVlMF)Ihi8ySZ_ZKol2h>Vb@otmRurCXe+yf}PUGEKoy^9W zUD*GcBN2`)$7!RfSYhUOyylH2Iy}>YJge6yI>zS!9S(!E#zx2rFQKdpA2COr^Kkw2 z?^L-lg{1$;W9_f$sNdri&bYh_A24L7z&HiwuX#5l_)jORs=M$4w?p_yWeEP98;(l1 zWbr~{C3)*7((#7%MbsXtC}b&J2upI7QQlLgk>f9eK(2ikHd{Oe?XZ~&w>ND@VcUxt z8@>Y&=OS7peNgRVJPYu`#V#*srlK`La{ z2SqYsunyI~5+Wfz_Pp7{Ih;R@YDjdu3z1HrihL7)W6gjK#A|yoHcFfd?ep*8%{wRG z`3t3x!O3@!k)Vs390Qmi#}n}8*?;kOha;51oK-O7!N*h@@1+bDtVQyB5Aa@}vV*H( zo_J(w7V$9|#pF{4)Sex}D>bUHU4sqj*#8qmk88rK_lD&2%$Jm-r-l-&?q@)!#i zWfO_$eYnNPnfLn3N+NQY!1KxY(b}tnO!oz#H8-TtkK+4y`Nju~x`jHK?{pR(Ih5kN zEz6gaiTV#r}@4_MTUft7?4F>-wg2OF+n*&sW#Q8oehhL3R`I;TSD z>nky&eOkpIVU2=lT-f1)zcfxantyLA#de>sIW zFIAs({`mwkYEH!IxfI&E=PW2F>Y?!YGniYnFq0S4PrWiT!)vc?f|9Xkuv6td-ne-J zKaaFUsb@vepB24WVv8mj86ARkoac=2_buq^?^2li{TJT#HG{z1T6{m7;Vh&O0Yt~WdDyO|rQL!7GgXZYgx zEHJ&&4Ucm7Fvs?DNw@Gg?9nzvb2c1AkoG$sX|_Cxg7v&fDEMGG3WfNX|#tXet_>jj-sP6vQ7? z1#|wsfH?P?oXlIj@Hl=mF&W?^dM;Hs%+H(*>F~hfj5j`$@(80A71(@i6*3alfD@Me zpg-Y@5^XOt2(KWvFC4IM#uAkG(;ej7?1|3iuh=8*2&~Th1c&c;Vaq4Y*f3fMh?grl zdd?hm`8rX({dLeBNK%h0+f=W(~m05-h)nQAtD(D=G#0t{x)BH@#Bx7gl4*lBXT z@b{!7+Ly1#>>n$l&Lql^EBC7@s!EdZ=XK!LQE_N7U5>10UjeU3FSwqb4^RDrh)%FG z@UBPNsqCggC?d~7;6pSWUGqNG=h-x++tyII zJ{4Jg*^h6h0+V5BMcjQBAwKWc}#1RH`H+>^<^8xL`fXDAU_ zb^@=AR-xuB9A}>SZN!l=eE556DmKZ!3Im5TN#KqMxc170V&bh)=f)hoY0FfU*DA;a zjvB$A%fYa)y%1Uw7Qt1OJkC(f6k?*#i{tc?sFUYs!V*y%ve(Oaa^FA-riJp8D64OH zX=o;H*H%G79tr3`z%IO9%z-5NO+}k?zQBp*TfnoDC-L)V5cf0+emp9|z9HdcWsNs} zO5K1{Yb}V;nlc>EI}2`Rf{0FPgWH$9km+kdwD03RXjhU)FZIS5w<25gt!E*Ll)TO? zBsTb~WHlr(CWy#h#cyRiQE_P`)^gxSLXvaQwr}4-%wPx>hwnstB}-tx=vj*PsKT9E zUU<2R29u#(2##i+C~etASRAyKv?LASiXd^+rWh=%zG_K0Jj*FNBcoA*PDzdu3&y=%z|>m z4aly;pLlRJ(K@R>CVHbMD%iM?Tv~VycR1EJKA{ZB;O{edrPy{9niNh|2fU`fKK_m8 z#QlMd9u<_maxGlBtB6e6Qh4g$`eBvK46?OnGXG4@gxNVsjH$aZ`nh3%w|`ujJW4)} zKXuPQqJLZPY!y%9KX*3nGu1&_S)X}J(Nh@qPXPRO23~159r4SG6Qfz0=%}#}F*zxN zl=nTs?ea^|>*8Fz$D*DozTrxy$lt^_z0TlgrFY?^#x8Va-WeQz%mVAr{*IkxjspKJ zQ8a9M#%@`18&&n|IbKQ4CciYoaPZ$n_NEIr-SnLrs23ljZFRhX}nXK6R;uwI`iA4vw;kYqq$1caEXHu$!IMF>(b?LZE_#q z@G3t(v|Ru>JbD3dO5QTroBF_bPzO&5-w6E$ft*(t9T?NfYFKj75)Z4oF)iZBm_43@ z&4u+j184M5%H~@z)05z2uM^;QZ7Qmr=M1eOW~9zro7C@l1ke8{lJZaubZfYf^Y+1E zxcp`}YznKUk`{b{rWMDjzRX~v@gNaGmst=IJ!`c0s28a)7RG09>Y{bq4N&0fa$p;O zVCl7bNU~iAIXycJ=9kP#o|!&~aSrwYe%&HWDvW8T-@S{5UxqWhHdrOz$TF_ z8}??BKkb2y0{+Bymk!BVlZL`pu7djwtB6Nb9NJ6`z!#i=N7d7D%iG1|TxK%t4_E^C zM^EEeei1?jEU_dt0=j;=D75%6{#z&lJr9mEGHE9m!QVPaDJz30UTVauBNGgNk1w9H z^DotLeJTEJwgGXjOh=DZC72n}Hb_}J0|Y)~@-9eEh03X77*17Eh0Q16HS}}DJxz)I z+RgB$>ZZLPKLMuhA`WZKB-d#jQrp^sFI~NYJ?n2nTRH1CGyK(zaQ6t2P(2`r(#vt*r4kjlC6u-y{+>FJXioHNKf3*)ezW_Ww_N$jjUfSJS1yn7F}!7I%KYG}GYzWP%d zyD50%{u~QN#8{MkBlXx_bS9aw5d->91ynvsd7w6}4dRqyU=~4m0ERhe55jmO6aI0)468O{U1+#V`7{ zGr=4oGOCw@q|Y3Lo4ecz-?}`!C}##$I-!i#QKsbQ?vptB)>qD<1?kiyx(SxtNJYEb zzA(XF>DXZ3EEKR^0slR*9l18{qV65nK=5@YQX5wzPILwtJ=67ysa{lnbMn%6I`Vrv0njp zd)L6F%_4N3b-kk1McxaG<{T-S9MCroYR70=Z~uWf;B z%~VBerSlkN?O-_ltsUFH(!zqKb=d00JY4@~I^Jic49d=8#Jy-czHI%A;k7%!f&3v# zAkmoktd~NibA!aQwB-J!&qd+V^~?z$-5bsOFEqV@!HZW@Mp_BlviqiMxCm$ zld1-izG_WsLla?++!-oC%7hpR`{R#kYLKVx1P<3s(V)t9+`dH>9dX!89f}QQ>JN3} z9S1*C8Oq*x)!Q@7{!##>w}LNJY^o zvvw0ANq4Q#?T=#Q)W|)aD^rMveDv^w);G+@k#zi4e+n~UluvCvDg)1U*I>(CCSbQ< zG3S}|3d%Vf5y6Lf@b&KjeCcHY^|o#vGx3WiW8FT)XuCD2$@0TSI~iEN^$b`%c*?qK#}A9l>icspQHSAA5PXbMfKzPc;=`%vb9+TuSev`gY~O` zr=W^6OwM43&kSUK4F!9%VXDdZG5&S75|Uz7$dI@qsvB}5p)2R0Te4fwkJeJk(<>VM zd0QZOs}eMydW0XY(kFr0);PMH1M#0F$zZPs8jY*KvyYWi*$qj^cnd!gh?_=?3bOG~ zOB-c^msMZ`07%P*(rG4P3 zrjOR=JCY^F!62bFlQiUA0%2zjawg**Osq>nPR)DqZH9xrH>;wbF)PWdv3|-T<0>Ba z&qj9=x*72Y$C&q#(_yek4K1;@MlR!hpfIo+289an3rSOywAPA@iA-QGv>y*loPui) zg0R!3&rqM*fJe8zhu2%eh@rnHN=|)9^+>Ho({CoAZ0b3z)EfYu9mCX|S&2wFYCl}h zsKh=$Tw&=RO=O_uP3(=-;E2`~^udvW1>@>uRZ|sc4>f?z?cKzC{S&;mUXB?ke*@bO zN0W`}Ca6|>1piU909m0v@CsW6txap;){O^{`Q;hjwN3)8^8{e2lpIOfFN3P8czC-^ z7_%#LKDxj?OC3-NL09&s!I+mCb?=HN=Ih=H>Sv7cnW%9}A?hpUkQNXQQ$$T7hA?A? zB;i&sAjw)&nI8@OcoUm~Biof>$?8+E^Fue5I5GbrK#CTZQ9Mx&HVXQJ*ABRn$BZ43tza3M zkDKwIos*ir{CQAG=3$fbVmP`Z4i8=`#ypXaSbXbdnD~>2^a7@#%`=Wd(39g(^-u&? z`zzwXKT-JRjb_}{D2!s)4`Ry;MNFr{I4=JNXyI!g9O*R&t(%=hmTxsjC%cAm*9}3E zd%J=&rZWQ{Hfd(M)=Vb>%~I4VErQ>!T8uT0x4<7^6Jq>|#8tAaM}c!fJ#h8kb}cHkuDNt1)T9OycO$VaJl*t7g0zVUcHS#sW4`_ zI?0#CEaO#yUac$PoLWtWlI4(}=Qr4Pp2kW$vcdLp8~%GP3PNXckVWKT6l9kTL3@g^ zz3e)o6J$v4Yvsa($T~7#oeygrGr$!_D%5W8g*bC~6c;ZZW%>{8X3lf7?Ce|uV0*(F zG(FCXbZD%{I}@{b$z%rPy?MhSCbz-aP?nU{m=lFNiHLJP8vi=b%`AVo8l60#K(_rz zVEl`W@v|+$~t_1fkrJjDRlOnt*BH?uJKBFJW_ME>2Bfg}iGPF-k(I z$jdO5So-iF_u&xkh1y09q)M)&2U2z32Z33(2!Gxph4#A6K&Dnb%%+4iwE6UXYPgKY zQ?0YX(@vG*cD^a-?CCmuvsE15kJtvI>j8BnNFj+gH-Y?-V+wLl;dcHMe8f~9FZ8I{1V z?pdSx^JilxS50*3+I%!DEQt1gbSK6yenQEkwRpmQ9r+U{1Dm&*L3&C&)jA+WrZ12p z73+iGz-ek# zLJK6?V0%#$wbSODebC8B=uIoZOP^|!<$)Bqe<@|Egr<|SDN;mwS2bQPYmc91su8|@ zF0i6A%*%x5r#kz_UiD9z20d2lwNOi}Bd+{RGy&b`p2qzJWEKO(pjpXrPxBdT=2o3CGK< zCZ;o_bJ9wo}dC<$Zf6! zC&@41d9V_9X^k^Z-)%^=-(7ga`~j!*@36vP5!|TZC-HJy;YWZrzSd%e&~ZH&Ke7&m zxZJZFKq`b!Z-n94U13xPREV$j0^X3@4D21%#DKgpiki5FpKK6A#Y?85H-c|?pQYRJ zm_0=d8bi^?U%L>jyNrWJLc!vYKbiY>8d5x?gq&0bN%rcSl+}iF%{*#DI*`r*79H9{(Sq_>Gu@2Z;= z*A^lhdwp^%Di%AHg(GppMJVm)Ro-1sTf{iD;h{VOqLHWxL#JK9a>9<$n*5eP=Upup z64Jw`gZJ>506VyTDv`|6O~$XjG~;gtf=KuLQ~c_d8ku!V7zJl4psrM5(ibR6rt3Fh zQ^P0BYobW9=j)Ss`+J#O9WDwvQU^`43MgyEEx4ykV5@viFe`{Zi zA9sro6RTW(i>8gb^F<{dl8p$}xC_ND;6=l`8`JkI?1I&t{&cu z2BOx~d-$z@FDG%uFMR7^HO{D;OSIJT$+7)uNbK1%D0q9EN*Ja%J68?iUsi9JPw$)| z=)f4R_6jHa=c}OjfRk|DClEz{Q9~MMZ&L5(yo3Vj3m`FY0Xnt{<2Cc^DLuL4n3Gom zXOKL)pT7f7EZ+cC$-&6Odn3d~*$`8&Ja~Vw723vQ@r+O#65m=2KgF+57wa|2r}{K# zefkFi*Sg{Llkd>m-%4@!+62szsRYqa6bc`lOZruZa6*g*dRur72QJFTO%1vr^|zN1 z+Z%xM)p{G>i)N6+3Z9f^jx%RfTN&=V<4l?cJ8<~qVnHEtD#(V0Ldd>cJZ0`?v`@$h z=U)^@yO&jihps7lk+GJ@pKQcUCNjiER{~2YrJ#t}ll$*IfOe{lW6{YvsT*5+nOh6w z(QNU1@N~fj_;7C;u`%5O%6-~MENC?vwoyW=g7cU|KMjfPzB@R%+ip@zdtY;Ezu1d4UZ- zM!==E0(kvxnA4&92D;zB!95?&fNpXR^~7Eg_a4PI$YwN51q=3KzW zzZQ*P!1L-T!XD)=#f18|x z(~iQYf(*?Ck#e@O9m#FC};KcK}`6nSecBQ0Bv(Wntc+_k4rs(zDmOyd#I6K=($Mhd9?s|-Y3 zG$g4*^Rd$0CU|}~43*4k#dBRO;Httw%6Hmp*f*Ak+S7MIzLhMQS(uI%zyFI>?GvGK z`4;4rzX}#d^+VGqOKM_{8(uT|2i$qX)FFR+@(^8yuoD5`krPIyA3-ST03VWBoB^|^ z31Wvx1r(HV14{0$qkOt2YbY8;(Tee()RcimMCLPQ2Kws2B10W5Y8OR%x&!dmD+j$$ zJPvUJZph}ybfURBlT1g_q{3w@ygTX+JGwNH_A5Q&ojweXTXpc7vUEKEQWB}H+Cx3z z=2Ngu0yw7=u;7+uaO98$B>R0~CUra{6<$eY8Mu=m13)dE*5uFI1T@;P9D6=6z(tOp zu+Om%@?~Gcz+{f4#P<3G#3q_E=*cgkgH?@5iz^VTuW6#N*gCNg1RWOCe{Hb3~TZZ zIK)rZEkuyavJ{-vy$>#kPRF9xZ-cz&BRuqEA4gi>6}kT|0`;4cBYH$&v~M&iR{WMwdAUfD3Lsog^Prwn3_RZY|u|GL3bm3y3q^g z-UuLTt8i@jp$*iDE}1P5iq>Xok`v=w;qi{$F#P8@r2qQtxcw0vnmbjpd3KD#f58V$h=9?e` zM;R=1>IQRP{vn>1XbBk}8u+BL7S7Z;1&SOQa_Qzv9H8$&dXmM+rJq`)H~Jl}pVTZC zCcnTx4ox8?9ScFCeFo}Qt%Rq(@#GQHgj*z6;v_#7ryjF`?_a0G{fy{_-R=rVZh9qb zJg#R#O{w@1~shRCk-DHA%GBTZC`qOe{3q&zv97%vmWpWn_VVl__Ks;-FX z-~Ak{6Zhgyn+;^R#sPV+HAYiIQ>nAN)?sbiNxr`n(C`DbNe#pa9oS!kPZS=U+G##1EW?Xrg;tkyQF;;)O-$!0PpGN-c^5Zk1EXm2H`fX_4$?U9K=P->*%reRY>} zb@d|LqoIbfPx>(U3I#eczbIKjX*4h#2s{H(bk?bbS;f&IdI92S=%h3A89U;`CC2z7 z|01HU7LNXYT7X2Z*TKVQA0R@x84i9HAxk>@!9TH*3f(G+F4!30p2=9BH>qK8k9`5@ zkWi-Pn>d;ppHEIrVx$hOVv1xj935TzHJb@YA%C_ zS%PT&qI#zGXc4yGP)IF2j3DmYK6quOLAvFg$jAIH;QcYf;X-;y<-80@a!`R96Gt-m zU4Rts5<>O$?J)jEgB;C0O#MDMz|5&uK$**un9H}GGOT+wV|QPjWOR;D{k^(qyJ;|7 zO^rnJWhKeJzWs1wPA)e3T^*<;& z?|81hKaP{V_pU^UvQml9eV?p`D56MdC?S=K_8??SWM!mjq{TN0pZh+clnNy)rD#Y( zq@7Z~`}^C&!8aP zubzayI|gv9v5Hx@s{}ho{()8HK8)IM1cHx$K-V9ERMBvdy>FC)`k^;qerF``W~kG9 zTO~>Txh&X!_dbYSyud$q$&TDmNn&m$YSH}lMXcVGLNvSo#%fU3mTuL#1y=i zm>hKzS{OW=EbqPzT4htorPCSM@`>RO>%j(|p{E{Xecm=?ZU z%k}dvfX%&dvbFjsJf86gjXD%@)W?N>Fd4>$V`hxw%0&A8VI*o)o&j&kW^B=}f-18D zM#a4y#?3B6)PfEO=iV_%%N2NVnkOE7*vYnT%Gt<3D_oO|ZhD7pA{RXTw)dVEdQK!@j%%Zhw@ajZ=qVEZ%{aB&0`_UXH?y5NVp> zS_oZi1Uc)AY_F^h*`xo0P161YPbYoAW$LZC;HEqMTE85kOf0F?LS17+o3;fgOIL~`#{Xv~Ucr=E49(=#LKg0~l-T-lfozE&l4V-osrYQ~Ob z!5~;24@#HkVa>U}=+ZfX!K!XhePT-a>Q>-2=uZCq6(+%3k3zdu62+mP?B8`$uvKLZ z{q;&o6|=XDzU;L<3H&_9U&ego)$kuhv8jT5{+lxc!v{ zjmgsFN*;p;YY?D6*Vr5Gq zbZi98Ot`aeq6LWGj>YRJ41-EXAp4XZtXNrykEWQB%)`-4_xvm-Yjz}rSMSHD)a|sd zZ5Jx&NnrDmJ5cc1hh16_iGPyqNUU}o)&4LC|E^lf_(2YDR??cPx(|Bn8n4M@TZ}gi z=W9^M0s*!jkC9(Nt7!T~2ddY=5aVqZShJ-Yk?8tiU#$Y^wAf85Osc_7T!6-W+d>j( z1)I`iLf&-)%^ojgjH?eZ(_XztVblL$`}r%l>qa$bNAi)e2xdiWxbt%IBC>oLLqh#S z34ib>-c4EvLdwTL=!qe@+a*lirEi8j)5l=+<^>uaID?Dc$C2R4xX?KAE*{|pzZ4nmQ&K8^dKNXKmkS%bkh zY?8!Z0C8u!#$Yp>>Q{~`Z)U*5fip0Atc9%|ab=9vIv_P?7oN6%jR_8Yp!Zw{Vg_>< zj}wAKdbK%uYZe7}^aw~!x&$wqp2OFWT7IU;R3hLM&&E*&tjV=x#cDQDb&oh$JzR}j z|9g+~N7u6h5*;Y(brzc6G{Mk8K(9JZuWsBA&FNOGnS~@-GBcK0dc&Od?>AyU@9D>B zsjuMEYejNpEDf?|MT2CI8qNRL0nI~&sB%G!m`$5ZU+xN=Zm-Y5M=b;RjU_z!;AT)bb{A- zH+)H$C(kic39XXNuBnof!_RB#&!KPbbWma?$rKe1}f%~o;#IbT`>boH< zDV82#tk9=L7Z$kuLa!qlWK47p`RJuerq9%d-_FOdQy;PO<92X)Gly!dQ)4Bk^s@02 zOlaPzMPyUj7w~%|LWl27h4xA{P>7jApYGKJoz;5uefBwEjy=ZQ@{5phRS6Q;iZYQ0 zB}wpzGCd>}MRmg^@Tay4)gI_)zkN2NVK>F8s>gKPdg4BlV!8?swEu>H<-)|RP>mjs znt)qW4Cua+eU*>S!~trG^Nuq!kq*9O<&57lwqnbujja=ju6LlcO&t18x1hP28nvKh zAgWQwjO>sSrwS*|!2bMk9A3GBJU()S zv~k&*>RMg$D|8qAK6e*Dm=o>kP@}8PNt4zI9^jbfLSLP+qxH|)SUW>OGEYy4o@SGY zQH3Xb&Zz)bA0g(%k()^GE~Ehs7vamz-%PLVK0NgB2flGS%@dQ@MH?p;vEvKAWASVT zHRd)$!%276A~BjV`q0RCUX+Du;wKrS2X8?xHh^sARk0Qi*OCDCDK3mWja5(2;2LK| z^1a)EwH%&ABXZraUR|B&xeT10?g9PB&f;dycRBmdj<)v@nphJ7 z0eu1V$hjrt<+?uLKU&5`_MITovKBqAUt*F54-=Z-14=55SX91(XddmyvM~v|yrmc< z*Bqek&U4A3i0ztk89WHFnb>gQ}qc+yy&8Stcg(|k%j3% zeI&>Podm|TEsLFns-!{7mMX7*2nrr4*y4YLJ>HuR*2nc|>#Kw0kjG2x-*S<^y@OD> zN;A^>ijNIylVHV?zo;-}HM&?`Mm_O~B*le0kIp%dA!%{M(6}5oJktfA-%4~ztSbF; zw*!CQQY0(Fd>NPNrJ(&OvudFK3RIh$usWMKos!#!Ox7n-#wn8x9y|lSU)+i0>3HTw zdMLErOrWNfs-!*WJ_H=H#%;5svGVi->{l;k4U%rLE4P-wlE1fMuHZCUZtH=1Bjr%W zWq3pM&ogc3RO!8o+}<)Wiu1(7Kxy%8(hzb6@(uIAGM&?xM_!`FyHwZ|(+#B-W@J-A zI@(Hw(yb0anxo~&9BE0y?kt zI$2>oz_f9Btd?CG{9UXj$KEP~`J2t?R22gZ?7D;JDTSToo4B)?9-gu8LSLhuWc*br zh^&f$tw}40RNYZ@c5@+r6bd1!aTP1)WJ~xXq13bM0)9C1n)%+XO8WZa@$Rwt(CGP% zJ?0!sH3L`Bn7mqCb#NoGrU`_R)n>JB7K6gO0e05uIV7O-J6m-#gnhI#8bjI~NT}Zv z+$P|Jt+vW^pT`&kdW_)&v%S<)P?OY)&1I+O5KR5O11{iB7*GImaX1xrz8Yj#)J{QR z<#puiv!yV9dMi^~`xy?!+razyYW%;pG5^4Hs>+B=lNAZ zwtJY`=k{dn909U#EP>8l*aS|W|Ki>mD~QhlUGBV^2}Y}rfXv1vIC>x)c8rW5u@;2H zd*8q!7PL3h!)XnJBhNvueRhVE(PkJ&tG8S?_G=j0;4 zMGIYgBQWugJb_Fj(0?UD*B4gfSj{L;+HDYK=~}?j$jvlNISphsR71A19tisRz?{8F zSbjN*JX297!&_oVjw}JY3Q^iMrwEUfTx8Sb@?qs&IkNaxEC~{iBh1qiurD*3d`NaB zISMKCWSkNPCxt`V?_)T)!kq59R)*n%^WkFNdwl2i3ssixCCaPcGRIYX$dnb1Fe%28 z400Wy$B+Kt#28O_*Eoqt#qJ@EhVPl*%Fc9ACBWPAXxy|Uj!4KF!-OXj=^M<3rDdy_ zbs5oUyeOAxR!n0W8iC$TD(aQnnT$THNo*#I@I+bZpb1n`{ZY3SRt7Ngw`!$DakY}|ScpU=Kt z71=$&XuV5>bpLl&ei_D~nC1nGT%MtHR|8B3EqeOGYTEf%mLAd+qH6bc;@y$c_&K-_ zq-Gz7vX`mw%OQeoY`hOsh66w#s{yTUOd%6_Q^|S!0zdc?%HMaaajqwefw4Tt!c$aa&cAPd6=TPlzh9F zhl$C#{BP3h*_80}`0!L0RwN{oa#e!?cDD@m_VFY?e`Uk%C#$Kde0tUMH|J5h@iIC# z#{sXU6jT0w<~KzCghfN9G^xrN{kKduI{F{wF)SMwLdS%A>;abiBc> zy~7J7;7O+#YV%@Q`<71hv~~c$)CR`0(~^n?7|_7GbBM_cD{|Swgua;`&ilD85LQ}j zfrBj*$f<|wNUwOu@G}|(x5-%IaLCIV-(y>&A zZhaijK6YS8#qbJ(J^~P1AV!Zhw(|_Hy~B4Qq3Ds2%d-)8B(pO_dA*%K(QuIkol5~PyYnkqQd1!wy$R~ z4REB;Y~V>G8$aSi#Q^3##4@vwOF__&#js5x1k6>%arzWF;^X7aWpgsvG?x~1jh)LR ztFET^tWBxMM|*l@o(_$2iKczwxmfnx2fuI`mj<6zc%`fxVlouzkq!FPq3;2d|F{6R zB=#}sPru+mb{s6kL^RBPhytw=Xcno$rpj}Sgx_r8ou z3(+~M4>v6C=8r!d;9D3(@Z2ZJlH;o_$v|X0KeUs}A}p_BeoXIVOegd)rct{gZr^Hp z@Jbyh&5WQ!b}_Vb>@MDs&;e#KH@*eC&^gVAz{p0O9y8Ua&x?!jR>Ukkw8e(3+)*WKz|1YV!O!6t-l-=w_~uY-vW~k}5DT@(`R|tVHu) zD$^I7&!H0V1@GI*Fd5mp%!y65Y^|IjI2cP~#al!EK%fwD4a$UP7GH2;&nMJf$58V= zf4V=8M?JJe$jHifgzGF$ZpV>jsy z`_RmfJdl)RG}O%L)oyvZROuo^e~7Y92dkN5ZQkVfVM7`wr$TR;>60n5mhj?&`a$4E zGun-BBBg7-u%E}-JxOWSkX#WMIHIwKc2CV>^*tu2aj=@#75|J+(HAj zxtyiaFVJ3o3XV&j#T$nv)8Qptj{Ua^QI2?u_BHmbylDgTa^f!XL|_%Il1U*CNHu%v zbp<2&<0Ae$y@i&Z5yCC`%joTP3nD5lL6XYyae(u@CR~=KxeLZYhRe?U$3(I}M^eD2 zX-P1zR$?S|3sA^$@?H zpce$RlQ2Xv72NW7Vv1q|T)ET2>7x=5IKBqLV4y zSp;b{3#s$HJa}buj%n}8geS+N=}QqgD!4?5M6arb;1#{F*FTW4Hi(9)H)UvPd<=V7 zUJcw`t1wlr5Z0D=g6vE#3paiS<6r%U72o$m@pg6ay^zbx;kpamd*s1&cO08|<_}wE zc@keJbV8YV5^iVY18rzd^Kt6Y zY)Td9bukgo_VRab6Q>unA97Z$4DmL&g};8?#lbm3AmhqldqpjKy1?yKOoPo|ToN(L9xm3w!qfAJ$re9a z>kl;J_VX(K@KKmyKan*KaR$k9F|uX59PQrYO;_CV$FHp?;8&O)H8*vq>8*8e!bj+aaJMc9SM~6`$PQoIncw~4rLu1$#UaASjzEG_8mD7q23$tz_t-=o%IG~ zXSlNu?iXW;_-?Yna~}CI=?ap4TX0$1bC~0L8gbozCgNfzqiFhypBAByZ{!@QP~cG- zYAQ(|?y{uk_8f)#u#HUFdy98s=qpNB8`6TiK-D>qy64ALw$#K9c{1tDkosEk@yug}Z^qG~8w_HfNiVghlySRIEsM`!ht>v>7|E zFD6})U6^Bf2-}sm!Lhy$tSF~!qGKR_|HjQT`v%!awcC)UE=z*`*^?(bfHVe8A+NlQ z=-*vm@RcR^^B3|oQo0lW+rWb>M{l4}m@X8HNfFO2No*yrgVmT<$i^D2qH+5*h;-F= z5Oen=d4mh+fA_>N{*ETm8*9W{Gaae%{umq`--Z)Ll`vk<2?HA6;7N}ebl^!C3Avtu z(E>qa^}kWJ?RGLzcJqLhsuNlM`j0sM!3TU~+78Vd_mOPdlW3sy8`#Z5n9_40af#a*usI}2v)cLKq87t+ z6>J7BGeeuZ?a1L7Ui8I{iS)XZG8Kq7Ciu>dseU^$QYU$em@@ex8iN{?AY+A_M*A z`I7D%f+RGm7~-z`vi2Pk(5VpwZ|#ThY?~c@I{J=9?)#i8-oqrWbY)fWJ2fdQXGep- zPr*mQ+vqdbDKw#D18Gs4O!N5vp!>Elz53T1^tXTI8Q(CW^?^%K+1-|?O6J{sJ9YsD zrwX&{BL`vbm4kGXkq@ztyozGiMevJB7la$1;Frj3W=*5#k)hiXR8C8V>=Mqxo=aR` z=Xy83kXI(AIt_TtB^1v6mVvO`GmQ52HmDs`$79uYZ0Al{;wV@ME~_TcvCuGj_M<(l z%A8J5)YuZ4+SMc>M3Tw|*F)f#A3AMDRJ{8NN>pbPwSynAlxL0mu8C36@h}YIc!ZAS zj;QH*lr9xhpcz>*)TLCL+;LZ+joTja&y{CGa&IZ@nnE!z{VJHW=7FeR2rRstgN14$ z?BtGF^wH@vSZZYm{d?lDgVWl)eb>PcO+DP-aE6Wh5yNFMf!2-OfU6fde$Pz}vZS^a z1bu$u%k9P>1_*Im%=oZcg3k0bB^&TX)sAL)`Zs72-Dvk6E;Jhj>H`y*wX^hLx9AjNbM++43mFHc>w0kO zzv$!UB5R|#cQu?9>s6#fS6r~=7RSYEEko_f z5QrU7#+6Tbc>mET_s4$6!>@RJn~uZqLb(QO{-stHtV@S&_U%|t%yG`VJCJ)chV=hr z;kd#X@ORI}W|1BAudArc)>t*@xFy38$NIb6YtC`Bq^0_G55SIF5F@ z8Ix<0z?^zzNk3f7 zt`P43O|}kg{>@ym`3o!67F(CE2%~hz0_-_7m)69{;a;8t30Qp(^^e#w!Y-$k}0&H__H#i>h$4i!A&K&O*Ds2bxzYvC|&=y(&PE9epx+ne0n{2G=wT*d2g zm3VD@DKmY00p49@%o;{~;?L{;51ch6aL*)hy3QgF(!Lg9>zB7oS@t|A$*260zBq=M z@1WUj$1s=u!qP2E=oW=()KFFsy!WMI`bJF}bZsViDH%?mFMh!*`qWVsXOKvx!&S-i zGHvRqB@Es=6R7ltAMkKYgLaReV-2EH@S=hgvG?OLXFpysJ}&N@wjoU8+ACmo{89Fa zQ3S5qS&XGS3USvVci7WAk(|1ygI-IvF;C8}B-3ibiSSP$+LhFg#S^&P*VQyQyqoJ< zeT=1Fl(yrOEL%u6*h0%|fK1WeM(4gMfGmsI)J^yUd#k{TzQ4E@=NuYm7GIDe$JI>8 z1dgfIb6b<}&6^on@0YyzygsgjcLSc?&&QkZUbE}@#qh2{kA`fVLT2>+;6L~_7rr;k z(xnHu?q|CyE}dGy8o8L!&qXGL-CqmIQ~xm;m4ETMWe(5lfDqlvv4T&&t^)D9nV9+| z8z1esi*@$`=!M!!6c^B-Str!Vn_D{Mg~WbTKIuh8&JUnty*d;`OG5tV*^pLw9NtV4 zCcpiour|k;|M+hLyjAJs)?aB7UM~ud=EM`8wnyVFt~mX@o#OP$B}7?U zhO%cSlM3Y%?5TIoj1*Z0(uqgur(DiM&X$I^yu)fgG)Gw!v)E4Dj3GMXT+m z5Z7Jb;9E}|cFo=nYor-C_skQY8t9@`$SZCyCxpo~#g+xTQMG0f&D^jXZ?s7fyUADa zZrlvi$rU9|Yo9^LG&zpfewf~N{g3mcfJnVsMlxq?K`FS+jCLr}UBq^%2R{ageX||;Qc_Lst>}+*Wr)s?2C&?`^YM~!si9#!!Ks?Y*+I7 zJjYd(y1)kSGNo((_MoDc8P1;24<7^ca;K_=kb zI|56#7?XY(G5F_9thBw}?Or|~LL z&C)`ACQ^ZBuWrJgSj-iGBkzN3oO z9ymgRxSna+wjtI%a0ab&F9p%V!gRXJD-76{hdnx?^hSy`mDRor0a@BqPC$aJr@h!- zm`L=j6iBJadQzcVf=$jZ@ZrCB6sYzf-)A@xHT5Or-JTr$^2(6byqriDDCFS5EkbCs zel|!m@$6-brR1b&HhSKj%9vm8WeD#sC}lM8wsT%tAg#n{o)wi>xXkwPFR#&K^#a;4 z%7dG48?f$h5kAz3r2S3Pz`QmU0+h_5bL}BGnsW>V27&I~d;}9a>}lx+agq=o${Gir z1YYB7Ff|ZBzkN%f*0_sR%vU1^CL2+4AtSQ%RT4^njAm3Kx3NRy4BlSdi5Y#*;ebvF zZVnhh!9o?>EnbMtYq{rc;!Z#8n?_DmIkQQMyBWQLU+`Q}fZlwoi6!6laQ<&a^4dO` zv9ss=UPm?Rz8rwxwupYnjv!7xv%ti4Xm;Z% zh&VU{LuaC?*>w8n?@v}MN`TD0EdicyT<~y5G76?tV%nrC)-ki6FXO$0Uim9QhZN(; z7vG0i|6&dFKli1l1|x`&)N1mHtR@vk~N zXdi-ECjIbBFOIcOyp5MeXHm7SCJ^Y?487BL&`TZhtX0rjfO$)))~Q(}2Oi+c$`>%Q za|eA_(!lXvW>G86i|^g$6hb6~glQ^YD0q8%cSZzz9n;VpsAlsERX2!`l}@G*t`L z=byy3M^)gP(1Y35!Zheg8P;vmBTxPMaBtBCC?bNmxa=F~W!Ay3;`8wDjw2jYdI@We zZNej0O=y{}8< z)UE|z{}5zpUlK1 zg3S2bhgG(bJl{#mWK*UvGLF(D+wHCOs(%Y;1h@!r{WUJP+&h>6d9qPFYlWGl?ov5pdS(zqtkb|CZ7tcSuS`nLyOH^R z4$PCq51EmgmoQuW3S%>%MpH+v$VTa6^ z9gM|OXryTsdppB`Jm`6Z3RjmvmB?~@l;{D~k=vjxs{z_ReMbM5CXQ1eMEia;@$=TJ z6Qgzy;%8QXvLf8M;revaU)7HatSB`<{)Qd+n}G@Ey&>%m$NC!>VPBL>Ljsq9`J$VE z^_LxJmgYOqdc7X>Q{{>NX&?GUd=M5(-DM_}?;wfaKEWk}r|gU!=IqO9Wvu!#YY13< ziS4ZOBW+n5Xxk1E=5fMCs1X?CZ(P*M?6Nl^mt!rcZS*91cGDC(UMI$}3e@PId=tFb z!m&FS3evr4c0?s}59wizxXj~fnD8KhMlCsmqUV>BJhx@^-X&Ai`o0xJLT*C!?iJ+G zKoBJUcN7}lOd@W%f}l26oPLVZ#4WS;5oRvO9@}b0Oyu3^(APBhb_$vH!>O2~uL*^h z?4jDdo?r9v09=V(3KGkofRKO{IcO<{Cc2YgE0<-l+llooGL~P9sc=a_4 z>%M%d%xpPXC@u>{f1Nt`-Jka?LunM8O;1%xQtA_q))#DjfYzPKW)fjyQjSkHWvy=)r(- zc6R&>T%Q_&O=qt&%kJ)HYmIz~kC!MtDxMBE!uzl$Y%$GCUQGNBZ=$U?M5y!(VOpWU zveyjPk>v)?XyP;q+wH~B?3z3kKa>xpye%+=mABq1qff4ec~Q#;&(T7%9GIRr@G-lB zH*Qq{QDyOXZ(B8H+-1l+6DhWql|d8fxwyBb48Jq6l-Kx&efI;ILl5-G3a>mA>X%_= zPw}AZR1*MgAHdYGDvZ^OWr|~Zm~H2OTCef;;gDY;%wJankQfbxOT$K_Bj1`>b$y1} zi=IREQW;Y2=R}V$o(#)0`K;rdp}$F z$r@p77TZF5zYDn;5kwzvyvyvmL2wG^QJm47jr$CQAvWC|Uqx&rU->$0;0$TD_IeY? zY`ujmA3VVg^Y1dJ4;{lcuWW#jpKN586jAei!!}%7L|%sVF^{UY!NT|x@Hj1iXuk=g z*{d8G$ygy2wJ(9m|EYkgPzxq!m%w22Cg$hjTBc^L0hwU4n&~!JK`VYuq>sQHKfkyD z&#rG}D#x1ese}xDQoobz6AYt0HAz?#?uCEmY=nnfuS3|uA{gtGp*6Y{tc!FXrWiV* zq_QPZEWN~m?^LPB32mA-Rhb-`Sc`$LOz}>?GA_S1k^d@j7W%rXz>)+za{K2TTysu^ z{Q9|tIE_a`|NRvheryhn;qnnlw}oid14pWJ`5jNne<35jtq_$tR;Z1O5pC)*qaV7D zlX*K;>DruY*n80#ebti5e^L^-O8yoIR(122CO3m^439}!&m-T4r;-gDRB7V3R^0Vd zi=O1TVZvMAq4+TyqWz0wOg2Z5-t;S2|KbgLl5JKAzooE_t0)KQeI9Ah4 z+TkFJvQzhS-TCF@p}P{to_+=&9<;O0-HU0WRw}PiC5lw%EFumf(}|>jFMX&#ge%0~ zK!th~{@tL-`NM-4Uj7A&R;|Q%zlC&VzXu8qe&qjAZoxDwOO%M#f)_SsRQDa%F)lLX zt0yYcwR)Y_V*3%rCbsccK$2atw5CNJ$P zQu}T++s*mWh0~!sR0nppZKl0y!iU{Qrvk22%R66s_YX(5P&#e=%JL8Ry9) zFs+x-*KUV82@?|e(-1aJbR{QU#p!aNlZ>>78ePY+D<151AlHnFv7uQ3GX@37QFwt* z6K7MejJnDh+WT4FvP3HDp-)me-?2ZX*C0*WgROFdtXlUel=$pOJzW1WX`fu_y%lb> zhvO!>9iBv{y1LQQ(vNT_*A-P)UT4mE3X|&WlW;O~EiE`(%0H+-mt6a7PLrl?q}pT? ztLAhb59!&$+ObW{yl>Cgvpsrr?;>4#PJcFxKkC7yKlcIOO%cm8*HL-KgPc zskmr2zW%uk&6*})K=miAs~_RrTKEw~ha+LWxEn0d{KI!Rb$}kcF3EX!eW2Yeg!im! z@bMB`=IfJAyy*U(=eghoqJ}DrM9+fsZ&$D~Ly3GkVa6W+u1MY&+tb8SU3&GZFcET) zrMK;4n7Q2CwzZ@h`bQ_C<_A+^$1&~alKrsLM1y(uNDORaV<@kDF5FI&BC#*ml5Cd& zn6ds3MzlP{&C6GjrvbLil+*_Jz22BKM?3-3HMY?6UV$BY(9HaE+`&xEI08+lIbZLZ z2x@CdF&?MgJJhH0R=%o6Ut$c5Yq>Jfx4P$ZWwn^6AGbT;eNDC6Hb z4fC((pt5TwzAd~DaQO^QZ-2>*w@Z=cE;-tlTF$S}(3uyQAJ+r5u@3aqo;E0*oR7t^!sN8^4JiEF!n!{Ah|Q;G(6%Tya=`iq zww?kaaU~SySnj0~Y4`an+M*awvodIOx`V6H9IUrbB+Z`hxbF50I%>F>RDH@+RsqErw{WcOjz57x8QEBxc#r4KO^P zL_c>s@e{W`!T4Nd_-JiQ8U~)R@{2}b^9%!8XQ|5USo(|kHYtQYoS(v&f4;=8IPegw zJ$F=D3l}pFRs__Tr{FeWdiJx(iwU7sR9JvDx z_9u`ru~1T`@)~|~xz;4-Mtt4R;|(A5r4d=Wa8oy$QNC*id*0uL4b5$=_+S=2$wQ@Q(B;Y zu`e_A!2qtfd=+2TIB}W(RlKWB@x15p3<>%C4W8XihiMx=fUMUWe0rdj7cr?Cr)*o# zI5%lS#Tq_i(QQeHPcT{WEt}2h;BJbP6(b^%bBWgx4`xVDl$16(;6d@PutWSme$@U% zBBbMnhSpJ3WvMU*M_$ArPBJshyp4I&meCm|7U(q17P{&*$TCKPNnW#r`Rw0|ix_>b zAM$}+Vz7?tP5qDE+s(aC?|<{6iwTD8Jb_yS?OA(|#Y9wa500hYWXC^b08gU|R=gFj zTG6NuZ&%Ntol^GnbAU{PA!rif28J(Ut49Xi*b) zzrxK3q8eAs+rD`!88wZE<^L=Rf5tC7+{JZ|4}HX8-6inI_6l<}&51k@Si z>SL~94VRN$jx}Iovj%hLdqL%;crF`rg+1tTlXqyQ7g7HwPUSC+u_B{!@O+XZNqW_d zb%D!y@*Ll3?w%S}wU}c|drFW8A#bpsb!3(dNRlnbWJuJD{;K@dTZrDj2rx4gf(xl@ zv78Abj>Cz>OXLdt&_Bjy4;P`I=WgclkP9QaPn!(?k|s@y#ptIu!o>fl0NuRCAC^_9 z(YG9f_mM_3dYnFru8JCvd;1V*-kZkxjZttaNfK_%jHE9GH*(*60c^D_WZk3h@Fty1 z2h}}NsBU417S&EnnS?Dpa8nwdjaI>hTUFR|`ny%4w>Ec9b|Xj51~Mithspk#d(cdA zG3I`$g}vh0@XWj(Bflt6l_)>R4j}Zr;eJ-m&j6S}5pwRE9(mkpkCnsHn9sanYZ5*| zWAScw;7=lHl@v#zuhZxZpGoxZiF07eUI7E<2`)_XgM$Agh?Y$$*6-o^*^6wc_v2C6 z^0Nm2j-AKQbQvOFbQLaMxCqK~T;Qy+8o3k{M`YuE@k=c9=*2hh+1*znU|E?M@!$R* zzLY(L=Hs7G#B>(zyA=!mZUFh}TSWojFp!@5upjCJd-sbi(LZNG@ z_bWki;KFhe3k~q|*GkG{x`QvLzm6v!0D<{k*7lRR?2y_7nyIk_$yINn&=E;RzdnVu zIf7VqvKRlkOkj_Nnvy3q4)m-_H%6HYlk$01bo&**D!Sz#?vnY=Zhjd9(k@YWCFKGX zT$Q0Gn|x?aHAAQRmSOFs!(iMdOqdaCyp%A&8*NTw2HN5nLrrH~WOR!i^(cnEp3U4n z4YxS{Wewa)MZBMoj>2^u&ye$;HvT>fRxu-(OOnAyP?9V=MR50>Hy{q7aCN;FtnG5c zF|R$eNg)ot58JXmy7CzOXcx`QZf2xaKEa)m=dki%45r5ql zd5dEqz(fG{%5gc}>Iif}E+SNS_FZMh5GyQyvtP`wgSqiv>C4$EbE~j^h zftoL?S-GQ8z<|OT8J}}>S*S#%~I;LS-j5F%IRUn(znWNuf6~-kZj@%91gO+9D zH0Y=(^-^j>KPe58D|3;{Or+qJr+TEVN}Xhs+-HBCn2HVkPf^83kpDF`7=Ek^XE)Gw zWGKs)SRTE}WU7YKche_>VAwJ)ceR2yc`nzT3Ok7&pRa?u(I{_+eG0$w@&s}ZI6hTE zF20P2qQo~89^2+X@pwGbvO^C?qMNb&^(fq%b_+hmSdr$}-2e591}@Wepq8zh$kp$X zjMV|PANXVfM$s%xl=li?wV&v%Pf!KldEshxJ4A+aNl_Y>w)%k2$UBVF&u6H z!wkN%cg(`^O?(Pu@%eauo+c^0phSnItMOEb6*)OE5qISqkm=NzY&LI!mTd>n$?h&{ zy=iBA4XP@CCd~oeumtAmN^4f2P>||BoDZ)KL~|LnDO7|NW2zcVa6w2o>!z1Ni|m#W zIUj41xnd`^TmA;7WVsT)z6`0^wuD$4YLOEjYd}Qx89dHl!SJ;nl@80rV{ZGXbZi$M zS-6y%^(qqqSwq@Cxfj!_vY50d0d)I;T_iJIjGm1NBD%7t@Z96~s4>%sci8?c-v2wm zmK}?MdR`bb>;1w@n~NFo-UuL-R(SC0Fq*rrC1Wwkj8dxz(aM}enle6PPfIvy=^MwF zLo@L5qzZUh9aeF#t{dKm*kD$@8tLOYTT-$IXvTKVw?FTTx9|OiSIJhyW4$m~aJqiu z#A1$Vxq|d093_5N&$H9GTqj&EX8zbY&^eckXxBw~cEZC9e9v`A+g<1L=MRU_$zpP( zHL01&2|zftwuTA4;!h8c|HJHM=IncsWGbk&i~XRzlFaleW1j1&lc`%~k-#}6SSULQ zqXNg!XRZ-$ZMcUi`O`SfU5N(1oJl0Oe&95hKsY8j01k!Wq(1c;z8jAt58G#v0v$)* z)A|AcO?~F>Omn=ayq3%^v?0f`-8e?HF>Q8AM*V1p*)z;z{WC1c-`r5VEnfr=>W)F6 zX+Opfd-8mWBG_r0mJp+t>)^wuj2+{kN7#`=9_zeW(!?Q8Od%YNWx?l+;Xo%36T^BJwU@-{JoJA+8$I-H~ zdNjFWf!?P%=929V(7E2sgnYV*e`V|N%eXBKF8zQeLisGuX#q}JR>?F9$k5u4w>TX| zk0{3+w$7y=U|aBX(x%}}VkIAf;FqIREp!CpFE_&fC^`?np8hY6w`po>s6<=ZMt$yi zH?$Bcl{7>Yg+gYD_Lc?>MM9F5mDJ~+H$r42qm;hcB2lSS%KF{kpYZ9?=X38l=knlTovpG(b&_`(Hyem&^k3i0yU9^G!99G14vMqTtXxQUQMnq;d zy_91>CBw@w;o2?Wt!ThyuX1tUGmfhf6OI2#%8;QhGottB2yS{_%akt8fX9W~iB98d zc$4=8*asKUh{Jhn=a%7`PaJ!G`Wg0z(Mr1dvJ0G`;gG1JgW)@3>2T>x`byV`jgS3= z*L9W1nzC0`KeeCZ-*`WIB;1tdjHuDZMK-uNA`V9%7VE3S|na{K*u(olJgN zErwbD!j*59P=nzFl>3ADbKoU*e)$YWn?8V_{auu7<8Ji#mf(u8$7od}NMemoz*8M1 z5+h{`FQdP)so63#{5>JJ1|LGj8c8xYCWd~wv7gJ0C6iObn@C~hIUettH#xGb4~9bo z$c5|QS&Pta#$fq)+TF(;8G!2H3ez zhy)pKASG7suq1&(s`Ldo(_IIv?g^1|k5kz1m$=^Rmf;%rMy_wJ+=LDOQ^>p7n&{Ns z3F^ki>=pVH9QgL+(E(|?^|vpv(>?;qS{lS#it9CgG$JvN{==r&CB$hYm|R#SK(mBi zVXlKLd9iL0n_l=EmQ{rklkg+V_OnZAuK8R3^I58N^NKTQ^``^t6aq-|#7*Re@(0$Y zW(!%N_z9M#w_*NLcediP9Oa!jgsTIF@KyRG5bYS`4F}JsevV)GtAZ4%*8M3Eo>#*^ zcfFhA8x*j|0t`UP)QM>hdkX0mvLt-z}ic;zimj zg5j`RA{&;G1>vQ8>CMVsaMa*9W9h|=l9@hzF)f(vx8eF8dY(A0piTk~OeF0ic_@{< z4rUq0Vnpg)9Iac!%=i>cd(B^gdG=K}BXAzBs2zd$)75w=e;$924gBEGwC%+PVy?;MsMI%MK<+J=GrA0XUw_5BS=wav6m=3G)MLFuOPqKq9R!n!RjAUe zO67j#vE_0iI6H0<9<}gcdMzF4CQq)TEm(`@lH1_sqY#|v7K(vmL+m0c33@C0pgV^u@TQQ=3@b8C_!|uEKE(P&moclYI1cE}6O6U(Iyz7g z%GSLyr-Ql9^x-l|P|2Q&PFwgeCVqrH`7s}^y|tmCA-^Gbc|Xja_!)V)f>h*tP`R~t zVN9(8Q+HcXvcjMJP&A*rd8{Gc-$LQ!aV`Vuv6^o@uEAc)cB7fXH+YK!o6x7qpX&C$ z8aqmcAj8fWgE}| z;c2*9?=<^$?>(kn@hWaH)ur1OM1tJCK=2lGhiF4FuwYU^;y**YJZMY9V|Z}FtBGlP zeT4Du{syOo)cJbyvQ%|J1YUnz$O?U&K~L?E#Ahd6h@xd96yM3fl4)AR`=Sdtyp<+L z12$9Mt-DOQ;A-N%Y6kuD+LD-7O(0WEkD=f_ak6jCJa+mH0g7%I2d?1hN3$Y$byv-L`;bxdhieIzFYvI zU&ipXFpi$Ns6vz+UvpWY8b(yPoB1#G0rNq#5szOor?qPHaE68e`7kw~k(<-Yn!0Gx zPa&@$In$5du_y(Z!hV#))imeXB@A3Ul^*Bzm@bz&&ZbEsblIuF=i^JD<9I)4rG7`% z{tDdGJ_$V+N>R!0bI9X>rL=y@1#U(*ViNqg4(gsau)BaF`-97%c*@{p*$?dcpr>qM zEBCx|2_TsPfu!vKWpcdlXMY@)HMa_Tg)T zetcp*0B-lp=zY^D5`K^mj*>p;A}Gi_7g@!2>iq@3{spu)%?}#4P9+05id5(RRQhvd z3^x1@!oDCruDfgsJs*_FF8!C7x8i zpiWiMaG%H$Qx8cJwk&`>c=$OSJKqTBjxWTV-)r#fdX7_kTY+?xIng;@-2O?{otzqZ zz&NJrP=EV))UmjO4-{`OtL4Yx(Z#3uB$!9;|NaGP*DKNZ#Y9|UEX)2k{2j*<_rkg* zwGi^Gl+{hX3(jjqXu?S`8e8f_%F{e?nSR8atqvi*e77 zLyJliE^MXPxb-2v8MY&9YiD8l-zntiqoqVkzXX52e}V1MdN`Y7@T7*8G09r#%&ae- zn6XHldatl$Thfk$=j+EPI^zPgyv!i+t|s(>ogI`+qv$5HlDa9SLq@hd8yvlhYFB@R zv!55TU!7ONx=UVInSH3vv#+23ytJFq`=rS8>S+L{08NrUEK8?N{)IO0xU2$Sg6>$Q z0^+!y%3M8$zC;%<=48UnF%_DUW&_V|W-wt*`gBERAXP5h%w9kC5$DL=Mwf-b?Eb^A z;bqiB`i`AVmzsB@p|cv@W>*KxVrp1o6@q&>uWjQWXQ+&sP8N&51;gKMydQd+)Nw5X zBT5u<%T?&}iS_7KrcY-+9b^VRXMpsYue{INHLQ*PVenOvBKCZ5vg%nPeeXUS#D)~f zC9;lKN|xh~x_1oUWg3|>cG{IU*X$HvjmFbc*$euXzp-I%C0j>m+TF!hyMr0RzWDXFglPwun$>hEOSpGw1> z>9fGTOoZUSE4a(2f`^Ow?9jg`cC%MIwq|Z5`;Wvhm)s>t^zu#^GpWF_>E2i@K7-i( z{s^iT18~~KnN$zuutBfyGbL|AQC3TyR-D(xLwXWKiMNcLqv~X4x;*nvU;}OB3y}{8 zuHpD;JND;NEjpX!@{v_X;Hbg@6z{$QB+-hjnp?+sicQ8AuD4k5Y!`E7f-6l=kOF~{ zH@H^lFD`33!#QI@amAfiF!Xv6xzaj;%ov;jpVjx!v?dXH_}@Wb+eL})s2VvoU6iKZ zw}+{Bw4m=U=UaKNLHBnZXRZ3*F@IGQP-Dg;RH@wz_a9&5=JO38CCkSH9IxngV?G9t zXfv-q9mC!qnUMOn6g^bs7=frraKEvWb!w4;it|@^&nq?PmoxRinXvGT$}Lnq_>D=c zpH5jF13KOQIn=qmgqw2Sctcl(!Xz1JFE|Yr8=f)WyN;s#atC@^@dF-JZiLqL4BR#> zV_T+MV(?~5>RzV?h0-n@GlRv>JqaMJBL=Y-L#WztF8EFprWGz$bi7cSNd1Y!e?AnN zTv~8ywgM>-T0^AEPebEPQ&|6B90Z+?!Qm&;jPghoP85tMzn{ub8z8u$psKFcP#o5d ztI?PrtKsF$U)H|j3*h@CMG~;A7%vGvf_CRfx_FHyz3+1vx1ZF)_hJL=)NU1epyECR z-~SBLBJ}AKlP>JGzYWz-65)&MTdb-cVYRNMpr7VfINKmbCw;wufp-49vL!n7L;5vn&yil0swUyKGwdaZEe?7F_0=)7ZC&Y;D_ZuHyZNDgP$W1xx`yn3L9DfAjjtXBYUKYywz za}9RBc#R{K_wnX4MG!8UP37a`NtcHhS-$Q!E|y*Ztv~0HFH&lB|5Hu+e$)q5ldhn_ z?sg_m+L$~}7scfoq0}fAVfOwW@V(K79R_XeYb{Y4bk>P}((A;}pZ3AZ?$vaYs|8JF zIp5f)e*9Us4lSaiAn3j=%EdRL-MKd~;t)zFM^%8*)?A3Yt3#^4$3gG{PpW=R2Hh7< zWb8Lh!Yzk1iOtId0MOo+)yN&HoZoMCu>#8D&RU}M%Glu+|N0>h zK=TGq5->WE9JIG0l}~cuUyLyQZ0$tq=c-aEF+v&|SCeB7P;7g}r7iGmrnG_cN#Ib-b2 z&1;Xre_(v-!7v{^h|(>CkR~to$N_6o9^%C9{Vp+z z9J^<$#cWb!5`j&V>$vaT6$M0UA8R`nRtTLqhtZ*4(6%F}S6 zw;lX()WXRZKH=*n9NVng4c4mN{=XlJn9+CE|8BkKGMZL6XPyDI5fdZvrux)4WhL%4 z+Cpq!E~Al~-l5&+8~n)e%e<+X%H+Z6OYCrOK2&Uy!@d{hV0-Zn>y&H)J8>cL+hT!5 z2c_s62abUf8A$GD%%D0y3c#T!5bmhQQ$hI;OjJTJM7&xLJ{#4cw|6-m+G|Ku4u+Ad znHyQFcXz>ZycsW;{DhbX8id~4kMV~JU{0-o7^yXb`kWU6N zj(6duQv%Jji-kuLG{(b$$aUG$1qmV~!JW%wTkFuc2cne8_=y_!F4QNn7_WVa=iL&I zB*%(7q4nV))SL~2f=5NnHUU+7c$9PNyfL9m|C&H}zXd(xn2F^QlOU|=4r(iyki3Vo*DjUL^J;<}p0g?> zjiyVhl$h`y5jwHnfm}YmmCmwU&E@j;P$TY{H6u-mevMVn z&nLAKs ztvglEok@4@RUt7Z5AjFC6Vz0%!@xt?P_=Li`*YD^;w#b0T)jUXkL+s#`$Q}3dtAf@ z*qvf7d`rgbXR_&WAyFnF5< zf{{4qC_V)1-))B4t~SP6rI>y1kwr^?MKhLXWKo^lalCR6BQ@-FnjC2df>*9Giw33X zacd=$BfI@c9L};0qd~9UVo=I%NGc76dhcg=&9$4k z?8*%I@1_Z<@3drpD=47H^*|`*7&v986o~BFVZ2>-3KuLRG@y^$yWE>c zF1*zPm(gZMFy5GawS0{p_m?w&6CKl@^ETta}$bvmJsE=Z!xYT7B+9&$90w6 z!TII|Y*{Nr|C(vig{SmL=zmGXc#Q(u3B5tFTa%&K>OJaro6*0&w1~Wnz4h{AYINR( zN?hk2jdX$rdU^23)eVCv#hQ^|pL~4x$PLfD+=xeZ-oW*|t7vy;Cp|3VNEMZL<0tQM zG|Te_J-Zy3FB1zQ;-2*D>mOJj_6WRC* zJ9-ojFPMhM{O`dj(`T^RZ8cg~H^Z?WZ94s)ACp-TB?)%x~aIoNf^8R<*0<{c`TcJ3lA6AQtxwzP^U9U2MM@$C!RX+ZsUv?8G6E)q1$UO^KO)QP{00ZWbgNL zST5dy2UbVnTc=H!x1|w_4{NgHH*ILvuski9dY{euA&3*xYN11X6iiAUUOTbzGX$9W z5}hgXaH#AsciYt>X=2j2LunFGjFqFua(l4)p&-qv=*EorRaDA3m_4(R`yVOpf_1hMIFGaTzE_|YSFXUVT18lyM97rJHOzmrD^b~&bEE81A?BIZH09VFTJrV~Ju@Y`Eixhd}0rn3$lHa4=Q%8T*-iUvkp#|F;t9Kr3bT<6t%4dw(n zGP6&7M630xfbT{y>nFGWoH384SG~dqXA?p1+ct9SZyn~$%&q+sd>SQsjfn}7g9lw) zS9YZk>0Zh4J+A6v^;0X7zG@cway#nvHWGN^`!M))@o|#936^O1v$vcjh~fN5(lmJl zY?M5#RZ1B6I;ohc-uf5jMq1;5*cg1ak)W@&+VINoS~hDyiBt!fl19_NP#5_P&pTve zk8c*5ig)v`=E-CGmpG_i{{au#{pIdevuNq9IGnOhoEW|fhlelD;sW&(%&ph*jD8*; zPeL^m<>`|hkzwTP*D>D9S7KD@uo7Kw{ReNU6{F@cVY;wk1O)$8@&u;6~ z${Xq1gf+&-)U)Ca;}c)Yc-=h1NE~cN0SQl-%j0HKof{aTFU4hkj^oT*!L(mYj!u|& z9#i7t@z7g$CjUhiyKId)9p|#8>o1+e==U*nL`8yHal43;wR_2`v5Tk{+0A;HiBJa@ zZ+as88e`KK#=^=9_JnON#+;y7m^lXmvTwlaCEwUbU$$9GKVsQ;l2UZ*;bPpnDUxj3 zoysb^uI2q(6h}W_Q=_{MJcnu0uC!+)j*gFAf!aAD)WYEIbiy@*eW+3-DvuH*k>|W6$1PLRLRw=sgofa(+R0UB^%(=Khxj8^Hm8 zlxOpH{YYS!d=4b%FZ#kR_`weMXX6~_OFZ7JEPCg;Gcn<2qbKj;NcvY<{P8#tdon*l z-r0|+_i_%$OXIc%<(#vlRSTliIIbd%1;y<15Yqn@Ps~HstY3#%?>mn@t)(z2d{GWW%1T^j-tJZ z7`^#Ph&=K50-IALN$&TL@Y&UsoVwcpJT7w)x_%##m2bg2`;h4kyUXz#6zMO~UyvNS ziPn9yVvpu25HqJlxE@vn*{3I>MT!)i6Dh*ofU4L{dK`1*RtmiFr7&MolRn9v%tnpI z!(-t#kpCVJJWESlKX?mma>Yrft~-eJbwZ?8E3A@U%_vVwqWhmL!||16AX0e**G-r} z7Us**x2h>5Nj?qqa3+(JRSz>b{-*J<=_FU|5YuAhMmu~CLxawEos~fc#OzIFp9$*H zwp|sVJUWG}oMJ(v)u+<#zNgS|+zrlusKIuHbSkm+H7<_dO+Jr)1^%JcRLkN7JDr^j zW4SA+OoKR^bB^QqZ>+<$EeYT)-Hs((FY7$lr7Ej`2!WFyvO)`Q!$yfjv>umdWpwoE zEKgJVpNuZ8$TJ}2_7{NMSU^wjnUD2x4E0pbhMpZpxGjGVeq8PYjW3eeHHW156IM#$ ztyB4opWtPTlc{IV)c$2gRbwIPZV%otY2y5Q24u2LI-M{#6v|Vxndg3mpfo{~&zJcJ zbCktNB+Mrrn?v}vOS>5hMKwlkVi+x6yo0sQm`fJh^s{H^J?5p*B|MzF1pCsh*@{KO z?B!~X!*gZ=4T|-p3uN_CQ`m*B`~94~C<}DzveVWdR~!Y^7beWhnGDQdd>UUq%BGhe zWYU()LAb+t9KBawsS9$Q#jz8b_`5bV;+h?0P@|W}Ugh}h%L4W?Dze|$nKMSXjKC2l z*Ot#r7}KGm3v}q;o&sE1AOm*O3Yco?NO;)5IWmuxGX2?EV7QqNx>_n^hto3hAwq=s z)=Q#$D`ijGkAq3fY4{$#oH%$X(C2I``*iC@_&ZI2B%c_<)7~>Ma9R;`)9uimew=x{ z+yqw_*2Bx4&rsl#IeF#GG5N~wGb@|xAoR+94D6F4x77Zj&FXL5UD6zWR&4|6?s&#Y zAO)14Pb2ZqCX>MuJA8UP3~mqRLG8V-C?_dTPfq&>`R*&3LPswm^mQ7oFF*vjB_!A5Z5?xJ-v_y3wnY`oFe+IDu=rhroeka zXY&4iA|z$>GIlZr%nYMbcq)tAd5xvvk`x6Z?K6p)A;`yrq8wx9fhKsI9)}|~k8$n! z)0o6xh~!fW9d!K#G9Nm5N<03+Sw@6Z487(UB8m{&PO$s4FxDD;WIB~4h}DK4@H}@m zIn{TO$yAy`TcaX)cNVlT{skx5^`Dl|ujR8)BR7f8Q?Y@!oJaM~tz2|FvIuqFsFG-7 zSC}29Lmu4rB9|m);IQO5bZ(tZ{RG>Y4)PuIx7@~`l49(wNCP^t(u~+tpWz8e*RWI8 zy=R(Yor%NVgZS7&jqLd=K?2pU;NRp527kzr)Ph);|BIlfgd^;ZRAy$qG^9ftxeRE{ zTCDB;gGcTx!nCqv9PJ^j+T3P%*Wg6V)i_S^3LoN-w~N|Fe8ud9d0@R$j5g2ONN=bq z5C@F}I=QX~zRH-AT)jE;UFtn_+WFp!62KTx>g-BTi ztmCWG?s9JuAE3eX9GFWRB5vYR&iO6ZBaBB1)F99&igzM)m?>C6I7Xm6*1QmAw}ywK zBT1r1SMagL;3^Ys1@r|B!r~N9G87|49JYrO*&zq~c~*v*dtRI9iN9dtOYfm??`%@L zK@LqX%5pvrA6Rnb5_3J`5UMjh7+LENg}vwDS<7d}`-3sr?P7(uH_l=IwfHe2MLbeB zn8~>#CeV#a%JkZX3uxsa!812dr+Jw%&~vi{-2WwD5yzmhF<41{K9^<;LOQ^~X$jd- z*@oT1N5N@8g&cc@9K%(ZWai3}hyE4#QCyXnT@)d$b3}2}S~!EQHO|JJ&N5(WI*~Xm^r9o`EzolL z3f|WiWG-wTM>(VW80@i@%kwMJnTH+8pADY)>Fsp-L39giVs;sS#P`9Pz(QO!-;Eg0 z<$P~r^HFA_3;TAzG2MFVA+wMvfc36?_Hg<)=GpF02yE5GCH5hB>1Yr5RUL1+${aHL|sz7z*P4Vd7 zSMax2k$f4^Clg!_vf|4*R?vfRYQM&p^vfDhBW{15X)2Frm&_s>CJX86eeu>u3zC?c zo<|`2a3gN_48m2_;k!o0wgAXALm<u z8+O~fodi7Bq%Ch>qKu;e?~8W;6|>t+#gb;@1xF{kNPdu+Wh+EgB!4rx*8*ryfi_if zDFErJ*BHofEYsNvR65n2e3*EZrOU1}yh?R))8G>Je3=UOR~XYR+8pco;5#_*KA4ss z(}rJ@7DBV_UA!6O%^H3T!5ec5;PVejNKeRuzmpH*I6sa|d|?E8Unjw_z6uO(e9Gw8 zl)<6P3yIURWa_%F53emzq4MvqqJT{hd(nI+?BezxGJZ|4T&Dxi&$dG%Re+X)rX(S3 z5!sVtK{{6dfJKFEpmSB5Ua8+s$8r?N`EagxJ@5`t_9Yx#F_YX3e`}pM8VX|^3o2@C zF`f$_XQQGP)83kR)LxMYmY=lAQ_CitXeLR=9~MAQ?muSFoG9MQP06IiHl1nfT8yfb zJjo3%^YAH2h=z|(qD*ryo^(thmsF=f{--$7E8opBoC9E$^-^fycAv!?gK4dQI5O#X zd6BQS!>>wJG#~zkHY?nLH-80b_T^#SdO5l*Jd@52dc(NsZX~@ARN+Wh8EPpmC;3X9 z;I8rwE%mc-X`u~PFss=T{spM^h`{r=JmJbgUmEz|1hVqF$u&z=K@vS5;7ac`G?=GN zV*T8SYdDbiqv}L^&kmegtVt(4d<=TG#~`th#S7QC9?^_E*0)WX4sf4m!j6ZSXDCf| z=dC7N|4Sxr#dqQ6^%Hn|of*~skq4?vy&(9|JFE-Z2>dnr^ySl1z%Me;o7n?!ne&W} z7w#NB4K2ey z)P?laqa;$mGXbTvpZwA@%gE>IU!Y;yfU<&oxKU+DefHEs$)qFrH*qoqW4=NP!<~}9r`1C#OR5B#y3)*3MYd-#!Jqdl) zqSir6a-dE0Ge5LWns(inBto-{NcK=VM$h50Q#~3~esdrk+ti7c9OhXc@^E8xCVp|< z#ulDp$c2U_aAQRTZOUkRP8|Zd>96psy)d+f3P5*?AYbX97{0Gd!!7WW>DlE* zS5El?&xaEsmX_e7_V=)Bnkx}5T!@hl`>5WmcC@{-ms&a&vy(an;F@q3C@RE!*ZY6E@zNM4UQGYDL}(bBy)h%s0o4 zblcx0;P~u6#&PpfA|uMe)R*FP_dQn<_azDCCS(wj^ctVFL{q(LVWv310^WS!k?mW~ zqhED;-O#iK%zWI0;4q7=5c~$>GtV(cf0_}sRiQLPjC0VBzGrkx7Q@_X9TKunlVfDK zQgfln=4j5v^IC@Ph1`6mQ-?NS*VUn5*R#}q=3M#9QTanzwq zh^!ekMSE^f*JE!<6Zc+bYu>ptjxmpM!d^u-v*9s*F0X=T%U6-ay&53!a2mdEMBJRl zW9wd*;|{kHrozRT@`^jLSHg^Z&z?!l$wESWL}<`H7F8-t(Bek|d7-r9s0y8v@QfV_vw}r|hcROQ4&0m{P24^?(h{+?MtLToYrqiL*5qJNjtW%> zNM)CvF{A1MufS0DiuLi2JR+bc2TKfJTALiqAhVN=*(ujH)5XYv#G+Hi*jKRv?v9$^5ATU|%wcIPY2}eETLuzaC#h zv@$kg(A#2Z(7Jya29(_# z-$I$blK{GFb_h*Uc)+*Muz<&0zUzAGPmI_q$-HP?M-HoSyvKe8a2kt^xiO3*>6bgwbQ6- zKj%El-N}wLJ_2+7cvkdk7CzaK#2+|0#%##QWfqEhFiEn~5aLk-UtT4Ufc9N98U3SIQplW7u%q2j}3oIBGVV?T&c`{!w}3NPc4Wz%rZxC4HB^c_qi zbMSMpFS(Z~LhI9J<0JtK;wXIur0*Q$?>My;2g`oIo3{+)hH{=&w4l}DquABt#9UG} zrY0NC;L!9uIO#DN=M5g>*BJYwN6U6#h3CK}c_Y}Wx|CyY8^W#3bnAbE8)%bc4u+}N zk)Bfzu=`Lju1Jl?(rzwSzOWn)+MdAg!k2OBmWeP)LWT?oS~71=36idJ*WiMeH4HD_ zM$)q+Nk<9SLmm+)9bD%A`fWW(D0_{3Wp}brPlu!@_>;+2`$7C*15ZG29?>~Ji8=<( zgssvuVcoh^=7>#uUC6qvWdGa$Gd)DWa{~{p;VLu>=;3i`j#&~cPg;YW!FedHws`U~ zI(T$3wcezT8=stp_P^oexK1HX6%{7)go@yKtu?FuVhio&G9n5|4b0!9>0H;M8DnZ@ zLY~HM+L2>TKE}7RJ|kRC=I&w4xi*Ky_soOhrzas}Zx_3{LXMl?Rd9p#atLkFBOlB3 zNT0tPaj&l7x7{Vg?1cyN4#|MrbFN>^+c2`(kbKhAKy-@O zgd6eDl$?QTnTyC57-!>q=ED2$)_CF@kIHsb;z5-$e7$`GiFoS;bz$$A{rxNP`jul$ zqJaWEH=axuuldDv&66S}DrI=IN{z^m-Nq+pH<7#19OF`N1AGqAW~I}$$^A)xc`sEH z*s*(ccrJB_HJrGNY&YZ4qmNBUV|WP5aYCU#ND6~owTbhf7hSol3uHK!`pGI5z7{Ew zW*h;=IFv5EqeX)aIPOu~CHUSskBqKqh4#6pn3-GS@OSDYx_xvysHf-R>DxCzslFV) zXK7*ZlLOG(CqRd%y0C(g-8iMuj%fOZQnw#f_pw%;=1B5GF%?e; zD^Xiz9q`(*2lrhZ#eE8)C(SN-BkQYpy&QUx)CkSR9 z+YNzD_T>7aN%T>QGqXwVBgbwqBQmPFY}V*;mTXGF{q9ZdlBY+pQdyU17)+(o^S^WZ z^-%V-c+!k`<}x zsYl1XNi_d_6x$nWOC#2O$7k*YX)2tvQJq%@ttt;x9Y%ya99ewhkFc z{EPvIkFyK5Ex|7n1;~QrZ@i{C$?$Td1XdNwVPYTR?HSpywJ(J}5@7I^2GEmxr&4+J zhv5g}ConQnBxik&lM z2|p+j^c9nE%2}>67AZy67D&Uv(%H;}3-6iNx*a&D=pyQtPN&Z^y-B#T2@|Y$3{H;8 zQcd$oWJj|(X)fG_irk)I!n11_-RlN=H9Q)sXh>oObh-VS3EAc@L&qkIut{BiAw#eq z8{$%-`L`(~#wX*7=nZ7j)J;$`N0q#7_=Qeh6`=mLh;iDXN0;v4LB*r3Xcse=OkHJw z{#oy#Q1~R9oe@o?WD0R*S2p=mElK`yx9@u&{xI6gaWsTuz3sVXM*E_b*tZW#AwE%x z`k2gSy-(_qyOKKGT`w8e2U(KaS@NW~>?8aqa1s|4sMFW^O`irqe$9s??wAFnngae+2RKO@+IK?z{l~UMN+NpeLl}(i#<2x{fFlCoabwDkV&B zRH;+($cbca%X|njp2%8o9+`5zZ*{@JOG)|xE%eQuMsuZKkA2Z^bqH6+&hh$=GP#v(g1CHk3#6cB^;Pj3lhUa@Njb?IX`t64%G>gf~VJc zvH!W^hjIb3&uRdc?+t~>J=UnFbQ|7P7GRiaAFPavVU@q%Wj_t@i2al|`1OMW%@GNt z-;U?7N|g!B7YA{ATVIEcUP`h)d{2_JY*K(HDSrIkYhh&9nMknCJPit`IaiJIDhv;t z$y3o$hgr#gVeffo-jj55R-xuUyqU?uq6bTv^;c)XWM__PQm~2s*qsIvQ-aa&+y>Sp zp%D8V9>SqJ0z|8a!NlqJK`0U7P4OSbB=`Fm6Da+|-bSk($Tg^temw^1@C~{8bB5rOYY3jF@fB+h#q z@!Y&rVrA5Yai{mv)b9YNmQJJt{ZDYP$`AU4g;}rO$zT$k$(rt;%(2AJ1Mh1Htx9L; zx|UcJHB6xE0=JUFQ{3HP;WhpYHUdRGd~j*M4ox3Xr;!eceBBu(Fn#$8{*>Z2XyvEk zO8as&>Ix+zdA_)(Bo41Vd5bM~JD~fr5-z;1N1HA=fc*MVw*5ji>}tOaTZ*jcnj>yB zV2LcvG|D6&xXy)ItSpUuDoFoJ&7%Fcy*bC^SKQsEON;emnfUX=aGBYS6<&niLFlvTjQJl^mOW`hM1S3cfZTM-n)AWZ;w-xL zodvtQJwO|-u-5E2+#8c4TRnqFTuv0J+Q@w;9_it-^=+(E?p=&t7)L8Nw)0P4^CL%+ z_2{bH3}`sKpH3*%qL*x5z$bkRa$PqZJ047c&?vzZp<#r3l+XvuV;5cSf`x2LsNUo13 zLBE@;P)$2u`htH6B0B@9z}9QbK=VcLE!M%vIjw9bm-SqtydL2h;Xhz!um_g)Ph^iznF9~5Pv#4sI?Xm7s)9-R#zeJ5 zl8q1TgcA|&tn8_|Y}Q}|Y zIj7TR>?@c7OFaZYXv7Q0dXjK0*Tvo|=s}k=!MHi`11qujFSML{k3R!MS)sqfaNf4a zx~fK){w%Vky{r9Tp5hvteXs0)haS7LAg=7%inaUhYbwtPRRFIm$BkSt@>1`uz5Z%U*WaT4p7-x|^4z29! zAHSf@;x*j5?LlH@RYUC8T3Axg^?FCHLiG2~c%teWYCbH1xD0m^rJMs#O~vWxHf^SL z<^`N@qC*{zxZ~s3Zj9g;X`VBeHL&~F#KiZ1!@p_bIQln~&|))wr;& z5vkPZ%^cR?@fsMm6CyXaM__1<21Y*~W_+BLVaxmS+5+o1OlgoMg_BQXsfjtASF)R* zx_T}CjTnNT&v;}PP)Cjfm6xKyUjB^j1+I7hdmrCZdkC{Dn^0BH7W7W$ zGh*-kv7~Q^jrngD-R?cg7%2%LUq_K%AFPG`DW_4_q5^8ak6=L+mn-4k7wSKUQ8!y- z>`C5CZd;|m} zS#;C-gX#*rBI`!BYZ}o|nf+w$^|z>eSd6i;UP>-McH)#^TEyM37`}!0k(gT+uqSIV z*=gF1=UTs`_QA98&N&|6!sVt^%!>x7f2Gga|rY&@`=CWRk%}aM^bN zSM3v`eI$ekWTn7n{#kr^^dRc)O2)-|JaJXZ791(sh;OZq!1~@4NGp|Qb_n>Aiq4~$ zcd&!qzRnktMOc_V*oLu>r{KZ*V*EYq2>mNduyjlrpM9G~jgDGVmu4RAJJ&Xgox_8G(}bP+l4y@&WrDqCY>zVKf?-g+GVsV%@x_hst4B#atRbU!BNUd82QL z8Z)%h4b!E$`)x}!C^Qy>O+!9dFZLo47L!O~>{`O=P347k1!Lv(e^5}q5!M%a(cSHF zbaT*8{BCQ?PY+ZeRy!Bdlw42h$?azHM?|b84sqRed3AD@R$!Uy2T=thi z=1Ef#G>5!rOC&w=Xqvqt(}DEDtNJ=v9y zi9Zd9@yjp3^Et`H%$Fzs+MYr%$9UCkisxzH{RDkM`(R>40H`N9&^w2mF+MSod3bj| z{0!mTY-vx}Wn3@8?QI+}&+;LLc_-lTHUol}4`Jr~bf_N_qzB*Zz>=G1@IlBf`nF$% ze?hSfa6Y$_()`ARyw|4vMG{~x>_w_=R?@Z^=Wyrq61ZZS|yOui<1)4)ygPW=(boUjE z%~2FAR z5^PuxQ;!17W*`j&2_0sAO;3IDD0ldpn|%RABK&o@{QX9!~JayYiVAUX3b zi3C5sf>1t#c4}tQ(>G+G@ca-uwJ)tRUptqyzFWoac_vJ&Zym>L&+X~I4r$zeI)i#; zW`T`uH`Xt_2~jp*nLX`^@M$Q5`E6JQVN{gL&2c4KT%Mw&kz<9Z?;_^y2O%I@ocw1a zO-^o^z%+aBBZgdOW*?W8`>8NVA2BRe^g2opUz(7-V7 z9*B}vKaAN0woM>YmxnjFo!W1=7f8*L!RWmUf6#0Zk0bDAKIl<+DKGNilQkc{m$=yF0RkT zd7g9M@AvEFhaFkf;P6z1yWcr5V)st7PsHzms_tP>iVCHVl>KeJChsx(-(D~vH?@`etdcOHDm?k-r_|3wo$Jk;C3zFDG|tI z+ic7mZGz=j7GrLa3%xnN8ow^oAlfqjz+%~RJR7D=?0AlF-DDxD&J;iKXggPx6^1=B# zFT-0)1Ny+Cl5aHh0bX?<28DM{m=)c?G;FjX4q975Z8!l8)MnA9eJ7!=qnYvErbi04 zo}4_^oz2vP zQ1t}(ebIuu18}|TdSQmsK;z2esb)rCkURLk z=Mc`jy9uSvx#1JB0%G!eHEbRiriAN-ItV0Et~o(-o3BDukRx64zMkzJokPYy3DfkT zad@=Ef!KR@Fe}`iqJCgH{CccNh2-UlPYqk;YDnSm2?LrJ982G}sKRqUMUG!_ z0{_+=gt(a(!BtnAY8GuK4{98!w%Q|9S$`5mR=ffEAZ2($9w zJI2NCIj-n#V0%nCM>uy+jpsab?_?ZcrI#2iRR4oN7f)x5N?eKZ+YZbe%Vs+7?cyKn z2*y@dIhsB7Ff=^Z=2$99Ah_i^$TPaMQEL`1^&Z8ror882+!IrFP<6+dP?4nL+l5qbSi=8kMGIY)D8()o`}$Hp3n-nj^Cx>d=? zU=}J@aBKv-__K__sJywG+l<&wCL%N$&)%`K?BWoVMeu z1P2n-mjXW*>yp^s2TVwE9Bq#M#W)N)LXqoJG}~N_v3_B2Y19ghrfg@nIelQYYk%Up zkXo3;WuhEr^+40DZvnk~UId*7 zIA^>R=jNuG=yZ`~wZ=GZ?gurRwdyN4{HF>IRxD25qCx!%y1^t(2+J3CKubCB59!=R zN6Tg8YQ;T>?9ikd^{dfJU=%#kMTq*1cUXBrmws-l1n0`H_~QFr-iVY5ybWJYqFvR= zv2z}@?(9FNV!=W(VZ%h)WI2ZX2~sqDe-zDA35Lqi38ebtMYxgABS{5m>`q%bGFd2s zPPM7Vjgku>^u8&TEgxi_83aLZToKbX@)?wzU9slhPI~C-1oFFbJ(r7>CjoIRUv%UM znp&@;zwfb3|2IqY_&3Td*+Nw|$U$^-F`O+h zCW6TpWJ^ONF8=FIOl}C%z?qwvsuzn;aKx5exwH=*3mu5n{de$oNCC`DZbH`eSrDFl zo0-zK9W;ASVRzYYcpW^Ky8BGUb>+r%=bm8DNZUwy^>^ZknE;ugvxvDKvzRC}>(PCy zl9|0)XE8~o1D|Otk~r5CCdbf>1pWKQUYe*!i^PK9&V?p;Ul~B-Ql}G*JHDjf$Qr%{ zxX}%Ti}`QWgo()61QdFtMm)Rc(n?WlvSdgVm(BAcWyd(5nK5bF4?0`CP>HEmP;O>I$p~iGwT?!m8@5Mp?i}?IV0kQen#ap@7jocklV~f_F z#~-G%sn8V}rcu0siTB;lK0LjSRqV-vy@RWmAx%4)SZ9Py;~BW<(+K+iFeh=H7s2-H zFZA1{Naxym(~(EpVR~f{y*(~MuW&za*xZgjVKS+oM;gvL(+JHglc`qKB&xe`G42^W z24Q0x*yt%4xXz%2G4?CP)!#&kd)+noyIY4H;l9T=K8g-n1)^Q&B(nciBst{j1XU3) zU_j+3Y)$*X3Ub^Hi#9|1qg%}=4XuS&(v^59*q=%&IN|yF3*c0D4zfobLHc_tIa}NT zX~m1k!r)S-+{>FBHm}4&t^4eXd9kD~b|KT7yiwmo9ORyyP^oCNg)2BedB#}nGLzEi&0ZNP*dC;ju(>&*r-`;2WJ}D(= z=%|iivl@9%g=@jd&57u!854fQA}aJppM6&Nf#pxpg#HP+?4j`|81crCyPsV}spZ<3 zn`en-BI49lmOB%2CCMyFDYmUwoa8JGWM9b2Q{f&JNLccaJtmxkdj){4*B79far0?G zEDzY#d2nO&C)eR~B$exHc(YD#r&|wBqtT96@rdU}=AOqQTv%Ypru!DLa^4F`N5X7k z(#+ifT_53mp%!iyAYS!v-22PNk1CL}5pqJ9Jzx!WelNoM$i03ff#X*(ZKs-1!(pY#SY@-IFsQ+nG;{_wjJ6=yVu+XODX~u7KC8$MLD} z3UoWr!Z}~2!g?5H1C-`7>nbDpy~y+bo3c~>#lt=_V-RvZ(}tBY0JbB1-9JeeMz zCP|l7t59Q&1?0e>7lk9Z;HCppd{!$bnmL;W@CBu zh{1iPVIf1yi4}QyLWp@>q(C~%df2vW`7pLlkhRHf<7XHNliL#ec_!CiVY~7he#VVh zSgW`n>6xpnmyiU>o19H1DRR%1lu6{>zN`3ZOAzx-%^Ox9KL%4*oM7`B3h3=F&igLk z26_Me;kT|RWu=1cVBsCkg}O2phi*65m=%w}-1h$>v}EZ}-TtC^yeRg9&fC~Y%PBYW8I(A)on8R^`^ z8m9KZV&6izzHtm!d}~7ihRf&fDTFD-1I(&@#q0r1YvNfwk8+*~ZWrZ4Oz&7D?XJP5 z1X)_i^#Oi(bub;5Hc`PV)Rz(X$TwINcFGx;aDq={IK3Z%bnt>r@A@FMXM7Lt;zIF=3Kbld>$;o zi&{L8A)*n+=tAr|b-s=cSD|*<~#X3xNWGd|NQYWj^ z|AAP=2pCF+qqM*Uru#z}&QZC@HruGv+mqMRl`85?w^TBI72Cmje^JI+o1fs0>prZc z@k5;DWk>c^58|7&KJ5Hi2*+zeajm5g6B~;u{d8jFX|h?iuf1 z5#2%JWIwRKA55acoO`)_T{?{S|Kc1jh1}C* zHLYI!9d>TWMD~#e`O=(D%)b5Qoi?)M^1nN9=vX=~@?Jxp+HmgDEmb%!-pcGxX#~A( zD_}x&3v5Vez(-4J8O|XIix*kZ=Vg?6c0ZqE-DiS>;TV%+QOWb0y`4z><`@;Bli7hh zKxIZ9P3H-M;`l#Mve1WNtK;nTsghuNa5p^RXHvuQZ|MC-g^UX-(ESn}V05#A?Je8O z@wa5D^yxvk!tltP(^jZik;)`F9>5Dtei(X6lO8+DZf;$7C%|(L3TMtnle}n5xnxgG{z;INBZ=hGUsp1<>LIgy*+tNkvBk5K zpJJeoDE)JH79s0)pvMa(sN?bgxlVUMb&(Ls^9-U2FK0rCm=%2({DBqWoXCe=&Y*f| zD;uIUnVCEN34HE6#=crCNUPM|GfQ5qWv1OwftV)+Y+>XGq*Wcj1DAHN>Rk6;WAQb1 zvXdgH$6MfV=N$61yM;AvJIc$mSi!kWm04fZz*FmViF>L8=)T=TWw^dTdu{{vw?ATU zN%n)#tpl(sgxmLU`NpR)nJ`j5!qzU|N(!2$(A3sKd!0N9S`g|^#c(?@2Gio8R?c`um^i3r@)#rd+p_gBaG;ddp;m5h5(J3NB_sw3$4YCD_>7{{Z= zoxIQG5opw_MR(qGC%a42Q0K=)y3)1+ucS-jz)4qX_)mfMXMKace=MnRaUdqeMq)}x zKCC&pk-oj)OUfnl6c0<6Sn=s|WI@VA5H2Wag zmWZueN^^3Dz{IK_W*!n^zN8D1n4G2LsJb^1Fv|op(Ot}9$5o)WtRLb|Mss_nKOC<> zga)a{;`z(oFqUkLtDUC6gpp8?IW&#_$@QsIJy1N#G&X6aAzSGH+2easwYh7&HYb}zuMU?nnd)=c9HnuruGEsATgk?Xk zvC6G79Iw5bd+zvhyOJ2(v%UeV?wgT!e;c95X)W{cN(r8qH728drbr*Cl3z<}F#YRv zVpeJnxANm5XF!%rjn9M@za^B~r^9Z00yF=7XIgKZ>kUUX|of|%Z z7++d~FMkTs*8$q(WO)G0Kl%&>ceI1uyhR{VJb^!!G{U@Z52fAmQrOyW1L;=cwBoKT ztvyo*=BDq#Db#@)xaCs2)AyKKu>yP%-HVqT>sc4dxm#{l!T$2=kQ4YCH+zWEV-n|a z)}~g}=FdQ>WN-Ru0ZWkwK@dK-r$Ayo6$Et{m4oR5lP27U4`+>5F=^J<*`OW^+8Q&!_IStOnorg=($j}KH@N<| z-3-(dP@qc#7r^pPOWMbtM5{a=)oF-kV@N2dciGUFi-`b+I?#esnWt_s79=u3r{=KaJd*=#NJ%r_i)0M|>#v z5>0*ylTP!0c+=FLw`3@g_#{2TZ*P-PFzF_4+ib@E%Cut7_2om$`aO)K`g|zTtc4#d zl;|9m+cT!=kf`Yc7s@@T=}&*# z?!fGi-!XT#DqYI2WF%R85cJO?=S2jm$+JjCd4?-J6CKQE_07bKCUtOp?iy6S??#XK zed8(gedZP0&LswZqSW<47w+3z%67z@!-1s*#QT6d{r&3~XgM@9zYM1EyDo=(M_T7QlU`E4_&P!KH+FK17qFVw@Wwht5%Y0@pGm=?Y5`8WZ7w8jdo=Hf{#J zvLgz-wg{4lC;^%+E=c>7&ZufqO?N#_PKxSxbQn?8%+!*k*Sa*O=$xi+N+X zS34bKB~#%0(tq$`Z7|nQmZlX^lW;IQ4_^gOBp(82va_^2h_#pj^CWyX+)ni&&ks)} zo9D?hf4(folUe_PJ=2AHcKujd5=?Jd&1K$PUq^lJ$-(;iHH0k>C1JDH;qzS?kbU(R z)Sb{G8Qecq__aWcuJa~R)w*=4N)*f~e$VFnFUL0z#$f9<1tJ*u94hLm{k~0qF?QN9 zfQ!cV?^E>1*E73Ohs{U)D{1$7&kSy+=0Sao9I2gR4=c2$mi_!{7WHs}O zqh0Y%PU{?%~@7vxxZ-Luwln##T=Okoh!$ zS(=f;J7dpzcpJHyl1v!ntdNHBn=`1;t~}UedJx(*vvGXyHn`S$m8Umx7U|N9z~|c- zFe`JS%ZFsqH18BfWJQvWT^^8|F_lTmlB80zccQ9)1#WpJO-_kSAOqj#(D7kD{E~`< zGAWL;mgCykFI9sX5-m;I{d>lhpa5tWFJSq0`0YG827>eXKdcf z+?rIvah04Q>isEjYwlqMmHzQBh3^8>t(tU~hyz+J-w#6ST6EH?e0XlG207pMgVUct z{B~4{XmG5FX{I`~(yxXE{Z{6izYtBForp_orlXWoCH{B27QPipkgpC527k(7RvSOY zIJf`sqTvFV|KCQq;**F%8f#%r`(@zII>ugh^daxB&PM5UWPk7_QGCBIshACq)~4(NpReY&zuI(v-$--x;?^t>OK$0NCb@)OTz85 zZh`8i+hDq|9EKQU)ZdXxcbt$Wyxeziucrc={xe~uLk~fi$t!-%!7*%W*+${t9;nWq zg)X!iu9YeiHT!gYmZ8tAyrc*w|2g6DAB*u_;T*apRg(n#ZDUn?KG|>5)Le?|DcCC4<|kR*<+J8V42|#4CXjf>)q8c zRx}5%=AFgnjJfDECQOzm_+ib2B)nav%&wT$%f9^inU(sogoN!`M-FG)W1_vdIZ~V+ zjT*3{_Z5$_S(ZQHW9DmIXswS)UrsVl8eCw#$|S-oG$#{%SJ4;UGicK*3u?V=*xp1x zovyv6f@j?8(b%_=J@jKX`RJ`dn(x1YjlWV+V-|M?ZRYa9N7G1*DjyX5H?iZFbYP}L z8-%4Tplxl|q$qJVE&95OdSChij_Y!}X{(=cjwm zhq?+S`buMJP84*IX*^(O^9KU|+4*lbvOZEfnBKK4o_}OUCGl}guKgmGY#L^ERVl!#gjVPtbECII*KvHE6nJzq7WaKmfb@+XM8fZR9XL}9h zjnCrf?w_nlupnJMC6>NwSx>L3^1DiR*S%+*^k*s+Jr<#(Xa z_FZtpa4k3AP$ZKr^GH*rB{{Uan2q`8Ldr_Jz&s&~CpO!b>h_Lf%kD0A@xx9W3C?AH zhVDRzU+dtg9ceq8>lwww;rQ{f22^8*&a)Go0}L;8N(` zq>3fsi|`ZM%cxlflM3^>?8Bc1n)JrrRjiHj5ypC-EPch@0WBmx!`Z=FxYMObRF81I z!dR~JF58ag1CsO$zYBlQ@x_#qG@@CZh2LtE$i)-0iC6wGigP~IGO6X{WBo*OW=aG- zqT@_f6z+%oVoQ`W-_L9RvlK!vR^lBaUHFx70k;;J;?7l$oO5Xh$X!z;4Qa!m`6rj_ zg2<43i5&FnTu$#DEC!w2kL=RG_h=pb4PVCIL}#bV?AP7_+~rupScD6L@Y^>2ymQ>{ zF5Zj$lQ;}tn(HxrF~`&?mBTTYldx_=CF}bA6m!?~JGgeQg;(D4RAhB99hj1X9&PDl z*y|G*TXTGiz}=AA+rntG$Dp8U1#>9QpMS*P0G8HI!0*joSd(r_ov)km-Z(9wjOH$k zyC6W`TmM79fk*h`MJVA+xv<(rm{)&$1@S2KXYs#wRN%6i9||HtDc_W6HhAMrVNa_3 zK^PtjJFxb;CUEn_7{1#lL(M+?WOn^nj=W?0p(*wjtW%pu)E8>dGP4e7zbZp!HI{?t za~tgZV?}J1=@Fp>A=+Yd5gtYk;Elb$4HlIjQDAN;-5M7FRjWT@``3dwBh?S)d#xsW zaSfb#@ev<%-ef+!SxAR3y+?cA893;a#<8D&qsQhBeB)h&T?3~;4z^&>K|#9I!i`Mp zT}AuGbK%=^VdAvKneG#`g$1{r$>NSy4B)u)scYI$IA=4Jy{1Mo_f2M2uT5nZKHFq( zbKx1kQsM}dP1=haj7#yLu^#$V%wlNv7-&SQ;~Kh;Ij~8R+&lS-jf;K9H88ds==Fc!;MJ~ZIPMUNo<|QuT*Ive6B&JiFu)RH&#vI5685c)bH8Pz}vfYVmnWSL6pKDE=`>YBlku%pGCc z1)Ry>+=(>s??STRjw(4~znq@Am`)9pOTgz{YUAYSEyR5L1~_%x3&ov|R(YYNKTsVtuCQo_QYx+Rr zW_{D6OULDj4S5P8wHib@OP`7yO~InjLAGvt0$SZ1h78-HnbQsu(2O_*M>Q8x`?PBC zIkB1gjZJ3-y3^^O@BZwT3VG^~G0aSE`T*6OYwy6za5~s1M*|hAm`2+@tjw$c@_Okl zyfp7FsPFVAkzB@9{@E_nEkxY3MxM^!s}0iM9x;~`77~$7{qUk^J1IJP0%L=2GQJxQ zGr5)NDE#pnb39=OIc1_lmrb|f{}3L4Ki|5bOL!$1wsG0lQ!?~pggYjUWz!=bEK7sR`|+BwHt7F$WlwSU zzDX)#bh-Hq!f^S3jU~z)6W5WQd2$l{&uJsRtP>;URsVP^y-caq6$)n7k)#4ONo}b= z`dFl4##}MFA$K~BDPYa+1KsXxG}_#iC@7ZUl;yiml=jbseCGI zYplduDk5O!^qdu|%)|{=Ze;wW13t18WF5TIICnFl2QHbziy~vl_!LXE#LZ~>%Y{t1 zehviQNWg^=5p>A48Cy6%fkUhT&+NSv-S;&Ro+Z2D{QaD}8cgV5CwC{{_Oy2C0VMkC zUFd2VW4^5yB;EJde zRR5_uu~tC5w5J-JCuq>AwR|+ca}N|dB>4OwOPbbGg$J(NkOFH}R!5_dtm<|p%TCr| zG7%!vJh=>4WfexdEaK-1UV`qzxkONXHv9X-Yv#!PU}%)zN&@;-P}@%#LMOLi1#5!~ zx=*7;Rui0)T8d)b*KqB+sif=yGWM(0*}&m>B)L+H6zrY>k%rmS>b@%3*OY+lpN-7# z=u5cgLK@y4O@WK?nI!Bwch;zDk!&3?%IhzN4Gk$wu7egMbkL0kbc9pIXeILFMlxB~ zU5AgYo#?*dm3*JNc6OVH4bgtvi2dtt!;wqg^w^IzMAtO{Q#}hAqnUc_MZGyt_8^`y zwAIInS&iJjb1B=hgAW}Gji`&zQ=DXUn@zB&hJtyM=@(fbwPrH7B7TH@7kdg-w=W~N ztm|Q)S`V(aT?K2LbBUO&FmYJ*89TEdLU*(cz0ci`BtK8Zz$jZdt?UU0i$+nvB9fUt zEfv>$wcyx{P4rBZA3o#%f`2Y9IAL`adJ0xzqwiJb*b{M__OlEXPjWk3Je1dvGRZv0ZTLddbYowXb2iCD-d)hD?;58ksbk+e-C2$XQx7`hD$6XHsz{xCbLsfTCDiUfE%YYF;@!B9=h$1rSq97*Jz;-ujVM_-Nru+S2GTwg0jf4%3?=^Y z$mG`hjE_Y&QA>`e(Ho3u!FzG?eO#8-$2YRZ1tN5!UMXY%;B z3a;694zffWIcMiQv~rz9(;r9Dw;gK4!&;B-4qJ(JK6{w2&3~ANoa6YP!A^|qnnVu$ ztYs|guV7WsR}i=Uf(K6|ft!6OsW-636ANFmf0L$=C+j@vkKV(LbE}hCp%ETkD7AvD zG>#|wzZP(_E;Tw`CkJ!AJD~X8adz2-Im|(h3-2D#)UbJ3D)sI6L7!6#xz1q$WW1@w zp}vpo;ZOx~@%>iHIKG6p7aE{nbUU$*ddWZB^p5@Da}&IKgV-Z#p^*1b2r^x`9F5sJ zJh3jF?w)ZJm(L8Qu3jxeD1oHm0~hxew%qL+GineDb_% zJx{@*2@}NEQ1#7|Kp=b~jSER3D)T#-+CB9+_4_R*`;Z0BS?R+(l9>kgPvx{c}=0jW2bIZM4ruT3si2>xVViAB)T zwU;1nIXtI+wD7EFo6TdY4o_HQ+?H-4z=GCzNga_4F zn9sH>YQ{6EX=KfvapvAGSNkvD_+Xf`g`QJfO~(@kn0b?)aqfuMa3<>?Yq{bh(_*p& zIvS=yV1*>Hk#B@KRG+wCaKIgB@`y#SK8Z`-$xmplLBlmQ=zBpBy@q1QlT)LtS3)n_ z-N|uOe+hxH9Oup8-fjN-q{kQ=&V;MZ1w>yupU&T0j`2sw@Mmp5j7nUDAIh)!wwJ2m z>Yh&+Ges26HwKWuo8yUZx-nh8W);_k-G&WOZ7`T)PR^-4VUHc&%L;EdfT6W!#O~xr zh!F!6U(*ER+hs_Xw<2}Sbz|ma_HwRA9a1I8^A5dVEGlDYm%BaZmH7m}8)eCl zq7CGjgC_i*r9w1If-pYtFn{@x4BqUORu~ed3(2ZS*lmAb<9K}{S*r1vzj?MTI2OKO zHz`iZFFr6;LTVqn$ed{j%6j&6(iH=#lG{n zP+W*;My6xVpgL7G7oY=W#W2D707hMugZsBfVV=oZX1QYuXeva4+AlriS2#gU*r$fQ zT{^UU@DC3B$-+aS)+FL}3&hS7CgT=7Na?)+X&is5ylNDlw#MLwo>V+GAcWo{Ul==A zZ5%zm7oESF@wDH{vq$8;*(YlXiCDfEUQujkY`MMJ)~O?mli60{^8Ga{F}9bb-oK$* zuBP#UX+8e^!(~g3+n_|2B-y=n2RYkTNV9!hiHUIyDo82O^p1(xm_LU0YsHDms`ZR@ z=_l5zbt$=0>O*Fv?1BZ>=h?Bin~Wv<4e-)dv(xa_mQ2U zlSQKr7GUFyHlFDIId*4`1j6F%Q5-fYLajD+@<^SbQ+mwVc^5MIjl1kf;>i+7jQs|^ zH|^m`>jan<_?sURn2TBASt$2x0b~r8uxaDNSk#z;vtMg4<#S%K{{>Eg`SKK#7k8q3 z(h@52=^}fD>v7Fnz~E;tA1p91i5BrysnBIr;=}R#|0K;u=M6miSojwR)MP=LkU4uu za1$y2;7_wuBFHDDC^U?e1@AMIbN{6gm6ws^;pYrCd?Xr@aU+>lrA6MwKLr_SJ`wS(HG>&aRL_rNQMGTu+L07&aUa8r>%TyY`&>FNVl7m(g_4iPliBm9?!f1T z8N^cRKPdgA4*hMnq}6Xl_TG10X{z^PTWLe&~m{}^t~%b>niSm zz^zdjQSm0~m2a>`5N5L_wfsNB{#wx2z_|?l8MpF{0PL~4wIr^N5 z`K?Uz6GCa*mq<7(okv^VSyEUhMbv-XN5Ai8{Nz4Kx%Vj=407SR%O@EV z`K{c{a`OxjQ;+6Vcf3KjOgXaOb0X&i)+4!z^N8)>BG{Xyjnz|K>5B0-cyuC)8Jov> z#Vt9X`6NT?d$^IWDrZb0bZ;@AHY!k`e(mrW4qkqoyQw3tfEQPk;CBvrpY;XrpXwi-%Mjmrg0 zldA+{d@%->hrNgIhqs_gstrjhuYSQ0@zt>nMBtzmHmYoZ5$>Jhu&X%P{YabqT6vSTTKp7w^EaU;^Bz2&?!%pC``ALC z*ZeMrrSMsWK=*Az{#`F)6zv`8RN2jZ(U~%s9}>($VKNrUxZrH@OKj@uVg6rlY5ep~ zmAcq-jDb`;I_rG|lkM+|)5mXc+;%&%0 z+!k(o$dQ7l+u>)^K2|th6}LMY&^CSq@lu~=KX3d8Dgu8iw$$kVT%xGcXy z0^ej<1Ei!bL#}ZMRvaxPYoc2iqqIO0uzNMH_u@?YdWt!#WLE{=9?E1)S%Un1s0=*W zdYJ2T77LfHrr`pf*mBE<>*I3It$SSF%=i&3bykrN@znB@~U5xI;GG0Kl0A09K zj@lj&XsmnCjLZJJ!8l)iz}oJeM~3`F*hMEFK+gp!GC#i%tyJwu+JaD0nx9RzR^NqC zE_V}n*PgWGIzd9Z9JtR8pzC{=Fxxn1y-|=iKYxw^zhG`1RyEEg?IKlh>7z6~H?WCg zfJAYtS5=ZeTbq7ev>Yeeh%xt>dhGjd0pEr~=%@Fobh*t+(*18I5zRV)Hp(Tq?#OKoUaHy4#x~z&o$dzF zN^ak;sO&^@V?SZ*%Q_zav>|Z9BQzdZOdBSRpaJc~Q|?mCJk{gO@C{pHJ3kjzrdVKo zw<;AF$wN7}&CHkiZlr8sF}5F4XI$;_@z=Ig^k+nf=@}KeLE4$_;<6MA>Q=)3=~j%W z4+|Tgu0Y*s9@NBfF3D`zhYzn9)B85_!Ou{Es(Gw~OGgX{Z?Yg+_wy$2&TCKZ{4T*| z9hn&PJeK^I$mPc_++w#jR`5hyFS5ja8vgkA3>G-1kP!JNVA=W?Q-T%fn`hB*B&!+B zmkx2f*10rl@hv=XWfa^rwc(Y)F1UR!5(=VMFcpX20_m7cPxfyo@+m2J^Y(q#bWy=ElfFfjg8uEK~~Mv zr;bjlbdFvM`MBaHJd%A1>tFn24~ZJ$yZJQ5fN*|zVY81aNRDy_>Jcq126Iw0# z6rL>F%=1nW!kwQ@(YB8Mma zPVrL2g0f*0`TPKP+vh-N=2Yr;@*udGWK#aSpYSdB2uKGlrdK;T2jVVaKI#r9s zmtV8#pvfN4O-{sK$2$;Z5KV$?w`0Pw0p7G3eGnCW5*{yb!{f24G+9oDgtTYUnb9Md z5ZQ(|H!86g7u;pXy)x;I17#c=g|N4ZZt-KbuYq9$O}IZ`OQx(=Aj@a0=3Jce#OIbf zYR~%%;SI~_^pZ0emo$N<9`I&->l@i;GKnNcq>?=CQ{fkSN8gagd3x_s)I_m*{$Hs!CKuErn!nML@+4B7c5 zgp8LwfwiuAOj0?=atd@{>odOcEs}4-y-i`Te#22P-_-zx#jWTqHJ22~3laQy5&|t# z>78o};i=LQ=5%8*?{sJj`pA(zwVD(Tt7PeL_sW88J8hIPJ3`7KV ziISQgSWG{~ez~WP-WBtS^vMp+moy)*C!R*Ph}&p++z1q-D;c9fEi8DbKtj8Y%zX7! z2HRJg(u2qQKz8&3t17abmevRmp}E#1YK;s19{YifF_57dnm{J({mQ<%V@tFq=)+C8N9}c#WncdY8Pfvdxmzug+oj~+FT?g-^L2>rX`?1x7#oO zV1>3>Zp{8XQ{q?BfT?!*oTu6k|E`IkyH*8*X88tiI6ax%&YeldJafzC^}bX$GyNK0hVuh}wq*+P!SSnfp2^AY$rUx^gtePEJh z&tTE~In2O|A+YOrBt5ZSL}y?!`*)f-*;;!NZ+ke?PiKr#>h)vZ;AWuv?;7DsC2KIw zD2FS-u5j+70QqJlKq6;^lNBmgQRLxd(px{7Ru0c$TU^ew6Z1XkP--!1-uMCYeh6XP zhCjG-!Z>bilAt&GJm`gG*HO8A5tU|L$;!Y$D)=Cc%lKU4UoQE8?D%6gLDYaPPFzfW zItkMak1}`y2b5{wzFf4oJ&Z|Rp14jR6})%dY7`Q6Aljvt!2djv1|R6gb3bQ;RAvIK zEq3&{Si!75smIPt7)C|0Q}|}{Gf4G`rJMD~nTt75bl--_)aNSeteM@-z>{(Q1rwT63FKLcx8c%UsHPE|b# zJbXBWo!dM3%fzp--&Um&o1#p3YQK&ayv%}{Wue%ARGlWZDA7IBckwnJP@*C8xPFuT z96YgqAuBDMYELLL^2 zkl{c@dRD=cyld>@RlaTc=Ptp<3K8&f+EefAbd(90sz*WwS7G;| z1jdbRgQmhj>Z+zlH`-o;zQmu6UJ6&>&|zE7DXoK+s>bxlpb-&_o`wf9j`N`2aso z_T=^1o&uft{o_4io@cDy~L0XE8KX*o7wN8q!PG(=g7ep3T>oL`-}Y zQC!HD#y9Q9_t|2!uyG3gd|H(*+S7-J7Ty77<9l|(l|;sJnFkRV;(Q5-)94I-HOyA~ zkN4&3c8;g+KtuhrITnKqZH>B#GZ&}f(f&M8FL{P{xnAF%Gn`vvt}!wiZ{Yc+Oum3+ z4D-H#!K-0snY}`M*zWU@{qQxP%Gp0=liymAE8YQAb(bADzB$V7aB-(MXB-Du=0enE zZ9yPN6BYSuP~e;+27fdqeKwsqG(8*+&-#mtR6QU+ayIQRILJgl=Q^X0cY>~68I!na z6{yt?u%^z&aCogR*>9E(rmlqUP)#7`s&hzpuPRM!D8`t`hof4413@AX`Eb?Os- zsr+O*tm#fhBZQbXsz=f$JV0eoqFn(2AnpKO!d1>sHh}~C!-2!-lqRibl!njy>A#N zB9SdSWY44&@3|j)HDpAgk`ig4Bt<2A%P6TNQX(TF%6sl7v^0p)(2%03J$#$$cYc5R z_jTUqJqCx_O^f%jq_sIKrG zt=vukh;^ZT(q1Sm6^BcgD2gu3CVM{zK|q`##-IPlYRF#Yt9;xCb8>7+Mriy*&S7b83E8oU|={58A&ZXr<v4B6&b?26A07(=CV&&En1?U5AM@e(>8?@$lrY$r6C`p zlAl4{#p5v7*Ow*+?7#!+d8}0LI&$5?6~9MUvBn=al0|K0Y`bJMt$CJ6#e9+(u@9?h z%q|`Y%eui_yfGKnKdFJRyJ5tBdmS8mco#LNs?x^TN6?ZrgqIl)lFHUV-@O0O`9cTV zdi)9;*{wz^yI$eb51Z+A)dpCR+T7MZG=v__V>rNNH`A6`AX%V7cAWZwu{))(F-M%L zRm~*d^2f2nsh2%ExeVq0EJBErp@L7Bk-JmZQPP`C(NB#!v@6lStG~kx&WU_M^DtKC zH1T5AXF=8*jt#=`E*6)~;kAz|frvv6Tr5*3>vaF3x~2lTqJ9lN-z~%|kbvgB2OvRe zE%~@KnFR2@vy-$rU)Q84sIS=p)y7Amb#EUI&6`f=dy3-8hr_I|sR*6DD+vE4ts?Ig zp5hT;KR&{d7#qrXl|b&bTK>h!F&vj9 zfLL@(klPQ3SZ9r5C?7Pz>S1~MUHd)8ehZ`{wL8d)r}NOozZ)O@tp|evjxVjW8>z}d zHoVP;9JA|&eILBgrs67SoNfXAukYaj*T(g|I@$+`Z zz4s6Hl*?lxx91Khx{8Xj)u?n)fCx&;lXxQyQaRxgqsB2l+H2-ouP-f#gG{D_te!fXcI8;_KQkZ2rO}bc?zHdpoBUoF`{NE`J8dQZd78cp#>ayy@2|M{$jW_ z5pAz0WUp)*aT=e3$?6is+dTwd{5#LI>&$@gpBq3Z!iK)EYQ-{*GUnuv3)t_u1rygO z(q@S{IEg7v4qQGIg64qn~gNEKa$-v*_!m5)nmTu7aWOGATQJT zATg9cC*&N1JK+sH>G;jKQqB!M_UMw6*HVe4(N^e~kwV@~?PEVaya^rW-U3sd#|sfk zgqn%Zt(@gOX=-*HYa1Da~*2%};@nnbh}| z1sMwpN6X(0xIJ$!J%059=x)iwh}Vk5vYXI!v(;pEL;)6@-^~8&Im9Z8 zcd#WNOW-Z^Lf#Z$_ZHsahs`*ND<;*zDi>86IT3*|JP6CfZt@R_XVaohy?8kI8#0$w zu+G88o{u5Ui6L$`y!kJ0-$dC^j%O)mKqfX_0F(1+#CEsQ*}*Uw zp3|b0L|r$WTwY;IFa6-Mi})D*YIAAToXI5M#X(?O1gO$f6}bJyf+NF=5aoR;Y}=oY zAS+%Ef~yV4fW9|*x51jcd~=Gw`%^C?T%L|Kx)$X2%rN%b(KG6Vzt)NFJ)%dvx)(5xcEU7v;Y1Si>ltGTskF#Wjh)z?0tO$0X?IW;gv>Ak zt?>qC=FV(JdEy4gUZyRRfm?iWwJiJmE@16GZjCRh3AtC@RZFV^xg0g zMG9uqY2!VRA(01Ug)~IZPsg`|xp3Xdo?hRU$lM*QhR~fN*u$M4F`Mg|&9mR2jFmne z5UgfGOlOiUqh3VP&X+ztodEBaSTSnpS3u({L+Tbg(gElF{LelonRA2XDB!pbIy+{f zaKUjnxNI#Wk`fBSYyjdXV(`Kd6&i4I1ZV4Pf~-AH@M@wb**m0%{U0;vxjP|b z=~4>_cDlwm)>Ol32TR&m7svPuC6IF%NWDOtj=C6HqG%)B-L+iFxfa74bGJ@V@7w`oHG+?lD9HX zFSH6(X4sKo`v;J~f54aWv?ekeYgoyJRiIl~&v8KausSZ1M6dn=S{;+*oa-id?qn+U z+G3AW&n;zFhouq_NxoSFIUyE@WTwcnq5=)K2fS8QnRn+>4mFrSLK>(Drl zUiKH*lTfaU(C*{S`shy}$`&GIRQ?8pAKeI#>k8m&LMi+>>djb``_k1i{W!l)h_~tj z=cGJ*hd*5Q8)m9(1?BMuEbX3%=907MCm&0)B>fv5sA7tfIqDXVY zO4`hdfYFCByg%+tlg2}6Fe6POMGwI3UC!3tKc|rV=zXBDa6Ro!xq@a_bJ_RCwT#*h zJF@me9jqUkLL2_*Q&pks?2$c>K<-=_uBeoN<1RP(kB%;a|-c6_X3-h7`)R|Uz?Sq_5a&t**-GOY!F zwr_*Jxm6&Ss7yU?w&BkS2F$S>OS&NN8S{h7?I+(U2AK_iv9CXt$WCj8!Q(SYZnZV} zbxMLw4ne!I!nP-Ga3pSR#_ z^hEkT(43|eE@u0joM?XEF^C+!f$j%VsIKk~x@P1Ae&#Y@3nT={*v0GUkgyN8r9h8Ww+(@Y;J`dGt;;GRg1?c@M+1g~I9couVcu_z5^ zmC1#Jaj5mE3WF_6;b!qXdfGReeh_@ap6&>sDl$Gi52J7_S!GTN!N^`@uvT1jONpQ0QnJ}EsJexRxa;`d% z{fl$x_ROW~&;4k5VFsqmxd4lYlc>SIX-PVAH4vAFnziT*c zUo^TlPXgWG43c%8_gy&X%4@K=SwLRXE#!i{-37wfc$pQTf zXz#X){5(5|E@58u;68V{EBzfvQ(JOZxf2tr5^<(pDM*;dz_UMh@ulK&5-p*O>5C{M z&^HKM?ep=0^(QQMUq{O~8qg_oCPH(h2W^;`iTOuQvwEQ|ypbph4}&Jb&5Ueve7-c8 ztk_DcZ<&#wVge+gR)cO&b=A zeYarAsyMpko)%60I*gs|A=J5UA>=P~A_w0n;J25-5O9F&>HJN=C9ihikm+u+!FU{0 z9AxQ=!L2CQ8bJDd)8LJ~4B3^j4I~V#X=j23{VdJ#(#~z+dq1hb2*F0C>)LMW!Oc~s zNF7F132p{4-v`SQLa;P`3kfO-hifK%tZivE4)iX-^k08*@4myJYbZwC*J+T0Y!{>b zRe?4+^zwDL9O8@rDgv2Bsmz0nFp&Omh3Deri;Z;*ksJ%4^BP<5LVh%(^YjJlyt)+U zh62PcpGAhRTxH&^T0?%GFTt~ix5BErQW*Zej6C9Pr+-G=soaHQ?9RQypnI>bb=c1i z)wNW}{>u4GbQ<;z^qS@fUtjZL?dG* zgldR_v0oL-=&m9I+ryZ^sulcIrONbV<1F$p{Q&!YI?J=vmZsv$wp4xXAmnaCXi#dmPLmT+EUl)Lx*I!_~LSSvS1v77!Dy_e92;Z+UrK>)l0;R_s@2rfw z!-WUo=eq)A&nx7cOp<1|ON+viE7gq8(om`;_a1%*)j{?@Y20y8l%`pJcb)u&*V-)Fd2WyeO@ZlVT1KeN5xBcb=g5ZZpsVh4Sb zVDRr1%&^J8e^&l9tkIj8iUE6CO@O33-b4d4b)sT+3GK^L;fP5)URfB+A1PFT#v6pS zX)wSq@{?&`nGm^f`ysA)<^Ts9j^Sg~oA5bMj#@75ocxI4Kmuk#bv_ zP;!F(*EE5D!dHp1|4K0;JQb%hmL&VE2*-^MBeoBoz`;dQP&YmZy6&bpqpus@%w*9% zSct|xUC7_IsRdrllgG1Z-|&xsDX#K)$`CO!4!%#nbcYRHu z8(ULgDD5;X{Ur%M0}bevW%W=#@h|S)6;196tC7j;ikP}NiR8>t1#a}Vg`9hqN-p`e zV|Pm+Ya*Bj|7ID{uO9-i>Vz@V3bL zowX;H{#rDRR!lp@2xhp-VMyyMlt#Z4>=(}%YUMLGN`UMdfX(mcVuBFg5it429$25*<`2+jobZB5<2`KH) zVK!7BfCJ_$V40f{%?a^CiBFoeZ|irMw{j)7!*rybA{~rr#zUCVl}5DF`D}2PFL=C- zqK%xt!s)gt`Ix+b7c1m}%}L@=zF&v_o->&)o>YKIt#VLObP)b>tgF}!N1=C{2+}$B(~$4?n+c0|HRI{pF`Jh?kLB>Jmej{h928=>H60Q z{MY4h?GM5K0<_6Czb+gfwkHe5m%y%(SvVec2P4b^=)Wy9$nhsfVaPLzrVsSM!VCv` z;NEe3dwr07C%G5owq;SNgMT1?-5|CWA`~n&qbXPFAo6=8O^FmGHv{xZys9Y<7g`b% zF?~jDyco~*?E(4lM{vseGy6HH5d%11+|lTXq@uxu6fe)Gm)2?0aN#`c?X{(LY68?P zWG>i=bz@abDIT_%2W=t0P_g_7V{0Z&_S8OQrC8v6NNZ_-mwt(x2Z%Fqw z!)Prxk6rYE*_-_if2EwnK5qALJ==&bJSa+NVHy=Ve;ht99m9{k_4xAAPjvBKO6Dj` zCKDIA(&bxTzHXfiGFNqd)mkvwLyL^B+(>S&o>R zSHSG+Jd&hVgyL%PGM`t%7B%p`2qU@I!9i;o zypcHwj&{?*hdWcgm`c;Ti~r!~*LOLt?HA13u#)|x*$eGOV$k+D8=Z&RVegsUs3XOl zb0d3bujC}kzbOsw9Q(pgQIdM8RI^d6BTf74LW{+kIj@i`QQlDwJ(UTpPL35GTs8s& zlYFrC#Ziu{zKkmW5Fup}CgjY$Fx0aVz`H^$+T2p4zaIPJnj@zKpexa*olTCQwh=dC&c z!7h%dx^p+QM3@rhx;BmSTZx9(S{WM@AnLn1z}U4N_Z?Rv_FPwEfa&{>viUtAmNWQ6KYi9``Km z31E&4KSJGQ+eqB$cbHU>%#IrG04*?rV|^R(`wCZJS8@4#mH(hX|1dC*yg=kcDAtMv zvLSQ(*<4vy@`5i)&9@km!H(I?aqhkKsdzj3bIjJ2H$7nU>KcfYvY~%ZAbUS34;Q#P zll8`$#79zq#@(2Uf8V!a(~?jko;tu-PS}H;M?$Fg^8mJTe;g4C6(jnLI2N|r^UAMg zQKwDIP{j2D^K|2O+9H+4yl=>-!?z0gF)N(N;Bh^?Z*(5MoYJ7`5{5*&Cy(9ubprc4 zOptnriPQNaIaENf81<*>Qmq4L5xw6qZv6uExwZtbA#Eteaj3642C#<$oS3D{@CB$0bBH*Y(`_A6IlftE4ICf1;4p(GqkzXz8VKH)m4i-1mcqggjM;HLAd zp)>9#>&@@v9iK9XCn-I~9{76}kIN|F;ix=h=EY-m(-c&=ZbL7>m`dIy-i9h;Va{VA zPQ>a@qwJZlaJX|P1my(5;+Yz>^6(_8Sd|KXfeJMH^$7^IK8|Z4mHnP@hW9e&qi`}qh(6*K!IBmYGAw=ua+l7c zjLB|tmt%Vfa6I)_FE(NN)3Ua}Pb)~z++^mP*(2B`_8+e%!JBOQT>)dy)2Z`I6SSDU z0ctb(Xke|54K>|NQD+dC+|x$yL}wN2Sr9-SpyjhqA@g) zoZo#NygRDdN|kV^^}7ul7Q8qv&H z&e_jCu3QW6p3eoB@4jGO;g3RW3LfJ)o$I89nUP=DSm{s^#^>n-vNpns6q|%ndsRd7 za9#(7SH;7>43_^O;wN(UL%MB#Cj94c3tSE_!sv*%%$v=&^e#6eY6>m6lM zcSC`88i^Cle{s~+@ebz*4WUI*e5{Z6M(O8w@apkY-tvlGBv;hfI2?M}DsTt!oV2EdMoO7us;85n3v$7wHANzoGSJ)trii^_UycrU^8V{}31E zFQ=XrOL5hw9NKbIj!5<-QASmcCdTsUSZFT2w>p8@InA7gItY;56S{Ed!wGoA|IFUH z)Cikj+mYEO$?WfSIcRe19B5Q*hhJrV&?{2JMxAM8p2_Ajw&QZ_9Cid;0^-P`b;_)& zyD~lOphQCj=0Kc!8$?e$#yc=GhTcCnkNp}khzajcFsC1J&r+Q}Q8ARJZ;cM&wvs~5 zHS`w#Q6C-+6ab6aKG+zXOfIVmlJ1cpq9LsTyB3^cM`Uk9R{-buW3Q#)7^xs7Oi zpF*FsN6!#FIVcF4HMDBGMC}quGp3OOIlj6(1Gf zM1Y}O5q5iVKGN1C`f8mX$9}iM=Q%dS=+P(`^s(r;!IOmlTZ$tR9?V#5ALNdjqN~<0 zPUmK)ue;0UEX0f%*hbLH_m4?Ur(kBcX3Y229Bxfb)Q#% z-vZX>>M%Byi^#(dTM^!Lu)i(zal-~ru2W-72HH$%!^{Y@TjImWSzObiPt~n+iM-N62vyM{Sq~;sT?Ie1t=)t1FS5w!)rWX&^I0HQey}@d zaZZLYH9l8qWA=1!A&+XlGTAz_sq5Zdyb$Y$@M3QhgwOD&zW%_{sM^nnza_QHb5ueRe0@E)*pQiR~}O)+9z zWWjbnc*sgCcH(lm|3L5E7myHo#AOXQuBZEM*yWYPM(K~DQ*RU$aD9ZMGIh-JJUu$E zN06{NskrfD4s4b;r=Kt8VzqQGw#@U#rn+2M+?E84f?%jhWuIWJtIYbhaqKwt5%Zoa9DkuMLEPwn#ju(95(uFbBI@GkS8dHJuw2 zi@^@?^s{R^!kFyxNcpE#bN__~92kShI}Q3`wEdNI04-jH9pEDm0J3+9p3D zOtuUJ5chy-^ydk2V(4N58zovf*2ZF5J2K4JmjuGm4eRK?<{;ADZ$`@c*Tc@>Ag1N2 zH{$m#Ae6Y7&#PKP>UU-muQ}>u?(BJZXO}I`{b5G--0tS@;3r^*Q3mr!TNST!+1Q@v z&gig5sEv2$6FmD82~Tf)VwL9h@jeB#ENFp!=C`2Xw-+4PN@4mO zTROF4AyIv)M@JuN;NrkPQ1VBM4p2M#Z-pP>-w)${lkPzD><}tcyB{m{OzFn7$NXo1 z4}Fi)u{RBmbT^v1B^^pEyvdy z1XT~9?;d9||Mj{9UHgGKBm03-x++JcKB;jI^p&vrvN{e)TQIBTjL78f42UVa4C{?F zaGI$Y4ZAl1#w$9pr*D7>_?SiZEV>5WqQI_`kD||CaV#&TdXTHXi|cG3;Q13*(7wcz zHo0q4(_fX`|7;3T-z6oP&Af> zd$w!PU5V4l%`gdiJpV2ldkn+s<^>RQJg?2@?;RBN)qz`@4|6p3`f>sXYlei|gvn;`TdoIUKXg{xmw!@BB|T*YMQJ8km=L4&G1e0oRm7 z(xq_{zfCp6c+W%NeYqW%WO!58OnLHq?E<_jDoERY?}xw3BEd6H2YKmlv4_iZmv>ac z^!`GaTp@;mgMKLbyBMD<#(xSbjQ+O8zRMG0r8aiGxomN~GpkLN~!4r3$!`_T~Xfzg} z77Kbn=z|zJx$h^(6!N8hE6u5dSOVGmb1{DYZcpP53q$SeMm+Fn0%=+_8B+vHu|ltt zk^B|~QVMERs8ltIwQKcTAVw^}PbdkjV&^2obq@im#<(aQ3vzcx+iGuYc=KdfL$u7VTFD zk-&>=#|lS6AB{2ZJGoBXk4~QSc~d&DN1pTwPNjvv^YFk&hFqRKmAtCrSaaVyU{1wm z{>142P-O2}cyq54<9MO8%Xombc3()pNBjo)`AYP?WExBw_<_~O3B8nX2~T9HL-cJf z!`G9B`$j3F%(=8ZZAY2RuPdkz)6SnPHV0PNE0a^V1ni7jm` zRi~wM+R;J4ghmuv5XblwVk|KZCqDm&4(%c^YknXKq+e%e-HQZ~s8>*T=`F_&NC$kJ zNF^M8f$%Q;>}2(Zyz zczMD`sGK$n$=(B8$PRqwF93-e%cX-v+=5*Dzhw$LC z4*et`2Y17gXhgR(sh7Nj1+6Ch3DxaPj{X!n*_R=~>(kg9)dg(Nbry9wFH7l@#V}l` zL8hPOb~$<(xQJ=T8vk5Ib7=gd4VQNuu-)z=q;qaOjaMJ#RRP z2v;kU=p(=QYbEZpDtE-r$>rU|-M2jWhNiXb^@_uAcl&IVTFzyu6y3d3X}(muf>hT}9fa zsLtH@n~9#Q))E6%8&YbOOsA~efCm2g9QP=NH@wY-PMO$f{pj}xcK1`R$FkXlKJmEC z@8q~VhUT6$I7^kji4~_Gxt$$PR)*A!NVfUwJwR8b&yeXXL^6Zlf@EDYyqo<8T6n{( z)+2oeo=U;09z8m8;y!qBtb^K!RF=klM(-XAqI&Q!u3R|>#x{=d|90MC{`&+tG*-aa zsrgc6gHe8Sl@~d?N`Z#!l)xEFYEGdRJ0@Y(;e%-T@CZ}0 zcQfk09YA}B^)xkLKMwzU1qzMPkoamkZ5r>yymd*~w{i`Q@Xvr6Sy{a9ydUfGbx}cc z6MkByPaXPX=sPZV|7p-2Z_Q1Ikb(E8(*Dl+=J6ghdqp7cVGn<$?R~HlT>-g_6I2^C zV0u<{XY`megs)Ct1erC# z#CD1xkv$_sPP=*%qZ;VTYJK4?MGnlAvFJMie z8!4YlV1lJ7$%#%ve-#aKyv>3PTiKGr&Pi0kGZ{w&Z?gKQN8!E5W_I$Sf zA?9s$+u|%aB4eEhvja?uhyASVq!e<*&H%ekR>7O447pz>NmoU7v7Za4 z(5UG)WNWz_n<#pmm(+X*Z=D~&|8_9MMLLLH8j?Vh;&x`oE)UQMl_VR;YFhJ9RSJcoWfBC1X46+xPCBNlczhS9hQ>9l&)bN0XFCN{>& zpPJ2V1ey0WQ0CSSj%k@pzUEpw(&PdaHzv?lX;a!KR{`1Wj=bXBdbE91hQ$iaXjLyn zC1+}p$kV=bTER_F3)_VkG#uHnbVF)gPv~wB8KQ1AodyaoBH^8j@$Lg#TI}mV1q-z4 z7x|+w+EmNNJyIu1-1~^jL@iw2^M^SXwv)}8{{aK1B1(2--~-~wQh2$`lPEkp2lH3o4&GK)nnF>bX?}N|KJU$)|6FWwkqf z)W_iZ^WjwcjW5;o+=lInsr1E}OYGHG{jjNa3iGw5gzu0v3MVuy$bTD^V77)ad-9<+ z$x1uOTpVKP7;^z;CNE@8HSNTf|DLzyG{3A0i10)pM2Vu|K@So!J=M74FH zag+ldOZ~;@AIb&0mqRe;jw}<~SHo%@8OPuAs`y4fMxgb!2J6sj%mj%#kdEoYaO>d# zkh3$Txlso(R#1k_=WCEi89}&Z8b^J4IN!j15uV0NaWb-$(~Av6p!xmvgtOT>sXF?EW#tSc>+ov^PnBZ`Vcv$h;dQ3(Dy?zJTDd|8&B;du7dLH&yNeJk3l`u zr*>oa^dh#RESsr#Z$M`6T>%Rws?+z$8kkzaW%RlI^v}5u@VnAl7<{Hgm;RfJoAs74 zY{pg^X8Yzc?)!BEN^@%1y!*oR&GtI%%3~qbDixOAUQVZ* zx)bfk%i!0RXN>h@j;nji90V>8foZlW&9;7ovBCMgMVW^5oBdq+xYU&tK}ny56sq+iW!UHeOnlfkE^zBD9^{m ztrO^hMjjcy(F96P=d7QHe88orI2ZN~FT8%|B|9m^9G94{XCre~z~6B(ta~FyDuqOG zry%Ej^m_#*3Op=g+{=m$-3Dn&4KB8DCHCW}3hJ%i5OHp zQ=K`BD9XNpYngv>cux?WEZ|BF_76b!!9&d653VHP>~{K3O%j#Ld-*er^H8V8h~8ie z!D%p)`@Ou3Tjn=$S-@acGR+ervIALBrS0tY5Lx`|)QAc{L}|`~n{ZyI04;mg0DpEd zoXFaBF6xU7Mtzl~Qrf!oky98x zc2bWzTW;w}GC8)*9@)*HtLco#>dvFO@_(TCE|@aSmNc&ZB{=Kn^6qhdD%t1eq$!ch zd(E9iwqE}S8E@_BqFwvYa`_~BLi;ltyrzxa_xcaKFH@z)0m_75+RH>I2IF&+ejU^=S72_u+fDoTTF}{Ct|+T3g??Qu zh(T2CfvNM{E9}tn3=DYtskkti|c|NR9#X?I&EnAWi2erBRE1YaE$*5&bqiLzmcS zHpyR;j48|~IXo@gFXaatmaeANrxeI)vpLWdHwzPIsKb8q8a95r3i;Swi0nFhGQZ0;aXt#lh=Z^w}w(xO)-(wE`ueQl1=wF(R$D!FE6ga z4<`BWy-JNnsN2Kudd>;<_6Yy&G(}v#T>}fBsX4|(*Ct` zD4qQh7fZ4{`AADj%3aCm^gA#&jdN%{odf;BD@p6z9{5)zOEzoGAv#*2II=$y7{xv3 zy3N#hB9E`JMly~3(V5=7{AKyXJ!mCtS^6LR6%!`;p(X71!5EZ#y@iG@Ph^v{&tkf@ zG)+q{0w1Lp%p?gBq9q?qZF}#dOMe1JPqL!5LDO+pJa-q|Qi!->7MpisF0VsmFVy@Q>e_z0`w4bfUyJF zc-(Uu9c=G~XxB29|Gf!ztBVjtFHijSNC>??4@3NbHuGNf1=gw_O*+PY|GpG~$xsH0!-&|OrAAq9M{6Ob{JMP`!P6wo$G2$S1cPCHb%ZDrQdTKnZ z-FT6o^d}YkHAV6JA476ostkM27z5ud6AHDzumf=m$iTe`STT7VE^S&$z4nBl_2E+Z zu;vO*$vwh$NuI>lPVzJ&mj+@u^v#*^W!$NB%tTw1+XPvWUkENc%H#`HL z)iOnL_=OQk{Q3>mT-Nc+P3JQw{yKw$`b;ckXVEV)O?Wbob7H%CVEBQlFtElBr*65< zu6DGbKScFOhRtN&g?S$t%Z}f8NI{*}6#LURiHg*ya044RB28Y6-DiKzTmZB^h&3EC zA=Wdvz0N>_wUfUd)+kMZk6Zskd-gkT<%<@M+rEqJKl_p?kdUC7<=f%A&TU*)+=s_L ze1@5;Z{hFP6_DjP5u(EfKvsS=vt(NhL=1H?mR90)qF5I!*;5bNj3k~>I}d3~(&+rs za`c*g1S4H1lW8GI%(JI@*phLV&Ajy$`$Y(mx8l*CpiBE+Dbd||d#DQsu>5^Q98?w$ zu*#X$*k-qc&h0+~6Z7?OUA7yewCN=8c()sl)z6@fn|IUCf0O88gBNIEYRI(9o1<@> z8SVFqAytDhCwX5=^(OpN_!H?SH@}hSfmz)y_K5b<;`As1%OZ{W8SX@oUT8ZQbl(*K3x(JO#&Z z-baPce;8|_jm+DS6(I7d*E*>|gxC#;z(R>%Y~e}wHp4}tbjGv>+;V;ry_+(QKQFyR zyHYc>QC~(CA0jGyN|WqMJ7HJe8;I2oBV=L;Ce7Tye#tea9+h8l^Qm{xa;pqZrPFg) zU$&#;%{$D-B?sq_A!F?+aY*N?kzO2cBEpB0G%*sOTLJi(ErN)Aol8Z+?S(8>h8#peLnk{HY;`7 zVc(AIe0g*}m)Dj}6);#QkvvrT19$)JhVd8!qNNuKE5xli&$S_4x?msNvF98{93SFQ zvKk%amf1r!9n2$#^+fpSbn;v61T1W2+q8rF`OEE(urnT1gW{|pCg?*X-l>zMw`%sY zv(0nbZo~@_;ZRLf^5vKnmj!5XKqXquXu#Z~#W32L4qv8k=HH9x2RyAn+Zvx+^`#~9 zuX{Ygi(^ihr5J{HHfhuUL}f|vQVS}oGNY|_#yEPt8^GIA_K^1EChCc+@_o5^Y^A$A z>6hz=iw|?a=W8j79&ZKXZHqWQju*1E0=VSeZt$0VzzCajInU8+u-pu&%)dp{XUc0( z(9;4Lw-mNj{60jkG9#W-=Mk@cn?c4flZL(a=8x4J!}>BC>poXSkj>4<@2@+V?Gb}O ze;49koeB6iVm7@iWlaipoPe(_-tab33(N$2nYO@qsbg_*hU3s@W! zXM_e0~w)YiB~U>mj@sbO4)EuCnee z+Ax!zfH}d!bd7g9Rt-v!L)BF{-?|$j9%nGF)e0ozxDD<*qsz=YKO0r^bpSOVi?P$j+ z$YXh5M(OWPekyl==oEawCe>YmTEPf3{Om{lvc0MB5)ZIyl%?r$6R3y$T6R-q0(tp- z2CSJNjbR(3>50g*aQ1}-74V+Hwe28R5HZ zb!VN|jliPDn^n%+N-Ye;)(4+=#?6X~AMBhpf7N~mav zgvu&1N;^@gXoznpDY6RP^Ld0+Dx+wTl!$~fQ$~L0_a}5a_nznZeBSTZ>qNq2F!w@6 zW96U}EWAoV&6UMG`kmkrF7x( zm1wf=(KVdCNfM@2?PlgiaZFbi2T(q-jO_M)#cqsHr1Awmq(JExW<85!|1Qxd{h5Ab z;Ls1oUg09f$w;IArCs>9DxZJ-dlwXVG~&7~3u(9GLDbtEi%a$AgSA^JXhi>lnK9QH z0+z(eR2j!5nwjLS%Yk3C3coqEW4XmTTD_HHM_pmSJ30ivemn}f2ffIWrb1RGEP))U z5uuqa)8Xue(`>WYS1{5VWYczUr{*5B;Khsibc1~{^lnV#wcOqhUmwNM&W{c3Sa&pZ zn`)5UA4()UR~F8yjq_fb=AgmzVD{cIZCdO*0~`y_@oRrB#nf;Auq*v8*k4wsKg@v& zn@u7cQpFdQz8F;STK?63-QL*H!yHM1G;7v!+OnpJSJShNVq$aDW~tV zl}T%Gr;#d6Ql3V6dsdKb^Mk3*#rsfR)`K0_OBlA)5^i2Q!1>boIDRFCD5knXO^X%l zXQ)Nq?{9~Fv7-2-s{*fbnKt*|JDKKPa`fG|5IS^w2`u+6MUPYwSWx`|I>S2{vqd7f z_Z%PFoY$d5L_BMLitCX+qwx8BB5$777c{>g1TQTNh}LssSd%Wsw)a?J#y}dKws;9W zMz5m7HF^3c*^YRG9fL1_e!yvgeEgHK8o3D(jV)P7rG>+pDk%c9Vgj&tW*xS4?;w-z zyu^7jPQ2&n>iBNF4S{FNE`z z8%fl{1YnhSp`o53xp;AaUHkMs*v*U~r{=2i#ZnK`{T)J_v#*WoOZ~zlnX+`N$vVjU zVhB+Im)Vf#ocqG56(>9&XGa|;(zAWsY8oYfk9NXDvUV;_GReY6T)v&z!u9u-sv~;wvN}_$d%AXM!e&_(~`FW7}vHKqeWG;Z? zg_~G~uOAtSAtSiCHG#fsx`rlBhp}l$oXp610UgEtRp3)kNMTnAyWy7%op<#xakIB0 z)eeUd_j3KO(XS}ixgJs#ZiCs3Jsew=&$M;g6X(i_q^UNS7xXa+EF-MRJo`NMTeTqW z`OlJ;>Y9V+ttQKnFmrZN=mhL^ae>FKhp|tg7@WF(!nF2(ps5)HOZOWR*Xwbt{f7-C zV>BB+cgDh`-QD=ZPK6n~l+3i{ zQ%dnw*bJiZy9ds28O`X1*X$0j9MEg6g#s?C`@2+vvFZEEd~~19Thnw0-eijrlafjJ z_+vFb>HovX8Sf_wL17@3p+ff`D1i*q)i@)=id0P7LDw}uK!vx0G;Ovi{aUZg8=E|b zr2OQZA=y^+b*KVS`}_!c_Q>Nd@l}lM$5Pv0}8V{;c@>G zw$VO}ZaBIRjNDd|x0jq~j$9O+=}BYmek=iRXAQdczCBcTS)i5DL%98Z7K!3`UprQ+ zkvQ!)V7Jx>i$^4?T!hw=)9-a@wyHH(fOdsFI66m$+@JE3-RAB_1PZ0-=T=b~SE`D&i>I(Q5qjiz_(~`+ znM@KoUNL=tXR)T8w(PXeYe}Sl1!E8}2MT&$FmE?15{FbLMnSU=;n-hhg_$uv*?11K zcKSiX)3wyzKLx%#&;-5v^C0!<6*g^^A9?#X5a#VX0~=qTXIj)Yvu+*Lc<=oi*xytO z#TLu)faFc=S*=U8Q;qRa%nF`Fs}M>1qEDY5-hsKd)To)L5NM@w-nKyr&{pBxNvEIS zt(FE9W=9ySe|?xA)eMmv#pzS8EEwzM<`)Kvv}k21|Jt2C@YYVpn=}KjK0gVfOUm(| zVLEE(<$|lGIt+8&DOJZLs$T6y?wIPbyUr{Eb8bI3HnalYeVa(;tk$AiJAQyL*Y8mM zHp*0(wo;`6k8Ds+@2`FUZCo4-Bsy+jcE;+Fi*Y)D-`?}n2 zU4ax5Kbkw)9}3g&ICg~ou`lbWu{+U5iI41B91%{WN(#-Un4`;3xts+l1 zN>GuC8T9o9mRbKx0V`^^(hfYpx(g0M(29Lz&B`*kBD|5b99_!}&mLiF-wfisE*Y|X zttYM%+DQHC$GI7)5*Zux#M`%n==qgKv}wHy`pRfg+pkuP(qIk9?9are>t}I=b0!*` z(k99$>L7iB6urJBgs9b=!Iq;Zv6)DKpZ+^WuU3Q1=t9m(lFn4INU)V)EMd?=10p$m-2v!XI)iSTv2xXgmK)JpL`{LX;WY0gCdQ5!6C z$>Ql<=X@|jm+;J09nzLyNqsnfn(A5+X!@>B78EX`M{@q4y)}0|U`$ZqeFkGAdj(oL z1Ht{7He3w6iofpY(tOSba_hlzjI`Uw)O=PXH96*F`nd^gq_R12@8Zt>?V}i_If@#& z+sXSq6Pco`FYr#50hA5bf^h3HGQH^@dpB?^4G_-6-ViY=V6Dp}>kKo}YHm39f;P`k zE(<@#t)u}}u{6fA2lelth3LbDIL5?La|<){u+syP-}mwH3PQGXXAKkk3fw<-kS+dU zN*Cb)OsY%)5tB69t$GcXIX&T@ingcfAsmao)|uKU41==s5&WR8MfI=i(HHgi@u(cf z#Le|biM842_-8lA?v`O*iHWf-0nWI+U7TjQzrpW2&oW**ml!d+0*)`9PJ-Wk$H2zL z=aimo`e9-v!8KOO!ne6+0 zOu=>rYA2boYX|=0=nik0#LW4m?d)Wty5|PW(N(94j63;jW5GUnIKZOZFIM<=99?!| zB`>S7gm-@ED|p1Z;tm6IYHMVSQ$pfMXWw^FwoHJAb=*vL=sBEO#dVr?c#^QRQp;q$ zI9j*Y5GHt^hmEe5imUcxhp{X5<#?VK9~JQBT=mF**H_}H&bh>P zLq8NW?gDF>XE-zQKKyh*_ES2+^P?-6+LwrzsV>C4KgX1+hZ37?B|2f-PU4r9b`2Fcr?$8=iYpX)G zu8KyXWJlOHFoz0We+jBHrm+wrLau-V75rBU;-W7h_JbCRS2~02b_KGEyR-E+?!`cv zYWS`{nH<#n3QM9La6H%-dNxm{-Al*eaf3Q+nYWtAH!mPXpKrr2VG|-K8cmn$hVzG? z3oz=r+vv%`cKA|mL;8Y4@cwOA^89Qo9_QFbiL2G=;)4P7-TJ*mf8sg3KCVy8_vf)R z;{+?ZcRe{8nFd3qL+l~B9eB2UF?CU_2FtbyRP)*@IQpXm7hD=*{^_^j#VT3Wl-QA> z@i1bX^psJ2DaY|CS76AUz3A6-0VR??urAyVVwI2>xt4hdQ_n_`O_g$_PsbJF!c5_k zR1B#QP#`MEqxa&jqGoFfNR3&LA^9j;|0f1K0zElD+E!ezM&K^zz;bf`%v5fdfy%)D z@b&2-xE0Oygcgl~zpEM9U33qdUi7o)-mimGpWninqa0JV`6ARTb0T+l&jpX88<>TU zPhd~47Zdhl1W)a$VOy@4V7v7MG{37$ZF5ygv!56vnehltXjfqWj57z_dnCec}s9jK}C7YI`2Bh*ZU z#MTs|AN>y#WW>2Mxfzw|`2w%QMxptuJ5?Op2|12eVA9YY=9zpfJmfM(>N}2N7V6XK z6218RGY>xP&VvcI((rc87_WH>_gTaQ(B-xbD6#4evwe9H+x;LLhSCkF+F&lu-rR|< z?P;up(JSmw(WV=AI%09%0BjUtsMo_b&|Nl}*q8l8i5M5!0yQeO z^n2-BhQInEIC*lc{kOU#ExZgm@*L@WuG8)wa}>*d>5}C-bvOGjf9JCHP27(70juR9 zKzb_+fUhjc${lc`Usm`My#)hMHCYOljtGO-_Y$bSH5(0<{ls)pFZO#*2kK|9Am9E4 z;OEL-yx(vRB3*N#cYi!SyH>?MY{-J>Td&cY`~9+;t4XSzITlvZ3fgM+375wN(RIh$ z(LJ=3nX!Bq?-Hzq!nWBskiU@;l7ER;_GZJ)@D0=?oSP5F<-oeJRGi#Cf=>rmQZuay zR9NN>G|6p3?<7$=Zpt~5GTZPQ_k9&j{f)BZKE@6npvM&@F|h*3nSn2id&qQhKB^JS zy3A>=p%}i3ktf6dy<%n8x3a!Zf_V>X)7ZxA{SXoHm?a#i=Dpor*!@PIwp`F8Tie!< zw^bTAQr*ogQQJr8G&>ak7XI@LqvH#+T#ZTmc|($x$#L1; zbxE>%JPd?C!ko^>Y|hhH49~L-XO-;76ka+eE%ky=AJwT_vMFJo`{MJ)x4fz>QJSZ^ z9d%27G2KDs@Muc~Yp$h_t+DOMNEgBr4Q^VU@;daJw5pzq$`&tk>V4N`ObPAxw+P?yWFf>ro~5w(8K%f-_8bwh;}iSVV7{RxmQ&dgP_T+{!%ZQEkrEeK!tc z=S^j*BRiRmZIWbbSu8zzUys^-3(1rhC$Xc7;TvW_2 z+I0gqTV2Pp1Mws&lFLu7dWq*1*CFqbHN>ijQfJR=cw)Lg<9c}#wqL77<%(NuYGxKb zQ;#RZc!)em-$0UAZQxjUHaK@Lmn%IaK|VqnUwZR1CRJS**QLe4vA$q7?Oqh{^PaHB zk6eO}^ZsIC>MmMouf%_sW=_Rk_CkE4{>>eA8T7{mV>0!D4R874Y<%+k2rP8Y!y^U5 z%p{{jASoD02AHp?AF~*O>%7R}y@jmKKOWWn@ElbiYiP}DyK<~Ya z;c`yBcg}yXt z$B~9l7{&D>PRO`$9h}XW@yw8%)m0+TN>iD%-wp7}Ya1LjItl(0#AzlsgVRe;r_L|* z=xK#cypp#Bat55q>rH>zfuJMcAK1hCaJ$mAZ}MQ??^_spEEasEjxf7^-NWAPlH^QB z3QW1I&iN5#souvx81enX9{jvp|sc$F+(X%JOwK|cv>Q*Ma(lnrJ;*jxE`4T7PCgMyM*K7xR=6){8ZT7`G8q+_&S82 z-%9$q`ChYnFS9H=7AsJhdo#Ir==PWV znDP7UmAMZ1-SR$`KkH=$-IEx%NmrO(uase#59cbostg#o2^S?E#1&0XVO;+|wk0GR z6pdx&+T0z}xhfgjg-8R$N)4`xY-)8jMTJ2{sb{vkr|8z_>|LnDm+UK3*B zQi6jA@1Rz7I>e1>Gxv;&SZOZ1RK?AKr9Q?ou0A{u6#=cB`kJ<*s-nR$}%{L)w0v0)K}METQ|(TAah5Ec>}Ikav%1D8sYZH)pSvAI{s8#12S!hlMb(^e?Q8T zRj$c6;Ta#am!)#|&LcRS_ltk3uaVLGu7|CP`tXIykXKXF@B~kt^c!8p>8jfxs^$;8 z_%Zhqza513|K<~g zm=5R*Qe=K6tc2(70U$0FfwgW1q`K`7t0OZ97X?exx@n?B);5T_-4RVn3XDkY!=$_0q3!?UW`;;<+?-} zThVKAEzG&|9m-a_Vd2qUzT4ajU>uyrG~CL77WoF|PU`{~?iz(&i#q;UdjYaT{xzKA zUaOz~69ok4he(vfa@S`PuPVx;BfeeAsR79yj|k&9*0%fq3>R7HpQ)c?nJ zUzkSLt1p3XHBVt;oHe|S{g1blIfh{h`QSTY2B{Y*<69pw!>*ha7@Dvbb0gECaQS{n z3u*-)b1^(xt4Vz=Wr%B>7(M+chE~jafn7_clAuj%iT;8pTHE3UKED@IiN=Yf;nQx` zxzUS!RE(orWX#wT!=0emB!fW`A)t}#1qJO=5VoX^p-}=Po!c)MMcIP%$P_$kJr{SZ z5TXBc789kS{p8smJxJJGz}Tr|aE`0180O%CM>eX_pT{}Y_+uTK^GJhQJ#E3rv+wbe ztrP_2rjbfDL$b<1l-rfd!@^f1@SrUhd)<;y$9EpFSd&JIee_9ecRwCA9cK%Zm*SRt z2Z+JKd!V$n5~;>wUb#dl4X^j4MTTcE`_gNE)v60v&Q~5=-G^}5(32ch64HKk_^-Kq~WO3D4Z$d`fFbf;j~Am z*!3D!5Y*8F-$W{4&gzFCmf}e_7ffQSWOjD2Sk)c}bAZWx{VfJ&%5?9&=ep0c6j1@tuqAV#AJ=bkU_?YJR+)w?k|K4V`s> zF3abhrSo!BHE@XIP1_N#!z-aWUYREE7(?HiSGnIhEzXl5K?;^;!#B05c-8h2TeifA z8JMaC;{gY7%{m9F|L6{el-`5X3(lleUW$4@(FPeERjSW?;(B}s*-{ln8d@WdU(K#T zA=mRPHPACv>cEg zvc;0U|1t9?9w0T%(&TQHIZ6Gk0l6zII1jiwqhgxG%70!+J%_&Xwtmp1 zbc!22QYlOot4d*Xp&gC7CIJe460~N#0_^8DvXODKam%tW;@NtMca9f9Bxan&zBzWZ zS?v+(aemU07yU4e*8m;tRh+~5DzDDF!1i24++kddTYviz|H)gpzV2M&e@crMHzeVL zHGPo2HijOvIuDw<$JmYa$@I3~Y8q^Fm%XB*NsP{IgmUk|e`NptYgTp^C(T?cfUBbD2$fkTS<{0ZjeaGVlXRN~ZtJ+r0h z$h+la-}*JEUR8mK`zY*=w^*wAX8^ZfHK9=khq2|eBCb1YMc+>@z{OXRK>2t#Z|Z3;xRBSvNaaL=Q{*;w zLW&`r;^|2nk~J;Hloc*`id zvX=DG(hjuoJ51X~JvnY+GX8ffhOV;S!R`7_viq;^g#l4M1YMX&9LJBd($Yz!!uuxl zdj7z=4#1!4GURS1<@=|I(w;yIQk~Jky1kwP>+d^~z8RIsjIAI~IB%oYL=%#+ryeff zInOw8`=OoR0;rpbC7Jop2irsQv8YUpo0s2DpDhVjGy9*JPEv~k%}_5 zmvPsznM9~O7;UD`pxyhnvp%dVIUyJip)FFB&DwnP)IS#3d@I_2^e*O)y@1^<8yL-m zYj|%%Bd+Lv4;iO@@YviYCRBkt$0-c4kpogBaOE{<3=AcgJyIdr>K9Bqr-tA5FQcvh zR-?iE5mff)x;bFjcI%MxfeL;RF}Ucokqiso!gq}v zbGy!%9`s+27Lz}*(p}&1(zNq%c{l`JE+#=w;%zw8?#>Ro%TZ&=0DN%Ro*pRp2v1|B ziLZ$<+bX}By0aG|Pn-JYMMV>ySC3w3_m8z8J;`l@z33p|`6g^_%sOd1kk& z>X&jnM11~%nPO!ST*4zKZvMg92J6xEX&f>9umXRkilLpNEw0hK2^F7T8(bFuY<*S0{bIK4wg#zpuZMh&Q>E}G8=omI zqUF}3%n^?<@XJ;w(~gJ3+QX6@uThZ65>zJfPBvtBsvm26VFf&2-i*@YjmS?=#EVIa zM5S8{56|g?U20L($=9A^@ZV$j+NQM7+>O8w57a*4MNPc2;dVg^yZ56hy=QEX_2X0N z^XYC!ZE^NT8d$-RU8RRpO_Vl^0`IrUn8m#OmB!f>$$Hf?1Gh}XZy$nhs2w#f>4 z+UHGZ#{Zs^?{Nn(Z7XuN(c2xKh4$W^Uqvtbndw+y2HA5WS+?=nsf znGI(e2H?k!WGdmQO^*49&_BoCGC!4`aO$Qe@OU|!_-F;tlO-xdjN8xAU0l|O(T4k7 z86@-B5iAM20^5fIz;>Mj=1(}t1z9TC&TtKsO;}8|yL{O6!-r5H`!{Uyy3HSy+DL3| zU13qGFcJLj#XYzAkXu&8f12(rxV~~y)tvf`yd``mb)nN?kS9gcmD~%jC?K+5~fW?tnPwAdj=MC$VLUC;9qQ zfebkrk_S8^9IlXNt9owZ=hs}O*r$uVHQvIMw9Fx~oi^ZZpiZs_Po&J?4Jh6xNd;3B zVQc1bHeM+b{xN?rO+%b^m0V#I-!3NxrAZ7>aBW^~LU(PfP{od%TfN!&Zmd&T7{a-T5WqVq`1xHQq8VL;>6 z-eLFd#jyQ-7u?^cMJJhXGuB&)c;xg8NNMYbO~Nrei$$N=!^@?}{t<#ruT}B%iWGXe zB7vk_(B*5(}uSI4hE$1zT>BnB6obNNZo)Cl9E<*I(c^z7?SrEoQYe8qIIkTo9 zm^3`AfaJm!khA5wOO7(WyjSRFvtATywfv0__dp;wp21UEb=8@_hN9?ztvTP166o>{5ROD`_k1P zTfsk^^Xs0S0&jMmVqN+t)1Kd9B#WCPPs#IvhK(CY0?A~}P42Vf-)7R#u@pvNv=R+X z7n7UIv{)I>N0#9_6Y#dB9t3$OlZ!PD%)D_6Vy-`l9C?&Zex`ddG3iAZYOan8Y_!?) z^3_c7lMoi z=Y!yzf&@=^sw6Qt$bblrT|F*P2c}=^(RS)|qO@)vig(-toz=hbxZ`Bf-%y5w;t^!9 zOM!G47|r0p7@R3W2{EL z=pSL&pUU*d`}st4&3Q1tV95F>s*$VbR`J)ww&179EI)66R)cL{h`k8~;X$wfEPX;8N2Ab!xPh?ZEbZRdD#^#sI4WUFTyL=9PFSG?85A^ZQFqFAE z+%tc=`b^rrrUDP{cV^6ulIf<*k8E%dw~IGFivMLl;&?@FL{Hg?AGQ29X!=#)y{g0P zeUYW~!(2@`lj%f72I6q;VHMJEu$(rhJjIAhD|jW`JT4^BoBCDhQl(jnH1o(x(k9`- z>^hhQx$D=XoPZ|Tbxnvw@fG>=GFsUsZKjyqeE?U9n-D4G+vpIulm2O#iLcc^Vc%&< zl-(JD!Ed6tv)eo>mX`rTY&vdH79?JdNtjqvkIzS%P~=cKYv$JoT58_tm^A}O_Fl!9 z^)pGB4?{iA{lJ})4>|VmUiwnjnA8aOdf|CO&^f}I}n%xL5G}pr|n>=I> zFW_Z~55rT55Tbb}1C|ao;}ou&5+vjRZ}%vW=l?B6t^O3&XrUXq+_iyu?&LssmM1Y zxsDk!Y{yqGt~1s93Q<7zCoJ7IfesdZWp_RDpf^Nh$%5?2 z=t~WA>}4V<>`TTQMVxE!zb(vbF)d#E-H({6Fb7;ybm8Ue7*<`GWx|rY$fB!vnB;3p zFjZ;^o`189>^i%HoxMq$rQJj7u4&OschMIbnUtfTW9)0k-izW>SQrt zkuG>}a2T#+HNayRMfThUakB2TH~E)Z05?uAAiyuc@0aJ$cfpA?zCV~qS1cyy^-57s z?jirEG-c?2%NSAJSS(p0LCqB6;RAP8QH;3+rso96t{=|mDG^Uq9nORErDQzC%_EDy zp1_8)TVbR4cP7kSopp0AK|hYK(w}3;=FZOJU9As8+r1+6bZs#^GE^`#Fk}}A>J-^}7RvuB>{0#;tPo^pNZejU@F=pz)1Ufp;f-FwnNOxIW!55l_9QVb5 zS}j-#p6KZ#%9eBTBt6Cwot`%NHP(T+^&5utTxrm0^Y=pj(IrzHGfX>(<4W)mzd9F?NWbUDf zbobY-*xFupeT~r?a^PketXbR)aXK+nWZrW6aUdVhcKcAlhpObWcPr!b`Y66BwILFF zW|H4rk85AD2&paiz`v$DaenGz61S1#qu;57A6xQPCNS=hF)<|kqPhNX;KN69yO#dv`@l9&V!=< zd<|X|(5BOV*$^e|ZX6JtjvHQ7u?^oGnOP=}nEAJYv8TrkqfC3)M*%0X{8U_({e@+8 zaAO=5T1`mUfDO*B@kY~V1-7y}d49Pd(AG`6VEV{iSg}_VR9B>c^^ciM-4Bj|nY9HY zZm*-(xiVz<;$z$sS_@dkBMm^=^zpf%ByCDJZrzP3^5*V3Kmc3lUdQsD*bG3yI|Qk@F%-oCKv*|l!AvoHV? zJFA(%6&8HK*DJ}32R^8KJQ)i3`Rw6sGVF`XvyeS?5;lEFB239ENRNLDZIUK*`GaOw zag7w|FixZAA`jEOUxnyNe>FU1I+fh1Nu?>;U0C@;oAkaFV7BCEK)cjja?sb6yt5Ie zYICw6{r6`4KC;X*pKnFws}*RX5itDeAJq1rBBB;eq01Op(th zE)twZ4j4J1_?6?}zt@AJh9(?%U_~Z!v(_!`GIV_yLqq?qp+{t95Vb-RvQ16^i$t7w zsty8lk6s(R-?tFNxBmpC8$oRQ=5d~#D4}UFkr?4C4~h{>>6sO_gc$yG4onp7of&l7~x73*c9DD-4}G!Mi@~B2+Ke zB69;ypjg`mMtW09)x6>;P@X%P5$tw=$VGjy_iZ|M?$N-z+trBo6bD*gp$z`ATn6uC zD)qfs4oNlNAS*VSUVQQgt#+Q|6={^>=3xt1v1cQBuxcCz=P%?R4ZRHHZ!WWS(Gz5| zet~)2C|Z5)fGK(M{k?LlvidJS>k)=-}xi|E3)?lkDz6dJSq6<#M?FWSMGm~aeRYn@Rz7ncI} zzVTp*%@k-b%f%;iJaM&xI$11jNM8T$!=T;;#8g2XhjU`dwy)8=dj*q7S<*!6t9%wi zEEW*2imU7{y-wVwnt{6?h|tO+AGR`fI?c)70FL8U9OqSq{B4|%@~gA3s5=A(Jj?NQ zj0j!y=pU1J%Z)xsX-3E5Wa{tRS+!HB0fOCv=&93|OmWgHrmobKBm`OD(^d{IJ0eVu zzrTmB?8OM%`ynJCe}gy(l4gmwx#k)?_H zOd97=FJ-pSh`1n13iOyTxkWVXw-Gt$B1oEwlSssMZnxtqfge((==;cdT;BK|JC}q~ z#mQSB-|$bB@h>6Lo~!}8_nhU~%ITBBJzS?`?PHGlWJ*?hCDGGklUUCu3poCYG0d;z zc!`faD5Q4qp3D+PH-pLKzwO@e=QE3&W+hLCS2nLm=n*OM(nfYdR{J&x;WmKpG}NoANG$Y+nk=e_n}~0&TI#%ZGJw)S$Bj zxbJzvZI-vuo$K+vX8!v-#>~7ZLO*^sLH5~iND)eB8>_B^ueT{JwU(q(c{6F(W^w-f zyX*M<9@D{=^OscR#X(1dD%X)7#*@~WO!i7;-0YhOQ~9fDjj89|S zM+uT)c@HY5>JXiJO;&rZKhDnwx?s0DIUIV47xGq`1ngK$7uS9SwcKi)&X+gw3C3m(2(o(Hby6o`S{N{o2p3R$yNV6%Te`YF2uf3F@By-AC_ zxTj7&lvvS^vvdgG*_*Na&AE4b1;}oZNWAZ-MGl1c!9rs{Mlh!pA0?b&;^x#><)yA6 zQxBep2`TlUJ0%wtg{siRiwDKI87w}EBPp*Aa?Z3C5Y8FE5FhUR!~JfH2ApSZ{NI07 zw~&~Ai^kc}OWBM2V`26rVfu1a-_1{Bk9jT;3_bsQA)Hys|rnNo}fl2Z8!!>&YO6lTpwbCpfNQQ9)LOGi>PiyC2VYzp+B;gfcPlG zY<{@_V@#^C-hDaL+NG0;ca!iy;aR*}!y_#kNpyaQ3u%~1$Z?-jpqwz5O5BU3@1x|X z$dc9UXif+3{^44@)8EZhezk$1b@uelo=~hwzla&Htf0ly3fJq$Q@vPavg6%$dRqB3 z_{ToRD>C!x#+fzv_rEkoj;m;h+kR%AwEo9zU*ibPH}cs!W+lCLaw_@MsZRg1ehiD9 z&B)#&AzUHsL}o|35n**b`a@b3B;)zGb5$&^J$D9_l8144uQpSjSq9H=5aJFD;Dx>j z@>|%CY|G)H(6dsW$}G-9y`15uUQcO?7 z8SWka=9n988Q#haZjY~;^sX1rrHYXNW(COAjqU zUoMiCcL-pRu{v3*G=n*B^bLe%-|=#v05ueGB(tZELWFrC=M31!zYwTKq#M71RpVsx zE^QlbKj=jE&FhA^?Z0@-*GOTXqz49U;ykbkRx~v(6$UTv$Ky9#$?rifBguOVy0+87 zM|cORGz-Rr#jmP#@8q$Q8oQYAd)sj+Lz6B)l?h?iJ{a?p>xD-fQ0rUb$Xr`N4W2u} zOi5Yl!0m?wT_@nQ?N;>gxCoDVE zz+S^Hthy;f#=2LLfhFPK_yMoA7p~lTiI#F*J0sQQL3+~ zjIq-$W0X)ToqV8|;a_)SeK+}YoS&KCRgSDSw|l+uX$h^*p2{ZtDMID?1@wu8Bk`VU zfdZCgkbS|3lyu)_iWgMDo2xJmGt$EX>t&zMN{pCf$Y_(LY9F1SRGM=!gdt2*WI8(^rO;|a*n zB;ID4`FJ<6wHgQMn-8(u$D6F1qE1Bb=P?z=zu2}t8L;?FGs=38gHN9>$4j`6Y5{$W z!lzrz%o7WU(m!Du$lc>@E61T)LkHz-bnvIg1sJuFqc#rF@aXFS{<)qwl5}x4XvXnT zm#f(iKYYvGgSmch-CCGxbd+;9b6!LHrMUG{6Hm193)nRaP??zZWS&MIS0RUnB+2*jN(=}ehl!7rc{_q(e*iSFB<1BbgafPdK*XK1Nmh}uECRGf%L++1|-J2(1c zsVOP9m`@JX=dmp}yqV$$SGeBRF;;6pf%w%~(DSvgA--Y@$qSxNn>WUhYAdeSzkdpv z^mETwRupXQbs@2C6?kPb=jxm;P6N|-B5x>)b{zbKC;c2~;)kP5i_s+dt<{3;R=$fK zgVnfy&Lf=Bkpz0}SK(BD90=&HCt=+hjPFuMDqIu?bDDbbXZ0X1keG?{b}FKP(=8}@ zSpX8dCXx7dzGcjnjr3}b7aJYhi-RZ1z-yiW@t&(lv>d(h6_@Lo-qnL!d!=Za%_h3g z|0H%e&Bcu6qinIM4sQRcMndcK$-;>ibWxTrRB{eHxKjxUEOdRY`)%6%D-7(avR#q^a;HF3W#5 z0w)fS;p213FeCHyjm)S%j_;&{>mrxZ;X^l=XcrZdw01kOVcPNZYHQpvqD20BKjHGx zAFxnq8h$H1gclxi-oDyE`rRo4^m1EaHg`8lT6&gA-(YJwYmWc~oGNBQByvFFYAg+n zT1;&(b2;#CNj$vHj+AADva|dnK}c;A#Pg1`UR*D#YPJwLTW3azoFP=Ny$x+oW{^{B zTUfDk)~MuXNSuDg(u2hbSQ=?c{?(=7>TyTVRm`BWN$W^C#|sJIyymnyn24O>xO&r! zV4=uG%l4~ECZgM4aYqVe0(k!Mb$1Ekc!)9VE5g>;1T0YEbFF|7ia#U-Mav6PEE%klmU^8 z;-rM@F3w@zU{26h7*Cu<=M-`LRsU!FO)sUXahMtGJST&*B74BaEuP$ayNy0?90Kv@ z_QZ1(@lz7=wWnEv)&_Z+etH%mF_Z9n*GX2XdkWP)vI_dTlZd^YI$hfMjxTU6ntdl- z!)!Xk!irg%MAY{bWIshn3{9a5_zemb3`pWXRbJhM*I;F8PAhLb#(ks5VCTyi&^DHW z^}l8T%JQME3Fu^*Pnasd0Gg_d$j|0GvCFfaAvmj$Ey{F*!Mn4`jFH2P;ophWEdDZx=$LUc z#T!g-j~%XyFk?2}dc{62Urux%O{3?cZb8K@QR4K-67=sJz{ul{}NcT&B!>%O|P6*kIruOXjnKA z_vm!;68-uy`-%l=_fH@?m(Q>c|D)(k+-iEeFx;R)lhTMvDHW*{O=qt+B_TAD3Kc4( zA`~Jt&!Z_Cg-l5bsk7H>3MnZxk)%Q9p+vus`u6u1bX{k6-m~Aep69-Ix8f4}k6^*Q zi=Dp|feTOCkc|mWC}PW^@Ao! zW5<{Mf;1U7=E)3m{)$!OxaFxOEt1m5{!s$D_qRdyoIA|VCx*;z9ELZKCnM`24rhbb z!+!$5AU0?(iOl@V>>f7%{2cTkuX-1_Yj6OvXi%;O>^Cv{LsT!{A&-TVETPkxO3|_p8L0h z*oSN(-c2pcy;w^!P17ItPF)2f_AOBBB*&VDj)Bw45_YNF1akDO3%Q_U4r_NTw+yWt zWS+Y&C1FMqWKI*8hf|2=RqH8HKO>HdyR!^E-v0;2^TU`7-yhK2YfsvjNYS?y<*Y*O zHfF&SH(*4>$fW=UA{aNDw&|U&I(9gN>6w}f-?@FviFw9ENQC>{yWv2ppJ-80mo_jy zdl^q>_>yG(Zt_&zh?re@28%j+A^FNWaJ!cT0aNaOcc9HzrHEbTV(k3ag57`hpy<9K z4E>ZL+h0$m$t4nG-n8qmWs3&0YWOa6Tcu+T*LT)x45Cg?^Dto7dNMI%3A%hzVb)&| zC%rPwQ6vor4XV)fz(R&BI+krODU#- zzKt);d36hC^p?Yq>LS!$@62&&SK_rTvuMX<73@^nkI(DH$-hU_NH}jgtyHoA{fkS< z*BxGT+td9tbKWa3A>1zd9R-bBt?Yvq9x;5go_c4;(371m{4G+;f#>E8x$4Pq)o~HN z_2n|&WDi-??en+q#_E?YP3&Z_T9IA{l62Z-xQq zH1Obc?sG6&h>7c6!7buFZUb3jw7d%>-$qj-b0r{|K>Z&j;mB7_j_=@xwZ>(ff4q;) zIJOElPg_Au0)N1SAIcZ+2% z^RD<1gl=^orr(GDhc>9}p(3dkC@Bkd8oG=Fy+zE_I@olaF4Rp^7bv;M^K{1xEvFq26jL@E)b`CRP+fQqOPbHxs4XGUQOfvk%`J3DWg{ zrqiviB2*{z2^7kd!-oCw{Ivli>_C?mnR#I^{I#%vyj7lbRe3IspRDCvKcX~#wj@7w@E1NDb}W+%Xbmi7^Uj^XD4yj zIJYm_cLP>Unoe(8q>))VQ<>)uzQkE%GOglsovA?jMHR79EUMDQTxo;y|! zOef1x20rJrXtYy=R;WDzfnsSkRi_KBP54-=TY}OEp%80CGf>@C~h%%qa{ zTo(2t=B!+g3Sal}6XmR_?BPf3L*36HyKpXX@_Y>EC(TBmlj0=G$Phk``{I5*Zq{~h zKT+ens5bGweM2NHl-~kE zKhA?gsS_*>QG-s-qiZBJ7ry>_4CANsnAJfGi0&+9;=3k_>eaU}Dn<9%J)iVwRRj0; z66#@bXbJa!olF{vC&8I)VauvdBCPINQ}kOD1}-ZWf+`%Yf#5!*7#+@#;p$u*Nt8bh5R{-DkgG@tObN+S_D2P;#2DN@f#dQE%SGxEc(; zwGLf98({1ZpPSk2#GFmNjI*dd)Ezy5dd8e%UT!Bn7jKQ;ztV71z!o~Ci0dg#$%O&c z38XryiMe@8k*du$28Fytni5{atn(w#WoSoi6?ru0q#{P#R!9DZDOA6}1lN5X2Ehz*5@oF9oznMrHmASBOKefY!B_+*p6As zf@IE-w@CM@l2>5?)ZA@7oVBz?J&$D=nwW&;$G2nbpEo%3x**5*b%AfW4rEPH0%Y7S zfH{pZ#JOY&&4_u*kLnOY+1$Ixe`QLQh1BUKt#o>WmBg+WOQ0m~A}0UQhQjql#{)1x^PQszXh45OJ4}ZigX_hlXpFh)v zL&MSZQt=C{{?*2QyP!*_9LvPfyWiQgToZbBv>c?&FEU04uCm7iLsYKCO^dss$O{bQaeN+SP4IDIX=42KcFh<_~(2C)%Oshws9r!k8VKc$0qdV zqDr=Tras5`)x+#1$Kl{6RoukRAb$?2fM$+31e{Ca-=FGB4JL0v&A)LZLGv$9bA2l$ zYAcdjGkZ)KHUWM|DDI5lL)#2H=BCD1CRXY_il#S1ZYswSxo<-s-~>8kUJQFL-hug* z`)ESlUEpp97<_gK)Q}o>yGiyUn7FP2LPIcn?~v^~I>v?|gyxrO0jn;cNX{#)CH<4G+cO`~X?xJo? z0b?R_PM*^!DMCF*MNkM9$=j9#&xjS;&7Nq>ePyI+%=fgYGZT-Yjh z)1A2_;T$rKPgAJZl34m?i5GO47V{ni#9&ZD0#tj+fcvN8+%>@&^Qe!|P+4sD{@X+Wtg}o{x(s zdmA^?0eN@&-)$MP-+3lI?5#p=!gX*c@&T(bGR90@QH|@|szC>UDmEz)rTe0!OZE*c z3qK5P8dI4STPD+YN88y~m1h}&_HZ^jLj-dlJp{uq+m7HbN$iM@a|6?NsjLv~Sc=G!Z?I@echL|Ga(^LLOZ8^vgS{WaL%Rsi=mM{&F1 z6qIZkXX+INX_WqR-ktC_?1r1Bv~#Bx2J0SW=bAdv&$HgM;iF!}h6a#7zunPe%_=BL z$m2cUSBelppdpr<&sRuMtzcoY;h#D&Hi@IUtFJPFCL<^{Z4?g!`2^b5n1RQKg`{Kj zI_PHifTtP#PKhdUJHEatT!7$+U5r=e7Ksl ze-~nZ{3i<+zoo!|^J&P}utCG4h1l;6RN=`k80|`gT#va#Y%)UwdQ{1(mHE7H7bR#_ zUo5SC??~?Ky33f)=XwrL0%3Q03%@9K3$FFjrswY(QR#|XsHeIR_B42t_KWLTmFOR^ zen9~B85#uJTe-~XFhO#HV_h|+?It;L5Ag`!jT&%$!a%ogu*YAXx?Z@9x{+MIJ^vd! z7;8@$omdc?p4&frAxw!dl|(I8T4zm5qm{NnAWu_Qmb|973EPZc@FsAK^1u2#5aIePay9 z!g1VjdL8+8*9g@vIHHw%J!+YbF-J>|GbJB=z<-x7N*dqbZTMh~=Z`(Z)1}Kvv%_la znD0e4>rSK_Gv*Qf{o7zulQ84_?g9MPxzEN6E~Gnq(#T9lL3&8Aj+>L7V#Kf9fS+6j z_tzRroXy)nnvZ_LjnU_rm6~I)`%VL*^KE#1Xf>q#dW3eZubKRu5=^gLi9IjZ!isIF zeDU5kR>)YL+;MfMg{&HfCf`jG6SCOz-b)xYf9{Ogas>3B8&M0(op{yD3%7q0A)5?+ z;9|5r{cE=csAdPE? zrFk;wCC-KNI1}&^(4~5(Q;C@VxmEZq~s=_x+LRvdoJ2ws9eO zsWk_7$4wwq{t#`t?8!M#bm5o98QfYSK>J_*#LsKhse!3EJz1E;HdGeD9=%nl_RE7h z>^KF(%cZID{8w;2P=T&W_r>?Rj-)KN7w7T5qV(qpRP(SO33lJeG;B~H^%E?pjqiWF zV6RlnzIqm6 z)`R%05Ab-%9pyiAp0;dDIK7ec>5iPj57XlC1jl^!U!Q`5n*zzdz1HN*#2{96*D0p- zbOm-=TS9B2KNakZMu8cMRI}BPDqN2u6lG|%+9PHx)|li8bVKLzPDri4Z7I2L8a4VI z39Dl*suTANG3T>#(aUTV-k+ci=bisS>g9>#^>$y-`230OHL}E|Lm%MNvVG*_@fyg! zGlN9Uw4$1oR(SQq4~{1tj!DjnvVxRdL9h3$}_ zhnyXtxygo9=bg4Zx>lN+$?PK^W{Eh6DaRSLq0q{}7t=l%-bm0BJlrd%Ir{d2VczL~b({>I4Nwk7d9(us|TAGl94 zL+4fza#25$?tUmqcdfCa2@URa*`p@~xT)bv@7xY1Aro^Utxknq@mUmoj66xm zwk2e)oDk>h`Y-bcKyU&szGcf z*j9bx<{I;Q+xWhsDkRIR2dakbKvd)jTpKuoS5HL|PqPPDAreKaXgR7B2w`B*77WV@ zB5S0yXvrKYGJC}wYE?Fl@>z|Tc2)&;-)@3qL!m_Q<2009EyT8ne`K<*v|-JzI<}VM zjhq=i#lA53iqnJ$vr2UVF}th6(jEFl;pSfmw&8LdXBts>Bot+u7?RFqYbJbX#Oz3W za;rOooI9|FUiS0FsUEM{c|OD7G_(i|b(N^nAr<2DK$&bhY(P(Cucq~!Yv9vrLSLw? zB{AhUV6~S3RSjJY)(+a#@7`*-(4Ne+M2OL^J!zQbevv&;@dIe_b!JJ80mgoCrV~Aa zfTY~R!L1v}`wQ;mY=i}9f-mY6b;9mQXVT6wVi!c0;C@R!#%t)2w{^)lc&wN8x}1Y< zFv`DVA%{zT)WTw}{+oQP5tKA1V1S7`qaYVar?wo!A?IsYtSZS z8-%ZIdZ7R89X7wG9j<7J(GDS1%4KKBxOX|vKE;&e#m}XAlBQT8K9$Cu)SzL@TtL28 z9vx?mVe~%}dI&?vy0{N`sU&3bFkkjpUZNLbL@Oe$TdyHifoQGx5tpTd7}z(mcD_ZN9Aef?_B1i^$A=*aXquA zUz+ZlKEgb%{f^&u-@)Uea@0LE475a>SoEzyJYLKXP~&*{!w9)n4^a1&Cmw56Ay?C9 zGRI^>*;iWT_LmGR4s z2e&U0^nA@W5|bHC1D89&3(v{awB-wHm2vvBA* zcs+wYJbff69Td>*g0k^L_26mSjT;R+3 z&fjTs8Ifb~af>K>iI+}h42+=jnI3p0t&Z0+;vintntl-a0|E=`nA`T#VZ`ShXvFA{ zYi{vGRY?GwvyPzrDK4x1ekc9+n1zwkru2E+0DSWv1cehiD50oNUS&!` z-i|Wvl2>rsKs@-rSqKA($*g#MB(W{BA(!v`f%ADuJfZJn?0Bg(NuArvL_KN4u#YLU zwD&N3=wd2QLXXFIe#(TwyuEyZ!`Co*nhD*U<3cx2{)7r=l<>aVdz>D86u&vng_15^ z=%0HM9|m(g%dm8^VU__*bpFfSD~-n}4F!^tIg`p=4}q8I^D$699wb(jKaFI&M58>f++=b7-6%N~n$N#fC5N4`5NOj8}#pt9yB zsF=SF*Eb8(X&%P()=N2Z;`baVKUct*Zxo?)T^&3-83eNSJ3;jEYcQAVW7e)6!|So8|t?fJmY9;<^V+&MSsp$7l98B)`KyK%R|B+fl$MqheL;-4Wo+#w#wa+2CgnBbxHHV=&Vg}COc0VVM4KlyC7-HBSjCwY#=s|cYjMzm(pdsv-O$-dhy!M^x32!VHlsmil9 zet+;o-0Ack4h#1|{Gw#~%Y7>xs`3Q6ib9Oc79$0pFW^ZZF^F=Rh?V9uh-^?HFEZ;i zqwjVVq&eq+j)gfDE%n2NYuE9b9b@S2<$#fJ5;J|xC&nVS3ey}raKVp7 z#Op~L)MlPX8P{m;?)wNA%kQA}vnCNQXJ2w*&JA2T_Zicn{Fb#ba%3`WXOZ~kr9}Nv z5lSzzq_3x5MUy?c&b{i+IYG+c)O-Z}u zD^TFLs;}CYQ~Ox1w+0I26=Op|ift zgAyT#x*yM` zR(A2MOC`rZ;+V|QiZgky{wroAPUV7P%tV-Z^)Gz3^<Z z8sW2&87~Udi8qILF;Eivu+j7pz(&`++* zQAv9?@p$VE>jUpHX+H9V-~9m~WG0c@szrY~WJ7!;!&KbbL2F;nV_R1-v?1{tSa94= z0oz`P>S|yQMBZgD)o0=kH7>8%gW5~I1hk3op zg@!*irIR!!kW}~eFt}TZdW+1a{*%_?X*LtDf2zfn!~jNe>v`~8pGNxUDw0DB-LUCt zH)`&l&w5lQqxas4%;j79G-PNp&InFr_S)GK$680oGS;C26{<8iN|u(~X$BwBTVVW% z;{vB^(63g$3uT^6p^^m% zrsEe-wLO_E|0+xO$$Y_^{l)Cw;#{J0(n|lFji@lO(-ll1{3I#~~uJh7mK;q1p-*ZH}}-=Fjcq@6jy$ zRP>AU*~!txOe`DQF_Y@sZYA~$Ey=;MdZuV3jde4xfZ?UTvFoBVed;s^6^diXEN~*! zcLKR}TmWXS^?`Q%=lJpp*YPtiLDl(hz#=^r1&ZcFc|6C*ELWrcAHAvgf34tjumbx_ zZ!`TNYe+G7M@Y%Pf*bnOS*_nfOzChA?=Jr(*iMP(n_eHpIWLrn+CT-Amz!Wmn+=t< zGN3>Hdf|(UCwR9k-=o-=H??C^@X5oAkiEVhtwTAE_NxQrNy2hkTONf198=qrV`a)3 zwJ=xXjoH_;gox_&V0`vpB9?3zh50k*@?R@_25ax1_=(F-v|1nxwkgs4rOchxrRixi zS=J_I2dR53h%E|dAz`$Noz$gCBl5q1xU3Y|<;D|7m&kC;t1jVa=n z<-6(UtMf=+!7bJ<_cZH&e1CY)wWJ$R%sH3}U^KX9|Ld|WMf zlVdwx#jEr6nQ#L+mY?iS^k)x4&J}gIsvtx?jd)b>kpelV{fl2cvk=Pa^+=mfCYO6! zLiHcoqEYsI8o@k)pM!0zxv&In^U23;!U9mdMG)>9d}0CLGcsV>B=_BG7Vf@2_H_Lg1tLWLf=)xl-Bl;{f0&5T3003A(@B$qk<>Hg^R zmNoOXL586={239Y623Rs-0d90CTbj(e13)A-F3`0L0SCVo5*AR1<<-I2M*QSkr&0^ zL38Ll(;q*DY_H_rF7D|}?Wmn|f_9;NXGo0D*IZ5z1NO(54~@*rH`CF5y45!S{RF>3_w zFn<5plY$~MtX=sAuC3&_OFrJvw_q)`^bjD;dv?Ok)?u8uma-4k2&@cy2;Y4=QMceT z3je1{LoVsV&{hNbII9>%{O^O7-Yd3AGK1=z2%wKwY7!mJZx)-*vAC5tu-R|s5>Vd& z)79sY?BZgao1ui?p6IedCqLuh-3AE#yNnpmT?8SP^EpnP2bZR5 z@y&fPc&z41%8n)y5uP@F6*x##Ck~+wjU^uw#L2?c3;gF@eQa*h6inX|#7rAeBkNuY zk=KRuNwbzBRJ5;ThwcHbD>A`y;RSH;?{`$NYk|=@9!$a(Ycj0*n_qDtng4odG5r;# zM}?Q!@s*zm)5&wxiI=Sk$;=sG+(V7jW%F5LlGo9+F?S^8IEHS2%|o- zbi$MARNf+qTr&Q{*gyl`zG_c~LzmL3>MbOn+r=0j-bK?lEGAt+mh|!^fbWh|ka6NV z4wK&Sp1+PG{d)xYB2O1!_qaQ2mXgJ6?bo0m$D`m}(?0ULi;t1*x-@0332p5PW7Qip zp-1WrGxy5^eDcU2x0|NI714NRdfz3`F*yp2^D=Rpsw@o+xB=5*73sqF8Xz+*h1RUg zpxvQkSb4>V@}mWa%IrsYC?FkT#ow`Ela`?v*U3nIVMxphd~mae2A0QNV$L}?ad*AM zRvR0m$~_C3qp^s19tZ)QDG9{7(}Lfs$n_>Rrm!EK%&BU}6No$T6ff`SVBb$JM&IMo z#8^mxarZilu^msLGn7XHer=}hokfgrYXB@+xq!<1dt#c_cecOVkiDrp4NSQ^f#25x zC|aXORP#)U`j{T=dO4ew`V@n=Z^uG|+%EXM>J+-}T1>;1ZYHMgX>8uP0^IL-1;#em zlR68)8zQ+?L0$ng*xH-U-+Ti2Zf&6Jun`RR*+Bm6sh~467sa_QL6zDmoVBBt>tke6 zr{-$T(^kbKJ~%|f4>0J+`AjALY@nBf{FqRiIDS^x3Dmn8hW%$0Y0N88uv6Pk+Tzq` z!E;l*Hg_`oc|Qfawk~COhKE>(J?St@MS<-3&m8U=+j5<^LabW#7!UIQ;gKh{xGPbO z(oQ>i;CwDS>+=QnfR7b<@7@N@Up}C8+ZMWQ<~D51ibko#7}y&jNq(=~PUc=rK#$f; z@PGZ7ZQS?>w)FU+{qE{S?R}`* zhwCUDu1A(ho=3eesm#ylos8MWE)aLYETreSzVwALH4@L5LRfe=Yt4w&ic%b9IL4{cLK5lc z$b0R!h`wAUM=lQCg2U1wkbm(XmSvc;PYN43mg!Br|85ew7WNy&pOrEhf0HoU-~j4Z z3J@O&FX&-qkco*W0_)C$MXD{!B&WWve!y{Bn*Dun@nUzPy#G2IQS}nKhjY+xa1yy}_zw)a z9dGfAazAvFDohYo{L{FdoTI`+(C8#kZ8D-p`zU?67sX8ICh0WvW0Ru=ZX zUdq1l7N?)0x6@oDKRo}b4cQSZ=xPq49U3>_uu>8K-;8r?X6kmbpy&Wjf6&9_oNmC? zq_wm>s~Qe-Jer)ScnGadqd&MT$-7-F)Em6u<$X7yrtu-Pv1lhQcpFLO1NTAKS{Jg% zZ9f^;)udJfGcDzA<}e1Q>fk}vTX0fKXOcn+neadsO2fM#E`dkptrH>*|Av@Kcdp;L zR*dXSR>F5e8*%faM2s@*hm&7*=!Huh5AXY2s%D;oL1OoDVV^o(B9{$ImK$N@(s20j zCz6(2-9beQs-bWw8pi|osCXC_C&m*wwQ+gSw)N%%%^`2 z&w>Q5N4w{^DM)fYQpeRZ$=<`qaY>;*=fnJgPfPNc7dy{j*J5CHuJ^`E*Ef>K?z`Et z-Ww=0@j9j)Jp-=q4QONGAMCzXgLUDf)eWaO_Vj5Uoj0)sjqS6*NOjQS%|kwv`;~)) z{R(nD+mi;K-2sOZ&%#GNdD=b0n*JE@CmZ_jay^kG8s;TIE;XOS$|?hV*82m#-sp!b zcCVO;qJJU5Z3-2eq=ZR<|FJ8*#?gP>cCfVI^M;<4L0G&fJuF@ce$PsvkMl2UiN40S z`D;kc0X2O6xCpMBtc1f2Bk*&{6MS6D?H@D~$OF!;E#DnM%NJZ>Ob>14?zjTV+h z0lhH@Ni{{0$PPTkGVmN7j%L6}AbuPCOEm6&Ttp5!j2b*mRs|9#VO z*M$$Tt#A-;o4DamKQG*=^@VwsH;q~UW(!u>{=z3>k@QMXHqM$a0pFiHkZD7NN^oaS z<&=-GE3gs9Te$bc?c2!LoF)izRK)zHMnpTp7UoSY;Y;1`K>P0dO!7-pT(NE@I&RSg z%A3M8EH{FwwjB3)_ay9GWy46ui?Q$Iim>dh0DaV*Lgp;!<1Jco9yL6yS+m$&ut*j` z9VI>3)w!9b7s!yb4;-^$Y%|{bnZ+?B@LE<<003V?=)L zSc7Y`^yu{DGpLh42$Hvx;3xTw|N5NR3lnVUH#1Y_U$y{Qr*e;3e{U6;*cpN6)0?Vi zw$);?JLKTIZz5*=2>~yqe(=ueM1hcMC>=TnbMzaq^0^D0u~Ho5SO10`GGQFsax*e@ z&p||SGgV)iiQ3#Vy}U=6<9IiKRbwH_x)s61@t-JQase~a*AcZP;$+gMziek(D#t_B zB1yWXc!aEAT&WrN{XK==YXZP^o*4O6BM7~79i=9wy4R|s@B}INAnS?4$NW(0w+tS>p~*3AvLV?r42qnxA-wu3 zS~!gIv-}2d#;P$exc`e)dBKOx`#2{C(*jc~#4*X}J*rO*0(f^CJ9=a23E>8exvc}z zf($jLRv6K(4+*|*WU^j2b0$?6EIB9kko9D;p;rJVUzR40UpNQPj5l~|k^psj{Dgo0 z29I2`4M&&OaM)_b!@D=iVM?4nv!Z?gKXU$n;u9BOchxA*{mxlFd)OP6ByIp=Jef$8 zo8#UAVf4+sgyJ{L*|HbQ!N6*W`4#dXY)XhnLtiJn@W_>>e0|G&o}5X4@eQ$0E{fv{ zoJLRKjnHsM4*%o{(XiMOI5o+K+%f^q5n(`16}XUy$sE&osx(Rjjxwuxd(m-%6h@w? zsV;ASf}8xr$@W4?3a!?3bSAey)?JJ8hJsXi`&Z24KG&yk3_2ThNCs;S!RPJa?3Al~ zs2Re&9f!!q*n;Uz?CqQORHA1- zah%=&+udDYS?B=onbTU_x;GO`SRo>6x)SrArsHU~IIWm*gt^=w317K+(H6@CjJ233 z9h1$5&u@J3%`{V3ytx3hF1}~#r1glsqYXLGU&ZTOnt?ipOF?fSn5IgY;ll};_>^*t zk)g-fkdn$|TiH{u#6a4;-~%o$oC1FyjKI87S?VUb4RmCVfsdOibnl%_bhmIG<*Z`R z+3<>aJUJQqc($~(YX{k=wG?*Vj3&EOPB3F(5tx>r=~v>#Dt+oYJKuVI7USEO2ZGEiEP!zF!kpU^6oXOHLeQleht6}As!fV9h&Q6 zGNjRDn9H;kg3}aJdhUK5PauT>=TAE^#YUKla6Qqb*#&&t*a-w*$x(%S1(3uCz8+weQ%05?lp;QO*nCUWHfySzvj zuWM&u_g4XCP;v#Cp3S*eR^9>BSln3gV2QJ0c_ms4G*10M8yZsr*Gjzc`*8&*x9EMSy z2&B#3up_~U(NCL(`?^FSZs-WOHQvH!{Mpp<+A^3NR>o|#Fr(8VE<%00AUbfrBbK_+ zw8ABc25w(OUOls>LpNLDw3awsRj)<2UzB6Ik8|vXZ;@o#0Y~ngN(l5znwU=`M)cFk zU|KaX3m@*ThwVES(~j*@WWm#VR@5^YuJK0E>E>tjcASWNt&+LmOC%vbuUVGgs^HE_ zG0L<12var8upnKY^mIH!k^E^mJ{ZC6T!o>bSfAQ%i-#YQI^^CRuEXAOg)y5eM~;-v zAR;AMSj-8rZGM*`6U-3V>gRZ4Fb>)uo@Fb?)A{4hC&0phM>Q2zlZfnduoUXCp!+{e zs-8g1O(vn0@(6ThaCxPta%A6*rBs@mvEa$31JkvVZtfGnr| za9epJGi&N}QZUaA!q01y@I{&Mz(bEFSJlEcZufKfK{X1#@TPn5BwC7Iha1|D1U^L=?Faz!EZ@DgdU{S?q?kRv{?Kk{NmkMaa( z9-zJTChVLO#>DXSDV+2vA1=KUBCf_M0FmB|N%KvR{@_nHCE0?Fwgc&In?@IX)}RN9 zHskzfuhsI)kp{3wxq%ROU3`L(o1 zjG_4nL3n0!F`6FNfp(QeME}7FXkPOgKHm0c^DhFk;7A?b>3fOQuV-Sp+E(6t7lQE> zCHTZ!m9~D=BPphwGcsd44a}cTQgNiCLOf#-f)10`kqS8(dNn1I3?4rOj)OPR!)zI7P2=-yg!3RRHpPPfp@NcH6>=yoyqVw>l@_plY zRz~*DN+Oan3+K6RLPXhF4O&PV@>NulkxfRT&``8wL`lwb-H~W$mzGi~MJh@qmGpal z|A5zVUgsJ2b$veXw}jL(@@KUWiRzyOYfF`g@gr;MHuM+5uPM>hUsu9G(QmlKfpb|J z$G7_#S~G_4gjL5N=gdlS}^69qhs>dWPizOTc!W*<;EV1c$>?Ra|I8(!{?U3A~+ zMXb~#6S}xGh-`}%Aid9C@S`_!yhw2m+TA*bgiR43*B%Sffm_dD-&|+XGiJklQ98}U z-dat(Gv8W&wkZVuvus?l>jRTn^a3WH@+DbL|M)fA(^0Li7DlX9>AL>o%xaE9p~Pn~ zQR@aqr(J+fw-{o*!3okikN7gKn2ARuram_hJ2sFjAp#9%9TgXjH=){PDOJcdOX4sRs^1%1Z^P zwRr-!RG+Z1(1wGXFEZLKGwAC1-RwW#0aV*|iG65W$l8pTLyV*XX&=@jCyH9FlV(o> zo|hh3VZD=XEoefGS7LN%3X6RS$8o{=6vlPF3_JeCh<2XdicdceLqo<<=D3Y7j*V}p zS4yoI(rb zV&4W?vMFABA^+Y~_SA}`DICOh1Bo{pYGdZ{{yt~R0_Pg8M`WgA~+O&-w{j6{<&sx))*XGY?)72L8(rop*qL1EMo z?Pg^op6SIydj>f+eL9)jSj&`T3)5vw#+cSh2MQ%c7@oO`-23+lzRYRHdKCqjleCoN zug<_ZJ$KNj&l+=AR57-%xHGL`Ihy95NB^;ZxaEl_xgy&FrzfQm|An8JQz4GDCFdAZ zq%)EDpE?X-ay{VgVnf||pCRMd3wC>nAh4P~n01Jt-VZ+<=X^VQ+)O7ib~5!`dk_tK zI$-~FuAjY(%kOdw&%WO}f&5gcJ{|LM0 zWWlTZKXJtlPr7~|iy{jPQHDXhcE1dA6vI$HMwMs^tI+%%( z;Ok?!N3$1a^ErOV>-jW8QJWsT62i=jy3ST<>rtU|l5|G;Z%Ap_f(B+R`$5W#SO4`Y z)Noy!)H8%kmb7NKT~y-yS9-jOLrOS-?}5%|_32s7h4l2HOd9X#j`@Kvp{L`Kb+qAI z{?Ep26t;>Y1$()Uc&ZtVd2#=oaH9o2Rk1h<~McwUuNK=Fi zEZ@TM`qfV1@%|$?+eMighreXJRdlGjNDjJ2y~o8xl4QG20GHq9kw=0+B2^xsRrx*k zvEo-|&AO@N(w~etpCGYaMfos9ddwCF(fJ@o9SXXE-%mRj(5rVvzu)5sMdmX zx`w;WD#zu`T$6{zxu_c!2?&#S4f=WXxWRKI5J6m6=Axzz{z|Q%HsoR zcjs6lE!K33{S1WPJh~_36uur9Hroti(0&i}(>7pv^1E1` zj~idc~9BinO|9>Hf|p)Ho|uQ-UofBzT&f!uTfq*m-Ran$`<&F0oWD64c~H>>2QI* zYCpL6Uoh?cw*!~S_)}RXfFC(2g!Vj2AvP~`Nql`1KmPf0*yh~;e}2uNAY}@xgaFa(35Mn8Rk`RtatDz%S=I zuiQ)>tThYyZyK$Mb7I9t3otrWZv4l;;f+vSLX7N)fEUpBb z>?yQgHjJ&ivJIR#zu=LQtGM-&5p+5l(%S-Q%%w%TRBX#f@GprZb_oIy)wPo57_Y#D z_x6cs*SeOxTV(<4qG^2Z z$->0{t~1ja*8p=D*s()mOSyh3*Ke%(g`a0U#QMAcV6N$HEcts3;tVQqxO;N31msvCHOiBa$+UotmQt#(*mWQ%%sU`5%4_o3+o!n<$jxo*aN&^D4cc;r=79JEteFz%+yyb ztauDEE<)gw_nX;Y@C)}Hyu!RK)gUXQBxt+hFtQ#JbmYu1Z{tZd^sEym0jI^u7Hb3I zWOW%urtF9GZ--HMM+^_t;?S4#jbyLbZ*xqN|XwJJo)bOv-j-GavL<|LkqQ%`Xb zk|gIzHtl3#;$A{aco9_KTP~QlrNYi6W7=G`gxvGf!RH@t@=G>#VNCT@5dH5g^KSMh z=(RqH#=kbx;J{&4jC?a1P0OOH1a1LpL<|O(N>-OSWWgD2cluOdl`# zf=!n*xZc5HdV?l&d((9K^S@XqYIfdx{7-Me@7W38YRq$zQ0{5H! z2jM5b!F9O})KOKA+}NN(>nC&0qOCcgW=()L2=Hsu87B#KZ`MNv{aIDny|5$IQyhg-r|;`qz$R5x@QSZ&`nq} zlBy4w(Yq0}*w=#|eLu{5wr&{yUYg7(SN&#cL?6N4rKK=F&5N{KEhAZSKE&^}78*)3Uh=MtrvDtO=Dk;IBO zGIj6&Bl9%#sNe$=xMH6SbIj~HUiWznKuhM+e0z@lFTuzho6kJ-nF*15+=z3`J$8wg zB@B0OBa2HPaNR2jVv_Gl1}{jF<1=QFwy~wqIg$n)AEIdHh7_LqI%!5+hTG|L+3MFv z5SH6M;@e$)$~v6wgn|t=WTeUh%vUWYSD90&sI#7&{3Ae@3PqEHnZ|hg)0H|Afs_6U4oz}mVbEAZkvz^VDq0cefJKc%4v;}v)KV^2A%pg-HZKQuo z08=(hBQcBbLE=pbvNFaPj&QrzyvycpGl*6sKjY zrMP_ORd&A8GdNJFKz&cik+);hNtr-6xju3ZXMg_%HB-#2TITGeO0Ows-gak%mSy7E zm|ZY$*>R5ZI+>?)Fpa(~UrxohbF4kP8r(egfb}u#h5lT9M!tUmEGuq=Bw2Bu?Zw5k z+-DZ$uM(yW*DQ#e-AeN7g(1CAkqo|lb6`)f0CAZ6j7_!R!7tBhOnUc{S;O&BH?;-R zXRXpuqi2Be+`e&|UOG8G+Zub*?Wk7E)lBD(Hf%NQ( z4BRQbAF`I0G9BO4Q7d3Ro{jJ&uN7t4wJm%$hMN~U#_7=>;|z50I>FvhQY6Rp{osa2 zBHd59o=MFf+MJvP^~-d~`uJ{`%uIuubG%4EtRfCvILn?Z+)T?`cCs3tPw>onTL|kY zVa+(7^48&pxNTR3)z^4kyi&(`PsbF--Qb@rqe0L&UEHp zLF)N9i+HJS;8;iLnQuvhN`?;;BVz=yv!i>`ksM*^dwOE19*2=JEr5 zjxe=cPk7RTRA>?6Iz1aRN!^Ps#(F|EO3dtm^se*xThWIs4BiT}g+=J(kOyo{q&oEe znoq5M^!6tnF%MAkwF54gDn`yOKM$)dTVc0e zAo1CCgneq6KzdIKQIm~8AD_v=n9?$M`C1Cpk;{KQ<$N@6FES4zrKm5*(Qp`Y!)=y1 zP}w`iZu{2^b=x;#l8_2Hl3fhaYvt%8o+4Wp?dO5Ya^;)fJDn(|AK=6%v4hfB+u zn8n|*L$C$nxs2z?{zGu!Q73yy>7&(7?MJXAr4^IrSkN-Ft2n1~9}aI{ONXQNiTms_ zj4P~wO6e?Ox<-p?4v11C-?J#&z@l)@b(kUX6je+K3(!YeiDf? zN~6OiTPV42LQbojpuLMW37EEtw5@-I_v8fVTva)C(dGS&zWz$0kP9@n;WNDT%|L65 zbUN0z2y13~U{~cWuxpUvu)l?jWS1kTFH{0{jtJe%v38vHWRhz!(u{FS5-vE1uvI#l z+E(p{3jKSqPV)zdG~VGGXi}_pJPJY9PdOfR3Yl7aj_Ee8LJQ##Cg7qEPExR-^J?$m zo%C{uUCuUQU19XPbq4JG2?bP5Jqmct`6KJI*`&)XFm8tMJvl?B+)dp8K(WxkUd>UCk_8A~es>>sO^n#6Hc zmy(&U22jZ25_l%WLztKey_PeROt%vR<#Vc(dBNbWikrCD_6Rz;O(c;aJLpT-r!bi7 z$L?;dVq9-drPeaD$@%!zV02^%G?cvPWvhPPE_GGfHjoC%e{8ABjW9;K;R~D%z0aOB zKfoM2JB?!j>av$3Twn#4zZ?BwK??axsi@FVFdeZ6@qO*=tHO<7q2WmPY^a9WlI9>8 zQG=!fXCb-o6&OcNgyDEW=A@A#-k>&Ir@|Hki{*%T{|Q!9Y_Ij^DSzSig%hCY+K7oG zm#{v;46Y_Uhox0NAtgD0xjMQE&M@of@69gY_@D+SE0se|(QN3?Zh-UUZ{XWwC%*a0 zFxaJ>fenUT(A%2?;T!IOX!b(Xk7@(|-dQMkXEhP(DuWapE{0;v1&x6|T=m(**^O&2DLrA~ZLzJJE zj8@a#NU=>CbL+%CIQ6Rsme^XL{f1kxIQ%E5zLIN-_G$#Bff2B_JIlzPx{R|I>(lDm z|H#tvMv(J&rT;un!Flxw%*~-BVxnD%rK_~)dnEzd6>}9lXDq-oFBIq@4Gms3mwh(K zlcrlnc4M3O7~AKq3{91HVcM>L{A$BsDzEScdR14k8EVSxPmZCuQp=7ODIJ6OhF4gp zuK^niTTsSLjlO>!3n8-a;I69+u?hBL`)e4Qv;vvGan0<{BR^oyL=DnDOPw6(K8nZ_ z$0vhEq+c(HCa9Oe?|5r67)3~9XevGAQi_reMR+gj3FCd_8B^_%Ni97!NRf0fq275k zW85CY{>ucT;LV`fAr1z!I&jX4r5rmt8M-T{&<}%t%&I+vsBYll(;XQ^_f;S@_L&Rj z>HwFfuA!dXZ&Y`QH?fg?4##&QPtoK(vO_ZDakVn|tGwcz*EeDIiz~QftPPy)v|(?7 zIeu7egnSEWnq$lLXzz%S=rM{D-f?G!@g638dpjgGU&5|`0;FE1h3!7Okox+ppmF(} zJN3~LB9wClcxRW9N|*)Era(V@X@xX}7$(<;yI!AYlA)|ByfZnNO6%-~E@lV0IZAQv z?oQzQe? z|KR4;J#_p<1+@Q|OCM-$hGU<&O7#X6SoJasmq^)>p#^WjSu~L@FdfE{9$)yCe*nyy z-+(Rm`GO5rfKIV7QA?b~eB5P6_w66W2@_*U*Ty_LtlkdIAGqgl-AsZ%ZE56JU+5on zhwDZrBE#*c*TD4dy?CX30nFWZ3rsb~Kv+y;&`v5BW=gdX7^jlcY}N|jhMzW`4kiJ+^0>Vju%HGY2d zh}mA8M}Gf4&MIw}q_Y&Sv1Lb2faY0!_~w$s=IN#L{*bO}9{tU-TSjDn8)Y3!4Tq7l_w@W(?ly6mtpiO=7P?`5UPS=BeVf-i~# zo4&Ej3~oaDxeR5=kJoV*9oe>?UTU2RJG{;4u3JLrGIkVhkQPQi`~d!p-G>=I zv#H*H8%gdXFV;HdDBm>tH+yfq0H&1^T(Nc$O`moZn`gDcdcHR)O!Q)pily_VYJQ`W zGRK68D1eU-g7}K5R#bSSFmX9Lg`ejrOE<~ggu84t2K2^5!5i)!>0bhEOv(VgH9I(e z!E;D`aT=b-JO%IP2SAC-a;_^@q$gZ&Ldo^*pc?c77AdFWjY0az$>AcsEx zsZ8$_eZpO?_n7;O{=mG&n~7|J490YNgJ+35uWG9*l#Bnt`lky?-qbAeUsWHZ${z%~ zE3veF|0>43BoBk%o#UA4DL5;<8fr}ap;*g=*mFSYjra|-43985Ws0PJUkG*T%jaJ= z6hu6F9`q(#!}pQhaDL`>+`M0nF!7>fc?jnPj@!pvm$s%FZVY>_Sr(&aDAKbgiS&5W z7p8B%Gn@I=i`17JU|szP{yjefMtWFQ(q0`R3^GbYf+s3}y=ufwC&aE2D zYgX33W>XpdJeKFev3D(cF!#O~-74_`(?+A1BZ}Niqq~qv{Sd;O`<}z>i&wy@yA?^H z@ou{6VGe#w(WWgg#^KZ54EX1uOB@T;s8JP<_U69<<5kkCupyY-5hLZ|Txoo8dC+GD6N4kn76bN9m0 z8E}5-7GmY}iy3Tnz~6mB%;_s{`F%q^tof%xgyS*P@WOmDO}q%xW0#P5)Da|~2{SMA z^P5I4D1uUE0I&97G$TALO|L3mg;NqDRH?cV%s!^zHjW{2b;dB}trfurUo}qLb_qND zlu7%Bhpeq;7h}|MhdDPT8B_+M*@q|iU{$Tc9532VvFjJU8Vn#6(;YegvmA(i-wo0s zs-(Tl0MaVo;hJPuob9@W{1+O-7|!{R_0s%+LBc}h)JJnX_GBx*xO))$dgtQ`g_W?y zPMf?`ABJtU(v0N5G)xRW!uI)gqhq=$`sxT%+m1lafqaIYd(4RFI=ADRfY(g3=Mvmj z*3McCFJ;aPXpyweBX~wygt^RdvO2PQp?mrq`e}zV{WZRd1g)3D?2>){iO$)iHK!)3zotLQ)6 z`{XU2lOKeH4Vy^ER%ue7y#@qti;;wQ&Oy3Zh?~pZhVoP&CT{f<^qc$;;vLRI`)~|> zJz9YaHw;4EdL3N;RF2p817Knc*GcTSfoESdh0WVR);vZvC|A-O2-vVa>WZ75B3FMMCm&?_ZrUg0@aDCNMvi{sU zOpCW7*R>tc+$WR0r>0JwGIle!#h-w3zykXIMIJlco&$#xB}q-EK2oE#bl%oUv~YPR zSn0gx{070)+5HY$+07z+cXMU|W{^*Fb12LSMKz;rkh=FAM>lK)Uh-Xhn0gZKP3~fR zZtcP4GnA?0y!&8g%H=r*-RK=rj@kWc0hNr_gz5Iz;iQ8kHD}s^=l2Z?BI}sc&|~=B zHV}UKeP<_m7lFg`ugvLiQ>|53hD_+?7`kr%dU|hUC4DtuN4kc0gXug|unnH{YNzzjX4DkooJK6EEl_z&khwi*`gJ+g*LbNV7LED^M=8$X~oILK2 zualoJLfW>tR6~;{j)qawWJ$W=N;H{?*nNoe3&|0Q>z-t#f*BnM&Vt-Ov#{_d7Zehcg1&~=IK1^YCO3Febo7IC zjyv{Wu{C-4q6VA0rs905L90LW_A)n&8}PdIS}LV(LpM+Q4cD!8xLsr?xI+a~uaHPO zYM(I6jU(yzccS#h%-v-0lmaa36u@t8T-T{fg8u9Xr%@rVL5$HP>8_U;vk7n6;ZGBx zCA9^9y*Z7pdC?$Mp$D3o&lv^tShC`y75qL`#w?6(Mv?iU@UAMK;{@u0UF>??@aiV( zd3O$pRcwNu&}yD|!89^|l=CsG4a3k%PyYAsoTte<4W6!4Wd?i>AYO#xkDh?{?v+Qvm0RNyg*i5C6;^=^T1~-wP16_6w?3p0G6fhfp^U@ z)Z|P$xnN_(9C!34m$NS8gUhEtJTHVEEVRMO_f@Etv4@MWZ0uu9H{H)AYAVl2MZZXVi~j-P5CEykC(tO(Wk6O6GKaiHF4nbj>^qzej~x5p{l~!He7L8QI<$a3@Bc`XnaMKJ8Cnf0%RH``Qv$ zu5X&DegzCRWzwlH^FX6l4H_1HfnWXc@FwLrNdGOtPc|Bu$jx$p+uP8o3)8`u;}FdG z=tUY|DN#W|SrYa6Dvk$Bki!M-%s<<$nw4ZI5+khf(>Tv2H zg@%8paJWmBbjAOKYOz*E@rpk!TW(Fu40P$TdmEu~SRcNA*+c@@D=}*(q+n{{6f6~y zgQx|zc-T*xR-Svp<~!X(iRrOSPNP4Ir2}};!H^jCiqb<}u4K1VzO`(^8ah&xfmS9q z^zG_mXbXCQWjT{@qStJC*EALDd&KDzeN9@cWlp~|88G*a=aU2e_Tys`2WoFQM2DT zV!`cEc0Yo?ITeuc@e*{7Ct#D78@azW3qx-kljr~6p?tc|?X3h+=#CCCnI1~?;wO@y z=l8MBT@z_oR5k4Tq68WwjFukw#k-Q00}7_gsQN1}YI<82u6Uloi%X{xjqQu++NZmi z;Qkb9#17)v=p16Znd_YtU*K5{6rojlrQSKP* zCXRIdDq&PRag?zk=7hbcObrh#1g*c>*jKg-+q^ix_f2)$*jd-K!euI5bF7kiJ6{wv zzt_T@0t@nT{ch-JtzpCas$oLX5r`~MA$vDIVXq$GQTyF%Nyi;^+SxY1RvcSJ@27^6 z(sgIqEA{gDbsxvpJ1I!FIBq7l=9s|K(IuqyNERJe@`dgr+ljjT4yJyF567SKC!J1{ zY1oM|c6h!NJv=Nz{Wm4^szT-I+Rz!K;YcJ&`D{naZo3c;4#7Ngmm(^^IF@QSmmy?Q zuva$)V((=`6bCU-^Z$l*l^QsybrRCsIf(9_KR9Vz(UGgW30N*b~No*&P8>iO$=J^uX?wR5MJ5^lx4d zb;AcB$03=->K+1|$Fi&5C6Te-@8Ih2hqo(E8P73Q@LpM;B!2E;bd`p{(ReEHF;zri zpCP6#ZZ`cQy@A&DFD7-fby0NWXc^+_GEo6Y85lw|164+B$SyLuKRX8 zb~dRgCp2+hAr2%Ju|?kuiN$~*DZHydj?B|1Z5Ku8q4hZ!tssowOLQQv@d6kwI*pQR z7hw8z9lX*_!RPf=Eb9BtNdB3@zgR0sz6x^A^SK^y+TkJm&N$674JFJ*D{Jy<<1Ai- zi#pwYW)E$bO~?5shB#kfE3;+;$7lHO3hQ-!IqT^Cku6@~OEUuJ!vGZ^qBDd*^RzBW zE!QL9r9qyy$%79s7mGvd*^Y<*;i%(N5V&~=m2@Onjc9H5y3ZUkmO)^iqaQZ3x#IM? zlkBlC%Jfr}CE3n77hVMz(h~iMug#? z;HSK=fNLLja#@X5)ZFGtVrIX=oyF5=O78+Q#o$Wlb&_D`HW*x`;0YiG4^RD`8t)&-BhOl~;38|Of4d(=G;L|G> z9&|2Z48A+k1hGU$Y^gJIetHC94AbG{DRw9Xex99*YEfV+kBHr&u)7FzBD}PDAO5aO#?<2)7m}SbV|f=^u6>EGP|>w zd9R8Yl~WXFEDNJ&=W_4T&ATz_#tO8ZSP6c^nrQi@o9Q?cz_$Kju`9j|Bigq%t>%AZ zy?VR2-nlzT@0B1z(nrwZuQN00^K9~WlM&A)`w;Z{3NxK+^Z-SH_I$A>i{{;7Voqce zohP?>@ykkCtK=GNebbMjF=|Om~Y8F z2TowT$V<+1>p^+T>zI$x)?mtcoF8rU!5GK2)W3ZrtbZIxVjcH_c)?!`$rdKzbt!c6 z!l5Rwy>H=M%x6$Lq{1YaiIWLmw!&LWRmNXnCW%X*&yP8_o|?MJP?HCe6tAjLFW1LB z;iCgkG+GCpnX{S0jsV|MB{AlSKb`Wyfm!PI3hMeSaY}Xtek<=rnRD;Z%RL8LL=n<% zbi%}@UVLIF!9Ef?jk~dh+oAVjW_uFd^jUxmcK<;0;x3GhoP@5EoN@4i3G;FM6|_HJ zOkQ89hehin$ttZoAPmRw+}t8CiCc#2#%0K~GF?y`o~)HFTZI!_f8rV?*qb|t05)9IZZpZI*8Uc8fLMNd{3fluZKkozM@RhER2fay6r*}Ob1r;|x^Cj{dX z%Ofz&Ze<}SVI;S& z1%2-?#+a|cMDu4AbKpc26@GsS(>b6D5`r;~#}r*=O!7Un$C-f4GudNGoESn*L&IY69(;Fp0jN z5>HyrwXhQ|#6wNPVT{R&BzwExLsm;E6CQL1{w==CqS!MM`5q9CI4~HJktHZk|dE$ux*19d{dcCafIW1&i=yopWyaM+`Ukz^&)O6 zGUONcsxXtDEv9+@6mW6&V=Q!9Ot%;az}F!LJ&zrR%R}d_bDtS9x1%i>jprWtw>S%n~e&yV|^7L0+kzM-bpEw=)Qe$ zxk?R(pq4F&%Oq~rKG3T22rkU|1e@N=k|o0X`O2H+h)7{3@irth;zT>tl*Hn8gQvV+ zQ)428zT|-3M*35|443VxXP>v9;adj0=gnJ_#GTjTspU!q7$9{p#ZZw*w!1(#y@X8? z&V=_;nrxjh46YY$VF}kc>+U(kOSq%Uxq0t{_M3-zAVY)8{5@gr3P_MT`$cr?h#CHV z9R^35R>S$PKcV5%Tw>j@hIJ9!3p#%+N&Y96r@BeeYAEFr8^7KO{u_IOBL6@|t{zP|#W|48DNdp_V*>Bsd&H2E^!-0W7GVAV1{ z93DCb9Y)JZ)2o}XV1*TPcFH3R*Ihz%=DEVkJLj0eTEJ259rWAn$!Kj=jvISanK<)n z(9Sv8LN|56S2Un57ZwxuyRvY7Xd#KI;Y@(9W$DhzYMA#kid{9pF}8(Y;<27=R1sOl z`6{@6f5|%3{e^H%GMsiO4&dy{8*o`?Ia6$BidS@`$O?rmQ2s~^p6YyutdL0fk@F60 zR@%`^N)w4(>_O(TX#xh#;9M};dtgq@O#A^c^svnWw9m_64L&@B%g!et?VuLjJK{nP zxrZ@pCb?kjz$p8)`X(+{P$pZBt|u3d^`d==58l)@W6%C_q+5Pw!3wh)fPLmv;A|;$ zP;KmV%ZC%*o6$^>+cTG^)13dL;E3`JvcFb@Bp>DYDbeESSsIJ`8WKs+%6Dw|oUd@U zPK%Pvet29QhDJAd;I9eHIg=zsQ!mDg1J?BUc_0N*W+c#a z2VMU1AiM{a3QnW`p}EwtAq(X$wlR@&w7H(<7IJ*+ z5B7|QCH|MLiBiRssq0+>kLy>{hsF2tafu#LdpZSH>8R4nA%z&D^ByO&X7p}y1H72+ z1LqcUF8H{a^zPTUjGLVf$CekMdB!WKcytArEW5{U-qniXDq=W`tf1*b^~ia_*_^=w zUhphIdZTDPJ#+UO{7^Nc$M_rrNhORdj}NA0b2%48qz3-LT2xHu_?+Wa?5*J*>z8s1 z_?hAh!Ej41erZ}vUs+W!n^XeHmJ607p;wHU9@M2fy{)--A0e{8%#ZcDD}z5q>}mHr z1#&l?J8QS-(5x6I=FpZ_w6ikeRVh1>Zu###+X)kiOIZNXo%x6{J12rvyAW@YN6=L! ziQm6WClUKx@es%C3l%paKd#N9A}=d3-7}KvZ{eH@V+f%|Q>n{JYktjT4@Otao2L9z zp?|Z(!QxaM$OT%`HjV8}@6!QD(bcER?UOJM?P<}P0gxEE#)L^5GR=1uk+`k%sED!( zsrpgRhCJF2HSx!BTBs)(_bCU9%4XcOE(#PpC(vv|0qXOa;SayPh{+FU(t^A0TxEL3Y{L~rM%t`=U3T~ zsp`fwb%G2$81sZ%>pZx9K?Ejb#juTnbBWVPAt>mYq1G8eNFENM%P-!;-x^b>)Z+w{ zthfm$zI^8xPTZ_UeKKAVQzg0y^Jp$JfDMB@zJ+%>v!<;G^SOJ(@7Y=8eGx^ya7{cu z@e)%!xebmz6CzuTI9J$YSMoW=io{BXqUE?A_1F?Y?yEH8)4(!3)h|h`y6&?N7AVsE zOmSH7fx!U3^Dz2S5Km}Ngz%VDa&nO#?)zbmN;6Nv`b`=PTM$Tl_BxON?yNI6UYYKh z!SP=rbwQ^8E_*}&1N<_bMwZ_fp`2}iO6DZf_4gI1;r4QVXr~~ZmN~?6*$%^PSyOUQ znQ+XVqj>p3CvL8=!Th(IaQ9m`!c$40=VY^St!a8wo5?Kldqp1>nr$LC+h)^GmrL0n?YGe&r`9Xx_ZnA{q17_rVvk?iH zl0_QFCgY-woRdu$sKts+?8WXyxG}2(PlT-j<(a`~bL=^+lX}A>FZjlY@7~BPb1Ox+ zvS_~7XeAUmAICS@fsj7b1NU!6pq^7Uemrm)ZhjZT#xIg@R~yZ z%p@)xYj5O+D(SPUW%J$Q!A3q2JnTftX{RL8lAla3x=)2jojt_(q7#{Ew*rPkKg0U{ zqQvT6Ghej3nw9-qiMCe@__B9Z>3_T&;vshl@`aV?IR_K^uEmQgmm(H#cV*_^?_zGr zCV<)rV@!R^dBo2taQ)&m=>8N%V>&Z%*u)3y6`x>ofFI3tSVI-U?xDeZ?t5soBe#q0 zz&q&;=#Wzl^UadT)paGjfaG;37%7a6(NP%RwFE|ndH66!jZBS+fLmslaA&UtSrqC? zqxViGjv+yeTjPnSd1x#3Ju;4=d-ec*uTG=qS%CevjFX@qT_kAF zXh=R{*HnoU?F1@dex)3nCRiICzR=7xtVw3OiL<9PGXfd!sW+>|t55O~ito z@MtH@Tjx)#xjBga_i5Cz^gVpZ_`|%|#yNau8Peab5=3OaFHP{TU=NkZk&-hUw^Do( z5w?E?*R{LYQ(=v8BP@Yk9^r{4uH9H(E64=&=)iwjL3H%Oap=;wAi+zW>Ag?gXzi*> z4|4v*u=sA~#|AYz_pS!neqWKu9zTzHe^hz{^0(s|jdlB_<_Iv7~71MX2)X2H2Kw>!YQw0Gzs z<^UJT9Z31GfL!cqvkqIi8vMMp@X(KSq)#v!R)#&mM^i>2_Pq?v5jG{Oe+m%~lU1zT zvlcei(T+G>zYY7`%fS5a96CkZm^{JY$1c;rAO% z{`(1&eplc>k4Sv0KaYIix)y~~=Qn-O)}iI!mojGrYEsYo+8R{OJgZ0JhmaiFIPZ*TsQwygAX$%!u1ek;@Kbe%gF}2ax5L=Vk2Q* zRJ$gIPEcqBzEw4>PR_-hEplvSXBodNX(qX5E`@$~DwxZT-=Il(3cdTE0w!{~*L8lL zY?ic3dd&HKtadWt-uE#=?h{DF>5_X!TYTt*Ug*`W+>7!Q5FlzG|2p__iUK`7x1GuVbX0|ntf;% z2D^LEpe8pu#EdJ2v6rPzCE9;C8ifVq1Wh@A3CW>`?3 z%xM%OL!T7rEZha}O~t6b-D0jYoK7~Gm4m>tmmKBo1{C{B2)V!$yY%%_`^Ga{#; zO-q>cmUXbLMdI{Yhzkwcx*JCu9BD-MHnO+XfG{;n2r_qL#UVW7aqNnApY8(kW`fj+stp{w|Z~fX&i^` zPHRc5m=))ZjG_kLo-y&y7jtvs8DyEwPY9`4M*6nwXO?&mLQuLM@fz8{PBJ6#x;~8E ze{m|@*|Z<>rmE8%^^?qpK5gQ;shV}LUPL@Z70LJCH*rW;4wjqp&}CIU9$Cq<6ZcfH zNri57dhb^7b)QL}OqfloE_cJ%f1-2;cfCQ@Aa-!9GOsgdFeP1-E;v?-{a$X^@@WAn zX^Em@GiKpQ!2pmnYh*8Ei4y8xZej53`G_kQn&lk+ZYmyei{c~~gKNoO`&mnbz?>6iFVH*Dd+0%(Ul<&O2H5W@V&uhZdUb>V^899wq z@2TLF5M#VCUW&8-r_3ImyG9$nH^I1<*J;hW5$tezJa6z|C+C}40#j3mV5^1~JpFWn z5})lQ&u@-6qU;4oh05WDySqX4KYy`uy)VDc5PmR;$*rYqt zpfgH??U%a8O9gL*mshB=uy4(Ef{L3`(+jSV#?7@cKz0DQb3ElQgbvUFe zbP4>|!Q)6B8lR8YA1UlR=W6h&AJg!FoC@V!Ov5P~2VqvQG<)?!8~>!`ldr1{?~bPU zZpUNTx5*B-m5rcQtvdcvL=+@NoPwt>jB)(8XVfty3VTKkV_tUMe4)z``0SpE`OOjH zcZKexFk~z$lt$5uz$;X{_5hR{OR>pKu40wEIT#b&%PT)nXI%0hlGyZyAJHg@byMr; z?)72x%hZouAEb-syUbyYv(R~JNu=M6+d%e20So;xjZ?9HPE!^RL#NEqtid`PEz@|u z>t_-FW7{w~qHah20_V_Y&~ut`{x<)rE{#g&TEO0rAy8X>9W?s4(-g4+OO6T!r-%0_ zrrZJMZ~6@LY!p%d=M4&~(r3ejUQ)EnHEz1oB5uYe7tqMCW|wgx$Ze2^#hLEt zVyp?{Kdj_W#8-plp=V@}wVCprC*xk9ULN@5$zD3Ttb$q@jl{(U15G zWOru(HVfy4MYd9G$-HNrIX*O}$C{rKHIL-3cfjtGL-FM#8|b7lgB~4&#|@%1Q9bc4l*aFJY?&h0vGJ#ll%+FlSem$S%yfK;EVU&|3Qf z3_hU)%UU6UjY#4yQI}5_>iAJ$-&8>Q`xk6MwmN21@@od_0(o}*=m@%%lSx(n>ge=q5DU=khb3PY@Q=#F zm{Fu6bl27JThN4EG%Ud4SHkZ6eJfx7cmS)_l&9D6@9Dz8x7236gt-p1$Ch_9nE%av zyp-&9SiZ^?zy9eWd&_;|ak3$tXVpjg(>)9q+_!gro~giAW!@Dn8WO^G8CtQ3F(zQ_ zy%^3*KLGVPZfGV_Wyb0^X^vzJ9u2a^AEx>A{p(_3etSr5IF0siJ1sW2r_P_Py;d{l zNI7jEGK)3U@CNiIo?Siv3R#EjI-XgIrqnTtL$u`wQBKAHZDS4D9+Yfl=XyV9dJRbU(z3 zN?ddK+O!B~gCPdYP^zC4e!nBbsn+;tj6S-Z3}wgqGPs7$0QOAC<*v&-3PYCG(3MnK zVNIsUA~nmoukB_ySLqluyGvl*i-pQ?OBBByZaMnUZS<;GLHbX_B-p z$doCf?VRDbeNBdFRJ|gLlYB*Y<`r=fB6X3YLOob-N@BMbKPR0FR(yisv>Di7D!%1% z8D>c9kbcWgr)lS!xau9&f-h1BRc0sR*pMvn37yKCG^Oc6j-TM-(SX2t$Rg~e+32@2 zI9BNRsP9}R&Zv7qhOavLhH*uZv%(AtbTXkrZ317BJ`YA}?1QVDmqTN%1M5wkzykcr z#KmvUP;rkE{rzGt`l_42X0%9Z@(9d+OuclTf-{){t4 z#pl+HUG~Bm99$%7mnl}r~bB$Xf9!n=g-V%%BowqkdoUp zGuao1zT6H~`-|w8ga->xHpUmNS!|i}efSgqi2gjT6ED7%O+$W9XO}fOh>Ez!&)<8O zuXA5SH(e*9dY&)aHPDBD5WN%9ze=FD!5{KGSVu~40cD>B(x{~Ce23i_ez@Os+Et*% z9)DTFOOIIhVF8^f5dBdc1z+4+4%+`01Ludu2mhQ;)`Vu->4 z(dLdwEI+M}^L!_=2Zt>%LNpPE=B>t_3`csma0)wk(wb=)BA#7kDDZSlxVwir*3zX$ z3L^(o%F8^sGh_iIeyN76JEi=KzzO_S$N4mSXD5FyvJtv>)^N*$2BCR~2{RXVLd!hO z*~YJ`V5A#B%MHiVc&!KgUBwo1Y!RN_Ll1jo-w zQuo6s=KWt6|9tl;zP?ppU8hav$1l<+E%8gf(Krkn=cRy;+B?u$6TyC7J6ViFqTA=fUP`AW~G)OsiIF1KLcI+xLY|4P26sjg!X5W(oe!M$zX4D>?--1- z$)n*XW>GoUBkW3s^5X)A;bS2m`hMzKw(|2zRvqF_V^571?#~Z(V^hPEPQ)D3s+# zO=r#ehv2GO46}Z8j}N?P0S1S%X|l2j8#wYR*nHBW{VmsEzj~(V>q$>gc|Vd8I$hzw zgA_U>^i#Ud1I}EaO0uFsY|^Ab?8%$aFld4yGi#A&+w2Cyg{3x(qR+q`$4XdnPzk0x zCX-C(X&COFhYG@ToSG&D(pLn2_`+Vexl#_pZ9K8*!(Xr#@_TujMb5?LW;k46twy}t zM(2YTqoTqrkeAn_wCpR85VwkJv{{S)y7rM%c?@g~{Y^20GT^i@cbzjIj7sAKw^r7C zY<5p&$%n@XJJ?9vQk^HR{J25XYWW-T8^fmNQ&ZUGw86qvEL^wEC z=)0PY#Ww-FxzOKk26uKI6tgQoAgiC#Nfs`j546DNKQV>Nx{r!X??1R|(8@sy;4;vAFGeJ}us3 z#C9eR!c|7n_&Ll6?Y=yR%@4oQ*W)4_BM}N_l5e2lK{^jb&ZnRiBS}e}9vj;H^0%9?ksYW!T>xGfCBZ zp1@TW4FT)Ai65Z_HZb`l;pF z6#Ug|%wjG$fK9~|wzRo{|JFDadY&#L$lvH3Js_3~UmS?j_xs{y;qK%^NGfY3u`uI} zBQL*L=*#be@oXiisUxlSGh-f`MeM3-F7_sGhPHK+A^Fh)Ui0q(I{(2L7b!b`O(!BwL zcDP{AA3gkU(k|}ZQ4{*)^qj7puxG1fQb8km4S&8gmEF9~bH5jM!*Iv74!zFn}O zxG9M9&!iHcu!|};_@j!?0q6;T17(J3Y<6k{dfgj|e#0|C>vJ}xa&6EobOpXFkEKU- zlhJy+DrE@FV~wy>>PS>$2{)$E`^qF(UEN8~lP^PYuzz=|Xlys>FF$@jYhGjL<_6H`zjeKv|lOJGKAw&>Nka&QEFoMokfY2!*^%bz%2 zX5&l&g)9f}OJ&e*%opd39sfAjq)2qyIS#Lts8nHC3Xrsr@k1JR%F0vM-@m^iiDaI2Dt6+ z1u%471Z}rI!kw8BaD4v_UScia)$(63jh_OtiPMDV`+@4D<{$O z)#F&Bo(iUm7SQ%D!uN{GPq18i6Q+6B@Yd5;VbGOVurI!ddgDttpYCJu{Xh{vtFIp( zXiQQ&VwQJ)RPEt~kfVLbUBF`*yYHqdZtH&qTf zL-k9_Xpnm!WM}8mV{;x16b=%9dk}sYWlM(J$FqlX{rF!CH_>R5ZqR$G%mSlHEdB2` zS?b2(+f(}F)7Zz)i_xOGH&J+Ym=e{wDWl%|-&_Qx^FvoDGQaDWAxK|`g($Rf7g~;z z%p-Zo_Wucsj0WS(AJ@n}X9pdC!AxOKHZ@-E74;3M=MU;AGqK+(zH{yvA(trhc|zmG z^OT+07(FFiG+2wDxjT;6x}bt`a&cnUuhrb0?%UjfnmN=|Cx^ds^=Mx}AZk1_LixE} z&>N>nvls56nyC_`_b-`pM_q)C^9)lQ3<56^7KnjBk4mETR+FH0BUp8N!SUSbQE&v>xq`$C!7Noo8TrOR6W3?j$*2l-F; zhvBa>Wpv&eM&oY|z}n+N$6o#|w0Ki# zxX1?@EBDZ^QBusz>^$|gZU^NnJ7HApbiof2!AhoH;xB0~hsph!FvrgtkKUM$iAg)a zZ+S2Guwn;%(@6)XQ3F^{TOIdK=bor}OE5oBeG-L@vSfV2K>Q%4DjFde57T$mN#XE`W)5&(mZObxQ%5OaH1FoUo5-3@(aem;#@#7*8IG z&R8$%;qyxtQ~mG-6eS%^Ycwh;!odOOSKG2FyFA!#lpvoQd89sPFk3fmE^D&@ko-7> z)}=k7C7svlMKiQsQ+ieY<&T-iP+)|2ePg35)y zoBbmQ=~ZV7=FMXJ9NNKS{z?oRv6Tx*E~FU|K=TI0Q_vS3Oz+TSN`r^Oj2C+BTWK`* zCtV0-zA8h9j%Wu0rY>B+P%=frKRW&uBvDgrxjsBl7VBH!Iym$qF4kvdFq_0q7Rli zDzODmZSa2FO}Mng0SCH=3jVoSTtbr?u8?Y{$Qc2k6E+%CbAwpgsl`-$bqQB=x0Z{l z%@vtQjAU7>^kK>$3BE_I5axa)@Gd$<=ZlkY@P@C>eA8h3@nc6qTYc7j`KZ##MGDl^0P^+vqfGqCEvas(PAX?~& z&7P-3VJcZnQnlT2sn+H4JJlCA;WPH1x_3x)wS|@LPA%nX(Py zJz5$tDSSU@H=GCM>yJs^Ish()yo0tg%w)@J?xSrg zi+*Q}TL0EC^K2XVIzAh!*Xy&l3kC?j*RiMuZfw9veVV*&1om!?Vty4>v;`_?;?0Q| z8C^mTqo#?nY=Tf>W)}PT>L-{DKfryjmLqkcugXSVsb+WhR`d&ukp{N(oLmrBRCQBN$?#BV@*fbBoGAnChcI&54@a zhd*MdOx#a4$4>IkrD{oUlpK?pe2~|5`~X+~e4<7vV`hG!h)VoyaA8m?%TSsICvRrM z?t6pS-kme>#U3@Ndfq`>PN(9Zf*QTK1q9d8At?^m;m(eEH^v?AlisnaYeeV9{M z03(;CqUo2)w{Kbz) zp)*?(9~z06mB0y57KBJkwn6 zitsL3#sqfG*|nHzdqbE9??KhGFQDNq0jmQS({# zq*e?O=|vPg>^Nlj|ACb{1x)_%8j55(Y*<@5-OTX8_T3tEE!P^_qxVBWfHRkor^ez> zd%~~_=`iE1lW>RF2yd=A(&F##T;CdfQO7(J;?lx#zzs_{+mS+X!aPvzZb=8f1~KJt z`TP{p#%}jEzHhrG-sl(#rsD#+5z;qkWX>37a^oGu80mrbEuIo%m2gVUVNQGeb}}(I zB2K!YLoY+^5XU~Gm$CJ*VVU5svCrfWT+WA$n~u|x_oHZ0Rx|7^&*V!I=hNT6B5rR* zKHvIrHc4g=gVsB z^FGo&pkDn5l9OMF-^c=H>?DAPAXYg?GGEAZajzXCq{3so=M8 zH~YxxJ>hvT*A^~Y$nqtLqPYu)RLRrSj%?)-lAkYu&=MmyR(3Jxx$_p?ktw3`_Sc;8 zx2;^m(mwL!TA(H50zBU_Q0!%pChY2@aZ~q9(P*{1+^AYNF3qWtu6@}9nm#(zcG3@8 z-v_}v*FX%>?&H0pZJ1)qcz)*ANH%h020!4WJ{Ik8WzoTfyv4tJQ2g{Eefw`Ash_ZA zJy#jjywJpqWPdh(wJW=`YXu)%_JIx>h45$gFXvj0Te0a0f-kpVXf6BD4fj+I!NL`D znWn!gjq7#B>bqqG>iftJvq-=4E!VKS4idM{VWUjrm}0^}hB2wIt~7^c4SfgegzjzH zK{@V%<766AWQ+wLMzU=Ka>;Pl1}vODOyE!|k@W9qT5wneKb+4ME&I3;LT(F;oXy8- zB`4?d)oaUOj;lOQNE32LEJNJvJ%D}bHfD?cG^xyQDoYrcO0|)mP`IYnd7l4S@cGh0 zm&b)bmt6rHljOllKW*pc3eJM~dFt#%gy4ZmoeQQx3uvP0ZPL1Um>lcYveC~hczYp# zYWw>D9W9jS=Z+8rMiScW^s_3;i&tl5AIGx6tH-l~Ps(igWk+_neK*+c)S-)S8~K|a zmiT?cZ{AJyKL6o#EPLoNla|=3;^IAj#XGLIlc`r8%Y4-g0j~`4cCjC>uJwaB_aT^J zlF8e8J>tbNYOEvc1Eh9^Q0IzN@y+IUpc?PbSyv{aSagtkS*3=Ne%r;L7Yrr8Pl2pA zc!|)--ozFxTtv512Q&A)0Bm^L4!TthqWi(EWcT$K`8esYb}PY0xcvs~R_%m)W+mLg z?s_VfHNw43Rxrb;fL+uw5lIAhbB%*GbE^{k8FwNH)>^&cV?QY{jct{p)O$76xC+iLk#zU>TUeB*j3utYApI>0rCi+j)DKxC{ZbLc zmo-5qtdks;r}95#FLOt02Xd3Rn>4TIJ+0fDK_&+aM4F={sp6fGy-47M{l5`e={T_S ziHfwdV>c9+C8NR58u}hK0f*R%p|@c$-rGJJZ~O^ma_X;0KJ5&5K$x$kbgkI}uZ>td zLl!GP4aM8~^H68D6I*oPIXCl>BA;m#M)SoDu#*4I=?%-L`7`&E?!x)3ZcR7+I{2LX zJ2(Wgr&=?IA5Uq*Tfs*(_5x(YNMOCR8m^JetCbMC)zvrS*tn8w5WRjKjgYj&H$twx zPv;K(mpK{U{W(oqu9`G(=t5Sxrx$G9?vnYtP~^|7z_!vHycyt#{b!xn$#?3QG&z*L zfA^lxI(rpL53LvSeeUdI{bVfPn$3GWj^bv9HuGE3o59fZ4}Cdij%fkreEZb1wAWV~ z+@nuYYx`U_vg|GCtg^+m*7wBr$2AyJ8p)P;E`$FrzU0FA8s4fkooU?Ya=IXJBL=9b zh}$btM9ONzL`SmBdE<@C@y*W|`t&aXu6!$pmYbQi6QjhmB>fJJknV%cIo`Zx<6!35 zV2OJTRx;@~SExiUNBBDhA@@F--KAihe9MAP1nr{0;j#Gb?l%5KY%YJwc_|)nS7Wad z(%Ig#JE7T8h7>yI(dQ;D8hh+Ax9e!DNMf@z`}Dz@jvfr=G}j!X@MB`GY>ff>SE-;) zrkLDUMbaa3#^eA2NiYX_M&C9@S{CgVC^2FJvSxJ>w6rykja zyIaDbpqayp8unLc6)zB7;%^;Wd1UfF> zhZljfz$-_AZJAR~#w%;6Y`Zf1+#R0;O5BbW_U!A5QZ72?8C2~QJT$LT(Di}PY4kh|osPpqFE*Ca)!BzB@OU<9ms+Fx zHbZ8!Usm{RNa2&9JLG47g$yRfF?)f#u{y*W{MR^wdS4HBea;d1KEo2PbuMMAer3WX zqZqg*=Pw@nU_aOQ=mdY~Y7rMzJs7@yYz4MY9^;$i@Y9BueClQ|T;&$SZ(X$%Cd`bX z^0@)1t5(NJ#Rs8j;sL=!mO!P(#@=z`f*PRFZ<*sp= zOApYGM@3u(b71b4(L&!T2rlFng8Z7ly&4Uo={J?wnHfgx`O*Z+IIf09k|S}*j8aG!ynT1Fdf@s7O>u;WDNEs} z(Pm%9ZMPhR&1dhEjlyH<5D#YOclxuLf8=oZMmu(nuROOTd=B2t&EpR*;o+vwOLG0z z&OP2S8_l|gvVPmmure_K6$GENcB>_7IE@j`n$zgoN?$g5>u3y~p^AM$mSSm(CAh4p z1HRs^qyg{8fULX)$FI#5?|;9Zg04mJy3swfYmYEDEIdzI`#*xR;BIu)pTWPF7R+=z zk3ib2E^xhb7l!l}ItTxJ2Q*5OJAPE2b+zo`#{V>Pu9$d)Yf^c~8O3@#*ECILMsB=# z@ytrPQgI3H*Azj=87J83p}^kjh47OKWzl?-8YZ6<(?hdmpzBmid(Y>v>#c{tcXK+r zygdPuW|@>(lok#1afuyP{-c}UGH|-I#884d=Qb#iaMwnVGO=R{2KCyuC_|D#uuivxaliRme@l#Qi8DKrKQ-@<0O}E zd>t-1yP(6$o78f$9-2lr!*-Q)au_Iy9tRNH)D78?XZJ*B%lDJgLkZe8Sf0%(TgJ9p zyoLGS4uT*yJP#n0jQAzz;&z^Wz16p*Ie0_(}st=Zhl+N6n9!ifn+! zPwE}>0`i`>aMQLu;*^uz(7z~6)d*9;K_EHZsHPzexriV0xlpdfjKHQ39P3O=>KH|Nk{T= zn8_m=7^uiXUk_%!>Vuilm`)mZKay6HyZGl0C0td&IMvgUY)9qor`{8pqlEQ{)tdUFRjiEuS5IfC3{G(0H{p`AYub zV^e?gul8+V|DI^F`-x7>Z7RWmx$3E}V zfD(l?K5Mlp>!_M5-s=!ZM_z}Z!&52dd(W5El;=T#5#t@ZD#rIaor4>@ufT5-mzOHJtmsDhU4d&tSLmkJDdCGv;N{ zPWStcQs+q-*4!hU9io~!&HZYyV}~7gml?6t@$z8ozl0=~tz*FwvUu%+BU2Q-mVfS8 zFoU_W;9I7F3*9Bykd$#$o-vJiTp7bTZkkR%g#79pmDeQgzl1$;GD6>sGQMYf8)UUh z;FEuo*l!sjBY!!GIrn|7jg3nOdHw@M4lgEG^J(m^^%77ns;5A+{S>)R3OD(_f#VXE zWRtE55qZ or3JXeO`M_Dt)?Z&<1)f$iHAvHQ+H2u+Cqz0TF3x;_;~ zE%*->3Nyg7%ld4^wlNsk>x&N-3}Uf)x%fjhoc4Hap^g+|b}FKiwyqqFDo=Ut*}6kC zAfo~rG6vvs$#b0B#|U;x^hmtv;8w6|-T=n~Ghp27FgT;<#g?oq6!&W$r_oj;g!2kd zE~}havdVR?El~$MZXc$CpIfLlXcG(Cslhh>%fZl?I9fX4C=C3Y2-Bx&Vd=cD(0uVL zWk)&TB-w4;{Gk%~?d&jScus-Ul!nsAT~kTr-@tR1Bb#XaIS2L$J1Ny;C~mpt0=|nb zLE*T&T=^jj_V(kk?n|~=F!wf6`n9)TKAB;LXgP*@?0(&xT;j2F>6Y54f-Qt=$AHO$cU)Y8D_3pU1G}ncVC7V0 z(2N;GlHs{9d;1IcHp+`+mbb#Ir`yE6R~_)^fk7-L`XVsJRZu?oAL*@Dz!mY@FjC40 zPxXYbU?n5Gd9sLB%cQeOF}0MImLbC+PQ3ymKUGbHwg{8C!pU57eTIdlcWZCp=F5- zg*fxT51C8-5kk;^mozp6%w@e9ZtR0?4<90QbY56qAT_UGvi$Oy|JvYzu@eN}#==I@ zD%&f1)#-xe6NJvfC22T0vXf6e>j+H}1F&sZxRCu*fnh!VIAX3W^Jo5WywMzm1Q_(( zH(%?icuBhH)ZBQa+^Z*#gcRGFj!mv252}HFO(@ zg8R7&&Kh=;dAcReeLey9HV2TD_e&73b;XU(zCzdrXN+FdMC(pIgA+p@^4D?>lBOvm zqhu+z;dB|ujO(MYoZ)=xgiD;+e=685_&sk~Rq_9v+)=0FD`fco;7@$upl|6hegMvZ z8A$`!t+Bo=!YGT6s4SpASsCw3e{2y?8Y=-Z`zEoH$T4EMqw_&=qqK0(GLV%lNaY~t2zkmVQR3>E0%KRm zq3&M+HxF+oQ~S@{zqtjNf6$n@4ebT{`W71fd?CN4dns#C$buaoJK&PZ6nb=cJd0O3 z18-IqLhUXYY_6;Ti-P&wa*KzY-@734f+X6-+~J%4Ua{!RIcj4xs3>hpBGOK@-; z&4RjzFJQ~PjnJO&!CYh~v(jx7$mES2E10Lqe5O{2QXaVAn$ziYr%;Z44@dzG8+ZD_ zuL3ReYW{>)K8bFHvzj?SVbC5`c4m(qjSQ*+iLGB@K5q^@i^DPEd78tIo2=4MPrYPR(Y*~0J{;)Y{h+uw4aY33XYMt&1$-u zuFjTvPDRmw8`$7IgPB^_cCKR1d_Gz82Ka53rEwP{ndHw)H19*V$i}Cg#6iC~cBP(F zPq^W?E6SMmAdczEmDBY7uLQ@$M!fp!5|`s7xap7op>J(roNmTd@$AS1G9B&-Bg!M7 z{he&q+QoXd8bZx`)eI*cvoJ|dp|%$vK}=fWzzj>BHX0pjvg znylUQ9arQTD)?gyp<$qg*n0OHTBiCK?wooCNw?$(Um4)@>#FC<1gC0cVF|tdo6kNi z$N`s~gYn?+ZlU95!&V7%<1IaF>ap2Ei7LB5TQ3YhUfV+wS0uSJ=hl&9cM99_I*08F zZl>G+wehCCVPLN*u&4uKFyCPV6~A=CS664SO$oj*t6%szu@mTE^A6x_|Iy37%XGx0 zhu4kd!QK8bcd{TCm_{u2k3CPn^-Un8qlK;q$ARR#Ji+_c4Hj<(vKXy_+=6K>6dpVW z4!K?j8P8U6nEIbuw|6nz#&cI-oq+@A=FmaKv#Y3I=MCA`0sA%DiA9C!p>6jUYKYeY zr|>>_9yuD9pZkyAq*lW1r=uu)(|C}1V~)Rq2e3j@HKk_mpn_z zrD$CvlYDi4%HMoSzSv0q)>3FF@K-iT+QQ6lcj&~}Lv;H4AU3rY1b0vbjPzJ6%#_bT zwC*$~`)n~P4WC9vi%!y4&*!u)vW&VSIau_05z8~q6~Bq?7d=})8P<&b!CS`1v+b3B zWWN8WD94cp+m!-$<*`0ntk)o((C&mOuO0BOiasa_pS_Qk2Ou;#l@*1juvnG-+@tp) z@bu16n%W_}fS*3%EqXTKrH{gQ;j0wbmo}5iU3xiH^nf6btze|{x7O>_8a5UFabbVw zz>_m)snn}d9Hnhf?HiGPj=4>BRk=8E+)$?Ll#jYOBbm%$qOj#r_^XL%ifjpa_(O@yGi`G8K3WMCmvD2Cmr+l!2czV52% zRzwCD)~*RQgFCo82_HcGM3r?-5L_~0Wi->Q0QQ`k$P$;tfY&!8G_r^l`~`*8AITW^SdA(G7QX)G3Xoaa^j zHJO6#Bud;B3R^QJF-7eW=UTB90)12Q{FiZTq+dR5I(mib`(;p4H3RH!*rK+?TMn3Uj^Sp%10>Glc(@&mGFMv2#g$IN}j^G>f&#~ zu{%;*@Iu^l&WgUvmtJ~F3iSeG$y65KKAsMXUG~wKOTD5J?@C&Gfzw;!2WGk)v?*N(p zIm~Osr}6P$x*<%+dm0wp;IgG3)2|7DUmTy&q0h@`u(bhm99RX-@7~da(SF#acZPdb zumyZ4tpHcewV*m@18jX;PTy(-UctNB0%v-+JkZZGD_-M0W+8O(D#97sdt(clL!zv0v*Zta?E=;`Jfu&_P5i~Atj_z zyBgGcL~wWPddL;HI%bw*P{pE$6Hi*kDn1NDVfV%-o*7CPk{9yw(`T}__Zvy#V5c~; z@h7+9pbH+mT1i@ig*%qo>HL%Y7yMfLa@aN}l3ev<5gtrN<=qPCel-|Pa+ZOCqY?fZ zAaE)|LivaVfoKqa3id~~@gF^Nx#vyC;90XT>&*5dz2#Qq@#+Kx++WE;jvR-oBoDk2 z@{tZE8R3du$~0EGkE(5*;YE!Y9(oSs7X9vr0|Mwsrbf)O}hD<&@fmOW95^LmUaqiy* zUi>v*ypS(TgGS{uEB$JI=QRy__BnzkFWE(3(*5xaeWhdR`lxy-j(Pj&fd0Sjn%7_RYkjA57VtoJ-JpNf8%$kA<=!=fziB)z~71Z30iI8rrqfAjZm` zHr#WdCS!T7akUvQD==>_Zc(JQ8HxC3&lDyyR%OF-RN1~s!W~FuGKMYhp}ILADCOo( z;tT9p`h;$(Z(q--Se@bmmGExj09-3J!O!0{S?U#iJf0_Db83l8IQ2!9oTx0c@J_^A&;-?f}G^wMD4&JSn*j-H0E&o&9Hm3{D} zOh&kKAH}B6NM#8UZ|Lo-;VjCp31seZOuFwm4Be5!pXm~MOkhFI4kKW@#vmMHn@lg{ zW`oLsvF!FJeSDiW2puIYx!1w_!RE6Reztr_%5w#4qly9BJI4b~)-3>Kt-Y{XBNhLC zZ6(>@XK?7ECRAypqr}r*?nc8=2wEj%A5xQCfx9fVkQ0-mcHkZy3!RSp4Fg8y(0!@BvsYh+^D*bymDm6-9AP;s@ItSYq6M zF16|?mmvKT6auHg;y!g|Q&q|@$-B-wWGHceb0?tPRDogiMICleSHLjAU+3rd1r*Ir z!KkE8nsIRuX;viQ?SfyTzty{F(Yt!kSGS>--ww=bSrKKicKC9J6peg) zm>)msG#AK*P?3oPT{9@6%VU>ORLdrM-L{4E-x9)BS?%GO`qkQ>cD6Y0(J=l$3tcuh z*9zR`#tPlo-Js505?{M}By%%z-mtJOa{@dQsq8N9qY|dZ@YEr?l z?kKpF9iY|IrlDlRAS?{XVc$zX^Zb`X@Y{0;=7eanQSWlG-W<{U65_VVcJjHH!Jj%b zlPTRvhxf*>;ZE!`7`baS|L?vE^A4TKJTsKJGBX)?aCQnZMQ^I!v5g*AFQtp8h3s6Q z8QZfk9A_tmvgB`SOn-GA=t-1ASgjYN{XPdnO9#`6pPdx<ueVJ`xyB*sj*2Ck?g^L0eD^A6BFv< z*!*yBH2V7ld@kFP{hn6tQ{+PY>$HJ9gHtfusE_EqNN^trKJnVaoZi1n&>x%$9m9iI zxZ^)~deEp3lC$&4lT1i)h6PT|A#jl(I?ay45V=Tl4mbZ%&Y6uiN#Y zK01#EEcM}DELg?o-aG;APl4vQEMsc-$77GAOeN)N0S_yd7lM_m`S zRPH9;C7C4+@I>X!3G`6-+*e;*Bo29(3y!-b*#WgC8Y1KX?|(6bvvPv_>~I(@h^~h{ z52GOG?oXPwekuF#Ba7_ZF7XzYcBr@hFsOaLKuK##Xm4s4Z~E#D9jtPM%JsJ78I;P0 zHaX!J{YF~$bRFFv^$Dau=CX;mM$(#6CQ2AOfj5-g3p=};`2U2A+;kmHwpB6+=lu=D z;pKC1^D0Zck~E#kjuCc-Z-!$^mJXgcq|e&?tGKaEmIC7dsiKs_pV8Mq{i7ahd+I>- zcEWrj|AD_*9L}l>E4iQB&kD|*?V_P$N3cKV&1tGmGiP|ejN*iE)0YCrGR7(ne2?7V zdT;x%ae`khAzuN{2lL`XT{E!M-iIBCRif$b+IZPCm7kz_lv{LK4&|e-@weA%quMl% zk1#U97LPFW_k0HdU&|?P^(nf&@FZ=Y{Fc@<6@&VewW#$=l0_uRGB#ZW%X0mBuC|um z+qlE5gJq&G3c)1zIFFC<3x$rZcz#IHIeI-pcwcy^vbr#9*yB8l1-b6zUY!5Riz_`z z#<2oEDShMRFZ(&ms)fVlr?;T*f$%PWDCS(-p78drM4CAQkvHQUHOIFNONB> zbF8GdiN$~w?X=9mhWl1BkStX+ng1+loOwK&yDSw*Yc45c)tffR)#KtOYdo|v z9bUvXLB=fZzMET(x1JWm#FVwPHP;<(ifZ$FKmK8unsX3(Y!w|?o=JwvGU?*hX1ti4 zLY~Eqpw{U&804j}17k@rvwb!>@rTPT_x(hXt>xG^r5U~X26#5*7k|#F2fIb(Q|1*<1~J2n_#%ehKZFm} zxcr;x5~$Mrj5@_3MC`%+>6U>C6MX z$>+Ts$AVY3OuY0Uyem0{3jHqNq!CE2DCR)1QU+wzn33{PMY_T|gX8@Uv5Qvh#0_PZ zRJL9WRW)5{Ti!9&dLhS`4U~YeC8M}CDjv*DFT<{*y|^L#F(ecU;?QAfh<5H{dt8>F zYHAmz*JRP-Jcf20uEh{>adK7TG<-d<2pon2NO)}yh#!-sn>fCY_zVM5u*QvP=5}sO zs5ag^Do87MUQh;m89nhEkjt^N5`-vvz4s&QOjppVa}QzH0tfP-XDPZT+=4Ep0W?1{ zm39cokf+fb=#@!%WK>`X1vbwl!!keF%|CU3K4}7t$7vXSyAzg<4&#+kcak-jiZ09q ze3}^n`ELkeBlckj9>B^KBD5f78r}8XldMo!&2*hxL9G1@N%rAlJUO?NZ5;j8^npGG zN6$Xkk|0WRUOC{}EsA{mICc86^8@>^N}bV`aU-t<^zo*?3%kc;KJTpHSDvf%L)hW; z1HuIJ;DcxoX$|{;i^F(y`RxT%c&0L$Cu~A^9$FYF%=IZxUFNc0+%76ilxcD2yyrWN zkdYCCe;-7Njg|~OSENEi_Ku>mt2U$FevQfdP{g0xya!*S9%-z90OnSiO!rJ}2-!9Y zDOE+FFUe8xU-)K_(U$(vO?I;#0*E)X204aXYS8 z>AZ@G^yX0#okVBroMa$9l`amCfs#kpFr&#G2Mq1uVwoj5-?^5mN#|k2@fUbxsUFE! zOylNd)^wKKeOS9jl|0F-hyME)nD+l>z=b;+1a)e#HB6g)TQHB#Q4xZLvsCfnp1@4N{CH}X5aCIiR<(Pa%Di0N#mG5t`EF$nolX#MhtUViW*+{^*iW2 z!eybSR>QZ~hqyh^EU>83hv&6*TvAwuJbY|Sx(uaYR%sGsuWN$gO-eM#KahNa99XWo zow%&zQAc|fYU5st3$_!3GDat)V6z}jY-`X)tW+@QI`D55$u1k`?YLG=o_Ve_O zQke3Su~4nQ1_%8f;iM~aWWV+}7Pi@vLfPFcTwbK3GdjUcer{$f#ihDA?n;cB5BeDoTFm`H&pGI*zi>tRpdc^OYY*pm^%&R z+}Xtj_t>e!CU7cH5L-G%*v%p>m^^a`E%E}{j)-XHatC*2Ty+AB{v=R`!;3i|!bN`d z^eVivV~p{B`Hi)nZ2>-e4BE zxEbW6@<)hu{)f*Tgvj&YVaEEc02$}zaXy7obVF%AfhzUtXg-w0c}Ia`NnmD-c| z1Ml$TJ59L!+mcwD&Y_2oTGFnt{p`?=dT2j^_^#OzR4y0eR^KH0yj+BKzZ1f9oXd?N z3+bB$582#n=NTVi9Xj!p8cEum1t$6<_-o=STJtWA_MSSxZ2oIUXGJ=YhLb0GDQ1)5 zsag~K3O&eNmz&ACG*eiMH8NDUTbY!1CXvQ6FILMjj4G^+pub~F8Btpu+W7AnBrSc6 z0UD=Sm#|{YV~xr3pF2s?RzXHO|016EoP(F{E~O`ZbfEc%B=Jmth7XiXsl3cp>^?h< zp8J-IQ`^75y?Y$9MR^&VaJtFU2VS5t!fMT_iF*S0^!l`UJh6GJ! z8yDMh`N?E>=iUn4@^R$)co&`x^=8(E9cDMxS=g}!EUxY@WGuBUsC<_#Rkr`l3r}9e zGzeRP*7}8{p{^fWcD!NZy<-615?n2=1Fw&cuzlA*qx7#5w(H;>4396tf^1a~SoaDZ zUc1h$6ugYHe=Z`1JEpR$D(mbb&npupV>z(a?F7eDYrt!h1S}Z6jhj=%!EMrAe6nO0 z8>A@%k9$*y%IFcubrQt%2T$QcyEy)Pt^^-qQc1NnWfngbrF93Qsrkz9Y|FQJx;)2; zc{0m`eC5Bum)@_DQEbGnM`zgCQ*z;)pbg4Sln1pM&TCxh0pm*_Fwr*a@%q6{kjTw` zEY{0{64xzrjX8@My9nJ0e_-W}=V&5*muVaAL|)=oHd8H%s&(+#d9~#*ZOMCBaH9hQ zYliKrmc{Y4#xn4WZY5-x_kiGGC910?Ov2wUgMvC?QeVClPK9>B#*z%O*2IgrCg))G zwq)3PF$j-!jnpBjUVX43>osl~NTB;> zh|_)Z_wzbGxS)Qm2MwiB`X4mk6E_|)?ig9>5J21Mm;CCnzS zZ|d+zkUm<@hs5JkAuZgIj<()Lso8(Ar)vrvh-<u!5Yd3Ge{af7VXvIc$dTJtF9coYRKKKojxcN`-1!p3ESb{En7Yxdx3XDP3I`TOB7x-*_1l#xZVneU zI+gl!4Bia-!GqUHAa}ABR^1dOL6?re=4aCQ`PFp%a^n$`qsF-?Ke%D%*}25KuLKp( zZXo_bZEX90ci1IcEosE8v-oM)6BC<$;{$<3@o^o@> z;Af{m*Z3PAdL=^cb^T%QG-%?XH~mmNdp_N(E=zM9&tS-fqju$!;$iFc=jnqvWH#}(MOF7zA zzn+xnhEtnUawN*I7gkzj5ev$|sqyz{+G0T``lmqk+Usz7Q5~q%Ol8(9b>g=fgUpQI zv2?G)Dn{LJ3mMUiW#1a8GjpEGQ!!>y`hUHAg(Z?8a{b9S>@ z6NRXd%_sggx3kP4nOBg@&G#i9rqNHU!|8)RmZYy%iM~*mq@~Zc6R&AMV0ec*&ROe< z>Npi2+_{U^8*ecF4>a)$o`u!x7ohKfJR)1@K-NFI%evSpFs-dSsez~w(LO55D=Fjd z>4!q;j&>(f@-3A*sWGs1VHj@8eZhH5+8FVYE(}sH0J+U;!A8`SoKA2cT7Gr#pYaOf zweu-Ve)@rxcJZY5bR>z__E+#Ox(VN?{{VvsEu0wgjF+&>kM?spf;Xn0_=-_tr02N> z9R@QRWTQeO?hmnzwz0I;V*wT2>PHU-rP1rH1%%h4ijEv3G5G=?HdYB@y#7n}sInF# z+V+L5v0Xqu|N4qUoqcw^#)%}+tOrJt{=npf1DG`M9B!9}5Up1_)GqE0gzn^+4h4_# z+Ky}(FCPP!)tcyDwe+<&`X4F07AdHz-VDPjI80WZy&!%wQ*$4SV z*C-dQx#!ry+q2lpr%*@IX@wZqPf(fbKtB>^z_J zv|Q&Wz7`>l9&PNN^FP5p$dxvq;qK-d*6h#{0!4qPQG@Jh^4gDW84Qk8TqyU6&=bq8LyC#V@;fH|9jG3ucd9N*alS~Do-CH+9X4Q^zI ze!p#+za`_5Cr6${Poal0GNESgKRC2Z7M6LL(3(B@G*yhj+cM(xOxsRSR7|4_#iC$M z{uEMKp~$x&)ohO+L)tT+fO@F~RFq$X@(n{6mLiOQ*KoO%if?Fp+7>mByvD>pD;zO@ zje@oWFZbe}dE!quM+?ly4pcYOptAfe1r15F?sfMeAEtr?3f!?O! zjKMN4-z>t-Jj!jEz%S*n_C*2H?!J{B@4d{l-_?SHJ9`)o?~W>K+A-oIp(%Owcue&g zV|b?*ozu<1cPxP&v(;p0O|q=sPW2|pix>kUIK}+S}=RAE4y$F$NtltPo^E! z1nV{#_O|V5n7U~qt-9C^?NWp6UdeNf$=}w{RniA>+*688*2=`O07B-xIgah6>6GzT zBMp6BF!}pJ>TS6fC@&5|w#y|fc?D1M+FZ-Qs{AGmc9~i7V?oT{#rclBo zbody;qv<4GAV2SI;5eHG(i|(kn)?F`8ru1G&a|IJzrX zXqA#9x`KP5{l8i8FesC$SS$jzjfK!PC4-i=iZVt$%iw|kIY{n8{BN%rHSvsPZpw0= z57#h~dPA4pc_W-2iK<84?f(wGOTq_yoddKERDh3PkMFdfrOy3d|W(AvA{19x~D-C3lb6 zt!Tag0dLx|Z)_6l-!uqW@8!tL$Z7PF{zqPsXeT@w*JBUNnL_j;5980l3XIvOhTnBw z@?)1v61{IFs3mqAg$Kv+=DsTCg^wiFsk(&u%{eH))S9N-O(4eESMk?A0StPQ4G-tG zV`X?WoBczK6n{Mnw?E6VX;WJmRXuy!$5W;A#We8T%JYm+)B!l1F@e~cv+&3HGxOmy zmtkv}Mxs>A$ejLkVrn=5jTMhzQPeqBan@=Y&z;3?wRb}1SwS#LFd~aMp3UvRTvW>v zBlgALz+5&5eSB}>9sf0;=q^Sgb~mtI`AW>S$a_rDD^FV85ka2wk72Ti06qFwnRary zOqt;*m}TKj>a^sD^8sCIal8UNM&7X9RY_o`SA@T=ck*HP2+OQBqVCCjJW zEH#a6Y}<^-ijQLUfmBlWMuM0VZbx&;mHG1NApHD$5oNBg-j1*scwVVwQczRvoXDLLJA6YO{1E6N4V!t5Iw#$pKOhOg1*O;iOt+4sPOhY zW03fb-*a^;Jw0Uafx0BkA%lAQ9DtQ<7km5PCU*Zb zIgXj`MlDY%k=lN1`e2_9srfJgZ^|gpubGd*YAp|%Vni{@+k^T%I?tM}k%P_)TExAu zmdq4y^Ee(Oh|=m z72e9}WsQ&I((3}*xP9j>T(1vsSolBiB+1}?ZZ*evzsi&480KpfHDN?ph#okkN`}o8 zseBKzt(BQXB6ADLuT3S>GR5h?x{1V7HjExGDFmr!>5K$?z?pHP3x*(Igv zS~ZTFt|vsY9E;&&i3)Z4na(=>e29K4UtnPE3O+yW0B>q^Gizq%O`Y4dSm(Yv)Dm!D z#QzP#RJFZKOSdvAaPz;tiW($s@pKXtwVHhyyM^{`QX+y+FXE5pGk9Jo8um7&0-sZ2 z-7>w#hGeay!)|&M^xK*iJ#Jy*=I9d%!3i{l>n;zh52u>#5pZkQUOZaKF$F!2k!$y&iJOxg@fZQx!wpWi&wIjzj1;kdxHG2qs?WCNq!vssYLMs2$1y(c z77EoIWwf|`>ilk1D!pnsuh};rHhWAYx}Tq6ikbuIZ8xNUvMuQYZme9PyN#`q-a^;O zIFL1q=ac`+l8MiYG-~*ZM{E+MX`1i@*4IOwZ1{4RnGyCB`;MAYmnBa@V?`HedX_PF zrg+e8$4)|CmlIX$dj&Gk~%_cwC4APmgP~h!EYnla!c6=F^0aPPj z_T`a-@D7LHaoznr@v!pB4w(P&IQu((3)DPVK(9V9=6G#uxgMJgDVgcY2Aj`d&PvIE z-eMJcq5L&!UJZup;{s&ii)a%5708XEcq=5ts6n)j>}Q`R_M-f3hHjqx8{|$E@UKoT!hWY~IN$IctG=U%S+GQt z*d-~@?^(h4JwFkq?`Vb%e-}~difyET>qROCo`=?5Pe33gjS;y#m3e(gjW~F@7)T}XFy2cv|c6&CN&V&xLsziP&L%Ov#eF~{*<2eK{H4l;X1fc0OC@25{7GZz)% z3g$mJ{@0&oC@dq+!@)GN^&X@oXETQPl0faUF;Vyw1{cHIno_ykd}X^Gs-h>o!mt*$1hOmh_P1TsB1KI3s^7kj9}d(b(0? zge3{mX<`AS*j)^t2+W3TQx*EH(}*edZ@`A`Kk%mR2*$4BK0_88(vE}9*!$<3@TA>cd_2>G%6+O!HQ_qAKO>|5-V9SUS$&~#{h9?o1y5hg~^1sR<-P9qN(lRAGXYy!bViCMHbHk((OQuTy3aWZ7rG9c&Fww9A zN1cVicFP6Kkud{k%)~fu=GP^tO{|YCp&wlT@MBKj<5~NO(r?^7&~1q@tqM4SORXBf z>+@V*^}PvX@Tm?&h3F95!}nqJluR@&m!O6FtjVgOVz$~!g5FMxC7V_rvnESZouI5cgU!^{Ag%G7 zyI_Ff&)pQ8`@p+QIukf(QsrO>b>i5UGO z?6nVCJVO6xdw_e5||GI zGbo%~PYn}AXt<=KBhr`Auvv>(A;?&-AoRwG0lR3lzTe(}{s>lv+TPog6mz$WLp z5LX&Vr;94ml&59<=1mV#a!`&fxo1v?{TEhHXBIc;R3c~N*d=)Ri!Z% zv$3uxn6~5;!orgQcy5C^sug_Z+yI`Wkz=mhO!6awS=uCo*-r8+>v3?+23!*zKqG_l zL8*E_R4>>8f8zIZF3mCgVy}tbT+Xed?=-5nM1$w0O4h<=B^wc`%QlAH0{N_DcJamU z&_4MeY%7%^zHT+>y3-y1*q?^D>-TZ{_KkSaXbuft5(-+o1W0~{9las1h3b)te2E?% zx=(o`3Q6@cn@nE89K!^9$}bNOY>=dnj0ET%&oVq}{sg>zq~O&7t{d{;5J)O=OlQ*$ z=DDp7Gn6b2U-!Jj`EOUyhOw8deJa84)mx~?zUidE-++wS<|2vX{Jwhn}uz1`Dki*gLU7mPVKLW z!@i&Yz*Sp{N-4L)EeknfSXPS}+u9*m=_-H9%9&1nJ#Rv3cm z@n>kWGRjV{IgOq@J)Q5^A4TquucM9iVR*`04OKZGhfw=0rc7}cKE5yy8?9U6-lE;G ztZbZp=dDYnJEzkGnOKr1;KXJ`BvQBM+`cCDC^K3sgD$^5FbyXMSfNm^lQYSIoc$?C z?D=;v&galw(K<&MLvR&(O5#^nn~RH@PL9P;OhGo0-xvFn|CANIGJ5YTg{J_R@M z#q67`!M;N<#N8=EANIndPl@O~v=3l<4Dnp=0_xto>9=c^^rwa~(dy&+C=Juen$kn? zDCQv?N&Co{JDK8`bPXG-zJjLiNh0|@pD<;G1Zb|aLu=!9_F-l!P0F^X2i#V%M$W(R zvi)SJ&T3&V9g^&qSLj%5ooA#1TSDpsbFuF8v8c{z&C zbo#_oPd*M)4-;x1JQL>bvcyI9pJD%uaTqj{#ye*&W5IlB5}5DLKODakrRFX}QSq6q z$nC9MM%{{3|MTQ8;%4YROSaMC`Ic0S?P0E+bHLQNB%(gJoXWDlnfF@}q^y%LPX85_ z+#f@~VtxL?P!pOXCQds3NYEP>tm$Ac$K>uVXT<_bQSQkoih8fX+@v8eC>4bW`yBG{ z*;U@dc|{;NKL94bF2_%&d9b8OoV3j>$HH}%b{3z1LE@+$2zW^_NuDWq+De6-j-A3E zj=aK;bpl%4{frR|;angx^Y{|CR+3vIMNPqW&CuE2#V*#DB0Uaw67gNiknLSRVP$VRh6Oj`O4b`hyNfYY=?!e0HW~O_7x{?l zGOWIHnK5X#BZnkZNz6W5$T*)wr`f&4w{Nx*FONuyBEA^=_$t$!XTdo|+(AChk#(7M z6eD(aqW=*VJ9dk+r#EwVmxO1`jm;wT$-6n^w8wm6I@AVrp~j@%umv7}G=->0d8+rO z14Sj?p}~1Rnn~(#+1^ezFEa}r_i}T>ayNRfa|V}b+>MLxRWi$-+LO`E(ezWx6Qn-k zaI9wy$5q&Y&P^XdV$vG$4*H0F--Ss1FDu-YHHAFAHI>ecnMBjyI+I7hRX#k zm>0_?k#74@E)!D@abtG$pu;JcWs*wr`?5*B|0IaIdx~+YJcfQJ)o|DIY-m;MV@oyF z>A)msFfHze>k76c?$SnlZOM5FJsfGqcmOKh-VU~Z7SbQ<-oat#^Sld(ZD7j-1?ui3 zOyW`m$coluSV&BX{>gl1;mKA^S~-N51SN_6?lbsG#D;KsY-%mEkhMH?3g4fZ4sG(Y z>9%F+#CQ5H{Pn&G`{rk`w>Zy5e|IEl;kX^T=c?FhGcP#$gWJ7$tI!)?Wmp5lI#8(C zNxt~nf;|5j_@i7W%Vx>ajJrwf8U6t@Y7wUivw;g*mB z?sj~OPG_vhyS`io5s-iyK)RXB!D8jU}1491)05pkJMa7gSl*U9dMao<5G{c#Q@ zUvdoA^mOK5aTqo6;v6)CuVL@vZ)h3ThBo8ZIL0c1q)Q@no;w9|YzpYT$C03kharNS zJuRJ_#1!~`W8+(NSyP2v)=&(&|KlcP>*pNyS!f0=d29^VbA9-YMS^z0=l;VxtsBtQ zd_R|65+&|~#W0ZN$I$xA_%AbzR?V@*os!>i!ATP`y67xEk9i4Y#_t)6(7W*K`v16I z-m=sB?8xxh!!S^7!W1l;izoX!(E7MKZG4qR!-j0wgom%0JMZd2f8U$!_@#Xb zV?N6XQT{H+hL}&E%n@X-sON&@=ZSE`r4hz5(n-KBb^b1)>2&s;Mm9EK9mKQ3l=Eqz zMU4r$_(dK@%9W|@^>+NTb06BS=)|+_LPY(l1vxaQtEuHQmJtzE5F)unT%0)T7a9gti`g&&~|LgL27=RNc>s97q+S%^O~T zjzKk}{W*geyyUv&XWaO@Ph!}E*1PG|vI7`wp-7kLDv@xReu&w5ot^8<^<(~JK=G1b znAPe=?vKsGFbGAB7s~+R#X!-E;xC`&cuKSdI|fC_wPz}H^v@(B{&bVS7WLxRL^AcI?n`}m1JkkhjeG3P<9(ih)AhR<11#0&QP}3n*&K+q( zb4_e$2j2^|Z5!eI41hglX{h5a4rQ18Xm(Hp@mnWOf;D&Jsvq0v`gJqF`#D1%MmjPt zFO`AqZ55{P{xp8&8AV2TXcKfzFs2LAxxRFcHl4nx4in5K(0$LRprfWU$PVs8Bav<_ zyW0TALcFN;$OKF->tY0Q4|5*9WO_&R2PC%*;7dVaHq0rX7>HNFkgNm^e`?M)*sce? z#uKnh%?#d(N^xB^ePYNQF%^Bqny`Dj74^27 zK#tZ3!6VLh;i+MUJD((j@$gSHlQO35v+c;`@`IqgS%Ns|_Q5~Siy(VYrf>`DPS|x znu>EwI)l5mEAn)|D0e_^@140M*n0SnvmmUB=i&Tg|u7ZiST*lXAl+@r1kAr_$FU6 zNJM}QE0~J7T_YdrUiyOHw?uYm$`;n+VLI92UCX409)_rUal}g_lU$tI&0G6P8=|bU z+2Q&)_E9Nw#}Z4gBIyr;j$>ht$aJbXrgsJG7vYz35tp z%8ysj&iY++&%0N^TR0QO;!d%7E&mve1UbefErZdK45NE0``~A4B3w1R3$w4qplU-W zQ^GMdYB>+<0wPHdoZE{^>1(j*7sr!q9z-LDSEv$q2)fd^`KV|jk#;{0<(|7(iR=Y% z%&H!ihEpipYYg*VtS6h_EP}^FNz`SnDqTGyM;pAKLudXZC_MTHa<j?`8{LlCQ*Za%x|NZ{5Trf{hVA=oA*AdsJY~jQoXz#O=UJ>;tL!J4QC&pz)_s*uP zBn@$3X)j8t_e1!Q7zAhAlJp)U{FD0`5;M;u+kXV^7JXxz6X($u@#`oZu?F{9o&!eJ zlCL*oB02M%JNHVNfbQ~s^!^=%N&`pH-C6|hIRwx~VLLXo_BOLK?<#01M8P$K3()`m zJ1o{{!CY=8{Q8jx-EjOLsvLaH#MdmR{Ju`o+gBx%rn9<86)!n&y31Ci6KiNQm6y77%R>2U6VVdtOx zI_D62uSuHX+!%U)N<7(^seq1>Ol3ZHk zL{ytUv2o6GKvjGRKTR&5ac-JMBND^O;7%nx+-^*sRSD376~*W%-3N+Sa;eh#NMgA6 zHrVgk%l_AKn^(CZ32YV}##1*^+0FKbc<1d+JRPGB;c=31n{{BGZqH(V*UFPeP5q3| zM+4&RdxyFEgxe8L@4@&36Bj;n}|UozI;U5CYA&ax?XsvHxan>UD@6#4HrQz4m0 zO6N|4^WVp?J;WJiOyP3e|L(%H9vc#qu^nYsp210PGV%6iOQb6w!>y&IV591W^3AVhZCDq($tFWnOM9c)H?=$7Ox z{&=z%Y`#AN@`_7g>6FiW3)eGE%c>^BchA{81FlO_Fl0$`mki=aA(z=fAA07k0cyMI zkbM%uyvx&cX?2VbnR{Xqbr^ofN_4L#@}!M@V4;nFOT*|V-Xd6+=t@rn{zR`J2N>FC z4L2s_qHWnWBG{@!?5{1y1$AdwO~C=ww+^MYn+14J^Cn_Y%SSxAR*LOedJEHsft3C| z4h0X)$?j>ntn$Yr@a0Jq`E0q8tTh$`KK6{xh}f3udhPhE*F6qD@&91Z!~_m%UMB)=3>^>Wy?5sA*uN zQZ(q@Q;t+Dj)lVChV+K>9VlcpaGXfcBHkRjRTR;E`B!Y=stn6^XRwuy@`V3TlAOqt zq<#yov8!emVnF8;_FCFxFjl#Mwo@*{@8=aTZ{Y~*oup4^@3@H_Z=>L8e-kg`!!>NV zHU~2#HsUrxds5@DlP)-IK;JGc!TC3m7&^rh+#~C`E`1^`tsKOyU0+czZ!hP%+X1_5 z!pIDD9r{nli=J=W2aP9PY1rioc<(Jv9#*a;B3DO2=GI5t;ClmR?|H?$@x2duw|La> z=OP*#6~!O?E=n)YmLmyWOQiayq)5*+=HO#c*#UNmo(j;bj5Ci{-lS_`G#Ds+7m2oMu zejS%deJIP?|CmH8YCD-ffz9Yp9)z;LHj)k(H!Aiwmi)UPTfq4%F|hPWfo|%H?^qdz}VZoO2HR7j!Uo4tMa9Z!q~0>q!h^ zXOp(_XN+vA3wKA&;;$R-XH^8+G5)LvewgErbN_JpE3q;-9#al-Aun;od>`yt_6tVe zE}+4or{I3fI4(2aKtz@oGg;x$@M>8a!*AJ5QjHVnZfZjw2UtR7HVby6Iat`YknR=~ zhbzj*u_yi@n}6JmwuB5}95v?q$!O3k*8`}Bga?W4H>W{&%GvY}SJ6!3Hv}piV4gNd zP%0Wtmhf!IXrTgsd|)*tMeVHE=82@&RFdxZOJ{9wWw8coUi9ax-*B$m4h-#XGD4Tk zXxC3qA{X+MJ^Z8s?oWD$ahoqd%Eb93-Mp6BE-OZ>Prktp2SHM`MVzX4*W<(oom^IU zDQGjhy4|Hwbiqz^5bCYFXDnbuh+8wHf6w>AUB$J zDi{B}T1T_F-t)I>Pmpa`MM6YE(a^_@SrG9ES!z!V9Dkzbsne(=sZToJ?Pf0+D&W!A zGBi>mk4-$a59J3c!0dKD8U?*!x;L0WXb9{##Rqc}GI>gJd;?qVH&*Hql0;SA@Vn=e;(;w=qR6vj4&IFuj zhlN|u;qjlh@jxx-eRho}XP!DT7jPafY2M3?%GzK@izL`dq=4neo47x#5B#27Vb2>H z(39uSV_D&PSaT|YRpPqtbA_GBHN9EXkXG|7lG*AITb)bK&kL`JH0 zEhhKL5OKv=ycU{AJ}j^%-gbnB3QZuUIqu|TkP&!F$r2x0f`|UB!s;hWsS3Lal&_Dn z1(y2A8;+x%redVMD3=_oQz94K9Pmb)5P5SPKAV2}1vDeo9`oAj+1Il&*n(08w!V2jX(@V-@|IkWzO?2}=>`HL8(c5@Y%C;x?5F`P^UN7sY&5! zEw)%m(nIT{sCFZFexDP{hW}T}#9eO0FGanuCs>I)$FD}~Ev}?#S1!mbaHi>+U*R)% zE`8nTPKJ+~)80IB`shnB1lXuiWzxyE)Np;6HHToi)9`{CO;iW>Wo70y&wfzS>E_Tfe&w|o?_yh@3Q6U zt|ar`R$j7TGn$yDL8-__xFIizLuSvQ{(LLs8f!rvw`02&%!<70$qio$Y8 zk{ti2icS+lX~CSg@GDP*Y{}?n7mcgXFue)<+mc4?{q=U(<|j?mSDyhvC4G`HX&o9~ zEd$ul36b}%!hdsD;mti7WZw@FdiBI=>b3JKdre0e;oEEo37kil8{C2@32xUFREeAP zFEa0ZOqs~6`zWeEjpj_`TviJ9C@|ZBiSFn_li~(^wIm5W)Le*C&*QKjlHOLd4cWMkPXb$F%77--asCRT9fag$8g}> zcaYKJG8SElL{~o)^OIK4Qz@c2c6|lID-xhrRHo6ErLHus!XDe)aBFRqxFVN z;5+KGDYZqwYn#emdzXOXiSFdlS7~~#S%J7ObH-0iN1^z@I%exVH(Kj;l|Stj(8;Gw z*fMS(IK<^Quk3DNMI9_jSmSkI`>L?^vNEpYSZK!#T)?>~LAc3}eDzI#6}`4U4~lyVrS52?IUf4M?!ohaEf$(T`B_=M}kzw*;l z99b=EM=DXKK`U=cQ}gyX)C&j=~T$848p}H`Ji*pfGAA6&u->85}%qc zaUBUcvRq{hKgAl;PQNm?P3AevR|_M`8@q6Ztu1}@SC_HSJ_{e_z2m>~vuf(~f5O}_ z9c30+JmQ!_pIIkG5xC#2jgR!aLEwW5$vgRgt@#@Pavct!cUKI|z8Uj>dS$@&q$K9> zxnC^rl_9wpwvnhU@#Oi6RDtnAHSQjBlSzCpMn2uxgVzI)|6*h@xuU{{jC<>dSTsXE z3jM%2xu0>Y$&kbr*Yc)VB@!oLHM->QFdO#Q()LA3EPkn72Tr!0R5H^6M)va2;>2c1 z?8qS+x{)NfMVNZs$bbuPpTYFDTv!mHOe5nD;Klf*C^$F{nQ6u(_p>tfjC>8-7E6## z%j@tTSD0A;dl#1p_zA&*jTm_C1y;zaLvEHA)=XPXzc^IkXA`c+-<}9x^K}TSU4*=X zR$MkY4ks4bk-STZ?6JGP_z1*ktfde;9_GT<>gS`P-#+{#nF&?9RH#dlG<%`%A69rs z(ci)StbM*2Xr{cux3;CsVz)$+J>QJRJx{?0dMlvg_8j~c6^lJ>4s=RU1|<7Uq<7Q;cs5#mPx40Ww!T6`ox+p(=+ph~bN3#&GOE$P-f} zx36cwP=FoWUvd?`N@f!&zr)a(YCso1ONFhEEy%o^XF#TS7R=xNoxNP;O}y^zfaCer zsK{}RUXAU79o23q)7He2N`LA%{}mXtcQ7jwG~l120qJ~yho7;djD7Sx4g}K8z)ouw zm*)-TIyHqz3XJKCyYe{8ca(82$YY3t3pq3BgdUNJcrq%DzHq9?s4pKuGKGO@l|VR| zCFJJWljr%pAZ5xq=l@S_Zyrw7`~H2K$(#%sD)SVQA?$Tt4OGS=$q|USW@4kP>_xnAL`#GNHk7qm9v9Z=^*?V7muj@L` z_xXD3h|`rqx1jrYAull4mx^7GhDJ>b(s#|4zUh&mvIXbicexDNcv6VoQ>|kHTX^)3 z77MAOi@2WL98`7s!HA0IF&|>vVW+`qNS)vSqG{h@SA8wq(h?v{++Tjx0ZIJf5Cm2q z-r6nZSS7zrx-fBrJ;l@@9Q?GDcKzA{FJH_ePZthB=$vNkC{-l27AM)J4Q}Lk;X-iE z<@$x(rW1#-DB4vc!C%jhLm9axM49!VA6E3S$=0bvaHTr+O>AdHIH+~kq&ylG*9#A)712+2``DBc+T~rzE`0ZHy+}n8=CaJ@;7vf zF(u~@04iCgfuzW7z5P zml@z4!a3W%@*Q6pp@HWn%-Pt@XeM4_+$|JQ%`6+|kC!ETcYQ>o1Sxnnp`DdEvIph7 zzanK{VysUP#J!3oWZ7-ZIWr0GO%TCV&As^Jks7`4ZH4M`ZEVZKFtoVw8rR)6pmtC8 z;fLZ77?zn#jQ^B?;+GrXUFU{_ug6j2&)ebi;e*WdcMXi@cLIVg;m|oNh@1^6$Li~; zXy6qEdJ#vNvfX<0w^}&y+4BJoU*{Mu%G)4qk`4W)6G`fJPo~p%<)HCTTO#8*9$!Ud z@{5X;1iP^O%ci*d z(k>XceiSak4YaMxBXYXU@I(CqoaYd`=p@G0{o+5_ z?ZjLP5+TXY3F*A}9DVXa>8ceAiQmagh#HWirPd|543hSV&Qpp|}xWZS|m5TwP2_UCb!d(VKB z3`CRJGGo}<`9idM&SJWvJ%u)YTMhj73S7W3(A~_Fh>XJ;QgBd`D0hb7UH4e5^E4&V ztQqySwSn@mPQ2q9K}Wgy-0oQ=u)_Zu7}rjrV{6pdJrWtP!1x^APl~6F=S{%&gD-h? zZUjYYwgL>_!_;nJcvbWh2FoW<+Yhbicpwu)b0R@^qXLx))28%3#RJyk(ZflCuK8w4 z_q-m#p2yDM+{fud#Kc*v;q`R>Qie(tB!T#;XJBUm|GlEBWqhTd?`=|!B!Dk zBe<4m53L02U{2TB)x!GcsuQuP3Ai!(CHn5)j{BZZWmEGbxCPro>+mwCEBS1ngU$AycB)3NSzEOVc==t*Zu6cc z)Ff;zushYru{Q_!M$XQ7Ov;oR45`tPT|fB#GqY%1!wX~;w!?upT+Yo-hZr4AgG<%2 zfM+zCT-AKQWSTr_>&gY$w<(PJ^9(X=-%IA+dA&x~>oPmK-4iJfH97; zRBf6f`fPV3Z#h5smtF$&{S0B^l2*fWCtMGBUkYijHpTjtzd-y>6x?!kAPai_;`xuW z$*~wAkV}q)0nI_CF)X|hEtu{^nlSZFbKEf^YN8rk%VrKK@X(;V`494CM zC$GD`LHcDm>|ACHl~=Oph7)Z(`5{lp|Mi!jF?=8HAIU_6sjnH0VSRF`B!nia*03!v zG|28v?^)^DOUdA4h8{jXz-S8^!3H6qvs&*llTRN9Z?(C2(T9Pm`f(&$mD4`G^`*Y? zk5RoQk(hZ$KuE+S+GD)~PrY%ark&O_b6`GRdpZR=jwN&bAZ3h5Tq;o_2O-QzmN`*e z2z!h6BTwkBT|d#M$NcZ(#P%pEGn zIz537dqvP$p|bR_^d#J^*Z|7G9M1K^KDg4u`6th6!4r$d@qBIcc$GnFSbV7gpB0|L ztp_?F%gPZ9PXAiKGj$5>ZD5@E-$x4QJgB(M>z-)ZyV-M?H zrlOVQ1ft^Z#FWv~pwzkoT4raXWOZcZNXY+FG8ToptgR;5cas<5_{#q-KNyaS(X zkdD_R0cT$@ewy}F^W;G=O<}Z8Z>Q6kzMDM1_Z{q?V~fe6^)txGKs6p5ybphF z?#C6MqNv2oHlE|G_v~&tTNLfjgVw})$OyNfcHPSK(y1!e+FXv2+ayY^J3BO6)6vFWUsZ~dNh$ocI$Mn5dL=%cGk~!wVZ`p21NYo-#M#T|kUe$gsO0;S zZSXf^_gbZ6#yx^^(~`)C=VdU>?-LZISwY_?NvK*9Lf5*BP|;Q`@>P92{QP_aN!UV| zctM6edge49C>5jgKJ>7|2I^@0?G}6I+d-V5JcQr3|6)_rn&Hk1GrUp~2X5Qly(SR>j5X4gKWd9oe8 zS6EGspISoY-q=zXuQd8>&J*}v{e&GidI1D4g#d4T7rQ14tLTT4E|4>0C_59G*#ky)&W7ruVVsa~{?o^`cc4THt-X3m5x_GFg$5 zr2dCIjaFX>-PvWZ<)SsT2DwsgR}bc9&31b8tu;LC9?S6o#*pPbR`lYUC=y-ZKr1bD ziCyL-VyiUD7Jx0`4fUdX*DE}-xC+FydvV@3Wo(dI0AjDKaaCgyZobur(gE2_Rc09| zPq8IJ%7JuT&qZ9kGZ!B{%cNUbKJV{KMWQK-AbmoKL`m}Kvz>=1lIqAGQNqC{ut&Y~Sp!pXT= zmXPZ76J+nq#Dse$?5wdDm=JR(vKh3ff#fkZ+OCv$RY2L!p#LAr9YV1@DR>- zPKHRt0kMmeqvk4hH1*UfdQm7A4y>%gK8b5kwUq1I4Hcj*ylfQOWJ70#Tt~m{qA2@m zKO8FQfM(-4)bOVbEfM{Sv$H3W!2KWC3o2ZWXJZU`Rpd{k#u%gGU=if_3X*jle_;-r z%?gV@=Gfsa@bq~r@6jET z4te0QohN^&l)YmrKv!;j4c!w{XwzmZ+9xaku`T&@B3}yEx_NQBOd;&rkPL6)vZzw) zHKuK4Hh9N3!HRoQ92(*`$S;b7eRKS%Vo)cDb&sNXvh8So+v( zgkd>77+-1<8<%?#zS*^9O)F1vyl014@7S1gnQnp zgh_9PvL75naLy$1r|~5}!9auAWe`V3riXxCO$wXG`C_W17%>Wg?_lzcS$2lI0*LbO zG|=62lE1=n3QRbjhN3T=>Esvrpt}DC^jCL5hKLe*-4{R~s#=n~az1(l1vjna9#3_a z4slr`&7-H15ufqMxG@Led00vl>nH{v7Wz_+(FNpE#a%ek6i0d{PNCjL=H%J9W;U8DcP|0ec2m#=p#xVbu3BI*unHIb|NL8d^nX+1b-e<4-Vo zOC+h{epC8!P?i2D)&SQZD^NXg1!_K!M2#p1TwN0l_HVDVcDFqmuYL4}F;XJ%c1|!m zym0{yojH+O*5yIKSwUKRgU1F8UWeu$Lwc}w0JOIf+ROEc+D&{7OZJ?_q9ZM^=KL(e zGr7(_ZrB0Rrk3QZGRx!}CepKKl*rw*#qemd2|G@ZEiq}M*pIR#3(pGHQKBHV)8O4alIq5L|8|hs5xdfB`^L1 zhsLUqC-ow5RY{vH=(b>GmuL~!l`gPpY6T;`*%$Wye8fETT1Q02-)HTPU17ehs$<5q zjOB}WE&?rcNn-Fa9gi6&ko)s)!<(7}oFL9Z=BqrA(SC$OKLNr6UqG*_7DU9hpy*va zP^t~(ci&K^*Gn|%=7T%oy;%SUXFrSvObXvt_Gc3f$aAmP1xsq zja_|GmHBM0KumR3(u}EbAjtKOuh1%EN=(*Lk$77gIKhJ0EqIBae2kgm_zeCF-)UI5 z(v;rJImSjwEMzr$i!l02KO}g@6E}VvEIi6F4nohv%z_25(|I|Y%9}~($7VPaMyR}M z6~4~VqV@M?F$q!i;IUYaeX>B23>Le?o)^x?$1XOCgfqOM~ z*#*}UDHA`Q$YtJ!)Z^E2oaG_r%ZyWYGq%V;oR=(FGtddMon@He_ASKhg*av;+Q4yF zEt>R7i#Fd5CK|cBAx~9|!6_M4DM?K;>bGlnMAzh=Ac7J#L}cU-c47I_nK z6n>4h=GazYXgk9lR&nc)v0g@;wx=FboW@b9G7%V~Sc$TCCt$)~0SL6%4@Yjc;DU}o zx}(ShX?ZD|qjHtWyrqN-n$1Du&?0)5-p38vWf1#g57Vx4lv%s?5X|)uL61)yi)=)j znbWot2bC7nluKn8eJ6>&tW2bd<^j-k&L8cJzQXUA40wBn2lGG5Q-0?z{$nQOA zWS-C>vn#&Bl;Ui>^n3zw&EQyjCmo6Y^$WWfgnN*HF`^txna3m*-Dae>+o0Hb&WB0a8gHzVAt|9-ux*+i zp08DA0y~Rv<(#X``4}FAzlb0~-EBD4UX88He8-cr*P)F$Luhr{m3~*%g^-dx(5`7j zY@#Zep@3S5#atsRrWyf1u;x)R)(64Y<4-%pXawAllFOD2jj6;)IU^^Y`T&|zB+7Yugb}QbxAGW zh$4*1Pe(eRt0V1yrb$;H*CSR=TiKc9GklT}Ax=+QS*tr7)5A3fZ+BnED|TFMwoe;F zf_z}cy(S1KNW!$gCiKIG7^qrxno;EPPzmEjNP2D{ysCC(2doa^_v9Ab_?oL*t9cNW zNuqS(WJ!8`^K-T{eFmvt6if{|7m>csATY=jg`LCBtmm{rX8XH%D!AtaCR}qStL_?* zom`)de!VPR^T(M!eQOMzvIF2apYrPtnKepsYY9JbGjel41Lv0ZH%(IL2Q3TEZwUJF z#fK(b{KA9GU+GHk6zynCByp(2>0*VKi&3TH-lQz`9?bunjkkj1u#}a6=}uqK+_x8f z|70V7k~vpL<~%S(16g~0OIr4*kWKVR#wxB?@Jb=)yJ@xzkBk@)TaI=2{@xOh*yv4p ze)I4{>=w>L-HhXzT(r{Q^-&V ztvR<6Vt<9;Gi5!dc>m+fq? zo0Fv-?|433uCi*$XUNI9g(a^C;DF>|xN>KSQp-sq?z*FSk0Y%dmpMjYH^%=Jrt5Nla{Vy!^u75BNO5(Ck}0(~r)4SIHBbk$!#9y54V=D| zbmCCiLDu-5Ai3DP1>Gty!x&)|GW@34%TnyK7oi7M zs*sr?9LH$98WeKAHwE=lO{y0tD%A=R)0s*bYhpv6tiA%Gl{z%1{XO(2$f5mEIlkO0 zK(=;qUck*S*)JvIXwNh;dM9H(9poG0yo(HZuwygvHZvsWr})A0-LCZ9oi=8tPy>^( zDwl{)_=fHN+vsHDI)2l5H{$m-m|kCfk*o2XV)Q%>NK%m`=YuFtwwxPFCS^78?)nYj z>#lqp`gj`6>mAAI51vH$#S~PVIfXnl^up>0X;M=;hD57tQ%7#jE@Q#zm5AbHkEE2Xnl8g&5#d|huiB+vW@hgA8?tryO z1G%0XM{Zu<&ogl6;7%*``Q`7O2t=ZhhP8 zeHcDI*$M&%UCi0VXQ5T$3LX?RBzp^PL;mYXa{GEDmAl=MSbMsF>do+@&*sLH5IIYneQ7nko{Ok{Y!Y_d zIE&K+3IUoE=)*HU)TlI=C~~#q0C{0tarQYlhzgOU<=Kqbr4-bZmnPNfQ~2K|_|VIn zG;x)B21@Pu1obBug6>W^GBJ7vHCZKux9d~EQiWrkx=%!z?Q5uhzcBF9Qt7dAet3iC zz>P;*9JAb?xcJ@1pG6LAM3pCATfs@`59sq|kDH9|svW_wdWcaSFu|wk57@zMW0<_g zk$&@6rw-zy?1@-O_Ctg*R;O;l_C^udc=`s!shoh=`(li1xe@I%ai=%Nhq4FbGU+4L z(39pFvN+vF9^nAIU>^rNu-EDc%jwgyYMCG4zlj8JDd@E#62G z9mwF7TnwTIET1uKb|5KiAncj)7`je#JMP|_hmTEuV5!g!@?2&LuX3L#b@*D!t{?lG zd3*8_tCHCOi32Lc+TVm@`I*zjK^i1G%Y{m9lqZEZ_G3acLk2jO^JG?ow57P(^=PV!3jD1<4J#H;Aw5}tFw-UwvuBU+Gd;dB zcj_+j3$A7ml|pM`^;nANi{FHJ`KPcXz89`bB;(*UMfUEOb1;66DgE7%gDa9pczS`W z7@7TIWXa_`*kyJWi#K}^$p$+-W-3H&#k<*%mS(7rwV{WyKk-^-Z=f?~iNL(85X<f~vP2Mfp2CSMu) zEFzVP{}@L7V?}sXb_jmZWmqu%BYN7crP*rYw8kg|#;mj^5|NW(nrJ5omoKD$3PWl3 z?MB4w<5AN~#CEq!0c>gI*8UD2^x693Y`peKOfauud-P8-QmfXJf=`a*Mo1PKEe;}| zUMN7dybu+bQ;KSP_QA_%PUMKTICV1AVW%GuL#y?Z>Arwbtlsw*R<%ixchwW1T{j&q zl9Z^zMqSz#QO?iL<#PMy!|Wm_?L|?|JiMMb%KX-rBAuQ7RKc_y)xKu2?Q68rv$P35 z&*;Fkrck=Ou?dq1j|!76_+>B?64&>zE(v#0^Mo3>?{LSHL-!F+KY-OA2XMl;18mlZfT}>CDw^FR(u~j+&2*p~3$5_{*S zcYmTI|0C?!8%RediIcWT(OA>o20_7`NABd#gISDXBBk)ck$eNY)BK#PCclf$RC!;u;xa(pT0d+L7*rK>;Vtr2mWpAiL1 zTNO#mVnW6jCNSlko^@KV4>a991UK%#fR|g`n1L&g*+Zju`14~7Nd2P>0{aogxw>BG zh$1YqJB0W8s(I(DuQS8zxV-+vvup;B!mBDv>cZ8puU;MphHf3`GVvO-YLw8@1Qxp2 z3sW&^4ff{rJ7{103kNnC;@glC2-zkGE!i2^`_`KX=PMEQXXz9cF*vXLp@sZ2H)H#0}Ne}Rwc941si9IXEaqFqBXxE`59F02#d73!DZ zv@u3ZrqW~*P-Vr4tXKp0r{;miY|0PUkERixHngZho#eaBAd6FXkln(W?EFoqF#gv* z?<%IFdpKx&EBsb<=2Mhdq6sFG%!_b1}BG2rLzIdC_54K-qsB zfis5Wy;dmGp{WAZ9?oR`q`BB(T+HsgcO4en*>D{43!u@Fj4vL|r>CR@iBwt@Zt|BS z@)8`&%gTq8lN_+r$a0nX_yXP>D>J9pf{X&_<(VLAIx}b%z$BywUlStj5W3u=&qKHd&Aa}1xrd` znvW>em~b23hszNY>#yigVhYnQePExcPN44I3e@{I!2S&rFmtmAWIdIm_XEzNUbPK; z^-Y7NvI9M*i%i(&i7BzC@w@P5bblK~bw52ohYoeBwssUP%btKv z@jYBMm`N-+50!Uojo@pgJ9KZE%Py6gPYlPbWs>5=;kx-UVifTUcrCpke9(m4F_U6W z1)T+n-63qBq|s((=w|c~fLFH{}*zk`htn*8|RNn@6{L=S$Co-Pc z*-fRNt~p`6Y#)@*4kDKFJJ^M+9^Uw zq%**ptWROcy~3~Tm@}G?_$i)Qm)yy|v9iTkoodY6PjksC`|H5JW2ztyg}+6$zrl3;9SL0h zFbC)L>hP_;O(lV0vUJOoQOtWTLME#}1D`o3;X?guJY`zNCjHVNWn9m}tIL;(<14~bIGv#wh#^bvWKzEaRYX9AuQh}Nu)>YN$ApRP-J)# z>E-eC#r$8;FlQy3d~pG-7VTxO%r>BH8LL_6EIX2sr-bbz8(?MAHym)wXXHlph<)f{ zCLwMPt(EV_%<(ov$I6y!{H}!`A5Mdz^9*`rawsq9xh(O^e1eYQk0IhgAes1TDm=6q z;q701zG-oAB5c_iOGG>&K>3#gVDF@M8+<{JG73F28^^QkUVy z+i0vR&*P7D%%b|OoQI&#K70_DK_6vCP>cI&^kQ)<4&9thPV2^#ULruIzlowM4{eFo zPC??Ha~@`d=aa8%G;pcSb94|3g6>n(Xh(w$?P*b@_pDjie|sERl3j*-61mS54t(>x3ez6#!9wY-NF2{`j|D1BSDAN!&s9U!d^`MkG01L>sXB=_u;usi z+^MiD&FDc0w?sa6XZ0Ijcii?Zs_pe!;tx5_{yje zmqdNUgC#st>(~y7f4#|}@MN~mX9pCti<5PYj&$eJcsR4J5&RQ#@t_0eo405hst(Q~ zP7>Q;)OI=%n6ne-FUVwT=Oi+AqyaKlyRxNXm*6lg#hMQ*F}MCcYAT%s?@&KXS~7)5 z?h~Nlrjf)_#hl#rt3>&9KN@4gtvzERm=48ewrl7Hv)M<2$m~kPvp&Yq>taIgy)Yn0 z4{@xQ9z*s$F{G2blsF&2Tevqwm`+?%%M`6V27Z}_So9&8=~b$L#WLAM)2I_Tn`64{ zq6-;_%Obt{sl0ZPhp1|{0IpDBcz0zR|8nP4*kL9MmcWTGq^-kpJjTZ|p+?cc^+(m>JvtksWtff=NHK14ZXP zXUBO4(^U@RxVqQ_95`PAGxl=+qs5QGPDh5U%t@k!r3=Yf17Yw@T0+Kc-HHMJBVhbM zgp_;>BG)%b(@R{vE_p^fx~%_z729&5Z8(Qk50#+$U>?!rnC&^Gr(xqeLF$?~g)UeI zRLUd^&De3IZTBgr!?GI{+Z0H9|15G;>?%%pHJ|*N$A^4l0v^-+sqAr85Zs#z;w{>A zkCQc#Zg!=?_vEPOCl&TUpC&DH--kN#+nHy_JuyJ>4f;ILVYV+^PM*#jPaHzNqSx)c zyarEY+8E$K4n?|?VwbVRI!Blu^ZJHb>Vt?+S92UARboA&fjshq-H8WG7rJZFJDhP#mqrFnqC3vK1Hlm~ zYI;c;Y;Wh&y5L`!c2%3n{Az*K%2s5}LVY;Z(+AtkxSX{hN6qVsqQ^@UG0x-?fA^|I z@LQ-2`2B@&sv!sB9!zB8!#tUc^cA=&ECRZQj5rPY1QPaH9()>&sE3mx{a7N+HYlf| zY44`qJh#bKsN=_--|b^cZEx^&{GBs)#411AMq1Hi<^64o{yMn4s^WA%J!N>cQWb4GyYSku`Y!XHe(?-xp4N%3|8ed*Hd@y z6KtEWMsl@x54&a)W-%=h<(V_2S3ib^a4p^qLIak=qAp=#R(hcVMYaCcDV867uyl z(Y7NQHg`$Uq(Lik_3BSnQqK&Wk7+NBCoe^;?@PO6crb8jKm&49#ej$nm%9s5_e0X7PZr{{UL zxJfhy#kqconI_vc<$FIm_E%x!+=jHzUvyC(7wRs z)jr@yj)h;Bv4d1cJM%wV6{4F`KfEv0N3%ZyRO56o?f+wM8(fx!k*D0qYM)GMdoc`! zy8B_5b~A4D>S1;&Cb4sj!|BJ6N|fy0i0*4_uyI=|F0Jgu^65Qzx;6vry-tIhy(x_@ z6(idP|FUA`zA#-shvq!J%8M);OZW{%a3eUl#>t3Gdn4 zRfkv|u{y|^y%|1oK1r!(%1~B#GwF=aViZFsvAY(W;Fs$m92|`%Ssde2?r1K=r|J`D zuTl1pD-T946k(TjK0jpm0LI3tkZ;uvWJ`r0@mku$iblqv;i}0*f9)jtyS9tD}nsA2bdf?fA&wGEZMDk5$7(w0>0gD9ASGFDy(?Se0?HHMA=l# z5ZFwvI%kokj^oMx_AiWLnhw2r#u3{_Cy*T|OT*r9*SAPB`gF`2$a^gW3Qt>^FY9WV z&-FJ!^qdk|-q6n5$N8o-)U>0Dk|^E0@DZHS=P_=nRk*t=6pee3b(yC#`^r z*gUBJyq$yvSisSx2rKs3P+qYe73288J8Qb(yxbJx&Hn+OCy$}V<2?`z45x*%g`yFAAE%NBu$Wh#l*yN1s1*WygW0x(fb zptny8vVr|Na4q!)Z)u4!^*ED8NAj0K#!WstYE1;07Y*pj_0_ec{A4#>HOJzyhhW+A ztx(Z?nOV@$idK6vah%sx+$I$+gH zQLs!NWp`vtQokcnREcAC9+3$p83*gIUP6J~(Kn;zMIz)-9H&dSQ6s@N&9HC@L2;MA zOyWWu`)8oJNM@%;^*t9=|K}0h9Pggg8W~(A@<# z^!A(e%)I7_q@Jte9UqV+;mciU+#Ve`5p{>sgxIh|m5xd8;H&Z9$9IJ~p9IZ-$KfHlKc&|m!< zli-tz(cf<{+nuuMK=BtS>V3>E&=Y`g-97MqoG$0lX-CfmYv9>yZQ%Fp8azAG#_l?9 zOLF6ln5kpjsVHw5Z?>>LY+G`N(GLOi=TnxLCE@D=Z zKNWvGg-T8k<`{?}=-9RwcIXws&7OX4Y@b8h$-|s>rV>BL$nkSE3mffy{uB{}Ah_NEdl84vn-!wOK+Q6Piwk1(F+YvFTvBP&s`hbhg~rH3B< zWXyM6<|PiTX6~)r0Rb7@eqKJ4Yc+>w^6ow6D0#pq!HM(Ovrg6M? zF=92O1bYQ%l8E6`On8Y6Xzt4(BUYkVlTwK=@R-xim=foRetxgicw%aP8h$UBPF)wh zK!MYS{Jc%ZP&#=r*?w#)9&T)ffKCHOzD))HEP9XZ;(5$0m2${(H~<&VUgXvAN{&0|y4^?cu5uJD>OF_`<9i`FlHyF+@7Vpqk?y)#4Vt}@G&Y4} z)K1K$@-dzm{K|$T1pUUD{hV)f#d+k(v_Mtc5_lE$o4NnxA+DQ{#X8n)rSm>V(B9h; z^!U9%yOSjbFzExgX4RL6BMvLk&wc{^n=Q=wk!a8`ip=?|f)Et(8Wt{J&r~LeGL^40 zz~iSFIcISMb*GA=c$xxvl*ILPb?GCBSdratxjxm+`=H6ifkf7MvoY^uNO^_;wac;t zb*?sk`A;BIjODnvt_Rn<+{a5RwOFUgcQL=>B3@cy&8}FWLL@U4$laCY%$T)SL~eF3 z$3EP`OpG36XRn@4r?l6wMZr15$%oTLOU<-XHT0r)bu;0u=Oh-j+TcN4Egpz@h+89C zpus;4Q2?m>%q6(Lpo~4>faoZ*70bq^(YKz$R9iKjx+pk6w}}ETsdNQt-1r5&#rNY> zll|!R>jpCq#K_Ny?;zG#fE%tOaFyluKVByYG5 z&c9F~#oT+XZ(kEya-Pp!)<0qE=o2>Xa4+UQ;@;a8p2XqyE-0u-f||9%P{?_N7;%1S zz3DQjy{Q2redeLggkvy!=>t5S$!Q?3JCd27r$gr4Uf2LzIp0|&w3Y-sd)=9QJrc}m zxs~uA{@gORFj>v{9Dk16{y*^NtpDHe=l%tC_y60s|2&3kZCNGDxikE;{ofyhoWP|2 zVdy1 z-){d(#59Xt8TFsh$9+58_sR(f{IkjaTjWyiZ~kla6!ZU#{a0?_zsJUo`;XYE{~7zQ zOwWIh{h<6Gu@?Us`+q;LN0!jP|Ie2H8T+qe?cdLPGUq>Ht^PCiUti;Yk2UZ6k67#f zjJ@?QuORnS|I6*mzr_CUG2!7VCT8-#51A-z`acK%AL{A Date: Wed, 16 Nov 2022 11:59:14 +0800 Subject: [PATCH 29/68] :tada: add reinforce causal mech and corresponding tests --- cmrl/models/causal_mech/CMI_test.py | 18 +- cmrl/models/causal_mech/neural_causal_mech.py | 19 +- cmrl/models/causal_mech/reinforce.py | 380 ++++++++++++++++++ cmrl/models/dynamics.py | 2 +- .../test_causal_mech/test_CMI_test.py | 12 +- .../test_causal_mech/test_plain_mech.py | 4 + .../test_causal_mech/test_reinforce.py | 176 ++++++++ .../test_graphs/test_binary_graph.py | 18 +- .../test_graphs/test_neural_graph.py | 18 +- .../test_graphs/test_weight_graph.py | 16 +- 10 files changed, 640 insertions(+), 23 deletions(-) create mode 100644 cmrl/models/causal_mech/reinforce.py create mode 100644 tests/test_models/test_causal_mech/test_reinforce.py diff --git a/cmrl/models/causal_mech/CMI_test.py b/cmrl/models/causal_mech/CMI_test.py index 66c1568..a285e3a 100644 --- a/cmrl/models/causal_mech/CMI_test.py +++ b/cmrl/models/causal_mech/CMI_test.py @@ -121,7 +121,12 @@ def CMI_single_step_forward(self, inputs: MutableMapping[str, torch.Tensor]) -> inputs_tensor[..., i, :] = out if len(extra_dim) == 0: - reduced_inputs_tensor = self.reduce_encoder_output(inputs_tensor, self.CMI_mask) + # [..., output-var-num, input-var-num] + mask = self.CMI_mask + # [..., output-var-num, ensemble-num, batch-size, input-var-num] + mask = mask.unsqueeze(-2).unsqueeze(-2) + mask = mask.repeat((1,) * len(mask.shape[:-3]) + (self.ensemble_num, batch_size, 1)) + reduced_inputs_tensor = self.reduce_encoder_output(inputs_tensor, mask) assert ( not torch.isinf(reduced_inputs_tensor).any() and not torch.isnan(reduced_inputs_tensor).any() ), "tensor must not be inf or nan" @@ -130,16 +135,23 @@ def CMI_single_step_forward(self, inputs: MutableMapping[str, torch.Tensor]) -> output_tensor = torch.empty( *extra_dim, self.output_var_num, self.ensemble_num, batch_size, self.decoder_input_dim ).to(self.device) + + CMI_mask = self.CMI_mask for i in range(self.input_var_num + 1): + # [..., output-var-num, input-var-num] + mask = CMI_mask[i] + # [..., output-var-num, ensemble-num, batch-size, input-var-num] + mask = mask.unsqueeze(-2).unsqueeze(-2) + mask = mask.repeat((1,) * len(mask.shape[:-3]) + (self.ensemble_num, batch_size, 1)) if i == len(inputs_tensor) - 1: - reduced_inputs_tensor = self.reduce_encoder_output(inputs_tensor[i], self.CMI_mask[i]) + reduced_inputs_tensor = self.reduce_encoder_output(inputs_tensor[i], mask) outs = self.network(reduced_inputs_tensor) output_tensor[i] = outs else: for j in range(self.output_var_num): ins = inputs_tensor[-1] ins[:, :, j] = inputs_tensor[i, :, :, j, :] - reduced_inputs_tensor = self.reduce_encoder_output(inputs_tensor[i], self.CMI_mask[i]) + reduced_inputs_tensor = self.reduce_encoder_output(inputs_tensor[i], mask) outs = self.network(reduced_inputs_tensor) output_tensor[i, j] = outs[j] diff --git a/cmrl/models/causal_mech/neural_causal_mech.py b/cmrl/models/causal_mech/neural_causal_mech.py index eca665f..5a775f2 100644 --- a/cmrl/models/causal_mech/neural_causal_mech.py +++ b/cmrl/models/causal_mech/neural_causal_mech.py @@ -128,7 +128,7 @@ def __init__( self.elite_indices: List[int] = [] def single_step_forward(self, inputs: MutableMapping[str, torch.Tensor]) -> Dict[str, torch.Tensor]: - batch_size = self.get_inputs_batch_size(inputs) + batch_size, _ = self.get_inputs_batch_size(inputs) inputs_tensor = torch.zeros(self.ensemble_num, batch_size, self.input_var_num, self.encoder_output_dim).to(self.device) for i, var in enumerate(self.input_variables): @@ -153,7 +153,6 @@ def forward(self, inputs: MutableMapping[str, torch.Tensor]) -> Dict[str, torch. outputs = {} for step in range(step_num): outputs = self.single_step_forward(inputs) - inputs = {} if step < step_num - 1: for name in filter(lambda s: s.startswith("obs"), inputs.keys()): assert inputs[name].shape[:2] == outputs["next_{}".format(name)].shape[:2] @@ -337,15 +336,19 @@ def reduce_encoder_output( ) if mask is None: + # [..., input-var-num] mask = self.forward_mask + # [..., ensemble-num, batch-size, input-var-num] + mask = mask.unsqueeze(-2).unsqueeze(-2) + mask = mask.repeat((1,) * len(mask.shape[:-3]) + (*encoder_output.shape[:2], 1)) - assert mask.shape[-1] == self.input_var_num + # mask shape [..., ensemble-num, batch-size, input-var-num] + assert ( + mask.shape[-3:] == encoder_output.shape[:-1] + ), "mask shape should be (..., ensemble-num, batch-size, input-var-num)" - # [..., ensemble_num, batch_size, input_var_num, encoder_output_dim] - mask = mask.unsqueeze(-1).unsqueeze(-3).unsqueeze(-4) - mask = mask.repeat((1,) * len(mask.shape[:-4]) + (*encoder_output.shape[:-2], 1, encoder_output.shape[-1])) - - # [*mask_extra_dims, ensemble_num, batch_size, input_var_num, encoder_output_dim] + # [*mask-extra-dims, ensemble-num, batch-size, input-var-num, encoder-output-dim] + mask = mask[..., None].repeat([1] * len(mask.shape) + [encoder_output.shape[-1]]) masked_encoder_output = encoder_output.repeat(tuple(mask.shape[:-4]) + (1,) * 4) # choose mask value diff --git a/cmrl/models/causal_mech/reinforce.py b/cmrl/models/causal_mech/reinforce.py new file mode 100644 index 0000000..e784879 --- /dev/null +++ b/cmrl/models/causal_mech/reinforce.py @@ -0,0 +1,380 @@ +from typing import List, Optional, Dict, Union, MutableMapping, Tuple +import math +import pathlib +from itertools import count +from functools import partial +import copy + +import torch +import numpy as np +from torch.utils.data import DataLoader +from stable_baselines3.common.logger import Logger +from omegaconf import DictConfig +from hydra.utils import instantiate + +from cmrl.utils.variables import Variable +from cmrl.models.causal_mech.neural_causal_mech import NeuralCausalMech, default_optimizer_cfg +from cmrl.models.graphs.prob_graph import BernoulliGraph +from cmrl.models.causal_mech.util import variable_loss_func, train_func, eval_func + + +default_graph_optimizer_cfg = DictConfig( + dict( + _target_="torch.optim.Adam", + _partial_=True, + lr=1e-3, + weight_decay=0.0, + eps=1e-8, + ) +) + + +class ReinforceCausalMech(NeuralCausalMech): + def __init__( + self, + name: str, + input_variables: List[Variable], + output_variables: List[Variable], + # ensemble + ensemble_num: int = 7, + elite_num: int = 5, + # cfgs + network_cfg: Optional[DictConfig] = None, + encoder_cfg: Optional[DictConfig] = None, + decoder_cfg: Optional[DictConfig] = None, + optimizer_cfg: Optional[DictConfig] = None, + graph_optimizer_cfg: Optional[DictConfig] = default_graph_optimizer_cfg, + # graph params + concat_mask: bool = True, + graph_MC_samples: int = 100, + graph_max_stack: int = 200, + lambda_sparse: float = 1e-3, + # forward method + residual: bool = True, + encoder_reduction: str = "sum", + multi_step: str = "none", + # logger + logger: Optional[Logger] = None, + # others + device: Union[str, torch.device] = "cpu", + **kwargs + ): + if multi_step == "none": + multi_step = "forward-euler 1" + + # cfgs + self.graph_optimizer_cfg = graph_optimizer_cfg + + # graph params + self._concat_mask = concat_mask + self._graph_MC_samples = graph_MC_samples + self._graph_max_stack = graph_max_stack + self._lambda_sparse = lambda_sparse + + super(ReinforceCausalMech, self).__init__( + name=name, + input_variables=input_variables, + output_variables=output_variables, + ensemble_num=ensemble_num, + elite_num=elite_num, + network_cfg=network_cfg, + encoder_cfg=encoder_cfg, + decoder_cfg=decoder_cfg, + optimizer_cfg=optimizer_cfg, + residual=residual, + encoder_reduction=encoder_reduction, + multi_step=multi_step, + logger=logger, + device=device, + **kwargs + ) + + def build_network(self): + input_dim = self.encoder_output_dim + if self._concat_mask: + input_dim += self.input_var_num + + self.network = instantiate(self.network_cfg)( + input_dim=input_dim, + output_dim=self.decoder_input_dim, + extra_dims=[self.output_var_num, self.ensemble_num], + ).to(self.device) + + def build_graph(self): + self.graph = BernoulliGraph( + in_dim=self.input_var_num, + out_dim=self.output_var_num, + include_input=False, + init_param=1e-6, + requires_grad=True, + device=self.device, + ) + + def build_optimizer(self): + assert ( + self.network is not None and self.graph is not None + ), "network and graph are both required when building optimizer" + super().build_optimizer() + + # graph optimizer + self.graph_optimizer = instantiate(self.graph_optimizer_cfg)(self.graph.parameters) + + @property + def causal_graph(self) -> torch.Tensor: + """property causal graph""" + assert self.graph is not None, "graph incorrectly initialized" + + return self.graph.get_binary_adj_matrix(threshold=0.5) + + def single_step_forward( + self, + inputs: MutableMapping[str, torch.Tensor], + train: bool = False, + mask: Optional[torch.Tensor] = None, + ) -> Dict[str, torch.Tensor]: + batch_size, extra_dim = self.get_inputs_batch_size(inputs) + assert len(extra_dim) == 0, "unexpected dimension in the inputs" + + inputs_tensor = torch.zeros(self.ensemble_num, batch_size, self.input_var_num, self.encoder_output_dim).to(self.device) + for i, var in enumerate(self.input_variables): + out = self.variable_encoders[var.name](inputs[var.name]) + inputs_tensor[..., i, :] = out + + if train: + # [ensemble-num, batch-size, input-var-num, output-var-num] + adj_matrix = self.graph.sample(None, sample_size=(self.ensemble_num, batch_size)) + # [ensemble-num, batch-size, output-var-num, input-var-num] + mask = adj_matrix.transpose(-1, -2) + # [output-var-num, ensemble-num, batch-size, input-var-num] + mask = mask.permute(2, 0, 1, 3) + else: + if mask is None: + mask = self.forward_mask + mask = mask.unsqueeze(-2).unsqueeze(-2) + mask = mask.repeat(1, self.ensemble_num, batch_size, 1) + + # [output-var-num, ensemble-num, batch-size, encoder-output-dim] + reduced_inputs_tensor = self.reduce_encoder_output(inputs_tensor, mask=mask) + if self._concat_mask: + # [output-var-num, ensemble-num, batch-size, encoder-output-dim + input-var-num] + reduced_inputs_tensor = torch.cat([reduced_inputs_tensor, mask], dim=-1) + output_tensor = self.network(reduced_inputs_tensor) + + outputs = {} + for i, var in enumerate(self.output_variables): + hid = output_tensor[i] + outputs[var.name] = self.variable_decoders[var.name](hid) + + if self.residual: + outputs = self.residual_outputs(inputs, outputs) + return outputs + + def forward( + self, + inputs: MutableMapping[str, torch.Tensor], + train: bool = False, + mask: Optional[torch.Tensor] = None, + ) -> Dict[str, torch.Tensor]: + if self.multi_step.startswith("forward-euler"): + step_num = int(self.multi_step.split()[-1]) + + outputs = {} + for step in range(step_num): + outputs = self.single_step_forward(inputs, train=train, mask=mask) + if step < step_num - 1: + for name in filter(lambda s: s.startswith("obs"), inputs.keys()): + inputs[name] = outputs["next_{}".format(name)][..., : inputs[name].shape[-1]] + else: + raise NotImplementedError("multi-step method {} is not supported".format(self.multi_step)) + + return outputs + + def train_graph(self, loader: DataLoader, data_ratio: float): + num_batches = len(loader) + train_num = int(num_batches * data_ratio) + + grads = torch.tensor([0], dtype=torch.float32) + for i, (inputs, targets) in enumerate(loader): + if train_num <= i: + break + + grads = grads + self._update_graph(inputs, targets) + + return grads + + def _update_graph( + self, + inputs: MutableMapping[str, torch.Tensor], + targets: MutableMapping[str, torch.Tensor], + ) -> torch.Tensor: + # do Monte-Carlo sampling to obtain adjacent matrices and corresponding model losses + adj_matrices, losses = self._MC_sample(inputs, targets) + + # calculate graph gradients + graph_grads = self._estimate_graph_grads(adj_matrices, losses) + + # update graph + graph_params = self.graph.parameters[0] # only one tensor parameter + self.graph_optimizer.zero_grad() + graph_params.grad = graph_grads + self.graph_optimizer.step() + + return graph_grads.detach().cpu() + + def _MC_sample( + self, + inputs: MutableMapping[str, torch.Tensor], + targets: MutableMapping[str, torch.Tensor], + ) -> Tuple[torch.Tensor]: + num_graph_list = [ + min(self._graph_max_stack, self._graph_MC_samples - i * self._graph_max_stack) + for i in range(math.ceil(self._graph_MC_samples / self._graph_max_stack)) + ] + num_graph_list = [(num_graph_list[i], sum(num_graph_list[:i])) for i in range(len(num_graph_list))] + + # sample graphs + adj_mats = self.graph.sample(None, sample_size=self._graph_MC_samples) + + # evaluate scores using the sampled adjacency matrices and data + batch_size, extra_dim = self.get_inputs_batch_size(inputs) + assert len(extra_dim) == 0, "unexpected dimension in the inputs" + + losses = [] + for graph_count, start_idx in num_graph_list: + # [ensemble-num, samples*batch_size, input-var-num, output-var-num] + expanded_adj_mats = ( + adj_mats[None, start_idx : start_idx + graph_count, None] + .expand(self.ensemble_num, -1, batch_size, -1, -1) + .flatten(1, 2) + ) + expanded_masks = expanded_adj_mats.transpose(-1, -2).permute(2, 0, 1, 3) + + expanded_inputs = {} + expanded_targets = {} + # expand inputs and targets + for in_key in inputs: + expanded_inputs[in_key] = inputs[in_key].repeat(1, graph_count, 1) + for tar_key in targets: + expanded_targets[tar_key] = targets[tar_key].repeat(1, graph_count, 1) + + with torch.no_grad(): + outputs = self.forward(expanded_inputs, train=False, mask=expanded_masks) + loss = variable_loss_func(outputs, expanded_targets, self.output_variables) + loss = loss.reshape(loss.shape[0], graph_count, batch_size, -1) + losses.append(loss.mean(dim=(0, 2))) + losses = sum(losses) + + return adj_mats, losses + + def _estimate_graph_grads( + self, + adj_matrices: torch.Tensor, + losses: torch.Tensor, + ) -> torch.Tensor: + """Use MC samples and corresponding losses to estimate gradients via REINFORCE. + + Args: + adj_matrices (tensor): MC sampled adjacent matrices from current graph, + shaped [num-samples, input-var-num, output-var-num]. + losses (tensor): the model losses corresponding to the adjacent matrices, + shaped [num-samples, output-var-num] + + """ + num_graphs = adj_matrices.shape[0] + losses = losses.unsqueeze(dim=1) + + # calculate graph gradients + edge_prob = self.graph.get_adj_matrix() + num_pos = adj_matrices.sum(dim=0) + num_neg = num_graphs - num_pos + mask = ((num_pos > 0) * (num_neg > 0)).float() + pos_grads = (losses * adj_matrices).sum(dim=0) / num_pos.clamp_(min=1e-5) + neg_grads = (losses * (1 - adj_matrices)).sum(dim=0) / num_neg.clamp_(min=1e-5) + graph_grads = mask * edge_prob * (1 - edge_prob) * (pos_grads - neg_grads + self._lambda_sparse) + + return graph_grads + + def learn( + self, + train_loader: DataLoader, + valid_loader: DataLoader, + longest_epoch: int = -1, + improvement_threshold: float = 0.01, + patience: int = 5, + graph_data_ratio: float = 0.5, + train_graph_freq: int = 2, + work_dir: Optional[Union[str, pathlib.Path]] = None, + **kwargs + ): + assert 0 <= graph_data_ratio <= 1, "graph data ratio should be in [0, 1]" + + best_weights: Optional[Dict] = None + epoch_iter = range(longest_epoch) if longest_epoch >= 0 else count() + epochs_since_update = 0 + + loss_fn = partial(variable_loss_func, output_variables=self.output_variables, device=self.device) + train_fn = partial(train_func, forward=partial(self.forward, train=True), optimizer=self.optimizer, loss_func=loss_fn) + eval_fn = partial(eval_func, forward=partial(self.forward, train=False), loss_func=loss_fn) + + best_eval_loss = eval_fn(valid_loader).mean(dim=(-2, -1)) + for epoch in epoch_iter: + if epoch % train_graph_freq == 0: + grads = self.train_graph(train_loader, data_ratio=graph_data_ratio) + + train_loss = train_fn(train_loader) + eval_loss = eval_fn(valid_loader) + + maybe_best_weights = self._maybe_get_best_weights( + best_eval_loss, eval_loss.mean(dim=(-2, -1)), improvement_threshold + ) + if maybe_best_weights: + # best loss + best_eval_loss = torch.minimum(best_eval_loss, eval_loss.mean(dim=(-2, -1))) + best_weights = maybe_best_weights + epochs_since_update = 0 + else: + epochs_since_update += 1 + + # log + self.total_epoch += 1 + if self.logger is not None: + self.logger.record("{}/epoch".format(self.name), epoch) + self.logger.record("{}/epoch_since_update".format(self.name), epochs_since_update) + self.logger.record("{}/train_dataset_size".format(self.name), len(train_loader.dataset)) + self.logger.record("{}/valid_dataset_size".format(self.name), len(valid_loader.dataset)) + self.logger.record("{}/train_loss".format(self.name), train_loss.mean().item()) + self.logger.record("{}/val_loss".format(self.name), eval_loss.mean().item()) + self.logger.record("{}/best_val_loss".format(self.name), best_eval_loss.mean().item()) + + if epoch % train_graph_freq == 0: + self.logger.record("{}/graph_update_grads".format(self.name), grads.abs().mean().item()) + + self.logger.dump(self.total_epoch) + + if patience and epochs_since_update >= patience: + break + + # saving the best models + self._maybe_set_best_weights_and_elite(best_weights, best_eval_loss) + + def _maybe_get_best_weights( + self, best_val_loss: torch.Tensor, val_loss: torch.Tensor, threshold: float = 0.01 + ) -> Optional[Dict]: + improvement = (best_val_loss - val_loss) / torch.abs(best_val_loss) + if (improvement > threshold).any().item(): + best_weights = { + "graph": copy.deepcopy(self.graph.parameters[0].detach().clone()), + "model": copy.deepcopy(self.network.state_dict()), + } + else: + best_weights = None + + return best_weights + + def _maybe_set_best_weights_and_elite(self, best_weights: Optional[Dict], best_val_score: torch.Tensor): + if best_weights is not None: + self.network.load_state_dict(best_weights["model"]) + self.graph.set_data(best_weights["graph"]) + + sorted_indices = np.argsort(best_val_score.tolist()) + self.elite_indices = sorted_indices[: self.elite_num] diff --git a/cmrl/models/dynamics.py b/cmrl/models/dynamics.py index f8acb5a..4ac1dad 100644 --- a/cmrl/models/dynamics.py +++ b/cmrl/models/dynamics.py @@ -51,7 +51,7 @@ def get_loader(self, real_replay_buffer, mech: str, batch_size: int = 128): ensemble_num=self.transition.ensemble_num, seed=self.seed, ) - train_loader = DataLoader(train_dataset, batch_size=batch_size, collate_fn=collate_fn) + train_loader = DataLoader(train_dataset, batch_size=batch_size, collate_fn=collate_fn, shuffle=True) valid_dataset = BufferDataset( real_replay_buffer, self.observation_space, diff --git a/tests/test_models/test_causal_mech/test_CMI_test.py b/tests/test_models/test_causal_mech/test_CMI_test.py index 73be7cd..f38041f 100644 --- a/tests/test_models/test_causal_mech/test_CMI_test.py +++ b/tests/test_models/test_causal_mech/test_CMI_test.py @@ -49,7 +49,7 @@ def test_mask(): ) for inputs, targets in train_loader: - batch_size = mech.get_inputs_batch_size(inputs) + batch_size, extra_dim = mech.get_inputs_batch_size(inputs) inputs_tensor = torch.zeros(mech.ensemble_num, batch_size, mech.input_var_num, mech.encoder_output_dim).to(mech.device) for i, var in enumerate(mech.input_variables): @@ -58,17 +58,19 @@ def test_mask(): mask = None masked_inputs_tensor = mech.reduce_encoder_output(inputs_tensor, mask) - assert masked_inputs_tensor.shape == (mech.ensemble_num, batch_size, mech.encoder_output_dim) + assert masked_inputs_tensor.shape == (mech.output_var_num, mech.ensemble_num, batch_size, mech.encoder_output_dim) - mask = torch.ones(mech.input_var_num).to(mech.device) + mask = torch.ones(mech.ensemble_num, batch_size, mech.input_var_num).to(mech.device) masked_inputs_tensor = mech.reduce_encoder_output(inputs_tensor, mask) assert masked_inputs_tensor.shape == (mech.ensemble_num, batch_size, mech.encoder_output_dim) - mask = torch.ones(mech.output_var_num, mech.input_var_num).to(mech.device) + mask = torch.ones(mech.output_var_num, mech.ensemble_num, batch_size, mech.input_var_num).to(mech.device) masked_inputs_tensor = mech.reduce_encoder_output(inputs_tensor, mask) assert masked_inputs_tensor.shape == (mech.output_var_num, mech.ensemble_num, batch_size, mech.encoder_output_dim) - mask = torch.ones(mech.input_var_num + 1, mech.output_var_num, mech.input_var_num).to(mech.device) + mask = torch.ones(mech.input_var_num + 1, mech.output_var_num, mech.ensemble_num, batch_size, mech.input_var_num).to( + mech.device + ) masked_inputs_tensor = mech.reduce_encoder_output(inputs_tensor, mask) assert masked_inputs_tensor.shape == ( mech.input_var_num + 1, diff --git a/tests/test_models/test_causal_mech/test_plain_mech.py b/tests/test_models/test_causal_mech/test_plain_mech.py index 58257b5..69b0d52 100644 --- a/tests/test_models/test_causal_mech/test_plain_mech.py +++ b/tests/test_models/test_causal_mech/test_plain_mech.py @@ -60,3 +60,7 @@ def test_inv_pendulum_multi_step(): ) mech.learn(train_loader, valid_loader, longest_epoch=1) + + +if __name__ == "__main__": + test_inv_pendulum_multi_step() diff --git a/tests/test_models/test_causal_mech/test_reinforce.py b/tests/test_models/test_causal_mech/test_reinforce.py new file mode 100644 index 0000000..feae1f4 --- /dev/null +++ b/tests/test_models/test_causal_mech/test_reinforce.py @@ -0,0 +1,176 @@ +import gym +from stable_baselines3.common.buffers import ReplayBuffer +import torch +from torch.utils.data import DataLoader + +from cmrl.models.causal_mech.reinforce import ReinforceCausalMech +from cmrl.models.data_loader import BufferDataset, EnsembleBufferDataset, collate_fn +from cmrl.utils.creator import parse_space +from cmrl.utils.env import load_offline_data +from cmrl.models.causal_mech.util import variable_loss_func + + +def prepare(freq_rate): + env = gym.make("BoundaryInvertedPendulumSwingUp-v0", freq_rate=freq_rate, time_step=0.02) + + real_replay_buffer = ReplayBuffer( + 1000, env.observation_space, env.action_space, device="cpu", handle_timeout_termination=False + ) + load_offline_data(env, real_replay_buffer, "SAC-expert-replay", use_ratio=0.001) + + ensemble_num = 7 + # test for transition + train_dataset = EnsembleBufferDataset( + real_replay_buffer, + env.observation_space, + env.action_space, + is_valid=False, + mech="transition", + train_ensemble=True, + ensemble_num=ensemble_num, + ) + train_loader = DataLoader(train_dataset, batch_size=8, collate_fn=collate_fn) + valid_dataset = BufferDataset( + real_replay_buffer, env.observation_space, env.action_space, is_valid=True, mech="transition", repeat=ensemble_num + ) + valid_loader = DataLoader(valid_dataset, batch_size=8, collate_fn=collate_fn) + + input_variables = parse_space(env.observation_space, "obs") + parse_space(env.action_space, "act") + output_variables = parse_space(env.observation_space, "next_obs") + + return input_variables, output_variables, train_loader, valid_loader + + +def test_init(): + input_variables, output_variables, _, _ = prepare(freq_rate=1) + + mech = ReinforceCausalMech( + name="test", + input_variables=input_variables, + output_variables=output_variables, + ) + + assert mech.network, "network incorrectly initialized" + assert mech.graph, "graph incorrectly initialized" + assert mech.optimizer, "network optimizer incorrectly initialized" + assert mech.graph_optimizer, "graph optimizer incorrectly initialized" + + +def test_causal_graph(): + input_variables, output_variables, _, _ = prepare(freq_rate=1) + + mech = ReinforceCausalMech( + name="test", + input_variables=input_variables, + output_variables=output_variables, + ) + + assert mech.causal_graph is not None, "cannot obtain causal graph correctly" + assert mech.causal_graph.all(), "incorrect initial causal graph" + + +def test_single_step_forward(): + input_variables, output_variables, train_loader, _ = prepare(freq_rate=1) + + mech = ReinforceCausalMech( + name="test", + input_variables=input_variables, + output_variables=output_variables, + ) + train_iter = iter(train_loader) + inputs, targets = next(train_iter) + + # test training + outputs = mech.single_step_forward(inputs, train=True) + assert len(outputs.keys() ^ targets.keys()) == 0, "single forward output keys mismatch when training" + loss = variable_loss_func(outputs, targets, mech.output_variables) + assert loss is not None, "single forward output is incorrect when training" + + # test evaluating + outputs = mech.single_step_forward(inputs, train=False) + assert len(outputs.keys() ^ targets.keys()) == 0, "single forward output keys mismatch when evaluating" + loss = variable_loss_func(outputs, targets, mech.output_variables) + assert loss is not None, "single forward outupt is incorrect when evaluating" + + # test evaluating with fixed mask + extra_dims = next(iter(inputs.values())).shape[:-1] + mask = torch.randint(0, 2, size=(len(targets), *extra_dims, len(inputs))) + outputs = mech.single_step_forward(inputs, train=False, mask=mask) + assert len(outputs.keys() ^ targets.keys()) == 0, "single forward output keys mismatch when evaluating with given mask" + loss = variable_loss_func(outputs, targets, mech.output_variables) + assert loss is not None, "single forward output is incorrect when evaluating with given mask" + + +def test_forward(): + input_variables, output_variables, train_loader, _ = prepare(freq_rate=1) + + # test single step + mech = ReinforceCausalMech( + name="test", + input_variables=input_variables, + output_variables=output_variables, + ) + train_iter = iter(train_loader) + inputs, targets = next(train_iter) + + outputs = mech.forward(inputs, train=True) + assert len(outputs.keys() ^ targets.keys()) == 0, "forward output keys mismatch when evaluating (single step)" + loss = variable_loss_func(outputs, targets, mech.output_variables) + assert loss is not None, "forward outupt is incorrect when evaluating (single step)" + + # test multi-step + mech = ReinforceCausalMech( + name="test", input_variables=input_variables, output_variables=output_variables, multi_step="forward-euler 2" + ) + train_iter = iter(train_loader) + inputs, targets = next(train_iter) + + outputs = mech.forward(inputs, train=True) + assert len(outputs.keys() ^ targets.keys()) == 0, "forward output keys mismatch when evaluating (2 steps)" + loss = variable_loss_func(outputs, targets, mech.output_variables) + assert loss is not None, "forward outupt is incorrect when evaluating (2 steps)" + + +def test_train_graph(): + input_variables, output_variables, train_loader, _ = prepare(freq_rate=1) + + # test single step + mech = ReinforceCausalMech( + name="test", + input_variables=input_variables, + output_variables=output_variables, + ) + + grads = mech.train_graph(train_loader, data_ratio=1.0) + assert grads is not None, "train graph single step failed" + + # test multi-step + mech = ReinforceCausalMech( + name="test", + input_variables=input_variables, + output_variables=output_variables, + multi_step="forward-euler 2", + ) + + grads = mech.train_graph(train_loader, data_ratio=0.5) + assert grads is not None, "train graph multi-step (2) failed" + + +def test_learn(): + input_variables, output_variables, train_loader, valid_loader = prepare(freq_rate=1) + + # test single step + mech = ReinforceCausalMech( + name="test", + input_variables=input_variables, + output_variables=output_variables, + ) + + mech.learn(train_loader, valid_loader) + + # test multi-step + mech = ReinforceCausalMech( + name="test", input_variables=input_variables, output_variables=output_variables, multi_step="forward-euler 2" + ) + + mech.learn(train_loader, valid_loader) diff --git a/tests/test_models/test_graphs/test_binary_graph.py b/tests/test_models/test_graphs/test_binary_graph.py index 10ce251..ff224d1 100644 --- a/tests/test_models/test_graphs/test_binary_graph.py +++ b/tests/test_models/test_graphs/test_binary_graph.py @@ -1,3 +1,7 @@ +import os +import time +import shutil + import torch from cmrl.models.graphs.binary_graph import BinaryGraph @@ -60,10 +64,20 @@ def test_set_data(): def test_save_load(): + # create a temp folder + while True: + save_dir = "./tmp" + str(time.time()) + if not os.path.exists(save_dir): + os.mkdir(save_dir) + break + g = BinaryGraph(5, 5, include_input=True, init_param=1) - g.save("./") + g.save(save_dir) old_graph = g.graph - g.load("./") + g.load(save_dir) assert g.graph is not old_graph assert g.graph.equal(old_graph) + + # clear the temp folder + shutil.rmtree(save_dir) diff --git a/tests/test_models/test_graphs/test_neural_graph.py b/tests/test_models/test_graphs/test_neural_graph.py index 4d385fc..f4cca42 100644 --- a/tests/test_models/test_graphs/test_neural_graph.py +++ b/tests/test_models/test_graphs/test_neural_graph.py @@ -1,3 +1,7 @@ +import os +import time +import shutil + import torch import torch.nn as nn @@ -35,15 +39,25 @@ def test_adj_matrix(): def test_save_load(): + # create a temp folder + while True: + save_dir = "./tmp" + str(time.time()) + if not os.path.exists(save_dir): + os.mkdir(save_dir) + break + g = NeuralGraph(5, 5, include_input=True) - g.save("./") + g.save(save_dir) old_graph = g.graph state_dict = old_graph.state_dict() - g.load("./") + g.load(save_dir) assert g.graph is old_graph assert g.graph.state_dict() is not state_dict + # clear the temp folder + shutil.rmtree(save_dir) + if __name__ == "__main__": test_init() diff --git a/tests/test_models/test_graphs/test_weight_graph.py b/tests/test_models/test_graphs/test_weight_graph.py index 3926091..171f46b 100644 --- a/tests/test_models/test_graphs/test_weight_graph.py +++ b/tests/test_models/test_graphs/test_weight_graph.py @@ -1,4 +1,6 @@ import os +import time +import shutil import torch @@ -101,10 +103,20 @@ def test_grad(): def test_save_load(): + # create a temp folder + while True: + save_dir = "./tmp" + str(time.time()) + if not os.path.exists(save_dir): + os.mkdir(save_dir) + break + g = WeightGraph(5, 5, include_input=True, init_param=0.5) - g.save("./") + g.save(save_dir) old_graph = g.graph - g.load("./") + g.load(save_dir) assert g.graph is not old_graph assert g.graph.equal(old_graph) + + # clear the temp folder + shutil.rmtree(save_dir) From e168b2f7e610e645f2df211ecbd453396ae4c978 Mon Sep 17 00:00:00 2001 From: wz139704646 <632291793@qq.com> Date: Mon, 21 Nov 2022 11:46:04 +0800 Subject: [PATCH 30/68] :wrench: add reinforce config file, fix some bugs in reinforce and loss util, add reinforce test using cuda device --- cmrl/examples/conf/transition/reinforce.yaml | 77 +++++++++++++++++++ cmrl/models/causal_mech/__init__.py | 1 + cmrl/models/causal_mech/reinforce.py | 4 +- cmrl/models/causal_mech/util.py | 2 +- .../test_causal_mech/test_reinforce.py | 15 +++- 5 files changed, 95 insertions(+), 4 deletions(-) create mode 100644 cmrl/examples/conf/transition/reinforce.yaml diff --git a/cmrl/examples/conf/transition/reinforce.yaml b/cmrl/examples/conf/transition/reinforce.yaml new file mode 100644 index 0000000..08d1bc2 --- /dev/null +++ b/cmrl/examples/conf/transition/reinforce.yaml @@ -0,0 +1,77 @@ +name: "CMI_test_transition" +learn: true + +encoder_cfg: + _partial_: true + _recursive_: false + _target_: cmrl.models.networks.VariableEncoder + output_dim: 100 + hidden_dims: [ 100 ] + bias: true + activation_fn_cfg: + _target_: torch.nn.SiLU + +decoder_cfg: + _partial_: true + _recursive_: false + _target_: cmrl.models.networks.VariableDecoder + identity: false + input_dim: 100 + hidden_dims: [ 100 ] + bias: true + activation_fn_cfg: + _target_: torch.nn.SiLU + +network_cfg: + _partial_: true + _recursive_: false + _target_: cmrl.models.networks.ParallelMLP + hidden_dims: [ 200, 200 ] + bias: true + activation_fn_cfg: + _target_: torch.nn.SiLU + +optimizer_cfg: + _partial_: true + _target_: torch.optim.Adam + lr: 1e-4 + weight_decay: 1e-5 + eps: 1e-8 + +graph_optimizer_cfg: + _partial_: true + _target_: torch.optim.Adam + lr: 1e-3 + weight_decay: 0.0 + eps: 1e-8 + +mech: + _partial_: true + _recursive_: false + _target_: cmrl.models.causal_mech.ReinforceCausalMech + # base causal-mech params + name: transition + input_variables: ??? + output_variables: ??? + # ensemble + ensemble_num: 7 + elite_num: 5 + # cfgs + network_cfg: ${transition.network_cfg} + encoder_cfg: ${transition.encoder_cfg} + decoder_cfg: ${transition.decoder_cfg} + optimizer_cfg: ${transition.optimizer_cfg} + graph_optimizer_cfg: ${transition.graph_optimizer_cfg} + # graph params + concat_mask: true + graph_MC_samples: 20 + graph_max_stack: 20 + lambda_sparse: 1e-3 + # forward method + residual: true + encoder_reduction: "sum" + multi_step: "forward-euler 1" + # logger + logger: ??? + # others + device: ${device} diff --git a/cmrl/models/causal_mech/__init__.py b/cmrl/models/causal_mech/__init__.py index 5f93938..2b5c138 100644 --- a/cmrl/models/causal_mech/__init__.py +++ b/cmrl/models/causal_mech/__init__.py @@ -1,2 +1,3 @@ from cmrl.models.causal_mech.plain_mech import PlainMech from cmrl.models.causal_mech.CMI_test import CMITest +from cmrl.models.causal_mech.reinforce import ReinforceCausalMech diff --git a/cmrl/models/causal_mech/reinforce.py b/cmrl/models/causal_mech/reinforce.py index e784879..5af24d7 100644 --- a/cmrl/models/causal_mech/reinforce.py +++ b/cmrl/models/causal_mech/reinforce.py @@ -137,7 +137,7 @@ def single_step_forward( inputs_tensor = torch.zeros(self.ensemble_num, batch_size, self.input_var_num, self.encoder_output_dim).to(self.device) for i, var in enumerate(self.input_variables): - out = self.variable_encoders[var.name](inputs[var.name]) + out = self.variable_encoders[var.name](inputs[var.name].to(self.device)) inputs_tensor[..., i, :] = out if train: @@ -259,7 +259,7 @@ def _MC_sample( with torch.no_grad(): outputs = self.forward(expanded_inputs, train=False, mask=expanded_masks) - loss = variable_loss_func(outputs, expanded_targets, self.output_variables) + loss = variable_loss_func(outputs, expanded_targets, self.output_variables, device=self.device) loss = loss.reshape(loss.shape[0], graph_count, batch_size, -1) losses.append(loss.mean(dim=(0, 2))) losses = sum(losses) diff --git a/cmrl/models/causal_mech/util.py b/cmrl/models/causal_mech/util.py index 392e282..dbe647f 100644 --- a/cmrl/models/causal_mech/util.py +++ b/cmrl/models/causal_mech/util.py @@ -16,7 +16,7 @@ def variable_loss_func( device: Union[str, torch.device] = "cpu", ): dims = list(outputs.values())[0].shape[:-1] - total_loss = torch.zeros(*dims, len(outputs)) + total_loss = torch.zeros(*dims, len(outputs)).to(device) for i, var in enumerate(output_variables): output = outputs[var.name] diff --git a/tests/test_models/test_causal_mech/test_reinforce.py b/tests/test_models/test_causal_mech/test_reinforce.py index feae1f4..47fd9f9 100644 --- a/tests/test_models/test_causal_mech/test_reinforce.py +++ b/tests/test_models/test_causal_mech/test_reinforce.py @@ -170,7 +170,20 @@ def test_learn(): # test multi-step mech = ReinforceCausalMech( - name="test", input_variables=input_variables, output_variables=output_variables, multi_step="forward-euler 2" + name="test", + input_variables=input_variables, + output_variables=output_variables, + multi_step="forward-euler 2", + ) + + mech.learn(train_loader, valid_loader) + + # test cuda + mech = ReinforceCausalMech( + name="test", + input_variables=input_variables, + output_variables=output_variables, + device="cuda:0", ) mech.learn(train_loader, valid_loader) From be1ccf107e0e974445b8fa504312df159e1a45b0 Mon Sep 17 00:00:00 2001 From: frank Date: Wed, 23 Nov 2022 09:43:11 +0800 Subject: [PATCH 31/68] :tada: update configuration --- cmrl/examples/conf/reward_mech/plain.yaml | 1 + .../examples/conf/termination_mech/plain.yaml | 1 + cmrl/examples/conf/transition/CMI_test.yaml | 1 + cmrl/examples/conf/transition/plain.yaml | 1 + cmrl/models/causal_mech/CMI_test.py | 36 ++++++++++--------- cmrl/models/causal_mech/base_causal_mech.py | 4 +++ cmrl/models/dynamics.py | 2 ++ 7 files changed, 30 insertions(+), 16 deletions(-) diff --git a/cmrl/examples/conf/reward_mech/plain.yaml b/cmrl/examples/conf/reward_mech/plain.yaml index 1e447d2..75a4387 100644 --- a/cmrl/examples/conf/reward_mech/plain.yaml +++ b/cmrl/examples/conf/reward_mech/plain.yaml @@ -1,5 +1,6 @@ name: "plain_reward_mech" learn: false +discovery: false encoder_cfg: _partial_: true diff --git a/cmrl/examples/conf/termination_mech/plain.yaml b/cmrl/examples/conf/termination_mech/plain.yaml index 44b97e5..2b81870 100644 --- a/cmrl/examples/conf/termination_mech/plain.yaml +++ b/cmrl/examples/conf/termination_mech/plain.yaml @@ -1,5 +1,6 @@ name: "plain_termination_mech" learn: false +discovery: false encoder_cfg: _partial_: true diff --git a/cmrl/examples/conf/transition/CMI_test.yaml b/cmrl/examples/conf/transition/CMI_test.yaml index 945d5c4..ab2aaab 100644 --- a/cmrl/examples/conf/transition/CMI_test.yaml +++ b/cmrl/examples/conf/transition/CMI_test.yaml @@ -1,5 +1,6 @@ name: "CMI_test_transition" learn: true +discovery: false encoder_cfg: _partial_: true diff --git a/cmrl/examples/conf/transition/plain.yaml b/cmrl/examples/conf/transition/plain.yaml index 6104811..efde025 100644 --- a/cmrl/examples/conf/transition/plain.yaml +++ b/cmrl/examples/conf/transition/plain.yaml @@ -1,5 +1,6 @@ name: "plain_transition" learn: true +discovery: false encoder_cfg: _partial_: true diff --git a/cmrl/models/causal_mech/CMI_test.py b/cmrl/models/causal_mech/CMI_test.py index a285e3a..bae6c2c 100644 --- a/cmrl/models/causal_mech/CMI_test.py +++ b/cmrl/models/causal_mech/CMI_test.py @@ -72,7 +72,7 @@ def build_graph(self): self.graph = None def single_step_forward(self, inputs: MutableMapping[str, torch.Tensor]) -> Dict[str, torch.Tensor]: - batch_size = self.get_inputs_batch_size(inputs) + batch_size, _ = self.get_inputs_batch_size(inputs) inputs_tensor = torch.zeros(self.ensemble_num, batch_size, self.input_var_num, self.encoder_output_dim).to(self.device) for i, var in enumerate(self.input_variables): @@ -193,11 +193,10 @@ def CMI_forward(self, inputs: MutableMapping[str, torch.Tensor]) -> Dict[str, to return outputs def calculate_CMI(self, nll_loss: torch.Tensor): - print("fc loss", nll_loss[-1].mean(dim=(0, 1))) - print("mask", nll_loss[:-1].mean(dim=(1, 2))) nll_loss_diff = nll_loss[:-1] - nll_loss[-1] - print("cmi", nll_loss_diff.mean(dim=(1, 2))) - pass + self.forward_mask = (nll_loss_diff.mean(dim=(1, 2)) > 1).to(torch.long) + + print(self.forward_mask) def learn( self, @@ -223,12 +222,13 @@ def learn( for epoch in epoch_iter: train_loss = train(train_loader) eval_loss = eval(valid_loader) - self.calculate_CMI(eval_loss) improvement = (best_eval_loss - eval_loss.mean(dim=(0, 2, 3))) / torch.abs(best_eval_loss) if (improvement > improvement_threshold).any().item(): best_eval_loss = torch.minimum(best_eval_loss, eval_loss.mean(dim=(0, 2, 3))) epochs_since_update = 0 + + self.calculate_CMI(eval_loss) else: epochs_since_update += 1 @@ -248,13 +248,17 @@ def learn( if patience and epochs_since_update >= patience: break - # super(CMITest, self).learn( - # train_loader=train_loader, - # valid_loader=valid_loader, - # # model learning - # longest_epoch=longest_epoch, - # improvement_threshold=improvement_threshold, - # patience=patience, - # work_dir=work_dir, - # **kwargs - # ) + super(CMITest, self).learn( + train_loader=train_loader, + valid_loader=valid_loader, + # model learning + longest_epoch=longest_epoch, + improvement_threshold=improvement_threshold, + patience=patience, + work_dir=work_dir, + **kwargs + ) + + def set_oracle_graph(self, graph): + self._oracle_graph = graph + pass diff --git a/cmrl/models/causal_mech/base_causal_mech.py b/cmrl/models/causal_mech/base_causal_mech.py index 19077dd..753cdb7 100644 --- a/cmrl/models/causal_mech/base_causal_mech.py +++ b/cmrl/models/causal_mech/base_causal_mech.py @@ -33,6 +33,10 @@ def learn(self, train_loader: DataLoader, valid_loader: DataLoader, **kwargs): def forward(self, inputs: MutableMapping[str, torch.Tensor]) -> Dict[str, torch.Tensor]: raise NotImplementedError + @abstractmethod + def set_oracle_graph(self, graph): + pass + @property def causal_graph(self) -> torch.Tensor: """property causal graph""" diff --git a/cmrl/models/dynamics.py b/cmrl/models/dynamics.py index 4ac1dad..cf843e8 100644 --- a/cmrl/models/dynamics.py +++ b/cmrl/models/dynamics.py @@ -113,3 +113,5 @@ def step(self, batch_obs, batch_action): info = {"origin-next_obs": torch.concat([tensor[:, :, :1] for tensor in outputs.values()], dim=-1).cpu().numpy()} return torch.concat([tensor.mean(dim=0)[:, :1] for tensor in outputs.values()], dim=-1).cpu().numpy(), None, None, info + + # def set_oracle_graph(self, graph): From 2927e04dff1964d54d5572a837a8d61cc2b80a1c Mon Sep 17 00:00:00 2001 From: wz139704646 <632291793@qq.com> Date: Fri, 25 Nov 2022 16:09:04 +0800 Subject: [PATCH 32/68] :bug: fix cmi test bug in single step forward --- cmrl/examples/conf/transition/reinforce.yaml | 4 ++-- cmrl/models/causal_mech/reinforce.py | 2 ++ cmrl/models/graphs/prob_graph.py | 2 +- .../test_causal_mech/test_reinforce.py | 23 ++++++++++++++----- 4 files changed, 22 insertions(+), 9 deletions(-) diff --git a/cmrl/examples/conf/transition/reinforce.yaml b/cmrl/examples/conf/transition/reinforce.yaml index 08d1bc2..dac8e51 100644 --- a/cmrl/examples/conf/transition/reinforce.yaml +++ b/cmrl/examples/conf/transition/reinforce.yaml @@ -1,4 +1,4 @@ -name: "CMI_test_transition" +name: "reinforce_transition" learn: true encoder_cfg: @@ -66,7 +66,7 @@ mech: concat_mask: true graph_MC_samples: 20 graph_max_stack: 20 - lambda_sparse: 1e-3 + lambda_sparse: 5e-2 # forward method residual: true encoder_reduction: "sum" diff --git a/cmrl/models/causal_mech/reinforce.py b/cmrl/models/causal_mech/reinforce.py index 5af24d7..b973df8 100644 --- a/cmrl/models/causal_mech/reinforce.py +++ b/cmrl/models/causal_mech/reinforce.py @@ -320,6 +320,8 @@ def learn( for epoch in epoch_iter: if epoch % train_graph_freq == 0: grads = self.train_graph(train_loader, data_ratio=graph_data_ratio) + print(self.graph.parameters[0]) + print(self.graph.get_binary_adj_matrix()) train_loss = train_fn(train_loader) eval_loss = eval_fn(valid_loader) diff --git a/cmrl/models/graphs/prob_graph.py b/cmrl/models/graphs/prob_graph.py index 90ea326..b7ee533 100644 --- a/cmrl/models/graphs/prob_graph.py +++ b/cmrl/models/graphs/prob_graph.py @@ -75,7 +75,7 @@ def __init__( def get_adj_matrix(self, *args, **kwargs) -> torch.Tensor: return torch.sigmoid(self.graph) - def get_binary_adj_matrix(self, threshold: float, *args, **kwargs) -> torch.Tensor: + def get_binary_adj_matrix(self, threshold: float = 0.5, *args, **kwargs) -> torch.Tensor: assert 0 <= threshold <= 1, "threshold of bernoulli graph should be in [0, 1]" return super().get_binary_adj_matrix(threshold, *args, **kwargs) diff --git a/tests/test_models/test_causal_mech/test_reinforce.py b/tests/test_models/test_causal_mech/test_reinforce.py index 47fd9f9..d5bf094 100644 --- a/tests/test_models/test_causal_mech/test_reinforce.py +++ b/tests/test_models/test_causal_mech/test_reinforce.py @@ -159,18 +159,18 @@ def test_train_graph(): def test_learn(): input_variables, output_variables, train_loader, valid_loader = prepare(freq_rate=1) - # test single step + # test single step on cpu mech = ReinforceCausalMech( - name="test", + name="test single on cpu", input_variables=input_variables, output_variables=output_variables, ) mech.learn(train_loader, valid_loader) - # test multi-step + # test multi-step on cpu mech = ReinforceCausalMech( - name="test", + name="test multi on cpu", input_variables=input_variables, output_variables=output_variables, multi_step="forward-euler 2", @@ -178,11 +178,22 @@ def test_learn(): mech.learn(train_loader, valid_loader) - # test cuda + # test single step on cuda mech = ReinforceCausalMech( - name="test", + name="test single on cuda", + input_variables=input_variables, + output_variables=output_variables, + device="cuda:0", + ) + + mech.learn(train_loader, valid_loader) + + # test multi-step on cuda + mech = ReinforceCausalMech( + name="test multi on cuda", input_variables=input_variables, output_variables=output_variables, + multi_step="forward-euler 2", device="cuda:0", ) From c5bdfb1ae79acca8dad1c910a7ca1b2f6e5f7b26 Mon Sep 17 00:00:00 2001 From: frank Date: Mon, 28 Nov 2022 15:54:12 +0800 Subject: [PATCH 33/68] :wrench: update .gitignore --- .gitignore | 2 -- 1 file changed, 2 deletions(-) diff --git a/.gitignore b/.gitignore index a25976e..0ad959e 100644 --- a/.gitignore +++ b/.gitignore @@ -138,6 +138,4 @@ cython_debug/ .vscode .idea -/cmrl.egg-info/ /exp/ -/stable-baselines3/ From 3fe361288e2b7fcbd35a7cba208c676eb21d2d20 Mon Sep 17 00:00:00 2001 From: frank Date: Mon, 28 Nov 2022 17:09:08 +0800 Subject: [PATCH 34/68] :wrench: update hydra's config --- cmrl/examples/conf/main.yaml | 4 ++-- cmrl/examples/conf/task/BI2PB.yaml | 11 +++++++--- cmrl/examples/conf/task/BI2PS.yaml | 11 ++++++++-- cmrl/examples/conf/task/BIPB.yaml | 11 +++++++--- cmrl/examples/conf/task/BIPS.yaml | 11 +++++++--- cmrl/examples/main.py | 2 ++ cmrl/utils/env.py | 33 +++++------------------------- 7 files changed, 42 insertions(+), 41 deletions(-) diff --git a/cmrl/examples/conf/main.yaml b/cmrl/examples/conf/main.yaml index 91ce51f..fb64a4f 100644 --- a/cmrl/examples/conf/main.yaml +++ b/cmrl/examples/conf/main.yaml @@ -15,8 +15,8 @@ wandb: false root_dir: "./exp" hydra: run: - dir: ${root_dir}/${exp_name}/${task.env}/${now:%Y.%m.%d}/${now:%H.%M.%S} + dir: ${root_dir}/${exp_name}/${task.env_id}/${to_str:${task.params}}/${task.dataset}/${now:%Y.%m.%d.%H%M%S} sweep: - dir: ${root_dir}/${exp_name}/${task.env}/${now:%Y.%m.%d}/${now:%H.%M.%S} + dir: ${root_dir}/${exp_name}/${task.env_id}/${to_str:${task.params}}/${task.dataset}/${now:%Y.%m.%d.%H%M%S} job: chdir: true diff --git a/cmrl/examples/conf/task/BI2PB.yaml b/cmrl/examples/conf/task/BI2PB.yaml index 8d13511..8684ca1 100644 --- a/cmrl/examples/conf/task/BI2PB.yaml +++ b/cmrl/examples/conf/task/BI2PB.yaml @@ -1,7 +1,12 @@ -env: "emei___BoundaryInvertedDoublePendulumBalancing-v0___freq_rate=${task.freq_rate}&time_step=${task.time_step}___${task.dataset}" +# env parameters +env_id: "BoundaryInvertedDoublePendulumBalancing-v0" + +params: + freq_rate: 1 + real_time_scale: 0.02 + integrator: "euler" + dataset: "SAC-expert-replay" -freq_rate: 1 -time_step: 0.02 # basic RL params num_steps: 3000000 diff --git a/cmrl/examples/conf/task/BI2PS.yaml b/cmrl/examples/conf/task/BI2PS.yaml index 06918ab..c5feda6 100644 --- a/cmrl/examples/conf/task/BI2PS.yaml +++ b/cmrl/examples/conf/task/BI2PS.yaml @@ -1,5 +1,12 @@ -env: "emei___BoundaryInvertedDoublePendulumSwingUp-v0___freq_rate=1&time_step=0.02___${task.dataset}" -dataset: "expert-replay" +# env parameters +env_id: "BoundaryInvertedDoublePendulumSwingUp-v0" + +params: + freq_rate: 1 + real_time_scale: 0.02 + integrator: "euler" + +dataset: "SAC-expert-replay" # basic RL params num_steps: 1000000 diff --git a/cmrl/examples/conf/task/BIPB.yaml b/cmrl/examples/conf/task/BIPB.yaml index a0536e4..321c5d2 100644 --- a/cmrl/examples/conf/task/BIPB.yaml +++ b/cmrl/examples/conf/task/BIPB.yaml @@ -1,7 +1,12 @@ -env: "emei___BoundaryInvertedPendulumBalancing-v0___freq_rate=${task.freq_rate}&time_step=${task.time_step}___${task.dataset}" +# env parameters +env_id: "BoundaryInvertedPendulumBalancing-v0" + +params: + freq_rate: 1 + real_time_scale: 0.02 + integrator: "euler" + dataset: "SAC-expert-replay" -freq_rate: 1 -time_step: 0.02 # basic RL params num_steps: 2000000 diff --git a/cmrl/examples/conf/task/BIPS.yaml b/cmrl/examples/conf/task/BIPS.yaml index 1b28ead..5c0e95b 100644 --- a/cmrl/examples/conf/task/BIPS.yaml +++ b/cmrl/examples/conf/task/BIPS.yaml @@ -1,7 +1,12 @@ -env: "emei___BoundaryInvertedPendulumSwingUp-v0___freq_rate=${task.freq_rate}&time_step=${task.time_step}___${task.dataset}" +# env parameters +env_id: "BoundaryInvertedPendulumSwingUp-v0" + +params: + freq_rate: 1 + real_time_scale: 0.02 + integrator: "euler" + dataset: "SAC-expert-replay" -freq_rate: 1 -time_step: 0.02 # basic RL params num_steps: 300000 diff --git a/cmrl/examples/main.py b/cmrl/examples/main.py index fac471d..544c018 100644 --- a/cmrl/examples/main.py +++ b/cmrl/examples/main.py @@ -2,6 +2,7 @@ from hydra.utils import instantiate import wandb from omegaconf import DictConfig, OmegaConf +from emei.core import get_params_str @hydra.main(version_base=None, config_path="conf", config_name="main") @@ -19,4 +20,5 @@ def run(cfg: DictConfig): if __name__ == "__main__": + OmegaConf.register_new_resolver("to_str", get_params_str) run() diff --git a/cmrl/utils/env.py b/cmrl/utils/env.py index 48bf4f1..af25ed3 100644 --- a/cmrl/utils/env.py +++ b/cmrl/utils/env.py @@ -10,37 +10,14 @@ from cmrl.types import TermFnType, RewardFnType, InitObsFnType -def to_num(s): - try: - return int(s) - except ValueError: - return float(s) - - -def get_term_and_reward_fn( - cfg: omegaconf.DictConfig, -) -> Tuple[Optional[TermFnType], Optional[RewardFnType]]: - return None, None - - def make_env( cfg: omegaconf.DictConfig, ) -> Tuple[emei.EmeiEnv, TermFnType, Optional[RewardFnType], Optional[InitObsFnType],]: - if "gym___" in cfg.task.env: - env = gym.make(cfg.task.env.split("___")[1]) - term_fn, reward_fn = get_term_and_reward_fn(cfg) - init_obs_fn = None - elif "emei___" in cfg.task.env: - env_name, params, = cfg.task.env.split( - "___" - )[1:3] - kwargs = dict([(item.split("=")[0], to_num(item.split("=")[1])) for item in params.split("&")]) - env = cast(emei.EmeiEnv, gym.make(env_name, **kwargs)) - reward_fn = env.get_reward - term_fn = env.get_terminal - init_obs_fn = env.get_batch_init_obs - else: - raise NotImplementedError + env = cast(emei.EmeiEnv, gym.make(cfg.task.env_id, **cfg.task.params)) + + reward_fn = env.get_batch_reward + term_fn = env.get_batch_terminal + init_obs_fn = env.get_batch_init_obs # set seed env.reset(seed=cfg.seed) From 1c929ba6fed9436b7014c85fedbc39274460dc35 Mon Sep 17 00:00:00 2001 From: frank Date: Mon, 28 Nov 2022 21:05:04 +0800 Subject: [PATCH 35/68] :tada: fit emei's oracle causal graph --- cmrl/algorithms/base_algorithm.py | 7 +++ cmrl/models/causal_mech/CMI_test.py | 63 ++++++++++----------- cmrl/models/causal_mech/base_causal_mech.py | 11 ++-- cmrl/models/causal_mech/reinforce.py | 3 +- cmrl/models/graphs/base_graph.py | 4 +- 5 files changed, 48 insertions(+), 40 deletions(-) diff --git a/cmrl/algorithms/base_algorithm.py b/cmrl/algorithms/base_algorithm.py index 3441eae..a4aba58 100644 --- a/cmrl/algorithms/base_algorithm.py +++ b/cmrl/algorithms/base_algorithm.py @@ -34,6 +34,13 @@ def __init__( # create ``cmrl`` dynamics self.dynamics = create_dynamics(self.cfg, self.env.observation_space, self.env.action_space, logger=self.logger) + if not self.cfg.transition.discovery: + self.dynamics.transition.set_oracle_graph(self.env.get_transition_graph()) + if self.cfg.reward_mech.learn and not self.cfg.reward_mech.discovery: + self.dynamics.reward_mech.set_oracle_graph(self.env.get_reward_mech_graph()) + if self.cfg.termination_mech.learn and not self.cfg.termination_mech.discovery: + self.dynamics.termination_mech.set_oracle_graph(self.env.get_termination_mech_graph()) + # create sb3's replay buffer for real offline data self.real_replay_buffer = ReplayBuffer( cfg.task.num_steps, diff --git a/cmrl/models/causal_mech/CMI_test.py b/cmrl/models/causal_mech/CMI_test.py index bae6c2c..5039d7a 100644 --- a/cmrl/models/causal_mech/CMI_test.py +++ b/cmrl/models/causal_mech/CMI_test.py @@ -210,43 +210,44 @@ def learn( work_dir: Optional[Union[str, pathlib.Path]] = None, **kwargs ): - epoch_iter = range(longest_epoch) if longest_epoch >= 0 else count() - epochs_since_update = 0 + if self.discovery: + epoch_iter = range(longest_epoch) if longest_epoch >= 0 else count() + epochs_since_update = 0 - loss_func = partial(variable_loss_func, output_variables=self.output_variables, device=self.device) - train = partial(train_func, forward=self.CMI_forward, optimizer=self.optimizer, loss_func=loss_func) - eval = partial(eval_func, forward=self.CMI_forward, loss_func=loss_func) + loss_func = partial(variable_loss_func, output_variables=self.output_variables, device=self.device) + train = partial(train_func, forward=self.CMI_forward, optimizer=self.optimizer, loss_func=loss_func) + eval = partial(eval_func, forward=self.CMI_forward, loss_func=loss_func) - best_eval_loss = eval(valid_loader).mean(dim=(0, 2, 3)) + best_eval_loss = eval(valid_loader).mean(dim=(0, 2, 3)) - for epoch in epoch_iter: - train_loss = train(train_loader) - eval_loss = eval(valid_loader) + for epoch in epoch_iter: + train_loss = train(train_loader) + eval_loss = eval(valid_loader) - improvement = (best_eval_loss - eval_loss.mean(dim=(0, 2, 3))) / torch.abs(best_eval_loss) - if (improvement > improvement_threshold).any().item(): - best_eval_loss = torch.minimum(best_eval_loss, eval_loss.mean(dim=(0, 2, 3))) - epochs_since_update = 0 + improvement = (best_eval_loss - eval_loss.mean(dim=(0, 2, 3))) / torch.abs(best_eval_loss) + if (improvement > improvement_threshold).any().item(): + best_eval_loss = torch.minimum(best_eval_loss, eval_loss.mean(dim=(0, 2, 3))) + epochs_since_update = 0 - self.calculate_CMI(eval_loss) - else: - epochs_since_update += 1 + self.calculate_CMI(eval_loss) + else: + epochs_since_update += 1 - # log - self.total_CMI_epoch += 1 - if self.logger is not None: - self.logger.record("{}-CMI-test/epoch".format(self.name), epoch) - self.logger.record("{}-CMI-test/epochs_since_update".format(self.name), epochs_since_update) - self.logger.record("{}-CMI-test/train_dataset_size".format(self.name), len(train_loader.dataset)) - self.logger.record("{}-CMI-test/valid_dataset_size".format(self.name), len(valid_loader.dataset)) - self.logger.record("{}-CMI-test/train_loss".format(self.name), train_loss.mean().item()) - self.logger.record("{}-CMI-test/val_loss".format(self.name), eval_loss.mean().item()) - self.logger.record("{}-CMI-test/best_val_loss".format(self.name), best_eval_loss.mean().item()) + # log + self.total_CMI_epoch += 1 + if self.logger is not None: + self.logger.record("{}-CMI-test/epoch".format(self.name), epoch) + self.logger.record("{}-CMI-test/epochs_since_update".format(self.name), epochs_since_update) + self.logger.record("{}-CMI-test/train_dataset_size".format(self.name), len(train_loader.dataset)) + self.logger.record("{}-CMI-test/valid_dataset_size".format(self.name), len(valid_loader.dataset)) + self.logger.record("{}-CMI-test/train_loss".format(self.name), train_loss.mean().item()) + self.logger.record("{}-CMI-test/val_loss".format(self.name), eval_loss.mean().item()) + self.logger.record("{}-CMI-test/best_val_loss".format(self.name), best_eval_loss.mean().item()) - self.logger.dump(self.total_CMI_epoch) + self.logger.dump(self.total_CMI_epoch) - if patience and epochs_since_update >= patience: - break + if patience and epochs_since_update >= patience: + break super(CMITest, self).learn( train_loader=train_loader, @@ -258,7 +259,3 @@ def learn( work_dir=work_dir, **kwargs ) - - def set_oracle_graph(self, graph): - self._oracle_graph = graph - pass diff --git a/cmrl/models/causal_mech/base_causal_mech.py b/cmrl/models/causal_mech/base_causal_mech.py index 753cdb7..a467c48 100644 --- a/cmrl/models/causal_mech/base_causal_mech.py +++ b/cmrl/models/causal_mech/base_causal_mech.py @@ -5,6 +5,7 @@ from torch.utils.data import DataLoader from cmrl.models.graphs.base_graph import BaseGraph +from cmrl.models.graphs.binary_graph import BinaryGraph from cmrl.utils.variables import Variable @@ -24,6 +25,7 @@ def __init__( self.input_var_num = len(self.input_variables) self.output_var_num = len(self.output_variables) self.graph: Optional[BaseGraph] = None + self.discovery: bool = True @abstractmethod def learn(self, train_loader: DataLoader, valid_loader: DataLoader, **kwargs): @@ -33,10 +35,6 @@ def learn(self, train_loader: DataLoader, valid_loader: DataLoader, **kwargs): def forward(self, inputs: MutableMapping[str, torch.Tensor]) -> Dict[str, torch.Tensor]: raise NotImplementedError - @abstractmethod - def set_oracle_graph(self, graph): - pass - @property def causal_graph(self) -> torch.Tensor: """property causal graph""" @@ -49,3 +47,8 @@ def causal_graph(self) -> torch.Tensor: def forward_mask(self) -> torch.Tensor: """property input masks""" return self.causal_graph.T + + def set_oracle_graph(self, graph_data): + self.discovery = False + self.graph = BinaryGraph(self.input_var_num, self.output_var_num) + self.graph.set_data(graph_data=graph_data) diff --git a/cmrl/models/causal_mech/reinforce.py b/cmrl/models/causal_mech/reinforce.py index b973df8..0c636a6 100644 --- a/cmrl/models/causal_mech/reinforce.py +++ b/cmrl/models/causal_mech/reinforce.py @@ -17,7 +17,6 @@ from cmrl.models.graphs.prob_graph import BernoulliGraph from cmrl.models.causal_mech.util import variable_loss_func, train_func, eval_func - default_graph_optimizer_cfg = DictConfig( dict( _target_="torch.optim.Adam", @@ -71,6 +70,8 @@ def __init__( self._graph_max_stack = graph_max_stack self._lambda_sparse = lambda_sparse + self.graph_optimizer = None + super(ReinforceCausalMech, self).__init__( name=name, input_variables=input_variables, diff --git a/cmrl/models/graphs/base_graph.py b/cmrl/models/graphs/base_graph.py index 2f27eb8..a75eec2 100644 --- a/cmrl/models/graphs/base_graph.py +++ b/cmrl/models/graphs/base_graph.py @@ -20,7 +20,7 @@ class BaseGraph(abc.ABC): in_dim (int): input dimension. out_dim (int): output dimension. extra_dim (int | tuple(int) | None): extra dimensions (multi-graph). - include_input (bool): whether inlcude input variables in the output variables. + include_input (bool): whether include input variables in the output variables. """ def __init__( @@ -37,7 +37,7 @@ def __init__( self._extra_dim = extra_dim self._include_input = include_input - assert not (include_input and out_dim < in_dim), "Once include input, the out dimesnion must >= in dimensino" + assert not (include_input and out_dim < in_dim), "Once include input, the out dimension must >= in dimension" @property @abc.abstractmethod From a3f1f9aeea1b02ce2a5c9d219380dfcf9de2e8a2 Mon Sep 17 00:00:00 2001 From: frank Date: Wed, 30 Nov 2022 00:19:12 +0800 Subject: [PATCH 36/68] :tada: add RadianVariable and von_mises_nll_loss --- cmrl/examples/conf/main.yaml | 2 +- cmrl/examples/conf/task/BIPS.yaml | 2 +- cmrl/models/causal_mech/neural_causal_mech.py | 7 ++ cmrl/models/causal_mech/util.py | 89 ++++++++++++++++++- cmrl/models/fake_env.py | 5 ++ cmrl/models/layers.py | 15 +++- cmrl/models/networks/coder.py | 8 +- cmrl/utils/creator.py | 3 +- cmrl/utils/variables.py | 10 ++- 9 files changed, 131 insertions(+), 10 deletions(-) diff --git a/cmrl/examples/conf/main.yaml b/cmrl/examples/conf/main.yaml index fb64a4f..f5ab3c2 100644 --- a/cmrl/examples/conf/main.yaml +++ b/cmrl/examples/conf/main.yaml @@ -1,7 +1,7 @@ defaults: - algorithm: off_dyna - task: BIPS - - transition: CMI_test + - transition: plain - reward_mech: plain - termination_mech: plain - _self_ diff --git a/cmrl/examples/conf/task/BIPS.yaml b/cmrl/examples/conf/task/BIPS.yaml index 5c0e95b..d383aa2 100644 --- a/cmrl/examples/conf/task/BIPS.yaml +++ b/cmrl/examples/conf/task/BIPS.yaml @@ -9,7 +9,7 @@ params: dataset: "SAC-expert-replay" # basic RL params -num_steps: 300000 +num_steps: 1000000 online_num_steps: 10000 epoch_length: 10000 n_eval_episodes: 8 diff --git a/cmrl/models/causal_mech/neural_causal_mech.py b/cmrl/models/causal_mech/neural_causal_mech.py index 5a775f2..490610e 100644 --- a/cmrl/models/causal_mech/neural_causal_mech.py +++ b/cmrl/models/causal_mech/neural_causal_mech.py @@ -135,6 +135,13 @@ def single_step_forward(self, inputs: MutableMapping[str, torch.Tensor]) -> Dict out = self.variable_encoders[var.name](inputs[var.name].to(self.device)) inputs_tensor[:, :, i] = out + for name, param in self.network.named_parameters(): + if param.grad is not None and torch.isnan(param.grad).any(): + print("nan gradient found") + print("name:", name) + print("param:", param.grad) + raise SystemExit + output_tensor = self.network(self.reduce_encoder_output(inputs_tensor)) outputs = {} diff --git a/cmrl/models/causal_mech/util.py b/cmrl/models/causal_mech/util.py index dbe647f..d6e194f 100644 --- a/cmrl/models/causal_mech/util.py +++ b/cmrl/models/causal_mech/util.py @@ -1,12 +1,91 @@ from typing import Callable, Dict, List, Union, MutableMapping from collections import defaultdict +import math import torch +from torch import Tensor import torch.nn.functional as F from torch.utils.data import DataLoader from torch.optim import Optimizer +from torch.distributions.von_mises import _log_modified_bessel_fn -from cmrl.utils.variables import Variable, ContinuousVariable, DiscreteVariable, BinaryVariable +from cmrl.utils.variables import Variable, ContinuousVariable, DiscreteVariable, BinaryVariable, RadianVariable + + +def von_mises_nll_loss( + input: Tensor, + target: Tensor, + var: Tensor, + full: bool = False, + eps: float = 1e-6, + reduction: str = "mean", +) -> Tensor: + r"""Von Mises negative log likelihood loss. + + Args: + input: loc of the Von Mises distribution. + target: sample from the Von Mises distribution. + var: tensor of positive var(s), one for each of the expectations + in the input (heteroscedastic), or a single one (homoscedastic). + full (bool, optional): include the constant term in the loss calculation. Default: ``False``. + eps (float, optional): value added to var, for stability. Default: 1e-6. + reduction (string, optional): specifies the reduction to apply to the output: + ``'none'`` | ``'mean'`` | ``'sum'``. ``'none'``: no reduction will be applied, + ``'mean'``: the output is the average of all batch member losses, + ``'sum'``: the output is the sum of all batch member losses. + Default: ``'mean'``. + """ + # Entries of var must be non-negative + if torch.any(var < 0): + raise ValueError("var has negative entry/entries") + + # Clamp for stability + var = var.clone() + with torch.no_grad(): + var.clamp_(min=eps) + + concentration = 1 / var + loss = -concentration * torch.cos(input - target) + _log_modified_bessel_fn(concentration, order=0) + if full: + loss += math.log(2 * math.pi) + + if reduction == "mean": + return loss.mean() + elif reduction == "sum": + return loss.sum() + else: + return loss + + +def circular_gaussian_nll_loss( + input: Tensor, + target: Tensor, + var: Tensor, + full: bool = False, + eps: float = 1e-6, + reduction: str = "mean", +) -> Tensor: + # Entries of var must be non-negative + if torch.any(var < 0): + raise ValueError("var has negative entry/entries") + + # Clamp for stability + var = var.clone() + with torch.no_grad(): + var.clamp_(min=eps) + + diff = torch.remainder(input - target, 2 * torch.pi) + diff[diff > torch.pi] = 2 * torch.pi - diff[diff > torch.pi] + loss = 0.5 * (torch.log(var) + diff**2 / var) + if full: + loss += 0.5 * math.log(2 * math.pi) + + if reduction == "mean": + return loss.mean() + elif reduction == "sum": + return loss.sum() + else: + return loss def variable_loss_func( @@ -25,7 +104,13 @@ def variable_loss_func( dim = target.shape[-1] # (xxx, ensemble-num, batch-size, dim) assert output.shape[-1] == 2 * dim mean, log_var = output[..., :dim], output[..., dim:] - loss = F.gaussian_nll_loss(mean, target, log_var.exp(), reduction="none").mean(dim=-1) + loss = F.gaussian_nll_loss(mean, target, log_var.exp(), reduction="none", full=True).mean(dim=-1) + total_loss[..., i] = loss + elif isinstance(var, RadianVariable): + dim = target.shape[-1] # (xxx, ensemble-num, batch-size, dim) + assert output.shape[-1] == 2 * dim + mean, log_var = output[..., :dim], output[..., dim:] + loss = circular_gaussian_nll_loss(mean, target, log_var.exp(), reduction="none").mean(dim=-1) total_loss[..., i] = loss elif isinstance(var, DiscreteVariable): # TODO: onehot to int? diff --git a/cmrl/models/fake_env.py b/cmrl/models/fake_env.py index 8cb0b3a..d1e4bfb 100644 --- a/cmrl/models/fake_env.py +++ b/cmrl/models/fake_env.py @@ -2,6 +2,7 @@ import gym import numpy as np +import torch from stable_baselines3.common.vec_env.base_vec_env import VecEnv, VecEnvIndices from stable_baselines3.common.logger import Logger from stable_baselines3.common.buffers import ReplayBuffer @@ -86,6 +87,10 @@ def step_wait(self): if self.logger is not None: self.logger.record_mean("rollout/penalty", penalty.mean().item()) + assert not np.isnan(batch_next_obs).any(), "next obs of fake env should not be nan." + assert not np.isnan(batch_reward).any(), "reward of fake env should not be nan." + assert not np.isnan(batch_terminal).any(), "terminal of fake env should not be nan." + self._current_batch_obs = batch_next_obs.copy() batch_reward = batch_reward.reshape(self.num_envs) batch_terminal = batch_terminal.reshape(self.num_envs) diff --git a/cmrl/models/layers.py b/cmrl/models/layers.py index f93858e..b92dca7 100644 --- a/cmrl/models/layers.py +++ b/cmrl/models/layers.py @@ -3,6 +3,7 @@ import numpy as np import torch from torch import nn as nn +from torch import Tensor from itertools import product from cmrl.models.util import truncated_normal_ @@ -59,7 +60,7 @@ def init_params(self): else: raise NotImplementedError - def forward(self, x): + def forward(self, x: Tensor) -> Tensor: xw = x.matmul(self.weight) if self.use_bias: return xw + self.bias @@ -77,7 +78,15 @@ def device(self) -> torch.device: return param.device return torch.device("cpu") - def __repr__(self): - return 'ParallelLinear(input_dims={}, output_dims={}, extra_dims={}, bias={}, init_type="{}")'.format( + def extra_repr(self): + return 'input_dims={}, output_dims={}, extra_dims={}, bias={}, init_type="{}"'.format( self.input_dim, self.output_dim, str(self.extra_dims), self.use_bias, self.init_type ) + + +class RadianLayer(nn.Module): + def __init__(self) -> None: + super(RadianLayer, self).__init__() + + def forward(self, input: Tensor) -> Tensor: + return torch.remainder(input + torch.pi, 2 * torch.pi) - torch.pi diff --git a/cmrl/models/networks/coder.py b/cmrl/models/networks/coder.py index 2f8935a..e4c111b 100644 --- a/cmrl/models/networks/coder.py +++ b/cmrl/models/networks/coder.py @@ -3,8 +3,9 @@ import torch.nn as nn from omegaconf import DictConfig -from cmrl.utils.variables import Variable, DiscreteVariable, ContinuousVariable, BinaryVariable +from cmrl.utils.variables import Variable, DiscreteVariable, ContinuousVariable, BinaryVariable, RadianVariable from cmrl.models.networks.base_network import BaseNetwork, create_activation +from cmrl.models.layers import RadianLayer class VariableEncoder(BaseNetwork): @@ -36,6 +37,9 @@ def build(self): if isinstance(self.variable, ContinuousVariable): layers.append(nn.Linear(self.variable.dim, hidden_dim)) + elif isinstance(self.variable, RadianVariable): + layers.append(RadianLayer()) + layers.append(nn.Linear(self.variable.dim, hidden_dim)) elif isinstance(self.variable, DiscreteVariable): layers.append(nn.Linear(self.variable.n, hidden_dim)) elif isinstance(self.variable, BinaryVariable): @@ -93,6 +97,8 @@ def build(self): if isinstance(self.variable, ContinuousVariable): layers.append(nn.Linear(hidden_dim, self.variable.dim * 2)) + elif isinstance(self.variable, RadianVariable): + layers.append(nn.Linear(hidden_dim, self.variable.dim * 2)) elif isinstance(self.variable, DiscreteVariable): layers.append(nn.Linear(hidden_dim, self.variable.n)) layers.append(nn.Softmax()) diff --git a/cmrl/utils/creator.py b/cmrl/utils/creator.py index a6fffa6..5f7d943 100644 --- a/cmrl/utils/creator.py +++ b/cmrl/utils/creator.py @@ -4,6 +4,7 @@ from hydra.utils import instantiate from omegaconf import DictConfig import numpy as np +from stable_baselines3.common.vec_env import VecMonitor from stable_baselines3.common.logger import Logger from stable_baselines3.common.base_class import BaseAlgorithm @@ -14,7 +15,7 @@ def create_agent(cfg: DictConfig, fake_env: VecFakeEnv, logger: Optional[Logger] = None): - agent = instantiate(cfg.algorithm.agent)(env=fake_env) + agent = instantiate(cfg.algorithm.agent)(env=VecMonitor(fake_env)) agent = cast(BaseAlgorithm, agent) agent.set_logger(logger) diff --git a/cmrl/utils/variables.py b/cmrl/utils/variables.py index 1ecdc3d..cfc26aa 100644 --- a/cmrl/utils/variables.py +++ b/cmrl/utils/variables.py @@ -19,6 +19,11 @@ class ContinuousVariable(Variable): high: np.ndarray = None +@dataclass +class RadianVariable(Variable): + dim: int + + @dataclass class BinaryVariable(Variable): pass @@ -33,7 +38,10 @@ def parse_space(space: spaces.Space, prefix="obs") -> List[Variable]: variables = [] if isinstance(space, spaces.Box): for i, (low, high) in enumerate(zip(space.low, space.high)): - variables.append(ContinuousVariable(dim=1, low=low, high=high, name="{}_{}".format(prefix, i))) + if np.isclose(low, -np.pi) and np.isclose(high, np.pi): + variables.append(RadianVariable(dim=1, name="{}_{}".format(prefix, i))) + else: + variables.append(ContinuousVariable(dim=1, low=low, high=high, name="{}_{}".format(prefix, i))) elif isinstance(space, spaces.Discrete): variables.append(DiscreteVariable(n=space.n, name="{}_0".format(prefix))) elif isinstance(space, spaces.MultiDiscrete): From dd1ee0cbba4d7b0425f4f71b15be5fb2a2e255f2 Mon Sep 17 00:00:00 2001 From: wz139704646 <632291793@qq.com> Date: Wed, 30 Nov 2022 10:09:17 +0800 Subject: [PATCH 37/68] :wrench: add discovery param in reinforce --- cmrl/examples/conf/transition/reinforce.yaml | 1 + cmrl/models/causal_mech/base_causal_mech.py | 2 +- cmrl/models/causal_mech/reinforce.py | 6 +++--- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/cmrl/examples/conf/transition/reinforce.yaml b/cmrl/examples/conf/transition/reinforce.yaml index dac8e51..eb8cf3d 100644 --- a/cmrl/examples/conf/transition/reinforce.yaml +++ b/cmrl/examples/conf/transition/reinforce.yaml @@ -1,5 +1,6 @@ name: "reinforce_transition" learn: true +discovery: true encoder_cfg: _partial_: true diff --git a/cmrl/models/causal_mech/base_causal_mech.py b/cmrl/models/causal_mech/base_causal_mech.py index a467c48..72eb916 100644 --- a/cmrl/models/causal_mech/base_causal_mech.py +++ b/cmrl/models/causal_mech/base_causal_mech.py @@ -50,5 +50,5 @@ def forward_mask(self) -> torch.Tensor: def set_oracle_graph(self, graph_data): self.discovery = False - self.graph = BinaryGraph(self.input_var_num, self.output_var_num) + self.graph = BinaryGraph(self.input_var_num, self.output_var_num, device=self.device) self.graph.set_data(graph_data=graph_data) diff --git a/cmrl/models/causal_mech/reinforce.py b/cmrl/models/causal_mech/reinforce.py index 0c636a6..3197c87 100644 --- a/cmrl/models/causal_mech/reinforce.py +++ b/cmrl/models/causal_mech/reinforce.py @@ -141,7 +141,7 @@ def single_step_forward( out = self.variable_encoders[var.name](inputs[var.name].to(self.device)) inputs_tensor[..., i, :] = out - if train: + if train and self.discovery: # [ensemble-num, batch-size, input-var-num, output-var-num] adj_matrix = self.graph.sample(None, sample_size=(self.ensemble_num, batch_size)) # [ensemble-num, batch-size, output-var-num, input-var-num] @@ -319,7 +319,7 @@ def learn( best_eval_loss = eval_fn(valid_loader).mean(dim=(-2, -1)) for epoch in epoch_iter: - if epoch % train_graph_freq == 0: + if self.discovery and epoch % train_graph_freq == 0: grads = self.train_graph(train_loader, data_ratio=graph_data_ratio) print(self.graph.parameters[0]) print(self.graph.get_binary_adj_matrix()) @@ -349,7 +349,7 @@ def learn( self.logger.record("{}/val_loss".format(self.name), eval_loss.mean().item()) self.logger.record("{}/best_val_loss".format(self.name), best_eval_loss.mean().item()) - if epoch % train_graph_freq == 0: + if self.discovery and epoch % train_graph_freq == 0: self.logger.record("{}/graph_update_grads".format(self.name), grads.abs().mean().item()) self.logger.dump(self.total_epoch) From a4d5310be0ff03dc10f0aa7c502b731d2d8d987c Mon Sep 17 00:00:00 2001 From: frank Date: Fri, 2 Dec 2022 00:34:02 +0800 Subject: [PATCH 38/68] :tada: add save and auto-load --- cmrl/algorithms/off_dyna.py | 18 ++-- cmrl/algorithms/util.py | 77 ++++++------- cmrl/examples/conf/algorithm/off_dyna.yaml | 2 +- cmrl/examples/conf/task/BIPS.yaml | 2 +- cmrl/models/causal_mech/base_causal_mech.py | 7 ++ cmrl/models/causal_mech/neural_causal_mech.py | 20 ++++ cmrl/models/networks/mlp.py | 102 ------------------ 7 files changed, 79 insertions(+), 149 deletions(-) delete mode 100644 cmrl/models/networks/mlp.py diff --git a/cmrl/algorithms/off_dyna.py b/cmrl/algorithms/off_dyna.py index e9de279..4eeb010 100644 --- a/cmrl/algorithms/off_dyna.py +++ b/cmrl/algorithms/off_dyna.py @@ -5,6 +5,7 @@ from cmrl.models.fake_env import VecFakeEnv from cmrl.algorithms.base_algorithm import BaseAlgorithm from cmrl.utils.env import load_offline_data +from cmrl.algorithms.util import maybe_load_offline_model class OfflineDyna(BaseAlgorithm): @@ -18,10 +19,13 @@ def __init__( def _setup_learn(self): load_offline_data(self.env, self.real_replay_buffer, self.cfg.task.dataset, self.cfg.task.use_ratio) - self.dynamics.learn( - real_replay_buffer=self.real_replay_buffer, - longest_epoch=self.cfg.task.longest_epoch, - improvement_threshold=self.cfg.task.improvement_threshold, - patience=self.cfg.task.patience, - work_dir=self.work_dir, - ) + existed_trained_model = maybe_load_offline_model(self.dynamics, self.cfg, work_dir=self.work_dir) + + if not existed_trained_model: + self.dynamics.learn( + real_replay_buffer=self.real_replay_buffer, + longest_epoch=self.cfg.task.longest_epoch, + improvement_threshold=self.cfg.task.improvement_threshold, + patience=self.cfg.task.patience, + work_dir=self.work_dir, + ) diff --git a/cmrl/algorithms/util.py b/cmrl/algorithms/util.py index 1f9a43f..8e708ae 100644 --- a/cmrl/algorithms/util.py +++ b/cmrl/algorithms/util.py @@ -1,8 +1,9 @@ from typing import Optional, cast from copy import deepcopy +import pathlib import hydra -from omegaconf import DictConfig +from omegaconf import DictConfig, OmegaConf from stable_baselines3.common.vec_env.vec_monitor import VecMonitor from stable_baselines3.common.base_class import BaseAlgorithm @@ -10,17 +11,17 @@ from cmrl.types import InitObsFnType, RewardFnType, TermFnType -# from cmrl.models.dynamics import BaseDynamics -from cmrl.models.fake_env import VecFakeEnv +from cmrl.models.dynamics import Dynamics +from cmrl.utils.config import load_hydra_cfg -def is_same_dict(dict1, dict2): +def compare_dict(dict1, dict2): for key in dict1: if key not in dict2: return False else: - if isinstance(dict1[key], DictConfig) and isinstance(dict2[key], DictConfig): - if not is_same_dict(dict1[key], dict2[key]): + if isinstance(dict1[key], dict) and isinstance(dict2[key], dict): + if not compare_dict(dict1[key], dict2[key]): return False else: if dict1[key] != dict2[key]: @@ -28,35 +29,35 @@ def is_same_dict(dict1, dict2): return True -# def maybe_load_trained_offline_model(dynamics: BaseDynamics, cfg, obs_shape, act_shape, work_dir): -# work_dir = pathlib.Path(work_dir) -# if "." not in work_dir.name: # exp by hydra's MULTIRUN mode -# task_exp_dir = work_dir.parent.parent.parent -# else: -# task_exp_dir = work_dir.parent.parent -# dynamics_cfg = cfg.dynamics -# -# for date_dir in task_exp_dir.glob(r"*"): -# for time_dir in date_dir.glob(r"*"): -# if (time_dir / "multirun.yaml").exists(): # exp by hydra's MULTIRUN mode, multi exp in this time -# this_time_exp_dir_list = list(time_dir.glob(r"*")) -# else: # only one exp in this time -# this_time_exp_dir_list = [time_dir] -# -# for exp_dir in this_time_exp_dir_list: -# if not (exp_dir / ".hydra").exists(): -# continue -# exp_cfg = load_hydra_cfg(exp_dir) -# exp_dynamics_cfg = get_complete_dynamics_cfg(exp_cfg.dynamics, obs_shape, act_shape) -# -# if exp_cfg.seed == cfg.seed and is_same_dict(dynamics_cfg, exp_dynamics_cfg): -# exist_model_file = True -# for mech in dynamics.learn_mech: -# mech_file_name = getattr(dynamics, mech).model_file_name -# if not (exp_dir / mech_file_name).exists(): -# exist_model_file = False -# if exist_model_file: -# dynamics.load(exp_dir) -# print("loaded dynamics from {}".format(exp_dir)) -# return True -# return False +def maybe_load_offline_model( + dynamics: Dynamics, + cfg: DictConfig, + # obs_shape, + # act_shape, + work_dir, +): + work_dir = pathlib.Path(work_dir) + if "." not in work_dir.name: # exp by hydra's MULTIRUN mode + task_exp_dir = work_dir.parent.parent + else: + task_exp_dir = work_dir.parent + + transition_cfg = OmegaConf.to_container(cfg.transition.mech, resolve=True) + + for time_dir in task_exp_dir.glob(r"*"): + if (time_dir / "multirun.yaml").exists(): # exp by hydra's MULTIRUN mode, multi exp in this time + this_time_exp_dir_list = list(time_dir.glob(r"*")) + else: # only one exp in this time + this_time_exp_dir_list = [time_dir] + + for exp_dir in this_time_exp_dir_list: + if not (exp_dir / ".hydra").exists(): + continue + exp_cfg = load_hydra_cfg(exp_dir) + + exp_transition_dir = OmegaConf.to_container(exp_cfg.transition.mech, resolve=True) + if compare_dict(exp_transition_dir, transition_cfg) and (exp_dir / "transition").exists(): + dynamics.transition.load(exp_dir / "transition") + print("loaded dynamics from {}".format(exp_dir)) + return True + return False diff --git a/cmrl/examples/conf/algorithm/off_dyna.yaml b/cmrl/examples/conf/algorithm/off_dyna.yaml index 445ff78..43799c2 100644 --- a/cmrl/examples/conf/algorithm/off_dyna.yaml +++ b/cmrl/examples/conf/algorithm/off_dyna.yaml @@ -9,7 +9,7 @@ num_eval_episodes: 5 dataset_size: 1000000 penalty_coeff: ${task.penalty_coeff} -num_envs: 16 +num_envs: 32 deterministic: false agent: _partial_: true diff --git a/cmrl/examples/conf/task/BIPS.yaml b/cmrl/examples/conf/task/BIPS.yaml index d383aa2..13de2af 100644 --- a/cmrl/examples/conf/task/BIPS.yaml +++ b/cmrl/examples/conf/task/BIPS.yaml @@ -9,7 +9,7 @@ params: dataset: "SAC-expert-replay" # basic RL params -num_steps: 1000000 +num_steps: 10000000 online_num_steps: 10000 epoch_length: 10000 n_eval_episodes: 8 diff --git a/cmrl/models/causal_mech/base_causal_mech.py b/cmrl/models/causal_mech/base_causal_mech.py index a467c48..52b563e 100644 --- a/cmrl/models/causal_mech/base_causal_mech.py +++ b/cmrl/models/causal_mech/base_causal_mech.py @@ -1,5 +1,6 @@ from typing import Optional, List, Dict, Union, MutableMapping from abc import abstractmethod, ABC +import pathlib import torch from torch.utils.data import DataLoader @@ -52,3 +53,9 @@ def set_oracle_graph(self, graph_data): self.discovery = False self.graph = BinaryGraph(self.input_var_num, self.output_var_num) self.graph.set_data(graph_data=graph_data) + + def save(self): + pass + + def load(self, load_dir: Union[str, pathlib.Path]): + pass diff --git a/cmrl/models/causal_mech/neural_causal_mech.py b/cmrl/models/causal_mech/neural_causal_mech.py index 490610e..109cc38 100644 --- a/cmrl/models/causal_mech/neural_causal_mech.py +++ b/cmrl/models/causal_mech/neural_causal_mech.py @@ -277,6 +277,26 @@ def learn( # saving the best models: self._maybe_set_best_weights_and_elite(best_weights, best_eval_loss) + def save(self): + save_dir = pathlib.Path(self.name) + save_dir.mkdir(exist_ok=True) + self.network.save(save_dir) + for coder in self.variable_encoders.values(): + coder.save(save_dir) + for coder in self.variable_decoders.values(): + coder.save(save_dir) + + def load(self, load_dir: Union[str, pathlib.Path]): + if isinstance(load_dir, str): + load_dir = pathlib.Path(load_dir) + assert load_dir.exists() + + self.network.load(load_dir) + for coder in self.variable_encoders.values(): + coder.load(load_dir) + for coder in self.variable_decoders.values(): + coder.load(load_dir) + def _maybe_get_best_weights( self, best_val_loss: torch.Tensor, diff --git a/cmrl/models/networks/mlp.py b/cmrl/models/networks/mlp.py deleted file mode 100644 index ddce2f9..0000000 --- a/cmrl/models/networks/mlp.py +++ /dev/null @@ -1,102 +0,0 @@ -import pathlib -from typing import Dict, Optional, Sequence, Union -from abc import abstractmethod - -import torch -import torch.nn as nn -import torch.nn.functional as F -import numpy as np -from omegaconf import DictConfig - -from cmrl.models.layers import ParallelLinear -from cmrl.models.networks.base_network import BaseNetwork - - -class EnsembleMLP(nn.Module): - _MODEL_FILENAME = "ensemble_mlp.pth" - - def __init__( - self, - ensemble_num: int = 7, - elite_num: int = 5, - device: Union[str, torch.device] = "cpu", - ): - super(EnsembleMLP, self).__init__() - self.ensemble_num = ensemble_num - self.elite_num = elite_num - self.device = device - - self._elite_members: Optional[Sequence[int]] = np.random.permutation(ensemble_num)[:elite_num] - - self._model_save_attrs = ["elite_members", "state_dict"] - - def set_elite_members(self, elite_indices: Sequence[int]): - if len(elite_indices) != self.ensemble_num: - assert len(elite_indices) == self.elite_num - self._elite_members = list(elite_indices) - - @property - def elite_members(self): - return self._elite_members - - def get_random_index(self, batch_size: int, numpy_generator: Optional[np.random.Generator] = None): - if numpy_generator: - return numpy_generator.choice(self._elite_members, size=batch_size) - else: - return np.random.choice(self._elite_members, size=batch_size) - - def save(self, save_dir: Union[str, pathlib.Path]): - """Saves the model to the given directory.""" - model_dict = {} - for attr in self._model_save_attrs: - if attr == "state_dict": - model_dict["state_dict"] = self.state_dict() - else: - model_dict[attr] = getattr(self, attr) - torch.save(model_dict, pathlib.Path(save_dir) / self._MODEL_FILENAME) - - def load(self, load_dir: Union[str, pathlib.Path], load_device: Optional[str] = None): - """Loads the model from the given path.""" - model_dict = torch.load(pathlib.Path(load_dir) / self._MODEL_FILENAME, map_location=load_device) - for attr in model_dict: - if attr == "state_dict": - self.load_state_dict(model_dict["state_dict"]) - else: - getattr(self, "set_" + attr)(model_dict[attr]) - - def create_linear_layer(self, l_in, l_out): - return ParallelLinear(l_in, l_out, extra_dims=[self.ensemble_num]) - - def get_mse_loss(self, model_in: Dict[(str, torch.Tensor)], target: torch.Tensor) -> torch.Tensor: - pred_mean, pred_logvar = self.forward(**model_in) - return F.mse_loss(pred_mean, target, reduction="none") - - def get_nll_loss(self, model_in: Dict[(str, torch.Tensor)], target: torch.Tensor) -> torch.Tensor: - pred_mean, pred_logvar = self.forward(**model_in) - nll_loss = gaussian_nll(pred_mean, pred_logvar, target, reduce=False) - nll_loss += 0.01 * (self.max_logvar.sum() - self.min_logvar.sum()) - return nll_loss - - def add_save_attr(self, attr: str): - assert hasattr(self, attr), "Class must has attribute {}".format(attr) - assert attr not in self._model_save_attrs, "Attribute {} has been in model-save-list".format(attr) - self._model_save_attrs.append(attr) - - @property - def save_attr(self): - return self._model_save_attrs - - @property - def model_file_name(self): - return self._MODEL_FILENAME - - -class ExternalMaskEnsembleMLP(EnsembleMLP): - """Ensemble of multi-layer perceptrons with input mask inside - - Args: - TODO - """ - - def __init__(self, ensemble_num: int = 7, elite_num: int = 5, device: Union[str, torch.device] = "cpu"): - super().__init__(ensemble_num, elite_num, device) From 5db680328c4e20e1b54b8b12b4f90e76469531a5 Mon Sep 17 00:00:00 2001 From: frank Date: Fri, 2 Dec 2022 15:41:32 +0800 Subject: [PATCH 39/68] :bug: save after learn --- cmrl/algorithms/base_algorithm.py | 11 ++++++++++- cmrl/examples/main.py | 9 --------- cmrl/models/causal_mech/neural_causal_mech.py | 2 ++ 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/cmrl/algorithms/base_algorithm.py b/cmrl/algorithms/base_algorithm.py index a4aba58..af51b82 100644 --- a/cmrl/algorithms/base_algorithm.py +++ b/cmrl/algorithms/base_algorithm.py @@ -4,9 +4,10 @@ import numpy as np import torch -from omegaconf import DictConfig +from omegaconf import DictConfig, OmegaConf from stable_baselines3.common.buffers import ReplayBuffer from stable_baselines3.common.callbacks import BaseCallback +import wandb from cmrl.models.fake_env import VecFakeEnv from cmrl.sb3_extension.logger import configure as logger_configure @@ -31,6 +32,14 @@ def __init__( self.logger = logger_configure("log", ["tensorboard", "multi_csv", "stdout"]) + if cfg.wandb: + wandb.init( + project="causal-mbrl", + group=cfg.exp_name, + config=OmegaConf.to_container(cfg, resolve=True), + sync_tensorboard=True, + ) + # create ``cmrl`` dynamics self.dynamics = create_dynamics(self.cfg, self.env.observation_space, self.env.action_space, logger=self.logger) diff --git a/cmrl/examples/main.py b/cmrl/examples/main.py index 544c018..e64eb57 100644 --- a/cmrl/examples/main.py +++ b/cmrl/examples/main.py @@ -1,20 +1,11 @@ import hydra from hydra.utils import instantiate -import wandb from omegaconf import DictConfig, OmegaConf from emei.core import get_params_str @hydra.main(version_base=None, config_path="conf", config_name="main") def run(cfg: DictConfig): - if cfg.wandb: - wandb.init( - project="causal-mbrl", - group=cfg.exp_name, - config=OmegaConf.to_container(cfg, resolve=True), - sync_tensorboard=True, - ) - algo = instantiate(cfg.algorithm.algo)(cfg=cfg) algo.learn() diff --git a/cmrl/models/causal_mech/neural_causal_mech.py b/cmrl/models/causal_mech/neural_causal_mech.py index 109cc38..426c89e 100644 --- a/cmrl/models/causal_mech/neural_causal_mech.py +++ b/cmrl/models/causal_mech/neural_causal_mech.py @@ -277,6 +277,8 @@ def learn( # saving the best models: self._maybe_set_best_weights_and_elite(best_weights, best_eval_loss) + self.save() + def save(self): save_dir = pathlib.Path(self.name) save_dir.mkdir(exist_ok=True) From 55f669838cd2dbeb7eac84966c6cbb4c569a4fe8 Mon Sep 17 00:00:00 2001 From: frank Date: Sat, 3 Dec 2022 20:31:53 +0800 Subject: [PATCH 40/68] :bug: fix eval_model_on_space.py action bug --- cmrl/diagnostics/eval_model_on_space.py | 91 +++++++++---------- cmrl/examples/conf/main.yaml | 2 +- .../task/continuous_cart_pole_swingup.yaml | 43 +++++++++ cmrl/models/causal_mech/base_causal_mech.py | 1 + cmrl/models/fake_env.py | 40 ++------ .../test_diagnostics/test_base_diagnostics.py | 5 + 6 files changed, 102 insertions(+), 80 deletions(-) create mode 100644 cmrl/examples/conf/task/continuous_cart_pole_swingup.yaml create mode 100644 tests/test_diagnostics/test_base_diagnostics.py diff --git a/cmrl/diagnostics/eval_model_on_space.py b/cmrl/diagnostics/eval_model_on_space.py index c883d22..09f23ca 100644 --- a/cmrl/diagnostics/eval_model_on_space.py +++ b/cmrl/diagnostics/eval_model_on_space.py @@ -14,19 +14,13 @@ import cmrl.utils.creator import cmrl.utils.env from cmrl.utils.config import load_hydra_cfg +from cmrl.utils.creator import create_dynamics, create_agent +from cmrl.models.fake_env import get_penalty -mpl.use("Qt5Agg") +mpl.use("TKAgg") SIN_COS_BINDINGS = {"BoundaryInvertedPendulumSwingUp-v0": [1]} -def calculate_penalty(ensemble_mean): - avg_ensemble_mean = np.mean(ensemble_mean, axis=0) # average predictions over models - diffs = ensemble_mean - avg_ensemble_mean - dists = np.linalg.norm(diffs, axis=2) # distance in obs space - penalty = np.max(dists, axis=0) # max distances over models - return penalty - - def set_ylim(y_min, y_max, ax): if y_max - y_min > 0.1: obs_y_lim = [y_min - 0.05, y_max + 0.05] @@ -73,19 +67,24 @@ def __init__( self.cfg = load_hydra_cfg(self.model_path) self.cfg.device = device - self.env, *_ = cmrl.util.env.make_env(self.cfg) + self.env, *_ = cmrl.utils.env.make_env(self.cfg) if penalty_coeff is None: self.penalty_coeff = self.cfg.task.penalty_coeff else: self.penalty_coeff = penalty_coeff - self.dynamics = cmrl.util.creator.create_dynamics( - self.cfg.dynamics, - self.env.observation_space.shape, - self.env.action_space.shape, - load_dir=self.model_path, - load_device=device, + self.dynamics = create_dynamics( + self.cfg, + self.env.observation_space, + self.env.action_space, ) + if not self.cfg.transition.discovery: + self.dynamics.transition.set_oracle_graph(self.env.get_transition_graph()) + if self.cfg.reward_mech.learn and not self.cfg.reward_mech.discovery: + self.dynamics.reward_mech.set_oracle_graph(self.env.get_reward_mech_graph()) + if self.cfg.termination_mech.learn and not self.cfg.termination_mech.discovery: + self.dynamics.termination_mech.set_oracle_graph(self.env.get_termination_mech_graph()) + self.dynamics.transition.load(self.model_path / "transition") self.bindings = [] self.obs_range, self.action_range = self.get_range() @@ -213,20 +212,19 @@ def slider_changed(value, dim=dim): self.draw_button.on_clicked(self.draw) def get_range(self, dataset_type="SAC-expert-replay"): - universe, basic_env_name, params, origin_dataset_type = self.cfg.task.env.split("___") - data_dict = self.env.get_dataset("{}-{}".format(params, dataset_type)) + data_dict = self.env.get_dataset(dataset_type) obs_min = np.percentile(data_dict["observations"], self.range_quantile, axis=0) obs_max = np.percentile(data_dict["observations"], 100 - self.range_quantile, axis=0) action_min = np.percentile(data_dict["actions"], self.range_quantile, axis=0) action_max = np.percentile(data_dict["actions"], 100 - self.range_quantile, axis=0) obs_range, action_range = np.array(list(zip(obs_min, obs_max))), np.array(list(zip(action_min, action_max))) - if basic_env_name in SIN_COS_BINDINGS: - self.bindings = SIN_COS_BINDINGS[basic_env_name] - for idx, binding_idx in enumerate(self.bindings): - theta_idx = binding_idx - idx - obs_range = np.delete(obs_range, [binding_idx, binding_idx + 1], axis=0) - obs_range = np.insert(obs_range, theta_idx, np.array([0, 2 * np.pi]), axis=0) + # if basic_env_name in SIN_COS_BINDINGS: + # self.bindings = SIN_COS_BINDINGS[basic_env_name] + # for idx, binding_idx in enumerate(self.bindings): + # theta_idx = binding_idx - idx + # obs_range = np.delete(obs_range, [binding_idx, binding_idx + 1], axis=0) + # obs_range = np.insert(obs_range, theta_idx, np.array([0, 2 * np.pi]), axis=0) return obs_range, action_range def build_model_in(self): @@ -250,7 +248,7 @@ def build_model_in(self): real_model_in[:, dim] = np.cos(compact_model_in[:, compact_dim].copy()) else: # is an action compact_dim = dim - (self.real_obs_dim_num - self.compact_obs_dim_num) - real_model_in[:, dim] = np.cos(compact_model_in[:, compact_dim].copy()) + real_model_in[:, dim] = compact_model_in[:, compact_dim].copy() return x, real_model_in def draw(self, event): @@ -285,32 +283,29 @@ def get_model_out(self, model_in): penalized_reward = np.empty(self.plot_dot_num) for batch_idx in range(batch_num): + f, t = self.batch_size * batch_idx, self.batch_size * (batch_idx + 1) + batch_input = model_in[self.batch_size * batch_idx : self.batch_size * (batch_idx + 1)] batch_obs, batch_action = ( batch_input[:, : self.real_obs_dim_num], batch_input[:, self.real_obs_dim_num :], ) - dynamics_result = self.dynamics.query(batch_obs, batch_action, return_as_np=True) - gt_next_obs, gt_reward, gt_terminal, gt_truncated, _ = self.env.query(batch_obs, batch_action) - # predict and ground truth - batch_predict_obs = dynamics_result["batch_next_obs"]["mean"].mean(0) - batch_gt_obs = gt_next_obs + + predict_next_obs, predict_reward, terminal, info = self.dynamics.step(batch_obs, batch_action) + gt_next_obs = self.env.get_batch_next_obs(batch_obs, batch_action) + gt_reward = self.env.get_batch_reward(gt_next_obs) + if self.draw_diff: - batch_predict_obs -= batch_obs - batch_gt_obs -= batch_obs - batch_predict = batch_predict_obs[:, self.current_out_dim] - batch_ground_truth = batch_gt_obs[:, self.current_out_dim] - # reward - # batch_reward = dynamics_result["batch_reward"]["mean"].mean(0)[:, 0] - batch_reward = gt_reward - # penalized_reward - batch_penalty = calculate_penalty(dynamics_result["batch_next_obs"]["mean"]) - batch_penalized_reward = batch_reward - batch_penalty * self.penalty_coeff - - predict[self.batch_size * batch_idx : self.batch_size * (batch_idx + 1)] = batch_predict - ground_truth[self.batch_size * batch_idx : self.batch_size * (batch_idx + 1)] = batch_ground_truth - reward[self.batch_size * batch_idx : self.batch_size * (batch_idx + 1)] = batch_reward - penalized_reward[self.batch_size * batch_idx : self.batch_size * (batch_idx + 1)] = batch_penalized_reward + predict_next_obs -= batch_obs + gt_next_obs -= batch_obs + + batch_penalty = get_penalty(info["origin-next_obs"]) + + predict[f:t] = predict_next_obs[:, self.current_out_dim] + ground_truth[f:t] = gt_next_obs[:, self.current_out_dim] + reward[f:t] = gt_reward[:, 0] + penalized_reward[f:t] = gt_reward[:, 0] - batch_penalty * self.penalty_coeff + return predict, ground_truth, reward, penalized_reward def run(self): @@ -326,7 +321,7 @@ def run(self): np.linspace(0, 1, 100), color="black", lw=2, - label="gt", + label="ground truth", ) (self.reward_line,) = self.reward_ax.plot( np.linspace(0, 1, 100), @@ -358,10 +353,10 @@ def run(self): parser = argparse.ArgumentParser() parser.add_argument("model_dir", type=str, default=None) parser.add_argument("--penalty_coeff", type=float, default=None) - parser.add_argument("--draw_diff", action="store_true") + parser.add_argument("--not_draw_diff", action="store_true", default=False) args = parser.parse_args() - evaluator = DatasetEvaluator(args.model_dir, penalty_coeff=args.penalty_coeff, draw_diff=args.draw_diff) + evaluator = DatasetEvaluator(args.model_dir, penalty_coeff=args.penalty_coeff, draw_diff=not args.not_draw_diff) mpl.rcParams["figure.facecolor"] = "white" mpl.rcParams["font.size"] = 14 diff --git a/cmrl/examples/conf/main.yaml b/cmrl/examples/conf/main.yaml index f5ab3c2..988eb9f 100644 --- a/cmrl/examples/conf/main.yaml +++ b/cmrl/examples/conf/main.yaml @@ -1,6 +1,6 @@ defaults: - algorithm: off_dyna - - task: BIPS + - task: continuous_cart_pole_swingup - transition: plain - reward_mech: plain - termination_mech: plain diff --git a/cmrl/examples/conf/task/continuous_cart_pole_swingup.yaml b/cmrl/examples/conf/task/continuous_cart_pole_swingup.yaml new file mode 100644 index 0000000..820b7a2 --- /dev/null +++ b/cmrl/examples/conf/task/continuous_cart_pole_swingup.yaml @@ -0,0 +1,43 @@ +# env parameters +env_id: "ContinuousCartPoleSwingUp-v0" + +params: + freq_rate: 1 + real_time_scale: 0.02 + integrator: "euler" + +dataset: "SAC-expert-replay" + +# basic RL params +num_steps: 10000000 +online_num_steps: 10000 +epoch_length: 10000 +n_eval_episodes: 8 +eval_freq: 100 + +# dynamics +learning_reward: false +learning_terminal: false +ensemble_num: 7 +elite_num: 5 +multi_step: "forward_euler_5" + +# conditional mutual information test(causal discovery) +oracle: true +cit_threshold: 0.02 +test_freq: 100 +# causal +update_causal_mask_ratio: 0.25 +discovery_schedule: [ 1, 30, 250, 250 ] + +# offline +penalty_coeff: 0.2 +use_ratio: 1 + +# dyna +freq_train_model: 100 + +# model learning +patience: 20 +longest_epoch: -1 +improvement_threshold: 0.01 diff --git a/cmrl/models/causal_mech/base_causal_mech.py b/cmrl/models/causal_mech/base_causal_mech.py index 53b78d7..64e999c 100644 --- a/cmrl/models/causal_mech/base_causal_mech.py +++ b/cmrl/models/causal_mech/base_causal_mech.py @@ -53,6 +53,7 @@ def set_oracle_graph(self, graph_data): self.discovery = False self.graph = BinaryGraph(self.input_var_num, self.output_var_num, device=self.device) self.graph.set_data(graph_data=graph_data) + print("set oracle causal graph successfully: \n{}".format(graph_data)) def save(self): pass diff --git a/cmrl/models/fake_env.py b/cmrl/models/fake_env.py index d1e4bfb..b6d37a0 100644 --- a/cmrl/models/fake_env.py +++ b/cmrl/models/fake_env.py @@ -11,6 +11,14 @@ from cmrl.models.dynamics import Dynamics +def get_penalty(ensemble_batch_next_obs): + avg = np.mean(ensemble_batch_next_obs, axis=0) # average predictions over models + diffs = ensemble_batch_next_obs - avg + dists = np.linalg.norm(diffs, axis=2) # distance in obs space + penalty = np.max(dists, axis=0) # max distances over models + return penalty + + class VecFakeEnv(VecEnv): def __init__( self, @@ -81,7 +89,7 @@ def step_wait(self): batch_terminal = self.termination_fn(batch_next_obs, self._current_batch_obs, self._current_batch_action) if self.penalty_coeff != 0: - penalty = self.get_penalty(info["origin-next_obs"]).reshape(batch_reward.shape) + penalty = get_penalty(info["origin-next_obs"]).reshape(batch_reward.shape) batch_reward -= penalty * self.penalty_coeff if self.logger is not None: @@ -154,36 +162,6 @@ def single_reset(self, idx): def render(self, mode="human"): raise NotImplementedError - @staticmethod - def get_penalty(ensemble_batch_next_obs): - avg = np.mean(ensemble_batch_next_obs, axis=0) # average predictions over models - diffs = ensemble_batch_next_obs - avg - dists = np.linalg.norm(diffs, axis=2) # distance in obs space - penalty = np.max(dists, axis=0) # max distances over models - return penalty - - def get_dynamics_predict( - self, - origin_predict: Dict, - mech: str, - deterministic: bool = False, - ): - variable = self.dynamics.get_variable_by_mech(mech) - ensemble_mean, ensemble_logvar = ( - origin_predict[variable]["mean"], - origin_predict[variable]["logvar"], - ) - batch_size = ensemble_mean.shape[1] - random_index = getattr(self.dynamics, mech).get_random_index(batch_size, self.generator) - if deterministic: - pred = ensemble_mean[random_index, np.arange(batch_size)] - else: - ensemble_std = np.sqrt(np.exp(ensemble_logvar)) - pred = ensemble_mean[random_index, np.arange(batch_size)] + ensemble_std[ - random_index, np.arange(batch_size) - ] * self.generator.normal(size=ensemble_mean.shape[1:]).astype(np.float32) - return pred - def env_method( self, method_name: str, diff --git a/tests/test_diagnostics/test_base_diagnostics.py b/tests/test_diagnostics/test_base_diagnostics.py new file mode 100644 index 0000000..92f8b3f --- /dev/null +++ b/tests/test_diagnostics/test_base_diagnostics.py @@ -0,0 +1,5 @@ +from cmrl.diagnostics.base_diagnostic import BaseDiagnostic + + +def test_base_diagnostics(): + pass From 9f5750a4d942aa34728f56d2de456ca62166848d Mon Sep 17 00:00:00 2001 From: frank Date: Sat, 3 Dec 2022 20:31:59 +0800 Subject: [PATCH 41/68] :bug: fix eval_model_on_space.py action bug --- cmrl/diagnostics/base_diagnostic.py | 11 +++++++++++ .../__init__.py} | 0 2 files changed, 11 insertions(+) create mode 100644 cmrl/diagnostics/base_diagnostic.py rename tests/{test_diagnostics.py => test_diagnostics/__init__.py} (100%) diff --git a/cmrl/diagnostics/base_diagnostic.py b/cmrl/diagnostics/base_diagnostic.py new file mode 100644 index 0000000..3039cd4 --- /dev/null +++ b/cmrl/diagnostics/base_diagnostic.py @@ -0,0 +1,11 @@ +from typing import Union +import pathlib + + +class BaseDiagnostic: + def __init__(self, exp_dir: Union[str, pathlib.Path]): + if isinstance(exp_dir, str): + self.exp_dir = pathlib.Path(exp_dir) + else: + self.exp_dir = exp_dir + pass diff --git a/tests/test_diagnostics.py b/tests/test_diagnostics/__init__.py similarity index 100% rename from tests/test_diagnostics.py rename to tests/test_diagnostics/__init__.py From 05c0d0bc233ffc16ffbc32d28100eccb80349779 Mon Sep 17 00:00:00 2001 From: frank Date: Sun, 4 Dec 2022 20:54:59 +0800 Subject: [PATCH 42/68] :tada: add scheduler for optimizer --- cmrl/algorithms/off_dyna.py | 3 -- cmrl/algorithms/util.py | 14 +++-- cmrl/examples/conf/main.yaml | 2 +- .../task/continuous_cart_pole_swingup.yaml | 22 +------- cmrl/examples/conf/transition/CMI_test.yaml | 13 ++++- cmrl/examples/conf/transition/plain.yaml | 12 +++++ cmrl/examples/conf/transition/reinforce.yaml | 4 ++ cmrl/models/causal_mech/CMI_test.py | 51 +++++++++++-------- cmrl/models/causal_mech/neural_causal_mech.py | 35 ++++++++++--- cmrl/models/causal_mech/plain_mech.py | 8 +++ cmrl/models/causal_mech/reinforce.py | 16 +++--- cmrl/models/dynamics.py | 35 ++----------- 12 files changed, 118 insertions(+), 97 deletions(-) diff --git a/cmrl/algorithms/off_dyna.py b/cmrl/algorithms/off_dyna.py index 4eeb010..e4ef2a4 100644 --- a/cmrl/algorithms/off_dyna.py +++ b/cmrl/algorithms/off_dyna.py @@ -24,8 +24,5 @@ def _setup_learn(self): if not existed_trained_model: self.dynamics.learn( real_replay_buffer=self.real_replay_buffer, - longest_epoch=self.cfg.task.longest_epoch, - improvement_threshold=self.cfg.task.improvement_threshold, - patience=self.cfg.task.patience, work_dir=self.work_dir, ) diff --git a/cmrl/algorithms/util.py b/cmrl/algorithms/util.py index 8e708ae..9c83afe 100644 --- a/cmrl/algorithms/util.py +++ b/cmrl/algorithms/util.py @@ -16,6 +16,8 @@ def compare_dict(dict1, dict2): + if len(list(dict1)) != len(list(dict2)): + return False for key in dict1: if key not in dict2: return False @@ -32,8 +34,6 @@ def compare_dict(dict1, dict2): def maybe_load_offline_model( dynamics: Dynamics, cfg: DictConfig, - # obs_shape, - # act_shape, work_dir, ): work_dir = pathlib.Path(work_dir) @@ -42,7 +42,7 @@ def maybe_load_offline_model( else: task_exp_dir = work_dir.parent - transition_cfg = OmegaConf.to_container(cfg.transition.mech, resolve=True) + transition_cfg = OmegaConf.to_container(cfg.transition, resolve=True) for time_dir in task_exp_dir.glob(r"*"): if (time_dir / "multirun.yaml").exists(): # exp by hydra's MULTIRUN mode, multi exp in this time @@ -55,8 +55,12 @@ def maybe_load_offline_model( continue exp_cfg = load_hydra_cfg(exp_dir) - exp_transition_dir = OmegaConf.to_container(exp_cfg.transition.mech, resolve=True) - if compare_dict(exp_transition_dir, transition_cfg) and (exp_dir / "transition").exists(): + exp_transition_dir = OmegaConf.to_container(exp_cfg.transition, resolve=True) + if ( + cfg.seed == exp_cfg.seed + and compare_dict(exp_transition_dir, transition_cfg) + and (exp_dir / "transition").exists() + ): dynamics.transition.load(exp_dir / "transition") print("loaded dynamics from {}".format(exp_dir)) return True diff --git a/cmrl/examples/conf/main.yaml b/cmrl/examples/conf/main.yaml index 988eb9f..1a9f794 100644 --- a/cmrl/examples/conf/main.yaml +++ b/cmrl/examples/conf/main.yaml @@ -9,7 +9,7 @@ defaults: seed: 0 device: "cuda:0" -exp_name: refactor +exp_name: default wandb: false root_dir: "./exp" diff --git a/cmrl/examples/conf/task/continuous_cart_pole_swingup.yaml b/cmrl/examples/conf/task/continuous_cart_pole_swingup.yaml index 820b7a2..5b55618 100644 --- a/cmrl/examples/conf/task/continuous_cart_pole_swingup.yaml +++ b/cmrl/examples/conf/task/continuous_cart_pole_swingup.yaml @@ -15,29 +15,9 @@ epoch_length: 10000 n_eval_episodes: 8 eval_freq: 100 -# dynamics -learning_reward: false -learning_terminal: false -ensemble_num: 7 -elite_num: 5 -multi_step: "forward_euler_5" - -# conditional mutual information test(causal discovery) -oracle: true -cit_threshold: 0.02 -test_freq: 100 -# causal -update_causal_mask_ratio: 0.25 -discovery_schedule: [ 1, 30, 250, 250 ] - # offline -penalty_coeff: 0.2 +penalty_coeff: 1 use_ratio: 1 # dyna freq_train_model: 100 - -# model learning -patience: 20 -longest_epoch: -1 -improvement_threshold: 0.01 diff --git a/cmrl/examples/conf/transition/CMI_test.yaml b/cmrl/examples/conf/transition/CMI_test.yaml index ab2aaab..15c10bc 100644 --- a/cmrl/examples/conf/transition/CMI_test.yaml +++ b/cmrl/examples/conf/transition/CMI_test.yaml @@ -1,6 +1,6 @@ name: "CMI_test_transition" learn: true -discovery: false +discovery: true encoder_cfg: _partial_: true @@ -39,6 +39,12 @@ optimizer_cfg: weight_decay: 1e-5 eps: 1e-8 +scheduler_cfg: + _partial_: true + _target_: torch.optim.lr_scheduler.StepLR + step_size: 1 + gamma: 0.95 + mech: _partial_: true _recursive_: false @@ -47,6 +53,10 @@ mech: name: transition input_variables: ??? output_variables: ??? + # model learning + patience: 20 + longest_epoch: -1 + improvement_threshold: 0.01 # mask mask_method: "zero" # ensemble @@ -57,6 +67,7 @@ mech: encoder_cfg: ${transition.encoder_cfg} decoder_cfg: ${transition.decoder_cfg} optimizer_cfg: ${transition.optimizer_cfg} + scheduler_cfg: ${transition.scheduler_cfg} # forward method residual: true encoder_reduction: "sum" diff --git a/cmrl/examples/conf/transition/plain.yaml b/cmrl/examples/conf/transition/plain.yaml index efde025..bd4a824 100644 --- a/cmrl/examples/conf/transition/plain.yaml +++ b/cmrl/examples/conf/transition/plain.yaml @@ -39,6 +39,8 @@ optimizer_cfg: weight_decay: 1e-5 eps: 1e-8 + + mech: _partial_: true _recursive_: false @@ -47,6 +49,11 @@ mech: name: transition input_variables: ??? output_variables: ??? + # model learning + patience: 20 + longest_epoch: -1 + improvement_threshold: 0.01 + # ensemble ensemble_num: 7 elite_num: 5 # cfgs @@ -61,3 +68,8 @@ mech: logger: ??? # others device: ${device} + +# model learning +patience: 20 +longest_epoch: -1 +improvement_threshold: 0.01 diff --git a/cmrl/examples/conf/transition/reinforce.yaml b/cmrl/examples/conf/transition/reinforce.yaml index eb8cf3d..8bfb69e 100644 --- a/cmrl/examples/conf/transition/reinforce.yaml +++ b/cmrl/examples/conf/transition/reinforce.yaml @@ -54,6 +54,10 @@ mech: name: transition input_variables: ??? output_variables: ??? + # model learning + patience: 20 + longest_epoch: -1 + improvement_threshold: 0.01 # ensemble ensemble_num: 7 elite_num: 5 diff --git a/cmrl/models/causal_mech/CMI_test.py b/cmrl/models/causal_mech/CMI_test.py index 5039d7a..6efccfd 100644 --- a/cmrl/models/causal_mech/CMI_test.py +++ b/cmrl/models/causal_mech/CMI_test.py @@ -20,6 +20,10 @@ def __init__( name: str, input_variables: List[Variable], output_variables: List[Variable], + # model learning + longest_epoch: int = -1, + improvement_threshold: float = 0.01, + patience: int = 5, # ensemble ensemble_num: int = 7, elite_num: int = 5, @@ -47,6 +51,9 @@ def __init__( name=name, input_variables=input_variables, output_variables=output_variables, + longest_epoch=longest_epoch, + improvement_threshold=improvement_threshold, + patience=patience, ensemble_num=ensemble_num, elite_num=elite_num, network_cfg=network_cfg, @@ -192,26 +199,23 @@ def CMI_forward(self, inputs: MutableMapping[str, torch.Tensor]) -> Dict[str, to return outputs - def calculate_CMI(self, nll_loss: torch.Tensor): + def calculate_CMI(self, nll_loss: torch.Tensor, threshold=1): nll_loss_diff = nll_loss[:-1] - nll_loss[-1] - self.forward_mask = (nll_loss_diff.mean(dim=(1, 2)) > 1).to(torch.long) - - print(self.forward_mask) + graph_data = (nll_loss_diff.mean(dim=(1, 2)) > threshold).to(torch.long) + return graph_data, nll_loss_diff.mean(dim=(1, 2)) def learn( self, # loader train_loader: DataLoader, valid_loader: DataLoader, - # model learning - longest_epoch: int = -1, - improvement_threshold: float = 0.01, - patience: int = 5, work_dir: Optional[Union[str, pathlib.Path]] = None, **kwargs ): if self.discovery: - epoch_iter = range(longest_epoch) if longest_epoch >= 0 else count() + final_graph_data = None + + epoch_iter = range(self.longest_epoch) if self.longest_epoch >= 0 else count() epochs_since_update = 0 loss_func = partial(variable_loss_func, output_variables=self.output_variables, device=self.device) @@ -225,11 +229,16 @@ def learn( eval_loss = eval(valid_loader) improvement = (best_eval_loss - eval_loss.mean(dim=(0, 2, 3))) / torch.abs(best_eval_loss) - if (improvement > improvement_threshold).any().item(): + if (improvement > self.improvement_threshold).any().item(): best_eval_loss = torch.minimum(best_eval_loss, eval_loss.mean(dim=(0, 2, 3))) epochs_since_update = 0 - self.calculate_CMI(eval_loss) + final_graph_data, mean_nll_loss_diff = self.calculate_CMI(eval_loss) + print( + "new best valid, CMI test result:\n{}\nwith mean nll loss diff:\n{}".format( + final_graph_data, mean_nll_loss_diff + ) + ) else: epochs_since_update += 1 @@ -243,19 +252,17 @@ def learn( self.logger.record("{}-CMI-test/train_loss".format(self.name), train_loss.mean().item()) self.logger.record("{}-CMI-test/val_loss".format(self.name), eval_loss.mean().item()) self.logger.record("{}-CMI-test/best_val_loss".format(self.name), best_eval_loss.mean().item()) + self.logger.record("{}-CMI-test/lr".format(self.name), self.optimizer.param_groups[0]["lr"]) self.logger.dump(self.total_CMI_epoch) - if patience and epochs_since_update >= patience: + if self.patience and epochs_since_update >= self.patience: break - super(CMITest, self).learn( - train_loader=train_loader, - valid_loader=valid_loader, - # model learning - longest_epoch=longest_epoch, - improvement_threshold=improvement_threshold, - patience=patience, - work_dir=work_dir, - **kwargs - ) + self.scheduler.step() + + assert final_graph_data is not None + self.set_oracle_graph(final_graph_data) + self.build_optimizer() + + super(CMITest, self).learn(train_loader=train_loader, valid_loader=valid_loader, work_dir=work_dir, **kwargs) diff --git a/cmrl/models/causal_mech/neural_causal_mech.py b/cmrl/models/causal_mech/neural_causal_mech.py index 426c89e..b55bc89 100644 --- a/cmrl/models/causal_mech/neural_causal_mech.py +++ b/cmrl/models/causal_mech/neural_causal_mech.py @@ -67,6 +67,15 @@ ) ) +default_scheduler_cfg = DictConfig( + dict( + _target_="torch.optim.lr_scheduler.StepLR", + _partial_=True, + step_size=1, + gamma=1, + ) +) + class NeuralCausalMech(BaseCausalMech): def __init__( @@ -74,6 +83,10 @@ def __init__( name: str, input_variables: List[Variable], output_variables: List[Variable], + # model learning + longest_epoch: int = -1, + improvement_threshold: float = 0.01, + patience: int = 5, # ensemble ensemble_num: int = 7, elite_num: int = 5, @@ -82,6 +95,7 @@ def __init__( encoder_cfg: Optional[DictConfig] = None, decoder_cfg: Optional[DictConfig] = None, optimizer_cfg: Optional[DictConfig] = None, + scheduler_cfg: Optional[DictConfig] = None, # forward method residual: bool = True, encoder_reduction: str = "sum", @@ -98,6 +112,10 @@ def __init__( output_variables=output_variables, device=device, ) + # model learning + self.longest_epoch = longest_epoch + self.improvement_threshold = improvement_threshold + self.patience = patience # ensemble self.ensemble_num = ensemble_num self.elite_num = elite_num @@ -106,6 +124,7 @@ def __init__( self.encoder_cfg = default_encoder_cfg if encoder_cfg is None else encoder_cfg self.decoder_cfg = default_decoder_cfg if decoder_cfg is None else decoder_cfg self.optimizer_cfg = default_optimizer_cfg if optimizer_cfg is None else optimizer_cfg + self.scheduler_cfg = default_scheduler_cfg if scheduler_cfg is None else scheduler_cfg # forward method self.residual = residual self.encoder_reduction = encoder_reduction @@ -119,6 +138,7 @@ def __init__( self.network: Optional[BaseNetwork] = None self.graph: Optional[BaseGraph] = None self.optimizer: Optional[Optimizer] = None + self.scheduler: Optional[object] = None self.build_coder() self.build_network() self.build_graph() @@ -183,6 +203,7 @@ def build_optimizer(self): ) self.optimizer = instantiate(self.optimizer_cfg)(params=chain(*params)) + self.scheduler = instantiate(self.scheduler_cfg)(optimizer=self.optimizer) @abstractmethod def build_graph(self): @@ -227,15 +248,11 @@ def learn( # loader train_loader: DataLoader, valid_loader: DataLoader, - # model learning - longest_epoch: int = -1, - improvement_threshold: float = 0.01, - patience: int = 5, work_dir: Optional[Union[str, pathlib.Path]] = None, **kwargs ): best_weights: Optional[Dict] = None - epoch_iter = range(longest_epoch) if longest_epoch >= 0 else count() + epoch_iter = range(self.longest_epoch) if self.longest_epoch >= 0 else count() epochs_since_update = 0 loss_func = partial(variable_loss_func, output_variables=self.output_variables, device=self.device) @@ -247,8 +264,9 @@ def learn( for epoch in epoch_iter: train_loss = train(train_loader) eval_loss = eval(valid_loader) + maybe_best_weights = self._maybe_get_best_weights( - best_eval_loss, eval_loss.mean(dim=(-2, -1)), improvement_threshold + best_eval_loss, eval_loss.mean(dim=(-2, -1)), self.improvement_threshold ) if maybe_best_weights: # best loss @@ -268,12 +286,15 @@ def learn( self.logger.record("{}/train_loss".format(self.name), train_loss.mean().item()) self.logger.record("{}/val_loss".format(self.name), eval_loss.mean().item()) self.logger.record("{}/best_val_loss".format(self.name), best_eval_loss.mean().item()) + self.logger.record("{}/lr".format(self.name), self.optimizer.param_groups[0]["lr"]) self.logger.dump(self.total_epoch) - if patience and epochs_since_update >= patience: + if self.patience and epochs_since_update >= self.patience: break + self.scheduler.step() + # saving the best models: self._maybe_set_best_weights_and_elite(best_weights, best_eval_loss) diff --git a/cmrl/models/causal_mech/plain_mech.py b/cmrl/models/causal_mech/plain_mech.py index 5393ba6..f99961c 100644 --- a/cmrl/models/causal_mech/plain_mech.py +++ b/cmrl/models/causal_mech/plain_mech.py @@ -15,6 +15,11 @@ def __init__( name: str, input_variables: List[Variable], output_variables: List[Variable], + # model learning + longest_epoch: int = -1, + improvement_threshold: float = 0.01, + patience: int = 5, + # ensemble ensemble_num: int = 7, elite_num: int = 5, # cfgs @@ -39,6 +44,9 @@ def __init__( name=name, input_variables=input_variables, output_variables=output_variables, + longest_epoch=longest_epoch, + improvement_threshold=improvement_threshold, + patience=patience, ensemble_num=ensemble_num, elite_num=elite_num, network_cfg=network_cfg, diff --git a/cmrl/models/causal_mech/reinforce.py b/cmrl/models/causal_mech/reinforce.py index 3197c87..0494a28 100644 --- a/cmrl/models/causal_mech/reinforce.py +++ b/cmrl/models/causal_mech/reinforce.py @@ -34,6 +34,10 @@ def __init__( name: str, input_variables: List[Variable], output_variables: List[Variable], + # model learning + longest_epoch: int = -1, + improvement_threshold: float = 0.01, + patience: int = 5, # ensemble ensemble_num: int = 7, elite_num: int = 5, @@ -76,6 +80,9 @@ def __init__( name=name, input_variables=input_variables, output_variables=output_variables, + longest_epoch=longest_epoch, + improvement_threshold=improvement_threshold, + patience=patience, ensemble_num=ensemble_num, elite_num=elite_num, network_cfg=network_cfg, @@ -299,9 +306,6 @@ def learn( self, train_loader: DataLoader, valid_loader: DataLoader, - longest_epoch: int = -1, - improvement_threshold: float = 0.01, - patience: int = 5, graph_data_ratio: float = 0.5, train_graph_freq: int = 2, work_dir: Optional[Union[str, pathlib.Path]] = None, @@ -310,7 +314,7 @@ def learn( assert 0 <= graph_data_ratio <= 1, "graph data ratio should be in [0, 1]" best_weights: Optional[Dict] = None - epoch_iter = range(longest_epoch) if longest_epoch >= 0 else count() + epoch_iter = range(self.longest_epoch) if self.longest_epoch >= 0 else count() epochs_since_update = 0 loss_fn = partial(variable_loss_func, output_variables=self.output_variables, device=self.device) @@ -328,7 +332,7 @@ def learn( eval_loss = eval_fn(valid_loader) maybe_best_weights = self._maybe_get_best_weights( - best_eval_loss, eval_loss.mean(dim=(-2, -1)), improvement_threshold + best_eval_loss, eval_loss.mean(dim=(-2, -1)), self.improvement_threshold ) if maybe_best_weights: # best loss @@ -354,7 +358,7 @@ def learn( self.logger.dump(self.total_epoch) - if patience and epochs_since_update >= patience: + if self.patience and epochs_since_update >= self.patience: break # saving the best models diff --git a/cmrl/models/dynamics.py b/cmrl/models/dynamics.py index cf843e8..55c670c 100644 --- a/cmrl/models/dynamics.py +++ b/cmrl/models/dynamics.py @@ -65,42 +65,15 @@ def get_loader(self, real_replay_buffer, mech: str, batch_size: int = 128): return train_loader, valid_loader - def learn( - self, - real_replay_buffer: ReplayBuffer, - # model learning - longest_epoch: int = -1, - improvement_threshold: float = 0.01, - patience: int = 5, - work_dir: Optional[Union[str, pathlib.Path]] = None, - **kwargs - ): + def learn(self, real_replay_buffer: ReplayBuffer, work_dir: Optional[Union[str, pathlib.Path]] = None, **kwargs): # transition - self.transition.learn( - *self.get_loader(real_replay_buffer, "transition"), - longest_epoch=longest_epoch, - improvement_threshold=improvement_threshold, - patience=patience, - work_dir=work_dir - ) + self.transition.learn(*self.get_loader(real_replay_buffer, "transition"), work_dir=work_dir) # reward-mech if self.learn_reward: - self.reward_mech.learn( - *self.get_loader(real_replay_buffer, "reward_mech"), - longest_epoch=longest_epoch, - improvement_threshold=improvement_threshold, - patience=patience, - work_dir=work_dir - ) + self.reward_mech.learn(*self.get_loader(real_replay_buffer, "reward_mech"), work_dir=work_dir) # termination-mech if self.learn_termination: - self.termination_mech.learn( - *self.get_loader(real_replay_buffer, "termination_mech"), - longest_epoch=longest_epoch, - improvement_threshold=improvement_threshold, - patience=patience, - work_dir=work_dir - ) + self.termination_mech.learn(*self.get_loader(real_replay_buffer, "termination_mech"), work_dir=work_dir) def step(self, batch_obs, batch_action): with torch.no_grad(): From 4ca25e077767ab9b455783506be7080acb3cf1d7 Mon Sep 17 00:00:00 2001 From: frank Date: Tue, 6 Dec 2022 14:18:13 +0800 Subject: [PATCH 43/68] :tada: save to work dir --- cmrl/examples/conf/transition/CMI_test.yaml | 2 +- cmrl/models/causal_mech/base_causal_mech.py | 2 +- cmrl/models/causal_mech/neural_causal_mech.py | 9 ++++++--- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/cmrl/examples/conf/transition/CMI_test.yaml b/cmrl/examples/conf/transition/CMI_test.yaml index 15c10bc..11816e3 100644 --- a/cmrl/examples/conf/transition/CMI_test.yaml +++ b/cmrl/examples/conf/transition/CMI_test.yaml @@ -43,7 +43,7 @@ scheduler_cfg: _partial_: true _target_: torch.optim.lr_scheduler.StepLR step_size: 1 - gamma: 0.95 + gamma: 1 mech: _partial_: true diff --git a/cmrl/models/causal_mech/base_causal_mech.py b/cmrl/models/causal_mech/base_causal_mech.py index 64e999c..918d914 100644 --- a/cmrl/models/causal_mech/base_causal_mech.py +++ b/cmrl/models/causal_mech/base_causal_mech.py @@ -55,7 +55,7 @@ def set_oracle_graph(self, graph_data): self.graph.set_data(graph_data=graph_data) print("set oracle causal graph successfully: \n{}".format(graph_data)) - def save(self): + def save(self, save_dir: Union[str, pathlib.Path]): pass def load(self, load_dir: Union[str, pathlib.Path]): diff --git a/cmrl/models/causal_mech/neural_causal_mech.py b/cmrl/models/causal_mech/neural_causal_mech.py index b55bc89..8dc8eb7 100644 --- a/cmrl/models/causal_mech/neural_causal_mech.py +++ b/cmrl/models/causal_mech/neural_causal_mech.py @@ -298,11 +298,14 @@ def learn( # saving the best models: self._maybe_set_best_weights_and_elite(best_weights, best_eval_loss) - self.save() + self.save(save_dir=work_dir) - def save(self): - save_dir = pathlib.Path(self.name) + def save(self, save_dir: Union[str, pathlib.Path]): + if isinstance(save_dir, str): + save_dir = pathlib.Path(save_dir) + save_dir = save_dir / pathlib.Path(self.name) save_dir.mkdir(exist_ok=True) + self.network.save(save_dir) for coder in self.variable_encoders.values(): coder.save(save_dir) From 289c5b280567d22b5885614300e7d9d2b0ea6420 Mon Sep 17 00:00:00 2001 From: wz139704646 <632291793@qq.com> Date: Wed, 7 Dec 2022 12:26:59 +0800 Subject: [PATCH 44/68] :wrench: add graph saving and loading --- cmrl/models/causal_mech/neural_causal_mech.py | 4 ++++ cmrl/models/causal_mech/reinforce.py | 2 ++ 2 files changed, 6 insertions(+) diff --git a/cmrl/models/causal_mech/neural_causal_mech.py b/cmrl/models/causal_mech/neural_causal_mech.py index 8dc8eb7..9a852be 100644 --- a/cmrl/models/causal_mech/neural_causal_mech.py +++ b/cmrl/models/causal_mech/neural_causal_mech.py @@ -307,6 +307,8 @@ def save(self, save_dir: Union[str, pathlib.Path]): save_dir.mkdir(exist_ok=True) self.network.save(save_dir) + if self.graph is not None: + self.graph.save(save_dir) for coder in self.variable_encoders.values(): coder.save(save_dir) for coder in self.variable_decoders.values(): @@ -318,6 +320,8 @@ def load(self, load_dir: Union[str, pathlib.Path]): assert load_dir.exists() self.network.load(load_dir) + if self.graph is not None: + self.graph.load(load_dir) for coder in self.variable_encoders.values(): coder.load(load_dir) for coder in self.variable_decoders.values(): diff --git a/cmrl/models/causal_mech/reinforce.py b/cmrl/models/causal_mech/reinforce.py index 0494a28..e923853 100644 --- a/cmrl/models/causal_mech/reinforce.py +++ b/cmrl/models/causal_mech/reinforce.py @@ -364,6 +364,8 @@ def learn( # saving the best models self._maybe_set_best_weights_and_elite(best_weights, best_eval_loss) + self.save(save_dir=work_dir) + def _maybe_get_best_weights( self, best_val_loss: torch.Tensor, val_loss: torch.Tensor, threshold: float = 0.01 ) -> Optional[Dict]: From bb5203d98526d1d1e3b32dfa21bd540d9cd5d3a0 Mon Sep 17 00:00:00 2001 From: acez Date: Sat, 17 Dec 2022 16:02:28 +0800 Subject: [PATCH 45/68] :tada: add neural bernoulli graph --- cmrl/models/causal_mech/CMI_test.py | 5 +- cmrl/models/graphs/neural_graph.py | 109 ++++++++++++++++++ cmrl/models/graphs/prob_graph.py | 6 +- .../test_graphs/test_neural_graph.py | 21 +++- 4 files changed, 136 insertions(+), 5 deletions(-) diff --git a/cmrl/models/causal_mech/CMI_test.py b/cmrl/models/causal_mech/CMI_test.py index 6efccfd..761d57c 100644 --- a/cmrl/models/causal_mech/CMI_test.py +++ b/cmrl/models/causal_mech/CMI_test.py @@ -11,6 +11,7 @@ from cmrl.utils.variables import Variable from cmrl.models.causal_mech.neural_causal_mech import NeuralCausalMech +from cmrl.models.graphs.binary_graph import BinaryGraph from cmrl.models.causal_mech.util import variable_loss_func, train_func, eval_func @@ -76,7 +77,7 @@ def build_network(self): ).to(self.device) def build_graph(self): - self.graph = None + self.graph = BinaryGraph(self.input_var_num, self.output_var_num, device=self.device) def single_step_forward(self, inputs: MutableMapping[str, torch.Tensor]) -> Dict[str, torch.Tensor]: batch_size, _ = self.get_inputs_batch_size(inputs) @@ -262,7 +263,7 @@ def learn( self.scheduler.step() assert final_graph_data is not None - self.set_oracle_graph(final_graph_data) + self.graph.set_data(final_graph_data) self.build_optimizer() super(CMITest, self).learn(train_loader=train_loader, valid_loader=valid_loader, work_dir=work_dir, **kwargs) diff --git a/cmrl/models/graphs/neural_graph.py b/cmrl/models/graphs/neural_graph.py index 5ea56a9..84935cc 100644 --- a/cmrl/models/graphs/neural_graph.py +++ b/cmrl/models/graphs/neural_graph.py @@ -2,10 +2,13 @@ from typing import Optional, Union, Tuple import torch +import torch.nn as nn +import torch.nn.functional as F from omegaconf import DictConfig from hydra.utils import instantiate from cmrl.models.graphs.base_graph import BaseGraph +from cmrl.models.graphs.prob_graph import BaseProbGraph default_network_cfg = DictConfig( dict( @@ -75,3 +78,109 @@ def save(self, save_dir: Union[str, pathlib.Path]): def load(self, load_dir: Union[str, pathlib.Path]): data_dict = torch.load(pathlib.Path(load_dir) / "graph.pth", map_location=self.device) self.graph.load_state_dict(data_dict["graph_network"]) + + +class NeuralBernoulliGraph(NeuralGraph, BaseProbGraph): + + _MASK_VALUE = -9e15 + + def __init__( + self, + in_dim: int, + out_dim: int, + extra_dim: Optional[Union[int, Tuple[int]]] = None, + include_input: bool = False, + network_cfg: Optional[DictConfig] = default_network_cfg, + device: Union[str, torch.device] = "cpu", + *args, + **kwargs + ) -> None: + super().__init__( + in_dim=in_dim, + out_dim=out_dim, + extra_dim=extra_dim, + include_input=include_input, + network_cfg=network_cfg, + device=device, + *args, + **kwargs + ) + + def _build_graph_network(self): + super()._build_graph_network() + + def init_weights_zero(layer): + for pname, params in layer.named_parameters(): + if "weight" in pname: + nn.init.zeros_(params) + + self.graph.apply(init_weights_zero) + + def get_adj_matrix(self, inputs: torch.Tensor, *args, **kwargs) -> torch.Tensor: + return torch.sigmoid(super().get_adj_matrix(inputs, *args, **kwargs)) + + def get_binary_adj_matrix(self, inputs: torch.Tensor, threshold: float = 0.5, *args, **kwargs) -> torch.Tensor: + """return the binary adjacency matrices corresponding to the inputs (w/o grad.)""" + return super().get_binary_adj_matrix(inputs, threshold, *args, **kwargs) + + def sample( + self, + prob_matrix: Optional[torch.Tensor], + sample_size: Union[Tuple[int], int], + reparameterization: Optional[str] = None, + *args, + **kwargs + ) -> torch.Tensor: + """sample from given or current graph probability (Bernoulli distribution). + + Args: + prob_matrix (tensor), graph probability, can not be empty here. + sample_size (tuple(int) or int), extra size of sampled graphs. + + Return: + (tensor): [*sample_size, *extra_dim, in_dim, out_dim] shaped multiple graphs. + """ + if prob_matrix is None: + raise ValueError("Porb. matrix can not be empty") + + if isinstance(sample_size, int): + sample_size = (sample_size,) + + sample_prob = prob_matrix[None].expand(*sample_size, -1, -1) + + if reparameterization is None: + return torch.bernoulli(sample_prob) + elif reparameterization == "gumbel-softmax": + return F.gumbel_softmax(torch.stack((sample_prob, 1 - sample_prob)), hard=True, dim=0)[0] + else: + raise NotImplementedError + + def sample_from_inputs( + self, + inputs: torch.Tensor, + sample_size: Union[Tuple[int], int], + reparameterization: Optional[str] = "gumbel-softmax", + *args, + **kwargs + ) -> torch.Tensor: + """sample adjacency matrix from inputs (genereated Bernoulli distribution given the inputs). + + Args: + inputs (tensor), input samples. + sample_size (tuple(int) or int), extra size of sampled graphs. + + Return: + (tensor): [*sample_size, *extra_dim, in_dim, out_dim] shaped multiple graphs. + """ + if isinstance(sample_size, int): + sample_size = (sample_size,) + + inputs = inputs[None].expand(*sample_size, *((-1,) * len(inputs.shape))) + sample_prob = self.get_adj_matrix(inputs) + + if reparameterization is None: + return torch.bernoulli(sample_prob) + elif reparameterization == "gumbel-softmax": + return F.gumbel_softmax(torch.stack((sample_prob, 1 - sample_prob)), hard=True, dim=0)[0] + else: + raise NotImplementedError diff --git a/cmrl/models/graphs/prob_graph.py b/cmrl/models/graphs/prob_graph.py index b7ee533..6bfc65e 100644 --- a/cmrl/models/graphs/prob_graph.py +++ b/cmrl/models/graphs/prob_graph.py @@ -3,6 +3,7 @@ import torch import numpy as np +import torch.nn.functional as F from cmrl.models.graphs.base_graph import BaseGraph from cmrl.models.graphs.weight_graph import WeightGraph @@ -103,10 +104,11 @@ def sample( if isinstance(sample_size, int): sample_size = (sample_size,) - sample_prob = prob_matrix[None].expand(*sample_size, -1, -1) + sample_prob = prob_matrix[None].expand(*sample_size, *((-1,) * len(prob_matrix.shape))) if reparameterization is None: return torch.bernoulli(sample_prob) + elif reparameterization == "gumbel-softmax": + return F.gumbel_softmax(torch.stack((sample_prob, 1 - sample_prob)), hard=True, dim=0)[0] else: - # TODO: reparameterization for bernoulli distribution raise NotImplementedError diff --git a/tests/test_models/test_graphs/test_neural_graph.py b/tests/test_models/test_graphs/test_neural_graph.py index f4cca42..d8709ca 100644 --- a/tests/test_models/test_graphs/test_neural_graph.py +++ b/tests/test_models/test_graphs/test_neural_graph.py @@ -5,7 +5,7 @@ import torch import torch.nn as nn -from cmrl.models.graphs.neural_graph import NeuralGraph +from cmrl.models.graphs.neural_graph import NeuralGraph, NeuralBernoulliGraph def test_init(): @@ -59,6 +59,25 @@ def test_save_load(): shutil.rmtree(save_dir) +def test_bernoulli(): + g = NeuralBernoulliGraph(5, 5, include_input=True) + assert next(g.parameters).grad is None + + inputs = torch.ones(2, 5) + adj_mat = g.get_adj_matrix(inputs) + + assert adj_mat.size() == (2, 5, 5), "get_adj_matrix failed" + assert (adj_mat[:, torch.arange(5), torch.arange(5)] == 0).all() + assert ((adj_mat >= 0) & (adj_mat <= 1)).all() + b = adj_mat.sum() + b.backward() + assert next(g.graph.parameters()).grad is not None + + binary_adj_matrix = g.get_binary_adj_matrix(inputs, 0.5) + + assert binary_adj_matrix.size() == (2, 5, 5), "get_binary_adj_matrix failed" + + if __name__ == "__main__": test_init() From ea72762065b90299cbd1fee2186ce8cfb9a579f2 Mon Sep 17 00:00:00 2001 From: acez Date: Sat, 17 Dec 2022 16:23:39 +0800 Subject: [PATCH 46/68] :wrench: fix dev requirements --- requirements/dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/dev.txt b/requirements/dev.txt index e47eb23..f7916c3 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -3,6 +3,6 @@ pytest>=7.1.3 pytest-cov>=4.0.0 flake8>=5.0.4 mkdocs>=1.4.1 -Pygments>2.13.0 +Pygments>=2.13.0 mkdocstrings>=0.19.0 mkdocstrings-python>=1.0.14 From 711d0f286a00da9a6dbfa00a6e03b485b9fdd00b Mon Sep 17 00:00:00 2001 From: frank Date: Thu, 12 Jan 2023 11:07:38 +0800 Subject: [PATCH 47/68] :tada: update config --- README.md | 2 +- cmrl/algorithms/base_algorithm.py | 2 +- cmrl/examples/conf/algorithm/off_dyna.yaml | 2 +- .../task/continuous_cart_pole_swingup.yaml | 10 +-- cmrl/examples/conf/transition/CMI_test.yaml | 4 +- cmrl/examples/conf/transition/plain.yaml | 7 +- cmrl/examples/conf/transition/reinforce.yaml | 2 +- cmrl/models/fake_env.py | 64 +++++++++---------- requirements/main.txt | 1 - 9 files changed, 45 insertions(+), 49 deletions(-) diff --git a/README.md b/README.md index 6925e6e..28361e1 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ `cmrl`(short for `Causal-MBRL`) is a toolbox for facilitating the development of Causal Model-based Reinforcement -learning algorithms. It use [Stable-Baselines3](https://github.com/DLR-RM/stable-baselines3) as model-free engine and +learning algorithms. It uses [Stable-Baselines3](https://github.com/DLR-RM/stable-baselines3) as model-free engine and allows flexible use of causal models. `cmrl` is inspired by [MBRL-Lib](https://github.com/facebookresearch/mbrl-lib). Unlike MBRL-Lib, `cmrl` focuses on the diff --git a/cmrl/algorithms/base_algorithm.py b/cmrl/algorithms/base_algorithm.py index af51b82..aa1d370 100644 --- a/cmrl/algorithms/base_algorithm.py +++ b/cmrl/algorithms/base_algorithm.py @@ -92,7 +92,7 @@ def callback(self) -> BaseCallback: fake_eval_env, n_eval_episodes=self.cfg.task.n_eval_episodes, best_model_save_path="./", - eval_freq=1000, + eval_freq=self.cfg.task.eval_freq, deterministic=True, render=False, ) diff --git a/cmrl/examples/conf/algorithm/off_dyna.yaml b/cmrl/examples/conf/algorithm/off_dyna.yaml index 43799c2..445ff78 100644 --- a/cmrl/examples/conf/algorithm/off_dyna.yaml +++ b/cmrl/examples/conf/algorithm/off_dyna.yaml @@ -9,7 +9,7 @@ num_eval_episodes: 5 dataset_size: 1000000 penalty_coeff: ${task.penalty_coeff} -num_envs: 32 +num_envs: 16 deterministic: false agent: _partial_: true diff --git a/cmrl/examples/conf/task/continuous_cart_pole_swingup.yaml b/cmrl/examples/conf/task/continuous_cart_pole_swingup.yaml index 5b55618..f95940c 100644 --- a/cmrl/examples/conf/task/continuous_cart_pole_swingup.yaml +++ b/cmrl/examples/conf/task/continuous_cart_pole_swingup.yaml @@ -5,15 +5,17 @@ params: freq_rate: 1 real_time_scale: 0.02 integrator: "euler" + gravity: 9.8 + length: 0.5 + force_mag: 10.0 dataset: "SAC-expert-replay" # basic RL params -num_steps: 10000000 +num_steps: 3000000 online_num_steps: 10000 -epoch_length: 10000 -n_eval_episodes: 8 -eval_freq: 100 +n_eval_episodes: 5 +eval_freq: 10000 # offline penalty_coeff: 1 diff --git a/cmrl/examples/conf/transition/CMI_test.yaml b/cmrl/examples/conf/transition/CMI_test.yaml index 11816e3..57afa5d 100644 --- a/cmrl/examples/conf/transition/CMI_test.yaml +++ b/cmrl/examples/conf/transition/CMI_test.yaml @@ -54,7 +54,7 @@ mech: input_variables: ??? output_variables: ??? # model learning - patience: 20 + patience: 5 longest_epoch: -1 improvement_threshold: 0.01 # mask @@ -70,7 +70,7 @@ mech: scheduler_cfg: ${transition.scheduler_cfg} # forward method residual: true - encoder_reduction: "sum" + encoder_reduction: "max" multi_step: "forward-euler 1" # logger logger: ??? diff --git a/cmrl/examples/conf/transition/plain.yaml b/cmrl/examples/conf/transition/plain.yaml index bd4a824..d56132d 100644 --- a/cmrl/examples/conf/transition/plain.yaml +++ b/cmrl/examples/conf/transition/plain.yaml @@ -50,7 +50,7 @@ mech: input_variables: ??? output_variables: ??? # model learning - patience: 20 + patience: 5 longest_epoch: -1 improvement_threshold: 0.01 # ensemble @@ -68,8 +68,3 @@ mech: logger: ??? # others device: ${device} - -# model learning -patience: 20 -longest_epoch: -1 -improvement_threshold: 0.01 diff --git a/cmrl/examples/conf/transition/reinforce.yaml b/cmrl/examples/conf/transition/reinforce.yaml index 8bfb69e..e0ac544 100644 --- a/cmrl/examples/conf/transition/reinforce.yaml +++ b/cmrl/examples/conf/transition/reinforce.yaml @@ -55,7 +55,7 @@ mech: input_variables: ??? output_variables: ??? # model learning - patience: 20 + patience: 5 longest_epoch: -1 improvement_threshold: 0.01 # ensemble diff --git a/cmrl/models/fake_env.py b/cmrl/models/fake_env.py index b6d37a0..da483c2 100644 --- a/cmrl/models/fake_env.py +++ b/cmrl/models/fake_env.py @@ -21,26 +21,26 @@ def get_penalty(ensemble_batch_next_obs): class VecFakeEnv(VecEnv): def __init__( - self, - # for need of sb3's agent - num_envs: int, - observation_space: gym.spaces.Space, - action_space: gym.spaces.Space, - # for dynamics - dynamics: Dynamics, - reward_fn: Optional[RewardFnType] = None, - termination_fn: Optional[TermFnType] = None, - get_init_obs_fn: Optional[InitObsFnType] = None, - real_replay_buffer: Optional[ReplayBuffer] = None, - # for offline - penalty_coeff: float = 0.0, - # for behaviour - deterministic: bool = False, - max_episode_steps: int = 1000, - branch_rollout: bool = False, - # others - logger: Optional[Logger] = None, - **kwargs, + self, + # for need of sb3's agent + num_envs: int, + observation_space: gym.spaces.Space, + action_space: gym.spaces.Space, + # for dynamics + dynamics: Dynamics, + reward_fn: Optional[RewardFnType] = None, + termination_fn: Optional[TermFnType] = None, + get_init_obs_fn: Optional[InitObsFnType] = None, + real_replay_buffer: Optional[ReplayBuffer] = None, + # for offline + penalty_coeff: float = 0.0, + # for behaviour + deterministic: bool = False, + max_episode_steps: int = 1000, + branch_rollout: bool = False, + # others + logger: Optional[Logger] = None, + **kwargs, ): super(VecFakeEnv, self).__init__( num_envs=num_envs, @@ -89,8 +89,8 @@ def step_wait(self): batch_terminal = self.termination_fn(batch_next_obs, self._current_batch_obs, self._current_batch_action) if self.penalty_coeff != 0: - penalty = get_penalty(info["origin-next_obs"]).reshape(batch_reward.shape) - batch_reward -= penalty * self.penalty_coeff + penalty = get_penalty(info["origin-next_obs"]).reshape(batch_reward.shape) * self.penalty_coeff + batch_reward -= penalty if self.logger is not None: self.logger.record_mean("rollout/penalty", penalty.mean().item()) @@ -121,11 +121,11 @@ def step_wait(self): ) def reset( - self, - *, - seed: Optional[int] = None, - return_info: bool = False, - options: Optional[dict] = None, + self, + *, + seed: Optional[int] = None, + return_info: bool = False, + options: Optional[dict] = None, ): if self.branch_rollout: upper_bound = self.replay_buffer.buffer_size if self.replay_buffer.full else self.replay_buffer.pos @@ -163,11 +163,11 @@ def render(self, mode="human"): raise NotImplementedError def env_method( - self, - method_name: str, - *method_args, - indices: VecEnvIndices = None, - **method_kwargs, + self, + method_name: str, + *method_args, + indices: VecEnvIndices = None, + **method_kwargs, ) -> List[Any]: pass diff --git a/requirements/main.txt b/requirements/main.txt index 4d7f2d4..787a866 100644 --- a/requirements/main.txt +++ b/requirements/main.txt @@ -8,6 +8,5 @@ imageio>=2.19.0 tensorboard>=2.9.0 mujoco >= 2.2.0 wandb >= 0.13 -emei @ git+https://github.com/FrankTianTT/emei@dev stable-baselines3 @ git+https://github.com/carlosluis/stable-baselines3@fix_tests PyQt5>=5.15.7 From 485515282e433a79f2a6a09191198e290354f8d8 Mon Sep 17 00:00:00 2001 From: frank Date: Sun, 15 Jan 2023 12:53:27 +0800 Subject: [PATCH 48/68] :tada: add exp_collect --- cmrl/examples/conf/transition/CMI_test.yaml | 4 +-- cmrl/utils/config.py | 39 +++++++++++++++++++++ 2 files changed, 41 insertions(+), 2 deletions(-) diff --git a/cmrl/examples/conf/transition/CMI_test.yaml b/cmrl/examples/conf/transition/CMI_test.yaml index 57afa5d..9ad6a76 100644 --- a/cmrl/examples/conf/transition/CMI_test.yaml +++ b/cmrl/examples/conf/transition/CMI_test.yaml @@ -1,6 +1,6 @@ name: "CMI_test_transition" learn: true -discovery: true +discovery: false encoder_cfg: _partial_: true @@ -27,7 +27,7 @@ network_cfg: _partial_: true _recursive_: false _target_: cmrl.models.networks.ParallelMLP - hidden_dims: [ 200, 200 ] + hidden_dims: [ 100, 100 ] bias: true activation_fn_cfg: _target_: torch.nn.SiLU diff --git a/cmrl/utils/config.py b/cmrl/utils/config.py index b62d9a4..befe862 100644 --- a/cmrl/utils/config.py +++ b/cmrl/utils/config.py @@ -1,8 +1,13 @@ import pathlib from typing import Dict, Union, Optional +from collections import defaultdict import omegaconf from omegaconf import DictConfig +import pandas as pd +import numpy as np + +PACKAGE_PATH = pathlib.Path(__file__).parent.parent.parent def load_hydra_cfg(results_dir: Union[str, pathlib.Path]) -> omegaconf.DictConfig: @@ -23,3 +28,37 @@ def load_hydra_cfg(results_dir: Union[str, pathlib.Path]) -> omegaconf.DictConfi if not isinstance(cfg, omegaconf.DictConfig): raise RuntimeError("Configuration format not a omegaconf.DictConf") return cfg + + +def exp_collect(cfg_extractor, + csv_extractor, + env_name="ContinuousCartPoleSwingUp-v0", + exp_name="default", + exp_path=None): + data = defaultdict(list) + + if exp_path is None: + exp_path = PACKAGE_PATH / "exp" + exp_dir = exp_path / exp_name + env_dir = exp_dir / env_name + + for params_dir in env_dir.glob("*"): + for dataset_dir in params_dir.glob("*"): + for time_dir in dataset_dir.glob("*"): + if not (time_dir / ".hydra").exists(): # exp by hydra's MULTIRUN mode, multi exp in this time + time_dir_list = list(time_dir.glob("*")) + else: # only one exp in this time + time_dir_list = [time_dir] + + for single_dir in time_dir_list: + if single_dir.name == "multirun.yaml": + continue + + cfg = load_hydra_cfg(single_dir) + + key = cfg_extractor(cfg, params_dir.name, dataset_dir.name, time_dir.name) + if not key: + continue + + data[key] = csv_extractor(single_dir / "log") + return data From f996955a2201ceaa7d0353c7a11426e82cf97d55 Mon Sep 17 00:00:00 2001 From: frank Date: Mon, 6 Feb 2023 00:10:36 +0800 Subject: [PATCH 49/68] :tada: update config --- cmrl/examples/conf/algorithm/off_dyna.yaml | 4 +- cmrl/examples/conf/task/hopper.yaml | 43 +++++++++++++++++++++ cmrl/examples/conf/transition/CMI_test.yaml | 10 ++--- cmrl/examples/conf/transition/plain.yaml | 4 +- cmrl/utils/env.py | 3 ++ 5 files changed, 54 insertions(+), 10 deletions(-) create mode 100644 cmrl/examples/conf/task/hopper.yaml diff --git a/cmrl/examples/conf/algorithm/off_dyna.yaml b/cmrl/examples/conf/algorithm/off_dyna.yaml index 445ff78..c4d0bec 100644 --- a/cmrl/examples/conf/algorithm/off_dyna.yaml +++ b/cmrl/examples/conf/algorithm/off_dyna.yaml @@ -4,12 +4,10 @@ algo: _partial_: true _target_: cmrl.algorithms.OfflineDyna -num_eval_episodes: 5 - dataset_size: 1000000 penalty_coeff: ${task.penalty_coeff} -num_envs: 16 +num_envs: 8 deterministic: false agent: _partial_: true diff --git a/cmrl/examples/conf/task/hopper.yaml b/cmrl/examples/conf/task/hopper.yaml new file mode 100644 index 0000000..cb23ddf --- /dev/null +++ b/cmrl/examples/conf/task/hopper.yaml @@ -0,0 +1,43 @@ +# env parameters +env_id: "HopperRunning-v0" + +params: + freq_rate: 1 + real_time_scale: 0.01 + integrator: "euler" + +dataset: "SAC-medium" + +# basic RL params +num_steps: 10000000 +online_num_steps: 10000 +epoch_length: 10000 +n_eval_episodes: 8 +eval_freq: 10000 + +# dynamics +learning_reward: false +learning_terminal: false +ensemble_num: 7 +elite_num: 5 +multi_step: "none" + +# conditional mutual information test(causal discovery) +oracle: true +cit_threshold: 0.02 +test_freq: 100 +# causal +update_causal_mask_ratio: 0.25 +discovery_schedule: [ 1, 30, 250, 250 ] + +# offline +penalty_coeff: 1.0 +use_ratio: 1 + +# dyna +freq_train_model: 100 + +# model learning +patience: 10 +longest_epoch: -1 +improvement_threshold: 0.01 \ No newline at end of file diff --git a/cmrl/examples/conf/transition/CMI_test.yaml b/cmrl/examples/conf/transition/CMI_test.yaml index 9ad6a76..25c7dbd 100644 --- a/cmrl/examples/conf/transition/CMI_test.yaml +++ b/cmrl/examples/conf/transition/CMI_test.yaml @@ -1,12 +1,12 @@ name: "CMI_test_transition" learn: true -discovery: false +discovery: true encoder_cfg: _partial_: true _recursive_: false _target_: cmrl.models.networks.VariableEncoder - output_dim: 100 + output_dim: 200 hidden_dims: [ 100 ] bias: true activation_fn_cfg: @@ -54,14 +54,14 @@ mech: input_variables: ??? output_variables: ??? # model learning - patience: 5 + patience: 20 longest_epoch: -1 improvement_threshold: 0.01 # mask mask_method: "zero" # ensemble ensemble_num: 7 - elite_num: 5 + elite_num: 20 # cfgs network_cfg: ${transition.network_cfg} encoder_cfg: ${transition.encoder_cfg} @@ -70,7 +70,7 @@ mech: scheduler_cfg: ${transition.scheduler_cfg} # forward method residual: true - encoder_reduction: "max" + encoder_reduction: "sum" multi_step: "forward-euler 1" # logger logger: ??? diff --git a/cmrl/examples/conf/transition/plain.yaml b/cmrl/examples/conf/transition/plain.yaml index d56132d..4dd7efc 100644 --- a/cmrl/examples/conf/transition/plain.yaml +++ b/cmrl/examples/conf/transition/plain.yaml @@ -1,6 +1,6 @@ name: "plain_transition" learn: true -discovery: false +discovery: true encoder_cfg: _partial_: true @@ -50,7 +50,7 @@ mech: input_variables: ??? output_variables: ??? # model learning - patience: 5 + patience: 20 longest_epoch: -1 improvement_threshold: 0.01 # ensemble diff --git a/cmrl/utils/env.py b/cmrl/utils/env.py index af25ed3..9899681 100644 --- a/cmrl/utils/env.py +++ b/cmrl/utils/env.py @@ -45,4 +45,7 @@ def load_offline_data(env, replay_buffer: ReplayBuffer, dataset_name: str, use_r # set all data for attr in ["observations", "next_observations", "actions", "rewards", "dones", "timeouts"]: + if attr == "dones" and attr not in data_dict and "terminals" in data_dict: + replay_buffer.dones[:sample_data_num, 0] = data_dict["terminals"][sample_idx] + continue getattr(replay_buffer, attr)[:sample_data_num, 0] = data_dict[attr][sample_idx] From ec49fd9ecbbe51ac2a8ffb3671801770a8ff45f8 Mon Sep 17 00:00:00 2001 From: frank Date: Mon, 6 Feb 2023 22:35:42 +0800 Subject: [PATCH 50/68] :tada: update config --- cmrl/algorithms/mopo.py | 21 +++++++++++---------- cmrl/examples/conf/algorithm/mopo.yaml | 6 +++--- cmrl/examples/conf/task/hopper.yaml | 2 +- 3 files changed, 15 insertions(+), 14 deletions(-) diff --git a/cmrl/algorithms/mopo.py b/cmrl/algorithms/mopo.py index c7ecd58..abdcf07 100644 --- a/cmrl/algorithms/mopo.py +++ b/cmrl/algorithms/mopo.py @@ -5,13 +5,14 @@ from cmrl.models.fake_env import VecFakeEnv from cmrl.algorithms.base_algorithm import BaseAlgorithm from cmrl.utils.env import load_offline_data +from cmrl.algorithms.util import maybe_load_offline_model class MOPO(BaseAlgorithm): def __init__( - self, - cfg: DictConfig, - work_dir: Optional[str] = None, + self, + cfg: DictConfig, + work_dir: Optional[str] = None, ): super(MOPO, self).__init__(cfg, work_dir) @@ -26,10 +27,10 @@ def fake_env(self) -> VecFakeEnv: def _setup_learn(self): load_offline_data(self.env, self.real_replay_buffer, self.cfg.task.dataset, self.cfg.task.use_ratio) - self.dynamics.learn( - real_replay_buffer=self.real_replay_buffer, - longest_epoch=self.cfg.task.longest_epoch, - improvement_threshold=self.cfg.task.improvement_threshold, - patience=self.cfg.task.patience, - work_dir=self.work_dir, - ) + existed_trained_model = maybe_load_offline_model(self.dynamics, self.cfg, work_dir=self.work_dir) + + if not existed_trained_model: + self.dynamics.learn( + real_replay_buffer=self.real_replay_buffer, + work_dir=self.work_dir, + ) diff --git a/cmrl/examples/conf/algorithm/mopo.yaml b/cmrl/examples/conf/algorithm/mopo.yaml index 270e1ac..9ff83d8 100644 --- a/cmrl/examples/conf/algorithm/mopo.yaml +++ b/cmrl/examples/conf/algorithm/mopo.yaml @@ -4,12 +4,12 @@ algo: _partial_: true _target_: cmrl.algorithms.MOPO -num_eval_episodes: 5 - dataset_size: 1000000 penalty_coeff: ${task.penalty_coeff} -num_envs: 1000 +branch_rollout_length: 5 + +num_envs: 100 deterministic: false agent: _partial_: true diff --git a/cmrl/examples/conf/task/hopper.yaml b/cmrl/examples/conf/task/hopper.yaml index cb23ddf..9a851ce 100644 --- a/cmrl/examples/conf/task/hopper.yaml +++ b/cmrl/examples/conf/task/hopper.yaml @@ -13,7 +13,7 @@ num_steps: 10000000 online_num_steps: 10000 epoch_length: 10000 n_eval_episodes: 8 -eval_freq: 10000 +eval_freq: 100 # dynamics learning_reward: false From 87fa146c558df5abf2a266c276def20c42cdde41 Mon Sep 17 00:00:00 2001 From: frank Date: Tue, 28 Feb 2023 13:51:46 +0800 Subject: [PATCH 51/68] :hammer: update causal mech framework --- README.md | 8 +- cmrl/algorithms/base_algorithm.py | 11 +- cmrl/algorithms/off_dyna.py | 1 - cmrl/examples/conf/main.yaml | 2 +- cmrl/examples/conf/reward_mech/plain.yaml | 1 - .../examples/conf/termination_mech/plain.yaml | 1 - cmrl/examples/conf/transition/CMI_test.yaml | 8 +- .../examples/conf/transition/kernal_test.yaml | 78 ++++ cmrl/examples/conf/transition/plain.yaml | 3 +- cmrl/examples/conf/transition/reinforce.yaml | 1 - cmrl/models/causal_mech/CMI_test.py | 271 +++++++----- cmrl/models/causal_mech/__init__.py | 5 +- cmrl/models/causal_mech/base.py | 413 ++++++++++++++++++ cmrl/models/causal_mech/base_causal_mech.py | 62 --- cmrl/models/causal_mech/kernel_test.py | 170 +++++++ cmrl/models/causal_mech/neural_causal_mech.py | 154 +++---- cmrl/models/causal_mech/plain_mech.py | 144 ++++-- cmrl/models/causal_mech/reinforce.py | 2 +- cmrl/models/constant.py | 55 +++ cmrl/models/data_loader.py | 201 ++++----- cmrl/models/dynamics.py | 73 ++-- cmrl/models/networks/coder.py | 7 - cmrl/types.py | 1 + cmrl/utils/creator.py | 13 +- cmrl/utils/env.py | 23 +- cmrl/utils/variables.py | 25 +- requirements/dev.txt | 4 +- requirements/main.txt | 2 +- setup.py | 3 +- tests/__init__.py | 0 .../test_causal_mech/test_CMI_test.py | 16 +- .../test_causal_mech/test_plain_mech.py | 8 +- .../test_causal_mech/test_reinforce.py | 8 +- tests/test_models/test_data_loader.py | 26 +- 34 files changed, 1233 insertions(+), 567 deletions(-) create mode 100644 cmrl/examples/conf/transition/kernal_test.yaml create mode 100644 cmrl/models/causal_mech/base.py delete mode 100644 cmrl/models/causal_mech/base_causal_mech.py create mode 100644 cmrl/models/causal_mech/kernel_test.py create mode 100644 cmrl/models/constant.py delete mode 100644 tests/__init__.py diff --git a/README.md b/README.md index 28361e1..fa20294 100644 --- a/README.md +++ b/README.md @@ -111,6 +111,8 @@ cd causal-mbrl # create conda env conda create -n cmrl python=3.8 conda activate cmrl +# install torch +conda install pytorch -c pytorch # install cmrl and its dependent packages pip install -e . ``` @@ -119,8 +121,10 @@ If there is no `cuda` in your device, it's convenient to install `cuda` and `pyt to [pytorch](https://pytorch.org/get-started/locally/)): ````shell -# for example, in the case of cuda=11.3 -conda install pytorch cudatoolkit=11.3 -c pytorch +# for MacOS +conda install pytorch -c pytorch +# for Linux +conda install pytorch pytorch-cuda=11.6 -c pytorch -c nvidia ```` ## install using pip diff --git a/cmrl/algorithms/base_algorithm.py b/cmrl/algorithms/base_algorithm.py index aa1d370..a6eabc1 100644 --- a/cmrl/algorithms/base_algorithm.py +++ b/cmrl/algorithms/base_algorithm.py @@ -18,14 +18,14 @@ class BaseAlgorithm: def __init__( - self, - cfg: DictConfig, - work_dir: Optional[str] = None, + self, + cfg: DictConfig, + work_dir: Optional[str] = None, ): self.cfg = cfg self.work_dir = work_dir or os.getcwd() - self.env, self.reward_fn, self.termination_fn, self.get_init_obs_fn = make_env(self.cfg) + self.env, self.reward_fn, self.termination_fn, self.get_init_obs_fn, self.obs2state_fn = make_env(self.cfg) self.eval_env, *_ = make_env(self.cfg) np.random.seed(self.cfg.seed) torch.manual_seed(self.cfg.seed) @@ -41,7 +41,8 @@ def __init__( ) # create ``cmrl`` dynamics - self.dynamics = create_dynamics(self.cfg, self.env.observation_space, self.env.action_space, logger=self.logger) + self.dynamics = create_dynamics(self.cfg, self.env.observation_space, self.env.action_space, self.obs2state_fn, + logger=self.logger) if not self.cfg.transition.discovery: self.dynamics.transition.set_oracle_graph(self.env.get_transition_graph()) diff --git a/cmrl/algorithms/off_dyna.py b/cmrl/algorithms/off_dyna.py index e4ef2a4..a34dffc 100644 --- a/cmrl/algorithms/off_dyna.py +++ b/cmrl/algorithms/off_dyna.py @@ -2,7 +2,6 @@ from omegaconf import DictConfig -from cmrl.models.fake_env import VecFakeEnv from cmrl.algorithms.base_algorithm import BaseAlgorithm from cmrl.utils.env import load_offline_data from cmrl.algorithms.util import maybe_load_offline_model diff --git a/cmrl/examples/conf/main.yaml b/cmrl/examples/conf/main.yaml index 1a9f794..e2091ab 100644 --- a/cmrl/examples/conf/main.yaml +++ b/cmrl/examples/conf/main.yaml @@ -7,7 +7,7 @@ defaults: - _self_ seed: 0 -device: "cuda:0" +device: "cpu" exp_name: default wandb: false diff --git a/cmrl/examples/conf/reward_mech/plain.yaml b/cmrl/examples/conf/reward_mech/plain.yaml index 75a4387..5658d7c 100644 --- a/cmrl/examples/conf/reward_mech/plain.yaml +++ b/cmrl/examples/conf/reward_mech/plain.yaml @@ -16,7 +16,6 @@ decoder_cfg: _partial_: true _recursive_: false _target_: cmrl.models.networks.VariableDecoder - identity: false input_dim: 100 hidden_dims: [ 100 ] bias: true diff --git a/cmrl/examples/conf/termination_mech/plain.yaml b/cmrl/examples/conf/termination_mech/plain.yaml index 2b81870..5d3081b 100644 --- a/cmrl/examples/conf/termination_mech/plain.yaml +++ b/cmrl/examples/conf/termination_mech/plain.yaml @@ -16,7 +16,6 @@ decoder_cfg: _partial_: true _recursive_: false _target_: cmrl.models.networks.VariableDecoder - identity: false input_dim: 100 hidden_dims: [ 100 ] bias: true diff --git a/cmrl/examples/conf/transition/CMI_test.yaml b/cmrl/examples/conf/transition/CMI_test.yaml index 25c7dbd..29b7dd5 100644 --- a/cmrl/examples/conf/transition/CMI_test.yaml +++ b/cmrl/examples/conf/transition/CMI_test.yaml @@ -16,7 +16,6 @@ decoder_cfg: _partial_: true _recursive_: false _target_: cmrl.models.networks.VariableDecoder - identity: false input_dim: 100 hidden_dims: [ 100 ] bias: true @@ -48,20 +47,20 @@ scheduler_cfg: mech: _partial_: true _recursive_: false - _target_: cmrl.models.causal_mech.CMITest + _target_: cmrl.models.causal_mech.CMITestMEch # base causal-mech params name: transition input_variables: ??? output_variables: ??? # model learning - patience: 20 + patience: 5 longest_epoch: -1 improvement_threshold: 0.01 # mask mask_method: "zero" # ensemble ensemble_num: 7 - elite_num: 20 + elite_num: 5 # cfgs network_cfg: ${transition.network_cfg} encoder_cfg: ${transition.encoder_cfg} @@ -71,7 +70,6 @@ mech: # forward method residual: true encoder_reduction: "sum" - multi_step: "forward-euler 1" # logger logger: ??? # others diff --git a/cmrl/examples/conf/transition/kernal_test.yaml b/cmrl/examples/conf/transition/kernal_test.yaml new file mode 100644 index 0000000..48ef2e0 --- /dev/null +++ b/cmrl/examples/conf/transition/kernal_test.yaml @@ -0,0 +1,78 @@ +name: "kernal_test_transition" +learn: true +discovery: true + +encoder_cfg: + _partial_: true + _recursive_: false + _target_: cmrl.models.networks.VariableEncoder + output_dim: 200 + hidden_dims: [ 100 ] + bias: true + activation_fn_cfg: + _target_: torch.nn.SiLU + +decoder_cfg: + _partial_: true + _recursive_: false + _target_: cmrl.models.networks.VariableDecoder + input_dim: 100 + hidden_dims: [ 100 ] + bias: true + activation_fn_cfg: + _target_: torch.nn.SiLU + +network_cfg: + _partial_: true + _recursive_: false + _target_: cmrl.models.networks.ParallelMLP + hidden_dims: [ 100, 100 ] + bias: true + activation_fn_cfg: + _target_: torch.nn.SiLU + +optimizer_cfg: + _partial_: true + _target_: torch.optim.Adam + lr: 1e-4 + weight_decay: 1e-5 + eps: 1e-8 + +scheduler_cfg: + _partial_: true + _target_: torch.optim.lr_scheduler.StepLR + step_size: 1 + gamma: 1 + +mech: + _partial_: true + _recursive_: false + _target_: cmrl.models.causal_mech.KernelTestMech + # base causal-mech params + name: transition + input_variables: ??? + output_variables: ??? + logger: ??? + # model learning + patience: 5 + longest_epoch: -1 + improvement_threshold: 0.01 + # mask + mask_method: "zero" + # ensemble + ensemble_num: 7 + elite_num: 5 + # cfgs + network_cfg: ${transition.network_cfg} + encoder_cfg: ${transition.encoder_cfg} + decoder_cfg: ${transition.decoder_cfg} + optimizer_cfg: ${transition.optimizer_cfg} + scheduler_cfg: ${transition.scheduler_cfg} + # forward method + residual: true + encoder_reduction: "sum" + # others + device: ${device} + # KCI + sample_num: 2000 + kci_times: 10 diff --git a/cmrl/examples/conf/transition/plain.yaml b/cmrl/examples/conf/transition/plain.yaml index 4dd7efc..4e8b77c 100644 --- a/cmrl/examples/conf/transition/plain.yaml +++ b/cmrl/examples/conf/transition/plain.yaml @@ -16,7 +16,6 @@ decoder_cfg: _partial_: true _recursive_: false _target_: cmrl.models.networks.VariableDecoder - identity: false input_dim: 100 hidden_dims: [ 100 ] bias: true @@ -50,7 +49,7 @@ mech: input_variables: ??? output_variables: ??? # model learning - patience: 20 + patience: 5 longest_epoch: -1 improvement_threshold: 0.01 # ensemble diff --git a/cmrl/examples/conf/transition/reinforce.yaml b/cmrl/examples/conf/transition/reinforce.yaml index e0ac544..44f9102 100644 --- a/cmrl/examples/conf/transition/reinforce.yaml +++ b/cmrl/examples/conf/transition/reinforce.yaml @@ -16,7 +16,6 @@ decoder_cfg: _partial_: true _recursive_: false _target_: cmrl.models.networks.VariableDecoder - identity: false input_dim: 100 hidden_dims: [ 100 ] bias: true diff --git a/cmrl/models/causal_mech/CMI_test.py b/cmrl/models/causal_mech/CMI_test.py index 761d57c..3ac1195 100644 --- a/cmrl/models/causal_mech/CMI_test.py +++ b/cmrl/models/causal_mech/CMI_test.py @@ -4,54 +4,51 @@ from itertools import count import torch +import numpy as np from torch.utils.data import DataLoader from omegaconf import DictConfig from hydra.utils import instantiate from stable_baselines3.common.logger import Logger from cmrl.utils.variables import Variable -from cmrl.models.causal_mech.neural_causal_mech import NeuralCausalMech +from cmrl.models.causal_mech.base import EnsembleNeuralMech from cmrl.models.graphs.binary_graph import BinaryGraph from cmrl.models.causal_mech.util import variable_loss_func, train_func, eval_func -class CMITest(NeuralCausalMech): +class CMITestMech(EnsembleNeuralMech): def __init__( - self, - name: str, - input_variables: List[Variable], - output_variables: List[Variable], - # model learning - longest_epoch: int = -1, - improvement_threshold: float = 0.01, - patience: int = 5, - # ensemble - ensemble_num: int = 7, - elite_num: int = 5, - # cfgs - network_cfg: Optional[DictConfig] = None, - encoder_cfg: Optional[DictConfig] = None, - decoder_cfg: Optional[DictConfig] = None, - optimizer_cfg: Optional[DictConfig] = None, - # forward method - residual: bool = True, - encoder_reduction: str = "sum", - multi_step: str = "none", - # logger - logger: Optional[Logger] = None, - # others - device: Union[str, torch.device] = "cpu", - **kwargs + self, + name: str, + input_variables: List[Variable], + output_variables: List[Variable], + logger: Optional[Logger] = None, + # model learning + longest_epoch: int = -1, + improvement_threshold: float = 0.01, + patience: int = 5, + # ensemble + ensemble_num: int = 7, + elite_num: int = 5, + # cfgs + network_cfg: Optional[DictConfig] = None, + encoder_cfg: Optional[DictConfig] = None, + decoder_cfg: Optional[DictConfig] = None, + optimizer_cfg: Optional[DictConfig] = None, + # forward method + residual: bool = True, + encoder_reduction: str = "sum", + multi_step: str = "none", + # others + device: Union[str, torch.device] = "cpu", + **kwargs ): - if multi_step == "none": - multi_step = "forward-euler 1" - - self.total_CMI_epoch = 0 - - super(CMITest, self).__init__( + EnsembleNeuralMech.__init__( + self, name=name, input_variables=input_variables, output_variables=output_variables, + logger=logger, longest_epoch=longest_epoch, improvement_threshold=improvement_threshold, patience=patience, @@ -63,12 +60,12 @@ def __init__( optimizer_cfg=optimizer_cfg, residual=residual, encoder_reduction=encoder_reduction, - multi_step=multi_step, - logger=logger, device=device, **kwargs ) + self.total_CMI_epoch = 0 + def build_network(self): self.network = instantiate(self.network_cfg)( input_dim=self.encoder_output_dim, @@ -79,16 +76,16 @@ def build_network(self): def build_graph(self): self.graph = BinaryGraph(self.input_var_num, self.output_var_num, device=self.device) - def single_step_forward(self, inputs: MutableMapping[str, torch.Tensor]) -> Dict[str, torch.Tensor]: + def forward(self, inputs: MutableMapping[str, torch.Tensor]) -> Dict[str, torch.Tensor]: batch_size, _ = self.get_inputs_batch_size(inputs) - inputs_tensor = torch.zeros(self.ensemble_num, batch_size, self.input_var_num, self.encoder_output_dim).to(self.device) + inputs_tensor = torch.zeros(self.ensemble_num, batch_size, self.input_var_num, self.encoder_output_dim).to( + self.device) for i, var in enumerate(self.input_variables): out = self.variable_encoders[var.name](inputs[var.name].to(self.device)) inputs_tensor[:, :, i] = out - reduced_inputs_tensor = self.reduce_encoder_output(inputs_tensor) - output_tensor = self.network(reduced_inputs_tensor) + output_tensor = self.network(self.reduce_encoder_output(inputs_tensor)) outputs = {} for i, var in enumerate(self.output_variables): @@ -109,7 +106,7 @@ def CMI_mask(self) -> torch.Tensor: mask[i] = m return mask.to(self.device) - def CMI_single_step_forward(self, inputs: MutableMapping[str, torch.Tensor]) -> Dict[str, torch.Tensor]: + def multi_graph_forward(self, inputs: MutableMapping[str, torch.Tensor]) -> Dict[str, torch.Tensor]: """when first step, inputs should be dict of str and Tensor with (ensemble-num, batch-size, specific-dim) shape, since twice step, the shape of Tensor becomes (input-var-num + 1, ensemble-num, batch-size, specific-dim) @@ -119,49 +116,59 @@ def CMI_single_step_forward(self, inputs: MutableMapping[str, torch.Tensor]) -> Returns: """ - batch_size, extra_dim = self.get_inputs_batch_size(inputs) + batch_size, extra_dim = self.get_inputs_info(inputs) - inputs_tensor = torch.empty(*extra_dim, self.ensemble_num, batch_size, self.input_var_num, self.encoder_output_dim).to( - self.device - ) + inputs_tensor = torch.empty(*extra_dim, self.ensemble_num, batch_size, self.input_var_num, + self.encoder_output_dim).to( + self.device) for i, var in enumerate(self.input_variables): out = self.variable_encoders[var.name](inputs[var.name].to(self.device)) inputs_tensor[..., i, :] = out - if len(extra_dim) == 0: - # [..., output-var-num, input-var-num] - mask = self.CMI_mask - # [..., output-var-num, ensemble-num, batch-size, input-var-num] - mask = mask.unsqueeze(-2).unsqueeze(-2) - mask = mask.repeat((1,) * len(mask.shape[:-3]) + (self.ensemble_num, batch_size, 1)) - reduced_inputs_tensor = self.reduce_encoder_output(inputs_tensor, mask) - assert ( + # if len(extra_dim) == 0: + # # [..., output-var-num, input-var-num] + # mask = self.CMI_mask + # # [..., output-var-num, ensemble-num, batch-size, input-var-num] + # mask = mask.unsqueeze(-2).unsqueeze(-2) + # mask = mask.repeat((1,) * len(mask.shape[:-3]) + (self.ensemble_num, batch_size, 1)) + # reduced_inputs_tensor = self.reduce_encoder_output(inputs_tensor, mask) + # assert ( + # not torch.isinf(reduced_inputs_tensor).any() and not torch.isnan(reduced_inputs_tensor).any() + # ), "tensor must not be inf or nan" + # output_tensor = self.network(reduced_inputs_tensor) + # else: + # output_tensor = torch.empty( + # *extra_dim, self.output_var_num, self.ensemble_num, batch_size, self.decoder_input_dim + # ).to(self.device) + # + # CMI_mask = self.CMI_mask + # for i in range(self.input_var_num + 1): + # # [..., output-var-num, input-var-num] + # mask = CMI_mask[i] + # # [..., output-var-num, ensemble-num, batch-size, input-var-num] + # mask = mask.unsqueeze(-2).unsqueeze(-2) + # mask = mask.repeat((1,) * len(mask.shape[:-3]) + (self.ensemble_num, batch_size, 1)) + # if i == len(inputs_tensor) - 1: + # reduced_inputs_tensor = self.reduce_encoder_output(inputs_tensor[i], mask) + # outs = self.network(reduced_inputs_tensor) + # output_tensor[i] = outs + # else: + # for j in range(self.output_var_num): + # ins = inputs_tensor[-1] + # ins[:, :, j] = inputs_tensor[i, :, :, j, :] + # reduced_inputs_tensor = self.reduce_encoder_output(inputs_tensor[i], mask) + # outs = self.network(reduced_inputs_tensor) + # output_tensor[i, j] = outs[j] + + mask = self.CMI_mask + # [..., output-var-num, ensemble-num, batch-size, input-var-num] + mask = mask.unsqueeze(-2).unsqueeze(-2) + mask = mask.repeat((1,) * len(mask.shape[:-3]) + (self.ensemble_num, batch_size, 1)) + reduced_inputs_tensor = self.reduce_encoder_output(inputs_tensor, mask) + assert ( not torch.isinf(reduced_inputs_tensor).any() and not torch.isnan(reduced_inputs_tensor).any() - ), "tensor must not be inf or nan" - output_tensor = self.network(reduced_inputs_tensor) - else: - output_tensor = torch.empty( - *extra_dim, self.output_var_num, self.ensemble_num, batch_size, self.decoder_input_dim - ).to(self.device) - - CMI_mask = self.CMI_mask - for i in range(self.input_var_num + 1): - # [..., output-var-num, input-var-num] - mask = CMI_mask[i] - # [..., output-var-num, ensemble-num, batch-size, input-var-num] - mask = mask.unsqueeze(-2).unsqueeze(-2) - mask = mask.repeat((1,) * len(mask.shape[:-3]) + (self.ensemble_num, batch_size, 1)) - if i == len(inputs_tensor) - 1: - reduced_inputs_tensor = self.reduce_encoder_output(inputs_tensor[i], mask) - outs = self.network(reduced_inputs_tensor) - output_tensor[i] = outs - else: - for j in range(self.output_var_num): - ins = inputs_tensor[-1] - ins[:, :, j] = inputs_tensor[i, :, :, j, :] - reduced_inputs_tensor = self.reduce_encoder_output(inputs_tensor[i], mask) - outs = self.network(reduced_inputs_tensor) - output_tensor[i, j] = outs[j] + ), "tensor must not be inf or nan" + output_tensor = self.network(reduced_inputs_tensor) outputs = {} for i, var in enumerate(self.output_variables): @@ -172,33 +179,33 @@ def CMI_single_step_forward(self, inputs: MutableMapping[str, torch.Tensor]) -> outputs = self.residual_outputs(inputs, outputs) return outputs - def CMI_forward(self, inputs: MutableMapping[str, torch.Tensor]) -> Dict[str, torch.Tensor]: - """inputs should be dict of str and Tensor with (ensemble-num, batch-size, specific-dim) shape - - Args: - inputs: - - Returns: - - """ - if self.multi_step.startswith("forward-euler"): - step_num = int(self.multi_step.split()[-1]) - - outputs = {} - for step in range(step_num): - outputs = self.CMI_single_step_forward(inputs) - # outputs shape: (input-var-num + 1, ensemble-num, batch-size, specific-dim * 2) - # new inputs shape: (input-var-num + 1, ensemble-num, batch-size, specific-dim) - if step == 0: - for name in filter(lambda s: s.startswith("act"), inputs.keys()): - inputs[name] = inputs[name][None, ...].repeat([self.input_var_num + 1, 1, 1, 1]) - if step < step_num - 1: - for name in filter(lambda s: s.startswith("obs"), inputs.keys()): - inputs[name] = outputs["next_{}".format(name)][..., : inputs[name].shape[-1]] - else: - raise NotImplementedError("multi-step method {} is not supported".format(self.multi_step)) - - return outputs + # def CMI_forward(self, inputs: MutableMapping[str, torch.Tensor]) -> Dict[str, torch.Tensor]: + # """inputs should be dict of str and Tensor with (ensemble-num, batch-size, specific-dim) shape + # + # Args: + # inputs: + # + # Returns: + # + # """ + # if self.multi_step.startswith("forward-euler"): + # step_num = int(self.multi_step.split()[-1]) + # + # outputs = {} + # for step in range(step_num): + # outputs = self.CMI_single_step_forward(inputs) + # # outputs shape: (input-var-num + 1, ensemble-num, batch-size, specific-dim * 2) + # # new inputs shape: (input-var-num + 1, ensemble-num, batch-size, specific-dim) + # if step == 0: + # for name in filter(lambda s: s.startswith("act"), inputs.keys()): + # inputs[name] = inputs[name][None, ...].repeat([self.input_var_num + 1, 1, 1, 1]) + # if step < step_num - 1: + # for name in filter(lambda s: s.startswith("obs"), inputs.keys()): + # inputs[name] = outputs["next_{}".format(name)][..., : inputs[name].shape[-1]] + # else: + # raise NotImplementedError("multi-step method {} is not supported".format(self.multi_step)) + # + # return outputs def calculate_CMI(self, nll_loss: torch.Tensor, threshold=1): nll_loss_diff = nll_loss[:-1] - nll_loss[-1] @@ -206,22 +213,24 @@ def calculate_CMI(self, nll_loss: torch.Tensor, threshold=1): return graph_data, nll_loss_diff.mean(dim=(1, 2)) def learn( - self, - # loader - train_loader: DataLoader, - valid_loader: DataLoader, - work_dir: Optional[Union[str, pathlib.Path]] = None, - **kwargs + self, + # loader + inputs: MutableMapping[str, np.ndarray], + outputs: MutableMapping[str, np.ndarray], + work_dir: Optional[Union[str, pathlib.Path]] = None, + **kwargs ): if self.discovery: + train_loader, valid_loader = self.get_data_loaders(inputs, outputs) + final_graph_data = None epoch_iter = range(self.longest_epoch) if self.longest_epoch >= 0 else count() epochs_since_update = 0 loss_func = partial(variable_loss_func, output_variables=self.output_variables, device=self.device) - train = partial(train_func, forward=self.CMI_forward, optimizer=self.optimizer, loss_func=loss_func) - eval = partial(eval_func, forward=self.CMI_forward, loss_func=loss_func) + train = partial(train_func, forward=self.multi_graph_forward, optimizer=self.optimizer, loss_func=loss_func) + eval = partial(eval_func, forward=self.multi_graph_forward, loss_func=loss_func) best_eval_loss = eval(valid_loader).mean(dim=(0, 2, 3)) @@ -266,4 +275,40 @@ def learn( self.graph.set_data(final_graph_data) self.build_optimizer() - super(CMITest, self).learn(train_loader=train_loader, valid_loader=valid_loader, work_dir=work_dir, **kwargs) + super(CMITestMech, self).learn(inputs, outputs, work_dir=work_dir, **kwargs) + + +if __name__ == '__main__': + import gym + from stable_baselines3.common.buffers import ReplayBuffer + from torch.utils.data import DataLoader + + from cmrl.models.causal_mech.reinforce import ReinforceCausalMech + from cmrl.models.data_loader import EnsembleBufferDataset, collate_fn, buffer_to_dict + from cmrl.utils.creator import parse_space + from cmrl.utils.env import load_offline_data + from cmrl.models.causal_mech.util import variable_loss_func + + env = gym.make("ContinuousCartPoleSwingUp-v0", real_time_scale=0.02) + real_replay_buffer = ReplayBuffer(int(1e6), env.observation_space, env.action_space, "cpu", + handle_timeout_termination=False) + load_offline_data(env, real_replay_buffer, "SAC-expert", use_ratio=1) + + input_variables = parse_space(env.observation_space, "obs") + parse_space(env.action_space, "act") + output_variables = parse_space(env.observation_space, "next_obs") + + mech = CMITestMech( + "kernel_test_mech", + input_variables, + output_variables, + ) + + inputs, outputs = buffer_to_dict( + env.observation_space, + env.action_space, + env.obs2state, + real_replay_buffer, + "transition" + ) + + mech.learn(inputs, outputs) diff --git a/cmrl/models/causal_mech/__init__.py b/cmrl/models/causal_mech/__init__.py index 2b5c138..ffa88a9 100644 --- a/cmrl/models/causal_mech/__init__.py +++ b/cmrl/models/causal_mech/__init__.py @@ -1,3 +1,4 @@ from cmrl.models.causal_mech.plain_mech import PlainMech -from cmrl.models.causal_mech.CMI_test import CMITest -from cmrl.models.causal_mech.reinforce import ReinforceCausalMech +from cmrl.models.causal_mech.CMI_test import CMITestMech +# from cmrl.models.causal_mech.reinforce import ReinforceCausalMech +from cmrl.models.causal_mech.kernel_test import KernelTestMech \ No newline at end of file diff --git a/cmrl/models/causal_mech/base.py b/cmrl/models/causal_mech/base.py new file mode 100644 index 0000000..4e691ea --- /dev/null +++ b/cmrl/models/causal_mech/base.py @@ -0,0 +1,413 @@ +from typing import Optional, List, Dict, Union, MutableMapping +from abc import abstractmethod, ABC +from itertools import chain, count +import pathlib +from functools import partial +import copy + +import numpy as np +import torch +from torch.utils.data import DataLoader +from torch.optim import Optimizer +from omegaconf import DictConfig +from stable_baselines3.common.logger import Logger +from hydra.utils import instantiate + +from cmrl.models.graphs.base_graph import BaseGraph +from cmrl.models.graphs.binary_graph import BinaryGraph +from cmrl.utils.variables import Variable +from cmrl.models.constant import NETWORK_CFG, ENCODER_CFG, DECODER_CFG, OPTIMIZER_CFG, SCHEDULER_CFG +from cmrl.models.networks.base_network import BaseNetwork +from cmrl.models.graphs.base_graph import BaseGraph +from cmrl.models.networks.coder import VariableEncoder, VariableDecoder +from cmrl.utils.variables import Variable, ContinuousVariable, DiscreteVariable, BinaryVariable +from cmrl.models.causal_mech.util import variable_loss_func, train_func, eval_func +from cmrl.models.data_loader import EnsembleBufferDataset, collate_fn + + +class BaseCausalMech(ABC): + """The base class of causal-mech learned by neural networks. + Pay attention that the causal discovery maybe not realized through a neural way. + """ + + def __init__( + self, + name: str, + input_variables: List[Variable], + output_variables: List[Variable], + logger: Optional[Logger] = None, + ): + self.name = name + self.input_variables = input_variables + self.output_variables = output_variables + self.logger=logger + + self.input_variables_dict = dict([(v.name, v) for v in self.input_variables]) + self.output_variables_dict = dict([(v.name, v) for v in self.output_variables]) + + self.input_var_num = len(self.input_variables) + self.output_var_num = len(self.output_variables) + self.graph: Optional[BaseGraph] = None + self.discovery: bool = True + + @abstractmethod + def learn( + self, + inputs: MutableMapping[str, np.ndarray], + outputs: MutableMapping[str, np.ndarray], + work_dir: Optional[Union[str, pathlib.Path]] = None, + **kwargs + ): + raise NotImplementedError + + @abstractmethod + def forward( + self, + inputs: MutableMapping[str, np.ndarray] + ) -> Dict[str, torch.Tensor]: + raise NotImplementedError + + @property + def causal_graph(self) -> torch.Tensor: + """property causal graph""" + if self.graph is None: + return torch.ones(len(self.input_variables), len(self.output_variables), dtype=torch.int, + device=self.device) + else: + return self.graph.get_binary_adj_matrix() + + def set_oracle_graph(self, graph_data): + self.discovery = False + self.graph = BinaryGraph(self.input_var_num, self.output_var_num, device=self.device) + self.graph.set_data(graph_data=graph_data) + print("set oracle causal graph successfully: \n{}".format(graph_data)) + + def save(self, save_dir: Union[str, pathlib.Path]): + pass + + def load(self, load_dir: Union[str, pathlib.Path]): + pass + + +class EnsembleNeuralMech(BaseCausalMech): + def __init__( + self, + # base + name: str, + input_variables: List[Variable], + output_variables: List[Variable], + logger: Optional[Logger] = None, + # model learning + longest_epoch: int = -1, + improvement_threshold: float = 0.01, + patience: int = 5, + # ensemble + ensemble_num: int = 7, + elite_num: int = 5, + # cfgs + network_cfg: Optional[DictConfig] = None, + encoder_cfg: Optional[DictConfig] = None, + decoder_cfg: Optional[DictConfig] = None, + optimizer_cfg: Optional[DictConfig] = None, + scheduler_cfg: Optional[DictConfig] = None, + # forward method + residual: bool = True, + encoder_reduction: str = "sum", + # others + device: Union[str, torch.device] = "cpu", + ): + BaseCausalMech.__init__( + self, + name=name, + input_variables=input_variables, + output_variables=output_variables, + logger=logger + ) + # model learning + self.longest_epoch = longest_epoch + self.improvement_threshold = improvement_threshold + self.patience = patience + # ensemble + self.ensemble_num = ensemble_num + self.elite_num = elite_num + # cfgs + self.network_cfg = NETWORK_CFG if network_cfg is None else network_cfg + self.encoder_cfg = ENCODER_CFG if encoder_cfg is None else encoder_cfg + self.decoder_cfg = DECODER_CFG if decoder_cfg is None else decoder_cfg + self.optimizer_cfg = OPTIMIZER_CFG if optimizer_cfg is None else optimizer_cfg + self.scheduler_cfg = SCHEDULER_CFG if scheduler_cfg is None else scheduler_cfg + # forward method + self.residual = residual + self.encoder_reduction = encoder_reduction + # others + self.device = device + + # build member object + self.variable_encoders: Optional[Dict[str, VariableEncoder]] = None + self.variable_decoders: Optional[Dict[str, VariableEncoder]] = None + self.network: Optional[BaseNetwork] = None + self.graph: Optional[BaseGraph] = None + self.optimizer: Optional[Optimizer] = None + self.scheduler: Optional[object] = None + self.build_coders() + self.build_network() + self.build_graph() + self.build_optimizer() + + self.total_epoch = 0 + self.elite_indices: List[int] = [] + + @property + def encoder_output_dim(self): + return self.encoder_cfg.output_dim + + @property + def decoder_input_dim(self): + return self.decoder_cfg.input_dim + + def build_network(self): + self.network = instantiate(self.network_cfg)( + input_dim=self.encoder_output_dim, + output_dim=self.decoder_input_dim, + extra_dims=[self.ensemble_num], + ).to(self.device) + + def build_optimizer(self): + assert self.network, "you must build network first" + assert self.variable_encoders and self.variable_decoders, "you must build coders first" + params = ( + [self.network.parameters()] + + [encoder.parameters() for encoder in self.variable_encoders.values()] + + [decoder.parameters() for decoder in self.variable_decoders.values()] + ) + + self.optimizer = instantiate(self.optimizer_cfg)(params=chain(*params)) + self.scheduler = instantiate(self.scheduler_cfg)(optimizer=self.optimizer) + + def build_graph(self): + pass + + def build_coders(self): + self.variable_encoders = {} + for var in self.input_variables: + assert var.name not in self.variable_encoders, "duplicate name in encoders: {}".format(var.name) + self.variable_encoders[var.name] = instantiate(self.encoder_cfg)(variable=var).to(self.device) + + self.variable_decoders = {} + for var in self.output_variables: + assert var.name not in self.variable_decoders, "duplicate name in decoders: {}".format(var.name) + self.variable_decoders[var.name] = instantiate(self.decoder_cfg)(variable=var).to(self.device) + + def save(self, save_dir: Union[str, pathlib.Path]): + if isinstance(save_dir, str): + save_dir = pathlib.Path(save_dir) + save_dir = save_dir / pathlib.Path(self.name) + save_dir.mkdir(exist_ok=True) + + self.network.save(save_dir) + if self.graph is not None: + self.graph.save(save_dir) + for coder in self.variable_encoders.values(): + coder.save(save_dir) + for coder in self.variable_decoders.values(): + coder.save(save_dir) + + def load(self, load_dir: Union[str, pathlib.Path]): + if isinstance(load_dir, str): + load_dir = pathlib.Path(load_dir) + assert load_dir.exists() + + self.network.load(load_dir) + if self.graph is not None: + self.graph.load(load_dir) + for coder in self.variable_encoders.values(): + coder.load(load_dir) + for coder in self.variable_decoders.values(): + coder.load(load_dir) + + def get_inputs_info(self, inputs: MutableMapping[str, torch.Tensor]): + assert len(set(inputs.keys()) & set(self.input_variables_dict.keys())) == len(inputs) + data_shape = next(iter(inputs.values())).shape + # assert len(data_shape) == 3, "{}".format(data_shape) # ensemble-num, batch-size, specific-dim + ensemble, batch_size, specific_dim = data_shape[-3:] + assert ensemble == self.ensemble_num + + return batch_size, data_shape[:-3] + + def residual_outputs( + self, + inputs: MutableMapping[str, torch.Tensor], + outputs: MutableMapping[str, torch.Tensor], + ) -> MutableMapping[str, torch.Tensor]: + for name in filter(lambda s: s.startswith("obs"), inputs.keys()): + # assert inputs[name].shape[:2] == outputs["next_{}".format(name)].shape[:2] + # assert inputs[name].shape[2] * 2 == outputs["next_{}".format(name)].shape[2] + var_dim = inputs[name].shape[-1] + outputs["next_{}".format(name)][..., :var_dim] += inputs[name].to(self.device) + return outputs + + def reduce_encoder_output( + self, + encoder_output: torch.Tensor, + mask: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + assert len(encoder_output.shape) == 4, ( + "shape of `encoder_output` should be (ensemble-num, batch-size, input-var-num, encoder-output-dim), " + "rather than {}".format(encoder_output.shape) + ) + + if mask is None: + # [..., input-var-num] + mask = self.forward_mask + # [..., ensemble-num, batch-size, input-var-num] + mask = mask.unsqueeze(-2).unsqueeze(-2) + mask = mask.repeat((1,) * len(mask.shape[:-3]) + (*encoder_output.shape[:2], 1)) + + # mask shape [..., ensemble-num, batch-size, input-var-num] + assert ( + mask.shape[-3:] == encoder_output.shape[:-1] + ), "mask shape should be (..., ensemble-num, batch-size, input-var-num)" + + # [*mask-extra-dims, ensemble-num, batch-size, input-var-num, encoder-output-dim] + mask = mask[..., None].repeat([1] * len(mask.shape) + [encoder_output.shape[-1]]) + masked_encoder_output = encoder_output.repeat(tuple(mask.shape[:-4]) + (1,) * 4) + + # choose mask value + mask_value = 0 + if self.encoder_reduction == "max": + mask_value = -float("inf") + masked_encoder_output[mask == 0] = mask_value + + if self.encoder_reduction == "sum": + return masked_encoder_output.sum(-2) + elif self.encoder_reduction == "mean": + return masked_encoder_output.mean(-2) + elif self.encoder_reduction == "max": + values, indices = masked_encoder_output.max(-2) + return values + else: + raise NotImplementedError("not implemented encoder reduction method: {}".format(self.encoder_reduction)) + + @property + def forward_mask(self) -> torch.Tensor: + """property input masks""" + return self.causal_graph.T + + def get_data_loaders( + self, + inputs: MutableMapping[str, np.ndarray], + outputs: MutableMapping[str, np.ndarray], + ): + train_set = EnsembleBufferDataset( + inputs=inputs, + outputs=outputs, + training=True, + train_ratio=0.8, + ensemble_num=7, + seed=1 + ) + valid_set = EnsembleBufferDataset( + inputs=inputs, + outputs=outputs, + training=False, + train_ratio=0.8, + ensemble_num=7, + seed=1 + ) + + train_loader = DataLoader(train_set, batch_size=32, collate_fn=collate_fn) + valid_loader = DataLoader(valid_set, batch_size=32, collate_fn=collate_fn) + + return train_loader, valid_loader + + def learn( + self, + inputs: MutableMapping[str, np.ndarray], + outputs: MutableMapping[str, np.ndarray], + work_dir: Optional[Union[str, pathlib.Path]] = None, + **kwargs + ): + train_loader, valid_loader = self.get_data_loaders(inputs, outputs) + + best_weights: Optional[Dict] = None + epoch_iter = range(self.longest_epoch) if self.longest_epoch >= 0 else count() + epochs_since_update = 0 + + loss_func = partial(variable_loss_func, output_variables=self.output_variables, device=self.device) + train = partial(train_func, forward=self.forward, optimizer=self.optimizer, loss_func=loss_func) + eval = partial(eval_func, forward=self.forward, loss_func=loss_func) + + best_eval_loss = eval(valid_loader).mean(dim=(-2, -1)) + + for epoch in epoch_iter: + train_loss = train(train_loader) + eval_loss = eval(valid_loader) + + maybe_best_weights = self._maybe_get_best_weights( + best_eval_loss, eval_loss.mean(dim=(-2, -1)), self.improvement_threshold + ) + if maybe_best_weights: + # best loss + best_eval_loss = torch.minimum(best_eval_loss, eval_loss.mean(dim=(-2, -1))) + best_weights = maybe_best_weights + epochs_since_update = 0 + else: + epochs_since_update += 1 + + # log + self.total_epoch += 1 + if self.logger is not None: + self.logger.record("{}/epoch".format(self.name), epoch) + self.logger.record("{}/epochs_since_update".format(self.name), epochs_since_update) + self.logger.record("{}/train_dataset_size".format(self.name), len(train_loader.dataset)) + self.logger.record("{}/valid_dataset_size".format(self.name), len(valid_loader.dataset)) + self.logger.record("{}/train_loss".format(self.name), train_loss.mean().item()) + self.logger.record("{}/val_loss".format(self.name), eval_loss.mean().item()) + self.logger.record("{}/best_val_loss".format(self.name), best_eval_loss.mean().item()) + self.logger.record("{}/lr".format(self.name), self.optimizer.param_groups[0]["lr"]) + + self.logger.dump(self.total_epoch) + + if self.patience and epochs_since_update >= self.patience: + break + + self.scheduler.step() + + # saving the best models: + self._maybe_set_best_weights_and_elite(best_weights, best_eval_loss) + + self.save(save_dir=work_dir) + + def _maybe_get_best_weights( + self, + best_val_loss: torch.Tensor, + val_loss: torch.Tensor, + threshold: float = 0.01, + ) -> Optional[Dict]: + """Return the current model state dict if the validation score improves. + For ensembles, this checks the validation for each ensemble member separately. + Copy from https://github.com/facebookresearch/mbrl-lib/blob/main/mbrl/models/model_trainer.py + + Args: + best_val_score (tensor): the current best validation losses per model. + val_score (tensor): the new validation loss per model. + threshold (float): the threshold for relative improvement. + Returns: + (dict, optional): if the validation score's relative improvement over the + best validation score is higher than the threshold, returns the state dictionary + of the stored model, otherwise returns ``None``. + """ + improvement = (best_val_loss - val_loss) / torch.abs(best_val_loss) + if (improvement > threshold).any().item(): + best_weights = copy.deepcopy(self.network.state_dict()) + else: + best_weights = None + + return best_weights + + def _maybe_set_best_weights_and_elite(self, best_weights: Optional[Dict], best_val_score: torch.Tensor): + if best_weights is not None: + self.network.load_state_dict(best_weights) + + sorted_indices = np.argsort(best_val_score.tolist()) + self.elite_indices = sorted_indices[: self.elite_num] diff --git a/cmrl/models/causal_mech/base_causal_mech.py b/cmrl/models/causal_mech/base_causal_mech.py deleted file mode 100644 index 918d914..0000000 --- a/cmrl/models/causal_mech/base_causal_mech.py +++ /dev/null @@ -1,62 +0,0 @@ -from typing import Optional, List, Dict, Union, MutableMapping -from abc import abstractmethod, ABC -import pathlib - -import torch -from torch.utils.data import DataLoader - -from cmrl.models.graphs.base_graph import BaseGraph -from cmrl.models.graphs.binary_graph import BinaryGraph -from cmrl.utils.variables import Variable - - -class BaseCausalMech(ABC): - def __init__( - self, - name: str, - input_variables: List[Variable], - output_variables: List[Variable], - device: Union[str, torch.device] = "cpu", - ): - self.name = name - self.input_variables = input_variables - self.output_variables = output_variables - self.device = device - - self.input_var_num = len(self.input_variables) - self.output_var_num = len(self.output_variables) - self.graph: Optional[BaseGraph] = None - self.discovery: bool = True - - @abstractmethod - def learn(self, train_loader: DataLoader, valid_loader: DataLoader, **kwargs): - raise NotImplementedError - - @abstractmethod - def forward(self, inputs: MutableMapping[str, torch.Tensor]) -> Dict[str, torch.Tensor]: - raise NotImplementedError - - @property - def causal_graph(self) -> torch.Tensor: - """property causal graph""" - if self.graph is None: - return torch.ones(len(self.input_variables), len(self.output_variables), dtype=torch.int, device=self.device) - else: - return self.graph.get_binary_adj_matrix() - - @property - def forward_mask(self) -> torch.Tensor: - """property input masks""" - return self.causal_graph.T - - def set_oracle_graph(self, graph_data): - self.discovery = False - self.graph = BinaryGraph(self.input_var_num, self.output_var_num, device=self.device) - self.graph.set_data(graph_data=graph_data) - print("set oracle causal graph successfully: \n{}".format(graph_data)) - - def save(self, save_dir: Union[str, pathlib.Path]): - pass - - def load(self, load_dir: Union[str, pathlib.Path]): - pass diff --git a/cmrl/models/causal_mech/kernel_test.py b/cmrl/models/causal_mech/kernel_test.py new file mode 100644 index 0000000..0d92d22 --- /dev/null +++ b/cmrl/models/causal_mech/kernel_test.py @@ -0,0 +1,170 @@ +from typing import Optional, List, Dict, Union, MutableMapping + +import numpy +import numpy as np +import torch +from omegaconf import DictConfig +from stable_baselines3.common.logger import Logger +from hydra.utils import instantiate +from causallearn.utils.KCI.KCI import KCI_CInd +from tqdm import tqdm + +from cmrl.models.causal_mech.base import EnsembleNeuralMech +from cmrl.utils.variables import Variable, ContinuousVariable, DiscreteVariable, BinaryVariable, RadianVariable + + +class KernelTestMech(EnsembleNeuralMech): + def __init__( + self, + # base + name: str, + input_variables: List[Variable], + output_variables: List[Variable], + logger: Optional[Logger] = None, + # model learning + longest_epoch: int = -1, + improvement_threshold: float = 0.01, + patience: int = 5, + # ensemble + ensemble_num: int = 7, + elite_num: int = 5, + # cfgs + network_cfg: Optional[DictConfig] = None, + encoder_cfg: Optional[DictConfig] = None, + decoder_cfg: Optional[DictConfig] = None, + optimizer_cfg: Optional[DictConfig] = None, + scheduler_cfg: Optional[DictConfig] = None, + # forward method + residual: bool = True, + encoder_reduction: str = "sum", + # others + device: Union[str, torch.device] = "cpu", + # KCI + sample_num=2000, + kci_times=10 + ): + EnsembleNeuralMech.__init__( + self, + name=name, + input_variables=input_variables, + output_variables=output_variables, + logger=logger, + longest_epoch=longest_epoch, + improvement_threshold=improvement_threshold, + patience=patience, + ensemble_num=ensemble_num, + elite_num=elite_num, + network_cfg=network_cfg, + encoder_cfg=encoder_cfg, + decoder_cfg=decoder_cfg, + optimizer_cfg=optimizer_cfg, + scheduler_cfg=scheduler_cfg, + residual=residual, + encoder_reduction=encoder_reduction, + device=device, + ) + self.sample_num = sample_num + self.kci_times = kci_times + + def kci_compute_graph( + self, + inputs: MutableMapping[str, numpy.ndarray], + outputs: MutableMapping[str, numpy.ndarray], + **kwargs + ): + + # [[0, 0, 0, 0], + # [0, 0, 1, 1], + # [1, 0, 0, 0], + # [0, 1, 1, 1], + # [0, 0, 1, 1]] + + length = next(iter(inputs.values())).shape[0] + sample_length = min(length, self.sample_num) + + def deal_with_radian_input(name, data): + if isinstance(self.input_variables_dict[name], RadianVariable): + return (data + np.pi) % (2 * np.pi) - np.pi + else: + return data + + p_values_array = np.empty((self.kci_times, self.input_var_num, self.output_var_num)) + with tqdm(total=self.kci_times * self.input_var_num * self.output_var_num, desc="kci") as pbar: + for time in range(self.kci_times): + sample_indices = np.random.permutation(length)[:sample_length] + for out_idx, out_name in enumerate(outputs): + for in_idx, in_name in enumerate(inputs): + if self.residual: + data_x = outputs[out_name][sample_indices] - inputs[out_name.replace("next_", "")][ + sample_indices] + data_x = data_x + else: + data_x = outputs[out_name][sample_indices] + data_y = deal_with_radian_input(in_name, inputs[in_name])[sample_indices] + data_z = [deal_with_radian_input(other_in_name, in_data)[sample_indices] + for other_in_name, in_data in inputs.items() if other_in_name != in_name] + data_z = np.concatenate(data_z, axis=1) + + # data_x = (data_x - data_x.mean(axis=0)) / data_x.std(axis=0) + # data_y = (data_y - data_y.mean(axis=0)) / data_y.std(axis=0) + # data_z = (data_z - data_z.mean(axis=0)) / data_z.std(axis=0) + + kci = KCI_CInd() + p_value, test_stat = kci.compute_pvalue(data_x, data_y, data_z) + p_values_array[time][in_idx][out_idx] = p_value + + pbar.update(1) + final_p_values = (p_values_array < 0.05).mean(axis=0) + print(final_p_values) + pass + + def build_network(self): + self.network = instantiate(self.network_cfg)( + input_dim=self.encoder_output_dim, + output_dim=self.decoder_input_dim, + extra_dims=[self.ensemble_num], + ).to(self.device) + + def forward( + self, + inputs: MutableMapping[str, numpy.ndarray] + ) -> Dict[str, torch.Tensor]: + pass + + +if __name__ == '__main__': + import gym + from stable_baselines3.common.buffers import ReplayBuffer + from torch.utils.data import DataLoader + + from cmrl.models.causal_mech.reinforce import ReinforceCausalMech + from cmrl.models.data_loader import EnsembleBufferDataset, collate_fn, buffer_to_dict + from cmrl.utils.creator import parse_space + from cmrl.utils.env import load_offline_data + from cmrl.models.causal_mech.util import variable_loss_func + + env = gym.make("ContinuousCartPoleSwingUp-v0", real_time_scale=0.02) + real_replay_buffer = ReplayBuffer(int(1e6), env.observation_space, env.action_space, "cpu", + handle_timeout_termination=False) + load_offline_data(env, real_replay_buffer, "SAC-expert", use_ratio=1) + + input_variables = parse_space(env.observation_space, "obs") + parse_space(env.action_space, "act") + output_variables = parse_space(env.observation_space, "next_obs") + + mech = KernelTestMech( + "kernel_test_mech", + input_variables, + output_variables, + sample_num=1000, + kci_times=20 + ) + + inputs, outputs = buffer_to_dict( + env.observation_space, + env.action_space, + env.obs2state, + real_replay_buffer, + "transition" + ) + + mech.kci_compute_graph(inputs, outputs) diff --git a/cmrl/models/causal_mech/neural_causal_mech.py b/cmrl/models/causal_mech/neural_causal_mech.py index 9a852be..9a19a1d 100644 --- a/cmrl/models/causal_mech/neural_causal_mech.py +++ b/cmrl/models/causal_mech/neural_causal_mech.py @@ -14,97 +14,42 @@ from omegaconf import DictConfig from hydra.utils import instantiate -from cmrl.models.causal_mech.base_causal_mech import BaseCausalMech +from cmrl.models.causal_mech.base import BaseCausalMech from cmrl.models.networks.base_network import BaseNetwork from cmrl.models.graphs.base_graph import BaseGraph from cmrl.models.networks.coder import VariableEncoder, VariableDecoder from cmrl.utils.variables import Variable, ContinuousVariable, DiscreteVariable, BinaryVariable from cmrl.models.causal_mech.util import variable_loss_func, train_func, eval_func -default_network_cfg = DictConfig( - dict( - _target_="cmrl.models.networks.ParallelMLP", - _partial_=True, - _recursive_=False, - hidden_dims=[200, 200], - bias=True, - activation_fn_cfg=dict(_target_="torch.nn.SiLU"), - ) -) - -default_encoder_cfg = DictConfig( - dict( - _target_="cmrl.models.networks.VariableEncoder", - _partial_=True, - _recursive_=False, - output_dim=100, - hidden_dims=[100], - bias=True, - activation_fn_cfg=dict(_target_="torch.nn.SiLU"), - ) -) - -default_decoder_cfg = DictConfig( - dict( - _target_="cmrl.models.networks.VariableDecoder", - _partial_=True, - _recursive_=False, - input_dim=100, - hidden_dims=[100], - bias=True, - identity=False, - activation_fn_cfg=dict(_target_="torch.nn.SiLU"), - ) -) - -default_optimizer_cfg = DictConfig( - dict( - _target_="torch.optim.Adam", - _partial_=True, - lr=1e-4, - weight_decay=1e-5, - eps=1e-8, - ) -) - -default_scheduler_cfg = DictConfig( - dict( - _target_="torch.optim.lr_scheduler.StepLR", - _partial_=True, - step_size=1, - gamma=1, - ) -) - class NeuralCausalMech(BaseCausalMech): def __init__( - self, - name: str, - input_variables: List[Variable], - output_variables: List[Variable], - # model learning - longest_epoch: int = -1, - improvement_threshold: float = 0.01, - patience: int = 5, - # ensemble - ensemble_num: int = 7, - elite_num: int = 5, - # cfgs - network_cfg: Optional[DictConfig] = None, - encoder_cfg: Optional[DictConfig] = None, - decoder_cfg: Optional[DictConfig] = None, - optimizer_cfg: Optional[DictConfig] = None, - scheduler_cfg: Optional[DictConfig] = None, - # forward method - residual: bool = True, - encoder_reduction: str = "sum", - multi_step: str = "none", - # logger - logger: Optional[Logger] = None, - # others - device: Union[str, torch.device] = "cpu", - **kwargs + self, + name: str, + input_variables: List[Variable], + output_variables: List[Variable], + # model learning + longest_epoch: int = -1, + improvement_threshold: float = 0.01, + patience: int = 5, + # ensemble + ensemble_num: int = 7, + elite_num: int = 5, + # cfgs + network_cfg: Optional[DictConfig] = None, + encoder_cfg: Optional[DictConfig] = None, + decoder_cfg: Optional[DictConfig] = None, + optimizer_cfg: Optional[DictConfig] = None, + scheduler_cfg: Optional[DictConfig] = None, + # forward method + residual: bool = True, + encoder_reduction: str = "sum", + multi_step: str = "none", + # logger + logger: Optional[Logger] = None, + # others + device: Union[str, torch.device] = "cpu", + **kwargs ): super(NeuralCausalMech, self).__init__( name=name, @@ -150,7 +95,8 @@ def __init__( def single_step_forward(self, inputs: MutableMapping[str, torch.Tensor]) -> Dict[str, torch.Tensor]: batch_size, _ = self.get_inputs_batch_size(inputs) - inputs_tensor = torch.zeros(self.ensemble_num, batch_size, self.input_var_num, self.encoder_output_dim).to(self.device) + inputs_tensor = torch.zeros(self.ensemble_num, batch_size, self.input_var_num, self.encoder_output_dim).to( + self.device) for i, var in enumerate(self.input_variables): out = self.variable_encoders[var.name](inputs[var.name].to(self.device)) inputs_tensor[:, :, i] = out @@ -166,7 +112,7 @@ def single_step_forward(self, inputs: MutableMapping[str, torch.Tensor]) -> Dict outputs = {} for i, var in enumerate(self.output_variables): - hid = output_tensor[:, :, i * self.decoder_input_dim : (i + 1) * self.decoder_input_dim] + hid = output_tensor[:, :, i * self.decoder_input_dim: (i + 1) * self.decoder_input_dim] outputs[var.name] = self.variable_decoders[var.name](hid) if self.residual: @@ -197,9 +143,9 @@ def build_network(self): def build_optimizer(self): assert self.network is not None, "you must build network first" params = ( - [self.network.parameters()] - + [encoder.parameters() for encoder in self.variable_encoders.values()] - + [decoder.parameters() for decoder in self.variable_decoders.values()] + [self.network.parameters()] + + [encoder.parameters() for encoder in self.variable_encoders.values()] + + [decoder.parameters() for decoder in self.variable_decoders.values()] ) self.optimizer = instantiate(self.optimizer_cfg)(params=chain(*params)) @@ -232,9 +178,9 @@ def get_inputs_batch_size(self, inputs: MutableMapping[str, torch.Tensor]) -> in return batch_size, data_shape[:-3] def residual_outputs( - self, - inputs: MutableMapping[str, torch.Tensor], - outputs: MutableMapping[str, torch.Tensor], + self, + inputs: MutableMapping[str, torch.Tensor], + outputs: MutableMapping[str, torch.Tensor], ) -> MutableMapping[str, torch.Tensor]: for name in filter(lambda s: s.startswith("obs"), inputs.keys()): # assert inputs[name].shape[:2] == outputs["next_{}".format(name)].shape[:2] @@ -244,12 +190,12 @@ def residual_outputs( return outputs def learn( - self, - # loader - train_loader: DataLoader, - valid_loader: DataLoader, - work_dir: Optional[Union[str, pathlib.Path]] = None, - **kwargs + self, + # loader + train_loader: DataLoader, + valid_loader: DataLoader, + work_dir: Optional[Union[str, pathlib.Path]] = None, + **kwargs ): best_weights: Optional[Dict] = None epoch_iter = range(self.longest_epoch) if self.longest_epoch >= 0 else count() @@ -328,10 +274,10 @@ def load(self, load_dir: Union[str, pathlib.Path]): coder.load(load_dir) def _maybe_get_best_weights( - self, - best_val_loss: torch.Tensor, - val_loss: torch.Tensor, - threshold: float = 0.01, + self, + best_val_loss: torch.Tensor, + val_loss: torch.Tensor, + threshold: float = 0.01, ) -> Optional[Dict]: """Return the current model state dict if the validation score improves. For ensembles, this checks the validation for each ensemble member separately. @@ -383,9 +329,9 @@ def decoder_input_dim(self): return self.decoder_cfg.input_dim def reduce_encoder_output( - self, - encoder_output: torch.Tensor, - mask: Optional[torch.Tensor] = None, + self, + encoder_output: torch.Tensor, + mask: Optional[torch.Tensor] = None, ) -> torch.Tensor: assert len(encoder_output.shape) == 4, ( "shape of encoder_output should be (ensemble-num, batch-size, input-var-num, encoder-output-dim), " @@ -401,7 +347,7 @@ def reduce_encoder_output( # mask shape [..., ensemble-num, batch-size, input-var-num] assert ( - mask.shape[-3:] == encoder_output.shape[:-1] + mask.shape[-3:] == encoder_output.shape[:-1] ), "mask shape should be (..., ensemble-num, batch-size, input-var-num)" # [*mask-extra-dims, ensemble-num, batch-size, input-var-num, encoder-output-dim] diff --git a/cmrl/models/causal_mech/plain_mech.py b/cmrl/models/causal_mech/plain_mech.py index f99961c..b745457 100644 --- a/cmrl/models/causal_mech/plain_mech.py +++ b/cmrl/models/causal_mech/plain_mech.py @@ -1,49 +1,50 @@ from typing import Optional, List, Dict, Union, MutableMapping import torch +from torch.utils.data import DataLoader +import numpy as np from omegaconf import DictConfig from hydra.utils import instantiate from stable_baselines3.common.logger import Logger from cmrl.utils.variables import Variable -from cmrl.models.causal_mech.neural_causal_mech import NeuralCausalMech +from cmrl.models.causal_mech.base import EnsembleNeuralMech +from cmrl.models.data_loader import EnsembleBufferDataset, collate_fn -class PlainMech(NeuralCausalMech): +class PlainMech(EnsembleNeuralMech): def __init__( - self, - name: str, - input_variables: List[Variable], - output_variables: List[Variable], - # model learning - longest_epoch: int = -1, - improvement_threshold: float = 0.01, - patience: int = 5, - # ensemble - ensemble_num: int = 7, - elite_num: int = 5, - # cfgs - network_cfg: Optional[DictConfig] = None, - encoder_cfg: Optional[DictConfig] = None, - decoder_cfg: Optional[DictConfig] = None, - optimizer_cfg: Optional[DictConfig] = None, - # forward method - residual: bool = True, - encoder_reduction: str = "sum", - multi_step: str = "none", - # logger - logger: Optional[Logger] = None, - # others - device: Union[str, torch.device] = "cpu", - **kwargs + self, + name: str, + input_variables: List[Variable], + output_variables: List[Variable], + logger: Optional[Logger] = None, + # model learning + longest_epoch: int = -1, + improvement_threshold: float = 0.01, + patience: int = 5, + # ensemble + ensemble_num: int = 7, + elite_num: int = 5, + # cfgs + network_cfg: Optional[DictConfig] = None, + encoder_cfg: Optional[DictConfig] = None, + decoder_cfg: Optional[DictConfig] = None, + optimizer_cfg: Optional[DictConfig] = None, + scheduler_cfg: Optional[DictConfig] = None, + # forward method + residual: bool = True, + encoder_reduction: str = "sum", + # others + device: Union[str, torch.device] = "cpu", + **kwargs ): - if multi_step == "none": - multi_step = "forward-euler 1" - - super(PlainMech, self).__init__( + EnsembleNeuralMech.__init__( + self, name=name, input_variables=input_variables, output_variables=output_variables, + logger=logger, longest_epoch=longest_epoch, improvement_threshold=improvement_threshold, patience=patience, @@ -53,24 +54,83 @@ def __init__( encoder_cfg=encoder_cfg, decoder_cfg=decoder_cfg, optimizer_cfg=optimizer_cfg, + scheduler_cfg=scheduler_cfg, residual=residual, encoder_reduction=encoder_reduction, - multi_step=multi_step, - logger=logger, device=device, - **kwargs ) - def build_network(self): - self.network = instantiate(self.network_cfg)( - input_dim=self.encoder_output_dim, - output_dim=self.output_var_num * self.decoder_input_dim, - extra_dims=[self.ensemble_num], - ).to(self.device) + def forward( + self, + inputs: MutableMapping[str, torch.Tensor] + ) -> Dict[str, torch.Tensor]: + batch_size, _ = self.get_inputs_info(inputs) + + inputs_tensor = torch.zeros(self.ensemble_num, batch_size, self.input_var_num, self.encoder_output_dim).to( + self.device) + for i, var in enumerate(self.input_variables): + out = self.variable_encoders[var.name](inputs[var.name].to(self.device)) + inputs_tensor[:, :, i] = out + + # for name, param in self.network.named_parameters(): + # if param.grad is not None and torch.isnan(param.grad).any(): + # print("nan gradient found") + # print("name:", name) + # print("param:", param.grad) + # raise SystemExit + + output_tensor = self.network(self.reduce_encoder_output(inputs_tensor)) + + outputs = {} + for i, var in enumerate(self.output_variables): + outputs[var.name] = self.variable_decoders[var.name](output_tensor) - def build_graph(self): - self.graph = None + if self.residual: + outputs = self.residual_outputs(inputs, outputs) + return outputs @property def forward_mask(self): return torch.ones(self.input_var_num).to(self.device) + + +if __name__ == '__main__': + import gym + from stable_baselines3.common.buffers import ReplayBuffer + from torch.utils.data import DataLoader + + from cmrl.models.causal_mech.reinforce import ReinforceCausalMech + from cmrl.models.data_loader import EnsembleBufferDataset, collate_fn, buffer_to_dict + from cmrl.utils.creator import parse_space + from cmrl.utils.env import load_offline_data + from cmrl.models.causal_mech.util import variable_loss_func + from cmrl.sb3_extension.logger import configure as logger_configure + + env = gym.make("ContinuousCartPoleSwingUp-v0", real_time_scale=0.02) + real_replay_buffer = ReplayBuffer(int(1e6), env.observation_space, env.action_space, "cpu", + handle_timeout_termination=False) + load_offline_data(env, real_replay_buffer, "SAC-expert-replay", use_ratio=1) + + input_variables = parse_space(env.observation_space, "obs") + parse_space(env.action_space, "act") + output_variables = parse_space(env.observation_space, "next_obs") + + logger = logger_configure("log", ["tensorboard", "stdout"]) + + mech = PlainMech( + "plain_mech", + input_variables, + output_variables, + logger=logger, + sample_num=2000, + kci_times=20, + ) + + inputs, outputs = buffer_to_dict( + env.observation_space, + env.action_space, + env.obs2state, + real_replay_buffer, + "transition" + ) + + mech.learn(inputs, outputs) diff --git a/cmrl/models/causal_mech/reinforce.py b/cmrl/models/causal_mech/reinforce.py index e923853..cc089a8 100644 --- a/cmrl/models/causal_mech/reinforce.py +++ b/cmrl/models/causal_mech/reinforce.py @@ -13,7 +13,7 @@ from hydra.utils import instantiate from cmrl.utils.variables import Variable -from cmrl.models.causal_mech.neural_causal_mech import NeuralCausalMech, default_optimizer_cfg +from cmrl.models.causal_mech.neural_causal_mech import NeuralCausalMech from cmrl.models.graphs.prob_graph import BernoulliGraph from cmrl.models.causal_mech.util import variable_loss_func, train_func, eval_func diff --git a/cmrl/models/constant.py b/cmrl/models/constant.py new file mode 100644 index 0000000..7003ccd --- /dev/null +++ b/cmrl/models/constant.py @@ -0,0 +1,55 @@ +from omegaconf import DictConfig + +NETWORK_CFG = DictConfig( + dict( + _target_="cmrl.models.networks.ParallelMLP", + _partial_=True, + _recursive_=False, + hidden_dims=[200, 200], + bias=True, + activation_fn_cfg=dict(_target_="torch.nn.SiLU"), + ) +) + +ENCODER_CFG = DictConfig( + dict( + _target_="cmrl.models.networks.VariableEncoder", + _partial_=True, + _recursive_=False, + output_dim=100, + hidden_dims=[100], + bias=True, + activation_fn_cfg=dict(_target_="torch.nn.SiLU"), + ) +) + +DECODER_CFG = DictConfig( + dict( + _target_="cmrl.models.networks.VariableDecoder", + _partial_=True, + _recursive_=False, + input_dim=100, + hidden_dims=[100], + bias=True, + activation_fn_cfg=dict(_target_="torch.nn.SiLU"), + ) +) + +OPTIMIZER_CFG = DictConfig( + dict( + _target_="torch.optim.Adam", + _partial_=True, + lr=1e-4, + weight_decay=1e-5, + eps=1e-8, + ) +) + +SCHEDULER_CFG = DictConfig( + dict( + _target_="torch.optim.lr_scheduler.StepLR", + _partial_=True, + step_size=1, + gamma=1, + ) +) diff --git a/cmrl/models/data_loader.py b/cmrl/models/data_loader.py index 062b905..4f386f7 100644 --- a/cmrl/models/data_loader.py +++ b/cmrl/models/data_loader.py @@ -1,102 +1,101 @@ -from typing import Optional +from typing import Optional, MutableMapping -from gym import spaces +from gym import spaces, Env import torch from torch.utils.data import Dataset, default_collate import numpy as np from stable_baselines3.common.buffers import ReplayBuffer, DictReplayBuffer -from cmrl.utils.variables import space2dict +from cmrl.utils.variables import to_dict_by_space -class BufferDataset(Dataset): - def __init__( - self, +def buffer_to_dict( + observation_space, + action_space, + obs2state_fn, replay_buffer: ReplayBuffer, - observation_space: spaces.Space, - action_space: spaces.Space, mech: str, - is_valid: bool = False, - train_ratio: float = 0.8, - seed: int = 10086, - repeat: Optional[int] = None, +): + assert mech in ["transition", "reward_mech", "termination_mech"] + # dict action is not supported by SB3(so not done by cmrl) + assert not isinstance(action_space, spaces.Dict) + assert hasattr(replay_buffer, "extra_obs") + assert hasattr(replay_buffer, "next_extra_obs") + + real_buffer_size = replay_buffer.buffer_size if replay_buffer.full else replay_buffer.pos + + if hasattr(replay_buffer, "extra_obs"): + observations = obs2state_fn(replay_buffer.observations[: real_buffer_size, 0], + replay_buffer.extra_obs[: real_buffer_size, 0]) + else: + observations = replay_buffer.observations[: real_buffer_size, 0] + obs_dict = to_dict_by_space(observations, observation_space, prefix="obs", ) + act_dict = to_dict_by_space( + replay_buffer.actions[: real_buffer_size, 0], + action_space, + prefix="act", + ) + if hasattr(replay_buffer, "next_extra_obs"): + next_observations = obs2state_fn(replay_buffer.next_observations[: real_buffer_size, 0], + replay_buffer.next_extra_obs[: real_buffer_size, 0]) + else: + next_observations = replay_buffer.next_observations[: real_buffer_size, 0] + next_obs_dict = to_dict_by_space(next_observations, observation_space, prefix="next_obs") + + inputs = {} + inputs.update(obs_dict) + inputs.update(act_dict) + + if mech == "transition": + outputs = next_obs_dict + elif mech == "reward_mech": + rewards = replay_buffer.rewards[: real_buffer_size, 0] + rewards_dict = {"reward": torch.from_numpy(rewards[:, None])} + inputs.update(next_obs_dict) + outputs = rewards_dict + elif mech == "termination_mech": + terminals = replay_buffer.dones[: real_buffer_size, 0] * (1 - replay_buffer.timeouts[: real_buffer_size, 0]) + terminals_dict = {"terminal": torch.from_numpy(terminals[:, None])} + inputs.update(next_obs_dict) + outputs = terminals_dict + else: + raise NotImplementedError("support mechs in [transition, reward_mech, termination_mech] only") + + return inputs, outputs + + +class EnsembleBufferDataset(Dataset): + def __init__( + self, + inputs: MutableMapping, + outputs: MutableMapping, + training: bool = False, + train_ratio: float = 0.8, + ensemble_num: int = 7, + seed: int = 10086, ): - assert mech in ["transition", "reward_mech", "termination_mech"] - # dict action is not supported by SB3(so not done by cmrl) - assert not isinstance(action_space, spaces.Dict) - - self.replay_buffer = replay_buffer - self.observation_space = observation_space - self.action_space = action_space - self.mech = mech - self.is_valid = is_valid + self.inputs = inputs + self.outputs = outputs + self.training = training self.train_ratio = train_ratio + self.ensemble_num = ensemble_num self.seed = seed - self.repeat = repeat - - if self.repeat: - assert self.repeat > 1, "repeat must be a int greater than 1" - - self.size = self.replay_buffer.buffer_size if self.replay_buffer.full else self.replay_buffer.pos - - self.inputs = None - self.outputs = None - self.load_from_buffer() - self.indexes = None - self.build_indexes() - def build_indexes(self): + size = next(iter(inputs.values())).shape[0] + np.random.seed(self.seed) - permutation = np.random.permutation(self.size) - if self.is_valid: # for valid set - self.indexes = permutation[int(self.size * self.train_ratio) :] - else: # for train set - self.indexes = permutation[: int(self.size * self.train_ratio)] - - def load_from_buffer(self): - obs_dict = space2dict( - self.replay_buffer.observations[: self.size, 0], - self.observation_space, - prefix="obs", - to_tensor=True, - ) - act_dict = space2dict( - self.replay_buffer.actions[: self.size, 0], - self.action_space, - prefix="act", - to_tensor=True, - ) - next_obs_dict = space2dict( - self.replay_buffer.next_observations[: self.size, 0], - self.observation_space, - prefix="next_obs", - to_tensor=True, - # device=self.device - ) - - self.inputs = {} - self.inputs.update(obs_dict) - self.inputs.update(act_dict) - - if self.mech == "transition": - self.outputs = next_obs_dict - elif self.mech == "reward_mech": - rewards = self.replay_buffer.rewards[: self.size, 0] - rewards_dict = {"reward": torch.from_numpy(rewards[:, None])} - self.inputs.update(next_obs_dict) - self.outputs = rewards_dict + permutation = np.random.permutation(size) + if self.training: + train_indexes = permutation[:int(size * self.train_ratio)] + indexes = [np.random.permutation(train_indexes) for _ in range(self.ensemble_num)] else: - terminals = self.replay_buffer.dones[: self.size, 0] * (1 - self.replay_buffer.timeouts[: self.size, 0]) - terminals_dict = {"terminal": torch.from_numpy(terminals[:, None])} - self.inputs.update(next_obs_dict) - self.outputs = terminals_dict + valid_indexes = permutation[int(size * self.train_ratio):] + indexes = [valid_indexes for _ in range(self.ensemble_num)] + self.indexes = np.array(indexes).T def __getitem__(self, item): index = self.indexes[item] - if self.repeat: - assert len(self.indexes.shape) == 1, "repeating conflicts with ensemble" - index = np.tile(index, self.repeat) inputs = dict([(key, self.inputs[key][index]) for key in self.inputs]) outputs = dict([(key, self.outputs[key][index]) for key in self.outputs]) @@ -106,50 +105,6 @@ def __len__(self): return len(self.indexes) -class EnsembleBufferDataset(BufferDataset): - def __init__( - self, - replay_buffer: ReplayBuffer, - observation_space: spaces.Space, - action_space: spaces.Space, - mech: str, - is_valid: bool = False, - train_ratio: float = 0.8, - ensemble_num: int = 7, - train_ensemble: bool = True, - seed: int = 10086, - ): - self.ensemble_num = ensemble_num - self.train_ensemble = train_ensemble - - super(EnsembleBufferDataset, self).__init__( - replay_buffer=replay_buffer, - observation_space=observation_space, - action_space=action_space, - mech=mech, - is_valid=is_valid, - train_ratio=train_ratio, - seed=seed, - ) - - def build_indexes(self): - indexes = [] - np.random.seed(self.seed) - - if self.train_ensemble: # call ``np.random`` ensemble-num + 1 times - assert not self.is_valid - train_indexes = np.random.permutation(self.size)[: int(self.size * self.train_ratio)] - indexes = [np.random.permutation(train_indexes) for _ in range(self.ensemble_num)] - else: - for i in range(self.ensemble_num): - permutation = np.random.permutation(self.size) - if self.is_valid: - indexes.append(permutation[int(self.size * self.train_ratio) :]) - else: - indexes.append(permutation[: int(self.size * self.train_ratio)]) - self.indexes = np.array(indexes).T - - def collate_fn(data): inputs, outputs = default_collate(data) inputs = dict([(key, value.transpose(0, 1)) for key, value in inputs.items()]) diff --git a/cmrl/models/dynamics.py b/cmrl/models/dynamics.py index 55c670c..6ff55c8 100644 --- a/cmrl/models/dynamics.py +++ b/cmrl/models/dynamics.py @@ -2,6 +2,7 @@ from collections import ChainMap import pathlib from typing import Dict, List, Optional, Tuple, Union +from functools import partial import numpy as np import torch @@ -10,25 +11,28 @@ from stable_baselines3.common.logger import Logger from stable_baselines3.common.buffers import ReplayBuffer -from cmrl.utils.variables import space2dict -from cmrl.models.causal_mech.base_causal_mech import BaseCausalMech -from cmrl.models.data_loader import BufferDataset, EnsembleBufferDataset, collate_fn +from cmrl.utils.variables import to_dict_by_space +from cmrl.models.causal_mech.base import BaseCausalMech +from cmrl.models.data_loader import buffer_to_dict +from cmrl.types import Obs2StateFnType class Dynamics: def __init__( - self, - transition: BaseCausalMech, - observation_space: spaces.Space, - action_space: spaces.Space, - reward_mech: Optional[BaseCausalMech] = None, - termination_mech: Optional[BaseCausalMech] = None, - seed: int = 7, - logger: Optional[Logger] = None, + self, + transition: BaseCausalMech, + observation_space: spaces.Space, + action_space: spaces.Space, + obs2state_fn: Obs2StateFnType, + reward_mech: Optional[BaseCausalMech] = None, + termination_mech: Optional[BaseCausalMech] = None, + seed: int = 7, + logger: Optional[Logger] = None, ): self.transition = transition self.observation_space = observation_space self.action_space = action_space + self.obs2state_fn = obs2state_fn self.reward_mech = reward_mech self.termination_mech = termination_mech self.seed = seed @@ -40,51 +44,36 @@ def __init__( self.device = self.transition.device pass - def get_loader(self, real_replay_buffer, mech: str, batch_size: int = 128): - train_dataset = EnsembleBufferDataset( - real_replay_buffer, - self.observation_space, - self.action_space, - is_valid=False, - mech=mech, - train_ensemble=True, - ensemble_num=self.transition.ensemble_num, - seed=self.seed, - ) - train_loader = DataLoader(train_dataset, batch_size=batch_size, collate_fn=collate_fn, shuffle=True) - valid_dataset = BufferDataset( - real_replay_buffer, - self.observation_space, - self.action_space, - is_valid=True, - mech=mech, - repeat=self.transition.ensemble_num, - seed=self.seed, + def learn(self, real_replay_buffer: ReplayBuffer, work_dir: Optional[Union[str, pathlib.Path]] = None, **kwargs): + get_dataset = partial( + buffer_to_dict, + observation_space=self.observation_space, + action_space=self.action_space, + obs2state_fn=self.obs2state_fn, + replay_buffer=real_replay_buffer, ) - valid_loader = DataLoader(valid_dataset, batch_size=batch_size, collate_fn=collate_fn) - return train_loader, valid_loader - - def learn(self, real_replay_buffer: ReplayBuffer, work_dir: Optional[Union[str, pathlib.Path]] = None, **kwargs): # transition - self.transition.learn(*self.get_loader(real_replay_buffer, "transition"), work_dir=work_dir) + self.transition.learn(*get_dataset(mech="transition"), work_dir=work_dir) # reward-mech if self.learn_reward: - self.reward_mech.learn(*self.get_loader(real_replay_buffer, "reward_mech"), work_dir=work_dir) + self.reward_mech.learn(*get_dataset(mech="reward_mech"), work_dir=work_dir) # termination-mech if self.learn_termination: - self.termination_mech.learn(*self.get_loader(real_replay_buffer, "termination_mech"), work_dir=work_dir) + self.termination_mech.learn(*get_dataset(mech="termination_mech"), work_dir=work_dir) def step(self, batch_obs, batch_action): with torch.no_grad(): - obs_dict = space2dict(batch_obs, self.observation_space, "obs", repeat=7, to_tensor=True) - act_dict = space2dict(batch_action, self.action_space, "act", repeat=7, to_tensor=True) + obs_dict = to_dict_by_space(batch_obs, self.observation_space, "obs", repeat=7, to_tensor=True) + act_dict = to_dict_by_space(batch_action, self.action_space, "act", repeat=7, to_tensor=True) inputs = ChainMap(obs_dict, act_dict) outputs = self.transition.forward(inputs) - info = {"origin-next_obs": torch.concat([tensor[:, :, :1] for tensor in outputs.values()], dim=-1).cpu().numpy()} + info = { + "origin-next_obs": torch.concat([tensor[:, :, :1] for tensor in outputs.values()], dim=-1).cpu().numpy()} - return torch.concat([tensor.mean(dim=0)[:, :1] for tensor in outputs.values()], dim=-1).cpu().numpy(), None, None, info + return torch.concat([tensor.mean(dim=0)[:, :1] for tensor in outputs.values()], + dim=-1).cpu().numpy(), None, None, info # def set_oracle_graph(self, graph): diff --git a/cmrl/models/networks/coder.py b/cmrl/models/networks/coder.py index e4c111b..ca0a133 100644 --- a/cmrl/models/networks/coder.py +++ b/cmrl/models/networks/coder.py @@ -62,14 +62,12 @@ def __init__( input_dim: int = 100, hidden_dims: Optional[List[int]] = None, bias: bool = True, - identity: bool = False, activation_fn_cfg: Optional[DictConfig] = None, ): self.variable = variable self.input_dim = input_dim self.hidden_dims = hidden_dims if hidden_dims is not None else [] self.bias = bias - self.identity = identity self.activation_fn_cfg = activation_fn_cfg self.name = "{}_decoder".format(variable.name) @@ -78,11 +76,6 @@ def __init__( self._model_filename = "{}.pth".format(self.name) def build(self): - if self.identity: - assert isinstance(self.variable, ContinuousVariable), "only ContinuousVariable could use identity" - self._layers = nn.ModuleList([nn.Identity()]) - return - layers = [create_activation(self.activation_fn_cfg)] hidden_dims = [self.input_dim] + self.hidden_dims diff --git a/cmrl/types.py b/cmrl/types.py index f65d340..0508827 100644 --- a/cmrl/types.py +++ b/cmrl/types.py @@ -7,3 +7,4 @@ # (next_obs, pre_obs, action) -> terminal TermFnType = Callable[[torch.Tensor, torch.Tensor, torch.Tensor], torch.Tensor] InitObsFnType = Callable[[int], torch.Tensor] +Obs2StateFnType = Callable[[torch.Tensor, torch.Tensor], torch.Tensor] diff --git a/cmrl/utils/creator.py b/cmrl/utils/creator.py index 5f7d943..7af9402 100644 --- a/cmrl/utils/creator.py +++ b/cmrl/utils/creator.py @@ -8,9 +8,10 @@ from stable_baselines3.common.logger import Logger from stable_baselines3.common.base_class import BaseAlgorithm +from cmrl.types import Obs2StateFnType from cmrl.models.dynamics import Dynamics from cmrl.models.fake_env import VecFakeEnv -from cmrl.models.causal_mech.base_causal_mech import BaseCausalMech +from cmrl.models.causal_mech.base import BaseCausalMech from cmrl.utils.variables import ContinuousVariable, BinaryVariable, DiscreteVariable, Variable, parse_space @@ -23,10 +24,11 @@ def create_agent(cfg: DictConfig, fake_env: VecFakeEnv, logger: Optional[Logger] def create_dynamics( - cfg: DictConfig, - observation_space: spaces.Space, - action_space: spaces.Space, - logger: Optional[Logger] = None, + cfg: DictConfig, + observation_space: spaces.Space, + action_space: spaces.Space, + obs2state_fn: Obs2StateFnType, + logger: Optional[Logger] = None, ): obs_variables = parse_space(observation_space, "obs") act_variables = parse_space(action_space, "act") @@ -71,6 +73,7 @@ def create_dynamics( termination_mech=termination_mech, observation_space=observation_space, action_space=action_space, + obs2state_fn=obs2state_fn, logger=logger, ) diff --git a/cmrl/utils/env.py b/cmrl/utils/env.py index 9899681..bb12805 100644 --- a/cmrl/utils/env.py +++ b/cmrl/utils/env.py @@ -7,23 +7,24 @@ from stable_baselines3.common.buffers import ReplayBuffer import cmrl.utils.variables -from cmrl.types import TermFnType, RewardFnType, InitObsFnType +from cmrl.types import TermFnType, RewardFnType, InitObsFnType, Obs2StateFnType def make_env( - cfg: omegaconf.DictConfig, -) -> Tuple[emei.EmeiEnv, TermFnType, Optional[RewardFnType], Optional[InitObsFnType],]: + cfg: omegaconf.DictConfig, +) -> Tuple[emei.EmeiEnv, TermFnType, Optional[RewardFnType], Optional[InitObsFnType],Optional[Obs2StateFnType],]: env = cast(emei.EmeiEnv, gym.make(cfg.task.env_id, **cfg.task.params)) reward_fn = env.get_batch_reward term_fn = env.get_batch_terminal init_obs_fn = env.get_batch_init_obs + obs2state_fn = env.obs2state # set seed env.reset(seed=cfg.seed) env.observation_space.seed(cfg.seed + 1) env.action_space.seed(cfg.seed + 2) - return env, reward_fn, term_fn, init_obs_fn + return env, reward_fn, term_fn, init_obs_fn, obs2state_fn def load_offline_data(env, replay_buffer: ReplayBuffer, dataset_name: str, use_ratio: float = 1): @@ -45,7 +46,15 @@ def load_offline_data(env, replay_buffer: ReplayBuffer, dataset_name: str, use_r # set all data for attr in ["observations", "next_observations", "actions", "rewards", "dones", "timeouts"]: - if attr == "dones" and attr not in data_dict and "terminals" in data_dict: - replay_buffer.dones[:sample_data_num, 0] = data_dict["terminals"][sample_idx] - continue + # if attr == "dones" and attr not in data_dict and "terminals" in data_dict: + # replay_buffer.dones[:sample_data_num, 0] = data_dict["terminals"][sample_idx] + # continue + getattr(replay_buffer, attr)[:sample_data_num, 0] = data_dict[attr][sample_idx] + + for attr in ["extra_obs", "next_extra_obs"]: + setattr( + replay_buffer, + attr, + np.zeros((replay_buffer.buffer_size, replay_buffer.n_envs) + data_dict[attr].shape[1:], dtype=np.float32) + ) getattr(replay_buffer, attr)[:sample_data_num, 0] = data_dict[attr][sample_idx] diff --git a/cmrl/utils/variables.py b/cmrl/utils/variables.py index cfc26aa..40196d3 100644 --- a/cmrl/utils/variables.py +++ b/cmrl/utils/variables.py @@ -57,21 +57,33 @@ def parse_space(space: spaces.Space, prefix="obs") -> List[Variable]: return variables -def space2dict( +def to_dict_by_space( data: np.ndarray, space: spaces.Space, prefix="obs", repeat: Optional[int] = None, to_tensor: bool = False, - device: Union[str, torch.device] = "cpu", ) -> Dict[str, Union[np.ndarray, torch.Tensor]]: + """Transform the interaction data from its own type to python's dict, by the signature of space. + + Args: + data: interaction data from replay buffer + space: space of gym + prefix: prefix of the key in dict + repeat: copy data in a new dimension + to_tensor: transform the data from numpy's ndarray to torch's tensor + + Returns: interaction data organized in dictionary form + + """ if repeat: assert repeat > 1, "repeat must be a int greater than 1" dict_data = {} - if isinstance(space, spaces.Box): # shape: (batch-size, node-num), every node has exactly one dim + if isinstance(space, spaces.Box): + # shape of data: (batch-size, node-num), every node has exactly one dim for i, (low, high) in enumerate(zip(space.low, space.high)): - # shape: (batch-size, specific-dim) + # shape of dict_data['xxx']: (batch-size, 1) dict_data["{}_{}".format(prefix, i)] = data[:, i, None].astype(np.float32) else: # TODO @@ -79,10 +91,11 @@ def space2dict( for name in dict_data: if repeat: - # shape: (repeat-dim, batch-size, specific-dim) + # shape of dict_data['xxx']: (repeat-dim, batch-size, specific-dim) + # specific-dim is 1 for the case of spaces.Box dict_data[name] = np.tile(dict_data[name][None, :, :], [repeat, 1, 1]) if to_tensor: - dict_data[name] = torch.from_numpy(dict_data[name]).to(device) + dict_data[name] = torch.from_numpy(dict_data[name]) return dict_data diff --git a/requirements/dev.txt b/requirements/dev.txt index f7916c3..ae90ea9 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -4,5 +4,5 @@ pytest-cov>=4.0.0 flake8>=5.0.4 mkdocs>=1.4.1 Pygments>=2.13.0 -mkdocstrings>=0.19.0 -mkdocstrings-python>=1.0.14 +# mkdocstrings>=0.19.0 +# mkdocstrings-python>=1.0.14 diff --git a/requirements/main.txt b/requirements/main.txt index 787a866..c94a59c 100644 --- a/requirements/main.txt +++ b/requirements/main.txt @@ -9,4 +9,4 @@ tensorboard>=2.9.0 mujoco >= 2.2.0 wandb >= 0.13 stable-baselines3 @ git+https://github.com/carlosluis/stable-baselines3@fix_tests -PyQt5>=5.15.7 +causal-learn >= 0.1.3.3 \ No newline at end of file diff --git a/setup.py b/setup.py index de81688..b895d12 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,6 @@ from setuptools import find_packages, setup - def parse_requirements_file(path): return [line.rstrip() for line in open(path, "r")] @@ -25,7 +24,7 @@ def parse_requirements_file(path): long_description=long_description, long_description_content_type="text/markdown", url="https://github.com/FrankTianTT/causal-mbrl", - packages=find_packages(), + packages=[package for package in find_packages() if package.startswith("cmrl")], classifiers=[ "License :: OSI Approved :: MIT License", "Programming Language :: Python :: 3", diff --git a/tests/__init__.py b/tests/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/tests/test_models/test_causal_mech/test_CMI_test.py b/tests/test_models/test_causal_mech/test_CMI_test.py index f38041f..1322f62 100644 --- a/tests/test_models/test_causal_mech/test_CMI_test.py +++ b/tests/test_models/test_causal_mech/test_CMI_test.py @@ -3,8 +3,8 @@ import torch from torch.utils.data import DataLoader -from cmrl.models.causal_mech.CMI_test import CMITest -from cmrl.models.data_loader import BufferDataset, EnsembleBufferDataset, collate_fn +from cmrl.models.causal_mech.CMI_test import CMITestMech +from cmrl.models.data_loader import EnsembleBufferDataset, EnsembleBufferDataset, collate_fn from cmrl.utils.creator import parse_space from cmrl.utils.env import load_offline_data from cmrl.models.causal_mech.util import variable_loss_func @@ -22,14 +22,14 @@ def prepare(freq_rate): real_replay_buffer, env.observation_space, env.action_space, - is_valid=False, + training=False, mech="transition", train_ensemble=True, ensemble_num=ensemble_num, ) train_loader = DataLoader(train_dataset, batch_size=8, collate_fn=collate_fn) - valid_dataset = BufferDataset( - real_replay_buffer, env.observation_space, env.action_space, is_valid=True, mech="transition", repeat=ensemble_num + valid_dataset = EnsembleBufferDataset( + real_replay_buffer, env.observation_space, env.action_space, training=True, mech="transition", repeat=ensemble_num ) valid_loader = DataLoader(valid_dataset, batch_size=8, collate_fn=collate_fn) @@ -42,7 +42,7 @@ def prepare(freq_rate): def test_mask(): input_variables, output_variables, train_loader, valid_loader = prepare(freq_rate=1) - mech = CMITest( + mech = CMITestMech( name="test", input_variables=input_variables, output_variables=output_variables, @@ -86,7 +86,7 @@ def test_mask(): def test_CMI_forward(): input_variables, output_variables, train_loader, valid_loader = prepare(freq_rate=1) - mech = CMITest( + mech = CMITestMech( name="test", input_variables=input_variables, output_variables=output_variables, @@ -102,7 +102,7 @@ def test_CMI_forward(): def test_forward(): input_variables, output_variables, train_loader, valid_loader = prepare(freq_rate=1) - mech = CMITest( + mech = CMITestMech( name="test", input_variables=input_variables, output_variables=output_variables, diff --git a/tests/test_models/test_causal_mech/test_plain_mech.py b/tests/test_models/test_causal_mech/test_plain_mech.py index 69b0d52..52f0502 100644 --- a/tests/test_models/test_causal_mech/test_plain_mech.py +++ b/tests/test_models/test_causal_mech/test_plain_mech.py @@ -3,7 +3,7 @@ from torch.utils.data import DataLoader from cmrl.models.causal_mech.plain_mech import PlainMech -from cmrl.models.data_loader import BufferDataset, EnsembleBufferDataset, collate_fn +from cmrl.models.data_loader import EnsembleBufferDataset, EnsembleBufferDataset, collate_fn from cmrl.utils.creator import parse_space from cmrl.utils.env import load_offline_data @@ -20,14 +20,14 @@ def prepare(freq_rate): real_replay_buffer, env.observation_space, env.action_space, - is_valid=False, + training=False, mech="transition", train_ensemble=True, ensemble_num=ensemble_num, ) train_loader = DataLoader(train_dataset, batch_size=8, collate_fn=collate_fn) - valid_dataset = BufferDataset( - real_replay_buffer, env.observation_space, env.action_space, is_valid=True, mech="transition", repeat=ensemble_num + valid_dataset = EnsembleBufferDataset( + real_replay_buffer, env.observation_space, env.action_space, training=True, mech="transition", repeat=ensemble_num ) valid_loader = DataLoader(valid_dataset, batch_size=8, collate_fn=collate_fn) diff --git a/tests/test_models/test_causal_mech/test_reinforce.py b/tests/test_models/test_causal_mech/test_reinforce.py index d5bf094..414100e 100644 --- a/tests/test_models/test_causal_mech/test_reinforce.py +++ b/tests/test_models/test_causal_mech/test_reinforce.py @@ -4,7 +4,7 @@ from torch.utils.data import DataLoader from cmrl.models.causal_mech.reinforce import ReinforceCausalMech -from cmrl.models.data_loader import BufferDataset, EnsembleBufferDataset, collate_fn +from cmrl.models.data_loader import EnsembleBufferDataset, EnsembleBufferDataset, collate_fn from cmrl.utils.creator import parse_space from cmrl.utils.env import load_offline_data from cmrl.models.causal_mech.util import variable_loss_func @@ -24,14 +24,14 @@ def prepare(freq_rate): real_replay_buffer, env.observation_space, env.action_space, - is_valid=False, + training=False, mech="transition", train_ensemble=True, ensemble_num=ensemble_num, ) train_loader = DataLoader(train_dataset, batch_size=8, collate_fn=collate_fn) - valid_dataset = BufferDataset( - real_replay_buffer, env.observation_space, env.action_space, is_valid=True, mech="transition", repeat=ensemble_num + valid_dataset = EnsembleBufferDataset( + real_replay_buffer, env.observation_space, env.action_space, training=True, mech="transition", repeat=ensemble_num ) valid_loader = DataLoader(valid_dataset, batch_size=8, collate_fn=collate_fn) diff --git a/tests/test_models/test_data_loader.py b/tests/test_models/test_data_loader.py index f9c35f3..bf436cb 100644 --- a/tests/test_models/test_data_loader.py +++ b/tests/test_models/test_data_loader.py @@ -4,7 +4,7 @@ from stable_baselines3.common.buffers import ReplayBuffer from torch.utils.data import DataLoader -from cmrl.models.data_loader import BufferDataset, EnsembleBufferDataset +from cmrl.models.data_loader import EnsembleBufferDataset, EnsembleBufferDataset from cmrl.utils.env import load_offline_data @@ -17,7 +17,7 @@ def test_buffer_dataset(): load_offline_data(env, real_replay_buffer, "SAC-expert-replay", use_ratio=0.1) # test for transition - dataset = BufferDataset(real_replay_buffer, env.observation_space, env.action_space, mech="transition") + dataset = EnsembleBufferDataset(real_replay_buffer, env.observation_space, env.action_space, mech="transition") loader = DataLoader(dataset, batch_size=128, drop_last=True) for inputs, outputs in loader: @@ -29,7 +29,7 @@ def test_buffer_dataset(): assert outputs[key].shape == (128, 1) # test for reward - dataset = BufferDataset(real_replay_buffer, env.observation_space, env.action_space, mech="reward_mech") + dataset = EnsembleBufferDataset(real_replay_buffer, env.observation_space, env.action_space, mech="reward_mech") loader = DataLoader(dataset, batch_size=128, drop_last=True) for inputs, outputs in loader: @@ -53,7 +53,7 @@ def test_buffer_dataset(): assert outputs[key].shape == (128, 1) # test for termination - dataset = BufferDataset(real_replay_buffer, env.observation_space, env.action_space, mech="termination_mech") + dataset = EnsembleBufferDataset(real_replay_buffer, env.observation_space, env.action_space, mech="termination_mech") loader = DataLoader(dataset, batch_size=128, drop_last=True) for inputs, outputs in loader: @@ -156,11 +156,11 @@ def test_train_valid(): ) load_offline_data(env, real_replay_buffer, "SAC-expert-replay", use_ratio=0.1) - train_dataset = BufferDataset( - real_replay_buffer, env.observation_space, env.action_space, mech="transition", is_valid=False + train_dataset = EnsembleBufferDataset( + real_replay_buffer, env.observation_space, env.action_space, mech="transition", training=False ) - valid_dataset = BufferDataset( - real_replay_buffer, env.observation_space, env.action_space, mech="transition", is_valid=True + valid_dataset = EnsembleBufferDataset( + real_replay_buffer, env.observation_space, env.action_space, mech="transition", training=True ) buffer_size = real_replay_buffer.buffer_size if real_replay_buffer.full else real_replay_buffer.pos @@ -182,7 +182,7 @@ def test_ensemble_train_valid(): env.observation_space, env.action_space, mech="transition", - is_valid=False, + training=False, ensemble_num=ensemble_num, train_ensemble=False, ) @@ -191,7 +191,7 @@ def test_ensemble_train_valid(): env.observation_space, env.action_space, mech="transition", - is_valid=True, + training=True, ensemble_num=ensemble_num, train_ensemble=False, ) @@ -217,12 +217,12 @@ def test_mixed(): env.observation_space, env.action_space, mech="transition", - is_valid=False, + training=False, ensemble_num=ensemble_num, train_ensemble=True, ) - valid_dataset = BufferDataset( - real_replay_buffer, env.observation_space, env.action_space, mech="transition", is_valid=True + valid_dataset = EnsembleBufferDataset( + real_replay_buffer, env.observation_space, env.action_space, mech="transition", training=True ) buffer_size = real_replay_buffer.buffer_size if real_replay_buffer.full else real_replay_buffer.pos From e68768d1f739fb6bca19df6fbfb2a90323991b0b Mon Sep 17 00:00:00 2001 From: frank Date: Tue, 28 Feb 2023 19:02:34 +0800 Subject: [PATCH 52/68] :tada: add self-adaption --- cmrl/models/causal_mech/kernel_test.py | 221 +++++++++++++++---------- 1 file changed, 136 insertions(+), 85 deletions(-) diff --git a/cmrl/models/causal_mech/kernel_test.py b/cmrl/models/causal_mech/kernel_test.py index 0d92d22..e6bb07b 100644 --- a/cmrl/models/causal_mech/kernel_test.py +++ b/cmrl/models/causal_mech/kernel_test.py @@ -1,5 +1,8 @@ from typing import Optional, List, Dict, Union, MutableMapping +from functools import partial +from collections import defaultdict +import pathlib import numpy import numpy as np import torch @@ -11,37 +14,39 @@ from cmrl.models.causal_mech.base import EnsembleNeuralMech from cmrl.utils.variables import Variable, ContinuousVariable, DiscreteVariable, BinaryVariable, RadianVariable +from cmrl.models.graphs.binary_graph import BinaryGraph class KernelTestMech(EnsembleNeuralMech): def __init__( - self, - # base - name: str, - input_variables: List[Variable], - output_variables: List[Variable], - logger: Optional[Logger] = None, - # model learning - longest_epoch: int = -1, - improvement_threshold: float = 0.01, - patience: int = 5, - # ensemble - ensemble_num: int = 7, - elite_num: int = 5, - # cfgs - network_cfg: Optional[DictConfig] = None, - encoder_cfg: Optional[DictConfig] = None, - decoder_cfg: Optional[DictConfig] = None, - optimizer_cfg: Optional[DictConfig] = None, - scheduler_cfg: Optional[DictConfig] = None, - # forward method - residual: bool = True, - encoder_reduction: str = "sum", - # others - device: Union[str, torch.device] = "cpu", - # KCI - sample_num=2000, - kci_times=10 + self, + # base + name: str, + input_variables: List[Variable], + output_variables: List[Variable], + logger: Optional[Logger] = None, + # model learning + longest_epoch: int = -1, + improvement_threshold: float = 0.01, + patience: int = 5, + # ensemble + ensemble_num: int = 7, + elite_num: int = 5, + # cfgs + network_cfg: Optional[DictConfig] = None, + encoder_cfg: Optional[DictConfig] = None, + decoder_cfg: Optional[DictConfig] = None, + optimizer_cfg: Optional[DictConfig] = None, + scheduler_cfg: Optional[DictConfig] = None, + # forward method + residual: bool = True, + encoder_reduction: str = "sum", + # others + device: Union[str, torch.device] = "cpu", + # KCI + sample_num=2000, + kci_times=10, + not_confident_bound=0.2, ): EnsembleNeuralMech.__init__( self, @@ -65,12 +70,43 @@ def __init__( ) self.sample_num = sample_num self.kci_times = kci_times + self.not_confident_bound = not_confident_bound + + def kci( + self, + input_idx: int, + output_idx: int, + inputs: MutableMapping[str, numpy.ndarray], + outputs: MutableMapping[str, numpy.ndarray], + sample_indices: np.ndarray, + ): + in_name, out_name = list(inputs.keys())[input_idx], list(outputs.keys())[output_idx] + + if self.residual: + data_x = outputs[out_name][sample_indices] - inputs[out_name.replace("next_", "")][sample_indices] + else: + data_x = outputs[out_name][sample_indices] + + def deal_with_radian_input(name, data): + if isinstance(self.input_variables_dict[name], RadianVariable): + return (data + np.pi) % (2 * np.pi) - np.pi + else: + return data + + data_y = deal_with_radian_input(in_name, inputs[in_name])[sample_indices] + data_z = [ + deal_with_radian_input(other_in_name, in_data)[sample_indices] + for other_in_name, in_data in inputs.items() + if other_in_name != in_name + ] + data_z = np.concatenate(data_z, axis=1) + + kci = KCI_CInd() + p_value, test_stat = kci.compute_pvalue(data_x, data_y, data_z) + return p_value def kci_compute_graph( - self, - inputs: MutableMapping[str, numpy.ndarray], - outputs: MutableMapping[str, numpy.ndarray], - **kwargs + self, inputs: MutableMapping[str, numpy.ndarray], outputs: MutableMapping[str, numpy.ndarray], **kwargs ): # [[0, 0, 0, 0], @@ -82,41 +118,54 @@ def kci_compute_graph( length = next(iter(inputs.values())).shape[0] sample_length = min(length, self.sample_num) - def deal_with_radian_input(name, data): - if isinstance(self.input_variables_dict[name], RadianVariable): - return (data + np.pi) % (2 * np.pi) - np.pi - else: - return data - - p_values_array = np.empty((self.kci_times, self.input_var_num, self.output_var_num)) - with tqdm(total=self.kci_times * self.input_var_num * self.output_var_num, desc="kci") as pbar: + init_pvalues_array = np.empty((self.kci_times, self.input_var_num, self.output_var_num)) + with tqdm( + total=self.kci_times * self.input_var_num * self.output_var_num, + desc="init kci of {} samples".format(sample_length), + ) as pbar: for time in range(self.kci_times): sample_indices = np.random.permutation(length)[:sample_length] - for out_idx, out_name in enumerate(outputs): - for in_idx, in_name in enumerate(inputs): - if self.residual: - data_x = outputs[out_name][sample_indices] - inputs[out_name.replace("next_", "")][ - sample_indices] - data_x = data_x - else: - data_x = outputs[out_name][sample_indices] - data_y = deal_with_radian_input(in_name, inputs[in_name])[sample_indices] - data_z = [deal_with_radian_input(other_in_name, in_data)[sample_indices] - for other_in_name, in_data in inputs.items() if other_in_name != in_name] - data_z = np.concatenate(data_z, axis=1) - - # data_x = (data_x - data_x.mean(axis=0)) / data_x.std(axis=0) - # data_y = (data_y - data_y.mean(axis=0)) / data_y.std(axis=0) - # data_z = (data_z - data_z.mean(axis=0)) / data_z.std(axis=0) - - kci = KCI_CInd() - p_value, test_stat = kci.compute_pvalue(data_x, data_y, data_z) - p_values_array[time][in_idx][out_idx] = p_value + kci = partial(self.kci, inputs=inputs, outputs=outputs, sample_indices=sample_indices) + for out_idx in range(len(outputs)): + for in_idx in range(len(inputs)): + init_pvalues_array[time][in_idx][out_idx] = kci(in_idx, out_idx) + pbar.update(1) + votes = (init_pvalues_array < 0.05).mean(axis=0) + is_not_confident = np.logical_and(votes > self.not_confident_bound, votes < 1 - self.not_confident_bound) + not_confident_list = np.array(np.where(is_not_confident)).T + + print(votes) + + recompute_times = 1 + while len(not_confident_list) != 0: + new_sample_length = sample_length * (2**recompute_times) + if new_sample_length > length: + break + + pvalues_dict = defaultdict(list) + with tqdm( + total=self.kci_times * len(not_confident_list), + desc="{}th re-compute kci of {} samples".format(recompute_times, new_sample_length), + ) as pbar: + for time in range(self.kci_times): + sample_indices = np.random.permutation(length)[:new_sample_length] + kci = partial(self.kci, inputs=inputs, outputs=outputs, sample_indices=sample_indices) + for in_idx, out_idx in not_confident_list: + pvalues_dict[(in_idx, out_idx)].append(kci(in_idx, out_idx)) pbar.update(1) - final_p_values = (p_values_array < 0.05).mean(axis=0) - print(final_p_values) - pass + + not_confident_list = [] + for key, value in pvalues_dict.items(): + vote = (np.array(value) < 0.05).mean() + if self.not_confident_bound < vote < 1 - self.not_confident_bound: + not_confident_list.append(key) + else: + votes[key] = vote + print(votes) + recompute_times += 1 + + return votes > 0.5 def build_network(self): self.network = instantiate(self.network_cfg)( @@ -125,14 +174,27 @@ def build_network(self): extra_dims=[self.ensemble_num], ).to(self.device) - def forward( - self, - inputs: MutableMapping[str, numpy.ndarray] - ) -> Dict[str, torch.Tensor]: + def forward(self, inputs: MutableMapping[str, numpy.ndarray]) -> Dict[str, torch.Tensor]: pass + def build_graph(self): + self.graph = BinaryGraph(self.input_var_num, self.output_var_num, device=self.device) -if __name__ == '__main__': + def learn( + self, + inputs: MutableMapping[str, np.ndarray], + outputs: MutableMapping[str, np.ndarray], + work_dir: Optional[Union[str, pathlib.Path]] = None, + **kwargs + ): + if self.discovery: + graph = self.kci_compute_graph(inputs, outputs) + self.graph.set_data(graph) + + super(KernelTestMech, self).learn(inputs, outputs, work_dir=work_dir, **kwargs) + + +if __name__ == "__main__": import gym from stable_baselines3.common.buffers import ReplayBuffer from torch.utils.data import DataLoader @@ -144,27 +206,16 @@ def forward( from cmrl.models.causal_mech.util import variable_loss_func env = gym.make("ContinuousCartPoleSwingUp-v0", real_time_scale=0.02) - real_replay_buffer = ReplayBuffer(int(1e6), env.observation_space, env.action_space, "cpu", - handle_timeout_termination=False) - load_offline_data(env, real_replay_buffer, "SAC-expert", use_ratio=1) + real_replay_buffer = ReplayBuffer( + int(1e6), env.observation_space, env.action_space, "cpu", handle_timeout_termination=False + ) + load_offline_data(env, real_replay_buffer, "SAC-expert-replay", use_ratio=1) input_variables = parse_space(env.observation_space, "obs") + parse_space(env.action_space, "act") output_variables = parse_space(env.observation_space, "next_obs") - mech = KernelTestMech( - "kernel_test_mech", - input_variables, - output_variables, - sample_num=1000, - kci_times=20 - ) + mech = KernelTestMech("kernel_test_mech", input_variables, output_variables, sample_num=256, kci_times=16) - inputs, outputs = buffer_to_dict( - env.observation_space, - env.action_space, - env.obs2state, - real_replay_buffer, - "transition" - ) + inputs, outputs = buffer_to_dict(env.observation_space, env.action_space, env.obs2state, real_replay_buffer, "transition") - mech.kci_compute_graph(inputs, outputs) + mech.learn(inputs, outputs) From 5f9ce3ea01f806c650377ee41deac18c381bb0de Mon Sep 17 00:00:00 2001 From: frank Date: Thu, 2 Mar 2023 18:39:23 +0800 Subject: [PATCH 53/68] :hammer: update kci sample num --- cmrl/models/causal_mech/base.py | 9 ++++++++ cmrl/models/causal_mech/kernel_test.py | 29 ++++++++++++++++++++++---- cmrl/models/causal_mech/plain_mech.py | 2 +- 3 files changed, 35 insertions(+), 5 deletions(-) diff --git a/cmrl/models/causal_mech/base.py b/cmrl/models/causal_mech/base.py index 4e691ea..cf2a118 100644 --- a/cmrl/models/causal_mech/base.py +++ b/cmrl/models/causal_mech/base.py @@ -411,3 +411,12 @@ def _maybe_set_best_weights_and_elite(self, best_weights: Optional[Dict], best_v sorted_indices = np.argsort(best_val_score.tolist()) self.elite_indices = sorted_indices[: self.elite_num] + + def get_inputs_batch_size(self, inputs: MutableMapping[str, torch.Tensor]) -> int: + assert len(set(inputs.keys()) & set(self.variable_encoders.keys())) == len(inputs) + data_shape = list(inputs.values())[0].shape + # assert len(data_shape) == 3, "{}".format(data_shape) # ensemble-num, batch-size, specific-dim + ensemble, batch_size, specific_dim = data_shape[-3:] + assert ensemble == self.ensemble_num + + return batch_size, data_shape[:-3] diff --git a/cmrl/models/causal_mech/kernel_test.py b/cmrl/models/causal_mech/kernel_test.py index e6bb07b..28ced19 100644 --- a/cmrl/models/causal_mech/kernel_test.py +++ b/cmrl/models/causal_mech/kernel_test.py @@ -139,7 +139,7 @@ def kci_compute_graph( recompute_times = 1 while len(not_confident_list) != 0: - new_sample_length = sample_length * (2**recompute_times) + new_sample_length = int(sample_length * 1.5**recompute_times) if new_sample_length > length: break @@ -174,8 +174,25 @@ def build_network(self): extra_dims=[self.ensemble_num], ).to(self.device) - def forward(self, inputs: MutableMapping[str, numpy.ndarray]) -> Dict[str, torch.Tensor]: - pass + def forward(self, inputs: MutableMapping[str, torch.Tensor]) -> Dict[str, torch.Tensor]: + batch_size, _ = self.get_inputs_batch_size(inputs) + + inputs_tensor = torch.zeros(self.ensemble_num, batch_size, self.input_var_num, self.encoder_output_dim).to( + self.device) + for i, var in enumerate(self.input_variables): + out = self.variable_encoders[var.name](inputs[var.name].to(self.device)) + inputs_tensor[:, :, i] = out + + output_tensor = self.network(self.reduce_encoder_output(inputs_tensor)) + + outputs = {} + for i, var in enumerate(self.output_variables): + hid = output_tensor[i] + outputs[var.name] = self.variable_decoders[var.name](hid) + + if self.residual: + outputs = self.residual_outputs(inputs, outputs) + return outputs def build_graph(self): self.graph = BinaryGraph(self.input_var_num, self.output_var_num, device=self.device) @@ -203,6 +220,7 @@ def learn( from cmrl.models.data_loader import EnsembleBufferDataset, collate_fn, buffer_to_dict from cmrl.utils.creator import parse_space from cmrl.utils.env import load_offline_data + from cmrl.sb3_extension.logger import configure as logger_configure from cmrl.models.causal_mech.util import variable_loss_func env = gym.make("ContinuousCartPoleSwingUp-v0", real_time_scale=0.02) @@ -214,7 +232,10 @@ def learn( input_variables = parse_space(env.observation_space, "obs") + parse_space(env.action_space, "act") output_variables = parse_space(env.observation_space, "next_obs") - mech = KernelTestMech("kernel_test_mech", input_variables, output_variables, sample_num=256, kci_times=16) + logger = logger_configure("kci-log", ["tensorboard", "stdout"]) + + mech = KernelTestMech("kernel_test_mech", input_variables, output_variables, sample_num=256, kci_times=16, + logger=logger) inputs, outputs = buffer_to_dict(env.observation_space, env.action_space, env.obs2state, real_replay_buffer, "transition") diff --git a/cmrl/models/causal_mech/plain_mech.py b/cmrl/models/causal_mech/plain_mech.py index b745457..afc646b 100644 --- a/cmrl/models/causal_mech/plain_mech.py +++ b/cmrl/models/causal_mech/plain_mech.py @@ -114,7 +114,7 @@ def forward_mask(self): input_variables = parse_space(env.observation_space, "obs") + parse_space(env.action_space, "act") output_variables = parse_space(env.observation_space, "next_obs") - logger = logger_configure("log", ["tensorboard", "stdout"]) + logger = logger_configure("plain-log", ["tensorboard", "stdout"]) mech = PlainMech( "plain_mech", From 909600c61bb4af49ab23d4ce4a43f1b1f3284d9e Mon Sep 17 00:00:00 2001 From: frank Date: Sun, 12 Mar 2023 15:18:13 +0800 Subject: [PATCH 54/68] :hammer: add parallel --- cmrl/examples/conf/main.yaml | 2 +- .../conf/task/parallel_cart_pole.yaml | 23 ++++ cmrl/examples/conf/transition/CMI_test.yaml | 2 - .../examples/conf/transition/kernal_test.yaml | 7 +- cmrl/examples/conf/transition/plain.yaml | 7 +- cmrl/models/causal_mech/CMI_test.py | 3 - cmrl/models/causal_mech/base.py | 2 +- cmrl/models/causal_mech/kernel_test.py | 111 ++++++++++-------- cmrl/models/causal_mech/plain_mech.py | 1 - 9 files changed, 92 insertions(+), 66 deletions(-) create mode 100644 cmrl/examples/conf/task/parallel_cart_pole.yaml diff --git a/cmrl/examples/conf/main.yaml b/cmrl/examples/conf/main.yaml index e2091ab..eea827c 100644 --- a/cmrl/examples/conf/main.yaml +++ b/cmrl/examples/conf/main.yaml @@ -1,6 +1,6 @@ defaults: - algorithm: off_dyna - - task: continuous_cart_pole_swingup + - task: parallel_cart_pole - transition: plain - reward_mech: plain - termination_mech: plain diff --git a/cmrl/examples/conf/task/parallel_cart_pole.yaml b/cmrl/examples/conf/task/parallel_cart_pole.yaml new file mode 100644 index 0000000..ca6d235 --- /dev/null +++ b/cmrl/examples/conf/task/parallel_cart_pole.yaml @@ -0,0 +1,23 @@ +# env parameters +env_id: "ParallelContinuousCartPoleSwingUp-v0" + +params: + freq_rate: 1 + real_time_scale: 0.02 + integrator: "euler" + parallel_num: 3 + +dataset: "SAC-expert-replay" + +# basic RL params +num_steps: 3000000 +online_num_steps: 10000 +n_eval_episodes: 5 +eval_freq: 10000 + +# offline +penalty_coeff: 1 +use_ratio: 1 + +# dyna +freq_train_model: 100 diff --git a/cmrl/examples/conf/transition/CMI_test.yaml b/cmrl/examples/conf/transition/CMI_test.yaml index 29b7dd5..691e74f 100644 --- a/cmrl/examples/conf/transition/CMI_test.yaml +++ b/cmrl/examples/conf/transition/CMI_test.yaml @@ -56,8 +56,6 @@ mech: patience: 5 longest_epoch: -1 improvement_threshold: 0.01 - # mask - mask_method: "zero" # ensemble ensemble_num: 7 elite_num: 5 diff --git a/cmrl/examples/conf/transition/kernal_test.yaml b/cmrl/examples/conf/transition/kernal_test.yaml index 48ef2e0..8313a80 100644 --- a/cmrl/examples/conf/transition/kernal_test.yaml +++ b/cmrl/examples/conf/transition/kernal_test.yaml @@ -57,8 +57,6 @@ mech: patience: 5 longest_epoch: -1 improvement_threshold: 0.01 - # mask - mask_method: "zero" # ensemble ensemble_num: 7 elite_num: 5 @@ -74,5 +72,6 @@ mech: # others device: ${device} # KCI - sample_num: 2000 - kci_times: 10 + sample_num: 256 + kci_times: 16 + not_confident_bound: 0.2 diff --git a/cmrl/examples/conf/transition/plain.yaml b/cmrl/examples/conf/transition/plain.yaml index 4e8b77c..5e779bb 100644 --- a/cmrl/examples/conf/transition/plain.yaml +++ b/cmrl/examples/conf/transition/plain.yaml @@ -38,7 +38,11 @@ optimizer_cfg: weight_decay: 1e-5 eps: 1e-8 - +scheduler_cfg: + _partial_: true + _target_: torch.optim.lr_scheduler.StepLR + step_size: 1 + gamma: 1 mech: _partial_: true @@ -62,7 +66,6 @@ mech: optimizer_cfg: ${transition.optimizer_cfg} # forward method residual: true - multi_step: "none" # logger logger: ??? # others diff --git a/cmrl/models/causal_mech/CMI_test.py b/cmrl/models/causal_mech/CMI_test.py index 3ac1195..2f9a7a9 100644 --- a/cmrl/models/causal_mech/CMI_test.py +++ b/cmrl/models/causal_mech/CMI_test.py @@ -38,10 +38,8 @@ def __init__( # forward method residual: bool = True, encoder_reduction: str = "sum", - multi_step: str = "none", # others device: Union[str, torch.device] = "cpu", - **kwargs ): EnsembleNeuralMech.__init__( self, @@ -61,7 +59,6 @@ def __init__( residual=residual, encoder_reduction=encoder_reduction, device=device, - **kwargs ) self.total_CMI_epoch = 0 diff --git a/cmrl/models/causal_mech/base.py b/cmrl/models/causal_mech/base.py index cf2a118..1f6e7eb 100644 --- a/cmrl/models/causal_mech/base.py +++ b/cmrl/models/causal_mech/base.py @@ -40,7 +40,7 @@ def __init__( self.name = name self.input_variables = input_variables self.output_variables = output_variables - self.logger=logger + self.logger = logger self.input_variables_dict = dict([(v.name, v) for v in self.input_variables]) self.output_variables_dict = dict([(v.name, v) for v in self.output_variables]) diff --git a/cmrl/models/causal_mech/kernel_test.py b/cmrl/models/causal_mech/kernel_test.py index 28ced19..6ec07a0 100644 --- a/cmrl/models/causal_mech/kernel_test.py +++ b/cmrl/models/causal_mech/kernel_test.py @@ -19,34 +19,34 @@ class KernelTestMech(EnsembleNeuralMech): def __init__( - self, - # base - name: str, - input_variables: List[Variable], - output_variables: List[Variable], - logger: Optional[Logger] = None, - # model learning - longest_epoch: int = -1, - improvement_threshold: float = 0.01, - patience: int = 5, - # ensemble - ensemble_num: int = 7, - elite_num: int = 5, - # cfgs - network_cfg: Optional[DictConfig] = None, - encoder_cfg: Optional[DictConfig] = None, - decoder_cfg: Optional[DictConfig] = None, - optimizer_cfg: Optional[DictConfig] = None, - scheduler_cfg: Optional[DictConfig] = None, - # forward method - residual: bool = True, - encoder_reduction: str = "sum", - # others - device: Union[str, torch.device] = "cpu", - # KCI - sample_num=2000, - kci_times=10, - not_confident_bound=0.2, + self, + # base + name: str, + input_variables: List[Variable], + output_variables: List[Variable], + logger: Optional[Logger] = None, + # model learning + longest_epoch: int = -1, + improvement_threshold: float = 0.01, + patience: int = 5, + # ensemble + ensemble_num: int = 7, + elite_num: int = 5, + # cfgs + network_cfg: Optional[DictConfig] = None, + encoder_cfg: Optional[DictConfig] = None, + decoder_cfg: Optional[DictConfig] = None, + optimizer_cfg: Optional[DictConfig] = None, + scheduler_cfg: Optional[DictConfig] = None, + # forward method + residual: bool = True, + encoder_reduction: str = "sum", + # others + device: Union[str, torch.device] = "cpu", + # KCI + sample_num=2000, + kci_times=10, + not_confident_bound=0.2, ): EnsembleNeuralMech.__init__( self, @@ -73,12 +73,12 @@ def __init__( self.not_confident_bound = not_confident_bound def kci( - self, - input_idx: int, - output_idx: int, - inputs: MutableMapping[str, numpy.ndarray], - outputs: MutableMapping[str, numpy.ndarray], - sample_indices: np.ndarray, + self, + input_idx: int, + output_idx: int, + inputs: MutableMapping[str, numpy.ndarray], + outputs: MutableMapping[str, numpy.ndarray], + sample_indices: np.ndarray, ): in_name, out_name = list(inputs.keys())[input_idx], list(outputs.keys())[output_idx] @@ -106,7 +106,7 @@ def deal_with_radian_input(name, data): return p_value def kci_compute_graph( - self, inputs: MutableMapping[str, numpy.ndarray], outputs: MutableMapping[str, numpy.ndarray], **kwargs + self, inputs: MutableMapping[str, numpy.ndarray], outputs: MutableMapping[str, numpy.ndarray], **kwargs ): # [[0, 0, 0, 0], @@ -120,8 +120,8 @@ def kci_compute_graph( init_pvalues_array = np.empty((self.kci_times, self.input_var_num, self.output_var_num)) with tqdm( - total=self.kci_times * self.input_var_num * self.output_var_num, - desc="init kci of {} samples".format(sample_length), + total=self.kci_times * self.input_var_num * self.output_var_num, + desc="init kci of {} samples".format(sample_length), ) as pbar: for time in range(self.kci_times): sample_indices = np.random.permutation(length)[:sample_length] @@ -139,14 +139,14 @@ def kci_compute_graph( recompute_times = 1 while len(not_confident_list) != 0: - new_sample_length = int(sample_length * 1.5**recompute_times) + new_sample_length = int(sample_length * 1.5 ** recompute_times) if new_sample_length > length: break pvalues_dict = defaultdict(list) with tqdm( - total=self.kci_times * len(not_confident_list), - desc="{}th re-compute kci of {} samples".format(recompute_times, new_sample_length), + total=self.kci_times * len(not_confident_list), + desc="{}th re-compute kci of {} samples".format(recompute_times, new_sample_length), ) as pbar: for time in range(self.kci_times): sample_indices = np.random.permutation(length)[:new_sample_length] @@ -198,11 +198,11 @@ def build_graph(self): self.graph = BinaryGraph(self.input_var_num, self.output_var_num, device=self.device) def learn( - self, - inputs: MutableMapping[str, np.ndarray], - outputs: MutableMapping[str, np.ndarray], - work_dir: Optional[Union[str, pathlib.Path]] = None, - **kwargs + self, + inputs: MutableMapping[str, np.ndarray], + outputs: MutableMapping[str, np.ndarray], + work_dir: Optional[Union[str, pathlib.Path]] = None, + **kwargs ): if self.discovery: graph = self.kci_compute_graph(inputs, outputs) @@ -213,8 +213,10 @@ def learn( if __name__ == "__main__": import gym + from emei import EmeiEnv from stable_baselines3.common.buffers import ReplayBuffer from torch.utils.data import DataLoader + from typing import cast from cmrl.models.causal_mech.reinforce import ReinforceCausalMech from cmrl.models.data_loader import EnsembleBufferDataset, collate_fn, buffer_to_dict @@ -223,20 +225,25 @@ def learn( from cmrl.sb3_extension.logger import configure as logger_configure from cmrl.models.causal_mech.util import variable_loss_func - env = gym.make("ContinuousCartPoleSwingUp-v0", real_time_scale=0.02) + def unwrap_env(env): + while isinstance(env, gym.Wrapper): + env = env.env + return env + + env = unwrap_env(gym.make("ParallelContinuousCartPoleSwingUp-v0")) real_replay_buffer = ReplayBuffer( int(1e6), env.observation_space, env.action_space, "cpu", handle_timeout_termination=False ) - load_offline_data(env, real_replay_buffer, "SAC-expert-replay", use_ratio=1) + load_offline_data(env, real_replay_buffer, "SAC-expert", use_ratio=0.1) - input_variables = parse_space(env.observation_space, "obs") + parse_space(env.action_space, "act") - output_variables = parse_space(env.observation_space, "next_obs") + input_variables = parse_space(env.state_space, "obs") + parse_space(env.action_space, "act") + output_variables = parse_space(env.state_space, "next_obs") logger = logger_configure("kci-log", ["tensorboard", "stdout"]) - mech = KernelTestMech("kernel_test_mech", input_variables, output_variables, sample_num=256, kci_times=16, - logger=logger) + mech = KernelTestMech("kernel_test_mech", input_variables, output_variables, sample_num=256, kci_times=5, + logger=logger) - inputs, outputs = buffer_to_dict(env.observation_space, env.action_space, env.obs2state, real_replay_buffer, "transition") + inputs, outputs = buffer_to_dict(env.state_space, env.action_space, env.obs2state, real_replay_buffer, "transition") mech.learn(inputs, outputs) diff --git a/cmrl/models/causal_mech/plain_mech.py b/cmrl/models/causal_mech/plain_mech.py index afc646b..2d181b6 100644 --- a/cmrl/models/causal_mech/plain_mech.py +++ b/cmrl/models/causal_mech/plain_mech.py @@ -37,7 +37,6 @@ def __init__( encoder_reduction: str = "sum", # others device: Union[str, torch.device] = "cpu", - **kwargs ): EnsembleNeuralMech.__init__( self, From 8546dc6043183536e0d436dc8a083e38cbf67518 Mon Sep 17 00:00:00 2001 From: frank Date: Sun, 12 Mar 2023 18:11:48 +0800 Subject: [PATCH 55/68] :tada: update plain mech --- cmrl/examples/conf/transition/CMI_test.yaml | 1 + .../examples/conf/transition/kernal_test.yaml | 1 + cmrl/examples/conf/transition/plain.yaml | 5 +- cmrl/models/causal_mech/CMI_test.py | 152 +++---- cmrl/models/causal_mech/base.py | 167 ++++---- cmrl/models/causal_mech/kernel_test.py | 115 +++--- cmrl/models/causal_mech/neural_causal_mech.py | 371 ------------------ cmrl/models/causal_mech/plain_mech.py | 111 ++---- 8 files changed, 232 insertions(+), 691 deletions(-) delete mode 100644 cmrl/models/causal_mech/neural_causal_mech.py diff --git a/cmrl/examples/conf/transition/CMI_test.yaml b/cmrl/examples/conf/transition/CMI_test.yaml index 691e74f..faf71fd 100644 --- a/cmrl/examples/conf/transition/CMI_test.yaml +++ b/cmrl/examples/conf/transition/CMI_test.yaml @@ -56,6 +56,7 @@ mech: patience: 5 longest_epoch: -1 improvement_threshold: 0.01 + batch_size: 256 # ensemble ensemble_num: 7 elite_num: 5 diff --git a/cmrl/examples/conf/transition/kernal_test.yaml b/cmrl/examples/conf/transition/kernal_test.yaml index 8313a80..9dc9c8f 100644 --- a/cmrl/examples/conf/transition/kernal_test.yaml +++ b/cmrl/examples/conf/transition/kernal_test.yaml @@ -57,6 +57,7 @@ mech: patience: 5 longest_epoch: -1 improvement_threshold: 0.01 + batch_size: 256 # ensemble ensemble_num: 7 elite_num: 5 diff --git a/cmrl/examples/conf/transition/plain.yaml b/cmrl/examples/conf/transition/plain.yaml index 5e779bb..ad13f77 100644 --- a/cmrl/examples/conf/transition/plain.yaml +++ b/cmrl/examples/conf/transition/plain.yaml @@ -6,7 +6,7 @@ encoder_cfg: _partial_: true _recursive_: false _target_: cmrl.models.networks.VariableEncoder - output_dim: 100 + output_dim: 200 hidden_dims: [ 100 ] bias: true activation_fn_cfg: @@ -26,7 +26,7 @@ network_cfg: _partial_: true _recursive_: false _target_: cmrl.models.networks.ParallelMLP - hidden_dims: [ 200, 200 ] + hidden_dims: [ 100, 100] bias: true activation_fn_cfg: _target_: torch.nn.SiLU @@ -56,6 +56,7 @@ mech: patience: 5 longest_epoch: -1 improvement_threshold: 0.01 + batch_size: 256 # ensemble ensemble_num: 7 elite_num: 5 diff --git a/cmrl/models/causal_mech/CMI_test.py b/cmrl/models/causal_mech/CMI_test.py index 2f9a7a9..c4cadfc 100644 --- a/cmrl/models/causal_mech/CMI_test.py +++ b/cmrl/models/causal_mech/CMI_test.py @@ -18,28 +18,29 @@ class CMITestMech(EnsembleNeuralMech): def __init__( - self, - name: str, - input_variables: List[Variable], - output_variables: List[Variable], - logger: Optional[Logger] = None, - # model learning - longest_epoch: int = -1, - improvement_threshold: float = 0.01, - patience: int = 5, - # ensemble - ensemble_num: int = 7, - elite_num: int = 5, - # cfgs - network_cfg: Optional[DictConfig] = None, - encoder_cfg: Optional[DictConfig] = None, - decoder_cfg: Optional[DictConfig] = None, - optimizer_cfg: Optional[DictConfig] = None, - # forward method - residual: bool = True, - encoder_reduction: str = "sum", - # others - device: Union[str, torch.device] = "cpu", + self, + name: str, + input_variables: List[Variable], + output_variables: List[Variable], + logger: Optional[Logger] = None, + # model learning + longest_epoch: int = -1, + improvement_threshold: float = 0.01, + patience: int = 5, + batch_size: int = 256, + # ensemble + ensemble_num: int = 7, + elite_num: int = 5, + # cfgs + network_cfg: Optional[DictConfig] = None, + encoder_cfg: Optional[DictConfig] = None, + decoder_cfg: Optional[DictConfig] = None, + optimizer_cfg: Optional[DictConfig] = None, + # forward method + residual: bool = True, + encoder_reduction: str = "sum", + # others + device: Union[str, torch.device] = "cpu", ): EnsembleNeuralMech.__init__( self, @@ -50,6 +51,7 @@ def __init__( longest_epoch=longest_epoch, improvement_threshold=improvement_threshold, patience=patience, + batch_size=batch_size, ensemble_num=ensemble_num, elite_num=elite_num, network_cfg=network_cfg, @@ -73,26 +75,6 @@ def build_network(self): def build_graph(self): self.graph = BinaryGraph(self.input_var_num, self.output_var_num, device=self.device) - def forward(self, inputs: MutableMapping[str, torch.Tensor]) -> Dict[str, torch.Tensor]: - batch_size, _ = self.get_inputs_batch_size(inputs) - - inputs_tensor = torch.zeros(self.ensemble_num, batch_size, self.input_var_num, self.encoder_output_dim).to( - self.device) - for i, var in enumerate(self.input_variables): - out = self.variable_encoders[var.name](inputs[var.name].to(self.device)) - inputs_tensor[:, :, i] = out - - output_tensor = self.network(self.reduce_encoder_output(inputs_tensor)) - - outputs = {} - for i, var in enumerate(self.output_variables): - hid = output_tensor[i] - outputs[var.name] = self.variable_decoders[var.name](hid) - - if self.residual: - outputs = self.residual_outputs(inputs, outputs) - return outputs - @property def CMI_mask(self) -> torch.Tensor: mask = torch.zeros(self.input_var_num + 1, self.output_var_num, self.input_var_num, dtype=torch.long) @@ -115,9 +97,9 @@ def multi_graph_forward(self, inputs: MutableMapping[str, torch.Tensor]) -> Dict """ batch_size, extra_dim = self.get_inputs_info(inputs) - inputs_tensor = torch.empty(*extra_dim, self.ensemble_num, batch_size, self.input_var_num, - self.encoder_output_dim).to( - self.device) + inputs_tensor = torch.empty(*extra_dim, self.ensemble_num, batch_size, self.input_var_num, self.encoder_output_dim).to( + self.device + ) for i, var in enumerate(self.input_variables): out = self.variable_encoders[var.name](inputs[var.name].to(self.device)) inputs_tensor[..., i, :] = out @@ -163,7 +145,7 @@ def multi_graph_forward(self, inputs: MutableMapping[str, torch.Tensor]) -> Dict mask = mask.repeat((1,) * len(mask.shape[:-3]) + (self.ensemble_num, batch_size, 1)) reduced_inputs_tensor = self.reduce_encoder_output(inputs_tensor, mask) assert ( - not torch.isinf(reduced_inputs_tensor).any() and not torch.isnan(reduced_inputs_tensor).any() + not torch.isinf(reduced_inputs_tensor).any() and not torch.isnan(reduced_inputs_tensor).any() ), "tensor must not be inf or nan" output_tensor = self.network(reduced_inputs_tensor) @@ -176,46 +158,18 @@ def multi_graph_forward(self, inputs: MutableMapping[str, torch.Tensor]) -> Dict outputs = self.residual_outputs(inputs, outputs) return outputs - # def CMI_forward(self, inputs: MutableMapping[str, torch.Tensor]) -> Dict[str, torch.Tensor]: - # """inputs should be dict of str and Tensor with (ensemble-num, batch-size, specific-dim) shape - # - # Args: - # inputs: - # - # Returns: - # - # """ - # if self.multi_step.startswith("forward-euler"): - # step_num = int(self.multi_step.split()[-1]) - # - # outputs = {} - # for step in range(step_num): - # outputs = self.CMI_single_step_forward(inputs) - # # outputs shape: (input-var-num + 1, ensemble-num, batch-size, specific-dim * 2) - # # new inputs shape: (input-var-num + 1, ensemble-num, batch-size, specific-dim) - # if step == 0: - # for name in filter(lambda s: s.startswith("act"), inputs.keys()): - # inputs[name] = inputs[name][None, ...].repeat([self.input_var_num + 1, 1, 1, 1]) - # if step < step_num - 1: - # for name in filter(lambda s: s.startswith("obs"), inputs.keys()): - # inputs[name] = outputs["next_{}".format(name)][..., : inputs[name].shape[-1]] - # else: - # raise NotImplementedError("multi-step method {} is not supported".format(self.multi_step)) - # - # return outputs - def calculate_CMI(self, nll_loss: torch.Tensor, threshold=1): nll_loss_diff = nll_loss[:-1] - nll_loss[-1] graph_data = (nll_loss_diff.mean(dim=(1, 2)) > threshold).to(torch.long) return graph_data, nll_loss_diff.mean(dim=(1, 2)) def learn( - self, - # loader - inputs: MutableMapping[str, np.ndarray], - outputs: MutableMapping[str, np.ndarray], - work_dir: Optional[Union[str, pathlib.Path]] = None, - **kwargs + self, + # loader + inputs: MutableMapping[str, np.ndarray], + outputs: MutableMapping[str, np.ndarray], + work_dir: Optional[Union[str, pathlib.Path]] = None, + **kwargs ): if self.discovery: train_loader, valid_loader = self.get_data_loaders(inputs, outputs) @@ -275,7 +229,7 @@ def learn( super(CMITestMech, self).learn(inputs, outputs, work_dir=work_dir, **kwargs) -if __name__ == '__main__': +if __name__ == "__main__": import gym from stable_baselines3.common.buffers import ReplayBuffer from torch.utils.data import DataLoader @@ -283,29 +237,29 @@ def learn( from cmrl.models.causal_mech.reinforce import ReinforceCausalMech from cmrl.models.data_loader import EnsembleBufferDataset, collate_fn, buffer_to_dict from cmrl.utils.creator import parse_space + from cmrl.sb3_extension.logger import configure as logger_configure + from cmrl.utils.env import load_offline_data from cmrl.models.causal_mech.util import variable_loss_func - env = gym.make("ContinuousCartPoleSwingUp-v0", real_time_scale=0.02) - real_replay_buffer = ReplayBuffer(int(1e6), env.observation_space, env.action_space, "cpu", - handle_timeout_termination=False) - load_offline_data(env, real_replay_buffer, "SAC-expert", use_ratio=1) + def unwrap_env(env): + while isinstance(env, gym.Wrapper): + env = env.env + return env - input_variables = parse_space(env.observation_space, "obs") + parse_space(env.action_space, "act") - output_variables = parse_space(env.observation_space, "next_obs") - - mech = CMITestMech( - "kernel_test_mech", - input_variables, - output_variables, + env = unwrap_env(gym.make("ParallelContinuousCartPoleSwingUp-v0")) + real_replay_buffer = ReplayBuffer( + int(1e6), env.observation_space, env.action_space, "cpu", handle_timeout_termination=False ) + load_offline_data(env, real_replay_buffer, "SAC-expert", use_ratio=0.1) - inputs, outputs = buffer_to_dict( - env.observation_space, - env.action_space, - env.obs2state, - real_replay_buffer, - "transition" - ) + input_variables = parse_space(env.state_space, "obs") + parse_space(env.action_space, "act") + output_variables = parse_space(env.state_space, "next_obs") + + logger = logger_configure("kci-log", ["tensorboard", "stdout"]) + + mech = CMITestMech("kernel_test_mech", input_variables, output_variables, device="cuda") + + inputs, outputs = buffer_to_dict(env.state_space, env.action_space, env.obs2state, real_replay_buffer, "transition") mech.learn(inputs, outputs) diff --git a/cmrl/models/causal_mech/base.py b/cmrl/models/causal_mech/base.py index 1f6e7eb..59eca37 100644 --- a/cmrl/models/causal_mech/base.py +++ b/cmrl/models/causal_mech/base.py @@ -31,11 +31,11 @@ class BaseCausalMech(ABC): """ def __init__( - self, - name: str, - input_variables: List[Variable], - output_variables: List[Variable], - logger: Optional[Logger] = None, + self, + name: str, + input_variables: List[Variable], + output_variables: List[Variable], + logger: Optional[Logger] = None, ): self.name = name self.input_variables = input_variables @@ -52,27 +52,23 @@ def __init__( @abstractmethod def learn( - self, - inputs: MutableMapping[str, np.ndarray], - outputs: MutableMapping[str, np.ndarray], - work_dir: Optional[Union[str, pathlib.Path]] = None, - **kwargs + self, + inputs: MutableMapping[str, np.ndarray], + outputs: MutableMapping[str, np.ndarray], + work_dir: Optional[Union[str, pathlib.Path]] = None, + **kwargs ): raise NotImplementedError @abstractmethod - def forward( - self, - inputs: MutableMapping[str, np.ndarray] - ) -> Dict[str, torch.Tensor]: + def forward(self, inputs: MutableMapping[str, np.ndarray]) -> Dict[str, torch.Tensor]: raise NotImplementedError @property def causal_graph(self) -> torch.Tensor: """property causal graph""" if self.graph is None: - return torch.ones(len(self.input_variables), len(self.output_variables), dtype=torch.int, - device=self.device) + return torch.ones(len(self.input_variables), len(self.output_variables), dtype=torch.int, device=self.device) else: return self.graph.get_binary_adj_matrix() @@ -91,42 +87,40 @@ def load(self, load_dir: Union[str, pathlib.Path]): class EnsembleNeuralMech(BaseCausalMech): def __init__( - self, - # base - name: str, - input_variables: List[Variable], - output_variables: List[Variable], - logger: Optional[Logger] = None, - # model learning - longest_epoch: int = -1, - improvement_threshold: float = 0.01, - patience: int = 5, - # ensemble - ensemble_num: int = 7, - elite_num: int = 5, - # cfgs - network_cfg: Optional[DictConfig] = None, - encoder_cfg: Optional[DictConfig] = None, - decoder_cfg: Optional[DictConfig] = None, - optimizer_cfg: Optional[DictConfig] = None, - scheduler_cfg: Optional[DictConfig] = None, - # forward method - residual: bool = True, - encoder_reduction: str = "sum", - # others - device: Union[str, torch.device] = "cpu", + self, + # base + name: str, + input_variables: List[Variable], + output_variables: List[Variable], + logger: Optional[Logger] = None, + # model learning + longest_epoch: int = -1, + improvement_threshold: float = 0.01, + patience: int = 5, + batch_size: int = 256, + # ensemble + ensemble_num: int = 7, + elite_num: int = 5, + # cfgs + network_cfg: Optional[DictConfig] = None, + encoder_cfg: Optional[DictConfig] = None, + decoder_cfg: Optional[DictConfig] = None, + optimizer_cfg: Optional[DictConfig] = None, + scheduler_cfg: Optional[DictConfig] = None, + # forward method + residual: bool = True, + encoder_reduction: str = "sum", + # others + device: Union[str, torch.device] = "cpu", ): BaseCausalMech.__init__( - self, - name=name, - input_variables=input_variables, - output_variables=output_variables, - logger=logger + self, name=name, input_variables=input_variables, output_variables=output_variables, logger=logger ) # model learning self.longest_epoch = longest_epoch self.improvement_threshold = improvement_threshold self.patience = patience + self.batch_size = batch_size # ensemble self.ensemble_num = ensemble_num self.elite_num = elite_num @@ -169,21 +163,40 @@ def build_network(self): self.network = instantiate(self.network_cfg)( input_dim=self.encoder_output_dim, output_dim=self.decoder_input_dim, - extra_dims=[self.ensemble_num], + extra_dims=[self.output_var_num, self.ensemble_num], ).to(self.device) def build_optimizer(self): assert self.network, "you must build network first" assert self.variable_encoders and self.variable_decoders, "you must build coders first" params = ( - [self.network.parameters()] - + [encoder.parameters() for encoder in self.variable_encoders.values()] - + [decoder.parameters() for decoder in self.variable_decoders.values()] + [self.network.parameters()] + + [encoder.parameters() for encoder in self.variable_encoders.values()] + + [decoder.parameters() for decoder in self.variable_decoders.values()] ) self.optimizer = instantiate(self.optimizer_cfg)(params=chain(*params)) self.scheduler = instantiate(self.scheduler_cfg)(optimizer=self.optimizer) + def forward(self, inputs: MutableMapping[str, torch.Tensor]) -> Dict[str, torch.Tensor]: + batch_size, _ = self.get_inputs_batch_size(inputs) + + inputs_tensor = torch.zeros(self.ensemble_num, batch_size, self.input_var_num, self.encoder_output_dim).to(self.device) + for i, var in enumerate(self.input_variables): + out = self.variable_encoders[var.name](inputs[var.name].to(self.device)) + inputs_tensor[:, :, i] = out + + output_tensor = self.network(self.reduce_encoder_output(inputs_tensor)) + + outputs = {} + for i, var in enumerate(self.output_variables): + hid = output_tensor[i] + outputs[var.name] = self.variable_decoders[var.name](hid) + + if self.residual: + outputs = self.residual_outputs(inputs, outputs) + return outputs + def build_graph(self): pass @@ -235,9 +248,9 @@ def get_inputs_info(self, inputs: MutableMapping[str, torch.Tensor]): return batch_size, data_shape[:-3] def residual_outputs( - self, - inputs: MutableMapping[str, torch.Tensor], - outputs: MutableMapping[str, torch.Tensor], + self, + inputs: MutableMapping[str, torch.Tensor], + outputs: MutableMapping[str, torch.Tensor], ) -> MutableMapping[str, torch.Tensor]: for name in filter(lambda s: s.startswith("obs"), inputs.keys()): # assert inputs[name].shape[:2] == outputs["next_{}".format(name)].shape[:2] @@ -247,9 +260,9 @@ def residual_outputs( return outputs def reduce_encoder_output( - self, - encoder_output: torch.Tensor, - mask: Optional[torch.Tensor] = None, + self, + encoder_output: torch.Tensor, + mask: Optional[torch.Tensor] = None, ) -> torch.Tensor: assert len(encoder_output.shape) == 4, ( "shape of `encoder_output` should be (ensemble-num, batch-size, input-var-num, encoder-output-dim), " @@ -265,7 +278,7 @@ def reduce_encoder_output( # mask shape [..., ensemble-num, batch-size, input-var-num] assert ( - mask.shape[-3:] == encoder_output.shape[:-1] + mask.shape[-3:] == encoder_output.shape[:-1] ), "mask shape should be (..., ensemble-num, batch-size, input-var-num)" # [*mask-extra-dims, ensemble-num, batch-size, input-var-num, encoder-output-dim] @@ -294,38 +307,28 @@ def forward_mask(self) -> torch.Tensor: return self.causal_graph.T def get_data_loaders( - self, - inputs: MutableMapping[str, np.ndarray], - outputs: MutableMapping[str, np.ndarray], + self, + inputs: MutableMapping[str, np.ndarray], + outputs: MutableMapping[str, np.ndarray], ): train_set = EnsembleBufferDataset( - inputs=inputs, - outputs=outputs, - training=True, - train_ratio=0.8, - ensemble_num=7, - seed=1 + inputs=inputs, outputs=outputs, training=True, train_ratio=0.8, ensemble_num=self.ensemble_num, seed=1 ) valid_set = EnsembleBufferDataset( - inputs=inputs, - outputs=outputs, - training=False, - train_ratio=0.8, - ensemble_num=7, - seed=1 + inputs=inputs, outputs=outputs, training=False, train_ratio=0.8, ensemble_num=self.ensemble_num, seed=1 ) - train_loader = DataLoader(train_set, batch_size=32, collate_fn=collate_fn) - valid_loader = DataLoader(valid_set, batch_size=32, collate_fn=collate_fn) + train_loader = DataLoader(train_set, batch_size=self.batch_size, collate_fn=collate_fn) + valid_loader = DataLoader(valid_set, batch_size=self.batch_size, collate_fn=collate_fn) return train_loader, valid_loader def learn( - self, - inputs: MutableMapping[str, np.ndarray], - outputs: MutableMapping[str, np.ndarray], - work_dir: Optional[Union[str, pathlib.Path]] = None, - **kwargs + self, + inputs: MutableMapping[str, np.ndarray], + outputs: MutableMapping[str, np.ndarray], + work_dir: Optional[Union[str, pathlib.Path]] = None, + **kwargs ): train_loader, valid_loader = self.get_data_loaders(inputs, outputs) @@ -379,10 +382,10 @@ def learn( self.save(save_dir=work_dir) def _maybe_get_best_weights( - self, - best_val_loss: torch.Tensor, - val_loss: torch.Tensor, - threshold: float = 0.01, + self, + best_val_loss: torch.Tensor, + val_loss: torch.Tensor, + threshold: float = 0.01, ) -> Optional[Dict]: """Return the current model state dict if the validation score improves. For ensembles, this checks the validation for each ensemble member separately. diff --git a/cmrl/models/causal_mech/kernel_test.py b/cmrl/models/causal_mech/kernel_test.py index 6ec07a0..089ed3b 100644 --- a/cmrl/models/causal_mech/kernel_test.py +++ b/cmrl/models/causal_mech/kernel_test.py @@ -19,34 +19,35 @@ class KernelTestMech(EnsembleNeuralMech): def __init__( - self, - # base - name: str, - input_variables: List[Variable], - output_variables: List[Variable], - logger: Optional[Logger] = None, - # model learning - longest_epoch: int = -1, - improvement_threshold: float = 0.01, - patience: int = 5, - # ensemble - ensemble_num: int = 7, - elite_num: int = 5, - # cfgs - network_cfg: Optional[DictConfig] = None, - encoder_cfg: Optional[DictConfig] = None, - decoder_cfg: Optional[DictConfig] = None, - optimizer_cfg: Optional[DictConfig] = None, - scheduler_cfg: Optional[DictConfig] = None, - # forward method - residual: bool = True, - encoder_reduction: str = "sum", - # others - device: Union[str, torch.device] = "cpu", - # KCI - sample_num=2000, - kci_times=10, - not_confident_bound=0.2, + self, + # base + name: str, + input_variables: List[Variable], + output_variables: List[Variable], + logger: Optional[Logger] = None, + # model learning + longest_epoch: int = -1, + improvement_threshold: float = 0.01, + patience: int = 5, + batch_size: int = 256, + # ensemble + ensemble_num: int = 7, + elite_num: int = 5, + # cfgs + network_cfg: Optional[DictConfig] = None, + encoder_cfg: Optional[DictConfig] = None, + decoder_cfg: Optional[DictConfig] = None, + optimizer_cfg: Optional[DictConfig] = None, + scheduler_cfg: Optional[DictConfig] = None, + # forward method + residual: bool = True, + encoder_reduction: str = "sum", + # others + device: Union[str, torch.device] = "cpu", + # KCI + sample_num=2000, + kci_times=10, + not_confident_bound=0.2, ): EnsembleNeuralMech.__init__( self, @@ -57,6 +58,7 @@ def __init__( longest_epoch=longest_epoch, improvement_threshold=improvement_threshold, patience=patience, + batch_size=batch_size, ensemble_num=ensemble_num, elite_num=elite_num, network_cfg=network_cfg, @@ -73,12 +75,12 @@ def __init__( self.not_confident_bound = not_confident_bound def kci( - self, - input_idx: int, - output_idx: int, - inputs: MutableMapping[str, numpy.ndarray], - outputs: MutableMapping[str, numpy.ndarray], - sample_indices: np.ndarray, + self, + input_idx: int, + output_idx: int, + inputs: MutableMapping[str, numpy.ndarray], + outputs: MutableMapping[str, numpy.ndarray], + sample_indices: np.ndarray, ): in_name, out_name = list(inputs.keys())[input_idx], list(outputs.keys())[output_idx] @@ -106,7 +108,7 @@ def deal_with_radian_input(name, data): return p_value def kci_compute_graph( - self, inputs: MutableMapping[str, numpy.ndarray], outputs: MutableMapping[str, numpy.ndarray], **kwargs + self, inputs: MutableMapping[str, numpy.ndarray], outputs: MutableMapping[str, numpy.ndarray], **kwargs ): # [[0, 0, 0, 0], @@ -120,8 +122,8 @@ def kci_compute_graph( init_pvalues_array = np.empty((self.kci_times, self.input_var_num, self.output_var_num)) with tqdm( - total=self.kci_times * self.input_var_num * self.output_var_num, - desc="init kci of {} samples".format(sample_length), + total=self.kci_times * self.input_var_num * self.output_var_num, + desc="init kci of {} samples".format(sample_length), ) as pbar: for time in range(self.kci_times): sample_indices = np.random.permutation(length)[:sample_length] @@ -139,14 +141,14 @@ def kci_compute_graph( recompute_times = 1 while len(not_confident_list) != 0: - new_sample_length = int(sample_length * 1.5 ** recompute_times) + new_sample_length = int(sample_length * 1.5**recompute_times) if new_sample_length > length: break pvalues_dict = defaultdict(list) with tqdm( - total=self.kci_times * len(not_confident_list), - desc="{}th re-compute kci of {} samples".format(recompute_times, new_sample_length), + total=self.kci_times * len(not_confident_list), + desc="{}th re-compute kci of {} samples".format(recompute_times, new_sample_length), ) as pbar: for time in range(self.kci_times): sample_indices = np.random.permutation(length)[:new_sample_length] @@ -174,35 +176,15 @@ def build_network(self): extra_dims=[self.ensemble_num], ).to(self.device) - def forward(self, inputs: MutableMapping[str, torch.Tensor]) -> Dict[str, torch.Tensor]: - batch_size, _ = self.get_inputs_batch_size(inputs) - - inputs_tensor = torch.zeros(self.ensemble_num, batch_size, self.input_var_num, self.encoder_output_dim).to( - self.device) - for i, var in enumerate(self.input_variables): - out = self.variable_encoders[var.name](inputs[var.name].to(self.device)) - inputs_tensor[:, :, i] = out - - output_tensor = self.network(self.reduce_encoder_output(inputs_tensor)) - - outputs = {} - for i, var in enumerate(self.output_variables): - hid = output_tensor[i] - outputs[var.name] = self.variable_decoders[var.name](hid) - - if self.residual: - outputs = self.residual_outputs(inputs, outputs) - return outputs - def build_graph(self): self.graph = BinaryGraph(self.input_var_num, self.output_var_num, device=self.device) def learn( - self, - inputs: MutableMapping[str, np.ndarray], - outputs: MutableMapping[str, np.ndarray], - work_dir: Optional[Union[str, pathlib.Path]] = None, - **kwargs + self, + inputs: MutableMapping[str, np.ndarray], + outputs: MutableMapping[str, np.ndarray], + work_dir: Optional[Union[str, pathlib.Path]] = None, + **kwargs ): if self.discovery: graph = self.kci_compute_graph(inputs, outputs) @@ -241,8 +223,7 @@ def unwrap_env(env): logger = logger_configure("kci-log", ["tensorboard", "stdout"]) - mech = KernelTestMech("kernel_test_mech", input_variables, output_variables, sample_num=256, kci_times=5, - logger=logger) + mech = KernelTestMech("kernel_test_mech", input_variables, output_variables, sample_num=1024, kci_times=5, logger=logger) inputs, outputs = buffer_to_dict(env.state_space, env.action_space, env.obs2state, real_replay_buffer, "transition") diff --git a/cmrl/models/causal_mech/neural_causal_mech.py b/cmrl/models/causal_mech/neural_causal_mech.py deleted file mode 100644 index 9a19a1d..0000000 --- a/cmrl/models/causal_mech/neural_causal_mech.py +++ /dev/null @@ -1,371 +0,0 @@ -from typing import Optional, List, Dict, Union, MutableMapping -from abc import abstractmethod, ABC -from itertools import chain, count -from functools import partial -import pathlib -import copy - -import torch -import numpy as np -from torch.utils.data import DataLoader -import torch.nn.functional as F -from torch.optim import Optimizer -from stable_baselines3.common.logger import Logger -from omegaconf import DictConfig -from hydra.utils import instantiate - -from cmrl.models.causal_mech.base import BaseCausalMech -from cmrl.models.networks.base_network import BaseNetwork -from cmrl.models.graphs.base_graph import BaseGraph -from cmrl.models.networks.coder import VariableEncoder, VariableDecoder -from cmrl.utils.variables import Variable, ContinuousVariable, DiscreteVariable, BinaryVariable -from cmrl.models.causal_mech.util import variable_loss_func, train_func, eval_func - - -class NeuralCausalMech(BaseCausalMech): - def __init__( - self, - name: str, - input_variables: List[Variable], - output_variables: List[Variable], - # model learning - longest_epoch: int = -1, - improvement_threshold: float = 0.01, - patience: int = 5, - # ensemble - ensemble_num: int = 7, - elite_num: int = 5, - # cfgs - network_cfg: Optional[DictConfig] = None, - encoder_cfg: Optional[DictConfig] = None, - decoder_cfg: Optional[DictConfig] = None, - optimizer_cfg: Optional[DictConfig] = None, - scheduler_cfg: Optional[DictConfig] = None, - # forward method - residual: bool = True, - encoder_reduction: str = "sum", - multi_step: str = "none", - # logger - logger: Optional[Logger] = None, - # others - device: Union[str, torch.device] = "cpu", - **kwargs - ): - super(NeuralCausalMech, self).__init__( - name=name, - input_variables=input_variables, - output_variables=output_variables, - device=device, - ) - # model learning - self.longest_epoch = longest_epoch - self.improvement_threshold = improvement_threshold - self.patience = patience - # ensemble - self.ensemble_num = ensemble_num - self.elite_num = elite_num - # cfgs - self.network_cfg = default_network_cfg if network_cfg is None else network_cfg - self.encoder_cfg = default_encoder_cfg if encoder_cfg is None else encoder_cfg - self.decoder_cfg = default_decoder_cfg if decoder_cfg is None else decoder_cfg - self.optimizer_cfg = default_optimizer_cfg if optimizer_cfg is None else optimizer_cfg - self.scheduler_cfg = default_scheduler_cfg if scheduler_cfg is None else scheduler_cfg - # forward method - self.residual = residual - self.encoder_reduction = encoder_reduction - self.multi_step = multi_step - # logger - self.logger = logger - - # build member object - self.variable_encoders: Optional[Dict[str, VariableEncoder]] = None - self.variable_decoders: Optional[Dict[str, VariableEncoder]] = None - self.network: Optional[BaseNetwork] = None - self.graph: Optional[BaseGraph] = None - self.optimizer: Optional[Optimizer] = None - self.scheduler: Optional[object] = None - self.build_coder() - self.build_network() - self.build_graph() - self.build_optimizer() - - self.total_epoch = 0 - self.elite_indices: List[int] = [] - - def single_step_forward(self, inputs: MutableMapping[str, torch.Tensor]) -> Dict[str, torch.Tensor]: - batch_size, _ = self.get_inputs_batch_size(inputs) - - inputs_tensor = torch.zeros(self.ensemble_num, batch_size, self.input_var_num, self.encoder_output_dim).to( - self.device) - for i, var in enumerate(self.input_variables): - out = self.variable_encoders[var.name](inputs[var.name].to(self.device)) - inputs_tensor[:, :, i] = out - - for name, param in self.network.named_parameters(): - if param.grad is not None and torch.isnan(param.grad).any(): - print("nan gradient found") - print("name:", name) - print("param:", param.grad) - raise SystemExit - - output_tensor = self.network(self.reduce_encoder_output(inputs_tensor)) - - outputs = {} - for i, var in enumerate(self.output_variables): - hid = output_tensor[:, :, i * self.decoder_input_dim: (i + 1) * self.decoder_input_dim] - outputs[var.name] = self.variable_decoders[var.name](hid) - - if self.residual: - outputs = self.residual_outputs(inputs, outputs) - return outputs - - def forward(self, inputs: MutableMapping[str, torch.Tensor]) -> Dict[str, torch.Tensor]: - if self.multi_step.startswith("forward-euler"): - step_num = int(self.multi_step.split()[-1]) - - outputs = {} - for step in range(step_num): - outputs = self.single_step_forward(inputs) - if step < step_num - 1: - for name in filter(lambda s: s.startswith("obs"), inputs.keys()): - assert inputs[name].shape[:2] == outputs["next_{}".format(name)].shape[:2] - assert inputs[name].shape[2] * 2 == outputs["next_{}".format(name)].shape[2] - inputs[name] = outputs["next_{}".format(name)][:, :, : inputs[name].shape[2]] - else: - raise NotImplementedError("multi-step method {} is not supported".format(self.multi_step)) - - return outputs - - @abstractmethod - def build_network(self): - raise NotImplementedError - - def build_optimizer(self): - assert self.network is not None, "you must build network first" - params = ( - [self.network.parameters()] - + [encoder.parameters() for encoder in self.variable_encoders.values()] - + [decoder.parameters() for decoder in self.variable_decoders.values()] - ) - - self.optimizer = instantiate(self.optimizer_cfg)(params=chain(*params)) - self.scheduler = instantiate(self.scheduler_cfg)(optimizer=self.optimizer) - - @abstractmethod - def build_graph(self): - raise NotImplementedError - - def build_coder(self): - self.variable_encoders = {} - for var in self.input_variables: - assert var.name not in self.variable_encoders, "duplicate name in encoders: {}".format(var.name) - self.variable_encoders[var.name] = instantiate(self.encoder_cfg)(variable=var).to(self.device) - - assert self.decoder_input_dim - - self.variable_decoders = {} - for var in self.output_variables: - assert var.name not in self.variable_decoders, "duplicate name in decoders: {}".format(var.name) - self.variable_decoders[var.name] = instantiate(self.decoder_cfg)(variable=var).to(self.device) - - def get_inputs_batch_size(self, inputs: MutableMapping[str, torch.Tensor]) -> int: - assert len(set(inputs.keys()) & set(self.variable_encoders.keys())) == len(inputs) - data_shape = list(inputs.values())[0].shape - # assert len(data_shape) == 3, "{}".format(data_shape) # ensemble-num, batch-size, specific-dim - ensemble, batch_size, specific_dim = data_shape[-3:] - assert ensemble == self.ensemble_num - - return batch_size, data_shape[:-3] - - def residual_outputs( - self, - inputs: MutableMapping[str, torch.Tensor], - outputs: MutableMapping[str, torch.Tensor], - ) -> MutableMapping[str, torch.Tensor]: - for name in filter(lambda s: s.startswith("obs"), inputs.keys()): - # assert inputs[name].shape[:2] == outputs["next_{}".format(name)].shape[:2] - # assert inputs[name].shape[2] * 2 == outputs["next_{}".format(name)].shape[2] - var_dim = inputs[name].shape[-1] - outputs["next_{}".format(name)][..., :var_dim] += inputs[name].to(self.device) - return outputs - - def learn( - self, - # loader - train_loader: DataLoader, - valid_loader: DataLoader, - work_dir: Optional[Union[str, pathlib.Path]] = None, - **kwargs - ): - best_weights: Optional[Dict] = None - epoch_iter = range(self.longest_epoch) if self.longest_epoch >= 0 else count() - epochs_since_update = 0 - - loss_func = partial(variable_loss_func, output_variables=self.output_variables, device=self.device) - train = partial(train_func, forward=self.forward, optimizer=self.optimizer, loss_func=loss_func) - eval = partial(eval_func, forward=self.forward, loss_func=loss_func) - - best_eval_loss = eval(valid_loader).mean(dim=(-2, -1)) - - for epoch in epoch_iter: - train_loss = train(train_loader) - eval_loss = eval(valid_loader) - - maybe_best_weights = self._maybe_get_best_weights( - best_eval_loss, eval_loss.mean(dim=(-2, -1)), self.improvement_threshold - ) - if maybe_best_weights: - # best loss - best_eval_loss = torch.minimum(best_eval_loss, eval_loss.mean(dim=(-2, -1))) - best_weights = maybe_best_weights - epochs_since_update = 0 - else: - epochs_since_update += 1 - - # log - self.total_epoch += 1 - if self.logger is not None: - self.logger.record("{}/epoch".format(self.name), epoch) - self.logger.record("{}/epochs_since_update".format(self.name), epochs_since_update) - self.logger.record("{}/train_dataset_size".format(self.name), len(train_loader.dataset)) - self.logger.record("{}/valid_dataset_size".format(self.name), len(valid_loader.dataset)) - self.logger.record("{}/train_loss".format(self.name), train_loss.mean().item()) - self.logger.record("{}/val_loss".format(self.name), eval_loss.mean().item()) - self.logger.record("{}/best_val_loss".format(self.name), best_eval_loss.mean().item()) - self.logger.record("{}/lr".format(self.name), self.optimizer.param_groups[0]["lr"]) - - self.logger.dump(self.total_epoch) - - if self.patience and epochs_since_update >= self.patience: - break - - self.scheduler.step() - - # saving the best models: - self._maybe_set_best_weights_and_elite(best_weights, best_eval_loss) - - self.save(save_dir=work_dir) - - def save(self, save_dir: Union[str, pathlib.Path]): - if isinstance(save_dir, str): - save_dir = pathlib.Path(save_dir) - save_dir = save_dir / pathlib.Path(self.name) - save_dir.mkdir(exist_ok=True) - - self.network.save(save_dir) - if self.graph is not None: - self.graph.save(save_dir) - for coder in self.variable_encoders.values(): - coder.save(save_dir) - for coder in self.variable_decoders.values(): - coder.save(save_dir) - - def load(self, load_dir: Union[str, pathlib.Path]): - if isinstance(load_dir, str): - load_dir = pathlib.Path(load_dir) - assert load_dir.exists() - - self.network.load(load_dir) - if self.graph is not None: - self.graph.load(load_dir) - for coder in self.variable_encoders.values(): - coder.load(load_dir) - for coder in self.variable_decoders.values(): - coder.load(load_dir) - - def _maybe_get_best_weights( - self, - best_val_loss: torch.Tensor, - val_loss: torch.Tensor, - threshold: float = 0.01, - ) -> Optional[Dict]: - """Return the current model state dict if the validation score improves. - For ensembles, this checks the validation for each ensemble member separately. - Copy from https://github.com/facebookresearch/mbrl-lib/blob/main/mbrl/models/model_trainer.py - - Args: - best_val_score (tensor): the current best validation losses per model. - val_score (tensor): the new validation loss per model. - threshold (float): the threshold for relative improvement. - Returns: - (dict, optional): if the validation score's relative improvement over the - best validation score is higher than the threshold, returns the state dictionary - of the stored model, otherwise returns ``None``. - """ - improvement = (best_val_loss - val_loss) / torch.abs(best_val_loss) - if (improvement > threshold).any().item(): - best_weights = copy.deepcopy(self.network.state_dict()) - else: - best_weights = None - - return best_weights - - def _maybe_set_best_weights_and_elite(self, best_weights: Optional[Dict], best_val_score: torch.Tensor): - if best_weights is not None: - self.network.load_state_dict(best_weights) - - sorted_indices = np.argsort(best_val_score.tolist()) - self.elite_indices = sorted_indices[: self.elite_num] - - @property - def encoder_output_dim(self): - return self.encoder_cfg.output_dim - - @property - def union_output_var_dim(self): - # all output variables should be ContinuousVariable and have same variable.dim - output_dim = [] - for var in self.output_variables: - assert isinstance(var, ContinuousVariable), "all output variables should be ContinuousVariable" - output_dim.append(var.dim) - assert len(set(output_dim)) == 1, "all output variables should have same variable.dim" - return output_dim[0] - - @property - def decoder_input_dim(self): - if self.decoder_cfg.identity: - return self.union_output_var_dim * 2 - else: - return self.decoder_cfg.input_dim - - def reduce_encoder_output( - self, - encoder_output: torch.Tensor, - mask: Optional[torch.Tensor] = None, - ) -> torch.Tensor: - assert len(encoder_output.shape) == 4, ( - "shape of encoder_output should be (ensemble-num, batch-size, input-var-num, encoder-output-dim), " - "rather than {}".format(encoder_output.shape) - ) - - if mask is None: - # [..., input-var-num] - mask = self.forward_mask - # [..., ensemble-num, batch-size, input-var-num] - mask = mask.unsqueeze(-2).unsqueeze(-2) - mask = mask.repeat((1,) * len(mask.shape[:-3]) + (*encoder_output.shape[:2], 1)) - - # mask shape [..., ensemble-num, batch-size, input-var-num] - assert ( - mask.shape[-3:] == encoder_output.shape[:-1] - ), "mask shape should be (..., ensemble-num, batch-size, input-var-num)" - - # [*mask-extra-dims, ensemble-num, batch-size, input-var-num, encoder-output-dim] - mask = mask[..., None].repeat([1] * len(mask.shape) + [encoder_output.shape[-1]]) - masked_encoder_output = encoder_output.repeat(tuple(mask.shape[:-4]) + (1,) * 4) - - # choose mask value - mask_value = 0 - if self.encoder_reduction == "max": - mask_value = -float("inf") - masked_encoder_output[mask == 0] = mask_value - - if self.encoder_reduction == "sum": - return masked_encoder_output.sum(-2) - elif self.encoder_reduction == "mean": - return masked_encoder_output.mean(-2) - elif self.encoder_reduction == "max": - values, indices = masked_encoder_output.max(-2) - return values - else: - raise NotImplementedError("not implemented encoder reduction method: {}".format(self.encoder_reduction)) diff --git a/cmrl/models/causal_mech/plain_mech.py b/cmrl/models/causal_mech/plain_mech.py index 2d181b6..7cbf77c 100644 --- a/cmrl/models/causal_mech/plain_mech.py +++ b/cmrl/models/causal_mech/plain_mech.py @@ -14,29 +14,30 @@ class PlainMech(EnsembleNeuralMech): def __init__( - self, - name: str, - input_variables: List[Variable], - output_variables: List[Variable], - logger: Optional[Logger] = None, - # model learning - longest_epoch: int = -1, - improvement_threshold: float = 0.01, - patience: int = 5, - # ensemble - ensemble_num: int = 7, - elite_num: int = 5, - # cfgs - network_cfg: Optional[DictConfig] = None, - encoder_cfg: Optional[DictConfig] = None, - decoder_cfg: Optional[DictConfig] = None, - optimizer_cfg: Optional[DictConfig] = None, - scheduler_cfg: Optional[DictConfig] = None, - # forward method - residual: bool = True, - encoder_reduction: str = "sum", - # others - device: Union[str, torch.device] = "cpu", + self, + name: str, + input_variables: List[Variable], + output_variables: List[Variable], + logger: Optional[Logger] = None, + # model learning + longest_epoch: int = -1, + improvement_threshold: float = 0.01, + patience: int = 5, + batch_size: int = 256, + # ensemble + ensemble_num: int = 7, + elite_num: int = 5, + # cfgs + network_cfg: Optional[DictConfig] = None, + encoder_cfg: Optional[DictConfig] = None, + decoder_cfg: Optional[DictConfig] = None, + optimizer_cfg: Optional[DictConfig] = None, + scheduler_cfg: Optional[DictConfig] = None, + # forward method + residual: bool = True, + encoder_reduction: str = "sum", + # others + device: Union[str, torch.device] = "cpu", ): EnsembleNeuralMech.__init__( self, @@ -47,6 +48,7 @@ def __init__( longest_epoch=longest_epoch, improvement_threshold=improvement_threshold, patience=patience, + batch_size=batch_size, ensemble_num=ensemble_num, elite_num=elite_num, network_cfg=network_cfg, @@ -59,41 +61,12 @@ def __init__( device=device, ) - def forward( - self, - inputs: MutableMapping[str, torch.Tensor] - ) -> Dict[str, torch.Tensor]: - batch_size, _ = self.get_inputs_info(inputs) - - inputs_tensor = torch.zeros(self.ensemble_num, batch_size, self.input_var_num, self.encoder_output_dim).to( - self.device) - for i, var in enumerate(self.input_variables): - out = self.variable_encoders[var.name](inputs[var.name].to(self.device)) - inputs_tensor[:, :, i] = out - - # for name, param in self.network.named_parameters(): - # if param.grad is not None and torch.isnan(param.grad).any(): - # print("nan gradient found") - # print("name:", name) - # print("param:", param.grad) - # raise SystemExit - - output_tensor = self.network(self.reduce_encoder_output(inputs_tensor)) - - outputs = {} - for i, var in enumerate(self.output_variables): - outputs[var.name] = self.variable_decoders[var.name](output_tensor) - - if self.residual: - outputs = self.residual_outputs(inputs, outputs) - return outputs - @property def forward_mask(self): - return torch.ones(self.input_var_num).to(self.device) + return torch.ones(self.output_var_num, self.input_var_num).to(self.device) -if __name__ == '__main__': +if __name__ == "__main__": import gym from stable_baselines3.common.buffers import ReplayBuffer from torch.utils.data import DataLoader @@ -105,31 +78,29 @@ def forward_mask(self): from cmrl.models.causal_mech.util import variable_loss_func from cmrl.sb3_extension.logger import configure as logger_configure - env = gym.make("ContinuousCartPoleSwingUp-v0", real_time_scale=0.02) - real_replay_buffer = ReplayBuffer(int(1e6), env.observation_space, env.action_space, "cpu", - handle_timeout_termination=False) - load_offline_data(env, real_replay_buffer, "SAC-expert-replay", use_ratio=1) + def unwrap_env(env): + while isinstance(env, gym.Wrapper): + env = env.env + return env - input_variables = parse_space(env.observation_space, "obs") + parse_space(env.action_space, "act") - output_variables = parse_space(env.observation_space, "next_obs") + env = unwrap_env(gym.make("ParallelContinuousCartPoleSwingUp-v0")) + real_replay_buffer = ReplayBuffer( + int(1e6), env.observation_space, env.action_space, "cpu", handle_timeout_termination=False + ) + load_offline_data(env, real_replay_buffer, "SAC-expert", use_ratio=0.1) - logger = logger_configure("plain-log", ["tensorboard", "stdout"]) + input_variables = parse_space(env.state_space, "obs") + parse_space(env.action_space, "act") + output_variables = parse_space(env.state_space, "next_obs") + + logger = logger_configure("kci-log", ["tensorboard", "stdout"]) mech = PlainMech( "plain_mech", input_variables, output_variables, logger=logger, - sample_num=2000, - kci_times=20, ) - inputs, outputs = buffer_to_dict( - env.observation_space, - env.action_space, - env.obs2state, - real_replay_buffer, - "transition" - ) + inputs, outputs = buffer_to_dict(env.observation_space, env.action_space, env.obs2state, real_replay_buffer, "transition") mech.learn(inputs, outputs) From 92a5f60c5511d6e287e5f848b5b3b87d151e7589 Mon Sep 17 00:00:00 2001 From: frank Date: Tue, 21 Mar 2023 01:09:57 +0800 Subject: [PATCH 56/68] :hammer: add state2obs_fn --- README.md | 30 +- cmrl/algorithms/base_algorithm.py | 16 +- cmrl/diagnostics/eval_model_on_dataset.py | 4 +- cmrl/diagnostics/eval_model_on_space.py | 4 +- cmrl/diagnostics/run_trained_model.py | 2 +- cmrl/examples/conf/main.yaml | 4 +- .../task/continuous_cart_pole_swingup.yaml | 4 + .../conf/task/parallel_cart_pole.yaml | 6 + .../{kernal_test.yaml => kernel_test.yaml} | 0 cmrl/models/causal_mech/kernel_test.py | 20 +- cmrl/models/data_loader.py | 22 +- cmrl/models/dynamics.py | 20 +- cmrl/types.py | 1 + cmrl/utils/RCIT.py | 643 ++++++++++++++++++ cmrl/utils/creator.py | 15 +- cmrl/utils/env.py | 18 +- cmrl/utils/variables.py | 27 +- requirements/main.txt | 2 +- .../test_online_mb_callback.py | 10 +- 19 files changed, 776 insertions(+), 72 deletions(-) rename cmrl/examples/conf/transition/{kernal_test.yaml => kernel_test.yaml} (100%) create mode 100644 cmrl/utils/RCIT.py diff --git a/README.md b/README.md index fa20294..4091dd8 100644 --- a/README.md +++ b/README.md @@ -117,16 +117,38 @@ conda install pytorch -c pytorch pip install -e . ``` -If there is no `cuda` in your device, it's convenient to install `cuda` and `pytorch` from conda directly (refer -to [pytorch](https://pytorch.org/get-started/locally/)): +for pytorch -````shell +```shell # for MacOS conda install pytorch -c pytorch # for Linux conda install pytorch pytorch-cuda=11.6 -c pytorch -c nvidia -```` +``` + +for KCIT and RCIT + +```shell +conda install -c conda-forge r-base +conda install -c conda-forge r-devtools +R +``` +```shell +# Install the RCIT from Github. +install.packages("devtools") +library(devtools) +install_github("ericstrobl/RCIT") +library(RCIT) + +# Install R libraries for RCIT +install.packages("MASS") +install.packages("momentchi2") +install.packages("devtools") + +# test RCIT +RCIT(rnorm(1000),rnorm(1000),rnorm(1000)) +``` ## install using pip coming soon. diff --git a/cmrl/algorithms/base_algorithm.py b/cmrl/algorithms/base_algorithm.py index a6eabc1..8d99f0d 100644 --- a/cmrl/algorithms/base_algorithm.py +++ b/cmrl/algorithms/base_algorithm.py @@ -25,7 +25,9 @@ def __init__( self.cfg = cfg self.work_dir = work_dir or os.getcwd() - self.env, self.reward_fn, self.termination_fn, self.get_init_obs_fn, self.obs2state_fn = make_env(self.cfg) + self.env, fns = make_env(self.cfg) + self.reward_fn, self.termination_fn, self.get_init_obs_fn, self.obs2state_fn, self.state2obs_fn = fns + self.eval_env, *_ = make_env(self.cfg) np.random.seed(self.cfg.seed) torch.manual_seed(self.cfg.seed) @@ -41,7 +43,11 @@ def __init__( ) # create ``cmrl`` dynamics - self.dynamics = create_dynamics(self.cfg, self.env.observation_space, self.env.action_space, self.obs2state_fn, + self.dynamics = create_dynamics(self.cfg, + self.env.state_space, + self.env.action_space, + self.obs2state_fn, + self.state2obs_fn, logger=self.logger) if not self.cfg.transition.discovery: @@ -63,7 +69,7 @@ def __init__( self.partial_fake_env = partial( VecFakeEnv, self.cfg.algorithm.num_envs, - self.env.observation_space, + self.env.state_space, self.env.action_space, self.dynamics, self.reward_fn, @@ -86,7 +92,9 @@ def fake_env(self) -> VecFakeEnv: @property def callback(self) -> BaseCallback: fake_eval_env = self.partial_fake_env( - deterministic=True, max_episode_steps=self.env.spec.max_episode_steps, branch_rollout=False + deterministic=True, + max_episode_steps=self.env.spec.max_episode_steps, + branch_rollout=False ) return EvalCallback( self.eval_env, diff --git a/cmrl/diagnostics/eval_model_on_dataset.py b/cmrl/diagnostics/eval_model_on_dataset.py index d8531bc..faf379a 100644 --- a/cmrl/diagnostics/eval_model_on_dataset.py +++ b/cmrl/diagnostics/eval_model_on_dataset.py @@ -25,7 +25,7 @@ def __init__(self, model_dir: str, dataset: str, batch_size: int = 4096, device= self.dynamics = cmrl.util.creator.create_dynamics( self.cfg.dynamics, - self.env.observation_space.shape, + self.env.state_space.shape, self.env.action_space.shape, load_dir=self.model_path, load_device=device, @@ -33,7 +33,7 @@ def __init__(self, model_dir: str, dataset: str, batch_size: int = 4096, device= self.replay_buffer = cmrl.util.creator.create_replay_buffer( self.cfg, - self.env.observation_space.shape, + self.env.state_space.shape, self.env.action_space.shape, ) diff --git a/cmrl/diagnostics/eval_model_on_space.py b/cmrl/diagnostics/eval_model_on_space.py index 09f23ca..4e17e53 100644 --- a/cmrl/diagnostics/eval_model_on_space.py +++ b/cmrl/diagnostics/eval_model_on_space.py @@ -75,7 +75,7 @@ def __init__( self.dynamics = create_dynamics( self.cfg, - self.env.observation_space, + self.env.state_space, self.env.action_space, ) if not self.cfg.transition.discovery: @@ -90,7 +90,7 @@ def __init__( self.obs_range, self.action_range = self.get_range() self.range = np.concatenate([self.obs_range, self.action_range], axis=0) - self.real_obs_dim_num = self.env.observation_space.shape[0] + self.real_obs_dim_num = self.env.state_space.shape[0] self.compact_obs_dim_num, self.action_dim_num = ( self.obs_range.shape[0], self.action_range.shape[0], diff --git a/cmrl/diagnostics/run_trained_model.py b/cmrl/diagnostics/run_trained_model.py index ebdc9c0..0871d43 100644 --- a/cmrl/diagnostics/run_trained_model.py +++ b/cmrl/diagnostics/run_trained_model.py @@ -34,7 +34,7 @@ def __init__(self, model_dir: str, device: str = "cuda:0", render: bool = False) self.dynamics = cmrl.util.creator.create_dynamics( self.cfg.dynamics, - self.env.observation_space.shape, + self.env.state_space.shape, self.env.action_space.shape, load_dir=self.model_path, load_device=device, diff --git a/cmrl/examples/conf/main.yaml b/cmrl/examples/conf/main.yaml index eea827c..e55977e 100644 --- a/cmrl/examples/conf/main.yaml +++ b/cmrl/examples/conf/main.yaml @@ -1,7 +1,7 @@ defaults: - algorithm: off_dyna - - task: parallel_cart_pole - - transition: plain + - task: continuous_cart_pole_swingup + - transition: kernel_test - reward_mech: plain - termination_mech: plain - _self_ diff --git a/cmrl/examples/conf/task/continuous_cart_pole_swingup.yaml b/cmrl/examples/conf/task/continuous_cart_pole_swingup.yaml index f95940c..bdf1483 100644 --- a/cmrl/examples/conf/task/continuous_cart_pole_swingup.yaml +++ b/cmrl/examples/conf/task/continuous_cart_pole_swingup.yaml @@ -11,6 +11,10 @@ params: dataset: "SAC-expert-replay" +extra_variable_info: + Radian: + - "obs_1" + # basic RL params num_steps: 3000000 online_num_steps: 10000 diff --git a/cmrl/examples/conf/task/parallel_cart_pole.yaml b/cmrl/examples/conf/task/parallel_cart_pole.yaml index ca6d235..d48fcca 100644 --- a/cmrl/examples/conf/task/parallel_cart_pole.yaml +++ b/cmrl/examples/conf/task/parallel_cart_pole.yaml @@ -9,6 +9,12 @@ params: dataset: "SAC-expert-replay" +extra_variable_info: + Radian: + - "obs_1" + - "obs_5" + - "obs_9" + # basic RL params num_steps: 3000000 online_num_steps: 10000 diff --git a/cmrl/examples/conf/transition/kernal_test.yaml b/cmrl/examples/conf/transition/kernel_test.yaml similarity index 100% rename from cmrl/examples/conf/transition/kernal_test.yaml rename to cmrl/examples/conf/transition/kernel_test.yaml diff --git a/cmrl/models/causal_mech/kernel_test.py b/cmrl/models/causal_mech/kernel_test.py index 6ec07a0..c0a0971 100644 --- a/cmrl/models/causal_mech/kernel_test.py +++ b/cmrl/models/causal_mech/kernel_test.py @@ -9,7 +9,8 @@ from omegaconf import DictConfig from stable_baselines3.common.logger import Logger from hydra.utils import instantiate -from causallearn.utils.KCI.KCI import KCI_CInd +from cmrl.utils.RCIT import KCI_CInd +# from causallearn.utils.KCI.KCI import KCI_CInd from tqdm import tqdm from cmrl.models.causal_mech.base import EnsembleNeuralMech @@ -116,7 +117,7 @@ def kci_compute_graph( # [0, 0, 1, 1]] length = next(iter(inputs.values())).shape[0] - sample_length = min(length, self.sample_num) + sample_length = min(length, self.sample_num) if self.sample_num > 0 else length init_pvalues_array = np.empty((self.kci_times, self.input_var_num, self.output_var_num)) with tqdm( @@ -225,23 +226,28 @@ def learn( from cmrl.sb3_extension.logger import configure as logger_configure from cmrl.models.causal_mech.util import variable_loss_func + def unwrap_env(env): while isinstance(env, gym.Wrapper): env = env.env return env - env = unwrap_env(gym.make("ParallelContinuousCartPoleSwingUp-v0")) + + env = unwrap_env(gym.make("ContinuousCartPoleSwingUp-v0")) real_replay_buffer = ReplayBuffer( int(1e6), env.observation_space, env.action_space, "cpu", handle_timeout_termination=False ) - load_offline_data(env, real_replay_buffer, "SAC-expert", use_ratio=0.1) + load_offline_data(env, real_replay_buffer, "SAC-expert", use_ratio=1) + + # extra_info = {"Radian": ["obs_1", "obs_5", "obs_9"]} + extra_info = {"Radian": ["obs_1"]} - input_variables = parse_space(env.state_space, "obs") + parse_space(env.action_space, "act") - output_variables = parse_space(env.state_space, "next_obs") + input_variables = parse_space(env.state_space, "obs", extra_info=extra_info) + parse_space(env.action_space, "act") + output_variables = parse_space(env.state_space, "next_obs", extra_info=extra_info) logger = logger_configure("kci-log", ["tensorboard", "stdout"]) - mech = KernelTestMech("kernel_test_mech", input_variables, output_variables, sample_num=256, kci_times=5, + mech = KernelTestMech("kernel_test_mech", input_variables, output_variables, sample_num=1000, kci_times=20, logger=logger) inputs, outputs = buffer_to_dict(env.state_space, env.action_space, env.obs2state, real_replay_buffer, "transition") diff --git a/cmrl/models/data_loader.py b/cmrl/models/data_loader.py index 4f386f7..f9c6cd1 100644 --- a/cmrl/models/data_loader.py +++ b/cmrl/models/data_loader.py @@ -10,7 +10,7 @@ def buffer_to_dict( - observation_space, + state_space, action_space, obs2state_fn, replay_buffer: ReplayBuffer, @@ -25,38 +25,38 @@ def buffer_to_dict( real_buffer_size = replay_buffer.buffer_size if replay_buffer.full else replay_buffer.pos if hasattr(replay_buffer, "extra_obs"): - observations = obs2state_fn(replay_buffer.observations[: real_buffer_size, 0], + states = obs2state_fn(replay_buffer.observations[: real_buffer_size, 0], replay_buffer.extra_obs[: real_buffer_size, 0]) else: - observations = replay_buffer.observations[: real_buffer_size, 0] - obs_dict = to_dict_by_space(observations, observation_space, prefix="obs", ) + states = replay_buffer.observations[: real_buffer_size, 0] + state_dict = to_dict_by_space(states, state_space, prefix="obs", ) act_dict = to_dict_by_space( replay_buffer.actions[: real_buffer_size, 0], action_space, prefix="act", ) if hasattr(replay_buffer, "next_extra_obs"): - next_observations = obs2state_fn(replay_buffer.next_observations[: real_buffer_size, 0], + next_states = obs2state_fn(replay_buffer.next_observations[: real_buffer_size, 0], replay_buffer.next_extra_obs[: real_buffer_size, 0]) else: - next_observations = replay_buffer.next_observations[: real_buffer_size, 0] - next_obs_dict = to_dict_by_space(next_observations, observation_space, prefix="next_obs") + next_states = replay_buffer.next_observations[: real_buffer_size, 0] + next_state_dict = to_dict_by_space(next_states, state_space, prefix="next_obs") inputs = {} - inputs.update(obs_dict) + inputs.update(state_dict) inputs.update(act_dict) if mech == "transition": - outputs = next_obs_dict + outputs = next_state_dict elif mech == "reward_mech": rewards = replay_buffer.rewards[: real_buffer_size, 0] rewards_dict = {"reward": torch.from_numpy(rewards[:, None])} - inputs.update(next_obs_dict) + inputs.update(next_state_dict) outputs = rewards_dict elif mech == "termination_mech": terminals = replay_buffer.dones[: real_buffer_size, 0] * (1 - replay_buffer.timeouts[: real_buffer_size, 0]) terminals_dict = {"terminal": torch.from_numpy(terminals[:, None])} - inputs.update(next_obs_dict) + inputs.update(next_state_dict) outputs = terminals_dict else: raise NotImplementedError("support mechs in [transition, reward_mech, termination_mech] only") diff --git a/cmrl/models/dynamics.py b/cmrl/models/dynamics.py index 6ff55c8..2746f05 100644 --- a/cmrl/models/dynamics.py +++ b/cmrl/models/dynamics.py @@ -14,25 +14,27 @@ from cmrl.utils.variables import to_dict_by_space from cmrl.models.causal_mech.base import BaseCausalMech from cmrl.models.data_loader import buffer_to_dict -from cmrl.types import Obs2StateFnType +from cmrl.types import Obs2StateFnType, State2ObsFnType class Dynamics: def __init__( self, transition: BaseCausalMech, - observation_space: spaces.Space, + state_space: spaces.Space, action_space: spaces.Space, obs2state_fn: Obs2StateFnType, + state2obs_fn: State2ObsFnType, reward_mech: Optional[BaseCausalMech] = None, termination_mech: Optional[BaseCausalMech] = None, seed: int = 7, logger: Optional[Logger] = None, ): self.transition = transition - self.observation_space = observation_space + self.state_space = state_space self.action_space = action_space self.obs2state_fn = obs2state_fn + self.state2obs_fn = state2obs_fn self.reward_mech = reward_mech self.termination_mech = termination_mech self.seed = seed @@ -47,7 +49,7 @@ def __init__( def learn(self, real_replay_buffer: ReplayBuffer, work_dir: Optional[Union[str, pathlib.Path]] = None, **kwargs): get_dataset = partial( buffer_to_dict, - observation_space=self.observation_space, + state_space=self.state_space, action_space=self.action_space, obs2state_fn=self.obs2state_fn, replay_buffer=real_replay_buffer, @@ -64,16 +66,16 @@ def learn(self, real_replay_buffer: ReplayBuffer, work_dir: Optional[Union[str, def step(self, batch_obs, batch_action): with torch.no_grad(): - obs_dict = to_dict_by_space(batch_obs, self.observation_space, "obs", repeat=7, to_tensor=True) + obs_dict = to_dict_by_space(batch_obs, self.state_space, "obs", repeat=7, to_tensor=True) act_dict = to_dict_by_space(batch_action, self.action_space, "act", repeat=7, to_tensor=True) inputs = ChainMap(obs_dict, act_dict) outputs = self.transition.forward(inputs) + batch_next_state = torch.concat([tensor.mean(dim=0)[:, :1] for tensor in outputs.values()], + dim=-1).cpu().numpy() + batch_next_obs = self.state2obs_fn(batch_next_state) info = { "origin-next_obs": torch.concat([tensor[:, :, :1] for tensor in outputs.values()], dim=-1).cpu().numpy()} - return torch.concat([tensor.mean(dim=0)[:, :1] for tensor in outputs.values()], - dim=-1).cpu().numpy(), None, None, info - - # def set_oracle_graph(self, graph): + return batch_next_obs, None, None, info diff --git a/cmrl/types.py b/cmrl/types.py index 0508827..07a1c4b 100644 --- a/cmrl/types.py +++ b/cmrl/types.py @@ -8,3 +8,4 @@ TermFnType = Callable[[torch.Tensor, torch.Tensor, torch.Tensor], torch.Tensor] InitObsFnType = Callable[[int], torch.Tensor] Obs2StateFnType = Callable[[torch.Tensor, torch.Tensor], torch.Tensor] +State2ObsFnType = Callable[[torch.Tensor], torch.Tensor] diff --git a/cmrl/utils/RCIT.py b/cmrl/utils/RCIT.py new file mode 100644 index 0000000..76c6c23 --- /dev/null +++ b/cmrl/utils/RCIT.py @@ -0,0 +1,643 @@ +import numpy as np +from numpy import sqrt +from numpy.linalg import eigh, eigvalsh +from scipy import stats +from sklearn.gaussian_process import GaussianProcessRegressor +from sklearn.gaussian_process.kernels import RBF +from sklearn.gaussian_process.kernels import ConstantKernel as C +from sklearn.gaussian_process.kernels import WhiteKernel + +from causallearn.utils.KCI.GaussianKernel import GaussianKernel +from causallearn.utils.KCI.Kernel import Kernel +from causallearn.utils.KCI.LinearKernel import LinearKernel +from causallearn.utils.KCI.PolynomialKernel import PolynomialKernel +import random +import math +import time +from numpy.linalg import inv + +##################### For Random Feature ##################### +try: + import rpy2 + import rpy2.robjects + + rpy2.robjects.r['options'](warn=-1) + from rpy2.robjects.packages import importr + import rpy2.robjects.numpy2ri + + rpy2.robjects.numpy2ri.activate() +except: + print("Could not import rpy package") + +try: + importr('RCIT') +except: + print("Could not import r-package RCIT") +import random + + +def set_random_seed(seed): + random.seed(seed) + np.random.seed(seed) + + +############################################################### + + +# Cannot find reference 'xxx' in '__init__.pyi | __init__.pyi | __init__.pxd' is a bug in pycharm, please ignore +class KCI_UInd(object): + """ + Python implementation of Kernel-based Conditional Independence (KCI) test. Unconditional version. + The original Matlab implementation can be found in http://people.tuebingen.mpg.de/kzhang/KCI-test.zip + + References + ---------- + [1] K. Zhang, J. Peters, D. Janzing, and B. Schölkopf, + "A kernel-based conditional independence test and application in causal discovery," In UAI 2011. + [2] A. Gretton, K. Fukumizu, C.-H. Teo, L. Song, B. Schölkopf, and A. Smola, "A kernel + Statistical test of independence." In NIPS 21, 2007. + """ + + def __init__(self, kernelX='Gaussian', kernelY='Gaussian', null_ss=1000, approx=True, est_width='empirical', + polyd=2, kwidthx=None, kwidthy=None): + """ + Construct the KCI_UInd model. + + Parameters + ---------- + kernelX: kernel function for input data x + 'Gaussian': Gaussian kernel + 'Polynomial': Polynomial kernel + 'Linear': Linear kernel + kernelY: kernel function for input data y + est_width: set kernel width for Gaussian kernels + 'empirical': set kernel width using empirical rules + 'median': set kernel width using the median trick + 'manual': set by users + null_ss: sample size in simulating the null distribution + approx: whether to use gamma approximation (default=True) + polyd: polynomial kernel degrees (default=1) + kwidthx: kernel width for data x (standard deviation sigma) + kwidthy: kernel width for data y (standard deviation sigma) + """ + + self.kernelX = kernelX + self.kernelY = kernelY + self.est_width = est_width + self.polyd = polyd + self.kwidthx = kwidthx + self.kwidthy = kwidthy + self.nullss = null_ss + self.thresh = 1e-6 + self.approx = approx + + def compute_pvalue(self, data_x=None, data_y=None): + """ + Main function: compute the p value and return it together with the test statistic + + Parameters + ---------- + data_x: input data for x (nxd1 array) + data_y: input data for y (nxd2 array) + + Returns + _________ + pvalue: p value (scalar) + test_stat: test statistic (scalar) + + [Notes for speedup optimization] + Kx, Ky are both symmetric with diagonals equal to 1 (no matter what the kernel is) + Kxc, Kyc are both symmetric + """ + + Kx, Ky = self.kernel_matrix(data_x, data_y) + test_stat, Kxc, Kyc = self.HSIC_V_statistic(Kx, Ky) + + if self.approx: + k_appr, theta_appr = self.get_kappa(Kxc, Kyc) + pvalue = 1 - stats.gamma.cdf(test_stat, k_appr, 0, theta_appr) + else: + null_dstr = self.null_sample_spectral(Kxc, Kyc) + pvalue = sum(null_dstr.squeeze() > test_stat) / float(self.nullss) + return pvalue, test_stat + + def compute_pvalue_rf(self, data_x=None, data_y=None): + rit = rpy2.robjects.r['RIT'](data_x, data_y, approx="lpd4", seed=42) + sta = float(rit.rx2('Sta')[0]) + pval = float(rit.rx2('p')[0]) + return pval, sta + + def kernel_matrix(self, data_x, data_y): + """ + Compute kernel matrix for data x and data y + + Parameters + ---------- + data_x: input data for x (nxd1 array) + data_y: input data for y (nxd2 array) + + Returns + _________ + Kx: kernel matrix for data_x (nxn) + Ky: kernel matrix for data_y (nxn) + """ + if self.kernelX == 'Gaussian': + if self.est_width == 'manual': + if self.kwidthx is not None: + kernelX = GaussianKernel(self.kwidthx) + else: + raise Exception('specify kwidthx') + else: + kernelX = GaussianKernel() + if self.est_width == 'median': + kernelX.set_width_median(data_x) + elif self.est_width == 'empirical': + kernelX.set_width_empirical_hsic(data_x) + else: + raise Exception('Undefined kernel width estimation method') + elif self.kernelX == 'Polynomial': + kernelX = PolynomialKernel(self.polyd) + elif self.kernelX == 'Linear': + kernelX = LinearKernel() + else: + raise Exception('Undefined kernel function') + + if self.kernelY == 'Gaussian': + if self.est_width == 'manual': + if self.kwidthy is not None: + kernelY = GaussianKernel(self.kwidthy) + else: + raise Exception('specify kwidthy') + else: + kernelY = GaussianKernel() + if self.est_width == 'median': + kernelY.set_width_median(data_y) + elif self.est_width == 'empirical': + kernelY.set_width_empirical_hsic(data_y) + else: + raise Exception('Undefined kernel width estimation method') + elif self.kernelY == 'Polynomial': + kernelY = PolynomialKernel(self.polyd) + elif self.kernelY == 'Linear': + kernelY = LinearKernel() + else: + raise Exception('Undefined kernel function') + + data_x = stats.zscore(data_x, ddof=1, axis=0) + data_x[np.isnan(data_x)] = 0. # in case some dim of data_x is constant + data_y = stats.zscore(data_y, ddof=1, axis=0) + data_y[np.isnan(data_y)] = 0. + # We set 'ddof=1' to conform to the normalization way in the original Matlab implementation in + # http://people.tuebingen.mpg.de/kzhang/KCI-test.zip + + Kx = kernelX.kernel(data_x) + Ky = kernelY.kernel(data_y) + return Kx, Ky + + def HSIC_V_statistic(self, Kx, Ky): + """ + Compute V test statistic from kernel matrices Kx and Ky + Parameters + ---------- + Kx: kernel matrix for data_x (nxn) + Ky: kernel matrix for data_y (nxn) + + Returns + _________ + Vstat: HSIC v statistics + Kxc: centralized kernel matrix for data_x (nxn) + Kyc: centralized kernel matrix for data_y (nxn) + """ + Kxc = Kernel.center_kernel_matrix(Kx) + Kyc = Kernel.center_kernel_matrix(Ky) + V_stat = np.sum(Kxc * Kyc) + return V_stat, Kxc, Kyc + + def null_sample_spectral(self, Kxc, Kyc): + """ + Simulate data from null distribution + + Parameters + ---------- + Kxc: centralized kernel matrix for data_x (nxn) + Kyc: centralized kernel matrix for data_y (nxn) + + Returns + _________ + null_dstr: samples from the null distribution + + """ + T = Kxc.shape[0] + if T > 1000: + num_eig = np.int(np.floor(T / 2)) + else: + num_eig = T + lambdax = eigvalsh(Kxc) + lambday = eigvalsh(Kyc) + lambdax = -np.sort(-lambdax) + lambday = -np.sort(-lambday) + lambdax = lambdax[0:num_eig] + lambday = lambday[0:num_eig] + lambda_prod = np.dot(lambdax.reshape(num_eig, 1), lambday.reshape(1, num_eig)).reshape( + (num_eig ** 2, 1)) + lambda_prod = lambda_prod[lambda_prod > lambda_prod.max() * self.thresh] + f_rand = np.random.chisquare(1, (lambda_prod.shape[0], self.nullss)) + null_dstr = lambda_prod.T.dot(f_rand) / T + return null_dstr + + def get_kappa(self, Kx, Ky): + """ + Get parameters for the approximated gamma distribution + Parameters + ---------- + Kx: kernel matrix for data_x (nxn) + Ky: kernel matrix for data_y (nxn) + + Returns + _________ + k_appr, theta_appr: approximated parameters of the gamma distribution + + [Updated @Haoyue 06/24/2022] + equivalent to: + var_appr = 2 * np.trace(Kx.dot(Kx)) * np.trace(Ky.dot(Ky)) / T / T + based on the fact that: + np.trace(K.dot(K)) == np.sum(K * K.T), where here K is symmetric + we can save time on the dot product by only considering the diagonal entries of K.dot(K) + time complexity is reduced from O(n^3) (matrix dot) to O(n^2) (traverse each element), + where n is usually big (sample size). + """ + T = Kx.shape[0] + mean_appr = np.trace(Kx) * np.trace(Ky) / T + var_appr = 2 * np.sum(Kx ** 2) * np.sum(Ky ** 2) / T / T # same as np.sum(Kx * Kx.T) ..., here Kx is symmetric + k_appr = mean_appr ** 2 / var_appr + theta_appr = var_appr / mean_appr + return k_appr, theta_appr + + +class KCI_CInd(object): + """ + Python implementation of Kernel-based Conditional Independence (KCI) test. Conditional version. + The original Matlab implementation can be found in http://people.tuebingen.mpg.de/kzhang/KCI-test.zip + + References + ---------- + [1] K. Zhang, J. Peters, D. Janzing, and B. Schölkopf, "A kernel-based conditional independence test and application in causal discovery," In UAI 2011. + """ + + def __init__(self, kernelX='Gaussian', kernelY='Gaussian', kernelZ='Gaussian', nullss=5000, est_width='empirical', + use_gp=False, approx=True, polyd=2, kwidthx=None, kwidthy=None, kwidthz=None): + """ + Construct the KCI_CInd model. + Parameters + ---------- + kernelX: kernel function for input data x + 'Gaussian': Gaussian kernel + 'Polynomial': Polynomial kernel + 'Linear': Linear kernel + kernelY: kernel function for input data y + kernelZ: kernel function for input data z (conditional variable) + est_width: set kernel width for Gaussian kernels + 'empirical': set kernel width using empirical rules + 'median': set kernel width using the median trick + 'manual': set by users + null_ss: sample size in simulating the null distribution + use_gp: whether use gaussian process to determine kernel width for z + approx: whether to use gamma approximation (default=True) + polyd: polynomial kernel degrees (default=1) + kwidthx: kernel width for data x (standard deviation sigma, default None) + kwidthy: kernel width for data y (standard deviation sigma) + kwidthz: kernel width for data z (standard deviation sigma) + """ + self.kernelX = kernelX + self.kernelY = kernelY + self.kernelZ = kernelZ + self.est_width = est_width + self.polyd = polyd + self.kwidthx = kwidthx + self.kwidthy = kwidthy + self.kwidthz = kwidthz + self.nullss = nullss + self.epsilon_x = 1e-3 # To conform to the original Matlab implementation. + self.epsilon_y = 1e-3 + self.use_gp = use_gp + self.thresh = 1e-5 + self.approx = approx + + def compute_pvalue_rf(self, data_x=None, data_y=None, data_z=None): + rit = rpy2.robjects.r['RCIT'](data_x, data_y, data_z, num_f=10000, num_f2=200, approx="lpd4", seed=42) + sta = float(rit.rx2('Sta')[0]) + pval = float(rit.rx2('p')[0]) + print(pval) + return pval, sta + + def compute_pvalue(self, data_x=None, data_y=None, data_z=None): + """ + Main function: compute the p value and return it together with the test statistic + Parameters + ---------- + data_x: input data for x (nxd1 array) + data_y: input data for y (nxd2 array) + data_z: input data for z (nxd3 array) + + Returns + _________ + pvalue: p value + test_stat: test statistic + """ + Kx, Ky, Kzx, Kzy = self.kernel_matrix(data_x, data_y, data_z) + test_stat, KxR, KyR = self.KCI_V_statistic(Kx, Ky, Kzx, Kzy) + uu_prod, size_u = self.get_uuprod(KxR, KyR) + if self.approx: + k_appr, theta_appr = self.get_kappa(uu_prod) + pvalue = 1 - stats.gamma.cdf(test_stat, k_appr, 0, theta_appr) + else: + null_samples = self.null_sample_spectral(uu_prod, size_u, Kx.shape[0]) + pvalue = sum(null_samples > test_stat) / float(self.nullss) + return pvalue, test_stat + + def kernel_matrix(self, data_x, data_y, data_z): + """ + Compute kernel matrix for data x, data y, and data_z + Parameters + ---------- + data_x: input data for x (nxd1 array) + data_y: input data for y (nxd2 array) + data_z: input data for z (nxd3 array) + + Returns + _________ + Kx: kernel matrix for data_x (nxn) + Ky: kernel matrix for data_y (nxn) + Kzx: centering kernel matrix for data_x (nxn) + kzy: centering kernel matrix for data_y (nxn) + """ + # normalize the data + data_x = stats.zscore(data_x, ddof=1, axis=0) + data_x[np.isnan(data_x)] = 0. + + data_y = stats.zscore(data_y, ddof=1, axis=0) + data_y[np.isnan(data_y)] = 0. + + data_z = stats.zscore(data_z, ddof=1, axis=0) + data_z[np.isnan(data_z)] = 0. + # We set 'ddof=1' to conform to the normalization way in the original Matlab implementation in + # http://people.tuebingen.mpg.de/kzhang/KCI-test.zip + + # concatenate x and z + data_x = np.concatenate((data_x, 0.5 * data_z), axis=1) + if self.kernelX == 'Gaussian': + if self.est_width == 'manual': + if self.kwidthx is not None: + kernelX = GaussianKernel(self.kwidthx) + else: + raise Exception('specify kwidthx') + else: + kernelX = GaussianKernel() + if self.est_width == 'median': + kernelX.set_width_median(data_x) + elif self.est_width == 'empirical': + # kernelX's empirical width is determined by data_z's shape, please refer to the original code + # (http://people.tuebingen.mpg.de/kzhang/KCI-test.zip) in the file + # 'algorithms/CInd_test_new_withGP.m', Line 37 to 52. + kernelX.set_width_empirical_kci(data_z) + else: + raise Exception('Undefined kernel width estimation method') + elif self.kernelX == 'Polynomial': + kernelX = PolynomialKernel(self.polyd) + elif self.kernelX == 'Linear': + kernelX = LinearKernel() + else: + raise Exception('Undefined kernel function') + + if self.kernelY == 'Gaussian': + if self.est_width == 'manual': + if self.kwidthy is not None: + kernelY = GaussianKernel(self.kwidthy) + else: + raise Exception('specify kwidthy') + else: + kernelY = GaussianKernel() + if self.est_width == 'median': + kernelY.set_width_median(data_y) + elif self.est_width == 'empirical': + # kernelY's empirical width is determined by data_z's shape, please refer to the original code + # (http://people.tuebingen.mpg.de/kzhang/KCI-test.zip) in the file + # 'algorithms/CInd_test_new_withGP.m', Line 37 to 52. + kernelY.set_width_empirical_kci(data_z) + else: + raise Exception('Undefined kernel width estimation method') + elif self.kernelY == 'Polynomial': + kernelY = PolynomialKernel(self.polyd) + elif self.kernelY == 'Linear': + kernelY = LinearKernel() + else: + raise Exception('Undefined kernel function') + + Kx = kernelX.kernel(data_x) + Ky = kernelY.kernel(data_y) + + # centering kernel matrix + Kx = Kernel.center_kernel_matrix(Kx) + Ky = Kernel.center_kernel_matrix(Ky) + + if self.kernelZ == 'Gaussian': + if not self.use_gp: + if self.est_width == 'manual': + if self.kwidthz is not None: + kernelZ = GaussianKernel(self.kwidthz) + else: + raise Exception('specify kwidthz') + else: + kernelZ = GaussianKernel() + if self.est_width == 'median': + kernelZ.set_width_median(data_z) + elif self.est_width == 'empirical': + kernelZ.set_width_empirical_kci(data_z) + Kzx = kernelZ.kernel(data_z) + Kzx = Kernel.center_kernel_matrix(Kzx) + # centering kernel matrix to conform with the original Matlab implementation, + # specifically, Line 100 in the file 'algorithms/CInd_test_new_withGP.m' + Kzy = Kzx + else: + # learning the kernel width of Kz using Gaussian process + n, Dz = data_z.shape + if self.kernelX == 'Gaussian': + widthz = sqrt(1.0 / (kernelX.width * data_x.shape[1])) + else: + widthz = 1.0 + # Instantiate a Gaussian Process model for x + wx, vx = eigh(0.5 * (Kx + Kx.T)) + topkx = int(np.min((400, np.floor(n / 4)))) + idx = np.argsort(-wx) + wx = wx[idx] + vx = vx[:, idx] + wx = wx[0:topkx] + vx = vx[:, 0:topkx] + vx = vx[:, wx > wx.max() * self.thresh] + wx = wx[wx > wx.max() * self.thresh] + vx = 2 * sqrt(n) * vx.dot(np.diag(np.sqrt(wx))) / sqrt(wx[0]) + kernelx = C(1.0, (1e-3, 1e3)) * RBF(widthz * np.ones(Dz), (1e-2, 1e2)) + WhiteKernel(0.1, (1e-10, 1e+1)) + gpx = GaussianProcessRegressor(kernel=kernelx) + # fit Gaussian process, including hyperparameter optimization + gpx.fit(data_z, vx) + + # construct Gaussian kernels according to learned hyperparameters + Kzx = gpx.kernel_.k1(data_z, data_z) + self.epsilon_x = np.exp(gpx.kernel_.theta[-1]) + + # Instantiate a Gaussian Process model for y + wy, vy = eigh(0.5 * (Ky + Ky.T)) + topky = int(np.min((400, np.floor(n / 4)))) + idy = np.argsort(-wy) + wy = wy[idy] + vy = vy[:, idy] + wy = wy[0:topky] + vy = vy[:, 0:topky] + vy = vy[:, wy > wy.max() * self.thresh] + wy = wy[wy > wy.max() * self.thresh] + vy = 2 * sqrt(n) * vy.dot(np.diag(np.sqrt(wy))) / sqrt(wy[0]) + kernely = C(1.0, (1e-3, 1e3)) * RBF(widthz * np.ones(Dz), (1e-2, 1e2)) + WhiteKernel(0.1, (1e-10, 1e+1)) + gpy = GaussianProcessRegressor(kernel=kernely) + # fit Gaussian process, including hyperparameter optimization + gpy.fit(data_z, vy) + + # construct Gaussian kernels according to learned hyperparameters + Kzy = gpy.kernel_.k1(data_z, data_z) + self.epsilon_y = np.exp(gpy.kernel_.theta[-1]) + elif self.kernelZ == 'Polynomial': + kernelZ = PolynomialKernel(self.polyd) + Kzx = kernelZ.kernel(data_z) + Kzx = Kernel.center_kernel_matrix(Kzx) + Kzy = Kzx + elif self.kernelZ == 'Linear': + kernelZ = LinearKernel() + Kzx = kernelZ.kernel(data_z) + Kzx = Kernel.center_kernel_matrix(Kzx) + Kzy = Kzx + else: + raise Exception('Undefined kernel function') + return Kx, Ky, Kzx, Kzy + + def KCI_V_statistic(self, Kx, Ky, Kzx, Kzy): + """ + Compute V test statistic from kernel matrices Kx and Ky + Parameters + ---------- + Kx: kernel matrix for data_x (nxn) + Ky: kernel matrix for data_y (nxn) + Kzx: centering kernel matrix for data_x (nxn) + kzy: centering kernel matrix for data_y (nxn) + + Returns + _________ + Vstat: KCI v statistics + KxR: centralized kernel matrix for data_x (nxn) + KyR: centralized kernel matrix for data_y (nxn) + + [Updated @Haoyue 06/24/2022] + 1. Kx, Ky, Kzx, Kzy are all symmetric matrices. + - * Kx's diagonal elements are not the same, because the kernel Kx is centered. + * Before centering, Kx's all diagonal elements are 1 (because of exp(-0.5 * sq_dists * self.width)). + * The same applies to Ky. + - * If (self.kernelZ == 'Gaussian' and self.use_gp), then Kzx has all the same diagonal elements (not necessarily 1). + * The same applies to Kzy. + 2. If not (self.kernelZ == 'Gaussian' and self.use_gp): assert (Kzx == Kzy).all() + With this we could save one repeated calculation of pinv(Kzy+\epsilonI), which consumes most time. + """ + KxR, Rzx = Kernel.center_kernel_matrix_regression(Kx, Kzx, self.epsilon_x) + if self.epsilon_x != self.epsilon_y or (self.kernelZ == 'Gaussian' and self.use_gp): + KyR, _ = Kernel.center_kernel_matrix_regression(Ky, Kzy, self.epsilon_y) + else: + # assert np.all(Kzx == Kzy), 'Kzx and Kzy are the same' + KyR = Rzx.dot(Ky.dot(Rzx)) + Vstat = np.sum(KxR * KyR) + return Vstat, KxR, KyR + + def get_uuprod(self, Kx, Ky): + """ + Compute eigenvalues for null distribution estimation + + Parameters + ---------- + Kx: centralized kernel matrix for data_x (nxn) + Ky: centralized kernel matrix for data_y (nxn) + + Returns + _________ + uu_prod: product of the eigenvectors of Kx and Ky + size_u: number of producted eigenvectors + + """ + wx, vx = eigh(0.5 * (Kx + Kx.T)) + wy, vy = eigh(0.5 * (Ky + Ky.T)) + idx = np.argsort(-wx) + idy = np.argsort(-wy) + wx = wx[idx] + vx = vx[:, idx] + wy = wy[idy] + vy = vy[:, idy] + vx = vx[:, wx > np.max(wx) * self.thresh] + wx = wx[wx > np.max(wx) * self.thresh] + vy = vy[:, wy > np.max(wy) * self.thresh] + wy = wy[wy > np.max(wy) * self.thresh] + vx = vx.dot(np.diag(np.sqrt(wx))) + vy = vy.dot(np.diag(np.sqrt(wy))) + + # calculate their product + T = Kx.shape[0] + num_eigx = vx.shape[1] + num_eigy = vy.shape[1] + size_u = num_eigx * num_eigy + uu = np.zeros((T, size_u)) + for i in range(0, num_eigx): + for j in range(0, num_eigy): + uu[:, i * num_eigy + j] = vx[:, i] * vy[:, j] + + if size_u > T: + uu_prod = uu.dot(uu.T) + else: + uu_prod = uu.T.dot(uu) + + return uu_prod, size_u + + def null_sample_spectral(self, uu_prod, size_u, T): + """ + Simulate data from null distribution + + Parameters + ---------- + uu_prod: product of the eigenvectors of Kx and Ky + size_u: number of producted eigenvectors + T: sample size + + Returns + _________ + null_dstr: samples from the null distribution + + """ + eig_uu = eigvalsh(uu_prod) + eig_uu = -np.sort(-eig_uu) + eig_uu = eig_uu[0:np.min((T, size_u))] + eig_uu = eig_uu[eig_uu > np.max(eig_uu) * self.thresh] + + f_rand = np.random.chisquare(1, (eig_uu.shape[0], self.nullss)) + null_dstr = eig_uu.T.dot(f_rand) + return null_dstr + + def get_kappa(self, uu_prod): + """ + Get parameters for the approximated gamma distribution + Parameters + ---------- + uu_prod: product of the eigenvectors of Kx and Ky + + Returns + ---------- + k_appr, theta_appr: approximated parameters of the gamma distribution + + """ + mean_appr = np.trace(uu_prod) + var_appr = 2 * np.trace(uu_prod.dot(uu_prod)) + k_appr = mean_appr ** 2 / var_appr + theta_appr = var_appr / mean_appr + return k_appr, theta_appr diff --git a/cmrl/utils/creator.py b/cmrl/utils/creator.py index 7af9402..8b71dd8 100644 --- a/cmrl/utils/creator.py +++ b/cmrl/utils/creator.py @@ -8,7 +8,7 @@ from stable_baselines3.common.logger import Logger from stable_baselines3.common.base_class import BaseAlgorithm -from cmrl.types import Obs2StateFnType +from cmrl.types import Obs2StateFnType, State2ObsFnType from cmrl.models.dynamics import Dynamics from cmrl.models.fake_env import VecFakeEnv from cmrl.models.causal_mech.base import BaseCausalMech @@ -25,14 +25,16 @@ def create_agent(cfg: DictConfig, fake_env: VecFakeEnv, logger: Optional[Logger] def create_dynamics( cfg: DictConfig, - observation_space: spaces.Space, + state_space: spaces.Space, action_space: spaces.Space, obs2state_fn: Obs2StateFnType, + state2obs_fn: State2ObsFnType, logger: Optional[Logger] = None, ): - obs_variables = parse_space(observation_space, "obs") - act_variables = parse_space(action_space, "act") - next_obs_variables = parse_space(observation_space, "next_obs") + extra_info = cfg.task.get("extra_variable_info", {}) + obs_variables = parse_space(state_space, "obs", extra_info=extra_info) + act_variables = parse_space(action_space, "act", extra_info=extra_info) + next_obs_variables = parse_space(state_space, "next_obs", extra_info=extra_info) # transition assert cfg.transition.learn, "transition must be learned, or you should try model-free RL:)" @@ -71,9 +73,10 @@ def create_dynamics( transition=transition, reward_mech=reward_mech, termination_mech=termination_mech, - observation_space=observation_space, + state_space=state_space, action_space=action_space, obs2state_fn=obs2state_fn, + state2obs_fn=state2obs_fn, logger=logger, ) diff --git a/cmrl/utils/env.py b/cmrl/utils/env.py index bb12805..f5d68a0 100644 --- a/cmrl/utils/env.py +++ b/cmrl/utils/env.py @@ -12,19 +12,21 @@ def make_env( cfg: omegaconf.DictConfig, -) -> Tuple[emei.EmeiEnv, TermFnType, Optional[RewardFnType], Optional[InitObsFnType],Optional[Obs2StateFnType],]: +) -> Tuple[emei.EmeiEnv, tuple]: env = cast(emei.EmeiEnv, gym.make(cfg.task.env_id, **cfg.task.params)) - - reward_fn = env.get_batch_reward - term_fn = env.get_batch_terminal - init_obs_fn = env.get_batch_init_obs - obs2state_fn = env.obs2state + fns = ( + env.get_batch_reward, + env.get_batch_terminal, + env.get_batch_init_obs, + env.obs2state, + env.state2obs + ) # set seed env.reset(seed=cfg.seed) - env.observation_space.seed(cfg.seed + 1) + env.state_space.seed(cfg.seed + 1) env.action_space.seed(cfg.seed + 2) - return env, reward_fn, term_fn, init_obs_fn, obs2state_fn + return env, fns def load_offline_data(env, replay_buffer: ReplayBuffer, dataset_name: str, use_ratio: float = 1): diff --git a/cmrl/utils/variables.py b/cmrl/utils/variables.py index 40196d3..97b9c2b 100644 --- a/cmrl/utils/variables.py +++ b/cmrl/utils/variables.py @@ -34,14 +34,21 @@ class DiscreteVariable(Variable): n: int -def parse_space(space: spaces.Space, prefix="obs") -> List[Variable]: +def parse_space( + space: spaces.Space, + prefix="obs", + extra_info=None +) -> List[Variable]: + extra_info = extra_info if extra_info is not None else {} + variables = [] if isinstance(space, spaces.Box): for i, (low, high) in enumerate(zip(space.low, space.high)): - if np.isclose(low, -np.pi) and np.isclose(high, np.pi): - variables.append(RadianVariable(dim=1, name="{}_{}".format(prefix, i))) + name = "{}_{}".format(prefix, i) + if "Radian" in extra_info and name in extra_info["Radian"]: + variables.append(RadianVariable(dim=1, name=name)) else: - variables.append(ContinuousVariable(dim=1, low=low, high=high, name="{}_{}".format(prefix, i))) + variables.append(ContinuousVariable(dim=1, low=low, high=high, name=name)) elif isinstance(space, spaces.Discrete): variables.append(DiscreteVariable(n=space.n, name="{}_0".format(prefix))) elif isinstance(space, spaces.MultiDiscrete): @@ -58,11 +65,11 @@ def parse_space(space: spaces.Space, prefix="obs") -> List[Variable]: def to_dict_by_space( - data: np.ndarray, - space: spaces.Space, - prefix="obs", - repeat: Optional[int] = None, - to_tensor: bool = False, + data: np.ndarray, + space: spaces.Space, + prefix="obs", + repeat: Optional[int] = None, + to_tensor: bool = False, ) -> Dict[str, Union[np.ndarray, torch.Tensor]]: """Transform the interaction data from its own type to python's dict, by the signature of space. @@ -101,6 +108,6 @@ def to_dict_by_space( def dict2space( - data: Dict[str, Union[np.ndarray, torch.Tensor]], space: spaces.Space + data: Dict[str, Union[np.ndarray, torch.Tensor]], space: spaces.Space ) -> Dict[str, Union[np.ndarray, torch.Tensor]]: pass diff --git a/requirements/main.txt b/requirements/main.txt index c94a59c..4fbd491 100644 --- a/requirements/main.txt +++ b/requirements/main.txt @@ -8,5 +8,5 @@ imageio>=2.19.0 tensorboard>=2.9.0 mujoco >= 2.2.0 wandb >= 0.13 -stable-baselines3 @ git+https://github.com/carlosluis/stable-baselines3@fix_tests +stable-baselines3 @ git+https://gitee.com/franktian424/stable-baselines3 causal-learn >= 0.1.3.3 \ No newline at end of file diff --git a/tests/test_sb3_extension/test_online_mb_callback.py b/tests/test_sb3_extension/test_online_mb_callback.py index 95a0a7f..6746f51 100644 --- a/tests/test_sb3_extension/test_online_mb_callback.py +++ b/tests/test_sb3_extension/test_online_mb_callback.py @@ -19,9 +19,9 @@ def test_callback(): termination_fn = env.get_terminal get_init_obs_fn = env.get_batch_init_obs - obs_variables = parse_space(env.observation_space, "obs") + obs_variables = parse_space(env.state_space, "obs") act_variables = parse_space(env.action_space, "act") - next_obs_variables = parse_space(env.observation_space, "next_obs") + next_obs_variables = parse_space(env.state_space, "next_obs") transition = PlainMech( name="transition", @@ -29,14 +29,14 @@ def test_callback(): output_variables=next_obs_variables, ) - dynamics = Dynamics(transition, env.observation_space, env.action_space) + dynamics = Dynamics(transition, env.state_space, env.action_space) real_replay_buffer = ReplayBuffer( - 100, env.observation_space, env.action_space, device="cpu", handle_timeout_termination=False + 100, env.state_space, env.action_space, device="cpu", handle_timeout_termination=False ) fake_env = VecFakeEnv( num_envs=1, - observation_space=env.observation_space, + observation_space=env.state_space, action_space=env.action_space, dynamics=dynamics, reward_fn=reward_fn, From db217ad0646c7d12f43cc1eae965fdf72109b6af Mon Sep 17 00:00:00 2001 From: frank Date: Tue, 21 Mar 2023 01:25:55 +0800 Subject: [PATCH 57/68] :hammer: add state2obs_fn --- cmrl/models/causal_mech/kernel_test.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cmrl/models/causal_mech/kernel_test.py b/cmrl/models/causal_mech/kernel_test.py index c0a0971..733cec7 100644 --- a/cmrl/models/causal_mech/kernel_test.py +++ b/cmrl/models/causal_mech/kernel_test.py @@ -30,6 +30,7 @@ def __init__( longest_epoch: int = -1, improvement_threshold: float = 0.01, patience: int = 5, + batch_size: int = 256, # ensemble ensemble_num: int = 7, elite_num: int = 5, @@ -58,6 +59,7 @@ def __init__( longest_epoch=longest_epoch, improvement_threshold=improvement_threshold, patience=patience, + batch_size=batch_size, ensemble_num=ensemble_num, elite_num=elite_num, network_cfg=network_cfg, From 3060b61fe3ed939c991a6ddc0cf68c87563dcd67 Mon Sep 17 00:00:00 2001 From: frank Date: Tue, 21 Mar 2023 16:13:22 +0800 Subject: [PATCH 58/68] :bug: fix maybe_load_offline_model bug --- cmrl/algorithms/util.py | 13 +++++++------ cmrl/models/causal_mech/kernel_test.py | 9 ++++----- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/cmrl/algorithms/util.py b/cmrl/algorithms/util.py index 9c83afe..fd594f1 100644 --- a/cmrl/algorithms/util.py +++ b/cmrl/algorithms/util.py @@ -32,9 +32,9 @@ def compare_dict(dict1, dict2): def maybe_load_offline_model( - dynamics: Dynamics, - cfg: DictConfig, - work_dir, + dynamics: Dynamics, + cfg: DictConfig, + work_dir, ): work_dir = pathlib.Path(work_dir) if "." not in work_dir.name: # exp by hydra's MULTIRUN mode @@ -57,9 +57,10 @@ def maybe_load_offline_model( exp_transition_dir = OmegaConf.to_container(exp_cfg.transition, resolve=True) if ( - cfg.seed == exp_cfg.seed - and compare_dict(exp_transition_dir, transition_cfg) - and (exp_dir / "transition").exists() + cfg.seed == exp_cfg.seed + and cfg.task.use_ratio == exp_cfg.task.use_ratio + and compare_dict(exp_transition_dir, transition_cfg) + and (exp_dir / "transition").exists() ): dynamics.transition.load(exp_dir / "transition") print("loaded dynamics from {}".format(exp_dir)) diff --git a/cmrl/models/causal_mech/kernel_test.py b/cmrl/models/causal_mech/kernel_test.py index 733cec7..7d3e01d 100644 --- a/cmrl/models/causal_mech/kernel_test.py +++ b/cmrl/models/causal_mech/kernel_test.py @@ -221,7 +221,6 @@ def learn( from torch.utils.data import DataLoader from typing import cast - from cmrl.models.causal_mech.reinforce import ReinforceCausalMech from cmrl.models.data_loader import EnsembleBufferDataset, collate_fn, buffer_to_dict from cmrl.utils.creator import parse_space from cmrl.utils.env import load_offline_data @@ -235,21 +234,21 @@ def unwrap_env(env): return env - env = unwrap_env(gym.make("ContinuousCartPoleSwingUp-v0")) + env = unwrap_env(gym.make("ParallelContinuousCartPoleSwingUp-v0")) real_replay_buffer = ReplayBuffer( int(1e6), env.observation_space, env.action_space, "cpu", handle_timeout_termination=False ) load_offline_data(env, real_replay_buffer, "SAC-expert", use_ratio=1) - # extra_info = {"Radian": ["obs_1", "obs_5", "obs_9"]} - extra_info = {"Radian": ["obs_1"]} + extra_info = {"Radian": ["obs_1", "obs_5", "obs_9"]} + # extra_info = {"Radian": ["obs_1"]} input_variables = parse_space(env.state_space, "obs", extra_info=extra_info) + parse_space(env.action_space, "act") output_variables = parse_space(env.state_space, "next_obs", extra_info=extra_info) logger = logger_configure("kci-log", ["tensorboard", "stdout"]) - mech = KernelTestMech("kernel_test_mech", input_variables, output_variables, sample_num=1000, kci_times=20, + mech = KernelTestMech("kernel_test_mech", input_variables, output_variables, sample_num=100, kci_times=20, logger=logger) inputs, outputs = buffer_to_dict(env.state_space, env.action_space, env.obs2state, real_replay_buffer, "transition") From b519930b41b99d2c99cacf455f08e02b07b6e960 Mon Sep 17 00:00:00 2001 From: frank Date: Tue, 21 Mar 2023 18:06:00 +0800 Subject: [PATCH 59/68] :hammer: plain -> oracle --- cmrl/algorithms/base_algorithm.py | 15 +- cmrl/examples/conf/main.yaml | 6 +- .../reward_mech/{plain.yaml => oracle.yaml} | 4 +- .../{plain.yaml => oracle.yaml} | 4 +- cmrl/examples/conf/transition/CMI_test.yaml | 1 - .../examples/conf/transition/kernel_test.yaml | 1 - .../transition/{plain.yaml => oracle.yaml} | 6 +- cmrl/models/causal_mech/__init__.py | 2 +- cmrl/models/causal_mech/base.py | 129 +++++++++--------- .../{plain_mech.py => oracle_mech.py} | 68 +++++---- cmrl/models/data_loader.py | 13 +- cmrl/models/dynamics.py | 1 + cmrl/models/fake_env.py | 4 +- cmrl/utils/variables.py | 5 +- .../test_causal_mech/test_plain_mech.py | 6 +- .../test_online_mb_callback.py | 4 +- 16 files changed, 139 insertions(+), 130 deletions(-) rename cmrl/examples/conf/reward_mech/{plain.yaml => oracle.yaml} (94%) rename cmrl/examples/conf/termination_mech/{plain.yaml => oracle.yaml} (93%) rename cmrl/examples/conf/transition/{plain.yaml => oracle.yaml} (93%) rename cmrl/models/causal_mech/{plain_mech.py => oracle_mech.py} (62%) diff --git a/cmrl/algorithms/base_algorithm.py b/cmrl/algorithms/base_algorithm.py index 8d99f0d..4412ab3 100644 --- a/cmrl/algorithms/base_algorithm.py +++ b/cmrl/algorithms/base_algorithm.py @@ -50,12 +50,15 @@ def __init__( self.state2obs_fn, logger=self.logger) - if not self.cfg.transition.discovery: - self.dynamics.transition.set_oracle_graph(self.env.get_transition_graph()) - if self.cfg.reward_mech.learn and not self.cfg.reward_mech.discovery: - self.dynamics.reward_mech.set_oracle_graph(self.env.get_reward_mech_graph()) - if self.cfg.termination_mech.learn and not self.cfg.termination_mech.discovery: - self.dynamics.termination_mech.set_oracle_graph(self.env.get_termination_mech_graph()) + if self.cfg.transition.name == "oracle_transition": + graph = self.env.get_transition_graph() if self.cfg.transition.oracle == "truth" else None + self.dynamics.transition.set_oracle_graph(graph) + if self.cfg.reward_mech.learn and not self.cfg.reward_mech.name == "oracle_reward_mech": + graph = self.env.get_reward_mech_graph() if self.cfg.transition.oracle == "truth" else None + self.dynamics.reward_mech.set_oracle_graph(graph) + if self.cfg.termination_mech.learn and not self.cfg.termination_mech.name == "oracle_termination_mech": + graph = self.env.get_termination_mech_graph() if self.cfg.transition.oracle == "truth" else None + self.dynamics.termination_mech.set_oracle_graph(graph) # create sb3's replay buffer for real offline data self.real_replay_buffer = ReplayBuffer( diff --git a/cmrl/examples/conf/main.yaml b/cmrl/examples/conf/main.yaml index e55977e..cfc23fd 100644 --- a/cmrl/examples/conf/main.yaml +++ b/cmrl/examples/conf/main.yaml @@ -1,9 +1,9 @@ defaults: - algorithm: off_dyna - task: continuous_cart_pole_swingup - - transition: kernel_test - - reward_mech: plain - - termination_mech: plain + - transition: oracle + - reward_mech: oracle + - termination_mech: oracle - _self_ seed: 0 diff --git a/cmrl/examples/conf/reward_mech/plain.yaml b/cmrl/examples/conf/reward_mech/oracle.yaml similarity index 94% rename from cmrl/examples/conf/reward_mech/plain.yaml rename to cmrl/examples/conf/reward_mech/oracle.yaml index 5658d7c..ffe3f12 100644 --- a/cmrl/examples/conf/reward_mech/plain.yaml +++ b/cmrl/examples/conf/reward_mech/oracle.yaml @@ -1,4 +1,4 @@ -name: "plain_reward_mech" +name: "oracle_reward_mech" learn: false discovery: false @@ -41,7 +41,7 @@ optimizer_cfg: mech: _partial_: true _recursive_: false - _target_: cmrl.models.causal_mech.PlainMech + _target_: cmrl.models.causal_mech.OracleMech # base causal-mech params name: reward_mech input_variables: ??? diff --git a/cmrl/examples/conf/termination_mech/plain.yaml b/cmrl/examples/conf/termination_mech/oracle.yaml similarity index 93% rename from cmrl/examples/conf/termination_mech/plain.yaml rename to cmrl/examples/conf/termination_mech/oracle.yaml index 5d3081b..c6077ce 100644 --- a/cmrl/examples/conf/termination_mech/plain.yaml +++ b/cmrl/examples/conf/termination_mech/oracle.yaml @@ -1,4 +1,4 @@ -name: "plain_termination_mech" +name: "oracle_termination_mech" learn: false discovery: false @@ -41,7 +41,7 @@ optimizer_cfg: mech: _partial_: true _recursive_: false - _target_: cmrl.models.causal_mech.PlainMech + _target_: cmrl.models.causal_mech.OracleMech # base causal-mech params name: termination_mech input_variables: ??? diff --git a/cmrl/examples/conf/transition/CMI_test.yaml b/cmrl/examples/conf/transition/CMI_test.yaml index faf71fd..4461b49 100644 --- a/cmrl/examples/conf/transition/CMI_test.yaml +++ b/cmrl/examples/conf/transition/CMI_test.yaml @@ -1,6 +1,5 @@ name: "CMI_test_transition" learn: true -discovery: true encoder_cfg: _partial_: true diff --git a/cmrl/examples/conf/transition/kernel_test.yaml b/cmrl/examples/conf/transition/kernel_test.yaml index 9dc9c8f..f7d750b 100644 --- a/cmrl/examples/conf/transition/kernel_test.yaml +++ b/cmrl/examples/conf/transition/kernel_test.yaml @@ -1,6 +1,5 @@ name: "kernal_test_transition" learn: true -discovery: true encoder_cfg: _partial_: true diff --git a/cmrl/examples/conf/transition/plain.yaml b/cmrl/examples/conf/transition/oracle.yaml similarity index 93% rename from cmrl/examples/conf/transition/plain.yaml rename to cmrl/examples/conf/transition/oracle.yaml index ad13f77..26d8ea6 100644 --- a/cmrl/examples/conf/transition/plain.yaml +++ b/cmrl/examples/conf/transition/oracle.yaml @@ -1,6 +1,6 @@ -name: "plain_transition" +name: "oracle_transition" learn: true -discovery: true +oracle: "truth" encoder_cfg: _partial_: true @@ -47,7 +47,7 @@ scheduler_cfg: mech: _partial_: true _recursive_: false - _target_: cmrl.models.causal_mech.PlainMech + _target_: cmrl.models.causal_mech.OracleMech # base causal-mech params name: transition input_variables: ??? diff --git a/cmrl/models/causal_mech/__init__.py b/cmrl/models/causal_mech/__init__.py index ffa88a9..dd309df 100644 --- a/cmrl/models/causal_mech/__init__.py +++ b/cmrl/models/causal_mech/__init__.py @@ -1,4 +1,4 @@ -from cmrl.models.causal_mech.plain_mech import PlainMech +from cmrl.models.causal_mech.oracle_mech import OracleMech from cmrl.models.causal_mech.CMI_test import CMITestMech # from cmrl.models.causal_mech.reinforce import ReinforceCausalMech from cmrl.models.causal_mech.kernel_test import KernelTestMech \ No newline at end of file diff --git a/cmrl/models/causal_mech/base.py b/cmrl/models/causal_mech/base.py index 59eca37..4fe5072 100644 --- a/cmrl/models/causal_mech/base.py +++ b/cmrl/models/causal_mech/base.py @@ -31,11 +31,11 @@ class BaseCausalMech(ABC): """ def __init__( - self, - name: str, - input_variables: List[Variable], - output_variables: List[Variable], - logger: Optional[Logger] = None, + self, + name: str, + input_variables: List[Variable], + output_variables: List[Variable], + logger: Optional[Logger] = None, ): self.name = name self.input_variables = input_variables @@ -48,15 +48,14 @@ def __init__( self.input_var_num = len(self.input_variables) self.output_var_num = len(self.output_variables) self.graph: Optional[BaseGraph] = None - self.discovery: bool = True @abstractmethod def learn( - self, - inputs: MutableMapping[str, np.ndarray], - outputs: MutableMapping[str, np.ndarray], - work_dir: Optional[Union[str, pathlib.Path]] = None, - **kwargs + self, + inputs: MutableMapping[str, np.ndarray], + outputs: MutableMapping[str, np.ndarray], + work_dir: Optional[Union[str, pathlib.Path]] = None, + **kwargs ): raise NotImplementedError @@ -68,16 +67,11 @@ def forward(self, inputs: MutableMapping[str, np.ndarray]) -> Dict[str, torch.Te def causal_graph(self) -> torch.Tensor: """property causal graph""" if self.graph is None: - return torch.ones(len(self.input_variables), len(self.output_variables), dtype=torch.int, device=self.device) + return torch.ones(len(self.input_variables), len(self.output_variables), dtype=torch.int, + device=self.device) else: return self.graph.get_binary_adj_matrix() - def set_oracle_graph(self, graph_data): - self.discovery = False - self.graph = BinaryGraph(self.input_var_num, self.output_var_num, device=self.device) - self.graph.set_data(graph_data=graph_data) - print("set oracle causal graph successfully: \n{}".format(graph_data)) - def save(self, save_dir: Union[str, pathlib.Path]): pass @@ -87,31 +81,31 @@ def load(self, load_dir: Union[str, pathlib.Path]): class EnsembleNeuralMech(BaseCausalMech): def __init__( - self, - # base - name: str, - input_variables: List[Variable], - output_variables: List[Variable], - logger: Optional[Logger] = None, - # model learning - longest_epoch: int = -1, - improvement_threshold: float = 0.01, - patience: int = 5, - batch_size: int = 256, - # ensemble - ensemble_num: int = 7, - elite_num: int = 5, - # cfgs - network_cfg: Optional[DictConfig] = None, - encoder_cfg: Optional[DictConfig] = None, - decoder_cfg: Optional[DictConfig] = None, - optimizer_cfg: Optional[DictConfig] = None, - scheduler_cfg: Optional[DictConfig] = None, - # forward method - residual: bool = True, - encoder_reduction: str = "sum", - # others - device: Union[str, torch.device] = "cpu", + self, + # base + name: str, + input_variables: List[Variable], + output_variables: List[Variable], + logger: Optional[Logger] = None, + # model learning + longest_epoch: int = -1, + improvement_threshold: float = 0.01, + patience: int = 5, + batch_size: int = 256, + # ensemble + ensemble_num: int = 7, + elite_num: int = 5, + # cfgs + network_cfg: Optional[DictConfig] = None, + encoder_cfg: Optional[DictConfig] = None, + decoder_cfg: Optional[DictConfig] = None, + optimizer_cfg: Optional[DictConfig] = None, + scheduler_cfg: Optional[DictConfig] = None, + # forward method + residual: bool = True, + encoder_reduction: str = "sum", + # others + device: Union[str, torch.device] = "cpu", ): BaseCausalMech.__init__( self, name=name, input_variables=input_variables, output_variables=output_variables, logger=logger @@ -170,9 +164,9 @@ def build_optimizer(self): assert self.network, "you must build network first" assert self.variable_encoders and self.variable_decoders, "you must build coders first" params = ( - [self.network.parameters()] - + [encoder.parameters() for encoder in self.variable_encoders.values()] - + [decoder.parameters() for decoder in self.variable_decoders.values()] + [self.network.parameters()] + + [encoder.parameters() for encoder in self.variable_encoders.values()] + + [decoder.parameters() for decoder in self.variable_decoders.values()] ) self.optimizer = instantiate(self.optimizer_cfg)(params=chain(*params)) @@ -181,9 +175,10 @@ def build_optimizer(self): def forward(self, inputs: MutableMapping[str, torch.Tensor]) -> Dict[str, torch.Tensor]: batch_size, _ = self.get_inputs_batch_size(inputs) - inputs_tensor = torch.zeros(self.ensemble_num, batch_size, self.input_var_num, self.encoder_output_dim).to(self.device) + inputs_tensor = torch.zeros(self.ensemble_num, batch_size, self.input_var_num, self.encoder_output_dim).to( + self.device) for i, var in enumerate(self.input_variables): - out = self.variable_encoders[var.name](inputs[var.name].to(self.device)) + out = self.variable_encoders[var.name](inputs[var.name]) inputs_tensor[:, :, i] = out output_tensor = self.network(self.reduce_encoder_output(inputs_tensor)) @@ -248,9 +243,9 @@ def get_inputs_info(self, inputs: MutableMapping[str, torch.Tensor]): return batch_size, data_shape[:-3] def residual_outputs( - self, - inputs: MutableMapping[str, torch.Tensor], - outputs: MutableMapping[str, torch.Tensor], + self, + inputs: MutableMapping[str, torch.Tensor], + outputs: MutableMapping[str, torch.Tensor], ) -> MutableMapping[str, torch.Tensor]: for name in filter(lambda s: s.startswith("obs"), inputs.keys()): # assert inputs[name].shape[:2] == outputs["next_{}".format(name)].shape[:2] @@ -260,9 +255,9 @@ def residual_outputs( return outputs def reduce_encoder_output( - self, - encoder_output: torch.Tensor, - mask: Optional[torch.Tensor] = None, + self, + encoder_output: torch.Tensor, + mask: Optional[torch.Tensor] = None, ) -> torch.Tensor: assert len(encoder_output.shape) == 4, ( "shape of `encoder_output` should be (ensemble-num, batch-size, input-var-num, encoder-output-dim), " @@ -278,7 +273,7 @@ def reduce_encoder_output( # mask shape [..., ensemble-num, batch-size, input-var-num] assert ( - mask.shape[-3:] == encoder_output.shape[:-1] + mask.shape[-3:] == encoder_output.shape[:-1] ), "mask shape should be (..., ensemble-num, batch-size, input-var-num)" # [*mask-extra-dims, ensemble-num, batch-size, input-var-num, encoder-output-dim] @@ -307,9 +302,9 @@ def forward_mask(self) -> torch.Tensor: return self.causal_graph.T def get_data_loaders( - self, - inputs: MutableMapping[str, np.ndarray], - outputs: MutableMapping[str, np.ndarray], + self, + inputs: MutableMapping[str, np.ndarray], + outputs: MutableMapping[str, np.ndarray], ): train_set = EnsembleBufferDataset( inputs=inputs, outputs=outputs, training=True, train_ratio=0.8, ensemble_num=self.ensemble_num, seed=1 @@ -324,11 +319,11 @@ def get_data_loaders( return train_loader, valid_loader def learn( - self, - inputs: MutableMapping[str, np.ndarray], - outputs: MutableMapping[str, np.ndarray], - work_dir: Optional[Union[str, pathlib.Path]] = None, - **kwargs + self, + inputs: MutableMapping[str, np.ndarray], + outputs: MutableMapping[str, np.ndarray], + work_dir: Optional[Union[str, pathlib.Path]] = None, + **kwargs ): train_loader, valid_loader = self.get_data_loaders(inputs, outputs) @@ -382,10 +377,10 @@ def learn( self.save(save_dir=work_dir) def _maybe_get_best_weights( - self, - best_val_loss: torch.Tensor, - val_loss: torch.Tensor, - threshold: float = 0.01, + self, + best_val_loss: torch.Tensor, + val_loss: torch.Tensor, + threshold: float = 0.01, ) -> Optional[Dict]: """Return the current model state dict if the validation score improves. For ensembles, this checks the validation for each ensemble member separately. diff --git a/cmrl/models/causal_mech/plain_mech.py b/cmrl/models/causal_mech/oracle_mech.py similarity index 62% rename from cmrl/models/causal_mech/plain_mech.py rename to cmrl/models/causal_mech/oracle_mech.py index 7cbf77c..78defda 100644 --- a/cmrl/models/causal_mech/plain_mech.py +++ b/cmrl/models/causal_mech/oracle_mech.py @@ -1,5 +1,6 @@ from typing import Optional, List, Dict, Union, MutableMapping +import numpy import torch from torch.utils.data import DataLoader import numpy as np @@ -9,35 +10,36 @@ from cmrl.utils.variables import Variable from cmrl.models.causal_mech.base import EnsembleNeuralMech +from cmrl.models.graphs.binary_graph import BinaryGraph from cmrl.models.data_loader import EnsembleBufferDataset, collate_fn -class PlainMech(EnsembleNeuralMech): +class OracleMech(EnsembleNeuralMech): def __init__( - self, - name: str, - input_variables: List[Variable], - output_variables: List[Variable], - logger: Optional[Logger] = None, - # model learning - longest_epoch: int = -1, - improvement_threshold: float = 0.01, - patience: int = 5, - batch_size: int = 256, - # ensemble - ensemble_num: int = 7, - elite_num: int = 5, - # cfgs - network_cfg: Optional[DictConfig] = None, - encoder_cfg: Optional[DictConfig] = None, - decoder_cfg: Optional[DictConfig] = None, - optimizer_cfg: Optional[DictConfig] = None, - scheduler_cfg: Optional[DictConfig] = None, - # forward method - residual: bool = True, - encoder_reduction: str = "sum", - # others - device: Union[str, torch.device] = "cpu", + self, + name: str, + input_variables: List[Variable], + output_variables: List[Variable], + logger: Optional[Logger] = None, + # model learning + longest_epoch: int = -1, + improvement_threshold: float = 0.01, + patience: int = 5, + batch_size: int = 256, + # ensemble + ensemble_num: int = 7, + elite_num: int = 5, + # cfgs + network_cfg: Optional[DictConfig] = None, + encoder_cfg: Optional[DictConfig] = None, + decoder_cfg: Optional[DictConfig] = None, + optimizer_cfg: Optional[DictConfig] = None, + scheduler_cfg: Optional[DictConfig] = None, + # forward method + residual: bool = True, + encoder_reduction: str = "sum", + # others + device: Union[str, torch.device] = "cpu", ): EnsembleNeuralMech.__init__( self, @@ -61,9 +63,12 @@ def __init__( device=device, ) - @property - def forward_mask(self): - return torch.ones(self.output_var_num, self.input_var_num).to(self.device) + def set_oracle_graph(self, graph_data: Optional[numpy.ndarray]): + self.graph = BinaryGraph(self.input_var_num, self.output_var_num, device=self.device) + if graph_data is None: + graph_data = np.ones([self.input_var_num, self.output_var_num]) + self.graph.set_data(graph_data=graph_data) + print("set oracle causal graph successfully: \n{}".format(graph_data)) if __name__ == "__main__": @@ -78,11 +83,13 @@ def forward_mask(self): from cmrl.models.causal_mech.util import variable_loss_func from cmrl.sb3_extension.logger import configure as logger_configure + def unwrap_env(env): while isinstance(env, gym.Wrapper): env = env.env return env + env = unwrap_env(gym.make("ParallelContinuousCartPoleSwingUp-v0")) real_replay_buffer = ReplayBuffer( int(1e6), env.observation_space, env.action_space, "cpu", handle_timeout_termination=False @@ -94,13 +101,14 @@ def unwrap_env(env): logger = logger_configure("kci-log", ["tensorboard", "stdout"]) - mech = PlainMech( + mech = OracleMech( "plain_mech", input_variables, output_variables, logger=logger, ) - inputs, outputs = buffer_to_dict(env.observation_space, env.action_space, env.obs2state, real_replay_buffer, "transition") + inputs, outputs = buffer_to_dict(env.observation_space, env.action_space, env.obs2state, real_replay_buffer, + "transition") mech.learn(inputs, outputs) diff --git a/cmrl/models/data_loader.py b/cmrl/models/data_loader.py index f9c6cd1..42155b2 100644 --- a/cmrl/models/data_loader.py +++ b/cmrl/models/data_loader.py @@ -15,6 +15,7 @@ def buffer_to_dict( obs2state_fn, replay_buffer: ReplayBuffer, mech: str, + device: str = "cpu" ): assert mech in ["transition", "reward_mech", "termination_mech"] # dict action is not supported by SB3(so not done by cmrl) @@ -26,21 +27,21 @@ def buffer_to_dict( if hasattr(replay_buffer, "extra_obs"): states = obs2state_fn(replay_buffer.observations[: real_buffer_size, 0], - replay_buffer.extra_obs[: real_buffer_size, 0]) + replay_buffer.extra_obs[: real_buffer_size, 0]) else: states = replay_buffer.observations[: real_buffer_size, 0] - state_dict = to_dict_by_space(states, state_space, prefix="obs", ) + state_dict = to_dict_by_space(states, state_space, prefix="obs", to_tensor=True, device=device) act_dict = to_dict_by_space( replay_buffer.actions[: real_buffer_size, 0], action_space, - prefix="act", - ) + prefix="act", to_tensor=True, device=device) + if hasattr(replay_buffer, "next_extra_obs"): next_states = obs2state_fn(replay_buffer.next_observations[: real_buffer_size, 0], - replay_buffer.next_extra_obs[: real_buffer_size, 0]) + replay_buffer.next_extra_obs[: real_buffer_size, 0]) else: next_states = replay_buffer.next_observations[: real_buffer_size, 0] - next_state_dict = to_dict_by_space(next_states, state_space, prefix="next_obs") + next_state_dict = to_dict_by_space(next_states, state_space, prefix="next_obs", to_tensor=True, device=device) inputs = {} inputs.update(state_dict) diff --git a/cmrl/models/dynamics.py b/cmrl/models/dynamics.py index 2746f05..b114ef9 100644 --- a/cmrl/models/dynamics.py +++ b/cmrl/models/dynamics.py @@ -53,6 +53,7 @@ def learn(self, real_replay_buffer: ReplayBuffer, work_dir: Optional[Union[str, action_space=self.action_space, obs2state_fn=self.obs2state_fn, replay_buffer=real_replay_buffer, + device=self.device ) # transition diff --git a/cmrl/models/fake_env.py b/cmrl/models/fake_env.py index da483c2..0e6aefe 100644 --- a/cmrl/models/fake_env.py +++ b/cmrl/models/fake_env.py @@ -153,8 +153,8 @@ def single_reset(self, idx): self._envs_length[idx] = 0 if self.branch_rollout: upper_bound = self.replay_buffer.buffer_size if self.replay_buffer.full else self.replay_buffer.pos - batch_inds = np.random.randint(0, upper_bound) - self._current_batch_obs[idx] = self.replay_buffer.observations[batch_inds, 0] + batch_idxs = np.random.randint(0, upper_bound) + self._current_batch_obs[idx] = self.replay_buffer.observations[batch_idxs, 0] else: assert self.get_init_obs_fn is not None self._current_batch_obs[idx] = self.get_init_obs_fn(1) diff --git a/cmrl/utils/variables.py b/cmrl/utils/variables.py index 97b9c2b..534c316 100644 --- a/cmrl/utils/variables.py +++ b/cmrl/utils/variables.py @@ -70,6 +70,7 @@ def to_dict_by_space( prefix="obs", repeat: Optional[int] = None, to_tensor: bool = False, + device: str = "cpu" ) -> Dict[str, Union[np.ndarray, torch.Tensor]]: """Transform the interaction data from its own type to python's dict, by the signature of space. @@ -79,6 +80,8 @@ def to_dict_by_space( prefix: prefix of the key in dict repeat: copy data in a new dimension to_tensor: transform the data from numpy's ndarray to torch's tensor + device: device + Returns: interaction data organized in dictionary form @@ -102,7 +105,7 @@ def to_dict_by_space( # specific-dim is 1 for the case of spaces.Box dict_data[name] = np.tile(dict_data[name][None, :, :], [repeat, 1, 1]) if to_tensor: - dict_data[name] = torch.from_numpy(dict_data[name]) + dict_data[name] = torch.from_numpy(dict_data[name]).to(device) return dict_data diff --git a/tests/test_models/test_causal_mech/test_plain_mech.py b/tests/test_models/test_causal_mech/test_plain_mech.py index 52f0502..dad59c3 100644 --- a/tests/test_models/test_causal_mech/test_plain_mech.py +++ b/tests/test_models/test_causal_mech/test_plain_mech.py @@ -2,7 +2,7 @@ from stable_baselines3.common.buffers import ReplayBuffer from torch.utils.data import DataLoader -from cmrl.models.causal_mech.plain_mech import PlainMech +from cmrl.models.causal_mech.oracle_mech import OracleMech from cmrl.models.data_loader import EnsembleBufferDataset, EnsembleBufferDataset, collate_fn from cmrl.utils.creator import parse_space from cmrl.utils.env import load_offline_data @@ -40,7 +40,7 @@ def prepare(freq_rate): def test_inv_pendulum_single_step(): input_variables, output_variables, train_loader, valid_loader = prepare(freq_rate=1) - mech = PlainMech( + mech = OracleMech( name="test", input_variables=input_variables, output_variables=output_variables, @@ -52,7 +52,7 @@ def test_inv_pendulum_single_step(): def test_inv_pendulum_multi_step(): input_variables, output_variables, train_loader, valid_loader = prepare(freq_rate=2) - mech = PlainMech( + mech = OracleMech( name="test", input_variables=input_variables, output_variables=output_variables, diff --git a/tests/test_sb3_extension/test_online_mb_callback.py b/tests/test_sb3_extension/test_online_mb_callback.py index 6746f51..a2ac0ce 100644 --- a/tests/test_sb3_extension/test_online_mb_callback.py +++ b/tests/test_sb3_extension/test_online_mb_callback.py @@ -8,7 +8,7 @@ from cmrl.sb3_extension.online_mb_callback import OnlineModelBasedCallback from cmrl.utils.creator import parse_space -from cmrl.models.causal_mech.plain_mech import PlainMech +from cmrl.models.causal_mech.oracle_mech import OracleMech from cmrl.models.dynamics import Dynamics from cmrl.models.fake_env import VecFakeEnv @@ -23,7 +23,7 @@ def test_callback(): act_variables = parse_space(env.action_space, "act") next_obs_variables = parse_space(env.state_space, "next_obs") - transition = PlainMech( + transition = OracleMech( name="transition", input_variables=obs_variables + act_variables, output_variables=next_obs_variables, From 69d047ace08d26f8af4aed9948895cedd07c14a0 Mon Sep 17 00:00:00 2001 From: frank Date: Tue, 21 Mar 2023 18:53:07 +0800 Subject: [PATCH 60/68] :tada: save causal discovery history --- cmrl/models/causal_mech/CMI_test.py | 167 +++++++++++++------------ cmrl/models/causal_mech/kernel_test.py | 37 +++--- 2 files changed, 109 insertions(+), 95 deletions(-) diff --git a/cmrl/models/causal_mech/CMI_test.py b/cmrl/models/causal_mech/CMI_test.py index c4cadfc..3d33432 100644 --- a/cmrl/models/causal_mech/CMI_test.py +++ b/cmrl/models/causal_mech/CMI_test.py @@ -18,29 +18,29 @@ class CMITestMech(EnsembleNeuralMech): def __init__( - self, - name: str, - input_variables: List[Variable], - output_variables: List[Variable], - logger: Optional[Logger] = None, - # model learning - longest_epoch: int = -1, - improvement_threshold: float = 0.01, - patience: int = 5, - batch_size: int = 256, - # ensemble - ensemble_num: int = 7, - elite_num: int = 5, - # cfgs - network_cfg: Optional[DictConfig] = None, - encoder_cfg: Optional[DictConfig] = None, - decoder_cfg: Optional[DictConfig] = None, - optimizer_cfg: Optional[DictConfig] = None, - # forward method - residual: bool = True, - encoder_reduction: str = "sum", - # others - device: Union[str, torch.device] = "cpu", + self, + name: str, + input_variables: List[Variable], + output_variables: List[Variable], + logger: Optional[Logger] = None, + # model learning + longest_epoch: int = -1, + improvement_threshold: float = 0.01, + patience: int = 5, + batch_size: int = 256, + # ensemble + ensemble_num: int = 7, + elite_num: int = 5, + # cfgs + network_cfg: Optional[DictConfig] = None, + encoder_cfg: Optional[DictConfig] = None, + decoder_cfg: Optional[DictConfig] = None, + optimizer_cfg: Optional[DictConfig] = None, + # forward method + residual: bool = True, + encoder_reduction: str = "sum", + # others + device: Union[str, torch.device] = "cpu", ): EnsembleNeuralMech.__init__( self, @@ -97,7 +97,8 @@ def multi_graph_forward(self, inputs: MutableMapping[str, torch.Tensor]) -> Dict """ batch_size, extra_dim = self.get_inputs_info(inputs) - inputs_tensor = torch.empty(*extra_dim, self.ensemble_num, batch_size, self.input_var_num, self.encoder_output_dim).to( + inputs_tensor = torch.empty(*extra_dim, self.ensemble_num, batch_size, self.input_var_num, + self.encoder_output_dim).to( self.device ) for i, var in enumerate(self.input_variables): @@ -145,7 +146,7 @@ def multi_graph_forward(self, inputs: MutableMapping[str, torch.Tensor]) -> Dict mask = mask.repeat((1,) * len(mask.shape[:-3]) + (self.ensemble_num, batch_size, 1)) reduced_inputs_tensor = self.reduce_encoder_output(inputs_tensor, mask) assert ( - not torch.isinf(reduced_inputs_tensor).any() and not torch.isnan(reduced_inputs_tensor).any() + not torch.isinf(reduced_inputs_tensor).any() and not torch.isnan(reduced_inputs_tensor).any() ), "tensor must not be inf or nan" output_tensor = self.network(reduced_inputs_tensor) @@ -164,67 +165,73 @@ def calculate_CMI(self, nll_loss: torch.Tensor, threshold=1): return graph_data, nll_loss_diff.mean(dim=(1, 2)) def learn( - self, - # loader - inputs: MutableMapping[str, np.ndarray], - outputs: MutableMapping[str, np.ndarray], - work_dir: Optional[Union[str, pathlib.Path]] = None, - **kwargs + self, + inputs: MutableMapping[str, np.ndarray], + outputs: MutableMapping[str, np.ndarray], + work_dir: Optional[pathlib.Path] = None, + **kwargs ): - if self.discovery: - train_loader, valid_loader = self.get_data_loaders(inputs, outputs) + work_dir = pathlib.Path(".") if work_dir is None else work_dir + + open(work_dir / "history_mask.txt", "w") + open(work_dir / "history_cmi.txt", "w") + train_loader, valid_loader = self.get_data_loaders(inputs, outputs) - final_graph_data = None + final_graph_data = None - epoch_iter = range(self.longest_epoch) if self.longest_epoch >= 0 else count() - epochs_since_update = 0 + epoch_iter = range(self.longest_epoch) if self.longest_epoch >= 0 else count() + epochs_since_update = 0 - loss_func = partial(variable_loss_func, output_variables=self.output_variables, device=self.device) - train = partial(train_func, forward=self.multi_graph_forward, optimizer=self.optimizer, loss_func=loss_func) - eval = partial(eval_func, forward=self.multi_graph_forward, loss_func=loss_func) + loss_func = partial(variable_loss_func, output_variables=self.output_variables, device=self.device) + train = partial(train_func, forward=self.multi_graph_forward, optimizer=self.optimizer, loss_func=loss_func) + eval = partial(eval_func, forward=self.multi_graph_forward, loss_func=loss_func) - best_eval_loss = eval(valid_loader).mean(dim=(0, 2, 3)) + best_eval_loss = eval(valid_loader).mean(dim=(0, 2, 3)) - for epoch in epoch_iter: - train_loss = train(train_loader) - eval_loss = eval(valid_loader) + for epoch in epoch_iter: + train_loss = train(train_loader) + eval_loss = eval(valid_loader) - improvement = (best_eval_loss - eval_loss.mean(dim=(0, 2, 3))) / torch.abs(best_eval_loss) - if (improvement > self.improvement_threshold).any().item(): - best_eval_loss = torch.minimum(best_eval_loss, eval_loss.mean(dim=(0, 2, 3))) - epochs_since_update = 0 + improvement = (best_eval_loss - eval_loss.mean(dim=(0, 2, 3))) / torch.abs(best_eval_loss) + if (improvement > self.improvement_threshold).any().item(): + best_eval_loss = torch.minimum(best_eval_loss, eval_loss.mean(dim=(0, 2, 3))) + epochs_since_update = 0 - final_graph_data, mean_nll_loss_diff = self.calculate_CMI(eval_loss) - print( - "new best valid, CMI test result:\n{}\nwith mean nll loss diff:\n{}".format( - final_graph_data, mean_nll_loss_diff - ) + final_graph_data, mean_nll_loss_diff = self.calculate_CMI(eval_loss) + with open(work_dir / "history_mask.txt", "a") as f: + f.write(str(final_graph_data) + "\n") + with open(work_dir / "history_cmi.txt", "a") as f: + f.write(str(mean_nll_loss_diff) + "\n") + print( + "new best valid, CMI test result:\n{}\nwith mean nll loss diff:\n{}".format( + final_graph_data, mean_nll_loss_diff ) - else: - epochs_since_update += 1 + ) + else: + epochs_since_update += 1 - # log - self.total_CMI_epoch += 1 - if self.logger is not None: - self.logger.record("{}-CMI-test/epoch".format(self.name), epoch) - self.logger.record("{}-CMI-test/epochs_since_update".format(self.name), epochs_since_update) - self.logger.record("{}-CMI-test/train_dataset_size".format(self.name), len(train_loader.dataset)) - self.logger.record("{}-CMI-test/valid_dataset_size".format(self.name), len(valid_loader.dataset)) - self.logger.record("{}-CMI-test/train_loss".format(self.name), train_loss.mean().item()) - self.logger.record("{}-CMI-test/val_loss".format(self.name), eval_loss.mean().item()) - self.logger.record("{}-CMI-test/best_val_loss".format(self.name), best_eval_loss.mean().item()) - self.logger.record("{}-CMI-test/lr".format(self.name), self.optimizer.param_groups[0]["lr"]) + # log + self.total_CMI_epoch += 1 + if self.logger is not None: + self.logger.record("{}-CMI-test/epoch".format(self.name), epoch) + self.logger.record("{}-CMI-test/epochs_since_update".format(self.name), epochs_since_update) + self.logger.record("{}-CMI-test/train_dataset_size".format(self.name), len(train_loader.dataset)) + self.logger.record("{}-CMI-test/valid_dataset_size".format(self.name), len(valid_loader.dataset)) + self.logger.record("{}-CMI-test/train_loss".format(self.name), train_loss.mean().item()) + self.logger.record("{}-CMI-test/val_loss".format(self.name), eval_loss.mean().item()) + self.logger.record("{}-CMI-test/best_val_loss".format(self.name), best_eval_loss.mean().item()) + self.logger.record("{}-CMI-test/lr".format(self.name), self.optimizer.param_groups[0]["lr"]) - self.logger.dump(self.total_CMI_epoch) + self.logger.dump(self.total_CMI_epoch) - if self.patience and epochs_since_update >= self.patience: - break + if self.patience and epochs_since_update >= self.patience: + break - self.scheduler.step() + self.scheduler.step() - assert final_graph_data is not None - self.graph.set_data(final_graph_data) - self.build_optimizer() + assert final_graph_data is not None + self.graph.set_data(final_graph_data) + self.build_optimizer() super(CMITestMech, self).learn(inputs, outputs, work_dir=work_dir, **kwargs) @@ -234,7 +241,6 @@ def learn( from stable_baselines3.common.buffers import ReplayBuffer from torch.utils.data import DataLoader - from cmrl.models.causal_mech.reinforce import ReinforceCausalMech from cmrl.models.data_loader import EnsembleBufferDataset, collate_fn, buffer_to_dict from cmrl.utils.creator import parse_space from cmrl.sb3_extension.logger import configure as logger_configure @@ -242,23 +248,28 @@ def learn( from cmrl.utils.env import load_offline_data from cmrl.models.causal_mech.util import variable_loss_func + def unwrap_env(env): while isinstance(env, gym.Wrapper): env = env.env return env + env = unwrap_env(gym.make("ParallelContinuousCartPoleSwingUp-v0")) real_replay_buffer = ReplayBuffer( int(1e6), env.observation_space, env.action_space, "cpu", handle_timeout_termination=False ) - load_offline_data(env, real_replay_buffer, "SAC-expert", use_ratio=0.1) + load_offline_data(env, real_replay_buffer, "SAC-expert", use_ratio=0.01) + + extra_info = {"Radian": ["obs_1", "obs_5", "obs_9"]} + # extra_info = {"Radian": ["obs_1"]} - input_variables = parse_space(env.state_space, "obs") + parse_space(env.action_space, "act") - output_variables = parse_space(env.state_space, "next_obs") + input_variables = parse_space(env.state_space, "obs", extra_info=extra_info) + parse_space(env.action_space, "act") + output_variables = parse_space(env.state_space, "next_obs", extra_info=extra_info) - logger = logger_configure("kci-log", ["tensorboard", "stdout"]) + logger = logger_configure("cmi-log", ["tensorboard", "stdout"]) - mech = CMITestMech("kernel_test_mech", input_variables, output_variables, device="cuda") + mech = CMITestMech("kernel_test_mech", input_variables, output_variables) inputs, outputs = buffer_to_dict(env.state_space, env.action_space, env.obs2state, real_replay_buffer, "transition") diff --git a/cmrl/models/causal_mech/kernel_test.py b/cmrl/models/causal_mech/kernel_test.py index 7d3e01d..abf6252 100644 --- a/cmrl/models/causal_mech/kernel_test.py +++ b/cmrl/models/causal_mech/kernel_test.py @@ -46,9 +46,10 @@ def __init__( # others device: Union[str, torch.device] = "cpu", # KCI - sample_num=2000, - kci_times=10, - not_confident_bound=0.2, + sample_num: int = 2000, + kci_times: int = 10, + not_confident_bound: float = 0.25, + longest_sample: int = 5000 ): EnsembleNeuralMech.__init__( self, @@ -74,6 +75,7 @@ def __init__( self.sample_num = sample_num self.kci_times = kci_times self.not_confident_bound = not_confident_bound + self.longest_sample = longest_sample def kci( self, @@ -109,14 +111,14 @@ def deal_with_radian_input(name, data): return p_value def kci_compute_graph( - self, inputs: MutableMapping[str, numpy.ndarray], outputs: MutableMapping[str, numpy.ndarray], **kwargs + self, + inputs: MutableMapping[str, numpy.ndarray], + outputs: MutableMapping[str, numpy.ndarray], + work_dir: Optional[pathlib.Path] = None, + **kwargs ): - # [[0, 0, 0, 0], - # [0, 0, 1, 1], - # [1, 0, 0, 0], - # [0, 1, 1, 1], - # [0, 0, 1, 1]] + open(work_dir / "history_vote.txt", "w") length = next(iter(inputs.values())).shape[0] sample_length = min(length, self.sample_num) if self.sample_num > 0 else length @@ -138,12 +140,14 @@ def kci_compute_graph( is_not_confident = np.logical_and(votes > self.not_confident_bound, votes < 1 - self.not_confident_bound) not_confident_list = np.array(np.where(is_not_confident)).T - print(votes) - recompute_times = 1 while len(not_confident_list) != 0: + with open(work_dir / "history_vote.txt", "a") as f: + f.write(str(votes) + "\n") + print(votes) + new_sample_length = int(sample_length * 1.5 ** recompute_times) - if new_sample_length > length: + if new_sample_length > min(self.longest_sample, length): break pvalues_dict = defaultdict(list) @@ -165,7 +169,6 @@ def kci_compute_graph( not_confident_list.append(key) else: votes[key] = vote - print(votes) recompute_times += 1 return votes > 0.5 @@ -204,12 +207,12 @@ def learn( self, inputs: MutableMapping[str, np.ndarray], outputs: MutableMapping[str, np.ndarray], - work_dir: Optional[Union[str, pathlib.Path]] = None, + work_dir: Optional[pathlib.Path] = None, **kwargs ): - if self.discovery: - graph = self.kci_compute_graph(inputs, outputs) - self.graph.set_data(graph) + work_dir = pathlib.Path(".") if work_dir is None else work_dir + graph = self.kci_compute_graph(inputs, outputs, work_dir) + self.graph.set_data(graph) super(KernelTestMech, self).learn(inputs, outputs, work_dir=work_dir, **kwargs) From 7995e86255f2fe4f4bacd629e58d12a762e6bd6d Mon Sep 17 00:00:00 2001 From: frank Date: Mon, 27 Mar 2023 17:07:41 +0800 Subject: [PATCH 61/68] :bug: fix device bug --- cmrl/models/causal_mech/util.py | 50 +++++++++++++++++---------------- cmrl/models/dynamics.py | 6 ++-- 2 files changed, 30 insertions(+), 26 deletions(-) diff --git a/cmrl/models/causal_mech/util.py b/cmrl/models/causal_mech/util.py index d6e194f..0fcebb4 100644 --- a/cmrl/models/causal_mech/util.py +++ b/cmrl/models/causal_mech/util.py @@ -1,6 +1,7 @@ from typing import Callable, Dict, List, Union, MutableMapping from collections import defaultdict import math +import time import torch from torch import Tensor @@ -13,12 +14,12 @@ def von_mises_nll_loss( - input: Tensor, - target: Tensor, - var: Tensor, - full: bool = False, - eps: float = 1e-6, - reduction: str = "mean", + input: Tensor, + target: Tensor, + var: Tensor, + full: bool = False, + eps: float = 1e-6, + reduction: str = "mean", ) -> Tensor: r"""Von Mises negative log likelihood loss. @@ -58,12 +59,12 @@ def von_mises_nll_loss( def circular_gaussian_nll_loss( - input: Tensor, - target: Tensor, - var: Tensor, - full: bool = False, - eps: float = 1e-6, - reduction: str = "mean", + input: Tensor, + target: Tensor, + var: Tensor, + full: bool = False, + eps: float = 1e-6, + reduction: str = "mean", ) -> Tensor: # Entries of var must be non-negative if torch.any(var < 0): @@ -76,7 +77,7 @@ def circular_gaussian_nll_loss( diff = torch.remainder(input - target, 2 * torch.pi) diff[diff > torch.pi] = 2 * torch.pi - diff[diff > torch.pi] - loss = 0.5 * (torch.log(var) + diff**2 / var) + loss = 0.5 * (torch.log(var) + diff ** 2 / var) if full: loss += 0.5 * math.log(2 * math.pi) @@ -89,10 +90,10 @@ def circular_gaussian_nll_loss( def variable_loss_func( - outputs: Dict[str, torch.Tensor], - targets: Dict[str, torch.Tensor], - output_variables: List[Variable], - device: Union[str, torch.device] = "cpu", + outputs: Dict[str, torch.Tensor], + targets: Dict[str, torch.Tensor], + output_variables: List[Variable], + device: Union[str, torch.device] = "cpu", ): dims = list(outputs.values())[0].shape[:-1] total_loss = torch.zeros(*dims, len(outputs)).to(device) @@ -124,10 +125,10 @@ def variable_loss_func( def train_func( - loader: DataLoader, - forward: Callable[[MutableMapping[str, torch.Tensor]], Dict[str, torch.Tensor]], - optimizer: Optimizer, - loss_func: Callable[[MutableMapping[str, torch.Tensor], MutableMapping[str, torch.Tensor]], torch.Tensor], + loader: DataLoader, + forward: Callable[[MutableMapping[str, torch.Tensor]], Dict[str, torch.Tensor]], + optimizer: Optimizer, + loss_func: Callable[[MutableMapping[str, torch.Tensor], MutableMapping[str, torch.Tensor]], torch.Tensor], ): """train for data @@ -150,13 +151,14 @@ def train_func( optimizer.step() batch_loss_list.append(loss) + return torch.cat(batch_loss_list, dim=-2).detach().cpu() def eval_func( - loader: DataLoader, - forward: Callable[[MutableMapping[str, torch.Tensor]], Dict[str, torch.Tensor]], - loss_func: Callable[[MutableMapping[str, torch.Tensor], MutableMapping[str, torch.Tensor]], torch.Tensor], + loader: DataLoader, + forward: Callable[[MutableMapping[str, torch.Tensor]], Dict[str, torch.Tensor]], + loss_func: Callable[[MutableMapping[str, torch.Tensor], MutableMapping[str, torch.Tensor]], torch.Tensor], ): """evaluate for data diff --git a/cmrl/models/dynamics.py b/cmrl/models/dynamics.py index b114ef9..10367dd 100644 --- a/cmrl/models/dynamics.py +++ b/cmrl/models/dynamics.py @@ -67,8 +67,10 @@ def learn(self, real_replay_buffer: ReplayBuffer, work_dir: Optional[Union[str, def step(self, batch_obs, batch_action): with torch.no_grad(): - obs_dict = to_dict_by_space(batch_obs, self.state_space, "obs", repeat=7, to_tensor=True) - act_dict = to_dict_by_space(batch_action, self.action_space, "act", repeat=7, to_tensor=True) + obs_dict = to_dict_by_space(batch_obs, self.state_space, "obs", + repeat=7, to_tensor=True, device=self.device) + act_dict = to_dict_by_space(batch_action, self.action_space, "act", + repeat=7, to_tensor=True, device=self.device) inputs = ChainMap(obs_dict, act_dict) outputs = self.transition.forward(inputs) From dd0be11da3673814f6d9160030b031fb30cf06c7 Mon Sep 17 00:00:00 2001 From: frank Date: Wed, 29 Mar 2023 23:51:54 +0800 Subject: [PATCH 62/68] :tada: add exp_reader --- exp_reader.ipynb | 166 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 166 insertions(+) create mode 100644 exp_reader.ipynb diff --git a/exp_reader.ipynb b/exp_reader.ipynb new file mode 100644 index 0000000..a585a3b --- /dev/null +++ b/exp_reader.ipynb @@ -0,0 +1,166 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 17, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "import pandas as pd\n", + "import numpy as np\n", + "import cmrl\n", + "from emei.core import get_params_str\n", + "from pathlib import Path\n", + "import matplotlib.pyplot as plt\n", + "import yaml" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "outputs": [], + "source": [ + "# 递归判断a字典中存在的值是否与b字典相等\n", + "def dict_equal(a, b):\n", + " for k, v in a.items():\n", + " if isinstance(v, dict):\n", + " if not dict_equal(v, b[k]):\n", + " return False\n", + " elif v != b[k]:\n", + " return False\n", + " return True\n", + "\n", + "\n", + "def get_value(d, key):\n", + " if isinstance(key, str):\n", + " if key not in d:\n", + " raise ValueError(f\"{key} not in dict\")\n", + " return d[key]\n", + " elif isinstance(key, tuple):\n", + " if key[0] not in d:\n", + " raise ValueError(f\"{key[0]} not in dict\")\n", + " return get_value(d[key[0]], key[1:])\n", + " else:\n", + " raise ValueError(\"key must be str or tuple\")" + ], + "metadata": { + "collapsed": false + } + }, + { + "cell_type": "code", + "execution_count": 42, + "outputs": [], + "source": [ + "default_params = dict(freq_rate=1,\n", + " real_time_scale=0.02,\n", + " integrator=\"euler\",\n", + " gravity=9.8,\n", + " length=0.5,\n", + " force_mag=10.0)\n", + "default_custom_cfg = {}\n", + "default_result_key = [\"seed\"]\n", + "\n", + "\n", + "def load_log(exp_name=\"default\",\n", + " task_name=\"ContinuousCartPoleSwingUp-v0\",\n", + " params=default_params,\n", + " dataset=\"SAC-expert-replay\",\n", + " custom_cfg=default_custom_cfg,\n", + " log_file=\"rollout.csv\",\n", + " log_key=\"ep_rew_mean\",\n", + " result_key=default_result_key):\n", + " path = Path(\"./exp\") / exp_name / task_name / get_params_str(params) / dataset\n", + "\n", + " result_dict = {}\n", + " for time_dir in path.glob(r\"*\"):\n", + " if not time_dir.is_dir() or not (time_dir / \".hydra\").exists():\n", + " continue\n", + "\n", + " config_path = time_dir / \".hydra\" / \"config.yaml\"\n", + " with open(config_path, \"r\") as f:\n", + " cfg = yaml.load(f, Loader=yaml.FullLoader)\n", + "\n", + " if not dict_equal(custom_cfg, cfg):\n", + " continue\n", + "\n", + " log_path = time_dir / \"log\" / log_file\n", + " if not log_path.exists():\n", + " continue\n", + "\n", + " key_name = tuple([str(get_value(cfg, k)) for k in result_key])\n", + " df = pd.read_csv(log_path)\n", + " result_dict[key_name] = df[log_key].to_numpy()\n", + " return result_dict" + ], + "metadata": { + "collapsed": false + } + }, + { + "cell_type": "code", + "execution_count": 54, + "outputs": [ + { + "data": { + "text/plain": "" + }, + "execution_count": 54, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "text/plain": "
", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAiwAAAGsCAYAAAD+L/ysAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy88F64QAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA25ElEQVR4nO3deXRU9f3/8dfMZJkkJGELWSALyxdQwIQ1JGARRVO1tqkboiKytfpVq6XVQr+/it9v+5V6jrbayreUHUUrCkpdUcSKZAEkEGWvCgkhIYlsCSSQZeb+/ghEowEykOTemXk+zplzZPK5k9fcM2fy8s6977EZhmEIAADAwuxmBwAAALgQCgsAALA8CgsAALA8CgsAALA8CgsAALA8CgsAALA8CgsAALA8CgsAALA8CgsAALA8CgsAALA8CoukmpoapaSkyGazKT8//7xrf/7zn6t3794KCQlRVFSUfvKTn2jPnj1N1thstu/dXnnllcaff/zxx82uKS0tbXHmzz77TBMmTFB8fLxCQkJ02WWX6bnnnvPoeQMA4C18vrBcddVVWrp06XnXPPbYY4qLi2vR4w0dOlRLlizR7t279f7778swDF133XVyuVxN1i1ZskSHDh1qvGVmZn7vsfbu3dtkTbdu3Vr6tJSXl6du3bpp+fLl2rlzp/7rv/5Ls2bN0vPPP9/ixwAAwFsEmB3AbO+9954++OADrVq1Su+9994F1//sZz9r/O+kpCT94Q9/UHJysgoKCtS7d+/Gn3Xs2FExMTHnfaxu3bqpY8eOzf7M7Xbrqaee0vz581VaWqq+ffvqd7/7nW699VZJ0pQpU5qs79Wrl3Jzc/X666/rwQcfvODzAADAm/j8EZbzKSsr0/Tp0/Xiiy8qNDTU4+2rqqq0ZMkS9ezZU/Hx8U1+9sADD6hr164aMWKEFi9erOa+FDslJUWxsbG69tprlZ2d3eRnc+bM0QsvvKB58+Zp586d+uUvf6m7775b69evP2eeiooKde7c2ePnAQCA1fntERbDMHTvvffqvvvu07Bhw1RQUNDibf/v//5Pjz32mKqqqtSvXz+tXbtWQUFBjT//n//5H1199dUKDQ3VBx98oP/8z//UyZMn9Ytf/EKSFBsbq3nz5mnYsGGqqanRwoULddVVV2nTpk0aMmSIampq9OSTT+rDDz9UWlqapIYjKFlZWfr73/+uMWPGfC9TTk6OVqxYoXfeeefSdgwAABZkM5r7X38v9uSTT+rJJ59s/PepU6cUGBiogIBvutmuXbu0evVqvfrqq1q/fr0cDocKCgrUs2dPbdu2TSkpKef9HRUVFSovL9ehQ4f09NNPq7i4WNnZ2XI6nc2uf/zxx7VkyRIVFRWd8zHHjBmjhIQEvfjii9q5c6cGDhyosLCwJmtqa2s1ePBgbdq0qcn9O3bs0NixY/Xwww/r//2//3fe7AAAeCOfKyxHjx7V0aNHG/9911136ZZbbtHNN9/ceF9SUpJuvfVWvfXWW7LZbI33u1wuORwO3XXXXVq2bFmLfl9tba06deqkhQsXasKECc2ueeedd/SjH/1Ip0+fVnBwcLNrHn30UWVlZSk3N1ebNm3SyJEj9fHHH6t79+5N1gUHBzf5+GnXrl0aO3aspk2bpv/93/9tUWYAALyNz30k1Llz5ybncYSEhKhbt27q06dPk3V/+ctf9Ic//KHx3yUlJcrIyNCKFSuUmpra4t9nGIYMw1BNTc051+Tn56tTp07nLCtn18TGxkqSLr/8cgUHB+vAgQPNfvxz1s6dO3X11Vdr0qRJlBUAgE/zucLSUgkJCU3+3aFDB0lS79691aNHD0lScXGxrrnmGr3wwgsaMWKE9u3bpxUrVui6665TVFSUDh48qD/+8Y8KCQnRDTfcIEl66623VFZWppEjR8rpdGrt2rV68skn9etf/7rxdz377LPq2bOnBgwYoNOnT2vhwoX66KOP9MEHH0iSwsPD9etf/1q//OUv5Xa7NXr0aFVUVCg7O1sRERGaNGmSduzYoauvvloZGRmaMWNG4wwXh8OhqKioNt9/AAC0J78tLC1RV1envXv3qrq6WpLkdDq1YcMGPfvsszp27Jiio6P1gx/8QDk5OY0zVAIDAzV37lz98pe/lGEY6tOnj/70pz9p+vTpjY9bW1urX/3qVyouLlZoaKiuuOIKffjhhxo7dmzjmt///veKiorSnDlztG/fPnXs2FFDhgzRb3/7W0nSypUr9fXXX2v58uVavnx543aJiYkenUAMAIA38LlzWAAAgO/x6zksAADAO1BYAACA5fnEOSxut1slJSUKDw9vcpkyAACwLsMwdOLECcXFxcluP/8xFJ8oLCUlJd8bjQ8AALxDUVFR4xW65+IThSU8PFxSwxOOiIgwOQ0AAGiJyspKxcfHN/4dPx+fKCxnPwaKiIigsAAA4GVacjoHJ90CAADLo7AAAADLo7AAAADLo7AAAADLo7AAAADLo7AAAADLo7AAAADLo7AAAADLo7AAAADLo7AAAADLo7AAAADLo7AAAADL84kvP2wr9S63/vfd3WbHgI/qGBKkn4/pJWegw+woAGB5FJbzcBvSkuwCs2PAhwU4bHpgbB+zYwCA5VFYzsNukx4Y29vsGPBBxcdOaXV+iZblFGj6lb0UFMCnswBwPhSW8whw2PVoRn+zY8AH1da7lfPVEZWfqNFbn5XolqE9zI4EAJbG/9YBJggKsGtSepIkacGGfTIMw9xAAGBxFBbAJHelJigk0KE9pSeU89URs+MAgKVRWACTdAwN0m3DGj4KWrBhn8lpAMDaKCyAiaaM6imbTfp479f6ouyE2XEAwLIoLICJkrqG6drLoiVJi7L2m5wGAKyLwgKYbPoPekmSXt9WrMMna0xOAwDWRGEBTDYssZOSe0Sqtt6tF3MLzY4DAJZEYQFMZrPZNO3KhqMsyzcW6nSdy+REAGA9FBbAAq4fGKPuHUN0pKpWb2wrNjsOAFgOhQWwgACHXZNHJUlqOPnW7WaQHAB8G4UFsIjbh8erQ3CAviw/qfX//trsOABgKRQWwCIinIG6Y3i8JGlhFoPkAODbKCyAhdw7KkkOu03ZXx7RzpIKs+MAgGVQWAAL6dEpVNcPjJHEIDkA+DYKC2AxZy9xfuuzEpVVnjY5DQBYA4UFsJiU+I4antRJdS5Dy3IKzI4DAJZAYQEsaOrohqMsL206oOraepPTAID5KCyABV17ebQSu4Sq4lSdXtty0Ow4AGA6CgtgQQ67TVNG9ZQkLc7eLxeD5AD4OQoLYFG3DeuhyJBAFR6p1tpdZWbHAQBTUVgAiwoNCtCdqQmSpEUMkgPg5ygsgIXdm56kQIdNnxYcU37RcbPjAIBpKCyAhUVHOHXTFXGSpIUbOMoCwH9RWACLm3plw8m37+0o1cFj1SanAQBzUFgAixsQF6n03l3kchtaml1gdhwAMAWFBfAC08+M63/l0yJVnq4zOQ0AtD8KC+AFxvSNUu+oMJ2sqdernxaZHQcA2h2FBfACdrut8UsRl2QXqN7lNjkRALQvCgvgJX46uLu6hAWp+Pgpvbej1Ow4ANCuKCyAl3AGOnT3yERJDZc4Gwbj+gH4DwoL4EUmpiUqKMCuzw5WaEvhMbPjAEC7obAAXqRrh2DdPLi7JGnBJwySA+A/KCyAl5k6umGQ3NrdZSo4XGVyGgBoHxQWwMv8R3S4ruoXJcOQFmfvNzsOALQLCgvghc4Okntty0Edr641OQ0AtD0KC+CF0nt3Uf+YcJ2qc+mlTQfMjgMAbY7CAnghm83WeJRlWU6BausZJAfAt1FYAC91U3KcuoUHq/xEjd76rMTsOADQpigsgJcKCrBrUnqSJGlh1n4GyQHwaRQWwIvdlZqgkECHdh+qVM5XR8yOAwBthsICeLGOoUG6bVgPSQ3j+gHAV1FYAC83ZVRP2WzSv/Z+rS/LT5gdBwDaBIUF8HJJXcN07WXRkqRFWQySA+CbKCyAD5h25hLnVVuLdfhkjclpAKD1UVgAHzA8qZOSe0Sqtt6t5RsLzY4DAK2OwgL4AJvNpqlnjrK8mFuo03UukxMBQOuisAA+4oaBMereMURHqmq1elux2XEAoFVRWAAfEeCw695vDZJzuxkkB8B3UFgAHzJ+RLw6BAfoy/KTWv/F12bHAYBWQ2EBfEiEM1Djh8dLYpAcAN9CYQF8zORRSbLbpOwvj2hnSYXZcQCgVVxUYZk7d66SkpLkdDqVmpqqzZs3n3f9a6+9pv79+8vpdGrQoEF69913z7n2vvvuk81m07PPPnsx0QC/16NTqK4fFCuJQXIAfIfHhWXFihWaMWOGZs+era1btyo5OVkZGRkqLy9vdn1OTo4mTJigqVOnatu2bcrMzFRmZqZ27NjxvbVvvPGGNm7cqLi4OM+fCYBG089c4vzWZyUqqzxtchoAuHQeF5Y//elPmj59uiZPnqzLL79c8+bNU2hoqBYvXtzs+ueee04//OEP9eijj+qyyy7T73//ew0ZMkTPP/98k3XFxcV66KGH9NJLLykwMPDing0ASVJKfEcNS+ykOpehZTkFZscBgEvmUWGpra1VXl6exo0b980D2O0aN26ccnNzm90mNze3yXpJysjIaLLe7XZr4sSJevTRRzVgwIAL5qipqVFlZWWTG4Cmzo7rf2nTAVXX1pucBgAujUeF5fDhw3K5XIqOjm5yf3R0tEpLS5vdprS09ILrn3rqKQUEBOgXv/hFi3LMmTNHkZGRjbf4+HhPngbgF669PFqJXUJVcapOK/MOmh0HAC6J6VcJ5eXl6bnnntPSpUtls9latM2sWbNUUVHReCsqKmrjlID3cdhtmjKqp6SGk29dDJID4MU8Kixdu3aVw+FQWVlZk/vLysoUExPT7DYxMTHnXb9hwwaVl5crISFBAQEBCggIUGFhoX71q18pKSmp2ccMDg5WREREkxuA77ttWA9FhgSq8Ei1PtxdduENAMCiPCosQUFBGjp0qNatW9d4n9vt1rp165SWltbsNmlpaU3WS9LatWsb10+cOFGff/658vPzG29xcXF69NFH9f7773v6fAB8S2hQgO5MTZDEIDkA3i3A0w1mzJihSZMmadiwYRoxYoSeffZZVVVVafLkyZKke+65R927d9ecOXMkSQ8//LDGjBmjZ555RjfeeKNeeeUVbdmyRfPnz5ckdenSRV26dGnyOwIDAxUTE6N+/fpd6vMD/N696UlauGGfPi04pvyi40qJ72h2JADwmMfnsIwfP15PP/20Hn/8caWkpCg/P19r1qxpPLH2wIEDOnToUOP69PR0vfzyy5o/f76Sk5O1cuVKrV69WgMHDmy9ZwHgnKIjnLrpiobZRhxlAeCtbIZheP2ZeJWVlYqMjFRFRQXnswDN2FlSoRv/kiWH3ab1j16lHp1CzY4EAB79/Tb9KiEAbW9AXKTSe3eRy21oaXaB2XEAwGMUFsBPnB3X/8qnRTpxus7kNADgGQoL4CfG9I1S76gwnayp14pPmV0EwLtQWAA/YbfbGsf1L8kuUL3LbXIiAGg5CgvgR346uLu6hAWp+Pgpvbej+a/TAAArorAAfsQZ6NDdIxMlNVzi7AMXCQLwExQWwM9MTEtUUIBdnx2s0JbCY2bHAYAWobAAfqZrh2DdPLi7JAbJAfAeFBbAD00d3fAtzh/sKlPB4SqT0wDAhVFYAD/0H9HhuqpflAxDWpK93+w4AHBBFBbAT00b3XCJ86tbDup4da3JaQDg/CgsgJ8a1aeL+seE61SdSy9vPmB2HAA4LwoL4Kdstm8GyS3LKVBtPYPkAFgXhQXwYz9OjlO38GCVVdbo7c9LzI4DAOdEYQH8WFCAXZPSkyRJCzbsZ5AcAMuisAB+7q7UBIUEOrT7UKVyvzpidhwAaBaFBfBzHUODdOvQHpKkBQySA2BRFBYAmjK6p2w26V97v9aX5SfMjgMA30NhAaCeXcM07rJoSdKiLAbJAbAeCgsASdL0M5c4r9parMMna0xOAwBNUVgASJKGJ3XSFT0iVVvv1vKNhWbHAYAmKCwAJDUdJPdibqFO17lMTgQA36CwAGh0/cAYxUU6daSqVqu3FZsdBwAaUVgANAp02DV5VE9J0sKs/XK7GSQHwBooLACaGD8iXh2CA/Rl+Umt/+Jrs+MAgCQKC4DviHAGavzweEnSQgbJAbAICguA75k8Kkl2m5T95RHtKqk0Ow4AUFgAfF+PTqG6flCsJGlhFkdZAJiPwgKgWWcHyb31WYnKKk+bnAaAv6OwAGhWSnxHDUvspDqXoWU5BWbHAeDnKCwAzunsILmXNh1QdW29yWkA+DMKC4BzuvbyaCV2CVXFqTqtzDtodhwAfozCAuCcHHabppwZJLc4a79cDJIDYBIKC4DzunVoD0U4A1RwpFof7i4zOw4AP0VhAXBeYcEBumtkoiRp0Yb9JqcB4K8oLAAuaFJakgLsNm0uOKrPio6bHQeAH6KwALigmEinfpwcJ6nhSxEBoL1RWAC0yNQrG06+fXf7IRUfP2VyGgD+hsICoEUGxEUqvXcXudyGlmZzlAVA+6KwAGixaWeOsryyuUgnTteZnAaAP6GwAGixq/p2U++oMJ2oqdeKT4vMjgPAj1BYALSY3W7T1NEN4/qXZBeo3uU2OREAf0FhAeCRm4d0V+ewIBUfP6U1O0vNjgPAT1BYAHjEGejQ3WcGyS3YsF+Gwbh+AG2PwgLAYxNHJioowK7Pio4rr/CY2XEA+AEKCwCPRYUH66cp3SVJCzbsMzkNAH9AYQFwUc4OkvtgV5kKj1SZnAaAr6OwALgofaPDNaZvlAxDWsy4fgBtjMIC4KJNv7LhEudXtxxURTWD5AC0HQoLgIs2qk8X9Y8J16k6l17aXGh2HAA+jMIC4KLZbDZNO3OUZVlOgWrrGSQHoG1QWABckpuSYxUVHqyyyhq9/XmJ2XEA+CgKC4BLEhzg0L3pSZIYJAeg7VBYAFyyO0ckyBlo1+5Dlcr96ojZcQD4IAoLgEvWKSxItw2Nl8QgOQBtg8ICoFVMGd1TNpv0r71f68vyE2bHAeBjKCwAWkXPrmEad1m0JGkRg+QAtDIKC4BWc3aQ3KqtxTpyssbkNAB8CYUFQKsZntRJV/SIVG29Wy9uZJAcgNZDYQHQar49SO7F3EKdrnOZnAiAr6CwAGhV1w+MUVykU0eqarV6W7HZcQD4CAoLgFYV6LBr8qiekqSFWQySA9A6KCwAWt34EfHqEBygL8tP6uN/f212HAA+gMICoNVFOAM1fnjDILlFG7jEGcClo7AAaBP3pifJbpOyvjysXSWVZscB4OUuqrDMnTtXSUlJcjqdSk1N1ebNm8+7/rXXXlP//v3ldDo1aNAgvfvuu01+/sQTT6h///4KCwtTp06dNG7cOG3atOliogGwiPjOobp+UKwkBskBuHQeF5YVK1ZoxowZmj17trZu3ark5GRlZGSovLy82fU5OTmaMGGCpk6dqm3btikzM1OZmZnasWNH45q+ffvq+eef1/bt25WVlaWkpCRdd911+vprPvsGvNm00Q0n3775WbHKKk+bnAaAN7MZHp7Cn5qaquHDh+v555+XJLndbsXHx+uhhx7SzJkzv7d+/Pjxqqqq0ttvv91438iRI5WSkqJ58+Y1+zsqKysVGRmpDz/8UNdcc80FM51dX1FRoYiICE+eDoA2duvfcrSl8JgeGNtbj2b0NzsOAAvx5O+3R0dYamtrlZeXp3Hjxn3zAHa7xo0bp9zc3Ga3yc3NbbJekjIyMs65vra2VvPnz1dkZKSSk5ObXVNTU6PKysomNwDWNO3KhqMsyzceUHVtvclpAHgrjwrL4cOH5XK5FB0d3eT+6OholZaWNrtNaWlpi9a//fbb6tChg5xOp/785z9r7dq16tq1a7OPOWfOHEVGRjbe4uPjPXkaANrRtZfHKKFzqCpO1WlV3kGz4wDwUpa5Smjs2LHKz89XTk6OfvjDH+r2228/53kxs2bNUkVFReOtqKiondMCaCmH3aYpo5IkNZx863IzSA6A5zwqLF27dpXD4VBZWVmT+8vKyhQTE9PsNjExMS1aHxYWpj59+mjkyJFatGiRAgICtGjRomYfMzg4WBEREU1uAKzrtmHxinAGqOBItdbtLrvwBgDwHR4VlqCgIA0dOlTr1q1rvM/tdmvdunVKS0trdpu0tLQm6yVp7dq151z/7cetqeHr6QFfEBYcoDtTEyVJCxkkB+AiePyR0IwZM7RgwQItW7ZMu3fv1v3336+qqipNnjxZknTPPfdo1qxZjesffvhhrVmzRs8884z27NmjJ554Qlu2bNGDDz4oSaqqqtJvf/tbbdy4UYWFhcrLy9OUKVNUXFys2267rZWeJgCz3ZuepAC7TZsLjuqzouNmxwHgZTwuLOPHj9fTTz+txx9/XCkpKcrPz9eaNWsaT6w9cOCADh061Lg+PT1dL7/8subPn6/k5GStXLlSq1ev1sCBAyVJDodDe/bs0S233KK+ffvqpptu0pEjR7RhwwYNGDCglZ4mALPFRDp1U3KcpIYvRQQAT3g8h8WKmMMCeIcdxRX60V+z5LDb9MljY9W9Y4jZkQCYqM3msADApRjYPVJpvbrI5Ta0NJujLABajsICoF1N/0HDILlXNhfpxOk6k9MA8BYUFgDt6qq+3dQrKkwnauq14lNmKAFoGQoLgHZlt9s0bXQvSdKS7ALVu9wmJwLgDSgsANrdzUO6q3NYkIqPn9Kanc1/rQcAfBuFBUC7cwY6dPfIhkFyCzbslw9crAigjVFYAJhi4shEBQXY9VnRceUVHjM7DgCLo7AAMEVUeLB+mtJdkrRgwz6T0wCwOgoLANNMvbLhEucPdpWp8EiVyWkAWBmFBYBp+kaHa0zfKBmGtJhx/QDOg8ICwFTTr2y4xPnVLQdVUc0gOQDNo7AAMNWoPl3UPyZcp+pcemlzodlxAFgUhQWAqWw2m6adOcqyLKdAtfUMkgPwfRQWAKa7KTlWUeHBKqus0dufl5gdB4AFUVgAmC44wKF705MkSQsZJAegGRQWAJZw54gEOQPt2nWoUrlfHTE7DgCLobAAsIROYUG6bWi8JGkhlzgD+A4KCwDLmDK6p2w26aM95fqy/ITZcQBYCIUFgGX07BqmcZdFS5IWZRWYGwaApVBYAFjKtNEN4/pf33pQR07WmJwGgFVQWABYyoienXVFj0jV1Lu1fOMBs+MAsAgKCwBLsdlsmnrmKMuLGwt0us5lciIAVkBhAWA5NwyKVVykU4dP1uqf+cVmxwFgARQWAJYT6LDr3lFJkhgkB6ABhQWAJd0xIkFhQQ59UX5S6//9tdlxAJiMwgLAkiKcgRo/PEFSw1EWAP6NwgLAsiaPSpLdJmV9eVi7D1WaHQeAiSgsACwrvnOorh8YK4mjLIC/o7AAsLRpVzZc4vzmZ8UqrzxtchoAZqGwALC0wQmdNDSxk+pchpblFpgdB4BJKCwALG/6maMsL206oOraepPTADADhQWA5V17eYwSOofqeHWdVuUdNDsOABNQWABYnsNu05Qzg+QWZe2X280gOcDfUFgAeIXbhsUrwhmggiPV+nB3mdlxALQzCgsArxAWHKA7UxMlSQuzuMQZ8DcUFgBeY1J6ogLsNm3ef1SfHzxudhwA7YjCAsBrxEaG6KbkOEkMkgP8DYUFgFeZOrrhEud3th9S8fFTJqcB0F4oLAC8ysDukUrr1UUut6FlOQVmxwHQTigsALzO9B80HGX5x6YDOnG6zuQ0ANoDhQWA17mqbzf1igrTiZp6rfi0yOw4ANoBhQWA17HbbZo2upckaUl2gepdbpMTAWhrFBYAXunmId3VOSxIxcdPac3OUrPjAGhjFBYAXskZ6NDdIxsGyS3YsF+Gwbh+wJdRWAB4rYkjExUUYNdnRceVV3jM7DgA2hCFBYDXigoP1k9TuktikBzg6ygsALza1CsbLnF+f1epCo9UmZwGQFuhsADwan2jwzWmb5QMo+GKIQC+icICwOtNO3OU5dUtRaqoZpAc4IsoLAC83ug+XdU/JlzVtS69vPmA2XEAtAEKCwCvZ7PZGr8UcWnOftXWM0gO8DUUFgA+4ccpcYoKD1ZZZY3e2V5idhwArYzCAsAnBAc4NCntzCC5TxgkB/gaCgsAn3FXaqKcgXbtOlSp3H1HzI4DoBVRWAD4jE5hQbp1aA9JDJIDfA2FBYBPmTKqp2w26aM95fqy/KTZcQC0EgoLAJ/SK6qDrukfLUlalMVRFsBXUFgA+JzpZwbJvb71oI6crDE5DYDWQGEB4HNG9OysQd0jVVPv1vKNDJIDfAGFBYDPsdlsjeP6X9xYoNN1LpMTAbhUFBYAPumGQbGKjXTq8Mla/TO/2Ow4AC4RhQWATwp02DV5VJKkhkucGSQHeDcKCwCfNX54gsKCHPqi/KTW//trs+MAuAQUFgA+KzIkUOOHJ0jiEmfA21FYAPi0yaOSZLdJG744rN2HKs2OA+AiXVRhmTt3rpKSkuR0OpWamqrNmzefd/1rr72m/v37y+l0atCgQXr33Xcbf1ZXV6ff/OY3GjRokMLCwhQXF6d77rlHJSV82yqASxffOVTXD4yVxFEWwJt5XFhWrFihGTNmaPbs2dq6dauSk5OVkZGh8vLyZtfn5ORowoQJmjp1qrZt26bMzExlZmZqx44dkqTq6mpt3bpVv/vd77R161a9/vrr2rt3r3784x9f2jMDgDOmnrnE+Z/5xSqvPG1yGgAXw2Z4eOp8amqqhg8frueff16S5Ha7FR8fr4ceekgzZ8783vrx48erqqpKb7/9duN9I0eOVEpKiubNm9fs7/j00081YsQIFRYWKiEh4YKZKisrFRkZqYqKCkVERHjydAD4iVv+lqO8wmN6cGwf/Tqjn9lxAMizv98eHWGpra1VXl6exo0b980D2O0aN26ccnNzm90mNze3yXpJysjIOOd6SaqoqJDNZlPHjh2b/XlNTY0qKyub3ADgfM6O61++qVDVtfUmpwHgKY8Ky+HDh+VyuRQdHd3k/ujoaJWWlja7TWlpqUfrT58+rd/85jeaMGHCOdvWnDlzFBkZ2XiLj4/35GkA8EPXXh6jhM6hOl5dp1VbGSQHeBtLXSVUV1en22+/XYZh6G9/+9s5182aNUsVFRWNt6KionZMCcAbOew2TTkzSG5x1n653QySA7yJR4Wla9eucjgcKisra3J/WVmZYmJimt0mJiamRevPlpXCwkKtXbv2vJ9lBQcHKyIioskNAC7ktmHxinAGaP/hKn24u+zCGwCwDI8KS1BQkIYOHap169Y13ud2u7Vu3TqlpaU1u01aWlqT9ZK0du3aJuvPlpUvvvhCH374obp06eJJLABokbDgAN2ZmihJWsglzoBX8fgjoRkzZmjBggVatmyZdu/erfvvv19VVVWaPHmyJOmee+7RrFmzGtc//PDDWrNmjZ555hnt2bNHTzzxhLZs2aIHH3xQUkNZufXWW7Vlyxa99NJLcrlcKi0tVWlpqWpra1vpaQJAg0npiQqw27R5/1F9fvC42XEAtJDHhWX8+PF6+umn9fjjjyslJUX5+flas2ZN44m1Bw4c0KFDhxrXp6en6+WXX9b8+fOVnJyslStXavXq1Ro4cKAkqbi4WG+++aYOHjyolJQUxcbGNt5ycnJa6WkCQIPYyBDdlBwnqeFLEQF4B4/nsFgRc1gAeGJHcYV+9NcsOew2ffLYWHXvGGJ2JMAvtdkcFgDwBQO7RyqtVxe53IaW5RSYHQdAC1BYAPilaWcGyf1j0wGdOF1nchoAF0JhAeCXxvbrpl5RYTpRU69Xtxw0Ow6AC6CwAPBLdrtNU0c3HGVZnLVf9S63yYkAnA+FBYDfumVID3UKDVTx8VN6fyeD5AAro7AA8FvOQIcmjmwYJLdgwz75wEWTgM+isADwaxPTkhTksCu/6Li2HjhmdhwA50BhAeDXosKDlTm4YZDcgk8YJAdYFYUFgN+bdmUvSdL7u0pVeKTK5DQAmkNhAeD3+kaH6wd9o2QY0pLsArPjAGgGhQUAJE0/M0ju1S1FqqhmkBxgNRQWAJA0uk9X9Y8JV3WtSy9vPmB2HADfQWEBAEk22zeD5Jbm7FdtPYPkACuhsADAGT9OiVNUeLDKKmv0zvYSs+MA+BYKCwCcERzg0KS0hkFyCzfsZ5AcYCEUFgD4lrtSE+UMtGtnSaVy9x0xOw6AMygsAPAtncKCdOvQHpKkRRsYJAdYBYUFAL5jyqiestmkdXvK9WX5SbPjABCFBQC+p1dUB13TP1qStDiboyyAFVBYAKAZ084MkluVd1BHTtaYnAYAhQUAmpHas7MGdY9UTb1bL21ikBxgNgoLADTDZrM1HmV5IbdAp+tcJicC/BuFBQDO4YZBsYqNdOrwyVq9mc8gOcBMFBYAOIdAh12TRyVJkhZm7WOQHGAiCgsAnMf44QkKC3Lo32Un9ckXh82OA/gtCgsAnEdkSKDGD0+QJC3csM/kNID/orAAwAVMHpUku03a8MVh7SmtNDsO4JcoLABwAfGdQ3X9wFhJDV+KCKD9UVgAoAWmnrnE+Z/5xSqvPG1yGsD/UFgAoAWGJHTS0MROqnMZeiG30Ow4gN+hsABAC00b3XCUZfmmQlXX1pucBvAvFBYAaKHrBsQovnOIjlfXadXWYrPjAH6FwgIALeSw2zRlVMNRlsVZ++V2M0gOaC8UFgDwwO3D4hXuDND+w1Vat6fc7DiA36CwAIAHwoIDdGdqwyC5BQySA9oNhQUAPHRvepIC7DZt3n9Unx88bnYcwC9QWADAQ7GRIfrRFQySA9oThQUALsK0K3tJkt7Zfkglx0+ZnAbwfRQWALgIA7tHamSvznK5DS3NKTA7DuDzKCwAcJGmnznK8o9NB3SyhkFyQFuisADARRrbr5t6RYXpRE29VnxaZHYcwKdRWADgItntNk09M65/SfZ+1bvcJicCfBeFBQAuwc2De6hTaKAOHjul93eWmR0H8FkUFgC4BCFBDk0cmShJWpjFIDmgrVBYAOAS3Z2WqCCHXdsOHFde4VGz4wA+icICAJeoW7hTmYPjJDFIDmgrFBYAaAVTRzdc4vz+zlIdOFJtchrA91BYAKAV9IsJ1w/6RsltSIuzOcoCtDYKCwC0kmlnLnF+dUuRKqrrTE4D+BYKCwC0kiv/o6v6x4Srutalf3x6wOw4gE+hsABAK7HZvhkktzS7QLX1DJIDWguFBQBa0Y9T4hQVHqzSytN6d/shs+MAPoPCAgCtKDjAoUlpDYPkFmzYJ8MwTE4E+AYKCwC0srtSE+UMtGtnSaU27mOQHNAaKCwA0Mo6hQXp1qE9JEkLNzCuH2gNFBYAaANTRvWUzSat21Our74+aXYcwOtRWACgDfSK6qBr+kdLkhZlMUgOuFQUFgBoI9OubLjEeVXeQR2tqjU5DeDdKCwA0EZSe3bWoO6Rqql3a/nGQrPjAF6NwgIAbcRmszUeZXkht0Cn61wmJwK8F4UFANrQDYNiFRvp1OGTtXozv8TsOIDXorAAQBsKdNh1b3qSJGlhFoPkgItFYQGANnbHiASFBTn077KT+uSLw2bHAbwShQUA2lhkSKBuHx4viUFywMW6qMIyd+5cJSUlyel0KjU1VZs3bz7v+tdee039+/eX0+nUoEGD9O677zb5+euvv67rrrtOXbp0kc1mU35+/sXEAgDLmjKqp+w2acMXh7WntNLsOIDX8biwrFixQjNmzNDs2bO1detWJScnKyMjQ+Xl5c2uz8nJ0YQJEzR16lRt27ZNmZmZyszM1I4dOxrXVFVVafTo0Xrqqacu/pkAgIXFdw7VDwfGSJIWbmCQHOApm+HhGWCpqakaPny4nn/+eUmS2+1WfHy8HnroIc2cOfN768ePH6+qqiq9/fbbjfeNHDlSKSkpmjdvXpO1BQUF6tmzp7Zt26aUlJQWZ6qsrFRkZKQqKioUERHhydMBgHaz9cAx3fx/OQp02JT9m6vVLcJpdiTAVJ78/fboCEttba3y8vI0bty4bx7Abte4ceOUm5vb7Da5ublN1ktSRkbGOde3RE1NjSorK5vcAMDqhiR00pCEjqpzGXohl0FygCc8KiyHDx+Wy+VSdHR0k/ujo6NVWlra7DalpaUerW+JOXPmKDIysvEWHx9/0Y8FAO1p+pW9JEnLNxXqVC2D5ICW8sqrhGbNmqWKiorGW1FRkdmRAKBFrhsQo/jOITpeXaeVWw+aHQfwGh4Vlq5du8rhcKisrKzJ/WVlZYqJiWl2m5iYGI/Wt0RwcLAiIiKa3ADAGzjsNk0Z1TCuf3HWfrndDJIDWsKjwhIUFKShQ4dq3bp1jfe53W6tW7dOaWlpzW6TlpbWZL0krV279pzrAcDX3TYsXuHOAO0/XKV1e5q/whJAUx5/JDRjxgwtWLBAy5Yt0+7du3X//ferqqpKkydPliTdc889mjVrVuP6hx9+WGvWrNEzzzyjPXv26IknntCWLVv04IMPNq45evSo8vPztWvXLknS3r17lZ+ff0nnuQCAVXUIDtCdqQmSGCQHtJTHhWX8+PF6+umn9fjjjyslJUX5+flas2ZN44m1Bw4c0KFDhxrXp6en6+WXX9b8+fOVnJyslStXavXq1Ro4cGDjmjfffFODBw/WjTfeKEm64447NHjw4O9d9gwAvuLe9CQF2G3atP+oth+sMDsOYHkez2GxIuawAPBGj7yyTavzS/STlDg9d8dgs+MA7a7N5rAAAFrPtDOXOL/9+SGVHD9lchrA2igsAGCSgd0jNbJXZ7nchpblFJgdB7A0CgsAmGja6IajLC9vPqCTNfUmpwGsi8ICACa6un839eoaphOn6/XqpwzBBM6FwgIAJrLbbZoy+swguez9qne5TU4EWBOFBQBMdsuQHuoUGqiDx07pg11lF94A8EMUFgAwWUiQQxNHJkqSFjBIDmgWhQUALODutEQFOezaduC48gqPmR0HsBwKCwBYQLdwpzIHx0liXD/QHAoLAFjE1DOXOL+/s1QHjlSbnAawFgoLAFhEv5hw/aBvlNxGwxVDAL5BYQEAC5l25hLnV7cUqeJUnclpAOugsACAhVz5H13VLzpc1bUu/WPzAbPjAJZBYQEAC7HZbJp6ZcNRlqXZBapjkBwgicICAJbzk5Q4de0QrNLK03rn80NmxwEsgcICABYTHODQpLSGQXILs/bJMAyTEwHmo7AAgAXdNTJRzkC7dhRXauO+o2bHAUxHYQEAC+ocFqRbhvSQxCA5QKKwAIBlTT1zifO6PeX66uuTJqcBzEVhAQCL6hXVQeMu6yZJWpTFIDn4NwoLAFjYtCsbxvWvyjuoo1W1JqcBzENhAQALS+3ZWQO7R6im3q3lGwvNjgOYhsICABZms9k0/cxRlhdyC3S6zmVyIsAcFBYAsLgbBsUqNtKpwydr9WZ+idlxAFNQWADA4gIddt2bniSJQXLwXxQWAPACd4xIUFiQQ/8uO6lPvjhsdhyg3VFYAMALRIYE6vbh8ZIYJAf/RGEBAC8xZVRP2W3Shi8Oa09ppdlxgHZFYQEALxHfOVQ/HBgjSVq0gUFy8C8UFgDwIlNHN1zi/M/8EpWfOG1yGqD9UFgAwIsMTeykIQkdVety68VcBsnBf1BYAMDLnB3Xv3xjoU7VMkgO/oHCAgBeJmNAjOI7h+hYdZ1WbT1odhygXVBYAMDLOOw2TU7vKUlanLVfbjeD5OD7KCwA4IVuHx6vcGeA9h2u0kd7ys2OA7Q5CgsAeKEOwQG6MzVBkrSAQXLwAxQWAPBS96YnKcBu06b9R7X9YIXZcYA2RWEBAC8VGxmiH10RK6nhSxEBX0ZhAQAvdvYS53c+P6SS46dMTgO0HQoLAHixgd0jNbJXZ9W7DS3LKTA7DtBmKCwA4OWmnRnX//LmAzpZU29yGqBtUFgAwMtd3b+benUN04nT9Xr10yKz4wBtgsICAF7Obrdpyugzg+Sy98vFIDn4oACzAwAALt0tQ3romQ/26uCxU3pkRb66dggyOxJ8TIDdpv+68XLzfr9pvxkA0GpCghyaODJRf/noS731WYnZceCDggLsFBYAwKX7z7F9FBocoBOn68yOAh/ksJt7FgmFBQB8hDPQofvG9DY7BtAmOOkWAABYHoUFAABYHoUFAABYHoUFAABYHoUFAABYHoUFAABYHoUFAABYHoUFAABYHoUFAABYHoUFAABYHoUFAABYHoUFAABYHoUFAABYnk98W7NhGJKkyspKk5MAAICWOvt3++zf8fPxicJy4sQJSVJ8fLzJSQAAgKdOnDihyMjI866xGS2pNRbndrtVUlKi8PBw2Wy2Vn3syspKxcfHq6ioSBEREa362L6GfdVy7KuWY195hv3VcuyrlmurfWUYhk6cOKG4uDjZ7ec/S8UnjrDY7Xb16NGjTX9HREQEL+gWYl+1HPuq5dhXnmF/tRz7quXaYl9d6MjKWZx0CwAALI/CAgAALI/CcgHBwcGaPXu2goODzY5ieeyrlmNftRz7yjPsr5ZjX7WcFfaVT5x0CwAAfBtHWAAAgOVRWAAAgOVRWAAAgOVRWAAAgOVRWCTNnTtXSUlJcjqdSk1N1ebNm8+7/rXXXlP//v3ldDo1aNAgvfvuu+2U1Hye7KulS5fKZrM1uTmdznZMa55PPvlEN910k+Li4mSz2bR69eoLbvPxxx9ryJAhCg4OVp8+fbR06dI2z2kFnu6rjz/++HuvK5vNptLS0vYJbKI5c+Zo+PDhCg8PV7du3ZSZmam9e/decDt/fM+6mH3lr+9Zf/vb33TFFVc0DoVLS0vTe++9d95tzHhN+X1hWbFihWbMmKHZs2dr69atSk5OVkZGhsrLy5tdn5OTowkTJmjq1Knatm2bMjMzlZmZqR07drRz8vbn6b6SGqYiHjp0qPFWWFjYjonNU1VVpeTkZM2dO7dF6/fv368bb7xRY8eOVX5+vh555BFNmzZN77//fhsnNZ+n++qsvXv3NnltdevWrY0SWsf69ev1wAMPaOPGjVq7dq3q6up03XXXqaqq6pzb+Ot71sXsK8k/37N69OihP/7xj8rLy9OWLVt09dVX6yc/+Yl27tzZ7HrTXlOGnxsxYoTxwAMPNP7b5XIZcXFxxpw5c5pdf/vttxs33nhjk/tSU1ONn//8522a0wo83VdLliwxIiMj2ymddUky3njjjfOueeyxx4wBAwY0uW/8+PFGRkZGGyaznpbsq3/961+GJOPYsWPtksnKysvLDUnG+vXrz7nGn9+zvq0l+4r3rG906tTJWLhwYbM/M+s15ddHWGpra5WXl6dx48Y13me32zVu3Djl5uY2u01ubm6T9ZKUkZFxzvW+4mL2lSSdPHlSiYmJio+PP29j93f++rq6FCkpKYqNjdW1116r7Oxss+OYoqKiQpLUuXPnc67htdWgJftK4j3L5XLplVdeUVVVldLS0ppdY9Zryq8Ly+HDh+VyuRQdHd3k/ujo6HN+Hl5aWurRel9xMfuqX79+Wrx4sf75z39q+fLlcrvdSk9P18GDB9sjslc51+uqsrJSp06dMimVNcXGxmrevHlatWqVVq1apfj4eF111VXaunWr2dHaldvt1iOPPKJRo0Zp4MCB51znr+9Z39bSfeXP71nbt29Xhw4dFBwcrPvuu09vvPGGLr/88mbXmvWa8olva4Y1paWlNWno6enpuuyyy/T3v/9dv//9701MBm/Wr18/9evXr/Hf6enp+uqrr/TnP/9ZL774oonJ2tcDDzygHTt2KCsry+woltfSfeXP71n9+vVTfn6+KioqtHLlSk2aNEnr168/Z2kxg18fYenatascDofKysqa3F9WVqaYmJhmt4mJifFova+4mH31XYGBgRo8eLC+/PLLtojo1c71uoqIiFBISIhJqbzHiBEj/Op19eCDD+rtt9/Wv/71L/Xo0eO8a/31PessT/bVd/nTe1ZQUJD69OmjoUOHas6cOUpOTtZzzz3X7FqzXlN+XViCgoI0dOhQrVu3rvE+t9utdevWnfOzu7S0tCbrJWnt2rXnXO8rLmZffZfL5dL27dsVGxvbVjG9lr++rlpLfn6+X7yuDMPQgw8+qDfeeEMfffSRevbsecFt/PW1dTH76rv8+T3L7Xarpqam2Z+Z9ppq01N6vcArr7xiBAcHG0uXLjV27dpl/OxnPzM6duxolJaWGoZhGBMnTjRmzpzZuD47O9sICAgwnn76aWP37t3G7NmzjcDAQGP79u1mPYV24+m++u///m/j/fffN7766isjLy/PuOOOOwyn02ns3LnTrKfQbk6cOGFs27bN2LZtmyHJ+NOf/mRs27bNKCwsNAzDMGbOnGlMnDixcf2+ffuM0NBQ49FHHzV2795tzJ0713A4HMaaNWvMegrtxtN99ec//9lYvXq18cUXXxjbt283Hn74YcNutxsffvihWU+h3dx///1GZGSk8fHHHxuHDh1qvFVXVzeu4T2rwcXsK399z5o5c6axfv16Y//+/cbnn39uzJw507DZbMYHH3xgGIZ1XlN+X1gMwzD++te/GgkJCUZQUJAxYsQIY+PGjY0/GzNmjDFp0qQm61999VWjb9++RlBQkDFgwADjnXfeaefE5vFkXz3yyCONa6Ojo40bbrjB2Lp1qwmp29/ZS2+/ezu7fyZNmmSMGTPme9ukpKQYQUFBRq9evYwlS5a0e24zeLqvnnrqKaN3796G0+k0OnfubFx11VXGRx99ZE74dtbcfpLU5LXCe1aDi9lX/vqeNWXKFCMxMdEICgoyoqKijGuuuaaxrBiGdV5TNsMwjLY9hgMAAHBp/PocFgAA4B0oLAAAwPIoLAAAwPIoLAAAwPIoLAAAwPIoLAAAwPIoLAAAwPIoLAAAwPIoLAAAwPIoLAAAwPIoLAAAwPIoLAAAwPL+P0kElGl5EZF4AAAAAElFTkSuQmCC\n" + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "re = load_log(log_file=\"rollout.csv\", log_key=\"ep_rew_mean\")\n", + "mean = np.stack(list(re.values())).mean(axis=0)\n", + "std = np.stack(list(re.values())).std(axis=0)\n", + "\n", + "plt.plot(mean)\n", + "plt.fill_between(np.arange(len(mean)), mean - std, mean + std, alpha=0.5)" + ], + "metadata": { + "collapsed": false + } + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [], + "metadata": { + "collapsed": false + } + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 2 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython2", + "version": "2.7.6" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} From d1151dd94d966a98997812845410677dbc397d80 Mon Sep 17 00:00:00 2001 From: frank Date: Thu, 30 Mar 2023 22:26:34 +0800 Subject: [PATCH 63/68] :tada: add exp_reader --- exp_reader.ipynb | 88 +++++++++++++++++++++++++++++++++--------------- 1 file changed, 61 insertions(+), 27 deletions(-) diff --git a/exp_reader.ipynb b/exp_reader.ipynb index a585a3b..99b7cc0 100644 --- a/exp_reader.ipynb +++ b/exp_reader.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 17, + "execution_count": 67, "metadata": { "collapsed": true }, @@ -14,12 +14,13 @@ "from emei.core import get_params_str\n", "from pathlib import Path\n", "import matplotlib.pyplot as plt\n", - "import yaml" + "import yaml\n", + "from collections import defaultdict" ] }, { "cell_type": "code", - "execution_count": 34, + "execution_count": 109, "outputs": [], "source": [ "# 递归判断a字典中存在的值是否与b字典相等\n", @@ -41,9 +42,28 @@ " elif isinstance(key, tuple):\n", " if key[0] not in d:\n", " raise ValueError(f\"{key[0]} not in dict\")\n", - " return get_value(d[key[0]], key[1:])\n", + " if len(key) <= 1:\n", + " raise ValueError(\"length of tuple-key must be 2\")\n", + " return get_value(d[key[0]], key[1])\n", " else:\n", - " raise ValueError(\"key must be str or tuple\")" + " raise ValueError(\"key must be str or tuple\")\n", + "\n", + "\n", + "# 返回多个字典中不同的value对应的key, 通过递归的方法\n", + "def get_diff_key(dicts):\n", + " if len(dicts) <= 1:\n", + " return []\n", + " keys = set(dicts[0].keys())\n", + " for d in dicts[1:]:\n", + " keys = keys & set(d.keys())\n", + "\n", + " diff_keys = []\n", + " for k in keys:\n", + " if isinstance(dicts[0][k], dict):\n", + " diff_keys += [(k, dk) for dk in get_diff_key([d[k] for d in dicts])]\n", + " elif not all([dicts[0][k] == d[k] for d in dicts[1:]]):\n", + " diff_keys.append(k)\n", + " return diff_keys" ], "metadata": { "collapsed": false @@ -51,7 +71,7 @@ }, { "cell_type": "code", - "execution_count": 42, + "execution_count": 125, "outputs": [], "source": [ "default_params = dict(freq_rate=1,\n", @@ -70,11 +90,11 @@ " dataset=\"SAC-expert-replay\",\n", " custom_cfg=default_custom_cfg,\n", " log_file=\"rollout.csv\",\n", - " log_key=\"ep_rew_mean\",\n", - " result_key=default_result_key):\n", + " log_key=\"ep_rew_mean\"):\n", " path = Path(\"./exp\") / exp_name / task_name / get_params_str(params) / dataset\n", "\n", - " result_dict = {}\n", + " result_list = []\n", + " cfg_list = []\n", " for time_dir in path.glob(r\"*\"):\n", " if not time_dir.is_dir() or not (time_dir / \".hydra\").exists():\n", " continue\n", @@ -90,10 +110,15 @@ " if not log_path.exists():\n", " continue\n", "\n", - " key_name = tuple([str(get_value(cfg, k)) for k in result_key])\n", " df = pd.read_csv(log_path)\n", - " result_dict[key_name] = df[log_key].to_numpy()\n", - " return result_dict" + " result_list.append(df[log_key].to_numpy())\n", + " cfg_list.append(cfg)\n", + "\n", + " diff_key = get_diff_key(cfg_list)\n", + " result_dict = {}\n", + " for i, cfg in enumerate(cfg_list):\n", + " result_dict[tuple([get_value(cfg, key) for key in diff_key])] = result_list[i]\n", + " return diff_key, result_dict" ], "metadata": { "collapsed": false @@ -101,32 +126,41 @@ }, { "cell_type": "code", - "execution_count": 54, + "execution_count": 136, "outputs": [ - { - "data": { - "text/plain": "" - }, - "execution_count": 54, - "metadata": {}, - "output_type": "execute_result" - }, { "data": { "text/plain": "
", - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAiwAAAGsCAYAAAD+L/ysAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy88F64QAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA25ElEQVR4nO3deXRU9f3/8dfMZJkkJGELWSALyxdQwIQ1JGARRVO1tqkboiKytfpVq6XVQr+/it9v+5V6jrbayreUHUUrCkpdUcSKZAEkEGWvCgkhIYlsCSSQZeb+/ghEowEykOTemXk+zplzZPK5k9fcM2fy8s6977EZhmEIAADAwuxmBwAAALgQCgsAALA8CgsAALA8CgsAALA8CgsAALA8CgsAALA8CgsAALA8CgsAALA8CgsAALA8CgsAALA8CoukmpoapaSkyGazKT8//7xrf/7zn6t3794KCQlRVFSUfvKTn2jPnj1N1thstu/dXnnllcaff/zxx82uKS0tbXHmzz77TBMmTFB8fLxCQkJ02WWX6bnnnvPoeQMA4C18vrBcddVVWrp06XnXPPbYY4qLi2vR4w0dOlRLlizR7t279f7778swDF133XVyuVxN1i1ZskSHDh1qvGVmZn7vsfbu3dtkTbdu3Vr6tJSXl6du3bpp+fLl2rlzp/7rv/5Ls2bN0vPPP9/ixwAAwFsEmB3AbO+9954++OADrVq1Su+9994F1//sZz9r/O+kpCT94Q9/UHJysgoKCtS7d+/Gn3Xs2FExMTHnfaxu3bqpY8eOzf7M7Xbrqaee0vz581VaWqq+ffvqd7/7nW699VZJ0pQpU5qs79Wrl3Jzc/X666/rwQcfvODzAADAm/j8EZbzKSsr0/Tp0/Xiiy8qNDTU4+2rqqq0ZMkS9ezZU/Hx8U1+9sADD6hr164aMWKEFi9erOa+FDslJUWxsbG69tprlZ2d3eRnc+bM0QsvvKB58+Zp586d+uUvf6m7775b69evP2eeiooKde7c2ePnAQCA1fntERbDMHTvvffqvvvu07Bhw1RQUNDibf/v//5Pjz32mKqqqtSvXz+tXbtWQUFBjT//n//5H1199dUKDQ3VBx98oP/8z//UyZMn9Ytf/EKSFBsbq3nz5mnYsGGqqanRwoULddVVV2nTpk0aMmSIampq9OSTT+rDDz9UWlqapIYjKFlZWfr73/+uMWPGfC9TTk6OVqxYoXfeeefSdgwAABZkM5r7X38v9uSTT+rJJ59s/PepU6cUGBiogIBvutmuXbu0evVqvfrqq1q/fr0cDocKCgrUs2dPbdu2TSkpKef9HRUVFSovL9ehQ4f09NNPq7i4WNnZ2XI6nc2uf/zxx7VkyRIVFRWd8zHHjBmjhIQEvfjii9q5c6cGDhyosLCwJmtqa2s1ePBgbdq0qcn9O3bs0NixY/Xwww/r//2//3fe7AAAeCOfKyxHjx7V0aNHG/9911136ZZbbtHNN9/ceF9SUpJuvfVWvfXWW7LZbI33u1wuORwO3XXXXVq2bFmLfl9tba06deqkhQsXasKECc2ueeedd/SjH/1Ip0+fVnBwcLNrHn30UWVlZSk3N1ebNm3SyJEj9fHHH6t79+5N1gUHBzf5+GnXrl0aO3aspk2bpv/93/9tUWYAALyNz30k1Llz5ybncYSEhKhbt27q06dPk3V/+ctf9Ic//KHx3yUlJcrIyNCKFSuUmpra4t9nGIYMw1BNTc051+Tn56tTp07nLCtn18TGxkqSLr/8cgUHB+vAgQPNfvxz1s6dO3X11Vdr0qRJlBUAgE/zucLSUgkJCU3+3aFDB0lS79691aNHD0lScXGxrrnmGr3wwgsaMWKE9u3bpxUrVui6665TVFSUDh48qD/+8Y8KCQnRDTfcIEl66623VFZWppEjR8rpdGrt2rV68skn9etf/7rxdz377LPq2bOnBgwYoNOnT2vhwoX66KOP9MEHH0iSwsPD9etf/1q//OUv5Xa7NXr0aFVUVCg7O1sRERGaNGmSduzYoauvvloZGRmaMWNG4wwXh8OhqKioNt9/AAC0J78tLC1RV1envXv3qrq6WpLkdDq1YcMGPfvsszp27Jiio6P1gx/8QDk5OY0zVAIDAzV37lz98pe/lGEY6tOnj/70pz9p+vTpjY9bW1urX/3qVyouLlZoaKiuuOIKffjhhxo7dmzjmt///veKiorSnDlztG/fPnXs2FFDhgzRb3/7W0nSypUr9fXXX2v58uVavnx543aJiYkenUAMAIA38LlzWAAAgO/x6zksAADAO1BYAACA5fnEOSxut1slJSUKDw9vcpkyAACwLsMwdOLECcXFxcluP/8xFJ8oLCUlJd8bjQ8AALxDUVFR4xW65+IThSU8PFxSwxOOiIgwOQ0AAGiJyspKxcfHN/4dPx+fKCxnPwaKiIigsAAA4GVacjoHJ90CAADLo7AAAADLo7AAAADLo7AAAADLo7AAAADLo7AAAADLo7AAAADLo7AAAADLo7AAAADLo7AAAADLo7AAAADLo7AAAADL84kvP2wr9S63/vfd3WbHgI/qGBKkn4/pJWegw+woAGB5FJbzcBvSkuwCs2PAhwU4bHpgbB+zYwCA5VFYzsNukx4Y29vsGPBBxcdOaXV+iZblFGj6lb0UFMCnswBwPhSW8whw2PVoRn+zY8AH1da7lfPVEZWfqNFbn5XolqE9zI4EAJbG/9YBJggKsGtSepIkacGGfTIMw9xAAGBxFBbAJHelJigk0KE9pSeU89URs+MAgKVRWACTdAwN0m3DGj4KWrBhn8lpAMDaKCyAiaaM6imbTfp479f6ouyE2XEAwLIoLICJkrqG6drLoiVJi7L2m5wGAKyLwgKYbPoPekmSXt9WrMMna0xOAwDWRGEBTDYssZOSe0Sqtt6tF3MLzY4DAJZEYQFMZrPZNO3KhqMsyzcW6nSdy+REAGA9FBbAAq4fGKPuHUN0pKpWb2wrNjsOAFgOhQWwgACHXZNHJUlqOPnW7WaQHAB8G4UFsIjbh8erQ3CAviw/qfX//trsOABgKRQWwCIinIG6Y3i8JGlhFoPkAODbKCyAhdw7KkkOu03ZXx7RzpIKs+MAgGVQWAAL6dEpVNcPjJHEIDkA+DYKC2AxZy9xfuuzEpVVnjY5DQBYA4UFsJiU+I4antRJdS5Dy3IKzI4DAJZAYQEsaOrohqMsL206oOraepPTAID5KCyABV17ebQSu4Sq4lSdXtty0Ow4AGA6CgtgQQ67TVNG9ZQkLc7eLxeD5AD4OQoLYFG3DeuhyJBAFR6p1tpdZWbHAQBTUVgAiwoNCtCdqQmSpEUMkgPg5ygsgIXdm56kQIdNnxYcU37RcbPjAIBpKCyAhUVHOHXTFXGSpIUbOMoCwH9RWACLm3plw8m37+0o1cFj1SanAQBzUFgAixsQF6n03l3kchtaml1gdhwAMAWFBfAC08+M63/l0yJVnq4zOQ0AtD8KC+AFxvSNUu+oMJ2sqdernxaZHQcA2h2FBfACdrut8UsRl2QXqN7lNjkRALQvCgvgJX46uLu6hAWp+Pgpvbej1Ow4ANCuKCyAl3AGOnT3yERJDZc4Gwbj+gH4DwoL4EUmpiUqKMCuzw5WaEvhMbPjAEC7obAAXqRrh2DdPLi7JGnBJwySA+A/KCyAl5k6umGQ3NrdZSo4XGVyGgBoHxQWwMv8R3S4ruoXJcOQFmfvNzsOALQLCgvghc4Okntty0Edr641OQ0AtD0KC+CF0nt3Uf+YcJ2qc+mlTQfMjgMAbY7CAnghm83WeJRlWU6BausZJAfAt1FYAC91U3KcuoUHq/xEjd76rMTsOADQpigsgJcKCrBrUnqSJGlh1n4GyQHwaRQWwIvdlZqgkECHdh+qVM5XR8yOAwBthsICeLGOoUG6bVgPSQ3j+gHAV1FYAC83ZVRP2WzSv/Z+rS/LT5gdBwDaBIUF8HJJXcN07WXRkqRFWQySA+CbKCyAD5h25hLnVVuLdfhkjclpAKD1UVgAHzA8qZOSe0Sqtt6t5RsLzY4DAK2OwgL4AJvNpqlnjrK8mFuo03UukxMBQOuisAA+4oaBMereMURHqmq1elux2XEAoFVRWAAfEeCw695vDZJzuxkkB8B3UFgAHzJ+RLw6BAfoy/KTWv/F12bHAYBWQ2EBfEiEM1Djh8dLYpAcAN9CYQF8zORRSbLbpOwvj2hnSYXZcQCgVVxUYZk7d66SkpLkdDqVmpqqzZs3n3f9a6+9pv79+8vpdGrQoEF69913z7n2vvvuk81m07PPPnsx0QC/16NTqK4fFCuJQXIAfIfHhWXFihWaMWOGZs+era1btyo5OVkZGRkqLy9vdn1OTo4mTJigqVOnatu2bcrMzFRmZqZ27NjxvbVvvPGGNm7cqLi4OM+fCYBG089c4vzWZyUqqzxtchoAuHQeF5Y//elPmj59uiZPnqzLL79c8+bNU2hoqBYvXtzs+ueee04//OEP9eijj+qyyy7T73//ew0ZMkTPP/98k3XFxcV66KGH9NJLLykwMPDing0ASVJKfEcNS+ykOpehZTkFZscBgEvmUWGpra1VXl6exo0b980D2O0aN26ccnNzm90mNze3yXpJysjIaLLe7XZr4sSJevTRRzVgwIAL5qipqVFlZWWTG4Cmzo7rf2nTAVXX1pucBgAujUeF5fDhw3K5XIqOjm5yf3R0tEpLS5vdprS09ILrn3rqKQUEBOgXv/hFi3LMmTNHkZGRjbf4+HhPngbgF669PFqJXUJVcapOK/MOmh0HAC6J6VcJ5eXl6bnnntPSpUtls9latM2sWbNUUVHReCsqKmrjlID3cdhtmjKqp6SGk29dDJID4MU8Kixdu3aVw+FQWVlZk/vLysoUExPT7DYxMTHnXb9hwwaVl5crISFBAQEBCggIUGFhoX71q18pKSmp2ccMDg5WREREkxuA77ttWA9FhgSq8Ei1PtxdduENAMCiPCosQUFBGjp0qNatW9d4n9vt1rp165SWltbsNmlpaU3WS9LatWsb10+cOFGff/658vPzG29xcXF69NFH9f7773v6fAB8S2hQgO5MTZDEIDkA3i3A0w1mzJihSZMmadiwYRoxYoSeffZZVVVVafLkyZKke+65R927d9ecOXMkSQ8//LDGjBmjZ555RjfeeKNeeeUVbdmyRfPnz5ckdenSRV26dGnyOwIDAxUTE6N+/fpd6vMD/N696UlauGGfPi04pvyi40qJ72h2JADwmMfnsIwfP15PP/20Hn/8caWkpCg/P19r1qxpPLH2wIEDOnToUOP69PR0vfzyy5o/f76Sk5O1cuVKrV69WgMHDmy9ZwHgnKIjnLrpiobZRhxlAeCtbIZheP2ZeJWVlYqMjFRFRQXnswDN2FlSoRv/kiWH3ab1j16lHp1CzY4EAB79/Tb9KiEAbW9AXKTSe3eRy21oaXaB2XEAwGMUFsBPnB3X/8qnRTpxus7kNADgGQoL4CfG9I1S76gwnayp14pPmV0EwLtQWAA/YbfbGsf1L8kuUL3LbXIiAGg5CgvgR346uLu6hAWp+Pgpvbej+a/TAAArorAAfsQZ6NDdIxMlNVzi7AMXCQLwExQWwM9MTEtUUIBdnx2s0JbCY2bHAYAWobAAfqZrh2DdPLi7JAbJAfAeFBbAD00d3fAtzh/sKlPB4SqT0wDAhVFYAD/0H9HhuqpflAxDWpK93+w4AHBBFBbAT00b3XCJ86tbDup4da3JaQDg/CgsgJ8a1aeL+seE61SdSy9vPmB2HAA4LwoL4Kdstm8GyS3LKVBtPYPkAFgXhQXwYz9OjlO38GCVVdbo7c9LzI4DAOdEYQH8WFCAXZPSkyRJCzbsZ5AcAMuisAB+7q7UBIUEOrT7UKVyvzpidhwAaBaFBfBzHUODdOvQHpKkBQySA2BRFBYAmjK6p2w26V97v9aX5SfMjgMA30NhAaCeXcM07rJoSdKiLAbJAbAeCgsASdL0M5c4r9parMMna0xOAwBNUVgASJKGJ3XSFT0iVVvv1vKNhWbHAYAmKCwAJDUdJPdibqFO17lMTgQA36CwAGh0/cAYxUU6daSqVqu3FZsdBwAaUVgANAp02DV5VE9J0sKs/XK7GSQHwBooLACaGD8iXh2CA/Rl+Umt/+Jrs+MAgCQKC4DviHAGavzweEnSQgbJAbAICguA75k8Kkl2m5T95RHtKqk0Ow4AUFgAfF+PTqG6flCsJGlhFkdZAJiPwgKgWWcHyb31WYnKKk+bnAaAv6OwAGhWSnxHDUvspDqXoWU5BWbHAeDnKCwAzunsILmXNh1QdW29yWkA+DMKC4BzuvbyaCV2CVXFqTqtzDtodhwAfozCAuCcHHabppwZJLc4a79cDJIDYBIKC4DzunVoD0U4A1RwpFof7i4zOw4AP0VhAXBeYcEBumtkoiRp0Yb9JqcB4K8oLAAuaFJakgLsNm0uOKrPio6bHQeAH6KwALigmEinfpwcJ6nhSxEBoL1RWAC0yNQrG06+fXf7IRUfP2VyGgD+hsICoEUGxEUqvXcXudyGlmZzlAVA+6KwAGixaWeOsryyuUgnTteZnAaAP6GwAGixq/p2U++oMJ2oqdeKT4vMjgPAj1BYALSY3W7T1NEN4/qXZBeo3uU2OREAf0FhAeCRm4d0V+ewIBUfP6U1O0vNjgPAT1BYAHjEGejQ3WcGyS3YsF+Gwbh+AG2PwgLAYxNHJioowK7Pio4rr/CY2XEA+AEKCwCPRYUH66cp3SVJCzbsMzkNAH9AYQFwUc4OkvtgV5kKj1SZnAaAr6OwALgofaPDNaZvlAxDWsy4fgBtjMIC4KJNv7LhEudXtxxURTWD5AC0HQoLgIs2qk8X9Y8J16k6l17aXGh2HAA+jMIC4KLZbDZNO3OUZVlOgWrrGSQHoG1QWABckpuSYxUVHqyyyhq9/XmJ2XEA+CgKC4BLEhzg0L3pSZIYJAeg7VBYAFyyO0ckyBlo1+5Dlcr96ojZcQD4IAoLgEvWKSxItw2Nl8QgOQBtg8ICoFVMGd1TNpv0r71f68vyE2bHAeBjKCwAWkXPrmEad1m0JGkRg+QAtDIKC4BWc3aQ3KqtxTpyssbkNAB8CYUFQKsZntRJV/SIVG29Wy9uZJAcgNZDYQHQar49SO7F3EKdrnOZnAiAr6CwAGhV1w+MUVykU0eqarV6W7HZcQD4CAoLgFYV6LBr8qiekqSFWQySA9A6KCwAWt34EfHqEBygL8tP6uN/f212HAA+gMICoNVFOAM1fnjDILlFG7jEGcClo7AAaBP3pifJbpOyvjysXSWVZscB4OUuqrDMnTtXSUlJcjqdSk1N1ebNm8+7/rXXXlP//v3ldDo1aNAgvfvuu01+/sQTT6h///4KCwtTp06dNG7cOG3atOliogGwiPjOobp+UKwkBskBuHQeF5YVK1ZoxowZmj17trZu3ark5GRlZGSovLy82fU5OTmaMGGCpk6dqm3btikzM1OZmZnasWNH45q+ffvq+eef1/bt25WVlaWkpCRdd911+vprPvsGvNm00Q0n3775WbHKKk+bnAaAN7MZHp7Cn5qaquHDh+v555+XJLndbsXHx+uhhx7SzJkzv7d+/Pjxqqqq0ttvv91438iRI5WSkqJ58+Y1+zsqKysVGRmpDz/8UNdcc80FM51dX1FRoYiICE+eDoA2duvfcrSl8JgeGNtbj2b0NzsOAAvx5O+3R0dYamtrlZeXp3Hjxn3zAHa7xo0bp9zc3Ga3yc3NbbJekjIyMs65vra2VvPnz1dkZKSSk5ObXVNTU6PKysomNwDWNO3KhqMsyzceUHVtvclpAHgrjwrL4cOH5XK5FB0d3eT+6OholZaWNrtNaWlpi9a//fbb6tChg5xOp/785z9r7dq16tq1a7OPOWfOHEVGRjbe4uPjPXkaANrRtZfHKKFzqCpO1WlV3kGz4wDwUpa5Smjs2LHKz89XTk6OfvjDH+r2228/53kxs2bNUkVFReOtqKiondMCaCmH3aYpo5IkNZx863IzSA6A5zwqLF27dpXD4VBZWVmT+8vKyhQTE9PsNjExMS1aHxYWpj59+mjkyJFatGiRAgICtGjRomYfMzg4WBEREU1uAKzrtmHxinAGqOBItdbtLrvwBgDwHR4VlqCgIA0dOlTr1q1rvM/tdmvdunVKS0trdpu0tLQm6yVp7dq151z/7cetqeHr6QFfEBYcoDtTEyVJCxkkB+AiePyR0IwZM7RgwQItW7ZMu3fv1v3336+qqipNnjxZknTPPfdo1qxZjesffvhhrVmzRs8884z27NmjJ554Qlu2bNGDDz4oSaqqqtJvf/tbbdy4UYWFhcrLy9OUKVNUXFys2267rZWeJgCz3ZuepAC7TZsLjuqzouNmxwHgZTwuLOPHj9fTTz+txx9/XCkpKcrPz9eaNWsaT6w9cOCADh061Lg+PT1dL7/8subPn6/k5GStXLlSq1ev1sCBAyVJDodDe/bs0S233KK+ffvqpptu0pEjR7RhwwYNGDCglZ4mALPFRDp1U3KcpIYvRQQAT3g8h8WKmMMCeIcdxRX60V+z5LDb9MljY9W9Y4jZkQCYqM3msADApRjYPVJpvbrI5Ta0NJujLABajsICoF1N/0HDILlXNhfpxOk6k9MA8BYUFgDt6qq+3dQrKkwnauq14lNmKAFoGQoLgHZlt9s0bXQvSdKS7ALVu9wmJwLgDSgsANrdzUO6q3NYkIqPn9Kanc1/rQcAfBuFBUC7cwY6dPfIhkFyCzbslw9crAigjVFYAJhi4shEBQXY9VnRceUVHjM7DgCLo7AAMEVUeLB+mtJdkrRgwz6T0wCwOgoLANNMvbLhEucPdpWp8EiVyWkAWBmFBYBp+kaHa0zfKBmGtJhx/QDOg8ICwFTTr2y4xPnVLQdVUc0gOQDNo7AAMNWoPl3UPyZcp+pcemlzodlxAFgUhQWAqWw2m6adOcqyLKdAtfUMkgPwfRQWAKa7KTlWUeHBKqus0dufl5gdB4AFUVgAmC44wKF705MkSQsZJAegGRQWAJZw54gEOQPt2nWoUrlfHTE7DgCLobAAsIROYUG6bWi8JGkhlzgD+A4KCwDLmDK6p2w26aM95fqy/ITZcQBYCIUFgGX07BqmcZdFS5IWZRWYGwaApVBYAFjKtNEN4/pf33pQR07WmJwGgFVQWABYyoienXVFj0jV1Lu1fOMBs+MAsAgKCwBLsdlsmnrmKMuLGwt0us5lciIAVkBhAWA5NwyKVVykU4dP1uqf+cVmxwFgARQWAJYT6LDr3lFJkhgkB6ABhQWAJd0xIkFhQQ59UX5S6//9tdlxAJiMwgLAkiKcgRo/PEFSw1EWAP6NwgLAsiaPSpLdJmV9eVi7D1WaHQeAiSgsACwrvnOorh8YK4mjLIC/o7AAsLRpVzZc4vzmZ8UqrzxtchoAZqGwALC0wQmdNDSxk+pchpblFpgdB4BJKCwALG/6maMsL206oOraepPTADADhQWA5V17eYwSOofqeHWdVuUdNDsOABNQWABYnsNu05Qzg+QWZe2X280gOcDfUFgAeIXbhsUrwhmggiPV+nB3mdlxALQzCgsArxAWHKA7UxMlSQuzuMQZ8DcUFgBeY1J6ogLsNm3ef1SfHzxudhwA7YjCAsBrxEaG6KbkOEkMkgP8DYUFgFeZOrrhEud3th9S8fFTJqcB0F4oLAC8ysDukUrr1UUut6FlOQVmxwHQTigsALzO9B80HGX5x6YDOnG6zuQ0ANoDhQWA17mqbzf1igrTiZp6rfi0yOw4ANoBhQWA17HbbZo2upckaUl2gepdbpMTAWhrFBYAXunmId3VOSxIxcdPac3OUrPjAGhjFBYAXskZ6NDdIxsGyS3YsF+Gwbh+wJdRWAB4rYkjExUUYNdnRceVV3jM7DgA2hCFBYDXigoP1k9TuktikBzg6ygsALza1CsbLnF+f1epCo9UmZwGQFuhsADwan2jwzWmb5QMo+GKIQC+icICwOtNO3OU5dUtRaqoZpAc4IsoLAC83ug+XdU/JlzVtS69vPmA2XEAtAEKCwCvZ7PZGr8UcWnOftXWM0gO8DUUFgA+4ccpcYoKD1ZZZY3e2V5idhwArYzCAsAnBAc4NCntzCC5TxgkB/gaCgsAn3FXaqKcgXbtOlSp3H1HzI4DoBVRWAD4jE5hQbp1aA9JDJIDfA2FBYBPmTKqp2w26aM95fqy/KTZcQC0EgoLAJ/SK6qDrukfLUlalMVRFsBXUFgA+JzpZwbJvb71oI6crDE5DYDWQGEB4HNG9OysQd0jVVPv1vKNDJIDfAGFBYDPsdlsjeP6X9xYoNN1LpMTAbhUFBYAPumGQbGKjXTq8Mla/TO/2Ow4AC4RhQWATwp02DV5VJKkhkucGSQHeDcKCwCfNX54gsKCHPqi/KTW//trs+MAuAQUFgA+KzIkUOOHJ0jiEmfA21FYAPi0yaOSZLdJG744rN2HKs2OA+AiXVRhmTt3rpKSkuR0OpWamqrNmzefd/1rr72m/v37y+l0atCgQXr33Xcbf1ZXV6ff/OY3GjRokMLCwhQXF6d77rlHJSV82yqASxffOVTXD4yVxFEWwJt5XFhWrFihGTNmaPbs2dq6dauSk5OVkZGh8vLyZtfn5ORowoQJmjp1qrZt26bMzExlZmZqx44dkqTq6mpt3bpVv/vd77R161a9/vrr2rt3r3784x9f2jMDgDOmnrnE+Z/5xSqvPG1yGgAXw2Z4eOp8amqqhg8frueff16S5Ha7FR8fr4ceekgzZ8783vrx48erqqpKb7/9duN9I0eOVEpKiubNm9fs7/j00081YsQIFRYWKiEh4YKZKisrFRkZqYqKCkVERHjydAD4iVv+lqO8wmN6cGwf/Tqjn9lxAMizv98eHWGpra1VXl6exo0b980D2O0aN26ccnNzm90mNze3yXpJysjIOOd6SaqoqJDNZlPHjh2b/XlNTY0qKyub3ADgfM6O61++qVDVtfUmpwHgKY8Ky+HDh+VyuRQdHd3k/ujoaJWWlja7TWlpqUfrT58+rd/85jeaMGHCOdvWnDlzFBkZ2XiLj4/35GkA8EPXXh6jhM6hOl5dp1VbGSQHeBtLXSVUV1en22+/XYZh6G9/+9s5182aNUsVFRWNt6KionZMCcAbOew2TTkzSG5x1n653QySA7yJR4Wla9eucjgcKisra3J/WVmZYmJimt0mJiamRevPlpXCwkKtXbv2vJ9lBQcHKyIioskNAC7ktmHxinAGaP/hKn24u+zCGwCwDI8KS1BQkIYOHap169Y13ud2u7Vu3TqlpaU1u01aWlqT9ZK0du3aJuvPlpUvvvhCH374obp06eJJLABokbDgAN2ZmihJWsglzoBX8fgjoRkzZmjBggVatmyZdu/erfvvv19VVVWaPHmyJOmee+7RrFmzGtc//PDDWrNmjZ555hnt2bNHTzzxhLZs2aIHH3xQUkNZufXWW7Vlyxa99NJLcrlcKi0tVWlpqWpra1vpaQJAg0npiQqw27R5/1F9fvC42XEAtJDHhWX8+PF6+umn9fjjjyslJUX5+flas2ZN44m1Bw4c0KFDhxrXp6en6+WXX9b8+fOVnJyslStXavXq1Ro4cKAkqbi4WG+++aYOHjyolJQUxcbGNt5ycnJa6WkCQIPYyBDdlBwnqeFLEQF4B4/nsFgRc1gAeGJHcYV+9NcsOew2ffLYWHXvGGJ2JMAvtdkcFgDwBQO7RyqtVxe53IaW5RSYHQdAC1BYAPilaWcGyf1j0wGdOF1nchoAF0JhAeCXxvbrpl5RYTpRU69Xtxw0Ow6AC6CwAPBLdrtNU0c3HGVZnLVf9S63yYkAnA+FBYDfumVID3UKDVTx8VN6fyeD5AAro7AA8FvOQIcmjmwYJLdgwz75wEWTgM+isADwaxPTkhTksCu/6Li2HjhmdhwA50BhAeDXosKDlTm4YZDcgk8YJAdYFYUFgN+bdmUvSdL7u0pVeKTK5DQAmkNhAeD3+kaH6wd9o2QY0pLsArPjAGgGhQUAJE0/M0ju1S1FqqhmkBxgNRQWAJA0uk9X9Y8JV3WtSy9vPmB2HADfQWEBAEk22zeD5Jbm7FdtPYPkACuhsADAGT9OiVNUeLDKKmv0zvYSs+MA+BYKCwCcERzg0KS0hkFyCzfsZ5AcYCEUFgD4lrtSE+UMtGtnSaVy9x0xOw6AMygsAPAtncKCdOvQHpKkRRsYJAdYBYUFAL5jyqiestmkdXvK9WX5SbPjABCFBQC+p1dUB13TP1qStDiboyyAFVBYAKAZ084MkluVd1BHTtaYnAYAhQUAmpHas7MGdY9UTb1bL21ikBxgNgoLADTDZrM1HmV5IbdAp+tcJicC/BuFBQDO4YZBsYqNdOrwyVq9mc8gOcBMFBYAOIdAh12TRyVJkhZm7WOQHGAiCgsAnMf44QkKC3Lo32Un9ckXh82OA/gtCgsAnEdkSKDGD0+QJC3csM/kNID/orAAwAVMHpUku03a8MVh7SmtNDsO4JcoLABwAfGdQ3X9wFhJDV+KCKD9UVgAoAWmnrnE+Z/5xSqvPG1yGsD/UFgAoAWGJHTS0MROqnMZeiG30Ow4gN+hsABAC00b3XCUZfmmQlXX1pucBvAvFBYAaKHrBsQovnOIjlfXadXWYrPjAH6FwgIALeSw2zRlVMNRlsVZ++V2M0gOaC8UFgDwwO3D4hXuDND+w1Vat6fc7DiA36CwAIAHwoIDdGdqwyC5BQySA9oNhQUAPHRvepIC7DZt3n9Unx88bnYcwC9QWADAQ7GRIfrRFQySA9oThQUALsK0K3tJkt7Zfkglx0+ZnAbwfRQWALgIA7tHamSvznK5DS3NKTA7DuDzKCwAcJGmnznK8o9NB3SyhkFyQFuisADARRrbr5t6RYXpRE29VnxaZHYcwKdRWADgItntNk09M65/SfZ+1bvcJicCfBeFBQAuwc2De6hTaKAOHjul93eWmR0H8FkUFgC4BCFBDk0cmShJWpjFIDmgrVBYAOAS3Z2WqCCHXdsOHFde4VGz4wA+icICAJeoW7hTmYPjJDFIDmgrFBYAaAVTRzdc4vz+zlIdOFJtchrA91BYAKAV9IsJ1w/6RsltSIuzOcoCtDYKCwC0kmlnLnF+dUuRKqrrTE4D+BYKCwC0kiv/o6v6x4Srutalf3x6wOw4gE+hsABAK7HZvhkktzS7QLX1DJIDWguFBQBa0Y9T4hQVHqzSytN6d/shs+MAPoPCAgCtKDjAoUlpDYPkFmzYJ8MwTE4E+AYKCwC0srtSE+UMtGtnSaU27mOQHNAaKCwA0Mo6hQXp1qE9JEkLNzCuH2gNFBYAaANTRvWUzSat21Our74+aXYcwOtRWACgDfSK6qBr+kdLkhZlMUgOuFQUFgBoI9OubLjEeVXeQR2tqjU5DeDdKCwA0EZSe3bWoO6Rqql3a/nGQrPjAF6NwgIAbcRmszUeZXkht0Cn61wmJwK8F4UFANrQDYNiFRvp1OGTtXozv8TsOIDXorAAQBsKdNh1b3qSJGlhFoPkgItFYQGANnbHiASFBTn077KT+uSLw2bHAbwShQUA2lhkSKBuHx4viUFywMW6qMIyd+5cJSUlyel0KjU1VZs3bz7v+tdee039+/eX0+nUoEGD9O677zb5+euvv67rrrtOXbp0kc1mU35+/sXEAgDLmjKqp+w2acMXh7WntNLsOIDX8biwrFixQjNmzNDs2bO1detWJScnKyMjQ+Xl5c2uz8nJ0YQJEzR16lRt27ZNmZmZyszM1I4dOxrXVFVVafTo0Xrqqacu/pkAgIXFdw7VDwfGSJIWbmCQHOApm+HhGWCpqakaPny4nn/+eUmS2+1WfHy8HnroIc2cOfN768ePH6+qqiq9/fbbjfeNHDlSKSkpmjdvXpO1BQUF6tmzp7Zt26aUlJQWZ6qsrFRkZKQqKioUERHhydMBgHaz9cAx3fx/OQp02JT9m6vVLcJpdiTAVJ78/fboCEttba3y8vI0bty4bx7Abte4ceOUm5vb7Da5ublN1ktSRkbGOde3RE1NjSorK5vcAMDqhiR00pCEjqpzGXohl0FygCc8KiyHDx+Wy+VSdHR0k/ujo6NVWlra7DalpaUerW+JOXPmKDIysvEWHx9/0Y8FAO1p+pW9JEnLNxXqVC2D5ICW8sqrhGbNmqWKiorGW1FRkdmRAKBFrhsQo/jOITpeXaeVWw+aHQfwGh4Vlq5du8rhcKisrKzJ/WVlZYqJiWl2m5iYGI/Wt0RwcLAiIiKa3ADAGzjsNk0Z1TCuf3HWfrndDJIDWsKjwhIUFKShQ4dq3bp1jfe53W6tW7dOaWlpzW6TlpbWZL0krV279pzrAcDX3TYsXuHOAO0/XKV1e5q/whJAUx5/JDRjxgwtWLBAy5Yt0+7du3X//ferqqpKkydPliTdc889mjVrVuP6hx9+WGvWrNEzzzyjPXv26IknntCWLVv04IMPNq45evSo8vPztWvXLknS3r17lZ+ff0nnuQCAVXUIDtCdqQmSGCQHtJTHhWX8+PF6+umn9fjjjyslJUX5+flas2ZN44m1Bw4c0KFDhxrXp6en6+WXX9b8+fOVnJyslStXavXq1Ro4cGDjmjfffFODBw/WjTfeKEm64447NHjw4O9d9gwAvuLe9CQF2G3atP+oth+sMDsOYHkez2GxIuawAPBGj7yyTavzS/STlDg9d8dgs+MA7a7N5rAAAFrPtDOXOL/9+SGVHD9lchrA2igsAGCSgd0jNbJXZ7nchpblFJgdB7A0CgsAmGja6IajLC9vPqCTNfUmpwGsi8ICACa6un839eoaphOn6/XqpwzBBM6FwgIAJrLbbZoy+swguez9qne5TU4EWBOFBQBMdsuQHuoUGqiDx07pg11lF94A8EMUFgAwWUiQQxNHJkqSFjBIDmgWhQUALODutEQFOezaduC48gqPmR0HsBwKCwBYQLdwpzIHx0liXD/QHAoLAFjE1DOXOL+/s1QHjlSbnAawFgoLAFhEv5hw/aBvlNxGwxVDAL5BYQEAC5l25hLnV7cUqeJUnclpAOugsACAhVz5H13VLzpc1bUu/WPzAbPjAJZBYQEAC7HZbJp6ZcNRlqXZBapjkBwgicICAJbzk5Q4de0QrNLK03rn80NmxwEsgcICABYTHODQpLSGQXILs/bJMAyTEwHmo7AAgAXdNTJRzkC7dhRXauO+o2bHAUxHYQEAC+ocFqRbhvSQxCA5QKKwAIBlTT1zifO6PeX66uuTJqcBzEVhAQCL6hXVQeMu6yZJWpTFIDn4NwoLAFjYtCsbxvWvyjuoo1W1JqcBzENhAQALS+3ZWQO7R6im3q3lGwvNjgOYhsICABZms9k0/cxRlhdyC3S6zmVyIsAcFBYAsLgbBsUqNtKpwydr9WZ+idlxAFNQWADA4gIddt2bniSJQXLwXxQWAPACd4xIUFiQQ/8uO6lPvjhsdhyg3VFYAMALRIYE6vbh8ZIYJAf/RGEBAC8xZVRP2W3Shi8Oa09ppdlxgHZFYQEALxHfOVQ/HBgjSVq0gUFy8C8UFgDwIlNHN1zi/M/8EpWfOG1yGqD9UFgAwIsMTeykIQkdVety68VcBsnBf1BYAMDLnB3Xv3xjoU7VMkgO/oHCAgBeJmNAjOI7h+hYdZ1WbT1odhygXVBYAMDLOOw2TU7vKUlanLVfbjeD5OD7KCwA4IVuHx6vcGeA9h2u0kd7ys2OA7Q5CgsAeKEOwQG6MzVBkrSAQXLwAxQWAPBS96YnKcBu06b9R7X9YIXZcYA2RWEBAC8VGxmiH10RK6nhSxEBX0ZhAQAvdvYS53c+P6SS46dMTgO0HQoLAHixgd0jNbJXZ9W7DS3LKTA7DtBmKCwA4OWmnRnX//LmAzpZU29yGqBtUFgAwMtd3b+benUN04nT9Xr10yKz4wBtgsICAF7Obrdpyugzg+Sy98vFIDn4oACzAwAALt0tQ3romQ/26uCxU3pkRb66dggyOxJ8TIDdpv+68XLzfr9pvxkA0GpCghyaODJRf/noS731WYnZceCDggLsFBYAwKX7z7F9FBocoBOn68yOAh/ksJt7FgmFBQB8hDPQofvG9DY7BtAmOOkWAABYHoUFAABYHoUFAABYHoUFAABYHoUFAABYHoUFAABYHoUFAABYHoUFAABYHoUFAABYHoUFAABYHoUFAABYHoUFAABYHoUFAABYnk98W7NhGJKkyspKk5MAAICWOvt3++zf8fPxicJy4sQJSVJ8fLzJSQAAgKdOnDihyMjI866xGS2pNRbndrtVUlKi8PBw2Wy2Vn3syspKxcfHq6ioSBEREa362L6GfdVy7KuWY195hv3VcuyrlmurfWUYhk6cOKG4uDjZ7ec/S8UnjrDY7Xb16NGjTX9HREQEL+gWYl+1HPuq5dhXnmF/tRz7quXaYl9d6MjKWZx0CwAALI/CAgAALI/CcgHBwcGaPXu2goODzY5ieeyrlmNftRz7yjPsr5ZjX7WcFfaVT5x0CwAAfBtHWAAAgOVRWAAAgOVRWAAAgOVRWAAAgOVRWCTNnTtXSUlJcjqdSk1N1ebNm8+7/rXXXlP//v3ldDo1aNAgvfvuu+2U1Hye7KulS5fKZrM1uTmdznZMa55PPvlEN910k+Li4mSz2bR69eoLbvPxxx9ryJAhCg4OVp8+fbR06dI2z2kFnu6rjz/++HuvK5vNptLS0vYJbKI5c+Zo+PDhCg8PV7du3ZSZmam9e/decDt/fM+6mH3lr+9Zf/vb33TFFVc0DoVLS0vTe++9d95tzHhN+X1hWbFihWbMmKHZs2dr69atSk5OVkZGhsrLy5tdn5OTowkTJmjq1Knatm2bMjMzlZmZqR07drRz8vbn6b6SGqYiHjp0qPFWWFjYjonNU1VVpeTkZM2dO7dF6/fv368bb7xRY8eOVX5+vh555BFNmzZN77//fhsnNZ+n++qsvXv3NnltdevWrY0SWsf69ev1wAMPaOPGjVq7dq3q6up03XXXqaqq6pzb+Ot71sXsK8k/37N69OihP/7xj8rLy9OWLVt09dVX6yc/+Yl27tzZ7HrTXlOGnxsxYoTxwAMPNP7b5XIZcXFxxpw5c5pdf/vttxs33nhjk/tSU1ONn//8522a0wo83VdLliwxIiMj2ymddUky3njjjfOueeyxx4wBAwY0uW/8+PFGRkZGGyaznpbsq3/961+GJOPYsWPtksnKysvLDUnG+vXrz7nGn9+zvq0l+4r3rG906tTJWLhwYbM/M+s15ddHWGpra5WXl6dx48Y13me32zVu3Djl5uY2u01ubm6T9ZKUkZFxzvW+4mL2lSSdPHlSiYmJio+PP29j93f++rq6FCkpKYqNjdW1116r7Oxss+OYoqKiQpLUuXPnc67htdWgJftK4j3L5XLplVdeUVVVldLS0ppdY9Zryq8Ly+HDh+VyuRQdHd3k/ujo6HN+Hl5aWurRel9xMfuqX79+Wrx4sf75z39q+fLlcrvdSk9P18GDB9sjslc51+uqsrJSp06dMimVNcXGxmrevHlatWqVVq1apfj4eF111VXaunWr2dHaldvt1iOPPKJRo0Zp4MCB51znr+9Z39bSfeXP71nbt29Xhw4dFBwcrPvuu09vvPGGLr/88mbXmvWa8olva4Y1paWlNWno6enpuuyyy/T3v/9dv//9701MBm/Wr18/9evXr/Hf6enp+uqrr/TnP/9ZL774oonJ2tcDDzygHTt2KCsry+woltfSfeXP71n9+vVTfn6+KioqtHLlSk2aNEnr168/Z2kxg18fYenatascDofKysqa3F9WVqaYmJhmt4mJifFova+4mH31XYGBgRo8eLC+/PLLtojo1c71uoqIiFBISIhJqbzHiBEj/Op19eCDD+rtt9/Wv/71L/Xo0eO8a/31PessT/bVd/nTe1ZQUJD69OmjoUOHas6cOUpOTtZzzz3X7FqzXlN+XViCgoI0dOhQrVu3rvE+t9utdevWnfOzu7S0tCbrJWnt2rXnXO8rLmZffZfL5dL27dsVGxvbVjG9lr++rlpLfn6+X7yuDMPQgw8+qDfeeEMfffSRevbsecFt/PW1dTH76rv8+T3L7Xarpqam2Z+Z9ppq01N6vcArr7xiBAcHG0uXLjV27dpl/OxnPzM6duxolJaWGoZhGBMnTjRmzpzZuD47O9sICAgwnn76aWP37t3G7NmzjcDAQGP79u1mPYV24+m++u///m/j/fffN7766isjLy/PuOOOOwyn02ns3LnTrKfQbk6cOGFs27bN2LZtmyHJ+NOf/mRs27bNKCwsNAzDMGbOnGlMnDixcf2+ffuM0NBQ49FHHzV2795tzJ0713A4HMaaNWvMegrtxtN99ec//9lYvXq18cUXXxjbt283Hn74YcNutxsffvihWU+h3dx///1GZGSk8fHHHxuHDh1qvFVXVzeu4T2rwcXsK399z5o5c6axfv16Y//+/cbnn39uzJw507DZbMYHH3xgGIZ1XlN+X1gMwzD++te/GgkJCUZQUJAxYsQIY+PGjY0/GzNmjDFp0qQm61999VWjb9++RlBQkDFgwADjnXfeaefE5vFkXz3yyCONa6Ojo40bbrjB2Lp1qwmp29/ZS2+/ezu7fyZNmmSMGTPme9ukpKQYQUFBRq9evYwlS5a0e24zeLqvnnrqKaN3796G0+k0OnfubFx11VXGRx99ZE74dtbcfpLU5LXCe1aDi9lX/vqeNWXKFCMxMdEICgoyoqKijGuuuaaxrBiGdV5TNsMwjLY9hgMAAHBp/PocFgAA4B0oLAAAwPIoLAAAwPIoLAAAwPIoLAAAwPIoLAAAwPIoLAAAwPIoLAAAwPIoLAAAwPIoLAAAwPIoLAAAwPIoLAAAwPL+P0kElGl5EZF4AAAAAElFTkSuQmCC\n" + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAigAAAGdCAYAAAA44ojeAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy88F64QAAAACXBIWXMAAA9hAAAPYQGoP6dpAABGY0lEQVR4nO3deXhU9d3+8fdkspGQRUI2ZJE1IbKDQnBDQQIEKo+2dkFBa9X6AxUXqlRaCy5Yi3UrVmtb0WofrPVxQ0AQBRcQYwANCAFZAyGENZMQss2c3x8nmSQQIBOSnFnu13XN5dlm5jOn08zNOefzPTbDMAxEREREvEiQ1QWIiIiInEwBRURERLyOAoqIiIh4HQUUERER8ToKKCIiIuJ1FFBERETE6yigiIiIiNdRQBERERGvE2x1AU3hcrnIz88nKioKm81mdTkiIiLSCIZhUFxcTIcOHQgKOvMxEp8MKPn5+XTq1MnqMkRERKQJ8vLy6Nix4xm38cmAEhUVBZgfMDo62uJqREREpDEcDgedOnVy/46fiU8GlJrTOtHR0QooIiIiPqYxl2foIlkRERHxOgooIiIi4nUUUERERMTr+OQ1KI1hGAZVVVU4nU6rS/Fbdrud4OBgtXqLiEiz88uAUlFRwf79+yktLbW6FL8XERFBcnIyoaGhVpciIiJ+xO8CisvlYufOndjtdjp06EBoaKj+hd8CDMOgoqKCgwcPsnPnTnr27HnWQXdEREQay+8CSkVFBS6Xi06dOhEREWF1OX6tTZs2hISEsHv3bioqKggPD7e6JBER8RN++09e/Wu+dWg/i4hIS/Do1+UPf/gDNput3iM1NdW9vqysjKlTpxIXF0fbtm257rrrOHDgQL3X2LNnD5mZmURERJCQkMCMGTOoqqpqnk8jIiIifsHjUzwXXnghH3/8ce0LBNe+xD333MOHH37IW2+9RUxMDNOmTePaa6/lyy+/BMDpdJKZmUlSUhKrV69m//79TJ48mZCQEB5//PFm+DgiIiLiDzw+Ph8cHExSUpL70b59ewCKior4xz/+wZ///GeuuuoqBg8ezCuvvMLq1av56quvAFi2bBnff/89r7/+OgMGDGDs2LE88sgjzJ8/n4qKiub9ZH5ixIgRTJ8+3eoyREREWpXHAWXbtm106NCBbt26MWnSJPbs2QNAdnY2lZWVjBo1yr1tamoqnTt3Zs2aNQCsWbOGvn37kpiY6N4mIyMDh8PBpk2bTvue5eXlOByOeg8RERHxXx4FlKFDh7JgwQKWLl3KX//6V3bu3Mlll11GcXExBQUFhIaGEhsbW+85iYmJFBQUAFBQUFAvnNSsr1l3OnPnziUmJsb96NSpkydli4iIiI/xKKCMHTuWn/zkJ/Tr14+MjAwWL17MsWPH+M9//tNS9QEwc+ZMioqK3I+8vDyPnm8YBqUVVa3+MAzDozqPHz/O5MmTadu2LcnJyTz11FP11peXl3P//fdz/vnnExkZydChQ1m5cqV7/YIFC4iNjeWjjz6id+/etG3bljFjxrB//373NitXruTiiy8mMjKS2NhYLrnkEnbv3u1e/9577zFo0CDCw8Pp1q0bs2fP1kXMIiKBprzE6grObRyU2NhYevXqxQ8//MDVV19NRUUFx44dq3cU5cCBAyQlJQGQlJTE119/Xe81arp8arZpSFhYGGFhYU2u80Slk7Tff9Tk5zfV93MyiAht/C6eMWMGq1at4r333iMhIYHf/va3rFu3jgEDBgAwbdo0vv/+exYuXEiHDh145513GDNmDDk5OfTs2ROA0tJS5s2bx7/+9S+CgoK44YYbuP/++3njjTeoqqpi4sSJ3Hrrrfzv//4vFRUVfP311+6B7D7//HMmT57Mc889x2WXXcb27du57bbbAHj44Yebd+eIiIh3On4IDmyEbiMsLeOcBrEoKSlh+/btJCcnM3jwYEJCQlixYoV7fW5uLnv27CE9PR2A9PR0cnJyKCwsdG+zfPlyoqOjSUtLO5dSfF5JSQn/+Mc/mDdvHiNHjqRv3768+uqr7qMXe/bs4ZVXXuGtt97isssuo3v37tx///1ceumlvPLKK+7Xqays5MUXX2TIkCEMGjSIadOmuf83cTgcFBUVMX78eLp3707v3r2ZMmUKnTt3BmD27Nk8+OCDTJkyhW7dunH11VfzyCOP8NJLL7X+DhERkdbncsHmD8Bl/ZFzj46g3H///UyYMIEuXbqQn5/Pww8/jN1u5+c//zkxMTHccsst3HvvvbRr147o6GjuvPNO0tPTGTZsGACjR48mLS2NG2+8kSeffJKCggJmzZrF1KlTz+kIydm0CbHz/ZyMFnv9M71vY23fvp2KigqGDh3qXtauXTtSUlIAyMnJwel00qtXr3rPKy8vJy4uzj0fERFB9+7d3fPJycnuQNiuXTtuuukmMjIyuPrqqxk1ahTXX389ycnJAHz77bd8+eWXPPbYY+7nO51OysrKKC0t1ci8IiL+Lu8rKC6AWOuv9fQooOzdu5ef//znHD58mPj4eC699FK++uor4uPjAXj66acJCgriuuuuo7y8nIyMDF544QX38+12O4sWLeKOO+4gPT2dyMhIpkyZwpw5c5r3U53EZrN5dKrFG5WUlGC328nOzsZurx982rZt654OCQmpt85ms9W7FuaVV17hrrvuYunSpbz55pvMmjWL5cuXM2zYMEpKSpg9ezbXXnvtKe+vYexFRPxcyUHY9YXVVbh59Ku9cOHCM64PDw9n/vz5zJ8//7TbdOnShcWLF3vytgGhe/fuhISEsHbtWvcpl6NHj7J161auuOIKBg4ciNPppLCwkMsuu+yc3mvgwIEMHDiQmTNnkp6ezr///W+GDRvGoEGDyM3NpUePHs3xkURExFe4XLDlA3A5ra7EzbcPK/iRtm3bcssttzBjxgzi4uJISEjgoYcect/rplevXkyaNInJkyfz1FNPMXDgQA4ePMiKFSvo168fmZmZZ32PnTt38re//Y0f/ehHdOjQgdzcXLZt28bkyZMB+P3vf8/48ePp3LkzP/7xjwkKCuLbb79l48aNPProoy36+UVExEJ7VkPxgbNv14oUULzIn/70J0pKSpgwYQJRUVHcd999FBUVude/8sorPProo9x3333s27eP9u3bM2zYMMaPH9+o14+IiGDLli28+uqrHD58mOTkZKZOncrtt98OmIPmLVq0iDlz5vDHP/6RkJAQUlNT+dWvftUin1dERLxA8QHYvbp23jCgYCP0GHX657QCm+HpYB1ewOFwEBMTQ1FREdHR0fXWlZWVsXPnTrp27arrJlqB9reIiA9zOSF7AZTUdteyfwNsXQoDboBr/gLVQ1E0hzP9fp/snNqMRURExIft/rJ+ODlxFH6oHi4kIbVZw4mnFFBEREQCUXEB7F5TO2/UjIFSCYkXwrCp1tWGAoqIiEjgcTlhyyIzlNTYswaK88EeBsPvhCBrI4ICioiISKDZ9YU57kkNx/7aMVB6Xg2R8dbUVYcCioiISCBx7Ic9X9XOOyvNMVAwID4VEi60rLS6FFBEREQCRUOndnZ8CieOQGhb6Jlh6YWxdSmgiIiIBIqdn5l3K65xZAfkrzOnUzIhpI01dTVAAUVERCQQOPIh7+va+coTkPuhOX3+YGjX1Zq6TkMBJYCsXLkSm83GsWPHrC5FRERak7MKtnxYe2rHMMzB2CqOQ0QcdB1Rf/uwMw+i1hoUULzIiBEjmD59ute9loiI+LhdJ53aObARDuWCLQhSJ4A9pHZdzPlw/pDWr/EkCig+xDAMqqqqrC5DRER8SdHe+qd2yo7BD8vN6S6XQlRS7brgMOj9I8vHQAEFFK9x0003sWrVKp599llsNhs2m40FCxZgs9lYsmQJgwcPJiwsjC+++IKbbrqJiRMn1nv+9OnTGTFixGlfa9euXe5ts7OzGTJkCBEREQwfPpzc3NzW+6AiItJ6nFWwZbF5SgfMUzxbPgRnBUSfD52H1d8+ZSy0iW31MhsSGHczNgyoLG399w2JaHS71rPPPsvWrVvp06cPc+bMAWDTpk0APPjgg8ybN49u3bpx3nnnNem14uPj3SHloYce4qmnniI+Pp5f//rX/PKXv+TLL79swgcUERGvtnMllB6und+bBUV5YA+F1PHmKZ4ayf0goXerl3g6gRFQKkvh8Q6t/76/zYfQyEZtGhMTQ2hoKBERESQlmYfbtmzZAsCcOXO4+uqrG/22Db1WXY899hhXXHEFYIafzMxMysrKdDdiERF/ciwP9n5TO19yAHauMqe7j4Q2df7BGxEHPRr/O9MadIrHBwwZ0rwXK/Xr1889nZycDEBhYeHpNhcREV/jrITcOqd2XFW1A7TF9YCk2t8BgoIh7RoIDrWm1tMIjCMoIRHm0Qwr3rcZREbWPwoTFBSEUfOlq1ZZWdn4skJqr9a2VZ+Ccrlcp9tcRER8zY5VUHqkdn7nKjh+0Pxd6jW2/uUH3UZAVGKrl3g2gRFQbLZGn2qxUmhoKE6n86zbxcfHs3HjxnrLNmzYUC94NPa1RETEzxzbA/vqnNo5usu89gQgZVz938O4HtDpolYtr7F0iseLXHDBBaxdu5Zdu3Zx6NCh0x7VuOqqq/jmm2947bXX2LZtGw8//PApgaWxryUiIn7EWVm/a6eqzDzVA5A8wAwkNUIjIXVcq5fYWAooXuT+++/HbreTlpZGfHw8e/bsaXC7jIwMfve73/Gb3/yGiy66iOLiYiZPntyk1xIRET+y/VM4cbR2fttyKHeYF8R2v6p2uc0GvSd49dkFm3HyxQw+wOFwEBMTQ1FREdHR9YfjLSsrY+fOnXTt2lVdKa1A+1tExEsc3Q3f/m/t0ZPCzbD5PcAGA28wxz2p0Xlo/cDSSs70+30yHUERERHxdVUV9bt2yoth21JzunN6/XASnQxdr2j9Gj2kgCIiIuLrdnwKJ46Z04Zh3qW4qtwcxr7LJbXbBYeaLcVBdkvK9IQCioiIiC87shPy19fO78s2O3eCgs0bAdYNIz0z6g/Q5sUUUERERHxVVTnkLqk9tXP8kDm8PUC3K80RYmsk9TEfPkIBRURExFdt/wTKisxplxO2fGCOGnteV+gwqHa7iHbm0RMf4rcBxQebk3yS9rOIiEWO7ID8DbXzu78w77cTHG4OyFYzWmyQHXr/yOuGsj8bvwsoNaOplpZacPfiAFSzn+uOYisiIi2s5tROjaK9sOcrc7rXGAiLql3X9XKzc8fH+N1Q93a7ndjYWPfN7yIiItz3m5HmYxgGpaWlFBYWEhsbi93u/VeEi4j4jR9WQJnDnK4qN0/tYEBiH4hPrd2uXVfoNNSSEs+V3wUUgKSkJEB36G0NsbGx7v0tIiKt4PB22P9t7fz2FeZ1KGHR0GNU7fLQCEgdX//GgD7ELwOKzWYjOTmZhIQEj+7yK54JCQnRkRMRkdZUWVb/1M6hrVDwnTmdOt68/gTMUJI6HsLatn6NzcQvA0oNu92uH1AREfEfP3xsjhILUFECW6vDSqehENu5druOQyCue+vX14z87iJZERERv3ToByjIMacNwzySUnkCIhPggstqt4tKNMdA8XEKKCIiIt6u8kTt0RKA/RvgyHaw2c27EgdVnxCxh0DaRJ8Yyv5sFFBERES83bblUF5iTpceMQdoA/Omf5Hxtdv1HG0OyuYHFFBERES82aFtcGCTOW24YMsicFWa15x0vKh2u8Q0SO5nTY0tQAFFRETEW1WeqN+1s3s1FOeDPQxSMmtbiNvEmgO0+REFFBEREW+1bRlUHDenHfth95fmdM/REB5jTtuCqoeyD7OmxhaigCIiIuKNDubCge/NaWdF7Wix8b0hIa12u66XQcz5lpTYkhRQREREvE1FKWxdWju/YyWcOAKhUebRk5pTO+d1gc7plpTY0hRQREREvM22j8yQAubQ9vnrzOmUcRDSxpwOaWO2GPvoUPZno4AiIiLiTQq3mA+AylLYuticPn+wefO/Gqnj69+12M8ooIiIiHiLiuPm0RMwR4vdutRcFhEHXUfUbtdxCLTvYUWFrUYBRURExFtsrXNq58BG82aAtiBInWCOEgvQNsEvhrI/GwUUERERb3Dge7NzB+DEMfhhuTl9wWUQlWRO24PNoeztfn2vX0ABRURExHrlJeaYJ2COFpu7yGwtjj7fvFNxjR6jIDLOmhpbmQKKiIiI1bYuNUeNBcj7Gor2gj3UPLVjq/6pTkiFDgOtq7GVKaCIiIhYqWCjeb8dgJIDsOszc7r7SHMIezBHje011pLyrKKAIiIiYpXyktprTVxVsPkD8xRPXE9Iqr7xny0I0n4EIeHW1WkBBRQRERGrbF0KlWXm9I5VUHoIQiLMoyU1A7BdcAnEdLSuRosooIiIiFihIKf21M7RXbAvy5xOGQehEeZ0bGfocokl5VlNAUVERKS1lRfDtupTO5VlkPuhOZ08AOKqB2ALCffroezPRgFFRESkteUuhapyc/qHZWZgaXMedL+qdpuUTAiPtqY+L6CAIiIi0pr2fwuHfzCnC783H9iqR4sNNZefPwjie1lWojdQQBEREWktZQ74YYU5Xe6ove9Ol+EQ3cGcjmxvthgHuHMKKE888QQ2m43p06e7lxUUFHDjjTeSlJREZGQkgwYN4u233673vCNHjjBp0iSio6OJjY3llltuoaSk5FxKERER8X65S8xTO4YBWz40p6OSofNwc30ADWV/Nk0OKFlZWbz00kv069ev3vLJkyeTm5vL+++/T05ODtdeey3XX38969evd28zadIkNm3axPLly1m0aBGfffYZt912W9M/hYiIiLfL3wBHdpjT+76BY7shKBhSx0OQ3Vze/SpoG29Zid6kSQGlpKSESZMm8fLLL3PeeefVW7d69WruvPNOLr74Yrp168asWbOIjY0lOzsbgM2bN7N06VL+/ve/M3ToUC699FKef/55Fi5cSH5+/rl/IhEREW9TVgTbq0/tHD8IO1aa092vgojqe+vE94LzB1tSnjdqUkCZOnUqmZmZjBo16pR1w4cP58033+TIkSO4XC4WLlxIWVkZI0aMAGDNmjXExsYyZMgQ93NGjRpFUFAQa9eubfD9ysvLcTgc9R4iIiI+I3cJVFWAywlbPgDDCed1g+Tqe+uERZnjn4ibxye5Fi5cyLp168jKympw/X/+8x9++tOfEhcXR3BwMBEREbzzzjv06GH2dRcUFJCQkFC/iOBg2rVrR0FBQYOvOXfuXGbPnu1pqSIiItbbtw6O7DSnd30BJYUQ3MYMJDab+Uj7EYS0sbZOL+PREZS8vDzuvvtu3njjDcLDG74nwO9+9zuOHTvGxx9/zDfffMO9997L9ddfT05OTpOLnDlzJkVFRe5HXl5ek19LRESk1Zw4Bts/MaeL8iDvK3O61xgIa2tOdxlujhgr9Xh0BCU7O5vCwkIGDRrkXuZ0Ovnss8/4y1/+Qm5uLn/5y1/YuHEjF154IQD9+/fn888/Z/78+bz44oskJSVRWFhY73Wrqqo4cuQISUlJDb5vWFgYYWFhnn42ERER6xgG5C4GZ6XZrbNlEWBAYh+ITzG3iekIXS61tExv5VFAGTly5ClHQm6++WZSU1N54IEHKC0tBSAoqP6BGbvdjsvlAiA9PZ1jx46RnZ3N4MHmxUCffPIJLpeLoUOHNvmDiIiIeJX8dXB0tzm9/WPzQtmwaOhxtbksOMwcyj5IQ5I1xKOAEhUVRZ8+feoti4yMJC4ujj59+lBZWUmPHj24/fbbmTdvHnFxcbz77rvudmKA3r17M2bMGG699VZefPFFKisrmTZtGj/72c/o0KFD830yERERq5w4Cts/NacP5Zo3BgRztNjg6jMCKeOgTawl5fmCZo1tISEhLF68mPj4eCZMmEC/fv147bXXePXVVxk3rvbq5DfeeIPU1FRGjhzJuHHjuPTSS/nb3/7WnKWIiIhYwzBgS/WpnYoS2LrUXN5pGMR2MqeT+0NCqnU1+gCbYRiG1UV4yuFwEBMTQ1FREdHRgXsjJRER8UJ7vzHvVGwYsPG/cGQ7RCbAoMnmwGyR7WHwTWAPsbrSVufJ77dOfImIiDSX0iOwo/rUzv4NZjix2auvNQk2H2nXBGQ48ZQCioiISHNwd+1UmUGlpr242xUQWT18ffcroW3C6V9D3BRQREREmsPeb+BYXu1osa5KiO0C519kro/rAR2HnPk1xE0BRURE5FyVHoGdK83pPWugeL/ZrZOSaY4UG9YWUjMtLdHXKKCIiIicC8MwB2FzVoEjH3Z/aS7vmQHh0WZA6T0BQiOsrdPHKKCIiIici7yvoWgfOCvMUzsYEN8bEtLM9Z2GwnkXWFmhT1JAERERaarjh2HnZ+b0jk/NAdpCo8yjJwDRHaDrFdbV58MUUERERJrC5TJP7biq4PB2yF9vLk/NhJBwCA4171KsoeybRHtNRESkKfZ+bV5zUllqthcDnD+k9nROrzHQ5jzLyvN1CigiIiKeOn4Idn5uXiC7dSlUHoeI9rWnc5L6QuKF1tbo4xRQREREPFH31M6BHDi0FWxBkDreHCE2oh30HG11lT5PAUVERMQTeV+BYz+cOAY/fGwuu+AyiEqCILs5lH1wqKUl+gMFFBERkcYqOQi7vgCj+iiKswKiO5qtxADdRphBRc6ZAoqIiEhjuE/tOCFvLTj2gj3UPLVjC4J23aDjRVZX6TcUUERERBpjzxooLjAfuz43l/UYBW1iITQSeo83R42VZqGAIiIicjYlheYQ9s5K8yiK4YL2vSCxb/VQ9uPNkCLNRgFFRETkTFzO2lM7O1dB6SEIiYSeY8xw0vEi8/SONCsFFBERkTPZvRqKD8DRXbDvG3NZyljz5n9RSeaFsdLsFFBEREROp/iAee1JZRls+dBcljwQ4nqYY56kXWO2FkuzU0ARERFpiMtp3p3Y5YQflkFFsTl0ffcrzfW9MsxB2aRFKKCIiIg0ZNcX5rgnhd+bD2yQOsFsLU680BzOXlqMAoqIiMjJHPthz1dQ5oBtH5nLugyH6A7mUZReGdbWFwAUUEREROqq27WT+yFUlUNUMnQeXj2U/Y8gOMzqKv2eAoqIiEhduz4371a87xs4thuCQsxTO0F285470R2srjAgKKCIiIjUcOTDnrVw/CDsWGku636VeTFsu67QeZil5QUSBRQREREAZ5XZSuysgM0fgOGEdt0heYA55kmqhrJvTQooIiIiALs+M0/t7PoCjhdCcBtzQLagIDOchLW1usKAooAiIiJStA/ysuBYHuR9ZS5LGQOhbeH8IRDX3dr6ApACioiIBLaaUzuVJyB3kbksqS+0T4GoxNqB2aRVKaCIiEhg27kKSg/DDx9DWRGEx0D3UeZQ9r01lL1VFFBERCRwFe2FvVlwMBcO5JjLUsab45z0vBoi46ytL4ApoIiISGByVpqndsqKYetSc1mnYRDbCRJ6Q3J/a+sLcAooIiISmHasguOHYetiqDoBbRPMgdjCY6DXGKurC3gKKCIiEniO7TFHit2/Ho7sAJu9+kaAIeZQ9iHhVlcY8BRQREQksDgrYcti8+jJ9k/MZd1GQGQ8XHApxHS0tDwxKaCIiEhg2bHSHJBtywfgqoLYLuZYJ7GdzTsWi1dQQBERkcBxdDfsy4Y9q6F4v9mtk5ppDmXfe4KGsvciCigiIhIYqiogd7E5auzu1eaynhkQFm2GlPBoa+uTehRQREQkMOz4FEoKzVM7GJCQZj7OHwzte1pdnZxEAUVERPzf0V2Qv968KPbEUQiLgh6joW08dL/K6uqkAQooIiLi36rKza6dQ9tg/wZzWUomhLeFtIlgD7ayOjkNBRQREfFv2z8Fx37IXWLOn38RnHcBdB8Jke0tLU1OTwFFRET815EdsG8dbF0Clcchoj10uwLiU+D8QVZXJ2eggCIiIv6pqtw8anIgBw5vA1uQ2Uoc0Q5SxlpdnZyFAoqIiPinH1bA0T3ww8fm/AWXQ1SyGVJC2lhbm5yVAoqIiPifw9vNrp0ti8BZYQ5f3+lic6TY2M5WVyeNoIAiIiL+pbLMPLWTtxYce8EeCqnj4bwu0OUSq6uTRlJAERER/7J9hdlSvOtzc77HKIhKMk/tBOlnz1fofykREfEfh7fD3mxztFjDBe17QWJfSBkH4TFWVyceUEARERH/UFlm3mtn5yooPQyhkdBrDJw/0GwrFp+igCIiIv7hh+WwPwf2fWPO9xpnXhDbY5S1dUmTKKCIiIjvO7QN8rIg90NzvkP1UZO0a8AeYm1t0iQKKCIi4tsqT8DWpbBtGVSUQJt20O1K8yaAbROsrk6aSAFFRER827ZlsGctHNwM2MyW4sQLoeNgqyuTc6CAIiIivuvgVtj9lRlSwBznJD7F7NoRn6aAIiIivqmi1OzayV0EznJzGPsul5jjnYRGWF2dnKNzCihPPPEENpuN6dOn11u+Zs0arrrqKiIjI4mOjubyyy/nxIkT7vVHjhxh0qRJREdHExsbyy233EJJScm5lCIiIoFm2zKzpfjYHggKgdQJcMEl5oix4vOaHFCysrJ46aWX6NevX73la9asYcyYMYwePZqvv/6arKwspk2bRlCd0fsmTZrEpk2bWL58OYsWLeKzzz7jtttua/qnEBGRwFK4BXZ+BjtWmfPdr4LkvnDBZdbWJc0muClPKikpYdKkSbz88ss8+uij9dbdc8893HXXXTz44IPuZSkptQPkbN68maVLl5KVlcWQIUMAeP755xk3bhzz5s2jQ4cOTSlJREQCRcVxs5148wdgOKFdd+g0FHr/SEPZ+5Em/S85depUMjMzGTWq/uA3hYWFrF27loSEBIYPH05iYiJXXHEFX3zxhXubNWvWEBsb6w4nAKNGjSIoKIi1a9c28WOIiEjA2PqR+TheCCFtIGUspI6DNrFWVybNyOMjKAsXLmTdunVkZWWdsm7Hjh0A/OEPf2DevHkMGDCA1157jZEjR7Jx40Z69uxJQUEBCQn1+9KDg4Np164dBQUFDb5neXk55eXl7nmHw+Fp2SIi4g8KN8O25eadigF6jYUuwyGht7V1SbPz6AhKXl4ed999N2+88Qbh4eGnrHe5XADcfvvt3HzzzQwcOJCnn36alJQU/vnPfza5yLlz5xITE+N+dOrUqcmvJSIiPqriOHz/HmxZZM4n9YPO6dDjamvrkhbhUUDJzs6msLCQQYMGERwcTHBwMKtWreK5554jODiYxMREANLS0uo9r3fv3uzZsweApKQkCgsL662vqqriyJEjJCUlNfi+M2fOpKioyP3Iy8vzpGwREfEHW5ea152UO8w7E/fMMIeyDw61ujJpAR6d4hk5ciQ5OTn1lt18882kpqbywAMP0K1bNzp06EBubm69bbZu3crYsWMBSE9P59ixY2RnZzN4sDnK3yeffILL5WLo0KENvm9YWBhhYWGelCoiIv7kwCb4/n04sBFztNgJ0CsDohKtrkxaiEcBJSoqij59+tRbFhkZSVxcnHv5jBkzePjhh+nfvz8DBgzg1VdfZcuWLfz3v/8FzKMpY8aM4dZbb+XFF1+ksrKSadOm8bOf/UwdPCIicqryEtj4tnkEBaDTMOg2AjpdZGlZ0rKa1GZ8JtOnT6esrIx77rmHI0eO0L9/f5YvX0737t3d27zxxhtMmzaNkSNHEhQUxHXXXcdzzz3X3KWIiIg/yF0CG9+BqjJomwi9RptdO+LXbIZhGFYX4SmHw0FMTAxFRUVER0dbXY6IiLSUghz4eA78sAxsdhjyS0ifCu26Wl2ZNIEnv98a0UZERLxTeTF8uxB2fGLOdxthHjlROAkICigiIuKdNi+Cjf8FVxXEXgC9x0PXK6yuSlqJAoqIiHif/d/B+n9BcQEEh8GFE+HC/4Egu9WVSStRQBEREe9S5oDsBbB7tTnfcwz0/Qm0Oc/SsqR1KaCIiIh3+f5d2PR/gAEJadDvJ5DU52zPEj+jgCIiIt4jfwNk/QNOHIWwKOj3U3PEWAk4CigiIuIdyopg7V9h/wZzvvePzICioewDkgKKiIh4h+/eNIezB+h4EQyaAtHJ1tYkllFAERER6+1bB2v/BpWlEBlvhpNOF1tdlVhIAUVERKx14hh88Wc4vA1sQWbHzoX/Azab1ZWJhRRQRETEOoYB616D3OobAXa9Ai76FYS1tbYusZwCioiIWGdvFqx9EVyVENMJht0Bcd3P/jzxewooIiJijRNH4dPHwbEP7KEw+CboMcrqqsRLKKCIiEjrMwz46kXYucqc7zXGvFOxhrKXagooIiLS+navgayXwXBB+xS47H6IaGd1VeJFFFBERKR1lR6Bjx+G0sMQGgmX3AUd+ltdlXgZBRQREWk9hgGfPwV7vzbn+15vthWLnEQBRUREWs/2T8w7FQN0GASXz4DgMEtLEu+kgCIiIq3j+GFYNgsqSqBNOxj5O4g53+qqxEspoIiISMszDFgxGwq/N0eLvfg26Hal1VWJF1NAERGRlrflQ/h2oTnd9QoYPk1D2csZKaCIiEjLKjkIH/0WnOUQ1QHGPAFhUVZXJV5OAUVERFqOywWL74djuyEoBK6cCQmpVlclPkABRUREWs53b8LmD8zpCydC/19YWo74DgUUERFpGUX7YPnvwXBCXE/IeALswVZXJT5CAUVERJqfywXvT4PjhRDSBsb+Edq2t7oq8SEKKCIi0vyyXjYHZQO46FboMdLaesTnKKCIiEjzOrwDPnnUnD5/CFz5kLX1iE9SQBERkebjcsE7t0G5A8JjYeILEBJudVXigxRQRESk+Xz2J9ibBdjMIyfxKVZXJD5KAUVERJpHQQ588bQ53XM0XHyrtfWIT1NAERGRc+d0wtu/gqoTEJVsntrRUPZyDhRQRETk3C37LRzcAkHBMP5ZiFRLsZwbBRQRETk3u1ZD1t/N6YE3QkqGtfWIX1BAERGRpqssg3duBVcVtO8FY5+0uiLxEwooIiLSdB/cBUV7ITgcrvsnBIdaXZH4CQUUERFpmtwl8N1/zOnLfwPJfa2tR/yKAoqIiHiurAjemwoY0DkdLr/P6orEzyigiIiI597+FZQehvAY+MkCq6sRP6SAIiIinln/BmxbZk5n/hmikqytR/ySAoqIiDSeYz8s+Y05nTYR+v7Y0nLEfymgiIhI4xgG/GcyVJRAVAeY+FerKxI/poAiIiKN8+VzsPdrsNnhJ69AaITVFYkfU0AREZGzO7gNPn3UnB52B3QeZm094vcUUERE5MycVfDmJHBWQHxvuHqO1RVJAFBAERGRM1sxGw7lmqPF/uzfEGS3uiIJAAooIiJyenuzYM1fzOmrH4G4btbWIwFDAUVERBpWUQr/mQKGC7peDhffanVFEkAUUEREpGGL7wPHvurRYl8Fm83qiiSAKKCIiMipti6DDf82pyf+FSLaWVuPBBwFFBERqa/0CLxzmznd76eQmmltPRKQFFBERKSWYcA7t8OJoxB9Pox/xuqKJEApoIiISK0N/zZvBGgLguv/pdFixTIKKCIiYjq2x7wwFuDSe6DjYGvrkYCmgCIiIuBywVs3Q+UJSOwDI35rdUUS4BRQREQEvnwW9n0DwWFw/WtgD7a6IglwCigiIoGuYGPtjQBHz4W47tbWI8I5BpQnnngCm83G9OnTT1lnGAZjx47FZrPx7rvv1lu3Z88eMjMziYiIICEhgRkzZlBVVXUupYiISFNUlsFbN4GrCrpdCRf90uqKRABo8jG8rKwsXnrpJfr169fg+meeeQZbA6MOOp1OMjMzSUpKYvXq1ezfv5/JkycTEhLC448/3tRyRESkKVbMhsPbIDwWrv2bRosVr9GkIyglJSVMmjSJl19+mfPOO++U9Rs2bOCpp57in//85ynrli1bxvfff8/rr7/OgAEDGDt2LI888gjz58+noqKiKeWIiEhT7PwcvvqrOT3xBWibYG09InU0KaBMnTqVzMxMRo0adcq60tJSfvGLXzB//nySkpJOWb9mzRr69u1LYmKie1lGRgYOh4NNmzY1pRwREfFUWRH8368AAwb8QqPFitfx+BTPwoULWbduHVlZWQ2uv+eeexg+fDjXXHNNg+sLCgrqhRPAPV9QUNDgc8rLyykvL3fPOxwOT8sWEZG6PrwPigsguiOM/ZPV1YicwqOAkpeXx913383y5csJDw8/Zf3777/PJ598wvr165utQIC5c+cye/bsZn1NEZGAtekdyHnLHC32x/+EsLZWVyRyCo9O8WRnZ1NYWMigQYMIDg4mODiYVatW8dxzzxEcHMzy5cvZvn07sbGx7vUA1113HSNGjAAgKSmJAwcO1HvdmvmGTgkBzJw5k6KiIvcjLy/P088pIiIAjnx4/y5z+pJ7oPNQa+sROQ2PjqCMHDmSnJycestuvvlmUlNTeeCBB2jfvj233357vfV9+/bl6aefZsKECQCkp6fz2GOPUVhYSEKCeUHW8uXLiY6OJi0trcH3DQsLIywszJNSRUTkZC4XvHsHlDsgsS9cOdPqikROy6OAEhUVRZ8+feoti4yMJC4uzr28oaMgnTt3pmvXrgCMHj2atLQ0brzxRp588kkKCgqYNWsWU6dOVQgREWlJWX+HHSvBHgo//gfYQ6yuSOS0Wn0kWbvdzqJFi7Db7aSnp3PDDTcwefJk5syZ09qliIgEjoO5sGyWOT36MYhPsbYekbOwGYZhWF2EpxwOBzExMRQVFREdHW11OSIi3q2qAv4+Egq+g24j4IZ3IEh3OpHW58nvt76hIiL+btUfzXASFgMTX1Q4EZ+gb6mIiD/bsxa++LM5/aNnITrZ2npEGkkBRUTEX5WXwP/dCoYL+v0ULvwfqysSaTQFFBERf/XRTDi2G6LPh3EaLVZ8iwKKiIg/2rIY1r0G2My7FIfHWF2RiEcUUERE/E1JIbw/zZwePg0uuNTaekSaQAFFRMSfGIY5lH3pYUhIg6t+Z3VFIk3i8d2M/dm2A8Ws3XnE6jIkQNhsJ81jO8t6D55/1ufazrK+8e918rqTefxeZ9gPZ3vuyVucfR+evjZP9kFDL36m9zqXfdDQ8+suSNi2kB5bl+AKCuG7i/9E6e5ioPjkZ4icVWJ0ON3jrbuRpAJKHVm7jjLr3Y1WlyEi0iRdbAUsDp0DNphb/hNe/m8RsNbqssRH/WJoZx7/n76Wvb8CSh0dz2tDxoWJVpchAeDk8ZtPHs751PGdjTOuP/X5xlnWe/b8U6o55flnqc/T7U95w9M/v6X3xckbnP35nu6Lk9c3bd8HGU7+fPwlIp3lbLD35bPo6+ll01l8abrEqHBL318BpY7Le8Vzea94q8sQEfHcqj/Bp7kQFsWAO/7NR7Gdra5I5JwoXouI+Lp962DVE+b0uKdA4UT8gAKKiIgvqyiF/7sNXFWQNhH6XW91RSLNQgFFRMSXLf89HN4GbZNg/NNnb6sS8RG6BkVExBs4q6CqrM6jHCpPmP+tOtHwsuICyHrZfP7EFyCinbWfQaQZKaCIiNQwDHBW1AkBdQJDZZ3gUHWigQBRftJ2pwsaNfMnbeOqanrdF98OPUY2334Q8QIKKCLifVzOOmGg7DRB4IT5qCg1l1eUVj/nRG0AqDwpODjL67xuefV8RZ3/VtBAo3Hrs4dCcDgEh0FwG/O/IeHVy8Jr14W0gfYp5nD2In5GAUUk0BmGGQgMZ+1/DZd5ysFZDhUnqn/0S6sfJ2p/8CtLT3/U4HRBwFlRGwacFeCsBFdl7bSz0qzBcjYIDgV7WHVQqBsW2piP4Jr/hp8+QNR9XkPbNfS8ILvVH17EcgooIq3BWWXeG+XkEOByNbDs5P9WnhQAqo8kVJafekSgqjoA1AsE1UHDWR0CXJW1QcBVZc67nOa0s6p6WfXDG44m2ILMIwr2EDMs2EPrB4e6P/Du0BABoW0gJBJCI8z/NiZAhNQJEvZQXXAqYiEFFJGW5tgPK2bD4e11fvyd9YNAQ4+asOAVRxOAoGDzX/ZBIea0OzTUDQ4NHW2IMMNBSET1kYeI6gARWefRtvoRUf/Igj0M7PozJRKI9P98kZZiGLD5A1j2Ozi2q5le1GYGgiB7dWA4w8N9pKHmeobQ6tBQc1qhTe2piprAENKmOijUBIcoCKueDw4zw4kCg4i0Av2lEWkJJ47Bx3+Ab/9tnnYJCoGkvmY4cAeI4DpHHdrUOQXRpvq0xMlHG9pWH1UIMZ8XFFI7bQ85dblOT4iID1NAEWluu1fD4hlwoPrO2FHJcNUs6DGqNkwE1YQJjZUoItIQBRSR5lJ5Ar54Gr76K5Q7ABt0vxLGzYO47lZXJyLiUxRQRJpD4Rb48D7Y/YU5Hx4Ll0yH9P9nnsIRERGPKKCInAtnFax7DVbOheOF5rIOgyDzKTh/kLW1iYj4MAUUkaZy7Iclv4EtH5qtwCFtYPDNcOVvISzK6upERHyaAoqIpwwDtiyCZbPg6C5zWVwPyJgLPa9W94yISDNQQBHxxIlj5qBrG96obR++cKIZTtrGW12diIjfUEARaazdX8GSGVDwnTlf0z7c/+e6d4qISDNTQBE5m8oT8OUzsOYFtQ+LiLQSBRSRMynMhcX3wa7PzfnwWLjkbkifqvZhEZEWpIAi0hC1D4uIWEoBReRkDbUPD7oJrnpI7cMiIq1EAUWkRkPtw+16wBi1D4uItDYFFBGAsiL4uKZ9uMy823DaRBjzhNqHRUQsoIAickr7cBJc+TsYoPZhERGrKKBI4Ko8AV8+C2vm17YPd7sSMtU+LCJiNQUUCUwNtg/fBenT1D4sIuIFFFAksDTYPjzQHHSt4xBraxMRETcFFAkcxQWw+Ddmp05N+/DAKWb7cHi01dWJiEgdCiji/wzDHNNk2Sw4utNc1q67eYO/XqPVPiwi4oUUUMS/Ndg+fA1kPAFRCVZXJyIip6GAIv6rwfbhWTDgF2ofFhHxcgoo4n8qy6rvPly3fXiEeR8dtQ+LiPgEBRTxL4e2waJ7Yddn5nx4LAy/02wfDgm3tDQREWk8BRTxD84qWP8arHwCSg6Yy9Q+LCLisxRQxPcVF8CSB2DzB2b7cHAbGDQZrpql9mERER+lgCK+yzBgy2JY9tBJ7cOPQ68MtQ+LiPgwBRTxTWVFsGIOrH+9tn249zXm3YfVPiwi4vMUUMT37FkLi2dAwbfmfNskczTYAZPUPiwi4icUUMR3VJbB6udg9fN12oevgHFPQfseVlcnIiLNSAFFfMOhbfDhfbBzlTkfHgPD71L7sIiIn1JAEe/mrIL1/zLvPlzTPpw8wGwf7nSRpaWJiEjLUUAR71XTPrxlEbiq1D4sIhJAFFDE+xgG5C6Bjx6CozvMZe26Q8Zj0GuM2odFRAKAAop4l7IiWPGIeVrH3T48AcY8qfZhEZEAEnQuT37iiSew2WxMnz4dgCNHjnDnnXeSkpJCmzZt6Ny5M3fddRdFRUX1nrdnzx4yMzOJiIggISGBGTNmUFVVdS6liD/I+xpenQBZL5vhpG0SZP4ZrvuHwomISIBp8hGUrKwsXnrpJfr16+delp+fT35+PvPmzSMtLY3du3fz61//mvz8fP773/8C4HQ6yczMJCkpidWrV7N//34mT55MSEgIjz/++Ll/IvE9Ne3Da/5iHkFR+7CISMCzGYZhePqkkpISBg0axAsvvMCjjz7KgAEDeOaZZxrc9q233uKGG27g+PHjBAcHs2TJEsaPH09+fj6JiYkAvPjiizzwwAMcPHiQ0NDQs76/w+EgJiaGoqIioqN1saRPa7B9+E5Iv1PtwyIifsaT3+8mneKZOnUqmZmZjBo16qzb1hQRHGwerFmzZg19+/Z1hxOAjIwMHA4HmzZtavA1ysvLcTgc9R7i45xV8M0CWJBZG06SB8Ckt+HyGQonIiIBzuNTPAsXLmTdunVkZWWdddtDhw7xyCOPcNttt7mXFRQU1AsngHu+oKCgwdeZO3cus2fP9rRU8VbFhbBkxkntwzfCVb9T+7CIiAAeHkHJy8vj7rvv5o033iA8/Mz/wnU4HGRmZpKWlsYf/vCHc6mRmTNnUlRU5H7k5eWd0+uJRWruPvzPDPj+XTOctOsGP3kFxj6pcCIiIm4eHUHJzs6msLCQQYMGuZc5nU4+++wz/vKXv1BeXo7dbqe4uJgxY8YQFRXFO++8Q0hIiHv7pKQkvv7663qve+DAAfe6hoSFhREWFuZJqeJtyhzVdx8+uX34jxCVePbni4hIQPEooIwcOZKcnJx6y26++WZSU1N54IEHsNvtOBwOMjIyCAsL4/333z/lSEt6ejqPPfYYhYWFJCSYraPLly8nOjqatLS0c/w44pXysmDxfbC/zt2Hr/ytefdhu4biERGRU3n06xAVFUWfPn3qLYuMjCQuLo4+ffrgcDgYPXo0paWlvP766/UuaI2Pj8dutzN69GjS0tK48cYbefLJJykoKGDWrFlMnTpVR0n8TUPtw10vh8ynoH1Pq6sTEREv1qz/fF23bh1r164FoEeP+uNX7Ny5kwsuuAC73c6iRYu44447SE9PJzIykilTpjBnzpzmLEWsdugH+PBe2PkZYJjtw+nTzDsQq0NHRETOoknjoFhN46B4MZcT1r8Onz4OJdVdWckDYNyfoNPFlpYmIiLW8uT3WxcASPMpLoSlv4HNH1S3D4fDwBth5O/MIygiIiKNpIAi567m7sPLHoIjNXcf7gajH4OUsbr7sIiIeEwBRc5NmQM+eRTWvar2YRERaTYKKNJ0eVmw+H7Yv8Gcb5sIVz6k9mERETln+hURz1WWwernYc3z1e3DQNcr1D4sIiLNRgFFPHPoB3PQtR2rAAPCos27D6t9WEREmpECijROg+3D/WHcPLUPi4hIs1NAkbMrLoSlD8Dm99U+LCIirUIBRU7PMGDrUvjoITiy3Vx2XjfIUPuwiIi0LAUUaZi7ffg1qDphtg+njoexf4Sohu86LSIi0lwUUORUDbUPj5hpntZR+7CIiLQC/dpIrcoy887Dq5+r0z58OWT+We3DIiLSqhRQxNRQ+3D6VLjkbghpY3V1IiISYBRQAp3LCRvegE8eq20fTupv3n2481BraxMRkYClgBLIGmwfvgFG/l7twyIiYikFlEBkGLB1GXw0s077cFfz7sOp49Q+LCIillNACTQntw/b7NB7PIx9Uu3DIiLiNRRQAsneb+DD+05qH34QBk5W+7CIiHgV/SoFgqpy8+7Dq5+HsmPmsgsuN+8+HN/L0tJEREQaooDi7w5vN4+a7FiJ2odFRMRXKKD4K5cTNvwbPn0Uimvah/vB2D9Bl2HW1iYiInIWCij+qOQgLHkANr9X2z484AYYpfZhERHxDQoo/uS07cOPQmqm2odFRMRnKKD4i/JiczTYdQugsrp9uObuw9HJVlcnIiLiEQUUf7D3G/Puw/nrzXm1D4uIiI/Tr5cvqyqHNfPhy2frtw+PmwcJKZaWJiIici4UUHyV2odFRMSPKaD4GpcTNvwvfPqI2odFRMRvKaD4kpKD5t2Hv6/bPjwJRj2s9mEREfErCii+wDBg23JYOhOO/GAuU/uwiIj4MQUUb9dg+3CmefdhtQ+LiIifUkDxZvvWwYf31rYPRybAiJkwSO3DIiLi3/Qr540abB++DMY9pfZhEREJCAoo3ubwjur24U+pbR/+f3DJdLUPi4hIwFBA8RYuJ3z7v/DJo1C831yW1Le6fTjd2tpERERamQKKNyg5CB89CJveA1clBIeZ7cMjH4Y2sVZXJyIi0uoUUKxU0z780Uw4XNM+fAFc/Qj0nqD2YRERCVgKKFZR+7CIiMhpKaBYYd8680LY/HXmvNqHRURE6tGvYWuqKoc1L8CXz9RpH760un041crKREREvIoCSms5vAMW3wfb67QPD7sDLr1H7cMiIiInUUBpaS4nfLcQVjxyUvvwk9BluLW1iYiIeCkFlJZUctDs0Nn0bm37cP9fwKg/qH1YRETkDBRQWoJhwLaPq9uHt5nLYi8w24fT1D4sIiJyNgooza28GD59HLIXQGWp2odFRESaQAGlOe1bB4vvh33Z5nxkPFzxIAy+Se3DIiIiHtCvZnOoaR9e/SycOGou63IpZKp9WEREpCkUUM7V4R3mUZPtn+BuHx76a7jsXrUPi4iINJECSlO5nPDdm7BiTm37cGIfGPcntQ+LiIicIwWUpig5VH334XfN9mF7GPT/OVw9W+3DIiIizUABxVNbl5/UPtwFrn5U7cMiIiLNSAGlscqL4dO5kP1KbftwyjgY9yREd7C6OhEREb+igNIY+9ab99Gp1z78QHX7cIilpYmIiPgjBZQzqSqHr/5q3n3Y3T58SXX7cG9LSxMREfFnCiinc2QnfHhfnfbhKBh6h9qHRUREWoECysnc7cOPQHG+uSyxjzlU/QWXWFubiIhIgFBAqcswYNUT8PnTah8WERGxUNC5PPmJJ57AZrMxffp097KysjKmTp1KXFwcbdu25brrruPAgQP1nrdnzx4yMzOJiIggISGBGTNmUFVVdS6lNI+PH4ZVT5rhJLYLXPs3mPCMwomIiEgra3JAycrK4qWXXqJfv371lt9zzz188MEHvPXWW6xatYr8/HyuvfZa93qn00lmZiYVFRWsXr2aV199lQULFvD73/++6Z+iufT/BYS2hdTx8MulcOFEjW0iIiJiAZthGIanTyopKWHQoEG88MILPProowwYMIBnnnmGoqIi4uPj+fe//82Pf/xjALZs2ULv3r1Zs2YNw4YNY8mSJYwfP578/HwSExMBePHFF3nggQc4ePAgoaGhZ31/h8NBTEwMRUVFREdHe1r+mR3bA1HJah8WERFpZp78fjfpCMrUqVPJzMxk1KhR9ZZnZ2dTWVlZb3lqaiqdO3dmzZo1AKxZs4a+ffu6wwlARkYGDoeDTZs2Nfh+5eXlOByOeo8WE9tZ4URERMRiHl8ku3DhQtatW0dWVtYp6woKCggNDSU2Nrbe8sTERAoKCtzb1A0nNetr1jVk7ty5zJ4929NSRURExEd5dAQlLy+Pu+++mzfeeIPw8PCWqukUM2fOpKioyP3Iy8trtfcWERGR1udRQMnOzqawsJBBgwYRHBxMcHAwq1at4rnnniM4OJjExEQqKio4duxYvecdOHCApKQkAJKSkk7p6qmZr9nmZGFhYURHR9d7iIiIiP/yKKCMHDmSnJwcNmzY4H4MGTKESZMmuadDQkJYsWKF+zm5ubns2bOH9PR0ANLT08nJyaGwsNC9zfLly4mOjiYtLa2ZPpaIiIj4Mo+uQYmKiqJPnz71lkVGRhIXF+defsstt3DvvffSrl07oqOjufPOO0lPT2fYsGEAjB49mrS0NG688UaefPJJCgoKmDVrFlOnTiUsLKyZPpaIiIj4smYfSfbpp58mKCiI6667jvLycjIyMnjhhRfc6+12O4sWLeKOO+4gPT2dyMhIpkyZwpw5c5q7FBEREfFRTRoHxWotOg6KiIiItIgWHwdFREREpCUpoIiIiIjXUUARERERr6OAIiIiIl5HAUVERES8jgKKiIiIeB0FFBEREfE6zT5QW2uoGbrF4XBYXImIiIg0Vs3vdmOGYPPJgFJcXAxAp06dLK5EREREPFVcXExMTMwZt/HJkWRdLhf5+flERUVhs9ma9bUdDgedOnUiLy9Po9SehfZV42lfNZ72VeNpXzWe9pVnWmp/GYZBcXExHTp0ICjozFeZ+OQRlKCgIDp27Nii7xEdHa0vcSNpXzWe9lXjaV81nvZV42lfeaYl9tfZjpzU0EWyIiIi4nUUUERERMTrKKCcJCwsjIcffpiwsDCrS/F62leNp33VeNpXjad91XjaV57xhv3lkxfJioiIiH/TERQRERHxOgooIiIi4nUUUERERMTrKKCIiIiI1wnIgDJ//nwuuOACwsPDGTp0KF9//fUZt3/rrbdITU0lPDycvn37snjx4laq1Hqe7KsFCxZgs9nqPcLDw1uxWut89tlnTJgwgQ4dOmCz2Xj33XfP+pyVK1cyaNAgwsLC6NGjBwsWLGjxOr2Bp/tq5cqVp3yvbDYbBQUFrVOwRebOnctFF11EVFQUCQkJTJw4kdzc3LM+LxD/XjVlXwXy36u//vWv9OvXzz0IW3p6OkuWLDnjc6z4XgVcQHnzzTe59957efjhh1m3bh39+/cnIyODwsLCBrdfvXo1P//5z7nllltYv349EydOZOLEiWzcuLGVK299nu4rMEcd3L9/v/uxe/fuVqzYOsePH6d///7Mnz+/Udvv3LmTzMxMrrzySjZs2MD06dP51a9+xUcffdTClVrP031VIzc3t953KyEhoYUq9A6rVq1i6tSpfPXVVyxfvpzKykpGjx7N8ePHT/ucQP171ZR9BYH796pjx4488cQTZGdn880333DVVVdxzTXXsGnTpga3t+x7ZQSYiy++2Jg6dap73ul0Gh06dDDmzp3b4PbXX3+9kZmZWW/Z0KFDjdtvv71F6/QGnu6rV155xYiJiWml6rwXYLzzzjtn3OY3v/mNceGFF9Zb9tOf/tTIyMhowcq8T2P21aeffmoAxtGjR1ulJm9VWFhoAMaqVatOu00g/72qqzH7Sn+v6jvvvPOMv//97w2us+p7FVBHUCoqKsjOzmbUqFHuZUFBQYwaNYo1a9Y0+Jw1a9bU2x4gIyPjtNv7i6bsK4CSkhK6dOlCp06dzpjIA12gfq/OxYABA0hOTubqq6/myy+/tLqcVldUVARAu3btTruNvlemxuwr0N8rAKfTycKFCzl+/Djp6ekNbmPV9yqgAsqhQ4dwOp0kJibWW56YmHja89kFBQUebe8vmrKvUlJS+Oc//8l7773H66+/jsvlYvjw4ezdu7c1SvYpp/teORwOTpw4YVFV3ik5OZkXX3yRt99+m7fffptOnToxYsQI1q1bZ3VprcblcjF9+nQuueQS+vTpc9rtAvXvVV2N3VeB/vcqJyeHtm3bEhYWxq9//Wveeecd0tLSGtzWqu+VT97NWLxTenp6vQQ+fPhwevfuzUsvvcQjjzxiYWXiy1JSUkhJSXHPDx8+nO3bt/P000/zr3/9y8LKWs/UqVPZuHEjX3zxhdWleL3G7qtA/3uVkpLChg0bKCoq4r///S9Tpkxh1apVpw0pVgioIyjt27fHbrdz4MCBessPHDhAUlJSg89JSkryaHt/0ZR9dbKQkBAGDhzIDz/80BIl+rTTfa+io6Np06aNRVX5josvvjhgvlfTpk1j0aJFfPrpp3Ts2PGM2wbq36sanuyrkwXa36vQ0FB69OjB4MGDmTt3Lv379+fZZ59tcFurvlcBFVBCQ0MZPHgwK1ascC9zuVysWLHitOfe0tPT620PsHz58tNu7y+asq9O5nQ6ycnJITk5uaXK9FmB+r1qLhs2bPD775VhGEybNo133nmHTz75hK5du571OYH6vWrKvjpZoP+9crlclJeXN7jOsu9Vi16C64UWLlxohIWFGQsWLDC+//5747bbbjNiY2ONgoICwzAM48YbbzQefPBB9/ZffvmlERwcbMybN8/YvHmz8fDDDxshISFGTk6OVR+h1Xi6r2bPnm189NFHxvbt243s7GzjZz/7mREeHm5s2rTJqo/QaoqLi43169cb69evNwDjz3/+s7F+/Xpj9+7dhmEYxoMPPmjceOON7u137NhhREREGDNmzDA2b95szJ8/37Db7cbSpUut+gitxtN99fTTTxvvvvuusW3bNiMnJ8e4++67jaCgIOPjjz+26iO0ijvuuMOIiYkxVq5caezfv9/9KC0tdW+jv1empuyrQP579eCDDxqrVq0ydu7caXz33XfGgw8+aNhsNmPZsmWGYXjP9yrgAophGMbzzz9vdO7c2QgNDTUuvvhi46uvvnKvu+KKK4wpU6bU2/4///mP0atXLyM0NNS48MILjQ8//LCVK7aOJ/tq+vTp7m0TExONcePGGevWrbOg6tZX0wp78qNm/0yZMsW44oorTnnOgAEDjNDQUKNbt27GK6+80up1W8HTffXHP/7R6N69uxEeHm60a9fOGDFihPHJJ59YU3wramgfAfW+J/p7ZWrKvgrkv1e//OUvjS5duhihoaFGfHy8MXLkSHc4MQzv+V7ZDMMwWvYYjYiIiIhnAuoaFBEREfENCigiIiLidRRQRERExOsooIiIiIjXUUARERERr6OAIiIiIl5HAUVERES8jgKKiIiIeB0FFBEREfE6CigiIiLidRRQRERExOsooIiIiIjX+f9kVrR8WR3SvQAAAABJRU5ErkJggg==\n" }, "metadata": {}, "output_type": "display_data" } ], "source": [ - "re = load_log(log_file=\"rollout.csv\", log_key=\"ep_rew_mean\")\n", - "mean = np.stack(list(re.values())).mean(axis=0)\n", - "std = np.stack(list(re.values())).std(axis=0)\n", + "result_key = []\n", + "diff_key, result_dict = load_log(\n", + " log_file=\"rollout.csv\",\n", + " log_key=\"ep_rew_mean\",\n", + ")\n", + "\n", + "idx = diff_key.index(('transition', 'oracle'))\n", + "for name in set([key[idx] for key in result_dict.keys()]):\n", + " values = np.stack([value for key, value in result_dict.items() if key[idx] == name])\n", + " plt.plot(values.mean(axis=0), label=name)\n", + " plt.fill_between(np.arange(len(values.mean(axis=0))), values.mean(axis=0) - values.std(axis=0),\n", + " values.mean(axis=0) + values.std(axis=0), alpha=0.5)\n", + " plt.legend()\n", "\n", - "plt.plot(mean)\n", - "plt.fill_between(np.arange(len(mean)), mean - std, mean + std, alpha=0.5)" + "# for key in result_key:\n", + "#\n", + "# print(re)\n", + "#\n", + "# mean = np.stack(list(re.values())).mean(axis=0)\n", + "# std = np.stack(list(re.values())).std(axis=0)\n", + "#\n", + "# plt.plot(mean)\n", + "# plt.fill_between(np.arange(len(mean)), mean - std, mean + std, alpha=0.5)" ], "metadata": { "collapsed": false From 52403f3b7c297f81b2c9bcc3ef600f45d3760505 Mon Sep 17 00:00:00 2001 From: frank Date: Thu, 6 Apr 2023 15:04:59 +0800 Subject: [PATCH 64/68] :tada: update notebook --- exp_reader.ipynb | 167 ++++++++++++++++++++++++++++++++--------------- 1 file changed, 115 insertions(+), 52 deletions(-) diff --git a/exp_reader.ipynb b/exp_reader.ipynb index 99b7cc0..cb2b37d 100644 --- a/exp_reader.ipynb +++ b/exp_reader.ipynb @@ -2,9 +2,11 @@ "cells": [ { "cell_type": "code", - "execution_count": 67, + "execution_count": 2, "metadata": { - "collapsed": true + "pycharm": { + "name": "#%%\n" + } }, "outputs": [], "source": [ @@ -20,7 +22,12 @@ }, { "cell_type": "code", - "execution_count": 109, + "execution_count": 31, + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, "outputs": [], "source": [ "# 递归判断a字典中存在的值是否与b字典相等\n", @@ -63,29 +70,33 @@ " diff_keys += [(k, dk) for dk in get_diff_key([d[k] for d in dicts])]\n", " elif not all([dicts[0][k] == d[k] for d in dicts[1:]]):\n", " diff_keys.append(k)\n", - " return diff_keys" - ], - "metadata": { - "collapsed": false - } + " return diff_keys\n", + "\n", + "\n", + "def argmax(l):\n", + " return max(l), l.index(max(l))" + ] }, { "cell_type": "code", - "execution_count": 125, + "execution_count": 21, + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, "outputs": [], "source": [ "default_params = dict(freq_rate=1,\n", " real_time_scale=0.02,\n", " integrator=\"euler\",\n", - " gravity=9.8,\n", - " length=0.5,\n", - " force_mag=10.0)\n", + " parallel_num=3)\n", "default_custom_cfg = {}\n", "default_result_key = [\"seed\"]\n", "\n", "\n", "def load_log(exp_name=\"default\",\n", - " task_name=\"ContinuousCartPoleSwingUp-v0\",\n", + " task_name=\"ParallelContinuousCartPoleSwingUp-v0\",\n", " params=default_params,\n", " dataset=\"SAC-expert-replay\",\n", " custom_cfg=default_custom_cfg,\n", @@ -104,6 +115,7 @@ " cfg = yaml.load(f, Loader=yaml.FullLoader)\n", "\n", " if not dict_equal(custom_cfg, cfg):\n", + " print(\"{} is passed cause its inconsistent cfg\".format(time_dir))\n", " continue\n", "\n", " log_path = time_dir / \"log\" / log_file\n", @@ -119,82 +131,133 @@ " for i, cfg in enumerate(cfg_list):\n", " result_dict[tuple([get_value(cfg, key) for key in diff_key])] = result_list[i]\n", " return diff_key, result_dict" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "outputs": [], + "source": [ + "def draw_result(exp_name=\"default\",\n", + " task_name=\"ParallelContinuousCartPoleSwingUp-v0\",\n", + " params=default_params,\n", + " dataset=\"SAC-expert-replay\",\n", + " custom_cfg=default_custom_cfg,\n", + " log_file=\"rollout.csv\",\n", + " log_key=\"ep_rew_mean\",\n", + " group_key=('transition', 'oracle')):\n", + " diff_key, result_dict = load_log(exp_name=exp_name,\n", + " task_name=task_name,\n", + " params=params,\n", + " dataset=dataset,\n", + " custom_cfg=custom_cfg,\n", + " log_file=log_file,\n", + " log_key=log_key)\n", + "\n", + " idx = diff_key.index(group_key)\n", + "\n", + " for name in set([key[idx] for key in result_dict.keys()]):\n", + " values = [value for key, value in result_dict.items() if key[idx] == name]\n", + " longest, longest_idx = argmax([value.shape[0] for value in values])\n", + " values_array = np.empty((len(values), longest))\n", + " for i, value in enumerate(values):\n", + " values_array[i, :len(value)] = value\n", + " values_array[i, len(value):] = values[longest_idx][len(value):]\n", + " plt.plot(values_array.mean(axis=0), label=name)\n", + " plt.fill_between(np.arange(len(values_array.mean(axis=0))),\n", + " values_array.mean(axis=0) - values_array.std(axis=0),\n", + " values_array.mean(axis=0) + values_array.std(axis=0), alpha=0.5)\n", + " plt.legend()" ], "metadata": { - "collapsed": false + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } } }, { "cell_type": "code", - "execution_count": 136, + "execution_count": 35, + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, "outputs": [ { "data": { "text/plain": "
", - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAigAAAGdCAYAAAA44ojeAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy88F64QAAAACXBIWXMAAA9hAAAPYQGoP6dpAABGY0lEQVR4nO3deXhU9d3+8fdkspGQRUI2ZJE1IbKDQnBDQQIEKo+2dkFBa9X6AxUXqlRaCy5Yi3UrVmtb0WofrPVxQ0AQBRcQYwANCAFZAyGENZMQss2c3x8nmSQQIBOSnFnu13XN5dlm5jOn08zNOefzPTbDMAxEREREvEiQ1QWIiIiInEwBRURERLyOAoqIiIh4HQUUERER8ToKKCIiIuJ1FFBERETE6yigiIiIiNdRQBERERGvE2x1AU3hcrnIz88nKioKm81mdTkiIiLSCIZhUFxcTIcOHQgKOvMxEp8MKPn5+XTq1MnqMkRERKQJ8vLy6Nix4xm38cmAEhUVBZgfMDo62uJqREREpDEcDgedOnVy/46fiU8GlJrTOtHR0QooIiIiPqYxl2foIlkRERHxOgooIiIi4nUUUERERMTr+OQ1KI1hGAZVVVU4nU6rS/Fbdrud4OBgtXqLiEiz88uAUlFRwf79+yktLbW6FL8XERFBcnIyoaGhVpciIiJ+xO8CisvlYufOndjtdjp06EBoaKj+hd8CDMOgoqKCgwcPsnPnTnr27HnWQXdEREQay+8CSkVFBS6Xi06dOhEREWF1OX6tTZs2hISEsHv3bioqKggPD7e6JBER8RN++09e/Wu+dWg/i4hIS/Do1+UPf/gDNput3iM1NdW9vqysjKlTpxIXF0fbtm257rrrOHDgQL3X2LNnD5mZmURERJCQkMCMGTOoqqpqnk8jIiIifsHjUzwXXnghH3/8ce0LBNe+xD333MOHH37IW2+9RUxMDNOmTePaa6/lyy+/BMDpdJKZmUlSUhKrV69m//79TJ48mZCQEB5//PFm+DgiIiLiDzw+Ph8cHExSUpL70b59ewCKior4xz/+wZ///GeuuuoqBg8ezCuvvMLq1av56quvAFi2bBnff/89r7/+OgMGDGDs2LE88sgjzJ8/n4qKiub9ZH5ixIgRTJ8+3eoyREREWpXHAWXbtm106NCBbt26MWnSJPbs2QNAdnY2lZWVjBo1yr1tamoqnTt3Zs2aNQCsWbOGvn37kpiY6N4mIyMDh8PBpk2bTvue5eXlOByOeg8RERHxXx4FlKFDh7JgwQKWLl3KX//6V3bu3Mlll11GcXExBQUFhIaGEhsbW+85iYmJFBQUAFBQUFAvnNSsr1l3OnPnziUmJsb96NSpkydli4iIiI/xKKCMHTuWn/zkJ/Tr14+MjAwWL17MsWPH+M9//tNS9QEwc+ZMioqK3I+8vDyPnm8YBqUVVa3+MAzDozqPHz/O5MmTadu2LcnJyTz11FP11peXl3P//fdz/vnnExkZydChQ1m5cqV7/YIFC4iNjeWjjz6id+/etG3bljFjxrB//373NitXruTiiy8mMjKS2NhYLrnkEnbv3u1e/9577zFo0CDCw8Pp1q0bs2fP1kXMIiKBprzE6grObRyU2NhYevXqxQ8//MDVV19NRUUFx44dq3cU5cCBAyQlJQGQlJTE119/Xe81arp8arZpSFhYGGFhYU2u80Slk7Tff9Tk5zfV93MyiAht/C6eMWMGq1at4r333iMhIYHf/va3rFu3jgEDBgAwbdo0vv/+exYuXEiHDh145513GDNmDDk5OfTs2ROA0tJS5s2bx7/+9S+CgoK44YYbuP/++3njjTeoqqpi4sSJ3Hrrrfzv//4vFRUVfP311+6B7D7//HMmT57Mc889x2WXXcb27du57bbbAHj44Yebd+eIiIh3On4IDmyEbiMsLeOcBrEoKSlh+/btJCcnM3jwYEJCQlixYoV7fW5uLnv27CE9PR2A9PR0cnJyKCwsdG+zfPlyoqOjSUtLO5dSfF5JSQn/+Mc/mDdvHiNHjqRv3768+uqr7qMXe/bs4ZVXXuGtt97isssuo3v37tx///1ceumlvPLKK+7Xqays5MUXX2TIkCEMGjSIadOmuf83cTgcFBUVMX78eLp3707v3r2ZMmUKnTt3BmD27Nk8+OCDTJkyhW7dunH11VfzyCOP8NJLL7X+DhERkdbncsHmD8Bl/ZFzj46g3H///UyYMIEuXbqQn5/Pww8/jN1u5+c//zkxMTHccsst3HvvvbRr147o6GjuvPNO0tPTGTZsGACjR48mLS2NG2+8kSeffJKCggJmzZrF1KlTz+kIydm0CbHz/ZyMFnv9M71vY23fvp2KigqGDh3qXtauXTtSUlIAyMnJwel00qtXr3rPKy8vJy4uzj0fERFB9+7d3fPJycnuQNiuXTtuuukmMjIyuPrqqxk1ahTXX389ycnJAHz77bd8+eWXPPbYY+7nO51OysrKKC0t1ci8IiL+Lu8rKC6AWOuv9fQooOzdu5ef//znHD58mPj4eC699FK++uor4uPjAXj66acJCgriuuuuo7y8nIyMDF544QX38+12O4sWLeKOO+4gPT2dyMhIpkyZwpw5c5r3U53EZrN5dKrFG5WUlGC328nOzsZurx982rZt654OCQmpt85ms9W7FuaVV17hrrvuYunSpbz55pvMmjWL5cuXM2zYMEpKSpg9ezbXXnvtKe+vYexFRPxcyUHY9YXVVbh59Ku9cOHCM64PDw9n/vz5zJ8//7TbdOnShcWLF3vytgGhe/fuhISEsHbtWvcpl6NHj7J161auuOIKBg4ciNPppLCwkMsuu+yc3mvgwIEMHDiQmTNnkp6ezr///W+GDRvGoEGDyM3NpUePHs3xkURExFe4XLDlA3A5ra7EzbcPK/iRtm3bcssttzBjxgzi4uJISEjgoYcect/rplevXkyaNInJkyfz1FNPMXDgQA4ePMiKFSvo168fmZmZZ32PnTt38re//Y0f/ehHdOjQgdzcXLZt28bkyZMB+P3vf8/48ePp3LkzP/7xjwkKCuLbb79l48aNPProoy36+UVExEJ7VkPxgbNv14oUULzIn/70J0pKSpgwYQJRUVHcd999FBUVude/8sorPProo9x3333s27eP9u3bM2zYMMaPH9+o14+IiGDLli28+uqrHD58mOTkZKZOncrtt98OmIPmLVq0iDlz5vDHP/6RkJAQUlNT+dWvftUin1dERLxA8QHYvbp23jCgYCP0GHX657QCm+HpYB1ewOFwEBMTQ1FREdHR0fXWlZWVsXPnTrp27arrJlqB9reIiA9zOSF7AZTUdteyfwNsXQoDboBr/gLVQ1E0hzP9fp/snNqMRURExIft/rJ+ODlxFH6oHi4kIbVZw4mnFFBEREQCUXEB7F5TO2/UjIFSCYkXwrCp1tWGAoqIiEjgcTlhyyIzlNTYswaK88EeBsPvhCBrI4ICioiISKDZ9YU57kkNx/7aMVB6Xg2R8dbUVYcCioiISCBx7Ic9X9XOOyvNMVAwID4VEi60rLS6FFBEREQCRUOndnZ8CieOQGhb6Jlh6YWxdSmgiIiIBIqdn5l3K65xZAfkrzOnUzIhpI01dTVAAUVERCQQOPIh7+va+coTkPuhOX3+YGjX1Zq6TkMBJYCsXLkSm83GsWPHrC5FRERak7MKtnxYe2rHMMzB2CqOQ0QcdB1Rf/uwMw+i1hoUULzIiBEjmD59ute9loiI+LhdJ53aObARDuWCLQhSJ4A9pHZdzPlw/pDWr/EkCig+xDAMqqqqrC5DRER8SdHe+qd2yo7BD8vN6S6XQlRS7brgMOj9I8vHQAEFFK9x0003sWrVKp599llsNhs2m40FCxZgs9lYsmQJgwcPJiwsjC+++IKbbrqJiRMn1nv+9OnTGTFixGlfa9euXe5ts7OzGTJkCBEREQwfPpzc3NzW+6AiItJ6nFWwZbF5SgfMUzxbPgRnBUSfD52H1d8+ZSy0iW31MhsSGHczNgyoLG399w2JaHS71rPPPsvWrVvp06cPc+bMAWDTpk0APPjgg8ybN49u3bpx3nnnNem14uPj3SHloYce4qmnniI+Pp5f//rX/PKXv+TLL79swgcUERGvtnMllB6und+bBUV5YA+F1PHmKZ4ayf0goXerl3g6gRFQKkvh8Q6t/76/zYfQyEZtGhMTQ2hoKBERESQlmYfbtmzZAsCcOXO4+uqrG/22Db1WXY899hhXXHEFYIafzMxMysrKdDdiERF/ciwP9n5TO19yAHauMqe7j4Q2df7BGxEHPRr/O9MadIrHBwwZ0rwXK/Xr1889nZycDEBhYeHpNhcREV/jrITcOqd2XFW1A7TF9YCk2t8BgoIh7RoIDrWm1tMIjCMoIRHm0Qwr3rcZREbWPwoTFBSEUfOlq1ZZWdn4skJqr9a2VZ+Ccrlcp9tcRER8zY5VUHqkdn7nKjh+0Pxd6jW2/uUH3UZAVGKrl3g2gRFQbLZGn2qxUmhoKE6n86zbxcfHs3HjxnrLNmzYUC94NPa1RETEzxzbA/vqnNo5usu89gQgZVz938O4HtDpolYtr7F0iseLXHDBBaxdu5Zdu3Zx6NCh0x7VuOqqq/jmm2947bXX2LZtGw8//PApgaWxryUiIn7EWVm/a6eqzDzVA5A8wAwkNUIjIXVcq5fYWAooXuT+++/HbreTlpZGfHw8e/bsaXC7jIwMfve73/Gb3/yGiy66iOLiYiZPntyk1xIRET+y/VM4cbR2fttyKHeYF8R2v6p2uc0GvSd49dkFm3HyxQw+wOFwEBMTQ1FREdHR9YfjLSsrY+fOnXTt2lVdKa1A+1tExEsc3Q3f/m/t0ZPCzbD5PcAGA28wxz2p0Xlo/cDSSs70+30yHUERERHxdVUV9bt2yoth21JzunN6/XASnQxdr2j9Gj2kgCIiIuLrdnwKJ46Z04Zh3qW4qtwcxr7LJbXbBYeaLcVBdkvK9IQCioiIiC87shPy19fO78s2O3eCgs0bAdYNIz0z6g/Q5sUUUERERHxVVTnkLqk9tXP8kDm8PUC3K80RYmsk9TEfPkIBRURExFdt/wTKisxplxO2fGCOGnteV+gwqHa7iHbm0RMf4rcBxQebk3yS9rOIiEWO7ID8DbXzu78w77cTHG4OyFYzWmyQHXr/yOuGsj8bvwsoNaOplpZacPfiAFSzn+uOYisiIi2s5tROjaK9sOcrc7rXGAiLql3X9XKzc8fH+N1Q93a7ndjYWPfN7yIiItz3m5HmYxgGpaWlFBYWEhsbi93u/VeEi4j4jR9WQJnDnK4qN0/tYEBiH4hPrd2uXVfoNNSSEs+V3wUUgKSkJEB36G0NsbGx7v0tIiKt4PB22P9t7fz2FeZ1KGHR0GNU7fLQCEgdX//GgD7ELwOKzWYjOTmZhIQEj+7yK54JCQnRkRMRkdZUWVb/1M6hrVDwnTmdOt68/gTMUJI6HsLatn6NzcQvA0oNu92uH1AREfEfP3xsjhILUFECW6vDSqehENu5druOQyCue+vX14z87iJZERERv3ToByjIMacNwzySUnkCIhPggstqt4tKNMdA8XEKKCIiIt6u8kTt0RKA/RvgyHaw2c27EgdVnxCxh0DaRJ8Yyv5sFFBERES83bblUF5iTpceMQdoA/Omf5Hxtdv1HG0OyuYHFFBERES82aFtcGCTOW24YMsicFWa15x0vKh2u8Q0SO5nTY0tQAFFRETEW1WeqN+1s3s1FOeDPQxSMmtbiNvEmgO0+REFFBEREW+1bRlUHDenHfth95fmdM/REB5jTtuCqoeyD7OmxhaigCIiIuKNDubCge/NaWdF7Wix8b0hIa12u66XQcz5lpTYkhRQREREvE1FKWxdWju/YyWcOAKhUebRk5pTO+d1gc7plpTY0hRQREREvM22j8yQAubQ9vnrzOmUcRDSxpwOaWO2GPvoUPZno4AiIiLiTQq3mA+AylLYuticPn+wefO/Gqnj69+12M8ooIiIiHiLiuPm0RMwR4vdutRcFhEHXUfUbtdxCLTvYUWFrUYBRURExFtsrXNq58BG82aAtiBInWCOEgvQNsEvhrI/GwUUERERb3Dge7NzB+DEMfhhuTl9wWUQlWRO24PNoeztfn2vX0ABRURExHrlJeaYJ2COFpu7yGwtjj7fvFNxjR6jIDLOmhpbmQKKiIiI1bYuNUeNBcj7Gor2gj3UPLVjq/6pTkiFDgOtq7GVKaCIiIhYqWCjeb8dgJIDsOszc7r7SHMIezBHje011pLyrKKAIiIiYpXyktprTVxVsPkD8xRPXE9Iqr7xny0I0n4EIeHW1WkBBRQRERGrbF0KlWXm9I5VUHoIQiLMoyU1A7BdcAnEdLSuRosooIiIiFihIKf21M7RXbAvy5xOGQehEeZ0bGfocokl5VlNAUVERKS1lRfDtupTO5VlkPuhOZ08AOKqB2ALCffroezPRgFFRESkteUuhapyc/qHZWZgaXMedL+qdpuUTAiPtqY+L6CAIiIi0pr2fwuHfzCnC783H9iqR4sNNZefPwjie1lWojdQQBEREWktZQ74YYU5Xe6ove9Ol+EQ3cGcjmxvthgHuHMKKE888QQ2m43p06e7lxUUFHDjjTeSlJREZGQkgwYN4u233673vCNHjjBp0iSio6OJjY3llltuoaSk5FxKERER8X65S8xTO4YBWz40p6OSofNwc30ADWV/Nk0OKFlZWbz00kv069ev3vLJkyeTm5vL+++/T05ODtdeey3XX38969evd28zadIkNm3axPLly1m0aBGfffYZt912W9M/hYiIiLfL3wBHdpjT+76BY7shKBhSx0OQ3Vze/SpoG29Zid6kSQGlpKSESZMm8fLLL3PeeefVW7d69WruvPNOLr74Yrp168asWbOIjY0lOzsbgM2bN7N06VL+/ve/M3ToUC699FKef/55Fi5cSH5+/rl/IhEREW9TVgTbq0/tHD8IO1aa092vgojqe+vE94LzB1tSnjdqUkCZOnUqmZmZjBo16pR1w4cP58033+TIkSO4XC4WLlxIWVkZI0aMAGDNmjXExsYyZMgQ93NGjRpFUFAQa9eubfD9ysvLcTgc9R4iIiI+I3cJVFWAywlbPgDDCed1g+Tqe+uERZnjn4ibxye5Fi5cyLp168jKympw/X/+8x9++tOfEhcXR3BwMBEREbzzzjv06GH2dRcUFJCQkFC/iOBg2rVrR0FBQYOvOXfuXGbPnu1pqSIiItbbtw6O7DSnd30BJYUQ3MYMJDab+Uj7EYS0sbZOL+PREZS8vDzuvvtu3njjDcLDG74nwO9+9zuOHTvGxx9/zDfffMO9997L9ddfT05OTpOLnDlzJkVFRe5HXl5ek19LRESk1Zw4Bts/MaeL8iDvK3O61xgIa2tOdxlujhgr9Xh0BCU7O5vCwkIGDRrkXuZ0Ovnss8/4y1/+Qm5uLn/5y1/YuHEjF154IQD9+/fn888/Z/78+bz44oskJSVRWFhY73Wrqqo4cuQISUlJDb5vWFgYYWFhnn42ERER6xgG5C4GZ6XZrbNlEWBAYh+ITzG3iekIXS61tExv5VFAGTly5ClHQm6++WZSU1N54IEHKC0tBSAoqP6BGbvdjsvlAiA9PZ1jx46RnZ3N4MHmxUCffPIJLpeLoUOHNvmDiIiIeJX8dXB0tzm9/WPzQtmwaOhxtbksOMwcyj5IQ5I1xKOAEhUVRZ8+feoti4yMJC4ujj59+lBZWUmPHj24/fbbmTdvHnFxcbz77rvudmKA3r17M2bMGG699VZefPFFKisrmTZtGj/72c/o0KFD830yERERq5w4Cts/NacP5Zo3BgRztNjg6jMCKeOgTawl5fmCZo1tISEhLF68mPj4eCZMmEC/fv147bXXePXVVxk3rvbq5DfeeIPU1FRGjhzJuHHjuPTSS/nb3/7WnKWIiIhYwzBgS/WpnYoS2LrUXN5pGMR2MqeT+0NCqnU1+gCbYRiG1UV4yuFwEBMTQ1FREdHRgXsjJRER8UJ7vzHvVGwYsPG/cGQ7RCbAoMnmwGyR7WHwTWAPsbrSVufJ77dOfImIiDSX0iOwo/rUzv4NZjix2auvNQk2H2nXBGQ48ZQCioiISHNwd+1UmUGlpr242xUQWT18ffcroW3C6V9D3BRQREREmsPeb+BYXu1osa5KiO0C519kro/rAR2HnPk1xE0BRURE5FyVHoGdK83pPWugeL/ZrZOSaY4UG9YWUjMtLdHXKKCIiIicC8MwB2FzVoEjH3Z/aS7vmQHh0WZA6T0BQiOsrdPHKKCIiIici7yvoWgfOCvMUzsYEN8bEtLM9Z2GwnkXWFmhT1JAERERaarjh2HnZ+b0jk/NAdpCo8yjJwDRHaDrFdbV58MUUERERJrC5TJP7biq4PB2yF9vLk/NhJBwCA4171KsoeybRHtNRESkKfZ+bV5zUllqthcDnD+k9nROrzHQ5jzLyvN1CigiIiKeOn4Idn5uXiC7dSlUHoeI9rWnc5L6QuKF1tbo4xRQREREPFH31M6BHDi0FWxBkDreHCE2oh30HG11lT5PAUVERMQTeV+BYz+cOAY/fGwuu+AyiEqCILs5lH1wqKUl+gMFFBERkcYqOQi7vgCj+iiKswKiO5qtxADdRphBRc6ZAoqIiEhjuE/tOCFvLTj2gj3UPLVjC4J23aDjRVZX6TcUUERERBpjzxooLjAfuz43l/UYBW1iITQSeo83R42VZqGAIiIicjYlheYQ9s5K8yiK4YL2vSCxb/VQ9uPNkCLNRgFFRETkTFzO2lM7O1dB6SEIiYSeY8xw0vEi8/SONCsFFBERkTPZvRqKD8DRXbDvG3NZyljz5n9RSeaFsdLsFFBEREROp/iAee1JZRls+dBcljwQ4nqYY56kXWO2FkuzU0ARERFpiMtp3p3Y5YQflkFFsTl0ffcrzfW9MsxB2aRFKKCIiIg0ZNcX5rgnhd+bD2yQOsFsLU680BzOXlqMAoqIiMjJHPthz1dQ5oBtH5nLugyH6A7mUZReGdbWFwAUUEREROqq27WT+yFUlUNUMnQeXj2U/Y8gOMzqKv2eAoqIiEhduz4371a87xs4thuCQsxTO0F285470R2srjAgKKCIiIjUcOTDnrVw/CDsWGku636VeTFsu67QeZil5QUSBRQREREAZ5XZSuysgM0fgOGEdt0heYA55kmqhrJvTQooIiIiALs+M0/t7PoCjhdCcBtzQLagIDOchLW1usKAooAiIiJStA/ysuBYHuR9ZS5LGQOhbeH8IRDX3dr6ApACioiIBLaaUzuVJyB3kbksqS+0T4GoxNqB2aRVKaCIiEhg27kKSg/DDx9DWRGEx0D3UeZQ9r01lL1VFFBERCRwFe2FvVlwMBcO5JjLUsab45z0vBoi46ytL4ApoIiISGByVpqndsqKYetSc1mnYRDbCRJ6Q3J/a+sLcAooIiISmHasguOHYetiqDoBbRPMgdjCY6DXGKurC3gKKCIiEniO7TFHit2/Ho7sAJu9+kaAIeZQ9iHhVlcY8BRQREQksDgrYcti8+jJ9k/MZd1GQGQ8XHApxHS0tDwxKaCIiEhg2bHSHJBtywfgqoLYLuZYJ7GdzTsWi1dQQBERkcBxdDfsy4Y9q6F4v9mtk5ppDmXfe4KGsvciCigiIhIYqiogd7E5auzu1eaynhkQFm2GlPBoa+uTehRQREQkMOz4FEoKzVM7GJCQZj7OHwzte1pdnZxEAUVERPzf0V2Qv968KPbEUQiLgh6joW08dL/K6uqkAQooIiLi36rKza6dQ9tg/wZzWUomhLeFtIlgD7ayOjkNBRQREfFv2z8Fx37IXWLOn38RnHcBdB8Jke0tLU1OTwFFRET815EdsG8dbF0Clcchoj10uwLiU+D8QVZXJ2eggCIiIv6pqtw8anIgBw5vA1uQ2Uoc0Q5SxlpdnZyFAoqIiPinH1bA0T3ww8fm/AWXQ1SyGVJC2lhbm5yVAoqIiPifw9vNrp0ti8BZYQ5f3+lic6TY2M5WVyeNoIAiIiL+pbLMPLWTtxYce8EeCqnj4bwu0OUSq6uTRlJAERER/7J9hdlSvOtzc77HKIhKMk/tBOlnz1fofykREfEfh7fD3mxztFjDBe17QWJfSBkH4TFWVyceUEARERH/UFlm3mtn5yooPQyhkdBrDJw/0GwrFp+igCIiIv7hh+WwPwf2fWPO9xpnXhDbY5S1dUmTKKCIiIjvO7QN8rIg90NzvkP1UZO0a8AeYm1t0iQKKCIi4tsqT8DWpbBtGVSUQJt20O1K8yaAbROsrk6aSAFFRER827ZlsGctHNwM2MyW4sQLoeNgqyuTc6CAIiIivuvgVtj9lRlSwBznJD7F7NoRn6aAIiIivqmi1OzayV0EznJzGPsul5jjnYRGWF2dnKNzCihPPPEENpuN6dOn11u+Zs0arrrqKiIjI4mOjubyyy/nxIkT7vVHjhxh0qRJREdHExsbyy233EJJScm5lCIiIoFm2zKzpfjYHggKgdQJcMEl5oix4vOaHFCysrJ46aWX6NevX73la9asYcyYMYwePZqvv/6arKwspk2bRlCd0fsmTZrEpk2bWL58OYsWLeKzzz7jtttua/qnEBGRwFK4BXZ+BjtWmfPdr4LkvnDBZdbWJc0muClPKikpYdKkSbz88ss8+uij9dbdc8893HXXXTz44IPuZSkptQPkbN68maVLl5KVlcWQIUMAeP755xk3bhzz5s2jQ4cOTSlJREQCRcVxs5148wdgOKFdd+g0FHr/SEPZ+5Em/S85depUMjMzGTWq/uA3hYWFrF27loSEBIYPH05iYiJXXHEFX3zxhXubNWvWEBsb6w4nAKNGjSIoKIi1a9c28WOIiEjA2PqR+TheCCFtIGUspI6DNrFWVybNyOMjKAsXLmTdunVkZWWdsm7Hjh0A/OEPf2DevHkMGDCA1157jZEjR7Jx40Z69uxJQUEBCQn1+9KDg4Np164dBQUFDb5neXk55eXl7nmHw+Fp2SIi4g8KN8O25eadigF6jYUuwyGht7V1SbPz6AhKXl4ed999N2+88Qbh4eGnrHe5XADcfvvt3HzzzQwcOJCnn36alJQU/vnPfza5yLlz5xITE+N+dOrUqcmvJSIiPqriOHz/HmxZZM4n9YPO6dDjamvrkhbhUUDJzs6msLCQQYMGERwcTHBwMKtWreK5554jODiYxMREANLS0uo9r3fv3uzZsweApKQkCgsL662vqqriyJEjJCUlNfi+M2fOpKioyP3Iy8vzpGwREfEHW5ea152UO8w7E/fMMIeyDw61ujJpAR6d4hk5ciQ5OTn1lt18882kpqbywAMP0K1bNzp06EBubm69bbZu3crYsWMBSE9P59ixY2RnZzN4sDnK3yeffILL5WLo0KENvm9YWBhhYWGelCoiIv7kwCb4/n04sBFztNgJ0CsDohKtrkxaiEcBJSoqij59+tRbFhkZSVxcnHv5jBkzePjhh+nfvz8DBgzg1VdfZcuWLfz3v/8FzKMpY8aM4dZbb+XFF1+ksrKSadOm8bOf/UwdPCIicqryEtj4tnkEBaDTMOg2AjpdZGlZ0rKa1GZ8JtOnT6esrIx77rmHI0eO0L9/f5YvX0737t3d27zxxhtMmzaNkSNHEhQUxHXXXcdzzz3X3KWIiIg/yF0CG9+BqjJomwi9RptdO+LXbIZhGFYX4SmHw0FMTAxFRUVER0dbXY6IiLSUghz4eA78sAxsdhjyS0ifCu26Wl2ZNIEnv98a0UZERLxTeTF8uxB2fGLOdxthHjlROAkICigiIuKdNi+Cjf8FVxXEXgC9x0PXK6yuSlqJAoqIiHif/d/B+n9BcQEEh8GFE+HC/4Egu9WVSStRQBEREe9S5oDsBbB7tTnfcwz0/Qm0Oc/SsqR1KaCIiIh3+f5d2PR/gAEJadDvJ5DU52zPEj+jgCIiIt4jfwNk/QNOHIWwKOj3U3PEWAk4CigiIuIdyopg7V9h/wZzvvePzICioewDkgKKiIh4h+/eNIezB+h4EQyaAtHJ1tYkllFAERER6+1bB2v/BpWlEBlvhpNOF1tdlVhIAUVERKx14hh88Wc4vA1sQWbHzoX/Azab1ZWJhRRQRETEOoYB616D3OobAXa9Ai76FYS1tbYusZwCioiIWGdvFqx9EVyVENMJht0Bcd3P/jzxewooIiJijRNH4dPHwbEP7KEw+CboMcrqqsRLKKCIiEjrMwz46kXYucqc7zXGvFOxhrKXagooIiLS+navgayXwXBB+xS47H6IaGd1VeJFFFBERKR1lR6Bjx+G0sMQGgmX3AUd+ltdlXgZBRQREWk9hgGfPwV7vzbn+15vthWLnEQBRUREWs/2T8w7FQN0GASXz4DgMEtLEu+kgCIiIq3j+GFYNgsqSqBNOxj5O4g53+qqxEspoIiISMszDFgxGwq/N0eLvfg26Hal1VWJF1NAERGRlrflQ/h2oTnd9QoYPk1D2csZKaCIiEjLKjkIH/0WnOUQ1QHGPAFhUVZXJV5OAUVERFqOywWL74djuyEoBK6cCQmpVlclPkABRUREWs53b8LmD8zpCydC/19YWo74DgUUERFpGUX7YPnvwXBCXE/IeALswVZXJT5CAUVERJqfywXvT4PjhRDSBsb+Edq2t7oq8SEKKCIi0vyyXjYHZQO46FboMdLaesTnKKCIiEjzOrwDPnnUnD5/CFz5kLX1iE9SQBERkebjcsE7t0G5A8JjYeILEBJudVXigxRQRESk+Xz2J9ibBdjMIyfxKVZXJD5KAUVERJpHQQ588bQ53XM0XHyrtfWIT1NAERGRc+d0wtu/gqoTEJVsntrRUPZyDhRQRETk3C37LRzcAkHBMP5ZiFRLsZwbBRQRETk3u1ZD1t/N6YE3QkqGtfWIX1BAERGRpqssg3duBVcVtO8FY5+0uiLxEwooIiLSdB/cBUV7ITgcrvsnBIdaXZH4CQUUERFpmtwl8N1/zOnLfwPJfa2tR/yKAoqIiHiurAjemwoY0DkdLr/P6orEzyigiIiI597+FZQehvAY+MkCq6sRP6SAIiIinln/BmxbZk5n/hmikqytR/ySAoqIiDSeYz8s+Y05nTYR+v7Y0nLEfymgiIhI4xgG/GcyVJRAVAeY+FerKxI/poAiIiKN8+VzsPdrsNnhJ69AaITVFYkfU0AREZGzO7gNPn3UnB52B3QeZm094vcUUERE5MycVfDmJHBWQHxvuHqO1RVJAFBAERGRM1sxGw7lmqPF/uzfEGS3uiIJAAooIiJyenuzYM1fzOmrH4G4btbWIwFDAUVERBpWUQr/mQKGC7peDhffanVFEkAUUEREpGGL7wPHvurRYl8Fm83qiiSAKKCIiMipti6DDf82pyf+FSLaWVuPBBwFFBERqa/0CLxzmznd76eQmmltPRKQFFBERKSWYcA7t8OJoxB9Pox/xuqKJEApoIiISK0N/zZvBGgLguv/pdFixTIKKCIiYjq2x7wwFuDSe6DjYGvrkYCmgCIiIuBywVs3Q+UJSOwDI35rdUUS4BRQREQEvnwW9n0DwWFw/WtgD7a6IglwCigiIoGuYGPtjQBHz4W47tbWI8I5BpQnnngCm83G9OnTT1lnGAZjx47FZrPx7rvv1lu3Z88eMjMziYiIICEhgRkzZlBVVXUupYiISFNUlsFbN4GrCrpdCRf90uqKRABo8jG8rKwsXnrpJfr169fg+meeeQZbA6MOOp1OMjMzSUpKYvXq1ezfv5/JkycTEhLC448/3tRyRESkKVbMhsPbIDwWrv2bRosVr9GkIyglJSVMmjSJl19+mfPOO++U9Rs2bOCpp57in//85ynrli1bxvfff8/rr7/OgAEDGDt2LI888gjz58+noqKiKeWIiEhT7PwcvvqrOT3xBWibYG09InU0KaBMnTqVzMxMRo0adcq60tJSfvGLXzB//nySkpJOWb9mzRr69u1LYmKie1lGRgYOh4NNmzY1pRwREfFUWRH8368AAwb8QqPFitfx+BTPwoULWbduHVlZWQ2uv+eeexg+fDjXXHNNg+sLCgrqhRPAPV9QUNDgc8rLyykvL3fPOxwOT8sWEZG6PrwPigsguiOM/ZPV1YicwqOAkpeXx913383y5csJDw8/Zf3777/PJ598wvr165utQIC5c+cye/bsZn1NEZGAtekdyHnLHC32x/+EsLZWVyRyCo9O8WRnZ1NYWMigQYMIDg4mODiYVatW8dxzzxEcHMzy5cvZvn07sbGx7vUA1113HSNGjAAgKSmJAwcO1HvdmvmGTgkBzJw5k6KiIvcjLy/P088pIiIAjnx4/y5z+pJ7oPNQa+sROQ2PjqCMHDmSnJycestuvvlmUlNTeeCBB2jfvj233357vfV9+/bl6aefZsKECQCkp6fz2GOPUVhYSEKCeUHW8uXLiY6OJi0trcH3DQsLIywszJNSRUTkZC4XvHsHlDsgsS9cOdPqikROy6OAEhUVRZ8+feoti4yMJC4uzr28oaMgnTt3pmvXrgCMHj2atLQ0brzxRp588kkKCgqYNWsWU6dOVQgREWlJWX+HHSvBHgo//gfYQ6yuSOS0Wn0kWbvdzqJFi7Db7aSnp3PDDTcwefJk5syZ09qliIgEjoO5sGyWOT36MYhPsbYekbOwGYZhWF2EpxwOBzExMRQVFREdHW11OSIi3q2qAv4+Egq+g24j4IZ3IEh3OpHW58nvt76hIiL+btUfzXASFgMTX1Q4EZ+gb6mIiD/bsxa++LM5/aNnITrZ2npEGkkBRUTEX5WXwP/dCoYL+v0ULvwfqysSaTQFFBERf/XRTDi2G6LPh3EaLVZ8iwKKiIg/2rIY1r0G2My7FIfHWF2RiEcUUERE/E1JIbw/zZwePg0uuNTaekSaQAFFRMSfGIY5lH3pYUhIg6t+Z3VFIk3i8d2M/dm2A8Ws3XnE6jIkQNhsJ81jO8t6D55/1ufazrK+8e918rqTefxeZ9gPZ3vuyVucfR+evjZP9kFDL36m9zqXfdDQ8+suSNi2kB5bl+AKCuG7i/9E6e5ioPjkZ4icVWJ0ON3jrbuRpAJKHVm7jjLr3Y1WlyEi0iRdbAUsDp0DNphb/hNe/m8RsNbqssRH/WJoZx7/n76Wvb8CSh0dz2tDxoWJVpchAeDk8ZtPHs751PGdjTOuP/X5xlnWe/b8U6o55flnqc/T7U95w9M/v6X3xckbnP35nu6Lk9c3bd8HGU7+fPwlIp3lbLD35bPo6+ll01l8abrEqHBL318BpY7Le8Vzea94q8sQEfHcqj/Bp7kQFsWAO/7NR7Gdra5I5JwoXouI+Lp962DVE+b0uKdA4UT8gAKKiIgvqyiF/7sNXFWQNhH6XW91RSLNQgFFRMSXLf89HN4GbZNg/NNnb6sS8RG6BkVExBs4q6CqrM6jHCpPmP+tOtHwsuICyHrZfP7EFyCinbWfQaQZKaCIiNQwDHBW1AkBdQJDZZ3gUHWigQBRftJ2pwsaNfMnbeOqanrdF98OPUY2334Q8QIKKCLifVzOOmGg7DRB4IT5qCg1l1eUVj/nRG0AqDwpODjL67xuefV8RZ3/VtBAo3Hrs4dCcDgEh0FwG/O/IeHVy8Jr14W0gfYp5nD2In5GAUUk0BmGGQgMZ+1/DZd5ysFZDhUnqn/0S6sfJ2p/8CtLT3/U4HRBwFlRGwacFeCsBFdl7bSz0qzBcjYIDgV7WHVQqBsW2piP4Jr/hp8+QNR9XkPbNfS8ILvVH17EcgooIq3BWWXeG+XkEOByNbDs5P9WnhQAqo8kVJafekSgqjoA1AsE1UHDWR0CXJW1QcBVZc67nOa0s6p6WfXDG44m2ILMIwr2EDMs2EPrB4e6P/Du0BABoW0gJBJCI8z/NiZAhNQJEvZQXXAqYiEFFJGW5tgPK2bD4e11fvyd9YNAQ4+asOAVRxOAoGDzX/ZBIea0OzTUDQ4NHW2IMMNBSET1kYeI6gARWefRtvoRUf/Igj0M7PozJRKI9P98kZZiGLD5A1j2Ozi2q5le1GYGgiB7dWA4w8N9pKHmeobQ6tBQc1qhTe2piprAENKmOijUBIcoCKueDw4zw4kCg4i0Av2lEWkJJ47Bx3+Ab/9tnnYJCoGkvmY4cAeI4DpHHdrUOQXRpvq0xMlHG9pWH1UIMZ8XFFI7bQ85dblOT4iID1NAEWluu1fD4hlwoPrO2FHJcNUs6DGqNkwE1YQJjZUoItIQBRSR5lJ5Ar54Gr76K5Q7ABt0vxLGzYO47lZXJyLiUxRQRJpD4Rb48D7Y/YU5Hx4Ll0yH9P9nnsIRERGPKKCInAtnFax7DVbOheOF5rIOgyDzKTh/kLW1iYj4MAUUkaZy7Iclv4EtH5qtwCFtYPDNcOVvISzK6upERHyaAoqIpwwDtiyCZbPg6C5zWVwPyJgLPa9W94yISDNQQBHxxIlj5qBrG96obR++cKIZTtrGW12diIjfUEARaazdX8GSGVDwnTlf0z7c/+e6d4qISDNTQBE5m8oT8OUzsOYFtQ+LiLQSBRSRMynMhcX3wa7PzfnwWLjkbkifqvZhEZEWpIAi0hC1D4uIWEoBReRkDbUPD7oJrnpI7cMiIq1EAUWkRkPtw+16wBi1D4uItDYFFBGAsiL4uKZ9uMy823DaRBjzhNqHRUQsoIAickr7cBJc+TsYoPZhERGrKKBI4Ko8AV8+C2vm17YPd7sSMtU+LCJiNQUUCUwNtg/fBenT1D4sIuIFFFAksDTYPjzQHHSt4xBraxMRETcFFAkcxQWw+Ddmp05N+/DAKWb7cHi01dWJiEgdCiji/wzDHNNk2Sw4utNc1q67eYO/XqPVPiwi4oUUUMS/Ndg+fA1kPAFRCVZXJyIip6GAIv6rwfbhWTDgF2ofFhHxcgoo4n8qy6rvPly3fXiEeR8dtQ+LiPgEBRTxL4e2waJ7Yddn5nx4LAy/02wfDgm3tDQREWk8BRTxD84qWP8arHwCSg6Yy9Q+LCLisxRQxPcVF8CSB2DzB2b7cHAbGDQZrpql9mERER+lgCK+yzBgy2JY9tBJ7cOPQ68MtQ+LiPgwBRTxTWVFsGIOrH+9tn249zXm3YfVPiwi4vMUUMT37FkLi2dAwbfmfNskczTYAZPUPiwi4icUUMR3VJbB6udg9fN12oevgHFPQfseVlcnIiLNSAFFfMOhbfDhfbBzlTkfHgPD71L7sIiIn1JAEe/mrIL1/zLvPlzTPpw8wGwf7nSRpaWJiEjLUUAR71XTPrxlEbiq1D4sIhJAFFDE+xgG5C6Bjx6CozvMZe26Q8Zj0GuM2odFRAKAAop4l7IiWPGIeVrH3T48AcY8qfZhEZEAEnQuT37iiSew2WxMnz4dgCNHjnDnnXeSkpJCmzZt6Ny5M3fddRdFRUX1nrdnzx4yMzOJiIggISGBGTNmUFVVdS6liD/I+xpenQBZL5vhpG0SZP4ZrvuHwomISIBp8hGUrKwsXnrpJfr16+delp+fT35+PvPmzSMtLY3du3fz61//mvz8fP773/8C4HQ6yczMJCkpidWrV7N//34mT55MSEgIjz/++Ll/IvE9Ne3Da/5iHkFR+7CISMCzGYZhePqkkpISBg0axAsvvMCjjz7KgAEDeOaZZxrc9q233uKGG27g+PHjBAcHs2TJEsaPH09+fj6JiYkAvPjiizzwwAMcPHiQ0NDQs76/w+EgJiaGoqIioqN1saRPa7B9+E5Iv1PtwyIifsaT3+8mneKZOnUqmZmZjBo16qzb1hQRHGwerFmzZg19+/Z1hxOAjIwMHA4HmzZtavA1ysvLcTgc9R7i45xV8M0CWJBZG06SB8Ckt+HyGQonIiIBzuNTPAsXLmTdunVkZWWdddtDhw7xyCOPcNttt7mXFRQU1AsngHu+oKCgwdeZO3cus2fP9rRU8VbFhbBkxkntwzfCVb9T+7CIiAAeHkHJy8vj7rvv5o033iA8/Mz/wnU4HGRmZpKWlsYf/vCHc6mRmTNnUlRU5H7k5eWd0+uJRWruPvzPDPj+XTOctOsGP3kFxj6pcCIiIm4eHUHJzs6msLCQQYMGuZc5nU4+++wz/vKXv1BeXo7dbqe4uJgxY8YQFRXFO++8Q0hIiHv7pKQkvv7663qve+DAAfe6hoSFhREWFuZJqeJtyhzVdx8+uX34jxCVePbni4hIQPEooIwcOZKcnJx6y26++WZSU1N54IEHsNvtOBwOMjIyCAsL4/333z/lSEt6ejqPPfYYhYWFJCSYraPLly8nOjqatLS0c/w44pXysmDxfbC/zt2Hr/ytefdhu4biERGRU3n06xAVFUWfPn3qLYuMjCQuLo4+ffrgcDgYPXo0paWlvP766/UuaI2Pj8dutzN69GjS0tK48cYbefLJJykoKGDWrFlMnTpVR0n8TUPtw10vh8ynoH1Pq6sTEREv1qz/fF23bh1r164FoEeP+uNX7Ny5kwsuuAC73c6iRYu44447SE9PJzIykilTpjBnzpzmLEWsdugH+PBe2PkZYJjtw+nTzDsQq0NHRETOoknjoFhN46B4MZcT1r8Onz4OJdVdWckDYNyfoNPFlpYmIiLW8uT3WxcASPMpLoSlv4HNH1S3D4fDwBth5O/MIygiIiKNpIAi567m7sPLHoIjNXcf7gajH4OUsbr7sIiIeEwBRc5NmQM+eRTWvar2YRERaTYKKNJ0eVmw+H7Yv8Gcb5sIVz6k9mERETln+hURz1WWwernYc3z1e3DQNcr1D4sIiLNRgFFPHPoB3PQtR2rAAPCos27D6t9WEREmpECijROg+3D/WHcPLUPi4hIs1NAkbMrLoSlD8Dm99U+LCIirUIBRU7PMGDrUvjoITiy3Vx2XjfIUPuwiIi0LAUUaZi7ffg1qDphtg+njoexf4Sohu86LSIi0lwUUORUDbUPj5hpntZR+7CIiLQC/dpIrcoy887Dq5+r0z58OWT+We3DIiLSqhRQxNRQ+3D6VLjkbghpY3V1IiISYBRQAp3LCRvegE8eq20fTupv3n2481BraxMRkYClgBLIGmwfvgFG/l7twyIiYikFlEBkGLB1GXw0s077cFfz7sOp49Q+LCIillNACTQntw/b7NB7PIx9Uu3DIiLiNRRQAsneb+DD+05qH34QBk5W+7CIiHgV/SoFgqpy8+7Dq5+HsmPmsgsuN+8+HN/L0tJEREQaooDi7w5vN4+a7FiJ2odFRMRXKKD4K5cTNvwbPn0Uimvah/vB2D9Bl2HW1iYiInIWCij+qOQgLHkANr9X2z484AYYpfZhERHxDQoo/uS07cOPQmqm2odFRMRnKKD4i/JiczTYdQugsrp9uObuw9HJVlcnIiLiEQUUf7D3G/Puw/nrzXm1D4uIiI/Tr5cvqyqHNfPhy2frtw+PmwcJKZaWJiIici4UUHyV2odFRMSPKaD4GpcTNvwvfPqI2odFRMRvKaD4kpKD5t2Hv6/bPjwJRj2s9mEREfErCii+wDBg23JYOhOO/GAuU/uwiIj4MQUUb9dg+3CmefdhtQ+LiIifUkDxZvvWwYf31rYPRybAiJkwSO3DIiLi3/Qr540abB++DMY9pfZhEREJCAoo3ubwjur24U+pbR/+f3DJdLUPi4hIwFBA8RYuJ3z7v/DJo1C831yW1Le6fTjd2tpERERamQKKNyg5CB89CJveA1clBIeZ7cMjH4Y2sVZXJyIi0uoUUKxU0z780Uw4XNM+fAFc/Qj0nqD2YRERCVgKKFZR+7CIiMhpKaBYYd8680LY/HXmvNqHRURE6tGvYWuqKoc1L8CXz9RpH760un041crKREREvIoCSms5vAMW3wfb67QPD7sDLr1H7cMiIiInUUBpaS4nfLcQVjxyUvvwk9BluLW1iYiIeCkFlJZUctDs0Nn0bm37cP9fwKg/qH1YRETkDBRQWoJhwLaPq9uHt5nLYi8w24fT1D4sIiJyNgooza28GD59HLIXQGWp2odFRESaQAGlOe1bB4vvh33Z5nxkPFzxIAy+Se3DIiIiHtCvZnOoaR9e/SycOGou63IpZKp9WEREpCkUUM7V4R3mUZPtn+BuHx76a7jsXrUPi4iINJECSlO5nPDdm7BiTm37cGIfGPcntQ+LiIicIwWUpig5VH334XfN9mF7GPT/OVw9W+3DIiIizUABxVNbl5/UPtwFrn5U7cMiIiLNSAGlscqL4dO5kP1KbftwyjgY9yREd7C6OhEREb+igNIY+9ab99Gp1z78QHX7cIilpYmIiPgjBZQzqSqHr/5q3n3Y3T58SXX7cG9LSxMREfFnCiinc2QnfHhfnfbhKBh6h9qHRUREWoECysnc7cOPQHG+uSyxjzlU/QWXWFubiIhIgFBAqcswYNUT8PnTah8WERGxUNC5PPmJJ57AZrMxffp097KysjKmTp1KXFwcbdu25brrruPAgQP1nrdnzx4yMzOJiIggISGBGTNmUFVVdS6lNI+PH4ZVT5rhJLYLXPs3mPCMwomIiEgra3JAycrK4qWXXqJfv371lt9zzz188MEHvPXWW6xatYr8/HyuvfZa93qn00lmZiYVFRWsXr2aV199lQULFvD73/++6Z+iufT/BYS2hdTx8MulcOFEjW0iIiJiAZthGIanTyopKWHQoEG88MILPProowwYMIBnnnmGoqIi4uPj+fe//82Pf/xjALZs2ULv3r1Zs2YNw4YNY8mSJYwfP578/HwSExMBePHFF3nggQc4ePAgoaGhZ31/h8NBTEwMRUVFREdHe1r+mR3bA1HJah8WERFpZp78fjfpCMrUqVPJzMxk1KhR9ZZnZ2dTWVlZb3lqaiqdO3dmzZo1AKxZs4a+ffu6wwlARkYGDoeDTZs2Nfh+5eXlOByOeo8WE9tZ4URERMRiHl8ku3DhQtatW0dWVtYp6woKCggNDSU2Nrbe8sTERAoKCtzb1A0nNetr1jVk7ty5zJ4929NSRURExEd5dAQlLy+Pu+++mzfeeIPw8PCWqukUM2fOpKioyP3Iy8trtfcWERGR1udRQMnOzqawsJBBgwYRHBxMcHAwq1at4rnnniM4OJjExEQqKio4duxYvecdOHCApKQkAJKSkk7p6qmZr9nmZGFhYURHR9d7iIiIiP/yKKCMHDmSnJwcNmzY4H4MGTKESZMmuadDQkJYsWKF+zm5ubns2bOH9PR0ANLT08nJyaGwsNC9zfLly4mOjiYtLa2ZPpaIiIj4Mo+uQYmKiqJPnz71lkVGRhIXF+defsstt3DvvffSrl07oqOjufPOO0lPT2fYsGEAjB49mrS0NG688UaefPJJCgoKmDVrFlOnTiUsLKyZPpaIiIj4smYfSfbpp58mKCiI6667jvLycjIyMnjhhRfc6+12O4sWLeKOO+4gPT2dyMhIpkyZwpw5c5q7FBEREfFRTRoHxWotOg6KiIiItIgWHwdFREREpCUpoIiIiIjXUUARERERr6OAIiIiIl5HAUVERES8jgKKiIiIeB0FFBEREfE6zT5QW2uoGbrF4XBYXImIiIg0Vs3vdmOGYPPJgFJcXAxAp06dLK5EREREPFVcXExMTMwZt/HJkWRdLhf5+flERUVhs9ma9bUdDgedOnUiLy9Po9SehfZV42lfNZ72VeNpXzWe9pVnWmp/GYZBcXExHTp0ICjozFeZ+OQRlKCgIDp27Nii7xEdHa0vcSNpXzWe9lXjaV81nvZV42lfeaYl9tfZjpzU0EWyIiIi4nUUUERERMTrKKCcJCwsjIcffpiwsDCrS/F62leNp33VeNpXjad91XjaV57xhv3lkxfJioiIiH/TERQRERHxOgooIiIi4nUUUERERMTrKKCIiIiI1wnIgDJ//nwuuOACwsPDGTp0KF9//fUZt3/rrbdITU0lPDycvn37snjx4laq1Hqe7KsFCxZgs9nqPcLDw1uxWut89tlnTJgwgQ4dOmCz2Xj33XfP+pyVK1cyaNAgwsLC6NGjBwsWLGjxOr2Bp/tq5cqVp3yvbDYbBQUFrVOwRebOnctFF11EVFQUCQkJTJw4kdzc3LM+LxD/XjVlXwXy36u//vWv9OvXzz0IW3p6OkuWLDnjc6z4XgVcQHnzzTe59957efjhh1m3bh39+/cnIyODwsLCBrdfvXo1P//5z7nllltYv349EydOZOLEiWzcuLGVK299nu4rMEcd3L9/v/uxe/fuVqzYOsePH6d///7Mnz+/Udvv3LmTzMxMrrzySjZs2MD06dP51a9+xUcffdTClVrP031VIzc3t953KyEhoYUq9A6rVq1i6tSpfPXVVyxfvpzKykpGjx7N8ePHT/ucQP171ZR9BYH796pjx4488cQTZGdn880333DVVVdxzTXXsGnTpga3t+x7ZQSYiy++2Jg6dap73ul0Gh06dDDmzp3b4PbXX3+9kZmZWW/Z0KFDjdtvv71F6/QGnu6rV155xYiJiWml6rwXYLzzzjtn3OY3v/mNceGFF9Zb9tOf/tTIyMhowcq8T2P21aeffmoAxtGjR1ulJm9VWFhoAMaqVatOu00g/72qqzH7Sn+v6jvvvPOMv//97w2us+p7FVBHUCoqKsjOzmbUqFHuZUFBQYwaNYo1a9Y0+Jw1a9bU2x4gIyPjtNv7i6bsK4CSkhK6dOlCp06dzpjIA12gfq/OxYABA0hOTubqq6/myy+/tLqcVldUVARAu3btTruNvlemxuwr0N8rAKfTycKFCzl+/Djp6ekNbmPV9yqgAsqhQ4dwOp0kJibWW56YmHja89kFBQUebe8vmrKvUlJS+Oc//8l7773H66+/jsvlYvjw4ezdu7c1SvYpp/teORwOTpw4YVFV3ik5OZkXX3yRt99+m7fffptOnToxYsQI1q1bZ3VprcblcjF9+nQuueQS+vTpc9rtAvXvVV2N3VeB/vcqJyeHtm3bEhYWxq9//Wveeecd0tLSGtzWqu+VT97NWLxTenp6vQQ+fPhwevfuzUsvvcQjjzxiYWXiy1JSUkhJSXHPDx8+nO3bt/P000/zr3/9y8LKWs/UqVPZuHEjX3zxhdWleL3G7qtA/3uVkpLChg0bKCoq4r///S9Tpkxh1apVpw0pVgioIyjt27fHbrdz4MCBessPHDhAUlJSg89JSkryaHt/0ZR9dbKQkBAGDhzIDz/80BIl+rTTfa+io6Np06aNRVX5josvvjhgvlfTpk1j0aJFfPrpp3Ts2PGM2wbq36sanuyrkwXa36vQ0FB69OjB4MGDmTt3Lv379+fZZ59tcFurvlcBFVBCQ0MZPHgwK1ascC9zuVysWLHitOfe0tPT620PsHz58tNu7y+asq9O5nQ6ycnJITk5uaXK9FmB+r1qLhs2bPD775VhGEybNo133nmHTz75hK5du571OYH6vWrKvjpZoP+9crlclJeXN7jOsu9Vi16C64UWLlxohIWFGQsWLDC+//5747bbbjNiY2ONgoICwzAM48YbbzQefPBB9/ZffvmlERwcbMybN8/YvHmz8fDDDxshISFGTk6OVR+h1Xi6r2bPnm189NFHxvbt243s7GzjZz/7mREeHm5s2rTJqo/QaoqLi43169cb69evNwDjz3/+s7F+/Xpj9+7dhmEYxoMPPmjceOON7u137NhhREREGDNmzDA2b95szJ8/37Db7cbSpUut+gitxtN99fTTTxvvvvuusW3bNiMnJ8e4++67jaCgIOPjjz+26iO0ijvuuMOIiYkxVq5caezfv9/9KC0tdW+jv1empuyrQP579eCDDxqrVq0ydu7caXz33XfGgw8+aNhsNmPZsmWGYXjP9yrgAophGMbzzz9vdO7c2QgNDTUuvvhi46uvvnKvu+KKK4wpU6bU2/4///mP0atXLyM0NNS48MILjQ8//LCVK7aOJ/tq+vTp7m0TExONcePGGevWrbOg6tZX0wp78qNm/0yZMsW44oorTnnOgAEDjNDQUKNbt27GK6+80up1W8HTffXHP/7R6N69uxEeHm60a9fOGDFihPHJJ59YU3wramgfAfW+J/p7ZWrKvgrkv1e//OUvjS5duhihoaFGfHy8MXLkSHc4MQzv+V7ZDMMwWvYYjYiIiIhnAuoaFBEREfENCigiIiLidRRQRERExOsooIiIiIjXUUARERERr6OAIiIiIl5HAUVERES8jgKKiIiIeB0FFBEREfE6CigiIiLidRRQRERExOsooIiIiIjX+f9kVrR8WR3SvQAAAABJRU5ErkJggg==\n" + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjAAAAGdCAYAAAAMm0nCAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8qNh9FAAAACXBIWXMAAA9hAAAPYQGoP6dpAACb+0lEQVR4nOzdd5ycVb348c/0srOzvZdkN703QhIIIUBICF1QRClSBLk3XEUEvSjyk6JcUVRQ1OtVAQUELIC0QCAkkJCEkEJ632Q32+vMzuz0eX5/PLuzmWyd3dnK9/165TUzz3OeZ85sAvPdc77nezSKoigIIYQQQowg2qHugBBCCCFErCSAEUIIIcSIIwGMEEIIIUYcCWCEEEIIMeJIACOEEEKIEUcCGCGEEEKMOBLACCGEEGLEkQBGCCGEECOOfqg7MFDC4TAVFRUkJiai0WiGujtCCCGE6AVFUWhubiY3NxettutxllEbwFRUVFBQUDDU3RBCCCFEH5SVlZGfn9/l+VEbwCQmJgLqD8Butw9xb4QQQgjRG06nk4KCgsj3eFdGbQDTNm1kt9slgBFCCCFGmJ7SP2JK4n300UeZP38+iYmJZGZmcuWVV3Lw4MGoNkuXLkWj0UT9ueOOO6LalJaWcskll2C1WsnMzOTee+8lGAxGtVm3bh1z587FZDIxfvx4nnnmmVi6KoQQQohRLKYAZv369axatYrNmzezZs0aAoEAy5cvx+12R7W77bbbqKysjPx57LHHIudCoRCXXHIJfr+fjz/+mGeffZZnnnmGBx54INKmpKSESy65hPPOO4+dO3dy11138fWvf5133nmnnx9XCCGEEKOBRlEUpa8X19bWkpmZyfr161myZAmgjsDMnj2bX/3qV51e8/bbb3PppZdSUVFBVlYWAL///e/53ve+R21tLUajke9973u8+eab7NmzJ3LdtddeS1NTE6tXr+5V35xOJ0lJSTgcDplCEkIIIUaI3n5/9ysHxuFwAJCamhp1/Pnnn+e5554jOzubyy67jB/+8IdYrVYANm3axIwZMyLBC8CKFSv4j//4D/bu3cucOXPYtGkTy5Yti7rnihUruOuuu/rT3Q4URSEYDBIKheJ6X9FOp9Oh1+tlKbsQQoi46nMAEw6Hueuuuzj77LOZPn165PhXv/pVxowZQ25uLrt27eJ73/seBw8e5F//+hcAVVVVUcELEHldVVXVbRun04nH48FisXToj8/nw+fzRV47nc5u++/3+6msrKSlpSWGTy36wmq1kpOTg9FoHOquCCGEGCX6HMCsWrWKPXv2sGHDhqjjt99+e+T5jBkzyMnJ4YILLuDo0aOMGzeu7z3twaOPPsqDDz7Yq7bhcJiSkhJ0Oh25ubkYjUYZIRgAiqLg9/upra2lpKSECRMmdFuUSAghhOitPgUwd955J2+88QYffvhht0VmABYsWADAkSNHGDduHNnZ2XzyySdRbaqrqwHIzs6OPLYdO7WN3W7vdPQF4L777uPuu++OvG5bR94Zv99POBymoKAgMrUlBobFYsFgMHDixAn8fj9ms3mouySEEGIUiOnXYUVRuPPOO3nllVdYu3YtRUVFPV6zc+dOAHJycgBYtGgRu3fvpqamJtJmzZo12O12pk6dGmnz/vvvR91nzZo1LFq0qMv3MZlMkZovva39IqMBg0N+zkIIIeItpm+WVatW8dxzz/HCCy+QmJhIVVUVVVVVeDweAI4ePcrDDz/Mtm3bOH78OP/+97+58cYbWbJkCTNnzgRg+fLlTJ06lRtuuIHPPvuMd955h/vvv59Vq1ZhMpkAuOOOOzh27Bjf/e53OXDgAL/97W95+eWX+fa3vx3njy+EEEKIkSimAOZ3v/sdDoeDpUuXkpOTE/nz0ksvAWA0GnnvvfdYvnw5kydP5jvf+Q5XX301r7/+euQeOp2ON954A51Ox6JFi7j++uu58cYbeeihhyJtioqKePPNN1mzZg2zZs3i8ccf549//CMrVqyI08cWQgghxEjWrzoww1l368i9Xi8lJSUUFRVJTsZp1q1bx3nnnUdjYyPJyclxuaf8vIUQQvRWb+vASHLCCLN06dK41cOJ572EEEKIwSQBzCjTVpxPCCGEGM0kgEH90m/xB4fkTywzeDfddBPr16/niSeeiGyU+cwzz6DRaHj77beZN28eJpOJDRs2cNNNN3HllVdGXX/XXXexdOnSLu91/PjxSNtt27ZxxhlnYLVaOeusszps2imEEKL3Tv9/fXmTJ6b//4uO+rWVwGjhCYSY+sDQbBS576EVWI29+2t44oknOHToENOnT48kPe/duxeA//7v/+bnP/85xcXFpKSk9OleGRkZkSDmBz/4AY8//jgZGRnccccd3HLLLWzcuLEPn1AIIcSGI3VMyk4EYG+Fkz0nHVw+O5cxaQlD3LORSwKYESQpKQmj0YjVao0U/Ttw4AAADz30EBdeeGG/7nWqH//4x5x77rmAGhxdcskleL1eScIVQogY1TR72VfhZEJmIiFF4Ui1i2BYISwDMP0iAQxgMejY99DQLNG2GHRxuc8ZZ5wRl/u0aavbA+1FCGtqaigsLIzr+wghxGi3v7KZQY1VDr8HZZshazpMvQJ8zbD777D7H2AwQ9ESSCqAvLmQWjyYPYsrCWAAjUbT62mc4SohIXoYUqvVdphfDQQCvb6fwWCIPG/bJyocDvejh0II8fl0ot49eG92dC08/0U4JWRStAY04UB0mzbX/AWmXK4+H2F7Ao7sb+3PIaPRSCgU6rFdRkYGe/bsiTq2c+fOqMCkt/cSQgjRN7tONlHv8mMxdj7afqzWxbv7qrlgciapCUbeP1DD7IJk6l1+mr0BxmXaGJuWgE7bi+Ai4EX5x61oThvv0YQDkD5RHW059A6nBje8fCMAoVvWoCuYDwffgrTxkDGp07doavHz543HybKbmD82lfEZNrS96dsAkABmhBk7dixbtmzh+PHj2Gy2LkdFzj//fH72s5/xl7/8hUWLFvHcc8+xZ88e5syZ0+W9UlNTB+tjCCHEqBcMhfnocF23bVy+IB5/iDd2VUaOTcm2U9HkobShhb0VTi6dmUOi2UBGoqnTQMblC/LkX17mjuofkRpsoF5J5FzfL0nRNONXDExI0zN34lx8wTCGaXfgV7RkBSu54fB/YVT8ADQ/cw3JigOUMKQUwTd3dDoi89zmEzz5/uHI6/svmcLXzxmaaShZRj3C3HPPPeh0OqZOnUpGRgalpaWdtluxYgU//OEP+e53v8v8+fNpbm7mxhtv7NO9hBBCxK6kzo0/GJ+p9/cPVPPh4VpCp2T+KorC3z8t4/wf/5s7T95NarAagF9qbuDaxdO48KyFVJPKhno7T649wv9+eIzfbGvhD9tdPLwrkfO9j/FxSN1EOTncqAYvAI0lvL9hA6v3VLGvwhnVj6O16nRYXrIFs0HL3DE9r3odKDICM8JMnDiRTZs2RR276aabOm374IMP8uCDD8Z0r7Fjx3bInZk9e7bUKxBCiBiVN3nier+dpU2U1rdw/uRMAP7z+e3sLnewTLsHu1F9r63zH+d7F3yNRLOaLjAt186WknrMBh0mvRazQYc3EEKr1ZCbNI3q0GyOrbmWYm0VZQnTcTU7maIt5YPV/+S5UPTKVr1WQ3aSuhL1+xdPYfm0LLRDmDcjAYwQQggxAAbi174Gt5+fvLWft/dUAWDWa7ktvxKqgHk3Mf+Sr0e1v3pePlfPy+/yfuHwGCav/hUp/jpqvMnM1ByjKDuFWst48hq8UUFYMKxwslF9XZhqxaAb2kkcCWCEEEKIOAqGwgT7WOSl2unlWK2ry/NVTm8keAFYc/e5FLz0iPpizNkxv59Wq6E4I4EDVer00THTZF76z2WYDToUReHjo/V4/CFe3FpKIKRg1GvJTDQxLbfrTRYHiwQwQgghRBztKncwLsMW83Un6t28urOCF7eWAVCUnoBeq2FfhYMkq4GNR+qpdHgBWFicyl9vXYDhzW9B9R5AA2MX96m/j31xJu/tqybZamTRuDTMrfXJNBoNZ49PB2DZ1Kw+3XsgSQAjhBBCxIk/GGbzsfo+BTDH61uiXpfUqQmzh2uiR2TMBi0//sIMDFoN7H1NPbhoFdhz+9TnmfnJzMxP7tO1Q0kCGCGEECJO9lc68QX6tvLIG1Drcs3IS+KMsSmcqG+hxRdkc0lDpM3ls3K5dGaOGiC568HnUE+cf3+/+z7SSAAjhBBCxEE4rLDtRGOfr28LYJZOymBsWgL21pVED10xjfcO1NDiC6HTashINKkXNBxVH+15YLD0q+8jkQQwQgghRBz4Q2Ecnp63bGnxB/n3ZxV8dLiWK2bnRY57W0du2pZAt9FoNCRZDPgCYdJsRsa27WBdsUN9zJgcnw8wwkgAI4QQQgyCQCjMttJGnt9SSk2zjxKg3uUnGFYoTk+gyaNWxbWbO/9qNuq1LJuSRUqCEVy1sO0Z9cTY2FcfjQYSwIwCS5cuZfbs2fzqV78a6q4IIcTnkssXROlh6fQfPzrGs5tORB070aAm7p5abyXRbIhMJ53q3IkZ5Ca3ThWt+wnU7ANzMkz/Yv86P0LJVgJCCCFEP5U1tBDupmL5K9vLo4KXs8en8b83zCPZaqAoPSFyvCDFwtzC5Mhro14b2ZIokvsCcHyD+njFU5AyJi6fYaSRERghhBCin7oLXkrq3BysbgZgbmEyK6fn4PIFmZiVyB9umMfGI/WUN3qwmnSsnJ5DTrKFxRPSaWzxMyEzsfUuGtJtrQHMq6ug7pD6vHDRAH6q4U1GYEYYt9vNjTfeiM1mIycnh8cffzzqvM/n45577iEvL4+EhAQWLFjAunXrIuefeeYZkpOTeeedd5gyZQo2m42LLrqIysr2nVDXrVvHmWeeSUJCAsnJyZx99tmcONH+m8Nrr73G3LlzMZvNFBcX8+CDDxIMBgf8swshxEjj8gZ5b7+6yeKKadn8/Euzos63FY3LS7GQYjVi0KnDLVl2M5Oz7ei0GnRaDeMzbepO1OEQ7HxOvThvHiSkDd6HGWZkBAZAUSDQ0nO7gWCwdrpleVfuvfde1q9fz2uvvUZmZibf//732b59O7NnzwbgzjvvZN++fbz44ovk5ubyyiuvcNFFF7F7924mTJgAQEtLCz//+c/561//ilar5frrr+eee+7h+eefJxgMcuWVV3Lbbbfxt7/9Db/fzyeffIKmtY8fffQRN954I08++STnnHMOR48e5fbbbwfg//2//xffn40QQowADW5/l+d2lDXS4g+RlmDkvpWTMei7HzfQ97S/UO0B9dFog1vXxNrVUUUCGFCDl5/0rYJhv32/AowJPbcDXC4Xf/rTn3juuee44IILAHj22WfJz1c36iotLeXpp5+mtLSU3Fz189xzzz2sXr2ap59+mp/85CcABAIBfv/73zNu3DhADXoeeughAJxOJw6Hg0svvTRyfsqUKZE+PPjgg/z3f/83X/va1wAoLi7m4Ycf5rvf/a4EMEKIz6WTjS3q6MhpgqEwB6vUqaMlEzMwG3SEuplqAnXH526VbVEf8+aBVten/o4WEsCMIEePHsXv97NgwYLIsdTUVCZNmgTA7t27CYVCTJw4Meo6n89HWlr7MKPVao0EJwA5OTnU1NRE7nfTTTexYsUKLrzwQpYtW8Y111xDTk4OAJ999hkbN27kxz/+ceT6UCiE1+ulpaUFq9Ua/w8uhBDDlD8Y5litmwlZHbcO+OR4A25/iASjjvGZvdtaoNsdnsMhWNP6i2LBmX3p7qgiAQyo0zjfrxi6944Tl8uFTqdj27Zt6HTRkbnN1v4fj8HQsUiScspvBU8//TTf/OY3Wb16NS+99BL3338/a9asYeHChbhcLh588EGuuuqqDu9vNpvj9lmEEGIkqGjy4A913DpAURTe3avmvswsSO50hKYzel037Q68CT6n+nzsOTH3dbSRAAbUHJReTuMMpXHjxmEwGNiyZQuFhYUANDY2cujQIc4991zmzJlDKBSipqaGc87p3z/uOXPmMGfOHO677z4WLVrECy+8wMKFC5k7dy4HDx5k/Pjx8fhIQggxolWcUr/lVNtONHKivgW9VsOMvKRe36/bKaT9r6uPxedB0ZJYujkqSQAzgthsNm699Vbuvfde0tLSyMzM5Ac/+AFarTrkOHHiRK677jpuvPFGHn/8cebMmUNtbS3vv/8+M2fO5JJLLunxPUpKSvjDH/7A5ZdfTm5uLgcPHuTw4cPceOONADzwwANceumlFBYW8sUvfhGtVstnn33Gnj17eOSRRwb08wshxHASCivsrXCSZDV0OPfXzerKzSk5diyG3ueqdJvE21SqPs77WkyLP0YrCWBGmJ/97Ge4XC4uu+wyEhMT+c53voPD4Yicf/rpp3nkkUf4zne+Q3l5Oenp6SxcuJBLL720V/e3Wq0cOHCAZ599lvr6enJycli1ahXf+MY3AFixYgVvvPEGDz30ED/96U8xGAxMnjyZr3/96wPyeYUQYrgqbWjB5Qt2CGC2lzZi0GkZl5HAnILkmO5p6G4Eprk11SFxiBadDDMaRekhJXqEcjqdJCUl4XA4sNvtUee8Xi8lJSUUFRVJ3sYgkJ+3EGI0en9/NbtOOshLsTAt105+soX//fAYf/yoBH8ozD0rJhIIql+xFqOOK2fnEVIU3tpVicsX5Mo5eTR7A7y/vyZyz6+dNZbUBGPHN1MUeCQTQn64azckFw7Wxxx03X1/n0oK2QkhhBAxOtnYQkmdu8PxQ9XN+ENh8lMsTMnp+su3jdWo44yxKSSY1GmmLpN4T2xUgxeNFmzZ/er7aCFTSEIIIUQvNLX42V7ayPmTs9h8rIFmb3QF8lBYYWdZEwBXz81H24s8lfGZiYzPTGRchg29ToO1q3yZfa+pj5MvAX0nIzSfQzICI4QQQpyipM7N1uMNUeUlgqEw7+6tZn9lc5fXvb6rgjqXH6Neyxfm5nX7HlajjpRTcmdyky1kJpq7TuJtKlMfi8/r/QcZ5SSAEUIIISASsKzZV8WGw3W8ubsSfzDMW7srqW72Ud7FkmmAepeP36w9AsCZY1NJsXY/SjI5x05yD22iOE6qj6M49yVWEsAIIYQQQLXTF/X6cLULly9IpcPb47U3/nkrtS51T6RZBd3XfdHGWBsGAEfrEuqk/NiuG8U+1wHMKF2ANezIz1kIMdwFQ2F2nWzq07Un6tuTea+YlYte2/1X65fnF3S+0qgrXid4W8tlSAAT8bkMYNpK6be0DNEO1J8zbT/n07cwEEKI4eJYnZv6bnaV7s6ecrW8/+Wzchmb3nVVd40GEow6bKYY1884y9VHczKYEvvUx9Hoc7kKSafTkZycHNnA0Gq1opGqhnGnKAotLS3U1NSQnJzcYX8mIYQYLg5Vd52c2x1/MEyD28+FU7O4+awxrDtU12XbZKsBk74P/x9sS+BNKuhTH0erz2UAA5Cdra6jbwtixMBJTk6O/LyFEGK4CYTCnKhviW1ap1VJnZvd5Q4a3H6K0hO6DWD6FLwAOFoDmGQJYE71uQ1gNBoNOTk5ZGZmEggEhro7o5bBYJCRFyHEsHa8zo0/2HFH6d5oG7k5e3xatyP5CbFOG52qLYCR/Jcon9sApo1Op5MvWCGE+BzzBvoWvDS2+DnWWo33vEmZXbZLMOlYPjWrT+8BtC+hlimkKJ/LJF4hhBCiv/ZWqMm7k7ITKUyzdtnuwqnZZNn7sQ9cJICREZhTSQAjhBBCxCgUVthfqQYwC4pSu2xnNugYk9p1cNMrDSXqY/KY/t1nlJEARgghxOfW8U42ZOyNfZVOWvwhrEZdt5s2js+0odX2Y5Wr1wGuKvV5+vi+32cUkgBGCCHE51Jts48dZY0xX7e/0smavWpQMSXHjq6bAGVOYXJfu6eqOaA+2rLBHGP13lHuc5/EK4QQ4vPpRL2bcIz5u2FF4aanP6Ha6UOn1TA9t+vRl3HpNtJtpv51cvfL6mPBmf27zygkAYwQQoiRzdMEQR+01EHWtF5fdqzOjS7GIqZajYYvzStg/aFapuQkdrshY3eJvb1WvU99nHpF/+81ykgAI4QQYmQ68THs/jt8+uf2Y1f8Vv2yN9m6vTQYClPR5KEgpfdBRpXDy/pDtRj0Gr51wUTKGrrejqY/aS9RWurVR1vXy7Q/rySAEUIIMfJ4nfDCl8HnjD7+2n+qf278NxSf2+mlLl+QUChMLPvMNrX4eWNXBW5/CFCr93Yl2Wog0Rynvd88DeqjpeuVTp9XksQrhBBi5Pn0z+3BS8FCsGVBYk77+be/2+WlpfUtxBC70OwNcN+/duP2h7Cb9bx+5+JuE3fHZ3Y/+tNrigItrQGMNS0+9xxFJIARQggxsrQ0wMdPqs+v/B3c+g7ctQcu/VV7G2clnWXoOloCbDpW3+u3qnR4+NV7h9l10oFGA1fOyWNGfhJajQaNEmLlwR9wzRvTSTz2FgA6rYZJWXHaMdrrAEUd8cEqIzCnkwBGCCHEyNHSAI8VqbkhSQUw40vqcb0RJl0E/7lFfe1zwO8WtY9gtFp3qAanp3f73wVCYR56fV/k9dKJGe0VdZUwE+reZ3LduwBkb/kxAGeMTSGzP1V3T9WW/2JIAH0/VzONQhLACCGEGDnWPtz+/Nzvge60XJPMyXDWN9XntQdg/U8jp8JhhZONnm5v//yWE2w8UseGI3X8dt1RtpSoAdCLty9kZn5ypF1OzYdccugHkdfG5jISdAoLi+I41eOqVh8lgbdTEsAIIYQYGdY/1r7i6Pz7Ye4Nnbe78CE47371+f7XI1NJ/lC4212nK5o8/PTtA6w/VMe2E+0F7mbmJ1Fw2nYAmbUfAxDStG8GrG8u61/V3dM5K9RHe1787jmKSAAjhBBi+At4YMOv1OfzbobFd3fdVqOBs+4Ekx2c5fD0Sijd0u3tHZ4Av157BKc3SEaiiUSzHqtRx23nFHPF7NMCCFcNVZnn8MLMZ3lu9gs0JU5Ujzce7/PH61QkgMnpvt3nlAQwQgghhr/D70LAjZJUQN3S/wGtrvv2BgvMaR2hKdsML3ypy6ZhReH9/ep0zZKJGdx2ThG3nF3EbecU8x9Lx0U3VhR47mrO2XIHZ5f+Foc5n1BirnrOebKvn65zzZXqY6IEMJ2RAEYIIcTwVncEVn8fgKMZF/Li1jJ8wRDeQKj765bc0/7c64BQx+TdsKLw2DsHKWvNjbnl7LHd3jLddRCqdgHg0SeTbLdhz2q9xhHnACYyApMb3/uOEhLACCGEGBb8wTBKa3W5YGuhuJaPfgO/mQfOk4RNSaxLuhIAlzfI9tY8FaWrinTWVPhuSeSlpuFIhyZv7qpk45E6AApTrSyZkNFtH1Na1Pv5jCm8O+GHXDg1C0NqoXoy3gGMjMB0K6YA5tFHH2X+/PkkJiaSmZnJlVdeycGDB6PaeL1eVq1aRVpaGjabjauvvprq6uqoNqWlpVxyySVYrVYyMzO59957CQaDUW3WrVvH3LlzMZlMjB8/nmeeeaZvn1AIIcSwtP5QLW/trqSm2Uuj28+GI7XUNvvYfdLB6r1VUPIR1vfbV/o4ljxIs7n9y3xXuQN/MMzucgf1Lh8t/iBHalw0ewOEw61BjTUVpWgpANpDq6PePxgKR1YZrZyezR3nFveYhJvkVYOUiqylJNpsZNvNkFKknqw73I+fRiecrQGMjMB0KqYAZv369axatYrNmzezZs0aAoEAy5cvx+12R9p8+9vf5vXXX+fvf/8769evp6KigquuuipyPhQKcckll+D3+/n444959tlneeaZZ3jggQcibUpKSrjkkks477zz2LlzJ3fddRdf//rXeeedd+LwkYUQQgwHh6ubOVjVTLM3yLv7qjhU7SKswHv7qzlW64Ytv4+03THmZjxTr4m63uMP4fYFCYYVdpY18YcPj/Huvio8gRBHal2RdiU5FwFgWPcwYxs3oigKobDCe/tr8ARCmA1aitIT0PRiY8ckTxkAbmsBi8alqde0bSBZs7/T4nl9Eg7LCEwPYtoLafXq6Oj1mWeeITMzk23btrFkyRIcDgd/+tOfeOGFFzj//PMBePrpp5kyZQqbN29m4cKFvPvuu+zbt4/33nuPrKwsZs+ezcMPP8z3vvc9fvSjH2E0Gvn9739PUVERjz/+OABTpkxhw4YN/PKXv2TFihVx+uhCCCGGi9OXNxtCLXDgDQBemPkMrvSZXKrp+nduTyCEohDZ3+jt3VXkJJnRaDS8Fz6T21vbfWHfXfzReisL1p5LED0a4HsXTaappefidjnOz5hc/SYA2UVTSc+2qydSx4HeAgE3vHUPXPqLmD57p1rqIRwANJCY3f/7jUL9yoFxOBwApKaqJY63bdtGIBBg2bJlkTaTJ0+msLCQTZs2AbBp0yZmzJhBVlZWpM2KFStwOp3s3bs30ubUe7S1abtHZ3w+H06nM+qPEEKIoXei3s3W4+pUTSAUxuUL9nAFLD72SwBCljRqbJNjfs+wohAOwwtbTuDWJXJs8u2Rc19v+RP36P8OwDVnFHDGmJ7L9Gt8Tq7d/fXI6/SCSe0ndXqYfIn6/NM/wYmuv6t6rbk1gTcho2OxPgH0I4AJh8PcddddnH322UyfPh2AqqoqjEYjycnJUW2zsrKoqqqKtDk1eGk733auuzZOpxOPp/Mqio8++ihJSUmRPwUFBX39aEIIIeLEHwzz6o4KSurUVINPjzfy9u7Kbq8xhFqYVKuW6G8464comh6WTHfD7VNXKh2e8R2+l/8CO8PqsuiV2i1MyU7kzKKegxddyEvWX0/b2Tq1KPr1ih+3Pz/+UZ/7GxHJf5Hpo670OYBZtWoVe/bs4cUXX4xnf/rsvvvuw+FwRP6UlZUNdZeEEOJzrbzJQ4s/iHLa3s++bqrhAsys+ifGkBtSx+Ge/MW49MXhCfCPYxpu8N9HWNEwRlvDFeN7ERiFAtywbjE61ylB14wvgSUlul1itlodGOJT0K5tBCZREni70qcA5s477+SNN97ggw8+ID8/P3I8Ozsbv99PU1NTVPvq6mqys7MjbU5fldT2uqc2drsdi8XSaZ9MJhN2uz3qjxBCiKGhKAprD9TEfmGghXnlz6nPF38busl7iYXbF+SMMSnYklJpsqgj9Omekh6uAn3pBnSKOuVVkryIPy/bCVf/sfPGbauRGo/Dgbfg+Wug/mjfOiwjMD2K6V+GoijceeedvPLKK6xdu5aioughtHnz5mEwGHj//fcjxw4ePEhpaSmLFi0CYNGiRezevZuamvZ/2GvWrMFutzN16tRIm1Pv0dam7R5CCCGGt+2ljdQ1+2K+zrrrLyQEGnCY82DWtXHrT5bdzLO3nMlVc/NotI4FIK2lBBQFa9Un6EOdpycYyjdHnh9OX8aC4m6mnFLU+1K+HV78Chx+B3Y817cOywhMj2JahbRq1SpeeOEFXnvtNRITEyM5K0lJSVgsFpKSkrj11lu5++67SU1NxW6381//9V8sWrSIhQsXArB8+XKmTp3KDTfcwGOPPUZVVRX3338/q1atwmRStwu/4447+M1vfsN3v/tdbrnlFtauXcvLL7/Mm2++GeePL4QQIt7qXD4+OlwX83WGoBvbp08CsLXgZpbpDIC/3/2pafby7MfHuWVxEXqtlgbLWMbxIeeV/Jzw8V+hVYKcnXMtB1N/0OFafelGALwFi6kuvorluUldv1HbCEzwlGCo7lDfOi0jMD2KaQTmd7/7HQ6Hg6VLl5KTkxP589JLL0Xa/PKXv+TSSy/l6quvZsmSJWRnZ/Ovf/0rcl6n0/HGG2+g0+lYtGgR119/PTfeeCMPPfRQpE1RURFvvvkma9asYdasWTz++OP88Y9/lCXUQggxApTUuemqOG535p74I1pPPY3mQvZnXRq3/uw66aDJE+BAlbo6dVd2e20ybev00JzKlzpcl+w5gb5yGwDNy37OBVN7CCasnYzONPQ8TdUpqQHTo5hGYLos13wKs9nMU089xVNPPdVlmzFjxvDWW291e5+lS5eyY8eOWLonhBBiiNW5fO1VcHtJ621kYunLzCpTp1s+HPtNFE1MX09d8vhDHKhqBuD6BWPYUtKA05yH5+zvYtn4WKRdSGNAo7TvraQPeVl56IdowkFOpi4if8yUnt9MowGNDk65DzX71C0GkvK7vq4zsg9Sj2QvJCGEEHERDIU5UuPqueEpDEE3+X+/lAX7f4xWCRFMncCx1HP63RdFUVAUhTX7qwmFFQpTrcwpTI6c9531HV6e/gdcVjWhV6/4ufDj68FVC8C8iufIdu0nbLJTdtZPev/Gt38AF/8cHmiE/DMBBUo+jK3z/hbwNqnPZQSmSxLACCGEiIs1+6ppdPc+Z0UTDrKw7I8YnMcBKEk/l8bL/9LvlUehsMKrOyuY/fCaSP2Zry4ojN4qQKOlPGkO71ywmpb0GQCkOfbAP24myX2cBWXqSiPPWfeSXxxDIb2cWXDmbaDVQkZrsbtYN3lsmz4yWMHcTc7N51x8xuiEEEJ8ru2vdBJWoIe9EAGwuUvhl5dwq6O9XtfWSfeyLfcrXJmSB5T2uR8ub5BFj66NOrZ4fDpF6QldXuNLnoC1brf64vhHfLm1EF2TOZ/w7JvITey8fEePkloLqjbF+Hnapo8Sc9RpKdEpGYERQgjRZ4qi4A2E8ARCPTduddbuH8IpwUvAlsfR/Cvj0p8KR/Ry6OsWFDJvTEoXrVV1M26nJLljmY4Piu9BazCj601U1pnk1gDGEWNh1WbZhbo3JIARQgjRZ7XNPlr8vQxeQkGSKjeQ2aQu0Gi0FvH8rL9y8svvENDb4tKf8Zk2nv/6mayYlsV73z6XdJupx2u8aVN5ddqTvHp+e/2xNyY9Sk3WEpIs/diHqG0KqXwHhHsf4EWNwIguyRSSEEKIPtt2opEFxWndtknw1bJw+5Owej1T22qkTL6Uf+Y9TLM3SNicAsReN6YzGmBCViKTs+1kJEYHL1qNhgVFqWwpaej0Wo85E774Zz6uMaBPm8PiDHt03kyscmaDKQl8DqjYCfnzendds9SA6Q0JYIQQQvRJg9vPoWpXjwHMsiOPUNj4ceR1UGtCv2hVf1JdOqUoCi9uPUmlw0uiqePXm1GvZdG4NPJSLNiMerStwUlUjDL9as6KV4e0Oph2hbqqKJYdpZ1Shbc3JIARQgjRJx8driXcQ30wfclaitqCly8/xxbtHI7VNvOVMVOg9Fhc+7OnwsnJRg//2l7OdQsKO22j0WgYk6Ym9F57ZgGKAtlJZsqbPJj0A5BVcfmvIRSAxhO9v0ZGYHpFAhghhBAxC4cVTtS3dNtmavXr2DaqVdabE8aQOPlSwscaCOljyAfpJacnwPpDag2XG88ag1kfvdN0us3Y4ZosuznyfFJWIsnWfuS7dMVdD7+YDOEgfL8CDL1Y0dS2jYCMwHRLkniFEELELKwohDqpuGv01mN0lTO2YSMrjqjBi9uQygeLnhmwJcHN3gAvbS0jFFbItpu5+eyxUee1Gg2Tsu3d3qMg1UqytWOQ02/WVDAlghKG2gM9tw+HwaXuMygjMN2TERghhBAx8QZCBEPhDsfTaj5m4YZbAJjTesw/6TKeTbmbdHPGgPXnLx+f4LOTDgDOm5wRyW1pMz7Thq2TnJhT9XmpdE80Gsiaplbjrd4LuXO6b++uVUdr0IAta2D6NErICIwQQoiYHKpuJnj66IuiMHP7D6PbpV1Ay8VP4Y/TEunOHKt18f6BGgDyki3kp1ijzuelWDhnYvqAvX+vZE5VH2v299y2uTWB15YZW+Lv55CMwAghhOi1GqeXHaVNFKa2Bwq6mj2kH/4Ya0s5AEcW/4KSyjr2ZH+Bm3uT89EPf954HIDMRBOXzuw45fKFOXkYdEP8u3pKkfrY1EMir6LAc19Un+fOHdg+jQISwAghhOiVBrefbScaaThlv6N8xzbsf1lFUusOzDXZS6kvvpI9Sv2A96epxc/be9SE15vOHosvED2tpdNphj54AUgZoz72tBKp6QS01IHWANf8ZeD7NcJJACOEEKJHe8odnVbcnVPxIprW4MWRPJU9cx+i612H4svpDXLepEw8gRCFqVYOV7fvhG026HrMexk0ya0BTEOJmqSr7SKoqtqjPmZOBv0AJBSPMsMgNBVCCDGYqp1e3t1bhdMbwBcMEegkIfd0x+vVXZ3Tqz7EEmgEn4vL9t/L+IZ1ABy49BU2nPdPfJbMgex6xL+2n+SVHeW8f6CGSzqZOspO6nkLgUGTPgFMdvA3Q/m2rttVtwYwWTMGp18j3DAJT4UQQgwGly/IqzvKsZr01Dh9ZNhMPLflBF86I5/MRHOH9v5gGAWFRrefsf7DTN94O/MBPoHk1ja+KVfTkjkHmryD8hn2Vjh5e09V5HWW3Yz3lM0ktRoN500anECqV3QGGHM2HHobKnZAwfzO21W17oidLQFMb8gIjBBCfE64fEGe23yiw1SQPxjG6QlS6fAQDitUOjwEQ2E8/hAt/iDljR4yGrYx/Y3LO9xza96NuC/+TUz9CIUVvv3STl7bWUFZQwtKD9V8T6UoCvsqnQDMzEti6w8uIDUherol1WocmJou/ZHYuiTa0/k+TMApAcz0ge/PKCAjMEII8Tnx8ZE6PN3sHP3e/hpm5CVR5fBytNZFMKTwtbPGQDjImQd+Gmm3fuxdLEj38pxyMc3mHMZp9YCv2/c+XufG4QlQ2+zlp6sPRIKo3eUOqpu9rJjac80TRVFYvacqUgF4el4S+k7ySbQDVdOlPyyp6mNLFwFMKABNrZtDZUwenD6NcBLACCHE54AvGMLfi1yXtqkYfzAcKQiXtOtPpDYfBGDT0hfYHpjArLPH0ty6hLk7iqLwWVkTHx6q47frjkadm5qTyL7KZjYeqWfjkXoyEk2MTbN2cSf49EQjh2rURN2Hr5hGncvfZdthx9oawHQ1AtNcCSigM4J1iOvWjBAyhSSEEJ8Dx+u637eoM5mu/Vjevou01v2MShb9hKa03tcnqW32Mf/H7/PBwVpCp0wTWQw6/n7HIq6am8/cwuSo9nc8tz2yNPp0z358HIA5hclcPGOEldnvaQTGodbQwZ7b9SolEUVGYIQQYhTzBkIoCrj9wZium1r9OsuPPIwGNfBotI2ndvw10NK7+9S5fLyxqyJSsXfJxHT+78YzeGbjcRJMeuaPTeWzsibOLEoly26OSsota/B02GfpX9vLOVDVjFYD88emxPRZhgVLa5+7GoFxtgUw+YPTn1FAwjwhhBjF1h2sjVqh0xuakJ/FJ36DBoVQ6gTqz/oh7837HWh1PV8M7DrZxIuflOH0qsHORdOzuXRmLia9jpn5yVFtTXod96yYxPdWTOKquXkkmvV4AiHe219NIBRmR2kj//fRMQ5UNQOQm2zBbh6BJfbbknidnY8uRQKYpLzB6c8oICMwQggxSm05Vs/+SicLilJ7fY3JeZwVr54PgEefhO/m9Ti8Cp4jdT1e6w2E+NOGksgqofwUC3+4YR7v7qvu8VqbWc/ErEQmZibyP6sPsLfCyYQfvB3V5pIZORSkDuzWBAMmtVh9dFWBzwWm0/aHOnUKSfSKjMAIIcQotPukg2ZvbNNGAHm7fxt5vjX/5l5vKLj+YC3/3H4yErzkJpm5fFYuucmxBRxXzsljxdQsjPror6dzJqTz3YsmYdL3bhRo2LGkgDVNfV5/pOP5yBSSjMD0lozACCHEKOLwBAiEwgTCPa84Op3J30j6sVcBeG3yzzmevpSZPVyz+Vg9T7x/uMPxC6dm9Xkfosk5dqbk2Ll0Vg7/2l5OaUMLM/OTenU/k34Y/16eMwuOroWDb0Pu7OhzjpPqY5LkwPTWMP6bFkIIEYtGt581+6p7tTVAZ3LqN6NVgjQnTeRY2rk9tnd6A2w9Hp2Uuqg4jRe+vqDfheS0Wg3jMxO55owCxmXYer6g1dQce7/ed0DN+JL6eGSN+hgKwP7X4aNfQOVO9ZhMIfWajMAIIcQI4wuGqGjykmDUkZJgjAQsb+6upLbZB6TFfE9NwM05n30PgLqsxT229/hDvLazgrbFQr+/YS6hEGg1oBuiQnJGvZaFxbF/9kGTf6b6WL4NPn0awkF4657oNrIKqdckgBFCiBHicHUzY9MTaGoJ8OqOcnRaDTcsHEOTJ8CHh2ppcPe9sJt9398izyvzV3ZZWDcQChNWFB5+cx8Nbj8JRh0v37EIXzBM5SDthdSVzETT8KzC2ya1WK0H42mAN+7qvI219wnXn3cSwAghxDAVCiuEw2HWHqwlLcHI5mP13LakOO7vk+Q5SdquRwEoOfNBnKkzoTUZ91S/X3+UtQdqoo6tnJ5DTpIlslv1UCpI7bqK77Cg1cLyh+G1VV230QzjAGyYkQBGCCGGIUVRKKlzk24zsq+iYzART7MrX0Yb9NJkG0fN+C/BaQM5jS1+/rShJLIHUeS6gmTyUobPsubi9ISh7kLPZn0VSj6EXS+1H7vop2oOzPhlQ9atkUgCGCGEGIb2VjgxGwZ+nYWm6QSzK18GYMeEb2LRm8EfiJz/8FAtr+wojwpezh6fxkOXT+PVnRUD3r/eshh0ZCSahrobPdNq4ao/qEFMc2tRu5SxsPCOIe3WSCQBjBBCDDP7KpzsLGtiYfEA50MoCpY196IlhCt7AWUZ5zKx9VQwFKas0cMv3zsUaX7fyskUplo5UNVMms0U2exxOBiXYUMzjPrTI1tWewBjyxzavoxQEsAIIcQwUunw8N7+alIT+rcMuTsNbj8Hq5txlOzkzpK1APxX6RI+OL6PnCQz4bBCdXN0Fm+23czsguR+JQoPpDHd7GI9LJ1xM7z+LciYDNk9VdsRnZEARgghholD1c0EQuEOGxnGy/bSRv65rRyHJ4AJPz8z/QdoYGNoGh+E5wBQ6WhfSaTTahiXkcDSSZnkp1iG9QjHcO5bp+Z+TR2FyZkFOvkq7gv5qQkhxBCrbfbh8ARw+YIDUkn2eJ2bq373ceS1jhBPGJ4iUeMBYLXlEs4rzkCj0bD2QA1zC1OYW5iMzaTnprPH0uD2s6EXeyGJGGg0MGnlUPdiRJMARgghhoDbFySkKLy2s4K6Zh9LJmYMyArag1XNUaX+p+fZeUD7NGfWbgXAN+0a0pKu4muzC9h0rJ5bFxfh8AQ4WNU8YkY1jH3cskCMbPK3LoQQg+RkYwst/iAn6t0crG7GHwxT19xFxbg4cHmDrN5bFXn9pXn5XDzewry6VwGoW/RDvJf8FkXTvkGiTqsZVsm5vVE40vJfRFzICIwQQgygtpwWs0HHv7aXEworTM5OJCvJPKDv6w+G+dPGEgDMBi3fvnAiBYkaFr1zBTolRFNCMU2zv0HGgPZicNjNvdsxW4wuMgIjhBADxBsI8eLWMqocg1tiv6TOzbV/2MzucgdaDfz++nmcazvJ2eu+Soq3FIDtk74tVV/FiCYjMEIIMQD8wTBv7qoc0CmizuytcETlvFwyM4dZmXqSn/wCGiUEwGuTf44z81wKB7VnQsSXBDBCCNFPiqLg8gXx+EM0+4IEQmE0aChtaOn54jg6WNXM/v17WaytolTJ5Mlx27HVlpL0f3sjwcuOnC9zLO1c0ge1ZwNnWG/eKAaUBDBCCNFH/mAYo17LR4frqG32MTErEU8ghF6nIcE4uP973Xikjnf3VbHW+CjF2tbE3ZPt5wOWDA5M/AbrbFcMar8GUn6KhSSL5L98XkkAI4QQp/AHwzi9AYIhhcxEE2FFQa/TEm4tLqfVanB4AryztwqrUcelM3MHrPBcb/iCIb7x1085VO3iC9oN7cELENbo0CohfFO/xP55D3LSrYFq15D1Nd5ykobPRpJi8EkAI4QQrZpa1IJth6tdLChORafV8O/PKvjy/AKe/fg4K6ZlA1DT7KW80cP4TNuQ9rekzs3CR9WtABZo9vOY8Y+Rcwcv+QdlCTPZfbKRm84uJlzvBjxD1NOBkWYbuO0WxPAnAYwQ4nPNGwixv9LJuEwbx+rcHc47PQFCIQV/MKy+9gY6tNGEA9CaYxJPiqLw+LsH8QbCWAw6jta6uPiJCswGHXsrnABY8fKI4c9crNuKgQCH087HdsNzuJ1+cPlBMzoXm+p12pG3/5GIKwlghBAjSjisoNVq2F7aSGaiifyUvn2J1bt8bD3ewPyxqaw7WMvYtISY72F1nYAX72XpgTfwGtNomnEzXn0i/rTJKIVnx3w/fUsNGmsyAO/sreLB1/cSCHU9PaUnxPf0f+Mq3QYAKhJn8PaEB/nSMA5aTAYtM/KS+n0fyX0REsAIIYa9epePDw7WMi3XzrFaNwuLU9l8rB672cBZ49LISDThD4ZJs5kIhsLoTyktHw4r+ENhvIEQpQ0tePwhFhSnUevycaTGxfyxqTH3x9BSzZd2ryLfuSNyzOyvJ3vbzyOvG6d9DZJW9brWSkLNNia+/RW0YT8fT/gtHx1O7tAm2WLg+oWFGHyN4Chj1bH/RK+oI0InMi/g38X/j5BuYAvk9ZXNpH7dzMxLivz9TMpO5NPjjUPZLTGCSQAjhBj2mjwByhpaKEpPIKwoUUmzba+f31LK5bNyaWjxq8m3YUi1GXlrdyUXTsmiyRNgR2kTvqAawPTH+M3fJ8W5AwUNmry5OD0Bmiz5JGu92MvUnJSUvc+yMr2Styc+0uk9NEEvhE0ke04wvn49kzf+OnLu64f/k63ab7MhPJ13v3cxf95wHKMmyFTnBi6u+yPakg/RBNvzWRwzb2Vd9jcJuv39+lzdOWNMKikJBsamWwnGmLRs0GlZUKQGiqcGl4vHp2Md5NVaYvSQfzlCiGHrRL2bMb2c2gmFFUKK+sVaWt9CSoKRTbvqqHbGt5CcueEAKeXrCKPlw/NfYemSpWw7UEOD28/ErET8rgYmv38TCbU7mVz3Li5jJsx9DGoPkdxSiltnJ/vkRuZsuQuAm7t4n/81/lJ98gT8EPDqbJhDHVcQBcYupeGcB+FwfVw/p1ajwWrUkWg2YNBpSTC1f11cODWLYEjNCUqzGdFpNOhPq8eSZjPi9ql5QRdNzybNZurwHhqNhtkFydS5Yvs7GimbTIqBJQGMEGJYanD7qXf7ex3ADJasPX8A4Eja+TQnTexwPmRK4uBlr1Kw+ykytv6MMyqeI/CXD8FZyo093PvYxFtZuesc/mT4GYt1e6POnRq8uOd+gzWeyQQMiSxbfhl445NAbNKroyNpNiNfP6cInVaDTquJCl4AzAYdil7LxTNymJSdCMBMazIub5Bkq5HUBCNTcu1YDOomkcnWrlcL6bQasuxmdFot0LvPUZAqybtCAhghxCCqaPKQm9xeu8PjD2E2aNFoNPiDYTz+EC5/kI8O1ZJkMQz4hoe9pXHXsvj4ryk4sBVT/X4APs27nsRurmmY9y2ajm5lQsM6DM7SDufd6TNR5tzAmnIDJ5IXsGBMMl95Ws2puS/hYe5amMgiWxWZ259AX/Ep1QmTeXPSo1x33kxcio2SLaXotJrWVUbxCWCuOaOgQw5RVzQaTSR4aXPW+L7X9/3ivHz+9knHn1PH94VF/ZwCFKODBDBCiLhoavFHftMOhRU8gVAkcbPe5UOv1fLS1jJuObuIRLOez042YTXq2XikjuwkM1Ny7ByrdWHUa6l0eIfNKhObrxrbc1cz39H+5epNyKM6cVq3AQzA25Meoe7kX5jNASxnfYPflhVi8VYz1ebGMP5cClOtHG85gS8YigQvADMKkvGY02kpnIJr6kU8vUHdVRqNBiwp0NJxKXd/TMpOJD9FDSx7E7wMBKNeG+lDdwpTrWQkdpyOEp8/EsAIIWIWDit8dKSOBUWpKAqsP1TD/spmlk3JIjfZzO5yB0a9liM1Ls6fnMm7e6u5am4eAIFwGG8wxIGqZuYWpuDwBNBpNUzJGfzPkeY+gvHgTpI9KcDYqHOakB/rxv/htk9/AYBPl4Dj7B/gsI6hIWkanOw+iFAUhZDWxObC28iZk8fY9ASCFYdpsoyhISORrNZ2/mCYl7aWRa578PJpNJyejNvPnA+NBtISjHj9Ic4Ym8L2E00ApCYYMRnUqaDh4Ozx6Tg83f9cizOGtnigGD4kgBFCxKSt8Nv2E43Mzk+mrLEFV2uy5ifHG1g+NQtf6x5B9S71i3joCu1HM/vqsbvLQSnGUrGJa7Zfh25biK+hw2e5F92YLwNmcp2fMffV+zG1qGX5FY2WtcXfY8L0r9HsDRAKhICGyH0rmjz8qLVmS7bdjMWoY9mULHyBEC3+EGGl65/A5pJ6GlsCWI06blg0hjPGpvDu3uq4feacJDNJFgPJVmMkn+iscenUNvtITRhelWzNBh2K0nW8ptVoKM4YXjlRYuhIACOE6LXjdW7q3T4qHd6h7krvKQr60g3kvfMdvtx0TD2208KYU5Yhawlh2fg/zNr4P0wwpmP110XONZgL0d25lQObSplwym09gRAn6t3sOtnEo28fiBScq3KqP5v/++hYpK03GOLXX5nToWv/3HaSHaVNANy1bAJn9KEmTXesJh3nF2R2WLWj02rIHib5RaezGHVkJJoiwe+pMhNNsvu0iJAARgjRK4qi0NTD8P5wNLZxI4kb7oo+2Bq8OE057L78LaxvrmJOy8cAkeDlZMI0bOffwz8qsyg64aC2WV3qG1YUqp1entt8ghZ/iJ+/ewgAs0HL+ZOzeGt3ZYc+vLGrskPBvLKGFh59+wAA88emMG9MfIMXrUbD3MIUzK0rgUaSWfnJrD1QE3VMg0aCFxFFAhghRI8OVjVT3tRCasIIS55UFM4oezrycse4/6TJmM154S2U+Sz8Z8kidv91P3Zu5iZdLkZNgE/Dk6hSUjngLYC/a7CZXLg2fArA9tJGDlQ1R73F1Bw7douey2bmMjM/ma8tGoMvGKbFH2L3ySZ2ljVR2tBCldNLlcNLYaqV2mYfj69RA5+xadYBWVUzJScRk37kBS8A0/OS2F/pjDo2Ndc+RL0Rw5UEMEKIbm070ciHh2opSk8YMQGMNugh54PvMeXQPwBQdEbKbtjMrgodTR4//647l9cOltNWUNZJAk+GruKBy6ay0qjjL5tOQOtmiS5fMHLfU4OXjEQTF0zO5IHLprLlWEMk8Vaj0WAz68mym9lf6WTemBRuWDSGb764U93uIMFI/SlJunMLU+JWmM1s0DE5O5EMm2nIVhPFy6m5LsUZCZzdjyXaYnSK+V/4hx9+yGWXXUZubi4ajYZXX3016vxNN92ERqOJ+nPRRRdFtWloaOC6667DbreTnJzMrbfeissVXWFy165dnHPOOZjNZgoKCnjsscdi/3RCiD4LhRVK61v48FDtUHclJpqgl8nrvkFya/AC4FnyQ0K2bLaXNvLr94/wyg41eMlNMvP76+fy9cVF7PjhhcwuSCbBpOeCyZl8+8IJ3LdyMudMSOfl2xdx2Ux1pU5esoXrFxbylfkFnN2LUvgajYaitASuXzAGICp4+cstZ8atKFuSxcB1Cws5b3LmiA9eAGYXpABqHs/ls3KHuDdiOIp5BMbtdjNr1ixuueUWrrrqqk7bXHTRRTz9dPuwrckU/VvbddddR2VlJWvWrCEQCHDzzTdz++2388ILLwDgdDpZvnw5y5Yt4/e//z27d+/mlltuITk5mdtvvz3WLgshutHo9pPSyWoUtU7LyJqC0AQ8THz7WhJqdwJwMus8DiYuInvcjXzjT1sob/SgABOzbNx09lgaXH7mj01lf2X0tJBGo2H+2FQmZCbS4g8xKTuR4gwb7357CS5fEG8gxJZjDR070AWTQccDl00lEAqz/lAtKVYD31w2gam5dradiM9mhudNzsRuHh61c+JB15rvkmEzydYBolMxBzArV65k5cqV3bYxmUxkZ2d3em7//v2sXr2arVu3csYZZwDw61//mosvvpif//zn5Obm8vzzz+P3+/nzn/+M0Whk2rRp7Ny5k1/84hcSwAgRR4FQmL9tLeX8yZkUplqxGvWUNbQMdbe6lVXxPgkVRzEYrgTU39K1TSdYdOL3jNn2L3TeRoIGGycvepot4clUNHl54tU9nGxUE3fnFibzz/84i4PVzby3L7blytp+fpFmJJr4wpw8JmcnktJNef2+GC6F/+JNghfRlQHJgVm3bh2ZmZmkpKRw/vnn88gjj5CWpiapbdq0ieTk5EjwArBs2TK0Wi1btmzhC1/4Aps2bWLJkiUYje3/ga9YsYKf/vSnNDY2kpKS0uE9fT4fPl/7hmBOp7NDGyGEyuMPsaWknga3H18gTI3TR4s/xNzCFNYeqGFqrj3yG/BwYi9by4QtdwJwG7/kaO7l5Ox1YC1bz8LWNgFzKgcXP4EudyGcbOIPHx7DE1Dr1KyYlsXcwuRR9aVoMmjJMoyM3CQh4inuAcxFF13EVVddRVFREUePHuX73/8+K1euZNOmTeh0OqqqqsjMzIzuhF5PamoqVVVq0aiqqiqKioqi2mRlZUXOdRbAPProozz44IPx/jhCjEpv76nkRP3wHmk5XUrLccZt/s+oY+Mq/h312l20goPn/gZvUEsKsPFIXSR4uemssRSlJ+ALxmffoOFAp9X0mIMjxGgV93/51157beT5jBkzmDlzJuPGjWPdunVccMEF8X67iPvuu4+777478trpdFJQUDBg7yfEYDlY1YzVqMOk15Ju618hr0AoTKPbj/uUlTUjgdbn4PL930Eb8uJImkJ4+U/Q//sOEn3qFFAodQK/nvxXLptTAJ4An5TU8qv3D0eu/6/zx3PV3PwRl5CcZTdjNuiYNyYFu8VAtdOLVqPBrNexoCiVCVk97cYkxOg14KF7cXEx6enpHDlyhAsuuIDs7GxqaqILFAWDQRoaGiJ5M9nZ2VRXR89Nt73uKrfGZDJ1SBYWYqTaWdbE9Fw7dS4/b+2u5IyxKfiDYdz+UJ9WZJyodxMKK6w/VMuSiRkD0OMBFAqQ/datWLylKGjYccb/MLVgIX9b9BZ2s4GFxamk20woG48DUFLnjgpezixK5eazxuLwjqygrTg9gcTWpNzT/86SrNp+7fwsxGgw4GvtTp48SX19PTk56hLERYsW0dTUxLZt2yJt1q5dSzgcZsGCBZE2H374IYFAe9XPNWvWMGnSpE6nj4QYTbadaOSDAzWRGiWnOlrj4khNc8cTXdhT7gDUpbuNLQGa4ryL8aD44CdYKjYR1JooOfdJmu0Tu2za4g/ynZc/A8Bq1HHR9CwWj08fcTkvGg3MLEge6m4IMazFHMC4XC527tzJzp07ASgpKWHnzp2Ulpbicrm499572bx5M8ePH+f999/niiuuYPz48axYsQKAKVOmcNFFF3HbbbfxySefsHHjRu68806uvfZacnPV3yy/+tWvYjQaufXWW9m7dy8vvfQSTzzxRNQUkRCjkdsXZOORum7bvP5ZJav3VOFoDUYURSEYClPj9OIPhgmFFRRFYevxBj463P29RoQ9/wRgzbjv01h8WZfNqp1eLvv1xsjrBy6dynmTMrtsP1xp0HDpzBxsJsltEaI7Mf8X8umnn3LeeedFXrcFFV/72tf43e9+x65du3j22WdpamoiNzeX5cuX8/DDD0dN7zz//PPceeedXHDBBWi1Wq6++mqefPLJyPmkpCTeffddVq1axbx580hPT+eBBx6QJdRiVAuFFQIhNQDpyf5KJ8UZCSRZDeytcJKdZOb5LaXcfPZYXt9ViV6rocrhxWQY2QXNtH4nNJ0AoCTlbDoLRw5WNbNq7Q4OnlIl95ErpzOrIJldJ5sGp6OtkiwG/KH+JQmnJhiH7UaLQgwnMQcwS5cuRelma/h33nmnx3ukpqZGitZ1ZebMmXz00Uexdk+IEeuzk00Upyf03PA0/lA46nW9y0c3/4mOGHM2f5Ps8ncBCCQW4DMkdWjzyo7yqEJwU3IS+eK8/D79HEGddjL0pYqtBhZPSCfLbiY90cixWnef3h/UHZeFED2TMUohhoEjNS62ljT0+Yt3tDEGXZHgBaBu8Y/gtNJOnx5vjApeLpySxX9fPBmHJ4DX37tRELvFgM2sZ+mkDOwWAykJRiZkJuLoxa7bGg2ktVYwnpGXRLpNDTysRj3Tcu0cqnZ1d3nn9wTZcVmIXhrZ48tCjBLbSxsj9UoETKl9q/3Fl56hpTh6P7X/++gYaw+oqxkNOg1vfnNxzLsVF6UncOOisZj0OuYUpkQK9+m0Gq6cnUfqadsrnF7Yb3J2ImcWpQJEgpc2Go2GwlQrU3JiW+YsOy4L0XsyAiPEEFIUJbKLsWg3t+JvAFTOupOcaV+AWnU04/8+Osb7+6sJhNQ5siUT07n/kilkJvY+ZySjdYrGpO96nyejXsu4DBtOT4BJ2YnYzHpWTs/GpNdiNxsoTLViM+m73TTRYtSxdFJmr1ZAmQxaClIt3fZJCBFNAhgxKtS5fLi8QcaOsCmYQ9WuXiXtfp7oAs0ke08CUDPtFnKAA1VOXt1ZHlU9OMli4MIpWTHtT6TVarhgSlav2hp0WtJsJi6eoZaA6EvROLNBx5ljU3uckpqSY48pCBNCyBSSGCXqXX5e2VGO0zs0dU5cfahs2+IP8sHBmp4b9lN4OARIShhNSx26cM+jTXn71Z3svZYsQuZUnt5Ywp3P74gEL0a9lre+uZhvnFscc32X6bn2Qd+xOSXBGBn16cokqagrRMxkBEaMeDVOb+R5aX0L0/M6rlYZKOGwQr3bz98+KeXWxUW4/UEyE800ewMY9dpupwRK6tx4epls2hfBcJgP9tTw3KYTjMu0kWQxxJQkHI7jUqYZJ54l6f0n+CbgzP4rTLm4y7bppW8DUJN9LvsrnVgMOhTUHJSvLRrL7IKkDjknvTUrP7lP1/VXQjc1XfJTLOQmWwaxN0KMDhLAiBFte2kjZQ0tTM5Wkx/rBymfZP2hWuaPSWHDkToK06yEwgrv7Ve3u7hidh57yp24fUGWTVWnK+pdPnRaDRqNhiSLOgIw0EudNx9rYNdJtRJvhUMN8qxGHU+uPUJ+soXpeUlMzLJ1GMVQFIXXdpTzwcFaHvvSzMjxvRUObGY9oXCYT0oaIiM7obBaSK87Zx55IvLc/soNeFLWAh23NMg68ToJDnUbgFfsN/DUv/dhNen46dUzqGjysmRiBhVNnph/FhoN2M36YbnCR7YEEKJvJIARI5aiKGw+Vk+KtX21SFPLwAcwR2qa2X6ikVn5STS4/RSmWQE4VuumqHWEIxAKs7vcwZzCZMIKvLCllLHpVgpSrRyvc3PV3PwB7WNNs5fPypoAGJtm5Xjr9EtL64jPySYPJ5s8rN4LOUlmDDoNi8al4fQGWH+olg8OqpseXvd/WzDoNJgNOpq9Qf7vo5Ko93luS2nkeZLFQLLVwC/ePcgXz8hnTFrraE84hIIGDe0Rm/Ht78DsZzv0O//wcwB4tVZ+uUUtTHdOYTrT85KoaPJ2aN9bSRbDsNxOwG42YNTLTL4QfSEBjBixmloC+ALRv/nXueIfwLRNE2UkmgiEwry3v/d5K8GwwvE6d9R0TOMA7kfUFjit3V+DAoxJs7L2O0vZebIJR0uAJ9ceJhxWMOi0bDvRiAJUOrz8/N1D6LUagp3kywRCCoFQzzk+Dk8gkqz67Mdq9dxzxqfzNcsGNCgoJjv/N+Nv3Lb9CnSVO0iYVAGkRq43VWwhuX4HAMs8P4kc/+31cznZGPuoy6mGY/ACSPAiRD9IACNGrHq3r8MxpydAszcQ2cU3Hk40tLC3wsGlM3NxeYMx560MVgrt3goH83/8fuR1aoKRK2bnRqZN7BYDSyZkcPnsXD4+UkdxRgJ7yp3sr3KiKESCF51W3YunMNXK6r1VBIJh8lOszC1MZkJWIkkWPcfq3FQ2eXlxaxnZdjMmgxZvIEQ4DEdq2wu4fXSkjl0nnmGpDtYkX0ODLo1Q6gT0dfuxu44SVqYB4AuGUFZ/H4AXgudRrmRy5thUVp0/rt9Li8e0jpANBya9NjIKJoToHwlgxIjV4O58JONEnBN5Gwd4Wmp/pZOfrj6ASa8lEAzj9ocieTK9pW7e2F6VdkyalfMnZWI2dP3ln5NkISfJwvmTM3lvfzVVTi+/vW4uh6tdTM21o9NquHhGNqv3qLk9RekJFKRaCYXDpNtMNLoDrP7WOdS6fByqdlHW0ILdYmD51Cz2VDhYs6+a1duPski7D4CflU7k8IljXJ6bx1T2o6new8onUkg067F5q7ndvJ+QouHx4DXcsHAMi8al9TsQtZsNLB1GGzpePS+ff25Tl4gPVUKxEKOFBDBixOoq3+VAVXPcAhhFUahr7jjS01+VDg8/ffsAbn+Q7aVNkeNPrTsKwJ3njY/pfvsqnZHpm28sKe42cDmdTqvh+xdPwajXsrA4jcOnlMDvaerFatSj1XT8e9BqNHxxXj4LAlsxHQ5wUknnsJIHwAvVBTxigHPLfseD+iP8r/cybtavBuBkwjR+cd0ytp9o6nX/u3P2hHQsxuFTHC7dZuKCKZm0+EMdKvsKIWIjE7BixGrqojhYtdMbt9on7++vYW+Fs+eGMfrbJ2V8dKQuKng51c6yzo935X/XHwNg5fTsPn9hD0Q+Rp5fTfr1Zs9nxw8vZFqunX+Fzomc/5p+Da9aHuTW1gCmac5/xFSYrit6nZYki2HQa770xvjMRGbK6IsQ/SYBjBiRwmGFOlfnIyP+YDguBe3KGlo4UBX/4AXg4hnZrJyeTZLFwLIpmdy+pJi7lk1g1XnjANhwpI6/bDqOu4cCeX/dfILfrTvKx0fr0WrgjnOLB6S/fZXkUadLnNZCNBoNSydlsO4Hl/C3yb9hl30pAJlKPVrChHUmGnKWxOV9c5PNUltFiFFOAhgxItW5fB1WIJ2qvwMwHn+Id/ZWRfbciSenJ8CkrERWnTeem84ay90XTmJ8pg2Aq+fmY20dQXnkzf3d1rUJKwr/89Z+1uxTc1Rm5CW1L10eJlK86jJrp7UQAL1Wi1aj4YqrvsLexb+m9Ip/0mTOQ0GDY96dhHV9K1B3umy7lOUXYrSTHBgx4jS1+PF2E7y0qXJ4yU7q2xfZh4drafbGvj1AT1zeIH/eWMLeCge3nF3U4XxqgpEbF41ha0kjRr0WfTd5Er5AmCvn5PHJ8Qa+ODcf1wD0tz+0gRaym/cCUJ84qdM2npwF/HPuK6Ql6DlnUnZk08b+6m6TRSHE6CABjBhx9lU4yU/peWnsm7sruXVxxyChN7yB+C91DYUVHn17P83eIEdr3V3mnJj0OhZPSOfSmTlc+MsPybKbONHgpqzBQ4Pbz+TsRHKSzGw8Use3LpjAySYPhalW/rrpRId7xXM5eazG7PgZesVPMDGfpoRxXTfUaFA0wyfRVggxMkgAI0YUbyDEkVpXrwKYvvIHex7d6Yv1h2rZX9mMUa/liWtn95is2pY8XO308eqOisjxkjp35HleioWC1K5/FguK1EJxKVYjDYO0zQKAxVVGzsG/AOBYfD8EBm/FTUYf90kSQowsMs4qRpSyhhZCMSS4ePwhlBg2HfrwUC37K+OfuBsKK7yztwqAFVOzerUK5eIZOfzxxjMwdTFSk59i6Xa5tM2kZ2xrTkxRegIpVsOgLd21N6m1X2oSJuCZeMWgvGeb7gI6IcToISMwYkQ5XBNbjkRjix+dVkNWD0mdDk+AKoeXbScaWTqp4yaD/bW73IE3EMao0zKjNXjpzWrh6XlJ3Lq4iEnZidhMerYca2BPhQOLUcc9yycxJcfO9tLGTq9dPi0rapqqOMPGrILkOHya7mndNRQd+D8AahKmMPDvqDIbdH3epVoIMfLICIwYMRrdfo7GGMCAuutzYxfTJ83eAJ+UNOALhiK7SQ+Ez042AXDT2WMjoyA9BVVtDDotqQnqhpUWo475Y1OZnpvU5WiKTqNhUnZipyuS5o9NwWYawN9bFIWMf19PUuNuwloj+zIvHbj3Os3SSRkxVzAWQoxcEsCIEeNYnavTzQZ7Y1e5o9PjZQ0egmE152Wgcl9qnF6aWgJoNXDOhPTI8XSbaUCmOxYUp3HRtOxOz1mNerLsZoozbHF/X4BMx2cYa3cDsGf585QnzRmQ9zmdRqNhSo59UN5LCDE8SAAjRoza5r4noXY1AqMMwlaLHx2uA2Bcho2E00Y/LpuVQ0qCgQSjPqby/92ZPzYlsoFjV84YmxL3yrv6kIfLPr0JgIaMM3FlzI3r/bumwSjLpoX43JH/6sWI0dnu073V4PbHbXuBWO2pUEd/5o1J6XDOpNeRk2Rh8fh0Vk7vfNQkVj3tX9T2vovHp/fYLhaXHLwPgJA5lT3zfxLXe3cnP8UyrPY7EkIMDglgxIgQCis0uPo+AuMPhdl0rD6y4eFgURSF43UtABSmdT1dlGQ1kBeH0vexjOLMyEviounZXSYTx7InUWLzUYobNwLQPPcOPAkFvb62P4x6bdxGroQQI4sEMGJEaHD7+5z/0uaTkgb+9kkpx2pdKIoyYDkvp6p0ePEEQug0mkh5+64STftbPTbJYojpy1yrVfNGMhKjV+4kmPSMSbWSZY8+rg80o3d3nug8tuwVADyGFJrn3BFjz/tu7DDbOkEIMXgkgBEjQrXTG5f7ePwhXttZQb3bz6Hq5rjcsztt75FmM0ZWDc0uSOqyfX/qtPR2VdPpzhiTyrjM9qTeL87LJ9NuJivxlPspYRZv/BpT/nEuBndV1PV6dxXFx18CYMPU/we6wVkJZNRpmZYribtCfF5JACNGhCpHfAKYNusO1rJvAArWne5Qtbrsu60+iVGnpSi96xVAiWYDiea+LXMe080UVXcmZSeSl2xh3Gn90mo1ZNhMFKUnUOD4lGTHAXRBD7aqLVHt0vb8CUOohUrbdErT47ObdG+cMTa1Q1K0EOLzQwIYMex5AyEOVMU32ChraInr/bqSbDEwNq19OiY32dLjKEtOUt9yYbrLsemNJGvHkZO0xl2c7d/IzOpXIseyd/8ebUgNKDXNlaTtfQaAT/JvAs3g/S9lZl7XI1lCiNFPfn0Rw16D208gNDQriPqjxull2dQs5hfNZNPReoBIQbruJHcSSPQkyWLAEO+lxDX74U/LyABOrU1sbTzA+IN/5GDaTehL3kcb8tGYNJVjqUsYjAmd8Zk20mymHpeKCyFGNxmBEYOjfDv8aTkcXhPzpY0tg7cJYbyEwgrv7a/h689+yoeHamO6ti/VZOMevAA8f02HQ41z/gOArMq1jGnchHX1XQBUZSzu3d4IcTAlxz5oezoJIYYvGYERg+O9H0HZFnj+i1B8HlzzLJh7NwXQ6B7cpc/xsPZADbUuH3azntkFyZGdpXsjN9mCQdf7L+hxA1FVN+ABR2nrG1wAYxeDNY2U8RfAjt+R5NjPVY5vRppXZQ187otBp4nLUnMhxOggAYwYeOEQHN/Q/vrYB/A/hWC0wQ2vQsH8bi+vitMKpMH0/gF1ufHtS4qxt46o9LauSmqCsdN9jLoyIPv/VKu7SZOQAdf/s310JRwGNHBKBWN/YiH1KbOh0cNAjYsUplr56oIxvZqCE0J8PsgUkhh4TSdACYFWD1Muaz/ud6kjM91QFCVuS6gHy9FaFxVNXrQaWDk9J3I8llVCE7JsgzUj07nKnepj9szoqSGtllODl9Alv8R/5w5sZgN5KRa+MCcPo14b90Dm1A0thRACJIARg6H+qPqYPhG+/BzctQfO+4F6rGwzhIJdXtrUEhiUgnPx9O5edfSlIMWK7ZQl0bGsLpqcbR+YvJbeqvxMfcyZ2fHc5NYdpheuQjf/FmwmPQuKUrl0Zg4pCUay7GZm5MdnhVCqVYIWIUTnJIARA6/hmPqYWqw+JhfAOfeAzgjhIDjLu7y0ztX3/Y+Gyt7WvY+KM6KngQZkqmcgVH4Ge/6lPi9c1PH8ysfg6j/B8kcih1ISjFiN0cGa3dz/zzs2XSrtCiE6JzkwYuA1nlAfU8a2H9NqIbkQ6o+oU0wpYzq9tKGLXaSHs998dS5njy/leJ07csxq1MV99+e4CwXg1f+A3X9XXyfmwPhlHdsl5cGML/Z4u+l5SX2qt6NBQ1F6AmeMTY35WiHE54cEMGLgNbUGMMmnBSkpRWoAU3sQijpfxeL0dj29NBy1bRaZl2yJqh6cMgD5G7Fsttgrb3+vPXgB9e9E27+NEi+anh1zDZ/ZhcnYpMKuEKIHw/xXQjEqNKnLcY8EUlGUU77M8uapj2WfdHnpYO8e3R/1Lh8/en0v//HcNsKnbTyZYTN1cVXfFKRaGZcRx+mVcAh2/6P9tVYP87/e79tqNBqMei29XRVuM+kleBFC9Ir8n0IMvNYRmI/rrCS7/ZF9gShcqD7ufhkmXwzTvtDh0pEUwOwoa0JRQFHoUCU2vY8BTLrNSLqtffRmZn4Sh2tcLChKRRPPEZh9r4HPASY73L1PDWAM8au58sUzCtDQfa27LLuJwtT+bYcghPj8kABGDCxPE3jVpFanOZfKJm/7l3n+fDUPpqkUdv29QwATDiu4RsgUkqIoHKtVc15uPGsMwdOmTfq6QaO+dfNHBYXJOXYybCbOHp+O2dC/qZ0onib4d2tRujNvA1Ni/O7dqm1U5bxJmR2WWGckmphdkByVBCyEED2RKSQxsBxlAHiNKQR01uiaLiYbXPaE+rz+cIdLm71BwsrI2AOpptmHJxDCqNNyxpiOyae2PgYwABOzbKRYjeQlWzDqtfENXgD2/xv8zZA2Ac797/je+zSzCpJJTTCiQa2qOyHTxnULCpkuGzMKIWIkv/KIgdW6AsltyQWguvm0onTpE9XH+qMQ9IO+fbpkpOyB5A+GWd+639H0PHunq436k9eRaTeTaTf3+foelW9TH6deHvXzHygajQaLUceSiRk9NxZCiC7ICIwYWK0JvG5LHgANLn90gmtiLlhS1Eq9256JurSmeWTUgHlu8wkqHWrl3eVTszucH5BRk3iqax39Sp80tP0QQogYSAAjBlZbAGNVA5hgWIkeWdFqYen31ecbfqlmwALv7q2KFIQbzhRF4aPDdQCcNS6d7KSOIyWJw3lVjaJA3SH1efr4oe2LEELEQAIYMbCaoqeQAHaXnxaYzPsait4MzRVQf5QjNS72VTppahn+K5AqHV4qHV50Wg0zuyifnxiHirQDpv4ouGvVqsiZU4e6N0II0WsSwIgBoyhKhykkoENgEtYacSS3fnme3MqGw7WMhNxdly/Ik2uPADAm1drl3kV9XYE0KI5/qD7mnxnXZdNCCDHQJIARA6be5eswhdSmoskTqfFyoKqZEnNrAPPqHYTDoUHtZ199erwh8nxOYXKX7azGYZj/oijgaYQ3vq2+LjpnaPsjhBAxkgBGDJjammrwOQFwW3KiztU0+/j3ZxXUNvvYcKSWavuMyLmxNe8Naj/7QlEU3thVCcAlM7LJT+m6AFtcC871hqsWDrwJJz8Ft5qfg9cB/lP2Jdr/b/jp2PbXEy8a1C4KIUR/DeOxbTGShcIKzbWteyBZUgnpLED0Euq6Zh9/+6SUUFihNGVh5Hhh7Xo+s58/iL2NTWOLn0+PN7KvUg3OFk9Ip8oxTFZMBbzw9Mr2ujrmZPjyc/DXL6h1d+7cBglpsOP59muWfh9yZw9Fb4UQos9kBEYMiHqXD31L62//tqwu24Val1QH9Db46ssApLiODnj/+qq80cPLn5ZFgpeFRanMKUwZ4l612vsK/DgruiigtwnWPgzhgDpldPgdaK6GI62jXMt+BEvuGYreCiFEv8gIjBgQx+rc2HxtAUwvC5a17lZt81YMUK/675G39uENhAFIthi4eEZOD1cMkqAPXvuvzs+VbWl/fngN+N1q3Z38+bD424PTPyGEiDMZgRED4litG6O3NYBJyOzdRckFAJiCzZiCzQPUs77zBkJsO9EIwFVz8vjfG+Z1WnV3QCkKvPcjePf+SM0cvE54JFPdDgBg4X/CvUfhqj92vH7vv2Dvq+rzccN3mk4IIXoiIzAi7vzBMLXNPozeevWArZcBjDEBrOnQUkeirxKfvvebCgZD4T70tPe8gRDbSxsJhBQSTDryUyxMyk6kpM49oO8bxd8CvzsLGkvU12fcCqlF8OFj7W1mfQUuelR9njun8/uc2KA+5swauL4KIcQAkxEYEXflTR7CitI+AtPbAAbU3akBu7ey15fUNHv56h+38MYudVVTvDW2+Hngtb2sO1jLg5dPY2FRWq9XFhm7qA3TK9V7IdS6G3coAP97TnvwAlCzD5qr4JNTRlpmfaX9edq46PutPCXQ0eigYEHf+yaEEENMAhgRd2UN6nJdo691BKa3U0gQCWCSfD3nwYQVhdpmH79ccxiHJ8DRWjev7izHF4hfHZkWf5C/f3qSFn+I3ScdXDozp9c7J1uNOlIS+rg5oqcRnr4YfrcI6o7AgTeg/kh0m4NvweOTIOiBlCK46U0oPrf9vEYD535PDVa++jJkTWs/N/UKSEjvW9+EEGIYkCkkEXfBsDqdY/KqOzRjy4TGXl7cyxEYfzDM3z89SZUzeml2iz/E3tYVQvFwstGDpzUg+tmXZqKNoaZLTnIfK9sqCrx5j7qCyNsEv5kXff7Mb8An/ws7nms/dsVTMPbsjvc6979h0Z1gtqu7fU9YDkn5sPzHfeubEEIMExLAiAETyYFJyIg9gPF1HsCohe/qeOSN/YRO2W9g1Xnj2VnWyMYj9VQ0efrT7Sh1LnVK6qxxaeSnWGl0+3u4ol26rY+jL+/9P9jzj47HNTr4zgEwJ8Gnf1aXRoM6bdRZ8ALqZplmu/pcb4Tr/t63PgkhxDAT8xTShx9+yGWXXUZubi4ajYZXX3016ryiKDzwwAPk5ORgsVhYtmwZhw8fjmrT0NDAddddh91uJzk5mVtvvRWXyxXVZteuXZxzzjmYzWYKCgp47LHHEMNfW10XlDAGX2up/ZhyYNSl1PZOppAcngAvfVrGthONkeBlYpaNa+cXcOnMHPJaRzxO1LdQ3uihxR/E4w9R1tBCpcPT3rcYVDrUEZ68PoympNtMMV9DSwNsfKL9tb11C4aChfBf29Sfpd4EGZPa21z6y9jfRwghRriYR2DcbjezZs3illtu4aqrrupw/rHHHuPJJ5/k2WefpaioiB/+8IesWLGCffv2YTabAbjuuuuorKxkzZo1BAIBbr75Zm6//XZeeOEFAJxOJ8uXL2fZsmX8/ve/Z/fu3dxyyy0kJydz++239/Mji4FU71ZHLCxBB1qlNRclIQPoZVJuF1NIYUXhg4M1UUFIgknHuRMzKM6wodVoyLabyUu2UN7k4Vsv7qDJE+Dxdw9F2i8qTuPMotRefxaXN0h5ozqaMy3P3uvrIh/FEuMu1DX71VVGAGjgtrWQN7fztisfg3WPqrkssgmjEOJzKOYAZuXKlaxcubLTc4qi8Ktf/Yr777+fK664AoC//OUvZGVl8eqrr3Lttdeyf/9+Vq9ezdatWznjjDMA+PWvf83FF1/Mz3/+c3Jzc3n++efx+/38+c9/xmg0Mm3aNHbu3MkvfvELCWCGuRqnGsBY/a3TR5ZU0MXwRd5aC8YcasYYdOHX2wDYX+nkRH0LWg3cdk4x0/OSOFTVjFbbnpOi0WhYNiWTZzed4Fgny5s3HaunscXPkgkZWHqxweLWEw0oQG6yuU+jKUnWGAOYt+4FpXU5+Pxbuw5eQJ0yuumNmPskhBCjRVxXIZWUlFBVVcWyZcsix5KSkliwYAGbNm0CYNOmTSQnJ0eCF4Bly5ah1WrZsmVLpM2SJUswGttzCFasWMHBgwdpbOw8mcLn8+F0OqP+iMEVDiuRHaatgT5MHwEYE/DrElrvUR853Fa6/8yxqWQnmZmVnxwVvLRJtho5Z7y6uibJYmB8ho2Lp2dHzh+oaubtverojscf4qWtZby2s5ywEj29tL/Sya6TDgDm9mGrAItRh0kfwy7UtYfg+Eftr09dDi2EEKKDuCbxVlVVAZCVFb33TVZWVuRcVVUVmZnRX2p6vZ7U1NSoNkVFRR3u0XYuJaXjF8qjjz7Kgw8+GJ8PIvrE4QlEpngiAUxCL7cROIXHlIaxxU2Cv54myxj8wTBVrbkoU3J6nsqZOyaF718yhQ8P1XLpzBx0Wi05yRbe2l1JpcNLWYOHPRUOvIEQG4+oQVJGohld6wojRVF4b181ADlJZorTE2L+DNl2c2wX7HtNfRy/TF1RlJjdfXshhPicGzV1YO677z4cDkfkT1lZ2VB36XOnsaV9hU6CP8YqvKfwGNURlITWERi1MJ46omLvZV5Jlt2MXqeNFJzLSDRxzRkFzMxXa7h8cKCG7KT2IOM3a4+ws6wJgDd2VXKk1o1Oo+Giadm9Llp3+vvHpOm4+liwQIIXIYTohbiOwGRnq//jra6uJienfZO76upqZs+eHWlTU1MTdV0wGKShoSFyfXZ2NtXV1VFt2l63tTmdyWTCZOrDqg8RN/WnLDFuH4GJPYBpMaYBkOQ9CcCJejWfpSCl/8mqSydmUNrQQlNLgH9uKyc/xcK0XDvv7K1m/aFadpY1RabBzhqX1uuA6XQx75HkKFcf21YdCSGE6FZcR2CKiorIzs7m/fffjxxzOp1s2bKFRYsWAbBo0SKamprYtm1bpM3atWsJh8MsWLAg0ubDDz8kEAhE2qxZs4ZJkyZ1On0khoe6U8r4R/JXersT9Snq7VMAmF79Gr5giL0Vav5LUUbsUzmn02g05J+yJHpMqpUbFo1hRmt13bbgBWBOYXK/36/XHGqwRlL+4L2nEEKMYDEHMC6Xi507d7Jz505ATdzduXMnpaWlaDQa7rrrLh555BH+/e9/s3v3bm688UZyc3O58sorAZgyZQoXXXQRt912G5988gkbN27kzjvv5NprryU3NxeAr371qxiNRm699Vb27t3LSy+9xBNPPMHdd98dtw8u4q/m1ADG35bEm9VF667tz7+GMDqSveX46ssIhhVsJj1Faf0PYAAWFKeRmWhCp9Fw+exctBoNT35lNhdMzsSg02A16rhv5eQ+TR31SSgITaXq89ZVWEIIIboX8xTSp59+ynnnnRd53RZUfO1rX+OZZ57hu9/9Lm63m9tvv52mpiYWL17M6tWrIzVgAJ5//nnuvPNOLrjgArRaLVdffTVPPvlk5HxSUhLvvvsuq1atYt68eaSnp/PAAw/IEuphzBsI0djip4i2FUR9n0IK6G3UJYwj032IpMbdwGSy7ea4BRQ2k55r5xeQZjMxPS8JhydAgknP9LwkxmXYOHtCGk0tAY7VDvBO04qi7ld04A0I+cCQAMljB/Y9hRBilIg5gFm6dCmK0nVFU41Gw0MPPcRDDz3UZZvU1NRI0bquzJw5k48++qjbNmL4qG32ceo/i4TIMurYp5AAmsyFZLoPoWuuACZHJdzGg0aj6bQWjMWow2420NQS6OSqOCr5EF68Hs6/H9b/VD2WVqyW/hdCCNEj2QtJxEX9aXsEbSy8g/HmZsa1bg0QK3drIq/Bo24ImR+HBN5ho2IHPHuZ+vzte9uPn3PP0PRHCCFGIAlgRFycvsnhvqzL0eUlMc6qlu5XFIVwWOm0+Fxn3AY1gElTGjHptWQkjpIVZo0n4PlrOh5PGQvTrhzs3gghxIglAYyIiyqnN/Jc72tCF/IB6sqeaqeXX71/GJc3yNJJmRT1ojBc21LqDI2D/GQL2sFKqB1o6x4Fdw0YrBBoaT++WBLUhRAiFjLhLjrVXZ7T6QKhMLWnrECatuk7fHPzOeQcfwWAx989SLXTh9sf4q3dlTS1+Lu6VYQ7EsA09W1X5yGi626EafX34bO/qc9XPgZffVl9nj4J5tww8J0TQohRREZgRAfHal04PAHm9HIPoMomb9Qu0Qaful+V35hMiz/I2gPthQsV4JOSBmb3cG+3Qa3Gm6FxkJZg7LbtcNLlVJenETY/pT7X6NTpIlMifLcEtHpJ3hVCiBjJ/zVFFEVRWLOvOqZVOKUNLVGvjT51BVLAlIrVqOeDe5by5TPyWT5VrQnz1p4qqk+ZcuqM26DmzqThID1h5MTZXW4hcHxD+/Pb16nBC4A1Fcw97+8khBAimgQwIkqFw0uLPxTTNYdrmqNet43ABEzqKEui2cCcwhTGZdgibb7x122RLQI6Ux2yEVY06DQK2foBrscSR+m2LkaLmqsADZx5O+TMHNQ+CSHEaCQBjIiy+2RTTO0dLYGo0Rpt0IMu5AHAb0qNamvUa5me2z7a8OrOCupcPjpT7QpRjzpKYQ/Wx9SnoWI26Eg0n7Z3UjgET18Mb90DObPgop8OTeeEEGKUkQBGRKlydD+1c7odZY1Rr9tGX0IaPc3hjvkg503K5M7zxkdev7GrEl8gesTH4w9xoMpJuaIWwUvxlsbUp6GS2Vn+S91hOLFRfa7VgRIe3E4JIcQoJQGMiAiGwjR5YqtAW9boiXqtbw1gmrVJfOvlz/j2Szujzmu1GsZn2njqq3OwGHQ4PAH+uKGEV3aU8+Dre3H7gvzvh0c5WutmX1gtgpfhPtT3DzWIcpM7KbbXWKI+ZkyB29aCbuTk8wghxHAm/zcVgJq8Gwwp9Hb19MYjdUzIsnU43jYC00giigIp1s5zQvJSrKycns17+6txeoMcqXFRUucm0aSn2RsEwGGfCC1rSXMf7duHGmSdrkBqPK4+pk8Y1L4IIcRoJyMwAoADVc14g50n7566RBrUkZo95Y5IoHEqQ+sKpNqQWqxu0bi0Lt+zINXK9QvHMCkrkbxkCw9ePo3ntqjTRedMSCdr7FQAknzlsX+gIdDpcu/aA60nxw1uZ4QQYpSTERgBQHmjh5wuNkw8WNVMeZMHo16LQashrNDlSqW2EZiaUCIaDZxZlNppu0h7nZaLpmdj1Gu5bGYueyucvL2nkpl5STj8eQAkecvbd24epkwGLUkWQ8cTFTvUx5zZg9ofIYQY7SSAEQBUdlOXRUFhT7kj8tqo73rgri2AaVASKUy1dv6l3oUkq4FHr5rBmDQrHn8IpykHBQ2GsA9roCGyvcBwNLcwpeM+TwEvVO9Tn+fOGfxOCSHEKCZTSII6l4+65s6XM3fGH+x6JU3bFFIjiUzOSuxTfww69Z9lWGugpbWgXYK/rk/3GiwdtjsIBWHfqxAOgCUVkguHpF9CCDFayQiM4JOShrjd69QRmEnZfQtgTuUyZpAQqMfmr6WWSf2+30CJyn9pKIEnZ7e/zp0zrKe/hBBiJJIRmM85hyfAkRpX3O6nuNWic40kMrGPIzCnchvVPZES/LX9vtdAsRh0JFtbp8pCAfjziugGk1YOfqeEEGKUkwBmlCo7bX+izngD6u7Qp68y6o+gSw00tAnpmA26ft+vLYBJ9Nf00HLo5CRZ0LSNsFTuAld1+8nZ18Oc64emY0IIMYrJFNIoVOXw8q/t5cwuTCYv2cL4zI71WkrrW1izvxpnjIXremIJNAGQnpkbl/s1WMYCkOk6EJf7DYSopOYj76mPlhS45R3IGL7TXkIIMZJJADMKbT3eQFhR2H6ikX0VTuwWPWkJJnStq2T2Vzp5d2814d5WreulYDBEkqJu7JiWkR2Xe5bbZwOQ27xr2C+lJhSE7c+qzy/6HwlehBBiAEkAM8r4g2FK6tp3b/YGQpTUutlR2oTDE8Bs0HE0jjkvpyqpqESrqMXt2nai7q+6BHXfJHPQiSXYhMcQn/vGS9Toy7F14CwHazpM+8KQ9UkIIT4PJAdmlClv8nSa0xIOK5Q3egYseAEIO6sAaNHaCOk6L4oXq5DWRLMxE2gtaDfMpJ66+qhyp/o4/gLQd7KtgBBCiLiRAGaUKT9tc8XBZGhRk1fdpoy43tdhbqvIezKu942HtIRTApXag+pjxuSh6YwQQnyOSAAzylQ0DU0AoygKJo+6UihozYrrvRst6q7UaS3H4nrf/tJqNBRnJLQfqN6rPkoAI4QQA05yYEaRQChMVTdbAgwkhydAhqIWxFNs8UngbVOToCbDDreVSBajLlI1GL8baverz2XbACGEGHAyAjOKVHSR/zIY6lx+CjTqCIwvIT5LqNvU2NQRjaKmTVyx79towvFd+h0rjaaT/aAqd4EShsQcsOcMTceEEOJzREZgRpET9T0XrxsoVU4vYzVqEq8ncUxc711rHR95Xty4gX3O/UB+XN8jFiun55CaYER/6uaN5dvUx9y5Q9MpIYT4nJERmFGkrHHoApjKJg9jtGoSb0vi2LjeO6QzU5MwIfI6wVsZ1/vHqjDVSkaiqb36LsDJT9THPJk+EkKIwSABzCjhDYSojWFH6XgKhRVqmj1koW7k6LXGfwrlvXH3R54neirifv/eshh1WIynbZEQ8Ko1YACKzh30PgkhxOeRBDCjRGlDC3EurNtrtc0+EsJu9JowAAFTatzfozpxKpvzbwEg0VMW8/VWo46r5+YzIy+pX/3odH+nV/8DvA5IzIW8ef26vxBCiN6RAGaUODmE00cVDg/pGgcAQUMiis7YwxV902AtBiDFdSTma5dPzaYwzcriCekdE3D7o+YA7P2X+vzyX4O2/xtYCiGE6JkEMKNEjXNopo9AXf2UiroH0kCMvoC68qdo2gIAUt1H1BU/MciyqwXnzAYds/KT49exQ2+rjxMvggnL4ndfIYQQ3ZIApg+UoZqr6UIorFDnGpoAJqwoVDR5SdM4AfCb0wbkfYrSE5gyfS7ojOiDLeidpb26TqOBJIshKuF2YXEqJkMc/um3NMB7P1Kfj13c//sJIYToNQlg+sAXjO23/4FW0eQhEBqaoKq22YcnECJDqwYwAfPAjMBMy7WDTh+pcmuo29er69ISTB2SbvU6Ldn22PZqyk+xqk+CPvjgUTi2Hp69vL1BwcKY7ieEEKJ/pA5MH/iC4c6TOYdIpWNoqu8ClNSqO1+PsXggMDBTSEkWA+MybOqLrOlQtQtz40HQzuzx2q7yXcakWTla6+703OnyUizMLkhWX/zzVtj/eieNpP6LEEIMJhmB6QNfMDTUXYjS4B66/JcdperS6UKjmkQc6GIK6dSSKbEqykhonwJKGweAzV3aYxBpPX258ymm5iRhNeoYk5aAqYek3vwUi/qkclfnwcu1f5PkXSGEGGQSwPSBPxjG4x8+QUydyz8k71vl8FLh8KLRQL5JHc3oagTm4hk5ZNrbd242G7RoexnVjG8bfQFIVVciaRqOcWZRSrfXdbdk2mLUsWRiBjaTnqJ0W5ftAHW6qWoP/OPmjidXbYXJF3d7vRBCiPiTKaQ+UBS1dH5RekLPjQdYIBSmfogCmK3H1c0bs+1mbKEmtT9dBDCJZgMrp+fwWVkTep2Gydl2jte52V/Z3O17JFsN7SMgEBmBoeEYM/KS+fR4Y5fXTsmxd3tvu9kAQGqCkZrmrqfh0mwm+ORtqG9dvr38ETAnq8FUxsRu30MIIcTAkBGYPggrCtVDtOvz6epcPsJDtCrqnb3q3kfFGQkk+OuB7lchpSYYOW9yJmeNSwdgUXF6j6MwE7MSo0v2pxSpjy11GIPNnFmUik6jYUyalStm55JmM6LRgF6nQaftx7xVK6Nei92sb99pOmcWTPsCzL0Bxp7d7/sLIYToGxmBiVEgFOaHr+0hM9HEwuKBWTIci+AQrT4KhNSVWFoNTMlKJLFCDWZ8CXk9XtsWWCRZDSwsTuXjo/Vdts1NtkQfMNshIQPctdBwjIlZM8i0mzEbdBRn2Eg0GwiEwpHRlf7KTTarAVTtQfXA0u9D0tBtJCmEEEIlAUyM3txVyYeH6gB45MoZw2o10mDyBcN86YwCWnxBdN5GjGEPAN6E3Jjuc2ZRKp+dbOp0GbhGAzlJnSx3Th2nBjD1R0nInUOCqf2fcUaiqWP7XuhqJGhchg0c5VC9Rz2QNbVP9xdCCBFfMoUUI09ATd7Npp7GI58MaV+aWoYm90VRFN7ZU8VDr+9jV7kDu0/dHdptSEPRxRZAaDQairtIok23mToPENNbd6auOxTTe3Un3WZiTmFy1LEZeUlMy02CP7Ru0JiQAUkFcXtPIYQQfScBTIyMOvVHttn8X+S8dBHhmvh9icaqvMkzJO9b1ujhZJMHs17L1Bw7Nn8tAM2mzKh2hl7uOTS2i2ToSdmJnV+QMUl9bJvWiZNF49JINLeP5kzJtaPzNqqjPQCTL+nfenAhhBBxIwFMjE7/Uq7Y//EQ9YQhW310tMYFwHmTM0mzmbD61dVILYbonKB0W+82dcxPsUTVYjHoNBj1Woq7WuWV1joC01gSY8+7Z9LrIhV385It5BncsOM59WT6JLjsibi+nxBCiL6THJgYqV+07fkaJxu9DFVKZ2OLnyIGdym3oigcrVUDmHMmphMKgzWgJuG6TwtgMhN7V67fbNBx/cIxkddZdjOTs43q8uXOJGarj83VMfa+Z0XpCQRCYebUvgLPfKv9xMI74v5eQggh+k5GYGJk0GkwEYi8bvQECIUHfyWQoig0tQR6bhhnVU4vbn8Io07LnAK1kJw10DoCY2yvAZNiNcSUUHtqrktxegJnFnWzJUFbAOOugXB8CwpOyLSp9WPW/U/7waRCmPXVuL6PEEKI/pERmBiFwjBeUx55HVY01Lt9oEBmjBsE9ke9209wkAMnRVHYfEwNVorSEyL7DCVEppBSaQtDijO6r27bnXE9XZuQARotKGFw10FiVp/f63RarQaaq6BZTUxm5WMw5XIwDN7frRBCiJ7JCEyM9O5K3jT9IPJaGw7w1q5KVrcWdRssQ1FI7/0DNZQ2qHsejctsn7pqm0JqMbZPIY1N6/vUlranAnRanRrEALji/HNvKoXHW5OE0yfCgm+APSe+7yGEEKLfJICJkdl5PPpAwENjS4BGdwBvYPD2R6oa5B2oj9W62FvhBNSdnItOCVCs/rYcGHXax6zXRZf/Hwi21lGXeObBuGrgt4vaX0+5LH73FkIIEVcSwMQoSHRdkmaX+qUeVhQ2Heu6omy8VQ3gCIwvGOLvn5bR4FZXObX4gry3vwaAeYUpXDk7D72u/Z9OJAemNYl3dmFyz6Mo/dWWBxOvEZhwGP68AvxqgjLWdDjrv+JzbyGEEHEnAUyMwqcljZ7T8K/I80NVzYQHIS8lEApT1zxwS6g/Pd7IW3uquOWZrSiKwsufnsQTCJFsUUv/n0ob8mEOqV/6bUm8VuMgpFbFewSmdBM0HFOf6y3wrZ1g6X63ayGEEENHApgYhUPRAUyOUo0x2PoF7g9RUu8e8D7UNg/cBo4uX5CdZU0A/OfScfz4zf38fdtJAOYXpUaNvABYWqePghoDPl0XhecGQjxHYI5vhHdb85qyZ8Kt74JpED+LEEKImEkAE6tOlu0ags2R56X1LQPehdpm34Dde0+5g2BYYXymjQunZvGlMwqwGnUsmZDO1Bx7h/ZmX2sCryEVNJoed5eOm9Rx6uOR96HkI+hrQNd4HJ65GCp2qK+XPww5M+PSRSGEEANHApgYnT6FBBBscUSeH693owzQ6EibttyUeAsrCvsr1Zye8ydlotFomJSdyDM3z2dOYefTKW0jMG3TR+mJvau+22+TLgKdUa3G++ylcODNvt1n37/bn+tMMPac+PRPCCHEgJIAJkbhcLjDsYCrMfK8qSXAoWrXgBa3q3UNzAjM9hONOL1BjDotc8ckR46futvz6cw+dWfutgTenKQBXn3UxpICE5a3vz6ypm/32f96+/Ob3lSXaAshhBj2JICJkRIOAnBEP4ESrVr+3t/SFNVm9Z4qtpTUEwiF475jtMsXpGKANnH892dq8bYJWTZM+t59kVtai9i5DWmYDFqSLIYB6Vunlj/c/nzPK7Dvtdiu/+xFONm6o/jdB6Bgfvz6JoQQYkBJABMjpTWJV6PV4TGq0ypNjdHLp8OKwtaSRl7+tIz6HqZ7SurcMdWPqXJ4+5zu0Z3yRg///qwCoNNcl65EcmCMqaQn9H7rgLhILYab31af+xzw8o3tK4m6Eg7Dlj/AH86DV76hHsufL8XqhBBihJEAJkZtU0garRaDNVl97nN2mDIKKwo1TnWqp6sARVEU1h+sYdPRevzBjlNTnakfgOmjQCjMP7afjLzOSep92XyLrxYAtzGd3ORBmj46VcECSCpof92WjNuVj5+Et++Fiu2g1as5Lxf9dGD7KIQQIu4kgIlV6xQSGh1JKekALNLupdnb9caKeyscnR4/WN1MY0uAnWVNvLqznGCo5yCmrDH+00cbj6h5LJmJJq5fUIgmhpVEllNyYIozBndnbEDNWbn2+fbX/7gFyj7pvK2iwJbfq89zZsOdW+GmNyB/3oB3UwghRHzFPYD50Y9+hEajifozefLkyHmv18uqVatIS0vDZrNx9dVXU10dXYystLSUSy65BKvVSmZmJvfeey/BYDDeXe2TsNIaZGh1BMcuBeBc7S6aPF0HMIerXZ0GOJ8eb0/+LW/08Mnxhm7fu8rhpawhvsu0a5t9fHZSDbAevHwaabbYpoHaAhifJYPsQdzMMkrOLPj62vbXe/7ZsU3tQXhqgbpJo84It7yjTkEJIYQYkQZkBGbatGlUVlZG/mzYsCFy7tvf/javv/46f//731m/fj0VFRVcddVVkfOhUIhLLrkEv9/Pxx9/zLPPPsszzzzDAw88MBBdjV3bMmqNltD4FQDYNF6OV9R0eUkgrPDS1jJqnF4cLQEURcHREuhQz+Vwtavbt95T3vlITl8pisL6Q+oU0MRMG4vGpfVwRYcbRAIYe0bewG8f0J38eXDhQ+rz8m0dz7/+Lag7qD4vWCC7SwshxAg3IAGMXq8nOzs78ic9XZ1qcTgc/OlPf+IXv/gF559/PvPmzePpp5/m448/ZvPmzQC8++677Nu3j+eee47Zs2ezcuVKHn74YZ566in8/oErn99bkWXUGh2mhET8GvWL0FlXgS/YdTJuszfIG7sqeXbTcY7WuthS0nHfpMYWP+WnrTDadqKR/ZVO9lc6OVDljN8HAY7UuChv8qDTajh7QnrM12sCLvRhdU+mgoKiuPatT9o2Xzy5Fcq3tx/f9Xd1qwCAaVfB5b8e/L4JIYSIqwEJYA4fPkxubi7FxcVcd911lJaWArBt2zYCgQDLli2LtJ08eTKFhYVs2qR+wWzatIkZM2aQlZUVabNixQqcTid79+7t8j19Ph9OpzPqz4BoHYFRNFosRn2kgFuK0sSO0qZuL3V4AoTCCh8cqOVAVXOH84oC/9x2kmO17SMxbl+Q1XuqWL2nikAofsuPgqEwH7XmvpwxJgW7Ofblz3q3OvXn0yWQmxnj6M1ASC2GiRepz49/pD4qCqz7ifp80Z3wpachdRgEW0IIIfol7gHMggULeOaZZ1i9ejW/+93vKCkp4ZxzzqG5uZmqqiqMRiPJyclR12RlZVFVpe5pU1VVFRW8tJ1vO9eVRx99lKSkpMifgoKCLtv2h3LKFJLVqKfFqH5xZ2iaON7LfZBcvmCXhe5CYQWXb+DzfbadaKTZG8Rm0jNvTN82LdS1qNNPHlN6nwKgAZHfWsulcpf66DipLq3W6mHpfUPXLyGEEHEV922DV65cGXk+c+ZMFixYwJgxY3j55ZexWAZume19993H3XffHXntdDoHJIhRwu1JvDqthhZzJjRDrqae95r9BEJhDLrhvbjLFwixrVRNID5nQnqf+6tvac37SciMV9f6ry2AOfIeBLzqdBJA1jQw2YauX0IIIeJqwL9pk5OTmThxIkeOHCE7Oxu/309TU1NUm+rqarKz1d2Fs7OzO6xKanvd1qYzJpMJu90e9WcgKEr7CAyAN0ENksYbagkpCrtOxjfRdiDsLncQCCmkJRiZkNn3L3Vd6xSSNS03Xl3rv7GLITEXvE3w9nfhHzerx/Olyq4QQowmAx7AuFwujh49Sk5ODvPmzcNgMPD+++9Hzh88eJDS0lIWLVoEwKJFi9i9ezc1Ne2retasWYPdbmfq1KkD3d0eKZEcGLXUvt+ubicw3aKOaOwudwz4Zo794fQE2Nq6fHtuYUpMNV9Op3epWw8YUwZmuq5PtDoYf4H6fPuz7cclgBFCiFEl7gHMPffcw/r16zl+/Dgff/wxX/jCF9DpdHzlK18hKSmJW2+9lbvvvpsPPviAbdu2cfPNN7No0SIWLlwIwPLly5k6dSo33HADn332Ge+88w73338/q1atwmQa5FL1nYmsQlJ/dKEUtZbImHApWo2aqOvopibMUPvwcC3+UJicJDOTcxL7dS+duzUnyT6MRmAAipd2PDbu/EHvhhBCiIET9wDm5MmTfOUrX2HSpElcc801pKWlsXnzZjIyMgD45S9/yaWXXsrVV1/NkiVLyM7O5l//+lfkep1OxxtvvIFOp2PRokVcf/313HjjjTz00EPx7mrfKG17Iak/uoQxcwFI9lUyxa4u8x6IarnxUOP0crRWTTS+YHIm2n6MvgDo3eoIzLALYCZdHJ2we8dGsA2jPB0hhBD9Fvck3hdffLHb82azmaeeeoqnnnqqyzZjxozhrbfeinfX4uL0KaSi/DwazQWkeMtYkFDFXkchJxtamJGXNJTd7MDpCfDWHnXEZFJ2YswVdztjcJapT+z5/b5XXBmtULNffX7xzyF7+tD2RwghRNzFPYAZ9dpGYFqnkCxGHTWWfFK8ZUw2NwKFHK9vwR8MY9QP/WqksKKwp8LB/7x9gEBIQa/VsKAotd/3NQea0LdNIWVM6vf94u5Lz6gVedPGD3VPhBBCDAAJYGJ05tgUqIXMJGvkmMeaB41QqK0j2WKgyRNgf6WTWQXJQ9ZPhyfA659VUO9ur15sM+m5eEY2KVZjv++f4T6sPkkZC+aBWfHVLxoN5J8x1L0QQggxQIZ+iGCEyU9Sv/wTLe1TMH6bOoVi91UyI1+dOtp4tK7bHaoH0tbjDTzz8fGo4AXg+oWF5CTFpxZPhvuQ+iRLpmeEEEIMPhmBiVU4ug4MAMmFgBrAzByXxMGqZmqafby1u4r5cZiuiUV5o4ePj6r7LJn0Wi6dmcOVs/Nw+YKdbl/QV5ERmOwZcbunEEII0VsyAhOrthovWl3kUEreOADs3gr0Oi1LJqgrrqqcXr75tx0crel+l+l48QfDvLtPzUuZkGnj9nOKyU+xDkguTqanNYCRERghhBBDQAKYWCkdR2CSstVEUZu/Fm04QF6KhS/MySMnyUworNDi73qX6rh1S1F4Z28VTm+QRLOeC6ZkotV2vkzaEmikuH49xmDfAittOECq+5j6QkZghBBCDAGZQoqV0lbIrn0ExpaWS0BrwhD2YfdV0GQZQ2GqldxkM+Mzbbh8IeqafQParU+ON3Cszo1Oq2Hl9GxMel2n7fQhL1/a/Q3SPCVU2qbx9+m/J6Qzx/Reme6DaJUgmJIi02dCCCHEYJIRmFh1kgOj1Wlx2tSKvGktJZHjOo0Gu8XA7pMOgm0VfAfAsVoXm481ALB0Uka3ibpLjv+KNI/axxzXXm7ddgW6cGzB1cS6NeqTCcvU1T5CCCHEIJMAJlZtU0ja6BEOb/IEANJajkUd/9bfdvLyp2U0ugdmRZLLF2TdoVoAZuUnMT236wJ6xfXrmVX1TwCOppwDQEKggezmfb1/QyXMxLr31OfTrupbp4UQQoh+kgAmVkr0XkhtNK3VXjNd+9uPaTQUZyQAUOeK/xRSKKzws9UHafYGsZv1nD0+vcu21uYSlh95GIBPc6/j31N/wbGUxQDkNO/q9Xumeo6T6K8BvQXGL+vfBxBCCCH6SAKYWHW2jBqwFC0A1GmZUxWnD0wAEwyH+clb+1l7sAYNsHJ6DgZd53+dttrtnPvuSixBBzUJE/l4zH8CUJakFnrL7UUAow0HsPmq+OKe/1AP5J8BhthyZ4QQQoh4kSTeWHWyjBogedx8wuiw+Wux+apxmbIAmJxjh50VHK11c/b49H5voNhma0lj5PnM/CSykzoPJoxBF+M2fCfy+oPi7xLSqsX4KuwzAchx7mr/XJ2w1OzgW5uuiD444cK+dl0IIYToNxmBiVUny6gBNMYEWlLUPYFynZ9Fji+ekI7FoMPhCXC8zh11TTAUZmdZE2/trmRLST1KN0HEqXyBELvLHQB8/+LJnDsxo9N2ma4D3Prp5ZibT+A3JvGXOS9SYZ8VOV+bMJEwOqzBJmz+6i7erJnC1TdHHQpnzYAzbu1VX4UQQoiBIAFMrCJTSB2XKZsmXQDArKp/RI5ZDDrmjUkBYEdZUyRICYbCvLWnivWHajlco64i+uvmExyvcxMKq20a3H7+suk4gVD7CqZwWGHN/mo8gRApVgMXTslC08WozvTqVzGH1Oq72xf+hnrruKjzIa2Jequ6eirLdaDjDZqr4dP/396dB0dV5XsA//aS3tLpJemkO0tng0hYAgYCMYA6DhkjMi7gOMpknAwqPphQwuBDQAv9YwZhtB7l8lDUKnDeU0GpAnQoxMcEBDOGBEISCGjYDVsSEJIOS7bu8/7I5EpLEpOQXm74fqq6qvve07d/9wd0/zj3nnPWQN3UPsJpX/R0VORWQDm7ENAau88TERGRD7GA6a2Om3iVNxYwIZkzAQBxrjJEX9cLk5kcDqUCOH3pGirPuAAAhUcv4MRPemQuXW3FZxVn8YfVJfjgXycw5+N9+Oe3dfh873Ekn1yHp/ZMgbF0JU6cb4RSAUxNj4O6i/tewq8ex5Dz/wcAOPyLd3Epcmyn7TouI42vXgXV2VIAgLG5BtHH1gNvjAK2LQEAFDmfQfnwhUgblNCjNBEREfkSC5je6uISEgDAmoBruigAwOMHnoah5QIAwGLQ4PZ/r0y9vaoOa0uqUXG6/RLQr0dGY+6kFDw1MQlWQwgAoLGpDaFaNZ679zYM19ZiQ+tsPHTmv2BqqcN/tPwdazV/xX2DQxFr7Xq+l18d/St07kacDUtDfdwvumx3NOIeAIDt6jGEfXgfhpzfit9U5iN1zwtA2zWp3WlTOhIiDF3O7ktERORPLGB6q4th1B1qx8yXnqfVbpKej4qzoOO3v+7fs/LGWfTSKCWjVo3fjYvHrLuT8crUEfjVMDvGOC34e/j/IEpR7/UZmcrv8Grt01CIzpcosF49iZjGA3Ar1Nic+iqEMqTL06m2ZOKfgxZDoD24+w8vgbWp2qtNQ/IDqAkbAWe4ocvjEBER+RNHIfWWp/sCxjrxaXxxvgWTD7+EtJqNqPO8AAAw6UMw557BONfQhOMXrsCkU2NotMnr/hW1Sgk1gBGxZlgMGuDQ57BdKkOL0oC/DfoAp9wReKx1EyadegvGlvO4t2wODJ6HYb52Gxr0cdJxYl3lAICzYSNxRdP13DAdDjimocY4HI//sBJtNQeha3Phh+i7EDH9PcAUjdNnG6A+fAGDInnfCxERBQcWML3VxUy8Hcz6EDSnPICrx1cgrKUOl08WAGifb0WhUCDGokeMpetLPz9+jgD2fwIAKI9+FMaoJAwFUOn5HYY0FCLOVYa4i0XAjiJMCU3Fx7f/r/TW6MYDAIBzYT1faPG8cQgu53yODwqPwdx0BkmDhuAXpmhpf4xFBxUvHxERUZDgJaTe6mQxx59KdITjoP1BAIDjy2egbbnY+8/ZsgD4bjOAH+9TAQCPUo31ae/hG+cz0jb7le8Qdd0oor4UMB2EQoV6fTyESuu1vbv1lYiIiPyNBUxvdTET7/VusxtxMOaR9maeNsTWfd2rjwi5fAbY8770us6YekObYufT2DL6fbREthcpd514HQCgb7koLdbYlwKmK4kRvP+FiIiCBwuY3upmGHUHg0aNSOcQfG9pX17A0FzXq4+wHtnQ/iRhIoqmFkF01tujUOBc+Fhc+PVquBVqOF2liGsoxaCLOwEAtaFDcU0T3qvP7UqsRY8oE5cNICKi4MECprce+m/g+RPA6D902ywzORyXw9t7QPTNF3p+fOGB5ejG9ucjf4tWfeez7HZwh8XhgP1hAEBW9bvSDbwnwif0/DN/hsWg6bdjERER9QcWML2lCQUM4UBI9/eE2IxaDB7UPvOtvvl8jw6tcjfhN5WzoWs4BmhNwPCpPXpfSdwMtCk0iHOVYdj5LQCAGuPwHr2XiIhIjljA+JDWGgsAMDT17BLSL4+/CqdrX/uLO58DdKYeve+KNgr7HdOk19fUZq81j4iIiAYaDqP2JWsSAMB09fv2YdHdrEStdl/DsLr23pPmsHhos/J79VHfJMyGSrSiVWXAfvs0NKvD+h43ERFRkGMB40u2FAAKaFsboG+91O1NtUmXvoESbjRoY/D9b3dhpKrr2XM706oyYPugRTcZMBERkTzwEpIvhegBayIAIPLK4S6badsacfeJFQCAw7bsbntq/EUX0vUoKyIiokBjAeNrznEAfpzevzNZ1e8irKUOl3RO7HY+7afAupcWZw50CERERF1iAeNrse3LCNiuHu2ySdKlfwEAdiXORZsq8DPeRhg1MGp5dZGIiIIXCxhfM7cvsmhs6XwkUuTlKliaTgMAzphH+y2s7jitnHWXiIiCG/+b7Wv/XhDR+JO5YExNZ3HP8deQfKkQAHAs/K6gGTnkDGcBQ0REwY09ML4WFgMAMLZegK61Xto8+fASqXhpUeqxPfn5QER3A7M+BMm20ECHQURE1C0WML4WaoNH1b6O0NjTfwcAxNcXI6ZxPwBgR9J/4sP0j3FZaw9YiNdLjgyFUhn4UVBERETd4SUkX1Oq0JT+JAx730bG2Q9Rr3fizpNvAgD226eiPOaxAAcIhKgUcHsAtVKBsYn9swAkERGRL7GA8QPDhFnA3rcBANnHlgFoXy36q+TnAhmW5J7UKIRq1AjVqqEIgjloiIiIfg4vIfmDNQG1MZO8Nn2dOAdupTZAAf3IFqbF8BgzEm2hLF6IiEg2WMD4yeWc13HekIImtQm7Ep/FKfPYTttp1Epo1f6bBTctlhPWERGR/PASkp8kOp1Yk/UpGpvaumyjUirwyOg4OMw6aVv0dc/7m16jwoiYnq14TUREFEzYA+MnKqUCQ6M7LxaGRodhTIIV00bHehUvAJBiD8PU9FiofDAyKNURBrWKfwWIiEh+2APjRyl2Iw6caYDbI+DxCIQbNVApFMhKtsFs6Hr16URbKDKTwvHNsR/6NZ5h7H0hIiKZYgHjR1FhOvxqmB1atRJuj0CsRd/jHpDRCVZUnK7HlWZ3v8Ri1ocgKsx3l6eIiIh8idcP/GxQpBFxVgMSIkJ7dfkmRKVEVrKt3+KI53IBREQkYyxgZCQtzow4a/+sVh1tYe8LERHJFwsYmemP+1ZCtSqkRAXHwpFERER9wQJGZgZFGhGiurkRSWMSrNCo+UdPRETyxV8xmdGFqDD4JntPBkey94WIiOSNBYwMDb+Jy0gRRk23Q7aJiIjkgAWMDMVZ9TDr+1aEDI409nM0RERE/scCRoYUCgUyk8P79N4UOy8fERGR/LGAkalUhwnhob3rhYmz6hEZFvgVsImIiG4WCxiZUikVuPu2qF69Z2JK/02ER0REFEgsYGQsPFTT47bDY8yINvfPJHhERESBxgJG5qLNOoTpfn5Jq/gILh1AREQDBwsYmVOrlEh1dD2sWqNWYnAURx4REdHAwgJmAMhItHbZC/PL1ChEGHnjLhERDSwsYAYAXYgKk4baofjJCgO3x1swNPrm104iIiIKNixgBogkWyjuSI6QXisUwNjEvs0VQ0REFOxYwAwgdyRHINbaPtJoXGI4jNqfv7mXiIhIjoK6gFm5ciUSExOh0+mQmZmJkpKSQIcU9O4ZEoWc4Q6MH8w5X4iIaOAK2gLmk08+wfz58/Hyyy9j3759GDVqFHJyclBXVxfo0IJaZJgWw25isUciIiI5CNoCZsWKFZg5cyZmzJiBYcOGYdWqVTAYDFi9enWgQyMiIqIAC8oCpqWlBaWlpcjOzpa2KZVKZGdno6ioqNP3NDc3w+VyeT2IiIhoYArKAubChQtwu92w2+1e2+12O2pqajp9z7Jly2A2m6WH0+n0R6hEREQUAEFZwPTF4sWL0dDQID1OnToV6JCIiIjIR4JynK3NZoNKpUJtba3X9traWjgcjk7fo9VqodVyxlkiIqJbQVD2wGg0GowZMwYFBQXSNo/Hg4KCAmRlZQUwMiIiIgoGQdkDAwDz589HXl4eMjIyMG7cOLz++uu4cuUKZsyYEejQiIiIKMCCtoB57LHHcP78ebz00kuoqanB7bffjq1bt95wYy8RERHdehRCCBHoIHzB5XLBbDajoaEBJhMndiMiIpKDnv5+B+U9MERERETdYQFDREREssMChoiIiGSHBQwRERHJDgsYIiIikp2gHUZ9szoGV3FRRyIiIvno+N3+uUHSA7aAaWxsBAAu6khERCRDjY2NMJvNXe4fsPPAeDwenD17FmFhYVAoFP12XJfLBafTiVOnTnF+GR9gfn2HufUd5ta3mF/fCcbcCiHQ2NiImJgYKJVd3+kyYHtglEol4uLifHZ8k8kUNH/YAxHz6zvMre8wt77F/PpOsOW2u56XDryJl4iIiGSHBQwRERHJDguYXtJqtXj55Zeh1WoDHcqAxPz6DnPrO8ytbzG/viPn3A7Ym3iJiIho4GIPDBEREckOCxgiIiKSHRYwREREJDssYIiIiEh2WMD00sqVK5GYmAidTofMzEyUlJQEOqSgt2zZMowdOxZhYWGIiorCww8/jKqqKq82TU1NyM/PR0REBIxGIx555BHU1tZ6tamursaUKVNgMBgQFRWFBQsWoK2tzZ+nEvSWL18OhUKBefPmSduY2747c+YMfv/73yMiIgJ6vR5paWnYu3evtF8IgZdeegnR0dHQ6/XIzs7GkSNHvI5x8eJF5ObmwmQywWKx4KmnnsLly5f9fSpBxe12Y8mSJUhKSoJer8egQYPwl7/8xWvtG+a253bt2oUHHngAMTExUCgU2LRpk9f+/srl/v37ceedd0Kn08HpdOLVV1/19al1T1CPrVu3Tmg0GrF69Wpx8OBBMXPmTGGxWERtbW2gQwtqOTk5Ys2aNaKyslKUl5eL+++/X8THx4vLly9LbWbNmiWcTqcoKCgQe/fuFXfccYcYP368tL+trU2MGDFCZGdni7KyMrFlyxZhs9nE4sWLA3FKQamkpEQkJiaKkSNHirlz50rbmdu+uXjxokhISBB//OMfRXFxsTh+/Lj48ssvxdGjR6U2y5cvF2azWWzatElUVFSIBx98UCQlJYlr165Jbe677z4xatQosXv3bvH111+LwYMHi+nTpwfilILG0qVLRUREhNi8ebM4ceKEWL9+vTAajeKNN96Q2jC3Pbdlyxbx4osvig0bNggAYuPGjV77+yOXDQ0Nwm63i9zcXFFZWSnWrl0r9Hq9ePfdd/11mjdgAdML48aNE/n5+dJrt9stYmJixLJlywIYlfzU1dUJAGLnzp1CCCHq6+tFSEiIWL9+vdTm22+/FQBEUVGREKL9H6hSqRQ1NTVSm3feeUeYTCbR3Nzs3xMIQo2NjSIlJUVs27ZN3H333VIBw9z23cKFC8XEiRO73O/xeITD4RCvvfaatK2+vl5otVqxdu1aIYQQhw4dEgDEnj17pDZffPGFUCgU4syZM74LPshNmTJFPPnkk17bpk2bJnJzc4UQzO3N+GkB01+5fPvtt4XVavX6Tli4cKEYMmSIj8+oa7yE1EMtLS0oLS1Fdna2tE2pVCI7OxtFRUUBjEx+GhoaAADh4eEAgNLSUrS2tnrlNjU1FfHx8VJui4qKkJaWBrvdLrXJycmBy+XCwYMH/Rh9cMrPz8eUKVO8cggwtzfj888/R0ZGBh599FFERUUhPT0d77//vrT/xIkTqKmp8cqt2WxGZmamV24tFgsyMjKkNtnZ2VAqlSguLvbfyQSZ8ePHo6CgAIcPHwYAVFRUoLCwEJMnTwbA3Pan/splUVER7rrrLmg0GqlNTk4OqqqqcOnSJT+djbcBu5hjf7tw4QLcbrfXlzwA2O12fPfddwGKSn48Hg/mzZuHCRMmYMSIEQCAmpoaaDQaWCwWr7Z2ux01NTVSm85y37HvVrZu3Trs27cPe/bsuWEfc9t3x48fxzvvvIP58+fjhRdewJ49e/Dss89Co9EgLy9Pyk1nubs+t1FRUV771Wo1wsPDb+ncLlq0CC6XC6mpqVCpVHC73Vi6dClyc3MBgLntR/2Vy5qaGiQlJd1wjI59VqvVJ/F3hwUM+VV+fj4qKytRWFgY6FAGhFOnTmHu3LnYtm0bdDpdoMMZUDweDzIyMvDKK68AANLT01FZWYlVq1YhLy8vwNHJ26effoqPPvoIH3/8MYYPH47y8nLMmzcPMTExzC31GC8h9ZDNZoNKpbph9EZtbS0cDkeAopKXOXPmYPPmzdixYwfi4uKk7Q6HAy0tLaivr/dqf31uHQ5Hp7nv2HerKi0tRV1dHUaPHg21Wg21Wo2dO3fizTffhFqtht1uZ277KDo6GsOGDfPaNnToUFRXVwP4MTfdfSc4HA7U1dV57W9ra8PFixdv6dwuWLAAixYtwuOPP460tDQ88cQT+POf/4xly5YBYG77U3/lMhi/J1jA9JBGo8GYMWNQUFAgbfN4PCgoKEBWVlYAIwt+QgjMmTMHGzduxPbt22/ohhwzZgxCQkK8cltVVYXq6mopt1lZWThw4IDXP7Jt27bBZDLd8CNzK5k0aRIOHDiA8vJy6ZGRkYHc3FzpOXPbNxMmTLhhuP/hw4eRkJAAAEhKSoLD4fDKrcvlQnFxsVdu6+vrUVpaKrXZvn07PB4PMjMz/XAWwenq1atQKr1/flQqFTweDwDmtj/1Vy6zsrKwa9cutLa2Sm22bduGIUOGBOTyEQAOo+6NdevWCa1WKz744ANx6NAh8cwzzwiLxeI1eoNuNHv2bGE2m8VXX30lzp07Jz2uXr0qtZk1a5aIj48X27dvF3v37hVZWVkiKytL2t8x1Pfee+8V5eXlYuvWrSIyMvKWH+rbmetHIQnB3PZVSUmJUKvVYunSpeLIkSPio48+EgaDQXz44YdSm+XLlwuLxSI+++wzsX//fvHQQw91Ojw1PT1dFBcXi8LCQpGSknJLDvW9Xl5enoiNjZWGUW/YsEHYbDbx/PPPS22Y255rbGwUZWVloqysTAAQK1asEGVlZeL7778XQvRPLuvr64XdbhdPPPGEqKysFOvWrRMGg4HDqOXkrbfeEvHx8UKj0Yhx48aJ3bt3BzqkoAeg08eaNWukNteuXRN/+tOfhNVqFQaDQUydOlWcO3fO6zgnT54UkydPFnq9XthsNvHcc8+J1tZWP59N8PtpAcPc9t0//vEPMWLECKHVakVqaqp47733vPZ7PB6xZMkSYbfbhVarFZMmTRJVVVVebX744Qcxffp0YTQahclkEjNmzBCNjY3+PI2g43K5xNy5c0V8fLzQ6XQiOTlZvPjii15DdJnbntuxY0en37F5eXlCiP7LZUVFhZg4caLQarUiNjZWLF++3F+n2CmFENdNfUhEREQkA7wHhoiIiGSHBQwRERHJDgsYIiIikh0WMERERCQ7LGCIiIhIdljAEBERkeywgCEiIiLZYQFDREREssMChoiIiGSHBQwRERHJDgsYIiIikh0WMERERCQ7/w+TluAa1PQ+SAAAAABJRU5ErkJggg==\n" }, "metadata": {}, "output_type": "display_data" } ], "source": [ - "result_key = []\n", - "diff_key, result_dict = load_log(\n", + "draw_result(\n", + " task_name=\"ParallelContinuousCartPoleSwingUp-v0\",\n", " log_file=\"rollout.csv\",\n", " log_key=\"ep_rew_mean\",\n", - ")\n", - "\n", - "idx = diff_key.index(('transition', 'oracle'))\n", - "for name in set([key[idx] for key in result_dict.keys()]):\n", - " values = np.stack([value for key, value in result_dict.items() if key[idx] == name])\n", - " plt.plot(values.mean(axis=0), label=name)\n", - " plt.fill_between(np.arange(len(values.mean(axis=0))), values.mean(axis=0) - values.std(axis=0),\n", - " values.mean(axis=0) + values.std(axis=0), alpha=0.5)\n", - " plt.legend()\n", - "\n", - "# for key in result_key:\n", - "#\n", - "# print(re)\n", - "#\n", - "# mean = np.stack(list(re.values())).mean(axis=0)\n", - "# std = np.stack(list(re.values())).std(axis=0)\n", - "#\n", - "# plt.plot(mean)\n", - "# plt.fill_between(np.arange(len(mean)), mean - std, mean + std, alpha=0.5)" - ], - "metadata": { - "collapsed": false - } + " params=dict(freq_rate=1,\n", + " real_time_scale=0.02,\n", + " integrator=\"euler\",\n", + " parallel_num=3),\n", + " group_key=('transition', 'oracle'),\n", + ")" + ] }, { "cell_type": "code", - "execution_count": null, - "outputs": [], - "source": [], + "execution_count": 37, "metadata": { - "collapsed": false - } + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [ + { + "data": { + "text/plain": "
", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjAAAAGdCAYAAAAMm0nCAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8qNh9FAAAACXBIWXMAAA9hAAAPYQGoP6dpAACp+klEQVR4nOzdd3hb13n48e/FJAESAPemRG1qT9vyijxieduJncRZtjOb1E6bnTo7TlI3idPEadO4+bWx49ROnOW9lyxbwxrWHpREcS+QBAEQe9z7+wMEREocWCRA8nyeh49I4OLeA4okXpzznveVFEVREARBEARBmEZUmR6AIAiCIAhCokQAIwiCIAjCtCMCGEEQBEEQph0RwAiCIAiCMO2IAEYQBEEQhGlHBDCCIAiCIEw7IoARBEEQBGHaEQGMIAiCIAjTjibTA5gssizT2dlJfn4+kiRlejiCIAiCIMRBURQGBweprKxEpRp7nmXGBjCdnZ3U1NRkehiCIAiCICShra2N6urqMe+fsQFMfn4+EPkGmEymDI9GEARBEIR4OJ1OampqYq/jY5mxAUx02chkMokARhAEQRCmmYnSP0QSryAIgiAI044IYARBEARBmHZEACMIgiAIwrQzY3Ng4qEoCqFQiHA4nOmhzFhqtRqNRiO2sguCIAhpNWsDmEAgQFdXFx6PJ9NDmfEMBgMVFRXodLpMD0UQBEGYIWZlACPLMk1NTajVaiorK9HpdGKGYBIoikIgEKC3t5empiYWLlw4blEiQRAEQYjXrAxgAoEAsixTU1ODwWDI9HBmtNzcXLRaLS0tLQQCAXJycjI9JEEQBGEGmNVvh8VswNQQ32dBEAQh3cQriyAIgiAI044IYARBEARBmHZEACOMsGXLFiRJwm63Z3oogiAIgjAmEcBMM5s2beKLX/xi1p1LEARBEKaSCGBmmGhxPkEQBEGYyUQAQ+RF3xMIZeRDUZS4x3nnnXfy5ptv8sADDyBJEpIk8fDDDyNJEi+88ALr1q1Dr9fz9ttvc+edd3LzzTePePwXv/hFNm3aNOa5mpubY8fu3buX9evXYzAYuPDCC2loaEjDd1oQBEHIRq39Hg61O+hyeAmE5EwPJy6zsg7M2bzBMEu/+1JGrn303s0YdPH9NzzwwAOcOHGC5cuXc++99wJw5MgRAP7lX/6F+++/n3nz5lFQUJDUuUpKSmJBzLe+9S1+/vOfU1JSwuc+9zk++clPsm3btiSeoSAIgpDNDrbbeeN4L/LQG2pJAlOOlqI8HSV5eory9BTn6Sgw6FCpsqfoqwhgphGz2YxOp8NgMFBeXg7A8ePHAbj33nt573vfm9K5hvvxj3/Me97zHiASHF133XX4fD5RiE4QBGGGUBSFbaf62d1sO+t2cHiDOLxBTve6Y7drVBIFRh3FeTqK8/QsqTCRp89cGCECGCBXq+bovZszdu10WL9+fVrOE7Vy5crY5xUVFQBYrVZqa2vTeh1BEARh6oVlhVeOdnOsazDux4Rkhd5BP72DfmCQAqOOvJK8yRvkBEQAA0iSFPcyTrYyGo0jvlapVOfk1wSDwbjPp9VqY59H+0TJ8vRYFxUEQRDG5guGeeZAJ+0D3kwPJSUiiXea0el0hMPhCY8rKSmhq6trxG379+9P6lyCIAjCzOD0BfnLnrZpH7yACGCmnblz5/LOO+/Q3NxMX1/fmLMil19+OXv27OGRRx7h5MmTfO973+Pw4cNJnUsQBEHInEBITmjH6lisgz4e39VGnyuQhlFlnghgppmvfvWrqNVqli5dSklJCa2traMet3nzZr7zne/w9a9/nQ0bNjA4OMjtt9+e1LkEQRCEzOi0e/n99mZ+t62Z7af6sHuSCz6a+9z8ZU87Lv/MqRMmKekI67KQ0+nEbDbjcDgwmUwj7vP5fDQ1NVFXVyd21UwB8f0WBEFI3IE2O2+e6CUsj3yZrrTksLTCzMKyPHLi2AhyuMPBa8essW3S6XLj6krmT0IS73iv38NN78xVQRAEQZhhQmGZ149bOdLpHPX+TruPTruPLQ1W5pfmUV9hYk6hYdQaLdsb+3jntG2Us0x/IoARBEEQhCzh9AV59kAXPU7fhMeGZIWG7kEaugcx6tUsKTdRX2GiJF9PWFZ49VgPR8cIgmYCEcAIgiAIQhZo7ffw/OEuvIHEd4e6/WH2tgywt2WAUpMerUpFh3367zQajwhgBEEQBCHD9jTb2HaqPy15KlanPw0jyn4igBEEQRCEDAmEZF452sOJnvgr4goRIoARBEEQhAwYcAd49mDnjKnLMtVEACMIgiAIU+x0r4sXj3TjD4oCoslKqJDdfffdx4YNG8jPz6e0tJSbb76ZhoaGEcds2rQJSZJGfHzuc58bcUxrayvXXXcdBoOB0tJSvva1rxEKjSyus2XLFtauXYter2fBggU8/PDDyT1DQRAEQZgkwbCM2x9i0Bfp3mz3BLC5A/S5Ik0PrU4f3Q4fnXYv7QMe2mwetp/q4+kDnSJ4SVFCMzBvvvkmd911Fxs2bCAUCvHNb36Tq666iqNHj45oJviZz3yGe++9N/a1wWCIfR4Oh7nuuusoLy9n+/btdHV1cfvtt6PVavnXf/1XAJqamrjuuuv43Oc+x6OPPsprr73Gpz/9aSoqKti8OTNdo7PZpk2bWL16Nb/85S8zPRRBEIQZzxcM09jr4pTVRWu/h5A8I+vBZr2EApgXX3xxxNcPP/wwpaWl7N27l0svvTR2u8FgoLy8fNRzvPzyyxw9epRXX32VsrIyVq9ezQ9/+EO+8Y1v8P3vfx+dTseDDz5IXV0dP//5zwGor6/n7bff5he/+IUIYARBEIQp5wmEaLS6OWkdpM3mTXtVWyFxKfVCcjgcABQWFo64/dFHH6W4uJjly5dzzz334PF4Yvft2LGDFStWUFZWFrtt8+bNOJ1Ojhw5EjvmyiuvHHHOzZs3s2PHjjHH4vf7cTqdIz4EQRAEIVkuf4h9rQP8ZU8b/29rE68e66Gl3yOClyyRdAAjyzJf/OIXueiii1i+fHns9o985CP83//9H2+88Qb33HMPf/jDH/jYxz4Wu7+7u3tE8ALEvu7u7h73GKfTidc7emGe++67D7PZHPuoqalJ9qllNbfbze23305eXh4VFRWxWaoov9/PV7/6VaqqqjAajZx//vls2bIldv/DDz+MxWLhpZdeor6+nry8PK6++mq6urpix2zZsoXzzjsPo9GIxWLhoosuoqWlJXb/U089xdq1a8nJyWHevHn84Ac/OCeHSRAEYTpyeIPsbbHx+O5W/uet02xp6KV9QMy4ZKOkdyHdddddHD58mLfffnvE7Z/97Gdjn69YsYKKigquuOIKGhsbmT9/fvIjncA999zDl7/85djXTqcz/iBGUSDomfi4yaA1gHRu/4qxfO1rX+PNN9/kqaeeorS0lG9+85u8++67rF69GoC7776bo0eP8qc//YnKykqeeOIJrr76ag4dOsTChQsB8Hg83H///fzhD39ApVLxsY99jK9+9as8+uijhEIhbr75Zj7zmc/wxz/+kUAgwK5du5CGxvjWW29x++2386tf/YpLLrmExsbG2P/59773vfR+bwRBEKZQICTzyPZmkdMyTSQVwNx99908++yzbN26lerq6nGPPf/88wE4deoU8+fPp7y8nF27do04pqenByCWN1NeXh67bfgxJpOJ3NzcUa+j1+vR6/XJPJ1I8PKvlck9NlXf7ASdceLjAJfLxf/+7//yf//3f1xxxRUA/P73v4/9H7S2tvLQQw/R2tpKZWXk+Xz1q1/lxRdf5KGHHoolSQeDQR588MFYQHn33XfHkq6dTicOh4Prr78+dn99fX1sDD/4wQ/4l3/5F+644w4A5s2bxw9/+EO+/vWviwBGEIRprdXmFsHLNJJQAKMoCl/4whd44okn2LJlC3V1dRM+Zv/+/QBUVFQAsHHjRn784x9jtVopLS0F4JVXXsFkMrF06dLYMc8///yI87zyyits3LgxkeHOOI2NjQQCgVhQCJH8o8WLFwNw6NAhwuEwixYtGvE4v99PUVFR7GuDwTBiNqyiogKr1Ro735133snmzZt573vfy5VXXskHP/jB2P/fgQMH2LZtGz/+8Y9jjw+Hw/h8Pjwez4gdZ4IgCNPJKas700MQEpBQAHPXXXfx2GOP8dRTT5Gfnx/LWTGbzeTm5tLY2Mhjjz3GtddeS1FREQcPHuRLX/oSl156KStXrgTgqquuYunSpXz84x/npz/9Kd3d3Xz729/mrrvuis2gfO5zn+M///M/+frXv84nP/lJXn/9df785z/z3HPPpfnpD9EaIjMhmaBN3wu+y+VCrVazd+9e1Gr1iPvy8vLOXFKrHXGfJEkow9Z3H3roIf7pn/6JF198kccff5xvf/vbvPLKK1xwwQW4XC5+8IMf8P73v/+c6+fk5KTtuQiCIEwlWVZo6hMBzHSSUADzm9/8BojUHRnuoYce4s4770Sn0/Hqq6/yy1/+ErfbTU1NDbfccgvf/va3Y8eq1WqeffZZPv/5z7Nx40aMRiN33HHHiLoxdXV1PPfcc3zpS1/igQceoLq6mv/5n/+ZvC3UkhT3Mk4mzZ8/H61WyzvvvENtbS0AAwMDnDhxgve85z2sWbOGcDiM1WrlkksuSelaa9asYc2aNdxzzz1s3LiRxx57jAsuuIC1a9fS0NDAggUL0vGUBEEQskKH3YsvmHgXaCFzEl5CGk9NTQ1vvvnmhOeZM2fOOUtEZ9u0aRP79u1LZHgzXl5eHp/61Kf42te+RlFREaWlpXzrW99CpYpsJlu0aBEf/ehHuf322/n5z3/OmjVr6O3t5bXXXmPlypVcd911E16jqamJ3/72t9x4441UVlbS0NDAyZMnuf322wH47ne/y/XXX09tbS233norKpWKAwcOcPjwYX70ox9N6vMXBEGYLI29rkwPQUiQ6IU0zfzsZz/D5XJxww03kJ+fz1e+8pVYPR6IzIb96Ec/4itf+QodHR0UFxdzwQUXcP3118d1foPBwPHjx/n9739Pf38/FRUV3HXXXfzDP/wDEKnH8+yzz3Lvvffyk5/8BK1Wy5IlS/j0pz89Kc9XEARhKjT2zvLlI0UGKaXScFNOUiaaVpmmnE4nZrMZh8OByWQacZ/P56OpqYm6ujqRtzEFxPdbEIRsZh308ejO1kwPY0powl4Kvc0UeZoo9DZR6GmiyNOEyd/BkdIbeW3BN+M+142rK5lfkjfxgQka7/V7ODEDIwiCIMxqp7N49kUXcnFp0y/Rhd0ENEYCaiMBdR5+TeTfgNqIX20koBn2udqICpnC4UHK0L9mf9eY11rR8yS7qj/BYE7FFD7D5IkARhAEQZjVsjn/5YK2/8cK61NpPadHW0B/bh02Qx223Dpshrmc3/a/VDv3saLnSbbP+XxarzdZRAAjCIIgzFpOXxCr05/pYYzK7G1nddefAdhVfSdBVQ76sBtdyI0u7EIXdg99PfJzFTIAg7pSbIa6EcFKv6EOn9ZyzrX0oUGqnftY3vMkO2s+jazSnnNMthEBjCAIgjBrZfPy0cUtv0athGi2XMC2OXfF9yBFQSP7kFAIquOvM9ZYuAm3tghjsJ8Fti2cKH5vkqOeOtMr5VgQBEEQ0qjRmp3LRxXOgyzqfxUZFVvn/nP8D5QkQurchIIXAFml4VDZTQCs7PpbQo/NlFkdwMzQDVhZR3yfBUHIRr5gmPYBb6aHcS5F4dLmXwJwpPQG+o1TUzj0cPn7kFFR49xLgad5Sq6ZilkZwERL6Xs8GepAPctEv89ntzAQBEHIpKY+N3IWvsFa2P86lYOHCKpy2FH7D1N23UF9OU0FFwGwsjv7Z2FmZQ6MWq3GYrHEGhgaDAYkScrwqGYeRVHweDxYrVYsFss5/ZkEQRAyKRvzX1RykItb/gOAPVUfx60vmdLrH6y4hfkDb7HU+hzb5txFSJ29tbtmZQADUF5eDhALYoTJY7FYYt9vQRCEbBCWFZr7sy+AWdX1Fyy+DlzaYvZUfXzKr99s2YhDX4nZ38mivpc5WnbjlI8hXrM2gJEkiYqKCkpLSwkGg5kezoyl1WrFzIsgCFmn1eYhEJIzPYwR9EEHF7T/LwDb53yOkDp36gchqThU/j4ubvk1q7r/JgKYbKZWq8ULrCAIwixzOguL153f/jtyQk76DPM5Whpf/7rJcLj0Rja2/jflrqOUuo5hzavP2FjGMyuTeAVBEITZS1GUrMt/MfvOFK3bOvefUaTMvbH26go5WXQFkN1bqkUAIwiCIMwq3U4fLn8o08MY4aLmM0XrWgo2Zno4HCy/BYAlfS+hDw1meDSjEwGMIAiCMKs0WrNr9qXCeZDF/a+iIPHW3H/K9HAA6DCtps8wD63so976fKaHMyoRwAiCIAizyum+LMp/URQubX4AiBSt6zMuzPCAhkhSbBZmZfffIAvr5YgARhAEQZg1BtwB+l2BTA8jZkH/G1QOHiSoymH7FBati8exkmsJqnIo8jZR5Xw308M5hwhgBEEQhFmjMYt2H6nkIJcMFa3bW/Ux3PrSDI9opIAmj+MlVwOwsvvvGR7NuUQAIwiCIMwa2bT7aFX3X7H42nFrCzNStC4e0WWkhf2vYwj0Z3g0I4kARhAEQch6Dk+Q/W32lM7hCYTodGRH80Z9yMn5bUNF62o/l3D36KlizVtCV94y1EqIZdanMz2cEUQAIwiCIGS9t0/1saXByilr8lt6T/e6syYX9by235EbctBnmMeRshsyPZxxRWdhVnQ/gaSEMzyaM0QAIwiCIGS1LoeXEz2DKAq8cKibTntysyjZkv8yvGjdW3P/CUXK7qL4DcXvxafOx+zvYs7AzkwPJ0YEMIIgCEJK/KEwVqdv0s7/1sm+2OchWeHpA50MuBPbSRQIybT2e9I9tKRc1PJfaJQgLZbzabZcmOnhTCiszom1NljV/dcMj+YMEcAIgiAIKWm0unnlWA+ynP71mVNWFx0DI2dcvIEwT+zrwBOIv5puq81NaBLGl6jywUMs7nsFBYmtc/8JJCnTQ4rLwfL3A1A3sI18X1eGRxMhAhhBEAQhJSetg1idft5tHUjreWVZYdupvlHvc3iDPLW/k2A4vo7Sp7Kk+u7G1t8CcKT0evqMizI8mvgNGObSal6PhMKKnicyPRxABDCCIAhCCnzBMC1DSzM7T/dj96SvSNyhDge2cZaKuh0+nj/UNeHMjywrNPdnRwBT4j4BwIGKD2R4JIk7WH4rAMt7nkIlBzM8GhHACIIgCCk4ZXURHgoggmGF145Z03JefyjMztMT1x053evmjYbxr9lh9+INZH73jEoOYQzaABjUl2d4NIlrLHwPbm0RxqCNBbYtmR6OCGAEQRCE5J3oGbmtudXm4XCHI+Xz7m0ewBNn0HGw3cGuJtuY92fL7iNDMLIcFpY0eDXmDI8mcbJKw6GymwBY2fW3DI9GBDCCIAhCkjyBEG22c7c0v3WyD7d/nARbWxM8dhu0jr4l1+UPJZxPs72xj2NdzlHvy5bqu3mBXgDcumKQpufL76Hy9yGjosa5F+3AqYyOZXp+BwVBEISMO2V1IY9SGc4XDPPmid6xH7jtl3DiBXjy8xA+N9DZfqqPYDixHUOKAq8c7aHNNnKrdO+gH4c38/kacCaAcelKMjyS5Ln05TQVXgyA6fAjGR2LCGAEQRCEpDR0j10Vt6F7kNOjLd3IMjS8EPncdhoO/mnE3X0uP0fHmEmZSFhWeOZgJ30uf+y2bFk+AjAGIktI7mkcwAAcGKrMm9/wVwhkrraOCGAEQRCEhLn8ITomqIj7+nEr/tBZeSwde8HVc+brN38CoTM7jd4+2ZdSuX9/UObJfR0M+iKzLtkUwMyEGRiAFssFOPSVqP0OOJy5XBgRwAiCIAgJOzlU2n88g74Q20+dtZPo+LORfxdfC3llYG+F/f8HQGu/h6a+1PNVBn0hntofmYmxOv0TP2CKGIfnwExnkop9FR9icOFNULYsY8MQAYwgCIKQsLN3H43lQLt9ZO+ihucj/664FS7+cuTzrfejBH28dWqcvJkE9Q76+eve9rSdLx3y/DNjBgZgX9VHsF71X1C1NmNjEAGMIAiCkBCnL0iXI77eR4oCrx3ridSK6T0BfSdApYUF74V1d0J+JTg76H7jt2mfLcmG2i/DzZQlpGwhAhhBEAQhIfEsHw3X5wqwu9kGDc9Fbqi7FHJMoM2BS78CgGnPA6jDk9cQMhvMlCTebCECGEEQBCEhDd2JJ8buarIRPDqU/7Lk2jN3rLkdv7ESY6CPld1/T9MIs48m7CMnHFl2EzMw6SECGEEQBCFudk+AHmfiMyU5vl40nXsjXyw+E8B4ZTXbqz4JwIaO36MJj7+zabqKJvAGVTkE1MYMj2ZmEAGMIAiCELcTPcltS55n24qEgrtkNZgqY7fvbOrnYNF12HOqMAZtrO76c5pGml3O5L+UgiRleDQzgwhgBEEQhLg1xLn76GzzbFsBOGS8KFajxe4JcKjdgazSsLPmMwCs7/gDulD21G5Jl2j+i2u6b6HOIiKAEQRBEOJicwfoG0x8p5A25KbWvguAhoJLeaMhMhux7VR/rJP18ZLN2HJqyQ05WN31ePoGnSXO9EES+S/pIgIYQRAEIS7jtQ4Yz1z7DjRKkIGcWmy5dTRaXbx1sndELRlF0rCz9rMArOt4FH0ouWtlq7yAFRAzMOkkAhhBEAQhLvEWrzvbfNubADQWXhrL/9jTfG636RPFV9JnmEdOeJC1nY8lP9AsJLZQp58IYARBEIQJ9Q76sbkDEx94FpUcos72NgCNRZvGPVaR1OysiczCrOn8IzlBe8LXy1aiiF36iQBGEARBmFCysy/Vzr3khF24tYV05S+f8PiTRZdhNS5EH3azrvPRpK6ZjcQMTPqJAEYQBEGYULIBTHT30emCS1Ak9cQPkFTsqPkHAFZ3Pk5u8NylpmlHUcQMzCQQAYwgCIIwrh6nD7snmPgDFYUF/VsAaCx6T9wPO114Kd159ehkL+vbf5/4dbOMPuxCK0eK/4kk3vQRAYwgCMIMFpYVPIFQSudIdvdRqfs4+QErAVUureYN8T9QkthROzQL0/3X2PLLdBWtwuvTmAirczI8mplDBDCCIAgzWKvNw593t+HwJjGDAiiKkvzuo/7I7qOWggsSfuFutlxIZ/4KNLKfDe0PJ3X9bHFm+UjMvqSTCGAEQRBmsEariwFPkD/vbqM3iSJ0XQ4fg77kZnDObJ/elPiDh83CrOj+O3n+nqTGkA1EAu/kEAGMIAjCDKUoCo29kbL8Ln+Iv+xto8OeWLPEZFsHmH3tlHhOIaPmdMFFSZ2j1Xwe7aY1aJQg57U/lNQ5soEoYjc5RAAjCIIwQ3U6fHgC4djX/qDME++2x4KaiSiKwqlkmzf2R3YftZvX4NeakzoHksT2oVmY5T1Pke/rSu48GXamD1Jphkcys4gARhAEYYY6ZT03+AiGFZ490MWRTseEj28f8OLyJ7d8tMC2BYDGwvh3H42mw7yOVvMG1EqI89v+N6VzZcqZPkhiBiadRAAjCIIwQzWOEsAAyIrCy0d62NNsG/fxySbv5gTtVDoPRMaQYgADsL32cwAssz6LydeR8vmm2pkZGJEDk04JBTD33XcfGzZsID8/n9LSUm6++WYaGhpGHOPz+bjrrrsoKioiLy+PW265hZ6ekclXra2tXHfddRgMBkpLS/na175GKDQyyt+yZQtr165Fr9ezYMECHn744eSeoSAIwixkHfRNuPPorZN9bD3Ri6Io59wny8qoMzjxmGd7CxUyVuMiBnMqkjrHcF2mlTRbLkBFmHUd0686b54/kgMjknjTK6EA5s033+Suu+5i586dvPLKKwSDQa666ircbnfsmC996Us888wz/OUvf+HNN9+ks7OT97///bH7w+Ew1113HYFAgO3bt/P73/+ehx9+mO9+97uxY5qamrjuuuu47LLL2L9/P1/84hf59Kc/zUsvvZSGpywIgjDzxRt87G0Z4KUjPcjyyCCmbcAzIn8mESntPhrDnqrbAVhmfXp6VedVZIxBMQMzGSRltNA7Tr29vZSWlvLmm29y6aWX4nA4KCkp4bHHHuPWW28F4Pjx49TX17Njxw4uuOACXnjhBa6//no6OzspKysD4MEHH+Qb3/gGvb296HQ6vvGNb/Dcc89x+PDh2LVuu+027HY7L774YlxjczqdmM1mHA4HJpMp2acoCIIwLf1hZwt9CWybnldi5NoVFWhb34YDf2Jn0c3s8M1N+LqasI/P7boSreznD6sfpc+4KOFzjEpR+PDBOyh3HWNn9afYMedz6TnvJMsN2Pjc7s0oSPxq43ZklSbTQ0qbG1dXMr8kL+3njff1O6UcGIcjkgRWWFgIwN69ewkGg1x55ZWxY5YsWUJtbS07duwAYMeOHaxYsSIWvABs3rwZp9PJkSNHYscMP0f0mOg5RuP3+3E6nSM+BEEQZiO7J5BQ8ALQ2dlB+0N3wu9vgP2PsnbLHVQ53k342rX2d9DKfhz6CvoMCxN+/Jgkid1VdwKwuvsvaMOe9J17EkUTeD3aghkVvGSDpAMYWZb54he/yEUXXcTy5ZEOo93d3eh0OiwWy4hjy8rK6O7ujh0zPHiJ3h+9b7xjnE4nXu/oNQzuu+8+zGZz7KOmpibZpyYIgjCtxbtNGgBFYWnP09z57q3UtT+FgoTfNBdd2MP7jv4zNfZdCV17xO4jSUrosRNpLHoPAzm15IScLO9+Mq3nniwigXfyJB3A3HXXXRw+fJg//elP6RxP0u655x4cDkfso62tLdNDEgRByIh4818KPM3cevhzbD71Q3JDDnoNC/nTyt/x30v/QJNlI1rZx83Hvsycge1xnU9SQsyzvQVAY9GmZIc/JkVSs6fqYwCs63wUlZxce4SpFC1iJxJ40y+pAObuu+/m2Wef5Y033qC6ujp2e3l5OYFAALvdPuL4np4eysvLY8ecvSsp+vVEx5hMJnJzc0cdk16vx2QyjfgQBEHIRoqi0OXwsqOxH18wuUTZsbj9IbocvnGPUct+NrY8yMf3f5ga57sEVTlsnftPPLbqEbrzlxNW5/BM/f00FlyCRvZz47GvMs+2dcJrVzoPkhty4NWY6TCtStdTGuFY6bW4tUXkB6ws6cv+jR15YgZm0iQUwCiKwt13380TTzzB66+/Tl1d3Yj7161bh1ar5bXXXovd1tDQQGtrKxs3bgRg48aNHDp0CKvVGjvmlVdewWQysXTp0tgxw88RPSZ6DkEQhOnGFwxzvNvJi4e7+O3W0/xpVxs7T/dzsH3ignKJaOx1Md7WjBr7Lj6+7yNc0P6/qJUQpwsu4pE1j7O36uMjcjTCKh3PLvkJJ4suR6MEuf7411nQ9/q4147uPmoquBhFmpx8j7BKz7uVHwFgffsjoMiTcp10McaK2IkAJt0S+gm76667eOyxx3jqqafIz8+P5ayYzWZyc3Mxm8186lOf4stf/jKFhYWYTCa+8IUvsHHjRi644AIArrrqKpYuXcrHP/5xfvrTn9Ld3c23v/1t7rrrLvR6PQCf+9zn+M///E++/vWv88lPfpLXX3+dP//5zzz33HNpfvqCIAiTx+r00dTnprnfTbfDjzxKZLGvdYC1tRY06vTUFR0r/yU3OMClTb9kae/zALi0xWyZ91VOFl0+Zq6KrNLy3OIfc/WJ77Ok7yWua/gmLyrfp6Hk6nMPVpRYAHOqKPXideM5WP5+zmv/HUXeJubZ3uL0JF8vFaIT9eRJKID5zW9+A8CmTZtG3P7QQw9x5513AvCLX/wClUrFLbfcgt/vZ/PmzfzXf/1X7Fi1Ws2zzz7L5z//eTZu3IjRaOSOO+7g3nvvjR1TV1fHc889x5e+9CUeeOABqqur+Z//+R82b96c5NMUBEGYfP5QmNZ+D019blr6PXGV4fcEwhzpdLKqxpLy9X3BMG22szY6KDLLrM9wSfN/kBtyoCBxoPxWts35RwKaibfAKpKGFxf9gLBKyzLrs1xz4ruo5SBHy24YcVyRpxGLr4OQSk+L5YKUn8t4Apo8DpbfwoaOR9jQ8UhWBzBiBmbyJBTAxFMyJicnh1//+tf8+te/HvOYOXPm8Pzzz497nk2bNrFv375EhicIgpAx1kEff3+3A28Sxd/2tgywosqMSpXarp3mfjfhYQXptGEPNxz7GnMckZ1EVuNCXp3/LXrylyV0XkVS8/KC7xCWtKzseYLNp+5FrQQ5VH6mSGl091GL+TxC6tFzFdNpX+WHWdP5RyoHD1Lp3E+nafWkXzMZZ2ZgRACTbqIXkiAIQoq6HT7+tje54AXA4Q1ywppc36Hhhu8+Uod93HT0y8xx7CKoyuHNuf/MY6seSTh4iZFUvDb/HvZVfAiAKxvvY3Xn47G75w91n26cotkQt66Yo6XXA7Ch/fdTcs1EqeQQhqGqwSKAST8RwAiCIKSgw+7lb++2p7ybaE9zauXxQ2GZlv5IcTe1HOCG41+nxrkXv9rIX5Y/yLtVH0s9sVaS2FL3FfZURrYyX9Z0P+s6/kCev5sy9zEUJE4XXJLaNRKwt+pjKEjMG3ibIvepKbtuvAzBfiQUwpIar9aS6eHMOCKAEQRBSFKbzcOT+zoIhFLfCdM76Ke5zz3xgWNosXkIhGRUcohrG75FnX0HQZWep+p/kfysy2gkibfm/hM7qz8JwKXNv+K6hm8C0GlahVdXmL5rTcCeWxtJQgbWd/xhyq4br+jykVtbDJJ4uU038R0VBEFIQnOfm6f2pyd4idrdbEv6saesLiQlzOaT32eBbQshScfT9T+nw7wmbeOLkSR2zPk822oj/YgqBw8B0Fh4afqvNYFok8clvS+R7+ua8uuPJ5rA69KXZngkM5MIYARBEBLU2Ovi6QOdBMNJ98IdVfuAly7H6O1SxiPLCk29g1x56scs6XuJsKTm2SX/Rqvl/LSO72y7aj7F1jn/FPv6VBq7T8erJ38preb1qAizrvPRKb/+eGIzMGIL9aQQAYwgCEICTvQM8tzBrhG7fdJpdxK5MB0DHi44/hOWW59BRsULi35MU+HU5KLsrf44Ty/5Gc8t/lccuZnpQbe7+k4Alvc8RU7QnpExjEb0QZpcIoARBEGI07EuJy8c6p604AXgdK8LmzsQ/wMUBeWV77C6+y8oSLy08HucLL5i0sY3msaiTZwofu+UXnO4VvN59BgXo5V9rO76S8bGcTYxAzO5RAAjCIIQh8MdDl460j1qNd10UpQEc2G2/Bu1x/8XgNfm/wvHS6+dpJFlMUmK5cKs7nocTTjxZbjJcKYGjMiBmQwigBEEQZjA/jY7rx7rGbfHUDo1dA8y6Iuj0/Lbv4A3/w2ALXVfGlFYbrY5WXw59pwqckMOlvc8lenhAMOr8IoZmMkgAhhBEIRx7G2x8cZx65QFLwBhWeHdVvv4B73z3/Dq9wF4e84/sm+oweFspUga9lZ+HIB1nY+ikidu4zDZRCfqySUCGEEQhDG8c7qfrSf6MnLtwx2OsYvjvfsIvPB1AA7O+wy7qz8xhSPLXkdKr8OtLcTk72ZR38sZHYs67CMn5AREH6TJIgIYQRCEs7TZPDyxr53tjf0ZG0MgJLO/zX7uHQf/Ak9Hti5713+e18o/M7UDy2JhdQ77Km4DYEPHI0zptNlZorMvQZUev3ripplC4kQAIwiCQKSWSkP3II+908pf97bT3OfJ9JDY32YnGB4qlBdww9b74Yl/ABRY/0kOLf0aSKk1gJxpDlbcQkBloNjTSN3AtoyNY0QCr/g/mhQpNsYQBEGY3oJhmSOdTt5tGcDhjSNxdgp5A2GOtPay2voUbP0ZuK2RO1Z9BK79OY172jM7wCzk15g4WP5+1nf+H+s7HqGp8OKMjEMk8E4+EcAIgjAreQNh9rUNcLDdkXQX6ckkKWHqe19g/t7/B77OyI2WOXDZN2HFBxkMhOlx+jI7yCz1buWHWdP1J6qd+6hwHqDLtGrKxyCK2E0+EcAIgjCr2D0B3m0d4GinM+2tANJCUVhge4MLWx6kyNsEQNBQivayb8Ca20GjA6Cx15nJFI+s5taXcqzkWpZbn+aaE9/lyaW/wGaYN6VjyAtEZsvEDMzkEQGMIAizgtXpY3fzAKesrkkvRpcURaHWsYuLWn5NuesYAF6Nmd1Vd9C+8CN8eP0SpGG5FKesrkyNdFrYUftZqp17sfg6uO3gJ3l28b/RWnDBlF3/zBZqUcRusogARhCEGS0Yltl2qo/9bfasnbGocB7kopb/osa5F4CAKpd3Kz/C3qqPEdDkgRea+tzMK4nsZvEFw3QMZEe12Wzl0pfxx5UPc+Pxr1Hl3M/7jn6RN+Z9lYMVt07J9WOdqMUS0qQRAYwgCDNWp93Ly0e6GfBkV3JuVJGnkYuaf838gbcACElaDpbfyq7qO/HqCkccu6d5IBbANPZm6SxSlvFpLfxt2a+58tS/srT3Oa44/RMKvc28WfdFFGlyX/6iMzDZsoSkKAqyEimSGFaUyL/DPxQFjUpCr1Gh16jRqqURM37ZSAQwgiDMOKGwzI7T/extGcjaWZd8Xxe3HfgEOtmLjJojZdfzTs2nGdSXj3p8h91Lh91LlSWXxl73FI92+gqrdLy08HvYDHO4uOW/WNP1OBZfG88v+nFkdmsyKErGZmD6XH7ebRmg3e4lFB4ZrCRCgkgwo1Wj16jQaVSx4EY/9HlIlnn/2mrml2Smzo0IYARBmFG6HT5ePtpNvyuBjs4ZsLHtv9HJXnqM9byw6F4GDHMnfMyeZhulKypo7RcBTEIkid3Vn8CeU8vmk9+jbmA7Hzr0aZ6q/3ecOZVpv5wu7EYnR5b4pqIKr6IodNp97Gmx0dwfX/0iSQK1JKFWnfkIhRX8oTCyAgrgC8n4QvKY59jZZGNRWb4IYARBEFIRlhV2nu5nT/NA1i+vFLlPsdT6PACvzf9GXMELRPJg9rYMZOfuqWngZPEVOPXl3HTsKxR7Grnt4Cd4ZsnP6DKtTOt1okXsfOp8QuqctJ57OEVRON3nZk/zAN3DttQvKM1jRZWZXK0ajWpkkBL9UI2xPKQMzdj4Q/LQRxh/cNjnw24vztMzp8g4ac9vIiKAEQQhrbY0WCk357Ck3DRl17Q6fbx0tIe+Qf+UXTMVF7X+BgmFk0WX05O/LO7HKQrsPJ259gYzQU/+Mh5b9TA3HfsKpe4T3Hr487y88Ls0lGxO2zUmu4hdSJZp6B5kb8tALL9LrZKor8hnbW0BBQZd0ueWJAmNWkKjVmHUj3/sjasrMzb7AiKAEQQhjZr73OxrtaOSJBQF6ismN4gJywrvNPWzuyn7Z12iKpwHmG/bioyKbbWfT/jx0+RpZjWXvpw/r/h/XHPiO8y3beXaE9+mwNvCzprPpKXs/2R1ofaHwhzucLKvbQC3P1J8UadRsbLKzOoaC0b97HpJn13PVhCESeMNhHnlaA8AsqLw0pFuFAWWVk5OENM76Oflo91YndNj1gUAReHill8DcKTshriXjoT0C6oNPLPkp1zc/J+s7/w/Nrb9Pwq8Lby84DuEU1z2iRaxc+nTE8C4/SH2t9k52OEgMJSTYtSrWVNTwPIqE3qNOi3XmW5EACMIQlq8eqwHlz8U+1pR4OWj3SgoLKs0p+06iqKwt2WA7Y39Ce+syLS5A9updu4jJOki7/aFjFIkNW/V/TMDuXO4/PS/saTvZfL9Pfx1+YPIquRfHo2xLdSpBzAnegZ55WgPoaGf9QKDlnVzClhcno9GNbv7MYsARhBmkaOdTqoKcjHnatN63sMdjlErwyoKvHK0B0WB5VWpBzEuf4iXDnfTapvkTtGKzOK+l3HpSukwr03bOaOzL/srPohLX5ae8wopO1x+M/acam48/lWqBg9Q7diTUtXevDRsoY4G6tsaIzlP5aYc1s8tYF6xMevrs0wVEcAIwiwx6AvyRoMVo07NBzfUYNCl59ff4Qny5oneMe9XlMjsjKLAiurkg5jTvS5ePtoz6Y0X1WEfm0/dy+K+VwhLav627Dd0mNekfN4lvS9R4jmJT53Hruo7Ux+okFbtlvWcLryE+t4XqRw8NGEAIysKVqef3kE/VQW5FBrPJM6mmsQrywpvnLByuMMJwJoaCxcvLB5z59BsJQIYQYjD6V5XrArqdPXmiV4CIZlASOaJfR3cuq465bVzWVZ48UhXbF1+LIoCrx3vQUFhZbUloWuEwjJvnepjf6s9+YHGKTc4wI3HvkLl4CEA1EqY6xr+hUdX/QG3PvmeNio5yIWtDwKwp+p2/Nr0LakJ6dOVv4L63hepGDx4zn2KomD3Bmm1eWizeWgf8OIf+rlXSXB+XRHr5hSgVknDZmAS/5kJhGSeP9xFy1A9l/csKmF1jSX5JzWDiQBGyCr9Lj9dDl9alhvSQVEU3miwcrTTyT+8Zz5a9fRcc27qc3Oy58wSj9Xp55kDXdy8uhJNCs9pd7ONTrtv4gOJBDGvH7eiKLAqzj/I/S4/zx/unpLt0YWeJm4++kXM/k586nxeXHQvF7X8mhLPKW44/g3+suK/CauS2566oucJzP5O3Noi9lXeluaRC+nSlR+pB1M+eBgUGXdApm3AMxS0eEfkeEFkB5A5V0vvoJ8dp/s51eviqiUlsRwYV4IzMC5fiKcOdNDnCqBRSVy9vDyj25SznQhghKwhywovH+2hx+lDVhJ/pz454+nmWNcgAK02z6T+MWnoHqS6IDftWyGDYZk3jlvPub3N5uGFw91ct6IClSrxqekep4+dp20JPSYaxMiKwpragnGPPdTu4M0T1ikp2lZj3831x79OTtiFPaeKJ+t/yYBhLrbcuXzkwO1UuA6z6fT9vLbgmwmfWxv2cH7b/wKws+bThNS56R7+rLCr2cagL8hli0snbSmlK2cefimHnLCL7Tu3s9szMk9JLUlUWHKoLTRQU2igNF+PBDT0DPJmQy+9g35e2nOUH+sjgY5HG38A0zvo5+kDnbj8IQw6NTesqqTcNHlF8GYCEcAIWWNv6wDdjsi7+deOWQnLE7/ITZZQWOa5Q12cHtZz5nSve1IDmAPtdg6227llbXVSAcVYdjfZcHhHb2Z4yuriteNW3rs0sYTSYFjmxcPdSdde2dLQiwKsHeX/1xcM8+qxnhEzRpNpac/TXNn4r6iVMJ35K3m6/n682si4HLnVPL/4R7zv6BdZ2fMEPXn1HC5/X0LnX9v5GMagDXtONYfLbp6EZzDz9Q762TGUzLqk3ESVZXKCwL8f6OGi0Dw2qo8yz3+U3ZRRmq+nptBATUEulZbcUWdhl5SbqCkw8EaDFV1fCwD9mOlxhynJn/hltqXfzfOHugmEZQoNOm5aXYkpzYn2M5EIYISs0O/ys7NxZIXRLQ29yIrCujmFYzxqcvhDYZ7e30n7gHfE7U19LhSldFJ2AHgCITrtXhQFdpzu56IF6angaXMH2NMyMO4xhzscGHTqhK751slebO7Ueg292dCLctb/b/uAhxcPdzPoC43zyDRRZC5q/Q3ntT8MwPHizby88DuEVSPLj7YUXMi2OZ/n4pb/4rLTP6PPuIDu/BVxXSInaGddx/8BsK328yltzZ3N9rScmemLNrRMt8jvoI93NQvYyFFuKmzDvGQeubr48sSMeg3XrahAe/oodEO3bOFPu1tZP7eQ8+YWoh7jTcnhTkdsabXakst1KyvI0c7Oui6Jmp4L+sKMIsvKiDoHw2090ceupsSWKVLhDYT5+7sd5wQvAG5/eES/kXQ63euOVVjd3WzjdG96Zh9eO9YTV62UXU029k4Q6EQ19bk50OZIdWhA5P93T7MNWVbY3tjH3/Z2TEnwog77uLbhW7HgZWf1p3hh0Q/PCV6idlfdycmiy9AoQW44/g0MQzkOEzmv/SH0YTc9xsWcKL4yXcOfVeyewIjZuM5RfjfTocMeOe9J3VIAFoeOxx28REmSxBJjZKyDuhJkJfK79cfdrfSc9bdDUSI/868diwQvS8rzuXlNlQheEiACGCHj3m0doMsxdmCw7VRfbPp4Mrn8If6yty22jDWa4UtK6TS8hoqiwEtHesZc9onX0U7nqIHYWN462cvRTue4x3gCIV452p3SuM69bh+P7GjmndO2KWkHkBuwceuRf2Rx/6uEJQ0vLfweO+Z8bvwS8pLESwu+R39uHXmBXq5vuAeVPP7/T76/m1VdfwFg25y7QJq5f24PtTvY02xDmYT/v70tAyiAZWhJpdPhRZ6EAoYdQ78rvZZIIm+Rtxl9MPFAPbqF2lhUzbXLy8nVqul3BXh8TxvbG/sIyTIhWealIz3sbo68aTi/rpCrlpaNOUsjjG7m/kYJ00K/yx9XcLLzdD/bTsX3rjcZdk+AP+9uo981/rJIumZGhvOHwrSdVZjNFwzz/KGupCvN+oJh3jo5dm2W0USLzo33HF89Zo31YEmnaEO6yVboaeLDBz9B5eAhfBoTf1/2nxwtvT6uxwY1Rp6u/xl+tZEq534ubf7luMdvbP1vNEqQNtM6WizJF0XLdr2Dfl5vsLKtsZ/GNAf4Ln8olkR/RX0peo2KYFjB6kr/rrToDIypsBxbTi0AFYOHEz7P8D5IC8vy+dgFtSwqzUNRYHfzAH/a1cbf3+2goWcQlQTvXVrGBfOKRHG6JIgFWSFjxls6Gs2uJhthWeHSReltkNbn8vPEux3nbJEc/dgADk8QsyF9CXbNfZ5RvwfdDh9bT/Zy2eLEa0m8fbIPTxIF32RF4flDXdy8porqAsOI+w53OGgcpdruVMsJ2rmk+VfkB6x4NRa8WgtejRmv1oJPaxl2mwWf1hzb+jxyp1E1T9b/IuFeRPbcOby46F5uOvYV1nT9GWte/agBUJGnkXrr8wC8PffutDQIzFa7m88s8b59qo+6YmPaZhL2tQ4QVhQqzDlUFxiotOTS1Oemc8Cb1h06vmCYvqE3L1WWXLqcKyj0tVIxeJDmwosSOtfZRewMOg3XrKhgodXF68et9A/ljunUKq5bWUFtoWHMcwnjEwGMkDETLR2NZm9L5A9aMi/qo+lyeHlyXye+YPwv9o19rlF3zyRrtBL8Uftb7VRZcllUlh/3+bocXg53Jp+jEgwrPH2gkw+sq6EkP5ITYvcExq22O1Xy/D28/8gXKPI2xf0Yv9qIT2MmL9CDWgnTkb+Kp+vvx6e1JDWG04WXsqPms2xs+y1XnLqPPsN8rHn1I465sOVBVMicLNxEd/7ypK4zHdjcAU4O/fzqNSoc3iAH2+1p2T3oC4Y51BH5Od4wN5LoXTUUwHTYvaydk77fwejsS4FBi1GvoSt/Jcusz1E5SkG7iYxVxG5BaR5VBbm8fbIPmzvAFfWlFOeNnnMlxEcEMEJG2NyBpPNa9rfaUYaCmFSmXdtsHp4+0DlhFdmzne51py2ACYVlmvvHn3Z/5WgPJXl6CowTF1GTZSWWFJgKf1DmiX3tfGh9Lfk5Gl483J3w9yndCjzNvP/oFzD5uxnUlbKz5jNowx5yQ3Zyg/Yz/wbt5IQc5AYdqAijD7vRhyPf42PFm3lllJ1GidpZ8ylKXceYP/AWNxz7Go+t/kNs63WF8yALbFuQUbF9zj+m/Lyz2Z6h2Zd5xUbqio28dtzKO0026itMKSejHmx3EAwrFOfpmFsUmaWI7j7qsHtRFCVtyy7R/Jfo+TtjBe2OIikhFCn+l8rx+iDlatUJlywQxiYCGGHKybLCy0e64146Gs2BNgdhGa6sTy6IOWV18cKhrqTG0DHgxRcMp2W3QIvNM2FgEAjJPHuoi9s21ExYCXhfm53eNFWtdfvD/H1fO3OLjQnPlKVbqesY7zvyTxhCdmy5c/j7sv9kUF8+/oMUGX3YFQtqwiotVuOS9CznSCpeXHQvHz5wB4W+Vq5t+CZ/X/YfKKi5uOU/AThaej02Q13q18pSDm+Q4z2R/JTz6gopydezv91OvyvAriZbSku9wbDM/jY7AOvnFMZ+x0vy9WjVEv6QTL87kLYZjOgMTFVBJICxGerwq43ow26K3Y305i2O6zySEsIQjAR1yfZBEuInAhhhyiWzdDSawx0OwrLCVUvLRhR+k2UFdyCEyx/C5QsxOPTv8K8HfcGkZylkRaG5382SclPKzyHenJK+QT9vHLdy1bKxX7QHfUF2nk7vbi27JzglPYjGU23fw03HvoJO9tBjrOeJZQ/EZjvGJanwa0z4NSbsubVpH1dAk8cz9T/jwwc+Qa1jD5c0/wetlvOodu4jJOnYUfuZtF8zm0R2HcGcIgNlQ/kolywo5sn9nRxot7Oy2ozFkFzrhSOdTrzBMOZcLQtLzxSPVKskKsy5tNo8dAx40xLA+EPhWNBfbYnM9CiSmu785cyxv0PF4KG4AxhDwIaEQlhS44nnZ1RIiQhghCmVytLRaI51OXH5Q+g1Klz+EG5/CLc/POnbcU/3ph7AyLLC6b74d20c6XRSackds0/UlobejC/zpNuC/te5puHbaJQgreb1PL3kfoIaY6aHFWMzzOOlRd/jhuPfYF3nYyzpfRGA/RUfwDXRDNE0NugLcrQrsuX+vLlnChHOKTIyp8hAS7+Hbaf6uW5lRcLnDstKrCbR2lrLOVWpqyxDAYzdG3dPrfF02n0ogDlXS17OmZfErvwVsQDmYMWtcZ0runzk1hbP6G3z2UJ8h4Upk46lo9G02TycsrrodvgY9IWmpJZIc7876S3OUR12L94EdwptabCOukR0utc1bjLwdLSs5ymuO34PGiXIycJNPLn0l1kVvESdKrqcd6o/AYAxaMOvNrK7+s7MDmqSvdtiRx6qHFt5VlXcSxYUIwGnel2xpZlENHQPxvoBLa04903C2XkwqYotH531PDqHqi0nksgbC2BGyX8R0k8EMMKUSdfSUTbwB+VY4l+ykgk4gmGF5w524g+FoXkbbPk3gu4BtjRMzg4hbdhDkfvUpJx7POvbH+GqUz9Chcyh0pt4bsl9KSfeTqYdtf9AU8GFAOyuvjPpHU7Tgdsf4tDQLrcNdee2+SjK07OsMhJ4vHWyN6EgQ1GUWNuANbWWUTull5n0qCUJTyCMPcVij3Amgbe6YGQAE20XYfG1kxuIrxq4MZbAK/JfpoIIYIQpke6lo2zQ2Jf8jIeiKDQmWRRvwBPk9YMt8PhHYct9BH97JQzEv604XiWuBm7f9yFu3/9h3nP650hK+gvYnUNRuKT5V1zS8h8A7K66nVcXfCuhXSCZoEhqnl5yP4+v+B92V92R6eFMqn1tdsKyQrkph5qC0XsSXTCvCK1aosfp50QCTTkbe90MeILoNSpWjLFUqlGrKDNHgtlU30QEQjI9g5E3VWfPwPg1+fTnRpKwKwYPxXW+aBE7MQMzNUQAI0y6yVo6yrRU2gr0OP0p9fzRHPkLeCN5AgbHKW478AkqnfuTPt/ZFvW9wocOfQqTP9I2YG3Xn7jh2NfQhj0TPDJ5khLivad+xPqOPwCwde4/8fbcL0ybInCySkunadW0GW8yvMEwB9vtQGTn0Vg7AI16DeuHmnRua+wjFJ44N2v47MvKajN6zdi7/IYvI6WiyxFpoJqfoxm1+3OnKbKdOt5lpNgMjF4EMFNBBDDCpJtJS0fDOb3BpLcsp5Svoiis6foTAEfq7qTHuARDyM4th/8xVv01+XPLXNjyG65r+CZa2U+z5QJeXvAdQpKO+QNv8YFDn8Xot6Z2jVGoZT/XH7+H5dankVHx0oLvsLfq42m/jpCa/W12gmGFkjx9rDbLWNbUWsjTaxj0hWJbosfTNuClx+lHo5JYPUFybroCmLHyX6K6hurBxD8DI3JgppIIYIRJNROXjoZLtjfSKetg0tesceyh2HOagCqXN0tv588rfhvrlHz1ye9xYctvQEl8N5Iu5OLGY1/l/PbfAbCn8mM8ufSXHCm7kb+seBCPtoAydwMfPvgJSlwNSY9/tOu+78g/s8C2hZCk49klP+Fo2Y1pO7+QHv5QmANDgciGuoIJ6y9p1SounF8ERHoAeQLjzzhGi+ItqzRh0I2/ZFhhzkWSYNAXwplCHky02WnVGEthXUN5MGWuo6jkiWdMxytiJ6SfCGCESXWowzHjlo6GS2QbdFS/y59S88Lo7MvR0uvwa/IJqXN5dvG/sWto58v57b/juoZvognHP+tl9rZx28FPMn/gLUKSjhcW/oC36v4ZRYpM43fnr+CPKx+iP7eO/ICVDx76DHW2t5J+DgAqOcSK7r9zx7sfoMa5F7/ayBPLfkVj0aaUzitMjoPtDvwhmUKDjgUleRM/AFhSnk9pvp5AWGbn6bETYbudPtoGvKgk4qpyrdOoKB1qc9GZ5CxMMCzT44z8jlSPMQNjy52DT2NCK/spcZ+Y8JzGWA6MSOKdCiKAESZV9A/ETNXj9MXVBHK4VJaPzN525g0FDvsrPnTmDknFtjl38dLC7xGWNCzqf40PHP5s7A/qeGoHdvKRA3dQ5G3CpSvhzyt+y/HSa885zplTxeMr/5dW8wZ0spcbj32V1Z1/SvxJKDKLel/m9n0f5MrG+8gL9uHQV/LX5Q/Sbl6X+PmESRcMy+wbKmgYz+xLlCRJXLIw8mJ+uNOBzT16t/fo7MvisvxRc1FGk+oyUrfDh6yAUa/GPNY1JRVdQ72sJlpGUst+ckOR3VliBmZqiABGmDSKoqStrH22UhRoSjCZ91SSy04Aq7r/goRCk2XjqJ2Uj5Zez9+W/RdejZly1zE+fOCOsZd7FIU1HY/xvqP/TE54kM78FTy26hF68peNeX2/Jp8nlv6KQ2U3oULmsqafs+n0/fHtUFKUWLB03YlvUeBrw60t5PV5X+PhtX/Fmrckzu+CMNUOdThilXEXlcbfWBSgusDAvGIjihLZVn02mztA49Dv0LoEGjSmGsAMz38ZLyDrjOXBjJ/IG10+Cqr0+NWJfY+E5IgARpg0A57gjKsMO5rTCWyndniDWJ3JBXXakJvlPU8BsL/yQ2Me12Fewx9XPkx/7tzYcs+8/jdHHKOW/Ww++QM2Nf8CFTKHS2/gr8sfjGvqW1ZpeHX+t9g6558AWNP1ODce+yra0NiBXPngYW458o/ccvQLlLmP41cb2V77Dzy07gkOVHwQWRXfu25h6oXCMu8OVcZdP7fgnMq48bh4YTEqCZr7PbTaRu5ki+48ml9ipCiB1gDRAnoDniDuBGdBYVj9F8v4ycjRPJiJZmCMw7dQz+CdaNlEBDDCpLEOzuzlo6jWfg/BOLaJAknXfgFY2vsc+rAbW04tzZaN4x7ryK3m8ZW/o8VyfmS55/jXWNfxB1AUjP5ePnDoH1ja+xwyat6o+wqvLPgOYVUCfWskib3VH+eZxT8hpNIzb+BtPnj4M+T5e0YcVuhp4vpjX+PDByP9gkKSlr2VH+F3657knZpPE1SP/+IhZN7RLifuQJg8vYb6JNtnFBh0rKyyAJFZmGi1bKcvSEN3JKE9uu06XjlaNcV5kZ/ZRPNgQrJM19Dy9lgJvFHdecuQUWH2d2H0j10wUiTwTj0RwAiTpifJmYbpJiQrtPTHVx8l6fwXRWZ15+PA0OxLHH1W/Jp8nqz/JQfKb0FC4dLmX3Ftwzf5yIHbqXAdwacx8fdlv2J/5W1Jv2M8VXw5f17+37i1hZS6T/LhA3dS6jpOnr+b9578IR/fdxsLbVuQUXG49AYeXvd3ttZ9aUZXqp1JwrLCnujsy5wC1EnMvkSdN68QvUZFnyvAsaE+SvuiLQkKcik35yR8zmSXkXocfsKygkGnpsAw/uxfUGOkzzgfGH8WxhjbQi0SeKdKwgHM1q1bueGGG6isrESSJJ588skR9995551IkjTi4+qrrx5xjM1m46Mf/SgmkwmLxcKnPvUpXK6Rf9gPHjzIJZdcQk5ODjU1Nfz0pz9N/NkJGWWd4Qm8w8WzndoTCCW9Y2KufQeFvlb8aiNHS66L+3GySsPr877BG3VfQUbF4v5XyQv20WeYx2Mrf0+b5bykxjNcT/4y/rjyYfoM88gL9vHBQ5/mE3tvYbn1aVTInCzcxB/W/JFXFn6XwRnc4HAmOt7tZNAX6UsUbQ+QrFytOtb4cUdjP05vkMNDLQnWJ5D7MlyyAUy8+S9RXXHkweT5xQzMVEs4gHG73axatYpf//rXYx5z9dVX09XVFfv44x//OOL+j370oxw5coRXXnmFZ599lq1bt/LZz342dr/T6eSqq65izpw57N27l5/97Gd8//vf57e//W2iwxUyRFEUrDM8gXe4pj73hD1fGq1uku09t2Zot8/hspsSb2goSeyvvI2nlv47g7pSjhdfxZ9W/A5HbnVygxnFYE4Fj6/4X5otF6CV/WiUAG2mdfxx5e94tv5n2Azz0nYtYWrIssLu5sjsy7raglH7EiVqZY0Zc64WdyDMX99tJyQrlObrqS1MbikxmgfT5wrgC8bf6qLdHpkxHauA3dm64mjsaBRLSFMu4QYj11xzDddcc824x+j1esrLR3+ndezYMV588UV2797N+vXrAfiP//gPrr32Wu6//34qKyt59NFHCQQC/O53v0On07Fs2TL279/Pv//7v48IdITsZZ8lCbxRnkCYLofvnM68wyWb/1LgaWaufScKEgcqPpDsEGkuuIj/Wf/spCUYBjR5PLn0FyzveRpHThWt5vNEMuM0dsI6iMMbJEerYvkYfYkSpVGpuGh+Ec8f7o610lg/N/5t2Wcz6jUUGLQMeIJ02r3Mi6M+TVhW6LLHl/8SFd2JVOo6jloOjJovJvogTb1JyYHZsmULpaWlLF68mM9//vP095+pxLpjxw4sFksseAG48sorUalUvPPOO7FjLr30UnS6Mz8kmzdvpqGhgYGBgckYspBms2n2JWq83kj+UPic3RfxWt0VyX05XXgJjpwUZ00mOaBQJA2Hyt9Pq+V8EbxMY4pyZvZlTU0BOk36XioWlOZRMZTvUmDQxl0UbyyJLiNZB32EZIUcjYoiY3yJ646cajwaCxolSKnr+KjHiE7UUy/tAczVV1/NI488wmuvvcZPfvIT3nzzTa655hrC4cj0Xnd3N6WlpSMeo9FoKCwspLu7O3ZMWVnZiGOiX0ePOZvf78fpdI74EDJnphewG81426mb+tyEk6hIrA8NstT6HAD7Km5LemyCkIjGXjc2dwCdRsWqmvTMvkRJksQVS0qpLTRw+ZLSpGdfohINYDqGtQ+I+9qSRJdp/L5I0RkYl6501PuF9Et7j/rbbjvzR3bFihWsXLmS+fPns2XLFq644op0Xy7mvvvu4wc/+MGknV9IzGycgel3BbB7AlgM576rS3b30bKep9HJXvoM82kzr5/4AYKQIkVR2DVUGXd1tWXcrtDJKsrT8741VWk5V+XQMpB10E8gJE84W9Q+QQPHsXTlr2C+beuoAYwu5EInR2ZYxS6kqTPp26jnzZtHcXExp06dAqC8vByrdWQ321AohM1mi+XNlJeX09Mzsp5E9OuxcmvuueceHA5H7KOtrS3dT0WIUySBN3tnYOb3v8G69khNlHSLVhTlqbvhlyuh6wChsBz3NuvhJCXM6q4/A7Cv4kNiSUaYdIO+IG+f6qN30I9WPXFX6GxgytGSn6NBUaDLMf4sjJxE/ktU5/BE3rP+dkSL2PnUeYTUiZ1XSN6kBzDt7e309/dTUVEBwMaNG7Hb7ezduzd2zOuvv44sy5x//vmxY7Zu3UoweKbh3SuvvMLixYspKBh9u51er8dkMo34EDLD4Q3iD2ZnAq8m7OOaE9/h0pZfMce+M+3nP93rgoEW2PcHsLfA72+k6/jOpBKa59newuzvxKsxc7xk/MR5QUiWLCuc7nXx9IFOHtrWzLtDPY9W11jI1aV/9mUyxLuM1OvyEwhHZmmKE6j6C9CTtwwZNXmBXvIDI99g58VqwIgE3qmUcADjcrnYv38/+/fvB6CpqYn9+/fT2tqKy+Xia1/7Gjt37qS5uZnXXnuNm266iQULFrB582YA6uvrufrqq/nMZz7Drl272LZtG3fffTe33XYblZWVAHzkIx9Bp9PxqU99iiNHjvD444/zwAMP8OUvfzl9z1yYNNm8fFTj2INWjoxvqfXZtJ+/0+4juG9Y2QCfnfInP0jZ4JGEzxVN3j1UdjMhdeJFvgRhPIO+IDtP9/PQ9maeOdgVKQVAJBjYvKyMjfOKMj3EuMUbwMTyXyy5qBKc0Qypc+g1LgKgwjlyO7XYQp0ZCefA7Nmzh8suuyz2dTSouOOOO/jNb37DwYMH+f3vf4/dbqeyspKrrrqKH/7wh+j1Z6LdRx99lLvvvpsrrrgClUrFLbfcwq9+9avY/WazmZdffpm77rqLdevWUVxczHe/+12xhXqayOYE3rqBt2OfL+jfgj7kxK9J32ydLMsoBx+NfHHt/SiH/oa2bQe3HLmLvy/7D7qHpqEnUuw+Sa1jDzJqDlTcmrbxCbObLCs097s51OGgpd9DdCEkR6tiaYWJ5ZVmCuLcmZNNostBPQ4/obA8Zs2aZPNfojpNKylzH6Ny8CAnSq6K3S5mYDIj4QBm06ZN4xbseumllyY8R2FhIY899ti4x6xcuZK33nor0eEJWSDZZoWTTlGoG9gGQEilRyP7Wdz7CgcrbknbJaqc+9E5W0GXB6s/QkftzfDYB6h27uP9R77Ak0sfoNO0asLzRNsGnCq6DJeoXiukyOkLcqTTydFOJ65hjQ+rLbksrzIzv9SIRjV9O8tYcrUYdGo8gTA9Tv+o+S2yosQqYSea/xLVlb+CNV2Pn5PImye2UGfE9P2JFbJWti4hFXkaMfm7Can0vFP9SQCWWp9J6zXqh7Y8y0tvAp2Rk3aFJ5Y+QKt5Pfqwm/cd+QJVjnfHPUdO0E5934sA7KsUW6eF5CmKwuvHrTy8rZldTTZc/hC5WjVray3cfsEcbllXzeLy/GkdvEBka/ZEy0j9rgD+kIxWLVGaYP5LVLQib4m7AXX4zEyzURSxy4jp/VMrZB2HJ5hQSe+pVDewHYA28zoOld1MWFJT4TpCoed0Ws6vCftY1P8qAD11t6AoCo29LkLqXJ6q/wUt5vPQyV7ed/SfqXbsHfM8K3qeQCP76TEuiVUAFYRkdDl8HOpwoBBpmHj1snI+efFcLllYMi2XisYzUQATvb3SnIsqyaaUTn0FLm0xaiVMuetY7HbRiTozRAAjpFU2b5+O5r80FVyEV1dIU8HFACxLUzLvgv430IfdOPSVHNMtpdvpi5VLD6lzeKr+50O9gnzcfPSfqbHvOuccKjnEqq6/AkOzL2LrtJCCo0Ndn5eU53PL2pkx2zKW6LJQl8M7atHI9gHPiOOSIkl0mSKzMMOXkWJJvHoRwEylmfmTPAOc7BlETqJya6b1ZGn+iz7kpHJo50A0cDlaej0A9dbnkZTQmI+NV3RX09HS6znd5z2neF1YncPT9fdzuuAitLKfm499mdqBkVu5F9jeID9gxa0t5ETxe1MekzB7BcMyJ3siP4OpdpKeDoqMOvQaFcGwQu9Zy9iKotAZrf+SZAJvVOfZnakVRSTxZogIYLLUoQ4HbzRYJz4wy2TrDMycgZ2oCNOfW4czJ7Jdv6ngYjzaAozBfuYOpFYTJs/fTa1jNwBHS69l0BfiYLvjnOPCKj3PLvkpjQWXoJH93HTsK8wdSiyGM12nD5W/f9SGcYIQr0ari0BYxpSjSflFezoYLw/G5g7gDYbRqCTKTKmVJIjmwVQMHgJFISfkQD30BsitnT5bz2cCEcBkKYc3yMF2B3uGSnpPF9k6AxPdfdRUcFHsNlml4dhQgbhlKSbz1ltfQEKhzbQWZ06kRPpYxevCKh3PLvkJpwo3oVEC3HDsa9TZ3qJs8CiVgwcJSxoOlKdvZ5QwOx0ZWj6qrzCl3G9ouhgrgIlun64w56BOMv8lypq3hLCkwRi0YfZ1xGZfPNoCZJU2pXMLiREBTBZSFCWWO/H2qT5O9Aym/RqhsExT39jdk5Ph8GZpAq8iM3cogbep8OIRd0WXkebZtpITtCd5fiWWRxM930RklZbnFt/HyaLL0ShBbjj+da5ovA+AE8XvxSO2YwopcHqDtA8VbVtaMfOXj6KifZE67F7kYeU+hjdwTFVYpacnrx6ILCOJInaZIwKYLOTyh2JJaIoCLx3ujrvTajz8oTBP7u9kS5qXqKxZWsCu3HUUQ8iOT51HZ/7IGix9xoX0GJegVkIs6Z24htFoKgYPUeBrJajK4WTR5XE/TlZpeH7Rj2kofi9qJUSZ+zgw1PdIEFIQTd6tLsjFlDt7ZgVK8/Ro1RKBkEy/KwBE3hBG/35WWwxpuc7wZSSxAylzRACThRze4IivQ7LCMwc6GXAHUj63yx/iL3vaabN5sHuC9LnSt+STrfVf6myR3UetlguQVefWbozOmiRbEyaavHuy6AqCGmNCj5VVGl5YdC/HiiOtNjryV9GTvyypcQgCRF6wjw0FMMtm0ewLgEolUWkeuYxk9wTxBMKoVRJlpuTqv5ytayiRt3Lw4LAEXjFrOtVEAJOFnN5zd8R4A2Ge3N+BN5D8Eo3NHeDx3W0jMvQbz9opk4psTeCN5b8UXjTq/cdLNhOStJS5Gyh2n0jo3Oqwj8V9LwNwJM7lo7MpkoaXFv2Ap5bcz3NL/i2pcwjTkz8UJhROb+PT9gEvTl8InVrF/NK8tJ57Ohi+jDT833JTzpgtBhIV7Uxd7D5FgbcVEDuQMkEEMFnI6QuOervdE+TpAx1J/cHrcnj58542nGfN7pzqTV8Ak40JvMZAX2xppsly4ajH+LQWThdeCsCynsRqwsy3bR2q/VJBu3lt0uNUJDWni94j3sXNAk5fkH2tA/x1bzv//eZpHt3VmtYgJrp8tKgsD22aXrCnk1gi74AXRVFS7n80Gre+FKeuDBVy7A2SWEKaerPvp3saODvIGK7T7uPFI93j9qM62+leF3/b2z7q7I3V6T9nySoZTl8wpdmhyRLdotydtxSvrnDM46LLSEt6X0Alx//9iO5eOlZyLUji10kYnc0dYFeTjT/uauWhbc1sPdlHh92LQuSNyYFRttwnwx8Kx+oPLZ0FtV9GU2bSo1ZJeINhBjzBtCbwDtdliiwj5YQiAaMIYKae+IubhSYKKE72uHjrZF9c5zrc4eCZA10Ew2MHPI1pmIXJ1gTeM9unLx73uOaCC3BrizCE7LHHTMTot1I7VE33aOl1qQ1UmFEURaHb4WPbqT4e2dHMH3a2sON0fyxPrMqSy6ULi7loQaRuyO5mG/407OA7aXURkhUKDFrKU6x3Ml1pVKrYcz/WFWleqZIiW6jTqeuszvJiCWnqJdyNWph8Tt/EVWH3tgxgztWyqsYy5jHvnO5ne2P/hOdqtLpYW1uQyBDPkY0dqFVyMBZgDK//MhpF0nCs9FrWd/yBZdZnaCzaNOH563tfQIVMh2k1jtyadAxZmMYURaF9wEtjr4vGXveIrs9qSaKmMJf5JXnMKzFi0EX+9MqKwvGuQfrdAfa0DHDRgtSWEI92RmYDls6i2i+jqbLk0mH3cqDdDkCZKSfty2ln9ykTnainnghgsowsK7jiCGAAtjT0kp+jYV7JyEQ9RVF4o8HKgbb4pqU77T68gTC5OnXC443qycIE3irnPvRhN25tIT15SyY8/kjp9azv+AN1tm0YAv14dONU1VQUlg51nk42eVeYOVptHt462Uuf68xOQa1aYm6RkfklecwtNqDXnPv7pZIkLpxfxDMHu9jfZmdVjYU8fXJ/lgc8AbocPiRgySzbfXS2qoJcaCY28zwZlYh7jYsIqfRoZD8yajzasZeohckhlpCyzKA/RJnjADcd/RIFnuZxj5UVhRcOd9MzbPkmFJZ59mBX3MFL9DypLiNl4wzMiOq7ceSn2Azz6MpbhoowS3pfHPfYMtdRirxNBFV6ThZdkZbxCtNPv8vPU/s7eGJfB32uADq1iqUVJm5YVcFnL5nHtSsqWFyeP2rwElVXbKTCnENIVnjn9MQzpmOJzr7UFhmSDoJmigpzDsML7lanOf8FIsUoowXt3LoikQOXAeI7nmWc3iAru//GvIG34ypvHwjJPLW/A6cvUgX37/s6zmkiGI9UAphBX6TOQraJ1n+ZKP9luKOlNwBDybnjJEpH/29OFV1GQDP7tqrOdp5AiNePW3l0VyvN/R5UEqyqNnPnhXN579Iy5hXnxb1lV5Kk2NLRkS5nUvWeZEXheHekYvdsq/0yGq1aRWl+JOdFkqDCPDm9oKLLSCKBNzNEAJNlHN4geYFIhdxCb3Ncj3H7wzy5r4O/7GmLZdwnqrXfM2bvnolk4/Zps7eNQl8rYUlNq+W8uB/XUHIVIUlHsaeR0qHt12dTy34W970CxN86QJgZQmGZ3c02fr+9hUMdDhQF5pcY+dgFc9i0uDTpZdgqSy51xUYUBbYnMQvTavPg8ofI0aioK0msmOJMFd11VJqvR6eZnJe6aPmFsxN6hakxu+cZs5DTF6TSHwlgJlpCGq7flVqV3pCs0NLvZmFZfsKPzcYCdtHlow7TmoRmSPyafE4VbWJJ38ss63kG69AU8XDzbG+RE3IyqCulzbw+bWMWspeiKDR0D7KtsT+WnFuar+eShcVUF6SnPP2F84to6nNzyuqi2+lLaBdRdPlocXk+GpV4XwqwvNJE+4CH9XMmLzel07SKB897Ga/GMmnXEMYmftKzjNMbJC8Q2SJt8XUkVJMkVckuI2V9/kuCjgwtIy3pewm1fG5gGE3ePVp6HYqUfOKzMD10DHj50+42Xjrag8sfIk+vYfOyMm7bUJO24AWgOE/PkvLIG4jtp+IrkwDgC4Y53RtpzDqbGjdOxGLQcduGWhZMcjVir7Ygsk4lTDkxA5NlPE47OtkDgIowFl87NkPdlFz7dJ+bsKwk3G4+22ZgtGEP1Y69QGL5L1Ftlg0M6krJD1iZZ9vKyeIrY/cZA33MHdgBDBWvE87h9ofQaVRZUwXW7Q+xu9lGU58btUpCp1Gh06jQq9WxzyNfD/tco0IlSRxot9M4FBzo1CrWzy1gTY0lbSXpz7ZxXhEne1y0DXhp6Xczp2ji5aCGnkHCikJxno6S/PT0+hGE6UAEMFlGdnaM+LrA2zxlAYw/KNM+4Inrj2bUoC+I259dCbw19t1olCD2nCoGcuck/HhFUnO09DrOb3+IpdZnRwQwS3pfQEWYzvwVDBjmpnHUkUabuVp1wgFkNmnud/PMgU5K83P4wLpqVBl8Lv5gmL2tA+xrtROS469cfTZJguWVZi6YVxir3zJZTLlaVlSb2d9mZ3tjP7WFhgnruYjaL8JsJQKYLBKWFSRX14jbCr3NNE7hGBp7XQkFMNnYgXreQHT30UVJT+0eLb2e89sfYu7ADoz+Xtz6khG1X9KdvNsx4OVv77azsCyPa5ZXpPXcU8XuCfDi4W5kBbqdPg6021mTYoHEZATDMgfa7expHsA/lJhebsphw9wCdBoVgZBMICTjD8kEwkP/Dn0evS/6dXGejgvnF1No1E3Z+DfMLeBopxProJ+TVheLxslL63P5sQ76UUmR/BdBmE1EAJNFnN4gef7eEbcVelqmdAyNVjeXLVbifieXdfkvisLcge1AcvkvUfbcWjryV1E1eID63ufZU30Hpe7jFHsaCUk6ThS/N10jBuBgux0FONHjYl2tj9JpVgY+EIrUH/KHZAw6NZ5AmB2n+1lQmkd+jnZKxhCWFY52OnmnqR/30Lb+QqOOC+cXMa/YOG1mJww6DWtrLexssrG9sZ/5JXljzspFGzfWFRsnfXZIELJNdixSC0BkB5JxKIDxqSPvpgri3EqdLi5/iO4E+hplW/5Lseck+QErQVUO7eZ1KZ3rSFkkmXep9dkRsy+nijbh16Tv3a4/FKaxzx37emeTLW3nngqKovDqsR763QEMOjUf3lBLhTmHYFhhS0PvxCdIw/Ubugf5w84WXm+w4g6EMeVouGppGR89v5b5JXnTJniJWlNbQK5WjcMb5Ejn6EUpw3KkDQGI5F1hdhIBTBZxekOxGjDRF99Cb/O4BdUmQ6PVPfFBQ7JtBqbOFtl91GreQFiVWkLjyaIrCKr0FHmbqXLui1XnTXfjxpNWF2FZIU+vQQKa+twjqitnu72tA5y0ulBJcN2KCvJyNFyxpBSVFEkMT0ez0NEoikJTn5vHdrXy4pFuHN4guVo171lUwsc3zqG+woRqmgUuUTqNivPrItt/32myEQyfW6Opud+NNxgmV6tOaNlXEGYKEcBkEacvSF4g8o613bwGGRX6sBtjMPny4sk4ZR2M6ziXPzSiYV02iOW/FCa+++hsAU1erE3AVad+SG7IgUtXQqvl/JTPPVz0XfSqanMsj+GdaTIL09LvZvupyM/nexaVUDnUc6YoTx9rELqloTfpIolj6XJ4+eu77Tx9oDNWwn/jvCLuvHAuq2ssM6IWyvIqM6YcDZ5AmH2t9nPujybv1lfkT+vEb0FI1vT/LZ9BhlfhdeircORUAYkVtEuHAU+QftfEMyvWLJslyAnaKR88DEBTwYVpOWe0JozF1w7AsZJr0lr7xekN0mGPVE9eXJ7PeXWF02YWxuEN8sLhbhRgWaWJFVXmEfefV1eIOVeLyx9iRwo9fs52yuriz3va6bT7UKsk1tZauPOiuZxXVzhpFVczQa2S2Dg/0lB0b8sA3uCZ3X6eQIjmflH7RZjdZs5v+wwwPInXpS+NbQGOt6VAOsXTTynbWgjMse9EhUyvYQEufXlaztluXotDXxn7Ot27j6L9a2oKcsnP0VJg0MWKme1M44t+ugXDMs8e7MQfkikz6dm0qOScPBOtWsVliyM9Yg602dMSkHXYvbx4pBuAhaV53LFxDpcsLCFXOzMLCi4uy6c4T0dgqIVB1PHuQWQFykx6ivJE7RdhdhIBTBYZ9HgxBCN/pNy6Emy5c4HMBDDR4l3jybYE3jPNG5PffXQOSRXLeenOW5rWmjyKonCsK7oMcOZd9Hl1hUgSNPd76HZk1/cYziTt9rkiSbvXr6gcs7DbnCIji8ryUIDXj1uRU6jHYnMHeOZAJ2FZYV6xkauXl0/ZDqdMkSSJi+ZHGj0ebHPg9AVRFGVE7RdBmK1EAJMlgmEZxdWLChkZNR5tAbahQmmZCGB6nD6cvvHbGPROQg0YXzCMO4m8GkkJM9e+E0hP/stwL5s/yO9Ut/JgwVfSet4epx+7N4hGJTG/5Ey5c8uwWZh3mrJvFmZfq50TPZGk3WuXR5J2x3PpwhL0GhXWQT8H2u1JXdPtD/Hk/g78IZlyUw5XLy+ftgm6iZpTZKDKkktYUXjntA3roJ9+dwC1Shq3RowgzHQigMkSkR5IkeUjt64IRVIzMDQDM9U5MFGN4ywjuf0hBn3pTeD1BcM8+k4rv9/RjM2dWHPKisHD5IYc+DQmuvKXp21MYVnh2QYn93rez5+ajaPuBklWdPZlQWneObkb583NzlmYVpuHt4f69Fy6sCTW8Xc8Rr2GixZEZhF2nO5ncILA+Gz+UJin9ncy6AthydVy46rKrGlTMBUkSeKiBZFcmGNdTrY3RoLa+SVGcmbo0pkgxGP2/BXIck7fmS3ULl0pALahHBhToAdt2DPlYxpvGWkyKvBua+zD5Q8RDCu8dKSbcALLDXVDu4+aLRtRpPQV9Hq3dSA20+QLyhzqGL0mR6LCssKJnkj+y5JRKqhaDDrqyyPLAzuzZBbG6Q3ywuEuFCI7X1ZWmyd8TNTySlOsNsybJ+KvDROWFZ4/1E2vy0+uVs3Na6rI1c2+F+0Kcy7zS4woRIJIEMtHgiACmCwxfAbGpY8kPvq0FjzayFbUAm/rlI+pY8CLLzh6n6N075DptHs53BGZkdCqJayD/hFJixOJ1n9JZ/6LzR2IbWeeWxTpOry3ZYBQGmZhmvvd+EIyRr2amsLROxpvmFuAJEFLv4cuhzfla6YikrTbhS8oU5qv5/LFpQkVh5MkicuHasM09sZXG0ZRFF471kOrzYNWLXHT6krMuTM752U8F84vJvodz9Nrxvy5EYTZQgQwWcLhDZLnHzkDA8QSeae6Ii+ArChjvtCkcwYmLCu8fjzy3JdWmLhiSRkAu5ptcVUFzvP3UOI5iYJEc8HGtIxJHkpUDcsKc4oMXL+ykvyhmhzR8u2piC4fLSkbu9ja8FmYd05nri6Moii8dtwamwW5fmVFUt2YixOsDbPjdD/HugeRhnJtyqZZe4V0KzTqWFYZ+XlYXjV9i/QJQrqIACZLDC9i59KVxG6PbaXOVB7MGMtI6awBs691gH53gBytiosXFLO4PD+yc0WBl490TzjjUTcQmX3pyl+OT2tJy5gOtjvocvjQqiMzB2qVxLqhF989LQMJLW+dzRsM0zTUOmBJxfhJmOfVFaKSoMWW+iyM0xfkxcPdPLW/g9eO9bDzdD+HOhyc7nNhdfpw+0PIo1R93t9mpyEaSKxIbedPvLVhDrbb2d08AMAVS0qZWywqzQJsWlzK+9ZUsWFOYaaHIggZJ7p/ZYnhbQSGBzBntlJPbVPHqNZ+N8GwPCJp0hNIXwKvwxuMLdNcsrAklt9w2eJSOga8DHiCbDvVz3sWl4x5jmgA01SQnt1HDm+QbUOJqhcvKMY09IK9rNLErmYbg74Qx7udLKuMPwdkuJM9kRoeJXl6iieo4WHO1VJfYeJIp5Odp228b01VUte0uQM8sa9jwsrJkgRGnQajXo1RpyFHq+ZYd2S26NKFJVQXpLZsEa0N8+T+Tg602VlSnn/OzEpjryvWQ+mCusKkv88zkVolUSuWjgQBEDMwWWPkDEx2LCEBBMMKLf0jZ2HS1f9IURTeaLASkhWqLbnUD0tmzdGquXJpZClpf7s9lrh4NrXsp9a+C0hP/ktkuaSHkKxQZckdUV1Wo1admYVpHhh1tiIex4ZaB0w0+xK1YW5kFqbV5qHTnvgsTLfDx1/2tOHyhyg06Lh8cSnn1xWyvNJEXbGR0nw9Rp0aiUjbLZc/RI/Tz+k+N0e7nChKJNF4VQJJu+MZrzZMl8Mbq+67vNLEeXVipkEQhNGJGZgs4A+F8QbCw6rwDltCMkSWkAq8rUhKOK1l7ON1yupmQemZF9t0JfCesrpo6fegHkrwPDspdG6RkRVVZg51OHjlaA8fO78W/VnbRqsd76KVfbh0JfQaF6U8piNdTtpsXtQqiSvqzx3T8iozu5tt2L1BTva4Yr2L4jXgCdDt9CFJkSqr8Rg+C/NOU2KzMC39bp471EUwrFBm0nPT6qoxq9bKsoJnqA5P5COMKxBCBaybU5DWjs6XLiyhpd8Tqw2zpraAAU+Ap4cK1c0tMnBZgonCgiDMLmIGJgs4vSF0IRc6OTLL4B62hOTUVxCSdGiUACZ/V0bG19TnHvEuOR0JvP5QmC1D22nXzy2gwKgb9bhLFhbHcia2jLL9dqn1WQBOF1wSWf9Igcsf4q2TkaWjjfOKKDCcOyadRsWaoVmY3c02lARnYaKNG2sLDRj18b9/OC+JWZgTPYM8faCTYFihttDA+9dUj1tyX6WSyNNrKDPlMK8kjxXVZjbOK+L8eUVJJe2O5+zaMD1OH0/u68AXjLQmuHZFBSrRoFAQhHGIACYLOH1BjIHIC6dfbSSoPrPGHSloVwtAgSczeTC+YJj2gTMvmumYgdl+qh9PIIzFoGX9nIIxj9OqVWxeVoZEpP/LyWGdso1+Kwv7XwPgYPn7UxqPoii8cdxKYKi3z5oay5jHrqo2o1Or6HcHON03ccuF4dc4PpRPEt1dFC9TrjZW9yOeujAH2u28cLgbWYFFpXncuKoy6xodDq8N8/ieNpy+EOZZWKhOEITkiL8SWcA5rAv18PyXqIEM9kSKOtUbCRy8gXDKCbzdDh8HhwrCXb64dMJ39xXmXNbPjQQ5rx+3xloNrOr+G2olTLtpDb15i1Ma04keF6f73KgkuLK+bNx3/3qtmlU1kXyQXU3xz8J02n04fSF0ahXzSxLfVRPNhWmzeWMdrM+mKAo7T/fHkmBXVpnZvLwcdRbOZgyvDaMoRArVra7EoBMr24IgTEwEMFnA4R19C3WULYNdqaNO97pRFCXl2RdZjiTJAtSX58ddjOv8uiKK83T4gjKvHbeiCvtY0f13APZV3pbSmDyBUKw67Ia5hRPuDAJYXWNBo4oU3Bsrwfhs0d08C0rzklqSGTELM8oWZEWJVLmN7uo6v66QTYtLsrpeSHGenk2LSikz6blxdSWWUZbtBEEQRiMCmCwwoo2AfpQAZqipY0GGtlIDDPoiO1NSzX/Z32anzxUgR6Pi4oXFcT9OrZLYvKwctSTR1OfGdOpJDCE7Tn05jYWXpjSmN0/04g2GKcrTsWFufLteDDoNK4Z25bwTxyxMKCxzsidSFLA+zt1Ho9kwVBemfcBLx7BlvbCs8OKRbg60R2a2Ni0q4YJ5RdMiCXZFtZnbNtRSPssL1Qnj06ikWdlGQhibCGCygNMbPLMDKUuXkCCya8g6mPwMjNMbjBUvu3hhccJLBcV5ejbOLwIULuj9KwD7yz+YUu+j070uTvS4kIgsHSWy1LK2tgC1JNHl8I25pBO7Tp+bQFgmP0dDlWXiBohjMeVoY3VRorkwwbDMMwc6Yx2ir15WzqpxcngEYTpaWJYf1+yoMHuIACYLjFWFNyqaxGsIDpATtE/l0EZo7HXRk2QNGEVR2HKiN1ZfJdlGdGtqLVxrOk291IIPHYdKb0jqPAD+YJjXGyIzX2trCxKeAcjTa1g6VNo9WjV2LLHWAeX5Kc+KrJ9bEJuFOWV18fd3O2ixedCoJG5cVZnw1m5BmA5W1ZixzOJeWMK5RACTYb5gGH9QHjeJN6g24NRFirplchnJ5g7g9AaTemxjr5umoSTZ0Wq+xEslSdxtiOw8+lvoEnZ0JV/S/61Tfbj9YSy5Wi6Yl1zBtPVzCmLbm7sdo89OeQIhWobyZBLdfTSa4bMwzx3qotvpI0ej4v1rq5hTJEruCzNPmSmHCnMuFoMIYIQzRACTYdGAIDoD4x5lBgZgYCgPJlMtBVIRqfkSCdDWzymkcIyaL/Ew+TpZYt8KwMPhzexo7KfPlfisUKvNw5HOyKzIlfVlSdc5MeVqYzMeu8bont3QPYiiQJlJP2a9m0RtmBtZvoLITNCt66qpMCe/NCUI2WzlUL6ZCGCE4UQAk2FOXxBJCWEIRPIZXPpzZ2BgeE+k5ikaWfrsbLTh9ocx52rZMHfsmi/xWNX1F1TItJjPI1S0mLCi8NKRboJhOe7tzMGwzGvHIjuhVlaZqSpI7YU/mvjb1Oemd5Qk5+PdkS3o6Zh9icrP0XL5klIWlebxgXXVFIncACGD8nM05OdMzvZ3vVYVe5NgzhW71IQzRMGFDHN4gxgCNlTIyKjxaEd/gY9upS7IUFfqZPU4fexvtwORpaNUKrpqwl6W9zwFRLZOX2Es5dF3WulzBfivLY1AZLeSWiWhGfr33M9V+IJhnL4Q+TlnqsGmosCgY1FZHid6XOxutnHtiorYff2uyM4tlQSL4mwdEK+llaZYDo4gZIpWHcm96nT4eOO4Ne3nX1phihU2tBi0SEN1gwRBzMBkWKQLdXT5qGjMXkcDGe5KnQxZVnh96A/a4vL8lLvoLrU+T054EHtONU0FF2HUa7iyvjS2lAKR7cSBkIxnqOCe3ROkzxWgx+mn0+6j1eaJbQW/Yklp2qrTRmdhTlpd2NyB2O3HhmZf5hYZxRZQYcaRJNi8rJxSUw7LK01pn4WRJFhVbYl9rVWrMIpCh8IQ8ZOQYRPtQIqKLiGZfR2o5QBhVfZPpR7tcmId9KPXqLgk1ZkORWF11+MA7K/4IEiRwGNeSR6f3zSfkCwTCiuEFYWwrEQ+l4c+l+XY55GvFUy52pQDquGK8/TMKzZyus/NnmYbVy0rR1YUGqLLR0nuuhKEbLZxXhELh2YWNWoV6+YUxKpAp0NNgeGcvDGzIdIbTRBEAJNhTm+QqtgOpLEDGLeuGL/aiD7sxuxrx2aYN1VDTNrRoa3DG+YWJtS4cDS1jl0UeZsIqAwcOWvrdGR5SE2Kl0jZhrpCTve5Od4zyPnzinB4g7j8IfQaFXOL0xcsCUI2qK/I5/x5RSNuW1FlZk/zQNoCjGjLjuEsudoRRRyF2UssIWVYpArv0AzMGAm8AEgSA1nQUiBeTm+QrqFtxemoS7Km808AHCm7gYAmL+XzTYZyUw61hQYUBfa02Dg+FMAtKstHoxK/asLMUWHO4cr6snNu16hVrEsxUT8qP0fDvOJzf9dFuwkhSvxVzSBPIEQgJMeq8I61hToquoyUqa7UiTjRE1k6qS7IJS/FqRGLt5V5A28DsL/iAymPbTKdN5QLc6xzkJPW1FsHCEK2yc/RcMOqyjET8ldUmTHqU8/3Wl5lHrWpqthKLUQlHMBs3bqVG264gcrKSiRJ4sknnxxxv6IofPe736WiooLc3FyuvPJKTp48OeIYm83GRz/6UUwmExaLhU996lO4XK4Rxxw8eJBLLrmEnJwcampq+OlPf5r4s8tyTm9kmnW8InbDTaet1CeG+v6kY+fN6q4/A3C64CLsQ7NQ2aqqIJcqSy5hJZJrY87Vih4/woyh06i4aXXVuEvCWrWKdXOSKwwZpVZJLK86d/kIENV4hZiEAxi3282qVav49a9/Per9P/3pT/nVr37Fgw8+yDvvvIPRaGTz5s34fGeqlH70ox/lyJEjvPLKKzz77LNs3bqVz372s7H7nU4nV111FXPmzGHv3r387Gc/4/vf/z6//e1vk3iK2csxVMTOGEcSL8CAYXosIdncAXpdka3DC0pTW+7RhVwstT4LwL6K1LpOT5XhtW7q09A6QBCygSTB1cvLKcmfuObQyurUZmHml+SNOXNrFjMwccnmLvTpkvDc/jXXXMM111wz6n2KovDLX/6Sb3/729x0000APPLII5SVlfHkk09y2223cezYMV588UV2797N+vXrAfiP//gPrr32Wu6//34qKyt59NFHCQQC/O53v0On07Fs2TL279/Pv//7v48IdKY7p29kFd6JApjYEpK3JVIIIUt/QBuGlo9qCw3kalObSl5qfRZ92E1/7lxaLeenY3iTrrbQQG2hgR6nj3pRp0WYIS5eUMz8kvjekGiHdiRtPdGX1LWilXdHo9eoMejUeALhpM49G1QX5HJFfRnPHuyk3xWY+AHTVFpzYJqamuju7ubKK6+M3WY2mzn//PPZsWMHADt27MBiscSCF4Arr7wSlUrFO++8Ezvm0ksvRac7k6y1efNmGhoaGBgYvWme3+/H6XSO+Mh2Tm8QbciNPuwGJkjiBRw51cio0YfdGAPJ/WGYbIqixPJfFqe6fKTIseWj/RUfytqA7WySJHHTqko+fXEdphzxblGY/pZVmlg/N7FloZXVFgxJ1D4qytNRM0GJA5EHM7Y8vYZrV1RQaNTxoQ011BXP3P5oaQ1guru7ASgrG5mdXlZWFruvu7ub0tKRL9QajYbCwsIRx4x2juHXONt9992H2WyOfdTU1KT+hCbZ8BowfrWRoHr8X9qwSocjpwqAQm/TpI8vGb2DfuyeIGqVxLw4362NpW5gOwW+NnzqPI6VXpumEU4NlUpKqeqwIGSLqqF384nSqlWsT2JH0ooxcl+GEy0FRqeSJK5dWRHLUdJr1Ny0ujKp/4fpYMb8hb3nnntwOByxj7a2tkwPaUIOz/AiduPPvkTZsnwrdXT5aF6xMeUqt2u6IlunD5fdNGFwJwhC+plztdywshL1KLuB4pHoLIxOo4qrPYaYgRndxQuLqLKM7O0mSRKXLCxh87JyNEn+P2artAYw5eXlAPT09Iy4vaenJ3ZfeXk5VuvIfhmhUAibzTbimNHOMfwaZ9Pr9ZhMphEf2UxRFAZ9w9sIxFepNtqVuiALWwpElo/Ss/uo0HOaOfZ3kFFxIMu3TgvCTBTZcVSZUguMaC5MvBaX5aPXTHw9EcCca0Fp3ri7v5ZWmrh1fXXKZS2ySVoDmLq6OsrLy3nttdditzmdTt555x02btwIwMaNG7Hb7ezduzd2zOuvv44sy5x//vmxY7Zu3UowGIwd88orr7B48WIKCmbGVJg7ECYkK3FvoY6KbaXOwqaOnQ4fLn8InVrF3KLUZkxiW6cLL8U5tGwmCMLUUEkS162oSEuX85XVlriDoJWjVN4djUUsIY1QYNBy1bKJl/kqzLncdl4NZTOktEPCAYzL5WL//v3s378fiCTu7t+/n9bWViRJ4otf/CI/+tGPePrppzl06BC33347lZWV3HzzzQDU19dz9dVX85nPfIZdu3axbds27r77bm677TYqKysB+MhHPoJOp+NTn/oUR44c4fHHH+eBBx7gy1/+ctqeeKY5z95CrR9/B1JUNi8hnRjq+zO/1JhS/oc+5GSp9Tkg0nVaEISpdcG8QuamKflTp4lvFqbSkkNpfnwvrGIG5gytWuK6lZVxzVwB5Odo+cD6apakoUJ6piU8l7Rnzx4uu+yy2NfRoOKOO+7g4Ycf5utf/zput5vPfvaz2O12Lr74Yl588UVycs78YD766KPcfffdXHHFFahUKm655RZ+9atfxe43m828/PLL3HXXXaxbt47i4mK++93vzqgt1NEaMHn+ifsgDRdtJ5AfsKINuQlqsiPDXJaVWOXZVJePlvc8hVb20WtYSLtpbTqGJwhCnAoM2oR3HE1kVbWFvS0DeMfZ+rxyWNfpieRo1eRo1fiCYiv1ZUtK46rNM5xWreKaoRm27Y19KMokDW6SJRzAbNq0CWWcZytJEvfeey/33nvvmMcUFhby2GOPjXudlStX8tZbbyU6vGkjOgOTaBKvT2vBoy3AEBygwNeKNa9+0saYiLYBD95gmFytmpqC5JePJCXEqq6/ALCvcvpsnRaEmeLSRSVJJ+2ORadRsba2gG2nRi//YNCpWZhg0UuLQUu3Y3YHMCuqzCyrjG/ZbTTn1RVSlKfjxcPdBEJyGkc2NWbMLqTpxumLthGIr4jdcNmYBxPdfbSgNC+lP37zbVsx+7vwaswcL96cruEJghCHumJjyuUPxrKqxkzOGIUtl1WaE152nu0tBUpNejYtjv91YyzzS/L40IYazGd9P9UqiRytGlOuluI8HZWWHOYUGVhYlsfSShOray0Z/z+YOenI04zTG0RSwhgD/UD8MzAQWUaqdu6jIEvyYEKyTKM1UowvleJ1khJiXcf/AXCo/H2E1TMj0UwQpgO1SuI9i1J/QRyLXqNmba2F7Y39I26XJFgxTuXdsczmlgI5WjXXrxi7oWaiivP0fPSCWnwBGZ1GhVY9PepYiQAmQxzeIIaADRVhZNR4dPGvOffn1gFQmCVbqVv6PQTCMnl6DZWW5IIOlRzimhPfoXLwECFJx4HyW9I8SkEQxrOm1kKBcXJ396yutfBuq31E7kpdsfGcd//xyNROJEmKzP6Um3MoNeVgytHy0pGpW4KRJNi8rCztAZxeo447EThbiAAmA2RZweUPUTy0hdqtK0KR4v/BObupo6IoDHiCGHTqMadoJ1PD0O6jRWV5STUuVMkhrj3xLRb2v05Y0vDckvtw6Uev9yMIQvoZ9WrOq0tv4u5o9Bo1a2ot7Bg2C5NI8u5wU7UTyZSrpcykp9yUQ5kph5J8/Tl/Z0NyKS8cGr1KfLptmFs4act8040IYDLAFQgRlpWk8l8A+vSRAMbsaeW5/W20OwL4QjKaoSngZZWmKeuAHAjJNPVFlo+S2X2kkoNc2/BNFtq2EJK0PLvkJzQVXpLuYQqCMI6LFhRP2bvv1TUW3m0dwB+UMedqk64ZNVkBzJwiA5WWXMpMOZSZ9Bh0E79MLik30WbzcrjDMSljiqopNLBxXtGkXmM6EQFMBjg8I2vAuCcIYEJhmR6nnw67lw67lx6Hh49rtORIQQK2FnxKGSoJQrLCa8ettNg8XLGkdEpmY073uQjJCuZcLaUJbuVTyUGua7iHBbY3CUk6nlnyU5oLL5qkkQqCMJoKcw5LK6aucnmOVs2amgJ2nu5nRbU56TdbBp0GvVaFP5i+pRu1SuKm1VVJbUTYtLiEbqePvkF/2sYzXKRJYzmqGdYOIBUigMkAp+/sLdQjAxhFUWi1eYYFLH7CI7auSzRrKlhCKzdWuuitWE9xvo4DbQ62N/Zxyuqi2+Hj6mXlVBWM7IuRbtHWAYvL8hP6Q6SWA1x3/F+YP/AWIUnH0/U/o6XgwskapiAIo5Ak2LS4dMpmbKPW1Fo41GFneQpbgCHSq8kaTF/AUJynT3oXpVat4roVFfxxV2va82GiTRrjmQ2aTcR3IwOc3ugW6qEidmdV4d3ZZGNXk23EbQadmipLbuSjIBepfRH0t7Ihv493zZHE2XVzCqguyOWFw904vEH+9m47G+oKOX9u4aRE7b5gmJb+6PJR/GuyajnA9ce/wbyBtwmp9Dy15H5aCy5I+/gEQRhffYWJcvPU7/bL0aq5eXVVSn2WIJLIa3WmL4ApM6XWOqHQqOOyxaW8dCR9+TAqSeKqZWXnNGkURACTEbEZGP/oRexO90ZmNeqKjcwvMVJpycWSqx3xLmnAMBf6z20pUGbK4SPn1bLlhJVjXYPsarLRZvNw9bJyTGnes3/K6kJWoDhPF3fPFLXsHwpetkWCl/qf02o5P63jEgRhYjqNiosXxNdEdjKUpqEfT7rzYNLRI2hppYm2AQ9HO50pn0ujkrhmRQULEizyN1tk/0bvGchxThXeMzMw3mCYPlcAgCuWlLKs0kyBQXfOFG+smN0oW6l1GhVXLS3n6mXl6NQquhw+Ht3VyomhYnPpEi1eF2/yrlr2c8OxrzFvYBtBlZ4n6/9dBC+CkCEXzCvEOM07Eyez/Xo8pSnOwERdvqSUorzUtnlr1RI3rq4Uwcs4RACTAWfaCJzbibp9wANAkVE37h+XaABTME413sXl+Xzk/FrKTTkEQjIvHO7mlaM9aVmfdftDtA94gfgCGHXYx43HvkqdfQdBVQ5P1v+SNst5KY9DEITEFRp1rK6ZuMFitkvnDIxWLVFsTE8Ao1WruHZFBVp1ckv3eq2K962tZk5RdvS6y1YigJliYVnB7Q+jDXvQhyP5I8NnYKJBQfUEybcDubUAGEJ2coL2MY8z52q5dV01G+ZG/lgd7XLyx92tWJ2+VJ5GrHFjuSlnwndBmrCPm45/lbn2nQRVOTyx9Je0W9andH1BEJL3nknod5QJFkP6itkV5+nTmitYnKdn0+L4K6xHGXRqbl1bLXJe4jC95w+nIZcvhKwomIe6UPvVxhEdpc8EMOPXRgipc3HqyzH5uyn0ttCptYx5rFolceH8YmoLDbx0pAe7J8jje9q4aH4xa2otce9AWNX1Z0pdDbh1xYS61OhVeVQVzCPPr8WjLUZWnfvjpAn7uOnYl6l17CagyuXJpQ/QYV4T1/UEQUi/eSVG5hbPjHf2eXoNOo0qLbPK6ch/OdvyKjPtAx6OdcW3fJ+fo+H9a6spnOSKyDOFCGCm2Hj5L25/CJs7kv8y0QwMRJaRTP5uCrzNdJpWTXh8dYGBj55fy6vHemjsdfPWqT5O97m5sr50wncyNfbdXH76Z7GvzwfQAT2RDwUJj7YAt64Yl64Et64Yt7aYaue7VDv3EVAZeGLZA3SaVk84TkEQJsdk9zvKBFOuNi21V9KV/3K2y5eU0eP0x/62j8Wcq+WWtdWzusdTokQAM8XOrgHjHmX5qCTv3FLVoxnIncNc+86EulLnaNVct6KCwx1Otp7spcPu5dF3Wtk4v4jVNRZUo8zGSEqITU0/B6DZcgEngiXIzm5qNA5qtA6MwT7UShhj0IYxaKPUfWLE4/1qI08sfYCuOIKs2SxHqx7RI0YQ0m1tbUFal12ygSVNAcxkzMBAZFPFtSsq+NOuVkKyMuoxRXk63r+2mrxpnlQ91cR3a4pFE3iNgXO3UEcTeKsL41v7PLMTqTmhMUiSxIpqM7VFBl491kP7gJe3TkYK4F1ZX3bO9OXK7r9T7GnEqzHz/KIf8bt37fQFA1wxv5TlVWZQZHJDDoyBXvICvRgDfbF/NbKf/RUfxJpXn9AYZ6Mr60t59ZhVBDHCpMjTa6ak39FUS0cir06jomgSl21K8iP5MK8e6znnvlKTnvevqU65Js5sJAKYKXbuEtKZOgxtcSbwRsV2IiXZldqcq+X9a6o40unkrZN9dDl8PLarlfPrCllXW4BKJZETtHNhy4MAbK/9PJ3+HPpcAVQSZ7b3SSq82gK82gL6jIuSGstsZzFoWVCaR9uAhwNtk9tPRZidLlpQjE4z8/ZtpKMrdUmeftKrEa+oNtM24Ik1vwWosuRy05rKadcFOlvMvJ/mLHdmCSlahTcyAzPoC+LwBpEk4s4+HzDMBcDs60Atj7++OhZJklheZeZjF9Qyp8hAWFbY3tjP43va6B30c2Hrg+SEB7EaF3Ko/OZY64A5RcaMdL6eqVZWR5Kpl1akVlpdEEZTacmhviLxZqvTQTpmYCYr/+VsV9SXUjA03rnFBt63tkoELykQAcwUi7UR8I9M4o3mv5Tm6+P+gXZri/CrjaiQsfjaUhpXfo6Wm1ZVctXSMvQaFdZBPwf2bGVF998B2FL3VWRUw4rXieJK6aLTqFhWGWmmV27OSbkAliAMl6l+R1MlHUmvk5X/cja9Rs21KytYUp7Pjauq0KrFS3AqxHdvCoXCMu7AWX2QhnJg2obyX2om2D49giQNK2iX3DLSyNNJ1FeY+PgFc5hfbOC7mkdQofCSdCHvSkuxDvpxeINoVBLzikUAky71FfkjZrPqp7AzsDDzzSvJm7IX6EzI12vQpFi/ZSq/P6X5OVyzomJG1OHJNBHATCGnL4SigKSEMQb6gcgMjKIocRewO9tAkom84zHqNXyp8gjnq47jVXR83/thHt/dxitHIwlodcXGGbmWngmSxDkVUesrTKPuBpsu9Frxs5FNokUsZypJklKahdFpVLFlHWF6EX9pplB0B5IhOICKMDIqPLpCnL4Qg74QKgkqE6y+aDPMAdIbwGjCXt7T/AAAu6rvIL9sDgrQP1THYHH5zFxLz4TaQsM5u77y9Bpqi6ZnFU6jXs2HN9SKZbAsUV2QS4V5ev4sJSKVnkil+ZOfwCtMDhHATKEzXagjy0ceXRGKpKHNFlk+KjflJLwmemYnUnPaxrmh/ffkB6w49JUcqLmda5ZXcP3KCvL0GorydMwpTGCZSxjX6hrLqLdPx2RevVbFzWuqKDDqWFs7s9/1Txcb5s68bdOjSaW2zUxeXpvpxDbqKRRN4DUGRk/grU4iMIgtIXlaiKxPpfZOwuTrYH3HHwDYWvfPhNWRX+75JXnML8lDVpRpvbyRTSwGLXVjlHSfX2JEr1XhD6ZeIn0qaNUSN62uojQ/8vOypDyf7Y19uP2ipk2mlJr0M6ZlwEQsKczAiABm+hIzMFPIMUoXakVRhiXwJj7Va8+pRkaNTvbEzpuKS5sfQKMEaDWv51ThZefcL4KX9FlVM3YfKo1axeI4unxnA5Ukce2KihHb/zVqFSurLZkb1FmqCnJZU2tJujvwdDRbZl8gta3UZVO0hVpIPzEDM4XO1IDpAyIzMAOeIJ5AGLVKojyJdwKySos9p4pCXyuF3mZc+rKkx1dj38XC/jeQUbOl7ispz+YIYxu+dXosSytNHGzP7qJ2kgRXLStjXsm5u9JWVVvY02wjGB69fPpky9NrqK8wsazSRMFQntG6OQVsO9XH8e5BlMwMa8T4Fpblcbx7EG8gvTNVBQYtC0tnz07BZJeQcrTqGddaYTYRAcwUcp4zA1MSm32pMOegSbImwIBhLoW+Vgq8LbRazk/qHMP7HR2ouJV+44KkziPEZ2mFacJ6PxXmXIrydPS7kitSOBXes6hkzG3fuTo1SytNU1pZWCVJ1JUYWVZpoq7IiOqsrar5OVquXl7BqhoLbzb00uXwTdnYIPI9WVCSx+LyfKoLcpEkiZJ8PS8fObfEfCrWzSmcVYmpphwNapVEeIxeQ2MRsy/TmwhgpkggJOMZepcVa+SoL6G9J5L/klD9l7PYcucyn60JNXU826quv1HsOY1XY2ZH7WeTPo8wscjWaUtcx9ZXmHj7ZN/kDihJF8wrYs0Eybpraws42O6Y9NmOQqOOZZUm6itMGONoiFdhzuVDG2o43j3ItlN9DPpCkzY2nUbF/BIji8tN1BYazqn/sazSzLGuwVgyf6ry9BqWTjC7N9NIkoQ5Vzthx+ezifyX6U0EMFMkunwEZ3YhDWpLzjRwTCL/JcqWm9pW6tzgABtb/xuAbXP+Eb9mdv3xm2pzigyxJY2J1FeY2H6qHznT6x1nWV1rYeP8ogmPsxh0zC/J45TVlfYx6DQqFpbmsbzKnHD5AThTuHFBaR57mgfY25K+5S6NKjITtLgsn7nFxgl3F16+pJRHd7aM2a04EWvnWGZlkTSLIZkARszATGcigJki0eUjOLMLqS1kwReU0aqllN4JRHciFbtPUj54mO68ZQnlr1zYEu13tIjDZTclPQ4hPmcXrhtPtCZMc1963p2nQ31FPpsWlcR9/Lo5BWkPYOaVGLlmeUVaCipq1So2zi9iWZWJbSf7aOhJPD9GkiLLU8V5OhaW5jO/1JhQj5tCo471cwvZebo/wdGPlKNVRzrEz0LJ1IIpFTMw05oIYKaIc2iKWhP2khOO/DE/5jYCXiotuSm9Y+o3zCMkaTGE7Hz44CcYyKnheMk1HCu5GkduzbiPLXE1sKLnCSDS70iRRGOxyVRg0DK3KLHlwqUV5qwJYOaVGLlqaXlC+RWVllwqLTl02tOTb5KjVXNFfVnaq0GbcrRcs6KC1bUWtjT00j1KfoxRH0n6LDDosBi0FBi0WAw6LLnapHPYojbMLeBEz2DCswjDrao2z9rmgIkm4xp0akw5ogLvdCYCmCly9hbqgMrAKUfkD14qy0cAAU0ef13+IKu6/8qC/jco8LWxse23bGz7LZ35KzhecjUNxVfh01pGPlBRuKzpfiQUjhdfRYd5TUrjECY23tbpsWRLTZiqglyuXVFxTmJsPNbWFtBp70rLOC5dVExeHHkuyaow53LbUH6M3ROkwKiNBSyTGRxo1CouX1LKX/e2J/V4rVqaMCdpJku0FozIf5n+RAAzRWI7kKJdqPUltNtTT+CN6jKtpMu0Em3Yw/z+N6nvfZ5a+y4qBw9ROXiI9zT9O82WCzleeg2NBZcQVuewqO8Vqpz7Car0vDX3CymPQRifTqNKKrkyWhMmk1uqS/L13LiqMunuuQtK87AYtNg9wYkPHsfcYgPLKid/iSSaHzPVagoN1FeYONblTPixyyrN5Opm5+wLJF4LplTkv0x7IoCZImdqwERmYGyqIgIhGZ1GRUl++n6RgmoDx0uv4XjpNRgCfSzue5l664uUuY8xf+At5g+8hV9t5GTR5cyxvwPArupP4NKXp20MqdBrVeg16hE5Q9ng0kXFtNm8NPW5kz7H0sqJt06P99hMBTAWg5b3raka0TE7UZIUmR1443jyxRZ1GhVX1Cdf52i6eM+iEpr73QnVhlFJEmvnzN7ZF4gsAaokKe6EdzEDM/2JSrxTJNpGILqFukuJVMmstuROWnVbj66YfZUf4bHVj/D7NX/mnepP4tBXoA+7WW59JtbvaG/lRyfl+sm4YF4RG+dNvLtlKhn1alZVW7hxVSWrapJ79y9JsDqFyrTRmjBTrdCo45Z11XFtTZ7IskpTSkHQxQuKZ0XOQq5OzcULihN6zOLy/JQaGs4EKpWEKTf+n1MRwEx/IoCZAt5AGF9wZA2YlmDkhTDV/Jd42Qx1bJ/zeX637kkeX/H/OFj2PnoNC3l5wXdi/Y4yrThPx+pqC/UV+VnVzXhNbQEatQqVSuLyJWW8Z3FJwkWK5xYZ4946PZapXtKoskRqpaQraNCqVayqTi4ArCrIZWWSj52OlleZ4/7bIEmRBGAh/mWkPL1mUvOohKkhApgpsK9tIPZ5NIBp9EX63FSnIf8lIZKKTtNqXlvwTf5vzWO0W9ZP7fXH8Z5FpahUEpIkcUGWzMLotapzXjjX1hZww6rKhHbBxFu4bjz1FaYp60W1oDSP969NbdloNKtqLGgSTALWqiXeW182qyrLAlxRXxbX7sS6YiNFeSKfA8CSG9+bBJH/MjOIAGaSeQNh9rXaY19Ha8B0hi3kaFUUZ9FMQyYtKM2jdtj24oWleWnNDUrWqmrLqHkr80vyuHVddVzv4gqNOuYkuHV6NNGaMJNtda2F61dWpLwteDRGvYbF5Yk1qdw4vyjl2avpKFIbZuKZlfPq/n97dx7c5HXvDfz7aLeszZItyfKGF2xjwAZMMM5CcoMvy+TmksBNKelNaZIhEwqdNjSZhE4bIJ0pGTqTbkObe9+2Ye5MGxI6pXmbpn0TSKBvExOKgbIlFBwSm+AFDHi3ZUvn/iEsIrxbj/Ro+X5mNBg9j6RzfDzST+c55/dLnqKN47FOcAaGl48SAwOYCDv86VV4B29ufx3Kwtsi7Mi2GZPuW+VItGoJi25JjCZJEm6fQKbXSNKoJMzNtY163GUxYPWCHKSPE2hNZev0aMoyI3cZRZKAu6an419KnBH9u6zMS5vwJTi31YC5k0j8l2gWTLMjbYwP5ey0FGRao3MZOh5MdCs1A5jEwAAmgjr7BnCi8frNO4QfqQOBujbNIg3Zdr7xAMC8vLQRFyAWZJjgsSn3RjMzywKjbuwZFotBiy/Nz8a09JFnWHQaFcpkXLsylBNGbmqVhGWz3Jg/LfLf5h0mPaY5UifUpn8tc00p70yi0KjH3nl1WxTGK55MNJkdSwgkBgYwEXT4wtWQ2ibGgatQCx98QsJl2GTJ/xLvzAbNmG/CtxdObjeGXFSShMrciX046DVqrKjIGnGR6UyPRdaMsUM5YeSk16rw4NwslLqjt0i4cgJbfhfk25HOtR3B3DC3clr0mJY+fiCYTKwp2nFn98wGzbhfTCg+MICJkOs9Xpy+FJqMamgB7xVYodfpxpwaThaLijPGTI6WYzcixx79QK/YZZrw9XQgsIVz8QwXFhWnB99AJ1N1ejLkrDRsNmjwUGVO1H/HOXbjmAsp0816zi58waLi9GELqufn8fdzK7VKgnmcXXO8fJQ4GMBEyKFP2uC7pbLs0PqXZmFHdlpK0q9/ybEbUTyB2YQ7iqK/Fmaql1Iq8+z4t3IPtGoJ+empk67PMhFy5YRJN+nwpdtyFFssPW+UtPcqScKSsontwEkWRp0Gd02/ORtpM2ox3WlSsEWxa7x1MAxgEgcDmAi40tWPj5s7h90/NAPTItKS/vKRSpJwT8nEKhpnWlNQkBG9qfL89NSwPtSLnCY8ND8noutJws0Jk52Wgofmy5fjZSpKXGaYDcOn8ivz0vghM4KZHguybuSGmZ9nT+q1QWMZLxcM178kDgYwEfBBfRtGymadcssMTDIrz7ZOan1DdaFj0snjpmoiW1fH47IYkGWL3BiHkxOmxG0OuzSAHFQj7PKyp+qwsICXRkYiSRIWlzphSdFiRqa866ASyfgBDIPjRMEARmbN7X2ob+0a8Zi6K1CN95rGkdRpv1N0alRPcou002zAdGfk37SzbCnRTy44BZPJCaOSJGSY9ZidZcXSmW4sn+WOSI6XqZiVZQ0ucpYkoKbMFTNti0UOkx7/UZnN39EYrGMks7OmaBUP3Ek+XIots/fPXxn1mL43MAMzaHRHLaNqLLq90DGlN5HqQgfOt3ZNuFjbVFTGUUr2skwrPr3SM+x+S4oWbosBbqsebmsKnGb9lKtIR5peo8bsLCvqPruGihxbRGetEkUyf/mZiLFmYDj7klgYwMio8WoPGq4O/0AZYh4IrIFRWTzRalLMcVoCMwFTYU/VoTTTjDO37O6SS7pJh4I42pZamJEKa4oWNmMgYHFZDXBbDLIUXoymObk2fHK5C3cotGWeEsvQVuqRvudw/Utiia93uhj3Qf3osy/9gz6k+9sACdDbs9AfxXbFknvCzPK6MN+Bs82dw3Z4yaEyzx5XO8M0ahUeuzNf6WaEzWLQ4qH5ObLmy6HkpVWrYNJr0Nk3OOwYZ2ASC98xZPLJ5S5cut436vHLV6/DKgVmZ4Q5OWdgSt3msC8RWI1azMqSP+GaJUWL0knW6CH5xNusEcW2kS6zSRKLOCYaBjAyEELgg/q2Mc/pa2sEAPTCAK86fi5TyEWnUeGu4oltmx7PgnwHtGp5Z0oq89K4LZUoQYyUf8mWoh2xMCvFLwYwMvhnSxcud459UWiw/RIAoF2bjqjtB44ht02zT6hy80SY9BqUZ9tkeS4AMOrUmCljdlsiUtZIC3l5+SjxMIAJk98vUDvG2hcA6BvwwdDbEvjZMHphtkRlM2oxb4yqzlNx2zS7bGsm5uTYYnaXDhFN3kjZeJ0MYBIO37XDdKapA9d6BsY85+K1XrilawCAXoMzGs2KKXcXZ8ietyJFp8ZcGeoM6TQqVESgXhERKWekOmbcgZR4GMCEYdDnx6FPxl77AgAXr/XALV0FAHTp5FkHEi8KMlJRkBGZmi3z8tLCTko1O8vKxFZECcZ2SzI7SQokw6TEInsAs3XrVkiSFHIrLS0NHu/r68OGDRvgcDhgMpmwatUqtLS0hDxHQ0MD7rvvPhiNRjidTjzzzDMYHBy+JU5pJz9vH3Gr3q0uXuuF68YMTJcueWZg5uba8G/lkdtxZdCqUZk39cRzapWEeWE8nohik06jQqr+5hcTe6qO2/QTUET2Ls6cORP79u27+SKamy/z1FNP4U9/+hP27NkDq9WKjRs3YuXKlXj//fcBAD6fD/fddx/cbjc++OADNDU14atf/Sq0Wi1+8IMfRKK5U+Id9OPvn14d97zOvgG0dXvh1t2YgdEn/gyMQavGkpkuFEZo5uWL5uTYcLzxGrr7fZN+7IxMi2wLi4kotthSdOju7wXA2ZdEFZF3b41GA7fbPez+9vZ2/OpXv8Jvf/tb3HvvvQCAV155BTNmzMChQ4ewcOFCvP322zhz5gz27dsHl8uFOXPm4Pvf/z6effZZbN26FTrd6HUuoul44/UJfWie/LwdAOBRBf5N9BmYLFsKls12R63KsU6jwr2lTpy+1IErXV509g2MmIHzVpIEzOfsC1HCshq1+Px6IIDh+pfEFJEA5ty5c/B4PDAYDKiursb27duRm5uLuro6DAwMoKamJnhuaWkpcnNzUVtbi4ULF6K2thazZ8+Gy3Vzt87SpUuxfv16nD59GnPnzh3xNfv7+9Hff3Mrc0dHZNLNA4FdRUc+G3/2ZdDnx6nPOyDBjwwMrYGJTLr0ErcZbd1eXBlnO3ekSFJgZ1B1gSPq+VSKnGYU3Sj06B3042q3F1e6+tHW7cXV7n60dXmHXeorcpqQlhobwTARye+LO5G4hToxyR7AVFVVYdeuXSgpKUFTUxO2bduGu+66C6dOnUJzczN0Oh1sNlvIY1wuF5qbmwEAzc3NIcHL0PGhY6PZvn07tm3bJm9nRnG88Tr6B/zjnvfP1i70DviQp++BGj4ISOjRyh/A6DQqLCrOwKdXuvHOmZbxHyCzVL0aS2e6kedQPkGfTqOC22qA2xr6htU34MPVbi/aurxo6+5HGfO+ECW0oWR2Q9XYKfHIHsAsX748+HN5eTmqqqqQl5eH119/HSkpkas0u3nzZmzatCn4/46ODuTk5ETktSaycFcIgX80XgcA3On0ApeBHq0dfpX8k163Fzpg0mtQ4jbj/5+7gr6Bya8HmapcuxHLZrljPhW8QauGx5YCD6sdEyWFoWR2dpOOeZ4SVMRH1Wazobi4GOfPn4fb7YbX68X169dDzmlpaQmumXG73cN2JQ39f6R1NUP0ej0sFkvITUlN7X1o7eyHWiWhwha4DhuJ9S9Oix4VN7LSatWqiNQJGolKknB7oQMr52XFfPBCRMlnqB6Si7MvCSviAUxXVxfq6+uRmZmJyspKaLVa7N+/P3j87NmzaGhoQHV1NQCguroaJ0+eRGtra/Ccd955BxaLBWVlZZFurmyGZl9K3WY4fIFcMXLvQJIk4N5SZ8iak/JsG1QRLlVgNmiwqjILVQWOuKreTETJw6BVI0Wn5vqXBCb7V+enn34a999/P/Ly8nDp0iVs2bIFarUaa9asgdVqxeOPP45NmzbBbrfDYrHgG9/4Bqqrq7Fw4UIAwJIlS1BWVoZHHnkEO3bsQHNzM7773e9iw4YN0OvjI5Lu6hvEuctdAICKbBtSrwaCMbmT2M3OsiLTGnpJxJqiRX5GKupbu2R9rSEFGalYUuZGio7J34gottlStAxgEpjsAczFixexZs0atLW1ISMjA3feeScOHTqEjIzAh/ePfvQjqFQqrFq1Cv39/Vi6dCl+/vOfBx+vVqvx5ptvYv369aiurkZqairWrl2LF154Qe6mRszJz9shRGBLcYZZD1PzZQDyXkIy6tS4o2jkBcFzc2wRCWAyrQb8e4WHsy5EFBfsqTou4E1gsgcwu3fvHvO4wWDAzp07sXPnzlHPycvLw1tvvSV306Ji0OcP5n6pyLYCAEzeoQBGvhmYO6enj5oCP8duRLpJhytdXtleb+g1GbwQUbwodJqgjnJaB4oeLs2W2bkbW6dNek0wE63Je+MSkl6eGZistBTM9FjHPEfuAoUFGanITjPK+pxERJE0LQZSO1DkMICRkRACx28s3i3PtgYX15q8VwDIMwOjVklYXDp+IDQj0yJbkUJJAm4vjEwCPiKiSOHsS2JjACOj5o6bW6dn3ZghUfv6YBgMZAWWYw3MvNw0OEzjX9PVqlWYKVOytlK3hdeRiYgopjCAkdHQ7EuJyxzcpZPb/ncAQJ/aDK86vOlMS4oWVQX2CZ9fkW1DuEtWNCoJtxc5wnsSIiIimTGAkUlX/yDO39j5U5FzY32KEKhu+D8AgFPuBxBuNHF3ccakMkpajVrkp4cXNJXn2KJWmJGIiGiiGMDI5OTFdvgF4LEagqXbC67+Fa7uj+BVpeBI1iNhPX9BRiqKnKZJP25uztQrLuu1KlTlT3zGh4iIKFoYwMhg0H9z6/Scod0/QqC68b8BAMczv4Re7dQDCa1awj0lU1s/k+swwmGaWtXl+Xl22RYCExERyYkBjAzOtdzcOl1wY+t04dUDcHb/E16VEXVZ/xnW81cVOIJ1PaZiqFbSZJj0GszNnfzjiIiIooEBTJi+uHV6drY1sG1P+INrX455vow+rW3Kz+8w6TAvd+qzN0BgS7VeO7mhXljgYAVXIiKKWfyEClPo1unAtuWitgPI6DmHfnUq6jwPh/X8/1LiDDuXgU6jQlnmxLdU21N1sm3BJiIiigQGMGEamn0pdplg1GkCsy831r4c86xBv3bsjLljmZFpQY5dnuy3c3ImvqX6jiJHSIVrIiKiWMMAJgxf3Do958Y6k+K2/UjvqUef2oSjN2ZfJClQ2n0yMykGrRqLiuXLfmsz6ia0pTrTakCR0yzb6xIREUWC7MUck8nJzwNbpzOtBjgtBkjCh4XBtS8Po18TCATy01OxYk4WAMDvF/D6/Bjw+THoExjw+THgFxgY9GPQ74d3UGDQ70eaUReY0ZFRRbYNn1zuHvOcO6ezZAAREcU+BjBTNOj34+TF0K3TxVf2wdF7AX1qM4561gTPLXXfXE+iUkkwqNSKbE/OcxhhT9XhavfIVapZsJGIiOIFLyFN0fmW0KrTkvChqvGXAIC6rK/Aqwlsp9ZpVCjMiI2KqJIkjVqlmgUbiYgonjCAmaLjF68DAGZnBbZOl1x+G47eT9GrseJ45urgedOdJmhiaDvyjEwzdJrh7WHBRiIiiiex88kaRz653IWWjn6oJQmzsiyQxCAWjjD7AgR2EsUSvUaNslu2SLNgIxERxRsGMFOw/+NWAECxO7B1uvTy/0NaX0Ng9sX9peB5ZoMG2WkpSjVzVHNuqVLNgo1ERBRvGMBMUmtHH+o+uwYgsKvni7MvR7IewYDm5nqXUrcFUpgVqCMhLVWHPEdgsS4LNhIRUTxiADNJv/mwAT6/QKbVAJfFgBmtb8HWdxE92jT8I/OhkHNnZMZuPpU5N6pUs2AjERHFIwYwk3S0ITD7MifHBpV/EAsbfwXgxuyL+uYWZKdFD4cpdhfFTnMYkZ2WwoKNREQUl5gHZpL+57EF2PlePfoGfJhx+f/C2n8J3Vo7/uH+j5Dzvpj7JRZJkoR/n+NhwUYiIopL/PSaJEmSUOI2Q4tBVN2Yffl71loMqm8u1lVJEkrdsXv5aIhew0tHREQUnxjATNHM1j/C2t+Ebq0DJ9wrQ47lOlKQqufkFhERUaQwgJkCyefFgouvAAD+nr0WPrUh5His5X4hIiJKNAxgpiDrwu9g6W9Gly4DJ1wPhhwLlA4wjfJIIiIikgMDmMka7Ef+Ry8DAA5nf23Y7EthhokLY4mIiCKMn7STdfR/YOhtRqfOiVOuFcMOl/HyERERUcQxgJkMIYDjvwUAHM5+FD5VaJ4Xs0GDHHvslQ4gIiJKNNwqMxmSBDz6Fj7683/htL5m2OFilzkmSwcQERElGs7ATJY2BReL1sCn0g07xN1HRERE0cEARibpZj0yzLFbOoCIiCiRMICRSVkMF24kIiJKNAxgZCBJQEmM1z4iIiJKJAxgZJCTZoSJpQOIiIiihgGMDLh4l4iIKLoYwIRJq5ZQ5GTpACIiomhiABOmwgwTdBr+GomIiKKJn7xh4uUjIiKi6GMAE4ZUvRq5dqPSzSAiIko6DGDCUOK2QKVi6QAiIqJoYwAThhluJq8jIiJSAgOYKXKYdHBaDEo3g4iIKCkxgJkiLt4lIiJSDgOYKZAAlPDyERERkWIYwExBtj0FFoNW6WYQERElLQYwU1CYwcy7RERESmIAMwVaNX9tRERESuInMREREcUdBjBEREQUdxjAEBERUdxhAENERERxhwEMERERxR0GMERERBR3GMAQERFR3InpAGbnzp2YNm0aDAYDqqqqcPjwYaWbRERERDEgZgOY1157DZs2bcKWLVtw9OhRVFRUYOnSpWhtbVW6aURERKSwmA1gXnrpJaxbtw6PPvooysrK8PLLL8NoNOLXv/610k0jIiIihcVkAOP1elFXV4eamprgfSqVCjU1NaitrR3xMf39/ejo6Ai5ERERUWKKyQDmypUr8Pl8cLlcIfe7XC40NzeP+Jjt27fDarUGbzk5OdFoKhERESkgJgOYqdi8eTPa29uDt8bGRqWbRERERBGiUboBI0lPT4darUZLS0vI/S0tLXC73SM+Rq/XQ6/XR6N5REREpLCYDGB0Oh0qKyuxf/9+PPDAAwAAv9+P/fv3Y+PGjRN6DiEEAHAtDBERURwZ+twe+hwfTUwGMACwadMmrF27FvPnz8eCBQvw4x//GN3d3Xj00Ucn9PjOzk4A4FoYIiKiONTZ2Qmr1Trq8ZgNYFavXo3Lly/j+eefR3NzM+bMmYO//OUvwxb2jsbj8aCxsRFmsxmSJMnWro6ODuTk5KCxsREWi0W2540H7Hvy9T1Z+w2w78nY92TtNxBbfRdCoLOzEx6PZ8zzJDHeHA2F6OjogNVqRXt7u+KDHG3se/L1PVn7DbDvydj3ZO03EJ99T5hdSERERJQ8GMAQERFR3GEAM0l6vR5btmxJyi3b7Hvy9T1Z+w2w78nY92TtNxCffecaGCIiIoo7nIEhIiKiuMMAhoiIiOIOAxgiIiKKOwxgiIiIKO4wgJmknTt3Ytq0aTAYDKiqqsLhw4eVblLEbd26FZIkhdxKS0uVbpbs/vrXv+L++++Hx+OBJEn4wx/+EHJcCIHnn38emZmZSElJQU1NDc6dO6dMY2U2Xt+/9rWvDfsbWLZsmTKNldH27dtx2223wWw2w+l04oEHHsDZs2dDzunr68OGDRvgcDhgMpmwatWqYYVm49FE+n7PPfcMG/cnn3xSoRbL4xe/+AXKy8thsVhgsVhQXV2NP//5z8HjiTrewPh9j7fxZgAzCa+99ho2bdqELVu24OjRo6ioqMDSpUvR2tqqdNMibubMmWhqagre/va3vyndJNl1d3ejoqICO3fuHPH4jh078NOf/hQvv/wyPvzwQ6SmpmLp0qXo6+uLckvlN17fAWDZsmUhfwOvvvpqFFsYGQcPHsSGDRtw6NAhvPPOOxgYGMCSJUvQ3d0dPOepp57CH//4R+zZswcHDx7EpUuXsHLlSgVbLY+J9B0A1q1bFzLuO3bsUKjF8sjOzsaLL76Iuro6HDlyBPfeey9WrFiB06dPA0jc8QbG7zsQZ+MtaMIWLFggNmzYEPy/z+cTHo9HbN++XcFWRd6WLVtERUWF0s2IKgBi7969wf/7/X7hdrvFD3/4w+B9169fF3q9Xrz66qsKtDBybu27EEKsXbtWrFixQpH2RFNra6sAIA4ePCiECIyxVqsVe/bsCZ7z0UcfCQCitrZWqWZGxK19F0KIu+++W3zzm99UrlFRkpaWJn75y18m1XgPGeq7EPE33pyBmSCv14u6ujrU1NQE71OpVKipqUFtba2CLYuOc+fOwePxoKCgAF/5ylfQ0NCgdJOi6sKFC2hubg4Zf6vViqqqqqQYfwA4cOAAnE4nSkpKsH79erS1tSndJNm1t7cDAOx2OwCgrq4OAwMDIeNeWlqK3NzchBv3W/s+5De/+Q3S09Mxa9YsbN68GT09PUo0LyJ8Ph92796N7u5uVFdXJ9V439r3IfE03jFbjTrWXLlyBT6fb1g1bJfLhY8//lihVkVHVVUVdu3ahZKSEjQ1NWHbtm246667cOrUKZjNZqWbFxXNzc0AMOL4Dx1LZMuWLcPKlSuRn5+P+vp6fOc738Hy5ctRW1sLtVqtdPNk4ff78a1vfQt33HEHZs2aBSAw7jqdDjabLeTcRBv3kfoOAA8//DDy8vLg8Xhw4sQJPPvsszh79ix+//vfK9ja8J08eRLV1dXo6+uDyWTC3r17UVZWhuPHjyf8eI/WdyD+xpsBDI1r+fLlwZ/Ly8tRVVWFvLw8vP7663j88ccVbBlFy5e//OXgz7Nnz0Z5eTkKCwtx4MABLF68WMGWyWfDhg04depUQq7vGs9ofX/iiSeCP8+ePRuZmZlYvHgx6uvrUVhYGO1myqakpATHjx9He3s7fve732Ht2rU4ePCg0s2KitH6XlZWFnfjzUtIE5Seng61Wj1sNXpLSwvcbrdCrVKGzWZDcXExzp8/r3RTomZojDn+AQUFBUhPT0+Yv4GNGzfizTffxHvvvYfs7Ozg/W63G16vF9evXw85P5HGfbS+j6SqqgoA4n7cdTodioqKUFlZie3bt6OiogI/+clPkmK8R+v7SGJ9vBnATJBOp0NlZSX2798fvM/v92P//v0h1w+TQVdXF+rr65GZmal0U6ImPz8fbrc7ZPw7Ojrw4YcfJt34A8DFixfR1tYW938DQghs3LgRe/fuxbvvvov8/PyQ45WVldBqtSHjfvbsWTQ0NMT9uI/X95EcP34cAOJ+3G/l9/vR39+f0OM9mqG+jyTmx1vpVcTxZPfu3UKv14tdu3aJM2fOiCeeeELYbDbR3NysdNMi6tvf/rY4cOCAuHDhgnj//fdFTU2NSE9PF62trUo3TVadnZ3i2LFj4tixYwKAeOmll8SxY8fEZ599JoQQ4sUXXxQ2m0288cYb4sSJE2LFihUiPz9f9Pb2Ktzy8I3V987OTvH000+L2tpaceHCBbFv3z4xb948MX36dNHX16d008Oyfv16YbVaxYEDB0RTU1Pw1tPTEzznySefFLm5ueLdd98VR44cEdXV1aK6ulrBVstjvL6fP39evPDCC+LIkSPiwoUL4o033hAFBQVi0aJFCrc8PM8995w4ePCguHDhgjhx4oR47rnnhCRJ4u233xZCJO54CzF23+NxvBnATNLPfvYzkZubK3Q6nViwYIE4dOiQ0k2KuNWrV4vMzEyh0+lEVlaWWL16tTh//rzSzZLde++9JwAMu61du1YIEdhK/b3vfU+4XC6h1+vF4sWLxdmzZ5VttEzG6ntPT49YsmSJyMjIEFqtVuTl5Yl169YlROA+Up8BiFdeeSV4Tm9vr/j6178u0tLShNFoFA8++KBoampSrtEyGa/vDQ0NYtGiRcJutwu9Xi+KiorEM888I9rb25VteJgee+wxkZeXJ3Q6ncjIyBCLFy8OBi9CJO54CzF23+NxvCUhhIjefA8RERFR+LgGhoiIiOIOAxgiIiKKOwxgiIiIKO4wgCEiIqK4wwCGiIiI4g4DGCIiIoo7DGCIiIgo7jCAISIiorjDAIaIiIjiDgMYIiIiijsMYIiIiCjuMIAhIiKiuPO/YMZDeA55PwAAAAAASUVORK5CYII=\n" + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "draw_result(\n", + " task_name=\"ParallelContinuousCartPoleSwingUp-v0\",\n", + " log_file=\"eval.csv\",\n", + " log_key=\"mean_reward\",\n", + " params=dict(freq_rate=1,\n", + " real_time_scale=0.02,\n", + " integrator=\"euler\",\n", + " parallel_num=3),\n", + " group_key=('transition', 'oracle'),\n", + ")" + ] } ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", - "version": 2 + "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", - "pygments_lexer": "ipython2", - "version": "2.7.6" + "pygments_lexer": "ipython3", + "version": "3.8.13" } }, "nbformat": 4, - "nbformat_minor": 0 -} + "nbformat_minor": 1 +} \ No newline at end of file From 5758629152464f73107efef49b1ed04e903e7b09 Mon Sep 17 00:00:00 2001 From: frank Date: Thu, 6 Apr 2023 16:10:19 +0800 Subject: [PATCH 65/68] :tada: multi-process loader --- cmrl/examples/conf/transition/oracle.yaml | 2 +- cmrl/models/causal_mech/base.py | 127 +++++++++++----------- cmrl/models/causal_mech/oracle_mech.py | 73 ++++++------- cmrl/models/causal_mech/util.py | 56 +++++----- cmrl/models/data_loader.py | 52 ++++----- 5 files changed, 146 insertions(+), 164 deletions(-) diff --git a/cmrl/examples/conf/transition/oracle.yaml b/cmrl/examples/conf/transition/oracle.yaml index 26d8ea6..7588695 100644 --- a/cmrl/examples/conf/transition/oracle.yaml +++ b/cmrl/examples/conf/transition/oracle.yaml @@ -56,7 +56,7 @@ mech: patience: 5 longest_epoch: -1 improvement_threshold: 0.01 - batch_size: 256 + batch_size: 1024 # ensemble ensemble_num: 7 elite_num: 5 diff --git a/cmrl/models/causal_mech/base.py b/cmrl/models/causal_mech/base.py index 4fe5072..4a46b5a 100644 --- a/cmrl/models/causal_mech/base.py +++ b/cmrl/models/causal_mech/base.py @@ -4,6 +4,7 @@ import pathlib from functools import partial import copy +from multiprocessing import cpu_count import numpy as np import torch @@ -31,11 +32,11 @@ class BaseCausalMech(ABC): """ def __init__( - self, - name: str, - input_variables: List[Variable], - output_variables: List[Variable], - logger: Optional[Logger] = None, + self, + name: str, + input_variables: List[Variable], + output_variables: List[Variable], + logger: Optional[Logger] = None, ): self.name = name self.input_variables = input_variables @@ -51,11 +52,11 @@ def __init__( @abstractmethod def learn( - self, - inputs: MutableMapping[str, np.ndarray], - outputs: MutableMapping[str, np.ndarray], - work_dir: Optional[Union[str, pathlib.Path]] = None, - **kwargs + self, + inputs: MutableMapping[str, np.ndarray], + outputs: MutableMapping[str, np.ndarray], + work_dir: Optional[Union[str, pathlib.Path]] = None, + **kwargs ): raise NotImplementedError @@ -67,8 +68,7 @@ def forward(self, inputs: MutableMapping[str, np.ndarray]) -> Dict[str, torch.Te def causal_graph(self) -> torch.Tensor: """property causal graph""" if self.graph is None: - return torch.ones(len(self.input_variables), len(self.output_variables), dtype=torch.int, - device=self.device) + return torch.ones(len(self.input_variables), len(self.output_variables), dtype=torch.int, device=self.device) else: return self.graph.get_binary_adj_matrix() @@ -81,31 +81,31 @@ def load(self, load_dir: Union[str, pathlib.Path]): class EnsembleNeuralMech(BaseCausalMech): def __init__( - self, - # base - name: str, - input_variables: List[Variable], - output_variables: List[Variable], - logger: Optional[Logger] = None, - # model learning - longest_epoch: int = -1, - improvement_threshold: float = 0.01, - patience: int = 5, - batch_size: int = 256, - # ensemble - ensemble_num: int = 7, - elite_num: int = 5, - # cfgs - network_cfg: Optional[DictConfig] = None, - encoder_cfg: Optional[DictConfig] = None, - decoder_cfg: Optional[DictConfig] = None, - optimizer_cfg: Optional[DictConfig] = None, - scheduler_cfg: Optional[DictConfig] = None, - # forward method - residual: bool = True, - encoder_reduction: str = "sum", - # others - device: Union[str, torch.device] = "cpu", + self, + # base + name: str, + input_variables: List[Variable], + output_variables: List[Variable], + logger: Optional[Logger] = None, + # model learning + longest_epoch: int = -1, + improvement_threshold: float = 0.01, + patience: int = 5, + batch_size: int = 256, + # ensemble + ensemble_num: int = 7, + elite_num: int = 5, + # cfgs + network_cfg: Optional[DictConfig] = None, + encoder_cfg: Optional[DictConfig] = None, + decoder_cfg: Optional[DictConfig] = None, + optimizer_cfg: Optional[DictConfig] = None, + scheduler_cfg: Optional[DictConfig] = None, + # forward method + residual: bool = True, + encoder_reduction: str = "sum", + # others + device: Union[str, torch.device] = "cpu", ): BaseCausalMech.__init__( self, name=name, input_variables=input_variables, output_variables=output_variables, logger=logger @@ -164,9 +164,9 @@ def build_optimizer(self): assert self.network, "you must build network first" assert self.variable_encoders and self.variable_decoders, "you must build coders first" params = ( - [self.network.parameters()] - + [encoder.parameters() for encoder in self.variable_encoders.values()] - + [decoder.parameters() for decoder in self.variable_decoders.values()] + [self.network.parameters()] + + [encoder.parameters() for encoder in self.variable_encoders.values()] + + [decoder.parameters() for decoder in self.variable_decoders.values()] ) self.optimizer = instantiate(self.optimizer_cfg)(params=chain(*params)) @@ -175,10 +175,9 @@ def build_optimizer(self): def forward(self, inputs: MutableMapping[str, torch.Tensor]) -> Dict[str, torch.Tensor]: batch_size, _ = self.get_inputs_batch_size(inputs) - inputs_tensor = torch.zeros(self.ensemble_num, batch_size, self.input_var_num, self.encoder_output_dim).to( - self.device) + inputs_tensor = torch.zeros(self.ensemble_num, batch_size, self.input_var_num, self.encoder_output_dim).to(self.device) for i, var in enumerate(self.input_variables): - out = self.variable_encoders[var.name](inputs[var.name]) + out = self.variable_encoders[var.name](inputs[var.name].to(self.device)) inputs_tensor[:, :, i] = out output_tensor = self.network(self.reduce_encoder_output(inputs_tensor)) @@ -243,9 +242,9 @@ def get_inputs_info(self, inputs: MutableMapping[str, torch.Tensor]): return batch_size, data_shape[:-3] def residual_outputs( - self, - inputs: MutableMapping[str, torch.Tensor], - outputs: MutableMapping[str, torch.Tensor], + self, + inputs: MutableMapping[str, torch.Tensor], + outputs: MutableMapping[str, torch.Tensor], ) -> MutableMapping[str, torch.Tensor]: for name in filter(lambda s: s.startswith("obs"), inputs.keys()): # assert inputs[name].shape[:2] == outputs["next_{}".format(name)].shape[:2] @@ -255,9 +254,9 @@ def residual_outputs( return outputs def reduce_encoder_output( - self, - encoder_output: torch.Tensor, - mask: Optional[torch.Tensor] = None, + self, + encoder_output: torch.Tensor, + mask: Optional[torch.Tensor] = None, ) -> torch.Tensor: assert len(encoder_output.shape) == 4, ( "shape of `encoder_output` should be (ensemble-num, batch-size, input-var-num, encoder-output-dim), " @@ -273,7 +272,7 @@ def reduce_encoder_output( # mask shape [..., ensemble-num, batch-size, input-var-num] assert ( - mask.shape[-3:] == encoder_output.shape[:-1] + mask.shape[-3:] == encoder_output.shape[:-1] ), "mask shape should be (..., ensemble-num, batch-size, input-var-num)" # [*mask-extra-dims, ensemble-num, batch-size, input-var-num, encoder-output-dim] @@ -302,9 +301,9 @@ def forward_mask(self) -> torch.Tensor: return self.causal_graph.T def get_data_loaders( - self, - inputs: MutableMapping[str, np.ndarray], - outputs: MutableMapping[str, np.ndarray], + self, + inputs: MutableMapping[str, np.ndarray], + outputs: MutableMapping[str, np.ndarray], ): train_set = EnsembleBufferDataset( inputs=inputs, outputs=outputs, training=True, train_ratio=0.8, ensemble_num=self.ensemble_num, seed=1 @@ -313,17 +312,17 @@ def get_data_loaders( inputs=inputs, outputs=outputs, training=False, train_ratio=0.8, ensemble_num=self.ensemble_num, seed=1 ) - train_loader = DataLoader(train_set, batch_size=self.batch_size, collate_fn=collate_fn) - valid_loader = DataLoader(valid_set, batch_size=self.batch_size, collate_fn=collate_fn) + train_loader = DataLoader(train_set, batch_size=self.batch_size, collate_fn=collate_fn, num_workers=cpu_count()) + valid_loader = DataLoader(valid_set, batch_size=self.batch_size, collate_fn=collate_fn, num_workers=cpu_count()) return train_loader, valid_loader def learn( - self, - inputs: MutableMapping[str, np.ndarray], - outputs: MutableMapping[str, np.ndarray], - work_dir: Optional[Union[str, pathlib.Path]] = None, - **kwargs + self, + inputs: MutableMapping[str, np.ndarray], + outputs: MutableMapping[str, np.ndarray], + work_dir: Optional[Union[str, pathlib.Path]] = None, + **kwargs ): train_loader, valid_loader = self.get_data_loaders(inputs, outputs) @@ -377,10 +376,10 @@ def learn( self.save(save_dir=work_dir) def _maybe_get_best_weights( - self, - best_val_loss: torch.Tensor, - val_loss: torch.Tensor, - threshold: float = 0.01, + self, + best_val_loss: torch.Tensor, + val_loss: torch.Tensor, + threshold: float = 0.01, ) -> Optional[Dict]: """Return the current model state dict if the validation score improves. For ensembles, this checks the validation for each ensemble member separately. diff --git a/cmrl/models/causal_mech/oracle_mech.py b/cmrl/models/causal_mech/oracle_mech.py index 78defda..dda20c1 100644 --- a/cmrl/models/causal_mech/oracle_mech.py +++ b/cmrl/models/causal_mech/oracle_mech.py @@ -16,30 +16,30 @@ class OracleMech(EnsembleNeuralMech): def __init__( - self, - name: str, - input_variables: List[Variable], - output_variables: List[Variable], - logger: Optional[Logger] = None, - # model learning - longest_epoch: int = -1, - improvement_threshold: float = 0.01, - patience: int = 5, - batch_size: int = 256, - # ensemble - ensemble_num: int = 7, - elite_num: int = 5, - # cfgs - network_cfg: Optional[DictConfig] = None, - encoder_cfg: Optional[DictConfig] = None, - decoder_cfg: Optional[DictConfig] = None, - optimizer_cfg: Optional[DictConfig] = None, - scheduler_cfg: Optional[DictConfig] = None, - # forward method - residual: bool = True, - encoder_reduction: str = "sum", - # others - device: Union[str, torch.device] = "cpu", + self, + name: str, + input_variables: List[Variable], + output_variables: List[Variable], + logger: Optional[Logger] = None, + # model learning + longest_epoch: int = -1, + improvement_threshold: float = 0.01, + patience: int = 5, + batch_size: int = 256, + # ensemble + ensemble_num: int = 7, + elite_num: int = 5, + # cfgs + network_cfg: Optional[DictConfig] = None, + encoder_cfg: Optional[DictConfig] = None, + decoder_cfg: Optional[DictConfig] = None, + optimizer_cfg: Optional[DictConfig] = None, + scheduler_cfg: Optional[DictConfig] = None, + # forward method + residual: bool = True, + encoder_reduction: str = "sum", + # others + device: Union[str, torch.device] = "cpu", ): EnsembleNeuralMech.__init__( self, @@ -72,43 +72,32 @@ def set_oracle_graph(self, graph_data: Optional[numpy.ndarray]): if __name__ == "__main__": + from typing import cast + import gym from stable_baselines3.common.buffers import ReplayBuffer from torch.utils.data import DataLoader + from emei import EmeiEnv - from cmrl.models.causal_mech.reinforce import ReinforceCausalMech from cmrl.models.data_loader import EnsembleBufferDataset, collate_fn, buffer_to_dict from cmrl.utils.creator import parse_space from cmrl.utils.env import load_offline_data from cmrl.models.causal_mech.util import variable_loss_func from cmrl.sb3_extension.logger import configure as logger_configure - - def unwrap_env(env): - while isinstance(env, gym.Wrapper): - env = env.env - return env - - - env = unwrap_env(gym.make("ParallelContinuousCartPoleSwingUp-v0")) + env = cast(EmeiEnv, gym.make("ParallelContinuousCartPoleSwingUp-v0")) real_replay_buffer = ReplayBuffer( int(1e6), env.observation_space, env.action_space, "cpu", handle_timeout_termination=False ) - load_offline_data(env, real_replay_buffer, "SAC-expert", use_ratio=0.1) + load_offline_data(env, real_replay_buffer, "SAC-expert", use_ratio=1) input_variables = parse_space(env.state_space, "obs") + parse_space(env.action_space, "act") output_variables = parse_space(env.state_space, "next_obs") logger = logger_configure("kci-log", ["tensorboard", "stdout"]) - mech = OracleMech( - "plain_mech", - input_variables, - output_variables, - logger=logger, - ) + mech = OracleMech("plain_mech", input_variables, output_variables, logger=logger, device="cuda:1") - inputs, outputs = buffer_to_dict(env.observation_space, env.action_space, env.obs2state, real_replay_buffer, - "transition") + inputs, outputs = buffer_to_dict(env.observation_space, env.action_space, env.obs2state, real_replay_buffer, "transition") mech.learn(inputs, outputs) diff --git a/cmrl/models/causal_mech/util.py b/cmrl/models/causal_mech/util.py index 0fcebb4..26480b3 100644 --- a/cmrl/models/causal_mech/util.py +++ b/cmrl/models/causal_mech/util.py @@ -9,17 +9,18 @@ from torch.utils.data import DataLoader from torch.optim import Optimizer from torch.distributions.von_mises import _log_modified_bessel_fn +from tqdm import tqdm from cmrl.utils.variables import Variable, ContinuousVariable, DiscreteVariable, BinaryVariable, RadianVariable def von_mises_nll_loss( - input: Tensor, - target: Tensor, - var: Tensor, - full: bool = False, - eps: float = 1e-6, - reduction: str = "mean", + input: Tensor, + target: Tensor, + var: Tensor, + full: bool = False, + eps: float = 1e-6, + reduction: str = "mean", ) -> Tensor: r"""Von Mises negative log likelihood loss. @@ -59,12 +60,12 @@ def von_mises_nll_loss( def circular_gaussian_nll_loss( - input: Tensor, - target: Tensor, - var: Tensor, - full: bool = False, - eps: float = 1e-6, - reduction: str = "mean", + input: Tensor, + target: Tensor, + var: Tensor, + full: bool = False, + eps: float = 1e-6, + reduction: str = "mean", ) -> Tensor: # Entries of var must be non-negative if torch.any(var < 0): @@ -77,7 +78,7 @@ def circular_gaussian_nll_loss( diff = torch.remainder(input - target, 2 * torch.pi) diff[diff > torch.pi] = 2 * torch.pi - diff[diff > torch.pi] - loss = 0.5 * (torch.log(var) + diff ** 2 / var) + loss = 0.5 * (torch.log(var) + diff**2 / var) if full: loss += 0.5 * math.log(2 * math.pi) @@ -90,10 +91,10 @@ def circular_gaussian_nll_loss( def variable_loss_func( - outputs: Dict[str, torch.Tensor], - targets: Dict[str, torch.Tensor], - output_variables: List[Variable], - device: Union[str, torch.device] = "cpu", + outputs: Dict[str, torch.Tensor], + targets: Dict[str, torch.Tensor], + output_variables: List[Variable], + device: Union[str, torch.device] = "cpu", ): dims = list(outputs.values())[0].shape[:-1] total_loss = torch.zeros(*dims, len(outputs)).to(device) @@ -121,14 +122,17 @@ def variable_loss_func( total_loss[..., i] = F.binary_cross_entropy(output, target, reduction="none") else: raise NotImplementedError + + if torch.isnan(total_loss[..., i]).any(): + raise ValueError(f"nan loss for {var.name}") return total_loss def train_func( - loader: DataLoader, - forward: Callable[[MutableMapping[str, torch.Tensor]], Dict[str, torch.Tensor]], - optimizer: Optimizer, - loss_func: Callable[[MutableMapping[str, torch.Tensor], MutableMapping[str, torch.Tensor]], torch.Tensor], + loader: DataLoader, + forward: Callable[[MutableMapping[str, torch.Tensor]], Dict[str, torch.Tensor]], + optimizer: Optimizer, + loss_func: Callable[[MutableMapping[str, torch.Tensor], MutableMapping[str, torch.Tensor]], torch.Tensor], ): """train for data @@ -142,7 +146,7 @@ def train_func( """ batch_loss_list = [] - for inputs, targets in loader: + for inputs, targets in tqdm(loader, desc="train"): outputs = forward(inputs) loss = loss_func(outputs, targets) # ensemble-num, batch-size, output-var-num @@ -156,9 +160,9 @@ def train_func( def eval_func( - loader: DataLoader, - forward: Callable[[MutableMapping[str, torch.Tensor]], Dict[str, torch.Tensor]], - loss_func: Callable[[MutableMapping[str, torch.Tensor], MutableMapping[str, torch.Tensor]], torch.Tensor], + loader: DataLoader, + forward: Callable[[MutableMapping[str, torch.Tensor]], Dict[str, torch.Tensor]], + loss_func: Callable[[MutableMapping[str, torch.Tensor], MutableMapping[str, torch.Tensor]], torch.Tensor], ): """evaluate for data @@ -172,7 +176,7 @@ def eval_func( """ batch_loss_list = [] with torch.no_grad(): - for inputs, targets in loader: + for inputs, targets in tqdm(loader, desc="eval"): outputs = forward(inputs) loss = loss_func(outputs, targets) # ensemble-num, batch-size, output-var-num diff --git a/cmrl/models/data_loader.py b/cmrl/models/data_loader.py index 42155b2..e9f5843 100644 --- a/cmrl/models/data_loader.py +++ b/cmrl/models/data_loader.py @@ -9,14 +9,7 @@ from cmrl.utils.variables import to_dict_by_space -def buffer_to_dict( - state_space, - action_space, - obs2state_fn, - replay_buffer: ReplayBuffer, - mech: str, - device: str = "cpu" -): +def buffer_to_dict(state_space, action_space, obs2state_fn, replay_buffer: ReplayBuffer, mech: str, device: str = "cpu"): assert mech in ["transition", "reward_mech", "termination_mech"] # dict action is not supported by SB3(so not done by cmrl) assert not isinstance(action_space, spaces.Dict) @@ -26,22 +19,19 @@ def buffer_to_dict( real_buffer_size = replay_buffer.buffer_size if replay_buffer.full else replay_buffer.pos if hasattr(replay_buffer, "extra_obs"): - states = obs2state_fn(replay_buffer.observations[: real_buffer_size, 0], - replay_buffer.extra_obs[: real_buffer_size, 0]) + states = obs2state_fn(replay_buffer.observations[:real_buffer_size, 0], replay_buffer.extra_obs[:real_buffer_size, 0]) else: - states = replay_buffer.observations[: real_buffer_size, 0] - state_dict = to_dict_by_space(states, state_space, prefix="obs", to_tensor=True, device=device) - act_dict = to_dict_by_space( - replay_buffer.actions[: real_buffer_size, 0], - action_space, - prefix="act", to_tensor=True, device=device) + states = replay_buffer.observations[:real_buffer_size, 0] + state_dict = to_dict_by_space(states, state_space, prefix="obs", to_tensor=True) + act_dict = to_dict_by_space(replay_buffer.actions[:real_buffer_size, 0], action_space, prefix="act", to_tensor=True) if hasattr(replay_buffer, "next_extra_obs"): - next_states = obs2state_fn(replay_buffer.next_observations[: real_buffer_size, 0], - replay_buffer.next_extra_obs[: real_buffer_size, 0]) + next_states = obs2state_fn( + replay_buffer.next_observations[:real_buffer_size, 0], replay_buffer.next_extra_obs[:real_buffer_size, 0] + ) else: - next_states = replay_buffer.next_observations[: real_buffer_size, 0] - next_state_dict = to_dict_by_space(next_states, state_space, prefix="next_obs", to_tensor=True, device=device) + next_states = replay_buffer.next_observations[:real_buffer_size, 0] + next_state_dict = to_dict_by_space(next_states, state_space, prefix="next_obs", to_tensor=True) inputs = {} inputs.update(state_dict) @@ -50,12 +40,12 @@ def buffer_to_dict( if mech == "transition": outputs = next_state_dict elif mech == "reward_mech": - rewards = replay_buffer.rewards[: real_buffer_size, 0] + rewards = replay_buffer.rewards[:real_buffer_size, 0] rewards_dict = {"reward": torch.from_numpy(rewards[:, None])} inputs.update(next_state_dict) outputs = rewards_dict elif mech == "termination_mech": - terminals = replay_buffer.dones[: real_buffer_size, 0] * (1 - replay_buffer.timeouts[: real_buffer_size, 0]) + terminals = replay_buffer.dones[:real_buffer_size, 0] * (1 - replay_buffer.timeouts[:real_buffer_size, 0]) terminals_dict = {"terminal": torch.from_numpy(terminals[:, None])} inputs.update(next_state_dict) outputs = terminals_dict @@ -67,13 +57,13 @@ def buffer_to_dict( class EnsembleBufferDataset(Dataset): def __init__( - self, - inputs: MutableMapping, - outputs: MutableMapping, - training: bool = False, - train_ratio: float = 0.8, - ensemble_num: int = 7, - seed: int = 10086, + self, + inputs: MutableMapping, + outputs: MutableMapping, + training: bool = False, + train_ratio: float = 0.8, + ensemble_num: int = 7, + seed: int = 10086, ): self.inputs = inputs self.outputs = outputs @@ -88,10 +78,10 @@ def __init__( np.random.seed(self.seed) permutation = np.random.permutation(size) if self.training: - train_indexes = permutation[:int(size * self.train_ratio)] + train_indexes = permutation[: int(size * self.train_ratio)] indexes = [np.random.permutation(train_indexes) for _ in range(self.ensemble_num)] else: - valid_indexes = permutation[int(size * self.train_ratio):] + valid_indexes = permutation[int(size * self.train_ratio) :] indexes = [valid_indexes for _ in range(self.ensemble_num)] self.indexes = np.array(indexes).T From d894c0ec6816e78bc23d2a667de1d7dc91ff8ce1 Mon Sep 17 00:00:00 2001 From: frank Date: Thu, 6 Apr 2023 19:35:01 +0800 Subject: [PATCH 66/68] :bug: clip log_var to avoid inf --- cmrl/models/causal_mech/util.py | 39 ++++++++++++++++++++------------- 1 file changed, 24 insertions(+), 15 deletions(-) diff --git a/cmrl/models/causal_mech/util.py b/cmrl/models/causal_mech/util.py index 26480b3..3ec9d6c 100644 --- a/cmrl/models/causal_mech/util.py +++ b/cmrl/models/causal_mech/util.py @@ -106,7 +106,9 @@ def variable_loss_func( dim = target.shape[-1] # (xxx, ensemble-num, batch-size, dim) assert output.shape[-1] == 2 * dim mean, log_var = output[..., :dim], output[..., dim:] - loss = F.gaussian_nll_loss(mean, target, log_var.exp(), reduction="none", full=True).mean(dim=-1) + # clip log_var to avoid nan loss + log_var = torch.clamp(log_var, min=-10, max=10) + loss = F.gaussian_nll_loss(mean, target, log_var.exp(), reduction="none", full=True, eps=1e-4).mean(dim=-1) total_loss[..., i] = loss elif isinstance(var, RadianVariable): dim = target.shape[-1] # (xxx, ensemble-num, batch-size, dim) @@ -117,14 +119,15 @@ def variable_loss_func( elif isinstance(var, DiscreteVariable): # TODO: onehot to int? raise NotImplementedError - total_loss[..., i] = F.cross_entropy(output, target, reduction="none") elif isinstance(var, BinaryVariable): total_loss[..., i] = F.binary_cross_entropy(output, target, reduction="none") else: raise NotImplementedError if torch.isnan(total_loss[..., i]).any(): - raise ValueError(f"nan loss for {var.name}") + raise ValueError(f"nan loss for {var.name} ({type(var)})") + elif torch.isinf(total_loss[..., i]).any(): + raise ValueError(f"inf loss for {var.name} ({type(var)})") return total_loss @@ -146,15 +149,18 @@ def train_func( """ batch_loss_list = [] - for inputs, targets in tqdm(loader, desc="train"): - outputs = forward(inputs) - loss = loss_func(outputs, targets) # ensemble-num, batch-size, output-var-num + with tqdm(loader) as pbar: + for inputs, targets in loader: + outputs = forward(inputs) + loss = loss_func(outputs, targets) # ensemble-num, batch-size, output-var-num - optimizer.zero_grad() - loss.mean().backward() - optimizer.step() + optimizer.zero_grad() + loss.mean().backward() + optimizer.step() + batch_loss_list.append(loss) - batch_loss_list.append(loss) + pbar.set_description(f"train loss: {loss.mean().item():.4f}") + pbar.update() return torch.cat(batch_loss_list, dim=-2).detach().cpu() @@ -176,9 +182,12 @@ def eval_func( """ batch_loss_list = [] with torch.no_grad(): - for inputs, targets in tqdm(loader, desc="eval"): - outputs = forward(inputs) - loss = loss_func(outputs, targets) # ensemble-num, batch-size, output-var-num - - batch_loss_list.append(loss) + with tqdm(loader) as pbar: + for inputs, targets in loader: + outputs = forward(inputs) + loss = loss_func(outputs, targets) # ensemble-num, batch-size, output-var-num + batch_loss_list.append(loss) + + pbar.set_description(f"eval loss: {loss.mean().item():.4f}") + pbar.update() return torch.cat(batch_loss_list, dim=-2).detach().cpu() From c7381718dcbafa02385c73165d36ae0f61630b3b Mon Sep 17 00:00:00 2001 From: frank Date: Fri, 7 Apr 2023 10:23:00 +0800 Subject: [PATCH 67/68] :tada: update new cfg --- cmrl/algorithms/base_algorithm.py | 24 ++-- cmrl/algorithms/mopo.py | 8 +- cmrl/examples/conf/main.yaml | 3 +- .../conf/task/parallel_cart_pole.yaml | 2 +- cmrl/examples/conf/transition/oracle.yaml | 2 +- cmrl/models/causal_mech/kernel_test.py | 115 +++++++++--------- 6 files changed, 75 insertions(+), 79 deletions(-) diff --git a/cmrl/algorithms/base_algorithm.py b/cmrl/algorithms/base_algorithm.py index 4412ab3..532ea13 100644 --- a/cmrl/algorithms/base_algorithm.py +++ b/cmrl/algorithms/base_algorithm.py @@ -18,9 +18,9 @@ class BaseAlgorithm: def __init__( - self, - cfg: DictConfig, - work_dir: Optional[str] = None, + self, + cfg: DictConfig, + work_dir: Optional[str] = None, ): self.cfg = cfg self.work_dir = work_dir or os.getcwd() @@ -32,7 +32,10 @@ def __init__( np.random.seed(self.cfg.seed) torch.manual_seed(self.cfg.seed) - self.logger = logger_configure("log", ["tensorboard", "multi_csv", "stdout"]) + format_strings = ["tensorboard", "multi_csv"] + if self.cfg.verbose: + format_strings += ["stdout"] + self.logger = logger_configure("log", format_strings) if cfg.wandb: wandb.init( @@ -43,12 +46,9 @@ def __init__( ) # create ``cmrl`` dynamics - self.dynamics = create_dynamics(self.cfg, - self.env.state_space, - self.env.action_space, - self.obs2state_fn, - self.state2obs_fn, - logger=self.logger) + self.dynamics = create_dynamics( + self.cfg, self.env.state_space, self.env.action_space, self.obs2state_fn, self.state2obs_fn, logger=self.logger + ) if self.cfg.transition.name == "oracle_transition": graph = self.env.get_transition_graph() if self.cfg.transition.oracle == "truth" else None @@ -95,9 +95,7 @@ def fake_env(self) -> VecFakeEnv: @property def callback(self) -> BaseCallback: fake_eval_env = self.partial_fake_env( - deterministic=True, - max_episode_steps=self.env.spec.max_episode_steps, - branch_rollout=False + deterministic=True, max_episode_steps=self.env.spec.max_episode_steps, branch_rollout=False ) return EvalCallback( self.eval_env, diff --git a/cmrl/algorithms/mopo.py b/cmrl/algorithms/mopo.py index abdcf07..3d1f729 100644 --- a/cmrl/algorithms/mopo.py +++ b/cmrl/algorithms/mopo.py @@ -10,9 +10,9 @@ class MOPO(BaseAlgorithm): def __init__( - self, - cfg: DictConfig, - work_dir: Optional[str] = None, + self, + cfg: DictConfig, + work_dir: Optional[str] = None, ): super(MOPO, self).__init__(cfg, work_dir) @@ -29,7 +29,7 @@ def _setup_learn(self): existed_trained_model = maybe_load_offline_model(self.dynamics, self.cfg, work_dir=self.work_dir) - if not existed_trained_model: + if not existed_trained_model and self.cfg.task.get("auto_load_offline_model", False): self.dynamics.learn( real_replay_buffer=self.real_replay_buffer, work_dir=self.work_dir, diff --git a/cmrl/examples/conf/main.yaml b/cmrl/examples/conf/main.yaml index cfc23fd..7ccd216 100644 --- a/cmrl/examples/conf/main.yaml +++ b/cmrl/examples/conf/main.yaml @@ -10,7 +10,8 @@ seed: 0 device: "cpu" exp_name: default -wandb: false +wandb: true +verbose: true root_dir: "./exp" hydra: diff --git a/cmrl/examples/conf/task/parallel_cart_pole.yaml b/cmrl/examples/conf/task/parallel_cart_pole.yaml index d48fcca..35145cd 100644 --- a/cmrl/examples/conf/task/parallel_cart_pole.yaml +++ b/cmrl/examples/conf/task/parallel_cart_pole.yaml @@ -16,7 +16,7 @@ extra_variable_info: - "obs_9" # basic RL params -num_steps: 3000000 +num_steps: 10000000 online_num_steps: 10000 n_eval_episodes: 5 eval_freq: 10000 diff --git a/cmrl/examples/conf/transition/oracle.yaml b/cmrl/examples/conf/transition/oracle.yaml index 7588695..7e45c68 100644 --- a/cmrl/examples/conf/transition/oracle.yaml +++ b/cmrl/examples/conf/transition/oracle.yaml @@ -42,7 +42,7 @@ scheduler_cfg: _partial_: true _target_: torch.optim.lr_scheduler.StepLR step_size: 1 - gamma: 1 + gamma: 0.9 mech: _partial_: true diff --git a/cmrl/models/causal_mech/kernel_test.py b/cmrl/models/causal_mech/kernel_test.py index abf6252..f4389c3 100644 --- a/cmrl/models/causal_mech/kernel_test.py +++ b/cmrl/models/causal_mech/kernel_test.py @@ -9,8 +9,9 @@ from omegaconf import DictConfig from stable_baselines3.common.logger import Logger from hydra.utils import instantiate -from cmrl.utils.RCIT import KCI_CInd -# from causallearn.utils.KCI.KCI import KCI_CInd + +# from cmrl.utils.RCIT import KCI_CInd +from causallearn.utils.KCI.KCI import KCI_CInd from tqdm import tqdm from cmrl.models.causal_mech.base import EnsembleNeuralMech @@ -20,36 +21,36 @@ class KernelTestMech(EnsembleNeuralMech): def __init__( - self, - # base - name: str, - input_variables: List[Variable], - output_variables: List[Variable], - logger: Optional[Logger] = None, - # model learning - longest_epoch: int = -1, - improvement_threshold: float = 0.01, - patience: int = 5, - batch_size: int = 256, - # ensemble - ensemble_num: int = 7, - elite_num: int = 5, - # cfgs - network_cfg: Optional[DictConfig] = None, - encoder_cfg: Optional[DictConfig] = None, - decoder_cfg: Optional[DictConfig] = None, - optimizer_cfg: Optional[DictConfig] = None, - scheduler_cfg: Optional[DictConfig] = None, - # forward method - residual: bool = True, - encoder_reduction: str = "sum", - # others - device: Union[str, torch.device] = "cpu", - # KCI - sample_num: int = 2000, - kci_times: int = 10, - not_confident_bound: float = 0.25, - longest_sample: int = 5000 + self, + # base + name: str, + input_variables: List[Variable], + output_variables: List[Variable], + logger: Optional[Logger] = None, + # model learning + longest_epoch: int = -1, + improvement_threshold: float = 0.01, + patience: int = 5, + batch_size: int = 256, + # ensemble + ensemble_num: int = 7, + elite_num: int = 5, + # cfgs + network_cfg: Optional[DictConfig] = None, + encoder_cfg: Optional[DictConfig] = None, + decoder_cfg: Optional[DictConfig] = None, + optimizer_cfg: Optional[DictConfig] = None, + scheduler_cfg: Optional[DictConfig] = None, + # forward method + residual: bool = True, + encoder_reduction: str = "sum", + # others + device: Union[str, torch.device] = "cpu", + # KCI + sample_num: int = 2000, + kci_times: int = 10, + not_confident_bound: float = 0.25, + longest_sample: int = 5000, ): EnsembleNeuralMech.__init__( self, @@ -78,12 +79,12 @@ def __init__( self.longest_sample = longest_sample def kci( - self, - input_idx: int, - output_idx: int, - inputs: MutableMapping[str, numpy.ndarray], - outputs: MutableMapping[str, numpy.ndarray], - sample_indices: np.ndarray, + self, + input_idx: int, + output_idx: int, + inputs: MutableMapping[str, numpy.ndarray], + outputs: MutableMapping[str, numpy.ndarray], + sample_indices: np.ndarray, ): in_name, out_name = list(inputs.keys())[input_idx], list(outputs.keys())[output_idx] @@ -111,11 +112,11 @@ def deal_with_radian_input(name, data): return p_value def kci_compute_graph( - self, - inputs: MutableMapping[str, numpy.ndarray], - outputs: MutableMapping[str, numpy.ndarray], - work_dir: Optional[pathlib.Path] = None, - **kwargs + self, + inputs: MutableMapping[str, numpy.ndarray], + outputs: MutableMapping[str, numpy.ndarray], + work_dir: Optional[pathlib.Path] = None, + **kwargs ): open(work_dir / "history_vote.txt", "w") @@ -125,8 +126,8 @@ def kci_compute_graph( init_pvalues_array = np.empty((self.kci_times, self.input_var_num, self.output_var_num)) with tqdm( - total=self.kci_times * self.input_var_num * self.output_var_num, - desc="init kci of {} samples".format(sample_length), + total=self.kci_times * self.input_var_num * self.output_var_num, + desc="init kci of {} samples".format(sample_length), ) as pbar: for time in range(self.kci_times): sample_indices = np.random.permutation(length)[:sample_length] @@ -146,14 +147,14 @@ def kci_compute_graph( f.write(str(votes) + "\n") print(votes) - new_sample_length = int(sample_length * 1.5 ** recompute_times) + new_sample_length = int(sample_length * 1.5**recompute_times) if new_sample_length > min(self.longest_sample, length): break pvalues_dict = defaultdict(list) with tqdm( - total=self.kci_times * len(not_confident_list), - desc="{}th re-compute kci of {} samples".format(recompute_times, new_sample_length), + total=self.kci_times * len(not_confident_list), + desc="{}th re-compute kci of {} samples".format(recompute_times, new_sample_length), ) as pbar: for time in range(self.kci_times): sample_indices = np.random.permutation(length)[:new_sample_length] @@ -183,8 +184,7 @@ def build_network(self): def forward(self, inputs: MutableMapping[str, torch.Tensor]) -> Dict[str, torch.Tensor]: batch_size, _ = self.get_inputs_batch_size(inputs) - inputs_tensor = torch.zeros(self.ensemble_num, batch_size, self.input_var_num, self.encoder_output_dim).to( - self.device) + inputs_tensor = torch.zeros(self.ensemble_num, batch_size, self.input_var_num, self.encoder_output_dim).to(self.device) for i, var in enumerate(self.input_variables): out = self.variable_encoders[var.name](inputs[var.name].to(self.device)) inputs_tensor[:, :, i] = out @@ -204,11 +204,11 @@ def build_graph(self): self.graph = BinaryGraph(self.input_var_num, self.output_var_num, device=self.device) def learn( - self, - inputs: MutableMapping[str, np.ndarray], - outputs: MutableMapping[str, np.ndarray], - work_dir: Optional[pathlib.Path] = None, - **kwargs + self, + inputs: MutableMapping[str, np.ndarray], + outputs: MutableMapping[str, np.ndarray], + work_dir: Optional[pathlib.Path] = None, + **kwargs ): work_dir = pathlib.Path(".") if work_dir is None else work_dir graph = self.kci_compute_graph(inputs, outputs, work_dir) @@ -230,13 +230,11 @@ def learn( from cmrl.sb3_extension.logger import configure as logger_configure from cmrl.models.causal_mech.util import variable_loss_func - def unwrap_env(env): while isinstance(env, gym.Wrapper): env = env.env return env - env = unwrap_env(gym.make("ParallelContinuousCartPoleSwingUp-v0")) real_replay_buffer = ReplayBuffer( int(1e6), env.observation_space, env.action_space, "cpu", handle_timeout_termination=False @@ -251,8 +249,7 @@ def unwrap_env(env): logger = logger_configure("kci-log", ["tensorboard", "stdout"]) - mech = KernelTestMech("kernel_test_mech", input_variables, output_variables, sample_num=100, kci_times=20, - logger=logger) + mech = KernelTestMech("kernel_test_mech", input_variables, output_variables, sample_num=100, kci_times=20, logger=logger) inputs, outputs = buffer_to_dict(env.state_space, env.action_space, env.obs2state, real_replay_buffer, "transition") From edfafa69a361df05b61bca1217cc11ae73d10381 Mon Sep 17 00:00:00 2001 From: frank Date: Fri, 7 Apr 2023 11:39:51 +0800 Subject: [PATCH 68/68] :tada: update new cfg --- cmrl/algorithms/mopo.py | 8 +-- cmrl/algorithms/off_dyna.py | 6 ++- cmrl/examples/conf/main.yaml | 4 +- cmrl/examples/conf/transition/oracle.yaml | 3 +- cmrl/models/causal_mech/CMI_test.py | 64 +++++++++++------------ 5 files changed, 44 insertions(+), 41 deletions(-) diff --git a/cmrl/algorithms/mopo.py b/cmrl/algorithms/mopo.py index 3d1f729..4a98517 100644 --- a/cmrl/algorithms/mopo.py +++ b/cmrl/algorithms/mopo.py @@ -27,9 +27,11 @@ def fake_env(self) -> VecFakeEnv: def _setup_learn(self): load_offline_data(self.env, self.real_replay_buffer, self.cfg.task.dataset, self.cfg.task.use_ratio) - existed_trained_model = maybe_load_offline_model(self.dynamics, self.cfg, work_dir=self.work_dir) - - if not existed_trained_model and self.cfg.task.get("auto_load_offline_model", False): + if self.cfg.task.get("auto_load_offline_model", False): + existed_trained_model = maybe_load_offline_model(self.dynamics, self.cfg, work_dir=self.work_dir) + else: + existed_trained_model = None + if not existed_trained_model: self.dynamics.learn( real_replay_buffer=self.real_replay_buffer, work_dir=self.work_dir, diff --git a/cmrl/algorithms/off_dyna.py b/cmrl/algorithms/off_dyna.py index a34dffc..3911c05 100644 --- a/cmrl/algorithms/off_dyna.py +++ b/cmrl/algorithms/off_dyna.py @@ -18,8 +18,10 @@ def __init__( def _setup_learn(self): load_offline_data(self.env, self.real_replay_buffer, self.cfg.task.dataset, self.cfg.task.use_ratio) - existed_trained_model = maybe_load_offline_model(self.dynamics, self.cfg, work_dir=self.work_dir) - + if self.cfg.task.get("auto_load_offline_model", False): + existed_trained_model = maybe_load_offline_model(self.dynamics, self.cfg, work_dir=self.work_dir) + else: + existed_trained_model = None if not existed_trained_model: self.dynamics.learn( real_replay_buffer=self.real_replay_buffer, diff --git a/cmrl/examples/conf/main.yaml b/cmrl/examples/conf/main.yaml index 7ccd216..39abbe5 100644 --- a/cmrl/examples/conf/main.yaml +++ b/cmrl/examples/conf/main.yaml @@ -10,8 +10,8 @@ seed: 0 device: "cpu" exp_name: default -wandb: true -verbose: true +wandb: false +verbose: false root_dir: "./exp" hydra: diff --git a/cmrl/examples/conf/transition/oracle.yaml b/cmrl/examples/conf/transition/oracle.yaml index 7e45c68..45fae3b 100644 --- a/cmrl/examples/conf/transition/oracle.yaml +++ b/cmrl/examples/conf/transition/oracle.yaml @@ -42,7 +42,7 @@ scheduler_cfg: _partial_: true _target_: torch.optim.lr_scheduler.StepLR step_size: 1 - gamma: 0.9 + gamma: 0.8 mech: _partial_: true @@ -65,6 +65,7 @@ mech: encoder_cfg: ${transition.encoder_cfg} decoder_cfg: ${transition.decoder_cfg} optimizer_cfg: ${transition.optimizer_cfg} + scheduler_cfg: ${transition.scheduler_cfg} # forward method residual: true # logger diff --git a/cmrl/models/causal_mech/CMI_test.py b/cmrl/models/causal_mech/CMI_test.py index 3d33432..4cadde4 100644 --- a/cmrl/models/causal_mech/CMI_test.py +++ b/cmrl/models/causal_mech/CMI_test.py @@ -18,29 +18,29 @@ class CMITestMech(EnsembleNeuralMech): def __init__( - self, - name: str, - input_variables: List[Variable], - output_variables: List[Variable], - logger: Optional[Logger] = None, - # model learning - longest_epoch: int = -1, - improvement_threshold: float = 0.01, - patience: int = 5, - batch_size: int = 256, - # ensemble - ensemble_num: int = 7, - elite_num: int = 5, - # cfgs - network_cfg: Optional[DictConfig] = None, - encoder_cfg: Optional[DictConfig] = None, - decoder_cfg: Optional[DictConfig] = None, - optimizer_cfg: Optional[DictConfig] = None, - # forward method - residual: bool = True, - encoder_reduction: str = "sum", - # others - device: Union[str, torch.device] = "cpu", + self, + name: str, + input_variables: List[Variable], + output_variables: List[Variable], + logger: Optional[Logger] = None, + # model learning + longest_epoch: int = -1, + improvement_threshold: float = 0.01, + patience: int = 5, + batch_size: int = 256, + # ensemble + ensemble_num: int = 7, + elite_num: int = 5, + # cfgs + network_cfg: Optional[DictConfig] = None, + encoder_cfg: Optional[DictConfig] = None, + decoder_cfg: Optional[DictConfig] = None, + optimizer_cfg: Optional[DictConfig] = None, + # forward method + residual: bool = True, + encoder_reduction: str = "sum", + # others + device: Union[str, torch.device] = "cpu", ): EnsembleNeuralMech.__init__( self, @@ -97,8 +97,7 @@ def multi_graph_forward(self, inputs: MutableMapping[str, torch.Tensor]) -> Dict """ batch_size, extra_dim = self.get_inputs_info(inputs) - inputs_tensor = torch.empty(*extra_dim, self.ensemble_num, batch_size, self.input_var_num, - self.encoder_output_dim).to( + inputs_tensor = torch.empty(*extra_dim, self.ensemble_num, batch_size, self.input_var_num, self.encoder_output_dim).to( self.device ) for i, var in enumerate(self.input_variables): @@ -146,7 +145,7 @@ def multi_graph_forward(self, inputs: MutableMapping[str, torch.Tensor]) -> Dict mask = mask.repeat((1,) * len(mask.shape[:-3]) + (self.ensemble_num, batch_size, 1)) reduced_inputs_tensor = self.reduce_encoder_output(inputs_tensor, mask) assert ( - not torch.isinf(reduced_inputs_tensor).any() and not torch.isnan(reduced_inputs_tensor).any() + not torch.isinf(reduced_inputs_tensor).any() and not torch.isnan(reduced_inputs_tensor).any() ), "tensor must not be inf or nan" output_tensor = self.network(reduced_inputs_tensor) @@ -165,11 +164,11 @@ def calculate_CMI(self, nll_loss: torch.Tensor, threshold=1): return graph_data, nll_loss_diff.mean(dim=(1, 2)) def learn( - self, - inputs: MutableMapping[str, np.ndarray], - outputs: MutableMapping[str, np.ndarray], - work_dir: Optional[pathlib.Path] = None, - **kwargs + self, + inputs: MutableMapping[str, np.ndarray], + outputs: MutableMapping[str, np.ndarray], + work_dir: Optional[pathlib.Path] = None, + **kwargs ): work_dir = pathlib.Path(".") if work_dir is None else work_dir @@ -228,6 +227,7 @@ def learn( break self.scheduler.step() + print(self.optimizer) assert final_graph_data is not None self.graph.set_data(final_graph_data) @@ -248,13 +248,11 @@ def learn( from cmrl.utils.env import load_offline_data from cmrl.models.causal_mech.util import variable_loss_func - def unwrap_env(env): while isinstance(env, gym.Wrapper): env = env.env return env - env = unwrap_env(gym.make("ParallelContinuousCartPoleSwingUp-v0")) real_replay_buffer = ReplayBuffer( int(1e6), env.observation_space, env.action_space, "cpu", handle_timeout_termination=False