diff --git a/datasets/uea.py b/datasets/uea.py new file mode 100644 index 0000000..584d34c --- /dev/null +++ b/datasets/uea.py @@ -0,0 +1,79 @@ +"""UEA Archive time series classification dataset. + +Uses tslearn to download and load any UEA multivariate dataset by name. +Each time series is returned as a ``(T, C)`` numpy array with ``C > 1`` +channels; labels are integers. + +The UEA archive and the UCR archive are served by the same tslearn loader +(``UCR_UEA_datasets``). This file is a dedicated entry point for the +multivariate UEA datasets so they appear as a separate dataset family in +benchopt results. + +Data contract output +-------------------- +X_train : List[np.ndarray (T, C)] one array per training sample +y_train : np.ndarray (N,) int class labels +X_test : List[np.ndarray (T, C)] +y_test : np.ndarray (M,) int +task : "classification" +metrics : ["accuracy", "balanced_accuracy", "f1_weighted"] +n_classes : int +""" + +import numpy as np +from benchopt import BaseDataset +from sklearn.preprocessing import LabelEncoder +from tslearn.datasets import UCR_UEA_datasets + + +class Dataset(BaseDataset): + """UEA Archive multivariate classification dataset. + + Parameters + ---------- + dataset_name : str + Name of a UEA multivariate dataset (e.g. "BasicMotions", + "EthanolConcentration", "NATOPS"). + debug : bool + If True, keep only the first 20 training samples for fast testing. + """ + + name = "UEA" + + requirements = ["pip::tslearn"] + + parameters = { + "dataset_name": ["BasicMotions"], + "debug": [False], + } + + def get_data(self): + + loader = UCR_UEA_datasets() + X_tr, y_tr, X_te, y_te = loader.load_dataset(self.dataset_name) + + # tslearn returns (N, T, C) — already the right layout. + X_tr = np.asarray(X_tr, dtype=np.float32) + X_te = np.asarray(X_te, dtype=np.float32) + + # Encode string labels to consecutive integers. + le = LabelEncoder() + y_tr_enc = le.fit_transform(y_tr).astype(np.int64) + y_te_enc = le.transform(y_te).astype(np.int64) + + if self.debug: + X_tr = X_tr[:20] + y_tr_enc = y_tr_enc[:20] + + # Convert to list of (T, C) arrays so variable-length datasets work too + X_train, X_test = list(X_tr), list(X_te) + + return dict( + X_train=X_train, + y_train=y_tr_enc, + X_test=X_test, + y_test=y_te_enc, + task="classification", + metrics=["accuracy", "balanced_accuracy", "f1_weighted"], + n_classes=int(len(le.classes_)), + ) diff --git a/solvers/tabicl.py b/solvers/tabicl.py new file mode 100644 index 0000000..c653623 --- /dev/null +++ b/solvers/tabicl.py @@ -0,0 +1,133 @@ +"""TabICL-v2 solver for UCR classification. + +TabICL is a tabular foundation model that uses in-context learning (ICL) to +classify new samples in a single forward pass. For the current UCR scope, +datasets are univariate and fixed-length, so each ``(T, 1)`` sample is +reshaped into a flat ``(T,)`` feature row and the full training matrix is +passed to ``TabICLClassifier``. + +References: + https://github.com/soda-inria/tabicl + https://huggingface.co/tabicl +""" + +import numpy as np +import torch +from benchopt import BaseSolver + +from tabicl import TabICLClassifier + +SUPPORTED_TASKS = {"classification"} + + +class _TabICLAdapter: + """Wrap a fitted TabICLClassifier behind the benchmark adapter API. + + ``predict`` receives the entire test list in one call because TabICL is + an in-context learner — all test rows are scored jointly against the + stored training context. + """ + + def __init__(self, model): + self.model = model + + def predict(self, X): + return self.model.predict(_to_tabular(X)) + + +def _to_tabular(X): + """Convert a list of benchmark samples into a 2-D feature matrix. + + Parameters + ---------- + X : list of np.ndarray, each shape (T, C) + UCR samples. ``C`` is 1 for the current scope. + + Returns + ------- + np.ndarray, shape (N, T * C) + Flat tabular matrix ready for TabICL. + """ + arr = np.asarray(X, dtype=np.float32) # (N, T) or (N, T, C) + if arr.ndim == 3: + arr = arr.reshape(arr.shape[0], -1) # flatten C into features + return arr + + +class Solver(BaseSolver): + """TabICL-v2 in-context learning solver for UCR classification. + + The classifier is instantiated once in ``set_objective`` (not timed) so + that any checkpoint download is excluded from benchmark timing. The + actual ``fit`` — which stores the training context — runs in ``run``. + """ + + name = "TabICL-v2" + + requirements = ["pip::tabicl"] + + sampling_strategy = "run_once" + + parameters = { + "checkpoint_version": ["tabicl-classifier-v2-20260212.ckpt"], + "n_estimators": [8], + } + + def skip(self, task, **kwargs): + if task not in SUPPORTED_TASKS: + return True, f"TabICL solver does not support task={task!r}" + return False, None + + def set_objective(self, task, X_train, y_train, **meta): + """Prepare the solver for a given dataset configuration. + + Classifier instantiation is done here (not inside ``run``) so that + the checkpoint download/init time is excluded from benchmark timing. + """ + self.task = task + self.X_train = X_train + self.y_train = y_train + self.meta = meta + + device = "cuda" if torch.cuda.is_available() else "cpu" + + # Reinstantiate when the checkpoint version or n_estimators changes. + current_key = (self.checkpoint_version, self.n_estimators) + should_reload = ( + not hasattr(self, "_classifier") + or not hasattr(self, "_loaded_key") + or self._loaded_key != current_key + ) + if should_reload: + try: + self._classifier = TabICLClassifier( + checkpoint_version=self.checkpoint_version, + n_estimators=self.n_estimators, + device=device, + random_state=42, + verbose=False, + ) + self._loaded_key = current_key + print( + f"\u2713 TabICL checkpoint ready: {self.checkpoint_version} " + f"on device: {device}" + ) + except Exception as e: + raise RuntimeError( + f"Failed to initialise TabICL checkpoint " + f"'{self.checkpoint_version}': {e}. " + "Make sure you have internet access and tabicl is installed." + ) from e + + self._device = device + self._adapter = _TabICLAdapter(self._classifier) + + def run(self, _): + """Fit TabICL on the training data (timed).""" + X_fit = _to_tabular(self.X_train) + y_fit = np.asarray(self.y_train) + self._classifier.fit(X_fit, y_fit) + + def get_result(self): + """Return the fitted adapter wrapping the TabICL classifier.""" + return {"model": self._adapter} diff --git a/solvers/tabpfn.py b/solvers/tabpfn.py new file mode 100644 index 0000000..958ca69 --- /dev/null +++ b/solvers/tabpfn.py @@ -0,0 +1,127 @@ +"""TabPFN-v2 solver for UCR/UEA classification. + +TabPFN is a Prior-fitted Network (PFN) — a Transformer pre-trained over +synthetic tabular datasets via meta-learning. For the UCR/UEA scope each +``(T, C)`` time-series sample is reshaped into a flat ``(T*C,)`` feature row +and the full training matrix is passed to ``TabPFNClassifier``. + +References: + https://github.com/PriorLabs/TabPFN + https://doi.org/10.1038/s41586-024-08328-6 +""" + +import numpy as np +from benchopt import BaseSolver + +from tabpfn import TabPFNClassifier +from tabpfn.constants import ModelVersion + +SUPPORTED_TASKS = {"classification"} + + +class _TabPFNAdapter: + """Wrap a fitted TabPFNClassifier behind the benchmark adapter API. + + ``predict`` receives the entire test list in one call — scoring all test + rows at once is significantly faster than individual calls because TabPFN + recomputes the training context on every call. + """ + + def __init__(self, model): + self.model = model + + def predict(self, X): + return self.model.predict(_to_tabular(X)) + + +def _to_tabular(X): + """Convert a list of benchmark samples into a 2-D feature matrix. + + Parameters + ---------- + X : list of np.ndarray, each shape (T, C) + UCR/UEA samples. ``C`` is 1 for univariate datasets. + + Returns + ------- + np.ndarray, shape (N, T * C) + Flat tabular matrix ready for TabPFN. + """ + arr = np.asarray(X, dtype=np.float32) # (N, T) or (N, T, C) + if arr.ndim == 3: + arr = arr.reshape(arr.shape[0], -1) # flatten C into features + return arr + + +class Solver(BaseSolver): + """TabPFN-v2 in-context learning solver for UCR/UEA classification. + + The classifier is instantiated once in ``set_objective`` (not timed) so + that any checkpoint download is excluded from benchmark timing. The + actual ``fit`` — which stores the training context — runs in ``run``. + """ + + name = "TabPFN-v2" + + requirements = ["pip::tabpfn"] + + sampling_strategy = "run_once" + + parameters = { + "n_estimators": [8], + } + + def skip(self, task, **kwargs): + if task not in SUPPORTED_TASKS: + return True, f"TabPFN solver does not support task={task!r}" + return False, None + + def set_objective(self, task, X_train, y_train, **meta): + """Prepare the solver for a given dataset configuration. + + Classifier instantiation is done here (not inside ``run``) so that + the checkpoint download/init time is excluded from benchmark timing. + """ + self.task = task + self.X_train = X_train + self.y_train = y_train + self.meta = meta + + # Reinstantiate only when n_estimators changes. + should_reload = ( + not hasattr(self, "_classifier") + or not hasattr(self, "_loaded_n_estimators") + or self._loaded_n_estimators != self.n_estimators + ) + if should_reload: + try: + self._classifier = TabPFNClassifier.create_default_for_version( + ModelVersion.V2, + n_estimators=self.n_estimators, + device="auto", + random_state=42, + ignore_pretraining_limits=True, + ) + self._loaded_n_estimators = self.n_estimators + print( + f"\u2713 TabPFN v2 ready " + f"(n_estimators={self.n_estimators}, device=auto)" + ) + except Exception as e: + raise RuntimeError( + f"Failed to initialise TabPFN v2: {e}. " + "Make sure tabpfn is installed and the v2 checkpoint " + "is available in the local cache (~/.cache/tabpfn/)." + ) from e + + self._adapter = _TabPFNAdapter(self._classifier) + + def run(self, _): + """Fit TabPFN on the training data (timed).""" + X_fit = _to_tabular(self.X_train) + y_fit = np.asarray(self.y_train) + self._classifier.fit(X_fit, y_fit) + + def get_result(self): + """Return the fitted adapter wrapping the TabPFN classifier.""" + return {"model": self._adapter} diff --git a/tests/solvers/test_tabicl.py b/tests/solvers/test_tabicl.py new file mode 100644 index 0000000..1558ee6 --- /dev/null +++ b/tests/solvers/test_tabicl.py @@ -0,0 +1,140 @@ +"""Shape and behaviour tests for the TabICL-v2 solver.""" + +import importlib.util +import sys +import types +from pathlib import Path + +import numpy as np + + +def _make_fake_tabicl_module(): + """Build a lightweight fake ``tabicl`` module for offline testing.""" + fake = types.ModuleType("tabicl") + + class FakeTabICLClassifier: + def __init__(self, **kwargs): + self.init_kwargs = kwargs + self.fit_X = None + self.fit_y = None + self.predict_calls = [] + + def fit(self, X, y): + self.fit_X = np.asarray(X) + self.fit_y = np.asarray(y) + return self + + def predict(self, X): + X_arr = np.asarray(X) + self.predict_calls.append(X_arr.copy()) + return np.zeros(X_arr.shape[0], dtype=np.int64) + + fake.TabICLClassifier = FakeTabICLClassifier + return fake + + +def _make_fake_benchopt_module(): + """Minimal fake ``benchopt`` module — only BaseSolver is needed.""" + fake = types.ModuleType("benchopt") + + class BaseSolver: + pass + + fake.BaseSolver = BaseSolver + return fake + + +def _make_fake_torch_module(): + """Minimal fake ``torch`` module — only cuda.is_available is needed.""" + fake = types.ModuleType("torch") + fake_cuda = types.ModuleType("torch.cuda") + fake_cuda.is_available = lambda: False + fake.cuda = fake_cuda + return fake + + +def _load_solver(monkeypatch): + """Import ``solvers/tabicl.py`` with fake modules injected.""" + monkeypatch.setitem(sys.modules, "benchopt", _make_fake_benchopt_module()) + monkeypatch.setitem(sys.modules, "torch", _make_fake_torch_module()) + monkeypatch.setitem(sys.modules, "tabicl", _make_fake_tabicl_module()) + solver_path = ( + Path(__file__).resolve().parents[2] / "solvers" / "tabicl.py" + ) + spec = importlib.util.spec_from_file_location( + "tabicl_solver_under_test", solver_path + ) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _make_solver(module): + """Instantiate Solver and inject default benchopt parameter attributes.""" + solver = module.Solver() + solver.n_estimators = 8 + solver.checkpoint_version = "tabicl-classifier-v2-20260212.ckpt" + return solver + + +def test_tabicl_solver_fit_shapes(monkeypatch): + """_to_tabular stacks (T, 1) lists correctly; TabICL receives (N, T).""" + module = _load_solver(monkeypatch) + solver = _make_solver(module) + + T = 6 + X_train = [np.arange(T, dtype=np.float32)[:, None] + i for i in range(4)] + y_train = np.array([0, 1, 0, 1], dtype=np.int64) + + solver.set_objective("classification", X_train, y_train) + solver.run(None) + + assert solver._classifier.fit_X.shape == (4, T) + assert solver._classifier.fit_y.shape == (4,) + + +def test_tabicl_solver_predict_shapes(monkeypatch): + """Adapter.predict converts the test list to (N, T) before TabICL.""" + module = _load_solver(monkeypatch) + solver = _make_solver(module) + + T = 6 + X_train = [np.arange(T, dtype=np.float32)[:, None] + i for i in range(4)] + y_train = np.array([0, 1, 0, 1], dtype=np.int64) + + solver.set_objective("classification", X_train, y_train) + solver.run(None) + + X_test = [np.arange(T, dtype=np.float32)[:, None] + i for i in range(3)] + adapter = solver.get_result()["model"] + y_pred = adapter.predict(X_test) + + assert y_pred.shape == (3,) + assert solver._classifier.predict_calls[-1].shape == (3, T) + + +def test_tabicl_classifier_reuse_across_datasets(monkeypatch): + """Classifier is reused when checkpoint_version and n_estimators are unchanged.""" + module = _load_solver(monkeypatch) + solver = _make_solver(module) + + T = 6 + X = [np.arange(T, dtype=np.float32)[:, None]] + y = np.array([0]) + + solver.set_objective("classification", X, y) + first_clf = solver._classifier + + solver.set_objective("classification", X, y) + assert solver._classifier is first_clf + + +def test_tabicl_solver_skip(monkeypatch): + """Unsupported tasks are skipped with an informative message.""" + module = _load_solver(monkeypatch) + solver = _make_solver(module) + + for task in ("forecasting", "anomaly_detection", "event_detection"): + should_skip, reason = solver.skip(task) + assert should_skip + assert f"task={task!r}" in reason diff --git a/tests/solvers/test_tabpfn.py b/tests/solvers/test_tabpfn.py new file mode 100644 index 0000000..ee1703b --- /dev/null +++ b/tests/solvers/test_tabpfn.py @@ -0,0 +1,172 @@ +"""Shape and behaviour tests for the TabPFN-v2 solver.""" + +import importlib.util +import sys +import types +from enum import Enum +from pathlib import Path + +import numpy as np + + +def _make_fake_tabpfn_module(): + """Build a lightweight fake ``tabpfn`` module for offline testing.""" + fake = types.ModuleType("tabpfn") + fake_constants = types.ModuleType("tabpfn.constants") + + class ModelVersion(Enum): + V2 = "v2" + V2_5 = "v2_5" + V2_6 = "v2_6" + V3 = "v3" + + # The solver only uses V2; keep the full enum for completeness. + + fake_constants.ModelVersion = ModelVersion + fake.constants = fake_constants + + class FakeTabPFNClassifier: + def __init__(self, **kwargs): + self.init_kwargs = kwargs + self.fit_X = None + self.fit_y = None + self.predict_calls = [] + + @classmethod + def create_default_for_version(cls, version, **kwargs): + obj = cls(**kwargs) + obj._version = version + return obj + + def fit(self, X, y): + self.fit_X = np.asarray(X) + self.fit_y = np.asarray(y) + return self + + def predict(self, X): + X_arr = np.asarray(X) + self.predict_calls.append(X_arr.copy()) + return np.zeros(X_arr.shape[0], dtype=np.int64) + + fake.TabPFNClassifier = FakeTabPFNClassifier + return fake, fake_constants + + +def _make_fake_benchopt_module(): + """Minimal fake ``benchopt`` module — only BaseSolver is needed.""" + fake = types.ModuleType("benchopt") + + class BaseSolver: + pass + + fake.BaseSolver = BaseSolver + return fake + + +def _load_solver(monkeypatch): + """Import ``solvers/tabpfn.py`` with fake modules injected.""" + fake_tabpfn, fake_constants = _make_fake_tabpfn_module() + monkeypatch.setitem(sys.modules, "benchopt", _make_fake_benchopt_module()) + monkeypatch.setitem(sys.modules, "tabpfn", fake_tabpfn) + monkeypatch.setitem(sys.modules, "tabpfn.constants", fake_constants) + solver_path = ( + Path(__file__).resolve().parents[2] / "solvers" / "tabpfn.py" + ) + spec = importlib.util.spec_from_file_location( + "tabpfn_solver_under_test", solver_path + ) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _make_solver(module): + """Instantiate Solver and inject default benchopt parameter attributes.""" + solver = module.Solver() + solver.n_estimators = 8 + return solver + + +def test_tabpfn_solver_fit_shapes(monkeypatch): + """_to_tabular stacks (T, 1) lists correctly; TabPFN receives (N, T).""" + module = _load_solver(monkeypatch) + solver = _make_solver(module) + + T = 6 + X_train = [np.arange(T, dtype=np.float32)[:, None] + i for i in range(4)] + y_train = np.array([0, 1, 0, 1], dtype=np.int64) + + solver.set_objective("classification", X_train, y_train) + solver.run(None) + + assert solver._classifier.fit_X.shape == (4, T) + assert solver._classifier.fit_y.shape == (4,) + + +def test_tabpfn_solver_predict_shapes(monkeypatch): + """Adapter.predict converts the test list to (N, T) before TabPFN.""" + module = _load_solver(monkeypatch) + solver = _make_solver(module) + + T = 6 + X_train = [np.arange(T, dtype=np.float32)[:, None] + i for i in range(4)] + y_train = np.array([0, 1, 0, 1], dtype=np.int64) + + solver.set_objective("classification", X_train, y_train) + solver.run(None) + + X_test = [np.arange(T, dtype=np.float32)[:, None] + i for i in range(3)] + adapter = solver.get_result()["model"] + y_pred = adapter.predict(X_test) + + assert y_pred.shape == (3,) + assert solver._classifier.predict_calls[-1].shape == (3, T) + + +def test_tabpfn_multivariate_shapes(monkeypatch): + """_to_tabular flattens (T, C) into T*C features for multivariate data.""" + module = _load_solver(monkeypatch) + solver = _make_solver(module) + + T, C = 5, 3 + X_train = [np.random.randn(T, C).astype(np.float32) for _ in range(4)] + y_train = np.array([0, 1, 0, 1], dtype=np.int64) + + solver.set_objective("classification", X_train, y_train) + solver.run(None) + + assert solver._classifier.fit_X.shape == (4, T * C) + + X_test = [np.random.randn(T, C).astype(np.float32) for _ in range(3)] + adapter = solver.get_result()["model"] + y_pred = adapter.predict(X_test) + + assert y_pred.shape == (3,) + assert solver._classifier.predict_calls[-1].shape == (3, T * C) + + +def test_tabpfn_classifier_reuse_across_datasets(monkeypatch): + """Classifier is reused when n_estimators is unchanged across datasets.""" + module = _load_solver(monkeypatch) + solver = _make_solver(module) + + T = 6 + X = [np.arange(T, dtype=np.float32)[:, None]] + y = np.array([0]) + + solver.set_objective("classification", X, y) + first_clf = solver._classifier + + solver.set_objective("classification", X, y) + assert solver._classifier is first_clf + + +def test_tabpfn_solver_skip(monkeypatch): + """Unsupported tasks are skipped with an informative message.""" + module = _load_solver(monkeypatch) + solver = _make_solver(module) + + for task in ("forecasting", "anomaly_detection", "event_detection"): + should_skip, reason = solver.skip(task) + assert should_skip + assert f"task={task!r}" in reason