From bc965148e9e21fd0d490f879eca29ea6da20ab30 Mon Sep 17 00:00:00 2001 From: Tony Bagnall Date: Thu, 3 Sep 2026 19:34:35 +0100 Subject: [PATCH 1/3] Port RankSCL The last of the survey's thirteen trustworthy papers that had no implementation here, and the one it singles out: "the only paper with no obvious problems in the exposed entry-point code that also provides usable results", reporting over five seeds. RankSCL is supervised contrastive learning with two departures. Positives are augmented in the embedding space rather than the input space, by jittering the representation of a randomly drawn same-class neighbour, and the loss is rank-based: for each positive it counts, through a sigmoid, the negatives that sit at least as close to the anchor, sums them, and compresses with arctan so a badly ranked positive saturates instead of dominating the batch. An SVM on the encoder representations does the classifying, the TS2Vec protocol the authors adopt, including its MAX_SAMPLES cap. Transcribed rather than vendored. The authors' modules import each other absolutely and pull in matplotlib and a logger writing to a hard-coded path, while the parts that matter are small: the FCN encoder, the augmentation and the loss. The loss is checked against a hand computation on four points on a line rather than only for running, since a transcription of it could be quietly wrong and still train. Two behaviours are ours: a single-class batch has nothing to rank and returns no loss where the original raises on an empty stack, and a collection smaller than batch_size is refused, since the original drops the last incomplete batch and would train on nothing. Defaults are the authors' UEA settings from scripts/uea.sh. 21/21 aeon checks, 14 tests. Co-Authored-By: Claude Opus 5 --- multiverse/classification/__init__.py | 2 + multiverse/classification/_rankscl.py | 490 ++++++++++++++++++ .../classification/tests/test_rankscl.py | 162 ++++++ 3 files changed, 654 insertions(+) create mode 100644 multiverse/classification/_rankscl.py create mode 100644 multiverse/classification/tests/test_rankscl.py diff --git a/multiverse/classification/__init__.py b/multiverse/classification/__init__.py index c97a596..0cf3557 100644 --- a/multiverse/classification/__init__.py +++ b/multiverse/classification/__init__.py @@ -7,6 +7,7 @@ "ConvTranClassifier", "DisjointCNNClassifier", "PatchMTSCClassifier", + "RankSCLClassifier", "TimesNetClassifier", "TS2VecClassifier", "XCMClassifier", @@ -16,6 +17,7 @@ from multiverse.classification._convtran import ConvTranClassifier from multiverse.classification._disjoint_cnn import DisjointCNNClassifier from multiverse.classification._patchmtsc import PatchMTSCClassifier +from multiverse.classification._rankscl import RankSCLClassifier from multiverse.classification._timesnet import TimesNetClassifier from multiverse.classification._ts2vec import TS2VecClassifier from multiverse.classification._xcm import XCMClassifier diff --git a/multiverse/classification/_rankscl.py b/multiverse/classification/_rankscl.py new file mode 100644 index 0000000..d64d9ef --- /dev/null +++ b/multiverse/classification/_rankscl.py @@ -0,0 +1,490 @@ +"""RankSCL classifier for aeon. + +Adapted from the authors' RankSCL implementation: +https://github.com/UConn-DSIS/Rank-Supervised-Contrastive-Learning-for-Time-Series-Classification + +RankSCL is supervised contrastive learning with two changes to the usual +recipe. Positives are augmented in the embedding space rather than the input +space, by jittering the representation of a same-class neighbour, and the loss +is rank-based: for each positive it counts, softly, how many negatives sit +closer to the anchor than that positive does. Training produces an encoder; +classification is an SVM fitted on the encoder's representations, as in +TS2Vec, whose evaluation protocol this inherits. + +The pieces are transcribed rather than vendored. The authors' modules import +each other absolutely and pull in matplotlib and a logging setup that writes to +a hard-coded path, none of which belongs in a library, and the parts that +matter are small: the FCN encoder, the embedding-space augmentation and the +ranking loss. + +This wrapper is designed for aeon and therefore assumes input X is a 3D NumPy +array with shape (n_cases, n_channels, n_timepoints), which is the layout the +authors' encoder expects after their transpose, so no reordering is needed. + +The original source is distributed under the MIT License. + +MIT License + +Copyright (c) 2024 UConn-DSIS + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +""" + +from __future__ import annotations + +__maintainer__ = ["TonyBagnall"] +__all__ = ["RankSCLClassifier"] + +import numpy as np +import torch +from aeon.classification import BaseClassifier +from sklearn.linear_model import LogisticRegression +from sklearn.model_selection import GridSearchCV, train_test_split +from sklearn.multiclass import OneVsRestClassifier +from sklearn.pipeline import make_pipeline +from sklearn.preprocessing import StandardScaler +from sklearn.svm import SVC +from sklearn.utils import check_random_state +from torch import nn +from torch.nn import functional +from torch.utils.data import DataLoader, TensorDataset + +#: The authors' UEA settings, from ``scripts/uea.sh``. +PAPER_DEFAULTS = { + "n_epochs": 100, + "batch_size": 4, + "learning_rate": 1e-4, + "weight_decay": 5e-4, + "aug_positives": 5, + "distance": "EU", +} + + +class _FCNEncoder(nn.Module): + """The authors' FCN encoder. + + A transcription of ``models/FCN.py::FCN``. Three dilated convolution blocks + widening 24 -> 64 -> 320, each with batch normalisation and ReLU, then + global average pooling to a 320 dimensional representation, and a + projection head of the same width used only during training. + + The dilations grow 2, 4, 8 while the kernels shrink 7, 5, 3, so the + receptive field widens without the parameter count following it. + + Defined at module level rather than inside a factory so that fitted + classifiers pickle, which aeon's estimator checks require. + """ + + def __init__(self, in_channels): + super().__init__() + self.encoder = nn.Sequential( + nn.Conv1d(in_channels, 24, kernel_size=7, padding=6, dilation=2), + nn.BatchNorm1d(24), + nn.ReLU(), + nn.Conv1d(24, 64, kernel_size=5, padding=8, dilation=4), + nn.BatchNorm1d(64), + nn.ReLU(), + nn.Conv1d(64, 320, kernel_size=3, padding=8, dilation=8), + nn.BatchNorm1d(320), + nn.ReLU(), + nn.AdaptiveAvgPool1d(1), + nn.Flatten(), + ) + self.projection = nn.Sequential( + nn.Linear(320, 320), + nn.BatchNorm1d(320), + nn.ReLU(), + nn.Linear(320, 320), + ) + + def forward(self, x): + """Return the projected embedding and the representation.""" + representation = self.encoder(x) + return self.projection(representation), representation + + +def _build_encoder(n_channels: int): + """Return a new encoder for a collection with this many channels.""" + return _FCNEncoder(n_channels) + + +def _same_class_neighbour(embeddings, labels, generator): + """Replace each embedding with that of a random same-class neighbour. + + ``utils/utils.py::generate_pos``. A case with no other member of its class + in the batch keeps its own embedding, which makes it its own positive. + """ + out = torch.zeros_like(embeddings) + for position in range(embeddings.shape[0]): + same = (labels == labels[position]).nonzero().flatten() + same = same[same != position] + if len(same) == 0: + out[position] = embeddings[position] + else: + pick = torch.randint( + len(same), (1,), generator=generator, device=same.device + ) + out[position] = embeddings[same[pick]] + return out + + +def _augment(embeddings, labels, n_positives, sigmas=(0.03, 0.05)): + """Augment positives in the embedding space by jittering. + + ``utils/augmentation.py::aug_data``. The normalised embeddings are kept, + and for each sigma ``n_positives`` jittered copies are appended, giving + ``2 * n_positives + 1`` blocks in all with the labels repeated to match. + + Note that the jitter is applied to the unnormalised embeddings while the + first block is normalised, which is what the authors do. + """ + stacked = [functional.normalize(embeddings, dim=1)] + for sigma in sigmas: + for _ in range(n_positives): + noise = torch.normal( + mean=0.0, std=sigma, size=embeddings.shape, device=embeddings.device + ) + stacked.append(embeddings + noise) + repeats = 1 + len(sigmas) * n_positives + return torch.cat(stacked, dim=0), labels.repeat(repeats) + + +def _ranking_loss(embeddings, labels, distance): + """The paper's rank-based contrastive loss. + + ``loss/Ranking_loss.py::Ranking_loss``. For every anchor and every positive + of that anchor, take the negatives that are at least as close to the anchor + as the positive is, which are the ones ranked wrongly, and sum a sigmoid of + how much closer they are. The per-positive sums are compressed with arctan + and averaged, so a badly ranked positive saturates rather than dominating. + + Returns None when the batch has no anchor with both a positive and a + negative, which the authors' version would raise on. + """ + if distance == "Cosine": + matrix = -torch.cosine_similarity( + embeddings.unsqueeze(1), embeddings.unsqueeze(0), dim=2 + ) + else: + matrix = torch.cdist(embeddings, embeddings, p=2) + + same = labels.reshape(1, -1) == labels.reshape(-1, 1) + violations = [] + for anchor in range(matrix.shape[0]): + negatives = matrix[anchor][~same[anchor]] + if negatives.numel() == 0: + continue + positives = same[anchor].nonzero().flatten() + for positive in positives[positives != anchor]: + gap = matrix[anchor, positive] + closer = negatives[negatives <= gap] + violations.append(torch.sigmoid(gap - closer).sum()) + + if not violations: + return None + return torch.atan(torch.stack(violations)).mean() + + +class RankSCLClassifier(BaseClassifier): + """Rank Supervised Contrastive Learning for time series classification. + + An FCN encoder is trained with a supervised contrastive objective in which + positives are augmented in the embedding space and the loss is rank-based, + counting how many negatives intrude on each positive. The representations + are then classified by an SVM, following the TS2Vec evaluation protocol the + authors adopt. + + Parameters + ---------- + n_epochs : int, default=100 + Encoder training epochs, the authors' ``epochs_up``. + batch_size : int, default=4 + Training batch size. The authors use 4 for the UEA archive. Batches + smaller than this are dropped, as in the original, so a collection with + fewer cases than ``batch_size`` cannot be trained on. + learning_rate : float, default=1e-4 + Adam learning rate. + weight_decay : float, default=5e-4 + Adam weight decay. + aug_positives : int, default=5 + Jittered copies generated per sigma, so the loss sees + ``2 * aug_positives + 1`` blocks per batch. + distance : {"EU", "Cosine"}, default="EU" + Distance the ranking is computed over. The authors use Euclidean for + the UEA archive. + probe : {"svm", "logistic"}, default="svm" + Classifier fitted on the representations, matching the protocol in + ``utils/_eval_protocols.py``. + probe_max_samples : int or None, default=None + Cap on the cases the probe is fitted on. None takes the authors' + values, 10000 for the SVM probe and 100000 for the logistic one, with + stratified subsampling above that. Their ``fit_svm`` carries the same + cap, inherited from TS2Vec. + device : {"auto", "cpu", "cuda"} or torch device string, default="auto" + Device used for training and encoding. + verbose : bool, default=False + Whether to print the loss every ten epochs, as the authors do. + random_state : int, RandomState instance or None, default=None + Seed controlling initialisation, batching, the neighbour draw and the + jitter. + + Attributes + ---------- + encoder_ : torch.nn.Module + The trained encoder. + probe_ : object + Classifier fitted on the encoded training collection. + probe_cases_ : int + Number of cases the probe was fitted on, after any subsampling. + history_ : list of dict + Mean loss per epoch. + device_ : str + Resolved device. + n_channels_ : int + Number of channels seen in ``fit``. + n_timepoints_ : int + Series length seen in ``fit``. + classes_ : np.ndarray + Class labels, from ``BaseClassifier``. + n_classes_ : int + Number of classes, from ``BaseClassifier``. + + References + ---------- + .. [1] Ren, Q., Luo, D. and Song, D. "Rank Supervised Contrastive Learning + for Time Series Classification." ICDM, 2024. + + Examples + -------- + >>> from aeon.testing.data_generation import make_example_3d_numpy + >>> from multiverse.classification import RankSCLClassifier + >>> X, y = make_example_3d_numpy(n_cases=8, n_channels=2, n_timepoints=20) + >>> clf = RankSCLClassifier(n_epochs=2) # doctest: +SKIP + >>> clf.fit(X, y) # doctest: +SKIP + """ + + _tags = { + "X_inner_type": "numpy3D", + "capability:multivariate": True, + "capability:unequal_length": False, + "algorithm_type": "deeplearning", + "non_deterministic": True, + "python_dependencies": "torch", + } + + def __init__( + self, + n_epochs: int = 100, + batch_size: int = 4, + learning_rate: float = 1e-4, + weight_decay: float = 5e-4, + aug_positives: int = 5, + distance: str = "EU", + probe: str = "svm", + probe_max_samples: int | None = None, + device: str = "auto", + verbose: bool = False, + random_state=None, + ): + self.n_epochs = n_epochs + self.batch_size = batch_size + self.learning_rate = learning_rate + self.weight_decay = weight_decay + self.aug_positives = aug_positives + self.distance = distance + self.probe = probe + self.probe_max_samples = probe_max_samples + self.device = device + self.verbose = verbose + self.random_state = random_state + super().__init__() + + def _validate_parameters(self) -> None: + """Check constructor parameters before any work is done.""" + for name in ["n_epochs", "batch_size"]: + value = getattr(self, name) + if not isinstance(value, int) or value <= 0: + raise ValueError(f"{name} must be a positive integer") + if not isinstance(self.aug_positives, int) or self.aug_positives < 0: + raise ValueError("aug_positives must be a non-negative integer") + if self.learning_rate < 0 or self.weight_decay < 0: + raise ValueError("learning_rate and weight_decay must be non-negative") + if self.distance not in ("EU", "Cosine"): + raise ValueError(f'distance must be "EU" or "Cosine", got {self.distance!r}') + if self.probe not in ("svm", "logistic"): + raise ValueError(f'probe must be "svm" or "logistic", got {self.probe!r}') + + def _resolve_device(self) -> str: + if self.device == "auto": + return "cuda" if torch.cuda.is_available() else "cpu" + if str(self.device).startswith("cuda") and not torch.cuda.is_available(): + raise RuntimeError("CUDA was requested but is not available") + return self.device + + def _encode(self, X: np.ndarray) -> np.ndarray: + """Return normalised encoder representations, as the probe expects.""" + self.encoder_.eval() + with torch.no_grad(): + batch = torch.as_tensor(X, dtype=torch.float32, device=self.device_) + _, representation = self.encoder_(batch) + representation = functional.normalize(representation, dim=1) + return representation.cpu().numpy() + + def _subsample(self, features, y): + """Cap the collection the probe is fitted on, as the authors do.""" + limit = self.probe_max_samples + if limit is None: + limit = 10_000 if self.probe == "svm" else 100_000 + if features.shape[0] <= limit: + return features, y + features, _, y, _ = train_test_split( + features, y, train_size=limit, random_state=0, stratify=y + ) + return features, y + + def _build_probe(self, n_cases: int, seed: int): + """Return the probe, following ``utils/_eval_protocols.py``.""" + if self.probe == "logistic": + return make_pipeline( + StandardScaler(), + OneVsRestClassifier( + LogisticRegression(max_iter=1000000, random_state=seed) + ), + ) + svm = SVC(C=np.inf, gamma="scale", probability=True, random_state=seed) + if n_cases // self.n_classes_ < 5 or n_cases < 50: + return svm + return GridSearchCV( + svm, + {"C": [0.0001, 0.001, 0.01, 0.1, 1, 10, 100, 1000, 10000, np.inf], + "kernel": ["rbf"], "gamma": ["scale"]}, + cv=5, + n_jobs=1, + ) + + def _fit(self, X: np.ndarray, y): + self._validate_parameters() + + rng = check_random_state(self.random_state) + seed = int(rng.randint(np.iinfo(np.int32).max)) + torch.manual_seed(seed) + np.random.seed(seed) + + self.device_ = self._resolve_device() + self.n_channels_, self.n_timepoints_ = X.shape[1], X.shape[2] + + encoded_y = np.asarray( + [self._class_dictionary[label] for label in y], dtype=np.int64 + ) + if X.shape[0] < self.batch_size: + raise ValueError( + f"batch_size={self.batch_size} exceeds the {X.shape[0]} training " + "cases, and the authors drop the last incomplete batch, so no " + "batch would be formed. Reduce batch_size." + ) + + self.encoder_ = _build_encoder(self.n_channels_).to(self.device_) + optimizer = torch.optim.Adam( + self.encoder_.parameters(), + lr=self.learning_rate, + weight_decay=self.weight_decay, + ) + generator = torch.Generator(device=self.device_).manual_seed(seed) + loader = DataLoader( + TensorDataset( + torch.as_tensor(X, dtype=torch.float32), + torch.as_tensor(encoded_y, dtype=torch.long), + ), + batch_size=self.batch_size, + shuffle=True, + drop_last=True, + ) + + self.history_ = [] + for epoch in range(self.n_epochs): + self.encoder_.train() + losses = [] + for batch, labels in loader: + batch = batch.to(self.device_) + labels = labels.to(self.device_) + optimizer.zero_grad() + + projected, _ = self.encoder_(batch) + neighbours = _same_class_neighbour(projected, labels, generator) + augmented, repeated = _augment( + neighbours, labels, self.aug_positives + ) + # The anchors themselves replace the first block, so the loss + # sees each anchor once and its jittered positives after it. + augmented = torch.cat( + [functional.normalize(projected, dim=1), + augmented[projected.shape[0]:]], + dim=0, + ) + loss = _ranking_loss(augmented, repeated, self.distance) + if loss is None: + continue + loss.backward() + optimizer.step() + losses.append(float(loss.item())) + + mean_loss = float(np.mean(losses)) if losses else float("nan") + self.history_.append({"epoch": epoch + 1, "loss": mean_loss}) + if self.verbose and epoch % 10 == 0: + print(f"Epoch {epoch + 1} ----- loss {mean_loss:.3f}") + + representations = self._encode(X) + fit_features, fit_y = self._subsample(representations, encoded_y) + self.probe_cases_ = int(fit_features.shape[0]) + self.probe_ = self._build_probe(self.probe_cases_, seed).fit( + fit_features, fit_y + ) + return self + + def _check_shape(self, X: np.ndarray) -> None: + if X.shape[1] != self.n_channels_: + raise ValueError( + f"X has {X.shape[1]} channels, but the classifier was fitted " + f"with {self.n_channels_}." + ) + if X.shape[2] != self.n_timepoints_: + raise ValueError( + f"X has length {X.shape[2]}, but the classifier was fitted with " + f"length {self.n_timepoints_}." + ) + + def _predict_proba(self, X: np.ndarray) -> np.ndarray: + self._check_shape(X) + return self.probe_.predict_proba(self._encode(X)) + + def _predict(self, X: np.ndarray): + self._check_shape(X) + return self.classes_[self.probe_.predict(self._encode(X))] + + @classmethod + def _get_test_params(cls, parameter_set: str = "default") -> dict: + """Return a small parameter set for aeon estimator checks.""" + return { + "n_epochs": 2, + "batch_size": 2, + "aug_positives": 1, + "probe": "logistic", + "device": "cpu", + "random_state": 0, + } diff --git a/multiverse/classification/tests/test_rankscl.py b/multiverse/classification/tests/test_rankscl.py new file mode 100644 index 0000000..f37cf7a --- /dev/null +++ b/multiverse/classification/tests/test_rankscl.py @@ -0,0 +1,162 @@ +"""Tests for the RankSCL port. + +The ranking loss is checked against a hand computation rather than only for +running, since it is the paper's contribution and a transcription of it is +exactly the kind of thing that can be subtly wrong while still training. +""" + +import math + +import numpy as np +import pytest + +pytest.importorskip("torch") + +import torch # noqa: E402 +from aeon.testing.data_generation import make_example_3d_numpy # noqa: E402 + +from multiverse.classification import RankSCLClassifier # noqa: E402 +from multiverse.classification._rankscl import ( # noqa: E402 + _augment, + _build_encoder, + _ranking_loss, + _same_class_neighbour, +) + +SMALL = { + "n_epochs": 2, + "batch_size": 4, + "aug_positives": 1, + "probe": "logistic", + "device": "cpu", + "random_state": 0, +} + + +def _data(n_cases=20, n_channels=2, n_timepoints=40, n_labels=2): + return make_example_3d_numpy( + n_cases=n_cases, + n_channels=n_channels, + n_timepoints=n_timepoints, + n_labels=n_labels, + random_state=0, + ) + + +def test_ranking_loss_matches_a_hand_computation(): + """Four points on a line, so every distance can be worked out by hand.""" + embeddings = torch.tensor([[0.0], [1.0], [0.5], [3.0]]) + labels = torch.tensor([0, 0, 1, 1]) + + def distance(i, j): + return abs(embeddings[i, 0].item() - embeddings[j, 0].item()) + + terms = [] + for anchor in range(4): + negatives = [distance(anchor, n) for n in range(4) if labels[n] != labels[anchor]] + for positive in range(4): + if positive == anchor or labels[positive] != labels[anchor]: + continue + gap = distance(anchor, positive) + closer = [n for n in negatives if n <= gap] + terms.append(sum(1 / (1 + math.exp(-(gap - c))) for c in closer)) + expected = sum(math.atan(t) for t in terms) / len(terms) + + assert float(_ranking_loss(embeddings, labels, "EU")) == pytest.approx(expected) + + +def test_ranking_loss_is_none_without_negatives(): + """A single-class batch has nothing to rank, where the original raises.""" + embeddings = torch.tensor([[0.0], [1.0]]) + assert _ranking_loss(embeddings, torch.tensor([0, 0]), "EU") is None + + +def test_same_class_neighbour_draws_within_the_class(): + """Every replacement comes from the same class, and never from itself.""" + embeddings = torch.arange(6, dtype=torch.float32).reshape(6, 1) + labels = torch.tensor([0, 0, 0, 1, 1, 1]) + generator = torch.Generator().manual_seed(0) + out = _same_class_neighbour(embeddings, labels, generator) + for position in range(6): + drawn = int(out[position, 0]) + assert labels[drawn] == labels[position] + assert drawn != position + + +def test_same_class_neighbour_keeps_a_singleton(): + """A class with one member in the batch becomes its own positive.""" + embeddings = torch.tensor([[0.0], [1.0], [2.0]]) + labels = torch.tensor([0, 0, 1]) + out = _same_class_neighbour(embeddings, labels, torch.Generator().manual_seed(0)) + assert float(out[2, 0]) == 2.0 + + +def test_augment_shapes_follow_the_positive_count(): + """The batch grows to 2 * aug_positives + 1 blocks, labels with it.""" + embeddings = torch.randn(4, 8) + labels = torch.tensor([0, 1, 0, 1]) + augmented, repeated = _augment(embeddings, labels, 3) + assert augmented.shape == (4 * (2 * 3 + 1), 8) + assert repeated.shape == (4 * (2 * 3 + 1),) + # the first block is the normalised input, and the rest are jittered + assert torch.allclose(augmented[:4], torch.nn.functional.normalize(embeddings, dim=1)) + assert not torch.allclose(augmented[4:8], embeddings) + + +def test_encoder_shapes(): + """The encoder gives a 320 wide representation and projection.""" + model = _build_encoder(3) + projected, representation = model(torch.randn(5, 3, 60)) + assert projected.shape == (5, 320) + assert representation.shape == (5, 320) + + +def test_fit_predict_proba(): + """Probabilities are well formed and predictions are known labels.""" + X, y = _data() + clf = RankSCLClassifier(**SMALL).fit(X, y) + proba = clf.predict_proba(X) + assert proba.shape == (len(y), clf.n_classes_) + assert np.allclose(proba.sum(axis=1), 1) + assert set(clf.predict(X)).issubset(set(clf.classes_)) + + +def test_repeatable_on_cpu(): + """The same seed gives the same probabilities.""" + X, y = _data() + first = RankSCLClassifier(**SMALL).fit(X, y).predict_proba(X) + second = RankSCLClassifier(**SMALL).fit(X, y).predict_proba(X) + assert np.allclose(first, second) + + +def test_batch_size_larger_than_the_collection_is_refused(): + """The original drops the last incomplete batch, so this trains on nothing.""" + X, y = _data(n_cases=3) + with pytest.raises(ValueError, match="exceeds the 3 training cases"): + RankSCLClassifier(**{**SMALL, "batch_size": 8}).fit(X, y) + + +@pytest.mark.parametrize( + "parameters,message", + [ + ({"distance": "manhattan"}, "distance"), + ({"probe": "forest"}, "probe"), + ({"aug_positives": -1}, "aug_positives"), + ({"n_epochs": 0}, "n_epochs"), + ], +) +def test_parameters_are_validated(parameters, message): + """Bad parameters are rejected before any training happens.""" + X, y = _data() + with pytest.raises(ValueError, match=message): + RankSCLClassifier(**{**SMALL, **parameters}).fit(X, y) + + +def test_shape_is_checked_at_predict(): + """A collection of a different shape is refused rather than mispredicted.""" + X, y = _data() + clf = RankSCLClassifier(**SMALL).fit(X, y) + with pytest.raises(ValueError, match="channels"): + clf.predict(np.random.random((4, 5, 40))) + with pytest.raises(ValueError, match="length"): + clf.predict(np.random.random((4, 2, 17))) From 420149124461ad5f1f10152c7c83e5365ffceec7 Mon Sep 17 00:00:00 2001 From: Tony Bagnall Date: Thu, 3 Sep 2026 22:04:39 +0100 Subject: [PATCH 2/3] Add a UEA leaderboard as its own page The UEA archive is the 30-dataset collection almost every published multivariate result is quoted on, so a table restricted to it is what a reader comparing against the literature needs. It is a subset view of the same runs behind the Multiverse-core leaderboard, not a separate experiment: the same function with a different dataset list. Coverage is stated rather than left to be inferred, because a partial-coverage table is precisely what the 2026 MTSC survey criticises in the literature. Four of the thirty are not in Multiverse-core and have no results at all here, BasicMotions, FingerMovements, InsectWingbeat and SelfRegulationSCP2, which the page names. Of the twenty-six that remain, twenty-three have a result for every estimator and those are what the ranking uses. leaderboard_markdown gains a collection parameter. Its caption was hard-coded to "Multiverse-core", so the UEA table would have claimed to average over Multiverse-core datasets while covering 23 UEA ones. HC2 leads on this subset too, though the order below it differs from Multiverse-core: RDST and MRHydra come second and third here. Co-Authored-By: Claude Opus 5 --- docs/leaderboard.md | 66 +++++++++++++ multiverse/experiments/tables.py | 30 +++++- results/multiverse/leaderboard_uea.html | 126 ++++++++++++++++++++++++ 3 files changed, 220 insertions(+), 2 deletions(-) create mode 100644 results/multiverse/leaderboard_uea.html diff --git a/docs/leaderboard.md b/docs/leaderboard.md index 2f0c747..61d204c 100644 --- a/docs/leaderboard.md +++ b/docs/leaderboard.md @@ -83,6 +83,72 @@ build your own table. ## Multiverse-core +## UEA + +The UEA archive is the older 30-dataset collection that almost every published +multivariate result is quoted on, so a table restricted to it is what a reader +comparing against the literature needs. This is a subset view of the same runs that +produce the Multiverse-core leaderboard, not a separate experiment, built by passing +`UEA` as the dataset list: + +```python +from aeon.datasets.tsc_datasets import UEA +from multiverse.experiments.tables import available_estimators, leaderboard + +leaderboard( + datasets=sorted(UEA), + estimators=available_estimators(exclude=("DisjointCNN-Aeon",)), + sort_by="accuracy", + title="UEA leaderboard", + output_path="results/multiverse/leaderboard_uea.html", +) +``` + +**Read the coverage before the ranking.** Four of the thirty are not in +Multiverse-core and have no results here at all: BasicMotions, FingerMovements, +InsectWingbeat and SelfRegulationSCP2. Of the twenty-six that remain, the table uses +only those with a result for every estimator, and the page lists what it dropped and +why. A partial-coverage table is exactly what the 2026 MTSC survey criticises in the +literature, so the number of datasets is stated on the page rather than left to be +inferred from the ranking. + + +| # | Estimator | Accuracy rank | Accuracy | Balanced accuracy | AUROC | F1 | Log loss ↓ | Sensitivity | Specificity | +|---|---|---|---|---|---|---|---|---|---| +| 1 | HC2 | **7.13** | **0.7665** | **0.7452** | 0.8823 | 0.7411 | **0.6692** | 0.7470 | **0.7703** | +| 2 | RDST | 8.50 | 0.7459 | 0.7294 | 0.8179 | 0.7243 | 9.1587 | 0.7263 | 0.7560 | +| 3 | MRHydra | 8.87 | 0.7523 | 0.7388 | 0.8236 | **0.7432** | 8.9285 | 0.7599 | 0.7360 | +| 4 | Arsenal | 9.98 | 0.7321 | 0.7134 | 0.8439 | 0.7116 | 5.5624 | 0.7119 | 0.7417 | +| 5 | ROCKET | 9.98 | 0.7317 | 0.7146 | 0.8089 | 0.7130 | 9.6705 | 0.7133 | 0.7393 | +| 6 | RIST | 10.20 | 0.7433 | 0.7278 | 0.8755 | 0.7325 | 0.7983 | 0.7454 | 0.7322 | +| 7 | H-InceptionTime | 10.28 | 0.7223 | 0.7230 | 0.8653 | 0.6967 | 1.5030 | 0.7053 | 0.7345 | +| 8 | CIF | 10.87 | 0.7525 | 0.7378 | 0.8825 | 0.7400 | 0.8488 | **0.7604** | 0.7349 | +| 9 | FreshPRINCE | 11.24 | 0.7422 | 0.7281 | 0.8796 | 0.7239 | 0.7764 | 0.7313 | 0.7457 | +| 10 | DrCIF | 11.39 | 0.7386 | 0.7252 | 0.8734 | 0.7246 | 0.8458 | 0.7384 | 0.7303 | +| 11 | LITETime-MV | 11.54 | 0.7073 | 0.7064 | 0.8568 | 0.6779 | 1.4779 | 0.6905 | 0.7218 | +| 12 | LiteTIME | 12.00 | 0.7087 | 0.7019 | 0.8576 | 0.6751 | 1.6854 | 0.6985 | 0.7204 | +| 13 | QUANT | 13.26 | 0.7285 | 0.7171 | **0.8888** | 0.7195 | 0.8041 | 0.7421 | 0.7074 | +| 14 | STSF | 13.61 | 0.7345 | 0.7223 | 0.8774 | 0.6934 | 0.8338 | 0.7007 | 0.7600 | +| 15 | TS2Vec | 14.11 | 0.7070 | 0.6913 | 0.8470 | 0.6917 | 0.8902 | 0.7150 | 0.6877 | +| 16 | TDE | 14.65 | 0.7079 | 0.6862 | 0.8484 | 0.6775 | 1.1475 | 0.6897 | 0.7095 | +| 17 | PatchMTSC | 14.72 | 0.7110 | 0.6986 | 0.8601 | 0.6899 | 0.7670 | 0.7192 | 0.6928 | +| 18 | ConvTran | 15.46 | 0.6931 | 0.6801 | 0.8552 | 0.6793 | 0.8155 | 0.7049 | 0.6736 | +| 19 | TSF | 15.54 | 0.7214 | 0.7076 | 0.8671 | 0.6917 | 0.9127 | 0.6977 | 0.7365 | +| 20 | STC | 15.57 | 0.7265 | 0.7036 | 0.8803 | 0.7035 | 0.8124 | 0.7186 | 0.7184 | +| 21 | Catch22 | 15.65 | 0.7006 | 0.6854 | 0.8557 | 0.6897 | 0.9814 | 0.7096 | 0.6802 | +| 22 | 1NN-DTW | 16.87 | 0.6848 | 0.6759 | 0.7785 | 0.6702 | 11.3600 | 0.6720 | 0.6879 | +| 23 | TimesURL | 17.50 | 0.6809 | 0.6658 | 0.8290 | 0.6539 | 1.3557 | 0.6698 | 0.6738 | +| 24 | Summary | 18.78 | 0.6477 | 0.6355 | 0.8295 | 0.6206 | 1.3000 | 0.6291 | 0.6589 | +| 25 | TimesNet | 19.35 | 0.6584 | 0.6504 | 0.8332 | 0.6386 | 1.1641 | 0.6628 | 0.6504 | +| 26 | Dummy | 23.96 | 0.2168 | 0.1980 | 0.5000 | 0.0800 | 1.9123 | 0.1853 | 0.2288 | + +Average over the 23 UEA datasets with results for every estimator on every metric, ordered by average accuracy rank. Best in each column in bold. + + +Sortable version with per-metric ranks: +[`results/multiverse/leaderboard_uea.html`](../results/multiverse/leaderboard_uea.html) +([preview](https://raw.githack.com/aeon-toolkit/multiverse/main/results/multiverse/leaderboard_uea.html)). + ## EEG archive diff --git a/multiverse/experiments/tables.py b/multiverse/experiments/tables.py index 5333722..d00508b 100644 --- a/multiverse/experiments/tables.py +++ b/multiverse/experiments/tables.py @@ -748,6 +748,7 @@ def leaderboard_markdown( sort_by: str = "accuracy", results_dir: Path | str = DEFAULT_RESULTS_DIR, decimals: int = 4, + collection: str = "Multiverse-core", ) -> str: """Return the leaderboard as a Markdown table. @@ -770,6 +771,9 @@ def leaderboard_markdown( Directory holding one sub-directory per estimator. decimals : int, default=4 Decimal places for scores. + collection : str + Name of the dataset collection, used in the caption so a table + built over a subset does not claim to cover the whole archive. Returns ------- @@ -820,7 +824,7 @@ def leaderboard_markdown( rows.append("") rows.append( - f"Average over the {len(common)} Multiverse-core datasets with results for every " + f"Average over the {len(common)} {collection} datasets with results for every " f"estimator on every metric, ordered by average {sort_label.lower()} rank. Best " "in each column in bold." ) @@ -1156,7 +1160,7 @@ def main() -> None: deleted, but listing them would read as a claim about the method. The Multiverse port of the same method reports under DisjointCNN. """ - from aeon.datasets.tsc_datasets import multiverse_core + from aeon.datasets.tsc_datasets import UEA, multiverse_core datasets = sorted(multiverse_core) estimators = available_estimators(exclude=("DisjointCNN-Aeon",)) @@ -1178,6 +1182,28 @@ def main() -> None: ) print(f"wrote {datasets_path}") + # The UEA archive is the older 30 dataset collection almost every published + # MTSC result is quoted on, so a table restricted to it is what a reader + # comparing against the literature actually needs. It is a subset view of + # the same runs, not a separate experiment. + uea_path = leaderboard( + sorted(UEA), + estimators, + sort_by="accuracy", + title="UEA leaderboard", + output_path=Path(DEFAULT_RESULTS_DIR) / "leaderboard_uea.html", + ) + print(f"wrote {uea_path}") + + docs = Path(__file__).resolve().parents[2] / "docs" / "leaderboard.md" + uea_table = leaderboard_markdown( + sorted(UEA), estimators, sort_by="accuracy", collection="UEA" + ) + if write_markdown_table(docs, uea_table, marker="UEA_LEADERBOARD"): + print(f"updated the UEA table in {docs}") + else: + print(f"no UEA_LEADERBOARD markers in {docs}; Markdown table not written") + table = leaderboard_markdown(datasets, estimators, sort_by="accuracy") readme = Path(__file__).resolve().parents[2] / "README.md" if write_markdown_table(readme, table): diff --git a/results/multiverse/leaderboard_uea.html b/results/multiverse/leaderboard_uea.html new file mode 100644 index 0000000..8422a45 --- /dev/null +++ b/results/multiverse/leaderboard_uea.html @@ -0,0 +1,126 @@ +UEA leaderboard

UEA leaderboard

26 estimators on 23 datasets · 7 metrics · ordered by average accuracy rank · built 2026-09-03

#EstimatorAccuracyBalanced accuracyAUROCF1Log loss ↓SensitivitySpecificity
ScoreRankScoreRankScoreRankScoreRankScoreRankScoreRankScoreRank
1HC20.76657.130.74528.090.88237.130.74118.110.66926.390.74708.540.77037.04
2RDST0.74598.500.72948.800.817918.740.72438.809.158720.910.72639.300.75608.57
3MRHydra0.75238.870.73888.780.823618.630.74328.178.928520.960.75998.220.73609.02
4Arsenal0.73219.980.713410.590.843915.130.711610.205.562418.090.711910.760.74179.35
5ROCKET0.73179.980.714610.410.808919.910.713010.709.670522.070.713311.350.73939.80
6RIST0.743310.200.727810.460.87558.670.732510.520.798310.090.745410.700.732211.24
7H-InceptionTime0.722310.280.72309.740.86538.760.696710.371.503011.480.705310.130.734510.78
8CIF0.752510.870.737811.070.88258.960.740010.720.848811.260.760411.040.734911.30
9FreshPRINCE0.742211.240.728111.480.87968.370.723912.130.77647.870.731312.110.745712.20
10DrCIF0.738611.390.725211.390.87349.910.724612.240.845811.520.738412.040.730312.48
11LITETime-MV0.707311.540.706410.670.85688.960.677910.411.477910.390.690510.170.721811.00
12LiteTIME0.708712.000.701911.130.85769.930.675111.571.685410.520.698510.480.720411.57
13QUANT0.728513.260.717112.980.88888.370.719513.070.80419.130.742112.540.707414.04
14STSF0.734513.610.722313.570.877411.720.693414.040.833810.130.700714.330.760013.20
15TS2Vec0.707014.110.691314.540.847013.850.691714.000.890212.000.715014.040.687714.41
16TDE0.707914.650.686214.780.848413.200.677514.571.147512.830.689714.800.709514.07
17PatchMTSC0.711014.720.698614.370.860111.330.689914.130.76708.000.719213.890.692814.87
18ConvTran0.693115.460.680115.110.855212.330.679314.500.815510.220.704914.800.673616.22
19TSF0.721415.540.707615.300.867112.220.691715.040.912711.520.697715.480.736514.91
20STC0.726515.570.703616.110.880311.780.703515.670.812411.000.718616.090.718415.11
21Catch220.700615.650.685416.200.855713.020.689715.670.981414.000.709615.260.680216.11
221NN-DTW0.684816.870.675916.300.778522.280.670216.4811.360023.850.672016.460.687916.43
23TimesURL0.680917.500.665817.480.829018.520.653917.481.355717.910.669818.090.673816.93
24Summary0.647718.780.635518.720.829517.700.620618.571.300016.610.629118.390.658918.33
25TimesNet0.658419.350.650418.700.833217.020.638619.261.164113.740.662818.460.650419.67
26Dummy0.216823.960.198024.240.500024.570.080024.591.912318.520.185323.520.228822.35

Average score and average rank over the 23 datasets with results for every estimator on every metric. Best in each column is highlighted. Metrics marked ↓ are better when lower.

Missing results

  • MRHydra — PenDigits (ValueError: n_timepoints must be >= 9, but found 8)
  • FreshPRINCE — FaceDetection (OOM at 128GB)
  • STSF — PenDigits (not recorded)
  • ConvTran — EigenWorms (CUDA out of memory)

Scoring uses the 23 datasets every estimator completed, so a dataset any one of them is missing is left out for all. Reasons are from the job logs of these runs.

4 requested dataset(s) have no results from any estimator: BasicMotions, FingerMovements, InsectWingbeat, SelfRegulationSCP2.

Reproducing this page

from multiverse.experiments.tables import leaderboard
+
+leaderboard(
+    datasets=sorted(UEA),
+    estimators=["HC2", "RDST", "MRHydra", "Arsenal", "ROCKET", "RIST", "H-InceptionTime", "CIF", "FreshPRINCE", "DrCIF", "LITETime-MV", "LiteTIME", "QUANT", "STSF", "TS2Vec", "TDE", "PatchMTSC", "ConvTran", "TSF", "STC", "Catch22", "1NN-DTW", "TimesURL", "Summary", "TimesNet", "Dummy"],
+    metrics=["accuracy", "balacc", "auroc", "f1", "logloss", "sensitivity", "specificity"],
+    sort_by="accuracy",
+)

Or python -m multiverse.experiments.tables to rebuild it with the defaults.

\ No newline at end of file From f8e7ef2a020a33e3b57f383ce33c35fd3efdba04 Mon Sep 17 00:00:00 2001 From: Tony Bagnall Date: Sat, 5 Sep 2026 14:07:06 +0100 Subject: [PATCH 3/3] Ingest the Disjoint-CNN port 65 of the 66 core datasets at resample 0; EmoPain is the only gap, and aeon rejects it before fit for every classifier on its variance check. The port enters the leaderboard at rank 13.63, between LiteTIME and ConvTran. Against aeon's estimator on the same 65 datasets it is +0.126 mean accuracy, 50 wins to 13 with 2 ties, Wilcoxon p = 3.3e-07, and on the 23 UEA datasets we hold, closest to the paper's own evaluation set, +0.170 at 20/2/1. ERing, the dataset the bug was traced on, goes 0.589 to 0.956 against 0.964 published. aeon's results stay under DisjointCNN-Aeon and stay excluded from the table. The common set is unchanged at 51, since EmoPain was already outside it. Co-Authored-By: Claude Opus 5 --- README.md | 53 +++++++-------- docs/leaderboard.md | 53 +++++++-------- .../DisjointCNN/DisjointCNN_accuracy.csv | 66 +++++++++++++++++++ .../DisjointCNN/DisjointCNN_auroc.csv | 66 +++++++++++++++++++ .../DisjointCNN/DisjointCNN_balacc.csv | 66 +++++++++++++++++++ .../multiverse/DisjointCNN/DisjointCNN_f1.csv | 66 +++++++++++++++++++ .../DisjointCNN/DisjointCNN_logloss.csv | 66 +++++++++++++++++++ .../DisjointCNN/DisjointCNN_sensitivity.csv | 66 +++++++++++++++++++ .../DisjointCNN/DisjointCNN_specificity.csv | 66 +++++++++++++++++++ results/multiverse/datasets.html | 2 +- results/multiverse/leaderboard.html | 4 +- results/multiverse/leaderboard_uea.html | 4 +- 12 files changed, 521 insertions(+), 57 deletions(-) create mode 100644 results/multiverse/DisjointCNN/DisjointCNN_accuracy.csv create mode 100644 results/multiverse/DisjointCNN/DisjointCNN_auroc.csv create mode 100644 results/multiverse/DisjointCNN/DisjointCNN_balacc.csv create mode 100644 results/multiverse/DisjointCNN/DisjointCNN_f1.csv create mode 100644 results/multiverse/DisjointCNN/DisjointCNN_logloss.csv create mode 100644 results/multiverse/DisjointCNN/DisjointCNN_sensitivity.csv create mode 100644 results/multiverse/DisjointCNN/DisjointCNN_specificity.csv diff --git a/README.md b/README.md index 6af806d..8ee3cdc 100644 --- a/README.md +++ b/README.md @@ -40,32 +40,33 @@ The current paper version describes: | # | Estimator | Accuracy rank | Accuracy | Balanced accuracy | AUROC | F1 | Log loss ↓ | Sensitivity | Specificity | |---|---|---|---|---|---|---|---|---|---| -| 1 | HC2 | **8.11** | **0.7887** | 0.7541 | **0.9000** | 0.7346 | **0.5440** | 0.7547 | **0.7910** | -| 2 | MRHydra | 8.92 | 0.7810 | **0.7579** | 0.8130 | **0.7368** | 7.8942 | **0.7715** | 0.7718 | -| 3 | RDST | 9.76 | 0.7707 | 0.7372 | 0.7963 | 0.7105 | 8.2660 | 0.7236 | 0.7833 | -| 4 | RIST | 10.38 | 0.7693 | 0.7422 | 0.8755 | 0.7221 | 0.6294 | 0.7504 | 0.7613 | -| 5 | DrCIF | 10.44 | 0.7721 | 0.7454 | 0.8821 | 0.7248 | 0.6558 | 0.7490 | 0.7669 | -| 6 | CIF | 10.61 | 0.7756 | 0.7497 | 0.8920 | 0.7288 | 0.6497 | 0.7536 | 0.7714 | -| 7 | FreshPRINCE | 10.63 | 0.7717 | 0.7516 | 0.8752 | 0.7293 | 0.6075 | 0.7515 | 0.7731 | -| 8 | Arsenal | 11.04 | 0.7654 | 0.7340 | 0.8471 | 0.7092 | 3.9265 | 0.7337 | 0.7696 | -| 9 | QUANT | 11.19 | 0.7693 | 0.7486 | 0.8839 | 0.7262 | 0.6238 | 0.7616 | 0.7539 | -| 10 | LITETime-MV | 11.60 | 0.7476 | 0.7312 | 0.8518 | 0.6875 | 1.3300 | 0.7200 | 0.7600 | -| 11 | ROCKET | 11.61 | 0.7661 | 0.7345 | 0.7955 | 0.7080 | 8.4299 | 0.7282 | 0.7724 | -| 12 | STSF | 12.08 | 0.7698 | 0.7503 | 0.8813 | 0.7155 | 0.6493 | 0.7439 | 0.7790 | -| 13 | H-InceptionTime | 12.28 | 0.7375 | 0.7205 | 0.8506 | 0.6897 | 1.3334 | 0.7303 | 0.7333 | -| 14 | LiteTIME | 12.79 | 0.7308 | 0.7122 | 0.8402 | 0.6746 | 1.4921 | 0.7199 | 0.7291 | -| 15 | ConvTran | 13.81 | 0.7430 | 0.7139 | 0.8606 | 0.6882 | 0.8300 | 0.7289 | 0.7295 | -| 16 | PatchMTSC | 13.92 | 0.7395 | 0.6934 | 0.8288 | 0.6660 | 0.7748 | 0.6985 | 0.7300 | -| 17 | Catch22 | 13.95 | 0.7442 | 0.7203 | 0.8703 | 0.6996 | 0.7238 | 0.7337 | 0.7326 | -| 18 | STC | 14.61 | 0.7516 | 0.7188 | 0.8748 | 0.7004 | 0.6447 | 0.7264 | 0.7496 | -| 19 | TSF | 14.73 | 0.7484 | 0.7257 | 0.8747 | 0.6952 | 0.7335 | 0.7179 | 0.7565 | -| 20 | TS2Vec | 15.26 | 0.7212 | 0.6849 | 0.8082 | 0.6588 | 0.7326 | 0.6980 | 0.7100 | -| 21 | TDE | 15.39 | 0.7230 | 0.6823 | 0.8383 | 0.6441 | 0.8859 | 0.6786 | 0.7301 | -| 22 | Summary | 17.85 | 0.6814 | 0.6586 | 0.8263 | 0.6294 | 0.9251 | 0.6661 | 0.6787 | -| 23 | TimesNet | 18.16 | 0.6971 | 0.6688 | 0.8280 | 0.6390 | 1.1785 | 0.6850 | 0.6838 | -| 24 | TimesURL | 18.25 | 0.6916 | 0.6563 | 0.7931 | 0.6084 | 1.0193 | 0.6379 | 0.6914 | -| 25 | 1NN-DTW | 19.71 | 0.6672 | 0.6457 | 0.7214 | 0.6193 | 11.9949 | 0.6584 | 0.6584 | -| 26 | Dummy | 23.91 | 0.3538 | 0.2991 | 0.5000 | 0.1537 | 1.4284 | 0.2911 | 0.3695 | +| 1 | HC2 | **8.40** | **0.7887** | 0.7541 | **0.9000** | 0.7346 | **0.5440** | 0.7547 | **0.7910** | +| 2 | MRHydra | 9.21 | 0.7810 | **0.7579** | 0.8130 | **0.7368** | 7.8942 | **0.7715** | 0.7718 | +| 3 | RDST | 10.10 | 0.7707 | 0.7372 | 0.7963 | 0.7105 | 8.2660 | 0.7236 | 0.7833 | +| 4 | RIST | 10.75 | 0.7693 | 0.7422 | 0.8755 | 0.7221 | 0.6294 | 0.7504 | 0.7613 | +| 5 | DrCIF | 10.87 | 0.7721 | 0.7454 | 0.8821 | 0.7248 | 0.6558 | 0.7490 | 0.7669 | +| 6 | CIF | 11.07 | 0.7756 | 0.7497 | 0.8920 | 0.7288 | 0.6497 | 0.7536 | 0.7714 | +| 7 | FreshPRINCE | 11.08 | 0.7717 | 0.7516 | 0.8752 | 0.7293 | 0.6075 | 0.7515 | 0.7731 | +| 8 | Arsenal | 11.42 | 0.7654 | 0.7340 | 0.8471 | 0.7092 | 3.9265 | 0.7337 | 0.7696 | +| 9 | QUANT | 11.65 | 0.7693 | 0.7486 | 0.8839 | 0.7262 | 0.6238 | 0.7616 | 0.7539 | +| 10 | LITETime-MV | 11.97 | 0.7476 | 0.7312 | 0.8518 | 0.6875 | 1.3300 | 0.7200 | 0.7600 | +| 11 | ROCKET | 12.01 | 0.7661 | 0.7345 | 0.7955 | 0.7080 | 8.4299 | 0.7282 | 0.7724 | +| 12 | STSF | 12.60 | 0.7698 | 0.7503 | 0.8813 | 0.7155 | 0.6493 | 0.7439 | 0.7790 | +| 13 | H-InceptionTime | 12.71 | 0.7375 | 0.7205 | 0.8506 | 0.6897 | 1.3334 | 0.7303 | 0.7333 | +| 14 | LiteTIME | 13.22 | 0.7308 | 0.7122 | 0.8402 | 0.6746 | 1.4921 | 0.7199 | 0.7291 | +| 15 | DisjointCNN | 13.63 | 0.7286 | 0.7061 | 0.8354 | 0.6688 | 1.9705 | 0.6889 | 0.7368 | +| 16 | ConvTran | 14.37 | 0.7430 | 0.7139 | 0.8606 | 0.6882 | 0.8300 | 0.7289 | 0.7295 | +| 17 | Catch22 | 14.50 | 0.7442 | 0.7203 | 0.8703 | 0.6996 | 0.7238 | 0.7337 | 0.7326 | +| 18 | PatchMTSC | 14.51 | 0.7395 | 0.6934 | 0.8288 | 0.6660 | 0.7748 | 0.6985 | 0.7300 | +| 19 | STC | 15.19 | 0.7516 | 0.7188 | 0.8748 | 0.7004 | 0.6447 | 0.7264 | 0.7496 | +| 20 | TSF | 15.36 | 0.7484 | 0.7257 | 0.8747 | 0.6952 | 0.7335 | 0.7179 | 0.7565 | +| 21 | TS2Vec | 15.87 | 0.7212 | 0.6849 | 0.8082 | 0.6588 | 0.7326 | 0.6980 | 0.7100 | +| 22 | TDE | 15.93 | 0.7230 | 0.6823 | 0.8383 | 0.6441 | 0.8859 | 0.6786 | 0.7301 | +| 23 | Summary | 18.54 | 0.6814 | 0.6586 | 0.8263 | 0.6294 | 0.9251 | 0.6661 | 0.6787 | +| 24 | TimesNet | 18.86 | 0.6971 | 0.6688 | 0.8280 | 0.6390 | 1.1785 | 0.6850 | 0.6838 | +| 25 | TimesURL | 18.95 | 0.6916 | 0.6563 | 0.7931 | 0.6084 | 1.0193 | 0.6379 | 0.6914 | +| 26 | 1NN-DTW | 20.47 | 0.6672 | 0.6457 | 0.7214 | 0.6193 | 11.9949 | 0.6584 | 0.6584 | +| 27 | Dummy | 24.77 | 0.3538 | 0.2991 | 0.5000 | 0.1537 | 1.4284 | 0.2911 | 0.3695 | Average over the 51 Multiverse-core datasets with results for every estimator on every metric, ordered by average accuracy rank. Best in each column in bold. diff --git a/docs/leaderboard.md b/docs/leaderboard.md index 61d204c..a055c2e 100644 --- a/docs/leaderboard.md +++ b/docs/leaderboard.md @@ -115,32 +115,33 @@ inferred from the ranking. | # | Estimator | Accuracy rank | Accuracy | Balanced accuracy | AUROC | F1 | Log loss ↓ | Sensitivity | Specificity | |---|---|---|---|---|---|---|---|---|---| -| 1 | HC2 | **7.13** | **0.7665** | **0.7452** | 0.8823 | 0.7411 | **0.6692** | 0.7470 | **0.7703** | -| 2 | RDST | 8.50 | 0.7459 | 0.7294 | 0.8179 | 0.7243 | 9.1587 | 0.7263 | 0.7560 | -| 3 | MRHydra | 8.87 | 0.7523 | 0.7388 | 0.8236 | **0.7432** | 8.9285 | 0.7599 | 0.7360 | -| 4 | Arsenal | 9.98 | 0.7321 | 0.7134 | 0.8439 | 0.7116 | 5.5624 | 0.7119 | 0.7417 | -| 5 | ROCKET | 9.98 | 0.7317 | 0.7146 | 0.8089 | 0.7130 | 9.6705 | 0.7133 | 0.7393 | -| 6 | RIST | 10.20 | 0.7433 | 0.7278 | 0.8755 | 0.7325 | 0.7983 | 0.7454 | 0.7322 | -| 7 | H-InceptionTime | 10.28 | 0.7223 | 0.7230 | 0.8653 | 0.6967 | 1.5030 | 0.7053 | 0.7345 | -| 8 | CIF | 10.87 | 0.7525 | 0.7378 | 0.8825 | 0.7400 | 0.8488 | **0.7604** | 0.7349 | -| 9 | FreshPRINCE | 11.24 | 0.7422 | 0.7281 | 0.8796 | 0.7239 | 0.7764 | 0.7313 | 0.7457 | -| 10 | DrCIF | 11.39 | 0.7386 | 0.7252 | 0.8734 | 0.7246 | 0.8458 | 0.7384 | 0.7303 | -| 11 | LITETime-MV | 11.54 | 0.7073 | 0.7064 | 0.8568 | 0.6779 | 1.4779 | 0.6905 | 0.7218 | -| 12 | LiteTIME | 12.00 | 0.7087 | 0.7019 | 0.8576 | 0.6751 | 1.6854 | 0.6985 | 0.7204 | -| 13 | QUANT | 13.26 | 0.7285 | 0.7171 | **0.8888** | 0.7195 | 0.8041 | 0.7421 | 0.7074 | -| 14 | STSF | 13.61 | 0.7345 | 0.7223 | 0.8774 | 0.6934 | 0.8338 | 0.7007 | 0.7600 | -| 15 | TS2Vec | 14.11 | 0.7070 | 0.6913 | 0.8470 | 0.6917 | 0.8902 | 0.7150 | 0.6877 | -| 16 | TDE | 14.65 | 0.7079 | 0.6862 | 0.8484 | 0.6775 | 1.1475 | 0.6897 | 0.7095 | -| 17 | PatchMTSC | 14.72 | 0.7110 | 0.6986 | 0.8601 | 0.6899 | 0.7670 | 0.7192 | 0.6928 | -| 18 | ConvTran | 15.46 | 0.6931 | 0.6801 | 0.8552 | 0.6793 | 0.8155 | 0.7049 | 0.6736 | -| 19 | TSF | 15.54 | 0.7214 | 0.7076 | 0.8671 | 0.6917 | 0.9127 | 0.6977 | 0.7365 | -| 20 | STC | 15.57 | 0.7265 | 0.7036 | 0.8803 | 0.7035 | 0.8124 | 0.7186 | 0.7184 | -| 21 | Catch22 | 15.65 | 0.7006 | 0.6854 | 0.8557 | 0.6897 | 0.9814 | 0.7096 | 0.6802 | -| 22 | 1NN-DTW | 16.87 | 0.6848 | 0.6759 | 0.7785 | 0.6702 | 11.3600 | 0.6720 | 0.6879 | -| 23 | TimesURL | 17.50 | 0.6809 | 0.6658 | 0.8290 | 0.6539 | 1.3557 | 0.6698 | 0.6738 | -| 24 | Summary | 18.78 | 0.6477 | 0.6355 | 0.8295 | 0.6206 | 1.3000 | 0.6291 | 0.6589 | -| 25 | TimesNet | 19.35 | 0.6584 | 0.6504 | 0.8332 | 0.6386 | 1.1641 | 0.6628 | 0.6504 | -| 26 | Dummy | 23.96 | 0.2168 | 0.1980 | 0.5000 | 0.0800 | 1.9123 | 0.1853 | 0.2288 | +| 1 | HC2 | **7.37** | **0.7665** | **0.7452** | 0.8823 | 0.7411 | **0.6692** | 0.7470 | **0.7703** | +| 2 | RDST | 8.80 | 0.7459 | 0.7294 | 0.8179 | 0.7243 | 9.1587 | 0.7263 | 0.7560 | +| 3 | MRHydra | 9.20 | 0.7523 | 0.7388 | 0.8236 | **0.7432** | 8.9285 | 0.7599 | 0.7360 | +| 4 | Arsenal | 10.26 | 0.7321 | 0.7134 | 0.8439 | 0.7116 | 5.5624 | 0.7119 | 0.7417 | +| 5 | ROCKET | 10.28 | 0.7317 | 0.7146 | 0.8089 | 0.7130 | 9.6705 | 0.7133 | 0.7393 | +| 6 | RIST | 10.57 | 0.7433 | 0.7278 | 0.8755 | 0.7325 | 0.7983 | 0.7454 | 0.7322 | +| 7 | H-InceptionTime | 10.67 | 0.7223 | 0.7230 | 0.8653 | 0.6967 | 1.5030 | 0.7053 | 0.7345 | +| 8 | CIF | 11.33 | 0.7525 | 0.7378 | 0.8825 | 0.7400 | 0.8488 | **0.7604** | 0.7349 | +| 9 | FreshPRINCE | 11.74 | 0.7422 | 0.7281 | 0.8796 | 0.7239 | 0.7764 | 0.7313 | 0.7457 | +| 10 | DrCIF | 11.83 | 0.7386 | 0.7252 | 0.8734 | 0.7246 | 0.8458 | 0.7384 | 0.7303 | +| 11 | LITETime-MV | 12.00 | 0.7073 | 0.7064 | 0.8568 | 0.6779 | 1.4779 | 0.6905 | 0.7218 | +| 12 | LiteTIME | 12.50 | 0.7087 | 0.7019 | 0.8576 | 0.6751 | 1.6854 | 0.6985 | 0.7204 | +| 13 | DisjointCNN | 13.20 | 0.7011 | 0.7030 | 0.8510 | 0.6704 | 1.7943 | 0.6938 | 0.7070 | +| 14 | QUANT | 13.78 | 0.7285 | 0.7171 | **0.8888** | 0.7195 | 0.8041 | 0.7421 | 0.7074 | +| 15 | STSF | 14.24 | 0.7345 | 0.7223 | 0.8774 | 0.6934 | 0.8338 | 0.7007 | 0.7600 | +| 16 | TS2Vec | 14.78 | 0.7070 | 0.6913 | 0.8470 | 0.6917 | 0.8902 | 0.7150 | 0.6877 | +| 17 | TDE | 15.15 | 0.7079 | 0.6862 | 0.8484 | 0.6775 | 1.1475 | 0.6897 | 0.7095 | +| 18 | PatchMTSC | 15.37 | 0.7110 | 0.6986 | 0.8601 | 0.6899 | 0.7670 | 0.7192 | 0.6928 | +| 19 | ConvTran | 16.11 | 0.6931 | 0.6801 | 0.8552 | 0.6793 | 0.8155 | 0.7049 | 0.6736 | +| 20 | STC | 16.15 | 0.7265 | 0.7036 | 0.8803 | 0.7035 | 0.8124 | 0.7186 | 0.7184 | +| 21 | TSF | 16.17 | 0.7214 | 0.7076 | 0.8671 | 0.6917 | 0.9127 | 0.6977 | 0.7365 | +| 22 | Catch22 | 16.30 | 0.7006 | 0.6854 | 0.8557 | 0.6897 | 0.9814 | 0.7096 | 0.6802 | +| 23 | 1NN-DTW | 17.61 | 0.6848 | 0.6759 | 0.7785 | 0.6702 | 11.3600 | 0.6720 | 0.6879 | +| 24 | TimesURL | 18.24 | 0.6809 | 0.6658 | 0.8290 | 0.6539 | 1.3557 | 0.6698 | 0.6738 | +| 25 | Summary | 19.48 | 0.6477 | 0.6355 | 0.8295 | 0.6206 | 1.3000 | 0.6291 | 0.6589 | +| 26 | TimesNet | 20.09 | 0.6584 | 0.6504 | 0.8332 | 0.6386 | 1.1641 | 0.6628 | 0.6504 | +| 27 | Dummy | 24.78 | 0.2168 | 0.1980 | 0.5000 | 0.0800 | 1.9123 | 0.1853 | 0.2288 | Average over the 23 UEA datasets with results for every estimator on every metric, ordered by average accuracy rank. Best in each column in bold. diff --git a/results/multiverse/DisjointCNN/DisjointCNN_accuracy.csv b/results/multiverse/DisjointCNN/DisjointCNN_accuracy.csv new file mode 100644 index 0000000..7f5f584 --- /dev/null +++ b/results/multiverse/DisjointCNN/DisjointCNN_accuracy.csv @@ -0,0 +1,66 @@ +Resamples:,0 +Alzheimers,0.27906976744186046 +AppliancesEnergy_disc,0.5476190476190477 +ArticularyWordRecognition,0.9866666666666667 +AsphaltObstaclesCoordinates,0.7749360613810742 +AsphaltRegularityCoordinates,0.9933422103861518 +AtrialFibrillation,0.3333333333333333 +AustraliaRainfall_disc,0.7509411201930076 +AutomotiveRoadTrials,0.7662337662337663 +BIDMC32HR_disc,0.8115881617340559 +BIDMC32SpO2_disc,0.5302209253855773 +BeijingPM10Quality_disc,0.8124009508716323 +BeijingPM25Quality_disc,0.8781695721077655 +BenzeneConcentration_disc,0.9742397830718574 +Blink,0.5711111111111111 +BoneIntensitiesAgeGroup,0.750561797752809 +BoneProbAgeGroup,0.6269662921348315 +CharacterTrajectories,0.9895543175487466 +CounterMovementJump,0.7597765363128491 +Cricket,0.9722222222222222 +CrowdSourced,0.7430285915990117 +DuckDuckGeese,0.6 +ERing,0.9555555555555556 +EigenWorms,0.6106870229007634 +Epilepsy,0.9782608695652174 +EthanolConcentration,0.27756653992395436 +EyesOpenShut,0.40476190476190477 +FaceDetection,0.5391600454029511 +FordChallenge,0.8789633305762338 +HandMovementDirection,0.3783783783783784 +Handwriting,0.44588235294117645 +Heartbeat,0.7170731707317073 +HouseholdPowerConsumption1_disc,0.8935860058309038 +HouseholdPowerConsumption2_disc,0.7871720116618076 +IEEEPPG_disc,0.4623493975903614 +IRDS-SFL,0.8275862068965517 +JapaneseVowels,0.9918918918918919 +KERAAL-RTK,0.7857142857142857 +KIMORE-PR-C,0.42857142857142855 +KINECAL-QSEO,0.8235294117647058 +LSST,0.26520681265206814 +Libras,0.9611111111111111 +Locust2022,0.89602909972719 +LowCost,0.505 +MindReading,0.6370597243491577 +MotionSenseHAR,0.9924528301886792 +MotorImagery,0.5 +NATOPS,0.9555555555555556 +PEMS-SF,0.7572254335260116 +PenDigits,0.9757004002287021 +PhonemeSpectra,0.2815389203698181 +PhotoStimulation,0.4166666666666667 +RacketSports,0.875 +STEW,0.6647025813692481 +SelfRegulationSCP1,0.7952218430034129 +Skoda,0.940926777502653 +SpokenArabicDigits,0.9945429740791268 +StandWalkJump,0.2 +TactileTextureRecognition,0.9985315712187959 +Tiselac,0.7835187057633973 +UCDHE-Rowing-MC,0.7954545454545454 +UCIActivity,0.9854227405247813 +UIPRMD-DS-C,0.6388888888888888 +USCActivity,0.7056500607533415 +UWaveGestureLibrary,0.9125 +WISDM,0.8686497971798339 diff --git a/results/multiverse/DisjointCNN/DisjointCNN_auroc.csv b/results/multiverse/DisjointCNN/DisjointCNN_auroc.csv new file mode 100644 index 0000000..aea739e --- /dev/null +++ b/results/multiverse/DisjointCNN/DisjointCNN_auroc.csv @@ -0,0 +1,66 @@ +Resamples:,0 +Alzheimers,0.44745188452285484 +AppliancesEnergy_disc,0.43014705882352944 +ArticularyWordRecognition,0.9999189814814813 +AsphaltObstaclesCoordinates,0.9309368445918137 +AsphaltRegularityCoordinates,0.9992338795488402 +AtrialFibrillation,0.39333333333333337 +AustraliaRainfall_disc,0.8508062557188008 +AutomotiveRoadTrials,0.8493647912885662 +BIDMC32HR_disc,0.7320075569947863 +BIDMC32SpO2_disc,0.35303849396089515 +BeijingPM10Quality_disc,0.883939154257941 +BeijingPM25Quality_disc,0.9352874982417204 +BenzeneConcentration_disc,0.9899352676607205 +Blink,0.62393 +BoneIntensitiesAgeGroup,0.9012074944408928 +BoneProbAgeGroup,0.7912919902391892 +CharacterTrajectories,0.9999715546200066 +CounterMovementJump,0.9433406882305995 +Cricket,0.9993686868686869 +CrowdSourced,0.8060149855068997 +DuckDuckGeese,0.8734999999999999 +ERing,0.9996049382716049 +EigenWorms,0.939828804098903 +Epilepsy,0.9996474772895348 +EthanolConcentration,0.5151667219374849 +EyesOpenShut,0.4331065759637188 +FaceDetection,0.5547579690296214 +FordChallenge,0.9357341067942236 +HandMovementDirection,0.67577215119588 +Handwriting,0.9097385683325577 +Heartbeat,0.7447842579421526 +HouseholdPowerConsumption1_disc,0.9769338027859599 +HouseholdPowerConsumption2_disc,0.8122203384447391 +IEEEPPG_disc,0.6233953127524545 +IRDS-SFL,0.8926630434782609 +JapaneseVowels,0.9999755739623883 +KERAAL-RTK,0.9375 +KIMORE-PR-C,0.5 +KINECAL-QSEO,0.875 +LSST,0.7541723637704801 +Libras,0.9995039682539684 +Locust2022,0.8299237702070262 +LowCost,0.5166222222222222 +MindReading,0.8699297658459535 +MotionSenseHAR,0.9939610232003022 +MotorImagery,0.5582 +NATOPS,0.9974814814814815 +PEMS-SF,0.9706842160399534 +PenDigits,0.9991311890042067 +PhonemeSpectra,0.850953370520314 +PhotoStimulation,0.41402523402523395 +RacketSports,0.9701683969097054 +STEW,0.8777091198177055 +SelfRegulationSCP1,0.9547572453638988 +Skoda,0.9953207540499943 +SpokenArabicDigits,0.9999761019582376 +StandWalkJump,0.41999999999999993 +TactileTextureRecognition,1.0 +Tiselac,0.9486528955005507 +UCDHE-Rowing-MC,0.9553559955104074 +UCIActivity,0.999603140870625 +UIPRMD-DS-C,0.6141975308641976 +USCActivity,0.9623691684519052 +UWaveGestureLibrary,0.9864062499999999 +WISDM,0.9463818263632254 diff --git a/results/multiverse/DisjointCNN/DisjointCNN_balacc.csv b/results/multiverse/DisjointCNN/DisjointCNN_balacc.csv new file mode 100644 index 0000000..4414cbd --- /dev/null +++ b/results/multiverse/DisjointCNN/DisjointCNN_balacc.csv @@ -0,0 +1,66 @@ +Resamples:,0 +Alzheimers,0.2551707551707552 +AppliancesEnergy_disc,0.48161764705882354 +ArticularyWordRecognition,0.9866666666666666 +AsphaltObstaclesCoordinates,0.7686636418132303 +AsphaltRegularityCoordinates,0.9933602894232816 +AtrialFibrillation,0.3333333333333333 +AustraliaRainfall_disc,0.4375972067066962 +AutomotiveRoadTrials,0.7740471869328494 +BIDMC32HR_disc,0.7967561331410166 +BIDMC32SpO2_disc,0.3732735054111336 +BeijingPM10Quality_disc,0.7182122646736285 +BeijingPM25Quality_disc,0.8582847155085094 +BenzeneConcentration_disc,0.9622667853052261 +Blink,0.5815 +BoneIntensitiesAgeGroup,0.7897389125654998 +BoneProbAgeGroup,0.6870424467165098 +CharacterTrajectories,0.9887006628524688 +CounterMovementJump,0.7602636534839924 +Cricket,0.9722222222222223 +CrowdSourced,0.743105311212915 +DuckDuckGeese,0.6 +ERing,0.9555555555555556 +EigenWorms,0.5226262626262625 +Epilepsy,0.9779411764705882 +EthanolConcentration,0.2776223776223776 +EyesOpenShut,0.40476190476190477 +FaceDetection,0.5391600454029511 +FordChallenge,0.8704105806031179 +HandMovementDirection,0.4083333333333333 +Handwriting,0.4409873802937574 +Heartbeat,0.7015765765765766 +HouseholdPowerConsumption1_disc,0.6833289843954878 +HouseholdPowerConsumption2_disc,0.6705825377058253 +IEEEPPG_disc,0.43294626822369026 +IRDS-SFL,0.8143115942028986 +JapaneseVowels,0.9930375180375179 +KERAAL-RTK,0.75 +KIMORE-PR-C,0.25 +KINECAL-QSEO,0.4375 +LSST,0.2845199123822816 +Libras,0.9611111111111109 +Locust2022,0.6934430342954592 +LowCost,0.505 +MindReading,0.6320268017140631 +MotionSenseHAR,0.9938271604938271 +MotorImagery,0.5 +NATOPS,0.9555555555555556 +PEMS-SF,0.7609580435667391 +PenDigits,0.9758028866203899 +PhonemeSpectra,0.28154617839980356 +PhotoStimulation,0.3333333333333333 +RacketSports,0.885610465116279 +STEW,0.664702581369248 +SelfRegulationSCP1,0.7958484763768521 +Skoda,0.9319823877791733 +SpokenArabicDigits,0.9945454545454545 +StandWalkJump,0.19999999999999998 +TactileTextureRecognition,0.9985569985569985 +Tiselac,0.6120547833855489 +UCDHE-Rowing-MC,0.8073333333333335 +UCIActivity,0.9859942794725404 +UIPRMD-DS-C,0.6388888888888888 +USCActivity,0.6982232821233808 +UWaveGestureLibrary,0.9125 +WISDM,0.7176410654759297 diff --git a/results/multiverse/DisjointCNN/DisjointCNN_f1.csv b/results/multiverse/DisjointCNN/DisjointCNN_f1.csv new file mode 100644 index 0000000..f31ecf7 --- /dev/null +++ b/results/multiverse/DisjointCNN/DisjointCNN_f1.csv @@ -0,0 +1,66 @@ +Resamples:,0 +Alzheimers,0.26988205715975894 +AppliancesEnergy_disc,0.24 +ArticularyWordRecognition,0.986485375494071 +AsphaltObstaclesCoordinates,0.7677470121043654 +AsphaltRegularityCoordinates,0.9932523616734144 +AtrialFibrillation,0.16666666666666666 +AustraliaRainfall_disc,0.7539881481588244 +AutomotiveRoadTrials,0.625 +BIDMC32HR_disc,0.8070473861924299 +BIDMC32SpO2_disc,0.010535557506584723 +BeijingPM10Quality_disc,0.6039314094521121 +BeijingPM25Quality_disc,0.8003894839337877 +BenzeneConcentration_disc,0.9573033707865168 +Blink,0.5831533477321814 +BoneIntensitiesAgeGroup,0.74680924905644 +BoneProbAgeGroup,0.6173447124759013 +CharacterTrajectories,0.9895998153435236 +CounterMovementJump,0.7518396048911654 +Cricket,0.9720279720279721 +CrowdSourced,0.7888631090487239 +DuckDuckGeese,0.5616361416361415 +ERing,0.9550962651682063 +EigenWorms,0.5339473211773573 +Epilepsy,0.9780092879822783 +EthanolConcentration,0.268875415691347 +EyesOpenShut,0.4186046511627907 +FaceDetection,0.4850982878883957 +FordChallenge,0.8387807565185457 +HandMovementDirection,0.37315945622096597 +Handwriting,0.40251549748568716 +Heartbeat,0.5671641791044776 +HouseholdPowerConsumption1_disc,0.8806776031721647 +HouseholdPowerConsumption2_disc,0.7755968062090511 +IEEEPPG_disc,0.43721301937093054 +IRDS-SFL,0.6551724137931034 +JapaneseVowels,0.9919216312806576 +KERAAL-RTK,0.6666666666666666 +KIMORE-PR-C,0.0 +KINECAL-QSEO,0.0 +LSST,0.24153333946468394 +Libras,0.9610724637681158 +Locust2022,0.43305785123966944 +LowCost,0.5676855895196506 +MindReading,0.6335634882598563 +MotionSenseHAR,0.9925613249927951 +MotorImagery,0.2857142857142857 +NATOPS,0.9555555555555556 +PEMS-SF,0.749377551863733 +PenDigits,0.9756088214440342 +PhonemeSpectra,0.2732075524903016 +PhotoStimulation,0.2450980392156863 +RacketSports,0.8750363108206245 +STEW,0.744220890410959 +SelfRegulationSCP1,0.8265895953757225 +Skoda,0.9409285132924602 +SpokenArabicDigits,0.9945449799429406 +StandWalkJump,0.13333333333333333 +TactileTextureRecognition,0.9985312340365269 +Tiselac,0.7741561042962818 +UCDHE-Rowing-MC,0.784762868748151 +UCIActivity,0.9853030818860898 +UIPRMD-DS-C,0.5517241379310345 +USCActivity,0.7015864523336914 +UWaveGestureLibrary,0.911019697330673 +WISDM,0.8635259033495417 diff --git a/results/multiverse/DisjointCNN/DisjointCNN_logloss.csv b/results/multiverse/DisjointCNN/DisjointCNN_logloss.csv new file mode 100644 index 0000000..6c07e9f --- /dev/null +++ b/results/multiverse/DisjointCNN/DisjointCNN_logloss.csv @@ -0,0 +1,66 @@ +Resamples:,0 +Alzheimers,3.1676739990510705 +AppliancesEnergy_disc,2.7128713822386468 +ArticularyWordRecognition,0.05662980524876592 +AsphaltObstaclesCoordinates,1.7623202952649653 +AsphaltRegularityCoordinates,0.0557362790197591 +AtrialFibrillation,10.956268038401936 +AustraliaRainfall_disc,0.5567941504467698 +AutomotiveRoadTrials,1.5324187641898397 +BIDMC32HR_disc,3.5662011879166196 +BIDMC32SpO2_disc,8.964836021178042 +BeijingPM10Quality_disc,0.42717580958924173 +BeijingPM25Quality_disc,0.2938736220114435 +BenzeneConcentration_disc,0.2866902990678911 +Blink,8.22921983813872 +BoneIntensitiesAgeGroup,1.544299270833766 +BoneProbAgeGroup,1.7500992396341042 +CharacterTrajectories,0.033192409526791665 +CounterMovementJump,1.0159187695111955 +Cricket,0.09033604959392025 +CrowdSourced,5.777233621887509 +DuckDuckGeese,2.043216171575822 +ERing,0.13835083108857268 +EigenWorms,1.1175401751396294 +Epilepsy,0.09597854665684544 +EthanolConcentration,2.8107859500435426 +EyesOpenShut,3.9011231082747133 +FaceDetection,3.944785508709933 +FordChallenge,0.8320643723859176 +HandMovementDirection,1.6688021164673719 +Handwriting,2.026299913497382 +Heartbeat,0.7300109666007285 +HouseholdPowerConsumption1_disc,0.36644725486183916 +HouseholdPowerConsumption2_disc,1.8761431324344888 +IEEEPPG_disc,9.676086026602054 +IRDS-SFL,1.0269409214259426 +JapaneseVowels,0.02601942956869396 +KERAAL-RTK,0.6585611385067799 +KIMORE-PR-C,1.8927008790684294 +KINECAL-QSEO,0.8156222117184887 +LSST,2.1614359684852524 +Libras,0.11301173007364324 +Locust2022,1.022409582099866 +LowCost,0.8940790719868018 +MindReading,2.7661419066377366 +MotionSenseHAR,0.1980690944234798 +MotorImagery,6.121015527982305 +NATOPS,0.1418758448646627 +PEMS-SF,0.8410462672598519 +PenDigits,0.15632686837343565 +PhonemeSpectra,7.573558512730689 +PhotoStimulation,1.4772163983369078 +RacketSports,0.4438046654876871 +STEW,1.9726013347772382 +SelfRegulationSCP1,1.1982375234610034 +Skoda,0.35845816649350576 +SpokenArabicDigits,0.03523146281288393 +StandWalkJump,1.547946852477762 +TactileTextureRecognition,0.0012990376577401627 +Tiselac,3.7134507029364854 +UCDHE-Rowing-MC,0.8283249887272893 +UCIActivity,0.05002875310206179 +UIPRMD-DS-C,6.3613193731957 +USCActivity,2.208241955248106 +UWaveGestureLibrary,0.41666040003765054 +WISDM,2.525803826296069 diff --git a/results/multiverse/DisjointCNN/DisjointCNN_sensitivity.csv b/results/multiverse/DisjointCNN/DisjointCNN_sensitivity.csv new file mode 100644 index 0000000..1025959 --- /dev/null +++ b/results/multiverse/DisjointCNN/DisjointCNN_sensitivity.csv @@ -0,0 +1,66 @@ +Resamples:,0 +Alzheimers,0.27906976744186046 +AppliancesEnergy_disc,0.375 +ArticularyWordRecognition,0.9866666666666667 +AsphaltObstaclesCoordinates,0.7749360613810742 +AsphaltRegularityCoordinates,0.9945945945945946 +AtrialFibrillation,0.3333333333333333 +AustraliaRainfall_disc,0.7509411201930076 +AutomotiveRoadTrials,0.7894736842105263 +BIDMC32HR_disc,0.8115881617340559 +BIDMC32SpO2_disc,0.008784773060029283 +BeijingPM10Quality_disc,0.49519890260631 +BeijingPM25Quality_disc,0.8079947575360419 +BenzeneConcentration_disc,0.9307116104868914 +Blink,0.675 +BoneIntensitiesAgeGroup,0.750561797752809 +BoneProbAgeGroup,0.6269662921348315 +CharacterTrajectories,0.9895543175487466 +CounterMovementJump,0.7597765363128491 +Cricket,0.9722222222222222 +CrowdSourced,0.96045197740113 +DuckDuckGeese,0.6 +ERing,0.9555555555555556 +EigenWorms,0.6106870229007634 +Epilepsy,0.9782608695652174 +EthanolConcentration,0.27756653992395436 +EyesOpenShut,0.42857142857142855 +FaceDetection,0.43416572077185017 +FordChallenge,0.8357116721551409 +HandMovementDirection,0.3783783783783784 +Handwriting,0.44588235294117645 +Heartbeat,0.6666666666666666 +HouseholdPowerConsumption1_disc,0.8935860058309038 +HouseholdPowerConsumption2_disc,0.7871720116618076 +IEEEPPG_disc,0.4623493975903614 +IRDS-SFL,0.7916666666666666 +JapaneseVowels,0.9918918918918919 +KERAAL-RTK,0.5 +KIMORE-PR-C,0.0 +KINECAL-QSEO,0.0 +LSST,0.26520681265206814 +Libras,0.9611111111111111 +Locust2022,0.447098976109215 +LowCost,0.65 +MindReading,0.6370597243491577 +MotionSenseHAR,0.9924528301886792 +MotorImagery,0.2 +NATOPS,0.9555555555555556 +PEMS-SF,0.7572254335260116 +PenDigits,0.9757004002287021 +PhonemeSpectra,0.2815389203698181 +PhotoStimulation,0.4166666666666667 +RacketSports,0.875 +STEW,0.9755892255892256 +SelfRegulationSCP1,0.9794520547945206 +Skoda,0.940926777502653 +SpokenArabicDigits,0.9945429740791268 +StandWalkJump,0.2 +TactileTextureRecognition,0.9985315712187959 +Tiselac,0.7835187057633973 +UCDHE-Rowing-MC,0.7954545454545454 +UCIActivity,0.9854227405247813 +UIPRMD-DS-C,0.4444444444444444 +USCActivity,0.7056500607533415 +UWaveGestureLibrary,0.9125 +WISDM,0.8686497971798339 diff --git a/results/multiverse/DisjointCNN/DisjointCNN_specificity.csv b/results/multiverse/DisjointCNN/DisjointCNN_specificity.csv new file mode 100644 index 0000000..2d69a80 --- /dev/null +++ b/results/multiverse/DisjointCNN/DisjointCNN_specificity.csv @@ -0,0 +1,66 @@ +Resamples:,0 +Alzheimers,0.27906976744186046 +AppliancesEnergy_disc,0.5882352941176471 +ArticularyWordRecognition,0.9866666666666667 +AsphaltObstaclesCoordinates,0.7749360613810742 +AsphaltRegularityCoordinates,0.9921259842519685 +AtrialFibrillation,0.3333333333333333 +AustraliaRainfall_disc,0.7509411201930076 +AutomotiveRoadTrials,0.7586206896551724 +BIDMC32HR_disc,0.8115881617340559 +BIDMC32SpO2_disc,0.7377622377622378 +BeijingPM10Quality_disc,0.9412256267409471 +BeijingPM25Quality_disc,0.9085746734809768 +BenzeneConcentration_disc,0.9938219601235608 +Blink,0.488 +BoneIntensitiesAgeGroup,0.750561797752809 +BoneProbAgeGroup,0.6269662921348315 +CharacterTrajectories,0.9895543175487466 +CounterMovementJump,0.7597765363128491 +Cricket,0.9722222222222222 +CrowdSourced,0.5257586450247 +DuckDuckGeese,0.6 +ERing,0.9555555555555556 +EigenWorms,0.6106870229007634 +Epilepsy,0.9782608695652174 +EthanolConcentration,0.27756653992395436 +EyesOpenShut,0.38095238095238093 +FaceDetection,0.6441543700340522 +FordChallenge,0.9051094890510949 +HandMovementDirection,0.3783783783783784 +Handwriting,0.44588235294117645 +Heartbeat,0.7364864864864865 +HouseholdPowerConsumption1_disc,0.8935860058309038 +HouseholdPowerConsumption2_disc,0.7871720116618076 +IEEEPPG_disc,0.4623493975903614 +IRDS-SFL,0.8369565217391305 +JapaneseVowels,0.9918918918918919 +KERAAL-RTK,1.0 +KIMORE-PR-C,0.5 +KINECAL-QSEO,0.875 +LSST,0.26520681265206814 +Libras,0.9611111111111111 +Locust2022,0.9397870924817032 +LowCost,0.36 +MindReading,0.6370597243491577 +MotionSenseHAR,0.9924528301886792 +MotorImagery,0.8 +NATOPS,0.9555555555555556 +PEMS-SF,0.7572254335260116 +PenDigits,0.9757004002287021 +PhonemeSpectra,0.2815389203698181 +PhotoStimulation,0.4166666666666667 +RacketSports,0.875 +STEW,0.3538159371492705 +SelfRegulationSCP1,0.6122448979591837 +Skoda,0.940926777502653 +SpokenArabicDigits,0.9945429740791268 +StandWalkJump,0.2 +TactileTextureRecognition,0.9985315712187959 +Tiselac,0.7835187057633973 +UCDHE-Rowing-MC,0.7954545454545454 +UCIActivity,0.9854227405247813 +UIPRMD-DS-C,0.8333333333333334 +USCActivity,0.7056500607533415 +UWaveGestureLibrary,0.9125 +WISDM,0.8686497971798339 diff --git a/results/multiverse/datasets.html b/results/multiverse/datasets.html index fde92bf..f024c1e 100644 --- a/results/multiverse/datasets.html +++ b/results/multiverse/datasets.html @@ -60,7 +60,7 @@ details { margin-top: .6rem; } summary { cursor: pointer; color: var(--accent); } code { font-family: ui-monospace, SFMono-Regular, Menlo, monospace; font-size: .9em; } -tr.nosignal td { background: rgba(214, 158, 46, .16); }tr.saturated td { background: rgba(56, 161, 105, .14); }

Multiverse-core datasets: accuracy

66 datasets · accuracy · best of up to 25 estimators against the Dummy baseline · built 2026-09-03

DatasetDummyMedianBestBest estimatorGain over dummySpreadEstimators
KINECAL-QSEO0.94120.94120.9412Arsenal0.00000.117625
BIDMC32SpO2_disc0.71530.66280.7203ROCKET0.00500.184223
Locust20220.91120.90850.9206MRHydra0.00940.045224
Heartbeat0.72200.74630.7854CIF0.06340.126825
HouseholdPowerConsumption2_disc0.72160.76680.7872HC20.06560.141425
AutomotiveRoadTrials0.75320.79220.8442CIF0.09090.233825
AustraliaRainfall_disc0.68600.77460.7808LITETime-MV0.09480.088116
EyesOpenShut0.50000.50000.5952STSF0.09520.190525
MotorImagery0.50000.51000.6000FreshPRINCE0.10000.140025
BeijingPM10Quality_disc0.71120.82470.8417FreshPRINCE0.13050.107025
Alzheimers0.41860.37210.5581MRHydra0.13950.302324
AppliancesEnergy_disc0.80950.83330.9524FreshPRINCE0.14290.452425
EmoPain0.78310.84080.92681NN-DTW0.14370.242320
PhotoStimulation0.41670.38890.5833ROCKET0.16670.388924
FaceDetection0.50000.63040.6850H-InceptionTime0.18500.170524
BeijingPM25Quality_disc0.69770.87560.8879ConvTran0.19020.128825
AtrialFibrillation0.33330.26670.5333TS2Vec0.20000.466725
HouseholdPowerConsumption1_disc0.77840.91250.9825FreshPRINCE0.20410.218725
LowCost0.50000.63330.7300TSF0.23000.248325
BoneProbAgeGroup0.47640.64940.7124H-InceptionTime0.23600.193325
StandWalkJump0.33330.40000.6000MRHydra0.26670.400025
CrowdSourced0.50020.71370.7734LITETime-MV0.27320.176825
BenzeneConcentration_disc0.68970.81950.9768STSF0.28700.577025
FordChallenge0.62320.88390.9360QUANT0.31280.312824
BIDMC32HR_disc0.65070.79070.9637RIST0.31300.635323
STEW0.50000.73650.8385Arsenal0.33850.210023
BoneIntensitiesAgeGroup0.47640.79330.8202HC20.34380.296625
PhonemeSpectra0.02560.27830.3746H-InceptionTime0.34890.291125
KERAAL-RTK0.57140.78570.9286HC20.35710.571425
LSST0.31510.62940.7040FreshPRINCE0.38890.480925
HandMovementDirection0.20270.41890.6081TSF0.40540.418925
DuckDuckGeese0.20000.46000.6400H-InceptionTime0.44000.480025
SelfRegulationSCP10.50170.85320.9454MRHydra0.44370.208225
IEEEPPG_disc0.26050.43220.7078ConvTran0.44730.438325
AsphaltRegularityCoordinates0.50730.98000.9947H-InceptionTime0.48740.291625
MindReading0.23120.52530.7243LITETime-MV0.49310.385925
EthanolConcentration0.25100.43350.7490STC0.49810.532325
UIPRMD-DS-C0.50000.83331.0000Catch220.50000.388925
WISDM0.36640.86420.8965MRHydra0.53000.131025
Blink0.44440.99111.0000Arsenal0.55560.415625
EigenWorms0.41980.87790.9771MRHydra0.55730.557324
KIMORE-PR-C0.14290.42860.7143LITETime-MV0.57140.571425
AsphaltObstaclesCoordinates0.28390.82100.8670MRHydra0.58310.289025
CounterMovementJump0.33520.74300.9274Arsenal0.59220.458125
Handwriting0.03760.37760.6529H-InceptionTime0.61530.478825
USCActivity0.11380.69240.7354LITETime-MV0.62160.137920
RacketSports0.28290.88160.9079RDST0.62500.125025
UCDHE-Rowing-MC0.20450.72950.8295PatchMTSC0.62500.338625
IRDS-SFL0.20690.79310.8621RDST0.65520.448325
Skoda0.23560.94690.9646H-InceptionTime0.72900.119924
Epilepsy0.26810.98551.0000HC20.73190.101425
Tiselac0.06280.81860.8373STSF0.77450.204419
MotionSenseHAR0.20380.98491.0000DrCIF0.79620.101925
NATOPS0.16670.88890.9667LITETime-MV0.80000.155625
UCIActivity0.19160.97580.9983LITETime-MV0.80670.174925
UWaveGestureLibrary0.12500.90940.9406Arsenal0.81560.553125
ERing0.16670.93330.9963MRHydra0.82960.237025
PEMS-SF0.11560.89601.0000CIF0.88440.317925
PenDigits0.10380.97860.9911H-InceptionTime0.88740.234423
SpokenArabicDigits0.10000.97910.9941RDST0.89400.128725
Libras0.06670.88890.9722RIST0.90560.338925
JapaneseVowels0.08380.96220.9946LiteTIME0.91080.208125
Cricket0.08330.98611.00001NN-DTW0.91670.069425
CharacterTrajectories0.06480.98960.9958H-InceptionTime0.93110.044625
TactileTextureRecognition0.05140.99851.0000H-InceptionTime0.94860.168925
ArticularyWordRecognition0.04000.98000.9933Arsenal0.95330.050025

One row per dataset. Dummy is the no-skill floor. Median, best and spread are over the other estimators, so the baseline cannot flatter them. Gain over dummy is best minus dummy, how much skill was found at all; spread is best minus worst, how much the choice of estimator mattered. The two answer different questions, and a single range would conflate them.

3 of 66 datasets gained 0.05 or less over the baseline (shaded amber) and 15 have a best of 0.99 or more (shaded green). Both separate estimators poorly, for opposite reasons. Best is a maximum over many estimators, so it is optimistic by construction: read it as what the archive can currently do on a problem, not as what any one method delivers.