From 11f06491732e9ce878f851c758ce0e8bee9c3254 Mon Sep 17 00:00:00 2001 From: Jayesh Suryavanshi Date: Fri, 31 Jul 2026 00:06:15 -0700 Subject: [PATCH 1/8] Add Deep SAD (Deep Semi-Supervised Anomaly Detection) detector Implement DeepSAD (Ruff et al., ICLR 2020), the semi-supervised extension of Deep SVDD, as a new PyOD BaseDetector. Unlabeled/normal samples use the unsupervised Deep SVDD objective (minimize squared distance to the hypersphere center c) while labeled anomalies use the inverse-distance term (eta * dist**-1), pushing them away from c. fit(X, y=None) treats y==1 as known anomalies and everything else as unlabeled; y=None recovers pure Deep SVDD behavior. Mirrors the structure and API of pyod/models/deep_svdd.py. - pyod/models/deep_sad.py: DeepSAD + InnerDeepSAD encoder - pyod/test/test_deep_sad.py: mirrors test_deepsvdd (semi-supervised fit) - examples/deep_sad_example.py: semi-supervised usage example - README.rst: model-table row + reference - docs/zreferences.bib: ruff2020deepsad citation Loss/architecture adapted from lukasruff/Deep-SAD-PyTorch (MIT) and DeepOD's dsad.py (BSD-3); written clean in PyOD style. --- README.rst | 7 + docs/zreferences.bib | 7 + examples/deep_sad_example.py | 64 ++++++ pyod/models/deep_sad.py | 404 +++++++++++++++++++++++++++++++++++ pyod/test/test_deep_sad.py | 162 ++++++++++++++ 5 files changed, 644 insertions(+) create mode 100644 examples/deep_sad_example.py create mode 100644 pyod/models/deep_sad.py create mode 100644 pyod/test/test_deep_sad.py diff --git a/README.rst b/README.rst index ab4fc01d..f5013c0b 100644 --- a/README.rst +++ b/README.rst @@ -485,6 +485,11 @@ PyOD is organized into two functional groups: **(i) Detection Algorithms**, with - Deep One-Class Classification (`example `__) - 2018 - [#Ruff2018Deep]_ + * - Neural Networks + - DeepSAD + - Deep Semi-Supervised Anomaly Detection (`example `__) + - 2020 + - [#Ruff2020Deep]_ * - Neural Networks - AnoGAN - Anomaly Detection with Generative Adversarial Networks @@ -884,6 +889,8 @@ Reference .. [#Ruff2018Deep] Ruff, L., Vandermeulen, R., Goernitz, N., Deecke, L., Siddiqui, S.A., Binder, A., Müller, E. and Kloft, M., 2018, July. Deep one-class classification. In *International conference on machine learning* (pp. 4393-4402). PMLR. +.. [#Ruff2020Deep] Ruff, L., Vandermeulen, R.A., Görnitz, N., Binder, A., Müller, E., Müller, K.R. and Kloft, M., 2020. Deep semi-supervised anomaly detection. In *International Conference on Learning Representations (ICLR)*. + .. [#Schlegl2017Unsupervised] Schlegl, T., Seeböck, P., Waldstein, S.M., Schmidt-Erfurth, U. and Langs, G., 2017, June. Unsupervised anomaly detection with generative adversarial networks to guide marker discovery. In International conference on information processing in medical imaging (pp. 146-157). Springer, Cham. .. [#Scholkopf2001Estimating] Scholkopf, B., Platt, J.C., Shawe-Taylor, J., Smola, A.J. and Williamson, R.C., 2001. Estimating the support of a high-dimensional distribution. *Neural Computation*, 13(7), pp.1443-1471. diff --git a/docs/zreferences.bib b/docs/zreferences.bib index fef40db7..c2fcbc7e 100644 --- a/docs/zreferences.bib +++ b/docs/zreferences.bib @@ -369,6 +369,13 @@ @article{ruff2018deepsvdd year={2018} } +@inproceedings{ruff2020deepsad, + title={Deep Semi-Supervised Anomaly Detection}, + author={Ruff, Lukas and Vandermeulen, Robert A. and Görnitz, Nico and Binder, Alexander and Müller, Emmanuel and Müller, Klaus-Robert and Kloft, Marius}, + booktitle={International Conference on Learning Representations}, + year={2020} +} + @article{birge2006many, title={How many bins should be put in a regular histogram}, author={Birg{\'e}, Lucien and Rozenholc, Yves}, diff --git a/examples/deep_sad_example.py b/examples/deep_sad_example.py new file mode 100644 index 00000000..64231ba6 --- /dev/null +++ b/examples/deep_sad_example.py @@ -0,0 +1,64 @@ +# -*- coding: utf-8 -*- +"""Example of using Deep SAD for semi-supervised outlier detection +""" +# Author: Jayesh Suryavanshi +# License: BSD 2 clause + +from __future__ import division +from __future__ import print_function + +import os +import sys + +import numpy as np + +# temporary solution for relative imports in case pyod is not installed +# if pyod is installed, no need to use the following line +sys.path.append( + os.path.abspath(os.path.join(os.path.dirname("__file__"), '..'))) + +from pyod.models.deep_sad import DeepSAD +from pyod.utils.data import generate_data +from pyod.utils.data import evaluate_print + +if __name__ == "__main__": + contamination = 0.1 # percentage of outliers + n_train = 20000 # number of training points + n_test = 2000 # number of testing points + n_features = 300 # number of features + random_state = 10 # if c is set to None use random_state + + # Generate sample data + X_train, X_test, y_train, y_test = \ + generate_data(n_train=n_train, + n_test=n_test, + n_features=n_features, + contamination=contamination, + random_state=42) + + # Deep SAD is semi-supervised: reveal a fraction of the training + # anomalies as labeled (1) and keep the rest unlabeled (0). + semi_y = np.zeros(n_train, dtype=int) + anomaly_idx = np.where(y_train == 1)[0] + revealed = anomaly_idx[:len(anomaly_idx) // 2] + semi_y[revealed] = 1 + + # train Deep SAD detector + clf_name = 'DeepSAD' + clf = DeepSAD(n_features=n_features, epochs=5, + contamination=contamination, random_state=random_state) + clf.fit(X_train, semi_y) + + # get the prediction labels and outlier scores of the training data + y_train_pred = clf.labels_ # binary labels (0: inliers, 1: outliers) + y_train_scores = clf.decision_scores_ # raw outlier scores + + # get the prediction on the test data + y_test_pred = clf.predict(X_test) # outlier labels (0 or 1) + y_test_scores = clf.decision_function(X_test) # outlier scores + + # evaluate and print the results + print("\nOn Training Data:") + evaluate_print(clf_name, y_train, y_train_scores) + print("\nOn Test Data:") + evaluate_print(clf_name, y_test, y_test_scores) diff --git a/pyod/models/deep_sad.py b/pyod/models/deep_sad.py new file mode 100644 index 00000000..ad5b3b01 --- /dev/null +++ b/pyod/models/deep_sad.py @@ -0,0 +1,404 @@ +# -*- coding: utf-8 -*- +"""Deep Semi-Supervised Anomaly Detection (Deep SAD) for outlier detection +""" +# Author: Jayesh Suryavanshi +# License: BSD 2 clause +# +# The Deep SAD objective and the hypersphere-center initialization are +# adapted from the authors' reference implementation +# https://github.com/lukasruff/Deep-SAD-PyTorch (MIT license) and from +# DeepOD's ``deepod/models/tabular/dsad.py`` (BSD-3 license). Neither is +# copied verbatim; the code below follows PyOD conventions and mirrors +# the structure of ``pyod/models/deep_svdd.py``. + + +import numpy as np + +try: + import torch +except ImportError: + print('please install torch first') + +import torch +import torch.nn as nn +import torch.optim as optim + +from sklearn.preprocessing import StandardScaler +from sklearn.utils import check_array +from torch.utils.data import DataLoader, TensorDataset + +from .base import BaseDetector +from ..utils.torch_utility import get_activation_by_name +from ..utils.utility import check_parameter + +optimizer_dict = { + 'sgd': optim.SGD, + 'adam': optim.Adam, + 'rmsprop': optim.RMSprop, + 'adagrad': optim.Adagrad, + 'adadelta': optim.Adadelta, + 'adamw': optim.AdamW, + 'nadam': optim.NAdam, + 'sparseadam': optim.SparseAdam, + 'asgd': optim.ASGD, + 'lbfgs': optim.LBFGS +} + + +class InnerDeepSAD(nn.Module): + """Inner encoder network for the Deep SAD model. + + The network maps the input into a low-dimensional representation space + in which the semi-supervised hypersphere objective is optimized. Unlike + Deep SVDD, no autoencoder branch is used: the Deep SAD loss operates + purely on the distance of the representation to the hypersphere center. + + Parameters + ---------- + n_features : int + Number of features in the input data. + + hidden_neurons : list, optional (default=[64, 32]) + The number of neurons per hidden layers. The last entry is the + dimensionality of the representation space. + + hidden_activation : str, optional (default='relu') + Activation function to use for hidden layers. + All hidden layers are forced to use the same type of activation. + + dropout_rate : float in (0., 1), optional (default=0.2) + The dropout to be used across all layers. + + l2_regularizer : float in (0., 1), optional (default=0.1) + The regularization strength of activity_regularizer + applied on each layer. By default, l2 regularizer is used. + """ + + def __init__(self, n_features, hidden_neurons, hidden_activation, + dropout_rate, l2_regularizer): + super(InnerDeepSAD, self).__init__() + self.n_features = n_features + self.hidden_neurons = hidden_neurons or [64, 32] + self.hidden_activation = hidden_activation + self.dropout_rate = dropout_rate + self.l2_regularizer = l2_regularizer + self.model = self._build_model() + + def _init_c(self, X_norm, eps=0.1): + """Initialize the hypersphere center as the mean of an initial + forward pass. Center components that are too close to zero are + pushed away by ``eps`` so that they cannot be trivially matched + by zero network weights. + """ + self.model.eval() + with torch.no_grad(): + out = self.model(X_norm) + c = torch.mean(out, dim=0) + c[(torch.abs(c) < eps) & (c < 0)] = -eps + c[(torch.abs(c) < eps) & (c > 0)] = eps + return c + + def _build_model(self): + layers = nn.Sequential() + layers.add_module('input_layer', + nn.Linear(self.n_features, self.hidden_neurons[0], + bias=False)) + layers.add_module('hidden_activation_e0', + get_activation_by_name(self.hidden_activation)) + for i in range(1, len(self.hidden_neurons) - 1): + layers.add_module(f'hidden_layer_e{i}', + nn.Linear(self.hidden_neurons[i - 1], + self.hidden_neurons[i], bias=False)) + layers.add_module(f'hidden_activation_e{i}', + get_activation_by_name(self.hidden_activation)) + layers.add_module(f'hidden_dropout_e{i}', + nn.Dropout(self.dropout_rate)) + layers.add_module('net_output', + nn.Linear(self.hidden_neurons[-2], + self.hidden_neurons[-1], bias=False)) + return layers + + def forward(self, x): + return self.model(x) + + +class DeepSAD(BaseDetector): + """Deep Semi-Supervised Anomaly Detection (Deep SAD) is the + semi-supervised extension of Deep SVDD. A neural network is trained to + map the data into a representation space where inliers are pulled + toward a fixed hypersphere center ``c`` while labeled anomalies are + pushed away from it. Unlabeled samples fall back on the unsupervised + Deep SVDD objective (minimize the squared distance to ``c``); labeled + anomalies use the inverse-distance term, so their contribution grows + as they approach the center. The distance to ``c`` is used as the + outlier score at inference time. + See :cite:`ruff2020deepsad` for details. + + Parameters + ---------- + n_features : int + Number of features in the input data. + + c : float, optional (default=None) + Deep SAD center. The default will be calculated based on network + initialization from a first forward pass. To get repeated results + set random_state when c is left as None. + + eta : float, optional (default=1.0) + Weight of the labeled term in the Deep SAD loss. Higher values + place more emphasis on the (few) labeled anomalies relative to the + unlabeled samples. + + hidden_neurons : list, optional (default=[64, 32]) + The number of neurons per hidden layers. The last entry is the + dimensionality of the representation space, e.g. [64, 32]. + + hidden_activation : str, optional (default='relu') + Activation function to use for hidden layers. + All hidden layers are forced to use the same type of activation. + See https://keras.io/activations/ + + optimizer : str, optional (default='adam') + String (name of optimizer) or optimizer instance. + See https://keras.io/optimizers/ + + epochs : int, optional (default=100) + Number of epochs to train the model. + + batch_size : int, optional (default=32) + Number of samples per gradient update. + + dropout_rate : float in (0., 1), optional (default=0.2) + The dropout to be used across all layers. + + l2_regularizer : float in (0., 1), optional (default=0.1) + The regularization strength of activity_regularizer + applied on each layer. By default, l2 regularizer is used. See + https://keras.io/regularizers/ + + validation_size : float in (0., 1), optional (default=0.1) + The percentage of data to be used for validation. + + preprocessing : bool, optional (default=True) + If True, apply standardization on the data. + + eps : float, optional (default=1e-6) + A small value added to the distance of labeled anomalies for + numerical stability in the inverse-distance term. + + verbose : int, optional (default=1) + Verbosity mode. + - 0 = silent + - 1 = progress bar + + random_state : random_state: int, RandomState instance or None, + optional (default=None) + If int, random_state is the seed used by the random + number generator; If RandomState instance, random_state is the + random number generator; If None, the random number generator is + the RandomState instance used by `np.random`. + + contamination : float in (0., 0.5), optional (default=0.1) + The amount of contamination of the data set, i.e. + the proportion of outliers in the data set. When fitting this is + used to define the threshold on the decision function. + + Attributes + ---------- + decision_scores_ : numpy array of shape (n_samples,) + The outlier scores of the training data. + The higher, the more abnormal. Outliers tend to have higher + scores. This value is available once the detector is + fitted. + + threshold_ : float + The threshold is based on ``contamination``. It is the + ``n_samples * contamination`` most abnormal samples in + ``decision_scores_``. The threshold is calculated for generating + binary outlier labels. + + labels_ : int, either 0 or 1 + The binary labels of the training data. 0 stands for inliers + and 1 for outliers/anomalies. It is generated by applying + ``threshold_`` on ``decision_scores_``. + """ + + def __init__(self, n_features, c=None, eta=1.0, hidden_neurons=None, + hidden_activation='relu', optimizer='adam', epochs=100, + batch_size=32, dropout_rate=0.2, l2_regularizer=0.1, + validation_size=0.1, preprocessing=True, eps=1e-6, + verbose=1, random_state=None, contamination=0.1): + super(DeepSAD, self).__init__(contamination=contamination) + + self.n_features = n_features + self.c = c + self.eta = eta + self.hidden_neurons = hidden_neurons or [64, 32] + self.hidden_activation = hidden_activation + self.optimizer = optimizer + self.epochs = epochs + self.batch_size = batch_size + self.dropout_rate = dropout_rate + self.l2_regularizer = l2_regularizer + self.validation_size = validation_size + self.preprocessing = preprocessing + self.eps = eps + self.verbose = verbose + self.random_state = random_state + self.model_ = None + self.best_model_dict = None + + if self.random_state is not None: + torch.manual_seed(self.random_state) + check_parameter(dropout_rate, 0, 1, param_name='dropout_rate', + include_left=True) + + def _process_semi_targets(self, y, n_samples): + """Turn the optional labels ``y`` into Deep SAD semi-supervised + targets in ``{-1, 0}``. + + - Unlabeled / normal samples receive a target of ``0`` and are + trained with the unsupervised Deep SVDD objective (minimize the + squared distance to the center). + - Labeled anomalies (``y == 1``) receive a target of ``-1`` and + are trained with the inverse-distance term, which pushes them + away from the center. + + Passing ``y=None`` makes every sample unlabeled, recovering the + unsupervised Deep SVDD behavior. + """ + semi_targets = np.zeros(n_samples, dtype=np.float32) + if y is not None: + y = np.asarray(y).ravel() + semi_targets[y == 1] = -1 + return semi_targets + + def fit(self, X, y=None): + """Fit detector. Deep SAD is semi-supervised: ``y`` is optional + and, when provided, marks labeled anomalies (``y == 1``). All other + samples are treated as unlabeled and use the unsupervised objective. + + Parameters + ---------- + X : numpy array of shape (n_samples, n_features) + The input samples. + + y : numpy array of shape (n_samples,), optional (default=None) + Optional labels, where 1 indicates a known anomaly and 0 marks + an unlabeled/normal sample. If None, every sample is unlabeled. + + Returns + ------- + self : object + Fitted estimator. + """ + # validate inputs X and y (optional) + X = check_array(X) + # Deep SAD is semi-supervised, so y carries partial supervision and + # is expected; treat this as a binary anomaly-detection problem. + self._classes = 2 + + self.n_samples_, self.n_features_ = X.shape[0], X.shape[1] + + semi_targets = self._process_semi_targets(y, self.n_samples_) + + # Standardize data for better performance + if self.preprocessing: + self.scaler_ = StandardScaler() + X_norm = self.scaler_.fit_transform(X) + else: + X_norm = np.copy(X) + + # Validate and complete the number of hidden neurons + if np.min(self.hidden_neurons) > self.n_features_: + raise ValueError("The number of neurons should not exceed " + "the number of features") + + # Build Deep SAD model & fit with X + self.model_ = InnerDeepSAD(self.n_features, + hidden_neurons=self.hidden_neurons, + hidden_activation=self.hidden_activation, + dropout_rate=self.dropout_rate, + l2_regularizer=self.l2_regularizer) + + X_norm = torch.tensor(X_norm, dtype=torch.float32) + semi_targets = torch.tensor(semi_targets, dtype=torch.float32) + + # Initialize the hypersphere center from a first forward pass + if self.c is None: + self.c_ = self.model_._init_c(X_norm) + elif not torch.is_tensor(self.c): + self.c_ = torch.tensor(self.c, dtype=torch.float32) + else: + self.c_ = self.c + + dataset = TensorDataset(X_norm, semi_targets) + dataloader = DataLoader(dataset, batch_size=self.batch_size, + shuffle=True) + + best_loss = float('inf') + best_model_dict = None + + optimizer = optimizer_dict[self.optimizer]( + self.model_.parameters(), weight_decay=self.l2_regularizer) + + for epoch in range(self.epochs): + self.model_.train() + epoch_loss = 0 + for batch_x, batch_semi in dataloader: + optimizer.zero_grad() + outputs = self.model_(batch_x) + dist = torch.sum((outputs - self.c_) ** 2, dim=-1) + # Deep SAD loss: unlabeled/normal points (semi == 0) + # minimize the distance; labeled anomalies (semi == -1) + # minimize the inverse distance, pushing them away from c. + losses = torch.where( + batch_semi == 0, dist, + self.eta * ((dist + self.eps) ** batch_semi)) + loss = torch.mean(losses) + loss.backward() + optimizer.step() + epoch_loss += loss.item() + if epoch_loss < best_loss: + best_loss = epoch_loss + best_model_dict = self.model_.state_dict() + if self.verbose: + print(f"Epoch {epoch + 1}/{self.epochs}, " + f"Loss: {epoch_loss}") + self.best_model_dict = best_model_dict + + self.decision_scores_ = self.decision_function(X) + self._process_decision_scores() + return self + + def decision_function(self, X): + """Predict raw anomaly score of X using the fitted detector. + + The anomaly score is the squared distance of the learned + representation to the hypersphere center. For consistency, outliers + are assigned with larger anomaly scores. + + Parameters + ---------- + X : numpy array of shape (n_samples, n_features) + The training input samples. Sparse matrices are accepted only + if they are supported by the base estimator. + + Returns + ------- + anomaly_scores : numpy array of shape (n_samples,) + The anomaly score of the input samples. + """ + X = check_array(X) + + if self.preprocessing: + X_norm = self.scaler_.transform(X) + else: + X_norm = np.copy(X) + X_norm = torch.tensor(X_norm, dtype=torch.float32) + self.model_.eval() + with torch.no_grad(): + outputs = self.model_(X_norm) + dist = torch.sum((outputs - self.c_) ** 2, dim=-1) + anomaly_scores = dist.numpy() + return anomaly_scores diff --git a/pyod/test/test_deep_sad.py b/pyod/test/test_deep_sad.py new file mode 100644 index 00000000..ef0d95a2 --- /dev/null +++ b/pyod/test/test_deep_sad.py @@ -0,0 +1,162 @@ +# -*- coding: utf-8 -*- + + +import os +import sys +import unittest + +import numpy as np +# noinspection PyProtectedMember +from numpy.testing import assert_equal +from numpy.testing import assert_raises +from sklearn.base import clone +from sklearn.metrics import roc_auc_score + +# temporary solution for relative imports in case pyod is not installed +# if pyod is installed, no need to use the following line +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) + +from pyod.models.deep_sad import DeepSAD +from pyod.utils.data import generate_data + + +class TestDeepSAD(unittest.TestCase): + def setUp(self): + self.n_train = 6000 + self.n_test = 1000 + self.n_features = 300 + self.contamination = 0.1 + self.roc_floor = 0.5 + self.X_train, self.X_test, self.y_train, self.y_test = generate_data( + n_train=self.n_train, n_test=self.n_test, + n_features=self.n_features, contamination=self.contamination, + random_state=42) + + # Deep SAD is semi-supervised: reveal a subset of the training + # anomalies as labeled (1) and leave everything else unlabeled (0). + self.semi_y = np.zeros(self.n_train, dtype=int) + anomaly_idx = np.where(self.y_train == 1)[0] + revealed = anomaly_idx[:len(anomaly_idx) // 2] + self.semi_y[revealed] = 1 + + self.clf = DeepSAD(n_features=self.n_features, epochs=10, + hidden_neurons=[64, 32], + contamination=self.contamination, + random_state=2021) + self.clf.fit(self.X_train, self.semi_y) + + # a second detector trained fully unsupervised (y=None) exercises + # the Deep SVDD fallback path. + self.clf_unsup = DeepSAD(n_features=self.n_features, epochs=5, + hidden_neurons=[32, 16], + contamination=self.contamination, + preprocessing=False) + self.clf_unsup.fit(self.X_train) + + def test_parameters(self): + assert (hasattr(self.clf, 'decision_scores_') and + self.clf.decision_scores_ is not None) + assert (hasattr(self.clf, 'labels_') and + self.clf.labels_ is not None) + assert (hasattr(self.clf, 'threshold_') and + self.clf.threshold_ is not None) + assert (hasattr(self.clf, '_mu') and + self.clf._mu is not None) + assert (hasattr(self.clf, '_sigma') and + self.clf._sigma is not None) + assert (hasattr(self.clf, 'model_') and + self.clf.model_ is not None) + + def test_train_scores(self): + assert_equal(len(self.clf.decision_scores_), self.X_train.shape[0]) + + def test_prediction_scores(self): + pred_scores = self.clf.decision_function(self.X_test) + + # check score shapes + assert_equal(pred_scores.shape[0], self.X_test.shape[0]) + + # check performance + assert (roc_auc_score(self.y_test, pred_scores) >= self.roc_floor) + + def test_prediction_labels(self): + pred_labels = self.clf.predict(self.X_test) + assert_equal(pred_labels.shape, self.y_test.shape) + + def test_prediction_proba(self): + pred_proba = self.clf.predict_proba(self.X_test) + assert (pred_proba.min() >= 0) + assert (pred_proba.max() <= 1) + + def test_prediction_proba_linear(self): + pred_proba = self.clf.predict_proba(self.X_test, method='linear') + assert (pred_proba.min() >= 0) + assert (pred_proba.max() <= 1) + + def test_prediction_proba_unify(self): + pred_proba = self.clf.predict_proba(self.X_test, method='unify') + assert (pred_proba.min() >= 0) + assert (pred_proba.max() <= 1) + + def test_prediction_proba_parameter(self): + with assert_raises(ValueError): + self.clf.predict_proba(self.X_test, method='something') + + def test_prediction_labels_confidence(self): + pred_labels, confidence = self.clf.predict(self.X_test, + return_confidence=True) + assert_equal(pred_labels.shape, self.y_test.shape) + assert_equal(confidence.shape, self.y_test.shape) + assert (confidence.min() >= 0) + assert (confidence.max() <= 1) + + def test_prediction_proba_linear_confidence(self): + pred_proba, confidence = self.clf.predict_proba(self.X_test, + method='linear', + return_confidence=True) + assert (pred_proba.min() >= 0) + assert (pred_proba.max() <= 1) + + assert_equal(confidence.shape, self.y_test.shape) + assert (confidence.min() >= 0) + assert (confidence.max() <= 1) + + def test_prediction_with_rejection(self): + pred_labels = self.clf.predict_with_rejection(self.X_test, + return_stats=False) + assert_equal(pred_labels.shape, self.y_test.shape) + + def test_prediction_with_rejection_stats(self): + _, [expected_rejrate, ub_rejrate, + ub_cost] = self.clf.predict_with_rejection(self.X_test, + return_stats=True) + assert (expected_rejrate >= 0) + assert (expected_rejrate <= 1) + assert (ub_rejrate >= 0) + assert (ub_rejrate <= 1) + assert (ub_cost >= 0) + + def test_fit_predict(self): + pred_labels = self.clf.fit_predict(self.X_train) + assert_equal(pred_labels.shape, self.y_train.shape) + + def test_fit_predict_score(self): + self.clf.fit_predict_score(self.X_test, self.y_test) + self.clf.fit_predict_score(self.X_test, self.y_test, + scoring='roc_auc_score') + self.clf.fit_predict_score(self.X_test, self.y_test, + scoring='prc_n_score') + with assert_raises(NotImplementedError): + self.clf.fit_predict_score(self.X_test, self.y_test, + scoring='something') + + def test_model_clone(self): + clone(self.clf) + clone(self.clf_unsup) + + def tearDown(self): + pass + + +if __name__ == '__main__': + unittest.main() From feb561aad8506860b305e94a3af9731808134803 Mon Sep 17 00:00:00 2001 From: Jayesh Suryavanshi Date: Sat, 1 Aug 2026 20:36:14 -0700 Subject: [PATCH 2/8] Address review: allow wide encoder layers and validate the center Per the automated review on the PR: - Remove the hidden_neurons > n_features check. Deep SAD uses only an encoder (no autoencoder branch like Deep SVDD), so hidden layers may legitimately be wider than the input dimension; the check wrongly rejected valid setups on datasets with fewer features than the default hidden width. - Validate a user-supplied c: it must match the representation dimension (hidden_neurons[-1]) and be finite, instead of silently broadcasting a scalar or a mismatched vector. Add tests for both. --- pyod/models/deep_sad.py | 30 ++++++++++++++++++++---------- pyod/test/test_deep_sad.py | 28 ++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 10 deletions(-) diff --git a/pyod/models/deep_sad.py b/pyod/models/deep_sad.py index ad5b3b01..75556277 100644 --- a/pyod/models/deep_sad.py +++ b/pyod/models/deep_sad.py @@ -309,12 +309,9 @@ def fit(self, X, y=None): else: X_norm = np.copy(X) - # Validate and complete the number of hidden neurons - if np.min(self.hidden_neurons) > self.n_features_: - raise ValueError("The number of neurons should not exceed " - "the number of features") - - # Build Deep SAD model & fit with X + # Build Deep SAD model & fit with X. Unlike Deep SVDD (which + # reverses the encoder in an optional autoencoder), Deep SAD uses + # only the encoder, so hidden layers may be wider than the input. self.model_ = InnerDeepSAD(self.n_features, hidden_neurons=self.hidden_neurons, hidden_activation=self.hidden_activation, @@ -324,13 +321,26 @@ def fit(self, X, y=None): X_norm = torch.tensor(X_norm, dtype=torch.float32) semi_targets = torch.tensor(semi_targets, dtype=torch.float32) - # Initialize the hypersphere center from a first forward pass + # Initialize the hypersphere center from a first forward pass, or + # validate the user-supplied center against the representation + # space. + rep_dim = self.hidden_neurons[-1] if self.c is None: self.c_ = self.model_._init_c(X_norm) - elif not torch.is_tensor(self.c): - self.c_ = torch.tensor(self.c, dtype=torch.float32) else: - self.c_ = self.c + if torch.is_tensor(self.c): + c_ = self.c.to(torch.float32) + else: + c_ = torch.tensor(self.c, dtype=torch.float32) + c_ = c_.flatten() + if c_.shape[0] != rep_dim: + raise ValueError( + "c must match the representation dimension " + "(hidden_neurons[-1] = {}), got {}".format( + rep_dim, c_.shape[0])) + if not torch.isfinite(c_).all(): + raise ValueError("c must be finite.") + self.c_ = c_ dataset = TensorDataset(X_norm, semi_targets) dataloader = DataLoader(dataset, batch_size=self.batch_size, diff --git a/pyod/test/test_deep_sad.py b/pyod/test/test_deep_sad.py index ef0d95a2..fea77b79 100644 --- a/pyod/test/test_deep_sad.py +++ b/pyod/test/test_deep_sad.py @@ -154,6 +154,34 @@ def test_model_clone(self): clone(self.clf) clone(self.clf_unsup) + def test_hidden_neurons_wider_than_input(self): + # Deep SAD uses only an encoder, so hidden layers may be wider + # than the input dimension; fitting must not raise. + X = self.X_train[:, :8] + clf = DeepSAD(n_features=8, epochs=2, hidden_neurons=[64, 32], + contamination=self.contamination) + clf.fit(X) + assert clf.decision_scores_ is not None + + def test_invalid_center_raises(self): + # A center that does not match the representation dimension is + # rejected rather than silently broadcasting. + with assert_raises(ValueError): + DeepSAD(n_features=self.n_features, c=0.0, epochs=1, + hidden_neurons=[64, 32]).fit(self.X_train) + with assert_raises(ValueError): + DeepSAD(n_features=self.n_features, c=[1.0, 2.0], epochs=1, + hidden_neurons=[64, 32]).fit(self.X_train) + + def test_valid_center_accepted(self): + # A correctly sized, finite center is accepted. + c = np.ones(32) + clf = DeepSAD(n_features=self.n_features, c=c, epochs=2, + hidden_neurons=[64, 32], + contamination=self.contamination) + clf.fit(self.X_train) + assert clf.decision_scores_ is not None + def tearDown(self): pass From 148f05fe69f333b7a85ae8384aaac0f3b8059888 Mon Sep 17 00:00:00 2001 From: Jayesh Suryavanshi Date: Fri, 11 Sep 2026 13:11:45 -0700 Subject: [PATCH 3/8] Address Codex review: restore the best epoch, expand scalar centers, seed torch from a RandomState, drop unsupported optimizers --- pyod/models/deep_sad.py | 83 +++++++++++++++++++++------------ pyod/test/test_deep_sad.py | 94 ++++++++++++++++++++++++++++++++++---- 2 files changed, 139 insertions(+), 38 deletions(-) diff --git a/pyod/models/deep_sad.py b/pyod/models/deep_sad.py index 75556277..1949d87f 100644 --- a/pyod/models/deep_sad.py +++ b/pyod/models/deep_sad.py @@ -12,6 +12,8 @@ # the structure of ``pyod/models/deep_svdd.py``. +import copy + import numpy as np try: @@ -24,7 +26,7 @@ import torch.optim as optim from sklearn.preprocessing import StandardScaler -from sklearn.utils import check_array +from sklearn.utils import check_array, check_random_state from torch.utils.data import DataLoader, TensorDataset from .base import BaseDetector @@ -39,9 +41,7 @@ 'adadelta': optim.Adadelta, 'adamw': optim.AdamW, 'nadam': optim.NAdam, - 'sparseadam': optim.SparseAdam, - 'asgd': optim.ASGD, - 'lbfgs': optim.LBFGS + 'asgd': optim.ASGD } @@ -139,10 +139,15 @@ class DeepSAD(BaseDetector): n_features : int Number of features in the input data. - c : float, optional (default=None) - Deep SAD center. The default will be calculated based on network - initialization from a first forward pass. To get repeated results - set random_state when c is left as None. + c : float, numpy array, or torch tensor, optional (default=None) + Deep SAD hypersphere center. When None (the default) the center + is initialized from the first forward pass of the freshly built + network; set ``random_state`` for repeatable results. A scalar is + expanded to the representation width ``hidden_neurons[-1]``; a + vector must match that width. The value must be finite and not + all zeros: an all-zero center is the trivial solution of the + objective for a bias-free network and collapses the hypersphere. + The fitted center is exposed as ``c_``. eta : float, optional (default=1.0) Weight of the labeled term in the Deep SAD loss. Higher values @@ -159,8 +164,8 @@ class DeepSAD(BaseDetector): See https://keras.io/activations/ optimizer : str, optional (default='adam') - String (name of optimizer) or optimizer instance. - See https://keras.io/optimizers/ + Name of the optimizer. One of 'sgd', 'adam', 'rmsprop', + 'adagrad', 'adadelta', 'adamw', 'nadam' or 'asgd'. epochs : int, optional (default=100) Number of epochs to train the model. @@ -194,9 +199,10 @@ class DeepSAD(BaseDetector): random_state : random_state: int, RandomState instance or None, optional (default=None) If int, random_state is the seed used by the random - number generator; If RandomState instance, random_state is the - random number generator; If None, the random number generator is - the RandomState instance used by `np.random`. + number generator; If RandomState instance, an integer seed is + drawn from it (advancing it) to seed torch; If None, the random + number generator is the RandomState instance used by + `np.random`. contamination : float in (0., 0.5), optional (default=0.1) The amount of contamination of the data set, i.e. @@ -221,6 +227,12 @@ class DeepSAD(BaseDetector): The binary labels of the training data. 0 stands for inliers and 1 for outliers/anomalies. It is generated by applying ``threshold_`` on ``decision_scores_``. + + c_ : torch tensor of shape (hidden_neurons[-1],) + The fitted hypersphere center used by the training objective and + by ``decision_function``. Either the center initialized from the + data (when ``c`` is None) or a validated copy of the + user-supplied ``c``. """ def __init__(self, n_features, c=None, eta=1.0, hidden_neurons=None, @@ -249,7 +261,13 @@ def __init__(self, n_features, c=None, eta=1.0, hidden_neurons=None, self.best_model_dict = None if self.random_state is not None: - torch.manual_seed(self.random_state) + if isinstance(self.random_state, (int, np.integer)): + torch_seed = self.random_state + else: + torch_seed = int( + check_random_state(self.random_state).randint( + 0, np.iinfo(np.int32).max)) + torch.manual_seed(torch_seed) check_parameter(dropout_rate, 0, 1, param_name='dropout_rate', include_left=True) @@ -326,21 +344,26 @@ def fit(self, X, y=None): # space. rep_dim = self.hidden_neurons[-1] if self.c is None: - self.c_ = self.model_._init_c(X_norm) + center = self.model_._init_c(X_norm) else: - if torch.is_tensor(self.c): - c_ = self.c.to(torch.float32) - else: - c_ = torch.tensor(self.c, dtype=torch.float32) - c_ = c_.flatten() - if c_.shape[0] != rep_dim: - raise ValueError( - "c must match the representation dimension " - "(hidden_neurons[-1] = {}), got {}".format( - rep_dim, c_.shape[0])) - if not torch.isfinite(c_).all(): - raise ValueError("c must be finite.") - self.c_ = c_ + center = torch.as_tensor(self.c, dtype=X_norm.dtype, + device=X_norm.device) + if center.ndim == 0: + center = center.repeat(rep_dim) + elif center.ndim != 1 or center.numel() != rep_dim: + raise ValueError( + "c must be a scalar or a vector of length {} " + "(hidden_neurons[-1]), got shape {}".format( + rep_dim, tuple(center.shape))) + if not torch.isfinite(center).all(): + raise ValueError("c must contain only finite values.") + if not torch.any(center != 0): + raise ValueError( + "c is all zeros, which makes the trivial all-zero-weights " + "solution optimal and collapses the hypersphere. Pass " + "c=None to initialize the center from the data, or " + "supply a non-zero center.") + self.c_ = center.detach().clone() dataset = TensorDataset(X_norm, semi_targets) dataloader = DataLoader(dataset, batch_size=self.batch_size, @@ -371,11 +394,13 @@ def fit(self, X, y=None): epoch_loss += loss.item() if epoch_loss < best_loss: best_loss = epoch_loss - best_model_dict = self.model_.state_dict() + best_model_dict = copy.deepcopy(self.model_.state_dict()) if self.verbose: print(f"Epoch {epoch + 1}/{self.epochs}, " f"Loss: {epoch_loss}") self.best_model_dict = best_model_dict + if self.best_model_dict is not None: + self.model_.load_state_dict(self.best_model_dict) self.decision_scores_ = self.decision_function(X) self._process_decision_scores() diff --git a/pyod/test/test_deep_sad.py b/pyod/test/test_deep_sad.py index fea77b79..484005d3 100644 --- a/pyod/test/test_deep_sad.py +++ b/pyod/test/test_deep_sad.py @@ -4,8 +4,10 @@ import os import sys import unittest +from unittest import mock import numpy as np +import torch # noinspection PyProtectedMember from numpy.testing import assert_equal from numpy.testing import assert_raises @@ -16,7 +18,7 @@ # if pyod is installed, no need to use the following line sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) -from pyod.models.deep_sad import DeepSAD +from pyod.models.deep_sad import DeepSAD, optimizer_dict from pyod.utils.data import generate_data @@ -166,21 +168,95 @@ def test_hidden_neurons_wider_than_input(self): def test_invalid_center_raises(self): # A center that does not match the representation dimension is # rejected rather than silently broadcasting. - with assert_raises(ValueError): - DeepSAD(n_features=self.n_features, c=0.0, epochs=1, - hidden_neurons=[64, 32]).fit(self.X_train) - with assert_raises(ValueError): - DeepSAD(n_features=self.n_features, c=[1.0, 2.0], epochs=1, - hidden_neurons=[64, 32]).fit(self.X_train) + for bad_c in ([1.0, 2.0], np.ones(33), float('nan'), + np.array([float('inf')] * 32)): + clf = DeepSAD(n_features=self.n_features, c=bad_c, epochs=1, + hidden_neurons=[64, 32], verbose=0) + with assert_raises(ValueError): + clf.fit(self.X_train) + + def test_all_zero_center_is_rejected(self): + for bad_c in (0.0, np.zeros(32, dtype=np.float32)): + clf = DeepSAD(n_features=self.n_features, c=bad_c, epochs=1, + hidden_neurons=[64, 32], verbose=0) + with assert_raises(ValueError): + clf.fit(self.X_train) + + def test_scalar_center_is_expanded(self): + clf = DeepSAD(n_features=self.n_features, c=1.0, epochs=2, + hidden_neurons=[64, 32], verbose=0, + contamination=self.contamination) + clf.fit(self.X_train) + assert_equal(tuple(clf.c_.shape), (32,)) + assert torch.equal(clf.c_, torch.ones(32)) + assert clf.decision_scores_ is not None def test_valid_center_accepted(self): # A correctly sized, finite center is accepted. - c = np.ones(32) + c = torch.ones(32) clf = DeepSAD(n_features=self.n_features, c=c, epochs=2, - hidden_neurons=[64, 32], + hidden_neurons=[64, 32], verbose=0, contamination=self.contamination) clf.fit(self.X_train) assert clf.decision_scores_ is not None + before = clf.c_.clone() + c[:] = 99.0 + assert torch.equal(before, clf.c_) + + def test_random_state_instance_is_accepted(self): + X = self.X_train[:600, :8] + scores = [] + for _ in range(2): + clf = DeepSAD(n_features=8, epochs=2, hidden_neurons=[16, 8], + verbose=0, + random_state=np.random.RandomState(0)) + clf.fit(X) + scores.append(clf.decision_scores_) + assert np.array_equal(scores[0], scores[1]) + clone(clf) + + def test_all_listed_optimizers_train(self): + assert 'lbfgs' not in optimizer_dict + assert 'sparseadam' not in optimizer_dict + X = self.X_train[:600, :8] + for name in optimizer_dict: + clf = DeepSAD(n_features=8, epochs=1, hidden_neurons=[16, 8], + optimizer=name, verbose=0) + clf.fit(X) + assert np.all(np.isfinite(clf.decision_scores_)), name + + def test_best_epoch_weights_are_restored(self): + X = self.X_train[:600, :8] + batch_size = 100 + steps_per_epoch = int(np.ceil(X.shape[0] / batch_size)) + + class DivergeAfterFirstEpoch(torch.optim.SGD): + def __init__(self, params, **kwargs): + super().__init__(params, **kwargs) + self.n_steps = 0 + + def step(self, closure=None): + self.n_steps += 1 + if self.n_steps <= steps_per_epoch: + return super().step(closure) + with torch.no_grad(): + for group in self.param_groups: + for p in group['params']: + p.fill_(100.0) + + with mock.patch.dict(optimizer_dict, + {'sgd': DivergeAfterFirstEpoch}): + clf = DeepSAD(n_features=8, epochs=3, hidden_neurons=[16, 8], + optimizer='sgd', batch_size=batch_size, + verbose=0, random_state=2021) + clf.fit(X) + + live = clf.model_.state_dict() + for key, snapshot in clf.best_model_dict.items(): + assert snapshot.data_ptr() != live[key].data_ptr(), key + assert torch.equal(snapshot, live[key]), key + assert not torch.any(snapshot == 100.0), key + assert np.all(np.isfinite(clf.decision_scores_)) def tearDown(self): pass From 3b1c56c8cfd8d3a4ccef7f7b2f0277c45254e911 Mon Sep 17 00:00:00 2001 From: Jayesh Suryavanshi Date: Fri, 11 Sep 2026 15:51:37 -0700 Subject: [PATCH 4/8] Address Codex review: use validation_size for best-epoch selection, apply dropout after every hidden activation, require positive eta, add DeepSAD to the docs, skip the tests without torch, train the test fixtures once --- docs/index.rst | 1 + docs/pyod.models.tabular.rst | 10 +++ pyod/models/deep_sad.py | 88 ++++++++++++++------ pyod/test/conftest.py | 1 + pyod/test/test_deep_sad.py | 155 ++++++++++++++++++++++++----------- 5 files changed, 184 insertions(+), 71 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index c39552a2..fb9a3a06 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -257,6 +257,7 @@ Neural Networks Beta-VAE Variational AutoEncoder (all customized l Neural Networks SO_GAAL Single-Objective Generative Adversarial Active Learning 2019 :class:`pyod.models.so_gaal.SO_GAAL` :cite:`a-liu2019generative` Neural Networks MO_GAAL Multiple-Objective Generative Adversarial Active Learning 2019 :class:`pyod.models.mo_gaal.MO_GAAL` :cite:`a-liu2019generative` Neural Networks DeepSVDD Deep One-Class Classification 2018 :class:`pyod.models.deep_svdd.DeepSVDD` :cite:`a-ruff2018deepsvdd` +Neural Networks DeepSAD Deep Semi-Supervised Anomaly Detection 2020 :class:`pyod.models.deep_sad.DeepSAD` :cite:`a-ruff2020deepsad` Neural Networks AnoGAN Anomaly Detection with Generative Adversarial Networks 2017 :class:`pyod.models.anogan.AnoGAN` :cite:`a-schlegl2017unsupervised` Neural Networks ALAD Adversarially learned anomaly detection 2018 :class:`pyod.models.alad.ALAD` :cite:`a-zenati2018adversarially` Neural Networks DevNet Deep Anomaly Detection with Deviation Networks 2019 :class:`pyod.models.devnet.DevNet` :cite:`a-pang2019deep` diff --git a/docs/pyod.models.tabular.rst b/docs/pyod.models.tabular.rst index b1ff7a63..cc134490 100644 --- a/docs/pyod.models.tabular.rst +++ b/docs/pyod.models.tabular.rst @@ -125,6 +125,16 @@ pyod.models.copod module :show-inheritance: +pyod.models.deep\_sad module +---------------------------- + +.. automodule:: pyod.models.deep_sad + :members: + :exclude-members: InnerDeepSAD + :undoc-members: + :show-inheritance: + + pyod.models.deep\_svdd module ----------------------------- diff --git a/pyod/models/deep_sad.py b/pyod/models/deep_sad.py index 1949d87f..9bbd99c2 100644 --- a/pyod/models/deep_sad.py +++ b/pyod/models/deep_sad.py @@ -66,8 +66,10 @@ class InnerDeepSAD(nn.Module): Activation function to use for hidden layers. All hidden layers are forced to use the same type of activation. - dropout_rate : float in (0., 1), optional (default=0.2) - The dropout to be used across all layers. + dropout_rate : float in [0., 1), optional (default=0.2) + The dropout applied after the activation of every hidden layer + (``hidden_neurons[:-1]``). The final representation layer + ``hidden_neurons[-1]`` has no activation and no dropout. l2_regularizer : float in (0., 1), optional (default=0.1) The regularization strength of activity_regularizer @@ -105,6 +107,7 @@ def _build_model(self): bias=False)) layers.add_module('hidden_activation_e0', get_activation_by_name(self.hidden_activation)) + layers.add_module('hidden_dropout_e0', nn.Dropout(self.dropout_rate)) for i in range(1, len(self.hidden_neurons) - 1): layers.add_module(f'hidden_layer_e{i}', nn.Linear(self.hidden_neurons[i - 1], @@ -150,9 +153,10 @@ class DeepSAD(BaseDetector): The fitted center is exposed as ``c_``. eta : float, optional (default=1.0) - Weight of the labeled term in the Deep SAD loss. Higher values - place more emphasis on the (few) labeled anomalies relative to the - unlabeled samples. + Weight of the labeled term in the Deep SAD loss. Must be positive: + a non-positive weight would reward moving labeled anomalies toward + the center. Higher values place more emphasis on the (few) labeled + anomalies relative to the unlabeled samples. hidden_neurons : list, optional (default=[64, 32]) The number of neurons per hidden layers. The last entry is the @@ -173,16 +177,24 @@ class DeepSAD(BaseDetector): batch_size : int, optional (default=32) Number of samples per gradient update. - dropout_rate : float in (0., 1), optional (default=0.2) - The dropout to be used across all layers. + dropout_rate : float in [0., 1), optional (default=0.2) + The dropout applied after the activation of every hidden layer + (``hidden_neurons[:-1]``). The final representation layer + ``hidden_neurons[-1]`` has no activation and no dropout. l2_regularizer : float in (0., 1), optional (default=0.1) The regularization strength of activity_regularizer applied on each layer. By default, l2 regularizer is used. See https://keras.io/regularizers/ - validation_size : float in (0., 1), optional (default=0.1) - The percentage of data to be used for validation. + validation_size : float in [0., 1), optional (default=0.1) + The fraction of the training samples (and their labels) held out + as a validation set, drawn with ``random_state``. The center is + initialized from and the network is trained on the remaining + samples; after every epoch the Deep SAD loss is computed on the + held-out samples and the weights of the epoch with the lowest + validation loss are restored at the end of training. Set to 0 to + train on every sample and select the best epoch by training loss. preprocessing : bool, optional (default=True) If True, apply standardization on the data. @@ -270,6 +282,9 @@ def __init__(self, n_features, c=None, eta=1.0, hidden_neurons=None, torch.manual_seed(torch_seed) check_parameter(dropout_rate, 0, 1, param_name='dropout_rate', include_left=True) + check_parameter(eta, 0, np.inf, param_name='eta') + check_parameter(validation_size, 0, 1, param_name='validation_size', + include_left=True) def _process_semi_targets(self, y, n_samples): """Turn the optional labels ``y`` into Deep SAD semi-supervised @@ -291,6 +306,17 @@ def _process_semi_targets(self, y, n_samples): semi_targets[y == 1] = -1 return semi_targets + def _deep_sad_loss(self, outputs, semi_targets): + """Deep SAD loss: unlabeled/normal points (target ``0``) minimize + the squared distance to the center; labeled anomalies (target + ``-1``) minimize the inverse distance, pushing them away from + ``c``. + """ + dist = torch.sum((outputs - self.c_) ** 2, dim=-1) + losses = torch.where(semi_targets == 0, dist, + self.eta * ((dist + self.eps) ** semi_targets)) + return torch.mean(losses) + def fit(self, X, y=None): """Fit detector. Deep SAD is semi-supervised: ``y`` is optional and, when provided, marks labeled anomalies (``y == 1``). All other @@ -339,12 +365,24 @@ def fit(self, X, y=None): X_norm = torch.tensor(X_norm, dtype=torch.float32) semi_targets = torch.tensor(semi_targets, dtype=torch.float32) + # Hold out the validation split used for best-epoch selection; the + # center and the training batches only see the remaining samples. + n_val = int(self.n_samples_ * self.validation_size) + if n_val > 0: + perm = check_random_state(self.random_state).permutation( + self.n_samples_) + X_val, semi_val = X_norm[perm[:n_val]], semi_targets[perm[:n_val]] + X_fit, semi_fit = X_norm[perm[n_val:]], semi_targets[perm[n_val:]] + else: + X_val, semi_val = None, None + X_fit, semi_fit = X_norm, semi_targets + # Initialize the hypersphere center from a first forward pass, or # validate the user-supplied center against the representation # space. rep_dim = self.hidden_neurons[-1] if self.c is None: - center = self.model_._init_c(X_norm) + center = self.model_._init_c(X_fit) else: center = torch.as_tensor(self.c, dtype=X_norm.dtype, device=X_norm.device) @@ -365,7 +403,7 @@ def fit(self, X, y=None): "supply a non-zero center.") self.c_ = center.detach().clone() - dataset = TensorDataset(X_norm, semi_targets) + dataset = TensorDataset(X_fit, semi_fit) dataloader = DataLoader(dataset, batch_size=self.batch_size, shuffle=True) @@ -380,24 +418,26 @@ def fit(self, X, y=None): epoch_loss = 0 for batch_x, batch_semi in dataloader: optimizer.zero_grad() - outputs = self.model_(batch_x) - dist = torch.sum((outputs - self.c_) ** 2, dim=-1) - # Deep SAD loss: unlabeled/normal points (semi == 0) - # minimize the distance; labeled anomalies (semi == -1) - # minimize the inverse distance, pushing them away from c. - losses = torch.where( - batch_semi == 0, dist, - self.eta * ((dist + self.eps) ** batch_semi)) - loss = torch.mean(losses) + loss = self._deep_sad_loss(self.model_(batch_x), batch_semi) loss.backward() optimizer.step() epoch_loss += loss.item() - if epoch_loss < best_loss: - best_loss = epoch_loss + if X_val is not None: + self.model_.eval() + with torch.no_grad(): + selection_loss = self._deep_sad_loss( + self.model_(X_val), semi_val).item() + else: + selection_loss = epoch_loss + if selection_loss < best_loss: + best_loss = selection_loss best_model_dict = copy.deepcopy(self.model_.state_dict()) if self.verbose: - print(f"Epoch {epoch + 1}/{self.epochs}, " - f"Loss: {epoch_loss}") + message = (f"Epoch {epoch + 1}/{self.epochs}, " + f"Loss: {epoch_loss}") + if X_val is not None: + message += f", Validation loss: {selection_loss}" + print(message) self.best_model_dict = best_model_dict if self.best_model_dict is not None: self.model_.load_state_dict(self.best_model_dict) diff --git a/pyod/test/conftest.py b/pyod/test/conftest.py index e533877f..e03c722c 100644 --- a/pyod/test/conftest.py +++ b/pyod/test/conftest.py @@ -36,6 +36,7 @@ # Deep learning detectors (torch-based) "test_auto_encoder.py", "test_vae.py", + "test_deep_sad.py", "test_deepsvdd.py", "test_so_gaal.py", "test_so_gaal_new.py", diff --git a/pyod/test/test_deep_sad.py b/pyod/test/test_deep_sad.py index 484005d3..a08437d2 100644 --- a/pyod/test/test_deep_sad.py +++ b/pyod/test/test_deep_sad.py @@ -8,6 +8,7 @@ import numpy as np import torch +import torch.nn as nn # noinspection PyProtectedMember from numpy.testing import assert_equal from numpy.testing import assert_raises @@ -18,42 +19,45 @@ # if pyod is installed, no need to use the following line sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) -from pyod.models.deep_sad import DeepSAD, optimizer_dict +from pyod.models.deep_sad import DeepSAD, InnerDeepSAD, optimizer_dict from pyod.utils.data import generate_data class TestDeepSAD(unittest.TestCase): - def setUp(self): - self.n_train = 6000 - self.n_test = 1000 - self.n_features = 300 - self.contamination = 0.1 - self.roc_floor = 0.5 - self.X_train, self.X_test, self.y_train, self.y_test = generate_data( - n_train=self.n_train, n_test=self.n_test, - n_features=self.n_features, contamination=self.contamination, + @classmethod + def setUpClass(cls): + cls.n_train = 6000 + cls.n_test = 1000 + cls.n_features = 300 + cls.contamination = 0.1 + cls.roc_floor = 0.5 + cls.X_train, cls.X_test, cls.y_train, cls.y_test = generate_data( + n_train=cls.n_train, n_test=cls.n_test, + n_features=cls.n_features, contamination=cls.contamination, random_state=42) # Deep SAD is semi-supervised: reveal a subset of the training # anomalies as labeled (1) and leave everything else unlabeled (0). - self.semi_y = np.zeros(self.n_train, dtype=int) - anomaly_idx = np.where(self.y_train == 1)[0] + cls.semi_y = np.zeros(cls.n_train, dtype=int) + anomaly_idx = np.where(cls.y_train == 1)[0] revealed = anomaly_idx[:len(anomaly_idx) // 2] - self.semi_y[revealed] = 1 + cls.semi_y[revealed] = 1 - self.clf = DeepSAD(n_features=self.n_features, epochs=10, - hidden_neurons=[64, 32], - contamination=self.contamination, - random_state=2021) - self.clf.fit(self.X_train, self.semi_y) + # The two fitted detectors are trained once and shared read-only + # by the tests; tests that refit build their own small instance. + cls.clf = DeepSAD(n_features=cls.n_features, epochs=10, + hidden_neurons=[64, 32], + contamination=cls.contamination, + random_state=2021) + cls.clf.fit(cls.X_train, cls.semi_y) # a second detector trained fully unsupervised (y=None) exercises # the Deep SVDD fallback path. - self.clf_unsup = DeepSAD(n_features=self.n_features, epochs=5, - hidden_neurons=[32, 16], - contamination=self.contamination, - preprocessing=False) - self.clf_unsup.fit(self.X_train) + cls.clf_unsup = DeepSAD(n_features=cls.n_features, epochs=5, + hidden_neurons=[32, 16], + contamination=cls.contamination, + preprocessing=False) + cls.clf_unsup.fit(cls.X_train) def test_parameters(self): assert (hasattr(self.clf, 'decision_scores_') and @@ -139,18 +143,21 @@ def test_prediction_with_rejection_stats(self): assert (ub_cost >= 0) def test_fit_predict(self): - pred_labels = self.clf.fit_predict(self.X_train) - assert_equal(pred_labels.shape, self.y_train.shape) + X, y = self.X_train[:600, :8], self.y_train[:600] + clf = DeepSAD(n_features=8, epochs=2, hidden_neurons=[16, 8], + verbose=0, contamination=self.contamination) + pred_labels = clf.fit_predict(X) + assert_equal(pred_labels.shape, y.shape) def test_fit_predict_score(self): - self.clf.fit_predict_score(self.X_test, self.y_test) - self.clf.fit_predict_score(self.X_test, self.y_test, - scoring='roc_auc_score') - self.clf.fit_predict_score(self.X_test, self.y_test, - scoring='prc_n_score') + X, y = self.X_test[:, :8], self.y_test + clf = DeepSAD(n_features=8, epochs=2, hidden_neurons=[16, 8], + verbose=0, contamination=self.contamination) + clf.fit_predict_score(X, y) + clf.fit_predict_score(X, y, scoring='roc_auc_score') + clf.fit_predict_score(X, y, scoring='prc_n_score') with assert_raises(NotImplementedError): - self.clf.fit_predict_score(self.X_test, self.y_test, - scoring='something') + clf.fit_predict_score(X, y, scoring='something') def test_model_clone(self): clone(self.clf) @@ -203,6 +210,52 @@ def test_valid_center_accepted(self): c[:] = 99.0 assert torch.equal(before, clf.c_) + def test_eta_must_be_positive(self): + for bad_eta in (-1, 0): + with assert_raises(ValueError): + DeepSAD(n_features=self.n_features, eta=bad_eta) + + def test_validation_size(self): + X, y = self.X_train[:600, :8], self.semi_y[:600] + original_init_c = InnerDeepSAD._init_c + n_center = [] + + def spy_init_c(model, X_norm, eps=0.1): + n_center.append(X_norm.shape[0]) + return original_init_c(model, X_norm, eps) + + # 0.2 keeps 120 of the 600 samples out of the center initialization + # and the training batches; 0 uses every sample. + for validation_size in (0.2, 0): + clf = DeepSAD(n_features=8, epochs=2, hidden_neurons=[16, 8], + validation_size=validation_size, verbose=0, + random_state=2021) + with mock.patch.object(InnerDeepSAD, '_init_c', spy_init_c): + clf.fit(X, y) + assert_equal(len(clf.decision_scores_), X.shape[0]) + assert np.all(np.isfinite(clf.decision_scores_)) + assert_equal(n_center, [480, 600]) + for bad_size in (-0.1, 1.0): + with assert_raises(ValueError): + DeepSAD(n_features=8, validation_size=bad_size) + + def test_dropout_follows_hidden_activations(self): + X = self.X_train[:600, :8] + # (hidden_neurons, dropout_rate); None is the default [64, 32] + for hidden_neurons, dropout_rate in ((None, 0.2), + ([64, 32, 16], 0.5), + (None, 0)): + clf = DeepSAD(n_features=8, epochs=1, + hidden_neurons=hidden_neurons, + dropout_rate=dropout_rate, verbose=0) + clf.fit(X) + modules = list(clf.model_.model) + dropouts = [m for m in modules if isinstance(m, nn.Dropout)] + assert_equal(len(dropouts), len(clf.hidden_neurons) - 1) + assert all(m.p == dropout_rate for m in dropouts) + assert isinstance(modules[-1], nn.Linear) + assert_equal(modules[-1].out_features, clf.hidden_neurons[-1]) + def test_random_state_instance_is_accepted(self): X = self.X_train[:600, :8] scores = [] @@ -228,35 +281,43 @@ def test_all_listed_optimizers_train(self): def test_best_epoch_weights_are_restored(self): X = self.X_train[:600, :8] batch_size = 100 - steps_per_epoch = int(np.ceil(X.shape[0] / batch_size)) class DivergeAfterFirstEpoch(torch.optim.SGD): + steps_per_epoch = None + def __init__(self, params, **kwargs): super().__init__(params, **kwargs) self.n_steps = 0 def step(self, closure=None): self.n_steps += 1 - if self.n_steps <= steps_per_epoch: + if self.n_steps <= self.steps_per_epoch: return super().step(closure) with torch.no_grad(): for group in self.param_groups: for p in group['params']: p.fill_(100.0) - with mock.patch.dict(optimizer_dict, - {'sgd': DivergeAfterFirstEpoch}): - clf = DeepSAD(n_features=8, epochs=3, hidden_neurons=[16, 8], - optimizer='sgd', batch_size=batch_size, - verbose=0, random_state=2021) - clf.fit(X) - - live = clf.model_.state_dict() - for key, snapshot in clf.best_model_dict.items(): - assert snapshot.data_ptr() != live[key].data_ptr(), key - assert torch.equal(snapshot, live[key]), key - assert not torch.any(snapshot == 100.0), key - assert np.all(np.isfinite(clf.decision_scores_)) + # validation_size=0 selects the epoch by training loss, 0.5 by the + # loss on the held-out half; both must restore the first epoch. + for validation_size in (0, 0.5): + n_fit = X.shape[0] - int(X.shape[0] * validation_size) + DivergeAfterFirstEpoch.steps_per_epoch = int( + np.ceil(n_fit / batch_size)) + with mock.patch.dict(optimizer_dict, + {'sgd': DivergeAfterFirstEpoch}): + clf = DeepSAD(n_features=8, epochs=3, hidden_neurons=[16, 8], + optimizer='sgd', batch_size=batch_size, + validation_size=validation_size, verbose=0, + random_state=2021) + clf.fit(X) + + live = clf.model_.state_dict() + for key, snapshot in clf.best_model_dict.items(): + assert snapshot.data_ptr() != live[key].data_ptr(), key + assert torch.equal(snapshot, live[key]), key + assert not torch.any(snapshot == 100.0), key + assert np.all(np.isfinite(clf.decision_scores_)) def tearDown(self): pass From 61e36bee54d8a69df4716effbc9b7c0f43ee3db5 Mon Sep 17 00:00:00 2001 From: Jayesh Suryavanshi Date: Fri, 11 Sep 2026 19:33:01 -0700 Subject: [PATCH 5/8] Address Codex review: stratify the validation split, fit the scaler on the training fold, weight the epoch loss by batch size, validate eps, handle a single-width network --- pyod/models/deep_sad.py | 103 ++++++++++++++++++++++--------- pyod/test/test_deep_sad.py | 121 +++++++++++++++++++++++++++++++++++++ 2 files changed, 195 insertions(+), 29 deletions(-) diff --git a/pyod/models/deep_sad.py b/pyod/models/deep_sad.py index 9bbd99c2..20464ba5 100644 --- a/pyod/models/deep_sad.py +++ b/pyod/models/deep_sad.py @@ -60,7 +60,9 @@ class InnerDeepSAD(nn.Module): hidden_neurons : list, optional (default=[64, 32]) The number of neurons per hidden layers. The last entry is the - dimensionality of the representation space. + dimensionality of the representation space. A single entry + ``[k]`` maps the input straight to the representation space + with one bias-free linear layer. hidden_activation : str, optional (default='relu') Activation function to use for hidden layers. @@ -102,6 +104,11 @@ def _init_c(self, X_norm, eps=0.1): def _build_model(self): layers = nn.Sequential() + if len(self.hidden_neurons) == 1: + layers.add_module('net_output', + nn.Linear(self.n_features, + self.hidden_neurons[0], bias=False)) + return layers layers.add_module('input_layer', nn.Linear(self.n_features, self.hidden_neurons[0], bias=False)) @@ -160,7 +167,9 @@ class DeepSAD(BaseDetector): hidden_neurons : list, optional (default=[64, 32]) The number of neurons per hidden layers. The last entry is the - dimensionality of the representation space, e.g. [64, 32]. + dimensionality of the representation space, e.g. [64, 32]. A + single entry, e.g. [32], maps the input straight to the + representation space with one bias-free linear layer. hidden_activation : str, optional (default='relu') Activation function to use for hidden layers. @@ -189,19 +198,25 @@ class DeepSAD(BaseDetector): validation_size : float in [0., 1), optional (default=0.1) The fraction of the training samples (and their labels) held out - as a validation set, drawn with ``random_state``. The center is - initialized from and the network is trained on the remaining - samples; after every epoch the Deep SAD loss is computed on the - held-out samples and the weights of the epoch with the lowest - validation loss are restored at the end of training. Set to 0 to - train on every sample and select the best epoch by training loss. + as a validation set, drawn with ``random_state`` and stratified + on the semi-supervised targets: the unlabeled samples and the + labeled anomalies are split separately, each holding out + ``round(n * validation_size)`` of its ``n`` members but never all + of them, so a lone labeled anomaly always stays in the training + split. The scaler is fitted on, the center is initialized from and the + network is trained on the remaining samples; after every epoch + the Deep SAD loss is computed on the held-out samples and the + weights of the epoch with the lowest validation loss are restored + at the end of training. Set to 0 to train on every sample and + select the best epoch by training loss. preprocessing : bool, optional (default=True) If True, apply standardization on the data. eps : float, optional (default=1e-6) A small value added to the distance of labeled anomalies for - numerical stability in the inverse-distance term. + numerical stability in the inverse-distance term. Must be a + positive finite float. verbose : int, optional (default=1) Verbosity mode. @@ -285,6 +300,11 @@ def __init__(self, n_features, c=None, eta=1.0, hidden_neurons=None, check_parameter(eta, 0, np.inf, param_name='eta') check_parameter(validation_size, 0, 1, param_name='validation_size', include_left=True) + check_parameter(eps, 0, np.inf, param_name='eps') + if not np.isfinite(eps): + raise ValueError( + 'eps is set to {}. Must be a positive finite float.'.format( + eps)) def _process_semi_targets(self, y, n_samples): """Turn the optional labels ``y`` into Deep SAD semi-supervised @@ -306,6 +326,29 @@ def _process_semi_targets(self, y, n_samples): semi_targets[y == 1] = -1 return semi_targets + def _split_validation(self, semi_targets): + """Split the sample indices into the training and the validation + fold, stratified on the semi-supervised targets. ``round(n * + validation_size)`` samples of each target class are held out, + but never all of them, so a lone labeled anomaly always stays in + the training fold. + """ + n_samples = len(semi_targets) + if self.validation_size == 0: + return np.arange(n_samples), np.arange(0) + random_state = check_random_state(self.random_state) + fit_idx, val_idx = [], [] + for target in np.unique(semi_targets): + idx = np.flatnonzero(semi_targets == target) + idx = idx[random_state.permutation(len(idx))] + n_val = min(int(round(len(idx) * self.validation_size)), + len(idx) - 1) + val_idx.append(idx[:n_val]) + fit_idx.append(idx[n_val:]) + fit_idx = np.sort(np.concatenate(fit_idx)) + val_idx = np.sort(np.concatenate(val_idx)) + return fit_idx, val_idx + def _deep_sad_loss(self, outputs, semi_targets): """Deep SAD loss: unlabeled/normal points (target ``0``) minimize the squared distance to the center; labeled anomalies (target @@ -346,12 +389,22 @@ def fit(self, X, y=None): semi_targets = self._process_semi_targets(y, self.n_samples_) + # Hold out the validation split used for best-epoch selection; the + # scaler, the center and the training batches only see the + # remaining samples. + fit_idx, val_idx = self._split_validation(semi_targets) + X_fit, semi_fit = X[fit_idx], semi_targets[fit_idx] + if len(val_idx) > 0: + X_val, semi_val = X[val_idx], semi_targets[val_idx] + else: + X_val, semi_val = None, None + # Standardize data for better performance if self.preprocessing: self.scaler_ = StandardScaler() - X_norm = self.scaler_.fit_transform(X) - else: - X_norm = np.copy(X) + X_fit = self.scaler_.fit_transform(X_fit) + if X_val is not None: + X_val = self.scaler_.transform(X_val) # Build Deep SAD model & fit with X. Unlike Deep SVDD (which # reverses the encoder in an optional autoencoder), Deep SAD uses @@ -362,20 +415,11 @@ def fit(self, X, y=None): dropout_rate=self.dropout_rate, l2_regularizer=self.l2_regularizer) - X_norm = torch.tensor(X_norm, dtype=torch.float32) - semi_targets = torch.tensor(semi_targets, dtype=torch.float32) - - # Hold out the validation split used for best-epoch selection; the - # center and the training batches only see the remaining samples. - n_val = int(self.n_samples_ * self.validation_size) - if n_val > 0: - perm = check_random_state(self.random_state).permutation( - self.n_samples_) - X_val, semi_val = X_norm[perm[:n_val]], semi_targets[perm[:n_val]] - X_fit, semi_fit = X_norm[perm[n_val:]], semi_targets[perm[n_val:]] - else: - X_val, semi_val = None, None - X_fit, semi_fit = X_norm, semi_targets + X_fit = torch.tensor(X_fit, dtype=torch.float32) + semi_fit = torch.tensor(semi_fit, dtype=torch.float32) + if X_val is not None: + X_val = torch.tensor(X_val, dtype=torch.float32) + semi_val = torch.tensor(semi_val, dtype=torch.float32) # Initialize the hypersphere center from a first forward pass, or # validate the user-supplied center against the representation @@ -384,8 +428,8 @@ def fit(self, X, y=None): if self.c is None: center = self.model_._init_c(X_fit) else: - center = torch.as_tensor(self.c, dtype=X_norm.dtype, - device=X_norm.device) + center = torch.as_tensor(self.c, dtype=X_fit.dtype, + device=X_fit.device) if center.ndim == 0: center = center.repeat(rep_dim) elif center.ndim != 1 or center.numel() != rep_dim: @@ -421,7 +465,8 @@ def fit(self, X, y=None): loss = self._deep_sad_loss(self.model_(batch_x), batch_semi) loss.backward() optimizer.step() - epoch_loss += loss.item() + epoch_loss += loss.item() * len(batch_x) + epoch_loss /= len(dataset) if X_val is not None: self.model_.eval() with torch.no_grad(): diff --git a/pyod/test/test_deep_sad.py b/pyod/test/test_deep_sad.py index a08437d2..1b20e65f 100644 --- a/pyod/test/test_deep_sad.py +++ b/pyod/test/test_deep_sad.py @@ -9,6 +9,7 @@ import numpy as np import torch import torch.nn as nn +from torch.utils.data import TensorDataset # noinspection PyProtectedMember from numpy.testing import assert_equal from numpy.testing import assert_raises @@ -215,6 +216,11 @@ def test_eta_must_be_positive(self): with assert_raises(ValueError): DeepSAD(n_features=self.n_features, eta=bad_eta) + def test_eps_must_be_positive_finite(self): + for bad_eps in (0, -1e-6, float('nan'), float('inf')): + with assert_raises(ValueError): + DeepSAD(n_features=self.n_features, eps=bad_eps) + def test_validation_size(self): X, y = self.X_train[:600, :8], self.semi_y[:600] original_init_c = InnerDeepSAD._init_c @@ -319,6 +325,121 @@ def step(self, closure=None): assert not torch.any(snapshot == 100.0), key assert np.all(np.isfinite(clf.decision_scores_)) + def test_validation_split_keeps_lone_labeled_anomaly(self): + # a single labeled anomaly can never be held out, so whatever the + # permutation it must reach the training targets; the 600 unlabeled + # samples are still split in half. + X = self.X_train[:601, :8] + y = np.zeros(601, dtype=int) + y[7] = 1 + for random_state in (0, 1, 2021): + clf = DeepSAD(n_features=8, epochs=1, hidden_neurons=[16, 8], + validation_size=0.5, verbose=0, + random_state=random_state) + with mock.patch('pyod.models.deep_sad.TensorDataset', + wraps=TensorDataset) as dataset: + clf.fit(X, y) + X_fit, semi_fit = dataset.call_args[0] + assert_equal(X_fit.shape, (301, 8)) + assert_equal(int((semi_fit == -1).sum()), 1) + assert_equal(len(clf.decision_scores_), 601) + + def test_validation_split_is_stratified(self): + # 0.2 of the 10 labeled anomalies and 0.2 of the 590 unlabeled + # samples are held out separately: 480 samples with 8 anomalies + # train and 120 samples with 2 anomalies are held out. + X = self.X_train[:600, :8] + y = np.zeros(600, dtype=int) + y[:10] = 1 + original_loss = DeepSAD._deep_sad_loss + semi_val = [] + + def spy_loss(clf, outputs, semi_targets): + if not clf.model_.training: + semi_val.append(semi_targets) + return original_loss(clf, outputs, semi_targets) + + clf = DeepSAD(n_features=8, epochs=1, hidden_neurons=[16, 8], + validation_size=0.2, verbose=0, random_state=2021) + with mock.patch('pyod.models.deep_sad.TensorDataset', + wraps=TensorDataset) as dataset, \ + mock.patch.object(DeepSAD, '_deep_sad_loss', spy_loss): + clf.fit(X, y) + X_fit, semi_fit = dataset.call_args[0] + assert_equal(X_fit.shape, (480, 8)) + assert_equal(int((semi_fit == -1).sum()), 8) + assert_equal(len(semi_val), 1) + assert_equal(len(semi_val[0]), 120) + assert_equal(int((semi_val[0] == -1).sum()), 2) + assert_equal(len(clf.decision_scores_), 600) + + def test_scaler_is_fitted_on_training_split(self): + X = self.X_train[:600, :8] + original_split = DeepSAD._split_validation + folds = [] + + def spy_split(clf, semi_targets): + fit_idx, val_idx = original_split(clf, semi_targets) + folds.append((fit_idx, val_idx)) + return fit_idx, val_idx + + for validation_size, n_fit in ((0.2, 480), (0, 600)): + clf = DeepSAD(n_features=8, epochs=1, hidden_neurons=[16, 8], + validation_size=validation_size, verbose=0, + random_state=2021) + with mock.patch.object(DeepSAD, '_split_validation', spy_split): + clf.fit(X) + fit_idx, val_idx = folds[-1] + assert_equal(len(fit_idx), n_fit) + assert_equal(np.sort(np.concatenate([fit_idx, val_idx])), + np.arange(600)) + assert_equal(clf.scaler_.n_samples_seen_, n_fit) + assert np.allclose(clf.scaler_.mean_, X[fit_idx].mean(axis=0)) + assert np.allclose(clf.scaler_.var_, X[fit_idx].var(axis=0)) + + def test_epoch_loss_is_mean_per_sample(self): + # 600 samples in batches of 256 leave an undersized last batch of + # 88; the reported epoch loss weights every batch by its size. + X = self.X_train[:600, :8] + original_loss = DeepSAD._deep_sad_loss + batches = [] + + def spy_loss(clf, outputs, semi_targets): + loss = original_loss(clf, outputs, semi_targets) + batches.append((loss.item(), len(semi_targets))) + return loss + + clf = DeepSAD(n_features=8, epochs=1, hidden_neurons=[16, 8], + batch_size=256, validation_size=0, verbose=1, + random_state=2021) + with mock.patch.object(DeepSAD, '_deep_sad_loss', spy_loss), \ + mock.patch('builtins.print') as mock_print: + clf.fit(X) + assert_equal([n for _, n in batches], [256, 256, 88]) + expected = sum(loss * n for loss, n in batches) / 600 + message = mock_print.call_args[0][0] + self.assertAlmostEqual(float(message.split('Loss: ')[1]), expected, + places=5) + + def test_single_hidden_layer(self): + # hidden_neurons=[k] is one bias-free linear layer straight into + # the representation space: no extra k -> k layer, activation or + # dropout. + X = self.X_train[:600, :8] + clf = DeepSAD(n_features=8, epochs=2, hidden_neurons=[16], verbose=0, + contamination=self.contamination) + clf.fit(X) + modules = list(clf.model_.model) + assert_equal(len(modules), 1) + assert isinstance(modules[0], nn.Linear) + assert_equal((modules[0].in_features, modules[0].out_features), + (8, 16)) + assert modules[0].bias is None + assert_equal(tuple(clf.c_.shape), (16,)) + scores = clf.decision_function(self.X_test[:, :8]) + assert_equal(scores.shape[0], self.X_test.shape[0]) + assert np.all(np.isfinite(scores)) + def tearDown(self): pass From ab8c2338369506bb028993fbf2790f3cb52ab058 Mon Sep 17 00:00:00 2001 From: Jayesh Suryavanshi Date: Tue, 22 Sep 2026 00:29:57 -0700 Subject: [PATCH 6/8] Update the author email in the DeepSAD header comments --- examples/deep_sad_example.py | 2 +- pyod/models/deep_sad.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/deep_sad_example.py b/examples/deep_sad_example.py index 64231ba6..59843b5e 100644 --- a/examples/deep_sad_example.py +++ b/examples/deep_sad_example.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- """Example of using Deep SAD for semi-supervised outlier detection """ -# Author: Jayesh Suryavanshi +# Author: Jayesh Suryavanshi # License: BSD 2 clause from __future__ import division diff --git a/pyod/models/deep_sad.py b/pyod/models/deep_sad.py index 20464ba5..b0eb9c65 100644 --- a/pyod/models/deep_sad.py +++ b/pyod/models/deep_sad.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- """Deep Semi-Supervised Anomaly Detection (Deep SAD) for outlier detection """ -# Author: Jayesh Suryavanshi +# Author: Jayesh Suryavanshi # License: BSD 2 clause # # The Deep SAD objective and the hypersphere-center initialization are From 2e362c3ebee277d88e06db6aaf84f0bdeb8c7f5c Mon Sep 17 00:00:00 2001 From: Jayesh Suryavanshi Date: Thu, 15 Oct 2026 15:42:42 -0700 Subject: [PATCH 7/8] Check fitted state in DeepSAD.decision_function and seed at fit time decision_function now raises NotFittedError on an unfitted estimator instead of an AttributeError. The torch seed moves from __init__ to the start of fit, so building other estimators before fitting no longer changes the result. Scores for an estimator built and fitted right away are unchanged. --- pyod/models/deep_sad.py | 26 +++++++++++++++++--------- pyod/test/test_deep_sad.py | 23 +++++++++++++++++++++++ 2 files changed, 40 insertions(+), 9 deletions(-) diff --git a/pyod/models/deep_sad.py b/pyod/models/deep_sad.py index b0eb9c65..5352f7ff 100644 --- a/pyod/models/deep_sad.py +++ b/pyod/models/deep_sad.py @@ -27,6 +27,7 @@ from sklearn.preprocessing import StandardScaler from sklearn.utils import check_array, check_random_state +from sklearn.utils.validation import check_is_fitted from torch.utils.data import DataLoader, TensorDataset from .base import BaseDetector @@ -229,7 +230,9 @@ class DeepSAD(BaseDetector): number generator; If RandomState instance, an integer seed is drawn from it (advancing it) to seed torch; If None, the random number generator is the RandomState instance used by - `np.random`. + `np.random`. The seed is applied when ``fit`` starts, so + building other estimators beforehand does not change the + result. contamination : float in (0., 0.5), optional (default=0.1) The amount of contamination of the data set, i.e. @@ -287,14 +290,6 @@ def __init__(self, n_features, c=None, eta=1.0, hidden_neurons=None, self.model_ = None self.best_model_dict = None - if self.random_state is not None: - if isinstance(self.random_state, (int, np.integer)): - torch_seed = self.random_state - else: - torch_seed = int( - check_random_state(self.random_state).randint( - 0, np.iinfo(np.int32).max)) - torch.manual_seed(torch_seed) check_parameter(dropout_rate, 0, 1, param_name='dropout_rate', include_left=True) check_parameter(eta, 0, np.inf, param_name='eta') @@ -381,6 +376,18 @@ def fit(self, X, y=None): """ # validate inputs X and y (optional) X = check_array(X) + + # Seed torch here rather than in __init__, so that building other + # estimators before this fit cannot change its result. + if self.random_state is not None: + if isinstance(self.random_state, (int, np.integer)): + torch_seed = self.random_state + else: + torch_seed = int( + check_random_state(self.random_state).randint( + 0, np.iinfo(np.int32).max)) + torch.manual_seed(torch_seed) + # Deep SAD is semi-supervised, so y carries partial supervision and # is expected; treat this as a binary anomaly-detection problem. self._classes = 2 @@ -509,6 +516,7 @@ def decision_function(self, X): anomaly_scores : numpy array of shape (n_samples,) The anomaly score of the input samples. """ + check_is_fitted(self, ['c_']) X = check_array(X) if self.preprocessing: diff --git a/pyod/test/test_deep_sad.py b/pyod/test/test_deep_sad.py index 1b20e65f..5405e6fb 100644 --- a/pyod/test/test_deep_sad.py +++ b/pyod/test/test_deep_sad.py @@ -14,6 +14,7 @@ from numpy.testing import assert_equal from numpy.testing import assert_raises from sklearn.base import clone +from sklearn.exceptions import NotFittedError from sklearn.metrics import roc_auc_score # temporary solution for relative imports in case pyod is not installed @@ -440,6 +441,28 @@ def test_single_hidden_layer(self): assert_equal(scores.shape[0], self.X_test.shape[0]) assert np.all(np.isfinite(scores)) + def test_decision_function_requires_fit(self): + clf = DeepSAD(n_features=8, hidden_neurons=[16, 8], verbose=0) + with assert_raises(NotFittedError): + clf.decision_function(self.X_test[:, :8]) + + def test_seed_is_applied_at_fit_time(self): + # Building a second seeded estimator before the first one is + # fitted must not change either result. + X = self.X_train[:600, :8] + + def build(seed): + return DeepSAD(n_features=8, epochs=2, hidden_neurons=[16, 8], + verbose=0, random_state=seed) + + first, second = build(0), build(1) + first.fit(X) + second.fit(X) + assert np.array_equal(first.decision_scores_, + build(0).fit(X).decision_scores_) + assert np.array_equal(second.decision_scores_, + build(1).fit(X).decision_scores_) + def tearDown(self): pass From aa6da4e997d1a362af173039eb40059883e7ced2 Mon Sep 17 00:00:00 2001 From: Jayesh Suryavanshi Date: Sat, 17 Oct 2026 14:41:04 -0700 Subject: [PATCH 8/8] Register DeepSAD in the knowledge base as experimental so ADEngine does not route it ADEngine passes no labels, so Deep SAD would run there as plain Deep SVDD. As an experimental entry it is listed in the knowledge base and by explain_detector, but rule routing, the routing context for LLM agents, and make_plan all skip it. Detector counts move to 62 (44 tabular) and the tabular skill reference is regenerated. --- README.rst | 2 +- docs/examples/tabular.rst | 2 +- docs/index.rst | 2 +- docs/install.rst | 2 +- examples/agentic_example.py | 2 +- pyod/skills/od_expert/SKILL.md | 2 +- pyod/skills/od_expert/references/tabular.md | 3 ++- pyod/test/test_ad_engine.py | 12 ++++++++++ pyod/utils/knowledge/algorithms.json | 25 +++++++++++++++++++++ 9 files changed, 45 insertions(+), 7 deletions(-) diff --git a/README.rst b/README.rst index 90885710..36972183 100644 --- a/README.rst +++ b/README.rst @@ -71,7 +71,7 @@ PyOD 3 is the most comprehensive Python library for anomaly detection. Four pill =========================== ======================================================================================== Pillar What it means =========================== ======================================================================================== -Multi-Modal 61 detectors across **tabular, time series, graph, text, image, and audio** data, one API +Multi-Modal 62 detectors across **tabular, time series, graph, text, image, and audio** data, one API Full Lifecycle From raw data to explained anomalies and next-step guidance in a single call Agentic ``od-expert`` turns natural-language requests into ADEngine workflows; MCP exposes structured tools for other agents Most Used 46+ million downloads; benchmark-backed routing (ADBench, TSB-AD, BOND, NLP-ADBench) diff --git a/docs/examples/tabular.rst b/docs/examples/tabular.rst index 8ed7e770..5860ff45 100644 --- a/docs/examples/tabular.rst +++ b/docs/examples/tabular.rst @@ -1,7 +1,7 @@ Layer 1: Tabular Anomaly Detection ==================================== -PyOD has 43 tabular detectors covering probabilistic, linear, proximity, ensemble, and deep learning approaches. All use the same ``fit``/``predict``/``decision_function`` API. +PyOD has 44 tabular detectors covering probabilistic, linear, proximity, ensemble, and deep learning approaches. All use the same ``fit``/``predict``/``decision_function`` API. .. code-block:: python diff --git a/docs/index.rst b/docs/index.rst index c49c8938..cd12717b 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -71,7 +71,7 @@ PyOD 3 is the most comprehensive Python library for anomaly detection. Four pill =========================== ======================================================================================== Pillar What it means =========================== ======================================================================================== -Multi-Modal 61 detectors across **tabular, time series, graph, text, image, and audio** data, one API +Multi-Modal 62 detectors across **tabular, time series, graph, text, image, and audio** data, one API Full Lifecycle From raw data to explained anomalies and next-step guidance in a single call Agentic Ask in plain English, and AI agents run the full detection workflow without OD expertise Most Used `46+ million downloads `_; benchmark-backed routing (ADBench, TSB-AD, BOND, NLP-ADBench) diff --git a/docs/install.rst b/docs/install.rst index 8e1dad0d..d2d9b81f 100644 --- a/docs/install.rst +++ b/docs/install.rst @@ -117,7 +117,7 @@ Example output: .. code-block:: text PyOD version: 3.1.0 - Detectors (ADEngine): 61 total (43 tabular, 7 time-series, 8 graph, 2 text, 2 image, 1 multimodal, 3 audio) + Detectors (ADEngine): 62 total (44 tabular, 7 time-series, 8 graph, 2 text, 2 image, 1 multimodal, 3 audio) Classic API: OK ADEngine (Layer 2): OK MCP extra: OK (run: pyod mcp serve) diff --git a/examples/agentic_example.py b/examples/agentic_example.py index 0527d388..ec291682 100644 --- a/examples/agentic_example.py +++ b/examples/agentic_example.py @@ -2,7 +2,7 @@ """Example: Agent-driven anomaly detection with PyOD 3 (Layer 3). Demonstrates what makes PyOD distinctive in the agentic workflow: - - 61 detectors across 6 data modalities + - 62 detectors across 6 data modalities - Benchmark-backed detector selection (ADBench, TSB-AD, BOND) - Multi-detector consensus with per-detector scores - Result quality assessment diff --git a/pyod/skills/od_expert/SKILL.md b/pyod/skills/od_expert/SKILL.md index 32a93b1d..8fa95492 100644 --- a/pyod/skills/od_expert/SKILL.md +++ b/pyod/skills/od_expert/SKILL.md @@ -19,7 +19,7 @@ Fire this skill when: ## What you have access to -PyOD ships 61 detectors across six modalities (43 tabular, 7 time series, 8 graph, 2 text, 2 image, 1 multimodal, 3 audio). Use the `ADEngine` session API to drive the full workflow: +PyOD ships 62 detectors across six modalities (44 tabular, 7 time series, 8 graph, 2 text, 2 image, 1 multimodal, 3 audio). Use the `ADEngine` session API to drive the full workflow: ```python from pyod.utils.ad_engine import ADEngine diff --git a/pyod/skills/od_expert/references/tabular.md b/pyod/skills/od_expert/references/tabular.md index 52948fff..e6186360 100644 --- a/pyod/skills/od_expert/references/tabular.md +++ b/pyod/skills/od_expert/references/tabular.md @@ -1,6 +1,6 @@ # Tabular anomaly detection reference -PyOD's largest modality (43 of 61 buildable detectors). The agent loads this file when the master decision tree (in SKILL.md) routes to tabular. +PyOD's largest modality (44 of 62 buildable detectors). The agent loads this file when the master decision tree (in SKILL.md) routes to tabular. ## Decision table by data shape (expert heuristics) @@ -29,6 +29,7 @@ These are rules of thumb for reasoning about which detectors a non-expert would - **COF** (Connectivity-Based Outlier Factor) — complexity: time O(n^2 * d), space O(n^2); best for: Datasets where outliers lie along sparse connectivity paths between clusters; avoid when: Dataset is large or simpler methods like LOF already perform well; paper: Tang et al., 2002 - **COPOD** (Copula-Based Outlier Detection) — complexity: time O(n * d * log(n)), space O(n * d); best for: Large-scale datasets where speed and interpretability matter and features are roughly independent; avoid when: Outliers only appear in joint distributions with strong feature dependencies; paper: Li et al., ICDM 2020 - **DIF** (Deep Isolation Forest) — complexity: time O(n * t * d * log(n)), space O(t * n); best for: Complex datasets where standard Isolation Forest misses anomalies due to axis-aligned splits; avoid when: Standard Isolation Forest performs well or dataset is small and simple; paper: Xu et al., TKDE 2023 +- **DeepSAD** (Deep Semi-Supervised Anomaly Detection) — complexity: time O(n * d * h * epochs), space O(d * h); best for: Semi-supervised detection where a few labeled anomalies are available; avoid when: No labeled anomalies are available; ADEngine passes no labels, so it does not route this detector; requires: pyod[torch]; paper: Ruff et al., ICLR 2020 - **DeepSVDD** (Deep Support Vector Data Description) — complexity: time O(n * d * h * epochs), space O(d * h); best for: One-class anomaly detection where a compact normal data description is desired; avoid when: Normal data is multi-modal or simpler one-class methods are sufficient; requires: pyod[torch]; paper: Ruff et al., ICML 2018 - **DevNet** (Deep Anomaly Detection with Deviation Networks) — complexity: time O(n * d * h * epochs), space O(d * h); best for: Semi-supervised anomaly detection where a small number of labeled anomalies are available; avoid when: No labeled anomalies are available or dataset is too small for deep learning; requires: pyod[torch]; paper: Pang et al., KDD 2019 - **ECOD** (Empirical Cumulative Distribution Functions) — complexity: time O(n * d * log(n)), space O(n * d); best for: General-purpose outlier detection when speed and interpretability are priorities; avoid when: Features are heavily correlated and outliers only manifest in joint distributions; paper: Li et al., TKDE 2022 diff --git a/pyod/test/test_ad_engine.py b/pyod/test/test_ad_engine.py index 7de7d794..4456b5e0 100644 --- a/pyod/test/test_ad_engine.py +++ b/pyod/test/test_ad_engine.py @@ -306,6 +306,18 @@ def test_get_benchmarks(self): benchmarks = self.engine.get_benchmarks() assert 'ADBench' in benchmarks + def test_deep_sad_is_listed_but_not_routed(self): + # DeepSAD needs labeled anomalies and ADEngine passes no labels, + # so it is listed as experimental and no routing path selects it. + info = self.engine.explain_detector('DeepSAD') + assert info['status'] == 'experimental' + profile = self.engine.profile_data(np.random.randn(500, 10)) + routing = self.engine.get_kb_for_routing(profile) + names = [d['name'] for d in routing['available_detectors']] + assert 'DeepSAD' not in names + with self.assertRaises(ValueError): + self.engine.make_plan(['DeepSAD']) + class TestRunDetection(unittest.TestCase): def setUp(self): diff --git a/pyod/utils/knowledge/algorithms.json b/pyod/utils/knowledge/algorithms.json index a981bd33..b6a92aee 100644 --- a/pyod/utils/knowledge/algorithms.json +++ b/pyod/utils/knowledge/algorithms.json @@ -990,6 +990,31 @@ "requires": ["torch"], "version_added": "0.7.5" }, + "DeepSAD": { + "class_path": "pyod.models.deep_sad.DeepSAD", + "full_name": "Deep Semi-Supervised Anomaly Detection", + "status": "experimental", + "data_types": ["tabular"], + "category": "deep_learning", + "complexity": {"time": "O(n * d * h * epochs)", "space": "O(d * h)"}, + "strengths": [ + "Learns from a few labeled anomalies by pushing them away from the hypersphere center", + "Works with partial labels and falls back to Deep SVDD when no labels are given" + ], + "weaknesses": [ + "Needs labeled anomalies to improve on Deep SVDD", + "Sensitive to network architecture choices and the eta weight" + ], + "best_for": "Semi-supervised detection where a few labeled anomalies are available", + "avoid_when": "No labeled anomalies are available; ADEngine passes no labels, so it does not route this detector", + "benchmark_refs": [], + "benchmark_rank": {}, + "paper": {"id": "deep_sad", "short": "Ruff et al., ICLR 2020"}, + "default_params": {"contamination": 0.1}, + "preprocessing_mode": "external", + "requires": ["torch"], + "version_added": "unreleased" + }, "AnoGAN": { "class_path": "pyod.models.anogan.AnoGAN", "full_name": "Anomaly Detection with Generative Adversarial Networks",