From 5e19d4ae2705051a27b937552f92a41f54854b38 Mon Sep 17 00:00:00 2001 From: abigailt Date: Thu, 7 Mar 2024 19:00:09 +0200 Subject: [PATCH] New model output types + single implementation of score method that supports multiple output types. Existing tests pass. Still need more tests for new types. Signed-off-by: abigailt --- apt/minimization/minimizer.py | 3 +- apt/utils/datasets/__init__.py | 2 +- apt/utils/datasets/datasets.py | 2 + apt/utils/models/__init__.py | 2 +- apt/utils/models/keras_model.py | 6 +- apt/utils/models/model.py | 164 +++++++++++++++++++++--------- apt/utils/models/pytorch_model.py | 109 +++++++++++++++----- apt/utils/models/sklearn_model.py | 8 +- tests/test_minimizer.py | 48 +++++---- tests/test_model.py | 47 +++++---- tests/test_pytorch.py | 38 ++++--- 11 files changed, 285 insertions(+), 144 deletions(-) diff --git a/apt/minimization/minimizer.py b/apt/minimization/minimizer.py index 4b9e657..e934085 100644 --- a/apt/minimization/minimizer.py +++ b/apt/minimization/minimizer.py @@ -93,7 +93,8 @@ class GeneralizeToRepresentative(BaseEstimator, MetaEstimatorMixin, TransformerM if is_regression: self.estimator = SklearnRegressor(estimator) else: - self.estimator = SklearnClassifier(estimator, ModelOutputType.CLASSIFIER_PROBABILITIES) + self.estimator = SklearnClassifier(estimator, + ModelOutputType.CLASSIFIER_SINGLE_OUTPUT_CLASS_PROBABILITIES) self.target_accuracy = target_accuracy self.cells = cells self.categorical_features = [] diff --git a/apt/utils/datasets/__init__.py b/apt/utils/datasets/__init__.py index 6252130..81f7d19 100644 --- a/apt/utils/datasets/__init__.py +++ b/apt/utils/datasets/__init__.py @@ -4,4 +4,4 @@ Implementation of datasets utility components for datasets creation, load, and s """ from apt.utils.datasets.datasets import Dataset, StoredDataset, DatasetFactory, Data, ArrayDataset, \ - DatasetWithPredictions, OUTPUT_DATA_ARRAY_TYPE, DATA_PANDAS_NUMPY_TYPE + DatasetWithPredictions, array2numpy, OUTPUT_DATA_ARRAY_TYPE, DATA_PANDAS_NUMPY_TYPE diff --git a/apt/utils/datasets/datasets.py b/apt/utils/datasets/datasets.py index b3278f4..cafdd45 100644 --- a/apt/utils/datasets/datasets.py +++ b/apt/utils/datasets/datasets.py @@ -266,6 +266,8 @@ class DatasetWithPredictions(Dataset): Dataset that is based on arrays (e.g., numpy/pandas/list...). Includes predictions from a model, and possibly also features and true labels. + :param pred: collection of model predictions + :type pred: numpy array or pandas DataFrame or list or pytorch Tensor :param x: collection of data samples :type x: numpy array or pandas DataFrame or list or pytorch Tensor :param y: collection of labels diff --git a/apt/utils/models/__init__.py b/apt/utils/models/__init__.py index 202ec90..9c1e40b 100644 --- a/apt/utils/models/__init__.py +++ b/apt/utils/models/__init__.py @@ -1,6 +1,6 @@ from apt.utils.models.model import Model, BlackboxClassifier, ModelOutputType, ScoringMethod, \ BlackboxClassifierPredictions, BlackboxClassifierPredictFunction, get_nb_classes, is_one_hot, \ - check_correct_model_output, is_multi_label, is_multi_label_binary + check_correct_model_output, is_multi_label, is_multi_label_binary, is_logits from apt.utils.models.sklearn_model import SklearnModel, SklearnClassifier, SklearnRegressor from apt.utils.models.keras_model import KerasClassifier, KerasRegressor from apt.utils.models.xgboost_model import XGBoostClassifier diff --git a/apt/utils/models/keras_model.py b/apt/utils/models/keras_model.py index 6f89a35..42d341e 100644 --- a/apt/utils/models/keras_model.py +++ b/apt/utils/models/keras_model.py @@ -4,7 +4,7 @@ import numpy as np from sklearn.metrics import mean_squared_error -from apt.utils.models import Model, ModelOutputType, ScoringMethod, check_correct_model_output +from apt.utils.models import Model, ModelOutputType, ScoringMethod, check_correct_model_output, is_logits from apt.utils.datasets import Dataset, OUTPUT_DATA_ARRAY_TYPE from art.utils import check_and_transform_label_format @@ -39,9 +39,7 @@ class KerasClassifier(KerasModel): def __init__(self, model: "keras.models.Model", output_type: ModelOutputType, black_box_access: Optional[bool] = True, unlimited_queries: Optional[bool] = True, **kwargs): super().__init__(model, output_type, black_box_access, unlimited_queries, **kwargs) - logits = False - if output_type == ModelOutputType.CLASSIFIER_LOGITS: - logits = True + logits = is_logits(output_type) self._art_model = ArtKerasClassifier(model, use_logits=logits) def fit(self, train_data: Dataset, **kwargs) -> None: diff --git a/apt/utils/models/model.py b/apt/utils/models/model.py index f470de7..6d75fef 100644 --- a/apt/utils/models/model.py +++ b/apt/utils/models/model.py @@ -2,8 +2,9 @@ from abc import ABCMeta, abstractmethod from typing import Any, Optional, Callable, Tuple, Union, TYPE_CHECKING from enum import Enum, auto import numpy as np +from scipy.special import expit -from apt.utils.datasets import Dataset, Data, OUTPUT_DATA_ARRAY_TYPE +from apt.utils.datasets import Dataset, Data, DatasetWithPredictions, array2numpy, OUTPUT_DATA_ARRAY_TYPE from art.estimators.classification import BlackBoxClassifier from art.utils import check_and_transform_label_format @@ -12,9 +13,16 @@ if TYPE_CHECKING: class ModelOutputType(Enum): - CLASSIFIER_PROBABILITIES = auto() # vector of probabilities - CLASSIFIER_LOGITS = auto() # vector of logits - CLASSIFIER_SCALAR = auto() # label only + CLASSIFIER_SINGLE_OUTPUT_CATEGORICAL = auto() # class labels + CLASSIFIER_SINGLE_OUTPUT_BINARY_PROBABILITIES = auto() # single binary probability + CLASSIFIER_SINGLE_OUTPUT_CLASS_PROBABILITIES = auto() # vector of class probabilities + CLASSIFIER_SINGLE_OUTPUT_BINARY_LOGITS = auto() # single binary logit + CLASSIFIER_SINGLE_OUTPUT_CLASS_LOGITS = auto() # vector of logits + CLASSIFIER_MULTI_OUTPUT_CATEGORICAL = auto() # vector of class labels + CLASSIFIER_MULTI_OUTPUT_BINARY_PROBABILITIES = auto() # vector of binary probabilities, 1 per output + CLASSIFIER_MULTI_OUTPUT_CLASS_PROBABILITIES = auto() # vector of class probabilities for multiple outputs + CLASSIFIER_MULTI_OUTPUT_BINARY_LOGITS = auto() # vector of binary logits + CLASSIFIER_MULTI_OUTPUT_CLASS_LOGITS = auto() # vector of logits for multiple outputs REGRESSOR_SCALAR = auto() # value @@ -32,15 +40,44 @@ def is_one_hot(y: OUTPUT_DATA_ARRAY_TYPE) -> bool: return len(y.shape) == 2 and y.shape[1] > 1 and np.all(np.around(np.sum(y, axis=1), decimals=4) == 1) -def is_multi_label(y: OUTPUT_DATA_ARRAY_TYPE) -> bool: - return len(y.shape) == 2 and y.shape[1] > 1 and not is_one_hot(y) +def is_multi_label(output_type: ModelOutputType) -> bool: + return (output_type == ModelOutputType.CLASSIFIER_MULTI_OUTPUT_CATEGORICAL or + output_type == ModelOutputType.CLASSIFIER_MULTI_OUTPUT_BINARY_PROBABILITIES or + output_type == ModelOutputType.CLASSIFIER_MULTI_OUTPUT_CLASS_PROBABILITIES or + output_type == ModelOutputType.CLASSIFIER_MULTI_OUTPUT_BINARY_LOGITS or + output_type == ModelOutputType.CLASSIFIER_MULTI_OUTPUT_CLASS_LOGITS) -def is_multi_label_binary(y: OUTPUT_DATA_ARRAY_TYPE) -> bool: - return is_multi_label(y) and np.max(y) <= 1 +def is_multi_label_binary(output_type: ModelOutputType) -> bool: + return (output_type == ModelOutputType.CLASSIFIER_MULTI_OUTPUT_BINARY_PROBABILITIES or + output_type == ModelOutputType.CLASSIFIER_MULTI_OUTPUT_BINARY_LOGITS) -def get_nb_classes(y: OUTPUT_DATA_ARRAY_TYPE) -> int: +def is_binary(output_type: ModelOutputType) -> bool: + return (output_type == ModelOutputType.CLASSIFIER_MULTI_OUTPUT_BINARY_PROBABILITIES or + output_type == ModelOutputType.CLASSIFIER_MULTI_OUTPUT_BINARY_LOGITS or + output_type == ModelOutputType.CLASSIFIER_SINGLE_OUTPUT_BINARY_PROBABILITIES or + output_type == ModelOutputType.CLASSIFIER_SINGLE_OUTPUT_BINARY_LOGITS) + + +def is_categorical(output_type: ModelOutputType) -> bool: + return (output_type == ModelOutputType.CLASSIFIER_SINGLE_OUTPUT_CATEGORICAL or + output_type == ModelOutputType.CLASSIFIER_MULTI_OUTPUT_CATEGORICAL) + + +def is_probabilities(output_type: ModelOutputType) -> bool: + return (output_type == ModelOutputType.CLASSIFIER_SINGLE_OUTPUT_CLASS_PROBABILITIES or + output_type == ModelOutputType.CLASSIFIER_MULTI_OUTPUT_CLASS_PROBABILITIES) + + +def is_logits(output_type: ModelOutputType) -> bool: + return (output_type == ModelOutputType.CLASSIFIER_SINGLE_OUTPUT_CLASS_LOGITS or + output_type == ModelOutputType.CLASSIFIER_MULTI_OUTPUT_CLASS_LOGITS or + output_type == ModelOutputType.CLASSIFIER_SINGLE_OUTPUT_BINARY_LOGITS or + output_type == ModelOutputType.CLASSIFIER_MULTI_OUTPUT_BINARY_LOGITS) + + +def get_nb_classes(y: OUTPUT_DATA_ARRAY_TYPE, output_type: ModelOutputType) -> int: """ Get the number of classes from an array of labels @@ -56,12 +93,14 @@ def get_nb_classes(y: OUTPUT_DATA_ARRAY_TYPE) -> int: if is_one_hot(y): return y.shape[1] - elif is_multi_label(y): + elif is_multi_label(output_type): # for now just return the number of labels return y.shape[1] # return [int(np.max(y.T[i]) + 1) for i in range(y.shape[1])] - else: + elif is_categorical(output_type): return int(np.max(y) + 1) + else: # binary + return 2 def check_correct_model_output(y: OUTPUT_DATA_ARRAY_TYPE, output_type: ModelOutputType): @@ -73,8 +112,8 @@ def check_correct_model_output(y: OUTPUT_DATA_ARRAY_TYPE, output_type: ModelOutp :type output_type: ModelOutputType :raises: ValueError (in case of mismatch) """ - if not is_one_hot(y) and not is_multi_label(y): # 1D array - if output_type == ModelOutputType.CLASSIFIER_PROBABILITIES or output_type == ModelOutputType.CLASSIFIER_LOGITS: + if not is_one_hot(y) and (output_type == ModelOutputType.CLASSIFIER_SINGLE_OUTPUT_CLASS_PROBABILITIES or + output_type == ModelOutputType.CLASSIFIER_SINGLE_OUTPUT_CLASS_LOGITS): raise ValueError("Incompatible model output types. Model outputs 1D array of categorical scalars while " "output type is set to ", output_type) @@ -127,16 +166,65 @@ class Model(metaclass=ABCMeta): """ raise NotImplementedError - @abstractmethod def score(self, test_data: Dataset, **kwargs): """ Score the model using test data. :param test_data: Test data. - :type train_data: `Dataset` + :type test_data: `Dataset` + :param predictions: Model predictions to score. If provided, these will be used instead of calling the model's + `predict` method. + :type predictions: `DatasetWithPredictions` with the `pred` field filled. + :param scoring_method: The method for scoring predictions. Default is ACCURACY. + :type scoring_method: `ScoringMethod`, optional + :param binary_threshold: The threshold to use on binary classification probabilities to assign the positive + class. + :type binary_threshold: float, optional. Default is 0.5. + :param apply_non_linearity: A non-linear function to apply to the result of the 'predict' method, in case the + model outputs logits (e.g., sigmoid). + :type apply_non_linearity: Callable, should be possible to apply directly to the numpy output of the 'predict' + method, optional. + :param nb_classes: number of classes (for classification models). + :type nb_classes: int, optional. :return: the score as float (for classifiers, between 0 and 1) """ - raise NotImplementedError + predictions = kwargs['predictions'] if 'predictions' in kwargs else None + nb_classes = kwargs['nb_classes'] if 'nb_classes' in kwargs else None + scoring_method = kwargs['scoring_method'] if 'scoring_method' in kwargs else ScoringMethod.ACCURACY + binary_threshold = kwargs['binary_threshold'] if 'binary_threshold' in kwargs else 0.5 + apply_non_linearity = kwargs['apply_non_linearity'] if 'apply_non_linearity' in kwargs else expit + + if test_data.get_samples() is None and predictions is None: + raise ValueError('score can only be computed when test data or predictions are available') + if test_data.get_labels() is None: + raise ValueError('score can only be computed when labels are available') + if predictions: + predicted = predictions.get_predictions() + else: + predicted = self.predict(test_data) + y = array2numpy(test_data.get_labels()) + + if scoring_method == ScoringMethod.ACCURACY: + if not is_multi_label(self.output_type) and not is_binary(self.output_type) and nb_classes is not None: + y = check_and_transform_label_format(y, nb_classes=nb_classes) + if (self.output_type == ModelOutputType.CLASSIFIER_SINGLE_OUTPUT_CLASS_PROBABILITIES or + self.output_type == ModelOutputType.CLASSIFIER_SINGLE_OUTPUT_CLASS_LOGITS or + self.output_type == ModelOutputType.CLASSIFIER_SINGLE_OUTPUT_CATEGORICAL): + # categorical has been 1-hot encoded by check_and_transform_label_format + return np.count_nonzero(np.argmax(y, axis=1) == np.argmax(predicted, axis=1)) / predicted.shape[0] + elif is_binary(self.output_type): + if is_logits(self.output_type): + if apply_non_linearity: + predicted = apply_non_linearity(predicted) + else: # apply sigmoid + predicted = expit(predicted) + predicted[predicted < binary_threshold] = 0 + predicted[predicted >= binary_threshold] = 1 + return np.count_nonzero(y == predicted) / (predicted.shape[0] * y.shape[1]) + else: + raise NotImplementedError('score method not implemented for output type: ', self.output_type) + else: + raise NotImplementedError('scoring method not implemented: ', scoring_method) @property def model(self) -> Any: @@ -260,6 +348,13 @@ class BlackboxClassifier(Model): """ return self._optimizer + def score(self, test_data: Dataset, **kwargs): + """ + Score the model using test data. + """ + kwargs ['nb_classes'] = self.nb_classes + return super().score(test_data, **kwargs) + def fit(self, train_data: Dataset, **kwargs) -> None: """ A blackbox model cannot be fit. @@ -279,37 +374,6 @@ class BlackboxClassifier(Model): check_correct_model_output(predictions, self.output_type) return predictions - def score(self, test_data: Dataset, scoring_method: Optional[ScoringMethod] = ScoringMethod.ACCURACY, - binary_threshold: Optional[float] = 0.5, **kwargs): - """ - Score the model using test data. - - :param test_data: Test data. - :type train_data: `Dataset` - :param scoring_method: The method for scoring predictions. Default is ACCURACY. - :type scoring_method: `ScoringMethod`, optional - :param binary_threshold: The threshold to use on binary classification probabilities to assign the positive - class. - :type binary_threshold: float, optional. Default is 0.5. - :return: the score as float (for classifiers, between 0 and 1) - """ - if test_data.get_samples() is None or test_data.get_labels() is None: - raise ValueError('score can only be computed when test data and labels are available') - predicted = self._art_model.predict(test_data.get_samples()) - y = test_data.get_labels() - if not is_multi_label(y): - y = check_and_transform_label_format(y, nb_classes=self._nb_classes) - if scoring_method == ScoringMethod.ACCURACY: - if not is_multi_label(y): - return np.count_nonzero(np.argmax(y, axis=1) == np.argmax(predicted, axis=1)) / predicted.shape[0] - else: - if is_multi_label_binary(y): - predicted[predicted < binary_threshold] = 0 - predicted[predicted >= binary_threshold] = 1 - return np.count_nonzero(y == predicted) / (predicted.shape[0] * y.shape[1]) - else: - raise NotImplementedError - @abstractmethod def get_predictions(self) -> Union[Callable, Tuple[OUTPUT_DATA_ARRAY_TYPE, OUTPUT_DATA_ARRAY_TYPE]]: """ @@ -356,11 +420,11 @@ class BlackboxClassifierPredictions(BlackboxClassifier): check_correct_model_output(y_test_pred, self.output_type) if y_train_pred is not None and len(y_train_pred.shape) == 1: - self._nb_classes = get_nb_classes(y_train_pred) + self._nb_classes = get_nb_classes(y_train_pred, self.output_type) y_train_pred = check_and_transform_label_format(y_train_pred, nb_classes=self._nb_classes) if y_test_pred is not None and len(y_test_pred.shape) == 1: if self._nb_classes is None: - self._nb_classes = get_nb_classes(y_test_pred) + self._nb_classes = get_nb_classes(y_test_pred, self.output_type) y_test_pred = check_and_transform_label_format(y_test_pred, nb_classes=self._nb_classes) if x_train_pred is not None and y_train_pred is not None and x_test_pred is not None and y_test_pred is not None: @@ -378,7 +442,7 @@ class BlackboxClassifierPredictions(BlackboxClassifier): else: raise NotImplementedError("Invalid data - None") - self._nb_classes = get_nb_classes(y_pred) + self._nb_classes = get_nb_classes(y_pred, self.output_type) self._input_shape = x_pred.shape[1:] self._x_pred = x_pred self._y_pred = y_pred diff --git a/apt/utils/models/pytorch_model.py b/apt/utils/models/pytorch_model.py index a7db4a9..2633af3 100644 --- a/apt/utils/models/pytorch_model.py +++ b/apt/utils/models/pytorch_model.py @@ -7,9 +7,10 @@ from typing import Optional, Tuple, Union, List, Callable import numpy as np import torch from torch.utils.data import DataLoader, TensorDataset +from collections.abc import Iterable from art.utils import check_and_transform_label_format -from apt.utils.datasets.datasets import PytorchData +from apt.utils.datasets.datasets import PytorchData, DatasetWithPredictions, ArrayDataset from apt.utils.models import Model, ModelOutputType, is_multi_label, is_multi_label_binary from apt.utils.datasets import OUTPUT_DATA_ARRAY_TYPE from art.estimators.classification.pytorch import PyTorchClassifier as ArtPyTorchClassifier @@ -36,6 +37,7 @@ class PyTorchClassifierWrapper(ArtPyTorchClassifier): loss: "torch.nn.modules.loss._Loss", input_shape: Tuple[int, ...], nb_classes: int, + output_type: ModelOutputType, optimizer: Optional["torch.optim.Optimizer"] = None, # type: ignore use_amp: bool = False, opt_level: str = "O1", @@ -50,9 +52,10 @@ class PyTorchClassifierWrapper(ArtPyTorchClassifier): super().__init__(model, loss, input_shape, nb_classes, optimizer, use_amp, opt_level, loss_scale, channels_first, clip_values, preprocessing_defences, postprocessing_defences, preprocessing, device_type) - self._is_single_binary = None - self._is_multi_label = None - self._is_multi_label_binary = None + self._is_single_binary = (output_type == ModelOutputType.CLASSIFIER_SINGLE_OUTPUT_BINARY_PROBABILITIES or + output_type == ModelOutputType.CLASSIFIER_SINGLE_OUTPUT_BINARY_LOGITS) + self._is_multi_label = is_multi_label(output_type) + self._is_multi_label_binary = is_multi_label_binary(output_type) def get_step_correct(self, outputs, targets) -> int: """ @@ -122,9 +125,6 @@ class PyTorchClassifierWrapper(ArtPyTorchClassifier): :param kwargs: Dictionary of framework-specific arguments. This parameter is not currently supported for PyTorch and providing it takes no effect. """ - self._is_single_binary = self.nb_classes == 2 and (len(y.shape) == 1 or y.shape[1] == 1) - self._is_multi_label = is_multi_label(y) - self._is_multi_label_binary = is_multi_label_binary(y) # Put the model in the training mode self._model.train() @@ -189,6 +189,58 @@ class PyTorchClassifierWrapper(ArtPyTorchClassifier): else: self.save_checkpoint_state_dict(is_best=best_acc <= val_acc, path=path) + def predict( + self, x: np.ndarray, batch_size: int = 128, training_mode: bool = False, **kwargs + ) -> np.ndarray: + """ + Perform prediction for a batch of inputs. + + :param x: Input samples. + :param batch_size: Size of batches. + :param training_mode: `True` for model set to training mode and `'False` for model set to evaluation mode. + :return: Array of predictions of shape `(nb_inputs, nb_classes)`. + """ + import torch + + # Set model mode + self._model.train(mode=training_mode) + + # Apply preprocessing + x_preprocessed, _ = self._apply_preprocessing(x, y=None, fit=False) + + results_list = [] + + # Run prediction with batch processing + num_batch = int(np.ceil(len(x_preprocessed) / float(batch_size))) + for m in range(num_batch): + # Batch indexes + begin, end = ( + m * batch_size, + min((m + 1) * batch_size, x_preprocessed.shape[0]), + ) + + with torch.no_grad(): + model_outputs = self._model(torch.from_numpy(x_preprocessed[begin:end]).to(self._device)) + output = model_outputs[-1] + if isinstance(output, Iterable): + for i, o in enumerate(output): + o = o.detach().cpu().numpy().astype(np.float32) + # if len(output.shape) == 1: + # o = np.expand_dims(o, axis=1).astype(np.float32) + else: + output = output.detach().cpu().numpy().astype(np.float32) + if len(output.shape) == 1: + output = np.expand_dims(output, axis=1).astype(np.float32) + + results_list.append(output) + + results = np.vstack(results_list) + + # Apply postprocessing + predictions = self._apply_postprocessing(preds=results, fit=False) + + return predictions + def save_checkpoint_state_dict(self, is_best: bool, path=os.getcwd(), filename="latest.tar") -> None: """ Saves checkpoint as latest.tar or best.tar. @@ -352,7 +404,8 @@ class PyTorchClassifier(PyTorchModel): super().__init__(model, output_type, black_box_access, unlimited_queries, **kwargs) self._loss = loss self._optimizer = optimizer - self._art_model = PyTorchClassifierWrapper(model, loss, input_shape, nb_classes, optimizer) + self._nb_classes = nb_classes + self._art_model = PyTorchClassifierWrapper(model, loss, input_shape, nb_classes, output_type, optimizer) @property def loss(self): @@ -433,8 +486,7 @@ class PyTorchClassifier(PyTorchModel): """ return self._art_model.predict(x.get_samples(), **kwargs) - def score(self, test_data: PytorchData, binary_threshold: Optional[float] = 0.5, - apply_non_linearity: Optional[Callable] = None, **kwargs): + def score(self, test_data: PytorchData, **kwargs): """ Score the model using test data. @@ -452,23 +504,26 @@ class PyTorchClassifier(PyTorchModel): # numpy arrays y = test_data.get_labels() predicted = self.predict(test_data) - if apply_non_linearity: - predicted = apply_non_linearity(predicted) - # binary classification, single column of probabilities - if self._art_model.nb_classes == 2 and (len(predicted.shape) == 1 or predicted.shape[1] == 1): - if len(predicted.shape) > 1: - y = check_and_transform_label_format(y, self._art_model.nb_classes, return_one_hot=False) - return np.count_nonzero(y == (predicted > binary_threshold)) / predicted.shape[0] - # multi column - else: - if not is_multi_label(y): - y = check_and_transform_label_format(y, self._art_model.nb_classes) - return np.count_nonzero(np.argmax(y, axis=1) == np.argmax(predicted, axis=1)) / predicted.shape[0] - else: - if is_multi_label_binary(y): - predicted[predicted < binary_threshold] = 0 - predicted[predicted >= binary_threshold] = 1 - return np.count_nonzero(y == predicted) / (predicted.shape[0] * y.shape[1]) + kwargs['predictions'] = DatasetWithPredictions(pred=predicted) + kwargs['nb_classes'] = self._nb_classes + return super().score(ArrayDataset(test_data.get_samples(), test_data.get_labels()), **kwargs) + # if apply_non_linearity: + # predicted = apply_non_linearity(predicted) + # # binary classification, single column of probabilities + # if self._art_model.nb_classes == 2 and (len(predicted.shape) == 1 or predicted.shape[1] == 1): + # if len(predicted.shape) > 1: + # y = check_and_transform_label_format(y, self._art_model.nb_classes, return_one_hot=False) + # return np.count_nonzero(y == (predicted > binary_threshold)) / predicted.shape[0] + # # multi column + # else: + # if not is_multi_label(y): + # y = check_and_transform_label_format(y, self._art_model.nb_classes) + # return np.count_nonzero(np.argmax(y, axis=1) == np.argmax(predicted, axis=1)) / predicted.shape[0] + # else: + # if is_multi_label_binary(y): + # predicted[predicted < binary_threshold] = 0 + # predicted[predicted >= binary_threshold] = 1 + # return np.count_nonzero(y == predicted) / (predicted.shape[0] * y.shape[1]) def load_checkpoint_state_dict_by_path(self, model_name: str, path: str = None): diff --git a/apt/utils/models/sklearn_model.py b/apt/utils/models/sklearn_model.py index 6f40c65..1c8fa39 100644 --- a/apt/utils/models/sklearn_model.py +++ b/apt/utils/models/sklearn_model.py @@ -3,7 +3,7 @@ from typing import Optional from sklearn.base import BaseEstimator from apt.utils.models import Model, ModelOutputType, get_nb_classes, check_correct_model_output -from apt.utils.datasets import Dataset, OUTPUT_DATA_ARRAY_TYPE +from apt.utils.datasets import Dataset, ArrayDataset, OUTPUT_DATA_ARRAY_TYPE from art.estimators.classification.scikitlearn import SklearnClassifier as ArtSklearnClassifier from art.estimators.regression.scikitlearn import ScikitlearnRegressor @@ -48,7 +48,7 @@ class SklearnClassifier(SklearnModel): super().__init__(model, output_type, black_box_access, unlimited_queries, **kwargs) self._art_model = ArtSklearnClassifier(model, preprocessing=None) - def fit(self, train_data: Dataset, **kwargs) -> None: + def fit(self, train_data: ArrayDataset, **kwargs) -> None: """ Fit the model using the training data. @@ -58,11 +58,11 @@ class SklearnClassifier(SklearnModel): :return: None """ y = train_data.get_labels() - self.nb_classes = get_nb_classes(y) + self.nb_classes = get_nb_classes(y, self.output_type) y_encoded = check_and_transform_label_format(y, nb_classes=self.nb_classes) self._art_model.fit(train_data.get_samples(), y_encoded, **kwargs) - def predict(self, x: Dataset, **kwargs) -> OUTPUT_DATA_ARRAY_TYPE: + def predict(self, x: ArrayDataset, **kwargs) -> OUTPUT_DATA_ARRAY_TYPE: """ Perform predictions using the model for input `x`. diff --git a/tests/test_minimizer.py b/tests/test_minimizer.py index 9d3240b..056f17c 100644 --- a/tests/test_minimizer.py +++ b/tests/test_minimizer.py @@ -216,7 +216,7 @@ def test_minimizer_params(cells): base_est = DecisionTreeClassifier(random_state=0, min_samples_split=2, min_samples_leaf=1) - model = SklearnClassifier(base_est, ModelOutputType.CLASSIFIER_PROBABILITIES) + model = SklearnClassifier(base_est, ModelOutputType.CLASSIFIER_SINGLE_OUTPUT_CLASS_PROBABILITIES) model.fit(ArrayDataset(x, y)) expected_generalizations = {'categories': {}, 'category_representatives': {}, @@ -258,7 +258,7 @@ def test_minimizer_params_not_transform(cells): samples = ArrayDataset(x, y, features) base_est = DecisionTreeClassifier(random_state=0, min_samples_split=2, min_samples_leaf=1) - model = SklearnClassifier(base_est, ModelOutputType.CLASSIFIER_PROBABILITIES) + model = SklearnClassifier(base_est, ModelOutputType.CLASSIFIER_SINGLE_OUTPUT_CLASS_PROBABILITIES) model.fit(ArrayDataset(x, y)) gen = GeneralizeToRepresentative(model, cells=cells, generalize_using_transform=False) @@ -270,7 +270,7 @@ def test_minimizer_fit(data_two_features): x, y, features, _ = data_two_features base_est = DecisionTreeClassifier(random_state=0, min_samples_split=2, min_samples_leaf=1) - model = SklearnClassifier(base_est, ModelOutputType.CLASSIFIER_PROBABILITIES) + model = SklearnClassifier(base_est, ModelOutputType.CLASSIFIER_SINGLE_OUTPUT_CLASS_PROBABILITIES) model.fit(ArrayDataset(x, y)) ad = ArrayDataset(x) predictions = model.predict(ad) @@ -299,7 +299,7 @@ def test_minimizer_ncp(data_two_features): base_est = DecisionTreeClassifier(random_state=0, min_samples_split=2, min_samples_leaf=1) - model = SklearnClassifier(base_est, ModelOutputType.CLASSIFIER_PROBABILITIES) + model = SklearnClassifier(base_est, ModelOutputType.CLASSIFIER_SINGLE_OUTPUT_CLASS_PROBABILITIES) model.fit(ArrayDataset(x, y)) ad = ArrayDataset(x) ad1 = ArrayDataset(x1, features_names=features) @@ -342,7 +342,7 @@ def test_minimizer_ncp_categorical(data_four_features): base_est = DecisionTreeClassifier(random_state=0, min_samples_split=2, min_samples_leaf=1) - model = SklearnClassifier(base_est, ModelOutputType.CLASSIFIER_PROBABILITIES) + model = SklearnClassifier(base_est, ModelOutputType.CLASSIFIER_SINGLE_OUTPUT_CLASS_PROBABILITIES) model.fit(ArrayDataset(encoded, y)) ad = ArrayDataset(x) ad1 = ArrayDataset(x1) @@ -382,7 +382,7 @@ def test_minimizer_fit_not_transform(data_two_features): x, y, features, x1 = data_two_features base_est = DecisionTreeClassifier(random_state=0, min_samples_split=2, min_samples_leaf=1) - model = SklearnClassifier(base_est, ModelOutputType.CLASSIFIER_PROBABILITIES) + model = SklearnClassifier(base_est, ModelOutputType.CLASSIFIER_SINGLE_OUTPUT_CLASS_PROBABILITIES) model.fit(ArrayDataset(x, y)) ad = ArrayDataset(x) predictions = model.predict(ad) @@ -412,7 +412,7 @@ def test_minimizer_fit_pandas(data_four_features): base_est = DecisionTreeClassifier(random_state=0, min_samples_split=2, min_samples_leaf=1) - model = SklearnClassifier(base_est, ModelOutputType.CLASSIFIER_PROBABILITIES) + model = SklearnClassifier(base_est, ModelOutputType.CLASSIFIER_SINGLE_OUTPUT_CLASS_PROBABILITIES) model.fit(ArrayDataset(encoded, y)) predictions = model.predict(ArrayDataset(encoded)) if predictions.shape[1] > 1: @@ -450,7 +450,7 @@ def test_minimizer_params_categorical(cells_categorical): preprocessor, encoded = create_encoder(numeric_features, categorical_features, x) base_est = DecisionTreeClassifier(random_state=0, min_samples_split=2, min_samples_leaf=1) - model = SklearnClassifier(base_est, ModelOutputType.CLASSIFIER_PROBABILITIES) + model = SklearnClassifier(base_est, ModelOutputType.CLASSIFIER_SINGLE_OUTPUT_CLASS_PROBABILITIES) model.fit(ArrayDataset(encoded, y)) predictions = model.predict(ArrayDataset(encoded)) if predictions.shape[1] > 1: @@ -474,7 +474,7 @@ def test_minimizer_fit_qi(data_three_features): qi = ['age', 'weight'] base_est = DecisionTreeClassifier(random_state=0, min_samples_split=2, min_samples_leaf=1) - model = SklearnClassifier(base_est, ModelOutputType.CLASSIFIER_PROBABILITIES) + model = SklearnClassifier(base_est, ModelOutputType.CLASSIFIER_SINGLE_OUTPUT_CLASS_PROBABILITIES) model.fit(ArrayDataset(x, y)) ad = ArrayDataset(x) predictions = model.predict(ad) @@ -508,7 +508,7 @@ def test_minimizer_fit_pandas_qi(data_five_features): base_est = DecisionTreeClassifier(random_state=0, min_samples_split=2, min_samples_leaf=1) - model = SklearnClassifier(base_est, ModelOutputType.CLASSIFIER_PROBABILITIES) + model = SklearnClassifier(base_est, ModelOutputType.CLASSIFIER_SINGLE_OUTPUT_CLASS_PROBABILITIES) model.fit(ArrayDataset(encoded, y)) predictions = model.predict(ArrayDataset(encoded)) if predictions.shape[1] > 1: @@ -543,7 +543,7 @@ def test_minimize_ndarray_iris(): qi = ['sepal length (cm)', 'petal length (cm)'] base_est = DecisionTreeClassifier(random_state=0, min_samples_split=2, min_samples_leaf=1) - model = SklearnClassifier(base_est, ModelOutputType.CLASSIFIER_PROBABILITIES) + model = SklearnClassifier(base_est, ModelOutputType.CLASSIFIER_SINGLE_OUTPUT_CATEGORICAL) model.fit(ArrayDataset(x_train, y_train)) predictions = model.predict(ArrayDataset(x_train)) if predictions.shape[1] > 1: @@ -586,7 +586,7 @@ def test_minimize_pandas_adult(): base_est = DecisionTreeClassifier(random_state=0, min_samples_split=2, min_samples_leaf=1) - model = SklearnClassifier(base_est, ModelOutputType.CLASSIFIER_PROBABILITIES) + model = SklearnClassifier(base_est, ModelOutputType.CLASSIFIER_SINGLE_OUTPUT_CLASS_PROBABILITIES) model.fit(ArrayDataset(encoded, y_train)) predictions = model.predict(ArrayDataset(encoded)) if predictions.shape[1] > 1: @@ -642,7 +642,7 @@ def test_german_credit_pandas(): base_est = DecisionTreeClassifier(random_state=0, min_samples_split=2, min_samples_leaf=1) - model = SklearnClassifier(base_est, ModelOutputType.CLASSIFIER_PROBABILITIES) + model = SklearnClassifier(base_est, ModelOutputType.CLASSIFIER_SINGLE_OUTPUT_CLASS_PROBABILITIES) model.fit(ArrayDataset(encoded, y_train)) predictions = model.predict(ArrayDataset(encoded)) if predictions.shape[1] > 1: @@ -760,7 +760,7 @@ def test_x_y(): qi = [0, 2] base_est = DecisionTreeClassifier(random_state=0, min_samples_split=2, min_samples_leaf=1) - model = SklearnClassifier(base_est, ModelOutputType.CLASSIFIER_PROBABILITIES) + model = SklearnClassifier(base_est, ModelOutputType.CLASSIFIER_SINGLE_OUTPUT_CLASS_PROBABILITIES) model.fit(ArrayDataset(x, y)) ad = ArrayDataset(x) predictions = model.predict(ad) @@ -800,7 +800,7 @@ def test_x_y_features_names(): qi = ['age', 'weight'] base_est = DecisionTreeClassifier(random_state=0, min_samples_split=2, min_samples_leaf=1) - model = SklearnClassifier(base_est, ModelOutputType.CLASSIFIER_PROBABILITIES) + model = SklearnClassifier(base_est, ModelOutputType.CLASSIFIER_SINGLE_OUTPUT_CLASS_PROBABILITIES) model.fit(ArrayDataset(x, y)) ad = ArrayDataset(x) predictions = model.predict(ad) @@ -1202,7 +1202,7 @@ def test_keras_model(): base_est.compile(loss="categorical_crossentropy", optimizer="adam", metrics=["accuracy"]) - model = KerasClassifier(base_est, ModelOutputType.CLASSIFIER_PROBABILITIES) + model = KerasClassifier(base_est, ModelOutputType.CLASSIFIER_SINGLE_OUTPUT_CLASS_PROBABILITIES) model.fit(ArrayDataset(x, y)) ad = ArrayDataset(x_test) predictions = model.predict(ad) @@ -1269,8 +1269,11 @@ def test_minimizer_pytorch(data_three_features): criterion = nn.CrossEntropyLoss() optimizer = optim.Adam(base_est.parameters(), lr=0.01) - model = PyTorchClassifier(model=base_est, output_type=ModelOutputType.CLASSIFIER_LOGITS, loss=criterion, - optimizer=optimizer, input_shape=(3,), + model = PyTorchClassifier(model=base_est, + output_type=ModelOutputType.CLASSIFIER_SINGLE_OUTPUT_CLASS_LOGITS, + loss=criterion, + optimizer=optimizer, + input_shape=(3,), nb_classes=2) model.fit(PytorchData(x, y), save_entire_model=False, nb_epochs=10) @@ -1308,8 +1311,11 @@ def test_minimizer_pytorch_iris(): criterion = nn.CrossEntropyLoss() optimizer = optim.Adam(base_est.parameters(), lr=0.01) - model = PyTorchClassifier(model=base_est, output_type=ModelOutputType.CLASSIFIER_LOGITS, loss=criterion, - optimizer=optimizer, input_shape=(4,), + model = PyTorchClassifier(model=base_est, + output_type=ModelOutputType.CLASSIFIER_SINGLE_OUTPUT_CLASS_LOGITS, + loss=criterion, + optimizer=optimizer, + input_shape=(4,), nb_classes=3) model.fit(PytorchData(x_train, y_train), save_entire_model=False, nb_epochs=10) @@ -1362,7 +1368,7 @@ def test_errors(): y = np.array([1, 1, 0, 1, 0, 0, 1, 1, 0, 1, 0]) base_est = DecisionTreeClassifier(random_state=0, min_samples_split=2, min_samples_leaf=1) - model = SklearnClassifier(base_est, ModelOutputType.CLASSIFIER_PROBABILITIES) + model = SklearnClassifier(base_est, ModelOutputType.CLASSIFIER_SINGLE_OUTPUT_CLASS_PROBABILITIES) model.fit(ArrayDataset(X, y)) ad = ArrayDataset(X) predictions = model.predict(ad) diff --git a/tests/test_model.py b/tests/test_model.py index 20f2ba9..efbc4c0 100644 --- a/tests/test_model.py +++ b/tests/test_model.py @@ -24,7 +24,7 @@ tf.compat.v1.disable_eager_execution() def test_sklearn_classifier(): (x_train, y_train), (x_test, y_test) = dataset_utils.get_iris_dataset_np() underlying_model = RandomForestClassifier() - model = SklearnClassifier(underlying_model, ModelOutputType.CLASSIFIER_PROBABILITIES) + model = SklearnClassifier(underlying_model, ModelOutputType.CLASSIFIER_SINGLE_OUTPUT_CATEGORICAL) train = ArrayDataset(x_train, y_train) test = ArrayDataset(x_test, y_test) model.fit(train) @@ -59,7 +59,7 @@ def test_keras_classifier(): underlying_model.compile(loss="categorical_crossentropy", optimizer="adam", metrics=["accuracy"]) - model = KerasClassifier(underlying_model, ModelOutputType.CLASSIFIER_PROBABILITIES) + model = KerasClassifier(underlying_model, ModelOutputType.CLASSIFIER_SINGLE_OUTPUT_CATEGORICAL) train = ArrayDataset(x_train, y_train) test = ArrayDataset(x_test, y_test) @@ -97,7 +97,8 @@ def test_xgboost_classifier(): (x_train, y_train), (x_test, y_test) = dataset_utils.get_iris_dataset_np() underlying_model = XGBClassifier() underlying_model.fit(x_train, y_train) - model = XGBoostClassifier(underlying_model, ModelOutputType.CLASSIFIER_PROBABILITIES, input_shape=(4,), nb_classes=3) + model = XGBoostClassifier(underlying_model, ModelOutputType.CLASSIFIER_SINGLE_OUTPUT_CATEGORICAL, + input_shape=(4,), nb_classes=3) train = ArrayDataset(x_train, y_train) test = ArrayDataset(x_test, y_test) pred = model.predict(test) @@ -115,7 +116,7 @@ def test_blackbox_classifier(): train = ArrayDataset(x_train, y_train) test = ArrayDataset(x_test, y_test) data = Data(train, test) - model = BlackboxClassifierPredictions(data, ModelOutputType.CLASSIFIER_SCALAR) + model = BlackboxClassifierPredictions(data, ModelOutputType.CLASSIFIER_SINGLE_OUTPUT_CATEGORICAL) pred = model.predict(test) assert (pred.shape[0] == x_test.shape[0]) @@ -131,7 +132,7 @@ def test_blackbox_classifier_predictions(): train = DatasetWithPredictions(y_train, x_train) test = DatasetWithPredictions(y_test, x_test) data = Data(train, test) - model = BlackboxClassifierPredictions(data, ModelOutputType.CLASSIFIER_SCALAR) + model = BlackboxClassifierPredictions(data, ModelOutputType.CLASSIFIER_SINGLE_OUTPUT_CATEGORICAL) pred = model.predict(test) assert (pred.shape[0] == x_test.shape[0]) assert model.model_type is None @@ -146,7 +147,7 @@ def test_blackbox_classifier_predictions_y(): train = DatasetWithPredictions(y_train, x_train, y_train) test = DatasetWithPredictions(y_test, x_test, y_test) data = Data(train, test) - model = BlackboxClassifierPredictions(data, ModelOutputType.CLASSIFIER_SCALAR) + model = BlackboxClassifierPredictions(data, ModelOutputType.CLASSIFIER_SINGLE_OUTPUT_CATEGORICAL) pred = model.predict(test) assert (pred.shape[0] == x_test.shape[0]) @@ -166,7 +167,7 @@ def test_blackbox_classifier_predictions_multi_label_cat(): train = DatasetWithPredictions(y_train, x_train, y_train) test = DatasetWithPredictions(y_test, x_test, y_test) data = Data(train, test) - model = BlackboxClassifierPredictions(data, ModelOutputType.CLASSIFIER_SCALAR) + model = BlackboxClassifierPredictions(data, ModelOutputType.CLASSIFIER_SINGLE_OUTPUT_CATEGORICAL) pred = model.predict(test) assert (pred.shape[0] == x_test.shape[0]) @@ -194,7 +195,7 @@ def test_blackbox_classifier_predictions_multi_label_binary(): train = DatasetWithPredictions(pred_train, x_train, y_train) test = DatasetWithPredictions(pred_test, x_test, y_test) data = Data(train, test) - model = BlackboxClassifierPredictions(data, ModelOutputType.CLASSIFIER_SCALAR) + model = BlackboxClassifierPredictions(data, ModelOutputType.CLASSIFIER_MULTI_OUTPUT_BINARY_PROBABILITIES) pred = model.predict(test) assert (pred.shape[0] == x_test.shape[0]) @@ -211,7 +212,7 @@ def test_blackbox_classifier_mismatch(): test = ArrayDataset(x_test, y_test) data = Data(train, test) with pytest.raises(ValueError): - BlackboxClassifierPredictions(data, ModelOutputType.CLASSIFIER_PROBABILITIES) + BlackboxClassifierPredictions(data, ModelOutputType.CLASSIFIER_SINGLE_OUTPUT_CLASS_PROBABILITIES) def test_blackbox_classifier_no_test(): @@ -220,7 +221,7 @@ def test_blackbox_classifier_no_test(): train = ArrayDataset(x_train, y_train) data = Data(train) - model = BlackboxClassifierPredictions(data, ModelOutputType.CLASSIFIER_SCALAR) + model = BlackboxClassifierPredictions(data, ModelOutputType.CLASSIFIER_SINGLE_OUTPUT_CATEGORICAL) pred = model.predict(train) assert (pred.shape[0] == x_train.shape[0]) @@ -237,7 +238,7 @@ def test_blackbox_classifier_no_train(): test = ArrayDataset(x_test, y_test) data = Data(test=test) - model = BlackboxClassifierPredictions(data, ModelOutputType.CLASSIFIER_SCALAR) + model = BlackboxClassifierPredictions(data, ModelOutputType.CLASSIFIER_SINGLE_OUTPUT_CATEGORICAL) pred = model.predict(test) assert (pred.shape[0] == x_test.shape[0]) @@ -255,7 +256,7 @@ def test_blackbox_classifier_no_test_y(): train = ArrayDataset(x_train, y_train) test = ArrayDataset(x_test) data = Data(train, test) - model = BlackboxClassifierPredictions(data, ModelOutputType.CLASSIFIER_SCALAR) + model = BlackboxClassifierPredictions(data, ModelOutputType.CLASSIFIER_SINGLE_OUTPUT_CATEGORICAL) pred = model.predict(train) assert (pred.shape[0] == x_train.shape[0]) @@ -278,7 +279,7 @@ def test_blackbox_classifier_no_train_y(): train = ArrayDataset(x_train) test = ArrayDataset(x_test, y_test) data = Data(train, test) - model = BlackboxClassifierPredictions(data, ModelOutputType.CLASSIFIER_SCALAR) + model = BlackboxClassifierPredictions(data, ModelOutputType.CLASSIFIER_SINGLE_OUTPUT_CATEGORICAL) pred = model.predict(test) assert (pred.shape[0] == x_test.shape[0]) @@ -302,7 +303,7 @@ def test_blackbox_classifier_probabilities(): train = ArrayDataset(x_train, y_train) data = Data(train) - model = BlackboxClassifierPredictions(data, ModelOutputType.CLASSIFIER_PROBABILITIES) + model = BlackboxClassifierPredictions(data, ModelOutputType.CLASSIFIER_SINGLE_OUTPUT_CLASS_PROBABILITIES) pred = model.predict(train) assert (pred.shape[0] == x_train.shape[0]) assert (0.0 < pred).all() @@ -321,7 +322,8 @@ def test_blackbox_classifier_predict(): train = ArrayDataset(x_train, y_train) - model = BlackboxClassifierPredictFunction(predict, ModelOutputType.CLASSIFIER_PROBABILITIES, (4,), 3) + model = BlackboxClassifierPredictFunction(predict, ModelOutputType.CLASSIFIER_SINGLE_OUTPUT_CLASS_PROBABILITIES, + (4,), 3) pred = model.predict(train) assert (pred.shape[0] == x_train.shape[0]) assert (0.0 < pred).all() @@ -340,7 +342,8 @@ def test_blackbox_classifier_predict_scalar(): train = ArrayDataset(x_train, y_train) - model = BlackboxClassifierPredictFunction(predict, ModelOutputType.CLASSIFIER_SCALAR, (4,), 3) + model = BlackboxClassifierPredictFunction(predict, ModelOutputType.CLASSIFIER_SINGLE_OUTPUT_CLASS_PROBABILITIES, + (4,), 3) pred = model.predict(train) assert (pred.shape[0] == x_train.shape[0]) @@ -358,23 +361,23 @@ def test_is_one_hot(): def test_get_nb_classes(): (_, y_train), (_, y_test) = dataset_utils.get_iris_dataset_np() - + output_type = ModelOutputType.CLASSIFIER_SINGLE_OUTPUT_CATEGORICAL # shape: (x,) - not 1-hot - nb_classes_test = get_nb_classes(y_test) - nb_classes_train = get_nb_classes(y_train) + nb_classes_test = get_nb_classes(y_test, output_type) + nb_classes_train = get_nb_classes(y_train, output_type) assert (nb_classes_test == nb_classes_train) assert (nb_classes_test == 3) # shape: (x,1) - not 1-hot - nb_classes_test = get_nb_classes(y_test.reshape(-1, 1)) + nb_classes_test = get_nb_classes(y_test.reshape(-1, 1), output_type) assert (nb_classes_test == 3) # shape: (x,3) - 1-hot y = to_categorical(y_test) - nb_classes = get_nb_classes(y) + nb_classes = get_nb_classes(y, output_type) assert (nb_classes == 3) # gaps: 1,2,4 (0,3 missing) y_test[y_test == 0] = 4 - nb_classes = get_nb_classes(y_test) + nb_classes = get_nb_classes(y_test, output_type) assert (nb_classes == 5) diff --git a/tests/test_pytorch.py b/tests/test_pytorch.py index c138a2d..65f64dd 100644 --- a/tests/test_pytorch.py +++ b/tests/test_pytorch.py @@ -1,7 +1,7 @@ import numpy as np -from torch import nn, optim, sigmoid, where +from torch import nn, optim, sigmoid, where, from_numpy from torch.nn import functional -# from torch.utils.data import DataLoader, TensorDataset +from torch.utils.data import DataLoader, TensorDataset from scipy.special import expit from apt.utils.datasets.datasets import PytorchData @@ -56,8 +56,11 @@ def test_pytorch_nursery_state_dict(): criterion = nn.CrossEntropyLoss() optimizer = optim.Adam(inner_model.parameters(), lr=0.01) - model = PyTorchClassifier(model=inner_model, output_type=ModelOutputType.CLASSIFIER_LOGITS, loss=criterion, - optimizer=optimizer, input_shape=(24,), + model = PyTorchClassifier(model=inner_model, + output_type=ModelOutputType.CLASSIFIER_SINGLE_OUTPUT_CLASS_LOGITS, + loss=criterion, + optimizer=optimizer, + input_shape=(24,), nb_classes=4) model.fit(PytorchData(x_train.astype(np.float32), y_train), save_entire_model=False, nb_epochs=10) model.load_latest_state_dict_checkpoint() @@ -66,7 +69,7 @@ def test_pytorch_nursery_state_dict(): assert (0 <= score <= 1) # python pytorch numpy model.load_best_state_dict_checkpoint() - score = model.score(PytorchData(x_test.astype(np.float32), y_test)) + score = model.score(PytorchData(x_test.astype(np.float32), y_test), apply_non_linearity=expit) print('best model accuracy: ', score) assert (0 <= score <= 1) @@ -86,8 +89,11 @@ def test_pytorch_nursery_save_entire_model(): criterion = nn.CrossEntropyLoss() optimizer = optim.Adam(model.parameters(), lr=0.01) - art_model = PyTorchClassifier(model=model, output_type=ModelOutputType.CLASSIFIER_LOGITS, loss=criterion, - optimizer=optimizer, input_shape=(24,), + art_model = PyTorchClassifier(model=model, + output_type=ModelOutputType.CLASSIFIER_SINGLE_OUTPUT_CLASS_LOGITS, + loss=criterion, + optimizer=optimizer, + input_shape=(24,), nb_classes=4) art_model.fit(PytorchData(x_train.astype(np.float32), y_train), save_entire_model=True, nb_epochs=10) @@ -95,7 +101,7 @@ def test_pytorch_nursery_save_entire_model(): print('Base model accuracy: ', score) assert (0 <= score <= 1) art_model.load_best_model_checkpoint() - score = art_model.score(PytorchData(x_test.astype(np.float32), y_test)) + score = art_model.score(PytorchData(x_test.astype(np.float32), y_test), apply_non_linearity=expit) print('best model accuracy: ', score) assert (0 <= score <= 1) @@ -126,7 +132,7 @@ def test_pytorch_nursery_save_entire_model(): # # make multi-label categorical # y_train = np.column_stack((y_train, y_train, y_train)) # y_test = np.column_stack((y_test, y_test, y_test)) -# test = PytorchData(x_test, y_test) +# test = PytorchData(x_test.astype(np.float32), y_test.astype(np.float32)) # # model = multi_label_cat_model(3, 4) # criterion = nn.CrossEntropyLoss() @@ -154,8 +160,11 @@ def test_pytorch_nursery_save_entire_model(): # # optimizer.step() # -# art_model = PyTorchClassifier(model=model, output_type=ModelOutputType.CLASSIFIER_LOGITS, loss=criterion, -# optimizer=optimizer, input_shape=(24,), +# art_model = PyTorchClassifier(model=model, +# output_type=ModelOutputType.CLASSIFIER_MULTI_OUTPUT_CLASS_LOGITS, +# loss=criterion, +# optimizer=optimizer, +# input_shape=(24,), # nb_classes=3) # # pred = art_model.predict(test) @@ -211,8 +220,11 @@ def test_pytorch_predictions_multi_label_binary(): criterion = FocalLoss() optimizer = optim.RMSprop(model.parameters(), lr=0.01) - art_model = PyTorchClassifier(model=model, output_type=ModelOutputType.CLASSIFIER_LOGITS, loss=criterion, - optimizer=optimizer, input_shape=(24,), + art_model = PyTorchClassifier(model=model, + output_type=ModelOutputType.CLASSIFIER_MULTI_OUTPUT_BINARY_LOGITS, + loss=criterion, + optimizer=optimizer, + input_shape=(24,), nb_classes=3) art_model.fit(PytorchData(x_train.astype(np.float32), y_train.astype(np.float32)), save_entire_model=False, nb_epochs=10)