mirror of
https://github.com/IBM/ai-privacy-toolkit.git
synced 2026-05-24 14:15:13 +02:00
Support for many new model output types (#93)
* General model wrappers and methods supporting multi-label classifiers * Support for pytorch multi-label binary classifier * New model output types + single implementation of score method that supports multiple output types. * Anonymization with pytorch multi-output binary model * Support for multi-label binary models in minimizer. * Support for multi-label logits/probabilities --------- Signed-off-by: abigailt <abigailt@il.ibm.com>
This commit is contained in:
abigailgold
GitHub
parent
e00535d120
commit
57e38ea4fa
13 changed files with 913 additions and 172 deletions
|
|
@ -1,6 +1,11 @@
|
|||
from apt.utils.models.model import Model, BlackboxClassifier, ModelOutputType, ScoringMethod, \
|
||||
BlackboxClassifierPredictions, BlackboxClassifierPredictFunction, get_nb_classes, is_one_hot, \
|
||||
check_correct_model_output
|
||||
check_correct_model_output, is_multi_label, is_multi_label_binary, is_logits, is_binary, \
|
||||
CLASSIFIER_SINGLE_OUTPUT_CATEGORICAL, CLASSIFIER_SINGLE_OUTPUT_BINARY_PROBABILITIES, \
|
||||
CLASSIFIER_SINGLE_OUTPUT_CLASS_PROBABILITIES, CLASSIFIER_SINGLE_OUTPUT_BINARY_LOGITS, \
|
||||
CLASSIFIER_SINGLE_OUTPUT_CLASS_LOGITS, CLASSIFIER_MULTI_OUTPUT_CATEGORICAL, \
|
||||
CLASSIFIER_MULTI_OUTPUT_BINARY_PROBABILITIES, CLASSIFIER_MULTI_OUTPUT_CLASS_PROBABILITIES, \
|
||||
CLASSIFIER_MULTI_OUTPUT_BINARY_LOGITS, CLASSIFIER_MULTI_OUTPUT_CLASS_LOGITS
|
||||
from apt.utils.models.sklearn_model import SklearnModel, SklearnClassifier, SklearnRegressor
|
||||
from apt.utils.models.keras_model import KerasClassifier, KerasRegressor
|
||||
from apt.utils.models.xgboost_model import XGBoostClassifier
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ import numpy as np
|
|||
|
||||
from sklearn.metrics import mean_squared_error
|
||||
|
||||
from apt.utils.models import Model, ModelOutputType, ScoringMethod, check_correct_model_output
|
||||
from apt.utils.models import Model, ModelOutputType, ScoringMethod, is_logits
|
||||
from apt.utils.datasets import Dataset, OUTPUT_DATA_ARRAY_TYPE
|
||||
|
||||
from art.utils import check_and_transform_label_format
|
||||
|
|
@ -39,9 +39,7 @@ class KerasClassifier(KerasModel):
|
|||
def __init__(self, model: "keras.models.Model", output_type: ModelOutputType, black_box_access: Optional[bool] = True,
|
||||
unlimited_queries: Optional[bool] = True, **kwargs):
|
||||
super().__init__(model, output_type, black_box_access, unlimited_queries, **kwargs)
|
||||
logits = False
|
||||
if output_type == ModelOutputType.CLASSIFIER_LOGITS:
|
||||
logits = True
|
||||
logits = is_logits(output_type)
|
||||
self._art_model = ArtKerasClassifier(model, use_logits=logits)
|
||||
|
||||
def fit(self, train_data: Dataset, **kwargs) -> None:
|
||||
|
|
@ -65,7 +63,6 @@ class KerasClassifier(KerasModel):
|
|||
:return: Predictions from the model as numpy array (class probabilities, if supported).
|
||||
"""
|
||||
predictions = self._art_model.predict(x.get_samples(), **kwargs)
|
||||
check_correct_model_output(predictions, self.output_type)
|
||||
return predictions
|
||||
|
||||
def score(self, test_data: Dataset, scoring_method: Optional[ScoringMethod] = ScoringMethod.ACCURACY, **kwargs):
|
||||
|
|
@ -104,7 +101,7 @@ class KerasRegressor(KerasModel):
|
|||
"""
|
||||
def __init__(self, model: "keras.models.Model", black_box_access: Optional[bool] = True,
|
||||
unlimited_queries: Optional[bool] = True, **kwargs):
|
||||
super().__init__(model, ModelOutputType.REGRESSOR_SCALAR, black_box_access, unlimited_queries, **kwargs)
|
||||
super().__init__(model, ModelOutputType.REGRESSION, black_box_access, unlimited_queries, **kwargs)
|
||||
self._art_model = ArtKerasRegressor(model)
|
||||
|
||||
def fit(self, train_data: Dataset, **kwargs) -> None:
|
||||
|
|
|
|||
|
|
@ -1,9 +1,10 @@
|
|||
from abc import ABCMeta, abstractmethod
|
||||
from typing import Any, Optional, Callable, Tuple, Union, TYPE_CHECKING
|
||||
from enum import Enum, auto
|
||||
from enum import Enum, Flag, auto
|
||||
import numpy as np
|
||||
from scipy.special import expit
|
||||
|
||||
from apt.utils.datasets import Dataset, Data, OUTPUT_DATA_ARRAY_TYPE
|
||||
from apt.utils.datasets import Dataset, Data, array2numpy, OUTPUT_DATA_ARRAY_TYPE
|
||||
from art.estimators.classification import BlackBoxClassifier
|
||||
from art.utils import check_and_transform_label_format
|
||||
|
||||
|
|
@ -11,11 +12,40 @@ if TYPE_CHECKING:
|
|||
import torch
|
||||
|
||||
|
||||
class ModelOutputType(Enum):
|
||||
CLASSIFIER_PROBABILITIES = auto() # vector of probabilities
|
||||
CLASSIFIER_LOGITS = auto() # vector of logits
|
||||
CLASSIFIER_SCALAR = auto() # label only
|
||||
REGRESSOR_SCALAR = auto() # value
|
||||
class ModelOutputType(Flag):
|
||||
CLASSIFIER = auto()
|
||||
MULTI_OUTPUT = auto()
|
||||
BINARY = auto()
|
||||
LOGITS = auto()
|
||||
PROBABILITIES = auto()
|
||||
REGRESSION = auto()
|
||||
|
||||
|
||||
# class labels
|
||||
CLASSIFIER_SINGLE_OUTPUT_CATEGORICAL = ModelOutputType.CLASSIFIER
|
||||
# single binary probability
|
||||
CLASSIFIER_SINGLE_OUTPUT_BINARY_PROBABILITIES = ModelOutputType.CLASSIFIER | ModelOutputType.BINARY | \
|
||||
ModelOutputType.PROBABILITIES
|
||||
# vector of class probabilities
|
||||
CLASSIFIER_SINGLE_OUTPUT_CLASS_PROBABILITIES = ModelOutputType.CLASSIFIER | ModelOutputType.PROBABILITIES
|
||||
# single binary logit
|
||||
CLASSIFIER_SINGLE_OUTPUT_BINARY_LOGITS = ModelOutputType.CLASSIFIER | ModelOutputType.BINARY | ModelOutputType.LOGITS
|
||||
# vector of logits
|
||||
CLASSIFIER_SINGLE_OUTPUT_CLASS_LOGITS = ModelOutputType.CLASSIFIER | ModelOutputType.LOGITS
|
||||
# vector of class labels
|
||||
CLASSIFIER_MULTI_OUTPUT_CATEGORICAL = ModelOutputType.MULTI_OUTPUT | ModelOutputType.CLASSIFIER
|
||||
# vector of binary probabilities, 1 per output
|
||||
CLASSIFIER_MULTI_OUTPUT_BINARY_PROBABILITIES = ModelOutputType.MULTI_OUTPUT | ModelOutputType.CLASSIFIER | \
|
||||
ModelOutputType.BINARY | ModelOutputType.PROBABILITIES
|
||||
# vector of class probabilities for multiple outputs
|
||||
CLASSIFIER_MULTI_OUTPUT_CLASS_PROBABILITIES = ModelOutputType.MULTI_OUTPUT | ModelOutputType.CLASSIFIER | \
|
||||
ModelOutputType.PROBABILITIES
|
||||
# vector of binary logits
|
||||
CLASSIFIER_MULTI_OUTPUT_BINARY_LOGITS = ModelOutputType.MULTI_OUTPUT | ModelOutputType.CLASSIFIER | \
|
||||
ModelOutputType.BINARY | ModelOutputType.LOGITS
|
||||
# vector of logits for multiple outputs
|
||||
CLASSIFIER_MULTI_OUTPUT_CLASS_LOGITS = ModelOutputType.MULTI_OUTPUT | ModelOutputType.CLASSIFIER | \
|
||||
ModelOutputType.LOGITS
|
||||
|
||||
|
||||
class ModelType(Enum):
|
||||
|
|
@ -29,16 +59,52 @@ class ScoringMethod(Enum):
|
|||
|
||||
|
||||
def is_one_hot(y: OUTPUT_DATA_ARRAY_TYPE) -> bool:
|
||||
return len(y.shape) == 2 and y.shape[1] > 1
|
||||
if not isinstance(y, list):
|
||||
return len(y.shape) == 2 and y.shape[1] > 1 and np.all(np.around(np.sum(y, axis=1), decimals=4) == 1)
|
||||
return False
|
||||
|
||||
|
||||
def get_nb_classes(y: OUTPUT_DATA_ARRAY_TYPE) -> int:
|
||||
def is_multi_label(output_type: ModelOutputType) -> bool:
|
||||
return ModelOutputType.MULTI_OUTPUT in output_type
|
||||
|
||||
|
||||
def is_multi_label_binary(output_type: ModelOutputType) -> bool:
|
||||
return (ModelOutputType.MULTI_OUTPUT in output_type
|
||||
and ModelOutputType.BINARY in output_type)
|
||||
|
||||
|
||||
def is_binary(output_type: ModelOutputType) -> bool:
|
||||
return ModelOutputType.BINARY in output_type
|
||||
|
||||
|
||||
def is_categorical(output_type: ModelOutputType) -> bool:
|
||||
return (ModelOutputType.CLASSIFIER in output_type
|
||||
and ModelOutputType.BINARY not in output_type
|
||||
and ModelOutputType.PROBABILITIES not in output_type
|
||||
and ModelOutputType.LOGITS not in output_type)
|
||||
|
||||
|
||||
def is_probabilities(output_type: ModelOutputType) -> bool:
|
||||
return ModelOutputType.PROBABILITIES in output_type
|
||||
|
||||
|
||||
def is_logits(output_type: ModelOutputType) -> bool:
|
||||
return ModelOutputType.LOGITS in output_type
|
||||
|
||||
|
||||
def is_logits_or_probabilities(output_type: ModelOutputType) -> bool:
|
||||
return is_probabilities(output_type) or is_logits(output_type)
|
||||
|
||||
|
||||
def get_nb_classes(y: OUTPUT_DATA_ARRAY_TYPE, output_type: ModelOutputType) -> int:
|
||||
"""
|
||||
Get the number of classes from an array of labels
|
||||
|
||||
:param y: The labels
|
||||
:type y: numpy array
|
||||
:return: The number of classes as integer
|
||||
:param output_type: The output type of the model, as provided by the user
|
||||
:type output_type: ModelOutputType
|
||||
:return: The number of classes as integer, or list of integers for multi-label
|
||||
"""
|
||||
if y is None:
|
||||
return 0
|
||||
|
|
@ -48,8 +114,13 @@ def get_nb_classes(y: OUTPUT_DATA_ARRAY_TYPE) -> int:
|
|||
|
||||
if is_one_hot(y):
|
||||
return y.shape[1]
|
||||
else:
|
||||
elif is_multi_label(output_type):
|
||||
# for now just return the prediction dimension - this works in most cases
|
||||
return y.shape[1]
|
||||
elif is_categorical(output_type):
|
||||
return int(np.max(y) + 1)
|
||||
else: # binary
|
||||
return 2
|
||||
|
||||
|
||||
def check_correct_model_output(y: OUTPUT_DATA_ARRAY_TYPE, output_type: ModelOutputType):
|
||||
|
|
@ -61,10 +132,9 @@ def check_correct_model_output(y: OUTPUT_DATA_ARRAY_TYPE, output_type: ModelOutp
|
|||
:type output_type: ModelOutputType
|
||||
:raises: ValueError (in case of mismatch)
|
||||
"""
|
||||
if not is_one_hot(y): # 1D array
|
||||
if output_type == ModelOutputType.CLASSIFIER_PROBABILITIES or output_type == ModelOutputType.CLASSIFIER_LOGITS:
|
||||
raise ValueError("Incompatible model output types. Model outputs 1D array of categorical scalars while "
|
||||
"output type is set to ", output_type)
|
||||
if not is_one_hot(y) and not is_multi_label(output_type) and is_categorical(output_type):
|
||||
raise ValueError("Incompatible model output types. Model outputs 1D array of categorical scalars while "
|
||||
"output type is set to ", output_type)
|
||||
|
||||
|
||||
class Model(metaclass=ABCMeta):
|
||||
|
|
@ -115,16 +185,81 @@ class Model(metaclass=ABCMeta):
|
|||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def score(self, test_data: Dataset, **kwargs):
|
||||
"""
|
||||
Score the model using test data.
|
||||
|
||||
:param test_data: Test data.
|
||||
:type train_data: `Dataset`
|
||||
:type test_data: `Dataset`
|
||||
:keyword predictions: Model predictions to score. If provided, these will be used instead of calling the model's
|
||||
`predict` method.
|
||||
:type predictions: `DatasetWithPredictions` with the `pred` field filled.
|
||||
:keyword scoring_method: The method for scoring predictions. Default is ACCURACY.
|
||||
:type scoring_method: `ScoringMethod`, optional
|
||||
:keyword binary_threshold: The threshold to use on binary classification probabilities to assign the positive
|
||||
class.
|
||||
:type binary_threshold: float, optional. Default is 0.5.
|
||||
:keyword apply_non_linearity: A non-linear function to apply to the result of the 'predict' method, in case the
|
||||
model outputs logits (e.g., sigmoid).
|
||||
:type apply_non_linearity: Callable, should be possible to apply directly to the numpy output of the 'predict'
|
||||
method, optional.
|
||||
:keyword nb_classes: number of classes (for classification models).
|
||||
:type nb_classes: int, optional.
|
||||
:return: the score as float (for classifiers, between 0 and 1)
|
||||
"""
|
||||
raise NotImplementedError
|
||||
predictions = kwargs.get('predictions')
|
||||
nb_classes = kwargs.get('nb_classes')
|
||||
scoring_method = kwargs.get('scoring_method', ScoringMethod.ACCURACY)
|
||||
binary_threshold = kwargs.get('binary_threshold', 0.5)
|
||||
apply_non_linearity = kwargs.get('apply_non_linearity', expit)
|
||||
|
||||
if test_data.get_samples() is None and predictions is None:
|
||||
raise ValueError('score can only be computed when test data or predictions are available')
|
||||
if test_data.get_labels() is None:
|
||||
raise ValueError('score can only be computed when labels are available')
|
||||
if predictions:
|
||||
predicted = predictions.get_predictions()
|
||||
else:
|
||||
predicted = self.predict(test_data)
|
||||
y = array2numpy(test_data.get_labels())
|
||||
|
||||
if scoring_method == ScoringMethod.ACCURACY:
|
||||
if not is_multi_label(self.output_type) and not is_binary(self.output_type):
|
||||
if nb_classes is not None:
|
||||
y = check_and_transform_label_format(y, nb_classes=nb_classes)
|
||||
# categorical has been 1-hot encoded by check_and_transform_label_format
|
||||
return np.count_nonzero(np.argmax(y, axis=1) == np.argmax(predicted, axis=1)) / predicted.shape[0]
|
||||
elif (is_multi_label(self.output_type) and not is_binary(self.output_type)
|
||||
and is_logits_or_probabilities(self.output_type)):
|
||||
if predicted.shape != y.shape:
|
||||
raise ValueError('Do not know how to compare arrays with different shapes')
|
||||
elif len(predicted.shape) < 3:
|
||||
raise ValueError('Do not know how to compare 2-D arrays for multi-output non-binary case')
|
||||
else:
|
||||
sum = 0
|
||||
count = 0
|
||||
for i in range(predicted.shape[1]):
|
||||
count += np.count_nonzero(np.argmax(y[:, i], axis=1) == np.argmax(predicted[:, i], axis=1))
|
||||
sum += predicted.shape[0] * predicted.shape[-1]
|
||||
return count / sum
|
||||
elif is_multi_label(self.output_type) and is_categorical(self.output_type):
|
||||
return np.count_nonzero(y == predicted) / (predicted.shape[0] * y.shape[1])
|
||||
elif is_binary(self.output_type):
|
||||
if is_logits(self.output_type):
|
||||
if apply_non_linearity:
|
||||
predicted = apply_non_linearity(predicted)
|
||||
else: # apply sigmoid
|
||||
predicted = expit(predicted)
|
||||
predicted[predicted < binary_threshold] = 0
|
||||
predicted[predicted >= binary_threshold] = 1
|
||||
if len(y.shape) > 1:
|
||||
return np.count_nonzero(y == predicted) / (predicted.shape[0] * y.shape[1])
|
||||
else:
|
||||
return np.count_nonzero(y == predicted.reshape(-1)) / (predicted.shape[0])
|
||||
else:
|
||||
raise NotImplementedError('score method not implemented for output type: ', self.output_type)
|
||||
else:
|
||||
raise NotImplementedError('scoring method not implemented: ', scoring_method)
|
||||
|
||||
@property
|
||||
def model(self) -> Any:
|
||||
|
|
@ -167,7 +302,8 @@ class Model(metaclass=ABCMeta):
|
|||
|
||||
class BlackboxClassifier(Model):
|
||||
"""
|
||||
Wrapper for black-box ML classification models.
|
||||
Wrapper for black-box ML classification models. This is an abstract class and must be instantiated as either
|
||||
BlackboxClassifierPredictFunction or BlackboxClassifierPredictions.
|
||||
|
||||
:param model: The training and/or test data along with the model's predictions for the data or a callable predict
|
||||
method.
|
||||
|
|
@ -247,6 +383,13 @@ class BlackboxClassifier(Model):
|
|||
"""
|
||||
return self._optimizer
|
||||
|
||||
def score(self, test_data: Dataset, **kwargs):
|
||||
"""
|
||||
Score the model using test data.
|
||||
"""
|
||||
kwargs['nb_classes'] = self.nb_classes
|
||||
return super().score(test_data, **kwargs)
|
||||
|
||||
def fit(self, train_data: Dataset, **kwargs) -> None:
|
||||
"""
|
||||
A blackbox model cannot be fit.
|
||||
|
|
@ -263,28 +406,8 @@ class BlackboxClassifier(Model):
|
|||
:return: Predictions from the model as numpy array.
|
||||
"""
|
||||
predictions = self._art_model.predict(x.get_samples())
|
||||
check_correct_model_output(predictions, self.output_type)
|
||||
return predictions
|
||||
|
||||
def score(self, test_data: Dataset, scoring_method: Optional[ScoringMethod] = ScoringMethod.ACCURACY, **kwargs):
|
||||
"""
|
||||
Score the model using test data.
|
||||
|
||||
:param test_data: Test data.
|
||||
:type train_data: `Dataset`
|
||||
:param scoring_method: The method for scoring predictions. Default is ACCURACY.
|
||||
:type scoring_method: `ScoringMethod`, optional
|
||||
:return: the score as float (for classifiers, between 0 and 1)
|
||||
"""
|
||||
if test_data.get_samples() is None or test_data.get_labels() is None:
|
||||
raise ValueError('score can only be computed when test data and labels are available')
|
||||
predicted = self._art_model.predict(test_data.get_samples())
|
||||
y = check_and_transform_label_format(test_data.get_labels(), nb_classes=self._nb_classes)
|
||||
if scoring_method == ScoringMethod.ACCURACY:
|
||||
return np.count_nonzero(np.argmax(y, axis=1) == np.argmax(predicted, axis=1)) / predicted.shape[0]
|
||||
else:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def get_predictions(self) -> Union[Callable, Tuple[OUTPUT_DATA_ARRAY_TYPE, OUTPUT_DATA_ARRAY_TYPE]]:
|
||||
"""
|
||||
|
|
@ -325,17 +448,9 @@ class BlackboxClassifierPredictions(BlackboxClassifier):
|
|||
if y_test_pred is None:
|
||||
y_test_pred = model.get_test_labels()
|
||||
|
||||
if y_train_pred is not None:
|
||||
check_correct_model_output(y_train_pred, self.output_type)
|
||||
if y_test_pred is not None:
|
||||
check_correct_model_output(y_test_pred, self.output_type)
|
||||
|
||||
if y_train_pred is not None and len(y_train_pred.shape) == 1:
|
||||
self._nb_classes = get_nb_classes(y_train_pred)
|
||||
y_train_pred = check_and_transform_label_format(y_train_pred, nb_classes=self._nb_classes)
|
||||
if y_test_pred is not None and len(y_test_pred.shape) == 1:
|
||||
if self._nb_classes is None:
|
||||
self._nb_classes = get_nb_classes(y_test_pred)
|
||||
y_test_pred = check_and_transform_label_format(y_test_pred, nb_classes=self._nb_classes)
|
||||
|
||||
if x_train_pred is not None and y_train_pred is not None and x_test_pred is not None and y_test_pred is not None:
|
||||
|
|
@ -353,7 +468,7 @@ class BlackboxClassifierPredictions(BlackboxClassifier):
|
|||
else:
|
||||
raise NotImplementedError("Invalid data - None")
|
||||
|
||||
self._nb_classes = get_nb_classes(y_pred)
|
||||
self._nb_classes = get_nb_classes(y_pred, self.output_type)
|
||||
self._input_shape = x_pred.shape[1:]
|
||||
self._x_pred = x_pred
|
||||
self._y_pred = y_pred
|
||||
|
|
|
|||
|
|
@ -3,17 +3,22 @@ import os
|
|||
import shutil
|
||||
import logging
|
||||
|
||||
from typing import Optional, Tuple
|
||||
from typing import Optional, Tuple, Union, List, TYPE_CHECKING
|
||||
import numpy as np
|
||||
import torch
|
||||
from torch.utils.data import DataLoader, TensorDataset
|
||||
|
||||
from art.utils import check_and_transform_label_format
|
||||
from apt.utils.datasets.datasets import PytorchData
|
||||
from apt.utils.models import Model, ModelOutputType
|
||||
from apt.utils.datasets import OUTPUT_DATA_ARRAY_TYPE
|
||||
from apt.utils.datasets.datasets import PytorchData, DatasetWithPredictions, ArrayDataset
|
||||
from apt.utils.models import Model, ModelOutputType, is_multi_label, is_multi_label_binary, is_binary
|
||||
from apt.utils.datasets import OUTPUT_DATA_ARRAY_TYPE, array2numpy
|
||||
from art.estimators.classification.pytorch import PyTorchClassifier as ArtPyTorchClassifier
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from art.utils import CLIP_VALUES_TYPE, PREPROCESSING_TYPE
|
||||
from art.defences.preprocessor import Preprocessor
|
||||
from art.defences.postprocessor import Postprocessor
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
|
@ -30,16 +35,46 @@ class PyTorchClassifierWrapper(ArtPyTorchClassifier):
|
|||
Extension for Pytorch ART model
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
model: "torch.nn.Module",
|
||||
loss: "torch.nn.modules.loss._Loss",
|
||||
input_shape: Tuple[int, ...],
|
||||
nb_classes: int,
|
||||
output_type: ModelOutputType,
|
||||
optimizer: Optional["torch.optim.Optimizer"] = None, # type: ignore
|
||||
use_amp: bool = False,
|
||||
opt_level: str = "O1",
|
||||
loss_scale: Optional[Union[float, str]] = "dynamic",
|
||||
channels_first: bool = True,
|
||||
clip_values: Optional["CLIP_VALUES_TYPE"] = None,
|
||||
preprocessing_defences: Union["Preprocessor", List["Preprocessor"], None] = None,
|
||||
postprocessing_defences: Union["Postprocessor", List["Postprocessor"], None] = None,
|
||||
preprocessing: "PREPROCESSING_TYPE" = (0.0, 1.0),
|
||||
device_type: str = "gpu",
|
||||
):
|
||||
super().__init__(model, loss, input_shape, nb_classes, optimizer, use_amp, opt_level, loss_scale,
|
||||
channels_first, clip_values, preprocessing_defences, postprocessing_defences, preprocessing,
|
||||
device_type)
|
||||
self._is_single_binary = not is_multi_label(output_type) and is_binary(output_type)
|
||||
self._is_multi_label = is_multi_label(output_type)
|
||||
self._is_multi_label_binary = is_multi_label_binary(output_type)
|
||||
|
||||
def get_step_correct(self, outputs, targets) -> int:
|
||||
"""
|
||||
Get number of correctly classified labels.
|
||||
"""
|
||||
# here everything is torch tensors
|
||||
if len(outputs) != len(targets):
|
||||
raise ValueError("outputs and targets should be the same length.")
|
||||
if self.nb_classes > 1:
|
||||
return int(torch.sum(torch.argmax(outputs, axis=-1) == targets).item())
|
||||
if self._is_single_binary:
|
||||
return int(torch.sum(torch.round(outputs) == targets).item())
|
||||
elif self._is_multi_label:
|
||||
if self._is_multi_label_binary:
|
||||
outputs = torch.round(outputs)
|
||||
return int(torch.sum(targets == outputs).item())
|
||||
else:
|
||||
return int(torch.sum(torch.round(outputs, axis=-1) == targets).item())
|
||||
return int(torch.sum(torch.argmax(outputs, axis=-1) == targets).item())
|
||||
|
||||
def _eval(self, loader: DataLoader):
|
||||
"""
|
||||
|
|
@ -93,6 +128,7 @@ class PyTorchClassifierWrapper(ArtPyTorchClassifier):
|
|||
:param kwargs: Dictionary of framework-specific arguments. This parameter is not currently
|
||||
supported for PyTorch and providing it takes no effect.
|
||||
"""
|
||||
|
||||
# Put the model in the training mode
|
||||
self._model.train()
|
||||
|
||||
|
|
@ -156,6 +192,61 @@ class PyTorchClassifierWrapper(ArtPyTorchClassifier):
|
|||
else:
|
||||
self.save_checkpoint_state_dict(is_best=best_acc <= val_acc, path=path)
|
||||
|
||||
def predict(
|
||||
self, x: np.ndarray, batch_size: int = 128, training_mode: bool = False, **kwargs
|
||||
) -> np.ndarray:
|
||||
"""
|
||||
Perform prediction for a batch of inputs.
|
||||
|
||||
:param x: Input samples.
|
||||
:param batch_size: Size of batches.
|
||||
:param training_mode: `True` for model set to training mode and `'False` for model set to evaluation mode.
|
||||
:return: Array of predictions of shape `(nb_inputs, nb_classes)`.
|
||||
"""
|
||||
import torch
|
||||
|
||||
# Set model mode
|
||||
self._model.train(mode=training_mode)
|
||||
|
||||
# Apply preprocessing
|
||||
x_preprocessed, _ = self._apply_preprocessing(x, y=None, fit=False)
|
||||
|
||||
results_list = []
|
||||
|
||||
# Run prediction with batch processing
|
||||
num_batch = int(np.ceil(len(x_preprocessed) / float(batch_size)))
|
||||
for m in range(num_batch):
|
||||
# Batch indexes
|
||||
begin, end = (
|
||||
m * batch_size,
|
||||
min((m + 1) * batch_size, x_preprocessed.shape[0]),
|
||||
)
|
||||
|
||||
with torch.no_grad():
|
||||
model_outputs = self._model(torch.from_numpy(x_preprocessed[begin:end]).to(self._device))
|
||||
output = model_outputs[-1]
|
||||
|
||||
if isinstance(output, tuple):
|
||||
output_list = []
|
||||
for o in output:
|
||||
o = o.detach().cpu().numpy().astype(np.float32)
|
||||
output_list.append(o)
|
||||
output_np = np.array(output_list)
|
||||
output_np = np.swapaxes(output_np, 0, 1)
|
||||
results_list.append(output_np)
|
||||
else:
|
||||
output = output.detach().cpu().numpy().astype(np.float32)
|
||||
if len(output.shape) == 1:
|
||||
output = np.expand_dims(output, axis=1).astype(np.float32)
|
||||
results_list.append(output)
|
||||
|
||||
results = np.vstack(results_list)
|
||||
|
||||
# Apply postprocessing
|
||||
predictions = self._apply_postprocessing(preds=results, fit=False)
|
||||
|
||||
return predictions
|
||||
|
||||
def save_checkpoint_state_dict(self, is_best: bool, path=os.getcwd(), filename="latest.tar") -> None:
|
||||
"""
|
||||
Saves checkpoint as latest.tar or best.tar.
|
||||
|
|
@ -319,7 +410,8 @@ class PyTorchClassifier(PyTorchModel):
|
|||
super().__init__(model, output_type, black_box_access, unlimited_queries, **kwargs)
|
||||
self._loss = loss
|
||||
self._optimizer = optimizer
|
||||
self._art_model = PyTorchClassifierWrapper(model, loss, input_shape, nb_classes, optimizer)
|
||||
self._nb_classes = nb_classes
|
||||
self._art_model = PyTorchClassifierWrapper(model, loss, input_shape, nb_classes, output_type, optimizer)
|
||||
|
||||
@property
|
||||
def loss(self):
|
||||
|
|
@ -398,7 +490,7 @@ class PyTorchClassifier(PyTorchModel):
|
|||
:type x: `np.ndarray` or `pandas.DataFrame`
|
||||
:return: Predictions from the model (class probabilities, if supported).
|
||||
"""
|
||||
return self._art_model.predict(x.get_samples(), **kwargs)
|
||||
return array2numpy(self._art_model.predict(x.get_samples(), **kwargs))
|
||||
|
||||
def score(self, test_data: PytorchData, **kwargs):
|
||||
"""
|
||||
|
|
@ -406,18 +498,20 @@ class PyTorchClassifier(PyTorchModel):
|
|||
|
||||
:param test_data: Test data.
|
||||
:type test_data: `PytorchData`
|
||||
:param binary_threshold: The threshold to use on binary classification probabilities to assign the positive
|
||||
class.
|
||||
:type binary_threshold: float, optional. Default is 0.5.
|
||||
:param apply_non_linearity: A non-linear function to apply to the result of the 'predict' method, in case the
|
||||
model outputs logits (e.g., sigmoid).
|
||||
:type apply_non_linearity: Callable, should be possible to apply directly to the numpy output of the 'predict'
|
||||
method, optional.
|
||||
:return: the score as float (between 0 and 1)
|
||||
"""
|
||||
y = test_data.get_labels()
|
||||
# numpy arrays
|
||||
predicted = self.predict(test_data)
|
||||
# binary classification, single column of probabilities
|
||||
if self._art_model.nb_classes == 2 and (len(predicted.shape) == 1 or predicted.shape[1] == 1):
|
||||
if len(predicted.shape) > 1:
|
||||
y = check_and_transform_label_format(y, self._art_model.nb_classes, return_one_hot=False)
|
||||
return np.count_nonzero(y == (predicted > 0.5)) / predicted.shape[0]
|
||||
else:
|
||||
y = check_and_transform_label_format(y, self._art_model.nb_classes)
|
||||
return np.count_nonzero(np.argmax(y, axis=1) == np.argmax(predicted, axis=1)) / predicted.shape[0]
|
||||
kwargs['predictions'] = DatasetWithPredictions(pred=predicted)
|
||||
kwargs['nb_classes'] = self._nb_classes
|
||||
return super().score(ArrayDataset(test_data.get_samples(), test_data.get_labels()), **kwargs)
|
||||
|
||||
def load_checkpoint_state_dict_by_path(self, model_name: str, path: str = None):
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -2,8 +2,8 @@ from typing import Optional
|
|||
|
||||
from sklearn.base import BaseEstimator
|
||||
|
||||
from apt.utils.models import Model, ModelOutputType, get_nb_classes, check_correct_model_output
|
||||
from apt.utils.datasets import Dataset, OUTPUT_DATA_ARRAY_TYPE
|
||||
from apt.utils.models import Model, ModelOutputType, get_nb_classes
|
||||
from apt.utils.datasets import Dataset, ArrayDataset, OUTPUT_DATA_ARRAY_TYPE
|
||||
|
||||
from art.estimators.classification.scikitlearn import SklearnClassifier as ArtSklearnClassifier
|
||||
from art.estimators.regression.scikitlearn import ScikitlearnRegressor
|
||||
|
|
@ -48,7 +48,7 @@ class SklearnClassifier(SklearnModel):
|
|||
super().__init__(model, output_type, black_box_access, unlimited_queries, **kwargs)
|
||||
self._art_model = ArtSklearnClassifier(model, preprocessing=None)
|
||||
|
||||
def fit(self, train_data: Dataset, **kwargs) -> None:
|
||||
def fit(self, train_data: ArrayDataset, **kwargs) -> None:
|
||||
"""
|
||||
Fit the model using the training data.
|
||||
|
||||
|
|
@ -58,11 +58,11 @@ class SklearnClassifier(SklearnModel):
|
|||
:return: None
|
||||
"""
|
||||
y = train_data.get_labels()
|
||||
self.nb_classes = get_nb_classes(y)
|
||||
self.nb_classes = get_nb_classes(y, self.output_type)
|
||||
y_encoded = check_and_transform_label_format(y, nb_classes=self.nb_classes)
|
||||
self._art_model.fit(train_data.get_samples(), y_encoded, **kwargs)
|
||||
|
||||
def predict(self, x: Dataset, **kwargs) -> OUTPUT_DATA_ARRAY_TYPE:
|
||||
def predict(self, x: ArrayDataset, **kwargs) -> OUTPUT_DATA_ARRAY_TYPE:
|
||||
"""
|
||||
Perform predictions using the model for input `x`.
|
||||
|
||||
|
|
@ -71,7 +71,7 @@ class SklearnClassifier(SklearnModel):
|
|||
:return: Predictions from the model as numpy array (class probabilities, if supported).
|
||||
"""
|
||||
predictions = self._art_model.predict(x.get_samples(), **kwargs)
|
||||
check_correct_model_output(predictions, self.output_type)
|
||||
# check_correct_model_output(predictions, self.output_type)
|
||||
return predictions
|
||||
|
||||
|
||||
|
|
@ -93,7 +93,7 @@ class SklearnRegressor(SklearnModel):
|
|||
"""
|
||||
def __init__(self, model: BaseEstimator, black_box_access: Optional[bool] = True,
|
||||
unlimited_queries: Optional[bool] = True, **kwargs):
|
||||
super().__init__(model, ModelOutputType.REGRESSOR_SCALAR, black_box_access, unlimited_queries, **kwargs)
|
||||
super().__init__(model, ModelOutputType.REGRESSION, black_box_access, unlimited_queries, **kwargs)
|
||||
self._art_model = ScikitlearnRegressor(model)
|
||||
|
||||
def fit(self, train_data: Dataset, **kwargs) -> None:
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
from typing import Optional, Tuple
|
||||
|
||||
from apt.utils.models import Model, ModelOutputType, ScoringMethod, check_correct_model_output, is_one_hot
|
||||
from apt.utils.models import Model, ModelOutputType, ScoringMethod, is_one_hot
|
||||
from apt.utils.datasets import Dataset, OUTPUT_DATA_ARRAY_TYPE
|
||||
|
||||
import numpy as np
|
||||
|
|
@ -63,7 +63,7 @@ class XGBoostClassifier(XGBoostModel):
|
|||
:return: Predictions from the model as numpy array (class probabilities, if supported).
|
||||
"""
|
||||
predictions = self._art_model.predict(x.get_samples(), **kwargs)
|
||||
check_correct_model_output(predictions, self.output_type)
|
||||
# check_correct_model_output(predictions, self.output_type)
|
||||
return predictions
|
||||
|
||||
def score(self, test_data: Dataset, scoring_method: Optional[ScoringMethod] = ScoringMethod.ACCURACY, **kwargs):
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue