mirror of
https://github.com/IBM/ai-privacy-toolkit.git
synced 2026-06-08 15:05:13 +02:00
New model output types + single implementation of score method that supports multiple output types. Existing tests pass. Still need more tests for new types.
Signed-off-by: abigailt <abigailt@il.ibm.com>
This commit is contained in:
abigailt
parent
79534b69db
commit
5e19d4ae27
11 changed files with 285 additions and 144 deletions
|
|
@ -93,7 +93,8 @@ class GeneralizeToRepresentative(BaseEstimator, MetaEstimatorMixin, TransformerM
|
|||
if is_regression:
|
||||
self.estimator = SklearnRegressor(estimator)
|
||||
else:
|
||||
self.estimator = SklearnClassifier(estimator, ModelOutputType.CLASSIFIER_PROBABILITIES)
|
||||
self.estimator = SklearnClassifier(estimator,
|
||||
ModelOutputType.CLASSIFIER_SINGLE_OUTPUT_CLASS_PROBABILITIES)
|
||||
self.target_accuracy = target_accuracy
|
||||
self.cells = cells
|
||||
self.categorical_features = []
|
||||
|
|
|
|||
|
|
@ -4,4 +4,4 @@ Implementation of datasets utility components for datasets creation, load, and s
|
|||
"""
|
||||
|
||||
from apt.utils.datasets.datasets import Dataset, StoredDataset, DatasetFactory, Data, ArrayDataset, \
|
||||
DatasetWithPredictions, OUTPUT_DATA_ARRAY_TYPE, DATA_PANDAS_NUMPY_TYPE
|
||||
DatasetWithPredictions, array2numpy, OUTPUT_DATA_ARRAY_TYPE, DATA_PANDAS_NUMPY_TYPE
|
||||
|
|
|
|||
|
|
@ -266,6 +266,8 @@ class DatasetWithPredictions(Dataset):
|
|||
Dataset that is based on arrays (e.g., numpy/pandas/list...). Includes predictions from a model, and possibly also
|
||||
features and true labels.
|
||||
|
||||
:param pred: collection of model predictions
|
||||
:type pred: numpy array or pandas DataFrame or list or pytorch Tensor
|
||||
:param x: collection of data samples
|
||||
:type x: numpy array or pandas DataFrame or list or pytorch Tensor
|
||||
:param y: collection of labels
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
from apt.utils.models.model import Model, BlackboxClassifier, ModelOutputType, ScoringMethod, \
|
||||
BlackboxClassifierPredictions, BlackboxClassifierPredictFunction, get_nb_classes, is_one_hot, \
|
||||
check_correct_model_output, is_multi_label, is_multi_label_binary
|
||||
check_correct_model_output, is_multi_label, is_multi_label_binary, is_logits
|
||||
from apt.utils.models.sklearn_model import SklearnModel, SklearnClassifier, SklearnRegressor
|
||||
from apt.utils.models.keras_model import KerasClassifier, KerasRegressor
|
||||
from apt.utils.models.xgboost_model import XGBoostClassifier
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ import numpy as np
|
|||
|
||||
from sklearn.metrics import mean_squared_error
|
||||
|
||||
from apt.utils.models import Model, ModelOutputType, ScoringMethod, check_correct_model_output
|
||||
from apt.utils.models import Model, ModelOutputType, ScoringMethod, check_correct_model_output, is_logits
|
||||
from apt.utils.datasets import Dataset, OUTPUT_DATA_ARRAY_TYPE
|
||||
|
||||
from art.utils import check_and_transform_label_format
|
||||
|
|
@ -39,9 +39,7 @@ class KerasClassifier(KerasModel):
|
|||
def __init__(self, model: "keras.models.Model", output_type: ModelOutputType, black_box_access: Optional[bool] = True,
|
||||
unlimited_queries: Optional[bool] = True, **kwargs):
|
||||
super().__init__(model, output_type, black_box_access, unlimited_queries, **kwargs)
|
||||
logits = False
|
||||
if output_type == ModelOutputType.CLASSIFIER_LOGITS:
|
||||
logits = True
|
||||
logits = is_logits(output_type)
|
||||
self._art_model = ArtKerasClassifier(model, use_logits=logits)
|
||||
|
||||
def fit(self, train_data: Dataset, **kwargs) -> None:
|
||||
|
|
|
|||
|
|
@ -2,8 +2,9 @@ from abc import ABCMeta, abstractmethod
|
|||
from typing import Any, Optional, Callable, Tuple, Union, TYPE_CHECKING
|
||||
from enum import Enum, auto
|
||||
import numpy as np
|
||||
from scipy.special import expit
|
||||
|
||||
from apt.utils.datasets import Dataset, Data, OUTPUT_DATA_ARRAY_TYPE
|
||||
from apt.utils.datasets import Dataset, Data, DatasetWithPredictions, array2numpy, OUTPUT_DATA_ARRAY_TYPE
|
||||
from art.estimators.classification import BlackBoxClassifier
|
||||
from art.utils import check_and_transform_label_format
|
||||
|
||||
|
|
@ -12,9 +13,16 @@ if TYPE_CHECKING:
|
|||
|
||||
|
||||
class ModelOutputType(Enum):
|
||||
CLASSIFIER_PROBABILITIES = auto() # vector of probabilities
|
||||
CLASSIFIER_LOGITS = auto() # vector of logits
|
||||
CLASSIFIER_SCALAR = auto() # label only
|
||||
CLASSIFIER_SINGLE_OUTPUT_CATEGORICAL = auto() # class labels
|
||||
CLASSIFIER_SINGLE_OUTPUT_BINARY_PROBABILITIES = auto() # single binary probability
|
||||
CLASSIFIER_SINGLE_OUTPUT_CLASS_PROBABILITIES = auto() # vector of class probabilities
|
||||
CLASSIFIER_SINGLE_OUTPUT_BINARY_LOGITS = auto() # single binary logit
|
||||
CLASSIFIER_SINGLE_OUTPUT_CLASS_LOGITS = auto() # vector of logits
|
||||
CLASSIFIER_MULTI_OUTPUT_CATEGORICAL = auto() # vector of class labels
|
||||
CLASSIFIER_MULTI_OUTPUT_BINARY_PROBABILITIES = auto() # vector of binary probabilities, 1 per output
|
||||
CLASSIFIER_MULTI_OUTPUT_CLASS_PROBABILITIES = auto() # vector of class probabilities for multiple outputs
|
||||
CLASSIFIER_MULTI_OUTPUT_BINARY_LOGITS = auto() # vector of binary logits
|
||||
CLASSIFIER_MULTI_OUTPUT_CLASS_LOGITS = auto() # vector of logits for multiple outputs
|
||||
REGRESSOR_SCALAR = auto() # value
|
||||
|
||||
|
||||
|
|
@ -32,15 +40,44 @@ def is_one_hot(y: OUTPUT_DATA_ARRAY_TYPE) -> bool:
|
|||
return len(y.shape) == 2 and y.shape[1] > 1 and np.all(np.around(np.sum(y, axis=1), decimals=4) == 1)
|
||||
|
||||
|
||||
def is_multi_label(y: OUTPUT_DATA_ARRAY_TYPE) -> bool:
|
||||
return len(y.shape) == 2 and y.shape[1] > 1 and not is_one_hot(y)
|
||||
def is_multi_label(output_type: ModelOutputType) -> bool:
|
||||
return (output_type == ModelOutputType.CLASSIFIER_MULTI_OUTPUT_CATEGORICAL or
|
||||
output_type == ModelOutputType.CLASSIFIER_MULTI_OUTPUT_BINARY_PROBABILITIES or
|
||||
output_type == ModelOutputType.CLASSIFIER_MULTI_OUTPUT_CLASS_PROBABILITIES or
|
||||
output_type == ModelOutputType.CLASSIFIER_MULTI_OUTPUT_BINARY_LOGITS or
|
||||
output_type == ModelOutputType.CLASSIFIER_MULTI_OUTPUT_CLASS_LOGITS)
|
||||
|
||||
|
||||
def is_multi_label_binary(y: OUTPUT_DATA_ARRAY_TYPE) -> bool:
|
||||
return is_multi_label(y) and np.max(y) <= 1
|
||||
def is_multi_label_binary(output_type: ModelOutputType) -> bool:
|
||||
return (output_type == ModelOutputType.CLASSIFIER_MULTI_OUTPUT_BINARY_PROBABILITIES or
|
||||
output_type == ModelOutputType.CLASSIFIER_MULTI_OUTPUT_BINARY_LOGITS)
|
||||
|
||||
|
||||
def get_nb_classes(y: OUTPUT_DATA_ARRAY_TYPE) -> int:
|
||||
def is_binary(output_type: ModelOutputType) -> bool:
|
||||
return (output_type == ModelOutputType.CLASSIFIER_MULTI_OUTPUT_BINARY_PROBABILITIES or
|
||||
output_type == ModelOutputType.CLASSIFIER_MULTI_OUTPUT_BINARY_LOGITS or
|
||||
output_type == ModelOutputType.CLASSIFIER_SINGLE_OUTPUT_BINARY_PROBABILITIES or
|
||||
output_type == ModelOutputType.CLASSIFIER_SINGLE_OUTPUT_BINARY_LOGITS)
|
||||
|
||||
|
||||
def is_categorical(output_type: ModelOutputType) -> bool:
|
||||
return (output_type == ModelOutputType.CLASSIFIER_SINGLE_OUTPUT_CATEGORICAL or
|
||||
output_type == ModelOutputType.CLASSIFIER_MULTI_OUTPUT_CATEGORICAL)
|
||||
|
||||
|
||||
def is_probabilities(output_type: ModelOutputType) -> bool:
|
||||
return (output_type == ModelOutputType.CLASSIFIER_SINGLE_OUTPUT_CLASS_PROBABILITIES or
|
||||
output_type == ModelOutputType.CLASSIFIER_MULTI_OUTPUT_CLASS_PROBABILITIES)
|
||||
|
||||
|
||||
def is_logits(output_type: ModelOutputType) -> bool:
|
||||
return (output_type == ModelOutputType.CLASSIFIER_SINGLE_OUTPUT_CLASS_LOGITS or
|
||||
output_type == ModelOutputType.CLASSIFIER_MULTI_OUTPUT_CLASS_LOGITS or
|
||||
output_type == ModelOutputType.CLASSIFIER_SINGLE_OUTPUT_BINARY_LOGITS or
|
||||
output_type == ModelOutputType.CLASSIFIER_MULTI_OUTPUT_BINARY_LOGITS)
|
||||
|
||||
|
||||
def get_nb_classes(y: OUTPUT_DATA_ARRAY_TYPE, output_type: ModelOutputType) -> int:
|
||||
"""
|
||||
Get the number of classes from an array of labels
|
||||
|
||||
|
|
@ -56,12 +93,14 @@ def get_nb_classes(y: OUTPUT_DATA_ARRAY_TYPE) -> int:
|
|||
|
||||
if is_one_hot(y):
|
||||
return y.shape[1]
|
||||
elif is_multi_label(y):
|
||||
elif is_multi_label(output_type):
|
||||
# for now just return the number of labels
|
||||
return y.shape[1]
|
||||
# return [int(np.max(y.T[i]) + 1) for i in range(y.shape[1])]
|
||||
else:
|
||||
elif is_categorical(output_type):
|
||||
return int(np.max(y) + 1)
|
||||
else: # binary
|
||||
return 2
|
||||
|
||||
|
||||
def check_correct_model_output(y: OUTPUT_DATA_ARRAY_TYPE, output_type: ModelOutputType):
|
||||
|
|
@ -73,8 +112,8 @@ def check_correct_model_output(y: OUTPUT_DATA_ARRAY_TYPE, output_type: ModelOutp
|
|||
:type output_type: ModelOutputType
|
||||
:raises: ValueError (in case of mismatch)
|
||||
"""
|
||||
if not is_one_hot(y) and not is_multi_label(y): # 1D array
|
||||
if output_type == ModelOutputType.CLASSIFIER_PROBABILITIES or output_type == ModelOutputType.CLASSIFIER_LOGITS:
|
||||
if not is_one_hot(y) and (output_type == ModelOutputType.CLASSIFIER_SINGLE_OUTPUT_CLASS_PROBABILITIES or
|
||||
output_type == ModelOutputType.CLASSIFIER_SINGLE_OUTPUT_CLASS_LOGITS):
|
||||
raise ValueError("Incompatible model output types. Model outputs 1D array of categorical scalars while "
|
||||
"output type is set to ", output_type)
|
||||
|
||||
|
|
@ -127,16 +166,65 @@ class Model(metaclass=ABCMeta):
|
|||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def score(self, test_data: Dataset, **kwargs):
|
||||
"""
|
||||
Score the model using test data.
|
||||
|
||||
:param test_data: Test data.
|
||||
:type train_data: `Dataset`
|
||||
:type test_data: `Dataset`
|
||||
:param predictions: Model predictions to score. If provided, these will be used instead of calling the model's
|
||||
`predict` method.
|
||||
:type predictions: `DatasetWithPredictions` with the `pred` field filled.
|
||||
:param scoring_method: The method for scoring predictions. Default is ACCURACY.
|
||||
:type scoring_method: `ScoringMethod`, optional
|
||||
:param binary_threshold: The threshold to use on binary classification probabilities to assign the positive
|
||||
class.
|
||||
:type binary_threshold: float, optional. Default is 0.5.
|
||||
:param apply_non_linearity: A non-linear function to apply to the result of the 'predict' method, in case the
|
||||
model outputs logits (e.g., sigmoid).
|
||||
:type apply_non_linearity: Callable, should be possible to apply directly to the numpy output of the 'predict'
|
||||
method, optional.
|
||||
:param nb_classes: number of classes (for classification models).
|
||||
:type nb_classes: int, optional.
|
||||
:return: the score as float (for classifiers, between 0 and 1)
|
||||
"""
|
||||
raise NotImplementedError
|
||||
predictions = kwargs['predictions'] if 'predictions' in kwargs else None
|
||||
nb_classes = kwargs['nb_classes'] if 'nb_classes' in kwargs else None
|
||||
scoring_method = kwargs['scoring_method'] if 'scoring_method' in kwargs else ScoringMethod.ACCURACY
|
||||
binary_threshold = kwargs['binary_threshold'] if 'binary_threshold' in kwargs else 0.5
|
||||
apply_non_linearity = kwargs['apply_non_linearity'] if 'apply_non_linearity' in kwargs else expit
|
||||
|
||||
if test_data.get_samples() is None and predictions is None:
|
||||
raise ValueError('score can only be computed when test data or predictions are available')
|
||||
if test_data.get_labels() is None:
|
||||
raise ValueError('score can only be computed when labels are available')
|
||||
if predictions:
|
||||
predicted = predictions.get_predictions()
|
||||
else:
|
||||
predicted = self.predict(test_data)
|
||||
y = array2numpy(test_data.get_labels())
|
||||
|
||||
if scoring_method == ScoringMethod.ACCURACY:
|
||||
if not is_multi_label(self.output_type) and not is_binary(self.output_type) and nb_classes is not None:
|
||||
y = check_and_transform_label_format(y, nb_classes=nb_classes)
|
||||
if (self.output_type == ModelOutputType.CLASSIFIER_SINGLE_OUTPUT_CLASS_PROBABILITIES or
|
||||
self.output_type == ModelOutputType.CLASSIFIER_SINGLE_OUTPUT_CLASS_LOGITS or
|
||||
self.output_type == ModelOutputType.CLASSIFIER_SINGLE_OUTPUT_CATEGORICAL):
|
||||
# categorical has been 1-hot encoded by check_and_transform_label_format
|
||||
return np.count_nonzero(np.argmax(y, axis=1) == np.argmax(predicted, axis=1)) / predicted.shape[0]
|
||||
elif is_binary(self.output_type):
|
||||
if is_logits(self.output_type):
|
||||
if apply_non_linearity:
|
||||
predicted = apply_non_linearity(predicted)
|
||||
else: # apply sigmoid
|
||||
predicted = expit(predicted)
|
||||
predicted[predicted < binary_threshold] = 0
|
||||
predicted[predicted >= binary_threshold] = 1
|
||||
return np.count_nonzero(y == predicted) / (predicted.shape[0] * y.shape[1])
|
||||
else:
|
||||
raise NotImplementedError('score method not implemented for output type: ', self.output_type)
|
||||
else:
|
||||
raise NotImplementedError('scoring method not implemented: ', scoring_method)
|
||||
|
||||
@property
|
||||
def model(self) -> Any:
|
||||
|
|
@ -260,6 +348,13 @@ class BlackboxClassifier(Model):
|
|||
"""
|
||||
return self._optimizer
|
||||
|
||||
def score(self, test_data: Dataset, **kwargs):
|
||||
"""
|
||||
Score the model using test data.
|
||||
"""
|
||||
kwargs ['nb_classes'] = self.nb_classes
|
||||
return super().score(test_data, **kwargs)
|
||||
|
||||
def fit(self, train_data: Dataset, **kwargs) -> None:
|
||||
"""
|
||||
A blackbox model cannot be fit.
|
||||
|
|
@ -279,37 +374,6 @@ class BlackboxClassifier(Model):
|
|||
check_correct_model_output(predictions, self.output_type)
|
||||
return predictions
|
||||
|
||||
def score(self, test_data: Dataset, scoring_method: Optional[ScoringMethod] = ScoringMethod.ACCURACY,
|
||||
binary_threshold: Optional[float] = 0.5, **kwargs):
|
||||
"""
|
||||
Score the model using test data.
|
||||
|
||||
:param test_data: Test data.
|
||||
:type train_data: `Dataset`
|
||||
:param scoring_method: The method for scoring predictions. Default is ACCURACY.
|
||||
:type scoring_method: `ScoringMethod`, optional
|
||||
:param binary_threshold: The threshold to use on binary classification probabilities to assign the positive
|
||||
class.
|
||||
:type binary_threshold: float, optional. Default is 0.5.
|
||||
:return: the score as float (for classifiers, between 0 and 1)
|
||||
"""
|
||||
if test_data.get_samples() is None or test_data.get_labels() is None:
|
||||
raise ValueError('score can only be computed when test data and labels are available')
|
||||
predicted = self._art_model.predict(test_data.get_samples())
|
||||
y = test_data.get_labels()
|
||||
if not is_multi_label(y):
|
||||
y = check_and_transform_label_format(y, nb_classes=self._nb_classes)
|
||||
if scoring_method == ScoringMethod.ACCURACY:
|
||||
if not is_multi_label(y):
|
||||
return np.count_nonzero(np.argmax(y, axis=1) == np.argmax(predicted, axis=1)) / predicted.shape[0]
|
||||
else:
|
||||
if is_multi_label_binary(y):
|
||||
predicted[predicted < binary_threshold] = 0
|
||||
predicted[predicted >= binary_threshold] = 1
|
||||
return np.count_nonzero(y == predicted) / (predicted.shape[0] * y.shape[1])
|
||||
else:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def get_predictions(self) -> Union[Callable, Tuple[OUTPUT_DATA_ARRAY_TYPE, OUTPUT_DATA_ARRAY_TYPE]]:
|
||||
"""
|
||||
|
|
@ -356,11 +420,11 @@ class BlackboxClassifierPredictions(BlackboxClassifier):
|
|||
check_correct_model_output(y_test_pred, self.output_type)
|
||||
|
||||
if y_train_pred is not None and len(y_train_pred.shape) == 1:
|
||||
self._nb_classes = get_nb_classes(y_train_pred)
|
||||
self._nb_classes = get_nb_classes(y_train_pred, self.output_type)
|
||||
y_train_pred = check_and_transform_label_format(y_train_pred, nb_classes=self._nb_classes)
|
||||
if y_test_pred is not None and len(y_test_pred.shape) == 1:
|
||||
if self._nb_classes is None:
|
||||
self._nb_classes = get_nb_classes(y_test_pred)
|
||||
self._nb_classes = get_nb_classes(y_test_pred, self.output_type)
|
||||
y_test_pred = check_and_transform_label_format(y_test_pred, nb_classes=self._nb_classes)
|
||||
|
||||
if x_train_pred is not None and y_train_pred is not None and x_test_pred is not None and y_test_pred is not None:
|
||||
|
|
@ -378,7 +442,7 @@ class BlackboxClassifierPredictions(BlackboxClassifier):
|
|||
else:
|
||||
raise NotImplementedError("Invalid data - None")
|
||||
|
||||
self._nb_classes = get_nb_classes(y_pred)
|
||||
self._nb_classes = get_nb_classes(y_pred, self.output_type)
|
||||
self._input_shape = x_pred.shape[1:]
|
||||
self._x_pred = x_pred
|
||||
self._y_pred = y_pred
|
||||
|
|
|
|||
|
|
@ -7,9 +7,10 @@ from typing import Optional, Tuple, Union, List, Callable
|
|||
import numpy as np
|
||||
import torch
|
||||
from torch.utils.data import DataLoader, TensorDataset
|
||||
from collections.abc import Iterable
|
||||
|
||||
from art.utils import check_and_transform_label_format
|
||||
from apt.utils.datasets.datasets import PytorchData
|
||||
from apt.utils.datasets.datasets import PytorchData, DatasetWithPredictions, ArrayDataset
|
||||
from apt.utils.models import Model, ModelOutputType, is_multi_label, is_multi_label_binary
|
||||
from apt.utils.datasets import OUTPUT_DATA_ARRAY_TYPE
|
||||
from art.estimators.classification.pytorch import PyTorchClassifier as ArtPyTorchClassifier
|
||||
|
|
@ -36,6 +37,7 @@ class PyTorchClassifierWrapper(ArtPyTorchClassifier):
|
|||
loss: "torch.nn.modules.loss._Loss",
|
||||
input_shape: Tuple[int, ...],
|
||||
nb_classes: int,
|
||||
output_type: ModelOutputType,
|
||||
optimizer: Optional["torch.optim.Optimizer"] = None, # type: ignore
|
||||
use_amp: bool = False,
|
||||
opt_level: str = "O1",
|
||||
|
|
@ -50,9 +52,10 @@ class PyTorchClassifierWrapper(ArtPyTorchClassifier):
|
|||
super().__init__(model, loss, input_shape, nb_classes, optimizer, use_amp, opt_level, loss_scale,
|
||||
channels_first, clip_values, preprocessing_defences, postprocessing_defences, preprocessing,
|
||||
device_type)
|
||||
self._is_single_binary = None
|
||||
self._is_multi_label = None
|
||||
self._is_multi_label_binary = None
|
||||
self._is_single_binary = (output_type == ModelOutputType.CLASSIFIER_SINGLE_OUTPUT_BINARY_PROBABILITIES or
|
||||
output_type == ModelOutputType.CLASSIFIER_SINGLE_OUTPUT_BINARY_LOGITS)
|
||||
self._is_multi_label = is_multi_label(output_type)
|
||||
self._is_multi_label_binary = is_multi_label_binary(output_type)
|
||||
|
||||
def get_step_correct(self, outputs, targets) -> int:
|
||||
"""
|
||||
|
|
@ -122,9 +125,6 @@ class PyTorchClassifierWrapper(ArtPyTorchClassifier):
|
|||
:param kwargs: Dictionary of framework-specific arguments. This parameter is not currently
|
||||
supported for PyTorch and providing it takes no effect.
|
||||
"""
|
||||
self._is_single_binary = self.nb_classes == 2 and (len(y.shape) == 1 or y.shape[1] == 1)
|
||||
self._is_multi_label = is_multi_label(y)
|
||||
self._is_multi_label_binary = is_multi_label_binary(y)
|
||||
|
||||
# Put the model in the training mode
|
||||
self._model.train()
|
||||
|
|
@ -189,6 +189,58 @@ class PyTorchClassifierWrapper(ArtPyTorchClassifier):
|
|||
else:
|
||||
self.save_checkpoint_state_dict(is_best=best_acc <= val_acc, path=path)
|
||||
|
||||
def predict(
|
||||
self, x: np.ndarray, batch_size: int = 128, training_mode: bool = False, **kwargs
|
||||
) -> np.ndarray:
|
||||
"""
|
||||
Perform prediction for a batch of inputs.
|
||||
|
||||
:param x: Input samples.
|
||||
:param batch_size: Size of batches.
|
||||
:param training_mode: `True` for model set to training mode and `'False` for model set to evaluation mode.
|
||||
:return: Array of predictions of shape `(nb_inputs, nb_classes)`.
|
||||
"""
|
||||
import torch
|
||||
|
||||
# Set model mode
|
||||
self._model.train(mode=training_mode)
|
||||
|
||||
# Apply preprocessing
|
||||
x_preprocessed, _ = self._apply_preprocessing(x, y=None, fit=False)
|
||||
|
||||
results_list = []
|
||||
|
||||
# Run prediction with batch processing
|
||||
num_batch = int(np.ceil(len(x_preprocessed) / float(batch_size)))
|
||||
for m in range(num_batch):
|
||||
# Batch indexes
|
||||
begin, end = (
|
||||
m * batch_size,
|
||||
min((m + 1) * batch_size, x_preprocessed.shape[0]),
|
||||
)
|
||||
|
||||
with torch.no_grad():
|
||||
model_outputs = self._model(torch.from_numpy(x_preprocessed[begin:end]).to(self._device))
|
||||
output = model_outputs[-1]
|
||||
if isinstance(output, Iterable):
|
||||
for i, o in enumerate(output):
|
||||
o = o.detach().cpu().numpy().astype(np.float32)
|
||||
# if len(output.shape) == 1:
|
||||
# o = np.expand_dims(o, axis=1).astype(np.float32)
|
||||
else:
|
||||
output = output.detach().cpu().numpy().astype(np.float32)
|
||||
if len(output.shape) == 1:
|
||||
output = np.expand_dims(output, axis=1).astype(np.float32)
|
||||
|
||||
results_list.append(output)
|
||||
|
||||
results = np.vstack(results_list)
|
||||
|
||||
# Apply postprocessing
|
||||
predictions = self._apply_postprocessing(preds=results, fit=False)
|
||||
|
||||
return predictions
|
||||
|
||||
def save_checkpoint_state_dict(self, is_best: bool, path=os.getcwd(), filename="latest.tar") -> None:
|
||||
"""
|
||||
Saves checkpoint as latest.tar or best.tar.
|
||||
|
|
@ -352,7 +404,8 @@ class PyTorchClassifier(PyTorchModel):
|
|||
super().__init__(model, output_type, black_box_access, unlimited_queries, **kwargs)
|
||||
self._loss = loss
|
||||
self._optimizer = optimizer
|
||||
self._art_model = PyTorchClassifierWrapper(model, loss, input_shape, nb_classes, optimizer)
|
||||
self._nb_classes = nb_classes
|
||||
self._art_model = PyTorchClassifierWrapper(model, loss, input_shape, nb_classes, output_type, optimizer)
|
||||
|
||||
@property
|
||||
def loss(self):
|
||||
|
|
@ -433,8 +486,7 @@ class PyTorchClassifier(PyTorchModel):
|
|||
"""
|
||||
return self._art_model.predict(x.get_samples(), **kwargs)
|
||||
|
||||
def score(self, test_data: PytorchData, binary_threshold: Optional[float] = 0.5,
|
||||
apply_non_linearity: Optional[Callable] = None, **kwargs):
|
||||
def score(self, test_data: PytorchData, **kwargs):
|
||||
"""
|
||||
Score the model using test data.
|
||||
|
||||
|
|
@ -452,23 +504,26 @@ class PyTorchClassifier(PyTorchModel):
|
|||
# numpy arrays
|
||||
y = test_data.get_labels()
|
||||
predicted = self.predict(test_data)
|
||||
if apply_non_linearity:
|
||||
predicted = apply_non_linearity(predicted)
|
||||
# binary classification, single column of probabilities
|
||||
if self._art_model.nb_classes == 2 and (len(predicted.shape) == 1 or predicted.shape[1] == 1):
|
||||
if len(predicted.shape) > 1:
|
||||
y = check_and_transform_label_format(y, self._art_model.nb_classes, return_one_hot=False)
|
||||
return np.count_nonzero(y == (predicted > binary_threshold)) / predicted.shape[0]
|
||||
# multi column
|
||||
else:
|
||||
if not is_multi_label(y):
|
||||
y = check_and_transform_label_format(y, self._art_model.nb_classes)
|
||||
return np.count_nonzero(np.argmax(y, axis=1) == np.argmax(predicted, axis=1)) / predicted.shape[0]
|
||||
else:
|
||||
if is_multi_label_binary(y):
|
||||
predicted[predicted < binary_threshold] = 0
|
||||
predicted[predicted >= binary_threshold] = 1
|
||||
return np.count_nonzero(y == predicted) / (predicted.shape[0] * y.shape[1])
|
||||
kwargs['predictions'] = DatasetWithPredictions(pred=predicted)
|
||||
kwargs['nb_classes'] = self._nb_classes
|
||||
return super().score(ArrayDataset(test_data.get_samples(), test_data.get_labels()), **kwargs)
|
||||
# if apply_non_linearity:
|
||||
# predicted = apply_non_linearity(predicted)
|
||||
# # binary classification, single column of probabilities
|
||||
# if self._art_model.nb_classes == 2 and (len(predicted.shape) == 1 or predicted.shape[1] == 1):
|
||||
# if len(predicted.shape) > 1:
|
||||
# y = check_and_transform_label_format(y, self._art_model.nb_classes, return_one_hot=False)
|
||||
# return np.count_nonzero(y == (predicted > binary_threshold)) / predicted.shape[0]
|
||||
# # multi column
|
||||
# else:
|
||||
# if not is_multi_label(y):
|
||||
# y = check_and_transform_label_format(y, self._art_model.nb_classes)
|
||||
# return np.count_nonzero(np.argmax(y, axis=1) == np.argmax(predicted, axis=1)) / predicted.shape[0]
|
||||
# else:
|
||||
# if is_multi_label_binary(y):
|
||||
# predicted[predicted < binary_threshold] = 0
|
||||
# predicted[predicted >= binary_threshold] = 1
|
||||
# return np.count_nonzero(y == predicted) / (predicted.shape[0] * y.shape[1])
|
||||
|
||||
|
||||
def load_checkpoint_state_dict_by_path(self, model_name: str, path: str = None):
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ from typing import Optional
|
|||
from sklearn.base import BaseEstimator
|
||||
|
||||
from apt.utils.models import Model, ModelOutputType, get_nb_classes, check_correct_model_output
|
||||
from apt.utils.datasets import Dataset, OUTPUT_DATA_ARRAY_TYPE
|
||||
from apt.utils.datasets import Dataset, ArrayDataset, OUTPUT_DATA_ARRAY_TYPE
|
||||
|
||||
from art.estimators.classification.scikitlearn import SklearnClassifier as ArtSklearnClassifier
|
||||
from art.estimators.regression.scikitlearn import ScikitlearnRegressor
|
||||
|
|
@ -48,7 +48,7 @@ class SklearnClassifier(SklearnModel):
|
|||
super().__init__(model, output_type, black_box_access, unlimited_queries, **kwargs)
|
||||
self._art_model = ArtSklearnClassifier(model, preprocessing=None)
|
||||
|
||||
def fit(self, train_data: Dataset, **kwargs) -> None:
|
||||
def fit(self, train_data: ArrayDataset, **kwargs) -> None:
|
||||
"""
|
||||
Fit the model using the training data.
|
||||
|
||||
|
|
@ -58,11 +58,11 @@ class SklearnClassifier(SklearnModel):
|
|||
:return: None
|
||||
"""
|
||||
y = train_data.get_labels()
|
||||
self.nb_classes = get_nb_classes(y)
|
||||
self.nb_classes = get_nb_classes(y, self.output_type)
|
||||
y_encoded = check_and_transform_label_format(y, nb_classes=self.nb_classes)
|
||||
self._art_model.fit(train_data.get_samples(), y_encoded, **kwargs)
|
||||
|
||||
def predict(self, x: Dataset, **kwargs) -> OUTPUT_DATA_ARRAY_TYPE:
|
||||
def predict(self, x: ArrayDataset, **kwargs) -> OUTPUT_DATA_ARRAY_TYPE:
|
||||
"""
|
||||
Perform predictions using the model for input `x`.
|
||||
|
||||
|
|
|
|||
|
|
@ -216,7 +216,7 @@ def test_minimizer_params(cells):
|
|||
|
||||
base_est = DecisionTreeClassifier(random_state=0, min_samples_split=2,
|
||||
min_samples_leaf=1)
|
||||
model = SklearnClassifier(base_est, ModelOutputType.CLASSIFIER_PROBABILITIES)
|
||||
model = SklearnClassifier(base_est, ModelOutputType.CLASSIFIER_SINGLE_OUTPUT_CLASS_PROBABILITIES)
|
||||
model.fit(ArrayDataset(x, y))
|
||||
|
||||
expected_generalizations = {'categories': {}, 'category_representatives': {},
|
||||
|
|
@ -258,7 +258,7 @@ def test_minimizer_params_not_transform(cells):
|
|||
samples = ArrayDataset(x, y, features)
|
||||
base_est = DecisionTreeClassifier(random_state=0, min_samples_split=2,
|
||||
min_samples_leaf=1)
|
||||
model = SklearnClassifier(base_est, ModelOutputType.CLASSIFIER_PROBABILITIES)
|
||||
model = SklearnClassifier(base_est, ModelOutputType.CLASSIFIER_SINGLE_OUTPUT_CLASS_PROBABILITIES)
|
||||
model.fit(ArrayDataset(x, y))
|
||||
|
||||
gen = GeneralizeToRepresentative(model, cells=cells, generalize_using_transform=False)
|
||||
|
|
@ -270,7 +270,7 @@ def test_minimizer_fit(data_two_features):
|
|||
x, y, features, _ = data_two_features
|
||||
base_est = DecisionTreeClassifier(random_state=0, min_samples_split=2,
|
||||
min_samples_leaf=1)
|
||||
model = SklearnClassifier(base_est, ModelOutputType.CLASSIFIER_PROBABILITIES)
|
||||
model = SklearnClassifier(base_est, ModelOutputType.CLASSIFIER_SINGLE_OUTPUT_CLASS_PROBABILITIES)
|
||||
model.fit(ArrayDataset(x, y))
|
||||
ad = ArrayDataset(x)
|
||||
predictions = model.predict(ad)
|
||||
|
|
@ -299,7 +299,7 @@ def test_minimizer_ncp(data_two_features):
|
|||
|
||||
base_est = DecisionTreeClassifier(random_state=0, min_samples_split=2,
|
||||
min_samples_leaf=1)
|
||||
model = SklearnClassifier(base_est, ModelOutputType.CLASSIFIER_PROBABILITIES)
|
||||
model = SklearnClassifier(base_est, ModelOutputType.CLASSIFIER_SINGLE_OUTPUT_CLASS_PROBABILITIES)
|
||||
model.fit(ArrayDataset(x, y))
|
||||
ad = ArrayDataset(x)
|
||||
ad1 = ArrayDataset(x1, features_names=features)
|
||||
|
|
@ -342,7 +342,7 @@ def test_minimizer_ncp_categorical(data_four_features):
|
|||
|
||||
base_est = DecisionTreeClassifier(random_state=0, min_samples_split=2,
|
||||
min_samples_leaf=1)
|
||||
model = SklearnClassifier(base_est, ModelOutputType.CLASSIFIER_PROBABILITIES)
|
||||
model = SklearnClassifier(base_est, ModelOutputType.CLASSIFIER_SINGLE_OUTPUT_CLASS_PROBABILITIES)
|
||||
model.fit(ArrayDataset(encoded, y))
|
||||
ad = ArrayDataset(x)
|
||||
ad1 = ArrayDataset(x1)
|
||||
|
|
@ -382,7 +382,7 @@ def test_minimizer_fit_not_transform(data_two_features):
|
|||
x, y, features, x1 = data_two_features
|
||||
base_est = DecisionTreeClassifier(random_state=0, min_samples_split=2,
|
||||
min_samples_leaf=1)
|
||||
model = SklearnClassifier(base_est, ModelOutputType.CLASSIFIER_PROBABILITIES)
|
||||
model = SklearnClassifier(base_est, ModelOutputType.CLASSIFIER_SINGLE_OUTPUT_CLASS_PROBABILITIES)
|
||||
model.fit(ArrayDataset(x, y))
|
||||
ad = ArrayDataset(x)
|
||||
predictions = model.predict(ad)
|
||||
|
|
@ -412,7 +412,7 @@ def test_minimizer_fit_pandas(data_four_features):
|
|||
|
||||
base_est = DecisionTreeClassifier(random_state=0, min_samples_split=2,
|
||||
min_samples_leaf=1)
|
||||
model = SklearnClassifier(base_est, ModelOutputType.CLASSIFIER_PROBABILITIES)
|
||||
model = SklearnClassifier(base_est, ModelOutputType.CLASSIFIER_SINGLE_OUTPUT_CLASS_PROBABILITIES)
|
||||
model.fit(ArrayDataset(encoded, y))
|
||||
predictions = model.predict(ArrayDataset(encoded))
|
||||
if predictions.shape[1] > 1:
|
||||
|
|
@ -450,7 +450,7 @@ def test_minimizer_params_categorical(cells_categorical):
|
|||
preprocessor, encoded = create_encoder(numeric_features, categorical_features, x)
|
||||
base_est = DecisionTreeClassifier(random_state=0, min_samples_split=2,
|
||||
min_samples_leaf=1)
|
||||
model = SklearnClassifier(base_est, ModelOutputType.CLASSIFIER_PROBABILITIES)
|
||||
model = SklearnClassifier(base_est, ModelOutputType.CLASSIFIER_SINGLE_OUTPUT_CLASS_PROBABILITIES)
|
||||
model.fit(ArrayDataset(encoded, y))
|
||||
predictions = model.predict(ArrayDataset(encoded))
|
||||
if predictions.shape[1] > 1:
|
||||
|
|
@ -474,7 +474,7 @@ def test_minimizer_fit_qi(data_three_features):
|
|||
qi = ['age', 'weight']
|
||||
base_est = DecisionTreeClassifier(random_state=0, min_samples_split=2,
|
||||
min_samples_leaf=1)
|
||||
model = SklearnClassifier(base_est, ModelOutputType.CLASSIFIER_PROBABILITIES)
|
||||
model = SklearnClassifier(base_est, ModelOutputType.CLASSIFIER_SINGLE_OUTPUT_CLASS_PROBABILITIES)
|
||||
model.fit(ArrayDataset(x, y))
|
||||
ad = ArrayDataset(x)
|
||||
predictions = model.predict(ad)
|
||||
|
|
@ -508,7 +508,7 @@ def test_minimizer_fit_pandas_qi(data_five_features):
|
|||
|
||||
base_est = DecisionTreeClassifier(random_state=0, min_samples_split=2,
|
||||
min_samples_leaf=1)
|
||||
model = SklearnClassifier(base_est, ModelOutputType.CLASSIFIER_PROBABILITIES)
|
||||
model = SklearnClassifier(base_est, ModelOutputType.CLASSIFIER_SINGLE_OUTPUT_CLASS_PROBABILITIES)
|
||||
model.fit(ArrayDataset(encoded, y))
|
||||
predictions = model.predict(ArrayDataset(encoded))
|
||||
if predictions.shape[1] > 1:
|
||||
|
|
@ -543,7 +543,7 @@ def test_minimize_ndarray_iris():
|
|||
qi = ['sepal length (cm)', 'petal length (cm)']
|
||||
base_est = DecisionTreeClassifier(random_state=0, min_samples_split=2,
|
||||
min_samples_leaf=1)
|
||||
model = SklearnClassifier(base_est, ModelOutputType.CLASSIFIER_PROBABILITIES)
|
||||
model = SklearnClassifier(base_est, ModelOutputType.CLASSIFIER_SINGLE_OUTPUT_CATEGORICAL)
|
||||
model.fit(ArrayDataset(x_train, y_train))
|
||||
predictions = model.predict(ArrayDataset(x_train))
|
||||
if predictions.shape[1] > 1:
|
||||
|
|
@ -586,7 +586,7 @@ def test_minimize_pandas_adult():
|
|||
|
||||
base_est = DecisionTreeClassifier(random_state=0, min_samples_split=2,
|
||||
min_samples_leaf=1)
|
||||
model = SklearnClassifier(base_est, ModelOutputType.CLASSIFIER_PROBABILITIES)
|
||||
model = SklearnClassifier(base_est, ModelOutputType.CLASSIFIER_SINGLE_OUTPUT_CLASS_PROBABILITIES)
|
||||
model.fit(ArrayDataset(encoded, y_train))
|
||||
predictions = model.predict(ArrayDataset(encoded))
|
||||
if predictions.shape[1] > 1:
|
||||
|
|
@ -642,7 +642,7 @@ def test_german_credit_pandas():
|
|||
|
||||
base_est = DecisionTreeClassifier(random_state=0, min_samples_split=2,
|
||||
min_samples_leaf=1)
|
||||
model = SklearnClassifier(base_est, ModelOutputType.CLASSIFIER_PROBABILITIES)
|
||||
model = SklearnClassifier(base_est, ModelOutputType.CLASSIFIER_SINGLE_OUTPUT_CLASS_PROBABILITIES)
|
||||
model.fit(ArrayDataset(encoded, y_train))
|
||||
predictions = model.predict(ArrayDataset(encoded))
|
||||
if predictions.shape[1] > 1:
|
||||
|
|
@ -760,7 +760,7 @@ def test_x_y():
|
|||
qi = [0, 2]
|
||||
base_est = DecisionTreeClassifier(random_state=0, min_samples_split=2,
|
||||
min_samples_leaf=1)
|
||||
model = SklearnClassifier(base_est, ModelOutputType.CLASSIFIER_PROBABILITIES)
|
||||
model = SklearnClassifier(base_est, ModelOutputType.CLASSIFIER_SINGLE_OUTPUT_CLASS_PROBABILITIES)
|
||||
model.fit(ArrayDataset(x, y))
|
||||
ad = ArrayDataset(x)
|
||||
predictions = model.predict(ad)
|
||||
|
|
@ -800,7 +800,7 @@ def test_x_y_features_names():
|
|||
qi = ['age', 'weight']
|
||||
base_est = DecisionTreeClassifier(random_state=0, min_samples_split=2,
|
||||
min_samples_leaf=1)
|
||||
model = SklearnClassifier(base_est, ModelOutputType.CLASSIFIER_PROBABILITIES)
|
||||
model = SklearnClassifier(base_est, ModelOutputType.CLASSIFIER_SINGLE_OUTPUT_CLASS_PROBABILITIES)
|
||||
model.fit(ArrayDataset(x, y))
|
||||
ad = ArrayDataset(x)
|
||||
predictions = model.predict(ad)
|
||||
|
|
@ -1202,7 +1202,7 @@ def test_keras_model():
|
|||
|
||||
base_est.compile(loss="categorical_crossentropy", optimizer="adam", metrics=["accuracy"])
|
||||
|
||||
model = KerasClassifier(base_est, ModelOutputType.CLASSIFIER_PROBABILITIES)
|
||||
model = KerasClassifier(base_est, ModelOutputType.CLASSIFIER_SINGLE_OUTPUT_CLASS_PROBABILITIES)
|
||||
model.fit(ArrayDataset(x, y))
|
||||
ad = ArrayDataset(x_test)
|
||||
predictions = model.predict(ad)
|
||||
|
|
@ -1269,8 +1269,11 @@ def test_minimizer_pytorch(data_three_features):
|
|||
criterion = nn.CrossEntropyLoss()
|
||||
optimizer = optim.Adam(base_est.parameters(), lr=0.01)
|
||||
|
||||
model = PyTorchClassifier(model=base_est, output_type=ModelOutputType.CLASSIFIER_LOGITS, loss=criterion,
|
||||
optimizer=optimizer, input_shape=(3,),
|
||||
model = PyTorchClassifier(model=base_est,
|
||||
output_type=ModelOutputType.CLASSIFIER_SINGLE_OUTPUT_CLASS_LOGITS,
|
||||
loss=criterion,
|
||||
optimizer=optimizer,
|
||||
input_shape=(3,),
|
||||
nb_classes=2)
|
||||
model.fit(PytorchData(x, y), save_entire_model=False, nb_epochs=10)
|
||||
|
||||
|
|
@ -1308,8 +1311,11 @@ def test_minimizer_pytorch_iris():
|
|||
criterion = nn.CrossEntropyLoss()
|
||||
optimizer = optim.Adam(base_est.parameters(), lr=0.01)
|
||||
|
||||
model = PyTorchClassifier(model=base_est, output_type=ModelOutputType.CLASSIFIER_LOGITS, loss=criterion,
|
||||
optimizer=optimizer, input_shape=(4,),
|
||||
model = PyTorchClassifier(model=base_est,
|
||||
output_type=ModelOutputType.CLASSIFIER_SINGLE_OUTPUT_CLASS_LOGITS,
|
||||
loss=criterion,
|
||||
optimizer=optimizer,
|
||||
input_shape=(4,),
|
||||
nb_classes=3)
|
||||
model.fit(PytorchData(x_train, y_train), save_entire_model=False, nb_epochs=10)
|
||||
|
||||
|
|
@ -1362,7 +1368,7 @@ def test_errors():
|
|||
y = np.array([1, 1, 0, 1, 0, 0, 1, 1, 0, 1, 0])
|
||||
base_est = DecisionTreeClassifier(random_state=0, min_samples_split=2,
|
||||
min_samples_leaf=1)
|
||||
model = SklearnClassifier(base_est, ModelOutputType.CLASSIFIER_PROBABILITIES)
|
||||
model = SklearnClassifier(base_est, ModelOutputType.CLASSIFIER_SINGLE_OUTPUT_CLASS_PROBABILITIES)
|
||||
model.fit(ArrayDataset(X, y))
|
||||
ad = ArrayDataset(X)
|
||||
predictions = model.predict(ad)
|
||||
|
|
|
|||
|
|
@ -24,7 +24,7 @@ tf.compat.v1.disable_eager_execution()
|
|||
def test_sklearn_classifier():
|
||||
(x_train, y_train), (x_test, y_test) = dataset_utils.get_iris_dataset_np()
|
||||
underlying_model = RandomForestClassifier()
|
||||
model = SklearnClassifier(underlying_model, ModelOutputType.CLASSIFIER_PROBABILITIES)
|
||||
model = SklearnClassifier(underlying_model, ModelOutputType.CLASSIFIER_SINGLE_OUTPUT_CATEGORICAL)
|
||||
train = ArrayDataset(x_train, y_train)
|
||||
test = ArrayDataset(x_test, y_test)
|
||||
model.fit(train)
|
||||
|
|
@ -59,7 +59,7 @@ def test_keras_classifier():
|
|||
|
||||
underlying_model.compile(loss="categorical_crossentropy", optimizer="adam", metrics=["accuracy"])
|
||||
|
||||
model = KerasClassifier(underlying_model, ModelOutputType.CLASSIFIER_PROBABILITIES)
|
||||
model = KerasClassifier(underlying_model, ModelOutputType.CLASSIFIER_SINGLE_OUTPUT_CATEGORICAL)
|
||||
|
||||
train = ArrayDataset(x_train, y_train)
|
||||
test = ArrayDataset(x_test, y_test)
|
||||
|
|
@ -97,7 +97,8 @@ def test_xgboost_classifier():
|
|||
(x_train, y_train), (x_test, y_test) = dataset_utils.get_iris_dataset_np()
|
||||
underlying_model = XGBClassifier()
|
||||
underlying_model.fit(x_train, y_train)
|
||||
model = XGBoostClassifier(underlying_model, ModelOutputType.CLASSIFIER_PROBABILITIES, input_shape=(4,), nb_classes=3)
|
||||
model = XGBoostClassifier(underlying_model, ModelOutputType.CLASSIFIER_SINGLE_OUTPUT_CATEGORICAL,
|
||||
input_shape=(4,), nb_classes=3)
|
||||
train = ArrayDataset(x_train, y_train)
|
||||
test = ArrayDataset(x_test, y_test)
|
||||
pred = model.predict(test)
|
||||
|
|
@ -115,7 +116,7 @@ def test_blackbox_classifier():
|
|||
train = ArrayDataset(x_train, y_train)
|
||||
test = ArrayDataset(x_test, y_test)
|
||||
data = Data(train, test)
|
||||
model = BlackboxClassifierPredictions(data, ModelOutputType.CLASSIFIER_SCALAR)
|
||||
model = BlackboxClassifierPredictions(data, ModelOutputType.CLASSIFIER_SINGLE_OUTPUT_CATEGORICAL)
|
||||
pred = model.predict(test)
|
||||
assert (pred.shape[0] == x_test.shape[0])
|
||||
|
||||
|
|
@ -131,7 +132,7 @@ def test_blackbox_classifier_predictions():
|
|||
train = DatasetWithPredictions(y_train, x_train)
|
||||
test = DatasetWithPredictions(y_test, x_test)
|
||||
data = Data(train, test)
|
||||
model = BlackboxClassifierPredictions(data, ModelOutputType.CLASSIFIER_SCALAR)
|
||||
model = BlackboxClassifierPredictions(data, ModelOutputType.CLASSIFIER_SINGLE_OUTPUT_CATEGORICAL)
|
||||
pred = model.predict(test)
|
||||
assert (pred.shape[0] == x_test.shape[0])
|
||||
assert model.model_type is None
|
||||
|
|
@ -146,7 +147,7 @@ def test_blackbox_classifier_predictions_y():
|
|||
train = DatasetWithPredictions(y_train, x_train, y_train)
|
||||
test = DatasetWithPredictions(y_test, x_test, y_test)
|
||||
data = Data(train, test)
|
||||
model = BlackboxClassifierPredictions(data, ModelOutputType.CLASSIFIER_SCALAR)
|
||||
model = BlackboxClassifierPredictions(data, ModelOutputType.CLASSIFIER_SINGLE_OUTPUT_CATEGORICAL)
|
||||
pred = model.predict(test)
|
||||
assert (pred.shape[0] == x_test.shape[0])
|
||||
|
||||
|
|
@ -166,7 +167,7 @@ def test_blackbox_classifier_predictions_multi_label_cat():
|
|||
train = DatasetWithPredictions(y_train, x_train, y_train)
|
||||
test = DatasetWithPredictions(y_test, x_test, y_test)
|
||||
data = Data(train, test)
|
||||
model = BlackboxClassifierPredictions(data, ModelOutputType.CLASSIFIER_SCALAR)
|
||||
model = BlackboxClassifierPredictions(data, ModelOutputType.CLASSIFIER_SINGLE_OUTPUT_CATEGORICAL)
|
||||
pred = model.predict(test)
|
||||
assert (pred.shape[0] == x_test.shape[0])
|
||||
|
||||
|
|
@ -194,7 +195,7 @@ def test_blackbox_classifier_predictions_multi_label_binary():
|
|||
train = DatasetWithPredictions(pred_train, x_train, y_train)
|
||||
test = DatasetWithPredictions(pred_test, x_test, y_test)
|
||||
data = Data(train, test)
|
||||
model = BlackboxClassifierPredictions(data, ModelOutputType.CLASSIFIER_SCALAR)
|
||||
model = BlackboxClassifierPredictions(data, ModelOutputType.CLASSIFIER_MULTI_OUTPUT_BINARY_PROBABILITIES)
|
||||
pred = model.predict(test)
|
||||
assert (pred.shape[0] == x_test.shape[0])
|
||||
|
||||
|
|
@ -211,7 +212,7 @@ def test_blackbox_classifier_mismatch():
|
|||
test = ArrayDataset(x_test, y_test)
|
||||
data = Data(train, test)
|
||||
with pytest.raises(ValueError):
|
||||
BlackboxClassifierPredictions(data, ModelOutputType.CLASSIFIER_PROBABILITIES)
|
||||
BlackboxClassifierPredictions(data, ModelOutputType.CLASSIFIER_SINGLE_OUTPUT_CLASS_PROBABILITIES)
|
||||
|
||||
|
||||
def test_blackbox_classifier_no_test():
|
||||
|
|
@ -220,7 +221,7 @@ def test_blackbox_classifier_no_test():
|
|||
train = ArrayDataset(x_train, y_train)
|
||||
|
||||
data = Data(train)
|
||||
model = BlackboxClassifierPredictions(data, ModelOutputType.CLASSIFIER_SCALAR)
|
||||
model = BlackboxClassifierPredictions(data, ModelOutputType.CLASSIFIER_SINGLE_OUTPUT_CATEGORICAL)
|
||||
pred = model.predict(train)
|
||||
assert (pred.shape[0] == x_train.shape[0])
|
||||
|
||||
|
|
@ -237,7 +238,7 @@ def test_blackbox_classifier_no_train():
|
|||
|
||||
test = ArrayDataset(x_test, y_test)
|
||||
data = Data(test=test)
|
||||
model = BlackboxClassifierPredictions(data, ModelOutputType.CLASSIFIER_SCALAR)
|
||||
model = BlackboxClassifierPredictions(data, ModelOutputType.CLASSIFIER_SINGLE_OUTPUT_CATEGORICAL)
|
||||
pred = model.predict(test)
|
||||
assert (pred.shape[0] == x_test.shape[0])
|
||||
|
||||
|
|
@ -255,7 +256,7 @@ def test_blackbox_classifier_no_test_y():
|
|||
train = ArrayDataset(x_train, y_train)
|
||||
test = ArrayDataset(x_test)
|
||||
data = Data(train, test)
|
||||
model = BlackboxClassifierPredictions(data, ModelOutputType.CLASSIFIER_SCALAR)
|
||||
model = BlackboxClassifierPredictions(data, ModelOutputType.CLASSIFIER_SINGLE_OUTPUT_CATEGORICAL)
|
||||
pred = model.predict(train)
|
||||
assert (pred.shape[0] == x_train.shape[0])
|
||||
|
||||
|
|
@ -278,7 +279,7 @@ def test_blackbox_classifier_no_train_y():
|
|||
train = ArrayDataset(x_train)
|
||||
test = ArrayDataset(x_test, y_test)
|
||||
data = Data(train, test)
|
||||
model = BlackboxClassifierPredictions(data, ModelOutputType.CLASSIFIER_SCALAR)
|
||||
model = BlackboxClassifierPredictions(data, ModelOutputType.CLASSIFIER_SINGLE_OUTPUT_CATEGORICAL)
|
||||
pred = model.predict(test)
|
||||
assert (pred.shape[0] == x_test.shape[0])
|
||||
|
||||
|
|
@ -302,7 +303,7 @@ def test_blackbox_classifier_probabilities():
|
|||
train = ArrayDataset(x_train, y_train)
|
||||
|
||||
data = Data(train)
|
||||
model = BlackboxClassifierPredictions(data, ModelOutputType.CLASSIFIER_PROBABILITIES)
|
||||
model = BlackboxClassifierPredictions(data, ModelOutputType.CLASSIFIER_SINGLE_OUTPUT_CLASS_PROBABILITIES)
|
||||
pred = model.predict(train)
|
||||
assert (pred.shape[0] == x_train.shape[0])
|
||||
assert (0.0 < pred).all()
|
||||
|
|
@ -321,7 +322,8 @@ def test_blackbox_classifier_predict():
|
|||
|
||||
train = ArrayDataset(x_train, y_train)
|
||||
|
||||
model = BlackboxClassifierPredictFunction(predict, ModelOutputType.CLASSIFIER_PROBABILITIES, (4,), 3)
|
||||
model = BlackboxClassifierPredictFunction(predict, ModelOutputType.CLASSIFIER_SINGLE_OUTPUT_CLASS_PROBABILITIES,
|
||||
(4,), 3)
|
||||
pred = model.predict(train)
|
||||
assert (pred.shape[0] == x_train.shape[0])
|
||||
assert (0.0 < pred).all()
|
||||
|
|
@ -340,7 +342,8 @@ def test_blackbox_classifier_predict_scalar():
|
|||
|
||||
train = ArrayDataset(x_train, y_train)
|
||||
|
||||
model = BlackboxClassifierPredictFunction(predict, ModelOutputType.CLASSIFIER_SCALAR, (4,), 3)
|
||||
model = BlackboxClassifierPredictFunction(predict, ModelOutputType.CLASSIFIER_SINGLE_OUTPUT_CLASS_PROBABILITIES,
|
||||
(4,), 3)
|
||||
pred = model.predict(train)
|
||||
assert (pred.shape[0] == x_train.shape[0])
|
||||
|
||||
|
|
@ -358,23 +361,23 @@ def test_is_one_hot():
|
|||
|
||||
def test_get_nb_classes():
|
||||
(_, y_train), (_, y_test) = dataset_utils.get_iris_dataset_np()
|
||||
|
||||
output_type = ModelOutputType.CLASSIFIER_SINGLE_OUTPUT_CATEGORICAL
|
||||
# shape: (x,) - not 1-hot
|
||||
nb_classes_test = get_nb_classes(y_test)
|
||||
nb_classes_train = get_nb_classes(y_train)
|
||||
nb_classes_test = get_nb_classes(y_test, output_type)
|
||||
nb_classes_train = get_nb_classes(y_train, output_type)
|
||||
assert (nb_classes_test == nb_classes_train)
|
||||
assert (nb_classes_test == 3)
|
||||
|
||||
# shape: (x,1) - not 1-hot
|
||||
nb_classes_test = get_nb_classes(y_test.reshape(-1, 1))
|
||||
nb_classes_test = get_nb_classes(y_test.reshape(-1, 1), output_type)
|
||||
assert (nb_classes_test == 3)
|
||||
|
||||
# shape: (x,3) - 1-hot
|
||||
y = to_categorical(y_test)
|
||||
nb_classes = get_nb_classes(y)
|
||||
nb_classes = get_nb_classes(y, output_type)
|
||||
assert (nb_classes == 3)
|
||||
|
||||
# gaps: 1,2,4 (0,3 missing)
|
||||
y_test[y_test == 0] = 4
|
||||
nb_classes = get_nb_classes(y_test)
|
||||
nb_classes = get_nb_classes(y_test, output_type)
|
||||
assert (nb_classes == 5)
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
import numpy as np
|
||||
from torch import nn, optim, sigmoid, where
|
||||
from torch import nn, optim, sigmoid, where, from_numpy
|
||||
from torch.nn import functional
|
||||
# from torch.utils.data import DataLoader, TensorDataset
|
||||
from torch.utils.data import DataLoader, TensorDataset
|
||||
from scipy.special import expit
|
||||
|
||||
from apt.utils.datasets.datasets import PytorchData
|
||||
|
|
@ -56,8 +56,11 @@ def test_pytorch_nursery_state_dict():
|
|||
criterion = nn.CrossEntropyLoss()
|
||||
optimizer = optim.Adam(inner_model.parameters(), lr=0.01)
|
||||
|
||||
model = PyTorchClassifier(model=inner_model, output_type=ModelOutputType.CLASSIFIER_LOGITS, loss=criterion,
|
||||
optimizer=optimizer, input_shape=(24,),
|
||||
model = PyTorchClassifier(model=inner_model,
|
||||
output_type=ModelOutputType.CLASSIFIER_SINGLE_OUTPUT_CLASS_LOGITS,
|
||||
loss=criterion,
|
||||
optimizer=optimizer,
|
||||
input_shape=(24,),
|
||||
nb_classes=4)
|
||||
model.fit(PytorchData(x_train.astype(np.float32), y_train), save_entire_model=False, nb_epochs=10)
|
||||
model.load_latest_state_dict_checkpoint()
|
||||
|
|
@ -66,7 +69,7 @@ def test_pytorch_nursery_state_dict():
|
|||
assert (0 <= score <= 1)
|
||||
# python pytorch numpy
|
||||
model.load_best_state_dict_checkpoint()
|
||||
score = model.score(PytorchData(x_test.astype(np.float32), y_test))
|
||||
score = model.score(PytorchData(x_test.astype(np.float32), y_test), apply_non_linearity=expit)
|
||||
print('best model accuracy: ', score)
|
||||
assert (0 <= score <= 1)
|
||||
|
||||
|
|
@ -86,8 +89,11 @@ def test_pytorch_nursery_save_entire_model():
|
|||
criterion = nn.CrossEntropyLoss()
|
||||
optimizer = optim.Adam(model.parameters(), lr=0.01)
|
||||
|
||||
art_model = PyTorchClassifier(model=model, output_type=ModelOutputType.CLASSIFIER_LOGITS, loss=criterion,
|
||||
optimizer=optimizer, input_shape=(24,),
|
||||
art_model = PyTorchClassifier(model=model,
|
||||
output_type=ModelOutputType.CLASSIFIER_SINGLE_OUTPUT_CLASS_LOGITS,
|
||||
loss=criterion,
|
||||
optimizer=optimizer,
|
||||
input_shape=(24,),
|
||||
nb_classes=4)
|
||||
art_model.fit(PytorchData(x_train.astype(np.float32), y_train), save_entire_model=True, nb_epochs=10)
|
||||
|
||||
|
|
@ -95,7 +101,7 @@ def test_pytorch_nursery_save_entire_model():
|
|||
print('Base model accuracy: ', score)
|
||||
assert (0 <= score <= 1)
|
||||
art_model.load_best_model_checkpoint()
|
||||
score = art_model.score(PytorchData(x_test.astype(np.float32), y_test))
|
||||
score = art_model.score(PytorchData(x_test.astype(np.float32), y_test), apply_non_linearity=expit)
|
||||
print('best model accuracy: ', score)
|
||||
assert (0 <= score <= 1)
|
||||
|
||||
|
|
@ -126,7 +132,7 @@ def test_pytorch_nursery_save_entire_model():
|
|||
# # make multi-label categorical
|
||||
# y_train = np.column_stack((y_train, y_train, y_train))
|
||||
# y_test = np.column_stack((y_test, y_test, y_test))
|
||||
# test = PytorchData(x_test, y_test)
|
||||
# test = PytorchData(x_test.astype(np.float32), y_test.astype(np.float32))
|
||||
#
|
||||
# model = multi_label_cat_model(3, 4)
|
||||
# criterion = nn.CrossEntropyLoss()
|
||||
|
|
@ -154,8 +160,11 @@ def test_pytorch_nursery_save_entire_model():
|
|||
#
|
||||
# optimizer.step()
|
||||
#
|
||||
# art_model = PyTorchClassifier(model=model, output_type=ModelOutputType.CLASSIFIER_LOGITS, loss=criterion,
|
||||
# optimizer=optimizer, input_shape=(24,),
|
||||
# art_model = PyTorchClassifier(model=model,
|
||||
# output_type=ModelOutputType.CLASSIFIER_MULTI_OUTPUT_CLASS_LOGITS,
|
||||
# loss=criterion,
|
||||
# optimizer=optimizer,
|
||||
# input_shape=(24,),
|
||||
# nb_classes=3)
|
||||
#
|
||||
# pred = art_model.predict(test)
|
||||
|
|
@ -211,8 +220,11 @@ def test_pytorch_predictions_multi_label_binary():
|
|||
criterion = FocalLoss()
|
||||
optimizer = optim.RMSprop(model.parameters(), lr=0.01)
|
||||
|
||||
art_model = PyTorchClassifier(model=model, output_type=ModelOutputType.CLASSIFIER_LOGITS, loss=criterion,
|
||||
optimizer=optimizer, input_shape=(24,),
|
||||
art_model = PyTorchClassifier(model=model,
|
||||
output_type=ModelOutputType.CLASSIFIER_MULTI_OUTPUT_BINARY_LOGITS,
|
||||
loss=criterion,
|
||||
optimizer=optimizer,
|
||||
input_shape=(24,),
|
||||
nb_classes=3)
|
||||
art_model.fit(PytorchData(x_train.astype(np.float32), y_train.astype(np.float32)), save_entire_model=False,
|
||||
nb_epochs=10)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue