mirror of
https://github.com/IBM/ai-privacy-toolkit.git
synced 2026-07-11 16:22:10 +02:00
Formatting
Signed-off-by: abigailt <abigailt@il.ibm.com>
This commit is contained in:
abigailt
parent
0f5a1bcaa0
commit
a4816878f9
6 changed files with 68 additions and 88 deletions
|
|
@ -4,7 +4,7 @@ from enum import Enum, auto
|
|||
import numpy as np
|
||||
from scipy.special import expit
|
||||
|
||||
from apt.utils.datasets import Dataset, Data, DatasetWithPredictions, array2numpy, OUTPUT_DATA_ARRAY_TYPE
|
||||
from apt.utils.datasets import Dataset, Data, array2numpy, OUTPUT_DATA_ARRAY_TYPE
|
||||
from art.estimators.classification import BlackBoxClassifier
|
||||
from art.utils import check_and_transform_label_format
|
||||
|
||||
|
|
@ -43,40 +43,40 @@ def is_one_hot(y: OUTPUT_DATA_ARRAY_TYPE) -> bool:
|
|||
|
||||
|
||||
def is_multi_label(output_type: ModelOutputType) -> bool:
|
||||
return (output_type == ModelOutputType.CLASSIFIER_MULTI_OUTPUT_CATEGORICAL or
|
||||
output_type == ModelOutputType.CLASSIFIER_MULTI_OUTPUT_BINARY_PROBABILITIES or
|
||||
output_type == ModelOutputType.CLASSIFIER_MULTI_OUTPUT_CLASS_PROBABILITIES or
|
||||
output_type == ModelOutputType.CLASSIFIER_MULTI_OUTPUT_BINARY_LOGITS or
|
||||
output_type == ModelOutputType.CLASSIFIER_MULTI_OUTPUT_CLASS_LOGITS)
|
||||
return (output_type == ModelOutputType.CLASSIFIER_MULTI_OUTPUT_CATEGORICAL
|
||||
or output_type == ModelOutputType.CLASSIFIER_MULTI_OUTPUT_BINARY_PROBABILITIES
|
||||
or output_type == ModelOutputType.CLASSIFIER_MULTI_OUTPUT_CLASS_PROBABILITIES
|
||||
or output_type == ModelOutputType.CLASSIFIER_MULTI_OUTPUT_BINARY_LOGITS
|
||||
or output_type == ModelOutputType.CLASSIFIER_MULTI_OUTPUT_CLASS_LOGITS)
|
||||
|
||||
|
||||
def is_multi_label_binary(output_type: ModelOutputType) -> bool:
|
||||
return (output_type == ModelOutputType.CLASSIFIER_MULTI_OUTPUT_BINARY_PROBABILITIES or
|
||||
output_type == ModelOutputType.CLASSIFIER_MULTI_OUTPUT_BINARY_LOGITS)
|
||||
return (output_type == ModelOutputType.CLASSIFIER_MULTI_OUTPUT_BINARY_PROBABILITIES
|
||||
or output_type == ModelOutputType.CLASSIFIER_MULTI_OUTPUT_BINARY_LOGITS)
|
||||
|
||||
|
||||
def is_binary(output_type: ModelOutputType) -> bool:
|
||||
return (output_type == ModelOutputType.CLASSIFIER_MULTI_OUTPUT_BINARY_PROBABILITIES or
|
||||
output_type == ModelOutputType.CLASSIFIER_MULTI_OUTPUT_BINARY_LOGITS or
|
||||
output_type == ModelOutputType.CLASSIFIER_SINGLE_OUTPUT_BINARY_PROBABILITIES or
|
||||
output_type == ModelOutputType.CLASSIFIER_SINGLE_OUTPUT_BINARY_LOGITS)
|
||||
return (output_type == ModelOutputType.CLASSIFIER_MULTI_OUTPUT_BINARY_PROBABILITIES
|
||||
or output_type == ModelOutputType.CLASSIFIER_MULTI_OUTPUT_BINARY_LOGITS
|
||||
or output_type == ModelOutputType.CLASSIFIER_SINGLE_OUTPUT_BINARY_PROBABILITIES
|
||||
or output_type == ModelOutputType.CLASSIFIER_SINGLE_OUTPUT_BINARY_LOGITS)
|
||||
|
||||
|
||||
def is_categorical(output_type: ModelOutputType) -> bool:
|
||||
return (output_type == ModelOutputType.CLASSIFIER_SINGLE_OUTPUT_CATEGORICAL or
|
||||
output_type == ModelOutputType.CLASSIFIER_MULTI_OUTPUT_CATEGORICAL)
|
||||
return (output_type == ModelOutputType.CLASSIFIER_SINGLE_OUTPUT_CATEGORICAL
|
||||
or output_type == ModelOutputType.CLASSIFIER_MULTI_OUTPUT_CATEGORICAL)
|
||||
|
||||
|
||||
def is_probabilities(output_type: ModelOutputType) -> bool:
|
||||
return (output_type == ModelOutputType.CLASSIFIER_SINGLE_OUTPUT_CLASS_PROBABILITIES or
|
||||
output_type == ModelOutputType.CLASSIFIER_MULTI_OUTPUT_CLASS_PROBABILITIES)
|
||||
return (output_type == ModelOutputType.CLASSIFIER_SINGLE_OUTPUT_CLASS_PROBABILITIES
|
||||
or output_type == ModelOutputType.CLASSIFIER_MULTI_OUTPUT_CLASS_PROBABILITIES)
|
||||
|
||||
|
||||
def is_logits(output_type: ModelOutputType) -> bool:
|
||||
return (output_type == ModelOutputType.CLASSIFIER_SINGLE_OUTPUT_CLASS_LOGITS or
|
||||
output_type == ModelOutputType.CLASSIFIER_MULTI_OUTPUT_CLASS_LOGITS or
|
||||
output_type == ModelOutputType.CLASSIFIER_SINGLE_OUTPUT_BINARY_LOGITS or
|
||||
output_type == ModelOutputType.CLASSIFIER_MULTI_OUTPUT_BINARY_LOGITS)
|
||||
return (output_type == ModelOutputType.CLASSIFIER_SINGLE_OUTPUT_CLASS_LOGITS
|
||||
or output_type == ModelOutputType.CLASSIFIER_MULTI_OUTPUT_CLASS_LOGITS
|
||||
or output_type == ModelOutputType.CLASSIFIER_SINGLE_OUTPUT_BINARY_LOGITS
|
||||
or output_type == ModelOutputType.CLASSIFIER_MULTI_OUTPUT_BINARY_LOGITS)
|
||||
|
||||
|
||||
def get_nb_classes(y: OUTPUT_DATA_ARRAY_TYPE, output_type: ModelOutputType) -> int:
|
||||
|
|
@ -114,10 +114,10 @@ def check_correct_model_output(y: OUTPUT_DATA_ARRAY_TYPE, output_type: ModelOutp
|
|||
:type output_type: ModelOutputType
|
||||
:raises: ValueError (in case of mismatch)
|
||||
"""
|
||||
if not is_one_hot(y) and (output_type == ModelOutputType.CLASSIFIER_SINGLE_OUTPUT_CLASS_PROBABILITIES or
|
||||
output_type == ModelOutputType.CLASSIFIER_SINGLE_OUTPUT_CLASS_LOGITS):
|
||||
raise ValueError("Incompatible model output types. Model outputs 1D array of categorical scalars while "
|
||||
"output type is set to ", output_type)
|
||||
if not is_one_hot(y) and (output_type == ModelOutputType.CLASSIFIER_SINGLE_OUTPUT_CLASS_PROBABILITIES
|
||||
or output_type == ModelOutputType.CLASSIFIER_SINGLE_OUTPUT_CLASS_LOGITS):
|
||||
raise ValueError("Incompatible model output types. Model outputs 1D array of categorical scalars while "
|
||||
"output type is set to ", output_type)
|
||||
|
||||
|
||||
class Model(metaclass=ABCMeta):
|
||||
|
|
@ -209,13 +209,13 @@ class Model(metaclass=ABCMeta):
|
|||
if scoring_method == ScoringMethod.ACCURACY:
|
||||
if not is_multi_label(self.output_type) and not is_binary(self.output_type) and nb_classes is not None:
|
||||
y = check_and_transform_label_format(y, nb_classes=nb_classes)
|
||||
if (self.output_type == ModelOutputType.CLASSIFIER_SINGLE_OUTPUT_CLASS_PROBABILITIES or
|
||||
self.output_type == ModelOutputType.CLASSIFIER_SINGLE_OUTPUT_CLASS_LOGITS or
|
||||
self.output_type == ModelOutputType.CLASSIFIER_SINGLE_OUTPUT_CATEGORICAL):
|
||||
if (self.output_type == ModelOutputType.CLASSIFIER_SINGLE_OUTPUT_CLASS_PROBABILITIES
|
||||
or self.output_type == ModelOutputType.CLASSIFIER_SINGLE_OUTPUT_CLASS_LOGITS
|
||||
or self.output_type == ModelOutputType.CLASSIFIER_SINGLE_OUTPUT_CATEGORICAL):
|
||||
# categorical has been 1-hot encoded by check_and_transform_label_format
|
||||
return np.count_nonzero(np.argmax(y, axis=1) == np.argmax(predicted, axis=1)) / predicted.shape[0]
|
||||
elif (self.output_type == ModelOutputType.CLASSIFIER_MULTI_OUTPUT_CLASS_LOGITS or
|
||||
self.output_type == ModelOutputType.CLASSIFIER_SINGLE_OUTPUT_CLASS_PROBABILITIES):
|
||||
elif (self.output_type == ModelOutputType.CLASSIFIER_MULTI_OUTPUT_CLASS_LOGITS
|
||||
or self.output_type == ModelOutputType.CLASSIFIER_SINGLE_OUTPUT_CLASS_PROBABILITIES):
|
||||
if predicted.shape != y.shape:
|
||||
raise ValueError('Do not know how to compare arrays with different shapes')
|
||||
elif len(predicted.shape) < 3:
|
||||
|
|
@ -372,7 +372,7 @@ class BlackboxClassifier(Model):
|
|||
"""
|
||||
Score the model using test data.
|
||||
"""
|
||||
kwargs ['nb_classes'] = self.nb_classes
|
||||
kwargs['nb_classes'] = self.nb_classes
|
||||
return super().score(test_data, **kwargs)
|
||||
|
||||
def fit(self, train_data: Dataset, **kwargs) -> None:
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ import os
|
|||
import shutil
|
||||
import logging
|
||||
|
||||
from typing import Optional, Tuple, Union, List
|
||||
from typing import Optional, Tuple, Union, List, TYPE_CHECKING
|
||||
import numpy as np
|
||||
import torch
|
||||
from torch.utils.data import DataLoader, TensorDataset
|
||||
|
|
@ -14,6 +14,11 @@ from apt.utils.models import Model, ModelOutputType, is_multi_label, is_multi_la
|
|||
from apt.utils.datasets import OUTPUT_DATA_ARRAY_TYPE, array2numpy
|
||||
from art.estimators.classification.pytorch import PyTorchClassifier as ArtPyTorchClassifier
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from art.utils import CLIP_VALUES_TYPE, PREPROCESSING_TYPE
|
||||
from art.defences.preprocessor import Preprocessor
|
||||
from art.defences.postprocessor import Postprocessor
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
|
@ -51,8 +56,8 @@ class PyTorchClassifierWrapper(ArtPyTorchClassifier):
|
|||
super().__init__(model, loss, input_shape, nb_classes, optimizer, use_amp, opt_level, loss_scale,
|
||||
channels_first, clip_values, preprocessing_defences, postprocessing_defences, preprocessing,
|
||||
device_type)
|
||||
self._is_single_binary = (output_type == ModelOutputType.CLASSIFIER_SINGLE_OUTPUT_BINARY_PROBABILITIES or
|
||||
output_type == ModelOutputType.CLASSIFIER_SINGLE_OUTPUT_BINARY_LOGITS)
|
||||
self._is_single_binary = (output_type == ModelOutputType.CLASSIFIER_SINGLE_OUTPUT_BINARY_PROBABILITIES
|
||||
or output_type == ModelOutputType.CLASSIFIER_SINGLE_OUTPUT_BINARY_LOGITS)
|
||||
self._is_multi_label = is_multi_label(output_type)
|
||||
self._is_multi_label_binary = is_multi_label_binary(output_type)
|
||||
|
||||
|
|
@ -504,29 +509,10 @@ class PyTorchClassifier(PyTorchModel):
|
|||
:return: the score as float (between 0 and 1)
|
||||
"""
|
||||
# numpy arrays
|
||||
y = test_data.get_labels()
|
||||
predicted = self.predict(test_data)
|
||||
kwargs['predictions'] = DatasetWithPredictions(pred=predicted)
|
||||
kwargs['nb_classes'] = self._nb_classes
|
||||
return super().score(ArrayDataset(test_data.get_samples(), test_data.get_labels()), **kwargs)
|
||||
# if apply_non_linearity:
|
||||
# predicted = apply_non_linearity(predicted)
|
||||
# # binary classification, single column of probabilities
|
||||
# if self._art_model.nb_classes == 2 and (len(predicted.shape) == 1 or predicted.shape[1] == 1):
|
||||
# if len(predicted.shape) > 1:
|
||||
# y = check_and_transform_label_format(y, self._art_model.nb_classes, return_one_hot=False)
|
||||
# return np.count_nonzero(y == (predicted > binary_threshold)) / predicted.shape[0]
|
||||
# # multi column
|
||||
# else:
|
||||
# if not is_multi_label(y):
|
||||
# y = check_and_transform_label_format(y, self._art_model.nb_classes)
|
||||
# return np.count_nonzero(np.argmax(y, axis=1) == np.argmax(predicted, axis=1)) / predicted.shape[0]
|
||||
# else:
|
||||
# if is_multi_label_binary(y):
|
||||
# predicted[predicted < binary_threshold] = 0
|
||||
# predicted[predicted >= binary_threshold] = 1
|
||||
# return np.count_nonzero(y == predicted) / (predicted.shape[0] * y.shape[1])
|
||||
|
||||
|
||||
def load_checkpoint_state_dict_by_path(self, model_name: str, path: str = None):
|
||||
"""
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue