mirror of
https://github.com/IBM/ai-privacy-toolkit.git
synced 2026-07-02 16:01:00 +02:00
Support for many new model output types (#93)
* General model wrappers and methods supporting multi-label classifiers * Support for pytorch multi-label binary classifier * New model output types + single implementation of score method that supports multiple output types. * Anonymization with pytorch multi-output binary model * Support for multi-label binary models in minimizer. * Support for multi-label logits/probabilities --------- Signed-off-by: abigailt <abigailt@il.ibm.com>
This commit is contained in:
abigailgold
GitHub
parent
e00535d120
commit
57e38ea4fa
13 changed files with 913 additions and 172 deletions
|
|
@ -6,11 +6,17 @@ from sklearn.impute import SimpleImputer
|
|||
from sklearn.pipeline import Pipeline
|
||||
from sklearn.tree import DecisionTreeClassifier, DecisionTreeRegressor
|
||||
from sklearn.preprocessing import OneHotEncoder
|
||||
|
||||
from apt.anonymization import Anonymize
|
||||
from apt.utils.dataset_utils import get_iris_dataset_np, get_adult_dataset_pd, get_nursery_dataset_pd
|
||||
from sklearn.datasets import load_diabetes
|
||||
from sklearn.model_selection import train_test_split
|
||||
from torch import nn, optim, sigmoid, where
|
||||
from torch.nn import functional
|
||||
from scipy.special import expit
|
||||
|
||||
from apt.utils.datasets.datasets import PytorchData
|
||||
from apt.utils.models import CLASSIFIER_MULTI_OUTPUT_BINARY_LOGITS
|
||||
from apt.utils.models.pytorch_model import PyTorchClassifier
|
||||
from apt.anonymization import Anonymize
|
||||
from apt.utils.dataset_utils import get_iris_dataset_np, get_adult_dataset_pd, get_nursery_dataset_pd
|
||||
from apt.utils.datasets import ArrayDataset
|
||||
|
||||
|
||||
|
|
@ -187,6 +193,72 @@ def test_anonymize_pandas_one_hot():
|
|||
assert ((np.min(anonymized_slice, axis=1) == 0).all())
|
||||
|
||||
|
||||
def test_anonymize_pytorch_multi_label_binary():
|
||||
class multi_label_binary_model(nn.Module):
|
||||
def __init__(self, num_labels, num_features):
|
||||
super(multi_label_binary_model, self).__init__()
|
||||
|
||||
self.fc1 = nn.Sequential(
|
||||
nn.Linear(num_features, 256),
|
||||
nn.Tanh(), )
|
||||
|
||||
self.classifier1 = nn.Linear(256, num_labels)
|
||||
|
||||
def forward(self, x):
|
||||
return self.classifier1(self.fc1(x))
|
||||
# missing sigmoid on each output
|
||||
|
||||
class FocalLoss(nn.Module):
|
||||
def __init__(self, gamma=2, alpha=0.5):
|
||||
super(FocalLoss, self).__init__()
|
||||
self.gamma = gamma
|
||||
self.alpha = alpha
|
||||
|
||||
def forward(self, input, target):
|
||||
bce_loss = functional.binary_cross_entropy_with_logits(input, target, reduction='none')
|
||||
|
||||
p = sigmoid(input)
|
||||
p = where(target >= 0.5, p, 1 - p)
|
||||
|
||||
modulating_factor = (1 - p) ** self.gamma
|
||||
alpha = self.alpha * target + (1 - self.alpha) * (1 - target)
|
||||
focal_loss = alpha * modulating_factor * bce_loss
|
||||
|
||||
return focal_loss.mean()
|
||||
|
||||
(x_train, y_train), _ = get_iris_dataset_np()
|
||||
|
||||
# make multi-label binary
|
||||
y_train = np.column_stack((y_train, y_train, y_train))
|
||||
y_train[y_train > 1] = 1
|
||||
|
||||
model = multi_label_binary_model(3, 4)
|
||||
criterion = FocalLoss()
|
||||
optimizer = optim.RMSprop(model.parameters(), lr=0.01)
|
||||
|
||||
art_model = PyTorchClassifier(model=model,
|
||||
output_type=CLASSIFIER_MULTI_OUTPUT_BINARY_LOGITS,
|
||||
loss=criterion,
|
||||
optimizer=optimizer,
|
||||
input_shape=(24,),
|
||||
nb_classes=3)
|
||||
art_model.fit(PytorchData(x_train.astype(np.float32), y_train.astype(np.float32)), save_entire_model=False,
|
||||
nb_epochs=10)
|
||||
pred = art_model.predict(PytorchData(x_train.astype(np.float32), y_train.astype(np.float32)))
|
||||
pred = expit(pred)
|
||||
pred[pred < 0.5] = 0
|
||||
pred[pred >= 0.5] = 1
|
||||
|
||||
k = 10
|
||||
QI = [0, 2]
|
||||
anonymizer = Anonymize(k, QI, train_only_QI=True)
|
||||
anon = anonymizer.anonymize(ArrayDataset(x_train, pred))
|
||||
assert (len(np.unique(anon[:, QI], axis=0)) < len(np.unique(x_train[:, QI], axis=0)))
|
||||
_, counts_elements = np.unique(anon[:, QI], return_counts=True)
|
||||
assert (np.min(counts_elements) >= k)
|
||||
assert ((np.delete(anon, QI, axis=1) == np.delete(x_train, QI, axis=1)).all())
|
||||
|
||||
|
||||
def test_errors():
|
||||
with pytest.raises(ValueError):
|
||||
Anonymize(1, [0, 2])
|
||||
|
|
|
|||
|
|
@ -4,25 +4,29 @@ import pandas as pd
|
|||
import scipy
|
||||
|
||||
from sklearn.compose import ColumnTransformer
|
||||
|
||||
from sklearn.datasets import load_diabetes
|
||||
from sklearn.impute import SimpleImputer
|
||||
from sklearn.model_selection import train_test_split
|
||||
from sklearn.pipeline import Pipeline
|
||||
from sklearn.preprocessing import OneHotEncoder
|
||||
from sklearn.tree import DecisionTreeClassifier, DecisionTreeRegressor
|
||||
|
||||
from torch import nn, optim
|
||||
from torch import nn, optim, sigmoid, where
|
||||
from torch.nn import functional
|
||||
from scipy.special import expit
|
||||
|
||||
import tensorflow as tf
|
||||
from tensorflow.keras.models import Sequential
|
||||
from tensorflow.keras.layers import Dense, Input
|
||||
|
||||
from apt.utils.datasets.datasets import PytorchData
|
||||
from apt.utils.models.pytorch_model import PyTorchClassifier
|
||||
from apt.minimization import GeneralizeToRepresentative
|
||||
from sklearn.tree import DecisionTreeClassifier, DecisionTreeRegressor
|
||||
from apt.utils.dataset_utils import get_iris_dataset_np, get_adult_dataset_pd, get_german_credit_dataset_pd
|
||||
from apt.utils.datasets import ArrayDataset
|
||||
from apt.utils.models import SklearnClassifier, ModelOutputType, SklearnRegressor, KerasClassifier
|
||||
|
||||
from apt.utils.models import SklearnClassifier, SklearnRegressor, KerasClassifier, \
|
||||
CLASSIFIER_SINGLE_OUTPUT_CLASS_PROBABILITIES, CLASSIFIER_SINGLE_OUTPUT_CATEGORICAL, \
|
||||
CLASSIFIER_SINGLE_OUTPUT_CLASS_LOGITS, CLASSIFIER_MULTI_OUTPUT_BINARY_LOGITS
|
||||
tf.compat.v1.disable_eager_execution()
|
||||
|
||||
|
||||
|
|
@ -216,7 +220,7 @@ def test_minimizer_params(cells):
|
|||
|
||||
base_est = DecisionTreeClassifier(random_state=0, min_samples_split=2,
|
||||
min_samples_leaf=1)
|
||||
model = SklearnClassifier(base_est, ModelOutputType.CLASSIFIER_PROBABILITIES)
|
||||
model = SklearnClassifier(base_est, CLASSIFIER_SINGLE_OUTPUT_CLASS_PROBABILITIES)
|
||||
model.fit(ArrayDataset(x, y))
|
||||
|
||||
expected_generalizations = {'categories': {}, 'category_representatives': {},
|
||||
|
|
@ -258,7 +262,7 @@ def test_minimizer_params_not_transform(cells):
|
|||
samples = ArrayDataset(x, y, features)
|
||||
base_est = DecisionTreeClassifier(random_state=0, min_samples_split=2,
|
||||
min_samples_leaf=1)
|
||||
model = SklearnClassifier(base_est, ModelOutputType.CLASSIFIER_PROBABILITIES)
|
||||
model = SklearnClassifier(base_est, CLASSIFIER_SINGLE_OUTPUT_CLASS_PROBABILITIES)
|
||||
model.fit(ArrayDataset(x, y))
|
||||
|
||||
gen = GeneralizeToRepresentative(model, cells=cells, generalize_using_transform=False)
|
||||
|
|
@ -270,7 +274,7 @@ def test_minimizer_fit(data_two_features):
|
|||
x, y, features, _ = data_two_features
|
||||
base_est = DecisionTreeClassifier(random_state=0, min_samples_split=2,
|
||||
min_samples_leaf=1)
|
||||
model = SklearnClassifier(base_est, ModelOutputType.CLASSIFIER_PROBABILITIES)
|
||||
model = SklearnClassifier(base_est, CLASSIFIER_SINGLE_OUTPUT_CLASS_PROBABILITIES)
|
||||
model.fit(ArrayDataset(x, y))
|
||||
ad = ArrayDataset(x)
|
||||
predictions = model.predict(ad)
|
||||
|
|
@ -287,6 +291,7 @@ def test_minimizer_fit(data_two_features):
|
|||
|
||||
compare_generalizations(gener, expected_generalizations)
|
||||
check_features(features, expected_generalizations, transformed, x)
|
||||
assert (np.equal(x, transformed).all())
|
||||
ncp = gen.ncp.transform_score
|
||||
check_ncp(ncp, expected_generalizations)
|
||||
|
||||
|
|
@ -299,7 +304,7 @@ def test_minimizer_ncp(data_two_features):
|
|||
|
||||
base_est = DecisionTreeClassifier(random_state=0, min_samples_split=2,
|
||||
min_samples_leaf=1)
|
||||
model = SklearnClassifier(base_est, ModelOutputType.CLASSIFIER_PROBABILITIES)
|
||||
model = SklearnClassifier(base_est, CLASSIFIER_SINGLE_OUTPUT_CLASS_PROBABILITIES)
|
||||
model.fit(ArrayDataset(x, y))
|
||||
ad = ArrayDataset(x)
|
||||
ad1 = ArrayDataset(x1, features_names=features)
|
||||
|
|
@ -342,7 +347,7 @@ def test_minimizer_ncp_categorical(data_four_features):
|
|||
|
||||
base_est = DecisionTreeClassifier(random_state=0, min_samples_split=2,
|
||||
min_samples_leaf=1)
|
||||
model = SklearnClassifier(base_est, ModelOutputType.CLASSIFIER_PROBABILITIES)
|
||||
model = SklearnClassifier(base_est, CLASSIFIER_SINGLE_OUTPUT_CLASS_PROBABILITIES)
|
||||
model.fit(ArrayDataset(encoded, y))
|
||||
ad = ArrayDataset(x)
|
||||
ad1 = ArrayDataset(x1)
|
||||
|
|
@ -382,7 +387,7 @@ def test_minimizer_fit_not_transform(data_two_features):
|
|||
x, y, features, x1 = data_two_features
|
||||
base_est = DecisionTreeClassifier(random_state=0, min_samples_split=2,
|
||||
min_samples_leaf=1)
|
||||
model = SklearnClassifier(base_est, ModelOutputType.CLASSIFIER_PROBABILITIES)
|
||||
model = SklearnClassifier(base_est, CLASSIFIER_SINGLE_OUTPUT_CLASS_PROBABILITIES)
|
||||
model.fit(ArrayDataset(x, y))
|
||||
ad = ArrayDataset(x)
|
||||
predictions = model.predict(ad)
|
||||
|
|
@ -412,7 +417,7 @@ def test_minimizer_fit_pandas(data_four_features):
|
|||
|
||||
base_est = DecisionTreeClassifier(random_state=0, min_samples_split=2,
|
||||
min_samples_leaf=1)
|
||||
model = SklearnClassifier(base_est, ModelOutputType.CLASSIFIER_PROBABILITIES)
|
||||
model = SklearnClassifier(base_est, CLASSIFIER_SINGLE_OUTPUT_CLASS_PROBABILITIES)
|
||||
model.fit(ArrayDataset(encoded, y))
|
||||
predictions = model.predict(ArrayDataset(encoded))
|
||||
if predictions.shape[1] > 1:
|
||||
|
|
@ -450,7 +455,7 @@ def test_minimizer_params_categorical(cells_categorical):
|
|||
preprocessor, encoded = create_encoder(numeric_features, categorical_features, x)
|
||||
base_est = DecisionTreeClassifier(random_state=0, min_samples_split=2,
|
||||
min_samples_leaf=1)
|
||||
model = SklearnClassifier(base_est, ModelOutputType.CLASSIFIER_PROBABILITIES)
|
||||
model = SklearnClassifier(base_est, CLASSIFIER_SINGLE_OUTPUT_CLASS_PROBABILITIES)
|
||||
model.fit(ArrayDataset(encoded, y))
|
||||
predictions = model.predict(ArrayDataset(encoded))
|
||||
if predictions.shape[1] > 1:
|
||||
|
|
@ -474,7 +479,7 @@ def test_minimizer_fit_qi(data_three_features):
|
|||
qi = ['age', 'weight']
|
||||
base_est = DecisionTreeClassifier(random_state=0, min_samples_split=2,
|
||||
min_samples_leaf=1)
|
||||
model = SklearnClassifier(base_est, ModelOutputType.CLASSIFIER_PROBABILITIES)
|
||||
model = SklearnClassifier(base_est, CLASSIFIER_SINGLE_OUTPUT_CLASS_PROBABILITIES)
|
||||
model.fit(ArrayDataset(x, y))
|
||||
ad = ArrayDataset(x)
|
||||
predictions = model.predict(ad)
|
||||
|
|
@ -508,7 +513,7 @@ def test_minimizer_fit_pandas_qi(data_five_features):
|
|||
|
||||
base_est = DecisionTreeClassifier(random_state=0, min_samples_split=2,
|
||||
min_samples_leaf=1)
|
||||
model = SklearnClassifier(base_est, ModelOutputType.CLASSIFIER_PROBABILITIES)
|
||||
model = SklearnClassifier(base_est, CLASSIFIER_SINGLE_OUTPUT_CLASS_PROBABILITIES)
|
||||
model.fit(ArrayDataset(encoded, y))
|
||||
predictions = model.predict(ArrayDataset(encoded))
|
||||
if predictions.shape[1] > 1:
|
||||
|
|
@ -543,7 +548,7 @@ def test_minimize_ndarray_iris():
|
|||
qi = ['sepal length (cm)', 'petal length (cm)']
|
||||
base_est = DecisionTreeClassifier(random_state=0, min_samples_split=2,
|
||||
min_samples_leaf=1)
|
||||
model = SklearnClassifier(base_est, ModelOutputType.CLASSIFIER_PROBABILITIES)
|
||||
model = SklearnClassifier(base_est, CLASSIFIER_SINGLE_OUTPUT_CATEGORICAL)
|
||||
model.fit(ArrayDataset(x_train, y_train))
|
||||
predictions = model.predict(ArrayDataset(x_train))
|
||||
if predictions.shape[1] > 1:
|
||||
|
|
@ -586,7 +591,7 @@ def test_minimize_pandas_adult():
|
|||
|
||||
base_est = DecisionTreeClassifier(random_state=0, min_samples_split=2,
|
||||
min_samples_leaf=1)
|
||||
model = SklearnClassifier(base_est, ModelOutputType.CLASSIFIER_PROBABILITIES)
|
||||
model = SklearnClassifier(base_est, CLASSIFIER_SINGLE_OUTPUT_CLASS_PROBABILITIES)
|
||||
model.fit(ArrayDataset(encoded, y_train))
|
||||
predictions = model.predict(ArrayDataset(encoded))
|
||||
if predictions.shape[1] > 1:
|
||||
|
|
@ -642,7 +647,7 @@ def test_german_credit_pandas():
|
|||
|
||||
base_est = DecisionTreeClassifier(random_state=0, min_samples_split=2,
|
||||
min_samples_leaf=1)
|
||||
model = SklearnClassifier(base_est, ModelOutputType.CLASSIFIER_PROBABILITIES)
|
||||
model = SklearnClassifier(base_est, CLASSIFIER_SINGLE_OUTPUT_CLASS_PROBABILITIES)
|
||||
model.fit(ArrayDataset(encoded, y_train))
|
||||
predictions = model.predict(ArrayDataset(encoded))
|
||||
if predictions.shape[1] > 1:
|
||||
|
|
@ -760,7 +765,7 @@ def test_x_y():
|
|||
qi = [0, 2]
|
||||
base_est = DecisionTreeClassifier(random_state=0, min_samples_split=2,
|
||||
min_samples_leaf=1)
|
||||
model = SklearnClassifier(base_est, ModelOutputType.CLASSIFIER_PROBABILITIES)
|
||||
model = SklearnClassifier(base_est, CLASSIFIER_SINGLE_OUTPUT_CLASS_PROBABILITIES)
|
||||
model.fit(ArrayDataset(x, y))
|
||||
ad = ArrayDataset(x)
|
||||
predictions = model.predict(ad)
|
||||
|
|
@ -800,7 +805,7 @@ def test_x_y_features_names():
|
|||
qi = ['age', 'weight']
|
||||
base_est = DecisionTreeClassifier(random_state=0, min_samples_split=2,
|
||||
min_samples_leaf=1)
|
||||
model = SklearnClassifier(base_est, ModelOutputType.CLASSIFIER_PROBABILITIES)
|
||||
model = SklearnClassifier(base_est, CLASSIFIER_SINGLE_OUTPUT_CLASS_PROBABILITIES)
|
||||
model.fit(ArrayDataset(x, y))
|
||||
ad = ArrayDataset(x)
|
||||
predictions = model.predict(ad)
|
||||
|
|
@ -1202,7 +1207,7 @@ def test_keras_model():
|
|||
|
||||
base_est.compile(loss="categorical_crossentropy", optimizer="adam", metrics=["accuracy"])
|
||||
|
||||
model = KerasClassifier(base_est, ModelOutputType.CLASSIFIER_PROBABILITIES)
|
||||
model = KerasClassifier(base_est, CLASSIFIER_SINGLE_OUTPUT_CLASS_PROBABILITIES)
|
||||
model.fit(ArrayDataset(x, y))
|
||||
ad = ArrayDataset(x_test)
|
||||
predictions = model.predict(ad)
|
||||
|
|
@ -1269,8 +1274,11 @@ def test_minimizer_pytorch(data_three_features):
|
|||
criterion = nn.CrossEntropyLoss()
|
||||
optimizer = optim.Adam(base_est.parameters(), lr=0.01)
|
||||
|
||||
model = PyTorchClassifier(model=base_est, output_type=ModelOutputType.CLASSIFIER_LOGITS, loss=criterion,
|
||||
optimizer=optimizer, input_shape=(3,),
|
||||
model = PyTorchClassifier(model=base_est,
|
||||
output_type=CLASSIFIER_SINGLE_OUTPUT_CLASS_LOGITS,
|
||||
loss=criterion,
|
||||
optimizer=optimizer,
|
||||
input_shape=(3,),
|
||||
nb_classes=2)
|
||||
model.fit(PytorchData(x, y), save_entire_model=False, nb_epochs=10)
|
||||
|
||||
|
|
@ -1308,8 +1316,11 @@ def test_minimizer_pytorch_iris():
|
|||
criterion = nn.CrossEntropyLoss()
|
||||
optimizer = optim.Adam(base_est.parameters(), lr=0.01)
|
||||
|
||||
model = PyTorchClassifier(model=base_est, output_type=ModelOutputType.CLASSIFIER_LOGITS, loss=criterion,
|
||||
optimizer=optimizer, input_shape=(4,),
|
||||
model = PyTorchClassifier(model=base_est,
|
||||
output_type=CLASSIFIER_SINGLE_OUTPUT_CLASS_LOGITS,
|
||||
loss=criterion,
|
||||
optimizer=optimizer,
|
||||
input_shape=(4,),
|
||||
nb_classes=3)
|
||||
model.fit(PytorchData(x_train, y_train), save_entire_model=False, nb_epochs=10)
|
||||
|
||||
|
|
@ -1329,6 +1340,78 @@ def test_minimizer_pytorch_iris():
|
|||
assert ((rel_accuracy >= target_accuracy) or (target_accuracy - rel_accuracy) <= ACCURACY_DIFF)
|
||||
|
||||
|
||||
def test_minimizer_pytorch_multi_label_binary():
|
||||
class multi_label_binary_model(nn.Module):
|
||||
def __init__(self, num_labels, num_features):
|
||||
super(multi_label_binary_model, self).__init__()
|
||||
|
||||
self.fc1 = nn.Sequential(
|
||||
nn.Linear(num_features, 256),
|
||||
nn.Tanh(), )
|
||||
|
||||
self.classifier1 = nn.Linear(256, num_labels)
|
||||
|
||||
def forward(self, x):
|
||||
return self.classifier1(self.fc1(x))
|
||||
# missing sigmoid on each output
|
||||
|
||||
class FocalLoss(nn.Module):
|
||||
def __init__(self, gamma=2, alpha=0.5):
|
||||
super(FocalLoss, self).__init__()
|
||||
self.gamma = gamma
|
||||
self.alpha = alpha
|
||||
|
||||
def forward(self, input, target):
|
||||
bce_loss = functional.binary_cross_entropy_with_logits(input, target, reduction='none')
|
||||
|
||||
p = sigmoid(input)
|
||||
p = where(target >= 0.5, p, 1 - p)
|
||||
|
||||
modulating_factor = (1 - p) ** self.gamma
|
||||
alpha = self.alpha * target + (1 - self.alpha) * (1 - target)
|
||||
focal_loss = alpha * modulating_factor * bce_loss
|
||||
|
||||
return focal_loss.mean()
|
||||
|
||||
(x_train, y_train), _ = get_iris_dataset_np()
|
||||
features = ['sepal length (cm)', 'sepal width (cm)', 'petal length (cm)', 'petal width (cm)']
|
||||
qi = ['sepal length (cm)', 'sepal width (cm)', 'petal length (cm)', 'petal width (cm)']
|
||||
|
||||
# make multi-label binary
|
||||
y_train = np.column_stack((y_train, y_train, y_train))
|
||||
y_train[y_train > 1] = 1
|
||||
x_train = x_train.astype(np.float32)
|
||||
y_train = y_train.astype(np.float32)
|
||||
|
||||
orig_model = multi_label_binary_model(3, 4)
|
||||
criterion = FocalLoss()
|
||||
optimizer = optim.RMSprop(orig_model.parameters(), lr=0.01)
|
||||
|
||||
model = PyTorchClassifier(model=orig_model,
|
||||
output_type=CLASSIFIER_MULTI_OUTPUT_BINARY_LOGITS,
|
||||
loss=criterion,
|
||||
optimizer=optimizer,
|
||||
input_shape=(24,),
|
||||
nb_classes=3)
|
||||
model.fit(PytorchData(x_train, y_train), save_entire_model=False, nb_epochs=10)
|
||||
predictions = model.predict(PytorchData(x_train, y_train))
|
||||
predictions = expit(predictions)
|
||||
predictions[predictions < 0.5] = 0
|
||||
predictions[predictions >= 0.5] = 1
|
||||
|
||||
target_accuracy = 0.99
|
||||
gen = GeneralizeToRepresentative(model, target_accuracy=target_accuracy, features_to_minimize=qi)
|
||||
transformed = gen.fit_transform(dataset=ArrayDataset(x_train, predictions, features_names=features))
|
||||
gener = gen.generalizations
|
||||
|
||||
check_features(features, gener, transformed, x_train)
|
||||
ncp = gen.ncp.transform_score
|
||||
check_ncp(ncp, gener)
|
||||
|
||||
rel_accuracy = model.score(ArrayDataset(transformed.astype(np.float32), predictions))
|
||||
assert ((rel_accuracy >= target_accuracy) or (target_accuracy - rel_accuracy) <= ACCURACY_DIFF)
|
||||
|
||||
|
||||
def test_untouched():
|
||||
cells = [{"id": 1, "ranges": {"age": {"start": None, "end": 38}}, "label": 0,
|
||||
'categories': {'gender': ['male']}, "representative": {"age": 26, "height": 149}},
|
||||
|
|
@ -1362,7 +1445,7 @@ def test_errors():
|
|||
y = np.array([1, 1, 0, 1, 0, 0, 1, 1, 0, 1, 0])
|
||||
base_est = DecisionTreeClassifier(random_state=0, min_samples_split=2,
|
||||
min_samples_leaf=1)
|
||||
model = SklearnClassifier(base_est, ModelOutputType.CLASSIFIER_PROBABILITIES)
|
||||
model = SklearnClassifier(base_est, CLASSIFIER_SINGLE_OUTPUT_CLASS_PROBABILITIES)
|
||||
model.fit(ArrayDataset(X, y))
|
||||
ad = ArrayDataset(X)
|
||||
predictions = model.predict(ad)
|
||||
|
|
|
|||
|
|
@ -1,8 +1,11 @@
|
|||
import pytest
|
||||
import numpy as np
|
||||
|
||||
from apt.utils.models import SklearnClassifier, SklearnRegressor, ModelOutputType, KerasClassifier, KerasRegressor, \
|
||||
BlackboxClassifierPredictions, BlackboxClassifierPredictFunction, is_one_hot, get_nb_classes, XGBoostClassifier
|
||||
from apt.utils.models import SklearnClassifier, SklearnRegressor, KerasClassifier, KerasRegressor, \
|
||||
BlackboxClassifierPredictions, BlackboxClassifierPredictFunction, is_one_hot, get_nb_classes, XGBoostClassifier, \
|
||||
CLASSIFIER_SINGLE_OUTPUT_CATEGORICAL, CLASSIFIER_SINGLE_OUTPUT_CLASS_PROBABILITIES, \
|
||||
CLASSIFIER_MULTI_OUTPUT_CATEGORICAL, CLASSIFIER_MULTI_OUTPUT_BINARY_PROBABILITIES, \
|
||||
CLASSIFIER_MULTI_OUTPUT_CLASS_PROBABILITIES
|
||||
from apt.utils.datasets import ArrayDataset, Data, DatasetWithPredictions
|
||||
from apt.utils import dataset_utils
|
||||
|
||||
|
|
@ -24,7 +27,7 @@ tf.compat.v1.disable_eager_execution()
|
|||
def test_sklearn_classifier():
|
||||
(x_train, y_train), (x_test, y_test) = dataset_utils.get_iris_dataset_np()
|
||||
underlying_model = RandomForestClassifier()
|
||||
model = SklearnClassifier(underlying_model, ModelOutputType.CLASSIFIER_PROBABILITIES)
|
||||
model = SklearnClassifier(underlying_model, CLASSIFIER_SINGLE_OUTPUT_CATEGORICAL)
|
||||
train = ArrayDataset(x_train, y_train)
|
||||
test = ArrayDataset(x_test, y_test)
|
||||
model.fit(train)
|
||||
|
|
@ -35,6 +38,28 @@ def test_sklearn_classifier():
|
|||
assert (0.0 <= score <= 1.0)
|
||||
|
||||
|
||||
# This test currently cannot pass due to ART dependency, so sklearn support will need to wait until ART is updated
|
||||
# def test_sklearn_classifier_predictions_multi_label_binary():
|
||||
# (x_train, y_train), (x_test, y_test) = dataset_utils.get_iris_dataset_np()
|
||||
#
|
||||
# # make multi-label binary
|
||||
# y_train = np.column_stack((y_train, y_train, y_train))
|
||||
# y_train[y_train > 1] = 1
|
||||
# y_test = np.column_stack((y_test, y_test, y_test))
|
||||
# y_test[y_test > 1] = 1
|
||||
#
|
||||
# test = ArrayDataset(x_test, y_test)
|
||||
#
|
||||
# underlying_model = RandomForestClassifier()
|
||||
# underlying_model.fit(x_train, y_train)
|
||||
# model = SklearnClassifier(underlying_model, ModelOutputType.CLASSIFIER_MULTI_OUTPUT_BINARY_PROBABILITIES)
|
||||
# pred = model.predict(test)
|
||||
# assert (pred[0].shape[0] == x_test.shape[0])
|
||||
#
|
||||
# score = model.score(test)
|
||||
# assert (score == 1.0)
|
||||
|
||||
|
||||
def test_sklearn_regressor():
|
||||
(x_train, y_train), (x_test, y_test) = dataset_utils.get_diabetes_dataset_np()
|
||||
underlying_model = DecisionTreeRegressor()
|
||||
|
|
@ -59,7 +84,7 @@ def test_keras_classifier():
|
|||
|
||||
underlying_model.compile(loss="categorical_crossentropy", optimizer="adam", metrics=["accuracy"])
|
||||
|
||||
model = KerasClassifier(underlying_model, ModelOutputType.CLASSIFIER_PROBABILITIES)
|
||||
model = KerasClassifier(underlying_model, CLASSIFIER_SINGLE_OUTPUT_CATEGORICAL)
|
||||
|
||||
train = ArrayDataset(x_train, y_train)
|
||||
test = ArrayDataset(x_test, y_test)
|
||||
|
|
@ -97,7 +122,8 @@ def test_xgboost_classifier():
|
|||
(x_train, y_train), (x_test, y_test) = dataset_utils.get_iris_dataset_np()
|
||||
underlying_model = XGBClassifier()
|
||||
underlying_model.fit(x_train, y_train)
|
||||
model = XGBoostClassifier(underlying_model, ModelOutputType.CLASSIFIER_PROBABILITIES, input_shape=(4,), nb_classes=3)
|
||||
model = XGBoostClassifier(underlying_model, CLASSIFIER_SINGLE_OUTPUT_CATEGORICAL,
|
||||
input_shape=(4,), nb_classes=3)
|
||||
train = ArrayDataset(x_train, y_train)
|
||||
test = ArrayDataset(x_test, y_test)
|
||||
pred = model.predict(test)
|
||||
|
|
@ -115,7 +141,7 @@ def test_blackbox_classifier():
|
|||
train = ArrayDataset(x_train, y_train)
|
||||
test = ArrayDataset(x_test, y_test)
|
||||
data = Data(train, test)
|
||||
model = BlackboxClassifierPredictions(data, ModelOutputType.CLASSIFIER_SCALAR)
|
||||
model = BlackboxClassifierPredictions(data, CLASSIFIER_SINGLE_OUTPUT_CATEGORICAL)
|
||||
pred = model.predict(test)
|
||||
assert (pred.shape[0] == x_test.shape[0])
|
||||
|
||||
|
|
@ -131,7 +157,7 @@ def test_blackbox_classifier_predictions():
|
|||
train = DatasetWithPredictions(y_train, x_train)
|
||||
test = DatasetWithPredictions(y_test, x_test)
|
||||
data = Data(train, test)
|
||||
model = BlackboxClassifierPredictions(data, ModelOutputType.CLASSIFIER_SCALAR)
|
||||
model = BlackboxClassifierPredictions(data, CLASSIFIER_SINGLE_OUTPUT_CATEGORICAL)
|
||||
pred = model.predict(test)
|
||||
assert (pred.shape[0] == x_test.shape[0])
|
||||
assert model.model_type is None
|
||||
|
|
@ -146,7 +172,7 @@ def test_blackbox_classifier_predictions_y():
|
|||
train = DatasetWithPredictions(y_train, x_train, y_train)
|
||||
test = DatasetWithPredictions(y_test, x_test, y_test)
|
||||
data = Data(train, test)
|
||||
model = BlackboxClassifierPredictions(data, ModelOutputType.CLASSIFIER_SCALAR)
|
||||
model = BlackboxClassifierPredictions(data, CLASSIFIER_SINGLE_OUTPUT_CATEGORICAL)
|
||||
pred = model.predict(test)
|
||||
assert (pred.shape[0] == x_test.shape[0])
|
||||
|
||||
|
|
@ -156,14 +182,62 @@ def test_blackbox_classifier_predictions_y():
|
|||
assert model.model_type is None
|
||||
|
||||
|
||||
def test_blackbox_classifier_mismatch():
|
||||
def test_blackbox_classifier_predictions_multi_label_cat():
|
||||
(x_train, y_train), (x_test, y_test) = dataset_utils.get_iris_dataset_np()
|
||||
|
||||
train = ArrayDataset(x_train, y_train)
|
||||
test = ArrayDataset(x_test, y_test)
|
||||
# make multi-label categorical
|
||||
y_train = np.column_stack((y_train, y_train, y_train))
|
||||
y_test = np.column_stack((y_test, y_test, y_test))
|
||||
|
||||
train = DatasetWithPredictions(y_train, x_train, y_train)
|
||||
test = DatasetWithPredictions(y_test, x_test, y_test)
|
||||
data = Data(train, test)
|
||||
with pytest.raises(ValueError):
|
||||
BlackboxClassifierPredictions(data, ModelOutputType.CLASSIFIER_PROBABILITIES)
|
||||
model = BlackboxClassifierPredictions(data, CLASSIFIER_MULTI_OUTPUT_CATEGORICAL)
|
||||
pred = model.predict(test)
|
||||
assert (pred.shape[0] == x_test.shape[0])
|
||||
|
||||
score = model.score(test)
|
||||
assert (score == 1.0)
|
||||
|
||||
assert model.model_type is None
|
||||
|
||||
|
||||
def test_blackbox_classifier_predictions_multi_label_binary():
|
||||
(x_train, y_train), (x_test, y_test) = dataset_utils.get_iris_dataset_np()
|
||||
|
||||
# make multi-label binary
|
||||
y_train = np.column_stack((y_train, y_train, y_train))
|
||||
y_train[y_train > 1] = 1
|
||||
pred_train = y_train.copy().astype(float)
|
||||
pred_train[pred_train == 0] = 0.2
|
||||
pred_train[pred_train == 1] = 0.6
|
||||
y_test = np.column_stack((y_test, y_test, y_test))
|
||||
y_test[y_test > 1] = 1
|
||||
pred_test = y_test.copy().astype(float)
|
||||
pred_test[pred_test == 0] = 0.2
|
||||
pred_test[pred_test == 1] = 0.6
|
||||
|
||||
train = DatasetWithPredictions(pred_train, x_train, y_train)
|
||||
test = DatasetWithPredictions(pred_test, x_test, y_test)
|
||||
data = Data(train, test)
|
||||
model = BlackboxClassifierPredictions(data, CLASSIFIER_MULTI_OUTPUT_BINARY_PROBABILITIES)
|
||||
pred = model.predict(test)
|
||||
assert (pred.shape[0] == x_test.shape[0])
|
||||
|
||||
score = model.score(test)
|
||||
assert (score == 1.0)
|
||||
|
||||
assert model.model_type is None
|
||||
|
||||
|
||||
# def test_blackbox_classifier_mismatch():
|
||||
# (x_train, y_train), (x_test, y_test) = dataset_utils.get_iris_dataset_np()
|
||||
#
|
||||
# train = ArrayDataset(x_train, y_train)
|
||||
# test = ArrayDataset(x_test, y_test)
|
||||
# data = Data(train, test)
|
||||
# with pytest.raises(ValueError):
|
||||
# BlackboxClassifierPredictions(data, ModelOutputType.CLASSIFIER_SINGLE_OUTPUT_CLASS_PROBABILITIES)
|
||||
|
||||
|
||||
def test_blackbox_classifier_no_test():
|
||||
|
|
@ -172,7 +246,7 @@ def test_blackbox_classifier_no_test():
|
|||
train = ArrayDataset(x_train, y_train)
|
||||
|
||||
data = Data(train)
|
||||
model = BlackboxClassifierPredictions(data, ModelOutputType.CLASSIFIER_SCALAR)
|
||||
model = BlackboxClassifierPredictions(data, CLASSIFIER_SINGLE_OUTPUT_CATEGORICAL)
|
||||
pred = model.predict(train)
|
||||
assert (pred.shape[0] == x_train.shape[0])
|
||||
|
||||
|
|
@ -189,7 +263,7 @@ def test_blackbox_classifier_no_train():
|
|||
|
||||
test = ArrayDataset(x_test, y_test)
|
||||
data = Data(test=test)
|
||||
model = BlackboxClassifierPredictions(data, ModelOutputType.CLASSIFIER_SCALAR)
|
||||
model = BlackboxClassifierPredictions(data, CLASSIFIER_SINGLE_OUTPUT_CATEGORICAL)
|
||||
pred = model.predict(test)
|
||||
assert (pred.shape[0] == x_test.shape[0])
|
||||
|
||||
|
|
@ -207,7 +281,7 @@ def test_blackbox_classifier_no_test_y():
|
|||
train = ArrayDataset(x_train, y_train)
|
||||
test = ArrayDataset(x_test)
|
||||
data = Data(train, test)
|
||||
model = BlackboxClassifierPredictions(data, ModelOutputType.CLASSIFIER_SCALAR)
|
||||
model = BlackboxClassifierPredictions(data, CLASSIFIER_SINGLE_OUTPUT_CATEGORICAL)
|
||||
pred = model.predict(train)
|
||||
assert (pred.shape[0] == x_train.shape[0])
|
||||
|
||||
|
|
@ -230,7 +304,7 @@ def test_blackbox_classifier_no_train_y():
|
|||
train = ArrayDataset(x_train)
|
||||
test = ArrayDataset(x_test, y_test)
|
||||
data = Data(train, test)
|
||||
model = BlackboxClassifierPredictions(data, ModelOutputType.CLASSIFIER_SCALAR)
|
||||
model = BlackboxClassifierPredictions(data, CLASSIFIER_SINGLE_OUTPUT_CATEGORICAL)
|
||||
pred = model.predict(test)
|
||||
assert (pred.shape[0] == x_test.shape[0])
|
||||
|
||||
|
|
@ -254,7 +328,7 @@ def test_blackbox_classifier_probabilities():
|
|||
train = ArrayDataset(x_train, y_train)
|
||||
|
||||
data = Data(train)
|
||||
model = BlackboxClassifierPredictions(data, ModelOutputType.CLASSIFIER_PROBABILITIES)
|
||||
model = BlackboxClassifierPredictions(data, CLASSIFIER_SINGLE_OUTPUT_CLASS_PROBABILITIES)
|
||||
pred = model.predict(train)
|
||||
assert (pred.shape[0] == x_train.shape[0])
|
||||
assert (0.0 < pred).all()
|
||||
|
|
@ -264,6 +338,23 @@ def test_blackbox_classifier_probabilities():
|
|||
assert (score == 1.0)
|
||||
|
||||
|
||||
def test_blackbox_classifier_multi_label_probabilities():
|
||||
(x_train, _), (_, _) = dataset_utils.get_iris_dataset_np()
|
||||
y_train = np.array([[0.23, 0.56, 0.21] for i in range(105)])
|
||||
|
||||
# make multi-label categorical
|
||||
y_train = np.column_stack((y_train, y_train, y_train))
|
||||
|
||||
train = ArrayDataset(x_train, y_train)
|
||||
|
||||
data = Data(train)
|
||||
model = BlackboxClassifierPredictions(data, CLASSIFIER_MULTI_OUTPUT_CLASS_PROBABILITIES)
|
||||
pred = model.predict(train)
|
||||
assert (pred.shape[0] == x_train.shape[0])
|
||||
assert (0.0 < pred).all()
|
||||
assert (pred < 1.0).all()
|
||||
|
||||
|
||||
def test_blackbox_classifier_predict():
|
||||
def predict(x):
|
||||
return np.array([[0.23, 0.56, 0.21] for i in range(x.shape[0])])
|
||||
|
|
@ -273,7 +364,8 @@ def test_blackbox_classifier_predict():
|
|||
|
||||
train = ArrayDataset(x_train, y_train)
|
||||
|
||||
model = BlackboxClassifierPredictFunction(predict, ModelOutputType.CLASSIFIER_PROBABILITIES, (4,), 3)
|
||||
model = BlackboxClassifierPredictFunction(predict, CLASSIFIER_SINGLE_OUTPUT_CLASS_PROBABILITIES,
|
||||
(4,), 3)
|
||||
pred = model.predict(train)
|
||||
assert (pred.shape[0] == x_train.shape[0])
|
||||
assert (0.0 < pred).all()
|
||||
|
|
@ -292,7 +384,8 @@ def test_blackbox_classifier_predict_scalar():
|
|||
|
||||
train = ArrayDataset(x_train, y_train)
|
||||
|
||||
model = BlackboxClassifierPredictFunction(predict, ModelOutputType.CLASSIFIER_SCALAR, (4,), 3)
|
||||
model = BlackboxClassifierPredictFunction(predict, CLASSIFIER_SINGLE_OUTPUT_CLASS_PROBABILITIES,
|
||||
(4,), 3)
|
||||
pred = model.predict(train)
|
||||
assert (pred.shape[0] == x_train.shape[0])
|
||||
|
||||
|
|
@ -310,23 +403,23 @@ def test_is_one_hot():
|
|||
|
||||
def test_get_nb_classes():
|
||||
(_, y_train), (_, y_test) = dataset_utils.get_iris_dataset_np()
|
||||
|
||||
output_type = CLASSIFIER_SINGLE_OUTPUT_CATEGORICAL
|
||||
# shape: (x,) - not 1-hot
|
||||
nb_classes_test = get_nb_classes(y_test)
|
||||
nb_classes_train = get_nb_classes(y_train)
|
||||
nb_classes_test = get_nb_classes(y_test, output_type)
|
||||
nb_classes_train = get_nb_classes(y_train, output_type)
|
||||
assert (nb_classes_test == nb_classes_train)
|
||||
assert (nb_classes_test == 3)
|
||||
|
||||
# shape: (x,1) - not 1-hot
|
||||
nb_classes_test = get_nb_classes(y_test.reshape(-1, 1))
|
||||
nb_classes_test = get_nb_classes(y_test.reshape(-1, 1), output_type)
|
||||
assert (nb_classes_test == 3)
|
||||
|
||||
# shape: (x,3) - 1-hot
|
||||
y = to_categorical(y_test)
|
||||
nb_classes = get_nb_classes(y)
|
||||
nb_classes = get_nb_classes(y, output_type)
|
||||
assert (nb_classes == 3)
|
||||
|
||||
# gaps: 1,2,4 (0,3 missing)
|
||||
y_test[y_test == 0] = 4
|
||||
nb_classes = get_nb_classes(y_test)
|
||||
nb_classes = get_nb_classes(y_test, output_type)
|
||||
assert (nb_classes == 5)
|
||||
|
|
|
|||
|
|
@ -1,16 +1,23 @@
|
|||
import numpy as np
|
||||
from torch import nn, optim
|
||||
from torch import nn, optim, sigmoid, where, from_numpy
|
||||
from torch.nn import functional
|
||||
from torch.utils.data import DataLoader, TensorDataset
|
||||
from scipy.special import expit
|
||||
|
||||
from art.utils import check_and_transform_label_format
|
||||
from apt.utils.datasets.datasets import PytorchData
|
||||
from apt.utils.models import ModelOutputType
|
||||
from apt.utils.models import CLASSIFIER_SINGLE_OUTPUT_CLASS_LOGITS, CLASSIFIER_SINGLE_OUTPUT_BINARY_LOGITS, \
|
||||
CLASSIFIER_SINGLE_OUTPUT_BINARY_PROBABILITIES, CLASSIFIER_MULTI_OUTPUT_CLASS_LOGITS, \
|
||||
CLASSIFIER_MULTI_OUTPUT_BINARY_LOGITS
|
||||
from apt.utils.models.pytorch_model import PyTorchClassifier
|
||||
from art.utils import load_nursery
|
||||
from apt.utils import dataset_utils
|
||||
|
||||
|
||||
class pytorch_model(nn.Module):
|
||||
class PytorchModel(nn.Module):
|
||||
|
||||
def __init__(self, num_classes, num_features):
|
||||
super(pytorch_model, self).__init__()
|
||||
super(PytorchModel, self).__init__()
|
||||
|
||||
self.fc1 = nn.Sequential(
|
||||
nn.Linear(num_features, 1024),
|
||||
|
|
@ -39,7 +46,77 @@ class pytorch_model(nn.Module):
|
|||
return self.classifier(out)
|
||||
|
||||
|
||||
def test_nursery_pytorch_state_dict():
|
||||
class PytorchModelBinary(nn.Module):
|
||||
|
||||
def __init__(self, num_features):
|
||||
super(PytorchModelBinary, self).__init__()
|
||||
|
||||
self.fc2 = nn.Sequential(
|
||||
nn.Linear(num_features, 256),
|
||||
nn.Tanh(), )
|
||||
|
||||
self.fc3 = nn.Sequential(
|
||||
nn.Linear(256, 128),
|
||||
nn.Tanh(), )
|
||||
|
||||
self.fc4 = nn.Sequential(
|
||||
nn.Linear(128, 1),
|
||||
nn.Tanh(),
|
||||
)
|
||||
|
||||
def forward(self, x):
|
||||
out = self.fc2(x)
|
||||
out = self.fc3(out)
|
||||
return self.fc4(out)
|
||||
|
||||
|
||||
class PytorchModelBinarySigmoid(nn.Module):
|
||||
|
||||
def __init__(self, num_features):
|
||||
super(PytorchModelBinarySigmoid, self).__init__()
|
||||
|
||||
self.fc2 = nn.Sequential(
|
||||
nn.Linear(num_features, 256),
|
||||
nn.Tanh(), )
|
||||
|
||||
self.fc3 = nn.Sequential(
|
||||
nn.Linear(256, 128),
|
||||
nn.Tanh(), )
|
||||
|
||||
self.fc4 = nn.Sequential(
|
||||
nn.Linear(128, 1),
|
||||
nn.Tanh(),
|
||||
)
|
||||
|
||||
self.classifier = nn.Sigmoid()
|
||||
|
||||
def forward(self, x):
|
||||
out = self.fc2(x)
|
||||
out = self.fc3(out)
|
||||
out = self.fc4(out)
|
||||
return self.classifier(out)
|
||||
|
||||
|
||||
class FocalLoss(nn.Module):
|
||||
def __init__(self, gamma=2, alpha=0.5):
|
||||
super(FocalLoss, self).__init__()
|
||||
self.gamma = gamma
|
||||
self.alpha = alpha
|
||||
|
||||
def forward(self, input, target):
|
||||
bce_loss = functional.binary_cross_entropy_with_logits(input, target, reduction='none')
|
||||
|
||||
p = sigmoid(input)
|
||||
p = where(target >= 0.5, p, 1 - p)
|
||||
|
||||
modulating_factor = (1 - p) ** self.gamma
|
||||
alpha = self.alpha * target + (1 - self.alpha) * (1 - target)
|
||||
focal_loss = alpha * modulating_factor * bce_loss
|
||||
|
||||
return focal_loss.mean()
|
||||
|
||||
|
||||
def test_pytorch_nursery_state_dict():
|
||||
(x_train, y_train), (x_test, y_test), _, _ = load_nursery(test_set=0.5)
|
||||
# reduce size of training set to make attack slightly better
|
||||
train_set_size = 500
|
||||
|
|
@ -48,12 +125,15 @@ def test_nursery_pytorch_state_dict():
|
|||
x_test = x_test[:train_set_size]
|
||||
y_test = y_test[:train_set_size]
|
||||
|
||||
inner_model = pytorch_model(4, 24)
|
||||
inner_model = PytorchModel(4, 24)
|
||||
criterion = nn.CrossEntropyLoss()
|
||||
optimizer = optim.Adam(inner_model.parameters(), lr=0.01)
|
||||
|
||||
model = PyTorchClassifier(model=inner_model, output_type=ModelOutputType.CLASSIFIER_LOGITS, loss=criterion,
|
||||
optimizer=optimizer, input_shape=(24,),
|
||||
model = PyTorchClassifier(model=inner_model,
|
||||
output_type=CLASSIFIER_SINGLE_OUTPUT_CLASS_LOGITS,
|
||||
loss=criterion,
|
||||
optimizer=optimizer,
|
||||
input_shape=(24,),
|
||||
nb_classes=4)
|
||||
model.fit(PytorchData(x_train.astype(np.float32), y_train), save_entire_model=False, nb_epochs=10)
|
||||
model.load_latest_state_dict_checkpoint()
|
||||
|
|
@ -62,12 +142,12 @@ def test_nursery_pytorch_state_dict():
|
|||
assert (0 <= score <= 1)
|
||||
# python pytorch numpy
|
||||
model.load_best_state_dict_checkpoint()
|
||||
score = model.score(PytorchData(x_test.astype(np.float32), y_test))
|
||||
score = model.score(PytorchData(x_test.astype(np.float32), y_test), apply_non_linearity=expit)
|
||||
print('best model accuracy: ', score)
|
||||
assert (0 <= score <= 1)
|
||||
|
||||
|
||||
def test_nursery_pytorch_save_entire_model():
|
||||
def test_pytorch_nursery_save_entire_model():
|
||||
|
||||
(x_train, y_train), (x_test, y_test), _, _ = load_nursery(test_set=0.5)
|
||||
# reduce size of training set to make attack slightly better
|
||||
|
|
@ -77,20 +157,208 @@ def test_nursery_pytorch_save_entire_model():
|
|||
x_test = x_test[:train_set_size]
|
||||
y_test = y_test[:train_set_size]
|
||||
|
||||
model = pytorch_model(4, 24)
|
||||
inner_model = PytorchModel(4, 24)
|
||||
# model = torch.nn.DataParallel(model)
|
||||
criterion = nn.CrossEntropyLoss()
|
||||
optimizer = optim.Adam(model.parameters(), lr=0.01)
|
||||
optimizer = optim.Adam(inner_model.parameters(), lr=0.01)
|
||||
|
||||
art_model = PyTorchClassifier(model=model, output_type=ModelOutputType.CLASSIFIER_LOGITS, loss=criterion,
|
||||
optimizer=optimizer, input_shape=(24,),
|
||||
nb_classes=4)
|
||||
art_model.fit(PytorchData(x_train.astype(np.float32), y_train), save_entire_model=True, nb_epochs=10)
|
||||
model = PyTorchClassifier(model=inner_model,
|
||||
output_type=CLASSIFIER_SINGLE_OUTPUT_CLASS_LOGITS,
|
||||
loss=criterion,
|
||||
optimizer=optimizer,
|
||||
input_shape=(24,),
|
||||
nb_classes=4)
|
||||
model.fit(PytorchData(x_train.astype(np.float32), y_train), save_entire_model=True, nb_epochs=10)
|
||||
|
||||
score = art_model.score(PytorchData(x_test.astype(np.float32), y_test))
|
||||
score = model.score(PytorchData(x_test.astype(np.float32), y_test))
|
||||
print('Base model accuracy: ', score)
|
||||
assert (0 <= score <= 1)
|
||||
art_model.load_best_model_checkpoint()
|
||||
score = art_model.score(PytorchData(x_test.astype(np.float32), y_test))
|
||||
model.load_best_model_checkpoint()
|
||||
score = model.score(PytorchData(x_test.astype(np.float32), y_test), apply_non_linearity=expit)
|
||||
print('best model accuracy: ', score)
|
||||
assert (0 <= score <= 1)
|
||||
|
||||
|
||||
def test_pytorch_predictions_single_label_binary():
|
||||
x = np.array([[23, 165, 70, 10],
|
||||
[45, 158, 67, 11],
|
||||
[56, 123, 65, 58],
|
||||
[67, 154, 90, 12],
|
||||
[45, 149, 67, 56],
|
||||
[42, 166, 58, 50],
|
||||
[73, 172, 68, 10],
|
||||
[94, 168, 69, 11],
|
||||
[69, 175, 80, 61],
|
||||
[24, 181, 95, 10],
|
||||
[18, 190, 102, 53],
|
||||
[22, 161, 95, 10],
|
||||
[24, 181, 103, 10],
|
||||
[28, 184, 108, 10]])
|
||||
x = from_numpy(x)
|
||||
y = np.array([1, 1, 0, 1, 0, 0, 1, 1, 0, 1, 0, 1, 1, 1])
|
||||
y = from_numpy(y)
|
||||
data = PytorchData(x, y)
|
||||
|
||||
inner_model = PytorchModelBinary(4)
|
||||
criterion = nn.BCEWithLogitsLoss()
|
||||
optimizer = optim.Adam(inner_model.parameters(), lr=0.01)
|
||||
|
||||
model = PyTorchClassifier(model=inner_model, output_type=CLASSIFIER_SINGLE_OUTPUT_BINARY_LOGITS,
|
||||
loss=criterion,
|
||||
optimizer=optimizer, input_shape=(4,),
|
||||
nb_classes=2)
|
||||
model.fit(data, save_entire_model=False, nb_epochs=1)
|
||||
|
||||
pred = model.predict(data)
|
||||
assert (pred.shape[0] == x.shape[0])
|
||||
score = model.score(data)
|
||||
assert (0 < score <= 1.0)
|
||||
|
||||
|
||||
def test_pytorch_predictions_single_label_binary_prob():
|
||||
x = np.array([[23, 165, 70, 10],
|
||||
[45, 158, 67, 11],
|
||||
[56, 123, 65, 58],
|
||||
[67, 154, 90, 12],
|
||||
[45, 149, 67, 56],
|
||||
[42, 166, 58, 50],
|
||||
[73, 172, 68, 10],
|
||||
[94, 168, 69, 11],
|
||||
[69, 175, 80, 61],
|
||||
[24, 181, 95, 10],
|
||||
[18, 190, 102, 53],
|
||||
[22, 161, 95, 10],
|
||||
[24, 181, 103, 10],
|
||||
[28, 184, 108, 10]])
|
||||
x = from_numpy(x)
|
||||
y = np.array([1, 1, 0, 1, 0, 0, 1, 1, 0, 1, 0, 1, 1, 1])
|
||||
y = from_numpy(y)
|
||||
data = PytorchData(x, y)
|
||||
|
||||
inner_model = PytorchModelBinarySigmoid(4)
|
||||
criterion = nn.BCELoss()
|
||||
optimizer = optim.Adam(inner_model.parameters(), lr=0.01)
|
||||
|
||||
model = PyTorchClassifier(model=inner_model,
|
||||
output_type=CLASSIFIER_SINGLE_OUTPUT_BINARY_PROBABILITIES,
|
||||
loss=criterion,
|
||||
optimizer=optimizer, input_shape=(4,),
|
||||
nb_classes=2)
|
||||
model.fit(data, save_entire_model=False, nb_epochs=1)
|
||||
|
||||
pred = model.predict(data)
|
||||
assert (pred.shape[0] == x.shape[0])
|
||||
score = model.score(data)
|
||||
assert (0 < score <= 1.0)
|
||||
|
||||
|
||||
def test_pytorch_predictions_multi_label_cat():
|
||||
# This kind of model requires special training and will not be supported using the 'fit' method.
|
||||
class MultiLabelCatModel(nn.Module):
|
||||
|
||||
def __init__(self, num_classes, num_features):
|
||||
super(MultiLabelCatModel, self).__init__()
|
||||
|
||||
self.fc1 = nn.Sequential(
|
||||
nn.Linear(num_features, 256),
|
||||
nn.Tanh(), )
|
||||
|
||||
self.classifier1 = nn.Linear(256, num_classes)
|
||||
self.classifier2 = nn.Linear(256, num_classes)
|
||||
|
||||
def forward(self, x):
|
||||
out1 = self.classifier1(self.fc1(x))
|
||||
out2 = self.classifier2(self.fc1(x))
|
||||
return out1, out2
|
||||
|
||||
(x_train, y_train), (x_test, y_test) = dataset_utils.get_iris_dataset_np()
|
||||
|
||||
# make multi-label categorical
|
||||
num_classes = 3
|
||||
y_train = check_and_transform_label_format(y_train, nb_classes=num_classes)
|
||||
y_test = check_and_transform_label_format(y_test, nb_classes=num_classes)
|
||||
y_train = np.column_stack((y_train, y_train))
|
||||
y_test = np.stack([y_test, y_test], axis=1)
|
||||
test = PytorchData(x_test.astype(np.float32), y_test.astype(np.float32))
|
||||
|
||||
inner_model = MultiLabelCatModel(num_classes, 4)
|
||||
criterion = nn.CrossEntropyLoss()
|
||||
optimizer = optim.Adam(inner_model.parameters(), lr=0.01)
|
||||
|
||||
# train model
|
||||
train_dataset = TensorDataset(from_numpy(x_train.astype(np.float32)), from_numpy(y_train.astype(np.float32)))
|
||||
train_loader = DataLoader(train_dataset, batch_size=100, shuffle=True)
|
||||
|
||||
for epoch in range(5):
|
||||
# Train for one epoch
|
||||
for inputs, targets in train_loader:
|
||||
# Zero the parameter gradients
|
||||
optimizer.zero_grad()
|
||||
|
||||
# Perform prediction
|
||||
model_outputs = inner_model(inputs)
|
||||
|
||||
# Form the loss function
|
||||
loss = 0
|
||||
for i, o in enumerate(model_outputs):
|
||||
t = targets[:, i * num_classes:(i + 1) * num_classes]
|
||||
loss += criterion(o, t)
|
||||
|
||||
loss.backward()
|
||||
|
||||
optimizer.step()
|
||||
|
||||
model = PyTorchClassifier(model=inner_model,
|
||||
output_type=CLASSIFIER_MULTI_OUTPUT_CLASS_LOGITS,
|
||||
loss=criterion,
|
||||
optimizer=optimizer,
|
||||
input_shape=(24,),
|
||||
nb_classes=3)
|
||||
|
||||
pred = model.predict(test)
|
||||
assert (pred.shape[0] == x_test.shape[0])
|
||||
|
||||
score = model.score(test, apply_non_linearity=expit)
|
||||
assert (0 < score <= 1.0)
|
||||
|
||||
|
||||
def test_pytorch_predictions_multi_label_binary():
|
||||
class MultiLabelBinaryModel(nn.Module):
|
||||
def __init__(self, num_labels, num_features):
|
||||
super(MultiLabelBinaryModel, self).__init__()
|
||||
|
||||
self.fc1 = nn.Sequential(
|
||||
nn.Linear(num_features, 256),
|
||||
nn.Tanh(), )
|
||||
|
||||
self.classifier1 = nn.Linear(256, num_labels)
|
||||
|
||||
def forward(self, x):
|
||||
return self.classifier1(self.fc1(x))
|
||||
|
||||
(x_train, y_train), (x_test, y_test) = dataset_utils.get_iris_dataset_np()
|
||||
|
||||
# make multi-label binary
|
||||
y_train = np.column_stack((y_train, y_train, y_train))
|
||||
y_train[y_train > 1] = 1
|
||||
y_test = np.column_stack((y_test, y_test, y_test))
|
||||
y_test[y_test > 1] = 1
|
||||
test = PytorchData(x_test.astype(np.float32), y_test)
|
||||
|
||||
inner_model = MultiLabelBinaryModel(3, 4)
|
||||
criterion = FocalLoss()
|
||||
optimizer = optim.RMSprop(inner_model.parameters(), lr=0.01)
|
||||
|
||||
model = PyTorchClassifier(model=inner_model,
|
||||
output_type=CLASSIFIER_MULTI_OUTPUT_BINARY_LOGITS,
|
||||
loss=criterion,
|
||||
optimizer=optimizer,
|
||||
input_shape=(24,),
|
||||
nb_classes=3)
|
||||
model.fit(PytorchData(x_train.astype(np.float32), y_train.astype(np.float32)), save_entire_model=False,
|
||||
nb_epochs=10)
|
||||
pred = model.predict(test)
|
||||
assert (pred.shape[0] == x_test.shape[0])
|
||||
|
||||
score = model.score(test, apply_non_linearity=expit)
|
||||
assert (score == 1.0)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue