mirror of
https://github.com/IBM/ai-privacy-toolkit.git
synced 2026-06-23 15:48:06 +02:00
Merge with main
This commit is contained in:
abigailt
commit
dc5cc793ee
30 changed files with 2819 additions and 1066 deletions
|
|
@ -7,14 +7,14 @@ from sklearn.tree import DecisionTreeClassifier, DecisionTreeRegressor
|
|||
from sklearn.preprocessing import OneHotEncoder
|
||||
|
||||
from apt.anonymization import Anonymize
|
||||
from apt.utils.dataset_utils import get_iris_dataset, get_adult_dataset, get_nursery_dataset
|
||||
from apt.utils.dataset_utils import get_iris_dataset_np, get_adult_dataset_pd, get_nursery_dataset_pd
|
||||
from sklearn.datasets import load_diabetes
|
||||
from sklearn.model_selection import train_test_split
|
||||
from apt.utils.datasets import ArrayDataset, DATA_PANDAS_NUMPY_TYPE
|
||||
from apt.utils.datasets import ArrayDataset
|
||||
|
||||
|
||||
def test_anonymize_ndarray_iris():
|
||||
(x_train, y_train), _ = get_iris_dataset()
|
||||
(x_train, y_train), _ = get_iris_dataset_np()
|
||||
|
||||
model = DecisionTreeClassifier()
|
||||
model.fit(x_train, y_train)
|
||||
|
|
@ -31,11 +31,11 @@ def test_anonymize_ndarray_iris():
|
|||
|
||||
|
||||
def test_anonymize_pandas_adult():
|
||||
(x_train, y_train), _ = get_adult_dataset()
|
||||
(x_train, y_train), _ = get_adult_dataset_pd()
|
||||
|
||||
k = 100
|
||||
features = ['age', 'workclass', 'education-num', 'marital-status', 'occupation',
|
||||
'relationship', 'race', 'sex', 'capital-gain', 'capital-loss', 'hours-per-week', 'native-country']
|
||||
features = ['age', 'workclass', 'education-num', 'marital-status', 'occupation', 'relationship', 'race', 'sex',
|
||||
'capital-gain', 'capital-loss', 'hours-per-week', 'native-country']
|
||||
QI = ['age', 'workclass', 'education-num', 'marital-status', 'occupation', 'relationship', 'race', 'sex',
|
||||
'native-country']
|
||||
categorical_features = ['workclass', 'marital-status', 'occupation', 'relationship', 'race', 'sex',
|
||||
|
|
@ -64,8 +64,9 @@ def test_anonymize_pandas_adult():
|
|||
assert (anon.loc[:, QI].value_counts().min() >= k)
|
||||
np.testing.assert_array_equal(anon.drop(QI, axis=1), x_train.drop(QI, axis=1))
|
||||
|
||||
|
||||
def test_anonymize_pandas_nursery():
|
||||
(x_train, y_train), _ = get_nursery_dataset()
|
||||
(x_train, y_train), _ = get_nursery_dataset_pd()
|
||||
x_train = x_train.astype(str)
|
||||
|
||||
k = 100
|
||||
|
|
@ -98,7 +99,6 @@ def test_anonymize_pandas_nursery():
|
|||
|
||||
|
||||
def test_regression():
|
||||
|
||||
dataset = load_diabetes()
|
||||
x_train, x_test, y_train, y_test = train_test_split(dataset.data, dataset.target, test_size=0.5, random_state=14)
|
||||
|
||||
|
|
@ -126,9 +126,9 @@ def test_errors():
|
|||
with pytest.raises(ValueError):
|
||||
Anonymize(2, None)
|
||||
anonymizer = Anonymize(10, [0, 2])
|
||||
(x_train, y_train), (x_test, y_test) = get_iris_dataset()
|
||||
(x_train, y_train), (x_test, y_test) = get_iris_dataset_np()
|
||||
with pytest.raises(ValueError):
|
||||
anonymizer.anonymize(dataset=ArrayDataset(x_train, y_test))
|
||||
(x_train, y_train), _ = get_adult_dataset()
|
||||
(x_train, y_train), _ = get_adult_dataset_pd()
|
||||
with pytest.raises(ValueError):
|
||||
anonymizer.anonymize(dataset=ArrayDataset(x_train, y_test))
|
||||
|
|
|
|||
41
tests/test_datasets.py
Normal file
41
tests/test_datasets.py
Normal file
|
|
@ -0,0 +1,41 @@
|
|||
import pytest
|
||||
import numpy as np
|
||||
|
||||
from apt.utils.datasets import Data, DatasetWithPredictions
|
||||
from apt.utils import dataset_utils
|
||||
|
||||
|
||||
def test_dataset_predictions():
|
||||
(x_train, y_train), (_, _) = dataset_utils.get_iris_dataset_np()
|
||||
pred = np.array([[0.23, 0.56, 0.21] for i in range(105)])
|
||||
|
||||
dataset = DatasetWithPredictions(pred)
|
||||
data = Data(train=dataset)
|
||||
|
||||
new_pred = data.get_train_set().get_predictions()
|
||||
|
||||
assert np.equal(pred, new_pred).all()
|
||||
|
||||
|
||||
def test_dataset_predictions_x():
|
||||
(x_train, y_train), (_, _) = dataset_utils.get_iris_dataset_np()
|
||||
pred = np.array([[0.23, 0.56, 0.21] for i in range(105)])
|
||||
|
||||
dataset = DatasetWithPredictions(pred, x=x_train)
|
||||
data = Data(train=dataset)
|
||||
|
||||
new_pred = data.get_train_set().get_predictions()
|
||||
|
||||
assert np.equal(pred, new_pred).all()
|
||||
|
||||
|
||||
def test_dataset_predictions_y():
|
||||
(x_train, y_train), (_, _) = dataset_utils.get_iris_dataset_np()
|
||||
pred = np.array([[0.23, 0.56, 0.21] for i in range(105)])
|
||||
|
||||
dataset = DatasetWithPredictions(pred, x=x_train, y=y_train)
|
||||
data = Data(train=dataset)
|
||||
|
||||
new_pred = data.get_train_set().get_predictions()
|
||||
|
||||
assert np.equal(pred, new_pred).all()
|
||||
|
|
@ -1,6 +1,8 @@
|
|||
import pytest
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from numpy.testing import assert_almost_equal
|
||||
|
||||
from sklearn.compose import ColumnTransformer
|
||||
|
||||
from sklearn.datasets import load_boston, load_diabetes
|
||||
|
|
@ -9,11 +11,15 @@ from sklearn.model_selection import train_test_split
|
|||
from sklearn.pipeline import Pipeline
|
||||
from sklearn.preprocessing import OneHotEncoder
|
||||
|
||||
from tensorflow.keras.models import Sequential
|
||||
from tensorflow.keras.layers import Dense, Input
|
||||
|
||||
from apt.minimization import GeneralizeToRepresentative
|
||||
from sklearn.tree import DecisionTreeClassifier, DecisionTreeRegressor
|
||||
from apt.utils.dataset_utils import get_iris_dataset, get_adult_dataset, get_nursery_dataset, get_german_credit_dataset
|
||||
from apt.utils.datasets import ArrayDataset
|
||||
from apt.utils.models import SklearnClassifier, ModelOutputType, SklearnRegressor
|
||||
from apt.utils.dataset_utils import get_iris_dataset_np, get_adult_dataset_pd, get_german_credit_dataset_pd
|
||||
from apt.utils.datasets import ArrayDataset, Data
|
||||
from apt.utils.models import SklearnClassifier, ModelOutputType, SklearnRegressor, KerasClassifier, \
|
||||
BlackboxClassifierPredictions
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
|
|
@ -39,7 +45,7 @@ def test_minimizer_params(data):
|
|||
y = [1, 1, 0]
|
||||
base_est = DecisionTreeClassifier(random_state=0, min_samples_split=2,
|
||||
min_samples_leaf=1)
|
||||
model = SklearnClassifier(base_est, ModelOutputType.CLASSIFIER_VECTOR)
|
||||
model = SklearnClassifier(base_est, ModelOutputType.CLASSIFIER_PROBABILITIES)
|
||||
model.fit(ArrayDataset(X, y))
|
||||
|
||||
gen = GeneralizeToRepresentative(model, cells=cells)
|
||||
|
|
@ -63,39 +69,43 @@ def test_minimizer_fit(data):
|
|||
y = np.array([1, 1, 0, 1, 0, 0, 1, 1, 0, 1, 0])
|
||||
base_est = DecisionTreeClassifier(random_state=0, min_samples_split=2,
|
||||
min_samples_leaf=1)
|
||||
model = SklearnClassifier(base_est, ModelOutputType.CLASSIFIER_VECTOR)
|
||||
model = SklearnClassifier(base_est, ModelOutputType.CLASSIFIER_PROBABILITIES)
|
||||
model.fit(ArrayDataset(X, y))
|
||||
predictions = model.predict(X)
|
||||
ad = ArrayDataset(X)
|
||||
predictions = model.predict(ad)
|
||||
if predictions.shape[1] > 1:
|
||||
predictions = np.argmax(predictions, axis=1)
|
||||
|
||||
gen = GeneralizeToRepresentative(model, target_accuracy=0.5)
|
||||
target_accuracy = 0.5
|
||||
gen = GeneralizeToRepresentative(model, target_accuracy=target_accuracy)
|
||||
train_dataset = ArrayDataset(X, predictions, features_names=features)
|
||||
|
||||
gen.fit(dataset=train_dataset)
|
||||
transformed = gen.transform(dataset=ArrayDataset(X))
|
||||
gener = gen.generalizations_
|
||||
expexted_generalizations = {'ranges': {}, 'categories': {}, 'untouched': ['height', 'age']}
|
||||
transformed = gen.transform(dataset=ad)
|
||||
gener = gen.generalizations
|
||||
expected_generalizations = {'ranges': {}, 'categories': {}, 'untouched': ['height', 'age']}
|
||||
|
||||
for key in expexted_generalizations['ranges']:
|
||||
assert (set(expexted_generalizations['ranges'][key]) == set(gener['ranges'][key]))
|
||||
for key in expexted_generalizations['categories']:
|
||||
assert (set([frozenset(sl) for sl in expexted_generalizations['categories'][key]]) ==
|
||||
for key in expected_generalizations['ranges']:
|
||||
assert (set(expected_generalizations['ranges'][key]) == set(gener['ranges'][key]))
|
||||
for key in expected_generalizations['categories']:
|
||||
assert (set([frozenset(sl) for sl in expected_generalizations['categories'][key]]) ==
|
||||
set([frozenset(sl) for sl in gener['categories'][key]]))
|
||||
assert (set(expexted_generalizations['untouched']) == set(gener['untouched']))
|
||||
assert (set(expected_generalizations['untouched']) == set(gener['untouched']))
|
||||
modified_features = [f for f in features if
|
||||
f in expexted_generalizations['categories'].keys() or f in expexted_generalizations[
|
||||
f in expected_generalizations['categories'].keys() or f in expected_generalizations[
|
||||
'ranges'].keys()]
|
||||
indexes = []
|
||||
for i in range(len(features)):
|
||||
if features[i] in modified_features:
|
||||
indexes.append(i)
|
||||
assert ((np.delete(transformed, indexes, axis=1) == np.delete(X, indexes, axis=1)).all())
|
||||
ncp = gen.ncp_
|
||||
if len(expexted_generalizations['ranges'].keys()) > 0 or len(expexted_generalizations['categories'].keys()) > 0:
|
||||
ncp = gen.ncp
|
||||
if len(expected_generalizations['ranges'].keys()) > 0 or len(expected_generalizations['categories'].keys()) > 0:
|
||||
assert (ncp > 0)
|
||||
assert (((transformed[indexes]) != (X[indexes])).any())
|
||||
|
||||
rel_accuracy = model.score(ArrayDataset(transformed, predictions))
|
||||
assert ((rel_accuracy >= target_accuracy) or (target_accuracy - rel_accuracy) <= 0.05)
|
||||
|
||||
|
||||
def test_minimizer_fit_pandas(data):
|
||||
features = ['age', 'height', 'sex', 'ola']
|
||||
|
|
@ -131,37 +141,42 @@ def test_minimizer_fit_pandas(data):
|
|||
encoded = pd.DataFrame(encoded)
|
||||
base_est = DecisionTreeClassifier(random_state=0, min_samples_split=2,
|
||||
min_samples_leaf=1)
|
||||
model = SklearnClassifier(base_est, ModelOutputType.CLASSIFIER_VECTOR)
|
||||
model = SklearnClassifier(base_est, ModelOutputType.CLASSIFIER_PROBABILITIES)
|
||||
model.fit(ArrayDataset(encoded, y))
|
||||
predictions = model.predict(encoded)
|
||||
predictions = model.predict(ArrayDataset(encoded))
|
||||
if predictions.shape[1] > 1:
|
||||
predictions = np.argmax(predictions, axis=1)
|
||||
|
||||
# Append classifier to preprocessing pipeline.
|
||||
# Now we have a full prediction pipeline.
|
||||
gen = GeneralizeToRepresentative(model, target_accuracy=0.5,
|
||||
target_accuracy = 0.5
|
||||
gen = GeneralizeToRepresentative(model, target_accuracy=target_accuracy,
|
||||
categorical_features=categorical_features)
|
||||
train_dataset = ArrayDataset(X, predictions)
|
||||
gen.fit(dataset=train_dataset)
|
||||
transformed = gen.transform(dataset=ArrayDataset(X))
|
||||
gener = gen.generalizations_
|
||||
expexted_generalizations = {'ranges': {'age': []}, 'categories': {}, 'untouched': ['ola', 'height', 'sex']}
|
||||
gener = gen.generalizations
|
||||
expected_generalizations = {'ranges': {'age': []}, 'categories': {'sex': [['f', 'm']], 'ola': [['aa', 'bb']]},
|
||||
'untouched': ['height']}
|
||||
|
||||
for key in expexted_generalizations['ranges']:
|
||||
assert (set(expexted_generalizations['ranges'][key]) == set(gener['ranges'][key]))
|
||||
for key in expexted_generalizations['categories']:
|
||||
assert (set([frozenset(sl) for sl in expexted_generalizations['categories'][key]]) ==
|
||||
for key in expected_generalizations['ranges']:
|
||||
assert (set(expected_generalizations['ranges'][key]) == set(gener['ranges'][key]))
|
||||
for key in expected_generalizations['categories']:
|
||||
assert (set([frozenset(sl) for sl in expected_generalizations['categories'][key]]) ==
|
||||
set([frozenset(sl) for sl in gener['categories'][key]]))
|
||||
assert (set(expexted_generalizations['untouched']) == set(gener['untouched']))
|
||||
assert (set(expected_generalizations['untouched']) == set(gener['untouched']))
|
||||
modified_features = [f for f in features if
|
||||
f in expexted_generalizations['categories'].keys() or f in expexted_generalizations[
|
||||
f in expected_generalizations['categories'].keys() or f in expected_generalizations[
|
||||
'ranges'].keys()]
|
||||
np.testing.assert_array_equal(transformed.drop(modified_features, axis=1), X.drop(modified_features, axis=1))
|
||||
ncp = gen.ncp_
|
||||
if len(expexted_generalizations['ranges'].keys()) > 0 or len(expexted_generalizations['categories'].keys()) > 0:
|
||||
ncp = gen.ncp
|
||||
if len(expected_generalizations['ranges'].keys()) > 0 or len(expected_generalizations['categories'].keys()) > 0:
|
||||
assert (ncp > 0)
|
||||
assert (((transformed[modified_features]).equals(X[modified_features])) == False)
|
||||
|
||||
rel_accuracy = model.score(ArrayDataset(preprocessor.transform(transformed), predictions))
|
||||
assert ((rel_accuracy >= target_accuracy) or (target_accuracy - rel_accuracy) <= 0.05)
|
||||
|
||||
|
||||
def test_minimizer_params_categorical(data):
|
||||
# Assume three features, age, sex and height, and boolean label
|
||||
|
|
@ -212,19 +227,23 @@ def test_minimizer_params_categorical(data):
|
|||
encoded = pd.DataFrame(encoded)
|
||||
base_est = DecisionTreeClassifier(random_state=0, min_samples_split=2,
|
||||
min_samples_leaf=1)
|
||||
model = SklearnClassifier(base_est, ModelOutputType.CLASSIFIER_VECTOR)
|
||||
model = SklearnClassifier(base_est, ModelOutputType.CLASSIFIER_PROBABILITIES)
|
||||
model.fit(ArrayDataset(encoded, y))
|
||||
predictions = model.predict(encoded)
|
||||
predictions = model.predict(ArrayDataset(encoded))
|
||||
if predictions.shape[1] > 1:
|
||||
predictions = np.argmax(predictions, axis=1)
|
||||
# Append classifier to preprocessing pipeline.
|
||||
# Now we have a full prediction pipeline.
|
||||
gen = GeneralizeToRepresentative(model, target_accuracy=0.5,
|
||||
target_accuracy = 0.5
|
||||
gen = GeneralizeToRepresentative(model, target_accuracy=target_accuracy,
|
||||
categorical_features=categorical_features, cells=cells)
|
||||
train_dataset = ArrayDataset(X, predictions)
|
||||
gen.fit(dataset=train_dataset)
|
||||
transformed = gen.transform(dataset=ArrayDataset(X))
|
||||
|
||||
rel_accuracy = model.score(ArrayDataset(preprocessor.transform(transformed), predictions))
|
||||
assert ((rel_accuracy >= target_accuracy) or (target_accuracy - rel_accuracy) <= 0.05)
|
||||
|
||||
|
||||
def test_minimizer_fit_QI(data):
|
||||
features = ['age', 'height', 'weight']
|
||||
|
|
@ -244,38 +263,42 @@ def test_minimizer_fit_QI(data):
|
|||
QI = ['age', 'weight']
|
||||
base_est = DecisionTreeClassifier(random_state=0, min_samples_split=2,
|
||||
min_samples_leaf=1)
|
||||
model = SklearnClassifier(base_est, ModelOutputType.CLASSIFIER_VECTOR)
|
||||
model = SklearnClassifier(base_est, ModelOutputType.CLASSIFIER_PROBABILITIES)
|
||||
model.fit(ArrayDataset(X, y))
|
||||
predictions = model.predict(X)
|
||||
ad = ArrayDataset(X)
|
||||
predictions = model.predict(ad)
|
||||
if predictions.shape[1] > 1:
|
||||
predictions = np.argmax(predictions, axis=1)
|
||||
|
||||
gen = GeneralizeToRepresentative(model, target_accuracy=0.5, features_to_minimize=QI)
|
||||
target_accuracy = 0.5
|
||||
gen = GeneralizeToRepresentative(model, target_accuracy=target_accuracy, features_to_minimize=QI)
|
||||
train_dataset = ArrayDataset(X, predictions, features_names=features)
|
||||
gen.fit(dataset=train_dataset)
|
||||
transformed = gen.transform(dataset=ArrayDataset(X))
|
||||
gener = gen.generalizations_
|
||||
expexted_generalizations = {'ranges': {'age': [], 'weight': [67.5]}, 'categories': {}, 'untouched': ['height']}
|
||||
for key in expexted_generalizations['ranges']:
|
||||
assert (set(expexted_generalizations['ranges'][key]) == set(gener['ranges'][key]))
|
||||
for key in expexted_generalizations['categories']:
|
||||
assert (set([frozenset(sl) for sl in expexted_generalizations['categories'][key]]) ==
|
||||
transformed = gen.transform(dataset=ad)
|
||||
gener = gen.generalizations
|
||||
expected_generalizations = {'ranges': {'age': [], 'weight': [67.5]}, 'categories': {}, 'untouched': ['height']}
|
||||
for key in expected_generalizations['ranges']:
|
||||
assert (set(expected_generalizations['ranges'][key]) == set(gener['ranges'][key]))
|
||||
for key in expected_generalizations['categories']:
|
||||
assert (set([frozenset(sl) for sl in expected_generalizations['categories'][key]]) ==
|
||||
set([frozenset(sl) for sl in gener['categories'][key]]))
|
||||
assert (set(expexted_generalizations['untouched']) == set(gener['untouched']))
|
||||
assert (set(expected_generalizations['untouched']) == set(gener['untouched']))
|
||||
assert ((np.delete(transformed, [0, 2], axis=1) == np.delete(X, [0, 2], axis=1)).all())
|
||||
modified_features = [f for f in features if
|
||||
f in expexted_generalizations['categories'].keys() or f in expexted_generalizations[
|
||||
f in expected_generalizations['categories'].keys() or f in expected_generalizations[
|
||||
'ranges'].keys()]
|
||||
indexes = []
|
||||
for i in range(len(features)):
|
||||
if features[i] in modified_features:
|
||||
indexes.append(i)
|
||||
assert ((np.delete(transformed, indexes, axis=1) == np.delete(X, indexes, axis=1)).all())
|
||||
ncp = gen.ncp_
|
||||
if len(expexted_generalizations['ranges'].keys()) > 0 or len(expexted_generalizations['categories'].keys()) > 0:
|
||||
ncp = gen.ncp
|
||||
if len(expected_generalizations['ranges'].keys()) > 0 or len(expected_generalizations['categories'].keys()) > 0:
|
||||
assert (ncp > 0)
|
||||
assert (((transformed[indexes]) != (X[indexes])).any())
|
||||
|
||||
rel_accuracy = model.score(ArrayDataset(transformed, predictions))
|
||||
assert ((rel_accuracy >= target_accuracy) or (target_accuracy - rel_accuracy) <= 0.05)
|
||||
|
||||
|
||||
def test_minimizer_fit_pandas_QI(data):
|
||||
features = ['age', 'height', 'weight', 'sex', 'ola']
|
||||
|
|
@ -313,85 +336,92 @@ def test_minimizer_fit_pandas_QI(data):
|
|||
encoded = pd.DataFrame(encoded)
|
||||
base_est = DecisionTreeClassifier(random_state=0, min_samples_split=2,
|
||||
min_samples_leaf=1)
|
||||
model = SklearnClassifier(base_est, ModelOutputType.CLASSIFIER_VECTOR)
|
||||
model = SklearnClassifier(base_est, ModelOutputType.CLASSIFIER_PROBABILITIES)
|
||||
model.fit(ArrayDataset(encoded, y))
|
||||
predictions = model.predict(encoded)
|
||||
predictions = model.predict(ArrayDataset(encoded))
|
||||
if predictions.shape[1] > 1:
|
||||
predictions = np.argmax(predictions, axis=1)
|
||||
|
||||
# Append classifier to preprocessing pipeline.
|
||||
# Now we have a full prediction pipeline.
|
||||
gen = GeneralizeToRepresentative(model, target_accuracy=0.5,
|
||||
target_accuracy = 0.5
|
||||
gen = GeneralizeToRepresentative(model, target_accuracy=target_accuracy,
|
||||
categorical_features=categorical_features, features_to_minimize=QI)
|
||||
train_dataset = ArrayDataset(X, predictions)
|
||||
gen.fit(dataset=train_dataset)
|
||||
transformed = gen.transform(dataset=ArrayDataset(X))
|
||||
gener = gen.generalizations_
|
||||
expexted_generalizations = {'ranges': {'age': [], 'weight': [47.0]}, 'categories': {'ola': [['bb', 'aa']]},
|
||||
gener = gen.generalizations
|
||||
expected_generalizations = {'ranges': {'age': [], 'weight': [47.0]}, 'categories': {'ola': [['bb', 'aa']]},
|
||||
'untouched': ['height', 'sex']}
|
||||
|
||||
for key in expexted_generalizations['ranges']:
|
||||
assert (set(expexted_generalizations['ranges'][key]) == set(gener['ranges'][key]))
|
||||
for key in expexted_generalizations['categories']:
|
||||
assert (set([frozenset(sl) for sl in expexted_generalizations['categories'][key]]) ==
|
||||
for key in expected_generalizations['ranges']:
|
||||
assert (set(expected_generalizations['ranges'][key]) == set(gener['ranges'][key]))
|
||||
for key in expected_generalizations['categories']:
|
||||
assert (set([frozenset(sl) for sl in expected_generalizations['categories'][key]]) ==
|
||||
set([frozenset(sl) for sl in gener['categories'][key]]))
|
||||
assert (set(expexted_generalizations['untouched']) == set(gener['untouched']))
|
||||
assert (set(expected_generalizations['untouched']) == set(gener['untouched']))
|
||||
# assert (transformed.drop(QI, axis=1).equals(X.drop(QI, axis=1)))
|
||||
np.testing.assert_array_equal(transformed.drop(QI, axis=1), X.drop(QI, axis=1))
|
||||
modified_features = [f for f in features if
|
||||
f in expexted_generalizations['categories'].keys() or f in expexted_generalizations[
|
||||
f in expected_generalizations['categories'].keys() or f in expected_generalizations[
|
||||
'ranges'].keys()]
|
||||
# assert (transformed.drop(modified_features, axis=1).equals(X.drop(modified_features, axis=1)))
|
||||
np.testing.assert_array_equal(transformed.drop(modified_features, axis=1), X.drop(modified_features, axis=1))
|
||||
ncp = gen.ncp_
|
||||
if len(expexted_generalizations['ranges'].keys()) > 0 or len(expexted_generalizations['categories'].keys()) > 0:
|
||||
ncp = gen.ncp
|
||||
if len(expected_generalizations['ranges'].keys()) > 0 or len(expected_generalizations['categories'].keys()) > 0:
|
||||
assert (ncp > 0)
|
||||
assert (((transformed[modified_features]).equals(X[modified_features])) == False)
|
||||
|
||||
rel_accuracy = model.score(ArrayDataset(preprocessor.transform(transformed), predictions))
|
||||
assert ((rel_accuracy >= target_accuracy) or (target_accuracy - rel_accuracy) <= 0.05)
|
||||
|
||||
|
||||
def test_minimize_ndarray_iris():
|
||||
features = ['sepal length (cm)', 'sepal width (cm)', 'petal length (cm)', 'petal width (cm)']
|
||||
(x_train, y_train), (x_test, y_test) = get_iris_dataset()
|
||||
(x_train, y_train), (x_test, y_test) = get_iris_dataset_np()
|
||||
QI = ['sepal length (cm)', 'petal length (cm)']
|
||||
base_est = DecisionTreeClassifier(random_state=0, min_samples_split=2,
|
||||
min_samples_leaf=1)
|
||||
model = SklearnClassifier(base_est, ModelOutputType.CLASSIFIER_VECTOR)
|
||||
model = SklearnClassifier(base_est, ModelOutputType.CLASSIFIER_PROBABILITIES)
|
||||
model.fit(ArrayDataset(x_train, y_train))
|
||||
predictions = model.predict(x_train)
|
||||
predictions = model.predict(ArrayDataset(x_train))
|
||||
if predictions.shape[1] > 1:
|
||||
predictions = np.argmax(predictions, axis=1)
|
||||
|
||||
gen = GeneralizeToRepresentative(model, target_accuracy=0.3, features_to_minimize=QI)
|
||||
target_accuracy = 0.3
|
||||
gen = GeneralizeToRepresentative(model, target_accuracy=target_accuracy, features_to_minimize=QI)
|
||||
# gen.fit(dataset=ArrayDataset(x_train, predictions))
|
||||
transformed = gen.fit_transform(dataset=ArrayDataset(x_train, predictions, features_names=features))
|
||||
gener = gen.generalizations_
|
||||
expexted_generalizations = {'ranges': {'sepal length (cm)': [], 'petal length (cm)': [2.449999988079071]},
|
||||
gener = gen.generalizations
|
||||
expected_generalizations = {'ranges': {'sepal length (cm)': [], 'petal length (cm)': [2.449999988079071]},
|
||||
'categories': {}, 'untouched': ['petal width (cm)', 'sepal width (cm)']}
|
||||
|
||||
for key in expexted_generalizations['ranges']:
|
||||
assert (set(expexted_generalizations['ranges'][key]) == set(gener['ranges'][key]))
|
||||
for key in expexted_generalizations['categories']:
|
||||
assert (set([frozenset(sl) for sl in expexted_generalizations['categories'][key]]) ==
|
||||
for key in expected_generalizations['ranges']:
|
||||
assert (set(expected_generalizations['ranges'][key]) == set(gener['ranges'][key]))
|
||||
for key in expected_generalizations['categories']:
|
||||
assert (set([frozenset(sl) for sl in expected_generalizations['categories'][key]]) ==
|
||||
set([frozenset(sl) for sl in gener['categories'][key]]))
|
||||
assert (set(expexted_generalizations['untouched']) == set(gener['untouched']))
|
||||
assert (set(expected_generalizations['untouched']) == set(gener['untouched']))
|
||||
assert ((np.delete(transformed, [0, 2], axis=1) == np.delete(x_train, [0, 2], axis=1)).all())
|
||||
|
||||
modified_features = [f for f in features if
|
||||
f in expexted_generalizations['categories'].keys() or f in expexted_generalizations[
|
||||
f in expected_generalizations['categories'].keys() or f in expected_generalizations[
|
||||
'ranges'].keys()]
|
||||
indexes = []
|
||||
for i in range(len(features)):
|
||||
if features[i] in modified_features:
|
||||
indexes.append(i)
|
||||
assert ((np.delete(transformed, indexes, axis=1) == np.delete(x_train, indexes, axis=1)).all())
|
||||
ncp = gen.ncp_
|
||||
if len(expexted_generalizations['ranges'].keys()) > 0 or len(expexted_generalizations['categories'].keys()) > 0:
|
||||
ncp = gen.ncp
|
||||
if len(expected_generalizations['ranges'].keys()) > 0 or len(expected_generalizations['categories'].keys()) > 0:
|
||||
assert (ncp > 0)
|
||||
assert (((transformed[indexes]) != (x_train[indexes])).any())
|
||||
|
||||
rel_accuracy = model.score(ArrayDataset(transformed, predictions))
|
||||
assert ((rel_accuracy >= target_accuracy) or (target_accuracy - rel_accuracy) <= 0.05)
|
||||
|
||||
|
||||
def test_minimize_pandas_adult():
|
||||
(x_train, y_train), (x_test, y_test) = get_adult_dataset()
|
||||
(x_train, y_train), (x_test, y_test) = get_adult_dataset_pd()
|
||||
x_train = x_train.head(1000)
|
||||
y_train = y_train.head(1000)
|
||||
|
||||
|
|
@ -420,18 +450,18 @@ def test_minimize_pandas_adult():
|
|||
encoded = pd.DataFrame(encoded)
|
||||
base_est = DecisionTreeClassifier(random_state=0, min_samples_split=2,
|
||||
min_samples_leaf=1)
|
||||
model = SklearnClassifier(base_est, ModelOutputType.CLASSIFIER_VECTOR)
|
||||
model = SklearnClassifier(base_est, ModelOutputType.CLASSIFIER_PROBABILITIES)
|
||||
model.fit(ArrayDataset(encoded, y_train))
|
||||
predictions = model.predict(encoded)
|
||||
predictions = model.predict(ArrayDataset(encoded))
|
||||
if predictions.shape[1] > 1:
|
||||
predictions = np.argmax(predictions, axis=1)
|
||||
|
||||
gen = GeneralizeToRepresentative(model, target_accuracy=0.7,
|
||||
target_accuracy = 0.7
|
||||
gen = GeneralizeToRepresentative(model, target_accuracy=target_accuracy,
|
||||
categorical_features=categorical_features, features_to_minimize=QI)
|
||||
gen.fit(dataset=ArrayDataset(x_train, predictions, features_names=features))
|
||||
transformed = gen.transform(dataset=ArrayDataset(x_train))
|
||||
gener = gen.generalizations_
|
||||
expexted_generalizations = {'ranges': {'age': [], 'education-num': []}, 'categories': {
|
||||
gener = gen.generalizations
|
||||
expected_generalizations = {'ranges': {'age': [], 'education-num': []}, 'categories': {
|
||||
'workclass': [['Self-emp-not-inc', 'Private', 'Federal-gov', 'Self-emp-inc', '?', 'Local-gov', 'State-gov']],
|
||||
'marital-status': [
|
||||
['Divorced', 'Married-AF-spouse', 'Married-spouse-absent', 'Widowed', 'Separated', 'Married-civ-spouse',
|
||||
|
|
@ -445,28 +475,31 @@ def test_minimize_pandas_adult():
|
|||
['Euro_1', 'LatinAmerica', 'BritishCommonwealth', 'SouthAmerica', 'UnitedStates', 'China', 'Euro_2',
|
||||
'SE_Asia', 'Other', 'Unknown']]}, 'untouched': ['capital-loss', 'hours-per-week', 'capital-gain']}
|
||||
|
||||
for key in expexted_generalizations['ranges']:
|
||||
assert (set(expexted_generalizations['ranges'][key]) == set(gener['ranges'][key]))
|
||||
for key in expexted_generalizations['categories']:
|
||||
assert (set([frozenset(sl) for sl in expexted_generalizations['categories'][key]]) ==
|
||||
for key in expected_generalizations['ranges']:
|
||||
assert (set(expected_generalizations['ranges'][key]) == set(gener['ranges'][key]))
|
||||
for key in expected_generalizations['categories']:
|
||||
assert (set([frozenset(sl) for sl in expected_generalizations['categories'][key]]) ==
|
||||
set([frozenset(sl) for sl in gener['categories'][key]]))
|
||||
assert (set(expexted_generalizations['untouched']) == set(gener['untouched']))
|
||||
assert (set(expected_generalizations['untouched']) == set(gener['untouched']))
|
||||
# assert (transformed.drop(QI, axis=1).equals(x_train.drop(QI, axis=1)))
|
||||
np.testing.assert_array_equal(transformed.drop(QI, axis=1), x_train.drop(QI, axis=1))
|
||||
|
||||
modified_features = [f for f in features if
|
||||
f in expexted_generalizations['categories'].keys() or f in expexted_generalizations[
|
||||
f in expected_generalizations['categories'].keys() or f in expected_generalizations[
|
||||
'ranges'].keys()]
|
||||
# assert (transformed.drop(modified_features, axis=1).equals(x_train.drop(modified_features, axis=1)))
|
||||
np.testing.assert_array_equal(transformed.drop(modified_features, axis=1), x_train.drop(modified_features, axis=1))
|
||||
ncp = gen.ncp_
|
||||
if len(expexted_generalizations['ranges'].keys()) > 0 or len(expexted_generalizations['categories'].keys()) > 0:
|
||||
ncp = gen.ncp
|
||||
if len(expected_generalizations['ranges'].keys()) > 0 or len(expected_generalizations['categories'].keys()) > 0:
|
||||
assert (ncp > 0)
|
||||
assert (((transformed[modified_features]).equals(x_train[modified_features])) == False)
|
||||
|
||||
rel_accuracy = model.score(ArrayDataset(preprocessor.transform(transformed), predictions))
|
||||
assert ((rel_accuracy >= target_accuracy) or (target_accuracy - rel_accuracy) <= 0.05)
|
||||
|
||||
|
||||
def test_german_credit_pandas():
|
||||
(x_train, y_train), (x_test, y_test) = get_german_credit_dataset()
|
||||
(x_train, y_train), (x_test, y_test) = get_german_credit_dataset_pd()
|
||||
features = ["Existing_checking_account", "Duration_in_month", "Credit_history", "Purpose", "Credit_amount",
|
||||
"Savings_account", "Present_employment_since", "Installment_rate", "Personal_status_sex", "debtors",
|
||||
"Present_residence", "Property", "Age", "Other_installment_plans", "Housing",
|
||||
|
|
@ -493,18 +526,18 @@ def test_german_credit_pandas():
|
|||
encoded = pd.DataFrame(encoded)
|
||||
base_est = DecisionTreeClassifier(random_state=0, min_samples_split=2,
|
||||
min_samples_leaf=1)
|
||||
model = SklearnClassifier(base_est, ModelOutputType.CLASSIFIER_VECTOR)
|
||||
model = SklearnClassifier(base_est, ModelOutputType.CLASSIFIER_PROBABILITIES)
|
||||
model.fit(ArrayDataset(encoded, y_train))
|
||||
predictions = model.predict(encoded)
|
||||
predictions = model.predict(ArrayDataset(encoded))
|
||||
if predictions.shape[1] > 1:
|
||||
predictions = np.argmax(predictions, axis=1)
|
||||
|
||||
gen = GeneralizeToRepresentative(model, target_accuracy=0.7,
|
||||
target_accuracy = 0.7
|
||||
gen = GeneralizeToRepresentative(model, target_accuracy=target_accuracy,
|
||||
categorical_features=categorical_features, features_to_minimize=QI)
|
||||
gen.fit(dataset=ArrayDataset(x_train, predictions))
|
||||
transformed = gen.transform(dataset=ArrayDataset(x_train))
|
||||
gener = gen.generalizations_
|
||||
expexted_generalizations = {'ranges': {'Duration_in_month': [31.5]},
|
||||
gener = gen.generalizations
|
||||
expected_generalizations = {'ranges': {'Duration_in_month': [31.5]},
|
||||
'categories': {'Credit_history': [['A30', 'A32', 'A31', 'A34', 'A33']], 'Purpose': [
|
||||
['A41', 'A46', 'A43', 'A40', 'A44', 'A410', 'A49', 'A45', 'A48', 'A42']],
|
||||
'debtors': [['A101', 'A102', 'A103']],
|
||||
|
|
@ -518,25 +551,28 @@ def test_german_credit_pandas():
|
|||
'Age', 'Existing_checking_account', 'Credit_amount',
|
||||
'Present_employment_since']}
|
||||
|
||||
for key in expexted_generalizations['ranges']:
|
||||
assert (set(expexted_generalizations['ranges'][key]) == set(gener['ranges'][key]))
|
||||
for key in expexted_generalizations['categories']:
|
||||
assert (set([frozenset(sl) for sl in expexted_generalizations['categories'][key]]) ==
|
||||
for key in expected_generalizations['ranges']:
|
||||
assert (set(expected_generalizations['ranges'][key]) == set(gener['ranges'][key]))
|
||||
for key in expected_generalizations['categories']:
|
||||
assert (set([frozenset(sl) for sl in expected_generalizations['categories'][key]]) ==
|
||||
set([frozenset(sl) for sl in gener['categories'][key]]))
|
||||
assert (set(expexted_generalizations['untouched']) == set(gener['untouched']))
|
||||
assert (set(expected_generalizations['untouched']) == set(gener['untouched']))
|
||||
# assert (transformed.drop(QI, axis=1).equals(x_train.drop(QI, axis=1)))
|
||||
np.testing.assert_array_equal(transformed.drop(QI, axis=1), x_train.drop(QI, axis=1))
|
||||
|
||||
modified_features = [f for f in features if
|
||||
f in expexted_generalizations['categories'].keys() or f in expexted_generalizations[
|
||||
f in expected_generalizations['categories'].keys() or f in expected_generalizations[
|
||||
'ranges'].keys()]
|
||||
# assert (transformed.drop(modified_features, axis=1).equals(x_train.drop(modified_features, axis=1)))
|
||||
np.testing.assert_array_equal(transformed.drop(modified_features, axis=1), x_train.drop(modified_features, axis=1))
|
||||
ncp = gen.ncp_
|
||||
if len(expexted_generalizations['ranges'].keys()) > 0 or len(expexted_generalizations['categories'].keys()) > 0:
|
||||
ncp = gen.ncp
|
||||
if len(expected_generalizations['ranges'].keys()) > 0 or len(expected_generalizations['categories'].keys()) > 0:
|
||||
assert (ncp > 0)
|
||||
assert (((transformed[modified_features]).equals(x_train[modified_features])) == False)
|
||||
|
||||
rel_accuracy = model.score(ArrayDataset(preprocessor.transform(transformed), predictions))
|
||||
assert ((rel_accuracy >= target_accuracy) or (target_accuracy - rel_accuracy) <= 0.05)
|
||||
|
||||
|
||||
def test_regression():
|
||||
dataset = load_diabetes()
|
||||
|
|
@ -545,20 +581,21 @@ def test_regression():
|
|||
base_est = DecisionTreeRegressor(random_state=10, min_samples_split=2)
|
||||
model = SklearnRegressor(base_est)
|
||||
model.fit(ArrayDataset(x_train, y_train))
|
||||
predictions = model.predict(x_train)
|
||||
predictions = model.predict(ArrayDataset(x_train))
|
||||
QI = ['age', 'bmi', 's2', 's5']
|
||||
features = ['age', 'sex', 'bmi', 'bp',
|
||||
's1', 's2', 's3', 's4', 's5', 's6']
|
||||
|
||||
gen = GeneralizeToRepresentative(model, target_accuracy=0.7, is_regression=True,
|
||||
target_accuracy = 0.7
|
||||
gen = GeneralizeToRepresentative(model, target_accuracy=target_accuracy, is_regression=True,
|
||||
features_to_minimize=QI)
|
||||
gen.fit(dataset=ArrayDataset(x_train, predictions, features_names=features))
|
||||
transformed = gen.transform(dataset=ArrayDataset(x_train, features_names=features))
|
||||
print('Base model accuracy (R2 score): ', model.score(ArrayDataset(x_test, y_test)))
|
||||
model.fit(ArrayDataset(transformed, y_train))
|
||||
print('Base model accuracy (R2 score) after anonymization: ', model.score(ArrayDataset(x_test, y_test)))
|
||||
gener = gen.generalizations_
|
||||
expexted_generalizations = {'ranges': {
|
||||
gener = gen.generalizations
|
||||
expected_generalizations = {'ranges': {
|
||||
'age': [-0.07816532626748085, -0.07090024650096893, -0.05637009255588055, -0.05092128552496433,
|
||||
-0.04728874587453902, -0.04547247663140297, -0.04183994047343731, -0.027309784665703773,
|
||||
-0.023677248042076826, -0.020044708624482155, -0.01641217083670199, -0.001882016600575298,
|
||||
|
|
@ -586,27 +623,30 @@ def test_regression():
|
|||
0.061315815430134535, 0.06272498145699501, 0.06460387445986271]}, 'categories': {},
|
||||
'untouched': ['s5', 's3', 'bp', 's1', 'sex', 's6', 's4']}
|
||||
|
||||
for key in expexted_generalizations['ranges']:
|
||||
assert (set(expexted_generalizations['ranges'][key]) == set(gener['ranges'][key]))
|
||||
for key in expexted_generalizations['categories']:
|
||||
assert (set([frozenset(sl) for sl in expexted_generalizations['categories'][key]]) ==
|
||||
for key in expected_generalizations['ranges']:
|
||||
assert (set(expected_generalizations['ranges'][key]) == set(gener['ranges'][key]))
|
||||
for key in expected_generalizations['categories']:
|
||||
assert (set([frozenset(sl) for sl in expected_generalizations['categories'][key]]) ==
|
||||
set([frozenset(sl) for sl in gener['categories'][key]]))
|
||||
assert (set(expexted_generalizations['untouched']) == set(gener['untouched']))
|
||||
assert (set(expected_generalizations['untouched']) == set(gener['untouched']))
|
||||
assert ((np.delete(transformed, [0, 2, 5, 8], axis=1) == np.delete(x_train, [0, 2, 5, 8], axis=1)).all())
|
||||
|
||||
modified_features = [f for f in features if
|
||||
f in expexted_generalizations['categories'].keys() or f in expexted_generalizations[
|
||||
f in expected_generalizations['categories'].keys() or f in expected_generalizations[
|
||||
'ranges'].keys()]
|
||||
indexes = []
|
||||
for i in range(len(features)):
|
||||
if features[i] in modified_features:
|
||||
indexes.append(i)
|
||||
assert ((np.delete(transformed, indexes, axis=1) == np.delete(x_train, indexes, axis=1)).all())
|
||||
ncp = gen.ncp_
|
||||
if len(expexted_generalizations['ranges'].keys()) > 0 or len(expexted_generalizations['categories'].keys()) > 0:
|
||||
ncp = gen.ncp
|
||||
if len(expected_generalizations['ranges'].keys()) > 0 or len(expected_generalizations['categories'].keys()) > 0:
|
||||
assert (ncp > 0)
|
||||
assert (((transformed[indexes]) != (x_train[indexes])).any())
|
||||
|
||||
rel_accuracy = model.score(ArrayDataset(transformed, predictions))
|
||||
assert ((rel_accuracy >= target_accuracy) or (target_accuracy - rel_accuracy) <= 0.05)
|
||||
|
||||
|
||||
def test_X_y(data):
|
||||
features = [0, 1, 2]
|
||||
|
|
@ -626,37 +666,41 @@ def test_X_y(data):
|
|||
QI = [0, 2]
|
||||
base_est = DecisionTreeClassifier(random_state=0, min_samples_split=2,
|
||||
min_samples_leaf=1)
|
||||
model = SklearnClassifier(base_est, ModelOutputType.CLASSIFIER_VECTOR)
|
||||
model = SklearnClassifier(base_est, ModelOutputType.CLASSIFIER_PROBABILITIES)
|
||||
model.fit(ArrayDataset(X, y))
|
||||
predictions = model.predict(X)
|
||||
ad = ArrayDataset(X)
|
||||
predictions = model.predict(ad)
|
||||
if predictions.shape[1] > 1:
|
||||
predictions = np.argmax(predictions, axis=1)
|
||||
|
||||
gen = GeneralizeToRepresentative(model, target_accuracy=0.5, features_to_minimize=QI)
|
||||
target_accuracy = 0.5
|
||||
gen = GeneralizeToRepresentative(model, target_accuracy=target_accuracy, features_to_minimize=QI)
|
||||
gen.fit(X=X, y=predictions)
|
||||
transformed = gen.transform(X)
|
||||
gener = gen.generalizations_
|
||||
expexted_generalizations = {'ranges': {'0': [], '2': [67.5]}, 'categories': {}, 'untouched': ['1']}
|
||||
for key in expexted_generalizations['ranges']:
|
||||
assert (set(expexted_generalizations['ranges'][key]) == set(gener['ranges'][key]))
|
||||
for key in expexted_generalizations['categories']:
|
||||
assert (set([frozenset(sl) for sl in expexted_generalizations['categories'][key]]) ==
|
||||
gener = gen.generalizations
|
||||
expected_generalizations = {'ranges': {'0': [], '2': [67.5]}, 'categories': {}, 'untouched': ['1']}
|
||||
for key in expected_generalizations['ranges']:
|
||||
assert (set(expected_generalizations['ranges'][key]) == set(gener['ranges'][key]))
|
||||
for key in expected_generalizations['categories']:
|
||||
assert (set([frozenset(sl) for sl in expected_generalizations['categories'][key]]) ==
|
||||
set([frozenset(sl) for sl in gener['categories'][key]]))
|
||||
assert (set(expexted_generalizations['untouched']) == set(gener['untouched']))
|
||||
assert (set(expected_generalizations['untouched']) == set(gener['untouched']))
|
||||
assert ((np.delete(transformed, [0, 2], axis=1) == np.delete(X, [0, 2], axis=1)).all())
|
||||
modified_features = [f for f in features if
|
||||
str(f) in expexted_generalizations['categories'].keys() or str(f) in expexted_generalizations[
|
||||
str(f) in expected_generalizations['categories'].keys() or str(f) in expected_generalizations[
|
||||
'ranges'].keys()]
|
||||
indexes = []
|
||||
for i in range(len(features)):
|
||||
if features[i] in modified_features:
|
||||
indexes.append(i)
|
||||
assert ((np.delete(transformed, indexes, axis=1) == np.delete(X, indexes, axis=1)).all())
|
||||
ncp = gen.ncp_
|
||||
if len(expexted_generalizations['ranges'].keys()) > 0 or len(expexted_generalizations['categories'].keys()) > 0:
|
||||
ncp = gen.ncp
|
||||
if len(expected_generalizations['ranges'].keys()) > 0 or len(expected_generalizations['categories'].keys()) > 0:
|
||||
assert (ncp > 0)
|
||||
assert (((transformed[indexes]) != (X[indexes])).any())
|
||||
|
||||
rel_accuracy = model.score(ArrayDataset(transformed, predictions))
|
||||
assert ((rel_accuracy >= target_accuracy) or (target_accuracy - rel_accuracy) <= 0.05)
|
||||
|
||||
|
||||
def test_X_y_features_names(data):
|
||||
features = ['age', 'height', 'weight']
|
||||
|
|
@ -676,37 +720,41 @@ def test_X_y_features_names(data):
|
|||
QI = ['age', 'weight']
|
||||
base_est = DecisionTreeClassifier(random_state=0, min_samples_split=2,
|
||||
min_samples_leaf=1)
|
||||
model = SklearnClassifier(base_est, ModelOutputType.CLASSIFIER_VECTOR)
|
||||
model = SklearnClassifier(base_est, ModelOutputType.CLASSIFIER_PROBABILITIES)
|
||||
model.fit(ArrayDataset(X, y))
|
||||
predictions = model.predict(X)
|
||||
ad = ArrayDataset(X)
|
||||
predictions = model.predict(ad)
|
||||
if predictions.shape[1] > 1:
|
||||
predictions = np.argmax(predictions, axis=1)
|
||||
|
||||
gen = GeneralizeToRepresentative(model, target_accuracy=0.5, features_to_minimize=QI)
|
||||
target_accuracy = 0.5
|
||||
gen = GeneralizeToRepresentative(model, target_accuracy=target_accuracy, features_to_minimize=QI)
|
||||
gen.fit(X=X, y=predictions, features_names=features)
|
||||
transformed = gen.transform(X=X, features_names=features)
|
||||
gener = gen.generalizations_
|
||||
expexted_generalizations = {'ranges': {'age': [], 'weight': [67.5]}, 'categories': {}, 'untouched': ['height']}
|
||||
for key in expexted_generalizations['ranges']:
|
||||
assert (set(expexted_generalizations['ranges'][key]) == set(gener['ranges'][key]))
|
||||
for key in expexted_generalizations['categories']:
|
||||
assert (set([frozenset(sl) for sl in expexted_generalizations['categories'][key]]) ==
|
||||
gener = gen.generalizations
|
||||
expected_generalizations = {'ranges': {'age': [], 'weight': [67.5]}, 'categories': {}, 'untouched': ['height']}
|
||||
for key in expected_generalizations['ranges']:
|
||||
assert (set(expected_generalizations['ranges'][key]) == set(gener['ranges'][key]))
|
||||
for key in expected_generalizations['categories']:
|
||||
assert (set([frozenset(sl) for sl in expected_generalizations['categories'][key]]) ==
|
||||
set([frozenset(sl) for sl in gener['categories'][key]]))
|
||||
assert (set(expexted_generalizations['untouched']) == set(gener['untouched']))
|
||||
assert (set(expected_generalizations['untouched']) == set(gener['untouched']))
|
||||
assert ((np.delete(transformed, [0, 2], axis=1) == np.delete(X, [0, 2], axis=1)).all())
|
||||
modified_features = [f for f in features if
|
||||
f in expexted_generalizations['categories'].keys() or f in expexted_generalizations[
|
||||
f in expected_generalizations['categories'].keys() or f in expected_generalizations[
|
||||
'ranges'].keys()]
|
||||
indexes = []
|
||||
for i in range(len(features)):
|
||||
if features[i] in modified_features:
|
||||
indexes.append(i)
|
||||
assert ((np.delete(transformed, indexes, axis=1) == np.delete(X, indexes, axis=1)).all())
|
||||
ncp = gen.ncp_
|
||||
if len(expexted_generalizations['ranges'].keys()) > 0 or len(expexted_generalizations['categories'].keys()) > 0:
|
||||
ncp = gen.ncp
|
||||
if len(expected_generalizations['ranges'].keys()) > 0 or len(expected_generalizations['categories'].keys()) > 0:
|
||||
assert (ncp > 0)
|
||||
assert (((transformed[indexes]) != (X[indexes])).any())
|
||||
|
||||
rel_accuracy = model.score(ArrayDataset(transformed, predictions))
|
||||
assert ((rel_accuracy >= target_accuracy) or (target_accuracy - rel_accuracy) <= 0.05)
|
||||
|
||||
|
||||
def test_BaseEstimator_classification(data):
|
||||
features = ['age', 'height', 'weight', 'sex', 'ola']
|
||||
|
|
@ -750,33 +798,37 @@ def test_BaseEstimator_classification(data):
|
|||
|
||||
# Append classifier to preprocessing pipeline.
|
||||
# Now we have a full prediction pipeline.
|
||||
gen = GeneralizeToRepresentative(model, target_accuracy=0.5,
|
||||
target_accuracy = 0.5
|
||||
gen = GeneralizeToRepresentative(model, target_accuracy=target_accuracy,
|
||||
categorical_features=categorical_features, features_to_minimize=QI)
|
||||
train_dataset = ArrayDataset(X, predictions)
|
||||
gen.fit(dataset=train_dataset)
|
||||
transformed = gen.transform(dataset=ArrayDataset(X))
|
||||
gener = gen.generalizations_
|
||||
expexted_generalizations = {'ranges': {'age': [], 'weight': [47.0]}, 'categories': {'ola': [['bb', 'aa']]},
|
||||
gener = gen.generalizations
|
||||
expected_generalizations = {'ranges': {'age': [], 'weight': [47.0]}, 'categories': {'ola': [['bb', 'aa']]},
|
||||
'untouched': ['height', 'sex']}
|
||||
|
||||
for key in expexted_generalizations['ranges']:
|
||||
assert (set(expexted_generalizations['ranges'][key]) == set(gener['ranges'][key]))
|
||||
for key in expexted_generalizations['categories']:
|
||||
assert (set([frozenset(sl) for sl in expexted_generalizations['categories'][key]]) ==
|
||||
for key in expected_generalizations['ranges']:
|
||||
assert (set(expected_generalizations['ranges'][key]) == set(gener['ranges'][key]))
|
||||
for key in expected_generalizations['categories']:
|
||||
assert (set([frozenset(sl) for sl in expected_generalizations['categories'][key]]) ==
|
||||
set([frozenset(sl) for sl in gener['categories'][key]]))
|
||||
assert (set(expexted_generalizations['untouched']) == set(gener['untouched']))
|
||||
assert (set(expected_generalizations['untouched']) == set(gener['untouched']))
|
||||
# assert (transformed.drop(QI, axis=1).equals(X.drop(QI, axis=1)))
|
||||
np.testing.assert_array_equal(transformed.drop(QI, axis=1), X.drop(QI, axis=1))
|
||||
modified_features = [f for f in features if
|
||||
f in expexted_generalizations['categories'].keys() or f in expexted_generalizations[
|
||||
f in expected_generalizations['categories'].keys() or f in expected_generalizations[
|
||||
'ranges'].keys()]
|
||||
# assert (transformed.drop(modified_features, axis=1).equals(X.drop(modified_features, axis=1)))
|
||||
np.testing.assert_array_equal(transformed.drop(modified_features, axis=1), X.drop(modified_features, axis=1))
|
||||
ncp = gen.ncp_
|
||||
if len(expexted_generalizations['ranges'].keys()) > 0 or len(expexted_generalizations['categories'].keys()) > 0:
|
||||
ncp = gen.ncp
|
||||
if len(expected_generalizations['ranges'].keys()) > 0 or len(expected_generalizations['categories'].keys()) > 0:
|
||||
assert (ncp > 0)
|
||||
assert (((transformed[modified_features]).equals(X[modified_features])) == False)
|
||||
|
||||
rel_accuracy = model.score(preprocessor.transform(transformed), predictions)
|
||||
assert ((rel_accuracy >= target_accuracy) or (target_accuracy - rel_accuracy) <= 0.05)
|
||||
|
||||
|
||||
def test_BaseEstimator_regression():
|
||||
dataset = load_diabetes()
|
||||
|
|
@ -789,16 +841,16 @@ def test_BaseEstimator_regression():
|
|||
QI = ['age', 'bmi', 's2', 's5']
|
||||
features = ['age', 'sex', 'bmi', 'bp',
|
||||
's1', 's2', 's3', 's4', 's5', 's6']
|
||||
|
||||
gen = GeneralizeToRepresentative(model, target_accuracy=0.7, is_regression=True,
|
||||
target_accuracy = 0.7
|
||||
gen = GeneralizeToRepresentative(model, target_accuracy=target_accuracy, is_regression=True,
|
||||
features_to_minimize=QI)
|
||||
gen.fit(dataset=ArrayDataset(x_train, predictions, features_names=features))
|
||||
transformed = gen.transform(dataset=ArrayDataset(x_train, features_names=features))
|
||||
print('Base model accuracy (R2 score): ', model.score(x_test, y_test))
|
||||
model.fit(transformed, y_train)
|
||||
print('Base model accuracy (R2 score) after minimization: ', model.score(x_test, y_test))
|
||||
gener = gen.generalizations_
|
||||
expexted_generalizations = {'ranges': {
|
||||
gener = gen.generalizations
|
||||
expected_generalizations = {'ranges': {
|
||||
'age': [-0.07816532626748085, -0.07090024650096893, -0.05637009255588055, -0.05092128552496433,
|
||||
-0.04728874587453902, -0.04547247663140297, -0.04183994047343731, -0.027309784665703773,
|
||||
-0.023677248042076826, -0.020044708624482155, -0.01641217083670199, -0.001882016600575298,
|
||||
|
|
@ -826,23 +878,89 @@ def test_BaseEstimator_regression():
|
|||
0.061315815430134535, 0.06272498145699501, 0.06460387445986271]}, 'categories': {},
|
||||
'untouched': ['s5', 's3', 'bp', 's1', 'sex', 's6', 's4']}
|
||||
|
||||
for key in expexted_generalizations['ranges']:
|
||||
assert (set(expexted_generalizations['ranges'][key]) == set(gener['ranges'][key]))
|
||||
for key in expexted_generalizations['categories']:
|
||||
assert (set([frozenset(sl) for sl in expexted_generalizations['categories'][key]]) ==
|
||||
for key in expected_generalizations['ranges']:
|
||||
assert (set(expected_generalizations['ranges'][key]) == set(gener['ranges'][key]))
|
||||
for key in expected_generalizations['categories']:
|
||||
assert (set([frozenset(sl) for sl in expected_generalizations['categories'][key]]) ==
|
||||
set([frozenset(sl) for sl in gener['categories'][key]]))
|
||||
assert (set(expexted_generalizations['untouched']) == set(gener['untouched']))
|
||||
assert (set(expected_generalizations['untouched']) == set(gener['untouched']))
|
||||
assert ((np.delete(transformed, [0, 2, 5, 8], axis=1) == np.delete(x_train, [0, 2, 5, 8], axis=1)).all())
|
||||
|
||||
modified_features = [f for f in features if
|
||||
f in expexted_generalizations['categories'].keys() or f in expexted_generalizations[
|
||||
f in expected_generalizations['categories'].keys() or f in expected_generalizations[
|
||||
'ranges'].keys()]
|
||||
indexes = []
|
||||
for i in range(len(features)):
|
||||
if features[i] in modified_features:
|
||||
indexes.append(i)
|
||||
assert ((np.delete(transformed, indexes, axis=1) == np.delete(x_train, indexes, axis=1)).all())
|
||||
ncp = gen.ncp_
|
||||
if len(expexted_generalizations['ranges'].keys()) > 0 or len(expexted_generalizations['categories'].keys()) > 0:
|
||||
ncp = gen.ncp
|
||||
if len(expected_generalizations['ranges'].keys()) > 0 or len(expected_generalizations['categories'].keys()) > 0:
|
||||
assert (ncp > 0)
|
||||
assert (((transformed[indexes]) != (x_train[indexes])).any())
|
||||
|
||||
rel_accuracy = model.score(transformed, predictions)
|
||||
assert ((rel_accuracy >= target_accuracy) or (target_accuracy - rel_accuracy) <= 0.05)
|
||||
|
||||
|
||||
def test_keras_model():
|
||||
(X, y), (x_test, y_test) = get_iris_dataset_np()
|
||||
|
||||
base_est = Sequential()
|
||||
base_est.add(Input(shape=(4,)))
|
||||
base_est.add(Dense(10, activation="relu"))
|
||||
base_est.add(Dense(3, activation='softmax'))
|
||||
|
||||
base_est.compile(loss="categorical_crossentropy", optimizer="adam", metrics=["accuracy"])
|
||||
|
||||
model = KerasClassifier(base_est, ModelOutputType.CLASSIFIER_PROBABILITIES)
|
||||
model.fit(ArrayDataset(X, y))
|
||||
ad = ArrayDataset(x_test)
|
||||
predictions = model.predict(ad)
|
||||
if predictions.shape[1] > 1:
|
||||
predictions = np.argmax(predictions, axis=1)
|
||||
target_accuracy = 0.5
|
||||
gen = GeneralizeToRepresentative(model, target_accuracy=target_accuracy)
|
||||
test_dataset = ArrayDataset(x_test, predictions)
|
||||
|
||||
gen.fit(dataset=test_dataset)
|
||||
transformed = gen.transform(dataset=ad)
|
||||
gener = gen.generalizations
|
||||
|
||||
features = ['0', '1', '2', '3']
|
||||
modified_features = [f for f in features if
|
||||
f in gener['categories'].keys() or f in gener['ranges'].keys()]
|
||||
indexes = []
|
||||
for i in range(len(features)):
|
||||
if features[i] in modified_features:
|
||||
indexes.append(i)
|
||||
assert ((np.delete(transformed, indexes, axis=1) == np.delete(x_test, indexes, axis=1)).all())
|
||||
ncp = gen.ncp
|
||||
if len(gener['ranges'].keys()) > 0 or len(gener['categories'].keys()) > 0:
|
||||
assert (ncp > 0)
|
||||
assert (((transformed[indexes]) != (X[indexes])).any())
|
||||
|
||||
rel_accuracy = model.score(ArrayDataset(transformed, predictions))
|
||||
assert ((rel_accuracy >= target_accuracy) or (target_accuracy - rel_accuracy) <= 0.05)
|
||||
|
||||
|
||||
def test_untouched():
|
||||
cells = [{"id": 1, "ranges": {"age": {"start": None, "end": 38}}, "label": 0,
|
||||
'categories': {'gender': ['male']}, "representative": {"age": 26, "height": 149}},
|
||||
{"id": 2, "ranges": {"age": {"start": 39, "end": None}}, "label": 1,
|
||||
'categories': {'gender': ['female']}, "representative": {"age": 58, "height": 163}},
|
||||
{"id": 3, "ranges": {"age": {"start": None, "end": 38}}, "label": 0,
|
||||
'categories': {'gender': ['male']}, "representative": {"age": 31, "height": 184}},
|
||||
{"id": 4, "ranges": {"age": {"start": 39, "end": None}}, "label": 1,
|
||||
'categories': {'gender': ['male', 'female']}, "representative": {"age": 45, "height": 176}}
|
||||
]
|
||||
gen = GeneralizeToRepresentative(cells=cells)
|
||||
gen._calculate_generalizations()
|
||||
gener = gen.generalizations
|
||||
expected_generalizations = {'ranges': {'age': [38, 39]}, 'categories': {}, 'untouched': ['gender']}
|
||||
for key in expected_generalizations['ranges']:
|
||||
assert (set(expected_generalizations['ranges'][key]) == set(gener['ranges'][key]))
|
||||
for key in expected_generalizations['categories']:
|
||||
assert (set([frozenset(sl) for sl in expected_generalizations['categories'][key]]) ==
|
||||
set([frozenset(sl) for sl in gener['categories'][key]]))
|
||||
assert (set(expected_generalizations['untouched']) == set(gener['untouched']))
|
||||
|
|
|
|||
|
|
@ -1,21 +1,32 @@
|
|||
import pytest
|
||||
import numpy as np
|
||||
|
||||
from apt.utils.models import SklearnClassifier, SklearnRegressor, ModelOutputType
|
||||
from apt.utils.datasets import ArrayDataset
|
||||
from apt.utils.models import SklearnClassifier, SklearnRegressor, ModelOutputType, KerasClassifier, KerasRegressor, \
|
||||
BlackboxClassifierPredictions, BlackboxClassifierPredictFunction, is_one_hot, get_nb_classes, XGBoostClassifier
|
||||
from apt.utils.datasets import ArrayDataset, Data, DatasetWithPredictions
|
||||
from apt.utils import dataset_utils
|
||||
|
||||
from sklearn.tree import DecisionTreeRegressor
|
||||
from sklearn.ensemble import RandomForestClassifier
|
||||
from xgboost import XGBClassifier
|
||||
|
||||
from tensorflow.keras.models import Sequential
|
||||
from tensorflow.keras.layers import Dense, Input
|
||||
|
||||
from art.utils import check_and_transform_label_format
|
||||
|
||||
|
||||
from art.utils import to_categorical
|
||||
|
||||
|
||||
def test_sklearn_classifier():
|
||||
(x_train, y_train), (x_test, y_test) = dataset_utils.get_iris_dataset()
|
||||
(x_train, y_train), (x_test, y_test) = dataset_utils.get_iris_dataset_np()
|
||||
underlying_model = RandomForestClassifier()
|
||||
model = SklearnClassifier(underlying_model, ModelOutputType.CLASSIFIER_VECTOR)
|
||||
model = SklearnClassifier(underlying_model, ModelOutputType.CLASSIFIER_PROBABILITIES)
|
||||
train = ArrayDataset(x_train, y_train)
|
||||
test = ArrayDataset(x_test, y_test)
|
||||
model.fit(train)
|
||||
pred = model.predict(x_test)
|
||||
pred = model.predict(test)
|
||||
assert(pred.shape[0] == x_test.shape[0])
|
||||
|
||||
score = model.score(test)
|
||||
|
|
@ -23,13 +34,296 @@ def test_sklearn_classifier():
|
|||
|
||||
|
||||
def test_sklearn_regressor():
|
||||
(x_train, y_train), (x_test, y_test) = dataset_utils.get_diabetes_dataset()
|
||||
(x_train, y_train), (x_test, y_test) = dataset_utils.get_diabetes_dataset_np()
|
||||
underlying_model = DecisionTreeRegressor()
|
||||
model = SklearnRegressor(underlying_model)
|
||||
train = ArrayDataset(x_train, y_train)
|
||||
test = ArrayDataset(x_test, y_test)
|
||||
model.fit(train)
|
||||
pred = model.predict(x_test)
|
||||
pred = model.predict(test)
|
||||
assert (pred.shape[0] == x_test.shape[0])
|
||||
|
||||
score = model.score(test)
|
||||
|
||||
|
||||
def test_keras_classifier():
|
||||
(x_train, y_train), (x_test, y_test) = dataset_utils.get_iris_dataset_np()
|
||||
|
||||
underlying_model = Sequential()
|
||||
underlying_model.add(Input(shape=(4,)))
|
||||
underlying_model.add(Dense(100, activation="relu"))
|
||||
underlying_model.add(Dense(10, activation="relu"))
|
||||
underlying_model.add(Dense(3, activation='softmax'))
|
||||
|
||||
underlying_model.compile(loss="categorical_crossentropy", optimizer="adam", metrics=["accuracy"])
|
||||
|
||||
model = KerasClassifier(underlying_model, ModelOutputType.CLASSIFIER_PROBABILITIES)
|
||||
|
||||
train = ArrayDataset(x_train, y_train)
|
||||
test = ArrayDataset(x_test, y_test)
|
||||
model.fit(train)
|
||||
pred = model.predict(test)
|
||||
assert(pred.shape[0] == x_test.shape[0])
|
||||
|
||||
score = model.score(test)
|
||||
assert(0.0 <= score <= 1.0)
|
||||
|
||||
|
||||
def test_keras_regressor():
|
||||
(x_train, y_train), (x_test, y_test) = dataset_utils.get_diabetes_dataset_np()
|
||||
|
||||
underlying_model = Sequential()
|
||||
underlying_model.add(Input(shape=(10,)))
|
||||
underlying_model.add(Dense(100, activation="relu"))
|
||||
underlying_model.add(Dense(10, activation="relu"))
|
||||
underlying_model.add(Dense(1))
|
||||
|
||||
underlying_model.compile(loss="mean_squared_error", optimizer="adam", metrics=["accuracy"])
|
||||
|
||||
model = KerasRegressor(underlying_model)
|
||||
|
||||
train = ArrayDataset(x_train, y_train)
|
||||
test = ArrayDataset(x_test, y_test)
|
||||
model.fit(train)
|
||||
pred = model.predict(test)
|
||||
assert (pred.shape[0] == x_test.shape[0])
|
||||
|
||||
score = model.score(test)
|
||||
|
||||
|
||||
def test_xgboost_classifier():
|
||||
(x_train, y_train), (x_test, y_test) = dataset_utils.get_iris_dataset_np()
|
||||
underlying_model = XGBClassifier()
|
||||
underlying_model.fit(x_train, y_train)
|
||||
model = XGBoostClassifier(underlying_model, ModelOutputType.CLASSIFIER_PROBABILITIES, input_shape=(4,), nb_classes=3)
|
||||
train = ArrayDataset(x_train, y_train)
|
||||
test = ArrayDataset(x_test, y_test)
|
||||
pred = model.predict(test)
|
||||
assert(pred.shape[0] == x_test.shape[0])
|
||||
|
||||
score = model.score(test)
|
||||
assert(0.0 <= score <= 1.0)
|
||||
|
||||
model.fit(train)
|
||||
|
||||
|
||||
def test_blackbox_classifier():
|
||||
(x_train, y_train), (x_test, y_test) = dataset_utils.get_iris_dataset_np()
|
||||
|
||||
train = ArrayDataset(x_train, y_train)
|
||||
test = ArrayDataset(x_test, y_test)
|
||||
data = Data(train, test)
|
||||
model = BlackboxClassifierPredictions(data, ModelOutputType.CLASSIFIER_SCALAR)
|
||||
pred = model.predict(test)
|
||||
assert(pred.shape[0] == x_test.shape[0])
|
||||
|
||||
score = model.score(test)
|
||||
assert(score == 1.0)
|
||||
|
||||
assert model.model_type is None
|
||||
|
||||
|
||||
def test_blackbox_classifier_predictions():
|
||||
(x_train, y_train), (x_test, y_test) = dataset_utils.get_iris_dataset_np()
|
||||
|
||||
train = DatasetWithPredictions(y_train, x_train)
|
||||
test = DatasetWithPredictions(y_test, x_test)
|
||||
data = Data(train, test)
|
||||
model = BlackboxClassifierPredictions(data, ModelOutputType.CLASSIFIER_SCALAR)
|
||||
pred = model.predict(test)
|
||||
assert(pred.shape[0] == x_test.shape[0])
|
||||
assert model.model_type is None
|
||||
|
||||
with pytest.raises(ValueError):
|
||||
model.score(test)
|
||||
|
||||
|
||||
def test_blackbox_classifier_predictions_y():
|
||||
(x_train, y_train), (x_test, y_test) = dataset_utils.get_iris_dataset_np()
|
||||
|
||||
train = DatasetWithPredictions(y_train, x_train, y_train)
|
||||
test = DatasetWithPredictions(y_test, x_test, y_test)
|
||||
data = Data(train, test)
|
||||
model = BlackboxClassifierPredictions(data, ModelOutputType.CLASSIFIER_SCALAR)
|
||||
pred = model.predict(test)
|
||||
assert(pred.shape[0] == x_test.shape[0])
|
||||
|
||||
score = model.score(test)
|
||||
assert(score == 1.0)
|
||||
|
||||
assert model.model_type is None
|
||||
|
||||
|
||||
def test_blackbox_classifier_mismatch():
|
||||
(x_train, y_train), (x_test, y_test) = dataset_utils.get_iris_dataset_np()
|
||||
|
||||
train = ArrayDataset(x_train, y_train)
|
||||
test = ArrayDataset(x_test, y_test)
|
||||
data = Data(train, test)
|
||||
with pytest.raises(ValueError):
|
||||
model = BlackboxClassifierPredictions(data, ModelOutputType.CLASSIFIER_PROBABILITIES)
|
||||
|
||||
|
||||
def test_blackbox_classifier_no_test():
|
||||
(x_train, y_train), (_, _) = dataset_utils.get_iris_dataset_np()
|
||||
|
||||
train = ArrayDataset(x_train, y_train)
|
||||
|
||||
data = Data(train)
|
||||
model = BlackboxClassifierPredictions(data, ModelOutputType.CLASSIFIER_SCALAR)
|
||||
pred = model.predict(train)
|
||||
assert(pred.shape[0] == x_train.shape[0])
|
||||
|
||||
score = model.score(train)
|
||||
assert (score == 1.0)
|
||||
|
||||
predictions_x, predictions_y = model.get_predictions()
|
||||
assert np.array_equal(predictions_x, x_train)
|
||||
assert np.array_equal(predictions_y, check_and_transform_label_format(y_train, nb_classes=3))
|
||||
|
||||
|
||||
def test_blackbox_classifier_no_train():
|
||||
(_, _), (x_test, y_test) = dataset_utils.get_iris_dataset_np()
|
||||
|
||||
test = ArrayDataset(x_test, y_test)
|
||||
data = Data(test=test)
|
||||
model = BlackboxClassifierPredictions(data, ModelOutputType.CLASSIFIER_SCALAR)
|
||||
pred = model.predict(test)
|
||||
assert(pred.shape[0] == x_test.shape[0])
|
||||
|
||||
score = model.score(test)
|
||||
assert (score == 1.0)
|
||||
|
||||
predictions_x, predictions_y = model.get_predictions()
|
||||
assert np.array_equal(predictions_x, x_test)
|
||||
assert np.array_equal(predictions_y, check_and_transform_label_format(y_test, nb_classes=3))
|
||||
|
||||
|
||||
def test_blackbox_classifier_no_test_y():
|
||||
(x_train, y_train), (x_test, _) = dataset_utils.get_iris_dataset_np()
|
||||
|
||||
train = ArrayDataset(x_train, y_train)
|
||||
test = ArrayDataset(x_test)
|
||||
data = Data(train, test)
|
||||
model = BlackboxClassifierPredictions(data, ModelOutputType.CLASSIFIER_SCALAR)
|
||||
pred = model.predict(train)
|
||||
assert(pred.shape[0] == x_train.shape[0])
|
||||
|
||||
score = model.score(train)
|
||||
assert (score == 1.0)
|
||||
|
||||
# since no test_y, BBC should use only test thus predict test should fail
|
||||
unable_to_predict_test = False
|
||||
try:
|
||||
model.predict(test)
|
||||
except BaseException:
|
||||
unable_to_predict_test = True
|
||||
|
||||
assert (unable_to_predict_test, True)
|
||||
|
||||
def test_blackbox_classifier_no_train_y():
|
||||
(x_train, _), (x_test, y_test) = dataset_utils.get_iris_dataset_np()
|
||||
|
||||
train = ArrayDataset(x_train)
|
||||
test = ArrayDataset(x_test, y_test)
|
||||
data = Data(train, test)
|
||||
model = BlackboxClassifierPredictions(data, ModelOutputType.CLASSIFIER_SCALAR)
|
||||
pred = model.predict(test)
|
||||
assert (pred.shape[0] == x_test.shape[0])
|
||||
|
||||
score = model.score(test)
|
||||
assert (score == 1.0)
|
||||
|
||||
# since no train_y, BBC should use only test thus predict train should fail
|
||||
unable_to_predict_train = False
|
||||
try:
|
||||
model.predict(train)
|
||||
except BaseException:
|
||||
unable_to_predict_train = True
|
||||
|
||||
assert(unable_to_predict_train,True)
|
||||
|
||||
def test_blackbox_classifier_probabilities():
|
||||
(x_train, _), (_, _) = dataset_utils.get_iris_dataset_np()
|
||||
y_train = np.array([[0.23, 0.56, 0.21] for i in range(105)])
|
||||
|
||||
train = ArrayDataset(x_train, y_train)
|
||||
|
||||
data = Data(train)
|
||||
model = BlackboxClassifierPredictions(data, ModelOutputType.CLASSIFIER_PROBABILITIES)
|
||||
pred = model.predict(train)
|
||||
assert (pred.shape[0] == x_train.shape[0])
|
||||
assert (0.0 < pred).all()
|
||||
assert (pred < 1.0).all()
|
||||
|
||||
score = model.score(train)
|
||||
assert (score == 1.0)
|
||||
|
||||
|
||||
def test_blackbox_classifier_predict():
|
||||
def predict(x):
|
||||
return np.array([[0.23, 0.56, 0.21] for i in range(x.shape[0])])
|
||||
|
||||
(x_train, y_train), (_, _) = dataset_utils.get_iris_dataset_np()
|
||||
y_train = np.array([[0.23, 0.56, 0.21] for i in range(105)])
|
||||
|
||||
train = ArrayDataset(x_train, y_train)
|
||||
|
||||
model = BlackboxClassifierPredictFunction(predict, ModelOutputType.CLASSIFIER_PROBABILITIES, (4,), 3)
|
||||
pred = model.predict(train)
|
||||
assert (pred.shape[0] == x_train.shape[0])
|
||||
assert (0.0 < pred).all()
|
||||
assert (pred < 1.0).all()
|
||||
|
||||
score = model.score(train)
|
||||
assert (score == 1.0)
|
||||
|
||||
|
||||
def test_blackbox_classifier_predict_scalar():
|
||||
def predict(x):
|
||||
return np.array([[1.0] for i in range(x.shape[0])])
|
||||
|
||||
(x_train, y_train), (_, _) = dataset_utils.get_iris_dataset_np()
|
||||
y_train = np.array([[0, 1, 0] for i in range(105)])
|
||||
|
||||
train = ArrayDataset(x_train, y_train)
|
||||
|
||||
model = BlackboxClassifierPredictFunction(predict, ModelOutputType.CLASSIFIER_SCALAR, (4,), 3)
|
||||
pred = model.predict(train)
|
||||
assert (pred.shape[0] == x_train.shape[0])
|
||||
|
||||
score = model.score(train)
|
||||
assert (score == 1.0)
|
||||
|
||||
|
||||
def test_is_one_hot():
|
||||
(_, y_train), (_, _) = dataset_utils.get_iris_dataset_np()
|
||||
|
||||
assert (not is_one_hot(y_train))
|
||||
assert (not is_one_hot(y_train.reshape(-1,1)))
|
||||
assert (is_one_hot(to_categorical(y_train)))
|
||||
|
||||
|
||||
def test_get_nb_classes():
|
||||
(_, y_train), (_, y_test) = dataset_utils.get_iris_dataset_np()
|
||||
|
||||
# shape: (x,) - not 1-hot
|
||||
nb_classes_test = get_nb_classes(y_test)
|
||||
nb_classes_train = get_nb_classes(y_train)
|
||||
assert (nb_classes_test == nb_classes_train)
|
||||
assert (nb_classes_test == 3)
|
||||
|
||||
# shape: (x,1) - not 1-hot
|
||||
nb_classes_test = get_nb_classes(y_test.reshape(-1,1))
|
||||
assert (nb_classes_test == 3)
|
||||
|
||||
# shape: (x,3) - 1-hot
|
||||
y = to_categorical(y_test)
|
||||
nb_classes = get_nb_classes(y)
|
||||
assert (nb_classes == 3)
|
||||
|
||||
# gaps: 1,2,4 (0,3 missing)
|
||||
y_test[y_test == 0] = 4
|
||||
nb_classes = get_nb_classes(y_test)
|
||||
assert (nb_classes == 5)
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue