From fd9f1343367b51800d5fca58c140e4a3ed647645 Mon Sep 17 00:00:00 2001 From: olasaadi Date: Tue, 1 Mar 2022 02:28:41 +0200 Subject: [PATCH 1/3] using dataset wrapper on anonymizer --- apt/anonymization/anonymizer.py | 10 ++--- apt/utils/dataset_utils.py | 31 +++++++++++---- tests/test_anonymizer.py | 70 ++++++++++++++++----------------- 3 files changed, 64 insertions(+), 47 deletions(-) diff --git a/apt/anonymization/anonymizer.py b/apt/anonymization/anonymizer.py index d7b1c88..a093402 100644 --- a/apt/anonymization/anonymizer.py +++ b/apt/anonymization/anonymizer.py @@ -5,6 +5,7 @@ from collections import Counter from sklearn.tree import DecisionTreeClassifier, DecisionTreeRegressor from sklearn.preprocessing import OneHotEncoder +from apt.utils.datasets import BaseDataset, Data from typing import Union, Optional @@ -37,8 +38,7 @@ class Anonymize: self.categorical_features = categorical_features self.is_regression = is_regression - def anonymize(self, x: Union[np.ndarray, pd.DataFrame], y: Union[np.ndarray, pd.DataFrame]) \ - -> Union[np.ndarray, pd.DataFrame]: + def anonymize(self, dataset: BaseDataset) -> Union[np.ndarray, pd.DataFrame]: """ Method for performing model-guided anonymization. @@ -47,12 +47,12 @@ class Anonymize: :param y: The predictions of the original model on the training data. :return: An array containing the anonymized training dataset. """ - if type(x) == np.ndarray: - return self._anonymize_ndarray(x.copy(), y) + if type(dataset.x) == np.ndarray: + return self._anonymize_ndarray(dataset.x.copy(), dataset.y) else: # pandas if not self.categorical_features: raise ValueError('When supplying a pandas dataframe, categorical_features must be defined') - return self._anonymize_pandas(x.copy(), y) + return self._anonymize_pandas(dataset.x.copy(), dataset.y) def _anonymize_ndarray(self, x, y): if x.shape[0] != y.shape[0]: diff --git a/apt/utils/dataset_utils.py b/apt/utils/dataset_utils.py index f99c6cc..3bafa7b 100644 --- a/apt/utils/dataset_utils.py +++ b/apt/utils/dataset_utils.py @@ -5,6 +5,8 @@ import ssl from os import path, mkdir from six.moves.urllib.request import urlretrieve +from apt.utils.datasets import BaseDataset, Data + def _load_iris(test_set_size: float = 0.3): iris = datasets.load_iris() @@ -14,8 +16,10 @@ def _load_iris(test_set_size: float = 0.3): # Split training and test sets x_train, x_test, y_train, y_test = model_selection.train_test_split(data, labels, test_size=test_set_size, random_state=18, stratify=labels) - - return (x_train, y_train), (x_test, y_test) + train_dataset = BaseDataset(x_train, y_train) + test_dataset = BaseDataset(x_test, y_test) + dataset = Data(train_dataset, test_dataset) + return dataset def get_iris_dataset(): @@ -37,7 +41,10 @@ def _load_diabetes(test_set_size: float = 0.3): x_train, x_test, y_train, y_test = model_selection.train_test_split(data, labels, test_size=test_set_size, random_state=18) - return (x_train, y_train), (x_test, y_test) + train_dataset = BaseDataset(x_train, y_train) + test_dataset = BaseDataset(x_test, y_test) + dataset = Data(train_dataset, test_dataset) + return dataset def get_diabetes_dataset(): @@ -97,7 +104,10 @@ def get_german_credit_dataset(test_set: float = 0.3): x_test.reset_index(drop=True, inplace=True) y_test.reset_index(drop=True, inplace=True) - return (x_train, y_train), (x_test, y_test) + train_dataset = BaseDataset(x_train, y_train) + test_dataset = BaseDataset(x_test, y_test) + dataset = Data(train_dataset, test_dataset) + return dataset def _modify_german_dataset(data): @@ -156,8 +166,10 @@ def get_adult_dataset(): y_train = train.loc[:, 'label'] x_test = test.drop(['label'], axis=1) y_test = test.loc[:, 'label'] - - return (x_train, y_train), (x_test, y_test) + train_dataset = BaseDataset(x_train, y_train) + test_dataset = BaseDataset(x_test, y_test) + dataset = Data(train_dataset, test_dataset) + return dataset def _modify_adult_dataset(data): @@ -315,5 +327,10 @@ def get_nursery_dataset(raw: bool = True, test_set: float = 0.2, transform_socia y_train = train.loc[:, "label"] x_test = test.drop(["label"], axis=1) y_test = test.loc[:, "label"] + x_train = x_train.astype(str) + x_test = x_test.astype(str) - return (x_train, y_train), (x_test, y_test) + train_dataset = BaseDataset(x_train, y_train) + test_dataset = BaseDataset(x_test, y_test) + dataset = Data(train_dataset, test_dataset) + return dataset diff --git a/tests/test_anonymizer.py b/tests/test_anonymizer.py index 4a96b9a..280309e 100644 --- a/tests/test_anonymizer.py +++ b/tests/test_anonymizer.py @@ -7,29 +7,29 @@ from apt.anonymization import Anonymize from apt.utils.dataset_utils import get_iris_dataset, get_adult_dataset, get_nursery_dataset from sklearn.datasets import load_diabetes from sklearn.model_selection import train_test_split - +from apt.utils.datasets import BaseDataset, Data def test_anonymize_ndarray_iris(): - (x_train, y_train), _ = get_iris_dataset() + dataset = get_iris_dataset() model = DecisionTreeClassifier() - model.fit(x_train, y_train) - pred = model.predict(x_train) + model.fit(dataset.get_train_samples(), dataset.get_train_labels()) + pred = model.predict(dataset.get_train_samples()) k = 10 QI = [0, 2] anonymizer = Anonymize(k, QI) - anon = anonymizer.anonymize(x_train, pred) - assert(len(np.unique(anon[:, QI], axis=0)) < len(np.unique(x_train[:, QI], axis=0))) + anon = anonymizer.anonymize(BaseDataset(dataset.get_train_samples(), pred)) + assert(len(np.unique(anon[:, QI], axis=0)) < len(np.unique(dataset.get_train_samples()[:, QI], axis=0))) _, counts_elements = np.unique(anon[:, QI], return_counts=True) assert (np.min(counts_elements) >= k) - assert ((np.delete(anon, QI, axis=1) == np.delete(x_train, QI, axis=1)).all()) + assert ((np.delete(anon, QI, axis=1) == np.delete(dataset.get_train_samples(), QI, axis=1)).all()) def test_anonymize_pandas_adult(): - (x_train, y_train), _ = get_adult_dataset() - encoded = OneHotEncoder().fit_transform(x_train) + dataset = get_adult_dataset() + encoded = OneHotEncoder().fit_transform(dataset.get_train_samples()) model = DecisionTreeClassifier() - model.fit(encoded, y_train) + model.fit(encoded, dataset.get_train_labels()) pred = model.predict(encoded) k = 100 @@ -38,51 +38,51 @@ def test_anonymize_pandas_adult(): categorical_features = ['workclass', 'marital-status', 'occupation', 'relationship', 'race', 'sex', 'native-country'] anonymizer = Anonymize(k, QI, categorical_features=categorical_features) - anon = anonymizer.anonymize(x_train, pred) + anon = anonymizer.anonymize(BaseDataset(dataset.get_train_samples(), pred)) - assert(anon.loc[:, QI].drop_duplicates().shape[0] < x_train.loc[:, QI].drop_duplicates().shape[0]) + assert(anon.loc[:, QI].drop_duplicates().shape[0] < dataset.get_train_samples().loc[:, QI].drop_duplicates().shape[0]) assert (anon.loc[:, QI].value_counts().min() >= k) - assert (anon.drop(QI, axis=1).equals(x_train.drop(QI, axis=1))) + assert (anon.drop(QI, axis=1).equals(dataset.get_train_samples().drop(QI, axis=1))) def test_anonymize_pandas_nursery(): - (x_train, y_train), _ = get_nursery_dataset() - x_train = x_train.astype(str) - encoded = OneHotEncoder().fit_transform(x_train) + dataset = get_nursery_dataset() + encoded = OneHotEncoder().fit_transform(dataset.get_train_samples()) model = DecisionTreeClassifier() - model.fit(encoded, y_train) + model.fit(encoded, dataset.get_train_labels()) pred = model.predict(encoded) k = 100 QI = ["finance", "social", "health"] categorical_features = ["parents", "has_nurs", "form", "housing", "finance", "social", "health", 'children'] anonymizer = Anonymize(k, QI, categorical_features=categorical_features) - anon = anonymizer.anonymize(x_train, pred) + anon = anonymizer.anonymize(BaseDataset(dataset.get_train_samples(), pred)) - assert(anon.loc[:, QI].drop_duplicates().shape[0] < x_train.loc[:, QI].drop_duplicates().shape[0]) + assert(anon.loc[:, QI].drop_duplicates().shape[0] < dataset.get_train_samples().loc[:, QI].drop_duplicates().shape[0]) assert (anon.loc[:, QI].value_counts().min() >= k) - assert (anon.drop(QI, axis=1).equals(x_train.drop(QI, axis=1))) + assert (anon.drop(QI, axis=1).equals(dataset.get_train_samples().drop(QI, axis=1))) def test_regression(): - dataset = load_diabetes() - x_train, x_test, y_train, y_test = train_test_split(dataset.data, dataset.target, test_size=0.5, random_state=14) - + x_train, x_test, y_train, y_test = train_test_split(load_diabetes().data, load_diabetes().target, test_size=0.5, random_state=14) + train_dataset = BaseDataset(x_train, y_train) + test_dataset = BaseDataset(x_test, y_test) + dataset = Data(train_dataset, test_dataset) model = DecisionTreeRegressor(random_state=10, min_samples_split=2) - model.fit(x_train, y_train) - pred = model.predict(x_train) + model.fit(dataset.get_train_samples(), dataset.get_train_labels()) + pred = model.predict(dataset.get_train_samples()) k = 10 QI = [0, 2, 5, 8] anonymizer = Anonymize(k, QI, is_regression=True) - anon = anonymizer.anonymize(x_train, pred) - print('Base model accuracy (R2 score): ', model.score(x_test, y_test)) - model.fit(anon, y_train) - print('Base model accuracy (R2 score) after anonymization: ', model.score(x_test, y_test)) - assert(len(np.unique(anon[:, QI], axis=0)) < len(np.unique(x_train[:, QI], axis=0))) + anon = anonymizer.anonymize(BaseDataset(dataset.get_train_samples(), pred)) + print('Base model accuracy (R2 score): ', model.score(dataset.get_test_samples(), dataset.get_test_labels())) + model.fit(anon, dataset.get_train_labels()) + print('Base model accuracy (R2 score) after anonymization: ', model.score(dataset.get_test_samples(), dataset.get_test_labels())) + assert(len(np.unique(anon[:, QI], axis=0)) < len(np.unique(dataset.get_train_samples()[:, QI], axis=0))) _, counts_elements = np.unique(anon[:, QI], return_counts=True) assert (np.min(counts_elements) >= k) - assert ((np.delete(anon, QI, axis=1) == np.delete(x_train, QI, axis=1)).all()) + assert ((np.delete(anon, QI, axis=1) == np.delete(dataset.get_train_samples(), QI, axis=1)).all()) def test_errors(): @@ -93,9 +93,9 @@ def test_errors(): with pytest.raises(ValueError): Anonymize(2, None) anonymizer = Anonymize(10, [0, 2]) - (x_train, y_train), (x_test, y_test) = get_iris_dataset() + dataset = get_iris_dataset() with pytest.raises(ValueError): - anonymizer.anonymize(x_train, y_test) - (x_train, y_train), _ = get_adult_dataset() + anonymizer.anonymize(BaseDataset(dataset.get_train_samples(), dataset.get_test_labels())) + dataset = get_adult_dataset() with pytest.raises(ValueError): - anonymizer.anonymize(x_train, y_train) + anonymizer.anonymize(BaseDataset(dataset.get_train_samples(), dataset.get_train_labels())) From 300e391432476692f0b8cef465762b8186b60662 Mon Sep 17 00:00:00 2001 From: olasaadi Date: Thu, 10 Mar 2022 12:56:41 +0200 Subject: [PATCH 2/3] fix bug and update test_model --- apt/anonymization/anonymizer.py | 3 +++ apt/utils/models/sklearn_model.py | 5 ++++- tests/test_model.py | 24 ++++++++++++------------ 3 files changed, 19 insertions(+), 13 deletions(-) diff --git a/apt/anonymization/anonymizer.py b/apt/anonymization/anonymizer.py index a093402..0f90a2d 100644 --- a/apt/anonymization/anonymizer.py +++ b/apt/anonymization/anonymizer.py @@ -66,6 +66,7 @@ class Anonymize: self.anonymizer = DecisionTreeRegressor(random_state=10, min_samples_split=2, min_samples_leaf=self.k) else: self.anonymizer = DecisionTreeClassifier(random_state=10, min_samples_split=2, min_samples_leaf=self.k) + self.anonymizer.fit(x_prepared, y) cells_by_id = self._calculate_cells(x, x_prepared) return self._anonymize_data_numpy(x, x_prepared, cells_by_id) @@ -80,6 +81,8 @@ class Anonymize: self.anonymizer = DecisionTreeRegressor(random_state=10, min_samples_split=2, min_samples_leaf=self.k) else: self.anonymizer = DecisionTreeClassifier(random_state=10, min_samples_split=2, min_samples_leaf=self.k) + if len(y.shape) > 1: + y = np.argmax(y, axis=1) self.anonymizer.fit(x_prepared, y) cells_by_id = self._calculate_cells(x, x_prepared) return self._anonymize_data_pandas(x, x_prepared, cells_by_id) diff --git a/apt/utils/models/sklearn_model.py b/apt/utils/models/sklearn_model.py index dffa356..238a49f 100644 --- a/apt/utils/models/sklearn_model.py +++ b/apt/utils/models/sklearn_model.py @@ -46,7 +46,10 @@ class SklearnClassifier(SklearnModel): :type y: `np.ndarray` or `pandas.DataFrame` """ encoder = OneHotEncoder(sparse=False) - y_encoded = encoder.fit_transform(y.reshape(-1, 1)) + if type(y) == np.ndarray: + y_encoded = encoder.fit_transform(y.reshape(-1, 1)) + else: + y_encoded = encoder.fit_transform(y.values.reshape(-1, 1)) self._art_model.fit(x, y_encoded, **kwargs) def predict(self, x: np.ndarray, **kwargs) -> np.ndarray: diff --git a/tests/test_model.py b/tests/test_model.py index d1dc6eb..f34f835 100644 --- a/tests/test_model.py +++ b/tests/test_model.py @@ -7,28 +7,28 @@ from sklearn.tree import DecisionTreeRegressor from sklearn.ensemble import RandomForestClassifier def test_sklearn_classifier(): - (x_train, y_train), (x_test, y_test) = dataset_utils.get_iris_dataset() + dataset = dataset_utils.get_iris_dataset() underlying_model = RandomForestClassifier() model = SklearnClassifier(underlying_model) - model.fit(x_train, y_train) - pred = model.predict(x_test) - assert(pred.shape[0] == x_test.shape[0]) + model.fit(dataset.get_train_samples(), dataset.get_train_labels()) + pred = model.predict(dataset.get_test_samples()) + assert(pred.shape[0] == dataset.get_test_samples().shape[0]) - score = model.score(x_test, y_test) + score = model.score(dataset.get_test_samples(), dataset.get_test_labels()) assert(0.0 <= score <= 1.0) def test_sklearn_regressor(): - (x_train, y_train), (x_test, y_test) = dataset_utils.get_diabetes_dataset() + dataset = dataset_utils.get_diabetes_dataset() underlying_model = DecisionTreeRegressor() model = SklearnRegressor(underlying_model) - model.fit(x_train, y_train) - pred = model.predict(x_test) - assert (pred.shape[0] == x_test.shape[0]) + model.fit(dataset.get_train_samples(), dataset.get_train_labels()) + pred = model.predict(dataset.get_test_samples()) + assert (pred.shape[0] == dataset.get_test_samples().shape[0]) - score = model.score(x_test, y_test) + score = model.score(dataset.get_test_samples(), dataset.get_test_labels()) - losses = model.loss(x_test, y_test) - assert (losses.shape[0] == x_test.shape[0]) + losses = model.loss(dataset.get_test_samples(), dataset.get_test_labels()) + assert (losses.shape[0] == dataset.get_test_samples().shape[0]) # Probably not needed for now, as we will not be using these wrappers directly in ART. From c0dbb200e77e6528915fefdd94f2c9d941c4dd47 Mon Sep 17 00:00:00 2001 From: olasaadi Date: Thu, 10 Mar 2022 13:04:39 +0200 Subject: [PATCH 3/3] add dataset wrapper to docstring --- apt/anonymization/anonymizer.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/apt/anonymization/anonymizer.py b/apt/anonymization/anonymizer.py index 0f90a2d..d7452b1 100644 --- a/apt/anonymization/anonymizer.py +++ b/apt/anonymization/anonymizer.py @@ -42,9 +42,9 @@ class Anonymize: """ Method for performing model-guided anonymization. - :param x: The training data for the model. If provided as a pandas dataframe, may contain both numeric and - categorical data. - :param y: The predictions of the original model on the training data. + :param dataset: Data wrapper Containing The training data for the model and ehe predictions of the + original model on the training data. If provided as a pandas + dataframe, may contain both numeric and categorical data. :return: An array containing the anonymized training dataset. """ if type(dataset.x) == np.ndarray: