diff --git a/apt/anonymization/anonymizer.py b/apt/anonymization/anonymizer.py
index 9f82c7c..02854f5 100644
--- a/apt/anonymization/anonymizer.py
+++ b/apt/anonymization/anonymizer.py
@@ -8,6 +8,7 @@ from sklearn.impute import SimpleImputer
from sklearn.pipeline import Pipeline
from sklearn.tree import DecisionTreeClassifier, DecisionTreeRegressor
from sklearn.preprocessing import OneHotEncoder
+from apt.utils.datasets import ArrayDataset, DATA_PANDAS_NUMPY_TYPE
from typing import Union, Optional
@@ -49,61 +50,64 @@ class Anonymize:
self.categorical_features = categorical_features
self.is_regression = is_regression
self.train_only_QI = train_only_QI
+ self.features_names = None
+ self.features = None
- def anonymize(self, x: Union[np.ndarray, pd.DataFrame], y: Union[np.ndarray, pd.DataFrame]) \
- -> Union[np.ndarray, pd.DataFrame]:
+ def anonymize(self, dataset: ArrayDataset) -> DATA_PANDAS_NUMPY_TYPE:
"""
Method for performing model-guided anonymization.
- :param x: The training data for the model. If provided as a pandas dataframe, may contain both numeric and
- categorical data.
- :param y: The predictions of the original model on the training data.
+ :param dataset: Data wrapper containing the training data for the model and the predictions of the
+ original model on the training data.
:return: An array containing the anonymized training dataset.
"""
- if type(x) == np.ndarray:
- self.features = [i for i in range(x.shape[1])]
- return self._anonymize_ndarray(x.copy(), y)
- else: # pandas
- self.features = x.columns
- if not self.categorical_features:
- raise ValueError('When supplying a pandas dataframe, categorical_features must be defined')
- return self._anonymize_pandas(x.copy(), y)
+ if dataset.get_samples().shape[1] != 0:
+ self.features = [i for i in range(dataset.get_samples().shape[1])]
+ else:
+ raise ValueError('No data provided')
- def _anonymize_ndarray(self, x, y):
+ if dataset.features_names is not None:
+ self.features_names = dataset.features_names
+ else: # if no names provided, use numbers instead
+ self.features_names = self.features
+
+ if not set(self.quasi_identifiers).issubset(set(self.features_names)):
+ raise ValueError('Quasi identifiers should bs a subset of the supplied features or indexes in range of '
+ 'the data columns')
+ if self.categorical_features and not set(self.categorical_features).issubset(set(self.features_names)):
+ raise ValueError('Categorical features should bs a subset of the supplied features or indexes in range of '
+ 'the data columns')
+ self.quasi_identifiers = [i for i, v in enumerate(self.features_names) if v in self.quasi_identifiers]
+ if self.categorical_features:
+ self.categorical_features = [i for i, v in enumerate(self.features_names) if v in self.categorical_features]
+
+ transformed = self._anonymize(dataset.get_samples().copy(), dataset.get_labels())
+ if dataset.is_pandas:
+ return pd.DataFrame(transformed, columns=self.features_names)
+ else:
+ return transformed
+
+ def _anonymize(self, x, y):
if x.shape[0] != y.shape[0]:
raise ValueError("x and y should have same number of rows")
- x_anonymizer_train = x
- if self.train_only_QI:
- # build DT just on QI features
- x_anonymizer_train = x[:, self.quasi_identifiers]
if x.dtype.kind not in 'iufc':
- x_prepared = self._modify_categorical_features(x_anonymizer_train)
+ if not self.categorical_features:
+ raise ValueError('when supplying an array with non-numeric data, categorical_features must be defined')
+ x_prepared = self._modify_categorical_features(x)
else:
- x_prepared = x_anonymizer_train
- if self.is_regression:
- self.anonymizer = DecisionTreeRegressor(random_state=10, min_samples_split=2, min_samples_leaf=self.k)
- else:
- self.anonymizer = DecisionTreeClassifier(random_state=10, min_samples_split=2, min_samples_leaf=self.k)
- self.anonymizer.fit(x_prepared, y)
- cells_by_id = self._calculate_cells(x, x_prepared)
- return self._anonymize_data_numpy(x, x_prepared, cells_by_id)
-
- def _anonymize_pandas(self, x, y):
- if x.shape[0] != y.shape[0]:
- raise ValueError("x and y should have same number of rows")
- x_anonymizer_train = x
+ x_prepared = x
+ x_anonymizer_train = x_prepared
if self.train_only_QI:
# build DT just on QI features
- x_anonymizer_train = x.loc[:, self.quasi_identifiers]
- # need to one-hot encode before training the decision tree
- x_prepared = self._modify_categorical_features(x_anonymizer_train)
+ x_anonymizer_train = x_prepared[:, self.quasi_identifiers]
if self.is_regression:
self.anonymizer = DecisionTreeRegressor(random_state=10, min_samples_split=2, min_samples_leaf=self.k)
else:
self.anonymizer = DecisionTreeClassifier(random_state=10, min_samples_split=2, min_samples_leaf=self.k)
- self.anonymizer.fit(x_prepared, y)
- cells_by_id = self._calculate_cells(x, x_prepared)
- return self._anonymize_data_pandas(x, x_prepared, cells_by_id)
+
+ self.anonymizer.fit(x_anonymizer_train, y)
+ cells_by_id = self._calculate_cells(x, x_anonymizer_train)
+ return self._anonymize_data(x, x_anonymizer_train, cells_by_id)
def _calculate_cells(self, x, x_anonymizer_train):
# x is original data, x_anonymizer_train is only QIs + 1-hot encoded
@@ -130,15 +134,9 @@ class Anonymize:
# get all rows in cell
indexes = [index for index, node_id in enumerate(node_ids) if node_id == cell['id']]
# TODO: should we filter only those with majority label? (using hist)
- if type(x) == np.ndarray:
- rows = x[indexes]
- else: # pandas
- rows = x.iloc[indexes]
+ rows = x[indexes]
for feature in self.quasi_identifiers:
- if type(x) == np.ndarray:
- values = rows[:, feature]
- else: # pandas
- values = rows.loc[:, feature]
+ values = rows[:, feature]
if self.categorical_features and feature in self.categorical_features:
# find most common value
cell['representative'][feature] = Counter(values).most_common(1)[0][0]
@@ -163,7 +161,7 @@ class Anonymize:
node_ids = self._find_sample_nodes(samples)
return [cells_by_id[node_id] for node_id in node_ids]
- def _anonymize_data_numpy(self, x, x_anonymizer_train, cells_by_id):
+ def _anonymize_data(self, x, x_anonymizer_train, cells_by_id):
cells = self._find_sample_cells(x_anonymizer_train, cells_by_id)
index = 0
for row in x:
@@ -173,22 +171,12 @@ class Anonymize:
row[feature] = cell['representative'][feature]
return x
- def _anonymize_data_pandas(self, x, x_anonymizer_train, cells_by_id):
- cells = self._find_sample_cells(x_anonymizer_train, cells_by_id)
- index = 0
- for i, row in x.iterrows():
- cell = cells[index]
- index += 1
- for feature in cell['representative']:
- x.at[i, feature] = cell['representative'][feature]
- return x
-
def _modify_categorical_features(self, x):
# prepare data for DT
used_features = self.features
if self.train_only_QI:
used_features = self.quasi_identifiers
- numeric_features = [f for f in x.columns if f in used_features and f not in self.categorical_features]
+ numeric_features = [f for f in self.features if f in used_features and f not in self.categorical_features]
categorical_features = [f for f in self.categorical_features if f in used_features]
numeric_transformer = Pipeline(
steps=[('imputer', SimpleImputer(strategy='constant', fill_value=0))]
diff --git a/apt/minimization/minimizer.py b/apt/minimization/minimizer.py
index d04cc03..27b6b6e 100644
--- a/apt/minimization/minimizer.py
+++ b/apt/minimization/minimizer.py
@@ -1,7 +1,7 @@
"""
This module implements all classes needed to perform data minimization
"""
-from typing import Union
+from typing import Union, Optional
import pandas as pd
import numpy as np
import copy
@@ -16,6 +16,9 @@ from sklearn.utils.validation import check_X_y, check_array, check_is_fitted
from sklearn.tree import DecisionTreeClassifier, DecisionTreeRegressor
from sklearn.model_selection import train_test_split
+from apt.utils.datasets import ArrayDataset, Data, DATA_PANDAS_NUMPY_TYPE
+from apt.utils.models import Model, SklearnRegressor, ModelOutputType, SklearnClassifier
+
class GeneralizeToRepresentative(BaseEstimator, MetaEstimatorMixin, TransformerMixin):
""" A transformer that generalizes data to representative points.
@@ -24,16 +27,13 @@ class GeneralizeToRepresentative(BaseEstimator, MetaEstimatorMixin, TransformerM
and a target accuracy. Once the generalizations are learned, can
receive one or more data records and transform them to representative
points based on the learned generalization.
-
- An alternative way to use the transformer is to supply ``cells`` and
- ``features`` in init or set_params and those will be used to transform
+ An alternative way to use the transformer is to supply ``cells`` in
+ init or set_params and those will be used to transform
data to representatives. In this case, fit must still be called but
there is no need to supply it with ``X`` and ``y``, and there is no
need to supply an existing ``estimator`` to init.
-
In summary, either ``estimator`` and ``target_accuracy`` should be
- supplied or ``cells`` and ``features`` should be supplied.
-
+ supplied or ``cells`` should be supplied.
Parameters
----------
estimator : estimator, optional
@@ -43,8 +43,6 @@ class GeneralizeToRepresentative(BaseEstimator, MetaEstimatorMixin, TransformerM
The required accuracy when applying the base model to the
generalized data. Accuracy is measured relative to the original
accuracy of the model.
- features : list of str, optional
- The feature names, in the order that they appear in the data.
categorical_features: list of str, optional
The list of categorical features should only be supplied when
passing data as a pandas dataframe.
@@ -67,28 +65,29 @@ class GeneralizeToRepresentative(BaseEstimator, MetaEstimatorMixin, TransformerM
Attributes
----------
+ features_ : list of str
+ The feature names, in the order that they appear in the data.
cells_ : list of object
The cells used to generalize records, as learned when calling fit.
-
ncp_ : float
The NCP (information loss) score of the resulting generalization,
as measured on the training data.
-
generalizations_ : object
The generalizations that were learned (actual feature ranges).
-
- Notes
- -----
-
-
"""
- def __init__(self, estimator=None, target_accuracy=0.998, features=None,
- cells=None, categorical_features=None, features_to_minimize: Union[np.ndarray, list] = None
- , train_only_QI=True, is_regression=False):
- self.estimator = estimator
+ def __init__(self, estimator: Union[BaseEstimator, Model] = None, target_accuracy: float = 0.998,
+ cells: list = None, categorical_features: Union[np.ndarray, list] = None,
+ features_to_minimize: Union[np.ndarray, list] = None, train_only_QI: bool = True,
+ is_regression: bool = False):
+ if issubclass(estimator.__class__, Model):
+ self.estimator = estimator
+ else:
+ if is_regression:
+ self.estimator = SklearnRegressor(estimator)
+ else:
+ self.estimator = SklearnClassifier(estimator, ModelOutputType.CLASSIFIER_VECTOR)
self.target_accuracy = target_accuracy
- self.features = features
self.cells = cells
self.categorical_features = []
if categorical_features:
@@ -114,11 +113,9 @@ class GeneralizeToRepresentative(BaseEstimator, MetaEstimatorMixin, TransformerM
ret = {}
ret['target_accuracy'] = self.target_accuracy
if deep:
- ret['features'] = copy.deepcopy(self.features)
ret['cells'] = copy.deepcopy(self.cells)
ret['estimator'] = self.estimator
else:
- ret['features'] = copy.copy(self.features)
ret['cells'] = copy.copy(self.cells)
return ret
@@ -132,8 +129,6 @@ class GeneralizeToRepresentative(BaseEstimator, MetaEstimatorMixin, TransformerM
"""
if 'target_accuracy' in params:
self.target_accuracy = params['target_accuracy']
- if 'features' in params:
- self.features = params['features']
if 'cells' in params:
self.cells = params['cells']
return self
@@ -142,7 +137,8 @@ class GeneralizeToRepresentative(BaseEstimator, MetaEstimatorMixin, TransformerM
def generalizations(self):
return self.generalizations_
- def fit_transform(self, X: Union[np.ndarray, pd.DataFrame] = None, y: Union[np.ndarray, pd.DataFrame] = None):
+ def fit_transform(self, X: Optional[DATA_PANDAS_NUMPY_TYPE] = None, y: Optional[DATA_PANDAS_NUMPY_TYPE] = None,
+ features_names: Optional = None, dataset: Optional[ArrayDataset] = None):
"""Learns the generalizations based on training data, and applies them to the data.
Parameters
@@ -152,17 +148,22 @@ class GeneralizeToRepresentative(BaseEstimator, MetaEstimatorMixin, TransformerM
y : array-like, shape (n_samples,), optional
The target values. An array of int.
This should contain the predictions of the original model on ``X``.
-
+ features_names : list of str, The feature names, in the order that they appear in the data,
+ provided just if X and y were provided (optional).
+ dataset : Data wrapper containing the training input samples and the predictions of the
+ original model on the training data.
+ Either X,y OR dataset need to be provided, not both.
Returns
-------
X_transformed : numpy or pandas according to the input type, shape (n_samples, n_features)
The array containing the representative values to which each record in
``X`` is mapped.
"""
- self.fit(X, y)
- return self.transform(X)
+ self.fit(X, y, features_names, dataset=dataset)
+ return self.transform(X, features_names, dataset=dataset)
- def fit(self, X: Union[np.ndarray, pd.DataFrame] = None, y: Union[np.ndarray, pd.DataFrame] = None):
+ def fit(self, X: Optional[DATA_PANDAS_NUMPY_TYPE] = None, y: Optional[DATA_PANDAS_NUMPY_TYPE] = None,
+ features_names: Optional = None, dataset: ArrayDataset = None):
"""Learns the generalizations based on training data.
Parameters
@@ -172,7 +173,11 @@ class GeneralizeToRepresentative(BaseEstimator, MetaEstimatorMixin, TransformerM
y : array-like, shape (n_samples,), optional
The target values. An array of int.
This should contain the predictions of the original model on ``X``.
-
+ features_names : list of str, The feature names, in the order that they appear in the data,
+ provided just if X and y were provided (optional).
+ dataset : Data wrapper containing the training input samples and the predictions of the
+ original model on the training data.
+ Either X,y OR dataset need to be provided, not both.
Returns
-------
X_transformed : numpy or pandas according to the input type, shape (n_samples, n_features)
@@ -181,26 +186,25 @@ class GeneralizeToRepresentative(BaseEstimator, MetaEstimatorMixin, TransformerM
"""
# take into account that estimator, X, y, cells, features may be None
- if X is not None:
- if type(X) == np.ndarray:
- self.is_numpy = True
- else:
- self.is_numpy = False
-
if X is not None and y is not None:
- if self.is_numpy:
- X, y = check_X_y(X, y, accept_sparse=True)
- self.n_features_ = X.shape[1]
- elif self.features:
- self.n_features_ = len(self.features)
+ if dataset is not None:
+ raise ValueError('Either X,y OR dataset need to be provided, not both')
+ else:
+ dataset = ArrayDataset(X, y, features_names)
+
+ if dataset and dataset.get_samples() is not None and dataset.get_labels() is not None:
+ self.n_features_ = dataset.get_samples().shape[1]
+
+ elif dataset and dataset.features_names:
+ self.n_features_ = len(dataset.features_names)
else:
self.n_features_ = 0
- if self.features:
- self._features = self.features
+ if dataset and dataset.features_names:
+ self._features = dataset.features_names
# if features is None, use numbers instead of names
elif self.n_features_ != 0:
- self._features = [i for i in range(self.n_features_)]
+ self._features = [str(i) for i in range(self.n_features_)]
else:
self._features = None
@@ -212,27 +216,24 @@ class GeneralizeToRepresentative(BaseEstimator, MetaEstimatorMixin, TransformerM
# Going to fit
# (currently not dealing with option to fit with only X and y and no estimator)
- if self.estimator and X is not None and y is not None:
+ if self.estimator and dataset and dataset.get_samples() is not None and dataset.get_labels() is not None:
+ x = pd.DataFrame(dataset.get_samples(), columns=self._features)
+ if not self.features_to_minimize:
+ self.features_to_minimize = self._features
+ self.features_to_minimize = [str(i) for i in self.features_to_minimize]
+ if not all(elem in self._features for elem in self.features_to_minimize):
+ raise ValueError('features to minimize should be a subset of features names')
+ x_QI = x.loc[:, self.features_to_minimize]
- if self.is_numpy:
- if not self.features_to_minimize:
- self.features_to_minimize = [i for i in range(len(self._features))]
- x_QI = X[:, self.features_to_minimize]
- self.features_to_minimize = [self._features[i] for i in self.features_to_minimize]
- X = pd.DataFrame(X, columns=self._features)
- else:
- if not self.features_to_minimize:
- self.features_to_minimize = self._features
- x_QI = X.loc[:, self.features_to_minimize]
- x_QI = pd.DataFrame(x_QI, columns=self.features_to_minimize)
# divide dataset into train and test
- used_data = X
+ used_data = x
if self.train_only_QI:
used_data = x_QI
if self.is_regression:
- X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.4, random_state=14)
+ X_train, X_test, y_train, y_test = train_test_split(x, dataset.get_labels(), test_size=0.4, random_state=14)
else:
- X_train, X_test, y_train, y_test = train_test_split(X, y, stratify=y, test_size=0.4, random_state=18)
+ X_train, X_test, y_train, y_test = train_test_split(x, dataset.get_labels(), stratify=dataset.get_labels(), test_size=0.4,
+ random_state=18)
X_train_QI = X_train.loc[:, self.features_to_minimize]
X_test_QI = X_test.loc[:, self.features_to_minimize]
@@ -246,7 +247,7 @@ class GeneralizeToRepresentative(BaseEstimator, MetaEstimatorMixin, TransformerM
for feature in self._features:
if feature not in feature_data.keys():
fd = {}
- values = list(X.loc[:, feature])
+ values = list(x.loc[:, feature])
if feature not in self.categorical_features:
fd['min'] = min(values)
fd['max'] = max(values)
@@ -259,7 +260,6 @@ class GeneralizeToRepresentative(BaseEstimator, MetaEstimatorMixin, TransformerM
categorical_features = [f for f in self._features if f in self.categorical_features and
f in self.features_to_minimize]
-
numeric_transformer = Pipeline(
steps=[('imputer', SimpleImputer(strategy='constant', fill_value=0))]
)
@@ -288,7 +288,7 @@ class GeneralizeToRepresentative(BaseEstimator, MetaEstimatorMixin, TransformerM
("cat", categorical_transformer, self.categorical_features),
]
)
- preprocessor.fit(X)
+ preprocessor.fit(x)
x_prepared = preprocessor.transform(X_train)
if self.train_only_QI:
x_prepared = preprocessor_QI_features.transform(X_train_QI)
@@ -300,7 +300,7 @@ class GeneralizeToRepresentative(BaseEstimator, MetaEstimatorMixin, TransformerM
self.dt_ = DecisionTreeRegressor(random_state=10, min_samples_split=2, min_samples_leaf=1)
else:
self.dt_ = DecisionTreeClassifier(random_state=0, min_samples_split=2,
- min_samples_leaf=1)
+ min_samples_leaf=1)
self.dt_.fit(x_prepared, y_train)
self._modify_categorical_features(used_data)
@@ -329,7 +329,7 @@ class GeneralizeToRepresentative(BaseEstimator, MetaEstimatorMixin, TransformerM
generalized = self._generalize(X_test, x_prepared_test, nodes, self.cells_, self.cells_by_id_)
# check accuracy
- accuracy = self.estimator.score(preprocessor.transform(generalized), y_test)
+ accuracy = self.estimator.score(ArrayDataset(preprocessor.transform(generalized), y_test))
print('Initial accuracy of model on generalized data, relative to original model predictions '
'(base generalization derived from tree, before improvements): %f' % accuracy)
@@ -349,7 +349,7 @@ class GeneralizeToRepresentative(BaseEstimator, MetaEstimatorMixin, TransformerM
self._calculate_generalizations()
generalized = self._generalize(X_test, x_prepared_test, nodes, self.cells_,
self.cells_by_id_)
- accuracy = self.estimator.score(preprocessor.transform(generalized), y_test)
+ accuracy = self.estimator.score(ArrayDataset(preprocessor.transform(generalized), y_test))
# if accuracy passed threshold roll back to previous iteration generalizations
if accuracy < self.target_accuracy:
self.cells_ = cells_previous_iter
@@ -375,7 +375,7 @@ class GeneralizeToRepresentative(BaseEstimator, MetaEstimatorMixin, TransformerM
self._calculate_generalizations()
generalized = self._generalize(X_test, x_prepared_test, nodes, self.cells_, self.cells_by_id_)
- accuracy = self.estimator.score(preprocessor.transform(generalized), y_test)
+ accuracy = self.estimator.score(ArrayDataset(preprocessor.transform(generalized), y_test))
print('Removed feature: %s, new relative accuracy: %f' % (removed_feature, accuracy))
# self.cells_ currently holds the chosen generalization based on target accuracy
@@ -386,7 +386,7 @@ class GeneralizeToRepresentative(BaseEstimator, MetaEstimatorMixin, TransformerM
# Return the transformer
return self
- def transform(self, X: Union[np.ndarray, pd.DataFrame]):
+ def transform(self, X: Optional[DATA_PANDAS_NUMPY_TYPE] = None, features_names: Optional = None, dataset: ArrayDataset = None):
""" Transforms data records to representative points.
Parameters
@@ -394,6 +394,10 @@ class GeneralizeToRepresentative(BaseEstimator, MetaEstimatorMixin, TransformerM
X : {array-like, sparse-matrix}, shape (n_samples, n_features), If provided as a pandas dataframe,
may contain both numeric and categorical data.
The input samples.
+ features_names : list of str, The feature names, in the order that they appear in the data,
+ provided just if X was provided (optional).
+ dataset : Data wrapper containing the training input samples.
+ Either X OR dataset need to be provided, not both.
Returns
-------
X_transformed : numpy or pandas according to the input type, shape (n_samples, n_features)
@@ -405,26 +409,30 @@ class GeneralizeToRepresentative(BaseEstimator, MetaEstimatorMixin, TransformerM
msg = 'This %(name)s instance is not initialized yet. ' \
'Call ‘fit’ or ‘set_params’ with ' \
'appropriate arguments before using this method.'
- check_is_fitted(self, ['cells', 'features'], msg=msg)
+ check_is_fitted(self, ['cells'], msg=msg)
- if type(X) == np.ndarray:
- # Input validation
- X = check_array(X, accept_sparse=True)
- self.is_numpy = True
- X = pd.DataFrame(X, columns=self._features)
- else:
- self.is_numpy = False
+ if X is not None:
+ if dataset is not None:
+ raise ValueError('Either X OR dataset need to be provided, not both')
+ else:
+ dataset = ArrayDataset(X, features_names=features_names)
+ elif dataset is None:
+ raise ValueError('Either X OR dataset need to be provided, not both')
+ if dataset and dataset.features_names:
+ self._features = dataset.features_names
+ if dataset and dataset.get_samples() is not None:
+ x = pd.DataFrame(dataset.get_samples(), columns=self._features)
- if X.shape[1] != self.n_features_ and self.n_features_ != 0:
+ if x.shape[1] != self.n_features_ and self.n_features_ != 0:
raise ValueError('Shape of input is different from what was seen'
'in `fit`')
if not self._features:
- self._features = [i for i in range(X.shape[1])]
+ self._features = [i for i in range(x.shape[1])]
representatives = pd.DataFrame(columns=self._features) # only columns
- generalized = pd.DataFrame(X, columns=self._features, copy=True) # original data
- mapped = np.zeros(X.shape[0]) # to mark records we already mapped
+ generalized = pd.DataFrame(x, columns=self._features, copy=True) # original data
+ mapped = np.zeros(x.shape[0]) # to mark records we already mapped
# iterate over cells (leaves in decision tree)
for i in range(len(self.cells_)):
@@ -443,7 +451,7 @@ class GeneralizeToRepresentative(BaseEstimator, MetaEstimatorMixin, TransformerM
representatives = representatives.drop(feature, axis=1)
# get the indexes of all records that map to this cell
- indexes = self._get_record_indexes_for_cell(X, self.cells_[i], mapped)
+ indexes = self._get_record_indexes_for_cell(x, self.cells_[i], mapped)
# replace the values in the representative columns with the representative
# values (leaves others untouched)
@@ -454,9 +462,11 @@ class GeneralizeToRepresentative(BaseEstimator, MetaEstimatorMixin, TransformerM
replace = representatives.loc[i].to_frame().T.reset_index(drop=True)
replace.index = indexes
generalized.loc[indexes, representatives.columns] = replace
- if self.is_numpy:
- return generalized.to_numpy()
- return generalized
+ if dataset and dataset.is_pandas:
+ return generalized
+ elif isinstance(X, pd.DataFrame):
+ return generalized
+ return generalized.to_numpy()
def _get_record_indexes_for_cell(self, X, cell, mapped):
indexes = []
@@ -640,7 +650,8 @@ class GeneralizeToRepresentative(BaseEstimator, MetaEstimatorMixin, TransformerM
# else: nothing to do, stay with previous cells
def _calculate_level_cell_label(self, left_cell, right_cell, new_cell):
- new_cell['hist'] = [x + y for x, y in zip(left_cell['hist'], right_cell['hist'])] if not self.is_regression else []
+ new_cell['hist'] = [x + y for x, y in
+ zip(left_cell['hist'], right_cell['hist'])] if not self.is_regression else []
new_cell['label'] = int(self.dt_.classes_[np.argmax(new_cell['hist'])]) if not self.is_regression else 1
def _get_nodes_level(self, level):
@@ -797,8 +808,8 @@ class GeneralizeToRepresentative(BaseEstimator, MetaEstimatorMixin, TransformerM
cells_by_id = copy.deepcopy(self.cells_by_id_)
GeneralizeToRepresentative._remove_feature_from_cells(new_cells, cells_by_id, feature)
generalized = self._generalize(original_data, prepared_data, nodes, new_cells, cells_by_id)
- accuracy_gain = self.estimator.score(self._preprocessor.transform(generalized),
- labels) - current_accuracy
+ accuracy_gain = self.estimator.score(ArrayDataset(self._preprocessor.transform(generalized),
+ labels)) - current_accuracy
if accuracy_gain < 0:
accuracy_gain = 0
if accuracy_gain != 0:
@@ -820,8 +831,8 @@ class GeneralizeToRepresentative(BaseEstimator, MetaEstimatorMixin, TransformerM
cells_by_id = copy.deepcopy(self.cells_by_id_)
GeneralizeToRepresentative._remove_feature_from_cells(new_cells, cells_by_id, feature)
generalized = self._generalize(original_data, prepared_data, nodes, new_cells, cells_by_id)
- accuracy_gain = self.estimator.score(self._preprocessor.transform(generalized),
- labels) - current_accuracy
+ accuracy_gain = self.estimator.score(ArrayDataset(self._preprocessor.transform(generalized),
+ labels)) - current_accuracy
if accuracy_gain < 0:
accuracy_gain = 0
diff --git a/apt/utils/__init__.py b/apt/utils/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/apt/utils.py b/apt/utils/dataset_utils.py
similarity index 93%
rename from apt/utils.py
rename to apt/utils/dataset_utils.py
index bc73cbc..e3eb959 100644
--- a/apt/utils.py
+++ b/apt/utils/dataset_utils.py
@@ -13,8 +13,7 @@ def _load_iris(test_set_size: float = 0.3):
# Split training and test sets
x_train, x_test, y_train, y_test = model_selection.train_test_split(data, labels, test_size=test_set_size,
- random_state=18, stratify=labels,
- shuffle=True)
+ random_state=18, stratify=labels)
return (x_train, y_train), (x_test, y_test)
@@ -29,6 +28,28 @@ def get_iris_dataset(test_set: float = 0.3):
return _load_iris(test_set)
+def _load_diabetes(test_set_size: float = 0.3):
+ diabetes = datasets.load_diabetes()
+ data = diabetes.data
+ labels = diabetes.target
+
+ # Split training and test sets
+ x_train, x_test, y_train, y_test = model_selection.train_test_split(data, labels, test_size=test_set_size,
+ random_state=18)
+
+ return (x_train, y_train), (x_test, y_test)
+
+
+def get_diabetes_dataset():
+ """
+ Loads the Iris dataset from scikit-learn.
+
+ :param test_set: Proportion of the data to use as validation split (value between 0 and 1).
+ :return: Entire dataset and labels as numpy array.
+ """
+ return _load_diabetes()
+
+
def get_german_credit_dataset(test_set: float = 0.3):
"""
Loads the UCI German_credit dataset from `tests/datasets/german` or downloads it if necessary.
@@ -253,7 +274,7 @@ def get_nursery_dataset(raw: bool = True, test_set: float = 0.2, transform_socia
raise Exception("Bad label value: %s" % value)
data["label"] = data["label"].apply(modify_label)
- data["children"] = data["children"].apply(lambda x: 4 if x == "more" else x)
+ data["children"] = data["children"].apply(lambda x: "4" if x == "more" else x)
if transform_social:
diff --git a/apt/utils/datasets/__init__.py b/apt/utils/datasets/__init__.py
new file mode 100644
index 0000000..6e7c640
--- /dev/null
+++ b/apt/utils/datasets/__init__.py
@@ -0,0 +1,7 @@
+"""
+The AI Privacy Toolbox (datasets).
+Implementation of datasets utility components for datasets creation, load, and store
+"""
+
+from apt.utils.datasets.datasets import Dataset, StoredDataset, DatasetFactory, Data, ArrayDataset, \
+ OUTPUT_DATA_ARRAY_TYPE, DATA_PANDAS_NUMPY_TYPE
diff --git a/apt/utils/datasets/datasets.py b/apt/utils/datasets/datasets.py
new file mode 100644
index 0000000..ff7c296
--- /dev/null
+++ b/apt/utils/datasets/datasets.py
@@ -0,0 +1,320 @@
+# !/usr/bin/env python
+"""
+The AI Privacy Toolbox (datasets).
+Implementation of utility classes for dataset handling
+"""
+
+from abc import ABCMeta, abstractmethod
+from typing import Callable, Collection, Any, Union, List, Optional
+
+import tarfile
+import os
+import urllib.request
+import numpy as np
+import pandas as pd
+import logging
+import torch
+from torch import Tensor
+
+logger = logging.getLogger(__name__)
+
+
+INPUT_DATA_ARRAY_TYPE = Union[np.ndarray, pd.DataFrame, List, Tensor]
+OUTPUT_DATA_ARRAY_TYPE = np.ndarray
+DATA_PANDAS_NUMPY_TYPE = Union[np.ndarray, pd.DataFrame]
+
+
+def array2numpy(self, arr: INPUT_DATA_ARRAY_TYPE) -> OUTPUT_DATA_ARRAY_TYPE:
+
+ """
+ converts from INPUT_DATA_ARRAY_TYPE to numpy array
+ """
+ if type(arr) == np.ndarray:
+ return arr
+ if type(arr) == pd.DataFrame or type(arr) == pd.Series:
+ self.is_pandas = True
+ return arr.to_numpy()
+ if isinstance(arr, list):
+ return np.array(arr)
+ if type(arr) == Tensor:
+ return arr.detach().cpu().numpy()
+
+ raise ValueError('Non supported type: ', type(arr).__name__)
+
+
+def array2torch_tensor(self, arr: INPUT_DATA_ARRAY_TYPE) -> Tensor:
+ """
+ converts from INPUT_DATA_ARRAY_TYPE to torch tensor array
+ """
+ if type(arr) == np.ndarray:
+ return torch.from_numpy(arr)
+ if type(arr) == pd.DataFrame or type(arr) == pd.Series:
+ self.is_pandas = True
+ return torch.from_numpy(arr.to_numpy())
+ if isinstance(arr, list):
+ return torch.tensor(arr)
+ if type(arr) == Tensor:
+ return arr
+
+ raise ValueError('Non supported type: ', type(arr).__name__)
+
+
+class Dataset(metaclass=ABCMeta):
+ """Base Abstract Class for Dataset"""
+
+ @abstractmethod
+ def __init__(self, **kwargs):
+ pass
+
+ @abstractmethod
+ def get_samples(self) -> Collection[Any]:
+ """Return data samples"""
+ pass
+
+ @abstractmethod
+ def get_labels(self) -> Collection[Any]:
+ """Return labels"""
+ pass
+
+
+class StoredDataset(Dataset):
+ """Abstract Class for Storable Dataset"""
+
+ @abstractmethod
+ def load_from_file(self, path: str):
+ """Load dataset from file"""
+ pass
+
+ @abstractmethod
+ def load(self, **kwargs):
+ """Load dataset"""
+ pass
+
+ @staticmethod
+ def download(url: str, dest_path: str, filename: str, unzip: bool = False) -> None:
+ """
+ Download the dataset from URL
+ :param url: dataset URL, the dataset will be requested from this URL
+ :param dest_path: local dataset destination path
+ :param filename: local dataset filename
+ :param unzip: flag whether or not perform extraction
+ :return: None
+ """
+ file_path = os.path.join(dest_path, filename)
+
+ if os.path.exists(file_path):
+ logger.warning("Files already downloaded, skipping downloading")
+
+ else:
+ os.makedirs(dest_path, exist_ok=True)
+ logger.info("Downloading the dataset...")
+ urllib.request.urlretrieve(url, file_path)
+ logger.info('Dataset Downloaded')
+
+ if unzip:
+ StoredDataset.extract_archive(zip_path=file_path, dest_path=dest_path, remove_archive=False)
+
+ @staticmethod
+ def extract_archive(zip_path: str, dest_path=None, remove_archive=False):
+ """
+ Extract dataset from archived file
+ :param zip_path: path to archived file
+ :param dest_path: directory path to uncompress the file to
+ :param remove_archive: whether remove the archive file after uncompress (default False)
+ :return: None
+ """
+ logger.info("Extracting the dataset...")
+ tar = tarfile.open(zip_path)
+ tar.extractall(path=dest_path)
+
+ logger.info("Dataset was extracted to {}".format(dest_path))
+ if remove_archive:
+ logger.info("Removing a zip file")
+ os.remove(zip_path)
+ logger.info("Extracted the dataset")
+
+ @staticmethod
+ def split_debug(datafile: str, dest_datafile: str, ratio: int, shuffle=True, delimiter=",", fmt=None) -> None:
+ """
+ Split the data and take only a part of it
+ :param datafile: dataset file path
+ :param dest_datafile: destination path for the partial dataset file
+ :param ratio: part of the dataset to save
+ :param shuffle: whether to shuffle the data or not (default True)
+ :param delimiter: dataset delimiter (default ",")
+ :param fmt: format for the correct data saving
+ :return: None
+ """
+ if os.path.isfile(dest_datafile):
+ logger.info(f"The partial debug split already exists {dest_datafile}")
+ return
+ else:
+ os.makedirs(os.path.dirname(dest_datafile), exist_ok=True)
+
+ data = np.genfromtxt(datafile, delimiter=delimiter)
+ if shuffle:
+ logger.info("Shuffling data")
+ np.random.shuffle(data)
+
+ debug_data = data[:int(len(data) * ratio)]
+ logger.info(f"Saving {ratio} of the data to {dest_datafile}")
+ np.savetxt(dest_datafile, debug_data, delimiter=delimiter, fmt=fmt)
+
+
+class ArrayDataset(Dataset):
+ """Dataset that is based on x and y arrays (e.g., numpy/pandas/list...)"""
+
+ def __init__(self, x: INPUT_DATA_ARRAY_TYPE, y: Optional[INPUT_DATA_ARRAY_TYPE] = None,
+ features_names: Optional = None, **kwargs):
+ """
+ ArrayDataset constructor.
+ :param x: collection of data samples
+ :param y: collection of labels (optional)
+ :param feature_names: list of str, The feature names, in the order that they appear in the data (optional)
+ :param kwargs: dataset parameters
+ """
+ self.is_pandas = False
+ self.features_names = features_names
+ self._y = array2numpy(self, y) if y is not None else None
+ self._x = array2numpy(self, x)
+ if self.is_pandas:
+ if features_names and not np.array_equal(features_names, x.columns):
+ raise ValueError("The supplied features are not the same as in the data features")
+ self.features_names = x.columns.to_list()
+
+ if y is not None and len(self._x) != len(self._y):
+ raise ValueError('Non equivalent lengths of x and y')
+
+ def get_samples(self) -> OUTPUT_DATA_ARRAY_TYPE:
+ """Return data samples as numpy array"""
+ return self._x
+
+ def get_labels(self) -> OUTPUT_DATA_ARRAY_TYPE:
+ """Return labels as numpy array"""
+ return self._y
+
+
+class PytorchData(Dataset):
+
+ def __init__(self, x: INPUT_DATA_ARRAY_TYPE, y: Optional[INPUT_DATA_ARRAY_TYPE] = None, **kwargs):
+ """
+ PytorchData constructor.
+ :param x: collection of data samples
+ :param y: collection of labels (optional)
+ :param kwargs: dataset parameters
+ """
+ self.is_pandas = False
+ self._y = array2torch_tensor(self, y) if y is not None else None
+ self._x = array2torch_tensor(self, x)
+ if self.is_pandas:
+ self.features_names = x.columns
+
+ if y is not None and len(self._x) != len(self._y):
+ raise ValueError('Non equivalent lengths of x and y')
+
+
+ if self._y is not None:
+ self.__getitem__ = self.get_item
+ else:
+ self.__getitem__ = self.get_sample_item
+
+
+ def get_samples(self) -> OUTPUT_DATA_ARRAY_TYPE:
+ """Return data samples as numpy array"""
+ return array2numpy(self._x)
+
+ def get_labels(self) -> OUTPUT_DATA_ARRAY_TYPE:
+ """Return labels as numpy array"""
+ return array2numpy(self._y) if self._y is not None else None
+
+ def get_sample_item(self, idx) -> Tensor:
+ return self.x[idx]
+
+ def get_item(self, idx) -> Tensor:
+ sample, label = self.x[idx], self.y[idx]
+ return sample, label
+
+ def __len__(self):
+ return len(self.x)
+
+
+class DatasetFactory:
+ """Factory class for dataset creation"""
+ registry = {}
+
+ @classmethod
+ def register(cls, name: str) -> Callable:
+ """
+ Class method to register Dataset to the internal registry
+ :param name: dataset name
+ :return:
+ """
+
+ def inner_wrapper(wrapped_class: Dataset) -> Any:
+ if name in cls.registry:
+ logger.warning('Dataset %s already exists. Will replace it', name)
+ cls.registry[name] = wrapped_class
+ return wrapped_class
+
+ return inner_wrapper
+
+ @classmethod
+ def create_dataset(cls, name: str, **kwargs) -> Dataset:
+ """
+ Factory command to create dataset instance.
+ This method gets the appropriate Dataset class from the registry
+ and creates an instance of it, while passing in the parameters
+ given in ``kwargs``.
+ :param name: The name of the dataset to create.
+ :param kwargs: dataset parameters
+ :return: An instance of the dataset that is created.
+ """
+ if name not in cls.registry:
+ msg = f'Dataset {name} does not exist in the registry'
+ logger.error(msg)
+ raise ValueError(msg)
+
+ exec_class = cls.registry[name]
+ executor = exec_class(**kwargs)
+ return executor
+
+
+class Data:
+ def __init__(self, train: Dataset = None, test: Dataset = None, **kwargs):
+ """
+ Data class constructor.
+ The class stores train and test datasets.
+ If neither of the datasets was provided,
+ Both train and test datasets will be create using
+ DatasetFactory to create a dataset instance
+ """
+ if train or test:
+ self.train = train
+ self.test = test
+ else:
+ self.train = DatasetFactory.create_dataset(train=True, **kwargs)
+ self.test = DatasetFactory.create_dataset(train=False, **kwargs)
+
+ def get_train_set(self) -> Dataset:
+ """Return train DatasetBase"""
+ return self.train
+
+ def get_test_set(self) -> Dataset:
+ """Return test DatasetBase"""
+ return self.test
+
+ def get_train_samples(self) -> Collection[Any]:
+ """Return train set samples"""
+ return self.train.get_samples()
+
+ def get_train_labels(self) -> Collection[Any]:
+ """Return train set labels"""
+ return self.train.get_labels()
+
+ def get_test_samples(self) -> Collection[Any]:
+ """Return test set samples"""
+ return self.test.get_samples()
+
+ def get_test_labels(self) -> Collection[Any]:
+ """Return test set labels"""
+ return self.test.get_labels()
diff --git a/apt/utils/models/__init__.py b/apt/utils/models/__init__.py
new file mode 100644
index 0000000..11efd5f
--- /dev/null
+++ b/apt/utils/models/__init__.py
@@ -0,0 +1,2 @@
+from apt.utils.models.model import Model, ModelOutputType
+from apt.utils.models.sklearn_model import SklearnModel, SklearnClassifier, SklearnRegressor
diff --git a/apt/utils/models/model.py b/apt/utils/models/model.py
new file mode 100644
index 0000000..9616459
--- /dev/null
+++ b/apt/utils/models/model.py
@@ -0,0 +1,109 @@
+from abc import ABCMeta, abstractmethod
+from typing import Any, Optional
+from enum import Enum, auto
+
+from apt.utils.datasets import Dataset, OUTPUT_DATA_ARRAY_TYPE
+
+
+class ModelOutputType(Enum):
+ CLASSIFIER_VECTOR = auto() # probabilities or logits
+ CLASSIFIER_SCALAR = auto() # label only
+ REGRESSOR_SCALAR = auto() # value
+
+
+class Model(metaclass=ABCMeta):
+ """
+ Abstract base class for ML model wrappers.
+ """
+
+ def __init__(self, model: Any, output_type: ModelOutputType, black_box_access: Optional[bool] = True,
+ unlimited_queries: Optional[bool] = True, **kwargs):
+ """
+ Initialize a `Model` wrapper object.
+
+ :param model: The original model object (of the underlying ML framework)
+ :param output_type: The type of output the model yields (vector/label only for classifiers,
+ value for regressors)
+ :param black_box_access: Boolean describing the type of deployment of the model (when in production).
+ Set to True if the model is only available via query (API) access, i.e.,
+ only the outputs of the model are exposed, and False if the model internals
+ are also available. Optional, Default is True.
+ :param unlimited_queries: If black_box_access is True, this boolean indicates whether a user can perform
+ unlimited queries to the model API or whether there is a limit to the number of
+ queries that can be submitted. Optional, Default is True.
+ """
+ self._model = model
+ self._output_type = output_type
+ self._black_box_access = black_box_access
+ self._unlimited_queries = unlimited_queries
+
+ @abstractmethod
+ def fit(self, train_data: Dataset, **kwargs) -> None:
+ """
+ Fit the model using the training data.
+
+ :param train_data: Training data.
+ :type train_data: `Dataset`
+ """
+ raise NotImplementedError
+
+ @abstractmethod
+ def predict(self, x: Dataset, **kwargs) -> OUTPUT_DATA_ARRAY_TYPE:
+ """
+ Perform predictions using the model for input `x`.
+
+ :param x: Input samples.
+ :type x: `np.ndarray` or `pandas.DataFrame`
+ :return: Predictions from the model.
+ """
+ raise NotImplementedError
+
+ @abstractmethod
+ def score(self, test_data: Dataset, **kwargs):
+ """
+ Score the model using test data.
+
+ :param test_data: Test data.
+ :type train_data: `Dataset`
+ """
+ return NotImplementedError
+
+ @property
+ def model(self) -> Any:
+ """
+ Return the model.
+
+ :return: The model.
+ """
+ return self._model
+
+ @property
+ def output_type(self) -> ModelOutputType:
+ """
+ Return the model's output type.
+
+ :return: The model's output type.
+ """
+ return self._output_type
+
+ @property
+ def black_box_access(self) -> bool:
+ """
+ Return True if the model is only available via query (API) access, i.e.,
+ only the outputs of the model are exposed, and False if the model internals are also available.
+
+ :return: True if the model is only available via query (API) access, i.e.,
+ only the outputs of the model are exposed, and False if the model internals are also available.
+ """
+ return self._black_box_access
+
+ @property
+ def unlimited_queries(self) -> bool:
+ """
+ If black_box_access is True, Return whether a user can perform unlimited queries to the model API
+ or whether there is a limit to the number of queries that can be submitted.
+
+ :return: If black_box_access is True, Return whether a user can perform unlimited queries to the model API
+ or whether there is a limit to the number of queries that can be submitted.
+ """
+ return self._unlimited_queries
diff --git a/apt/utils/models/sklearn_model.py b/apt/utils/models/sklearn_model.py
new file mode 100644
index 0000000..f7afaa6
--- /dev/null
+++ b/apt/utils/models/sklearn_model.py
@@ -0,0 +1,112 @@
+from typing import Optional
+
+import numpy as np
+
+from sklearn.preprocessing import OneHotEncoder
+from sklearn.base import BaseEstimator
+
+from apt.utils.models import Model, ModelOutputType
+from apt.utils.datasets import Dataset, OUTPUT_DATA_ARRAY_TYPE
+
+from art.estimators.classification.scikitlearn import SklearnClassifier as ArtSklearnClassifier
+from art.estimators.regression.scikitlearn import ScikitlearnRegressor
+
+
+class SklearnModel(Model):
+ """
+ Wrapper class for scikitlearn models.
+ """
+ def score(self, test_data: Dataset, **kwargs):
+ """
+ Score the model using test data.
+
+ :param test_data: Test data.
+ :type train_data: `Dataset`
+ """
+ return self.model.score(test_data.get_samples(), test_data.get_labels(), **kwargs)
+
+
+class SklearnClassifier(SklearnModel):
+ """
+ Wrapper class for scikitlearn classification models.
+ """
+ def __init__(self, model: BaseEstimator, output_type: ModelOutputType, black_box_access: Optional[bool] = True,
+ unlimited_queries: Optional[bool] = True, **kwargs):
+ """
+ Initialize a `SklearnClassifier` wrapper object.
+
+ :param model: The original sklearn model object.
+ :param output_type: The type of output the model yields (vector/label only for classifiers,
+ value for regressors)
+ :param black_box_access: Boolean describing the type of deployment of the model (when in production).
+ Set to True if the model is only available via query (API) access, i.e.,
+ only the outputs of the model are exposed, and False if the model internals
+ are also available. Optional, Default is True.
+ :param unlimited_queries: If black_box_access is True, this boolean indicates whether a user can perform
+ unlimited queries to the model API or whether there is a limit to the number of
+ queries that can be submitted. Optional, Default is True.
+ """
+ super().__init__(model, output_type, black_box_access, unlimited_queries, **kwargs)
+ self._art_model = ArtSklearnClassifier(model)
+
+ def fit(self, train_data: Dataset, **kwargs) -> None:
+ """
+ Fit the model using the training data.
+
+ :param train_data: Training data.
+ :type train_data: `Dataset`
+ """
+ encoder = OneHotEncoder(sparse=False)
+ y_encoded = encoder.fit_transform(train_data.get_labels().reshape(-1, 1))
+ self._art_model.fit(train_data.get_samples(), y_encoded, **kwargs)
+
+ def predict(self, x: Dataset, **kwargs) -> OUTPUT_DATA_ARRAY_TYPE:
+ """
+ Perform predictions using the model for input `x`.
+
+ :param x: Input samples.
+ :type x: `np.ndarray` or `pandas.DataFrame`
+ :return: Predictions from the model (class probabilities, if supported).
+ """
+ return self._art_model.predict(x, **kwargs)
+
+
+class SklearnRegressor(SklearnModel):
+ """
+ Wrapper class for scikitlearn regression models.
+ """
+ def __init__(self, model: BaseEstimator, black_box_access: Optional[bool] = True,
+ unlimited_queries: Optional[bool] = True, **kwargs):
+ """
+ Initialize a `SklearnRegressor` wrapper object.
+
+ :param model: The original sklearn model object.
+ :param black_box_access: Boolean describing the type of deployment of the model (when in production).
+ Set to True if the model is only available via query (API) access, i.e.,
+ only the outputs of the model are exposed, and False if the model internals
+ are also available. Optional, Default is True.
+ :param unlimited_queries: If black_box_access is True, this boolean indicates whether a user can perform
+ unlimited queries to the model API or whether there is a limit to the number of
+ queries that can be submitted. Optional, Default is True.
+ """
+ super().__init__(model, ModelOutputType.REGRESSOR_SCALAR, black_box_access, unlimited_queries, **kwargs)
+ self._art_model = ScikitlearnRegressor(model)
+
+ def fit(self, train_data: Dataset, **kwargs) -> None:
+ """
+ Fit the model using the training data.
+
+ :param train_data: Training data.
+ :type train_data: `Dataset`
+ """
+ self._art_model.fit(train_data.get_samples(), train_data.get_labels(), **kwargs)
+
+ def predict(self, x: Dataset, **kwargs) -> OUTPUT_DATA_ARRAY_TYPE:
+ """
+ Perform predictions using the model for input `x`.
+
+ :param x: Input samples.
+ :type x: `np.ndarray` or `pandas.DataFrame`
+ :return: Predictions from the model.
+ """
+ return self._art_model.predict(x, **kwargs)
diff --git a/notebooks/attribute_inference_anonymization_nursery.ipynb b/notebooks/attribute_inference_anonymization_nursery.ipynb
index 9952885..bfba540 100644
--- a/notebooks/attribute_inference_anonymization_nursery.ipynb
+++ b/notebooks/attribute_inference_anonymization_nursery.ipynb
@@ -29,198 +29,15 @@
},
{
"cell_type": "code",
- "execution_count": 61,
+ "execution_count": 1,
"metadata": {},
"outputs": [
{
"data": {
- "text/html": [
- "\n",
- "\n",
- "
\n",
- " \n",
- " \n",
- " | \n",
- " parents | \n",
- " has_nurs | \n",
- " form | \n",
- " children | \n",
- " housing | \n",
- " finance | \n",
- " social | \n",
- " health | \n",
- "
\n",
- " \n",
- " \n",
- " \n",
- " | 8450 | \n",
- " pretentious | \n",
- " very_crit | \n",
- " foster | \n",
- " 1 | \n",
- " less_conv | \n",
- " convenient | \n",
- " 1 | \n",
- " not_recom | \n",
- "
\n",
- " \n",
- " | 12147 | \n",
- " great_pret | \n",
- " very_crit | \n",
- " complete | \n",
- " 1 | \n",
- " critical | \n",
- " inconv | \n",
- " 1 | \n",
- " recommended | \n",
- "
\n",
- " \n",
- " | 2780 | \n",
- " usual | \n",
- " critical | \n",
- " complete | \n",
- " 4 | \n",
- " less_conv | \n",
- " convenient | \n",
- " 1 | \n",
- " not_recom | \n",
- "
\n",
- " \n",
- " | 11924 | \n",
- " great_pret | \n",
- " critical | \n",
- " foster | \n",
- " 1 | \n",
- " critical | \n",
- " convenient | \n",
- " 1 | \n",
- " not_recom | \n",
- "
\n",
- " \n",
- " | 59 | \n",
- " usual | \n",
- " proper | \n",
- " complete | \n",
- " 2 | \n",
- " convenient | \n",
- " convenient | \n",
- " 0 | \n",
- " not_recom | \n",
- "
\n",
- " \n",
- " | ... | \n",
- " ... | \n",
- " ... | \n",
- " ... | \n",
- " ... | \n",
- " ... | \n",
- " ... | \n",
- " ... | \n",
- " ... | \n",
- "
\n",
- " \n",
- " | 5193 | \n",
- " pretentious | \n",
- " less_proper | \n",
- " complete | \n",
- " 1 | \n",
- " convenient | \n",
- " inconv | \n",
- " 0 | \n",
- " recommended | \n",
- "
\n",
- " \n",
- " | 1375 | \n",
- " usual | \n",
- " less_proper | \n",
- " incomplete | \n",
- " 2 | \n",
- " less_conv | \n",
- " convenient | \n",
- " 1 | \n",
- " priority | \n",
- "
\n",
- " \n",
- " | 10318 | \n",
- " great_pret | \n",
- " less_proper | \n",
- " foster | \n",
- " 4 | \n",
- " convenient | \n",
- " convenient | \n",
- " 0 | \n",
- " priority | \n",
- "
\n",
- " \n",
- " | 6396 | \n",
- " pretentious | \n",
- " improper | \n",
- " completed | \n",
- " 3 | \n",
- " less_conv | \n",
- " convenient | \n",
- " 1 | \n",
- " recommended | \n",
- "
\n",
- " \n",
- " | 485 | \n",
- " usual | \n",
- " proper | \n",
- " incomplete | \n",
- " 1 | \n",
- " critical | \n",
- " inconv | \n",
- " 1 | \n",
- " not_recom | \n",
- "
\n",
- " \n",
- "
\n",
- "
10366 rows × 8 columns
\n",
- "
\n\n
\n \n \n | \n parents | \n has_nurs | \n form | \n children | \n housing | \n finance | \n social | \n health | \n
\n \n \n \n | 8450 | \n pretentious | \n very_crit | \n foster | \n 1 | \n less_conv | \n convenient | \n 1 | \n not_recom | \n
\n \n | 12147 | \n great_pret | \n very_crit | \n complete | \n 1 | \n critical | \n inconv | \n 1 | \n recommended | \n
\n \n | 2780 | \n usual | \n critical | \n complete | \n 4 | \n less_conv | \n convenient | \n 1 | \n not_recom | \n
\n \n | 11924 | \n great_pret | \n critical | \n foster | \n 1 | \n critical | \n convenient | \n 1 | \n not_recom | \n
\n \n | 59 | \n usual | \n proper | \n complete | \n 2 | \n convenient | \n convenient | \n 0 | \n not_recom | \n
\n \n | ... | \n ... | \n ... | \n ... | \n ... | \n ... | \n ... | \n ... | \n ... | \n
\n \n | 5193 | \n pretentious | \n less_proper | \n complete | \n 1 | \n convenient | \n inconv | \n 0 | \n recommended | \n
\n \n | 1375 | \n usual | \n less_proper | \n incomplete | \n 2 | \n less_conv | \n convenient | \n 1 | \n priority | \n
\n \n | 10318 | \n great_pret | \n less_proper | \n foster | \n 4 | \n convenient | \n convenient | \n 0 | \n priority | \n
\n \n | 6396 | \n pretentious | \n improper | \n completed | \n 3 | \n less_conv | \n convenient | \n 1 | \n recommended | \n
\n \n | 485 | \n usual | \n proper | \n incomplete | \n 1 | \n critical | \n inconv | \n 1 | \n not_recom | \n
\n \n
\n
10366 rows × 8 columns
\n
\n",
- "\n",
- "
\n",
- " \n",
- " \n",
- " | \n",
- " parents | \n",
- " has_nurs | \n",
- " form | \n",
- " children | \n",
- " housing | \n",
- " finance | \n",
- " social | \n",
- " health | \n",
- "
\n",
- " \n",
- " \n",
- " \n",
- " | 8450 | \n",
- " pretentious | \n",
- " very_crit | \n",
- " foster | \n",
- " 1 | \n",
- " less_conv | \n",
- " convenient | \n",
- " 0 | \n",
- " not_recom | \n",
- "
\n",
- " \n",
- " | 12147 | \n",
- " great_pret | \n",
- " very_crit | \n",
- " complete | \n",
- " 1 | \n",
- " critical | \n",
- " inconv | \n",
- " 1 | \n",
- " recommended | \n",
- "
\n",
- " \n",
- " | 2780 | \n",
- " usual | \n",
- " critical | \n",
- " complete | \n",
- " 4 | \n",
- " less_conv | \n",
- " convenient | \n",
- " 0 | \n",
- " not_recom | \n",
- "
\n",
- " \n",
- " | 11924 | \n",
- " great_pret | \n",
- " critical | \n",
- " foster | \n",
- " 1 | \n",
- " critical | \n",
- " convenient | \n",
- " 0 | \n",
- " not_recom | \n",
- "
\n",
- " \n",
- " | 59 | \n",
- " usual | \n",
- " proper | \n",
- " complete | \n",
- " 2 | \n",
- " convenient | \n",
- " convenient | \n",
- " 0 | \n",
- " not_recom | \n",
- "
\n",
- " \n",
- " | ... | \n",
- " ... | \n",
- " ... | \n",
- " ... | \n",
- " ... | \n",
- " ... | \n",
- " ... | \n",
- " ... | \n",
- " ... | \n",
- "
\n",
- " \n",
- " | 5193 | \n",
- " pretentious | \n",
- " less_proper | \n",
- " complete | \n",
- " 1 | \n",
- " convenient | \n",
- " inconv | \n",
- " 0 | \n",
- " recommended | \n",
- "
\n",
- " \n",
- " | 1375 | \n",
- " usual | \n",
- " less_proper | \n",
- " incomplete | \n",
- " 2 | \n",
- " less_conv | \n",
- " convenient | \n",
- " 1 | \n",
- " priority | \n",
- "
\n",
- " \n",
- " | 10318 | \n",
- " great_pret | \n",
- " less_proper | \n",
- " foster | \n",
- " 4 | \n",
- " convenient | \n",
- " convenient | \n",
- " 0 | \n",
- " priority | \n",
- "
\n",
- " \n",
- " | 6396 | \n",
- " pretentious | \n",
- " improper | \n",
- " completed | \n",
- " 3 | \n",
- " less_conv | \n",
- " convenient | \n",
- " 1 | \n",
- " recommended | \n",
- "
\n",
- " \n",
- " | 485 | \n",
- " usual | \n",
- " proper | \n",
- " incomplete | \n",
- " 1 | \n",
- " critical | \n",
- " convenient | \n",
- " 0 | \n",
- " not_recom | \n",
- "
\n",
- " \n",
- "
\n",
- "
10366 rows × 8 columns
\n",
- "
\n\n
\n \n \n | \n parents | \n has_nurs | \n form | \n children | \n housing | \n finance | \n social | \n health | \n
\n \n \n \n | 0 | \n pretentious | \n very_crit | \n foster | \n 1 | \n less_conv | \n convenient | \n 0 | \n not_recom | \n
\n \n | 1 | \n great_pret | \n very_crit | \n complete | \n 1 | \n critical | \n inconv | \n 1 | \n recommended | \n
\n \n | 2 | \n usual | \n critical | \n complete | \n 4 | \n less_conv | \n convenient | \n 0 | \n not_recom | \n
\n \n | 3 | \n great_pret | \n critical | \n foster | \n 1 | \n critical | \n convenient | \n 0 | \n not_recom | \n
\n \n | 4 | \n usual | \n proper | \n complete | \n 2 | \n convenient | \n convenient | \n 0 | \n not_recom | \n
\n \n | ... | \n ... | \n ... | \n ... | \n ... | \n ... | \n ... | \n ... | \n ... | \n
\n \n | 10361 | \n pretentious | \n less_proper | \n complete | \n 1 | \n convenient | \n inconv | \n 0 | \n recommended | \n
\n \n | 10362 | \n usual | \n less_proper | \n incomplete | \n 2 | \n less_conv | \n convenient | \n 1 | \n priority | \n
\n \n | 10363 | \n great_pret | \n less_proper | \n foster | \n 4 | \n convenient | \n convenient | \n 0 | \n priority | \n
\n \n | 10364 | \n pretentious | \n improper | \n completed | \n 3 | \n less_conv | \n convenient | \n 1 | \n recommended | \n
\n \n | 10365 | \n usual | \n proper | \n incomplete | \n 1 | \n critical | \n convenient | \n 0 | \n not_recom | \n
\n \n
\n
10366 rows × 8 columns
\n