apunkt/ai-privacy-toolkit
1
0
Fork 0
mirror of https://github.com/IBM/ai-privacy-toolkit.git synced 2026-06-08 15:05:13 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

Remove unused code, renaming and additional review comments

Browse source 
Signed-off-by: abigailt <abigailt@il.ibm.com>
This commit is contained in:
abigailt 2023-08-21 16:40:52 +03:00
parent 69e45d99e5
commit 256dfbbc71
2 changed files with 74 additions and 94 deletions
Download patch file Download diff file Expand all files Collapse all files

112
apt/minimization/minimizer.py
View file

@ -400,9 +400,9 @@ class GeneralizeToRepresentative(BaseEstimator, MetaEstimatorMixin, TransformerM
# self._cells currently holds the chosen generalization based on target accuracy
# calculate iLoss
X_test_dataset = ArrayDataset(x_test, features_names=self._features)
self._ncp_scores.fit_score = self.calculate_ncp(X_test_dataset, generalize_using_transform)
self._ncp_scores.generalizations_score = self.calculate_ncp(X_test_dataset, False)
x_test_dataset = ArrayDataset(x_test, features_names=self._features)
self._ncp_scores.fit_score = self.calculate_ncp(x_test_dataset, generalize_using_transform)
self._ncp_scores.generalizations_score = self.calculate_ncp(x_test_dataset, False)
# Return the transformer
return self
@ -477,7 +477,7 @@ class GeneralizeToRepresentative(BaseEstimator, MetaEstimatorMixin, TransformerM
return ncp
def _inner_transform(self, X: Optional[DATA_PANDAS_NUMPY_TYPE] = None, features_names: Optional[list] = None,
def _inner_transform(self, x: Optional[DATA_PANDAS_NUMPY_TYPE] = None, features_names: Optional[list] = None,
dataset: Optional[ArrayDataset] = None):
# Check if fit has been called
msg = 'This %(name)s instance is not initialized yet. ' \
@ -485,45 +485,45 @@ class GeneralizeToRepresentative(BaseEstimator, MetaEstimatorMixin, TransformerM
'appropriate arguments before using this method.'
check_is_fitted(self, ['cells'], msg=msg)
if X is not None:
if x is not None:
if dataset is not None:
raise ValueError('Either X OR dataset need to be provided, not both')
raise ValueError('Either x OR dataset need to be provided, not both')
else:
dataset = ArrayDataset(X, features_names=features_names)
dataset = ArrayDataset(x, features_names=features_names)
elif dataset is None:
raise ValueError('Either X OR dataset need to be provided, not both')
raise ValueError('Either x OR dataset need to be provided, not both')
if dataset and dataset.features_names:
if self._features is None:
self._features = dataset.features_names
if dataset and dataset.get_samples() is not None:
x = pd.DataFrame(dataset.get_samples(), columns=self._features)
x_pd = pd.DataFrame(dataset.get_samples(), columns=self._features)
if x.shape[1] != self._n_features and self._n_features != 0:
if x_pd.shape[1] != self._n_features and self._n_features != 0:
raise ValueError('Shape of input is different from what was seen'
'in `fit`')
if not self._features:
self._features = [i for i in range(x.shape[1])]
self._features = [i for i in range(x_pd.shape[1])]
if self._dt: # only works if fit was called previously (but much more efficient)
nodes = self._get_nodes_level(self._level)
QI = x.loc[:, self.features_to_minimize]
used_x = x
QI = x_pd.loc[:, self.features_to_minimize]
used_x = x_pd
if self.train_only_features_to_minimize:
used_x = QI
prepared = self._encode_categorical_features(used_x)
generalized = self._generalize_from_tree(x, prepared, nodes, self.cells, self._cells_by_id)
generalized = self._generalize_from_tree(x_pd, prepared, nodes, self.cells, self._cells_by_id)
else:
mapped = np.zeros(x.shape[0]) # to mark records we already mapped
mapped = np.zeros(x_pd.shape[0]) # to mark records we already mapped
all_indexes = []
for cell in self.cells:
indexes = self._get_record_indexes_for_cell(x, cell, mapped)
indexes = self._get_record_indexes_for_cell(x_pd, cell, mapped)
all_indexes.append(indexes)
generalized = self._generalize_indexes(x, self.cells, all_indexes)
generalized = self._generalize_indexes(x_pd, self.cells, all_indexes)
if dataset and dataset.is_pandas:
return generalized
elif isinstance(X, pd.DataFrame):
elif isinstance(x, pd.DataFrame):
return generalized
return generalized.to_numpy()
@ -584,39 +584,37 @@ class GeneralizeToRepresentative(BaseEstimator, MetaEstimatorMixin, TransformerM
feature_data[feature] = fd
return feature_data
def _get_record_indexes_for_cell(self, X, cell, mapped):
def _get_record_indexes_for_cell(self, x, cell, mapped):
indexes = []
for index, row in X.iterrows():
for index, row in x.iterrows():
if not mapped.item(index) and self._cell_contains(cell, row, index, mapped):
indexes.append(index)
return indexes
def _get_record_count_for_cell(self, X, cell, mapped):
def _get_record_count_for_cell(self, x, cell, mapped):
count = 0
index = 0
for _, row in X.iterrows():
for index, (_, row) in enumerate(x.iterrows()):
if not mapped.item(index) and self._cell_contains(cell, row, index, mapped):
count += 1
index += 1
return count
def _cell_contains(self, cell, x, index, mapped):
for i, f in enumerate(self._features):
if f in cell['ranges']:
if not self._cell_contains_numeric(i, cell['ranges'][f], x):
def _cell_contains(self, cell, row, index, mapped):
for i, feature in enumerate(self._features):
if feature in cell['ranges']:
if not self._cell_contains_numeric(i, cell['ranges'][feature], row):
return False
elif f in cell['categories']:
if not self._cell_contains_categorical(i, cell['categories'][f], x):
elif feature in cell['categories']:
if not self._cell_contains_categorical(i, cell['categories'][feature], row):
return False
elif f in cell['untouched']:
elif feature in cell['untouched']:
continue
else:
raise TypeError("feature " + f + "not found in cell" + cell['id'])
raise TypeError("feature " + feature + "not found in cell" + cell['id'])
# Mark as mapped
mapped.itemset(index, 1)
return True
def _encode_categorical_features(self, X, save_mapping=False):
def _encode_categorical_features(self, x, save_mapping=False):
if save_mapping:
self._categorical_values = {}
self._one_hot_vector_features_to_features = {}
@ -627,31 +625,31 @@ class GeneralizeToRepresentative(BaseEstimator, MetaEstimatorMixin, TransformerM
for feature in self.categorical_features:
if feature in used_features:
try:
all_values = X.loc[:, feature]
all_values = x.loc[:, feature]
values = list(all_values.unique())
if save_mapping:
self._categorical_values[feature] = values
X[feature] = pd.Categorical(X.loc[:, feature], categories=self._categorical_values[feature],
x[feature] = pd.Categorical(x.loc[:, feature], categories=self._categorical_values[feature],
ordered=False)
ohe = pd.get_dummies(X[feature], prefix=feature)
ohe = pd.get_dummies(x[feature], prefix=feature)
if save_mapping:
for one_hot_vector_feature in ohe.columns:
self._one_hot_vector_features_to_features[one_hot_vector_feature] = feature
X = pd.concat([X, ohe], axis=1)
x = pd.concat([x, ohe], axis=1)
features_to_remove.append(feature)
except KeyError:
print("feature " + feature + "not found in training data")
new_data = X.drop(features_to_remove, axis=1)
new_data = x.drop(features_to_remove, axis=1)
if save_mapping:
self._encoded_features = new_data.columns
return new_data
@staticmethod
def _cell_contains_numeric(i, range, x):
# convert x to ndarray to allow indexing
a = np.array(x)
value = a.item(i)
def _cell_contains_numeric(index, range, row):
# convert row to ndarray to allow indexing
a = np.array(row)
value = a.item(index)
if range['start']:
if value <= range['start']:
return False
@ -661,10 +659,10 @@ class GeneralizeToRepresentative(BaseEstimator, MetaEstimatorMixin, TransformerM
return True
@staticmethod
def _cell_contains_categorical(i, range, x):
# convert x to ndarray to allow indexing
a = np.array(x)
value = a.item(i)
def _cell_contains_categorical(index, range, row):
# convert row to ndarray to allow indexing
a = np.array(row)
value = a.item(index)
if value in range:
return True
return False
@ -863,14 +861,14 @@ class GeneralizeToRepresentative(BaseEstimator, MetaEstimatorMixin, TransformerM
generalizations['categories'])
original_data_generalized = pd.DataFrame(original_data, columns=self._features, copy=True)
for feature in self._generalizations['categories']:
if ('untouched' not in generalizations or feature not in generalizations['untouched']):
if 'untouched' not in generalizations or feature not in generalizations['untouched']:
for g_index, group in enumerate(generalizations['categories'][feature]):
indexes = [i for i, s in enumerate(sample_indexes) if s[feature] == g_index]
if indexes:
rows = original_data_generalized.iloc[indexes]
rows[feature] = generalizations['category_representatives'][feature][g_index]
for feature in self._generalizations['ranges']:
if ('untouched' not in generalizations or feature not in generalizations['untouched']):
if 'untouched' not in generalizations or feature not in generalizations['untouched']:
for r_index, range in enumerate(generalizations['ranges'][feature]):
indexes = [i for i, s in enumerate(sample_indexes) if s[feature] == r_index]
if indexes:
@ -941,16 +939,6 @@ class GeneralizeToRepresentative(BaseEstimator, MetaEstimatorMixin, TransformerM
if row[feature] in group:
sample_indexes[feature] = g_index
break
# found = False
# for g_index, group in enumerate(categories):
# sample_indexes[feature] = {}
# for c_index, category in enumerate(group):
# if row[feature] == group:
# sample_indexes[feature][g_index] = c_index
# found = True
# break
# if found:
# break
all_sample_indexes.append(sample_indexes)
return all_sample_indexes
@ -1087,20 +1075,12 @@ class GeneralizeToRepresentative(BaseEstimator, MetaEstimatorMixin, TransformerM
for feature in self._generalizations['categories']:
category_representatives[feature] = []
for g_index, group in enumerate(self._generalizations['categories'][feature]):
# max_count = 0
# for c_index in range(len(group)):
# indexes = [i for i, s in enumerate(sample_indexes) if s[feature][g_index] == c_index]
indexes = [i for i, s in enumerate(sample_indexes) if s[feature] == g_index]
if indexes:
rows = samples.iloc[indexes]
values = rows[feature]
category = Counter(values).most_common(1)[0][0]
category_representatives[feature].append(category)
# c_count = len([s for s in sample_indexes if s[feature][g_index] == c_index])
# if c_count > max_count:
# max_count = c_count
# category = c_index
# category_representatives[feature].append(group[category])
else:
category_representatives[feature].append(old_category_representatives[feature][g_index])

56
tests/test_minimizer.py
View file

@ -30,7 +30,7 @@ def diabetes_dataset():
@pytest.fixture
def get_cells():
def cells():
cells = [{"id": 1, "ranges": {"age": {"start": None, "end": 38}, "height": {"start": None, "end": 170}}, "label": 0,
'categories': {}, "representative": {"age": 26, "height": 149}},
{"id": 2, "ranges": {"age": {"start": 39, "end": None}, "height": {"start": None, "end": 170}}, "label": 1,
@ -49,7 +49,7 @@ def get_cells():
@pytest.fixture
def get_cells_categorical():
def cells_categorical():
cells = [{'id': 1, 'label': 0, 'ranges': {'age': {'start': None, 'end': None}},
'categories': {'sex': ['f', 'm']}, 'hist': [2, 0],
'representative': {'age': 45, 'height': 149, 'sex': 'f'},
@ -80,7 +80,7 @@ def get_cells_categorical():
@pytest.fixture
def get_data_two_features():
def data_two_features():
x = np.array([[23, 165],
[45, 158],
[56, 123],
@ -104,7 +104,7 @@ def get_data_two_features():
@pytest.fixture
def get_data_three_features():
def data_three_features():
features = ['age', 'height', 'weight']
x = np.array([[23, 165, 70],
[45, 158, 67],
@ -122,7 +122,7 @@ def get_data_three_features():
@pytest.fixture
def get_data_four_features():
def data_four_features():
features = ['age', 'height', 'sex', 'ola']
x = [[23, 165, 'f', 'aa'],
[45, 158, 'f', 'aa'],
@ -146,7 +146,7 @@ def get_data_four_features():
@pytest.fixture
def get_data_five_features():
def data_five_features():
features = ['age', 'height', 'weight', 'sex', 'ola']
x = [[23, 165, 65, 'f', 'aa'],
[45, 158, 76, 'f', 'aa'],
@ -205,9 +205,9 @@ def check_ncp(ncp, expected_generalizations):
assert (ncp > 0.0)
def test_minimizer_params(get_cells):
def test_minimizer_params(cells):
# Assume two features, age and height, and boolean label
cells, features, x, y = get_cells
cells, features, x, y = cells
base_est = DecisionTreeClassifier(random_state=0, min_samples_split=2,
min_samples_leaf=1)
@ -247,9 +247,9 @@ def create_encoder(numeric_features, categorical_features, x):
return preprocessor, encoded
def test_minimizer_params_not_transform(get_cells):
def test_minimizer_params_not_transform(cells):
# Assume two features, age and height, and boolean label
cells, features, x, y = get_cells
cells, features, x, y = cells
samples = ArrayDataset(x, y, features)
base_est = DecisionTreeClassifier(random_state=0, min_samples_split=2,
min_samples_leaf=1)
@ -261,8 +261,8 @@ def test_minimizer_params_not_transform(get_cells):
assert (ncp > 0.0)
def test_minimizer_fit(get_data_two_features):
x, y, features, _ = get_data_two_features
def test_minimizer_fit(data_two_features):
x, y, features, _ = data_two_features
base_est = DecisionTreeClassifier(random_state=0, min_samples_split=2,
min_samples_leaf=1)
model = SklearnClassifier(base_est, ModelOutputType.CLASSIFIER_PROBABILITIES)
@ -289,8 +289,8 @@ def test_minimizer_fit(get_data_two_features):
assert ((rel_accuracy >= target_accuracy) or (target_accuracy - rel_accuracy) <= 0.05)
def test_minimizer_ncp(get_data_two_features):
x, y, features, x1 = get_data_two_features
def test_minimizer_ncp(data_two_features):
x, y, features, x1 = data_two_features
base_est = DecisionTreeClassifier(random_state=0, min_samples_split=2,
min_samples_leaf=1)
@ -326,8 +326,8 @@ def test_minimizer_ncp(get_data_two_features):
assert (ncp6 == ncp4)
def test_minimizer_ncp_categorical(get_data_four_features):
x, y, features, x1 = get_data_four_features
def test_minimizer_ncp_categorical(data_four_features):
x, y, features, x1 = data_four_features
x = pd.DataFrame(x, columns=features)
x1 = pd.DataFrame(x1, columns=features)
@ -370,8 +370,8 @@ def test_minimizer_ncp_categorical(get_data_four_features):
assert (ncp6 == ncp4)
def test_minimizer_fit_not_transform(get_data_two_features):
x, y, features, x1 = get_data_two_features
def test_minimizer_fit_not_transform(data_two_features):
x, y, features, x1 = data_two_features
base_est = DecisionTreeClassifier(random_state=0, min_samples_split=2,
min_samples_leaf=1)
model = SklearnClassifier(base_est, ModelOutputType.CLASSIFIER_PROBABILITIES)
@ -394,8 +394,8 @@ def test_minimizer_fit_not_transform(get_data_two_features):
check_ncp(ncp, expected_generalizations)
def test_minimizer_fit_pandas(get_data_four_features):
x, y, features, _ = get_data_four_features
def test_minimizer_fit_pandas(data_four_features):
x, y, features, _ = data_four_features
x = pd.DataFrame(x, columns=features)
numeric_features = ["age", "height"]
@ -431,9 +431,9 @@ def test_minimizer_fit_pandas(get_data_four_features):
assert ((rel_accuracy >= target_accuracy) or (target_accuracy - rel_accuracy) <= 0.05)
def test_minimizer_params_categorical(get_cells_categorical):
def test_minimizer_params_categorical(cells_categorical):
# Assume three features, age, sex and height, and boolean label
cells, features, x, y = get_cells_categorical
cells, features, x, y = cells_categorical
x = pd.DataFrame(x, columns=features)
numeric_features = ["age", "height"]
@ -459,8 +459,8 @@ def test_minimizer_params_categorical(get_cells_categorical):
assert ((rel_accuracy >= target_accuracy) or (target_accuracy - rel_accuracy) <= 0.05)
def test_minimizer_fit_qi(get_data_three_features):
x, y, features = get_data_three_features
def test_minimizer_fit_qi(data_three_features):
x, y, features = data_three_features
qi = ['age', 'weight']
base_est = DecisionTreeClassifier(random_state=0, min_samples_split=2,
min_samples_leaf=1)
@ -487,8 +487,8 @@ def test_minimizer_fit_qi(get_data_three_features):
assert ((rel_accuracy >= target_accuracy) or (target_accuracy - rel_accuracy) <= 0.05)
def test_minimizer_fit_pandas_qi(get_data_five_features):
x, y, features = get_data_five_features
def test_minimizer_fit_pandas_qi(data_five_features):
x, y, features = data_five_features
x = pd.DataFrame(x, columns=features)
qi = ['age', 'weight', 'ola']
@ -809,8 +809,8 @@ def test_x_y_features_names():
assert ((rel_accuracy >= target_accuracy) or (target_accuracy - rel_accuracy) <= 0.05)
def test_BaseEstimator_classification(get_data_five_features):
x, y, features = get_data_five_features
def test_BaseEstimator_classification(data_five_features):
x, y, features = data_five_features
x = pd.DataFrame(x, columns=features)
QI = ['age', 'weight', 'ola']