From 256dfbbc71b5e3d0270dfe1f49f26017dfe1ac12 Mon Sep 17 00:00:00 2001 From: abigailt Date: Mon, 21 Aug 2023 16:40:52 +0300 Subject: [PATCH] Remove unused code, renaming and additional review comments Signed-off-by: abigailt --- apt/minimization/minimizer.py | 112 ++++++++++++++-------------------- tests/test_minimizer.py | 56 ++++++++--------- 2 files changed, 74 insertions(+), 94 deletions(-) diff --git a/apt/minimization/minimizer.py b/apt/minimization/minimizer.py index 75d7208..d9a1b14 100644 --- a/apt/minimization/minimizer.py +++ b/apt/minimization/minimizer.py @@ -400,9 +400,9 @@ class GeneralizeToRepresentative(BaseEstimator, MetaEstimatorMixin, TransformerM # self._cells currently holds the chosen generalization based on target accuracy # calculate iLoss - X_test_dataset = ArrayDataset(x_test, features_names=self._features) - self._ncp_scores.fit_score = self.calculate_ncp(X_test_dataset, generalize_using_transform) - self._ncp_scores.generalizations_score = self.calculate_ncp(X_test_dataset, False) + x_test_dataset = ArrayDataset(x_test, features_names=self._features) + self._ncp_scores.fit_score = self.calculate_ncp(x_test_dataset, generalize_using_transform) + self._ncp_scores.generalizations_score = self.calculate_ncp(x_test_dataset, False) # Return the transformer return self @@ -477,7 +477,7 @@ class GeneralizeToRepresentative(BaseEstimator, MetaEstimatorMixin, TransformerM return ncp - def _inner_transform(self, X: Optional[DATA_PANDAS_NUMPY_TYPE] = None, features_names: Optional[list] = None, + def _inner_transform(self, x: Optional[DATA_PANDAS_NUMPY_TYPE] = None, features_names: Optional[list] = None, dataset: Optional[ArrayDataset] = None): # Check if fit has been called msg = 'This %(name)s instance is not initialized yet. ' \ @@ -485,45 +485,45 @@ class GeneralizeToRepresentative(BaseEstimator, MetaEstimatorMixin, TransformerM 'appropriate arguments before using this method.' check_is_fitted(self, ['cells'], msg=msg) - if X is not None: + if x is not None: if dataset is not None: - raise ValueError('Either X OR dataset need to be provided, not both') + raise ValueError('Either x OR dataset need to be provided, not both') else: - dataset = ArrayDataset(X, features_names=features_names) + dataset = ArrayDataset(x, features_names=features_names) elif dataset is None: - raise ValueError('Either X OR dataset need to be provided, not both') + raise ValueError('Either x OR dataset need to be provided, not both') if dataset and dataset.features_names: if self._features is None: self._features = dataset.features_names if dataset and dataset.get_samples() is not None: - x = pd.DataFrame(dataset.get_samples(), columns=self._features) + x_pd = pd.DataFrame(dataset.get_samples(), columns=self._features) - if x.shape[1] != self._n_features and self._n_features != 0: + if x_pd.shape[1] != self._n_features and self._n_features != 0: raise ValueError('Shape of input is different from what was seen' 'in `fit`') if not self._features: - self._features = [i for i in range(x.shape[1])] + self._features = [i for i in range(x_pd.shape[1])] if self._dt: # only works if fit was called previously (but much more efficient) nodes = self._get_nodes_level(self._level) - QI = x.loc[:, self.features_to_minimize] - used_x = x + QI = x_pd.loc[:, self.features_to_minimize] + used_x = x_pd if self.train_only_features_to_minimize: used_x = QI prepared = self._encode_categorical_features(used_x) - generalized = self._generalize_from_tree(x, prepared, nodes, self.cells, self._cells_by_id) + generalized = self._generalize_from_tree(x_pd, prepared, nodes, self.cells, self._cells_by_id) else: - mapped = np.zeros(x.shape[0]) # to mark records we already mapped + mapped = np.zeros(x_pd.shape[0]) # to mark records we already mapped all_indexes = [] for cell in self.cells: - indexes = self._get_record_indexes_for_cell(x, cell, mapped) + indexes = self._get_record_indexes_for_cell(x_pd, cell, mapped) all_indexes.append(indexes) - generalized = self._generalize_indexes(x, self.cells, all_indexes) + generalized = self._generalize_indexes(x_pd, self.cells, all_indexes) if dataset and dataset.is_pandas: return generalized - elif isinstance(X, pd.DataFrame): + elif isinstance(x, pd.DataFrame): return generalized return generalized.to_numpy() @@ -584,39 +584,37 @@ class GeneralizeToRepresentative(BaseEstimator, MetaEstimatorMixin, TransformerM feature_data[feature] = fd return feature_data - def _get_record_indexes_for_cell(self, X, cell, mapped): + def _get_record_indexes_for_cell(self, x, cell, mapped): indexes = [] - for index, row in X.iterrows(): + for index, row in x.iterrows(): if not mapped.item(index) and self._cell_contains(cell, row, index, mapped): indexes.append(index) return indexes - def _get_record_count_for_cell(self, X, cell, mapped): + def _get_record_count_for_cell(self, x, cell, mapped): count = 0 - index = 0 - for _, row in X.iterrows(): + for index, (_, row) in enumerate(x.iterrows()): if not mapped.item(index) and self._cell_contains(cell, row, index, mapped): count += 1 - index += 1 return count - def _cell_contains(self, cell, x, index, mapped): - for i, f in enumerate(self._features): - if f in cell['ranges']: - if not self._cell_contains_numeric(i, cell['ranges'][f], x): + def _cell_contains(self, cell, row, index, mapped): + for i, feature in enumerate(self._features): + if feature in cell['ranges']: + if not self._cell_contains_numeric(i, cell['ranges'][feature], row): return False - elif f in cell['categories']: - if not self._cell_contains_categorical(i, cell['categories'][f], x): + elif feature in cell['categories']: + if not self._cell_contains_categorical(i, cell['categories'][feature], row): return False - elif f in cell['untouched']: + elif feature in cell['untouched']: continue else: - raise TypeError("feature " + f + "not found in cell" + cell['id']) + raise TypeError("feature " + feature + "not found in cell" + cell['id']) # Mark as mapped mapped.itemset(index, 1) return True - def _encode_categorical_features(self, X, save_mapping=False): + def _encode_categorical_features(self, x, save_mapping=False): if save_mapping: self._categorical_values = {} self._one_hot_vector_features_to_features = {} @@ -627,31 +625,31 @@ class GeneralizeToRepresentative(BaseEstimator, MetaEstimatorMixin, TransformerM for feature in self.categorical_features: if feature in used_features: try: - all_values = X.loc[:, feature] + all_values = x.loc[:, feature] values = list(all_values.unique()) if save_mapping: self._categorical_values[feature] = values - X[feature] = pd.Categorical(X.loc[:, feature], categories=self._categorical_values[feature], + x[feature] = pd.Categorical(x.loc[:, feature], categories=self._categorical_values[feature], ordered=False) - ohe = pd.get_dummies(X[feature], prefix=feature) + ohe = pd.get_dummies(x[feature], prefix=feature) if save_mapping: for one_hot_vector_feature in ohe.columns: self._one_hot_vector_features_to_features[one_hot_vector_feature] = feature - X = pd.concat([X, ohe], axis=1) + x = pd.concat([x, ohe], axis=1) features_to_remove.append(feature) except KeyError: print("feature " + feature + "not found in training data") - new_data = X.drop(features_to_remove, axis=1) + new_data = x.drop(features_to_remove, axis=1) if save_mapping: self._encoded_features = new_data.columns return new_data @staticmethod - def _cell_contains_numeric(i, range, x): - # convert x to ndarray to allow indexing - a = np.array(x) - value = a.item(i) + def _cell_contains_numeric(index, range, row): + # convert row to ndarray to allow indexing + a = np.array(row) + value = a.item(index) if range['start']: if value <= range['start']: return False @@ -661,10 +659,10 @@ class GeneralizeToRepresentative(BaseEstimator, MetaEstimatorMixin, TransformerM return True @staticmethod - def _cell_contains_categorical(i, range, x): - # convert x to ndarray to allow indexing - a = np.array(x) - value = a.item(i) + def _cell_contains_categorical(index, range, row): + # convert row to ndarray to allow indexing + a = np.array(row) + value = a.item(index) if value in range: return True return False @@ -863,14 +861,14 @@ class GeneralizeToRepresentative(BaseEstimator, MetaEstimatorMixin, TransformerM generalizations['categories']) original_data_generalized = pd.DataFrame(original_data, columns=self._features, copy=True) for feature in self._generalizations['categories']: - if ('untouched' not in generalizations or feature not in generalizations['untouched']): + if 'untouched' not in generalizations or feature not in generalizations['untouched']: for g_index, group in enumerate(generalizations['categories'][feature]): indexes = [i for i, s in enumerate(sample_indexes) if s[feature] == g_index] if indexes: rows = original_data_generalized.iloc[indexes] rows[feature] = generalizations['category_representatives'][feature][g_index] for feature in self._generalizations['ranges']: - if ('untouched' not in generalizations or feature not in generalizations['untouched']): + if 'untouched' not in generalizations or feature not in generalizations['untouched']: for r_index, range in enumerate(generalizations['ranges'][feature]): indexes = [i for i, s in enumerate(sample_indexes) if s[feature] == r_index] if indexes: @@ -941,16 +939,6 @@ class GeneralizeToRepresentative(BaseEstimator, MetaEstimatorMixin, TransformerM if row[feature] in group: sample_indexes[feature] = g_index break - # found = False - # for g_index, group in enumerate(categories): - # sample_indexes[feature] = {} - # for c_index, category in enumerate(group): - # if row[feature] == group: - # sample_indexes[feature][g_index] = c_index - # found = True - # break - # if found: - # break all_sample_indexes.append(sample_indexes) return all_sample_indexes @@ -1087,20 +1075,12 @@ class GeneralizeToRepresentative(BaseEstimator, MetaEstimatorMixin, TransformerM for feature in self._generalizations['categories']: category_representatives[feature] = [] for g_index, group in enumerate(self._generalizations['categories'][feature]): - # max_count = 0 - # for c_index in range(len(group)): - # indexes = [i for i, s in enumerate(sample_indexes) if s[feature][g_index] == c_index] indexes = [i for i, s in enumerate(sample_indexes) if s[feature] == g_index] if indexes: rows = samples.iloc[indexes] values = rows[feature] category = Counter(values).most_common(1)[0][0] category_representatives[feature].append(category) - # c_count = len([s for s in sample_indexes if s[feature][g_index] == c_index]) - # if c_count > max_count: - # max_count = c_count - # category = c_index - # category_representatives[feature].append(group[category]) else: category_representatives[feature].append(old_category_representatives[feature][g_index]) diff --git a/tests/test_minimizer.py b/tests/test_minimizer.py index 62b42a2..b4e44ba 100644 --- a/tests/test_minimizer.py +++ b/tests/test_minimizer.py @@ -30,7 +30,7 @@ def diabetes_dataset(): @pytest.fixture -def get_cells(): +def cells(): cells = [{"id": 1, "ranges": {"age": {"start": None, "end": 38}, "height": {"start": None, "end": 170}}, "label": 0, 'categories': {}, "representative": {"age": 26, "height": 149}}, {"id": 2, "ranges": {"age": {"start": 39, "end": None}, "height": {"start": None, "end": 170}}, "label": 1, @@ -49,7 +49,7 @@ def get_cells(): @pytest.fixture -def get_cells_categorical(): +def cells_categorical(): cells = [{'id': 1, 'label': 0, 'ranges': {'age': {'start': None, 'end': None}}, 'categories': {'sex': ['f', 'm']}, 'hist': [2, 0], 'representative': {'age': 45, 'height': 149, 'sex': 'f'}, @@ -80,7 +80,7 @@ def get_cells_categorical(): @pytest.fixture -def get_data_two_features(): +def data_two_features(): x = np.array([[23, 165], [45, 158], [56, 123], @@ -104,7 +104,7 @@ def get_data_two_features(): @pytest.fixture -def get_data_three_features(): +def data_three_features(): features = ['age', 'height', 'weight'] x = np.array([[23, 165, 70], [45, 158, 67], @@ -122,7 +122,7 @@ def get_data_three_features(): @pytest.fixture -def get_data_four_features(): +def data_four_features(): features = ['age', 'height', 'sex', 'ola'] x = [[23, 165, 'f', 'aa'], [45, 158, 'f', 'aa'], @@ -146,7 +146,7 @@ def get_data_four_features(): @pytest.fixture -def get_data_five_features(): +def data_five_features(): features = ['age', 'height', 'weight', 'sex', 'ola'] x = [[23, 165, 65, 'f', 'aa'], [45, 158, 76, 'f', 'aa'], @@ -205,9 +205,9 @@ def check_ncp(ncp, expected_generalizations): assert (ncp > 0.0) -def test_minimizer_params(get_cells): +def test_minimizer_params(cells): # Assume two features, age and height, and boolean label - cells, features, x, y = get_cells + cells, features, x, y = cells base_est = DecisionTreeClassifier(random_state=0, min_samples_split=2, min_samples_leaf=1) @@ -247,9 +247,9 @@ def create_encoder(numeric_features, categorical_features, x): return preprocessor, encoded -def test_minimizer_params_not_transform(get_cells): +def test_minimizer_params_not_transform(cells): # Assume two features, age and height, and boolean label - cells, features, x, y = get_cells + cells, features, x, y = cells samples = ArrayDataset(x, y, features) base_est = DecisionTreeClassifier(random_state=0, min_samples_split=2, min_samples_leaf=1) @@ -261,8 +261,8 @@ def test_minimizer_params_not_transform(get_cells): assert (ncp > 0.0) -def test_minimizer_fit(get_data_two_features): - x, y, features, _ = get_data_two_features +def test_minimizer_fit(data_two_features): + x, y, features, _ = data_two_features base_est = DecisionTreeClassifier(random_state=0, min_samples_split=2, min_samples_leaf=1) model = SklearnClassifier(base_est, ModelOutputType.CLASSIFIER_PROBABILITIES) @@ -289,8 +289,8 @@ def test_minimizer_fit(get_data_two_features): assert ((rel_accuracy >= target_accuracy) or (target_accuracy - rel_accuracy) <= 0.05) -def test_minimizer_ncp(get_data_two_features): - x, y, features, x1 = get_data_two_features +def test_minimizer_ncp(data_two_features): + x, y, features, x1 = data_two_features base_est = DecisionTreeClassifier(random_state=0, min_samples_split=2, min_samples_leaf=1) @@ -326,8 +326,8 @@ def test_minimizer_ncp(get_data_two_features): assert (ncp6 == ncp4) -def test_minimizer_ncp_categorical(get_data_four_features): - x, y, features, x1 = get_data_four_features +def test_minimizer_ncp_categorical(data_four_features): + x, y, features, x1 = data_four_features x = pd.DataFrame(x, columns=features) x1 = pd.DataFrame(x1, columns=features) @@ -370,8 +370,8 @@ def test_minimizer_ncp_categorical(get_data_four_features): assert (ncp6 == ncp4) -def test_minimizer_fit_not_transform(get_data_two_features): - x, y, features, x1 = get_data_two_features +def test_minimizer_fit_not_transform(data_two_features): + x, y, features, x1 = data_two_features base_est = DecisionTreeClassifier(random_state=0, min_samples_split=2, min_samples_leaf=1) model = SklearnClassifier(base_est, ModelOutputType.CLASSIFIER_PROBABILITIES) @@ -394,8 +394,8 @@ def test_minimizer_fit_not_transform(get_data_two_features): check_ncp(ncp, expected_generalizations) -def test_minimizer_fit_pandas(get_data_four_features): - x, y, features, _ = get_data_four_features +def test_minimizer_fit_pandas(data_four_features): + x, y, features, _ = data_four_features x = pd.DataFrame(x, columns=features) numeric_features = ["age", "height"] @@ -431,9 +431,9 @@ def test_minimizer_fit_pandas(get_data_four_features): assert ((rel_accuracy >= target_accuracy) or (target_accuracy - rel_accuracy) <= 0.05) -def test_minimizer_params_categorical(get_cells_categorical): +def test_minimizer_params_categorical(cells_categorical): # Assume three features, age, sex and height, and boolean label - cells, features, x, y = get_cells_categorical + cells, features, x, y = cells_categorical x = pd.DataFrame(x, columns=features) numeric_features = ["age", "height"] @@ -459,8 +459,8 @@ def test_minimizer_params_categorical(get_cells_categorical): assert ((rel_accuracy >= target_accuracy) or (target_accuracy - rel_accuracy) <= 0.05) -def test_minimizer_fit_qi(get_data_three_features): - x, y, features = get_data_three_features +def test_minimizer_fit_qi(data_three_features): + x, y, features = data_three_features qi = ['age', 'weight'] base_est = DecisionTreeClassifier(random_state=0, min_samples_split=2, min_samples_leaf=1) @@ -487,8 +487,8 @@ def test_minimizer_fit_qi(get_data_three_features): assert ((rel_accuracy >= target_accuracy) or (target_accuracy - rel_accuracy) <= 0.05) -def test_minimizer_fit_pandas_qi(get_data_five_features): - x, y, features = get_data_five_features +def test_minimizer_fit_pandas_qi(data_five_features): + x, y, features = data_five_features x = pd.DataFrame(x, columns=features) qi = ['age', 'weight', 'ola'] @@ -809,8 +809,8 @@ def test_x_y_features_names(): assert ((rel_accuracy >= target_accuracy) or (target_accuracy - rel_accuracy) <= 0.05) -def test_BaseEstimator_classification(get_data_five_features): - x, y, features = get_data_five_features +def test_BaseEstimator_classification(data_five_features): + x, y, features = data_five_features x = pd.DataFrame(x, columns=features) QI = ['age', 'weight', 'ola']