diff --git a/apt/utils/datasets/datasets.py b/apt/utils/datasets/datasets.py index 02d886e..6569b37 100644 --- a/apt/utils/datasets/datasets.py +++ b/apt/utils/datasets/datasets.py @@ -400,6 +400,8 @@ class Data: :return: training samples """ + if self.train is None: + return None return self.train.get_samples() def get_train_labels(self) -> Collection[Any]: @@ -408,6 +410,8 @@ class Data: :return: training labels """ + if self.train is None: + return None return self.train.get_labels() def get_test_samples(self) -> Collection[Any]: diff --git a/apt/utils/models/model.py b/apt/utils/models/model.py index 7af3a01..bba743b 100644 --- a/apt/utils/models/model.py +++ b/apt/utils/models/model.py @@ -153,19 +153,36 @@ class BlackboxClassifier(Model): def __init__(self, model: Data, output_type: ModelOutputType, black_box_access: Optional[bool] = True, unlimited_queries: Optional[bool] = True, **kwargs): super().__init__(model, output_type, black_box_access=True, unlimited_queries=False, **kwargs) - x = model.get_train_samples() - y = model.get_train_labels() - self.nb_classes = self.get_nb_classes(y) - y = check_and_transform_label_format(y, nb_classes=self.nb_classes) + x_train_pred = model.get_train_samples() + y_train_pred = model.get_train_labels() + x_test_pred = model.get_test_samples() + y_test_pred = model.get_test_labels() - if model.get_test_samples() is not None and type(x) == np.ndarray: - x = np.vstack((x, model.get_test_samples())) + if x_train_pred is not None and y_train_pred is not None and x_test_pred is not None and y_test_pred is not None: + if type(y_train_pred) != np.ndarray or type(y_test_pred) != np.ndarray \ + or type(y_train_pred) != np.ndarray or type(y_test_pred) != np.ndarray: + raise NotImplementedError("X/Y Data should be np ndarray") - if model.get_test_labels() is not None and type(y) == np.ndarray: - y = np.vstack((y, check_and_transform_label_format(model.get_test_labels(), nb_classes=self.nb_classes))) + self.nb_classes = self.get_nb_classes(y_train_pred) + y_train_pred = check_and_transform_label_format(y_train_pred, nb_classes=self.nb_classes) + y_test_pred = check_and_transform_label_format(y_test_pred, nb_classes=self.nb_classes) + x_pred = np.vstack((x_train_pred, x_test_pred)) + y_pred = np.vstack((y_train_pred, y_test_pred)) + elif x_test_pred is not None and y_test_pred is not None: + self.nb_classes = self.get_nb_classes(y_test_pred) + y_test_pred = check_and_transform_label_format(y_test_pred, nb_classes=self.nb_classes) + x_pred = x_test_pred + y_pred = y_test_pred + elif x_train_pred is not None and y_train_pred is not None: + self.nb_classes = self.get_nb_classes(y_train_pred) + y_train_pred = check_and_transform_label_format(y_train_pred, nb_classes=self.nb_classes) + x_pred = x_train_pred + y_pred = y_train_pred + else: + raise NotImplementedError("Invalid data - None") - predict_fn = (x, y) - self._art_model = BlackBoxClassifier(predict_fn, x.shape[1:], self.nb_classes, fuzzy_float_compare=True) + predict_fn = (x_pred, y_pred) + self._art_model = BlackBoxClassifier(predict_fn, x_pred.shape[1:], self.nb_classes, fuzzy_float_compare=True) def fit(self, train_data: Dataset, **kwargs) -> None: """ diff --git a/tests/test_model.py b/tests/test_model.py index 4470658..3a5e45c 100644 --- a/tests/test_model.py +++ b/tests/test_model.py @@ -73,3 +73,75 @@ def test_blackbox_classifier(): score = model.score(test) assert(0.0 <= score <= 1.0) + +def test_blackbox_classifier_no_test(): + (x_train, y_train), (_, _) = dataset_utils.get_iris_dataset_np() + + train = ArrayDataset(x_train, y_train) + + data = Data(train) + model = BlackboxClassifier(data, ModelOutputType.CLASSIFIER_PROBABILITIES) + pred = model.predict(train) + assert(pred.shape[0] == x_train.shape[0]) + + score = model.score(train) + assert(0.0 <= score <= 1.0) + + +def test_blackbox_classifier_no_train(): + (_, _), (x_test, y_test) = dataset_utils.get_iris_dataset_np() + + test = ArrayDataset(x_test, y_test) + data = Data(test=test) + model = BlackboxClassifier(data, ModelOutputType.CLASSIFIER_PROBABILITIES) + pred = model.predict(test) + assert(pred.shape[0] == x_test.shape[0]) + + score = model.score(test) + assert(0.0 <= score <= 1.0) + + +def test_blackbox_classifier_no_test_y(): + (x_train, y_train), (x_test, _) = dataset_utils.get_iris_dataset_np() + + train = ArrayDataset(x_train, y_train) + test = ArrayDataset(x_test) + data = Data(train, test) + model = BlackboxClassifier(data, ModelOutputType.CLASSIFIER_PROBABILITIES) + pred = model.predict(train) + assert(pred.shape[0] == x_train.shape[0]) + + score = model.score(train) + assert(0.0 <= score <= 1.0) + + # since no test_y, BBC should use only test thus predict test should fail + unable_to_predict_test = False + try: + model.predict(test) + except BaseException: + unable_to_predict_test = True + + assert (unable_to_predict_test, True) + +def test_blackbox_classifier_no_train_y(): + (x_train, _), (x_test, y_test) = dataset_utils.get_iris_dataset_np() + + train = ArrayDataset(x_train) + test = ArrayDataset(x_test, y_test) + data = Data(train, test) + model = BlackboxClassifier(data, ModelOutputType.CLASSIFIER_PROBABILITIES) + pred = model.predict(test) + assert (pred.shape[0] == x_test.shape[0]) + + score = model.score(test) + assert (0.0 <= score <= 1.0) + + # since no train_y, BBC should use only test thus predict train should fail + unable_to_predict_train = False + try: + model.predict(train) + except BaseException: + unable_to_predict_train = True + + assert(unable_to_predict_train,True) +