diff --git a/apt/utils/datasets/datasets.py b/apt/utils/datasets/datasets.py index 0ebc998..7eae95f 100644 --- a/apt/utils/datasets/datasets.py +++ b/apt/utils/datasets/datasets.py @@ -477,7 +477,7 @@ class Data: def get_train_samples(self) -> Collection[Any]: """ - Get train set samples + Get train set samples, or None if no training data provided :return: training samples """ @@ -487,7 +487,7 @@ class Data: def get_train_labels(self) -> Collection[Any]: """ - Get train set labels + Get train set labels, or None if no training labels provided :return: training labels """ @@ -495,6 +495,16 @@ class Data: return None return self.train.get_labels() + def get_train_predictions(self) -> Collection[Any]: + """ + Get train set predictions, or None if no training predictions provided + + :return: training labels + """ + if self.train is None: + return None + return self.train.get_predictions() + def get_test_samples(self) -> Collection[Any]: """ Get test set samples @@ -509,8 +519,18 @@ class Data: """ Get test set labels - :return: test labels, or None if no test data provided + :return: test labels, or None if no test labels provided """ if self.test is None: return None return self.test.get_labels() + + def get_test_predictions(self) -> Collection[Any]: + """ + Get test set predictions, or None if no test predictions provided + + :return: test labels + """ + if self.test is None: + return None + return self.test.get_predictions() diff --git a/apt/utils/models/model.py b/apt/utils/models/model.py index e81bc89..d9811e1 100644 --- a/apt/utils/models/model.py +++ b/apt/utils/models/model.py @@ -246,6 +246,8 @@ class BlackboxClassifier(Model): :type scoring_method: `ScoringMethod`, optional :return: the score as float (for classifiers, between 0 and 1) """ + if test_data.get_samples() is None or test_data.get_labels() is None: + raise ValueError('score can only be computed when test data and labels are available') predicted = self._art_model.predict(test_data.get_samples()) y = check_and_transform_label_format(test_data.get_labels(), nb_classes=self._nb_classes) if scoring_method == ScoringMethod.ACCURACY: @@ -276,9 +278,13 @@ class BlackboxClassifierPredictions(BlackboxClassifier): unlimited_queries: Optional[bool] = True, **kwargs): super().__init__(model, output_type, black_box_access=True, unlimited_queries=False, **kwargs) x_train_pred = model.get_train_samples() - y_train_pred = model.get_train_labels() + y_train_pred = model.get_train_predictions() + if y_train_pred is None: + y_train_pred = model.get_train_labels() x_test_pred = model.get_test_samples() - y_test_pred = model.get_test_labels() + y_test_pred = model.get_test_predictions() + if y_test_pred is None: + y_test_pred = model.get_test_labels() if y_train_pred is not None: check_correct_model_output(y_train_pred, self.output_type) diff --git a/tests/test_model.py b/tests/test_model.py index d5f4216..821c776 100644 --- a/tests/test_model.py +++ b/tests/test_model.py @@ -3,7 +3,7 @@ import numpy as np from apt.utils.models import SklearnClassifier, SklearnRegressor, ModelOutputType, KerasClassifier, KerasRegressor, \ BlackboxClassifierPredictions, BlackboxClassifierPredictFunction, is_one_hot, get_nb_classes -from apt.utils.datasets import ArrayDataset, Data +from apt.utils.datasets import ArrayDataset, Data, DatasetWithPredictions from apt.utils import dataset_utils from sklearn.tree import DecisionTreeRegressor @@ -104,6 +104,37 @@ def test_blackbox_classifier(): assert model.model_type is None +def test_blackbox_classifier_predictions(): + (x_train, y_train), (x_test, y_test) = dataset_utils.get_iris_dataset_np() + + train = DatasetWithPredictions(y_train, x_train) + test = DatasetWithPredictions(y_test, x_test) + data = Data(train, test) + model = BlackboxClassifierPredictions(data, ModelOutputType.CLASSIFIER_SCALAR) + pred = model.predict(test) + assert(pred.shape[0] == x_test.shape[0]) + assert model.model_type is None + + with pytest.raises(ValueError): + model.score(test) + + +def test_blackbox_classifier_predictions_y(): + (x_train, y_train), (x_test, y_test) = dataset_utils.get_iris_dataset_np() + + train = DatasetWithPredictions(y_train, x_train, y_train) + test = DatasetWithPredictions(y_test, x_test, y_test) + data = Data(train, test) + model = BlackboxClassifierPredictions(data, ModelOutputType.CLASSIFIER_SCALAR) + pred = model.predict(test) + assert(pred.shape[0] == x_test.shape[0]) + + score = model.score(test) + assert(score == 1.0) + + assert model.model_type is None + + def test_blackbox_classifier_mismatch(): (x_train, y_train), (x_test, y_test) = dataset_utils.get_iris_dataset_np()