BlackboxClassifier based on predictions to work with DatasetWithPredictions

apt/utils/datasets/datasets.py

@@ -477,7 +477,7 @@ class Data:
 
     def get_train_samples(self) -> Collection[Any]:
         """
-        Get train set samples
+        Get train set samples, or None if no training data provided
 
         :return: training samples
         """
@@ -487,7 +487,7 @@ class Data:
 
     def get_train_labels(self) -> Collection[Any]:
         """
-        Get train set labels
+        Get train set labels, or None if no training labels provided
 
         :return: training labels
         """
@@ -495,6 +495,16 @@ class Data:
             return None
         return self.train.get_labels()
 
+    def get_train_predictions(self) -> Collection[Any]:
+        """
+        Get train set predictions, or None if no training predictions provided
+
+        :return: training labels
+        """
+        if self.train is None:
+            return None
+        return self.train.get_predictions()
+
     def get_test_samples(self) -> Collection[Any]:
         """
         Get test set samples
@@ -509,8 +519,18 @@ class Data:
         """
         Get test set labels
 
-        :return: test labels, or None if no test data provided
+        :return: test labels, or None if no test labels provided
         """
         if self.test is None:
             return None
         return self.test.get_labels()
+
+    def get_test_predictions(self) -> Collection[Any]:
+        """
+        Get test set predictions, or None if no test predictions provided
+
+        :return: test labels
+        """
+        if self.test is None:
+            return None
+        return self.test.get_predictions()

apt/utils/models/model.py

@@ -246,6 +246,8 @@ class BlackboxClassifier(Model):
         :type scoring_method: `ScoringMethod`, optional
         :return: the score as float (for classifiers, between 0 and 1)
         """
+        if test_data.get_samples() is None or test_data.get_labels() is None:
+            raise ValueError('score can only be computed when test data and labels are available')
         predicted = self._art_model.predict(test_data.get_samples())
         y = check_and_transform_label_format(test_data.get_labels(), nb_classes=self._nb_classes)
         if scoring_method == ScoringMethod.ACCURACY:
@@ -276,9 +278,13 @@ class BlackboxClassifierPredictions(BlackboxClassifier):
                  unlimited_queries: Optional[bool] = True, **kwargs):
         super().__init__(model, output_type, black_box_access=True, unlimited_queries=False, **kwargs)
         x_train_pred = model.get_train_samples()
-        y_train_pred = model.get_train_labels()
+        y_train_pred = model.get_train_predictions()
+        if y_train_pred is None:
+            y_train_pred = model.get_train_labels()
         x_test_pred = model.get_test_samples()
-        y_test_pred = model.get_test_labels()
+        y_test_pred = model.get_test_predictions()
+        if y_test_pred is None:
+            y_test_pred = model.get_test_labels()
 
         if y_train_pred is not None:
             check_correct_model_output(y_train_pred, self.output_type)

tests/test_model.py

@@ -3,7 +3,7 @@ import numpy as np
 
 from apt.utils.models import SklearnClassifier, SklearnRegressor, ModelOutputType, KerasClassifier, KerasRegressor, \
     BlackboxClassifierPredictions, BlackboxClassifierPredictFunction, is_one_hot, get_nb_classes
-from apt.utils.datasets import ArrayDataset, Data
+from apt.utils.datasets import ArrayDataset, Data, DatasetWithPredictions
 from apt.utils import dataset_utils
 
 from sklearn.tree import DecisionTreeRegressor
@@ -104,6 +104,37 @@ def test_blackbox_classifier():
     assert model.model_type is None
 
 
+def test_blackbox_classifier_predictions():
+    (x_train, y_train), (x_test, y_test) = dataset_utils.get_iris_dataset_np()
+
+    train = DatasetWithPredictions(y_train, x_train)
+    test = DatasetWithPredictions(y_test, x_test)
+    data = Data(train, test)
+    model = BlackboxClassifierPredictions(data, ModelOutputType.CLASSIFIER_SCALAR)
+    pred = model.predict(test)
+    assert(pred.shape[0] == x_test.shape[0])
+    assert model.model_type is None
+
+    with pytest.raises(ValueError):
+        model.score(test)
+
+
+def test_blackbox_classifier_predictions_y():
+    (x_train, y_train), (x_test, y_test) = dataset_utils.get_iris_dataset_np()
+
+    train = DatasetWithPredictions(y_train, x_train, y_train)
+    test = DatasetWithPredictions(y_test, x_test, y_test)
+    data = Data(train, test)
+    model = BlackboxClassifierPredictions(data, ModelOutputType.CLASSIFIER_SCALAR)
+    pred = model.predict(test)
+    assert(pred.shape[0] == x_test.shape[0])
+
+    score = model.score(test)
+    assert(score == 1.0)
+
+    assert model.model_type is None
+
+
 def test_blackbox_classifier_mismatch():
     (x_train, y_train), (x_test, y_test) = dataset_utils.get_iris_dataset_np()