BlackboxClassifier based on predictions to work with DatasetWithPredictions

apt/utils/datasets/datasets.py

@@ -477,7 +477,7 @@ class Data:
 
     def get_train_samples(self) -> Collection[Any]:
         """
-        Get train set samples
+        Get train set samples, or None if no training data provided
 
         :return: training samples
         """
@@ -487,7 +487,7 @@ class Data:
 
     def get_train_labels(self) -> Collection[Any]:
         """
-        Get train set labels
+        Get train set labels, or None if no training labels provided
 
         :return: training labels
         """
@@ -495,6 +495,16 @@ class Data:
             return None
         return self.train.get_labels()
 
+    def get_train_predictions(self) -> Collection[Any]:
+        """
+        Get train set predictions, or None if no training predictions provided
+
+        :return: training labels
+        """
+        if self.train is None:
+            return None
+        return self.train.get_predictions()
+
     def get_test_samples(self) -> Collection[Any]:
         """
         Get test set samples
@@ -509,8 +519,18 @@ class Data:
         """
         Get test set labels
 
-        :return: test labels, or None if no test data provided
+        :return: test labels, or None if no test labels provided
         """
         if self.test is None:
             return None
         return self.test.get_labels()
+
+    def get_test_predictions(self) -> Collection[Any]:
+        """
+        Get test set predictions, or None if no test predictions provided
+
+        :return: test labels
+        """
+        if self.test is None:
+            return None
+        return self.test.get_predictions()

apt/utils/models/model.py

@@ -246,6 +246,8 @@ class BlackboxClassifier(Model):
         :type scoring_method: `ScoringMethod`, optional
         :return: the score as float (for classifiers, between 0 and 1)
         """
+        if test_data.get_samples() is None or test_data.get_labels() is None:
+            raise ValueError('score can only be computed when test data and labels are available')
         predicted = self._art_model.predict(test_data.get_samples())
         y = check_and_transform_label_format(test_data.get_labels(), nb_classes=self._nb_classes)
         if scoring_method == ScoringMethod.ACCURACY:
@@ -276,9 +278,13 @@ class BlackboxClassifierPredictions(BlackboxClassifier):
                  unlimited_queries: Optional[bool] = True, **kwargs):
         super().__init__(model, output_type, black_box_access=True, unlimited_queries=False, **kwargs)
         x_train_pred = model.get_train_samples()
-        y_train_pred = model.get_train_labels()
+        y_train_pred = model.get_train_predictions()
+        if y_train_pred is None:
+            y_train_pred = model.get_train_labels()
         x_test_pred = model.get_test_samples()
-        y_test_pred = model.get_test_labels()
+        y_test_pred = model.get_test_predictions()
+        if y_test_pred is None:
+            y_test_pred = model.get_test_labels()
 
         if y_train_pred is not None:
             check_correct_model_output(y_train_pred, self.output_type)