From 8b8b4611430fe5855b269f37ee62f74221991056 Mon Sep 17 00:00:00 2001 From: abigailt Date: Sun, 17 Mar 2024 11:49:05 +0200 Subject: [PATCH] Support for multi-label logits/probabilities Signed-off-by: abigailt --- apt/utils/models/model.py | 13 +++ apt/utils/models/pytorch_model.py | 22 ++--- tests/test_pytorch.py | 135 +++++++++++++++--------------- 3 files changed, 94 insertions(+), 76 deletions(-) diff --git a/apt/utils/models/model.py b/apt/utils/models/model.py index 6d75fef..fc48633 100644 --- a/apt/utils/models/model.py +++ b/apt/utils/models/model.py @@ -212,6 +212,19 @@ class Model(metaclass=ABCMeta): self.output_type == ModelOutputType.CLASSIFIER_SINGLE_OUTPUT_CATEGORICAL): # categorical has been 1-hot encoded by check_and_transform_label_format return np.count_nonzero(np.argmax(y, axis=1) == np.argmax(predicted, axis=1)) / predicted.shape[0] + elif (self.output_type == ModelOutputType.CLASSIFIER_MULTI_OUTPUT_CLASS_LOGITS or + self.output_type == ModelOutputType.CLASSIFIER_SINGLE_OUTPUT_CLASS_PROBABILITIES): + if predicted.shape != y.shape: + raise ValueError('Do not know how to compare arrays with different shapes') + elif len(predicted.shape) < 3: + raise ValueError('Do not know how to compare 2-D arrays for multi-output non-binary case') + else: + sum = 0 + count = 0 + for i in range(predicted.shape[1]): + count += np.count_nonzero(np.argmax(y[:, i], axis=1) == np.argmax(predicted[:, i], axis=1)) + sum += predicted.shape[0] * predicted.shape[-1] + return count / sum elif is_binary(self.output_type): if is_logits(self.output_type): if apply_non_linearity: diff --git a/apt/utils/models/pytorch_model.py b/apt/utils/models/pytorch_model.py index 2633af3..723619b 100644 --- a/apt/utils/models/pytorch_model.py +++ b/apt/utils/models/pytorch_model.py @@ -3,16 +3,15 @@ import os import shutil import logging -from typing import Optional, Tuple, Union, List, Callable +from typing import Optional, Tuple, Union, List import numpy as np import torch from torch.utils.data import DataLoader, TensorDataset -from collections.abc import Iterable from art.utils import check_and_transform_label_format from apt.utils.datasets.datasets import PytorchData, DatasetWithPredictions, ArrayDataset from apt.utils.models import Model, ModelOutputType, is_multi_label, is_multi_label_binary -from apt.utils.datasets import OUTPUT_DATA_ARRAY_TYPE +from apt.utils.datasets import OUTPUT_DATA_ARRAY_TYPE, array2numpy from art.estimators.classification.pytorch import PyTorchClassifier as ArtPyTorchClassifier @@ -222,17 +221,20 @@ class PyTorchClassifierWrapper(ArtPyTorchClassifier): with torch.no_grad(): model_outputs = self._model(torch.from_numpy(x_preprocessed[begin:end]).to(self._device)) output = model_outputs[-1] - if isinstance(output, Iterable): - for i, o in enumerate(output): + + if isinstance(output, tuple): + output_list = [] + for o in output: o = o.detach().cpu().numpy().astype(np.float32) - # if len(output.shape) == 1: - # o = np.expand_dims(o, axis=1).astype(np.float32) + output_list.append(o) + output_np = np.array(output_list) + output_np = np.swapaxes(output_np, 0, 1) + results_list.append(output_np) else: output = output.detach().cpu().numpy().astype(np.float32) if len(output.shape) == 1: output = np.expand_dims(output, axis=1).astype(np.float32) - - results_list.append(output) + results_list.append(output) results = np.vstack(results_list) @@ -484,7 +486,7 @@ class PyTorchClassifier(PyTorchModel): :type x: `np.ndarray` or `pandas.DataFrame` :return: Predictions from the model (class probabilities, if supported). """ - return self._art_model.predict(x.get_samples(), **kwargs) + return array2numpy(self._art_model.predict(x.get_samples(), **kwargs)) def score(self, test_data: PytorchData, **kwargs): """ diff --git a/tests/test_pytorch.py b/tests/test_pytorch.py index 65f64dd..d2ada1f 100644 --- a/tests/test_pytorch.py +++ b/tests/test_pytorch.py @@ -4,6 +4,7 @@ from torch.nn import functional from torch.utils.data import DataLoader, TensorDataset from scipy.special import expit +from art.utils import check_and_transform_label_format from apt.utils.datasets.datasets import PytorchData from apt.utils.models import ModelOutputType from apt.utils.models.pytorch_model import PyTorchClassifier @@ -106,72 +107,74 @@ def test_pytorch_nursery_save_entire_model(): assert (0 <= score <= 1) -# def test_pytorch_predictions_multi_label_cat(): -# # This kind of model requires special training and will not be supported using the 'fit' method. -# class multi_label_cat_model(nn.Module): -# -# def __init__(self, num_classes, num_features): -# super(multi_label_cat_model, self).__init__() -# -# self.fc1 = nn.Sequential( -# nn.Linear(num_features, 256), -# nn.Tanh(), ) -# -# self.classifier1 = nn.Linear(256, num_classes) -# self.classifier2 = nn.Linear(256, num_classes) -# self.classifier3 = nn.Linear(256, num_classes) -# -# def forward(self, x): -# out1 = self.classifier1(self.fc1(x)) -# out2 = self.classifier2(self.fc1(x)) -# out3 = self.classifier3(self.fc1(x)) -# return out1, out2, out3 -# -# (x_train, y_train), (x_test, y_test) = dataset_utils.get_iris_dataset_np() -# -# # make multi-label categorical -# y_train = np.column_stack((y_train, y_train, y_train)) -# y_test = np.column_stack((y_test, y_test, y_test)) -# test = PytorchData(x_test.astype(np.float32), y_test.astype(np.float32)) -# -# model = multi_label_cat_model(3, 4) -# criterion = nn.CrossEntropyLoss() -# optimizer = optim.Adam(model.parameters(), lr=0.01) -# -# # train model -# train_dataset = TensorDataset(from_numpy(x_train.astype(np.float32)), from_numpy(y_train.astype(np.float32))) -# train_loader = DataLoader(train_dataset, batch_size=100, shuffle=True) -# -# for epoch in range(5): -# # Train for one epoch -# for inputs, targets in train_loader: -# # Zero the parameter gradients -# optimizer.zero_grad() -# -# # Perform prediction -# model_outputs = model(inputs)[-1] -# -# # Form the loss function -# loss = 0 -# for i, o in enumerate(model_outputs): -# loss += criterion(o, targets[i]) -# -# loss.backward() -# -# optimizer.step() -# -# art_model = PyTorchClassifier(model=model, -# output_type=ModelOutputType.CLASSIFIER_MULTI_OUTPUT_CLASS_LOGITS, -# loss=criterion, -# optimizer=optimizer, -# input_shape=(24,), -# nb_classes=3) -# -# pred = art_model.predict(test) -# assert (pred.shape[0] == x_test.shape[0]) -# -# score = art_model.score(test, apply_non_linearity=expit) -# assert (score == 1.0) +def test_pytorch_predictions_multi_label_cat(): + # This kind of model requires special training and will not be supported using the 'fit' method. + class multi_label_cat_model(nn.Module): + + def __init__(self, num_classes, num_features): + super(multi_label_cat_model, self).__init__() + + self.fc1 = nn.Sequential( + nn.Linear(num_features, 256), + nn.Tanh(), ) + + self.classifier1 = nn.Linear(256, num_classes) + self.classifier2 = nn.Linear(256, num_classes) + + def forward(self, x): + out1 = self.classifier1(self.fc1(x)) + out2 = self.classifier2(self.fc1(x)) + return out1, out2 + + (x_train, y_train), (x_test, y_test) = dataset_utils.get_iris_dataset_np() + + # make multi-label categorical + num_classes = 3 + y_train = check_and_transform_label_format(y_train, nb_classes=num_classes) + y_test = check_and_transform_label_format(y_test, nb_classes=num_classes) + y_train = np.column_stack((y_train, y_train)) + y_test = np.stack([y_test, y_test], axis=1) + test = PytorchData(x_test.astype(np.float32), y_test.astype(np.float32)) + + model = multi_label_cat_model(num_classes, 4) + criterion = nn.CrossEntropyLoss() + optimizer = optim.Adam(model.parameters(), lr=0.01) + + # train model + train_dataset = TensorDataset(from_numpy(x_train.astype(np.float32)), from_numpy(y_train.astype(np.float32))) + train_loader = DataLoader(train_dataset, batch_size=100, shuffle=True) + + for epoch in range(5): + # Train for one epoch + for inputs, targets in train_loader: + # Zero the parameter gradients + optimizer.zero_grad() + + # Perform prediction + model_outputs = model(inputs) + + # Form the loss function + loss = 0 + for i, o in enumerate(model_outputs): + t = targets[:, i*num_classes:(i+1)*num_classes] + loss += criterion(o, t) + + loss.backward() + + optimizer.step() + + art_model = PyTorchClassifier(model=model, + output_type=ModelOutputType.CLASSIFIER_MULTI_OUTPUT_CLASS_LOGITS, + loss=criterion, + optimizer=optimizer, + input_shape=(24,), + nb_classes=3) + + pred = art_model.predict(test) + assert (pred.shape[0] == x_test.shape[0]) + + score = art_model.score(test, apply_non_linearity=expit) + assert (0 < score <= 1.0) def test_pytorch_predictions_multi_label_binary():