apunkt/ai-privacy-toolkit
1
0
Fork 0
mirror of https://github.com/IBM/ai-privacy-toolkit.git synced 2026-06-08 15:05:13 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

Support for multi-label logits/probabilities

Browse source 
Signed-off-by: abigailt <abigailt@il.ibm.com>
This commit is contained in:
abigailt 2024-03-17 11:49:05 +02:00
parent 7e34f0d2ff
commit 8b8b461143
3 changed files with 94 additions and 76 deletions
Download patch file Download diff file Expand all files Collapse all files

13
apt/utils/models/model.py
View file

@ -212,6 +212,19 @@ class Model(metaclass=ABCMeta):
self.output_type == ModelOutputType.CLASSIFIER_SINGLE_OUTPUT_CATEGORICAL):
# categorical has been 1-hot encoded by check_and_transform_label_format
return np.count_nonzero(np.argmax(y, axis=1) == np.argmax(predicted, axis=1)) / predicted.shape[0]
elif (self.output_type == ModelOutputType.CLASSIFIER_MULTI_OUTPUT_CLASS_LOGITS or
self.output_type == ModelOutputType.CLASSIFIER_SINGLE_OUTPUT_CLASS_PROBABILITIES):
if predicted.shape != y.shape:
raise ValueError('Do not know how to compare arrays with different shapes')
elif len(predicted.shape) < 3:
raise ValueError('Do not know how to compare 2-D arrays for multi-output non-binary case')
else:
sum = 0
count = 0
for i in range(predicted.shape[1]):
count += np.count_nonzero(np.argmax(y[:, i], axis=1) == np.argmax(predicted[:, i], axis=1))
sum += predicted.shape[0] * predicted.shape[-1]
return count / sum
elif is_binary(self.output_type):
if is_logits(self.output_type):
if apply_non_linearity:

22
apt/utils/models/pytorch_model.py
View file

@ -3,16 +3,15 @@ import os
import shutil
import logging
from typing import Optional, Tuple, Union, List, Callable
from typing import Optional, Tuple, Union, List
import numpy as np
import torch
from torch.utils.data import DataLoader, TensorDataset
from collections.abc import Iterable
from art.utils import check_and_transform_label_format
from apt.utils.datasets.datasets import PytorchData, DatasetWithPredictions, ArrayDataset
from apt.utils.models import Model, ModelOutputType, is_multi_label, is_multi_label_binary
from apt.utils.datasets import OUTPUT_DATA_ARRAY_TYPE
from apt.utils.datasets import OUTPUT_DATA_ARRAY_TYPE, array2numpy
from art.estimators.classification.pytorch import PyTorchClassifier as ArtPyTorchClassifier
@ -222,17 +221,20 @@ class PyTorchClassifierWrapper(ArtPyTorchClassifier):
with torch.no_grad():
model_outputs = self._model(torch.from_numpy(x_preprocessed[begin:end]).to(self._device))
output = model_outputs[-1]
if isinstance(output, Iterable):
for i, o in enumerate(output):
if isinstance(output, tuple):
output_list = []
for o in output:
o = o.detach().cpu().numpy().astype(np.float32)
# if len(output.shape) == 1:
# o = np.expand_dims(o, axis=1).astype(np.float32)
output_list.append(o)
output_np = np.array(output_list)
output_np = np.swapaxes(output_np, 0, 1)
results_list.append(output_np)
else:
output = output.detach().cpu().numpy().astype(np.float32)
if len(output.shape) == 1:
output = np.expand_dims(output, axis=1).astype(np.float32)
results_list.append(output)
results_list.append(output)
results = np.vstack(results_list)
@ -484,7 +486,7 @@ class PyTorchClassifier(PyTorchModel):
:type x: `np.ndarray` or `pandas.DataFrame`
:return: Predictions from the model (class probabilities, if supported).
"""
return self._art_model.predict(x.get_samples(), **kwargs)
return array2numpy(self._art_model.predict(x.get_samples(), **kwargs))
def score(self, test_data: PytorchData, **kwargs):
"""

135
tests/test_pytorch.py
View file

@ -4,6 +4,7 @@ from torch.nn import functional
from torch.utils.data import DataLoader, TensorDataset
from scipy.special import expit
from art.utils import check_and_transform_label_format
from apt.utils.datasets.datasets import PytorchData
from apt.utils.models import ModelOutputType
from apt.utils.models.pytorch_model import PyTorchClassifier
@ -106,72 +107,74 @@ def test_pytorch_nursery_save_entire_model():
assert (0 <= score <= 1)
# def test_pytorch_predictions_multi_label_cat():
# # This kind of model requires special training and will not be supported using the 'fit' method.
# class multi_label_cat_model(nn.Module):
#
# def __init__(self, num_classes, num_features):
# super(multi_label_cat_model, self).__init__()
#
# self.fc1 = nn.Sequential(
# nn.Linear(num_features, 256),
# nn.Tanh(), )
#
# self.classifier1 = nn.Linear(256, num_classes)
# self.classifier2 = nn.Linear(256, num_classes)
# self.classifier3 = nn.Linear(256, num_classes)
#
# def forward(self, x):
# out1 = self.classifier1(self.fc1(x))
# out2 = self.classifier2(self.fc1(x))
# out3 = self.classifier3(self.fc1(x))
# return out1, out2, out3
#
# (x_train, y_train), (x_test, y_test) = dataset_utils.get_iris_dataset_np()
#
# # make multi-label categorical
# y_train = np.column_stack((y_train, y_train, y_train))
# y_test = np.column_stack((y_test, y_test, y_test))
# test = PytorchData(x_test.astype(np.float32), y_test.astype(np.float32))
#
# model = multi_label_cat_model(3, 4)
# criterion = nn.CrossEntropyLoss()
# optimizer = optim.Adam(model.parameters(), lr=0.01)
#
# # train model
# train_dataset = TensorDataset(from_numpy(x_train.astype(np.float32)), from_numpy(y_train.astype(np.float32)))
# train_loader = DataLoader(train_dataset, batch_size=100, shuffle=True)
#
# for epoch in range(5):
# # Train for one epoch
# for inputs, targets in train_loader:
# # Zero the parameter gradients
# optimizer.zero_grad()
#
# # Perform prediction
# model_outputs = model(inputs)[-1]
#
# # Form the loss function
# loss = 0
# for i, o in enumerate(model_outputs):
# loss += criterion(o, targets[i])
#
# loss.backward()
#
# optimizer.step()
#
# art_model = PyTorchClassifier(model=model,
# output_type=ModelOutputType.CLASSIFIER_MULTI_OUTPUT_CLASS_LOGITS,
# loss=criterion,
# optimizer=optimizer,
# input_shape=(24,),
# nb_classes=3)
#
# pred = art_model.predict(test)
# assert (pred.shape[0] == x_test.shape[0])
#
# score = art_model.score(test, apply_non_linearity=expit)
# assert (score == 1.0)
def test_pytorch_predictions_multi_label_cat():
# This kind of model requires special training and will not be supported using the 'fit' method.
class multi_label_cat_model(nn.Module):
def __init__(self, num_classes, num_features):
super(multi_label_cat_model, self).__init__()
self.fc1 = nn.Sequential(
nn.Linear(num_features, 256),
nn.Tanh(), )
self.classifier1 = nn.Linear(256, num_classes)
self.classifier2 = nn.Linear(256, num_classes)
def forward(self, x):
out1 = self.classifier1(self.fc1(x))
out2 = self.classifier2(self.fc1(x))
return out1, out2
(x_train, y_train), (x_test, y_test) = dataset_utils.get_iris_dataset_np()
# make multi-label categorical
num_classes = 3
y_train = check_and_transform_label_format(y_train, nb_classes=num_classes)
y_test = check_and_transform_label_format(y_test, nb_classes=num_classes)
y_train = np.column_stack((y_train, y_train))
y_test = np.stack([y_test, y_test], axis=1)
test = PytorchData(x_test.astype(np.float32), y_test.astype(np.float32))
model = multi_label_cat_model(num_classes, 4)
criterion = nn.CrossEntropyLoss()
optimizer = optim.Adam(model.parameters(), lr=0.01)
# train model
train_dataset = TensorDataset(from_numpy(x_train.astype(np.float32)), from_numpy(y_train.astype(np.float32)))
train_loader = DataLoader(train_dataset, batch_size=100, shuffle=True)
for epoch in range(5):
# Train for one epoch
for inputs, targets in train_loader:
# Zero the parameter gradients
optimizer.zero_grad()
# Perform prediction
model_outputs = model(inputs)
# Form the loss function
loss = 0
for i, o in enumerate(model_outputs):
t = targets[:, i*num_classes:(i+1)*num_classes]
loss += criterion(o, t)
loss.backward()
optimizer.step()
art_model = PyTorchClassifier(model=model,
output_type=ModelOutputType.CLASSIFIER_MULTI_OUTPUT_CLASS_LOGITS,
loss=criterion,
optimizer=optimizer,
input_shape=(24,),
nb_classes=3)
pred = art_model.predict(test)
assert (pred.shape[0] == x_test.shape[0])
score = art_model.score(test, apply_non_linearity=expit)
assert (0 < score <= 1.0)
def test_pytorch_predictions_multi_label_binary():