Calculating AUC ROC

1

I am trying to calculate AUC ROC score and curve for my model which is trained to detect whether given image is not adversarial(label 0) and adversarial (label 1) for specific adversarial attack such as FGSM
I was trying to calculate this using following code :

import torch
import torch.nn as nn
from torch.utils.data import DataLoader
from torchvision.datasets import ImageFolder
from torchvision.transforms import transforms
from tqdm import tqdm
from sklearn.metrics import roc_auc_score, roc_curve, f1_score, recall_score, precision_score
from sklearn.metrics import confusion_matrix
import numpy as np


validation_dir = '/kaggle/input/piash-pgd-all-test/validation_pgd_all_1'


model_path = '/kaggle/input/piash-selu-new/custom_epoch_bit_8_SELU_399.pb'


transform = transforms.Compose([
    transforms.Resize((224, 224)),  # Adjust the size as needed
    transforms.ToTensor(),
])


validation_dataset = ImageFolder(validation_dir, transform=transform)
val_loader = DataLoader(validation_dataset, batch_size=32, shuffle=False)


model = ResNetWithAttention(ResidualBlockWithTransformerAttention, [3, 4, 6, 3], num_classes=2)

device = torch.device("cuda")

model.load_state_dict(torch.load(model_path,map_location=device))
model.to(device)  # Move the model to the GPU


model.eval()


criterion = nn.CrossEntropyLoss()


true_labels = []
predicted_probs = []

epoch_val_accuracy = 0
epoch_val_loss = 0

with tqdm(total=len(val_loader), unit='batch') as val_pbar:
    for j, (val_data, val_label) in enumerate(val_loader):
        val_data = val_data.to(device)
        val_label = val_label.to(device)

        val_output = model(val_data)
        val_probs = torch.nn.functional.softmax(val_output, dim=1)[:, 1]

        true_labels.extend(val_label.cpu().numpy())
        predicted_probs.extend(val_probs.detach().cpu().numpy())

        
 
        val_loss = criterion(val_output, val_label)

        acc = (val_output.argmax(dim=1) == val_label).float().mean()
        epoch_val_accuracy += acc.item()  # Accumulate validation accuracy
        epoch_val_loss += val_loss.item()  # Accumulate validation loss

        val_pbar.set_postfix(loss=val_loss.item(), acc=acc.item())
        val_pbar.update()

epoch_val_accuracy /= len(val_loader)
epoch_val_loss /= len(val_loader)




print(f'Accuracy: {epoch_val_accuracy:.4f}')
print(f'Average Loss: {epoch_val_loss:.4f}')


true_labels = np.array(true_labels)
predicted_probs = np.array(predicted_probs)


fpr, tpr, _ = roc_curve(true_labels, predicted_probs)
roc_auc = roc_auc_score(true_labels, predicted_probs)

print(f'ROC AUC Score: {roc_auc:.4f}')

I am confused about this output and not sure whether it is okay.

2

Could you explain your confusion and concerns a bit more, please?
Are these values not matching your expectation?

3

Thank you so much for your reply.
Could you please clarify the purpose of the following code snippet: `val_probs=torch.nn.functional.softmax(val_output, dim=1)[:, 1]
Is it intended to compute the probability of the positive class (label 1 for adversarial images) in the context of my task of detecting adversarial images?
Additionally, in the context of my task of detecting adversarial images,what should be provided as input parameters to the ROC AUC function of scikit-learn?