How to plot ROC Curve using PyTorch model

1

Hello,

I have semantic segmentation code, this code help me to test 25 images results (using confusion matrix). But I want to plot ROC Curve of testing datasets. But I am unable to do this job. Please check my shared code, and let me know, how I properly draw ROC curve by using this code.

import os
import cv2
import torch
import numpy as np
from glob import glob
from model import AI_Net
from operator import add
from crf import apply_crf
import matplotlib.pyplot as plt
from sklearn.metrics import roc_curve
from sklearn.metrics import roc_auc_score

device = torch.device(‘cuda’ if torch.cuda.is_available() else ‘cpu’)

“”" Load the checkpoint “”"
model = AI_Net()
model = model.to(device)
model.load_state_dict(torch.load(‘datasets/models/A_Net/Fold_1_Model.pth’, map_location=device))
model.eval()

def calculate_metrics(y_true, y_pred):
y_true = y_true.cpu().numpy()
y_pred = y_pred.cpu().numpy()

y_pred = y_pred > 0.5
y_pred = y_pred.reshape(-1)
y_pred = y_pred.astype(np.uint8)

y_true = y_true > 0.5
y_true = y_true.reshape(-1)
y_true = y_true.astype(np.uint8)

## Score
score_fpr, score_tpr, _ = roc_curve(y_true, y_pred)
score_roc_auc = roc_auc_score(y_true, y_pred)

return [score_fpr, score_tpr, score_roc_auc]

""" Load dataset """

root_path = ‘datasets/UDIAT/’
test_x = sorted(glob(os.path.join(root_path, “test/images”, “.png")))
test_y = sorted(glob(os.path.join(root_path, “test/masks”, ".png”)))

metrics_score = [0.0, 0.0, 0.0, 0.0, 0.0, 0.0]

for i, (x, y) in enumerate(zip(test_x, test_y)):
name = y.split(“/”)[-1].split(“.”)[0]
## Image
image = cv2.imread(x, cv2.IMREAD_GRAYSCALE)
image = cv2.resize(image, (128, 128))
ori_img1 = image
image1 = np.expand_dims(image, axis=0)
image1 = image1/255.0
image1 = np.expand_dims(image1, axis=0)
image1 = image1.astype(np.float32)
image1 = torch.from_numpy(image1)
image1 = image1.to(device)

  ## Mask
  mask = cv2.imread(y, cv2.IMREAD_GRAYSCALE)
  mask = cv2.resize(mask, (128, 128))
  ori_mask1 = mask
  mask1 = np.expand_dims(mask, axis=0)
  mask1 = mask1/255.0
  mask1 = np.expand_dims(mask1, axis=0)
  mask1 = mask1.astype(np.float32)
  mask1 = torch.from_numpy(mask1)
  mask1 = mask1.to(device)

  # Shape
  #print('image shape:', image1.shape, 'roi shape:', mask1.shape)
          
  with torch.no_grad():
      pred_y1 = torch.sigmoid(model(image1))
      score = calculate_metrics(mask1, pred_y1)
      metrics_score = list(map(add, metrics_score, score))

    tpr     = metrics_score[0]/len(test_x)
    fpr     = metrics_score[1]/len(test_x)
    roc_auc = metrics_score[2]/len(test_x)

algorithm = ‘CNN’
dataset = ‘BUS’
rocCurve = plt.figure()
plt.plot(fpr, tpr, ‘-’, label=algorithm + ‘_’ + dataset + ‘(AUC = %0.4f)’ % roc_auc)
plt.title(‘ROC curve’, fontsize=14)
plt.xlabel(“FPR (False Positive Rate)”, fontsize=15)
plt.ylabel(“TPR (True Positive Rate)”, fontsize=15)
plt.legend(loc=“lower right”)
plt.xticks(fontsize=15)
plt.yticks(fontsize=15)
plt.show()

2

You could use the ROC implementations from other libraries such as sklearn.metrics.roc_curve. To do so you could transform the predictions and targets to numpy arrays via tensor.numpy() and apply the mentioned method.

1 Like
3

Hello,

To store all iterations results of y_true, and y_pred, i added all_y_true, all_y_pred. But when i try to plot ROC Curve, it shows ValueError: continuous format is not supported, at line → 11 fpr, tpr, _ = roc_curve(y_true, y_pred)

Here is my code:

all_y_true =
all_y_pred =

for i, (x, y) in enumerate(zip(test_x, test_y)):
name = y.split(“/”)[-1].split(“.”)[0]
## Image
image = cv2.imread(x, cv2.IMREAD_GRAYSCALE)
image = cv2.resize(image, (128, 128))
image = np.expand_dims(image, axis=0)
image = image/255.0
image = np.expand_dims(image, axis=0)
image = image.astype(np.float32)
image = torch.from_numpy(image)
image = image.to(device)

## Mask
mask = cv2.imread(y, cv2.IMREAD_GRAYSCALE)
mask = cv2.resize(mask, (128, 128))
mask = np.expand_dims(mask, axis=0)
mask = mask/255.0
mask = np.expand_dims(mask, axis=0)
mask = mask.astype(np.float32)
mask = torch.from_numpy(mask)
#mask = mask.to(device)
all_y_true = np.append(all_y_true, mask)

# Shape
#print('image shape:', image1.shape, 'roi shape:', mask1.shape)

with torch.no_grad():
    pred_y1 = torch.sigmoid(model(image))
    y_pred = pred_y1.cpu().numpy()
    y_pred = y_pred > 0.5
    y_pred = y_pred.reshape(-1)
    y_pred = y_pred.astype(np.uint8)
    #y_pred.flatten()
    all_y_pred = np.append(all_y_pred, y_pred)

Calculate image-level ROC AUC score

y_true = all_y_true
y_pred = all_y_pred

fpr, tpr, _ = roc_curve(y_true, y_pred)
roc_auc = roc_auc_score(y_true, y_pred)

plt.figure(1)
plt.plot([0, 1], [0, 1], ‘k–’)
plt.plot(fpr, tpr, label=‘CNN(area = {:.3f})’.format(roc_auc))
plt.xlabel(‘False positive rate’)
plt.ylabel(‘True positive rate’)
plt.title(‘ROC curve’)
plt.legend(loc=‘best’)
plt.show()

4

I guess the inputs to roc_curve are wrong, so you would have to make sure they fit the expected arrays as described in the docs:

y_true ndarray of shape (n_samples,)
True binary labels. If labels are not either {-1, 1} or {0, 1}, then pos_label should be explicitly given.

y_score ndarray of shape (n_samples,)
Target scores, can either be probability estimates of the positive class, confidence values, or non-thresholded measure of decisions (as returned by “decision_function” on some classifiers).

5

I resolved error, but now i am getting this error

ValueError: multiclass format is not supported → Line 12 fpr, tpr, _ = roc_curve(y_true, y_pred)

6

This indicates a wrong shape of one of the inputs, so you would have to make sure to use the described shapes from my previous post.

7

Thanks very much, I transform my y_true, y_score into acceptable shapes, and issue is resolved.