# Using roc curve with leaving one out

Hey guys,
i m looking for help to correctly implement roc curves for my leaving one out code. Something doesn’t work well. My roc curve looks like:

I was looking in the internet for some instructions/examples how to implement the roc curves for leaving one out but what i have founded doesn’t match to my requirements.

https://scikit-learn.org/stable/auto_examples/model_selection/plot_roc_crossval.html
In my code i dont use any pipeline and so i dont can use this lines of code:

classifier.fit(X[train], y[train])
(classifier, X[test], y[test], name='ROC fold {}'.format(i), alpha=0.3, lw=1, ax=ax)


i’m also not really sure what’s about mean_fpr = numpy.linspace)(0, 1, 100)

Here is my full code:

import train
import model
import data_preparation
import plot_graph
import matplotlib.pyplot as plt
import yaml
import numpy as np
from sklearn.metrics import confusion_matrix
import torch
from sklearn import svm, datasets
from sklearn.metrics import auc
from sklearn.metrics import plot_roc_curve
# Function to print ROC and AUC
from sklearn.metrics import accuracy_score, roc_curve, auc

project_root =  "/mnt/XX/user/XX/XX"
with open(project_root+"/config.yml") as f:
import numpy as np
from sklearn.model_selection import LeaveOneOut
import pandas as pd

tprs = []
aucs = []
mean_fpr = np.linspace(0, 1, 100)

fig, ax = plt.subplots()

#for test purpose to avoid full loop of the whole data set
i = 0

# Load data and set labels
data_z['label'] = 0
data_a['label'] = 1

# Concatenate data
data = pd.concat([data_z, data_a], axis=0).reset_index(drop=True)

X = data.le.values
y = data.label.values

loo = LeaveOneOut()
loo.get_n_splits(X)

LeaveOneOut()
for train_index, test_index in loo.split(X):
print("TRAIN:", train_index, "TEST:", test_index)
X_train, X_test = X[train_index], X[test_index]
y_train, y_test = y[train_index], y[test_index]

#-------------------------------       --------------------------------
#-------------------------------       --------------------------------

# Convert other data types to torch.Tensor
train_labels = torch.tensor(y_train)
val_labels = torch.tensor(y_test)

# For fine-tuning BERT, the authors recommend a batch size of 16 or 32.
batch_size = params["training"]["batch_size"]

# Create the DataLoader for our training set
train_sampler = RandomSampler(train_data)

# Create the DataLoader for our validation set
val_sampler = SequentialSampler(val_data)

#Training Start

train.set_seed(params["training"]["seed"])    # Set seed for reproducibility
model.bert_classifier, train.optimizer, train.scheduler = train.initialize_model(epochs=params["training"]["epochs"])

i = i +1
print(i)

if i == 2:
break;

preds = probs[:, 1]
fpr, tpr, threshold = roc_curve(y_test, preds)
roc_auc = auc(fpr, tpr)
print(f'AUC: {roc_auc:.4f}')

# Get accuracy over the test set
y_pred = np.where(preds >= 0.5, 1, 0)
accuracy = accuracy_score(y_test, y_pred)
print(f'Accuracy: {accuracy*100:.2f}%')

interp_tpr = np.interp(mean_fpr, fpr, tpr)
interp_tpr[0] = 0.0
tprs.append(interp_tpr)
aucs.append(roc_auc)

ax.plot([0, 1], [0, 1], linestyle='--', lw=2, color='r',
label='Chance', alpha=.8)

mean_tpr = np.mean(tprs, axis=0)
mean_tpr[-1] = 1.0
mean_auc = auc(mean_fpr, mean_tpr)
std_auc = np.std(aucs)
ax.plot(mean_fpr, mean_tpr, color='b',
label=r'Mean ROC (AUC = %0.2f $\pm$ %0.2f)' % (mean_auc, std_auc),
lw=2, alpha=.8)

std_tpr = np.std(tprs, axis=0)
tprs_upper = np.minimum(mean_tpr + std_tpr, 1)
tprs_lower = np.maximum(mean_tpr - std_tpr, 0)
ax.fill_between(mean_fpr, tprs_lower, tprs_upper, color='grey', alpha=.2,
label=r'$\pm$ 1 std. dev.')

ax.set(xlim=[-0.05, 1.05], ylim=[-0.05, 1.05],