I have binary classification problem, My input is vector of 561 and out put is two classes are negative class is [1,0] and positive class is [0,1], I trained model that is return two numbers and I am trying to plot auc and find the best threshold , when I tries to use the probabilities in calculating roc curve i got 50% and when I replace it with predictions 0 or 1 i get 70 % and I do not know why that big difference. I do not know if I am calculating it wrong since I get two probability and I get the index of the largest one. I do not know if I am getting that difference because weighted random samples since I have unbalanced data. I would like to know how
Here is the training function
def run_gradient_descent(model, data_train, data_val, batch_size, learning_rate, weight_decay=0, num_epochs=20):
model = model.to(device)
#criterion = nn.CrossEntropyLoss()
criterion = nn.MSELoss()
#criterion = nn.BCEWithLogitsLoss()
optimizer = optim.SGD(model.parameters(), lr=learning_rate, weight_decay=weight_decay)
iters, losses, train_losses, test_losses = [], [], [], []
iters_sub, train_acc, val_acc = [], [] ,[]
print(batch_size)
# weight sampler
class0, class1 =labels_count(data_train)
dataset_counts = [class0, class1]
print(dataset_counts)
num_samples = sum(dataset_counts)
labels = [tag for _, tag in data_train]
class_weights = [1./dataset_counts[i] for i in range(len(dataset_counts))]
labels_indics = [i.index(max(i)) for i in labels ]
weights = [class_weights[i] for i in labels_indics] # labels.max(1, keepdim=True)[1]
weights = numpy.array(weights)
samples_weight = torch.from_numpy(weights)
samples_weigth = samples_weight.double()
sampler = torch.utils.data.sampler.WeightedRandomSampler(samples_weight, int(num_samples), replacement=True)
train_loader = torch.utils.data.DataLoader(
data_train,
batch_size=batch_size,
shuffle=False,
sampler = sampler,
collate_fn=lambda d: ([x[0] for x in d], [x[1] for x in d]),
num_workers=os.cpu_count()//2
)
# training
n = 0 # the number of iterations
for epoch in tqdm(range(num_epochs), desc="epoch"):
correct = 0
total = 0
for xs, ts in tqdm(train_loader, desc="train"):
xs = torch.FloatTensor(xs).to(device)
ts = torch.FloatTensor(ts).to(device)
# print("batch index {}, 0/1: {}/{}".format(n,ts.tolist().count([1,0]),ts.tolist().count([0,1])))
# if len(ts) != batch_size:
# print("ops")
# continue
model.train()
zs = model(xs)
zs = zs.to(device)
loss = criterion(zs, ts)
loss.backward()
optimizer.step()
optimizer.zero_grad()
torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)
iters.append(n)
loss.detach().cpu()
losses.append(float(loss)/len(ts)) # compute *average* loss
pred = zs.max(1, keepdim=True)[1] # get the index of the max logit
target = ts.max(1, keepdim=True)[1]
correct += pred.eq(target).sum().item()
total += int(ts.shape[0])
acc = correct / total
if (n % len(train_loader) == 0) and n>0 and epoch%2==0:
test_acc, test_loss = get_accuracy(model, data_val)
iters_sub.append(n)
train_acc.append(acc)
val_acc.append(test_acc)
train_losses.append(sum(losses)/len(losses))
test_losses.append(test_loss)
print("Epoch", epoch, "train_acc", acc)
print("Epoch", epoch, "test_acc", test_acc)
print("Epoch", epoch, "train_loss", sum(losses)/len(losses))
print("Epoch", epoch, "test_loss", test_loss)
# increment the iteration number
n += 1
torch.save(model.state_dict(), f"{MODEL_NAME}/checkpoint_epoch{epoch}.pt")
# plotting
plt.title("Training Curve (batch_size={}, lr={})".format(batch_size, learning_rate))
plt.plot(iters_sub, train_losses, label="Train")
plt.legend(loc='best')
plt.xlabel("Iterations")
plt.ylabel("Loss")
plt.savefig(f"{MODEL_NAME}/training_loss.png")
#
plt.clf()
plt.title("Validation Curve (batch_size={}, lr={})".format(batch_size, learning_rate))
plt.plot(iters_sub, test_losses, label="Test")
plt.xlabel("Iterations")
plt.ylabel("Loss")
plt.savefig(f"{MODEL_NAME}/test_loss.png")
#plt.show()
plt.clf()
plt.title("Training Curve (batch_size={}, lr={})".format(batch_size, learning_rate))
plt.plot(iters_sub, train_acc, label="Train")
plt.plot(iters_sub, val_acc, label="Test")
plt.xlabel("Iterations")
plt.ylabel("Accuracy")
plt.legend(loc='best')
plt.savefig(f"{MODEL_NAME}/training_acc.png")
#plt.show()
return model
HEre is how I get the prediction and calculate roc
def get_accuracy(model, data):
loader = torch.utils.data.DataLoader(
data,
batch_size=64,
collate_fn=lambda d: ([x[0] for x in d], [x[1] for x in d]),
num_workers=os.cpu_count() // 2
)
preds_prob = []
targets = []
preds = []
model.eval()
with torch.no_grad():
for xs, ts in tqdm(loader, desc="eval"):
xs = torch.FloatTensor(xs).to(device)
ts = torch.FloatTensor(ts).to(device)
zs = model(xs)
preds_prob.append([max(i) for i in zs.tolist()])
preds.append(flatten_list(zs.max(1, keepdim=True)[1].tolist())) # get the index of the max logit
targets.append(flatten_list(ts.max(1, keepdim=True)[1].tolist()))
return preds, preds_prob, targets
preds, preds_prob, targets= get_accuracy(model, test_set)
predictions = flatten_list(preds)
target = flatten_list(targets)
predictions_prob= flatten_list(preds_prob)
when I use the predictions only to calculaute auc
fpr, tpr, thresholds = roc_curve(target, predictions)
I get 70%
but if I use
fpr, tpr, thresholds = roc_curve(target, predictions_prob)
I get around 51%, also even if I try to fin the optimal threshold it did not change much.
I would like to know if i get prediction probability in wrong way since y classes are [0,1] and [1,0]?