I have an issue where if I feed data row by row, then my binary-classification LSTM model gets an AUC of 0.9+ within a few epochs. When I changed to code so that it accepts batches, the AUC gets stuck at 0.5 despite the loss decreasing.
I created a toy example of the LSTM model to try to test out this issue. I am suspecting that my model architecture is passing in the wrong information, because at some point it just predicts everything as “positive”, but I don’t know where. Code is below:
Import and helper functions:
import pandas as pd
import numpy as np
from sklearn.metrics import roc_curve, auc, roc_auc_score
import multiprocessing
import pickle
import ast
import s3fs
import json
import torch
import torch.autograd as autograd
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
import torch.nn.utils.rnn as rnn_utils
import random
from random import randint
import os
import math
import time
# returns the softmax output into a readable prediction
def get_max_prob_result(input, ix_to_word):
return ix_to_word[get_index_of_max(input)]
# calculates the roc_auc score for the predictions
def get_auroc(truth, pred):
assert len(truth) == len(pred)
auc_score = roc_auc_score(np.array(truth),np.array(pred))
return auc_score
def reorder_list(list, new_index_list):
new_list = []
for index in new_index_list:
new_list.append(list[index])
return new_list
def grab_batch(batch_size):
seq=[]
freq=[]
target=[]
time = []
for k in range(batch_size):
tseq, tfreq, ttime, ttarget = generate_patient()
seq.append(tseq)
freq.append(tfreq)
time.append(ttime)
target.append(ttarget)
return seq, freq, time, target
Function to randomly generate data with structure that matches my real use case (note that I created a rule to define when target is positive that the model will learn):
events_to_ix = {'<PAD>':0,'non':1,'othernon':2,'neutral':3,'trigger':4}
final_seq = []
final_freq = []
final_time = []
final_target = []
dict_keys = list(events_to_ix.keys())[1:]
def generate_patient():
num_seq = randint(1,100)
patient_seq = []
patient_freq = []
patient_time = []
patient_target = 0
final_seq = []
final_freq = []
final_time = []
for i in range(num_seq):
step_seq = []
step_freq = []
step_time = []
seq_length = randint(1,10)
for k in range(seq_length):
event = random.choice(dict_keys)
if events_to_ix[event] in step_seq:
continue
step_seq.append(events_to_ix[event])
step_freq.append(randint(1, 17))
step_time.append(randint(0,(5+seq_length-k)))
patient_seq.append(step_seq)
patient_freq.append(step_freq)
patient_time.append(step_time)
for index, item in enumerate(patient_seq[-1]):
if item == 4 and patient_freq[-1][index] > 15 and patient_time[-1][index] < 3:
patient_target = 1
break
# Loop through each concet in each timestep
for step_idx, step in enumerate(patient_seq):
concepts = [0]*len(events_to_ix)
frequencies = [0]*len(events_to_ix)
times = [0]*len(events_to_ix)
for event_idx, event in enumerate(step):
# Convert the textual concepts into their index representation
concepts[event] = event
# Append the frequencies and time encodings to their appropriate position in the list of 0's
frequencies[event] = patient_freq[step_idx][event_idx]
times[event] = patient_time[step_idx][event_idx]
final_seq.append(concepts)
final_freq.append(frequencies)
final_time.append(times)
#final_static.append(patient_static)
final_seq = torch.LongTensor(final_seq)
final_freq = torch.FloatTensor(final_freq).view(-1,len(events_to_ix),1)
final_time = torch.FloatTensor(final_time).view(-1,len(events_to_ix),1)
#final_static = torch.FloatTensor(final_static)
return final_seq, final_freq, final_time, patient_target
The LSTM model:
# Class containing the LSTM model initialization and feed-forward logic
class LSTMClassifier(nn.Module):
# LSTM initialization
def __init__(self, embedding_dim, hidden_dim, vocab_size, label_size):
super(LSTMClassifier, self).__init__()
# Setting the hidden layer dimension of the LSTM
self.hidden_dim = hidden_dim
# Initializing the embedding layer
self.embeddings = nn.Embedding(vocab_size, embedding_dim-2)
# Initializing the LSTM layer with one hidden layer
self.lstm = nn.LSTM(((embedding_dim*vocab_size)), hidden_dim, num_layers = 1, batch_first=False)
# Initializing linear linear that takes the hidden layer output
self.hidden2label = nn.Linear(hidden_dim, label_size)
# Defining the hidden state of the LSTM
def init_hidden(self,batch_size):
# the first is the hidden h
# the second is the cell c
return [autograd.Variable(torch.zeros(1,batch_size, self.hidden_dim).cuda()),
autograd.Variable(torch.zeros(1,batch_size, self.hidden_dim).cuda())]
# Defining the feed forward logic of the LSTM. It contains:
# 1. The embedding layer
# 2. The LSTM layer with one hidden layer
# 3. The softmax layer
def forward(self, seq, freq, time_data, seq_lengths):
# Grab the mini-batch length and max sequence length (pre-ordered)
# (need to do this in the forward logic because of data parallelism and how the GPU's will split up the batch)
sequence_length = seq.size()[1]
batch_length = seq.size()[0]
# reset the LSTM hidden state.
# Must be done before you run a new batch. Otherwise the LSTM will treat a new batch as a continuation of a sequence
self.hidden = self.init_hidden(batch_length)
# This is the pass to the embedding layer.
# The sequence is of dimension N and the output is N x Demb
embeds = self.embeddings(seq)
# Concatenate the embedding output with the time and frequency vectors
embeds = torch.cat((embeds,freq), dim=3)
embeds = torch.cat((embeds,time_data), dim=3)
# Because the LSTM excepts a dimension of (sequence length, batch size, feature size), and we have (batch size, seq length, feature size),
# we need to switch the first and second dimension so that we get the correct input format
embeds = torch.transpose(embeds, 0, 1)
# Flatten the embedding dimension so that the input to the LSTM remains 3D rather than 4D
x = embeds.view(sequence_length, batch_length, -1)
# pack the padded sequence so that paddings are ignored
packed_x = torch.nn.utils.rnn.pack_padded_sequence(x, seq_lengths, batch_first=False)
# Feed to the LSTM layer
lstm_out, self.hidden = self.lstm(packed_x, self.hidden)
# Feed the last layer of the LSTM into the linear layer
y = self.hidden2label(self.hidden[0][-1])
# Produce the softmax probabilities
log_probs = F.log_softmax(y)
return log_probs
Function to run a single epoch:
def train_epoch(model, loss_function, optimizer,batch_size, i):
# Set model to training mode and initialize variables
model.train()
avg_loss = 0.0
count = -1
truth_res = []
pred_res = []
# Group the dataframe into dataframe chunks of length batch size and loop through each batch
for j in range(1000):
count += 1
seq, freq, time_data, target = grab_batch(batch_size)
# Sort the batches by descending size
final_seq_ordered = sorted(enumerate(seq), key=lambda x: len(x[-1]), reverse=True)
# Grab the original indices from final_seq and see how they are now ordered
final_seq_indices = [item[0] for item in final_seq_ordered]
# Grab the actual values from the tupled master list of concepts
seq = [item[1] for item in final_seq_ordered]
freq = reorder_list(freq, final_seq_indices)
time_data = reorder_list(time_data, final_seq_indices)
target = reorder_list(target, final_seq_indices)
# Grab the list of lengths of sequences, for the purpose of packing the padded sequenes
seq_lengths = torch.LongTensor(list(map(len, seq)))
# Grab the targets into a list and append it into the truth_res list in order to measure AUC performance
truth_res.extend(target)
# Pad the sequences
seq = rnn_utils.pad_sequence(seq, batch_first = True)
freq = rnn_utils.pad_sequence(freq, batch_first = True)
time_data = rnn_utils.pad_sequence(time_data, batch_first = True)
# Put the padded sequences into Variable and Cuda cores
seq = autograd.Variable(seq.cuda())
freq = autograd.Variable(freq.cuda())
time_data = autograd.Variable(time_data.cuda())
target = autograd.Variable(torch.LongTensor(target).cuda())
# Feed the tensor Variables into the model
pred = model(seq,freq,time_data,seq_lengths)
# Append the predictions into a list for future AUC evaluation
pred_label = pred.detach().max(1)[1].cpu().numpy()
pred_res.extend(pred_label)
# Reset the model gradient
model.zero_grad()
# Compute the loss
loss = loss_function(pred, target)
# Backpropagate
loss.backward()
# Update weights
optimizer.step()
# Computes the average loss
avg_loss += loss.detach().item()
# Computes the AUC score
auc_score = get_auroc(truth_res, pred_res)
avg_loss /= (1000/batch_size)
print('epoch: %d done! \n train avg_loss:%g , auc:%g' % (i, avg_loss, auc_score))
Main training loop:
#############################
### Set hyper parameters ###
############################
EMBEDDING_DIM = 32
HIDDEN_DIM = 50
EPOCH = 10
BATCH_SIZE = 16
best_val_auc = 0.5
model = LSTMClassifier(embedding_dim=EMBEDDING_DIM, hidden_dim=HIDDEN_DIM, vocab_size=(len(events_to_ix)), label_size=2)
model = torch.nn.DataParallel(model.cuda())
weights = [(26/1000), 1]
class_weights = torch.FloatTensor(weights).cuda()
loss_function = nn.NLLLoss(weight=class_weights,reduction="sum",ignore_index=0)
optimizer = optim.Adam(model.parameters(), lr=0.001)
no_up = 0
#####################################################
### Set loop to determine number of EPOCHs to run ###
#####################################################
for i in range(EPOCH):
#############################################
### Run the training on the training data ###
#############################################
print('epoch: %d start!' % i)
start = time.time()
# Perform the training on the epoch
train_epoch(model,loss_function, optimizer,BATCH_SIZE, i)
print("1 epoch length of time")
print(time.time() - start)