My training loss keeps going slightly up and down around the same value using MSELoss(). On top of that my validation loss stays the exact same down to the 8th decimal point. Did I mess something up because I don’t think my model is training?
class LSTMSentenceEncoder(nn.Module):
def __init__(self,input_size,hidden_size,num_layers,vectors,word2idx):
super(LSTMSentenceEncoder,self).__init__()
self.embedding = nn.Embedding.from_pretrained(vectors,freeze=False,padding_idx=word2idx['_PAD'])
self.lstm = nn.LSTM(input_size,hidden_size,num_layers,batch_first=True)
self.hidden_size = hidden_size
self.num_layers = num_layers
self.word2idx = word2idx
self.input_size = input_size
def forward(self,x):
if torch.cuda.is_available():
h0 = Variable(torch.randn(self.num_layers,x.size(0),self.hidden_size),requires_grad=True).cuda()
c0 = Variable(torch.randn(self.num_layers,x.size(0),self.hidden_size),requires_grad=True).cuda()
embs = self.embedding(x)
embs = embs.view(x.size(0),-1,self.input_size).requires_grad_().cuda()
else:
h0 = Variable(torch.randn(self.num_layers,x.size(0),self.hidden_size),requires_grad=True)
c0 = Variable(torch.randn(self.num_layers,x.size(0),self.hidden_size),requires_grad=True)
embs = self.embedding(x)
embs = embs.view(x.size(0),-1,self.input_size)
out,(hn,cn) = self.lstm(embs,(h0,c0))
out = out[:, -1, :]
return out,hn
class SiameseLSTM(nn.Module):
def __init__(self,input_size,hidden_size,num_layers,vectors,word2idx):
super(SiameseLSTM, self).__init__()
self.encoder = LSTMSentenceEncoder(input_size,hid_size,num_layers,vectors,word2idx)
def forward(self, s1, s2):
outputs = []
for i in range(len(s1)):
v1,h1 = self.encoder(s1[i])
v2,h2 = self.encoder(s2[i])
# to select the final hidden representation
h1 = h1[-1,-1,:]
h2 = h2[-1,-1,:]
out = torch.exp(-torch.norm((h1 - h2), 1))
outputs.append(out)
return torch.stack(outputs).cuda()
model = SiameseLSTM(input_size,hid_size,num_layers,wordEmbs,word2idx)
if torch.cuda.is_available():
model = model.cuda()
model.float()
optimizer = torch.optim.Adam(model.parameters(),lr = 0.001)
criterion = torch.nn.MSELoss()
count = 0
seq_dim = 15
num_epochs = 5
for epoch in tqdm(range(num_epochs)):
for i, (s1,s2, labels) in enumerate(trainLoader):
if torch.cuda.is_available():
s1 = Variable(s1.view(-1, seq_dim, 1).cuda())
s2 = Variable(s2.view(-1, seq_dim, 1).cuda())
labels = Variable(labels.cuda())
else:
s1 = Variable(s1.view(-1, seq_dim, 1),)
s2 = Variable(s2.view(-1, seq_dim, 1),)
labels = Variable(labels)
optimizer.zero_grad()
outputs = model(s1,s2)
loss = criterion(outputs, labels)
loss.backward()
optimizer.step()
count += 1
if count % 10 == 0:
val_losses = []
correct = 0
total = 0
for s1,s2, labels in testLoader:
if torch.cuda.is_available():
s1 = Variable(s1.view(-1, seq_dim, 1).cuda())
s2 = Variable(s2.view(-1, seq_dim, 1).cuda())
labels = Variable(labels.cuda())
else:
s1 = Variable(s1.view(-1, seq_dim, 1),)
s2 = Variable(s2.view(-1, seq_dim, 1),)
labels = Variable(labels)
predicted = model(s1,s2)
total += labels.size(0)
vloss = criterion(predicted,labels)
val_losses.append(vloss.data.cpu())
print('Iteration: {}. Loss: {}. MSE: {}'.format(count, loss.item(), np.mean(val_losses)))