Hi everyone,
I built a simple sequence-to-sequence model, without attention, to translate senteces from german to english.
Here’s the model:
class Encoder(nn.Module):
def __init__(self, input_size, embedding_size, hidden_size):
super(Encoder, self).__init__()
self.embed = nn.Embedding(input_size, embedding_size)
self.dropout = nn.Dropout(p=0.5)
self.lstm = nn.LSTM(embedding_size, hidden_size, num_layers=2, dropout=0.5)
def forward(self, input):
# torch.Size([21, 32])
x = self.dropout(self.embed(input))
# torch.Size([21, 32, 300])
outputs, (hidden_state, cell_state) = self.lstm(x)
# torch.Size([21, 32, 1024]) torch.Size([2, 32, 1024]) torch.Size([2, 32, 1024])
return hidden_state, cell_state
class Decoder(nn.Module):
def __init__(self, input_size, embedding_size, hidden_size, output_size):
super(Decoder, self).__init__()
self.embed = nn.Embedding(input_size, embedding_size)
self.dropout = nn.Dropout(p=0.5)
self.lstm = nn.LSTM(embedding_size, hidden_size, num_layers=2, dropout=0.5)
self.fc = nn.Linear(hidden_size, output_size)
def forward(self, input, hidden_state_encoder, cell_state_encoder):
# torch.Size([1, 32])
x = self.dropout(self.embed(input))
# torch.Size([1, 32, 300])
outputs, (hidden_state, cell_state) = self.lstm(x, (hidden_state_encoder, cell_state_encoder))
# torch.Size([1, 32, 1024]) torch.Size([2, 32, 1024]) torch.Size([2, 32, 1024])
outputs = outputs.permute(1, 0, 2) # nn.Linear expects (batch_size, *, in_features)
# torch.Size([32, 1, 1024])
y_hat = self.fc(outputs)
# torch.Size([32, 1, 4556])
return y_hat, hidden_state, cell_state
class Seq2Seq(nn.Module):
def __init__(self, encoder, decoder, device, output_size):
super().__init__()
self.encoder = encoder
self.decoder = decoder
self.device = device
self.output_size = output_size
def forward(self, src, trg, teacher_forcing_ratio=0.5):
trg_seq_len = trg.size(0)
batch_size = src.size(1)
outputs = torch.zeros(trg_seq_len, batch_size, self.output_size).to(device)
# torch.Size([*, 32, 4556])
hidden, cell = self.encoder(src)
x = trg[0, :] # getting first tokens of batch as first input to decoder
x = x.unsqueeze(0)
# torch.Size([1, 32])
for i in range(1, trg_seq_len):
y, hidden, cell = self.decoder(x, hidden, cell)
y = y.permute(1, 0, 2)
# torch.Size([1, 32, 4556])
outputs[i] = y
teacher_force = random.random() < teacher_forcing_ratio
x = trg[i].unsqueeze(0) if teacher_force else y.argmax(2)
return outputs
The network seems to be learning, loss is decreasing nicely on the training set, however I’m having trouble decreasing loss beyond a certain point on the validation set. As you can see I’ve already added some regularization and while that has helped a tiny bit, it hasn’t really improvied the results by much.
Especially looking at the actual translations it produces, the results aren’t all that great (though some are 100% correct).
Now, without writing a more complex model (I’m trying to work my way there, just starting out with deep learning), is there anything I can do, to improve the results?
Is there anything I’m doing wrong maybe?
I’m using the Multi30k dataset and the following:
criterion = nn.CrossEntropyLoss(ignore_index=TRG_PAD_IDX)
optimizer = torch.optim.Adam(seq2seq_model.parameters(), lr=0.003)
and also
torch.nn.utils.clip_grad_norm_(seq2seq_model.parameters(), 1)
during training.
Here is the progress I’m seeing while training:
1, progress:5%, train-loss: 4.4046, vald-loss: 3.9111, train-perp: 81.8246, vald-perp: 49.9540
2, progress:10%, train-loss: 3.8298, vald-loss: 3.5941, train-perp: 46.0512, vald-perp: 36.3840
3, progress:15%, train-loss: 3.6142, vald-loss: 3.4860, train-perp: 37.1226, vald-perp: 32.6539
4, progress:20%, train-loss: 3.4447, vald-loss: 3.3735, train-perp: 31.3325, vald-perp: 29.1815
5, progress:25%, train-loss: 3.3201, vald-loss: 3.2923, train-perp: 27.6621, vald-perp: 26.9046
6, progress:30%, train-loss: 3.2181, vald-loss: 3.2581, train-perp: 24.9803, vald-perp: 25.9999
7, progress:35%, train-loss: 3.1210, vald-loss: 3.1831, train-perp: 22.6684, vald-perp: 24.1222
8, progress:40%, train-loss: 3.0291, vald-loss: 3.2246, train-perp: 20.6779, vald-perp: 25.1444
9, progress:45%, train-loss: 2.9529, vald-loss: 3.1308, train-perp: 19.1617, vald-perp: 22.8918
10, progress:50%, train-loss: 2.8898, vald-loss: 3.1745, train-perp: 17.9897, vald-perp: 23.9156
11, progress:55%, train-loss: 2.8239, vald-loss: 3.1459, train-perp: 16.8417, vald-perp: 23.2398
12, progress:60%, train-loss: 2.7759, vald-loss: 3.1733, train-perp: 16.0525, vald-perp: 23.8866
13, progress:65%, train-loss: 2.7048, vald-loss: 3.2212, train-perp: 14.9520, vald-perp: 25.0574
...
Any pointers or feedback would be much appreciated!