Code for loss function
def batch_loss(encoder, decoder, X, Y, loss):
batch_size = X.shape[0]
enc_state = None
enc_outputs, enc_state = encoder(X, enc_state)
# 初始化解码器的隐藏状态
dec_state = decoder.begin_state(enc_state)
# 解码器在最初时间步的输入是BOS
dec_input = torch.tensor([out_vocab.stoi[BOS]] * batch_size)
# 我们将使用掩码变量mask来忽略掉标签为填充项PAD的损失
mask, num_not_pad_tokens = torch.ones(batch_size), 0
l = torch.tensor([0])
for y in Y.t():
dec_output, dec_state = decoder(dec_input, dec_state, enc_outputs)
l = l + (mask * loss(dec_output, y)).sum()
dec_input = y # 使用强制教学
num_not_pad_tokens += mask.sum().item()
# 当遇到EOS时，序列后面的词将均为PAD，相应位置的掩码设成0
mask = mask * (y != out_vocab.stoi[EOS]).float()
return l / num_not_pad_tokens
Code for train function
def train(encoder, decoder, dataset, lr, batch_size, num_epochs):
d2lt.params_init(encoder, init=nn.init.xavier_uniform_)
d2lt.params_init(decoder, init=nn.init.xavier_uniform_)
enc_optimizer = optim.Adam(encoder.parameters(), lr=lr)
dec_optimizer = optim.Adam(decoder.parameters(), lr=lr)
loss = nn.CrossEntropyLoss(reduction='none')
data_iter = tdata.DataLoader(dataset, batch_size, shuffle=True)
for epoch in range(num_epochs):
l_sum = 0.0
for X, Y in data_iter:
enc_optimizer.zero_grad()
dec_optimizer.zero_grad()
l = batch_loss(encoder, decoder, X, Y, loss)
l.backward()
enc_optimizer.step()
dec_optimizer.step()
l_sum += l.item()
if (epoch + 1) % 10 == 0:
print("epoch %d, loss %.3f" % (epoch + 1, l_sum / len(data_iter)))
Error
RuntimeError Traceback (most recent call last)
<ipython-input-49-b0ce1fe22758> in <module>
5 decoder = Decoder(len(out_vocab), embed_size, num_hiddens, num_layers,
6 attention_size, drop_prob)
----> 7 train(encoder, decoder, dataset, lr, batch_size, num_epochs)
<ipython-input-48-0faa11e92493> in train(encoder, decoder, dataset, lr, batch_size, num_epochs)
13 dec_optimizer.zero_grad()
14 l = batch_loss(encoder, decoder, X, Y, loss)
---> 15 l.backward()
16 enc_optimizer.step()
17 dec_optimizer.step()
/usr/lib64/python3.6/site-packages/torch/tensor.py in backward(self, gradient, retain_graph, create_graph)
105 products. Defaults to ``False``.
106 """
--> 107 torch.autograd.backward(self, gradient, retain_graph, create_graph)
108
109 def register_hook(self, hook):
/usr/lib64/python3.6/site-packages/torch/autograd/__init__.py in backward(tensors, grad_tensors, retain_graph, create_graph, grad_variables)
91 Variable._execution_engine.run_backward(
92 tensors, grad_tensors, retain_graph, create_graph,
---> 93 allow_unreachable=True) # allow_unreachable flag
94
95
RuntimeError: Function AddBackward0 returned an invalid gradient at index 1 - expected type torch.FloatTensor but got torch.LongTensor
Looking forward to positive helpful.