Hi, PyTorch community!
I am new to PyTorch and deep learning in general. I am using Elman RNN (Ref) in a regression analysis problem. However, the RNN is always predicting a constant output. I have tried -
- Changing batch size
- Scaling the input and output by a constant factor
but still, the issue persists.
class RNN(nn.Module):
def __init__(self, input_size, output_size, hidden_dim, n_layers):
super(RNN, self).__init__()
self.hidden_dim=hidden_dim
self.rnn = nn.RNN(input_size, hidden_dim, n_layers, batch_first=True)
self.fc = nn.Linear(hidden_dim, output_size)
def forward(self, x, hidden):
# x (batch_size, seq_length, input_size)
# hidden (n_layers, batch_size, hidden_dim)
# r_out (batch_size, time_step, hidden_size)
batch_size = x.size(0)
r_out, hidden = self.rnn(x, hidden)
output = self.fc(r_out)
return output, hidden
# hyperparameters
input_size=3
output_size=3
hidden_dim=14
n_layers=1
lr=0.1
# instantiate an RNN
rnn = RNN(input_size, output_size, hidden_dim, n_layers)
print(rnn)
# MSE loss and Adam optimizer with a learning rate of 0.01
criterion = nn.MSELoss()
optimizer = torch.optim.Adam(rnn.parameters(), lr=lr)
final_hidden = None
# train the RNN
def train(rnn, n_steps, print_every):
hidden = None
for batch_i, step in enumerate(range(n_steps)):
prediction, hidden = rnn(x_tensor, hidden)
final_hidden = hidden
## Representing Memory ##
# make a new variable for hidden and detach the hidden state from its history
# this way, we don't backpropagate through the entire history
hidden = hidden.data
loss = criterion(prediction, y_tensor)
loss_val.append(loss.item())
optimizer.zero_grad()
loss.backward()
optimizer.step()
if batch_i%print_every == 0 or batch_i == n_steps-1:
print("Epoch: {0}/{1}".format(batch_i+1, n_steps))
print('Loss: ', loss.item())
print(prediction.data)
return rnn
Thanks for any advice in advance!