Hi,
I have a time series dataset consisting 72 input values and 72 output values for each entry and I’m trying to train a seq2seq Encoder Decoder network that forecasts the output using the input. Here’s what I have so far.
model;
import torch
import torch.nn as nn
import torch.nn.functional as F
class EncoderRNN(nn.Module):
def __init__(self, input_size, hidden_size, n_layers, dropout=0.1):
super(EncoderRNN, self).__init__()
self.hidden_size = hidden_size
self.num_layers = n_layers
self.gru = nn.GRU(input_size=input_size, hidden_size=hidden_size,
num_layers=n_layers, dropout=dropout)
def forward(self, input):
self.gru.flatten_parameters()
hidden = torch.zeros(self.num_layers, 1, self.hidden_size).cuda()
_, hidden = self.gru(input.unsqueeze(1), hidden)
return hidden
class DecoderRNN(nn.Module):
def __init__(self, output_size, hidden_size, n_layers, dropout=0.1):
super(DecoderRNN, self).__init__()
self.gru = nn.GRU(input_size=output_size, hidden_size=hidden_size,
num_layers=n_layers, dropout=dropout)
self.out = nn.Linear(hidden_size, output_size)
def forward(self, input, hidden):
self.gru.flatten_parameters()
output = input.unsqueeze(1).unsqueeze(1)
output, hidden = self.gru(output, hidden)
output = self.out(output)
output = output.squeeze()
return output, hidden
class Seq2Seq(nn.Module):
def __init__(self, hidden=256, n_layers=2, dropout=0.1):
super().__init__()
self.encoder = EncoderRNN(72, hidden, n_layers, dropout)
self.decoder = DecoderRNN(1, hidden, n_layers, dropout)
def forward(self, src):
outputs = torch.zeros(src.shape[0], src.shape[1]).cuda()
hidden = self.encoder(src)
input = src[:,-1]
output, hidden = self.decoder(input, hidden)
outputs[:,0] = output
for t in range(1,src.shape[1]):
input = src[:,t]
output, hidden = self.decoder(input, hidden)
outputs[:,t] = output
return outputs
simplified training code;
import torch.optim as optim
import torch.utils.data as loader
import data as datapy
trainloader = ...
testloader = ...
model = Seq2Seq(hidden=144, n_layers=2, dropout=0.1)
model.cuda()
mse = torch.nn.MSELoss(reduction='mean').cuda()
optimizer = optim.SGD(model.parameters(), lr=0.001)
num_epochs = 5
for epoch in range(1,num_epochs+1):
model.train()
for i, data in enumerate(trainloader, 1):
input = data[:,:72].cuda()
target = data[:,72:].cuda()
model.zero_grad()
outputs = model(input, target)
loss = mse(outputs, target.float())
loss.backward()
torch.nn.utils.clip_grad_value_(model.parameters(), 0.03)
optimizer.step()
model.eval()
with torch.no_grad():
for i, data in enumerate(testloader, 1):
input = data[:,:72].cuda()
target = data[:,72:].cuda()
outputs = model(input, target)
loss = mse(outputs, target)
Starting from first epoch the network gets stucked at some point and changing lr doesn’t seem to be solving the issue. My dataset consists of only a sensor’s measurements which are always non-zero. My data loader basically normalizes it by dividing each measurement by the maximum_value/2. Any data tensor dataloader gives has the shape of [batch_size, 144], 144 meaning 144 hours of data, I split them by 72 and 72 on train and test runs.
Here’re some plots showing two separate single forward passes for entries from both training and test sets.
Train; blue: actual data, green: forecasts
Test;
Outputs are not exactly same but they are a straight line all the time.
I’m open to any suggestions regarding the network and/or the training process.