I’m using Dataset, and I have:
def __getitem__(self, index):
random_tensor = torch.rand(self.seq_length, self.feature_dim) * 2
random_tensor = random_tensor - 0.5
return random_tensor, random_tensor * 2
So all I’m doing is learning how to multiply a tensor by 2.
My model is:
class BaselineModel(nn.Module):
def __init__(self, feature_dim=1, hidden_size=1, num_layers=2):
super(BaselineModel, self).__init__()
self.num_layers = num_layers
self.hidden_size = hidden_size
self.lstm = nn.LSTM(input_size=feature_dim,
hidden_size=hidden_size, num_layers=num_layers)
def forward(self, x, hidden):
lstm_out, hidden = self.lstm(x, hidden)
return lstm_out, hidden
def init_hidden(self, batch_size):
hidden = torch.zeros(2, 1, 1)
cell = torch.zeros(2, 1, 1)
return (hidden, cell)
So basically, a simple 1 cell, 1 dimension LSTM. My training loop is:
model = BaselineModel(feature_dim=FEATURE_DIM, hidden_size=FEATURE_DIM)
optimizer = optim.Adam(model.parameters(), lr=0.01, weight_decay=0.0001)
loss_fn = torch.nn.MSELoss(reduction='sum')
wandb.watch(model)
for epoch in range(250):
model.train(True) # Set model to training mode
running_loss = 0.0
for i, data in enumerate(data_loaders['train']):
hidden = model.init_hidden(13)
inputs = data[0]
outputs = data[1]
optimizer.zero_grad()
pred, hidden = model(inputs, hidden)
loss = loss_fn(pred, outputs)
torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)
loss.backward()
optimizer.step()
running_loss += loss.data
epoch_loss = running_loss / len(data_loaders['train'])
wandb.log({"Training Loss": epoch_loss})
print('Epoch: {}\tLoss: {:.4f}'.format(epoch, epoch_loss))
However this doesn’t really decrease the LOSS. Starts at 0.9490 and after 250 epochs, stays at 0.92. I realize that an LSTM is overkill for this, but I want to get a VERY basic version working before augmenting with my sequence data