Size issues with RNN module

1

We are modifying an example found here to use our own data:

Our data x_data and y_data are 47x13 and 47x5

This is the code:

import torch.nn as nn
from torch.nn import functional as F
from torch.autograd import Variable
from torch import optim
import numpy as np
import math, random
import pandas

x_csv = pandas.read_csv('X_train_3.csv')
x_data = torch.Tensor(x_csv.values)

y_csv = pandas.read_csv('Y_train_3.csv')
y_data = torch.Tensor(y_csv.values)

#x = Variable(x_data)
#y = Variable(y_data)

# Define the model


class SimpleRNN(nn.Module):
    def __init__(self, hidden_size):
        super(SimpleRNN, self).__init__()
        self.hidden_size = hidden_size

        self.inp = nn.Linear(1, hidden_size)
        self.rnn = nn.LSTM(hidden_size, hidden_size, 2, dropout=0.05)
        self.out = nn.Linear(hidden_size, 1)

    def step(self, input, hidden=None):
        input = self.inp(input.view(1, -1)).unsqueeze(1)
        output, hidden = self.rnn(input, hidden)
        output = self.out(output.squeeze(1))
        return output, hidden

    def forward(self, inputs, hidden=None, force=True, steps=0):
        if force or steps == 0: steps = len(inputs)
        outputs = Variable(torch.zeros(steps, 1, 1))
        for i in range(steps):
            if force or i == 0:
                input = inputs[i]
            else:
                input = output
            output, hidden = self.step(input, hidden)
            outputs[i] = output
        return outputs, hidden

n_epochs = 100
n_iters = 50
hidden_size = 13

model = SimpleRNN(hidden_size)
criterion = nn.MSELoss()
optimizer = optim.SGD(model.parameters(), lr=0.01)

losses = np.zeros(n_epochs) # For plotting

for epoch in range(n_epochs):

    for iter in range(n_iters):
        inputs = Variable((x_data[:-1]).float())
        targets = Variable((y_data[1:]).float())

        # Use teacher forcing 50% of the time
        force = random.random() < 0.5
        outputs, hidden = model(inputs, None, force)

        optimizer.zero_grad()
        loss = criterion(outputs, targets)
        loss.backward()
        optimizer.step()

        losses[epoch] += loss.data[0]

    if epoch > 0:
        print(epoch, loss.data[0])

We are having size issues in the forward function at self.step(…).

2

maybe the size issues are indeed expected. It’s possible that the script’s model expects a minimum size in a particular dimension.

3

I am having the same issue using the updated torch RNN. I am sure there is something fundamental I am missing.

The issue in with updated torch RNN occurs in loss function loss_fn. Where the results of the RNN gives a 3D matrix while the function is expecting a 2D results. I am unsure how to deal with this.

Example below:

import torch
from torch.autograd import Variable
import pandas
from torch.optim.lr_scheduler import ReduceLROnPlateau
import math, random

x_csv = pandas.read_csv(‘X_train_3.csv’)
x_data = torch.Tensor(x_csv.values)

y_csv = pandas.read_csv(‘Y_train_3.csv’)
y_data = torch.Tensor(y_csv.values)

D_in = 13
H = 6
D_out = 5

x = Variable(x_data)
y = Variable(y_data)

rnn = torch.nn.RNN(input_size=13, hidden_size=5, num_layers=1, nonlinearity=‘tanh’, bias=True, batch_first=False, bidirectional=False)

loss_fn = torch.nn.MSELoss(size_average=False)

learning_rate = 1e-2
optimizer = torch.optim.SGD(rnn.parameters(), lr=learning_rate, momentum=0.25, nesterov=True)
scheduler = ReduceLROnPlateau(optimizer, ‘min’, patience=0, threshold = 1e-1, factor = 0.90, min_lr=1e-25, verbose = False) #

for epoch in range(5000000):
# Forward pass: compute predicted y by passing x to the model.

for i in range(47):
    y_pred, h_n = rnn(x)

    # Compute and print loss.
    loss = loss_fn(y_pred, y)

    if (epoch%10000==0):
        print(epoch, loss.data[0])

    prev_lr = optimizer.param_groups[0]['lr']
    # Before the backward pass, use the optimizer object to zero all of the
    # gradients for the variables it will update (which are the learnable weights
    # of the model)
    optimizer.zero_grad()

    # Backward pass: compute gradient of the loss with respect to model
    # parameters
    loss.backward()


    # Calling the step function on an Optimizer makes an update to its
    # parameters

    #optimizer.step()
    #scheduler.step(loss.data[0])
    scheduler.step(loss.data[0], epoch)

    new_lr = optimizer.param_groups[0]['lr']

    if prev_lr == new_lr:
        optimizer.param_groups[0]['lr'] = new_lr * 1.001

    optimizer.step()

pass