Pytorch rnn error

I am getting the following error , trying to create a simple rnn for implementing ctc based neural net matrices expected, got 3D, 2D tensors

import torch 
import torch.nn as nn
import torchvision.datasets as dsets
import torchvision.transforms as transforms
from torch.autograd import Variable
import torch.optim as optim
import numpy as np
import matplotlib
from __future__ import print_function

# Hyper Parameters
sequence_length = 2
input_size = 13
hidden_size = 4
num_layers = 2
num_classes = 3
batch_size = 1
num_epochs = 2
learning_rate = 0.01

class RNN(nn.Module):
    def __init__(self, input_size, hidden_size, num_layers, num_classes):
        super(RNN, self).__init__()
        self.hidden_size = hidden_size
        self.num_layers = num_layers
        self.rnn = nn.RNN(input_size, hidden_size, num_layers)
        self.fc = nn.Linear(hidden_size, num_classes)
    
    def forward(self, x):
        # Set initial states 
        h0 = Variable(torch.zeros(self.num_layers, 1 ,1, self.hidden_size)) 
        
        
        # Forward propagate RNN
        out, out2 = self.rnn(x, h0)  
        
        # Decode hidden state of last time step
       # out = self.fc(out[:, -1, :])  
        return out

rnn = RNN(input_size, hidden_size, num_layers, num_classes)

criterion = nn.CrossEntropyLoss()
optimizer = torch.optim.Adam(rnn.parameters(), lr=learning_rate)

input = Variable(torch.randn(sequence_length, 1, 13))
h0 = Variable(torch.randn(2, 1, num_classes))
print(input)
output = rnn.forward(input)

your variable out has 3 dimensions (sequence length, batch, hidden_size), and you must feed your linear layer with a 2 dimensions tensor.

You should, for example, reshape your output like this:

out = out.view(-1, hidden_size) 
1 Like

Thanks for your reply , but I dont think thats the problem , that line is commented out , so it should atleast result intermediate out variable , when I try printing that out I get this error

can you post the error message stack?

I think you got your hidden state Variable with wrong dimensions. I haven’t executed your code, but I think it should be:
h0 = Variable(torch.zeros(self.num_layers, 1 ,self.hidden_size)) as it stands for num_layers, batch_size, hidden_size.

Regards

This is the error message stack

` RuntimeError Traceback (most recent call last)
in ()
3 h0 = Variable(torch.randn(2, 1, num_classes))
4
----> 5 print( rnn.forward(input))

in forward(self, x)
13
14 # Forward propagate RNN
—> 15 out, out2 = self.rnn(x, h0)
16
17 # Decode hidden state of last time step

/usr/local/lib/python3.5/dist-packages/torch/nn/modules/module.py in call(self, *input, **kwargs)
204
205 def call(self, *input, **kwargs):
–> 206 result = self.forward(*input, **kwargs)
207 for hook in self._forward_hooks.values():
208 hook_result = hook(self, input, result)

/usr/local/lib/python3.5/dist-packages/torch/nn/modules/rnn.py in forward(self, input, hx)
89 dropout_state=self.dropout_state
90 )
—> 91 output, hidden = func(input, self.all_weights, hx)
92 if is_packed:
93 output = PackedSequence(output, batch_sizes)

/usr/local/lib/python3.5/dist-packages/torch/nn/_functions/rnn.py in forward(input, *fargs, **fkwargs)
341 else:
342 func = AutogradRNN(*args, **kwargs)
–> 343 return func(input, *fargs, **fkwargs)
344
345 return forward

/usr/local/lib/python3.5/dist-packages/torch/nn/_functions/rnn.py in forward(input, weight, hidden)
241 input = input.transpose(0, 1)
242
–> 243 nexth, output = func(input, hidden, weight)
244
245 if batch_first and batch_sizes is None:

/usr/local/lib/python3.5/dist-packages/torch/nn/_functions/rnn.py in forward(input, hidden, weight)
81 l = i * num_directions + j
82
—> 83 hy, output = inner(input, hidden[l], weight[l])
84 next_hidden.append(hy)
85 all_output.append(output)

/usr/local/lib/python3.5/dist-packages/torch/nn/_functions/rnn.py in forward(input, hidden, weight)
110 steps = range(input.size(0) - 1, -1, -1) if reverse else range(input.size(0))
111 for i in steps:
–> 112 hidden = inner(input[i], hidden, *weight)
113 # hack to handle LSTM
114 output.append(isinstance(hidden, tuple) and hidden[0] or hidden)

/usr/local/lib/python3.5/dist-packages/torch/nn/_functions/rnn.py in RNNTanhCell(input, hidden, w_ih, w_hh, b_ih, b_hh)
16
17 def RNNTanhCell(input, hidden, w_ih, w_hh, b_ih=None, b_hh=None):
—> 18 hy = F.tanh(F.linear(input, w_ih, b_ih) + F.linear(hidden, w_hh, b_hh))
19 return hy
20

/usr/local/lib/python3.5/dist-packages/torch/nn/functional.py in linear(input, weight, bias)
447 def linear(input, weight, bias=None):
448 state = _functions.linear.Linear()
–> 449 return state(input, weight) if bias is None else state(input, weight, bias)
450
451

/usr/local/lib/python3.5/dist-packages/torch/nn/functions/linear.py in forward(self, input, weight, bias)
8 self.save_for_backward(input, weight, bias)
9 output = input.new(input.size(0), weight.size(0))
—> 10 output.addmm
(0, 1, input, weight.t())
11 if bias is not None:
12 # cuBLAS doesn’t support 0 strides in sger, so we can’t use expand

RuntimeError: matrices expected, got 3D, 2D tensors at /b/wheel/pytorch-src/torch/lib/TH/generic/THTensorMath.c:1232 `

@santi-pdp Thanks it worked , I am new to pytorch and deeplearning framework trying hard to learn :slight_smile:

I found out what I did wrong

from docs
" h_0 (num_layers * num_directions, batch, hidden_size): tensor containing the initial hidden state for each element in the batch. "

I got confused between num_layers * num_directions and treated them as two params instead of one , Thanks for help again

@saurabhvyas I’m having similar issues, can you point me to where in the docs you saw num_layers * num_directions? Specifically, what does num_directions refer to here?

thanks.

I found the source here. num_directions == 2 if it’s bi-directional.

1 Like

So can the num_directions be 3 or more? for multh bi-directional rnn model?