Hi, I’m trying to do some tutorial for myself with my recent research topic.
I have sequential images, classify the labels using LSTM.
trying to extract CNN features from the sequential images and put them into LSTM for classification, and facing some problem.
here is my code.
class CNN_RNN(nn.Module):
def __init__(self, input_size, hidden_size, num_layers, num_classes):
super(CNN_RNN, self).__init__()
self.hidden_size = hidden_size
self.num_layers = num_layers
self.conv1 = nn.Conv2d(3,6,5)
self.pool = nn.MaxPool2d(2,2)
self.conv2 = nn.Conv2d(6,16,5)
self.relu = nn.ReLU()
self.bfc1 = nn.Linear(53,120)
self.lstm = nn.LSTM(120, hidden_size, num_layers, batch_first=True)
self.fc = nn.Linear(hidden_size, num_classes)
def forward(self, x):
#CNN
x = self.conv1(x)
x = F.relu(x)
x = self.pool(x)
x = self.conv2(x)
x = F.relu(x)
x = self.pool(x)
x = self.bfc1(x)
h0 = Variable(torch.zeros(self.num_layers, x.size(0), self.hidden_size))
c0 = Variable(torch.zeros(self.num_layers, x.size(0), self.hidden_size))
#forward propagate rnn
out,_ = self.lstm(x, (h0, c0))
#decode hidden state of last time step
out = self.fc(out[:,-1, :])
return out
input_size = [244,244]
hidden_size = 6
output_size = len(loaded.classes)
num_layers = 3
model = CNN_RNN(input_size, hidden_size, num_layers, output_size)
loss_function = nn.NLLLoss()
optimizer = optim.SGD(model.parameters(), lr=0.1)
# See what the scores are before training
print trainloader
dataiter = iter(trainloader)
images, labels = dataiter.next()
print images.size()
inputs = (images[0][0], labels)
print inputs
print '-----------------------'
tag_scores = model(Variable(images))
print(tag_scores)
What I just simply want to see is it running, but it isn’t.
here is the error message.
---------------------------------------------------------------------------
RuntimeError Traceback (most recent call last)
<ipython-input-225-35a06775e1f4> in <module>()
21 print inputs
22 print '-----------------------'
---> 23 tag_scores = model(Variable(images))
24 print(tag_scores)
25
/usr/local/lib/python2.7/dist-packages/torch/nn/modules/module.pyc in __call__(self, *input, **kwargs)
222 for hook in self._forward_pre_hooks.values():
223 hook(self, input)
--> 224 result = self.forward(*input, **kwargs)
225 for hook in self._forward_hooks.values():
226 hook_result = hook(self, input, result)
<ipython-input-224-d5df1538679a> in forward(self, x)
39
40 #forward propagate rnn
---> 41 out,_ = self.lstm(x, (h0, c0))
42
43 #decode hidden state of last time step
/usr/local/lib/python2.7/dist-packages/torch/nn/modules/module.pyc in __call__(self, *input, **kwargs)
222 for hook in self._forward_pre_hooks.values():
223 hook(self, input)
--> 224 result = self.forward(*input, **kwargs)
225 for hook in self._forward_hooks.values():
226 hook_result = hook(self, input, result)
/usr/local/lib/python2.7/dist-packages/torch/nn/modules/rnn.pyc in forward(self, input, hx)
160 flat_weight=flat_weight
161 )
--> 162 output, hidden = func(input, self.all_weights, hx)
163 if is_packed:
164 output = PackedSequence(output, batch_sizes)
/usr/local/lib/python2.7/dist-packages/torch/nn/_functions/rnn.pyc in forward(input, *fargs, **fkwargs)
349 else:
350 func = AutogradRNN(*args, **kwargs)
--> 351 return func(input, *fargs, **fkwargs)
352
353 return forward
/usr/local/lib/python2.7/dist-packages/torch/nn/_functions/rnn.pyc in forward(input, weight, hidden)
242 input = input.transpose(0, 1)
243
--> 244 nexth, output = func(input, hidden, weight)
245
246 if batch_first and batch_sizes is None:
/usr/local/lib/python2.7/dist-packages/torch/nn/_functions/rnn.pyc in forward(input, hidden, weight)
82 l = i * num_directions + j
83
---> 84 hy, output = inner(input, hidden[l], weight[l])
85 next_hidden.append(hy)
86 all_output.append(output)
/usr/local/lib/python2.7/dist-packages/torch/nn/_functions/rnn.pyc in forward(input, hidden, weight)
111 steps = range(input.size(0) - 1, -1, -1) if reverse else range(input.size(0))
112 for i in steps:
--> 113 hidden = inner(input[i], hidden, *weight)
114 # hack to handle LSTM
115 output.append(hidden[0] if isinstance(hidden, tuple) else hidden)
/usr/local/lib/python2.7/dist-packages/torch/nn/_functions/rnn.pyc in LSTMCell(input, hidden, w_ih, w_hh, b_ih, b_hh)
38 outgate = F.sigmoid(outgate)
39
---> 40 cy = (forgetgate * cx) + (ingate * cellgate)
41 hy = outgate * F.tanh(cy)
42
/usr/local/lib/python2.7/dist-packages/torch/autograd/variable.pyc in __mul__(self, other)
827
828 def __mul__(self, other):
--> 829 return self.mul(other)
830 __rmul__ = __mul__
831
/usr/local/lib/python2.7/dist-packages/torch/autograd/variable.pyc in mul(self, other)
337 def mul(self, other):
338 if isinstance(other, Variable):
--> 339 return Mul.apply(self, other)
340 else:
341 assert not torch.is_tensor(other)
/usr/local/lib/python2.7/dist-packages/torch/autograd/_functions/basic_ops.pyc in forward(ctx, a, b)
46 ctx.b_size = b.size()
47 ctx.save_for_backward(a, b)
---> 48 return a.mul(b)
49
50 @staticmethod
RuntimeError: inconsistent tensor size, expected r_ [1 x 14 x 24], t [1 x 14 x 24] and src [1 x 6] to have the same number of elements, but got 336, 336 and 6 elements respectively at /pytorch/torch/lib/TH/generic/THTensorMath.c:902
It seems like I had a mistake about tensor size, but [1*14*24] is equal to 336. So the error message is quite awkward for me (I’m new to python language)
I’m guessing I made plenty of errors while writing this code, but I don’t know where and how.
I’m waiting for your kind reply.