Input_size error in LSTM: RuntimeError: shape '[10, 30, 1]' is invalid for input of size 150

Hi, everyone,
I am using LSTM to predict the stock index of someday using the ones of 30 days before it as the input only. I think in this example, the size of LSTM input should be [10,30,1],so I use t_x=x.view(10,30,1) to reshape the input. But there is an RuntimeError( shape '[10, 30, 1]' is invalid for input of size 150) when I run the code below, could you please help me find what’s the problem? Thank you:)

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.utils.data import Dataset, DataLoader
from torch.utils.data import TensorDataset


dataread_df=pd.read_csv('D:/Desktop/399300.csv')
dataread_series=pd.Series(dataread_df['value'].values)
plt.plot(dataread_series)
plt.show()


def generate_data_df(series, n):
    if len(series) <= n:
        raise Exception("The Length of series is %d, while affect by (n=%d)." % (len(series), n))
    df = pd.DataFrame()
    for i in range(n):
        df['x%d' % i] = series.tolist()[i:-(n - i)]
    df['y'] = series.tolist()[n:]
    return df
data_df = generate_data_df(dataread_series, 30)

data_numpy=np.array(data_df)
mean=np.mean(data_numpy)
std=np.std(data_numpy)
data_numpy = (data_numpy-mean)/std
train_size=int(len(data_numpy)*0.7)
test_size=len(data_numpy)-train_size
trainset_np=data_numpy[:train_size]
testset_np=data_numpy[train_size:]
train_x_np=trainset_np[:,:30]
train_y_np=trainset_np[:,30:]
test_x_np=testset_np[:,:30]
test_y_np=testset_np[:,30:]

train_x=torch.Tensor(train_x_np)
train_y=torch.Tensor(train_y_np)
test_x=torch.Tensor(test_x_np)
test_y=torch.Tensor(test_y_np)
trainset=TensorDataset(train_x,train_y)
testset=TensorDataset(test_x,test_y)
trainloader = DataLoader(trainset, batch_size=10, shuffle=True)
testloader=DataLoader(testset,batch_size=10,shuffle=True)

class Net(nn.Module):
    def __init__(self):
        super(Net,self).__init__()
        self.rnn=nn.LSTM(input_size=1,hidden_size=64,num_layers=1,batch_first=True)
        self.out=nn.Sequential(nn.Linear(64,1))
    def forward(self,x):
        r_out,(h_n,h_c)=self.rnn(x,None)
        out=self.out(r_out[:,-1,:])
        return out
rnn = Net()
print(rnn)

optimizer = torch.optim.Adam(rnn.parameters(), lr=0.0001)  
criterion = nn.MSELoss()
train_correct=0
test_correct=0
train_total=0
test_total=0
prediction_list=[]

for epoch in range(10):
    running_loss_train=0
    running_loss_test=0
    for i,(x1,y1) in enumerate(trainloader):
        t_x1=x1.view(10,30,1)
        output=rnn(t_x1)
        loss_train=criterion(output,y1)
        optimizer.zero_grad() 
        loss_train.backward() 
        optimizer.step()
        running_loss_train+=loss_train.item()
    for i,(x2,y2) in enumerate(testloader):
        t_x2=x2.view(10,30,1)
        prediction=rnn(t_x2)
        loss_test=criterion(prediction,y2)
        running_loss_test+=loss_test.item()
        prediction_list.append(prediction)
    print('Epoch {} Train Loss:{}, Test Loss:{}'.format(epoch+1,running_loss_train,running_loss_test))
    prediction_list_plot=np.array(prediction_list)
    plt.plot(test_y_np.flatten(),'r-',linewidth=0.1,label='real data')
    plt.plot(prediction_list_plot.flatten(),'b-',linewidth=0.1,label='predicted data')
    plt.show()
print('Finish training')

RuntimeError:

RuntimeError                              Traceback (most recent call last)
<ipython-input-3-fb8cb4c93775> in <module>
     71     running_loss_test=0
     72     for i,(x1,y1) in enumerate(trainloader):
---> 73         t_x1=x1.view(10,30,1)
     74         output=rnn(t_x1)
     75         loss_train=criterion(output,y1)

RuntimeError: shape '[10, 30, 1]' is invalid for input of size 150

It seems the data provided by the DataLoader does not contain enough values to be reshaped to [10, 30, 1]. Could you check the shape of train_x_np?

However, if the error occurs later in the training and the first iterations are running successful, the last batch might be smaller, thus yielding this error.
In this case, you could drop the last batch using drop_last=True in your DataLoader or alternatively reshape to a variable shaped tensor:

t_x1 = x1.view(-1, 30, 1)

As a small side note: if you would like to create tensors from numpy arrays, you should use torch.from_numpy(array).

2 Likes

Thank you for your reply:)
I check the shape of train_x_np and it is (2895, 30).I use torch.from_numpy(array) to create tensors and set drop_last =True, but it has the error :Expected object of scalar type Float but got scalar type Double for argument #4 'mat1', the same if I use t_x = x.view(-1, 30, 1). But I check the type of train_x using print(train_x.dtype), it istorch.float64 rather than double.:face_with_head_bandage: Complete error is like this:

RuntimeError                              Traceback (most recent call last)
<ipython-input-13-9266d8119db2> in <module>
     79     for i,(x1,y1) in enumerate(trainloader):
     80         t_x1=x1.view(10,30,1)
---> 81         output=rnn(t_x1)
     82         loss_train=criterion(output,y1)
     83         optimizer.zero_grad()

D:\Anaconda3\lib\site-packages\torch\nn\modules\module.py in __call__(self, *input, **kwargs)
    487             result = self._slow_forward(*input, **kwargs)
    488         else:
--> 489             result = self.forward(*input, **kwargs)
    490         for hook in self._forward_hooks.values():
    491             hook_result = hook(self, input, result)

<ipython-input-13-9266d8119db2> in forward(self, x)
     59         self.out=nn.Sequential(nn.Linear(64,1))
     60     def forward(self,x):
---> 61         r_out,(h_n,h_c)=self.rnn(x,None)
     62         out=self.out(r_out[:,-1,:])
     63         return out

D:\Anaconda3\lib\site-packages\torch\nn\modules\module.py in __call__(self, *input, **kwargs)
    487             result = self._slow_forward(*input, **kwargs)
    488         else:
--> 489             result = self.forward(*input, **kwargs)
    490         for hook in self._forward_hooks.values():
    491             hook_result = hook(self, input, result)

D:\Anaconda3\lib\site-packages\torch\nn\modules\rnn.py in forward(self, input, hx)
    177         if batch_sizes is None:
    178             result = _impl(input, hx, self._flat_weights, self.bias, self.num_layers,
--> 179                            self.dropout, self.training, self.bidirectional, self.batch_first)
    180         else:
    181             result = _impl(input, batch_sizes, hx, self._flat_weights, self.bias,

RuntimeError: Expected object of scalar type Float but got scalar type Double for argument #4 'mat1'

You would have to cast your tensor to a FloatTensor, since from_numpy keeps the original dtype (which seems to be float64) using: tensor = torch.from_numpy(array).float().

1 Like

Thank you! If I delete the plotting part listed below, It works! But if I include them, there is a ValueError: ValueError: only one element tensors can be converted to Python scalars

    prediction_list_plot=np.array(prediction_list)
    plt.plot(test_y_np.flatten(),'r-',linewidth=0.1,label='real data')
    plt.plot(prediction_list_plot.flatten(),'b-',linewidth=0.1,label='predicted data')
    plt.show()

The complete error is like this:

Epoch 1 Train Loss:160.0811848245794, Test Loss:6.106114995200187
---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
ValueError: only one element tensors can be converted to Python scalars

The above exception was the direct cause of the following exception:

SystemError                               Traceback (most recent call last)
<ipython-input-18-cd9cea2f2ee0> in <module>
     96     prediction_list_plot=np.array(prediction_list)
     97     plt.plot(test_y_np.flatten(),'r-',linewidth=0.1,label='real data')
---> 98     plt.plot(prediction_list_plot.flatten(),'b-',linewidth=0.1,label='predicted data')
     99     plt.show()
    100 print('Finish training')

D:\Anaconda3\lib\site-packages\matplotlib\pyplot.py in plot(*args, **kwargs)
   3361                       mplDeprecation)
   3362     try:
-> 3363         ret = ax.plot(*args, **kwargs)
   3364     finally:
   3365         ax._hold = washold

D:\Anaconda3\lib\site-packages\matplotlib\__init__.py in inner(ax, *args, **kwargs)
   1865                         "the Matplotlib list!)" % (label_namer, func.__name__),
   1866                         RuntimeWarning, stacklevel=2)
-> 1867             return func(ax, *args, **kwargs)
   1868 
   1869         inner.__doc__ = _add_data_doc(inner.__doc__,

D:\Anaconda3\lib\site-packages\matplotlib\axes\_axes.py in plot(self, *args, **kwargs)
   1527 
   1528         for line in self._get_lines(*args, **kwargs):
-> 1529             self.add_line(line)
   1530             lines.append(line)
   1531 

D:\Anaconda3\lib\site-packages\matplotlib\axes\_base.py in add_line(self, line)
   1958             line.set_clip_path(self.patch)
   1959 
-> 1960         self._update_line_limits(line)
   1961         if not line.get_label():
   1962             line.set_label('_line%d' % len(self.lines))

D:\Anaconda3\lib\site-packages\matplotlib\axes\_base.py in _update_line_limits(self, line)
   1980         Figures out the data limit of the given line, updating self.dataLim.
   1981         """
-> 1982         path = line.get_path()
   1983         if path.vertices.size == 0:
   1984             return

D:\Anaconda3\lib\site-packages\matplotlib\lines.py in get_path(self)
    954         """
    955         if self._invalidy or self._invalidx:
--> 956             self.recache()
    957         return self._path
    958 

D:\Anaconda3\lib\site-packages\matplotlib\lines.py in recache(self, always)
    655         if always or self._invalidy:
    656             yconv = self.convert_yunits(self._yorig)
--> 657             y = _to_unmasked_float_array(yconv).ravel()
    658         else:
    659             y = self._y

D:\Anaconda3\lib\site-packages\matplotlib\cbook\__init__.py in _to_unmasked_float_array(x)
   2050         return np.ma.asarray(x, float).filled(np.nan)
   2051     else:
-> 2052         return np.asarray(x, float)
   2053 
   2054 

D:\Anaconda3\lib\site-packages\numpy\core\numeric.py in asarray(a, dtype, order)
    480 
    481     """
--> 482     return array(a, dtype, copy=False, order=order)
    483 
    484 def asanyarray(a, dtype=None, order=None):

SystemError: <built-in method __float__ of Tensor object at 0x00000140EF486558> returned a result with an error set

image

Could you try to append prediction as:

prediction_list.append(prediction.detach().cpu().numpy())

Currently you are also storing the computation graph in the list, which will grow your memory usage unnecessarily. I’m not sure, how numpy handles a list of tensors, but I assume this might be the reason for the error.

1 Like

It turns out another error:ValueError: setting an array element with a sequence. Or do you have suggestions for plotting the test_y curve and prediction curve instead of using my method?

Epoch 1 Train Loss:174.92422260111198, Test Loss:18.06840141862631
---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
<ipython-input-19-f1c9c6efca6c> in <module>
     96     prediction_list_plot=np.array(prediction_list)
     97     plt.plot(test_y_np.flatten(),'r-',linewidth=0.1,label='real data')
---> 98     plt.plot(prediction_list_plot.flatten(),'b-',linewidth=0.1,label='predicted data')
     99     plt.show()
    100 print('Finish training')

D:\Anaconda3\lib\site-packages\matplotlib\pyplot.py in plot(*args, **kwargs)
   3361                       mplDeprecation)
   3362     try:
-> 3363         ret = ax.plot(*args, **kwargs)
   3364     finally:
   3365         ax._hold = washold

D:\Anaconda3\lib\site-packages\matplotlib\__init__.py in inner(ax, *args, **kwargs)
   1865                         "the Matplotlib list!)" % (label_namer, func.__name__),
   1866                         RuntimeWarning, stacklevel=2)
-> 1867             return func(ax, *args, **kwargs)
   1868 
   1869         inner.__doc__ = _add_data_doc(inner.__doc__,

D:\Anaconda3\lib\site-packages\matplotlib\axes\_axes.py in plot(self, *args, **kwargs)
   1527 
   1528         for line in self._get_lines(*args, **kwargs):
-> 1529             self.add_line(line)
   1530             lines.append(line)
   1531 

D:\Anaconda3\lib\site-packages\matplotlib\axes\_base.py in add_line(self, line)
   1958             line.set_clip_path(self.patch)
   1959 
-> 1960         self._update_line_limits(line)
   1961         if not line.get_label():
   1962             line.set_label('_line%d' % len(self.lines))

D:\Anaconda3\lib\site-packages\matplotlib\axes\_base.py in _update_line_limits(self, line)
   1980         Figures out the data limit of the given line, updating self.dataLim.
   1981         """
-> 1982         path = line.get_path()
   1983         if path.vertices.size == 0:
   1984             return

D:\Anaconda3\lib\site-packages\matplotlib\lines.py in get_path(self)
    954         """
    955         if self._invalidy or self._invalidx:
--> 956             self.recache()
    957         return self._path
    958 

D:\Anaconda3\lib\site-packages\matplotlib\lines.py in recache(self, always)
    655         if always or self._invalidy:
    656             yconv = self.convert_yunits(self._yorig)
--> 657             y = _to_unmasked_float_array(yconv).ravel()
    658         else:
    659             y = self._y

D:\Anaconda3\lib\site-packages\matplotlib\cbook\__init__.py in _to_unmasked_float_array(x)
   2050         return np.ma.asarray(x, float).filled(np.nan)
   2051     else:
-> 2052         return np.asarray(x, float)
   2053 
   2054 

D:\Anaconda3\lib\site-packages\numpy\core\numeric.py in asarray(a, dtype, order)
    480 
    481     """
--> 482     return array(a, dtype, copy=False, order=order)
    483 
    484 def asanyarray(a, dtype=None, order=None):

ValueError: setting an array element with a sequence.

I guess the last batch might have another shape compared to the orhers, thus numpy cannot flatten the array properly.
Assuming your model output has the shape [batch_size, 1], try the following:

prediction_list.append(prediction.detach().cpu())
...
predictions = torch.cat(prediction_list).numpy().flatten()
plt.plot(predictions, 'b-')
1 Like

It works! Thank you very much!