RuntimeError: shape '[20, 15, -1]' is invalid for input of size 1216

I have a little experience with using recurrent neural networks. I am trying to use RNN for reward predictions for reinforcement learning. The inputs of forward module are observations and actions. So I tried to build batch size and sequence length from data to feed the right size of inputs to the RNN network but apparently when after making batches of dataset at the end of training in the first epoch I got error

class MlpRnnEncoder(nn.Module):
    def __init__(self, seq_len, input_dim, hidden_size, num_layers, n_action=0, embed_dim=4, dropout_rate=0.25, rnn_cell_type='GRU',  normalize=False):

        super(MlpRnnEncoder, self).__init__()
        self.normalize = normalize
        self.seq_len = seq_len
        self.input_dim = input_dim
        self.hidden_dim = hidden_size
        self.num_layers = num_layers
        self.rnn_cell_type = rnn_cell_type
        self.n_action = n_action
        if n_action != 0:
            self.act_embedding = nn.Embedding(n_action, embed_dim)

        self.encoder = nn.Sequential()
        self.encoder.add_module('mlp_0', nn.Linear(input_dim, hidden_size))
        self.encoder.add_module('relu', nn.ReLU())
        self.encoder.add_module('dropout', nn.Dropout(dropout_rate))

        if self.rnn_cell_type == 'GRU':
            print('Using GRU as the recurrent layer.')
            self.rnn = nn.GRU(input_size=hidden_size, hidden_size=hidden_size, num_layers =num_layers, batch_first=True)
        elif self.rnn_cell_type == 'LSTM':
            print('Using LSTM as the recurrent layer.')
            self.rnn = nn.LSTM(input_size=hidden_size, hidden_size=hidden_size, num_layers =num_layers, batch_first=True)
            print('Using the default recurrent layer: RNN.')
            self.rnn = nn.RNN(input_size=hidden_size, hidden_size=hidden_size, num_layers =num_layers, batch_first=True)
            self.rnn_cell_type = 'RNN'
        #using information from all hidden states 
        self.traj_embed_layer = nn.Linear(hidden_size*self.seq_len, hidden_size)

    def forward(self, x, y):

        if self.normalize:
            mean = torch.mean(x, dim=(0, 1))[None, None, :]
            std = torch.std(x, dim=(0, 1))[None, None, :]
            x = (x - mean)/std
        if self.n_action != 0:
            y = self.act_embedding(y)
        if len(x.shape) != len(y.shape):
            y = torch.unsqueeze(y,-1)
        input =, y), -1).float()
        state_size =input.shape[-1]
        mlp_encoded = self.encoder(input) # (N, T, Hiddens[-2]) get the hidden representation of every time step.
        mlp_encoded=mlp_encoded.view(self._batch_size, self.seq_len, -1)

        h0 = torch.zeros(self.num_layers, mlp_encoded.size(0),self.hidden_dim ).to(device)
        #forward prop
        if self.rnn_cell_type in ['GRU', 'RNN']:
            step_embed, traj_embed = self.rnn(mlp_encoded, h0) #the first one is important and secound one is the hidden state
            c0 = torch.zeros(self.num_layers, mlp_encoded.size(0) , self.hidden_dim ).to(device)
            step_embed, traj_embed = self.rnn(mlp_encoded, (h0, c0))
        new_input = step_embed.reshape(step_embed.shape[0], -1)
        step_embed = self.traj_embed_layer(new_input) # (batch_size, hidden_dim)
        return step_embed, traj_embed

epochs_iter = tqdm.notebook.tqdm(range(num_epochs), desc="GP training Epoch")
for i in epochs_iter:
    loss_avg = 0.
    batch_counter = 0.
    minibatch_iter = tqdm.notebook.tqdm(train_loader, desc="Minibatch", leave=False)  
    for x_batch, y_batch in train_loader:
        pred = model(x_batch)
        y_batch_norm=model.normalize(y_batch.cpu().numpy() )        output=torch.from_numpy(y_batch_norm).float().to(device).view(model.encoderRNN._batch_size, model.encoderRNN.seq_len, -1).squeeze()
        loss = -mll(pred, output)

Here is my error:

torch.Size([300, 14])
torch.Size([300, 14])
torch.Size([19, 14])
RuntimeError                              Traceback (most recent call last)
<ipython-input-15-09a76dd4a16c> in <module>
    483         hyperparameter_optimizer.zero_grad()
    484         print(x_batch.shape)
--> 485         pred = model(x_batch)
    486         y_batch_norm=torch.from_numpy(model.normalize(y_batch.cpu().numpy() )).to(device)

3 frames
<ipython-input-15-09a76dd4a16c> in forward(self, x, y)
    107         if self._batch_size%self.seq_len != 0.0 :
    108             raise ValueError("Invalid batch size or sequence length")
--> 109         mlp_encoded=mlp_encoded.view(batch, self.seq_len, -1)
    111         seq_length=torch.tensor([batch,self.seq_len],dtype=torch.int64, device='cpu')

RuntimeError: shape '[20, 15, -1]' is invalid for input of size 1216

How can I fix this problem?

I’m not sure what exactly these shapes represent, but I assume dim0 is the batch size:

torch.Size([300, 14])
torch.Size([300, 14])
torch.Size([19, 14])

If so, then note that you are apparently changing the batch size which sounds wrong.
Also, could you explain the general shape of the intermediate tensors as it seems dim0 is representing a “length”:


while you are later trying to split the dimension to a batch size and a length:

        mlp_encoded=mlp_encoded.view(self._batch_size, self.seq_len, -1)

The length of this data was roughly ~60000. I am not pretty sure because I generated a new dataset by running the gym environment and each episode has different lengths even by keeping the total 2000 episodes fixed.