RuntimeError: input must have 3 dimensions, got 2

I am getting the error input must have 3 dimensions, got2, How to solving this error?

class LSTMLanguageModel(nn.Module):
    def __init__(self, hidden_dim = 100, TEXT = TEXT, batch_size = 10):
        super(LSTMLanguageModel, self).__init__()
        self.hidden_dim = hidden_dim
        self.batch_size = batch_size
        
        vocab_size, embedding_dim = TEXT.vocab.vectors.shape
        self.embeddings = nn.Embedding(vocab_size, embedding_dim)
        self.embeddings.weight.data.copy_(TEXT.vocab.vectors)
        
        self.lstm = nn.LSTM(input_size = embedding_dim, hidden_size = self.hidden_dim, dropout = 0.50)
        self.linear = nn.Linear(in_features = self.hidden_dim, out_features = vocab_size)
        self.drop = nn.Dropout(p = 0.50)

    def init_hidden(self):
        direction = 2 if self.lstm.bidirectional else 1
        if use_cuda:
            return (Variable(torch.zeros(direction*self.lstm.num_layers, self.batch_size, self.hidden_dim)).cuda(), 
                    Variable(torch.zeros(direction*self.lstm.num_layers, self.batch_size, self.hidden_dim)).cuda())
        else:
            return (Variable(torch.zeros(direction*self.lstm.num_layers, self.batch_size, self.hidden_dim)), 
                    Variable(torch.zeros(direction*self.lstm.num_layers, self.batch_size, self.hidden_dim)))
    
    def detach_hidden(self, hidden):
        """ util function to keep down number of graphs """
        return tuple([h.detach() for h in hidden])
        
    def forward(self, x, hidden, train = True):
        """ predict, return hidden state so it can be used to intialize the next hidden state """
        embedded = self.embeddings(x)
        embedded = self.drop(embedded) if train else embedded
        
        lstm_output, hdn = self.lstm(embedded, hidden)
        reshaped = lstm_output.view(-1, lstm_output.size(2))
        dropped = self.drop(reshaped) if train else reshaped
        
        decoded = self.linear(dropped)
        
        logits = F.log_softmax(decoded, dim = 1)
                
        return logits, self.detach_hidden(hdn)    
    
class Trainer:
    def __init__(self, train_iter, val_iter):
        self.train_iter = train_iter
        self.val_iter = val_iter
        
    def string_to_batch(self, string):
        relevant_split = string.split() # last two words, ignore ___
        ids = [self.word_to_id(word) for word in relevant_split]
        if use_cuda:
            return Variable(torch.LongTensor(ids)).cuda()
        else:
            return Variable(torch.LongTensor(ids))
        
    def word_to_id(self, word, TEXT = TEXT):
        return TEXT.vocab.stoi[word]
    
    def batch_to_input(self, batch):
        ngrams = self.collect_batch_ngrams(batch)
        x = Variable(torch.LongTensor([ngram[:-1] for ngram in ngrams]))
        y = Variable(torch.LongTensor([ngram[-1] for ngram in ngrams]))
        if use_cuda:
            return x.cuda(), y.cuda()
        else:
            return x, y
    
    def collect_batch_ngrams(self, batch, n = 3):
        data = batch.text.view(-1).data.tolist()
        return [tuple(data[idx:idx + n]) for idx in range(0, len(data) - n + 1)]
    
    def train_model(self, model, num_epochs):
        parameters = filter(lambda p: p.requires_grad, model.parameters())
        optimizer = torch.optim.Adam(params = parameters, lr=1e-3)
        criterion = nn.NLLLoss()
        
        for epoch in tqdm_notebook(range(num_epochs)):

            epoch_loss = []
            hidden = model.init_hidden()
            model.train()

            for batch in tqdm_notebook(train_iter):
                x, y = batch.text, batch.target.view(-1)
                if use_cuda: x, y = x.cuda(), y.cuda()

                optimizer.zero_grad()

                y_pred, hidden = model.forward(x, hidden, train = True)

                loss = criterion(y_pred, y)
                loss.backward()

                torch.nn.utils.clip_grad_norm(model.lstm.parameters(), 1)

                optimizer.step()

                epoch_loss.append(loss.data[0])
                
            model.eval()
            train_ppl = np.exp(np.mean(epoch_loss))
            val_ppl = self.validate(model)

            print('Epoch {0} | Loss: {1} | Train PPL: {2} | Val PPL: {3}'.format(epoch+1, np.mean(epoch_loss), train_ppl,  val_ppl))
    
        print('Model trained.')
        self.write_kaggle(model)
        print('Output saved.')
        
    def validate(self, model):
        criterion = nn.NLLLoss()
        hidden = model.init_hidden()
        aggregate_loss = []
        for batch in self.val_iter:
            y_p, _ = model.forward(batch.text, hidden, train = False)
            y_t = batch.target.view(-1)
            
            loss = criterion(y_p, y_t)
            aggregate_loss.append(loss.data[0])        
        val_ppl = np.exp(np.mean(aggregate_loss))
        return val_ppl
    
    def predict_sentence(self, string, model, TEXT = TEXT):
        string = string[:-4]
        model.batch_size = 1
        hidden = model.init_hidden()
        x = self.string_to_batch(string)
        logits, _ = model.forward(x, hidden, train = False)
        argsort_ids = np.argsort(logits[-1].data.tolist())
        out_ids = argsort_ids[-20:][::-1]
        out_words = ' '.join([TEXT.vocab.itos[out_id] for out_id in out_ids])
        return out_words
    
    def write_kaggle(self, model, input_file = '../data/input.txt'):        
        inputs = open(input_file, 'r').read().splitlines()
        outputs = [self.predict_sentence(sentence, model) for sentence in inputs]
        with open('lstm_output.txt', 'w') as f:
            f.write('id,word')
            for idx, line in enumerate(outputs):
                f.write('\n')
                f.write(str(idx) + ',')
                f.write(line)
1 Like

it is not working for me.

Yes that was my fault i didnt see u pass good hidden state and that was pure input fault.
Try change emmbedded in forward function before lstm to 3d maybe something like this -> self.embeddings(x).view(1, 1, -1)
This will be a lot easier if you send full error message with code lines

Here is the which I am getting wrt the I have mentioned above:

Epoch 1 | Loss: 5.17788553238 | Train PPL: 177.307510376 | Val PPL: 129.847122192

Model trained.
('string is', "but while the new york stock exchange did n't fall")
---------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
<ipython-input-21-6c5c0b136b8a> in <module>()
      3     model.cuda()
      4 trainer = Trainer(train_iter = train_iter, val_iter = val_iter)
----> 5 trainer.train_model(model = model, num_epochs = 1)

<ipython-input-20-d6fd9f7bb41a> in train_model(self, model, num_epochs)
     63 
     64         print('Model trained.')
---> 65         self.write_kaggle(model)
     66         print('Output saved.')
     67 

<ipython-input-20-d6fd9f7bb41a> in write_kaggle(self, model, input_file)
     93     def write_kaggle(self, model, input_file = '../data/input.txt'):
     94         inputs = open(input_file, 'r').read().splitlines()
---> 95         outputs = [self.predict_sentence(sentence, model) for sentence in inputs]
     96         with open('lstm_output.txt', 'w') as f:
     97             f.write('id,word')

<ipython-input-20-d6fd9f7bb41a> in predict_sentence(self, string, model, TEXT)
     85         hidden = model.init_hidden()
     86         x = self.string_to_batch(string)
---> 87         logits, _ = model.forward(x, hidden, train = False)
     88         argsort_ids = np.argsort(logits[-1].data.tolist())
     89         out_ids = argsort_ids[-20:][::-1]

<ipython-input-19-70af3154bb0b> in forward(self, x, hidden, train)
     33         embedded = self.drop(embedded) if train else embedded
     34 
---> 35         lstm_output, hdn = self.lstm(embedded, hidden)
     36         reshaped = lstm_output.view(-1, lstm_output.size(2))
     37         dropped = self.drop(reshaped) if train else reshaped

/home/naditya/anaconda2/lib/python2.7/site-packages/torch/nn/modules/module.pyc in __call__(self, *input, **kwargs)
    489             result = self._slow_forward(*input, **kwargs)
    490         else:
--> 491             result = self.forward(*input, **kwargs)
    492         for hook in self._forward_hooks.values():
    493             hook_result = hook(self, input, result)

/home/naditya/anaconda2/lib/python2.7/site-packages/torch/nn/modules/rnn.pyc in forward(self, input, hx)
    176             flat_weight = None
    177 
--> 178         self.check_forward_args(input, hx, batch_sizes)
    179         func = self._backend.RNN(
    180             self.mode,

/home/naditya/anaconda2/lib/python2.7/site-packages/torch/nn/modules/rnn.pyc in check_forward_args(self, input, hidden, batch_sizes)
    124             raise RuntimeError(
    125                 'input must have {} dimensions, got {}'.format(
--> 126                     expected_input_dim, input.dim()))
    127         if self.input_size != input.size(-1):
    128             raise RuntimeError(

RuntimeError: input must have 3 dimensions, got 2

After applying previous reply embembedded = self.embeddings(x).view(1, 1, -1).I got the error.

 def forward(self, x, hidden, train = True):
        """ predict, return hidden state so it can be used to intialize the next hidden state """
      
        embedded = self.embeddings(x).view(1, 1, -1)
        embedded = self.drop(embedded) if train else embedded
        
        lstm_output, hdn = self.lstm(embedded, hidden)
        reshaped = lstm_output.view(-1, lstm_output.size(2))
        dropped = self.drop(reshaped) if train else reshaped
    
        decoded = self.linear(dropped)
        
        logits = F.log_softmax(decoded, dim = 1)
                
        return logits, self.detach_hidden(hdn)    
    
RuntimeError                              Traceback (most recent call last)
<ipython-input-24-6c5c0b136b8a> in <module>()
      3     model.cuda()
      4 trainer = Trainer(train_iter = train_iter, val_iter = val_iter)
----> 5 trainer.train_model(model = model, num_epochs = 1)

<ipython-input-23-d6fd9f7bb41a> in train_model(self, model, num_epochs)
     45                 optimizer.zero_grad()
     46 
---> 47                 y_pred, hidden = model.forward(x, hidden, train = False)
     48 
     49                 loss = criterion(y_pred, y)

<ipython-input-22-a749ecb174fe> in forward(self, x, hidden, train)
     33         embedded = self.drop(embedded) if train else embedded
     34 
---> 35         lstm_output, hdn = self.lstm(embedded, hidden)
     36         reshaped = lstm_output.view(-1, lstm_output.size(2))
     37         dropped = self.drop(reshaped) if train else reshaped

/home/naditya/anaconda2/lib/python2.7/site-packages/torch/nn/modules/module.pyc in __call__(self, *input, **kwargs)
    489             result = self._slow_forward(*input, **kwargs)
    490         else:
--> 491             result = self.forward(*input, **kwargs)
    492         for hook in self._forward_hooks.values():
    493             hook_result = hook(self, input, result)

/home/naditya/anaconda2/lib/python2.7/site-packages/torch/nn/modules/rnn.pyc in forward(self, input, hx)
    176             flat_weight = None
    177 
--> 178         self.check_forward_args(input, hx, batch_sizes)
    179         func = self._backend.RNN(
    180             self.mode,

/home/naditya/anaconda2/lib/python2.7/site-packages/torch/nn/modules/rnn.pyc in check_forward_args(self, input, hidden, batch_sizes)
    128             raise RuntimeError(
    129                 'input.size(-1) must be equal to input_size. Expected {}, got {}'.format(
--> 130                     self.input_size, input.size(-1)))
    131 
    132         if is_input_packed:

RuntimeError: input.size(-1) must be equal to input_size. Expected 300, got 96000


                    

Try to convert ur size via .view() method to format like this -> (input, hx, batch_sizes).
Check https://pytorch.org/docs/stable/tensors.html#torch.Tensor.view

can you please broadly explain?

So error message clearly say LSTM cell expect input to be in 3 dimensions not 2
and following the pytorch docs β†’ input have shape (seq_len, batch, input_size)
So you use .view() method to manipulate ur emmbeding output to have 3d shape with this order (seq_len, batch, input_size)
And this order corresponde to:
seq_len = time dimension (how many time steps do you have)
batch = mini-batch dimension (how much do you split ur data)
input_size = feature dimension (so how many features do you have)
So lets say you have some time series forecasting problem you split your data and you want to predict next 5 time steps using last 5 time steps number of features is 10 and u split ur data into batches of size 10 and total numbers of exaples = 1000.
Your input should have shape β†’ 5 x 100 x 10
Im not doing much NLP so i cant help you with ur embedding output but u can check it manualy and confirm it here torch.nn β€” PyTorch 2.1 documentation

And there are crucial informations about problem

/home/naditya/anaconda2/lib/python2.7/site-packages/torch/nn/modules/rnn.pyc in forward(self, input, hx)
176 flat_weight = None
177
β†’ 178 self.check_forward_args(input, hx, batch_sizes)
179 func = self._backend.RNN(
180 self.mode,

/home/naditya/anaconda2/lib/python2.7/site-packages/torch/nn/modules/rnn.pyc in check_forward_args(self, input, hidden, batch_sizes)
124 raise RuntimeError(
125 β€˜input must have {} dimensions, got {}’.format(
β†’ 126 expected_input_dim, input.dim()))
127 if self.input_size != input.size(-1):
128 raise RuntimeError(

RuntimeError: input must have 3 dimensions, got 2

Thats why i focus on β†’ self.check_forward_args(input, hx, batch_sizes) β†’ cause u fail this check

1 Like