Embeddings index out of range error

1 
class LSTMModel(nn.Module):
    def __init__(self, lstm_size=50, linear_size=50, vocab_size=20000):
        super(LSTMModel, self).__init__()
        
        self.linear_size = linear_size
        self.lstm_size = lstm_size
        
        self.embeds = nn.Embedding(vocab_size, 128)
        self.lstm = nn.LSTM(128, lstm_size)
        self.dropout = nn.Dropout(.1)
        self.fc1 = nn.Linear(lstm_size, linear_size) # * 2 for bidirection
        self.fc2 = nn.Linear(linear_size, 6)
        
    def forward(self, x):
        h0 = Variable(torch.zeros(1, x.size(1), self.lstm_size)) # 2 for bidirection 
        c0 = Variable(torch.zeros(1, x.size(1), self.lstm_size))
        out = self.embeds(x)
        out = out.view(len(x), x.size(1), -1)

        
        out, _ = self.lstm(out, (h0, c0))
        out = self.dropout(out[-1])
                
        out = self.fc1(out)
        out = torch.nn.functional.relu(out)
        out = self.dropout(out)
        
        out = self.fc2(out)
        out = torch.nn.functional.sigmoid(out)
        
        return out

If I forward pass with a batch size bigger than 1 I get this error:

    RuntimeError                              Traceback (most recent call last)
    <ipython-input-219-ce43ac67861e> in <module>()
          1 t = dataset[:10][0]
    ----> 2 model(t.t())

~/anaconda3/lib/python3.6/site-packages/torch/nn/modules/module.py in __call__(self, *input, **kwargs)
    323         for hook in self._forward_pre_hooks.values():
    324             hook(self, input)
--> 325         result = self.forward(*input, **kwargs)
    326         for hook in self._forward_hooks.values():
    327             hook_result = hook(self, input, result)

<ipython-input-217-478ca91c0a21> in forward(self, x)
     20         h0 = Variable(torch.zeros(1, x.size(1), self.lstm_size)) # 2 for bidirection
     21         c0 = Variable(torch.zeros(1, x.size(1), self.lstm_size))
---> 22         out = self.embeds(x)
     23         out = out.view(len(x), x.size(1), -1)

~/anaconda3/lib/python3.6/site-packages/torch/nn/modules/module.py in __call__(self, *input, **kwargs)
    323         for hook in self._forward_pre_hooks.values():
    324             hook(self, input)
--> 325         result = self.forward(*input, **kwargs)
    326         for hook in self._forward_hooks.values():
    327             hook_result = hook(self, input, result)

~/anaconda3/lib/python3.6/site-packages/torch/nn/modules/sparse.py in forward(self, input)
    101             input, self.weight,
    102             padding_idx, self.max_norm, self.norm_type,
--> 103             self.scale_grad_by_freq, self.sparse
    104         )
    105 

~/anaconda3/lib/python3.6/site-packages/torch/nn/_functions/thnn/sparse.py in forward(cls, ctx, indices, weight, padding_idx, max_norm, norm_type, scale_grad_by_freq, sparse)
     57             output = torch.index_select(weight, 0, indices)
     58         else:
---> 59             output = torch.index_select(weight, 0, indices.view(-1))
     60             output = output.view(indices.size(0), indices.size(1), weight.size(1))
     61 

RuntimeError: index out of range at /Users/soumith/minicondabuild3/conda-bld/pytorch_1512381214802/work/torch/lib/TH/generic/THTensorMath.c:277
3 Likes
2

The traceback indicates that (in sparse.py) you’re trying to index_select weight with something that’s out of range. What is the shape of your input to forward?

1 Like
3

For anyone else getting this error: check that you’re not passing in any lookup indices larger than the size of your embedding matrix. This might happen if you’re encoding your input with an additional UNK index but didn’t include in when specifying the size of your embedding layer.

10 Likes
4

@cosmoserdean Did you find a fix to this? Having the exact same problem.

5

That’s what I was doing wrong – thanks! AveryLiu’s response to this stackoverflow post has a good example of how to use nn.Embedding properly, to make sure you’re initializing it with the right arguments.

2 Likes
6

I have the same problem. When I pass batch size of more than 1, I get an error index out of range. Did any one found a solution?

1 Like
7

@Sohaib_Mian This shouldn’t have anything to do with your batch size, but the values that your input tensor contains. How are you representing your input to the embedding lookup?

8

What if my index is very large, e.g. [10000]? Does the Embedding layer generates a matrix like (10001, embedding_size)? If so, how to improve the efficiency？

9

This kind of error is happening to me help!!!

It runs good in my size of this

        vocab_size:	20
	embedding_dim:	15
	hidden_dim:	10
	n_layers:	2
	output_size:	10

but gives wrong output and error like

RuntimeError: index out of range at /opt/conda/conda-bld/pytorch_1524584710464/work/aten/src/TH/generic/THTensorMath.c:343

in this sizes

vocab_size:	72	
embedding_dim:	256	
hidden_dim:	256	
n_layers:	3	
output_size:	72

My code is below

import torch.nn as nn

class RNN(nn.Module):
    
    def __init__(self, vocab_size, output_size, embedding_dim, hidden_dim, n_layers, dropout=0.5):
        """
        :param vocab_size: The number of input dimensions of the neural network (the size of the vocabulary)
        :param output_size: The number of output dimensions of the neural network
        :param embedding_dim: The size of embeddings, should you choose to use them        
        :param hidden_dim: The size of the hidden layer outputs
        :param dropout: dropout to add in between LSTM/GRU layers
        """
        super(RNN, self).__init__()
        # TODO: Implement function

        
        # set class variables
        self.output_size = output_size
        self.n_layers = n_layers
        self.hidden_dim = hidden_dim
        self.embedding_dim = embedding_dim
        self.vocab_size = vocab_size
        print('vocab_size:', self.vocab_size,
              'embedding_dim:', self.embedding_dim,
              'hidden_dim:',self.hidden_dim,
              'n_layers:',self.n_layers,
              'output_size:',self.output_size,
              sep='\t'
             )
        # define model layers
        self.embedding = nn.Embedding(vocab_size, embedding_dim)
        self.lstm = nn.LSTM(embedding_dim, hidden_dim, n_layers, dropout=dropout, batch_first=True)
        
         
        #linear layer
        self.dropout = nn.Dropout(0.3)
        self.fc = nn.Linear(hidden_dim,output_size)
        self.sig = nn.Sigmoid()
    
    
    def forward(self, nn_input, hidden):
        """
        Forward propagation of the neural network
        :param nn_input: The input to the neural network
        :param hidden: The hidden state        
        :return: Two Tensors, the output of the neural network and the latest hidden state
        """

        # TODO: Implement function  
        #get the batch size
        batch_size = nn_input.size(0)

        #change to long tensor
        nn_input = nn_input.long()
        
        #embedding and lstm_out
        embeds = self.embedding(nn_input)
        
        lstm_out, hidden = self.lstm(embeds,hidden)
        
        #stack up lstm outputs
        lstm_out = lstm_out.contiguous().view(-1, self.hidden_dim)
        
        #dropout and fully-connected layer
        out = self.dropout(lstm_out)
        out = self.fc(out)
        
        out = out.view(batch_size, -1, self.output_size)
        
        out = out[:, -1] # last batch of labels
        
        
        # return one batch of output word scores and the hidden state
        return out, hidden
    
    
    def init_hidden(self, batch_size):
        '''
        Initialize the hidden state of an LSTM/GRU
        :param batch_size: The batch_size of the hidden state
        :return: hidden state of dims (n_layers, batch_size, hidden_dim)
        '''
        # Implement function
        weight = next(self.parameters()).data
        # initialize hidden state with zero weights, and move to GPU if available
        if(train_on_gpu):
            hidden = (weight.new(self.n_layers, batch_size, self.hidden_dim).zero_().cuda(),
                      weight.new(self.n_layers, batch_size, self.hidden_dim).zero_().cuda())
        else:
            hidden = (weight.new(self.n_layers, batch_size, self.hidden_dim).zero_(),
                      weight.new(self.n_layers, batch_size, self.hidden_dim).zero_())
            
        return hidden

I tried to debug it it specifically started giving error in
self.embedding = nn.Embedding(vocab_size, embedding_dim)

Any help will be appretiated and Thank your for response.

2 Likes
10

Same problem for me ‘nn.Embedding’ not working

no_layers = 3
vocab_size = padded_rev.shape[0] + 1 #extra 1 for padding
embedding_dim = 400
output_dim = 1
hidden_dim = 256
train_on_gpu = False

model = SentimentRNN(no_layers,seq_length,embedding_dim,hidden_dim,drop_prob=0.5,
                 train_on_gpu=train_on_gpu)
print(model)


SentimentRNN(
  (embedding): Embedding(49975, 400)
  (lstm): LSTM(400, 256, num_layers=3, batch_first=True, dropout=0.5)
  (dropout): Dropout(p=0.3, inplace=False)
  (fc): Linear(in_features=256, out_features=1, bias=True)
  (sig): Sigmoid()
)

Unable to train model.It is giving index error.

11

Could you check the min and max values of the input tensors you are passing to the embedding layer and make sure their values are in the valid range [0, num_embeddings-1]?

CC @mathematics

3 Likes
12

@ptrblck tx for the response.I figured out my mistake.The value i had given for embedding layer was wrong.Now I made it correct.

13

Hi @ptrblck,

I’ve found error was on my batch_size function and its internal logic. I was making batches of words given text as
batch_data(words, sequence_length, batch_size)

I was given a problem to Implement the batch_data function to batch words data into chunks of size batch_size using the TensorDataset and DataLoader classes

My illogical code was

def batch_data(words, sequence_length, batch_size):
    batch_size_total = batch_size * sequence_length
    n_batches = len(words) // batch_size_total
    words = words[:n_batches*batch_size_total]
    x = np.arange(len(words)).reshape(batch_size,sequence_length)
    y = x.T[-1] + 1
    feature_tensors = torch.from_numpy(x)
    target_tensors =   torch.from_numpy(y)

    data = TensorDataset(feature_tensors, target_tensors)
    data_loader = torch.utils.data.DataLoader(data,  batch_size=batch_size, shuffle=True)
    return data_loader

I’m trying to solving the problem. Thanks and appretitate for you support :slight_smile: .

edit:
I think i have solved it . I had to change the x values.

14

@ptrblck
Hi, i have the same problem, but i tried your suggestion and the error released again, can you help me please?
Thanks

max_len=250
min_count=50
batch_size=11

LSTM parameters

embed_size =251
hidden_size = 256
num_layers = 1

training parameters

lr = 0.001
num_epochs = 10

model = LSTMClassifier(embed_size=embed_size,
hidden_size=hidden_size,
vocab_size=vocab_size, # my vocab_size is 67
num_layers=num_layers,
num_classes=train_ds.num_classes,
batch_size=batch_size)

if use_gpu:
model = model.cuda()

criterion = nn.CrossEntropyLoss()
if use_gpu:
criterion = criterion.cuda()

optimizer = optim.Adam(model.parameters(), lr=lr, betas=(0.7, 0.99))
scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=1, gamma=0.975)

`hist = train(model, train_dl, valid_dl, criterion, optimizer, scheduler, num_epochs)`

IndexError                                Traceback (most recent call last)
<ipython-input-192-7e0f888e140e> in <module>
----> 1 hist = train(model, train_dl, valid_dl, criterion, optimizer, scheduler, num_epochs)

~\train_utils.py in train(model, train_dl, valid_dl, criterion, optimizer, scheduler, num_epochs)
     85 
     86         ## perform one epoch of training and validation
---> 87         trn_loss, trn_acc = train_step(model, train_dl, criterion, optimizer, scheduler)
     88         val_loss, val_acc = validate_step(model, valid_dl, criterion)
     89 

~\train_utils.py in train_step(model, train_dl, criterion, optimizer, scheduler)
     29         model.zero_grad()
     30 
---> 31         output = model(train_inputs.t())
     32 
     33         loss = criterion(output, train_labels)

C:\ProgramData\Anaconda3\lib\site-packages\torch\nn\modules\module.py in _call_impl(self, *input, **kwargs)
    720             result = self._slow_forward(*input, **kwargs)
    721         else:
--> 722             result = self.forward(*input, **kwargs)
    723         for hook in itertools.chain(
    724                 _global_forward_hooks.values(),

<ipython-input-181-360ed93de0e5> in forward(self, x)
     24 
     25     def forward(self, x):
---> 26         x = self.embedding(x)
     27         x, self.hidden = self.lstm(x, self.hidden)
     28         x = self.fc(x[-1])  # select the last output

C:\ProgramData\Anaconda3\lib\site-packages\torch\nn\modules\module.py in _call_impl(self, *input, **kwargs)
    720             result = self._slow_forward(*input, **kwargs)
    721         else:
--> 722             result = self.forward(*input, **kwargs)
    723         for hook in itertools.chain(
    724                 _global_forward_hooks.values(),

C:\ProgramData\Anaconda3\lib\site-packages\torch\nn\modules\sparse.py in forward(self, input)
    122 
    123     def forward(self, input: Tensor) -> Tensor:
--> 124         return F.embedding(
    125             input, self.weight, self.padding_idx, self.max_norm,
    126             self.norm_type, self.scale_grad_by_freq, self.sparse)

C:\ProgramData\Anaconda3\lib\site-packages\torch\nn\functional.py in embedding(input, weight, padding_idx, max_norm, norm_type, scale_grad_by_freq, sparse)
   1812         # remove once script supports set_grad_enabled
   1813         _no_grad_embedding_renorm_(weight, input, max_norm, norm_type)
-> 1814     return torch.embedding(weight, input, padding_idx, scale_grad_by_freq, sparse)
   1815 
   1816 

IndexError: index out of range in self
15

Could you print the min and max values for all inputs as well as the embedding size?
Based on the error message you are passing word indices to an embedding layer, which are not in the expected range.

16

print(len(train_ds))
44

print(len(valid_ds))
22

vocab_size = 2 + len([w for (w, c) in train_ds.vocab.word2count.items() if c >= min_count])
print(vocab_size)
67
i don’t have any other input…
@ptrblck

17

These values would be interesting to see:

print(train_inputs.min())
print(train_inputs.max())

before using model(train_inputs.t()).

18

Thanks for your reply, My inputs are as shown below:
@ptrblck

print(train_inputs.size())
print(train_inputs.min())
print(train_inputs.max())
print(train_inputs)

torch.Size([11, 250])
tensor(0)
tensor(8427)
tensor([[ 243, 97, 1999, …, 3, 19, 166],
[ 0, 0, 0, …, 8083, 2, 8084],
[8293, 1146, 2027, …, 2, 8225, 2],
…,
[ 0, 0, 0, …, 2, 597, 598],
[ 4, 4, 4, …, 34, 2019, 5802],
[5735, 6, 5878, …, 26, 10, 228]])

19

Thanks for the update.
In a previous post you’ve posted: # my vocab_size is 67. If that value is used as num_embeddings in an nn.Embedding module, the error is expected, as your inputs have to stay in the expected range [0, num_embeddings-1], so [0, 66] in your case, while the max. value is 8427.

20

@ptrblck
Thanks for your attention,
Sorry for my questions, i am new in python and deep learning.
can you help me that this code related to earn vocab_size, is wrong?

vocab_size = 2 + len([w for (w, c) in train_ds.vocab.word2count.items() if c >= min_count])

I should mention that my project is text classification, and i am using persian language.