TypeError: expected torch.LongTensor (got torch.FloatTensor)

nlp
1

From creating the following LSTM code

# Hyper-parameters
layer_dim = 2
time_step = 28
batch_size = 40
num_epochs = 100
input_dim = 1152
hidden_dim = 2048
output_dim = 3846
learning_rate = 0.003

lstm_dataset = data_utils.TensorDataset(torch.Tensor(x), torch.Tensor(y))
lstm_loader = torch.utils.data.DataLoader(dataset=dataset, 
                                           batch_size=batch_size, 
                                           shuffle=True)

class LSTMModel(nn.Module):
    def __init__(self, input_dim, hidden_dim, layer_dim, output_dim):
        super(LSTMModel, self).__init__()
        # Hidden dimensions
        self.hidden_dim = hidden_dim
        
        # Number of hidden layers
        self.layer_dim = layer_dim
        
        # Building your LSTM
        # batch_first=True causes input/output tensors to be of shape
        # (batch_dim, seq_dim, feature_dim)
        self.lstm = nn.LSTM(input_dim, hidden_dim, layer_dim, batch_first=True)
        
        # Readout layer
        self.fc = nn.Linear(hidden_dim, output_dim)
    
    def forward(self, x):
        # Initialize hidden state with zeros
        #######################
        #  USE GPU FOR MODEL  #
        #######################
        h0 = torch.zeros(self.layer_dim, x.size(0), self.hidden_dim)
        
        # Initialize cell state
        c0 = torch.zeros(self.layer_dim, x.size(0), self.hidden_dim)
        
        out, (hn, cn) = self.lstm(x, (h0,c0))
        
        print(out.shape)
        
        # Index hidden state of last time step
        # out.size() --> batch, input(features), hidden_dim
        # out[:, -1, :] --> 100, 100 --> just want last time step hidden states! 
        out = self.fc(out[:, -1, :]) 
        # out.size() --> 100, 10
        return out

lstm_model = LSTMModel(input_dim, hidden_dim, layer_dim, output_dim)


criterion = nn.CrossEntropyLoss()
optimizer = torch.optim.SGD(lstm_model.parameters(), lr=learning_rate)  

epochs = 5
for epoch in range(epochs):
    nn_losses = []
    for i, (x, y) in enumerate(lstm_loader):
       
        output = lstm_model(x)

        loss = torch.LongTensor(criterion(torch.LongTensor(output), torch.LongTensor(output)))

        optimizer.zero_grad()
        loss.backward()
        optimizer.step()
        losses.append(loss.data.mean())
    print('[%d/%d] Loss: %.3f' % (epoch+1, epochs, np.mean(losses)))

I keep getting the following error

---------------------------------------------------------------
RuntimeError                  Traceback (most recent call last)
<ipython-input-65-ffde537e0746> in <module>()
     11         print(y)
     12 
---> 13         loss = criterion(output, output)
     14 
     15         optimizer.zero_grad()

~/anaconda3/envs/fastai/lib/python3.6/site-packages/torch/nn/modules/module.py in __call__(self, *input, **kwargs)
    475             result = self._slow_forward(*input, **kwargs)
    476         else:
--> 477             result = self.forward(*input, **kwargs)
    478         for hook in self._forward_hooks.values():
    479             hook_result = hook(self, input, result)

~/anaconda3/envs/fastai/lib/python3.6/site-packages/torch/nn/modules/loss.py in forward(self, input, target)
    860     def forward(self, input, target):
    861         return F.cross_entropy(input, target, weight=self.weight,
--> 862                                ignore_index=self.ignore_index, reduction=self.reduction)
    863 
    864 

~/anaconda3/envs/fastai/lib/python3.6/site-packages/torch/nn/functional.py in cross_entropy(input, target, weight, size_average, ignore_index, reduce, reduction)
   1548     if size_average is not None or reduce is not None:
   1549         reduction = _Reduction.legacy_get_string(size_average, reduce)
-> 1550     return nll_loss(log_softmax(input, 1), target, weight, None, ignore_index, None, reduction)
   1551 
   1552 

~/anaconda3/envs/fastai/lib/python3.6/site-packages/torch/nn/functional.py in nll_loss(input, target, weight, size_average, ignore_index, reduce, reduction)
   1405                          .format(input.size(0), target.size(0)))
   1406     if dim == 2:
-> 1407         return torch._C._nn.nll_loss(input, target, weight, _Reduction.get_enum(reduction), ignore_index)
   1408     elif dim == 4:
   1409         return torch._C._nn.nll_loss2d(input, target, weight, _Reduction.get_enum(reduction), ignore_index)

RuntimeError: Expected object of type torch.LongTensor but found type torch.FloatTensor for argument #2 'target'

However I try to cast my target variable (y) with torch.LongTensor(y) but keep getting the same error. I’ve found post online to downgrade my version of pytorch to 0.3 but I prefer 1.0

2

I’m not sure what exactly should be calculated in the criterion, but currently you are trying to pass the output of your model as the input and target to nn.CrossEntropyLoss.

Basically the error message comes from torch.LongTensor(output). I wouldn’t recommend to use the “type” tensors but rather cast your tensors using .long() etc.
So output.long() would work, but then the target won’t for your criterion.

After fixing some minor issues in your code (e.g. your model does not return anything in forward), this works for me:

x = torch.randn(batch_size, time_step, input_dim)
y = torch.empty(batch_size, dtype=torch.long).random_(output_dim)

output = lstm_model(x)
loss = criterion(output, y)

Probably the shapes, number of classes etc. might be wrong as I don’t know your exact use case.

3 Likes
4

My criterion is down below. As for the forward function, I didn’t copy the entire function. I edit the post just now

criterion = nn.CrossEntropyLoss()
optimizer = torch.optim.SGD(lstm_model.parameters(), lr=learning_rate)
5

After changing my criterion to the following

optimizer = optim.Adam(lstm_model.parameters())
criterion = nn.MultiLabelSoftMarginLoss()

My bug disappear :slight_smile: Thanks for pointing out the criterion!

1 Like
6

Me too, but I don’t know why. It’s really strange.

1 Like