Incompatible addition error

Hi , I am new to deep learning and pytorch I have been trying to implement CBOW model for creating word embeddings
Please help me out I am having an incompatible addition error.Thanks in advance

Code:

import torch
from torch.autograd import Variable
import torch.nn as nn
import torch.nn.functional as F
 import torch.optim as optim
class CBOW(nn.Module):
    def __init__(self,vocab_size,embedding_size,context_size):
        super(CBOW,self).__init__()
        self.fc1 = nn.Linear(vocab_size,embedding_size)
        self.fc2 = nn.Linear(embedding_size,vocab_size)

    def forward(self,x):
        y = []
        for i in xrange(0,174,29):
            y.append(self.fc1(x[:,i:i+29]))
    
        embedding = Variable(torch.zeros(1,128))
        for i in xrange(len(y)):
            embedding = embedding + y[i]
        
        embedding = embedding/len(y)
        x = self.fc2(embedding)
        return [F.softmax(x),embedding]

def make_corpa(data):
    vocab = ""
    for i in data:
        vocab = vocab + " " + i
    vocab.strip(" ")
    corpa = {}
    all_words = list(set(vocab.split(" ")))
    for i in xrange(len(all_words)):
        corpa[all_words[i]] = i

    return [corpa,len(corpa),corpa.keys()]

def conv_vect(word,corpa):
    temp = torch.LongTensor(1,len(corpa)).zero_()
    temp[0][corpa[word]] = 1
    return temp

def train_word2vec(vocab_size,embedding_dim,number_of_epochs,data):
    model = CBOW(vocab_size,embedding_dim,6)
    loss = nn.CrossEntropyLoss()
    context,word = make_training_data(data,3)
    corpa = make_corpa(data)[0]
    optimizer = optim.SGD(model.parameters(),lr= 0.01)
    for epoch in xrange(number_of_epochs):
        for i in xrange(len(context)):
            context_vec_tmp = [conv_vect(j,corpa) for j in context[i]]
            context_vec = Variable(torch.cat(tuple([context_vec_tmp[j] for j in xrange(len(context_vec_tmp))]),1))
            word_vec = Variable(conv_vect(word[i],corpa))
            predict = model(context_vec)[0]
            predicted = torch.LongTensor(predict.size()[0],predict.size()[1]).zero_()
            for i in xrange(predict.size()[1]):
                predicted[0][i] = float(int(predict[0][i].data[0]/torch.max(predict.data[0])))
        
            predicted = Variable(predicted)
            model.zero_grad()
            l = loss(predicted,word_vec)
            l.backward()
            optimizer.step()

    return model

def make_training_data(data,context_size):
    context = []
    word = []
    for i in data:
        temp = i.split(" ")
        for j in xrange(context_size,len(temp)-context_size,1):
             context.append([temp[j - context_size],temp[j - context_size + 1],temp[j - context_size + 2],temp[j +   context_size - 2],temp[j + context_size - 1],temp[j + context_size]])
             word.append(temp[j])

    return context,word

train_word2vec(make_corpa(po)[1],128,10000,po)
# po is a list of sentences

error comes out to be this:

TypeError                                 Traceback (most recent call last)
<ipython-input-12-c4d942812d63> in <module>()
----> 1 train_word2vec(make_corpa(po)[1],128,10000,po)

<ipython-input-10-181483152376> in train_word2vec(vocab_size, embedding_dim, number_of_epochs, data)
     10             context_vec = Variable(torch.cat(tuple([context_vec_tmp[j] for j in xrange(len(context_vec_tmp))]),1))
     11             word_vec = Variable(conv_vect(word[i],corpa))
---> 12             predict = model(context_vec)[0]
     13             predicted = torch.LongTensor(predict.size()[0],predict.size()[1]).zero_()
     14             for i in xrange(predict.size()[1]):

/usr/local/lib/python2.7/dist-packages/torch/nn/modules/module.pyc in __call__(self, *input, **kwargs)
    204 
    205     def __call__(self, *input, **kwargs):
--> 206         result = self.forward(*input, **kwargs)
    207         for hook in self._forward_hooks.values():
    208             hook_result = hook(self, input, result)

<ipython-input-7-33d7cb28d3b4> in forward(self, x)
      8         y = []
      9         for i in xrange(0,174,29):
---> 10             y.append(self.fc1(x[:,i:i+29]))
     11 
     12         embedding = Variable(torch.zeros(1,128))

/usr/local/lib/python2.7/dist-packages/torch/nn/modules/module.pyc in __call__(self, *input, **kwargs)
    204 
    205     def __call__(self, *input, **kwargs):
--> 206         result = self.forward(*input, **kwargs)
    207         for hook in self._forward_hooks.values():
   208             hook_result = hook(self, input, result)

/usr/local/lib/python2.7/dist-packages/torch/nn/modules/linear.pyc in forward(self, input)
     52             return self._backend.Linear()(input, self.weight)
     53         else:
---> 54             return self._backend.Linear()(input, self.weight, self.bias)
     55 
     56     def __repr__(self):

/usr/local/lib/python2.7/dist-packages/torch/nn/_functions/linear.pyc in forward(self, input, weight, bias)
      8         self.save_for_backward(input, weight, bias)
      9         output = input.new(input.size(0), weight.size(0))
   ---> 10         output.addmm_(0, 1, input, weight.t())
     11         if bias is not None:
     12             # cuBLAS doesn't support 0 strides in sger, so we can't use expand

TypeError: addmm_ received an invalid combination of arguments - got (int, int, torch.LongTensor, torch.FloatTensor),     but expected one of:
 * (torch.LongTensor mat1, torch.LongTensor mat2)
 * (torch.SparseLongTensor mat1, torch.LongTensor mat2)
 * (int beta, torch.LongTensor mat1, torch.LongTensor mat2)
 * (int alpha, torch.LongTensor mat1, torch.LongTensor mat2)
 * (int beta, torch.SparseLongTensor mat1, torch.LongTensor mat2)
 * (int alpha, torch.SparseLongTensor mat1, torch.LongTensor mat2)
 * (int beta, int alpha, torch.LongTensor mat1, torch.LongTensor mat2)
  didn't match because some of the arguments have invalid types: (int, int, torch.LongTensor, torch.FloatTensor)
 * (int beta, int alpha, torch.SparseLongTensor mat1, torch.LongTensor mat2)
  didn't match because some of the arguments have invalid types: (int, int, torch.LongTensor, torch.FloatTensor)

Try:

predict = model(context_vec.float())[0]

The weights of your network are FloatTensor, and you are trying to multiply/sum them with your input which is a LongTensor.

Hey thanks for looking into the code ,actually first I did go with FloatTensor but in that case I got a different error like this:

TypeError: FloatClassNLLCriterion_updateOutput received an invalid combination of arguments - got (int, torch.FloatTensor, torch.FloatTensor, torch.FloatTensor, bool, NoneType, torch.FloatTensor), but expected (int state, torch.FloatTensor input, torch.LongTensor target, torch.FloatTensor output, bool sizeAverage, [torch.FloatTensor weights or None], torch.FloatTensor total_weight)

Yes, cross-entropy loss requires longs for the target. I assume your target is word_vec. If you just turn your input to float it should not modify the content of the target.

1 Like

So as in the input to the model should be a float Tensor where as the labels should be long am I correct?

That’s right, just as you said!

Thanks it worked out :slight_smile:

1 Like