Hi , I am new to deep learning and pytorch I have been trying to implement CBOW model for creating word embeddings
Please help me out I am having an incompatible addition error.Thanks in advance
Code:
import torch
from torch.autograd import Variable
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
class CBOW(nn.Module):
def __init__(self,vocab_size,embedding_size,context_size):
super(CBOW,self).__init__()
self.fc1 = nn.Linear(vocab_size,embedding_size)
self.fc2 = nn.Linear(embedding_size,vocab_size)
def forward(self,x):
y = []
for i in xrange(0,174,29):
y.append(self.fc1(x[:,i:i+29]))
embedding = Variable(torch.zeros(1,128))
for i in xrange(len(y)):
embedding = embedding + y[i]
embedding = embedding/len(y)
x = self.fc2(embedding)
return [F.softmax(x),embedding]
def make_corpa(data):
vocab = ""
for i in data:
vocab = vocab + " " + i
vocab.strip(" ")
corpa = {}
all_words = list(set(vocab.split(" ")))
for i in xrange(len(all_words)):
corpa[all_words[i]] = i
return [corpa,len(corpa),corpa.keys()]
def conv_vect(word,corpa):
temp = torch.LongTensor(1,len(corpa)).zero_()
temp[0][corpa[word]] = 1
return temp
def train_word2vec(vocab_size,embedding_dim,number_of_epochs,data):
model = CBOW(vocab_size,embedding_dim,6)
loss = nn.CrossEntropyLoss()
context,word = make_training_data(data,3)
corpa = make_corpa(data)[0]
optimizer = optim.SGD(model.parameters(),lr= 0.01)
for epoch in xrange(number_of_epochs):
for i in xrange(len(context)):
context_vec_tmp = [conv_vect(j,corpa) for j in context[i]]
context_vec = Variable(torch.cat(tuple([context_vec_tmp[j] for j in xrange(len(context_vec_tmp))]),1))
word_vec = Variable(conv_vect(word[i],corpa))
predict = model(context_vec)[0]
predicted = torch.LongTensor(predict.size()[0],predict.size()[1]).zero_()
for i in xrange(predict.size()[1]):
predicted[0][i] = float(int(predict[0][i].data[0]/torch.max(predict.data[0])))
predicted = Variable(predicted)
model.zero_grad()
l = loss(predicted,word_vec)
l.backward()
optimizer.step()
return model
def make_training_data(data,context_size):
context = []
word = []
for i in data:
temp = i.split(" ")
for j in xrange(context_size,len(temp)-context_size,1):
context.append([temp[j - context_size],temp[j - context_size + 1],temp[j - context_size + 2],temp[j + context_size - 2],temp[j + context_size - 1],temp[j + context_size]])
word.append(temp[j])
return context,word
train_word2vec(make_corpa(po)[1],128,10000,po)
# po is a list of sentences
error comes out to be this:
TypeError Traceback (most recent call last)
<ipython-input-12-c4d942812d63> in <module>()
----> 1 train_word2vec(make_corpa(po)[1],128,10000,po)
<ipython-input-10-181483152376> in train_word2vec(vocab_size, embedding_dim, number_of_epochs, data)
10 context_vec = Variable(torch.cat(tuple([context_vec_tmp[j] for j in xrange(len(context_vec_tmp))]),1))
11 word_vec = Variable(conv_vect(word[i],corpa))
---> 12 predict = model(context_vec)[0]
13 predicted = torch.LongTensor(predict.size()[0],predict.size()[1]).zero_()
14 for i in xrange(predict.size()[1]):
/usr/local/lib/python2.7/dist-packages/torch/nn/modules/module.pyc in __call__(self, *input, **kwargs)
204
205 def __call__(self, *input, **kwargs):
--> 206 result = self.forward(*input, **kwargs)
207 for hook in self._forward_hooks.values():
208 hook_result = hook(self, input, result)
<ipython-input-7-33d7cb28d3b4> in forward(self, x)
8 y = []
9 for i in xrange(0,174,29):
---> 10 y.append(self.fc1(x[:,i:i+29]))
11
12 embedding = Variable(torch.zeros(1,128))
/usr/local/lib/python2.7/dist-packages/torch/nn/modules/module.pyc in __call__(self, *input, **kwargs)
204
205 def __call__(self, *input, **kwargs):
--> 206 result = self.forward(*input, **kwargs)
207 for hook in self._forward_hooks.values():
208 hook_result = hook(self, input, result)
/usr/local/lib/python2.7/dist-packages/torch/nn/modules/linear.pyc in forward(self, input)
52 return self._backend.Linear()(input, self.weight)
53 else:
---> 54 return self._backend.Linear()(input, self.weight, self.bias)
55
56 def __repr__(self):
/usr/local/lib/python2.7/dist-packages/torch/nn/_functions/linear.pyc in forward(self, input, weight, bias)
8 self.save_for_backward(input, weight, bias)
9 output = input.new(input.size(0), weight.size(0))
---> 10 output.addmm_(0, 1, input, weight.t())
11 if bias is not None:
12 # cuBLAS doesn't support 0 strides in sger, so we can't use expand
TypeError: addmm_ received an invalid combination of arguments - got (int, int, torch.LongTensor, torch.FloatTensor), but expected one of:
* (torch.LongTensor mat1, torch.LongTensor mat2)
* (torch.SparseLongTensor mat1, torch.LongTensor mat2)
* (int beta, torch.LongTensor mat1, torch.LongTensor mat2)
* (int alpha, torch.LongTensor mat1, torch.LongTensor mat2)
* (int beta, torch.SparseLongTensor mat1, torch.LongTensor mat2)
* (int alpha, torch.SparseLongTensor mat1, torch.LongTensor mat2)
* (int beta, int alpha, torch.LongTensor mat1, torch.LongTensor mat2)
didn't match because some of the arguments have invalid types: (int, int, torch.LongTensor, torch.FloatTensor)
* (int beta, int alpha, torch.SparseLongTensor mat1, torch.LongTensor mat2)
didn't match because some of the arguments have invalid types: (int, int, torch.LongTensor, torch.FloatTensor)