I am following a seq2seq tutorial here.
I want to use pretrained vectors. I have edited the code to get the vector of the word rather than index. Following is the code:
#This piece of code loads the vectors from a json file {'word':[vector]..}
class Lang:
def __init__(self, name, savedVectorsFile):
def getSavedVectors(filename):
import json
word2vec = {}
with open(filename) as json_data:
word2vec = json.load(json_data)
return word2vec
self.name = name
self.word2vector = {}
self.word2count = {}
self.index2word = {0: "SOS", 1: "EOS"}
self.n_words = 2 # Count SOS and EOS
self.get_saved_vector = getSavedVectors(savedVectorsFile)
self.word2vector['unknown'] = self.get_saved_vector['unknown']
def addSentence(self, sentence):
for word in sentence.split(' '):
self.addWord(word)
def addWord(self, word):
if word not in self.word2vector:
self.word2vector[word] = self.get_saved_vector[word]
self.word2count[word] = 1
self.index2word[self.n_words] = word
self.n_words += 1
else:
self.word2count[word] += 1
# this piece deals with returning a vector of given word, all vectors are just concatenated into one giant vector
def vectorFromSentence(lang, sentence):
vectors = []
for word in sentence.split(' '):
if word not in lang.word2vector:
vectors += (lang.word2vector["unknown"])
else:
vectors += (lang.word2vector[word])
return vectors
# in the train method, I am passing a vector instead of index to the encoder
for ei in range(0, input_length, VEC_SIZE):
encoder_output, encoder_hidden = encoder(input_variable[ei*VEC_SIZE:(ei+1)*VEC_SIZE], encoder_hidden)
encoder_outputs[ei] = encoder_output[0][0]
Now, I cannot figure out what should be the input size of my encoder. I tried with using VEC_SIZE instead of input_size, but to no avail.
I get this error that
TypeError: torch.index_select received an invalid combination of arguments - got (torch.cuda.FloatTensor, int, torch.cuda.FloatTensor), but expected (torch.cuda.FloatTensor source, int dim, torch.cuda.LongTensor index)
Following is the trace:
Traceback (most recent call last):
File "scapula_generation_pretrained_vectors.py", line 710, in <module>
trainIters(encoder1, attn_decoder1, 50000, print_every=1000)
File "scapula_generation_pretrained_vectors.py", line 566, in trainIters
decoder, encoder_optimizer, decoder_optimizer, criterion)
File "scapula_generation_pretrained_vectors.py", line 472, in train
encoder_output, encoder_hidden = encoder(input_variable[ei*VEC_SIZE:(ei+1)*VEC_SIZE], encoder_hidden)
File "/home/sagar/anaconda3/lib/python3.6/site-packages/torch/nn/modules/module.py", line 224, in __call__
result = self.forward(*input, **kwargs)
File "scapula_generation_pretrained_vectors.py", line 214, in forward
embedded = self.embedding(input).view(1, 1, -1)
File "/home/sagar/anaconda3/lib/python3.6/site-packages/torch/nn/modules/module.py", line 224, in __call__
result = self.forward(*input, **kwargs)
File "/home/sagar/anaconda3/lib/python3.6/site-packages/torch/nn/modules/sparse.py", line 94, in forward
self.scale_grad_by_freq, self.sparse
File "/home/sagar/anaconda3/lib/python3.6/site-packages/torch/nn/_functions/thnn/sparse.py", line 53, in forward
output = torch.index_select(weight, 0, indices.view(-1))
How to write encoder and decoder to occupy the word2vec embeddings? Several things are involved, like does my output of encoder gru has to be a vector of size VEC_SIZE or not, in the decoder - my loss has to be calculated using some similarity metric. I think I will go for cosine similarity, but before that i will have to make sure that decoder takes the output of encoder and generates a vector of size VEC_SIZE.
I would appreciate if someone has already done this homework in the tutorial and has a code readily available?