#!/usr/bin/env python
# coding: utf-8
# In[1]:
get_ipython().run_line_magic('matplotlib', 'inline')
# In[2]:
from __future__ import unicode_literals, print_function, division
from io import open
import unicodedata
import string
import re
import random
import torch
import torch.nn as nn
from torch import optim
import torch.nn.functional as F
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
# In[3]:
SOS_token = 0
EOS_token = 1
class Lang:
def __init__(self, name):
self.name = name
self.word2index = {}
self.word2count = {}
self.index2word = {0: "SOS", 1: "EOS"}
self.n_words = 2 # Count SOS and EOS
def addSentence(self, sentence):
for word in sentence.split(' '):
self.addWord(word)
def addWord(self, word):
if word not in self.word2index:
# print(word)
self.word2index[word] = self.n_words
self.word2count[word] = 1
self.index2word[self.n_words] = word
self.n_words += 1
else:
self.word2count[word] += 1
# In[4]:
# Turn a Unicode string to plain ASCII, thanks to
# http://stackoverflow.com/a/518232/2809427
def unicodeToAscii(s):
return ''.join(
c for c in unicodedata.normalize('NFD', s)
if unicodedata.category(c) != 'Mn'
)
# Lowercase, trim, and remove non-letter characters
def normalizeString(s):
s = unicodeToAscii(s.lower().strip())
s = re.sub(r"([.!?])", r" \1", s)
s = re.sub(r"[^a-zA-Z.!?]+", r" ", s)
return s
# In[5]:
def reading_bigfiles(lang1, lang2, reverse=False):
print("Reading lines...")
input_lines = open("../data/english_rough_changed_updated", encoding='utf-8').read().strip().split('\n')
output_lines = open("../data/french_rough_changed_updated", encoding='utf-8').read().strip().split('\n')
print(len(input_lines),len(output_lines))
lines = []
for i,j in zip(input_lines,output_lines):
lines.append(i+"\t"+j)
print(random.choice(lines))
# Read the file and split into lines
# lines = open('data/%s-%s.txt' % (lang1, lang2), encoding='utf-8').\
# read().strip().split('\n')
# # Split every line into pairs and normalize
pairs = [[normalizeString(s) for s in l.split('\t')] for l in lines]
print(len(pairs),"pairs")
# Reverse pairs, make Lang instances
if reverse:
pairs = [list(reversed(p)) for p in pairs]
input_lang = Lang(lang2)
output_lang = Lang(lang1)
else:
input_lang = Lang(lang1)
output_lang = Lang(lang2)
return input_lang, output_lang, pairs
# In[6]:
# def readLangs(lang1, lang2, reverse=False):
# print("Reading lines...")
# # Read the file and split into lines
# lines = open('../sorted_entofrdata/combined_data', encoding='utf-8').\
# read().strip().split('\n')
# print(lines[10],"the lines value")
# # Split every line into pairs and normalize
# pairs = [[normalizeString(s) for s in l.split('\t')] for l in lines]
# print(len(pairs),"pairs")
# # Reverse pairs, make Lang instances
# if reverse:
# pairs = [list(reversed(p)) for p in pairs]
# input_lang = Lang(lang2)
# output_lang = Lang(lang1)
# else:
# input_lang = Lang(lang1)
# output_lang = Lang(lang2)
# return input_lang, output_lang, pairs
# In[7]:
# dummy = "B4-0012/98 - O-0009/98 et B4-0013/98 - O-0010/98 des députés Maij-Weggen, Moorhouse et Oomen-Ruijten, au nom du groupe du parti populaire européen, au Conseil et à la Commission, sur l'attitude de l'Union européenne à l'égard du Nigéria; -B4-0020/98 - O-0028/98 et B4-0122/98 - O-0029/98 de M. Bertens, au nom du groupe du parti européen des libéraux démocrates et réformateurs, au Conseil et à la Commission, sur l'attitude de l'Union européenne à l'égard du Nigéria; -B4-0123/98 - O-0030/98 et B4-0124/98 - O-0031/98 des députés Müller, Aelvoet, Telkämper et McKenna, au nom du groupe des verts au Parlement européen, au Conseil et à la Commission, sur l'attitude de l'Union européenne à l'égard du Nigéria; -B4-0127/98 - O-0035/98 des députés Carnero González et Pettinari, au nom du groupe confédéral de la gauche unitaire européenne - gauche verte nordique, au Conseil, sur les relations entre l'Union européenne et le Nigéria; -B4-0128/98 - O-0036/98 et B4-0129/98 - O-0037/98 des députés Macartney et Hory, au nom du groupe de l'alliance radicale européenne, au Conseil et à la Commission, sur l'attitude de l'Union européenne vis-à -vis du Nigéria; -B4-0134/98 - O-0042/98 et B4-0135/98 - O-0043/98 des députés Andrews et Girão Pereira, au nom du groupe Union pour l'Europe, au Conseil et à la Commission, sur l'attitude de l'Union européenne vis-à -vis du Nigéria; -B4-0137/98 - O-0046/98 et B4-0169/98 - O-0051/98 des députés Kinnock et Vecchi, au nom du groupe du parti des socialistes européens, au Conseil et à la Commission, sur la situation au Nigéria; -B4-0014/98 - O-0011/98 et B4-0015/98 - O-0012/98 des députés Maij-Weggen et Oomen-Ruijten, au nom du groupe du parti populaire européen, au Conseil et à la Commission, sur les violations des droits de l'homme en Birmanie; -B4-0018/98 - O-0026/98 et B4-0019/98 - O-0027/98 dude M. Bertens, au nom du groupe du parti européen des libéraux démocrates et réformateurs, au Conseil et à la Commission, sur les violations des droits de l'homme en Birmanie; -B4-0126/98 - O-0034/98 des députés Pettinari et Vinci, au nom du groupe confédéral de la gauche unitaire européenne - gauche verte nordique, au Conseil, sur les relations entre l'Union européenne et la Birmanie; -B4-0131/98 - O-0039/98 de M. Dupuis, au nom du groupe de l'alliance radicale européenne, à la Commission, sur la violation des droits de l'homme en Birmanie; -B4-0133/98 - O-0041/98 des députés Telkämper, Aelvoet, Kreissl-Dörfler et Lannoye, au nom du groupe des verts au Parlement européen, à la Commission, sur les violations des droits de l'homme en Birmanie; -B4-0136/98 - O-0045/98 et B4-0141/98 - O-0050/98 des députés Kinnock et Vecchi, au nom du groupe du parti des socialistes européens, au Conseil et à la Commission, sur la situation en Birmanie; -B4-0139/98 - O-0048/98) de Mme van Bladel, au nom du groupe Union pour l'Europe, au Conseil, sur la situation des droits de l'homme en Birmanie; -B4-0011/98 - O-0182/98 de M. Bertens, au nom du groupe du parti européen des libéraux démocrates et réformateurs, au Conseil, sur la position de l'Union européenne lors de la 54e Assemblée des Nations unies sur les droits de l'homme à Genève; -B4-0125/98 - O-0033/98 des députés Carnero González, Manisco et Alavanos, au nom du groupe confédéral de la gauche unitaire européenne - gauche verte nordique, au Conseil, sur la Commission des Droits de l'homme des Nations unies; -B4-0130/98 - O-0038/98 de M. Dupuis, au nom du groupe de l'alliance radicale européenne, au Conseil, sur la position de l'Union européenne lors de la 54e session de la Commission des Droits de l'homme des Nations unies à Genève; -B4-0132/98 - O-0040/98 des députés Aglietta, Müller, Kreissl-Dörfler et Ripa Di Meana, au nom du groupe des verts au Parlement européen, au Conseil, sur la position de l'Union européenne lors de la 54e session de la Commission des Droits de l'homme des Nations unies à Genève; -B4-0138/98 - O-0047/98 de Mme van Bladel, au nom du groupe Union pour l'Europe, au Conseil, sur la 54e session des Nations unies sur les droits de l'homme; -B4-0140/98 - O-0049/98 de M. Barros Moura, au nom du groupe du parti des socialistes européens, au Conseil, sur la position de l'Union européenne lors de la 54e Assemblée des Nations unies sur les droits de l'homme à Genève."
# print(len(dummy.split(" ")))
# In[8]:
MAX_LENGTH = 700
eng_prefixes = (
"i am ", "i m ",
"he is", "he s ",
"she is", "she s",
"you are", "you re ",
"we are", "we re ",
"they are", "they re "
)
def filterPair(p):
return len(p[0].split(' ')) < MAX_LENGTH and len(p[1].split(' ')) < MAX_LENGTH
def filterPairs(pairs):
return [pair for pair in pairs if filterPair(pair)]
# In[9]:
def prepareData(lang1, lang2, reverse=False):
input_lang, output_lang, pairs = reading_bigfiles(lang1, lang2, reverse)
print("Read %s sentence pairs" % len(pairs))
pairs = filterPairs(pairs)
print("Trimmed to %s sentence pairs" % len(pairs))
print("Counting words...")
for pair in pairs:
input_lang.addSentence(pair[0])
output_lang.addSentence(pair[1])
print("Counted words:")
print(input_lang.name, input_lang.n_words)
print(output_lang.name, output_lang.n_words)
return input_lang, output_lang, pairs
input_lang, output_lang, pairs = prepareData('eng', 'fra', True)
print(random.choice(pairs))
print(len(pairs),"after completion")
# In[10]:
print(output_lang.n_words,"number of words")
# In[11]:
import json
with open("../processed/input_index2word",'w') as f:
f.write(json.dumps(input_lang.index2word))
with open("../processed/input_word2index",'w') as f:
f.write(json.dumps(input_lang.word2index))
# add_text_to_files = open('./data/technical_text.txt', encoding='UTF-8').read().strip().split('\n')
with open("../processed/output_index2word",'w') as f:
f.write(json.dumps(output_lang.index2word))
with open("../processed/output_word2index",'w') as f:
f.write(json.dumps(output_lang.word2index))
# In[12]:
class EncoderRNN(nn.Module):
def __init__(self, input_size, hidden_size):
super(EncoderRNN, self).__init__()
self.hidden_size = hidden_size
self.embedding = nn.Embedding(input_size, hidden_size)
self.gru = nn.GRU(hidden_size, hidden_size)
def forward(self, input, hidden):
embedded = self.embedding(input).view(1, 1, -1)
output = embedded
output, hidden = self.gru(output, hidden)
return output, hidden
def initHidden(self):
return torch.zeros(1, 1, self.hidden_size, device=device)
# In[13]:
class DecoderRNN(nn.Module):
def __init__(self, hidden_size, output_size):
super(DecoderRNN, self).__init__()
self.hidden_size = hidden_size
self.embedding = nn.Embedding(output_size, hidden_size)
self.gru = nn.GRU(hidden_size, hidden_size)
self.out = nn.Linear(hidden_size, output_size)
self.softmax = nn.LogSoftmax(dim=1)
def forward(self, input, hidden):
output = self.embedding(input).view(1, 1, -1)
output = F.relu(output)
output, hidden = self.gru(output, hidden)
output = self.softmax(self.out(output[0]))
return output, hidden
def initHidden(self):
return torch.zeros(1, 1, self.hidden_size, device=device)
# In[14]:
class AttnDecoderRNN(nn.Module):
def __init__(self, hidden_size, output_size, dropout_p=0.1, max_length=MAX_LENGTH):
super(AttnDecoderRNN, self).__init__()
self.hidden_size = hidden_size
self.output_size = output_size
self.dropout_p = dropout_p
self.max_length = max_length
self.embedding = nn.Embedding(self.output_size, self.hidden_size)
self.attn = nn.Linear(self.hidden_size * 2, self.max_length)
self.attn_combine = nn.Linear(self.hidden_size * 2, self.hidden_size)
self.dropout = nn.Dropout(self.dropout_p)
self.gru = nn.GRU(self.hidden_size, self.hidden_size)
self.out = nn.Linear(self.hidden_size, self.output_size)
def forward(self, input, hidden, encoder_outputs):
embedded = self.embedding(input).view(1, 1, -1)
embedded = self.dropout(embedded)
attn_weights = F.softmax(
self.attn(torch.cat((embedded[0], hidden[0]), 1)), dim=1)
attn_applied = torch.bmm(attn_weights.unsqueeze(0),
encoder_outputs.unsqueeze(0))
output = torch.cat((embedded[0], attn_applied[0]), 1)
output = self.attn_combine(output).unsqueeze(0)
output = F.relu(output)
output, hidden = self.gru(output, hidden)
output = F.log_softmax(self.out(output[0]), dim=1)
return output, hidden, attn_weights
def initHidden(self):
return torch.zeros(1, 1, self.hidden_size, device=device)
# In[15]:
def indexesFromSentence(lang, sentence):
return [lang.word2index[word] for word in sentence.split(' ')]
def tensorFromSentence(lang, sentence):
indexes = indexesFromSentence(lang, sentence)
indexes.append(EOS_token)
return torch.tensor(indexes, dtype=torch.long, device=device).view(-1, 1)
def tensorsFromPair(pair):
input_tensor = tensorFromSentence(input_lang, pair[0])
target_tensor = tensorFromSentence(output_lang, pair[1])
return (input_tensor, target_tensor)
# In[16]:
teacher_forcing_ratio = 0.5
def train(input_tensor, target_tensor, encoder, decoder, encoder_optimizer, decoder_optimizer, criterion, max_length=MAX_LENGTH):
encoder_hidden = encoder.module.initHidden()
encoder_optimizer.zero_grad()
decoder_optimizer.zero_grad()
input_length = input_tensor.size(0)
target_length = target_tensor.size(0)
encoder_outputs = torch.zeros(max_length, encoder.module.hidden_size, device=device)
loss = 0
for ei in range(input_length):
encoder_output, encoder_hidden = encoder(
input_tensor[ei], encoder_hidden)
encoder_outputs[ei] = encoder_output[0, 0]
decoder_input = torch.tensor([[SOS_token]], device=device)
decoder_hidden = encoder_hidden
use_teacher_forcing = True if random.random() < teacher_forcing_ratio else False
if use_teacher_forcing:
# Teacher forcing: Feed the target as the next input
for di in range(target_length):
decoder_output, decoder_hidden, decoder_attention = decoder(
decoder_input, decoder_hidden, encoder_outputs)
loss += criterion(decoder_output, target_tensor[di])
decoder_input = target_tensor[di] # Teacher forcing
else:
# Without teacher forcing: use its own predictions as the next input
for di in range(target_length):
decoder_output, decoder_hidden, decoder_attention = decoder(
decoder_input, decoder_hidden, encoder_outputs)
topv, topi = decoder_output.topk(1)
decoder_input = topi.squeeze().detach() # detach from history as input
loss += criterion(decoder_output, target_tensor[di])
if decoder_input.item() == EOS_token:
break
loss.backward()
encoder_optimizer.step()
decoder_optimizer.step()
return loss.item() / target_length
# In[17]:
import time
import math
def asMinutes(s):
m = math.floor(s / 60)
s -= m * 60
return '%dm %ds' % (m, s)
def timeSince(since, percent):
now = time.time()
s = now - since
es = s / (percent)
rs = es - s
return '%s (- %s)' % (asMinutes(s), asMinutes(rs))
# In[18]:
def trainIters(encoder, decoder, n_iters, print_every=1000, plot_every=100, learning_rate=0.01):
start = time.time()
plot_losses = []
print_loss_total = 0 # Reset every print_every
plot_loss_total = 0 # Reset every plot_every
encoder_optimizer = optim.SGD(encoder.parameters(), lr=learning_rate)
decoder_optimizer = optim.SGD(decoder.parameters(), lr=learning_rate)
training_pairs = [tensorsFromPair(random.choice(pairs))
for i in range(n_iters)]
criterion = nn.NLLLoss()
for iter in range(1, n_iters + 1):
training_pair = training_pairs[iter - 1]
input_tensor = training_pair[0]
target_tensor = training_pair[1]
loss = train(input_tensor, target_tensor, encoder,
decoder, encoder_optimizer, decoder_optimizer, criterion)
print_loss_total += loss
plot_loss_total += loss
if iter % print_every == 0:
print_loss_avg = print_loss_total / print_every
print_loss_total = 0
print('%s (%d %d%%) %.4f' % (timeSince(start, iter / n_iters),
iter, iter / n_iters * 100, print_loss_avg))
if iter % plot_every == 0:
plot_loss_avg = plot_loss_total / plot_every
plot_losses.append(plot_loss_avg)
plot_loss_total = 0
showPlot(plot_losses)
# In[19]:
import matplotlib.pyplot as plt
plt.switch_backend('agg')
import matplotlib.ticker as ticker
import numpy as np
def showPlot(points):
plt.figure()
fig, ax = plt.subplots()
# this locator puts ticks at regular intervals
loc = ticker.MultipleLocator(base=0.2)
ax.yaxis.set_major_locator(loc)
plt.plot(points)
# In[20]:
def evaluate(encoder, decoder, sentence, max_length=MAX_LENGTH):
with torch.no_grad():
input_tensor = tensorFromSentence(input_lang, sentence)
input_length = input_tensor.size()[0]
encoder_hidden = encoder.initHidden()
encoder_outputs = torch.zeros(max_length, encoder.hidden_size, device=device)
for ei in range(input_length):
encoder_output, encoder_hidden = encoder(input_tensor[ei],
encoder_hidden)
encoder_outputs[ei] += encoder_output[0, 0]
decoder_input = torch.tensor([[SOS_token]], device=device) # SOS
decoder_hidden = encoder_hidden
decoded_words = []
decoder_attentions = torch.zeros(max_length, max_length)
for di in range(max_length):
decoder_output, decoder_hidden, decoder_attention = decoder(
decoder_input, decoder_hidden, encoder_outputs)
decoder_attentions[di] = decoder_attention.data
topv, topi = decoder_output.data.topk(1)
if topi.item() == EOS_token:
decoded_words.append('<EOS>')
break
else:
decoded_words.append(output_lang.index2word[topi.item()])
decoder_input = topi.squeeze().detach()
return decoded_words, decoder_attentions[:di + 1]
# In[21]:
def evaluateRandomly(encoder, decoder, n=10):
for i in range(n):
pair = random.choice(pairs)
print('>', pair[0])
print('=', pair[1])
output_words, attentions = evaluate(encoder, decoder, pair[0])
output_sentence = ' '.join(output_words)
print('<', output_sentence)
print('')
# In[22]:
hidden_size = 256
encoder1 = EncoderRNN(input_lang.n_words, hidden_size).to(device)
attn_decoder1 = AttnDecoderRNN(hidden_size, output_lang.n_words, dropout_p=0.1).to(device)
# In[23]:
encoder1.parameters()
# In[24]:
if torch.cuda.device_count() > 1:
print("Let's use", torch.cuda.device_count(), "GPUs!")
# dim = 0 [30, xxx] -> [10, ...], [10, ...], [10, ...] on 3 GPUs
encoder1 = nn.DataParallel(encoder1,device_ids=[0,1,2,3])
attn_decoder1 = nn.DataParallel(attn_decoder1)
encoder1.train()
# In[25]:
attn_decoder1.train()
# In[26]:
trainIters(encoder1, attn_decoder1, 80000, print_every=5000)
# In[ ]:
import os
model_path = './model/translate.pt'
# In[ ]:
if not os.path.exists('./model'):
os.makedirs('model')
torch.save({'encoder':encoder1.state_dict(),
'attentondecoder':attn_decoder1.state_dict()},model_path)
# In[ ]:
model_loaded = torch.load(model_path)
encoder1.load_state_dict(model_loaded['encoder'])
attn_decoder1.load_state_dict(model_loaded['attentondecoder'])
encoder1.eval()
# In[ ]:
attn_decoder1.eval()
# In[ ]:
evaluateRandomly(encoder1, attn_decoder1)
# In[ ]:
output_words, attentions = evaluate(
encoder1, attn_decoder1, "je suis trop froid .")
plt.matshow(attentions.numpy())
# In[ ]:
len(output_lang.index2word),len(output_lang.word2index),len(input_lang.index2word),len(input_lang.word2index)
# In[ ]:
add_text_pairs = [[ens for ens in s.split('\t')] for s in add_text_to_files]
# english_extra = [[output_lang.word2index[ens] for ens in end.split(' ')] for end,frd in add_text_pairs]
for end,frd in add_text_pairs:
output_lang.addSentence(end)
input_lang.addSentence(frd)
len(output_lang.index2word),len(output_lang.word2index),len(input_lang.index2word),len(input_lang.word2index)
# In[ ]:
def showAttention(input_sentence, output_words, attentions):
# Set up figure with colorbar
fig = plt.figure()
ax = fig.add_subplot(111)
cax = ax.matshow(attentions.numpy(), cmap='bone')
fig.colorbar(cax)
# Set up axes
ax.set_xticklabels([''] + input_sentence.split(' ') +
['<EOS>'], rotation=90)
ax.set_yticklabels([''] + output_words)
# Show label at every tick
ax.xaxis.set_major_locator(ticker.MultipleLocator(1))
ax.yaxis.set_major_locator(ticker.MultipleLocator(1))
plt.show()
def evaluateAndShowAttention(input_sentence):
output_words, attentions = evaluate(
encoder1, attn_decoder1, input_sentence)
print('input =', input_sentence)
print('output =', ' '.join(output_words))
showAttention(input_sentence, output_words, attentions)
evaluateAndShowAttention("elle a cinq ans de moins que moi .")
evaluateAndShowAttention("elle est trop petit .")
evaluateAndShowAttention("je ne crains pas de mourir .")
evaluateAndShowAttention("salut comment allez-vous")
# In[ ]:
# evaluateAndShowAttention("")
# In[ ]:
In this i used nn.dataparallel() to make the model to work with mult-gpu after changing the code i executed the program and its showing the error
RuntimeError: invalid argument 6: wrong matrix size at /pytorch/aten/src/THC/generic/THCTensorMathBlas.cu:492
I think that error is maybe of different lengths of text so using dataparallel api it doesn’t able to split them into equal parts maybe that was the error i am not sure.
could anybody help me to solve this error.