Hi
I have a neural network in which I am trying to implement multi-GPU capability with
model=torch.nn.DataParallel(model, device_ids=[0]) (currently I’m trying to get it to work on a computer with only one GPU, thus why device_ids=[0].)
But I get the error:
RuntimeError: Trying to backward through the graph a second time, but the buffers have already been freed. Specify retain_graph=True when calling backward the first time.
Stemming from:
loss.backward()
Worth noting, without implementing multi-GPU the code works fine with no errors.
(To implement multi-GPU I also had to switch some lines of code from model.something, to model.module.something, which could be the source of the problem)
The full code, in all its chaos:
import os
os.chdir(r'C:\Users\john\Desktop\Deep_Learning_A_Z\pytorch-0.3.1-py36_cuda80_cudnn6he774522_2\Lib\site-packages')
import torch
import torch.nn as nn
from torch import optim
import torch.autograd as autograd
import time, random
os.chdir(r'C:\Users\john\Desktop\Deep_Learning_A_Z\tqdm-master')
from tqdm import tqdm
os.chdir(r'C:\Users\john\Desktop\Deep_Learning_A_Z\pytorch-sentiment-classification-master')
import bilstm
import lstm
from lstm import LSTMSentiment
from bilstm import BiLSTMSentiment
os.chdir(r'C:\Users\john\Desktop\Deep_Learning_A_Z\text-master')
from torchtext import data
os.chdir(r'C:\Users\john\Desktop\Deep_Learning_A_Z')
import numpy as np
import argparse
import csv
import time
#import gensim
for x in range (1):
torch.set_num_threads(8)
torch.manual_seed(1)
random.seed(1)
#########################################################################
cuda = True
Difficulty = 'easy'
#########################################################################
#if Difficulty == 'easy':
def load_bin_vec(fname, vocab):
"""
Loads 300x1 word vecs from Google (Mikolov) word2vec
"""
# count = 0
word_vecs = {}
with open(fname, "rb") as f:
header = f.readline()
vocab_size, layer1_size = map(int, header.split())
binary_len = np.dtype('float32').itemsize * layer1_size
for line in range(vocab_size):
word = []
while True:
ch = f.read(1).decode('latin-1')
if ch == ' ':
word = ''.join(word)
break
if ch != '\n':
word.append(ch)
if word in vocab:
word_vecs[word] = np.fromstring(f.read(binary_len), dtype='float32')
else:
f.read(binary_len)
# count = count+1
# print(count)
return word_vecs
#if Difficulty == 'hard':
# def load_bin_vec(fname, vocab):
# count = 0
# success = 0
# """
# Loads 300x1 word vecs from Google (Mikolov) word2vec
# """
# word_vecs = {}
# w2v_model = gensim.models.KeyedVectors.load_word2vec_format(fname, binary=True)
#
# from gensim.models import word2vec
# w2v_model = word2vec.Word2Vec.load(fname)
# print(w2v_model.similarity('film', 'movie'))
#
# for word in vocab:
# print(word)
# here = 0
# find = 0
# for letter in word:
# here = here + 1
# if letter == ':':
# find = here
# try:
# word_vecs[word] = w2v_model[word[0:find]]
# success = success + 1
# except KeyError:
# count = count + 1
# some tokens from the dataset may not appear as an entry in the word embeds matrix, so i print and skip them
# print('Key error: {}'.format(word))
# continue
# print(count, 'problematic words', success, 'successful words')
# return word_vecs
def get_accuracy(truth, pred):
assert len(truth) == len(pred)
right = 0
for i in range(len(truth)):
if truth[i] == pred[i]:
right += 1.0
return right / len(truth)
def train_epoch_progress(model, train_iter, loss_function, optimizer, text_field, label_field, epoch):
model.train()
avg_loss = 0.0
truth_res = []
pred_res = []
count = 0
for batch in tqdm(train_iter, desc='Train epoch '+str(epoch+1)):
sent, label = batch.text, batch.label
label.data.sub_(1)
truth_res += list(label.data)
model.batch_size = len(label.data)
model.epoch = epoch+1
model.hidden = model.module.init_hidden()
model.count = count
pred = model.module(sent)
pred_label = pred.data.max(1)[1]
pred_res += [x for x in pred_label]
model.zero_grad()
loss = loss_function(pred, label)
avg_loss += loss.data[0]
count += 1
loss.backward()
optimizer.step()
avg_loss /= len(train_iter)
acc = get_accuracy(truth_res, pred_res)
# print(model.hidden)
return avg_loss, acc
def train_epoch(model, train_iter, loss_function, optimizer):
model.train()
avg_loss = 0.0
truth_res = []
pred_res = []
count = 0
for batch in train_iter:
sent, label = batch.text, batch.label
label.data.sub_(1)
truth_res += list(label.data)
model.batch_size = len(label.data)
model.hidden = model.module.init_hidden()
pred = model.module(sent)
pred_label = pred.data.max(1)[1].numpy()
pred_res += [x for x in pred_label]
model.zero_grad()
loss = loss_function(pred, label)
avg_loss += loss.data[0]
count += 1
loss.backward()
optimizer.step()
avg_loss /= len(train_iter)
acc = get_accuracy(truth_res, pred_res)
return avg_loss, acc
def evaluate(model, data, loss_function, name):
model.eval()
avg_loss = 0.0
truth_res = []
pred_res = []
for batch in data:
sent, label = batch.text, batch.label
label.data.sub_(1)
truth_res += list(label.data)
model.batch_size = len(label.data)
model.hidden = model.module.init_hidden()
pred = model.module(sent)
pred_label = pred.data.max(1)[1]
pred_res += [x for x in pred_label]
loss = loss_function(pred, label)
avg_loss += loss.data[0]
avg_loss /= len(data)
acc = get_accuracy(truth_res, pred_res)
print(name + ': loss %.2f acc %.1f' % (avg_loss, acc*100))
return acc
def load_sst(text_field, label_field, batch_size, Difficulty, cuda):
# for x in range(B_sizes):
if Difficulty == 'easy':
os.chdir(r'C:\Users\john\Desktop\Deep_Learning_A_Z\pytorch-sentiment-classification-master copy/data')
train, dev, test = data.TabularDataset.splits(path='./SST2/', train='train.tsv',
validation='dev.tsv', test='test.tsv', format='tsv',
fields=[('text', text_field), ('label', label_field)])
if Difficulty == 'medium':
os.chdir(r'C:\Users\john\Desktop\Deep_Learning_A_Z\pytorch-sentiment-classification-master')
train, dev, test = data.TabularDataset.splits(path='./data/Dataset/', train='train.tsv',
validation='test.tsv', test='test.tsv', format='tsv',
fields=[('text', text_field), ('label', label_field)])
if Difficulty == 'hard':
os.chdir(r'C:\Users\john\Desktop\Deep_Learning_A_Z')
train, dev, test = data.TabularDataset.splits(path='./aclImdb/', train='usable_train.tsv',
validation='usable_dev.tsv', test='usable_test.tsv', format='tsv',
fields=[('text', text_field), ('label', label_field)])
text_field.build_vocab(train, dev, test)
label_field.build_vocab(train, dev, test)
# batch_size = 10*(.1*x*(torch.cos(torch.FloatTensor([x]))*torch.sin(torch.FloatTensor([2*x])))+.08*x).round() + 1
## for GPU run
if cuda == True:
train_iter, dev_iter, test_iter = data.BucketIterator.splits((train, dev, test),
batch_sizes=(batch_size, len(dev), len(test)), sort_key=lambda x: len(x.text), repeat=False, device=None)
else:
train_iter, dev_iter, test_iter = data.BucketIterator.splits((train, dev, test),
batch_sizes=(batch_size, len(dev), len(test)), sort_key=lambda x: len(x.text), repeat=False, device=-1)
return train_iter, dev_iter, test_iter
#def adjust_learning_rate(learning_rate, optimizer, epoch):
# lr = learning_rate * (0.9 ** (epoch //1))
# for param_group in optimizer.param_groups:
# param_group['lr'] = lr
# return optimizer
#for clustering
#if Difficulty == 'easy':
# EPOCHS= 30
# BATCH_SIZE = 30
# HIDDEN_DIM = 150
if Difficulty == 'easy':
EPOCHS= 1000
BATCH_SIZE = 5
HIDDEN_DIM = 100
#HIDDEN_DIM = round((150*(4/6)))
#HIDDEN_DIM = round((150*(4/5)))
#HIDDEN_DIM = round((150*(4/8)))
if Difficulty == 'medium':
EPOCHS= 30
BATCH_SIZE = 5
HIDDEN_DIM = 50
#HIDDEN_DIM = round((150*(4/6)))
#HIDDEN_DIM = round((150*(4/5)))
#HIDDEN_DIM = round((150*(4/8)))
if Difficulty == 'hard':
EPOCHS= 30
BATCH_SIZE = 5
HIDDEN_DIM = 50
USE_GPU = torch.cuda.is_available()
EMBEDDING_DIM = 300
timestamp = str(int(time.time()))
best_dev_acc = 0.0
#B_sizes = np.ones(BATCH_SIZE) #BATCH SIZE MUST BE MINIMUM OF 50
#for x in range(BATCH_SIZE):
# B_sizes[x]= 10*(.1*x*(torch.cos(torch.FloatTensor([x]))*torch.sin(torch.FloatTensor([2*x])))+.08*x).round() + 1
#B_sizes = max(B_sizes)
#def weights_init(LSTMSentiment):
# classname = LSTMSentiment.__class__.__name__
# if classname.find('LSTMSentiment') != -1:
# LSTMSentiment.weight.data.fill_(.5)
# LSTMSentiment.bias.data.fill_(0)
#Using = BiLSTMSentiment
Using = LSTMSentiment
if cuda:
torch.set_default_tensor_type('torch.cuda.FloatTensor')
print('Using: ', Using)
text_field = data.Field(lower=True)
label_field = data.Field(sequential=False)
train_iter, dev_iter, test_iter = load_sst(text_field, label_field, BATCH_SIZE, Difficulty = Difficulty, cuda = cuda)
count = 0
model = Using(embedding_dim=EMBEDDING_DIM, hidden_dim=HIDDEN_DIM, vocab_size=len(text_field.vocab),
use_gpu=cuda, label_size=len(label_field.vocab)-1, batch_size=BATCH_SIZE)
if cuda:
model = model.cuda()
print('Load word embeddings...')
# # glove
#text_field.vocab.load_vectors('glove.6B.100d')
#text_field.vocab.load_vectors('hard_dataset_embeddings')
# word2vector
word_to_idx = text_field.vocab.stoi
pretrained_embeddings = np.random.uniform(-0.25, 0.25, (len(text_field.vocab), EMBEDDING_DIM))
pretrained_embeddings[0] = 0
#if Difficulty == 'easy':
os.chdir(r'C:\Users\john\Desktop\Deep_Learning_A_Z\pytorch-sentiment-classification-master')
word2vec = load_bin_vec('./data/GoogleNews-vectors-negative300.bin', word_to_idx)
#if Difficulty == 'hard':
# os.chdir(r'C:\Users\john\Desktop\Deep_Learning_A_Z')
# word2vec = load_bin_vec('hard_dataset_embeddings', word_to_idx)
for word, vector in word2vec.items():
pretrained_embeddings[word_to_idx[word]-1] = vector
# text_field.vocab.load_vectors(wv_type='', wv_dim=300)
model.embeddings.weight.data.copy_(torch.from_numpy(pretrained_embeddings))
# model.embeddings.weight.data = text_field.vocab.vectors
# model.embeddings.embed.weight.requires_grad = False
best_model = model
optimizer = optim.Adam(model.parameters(), lr=1e-3)
#optimizer = optim.Adam(model.parameters(), lr=1e-2,weight_decay=1e-5)
loss_function = nn.NLLLoss()
#loss_function = nn.CrossEntropyLoss()
model=torch.nn.DataParallel(model, device_ids=[0])
print('Training...')
out_dir = os.path.abspath(os.path.join(os.path.curdir, "runs", timestamp))
print("Writing to {}\n".format(out_dir))
time_start = time.time()
if not os.path.exists(out_dir):
os.makedirs(out_dir)
for epoch in range(EPOCHS):
# batch_modifer = epoch + 1
avg_loss, acc = train_epoch_progress(model, train_iter, loss_function, optimizer, text_field, label_field, epoch)
tqdm.write('Train: loss %.2f acc %.1f' % (avg_loss, acc*100))
dev_acc = evaluate(model, dev_iter, loss_function, 'Dev')
if dev_acc > best_dev_acc:
if best_dev_acc > 0:
os.system('rm '+ out_dir + '/best_model' + '.pth')
best_dev_acc = dev_acc
best_model = model
# torch.save(best_model.state_dict(), out_dir + '/best_model' + '.pth')
# evaluate on test with the best dev performance model
test_acc = evaluate(best_model, test_iter, loss_function, 'Test')
dev_acc = evaluate(model, dev_iter, loss_function, 'Dev')
test_acc = evaluate(best_model, test_iter, loss_function, 'Final Test')
time_finish = time.time()
total_time = time_finish - time_start
print('total time', (time_finish-time_start)/3600, 'hours')
total_time = (time_finish-time_start)/3600
os.chdir(r'C:\Users\john\Desktop\Deep_Learning_A_Z\pytorch-sentiment-classification-master/output/')
for x in range(1001):
if os.path.isfile('out_'+str(x)+'.csv') == False:
ID = x
break
eval(str(np.savetxt('out_'+str(ID)+'.csv', np.array([dev_acc,test_acc,total_time]), delimiter=',')))
#eval("np.savetxt('out_pm_%i.csv', np.array([dev_acc,test_acc,total_time]), delimiter=',')" %id)
Any help would be greatly appreciated!
Thank you!
UPDATE:
The error goes away when I plug retain_graphs = True into the two loss.backward() lines.
However, this causes the time per epoch to explode from 30 seconds to 20 minutes (understandably, considering what this command does). The whole thing I’m trying to achieve with the multi-GPU’s is faster run-times, so this is a problem.
Does someone perhaps know why the model=torch.nn.DataParallel(model, device_ids=[0]) line is forcing me to set retain_graphs = True'?
I feel that this is something to do with the fact that I had to turn model.init_hidden and model(sent) into mode.module.init_hidden and model.module(sent).
But without that addition, I was getting errors like:
AttributeError: 'DataParallel' object has no attribute 'init_hidden'
So I’m really not sure where to go from here…
SOLVED: (I think)
For anyone who is having a similar issue, my solution was to scrap the retain_graph = True entirely, and simply changed almost every model to model.module
WORKING CODE:
import os
os.chdir(r'C:\Users\john\Desktop\Deep_Learning_A_Z\pytorch-0.3.1-py36_cuda80_cudnn6he774522_2\Lib\site-packages')
import torch
import torch.nn as nn
from torch import optim
import torch.autograd as autograd
import time, random
os.chdir(r'C:\Users\john\Desktop\Deep_Learning_A_Z\tqdm-master')
from tqdm import tqdm
os.chdir(r'C:\Users\john\Desktop\Deep_Learning_A_Z\pytorch-sentiment-classification-master')
import bilstm
import lstm
from lstm import LSTMSentiment
from bilstm import BiLSTMSentiment
os.chdir(r'C:\Users\john\Desktop\Deep_Learning_A_Z\text-master')
from torchtext import data
os.chdir(r'C:\Users\john\Desktop\Deep_Learning_A_Z')
import numpy as np
import argparse
import csv
import time
#import gensim
for x in range (1):
torch.set_num_threads(8)
torch.manual_seed(1)
random.seed(1)
#########################################################################
cuda = True
Difficulty = 'easy'
#########################################################################
#if Difficulty == 'easy':
def load_bin_vec(fname, vocab):
"""
Loads 300x1 word vecs from Google (Mikolov) word2vec
"""
# count = 0
word_vecs = {}
with open(fname, "rb") as f:
header = f.readline()
vocab_size, layer1_size = map(int, header.split())
binary_len = np.dtype('float32').itemsize * layer1_size
for line in range(vocab_size):
word = []
while True:
ch = f.read(1).decode('latin-1')
if ch == ' ':
word = ''.join(word)
break
if ch != '\n':
word.append(ch)
if word in vocab:
word_vecs[word] = np.fromstring(f.read(binary_len), dtype='float32')
else:
f.read(binary_len)
# count = count+1
# print(count)
return word_vecs
#if Difficulty == 'hard':
# def load_bin_vec(fname, vocab):
# count = 0
# success = 0
# """
# Loads 300x1 word vecs from Google (Mikolov) word2vec
# """
# word_vecs = {}
# w2v_model = gensim.models.KeyedVectors.load_word2vec_format(fname, binary=True)
#
# from gensim.models import word2vec
# w2v_model = word2vec.Word2Vec.load(fname)
# print(w2v_model.similarity('film', 'movie'))
#
# for word in vocab:
# print(word)
# here = 0
# find = 0
# for letter in word:
# here = here + 1
# if letter == ':':
# find = here
# try:
# word_vecs[word] = w2v_model[word[0:find]]
# success = success + 1
# except KeyError:
# count = count + 1
# some tokens from the dataset may not appear as an entry in the word embeds matrix, so i print and skip them
# print('Key error: {}'.format(word))
# continue
# print(count, 'problematic words', success, 'successful words')
# return word_vecs
def get_accuracy(truth, pred):
assert len(truth) == len(pred)
right = 0
for i in range(len(truth)):
if truth[i] == pred[i]:
right += 1.0
return right / len(truth)
def train_epoch_progress(model, train_iter, loss_function, optimizer, text_field, label_field, epoch):
model.train()
avg_loss = 0.0
truth_res = []
pred_res = []
count = 0
for batch in tqdm(train_iter, desc='Train epoch '+str(epoch+1)):
sent, label = batch.text, batch.label
label.data.sub_(1)
truth_res += list(label.data)
model.module.batch_size = len(label.data)
model.module.epoch = epoch+1
model.module.hidden = model.module.init_hidden()
model.module.count = count
pred = model.module(sent)
pred_label = pred.data.max(1)[1]
pred_res += [x for x in pred_label]
model.module.zero_grad()
loss = loss_function(pred, label)
avg_loss += loss.data[0]
count += 1
loss.backward()
optimizer.step()
avg_loss /= len(train_iter)
acc = get_accuracy(truth_res, pred_res)
# print(model.hidden)
return avg_loss, acc
def train_epoch(model, train_iter, loss_function, optimizer):
model.train()
avg_loss = 0.0
truth_res = []
pred_res = []
count = 0
for batch in train_iter:
sent, label = batch.text, batch.label
label.data.sub_(1)
truth_res += list(label.data)
model.module.batch_size = len(label.data)
model.module.hidden = model.module.init_hidden()
pred = model.module(sent)
pred_label = pred.data.max(1)[1].numpy()
pred_res += [x for x in pred_label]
model.module.zero_grad()
loss = loss_function(pred, label)
avg_loss += loss.data[0]
count += 1
loss.backward()
optimizer.step()
avg_loss /= len(train_iter)
acc = get_accuracy(truth_res, pred_res)
return avg_loss, acc
def evaluate(model, data, loss_function, name):
model.eval()
avg_loss = 0.0
truth_res = []
pred_res = []
for batch in data:
sent, label = batch.text, batch.label
label.data.sub_(1)
truth_res += list(label.data)
model.module.batch_size = len(label.data)
model.module.hidden = model.module.init_hidden()
pred = model.module(sent)
pred_label = pred.data.max(1)[1]
pred_res += [x for x in pred_label]
loss = loss_function(pred, label)
avg_loss += loss.data[0]
avg_loss /= len(data)
acc = get_accuracy(truth_res, pred_res)
print(name + ': loss %.2f acc %.1f' % (avg_loss, acc*100))
return acc
def load_sst(text_field, label_field, batch_size, Difficulty, cuda):
# for x in range(B_sizes):
if Difficulty == 'easy':
os.chdir(r'C:\Users\john\Desktop\Deep_Learning_A_Z\pytorch-sentiment-classification-master copy/data')
train, dev, test = data.TabularDataset.splits(path='./SST2/', train='train.tsv',
validation='dev.tsv', test='test.tsv', format='tsv',
fields=[('text', text_field), ('label', label_field)])
if Difficulty == 'medium':
os.chdir(r'C:\Users\john\Desktop\Deep_Learning_A_Z\pytorch-sentiment-classification-master')
train, dev, test = data.TabularDataset.splits(path='./data/Dataset/', train='train.tsv',
validation='test.tsv', test='test.tsv', format='tsv',
fields=[('text', text_field), ('label', label_field)])
if Difficulty == 'hard':
os.chdir(r'C:\Users\john\Desktop\Deep_Learning_A_Z')
train, dev, test = data.TabularDataset.splits(path='./aclImdb/', train='usable_train.tsv',
validation='usable_dev.tsv', test='usable_test.tsv', format='tsv',
fields=[('text', text_field), ('label', label_field)])
text_field.build_vocab(train, dev, test)
label_field.build_vocab(train, dev, test)
# batch_size = 10*(.1*x*(torch.cos(torch.FloatTensor([x]))*torch.sin(torch.FloatTensor([2*x])))+.08*x).round() + 1
## for GPU run
if cuda == True:
train_iter, dev_iter, test_iter = data.BucketIterator.splits((train, dev, test),
batch_sizes=(batch_size, len(dev), len(test)), sort_key=lambda x: len(x.text), repeat=False, device=None)
else:
train_iter, dev_iter, test_iter = data.BucketIterator.splits((train, dev, test),
batch_sizes=(batch_size, len(dev), len(test)), sort_key=lambda x: len(x.text), repeat=False, device=-1)
return train_iter, dev_iter, test_iter
#def adjust_learning_rate(learning_rate, optimizer, epoch):
# lr = learning_rate * (0.9 ** (epoch //1))
# for param_group in optimizer.param_groups:
# param_group['lr'] = lr
# return optimizer
#for clustering
#if Difficulty == 'easy':
# EPOCHS= 30
# BATCH_SIZE = 30
# HIDDEN_DIM = 150
if Difficulty == 'easy':
EPOCHS= 1000
BATCH_SIZE = 5
HIDDEN_DIM = 100
#HIDDEN_DIM = round((150*(4/6)))
#HIDDEN_DIM = round((150*(4/5)))
#HIDDEN_DIM = round((150*(4/8)))
if Difficulty == 'medium':
EPOCHS= 30
BATCH_SIZE = 5
HIDDEN_DIM = 50
#HIDDEN_DIM = round((150*(4/6)))
#HIDDEN_DIM = round((150*(4/5)))
#HIDDEN_DIM = round((150*(4/8)))
if Difficulty == 'hard':
EPOCHS= 30
BATCH_SIZE = 5
HIDDEN_DIM = 50
USE_GPU = torch.cuda.is_available()
EMBEDDING_DIM = 300
timestamp = str(int(time.time()))
best_dev_acc = 0.0
#B_sizes = np.ones(BATCH_SIZE) #BATCH SIZE MUST BE MINIMUM OF 50
#for x in range(BATCH_SIZE):
# B_sizes[x]= 10*(.1*x*(torch.cos(torch.FloatTensor([x]))*torch.sin(torch.FloatTensor([2*x])))+.08*x).round() + 1
#B_sizes = max(B_sizes)
#def weights_init(LSTMSentiment):
# classname = LSTMSentiment.__class__.__name__
# if classname.find('LSTMSentiment') != -1:
# LSTMSentiment.weight.data.fill_(.5)
# LSTMSentiment.bias.data.fill_(0)
#Using = BiLSTMSentiment
Using = LSTMSentiment
if cuda:
torch.set_default_tensor_type('torch.cuda.FloatTensor')
print('Using: ', Using)
text_field = data.Field(lower=True)
label_field = data.Field(sequential=False)
train_iter, dev_iter, test_iter = load_sst(text_field, label_field, BATCH_SIZE, Difficulty = Difficulty, cuda = cuda)
count = 0
model = Using(embedding_dim=EMBEDDING_DIM, hidden_dim=HIDDEN_DIM, vocab_size=len(text_field.vocab),
use_gpu=cuda, label_size=len(label_field.vocab)-1, batch_size=BATCH_SIZE)
if cuda:
model = model.cuda()
model=torch.nn.DataParallel(model, device_ids=[0])
print('Load word embeddings...')
# # glove
#text_field.vocab.load_vectors('glove.6B.100d')
#text_field.vocab.load_vectors('hard_dataset_embeddings')
# word2vector
word_to_idx = text_field.vocab.stoi
pretrained_embeddings = np.random.uniform(-0.25, 0.25, (len(text_field.vocab), EMBEDDING_DIM))
pretrained_embeddings[0] = 0
#if Difficulty == 'easy':
os.chdir(r'C:\Users\john\Desktop\Deep_Learning_A_Z\pytorch-sentiment-classification-master')
word2vec = load_bin_vec('./data/GoogleNews-vectors-negative300.bin', word_to_idx)
#if Difficulty == 'hard':
# os.chdir(r'C:\Users\john\Desktop\Deep_Learning_A_Z')
# word2vec = load_bin_vec('hard_dataset_embeddings', word_to_idx)
for word, vector in word2vec.items():
pretrained_embeddings[word_to_idx[word]-1] = vector
# text_field.vocab.load_vectors(wv_type='', wv_dim=300)
model.module.embeddings.weight.data.copy_(torch.from_numpy(pretrained_embeddings))
# model.embeddings.weight.data = text_field.vocab.vectors
# model.embeddings.embed.weight.requires_grad = False
best_model = model
optimizer = optim.Adam(model.parameters(), lr=1e-3)
#optimizer = optim.Adam(model.parameters(), lr=1e-2,weight_decay=1e-5)
loss_function = nn.NLLLoss()
#loss_function = nn.CrossEntropyLoss()
print('Training...')
out_dir = os.path.abspath(os.path.join(os.path.curdir, "runs", timestamp))
print("Writing to {}\n".format(out_dir))
time_start = time.time()
if not os.path.exists(out_dir):
os.makedirs(out_dir)
for epoch in range(EPOCHS):
# batch_modifer = epoch + 1
avg_loss, acc = train_epoch_progress(model, train_iter, loss_function, optimizer, text_field, label_field, epoch)
tqdm.write('Train: loss %.2f acc %.1f' % (avg_loss, acc*100))
dev_acc = evaluate(model, dev_iter, loss_function, 'Dev')
if dev_acc > best_dev_acc:
if best_dev_acc > 0:
os.system('rm '+ out_dir + '/best_model' + '.pth')
best_dev_acc = dev_acc
best_model = model
# torch.save(best_model.state_dict(), out_dir + '/best_model' + '.pth')
# evaluate on test with the best dev performance model
test_acc = evaluate(best_model, test_iter, loss_function, 'Test')
dev_acc = evaluate(model, dev_iter, loss_function, 'Dev')
test_acc = evaluate(best_model, test_iter, loss_function, 'Final Test')
time_finish = time.time()
total_time = time_finish - time_start
print('total time', (time_finish-time_start)/3600, 'hours')
total_time = (time_finish-time_start)/3600
os.chdir(r'C:\Users\john\Desktop\Deep_Learning_A_Z\pytorch-sentiment-classification-master/output/')
for x in range(1001):
if os.path.isfile('out_'+str(x)+'.csv') == False:
ID = x
break
eval(str(np.savetxt('out_'+str(ID)+'.csv', np.array([dev_acc,test_acc,total_time]), delimiter=',')))
#eval("np.savetxt('out_pm_%i.csv', np.array([dev_acc,test_acc,total_time]), delimiter=',')" %id)