I am having a trouble with increasing memory issue.
From the first data read, the memory starts to grow continuously.
And if I keep watching through htop, the memory continues to increase every time I do training and validating.
Eventually, a dataloader killed error occurs, and the training cannot be completed until the end.
Also, it didn’t use gpu 100%… it only uses 50%~60%…
I changed the loss variable and used all of the cudas, but I don’t know what is the cause.
My training code is here.
def train(self):
train_gen = data.DataLoader(
dataset=self._train_data, shuffle=True,
batch_size=ARGS.train_batch,
num_workers=ARGS.num_workers, pin_memory=True)
val_gen = data.DataLoader(
dataset=self._val_data, shuffle=False,
batch_size=ARGS.test_batch, num_workers=ARGS.num_workers,
pin_memory=True)
# will train self._num_epochs copies of train data
to_train = chain.from_iterable(repeat(train_gen, self._num_epochs))
# consisting of total_steps batches
total_steps = len(train_gen) * self._num_epochs
print("total steps: ", total_steps)
num_steps = 0
self._train(to_train, len(train_gen), num_steps, val_gen)
cur_weight = self._model.state_dict()
torch.save(cur_weight, '{}{}.pt'.format(self._weight_path, self.step))
print('Current best weight: {}.pt, best auc: {}'.format(self.max_step, self.max_auc))
def _forward(self, batch):
#print('device: ', self._device)
batch = {k: t.to(self._device, non_blocking=True) for k, t in batch.items()}
#label = Variable(batch['label']).to(self._device) # shape: (batch_size, 1)
#inputs = Variable(batch['input']).to(self._device)
label = batch['label']
output = self._model(batch['input'], batch['target_id'])
pred = (torch.sigmoid(output) >= self._threshold).long() # shape: (batch_size, 1)
loss = self._loss_fn(output, label.float())
return label, output, pred, loss.mean()
def _get_loss(self, label, output):
loss = self._loss_fn(output, label.float())
return loss.mean()
def _train(self, batch_iter, num_iter, num_batches, val_gen):
start_time = time.time()
self._model.train()
losses = 0
num_corrects = 0
num_total = 0
labels = []
outs = []
data_time = AverageMeter()
batch_time = AverageMeter()
#losses = AverageMeter()
end = time.time()
for i, batch in enumerate(batch_iter):
i += 1
data_time.update(time.time() - end)
label, out, pred, train_loss = self._forward(batch)
#train_loss = self._get_loss(label, out)
losses += train_loss.item()
#compute gradient and do Noam step
self._opt.step(train_loss)
#measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
num_corrects += (pred == label).sum().item()
num_total += len(label)
if i%1000 == 0:
labels.extend(label.squeeze(-1).data.cpu().numpy())
outs.extend(out.squeeze(-1).data.cpu().numpy())
acc = num_corrects / num_total
auc = roc_auc_score(labels, outs)
losses = losses / 1000
training_time = time.time() - start_time
print('correct: {}, total: {}'.format(num_corrects, num_total))
#print('[Train] time: {}, loss: {}, acc: {}, auc: {}'.format(training_time, loss, acc, auc))
print('[Train] [{0}/{1}]\t'
'time {batch_time.avg:.3f}\t'
'data time {data_time.avg:.3f}\t'
'loss {loss:.4f}\t'
'acc {acc:.4f}\t'
'auc {auc:.4f}\t'
'training time {training_time:.4f}\t'.format(
i, num_iter, acc=acc, auc=auc, batch_time=batch_time, data_time=data_time, loss=losses, training_time=training_time
))
losses = 0
if i % 10000 == 0:
self._test('Validation', val_gen, i)
print('Current best weight: {}.pt, best auc: {}'.format(self.max_step, self.max_auc))
cur_weight = self._model.state_dict()
torch.save(cur_weight, '{}{}.pt'.format(self._weight_path, i))
mylist = []
mylist.append(self.max_auc)
mylist.append(self.max_acc)
mylist.append(i)
with open('output.csv', 'a', newline='') as file:
writer = csv.writer(file)
writer.writerow(mylist)
print('--------------------------------------------------------')
