Hi,
I create a dataloader to load features from local files by their file pathes.
The dataloader can be simplfied as:
(P.S. All codes were tested on Pytorch 1.0.0 and Pytorch 1.0.1. Memory capacity of my machine is 256Gb)
import numpy as np
import torch
import torch.utils.data as data
import time
class MyDataSet(data.Dataset):
def __init__(self):
super(MyDataSet, self).__init__()
# Assume that the self.infoset here contains the description information about the dataset.
# Here it is a list of strings. I set it aoubt 8Gb in memory.
# In my real project, this infoset may be 40Gb in memory.
self.infoset = [str(i).zfill(1024) for i in range(len(self))]
def __getitem__(self, index):
info = self.infoset[index] # problem is here
items = {}
items['features'] = self.load_feature(info)
return items
def load_feature(self, info):
'''
Load feature from files
'''
feature = torch.Tensor(np.ones([8, 4, 2], dtype=np.float32))
return feature
def __len__(self):
return 8000000
dataset = MyDataSet()
dataloader = data.DataLoader(dataset, batch_size=1024, shuffle=True, num_workers=16, pin_memory=True)
while True:
for i, sample in enumerate(dataloader):
print(i, len(dataloader))
time.sleep(0.05) # slow down the process to see the mem-usage increasing during one epoch
During each epoch, the memory usage is about 13GB at the very beginning and keeps inscreasing and finally up to about 46Gb. Although it will decrease to 13GB at the beginning of next epoch, this problem is serious because in my real project the infoset is about 40Gb due to the large number of samples and finally leads to Out of Memory (OOM).
I have found that the problem is caused by the first line of MyDataset.__getitem__(), in the following code, infoset is just read but not used, but the same problem still happen.
class MyDataSet(data.Dataset):
def __init__(self):
super(MyDataSet, self).__init__()
# Assume that the self.infoset here contains the description information about the dataset.
# Here it is a list of strings. I set it aoubt 8Gb in memory.
# In my real project, this infoset may be 40Gb in memory.
self.infoset = [str(i).zfill(1024) for i in range(len(self))]
def __getitem__(self, index):
info = self.infoset[index] # problem is here
items = {}
# items['features'] = self.load_feature(info)
return items
def load_feature(self, info):
'''
Load feature from files
'''
feature = torch.Tensor(np.ones([8, 4, 2], dtype=np.float32))
return feature
def __len__(self):
return 8000000
dataset = MyDataSet()
dataloader = data.DataLoader(dataset, batch_size=1024, shuffle=True, num_workers=16, pin_memory=True)
while True:
for i, sample in enumerate(dataloader):
print(i, len(dataloader))
time.sleep(0.05) # slow down the process to see the mem-usage increasing during one epoch
As a contrast, when I comment the first line of MyDataset.__getitem__(), the memory usage keeps stable:
class MyDataSet(data.Dataset):
def __init__(self):
super(MyDataSet, self).__init__()
# Assume that the self.infoset here contains the description information about the dataset.
# Here it is a list of strings. I set it aoubt 8Gb in memory.
# In my real project, this infoset may be 40Gb in memory.
self.infoset = [str(i).zfill(1024) for i in range(len(self))]
def __getitem__(self, index):
# info = self.infoset[index] # problem is here
info = 'fake info'
items = {}
items['features'] = self.load_feature(info)
return items
def load_feature(self, info):
'''
Load feature from files
'''
feature = torch.Tensor(np.ones([8, 4, 2], dtype=np.float32))
return feature
def __len__(self):
return 8000000
dataset = MyDataSet()
dataloader = data.DataLoader(dataset, batch_size=1024, shuffle=True, num_workers=16, pin_memory=True)
while True:
for i, sample in enumerate(dataloader):
print(i, len(dataloader))
time.sleep(0.05) # slow down the process to see the mem-usage increasing during one epoch
Any reasons and suggestions about this problems?