hi … I am using the data loader below, but while training my on GPU, the model takes too much time. I calculated the time for single line of for loop main it take 2 sec , this make GPU waiting for next batch like gpu performance reach to 100 and for a second and than zero in every step of iteration. I set the no of worker to zero. my cpu usage is about 1800%. the code for data loader and training loop is given below.
My dataset class is
class myDataSet(Dataset):
# 继承Dataset, é‡è½½__init__, __getitem__, __len__
def __init__(self, fileList, transform_mode=None, list_reader=PIL_list_reader, loader=img_loader, cuda=True):
self.channel_num = channel_num
self.image_size = img_size
self.loader = loader
self.Nz = 50
self.imgList, self.pose_label, self.id_label, self.Np, self.Nd = list_reader(fileList)
if transform_mode=='train_transform': self.transform = multiPIE_train_transform
elif transform_mode=='test_transform': self.transform = multiPIE_test_transform
elif transform_mode=='trainaug_transform': self.transform = multiPIE_train_aug_transform
elif transform_mode=='testaug_transform': self.transform = multiPIE_test_aug_transform
else: self.transform=None
def __getitem__(self, index):
#start=time.time()
imgPath = self.imgList[index]
img = self.loader(imgPath)
if self.transform is not None:
img = self.transform(img)
#print("time of get item ",time.time()- start)
return img, self.id_label[index], self.pose_label[index]
def __len__(self):
return len(self.imgList)
the training loop is
for i, [batch_image, batch_id_label, batch_pose_label] in enumerate(dataloader):
D_model.zero_grad()
G_model.zero_grad()
batch_size = batch_image.size(0)
batch_real_label = torch.ones(batch_size)
batch_sys_label = torch.zeros(batch_size)
# generate noise code and pose code, label: LongTensor, input: FloatTensor
noise = torch.FloatTensor(np.random.uniform(-1,1, (batch_size, Nz)))
pose_code_label = np.random.randint(Np, size=batch_size) # get a list of int in range(Np)
pose_code = np.zeros((batch_size, Np))
pose_code[range(batch_size), pose_code_label] = 1
pose_code_label = torch.LongTensor(pose_code_label.tolist())
pose_code = torch.FloatTensor(pose_code.tolist())
batch_pose_code = np.zeros((batch_size, Np))
batch_pose_code[range(batch_size), batch_pose_label] = 1
batch_pose_code = torch.FloatTensor(pose_code.tolist())
# use cuda for label and input
if args.cuda:
batch_image, batch_id_label, batch_pose_label, batch_real_label, batch_sys_label = \
batch_image.cuda(), batch_id_label.cuda(), batch_pose_label.cuda(), batch_real_label.cuda(), batch_sys_label.cuda()
noise, pose_code, pose_code_label, batch_pose_code = \
noise.cuda(), pose_code.cuda(), pose_code_label.cuda(), batch_pose_code.cuda()
# use Variable for label and input
batch_image, batch_id_label, batch_pose_label, batch_real_label, batch_sys_label = \
Variable(batch_image), Variable(batch_id_label), Variable(batch_pose_label), Variable(batch_real_label), Variable(batch_sys_label)
noise, pose_code, pose_code_label, batch_pose_code = \
Variable(noise), Variable(pose_code), Variable(pose_code_label), Variable(batch_pose_code)
# generator forward
generated = G_model(batch_image, pose_code, noise) #forward