class DeephomographyDataset(Dataset):
'''
DeepHomography Dataset
'''
def __init__(self,hdf5file,imgs_key='images',labels_key='labels',
transform=None):
'''
:argument
:param hdf5file: the hdf5 file including the images and the label.
:param transform (callable, optional): Optional transform to be
applied on a sample
'''
self.db=h5py.File(hdf5file,'r') # store the images and the labels
keys=list(self.db.keys())
if imgs_key not in keys:
raise(' the ims_key should not be {}, should be one of {}'
.format(imgs_key,keys))
if labels_key not in keys:
raise(' the labels_key should not be {}, should be one of {}'
.format(labels_key,keys))
self.imgs_key=imgs_key
self.labels_key=labels_key
self.transform=transform
def __len__(self):
return len(self.db[self.labels_key])
def __getitem__(self, idx):
image=self.db[self.imgs_key][idx]
label=self.db[self.labels_key][idx]
sample={'images':image,'labels':label}
if self.transform:
sample=self.transform(sample)
return sample
batchSize=30
trainDataset=DeephomographyDataset(pth_config.TRAIN_H5_FILE,
transform=transforms.Lambda(
lambda x: toTensor(x)))
traindataloader=DataLoader(trainDataset,batch_size=batchSize,shuffle=True,
num_workers=20)
# samplesss=trainDataset[0]
for i, sample in enumerate(trainDataset):
....
some errors are repoted as follows:
Traceback (most recent call last):
File "/home/dler/pytorch_codes_from_pc4/DeepHomography/my_Model/training_deephomography.py", line 138, in <module>
epoch_num)
File "/home/dler/pytorch_codes_from_pc4/DeepHomography/my_Model/training_deephomography.py", line 49, in train_model
for i, sample in enumerate(dataloaders[phase]):
File "/home/dler/anaconda3/envs/pytorch4/lib/python3.6/site-packages/torch/utils/data/dataloader.py", line 322, in __next__
return self._process_next_batch(batch)
File "/home/dler/anaconda3/envs/pytorch4/lib/python3.6/site-packages/torch/utils/data/dataloader.py", line 357, in _process_next_batch
raise batch.exc_type(batch.exc_msg)
KeyError: 'Traceback (most recent call last):\n File "/home/dler/anaconda3/envs/pytorch4/lib/python3.6/site-packages/torch/utils/data/dataloader.py", line 106, in _worker_loop\n samples = collate_fn([dataset[i] for i in batch_indices])\n File "/home/dler/anaconda3/envs/pytorch4/lib/python3.6/site-packages/torch/utils/data/dataloader.py", line 106, in <listcomp>\n samples = collate_fn([dataset[i] for i in batch_indices])\n File "/home/dler/pytorch_codes_from_pc4/DeepHomography/preprocessing/generate_dataset.py", line 34, in __getitem__\n label=self.db[self.labels_key][idx]\n File "h5py/_objects.pyx", line 54, in h5py._objects.with_phil.wrapper\n File "h5py/_objects.pyx", line 55, in h5py._objects.with_phil.wrapper\n File "/home/dler/anaconda3/envs/pytorch4/lib/python3.6/site-packages/h5py/_hl/group.py", line 167, in __getitem__\n oid = h5o.open(self.id, self._e(name), lapl=self._lapl)\n File "h5py/_objects.pyx", line 54, in h5py._objects.with_phil.wrapper\n File "h5py/_objects.pyx", line 55, in h5py._objects.with_phil.wrapper\n File "h5py/h5o.pyx", line 190, in h5py.h5o.open\nKeyError: \'Unable to open object (bad object header version number)\'\n'
Exception ignored in: <bound method _DataLoaderIter.__del__ of <torch.utils.data.dataloader._DataLoaderIter object at 0x7f09bc58dac8>>
Traceback (most recent call last):
File "/home/dler/anaconda3/envs/pytorch4/lib/python3.6/site-packages/torch/utils/data/dataloader.py", line 399, in __del__
self._shutdown_workers()
File "/home/dler/anaconda3/envs/pytorch4/lib/python3.6/site-packages/torch/utils/data/dataloader.py", line 378, in _shutdown_workers
self.worker_result_queue.get()
File "/home/dler/anaconda3/envs/pytorch4/lib/python3.6/multiprocessing/queues.py", line 337, in get
return _ForkingPickler.loads(res)
File "/home/dler/anaconda3/envs/pytorch4/lib/python3.6/site-packages/torch/multiprocessing/reductions.py", line 151, in rebuild_storage_fd
fd = df.detach()
File "/home/dler/anaconda3/envs/pytorch4/lib/python3.6/multiprocessing/resource_sharer.py", line 57, in detach
with _resource_sharer.get_connection(self._id) as conn:
File "/home/dler/anaconda3/envs/pytorch4/lib/python3.6/multiprocessing/resource_sharer.py", line 87, in get_connection
c = Client(address, authkey=process.current_process().authkey)
File "/home/dler/anaconda3/envs/pytorch4/lib/python3.6/multiprocessing/connection.py", line 487, in Client
c = SocketClient(address)
File "/home/dler/anaconda3/envs/pytorch4/lib/python3.6/multiprocessing/connection.py", line 614, in SocketClient
s.connect(address)
ConnectionRefusedError: [Errno 111] Connection refused
Process finished with exit code 1
Interestingly, when the code samplesss=trainDataset[0]; ` uncomment, there is no error reported! But, the dataset traindataloader returned by the DataLoader is wrong, namely some data is not the raw data.
So, I know how the num_wokers in DataLoader affect the code?
in addition, my computer have 32 cpu cores. I set num_workers to 20.
My OS is unbuntu 16.0, the version of pytorch is 0.4! python is 3.6