I am getting an error when I enumerate my custom data set. My Dataset has 13 pickle files which I load and then processing it using my custom build Dataset class. However when i tried to enumerate my dataset I am ran out of input.
Here is the trace back of the error
Traceback (most recent call last):
File "train_2.py", line 137, in <module>
train(model, device,criterion, trainLoader, optimizer, epoch,losses)
File "train_2.py", line 33, in train
for batchIdx, (data, target) in enumerate(trainLoader):
File "C:\Users\user_name\AppData\Local\Continuum\anaconda3\lib\site-packages\torch\utils\data\dataloader.py", line 501, in __iter__
__mp_main__
return _DataLoaderIter(self)
File "C:\Users\user_name\AppData\Local\Continuum\anaconda3\lib\site-packages\torch\utils\data\dataloader.py", line 289, in __init__
w.start()
File "C:\Users\user_name\AppData\Local\Continuum\anaconda3\lib\multiprocessing\process.py", line 105, in start
self._popen = self._Popen(self)
File "C:\Users\user_name\AppData\Local\Continuum\anaconda3\lib\multiprocessing\context.py", line 223, in _Popen
return _default_context.get_context().Process._Popen(process_obj)
File "C:\Users\user_name\AppData\Local\Continuum\anaconda3\lib\multiprocessing\context.py", line 322, in _Popen
return Popen(process_obj)
File "C:\Users\user_name\AppData\Local\Continuum\anaconda3\lib\multiprocessing\popen_spawn_win32.py", line 65, in __init__
reduction.dump(process_obj, to_child)
File "C:\Users\user_name\AppData\Local\Continuum\anaconda3\lib\multiprocessing\reduction.py", line 60, in dump
ForkingPickler(file, protocol).dump(obj)
OSError: [Errno 22] Invalid argument
C:\Users\user_name\AppData\Local\Continuum\anaconda3\lib\site-packages\h5py\__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.
from ._conv import register_converters as _register_converters
Traceback (most recent call last):
File "<string>", line 1, in <module>
File "C:\Users\user_name\AppData\Local\Continuum\anaconda3\lib\multiprocessing\spawn.py", line 105, in spawn_main
exitcode = _main(fd)
File "C:\Users\user_namer\AppData\Local\Continuum\anaconda3\lib\multiprocessing\spawn.py", line 115, in _main
self = reduction.pickle.load(from_parent)
EOFError: Ran out of input
The code for the data loading is the following:-
Here my data set is divided into 13 pickle files.
class PATCHLABEL(data.Dataset):
base_folder = 'data_batchs'
train_list = [
['data_batch_1'],
['data_batch_2'],
['data_batch_3'],
['data_batch_4'],
['data_batch_5'],
['data_batch_6'],
['data_batch_7'],
['data_batch_8'],
['data_batch_9'],
['data_batch_10'],
['data_batch_11'],
['data_batch_12'],
['data_batch_13'],
]
test_list = [
['test_batch'],
]
patch_dim = 1299
train_size = 37922*13#492986
test_size = 31468
@property
def targets(self):
if self.train:
return self.train_labels
else:
return self.test_labels
def __init__(self, root, train=True,
transform=None, target_transform=None):
self.root = os.path.expanduser(root)
self.transform = transform
self.target_transform = target_transform
self.train = train # training set or test set
if self.train:
self.train_data = []
self.train_labels = []
for fentry in self.train_list:
f = fentry[0]
file = os.path.join(self.root, self.base_folder, f)
fo = open(file, 'rb')
entry = pickle.load(fo,encoding='latin1')
xTrain = entry[:,:self.patch_dim] #Extracting Images
yTrain = entry[:,self.patch_dim:] #Extracting Labels
self.train_data.append(xTrain[:,3:])
self.train_labels += list(yTrain[:,2])
fo.close()
self.train_data = np.concatenate(self.train_data)
self.train_data = self.train_data.reshape((self.train_size, 36, 36))
else:
f = self.test_list[0][0]
file = os.path.join(self.root, self.base_folder, f)
fo = open(file, 'rb')
entry = pickle.load(fo,encoding='latin1')
xTrain = entry[:,:self.patch_dim] #Extracting Images
yTrain = entry[:,self.patch_dim:] #Extracting Labels
self.test_data = xTrain[:,3:]
self.test_labels = list(yTrain[:,2])
fo.close()
self.test_data = self.test_data.reshape((self.test_size,36, 36))
def __getitem__(self, index):
"""
Args:
index (int): Index
Returns:
tuple: (image, target) where target is index of the target class.
"""
if self.train:
patch, target = self.train_data[index], self.train_labels[index]
else:
patch, target = self.test_data[index], self.test_labels[index]
if self.transform is not None:
patch = self.transform(patch)
if self.target_transform is not None:
target = self.target_transform(target)
return patch, target
def __len__(self):
if self.train:
return len(self.train_data)
else:
return len(self.test_data)
I am using PyTorch version 0.4.1 and Windows 10.
Any suggestions on how i can solve this error ?
Thanks