I’m getting a runtime error that :
An attempt has been made to start a new process before the current process has finished its bootstrapping phase.
When num_workers = 0, there is no problem at all. But when I increase it to 1, I’m getting the above error.
Code used :
import torch
import torchvision
import torchvision.transforms as transforms
import torch.nn as nn
from torch.utils.data import DataLoader
from torch.utils.data import Dataset
class MyDataset(Dataset):
def __init__(self, nb_images):
data = torch.ones(5, 5).expand(nb_images, -1, -1)
self.data = data * torch.arange(nb_images)[:, None, None].expand_as(data)
self.target = torch.arange(nb_images)
def __getitem__(self, index):
x = self.data[index]
y = self.target[index]
return x, y
def __len__(self):
return len(self.data)
train_set = MyDataset(nb_images=12)
loader = DataLoader(
train_set,
batch_size=1,
num_workers=1,
# prefetch_factor=None
)
image, label = next(iter(loader))
print(image.shape)
print(label.shape)
Looks like error is due to not checking for main in spawn.py ?
The second part of execption where it complains that queue is empty is understandable since the worker failed to fetch, the data queue would be empty
I haven’t modified any source code in pytorch while testing the above
Complete Error Trace :
image, label = next(iter(loader))
File "C:\Users\V Hegde\AppData\Local\Programs\Python\Python39\lib\site-packages\torch\utils\data\dataloader.py", line 437, in __iter__
return self._get_iterator()
File "C:\Users\V Hegde\AppData\Local\Programs\Python\Python39\lib\site-packages\torch\utils\data\dataloader.py", line 383, in _get_iterator
return _MultiProcessingDataLoaderIter(self)
File "C:\Users\V Hegde\AppData\Local\Programs\Python\Python39\lib\site-packages\torch\utils\data\dataloader.py", line 1037, in __init__
w.start()
File "C:\Users\V Hegde\AppData\Local\Programs\Python\Python39\lib\multiprocessing\process.py", line 121, in start
self._popen = self._Popen(self)
File "C:\Users\V Hegde\AppData\Local\Programs\Python\Python39\lib\multiprocessing\context.py", line 224, in _Popen
return _default_context.get_context().Process._Popen(process_obj)
File "C:\Users\V Hegde\AppData\Local\Programs\Python\Python39\lib\multiprocessing\context.py", line 327, in _Popen
return Popen(process_obj)
File "C:\Users\V Hegde\AppData\Local\Programs\Python\Python39\lib\multiprocessing\popen_spawn_win32.py", line 45, in __init__
prep_data = spawn.get_preparation_data(process_obj._name)
File "C:\Users\V Hegde\AppData\Local\Programs\Python\Python39\lib\multiprocessing\spawn.py", line 154, in get_preparation_data
_check_not_importing_main()
File "C:\Users\V Hegde\AppData\Local\Programs\Python\Python39\lib\multiprocessing\spawn.py", line 134, in _check_not_importing_main
raise RuntimeError('''
RuntimeError:
An attempt has been made to start a new process before the
current process has finished its bootstrapping phase.
This probably means that you are not using fork to start your
child processes and you have forgotten to use the proper idiom
in the main module:
if __name__ == '__main__':
freeze_support()
...
The "freeze_support()" line can be omitted if the program
is not going to be frozen to produce an executable.
Traceback (most recent call last):
File "C:\Users\V Hegde\AppData\Local\Programs\Python\Python39\lib\site-packages\torch\utils\data\dataloader.py", line 1123, in _try_get_data
data = self._data_queue.get(timeout=timeout)
File "C:\Users\V Hegde\AppData\Local\Programs\Python\Python39\lib\multiprocessing\queues.py", line 114, in get
raise Empty
_queue.Empty
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "d:\DREAM\pytorch\mycode\dataloader_test_1.py", line 47, in <module>
image, label = next(iter(loader))
File "C:\Users\V Hegde\AppData\Local\Programs\Python\Python39\lib\site-packages\torch\utils\data\dataloader.py", line 630, in __next__
data = self._next_data()
File "C:\Users\V Hegde\AppData\Local\Programs\Python\Python39\lib\site-packages\torch\utils\data\dataloader.py", line 1319, in _next_data
idx, data = self._get_data()
File "C:\Users\V Hegde\AppData\Local\Programs\Python\Python39\lib\site-packages\torch\utils\data\dataloader.py", line 1285, in _get_data
success, data = self._try_get_data()
File "C:\Users\V Hegde\AppData\Local\Programs\Python\Python39\lib\site-packages\torch\utils\data\dataloader.py", line 1136, in _try_get_data
raise RuntimeError('DataLoader worker (pid(s) {}) exited unexpectedly'.format(pids_str)) from e
RuntimeError: DataLoader worker (pid(s) 8984) exited unexpectedly`