If you do not delete or comment the piece of code:
dataloader = torch.utils.data.DataLoader(
H5Dataset('test.h5'),
batch_size=32,
num_workers=0,
shuffle=True
)
count1=0
for i, (data, target) in enumerate(dataloader):
# print(data.shape)
count1+=target
print('count1 is equal to \n{}:'.format(count1))
There is no error reported, but the target returned is no right. This is can be illustrated in the following script:
import h5py
import numpy as np
import torch
from torch.utils.data import Dataset, DataLoader
class H5Dataset(Dataset):
def __init__(self, h5_path):
self.h5_path = h5_path
self.h5_file = h5py.File(h5_path, 'r')
self.length = len(h5py.File(h5_path, 'r'))
def __getitem__(self, index):
record = self.h5_file[str(index)]
return (
record['data'].value,
record['target'].value,
)
def __len__(self):
return self.length
# --
# Make data
# f = h5py.File('test.h5')
# for i in range(256):
# f['%s/data' % i] = np.random.uniform(0, 1, (1024, 1024))
# f['%s/target' % i] = np.random.choice(10)
# Runs correctly
dataloader = torch.utils.data.DataLoader(
H5Dataset('test.h5'),
batch_size=32,
num_workers=0,
shuffle=True
)
count1=0
for i, (data, target) in enumerate(dataloader):
# print(data.shape)
count1+=target
print('count1 is equal to \n{}:'.format(count1))
# if i > 10:
# break
# Throws error (sometimes, may have to restart python)
dataloader = torch.utils.data.DataLoader(
H5Dataset('test.h5'),
batch_size=32,
num_workers=24,
shuffle=True
)
count2=0
for i, (data, target) in enumerate(dataloader):
# print(data.shape)
# print(target.shape)
count2+=target
# if i > 10:
# break
print('count2 is equal to :\n{}'.format(count2))
The output is as follows:
count1 is equal to
tensor([49, 24, 51, 44, 38, 37, 33, 26, 44, 37, 44, 39, 53, 46, 29, 28, 31, 35,
28, 42, 32, 34, 35, 28, 44, 22, 32, 39, 40, 34, 34, 30]):
count2 is equal to :
tensor([ -40011464938561591, 36, -4640496823984442177,
4596629407495505507, 9180213616633249665, -4690268498061449214,
4565213464737575129, -4677703075192396625, 4551168039923175676,
-50017473233541162, 4552094186216834730, 4556169682132194915,
9164131086913356889, -45353263880063689, -4707432270418525107,
-84909258148334466, 4567934797786953787, 4561996622320321486,
4567793662555240134, 9170571099677535639, 4549180305499652268,
9149618509619540126, -38544862363653458, -4641323993829215895,
-33862949871259449, 4560804812378475854, 4573418027288026608,
4559805757876447575, 4564365524252319916, 9136588953091962375,
-4645116466038797201, 9210419057565000560])
In fact, count1 should be equal to count2.
I hope these reproduced errors are helpful for your developers! Thank you in advance.