There is such a class for forming a dataset:
class SiameseNetworkDataset(Dataset):
def __init__(self, imageFolderDataset, transform=None, should_invert=True):
self.imageFolderDataset = imageFolderDataset
self.transform = transform
self.should_invert = should_invert
np.random.seed(123)
random.seed(123)
# Defines a set of random logos by index (for each epoch)
self.random_indexes = np.random.randint(
len(self.imageFolderDataset.imgs),
size=int((len(self.imageFolderDataset.imgs)) / Config.train_batch_size) + 1)
def __getitem__(self, index):
# reset the indexes every epoch
if index == 0:
self.random_indexes = np.random.randint(len(self.imageFolderDataset.imgs), size=int(
(len(self.imageFolderDataset.imgs)) / Config.train_batch_size) + 1)
# get the index for the current batch
img0_tuple = self.imageFolderDataset.imgs[self.random_indexes[int(index/Config.train_batch_size)]]
# we need to make sure approx 50% of images are in the same class
should_get_same_class = random.randint(0, 1)
# Search for class by looping random indexes
if should_get_same_class:
while True:
# keep looping till the same class image is found
img1_tuple = random.choice(self.imageFolderDataset.imgs)
if img0_tuple[1] == img1_tuple[1]:
break
else:
while True:
# keep looping till a different class image is found
img1_tuple = random.choice(self.imageFolderDataset.imgs)
if img0_tuple[1] != img1_tuple[1]:
break
img0 = Image.open(img0_tuple[0])
img1 = Image.open(img1_tuple[0])
#img0 = img0.convert("L")
#img1 = img1.convert("L")
if self.should_invert:
img0 = PIL.ImageOps.invert(img0)
img1 = PIL.ImageOps.invert(img1)
if self.transform is not None:
img0 = self.transform(img0)
img1 = self.transform(img1)
return img0, img1, torch.from_numpy(np.array([int(img1_tuple[1] != img0_tuple[1])], dtype=np.float32))
def __len__(self):
return len(self.imageFolderDataset.imgs)
This code is executed instantly:
folder_dataset = torchvision.datasets.ImageFolder(root = “./openlogo_100”)
siamese_dataset = SiameseNetworkDataset(imageFolderDataset = folder_dataset,
transform=transforms.Compose([transforms.Resize((Config.im_w, Config.im_h)), transforms.ToTensor()]))
train_dataloader = DataLoader(siamese_dataset, pin_memory = False,
shuffle=False)
But the execution of this code takes infinity of time:
for batch_idx, samples in enumerate(train_dataloader):
print(batch_idx)
Sometimes in the console I see this: 1, 2, 3 or 1, 2 or 1
But it never goes beyond 3
often the output is generally empty, and at the same time
At the same time, the processor is loaded about 20% and the load does not disappear after the kernel is rebooted
But when I see some number in the console, I stop the execution and run the following code:
np.shape(samples)
In this case, I see the normal part of the dataset as [tensor(img_1), tensor(img_2), tensor(0 or 1)]
What is the problem? I tried to solve it by controlling the parameters num_workers, pin_memory, shuffle, I also tried to do it on a Mac OS computer. Nothing helped.