DatasetFolder pickle loader

vision
1

I’m trying to use DatasetFolder in order to use a pickle data loader with my transform and batch_size.

( I also tried adding to the transform: transforms.ToPILImage())

I tried this code which results in error because the whole list is loaded as one element (the data size is 1 and should be 500)

def pickle_loader(input):
    return pickle.load(open(input))

test_data= torchvision.datasets.DatasetFolder(root='.', loader=pickle_loader, extensions='.pickle', transform=transform)
test_loader = torch.utils.data.DataLoader(
        dataset=test_data,
        batch_size=batch_size,
        shuffle=False)



test_labels = []
for x in test_loader:
    x = Variable(x)
    out = model_conv(x)
    _, pred_label = torch.max(out.data, 1)
    test_labels.append(pred_label)

Traceback (most recent call last):
  File "/home/noay/PycharmProjects/ML3/ex_9_code.py", line 206, in main
    for x in test_loader:
  File "/home/noay/anaconda2/lib/python2.7/site-packages/torch/utils/data/dataloader.py", line 179, in __next__
    batch = self.collate_fn([self.dataset[i] for i in indices])
  File "/home/noay/anaconda2/lib/python2.7/site-packages/torchvision/datasets/folder.py", line 103, in __getitem__
    sample = self.transform(sample)
  File "/home/noay/anaconda2/lib/python2.7/site-packages/torchvision/transforms/transforms.py", line 49, in __call__
    img = t(img)
  File "/home/noay/anaconda2/lib/python2.7/site-packages/torchvision/transforms/transforms.py", line 76, in __call__
    return F.to_tensor(pic)
  File "/home/noay/anaconda2/lib/python2.7/site-packages/torchvision/transforms/functional.py", line 44, in to_tensor
    raise TypeError('pic should be PIL Image or ndarray. Got {}'.format(type(pic)))
TypeError: pic should be PIL Image or ndarray. Got <type 'list'>
2

If you are ok with your datum being a numpy array, you can change your loader to:

def pickle_loader(input):
    item = pickle.load(open(input, 'rb'))
    return item.values
3

You can use the pil_loader directly or if possible the default_loader that will try to use high performance image loader accimage_loader

def pil_loader(path):
    # open path as file to avoid ResourceWarning (https://github.com/python-pillow/Pillow/issues/835)
    with open(path, 'rb') as f:
        img = Image.open(f)
        return img.convert('RGB')


def accimage_loader(path):
    import accimage
    try:
        return accimage.Image(path)
    except IOError:
        # Potentially a decoding problem, fall back to PIL.Image
        return pil_loader(path)


def default_loader(path):
    from torchvision import get_image_backend
    if get_image_backend() == 'accimage':
        return accimage_loader(path)
    else:
        return pil_loader(path)