.mat image file reading: TypeError: batch must contain tensors, numbers, dicts or lists; found <type 'NoneType'>

I have image dataset that is in .mat format. But when I use my dataloader, I get the following error:

Traceback (most recent call last):
  File "vanilla_triplet_vae.py", line 808, in <module>
  File "vanilla_triplet_vae.py", line 779, in main
    train(train_loader, tnet,decoder,descriptor, criterion, optimizer, epoch)
  File "vanilla_triplet_vae.py", line 418, in train
    for batch_idx, (data1, data2, data3) in enumerate(train_loader):
  File "/home/hmishfaq/anaconda2/lib/python2.7/site-packages/torch/utils/data/dataloader.py", line 201, in __next__
    return self._process_next_batch(batch)
  File "/home/hmishfaq/anaconda2/lib/python2.7/site-packages/torch/utils/data/dataloader.py", line 221, in _process_next_batch
    raise batch.exc_type(batch.exc_msg)
TypeError: Traceback (most recent call last):
  File "/home/hmishfaq/anaconda2/lib/python2.7/site-packages/torch/utils/data/dataloader.py", line 40, in _worker_loop
    samples = collate_fn([dataset[i] for i in batch_indices])
  File "/home/hmishfaq/anaconda2/lib/python2.7/site-packages/torch/utils/data/dataloader.py", line 112, in default_collate
TypeError: batch must contain tensors, numbers, dicts or lists; found <type 'NoneType'>

My dataloader code is this:

from PIL import Image
import os
import os.path
import torch.utils.data
import torchvision.transforms as transforms
import numpy as np
import scipy.io as sio
import torch

filenames = {'train': ['triplet_train.csv'],
             'val': ['triplet_val.csv'],
             'test': ['triplet_test.csv']}

def default_image_loader(path):
    data = sio.loadmat(path) #example path /data/p110.mat
    return data['NormImg']
    # return Image.open(path).convert('RGB')

class TripletImageLoader(torch.utils.data.Dataset):
    def __init__(self, root, base_path, filenames_filename, split, n_triplets, transform=None,
        """ filenames_filename: A text file with each line containing the path to an image e.g.,
            triplets_file_name: A text file with each line containing three integers, 
                where integer i refers to the i-th image in the filenames file. 
                For a line of intergers 'a b c', a triplet is defined such that image a is more 
                similar to image c than it is to image b, e.g., 
                0 2017 42 """
        self.root = root
        self.base_path = base_path  
        self.filenamelist = []
        print(os.path.join(self.root, filenames_filename))
        for line in open(os.path.join(self.root, filenames_filename)):
        triplets = []
        if split == 'train':
            fnames = filenames['train']
        elif split == 'val':
            fnames = filenames['val']
            fnames = filenames['test']

        for line in open(os.path.join(self.root, split, fnames[0])):
            triplets.append((line.split(',')[0], line.split(',')[1], line.split(',')[2])) # anchor, far, close   
        self.triplets = triplets[:int(n_triplets)]
        self.transform = transform
        self.loader = loader

    def __getitem__(self, index):
        path1, path2, path3 = self.triplets[index]

        if os.path.exists(os.path.join('..', self.filenamelist[int(path1)])) and os.path.exists(os.path.join('..', self.filenamelist[int(path2)])) and os.path.exists(os.path.join('..', self.filenamelist[int(path3)])):
            img1 = self.loader(os.path.join('..', self.filenamelist[int(path1)]))
            img2 = self.loader(os.path.join('..', self.filenamelist[int(path2)]))
            img3 = self.loader(os.path.join('..', self.filenamelist[int(path3)]))

            img1 = Image.fromarray(img1) #torch.from_numpy(img1)
            img2 = Image.fromarray(img2) #torch.from_numpy(img2)
            img3 = Image.fromarray(img3) #torch.from_numpy(img3)

            if self.transform is not None:
                img1 = self.transform(img1)
                img2 = self.transform(img2)
                img3 = self.transform(img3)
            return img1, img2, img3
            return None

    def __len__(self):
        return len(self.triplets)

Not sure where the error is coming from. I used the exact same dataloader for jpeg format images and it was working fine.

It seems that your Dataset returns None, because the specified paths do not exist in the __getitem__ method.
If you really need this condition, try to return a zero tensor instead of None.

1 Like