DataLoaders - Multiple files, and multiple rows per column with lazy evaluation

8 
class LazyTextDataset(Dataset):
    def __init__(self, filename):
        self._filename = filename
        self._total_data = int(subprocess.check_output("wc -l " + filename, shell=True).split()[0]) - 1

    def __getitem__(self, idx):
        line = linecache.getline(self._filename, idx + 1)
        csv_line = csv.reader([line])
        return next(csv_line)
      
    def __len__(self):
        return self._total_data

path = /where_csv_files_are_dumped/
files = list(map(lambda x : path + x, (filter(lambda x : x.endswith("csv"), os.listdir(path)))))
datasets = list(map(lambda x : LazyTextDataset(x), files))
dataset = ConcatDataset(datasets)

This should work for multiple csv files where each row is represented by a training example.