Getting error in Dataset loader

I am trying to implement this following code on my system.

In the original code, they used two datasets named “DIV2k” and “Flicker2K” for training. But I want to use only the DIV2K dataset for training. To do so, I have first divide the dataset images into sub-images and then change the init.py file accordingly:

 from easydict import EasyDict as edict
 import importlib
 import os
from utils.common import scandir
dataset_root = os.path.dirname(os.path.abspath(__file__))
print(dataset_root)
dataset_filenames = [
os.path.splitext(os.path.basename(v))[0] for v in scandir(dataset_root)
    if v.endswith('_dataset.py')
 ]
dataset_modules = [
       importlib.import_module(f'datasets.{file_name}')
       for file_name in dataset_filenames
]
class DATASET:
     LEGAL = ['DIV2K', 'Flickr2K', 'Set5', 'Set14', 'BSDS100', 'Urban100', 'Manga109']
    # training dataset
DIV2K = edict()
**DIV2K.TRAIN = edict()**
**DIV2K.TRAIN.HRx2 = '/home/BebyGAN/Simple-SR-master/datasets/DIV2K/DIV2K_train_HR_sub'**
**DIV2K.TRAIN.HRx3 = '/home/BebyGAN/Simple-SR-master/datasets/DIV2K/DIV2K_train_HR_sub'**
**DIV2K.TRAIN.HRx4 = '/home/BebyGAN/Simple-SR-master/datasets/DIV2K/DIV2K_train_HR_sub'**
**DIV2K.TRAIN.LRx2 = '/home/BebyGAN/Simple-SR-master/datasets/DIV2K/DIV2K_train_LR_bicubic_sub/X2'**
**DIV2K.TRAIN.LRx3 = '/home/BebyGAN/Simple-SR-master/datasets/DIV2K/DIV2K_train_LR_bicubic_sub/X3'**
**DIV2K.TRAIN.LRx4 = '/home/BebyGAN/Simple-SR-master/datasets/DIV2K/DIV2K_train_LR_bicubic_sub/X4' # testing dataset**
Set5 = edict()
Set5.VAL = edict()
Set5.VAL.HRx2 = None
Set5.VAL.HRx3 = None
Set5.VAL.HRx4 = None
Set5.VAL.LRx2 = None
Set5.VAL.LRx3 = None
Set5.VAL.LRx4 = None
Set14 = edict()
Set14.VAL = edict()
Set14.VAL.HRx2 = None
Set14.VAL.HRx3 = None
Set14.VAL.HRx4 = None
Set14.VAL.LRx2 = None
Set14.VAL.LRx3 = None
Set14.VAL.LRx4 = None
 BSDS100 = edict()
BSDS100.VAL = edict()
BSDS100.VAL.HRx2 = '/data/liwenbo/datasets/benchmark_SR/BSDS100/HR/modX2'
BSDS100.VAL.HRx3 = '/data/liwenbo/datasets/benchmark_SR/BSDS100/HR/modX3'
BSDS100.VAL.HRx4 = '/data/liwenbo/datasets/benchmark_SR/BSDS100/HR/modX4'
BSDS100.VAL.LRx2 = '/data/liwenbo/datasets/benchmark_SR/BSDS100/LR_bicubic/X2'
BSDS100.VAL.LRx3 = '/data/liwenbo/datasets/benchmark_SR/BSDS100/LR_bicubic/X3'
BSDS100.VAL.LRx4 = '/data/liwenbo/datasets/benchmark_SR/BSDS100/LR_bicubic/X4'
Urban100 = edict()
Urban100.VAL = edict()
Urban100.VAL.HRx2 = None
Urban100.VAL.HRx3 = None
Urban100.VAL.HRx4 = None
Urban100.VAL.LRx2 = None
Urban100.VAL.LRx3 = None
Urban100.VAL.LRx4 = None
Manga109 = edict()
Manga109.VAL = dict()
Manga109.VAL.HRx2 = None
Manga109.VAL.HRx3 = None
Manga109.VAL.HRx4 = None
Manga109.VAL.LRx2 = None
Manga109.VAL.LRx3 = None
Manga109.VAL.LRx4 = None
def get_dataset(config):
     dataset_type = config.TYPE
     dataset_cls = None
      for module in _dataset_modules:
         dataset_cls = getattr(module, dataset_type, None)
         if dataset_cls is not None:
            break
         if dataset_cls is None:
            raise ValueError(f'Dataset {dataset_type} is not found.')
  hr_paths = []
  lr_paths = []
D = DATASET()
for dataset, split in zip(config.DATASETS, config.SPLITS):
    if dataset not in D.LEGAL or split not in eval('D.%s' % dataset):
        raise ValueError('Illegal dataset.')
    hr_paths.append(eval('D.%s.%s.HRx%d' % (dataset, split, config.SCALE)))
    lr_paths.append(eval('D.%s.%s.LRx%d' % (dataset, split, config.SCALE)))
return dataset_cls(hr_paths, lr_paths, config)`

The config.py file is changed accordingly as shown in below:

from easydict import EasyDict as edict

class Config:
# dataset
DATASET = edict()
DATASET.TYPE = 'MixDataset'
DATASET.DATASETS = ['DIV2K']
DATASET.SPLITS = ['TRAIN']
DATASET.PHASE = 'train'
DATASET.INPUT_HEIGHT = 48
DATASET.INPUT_WIDTH = 48
DATASET.SCALE = 4
DATASET.REPEAT = 1
DATASET.VALUE_RANGE = 255.0
DATASET.SEED = 100`

But, I have got the following error:

Traceback (most recent call last):
File “train.py”, line 17, in
from dataset import get_dataset
File “/home/BebyGAN/Simple-SR-master/exps/BebyGAN/…/…/dataset/init.py”, line 16, in
for file_name in dataset_filenames
File “/home/BebyGAN/Simple-SR-master/exps/BebyGAN/…/…/dataset/init.py”, line 16, in
for file_name in dataset_filenames
File “/home/anaconda3/envs/Bebygan/lib/python3.6/importlib/init.py”, line 126, in import_module
return _bootstrap._gcd_import(name[level:], package, level)
ModuleNotFoundError: No module named 'datasets.mix_dataset

Please help me solve this error @ptrblck . I have used the same packages mentioned in the code’s requirement file.

dataset_filenames apparently contains mix_dataset which importlib.import_module cannot load so make sure to pass only valid names to:

dataset_filenames = [
os.path.splitext(os.path.basename(v))[0] for v in scandir(dataset_root)
    if v.endswith('_dataset.py')
 ]
dataset_modules = [
       importlib.import_module(f'datasets.{file_name}')
       for file_name in dataset_filenames
]
1 Like

1
The file name is same as mix_dataset.py as shown in the image.

I guess import datasets.mix_dataset doesn’t work so check the module name, folder name etc. and make sure you can import the file.

1 Like