Applying the train_transform to train_loader not to the whole dataset for a custom dataset

1

How should I apply the transform_train to my train_loader or train_dataset here?


rgb_mean = (97.13, 97.15, 97.15)
rgb_std = (28.74, 28.79, 28.8)


transform_train = transforms.Compose([
    transforms.RandomHorizontalFlip(),
    transforms.ToTensor(),
    transforms.Normalize(rgb_mean, rgb_std),
])
class MothLandmarksDataset(Dataset):
    """Face Landmarks dataset."""

    def __init__(self, csv_file, root_dir, transform=None):
        """
        Args:
            csv_file (string): Path to the csv file with annotations.
            root_dir (string): Directory with all the images.
            transform (callable, optional): Optional transform to be applied
                on a sample.
        """
        self.landmarks_frame = pd.read_csv(csv_file)
        self.root_dir = root_dir
        self.transform = transform

    def __len__(self):
        return len(self.landmarks_frame)

    def __getitem__(self, idx):
        if torch.is_tensor(idx):
            idx = idx.tolist()

        img_name = os.path.join(self.root_dir, self.landmarks_frame.iloc[idx, 0])
        image = io.imread(img_name)
        landmarks = self.landmarks_frame.iloc[idx, 1:]
        landmarks = np.array([landmarks])
        landmarks = landmarks.astype('float').reshape(-1, 2)
        sample = {'image': image, 'landmarks': landmarks}

        if self.transform:
            sample = self.transform(sample)

        return sample
dataset = MothLandmarksDataset('moth_gt.csv', '.', transform=transform_train)

# Device configuration
device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')
seed = 42
np.random.seed(seed)
torch.manual_seed(seed)

# split the dataset into validation and test sets
len_valid_set = int(0.1*len(dataset))
len_train_set = len(dataset) - len_valid_set



train_dataset , valid_dataset,  = torch.utils.data.random_split(dataset , [len_train_set, len_valid_set])





# shuffle and batch the datasets
train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=8, shuffle=True, num_workers=4)
test_loader = torch.utils.data.DataLoader(valid_dataset, batch_size=8, shuffle=True, num_workers=4)

The main problem with above code is I cannot segregate the transform_train and transform_test. How can I differentiate them in MothLandmarksDataset? Mainly I think the problem happens because MothLandmarksDataset is instantiated on the whole dataset not train_dataset or test_dataset. Any help is really appreciated.

2

One simple idea would be to split the data beforehand and have train.csv and test.csv.
Then you can write:

train_dataset =  MothLandmarksDataset('train.csv', '.', transform=transform_train)
test_dataset =  MothLandmarksDataset('test.csv', '.', transform=transform_test)
1 Like
3

right but I was hoping I don’t have to do manually and could use pytorch to do that automatically. is there an automatic way for doing what you mean via torch?

4

After this line:

train_dataset , valid_dataset,  = torch.utils.data.random_split(dataset , [len_train_set, len_valid_set])

You could modify the transforms, something like:

train_dataset.transforms = train_transform
valid_dataset.transforms = test_transform
1 Like