Custom dataset and DataLoader

nlp
1

Hello,
I want to use a custom dataset with DataLoader. My dataset contains text and labels. Any suggestion?

2

Welcome Wan
Sure, you can try the below

import torch
from torch.utils.data import Dataset
class CustomDataloader(Dataset):
    def __init__(self, text, labels):
        self.text = text
        self.labels = labels

    def __len__(self):
        return len(self.text)

    def __getitem__(self, idx):
        text = self.text[idx]
        labels = self.labels[idx]
        sample = {"text": text, "labels": labels}
        return sample

all_text = []
all_labels = []

#Generate data
for i in range(4):
    all_text.append(torch.rand(2,1))
    if i%1 ==0:
        all_labels.append(1)
    else:
        all_labels.append(0)

DataClass = CustomDataloader(all_text,all_labels)
trainloader = torch.utils.data.DataLoader(DataClass,batch_size=2)
#get the data for training
for i ,data in enumerate(trainloader):
    text = data['text']
    labels = data['labels']
    print(text)
    print(labels)