Image file is truncated when load data using multiprocessing

1

Dear all,

When I use dataloader to load data, when settiing num_workers = 0, nothing happened. But when setting num_workers = 1 or more, the error occures in the following:

RuntimeError: DataLoader worker (pid 5520) exited unexpectedly with exit code 1. Details are lost due to multiprocessing. Rerunning with num_workers=0 may give better error trace.

That is,

test_data = Data(IMGS_TEST, GT_TEST, V_TEST, mode="test")
data_loader = DataLoader(test_data, num_workers=10) # get error
data_loader = DataLoader(test_data, num_workers=0) # OK

And the code of Dataset is as follows:

class Data(data.Dataset):  # vis: vessels of images

    def __init__(self, imgs_root, gt_root, vessels_path, mode="train"):
        # self.imgs = [os.path.join(imgs_root, img) for img in os.listdir(imgs_root)]
        self.img_paths = sorted(glob.glob(os.path.join(imgs_root, '*.png')))
        self.gt_paths = [os.path.join(gt_root, img.split("/")[-1]) for img in self.img_paths]
        vis = load_pkl(vessels_path)

        split = int(0.7 * len(self.img_paths))

        for split2 in range(len(vis)):
            point = vis[split2]
            if point[2] >= split:
                break

        self.mode = mode

        if mode == "train":
            # self.img_paths = img_paths[:split]
            # self.gt_paths = gt_paths[:split]
            self.vis = vis[:split2]
        elif mode == "val":
            # self.img_paths = img_paths[split:]
            # self.gt_paths = gt_paths[split:]
            self.vis = vis[split2:]
        elif mode == "test":
            # self.img_paths = img_paths
            # self.gt_paths = gt_paths
            self.vis = vis
        else:
            print("the mode must be train / val / test.")
            exit()

        self.imgs = [T.Grayscale()(Image.open(img_path)) for img_path in self.img_paths]
        self.gts = [Image.open(gt_path) for gt_path in self.gt_paths]

        self.len = len(self.vis)

    def transform(self, image, center_point):

        pad = T.Pad(padding=PATCH_SIZE // 2)
        image = pad(image)

        i, j, h, w = int(center_point[0]) + PATCH_SIZE // 2, \
                     int(center_point[1]) + PATCH_SIZE // 2, \
                     PATCH_SIZE, \
                     PATCH_SIZE

        patch = F.crop(image, i, j, h, w)  # (40, 40)

        totensor = T.ToTensor()
        patch = totensor(patch)  # torch.Size([1, 40, 40])

        return patch

    def __getitem__(self, index):

        x, y, img_num, v_num = self.vis[index]

        if index == self.len - 1 or v_num != self.vis[index + 1][3]:
            deta_x, deta_y = 0, 0
        else:
            deta_x = self.vis[index + 1][0] - self.vis[index][0]
            deta_y = self.vis[index + 1][1] - self.vis[index][1]

        try:
            patch = self.transform(self.imgs[img_num], (x, y))
            segmentation = self.transform(self.gts[img_num], (x, y))
            label = t.Tensor([deta_x, deta_y])
        except Exception as e:
            print(e)
            print(self.mode)
            print(self.len)
            print("index and img_num", index, img_num)
            exit()

        return patch, segmentation, label

    def __len__(self):
        return len(self.vis)

Can anyone help me?

Regards,
Pt

1 Like