I’m dealing with MRI data, and I converted these files into numpy files and saved them.
my data consisted of input : [3600, 512, 512] (N,H,W) -1.8G and mask : [3600,8,512,512] (N,classes,H,W)-28.125G.
I’m using U-Net for segmentation and GPU is ‘NVIDIA GeForce RTX 2080Ti’ with 18G ram.
Data has been loaded as follows:
class trainDataset(torch.utils.data.Dataset):
def __init__(self, data, target, transform=None):
self.data = data.astype(np.float32)
self.data = normalize(data)
self.target = target.astype(np.float32)
self.transform = transform
def __getitem__(self, index):
x = self.data[index]
y = self.target[index]
if self.transform:
x = self.transform(x)
return x, y
def __len__(self):
return len(self.data)
numpy_data = np.load(image_path+'MRtrain.npy')
numpy_target = np.load(mask_path+'RStrain.npy')
traindataset = trainDataset(numpy_data, numpy_target ,transform = transform)
trainloader = torch.utils.data.DataLoader(traindataset, batch_size = batch_size, shuffle=True, num_workers=0, pin_memory=False)
and train loop as follow :
def fit(epoch,model,data_loader,phase='train',volatile=False):
if phase == 'train':
exp_lr_scheduler.step()
model.train()
if phase == 'valid':
model.eval()
running_loss = 0.0
for batch_idx , (data,target) in enumerate(data_loader):
inputs,target = data.cpu(),target.cpu()
if is_cuda:
inputs,target = data.cuda(),target.cuda()
inputs , target = Variable(inputs),Variable(target)
if phase == 'train':
optimizer.zero_grad()
output = model(inputs)
pred = torch.sigmoid(output)
loss = dice(pred,target)
running_loss += loss.data.item()
if phase == 'train':
loss.backward()
optimizer.step()
loss = running_loss/len(data_loader.dataset)
print('{} Dice_Loss: {:.4f}'.format(
phase, loss))
return loss
Training to 515x515 size will cause cuda memory error.
I think it’s inefficient to deal with numpy file, but is there a good way?
