Hi, I am new to machine learning and i use a unet as a toy model in order to deepen my knowledge. I woud like to perform various normalization techniques in order to understand practically their effects on my grayscale dataset.
-
Please could you tell me how many data normalization techniques for image processing are available on Pytorch ? Do we have any practical documentation on them for beginners ?
-
How can i implement them on my simple unet architecture ? I heard about Batch normalisation, layer normalisation, spectral normalisation, instance normalisation and weight normalisation. Where in my code is it more interesting to apply them please ?
Thank you
# architecture +++++++++++++++++++++++++++++++++++++
def double_conv(in_channels, out_channels):
return nn.Sequential(
nn.Conv2d(in_channels, out_channels, 5, padding=2),
nn.ReLU(inplace=True),
nn.Conv2d(out_channels, out_channels, 5, padding=2),
nn.ReLU(inplace=True)
)
class UNet(nn.Module):
def __init__(self, n_class):
super().__init__()
dim_0,dim_1,dim_2,dim_3,dim_4 = 1,64,128,256,512
self.db_conv_1 = double_conv(dim_0, dim_1)
self.db_conv_2 = double_conv(dim_1, dim_2)
self.db_conv_3 = double_conv(dim_2, dim_3)
self.db_conv_4 = double_conv(dim_3, dim_4)
self.maxpool = nn.MaxPool2d(2)
self.upsample = nn.Upsample(scale_factor=2, mode='nearest')
self.db_transpose_conv_3 = double_conv(dim_3 + dim_4, dim_3)
self.db_transpose_conv_2 = double_conv(dim_2 + dim_3, dim_2)
self.db_transpose_conv_1 = double_conv(dim_2 + dim_1, dim_1)
self.conv_last = nn.Conv2d(dim_1, n_class, 1)
def forward(self, x): #
conv1 = self.db_conv_1(x)
x = self.maxpool(conv1)
conv2 = self.db_conv_2(x)
x = self.maxpool(conv2)
conv3 = self.db_conv_3(x)
x = self.maxpool(conv3)
conv4 = self.db_conv_4(x)
x = self.upsample(x)
x = torch.cat([x, conv3], dim=1) # skips connections betweent layers
x = self.db_transpose_conv_3(x)
x = self.upsample(x)
x = torch.cat([x, conv2], dim=1)
x = self.db_transpose_conv_2(x)
x = self.upsample(x)
x = torch.cat([x, conv1], dim=1)
x = self.db_transpose_conv_1(x)
x = self.conv_last(x)
result = torch.sigmoid(x)
return result
# training function ++++++++++++++++++++++++++++++++++
def train_model(model, optimizer, scheduler, num_epochs=25):
best_model_wts = copy.deepcopy(model.state_dict())
best_loss = 1e10
train_loss,val_loss = [],[]
for epoch in range(num_epochs):
print('Epoch {}/{}'.format(epoch+1,num_epochs))
print('-' * 10)
since = time.time()
# Each epoch has a training and validation phase
for phase in ['train','val']:
if phase == 'train':
# print value of learning rate
for param_group in optimizer.param_groups:
print("LR", param_group['lr'])
# Set model to training mode
model.train()
else:
# Set model to evaluate mode
model.eval()
metrics, epoch_samples = defaultdict(float), 0
for inputs, labels in dataloaders[phase]:
inputs, labels = (inputs.type(torch.FloatTensor)).to(device), labels.to(device)
# zero the parameter gradients before each weight update and avoid loss accumulation
optimizer.zero_grad()
# forward: make a prediction
with torch.set_grad_enabled(phase == 'train'):
# compute outputs
outputs = model(inputs)
# track losses for each phase
if phase=='train':
loss = calc_loss(outputs, labels, metrics)
else:
loss = calc_loss(outputs, labels, metrics)
# backward: backpropagation to compute loss
# then optimize: tune hyper parameters of the model
# and call scheduler for l_r (only during training phase)
if phase == 'train':
loss.backward(); optimizer.step(); scheduler.step()
# count the number of samples used in this training phase
epoch_samples += inputs.size(0)
# print and save loss for model actualisation
print_metrics(metrics, epoch_samples, phase)
epoch_loss = metrics['loss'] / epoch_samples
if phase == 'train':
train_loss.append(metrics['loss'] / epoch_samples)
else:
val_loss.append(metrics['loss'] / epoch_samples)
# deep copy the model
if phase == 'val' and epoch_loss < best_loss:
print("saving best model")
best_loss = epoch_loss
best_model_wts = copy.deepcopy(model.state_dict())
time_elapsed = time.time() - since
print('{:.0f}m {:.0f}s'.format(time_elapsed // 60, time_elapsed % 60))
r_t = (time_elapsed % 60)*num_epochs-(time_elapsed % 60)*(epoch+1)
r_t = r_t/60
print("Remaining time: %i minute(s)" %r_t )
print('Best val loss: {:4f}'.format(best_loss))
# load best model weights
model.load_state_dict(best_model_wts)
return model, np.array(train_loss), np.array(val_loss)