Understanding the effect of various normalization techniques

Hi, I am new to machine learning and i use a unet as a toy model in order to deepen my knowledge. I woud like to perform various normalization techniques in order to understand practically their effects on my grayscale dataset.

  1. Please could you tell me how many data normalization techniques for image processing are available on Pytorch ? Do we have any practical documentation on them for beginners ?

  2. How can i implement them on my simple unet architecture ? I heard about Batch normalisation, layer normalisation, spectral normalisation, instance normalisation and weight normalisation. Where in my code is it more interesting to apply them please ?

Thank you

# architecture +++++++++++++++++++++++++++++++++++++
def double_conv(in_channels, out_channels):                  
    return nn.Sequential(
        nn.Conv2d(in_channels, out_channels, 5, padding=2),
        nn.ReLU(inplace=True),
        nn.Conv2d(out_channels, out_channels, 5, padding=2),
        nn.ReLU(inplace=True)
    )   

class UNet(nn.Module):                  

    def __init__(self, n_class):        
        super().__init__()
        dim_0,dim_1,dim_2,dim_3,dim_4 = 1,64,128,256,512
        self.db_conv_1 = double_conv(dim_0, dim_1)
        self.db_conv_2 = double_conv(dim_1, dim_2)
        self.db_conv_3 = double_conv(dim_2, dim_3)
        self.db_conv_4 = double_conv(dim_3, dim_4)   
        self.maxpool = nn.MaxPool2d(2)                        
        self.upsample = nn.Upsample(scale_factor=2, mode='nearest')       
        self.db_transpose_conv_3 = double_conv(dim_3 + dim_4, dim_3)
        self.db_transpose_conv_2 = double_conv(dim_2 + dim_3, dim_2)
        self.db_transpose_conv_1 = double_conv(dim_2 + dim_1, dim_1)           
        self.conv_last = nn.Conv2d(dim_1, n_class, 1)      
        
        
    def forward(self, x):                             #
        conv1 = self.db_conv_1(x)
        x = self.maxpool(conv1)

        conv2 = self.db_conv_2(x)
        x = self.maxpool(conv2)
     
        conv3 = self.db_conv_3(x)
        x = self.maxpool(conv3)   

        conv4 = self.db_conv_4(x)                                                                              
        x = self.upsample(x) 

        x = torch.cat([x, conv3], dim=1)          # skips connections betweent layers 
        x = self.db_transpose_conv_3(x)
        x = self.upsample(x)  

        x = torch.cat([x, conv2], dim=1)     
        x = self.db_transpose_conv_2(x)
        x = self.upsample(x)  

        x = torch.cat([x, conv1], dim=1)   
        x = self.db_transpose_conv_1(x)
    
        x = self.conv_last(x)

        result = torch.sigmoid(x)
        
        return result

# training function ++++++++++++++++++++++++++++++++++
def train_model(model, optimizer, scheduler, num_epochs=25):
    best_model_wts = copy.deepcopy(model.state_dict())
    best_loss = 1e10
    train_loss,val_loss = [],[]
    for epoch in range(num_epochs):
        print('Epoch {}/{}'.format(epoch+1,num_epochs))
        print('-' * 10)
        since = time.time()
        # Each epoch has a training and validation phase
        for phase in ['train','val']:
            if phase == 'train':
                # print value of learning rate
                for param_group in optimizer.param_groups:
                    print("LR", param_group['lr'])   
                # Set model to training mode           
                model.train()  
            else:
                # Set model to evaluate mode
                model.eval()   

            metrics, epoch_samples = defaultdict(float), 0
            for inputs, labels in dataloaders[phase]:
                inputs, labels = (inputs.type(torch.FloatTensor)).to(device), labels.to(device)         
                # zero the parameter gradients before each weight update and avoid loss accumulation
                optimizer.zero_grad()
                # forward: make a prediction
                with torch.set_grad_enabled(phase == 'train'):
                    # compute outputs
                    outputs = model(inputs)
                    # track losses for each phase
                    if phase=='train':
                      loss = calc_loss(outputs, labels, metrics)
                    else:
                      loss = calc_loss(outputs, labels, metrics)

                    # backward: backpropagation to compute loss
                    # then optimize: tune hyper parameters of the model
                    # and call scheduler for l_r (only during training phase)
                    if phase == 'train':
                        loss.backward(); optimizer.step(); scheduler.step()
                # count the number of samples used in this training phase
                epoch_samples += inputs.size(0)

            # print and save loss for model actualisation 
            print_metrics(metrics, epoch_samples, phase)
            epoch_loss = metrics['loss'] / epoch_samples
            if phase == 'train':
              train_loss.append(metrics['loss'] / epoch_samples)
            else:
              val_loss.append(metrics['loss'] / epoch_samples)
            # deep copy the model
            if phase == 'val' and epoch_loss < best_loss:
                print("saving best model")
                best_loss = epoch_loss
                best_model_wts = copy.deepcopy(model.state_dict())

        time_elapsed = time.time() - since
        print('{:.0f}m {:.0f}s'.format(time_elapsed // 60, time_elapsed % 60))
        r_t = (time_elapsed % 60)*num_epochs-(time_elapsed % 60)*(epoch+1)
        r_t = r_t/60
        print("Remaining time: %i minute(s)" %r_t )

    print('Best val loss: {:4f}'.format(best_loss))

    # load best model weights
    model.load_state_dict(best_model_wts)
    return model, np.array(train_loss), np.array(val_loss)

Hi, welcome! Take a look at torch.nn — PyTorch 1.11.0 documentation and click into any of the layers to understand more. Clicking on [SOURCE] near the API definition will show you the implementation in the codebase.