Weight Intialisation

1

Can someone please explain how the weights are being intialised here? How do I set the weights using manual seed or so, so that I can reproduce the same intialisation later?

import math
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.utils.model_zoo as model_zoo

class ResNet(nn.Module):
    def __init__(self, block, layers, num_classes, KaimingInit=False):

        self.inplanes = 16

        super(ResNet, self).__init__()

        self.conv1 = nn.Conv2d(1, 16, kernel_size=7, stride=2, padding=3, bias=False)
        self.bn1 = nn.BatchNorm2d(16)
        self.relu = nn.ReLU(inplace=True)
        self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)

        self.layer1 = self._make_layer(block, 16, layers[0])
        self.layer2 = self._make_layer(block, 32, layers[1], stride=2)
        self.layer3 = self._make_layer(block, 64, layers[2], stride=2)
        self.layer4 = self._make_layer(block, 128, layers[3], stride=2)
        self.avgpool = nn.AdaptiveAvgPool2d((1, 1))
        self.middle_fc1 = nn.Linear(16* block.expansion, 2)
        self.middle_fc2=nn.Linear(32* block.expansion, 2)
        self.middle_fc3=nn.Linear(64* block.expansion, 2)
        self.classifier = nn.Linear(128 * block.expansion, 2)

        if KaimingInit == True:
            print('Using Kaiming Initialization.')
            for m in self.modules():
                if isinstance(m, nn.Conv2d):
                    nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu')

        for m in self.modules():
            if isinstance(m, nn.BatchNorm2d):
                nn.init.constant_(m.weight, 1)
                nn.init.constant_(m.bias, 0)

    def _make_layer(self, block, planes, blocks, stride=1):
        downsample = None
        if stride != 1 or self.inplanes != planes * block.expansion:
            downsample = nn.Sequential(
                nn.Conv2d(self.inplanes, planes * block.expansion,
                          kernel_size=1, stride=stride, bias=False),
                nn.BatchNorm2d(planes * block.expansion),
            )

        layers = []
        layers.append(block(self.inplanes, planes, stride, downsample))
        self.inplanes = planes * block.expansion
        for i in range(1, blocks):
            layers.append(block(self.inplanes, planes))

        return nn.Sequential(*layers)

    def forward(self, x):
        #print(x.size())
        x = self.conv1(x)
        x = self.bn1(x)
        x = self.relu(x)
        x = self.maxpool(x)
        #print(x.size())

        x = self.layer1(x)
        # middle_output1 = torch.flatten(middle_output1, 1)
        middle_output1=self.avgpool(x).view(x.size()[0], -1)
        middle_output1 = self.middle_fc1(middle_output1)
        #print(x.size())
        x = self.layer2(x)
        middle_output2=self.avgpool(x).view(x.size()[0], -1)
        middle_output2 = self.middle_fc2(middle_output2)
        #print(x.size())
        x = self.layer3(x)
        middle_output3=self.avgpool(x).view(x.size()[0], -1)
        middle_output3 = self.middle_fc3(middle_output3)
        #print(x.size())
        x = self.layer4(x)
        #print(x.size())
        x = self.avgpool(x).view(x.size()[0], -1)
        #print(x.shape)
        out = self.classifier(x)
        #print(out.shape)
        return middle_output1,middle_output2,middle_output3,out,x

def eca_resnet18():
    model = ResNet(ECABasicBlock, [2, 2, 2, 2], 2)
    return model

def eca_resnet34():
    model = ResNet(ECABasicBlock, [3, 4, 6, 3], 2)
    return model
2

Hi, as per the nn.conv2D documentation, the weights are by default initialised from a distribution described in the Variables section.

To set the manual seed, you can do something like
torch.manual_seed(42) at the beginning so that the random number generation is the same every time the code is run (note that this is only for the same process, for running in different process/instance saving and loading weights should ensure same initialisation). After setting the random seed, you can have default or custom random weight initialisation and expect them to be the same in each run (for that process).

1 Like
3

Thanks for answering!
Alright so no matter the initialization technique Iā€™m using, I still need to set manual seed to reproduce results later, like in the below case. Have I gotten this correct?

class Classifier(nn.Module):
    def __init__(self):
        super(Discriminator, self).__init__()
        self.fc1 = nn.Linear(128 , 128 )
        self.fc1.weight.data.normal_(0, 0.01)
        self.fc1.bias.data.fill_(0.0)
        self.fc2 = nn.Linear(128 , 3)
        self.fc2.weight.data.normal_(0, 0.3)
        self.fc2.bias.data.fill_(0.0)
        self.relu = nn.ReLU()
        self.dropout = nn.Dropout(0.3)
        self.grl_layer = GradientReverse(alpha)

    def forward(self, feature):
        feature = self.grl_layer.forward(feature)
        feature = self.fc1(feature)
        feature = self.relu(feature)
        feature = self.dropout(feature)
        feature = self.fc2(feature)
        return feature
def clf():
    torch.manual_seed(0)
    model =Classifier()
    return model

Thus, trraining the model will influence and change the weights, but when I initialize the same model with the same seed again, the initial weights will be the same as before?

4

Hi, I believe the manual seed should be set at the global level, outside any class or function. Additionally, you can try these statements as well to ensure the same initialisation -

import random, os, torch, numpy
random.seed(seed)
os.environ["PYTHONHASHSEED"] = str(seed)
numpy.random.seed(seed)
torch.manual_seed(seed)

Also, you can check and print the model weights like conv_layer.weight to check if they are being initialised the same in each run.