Parameters initialised by nn.Parameter not present in the model.parameters()

1

Hi,
I have defined the weight parameters as follows but still these trainable parameters are not listed in the model.parameters().

 class Module(nn.Module):
    def __init__(input_dim, output_dim)
    #some variables

    def build(self):
       self.wt_dict = nn.ParameterDict()
       # self.wt_dict = {}
       for i in range(3):
          self.wt_dict["weight_%s"%i] = nn.Parameter(torch.FloatTensor(self.input_dim,self.output_dim))            
          for j in range(14):
             self.wt_dict["weight_left_%s%s"%(i,j)] = nn.Parameter(torch.FloatTensor(self.output_dim,1))
             self.wt_dict["weight_right_%s%s"%(i,j)] = nn.Parameter(torch.FloatTensor(self.output_dim,1))

I have tried it using the ParameterDict() and also with the standard dict, but the parameters are not reflected in the model parameters.
Even used the self.register_parameter() to register the parameters but with no success.

Any help would be appreciated.
Thanks!

1 Like
2

Your code works fine after adding the self.build() call and the missing super().__init__():

class Module(nn.Module):
   def __init__(self):
       super().__init__()
       self.build()

   def build(self):
      self.wt_dict = nn.ParameterDict()
      for i in range(3):
         self.wt_dict["weight_%s"%i] = nn.Parameter(torch.randn(1, 1)) 
         for j in range(14):
            self.wt_dict["weight_left_%s%s"%(i,j)] = nn.Parameter(torch.randn(1, 1))
            self.wt_dict["weight_right_%s%s"%(i,j)] = nn.Parameter(torch.randn(1, 1))
    
model = Module()
print(dict(model.named_parameters()))
> {'wt_dict.weight_0': Parameter containing:
  tensor([[-0.6057]], requires_grad=True), 'wt_dict.weight_left_00': Parameter containing:
  tensor([[1.0553]], requires_grad=True), 'wt_dict.weight_right_00': Parameter containing:
  ...
1 Like
3

However, when I call the “Module” from other custom class, these parameters do not appear in the list.

So if I want to call the function “Module2” from some other custom class

  self.conv = Module1(input_dim, output_dim, support1, support2)
  

   class Module1(nn.Module):
       def __init__(self, input_dim, output_dim, support1, support2, iter):
           super(Module1, self).__init__()
              self.input_dim = input_dim
              self.output_dim = output_dim
              self.support1 = support1
              self.support2 = support2
              self.iter = iter
              self.build()

     def build(self):
        self.dict = {}
        for i in range(self.iter):
             self.dict["no_%s"%i] = Module2(input_dim=self.input_dim,
                                                    output_dim=self.output_dim,
                                                    support1=self.support1,
                                                    support2=self.support2,
                                                    )
class Module2(nn.Module):
    def __init__(self, self, input_dim, output_dim, support1, support2):
       super(Module2,self).__init__()
       self.build()

  def build(self):
     self.wt_dict = nn.ParameterDict()
     for i in range(3):
        self.wt_dict["weight_%s"%i] = nn.Parameter(torch.randn(output_dim, input_dim)) 
        for j in range(14):
           self.wt_dict["weight_left_%s%s"%(i,j)] = nn.Parameter(torch.randn(output_dim, 1))
           self.wt_dict["weight_right_%s%s"%(i,j)] = nn.Parameter(torch.randn(output_dim, 1))

Are there any things I need to keep in mind to have the parameters from the custom layers to be present in the model parameters when finally we call the model.??

Thanks!

4

In Module1 you are assigning instances of Module2 to a plain Python dict, which won’t register the parameters, so you would have to use nn.ModuleDict instead.

5

Yes, Thank you it worked for the plain dict part.

So for assigning instances of Module2 in module1, and assigning Module1 instance from some other class (apart from a dictionary), nn.ModuleList should be used.

Eg: -

self.mod1 = nn.ModuleList(Module2(input_dim=self.input_dim,
                                                output_dim=self.output_dim,
                                                support1=self.support1,
                                                support2=self.support2,))

self.mod2 = nn.ModuleList(Module1(input_dim, output_dim, support1, support2))

Thanks.

6

Yes, plain Python “containers” such as list, dict etc. won’t (recursively) register the parameters and buffers, so you should use the PyTorch equivalents instead such as nn.ParameterList, nn.ModuleList, nn.ModuleDict etc.

1 Like
7

Thank you so much !!!

8

If we are initialising a class instance of Module 1 in Module 2’s forward layer, then Module1’s parameters along with Layer 1 and Layer2’s don’t get registered.

class Layer1(nn.Module):
  def __init__(self, a, n, dim):
    super(Layer1, self).__init__()
    self.a = a
    self.n = n
    self.dim = dim
    self.w_input = nn.Parameter(torch.FloatTensor(self.dim, self.n,1,1))
    if a:
      self.wt = nn.Parameter(torch.FloatTensor(2 * self.dim, n, 1))

class Layer2(nn.Module):
  def __init__(self, n, dim):
    super(Layer1, self).__init__()
    self.n = n
    self.dim = dim
    self.g = nn.Parameter(torch.ones((1,dim,1,n)))


class Module1(nn.Module):
  def __init__(self, a, n, dim):
    super(Module1, self).__init__()
    self.a = a
    self.n = n
    self.dim = dim
    self.layer1 = Layer1(a, dim, dim)
    self.layer2 = Layer2(n, dim)

def forward(self, input):
    x = self.layer1(input)
    x = self.layer2(x)
    return x


 class Module2(nn.Module):
    def __init__():
       super(Module2, self).__init__()
       self.a = a
       self.n=n
       self.dim = dim
       self.layer3 = Layer3(a,n)
       
    def forward(self, input):
       x = self.layer3(input)
       inp_size = list(x.size())
       _, _, _, ndim = inp_size[0],inp_size[1],inp_size[2],inp_size[3]
       k = 2*(ndim-self.dim)
       if(k>1):
          layer_output = Module1(self.a, k, self.dim)
          output = layer_output(x)
       return output

The value of k (in Module2 forward layer) can only be determined from the values obtained in this forward layer, so we cannot define Module 1 instance in the init layer of Module 2.

So how do we register the parameters of the instance initialised in the forward layer?

Thanks!

9

You would have to assign the module to self in the forward.
Also, make sure to register this module only once. Otherwise you would be overwriting the module and no training will be performed.
Since you are now defining new models after the __init__, you should also take care of passing these new parameters to the optimizer after all modules were created in the forward method.

10

That was very helpful, thank you!