How to change parameter sharing module to resuse?

When using JIT tracing a python model, it throw out an error:

ValueError: TracedModules don't support parameter sharing between modules

But I do not understand what is the difference between paramaters sharing and reuse, doesn’t they are the same thing?

we define a Model into a class, and defined some modules when init, then call it one by one in forward, does this not resuse? Then what is the sharing parameters case?

In my model, How do I find those layers sharing parameters where everywhere I thought it was reuse module rather than parameters sharing.

From this architecture, does it can figure out whether it shared parameters or not?

unet8:  TUM(
  (layers): Sequential(
    (0): BasicConv(
      (conv): Conv2d(384, 256, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)
      (bn): BatchNorm2d(256, eps=1e-05, momentum=0.01, affine=True, track_running_stats=True)
      (relu): ReLU(inplace)
    )
    (1): BasicConv(
      (conv): Conv2d(256, 256, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)
      (bn): BatchNorm2d(256, eps=1e-05, momentum=0.01, affine=True, track_running_stats=True)
      (relu): ReLU(inplace)
    )
    (2): BasicConv(
      (conv): Conv2d(256, 256, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)
      (bn): BatchNorm2d(256, eps=1e-05, momentum=0.01, affine=True, track_running_stats=True)
      (relu): ReLU(inplace)
    )
    (3): BasicConv(
      (conv): Conv2d(256, 256, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)
      (bn): BatchNorm2d(256, eps=1e-05, momentum=0.01, affine=True, track_running_stats=True)
      (relu): ReLU(inplace)
    )
    (4): BasicConv(
      (conv): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), bias=False)
      (bn): BatchNorm2d(256, eps=1e-05, momentum=0.01, affine=True, track_running_stats=True)
      (relu): ReLU(inplace)
    )
  )
  (toplayer): Sequential(
    (0): BasicConv(
      (conv): Conv2d(256, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)
      (bn): BatchNorm2d(256, eps=1e-05, momentum=0.01, affine=True, track_running_stats=True)
      (relu): ReLU(inplace)
    )
  )
  (latlayer): Sequential(
    (0): BasicConv(
      (conv): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
      (bn): BatchNorm2d(256, eps=1e-05, momentum=0.01, affine=True, track_running_stats=True)
      (relu): ReLU(inplace)
    )
    (1): BasicConv(
      (conv): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
      (bn): BatchNorm2d(256, eps=1e-05, momentum=0.01, affine=True, track_running_stats=True)
      (relu): ReLU(inplace)
    )
    (2): BasicConv(
      (conv): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
      (bn): BatchNorm2d(256, eps=1e-05, momentum=0.01, affine=True, track_running_stats=True)
      (relu): ReLU(inplace)
    )
    (3): BasicConv(
      (conv): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
      (bn): BatchNorm2d(256, eps=1e-05, momentum=0.01, affine=True, track_running_stats=True)
      (relu): ReLU(inplace)
    )
    (4): BasicConv(
      (conv): Conv2d(384, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
      (bn): BatchNorm2d(256, eps=1e-05, momentum=0.01, affine=True, track_running_stats=True)
      (relu): ReLU(inplace)
    )
  )
  (smooth): Sequential(
    (0): BasicConv(
      (conv): Conv2d(256, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)
      (bn): BatchNorm2d(256, eps=1e-05, momentum=0.01, affine=True, track_running_stats=True)
      (relu): ReLU(inplace)
    )
    (1): BasicConv(
      (conv): Conv2d(256, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)
      (bn): BatchNorm2d(256, eps=1e-05, momentum=0.01, affine=True, track_running_stats=True)
      (relu): ReLU(inplace)
    )
    (2): BasicConv(
      (conv): Conv2d(256, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)
      (bn): BatchNorm2d(256, eps=1e-05, momentum=0.01, affine=True, track_running_stats=True)
      (relu): ReLU(inplace)
    )
    (3): BasicConv(
      (conv): Conv2d(256, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)
      (bn): BatchNorm2d(256, eps=1e-05, momentum=0.01, affine=True, track_running_stats=True)
      (relu): ReLU(inplace)
    )
    (4): BasicConv(
      (conv): Conv2d(256, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)
      (bn): BatchNorm2d(256, eps=1e-05, momentum=0.01, affine=True, track_running_stats=True)
      (relu): ReLU(inplace)
    )
  )
)

anybody could help?9999999

Could you post the code to initialize this model so that we could check for parameter sharing?

Thanks for you reply ptrblck!

I made an experiment on this:

from torch import nn
from alfred.utils.log import logger as logging
from alfred.dl.torch.common import device
import torch


class BasicConv(nn.Module):

    def __init__(self, in_planes, out_planes, kernel_size, stride=1, padding=0, dilation=1,
                 groups=1, relu=True, bn=True, bias=False):
        super(BasicConv, self).__init__()
        self.out_channels = out_planes
        self.conv = nn.Conv2d(in_planes, out_planes, kernel_size=kernel_size,
                              stride=stride, padding=padding, dilation=dilation, groups=groups, bias=bias)
        self.bn = nn.BatchNorm2d(out_planes, eps=1e-5, momentum=0.01, affine=True) if bn else None
        self.relu = nn.ReLU(inplace=True) if relu else None

    def forward(self, x):
        x = self.conv(x)
        if self.bn is not None:
            x = self.bn(x)
        if self.relu is not None:
            x = self.relu(x)
        return x


class FuckNet(nn.Module):

    def __init__(self):
        super(FuckNet, self).__init__()

        self.welcome_layer = BasicConv(3, 256, 3, 1, 1)
        self.fuck_layers = nn.Sequential()
        for i in range(5):
            self.fuck_layers.add_module('{}'.format(i), BasicConv(256, 256, 3, 1, 1))

    def forward(self, x):
        x = self.welcome_layer(x)
        return self.fuck_layers(x)


class FuckNet2(nn.Module):

    def __init__(self):
        super(FuckNet2, self).__init__()

        self.welcome_layer = BasicConv(3, 256, 3, 1, 1)
        self.block_1 = BasicConv(256, 256, 3, 1, 1)
        self.fuck_layers = nn.Sequential()
        for i in range(5):
            self.fuck_layers.add_module('{}'.format(i), self.block_1)

    def forward(self, x):
        x = self.welcome_layer(x)
        return self.fuck_layers(x)



if __name__ == '__main__':
    model1 = FuckNet()
    model2 = FuckNet2()

    model2.eval().to(device)

    # start to trace model
    example = torch.rand(1, 3, 512, 512).to(device)

    traced_script_module = torch.jit.trace(model2, example)
    traced_script_module.save('test.pt')

The Net2 can not be traced, error:

    self._modules[name] = TracedModule(submodule, id_set, optimize=optimize)
  File "/usr/local/lib/python3.6/dist-packages/torch/jit/__init__.py", line 1046, in init_then_register
    original_init(self, *args, **kwargs)
  File "/usr/local/lib/python3.6/dist-packages/torch/jit/__init__.py", line 1469, in __init__
    check_unique(param)
  File "/usr/local/lib/python3.6/dist-packages/torch/jit/__init__.py", line 1461, in check_unique
    raise ValueError("TracedModules don't support parameter sharing between modules")
ValueError: TracedModules don't support parameter sharing between modules

My question is:

  1. Why net2 can not be traced, that operation is common in building models;
  2. How do I solve the problem once my model all written in the Net2 format?

Answered here.