How can I replace the Conv2d, and MaxPool2d in a VGG model?

1

I want to change the Conv2d layers into SpatialConvolution layers, and the MaxPool2d layers into SpatialMaxPooling layers:

Conv2d --> SpatialConvolution

MaxPool2d --> SpatialMaxPooling

If I load the model like this:

import torch.legacy.nn as lnn
import torch.nn as nn

cnn = models.vgg19(pretrained=True)

Then how do I perform the above changes?

I am not sure if I need anything like the cfg dictionary from this script: https://github.com/pytorch/vision/blob/master/torchvision/models/vgg.py

This is PyTorch’s legacy SpatialConvolution code: https://github.com/pytorch/pytorch/blob/master/torch/legacy/nn/SpatialConvolution.py

The non legacy Conv code is found here: https://github.com/pytorch/pytorch/blob/master/torch/nn/modules/conv.py

This is PyTorch’s legacy SpatialMaxPooling code: https://github.com/pytorch/pytorch/blob/master/torch/legacy/nn/SpatialMaxPooling.py

The non legacy pooling code is found here: https://github.com/pytorch/pytorch/blob/master/torch/nn/modules/pooling.py

2

When I try the code below:

import torch
import torch.legacy.nn as lnn
import torch.nn as nn

import torchvision
import torchvision.models as models


cnn = models.vgg19(pretrained=True)#.features
print(cnn)

for idx, module in cnn.features._modules.items():
	if module.__class__.__name__ == 'ReLU':
            print("ReLU!")
        if module.__class__.__name__ == 'Conv2d':
            print("Conv!")
            cnn.features._modules[idx] = lnn.SpatialConvolution
        if module.__class__.__name__ == 'MaxPool2d':
            print("Pool!")
            cnn.features._modules[idx] = lnn.SpatialMaxPooling

print(cnn)

I get this error when I run print(cnn) after changing the layers:

TypeError: unbound method __repr__() must be called with SpatialConvolution instance as first argument (got nothing instead)
3

It’s probably easier to write a new class that mirrors the code: https://github.com/pytorch/vision/blob/master/torchvision/models/vgg.py

4

I tried to that earlier:

def make_layers_legacy(cfg, batch_norm=False):
    layers = []
    in_channels = 3
    for v in cfg:
        if v == 'M':
            layers += [lnn.SpatialMaxPooling(kW=0, kH=0, dW=None, dH=None, padW=0, padH=0)]
        else:
            conv = lnn.SpatialConvolution(nInputPlane, nOutputPlane, kW, kH, dW=1, dH=1, padW=0, padH=None)
            if batch_norm:
                layers += [conv, lnn.SpatialBatchNormalization(v), lnn.ReLU(inplace=True)]
            else:
                layers += [conv, lnn.ReLU(inplace=True)]
            in_channels = v
    return lnn.Sequential(*layers)

cfg = {
    'A': [64, 'M', 128, 'M', 256, 256, 'M', 512, 512, 'M', 512, 512, 'M'],
    'B': [64, 64, 'M', 128, 128, 'M', 256, 256, 'M', 512, 512, 'M', 512, 512, 'M'],
    'D': [64, 64, 'M', 128, 128, 'M', 256, 256, 256, 'M', 512, 512, 512, 'M', 512, 512, 512, 'M'],
    'E': [64, 64, 'M', 128, 128, 'M', 256, 256, 256, 256, 'M', 512, 512, 512, 512, 'M', 512, 512, 512, 512, 'M'],
}

model = VGG(make_layers_legacy(cfg['E']), **kwargs)

But I can’t seem to figure out how to setup the code from that script in a way that replaces the appropriate layers:

NameError: global name 'nInputPlane' is not defined

I tried looking at Torch to PyTorch conversion scripts, but they didn’t seem to have the relevant information on how I can properly setup the code in make_layers_legacy.

I also have no idea where to start on changing:

    def _initialize_weights(self):
        for m in self.modules():
            if isinstance(m, nn.Conv2d):
                #n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels
                m.weight.data.normal_(0, math.sqrt(2. / n))
                if m.bias is not None:
                    m.bias.data.zero_()
            elif isinstance(m, nn.BatchNorm2d):
                m.weight.data.fill_(1)
                m.bias.data.zero_()
            elif isinstance(m, nn.Linear):
                m.weight.data.normal_(0, 0.01)
                m.bias.data.zero_()

It would nice is there was a torchvision.legacy.models like there is for nn and optim.

5

For the first method:

            print("Conv!")
            print(cnn.features._modules[idx])
            cnn.features._modules[idx] = lnn.SpatialConvolution
            print(cnn.features._modules[idx])

The cnn.features._modules[idx] value is being changed to <class 'torch.legacy.nn.SpatialConvolution.SpatialConvolution'>, which I suspect is not correct:

Conv!
Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
<class 'torch.legacy.nn.SpatialConvolution.SpatialConvolution'>

So I should instead be taking those variables from Conv2d and putting them into SpatialConvolution()? Thought I’m not sure where to start for that?

6

I think I figured it out somewhat:

Using the pattern generated by Torch7’s loadcaffe (Lua), it looks like only 2 values need to be changed, and they are predictable.

import torch
import torch.legacy.nn as lnn
import torch.nn as nn

import torchvision
import torchvision.models as models

from torch.legacy.nn import SpatialConvolution
from torch.legacy.nn import SpatialMaxPooling

cnn = models.vgg19(pretrained=True)#.features
cnn.cuda()
print(cnn)

#VGG19 Pattern
num_channel1 = [3, 64, 64, 128, 128, 256, 256, 256, 256, 512, 512, 512, 512, 512, 512, 512]
num_channel2 = [64, 64, 128, 128, 256, 256, 256, 256, 512, 512, 512, 512, 512, 512, 512, 512]

conv_num = [1]
for idx, module in cnn.features._modules.items():
	#if module.__class__.__name__ == 'ReLU':
            #print("ReLU!")
        if module.__class__.__name__ == 'Conv2d':
            #print("Conv!")
            print(cnn.features._modules[idx])
            c = conv_num[0]
            channels_1 = num_channel1[c]
            channels_2 = num_channel2[c]
            print(channels_1, " ", channels_2)
            cnn.features._modules[idx] = SpatialConvolution(channels_1, channels_2, 3, 3, 1, 1, 1, 1)
            print(cnn.features._modules[idx])
            conv_num[0] += 1
        if module.__class__.__name__ == 'MaxPool2d':
            #print("Pool!")
            cnn.features._modules[idx] = SpatialMaxPooling(2, 2, 2, 2, 0, 0)
print(cnn)

Now there is this error:

cnn.features._modules[idx] = SpatialConvolution(channels_1, channels_2, 3, 3, 1, 1, 1, 1, 1)
TypeError: __init__() takes at most 9 arguments (10 given)

Looking at the loadcaffe values, it appears that 10 are required, and PyTorch’s SpatialConvolution is meant to be like Lua’s SpatialConvolution:

table.insert(model, {'conv2_2', cudnn.SpatialConvolution(128, 128, 3, 3, 1, 1, 1, 1, 1)})

Omitting one of the 1s from (channels_1, channels_2, 3, 3, 1, 1, 1, 1, 1) makes things run smoothly until another error occurs.

Adding another 512 to num_channel1 and num_channel2, fixes that error.

7

This seems to work:

import torch
import torch.nn as nn
from torch.legacy.nn import SpatialConvolution
from torch.legacy.nn import SpatialMaxPooling
import torchvision
import torchvision.models as models

num_channel1 = [3, 64, 64, 128, 128, 256, 256, 256, 256, 512, 512, 512, 512, 512, 512, 512, 512]
num_channel2 = [64, 64, 128, 128, 256, 256, 256, 256, 512, 512, 512, 512, 512, 512, 512, 512, 512]

print("Changing Layers")
conv_num = [1]
for idx, module in cnn.features._modules.items():
        if module.__class__.__name__ == 'Conv2d':
            c = conv_num[0]
            channels_1 = num_channel1[c]
            channels_2 = num_channel2[c]
            cnn.features._modules[idx] = SpatialConvolution(channels_1, channels_2, 3, 3, 1, 1, 1, 1)
            conv_num[0] += 1
        if module.__class__.__name__ == 'MaxPool2d':
            cnn.features._modules[idx] = SpatialMaxPooling(2, 2, 2, 2, 0, 0)
print("Layers Changed")
cnn = cnn.cuda

However when I try to use copy.deepcopy, I a new error:

    cnn = copy.deepcopy(cnn)
  File "/usr/lib/python2.7/copy.py", line 190, in deepcopy
    y = _reconstruct(x, rv, 1, memo)
  File "/usr/lib/python2.7/copy.py", line 334, in _reconstruct
    state = deepcopy(state, memo)
  File "/usr/lib/python2.7/copy.py", line 163, in deepcopy
    y = copier(x, memo)
  File "/usr/lib/python2.7/copy.py", line 257, in _deepcopy_dict
    y[deepcopy(key, memo)] = deepcopy(value, memo)
  File "/usr/lib/python2.7/copy.py", line 190, in deepcopy
    y = _reconstruct(x, rv, 1, memo)
  File "/usr/lib/python2.7/copy.py", line 328, in _reconstruct
    args = deepcopy(args, memo)
  File "/usr/lib/python2.7/copy.py", line 163, in deepcopy
    y = copier(x, memo)
  File "/usr/lib/python2.7/copy.py", line 237, in _deepcopy_tuple
    y.append(deepcopy(a, memo))
  File "/usr/lib/python2.7/copy.py", line 163, in deepcopy
    y = copier(x, memo)
  File "/usr/lib/python2.7/copy.py", line 230, in _deepcopy_list
    y.append(deepcopy(a, memo))
  File "/usr/lib/python2.7/copy.py", line 163, in deepcopy
    y = copier(x, memo)
  File "/usr/lib/python2.7/copy.py", line 230, in _deepcopy_list
    y.append(deepcopy(a, memo))
  File "/usr/lib/python2.7/copy.py", line 190, in deepcopy
    y = _reconstruct(x, rv, 1, memo)
  File "/usr/lib/python2.7/copy.py", line 334, in _reconstruct
    state = deepcopy(state, memo)
  File "/usr/lib/python2.7/copy.py", line 163, in deepcopy
    y = copier(x, memo)
  File "/usr/lib/python2.7/copy.py", line 257, in _deepcopy_dict
    y[deepcopy(key, memo)] = deepcopy(value, memo)
  File "/usr/lib/python2.7/copy.py", line 190, in deepcopy
    y = _reconstruct(x, rv, 1, memo)
  File "/usr/lib/python2.7/copy.py", line 328, in _reconstruct
    args = deepcopy(args, memo)
  File "/usr/lib/python2.7/copy.py", line 163, in deepcopy
    y = copier(x, memo)
  File "/usr/lib/python2.7/copy.py", line 237, in _deepcopy_tuple
    y.append(deepcopy(a, memo))
  File "/usr/lib/python2.7/copy.py", line 163, in deepcopy
    y = copier(x, memo)
  File "/usr/lib/python2.7/copy.py", line 230, in _deepcopy_list
    y.append(deepcopy(a, memo))
  File "/usr/lib/python2.7/copy.py", line 163, in deepcopy
    y = copier(x, memo)
  File "/usr/lib/python2.7/copy.py", line 230, in _deepcopy_list
    y.append(deepcopy(a, memo))
  File "/usr/lib/python2.7/copy.py", line 190, in deepcopy
    y = _reconstruct(x, rv, 1, memo)
  File "/usr/lib/python2.7/copy.py", line 334, in _reconstruct
    state = deepcopy(state, memo)
  File "/usr/lib/python2.7/copy.py", line 163, in deepcopy
    y = copier(x, memo)
  File "/usr/lib/python2.7/copy.py", line 257, in _deepcopy_dict
    y[deepcopy(key, memo)] = deepcopy(value, memo)
  File "/usr/lib/python2.7/copy.py", line 172, in deepcopy
    copier = getattr(x, "__deepcopy__", None)
  File "/usr/local/lib/python2.7/dist-packages/torch/_thnn/utils.py", line 22, in __getattr__
    raise NotImplementedError
NotImplementedError