How to change forward class? (Skip Net)

hix · August 9, 2021, 3:49am

This is Skip Net nn.Sequential function

from numpy.testing.private.utils import assert

import torch

import torch.nn as nn

from .common import *

def Skip(

    num_input_channels=1, num_output_channels=3,

    num_channels_down=[16, 32, 64, 128, 128], num_channels_up=[16, 32, 64, 128, 128], num_channels_skip=[4, 4, 4, 4, 4], 

    filter_size_down=3, filter_size_up=3, filter_skip_size=1,

    need_sigmoid=True, need_bias=True, 

    pad='zero', upsample_mode='nearest', downsample_mode='stride', act_fun='LeakyReLU', 

    need1x1_up=True):

"""Assembles encoder-decoder with skip connections.

Arguments:

    act_fun: Either string 'LeakyReLU|Swish|ELU|none' or module (e.g. nn.ReLU)

    pad (string): zero|reflection (default: 'zero')

    upsample_mode (string): 'nearest|bilinear' (default: 'nearest')

    downsample_mode (string): 'stride|avg|max|lanczos2' (default: 'stride')

"""

assert len(num_channels_down) == len(num_channels_up) == len(num_channels_skip)

n_scales = len(num_channels_down) 

if not (isinstance(upsample_mode, list) or isinstance(upsample_mode, tuple)) :

    upsample_mode   = [upsample_mode]*n_scales

if not (isinstance(downsample_mode, list)or isinstance(downsample_mode, tuple)):

    downsample_mode   = [downsample_mode]*n_scales



if not (isinstance(filter_size_down, list) or isinstance(filter_size_down, tuple)) :

    filter_size_down   = [filter_size_down]*n_scales

if not (isinstance(filter_size_up, list) or isinstance(filter_size_up, tuple)) :

    filter_size_up   = [filter_size_up]*n_scales

last_scale = n_scales - 1 

cur_depth = None

model = nn.Sequential()

model_tmp = model

input_depth = num_input_channels

for i in range(len(num_channels_down)):

    deeper = nn.Sequential()

    skip = nn.Sequential()

    if num_channels_skip[i] != 0:

        model_tmp.add(Concat(1, skip, deeper))

    else:

        model_tmp.add(deeper)

    

    model_tmp.add(bn(num_channels_skip[i] + (num_channels_up[i + 1] if i < last_scale else num_channels_down[i])))

 

    if num_channels_skip[i] != 0:

        skip.add(conv(input_depth, num_channels_skip[i], filter_skip_size, bias=need_bias, pad=pad))

        skip.add(bn(num_channels_skip[i]))

        skip.add(act(act_fun))

        

    # skip.add(Concat(2, GenNoise(nums_noise[i]), skip_part))

    deeper.add(conv(input_depth, num_channels_down[i], filter_size_down[i], 2, bias=need_bias, pad=pad, downsample_mode=downsample_mode[i]))

    deeper.add(bn(num_channels_down[i]))

    deeper.add(act(act_fun))

    deeper.add(conv(num_channels_down[i], num_channels_down[i], filter_size_down[i], bias=need_bias, pad=pad))

    deeper.add(bn(num_channels_down[i]))

    deeper.add(act(act_fun))

    deeper_main = nn.Sequential()

    if i == len(num_channels_down) - 1:

        # The deepest

        k = num_channels_down[i]

    else:

        deeper.add(deeper_main)

        k = num_channels_up[i + 1]

    deeper.add(nn.Upsample(scale_factor=2, mode=upsample_mode[i]))

    model_tmp.add(conv(num_channels_skip[i] + k, num_channels_up[i], filter_size_up[i], 1, bias=need_bias, pad=pad))

    model_tmp.add(bn(num_channels_up[i]))

    model_tmp.add(act(act_fun))

    if need1x1_up:

        model_tmp.add(conv(num_channels_up[i], num_channels_up[i], 1, bias=need_bias, pad=pad))

        model_tmp.add(bn(num_channels_up[i]))

        model_tmp.add(act(act_fun))

    input_depth = num_channels_down[i]

    model_tmp = deeper_main

model.add(conv(num_channels_up[0], num_output_channels, 1, bias=need_bias, pad=pad))

if need_sigmoid:

    model.add(nn.Sigmoid())

return model

I want change forward class How to convert forward class? Or How to Debugging nn.Sequential function

ooodragon · August 9, 2021, 5:16am

잘 이해를 못했는데 다시 한번 적어주실 수 있나요?
So, from what I understand, you want to change some modules within the nn.sequential module?
if so, I think you can change module with following
model._modules[wanted_change_idx] = <ANY MODULE YOU WANT (but it needs to be in correct shape>

hix · August 9, 2021, 5:32am

Dear ooodragon

I want change Forward class

Example:

def forward(self, t):
    t = F.relu(self.conv1(t))
    t = F.max_pool2d(t, kernel_size=2, stride=2)
    t = F.relu(self.conv2(t))
    t = F.max_pool2d(t, kernel_size=2, stride=2)
    t = t.flatten(start_dim=1)
    t = F.relu(self.fc1(t))
    t = F.relu(self.fc2(t))
    t = self.out(t)
    return t

I want make a Forward class of Skip Net Because Skip Net can’t debug input tensor

ooodragon · August 9, 2021, 5:49am

Still not quite sure what you mean…
So you want to “debug” tensors?
you can add hook functions within your modules and ask them to print it

or if you need forward function, I’m not aware if you can override one from nn.Sequential, you might need to do some research

hix · August 9, 2021, 5:58am

Yes. Exactly

I want just debug tensors so i’m add some hook

Example:

from numpy.testing._private.utils import assert_

 import torch

 import torch.nn as nn

 from .common import *

def Skip(

    num_input_channels=1, num_output_channels=3,

    num_channels_down=[16, 32, 64, 128, 128], num_channels_up=

[16, 32, 64, 128, 128], num_channels_skip=[4, 4, 4, 4, 4],

    filter_size_down=3, filter_size_up=3, filter_skip_size=1,

    need_sigmoid=True, need_bias=True, 

    pad='zero', upsample_mode='nearest', downsample_mode='stride', act_fun='LeakyReLU', 

    need1x1_up=True):

"""Assembles encoder-decoder with skip connections.

Arguments:

    act_fun: Either string 'LeakyReLU|Swish|ELU|none' or module (e.g. nn.ReLU)

    pad (string): zero|reflection (default: 'zero')

    upsample_mode (string): 'nearest|bilinear' (default: 'nearest')

    downsample_mode (string): 'stride|avg|max|lanczos2' (default: 'stride')

"""

assert len(num_channels_down) == len(num_channels_up) == len(num_channels_skip)

n_scales = len(num_channels_down) 

if not (isinstance(upsample_mode, list) or isinstance(upsample_mode, tuple)) :

    upsample_mode   = [upsample_mode]*n_scales

if not (isinstance(downsample_mode, list)or isinstance(downsample_mode, tuple)):

    downsample_mode   = [downsample_mode]*n_scales



if not (isinstance(filter_size_down, list) or isinstance(filter_size_down, tuple)) :

    filter_size_down   = [filter_size_down]*n_scales

if not (isinstance(filter_size_up, list) or isinstance(filter_size_up, tuple)) :

    filter_size_up   = [filter_size_up]*n_scales

last_scale = n_scales - 1 

cur_depth = None

model = nn.Sequential()

model_tmp = model



def print_sizes(model_tmp, input_tensor):

    output = input_tensor

    for m in model_tmp.children():

        output = m(output)

        print(m, output.shape)

    return output

input_tensor = torch.rand(1,num_input_channels,512,512)



input_depth = num_input_channels

for i in range(len(num_channels_down)):

    deeper = nn.Sequential()

    skip = nn.Sequential()

    if num_channels_skip[i] != 0:

        model_tmp.add(Concat(1, skip, deeper))

    else:

        model_tmp.add(deeper)

    print_sizes(model_tmp, input_tensor)

    model_tmp.add(bn(num_channels_skip[i] + (num_channels_up[i + 1] if i < last_scale else num_channels_down[i])))

    if num_channels_skip[i] != 0:

        skip.add(conv(input_depth, num_channels_skip[i], filter_skip_size, bias=need_bias, pad=pad))

        skip.add(bn(num_channels_skip[i]))

        skip.add(act(act_fun))

        

    # skip.add(Concat(2, GenNoise(nums_noise[i]), skip_part))

    deeper.add(conv(input_depth, num_channels_down[i], filter_size_down[i], 2, bias=need_bias, pad=pad, downsample_mode=downsample_mode[i]))

    deeper.add(bn(num_channels_down[i]))

    deeper.add(act(act_fun))

    deeper.add(conv(num_channels_down[i], num_channels_down[i], filter_size_down[i], bias=need_bias, pad=pad))

    deeper.add(bn(num_channels_down[i]))

    deeper.add(act(act_fun))

    deeper_main = nn.Sequential()

    if i == len(num_channels_down) - 1:

        # The deepest

        k = num_channels_down[i]

    else:

        deeper.add(deeper_main)

        k = num_channels_up[i + 1]

    deeper.add(nn.Upsample(scale_factor=2, mode=upsample_mode[i]))

    model_tmp.add(conv(num_channels_skip[i] + k, num_channels_up[i], filter_size_up[i], 1, bias=need_bias, pad=pad))

    model_tmp.add(bn(num_channels_up[i]))

    model_tmp.add(act(act_fun))

    print_sizes(model_tmp, input_tensor)

    if need1x1_up:

        model_tmp.add(conv(num_channels_up[i], num_channels_up[i], 1, bias=need_bias, pad=pad))

        model_tmp.add(bn(num_channels_up[i]))

        model_tmp.add(act(act_fun))

    input_depth = num_channels_down[i]

    model_tmp = deeper_main

model.add(conv(num_channels_up[0], num_output_channels, 1, bias=need_bias, pad=pad))

if need_sigmoid:

    model.add(nn.Sigmoid())

return model

i’m add printsizes but i have some error…

ooodragon · August 9, 2021, 6:23am

plz show me your “error”
I’m sorry I can’t help you with code one by one

However, I don’t see you applied any hook functions, plz try adding them

hix · August 9, 2021, 6:33am

Sorry. This is My error How can i debug this code…

ooodragon · August 9, 2021, 6:40am

그냥 편하게 한국말로 할게요 ^^… 굳이 영어로 하라는 법은 없어서…
보니까 model.children 사용하셨는데, 이렇게 하시면 sequential 한 모델, (vgg같은 residual connection이나 다양한 operation이 없는 모델) 에서만 그렇게 가능하고 나머지에서는 불가능 해요

그래서 forward hook을 사용하셔야해요

여기에 친절하게 hook function 사용하는 방법, 어떻게 동작하는지까지 알려주니까 이 이상의 설명은 생략할게요

제가 코딩까지 해드릴정도로 시간이 많지는 않아서 ^^;;
에러를 보니 제가 말씀드린 부분이 맞을 것 같아요. for children 사용하지 마시구요 hook function 사용해보세요!

정말 다양한 코딩이 가능해지고, 더 고급 프로그래머가 되실꺼예요!

hix · August 9, 2021, 6:49am

좋은정보 감사합니다!

후크 펑션을 이용하여 디버깅을 시도해보겟습니다.

정말 많은도움이 되었습니다.