GAN spectral normalization Error

vision
1

I got the below error when I am trying to add spectral normalization to my GAN model, how can I solve it?

AttributeError: 'MyConvo2d' object has no attribute 'weight'

My residual block code

class ResidualBlock(nn.Module):
    def __init__(self, input_dim, output_dim, kernel_size, resample=None, hw=DIM):
        super(ResidualBlock, self).__init__()

        self.input_dim = input_dim
        self.output_dim = output_dim
        self.kernel_size = kernel_size
        self.resample = resample
        self.bn1 = None
        self.bn2 = None
        self.relu1 = nn.ReLU()
        self.relu2 = nn.ReLU()
        
        if resample == 'down':
            self.bn1 = nn.LayerNorm([input_dim, hw, hw])
            self.bn2 = nn.LayerNorm([input_dim, hw, hw])
        elif resample == 'up':
            self.bn1 = nn.BatchNorm2d(input_dim)
            self.bn2 = nn.BatchNorm2d(output_dim)
        elif resample == None:
            # TODO: ????
            self.bn1 = nn.BatchNorm2d(output_dim)
            self.bn2 = nn.LayerNorm([input_dim, hw, hw])
        else:
            raise Exception('invalid resample value')

        if resample == 'down':
            self.conv_shortcut = MeanPoolConv(input_dim, output_dim, kernel_size=1, he_init=False)
            self.conv_1 = MyConvo2d(input_dim, input_dim, kernel_size=kernel_size, bias=False)
            self.conv_2 = ConvMeanPool(input_dim, output_dim, kernel_size=kernel_size)
            
        elif resample == 'up':
            self.conv_shortcut = UpSampleConv(input_dim, output_dim, kernel_size=1, he_init=False)
            self.conv_1 = UpSampleConv(input_dim, output_dim, kernel_size=kernel_size, bias=False)
            self.conv_2 = MyConvo2d(output_dim, output_dim, kernel_size=kernel_size)
            
        elif resample == None:
            self.conv_shortcut = MyConvo2d(input_dim, output_dim, kernel_size=1, he_init=False)
            self.conv_1 = MyConvo2d(input_dim, input_dim, kernel_size=kernel_size, bias=False)
            self.conv_2 = MyConvo2d(input_dim, output_dim, kernel_size=kernel_size)
        else:
            raise Exception('invalid resample value')

    def forward(self, input):
        if self.input_dim == self.output_dim and self.resample == None:
            shortcut = input
        else:
            shortcut = self.conv_shortcut(input)

        output = input
        output = self.bn1(output)
        output = self.relu1(output)
        output = self.conv_1(output)
        output = self.bn2(output)
        output = self.relu2(output)
        output = self.conv_2(output)

        return shortcut + output

After adding spectral normalization

class ResidualBlock(nn.Module):
    def __init__(self, input_dim, output_dim, kernel_size, resample=None, hw=DIM):
        super(ResidualBlock, self).__init__()

        self.input_dim = input_dim
        self.output_dim = output_dim
        self.kernel_size = kernel_size
        self.resample = resample
     #   self.bn1 = None
      #  self.bn2 = None
        
        
        layer1, layer2, layer3 = [], [], []
         
        if resample == 'down':            
            self.conv_shortcut = SpectralNorm(MeanPoolConv(input_dim, output_dim, kernel_size=1, he_init=False))
            
            layer2.append(nn.LayerNorm([input_dim, hw, hw]))
            layer2.append(nn.ReLU())
            layer2.append(SpectralNorm(MyConvo2d(input_dim, input_dim, kernel_size=kernel_size, bias=False)))
            self.l2 = nn.Sequential(*layer2)
            
            layer3.append(nn.LayerNorm([input_dim, hw, hw]))
            layer3.append(nn.ReLU())
            layer3.append(SpectralNorm(ConvMeanPool(input_dim, output_dim, kernel_size=kernel_size)))
            self.l3 = nn.Sequential(*layer3)


        elif resample == 'up':
            self.conv_shortcut = SpectralNorm(UpSampleConv(input_dim, output_dim, kernel_size=1, he_init=False))
            
            layer2.append(nn.BatchNorm2d(input_dim))
            layer2.append(nn.ReLU())           
            layer2.append(SpectralNorm(UpSampleConv(input_dim, output_dim, kernel_size=kernel_size, bias=False)))
            self.l2 = nn.Sequential(*layer2)
            
            
            layer3.append(nn.BatchNorm2d(output_dim))
            layer3.append(nn.ReLU())
            layer3.append(SpectralNorm(MyConvo2d(output_dim, output_dim, kernel_size=kernel_size)))
            self.l3 = nn.Sequential(*layer3)

            
        elif resample == None:
            self.conv_shortcut = SpectralNorm(MyConvo2d(input_dim, output_dim, kernel_size=1, he_init=False))
            
            layer2.append(nn.BatchNorm2d(output_dim))
            layer2.append(nn.ReLU())
            layer2.append(SpectralNorm(MyConvo2d(input_dim, input_dim, kernel_size=kernel_size, bias=False)))
            self.l2 = nn.Sequential(*layer2)

            layer3.append(n.LayerNorm([input_dim, hw, hw]))
            layer3.append(nn.ReLU())
            layer3.append(SpectralNorm(MyConvo2d(input_dim, output_dim, kernel_size=kernel_size)))
            self.l3 = nn.Sequential(*layer3)

        else:
            raise Exception('invalid resample value')



    def forward(self, input):
        if self.input_dim == self.output_dim and self.resample == None:
            shortcut = input
        else:
            shortcut = self.conv_shortcut(input)

        output = input
        output = self.l2(output)
        output = self.l3(output)

        return shortcut + output
2

Hi,
I think your MyConvo2d is an nn.Sequential object. SpectralNorm works on weight of nn.Conv2d. So, if you call SpectralNorm on a sequence i.e.,MyConvo2d , it can not find the weight and hence the error is occurring.

3

For example, you can test with this code snippet.

import torch
from torch import nn
from torch.nn.utils import spectral_norm

x = torch.rand(size=(1, 3, 4, 4))
My_Conv = nn.Conv2d(in_channels=3, out_channels=16, kernel_size=3, stride=1, padding=1)
print(My_Conv(x).shape) # works

My_Conv_with_spectral_norm = spectral_norm(nn.Conv2d(in_channels=3, out_channels=16, kernel_size=3, stride=1, padding=1))
print(My_Conv_with_spectral_norm(x).shape) # works

My_Conv_sequential = nn.Sequential(
    nn.Conv2d(in_channels=3, out_channels=16, kernel_size=3, stride=1, padding=1)
)
print(My_Conv_sequential(x).shape) # works

My_Conv_sequential_spectral_norm = spectral_norm(
    nn.Sequential(
        nn.Conv2d(in_channels=3, out_channels=16, kernel_size=3, stride=1, padding=1)
    )
)
print(My_Conv_sequential_spectral_norm(x).shape) # will not work
4

I changed the mosel by the below code but I still got same problem

class ResidualBlock(nn.Module):
    def __init__(self, input_dim, output_dim, kernel_size, resample=None, hw=DIM):
        super(ResidualBlock, self).__init__()

        self.input_dim = input_dim
        self.output_dim = output_dim
        self.kernel_size = kernel_size
        self.resample = resample
        self.bn1 = None
        self.bn2 = None
        self.relu1 = nn.ReLU()
        self.relu2 = nn.ReLU()
        
        if resample == 'down':
            self.bn1 = nn.LayerNorm([input_dim, hw, hw])
            self.bn2 = nn.LayerNorm([input_dim, hw, hw])
        elif resample == 'up':
            self.bn1 = nn.BatchNorm2d(input_dim)
            self.bn2 = nn.BatchNorm2d(output_dim)
        elif resample == None:
            # TODO: ????
            self.bn1 = nn.BatchNorm2d(output_dim)
            self.bn2 = nn.LayerNorm([input_dim, hw, hw])
        else:
            raise Exception('invalid resample value')

        if resample == 'down':
            self.conv_shortcut = SpectralNorm(MeanPoolConv(input_dim, output_dim, kernel_size=1, he_init=False))
            self.conv_1 = SpectralNorm(MyConvo2d(input_dim, input_dim, kernel_size=kernel_size, bias=False))
            self.conv_2 = SpectralNorm(ConvMeanPool(input_dim, output_dim, kernel_size=kernel_size))
            
        elif resample == 'up':
            self.conv_shortcut = SpectralNorm(UpSampleConv(input_dim, output_dim, kernel_size=1, he_init=False))
            self.conv_1 = SpectralNorm(UpSampleConv(input_dim, output_dim, kernel_size=kernel_size, bias=False))
            self.conv_2 = SpectralNorm(MyConvo2d(output_dim, output_dim, kernel_size=kernel_size))
            
        elif resample == None:
            self.conv_shortcut = SpectralNorm(MyConvo2d(input_dim, output_dim, kernel_size=1, he_init=False))
            self.conv_1 = SpectralNorm(MyConvo2d(input_dim, input_dim, kernel_size=kernel_size, bias=False))
            self.conv_2 = SpectralNorm(MyConvo2d(input_dim, output_dim, kernel_size=kernel_size))
        else:
            raise Exception('invalid resample value')

    def forward(self, input):
        if self.input_dim == self.output_dim and self.resample == None:
            shortcut = input
        else:
            shortcut = self.conv_shortcut(input)

        output = input
        output = self.bn1(output)
        output = self.relu1(output)
        output = self.conv_1(output)
        output = self.bn2(output)
        output = self.relu2(output)
        output = self.conv_2(output)

        return shortcut + output
5

this is the weight initialization function

        
#weight initialization
def initialize_weights(m):
            if isinstance(m, nn.Conv2d):
                nn.init.kaiming_normal_(m.weight)
                if m.bias is not None:
                    nn.init.constant_(m.bias, 0)
            elif isinstance(m, nn.BatchNorm2d) or isinstance(m, nn.LayerNorm):
                nn.init.constant_(m.weight, 1)
                nn.init.constant_(m.bias, 0)
            elif isinstance(m, nn.Linear):
                nn.init.xavier_uniform_(m.weight)
                nn.init.constant_(m.bias, 0)
6

The easiest solution would be to put spectral_norm around the nn.Conv2d calls inside your Myconvo2d class. For example,

import torch
from torch import nn
from torch.nn.utils import spectral_norm


class MyConvo2d(nn.Module):
    def __init__(self, in_channels, out_channels):
        super().__init__()
        self.my_conv = nn.Sequential(
            spectral_norm(nn.Conv2d(in_channels=in_channels, out_channels=in_channels, kernel_size=3, stride=1, padding=1)),
            nn.ReLU(),
            spectral_norm(nn.Conv2d(in_channels=in_channels, out_channels=out_channels, kernel_size=3, stride=1, padding=1)),
            nn.ReLU()
        )

    def forward(self, x):
        return self.my_conv(x)


x = torch.rand(size=(1, 1, 4, 4))
Net = MyConvo2d(1, 1)
print(Net(x))
7

All the other functions are related to MyConvo2d, so should I define SpectralNorm just for MyConvo2d

# A custom layer: apply a padding and a two-dimensional convolutional layer to all inputs passes from
class MyConvo2d(nn.Module):
    def __init__(self, input_dim, output_dim, kernel_size, he_init=True,  stride=1, bias=True):
        
        super(MyConvo2d, self).__init__()
        self.he_init = he_init
        self.padding = int((kernel_size - 1)/2)
        self.conv = SpectralNorm(nn.Conv2d(input_dim, output_dim, kernel_size,stride=1, padding=self.padding, bias=bias))

    def forward(self, input):
        output = self.conv(input)
        return output
    

class ConvMeanPool(nn.Module):
    def __init__(self, input_dim, output_dim, kernel_size, he_init=True):
        super(ConvMeanPool, self).__init__()
        self.he_init = he_init
        self.conv = MyConvo2d(input_dim, output_dim, kernel_size, he_init=self.he_init)

    def forward(self, input):
        output = self.conv(input)
        output = (output[:, :, ::2, ::2] + output[:, :, 1::2, ::2] + output[:, :, ::2, 1::2] + output[:, :, 1::2, 1::2]) / 4
        return output


class MeanPoolConv(nn.Module):
    def __init__(self, input_dim, output_dim, kernel_size, he_init=True):
        super(MeanPoolConv, self).__init__()
        self.he_init = he_init
        self.conv = MyConvo2d(input_dim, output_dim,
                              kernel_size, he_init=self.he_init)

    def forward(self, input):
        output = input
        output = (output[:, :, ::2, ::2] + output[:, :, 1::2, ::2] +
                  output[:, :, ::2, 1::2] + output[:, :, 1::2, 1::2]) / 4
        output = self.conv(output)
        return output


class DepthToSpace(nn.Module):
    def __init__(self, block_size):
        super(DepthToSpace, self).__init__()
        self.block_size = block_size
        self.block_size_sq = block_size*block_size

    def forward(self, input):
        output = input.permute(0, 2, 3, 1)
        (batch_size, input_height, input_width, input_depth) = output.size()
        output_depth = int(input_depth / self.block_size_sq)
        output_width = int(input_width * self.block_size)
        output_height = int(input_height * self.block_size)
        t_1 = output.reshape(batch_size, input_height,
                             input_width, self.block_size_sq, output_depth)
        spl = t_1.split(self.block_size, 3)
        stacks = [t_t.reshape(batch_size, input_height,
                              output_width, output_depth) for t_t in spl]
        output = torch.stack(stacks, 0).transpose(0, 1).permute(0, 2, 1, 3, 4).reshape(
            batch_size, output_height, output_width, output_depth)
        output = output.permute(0, 3, 1, 2)
        return output


class UpSampleConv(nn.Module):
    def __init__(self, input_dim, output_dim, kernel_size, he_init=True, bias=True):
        super(UpSampleConv, self).__init__()
        self.he_init = he_init
        self.conv = MyConvo2d(input_dim, output_dim,kernel_size, he_init=self.he_init, bias=bias)
        self.depth_to_space = DepthToSpace(2)

    def forward(self, input):
        output = input
        output = torch.cat((output, output, output, output), 1)
        output = self.depth_to_space(output)
        output = self.conv(output)
        return output
    
    
class ReLULayer(nn.Module):
    def __init__(self, n_in, n_out):
        super(ReLULayer, self).__init__()
        self.n_in = n_in
        self.n_out = n_out
        self.linear = nn.Linear(n_in, n_out)
        self.relu = nn.ReLU()

    def forward(self, input):
        output = self.linear(input)
        output = self.relu(output)
        return output
8

Yes. I can see all the modules are related to MyConvo2d, so adding SpectralNorm in MyConvo2d is enough.
Also, you can put SpectralNorm around nn.Linear also, if you want to normalize the weights of the linear layers too.

9

I got an error while adding SpectralNorm to nn.Linear

AttributeError: 'Linear' object has no attribute 'weight'
10

Can you please try this?

import torch
from torch import nn
from torch.nn.utils import spectral_norm


class ReLULayer(nn.Module):
    def __init__(self, n_in, n_out):
        super(ReLULayer, self).__init__()
        self.n_in = n_in
        self.n_out = n_out
        self.linear = spectral_norm(nn.Linear(n_in, n_out)) #spectral norm in Linear Layer
        self.relu = nn.ReLU()

    def forward(self, input):
        output = self.linear(input)
        output = self.relu(output)
        return output

Also, check this link documentation.

11

Adding SpectralNorm in Linear Layer of ReLULayer works, but when adding it to Linear Layer of the Generator, I got the error.

    def __init__(self, dim=DIM, output_dim=OUTPUT_DIM):
        super(Generator, self).__init__()

        self.dim = dim
        self.ln1 = SpectralNorm(nn.Linear(128, 4*4*8*self.dim))
12

Try this,

from torch.nn.utils import spectral_norm

def __init__(self, dim=DIM, output_dim=OUTPUT_DIM):
        super(Generator, self).__init__()

        self.dim = dim
        self.ln1 = spectral_norm(nn.Linear(128, 4*4*8*self.dim))

I don’t know how SpectralNorm is defined in your code. Can you check it?

13

bellow is the code of my SpectralNorm, I got the same error with from torch.nn.utils import spectral_norm

def l2normalize(v, eps=1e-12):
    return v / (v.norm() + eps)

class SpectralNorm(nn.Module):
    def __init__(self, module, name='weight', power_iterations=1):
        super(SpectralNorm, self).__init__()
        self.module = module
        self.name = name
        self.power_iterations = power_iterations
        if not self._made_params():
            self._make_params()

    def _update_u_v(self):
        u = getattr(self.module, self.name + "_u")
        v = getattr(self.module, self.name + "_v")
        w = getattr(self.module, self.name + "_bar")

        height = w.data.shape[0]
        for _ in range(self.power_iterations):
            v.data = l2normalize(torch.mv(torch.t(w.view(height,-1).data), u.data))
            u.data = l2normalize(torch.mv(w.view(height,-1).data, v.data))

        # sigma = torch.dot(u.data, torch.mv(w.view(height,-1).data, v.data))
        sigma = u.dot(w.view(height, -1).mv(v))
        setattr(self.module, self.name, w / sigma.expand_as(w))

    def _made_params(self):
        try:
            u = getattr(self.module, self.name + "_u")
            v = getattr(self.module, self.name + "_v")
            w = getattr(self.module, self.name + "_bar")
            return True
        except AttributeError:
            return False


    def _make_params(self):
        w = getattr(self.module, self.name)

        height = w.data.shape[0]
        width = w.view(height, -1).data.shape[1]

        u = Parameter(w.data.new(height).normal_(0, 1), requires_grad=False)
        v = Parameter(w.data.new(width).normal_(0, 1), requires_grad=False)
        u.data = l2normalize(u.data)
        v.data = l2normalize(v.data)
        w_bar = Parameter(w.data)

        del self.module._parameters[self.name]

        self.module.register_parameter(self.name + "_u", u)
        self.module.register_parameter(self.name + "_v", v)
        self.module.register_parameter(self.name + "_bar", w_bar)


    def forward(self, *args):
        self._update_u_v()
        return self.module.forward(*args)
14

Can you please post the error?

15

This is the error

---------------------------------------------------------------------------
AttributeError                            Traceback (most recent call last)
<ipython-input-67-4c1227c7febc> in <module>
      1 G =Generator().to(device)
----> 2 G.apply(initialize_weights)
      3 
      4 summary(G)

~\anaconda3\envs\st\lib\site-packages\torch\nn\modules\module.py in apply(self, fn)
    471         """
    472         for module in self.children():
--> 473             module.apply(fn)
    474         fn(self)
    475         return self

~\anaconda3\envs\st\lib\site-packages\torch\nn\modules\module.py in apply(self, fn)
    471         """
    472         for module in self.children():
--> 473             module.apply(fn)
    474         fn(self)
    475         return self

~\anaconda3\envs\st\lib\site-packages\torch\nn\modules\module.py in apply(self, fn)
    472         for module in self.children():
    473             module.apply(fn)
--> 474         fn(self)
    475         return self
    476 

<ipython-input-57-77b5d20a0bc3> in initialize_weights(m)
     13 def initialize_weights(m):
     14     if isinstance(m, nn.Linear):
---> 15         nn.init.xavier_uniform_(m.weight.data)
     16         m.bias.data.fill_(0)
     17 

~\anaconda3\envs\st\lib\site-packages\torch\nn\modules\module.py in __getattr__(self, name)
    946                 return modules[name]
    947         raise AttributeError("'{}' object has no attribute '{}'".format(
--> 948             type(self).__name__, name))
    949 
    950     def __setattr__(self, name: str, value: Union[Tensor, 'Module']) -> None:

AttributeError: 'Linear' object has no attribute 'weight'
16

Hi, I couldn’t reproduce your error. Here’s my minimum working code.

import torch
from torch import nn
from torch.nn.utils import spectral_norm


# weight initialization
def initialize_weights(m):
    if isinstance(m, nn.Conv2d):
        nn.init.kaiming_normal_(m.weight)
        if m.bias is not None:
            nn.init.constant_(m.bias, 0)
    elif isinstance(m, nn.BatchNorm2d) or isinstance(m, nn.LayerNorm):
        nn.init.constant_(m.weight, 1)
        nn.init.constant_(m.bias, 0)
    elif isinstance(m, nn.Linear):
        nn.init.xavier_uniform_(m.weight)
        nn.init.constant_(m.bias, 0)
        print('No problem with initializing weights for linear....')


# My Conv with spectral norm
class MyConvo2d(nn.Module):
    def __init__(self, in_channels, out_channels):
        super().__init__()
        self.my_conv_with_sn = nn.Sequential(
            spectral_norm(nn.Conv2d(in_channels=in_channels, out_channels=out_channels, kernel_size=3, stride=1,
                                    padding=1)),
            nn.ReLU(),
        )

    def forward(self, x):
        return self.my_conv_with_sn(x)


# Generator with linear + SN
class Generator(nn.Module):
    def __init__(self):
        super().__init__()

        self.ln1 = spectral_norm(nn.Linear(16, 1))
        self.my_conv_with_sn = MyConvo2d(1, 1)

    def forward(self, x):
        x_size = x.shape
        x = self.my_conv_with_sn(x)
        x = x.view(x_size[0], -1)
        x = self.ln1(x)
        return x


x = torch.rand(size=(1, 1, 4, 4))
Net = Generator()
Net.apply(initialize_weights)  # applying weight initialization
out = Net(x)
print("result from generator: ", out)
print("Linear weight with spectral norm: ", Net.ln1.weight)  # with spectral norm
print("Linear weight - Original: ", Net.ln1.weight_orig)  # no spectral norm

Output:

No problem with initializing weights for linear....
result from generator:  tensor([[-0.2687]], grad_fn=<AddmmBackward>)
Linear weight with spectral norm:  tensor([[-0.1292, -0.3753,  0.3092, -0.1900,  0.3692,  0.0772, -0.3573, -0.1007,
          0.0139, -0.1800,  0.2348, -0.0897, -0.2418,  0.3746,  0.3307, -0.1635]],
       grad_fn=<DivBackward0>)
Linear weight - Original:  Parameter containing:
tensor([[-0.2020, -0.5868,  0.4834, -0.2970,  0.5773,  0.1206, -0.5586, -0.1575,
          0.0217, -0.2814,  0.3671, -0.1402, -0.3781,  0.5857,  0.5170, -0.2556]],
       requires_grad=True)

Can you please check your code?

17

There is no problem when I am using 'spectral_norm from ‘from torch.nn.utils import spectral_norm’ but using my defined SpectralNorm class, I got the errors above.