Error in Custom a Layer

autograd
1

Hello, I’m new to PyTorch and I’m trying to custom a layer, but it didn’t work. (pytorch version 0.3.1)
Here comes the thing:

In main.py, I feed data into the net:

#net = models.CIFAR10_VGG(<Args>)
output = net(data)

In models.py, I try to form the net by nn.Sequential:

        self.comb = nn.Sequential(
            dfxp.Conv2d_q(  
                name='conv1_1',
                bits=self.bits,
                ksize=[3, 3, 3, 128],
                strides=[1, 1, 1, 1],
                padding=1,
                weight_decay=self.weight_decay
            ),
            # import from dynamic_fixed_point.py
            dfxp.ReLU_q(),
            ...
         )

In dynamic_fixed_point.py, I want to achieve a convolution layer by ‘conv2d’ function from nn.functional:

class Conv2d_func(torch.autograd.Function):
    @staticmethod
    def forward(ctx, input, weight, bias, strides, padding):
        ctx.save_for_backward(input, weight, bias)
        output = Function.conv2d(Variable(input), Variable(weight), Variable(bias),
                                 stride=strides[1], padding=1)
        return output
    
    # temporarily I want to use the default backward function

class Conv2d_q(nn.Module):
    def __init__(self, name, bits, ksize, strides, padding=0, weight_decay=0):
        super(Conv2d_q, self).__init__()
        
        self.bits = bits
        self.ksize = ksize
        self.strides = strides
        self.padding = padding
        self.weight_decay = weight_decay

        # order: (out_channels, in_channels, H, W)
        self.weight = nn.Parameter(torch.Tensor(ksize[3], ksize[0], ksize[1], ksize[2]))
        self.bias = nn.Parameter(torch.Tensor(ksize[3]))
        # initialization
        self.weight.data.uniform_(-0.1, 0.1)
        self.bias.data.uniform_(-0.1, 0.1)

    def forward(self, input):
        return Conv2d_func.apply(input, self.weight, self.bias,
                                 self.strides, self.padding)

And the error message is:

Traceback (most recent call last):
File “main.py”, line 241, in
training_losses = train(epoch)
File “main.py”, line 161, in train
loss = optimizer.step(closure)
File “/content/halp.py”, line 139, in step
self._compute_full_grad(closure)
File “/content/halp.py”, line 116, in _compute_full_grad
closure(data, target)
File “main.py”, line 154, in closure
output = net(data)
File “/usr/local/lib/python2.7/dist-packages/torch/nn/modules/module.py”, line 357, in call
result = self.forward(*input, **kwargs)
File “/usr/local/lib/python2.7/dist-packages/torch/nn/parallel/data_parallel.py”, line 71, in forward
return self.module(*inputs[0], **kwargs[0])
File “/usr/local/lib/python2.7/dist-packages/torch/nn/modules/module.py”, line 357, in call
result = self.forward(*input, **kwargs)
File “/content/models.py”, line 125, in forward
out = self.comb(input)
File “/usr/local/lib/python2.7/dist-packages/torch/nn/modules/module.py”, line 357, in call
result = self.forward(*input, **kwargs)
File “/usr/local/lib/python2.7/dist-packages/torch/nn/modules/container.py”, line 67, in forward
input = module(input)
File “/usr/local/lib/python2.7/dist-packages/torch/nn/modules/module.py”, line 357, in call
result = self.forward(*input, **kwargs)
File “/content/dynamic_fixed_point.py”, line 39, in forward
self.strides, self.padding)
RuntimeError: data must be a Tensor

Could anyone help me out?

2

Your data must be a Tensor, how do you get this data ?

3

From a Dataloader, actually. It should be correct:

    testset = torchvision.datasets.CIFAR10(root='./data', train=False, download=True, transform=transform_test)
    testloader = torch.utils.data.DataLoader(testset, batch_size=100, shuffle=False, num_workers=2)
    ...
    for batch_idx, (data, target) in enumerate(trainloader):
    ...
4

But in your transform_test, what do you do ?

5 
transform_test = transforms.Compose([
    transforms.ToTensor(),
    transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)),
])
6

The error comes from the optimizer step, how do you train your model ?

7

I don’t think so. In fact, I have tested it with another ‘net’ before, and it worked fine.
The problem lies in the definition of the net, I’m afraid.

8

I understand that but in the error message the execution goes through

loss = optimizer.step(closure)

And then

output = net(data)

9

I tracked and printed the type of data, and the results:

print(type(data))    # <class 'torch.autograd.variable.Variable'>
output = net(data)

In models.py, ‘net’->forward:

    def forward(self, input):
        print(type(input))    # <class 'torch.autograd.variable.Variable'>
        out = self.comb(input)

In dynamic_fixed_point.py:

class Conv2d_func(torch.autograd.Function):

    @staticmethod
    def forward(ctx, input, weight, bias, strides, padding):
        ctx.save_for_backward(input, weight, bias)
        output = Function.conv2d(Variable(input), Variable(weight), Variable(bias),
                                 stride=strides[1], padding=1)
        print(type(input))    # also print <class 'torch.autograd.variable.Variable'>
        return output

class Conv2d_q(nn.Module):
    def __init__(self, name, bits, ksize, strides, padding=0, weight_decay=0):
        super(Conv2d_q, self).__init__()
        
        self.bits = bits
        self.ksize = ksize
        self.strides = strides
        self.padding = padding
        self.weight_decay = weight_decay

        # order: (out_channels, in_channels, H, W)
        self.weight = nn.Parameter(torch.Tensor(ksize[3], ksize[0], ksize[1], ksize[2]))
        self.bias = nn.Parameter(torch.Tensor(ksize[3]))
        # initialization
        self.weight.data.uniform_(-0.1, 0.1)
        self.bias.data.uniform_(-0.1, 0.1)

    def forward(self, input):       
        out = Conv2d_func.apply(input, self.weight, self.bias, self.strides, self.padding)
        print(type(out))    # <= it stopped here
        return out
10

:cry: It seems that I have fixed the problem. I replaced the custom conv layer with nn.conv2d and saw what would happen. After that, I found that the customized ReLU layer worked fine. When I tracked the type of data flowing between these layers. I found something different. See:

class ReLU_func(torch.autograd.Function):
    @staticmethod
    def forward(ctx, input):
        ctx.save_for_backward(input)
        output = torch.max(torch.Tensor([0.0]).cuda(), input)
        print("ReLU")
        print(type(output))    
        # it printed <class 'torch.cuda.FloatTensor'>, instead of <class 'torch.autograd.variable.Variable'>
        return output

class ReLU_q(nn.Module):
    def __init__(self):
        super(ReLU_q, self).__init__()

    def forward(self, input):
        return ReLU_func.apply(input)

Then I finally understand what the error message means. The data returned by customized functions should be tensor(rather than Variable), only then can ‘apply’ work normally. Here is the revised code, as you can see:

class Conv2d_func(torch.autograd.Function):

    @staticmethod
    def forward(ctx, input, weight, bias, strides, padding):
        ctx.save_for_backward(input, weight, bias)
        output = Function.conv2d(Variable(input), Variable(weight), Variable(bias),
                                 stride=strides[1], padding=1)
        # up to now the type of 'output' is variable, but tensor needed
        output = output.data.cuda()
        # here 'output' is converted to torch.cuda.FloatTensor, as is requested.
        return output

class Conv2d_q(nn.Module):
    def __init__(self, name, bits, ksize, strides, padding=0, weight_decay=0):
        super(Conv2d_q, self).__init__()
        
        self.bits = bits
        self.ksize = ksize
        self.strides = strides
        self.padding = padding
        self.weight_decay = weight_decay

        # order: (out_channels, in_channels, H, W)
        self.weight = nn.Parameter(torch.Tensor(ksize[3], ksize[0], ksize[1], ksize[2]))
        self.bias = nn.Parameter(torch.Tensor(ksize[3]))
        # initialization
        self.weight.data.uniform_(-0.1, 0.1)
        self.bias.data.uniform_(-0.1, 0.1)

    def forward(self, input):       
        return Conv2d_func.apply(input, self.weight, self.bias, self.strides, self.padding)

Thanks for reply anyway, I still don’t quite understand why it is designed in this way, though. :thinking: