RuntimeError: one of the variables needed for gradient computation has been modified by an inplace operation: [torch.cuda.FloatTensor [4, 80, 25]], which is output 0 of ReciprocalBackward0, is at version 1; expected version 0 instead

vision
1 
import torch
import torch.nn as nn
#from torch.autograd import Variable
#from torch.autograd import Function
#from torch.nn.modules.module import Module
#from torch.nn.parameter import Parameter
from torch.nn.functional import conv2d
import torch.nn.functional as F
import numpy as np
import pywt
import scipy
class Kerv2d(nn.Conv2d):
    '''
    kervolution with following options:
    kernel_type: [linear, polynomial, gaussian, etc.]
    default is convolution:
             kernel_type --> linear,
    balance, power, gamma is valid only when the kernel_type is specified
    if learnable_kernel = True,  they just be the initial value of learable parameters
    if learnable_kernel = False, they are the value of kernel_type's parameter
    the parameter [power] cannot be learned due to integer limitation
    '''
    def __init__(self, in_channels, out_channels, kernel_size, 
            stride=1, padding_mode='zeros', dilation=1, groups=1, bias=True,
            kernel_type='sigmoid', learnable_kernel=False, kernel_regularizer=False, power=3, gamma=1,kernel_fn=None, balance=1):

        super(Kerv2d, self).__init__(in_channels, out_channels, kernel_size, stride, dilation, groups, bias,padding_mode)
        self.kernel_type = kernel_type
        #self.learnable_kernel, self.kernel_regularizer = learnable_kernel, kernel_regularizer
        self.balance, self.power, self.gamma = balance, power, gamma

        # parameter for kernel type
        #if learnable_kernel == True:
            #self.balance = nn.Parameter(torch.cuda.FloatTensor([balance] * out_channels), requires_grad=True).view(-1, 1)
            #self.gamma   = nn.Parameter(torch.cuda.FloatTensor([gamma]   * out_channels), requires_grad=True).view(-1, 1)
    
    
    def forward(self, input):

        minibatch, in_channels, input_width, input_hight = input.size()
        assert(in_channels == self.in_channels)
        input_unfold = F.unfold(input, kernel_size=self.kernel_size, dilation=self.dilation, padding=self.padding, stride=self.stride)
        input_unfold = input_unfold.view(minibatch, 1, self.kernel_size[0]*self.kernel_size[1]*self.in_channels, -1)
        weight_flat  = self.weight.view(self.out_channels, -1, 1)
        output_width = (input_width - self.kernel_size[0] + 2 * self.padding[0]) // self.stride[0] + 1
        output_hight = (input_hight - self.kernel_size[1] + 2 * self.padding[1]) // self.stride[1] + 1

        if self.kernel_type == 'linear':
            output = (input_unfold * weight_flat).sum(dim=2)

        elif self.kernel_type == 'manhattan':
            output = -((input_unfold - weight_flat).abs().sum(dim=2))

        elif self.kernel_type == 'euclidean':
            output = -(((input_unfold - weight_flat)**2).sum(dim=2))

        elif self.kernel_type == 'polynomial':
            
            output = ((input_unfold * weight_flat).sum(dim=2) + self.balance)**self.power

        elif self.kernel_type == 'gaussian':
            output = (-self.gamma*((input_unfold - weight_flat)**2).sum(dim=2)).exp() + 0

        elif self.kernel_type == 'rbf':
            output = ((-self.gamma*(((input_unfold - weight_flat)**2).abs().sum(dim=2)).sqrt()).exp())

        elif self.kernel_type =='bessel':
            output = torch.special.i0(input_unfold - weight_flat).sum(dim=2)

        elif self.kernel_type =='sigmoid':
            #output = 1/(1 + ((input_unfold - weight_flat).sum(dim=2)).exp())
            
            output = (1 + ((input_unfold - weight_flat).sum(dim=2)).exp()).reciprocal()
            

       
            
            
            
            
        else:
            raise NotImplementedError(self.kernel_type+' kervolution not implemented')
        


        if self.bias is not None:
            output += self.bias.view(self.out_channels, -1)

        return output.view(minibatch, self.out_channels, output_width, output_hight)

Hello friends. Can anyone help me in resolving this issue while implementing the sigmoid Kernel for CNN.
RuntimeError: one of the variables needed for gradient computation has been modified by an inplace operation: [torch.cuda.FloatTensor [4, 80, 25]], which is output 0 of ReciprocalBackward0, is at version 1; expected version 0 instead.

2

Could you replace that with

output = output + self.bias.view(self.out_channels, -1)

Your bias operation is being done inplace.

3

Thank you for your reply. I have replaced my code with

output = output + self.bias.view(self.out_channels, -1)

Now I am getting the following error

RuntimeError: Function ‘ExpBackward0’ returned nan values in its 0th output.

4

whatever’s computed within this is causing torch.exp() to return a NaN value. I see you’re using sigmoid as your given function? If so, you’re missing the minus sign on your exponent.

Sigmoid is defined as 1/(1 + torch.exp(-x)) whereas you have 1/(1 + exp(x)) at the moment where x is defined as ((input_unfold - weight_flat).sum(dim=2)).

You could just pass it into torch.sigmoid directly, via,

output = torch.sigmoid( ((input_unfold - weight_flat).sum(dim=2)) )
5

I am getting the following error after making the changes as suggested by you.

RuntimeError: Function ‘ExpBackward0’ returned nan values in its 0th output.

6

Could you try running with torch.autograd.set_detect_anomaly? To find where the NaN is appearing? Automatic differentiation package - torch.autograd — PyTorch 1.10.0 documentation

7

As per your suggestion, I have included torch.autograd.set_detect_anomaly(True) in my code. Following are the details of the error,

RuntimeError                              Traceback (most recent call last)
<ipython-input-69-599bd4b36306> in <module>()
     29         # Backward and optimize
     30         optimizer.zero_grad()
---> 31         loss.backward()
     32         optimizer.step()
     33 

1 frames
/usr/local/lib/python3.7/dist-packages/torch/_tensor.py in backward(self, gradient, retain_graph, create_graph, inputs)
    305                 create_graph=create_graph,
    306                 inputs=inputs)
--> 307         torch.autograd.backward(self, gradient, retain_graph, create_graph, inputs=inputs)
    308 
    309     def register_hook(self, hook):

/usr/local/lib/python3.7/dist-packages/torch/autograd/__init__.py in backward(tensors, grad_tensors, retain_graph, create_graph, grad_variables, inputs)
    154     Variable._execution_engine.run_backward(
    155         tensors, grad_tensors_, retain_graph, create_graph, inputs,
--> 156         allow_unreachable=True, accumulate_grad=True)  # allow_unreachable flag
    157 
    158 

RuntimeError: Function 'ExpBackward0' returned nan values in its 0th output.
8

So it seems that the torch.exp call is returning an invalid gradient. Could you print out the output of the torch.exp(-x) call where x is defined as ((input_unfold - weight_flat).sum(dim=2))? and possibly input_unfold - weight_flat?

I have a feeling torch.exp is return an infinity which when passed to the backward causes the NaN.

Did you check that you’re using torch.exp(-x) rather than torch.exp(x), the minus sign in the exponent is needed.

This should be output = (1 + ((input_unfold - weight_flat).sum(dim=2)).mul(-1).exp()).reciprocal(), in order for it to be a sigmoid function!

9

I am getting the same error for other kernels like rbf, Gaussian. Can you please suggest any modifications in code. I would also request to suggest inclusion of other useful kernels in my code for improvement of classification accuracy.

10

The error is emerging from the calculation of the Loss. Could you print the loss value? (Before calling .backward()) And also print out the value of,

to check it’s finite. If the loss is infinite, it will become NaN in the backward pass.

11

As per your suggestion I have printed the value of (input_unfold - weight_flat)**2 for an rbf kernel as given below.

elif self.kernel_type == 'rbf':
            print((input_unfold - weight_flat)**2)
            output = torch.exp((self.gamma*(((input_unfold - weight_flat)**2).abs().sum(dim=2).mul(-1)).sqrt()))

I am getting the following tensor.

tensor([[[[1.5106e-03, 1.5106e-03, 1.5106e-03, …, 1.1569e-03,
3.0368e-04, 6.0389e-03],
[2.8391e-04, 2.8391e-04, 2.8391e-04, …, 5.3499e-03,
4.8374e-04, 1.9257e-05],
[1.5675e-03, 1.5675e-03, 1.5675e-03, …, 6.1521e-03,
2.7095e-03, 1.5675e-03],
…,
[7.2640e-03, 8.6801e-01, 3.9451e-01, …, 7.2640e-03,
7.2640e-03, 7.2640e-03],
[8.6283e-01, 3.9102e-01, 7.7187e-02, …, 6.7971e-03,
6.7971e-03, 6.7971e-03],
[3.0447e-01, 1.2345e-01, 6.5897e-02, …, 7.9489e-05,
7.9489e-05, 7.9489e-05]],

     [[7.3047e-06, 7.3047e-06, 7.3047e-06,  ..., 5.7090e-05,
       3.4805e-03, 1.3062e-03],
      [2.5936e-03, 2.5936e-03, 2.5936e-03,  ..., 2.8788e-05,
       8.0589e-03, 4.0182e-03],
      [3.9495e-03, 3.9495e-03, 3.9495e-03,  ..., 5.7607e-04,
       2.5385e-03, 3.9495e-03],
      ...,
      [3.7975e-03, 6.1594e-01, 2.3160e-01,  ..., 3.7975e-03,
       3.7975e-03, 3.7975e-03],
      [6.7872e-01, 2.7069e-01, 1.4658e-01,  ..., 5.1044e-04,
       5.1044e-04, 5.1044e-04],
      [3.3737e-01, 1.0388e-01, 5.1827e-02,  ..., 1.4412e-03,
       1.4412e-03, 1.4412e-03]],

     [[8.8988e-04, 8.8988e-04, 8.8988e-04,  ..., 6.2389e-04,
       7.0026e-04, 4.7162e-03],
      [2.9093e-04, 2.9093e-04, 2.9093e-04,  ..., 5.3802e-03,
       4.7467e-04, 2.1118e-05],
      [6.8946e-03, 6.8946e-03, 6.8946e-03,  ..., 1.4854e-02,
       9.1193e-03, 6.8946e-03],
      ...,
      [1.6353e-04, 7.3827e-01, 3.0876e-01,  ..., 1.6353e-04,
       1.6353e-04, 1.6353e-04],
      [7.1580e-01, 2.9429e-01, 1.3008e-01,  ..., 1.5248e-07,
       1.5248e-07, 1.5248e-07],
      [3.6435e-01, 8.9718e-02, 4.1975e-02,  ..., 3.6894e-03,
       3.6894e-03, 3.6894e-03]],

     ...,

     [[3.3553e-03, 3.3553e-03, 3.3553e-03,  ..., 3.9410e-03,
       1.3046e-02, 3.6408e-04],
      [8.0142e-04, 8.0142e-04, 8.0142e-04,  ..., 7.1576e-03,
       1.1097e-04, 2.5116e-04],
      [6.5826e-04, 6.5826e-04, 6.5826e-04,  ..., 4.1603e-03,
       1.4530e-03, 6.5826e-04],
      ...,
      [1.3704e-03, 6.5516e-01, 2.5589e-01,  ..., 1.3704e-03,
       1.3704e-03, 1.3704e-03],
      [8.3160e-01, 3.7010e-01, 8.6902e-02,  ..., 4.2876e-03,
       4.2876e-03, 4.2876e-03],
      [3.8677e-01, 7.9096e-02, 3.4815e-02,  ..., 6.2459e-03,
       6.2459e-03, 6.2459e-03]],

     [[4.5563e-03, 4.5563e-03, 4.5563e-03,  ..., 5.2350e-03,
       1.5325e-02, 8.2121e-04],
      [4.1114e-03, 4.1114e-03, 4.1114e-03,  ..., 6.1266e-05,
       1.0602e-02, 5.8648e-03],
      [1.6336e-03, 1.6336e-03, 1.6336e-03,  ..., 6.2824e-03,
       2.7962e-03, 1.6336e-03],
      ...,
      [7.2603e-03, 8.6797e-01, 3.9449e-01,  ..., 7.2603e-03,
       7.2603e-03, 7.2603e-03],
      [7.4526e-01, 3.1328e-01, 1.1794e-01,  ..., 2.8369e-04,
       2.8369e-04, 2.8369e-04],
      [2.6591e-01, 1.5014e-01, 8.5750e-02,  ..., 7.4053e-04,
       7.4053e-04, 7.4053e-04]],

     [[4.6240e-04, 4.6240e-04, 4.6240e-04,  ..., 2.7724e-04,
       1.2103e-03, 3.6418e-03],
      [6.0864e-03, 6.0864e-03, 6.0864e-03,  ..., 1.8039e-02,
       1.5344e-03, 4.2973e-03],
      [6.5879e-03, 6.5879e-03, 6.5879e-03,  ..., 1.7912e-03,
       4.7203e-03, 6.5879e-03],
      ...,
      [7.2810e-04, 7.6287e-01, 3.2474e-01,  ..., 7.2810e-04,
       7.2810e-04, 7.2810e-04],
      [7.7364e-01, 3.3178e-01, 1.0702e-01,  ..., 1.0974e-03,
       1.0974e-03, 1.0974e-03],
      [2.4592e-01, 1.6585e-01, 9.7714e-02,  ..., 2.2065e-03,
       2.2065e-03, 2.2065e-03]]],


    [[[1.5106e-03, 1.5106e-03, 1.5106e-03,  ..., 1.6799e-04,
       3.5862e-02, 1.3757e-03],
      [2.8391e-04, 2.8391e-04, 2.8391e-04,  ..., 1.7864e-02,
       3.4694e-04, 6.5997e-05],
      [1.5675e-03, 1.5675e-03, 1.5675e-03,  ..., 1.4300e-03,
       4.1687e-03, 1.5675e-03],
      ...,
      [7.2640e-03, 1.0410e+00, 1.0757e-02,  ..., 7.2640e-03,
       7.2640e-03, 7.2640e-03],
      [1.0353e+00, 1.0187e-02, 3.2086e-01,  ..., 6.7971e-03,
       6.7971e-03, 6.7971e-03],
      [7.5077e-04, 2.4296e-01, 4.0355e-02,  ..., 7.9489e-05,
       7.9489e-05, 7.9489e-05]],

     [[7.3047e-06, 7.3047e-06, 7.3047e-06,  ..., 8.1844e-04,
       2.1846e-02, 2.0065e-05],
      [2.5936e-03, 2.5936e-03, 2.5936e-03,  ..., 4.0575e-02,
       2.4158e-03, 5.7610e-03],
      [3.9495e-03, 3.9495e-03, 3.9495e-03,  ..., 4.1760e-03,
       1.4343e-03, 3.9495e-03],
      ...,
      [3.7975e-03, 7.6289e-01, 1.8610e-03,  ..., 3.7975e-03,
       3.7975e-03, 3.7975e-03],
      [8.3260e-01, 1.6878e-05, 2.1289e-01,  ..., 5.1044e-04,
       5.1044e-04, 5.1044e-04],
      [3.1863e-03, 2.7244e-01, 2.9529e-02,  ..., 1.4412e-03,
       1.4412e-03, 1.4412e-03]],

     [[8.8988e-04, 8.8988e-04, 8.8988e-04,  ..., 1.5406e-05,
       3.2521e-02, 7.8703e-04],
      [2.9093e-04, 2.9093e-04, 2.9093e-04,  ..., 1.7808e-02,
       3.5470e-04, 6.2674e-05],
      [6.8946e-03, 6.8946e-03, 6.8946e-03,  ..., 6.6027e-03,
       1.1666e-02, 6.8946e-03],
      ...,
      [1.6353e-04, 8.9842e-01, 9.7796e-04,  ..., 1.6353e-04,
       1.6353e-04, 1.6353e-04],
      [8.7361e-01, 3.2740e-04, 2.3387e-01,  ..., 1.5248e-07,
       1.5248e-07, 1.5248e-07],
      [6.2766e-03, 2.9674e-01, 2.2219e-02,  ..., 3.6894e-03,
       3.6894e-03, 3.6894e-03]],

     ...,

     [[3.3553e-03, 3.3553e-03, 3.3553e-03,  ..., 7.0275e-03,
       8.5711e-03, 3.5642e-03],
      [8.0142e-04, 8.0142e-04, 8.0142e-04,  ..., 1.4932e-02,
       9.0518e-04, 1.1129e-05],
      [6.5826e-04, 6.5826e-04, 6.5826e-04,  ..., 5.7025e-04,
       2.5634e-03, 6.5826e-04],
      ...,
      [1.3704e-03, 8.0648e-01, 3.4350e-04,  ..., 1.3704e-03,
       1.3704e-03, 1.3704e-03],
      [1.0011e+00, 7.0500e-03, 3.0192e-01,  ..., 4.2876e-03,
       4.2876e-03, 4.2876e-03],
      [9.5092e-03, 3.1700e-01, 1.7101e-02,  ..., 6.2459e-03,
       6.2459e-03, 6.2459e-03]],

     [[4.5563e-03, 4.5563e-03, 4.5563e-03,  ..., 8.7247e-03,
       6.8898e-03, 4.7993e-03],
      [4.1114e-03, 4.1114e-03, 4.1114e-03,  ..., 4.6064e-02,
       3.8867e-03, 7.9377e-03],
      [1.6336e-03, 1.6336e-03, 1.6336e-03,  ..., 1.4932e-03,
       4.2760e-03, 1.6336e-03],
      ...,
      [7.2603e-03, 1.0409e+00, 1.0752e-02,  ..., 7.2603e-03,
       7.2603e-03, 7.2603e-03],
      [9.0612e-01, 1.2480e-03, 2.5084e-01,  ..., 2.8369e-04,
       2.8369e-04, 2.8369e-04],
      [7.6180e-05, 2.0865e-01, 5.6176e-02,  ..., 7.4053e-04,
       7.4053e-04, 7.4053e-04]],

     [[4.6240e-04, 4.6240e-04, 4.6240e-04,  ..., 1.9379e-05,
       2.9587e-02, 3.8915e-04],
      [6.0864e-03, 6.0864e-03, 6.0864e-03,  ..., 5.2548e-03,
       6.3668e-03, 2.8134e-03],
      [6.5879e-03, 6.5879e-03, 6.5879e-03,  ..., 6.8795e-03,
       3.1576e-03, 6.5879e-03],
      ...,
      [7.2810e-04, 9.2553e-01, 2.0673e-03,  ..., 7.2810e-04,
       7.2810e-04, 7.2810e-04],
      [9.3739e-01, 2.6637e-03, 2.6742e-01,  ..., 1.0974e-03,
       1.0974e-03, 1.0974e-03],
      [8.1160e-04, 1.9099e-01, 6.5933e-02,  ..., 2.2065e-03,
       2.2065e-03, 2.2065e-03]]],


    [[[1.5106e-03, 1.5106e-03, 1.5106e-03,  ..., 1.1227e-03,
       4.5607e-03, 4.0703e-03],
      [2.8391e-04, 2.8391e-04, 2.8391e-04,  ..., 1.3964e-04,
       6.5322e-05, 6.6743e-04],
      [1.5675e-03, 1.5675e-03, 1.5675e-03,  ..., 4.1633e-03,
       9.3677e-04, 1.5675e-03],
      ...,
      [7.2640e-03, 7.9553e-01, 5.7596e-02,  ..., 7.2640e-03,
       7.2640e-03, 7.2640e-03],
      [7.9057e-01, 5.6267e-02, 5.9802e-02,  ..., 6.7971e-03,
       6.7971e-03, 6.7971e-03],
      [2.6790e-02, 2.9246e-02, 7.7913e-02,  ..., 7.9489e-05,
       7.9489e-05, 7.9489e-05]],

     [[7.3047e-06, 7.3047e-06, 7.3047e-06,  ..., 6.5016e-05,
       6.7411e-04, 4.9413e-04],
      [2.5936e-03, 2.5936e-03, 2.5936e-03,  ..., 6.3352e-03,
       5.7547e-03, 1.7592e-03],
      [3.9495e-03, 3.9495e-03, 3.9495e-03,  ..., 1.4374e-03,
       5.1596e-03, 3.9495e-03],
      ...,
      [3.7975e-03, 5.5513e-01, 8.6746e-03,  ..., 3.7975e-03,
       3.7975e-03, 3.7975e-03],
      [6.1481e-01, 1.7469e-02, 1.9462e-02,  ..., 5.1044e-04,
       5.1044e-04, 5.1044e-04],
      [3.7143e-02, 4.0025e-02, 6.2541e-02,  ..., 1.4412e-03,
       1.4412e-03, 1.4412e-03]],

     [[8.8988e-04, 8.8988e-04, 8.8988e-04,  ..., 5.9879e-04,
       3.4219e-03, 2.9989e-03],
      [2.9093e-04, 2.9093e-04, 2.9093e-04,  ..., 1.3479e-04,
       6.2016e-05, 6.7818e-04],
      [6.8946e-03, 6.8946e-03, 6.8946e-03,  ..., 1.1657e-02,
       5.4832e-03, 6.8946e-03],
      ...,
      [1.6353e-04, 6.7155e-01, 2.8073e-02,  ..., 1.6353e-04,
       1.6353e-04, 1.6353e-04],
      [6.5013e-01, 2.3830e-02, 2.6150e-02,  ..., 1.5248e-07,
       1.5248e-07, 1.5248e-07],
      [4.6441e-02, 4.9657e-02, 5.1667e-02,  ..., 3.6894e-03,
       3.6894e-03, 3.6894e-03]],

     ...,

     [[3.3553e-03, 3.3553e-03, 3.3553e-03,  ..., 4.0050e-03,
       8.5604e-04, 1.0885e-03],
      [8.0142e-04, 8.0142e-04, 8.0142e-04,  ..., 1.2746e-07,
       1.1409e-05, 1.3909e-03],
      [6.5826e-04, 6.5826e-04, 6.5826e-04,  ..., 2.5592e-03,
       2.7794e-04, 6.5826e-04],
      ...,
      [1.3704e-03, 5.9240e-01, 1.3864e-02,  ..., 1.3704e-03,
       1.3704e-03, 1.3704e-03],
      [7.6069e-01, 4.8506e-02, 5.1792e-02,  ..., 4.2876e-03,
       4.2876e-03, 4.2876e-03],
      [5.4659e-02, 5.8144e-02, 4.3687e-02,  ..., 6.2459e-03,
       6.2459e-03, 6.2459e-03]],

     [[4.5563e-03, 4.5563e-03, 4.5563e-03,  ..., 5.3087e-03,
       1.5081e-03, 1.8121e-03],
      [4.1114e-03, 4.1114e-03, 4.1114e-03,  ..., 8.6094e-03,
       7.9303e-03, 3.0399e-03],
      [1.6336e-03, 1.6336e-03, 1.6336e-03,  ..., 4.2706e-03,
       9.8803e-04, 1.6336e-03],
      ...,
      [7.2603e-03, 7.9549e-01, 5.7585e-02,  ..., 7.2603e-03,
       7.2603e-03, 7.2603e-03],
      [6.7821e-01, 2.9448e-02, 3.2020e-02,  ..., 2.8369e-04,
       2.8369e-04, 2.8369e-04],
      [1.6269e-02, 1.8194e-02, 9.9387e-02,  ..., 7.4053e-04,
       7.4053e-04, 7.4053e-04]],

     [[4.6240e-04, 4.6240e-04, 4.6240e-04,  ..., 2.6060e-04,
       2.5170e-03, 2.1562e-03],
      [6.0864e-03, 6.0864e-03, 6.0864e-03,  ..., 2.4353e-03,
       2.8179e-03, 7.5691e-03],
      [6.5879e-03, 6.5879e-03, 6.5879e-03,  ..., 3.1623e-03,
       8.1272e-03, 6.5879e-03],
      ...,
      [7.2810e-04, 6.9502e-01, 3.3031e-02,  ..., 7.2810e-04,
       7.2810e-04, 7.2810e-04],
      [7.0530e-01, 3.5302e-02, 3.8113e-02,  ..., 1.0974e-03,
       1.0974e-03, 1.0974e-03],
      [1.1618e-02, 1.3254e-02, 1.1224e-01,  ..., 2.2065e-03,
       2.2065e-03, 2.2065e-03]]],


    [[[1.5106e-03, 1.5106e-03, 1.5106e-03,  ..., 4.0845e-03,
       3.8873e-02, 2.6158e-03],
      [2.8391e-04, 2.8391e-04, 2.8391e-04,  ..., 2.0007e-02,
       2.0895e-05, 6.4618e-03],
      [1.5675e-03, 1.5675e-03, 1.5675e-03,  ..., 2.6905e-03,
       5.7331e-04, 1.5675e-03],
      ...,
      [7.2640e-03, 1.3240e-01, 2.2788e-02,  ..., 7.2640e-03,
       7.2640e-03, 7.2640e-03],
      [1.3443e-01, 2.1955e-02, 1.1336e-02,  ..., 6.7971e-03,
       6.7971e-03, 6.7971e-03],
      [5.5719e-03, 3.2399e-02, 1.2244e-02,  ..., 7.9489e-05,
       7.9489e-05, 7.9489e-05]],

     [[7.3047e-06, 7.3047e-06, 7.3047e-06,  ..., 4.9911e-04,
       2.4209e-02, 9.1694e-05],
      [2.5936e-03, 2.5936e-03, 2.5936e-03,  ..., 4.3775e-02,
       3.9950e-03, 1.5896e-04],
      [3.9495e-03, 3.9495e-03, 3.9495e-03,  ..., 2.5570e-03,
       1.5972e-02, 3.9495e-03],
      ...,
      [3.7975e-03, 2.6083e-01, 1.6852e-05,  ..., 3.7975e-03,
       3.7975e-03, 3.7975e-03],
      [2.2249e-01, 1.8607e-03, 4.4735e-02,  ..., 5.1044e-04,
       5.1044e-04, 5.1044e-04],
      [1.0752e-02, 2.2786e-02, 6.6593e-03,  ..., 1.4412e-03,
       1.4412e-03, 1.4412e-03]],

     [[8.8988e-04, 8.8988e-04, 8.8988e-04,  ..., 3.0112e-03,
       3.5392e-02, 1.7732e-03],
      [2.9093e-04, 2.9093e-04, 2.9093e-04,  ..., 1.9949e-02,
       2.2832e-05, 6.4951e-03],
      [6.8946e-03, 6.8946e-03, 6.8946e-03,  ..., 9.0845e-03,
       3.8018e-04, 6.8946e-03],
      ...,
      [1.6353e-04, 1.9036e-01, 6.1649e-03,  ..., 1.6353e-04,
       1.6353e-04, 1.6353e-04],
      [2.0204e-01, 4.2691e-03, 3.5836e-02,  ..., 1.5248e-07,
       1.5248e-07, 1.5248e-07],
      [1.5995e-02, 1.6428e-02, 3.4607e-03,  ..., 3.6894e-03,
       3.6894e-03, 3.6894e-03]],

     ...,

     [[3.3553e-03, 3.3553e-03, 3.3553e-03,  ..., 1.0812e-03,
       1.0074e-02, 2.0836e-03],
      [8.0142e-04, 8.0142e-04, 8.0142e-04,  ..., 1.6897e-02,
       2.5699e-04, 8.4355e-03],
      [6.5826e-04, 6.5826e-04, 6.5826e-04,  ..., 1.4391e-03,
       1.4348e-03, 6.5826e-04],
      ...,
      [1.3704e-03, 2.3630e-01, 8.2432e-04,  ..., 1.3704e-03,
       1.3704e-03, 1.3704e-03],
      [1.4716e-01, 1.7216e-02, 1.5236e-02,  ..., 4.2876e-03,
       4.2876e-03, 4.2876e-03],
      [2.0955e-02, 1.2074e-02, 1.6432e-03,  ..., 6.2459e-03,
       6.2459e-03, 6.2459e-03]],

     [[4.5563e-03, 4.5563e-03, 4.5563e-03,  ..., 1.8026e-03,
       8.2438e-03, 3.0495e-03],
      [4.1114e-03, 4.1114e-03, 4.1114e-03,  ..., 4.9469e-02,
       5.8368e-03, 3.4194e-07],
      [1.6336e-03, 1.6336e-03, 1.6336e-03,  ..., 2.7769e-03,
       5.3442e-04, 1.6336e-03],
      ...,
      [7.2603e-03, 1.3241e-01, 2.2782e-02,  ..., 7.2603e-03,
       7.2603e-03, 7.2603e-03],
      [1.8684e-01, 6.8182e-03, 2.9608e-02,  ..., 2.8369e-04,
       2.8369e-04, 2.8369e-04],
      [1.4835e-03, 4.6711e-02, 2.1544e-02,  ..., 7.4053e-04,
       7.4053e-04, 7.4053e-04]],

     [[4.6240e-04, 4.6240e-04, 4.6240e-04,  ..., 2.1666e-03,
       3.2328e-02, 1.1412e-03],
      [6.0864e-03, 6.0864e-03, 6.0864e-03,  ..., 6.4450e-03,
       4.3213e-03, 2.0037e-02],
      [6.5879e-03, 6.5879e-03, 6.5879e-03,  ..., 4.7455e-03,
       2.0939e-02, 6.5879e-03],
      ...,
      [7.2810e-04, 1.7818e-01, 8.5956e-03,  ..., 7.2810e-04,
       7.2810e-04, 7.2810e-04],
      [1.7303e-01, 9.7725e-03, 2.4270e-02,  ..., 1.0974e-03,
       1.0974e-03, 1.0974e-03],
      [3.5179e-04, 5.5643e-02, 2.7736e-02,  ..., 2.2065e-03,
       2.2065e-03, 2.2065e-03]]]], device='cuda:0', grad_fn=<PowBackward0>)

tensor([[[[3.9900e-04, 3.9900e-04, 3.9900e-04, …, nan,
3.9900e-04, nan],
[2.3724e-03, 2.3724e-03, 2.3724e-03, …, 2.3724e-03,
nan, nan],
[6.4523e-05, 6.4523e-05, 6.4523e-05, …, nan,
nan, 6.4523e-05],
…,
[3.4188e-03, nan, nan, …, 3.4188e-03,
3.4188e-03, 3.4188e-03],
[ nan, nan, nan, …, 1.7611e-04,
1.7611e-04, 1.7611e-04],
[ nan, nan, nan, …, 3.1767e-03,
3.1767e-03, 3.1767e-03]],

     [[8.7806e-07, 8.7806e-07, 8.7806e-07,  ...,        nan,
       8.7806e-07,        nan],
      [4.7228e-05, 4.7228e-05, 4.7228e-05,  ..., 4.7228e-05,
              nan,        nan],
      [2.5639e-03, 2.5639e-03, 2.5639e-03,  ...,        nan,
              nan, 2.5639e-03],
      ...,
      [4.1269e-08,        nan,        nan,  ..., 4.1269e-08,
       4.1269e-08, 4.1269e-08],
      [       nan,        nan,        nan,  ..., 1.3153e-03,
       1.3153e-03, 1.3153e-03],
      [       nan,        nan,        nan,  ..., 4.0486e-04,
       4.0486e-04, 4.0486e-04]],

     [[3.1932e-03, 3.1932e-03, 3.1932e-03,  ...,        nan,
       3.1932e-03,        nan],
      [2.6909e-03, 2.6909e-03, 2.6909e-03,  ..., 2.6909e-03,
              nan,        nan],
      [9.7583e-05, 9.7583e-05, 9.7583e-05,  ...,        nan,
              nan, 9.7583e-05],
      ...,
      [1.7329e-03,        nan,        nan,  ..., 1.7329e-03,
       1.7329e-03, 1.7329e-03],
      [       nan,        nan,        nan,  ..., 2.8461e-03,
       2.8461e-03, 2.8461e-03],
      [       nan,        nan,        nan,  ..., 1.0612e-05,
       1.0612e-05, 1.0612e-05]],

     ...,

     [[2.9565e-03, 2.9565e-03, 2.9565e-03,  ...,        nan,
       2.9565e-03,        nan],
      [3.2911e-03, 3.2911e-03, 3.2911e-03,  ..., 3.2911e-03,
              nan,        nan],
      [9.1167e-06, 9.1167e-06, 9.1167e-06,  ...,        nan,
              nan, 9.1167e-06],
      ...,
      [3.4284e-03,        nan,        nan,  ..., 3.4284e-03,
       3.4284e-03, 3.4284e-03],
      [       nan,        nan,        nan,  ..., 1.7568e-03,
       1.7568e-03, 1.7568e-03],
      [       nan,        nan,        nan,  ..., 1.5258e-06,
       1.5258e-06, 1.5258e-06]],

     [[5.7720e-04, 5.7720e-04, 5.7720e-04,  ...,        nan,
       5.7720e-04,        nan],
      [1.2828e-05, 1.2828e-05, 1.2828e-05,  ..., 1.2828e-05,
              nan,        nan],
      [1.6647e-03, 1.6647e-03, 1.6647e-03,  ...,        nan,
              nan, 1.6647e-03],
      ...,
      [1.0735e-03,        nan,        nan,  ..., 1.0735e-03,
       1.0735e-03, 1.0735e-03],
      [       nan,        nan,        nan,  ..., 1.4540e-04,
       1.4540e-04, 1.4540e-04],
      [       nan,        nan,        nan,  ..., 1.1156e-03,
       1.1156e-03, 1.1156e-03]],

     [[1.3076e-04, 1.3076e-04, 1.3076e-04,  ...,        nan,
       1.3076e-04,        nan],
      [2.4632e-03, 2.4632e-03, 2.4632e-03,  ..., 2.4632e-03,
              nan,        nan],
      [3.0567e-03, 3.0567e-03, 3.0567e-03,  ...,        nan,
              nan, 3.0567e-03],
      ...,
      [3.0679e-03,        nan,        nan,  ..., 3.0679e-03,
       3.0679e-03, 3.0679e-03],
      [       nan,        nan,        nan,  ..., 2.7698e-03,
       2.7698e-03, 2.7698e-03],
      [       nan,        nan,        nan,  ..., 3.1502e-03,
       3.1502e-03, 3.1502e-03]]],


    [[[3.9900e-04, 3.9900e-04, 3.9900e-04,  ...,        nan,
       3.9900e-04,        nan],
      [2.3724e-03, 2.3724e-03, 2.3724e-03,  ..., 2.3724e-03,
              nan,        nan],
      [6.4523e-05, 6.4523e-05, 6.4523e-05,  ...,        nan,
              nan, 6.4523e-05],
      ...,
      [3.4188e-03,        nan,        nan,  ..., 3.4188e-03,
       3.4188e-03, 3.4188e-03],
      [       nan,        nan,        nan,  ..., 1.7611e-04,
       1.7611e-04, 1.7611e-04],
      [       nan,        nan,        nan,  ..., 3.1767e-03,
       3.1767e-03, 3.1767e-03]],

     [[8.7806e-07, 8.7806e-07, 8.7806e-07,  ...,        nan,
       8.7806e-07,        nan],
      [4.7228e-05, 4.7228e-05, 4.7228e-05,  ..., 4.7228e-05,
              nan,        nan],
      [2.5639e-03, 2.5639e-03, 2.5639e-03,  ...,        nan,
              nan, 2.5639e-03],
      ...,
      [4.1269e-08,        nan,        nan,  ..., 4.1269e-08,
       4.1269e-08, 4.1269e-08],
      [       nan,        nan,        nan,  ..., 1.3153e-03,
       1.3153e-03, 1.3153e-03],
      [       nan,        nan,        nan,  ..., 4.0486e-04,
       4.0486e-04, 4.0486e-04]],

     [[3.1932e-03, 3.1932e-03, 3.1932e-03,  ...,        nan,
       3.1932e-03,        nan],
      [2.6909e-03, 2.6909e-03, 2.6909e-03,  ..., 2.6909e-03,
              nan,        nan],
      [9.7583e-05, 9.7583e-05, 9.7583e-05,  ...,        nan,
              nan, 9.7583e-05],
      ...,
      [1.7329e-03,        nan,        nan,  ..., 1.7329e-03,
       1.7329e-03, 1.7329e-03],
      [       nan,        nan,        nan,  ..., 2.8461e-03,
       2.8461e-03, 2.8461e-03],
      [       nan,        nan,        nan,  ..., 1.0612e-05,
       1.0612e-05, 1.0612e-05]],

     ...,

     [[2.9565e-03, 2.9565e-03, 2.9565e-03,  ...,        nan,
       2.9565e-03,        nan],
      [3.2911e-03, 3.2911e-03, 3.2911e-03,  ..., 3.2911e-03,
              nan,        nan],
      [9.1167e-06, 9.1167e-06, 9.1167e-06,  ...,        nan,
              nan, 9.1167e-06],
      ...,
      [3.4284e-03,        nan,        nan,  ..., 3.4284e-03,
       3.4284e-03, 3.4284e-03],
      [       nan,        nan,        nan,  ..., 1.7568e-03,
       1.7568e-03, 1.7568e-03],
      [       nan,        nan,        nan,  ..., 1.5258e-06,
       1.5258e-06, 1.5258e-06]],

     [[5.7720e-04, 5.7720e-04, 5.7720e-04,  ...,        nan,
       5.7720e-04,        nan],
      [1.2828e-05, 1.2828e-05, 1.2828e-05,  ..., 1.2828e-05,
              nan,        nan],
      [1.6647e-03, 1.6647e-03, 1.6647e-03,  ...,        nan,
              nan, 1.6647e-03],
      ...,
      [1.0735e-03,        nan,        nan,  ..., 1.0735e-03,
       1.0735e-03, 1.0735e-03],
      [       nan,        nan,        nan,  ..., 1.4540e-04,
       1.4540e-04, 1.4540e-04],
      [       nan,        nan,        nan,  ..., 1.1156e-03,
       1.1156e-03, 1.1156e-03]],

     [[1.3076e-04, 1.3076e-04, 1.3076e-04,  ...,        nan,
       1.3076e-04,        nan],
      [2.4632e-03, 2.4632e-03, 2.4632e-03,  ..., 2.4632e-03,
              nan,        nan],
      [3.0567e-03, 3.0567e-03, 3.0567e-03,  ...,        nan,
              nan, 3.0567e-03],
      ...,
      [3.0679e-03,        nan,        nan,  ..., 3.0679e-03,
       3.0679e-03, 3.0679e-03],
      [       nan,        nan,        nan,  ..., 2.7698e-03,
       2.7698e-03, 2.7698e-03],
      [       nan,        nan,        nan,  ..., 3.1502e-03,
       3.1502e-03, 3.1502e-03]]],


    [[[3.9900e-04, 3.9900e-04, 3.9900e-04,  ...,        nan,
              nan,        nan],
      [2.3724e-03, 2.3724e-03, 2.3724e-03,  ...,        nan,
              nan,        nan],
      [6.4523e-05, 6.4523e-05, 6.4523e-05,  ...,        nan,
              nan, 6.4523e-05],
      ...,
      [3.4188e-03,        nan,        nan,  ..., 3.4188e-03,
       3.4188e-03, 3.4188e-03],
      [       nan,        nan,        nan,  ..., 1.7611e-04,
       1.7611e-04, 1.7611e-04],
      [       nan,        nan,        nan,  ..., 3.1767e-03,
       3.1767e-03, 3.1767e-03]],

     [[8.7806e-07, 8.7806e-07, 8.7806e-07,  ...,        nan,
              nan,        nan],
      [4.7228e-05, 4.7228e-05, 4.7228e-05,  ...,        nan,
              nan,        nan],
      [2.5639e-03, 2.5639e-03, 2.5639e-03,  ...,        nan,
              nan, 2.5639e-03],
      ...,
      [4.1269e-08,        nan,        nan,  ..., 4.1269e-08,
       4.1269e-08, 4.1269e-08],
      [       nan,        nan,        nan,  ..., 1.3153e-03,
       1.3153e-03, 1.3153e-03],
      [       nan,        nan,        nan,  ..., 4.0486e-04,
       4.0486e-04, 4.0486e-04]],

     [[3.1932e-03, 3.1932e-03, 3.1932e-03,  ...,        nan,
              nan,        nan],
      [2.6909e-03, 2.6909e-03, 2.6909e-03,  ...,        nan,
              nan,        nan],
      [9.7583e-05, 9.7583e-05, 9.7583e-05,  ...,        nan,
              nan, 9.7583e-05],
      ...,
      [1.7329e-03,        nan,        nan,  ..., 1.7329e-03,
       1.7329e-03, 1.7329e-03],
      [       nan,        nan,        nan,  ..., 2.8461e-03,
       2.8461e-03, 2.8461e-03],
      [       nan,        nan,        nan,  ..., 1.0612e-05,
       1.0612e-05, 1.0612e-05]],

     ...,

     [[2.9565e-03, 2.9565e-03, 2.9565e-03,  ...,        nan,
              nan,        nan],
      [3.2911e-03, 3.2911e-03, 3.2911e-03,  ...,        nan,
              nan,        nan],
      [9.1167e-06, 9.1167e-06, 9.1167e-06,  ...,        nan,
              nan, 9.1167e-06],
      ...,
      [3.4284e-03,        nan,        nan,  ..., 3.4284e-03,
       3.4284e-03, 3.4284e-03],
      [       nan,        nan,        nan,  ..., 1.7568e-03,
       1.7568e-03, 1.7568e-03],
      [       nan,        nan,        nan,  ..., 1.5258e-06,
       1.5258e-06, 1.5258e-06]],

     [[5.7720e-04, 5.7720e-04, 5.7720e-04,  ...,        nan,
              nan,        nan],
      [1.2828e-05, 1.2828e-05, 1.2828e-05,  ...,        nan,
              nan,        nan],
      [1.6647e-03, 1.6647e-03, 1.6647e-03,  ...,        nan,
              nan, 1.6647e-03],
      ...,
      [1.0735e-03,        nan,        nan,  ..., 1.0735e-03,
       1.0735e-03, 1.0735e-03],
      [       nan,        nan,        nan,  ..., 1.4540e-04,
       1.4540e-04, 1.4540e-04],
      [       nan,        nan,        nan,  ..., 1.1156e-03,
       1.1156e-03, 1.1156e-03]],

     [[1.3076e-04, 1.3076e-04, 1.3076e-04,  ...,        nan,
              nan,        nan],
      [2.4632e-03, 2.4632e-03, 2.4632e-03,  ...,        nan,
              nan,        nan],
      [3.0567e-03, 3.0567e-03, 3.0567e-03,  ...,        nan,
              nan, 3.0567e-03],
      ...,
      [3.0679e-03,        nan,        nan,  ..., 3.0679e-03,
       3.0679e-03, 3.0679e-03],
      [       nan,        nan,        nan,  ..., 2.7698e-03,
       2.7698e-03, 2.7698e-03],
      [       nan,        nan,        nan,  ..., 3.1502e-03,
       3.1502e-03, 3.1502e-03]]],


    [[[3.9900e-04, 3.9900e-04, 3.9900e-04,  ...,        nan,
              nan,        nan],
      [2.3724e-03, 2.3724e-03, 2.3724e-03,  ...,        nan,
              nan,        nan],
      [6.4523e-05, 6.4523e-05, 6.4523e-05,  ...,        nan,
              nan, 6.4523e-05],
      ...,
      [3.4188e-03,        nan, 3.4188e-03,  ..., 3.4188e-03,
       3.4188e-03, 3.4188e-03],
      [       nan, 1.7611e-04,        nan,  ..., 1.7611e-04,
       1.7611e-04, 1.7611e-04],
      [3.1767e-03,        nan,        nan,  ..., 3.1767e-03,
       3.1767e-03, 3.1767e-03]],

     [[8.7806e-07, 8.7806e-07, 8.7806e-07,  ...,        nan,
              nan,        nan],
      [4.7228e-05, 4.7228e-05, 4.7228e-05,  ...,        nan,
              nan,        nan],
      [2.5639e-03, 2.5639e-03, 2.5639e-03,  ...,        nan,
              nan, 2.5639e-03],
      ...,
      [4.1269e-08,        nan, 4.1269e-08,  ..., 4.1269e-08,
       4.1269e-08, 4.1269e-08],
      [       nan, 1.3153e-03,        nan,  ..., 1.3153e-03,
       1.3153e-03, 1.3153e-03],
      [4.0486e-04,        nan,        nan,  ..., 4.0486e-04,
       4.0486e-04, 4.0486e-04]],

     [[3.1932e-03, 3.1932e-03, 3.1932e-03,  ...,        nan,
              nan,        nan],
      [2.6909e-03, 2.6909e-03, 2.6909e-03,  ...,        nan,
              nan,        nan],
      [9.7583e-05, 9.7583e-05, 9.7583e-05,  ...,        nan,
              nan, 9.7583e-05],
      ...,
      [1.7329e-03,        nan, 1.7329e-03,  ..., 1.7329e-03,
       1.7329e-03, 1.7329e-03],
      [       nan, 2.8461e-03,        nan,  ..., 2.8461e-03,
       2.8461e-03, 2.8461e-03],
      [1.0612e-05,        nan,        nan,  ..., 1.0612e-05,
       1.0612e-05, 1.0612e-05]],

     ...,

     [[2.9565e-03, 2.9565e-03, 2.9565e-03,  ...,        nan,
              nan,        nan],
      [3.2911e-03, 3.2911e-03, 3.2911e-03,  ...,        nan,
              nan,        nan],
      [9.1167e-06, 9.1167e-06, 9.1167e-06,  ...,        nan,
              nan, 9.1167e-06],
      ...,
      [3.4284e-03,        nan, 3.4284e-03,  ..., 3.4284e-03,
       3.4284e-03, 3.4284e-03],
      [       nan, 1.7568e-03,        nan,  ..., 1.7568e-03,
       1.7568e-03, 1.7568e-03],
      [1.5258e-06,        nan,        nan,  ..., 1.5258e-06,
       1.5258e-06, 1.5258e-06]],

     [[5.7720e-04, 5.7720e-04, 5.7720e-04,  ...,        nan,
              nan,        nan],
      [1.2828e-05, 1.2828e-05, 1.2828e-05,  ...,        nan,
              nan,        nan],
      [1.6647e-03, 1.6647e-03, 1.6647e-03,  ...,        nan,
              nan, 1.6647e-03],
      ...,
      [1.0735e-03,        nan, 1.0735e-03,  ..., 1.0735e-03,
       1.0735e-03, 1.0735e-03],
      [       nan, 1.4540e-04,        nan,  ..., 1.4540e-04,
       1.4540e-04, 1.4540e-04],
      [1.1156e-03,        nan,        nan,  ..., 1.1156e-03,
       1.1156e-03, 1.1156e-03]],

     [[1.3076e-04, 1.3076e-04, 1.3076e-04,  ...,        nan,
              nan,        nan],
      [2.4632e-03, 2.4632e-03, 2.4632e-03,  ...,        nan,
              nan,        nan],
      [3.0567e-03, 3.0567e-03, 3.0567e-03,  ...,        nan,
              nan, 3.0567e-03],
      ...,
      [3.0679e-03,        nan, 3.0679e-03,  ..., 3.0679e-03,
       3.0679e-03, 3.0679e-03],
      [       nan, 2.7698e-03,        nan,  ..., 2.7698e-03,
       2.7698e-03, 2.7698e-03],
      [3.1502e-03,        nan,        nan,  ..., 3.1502e-03,
       3.1502e-03, 3.1502e-03]]]]

How can I avoid ‘nan’ values?

12

You’ll need to check all input Tensors, so check how input_unfold and weight_flat and perhaps even self.gamma are defined/computed. I’d assume it’s coming from input_unfold but check them all.

You can do this easily by usingtorch.autograd.detect_anomaly as a context manager (see more Automatic differentiation package - torch.autograd — PyTorch 2.1 documentation)

13

input_unfold and weight_flat are as given below.

input_unfold =  tensor([[[[0.0000, 0.0000, 0.0000,  ..., 0.9194, 0.0503, 0.9589],
          [0.0000, 0.0000, 0.0000,  ..., 0.0503, 0.9589, 0.6398],
          [0.0000, 0.0000, 0.0000,  ..., 0.9589, 0.6398, 0.0000],
          ...,
          [0.0000, 0.7830, 0.6963,  ..., 0.0000, 0.0000, 0.0000],
          [0.7830, 0.6963, 0.6662,  ..., 0.0000, 0.0000, 0.0000],
          [0.6963, 0.6662, 0.8015,  ..., 0.0000, 0.0000, 0.0000]]],


        [[[0.0000, 0.0000, 0.0000,  ..., 0.7274, 0.0721, 0.7137],
          [0.0000, 0.0000, 0.0000,  ..., 0.0721, 0.7137, 0.8258],
          [0.0000, 0.0000, 0.0000,  ..., 0.7137, 0.8258, 0.0000],
          ...,
          [0.0000, 0.1822, 0.6842,  ..., 0.0000, 0.0000, 0.0000],
          [0.1822, 0.6842, 0.2491,  ..., 0.0000, 0.0000, 0.0000],
          [0.6842, 0.2491, 0.4927,  ..., 0.0000, 0.0000, 0.0000]]]],
       device='cuda:0')
weight_flat =  tensor([[[ 0.0389],
         [-0.0168],
         [ 0.0396],
         ...,
         [ 0.0852],
         [ 0.0824],
         [ 0.0089]],

        [[-0.0027],
         [ 0.0509],
         [-0.0628],
         ...,
         [-0.0616],
         [-0.0226],
         [ 0.0380]],

        [[ 0.0298],
         [-0.0171],
         [ 0.0830],
         ...,
         [ 0.0128],
         [-0.0004],
         [ 0.0607]],

        ...,

        [[-0.0579],
         [-0.0283],
         [ 0.0257],
         ...,
         [-0.0370],
         [ 0.0655],
         [ 0.0790]],

        [[-0.0675],
         [ 0.0641],
         [ 0.0404],
         ...,
         [ 0.0852],
         [ 0.0168],
         [-0.0272]],

        [[ 0.0215],
         [-0.0780],
         [-0.0812],
         ...,
         [ 0.0270],
         [ 0.0331],
         [-0.0470]]], device='cuda:0', grad_fn=<ViewBackward0>)

input_unfold.shape =  torch.Size([2, 1, 135, 225])
weight_flat.shape =  torch.Size([32, 135, 1])
14

You’re taking the inverse of a negative number here, that’s why you’re getting NaNs

15 
Can you please suggest the way by which we can avoid negative numbers in the weights of a model?
16

output = torch.exp(self.gamma*(((input_unfold - weight_flat)**2).abs().sum(dim=2)).sqrt().mul(-1))