Loss function error when backward

import cmath
import torch as t
import numpy as np
from torch.autograd import Function
from torch.nn import functional as F
cmath.sqrt(-1)

class loss(Function):
@staticmethod
def forward(ctx,x,INPUT):

    batch_size = x.shape[0]
    X = x.detach().numpy()
    input = INPUT.detach().numpy()
    Loss = 0
    for i in range(batch_size):
        t_R_r = input[i,0:4]
        R_r = t_R_r[np.newaxis,:]
        t_R_i = input[i,4:8]
        R_i = t_R_i[np.newaxis,:]
        t_H_r = input[i,8:12]
        H_r = t_H_r[np.newaxis,:]
        t_H_i = input[i,12:16]
        H_i = t_H_i[np.newaxis,:]

        t_T_r = input[i, 16:32]
        T_r = t_T_r.reshape(4,4)
        t_T_i = input[i, 32:48]
        T_i = t_T_i.reshape(4,4)
        
        R = np.concatenate((R_r, R_i), axis=1)
        H = np.concatenate((H_r, H_i), axis=1)


        temp_t1 = np.concatenate((T_r,T_i),axis=1)
        temp_t2 = np.concatenate((-T_i,T_r),axis=1)
        T = np.concatenate((temp_t1,temp_t2),axis=0)
        phi_r = np.zeros((4,4))
        row, col = np.diag_indices(4)
        phi_r[row,col] = X[i,0:4]
        phi_i = np.zeros((4, 4))
        row, col = np.diag_indices(4)
        phi_i[row, col] = 1 - np.power(X[i, 0:4],2)
    
        temp_phi1 = np.concatenate((phi_r,phi_i),axis=1)
        temp_phi2 = np.concatenate((-phi_i, phi_r), axis=1)
        phi = np.concatenate((temp_phi1,temp_phi2),axis=0)

        temp1 = np.matmul(R,phi)

        temp2 = np.matmul(temp1,T)  # error
        H_hat = H + temp2

        t_Q_r = np.zeros((4,4))
        t_Q_r[np.triu_indices(4,1)] = X[i,4:10]
        Q_r = t_Q_r + t_Q_r.T
        row,col = np.diag_indices(4)
        Q_r[row,col] = X[i,10:14]
        Q_i = np.zeros((4,4))
        Q_i[np.triu_indices(4,1)] = X[i,14:20]
        Q_i = Q_i - Q_i.T

        temp_Q1 = np.concatenate((Q_r,Q_i),axis=1)
        temp_Q2 = np.concatenate((-Q_i,Q_r),axis=1)
        Q = np.concatenate((temp_Q1,temp_Q2),axis=0)

        t_H_hat_r = H_hat[0,0:4]
        H_hat_r = t_H_hat_r[np.newaxis,:]
        t_H_hat_i= H_hat[0,4:8]
        H_hat_i = t_H_hat_i[np.newaxis,:]

        temp_H1 = np.concatenate((-H_hat_i.T,H_hat_r.T),axis=0)
        H_hat_H = np.concatenate((H_hat.T,temp_H1),axis=1)
        temp_result1 = np.matmul(H_hat,Q)
        temp_result2 = np.matmul(temp_result1,H_hat_H)
        
        Loss += np.log10(1+temp_result2[0][0])
    Loss = t.from_numpy(np.array(Loss / batch_size))
    return Loss
@staticmethod
def backward(ctx,grad_output):
    print('gradient')
    return grad_output

def criterion(output,input):
return loss.apply(output,input)

Traceback (most recent call last):
File “/Users/mrfang/channel_capacity/training.py”, line 24, in
loss.backward()
File “/Users/mrfang/anaconda3/lib/python3.6/site-packages/torch/tensor.py”, line 150, in backward
torch.autograd.backward(self, gradient, retain_graph, create_graph)
File “/Users/mrfang/anaconda3/lib/python3.6/site-packages/torch/autograd/init.py”, line 99, in backward
allow_unreachable=True) # allow_unreachable flag
RuntimeError: function lossBackward returned an incorrect number of gradients (expected 2, got 1)

How could I fix it. Thanks very much

Hi,

Your forward function takes 2 inputs: x and INPUT.
So the backward function should return 2 things: the gradients for x and the gradients for INPUT.

1 Like

Hi @ptrblck , @albanD
I have problem as

I have tried to make custom loss function of that tripletloss. forward() was working but when calling loss.backward()
it is giving alot of errors, I tried to debug it and as Alban said forward with two inputs returns 2 things, but my mind still cant comprehent what to do with that. Sorry, if i was annonying.

Also in custom autograd after getting two vectors v1 and v2 , I detached from Tensor and tried to do operations in numpy and at last converted numpy and returned torch tensor. .apply(v1,v2) . As that given autograd tutorial

    def forward(ctx, input):
        """
        In the forward pass we receive a Tensor containing the input and return
        a Tensor containing the output. ctx is a context object that can be used
        to stash information for backward computation. You can cache arbitrary
        objects for use in the backward pass using the ctx.save_for_backward method.
        """
        ctx.save_for_backward(input)
        return input.clamp(min=0)

input was directly fed but my case is I have done numpy operations on it,

I am also confused what should i used for ctx.save_for_backward() tried to use both last converted torch tensor and also v1 and v2 but I am not getting. I am also confused what things should be done in backward in my case.
Thank you for your help .

My custom autograd loss code is below

class TripletLoss(torch.autograd.Function):
    @staticmethod
    def forward(ctx, v1, v2, margin=0.25):
        
        scores = np.dot(v1.detach().numpy(), v2.detach().numpy().T)
        batch_size = len(scores)
        positive = np.diag(scores) # the positive ones (duplicates)
        negative_without_positive = scores - 2.0 * np.identity(batch_size)
        closest_negative = negative_without_positive.max(axis=1)
        negative_zero_on_duplicate =  scores * (1.0 - np.eye(batch_size))
        mean_negative = np.sum(negative_zero_on_duplicate, axis=1) / (batch_size - 1)
        triplet_loss1 = torch.Tensor(np.maximum(0.0, margin - positive + closest_negative))
        triplet_loss2 = torch.Tensor(np.maximum(0.0, margin - positive + mean_negative))
        triplet_loss = torch.mean(triplet_loss1 + triplet_loss2)
        triplet_loss.requires_grad=True
        ctx.save_for_backward(triplet_loss)
        return triplet_loss
    
    @staticmethod
    def backward(ctx, grad_output):
        input = ctx.saved_tensors
        grad_input = grad_output.clone()
        grad_input[v1<0] = 0
        grad_input[v2<0] = 0
        return grad_input        

Hi,

Your forward takes 3 arguments here so the backward should return 3 things: the gradient for v1, v2 and margin.
Since margin is a plain number, you cannot have gradients for it and you should return None. So the backward return statement should be return grad_v1, grad_v2, None.

You should use save_for_backward() if you need to save any input or output of the forward for the backward.

1 Like