Alright , thanks for the explanation 
so how is backward method inherited for custom functions? In case I have something more complicated like this:
from torch.nn.modules.loss import _Loss
class GaussianLoss(_Loss):
def __init__(self, sigma=None, abs_loss=None):
super(GaussianLoss, self).__init__()
assert sigma is not None
assert abs_loss is not None
self.sigma=sigma
def forward(self, d):
gaussian_val = torch.exp((-d).div(self.sigma))
return gaussian_val
In other words, does the autograd know how to take derivative of exp(-d/sigma) wrt d (which is -d/sigma exp(-d/sigma) btw) ?
Yes, Autograd will be able to backpropagate all PyTorch functions, if they have a valid .grad_fn (the majority of PyTorch ops has it unless the operation is not differentiable).
1 Like
Thanks, so as long as I’m using torch.exp and _Loss base class, it should work fine?
You don’t need to use _Loss as the base class, but can use nn.Module instead.
Is this because ‘reduce’ was deprecated in favor of ‘reduction’?
You are not using reduce or reduction anywhere in your code and just store the sigma value in the class, so I’m unsure why you would need the _Loss base class.
I guess I just assumed it’d a loss function’s base class that they all must inherit, since it’s what I saw in torch.nn.modules.loss
You could still derive from _Loss, if you want to set the reduction parameter using the legacy checks as seen here.
This would also mean that you would call:
super(MSELoss, self).__init__(size_average, reduce, reduction)
in the __init__ method and could use self.reduction in the forward.
However, if you don’t need the reduction argument, you can just use nn.Module.
In fact, you could even use a plain Object class, as no parameters or buffers are registered in your custom loss function.
2 Likes
Hi,@ptrblck,ptrblck,could you answer some questions about custom loss funtion ? I use a autoencoder to recontruct a signal,input:x,output:y,autoencoder is made by CNN,I wanted to change the weights of the autoencoder,that mean I must change the weights in the autoencoder.parameters() .I made a custom loss function using numpy and scipy ,but I don’t know how to write backward function about the weight of autoencoder .Here is my loss function.If you know how to write it,please tell me,it is great matter to me.Thank you!
class autoencoderlossFuction(Function):
def forward(self, x, y, M, T):
x = x.detach().numpy()
x_std = np.std(x)
x_mean = np.mean(x)
x = (x-x_mean)/x_std
y = y.detach().numpy()
y_std = np.std(y)
y_mean = np.mean(y)
y = (y-y_mean)/y_std
N = len(x)
XmT = np.zeros((N, M+1))
YmT = np.zeros((N, M+1))
for m in range(M+1):
XmT[m*T:,m] = x[0:N-m*T]
for m in range(M+1):
YmT[m*T:,m] = y[0:N-m*T]
self.save_for_backward(x, y)
ckx = np.sum(np.multiply(np.prod(XmT,1),np.prod(XmT,1)))/(np.sum(np.multiply(x,x)))**(M+1)
cky = np.sum(np.multiply(np.prod(YmT,1),np.prod(YmT,1)))/(np.sum(np.multiply(y,y)))**(M+1)
ckloss = 1/(cky-ckx)**2
x = torch.tensor(x)
y = torch.tensor(y)
loss = torch.Tensor(ckloss)
return loss
#question ???
def backward(self, grad_loss):
grad_output = grad_output.detach().numpy()
x, y = self.saved_tensors
x = x.numpy()
y = y.numpy()
grad_input =
return torch.Tensor(grad_input)
class autoencoderloss(nn.Module):
def init(self,M,T):
super(autoencoderloss,self).init()
self.M = M
self.T = T
return
def forward(self,x,y):
output = autoencoderlossFuction(x, y, self.M, self.T)
return output
Based on the provided code snippet I think you could replace all numpy operations with their PyTorch equivalent, which would automatically create the backward pass for you so that you don’t have to manually implement it.
I didn’t offer all the code,some codes are using scipy ,so I must wirte backward function
I delete all the code that using scipy and replace all numpy operations with torch like you said ,there are still errors:‘RuntimeError: element 0 of tensors does not require grad and does not have a grad_fn’,I don’t know why
Could you post an executable code snippet to reproduce this issue, please? You might be using non-differentiable operations, but it would be helpful to debug the code to be sure.
I send the main code to you in message,check it out if you have time,Thank you!
Could you post it here, please, so that also others could have a look? 
Oh,sorry,forgive my selfish,knowledge should share to everyone!Here is all code.
import torch.nn as nn
from torch.autograd import Variable
from torch import optim
import numpy as np
import scipy.io as sio
from scipy import fftpack
from scipy.fftpack import fft
import visdom
class AEGenerator(nn.Module):
def init(self,judge=True):
super(AEGenerator,self).init()
self.encode=nn.Sequential(
nn.Conv1d(in_channels=1, out_channels=16,
kernel_size=32, stride=2, padding=15),
nn.PReLU(),
nn.Conv1d(16, 32, 32, 2, 15),
nn.PReLU(),
nn.Conv1d(32, 32, 32, 2, 15),
nn.PReLU(),
nn.Conv1d(32, 64, 32, 2, 15),
nn.PReLU(),
nn.Conv1d(64, 64, 32, 2, 15),
nn.PReLU(),
nn.Conv1d(64, 128, 32, 2, 15),
nn.PReLU(),
nn.Conv1d(128, 128, 32, 2, 15),
nn.PReLU(),
nn.Conv1d(128, 256, 32, 2, 15),
nn.PReLU(),
nn.Conv1d(256, 256, 32, 2, 15),
nn.PReLU(),
nn.Conv1d(256, 512, 32, 2, 15),
nn.PReLU(),
)
self.decode=nn.Sequential(
nn.ConvTranspose1d(in_channels=512, out_channels=256,
kernel_size=32, stride=2, padding=15),#199
nn.PReLU(),
nn.ConvTranspose1d(256, 256, 32, 2, 15),
nn.PReLU(),
nn.ConvTranspose1d(256, 128, 32, 2, 15),
nn.PReLU(),
nn.ConvTranspose1d(128, 128, 32, 2, 15),
nn.PReLU(),
nn.ConvTranspose1d(128, 64, 32, 2, 15),
nn.PReLU(),
nn.ConvTranspose1d(64, 64, 32, 2, 15),
nn.PReLU(),
nn.ConvTranspose1d(64, 32, 32, 2, 15),
nn.PReLU(),
nn.ConvTranspose1d(32, 32, 32, 2, 15),
nn.PReLU(),
nn.ConvTranspose1d(32, 16, 32, 2, 15),
nn.PReLU(),
nn.ConvTranspose1d(16, 1, 32, 2, 15),
nn.Tanh(),
)
def forward(self,x,judge):
enOutputs=self.encode(x)
outputs=self.decode(enOutputs)
if judge:
return outputs
else:
return enOutputs
class autoencoderloss(nn.Module):
def init(self,M,T):
super(autoencoderloss, self).init()
self.M = M
self.T = T
return
def forward(self, x, y):
x = x.detach()
x_std = torch.std(x)
x_mean = torch.mean(x)
x = (x-x_mean)/x_std
y = y.detach()
y_std = torch.std(y)
y_mean = torch.mean(y)
y = (y-y_mean)/y_std
T = round(self.T)
N = len(x)
XmT = torch.zeros((N, self.M+1))
YmT = torch.zeros((N, self.M+1))
for m in range(self.M+1):
XmT[m*T:,m] = x[0:N-m*T]
for m in range(self.M+1):
YmT[m*T:,m] = y[0:N-m*T]
ckx = torch.sum(torch.mul(torch.prod(XmT,1),torch.prod(XmT,1)))/(torch.sum(torch.mul(x,x)))**(self.M+1)
cky = torch.sum(torch.mul(torch.prod(YmT,1),torch.prod(YmT,1)))/(torch.sum(torch.mul(y,y)))**(self.M+1)
ckloss = (cky-ckx)**2
xy = torch.tensor(1)
loss = torch.div(xy.float(), ckloss.float())
return loss
viz = visdom.Visdom(env=‘aemckd’)
x = sio.loadmat(‘D:/研一/算法/SEGAN_triandata/xinxinnati982.mat’)
x = x[‘x’]
x = np.array(x)
X = np.squeeze(x)
X_std = np.std(X)
X_mean = np.mean(X)
X = (X-X_mean)/X_std
X = X[0:8192]
signal_length = len(X)
x = X.reshape(-1,1)
x = x.T
if name == ‘main’:
batch_size = 8192
num_batch = signal_length / batch_size
num_epochs = 800
autoencoder = AEGenerator()
if torch.cuda.is_available():
autoencoder.cuda()
print("#autoencoder parameters:",sum(param.numel() for param in autoencoder.parameters()))
optimizer = optim.RMSprop(autoencoder.parameters(),lr = 1e-30 )
XX = []
loss_history = []
for epoch in range(num_epochs):
for batch in range(int(num_batch)):
x_ = x[batch_size * batch:batch_size * (batch + 1)]
x_t = torch.from_numpy(x_).type(torch.FloatTensor).unsqueeze(0)
if torch.cuda.is_available():
x_t = x_t.cuda()
x_t = Variable(x_t)
autoencoder.train()
optimizer.zero_grad()
# autoencoder.zero_grad()
y = autoencoder(x_t,judge=True)
y = Variable(y)
X = x_t.data.cpu().squeeze(0).squeeze(0)
Y = y.data.cpu().squeeze(0).squeeze(0)
criterion = MCKDloss(3, 96)
# criterion = nn.MSELoss()
loss = criterion(Y, X)
# loss = Variable(loss, requires_grad=True) #????
loss.backward()
optimizer.step()
XX.append(epoch)
loss_history.append(loss.item())
yy = y.detach().numpy().squeeze(0).squeeze(0)
yy_std = np.std(yy)
yy_mean = np.mean(yy)
yy = (yy-yy_mean)/yy_std
Loss = loss.detach().numpy()
viz.line(X=XX, Y=loss_history, win='AE', opts={"title":'AE--',
"xlabel":'Epochs',"ylabel":'Value'})
if (epoch+1) % 5 == 0:
print ('Epoch [{}/{}],Loss: {:.4f}'.format(epoch+1,num_epochs,loss.item()))Thanks for the code. In your autoencoderloss you are detaching the input tensor explicitly, via x = x.detach() so that the backward call would stop at this point (and might also raise the error you are seeing now, if no available parameters were used in the custom loss function). Could you remove this line of code and rerun it?
Thank for you reply,gentleman.I delete all the code that using detach,same error happen.Also,I change the loss function to nn.MSEloss,same error happen again.I am confuse 
I know where is wrong:X = x_t.data.cpu().squeeze(0).squeeze(0).X become tensor ,not Variable anymore,so it can’t backward.Thank for your patient reply.