In code below output = w * x which is given for loss calculation. So after loss.backward(), change in weight should be " w = w - (lr * dL/dw) ". Now dL/dw = dL/d(output) * d(output)/dw= -1 * x = -4.
Putting this value it should update w = w - (lr * dL/dw) = 1 - (0.2* -4) = 1.8 but its 1.2 and update constant like 1.2,1.4.1.6 etc
import torch
import torch.nn as nn
class Net(torch.nn.Module):
def __init__(self):
super(Net, self).__init__()
self.w=torch.nn.Parameter(torch.ones(1))
def forward(self, x):
print("weight and its gradient is ",self.w,self.w.grad)
out = torch.mul(x,self.w)
return out
learning_rate = 0.2
model=Net()
criterion=torch.nn.L1Loss()
optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)
x=4*torch.ones((1),requires_grad=True)
target=torch.FloatTensor([36])
for i in range (1,3):
optimizer.zero_grad()
out=model(x)
loss= criterion(target,out)
print("output and loss is ",out,loss)
loss.backward(retain_graph=True)
optimizer.step()