Computing the gradient with respect to a hidden layer's output

autograd
1

I’d like to compute the gradient with respect to a hidden layer’s output(pred_z in the example below). requires_grad is set to True but the gradient is still None. Why?

import torch
from torch.autograd import Variable

x_data = Variable(torch.Tensor([[1.0, 2.0], [2.0, 3.0], [3.0, 4.0]]))
y_data = Variable(torch.Tensor([[2.0], [4.0], [6.0]]))


class Model(torch.nn.Module):

    def __init__(self):
        super(Model, self).__init__()
        self.l1 = torch.nn.Linear(2, 5)
        self.l2 = torch.nn.Linear(5, 1)

    def forward(self, x):
        z = torch.nn.functional.relu(self.l1(x))
        y_pred = self.l2(z)
        return y_pred, z

# our model
our_model = Model()

criterion = torch.nn.MSELoss(size_average = False)
optimizer = torch.optim.SGD(our_model.parameters(), lr = 0.01)

for epoch in range(500):

    # Forward pass: Compute predicted y by passing
    # x to the model
    pred_y, pred_z = our_model(x_data)

    # Compute and print loss
    loss = criterion(pred_y, y_data)

    # Zero gradients, perform a backward pass,
    # and update the weights.
    optimizer.zero_grad()
    loss.backward()
    print('dL/dz', pred_z.grad)
    optimizer.step()
    print('epoch {}, loss {}'.format(epoch, loss.data[0]))

new_var = Variable(torch.Tensor([[4.0, 5.0]]))
pred_y = our_model(new_var)
print("predict (after training)", [4,5], our_model(new_var)[0].data[0])

I’d like to compute a gradient with of the loss w/ respect to z but I don’t actually want to apply it.

1 Like