Why .grad is None?

I cannot figure out why .grad is None while the weights have required_grad = True?

class LeNet(nn.Module):
    def __init__(self):
        super(LeNet, self).__init__()
        self.conv1 = nn.Conv2d(3, 6, 5)
        self.conv2 = nn.Conv2d(6, 16, 5)
        self.fc1   = nn.Linear(16*5*5, 120)
        self.fc2   = nn.Linear(120, 84)
        self.fc3   = nn.Linear(84, 10)

    def forward(self, x):
        out = F.relu(self.conv1(x))
        out = F.max_pool2d(out, 2)
        out = F.relu(self.conv2(out))
        out = F.max_pool2d(out, 2)
        out = out.view(out.size(0), -1)
        out = F.relu(self.fc1(out))
        out = F.relu(self.fc2(out))
        out = self.fc3(out)
        return out


def train_cnn(data, lr=0.01, epochs=100, momentum=0.9, weight_decay=0):
    model = LeNet()
    model.cuda()
    criterion = torch.nn.CrossEntropyLoss()

    model.train()
    optimizer = torch.optim.SGD(model.parameters(), lr=lr, momentum=momentum, weight_decay=weight_decay)

    for e in range(epochs):
        for data in trainloader:
            x, y = data
            x = x.cuda()
            y = y.cuda()

            outputs = model(x)
            loss = criterion(outputs, y)
            optimizer.zero_grad()
            loss.backward()

            print(model.conv1.weight[0].requires_grad)
            print(model.conv1.weight[0].grad)
            exit()

The output of this is

True
None

Hi,

The problem is that you index the weights when you do [1]. So you get a new Tensor that does not have a .grad field.
model.convos[1].weight.grad should not be None right?

2 Likes

Oh I see, grad is stored for the entire tensor, and not it’s inner dimensions, sorry, silly mistake.
I wanted to get a grad for a specific filter, but I should have sliced the grad itself.

1 Like