I have a simple model with is not learning

Hi there!

I am trying to training this simple model

class Classifier(nn.Module):
def init(self):
super().init()
self.fc1 = nn.Linear(784, 256)
self.fc2 = nn.Linear(256, 128)
self.fc3 = nn.Linear(128, 64)
self.out = nn.Linear(64, 10)
self.dropout = nn.Dropout(0.2)

def forward(self, t):
    t = t.view(t.shape[0], -1) # Flattern features
    t = self.dropout(F.relu(self.fc1(t)))
    t = self.dropout(F.relu(self.fc2(t)))
    t = self.dropout(F.relu(self.fc3(t)))
    t = self.out(t)
    
    return t

In Fashion dataset. Here is my code for training:

model = Classifier()
criterion = torch.nn.CrossEntropyLoss()
optimizer = torch.optim.Adam(model.parameters(), lr =.003)

epochs = 30

validation_losses = []
validation_accuracies = []

training_loss = []
training_accuracies = []

for e in range(epochs):

train_loss = .0
train_acc = .0
model.train()
for images, labels in trainloader:
    out = model(images)
    loss = criterion(out, labels)

    # Acc training
    _, pred = torch.max(out, dim = 1)
    equals = pred == labels
    train_acc += torch.mean(equals.type(torch.FloatTensor)).item()

    # Loss training
    train_loss += criterion(out, labels).item()
    
    # Update weights
    loss.backward()
    optimizer.step()

else:
    test_acc = .0
    test_loss = .0
    with torch.no_grad():
        model.eval()
        for images, labels in testloader:
            out = model(images)
            loss = criterion(out, labels)

            # Acc training
            _, pred = torch.max(out, dim = 1)
            equals = pred == labels
            test_acc += torch.mean(equals.type(torch.FloatTensor)).item()

            # Loss training
            test_loss += criterion(out, labels).item()
            
train_losses.append(train_loss/len(trainloader))
training_accuracies.append(train_acc/len(trainloader))
validation_losses.append(test_loss/len(testloader))
validation_accuracies.append(test_acc/len(testloader))
        
print("Epoch: {}/{}".format(e+1, epochs),
     "Training accuracy: {:.3f}".format(train_acc/len(trainloader)),
     "Training loss: {:.3f}".format(train_loss/len(trainloader)),
     "Test accuracy: {:.3f}".format(test_acc/len(testloader)),
     "Test loss: {:.3f}".format(test_loss/len(testloader)))

And this are my logs:

Epoch: 1/30 Training accuracy: 0.127 Training loss: 2.442 Test accuracy: 0.100 Test loss: 2.346
Epoch: 2/30 Training accuracy: 0.099 Training loss: 311.148 Test accuracy: 0.100 Test loss: 2.404
Epoch: 3/30 Training accuracy: 0.101 Training loss: 320.982 Test accuracy: 0.100 Test loss: 2.386
Epoch: 4/30 Training accuracy: 0.100 Training loss: 1291.109 Test accuracy: 0.100 Test loss: 2.457
Epoch: 5/30 Training accuracy: 0.098 Training loss: 2.422 Test accuracy: 0.100 Test loss: 2.446
Epoch: 6/30 Training accuracy: 0.100 Training loss: 15.440 Test accuracy: 0.100 Test loss: 2.429
Epoch: 7/30 Training accuracy: 0.099 Training loss: 2.459 Test accuracy: 0.100 Test loss: 2.455

I am starting with pytorch coding so i am sure something must be wrong, look the score.
Could you help me?
Ty!

You have forgotten to zero out the gradients, so that they will be accumulated in the .grad attributes of all parameters.
If you add optimizer.zero_grad() into your training loop, the training starts to converge:

    for images, labels in trainloader:
        optimizer.zero_grad()
        out = model(images)
        loss = criterion(out, labels)
       # Acc training
        _, pred = torch.max(out, dim = 1)
        equals = pred == labels
        train_acc += torch.mean(equals.type(torch.FloatTensor)).item()
    
        # Loss training
        train_loss += criterion(out, labels).item()
        
        # Update weights
        loss.backward()
        optimizer.step()
1 Like