Model is underfitting

Currently im building a CNN model to do classification with 6 classes, here’s my model, train & validation performance, and my training code.

Model

class Net(nn.Module):
  def __init__(self):
    super(Net, self).__init__()
    self.conv1 = nn.Conv2d(3, 64, 3)
    self.conv12 = nn.Conv2d(64, 64, 3)
    # self.bn1 = nn.BatchNorm2d(64)
    self.pool1 = nn.MaxPool2d(2,2)
    
    self.conv2 = nn.Conv2d(64, 128, 3)
    self.conv22 = nn.Conv2d(128, 128, 3)
    # self.bn2 = nn.BatchNorm2d(128)
    self.pool2 = nn.MaxPool2d(2,2)
    
    self.conv3 = nn.Conv2d(128, 256, 3)
    self.conv32 = nn.Conv2d(256, 256, 3)
    # self.bn3 = nn.BatchNorm2d(256)
    self.pool3 = nn.MaxPool2d(2,2)
    
    self.conv4 = nn.Conv2d(256, 512, 3)
    self.conv42 = nn.Conv2d(512, 512, 3)
    # self.bn4 = nn.BatchNorm2d(512)
    self.pool4 = nn.MaxPool2d(2,2)
    
    self.fc1 = nn.Linear(512*12*12, 128)
    # self.do1 = nn.Dropout(p=0.2)
    self.fc2 = nn.Linear(128, 6)
  def forward(self, x):
    x = self.pool1(F.relu(self.conv12(self.conv1(x))))
    x = self.pool2(F.relu(self.conv22(self.conv2(x))))
    x = self.pool3(F.relu(self.conv32(self.conv3(x))))
    x = self.pool4(F.relu(self.conv42(self.conv4(x))))
    # print(x.shape)
    x = x.view(-1, 512*12*12)
    x = F.relu(self.fc1(x))
    # x = self.do1(x)
    x = self.fc2(x)
    return x

net = Net()
net.to(device)

Training and validation code

def trainModel(model, criterion, optimizer, scheduler=None, epochs=10):
  T = time.time()
  epochLog = []
  tLossLog = []
  tAccLog = []
  vLossLog = []
  vAccLog = []

  
  # bestModel = copy.deepcopy(model.state_dict())
  bestAcc = .0

  for epoch in range(epochs):
    print(f'Epoch {epoch+1}/{epochs}')
    print('-' * 10)
    for phase in ['train', 'val']:
      tEpoch = time.time()
      if phase =='train':
        model.train()
      else:
        model.eval()
      
      runningLoss = .0
      runningCorrect = 0

      for inputs, labels in loader[phase]:
        inputs = inputs.to(device)
        # labels = torch.reshape(labels, (-1,))
        labels = labels.to(device)

        optimizer.zero_grad()

        with torch.set_grad_enabled(phase == 'train'):
          outputs = model(inputs)
          _, pred = torch.max(outputs, 1)
          # print(outputs.size())
          # print(labels.size())
          loss = criterion(outputs, labels)

          if phase == 'train':
            loss.backward()
            optimizer.step()
        
        runningLoss += loss.item() * inputs.size(0)
        runningCorrect += torch.sum(pred == labels.data)

      if phase == 'train' and scheduler:
        scheduler.step()

      epochLoss = runningLoss / size[phase]
      epochAcc = runningCorrect.double() / size[phase]

      epochLog.append(epoch)
      if phase == 'train':
        tAccLog.append(epochAcc)
        tLossLog.append(epochLoss)
      elif phase == 'val':
        vAccLog.append(epochAcc)
        vLossLog.append(epochLoss)

      elapsedEpoch = time.time()-tEpoch
      print(f'time per {phase} epoch : {elapsedEpoch//60:.0f}m.{elapsedEpoch%60:.0f}s, Loss : {epochLoss:.4f}, Acc : {epochAcc:.4f}%')

      if phase=='val' and epochAcc>bestAcc:
        bestAcc=epochAcc
        # bestModel = copy.deepcopy(model.state_dict())
      print()

  timeElapsed = time.time() - T
  print(f'Training complete in {timeElapsed//60:.0f}m {timeElapsed%60:.0f}s')
  print(f'best accuracy in : {bestAcc:.4f}%')

  return (epochLog, tAccLog, tLossLog, vAccLog, vLossLog)

and here’s the performance plot

as you can see in the performance plot, training and validation accuracy is bad and i think this is underfitting. I tried batch normalization and dropout but it’s not changing too much. Maybe anyone have some solution to handle this kind of problems?
(im using Adam as optimizer with 0.09 learning rate. 552 images on the dataset)

Hello Dicky,
for me it looks like your model is not learning a lot since random guessing would be at 1/6 ~ 16,7 %. I suppose that from what you have posted it is not obvious why one should/could conclude that your model is underfitting. Furthermore, if so, Dropout would not be appropriate because it mitigates the effect of Overfitting: Remeber: While using Dropout a certain percentage of the neurons is (randomly) disabled.
This “temporarily decreases the model complexity”, so that the model will not fit to a structure that is only present in the training data.
Rather check whether you are using the right loss function, at first - which one did you pick? CrossEntropy is probably suitable. Then check for the learning rate. Or maybe you need to normalize your data at first, if this did not already happen. If you still think your model is underfitting, you can always add a few layers (and/or increase the layer width).