Validation accuracy not improving in this basic binary classification task

i am new to pytorch and practical deep learning i want to know why my validation accuracy is not going above 80% with my custom cnn model is there any way to improve that?

Here is my code-
import torch
import torch.nn as nn
import torch.optim as optim
from torch.optim import lr_scheduler
import numpy as np
import torchvision
from torchvision import datasets, models,transforms
import matplotlib.pyplot as plt
import time
import os
import copy

  device=torch.device("cuda" if torch.cuda.is_available() else "cpu" )
    
  mean=np.array([0.485,0.456,0.406])
  std=np.array([0.229,0.224,0.225])
    
  data_transforms={
  "train": transforms.Compose([
    transforms.RandomResizedCrop(224),
    transforms.RandomHorizontalFlip(),
    transforms.ToTensor(),
    transforms.RandomRotation(20),
    transforms.Normalize(mean,std)
    
  ]),
  "val": transforms.Compose([
    transforms.Resize(256),
    transforms.CenterCrop(224),
    transforms.ToTensor(),
    transforms.Normalize(mean,std)
  ]),
  }
    
  # import data
    
  data_dir="C:\\torch files\\data\\hymenoptera_data"
  sets=['train','val']
  image_datasets={x: datasets.ImageFolder(os.path.join(data_dir,x),
                                    data_transforms[x])
             for x in ['train', 'val']}
    
  dataloaders = {x: torch.utils.data.DataLoader(image_datasets[x],batch_size=8,shuffle=True)
             for x in ['train','val']}
    
  dataset_sizes = {x:len(image_datasets[x]) for x in ['train', 'val']}
  class_names = image_datasets['train'].classes
  print(class_names)
    
    
  class CustomCNN(nn.Module):
  def __init__(self,num_classes=2):
    super(CustomCNN,self).__init__()
    self.features=nn.Sequential(
        nn.Conv2d(3,64,7,2,3),
        nn.BatchNorm2d(64),
        nn.ReLU(inplace=True),
        nn.MaxPool2d(3,2,1),
    
        nn.Conv2d(64,128,3,1,1),
        nn.BatchNorm2d(128),
        nn.ReLU(inplace=True),
        nn.MaxPool2d(2,2),
    
        nn.Conv2d(128,256,3,1,1),
        nn.BatchNorm2d(256),
        nn.ReLU(inplace=True),
        nn.MaxPool2d(2,2),
    
        nn.Conv2d(256,512,3,1,1),
        nn.BatchNorm2d(512),
        nn.ReLU(inplace=True),
        nn.MaxPool2d(2,2),
    
        nn.Conv2d(512,1024,3,1,1),
        nn.BatchNorm2d(1024),
        nn.ReLU(inplace=True),
        nn.MaxPool2d(2,2),
    
    )
    self.Classifier=nn.Sequential(
        nn.Dropout(),
        nn.Linear(9216,4096),
        nn.ReLU(inplace=True),
        nn.Dropout(p=0.5),
        nn.Linear(4096,num_classes)
    )
    
  def forward(self,x):
    x=self.features(x)
    x=x.view(x.size(0),-1)
    x=self.Classifier(x)
    
    return x 
    
  model=CustomCNN(num_classes=len(class_names))
  model=model.to(device)
    
  criterion=nn.CrossEntropyLoss()
  optimizer=optim.Adam(model.parameters(),lr=0.0001,weight_decay=1e-5)
  scheduler=lr_scheduler.ReduceLROnPlateau(optimizer,mode='min',factor=0.1,patience=5,verbose=True)
    
  def train_model(model,criterion,optimizer,scheduler,num_epochs):
  since = time.time()
    
    
  best_model_wts = copy.deepcopy(model.state_dict())
  best_acc = 0.0
    
    
  for epoch in range(num_epochs):
    print(f"Epoch {epoch}/{num_epochs-1}")
    print('-' * 10)
    
    #Each epoch hasa training and validation phase
    for phase in ['train','val']:
        if phase =="train":
            model.train()  #set model to training mode
        else:
            model.eval()   #set model to evaluate mode
    
        running_loss = 0.0
        running_corrects = 0
    
        #iterate over data
        for inputs,labels in dataloaders[phase]:
            inputs = inputs.to(device)
            labels = labels.to(device)
    
            #forward
            #track history if onlyin train
            with torch.set_grad_enabled(phase == 'train'):
                outputs = model(inputs)
                _, preds = torch.max(outputs,1)
                loss = criterion(outputs, labels)
    
                #backward + optimize only if in traing phase
                if phase == 'train':
                    optimizer.zero_grad()
                    loss.backward()
                    optimizer.step()
    
            #statistics
            running_loss += loss.item() * inputs.size(0)
            running_corrects += torch.sum(preds == labels)
    
        if phase == 'Train':
            scheduler.step()
    
        epoch_loss = running_loss / dataset_sizes[phase]
        epoch_acc = running_corrects.double() / dataset_sizes[phase]
    
        print(f"{phase} Loss: {epoch_loss:.4f} Acc: {epoch_acc:.4f}")
    
        # deep copy of the model
        if phase == 'val' and epoch_acc > best_acc:
            best_acc = epoch_acc
            best_model_wts = copy.deepcopy(model.state_dict())
    
    print()
    
  time_elapsed = time.time() - since
  print((f"Training complete in {time_elapsed//60:.0f}m {time_elapsed%60:.0f}s"))
  print(f"Best val Acc: {best_acc:4f}")
    
  # load best model weights
  model.load_state_dict(best_model_wts)
  return model
    
    
  model=train_model(model,criterion,optimizer,scheduler,num_epochs=48)

Have you tried an LR warmup for Adam?

I spoke about the validation accuracy.
What about the training accuracy? Does it increase? Does it remain constant? Do they diverge?
You need to consider both. Not only one.

It is well known that the Adam algorithm cannot improve validation accuracy when increasing learning rate. Have you tried the RAdam algorithm in PyTorch?

Training accuracy oscillate at around same 77 to 84% accuracy

No I did not try it can you explain how it would improve accuracy

The first update step of Adam is significant large and not good in direction. So, the model may be trapped in a local minimum in a very early stage. The ML practitioners often use Adam together with warmup.

Ok thanks i will try that.

I overcame problems with learning rate by using this learning rate schedule:
sched = lr_scheduler.OneCycleLR(optim, max_lr=0.1, steps_per_epoch=len(train_loader),
epochs=n_epochs)