Error is increasing in Single Object detection

Hi, I am trying to detect the largest item(Single object detection) and I have used resnet34.
and I am using L1 loss for bounding box and Cross-Entropy for classification but my loss in training is quite low around 20-30 but invalidation the loss is above 200. And Accuracy is always 0

Any suggestions on how can I solve this problem.

`Custom head for resnet34 -



custom_fc = nn.Sequential(
Flatten(),
nn.ReLU(),
nn.Dropout(0.5),
nn.Linear(512*7*7, 256),
nn.ReLU(),
nn.BatchNorm1d(256),
nn.Dropout(0.5),
nn.Linear(256, 4 + 20))

Complete Training Loop:

n_epochs = 100
lr = 0.001
bb_optimizer = torch.optim.Adam(model_bb.parameters(), lr = lr)
val_loss_min = np.Inf

total_train_loss = 0
train_total_reg_loss = 0
train_total_cls_loss = 0

total_val_loss = 0
total_val_reg_loss = 0
total_val_cls_loss = 0

cls_crct = 0
total_target = 0

step = 0
for e in range(n_epochs):
    running_loss = 0
    running_reg_loss = 0
    running_cls_loss = 0
    
    model_bb.train()
    
    for img, label in train_dl:
        
        
        if trn_gpu:
          img, label[0],label[1] = img.cuda(),label[0].cuda(),label[1].cuda()
        
        bb_optimizer.zero_grad()
        pred = model_bb(img)
        
        loss_t = combine_loss(predictions= pred, correct_label= label)
        bb_loss_t = reg_loss(predictions= pred, correct_label= label)
        img_loss_t = cls_loss(predictions= pred, correct_label= label)
               
        loss_t.backward()
        bb_optimizer.step()
        
        step += 1   
        running_loss += loss_t.item()
        running_reg_loss += bb_loss_t.item()
        running_cls_loss += img_loss_t        
        
    train_total_loss = running_loss / len(train_dl)
    train_total_reg_loss = running_reg_loss / len(train_dl)
    train_total_cls_loss = running_cls_loss / len(train_dl)
    
    print("---------------------------------------------------TRAINING----------------------------------------")
    print("Epoch: [{}/{}]  \tTotal Loss: {:.3f}  \tL1 Loss: {:.3f}  \tClassification Loss: {:.3f}".format((e+1), 
                                                                                              n_epochs, 
                                                                                              train_total_loss,
                                                                                              train_total_reg_loss,
                                                                                              train_total_cls_loss))
    
    model_bb.eval()
    with torch.no_grad():
        r_val, r_reg_val, r_cls_val = 0,0,0
        
        for img, label in val_dl:
            img = img.to(device)
            label[0] = label[0].to(device)
            label[1] = label[1].to(device)
            
            pred = model_bb(img)
            
            loss_v = combine_loss(predictions= pred, correct_label= label)
            bb_loss_v = reg_loss(predictions= pred, correct_label= label)
            img_loss_v = cls_loss(predictions= pred, correct_label= label)
            
            r_val += loss_v.item()
            r_reg_val += bb_loss_v.item()
            r_cls_val += img_loss_v
            
            crct_label, total_label = correct_batch(predictions= pred, correct_label= label)
            
            cls_crct += crct_label
            total_target += total_label
    
    #--------Validation loss and accuracy
    
    total_val_loss = r_val / len(val_dl)
    total_val_reg_loss = r_reg_val / len(val_dl)
    total_val_cls_loss = r_cls_val / len(val_dl)
    
    accuracy = 100*(cls_crct/total_target)
       
    print("-------------------------------------------------EVALUATION------------------------------------")
    print("Epoch: [{}/{}]  \tTotal Loss: {:.3f}  \tL1 Loss: {:.3f}  \t Classification Loss: {:.3f}  \tAccuracy: {}".format((e+1),
                                                                                                                n_epochs,
                                                                                                                total_val_loss,
                                                                                                                total_val_reg_loss,
                                                                                                                total_val_cls_loss,
                                                                                                                accuracy))
    
    if total_val_loss < val_loss_min:
        print("Model Saving....")
        torch.save(model_bb.state_dict(),'drive/My Drive/files/model.pt')
        val_loss_min = total_val_loss