The validation set loss obtained from the loaded model weights is not the same as the validation loss from the training process


As shown above, the validation loss obtained from the loaded weights and the training loss obtained from the training process.A simplified version of the source code is shown below

mean_valid_loss = []
valid_y = [None]*3
for step,(X_batch, y_batch, dtw) in enumerate(val_dataloader):
    loss, y_hat = engine.valid(X_batch,y_batch,dtw)
    mean_valid_loss.append(loss)
    if step==0:
        valid_y[0] = y_batch
        valid_y[1] = y_hat[0]
        valid_y[2] = y_hat[1]
    else:
        valid_y[0] = np.concatenate([valid_y[0], y_batch],axis=0)
        valid_y[1] = np.concatenate([valid_y[1], y_hat[0]],axis=0)
        valid_y[2] = np.concatenate([valid_y[2], y_hat[1]],axis=0)
mean_valid_loss = np.mean(mean_valid_loss)
valid_coverage, valid_width = compute_coverage(valid_y[0], valid_y[2], valid_y[1])

  
if mean_valid_loss <= min_valid_loss:
    overfitting_epoch = 0
    min_valid_loss = mean_valid_loss
    torch.save(deepcopy(engine.model), save_dir + 'best_model.pth') 
else:
    overfitting_epoch+=1
if overfitting_epoch>=early_stopping:
    break
logger.info("EPOCH: {:3d} TRAIN LOSS: {:.4f} TRAIN UPPER LOSS: {:.4f} TRAIN LOWER LOSS: {:.4f} VALID LOSS: {:.4f} MIN VALID LOSS: {:.4f}".format(epoch, mean_train_loss,mean_train_upper_loss, mean_train_lower_loss,mean_valid_loss, min_valid_loss))
 
# check_point = torch.load(save_dir + 'best_model.pth')
engine.model = torch.load(save_dir + 'best_model.pth')

mean_valid_loss = []
valid_y = [None]*3
for step,(X_batch, y_batch, dtw) in enumerate(val_dataloader):
  loss, y_hat = engine.valid(X_batch,y_batch,dtw)
  mean_valid_loss.append(loss)
  if step==0:
      valid_y[0] = y_batch
      valid_y[1] = y_hat[0]
      valid_y[2] = y_hat[1]
  else:
      valid_y[0] = np.concatenate([valid_y[0], y_batch],axis=0)
      valid_y[1] = np.concatenate([valid_y[1], y_hat[0]],axis=0)
      valid_y[2] = np.concatenate([valid_y[2], y_hat[1]],axis=0)
mean_valid_loss = np.mean(mean_valid_loss)
print("After loading model valid loss:",mean_valid_loss)

model.eval() and torch.no_grad() have been added to engine.valid() and engine.test()