As shown above, the validation loss obtained from the loaded weights and the training loss obtained from the training process.A simplified version of the source code is shown below
mean_valid_loss = []
valid_y = [None]*3
for step,(X_batch, y_batch, dtw) in enumerate(val_dataloader):
loss, y_hat = engine.valid(X_batch,y_batch,dtw)
mean_valid_loss.append(loss)
if step==0:
valid_y[0] = y_batch
valid_y[1] = y_hat[0]
valid_y[2] = y_hat[1]
else:
valid_y[0] = np.concatenate([valid_y[0], y_batch],axis=0)
valid_y[1] = np.concatenate([valid_y[1], y_hat[0]],axis=0)
valid_y[2] = np.concatenate([valid_y[2], y_hat[1]],axis=0)
mean_valid_loss = np.mean(mean_valid_loss)
valid_coverage, valid_width = compute_coverage(valid_y[0], valid_y[2], valid_y[1])
if mean_valid_loss <= min_valid_loss:
overfitting_epoch = 0
min_valid_loss = mean_valid_loss
torch.save(deepcopy(engine.model), save_dir + 'best_model.pth')
else:
overfitting_epoch+=1
if overfitting_epoch>=early_stopping:
break
logger.info("EPOCH: {:3d} TRAIN LOSS: {:.4f} TRAIN UPPER LOSS: {:.4f} TRAIN LOWER LOSS: {:.4f} VALID LOSS: {:.4f} MIN VALID LOSS: {:.4f}".format(epoch, mean_train_loss,mean_train_upper_loss, mean_train_lower_loss,mean_valid_loss, min_valid_loss))
# check_point = torch.load(save_dir + 'best_model.pth')
engine.model = torch.load(save_dir + 'best_model.pth')
mean_valid_loss = []
valid_y = [None]*3
for step,(X_batch, y_batch, dtw) in enumerate(val_dataloader):
loss, y_hat = engine.valid(X_batch,y_batch,dtw)
mean_valid_loss.append(loss)
if step==0:
valid_y[0] = y_batch
valid_y[1] = y_hat[0]
valid_y[2] = y_hat[1]
else:
valid_y[0] = np.concatenate([valid_y[0], y_batch],axis=0)
valid_y[1] = np.concatenate([valid_y[1], y_hat[0]],axis=0)
valid_y[2] = np.concatenate([valid_y[2], y_hat[1]],axis=0)
mean_valid_loss = np.mean(mean_valid_loss)
print("After loading model valid loss:",mean_valid_loss)
model.eval()
and torch.no_grad()
have been added to engine.valid()
and engine.test()