My main.py contains iteration of multiple training.
I use seed, but even with same training setups, the result is slightly different.
How can I fix it? Please help me 
I run multiple experiments inside main.py to compare.
Below is some outputs. Here debug1 and debug2 is experiments of completely identical configs.
The data loader result seems to be constant across experiments (the train_dl size and eval_dl size have no change. In my case this should mean they are exactly same data split).
But the AUC is slightly different.
(base) $ python main.py config/debug.json
INFO : 4547 : 22:12 : __main__ | Starting Experiment: debug1
INFO : 4547 : 22:12 : ksdkt/debug1 | PyTorch: 1.3.0
INFO : 4547 : 22:12 : ksdkt/debug1 | Using Device: cuda
INFO : 4547 : 22:12 : ksdkt/debug1 | train_dl.dataset size: 14312
INFO : 4547 : 22:12 : ksdkt/debug1 | eval_dl.dataset size: 3644
INFO : 4547 : 22:12 : ksdkt/debug1 | The model has 514,424 trainable parameters
INFO : 4547 : 22:12 : ksdkt/debug1 | Starting train
INFO : 4547 : 22:13 : ksdkt/debug1 | Epoch 100 Train Loss: 0.61461 AUC: 0.846741
INFO : 4547 : 22:13 : ksdkt/debug1 | Epoch 100 Valid Loss: 0.691517 AUC: 0.528118
INFO : 4547 : 22:13 : ksdkt/debug1 | Best AUC 0.528118 refreshed and saved!
INFO : 4547 : 22:13 : ksdkt/debug1 | At 22:13 0m 0s passed ( - 0m 0s til 22:13) (100epoch 100.0%)
INFO : 4547 : 22:13 : ksdkt/debug1 | Starting evaluation
INFO : 4547 : 22:13 : ksdkt/debug1 | Valid Loss: 0.691517 AUC: 0.528118
INFO : 4547 : 22:13 : ksdkt/debug1 | At 22:13 0m 0s passed ( - 0m 0s til 22:13)
INFO : 4547 : 22:13 : __main__ | Starting Experiment: debug2
INFO : 4547 : 22:13 : ksdkt/debug2 | PyTorch: 1.3.0
INFO : 4547 : 22:13 : ksdkt/debug2 | Using Device: cuda
INFO : 4547 : 22:13 : ksdkt/debug2 | train_dl.dataset size: 14312
INFO : 4547 : 22:13 : ksdkt/debug2 | eval_dl.dataset size: 3644
INFO : 4547 : 22:13 : ksdkt/debug2 | The model has 514,424 trainable parameters
INFO : 4547 : 22:13 : ksdkt/debug2 | Starting train
INFO : 4547 : 22:13 : ksdkt/debug2 | Epoch 100 Train Loss: 0.614401 AUC: 0.850618
INFO : 4547 : 22:13 : ksdkt/debug2 | Epoch 100 Valid Loss: 0.691517 AUC: 0.52813
INFO : 4547 : 22:13 : ksdkt/debug2 | Best AUC 0.52813 refreshed and saved!
INFO : 4547 : 22:13 : ksdkt/debug2 | At 22:13 0m 0s passed ( - 0m 0s til 22:13) (100epoch 100.0%)
INFO : 4547 : 22:13 : ksdkt/debug2 | Starting evaluation
INFO : 4547 : 22:13 : ksdkt/debug2 | Valid Loss: 0.691517 AUC: 0.52813
INFO : 4547 : 22:13 : ksdkt/debug2 | At 22:13 0m 0s passed ( - 0m 0s til 22:13)
INFO : 4547 : 22:13 : __main__ | All experiments done!
I attach the abstract code below.
I split train.py from main.py, but even if I run seed_everything inside train.py too it didn’t get fixed.
I deleted a lot of logger.info to simplify the code in this question.
# main.py
from src.train import Trainer
def seed_everything(seed: int=42):
random.seed(seed)
os.environ["PYTHONHASHSEED"] = str(seed)
np.random.seed(seed)
torch.manual_seed(seed)
torch.cuda.manual_seed(seed)
torch.backends.cudnn.deterministic = True
def main(configpath: Path):
with open('config.json', r) as f:
cfg = json.load(f)
experiments = cfg['experiments']
for exp_dict in experiments:
config = Config(exp_dict)
run(config)
logger.info('All experiments done!')
def run(config):
seed_everything()
trainer = Trainer(config)
if not config.load_model:
try:
trainer.train_model()
except KeyboardInterrupt as e:
print(e)
finally:
trainer.dump_report()
trainer.evaluate_model()
# src/train.py
class Trainer(object):
def __init__(self, config):
self.config = config
self.device = self.get_device(self.config)
self.train_dl, self.eval_dl = self.get_dataloader(
self.config, self.device)
model = self.get_model(self.config, self.device)
if self.config.load_model:
model.load_state_dict(torch.load(str(self.config.load_model_path)))
model = model.to(self.device)
self.model = model
self.opt = self.get_opt(self.model)
self._report = {
'config': self.config.as_dict(),
'indicator': defaultdict(list)
}
def get_device(self, config):
device = torch.device(
'cuda' if config.cuda and torch.cuda.is_available() else 'cpu')
return device
def get_model(self, config, device):
if config.model_name == 'MyModel':
model = MyModel(config, device).to(device)
else:
raise ValueError(f'model_name {config.model_name} is wrong')
return model
def get_dataloader(self, config, device):
train_dl, eval_dl = prepare_dataloader(
config, device=device, pad=config.pad)
return train_dl, eval_dl
def get_opt(self, model):
opt = torch.optim.SGD(model.parameters(), lr=self.config.lr)
return opt
def train_model(self, validate=True):
for epoch in range(1, self.config.epoch_size + 1):
self.model.train()
t_idc = self.exec_core(self.train_dl, self.opt)
t_loss, t_auc = t_idc['loss'], t_idc['auc']
if epoch % 10 == 0 and validate:
with torch.no_grad():
self.model.eval()
v_idc = self.exec_core(dl=self.eval_dl, opt=None)
v_loss, v_auc = v_idc['loss'], v_idc['auc']
def exec_core(self, dl, opt, only_eval=False):
arr_len = len(dl) if not self.config.debug else 1
pred_mx = np.zeros([arr_len, self.config.batch_size])
actu_mx = np.zeros([arr_len, self.config.batch_size])
pred_ls = []
actu_ls = []
pred_v_mx = np.zeros(
[arr_len, self.config.batch_size * self.config.n_skills])
actu_v_mx = np.zeros(
[arr_len, self.config.batch_size * self.config.n_skills])
loss_ar = np.zeros(arr_len)
wvn1_ar = np.zeros(arr_len)
wvn2_ar = np.zeros(arr_len)
ksv1_ar = np.zeros(arr_len)
if only_eval:
q_all_count = defaultdict(int)
q_cor_count = defaultdict(int)
q_pred_list = defaultdict(list)
for i, (xseq, yseq, mask) in enumerate(dl):
out = self.model.loss_batch(xseq, yseq, mask, opt=opt)
loss_ar[i] = out['loss'].item()
fpr, tpr, _thresholds = metrics.roc_curve(
torch.cat(actu_ls).detach().cpu().numpy().reshape(-1),
torch.cat(pred_ls).detach().cpu().numpy().reshape(-1), pos_label=1)
auc = metrics.auc(fpr, tpr)
indicators = {
'loss': loss_ar.mean(),
'auc': auc,
}
return indicators
def _train_model_simple(self):
'''simplest skelton'''
for epoch in range(1, self.config.epoch_size + 1):
self.model.train()
for i, (xseq, yseq) in enumerate(self.train_dl):
out = self.model.loss_batch(xseq, yseq, opt=self.opt)
def evaluate_model(self):
with torch.no_grad():
self.model.eval()
indicators = self.exec_core(
dl=self.eval_dl, opt=None, only_eval=True)
v_loss, v_auc = indicators['loss'], indicators['auc']