#I want to switch the optimizer in the middle of the training
#from LBFGS to Adam.
#Below is the code that I wrote.
optimizer = optim.LBFGS(model.parameters(), lr=0.003)
Use_Adam_optim_FirstTime=True
Use_LBFGS_optim=True
for epoch in range(30000):
loss_SUM = 0
for i, (x, t) in enumerate(GridLoader):
x = x.to(device)
t = t.to(device)
if Use_LBFGS_optim:
def closure():
optimizer.zero_grad()
lg, lb, li = problem_formulation(x, t, x_Array,t_Array,bndry,pi)
loss_total=lg+ lb+ li
loss_total.backward(retain_graph=True)
return loss_total
loss_out=optimizer.step(closure)
loss_SUM+=loss_out.item()
elif Use_Adam_optim_FirstTime:
Use_Adam_optim_FirstTime=False
optimizerAdam = optim.Adam(model.parameters(), lr=0.0003)
model.load_state_dict(checkpoint['model'])
optimizerAdam.zero_grad()
lg, lb, li = problem_formulation(x, t, x_Array,t_Array,bndry,pi)
lg.backward()
lb.backward()
li.backward()
optimizerAdam.step()
loss_SUM += lg.item()+lb.item()+li.item()
else:
optimizerAdam.zero_grad()
lg, lb, li = problem_formulation(x, t, x_Array,t_Array,bndry,pi)
lg.backward()
lb.backward()
li.backward()
optimizerAdam.step()
loss_SUM += lg.item()+lb.item()+li.item()
if loss_SUM<.3 and use_LBFGS_optim == True:
Use_LBFGS_optim=False
checkpoint = {'model': model.state_dict(),
'optimizer': optimizer.state_dict()}
#My questions are:
#Regading the closure function:
#(1) Is there a way that I can return more than one variable from the closure function?
#(2) Is there a way that I can make three backwards in the closure function instead of only one?
#(3) Why do I need to set retain_graph to True in loss_total.backward(retain_graph=True) of the closure function
#(4) Sometimes, the loss_SUM for the closure function approcheas to 1e+29. Is there a way to avoid this problem?
#-----------------------------
#Regading the switch from LBFGS to Adam:
#(5) When loss_SUM<0.3 and we go to the "elif" part, the loss reduces however after one epoch,
#loss_SUM dramatically increases (e.g., 20 times)
#What is the correct way of switching the optimizer from LBFGS to Adam
#In general, what are the problems of the above code and how I can impprove it.