I am trying to make a LRScheduler Object for cyclical learning rate with warm restarts. I have written the code and checked it does what it is supposed to but am having difficulty making it a _LRScheduler object. The errors I get when I do
model = Mnist_Logistic()
opt=optim.SGD(model.parameters(), lr=lr)
cylical=Cyc_lr(optimizer=opt , lr_max=0.5 , lr_min=0.01 , datasize=700 , bs=64 , epochs=1)
model,opt = get_model()
loss_func(model(xb), yb)
are :
AttributeError: 'Cyc_lr' object has no attribute 'max_lr' even when self.max_lr is defined in init and taken as input, in this case
AttributeError: 'cyc_lr' object has no attribute 'optimizer' Even though optimiser object has been passed
I have added a last_epoch=-1 argument as it wanted that. Can someone tell me how I can create this object? I don’t get what I am doing wrong.
I looked at the pytorch classes in the docs and followed the pattern.
Here is the Cyclical LR class( get_lr is abt hairy but what formal statements is this class missing?)
from torch.optim.lr_scheduler import _LRScheduler
class Cyc_lr(_LRScheduler):
def __init__(self,optimizer,lr_max,lr_min,datasize,bs,epochs,last_epoch=-1):
"""Args:
Cyclical Learning rate with warm restart
lr_max: maximum learning rate
lr_min: minimum learning rate
datasize: total number of datapoints in train dataset (excluding val/test)
bs: batchsize trained on
epochs: number of epochs to reset learning rate
"""
super(Cyc_lr, self).__init__(optimizer, last_epoch)
self.optimizer=optimizer
self.n = 0
self.last_epoch=-1
self.batches=int(datasize/bs) # how many batches in one epoch, one batch is one lr step
self.points=epochs*self.batches # how many lr steps for one max to min
self.min_lr = lr_min
self.max_lr = lr_max
if 'self.lr' not in locals(): self.lr=self.max_lr # create self.lr initially as max lr
self.miny=self.min_lr/self.max_lr # get the ratio and take arccos to know when to stop x in cos function
self.stopx=np.arccos(self.miny) # so that cos points are created only in between the required range
self.x=np.linspace(0,self.stopx+0.5,self.points)
self.y=0.5*np.cos(self.x)+0.5
self.maxn=len(self.y)-1 # length of cos list
def get_lr(self):
self.lr=self.max_lr*self.y[self.n] # multiply lr by decreasing cos ration
self.n+=1
if self.lr<self.min_lr: # incase somethings go wrong
self.n=0
self.lr=self.max_lr
self.points=2*self.points
self.x=np.linspace(0,self.stopx+0.5,self.points)
self.y=0.5*np.cos(self.x)+0.5
if self.n==self.maxn:
self.n=0
self.lr=self.max_lr
self.points=int(1.5*self.points) # increase points by 1.5x so reaching min takes longer
self.x=np.linspace(0,self.stopx+0.5,self.points)
self.y=0.5*np.cos(self.x)+0.5
self.maxn=len(self.y)-1
return self.lr