I am trying to make a LRScheduler Object for cyclical learning rate with warm restarts. I have written the code and checked it does what it is supposed to but am having difficulty making it a _LRScheduler object. The errors I get when I do
model = Mnist_Logistic() opt=optim.SGD(model.parameters(), lr=lr) cylical=Cyc_lr(optimizer=opt , lr_max=0.5 , lr_min=0.01 , datasize=700 , bs=64 , epochs=1) model,opt = get_model() loss_func(model(xb), yb)
AttributeError: 'Cyc_lr' object has no attribute 'max_lr' even when self.max_lr is defined in init and taken as input, in this case
AttributeError: 'cyc_lr' object has no attribute 'optimizer' Even though optimiser object has been passed
I have added a last_epoch=-1 argument as it wanted that. Can someone tell me how I can create this object? I don’t get what I am doing wrong.
I looked at the pytorch classes in the docs and followed the pattern.
Here is the Cyclical LR class( get_lr is abt hairy but what formal statements is this class missing?)
from torch.optim.lr_scheduler import _LRScheduler class Cyc_lr(_LRScheduler): def __init__(self,optimizer,lr_max,lr_min,datasize,bs,epochs,last_epoch=-1): """Args: Cyclical Learning rate with warm restart lr_max: maximum learning rate lr_min: minimum learning rate datasize: total number of datapoints in train dataset (excluding val/test) bs: batchsize trained on epochs: number of epochs to reset learning rate """ super(Cyc_lr, self).__init__(optimizer, last_epoch) self.optimizer=optimizer self.n = 0 self.last_epoch=-1 self.batches=int(datasize/bs) # how many batches in one epoch, one batch is one lr step self.points=epochs*self.batches # how many lr steps for one max to min self.min_lr = lr_min self.max_lr = lr_max if 'self.lr' not in locals(): self.lr=self.max_lr # create self.lr initially as max lr self.miny=self.min_lr/self.max_lr # get the ratio and take arccos to know when to stop x in cos function self.stopx=np.arccos(self.miny) # so that cos points are created only in between the required range self.x=np.linspace(0,self.stopx+0.5,self.points) self.y=0.5*np.cos(self.x)+0.5 self.maxn=len(self.y)-1 # length of cos list def get_lr(self): self.lr=self.max_lr*self.y[self.n] # multiply lr by decreasing cos ration self.n+=1 if self.lr<self.min_lr: # incase somethings go wrong self.n=0 self.lr=self.max_lr self.points=2*self.points self.x=np.linspace(0,self.stopx+0.5,self.points) self.y=0.5*np.cos(self.x)+0.5 if self.n==self.maxn: self.n=0 self.lr=self.max_lr self.points=int(1.5*self.points) # increase points by 1.5x so reaching min takes longer self.x=np.linspace(0,self.stopx+0.5,self.points) self.y=0.5*np.cos(self.x)+0.5 self.maxn=len(self.y)-1 return self.lr