How to use torch.optim.lr_scheduler.ExponentialLR?

6

Thanks! Here’s some more code…

        lstm = DSARNN(input_dims, sequence_length, cell_size)
        criterion = torch.nn.MSELoss()
        #optimiser = torch.optim.Adagrad(lstm.parameters(), lr=0.01)
        optimiser = torch.optim.SGD(lstm.parameters(), lr=0.1)
        scheduler = torch.optim.lr_scheduler.StepLR(optimiser, step_size=3, gamma=0.1)

        # register hooks

        lstm.softmax.register_forward_hook(monitorAttention)

        # init Tensorboard
        tensorboard_step = 0
        writer = SummaryWriter(comment="LSTM Cell + input attention, entropy " + str(bc.entropy()).replace('.', '_'))

        for epoch in range(40):

            scheduler.step(epoch)

            # train
            for minibatch, target in tqdm(Batches(train, target_input), total=len(train)):

                    optimiser.zero_grad()
                    output = lstm(minibatch)
                    loss = criterion(output, target)
                    loss.backward()
                    tensorboard_step += 1
                    writer.add_scalar('training loss', loss, tensorboard_step)
                    writer.add_scalar('learning rate', get_learning_rate(optimiser), tensorboard_step)
                    loss = optimiser.step()