Hi,
I am trying to find optimal hyperparameters for my deep learning model. I could able to optimize LR, BS and momentum. But I am strugging with optimization of number of layers and neurons within them. Could someone help me ?
Here is what I have written, what I expect is to modify this class to find optimal input_neurons and output_neurons within each layer.
class Net(nn.Module):
def __init__(self, n_lay=2, neur=32, dropout=0.1):
super().__init__()
self.name = "DNN"
self.init_args = {"n_lay": n_lay, "neur": neur}
self.fc = nn.ModuleList([nn.Linear(1105, neur)]) # 1105 features/descriptors
self.fc.extend([nn.Linear(neur, neur) for i in range(1, n_lay-1)])
self.fc.append(nn.Linear(neur, 1))
self.dropout = nn.Dropout(p=dropout)
def forward(self, x):
for i, l in enumerate(self.fc):
x = self.dropout(self.fc[i](x))
return x
parameters:
opt_BO = [{"name": "lr", "type": "continuous", "domain": (0.00001, 0.001)},
{"name": "bs", "type": "discrete", "domain": range(16, 512, 16)},
{"name": "n_lay", "type": "discrete", "domain": range(2, 6)},
{"name": "neur", "type": "discrete", "domain": range(8, 128, 8)},
{"name": "dropout", "type": "discrete", "domain": np.linspace(0, 0.5, 11)}]
The f_opt function is as below:
def f_opt(parameters):
parameters = parameters[0]
# Data loading:
trainloader = torch.utils.data.DataLoader(dataset_train, batch_size=int(parameters[1]), shuffle=True)
testloader = torch.utils.data.DataLoader(dataset_val, batch_size=int(parameters[1]), shuffle=False)
train_dl, valid_dl = (WrapDL(trainloader, to_gpu), WrapDL(testloader, to_gpu))
# Model definition:
model = Net(
n_lay=int(parameters[2]),
neur=int(parameters[3]),
dropout=parameters[4]
)
model.to(device)
# Optimizer:
opt = optim.SGD(model.parameters(), lr=parameters[0])
scheduler = optim.lr_scheduler.CyclicLR(opt, parameters[0], parameters[0] * 3,
step_size_up = int(len(train_dl) * int(parameters[1]) * epochs * 0.25))
# Fit:
score = fit(epochs, model, loss_func, scheduler, train_dl, valid_dl, calc_accuracy)
return np.array(score)