Hi guys, I’m trying to optimize the following parameters by keeping the L param always lower triangular with a positive diagonal and the noise param always with just positive diagonal, but they are not correctly updating in the forward pass. I guess I’m doing something wrong with the autograd mechanism. Any help appreciated. Here is a sample snippet.
import torch
from torch.autograd import Variable
from torch.nn.parameter import Parameter
from torch.nn import ParameterList
class Model(torch.nn.Module):
def __init__(self, dim):
"""
Constructor.
"""
super(Model, self).__init__()
self.noise_vector = Parameter(torch.tensor(torch.zeros(D).cuda(), requires_grad=True))
self.noise = Parameter(torch.tensor(torch.diag(torch.exp(self.noise_vector.data)).cuda(), requires_grad=True))
self.L_chol_cov_theta = Parameter(torch.tensor(torch.randn(dim, dim).cuda(), requires_grad=True))
self.log_diag_L_chol_cov_theta = Parameter(torch.tensor(torch.randn(dim).cuda(), requires_grad=True))
self.L = Parameter(torch.tensor(torch.randn(dim, dim).cuda(), requires_grad=True))
self.L_chol_cov_theta.data = torch.tril(self.L_chol_cov_theta.data)
self.L_chol_cov_theta.data -= torch.diag(torch.diag(self.L_chol_cov_theta.data))
self.L.data = self.L_chol_cov_theta.data + torch.diag(torch.exp(self.log_diag_L_chol_cov_theta.data))
def forward(self):
# update parameters
self.L_chol_cov_theta.data -= torch.diag(torch.diag(self.L_chol_cov_theta.data))
self.L.data = self.L_chol_cov_theta.data + torch.diag(torch.exp(self.log_diag_L_chol_cov_theta.data))
self.noise.data = torch.diag(torch.exp(self.noise_vector.data))
return torch.mm(self.L, self.noise_vector.view(-1,-1))
custom_net = Model(5)
optimizer = torch.optim.SGD(filter(lambda p: p.requires_grad, model.parameters()), lr=1e-3)
for epoch in range(10):
optimizer.zero_grad()
forward_pass_something = model()
loss = calc_likelihood(samples, a_ground_truth) # calc a custom loss
loss.backward()
optimizer.step()