I’m new to pytorch, and I’m trying to implement a VAE. My input is a matrix that looks like this:
array([[2.55210000e+00, 8.51190000e+00, 1.17499000e+02, ...,
1.24905000e+04, 2.10142000e+03, 8.60778000e+02],
[3.89330000e+00, 5.73050000e+00, 1.03911000e+02, ...,
1.53351000e+04, 2.02537000e+03, 8.47769000e+02],
[5.45940000e+00, 1.04417000e+01, 1.48825000e+02, ...,
2.04567000e+04, 1.90901000e+03, 8.18905000e+02],
...,
[6.46610247e+01, 5.59159696e+01, 2.04382482e+02, ...,
3.76147787e+03, 3.68634406e+03, 1.04819776e+03],
[1.02605040e+02, 6.04046079e+01, 1.12747868e+02, ...,
2.31106780e+03, 2.90486515e+03, 8.09890055e+02],
[2.95345151e+01, 3.85387327e+01, 3.38398587e+02, ...,
5.84942659e+03, 1.35516902e+03, 4.42895508e+02]])
Here is the code:
class VAE(nn.Module):
def __init__(self):
super(VAE, self).__init__()
self.en1 = nn.Linear(input_size, 200)
self.en_mu = nn.Linear(200, representation_size)
self.en_std = nn.Linear(200, representation_size)
self.de1 = nn.Linear(representation_size, 200)
self.de2 = nn.Linear(200, input_size)
self.relu = nn.ReLU()
self.sigmoid = nn.Sigmoid()
def encode(self, x):
"""Encode a batch of samples, and return posterior parameters for each point."""
h1 = self.relu(self.en1(x))
return self.en_mu(h1), self.en_std(h1)
def decode(self, z):
"""Decode a batch of latent variables"""
h2 = self.relu(self.de1(z))
return self.sigmoid(self.de2(h2))
def reparam(self, mu, logvar):
"""Reparameterisation trick to sample z values.
This is stochastic during training, and returns the mode during evaluation."""
if self.training:
std = logvar.mul(0.5).exp_()
eps = Variable(std.data.new(std.size()).normal_())
return eps.mul(std).add_(mu)
else:
return mu
def forward(self, x):
"""Takes a batch of samples, encodes them, and then decodes them again to compare."""
mu, logvar = self.encode(x.view(-1, input_size))
z = self.reparam(mu, logvar)
return self.decode(z), mu, logvar
def loss(self, reconstruction, x, mu, logvar):
"""ELBO assuming entries of x are binary variables, with closed form KLD."""
bce = torch.nn.functional.binary_cross_entropy(reconstruction, x.view(-1, input_size))
KLD = -0.5 * torch.sum(1 + logvar - mu.pow(2) - logvar.exp())
# Normalise by same number of elements as in reconstruction
KLD /= x.view(-1, input_size).data.shape[0] * input_size
return bce + KLD
def get_z(self, x):
"""Encode a batch of data points, x, into their z representations."""
mu, logvar = self.encode(x.view(-1, input_size))
return self.reparam(mu, logvar)
model = VAE()
optimizer = torch.optim.Adam(model.parameters(), lr=1e-3)
def train(epoch, batches_per_epoch = 501, log_interval=500):
model.train()
ind = np.arange(x.shape[0])
for i in range(batches_per_epoch):
data = torch.from_numpy(x[np.random.choice(ind, size=batch_size)])
data = Variable(data, requires_grad=False)
optimizer.zero_grad()
recon_batch, mu, logvar = model(data)
loss = model.loss(recon_batch, data, mu, logvar)
loss.backward()
#train_loss += loss.data[0]
optimizer.step()
if (i % log_interval == 0) and (epoch % 5 ==0):
#Print progress
print('Train Epoch: {} [{}/{}]\tLoss: {:.6f}'.format(
epoch, i * batch_size, batch_size*batches_per_epoch,
loss.data / len(data)))
print('====> Epoch: {} done!'.format(
epoch))
def test(epoch, batches_per_epoch=1):
model.eval()
ind = np.arange(x.shape[0])
for i in range(batches_per_epoch):
data = torch.from_numpy(x[np.random.choice(ind, size=batch_size)])
data = Variable(data, requires_grad=False)
recon_batch, mu, logvar = model(data)
if i == 0:
n = min(data.size(0), 2)
print(data.view(batch_size, 2,2)[:n])
print(recon_batch.view(batch_size, 2,2)[:n])
print_examples = False #Change to true if you want to see some examples at each step!
for epoch in range(1, 30):
train(epoch)
if print_examples:
test(epoch)
---------------------------------------------------------------------------
RuntimeError Traceback (most recent call last)
<ipython-input-63-a93465c748b0> in <module>()
1 print_examples = False #Change to true if you want to see some examples at each step!
2 for epoch in range(1, 30):
----> 3 train(epoch)
4 if print_examples:
5 test(epoch)
<ipython-input-62-c2ded286e850> in train(epoch, batches_per_epoch, log_interval)
8 optimizer.zero_grad()
9 recon_batch, mu, logvar = model(data)
---> 10 loss = model.loss(recon_batch, data, mu, logvar)
11 loss.backward()
12 #train_loss += loss.data[0]
<ipython-input-59-acbe3e619d52> in loss(self, reconstruction, x, mu, logvar)
42 """ELBO assuming entries of x are binary variables, with closed form KLD."""
43
---> 44 bce = torch.nn.functional.binary_cross_entropy(reconstruction, x.view(-1, input_size))
45 KLD = -0.5 * torch.sum(1 + logvar - mu.pow(2) - logvar.exp())
46 # Normalise by same number of elements as in reconstruction
/projects/sysbio/projects/czi/immune/anaconda2/envs/py36/lib/python3.6/site-packages/torch/nn/functional.py in binary_cross_entropy(input, target, weight, size_average, reduce, reduction)
2025
2026 return torch._C._nn.binary_cross_entropy(
-> 2027 input, target, weight, reduction_enum)
2028
2029
RuntimeError: Assertion `x >= 0. && x <= 1.' failed. input value should be between 0~1, but got -nan at /opt/conda/conda-bld/pytorch_1549627089062/work/aten/src/THNN/generic/BCECriterion.c:62