When you run the below code then the last line - print(test_output[0])- shows that the model returns all ‘nan’ values. Also when I added a print statement to the start of the forward method to print out a weight - print("preforward w: ", w[0][0][0]) - in the 4th round it throws an error RuntimeError: Overflow when unpacking long.
My main problem is that I don’t know why the nan values come. Any ideas?
import torch
import torchvision
import torch.nn as nn
import numpy as np
import torchvision.transforms as transforms
from torchvision import datasets
from torch.autograd import Variable
import pandas as pd
torch.manual_seed(1)
class MnistModel(nn.Module):
def __init__(self, w, b):
super(MnistModel, self).__init__()
self.w = torch.nn.Parameter(w)
self.b = torch.nn.Parameter(b)
self.w2 = torch.nn.Parameter(torch.rand(1000, 784, 10))
self.w3 = torch.nn.Parameter(torch.rand(1000, 784, 1))
def forward(self, x):
# print("preforward w: ", w[0][0][0])
layer = torch.mul(x , self.w)
layer = torch.bmm(layer, self.w2)
layer = layer.view(1000, 10, 784)
layer = torch.bmm(layer, self.w3)
output = layer.view(1000, 10)
return output
batch_size = 1000
classes = 10
train_data = datasets.MNIST('data', train=True,
download=True,
transform=transforms.ToTensor())
train_loader = torch.utils.data.DataLoader(train_data,
batch_size=batch_size,
shuffle=False)
test_data = datasets.MNIST('data', train=False, transform=transforms.ToTensor())
test_loader = torch.utils.data.DataLoader(test_data,batch_size=batch_size)
w = torch.randn(1, 784, requires_grad=True)
b = torch.randn(784, 1, requires_grad=True)
learning_rate = 0.001
model = MnistModel(w, b)
criterion = nn.MSELoss()
optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate)
rows = np.array(range(batch_size))
zeros = torch.zeros((batch_size, 10), dtype=torch.float64)
for i, (raw_data, raw_target) in enumerate(train_loader):
data = raw_data.view((batch_size, 784, 1))
logits = model(data)
zeros = pd.DataFrame(0, index=range(batch_size), columns=range(classes))
zeros.iloc[[rows, raw_target.numpy()]] = 1
zeros = torch.from_numpy(zeros.as_matrix())
target = zeros.float()
loss = criterion(logits, target)
# Backward and optimize
optimizer.zero_grad()
loss.backward()
optimizer.step()
if i == 3:
break
for i, (raw_data, raw_target) in enumerate(test_loader):
if i == 1:
break
data = raw_data.view((batch_size, 784, 1))
test_output = model(data)
print(test_output[0])