Hey I have a very simple NN with only one layer. I was going through someones tutorial and they did not take the log in the negative log likely hood loss. This is on the MNIST flattened dataset. When I take the log and train, my model does not learning anything while without the log it achieves 90% accuracy. How can log be messing up the learning, shouldn’t it be the same?
This is his and my loss:
def nll(input, target): return -input[range(target.shape[0]), target].mean()
def nll2(input,target): #my
x= -input[:,target].mean()
return torch.log(x)
def log_softmax(x): return x - x.exp().sum(-1).log().unsqueeze(-1) # ???
def model(xb): return log_softmax(torch.mm(xb,weights) + bias)
with gzip.open('data/mnist/mnist.pkl.gz', 'rb') as f:
((x_train, y_train), (x_valid, y_valid), _) = pickle.load(f)
bs=64 # batch size
xb = x_train[0:bs] # a mini-batch from x
preds = model(xb) # predictions
preds[0], preds.shape
weights = torch.randn(784,10)/math.sqrt(784) # dont want 784 to be in grad
weights.requires_grad_()
bias = torch.zeros(10, requires_grad=True)
def acc(input,target):
indices=torch.argmax(input,1)
return (indices==target).sum().float()/len(indices)
Training :
lr=0.015
for i in range(0,5000):
r=np.random.randint(0,50000)
ins=x_train[r:r+64]
tars=y_train[r:r+64]
outs=model(ins)
loss=nll(outs,tars) # ***** changing nll to nll2 does not make the model learn
if i%10==0:
print('loss: ',loss)
print('accuracy: ',acc(outs,tars))
loss.backward()
with torch.no_grad():
weights -= weights.grad * lr
bias -= bias.grad * lr
weights.grad.zero_()
bias.grad.zero_()