Hi,
As a simple test for multi-label classification I am trying to fit a simple linear neural network to random noise and randomly generated binary labels for a few categories. However, over each epoch the loss seems to not decrease at all. Here is a self-contained example:
import os
import torch
import torch.onnx as onnx
import torchvision.models as models
import numpy as np
from torch import nn
from torch.utils.data import DataLoader, TensorDataset
from torchvision import datasets
from torchvision.transforms import ToTensor, Lambda
import torch.nn.functional as F
from torch.autograd import Variable
X_np = np.random.rand(1000,214)
y_np = np.random.randint(2, size=(1000,15))
train_X = X_np[:900]
train_y = y_np[:900]
test_X = X_np[900:]
test_y = y_np[900:]
train_X_tensor = Variable(torch.from_numpy(train_X).float(), requires_grad=True)
train_y_tensor = Variable(torch.from_numpy(train_y).float(), requires_grad=False)
test_X_tensor = Variable(torch.from_numpy(test_X).float(), requires_grad=True)
test_y_tensor = Variable(torch.from_numpy(test_y).float(), requires_grad=False)
train_dataset = TensorDataset(train_X_tensor, train_y_tensor)
train_dataloader = DataLoader(train_dataset,batch_size=1)
test_dataset = TensorDataset(test_X_tensor, test_y_tensor)
test_dataloader = DataLoader(test_dataset,batch_size=1)
device='cpu'
learning_rate = 0.3
epochs = 5
num_triggers=len(train_y[0])
num_inputs=len(train_X[0])
lambda1 = 0.001
class NeuralNetwork(nn.Module):
def __init__(self):
super(NeuralNetwork, self).__init__()
self.linear1 = nn.Linear(num_inputs, num_triggers)
def forward(self, x):
x = self.linear1(x)
x2 = torch.sigmoid(x)
return x2
model = NeuralNetwork().to(device)
def train_loop(dataloader, model, loss_fn, optimizer):
size = len(dataloader.dataset)
for i in range(3):
model.train()
print("epoch", i+1)
for batch, (X, y) in enumerate(dataloader):
X, y = X.to(device), y.to(device)
# Compute prediction and loss
pred = model(X)
loss = loss_fn(pred, y)
# Backpropagation
optimizer.zero_grad()
loss.backward()
optimizer.step()
if batch % 100 == 0:
loss, current = loss.item(), batch * len(X)
print(f"loss: {loss:>7f} [{current:>5d}/{size:>5d}]")
test_loop(test_dataloader, model, loss_fn)
def test_loop(dataloader, model, loss_fn):
model.eval()
size = len(dataloader.dataset)
test_loss, correct = 0, 0
with torch.no_grad():
for X, y in dataloader:
pred = model(X)
test_loss += loss_fn(pred, y).item()
test_loss /= size
print(f"Test Error: \n Avg loss: {test_loss:>8f} \n")
return [],[]
loss_fn = nn.BCELoss()
optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate)
train_loop(train_dataloader, model, loss_fn, optimizer)
I feel like I might be misunderstanding something in either how I’m using the randomly generated numpy array or how I’m training the neural network. Any advice would be appreciated, thanks!
PS: It is pretty important that I read in the data from a numpy array as that is how I will receive my training data in my use-case.