Hey everyone! I am working on a binary classifier with simulated data. The dataset monitors COVID related symptoms. Originally the whole dataset was simulated, but then I found real-world data. However, I still needed to generate testing statuses, as these are not readily available to the public. Regardless, neither dataset seemed to be working with any of my models. The loss seemed to vary slightly between epochs but not in any significant way. And the accuracy stayed exactly the same the entire time. I am wondering whether this is an issue of me not knowing how to simulate data, implementing the Dataset
class incorrectly, or how I am training the models. I don’t think it’s how I define my models because they learn and the loss changes appropriately when I run these on the MNIST
data provided by PyTorch.
I’ve included all the relevant code below
#Loading the data
import torchvision.datasets as dsets
import torchvision.transforms as transforms
from torch.utils.data import Dataset, DataLoader
class dataSet(Dataset):
def __init__(self, data):
self.X = torch.from_numpy(data[:, 0:-1]).float()
self.Y = torch.from_numpy(data[:, -1]).float()
self.samples = data.shape[0]
def __getitem__(self, index):
return self.X[index], self.Y[index]
def __len__ (self):
return self.samples
#data is a Pandas dataframe
train_dataset = dataSet(data.to_numpy())
test_dataset = dataSet(data.to_numpy())
train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)
test_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False)
#Function for training and evaluating
def train_model(model, n_epochs):
iteration = 0
learning_rate = 0.001
optimizer = optim.Adam(baseline.parameters(), lr=learning_rate)
criterion = nn.BCEWithLogitsLoss()
for epochs in range(n_epochs):
model.train()
for inputs, targets in train_loader:
# Make sure they accumulate gradients!
inputs.requires_grad = True
# Clear gradients w.r.t. parameters
optimizer.zero_grad()
# Forward pass to get the output/logits
output = model(inputs)
# Calculate Loss
loss = criterion(output, targets.unsqueeze(1))
# Get gradients w.r.t. the parameters
loss.backward()
# Update the parameters
optimizer.step()
iteration += 1
# Every 200 iterations, check up on how the model is doing,
# by printing the loss and the training accuracy on the held out data.
# Accuracy = (number of test statuses correctly identified) / (total number of statuses)
if iteration % 200 == 0:
# Calculate Accuracy
correct = 0
total = 0
model.eval()
# Iterate through test dataset
for inputs, targets in test_loader:
# Load images with gradient accumulation capabilities
inputs.requires_grad = True
# Forward pass only to get logits/output
outputs = model(inputs)
# Get predictions from the maximum value
_, predictions = torch.max(outputs.data, 1)
# Get the total number of labels
total += targets.size(0)
# Calculate the total correct predictions
overlap = predictions.eq(targets)
correct += (overlap == True).sum().item()
accuracy = 100 * (correct/total)
# Print Loss
print('Iteration: {}. Loss: {}. Accuracy: {}'.format(iteration, loss.item(), accuracy))
Here is the resulting output when I run either of these models:
#Running the models
baseline = binaryClassifier(input_dim)
baseline
binaryClassifier(
(linear): Linear(in_features=26, out_features=1, bias=True)
)
list(baseline.parameters())
[Parameter containing:
tensor([[ 3.9646e-04, -4.4786e-04, -4.0808e-04, 3.5642e-04, 3.9394e-04,
-3.8129e-04, -4.5752e-04, 2.2144e-04, -1.7427e-04, 7.4252e-05,
2.8504e-04, 4.6275e-05, 7.3203e-06, -1.5663e-04, -5.1841e-05,
1.5425e-04, -3.3701e-04, 2.2288e-04, -4.6276e-04, 1.0501e-04,
1.7254e-04, -5.5671e-05, -1.9452e-04, 4.5088e-04, -1.3678e-04,
4.0610e-04]], requires_grad=True), Parameter containing:
tensor([-0.0816], requires_grad=True)]
train_model(baseline, epochs)
Iteration: 200. Loss: 0.08446517586708069. Accuracy: 95.11
Iteration: 400. Loss: 0.15068431198596954. Accuracy: 95.11
Iteration: 600. Loss: 0.1580568253993988. Accuracy: 95.11
Iteration: 800. Loss: 0.14487606287002563. Accuracy: 95.11
Iteration: 1000. Loss: 0.18062788248062134. Accuracy: 95.11
Iteration: 1200. Loss: 0.13961248099803925. Accuracy: 95.11
Iteration: 1400. Loss: 0.1084781289100647. Accuracy: 95.11
Iteration: 1600. Loss: 0.1737363040447235. Accuracy: 95.11
Iteration: 1800. Loss: 0.1317535638809204. Accuracy: 95.11
Iteration: 2000. Loss: 0.10618090629577637. Accuracy: 95.11
list(baseline.parameters())
[Parameter containing:
tensor([[ 0.3363, 0.1596, 0.0758, 0.0787, -0.5132, -0.7288, -0.1051, -0.0859,
-0.1446, -0.3361, -0.6885, -0.5501, -0.5627, -0.5177, -0.3476, -0.3683,
-0.5116, -0.4581, -0.5300, -0.5626, -0.3942, -0.6555, -0.3099, -0.5286,
-0.6720, -0.2889]], requires_grad=True), Parameter containing:
tensor([-0.6178], requires_grad=True)]
model2 = FFNN(input_dim, 500, act_fn="relu", apply_dropout=True)
learning_rate = 0.001
optimizer = optim.Adam(model2.parameters(), lr=learning_rate)
criterion = nn.BCEWithLogitsLoss()
train_model(model2, epochs)
Iteration: 200. Loss: 0.0984780415892601. Accuracy: 95.11
Iteration: 400. Loss: 0.24589982628822327. Accuracy: 95.11
Iteration: 600. Loss: 0.10863172262907028. Accuracy: 95.11
Iteration: 800. Loss: 0.05246538296341896. Accuracy: 95.11
Iteration: 1000. Loss: 0.130008727312088. Accuracy: 95.11
Iteration: 1200. Loss: 0.13363279402256012. Accuracy: 95.11
Iteration: 1400. Loss: 0.16624166071414948. Accuracy: 95.11
Iteration: 1600. Loss: 0.07791421562433243. Accuracy: 95.11
Iteration: 1800. Loss: 0.15000399947166443. Accuracy: 95.11
Iteration: 2000. Loss: 0.14199721813201904. Accuracy: 95.11