I have built this model with pytorch and I have trained it many times adjusting layers, batch size, learning rate, but for some reason, when I test it with a single batch the outputs, no matter the inputs, are always the same values.
Here is my model:
class model(nn.Module):
def __init__(self):
super().__init__()
self.conv1 = nn.Sequential(
nn.Conv1d(
in_channels=2,
out_channels=4,
kernel_size=3,
stride=1,
padding=2
),
nn.ReLU(),
nn.MaxPool1d(kernel_size=2)
)
self.conv2 = nn.Sequential(
nn.Conv1d(
in_channels=4,
out_channels=8,
kernel_size=3,
stride=1,
padding=2
),
nn.ReLU(),
nn.MaxPool1d(kernel_size=2)
)
self.flatten = nn.Flatten()
self.fc1 = nn.Linear(88, 22)
self.fc2 = nn.Linear(22, 2)
self.norm1 = nn.BatchNorm1d(2, 40)
self.norm2 = nn.BatchNorm1d(2)
def forward(self, x):
x = self.norm1(x)
x = self.conv1(x)
x = self.conv2(x)
x = self.flatten(x)
x = torch.relu(self.fc1(x))
x = self.fc2(x)
x = self.norm2(x)
return x
train_dataset = HeartBeatSensorDataset(root_dir='D:\Python\\arduino data collection\Good Data\Train\\')
train_dataloader = DataLoader(train_dataset, batch_size=32, shuffle=True, drop_last=True)
model = model()
optimizer = optim.Adam(model.parameters(), lr=1e-4)
My layers are very simple with a low amount of parameters since my data is really simple, and a large model wouldn’t converge. My input tensors are, (batch, 2, 40).
Here is my training loop:
running_loss = 0.0
model.to(device='cuda:0')
model.train(True)
criterion = nn.CrossEntropyLoss()
info_dict = {}
for epoch in range(50):
for i, data in enumerate(train_dataloader, 0):
inputs = data['data']
targets = data['targets']
targets = targets.to(dtype=torch.long, device='cuda:0')
inputs = inputs.to(dtype=torch.float, device='cuda:0')
optimizer.zero_grad()
outputs = model(inputs)
outputs = outputs.to(dtype=torch.double)
loss = criterion(outputs, targets)
loss.backward()
optimizer.step()
running_loss += loss.item()
if i % 10 == 0:
correct_guess = 0
guess_acc = []
for z, guess in enumerate(outputs):
index = torch.argmax(guess, dim=-1)
guess_acc.append(index)
guess_bool = index == targets[z] #check if guess was correct or not
guess_bool = guess_bool.item()
if guess_bool == True:
correct_guess += 1
accuracy = correct_guess / targets.size(dim=-1) * 100
info_dict['Loss'] = running_loss
info_dict['Epoch'] = epoch + 1
info_dict['Iteration'] = i
info_dict['Outputs'] = torch.flatten(torch.softmax(outputs, dim=-1))
info_dict['Guess'] = guess_acc
info_dict['Targets'] = targets
info_dict['Guess Accuracy'] = accuracy
for key, value in info_dict.items():
print((key, value))
print('===========================================')
running_loss = 0.0
print('training done')
model_script = torch.jit.trace(model, inputs)
model_script.save('model_script3.pt')
Here is my test script:
model_test = torch.jit.load('model_script3.pt')
test_dataset = HeartBeatSensorDataset(root_dir='D:\Python\\arduino data collection\Good Data\Test\\')
test_dataloader = DataLoader(test_dataset, batch_size=1, shuffle=True)
test = next(iter(test_dataloader))
model_test.eval()
model_test.cpu()
test['data'] = test['data'].to(dtype=torch.float)
with torch.no_grad():
outputs = model_test(test['data'])
outputs = torch.softmax(outputs, dim=-1)
info_dict = {}
prediction = torch.argmax(outputs, dim=-1)
correct = prediction == test['targets']
info_dict['Outputs'] = outputs
info_dict['Targets'] = test['targets']
info_dict['Prediction'] = prediction
info_dict['Correct'] = correct
print('=====================')
for key, value in info_dict.items():
print((key, value))
Here is one of my training outputs:
('Outputs', tensor([0.1118, 0.8882, 0.5427, 0.4573, 0.1863, 0.8137, 0.1619, 0.8381, 0.1118,
0.8882, 0.1118, 0.8882, 0.7964, 0.2036, 0.8400, 0.1600, 0.9656, 0.0344,
0.1285, 0.8715, 0.9341, 0.0659, 0.1346, 0.8654, 0.2828, 0.7172, 0.1118,
0.8882, 0.9202, 0.0798, 0.1863, 0.8137, 0.1346, 0.8654, 0.1863, 0.8137,
0.1118, 0.8882, 0.2117, 0.7883, 0.1863, 0.8137, 0.8192, 0.1808, 0.9621,
0.0379, 0.2284, 0.7716, 0.9797, 0.0203, 0.6406, 0.3594, 0.7735, 0.2265,
0.1346, 0.8654, 0.7547, 0.2453, 0.8942, 0.1058, 0.8306, 0.1694, 0.4298,
0.5702], device='cuda:0', dtype=torch.float64, grad_fn=<ViewBackward0>))
And my test outputs look like this with a single batch:
=====================
('Outputs', tensor([[0.4778, 0.5222]]))
('Targets', tensor([1]))
('Prediction', tensor([1]))
('Correct', tensor([True]))
=====================
('Outputs', tensor([[0.4778, 0.5222]]))
('Targets', tensor([0]))
('Prediction', tensor([1]))
('Correct', tensor([False]))
=====================
('Outputs', tensor([[0.4778, 0.5222]]))
('Targets', tensor([0]))
('Prediction', tensor([1]))
('Correct', tensor([False]))
That spread never changes and these are all different runs of the test script. If I retrain the model or adjust its layers then those values will slightly change, but the problem remains.
I have also tried training with a much smaller subset of my dataset and with random torch tensors (over fitting it) and had the same problem both times. So at this point it must be something wrong with my model structure or training method, but I am not sure.
In terms of my data, it is all in csv files and I don’t apply any transforms. I did collect it myself so I am not sure if that could be the problem or not. I am new to pytorch so if the problem is obvious, sorry.
Thank you for any help.