Hello,
I’m was making a neural network to to try a few option of pytorch, and then when I tried it on the classical Breast Cancer dataset my algo was just stuck. I simplified to the maximum my model, but accurracy seem to be blocked.
my model:
class Modela(torch.nn.Module):
def __init__(self):
super(Modela, self).__init__()
self.l1 = torch.nn.Linear(30,16)
self.l2 = torch.nn.Linear(16,4)
self.l3 = torch.nn.Linear(4,2)
self.sigmoid = torch.nn.Sigmoid()
self.relu = torch.nn.ReLU()
def forward(self, x):
out1 = self.relu(self.l1(x))
out2 = self.relu(self.l2(out1))
y_pred = self.sigmoid(self.l3(out2))
return y_pred
my training:
train_size = int(0.8 * len(full_dataset))
test_size = len(full_dataset) - train_size
train_dataset, test_dataset = torch.utils.data.random_split(full_dataset, [train_size, test_size])
model = Modela()
print(model)
#TRAIN
n_epoch = 50
criterion = torch.nn.CrossEntropyLoss()#torch.nn.BCELoss(reduction='sum')
def train(model,train_dataset,test_dataset,loss_fn,n_epoch=50):
optimizer = torch.optim.Adam(model.parameters(), lr=0.0001)
valid_losses = [] #list of the losses on valid
valid_acc = [] #list of the accuracy on valid
train_losses = [] #list of all the loss on training
train_loss_store = torch.zeros(1) #will store the current accuracy
for e in range(n_epoch):
print('n_epoch',e)
for i in range(len(train_dataset)):
#let's evaluate the performan,ce of my model
if (i%100 == 0):
score = torch.zeros(1)
valid_loss_store = torch.zeros(1)
with torch.no_grad():
for j in range(len(test_dataset)):
sample = test_dataset[j]
X, label = sample['X'], sample['label']
output = model(X)
output = torch.unsqueeze(output,0)
label = torch.unsqueeze(label,0)
loss = loss_fn(output, label)
valid_loss_store += loss
_, predicted = torch.max(output.data, 1)
score += predicted.eq(label).float()
#let's print a few result
score = score/len(test_dataset)
valid_acc += [score.item()]
valid_loss_store = valid_loss_store/len(test_dataset)
valid_losses += [valid_loss_store.item()]
train_loss_store = train_loss_store/100
print('acc', score, 'valid loss :',valid_loss_store, 'train_loss_store', train_loss_store )
train_losses += [train_loss_store.item()]
train_loss_store = torch.zeros(1)
# here we train on the training dataset
sample = train_dataset[i]
X, label = sample['X'], sample['label']
optimizer.zero_grad()
output = model(X)
output = torch.unsqueeze(output,0)
label = torch.unsqueeze(label,0)
loss = loss_fn(output, label)
train_loss_store += loss
loss.backward()
# for param in model.parameters():
# print(i,'gradient odg',param.grad.data.sum())
optimizer.step()
return train_losses, valid_losses, valid_acc
train_losses, valid_losses, valid_acc = train(model,train_dataset, test_dataset,criterion)
If I start this last part, sometimes I quickly grow to a 0.9 accurracy but most of the time I m just stuck to the exact first accuracy, with gradient close from zero. It looks like a local minimum maybe but in that case the fact of making a step for each data should enables it to escape this minimum. I don’t understand where it does come from.
I’ve tried to change the learning rate, and it does not change much.
Best regards
b