I’m was making a neural network to to try a few option of pytorch, and then when I tried it on the classical Breast Cancer dataset my algo was just stuck. I simplified to the maximum my model, but accurracy seem to be blocked.
class Modela(torch.nn.Module): def __init__(self): super(Modela, self).__init__() self.l1 = torch.nn.Linear(30,16) self.l2 = torch.nn.Linear(16,4) self.l3 = torch.nn.Linear(4,2) self.sigmoid = torch.nn.Sigmoid() self.relu = torch.nn.ReLU() def forward(self, x): out1 = self.relu(self.l1(x)) out2 = self.relu(self.l2(out1)) y_pred = self.sigmoid(self.l3(out2)) return y_pred
train_size = int(0.8 * len(full_dataset)) test_size = len(full_dataset) - train_size train_dataset, test_dataset = torch.utils.data.random_split(full_dataset, [train_size, test_size]) model = Modela() print(model) #TRAIN n_epoch = 50 criterion = torch.nn.CrossEntropyLoss()#torch.nn.BCELoss(reduction='sum') def train(model,train_dataset,test_dataset,loss_fn,n_epoch=50): optimizer = torch.optim.Adam(model.parameters(), lr=0.0001) valid_losses =  #list of the losses on valid valid_acc =  #list of the accuracy on valid train_losses =  #list of all the loss on training train_loss_store = torch.zeros(1) #will store the current accuracy for e in range(n_epoch): print('n_epoch',e) for i in range(len(train_dataset)): #let's evaluate the performan,ce of my model if (i%100 == 0): score = torch.zeros(1) valid_loss_store = torch.zeros(1) with torch.no_grad(): for j in range(len(test_dataset)): sample = test_dataset[j] X, label = sample['X'], sample['label'] output = model(X) output = torch.unsqueeze(output,0) label = torch.unsqueeze(label,0) loss = loss_fn(output, label) valid_loss_store += loss _, predicted = torch.max(output.data, 1) score += predicted.eq(label).float() #let's print a few result score = score/len(test_dataset) valid_acc += [score.item()] valid_loss_store = valid_loss_store/len(test_dataset) valid_losses += [valid_loss_store.item()] train_loss_store = train_loss_store/100 print('acc', score, 'valid loss :',valid_loss_store, 'train_loss_store', train_loss_store ) train_losses += [train_loss_store.item()] train_loss_store = torch.zeros(1) # here we train on the training dataset sample = train_dataset[i] X, label = sample['X'], sample['label'] optimizer.zero_grad() output = model(X) output = torch.unsqueeze(output,0) label = torch.unsqueeze(label,0) loss = loss_fn(output, label) train_loss_store += loss loss.backward() # for param in model.parameters(): # print(i,'gradient odg',param.grad.data.sum()) optimizer.step() return train_losses, valid_losses, valid_acc train_losses, valid_losses, valid_acc = train(model,train_dataset, test_dataset,criterion)
If I start this last part, sometimes I quickly grow to a 0.9 accurracy but most of the time I m just stuck to the exact first accuracy, with gradient close from zero. It looks like a local minimum maybe but in that case the fact of making a step for each data should enables it to escape this minimum. I don’t understand where it does come from.
I’ve tried to change the learning rate, and it does not change much.