Hi,
I am training a classification neural network but the loss function is barely changing at each epoch, I checked the gradients of my network params and I found them to be 0 or e-05,e-10… I have tried normalization, decreasing hidden layers, changing activation functions but still no improvements, and I don’t think it is a problem of requires_grad=true, here is the code below
model:
def init(self,input_dim,hidden_dim,output_dim=4):
super(Sic_Detector,self,).init()
self.input=nn.Linear(in_features=input_dim,out_features=hidden_dim)
self.hidden1 = nn.Linear(in_features=hidden_dim, out_features=hidden_dim*3)
self.output= nn.Linear(in_features=hidden_dim, out_features=output_dim)
self.softmax=nn.Softmax(dim=0)
def forward(self,x):
x=self.input(x)
#x=self.hidden1(relu(x))
x=self.output(relu(x))
return self.softmax(x)
Training block:
criterion = nn.CrossEntropyLoss()
optimizer = optim.Adam(model.parameters(),lr=0.005)
for epoch in range(10):
running_loss = 0.0
for i, data in enumerate(trainloader, 0):
inputs, labels = data
inputs=inputs.requires_grad_(True)
optimizer.zero_grad()
outputs = model(inputs)
loss = criterion(outputs, labels)
loss.backward()
running_loss += loss.item()
optimizer.step()