I am trying to perform a simple binary classification with a neural network on the make_moons dataset.
Because of random initialization, I expect the first values to be equally splited between correct / incorrect with 50% chance. This would lead to a value of the loss (cross entropy) as -ln(2)=0.69 but my initial loss is 1.684.
What could it be?
I am using a simple PyTorch 2 layers NN:
class TorchNet(nn.Module):
def __init__(self, inp:int, out:int, hid:int, n_layers:int, actf: str = 'relu':
super(TorchNet, self).__init__()
opt = ['relu', 'sigm', 'tanh']
err = 'Select a correct activation function from: {}'.format(opt)
assert actf in opt, err
self.n_lay = n_layers
self.fcInp = nn.Linear(inp, hid, bias=False)
self.fcHid = nn.ModuleList([nn.Linear(hid, hid, bias=False) for _ in range(self.n_lay)])
self.fcOut = nn.Linear(hid, out, bias=False)
if actf == 'relu': self.actf = nn.ReLU(inplace=True)
if actf == 'sigm': self.actf = nn.Sigmoid()
if actf == 'tanh': self.actf = nn.Tanh()
def forward(self, x):
# Input Layer
x = self.actf(self.fcInp(x))
# Hidden Layers
for l in range(self.n_lay):
x = self.actf(self.fcHid[l](x))
# Apply recursivity to the last layer
if l == max(range(self.n_lay)) and self.recursive is not None:
for _ in range(self.recursive):
x = self.actf(self.fcHid[l](x))
# Output Layer
x = self.fcOut(x)
return x