How do you pick your dimensions for a neural net

1

I’ve been struggling with this patch of code where I cannot seem to get the size of my matrices right. This one is based on a batch size of 4.

https://github.com/adomakor412/NERTO_CIMSS_GOES-R/blob/a35f882fe15f68cb6df22bc5c28d503ca18a88b3/AUC_model.ipynb

If you have trouble rendering the code use https://nbviewer.jupyter.org/

class Net(nn.Module):
    def __init__(self):
        super().__init__()
        self.conv1 = nn.Conv2d(3, 6, 5)
        self.pool = nn.MaxPool2d(2, 2)
        self.conv2 = nn.Conv2d(6, 16, 5)
        self.fc1 = nn.Linear(16 * 5 * 5, 120)
        self.fc2 = nn.Linear(120, 84)
        self.fc3 = nn.Linear(84, 2)
#         super(Net, self).__init__()
#         self.conv1 = nn.Conv2d(3, 32, 5)
#         self.pool = nn.MaxPool2d(2, 2)
#         self.conv2 = nn.Conv2d(32, 64, 5)
#         self.fc1 = nn.Linear(64*9*9, 1024)
#         self.fc2 = nn.Linear(1024, 7)
    def forward(self, x):
        x = self.pool(F.relu(self.conv1(x)))
        x = self.pool(F.relu(self.conv2(x)))
        x = torch.flatten(x, 1) # flatten all dimensions except batch
        x = F.relu(self.fc1(x))
        x = F.relu(self.fc2(x))
        x = self.fc3(x)
#         x = self.pool(F.relu(self.conv1(x)))
#         x = self.pool(F.relu(self.conv2(x)))
#         x = x.view(x.size(0), -1)
#         #x = x.view(-1, 64)
#         #x = F.relu(self.fc1(x))
#         #x = F.relu(self.fc2(x))
#         #x = self.fc2(x)
        return x
net = Net()

And when try the net() this way, I get an index out of range error:

class Net(nn.Module):
    def __init__(self):
#         super().__init__()
#         self.conv1 = nn.Conv2d(3, 6, 5)
#         self.pool = nn.MaxPool2d(2, 2)
#         self.conv2 = nn.Conv2d(6, 16, 5)
#         self.fc1 = nn.Linear(16 * 5 * 5, 120)
#         self.fc2 = nn.Linear(120, 84)
#         self.fc3 = nn.Linear(84, 2)
        super(Net, self).__init__()
        self.conv1 = nn.Conv2d(3, 32, 5)
        self.pool = nn.MaxPool2d(2, 2)
        self.conv2 = nn.Conv2d(32, 64, 5)
        self.fc1 = nn.Linear(64*9*9, 1024)
        self.fc2 = nn.Linear(1024, 7)
    def forward(self, x):
#         x = self.pool(F.relu(self.conv1(x)))
#         x = self.pool(F.relu(self.conv2(x)))
#         x = torch.flatten(x, 1) # flatten all dimensions except batch
#         x = F.relu(self.fc1(x))
#         x = F.relu(self.fc2(x))
#         x = self.fc3(x)
#         return x
        x = self.pool(F.relu(self.conv1(x)))
        x = self.pool(F.relu(self.conv2(x)))
        x = x.view(x.size(0), -1)
        #x = x.view(-1, 64)
        #x = F.relu(self.fc1(x))
        #x = F.relu(self.fc2(x))
        #x = self.fc2(x)
        return x
2
3

There are only two classes in my target label.

4

Your initial model works fine for an input shape of [batch_size, 3, 32, 32]:

class Net(nn.Module):
    def __init__(self):
        super().__init__()
        self.conv1 = nn.Conv2d(3, 6, 5)
        self.pool = nn.MaxPool2d(2, 2)
        self.conv2 = nn.Conv2d(6, 16, 5)
        self.fc1 = nn.Linear(16 * 5 * 5, 120)
        self.fc2 = nn.Linear(120, 84)
        self.fc3 = nn.Linear(84, 2)

    def forward(self, x):
        x = self.pool(F.relu(self.conv1(x)))
        x = self.pool(F.relu(self.conv2(x)))
        x = torch.flatten(x, 1) # flatten all dimensions except batch
        x = F.relu(self.fc1(x))
        x = F.relu(self.fc2(x))
        x = self.fc3(x)
        return x

net = Net()
x = torch.randn(3, 3, 32, 32)
out = net(x)
print(out.shape)
> torch.Size([3, 2])

What error are you getting and could you post an executable code snippet, which would reproduce it?

5

How did you figure out my initial model would work for an input shape of [batch_size, 3, 32, 32]?

6

Here is the error and snippet of code when attempted batch size of 4:

running_loss = 0.0
for epoch in range(1):  # loop over the dataset multiple times
    
    if len(list(enumerate(trainloader, 0))) < 10:
            print('Batch size too small (<10) for PR-curve')
            
    for i, data in enumerate(trainloader, 0):
        
        # get the inputs; data is a list of [inputs, labels]
        inputs, labels = data

        # zero the parameter gradients
        optimizer.zero_grad()

        # forward + backward + optimize
        outputs = net(inputs)
        loss = criterion(outputs, labels)
        loss.backward()
        optimizer.step()

        running_loss += loss.item()
        if i % 10 == 9:    # every 1000 mini-batches...
#         if i % 1000 == 999:    # every 1000 mini-batches...

            # ...log the running loss
            writer.add_scalar('training loss',
                            running_loss / 1000,
                            epoch * len(trainloader) + i)

            # ...log a Matplotlib Figure showing the model's predictions on a
            # random mini-batch
            writer.add_figure('predictions vs. actuals',
                            plot_classes_preds(net, inputs, labels),
                            global_step=epoch * len(trainloader) + i)
            running_loss = 0.0
        
print('Finished Training')

0 index of predictions with length 4: 49254
1 index of predictions with length 4: 79865
2 index of predictions with length 4: 44043
3 index of predictions with length 4: 43242

IndexError Traceback (most recent call last)
in
31 # random mini-batch
32 writer.add_figure(‘predictions vs. actuals’,
—> 33 plot_classes_preds(net, inputs, labels),
34 global_step=epoch * len(trainloader) + i)
35 running_loss = 0.0

in plot_classes_preds(net, images, labels)
31 matplotlib_imshow(images[idx], one_channel=True)
32 ax.set_title(“{0}, {1:.1f}%\n(label: {2})”.format(
—> 33 classes[preds[idx]],
34 probs[idx] * 100.0,
35 classes[labels[idx]]),

IndexError: list index out of range

7

So for the LeNet model this is how the convolution is designed for 32x32 image sizes:

Deep learning with Pytorch: understanding the neural network example

8

It was a guess based on your model definition.

The indexing error is raised in plot_classes_preds, where classes[preds[idx]] uses an invalid index.
Could you check the shape of preds and the value of idx as well as the shape of classes and the value of preds[idx] as one of them is using an invalid indexing?