Can't use BatchNorm1d despite using a batch size of 64 for training

Hello guys, you can find my code below. It throws the following error “Expected more than 1 value per channel when training, got input size torch.Size([1, 256])”

This is my data generator code:

class datagen(Dataset):
    def __init__(self, df, tfms):
        super(datagen, self).__init__()
        self.x = df['x']
        self.y = df['y']
        self.tfms = tfms
    def __len__(self):
        return len(self.x)
    def __getitem__(self, i):
        img = imageio.imread(self.x[i])
        stacked_img = np.repeat(img[..., np.newaxis], 3, -1)
        x = self.tfms(image = stacked_img)
        y = self.y[i]
        return (x, y)

train_set = datagen(train_df, ttfms)
train = iter(DataLoader(train_set, batch_size = 64, shuffle = False, drop_last = True, num_workers = 16, pin_memory = True))

val_set = datagen(val_df, vtfms)
val = iter(DataLoader(val_set, batch_size = 64, shuffle = False, drop_last = True, num_workers = 16, pin_memory = True))

This is my code for network and training:

class Network(nn.Module):
    def __init__(self):
        super(Network, self).__init__()
        self.conv = models.resnet101(pretrained = True)
        
        for param in self.conv.parameters():
            param.requires_grad = False
            
        self.fc1 = nn.Sequential(nn.Linear(64000, 2048),
                                 nn.Linear(2048, 1024),
                                 nn.Linear(1024, 256))
        
        self.bn1 = nn.BatchNorm1d(256)
        
        self.fc2 = nn.Sequential(nn.Linear(256, 64),
                                 nn.Linear(64, 16))
        
        self.bn2 = nn.BatchNorm1d(16)
        
        self.out = nn.Linear(16, 1)

    def forward(self, x):
        x = self.conv(x)
        x = x.view(1, 64000)
        x = F.relu(self.bn1(self.fc1(x)))
        x = self.bn2(self.fc2(x))
        x = F.sigmoid(self.out(x))
        return x

model = Network()
model = model.to(device)
criterion = nn.BCELoss()
optimizer = optim.Adam(model.parameters(), lr = 0.07)

for e in tqdm(range(10)):
    x, y = train.next()
    x, y = x['image'].float().to(device), y.float().to(device)
    optimizer.zero_grad()
    ps = model.forward(x)
    train_loss = criterion(ps, y.reshape(-1, 1))
    train_loss.backward()
    optimizer.step()

This view operations looks wrong:

x = x.view(1, 64000)

since you are flattening the tensor such that it’ll have a batch size of 1.
To keep the batch size equal, use x = x.view(x.size(0), -1) instead.

In case you would still run into the batchnorm issue, check if the last batch returned by the DataLoader might be smaller. If that’s the case you can drop it via drop_last=True.

1 Like

Thank you, you have always helped me and everyone :smiley: