How can I include additional data to my CNN at the fully-connected layer?

My dataset consists of samples in the form:
[(3x32x32 tensor), (20x1 tensor), float value)]
First is the image data, second is the metadata vector, last is the label.

I defined my CNN such that in the forward pass, the image tensor in extracted from the sample and then passed through the convolutional layers. When completed, it is reshaped into a vector and the metadata is concatenated. In my CNN definition I increase the first fully-connected layer’s size to compensate for the additional data.
I successfully trained the model but when I try to pass a sample(s) through the trained model I get the following error:

TypeError: conv2d() received an invalid combination of arguments - got (list, Parameter, Parameter, tuple, tuple, tuple, int), but expected one of:
 * (Tensor input, Tensor weight, Tensor bias, tuple of ints stride, tuple of ints padding, tuple of ints dilation, int groups)
      didn't match because some of the arguments have invalid types: (list, Parameter, Parameter, tuple, tuple, tuple, int)
 * (Tensor input, Tensor weight, Tensor bias, tuple of ints stride, str padding, tuple of ints dilation, int groups)
      didn't match because some of the arguments have invalid types: (list, Parameter, Parameter, tuple, tuple, tuple, int)

I searched this error but the solutions did not seem applicable. I am wondering if this has to do with the fact that my forward function is being given a list holding the tensor data as opposed to the tensor data itself. But that if that was the case I think the model would not be able to successfully train to begin with.

CNN definition:

class CNN(nn.Module):
    def __init__(self):
        super().__init__()

        self.conv1 = nn.Sequential(
            nn.Conv2d(in_channels=3, out_channels=28, kernel_size=3),
            nn.BatchNorm2d(28),
            nn.ReLU()
        )
        self.conv11 = nn.Sequential(
            nn.Conv2d(in_channels=28, out_channels=28, kernel_size=3),
            nn.BatchNorm2d(28),
            nn.ReLU()
        )

        self.conv2 = nn.Sequential(
            nn.Conv2d(in_channels=28, out_channels=56, kernel_size=3),
            nn.BatchNorm2d(56),
            nn.ReLU()
        )
        self.conv22 = nn.Sequential(
            nn.Conv2d(in_channels=56, out_channels=56, kernel_size=3),
            nn.BatchNorm2d(56),
            nn.ReLU()
        )

        self.conv3 = nn.Sequential(
            nn.Conv2d(in_channels=56, out_channels=112, kernel_size=3),
            nn.BatchNorm2d(112),
            nn.ReLU()
        )
        self.conv33 = nn.Sequential(
            nn.Conv2d(in_channels=112, out_channels=112, kernel_size=3),
            nn.BatchNorm2d(112),
            nn.ReLU()
        )

        self.size_x = self.size_y = 20
        self.mdata_size = 20
        self.fc1 = nn.Linear(112 * self.size_x * self.size_y + self.mdata_size, 512)
        self.fc2 = nn.Linear(512, 128)
        self.drop2 = nn.Dropout2d(0.1)
        self.fc3 = nn.Linear(128, 100)
        self.drop3 = nn.Dropout2d(0.1)
        self.out = nn.Linear(100, 100)

    def forward(self, x):
        v = x[1]
        x = self.conv1(x[0])
        x = self.conv11(x)
        x = self.conv2(x)
        x = self.conv22(x)
        x = self.conv3(x)
        x = self.conv33(x)

        x = torch.cat((x.reshape(-1, 112 * self.size_x * self.size_y), v), 1)

        x = F.relu(self.fc1(x))
        x = self.drop2(F.relu(self.fc2(x)))
        x = self.drop3(F.relu(self.fc3(x)))
        x = self.out(x)
        return x

Train method:

def train(model, optimizer, criterion, train_loader, test_loader, epochs):
    print(f"Training (using device: {device})...")
    trainlosses = []
    testlosses = []
    
    for epoch in range(epochs):
        # Train loop
        training_loss = 0
        for images, labels in tqdm(iter(train_loader)):
            images[0] = images[0].to(device)
            images[1] = images[1].to(device)
            labels = labels.to(device)
            
            optimizer.zero_grad()
            out = model(images)
            
            loss = criterion(out, labels.to(torch.long))
            training_loss += loss.cpu().item()
            loss.backward()
            optimizer.step()
        trainlosses.append(training_loss / len(train_loader))

        # Test loop
        testing_loss = 0
        for images, labels in tqdm(iter(test_loader)):
            images[0] = images[0].to(device)
            images[1] = images[1].to(device)
            labels = labels.to(device)
            outputs = model(images)
            test_loss = criterion(outputs, labels.to(torch.long))
            testing_loss += test_loss.cpu().item()
        testlosses.append(testing_loss / len(test_loader))

        print(f"Epoch {epoch + 1} of {epochs} | "
              f"Training Loss: {round(training_loss / len(train_loader), 4)} | "
              f"Testing Loss: {round(testing_loss / len(test_loader), 4)}")

Attempt at inference 1:

ds = test_loader.dataset
sample = ds[0]
print(sample)
model.eval() 
model(sample)
>>>[[tensor([[[0.0706, 0.0588, 0.0431,  ..., 0.0863, 0.0824, 0.0745],
         [0.0784, 0.0627, 0.0392,  ..., 0.0784, 0.0784, 0.0745],
         [0.0784, 0.0627, 0.0431,  ..., 0.0824, 0.0784, 0.0745],
         ...,
         [0.2667, 0.2157, 0.0510,  ..., 0.6078, 0.6980, 0.6667],
         [0.3686, 0.3020, 0.0745,  ..., 0.2196, 0.3647, 0.3961],
         [0.4588, 0.3137, 0.1137,  ..., 0.2627, 0.1961, 0.1882]],

        [[0.0902, 0.0745, 0.0549,  ..., 0.1020, 0.0941, 0.0863],
         [0.0980, 0.0784, 0.0549,  ..., 0.0980, 0.0941, 0.0863],
         [0.1020, 0.0824, 0.0588,  ..., 0.0980, 0.0941, 0.0902],
         ...,
         [0.1608, 0.1451, 0.0235,  ..., 0.5804, 0.6824, 0.6667],
         [0.3333, 0.2118, 0.0627,  ..., 0.1686, 0.2980, 0.3255],
         [0.4235, 0.1725, 0.0706,  ..., 0.2000, 0.1333, 0.1216]],

        [[0.1020, 0.0824, 0.0588,  ..., 0.0745, 0.0667, 0.0588],
         [0.1098, 0.0863, 0.0549,  ..., 0.0706, 0.0667, 0.0588],
         [0.1137, 0.0863, 0.0588,  ..., 0.0745, 0.0667, 0.0588],
         ...,
         [0.1529, 0.0902, 0.0118,  ..., 0.4431, 0.5373, 0.5216],
         [0.2980, 0.1490, 0.0275,  ..., 0.1294, 0.2353, 0.2510],
         [0.3686, 0.1490, 0.0392,  ..., 0.1843, 0.1216, 0.1294]]]), tensor([0., 0., 0., 0., 0., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
        0., 0.])], 26.0]
---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
~\AppData\Local\Temp/ipykernel_20644/1839303120.py in <module>
      4 
      5 model.eval()
----> 6 model(sample)

D:\ProgramData\Anaconda3\lib\site-packages\torch\nn\modules\module.py in _call_impl(self, *input, **kwargs)
   1100         if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
   1101                 or _global_forward_hooks or _global_forward_pre_hooks):
-> 1102             return forward_call(*input, **kwargs)
   1103         # Do not call functions when jit is used
   1104         full_backward_hooks, non_full_backward_hooks = [], []

~\AppData\Local\Temp/ipykernel_20644/4062736071.py in forward(self, x)
     46     def forward(self, x):
     47         v = x[1]
---> 48         x = self.conv1(x[0])
     49         x = self.conv11(x)
     50         x = self.conv2(x)

D:\ProgramData\Anaconda3\lib\site-packages\torch\nn\modules\module.py in _call_impl(self, *input, **kwargs)
   1100         if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
   1101                 or _global_forward_hooks or _global_forward_pre_hooks):
-> 1102             return forward_call(*input, **kwargs)
   1103         # Do not call functions when jit is used
   1104         full_backward_hooks, non_full_backward_hooks = [], []

D:\ProgramData\Anaconda3\lib\site-packages\torch\nn\modules\container.py in forward(self, input)
    139     def forward(self, input):
    140         for module in self:
--> 141             input = module(input)
    142         return input
    143 

D:\ProgramData\Anaconda3\lib\site-packages\torch\nn\modules\module.py in _call_impl(self, *input, **kwargs)
   1100         if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
   1101                 or _global_forward_hooks or _global_forward_pre_hooks):
-> 1102             return forward_call(*input, **kwargs)
   1103         # Do not call functions when jit is used
   1104         full_backward_hooks, non_full_backward_hooks = [], []

D:\ProgramData\Anaconda3\lib\site-packages\torch\nn\modules\conv.py in forward(self, input)
    444 
    445     def forward(self, input: Tensor) -> Tensor:
--> 446         return self._conv_forward(input, self.weight, self.bias)
    447 
    448 class Conv3d(_ConvNd):

D:\ProgramData\Anaconda3\lib\site-packages\torch\nn\modules\conv.py in _conv_forward(self, input, weight, bias)
    440                             weight, bias, self.stride,
    441                             _pair(0), self.dilation, self.groups)
--> 442         return F.conv2d(input, weight, bias, self.stride,
    443                         self.padding, self.dilation, self.groups)
    444 

TypeError: conv2d() received an invalid combination of arguments - got (list, Parameter, Parameter, tuple, tuple, tuple, int), but expected one of:
 * (Tensor input, Tensor weight, Tensor bias, tuple of ints stride, tuple of ints padding, tuple of ints dilation, int groups)
      didn't match because some of the arguments have invalid types: (list, Parameter, Parameter, tuple, tuple, tuple, int)
 * (Tensor input, Tensor weight, Tensor bias, tuple of ints stride, str padding, tuple of ints dilation, int groups)
      didn't match because some of the arguments have invalid types: (list, Parameter, Parameter, tuple, tuple, tuple, int)

Attempt at inference 2:

sample = next(iter(test_loader))
print(sample)
model.train() 
model(sample)
>>> [[tensor([[[[0.0706, 0.0588, 0.0431,  ..., 0.0863, 0.0824, 0.0745],
          [0.0784, 0.0627, 0.0392,  ..., 0.0784, 0.0784, 0.0745],
          [0.0784, 0.0627, 0.0431,  ..., 0.0824, 0.0784, 0.0745],
          ...,
          [0.2667, 0.2157, 0.0510,  ..., 0.6078, 0.6980, 0.6667],
          [0.3686, 0.3020, 0.0745,  ..., 0.2196, 0.3647, 0.3961],
          [0.4588, 0.3137, 0.1137,  ..., 0.2627, 0.1961, 0.1882]],

         [[0.0902, 0.0745, 0.0549,  ..., 0.1020, 0.0941, 0.0863],
          [0.0980, 0.0784, 0.0549,  ..., 0.0980, 0.0941, 0.0863],
          [0.1020, 0.0824, 0.0588,  ..., 0.0980, 0.0941, 0.0902],
          ...,
          [0.1608, 0.1451, 0.0235,  ..., 0.5804, 0.6824, 0.6667],
          [0.3333, 0.2118, 0.0627,  ..., 0.1686, 0.2980, 0.3255],
          [0.4235, 0.1725, 0.0706,  ..., 0.2000, 0.1333, 0.1216]],

         [[0.1020, 0.0824, 0.0588,  ..., 0.0745, 0.0667, 0.0588],
          [0.1098, 0.0863, 0.0549,  ..., 0.0706, 0.0667, 0.0588],
          [0.1137, 0.0863, 0.0588,  ..., 0.0745, 0.0667, 0.0588],
          ...,
          [0.1529, 0.0902, 0.0118,  ..., 0.4431, 0.5373, 0.5216],
          [0.2980, 0.1490, 0.0275,  ..., 0.1294, 0.2353, 0.2510],
          [0.3686, 0.1490, 0.0392,  ..., 0.1843, 0.1216, 0.1294]]],


        [[[0.8471, 0.8353, 0.8039,  ..., 0.4510, 0.5451, 0.6353],
          [0.8353, 0.7882, 0.7922,  ..., 0.4745, 0.4745, 0.5608],
          [0.8118, 0.7333, 0.7922,  ..., 0.4431, 0.4275, 0.4275],
          ...,
          [0.2000, 0.1961, 0.2353,  ..., 0.4275, 0.4784, 0.3137],
          [0.2235, 0.2078, 0.2510,  ..., 0.4157, 0.4353, 0.3255],
          [0.1686, 0.2392, 0.1647,  ..., 0.3882, 0.3725, 0.3647]],

         [[0.8471, 0.8314, 0.8039,  ..., 0.4588, 0.5608, 0.6549],
          [0.8314, 0.7882, 0.7882,  ..., 0.4745, 0.4784, 0.5686],
          [0.8039, 0.7294, 0.7882,  ..., 0.4353, 0.4275, 0.4275],
          ...,
          [0.1569, 0.1529, 0.1843,  ..., 0.4275, 0.4667, 0.3059],
          [0.1843, 0.1686, 0.1961,  ..., 0.4275, 0.4118, 0.3059],
          [0.1255, 0.2000, 0.1216,  ..., 0.3765, 0.3412, 0.3529]],

         [[0.9020, 0.8824, 0.8510,  ..., 0.4863, 0.6235, 0.7294],
          [0.8863, 0.8392, 0.8392,  ..., 0.4902, 0.5137, 0.6235],
          [0.8667, 0.7882, 0.8431,  ..., 0.4588, 0.4431, 0.4549],
          ...,
          [0.1059, 0.0902, 0.1333,  ..., 0.4039, 0.4431, 0.2627],
          [0.1294, 0.1333, 0.1412,  ..., 0.3882, 0.3412, 0.2706],
          [0.0941, 0.1569, 0.0863,  ..., 0.3098, 0.2431, 0.3098]]],


        [[[0.6235, 0.4431, 0.4627,  ..., 0.1137, 0.1451, 0.2824],
          [0.5216, 0.4471, 0.4510,  ..., 0.1176, 0.1412, 0.2784],
          [0.4824, 0.3412, 0.3961,  ..., 0.1373, 0.1490, 0.3098],
          ...,
          [0.3961, 0.4039, 0.4039,  ..., 0.3765, 0.4078, 0.4275],
          [0.3216, 0.3294, 0.3333,  ..., 0.4039, 0.4275, 0.4588],
          [0.2196, 0.2549, 0.2706,  ..., 0.4039, 0.3961, 0.4078]],

         [[0.7020, 0.5333, 0.5569,  ..., 0.2039, 0.2431, 0.3961],
          [0.6471, 0.5608, 0.5569,  ..., 0.2039, 0.2314, 0.3922],
          [0.6118, 0.4549, 0.4980,  ..., 0.2157, 0.2314, 0.4235],
          ...,
          [0.3451, 0.3490, 0.3490,  ..., 0.3451, 0.3765, 0.3961],
          [0.3137, 0.3137, 0.3176,  ..., 0.3569, 0.3922, 0.4275],
          [0.2353, 0.2588, 0.2706,  ..., 0.3529, 0.3529, 0.3765]],

         [[0.5882, 0.4000, 0.4000,  ..., 0.1176, 0.1412, 0.2902],
          [0.4784, 0.4078, 0.4000,  ..., 0.1216, 0.1412, 0.2667],
          [0.4627, 0.3412, 0.3804,  ..., 0.1333, 0.1490, 0.2706],
          ...,
          [0.2784, 0.2784, 0.2784,  ..., 0.2667, 0.2980, 0.3137],
          [0.2667, 0.2745, 0.2784,  ..., 0.2784, 0.3176, 0.3569],
          [0.1922, 0.2235, 0.2353,  ..., 0.2784, 0.2824, 0.3098]]],


        [[[0.8863, 0.8510, 0.8549,  ..., 0.2627, 0.2078, 0.1725],
          [0.7804, 0.7451, 0.7647,  ..., 0.2157, 0.1412, 0.1804],
          [0.5373, 0.5137, 0.5412,  ..., 0.1882, 0.1490, 0.2314],
          ...,
          [0.5059, 0.5098, 0.3490,  ..., 0.3176, 0.2549, 0.3961],
          [0.4745, 0.4431, 0.4784,  ..., 0.2902, 0.2588, 0.3647],
          [0.4980, 0.4824, 0.5059,  ..., 0.3451, 0.2980, 0.2588]],

         [[0.9020, 0.8706, 0.8627,  ..., 0.1725, 0.1412, 0.1216],
          [0.8196, 0.7882, 0.8000,  ..., 0.1765, 0.1451, 0.1451],
          [0.6431, 0.6157, 0.6235,  ..., 0.2196, 0.1490, 0.2392],
          ...,
          [0.4039, 0.4000, 0.2510,  ..., 0.2588, 0.2235, 0.2902],
          [0.3725, 0.3333, 0.3412,  ..., 0.2392, 0.2039, 0.2431],
          [0.3569, 0.3255, 0.3255,  ..., 0.2824, 0.2118, 0.1490]],

         [[0.6902, 0.6510, 0.6510,  ..., 0.0549, 0.0471, 0.0510],
          [0.5765, 0.5451, 0.5686,  ..., 0.0510, 0.0471, 0.0627],
          [0.3922, 0.3608, 0.3961,  ..., 0.0588, 0.0549, 0.1059],
          ...,
          [0.3686, 0.3608, 0.2431,  ..., 0.1843, 0.1098, 0.2039],
          [0.3451, 0.2941, 0.3059,  ..., 0.1451, 0.1176, 0.1882],
          [0.3176, 0.2863, 0.2667,  ..., 0.2039, 0.1725, 0.1137]]],


        [[[0.0980, 0.1294, 0.0745,  ..., 0.3098, 0.3255, 0.2706],
          [0.0941, 0.1098, 0.1137,  ..., 0.4353, 0.4157, 0.2824],
          [0.0902, 0.0824, 0.1922,  ..., 0.5569, 0.4196, 0.2157],
          ...,
          [0.4314, 0.1882, 0.1569,  ..., 0.0745, 0.0627, 0.2000],
          [0.3451, 0.1804, 0.1569,  ..., 0.2235, 0.0235, 0.0980],
          [0.3451, 0.1569, 0.1843,  ..., 0.3804, 0.1529, 0.0588]],

         [[0.1725, 0.2314, 0.1569,  ..., 0.4510, 0.4588, 0.3686],
          [0.1333, 0.1843, 0.1961,  ..., 0.5647, 0.5490, 0.3765],
          [0.1059, 0.1333, 0.2863,  ..., 0.6824, 0.5490, 0.3059],
          ...,
          [0.5333, 0.2784, 0.2392,  ..., 0.1529, 0.1373, 0.2980],
          [0.4353, 0.2667, 0.2392,  ..., 0.3098, 0.0980, 0.1922],
          [0.4275, 0.2392, 0.2667,  ..., 0.4706, 0.2353, 0.1490]],

         [[0.0039, 0.0118, 0.0196,  ..., 0.1882, 0.2157, 0.1608],
          [0.0118, 0.0078, 0.0431,  ..., 0.3647, 0.3255, 0.1765],
          [0.0235, 0.0157, 0.0549,  ..., 0.5255, 0.3412, 0.1216],
          ...,
          [0.2627, 0.0235, 0.0118,  ..., 0.0745, 0.0588, 0.1725],
          [0.1686, 0.0235, 0.0157,  ..., 0.2196, 0.0235, 0.0824],
          [0.1647, 0.0078, 0.0471,  ..., 0.3333, 0.1216, 0.0549]]]]), tensor([[0., 0., 0., 0., 0., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
         0., 0.],
        [0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
         0., 0.],
        [0., 0., 0., 0., 0., 0., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
         0., 0.],
        [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 0.,
         0., 0.],
        [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 0.,
         0., 0.]])], tensor([26., 42., 33., 80., 83.], dtype=torch.float64)]
---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
~\AppData\Local\Temp/ipykernel_20644/2736533719.py in <module>
      4 
      5 model.train()
----> 6 model(sample)

D:\ProgramData\Anaconda3\lib\site-packages\torch\nn\modules\module.py in _call_impl(self, *input, **kwargs)
   1100         if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
   1101                 or _global_forward_hooks or _global_forward_pre_hooks):
-> 1102             return forward_call(*input, **kwargs)
   1103         # Do not call functions when jit is used
   1104         full_backward_hooks, non_full_backward_hooks = [], []

~\AppData\Local\Temp/ipykernel_20644/4062736071.py in forward(self, x)
     46     def forward(self, x):
     47         v = x[1]
---> 48         x = self.conv1(x[0])
     49         x = self.conv11(x)
     50         x = self.conv2(x)

D:\ProgramData\Anaconda3\lib\site-packages\torch\nn\modules\module.py in _call_impl(self, *input, **kwargs)
   1100         if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
   1101                 or _global_forward_hooks or _global_forward_pre_hooks):
-> 1102             return forward_call(*input, **kwargs)
   1103         # Do not call functions when jit is used
   1104         full_backward_hooks, non_full_backward_hooks = [], []

D:\ProgramData\Anaconda3\lib\site-packages\torch\nn\modules\container.py in forward(self, input)
    139     def forward(self, input):
    140         for module in self:
--> 141             input = module(input)
    142         return input
    143 

D:\ProgramData\Anaconda3\lib\site-packages\torch\nn\modules\module.py in _call_impl(self, *input, **kwargs)
   1100         if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
   1101                 or _global_forward_hooks or _global_forward_pre_hooks):
-> 1102             return forward_call(*input, **kwargs)
   1103         # Do not call functions when jit is used
   1104         full_backward_hooks, non_full_backward_hooks = [], []

D:\ProgramData\Anaconda3\lib\site-packages\torch\nn\modules\conv.py in forward(self, input)
    444 
    445     def forward(self, input: Tensor) -> Tensor:
--> 446         return self._conv_forward(input, self.weight, self.bias)
    447 
    448 class Conv3d(_ConvNd):

D:\ProgramData\Anaconda3\lib\site-packages\torch\nn\modules\conv.py in _conv_forward(self, input, weight, bias)
    440                             weight, bias, self.stride,
    441                             _pair(0), self.dilation, self.groups)
--> 442         return F.conv2d(input, weight, bias, self.stride,
    443                         self.padding, self.dilation, self.groups)
    444 

TypeError: conv2d() received an invalid combination of arguments - got (list, Parameter, Parameter, tuple, tuple, tuple, int), but expected one of:
 * (Tensor input, Tensor weight, Tensor bias, tuple of ints stride, tuple of ints padding, tuple of ints dilation, int groups)
      didn't match because some of the arguments have invalid types: (list, Parameter, Parameter, tuple, tuple, tuple, int)
 * (Tensor input, Tensor weight, Tensor bias, tuple of ints stride, str padding, tuple of ints dilation, int groups)
      didn't match because some of the arguments have invalid types: (list, Parameter, Parameter, tuple, tuple, tuple, int)

Maybe in inference, a different method from my forward method is called so the manipulation I did there isn’t being applied? I’m not sure.
Any help appreciated!

In inference, don’t forget to unpack the images and labels yielded by the loader, just like you do in training.

ds = test_loader.dataset
sample, labels = ds[0]  # <-- unpacking
# here maybe you need:
# sample[0] = sample[0].to(device)
# sample[1] = sample[1].to(device)
# labels = labels.to(device)

model.eval() 
model(sample)
1 Like