RuntimeError: mat1 dim 1 must match mat2 dim 0 with image classification using resnet

Hi, I’m getting this error when I try to use a resnet net:

Traceback (most recent call last):
  File "test.py", line 58, in <module>
    _, concat_logits, _, _, _ = net(img)
  File "/home/julio/anaconda3/envs/logo2k/lib/python3.7/site-packages/torch/nn/modules/module.py", line 722, in _call_impl
    result = self.forward(*input, **kwargs)
  File "/home/julio/anaconda3/envs/logo2k/lib/python3.7/site-packages/torch/nn/parallel/data_parallel.py", line 153, in forward
    return self.module(*inputs[0], **kwargs[0])
  File "/home/julio/anaconda3/envs/logo2k/lib/python3.7/site-packages/torch/nn/modules/module.py", line 722, in _call_impl
    result = self.forward(*input, **kwargs)
  File "/home/julio/wdir/code/Logo-2k-plus-Dataset/Drna-master/core/model_resnet.py", line 84, in forward
    resnet_out, rpn_feature, feature = self.pretrained_model(x)
  File "/home/julio/anaconda3/envs/logo2k/lib/python3.7/site-packages/torch/nn/modules/module.py", line 722, in _call_impl
    result = self.forward(*input, **kwargs)
  File "/home/julio/wdir/code/Logo-2k-plus-Dataset/Drna-master/core/resnet.py", line 154, in forward
    x = self.fc(x)
  File "/home/julio/anaconda3/envs/logo2k/lib/python3.7/site-packages/torch/nn/modules/module.py", line 722, in _call_impl
    result = self.forward(*input, **kwargs)
  File "/home/julio/anaconda3/envs/logo2k/lib/python3.7/site-packages/torch/nn/modules/linear.py", line 91, in forward
    return F.linear(input, self.weight, self.bias)
  File "/home/julio/anaconda3/envs/logo2k/lib/python3.7/site-packages/torch/nn/functional.py", line 1674, in linear
    ret = torch.addmm(bias, input, weight.t())
RuntimeError: mat1 dim 1 must match mat2 dim 0

And here is my code:

class ResNet(nn.Module):
    def __init__(self, block, layers, num_classes=1000):
        self.inplanes = 64
        super(ResNet, self).__init__()
        self.conv1 = nn.Conv2d(3, 64, kernel_size=7, stride=2, padding=3,
                               bias=False)
        self.bn1 = nn.BatchNorm2d(64)
        self.relu = nn.ReLU(inplace=True)
        self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)
        self.layer1 = self._make_layer(block, 64, layers[0])
        self.layer2 = self._make_layer(block, 128, layers[1], stride=2)
        self.layer3 = self._make_layer(block, 256, layers[2], stride=2)
        self.layer4 = self._make_layer(block, 512, layers[3], stride=2)
        self.avgpool = nn.AvgPool2d(7)
        self.fc = nn.Linear(512 * block.expansion, num_classes)

        for m in self.modules():
            if isinstance(m, nn.Conv2d):
                n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels
                m.weight.data.normal_(0, math.sqrt(2. / n))
            elif isinstance(m, nn.BatchNorm2d):
                m.weight.data.fill_(1)
                m.bias.data.zero_()

    def _make_layer(self, block, planes, blocks, stride=1):
        downsample = None
        if stride != 1 or self.inplanes != planes * block.expansion:
            downsample = nn.Sequential(
                nn.Conv2d(self.inplanes, planes * block.expansion,
                          kernel_size=1, stride=stride, bias=False),
                nn.BatchNorm2d(planes * block.expansion),
            )

        layers = []
        layers.append(block(self.inplanes, planes, stride, downsample))
        self.inplanes = planes * block.expansion
        for i in range(1, blocks):
            layers.append(block(self.inplanes, planes))

        return nn.Sequential(*layers)

    def forward(self, x):
        x = self.conv1(x)
        x = self.bn1(x)
        x = self.relu(x)
        x = self.maxpool(x)

        x = self.layer1(x)
        x = self.layer2(x)
        x = self.layer3(x)
        x = self.layer4(x)
        feature1 = x
        x = self.avgpool(x)
        x = x.view(x.size(0), -1)
        x = nn.Dropout(p=0.5)(x)
        feature2 = x
        x = self.fc(x)

        return x, feature1, feature2

The error is raised when I run “_, concat_logits, _, _, _ = net(img)” so I investigated which was the shape of img. It is: “Torch.size([2, 3, 448, 448])”. In addition I see that the error originally comes from the file “resnet.py” in: " x = self.fc(x)" so I also investigated which was the shape of “x”. It is: “Torch.size([2, 2048])”. I think that might help you.

Where am I commiting a mistake? I saw other similar posts but I couldn’t find my error. Sorry for the inconvenience.

The in_features of self.fc are defined as 512, while the incoming number of features is 2048.
Change the linear layer and it should work:

class ResNet(nn.Module):
    def __init__(self, block, layers, num_classes=1000):
        self.inplanes = 64
        super(ResNet, self).__init__()
        self.conv1 = nn.Conv2d(3, 64, kernel_size=7, stride=2, padding=3,
                               bias=False)
        self.bn1 = nn.BatchNorm2d(64)
        self.relu = nn.ReLU(inplace=True)
        self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)
        self.layer1 = self._make_layer(block, 64, layers[0])
        self.layer2 = self._make_layer(block, 128, layers[1], stride=2)
        self.layer3 = self._make_layer(block, 256, layers[2], stride=2)
        self.layer4 = self._make_layer(block, 512, layers[3], stride=2)
        self.avgpool = nn.AvgPool2d(7)
        self.fc = nn.Linear(2048, num_classes)

        for m in self.modules():
            if isinstance(m, nn.Conv2d):
                n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels
                m.weight.data.normal_(0, math.sqrt(2. / n))
            elif isinstance(m, nn.BatchNorm2d):
                m.weight.data.fill_(1)
                m.bias.data.zero_()

    def _make_layer(self, block, planes, blocks, stride=1):
        downsample = None
        if stride != 1 or self.inplanes != planes * block.expansion:
            downsample = nn.Sequential(
                nn.Conv2d(self.inplanes, planes * block.expansion,
                          kernel_size=1, stride=stride, bias=False),
                nn.BatchNorm2d(planes * block.expansion),
            )

        layers = []
        layers.append(block(self.inplanes, planes, stride, downsample))
        self.inplanes = planes * block.expansion
        for i in range(1, blocks):
            layers.append(block(self.inplanes, planes))

        return nn.Sequential(*layers)

    def forward(self, x):
        x = self.conv1(x)
        x = self.bn1(x)
        x = self.relu(x)
        x = self.maxpool(x)

        x = self.layer1(x)
        x = self.layer2(x)
        x = self.layer3(x)
        x = self.layer4(x)
        feature1 = x
        x = self.avgpool(x)
        x = x.view(x.size(0), -1)
        x = nn.Dropout(p=0.5)(x)
        feature2 = x
        x = self.fc(x)

        return x, feature1, feature2

model = ResNet(torchvision.models.resnet.BasicBlock, [2, 2, 2, 2])
x = torch.randn(2, 3, 448, 448)
out = model(x)

Note that I don’t know, which block you are using or how you are initializing the model, so I just used the standard resnet18 approach.

Hi ptrblck, thank you for your answer.

I made the changes that you told me but the error persists.
For initializing the model I am using:

model = Resnet(torchvision.models.resnet.Bottleneck, [3,8,36,3])

I tried to initialize the model in your way but the error also persists.
I think the mistake I’m commiting might be in the forward function.
Here is the new error trace with some prints related to shapes:

Size of self.fc in the init function: Linear(in_features=2048, out_features=2341, bias=True)
The shape of the tensor passed to the model is: torch.Size([2, 3, 448, 448])
The shape of x in the forward function is: torch.Size([2, 2048])
Traceback (most recent call last):
  File "test.py", line 58, in <module>
    _, concat_logits, _, _, _ = net(img)
  File "/home/julio/anaconda3/envs/logo2k/lib/python3.7/site-packages/torch/nn/modules/module.py", line 722, in _call_impl
    result = self.forward(*input, **kwargs)
  File "/home/julio/anaconda3/envs/logo2k/lib/python3.7/site-packages/torch/nn/parallel/data_parallel.py", line 153, in forward
    return self.module(*inputs[0], **kwargs[0])
  File "/home/julio/anaconda3/envs/logo2k/lib/python3.7/site-packages/torch/nn/modules/module.py", line 722, in _call_impl
    result = self.forward(*input, **kwargs)
  File "/home/julio/wdir/code/Logo-2k-plus-Dataset/Drna-master/core/model_resnet.py", line 85, in forward
    resnet_out, rpn_feature, feature = self.pretrained_model(x)
  File "/home/julio/anaconda3/envs/logo2k/lib/python3.7/site-packages/torch/nn/modules/module.py", line 722, in _call_impl
    result = self.forward(*input, **kwargs)
  File "/home/julio/wdir/code/Logo-2k-plus-Dataset/Drna-master/core/resnet.py", line 156, in forward
    x = self.fc(x)
  File "/home/julio/anaconda3/envs/logo2k/lib/python3.7/site-packages/torch/nn/modules/module.py", line 722, in _call_impl
    result = self.forward(*input, **kwargs)
  File "/home/julio/anaconda3/envs/logo2k/lib/python3.7/site-packages/torch/nn/modules/linear.py", line 91, in forward
    return F.linear(input, self.weight, self.bias)
  File "/home/julio/anaconda3/envs/logo2k/lib/python3.7/site-packages/torch/nn/functional.py", line 1674, in linear
    ret = torch.addmm(bias, input, weight.t())
RuntimeError: mat1 dim 1 must match mat2 dim 0

I got a new shape mismatch using your initialization, which can be solved by setting the in_features of self.fc to 8192.
The error message gives the activation shape as well as the weight shape. If you want to keep the activation equal, you would have to adapt the in_features.

Sorry about this question but I’m a bit newbie:

This change has to be made in the init function? I think it can be here but I’m not sure:

self.fc = nn.Linear(8192, num_classes)

Yes, you would have to change it in the __init__ of your model while you are initializing the module. Your code should work or are you seeing any new issues (it worked in my case with this change)?

In my case it didn’t worked. I still get the same error. Maybe the mistake that I’m commiting isn’t there? Here is the new error trace:

Size of self.fc in the init function: Linear(in_features=8192, out_features=2341, bias=True)
The shape of the tensor passed to the model is: torch.Size([2, 3, 448, 448])
The shape of x in the forward function is: torch.Size([2, 2048])
Traceback (most recent call last):
  File "test.py", line 58, in <module>
    _, concat_logits, _, _, _ = net(img)
  File "/home/julio/anaconda3/envs/logo2k/lib/python3.7/site-packages/torch/nn/modules/module.py", line 722, in _call_impl
    result = self.forward(*input, **kwargs)
  File "/home/julio/anaconda3/envs/logo2k/lib/python3.7/site-packages/torch/nn/parallel/data_parallel.py", line 153, in forward
    return self.module(*inputs[0], **kwargs[0])
  File "/home/julio/anaconda3/envs/logo2k/lib/python3.7/site-packages/torch/nn/modules/module.py", line 722, in _call_impl
    result = self.forward(*input, **kwargs)
  File "/home/julio/wdir/code/Logo-2k-plus-Dataset/Drna-master/core/model_resnet.py", line 85, in forward
    resnet_out, rpn_feature, feature = self.pretrained_model(x)
  File "/home/julio/anaconda3/envs/logo2k/lib/python3.7/site-packages/torch/nn/modules/module.py", line 722, in _call_impl
    result = self.forward(*input, **kwargs)
  File "/home/julio/wdir/code/Logo-2k-plus-Dataset/Drna-master/core/resnet.py", line 156, in forward
    x = self.fc(x)
  File "/home/julio/anaconda3/envs/logo2k/lib/python3.7/site-packages/torch/nn/modules/module.py", line 722, in _call_impl
    result = self.forward(*input, **kwargs)
  File "/home/julio/anaconda3/envs/logo2k/lib/python3.7/site-packages/torch/nn/modules/linear.py", line 91, in forward
    return F.linear(input, self.weight, self.bias)
  File "/home/julio/anaconda3/envs/logo2k/lib/python3.7/site-packages/torch/nn/functional.py", line 1674, in linear
    ret = torch.addmm(bias, input, weight.t())
RuntimeError: mat1 dim 1 must match mat2 dim 0

Could you test my modified code and check, if you are still seeing the error?

class ResNet(nn.Module):
    def __init__(self, block, layers, num_classes=1000):
        self.inplanes = 64
        super(ResNet, self).__init__()
        self.conv1 = nn.Conv2d(3, 64, kernel_size=7, stride=2, padding=3,
                               bias=False)
        self.bn1 = nn.BatchNorm2d(64)
        self.relu = nn.ReLU(inplace=True)
        self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)
        self.layer1 = self._make_layer(block, 64, layers[0])
        self.layer2 = self._make_layer(block, 128, layers[1], stride=2)
        self.layer3 = self._make_layer(block, 256, layers[2], stride=2)
        self.layer4 = self._make_layer(block, 512, layers[3], stride=2)
        self.avgpool = nn.AvgPool2d(7)
        self.fc = nn.Linear(8192, num_classes)

        for m in self.modules():
            if isinstance(m, nn.Conv2d):
                n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels
                m.weight.data.normal_(0, math.sqrt(2. / n))
            elif isinstance(m, nn.BatchNorm2d):
                m.weight.data.fill_(1)
                m.bias.data.zero_()

    def _make_layer(self, block, planes, blocks, stride=1):
        downsample = None
        if stride != 1 or self.inplanes != planes * block.expansion:
            downsample = nn.Sequential(
                nn.Conv2d(self.inplanes, planes * block.expansion,
                          kernel_size=1, stride=stride, bias=False),
                nn.BatchNorm2d(planes * block.expansion),
            )

        layers = []
        layers.append(block(self.inplanes, planes, stride, downsample))
        self.inplanes = planes * block.expansion
        for i in range(1, blocks):
            layers.append(block(self.inplanes, planes))

        return nn.Sequential(*layers)

    def forward(self, x):
        x = self.conv1(x)
        x = self.bn1(x)
        x = self.relu(x)
        x = self.maxpool(x)

        x = self.layer1(x)
        x = self.layer2(x)
        x = self.layer3(x)
        x = self.layer4(x)
        feature1 = x
        x = self.avgpool(x)
        x = x.view(x.size(0), -1)
        x = nn.Dropout(p=0.5)(x)
        feature2 = x
        x = self.fc(x)

        return x, feature1, feature2

import math
model = ResNet(torchvision.models.resnet.Bottleneck, [3,8,36,3])
x = torch.randn(2, 3, 448, 448)
out = model(x)

First of all, thank you for your replies ptrblck.

Yes, I checked if your code works and it does not raise new errors doing this:

import math
model = ResNet(torchvision.models.resnet.Bottleneck, [3,8,36,3])
x = torch.randn(2, 3, 448, 448)
out = model(x)

The problem is that my error persists, that is why I think that the error can be located in a different line of the code.

Could the error be in the way the model is defined? Here are some lines of the test.py file in which the model is defined:

# define model
net = model.attention_net(topN=PROPOSAL_NUM)
ckpt = torch.load(test_model)
net.load_state_dict(ckpt, strict=False)
net = net.cuda()
net = DataParallel(net)
creterion = torch.nn.CrossEntropyLoss()

# evaluate on train set
train_loss = 0
train_correct = 0
total = 0
net.eval()

for i, data in enumerate(trainloader):
    with torch.no_grad():
        img, label = data[0].cuda(), data[1]
        batch_size = img.size(0)
        print('The shape of the tensor passed to the model is: ' + str(img.shape))
        _, concat_logits, _, _, _ = net(img)

In that last line, the error is raised.

Based on your code snippet it seems you are using another model by calling into attention_net.
Could you write a code snippet to reproduce this error with this model (same as before)?

Sure, here is the code snippet of the attention_net class:

class attention_net(nn.Module):
    def __init__(self, topN=4):
        super(attention_net, self).__init__()
        self.pretrained_model = resnet.ResNet(torchvision.models.resnet.Bottleneck, [3,8,36,3])
        self.pretrained_model.avgpool = nn.AdaptiveAvgPool2d(1)
        self.pretrained_model.fc = nn.Linear(2*512, NUM_CLASSES)
        self.proposal_net = ProposalNet()
        self.topN = topN
        #self.aug_net = aug_net(NUM_CLASSES)
        self.concat_net = nn.Linear(2048 * (CAT_NUM + 1), 2341)
        self.partcls_net = nn.Linear(512 * 2, 2341)
        _, edge_anchors, _ = generate_default_anchor_maps()
        self.pad_side = 224
        self.edge_anchors = (edge_anchors + 224).astype(np.int)

    def forward(self, x):
        resnet_out, rpn_feature, feature = self.pretrained_model(x)
        print(resnet_out.shape)
        x_pad = F.pad(x, (self.pad_side, self.pad_side, self.pad_side, self.pad_side), mode='constant', value=0)
        batch = x.size(0)
        # we will reshape rpn to shape: batch * nb_anchor
        rpn_score = self.proposal_net(rpn_feature.detach())
        all_cdds = [ np.concatenate((x.reshape(-1, 1), self.edge_anchors.copy(), np.arange(0, len(x)).reshape(-1, 1)), axis=1)
            for x in rpn_score.data.cpu().numpy()]
        top_n_cdds = [hard_nms(x, topn=self.topN, iou_thresh=0.25) for x in all_cdds]
        top_n_cdds = np.array(top_n_cdds)
        top_n_index = top_n_cdds[:, :, -1].astype(np.int)
        top_n_index = torch.from_numpy(top_n_index).cuda()
        top_n_prob = torch.gather(rpn_score, dim=1, index=top_n_index)
        part_imgs = torch.zeros([batch, self.topN, 3, 224, 224]).cuda()
        for i in range(batch):
            for j in range(self.topN):
                [y0, x0, y1, x1] = top_n_cdds[i][j, 1:5].astype(np.int)
                part_imgs[i:i + 1, j] = F.upsample(x_pad[i:i + 1, :, y0:y1, x0:x1], size=(224, 224), mode='bilinear', align_corners=True)
        part_imgs = part_imgs.view(batch * self.topN, 3, 224, 224)
        _, _, part_features = self.pretrained_model(part_imgs.detach())
        part_feature = part_features.view(batch, self.topN, -1)
        part_feature = part_feature[:, :CAT_NUM, ...].contiguous()
        part_feature = part_feature.view(batch, -1)
        # concat_logits have the shape: B*2341
        concat_out = torch.cat([part_feature, feature], dim=1)
        concat_logits = self.concat_net(concat_out)
        #print(concat_logits.shape)
        raw_logits = resnet_out
        aug_logits = self.aug_net(part_features)
        # part_logits have the shape: B*N*2341
        part_logits = self.partcls_net(part_features).view(batch, self.topN, -1)
        return [raw_logits, aug_logits, concat_logits, part_logits, top_n_index, top_n_prob]

I would recommend to add print statements for the current tensor shape in the forward methods in all custom modules to narrow down where the shape mismatch is coming from.
Your current code snippets uses another undefined model (ProposalNet) so that I cannot debug is myself.

kindly, did you find a solution to this problem??

Doaa