How to ensemble two model in pytorch?

1

I want to ensemble Mode1A and Model1B, But there is a run time error
Expected 4-dimensional input for 4-dimensional weight 8 3, but got 2-dimensional input of size [1, 25088] instead
Please help me

class MyModelA(nn.Module):
    def __init__(self):
        super(MyModelA, self).__init__()
        self.layer1 = nn.Sequential(
            nn.Conv2d(in_channels=3, out_channels=8, kernel_size=3,stride=1, padding=1),
            nn.BatchNorm2d(8),
            nn.ReLU(),
            nn.MaxPool2d(kernel_size=2))
        self.layer2 = nn.Sequential(
            nn.Conv2d(in_channels=8, out_channels=16, kernel_size=5, stride=1, padding=2),
            nn.BatchNorm2d(16),
            nn.ReLU(),
            nn.MaxPool2d(kernel_size=2)) 
        self.layer3 = nn.Sequential(
            nn.Conv2d(in_channels=16, out_channels=32, kernel_size=5, stride=1, padding=2),
            nn.BatchNorm2d(32),
            nn.ReLU(),
            nn.MaxPool2d(kernel_size=2))     
        self.fc = nn.Linear(25088, 2)
        
    def forward(self, x):
        out = self.layer1(x)
        out = self.layer2(out)
        out = self.layer3(out)
        #out = self.layer4(out)
        out = out.reshape(out.size(0), -1)
        out = self.fc(out)
        return out
class MyModelB(nn.Module):
..
class MyEnsemble(nn.Module):
    def __init__(self, modelA, modelB):
        super(MyEnsemble, self).__init__()
        self.modelA = modelA
        self.modelB = modelB
        #self.classifier = nn.Linear(4, 2)
        
    def forward(self, x1, x2):
        head1a, head1b = self.modelA(x1)
        head2 = self.modelB(head1a)
        x = torch.cat((head1b, head2), dim=1)
        return x

# Create models and load state_dicts    
modelA = MyModelA()
modelB = MyModelB()
# Load state dicts
modelA.load_state_dict(torch.load('checkpoint1.pt'))
modelB.load_state_dict(torch.load('checkpoint2.pt'))

model = MyEnsemble(modelA, modelB)
x1, x2 = torch.randn(1,25088), torch.randn(1, 25088)
output = model(x1, x2)
2

MyModelA uses a nn.Conv2d layer as the first layer, which expects a 4-dimensional input with the shape [batch_size, channels, height, width].
In your code snippet, you are defining x1 as a 2-dimensional tensor, which will throw this error.
Based on the shape, it seems as if you would only want to use self.fc on this input?

3

@ptrblck I saw your Ensemble code and try to ensemble my model with this code. I want to combine the classifier only. Can I use this for resent model? If yes, how to use?

4

Would you like to combine the penultimate activations and train a new classifier on top or did I misunderstand your use case?

5

I have 2 similar dataset A and B . So I want to train the model in A and B and save the model in checkpoint1 and checkpoint2. Now I want to ensemble this two model. My question is how to combine this two model?

6

Yes…you are right.I want to combine the penultimate activations and train a new classifier.

7

My example should do exactly this. Would that work for you?

3 Likes
8

Thank you… I will try this & Let you know

9

@ptrblck
Please check: is this correct?
‘’’
class MyEnsemble(nn.Module):
def init(self, modelA, modelB, nb_classes=2):
super(MyEnsemble, self).init()
self.modelA = modelA
self.modelB = modelB

    self.modelA.features[10] = nn.Identity() 
    model_ft.features=model_ft.features[:--11]
    
     self.modelB.features[10] = nn.Identity() 
    model_ft1.features=model_ft1.features[:--11]

    
    self.features = nn.Sequential(nn.Conv2d(256+256, 256+256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1)), nn.ReLU(inplace=True), nn.MaxPool2d(kernel_size=3, stride=2, padding=0, dilation=1, ceil_mode=False))
   
    self.classifier = nn.Sequential(   #this add multple fc layers
        nn.Dropout(p=0.5, inplace=False),
        nn.Linear(in_features=9216, out_features=4096, bias=True),
        nn.ReLU(inplace=True),
        nn.Dropout(p=0.5, inplace=False),
        nn.Linear(in_features=4096, out_features=4096, bias=True),
        nn.ReLU(inplace=True),
        nn.Linear(in_features=4096, out_features=1000, bias=True)
    )

    
def forward(self, x):
    x1 = self.modelA(x.clone())          
  x1 = x1.view(x1.size(0), -1)
    x2 = self.modelB(x)
    x2 = x2.view(x2.size(0), -1)
    x = torch.cat((x1, x2), dim=1)
    
    x = self.features(F.relu(x))
    return x

‘’’

10

It’s unclear how model_ft is defines, but features[:--11] looks wrong as you could also just remove the double minus signs or fix the typo.

Besides that self.classifier is never used.

11

@ptrblck Thank you for your reply.

This is how model look:

‘’’
MyEnsemble(
(modelA): AlexNet(
(features): Sequential(
(0): Conv2d(3, 64, kernel_size=(11, 11), stride=(4, 4), padding=(2, 2))
(1): ReLU(inplace=True)
(2): MaxPool2d(kernel_size=3, stride=2, padding=0, dilation=1, ceil_mode=False)
(3): Conv2d(64, 192, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2))
(4): ReLU(inplace=True)
(5): MaxPool2d(kernel_size=3, stride=2, padding=0, dilation=1, ceil_mode=False)
(6): Conv2d(192, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
(7): ReLU(inplace=True)
(8): Conv2d(384, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
(9): ReLU(inplace=True)
(10): Identity()
)
)
(modelB): AlexNet(
(features): Sequential(
(0): Conv2d(3, 64, kernel_size=(11, 11), stride=(4, 4), padding=(2, 2))
(1): ReLU(inplace=True)
(2): MaxPool2d(kernel_size=3, stride=2, padding=0, dilation=1, ceil_mode=False)
(3): Conv2d(64, 192, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2))
(4): ReLU(inplace=True)
(5): MaxPool2d(kernel_size=3, stride=2, padding=0, dilation=1, ceil_mode=False)
(6): Conv2d(192, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
(7): ReLU(inplace=True)
(8): Conv2d(384, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
(9): ReLU(inplace=True)
(10): Identity()
)
)
(features): Sequential(
(0): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
(1): ReLU(inplace=True)
(2): MaxPool2d(kernel_size=3, stride=2, padding=0, dilation=1, ceil_mode=False)
)
(classifier): Sequential(
(0): Dropout(p=0.5, inplace=False)
(1): Linear(in_features=9216, out_features=4096, bias=True)
(2): ReLU(inplace=True)
(3): Dropout(p=0.5, inplace=False)
(4): Linear(in_features=4096, out_features=4096, bias=True)
(5): ReLU(inplace=True)
(6): Linear(in_features=4096, out_features=1000, bias=True)
)
)

‘’‘’
Please help to concatenate the two alexnet model in last convolution layer. After this concatenate i added this convolution layer and all linear layers.
rest code is:

‘’’
model_ft = models.alexnet(pretrained=True)

model_ft1 = models.alexnet(pretrained=True)

del model_ft.classifier #delete all fully connected layer

del model_ft.avgpool #delete AdaptiveAvgPool2d

del model_ft1.classifier #delete all fully connected layer

del model_ft1.avgpool #delete AdaptiveAvgPool2d

num_ftrs = model_ft.features[10].get_parameter

num_ftrs1 = model_ft1.features[10].get_parameter

model_ft.features[10]= nn.Conv2d(256, 256, 2)

model_ft1.features[10]= nn.Conv2d(256, 256, 2)

model = MyEnsemble(model_ft,model_ft1).to(device)

criterion = nn.CrossEntropyLoss()

model = MyEnsemble(model_ft, model_ft1)

optimizer_ft = optim.SGD(model.parameters(), lr=0.001, momentum=0.9)

exp_lr_scheduler = lr_scheduler.StepLR(optimizer_ft, step_size=20, gamma=0.1)

print(model)

‘’’

12

@ptrblck
model_ft.features=model_ft.features[:–11]
This line removes 11 and 12th layer of 1st alexnet model features, i.e. ReLu and MaxPool2d layer.
I used self.classifier to add multple fc layers after concatenation.

13

@ptrblck
please tell me how to resolve this error?

usr/local/lib/python3.10/dist-packages/torch/nn/modules/conv.py in _conv_forward(self, input, weight, bias)
457 weight, bias, self.stride,
458 _pair(0), self.dilation, self.groups)
→ 459 return F.conv2d(input, weight, bias, self.stride,
460 self.padding, self.dilation, self.groups)
461

RuntimeError: Expected 3D (unbatched) or 4D (batched) input to conv2d, but got input of size: [16, 86528]

14

You are passing a 2D tensor to nn.Conv2d while either a batched 4D tensor in the shape [batch_size, channels, height, width] or an unbatched 3D tensor in the shape [channels, height, width] is expected.

15

Thank you @ptrblck .
This is the code I am passing:
‘’’
self.features = nn.Conv2d(in_channels=256+256, out_channels=256, kernel_size=3)
‘’

16

and @ptrblck

how to initialize it:
‘’’
def init(self, modelA, modelB, out_channels=256, kernel_size=3):
‘’’
correct??