Ensemble trained model train using image and a model trained using some numerical values

1

I have 3 trained models. 2 models are trained using images. Another model is trained using some numerical value. How to combine them? If I combine any two models then there is no problem.

class MyEnsemble(nn.Module):
    def __init__(self, modelA, modelB, modelC, nb_classes=2):
        super(MyEnsemble, self).__init__()
        self.modelA = modelA
        self.modelB = modelB
        self.modelC = modelC
        # Remove last linear layer
        
        self.modelA.classifier[6] = nn.Identity()
        self.modelB.classifier[6] = nn.Identity()
        self.modelC.fc3= nn.Identity ()
        
        # Create new classifier
        self.classifier = nn.Linear(4096+4096+100, nb_classes)
        
    def forward(self, x1, x2, x3):
        x1 = self.modelB(x1)  # clone to make sure x is not changed by inplace methods
        #x1 = x1.unsqueeze()
        x2 = self.modelC(x2)
        x3 = self.modelC(x3)
        x = torch.cat((x1, x2, x3), dim=1)  
        #print(x.shape)      
        x = self.classifier(F.relu(x))
        return x

# Train your separate models
# ...
# We use pretrained torchvision models here
modelA = models.vgg16(pretrained=True)
num_ftrs = modelA.classifier[6].in_features
modelA.classifier[6] = nn.Linear(num_ftrs,2)
modelA.load_state_dict(torch.load('checkpoint1.pt'))
modelA.features[0]= nn.Conv2d(1, 64, kernel_size=3, stride=1, padding=1)
modelB = models.vgg16(pretrained=True)
num_ftrs = modelB.classifier[6].in_features
modelB.classifier[6] = nn.Linear(num_ftrs,2)
modelB.load_state_dict(torch.load('checkpoint2.pt'))
class Net(nn.Module):
    # define nn
    def __init__(self):
        super(Net, self).__init__()
        self.fc1 = nn.Linear(2, 100)
        self.fc2 = nn.Linear(100, 100)
        self.fc3 = nn.Linear(100, 2)
        #self.softmax = nn.Softmax(dim=1)

    def forward(self, X):
        X = F.relu(self.fc1(X))
        X = self.fc2(X)
        X = self.fc3(X)
        #X = self.softmax(X)
        return X
modelC = Net() 
modelC.load_state_dict(torch.load('checkpoint_SR.pt')) 
model = MyEnsemble(modelA, modelB, modelC)
print(model)     
x1 = torch.randn(1, 3, 224, 224)
x2 = torch.randn(1, 3, 224, 224)
x3=torch.randn(1, 2)
print(x3)
out = model(x1, x2, x3)
2

The code looks alright, at least I cannot find any obvious issue. What kind of errors are you seeing?

3

@ptrblck Thanks for your reply. The following error is coming


  File "", line 106, in <module>
    out = model(x1, x2, x3)

  File "", line 532, in __call__
    result = self.forward(*input, **kwargs)

  File "", line 64, in forward
    x2 = self.modelC(x2)

  File "C:\Users\Sampa\anaconda3\lib\site-packages\torch\nn\modules\module.py", line 532, in __call__
    result = self.forward(*input, **kwargs)

  File "", line 93, in forward
    X = F.relu(self.fc1(X))

  File "C:\Users\Sampa\anaconda3\lib\site-packages\torch\nn\modules\module.py", line 532, in __call__
    result = self.forward(*input, **kwargs)

  File "C:\Users\Sampa\anaconda3\lib\site-packages\torch\nn\modules\linear.py", line 87, in forward
    return F.linear(input, self.weight, self.bias)

  File "C:\Users\Sampa\anaconda3\lib\site-packages\torch\nn\functional.py", line 1372, in linear
    output = input.matmul(weight.t())

RuntimeError: size mismatch, m1: [672 x 224], m2: [2 x 100] at C:\w\1\s\tmp_conda_3.7_100118\conda\conda-bld\pytorch_1579082551706\work\aten\src\TH/generic/THTensorMath.cpp:136
4

Thanks for the update.
You are reusing modelC in the forward method so change it to:

        x1 = self.modelA(x1)
        x2 = self.modelB(x2)
        x3 = self.modelC(x3)

Also, the input to modelA should have a single input channel, since you’ve replaced the first conv layer, so use:

x1 = torch.randn(1, 1, 224, 224)
1 Like
5

Oh no…casual mistake. Thank you so much…

6

Thank you so much @ptrblck. one more question. after combining the models, is it necessary to train the final classification layer again?

7

Yes, I would think so as otherwise the self.classifier layer will stay randomly initialized.

8

Is it possible to use majority voting using modelA, modelB, and modelC instead of stackwise ensemble? If yes, can you please tell me the process?

9

Yes, you could get the predictions of each model and then use e.g. sklearn.ensemble.VotingClassifier, which also shows an example how to use it.

10

Thanks, @ptrblck for your reply. I have done this way… But I am not sure about how to use ‘fit’ in this example. can you please help me.

modelA = models.vgg16(pretrained=True)
num_ftrs = modelA.classifier[6].in_features
modelA.classifier[6] = nn.Linear(num_ftrs,2)
modelA.load_state_dict(torch.load('checkpoint1.pt'))
modelB = models.vgg16(pretrained=True)
num_ftrs = modelB.classifier[6].in_features
modelB.classifier[6] = nn.Linear(num_ftrs,2)
modelB.load_state_dict(torch.load('checkpoint2.pt'))
class Net(nn.Module):
    def __init__(self):
        super(Net, self).__init__()
        self.fc1 = nn.Linear(1, 100)
        self.fc2 = nn.Linear(100, 100)
        self.fc3 = nn.Linear(100, 2)
        #self.softmax = nn.Softmax(dim=1)

    def forward(self, X):
        X = F.relu(self.fc1(X))
        X = self.fc2(X)
        X = self.fc3(X)
        #X = self.softmax(X)
        return X
modelC = Net() 
modelC.load_state_dict(torch.load('checkpoint_3.pt')) 
or param in modelA.parameters():
    param.requires_grad_(False)
for param in modelB.parameters():
    param.requires_grad_(False)
for param in modelC.parameters():
    param.requires_grad_(False)   
x1 = torch.randn(6, 3, 224, 224)
x2 = torch.randn(6, 3, 224, 224)
x3=torch.randn(6, 1)
y = np.array([1, 1, 1, 2, 2, 2])
eclf1 = VotingClassifier(estimators=[modelA, modelB, modelC], voting='hard')
eclf = eclf1.fit(.....)