here is the code:
‘’’
class MyEnsemble(nn.Module):
def init(self, modelA, modelB, nb_classes=4):
super(MyEnsemble, self).init()
self.modelA = modelA
self.modelB = modelB
# Remove last linear layer
#The nn. Identity module will just return the input without any
#manipulation and can be used to e.g. replace other layers.
#self.modelA.classifier = nn.Conv2d(64, 128, 5)
self.modelA.classifier = nn.Identity() #alexnet
self.modelB.classifier= nn.Identity() #googlenet
# Create new classifier
#nn. Linear(n,m) is a module that creates single layer feed forward network with n inputs and m output.
#self.classifier = nn.Linear(1920+2048, nb_classes) #Fully-connected layers, also known as linear layers.
#new line added for middle layer fusion ---------------
self.classifier = nn.MaxPool3d(4096+4096, nb_classes) # Middle layer (pooling layer)
#self.classifier = nn.Conv3d(1920+2048, nb_classes) #First layer (a convolutional layer)
#--------------------------
# mat1 and mat2 shapes cannot be multiplied (16x514 and 512x4096)
def forward(self, x):
x1 = self.modelA(x.clone()) # clone to make sure x is not changed by inplace methods
x1 = x1.view(x1.size(0), -1)
x2 = self.modelB(x)
x2 = x2.view(x2.size(0), -1)
x = torch.cat((x1, x2), dim=1)
x = self.classifier(F.relu(x))
return x
‘’’
#Load a pretrained model and reset final fully connected layer.
model_ft = models.alexnet(weights=True)
model_ft1 = models.vgg11(weights=True)
‘’’
model_ft
ct = 0
for child in model_ft.children():
ct += 1
if ct < 4:
for param in child.parameters():
param.requires_grad = False
#frozen model_ft1
ct = 0
for child in model_ft1.children():
ct += 1
if ct < 4:
for param in child.parameters():
param.requires_grad = False
‘’’
#num_ftrs = list(model_ft.children())[0][:-2]
num_ftrs = model_ft.classifier[6].in_features
#num_ftrs = self.axial.model.classifier[0].in_features
#num_ftrs = model_ft.classifier.in_features
num_ftrs1 = model_ft1.classifier[6].in_features
‘’’
Here the size of each output sample is set to 2.
Alternatively, it can be generalized to nn.Linear(num_ftrs, len(class_names)).
model_ft.classifier= nn.Linear(num_ftrs, 2)
model_ft1.fc= nn.Linear(num_ftrs1, 2)
‘’’
#intermediate new-----------------------
model_ft.classifier= nn.MaxPool3d(num_ftrs, 2)
model_ft1.classifier= nn.MaxPool3d(num_ftrs1, 2)
#----------------------------
#model_ft = model_ft.to(device)
model = MyEnsemble(model_ft,model_ft1).to(device)
criterion = nn.CrossEntropyLoss()
‘’’
Freeze these models
for param in model_ft.parameters():
param.requires_grad_(False)
for param in model_ft1.parameters():
param.requires_grad_(False)
‘’’
Create ensemble model
model = MyEnsemble(model_ft, model_ft1)
Observe that all parameters are being optimized
optimizer_ft = optim.SGD(model.parameters(), lr=0.001, momentum=0.9)
Decay LR by a factor of 0.1 every 7 epochs
exp_lr_scheduler = lr_scheduler.StepLR(optimizer_ft, step_size=20, gamma=0.1)
‘’’
model = train_model(model, criterion, optimizer_ft, exp_lr_scheduler, num_epochs=8)
‘’’
The error is:
Epoch 0/7 ----------
RuntimeError Traceback (most recent call last)
in ----> 1 model = train_model(model, criterion, optimizer_ft, exp_lr_scheduler, num_epochs=8)
6 frames
/usr/local/lib/python3.9/dist-packages/torch/nn/functional.py in _max_pool3d(input, kernel_size, stride, padding, dilation, ceil_mode, return_indices) 866 if stride is None: 867 stride = torch.jit.annotate(List[int], []) → 868 return torch.max_pool3d(input, kernel_size, stride, padding, dilation, ceil_mode) 869 870
RuntimeError: non-empty 4D or 5D (batch mode) tensor expected for input
‘’’