How to create a class for multiple inputs? the inputs are the 2 tensors: (331, 3, 224, 224) and (331, 3, 224, 224)

1

I have the DataLoader class for 1 image and labels:

class CoronalDataset(Dataset):
    def __init__(self, pet_type='interim', train=True, transform=None):
        train_valid = 'train' if train else 'valid'
        # Create the Tensor from the numpy array 
        self.x = torch.from_numpy(np.load(f'{DATASET_DIR}/{pet_type}/rgb/x_{train_valid}_coronal.npy')).permute(0, 3, 1, 2).type(torch.FloatTensor)
        print("Shape of X: ", self.x.shape)
        print(self.x.dtype)
        
        self.y = torch.from_numpy(np.load(f'{DATASET_DIR}/{pet_type}/rgb/y_{train_valid}_123_45.npy'))
        print("Shape of Y:", self.y.unique(return_counts=True))
        labellist = self.y.tolist()
        result = sorted([(x, labellist.count(x)) for x in [[1.0, 0.0],[0.0, 1.0]]], key=lambda y: y[1])
        for elem in result:
            print('{} {}'.format(elem[0], elem[1]))

        print(self.y.dtype)
        
        self.len = len(self.x)
        self.transform = transform
        
    def __getitem__(self, index):
        sample = self.x[index], self.y[index]

        if self.transform:
            sample = self.transform(sample1)

            
        return sample
          
    def __len__(self):
        return self.len
2

Could you describe your use case a bit more and in particular what kind of class you would like to implement? Would it be a model with two inputs or e.g. the transformation using two image tensors?

3

Yes sir, that is the model with two inputs.

4

In that case you could write a custom model and use these two inputs directly in the forward method.
Here is a small example:

class MyModel(nn.Module):
    def __init__(self):
        super().__init__()
        self.fc1 = nn.Linear(10, 10)
        self.fc2 = nn.Linear(10, 10)
        
    def forward(self, x1, x2):
        x1 = self.fc1(x1)
        x2 = self.fc2(x2)
        x = x1 + x2
        return x

model = MyModel()
x1 = torch.randn(1, 10)
x2 = torch.randn(1, 10)
out = model(x1, x2)
1 Like
5

Is this right, sir?

class ResNet(nn.Module):
    def __init__(self, block, num_blocks, num_classes=10):
        super(ResNet, self).__init__()
        self.in_planes = 64

        self.conv1 = nn.Conv2d(3, 64, kernel_size=3,
                               stride=1, padding=1, bias=False)
        self.conv2 = nn.Conv2d(3, 64, kernel_size=3,
                               stride=1, padding=1, bias=False)
        self.bn1 = nn.BatchNorm2d(64)
        self.layer1 = self._make_layer(block, 64, num_blocks[0], stride=1)
        self.layer2 = self._make_layer(block, 128, num_blocks[1], stride=2)
        self.layer3 = self._make_layer(block, 256, num_blocks[2], stride=2)
        self.layer4 = self._make_layer(block, 512, num_blocks[3], stride=2)
        self.linear = nn.Linear(512*block.expansion, num_classes)

    def _make_layer(self, block, planes, num_blocks, stride):
        strides = [stride] + [1]*(num_blocks-1)
        layers = []
        for stride in strides:
            layers.append(block(self.in_planes, planes, stride))
            self.in_planes = planes * block.expansion
        return nn.Sequential(*layers)

    def forward(self, x, y):
        out1 = F.relu(self.bn1(self.conv1(x)))
        out2 = F.relu(self.bn1(self.conv2(y)))
        out = self.layer1(out)
        out = self.layer2(out)
        out = self.layer3(out)
        out = self.layer4(out)
        out = F.avg_pool2d(out, 4)
        out = out.view(out.size(0), -1)
        out = self.linear(out)
        return out
6

No, since out1 and out2 are not used, while the undefined out is used instead, which would raise an error.

7

so I will have out = out1 + out2, is this right sir?

8

This could work assuming that both have the same shape or can be broadcasted.
To quickly check your model, you can create random tensors and execute the forward as well as backward pass.

9

Dear Sir, your suggestion is right. But how can I create the class DataLoader for input that has 2 images?

10

You can write a custom Dataset as described here and return two images in the __getitem__.

1 Like
11

Dear sir,
I have a problem: “mat1 dim 1 must match mat2 dim 0”
I have the inputs are 2 images (3, 224, 224) and (3, 224, 224), and the network ( I used Resnet18) is:

'''ResNet in PyTorch.
'''
import torch
import torch.nn as nn
import torch.nn.functional as F


class BasicBlock(nn.Module):
    expansion = 1

    def __init__(self, in_planes, planes, stride=1):
        super(BasicBlock, self).__init__()
        self.conv1 = nn.Conv2d(
            in_planes, planes, kernel_size=3, stride=stride, padding=1, bias=False)
        self.bn1 = nn.BatchNorm2d(planes)
        self.conv2 = nn.Conv2d(planes, planes, kernel_size=3,
                               stride=1, padding=1, bias=False)
        self.bn2 = nn.BatchNorm2d(planes)

        self.shortcut = nn.Sequential()
        if stride != 1 or in_planes != self.expansion*planes:
            self.shortcut = nn.Sequential(
                nn.Conv2d(in_planes, self.expansion*planes,
                          kernel_size=1, stride=stride, bias=False),
                nn.BatchNorm2d(self.expansion*planes)
            )

    def forward(self, x):
        out = F.relu(self.bn1(self.conv1(x)))
        out = self.bn2(self.conv2(out))
        out += self.shortcut(x)
        out = F.relu(out)
        return out


class Bottleneck(nn.Module):
    expansion = 4

    def __init__(self, in_planes, planes, stride=1):
        super(Bottleneck, self).__init__()
        self.conv1 = nn.Conv2d(in_planes, planes, kernel_size=1, bias=False)
        self.bn1 = nn.BatchNorm2d(planes)
        self.conv2 = nn.Conv2d(planes, planes, kernel_size=3,
                               stride=stride, padding=1, bias=False)
        self.bn2 = nn.BatchNorm2d(planes)
        self.conv3 = nn.Conv2d(planes, self.expansion *
                               planes, kernel_size=1, bias=False)
        self.bn3 = nn.BatchNorm2d(self.expansion*planes)

        self.shortcut = nn.Sequential()
        if stride != 1 or in_planes != self.expansion*planes:
            self.shortcut = nn.Sequential(
                nn.Conv2d(in_planes, self.expansion*planes,
                          kernel_size=1, stride=stride, bias=False),
                nn.BatchNorm2d(self.expansion*planes)
            )

    def forward(self, x):
        out = F.relu(self.bn1(self.conv1(x)))
        out = F.relu(self.bn2(self.conv2(out)))
        out = self.bn3(self.conv3(out))
        out += self.shortcut(x)
        out = F.relu(out)
        return out


class ResNet(nn.Module):
    def __init__(self, block, num_blocks, num_classes=10):
        super(ResNet, self).__init__()
        self.in_planes = 64

        self.conv1 = nn.Conv2d(3, 64, kernel_size=3,
                               stride=1, padding=1, bias=False)
        self.conv2 = nn.Conv2d(3, 64, kernel_size=3,
                               stride=1, padding=1, bias=False)
        self.bn1 = nn.BatchNorm2d(64)
        self.layer1 = self._make_layer(block, 64, num_blocks[0], stride=1)
        self.layer2 = self._make_layer(block, 128, num_blocks[1], stride=2)
        self.layer3 = self._make_layer(block, 256, num_blocks[2], stride=2)
        self.layer4 = self._make_layer(block, 512, num_blocks[3], stride=2)
        self.linear = nn.Linear(512*block.expansion, num_classes)

    def _make_layer(self, block, planes, num_blocks, stride):
        strides = [stride] + [1]*(num_blocks-1)
        layers = []
        for stride in strides:
            layers.append(block(self.in_planes, planes, stride))
            self.in_planes = planes * block.expansion
        return nn.Sequential(*layers)

    def forward(self, x, y):
        out1 = F.relu(self.bn1(self.conv1(x)))
        out2 = F.relu(self.bn1(self.conv2(y)))
        out = out1 + out2 
        out = self.layer1(out)
        out = self.layer2(out)
        out = self.layer3(out)
        out = self.layer4(out)
        out = F.avg_pool2d(out, 4)
        out = out.view(out.size(0), -1)
        out = self.linear(out)
        return out

def ResNet18():
    return ResNet(BasicBlock, [2, 2, 2, 2])