What is NotImplementedError?

1 
class MODEL(nn.Module):
    def __init__(self, attention_heads=4, attention_size=32, out_size=4):
        super(MODEL, self).__init__()
        self.conv1a = nn.Conv2d(kernel_size=(10, 2), in_channels=1, out_channels=16, padding=(4, 0))
        self.conv1b = nn.Conv2d(kernel_size=(2,8), in_channels=1, out_channels=16, padding=(0, 3))
        self.conv2 = nn.Conv2d(kernel_size=(3,3), in_channels=32, out_channels=32, padding=(1,1))
        self.conv3 = nn.Conv2d(kernel_size=(3,3), in_channels=32, out_channels=48, padding=(1,1))
        self.conv4 = nn.Conv2d(kernel_size=(3,3), in_channels=48, out_channels=64, padding=(1,1))
        self.conv5 = nn.Conv2d(kernel_size=(3,3), in_channels=64, out_channels=80, padding=(1,1))
            
        self.maxp = nn.MaxPool2d((2, 2))
        self.bn1a = nn.BatchNorm2d(3)
        self.bn1b = nn.BatchNorm2d(3)
        self.bn2 = nn.BatchNorm2d(3)
        self.bn3 = nn.BatchNorm2d(3)
        self.bn4 = nn.BatchNorm2d(3)
        self.bn5 = nn.BatchNorm2d(3)
        self.gap = nn.AdaptiveAvgPool2d(1)#(data_format='channels_last')
        self.flatten = nn.Flatten()
        self.fc = nn.Linear(10000,out_size)
        self.attention_query = []
        self.attention_key = []
        self.attention_value = []
        self.attention_heads = attention_heads
        self.attention_size = attention_size
        for i in range(self.attention_heads):
            self.attention_query.append(nn.Conv2d(in_channels=80,out_channels=self.attention_size,kernel_size=1))
            self.attention_key.append(nn.Conv2d(in_channels=80,out_channels=self.attention_size,kernel_size=1))
            self.attention_value.append(nn.Conv2d(in_channels=80,out_channels=self.attention_size,kernel_size=1))
            
            

    def call(self, *input):
        x = input[0]
        xa = self.conv1a(x)
        xa = self.bn1a(xa)
        xa=nn.relu(xa)
        xb = self.conv1b(x)
        xb = self.bn1b(xb)
        xb = nn.relu(xb)
        x = nn.concat([xa, xb], 1)
        x = self.conv2(x)
        x = self.bn2(x)
        x=nn.relu(x)
        x = self.maxp(x)
        x = self.conv3(x)
        x = self.bn3(x)
        x = nn.relu(x)
        x = self.maxp(x)
        x = self.conv4(x)
        x = self.bn4(x)
        x = nn.relu(x)
        x = self.conv5(x)
        x = self.bn5(x)
        x = nn.relu(x)

        attn = None
        for i in range(self.attention_heads):
            # Q = self.attention_query[i](x)
            # Q = tf.transpose(Q, perm=[0, 3, 1, 2])
            # K = self.attention_key[i](x)
            # K = tf.transpose(K, perm=[0, 3, 2, 1])
            # V = self.attention_value[i](x)
            # V = tf.transpose(V, perm=[0, 3, 1, 2])
            # attention = tf.nn.softmax(tf.matmul(Q, K))
            # attention = tf.matmul(attention, V)
            Q = self.attention_query[i](x)
            K = self.attention_key[i](x)
            V = self.attention_value[i](x)
            attention = nn.Softmax(torch.matmul(Q, K),dim=1)
            attention = torch.matmul(attention, V)
            if (attn is None):
                attn = attention
            else:
                attn = nn.concat([attn, attention], 2)
        x = torch. transpose(attn, perm=[0, 2, 3, 1])
        x = nn.relu(x)
        x = self.gap(x)
        x = self.flatten(x)
        x = self.fc(x)
        return x

raise NotImplementedError
I am not getting this error , my input is a 2d structure of size (23,63)

2

I guess this error is raised during the forward pass, since the forward method definition is missing, so you would have to change call to forward.

1 Like
3

@ptrblck Sir did that. But for input

I=torch.randn(32,1,23,63)
model=MODEL(4,32,4)
O=model(I)

I am now getting an error at line

attention = nn.Softmax(torch.matmul(Q, K),dim=1)

The error is :

RuntimeError: Expected batch2_sizes[0] == bs && batch2_sizes[1] == contraction_size to be true, but got false. (Could this error message be improved? If so, please report an enhancement request to PyTorch.)

Plz guide

4

This error is raised in the torch.matmul call, which gets tensors in a wrong shape.
Print the shape of both input tensors and make sure they have the expected shapes.

1 Like
5

Sir the shapes are

Q=torch.Size([32, 32, 5, 15])
K=torch.Size([32, 32, 5, 15])

and yes this size is inappropriate for matrix mul. I dont know what to do now.

6

You are confusing the shapes with the values in a tensor.
The first example won’t work, as the shapes are invalid for a matrix multipliciation:

A = torch.randn([32, 32, 5, 15])
B = torch.randn([32, 32, 5, 15])

C = torch.matmul(A, B)
> RuntimeError: Expected batch2_sizes[0] == bs && batch2_sizes[1] == contraction_size to be true, but got false.

So you would have to permute one of the tensors depending on the expected output shape:

C = torch.matmul(A.permute(0, 1, 3, 2), B)
print(C.shape)
> torch.Size([32, 32, 15, 15])

# or
C = torch.matmul(A, B.permute(0, 1, 3, 2))
print(C.shape)
> torch.Size([32, 32, 5, 5])

Your second example was using just 4 values and is thus returning a scalar.

1 Like
7

Yes sir I got it, but now plz guide how to multiply this softmax output with another matrix, it says it’s not possible. But this is ky definition of attention module @ptrblck sir.

8

As now I am getting this error:
TypeError: matmul(): argument ‘input’ (position 1) must be Tensor, not Softmax

At line:
attention = torch.matmul(attention, V.permute(0, 1, 3, 2))

This is for second matmul() method

9

That was my mistake , it should be F.softmax().