Large memory allocated of nn.Conv2d when kernel_size=7

When I run conv_mem(kernel_size=7, padding=3) in the following demo, I find the allocated max_memory is 1591.0751953125M while conv_mem(kernel_size=5, padding=2) is 69.2568359375M, is it normal?

# -*- coding: utf-8 -*-
import torch
import torch.nn as nn


def memory_usage_stats(gpu_id=0):
    allocated_mem = torch.cuda.memory_allocated(gpu_id) / 1024 / 1024
    max_allocated_mem = torch.cuda.max_memory_allocated(gpu_id) / 1024 / 1024
    return allocated_mem, max_allocated_mem


def conv_mem(kernel_size, padding):
    conv3d = nn.Conv2d(256, 256, kernel_size=kernel_size, padding=padding, dilation=1, groups=1, bias=False).cuda()
    x = torch.rand(1, 256, 132, 240, device='cuda')
    y = conv3d(x)
    allocated_mem, max_allocated_mem = memory_usage_stats(0)
    print("allocated: {}M".format(allocated_mem))
    print("allocated max_memory: {}M".format(max_allocated_mem))


if __name__ == '__main__':
    # conv_mem(kernel_size=1, padding=0)
    # result:
    # allocated: 63.0751953125M
    # allocated max_memory: 63.2568359375M

    # conv_mem(kernel_size=3, padding=1)
    # result:
    # allocated: 65.0751953125M
    # allocated max_memory: 71.4501953125M

    # conv_mem(kernel_size=5, padding=2)
    # result:
    # allocated: 69.0751953125M
    # allocated max_memory: 69.2568359375M

    conv_mem(kernel_size=7, padding=3)
    # result:
    # allocated: 75.0751953125M
    # allocated max_memory: 1591.0751953125M