Faster RCNN yields hundred maches every time

Transforming

def create_model(pretrained=False, num_classes=2):
        """Creates faster rcnn 50 model

        Args:
            pretrained (bool): To use Coco pretrained
            num_classes (int): Number of output classes
        """
        return tv.models.detection.fasterrcnn_resnet50_fpn(
            pretrained=pretrained,
            num_classes=num_classes,
            pretrained_backbone=True)

to

def create_model(pretrained=False, num_classes=2):
        """Creates faster rcnn 50 model

        Args:
            pretrained (bool): To use Coco pretrained
            num_classes (int): Number of output classes
        """
        return tv.models.detection.fasterrcnn_resnet50_fpn(
            pretrained=pretrained,
            num_classes=num_classes,
            pretrained_backbone=True,
            box_detections_per_img=1000)

solved first part of the problem.