I found out the reason of the memory growing…
It happens when inputs have different sizes.
The following code is with detectron2 but previous model works in the same way.
import detectron2
from detectron2.evaluation import COCOEvaluator, inference_on_dataset
from detectron2.utils.logger import setup_logger
from detectron2.utils.visualizer import ColorMode
from glob import glob
setup_logger()
import os
import numpy as np
import cv2
import random
import uuid
from detectron2 import model_zoo
from detectron2.engine import DefaultPredictor
from detectron2.config import get_cfg
from detectron2.data.catalog import Metadata
from memory_profiler import profile
import torch
import numpy as np
import psutil
CLASSES = ['short_sleeved_shirt', 'long_sleeved_shirt', 'short_sleeved_outwear', 'long_sleeved_outwear', 'vest', 'sling', 'shorts', 'trousers', 'skirt', 'short_sleeved_dress', 'long_sleeved_dress', 'vest_dress', 'sling_dress']
cfg = get_cfg()
cfg.merge_from_file(model_zoo.get_config_file("COCO-Detection/faster_rcnn_R_101_FPN_3x.yaml"))
metadata = Metadata()
cfg.MODEL.ROI_HEADS.NUM_CLASSES = len(CLASSES)
metadata.set(thing_classes=CLASSES)
cfg.MODEL.WEIGHTS = os.path.join('/home/algernone/DNNS/dpf2/model_final_40000.pth')
cfg.MODEL.ROI_HEADS.SCORE_THRESH_TEST = 0.55 # set the testing threshold for this model
cfg.MODEL.DEVICE = 'cpu'
predictor = DefaultPredictor(cfg)
process = psutil.Process(os.getpid())
for i in range(10):
for img_name in glob('content/*.jpg'):
im = cv2.imread(img_name)
outputs = predictor(im)
print(im.shape, process.memory_full_info().rss / 1024**2)
5 runs for one image:
(550, 367, 3) 1565.56640625
(550, 367, 3) 1684.4140625
(550, 367, 3) 1714.15625
(550, 367, 3) 1743.78125
(550, 367, 3) 1565.8203125
5 for another:
(1800, 3200, 3) 1583.125
(1800, 3200, 3) 1687.15234375
(1800, 3200, 3) 1671.32421875
(1800, 3200, 3) 1600.40234375
(1800, 3200, 3) 1567.5
it is strange that for a larger image it took less memory…
and 5 runs for both images:
(550, 367, 3) 1580.203125
(1800, 3200, 3) 2706.109375
(550, 367, 3) 2690.5078125
(1800, 3200, 3) 2785.1953125
(550, 367, 3) 2798.828125
(1800, 3200, 3) 2862.90625
(550, 367, 3) 2876.65234375
(1800, 3200, 3) 2690.60546875
(550, 367, 3) 2690.66796875
(1800, 3200, 3) 2785.04296875
the memory increased significantly compared to inferences for these images separately
but if I don’t stick to the same size then this does not solve my problem