Possible memory leak in torch.jit.trace

Hi

Maybe I’m doing something wrong, but I’ve noticed a continuous increase in the memory usage when calling torch.jit.trace(model, inputs) multiple times in the same process.
I’m using the following test code:


#Utilities 
import os
import psutil

#JIT trace test
import torch
import torchvision.models as model_zoo

with torch.no_grad():

  #Create a simple resnet 
  model = model_zoo.resnet18()
  model.eval

  #Create sample input
  sample_input = torch.randn(size=[2, 3, 480, 640], requires_grad=False)

  #Get process id
  process = psutil.Process(os.getpid())

  #Repeat tracing 
  for i in range(0, 1000):
    tr_model = torch.jit.trace(model, sample_input, check_trace=False)
    print("Iter: {} = {}".format(i, process.memory_full_info()))

I’m using PyTorch 1.4.0 installed with conda.
I get the same problem with check_trace=True and check_trace=False

Thanks for reporting this issue!
How large is the leak for each iteration?

Hi

Looking at the output of the script, it seems that there is not a constant increase in the memory usage.
For example, if I run the test above with check_trace=True for 80 iterations, I get the following output:

Iter: 0 = pfullmem(rss=326701056, vms=729473024, shared=75661312, text=2265088, lib=0, data=278097920, dirty=0, uss=319164416, pss=320976896, swap=0)
Iter: 1 = pfullmem(rss=334843904, vms=737370112, shared=76226560, text=2265088, lib=0, data=285995008, dirty=0, uss=326877184, pss=328689664, swap=0)
Iter: 2 = pfullmem(rss=342360064, vms=744742912, shared=76226560, text=2265088, lib=0, data=293367808, dirty=0, uss=334262272, pss=336074752, swap=0)
Iter: 3 = pfullmem(rss=379133952, vms=781746176, shared=76226560, text=2265088, lib=0, data=330371072, dirty=0, uss=371142656, pss=372955136, swap=0)
Iter: 4 = pfullmem(rss=407904256, vms=810496000, shared=76226560, text=2265088, lib=0, data=359120896, dirty=0, uss=399900672, pss=401713152, swap=0)
Iter: 5 = pfullmem(rss=417579008, vms=820097024, shared=76226560, text=2265088, lib=0, data=368721920, dirty=0, uss=409509888, pss=411322368, swap=0)
Iter: 6 = pfullmem(rss=408051712, vms=810266624, shared=76226560, text=2265088, lib=0, data=358891520, dirty=0, uss=399818752, pss=401631232, swap=0)
Iter: 7 = pfullmem(rss=427544576, vms=829927424, shared=76226560, text=2265088, lib=0, data=378552320, dirty=0, uss=419491840, pss=421304320, swap=0)
Iter: 8 = pfullmem(rss=442355712, vms=844673024, shared=76226560, text=2265088, lib=0, data=393297920, dirty=0, uss=434245632, pss=436058112, swap=0)
Iter: 9 = pfullmem(rss=449597440, vms=852045824, shared=76226560, text=2265088, lib=0, data=400670720, dirty=0, uss=441626624, pss=443439104, swap=0)
Iter: 10 = pfullmem(rss=449785856, vms=852045824, shared=76226560, text=2265088, lib=0, data=400670720, dirty=0, uss=441634816, pss=443447296, swap=0)
Iter: 11 = pfullmem(rss=457093120, vms=859418624, shared=76226560, text=2265088, lib=0, data=408043520, dirty=0, uss=449015808, pss=450828288, swap=0)
Iter: 12 = pfullmem(rss=457015296, vms=859418624, shared=76226560, text=2265088, lib=0, data=408043520, dirty=0, uss=449028096, pss=450840576, swap=0)
Iter: 13 = pfullmem(rss=496234496, vms=898740224, shared=76226560, text=2265088, lib=0, data=447365120, dirty=0, uss=488226816, pss=490039296, swap=0)
Iter: 14 = pfullmem(rss=511221760, vms=913670144, shared=76226560, text=2265088, lib=0, data=462295040, dirty=0, uss=503164928, pss=504977408, swap=0)
Iter: 15 = pfullmem(rss=511164416, vms=913670144, shared=76226560, text=2265088, lib=0, data=462295040, dirty=0, uss=503173120, pss=504985600, swap=0)
Iter: 16 = pfullmem(rss=550715392, vms=952991744, shared=76226560, text=2265088, lib=0, data=501616640, dirty=0, uss=542507008, pss=544319488, swap=0)
Iter: 17 = pfullmem(rss=550686720, vms=952991744, shared=76226560, text=2265088, lib=0, data=501616640, dirty=0, uss=542515200, pss=544327680, swap=0)
Iter: 18 = pfullmem(rss=550268928, vms=952578048, shared=76226560, text=2265088, lib=0, data=501202944, dirty=0, uss=542109696, pss=543922176, swap=0)
Iter: 19 = pfullmem(rss=559927296, vms=962408448, shared=76226560, text=2265088, lib=0, data=511033344, dirty=0, uss=551948288, pss=553760768, swap=0)
Iter: 20 = pfullmem(rss=530780160, vms=932917248, shared=76226560, text=2265088, lib=0, data=481542144, dirty=0, uss=522661888, pss=524474368, swap=0)
Iter: 21 = pfullmem(rss=530845696, vms=932917248, shared=76226560, text=2265088, lib=0, data=481542144, dirty=0, uss=522661888, pss=524474368, swap=0)
Iter: 22 = pfullmem(rss=535318528, vms=937832448, shared=76226560, text=2265088, lib=0, data=486457344, dirty=0, uss=527446016, pss=529258496, swap=0)
Iter: 23 = pfullmem(rss=550354944, vms=952578048, shared=76226560, text=2265088, lib=0, data=501202944, dirty=0, uss=542322688, pss=544135168, swap=0)
Iter: 24 = pfullmem(rss=557764608, vms=959950848, shared=76226560, text=2265088, lib=0, data=508575744, dirty=0, uss=549695488, pss=551507968, swap=0)
Iter: 25 = pfullmem(rss=565141504, vms=967323648, shared=76226560, text=2265088, lib=0, data=515948544, dirty=0, uss=557068288, pss=558880768, swap=0)
Iter: 26 = pfullmem(rss=572645376, vms=974696448, shared=76226560, text=2265088, lib=0, data=523321344, dirty=0, uss=564441088, pss=566253568, swap=0)
Iter: 27 = pfullmem(rss=579878912, vms=982069248, shared=76226560, text=2265088, lib=0, data=530694144, dirty=0, uss=571813888, pss=573626368, swap=0)
Iter: 28 = pfullmem(rss=594636800, vms=997076992, shared=76226560, text=2265088, lib=0, data=545701888, dirty=0, uss=586567680, pss=588380160, swap=0)
Iter: 29 = pfullmem(rss=602152960, vms=1004449792, shared=76226560, text=2265088, lib=0, data=553074688, dirty=0, uss=593948672, pss=595761152, swap=0)
Iter: 30 = pfullmem(rss=641359872, vms=1043771392, shared=76226560, text=2265088, lib=0, data=592396288, dirty=0, uss=633151488, pss=634963968, swap=0)
Iter: 31 = pfullmem(rss=616792064, vms=1019195392, shared=76226560, text=2265088, lib=0, data=567820288, dirty=0, uss=608706560, pss=610519040, swap=0)
Iter: 32 = pfullmem(rss=624295936, vms=1026568192, shared=76226560, text=2265088, lib=0, data=575193088, dirty=0, uss=616083456, pss=617895936, swap=0)
Iter: 33 = pfullmem(rss=641503232, vms=1043771392, shared=76226560, text=2265088, lib=0, data=592396288, dirty=0, uss=633294848, pss=635107328, swap=0)
Iter: 34 = pfullmem(rss=725028864, vms=1127489536, shared=76226560, text=2265088, lib=0, data=676114432, dirty=0, uss=716894208, pss=718706688, swap=0)
Iter: 35 = pfullmem(rss=725000192, vms=1127489536, shared=76226560, text=2265088, lib=0, data=676114432, dirty=0, uss=716902400, pss=718714880, swap=0)
Iter: 36 = pfullmem(rss=724959232, vms=1127489536, shared=76226560, text=2265088, lib=0, data=676114432, dirty=0, uss=716910592, pss=718723072, swap=0)
Iter: 37 = pfullmem(rss=695332864, vms=1097781248, shared=76226560, text=2265088, lib=0, data=646406144, dirty=0, uss=687341568, pss=689154048, swap=0)
Iter: 38 = pfullmem(rss=714858496, vms=1117442048, shared=76226560, text=2265088, lib=0, data=666066944, dirty=0, uss=706879488, pss=708691968, swap=0)
Iter: 39 = pfullmem(rss=695586816, vms=1097781248, shared=76226560, text=2265088, lib=0, data=646406144, dirty=0, uss=687362048, pss=689174528, swap=0)
Iter: 40 = pfullmem(rss=761667584, vms=1164136448, shared=76226560, text=2265088, lib=0, data=712761344, dirty=0, uss=753594368, pss=755406848, swap=0)
Iter: 41 = pfullmem(rss=751833088, vms=1154306048, shared=76226560, text=2265088, lib=0, data=702930944, dirty=0, uss=743772160, pss=745584640, swap=0)
Iter: 42 = pfullmem(rss=751779840, vms=1154306048, shared=76226560, text=2265088, lib=0, data=702930944, dirty=0, uss=743780352, pss=745592832, swap=0)
Iter: 43 = pfullmem(rss=732422144, vms=1134645248, shared=76226560, text=2265088, lib=0, data=683270144, dirty=0, uss=724258816, pss=726071296, swap=0)
Iter: 44 = pfullmem(rss=737099776, vms=1139560448, shared=76226560, text=2265088, lib=0, data=688185344, dirty=0, uss=729055232, pss=730867712, swap=0)
Iter: 45 = pfullmem(rss=751976448, vms=1154306048, shared=76226560, text=2265088, lib=0, data=702930944, dirty=0, uss=743940096, pss=745752576, swap=0)
Iter: 46 = pfullmem(rss=761769984, vms=1164136448, shared=76226560, text=2265088, lib=0, data=712761344, dirty=0, uss=753778688, pss=755591168, swap=0)
Iter: 47 = pfullmem(rss=781651968, vms=1183797248, shared=76226560, text=2265088, lib=0, data=732422144, dirty=0, uss=773447680, pss=775260160, swap=0)
Iter: 48 = pfullmem(rss=789053440, vms=1191170048, shared=76226560, text=2265088, lib=0, data=739794944, dirty=0, uss=780832768, pss=782645248, swap=0)
Iter: 49 = pfullmem(rss=788971520, vms=1191170048, shared=76226560, text=2265088, lib=0, data=739794944, dirty=0, uss=780840960, pss=782653440, swap=0)
Iter: 50 = pfullmem(rss=796258304, vms=1198542848, shared=76226560, text=2265088, lib=0, data=747167744, dirty=0, uss=788221952, pss=790034432, swap=0)
Iter: 51 = pfullmem(rss=811008000, vms=1213288448, shared=76226560, text=2265088, lib=0, data=761913344, dirty=0, uss=802975744, pss=804788224, swap=0)
Iter: 52 = pfullmem(rss=818520064, vms=1220661248, shared=76226560, text=2265088, lib=0, data=769286144, dirty=0, uss=810356736, pss=812169216, swap=0)
Iter: 53 = pfullmem(rss=825761792, vms=1228034048, shared=76226560, text=2265088, lib=0, data=776658944, dirty=0, uss=817741824, pss=819554304, swap=0)
Iter: 54 = pfullmem(rss=833257472, vms=1235406848, shared=76226560, text=2265088, lib=0, data=784031744, dirty=0, uss=825122816, pss=826934272, swap=0)
Iter: 55 = pfullmem(rss=840499200, vms=1242779648, shared=76226560, text=2265088, lib=0, data=791404544, dirty=0, uss=832503808, pss=834316288, swap=0)
Iter: 56 = pfullmem(rss=848011264, vms=1250152448, shared=76226560, text=2265088, lib=0, data=798777344, dirty=0, uss=839884800, pss=841694208, swap=0)
Iter: 57 = pfullmem(rss=855269376, vms=1257525248, shared=76226560, text=2265088, lib=0, data=806150144, dirty=0, uss=847265792, pss=849075200, swap=0)
Iter: 58 = pfullmem(rss=862777344, vms=1264898048, shared=76226560, text=2265088, lib=0, data=813522944, dirty=0, uss=854642688, pss=856452096, swap=0)
Iter: 59 = pfullmem(rss=933801984, vms=1336430592, shared=76226560, text=2265088, lib=0, data=885055488, dirty=0, uss=925798400, pss=927607808, swap=0)
Iter: 60 = pfullmem(rss=933892096, vms=1336430592, shared=76226560, text=2265088, lib=0, data=885055488, dirty=0, uss=925806592, pss=927616000, swap=0)
Iter: 61 = pfullmem(rss=933851136, vms=1336430592, shared=76226560, text=2265088, lib=0, data=885055488, dirty=0, uss=925814784, pss=927624192, swap=0)
Iter: 62 = pfullmem(rss=900866048, vms=1303351296, shared=76226560, text=2265088, lib=0, data=851976192, dirty=0, uss=892878848, pss=894691328, swap=0)
Iter: 63 = pfullmem(rss=905740288, vms=1308266496, shared=76226560, text=2265088, lib=0, data=856891392, dirty=0, uss=897671168, pss=899483648, swap=0)
Iter: 64 = pfullmem(rss=927948800, vms=1330384896, shared=76226560, text=2265088, lib=0, data=879009792, dirty=0, uss=919920640, pss=921733120, swap=0)
Iter: 65 = pfullmem(rss=935456768, vms=1337757696, shared=76226560, text=2265088, lib=0, data=886382592, dirty=0, uss=927301632, pss=929114112, swap=0)
Iter: 66 = pfullmem(rss=942710784, vms=1345130496, shared=76226560, text=2265088, lib=0, data=893755392, dirty=0, uss=934686720, pss=936499200, swap=0)
Iter: 67 = pfullmem(rss=950222848, vms=1352503296, shared=76226560, text=2265088, lib=0, data=901128192, dirty=0, uss=942067712, pss=943880192, swap=0)
Iter: 68 = pfullmem(rss=957468672, vms=1359876096, shared=76226560, text=2265088, lib=0, data=908500992, dirty=0, uss=949448704, pss=951261184, swap=0)
Iter: 69 = pfullmem(rss=972296192, vms=1374621696, shared=76226560, text=2265088, lib=0, data=923246592, dirty=0, uss=964202496, pss=966014976, swap=0)
Iter: 70 = pfullmem(rss=987045888, vms=1389367296, shared=76226560, text=2265088, lib=0, data=937992192, dirty=0, uss=978956288, pss=980768768, swap=0)
Iter: 71 = pfullmem(rss=994557952, vms=1396740096, shared=76226560, text=2265088, lib=0, data=945364992, dirty=0, uss=986341376, pss=988153856, swap=0)
Iter: 72 = pfullmem(rss=1001803776, vms=1404112896, shared=76226560, text=2265088, lib=0, data=952737792, dirty=0, uss=993722368, pss=995534848, swap=0)
Iter: 73 = pfullmem(rss=1033687040, vms=1436061696, shared=76226560, text=2265088, lib=0, data=984686592, dirty=0, uss=1025548288, pss=1027360768, swap=0)
Iter: 74 = pfullmem(rss=1009147904, vms=1411485696, shared=76226560, text=2265088, lib=0, data=960110592, dirty=0, uss=1001111552, pss=1002924032, swap=0)
Iter: 75 = pfullmem(rss=1016659968, vms=1418858496, shared=76226560, text=2265088, lib=0, data=967483392, dirty=0, uss=1008496640, pss=1010309120, swap=0)
Iter: 76 = pfullmem(rss=1023901696, vms=1426231296, shared=76226560, text=2265088, lib=0, data=974856192, dirty=0, uss=1015877632, pss=1017690112, swap=0)
Iter: 77 = pfullmem(rss=1038667776, vms=1440976896, shared=76226560, text=2265088, lib=0, data=989601792, dirty=0, uss=1030631424, pss=1032443904, swap=0)
Iter: 78 = pfullmem(rss=1046183936, vms=1448349696, shared=76226560, text=2265088, lib=0, data=996974592, dirty=0, uss=1038012416, pss=1039824896, swap=0)
Iter: 79 = pfullmem(rss=1085386752, vms=1487671296, shared=76226560, text=2265088, lib=0, data=1036296192, dirty=0, uss=1077211136, pss=1079023616, swap=0)
Iter: 80 = pfullmem(rss=1109991424, vms=1512247296, shared=76226560, text=2265088, lib=0, data=1060872192, dirty=0, uss=1101799424, pss=1103611904, swap=0)

As you can see it starts from ~ 300MB and reaches 1.1GB.
If it helps, I’ve noticed that, If I set check_trace=False, the memory usage increases more slowly and it takes more iterations to reach 1GB

Thanks

1 Like

Thanks for this information.
We’ll try to reproduce it.

I can reproduce it with the latest master build and have created a bug here.
Thanks again for reporting this issue here. :slight_smile:

1 Like