Output overflow and unstablity when use model.eval()

Hi there,

I have transfered a keras based model (with tensorflow backend) to pytorch! Actually it is the pretrained model of YOLOV2. When I forward a test image to understand how the network works well, I does not have any warnings (notice the forward path consists of Batchnorm layers), but when I move the network to evaluation mode using model.eval(), I face with this warning:

Warning: overflow encountered in exp return 1./(1 + np.exp(-1*inp)) /pytorch_model/yoloUtil.py:143: RuntimeWarning: overflow encountered in exp box_wh = np.exp(feature[..., 2:4])

The overflow warning causes the network fail to predict any bounding boxes. Could you please tell me, why do these problems happen?

Appreciating in advance for a any response!

what is inp and features? Even in eval mode, Batchnorm still works.

@chenyuntc Thanks for your response!
features is the final layer output. and inp is the input to the sigmoid function.

def yolo_head(feats, anchors, num_classes):
      """Convert final layer features to bounding box parameters.

      Parameters
      ----------
      feats : tensor
             Final convolutional layer features.
      anchors : array-like
             Anchor box widths and heights.
      num_classes : int
             Number of target classes.

      Returns
      -------
      box_xy : tensor
              x, y box predictions adjusted by spatial location in conv layer.
      box_wh : tensor
             w, h box predictions adjusted by anchors and conv spatial resolution.
      box_conf : tensor
             Probability estimate for whether each box contains any object.
      box_class_pred : tensor
             Probability distribution estimate for each box over class labels.
      """
      feature = feats.numpy()
      feature = feature.transpose(0,2,3,1)

      num_anchors = len(anchors)
      # Reshape to batch, height, width, num_anchors, box_params.
      anchors_tensor = anchors.reshape([1, 1, 1, num_anchors, 2])
      conv_dims = feature.shape[1:3]

      conv_height_index = np.arange(0, stop=conv_dims[0])
      conv_width_index = np.arange(0, stop=conv_dims[1])

      conv_height_index = np.tile(conv_height_index, [conv_dims[1]])
      conv_width_index = np.tile(
               np.expand_dims(conv_width_index, 0), [conv_dims[0], 1])
      conv_width_index = (np.transpose(conv_width_index)).flatten()
      conv_index = np.transpose(np.stack([conv_height_index, conv_width_index]))
      conv_index = np.reshape(conv_index, [1, conv_dims[0], conv_dims[1], 1, 2])
      conv_index = conv_index.astype(float)
      feature = np.reshape(
            feature, [-1, conv_dims[0], conv_dims[1], num_anchors, num_classes + 5])
      conv_dims = np.reshape(conv_dims, [1, 1, 1, 1, 2]).astype(float)

      box_xy = sigmoid(feature[..., :2])
      box_wh = np.exp(feature[..., 2:4])        ***** Here is the location of warning*****
      box_confidence = sigmoid(feature[..., 4:5])
      box_class_probs = np.apply_along_axis(softmax,4,feature[..., 5:])

      box_xy = (box_xy + conv_index) / conv_dims
      box_wh = box_wh * anchors_tensor / conv_dims
      return box_xy, box_wh, box_confidence, box_class_probs 

Also for Calculating the softmax and sigmoid along a dimension, I wrote 2 functions as bellow:

def sigmoid(inp):
      return 1./(1 + np.exp(-1*inp))        **** Here is the other location of warning******

def softmax(x):
      """Compute softmax values for each sets of scores in x."""
      e_x = np.exp(x - np.max(x))
return e_x / e_x.sum()

seem to be the problem of feature. You may print it and see what’s wrong.

Without model.eval():

`[[[[[ -1.07075989e-01 -4.39112991e-01 -9.40587878e-01 …,
3.13314486e+00 -1.54667079e+00 1.74327826e+00]
[ 1.02364635e+00 -8.85347366e-01 -1.57168433e-01 …,
2.75069904e+00 -1.75123787e+00 1.31727755e+00]
[ 1.76883888e+00 1.31513536e-01 -1.28749418e+00 …,
7.74755001e-01 -1.84538651e+00 2.71459341e+00]
[ 2.91872621e-02 1.59589723e-01 1.53984472e-01 …,
9.13461328e-01 -1.59088635e+00 1.63678157e+00]
[ 3.64127815e-01 8.80579948e-01 2.06380770e-01 …,
-1.29420459e-01 -2.28418446e+00 2.53179502e+00]]

[[ -1.59637094e+00 -2.83793718e-01 -3.61469179e-01 …,
2.42104936e+00 -1.50146878e+00 1.04749036e+00]
[ -2.38844544e-01 4.07841444e-01 -1.25134170e-01 …,
6.22990906e-01 -1.77230883e+00 2.31721014e-01]
[ 1.36048365e+00 -3.81075621e-01 -5.39776325e-01 …,
4.47489262e-01 -1.50398338e+00 -4.20431912e-01]
[ 2.17169657e-01 -1.81535196e+00 1.78491324e-02 …,
-2.13062286e-01 -1.95609546e+00 -6.64336801e-01]
[ 6.03908300e-03 3.47443670e-02 1.22472361e-01 …,
-4.82561469e-01 -2.42117286e+00 5.68968773e-01]]

[[ -1.06827986e+00 7.05153108e-01 -5.03763676e-01 …,
2.00297642e+00 -3.44963861e+00 -1.22004223e+00]
[ -1.68623328e-02 -6.48242235e-03 3.68758440e-01 …,
8.54735672e-01 -4.24609566e+00 -1.54878330e+00]
[ 5.85269928e-01 -7.14257777e-01 -2.85654664e-01 …,
5.52064121e-01 -3.39210773e+00 -1.36829436e+00]
[ 6.22348189e-01 -1.53005385e+00 -2.05970570e-01 …,
-4.38622177e-01 -2.86752367e+00 -9.15135264e-01]
[ 7.27358520e-01 2.65871972e-01 3.32212970e-02 …,
-1.58870578e+00 -3.09723973e+00 1.89628214e-01]]
…,
[[ 5.97730041e-01 2.70410627e-03 -6.01684928e-01 …,
-2.04850030e+00 -2.17200804e+00 9.50788185e-02]
[ -1.24553299e+00 3.14570904e+00 -2.15448156e-01 …,
-3.16923404e+00 -2.56539011e+00 2.00808263e+00]
[ -5.94927192e-01 1.77001309e+00 -4.54834878e-01 …,
-4.24447441e+00 -1.33400464e+00 2.14467502e+00]
[ 3.49495411e-01 1.07096577e+00 -9.58548486e-03 …,
-3.61487818e+00 -1.60971963e+00 4.42153335e-01]
[ 7.03387678e-01 -8.63260806e-01 -1.28527582e-02 …,
-3.03880906e+00 -1.64708936e+00 1.28562307e+00]]
…,
[[[ -9.89431083e-01 4.22683418e-01 -7.39042521e-01 …,
3.44297767e+00 -7.30107903e-01 -4.18882519e-01]
[ -1.73127711e-01 3.01834464e-01 -7.54745424e-01 …,
3.02644777e+00 -1.29641032e+00 -1.14082289e+00]
[ 1.04358292e+00 -9.05704558e-01 -9.51315284e-01 …,
2.23359179e+00 -1.43012857e+00 -9.47165728e-01]
[ -1.36387244e-01 -3.03453594e-01 -2.06832945e-01 …,
1.55085158e+00 -5.74167728e-01 -2.87570894e-01]
[ 8.87480527e-02 -8.30304503e-01 -1.67493373e-01 …,
1.74410915e+00 -6.33536935e-01 1.21541008e-01]]

[[ -6.48639053e-02 -8.53286743e-01 1.64060012e-01 …,
2.03146029e+00 -2.90256917e-01 -3.23312074e-01]
[ -4.62928921e-01 5.12159020e-02 3.26973081e-01 …,
1.73465192e+00 -6.56264246e-01 -5.83739102e-01]
[ -5.54190457e-01 -6.25617266e-01 -1.09632708e-01 …,
7.50372171e-01 -1.14373255e+00 -3.34591568e-01]
[ 5.69369197e-01 2.06014216e-02 -2.42602661e-01 …,
3.47952664e-01 -6.50111437e-01 2.22202986e-02]
[ 1.36280119e-01 -7.56513476e-01 -3.31164837e-01 …,
1.29209614e+00 -9.01850760e-01 3.78811538e-01]]

[[ -1.43173218e-01 7.72010535e-02 -3.40250641e-01 …,
5.43265581e-01 -1.49092317e+00 8.30179453e-03]
[ -2.54103988e-01 -4.35017437e-01 3.43804181e-01 …,
2.34699309e-01 -1.73454559e+00 -5.44458747e-01]
[ -4.06496316e-01 -8.23709130e-01 2.78647333e-01 …,
2.17914969e-01 -1.44878078e+00 -8.31533670e-02]
[ 8.87326360e-01 1.07999064e-01 -2.16917351e-01 …,
-8.14723074e-02 -1.00205600e+00 -3.16833258e-02]
[ 6.76401258e-01 -1.25771821e+00 -4.82933074e-01 …,
1.10465693e+00 -1.15929174e+00 6.48908377e-01]]

[[ 1.57525873e+00 -5.40540695e-01 -1.20892072e+00 …,
1.04007089e+00 -1.59711325e+00 -2.29269075e+00]
[ 7.97020912e-01 5.50970435e-03 -8.81440580e-01 …,
9.70211506e-01 -1.97356272e+00 -1.23058438e+00]
[ -7.06732720e-02 -9.18719172e-01 -9.84150767e-01 …,
3.32275331e-01 -1.64473248e+00 -5.06047189e-01]
[ 7.16224074e-01 -1.74189866e-01 1.74300093e-02 …,
-7.42350161e-01 -8.18396151e-01 -6.83267772e-01]
[ 4.48581547e-01 -2.33436674e-01 -5.44481985e-02 …,
-1.70234203e-01 -8.69583786e-01 -2.86695510e-01]]]

[[[ -1.15510321e+00 -4.45892811e-02 -5.27243614e-01 …,
2.25410175e+00 -6.72602355e-01 -2.73259282e-02]
[ 4.97808754e-01 -1.65608972e-01 -4.31176931e-01 …,
2.60243559e+00 -8.74305367e-01 -6.63845539e-01]
[ 9.31014121e-01 -9.65950966e-01 -9.37161326e-01 …,
2.03862977e+00 -7.79089451e-01 -4.67905223e-01]
[ 1.24568745e-01 -3.15860659e-01 -1.57195643e-01 …,
1.64764893e+00 -4.80436504e-01 -1.00497074e-01]
[ 1.79682627e-01 -5.20490885e-01 -1.43755063e-01 …,
1.53291631e+00 -4.07362163e-01 1.21973202e-01]]

[[ 1.39580086e-01 -2.45647937e-01 2.17374504e-01 …,
2.42638159e+00 4.73639071e-02 3.53267223e-01]
[ -5.22018254e-01 2.38307714e-01 3.52554500e-01 …,
2.10760283e+00 -5.09911597e-01 1.73100770e-01]
[ 2.76583910e-01 -6.26857162e-01 -6.54999986e-02 …,
1.73377800e+00 -6.14430785e-01 2.27044418e-01]
[ 7.48551965e-01 -2.00060338e-01 -3.26280743e-01 …,
1.32619882e+00 -3.96631807e-01 6.05146468e-01]
[ 2.96731710e-01 -2.42293268e-01 -3.06056291e-01 …,
1.86239004e+00 -2.64620543e-01 9.75538135e-01]]

[[ 2.50553995e-01 -7.46472597e-01 -1.48889571e-01 …,
1.16063213e+00 -4.20520544e-01 8.28358293e-01]
[ -8.39579701e-02 4.85758901e-01 5.89942932e-01 …,
9.11077380e-01 -9.60058689e-01 6.15663826e-01]
[ -6.37243390e-01 -3.94274235e-01 2.03115135e-01 …,
1.32119536e+00 -9.88202155e-01 3.89122695e-01]
[ 6.18680477e-01 2.52195656e-01 -1.67415440e-01 …,
8.67859423e-01 -8.90301168e-01 4.64680135e-01]
[ 4.15470600e-01 -1.50768459e-01 -2.57678390e-01 …,
1.78741479e+00 -8.29994082e-01 8.09115171e-01]]

…,
[[ -7.91447163e-02 9.78098512e-02 -2.80131727e-01 …,
-4.89220470e-02 -9.06519771e-01 -5.76316357e-01]
[ 3.48259658e-02 3.20821106e-01 1.95926860e-01 …,
-5.79015970e-01 -1.48590934e+00 -4.38588947e-01]
[ 9.25648510e-01 -3.35408926e-01 1.75966889e-01 …,
-8.52046132e-01 -1.19623518e+00 -4.92681563e-02]
[ 1.72467202e-01 4.47181463e-01 -1.97901145e-01 …,
-1.66926265e+00 -9.42209065e-01 -2.59209812e-01]
[ -1.42895579e-02 -3.33616614e-01 -3.29961449e-01 …,
-1.55633569e+00 -8.91426504e-01 1.31785035e-01]]

[[ -8.25410545e-01 -6.03218675e-02 2.63885319e-01 …,
-4.63727564e-02 -8.88435483e-01 -1.73753405e+00]
[ 4.06429082e-01 1.89955413e-01 1.92851394e-01 …,
6.18646085e-01 -1.83080256e+00 -1.91211462e+00]
[ -3.71373296e-01 -3.79637241e-01 -5.77083044e-02 …,
-6.13297105e-01 -1.49187517e+00 -4.53189075e-01]
[ -2.25246459e-01 1.82925940e-01 -3.60963553e-01 …,
-1.57162523e+00 -1.01040244e+00 -3.90981078e-01]
[ -3.81843835e-01 1.10523179e-02 -2.62450457e-01 …,
-2.04315782e+00 -7.68837094e-01 1.41253054e-01]]

[[ -1.18505210e-02 1.49443650e+00 2.22243965e-01 …,
1.21491241e+00 -6.66923523e-01 -2.62692714e+00]
[ 2.16134638e-01 -1.18031228e+00 -3.02660078e-01 …,
1.80655015e+00 -8.08886588e-01 -1.31399608e+00]
[ 1.94263801e-01 -5.11718750e-01 -8.90758932e-01 …,
-2.15077370e-01 -9.29562092e-01 -1.53487176e-01]
[ -3.84806991e-02 -5.45224905e-01 1.67636156e-01 …,
-1.22038198e+00 -6.96404874e-01 6.83081150e-03]
[ -1.63758278e-01 -3.24006081e-02 1.28940463e-01 …,
-1.59356236e+00 -8.51782918e-01 7.55040944e-01]]]]]`

the maximum of the output is: 16.2592
the minimum of the output is: -42.9159

with model.eval():
`
[[[[[ -3.57405243e+01 -3.40947647e+01 -1.80168018e+01 …,
-1.11552505e+02 -1.47317673e+02 1.15841808e+01]
[ -3.45298996e+01 -8.66607361e+01 -1.56450157e+01 …,
3.38557281e+01 -1.14936005e+02 7.51782608e+00]
[ -5.57367744e+01 3.30688400e+01 -2.05872498e+01 …,
4.67400856e+01 -8.21306000e+01 -3.18207779e+01]
[ -4.24697380e+01 3.34780769e+01 -9.04815388e+00 …,
-9.59447765e+00 -3.47139511e+01 -5.54042931e+01]
[ -4.58841629e+01 1.28552811e+02 -6.47773361e+00 …,
1.00611515e+01 -5.96798134e+01 -8.32996063e+01]]

[[ 2.68651199e+01 -1.63246651e+01 2.43751316e+01 …,
-1.63626968e+02 -2.72747986e+02 -3.68727036e+01]
[ -1.04041290e+01 -9.96350708e+01 -3.00881462e+01 …,
7.78560104e+01 -2.17356903e+02 -4.69987488e+01]
[ -3.31158714e+01 6.19809570e+01 -2.54234257e+01 …,
1.20234375e+02 -1.46192612e+02 -1.04720001e+02]
[ 3.76580048e+00 4.86019516e+01 -5.56028976e+01 …,
1.10028868e+01 -6.48364487e+01 -1.13399048e+02]
[ -1.31446609e+01 2.17165466e+02 -3.61693077e+01 …,
1.67402267e+01 -7.37028275e+01 -1.41293900e+02]]

[[ 3.28933296e+01 -1.87349072e+01 3.23689117e+01 …,
-2.01339417e+02 -3.07486755e+02 -4.66394157e+01]
[ -4.69960098e+01 -1.38499100e+02 -2.74664307e+01 …,
9.80660553e+01 -2.42747482e+02 -6.20588684e+01]
[ -8.69029541e+01 3.85177727e+01 -2.88304672e+01 …,
1.60042236e+02 -1.57490082e+02 -1.33510590e+02]
[ 1.17969360e+01 4.36259995e+01 -5.15230789e+01 …,
3.01130390e+01 -6.94463501e+01 -1.46284805e+02]
[ 5.86700630e+01 2.50478577e+02 -2.46847076e+01 …,
1.78839912e+01 -9.06006470e+01 -1.84669907e+02]]
…,
[[ 1.28096069e+02 -1.08034973e+02 1.00587769e+02 …,
-3.02942017e+02 -5.87585693e+02 -1.85980316e+02]
[ -1.07169044e+02 -1.92016525e+02 -1.07185005e+02 …,
3.07053619e+02 -4.95432129e+02 -1.48049164e+02]
[ -1.84443207e+02 1.46187561e+02 -3.77634125e+01 …,
4.00913208e+02 -3.21964172e+02 -2.51287964e+02]
[ 1.05675522e+02 7.63189240e+01 -5.44517975e+01 …,
2.01354736e+02 -1.78205566e+02 -3.71699127e+02]
[ 3.00891693e+02 2.60920654e+02 -7.02868652e+00 …,
1.77396393e+02 -2.07020905e+02 -4.55916260e+02]]

[[ 1.18607811e+02 -7.71134644e+01 1.03000252e+02 …,
-2.72433044e+02 -4.68416016e+02 -1.63925430e+02]
[ -1.24680687e+02 -1.35840088e+02 -6.77824707e+01 …,
2.23476318e+02 -3.85767761e+02 -8.15217590e+01]
[ -1.51540619e+02 1.24813019e+02 -4.20030098e+01 …,
3.52139038e+02 -2.55510162e+02 -1.79175507e+02]
[ 1.61691925e+02 5.56291580e+01 -4.61462326e+01 …,
1.81345627e+02 -1.42671478e+02 -2.94653198e+02]
[ 3.68817932e+02 1.83100464e+02 -3.00182953e+01 …,
1.79531754e+02 -1.65707153e+02 -3.74365234e+02]]

[[ 1.84909195e+02 -6.60677185e+01 -3.94966850e+01 …,
-2.23886200e+02 -3.13589722e+02 -1.47936615e+02]
[ -6.69442215e+01 -9.53987885e+01 -5.60369072e+01 …,
1.28458511e+02 -2.44234451e+02 -6.48127975e+01]
[ -1.80063736e+02 7.14935608e+01 -7.09763565e+01 …,
2.24825470e+02 -1.83780502e+02 -9.14833069e+01]
[ 5.47513046e+01 5.16017265e+01 -3.16933441e+00 …,
1.03527550e+02 -9.15430298e+01 -1.92175537e+02]
[ 1.57931335e+02 1.00216995e+02 -7.86803436e+00 …,
9.73970795e+01 -1.29235748e+02 -2.48961700e+02]]]

[[ 1.03647446e+02 -4.84287224e+01 4.29009857e+01 …,
-1.61768234e+02 -3.45710693e+02 -4.20045700e+01]
[ -8.61012115e+01 -7.23499603e+01 -4.41467285e-01 …,
1.81409683e+02 -3.13771912e+02 -1.03866043e+01]
[ -1.49256012e+02 1.40624847e+01 -2.46753464e+01 …,
2.51162933e+02 -2.11735046e+02 -1.00225426e+02]
[ 3.79445801e+01 6.66898727e+01 -1.85524902e+01 …,
1.29778748e+02 -1.48592590e+02 -1.96812775e+02]
[ 1.02250710e+02 -2.70945740e+01 7.59958649e+00 …,
8.03603287e+01 -1.37915009e+02 -2.35489746e+02]]

[[ 1.43202881e+02 -5.66237297e+01 -1.49130478e+01 …,
-1.37114746e+02 -2.42718002e+02 -6.97095337e+01]
[ -4.88247223e+01 -6.57657471e+01 -2.17019615e+01 …,
8.71873169e+01 -2.09040985e+02 -8.88125801e+00]
[ -9.23230133e+01 1.68656158e+00 -4.53515472e+01 …,
1.34258362e+02 -1.47102661e+02 -5.58313560e+01]
[ -2.39641571e+00 4.52565727e+01 -4.81935883e+00 …,
5.46552963e+01 -9.88356247e+01 -1.32153473e+02]
[ 3.77416306e+01 -2.55861816e+01 -8.08153534e+00 …,
3.26289711e+01 -9.94700623e+01 -1.55331451e+02]]]

[[[ -2.50377026e+01 -2.83629608e+01 7.04711914e+00 …,
-1.43002823e+02 -2.67520386e+02 1.07822800e+00]
[ -4.81338997e+01 -5.76092148e+00 -5.29888725e+00 …,
8.41980896e+01 -2.71234924e+02 -5.21333351e+01]
[ 4.67211151e+01 -8.86807404e+01 -1.72004662e+01 …,
6.92080460e+01 -1.48268478e+02 -8.12087784e+01]
[ 2.12886467e+01 -2.17681236e+01 2.25733852e+01 …,
8.13394451e+00 -1.31059464e+02 -1.64692322e+02]
[ 1.73150730e+01 -3.44890137e+01 -1.77896500e+00 …,
-3.54678726e+01 -1.00912758e+02 -1.50829758e+02]]

[[ 3.24036713e+01 -4.51581650e+01 6.58598785e+01 …,
-1.38405884e+02 -4.20851318e+02 -2.06448860e+01]
[ -4.55138474e+01 -6.77271500e+01 3.81010962e+00 …,
1.97721069e+02 -4.13602966e+02 -1.30372498e+02]
[ 6.08714828e+01 -8.89726562e+01 -1.72021904e+01 …,
2.11114685e+02 -2.62320099e+02 -2.01788208e+02]
[ 2.00961514e+01 5.09321175e+01 -5.40153770e+01 …,
8.94629059e+01 -1.81328033e+02 -2.61308228e+02]
[ 7.69575043e+01 -2.47528992e+01 -3.47502213e+01 …,
-9.40509605e+00 -1.30564438e+02 -2.40160339e+02]]

[[ 2.93432312e+01 -3.16786194e+00 7.86419296e+01 …,
-1.03176529e+02 -4.66820251e+02 -1.07993469e+01]
[ -5.57636032e+01 -7.17544556e+01 4.51091385e+00 …,
2.86591187e+02 -4.61342773e+02 -1.28324356e+02]
[ -2.46082764e+01 -8.43938751e+01 -1.04389076e+01 …,
3.07387573e+02 -2.92467529e+02 -2.25555603e+02]
[ 1.18098686e+02 7.97620087e+01 -5.05269623e+01 …,
1.61050934e+02 -1.99611679e+02 -2.88568939e+02]
[ 2.16817093e+02 -4.23579407e+01 -1.50130844e+01 …,
6.86450500e+01 -1.53752533e+02 -2.83708527e+02]]

…,
[[ 5.07347946e+01 6.57857513e+00 7.35759277e+01 …,
-1.00078987e+02 -3.23601624e+02 -2.64569664e+00]
[ -6.81458893e+01 -6.78410034e+01 1.08724518e+01 …,
1.83104355e+02 -3.08435303e+02 -3.35540657e+01]
[ -5.84255142e+01 -6.26393051e+01 -6.13491249e+00 …,
2.13446503e+02 -1.86155029e+02 -1.05150665e+02]
[ 5.09105148e+01 2.93376713e+01 -1.09460449e+01 …,
1.29562485e+02 -1.54337769e+02 -1.90460526e+02]
[ 1.05476334e+02 -6.08709869e+01 8.44407654e+00 …,
5.92027092e+01 -1.38292267e+02 -1.92579346e+02]]

[[ 5.85530128e+01 2.92981339e+00 7.23309860e+01 …,
-8.85582123e+01 -2.56360382e+02 -1.72532635e+01]
[ -8.04446182e+01 -5.68215256e+01 1.85313091e+01 …,
1.37098480e+02 -2.36876419e+02 -1.10650196e+01]
[ -1.00497459e+02 -4.35578537e+01 -9.97192860e+00 …,
1.73812210e+02 -1.40929184e+02 -6.65958405e+01]
[ 3.90408859e+01 2.00592747e+01 -3.38714600e+00 …,
1.05563202e+02 -1.23946381e+02 -1.41684906e+02]
[ 7.93414688e+01 -4.68648834e+01 9.23478699e+00 …,
4.81521721e+01 -1.10725357e+02 -1.53245071e+02]]

[[ 7.42268143e+01 -6.93738556e+00 1.75627327e+01 …,
-8.52064590e+01 -1.77411224e+02 -3.63268852e+01]
[ -5.00122452e+01 -2.74321156e+01 5.55675316e+00 …,
5.72885284e+01 -1.54153854e+02 -8.24862385e+00]
[ -7.10742340e+01 -2.71675797e+01 -2.48989258e+01 …,
9.00824585e+01 -9.69757538e+01 -3.72193451e+01]
[ -1.51661301e+00 1.67727280e+01 3.49994659e-01 …,
4.81879349e+01 -8.20349045e+01 -9.47352371e+01]
[ 3.75077133e+01 -3.13303528e+01 -4.80247116e+00 …,
1.94965630e+01 -7.80006332e+01 -9.93223724e+01]]]]]`

the maximum of the output is: 4817.28
the minimum of the output is: -13303.7

As you can see above, When the network changes to evaluation mode, the output of the network changes exponentially.

Can you show how you use Batchnorm in your model?
Are the inputs for the training model and the eval model the same?

As I have mentioned before, I have got a pretrained model in keras and I have transfered to the pytorch. I have never trained model so far but want to finetune for the future. Actually, the pretrained model was trained on mscoco dataset and my testing image is actually from it, i mean mscoco.

And here is the order of the layers and the usage of the batchnorm layer:

   path1 = ["conv2d_1", 
            "batch_normalization_1",
            "leaky_re_lu_1",
            "max_pooling2d_1",
            "conv2d_2", 
            "batch_normalization_2",
            "leaky_re_lu_2",
            "max_pooling2d_2",
            "conv2d_3", 
            "batch_normalization_3",
            "leaky_re_lu_3",
            "conv2d_4", 
            "batch_normalization_4",
            "leaky_re_lu_4",
            "conv2d_5", 
            "batch_normalization_5",
            "leaky_re_lu_5",
            "max_pooling2d_3",
            "conv2d_6", 
            "batch_normalization_6",
            "leaky_re_lu_6",
            "conv2d_7", 
            "batch_normalization_7",
            "leaky_re_lu_7",
            "conv2d_8", 
            "batch_normalization_8",
            "leaky_re_lu_8",
            "max_pooling2d_4",
            "conv2d_9", 
            "batch_normalization_9",
            "leaky_re_lu_9",
            "conv2d_10", 
            "batch_normalization_10",
            "leaky_re_lu_10",
            "conv2d_11", 
            "batch_normalization_11",
            "leaky_re_lu_11",
            "conv2d_12", 
            "batch_normalization_12",
            "leaky_re_lu_12",
            "conv2d_13", 
            "batch_normalization_13",
            "leaky_re_lu_13"
        ]
    paralle1 = ["max_pooling2d_5",
            "conv2d_14", 
            "batch_normalization_14",
            "leaky_re_lu_14",
            "conv2d_15", 
            "batch_normalization_15",
            "leaky_re_lu_15",
            "conv2d_16", 
            "batch_normalization_16",
            "leaky_re_lu_16",
            "conv2d_17", 
            "batch_normalization_17",
            "leaky_re_lu_17",
            "conv2d_18", 
            "batch_normalization_18",
            "leaky_re_lu_18",
            "conv2d_19", 
            "batch_normalization_19",
            "leaky_re_lu_19",
            "conv2d_20", 
            "batch_normalization_20",
            "leaky_re_lu_20"
        ]
    paralle2 = ["conv2d_21", 
            "batch_normalization_21",
            "leaky_re_lu_21",
            "space_to_depth_x2"
        ]
    path2 = [
            "conv2d_22", 
            "batch_normalization_22",
            "leaky_re_lu_22",
            "conv2d_23"
        ]

The other thing which is good to mention is that I have transfered the weight from keras using this code:

def loadWeights(self):
    model = load_model(self.modelUrl)        
    j = json.loads(model.to_json())
    for i, layer in enumerate(j['config']['layers']):
        ln = layer['name']
        l = model.get_layer(name=layer['name'])
        if layer['class_name'] != 'Concatenate':
            self.lid[ln] = l.input_shape[3]
        else:
            self.lid[ln] = l.input_shape[0][3]
        self.lod[ln] = l.output_shape[3]
        w = l.get_weights()
        if layer['class_name'] == 'Conv2D':
            filter_size = layer['config']['kernel_size'][0]
            if filter_size == 3:
                self.layers[ln] = nn.Conv2d(self.lid[ln],self.lod[ln], 
                    filter_size,padding=1,stride=1,bias=False)
            elif filter_size==1:
                self.layers[ln] = nn.Conv2d(self.lid[ln],self.lod[ln], 
                    filter_size,padding=0,stride=1,bias=False)
            self.layers[ln].weight.data = torch.from_numpy(w[0].transpose((3,2,0,1)))

        elif layer['class_name'] == 'BatchNormalization':
            self.layers[ln] = nn.BatchNorm2d(self.lid[ln])
            self.layers[ln].weight.data = torch.from_numpy(w[0])
            self.layers[ln].bias.data = torch.from_numpy(w[1])
            self.layers[ln].running_mean.data = torch.from_numpy(w[2])
            self.layers[ln].running_var.data = torch.from_numpy(w[3])

        elif layer['class_name'] == 'LeakyReLU':
            self.layers[ln] = nn.LeakyReLU(.1)
        elif layer['class_name'] == 'MaxPooling2D':
            self.layers[ln] = nn.MaxPool2d(2, 2)
        elif layer['class_name'] == 'Lambda':
            self.layers[ln] = scale_to_depth(2)

can you show your definition of PyTorch module/model?

from .model import Darknet19
import torch
import torch.nn as nn


class YoloV2(nn.Module):
    """Yolo version 2; It is an extented version of 
    yolo v1 capable of detecting 9000 object"""
    def __init__(self, modelUrl):
        super(YoloV2, self).__init__()
        self.modelUrl = modelUrl

        self.darknet19 = Darknet19(modelUrl)
        self.darknet19.loadWeights()
        self.weights = self.darknet19.layers
        arch = self.darknet19.arch

        self.path1 = self.makeSequence(arch[0])
        self.parallel1 = self.makeSequence(arch[1])
        self.parallel2 = self.makeSequence(arch[2])
        self.path2 = self.makeSequence(arch[3])

    def makeSequence(self, arch):
        layers = []
        for id, name in enumerate(arch):
            layers.append(self.weights[name])
        return nn.ModuleList(layers)

    def forward(self, input):
        out = input
        for layer in self.path1:
            out = layer(out)
        out1 = out.clone()
        for layer in self.parallel1:
            out1 = layer(out1)
        out2 = out.clone()
        for layer in self.parallel2:
            out2 = layer(out2)
        final = torch.cat([out2, out1], dim=1)
        for layer in self.path2:
            final = layer(final)
        return final


from keras.models import load_model
import torch
import tensorflow as tf
import json
import torch.nn as nn
import torch.autograd as autograd
import numpy as np

class scale_to_depth(nn.Module):
    def __init__(self, block_size=1):
        super(scale_to_depth, self).__init__()
        self.block_size = block_size        
    def forward(self, input):
        batch_size, in_channels, in_height, in_width = input.size()
        channels = in_channels * (self.block_size ** 2)

        out_height = int(in_height / self.block_size)
        out_width = int(in_width / self.block_size)

        input_view = input.contiguous().view(
            batch_size, in_channels, self.block_size, self.block_size,
            out_height, out_width)

        shuffle_out = input_view.permute(0, 1, 4, 2, 5, 3).contiguous()
        return shuffle_out.view(batch_size, channels, out_height, out_width)

class Darknet19:
    """This is the model to create the pretrained darknet19"""
    def __init__(self, modelUrl):
        super(Darknet19, self).__init__()
        self.modelUrl = modelUrl
        self.layers = {}
        self.lid = {}
        self.lod = {}
        self.lin = {}
        self.arch = self.makeArch()

    def makeArch(self):
        path1 = ["conv2d_1", 
                "batch_normalization_1",
                "leaky_re_lu_1",
                "max_pooling2d_1",
                "conv2d_2", 
                "batch_normalization_2",
                "leaky_re_lu_2",
                "max_pooling2d_2",
                "conv2d_3", 
                "batch_normalization_3",
                "leaky_re_lu_3",
                "conv2d_4", 
                "batch_normalization_4",
                "leaky_re_lu_4",
                "conv2d_5", 
                "batch_normalization_5",
                "leaky_re_lu_5",
                "max_pooling2d_3",
                "conv2d_6", 
                "batch_normalization_6",
                "leaky_re_lu_6",
                "conv2d_7", 
                "batch_normalization_7",
                "leaky_re_lu_7",
                "conv2d_8", 
                "batch_normalization_8",
                "leaky_re_lu_8",
                "max_pooling2d_4",
                "conv2d_9", 
                "batch_normalization_9",
                "leaky_re_lu_9",
                "conv2d_10", 
                "batch_normalization_10",
                "leaky_re_lu_10",
                "conv2d_11", 
                "batch_normalization_11",
                "leaky_re_lu_11",
                "conv2d_12", 
                "batch_normalization_12",
                "leaky_re_lu_12",
                "conv2d_13", 
                "batch_normalization_13",
                "leaky_re_lu_13"
            ]
        paralle1 = ["max_pooling2d_5",
                "conv2d_14", 
                "batch_normalization_14",
                "leaky_re_lu_14",
                "conv2d_15", 
                "batch_normalization_15",
                "leaky_re_lu_15",
                "conv2d_16", 
                "batch_normalization_16",
                "leaky_re_lu_16",
                "conv2d_17", 
                "batch_normalization_17",
                "leaky_re_lu_17",
                "conv2d_18", 
                "batch_normalization_18",
                "leaky_re_lu_18",
                "conv2d_19", 
                "batch_normalization_19",
                "leaky_re_lu_19",
                "conv2d_20", 
                "batch_normalization_20",
                "leaky_re_lu_20"
            ]
        paralle2 = ["conv2d_21", 
                "batch_normalization_21",
                "leaky_re_lu_21",
                "space_to_depth_x2"
            ]
        path2 = [
                "conv2d_22", 
                "batch_normalization_22",
                "leaky_re_lu_22",
                "conv2d_23"
            ]
        return path1, paralle1, paralle2, path2


    def loadWeights(self):
        model = load_model(self.modelUrl)        
        j = json.loads(model.to_json())


        for i, layer in enumerate(j['config']['layers']):
            ln = layer['name']
            l = model.get_layer(name=layer['name'])
            if layer['class_name'] != 'Concatenate':
                self.lid[ln] = l.input_shape[3]
            else:
                self.lid[ln] = l.input_shape[0][3]
            self.lod[ln] = l.output_shape[3]
            w = l.get_weights()
            if layer['class_name'] == 'Conv2D':
                filter_size = layer['config']['kernel_size'][0]
                if filter_size == 3:
                    self.layers[ln] = nn.Conv2d(self.lid[ln],self.lod[ln], 
                        filter_size,padding=1,stride=1,bias=False)
                elif filter_size==1:
                    self.layers[ln] = nn.Conv2d(self.lid[ln],self.lod[ln], 
                        filter_size,padding=0,stride=1,bias=False)
                self.layers[ln].weight.data = torch.from_numpy(w[0].transpose((3,2,0,1)))

            elif layer['class_name'] == 'BatchNormalization':
                self.layers[ln] = nn.BatchNorm2d(self.lid[ln])
                self.layers[ln].weight.data = torch.from_numpy(w[0])
                self.layers[ln].bias.data = torch.from_numpy(w[1])
                self.layers[ln].running_mean.data = torch.from_numpy(w[2])
                self.layers[ln].running_var.data = torch.from_numpy(w[3])

            elif layer['class_name'] == 'LeakyReLU':
                self.layers[ln] = nn.LeakyReLU(.1)
            elif layer['class_name'] == 'MaxPooling2D':
                self.layers[ln] = nn.MaxPool2d(2, 2)
            elif layer['class_name'] == 'Lambda':
                self.layers[ln] = scale_to_depth(2)

sorry, but I can’t find what’s wrong :pensive:

Have you solved it? I also have similar question. All my output is the same when I use model.eval().

maybe you can set the batchsize bigger