LSTM model output different between pytorch1.6 & 1.8

nlp
1

Hi,

LSTM model output different between pytorch1.6 & 1.8.
Could anyone help me? Thanks!

Ver 1.6

Python 3.8.5 (default, May 27 2021, 13:30:53) 
[GCC 9.3.0] on linux
Type "help", "copyright", "credits" or "license" for more information.
>>> import torch
>>> torch.__version__
**'1.6.0'**
>>> torch.manual_seed(0)
<torch._C.Generator object at 0xffff83c7c8b0>
>>> lstm = torch.nn.LSTM(10, 20, 2)
>>> input = torch.randn(5, 3, 10)
>>> h0 = torch.randn(2, 3, 20)
>>> c0 = torch.randn(2, 3, 20)
>>> output, (hn, cn) = lstm(input, (h0, c0))
>>> h0
tensor([[[ 0.7828, -1.1018,  1.1389,  0.0786,  0.3427,  1.1261, -0.3791,
           1.2558,  1.4030, -0.0642, -1.0546, -1.4373,  0.8565, -0.6454,
           0.9782, -0.4352,  0.5520, -2.6919,  0.7212,  0.8537],
         [ 1.5161,  0.3222,  0.4647,  0.5152, -0.8429,  0.9080, -1.6248,
          -0.7061,  1.1524, -2.2513,  0.1212, -1.2317,  0.6823, -0.6561,
          -1.2312,  0.0565,  0.2537,  0.2951, -0.2269,  0.3493],
         [ 0.2011,  0.6434, -0.6567, -0.3752,  1.1199,  0.8362, -1.4476,
           1.0720,  0.8041,  0.0208, -0.3491,  0.0881, -0.1052,  0.3467,
          -1.1734,  0.3854, -0.2713,  0.0072,  0.9125, -0.4537]],

        [[-1.3100, -2.1414,  1.0880,  0.1586, -0.1273,  1.1881, -0.1525,
          -0.5198,  0.6734,  0.6762,  0.3861,  0.4494, -0.0692, -0.3816,
          -0.0980, -0.3216,  1.7471,  0.8016, -1.4477,  0.1534],
         [-0.3492, -0.0731, -0.8770, -0.1190,  0.3472, -0.8273,  0.5216,
           0.8534,  1.0703,  0.4516,  0.8735, -0.0778,  1.4387, -0.8398,
           0.3218, -0.1580, -0.4267,  1.1945,  0.5842,  0.1174],
         [-0.4573,  0.4540, -0.1609,  0.6837, -0.8150,  0.4559,  1.3992,
          -0.6921, -0.5048,  1.1740, -0.0266,  0.6329,  0.7331,  0.0910,
           0.4534,  0.9553,  0.3396, -1.3020, -0.2528, -0.1419]]])
>>> c0
tensor([[[-4.0620e-01, -1.3253e-01,  3.6886e-01, -3.2871e-01, -3.2211e-01,
          -2.3720e-01, -2.1999e-01,  4.1985e-01,  1.4019e+00, -1.2868e+00,
           9.4751e-01, -7.5783e-01,  5.6568e-02, -1.7967e-01,  1.5641e+00,
          -4.7876e-01, -3.6283e-01, -1.8002e-01, -1.9621e-01,  4.0187e-01],
         [ 9.8542e-01, -1.5731e+00, -9.5435e-01, -9.9657e-01, -2.4342e-01,
          -1.8746e-01,  2.0924e+00, -4.3870e-01, -2.6173e+00, -2.9391e-01,
          -1.2045e+00,  3.9955e-01,  7.5689e-01,  4.2095e-01, -3.8234e-01,
          -2.0688e-01,  3.5718e-01,  1.5855e+00, -1.0789e+00, -2.0981e-01],
         [ 4.4348e-01, -2.5730e-01,  7.6807e-01, -8.7934e-04, -8.4371e-01,
          -5.9478e-01, -2.6929e+00, -2.0680e-01,  7.4818e-01,  5.8791e-01,
          -8.9436e-01, -2.3063e-01, -2.1472e-02,  5.6530e-01, -1.4501e-01,
           1.0794e+00, -8.8254e-01, -1.0448e+00, -1.2727e+00, -8.3937e-02]],

        [[-1.7817e-01, -1.8194e+00, -1.4392e-02,  1.3086e+00, -1.5097e+00,
           1.1350e+00,  1.2086e+00,  3.3407e-01,  9.6189e-01, -6.1815e-01,
           1.0697e+00, -1.8673e+00, -1.0265e+00, -5.7815e-01, -1.3305e+00,
           4.9888e-01, -1.2174e-01, -1.0897e-01, -6.0001e-01, -1.3004e+00],
         [-1.0210e+00, -6.1215e-01, -1.1412e-01,  1.1502e+00, -4.5270e-01,
           1.2160e+00,  8.0681e-01,  2.1000e+00,  1.6918e-01, -2.1890e+00,
          -5.1534e-01, -3.2971e-01, -9.5873e-01,  1.1979e-02,  4.8049e-01,
           3.6809e-01,  6.3906e-02, -4.6437e-02,  1.0845e+00,  1.9278e+00],
         [ 5.5433e-01,  5.3868e-02, -6.4048e-01,  1.9526e+00, -7.3037e-01,
           8.0987e-01, -8.5326e-01,  7.0106e-01, -1.1073e+00, -6.2357e-01,
           2.7765e+00, -3.7459e+00,  1.9703e+00, -1.1577e+00,  7.8968e-01,
           8.1870e-01, -6.6278e-01,  4.2700e-01, -1.6233e+00, -8.2866e-01]]])
**>>> hn**
tensor([[[ 1.8997e-01,  2.9446e-02,  3.6393e-02, -1.3239e-01, -7.6356e-02,
          -4.9796e-02, -1.5002e-02, -1.3821e-01, -6.7487e-03, -2.4862e-01,
           1.6773e-01, -8.7203e-02,  3.0214e-02,  1.0023e-01,  8.4150e-02,
          -2.5039e-01, -1.6382e-01,  1.3463e-02,  1.4638e-01, -1.2570e-01],
         [-5.2918e-02,  1.3052e-01, -1.8922e-01, -8.0349e-02,  1.7770e-01,
           1.6402e-01,  1.9102e-01, -3.1886e-01, -3.1571e-02, -1.4692e-01,
           5.1075e-02, -5.3673e-02,  7.2232e-02, -5.3237e-02,  3.9375e-01,
          -4.6529e-02,  7.9471e-02,  5.0720e-02,  1.1154e-02, -1.7962e-01],
         [-1.1576e-02,  1.7528e-01, -7.8073e-02, -1.7269e-01, -1.1405e-01,
           1.2984e-01, -1.8881e-01,  1.4909e-02, -8.5723e-02,  5.5809e-02,
           9.5540e-02, -1.3547e-01, -5.4608e-02, -8.4146e-02,  2.0542e-01,
          -4.0577e-02, -5.4770e-02,  3.8670e-02,  9.0426e-02,  6.7487e-02]],

        [[ 5.1958e-03, -1.2238e-01, -3.8408e-02, -7.0678e-02, -1.4338e-01,
          -1.7208e-02, -2.8793e-02,  1.2637e-02, -7.1470e-02, -6.5546e-02,
          -4.2220e-02, -4.7928e-02, -5.3376e-02,  1.9727e-01, -8.8258e-02,
          -5.1277e-02, -1.6312e-02, -4.3475e-02, -5.6749e-02,  3.8840e-02],
         [ 2.9578e-02, -1.1640e-01, -7.6083e-02, -6.1399e-02, -6.5441e-03,
           7.1630e-03, -5.1347e-02, -2.5899e-02, -1.1294e-01, -4.9712e-02,
          -9.9401e-02,  1.1858e-02, -1.7954e-02,  2.0904e-01, -3.7076e-02,
          -5.2638e-02,  2.6925e-02,  7.9106e-02, -7.9513e-02,  1.8975e-01],
         [ 6.9908e-02, -7.6080e-02, -2.3130e-02, -9.0202e-02, -1.3944e-01,
           2.0624e-02, -5.3879e-02,  6.0119e-02, -1.0089e-01,  2.2968e-04,
          -1.7591e-02, -9.6028e-02, -1.7794e-02,  1.6758e-01, -8.5724e-02,
          -4.9486e-02,  2.5974e-02, -4.7456e-02, -1.0742e-01, -5.8604e-02]]],
       grad_fn=<StackBackward>)
**>>> cn**
tensor([[[ 2.6768e-01,  1.1605e-01,  6.0007e-02, -2.8404e-01, -2.1064e-01,
          -1.4188e-01, -5.5860e-02, -2.6974e-01, -2.0897e-02, -3.6769e-01,
           2.5299e-01, -2.1963e-01,  8.4530e-02,  2.0038e-01,  1.5994e-01,
          -5.5976e-01, -2.4437e-01,  2.7045e-02,  2.5375e-01, -2.4539e-01],
         [-1.0101e-01,  2.4400e-01, -3.9518e-01, -2.0050e-01,  2.8268e-01,
           3.2269e-01,  3.2881e-01, -5.7887e-01, -7.2371e-02, -2.8195e-01,
           8.0358e-02, -1.7127e-01,  1.7305e-01, -9.3407e-02,  6.3815e-01,
          -8.0898e-02,  1.8861e-01,  1.4010e-01,  2.6388e-02, -3.7314e-01],
         [-4.1374e-02,  3.4264e-01, -1.5974e-01, -4.2603e-01, -1.9505e-01,
           2.0097e-01, -2.6677e-01,  2.7834e-02, -2.4039e-01,  1.1889e-01,
           1.6746e-01, -2.5576e-01, -1.7141e-01, -2.2255e-01,  3.5760e-01,
          -7.8496e-02, -1.5505e-01,  1.1910e-01,  1.8533e-01,  1.2208e-01]],

        [[ 9.7706e-03, -2.5779e-01, -7.1028e-02, -1.7264e-01, -3.7643e-01,
          -3.0571e-02, -5.7411e-02,  2.7361e-02, -1.4561e-01, -1.3843e-01,
          -9.5624e-02, -9.5882e-02, -1.1316e-01,  4.6548e-01, -1.8931e-01,
          -1.0731e-01, -3.6554e-02, -8.0840e-02, -9.5162e-02,  6.6051e-02],
         [ 5.3684e-02, -2.4183e-01, -1.3862e-01, -1.7321e-01, -1.5903e-02,
           1.2568e-02, -1.0623e-01, -5.5737e-02, -2.1350e-01, -9.9377e-02,
          -2.0017e-01,  2.1286e-02, -3.6946e-02,  4.8883e-01, -7.5772e-02,
          -1.0905e-01,  5.3231e-02,  1.4163e-01, -1.3482e-01,  3.4709e-01],
         [ 1.3303e-01, -1.5051e-01, -4.0423e-02, -2.1040e-01, -3.4075e-01,
           3.8102e-02, -1.0817e-01,  1.3218e-01, -2.1749e-01,  4.7524e-04,
          -3.8515e-02, -1.7484e-01, -3.8400e-02,  3.6918e-01, -1.8216e-01,
          -1.0429e-01,  5.7928e-02, -9.0111e-02, -1.7630e-01, -1.0191e-01]]],
       grad_fn=<StackBackward>)
**>>> output**
tensor([[[ 1.3763e-02, -4.7405e-01, -4.7226e-02,  2.1277e-01, -2.8291e-01,
           6.6150e-02,  1.2686e-01,  2.6877e-01,  2.5887e-01, -1.3825e-01,
           3.0942e-02, -3.2615e-01, -2.3796e-01, -8.8688e-02, -4.3774e-01,
           1.6219e-01, -2.5254e-01, -1.0656e-01,  6.6284e-03, -2.1903e-01],
         [-4.0682e-02, -6.1078e-02, -9.1357e-03,  1.7924e-01, -4.1341e-02,
           3.2040e-01,  2.1028e-01,  1.7965e-01,  6.6776e-02, -3.5610e-01,
          -2.6748e-01,  3.4839e-03, -2.4401e-01,  2.7502e-01, -3.1777e-02,
          -5.9150e-02,  8.2222e-02,  1.3290e-03,  2.6449e-01,  3.6030e-01],
         [ 9.9093e-02, -8.0816e-02, -1.5692e-01,  4.4483e-01, -1.3570e-01,
           1.8364e-01, -1.6354e-01,  1.8791e-01, -1.1599e-01,  1.9839e-02,
           3.4043e-01, -4.4461e-01,  2.1897e-01, -2.4662e-01,  7.7254e-02,
           1.9968e-01, -5.0982e-02, -1.1274e-02, -4.7323e-01, -4.2296e-01]],

        [[-8.5882e-03, -3.0510e-01, -5.4615e-02,  1.0872e-01, -2.6207e-01,
           3.0682e-02, -5.1152e-02,  6.6414e-02,  6.9161e-02, -1.1180e-01,
          -2.3297e-02, -1.6112e-01, -1.3799e-01,  4.8269e-02, -2.6084e-01,
           4.9673e-02, -1.3916e-01, -8.9519e-02, -1.0470e-01, -1.9094e-01],
         [-5.6798e-02, -1.2192e-01, -5.7072e-02,  4.4346e-02, -2.9637e-02,
           2.2989e-01,  2.3716e-02,  1.0543e-01, -6.7128e-02, -2.2290e-01,
          -1.6604e-01, -1.3996e-02, -9.3793e-02,  1.7330e-01, -4.8665e-02,
          -3.5167e-02,  6.1314e-02,  1.0554e-01,  7.0937e-02,  3.1433e-01],
         [ 9.8249e-02, -7.0406e-02, -7.7364e-02,  1.6208e-01, -1.3774e-01,
           9.2777e-02, -1.2063e-01,  9.0683e-02, -1.0360e-01,  3.6084e-02,
           1.8434e-01, -3.6086e-01,  1.1622e-01,  9.7443e-03, -2.8319e-02,
           1.1378e-02,  2.6941e-02, -6.3948e-02, -3.3294e-01, -2.3187e-01]],

        [[-1.2551e-03, -1.9535e-01, -4.1329e-02,  2.0645e-02, -2.0987e-01,
           1.0554e-02, -6.2501e-02,  3.4985e-02, -2.3207e-02, -1.0440e-01,
          -4.3044e-02, -1.1287e-01, -8.7696e-02,  1.2633e-01, -1.6148e-01,
          -1.6297e-02, -7.5600e-02, -7.2126e-02, -1.0574e-01, -1.2205e-01],
         [-2.5500e-02, -1.3617e-01, -7.9953e-02, -1.7516e-02, -2.3220e-02,
           1.3753e-01, -3.5743e-02,  3.3521e-02, -1.1098e-01, -1.4257e-01,
          -1.1949e-01, -7.4510e-03, -3.7011e-02,  1.8061e-01, -3.6939e-02,
          -4.1680e-02,  3.3600e-02,  1.1253e-01, -9.8322e-03,  2.6338e-01],
         [ 1.0436e-01, -9.7611e-02, -3.6283e-02,  9.0275e-03, -1.5239e-01,
           3.1421e-02, -6.1998e-02,  8.8978e-02, -7.9108e-02,  1.6959e-02,
           1.0767e-01, -2.1643e-01,  2.3294e-02,  9.8876e-02, -7.6316e-02,
          -3.8616e-02,  6.0168e-02, -8.4881e-02, -2.6125e-01, -1.6038e-01]],

        [[ 4.6940e-03, -1.4647e-01, -3.8358e-02, -2.8346e-02, -1.7929e-01,
           9.0697e-04, -6.2204e-02,  1.2483e-02, -8.0002e-02, -8.5253e-02,
          -5.1794e-02, -7.2530e-02, -5.1062e-02,  1.6837e-01, -1.0613e-01,
          -4.0402e-02, -3.9074e-02, -3.7650e-02, -1.0147e-01, -4.5290e-02],
         [ 4.4334e-03, -1.3210e-01, -8.2383e-02, -4.4179e-02, -8.1091e-03,
           5.3340e-02, -4.6708e-02, -4.5104e-03, -1.2363e-01, -8.4998e-02,
          -1.2310e-01,  7.5327e-03, -2.3798e-02,  2.0843e-01, -4.2913e-02,
          -4.4089e-02,  3.3974e-02,  1.0646e-01, -5.2370e-02,  2.1589e-01],
         [ 9.4963e-02, -9.0009e-02, -2.5523e-02, -6.2247e-02, -1.5550e-01,
           2.0828e-02, -3.7283e-02,  7.2984e-02, -8.4218e-02,  2.2734e-03,
           5.0392e-02, -1.3427e-01, -1.0935e-02,  1.4423e-01, -8.4868e-02,
          -5.6570e-02,  3.9734e-02, -7.7401e-02, -1.7031e-01, -9.5842e-02]],

        [[ 5.1958e-03, -1.2238e-01, -3.8408e-02, -7.0678e-02, -1.4338e-01,
          -1.7208e-02, -2.8793e-02,  1.2637e-02, -7.1470e-02, -6.5546e-02,
          -4.2220e-02, -4.7928e-02, -5.3376e-02,  1.9727e-01, -8.8258e-02,
          -5.1277e-02, -1.6312e-02, -4.3475e-02, -5.6749e-02,  3.8840e-02],
         [ 2.9578e-02, -1.1640e-01, -7.6083e-02, -6.1399e-02, -6.5441e-03,
           7.1630e-03, -5.1347e-02, -2.5899e-02, -1.1294e-01, -4.9712e-02,
          -9.9401e-02,  1.1858e-02, -1.7954e-02,  2.0904e-01, -3.7076e-02,
          -5.2638e-02,  2.6925e-02,  7.9106e-02, -7.9513e-02,  1.8975e-01],
         [ 6.9908e-02, -7.6080e-02, -2.3130e-02, -9.0202e-02, -1.3944e-01,
           2.0624e-02, -5.3879e-02,  6.0119e-02, -1.0089e-01,  2.2968e-04,
          -1.7591e-02, -9.6028e-02, -1.7794e-02,  1.6758e-01, -8.5724e-02,
          -4.9486e-02,  2.5974e-02, -4.7456e-02, -1.0742e-01, -5.8604e-02]]],
       grad_fn=<StackBackward>)

Ver 1.8

Python 3.8.5 (default, May 27 2021, 13:30:53) 
[GCC 9.3.0] on linux
Type "help", "copyright", "credits" or "license" for more information.
>>> import torch
>>> torch.__version__
**'1.8.1'**
>>> torch.manual_seed(0)
<torch._C.Generator object at 0xffffae885450>
>>> lstm = torch.nn.LSTM(10, 20, 2)
>>> input = torch.randn(5, 3, 10)
>>> h0 = torch.randn(2, 3, 20)
>>> c0 = torch.randn(2, 3, 20)
>>> output, (hn, cn) = lstm(input, (h0, c0))
>>> h0
tensor([[[ 0.7828, -1.1018,  1.1389,  0.0786,  0.3427,  1.1261, -0.3791,
           1.2558,  1.4030, -0.0642, -1.0546, -1.4373,  0.8565, -0.6454,
           0.9782, -0.4352,  0.5520, -2.6919,  0.7212,  0.8537],
         [ 1.5161,  0.3222,  0.4647,  0.5152, -0.8429,  0.9080, -1.6248,
          -0.7061,  1.1524, -2.2513,  0.1212, -1.2317,  0.6823, -0.6561,
          -1.2312,  0.0565,  0.2537,  0.2951, -0.2269,  0.3493],
         [ 0.2011,  0.6434, -0.6567, -0.3752,  1.1199,  0.8362, -1.4476,
           1.0720,  0.8041,  0.0208, -0.3491,  0.0881, -0.1052,  0.3467,
          -1.1734,  0.3854, -0.2713,  0.0072,  0.9125, -0.4537]],

        [[-1.3100, -2.1414,  1.0880,  0.1586, -0.1273,  1.1881, -0.1525,
          -0.5198,  0.6734,  0.6762,  0.3861,  0.4494, -0.0692, -0.3816,
          -0.0980, -0.3216,  1.7471,  0.8016, -1.4477,  0.1534],
         [-0.3492, -0.0731, -0.8770, -0.1190,  0.3472, -0.8273,  0.5216,
           0.8534,  1.0703,  0.4516,  0.8735, -0.0778,  1.4387, -0.8398,
           0.3218, -0.1580, -0.4267,  1.1945,  0.5842,  0.1174],
         [-0.4573,  0.4540, -0.1609,  0.6837, -0.8150,  0.4559,  1.3992,
          -0.6921, -0.5048,  1.1740, -0.0266,  0.6329,  0.7331,  0.0910,
           0.4534,  0.9553,  0.3396, -1.3020, -0.2528, -0.1419]]])
>>> c0
tensor([[[-4.0620e-01, -1.3253e-01,  3.6886e-01, -3.2871e-01, -3.2211e-01,
          -2.3720e-01, -2.1999e-01,  4.1985e-01,  1.4019e+00, -1.2868e+00,
           9.4751e-01, -7.5783e-01,  5.6568e-02, -1.7967e-01,  1.5641e+00,
          -4.7876e-01, -3.6283e-01, -1.8002e-01, -1.9621e-01,  4.0187e-01],
         [ 9.8542e-01, -1.5731e+00, -9.5435e-01, -9.9657e-01, -2.4342e-01,
          -1.8746e-01,  2.0924e+00, -4.3870e-01, -2.6173e+00, -2.9391e-01,
          -1.2045e+00,  3.9955e-01,  7.5689e-01,  4.2095e-01, -3.8234e-01,
          -2.0688e-01,  3.5718e-01,  1.5855e+00, -1.0789e+00, -2.0981e-01],
         [ 4.4348e-01, -2.5730e-01,  7.6807e-01, -8.7934e-04, -8.4371e-01,
          -5.9478e-01, -2.6929e+00, -2.0680e-01,  7.4818e-01,  5.8791e-01,
          -8.9436e-01, -2.3063e-01, -2.1472e-02,  5.6530e-01, -1.4501e-01,
           1.0794e+00, -8.8254e-01, -1.0448e+00, -1.2727e+00, -8.3937e-02]],

        [[-1.7817e-01, -1.8194e+00, -1.4392e-02,  1.3086e+00, -1.5097e+00,
           1.1350e+00,  1.2086e+00,  3.3407e-01,  9.6189e-01, -6.1815e-01,
           1.0697e+00, -1.8673e+00, -1.0265e+00, -5.7815e-01, -1.3305e+00,
           4.9888e-01, -1.2174e-01, -1.0897e-01, -6.0001e-01, -1.3004e+00],
         [-1.0210e+00, -6.1215e-01, -1.1412e-01,  1.1502e+00, -4.5270e-01,
           1.2160e+00,  8.0681e-01,  2.1000e+00,  1.6918e-01, -2.1890e+00,
          -5.1534e-01, -3.2971e-01, -9.5873e-01,  1.1979e-02,  4.8049e-01,
           3.6809e-01,  6.3906e-02, -4.6437e-02,  1.0845e+00,  1.9278e+00],
         [ 5.5433e-01,  5.3868e-02, -6.4048e-01,  1.9526e+00, -7.3037e-01,
           8.0987e-01, -8.5326e-01,  7.0106e-01, -1.1073e+00, -6.2357e-01,
           2.7765e+00, -3.7459e+00,  1.9703e+00, -1.1577e+00,  7.8968e-01,
           8.1870e-01, -6.6278e-01,  4.2700e-01, -1.6233e+00, -8.2866e-01]]])
**>>> hn**
tensor([[[ 0.2093,  0.0327,  0.0454, -0.1409, -0.0759, -0.0519, -0.0338,
          -0.1367, -0.0035, -0.0070,  0.0883, -0.0201,  0.0021,  0.0037,
           0.0028, -0.0486, -0.2056,  0.0053,  0.1457, -0.0915],
         [-0.0269,  0.1360, -0.1877, -0.0770,  0.1514,  0.1569,  0.1829,
          -0.3219, -0.0306, -0.0425,  0.0165, -0.0147, -0.0091, -0.0037,
           0.0569, -0.0409,  0.0629,  0.0417, -0.0020, -0.1851],
         [-0.0173,  0.1775, -0.0818, -0.1738, -0.1043,  0.1407, -0.1624,
           0.0088, -0.0899,  0.0345,  0.0085,  0.0056, -0.0197, -0.0139,
           0.1051,  0.0058, -0.0515,  0.0229,  0.0821,  0.0734]],

        [[ 0.0053, -0.1342, -0.0124, -0.0937, -0.1015, -0.0087, -0.0200,
           0.0461, -0.0846, -0.0110, -0.0172, -0.0039, -0.0103,  0.0469,
          -0.0266, -0.0184,  0.0051, -0.0234, -0.0333,  0.0216],
         [ 0.0389, -0.1299, -0.0406, -0.0799, -0.0075,  0.0020, -0.0396,
          -0.0058, -0.1006,  0.0013, -0.0099,  0.0037, -0.0047,  0.0465,
          -0.0135, -0.0080,  0.0440,  0.0518, -0.0809,  0.1619],
         [ 0.0333, -0.0836,  0.0029, -0.0802, -0.0985,  0.0326, -0.0688,
           0.0526, -0.1093, -0.0049, -0.0177, -0.0091, -0.0082,  0.0402,
          -0.0292, -0.0152,  0.0104, -0.0321, -0.0636, -0.0270]]],
       grad_fn=<StackBackward>)
**>>> cn**
tensor([[[ 0.2979,  0.1268,  0.0750, -0.3108, -0.2097, -0.1503, -0.1236,
          -0.2710, -0.0110, -0.1301,  0.2630, -0.0828,  0.0138,  0.0282,
           0.0421, -0.1742, -0.3070,  0.0106,  0.2508, -0.1782],
         [-0.0516,  0.2524, -0.3922, -0.1934,  0.2419,  0.3090,  0.3054,
          -0.5903, -0.0710, -0.1395,  0.0608, -0.0794, -0.0250, -0.0127,
           0.1640, -0.1195,  0.1476,  0.1167, -0.0048, -0.3749],
         [-0.0588,  0.3353, -0.1663, -0.4293, -0.1815,  0.2166, -0.2287,
           0.0164, -0.2457,  0.1135,  0.0320,  0.0266, -0.0601, -0.0373,
           0.2655,  0.0198, -0.1482,  0.0718,  0.1677,  0.1344]],

        [[ 0.0098, -0.2779, -0.0227, -0.2263, -0.2625, -0.0156, -0.0385,
           0.0959, -0.1789, -0.0417, -0.0567, -0.0192, -0.0578,  0.1519,
          -0.0943, -0.0724,  0.0124, -0.0450, -0.0551,  0.0368],
         [ 0.0719, -0.2636, -0.0747, -0.2012, -0.0185,  0.0034, -0.0786,
          -0.0125, -0.1968,  0.0049, -0.0327,  0.0198, -0.0257,  0.1440,
          -0.0497, -0.0332,  0.0897,  0.0960, -0.1384,  0.3007],
         [ 0.0627, -0.1655,  0.0051, -0.1862, -0.2407,  0.0595, -0.1374,
           0.1122, -0.2380, -0.0176, -0.0566, -0.0422, -0.0413,  0.1322,
          -0.1076, -0.0620,  0.0236, -0.0614, -0.1045, -0.0460]]],
       grad_fn=<StackBackward>)
**>>> output**
tensor([[[-2.6673e-02, -4.9636e-01, -3.7769e-02,  2.3962e-01, -2.6064e-01,
           7.6862e-02,  1.8103e-01,  2.9257e-01,  2.5556e-01, -1.9950e-02,
           1.0540e-01, -8.7910e-02, -6.5437e-02, -3.1375e-02, -1.1858e-01,
           9.0027e-02, -2.2028e-01, -7.4762e-02,  9.2547e-04, -2.2558e-01],
         [-4.4369e-02, -7.1114e-02,  7.7424e-03,  1.7355e-01, -4.2202e-02,
           3.2365e-01,  2.0331e-01,  1.6907e-01,  6.3443e-02, -1.4883e-01,
          -8.2721e-02,  2.1776e-02, -2.5007e-02,  2.6112e-02, -4.7448e-02,
          -1.0190e-02,  8.5294e-02, -8.4146e-03,  2.7708e-01,  3.5297e-01],
         [ 9.8084e-02, -8.9517e-02, -1.5645e-01,  4.4393e-01, -1.4666e-01,
           1.8609e-01, -1.5919e-01,  1.8339e-01, -1.1092e-01, -2.1365e-02,
           1.5143e-01, -2.6989e-01,  4.6150e-02, -5.8090e-02, -2.3952e-02,
           2.5324e-02, -5.6398e-02, -9.6794e-03, -4.7952e-01, -4.2185e-01]],

        [[-2.8448e-02, -3.2872e-01, -1.7473e-02,  1.1152e-01, -2.2227e-01,
           4.5737e-02, -4.0928e-04,  1.0374e-01,  6.3578e-02, -1.3689e-02,
           1.5962e-02, -3.3820e-02, -2.5140e-02,  1.5902e-02, -6.1120e-02,
           3.9927e-03, -8.5676e-02, -6.6305e-02, -9.4906e-02, -2.0794e-01],
         [-3.2044e-02, -1.2870e-01, -2.7621e-02,  2.4814e-02, -2.8480e-02,
           2.1466e-01,  2.5901e-02,  1.0573e-01, -4.1017e-02, -4.2168e-02,
          -2.5227e-02,  5.4264e-03, -1.2931e-02,  3.5586e-02, -2.6798e-02,
          -6.5280e-03,  6.5973e-02,  7.0923e-02,  7.0427e-02,  2.8321e-01],
         [ 6.6207e-02, -7.7442e-02, -7.7305e-02,  1.6759e-01, -1.5048e-01,
           1.1239e-01, -1.2306e-01,  7.9206e-02, -1.1039e-01, -5.2124e-03,
           1.7111e-02, -8.4097e-02,  2.9212e-03, -1.1128e-03, -2.8720e-02,
          -1.9616e-02, -7.3049e-04, -5.1871e-02, -3.0112e-01, -2.2233e-01]],

        [[-1.1235e-02, -2.2028e-01, -1.7289e-03,  3.9407e-03, -1.7297e-01,
           2.1011e-02, -3.2284e-02,  7.1805e-02, -2.7267e-02, -1.5747e-02,
          -1.0041e-02, -1.3044e-02, -1.1577e-02,  3.5592e-02, -3.8378e-02,
          -1.3451e-02, -3.8705e-02, -4.9134e-02, -7.8068e-02, -1.2882e-01],
         [ 4.0282e-03, -1.4168e-01, -4.2354e-02, -4.3460e-02, -2.5009e-02,
           1.2293e-01, -2.8206e-02,  4.7183e-02, -8.3002e-02, -1.8047e-02,
          -1.4284e-02,  4.0554e-03, -4.4495e-03,  3.8579e-02, -1.7993e-02,
          -5.9344e-03,  4.3580e-02,  7.3278e-02, -8.9519e-03,  2.3745e-01],
         [ 5.3321e-02, -9.2912e-02, -2.0832e-02,  2.8523e-02, -1.3333e-01,
           6.0907e-02, -8.0386e-02,  7.4926e-02, -1.0071e-01, -8.2051e-03,
          -1.8861e-03, -2.8327e-02, -9.7404e-03,  2.6251e-02, -3.2134e-02,
          -2.9391e-02,  2.1432e-02, -6.0290e-02, -1.9627e-01, -1.3897e-01]],

        [[ 6.4694e-03, -1.7008e-01, -5.5919e-03, -5.7946e-02, -1.4017e-01,
           8.8724e-03, -4.3120e-02,  4.4612e-02, -7.7500e-02, -1.2995e-02,
          -1.3913e-02, -4.0142e-03, -5.9943e-03,  4.3914e-02, -2.7149e-02,
          -1.4827e-02, -1.2854e-02, -2.7588e-02, -7.3126e-02, -4.8311e-02],
         [ 2.4794e-02, -1.4159e-01, -4.7556e-02, -6.7038e-02, -1.3367e-02,
           4.4180e-02, -3.5167e-02,  1.3268e-02, -1.0014e-01, -2.9925e-03,
          -1.4467e-02,  5.8539e-03, -4.7325e-03,  4.5160e-02, -2.0185e-02,
          -6.5915e-03,  4.6098e-02,  6.9080e-02, -5.2515e-02,  1.9137e-01],
         [ 4.7335e-02, -8.8521e-02, -1.3537e-03, -4.4283e-02, -1.1698e-01,
           4.1425e-02, -5.8499e-02,  6.1519e-02, -1.0522e-01, -9.2887e-03,
          -1.2032e-02, -1.3911e-02, -1.0989e-02,  3.5330e-02, -2.9526e-02,
          -2.3759e-02,  1.5512e-02, -5.2692e-02, -1.1012e-01, -6.9745e-02]],

        [[ 5.2770e-03, -1.3420e-01, -1.2387e-02, -9.3677e-02, -1.0147e-01,
          -8.6947e-03, -1.9955e-02,  4.6094e-02, -8.4619e-02, -1.1043e-02,
          -1.7218e-02, -3.8805e-03, -1.0286e-02,  4.6911e-02, -2.6573e-02,
          -1.8406e-02,  5.1034e-03, -2.3441e-02, -3.3302e-02,  2.1556e-02],
         [ 3.8912e-02, -1.2992e-01, -4.0570e-02, -7.9918e-02, -7.4524e-03,
           1.9874e-03, -3.9574e-02, -5.8284e-03, -1.0056e-01,  1.3243e-03,
          -9.8694e-03,  3.6674e-03, -4.7457e-03,  4.6518e-02, -1.3538e-02,
          -7.9770e-03,  4.3989e-02,  5.1840e-02, -8.0910e-02,  1.6187e-01],
         [ 3.3277e-02, -8.3621e-02,  2.8798e-03, -8.0173e-02, -9.8529e-02,
           3.2648e-02, -6.8788e-02,  5.2587e-02, -1.0930e-01, -4.8666e-03,
          -1.7717e-02, -9.0626e-03, -8.1795e-03,  4.0153e-02, -2.9160e-02,
          -1.5184e-02,  1.0386e-02, -3.2097e-02, -6.3629e-02, -2.7034e-02]]],
       grad_fn=<StackBackward>)
2

Could you check, if the .parameters() are also initialized in the same way in both versions?
Based on the output, I would guess that either the implementation of some random number generation methods was changed or the initialization of this module.

3

@ptrblck Thanks. I tried your guess, but it may not have much to do with parameter initialization, it seems that the initialization process is consistent.

and I just modify the size of input, h0, c0 tensor, the output is the same.

Ver 1.6

Python 3.8.5 (default, May 27 2021, 13:30:53) 
[GCC 9.3.0] on linux
Type "help", "copyright", "credits" or "license" for more information.
>>> import torch
>>> torch.__version__
'1.6.0'
>>> torch.manual_seed(0)
<torch._C.Generator object at 0xffff8d5e78b0>
>>> lstm = torch.nn.LSTM(2, 3, 2)
>>> input = torch.randn(2, 2, 2)
>>> h0 = torch.randn(2, 2, 3)
>>> c0 = torch.randn(2, 2, 3)
>>> output, (hn, cn) = lstm(input, (h0, c0))
>>> h0
tensor([[[-0.6731, -1.2095,  1.3441],
         [ 2.3832, -0.5665, -1.1536]],

        [[-2.5023,  0.8756, -2.6726],
         [-0.0313,  0.4988, -0.5233]]])
>>> c0
tensor([[[-0.2515, -1.0555, -0.5593],
         [-0.1197, -0.1635, -0.2505]],

        [[-1.0882,  0.0044,  0.2328],
         [-0.8201, -0.6790,  1.4251]]])
>>> hn
tensor([[[ 0.3527, -0.1816, -0.0543],
         [ 0.0832, -0.2947, -0.0653]],

        [[-0.3593, -0.1131,  0.0362],
         [-0.2696, -0.1553,  0.0931]]], grad_fn=<StackBackward>)
>>> cn
tensor([[[ 0.4743, -0.4206, -0.2134],
         [ 0.1192, -0.5162, -0.1760]],

        [[-0.9582, -0.1660,  0.0946],
         [-0.6909, -0.2449,  0.2589]]], grad_fn=<StackBackward>)
>>> output
tensor([[[-5.7776e-01, -8.8637e-02,  5.1628e-04],
         [-3.1305e-01, -1.5366e-01,  2.0715e-01]],

        [[-3.5934e-01, -1.1314e-01,  3.6213e-02],
         [-2.6963e-01, -1.5533e-01,  9.3064e-02]]], grad_fn=<StackBackward>)

Ver1.8

Python 3.8.5 (default, May 27 2021, 13:30:53) 
[GCC 9.3.0] on linux
Type "help", "copyright", "credits" or "license" for more information.
>>> import torch
>>> torch.__version__
'1.8.1'
>>> torch.manual_seed(0)
<torch._C.Generator object at 0xffffaaf233b0>
>>> lstm = torch.nn.LSTM(2, 3, 2)
>>> input = torch.randn(2, 2, 2)
>>> h0 = torch.randn(2, 2, 3)
>>> c0 = torch.randn(2, 2, 3)
>>> output, (hn, cn) = lstm(input, (h0, c0))
>>> h0
tensor([[[-0.6731, -1.2095,  1.3441],
         [ 2.3832, -0.5665, -1.1536]],

        [[-2.5023,  0.8756, -2.6726],
         [-0.0313,  0.4988, -0.5233]]])
>>> c0
tensor([[[-0.2515, -1.0555, -0.5593],
         [-0.1197, -0.1635, -0.2505]],

        [[-1.0882,  0.0044,  0.2328],
         [-0.8201, -0.6790,  1.4251]]])
>>> hn
tensor([[[ 0.3527, -0.1816, -0.0543],
         [ 0.0832, -0.2947, -0.0653]],

        [[-0.3593, -0.1131,  0.0362],
         [-0.2696, -0.1553,  0.0931]]], grad_fn=<StackBackward>)
>>> cn
tensor([[[ 0.4743, -0.4206, -0.2134],
         [ 0.1192, -0.5162, -0.1760]],

        [[-0.9582, -0.1660,  0.0946],
         [-0.6909, -0.2449,  0.2589]]], grad_fn=<StackBackward>)
>>> output
tensor([[[-5.7776e-01, -8.8637e-02,  5.1628e-04],
         [-3.1305e-01, -1.5366e-01,  2.0715e-01]],

        [[-3.5934e-01, -1.1314e-01,  3.6213e-02],
         [-2.6963e-01, -1.5533e-01,  9.3064e-02]]], grad_fn=<StackBackward>)

Hope for help!

4

In your code snippets you are not showing the internal parameters, so could you add:

for name, param in lstm.named_parameters():
    print(name, param)