Model.eval() giving different result when shuffle is True and False

2

I cannot reproduce the issue after calling model.eval() as neither the running stats are updated (as mentioned here nor is the output showing any difference when shuffling the inputs using this code snippet:

embedding_sizes = [(3, 2), (2, 1), (2, 1), (2, 1)]
model = testNet(embedding_sizes,6)
print(model)
model.eval()

for name, module in model.named_modules():
    if 'bn' in name:
        print(module.running_mean)
        print(module.running_var)

# first pass
x_cat = torch.randint(0, 2, (16, 4))
x_cont = torch.randn(16, 6)
out_ref = model(x_cat, x_cont)

# running stats are not updated
for name, module in model.named_modules():
    if 'bn' in name:
        print(module.running_mean)
        print(module.running_var)

# shuffle data
idx = torch.randperm(x_cat.size(0))
out = model(x_cat[idx], x_cont[idx])

# compare to reference output
print((out - out_ref[idx]).abs().max())
> tensor(0., grad_fn=<MaxBackward1>)

# check running stats
# running stats are not updated
for name, module in model.named_modules():
    if 'bn' in name:
        print(module.running_mean)
        print(module.running_var)
1 Like