Hi, I meet a strange thing when I use nn.DataParallel and multi-gpu (say, 2) and Adam with weight_dacay not equal to 0, to optimize an easy network, with its conv1 detached from loss function. The result shows that Adam optimizes conv1's weight. When I change weight_decay to 0, there’s no bug. When I run with single gpu, there’s no bug either. Here are the code and result, can anyone help me with this?
(pytorch 0.4 and 1.0 both)
code
#!/user/bin/python
# coding=utf-8
import os
import torch
import torch.nn.functional as F
from torch import nn
class Net(nn.Module):
def __init__(self):
super(Net, self).__init__()
self.conv1 = nn.Conv2d(3, 3, 3, 1, 1)
self.conv2 = nn.Conv2d(3, 3, 3, 1, 1)
self.init()
def init(self):
for m in self.modules():
if isinstance(m, nn.Conv2d):
nn.init.constant_(m.weight, 1)
def forward(self, x):
conv1 = self.conv1(x)
conv2 = self.conv2(conv1.detach())
return conv1, conv2
BS = 20
os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID" # see issue #152
os.environ["CUDA_VISIBLE_DEVICES"] = "0,2"
bce_loss = lambda a, b: F.binary_cross_entropy_with_logits(a, b)
net = nn.DataParallel(Net().cuda())
opt = torch.optim.Adam(
filter(lambda p: p.requires_grad, net.parameters()), lr=1e-3,
betas=(0.9, 0.99), weight_decay=2e-4)
inp = torch.zeros((BS, 3, 64, 64)).cuda()
inp.fill_(1)
gt = torch.zeros((BS, 3, 64, 64)).cuda()
gt.fill_(100)
for _ in range(5):
conv1, conv2 = net(inp)
print()
print('conv1.output:', conv1[0,0,0,0])
print('conv1.weight:', net.module.conv1.weight[0,0,0,0])
if net.module.conv1.weight.grad is not None:
print('conv1.grad :', net.module.conv1.weight.grad[0,0,0,0])
print('conv2.weight:', net.module.conv2.weight[0,0,0,0])
if net.module.conv2.weight.grad is not None:
print('conv2.grad :', net.module.conv2.weight.grad[0,0,0,0])
loss = bce_loss(conv2, gt).cuda()
print('loss :', loss.data.item())
opt.zero_grad()
loss.backward()
opt.step()
Result:
conv1.output: tensor(12.0751, device='cuda:0', grad_fn=<SelectBackward>)
conv1.weight: tensor(1., device='cuda:0', grad_fn=<SelectBackward>)
conv2.weight: tensor(1., device='cuda:0', grad_fn=<SelectBackward>)
loss : -69638.2421875
conv1.output: tensor(12.0621, device='cuda:0', grad_fn=<SelectBackward>)
conv1.weight: tensor(0.9990, device='cuda:0', grad_fn=<SelectBackward>)
conv1.grad : tensor(0., device='cuda:0')
conv2.weight: tensor(1.0010, device='cuda:0', grad_fn=<SelectBackward>)
conv2.grad : tensor(-856.6614, device='cuda:0')
loss : -69637.3984375
conv1.output: tensor(12.0491, device='cuda:0', grad_fn=<SelectBackward>)
conv1.weight: tensor(0.9980, device='cuda:0', grad_fn=<SelectBackward>)
conv1.grad : tensor(0., device='cuda:0')
conv2.weight: tensor(1.0020, device='cuda:0', grad_fn=<SelectBackward>)
conv2.grad : tensor(-855.7734, device='cuda:0')
loss : -69636.3984375
conv1.output: tensor(12.0361, device='cuda:0', grad_fn=<SelectBackward>)
conv1.weight: tensor(0.9970, device='cuda:0', grad_fn=<SelectBackward>)
conv1.grad : tensor(0., device='cuda:0')
conv2.weight: tensor(1.0030, device='cuda:0', grad_fn=<SelectBackward>)
conv2.grad : tensor(-854.8976, device='cuda:0')
loss : -69635.2890625
conv1.output: tensor(12.0231, device='cuda:0', grad_fn=<SelectBackward>)
conv1.weight: tensor(0.9960, device='cuda:0', grad_fn=<SelectBackward>)
conv1.grad : tensor(0., device='cuda:0')
conv2.weight: tensor(1.0040, device='cuda:0', grad_fn=<SelectBackward>)
conv2.grad : tensor(-854.0023, device='cuda:0')
loss : -69633.96875
Contrast Experiment: Single GPU
And when
os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID" # see issue #152
os.environ["CUDA_VISIBLE_DEVICES"] = "0"
Result:
conv1.output: tensor(12.0137, device='cuda:0', grad_fn=<SelectBackward>)
conv1.weight: tensor(1., device='cuda:0', grad_fn=<SelectBackward>)
conv2.weight: tensor(1., device='cuda:0', grad_fn=<SelectBackward>)
loss : -69860.8515625
conv1.output: tensor(12.0137, device='cuda:0', grad_fn=<SelectBackward>)
conv1.weight: tensor(1., device='cuda:0', grad_fn=<SelectBackward>)
conv2.weight: tensor(1.0010, device='cuda:0', grad_fn=<SelectBackward>)
conv2.grad : tensor(-854.7103, device='cuda:0')
loss : -69930.796875
conv1.output: tensor(12.0137, device='cuda:0', grad_fn=<SelectBackward>)
conv1.weight: tensor(1., device='cuda:0', grad_fn=<SelectBackward>)
conv2.weight: tensor(1.0020, device='cuda:0', grad_fn=<SelectBackward>)
conv2.grad : tensor(-854.7103, device='cuda:0')
loss : -70000.7890625
conv1.output: tensor(12.0137, device='cuda:0', grad_fn=<SelectBackward>)
conv1.weight: tensor(1., device='cuda:0', grad_fn=<SelectBackward>)
conv2.weight: tensor(1.0030, device='cuda:0', grad_fn=<SelectBackward>)
conv2.grad : tensor(-854.7103, device='cuda:0')
loss : -70070.734375
conv1.output: tensor(12.0137, device='cuda:0', grad_fn=<SelectBackward>)
conv1.weight: tensor(1., device='cuda:0', grad_fn=<SelectBackward>)
conv2.weight: tensor(1.0040, device='cuda:0', grad_fn=<SelectBackward>)
conv2.grad : tensor(-854.7103, device='cuda:0')
loss : -70140.640625
Contrast Experiment: Weight Decay
os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
os.environ["CUDA_VISIBLE_DEVICES"] = "0,2"
opt = torch.optim.Adam(
filter(lambda p: p.requires_grad, net.parameters()), lr=1e-3,
betas=(0.9, 0.99), weight_decay=0)
Result:
conv1.output: tensor(11.8135, device='cuda:0', grad_fn=<SelectBackward>)
conv1.weight: tensor(1., device='cuda:0', grad_fn=<SelectBackward>)
conv2.weight: tensor(1., device='cuda:0', grad_fn=<SelectBackward>)
loss : -69543.546875
conv1.output: tensor(11.8135, device='cuda:0', grad_fn=<SelectBackward>)
conv1.weight: tensor(1., device='cuda:0', grad_fn=<SelectBackward>)
conv1.grad : tensor(0., device='cuda:0')
conv2.weight: tensor(1.0010, device='cuda:0', grad_fn=<SelectBackward>)
conv2.grad : tensor(-848.2969, device='cuda:0')
loss : -69613.203125
conv1.output: tensor(11.8135, device='cuda:0', grad_fn=<SelectBackward>)
conv1.weight: tensor(1., device='cuda:0', grad_fn=<SelectBackward>)
conv1.grad : tensor(0., device='cuda:0')
conv2.weight: tensor(1.0020, device='cuda:0', grad_fn=<SelectBackward>)
conv2.grad : tensor(-848.2969, device='cuda:0')
loss : -69682.859375
conv1.output: tensor(11.8135, device='cuda:0', grad_fn=<SelectBackward>)
conv1.weight: tensor(1., device='cuda:0', grad_fn=<SelectBackward>)
conv1.grad : tensor(0., device='cuda:0')
conv2.weight: tensor(1.0030, device='cuda:0', grad_fn=<SelectBackward>)
conv2.grad : tensor(-848.2969, device='cuda:0')
loss : -69752.5
conv1.output: tensor(11.8135, device='cuda:0', grad_fn=<SelectBackward>)
conv1.weight: tensor(1., device='cuda:0', grad_fn=<SelectBackward>)
conv1.grad : tensor(0., device='cuda:0')
conv2.weight: tensor(1.0040, device='cuda:0', grad_fn=<SelectBackward>)
conv2.grad : tensor(-848.2969, device='cuda:0')
loss : -69822.1484375