Is there a way to enable nn.Sequential
to handle multiple inputs with dynamic number of blocks?
My code follows #19808, but it doesn’t work from the 2nd blocks.
Traceback (most recent call last):
File "/home/Workspace/one-implementation/models/cifar/resnet_stochastic_depth.py", line 380, in <module>
out = model(data)
File "/home/Workspace/one-implementation/venv/lib/python3.6/site-packages/torch/nn/modules/module.py", line 550, in __call__
result = self.forward(*input, **kwargs)
File "/home/Workspace/one-implementation/models/cifar/resnet_stochastic_depth.py", line 350, in forward
x = self.layer1(x, self.actives, self.probabilities)
File "/home/Workspace/one-implementation/venv/lib/python3.6/site-packages/torch/nn/modules/module.py", line 550, in __call__
result = self.forward(*input, **kwargs)
File "/home/Workspace/one-implementation/models/cifar/resnet_stochastic_depth.py", line 282, in forward
input = module(*input)
File "/home/Workspace/one-implementation/venv/lib/python3.6/site-packages/torch/nn/modules/module.py", line 550, in __call__
result = self.forward(*input, **kwargs)
TypeError: forward() takes 4 positional arguments but 513 were given
cnt = 1 module = BasicBlock_stochastic_depth(
(conv1): Conv2d(16, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
(bn1): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
(relu): ReLU(inplace=True)
(conv2): Conv2d(16, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
(bn2): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
)
Process finished with exit code 1
class mySequential(nn.Sequential):
def forward(self, *input):
cnt = 0
for module in self._modules.values():
input = module(*input)
cnt = cnt + 1
print('cnt = {} module = {}'.format(cnt, module))
return input
class BasicBlock_stochastic_depth(nn.Module):
expansion = 1
def __init__(self, inplanes, planes, stride=1, downsample=None):
super(BasicBlock_stochastic_depth, self).__init__()
self.conv1 = conv3x3(inplanes, planes, stride)
self.bn1 = nn.BatchNorm2d(planes)
self.relu = nn.ReLU(inplace=True)
self.conv2 = conv3x3(planes, planes)
self.bn2 = nn.BatchNorm2d(planes)
self.downsample = downsample
self.stride = stride
def forward(self, x, active, prob):
if self.training:
if active == 1:
print('active')
residual = x
out = self.conv1(x)
out = self.bn1(out)
out = self.relu(out)
out = self.conv2(out)
out = self.bn2(out)
if self.downsample is not None:
residual = self.downsample(x)
out += residual
out = self.relu(out)
else:
print('inactive')
out = x
if self.downsample is not None:
out = self.downsample(out)
out = self.relu(out)
else:
residual = x
out = self.conv1(x)
out = self.bn1(out)
out = self.relu(out)
out = self.conv2(out)
out = self.bn2(out)
if self.downsample is not None:
residual = self.downsample(x)
out = prob * out + residual
out = self.relu(out)
return (out)
class ResNetSD(nn.Module):
def __init__(self, depth, num_classes=1000, block_name='BasicBlock'):
super(ResNetSD, self).__init__()
# Model type specifies number of layers for CIFAR-10 model
if block_name.lower() == 'basicblock':
assert(depth - 2) % 6 == 0, 'When use basicblock, depth should be 6n+2, e.g. 20, 32, 44, 56, 110, 1202'
n = (depth - 2) // 6
block = BasicBlock_stochastic_depth
elif block_name.lower() == 'bottleneck':
assert (depth == 2) % 9 == 0, 'When use bottleneck, depth should be 9n + 2, e.g. 20, 29, 47, 56, 110, 1199'
n = (depth - 2) // 9
block = Bottleneck_stochastic_depth
else:
raise ValueError('block_name should be Basicblock or Bottleneck')
self.probabilities = torch.tensor(0.5)
self.actives = torch.bernoulli(self.probabilities)
self.inplanes = 16
self.conv1 = nn.Conv2d(3, 16, kernel_size=3, padding=1, bias=False)
self.bn1 = nn.BatchNorm2d(16)
self.relu = nn.ReLU(inplace=True)
self.layer1 = self._make_layer(block, 16, n)
self.layer2 = self._make_layer(block, 32, n, stride=2)
self.layer3 = self._make_layer(block, 64, n, stride=2)
self.avgpool = nn.AvgPool2d(8)
self.fc = nn.Linear(64 * block.expansion, num_classes)
for m in self.modules():
if isinstance(m, nn.Conv2d):
n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels
m.weight.data.normal_(0, math.sqrt(2./n))
elif isinstance(m, nn.BatchNorm2d):
m.weight.data.fill_(1)
m.bias.data.zero_()
def _make_layer(self, block, planes, blocks, stride=1):
downsample = None
if stride != 1 or self.inplanes != planes * block.expansion:
downsample = nn.Sequential(
nn.Conv2d(self.inplanes, planes * block.expansion,
kernel_size=1, stride=stride, bias=False),
nn.BatchNorm2d(planes * block.expansion)
)
layers = []
layers.append(block(self.inplanes, planes, stride, downsample))
self.inplanes = planes * block.expansion
for i in range(1, blocks):
layers.append(block(self.inplanes, planes))
group_layers = mySequential(*layers)
# group_layers = mySequential(nn.Sequential(*layers))
# return nn.Sequential(*layers)
return group_layers
def forward(self, x):
x = self.conv1(x)
x = self.bn1(x)
x = self.relu(x)
x = self.layer1(x, self.actives, self.probabilities)
x = self.layer2(x, self.actives, self.probabilities)
x = self.layer3(x, self.actives, self.probabilities)
x = self.avgpool(x)
x = x.view(x.size(0), -1)
x = self.fc(x)
return x