Hi,
I define a layer in init function but I don’t use it in forward function. why it still cause the number of trainable parameter change?
and what it mean? (dose it mean this layer is not in forward path but in backpropagation path exist?)
and what if I want use a layer conditionally (i.e. each 20 epoch, or if error is less than … )
i put the code here( its part of pytorch tutorial https://pytorch.org/tutorials/beginner/blitz/neural_networks_tutorial.html)
import torch
import torch.nn as nn
import torch.nn.functional as F
class Net(nn.Module):
def __init__(self):
super(Net, self).__init__()
# 1 input image channel, 6 output channels, 5x5 square convolution
# kernel
self.conv1 = nn.Conv2d(1, 6, 5)
self.conv2 = nn.Conv2d(6, 16, 5)
# an affine operation: y = Wx + b
self.fc1 = nn.Linear(16 * 5 * 5, 120) # 5*5 from image dimension
self.fc2 = nn.Linear(120, 84)
self.test_layer = nn.Linear(84, 84) #add or remove this layer which not exict in forward pass
self.fc3 = nn.Linear(84, 10)
def forward(self, x):
# Max pooling over a (2, 2) window
x = F.max_pool2d(F.relu(self.conv1(x)), (2, 2))
# If the size is a square, you can specify with a single number
x = F.max_pool2d(F.relu(self.conv2(x)), 2)
x = torch.flatten(x, 1) # flatten all dimensions except the batch dimension
x = F.relu(self.fc1(x))
x = F.relu(self.fc2(x))
x = self.fc3(x)
return x
model = Net()
print(model)
print ('model trainable parameters',sum(p.numel() for p in model.parameters() if p.requires_grad))
print ('model non-trainable parameters',sum(p.numel() for p in model.parameters() if p.requires_grad==False))
you can see with self.test_layer = nn.Linear(84, 84) output is
Net(
(conv1): Conv2d(1, 6, kernel_size=(5, 5), stride=(1, 1))
(conv2): Conv2d(6, 16, kernel_size=(5, 5), stride=(1, 1))
(fc1): Linear(in_features=400, out_features=120, bias=True)
(fc2): Linear(in_features=120, out_features=84, bias=True)
(test_layer): Linear(in_features=84, out_features=84, bias=True)
(fc3): Linear(in_features=84, out_features=10, bias=True)
)
model trainable parameters 68846
model non-trainable parameters 0
while without it
Net(
(conv1): Conv2d(1, 6, kernel_size=(5, 5), stride=(1, 1))
(conv2): Conv2d(6, 16, kernel_size=(5, 5), stride=(1, 1))
(fc1): Linear(in_features=400, out_features=120, bias=True)
(fc2): Linear(in_features=120, out_features=84, bias=True)
(fc3): Linear(in_features=84, out_features=10, bias=True)
)
model trainable parameters 61706
model non-trainable parameters 0