Update weights and Bias separately

#1

Before modification Net

G_network = torch.nn.Sequential(
torch.nn.Linear(lh1, lh2),
torch.nn.CELU(0.1),
torch.nn.Linear(lh2, lh3),
torch.nn.CELU(0.1),
torch.nn.Linear(lh3, lh4),
torch.nn.CELU(0.1),
torch.nn.Linear(lh4, lh5)
)

After modification for debugging.

class G_network(nn.Module):
    def __init__(self, lh1, lh2, lh3, lh4, lh5):
        super(G_network, self).__init__()
        self.fc1 = nn.Linear(lh1, lh2)
        self.fc2 = nn.Linear(lh2, lh3)
        self.fc3 = nn.Linear(lh3, lh4)
        self.fc4 = nn.Linear(lh4, lh5)

    def forward(self, data):
        data = F.celu(self.fc1(data), 0.1)
        data = F.celu(self.fc2(data), 0.1)
        data = F.celu(self.fc3(data), 0.1)
        data = self.fc4(data)
        return data

G_network = G_network(640, 128, 128, 64, 1)

After modification I am not able to update weights and bias separately.

AdamW = torchani.optim.AdamW([
    # O networks
    {'params': [G_network[0].weight]},
    {'params': [G_network[2].weight], 'weight_decay': 0.000001},
    {'params': [G_network[4].weight], 'weight_decay': 0.0000001},
    {'params': [G_network[6].weight]},
], lr=adam_lr)

SGD = torch.optim.SGD([
    # O networks
    {'params': [G_network[0].bias]},
    {'params': [G_network[2].bias]},
    {'params': [G_network[4].bias]},
    {'params': [G_network[6].bias]},
], lr=sgd_lr)
#2

Your model works fine using this code:

class G_networkDef(nn.Module):
    def __init__(self, lh1, lh2, lh3, lh4, lh5):
        super(G_networkDef, self).__init__()
        self.fc1 = nn.Linear(lh1, lh2)
        self.fc2 = nn.Linear(lh2, lh3)
        self.fc3 = nn.Linear(lh3, lh4)
        self.fc4 = nn.Linear(lh4, lh5)

    def forward(self, data):
        data = F.celu(self.fc1(data), 0.1)
        data = F.celu(self.fc2(data), 0.1)
        data = F.celu(self.fc3(data), 0.1)
        data = self.fc4(data)
        return data

G_network = G_networkDef(640, 128, 128, 64, 1)

AdamW = torch.optim.AdamW([
    # O networks
    {'params': [G_network.fc1.weight]},
    {'params': [G_network.fc2.weight], 'weight_decay': 0.000001},
    {'params': [G_network.fc3.weight], 'weight_decay': 0.0000001},
    {'params': [G_network.fc4.weight]},
], lr=1.)

SGD = torch.optim.SGD([
    # O networks
    {'params': [G_network.fc1.bias]},
    {'params': [G_network.fc2.bias]},
    {'params': [G_network.fc3.bias]},
    {'params': [G_network.fc4.bias]},
], lr=1.)

G_network(torch.randn(1, 640)).mean().backward()

print(G_network.fc1.weight.abs().sum(), G_network.fc1.bias.abs().sum())
> tensor(1622.0929, grad_fn=<SumBackward0>) tensor(2.2686, grad_fn=<SumBackward0>)

AdamW.step()
print(G_network.fc1.weight.abs().sum(), G_network.fc1.bias.abs().sum())
> tensor(80437.7969, grad_fn=<SumBackward0>) tensor(2.2686, grad_fn=<SumBackward0>)

SGD.step()
print(G_network.fc1.weight.abs().sum(), G_network.fc1.bias.abs().sum())
> tensor(80437.7969, grad_fn=<SumBackward0>) tensor(2.4376, grad_fn=<SumBackward0>)

Note that I changed the model class name, as you were overriding it with the model instance.