How to use network as an input parameter instead of a tensor for an optimizer?

1

According to this post using TensorFlow (code here), you can use a simple Compositional Pattern Producing Network (CPPN) as an input parameter for a classification network. By using the CPPN as the input parameter, you can create really smooth looking visualizations based on neuron activation.

I tried using the following in PyTorch, and while the loss seems to go down, nothing else changes:

       optimizer = optim.Adam([
       {'params': cppn_net.parameters()}], lr = learning_rate)  # Replaced image tensor

This is what cppn_net looks like:

import torch
import torch.nn as nn

		
def cppn_normal(l):
    if type(l) == nn.Linear:        
        nn.init.normal_(l.weight)

		
# Regular CPPN         
class CPPN(nn.Module):

    def __init__(self, size, input_range=1, num_channels=16, num_layers=6, activ_func=nn.Tanh()):
        super(CPPN, self).__init__()
        if type(size) is not tuple:
            size = (size, size)
        self.input_size = size
        self.input_range = input_range
        self.net = nn.Sequential()
        self.net.add_module(str(len(self.net)), nn.Linear(2, num_channels, bias=True))
        self.net.add_module(str(len(self.net)), activ_func)
        for l in range(num_layers - 1):
             self.net.add_module(str(len(self.net)), nn.Linear(num_channels, num_channels, bias=False))
             self.net.add_module(str(len(self.net)), activ_func)
        self.net.add_module(str(len(self.net)), nn.Linear(num_channels, 3, bias=False))
        self.net.add_module(str(len(self.net)), nn.Sigmoid())
        
    def create_input(self): 
        w = torch.arange(0, self.input_size[1])
        h = torch.arange(0, self.input_size[0]) 
        w_exp = w.unsqueeze(1).expand((self.input_size[1], self.input_size[0])).true_divide(self.input_size[0]) - 0.5
        h_exp = h.unsqueeze(0).expand((self.input_size[1], self.input_size[0])).true_divide(self.input_size[1]) - 0.5
        return torch.stack((w_exp, h_exp), -1).reshape(self.input_size[1] * self.input_size[0], 2)

    def forward(self):
        input = self.create_input()
        input.requires_grad= True
        return (self.net(input).reshape(self.input_size[1], self.input_size[0], 3).permute(2,1,0) * self.input_range).unsqueeze(0)
		
		
cppn_net = CPPN(size=(512, 512)).apply(cppn_normal)

I’m not sure if I need to setup the CPPN network differently, change how I’m integrating it with the main network, or do something else entirely?

2

It is possible to subclass any given optimizer.

3

So, I have to make changes to the optimizer class for it to work?

4

I’m unsure if there’s a built-in way to do this, but I’m sure you can make it work by subclassing.

5

So, looking at the example image, I think that I could probably use a CPPN as a function instead of a nn.Sequential.

example
example854×275 46.5 KB

Though I am not sure if creating Conv2d weights in a loop like this is a good idea, or if I need to do it a different way?

import torch
import torch.nn as nn
import torch.nn.functional as F

def create_input(input_size, mode=0):
    if mode == 0:
        if type(input_size) is not tuple and type(input_size) is not list:
            input_size = (input_size, input_size) 
        w = torch.arange(0, input_size[1])
        h = torch.arange(0, input_size[0]) 
        w_exp = w.unsqueeze(1).expand((input_size[1], input_size[0])).true_divide(input_size[0]) - 0.5
        h_exp = h.unsqueeze(0).expand((input_size[1], input_size[0])).true_divide(input_size[1]) - 0.5
        return torch.stack((w_exp, h_exp), -1).permute(2,1,0).unsqueeze(0)
    elif mode == 1: # TensorFlow/Lucid Creation method
        if type(input_size) is tuple or type(input_size) is list:
            input_size = input_size[0]
        r = 3.0**0.5
        coord_range = torch.linspace(-r, r, input_size)
        y, x = torch.meshgrid(coord_range, coord_range)
        tensor = torch.stack((x, y), -1).unsqueeze(0).permute(0,3,1,2)
        return tensor

def cppn_image(size, num_channels=16, num_layers=9):
    tensor = create_input(size)
    weight_val = nn.Parameter(torch.randn(num_channels,2,1,1))
    tensor = F.conv2d(tensor, weight_val).tanh()
    for i in range(num_layers):
        weight_val = nn.Parameter(torch.randn(num_channels,num_channels,1,1))
        tensor = F.conv2d(tensor, weight_val).tanh()
    weight_val = nn.Parameter(torch.randn(3,num_channels,1,1))
    tensor = F.conv2d(tensor, weight_val).sigmoid()
    return tensor


output = cppn_image((512,460))