I am trying to take the max of the previous layer and use it to normalize the activation of next layer in a model, as you see in a sample of code below :
here is the forward in the model where I record the max of each layer in a list ( thresh_list )
def forward(self, input, epoch):
x = self.conv1(input)
x = self.relu(x,1)
self.thresh_list[0] = max(self.thresh_list[0], x.max()) # here to get the max activation
x = self.conv_dropout(x)
x = self.conv2(x)
x = self.relu(x, self.thresh_list[0])
self.thresh_list[1] = max(self.thresh_list[1], x.max())
x = self.pool1(x)
x = self.conv_dropout(x)
x = self.conv3(x)
x = self.relu(x, self.thresh_list[1] )
self.thresh_list[2] = max(self.thresh_list[2], x.max())
The Relue function I call is a custom function as below :
self.relu = th_norm_ReLU(True)
and the norm_ReLU class is as below :
import torch
import torch.nn as nn
import torch.nn.functional as F
from argument_settings import *
device = "cuda" if torch.cuda.is_available() else "cpu"
class thReLU(torch.autograd.Function):
@staticmethod
def forward(ctx, input):
ctx.save_for_backward(input)
zero_tensor = torch.zeros(input.size()).to(device)
output = torch.maximum(input , zero_tensor)
return output
@staticmethod
def backward(ctx, grad_output):
input, = ctx.saved_tensors
grad_input = grad_output.clone()
diff = ( input >= 0 )
return grad_input * diff.float()
act_fun = thReLU.apply
class th_norm_ReLU(nn.Module):
def __init__(self, modify):
super(th_norm_ReLU, self).__init__()
self.norm = act_fun
def forward(self, input, prev_layer_max):
output = input * (prev_layer_max / input.max())
norm_output = self.norm (output)
return norm_output
if __name__ == '__main__':
xrelu = th_norm_ReLU(True)
z = torch.rand(3, 3) * 4
print(z)
y = xrelu(z, 1)
print(y)
Anyone please can help me to solve this problem, actually when I pass fixed value instead of the max of previous layer, it is working well with no error. But when I pass the max of previous layer I encounter this error. please, help me to solve this problem.