Please help me to implement Long range Connections in Linear layers (pytorch)
My Current Implementation without long range connections. I want to implement a random long range connections from one layer to another layer. How can i accomplish that.
def get_weights_tensor(nodes):
weights = torch.ones(nodes, nodes)
for i in range(nodes):
for j in range(nodes):
if i != j and i != j+1 and j != i+1:
weights[i, j] = 0
weight_tensor = weights.type(torch.float32)
return weight_tensor
class NN(nn.Module):
def __init__(self, input_size, num_classes, hidden_nodes=50):
super(NN, self).__init__()
self.hidden_nodes = hidden_nodes
self.input_layer = nn.Linear(input_size, hidden_nodes)
self.hl_1 = nn.Linear(hidden_nodes, hidden_nodes)
self.hl_1.weight = torch.nn.Parameter(get_weights_tensor(hidden_nodes))
self.hl_2 = nn.Linear(hidden_nodes, hidden_nodes)
self.hl_2.weight = torch.nn.Parameter(get_weights_tensor(hidden_nodes))
self.hl_3 = nn.Linear(hidden_nodes, hidden_nodes)
self.hl_3.weight = torch.nn.Parameter(get_weights_tensor(hidden_nodes))
self.hl_4 = nn.Linear(hidden_nodes, hidden_nodes)
self.hl_4.weight = torch.nn.Parameter(get_weights_tensor(hidden_nodes))
self.hl_5 = nn.Linear(hidden_nodes, hidden_nodes)
self.hl_5.weight = torch.nn.Parameter(get_weights_tensor(hidden_nodes))
self.output_layer = nn.Linear(hidden_nodes, num_classes)
def forward(self, x):
x = F.relu(self.input_layer(x))
x = F.relu(self.hl_1(x))
x = F.relu(self.hl_2(x))
x = F.relu(self.hl_3(x))
x = F.relu(self.hl_4(x))
x = F.relu(self.hl_5(x))
x = self.output_layer(x)
return x