retain_graph=True

autograd
1

Hi
I want to solve 2D Laplace equation with PINNs. I write this code but when I run the code, I faced this error “Trying to backward through the graph a second time (or directly access saved tensors after they have already been freed). Saved intermediate values of the graph are freed when you call .backward() or autograd.grad(). Specify retain_graph=True if you need to backward through the graph a second time or if you need to access saved tensors after calling backward.”
I’ve read the other posts, but unless I’m mistaken none of them solve my problem.
Here is my code:

import numpy as np
import matplotlib.pyplot as plt

import torch
import torch.nn as nn

num_points = 3 # Number of uniformly selected points for BC
x_min, x_max, y_min, y_max = 0, 10000, 0, 4
y_range, x_range =np.linspace(0,4,5)[1:-1] , np.linspace(0, 10000, 12)[1:-1]
x_zeros = np.zeros(10)
y_zeros = np.zeros(3)
x_ones = np.ones(10)
y_ones = np.ones(3)

bc_left = np.vstack([y_zeros, y_range, 5*y_ones]).T # Boundary left: h(x=0, y) = 5
bc_right = np.vstack([10000 * y_ones, y_range, y_ones]).T # Boundary right: u(x=10, y) = 1

initial_boundary_conditions = np.vstack([bc_left, bc_right])
x_initial_bc = initial_boundary_conditions[:, 0]
y_initial_bc = initial_boundary_conditions[:, 1]
u_initial_bc = initial_boundary_conditions[:, 2]

x_initial_bc = torch.autograd.Variable(torch.from_numpy(x_initial_bc).float(), requires_grad=True) #torch.Size([50000])
y_initial_bc = torch.autograd.Variable(torch.from_numpy(y_initial_bc).float(), requires_grad=True)
u_initial_bc = torch.autograd.Variable(torch.from_numpy(u_initial_bc).float(), requires_grad=True)
x_range = torch.autograd.Variable(torch.from_numpy(x_range).float(), requires_grad=True)
x_zeros = torch.autograd.Variable(torch.from_numpy(x_zeros).float(), requires_grad=True)
x_ones = torch.autograd.Variable(torch.from_numpy(x_ones).float(), requires_grad=True)

collocation_points = 30 # Number of uniformly selected collocation points to be evaluated for the PDE-based loss
class Net(nn.Module):
def init(self):
super(Net, self).init()
self.neurons_per_layer = 20
self.fc1 = nn.Linear(2, self.neurons_per_layer)
self.fc2 = nn.Linear(self.neurons_per_layer, self.neurons_per_layer)
self.fc3 = nn.Linear(self.neurons_per_layer, self.neurons_per_layer)
self.fc4 = nn.Linear(self.neurons_per_layer, self.neurons_per_layer)
self.fc5 = nn.Linear(self.neurons_per_layer, self.neurons_per_layer)
self.fc6 = nn.Linear(self.neurons_per_layer, self.neurons_per_layer)
self.fc7 = nn.Linear(self.neurons_per_layer, self.neurons_per_layer)
self.fc8 = nn.Linear(self.neurons_per_layer, 1)
self.relu = nn.ReLU()

def forward(self, x, y):
    inputs = torch.cat([x.reshape(-1, 1), y.reshape(-1, 1)], axis=1)
    output = self.relu(self.fc1(inputs))
    output = self.relu(self.fc2(output))
    output = self.relu(self.fc3(output))
    output = self.relu(self.fc4(output))
    output = self.relu(self.fc5(output))
    output = self.relu(self.fc6(output))
    output = self.relu(self.fc7(output))
    output = self.fc8(output)
    return output

net = Net()
epochs = 1000
optimizer = torch.optim.Adam(net.parameters(), lr=1e-3)
criterion = torch.nn.MSELoss()
#PDE loss
def f(x, y, net):
u = net(x, y)
u_x = torch.autograd.grad(u, x, create_graph=True, retain_graph=True, grad_outputs=torch.ones_like(u))[0]
u_xx = torch.autograd.grad(u_x, x, create_graph=True, retain_graph=True, grad_outputs=torch.ones_like(u_x))[0]
u_y = torch.autograd.grad(u, y, create_graph=True, retain_graph=True, grad_outputs=torch.ones_like(u))[0]
u_yy = torch.autograd.grad(u_y, y, create_graph=True, retain_graph=True, grad_outputs=torch.ones_like(u_y))[0]
loss_f = u_xx + u_yy
return loss_f

def neumann_bc_loss(net, x, y, target_gradient_value):
u = net(x, y)
u_y = torch.autograd.grad(u, y, create_graph=True, grad_outputs=torch.ones_like(u))[0]

neumann_loss = torch.nn.functional.mse_loss(u_y, target_gradient_value * torch.ones_like(u_y))
return neumann_loss

neumann_loss_top = neumann_bc_loss(net, x_range, 4*x_ones, 0)
neumann_loss_bottom = neumann_bc_loss(net, x_range, x_zeros, 0)

losses =
for epoch in range(epochs):
optimizer.zero_grad()
predictions_initial_bc = net(x_initial_bc, y_initial_bc)
mse_u = criterion(predictions_initial_bc.reshape(-1,), u_initial_bc) # This is the loss from boundary and initial conditions

x_collocation = torch.FloatTensor(collocation_points, ).uniform_(x_min, x_max)
y_collocation = torch.FloatTensor(collocation_points, ).uniform_(y_min, y_max)
x_collocation = torch.autograd.Variable(x_collocation, requires_grad=True)
y_collocation = torch.autograd.Variable(y_collocation, requires_grad=True)

f_out = f(x_collocation, y_collocation, net)
mse_f = criterion(torch.zeros_like(f_out), f_out) # This is the PDE-based loss evaluated at the randomly sampled collocation points

loss = mse_u + mse_f + neumann_loss_bottom + neumann_loss_top
losses.append(loss.item())
loss.backward()
optimizer.step()
if epoch % 100 == 0:
    
    
    print(f'Epoch {epoch}/{epochs}: Loss = {loss.item()}')

I tried loss.backward(retain_graph=True) but I received this error “one of the variables needed for gradient computation has been modified by an inplace operation: [torch.FloatTensor [20, 2]], which is output 0 of AsStridedBackward0, is at version 2; expected version 1 instead. Hint: enable anomaly detection to find the operation that failed to compute its gradient, with torch.autograd.set_detect_anomaly(True).” can someone please help me to solve this problem?

2

Have you tried running your code within a torch.autograd.set_detect_anomaly(True) context manager to isolate the in-place operation?

3

no, I didn’t try it. where should I implement torch.autograd.set_detect_anomaly(True)?

4

it should be noticed that when i added neumann_loss_bottom and neumann_loss_top to the loss this error was ocurred.

5

Hi @parisa71,

You can add this line before your training loss,

with torch.autograd.set_detect_anonmaly(True):
  for epoch in range(epochs):
    #code goes here
6

thank you. I tried it but received this error again “one of the variables needed for gradient computation has been modified by an inplace operation: [torch.FloatTensor [20, 2]], which is output 0 of AsStridedBackward0, is at version 2; expected version 1 instead. Hint: the backtrace further above shows the operation that failed to compute its gradient. The variable in question was changed in there or anywhere later.”

7

Should you have retain_graph=True in the neumann_bc_loss function?

8

That’s right. Thank you very much.

1 Like