Sum of losses, inplace operation error

Hi I don’t why this happens, someone can help me? Thanks in advance.

class MyEnsemble(nn.Module):
    def __init__(self, n_clients):
        super(MyEnsemble, self).__init__()
        self.models = nn.ModuleList([MyModelA() for _ in range(0,n_clients)])
        
    def forward(self, x1, id):
        output = self.models[id](x1)
        return output
    def get_model(self, id):
      return self.models[id]
    def disable_model_grad(self):
      for p in self.models.parameters():
        p.requires_grad_(False)
    def enable_model_grad(self, id):
      for p in self.models[id].parameters():
        p.requires_grad_(True)
ensemble = MyEnsemble(N_CLIENTS)
dataloader_list = [DataLoader(TensorDataset(torch.randn(10, 10), torch.randn(10, 2))) for _ in range(0,N_CLIENTS)]
optimizer = optim.SGD(ensemble.parameters(), lr=0.01, momentum=0.9)

all_losses = None
for client in range(0,N_CLIENTS):
  ensemble.disable_model_grad()
  for feature, target in dataloader_list[client]:
    ensemble.enable_model_grad(client)
    predicted = ensemble.forward(feature,id=client)
    optimizer.zero_grad()
    output = loss(predicted, target)
    output.backward(retain_graph=True)
    optimizer.step()
  if all_losses is None:
      all_losses = output
  else:
      all_losses += output

Now when I compute the backward on all_losses as follows:

all_losses.backward()

This error appears, RuntimeError: one of the variables needed for gradient computation has been modified by an inplace operation: [torch.FloatTensor [10, 2]], which is output 0 of TBackward, is at version 363;

The optimizer already executed its step() method using the calculated gradients from the output.backward() call and thus manipulated the parameters.
all_losses stores these output tensors (losses) and tries to calculate the gradients again, which won’t work since the parameters were already updated.