So I have the following code:
import torch
import torch.nn as nn
import math
torch.autograd.set_detect_anomaly(True)
def sigmoid(x):
return 1 / (1 + torch.exp(-x))
dtype = torch.float
device = torch.device("cpu")
# device = torch.device("cuda:0") # Uncomment this to run on the cpu
# N = batch size
# D_in = input size
# H = hidden dimension
# D_out = output dimension
Time, N, D_in, H, D_out = 7, 3, 10, 5, 2
# Create random Tensors to hold input and outputs.
# Setting requires_grad=False indicates that we do not need to compute gradients
# with respect to thes Tensors during the backward pass.
X = torch.randn(Time, N, D_in, device=device, dtype=dtype, requires_grad=False)
Y = torch.randn(Time, D_out, D_in, device=device, dtype=dtype, requires_grad=False)
Cell = torch.zeros(Time, H, D_in, device=device, dtype=dtype, requires_grad=False)
Hidden = torch.zeros(Time, H, D_in, device=device, dtype=dtype, requires_grad=False)
yHat = torch.zeros(Time, D_out, D_in, device=device, dtype=dtype, requires_grad=False)
# Create random Tensors for weights.
# Setting requires_grad=True indicates that we want to compute gradients with
# respect to these Tensors during the backward pass.
WeightsForget = torch.randn(H, H+N, device=device, dtype=dtype, requires_grad=True)
WeightsUpdate = torch.randn(H, H+N, device=device, dtype=dtype, requires_grad=True)
WeightsCell = torch.randn(H, H+N, device=device, dtype=dtype, requires_grad=True)
WeightsOutput = torch.randn(H, H+N, device=device, dtype=dtype, requires_grad=True)
# Create random Tensors for Bias.
BiasForget = torch.randn(H, 1, device=device, dtype=dtype, requires_grad=True)
BiasUpdate = torch.randn(H, 1, device=device, dtype=dtype, requires_grad=True)
BiasCell = torch.randn(H, 1, device=device, dtype=dtype, requires_grad=True)
BiasOutput = torch.randn(H, 1, device=device, dtype=dtype, requires_grad=True)
# Weights and Bias relating the hidden state to the output.
Weights_HiddenToOutput = torch.randn(D_out, H, device=device, dtype=dtype, requires_grad=True)
Bias_HiddenToOutput = torch.randn(D_out, 1, device=device, dtype=dtype, requires_grad=True)
# Initialize aNext and cNext
hLast = Hidden[0,:,:]
cLast = torch.zeros(hLast.shape)
learning_rate = 1e-6
for t in range(Time):
# Concat hLast and X(t)
concat = torch.zeros(H+N, D_in, device=device, dtype=dtype)
concat[: H, :] = hLast
concat[H :, :] = X[t,:,:]
forget = sigmoid(torch.mm(WeightsForget, concat) + BiasForget)
update = sigmoid(torch.mm(WeightsUpdate, concat) + BiasUpdate)
candCell = torch.tanh(torch.mm(WeightsCell, concat) + BiasCell)
cNext = forget * cLast + update * candCell
output = sigmoid(torch.mm(WeightsOutput, concat) + BiasOutput)
hNext = output * torch.tanh(cNext)
# Compute prediction of the LSTM Cell
prediction = torch.mm(Weights_HiddenToOutput, hNext) + Bias_HiddenToOutput
# Save the value of the next hidden state
Hidden[t,:,:] = hNext
# Save the value of the Prediction in Y
yHat[t,:,:] = prediction
# Save the value of the next Cell State
Cell[t,:,:] = cNext
# Compute and print loss using operations on Tensors.
# Now loss is a Tensor of shape(1,)
loss = (-Y[t,:,:] * torch.log(prediction) - (1-Y[t,:,:]) * torch.log(1-prediction)).sum()
print('t, loss.item = ', t, loss.item())
loss.backward(retain_graph=True)
with torch.no_grad():
# Update hLast and cLast
hLast = hNext
cLast = cNext
# Update Weights
WeightsCell -= learning_rate * WeightsCell.grad
WeightsOutput -= learning_rate * WeightsOutput.grad
WeightsUpdate -= learning_rate * WeightsUpdate.grad
Weights_HiddenToOutput -= learning_rate * Weights_HiddenToOutput.grad
# Update Bias
BiasCell -= learning_rate * BiasCell.grad
BiasForget -= learning_rate * BiasForget.grad
BiasOutput -= learning_rate * BiasOutput.grad
BiasUpdate -= learning_rate * BiasUpdate.grad
Bias_HiddenToOutput -= learning_rate * Bias_HiddenToOutput.grad
# Zero out the gradients
WeightsCell.grad.zero_()
WeightsForget.grad.zero_()
WeightsOutput.grad.zero_()
WeightsUpdate.grad.zero_()
Weights_HiddenToOutput.grad.zero_()
BiasCell.grad.zero_()
BiasForget.grad.zero_()
BiasOutput.grad.zero_()
BiasUpdate.grad.zero_()
print(Bias_HiddenToOutput.dtype)
Bias_HiddenToOutput.grad.zero_()
When I run it I get:
RuntimeError: one of the variables needed for gradient computation has been modified by an inplace operation: [torch.FloatTensor [5, 8]] is at version 1; expected version 0 instead. Hint: the backtrace further above shows the operation that failed to compute its gradient. The variable in question was changed in there or anywhere later. Good luck!
Commenting out the updates of the weights and biases will let the code run but doesn’t update the weights and biases. I’m new to pytorch and I understand that this error has something to do with the inplace operation incrementing the version of the Tensor during the loss.backward() call. But I thought the with torch.no_grad(): was supposed to let me update the Tensors without incrementing their version.
So I guess I have 2 questions… First, how do I get my code running?? Second, what am I missing about using with torch.no_grad()?? Because it doesn’t work the way I thought.
Thanks!
JP
