Today I read a lot about in place operation and optimizer, I try to debug my code but I still could not find any in place operation, the full network is available here:
class FeatureExtractor(nn.Module):
def __init__(self, main_input_channels, complementary_input_channels):
super(FeatureExtractor, self).__init__()
self.main_input_channels = main_input_channels
self.complementary_input_channels = complementary_input_channels
self.out_channels = 100
self.conv1 = nn.Conv2d(in_channels=main_input_channels, out_channels=8, kernel_size=3, padding_mode='circular')
self.conv2 = nn.Conv2d(in_channels=8, out_channels=16, kernel_size=3, padding_mode='circular')
self.conv3 = nn.Conv2d(in_channels=16, out_channels=32, kernel_size=3, padding_mode='circular')
self.pool = nn.MaxPool2d(2, 2)
self.fc = nn.Linear(32 + complementary_input_channels, self.out_channels)
def forward(self, x1, x2):
x1 = self.pool(F.relu(self.conv1(x1)))
x1 = self.pool(F.relu(self.conv2(x1)))
x1 = F.relu(self.conv3(x1))
x1 = x1.view(-1, 32)
x2 = x2.unsqueeze(0).expand(x1.shape[0], -1)
x = torch.cat((x1, x2), 1)
x = F.relu(self.fc(x))
return x
class ActorCriticNetwork(nn.Module):
def __init__(self, main_input_channels, complementary_input_channels):
super(ActorCriticNetwork, self).__init__()
self.feature_extractor_actor = FeatureExtractor(main_input_channels, complementary_input_channels)
self.fc_actor = nn.Linear(self.feature_extractor_actor.out_channels, 50)
self.fc_actor_ship = nn.Linear(50, 6)
self.fc_actor_shipyard = nn.Linear(50, 2)
self.feature_extractor_critic = FeatureExtractor(main_input_channels, complementary_input_channels)
self.fc_critic_1 = nn.Linear(self.feature_extractor_critic.out_channels, 50)
self.fc_critic_2 = nn.Linear(50, 1)
self.softmax = nn.Softmax(dim=1)
def forward_actor_shipyard(self, agent_state: AgentState):
main_input = torch.stack([self.translated(agent_state.main_input, shipyard.x, shipyard.y) for shipyard in agent_state.player.shipyards])
x = self.feature_extractor_actor(main_input, agent_state.complementary_input)
x = F.relu(self.fc_actor(x))
x = self.fc_actor_shipyard(x)
return self.softmax(x)
def forward_actor_ship(self, agent_state: AgentState):
main_input = torch.stack([self.translated(agent_state.main_input, ship.x, ship.y) for ship in agent_state.player.ships])
x = self.feature_extractor_actor(main_input, agent_state.complementary_input)
x = F.relu(self.fc_actor(x))
x = self.fc_actor_ship(x)
return self.softmax(x)
def forward_critic(self, agent_state: AgentState):
x = self.feature_extractor_critic(agent_state.main_input).unsqueeze(0), agent_state.complementary_input.clone())
x = F.relu(self.fc_critic_1(x))
x = self.fc_critic_2(x)
return x
@staticmethod
@torch.no_grad()
def translated(matrix, x, y):
return torch.roll(torch.roll(matrix, x, dims=-2), y, dims=-1)
Actions predicted by the network are stored inside a container:
class Agent:
def __init__(self, actor_critic_network: ActorCriticNetwork):
self.network = actor_critic_network
def __call__(self, agent_state: AgentState, counter=None) -> Action:
prediction_shipyards, prediction_ships = [], []
if agent_state.player.shipyards:
prediction_shipyards = self.network.forward_actor_shipyard(agent_state)
if agent_state.player.ships:
prediction_ships = self.network.forward_actor_ship(agent_state)
action = Action(prediction_shipyards, prediction_ships, agent_state, counter=counter)
return action
class Action:
def __init__(self, prediction_shipyards, prediction_ships, agent_state, counter=None):
self.prediction_shipyards = prediction_shipyards
self.prediction_ships = prediction_ships
...
and this method is used for learning:
def learn(self, state: AgentState, action: Action, state_: AgentState):
self.counter += 1
value_pred = self.network.forward_critic(state)
value_pred_ = self.network.forward_critic(state_)
reward = state.value - state_.value
target = reward + self.gamma * value_pred_.detach()
delta = target - value_pred.detach()
self.critic_loss += F.mse_loss(value_pred, target)
actor_loss_function = custom_loss(delta)
if state.player.shipyards:
self.actor_loss += actor_loss_function(action.prediction_shipyards, action.tensor_action_shipyards)
if state.player.ships:
self.actor_loss += actor_loss_function(action.prediction_ships, action.tensor_action_ships)
if self.counter % self.optimization_frequency == 0:
self.actor_loss.backward()
self.critic_loss.backward()
self.optimizer_actor.step()
self.optimizer_critic.step()
self.optimizer_actor.zero_grad()
self.optimizer_critic.zero_grad()
self.actor_loss = 0
self.critic_loss = 0
I get this trackback
Connected to pydev debugger (build 193.6494.30)
[W ..\torch\csrc\autograd\python_anomaly_mode.cpp:60] Warning: Error detected in AddmmBackward. Traceback of forward call that caused the error:
File "C:\Program Files\JetBrains\PyCharm 2019.3.3\plugins\python\helpers\pydev\pydevd.py", line 2127, in <module>
main()
File "C:\Program Files\JetBrains\PyCharm 2019.3.3\plugins\python\helpers\pydev\pydevd.py", line 2118, in main
globals = debugger.run(setup['file'], None, None, is_module)
File "C:\Program Files\JetBrains\PyCharm 2019.3.3\plugins\python\helpers\pydev\pydevd.py", line 1427, in run
return self._exec(is_module, entry_point_fn, module_name, file, globals, locals)
File "C:\Program Files\JetBrains\PyCharm 2019.3.3\plugins\python\helpers\pydev\pydevd.py", line 1434, in _exec
pydev_imports.execfile(file, globals, locals) # execute the script
File "C:\Program Files\JetBrains\PyCharm 2019.3.3\plugins\python\helpers\pydev\_pydev_imps\_pydev_execfile.py", line 18, in execfile
exec(compile(contents+"\n", file, 'exec'), glob, loc)
File "D:/Thomas/Python/NewHalite/code/A3C.py", line 124, in <module>
alt.run()
File "D:/Thomas/Python/NewHalite/code/A3C.py", line 86, in run
actions = self.agent(player_state[active_id], self.counter)
File "D:\Thomas\Python\NewHalite\code\agent.py", line 18, in __call__
prediction_ships = self.network.forward_actor_ship(agent_state)
File "D:\Thomas\Python\NewHalite\code\network.py", line 58, in forward_actor_ship
x = self.fc_actor_ship(x)
File "C:\ProgramData\Anaconda3\lib\site-packages\torch\nn\modules\module.py", line 722, in _call_impl
result = self.forward(*input, **kwargs)
File "C:\ProgramData\Anaconda3\lib\site-packages\torch\nn\modules\linear.py", line 91, in forward
return F.linear(input, self.weight, self.bias)
File "C:\ProgramData\Anaconda3\lib\site-packages\torch\nn\functional.py", line 1674, in linear
ret = torch.addmm(bias, input, weight.t())
(function print_stack)
Traceback (most recent call last):
File "D:/Thomas/Python/NewHalite/code/A3C.py", line 124, in <module>
alt.run()
File "D:/Thomas/Python/NewHalite/code/A3C.py", line 97, in run
self.learn(player_state[agent_id], player_actions[agent_id], player_state_[agent_id])
File "D:/Thomas/Python/NewHalite/code/A3C.py", line 64, in learn
self.actor_loss.backward()
File "C:\ProgramData\Anaconda3\lib\site-packages\torch\tensor.py", line 185, in backward
torch.autograd.backward(self, gradient, retain_graph, create_graph)
File "C:\ProgramData\Anaconda3\lib\site-packages\torch\autograd\__init__.py", line 127, in backward
allow_unreachable=True) # allow_unreachable flag
RuntimeError: one of the variables needed for gradient computation has been modified by an inplace operation: [torch.FloatTensor [50, 6]], which is output 0 of TBackward, is at version 3; expected version 1 instead. Hint: the backtrace further above shows the operation that failed to compute its gradient. The variable in question was changed in there or anywhere later. Good luck!
If you have any insights related to this issue I would be really interested!
Best,
Thomas
