I’m doing reinforcement learning
Batch size cannot be greater than 1
I’m trying a batch size of 32
I can’t find anything strange when I look at Ross
I’m in trouble
tensor([[1.3092e-03, 2.1749e-03, 2.6177e-06, 2.2517e-03, 2.1730e-03, 4.7358e-05,
1.3836e-03, 8.5324e-04, 1.0955e-03, 1.9888e-03, 1.3575e-03, 1.6514e-05,
2.5816e-04, 1.8731e-04, 1.9047e-03, 5.3802e-04, 6.7010e-04, 8.9404e-04,
4.5198e-04, 1.1572e-04, 3.0101e-04, 1.6938e-03, 4.7421e-04, 6.1655e-04,
3.7556e-03, 1.9652e-04, 3.1410e-04, 6.3015e-05, 3.7714e-04, 2.9124e-08,
6.6303e-04, 2.7347e-05]], device=‘cuda:0’, grad_fn=)
RuntimeError Traceback (most recent call last)
in
427 trin.Double_R2D2_IQN_pioritized_Nstep_NAF_replay(batch_size, gamma,step=episode,
428 state_size=state_,action_size=acthon,
→ 429 multireward_steps=multireward_steps)
430
431 if done or t==max_number_of_steps + 1:
in Double_R2D2_IQN_pioritized_Nstep_NAF_replay(self, batch_size, gamma, step, state_size, action_size, multireward_steps)
263 loss=loss*weights
264 print(loss)
→ 265 loss.backward() # ここを変更
266 optimizer.step()
267 self.Rs=[0 for _ in range(multireward_steps)]
~\Anaconda3\envs\pyflan\lib\site-packages\torch\tensor.py in backward(self, gradient, retain_graph, create_graph)
196 products. Defaults to False.
197 “”"
→ 198 torch.autograd.backward(self, gradient, retain_graph, create_graph)
199
200 def register_hook(self, hook):
~\Anaconda3\envs\pyflan\lib\site-packages\torch\autograd_init_.py in backward(tensors, grad_tensors, retain_graph, create_graph, grad_variables)
92 grad_tensors = list(grad_tensors)
93
—> 94 grad_tensors = _make_grads(tensors, grad_tensors)
95 if retain_graph is None:
96 retain_graph = create_graph
~\Anaconda3\envs\pyflan\lib\site-packages\torch\autograd_init_.py in _make_grads(outputs, grads)
33 if out.requires_grad:
34 if out.numel() != 1:
—> 35 raise RuntimeError(“grad can be implicitly created only for scalar outputs”)
36 new_grads.append(torch.ones_like(out, memory_format=torch.preserve_format))
37 else:
RuntimeError: grad can be implicitly created only for scalar outputs
The code is
targets= torch.zeros(batch_size,device='cuda:0')
trin_x=list(range(batch_size))
weights = torch.ones(batch_size,device='cuda:0')
idx=np.random.choice(np.arange(len(memory_TDerror.buffer)), size=batch_size,
replace=False,p = memory_TDerror.buffer/np.sum(memory_TDerror.buffer))
for i in idx:
inputs=(torch.cat([memory.buffer[i-multireward_steps][0]]),
torch.cat([memory.buffer[i-multireward_steps][1]]))
for _,i in enumerate(idx):
with torch.no_grad():
targets[_]=memory.buffer[i][2].to("cuda:0")+(gamma ** multireward_steps)*targetQN.forward(memory.buffer[i][0],"net_v")
priority = rank_sum(memory_TDerror.buffer[i], self.alpha)
weights[_]=(len(memory.buffer) * priority) ** (-self.beta)
weights = weights / max(weights)
#リプレイとおなじ
optimizer.zero_grad()
output = mainQN.forward(inputs,"net_q")
if self.IQN==True:
self.loss_IQN(target,output,weights)
else:
loss = huber(output-targets)
loss=loss*weights
print(loss)
loss.backward() # ここを変更
optimizer.step()
self.Rs=[0 for _ in range(multireward_steps)]