Just in case it’s useful, I can reproduce the results from calling .reinforce with the following code:
"""
Try some reinforce by hand and similar
"""
import torch
from torch import autograd, nn, optim
import torch.nn.functional as F
from torch.autograd import Variable
import numpy as np
def run_model(x, h1):
torch.manual_seed(123)
params = list(h1.parameters())
x1 = h1(Variable(x))
x2 = F.softmax(x1)
a = torch.multinomial(x2)
print('a', a)
return x1, x2, a
def run_by_hand(params, x, h1):
print('')
print('=======')
print('by hand')
h1.zero_grad()
x1, x2, a = run_model(x, h1)
g = torch.gather(x2, 1, Variable(a.data))
log_g = g.log()
log_g.backward(- torch.ones(4, 1))
print('params.grad', params.grad)
def run_pytorch_reinforce(params, x, h1):
print('')
print('=======')
print('pytorch reinforce')
h1.zero_grad()
x1, x2, a = run_model(x, h1)
a.reinforce(torch.ones(4, 1))
autograd.backward([a], [None])
print('params.grad', params.grad)
def run():
N = 4
K = 1
C = 3
torch.manual_seed(123)
x = torch.ones(N, K)
h1 = nn.Linear(K, C, bias=False)
params = list(h1.parameters())[0]
run_by_hand(params, x, h1)
run_pytorch_reinforce(params, x, h1)
if __name__ == '__main__':
run()
Result:
=======
by hand
a Variable containing:
1
1
0
1
[torch.LongTensor of size 4x1]
params.grad Variable containing:
0.6170
-1.3288
0.7117
[torch.FloatTensor of size 3x1]
=======
pytorch reinforce
a Variable containing:
1
1
0
1
[torch.LongTensor of size 4x1]
params.grad Variable containing:
0.6170
-1.3288
0.7117
[torch.FloatTensor of size 3x1]
(Its actually almost the same amount of code in fact?)
