I am trying to use A3C with LSTM for an environment where states has 12 inputs ranging from -5000 to 5000. I am using an LSTM layer of size 12 and then 2 fully connected hidden layers of size 256, then 1 fc for 3 action dim and 1 fc for 1 value function. The reward is in range (-1,1).

However during initial training I am unable to get good results.

My question is- Is this Neural Network good enough for this kind of environment? Or this bad performance initially is due to lstm?

Below is the code for Actor Critic

```
class ActorCritic(torch.nn.Module):
def __init__(self, params):
super(ActorCritic, self).__init__()
self.state_dim = params.state_dim
self.action_space = params.action_dim
self.hidden_size = params.hidden_size
state_dim = params.state_dim
self.lstm = nn.LSTMCell(state_dim, state_dim)
self.lstm.bias_ih.data.fill_(0)
self.lstm.bias_hh.data.fill_(0)
lst = [state_dim]
for i in range(params.layers):
lst.append(params.hidden_size)
self.hidden = nn.ModuleList()
for k in range(len(lst)-1):
self.hidden.append(nn.Linear(lst[k], lst[k+1]))
for layer in self.hidden:
layer.apply(init_weights)
self.critic_linear = nn.Linear(params.hidden_size, 1)
self.critic_linear.apply(init_weights)
self.actor_linear = nn.Linear(params.hidden_size, self.action_space)
self.actor_linear.apply(init_weights)
self.train()
def forward(self, inputs):
inputs, (hx, cx) = inputs
inputs = inputs.reshape(1,-1)
hx, cx = self.lstm(inputs, (hx, cx))
x = hx
for layer in self.hidden:
x = torch.tanh(layer(x))
return self.critic_linear(x), self.actor_linear(x), (hx, cx)
class Params():
def __init__(self):
self.lr = 0.0001
self.gamma = 0.99
self.tau = 1.
self.num_processes = os.cpu_count()
self.state_dim = 12
self.action_dim = 3
self.hidden_size = 256
self.layers = 2
self.lstm_layers = 1
self.lstm_size = self.state_dim
self.num_steps = 20
```