I need your help on how I can resolve this error:
next_state, reward, done, info = env.step(action)
TypeError: cannot unpack non-iterable int object
class QNetwork(nn.Module):
def init(self, state_size, action_size, seed):
super(QNetwork, self).init()
self.seed = torch.manual_seed(seed)
self.fc1 = nn.Linear(state_size, 32)
self.fc2 = nn.Linear(32, 64)
self.fc3 = nn.Linear(64, action_size)
def forward(self, x):
"""Forward pass"""
x = F.relu(self.fc1(x))
x = F.relu(self.fc2(x))
x = self.fc3(x)
return x
class ReplayBuffer:
def init(self, buffer_size, batch_size, seed):
self.batch_size = batch_size
self.seed = random.seed(seed)
self.memory = deque(maxlen=buffer_size)
self.experience = namedtuple(“Experience”, field_names=[“state”, “action”, “reward”, “next_state”, “done”])
def add(self, state, action, reward, next_state, done):
"""Add experience"""
experience = self.experience(state, action, reward, next_state, done)
self.memory.append(experience)
def sample(self):
experiences = random.sample(self.memory, k=self.batch_size)
# Convert to torch tensors
states = torch.from_numpy(
np.vstack([experience.state for experience in experiences if experience is not None])).float().to(device)
actions = torch.from_numpy(
np.vstack([experience.action for experience in experiences if experience is not None])).long().to(device)
rewards = torch.from_numpy(
np.vstack([experience.reward for experience in experiences if experience is not None])).float().to(device)
next_states = torch.from_numpy(
np.vstack([experience.next_state for experience in experiences if experience is not None])).float().to(
device)
# Convert done from boolean to int
dones = torch.from_numpy(
np.vstack([experience.done for experience in experiences if experience is not None]).astype(
np.uint8)).float().to(device)
return (states, actions, rewards, next_states, dones)
def __len__(self):
return len(self.memory)
BUFFER_SIZE = int(1e5) # Replay memory size
BATCH_SIZE = 64 # Number of experiences to sample from memory
GAMMA = 0.99 # Discount factor
TAU = 1e-3 # Soft update parameter for updating fixed q network
LR = 1e-4 # Q Network learning rate
UPDATE_EVERY = 4 # How often to update Q network
class DQNAgent:
def init(self, state_size, action_size, seed, learningRate, decayRate, numOfEpisodes, stepsPerEpisode, epsilon,
annealingConstant, annealAfter, checkpoint=’’):
self.state_size = state_size
self.action_size = action_size
random.seed(seed)
# Initialize Q and Fixed Q networks
self.q_network = QNetwork(state_size, action_size, seed).to(device)
self.fixed_network = QNetwork(state_size, action_size, seed).to(device)
self.optimizer = optim.Adam(self.q_network.parameters())
# Initialise memory
self.memory = ReplayBuffer(BUFFER_SIZE, BATCH_SIZE, seed)
self.timestep = 0
self.voltageRanges_2 = [’<0.85’, ‘0.85-0.874’, ‘0.875-0.899’, ‘0.9-0.924’, ‘0.925-0-949’, ‘0.95-0.974’,
‘0.975.0.999’, ‘1-1.024’, ‘1.025-1.049’, ‘1.05-1.074’, ‘1.075-1.1’, ‘>=1.1’]
self.loadingPercentRange = [‘0-9’, ‘10-19’, ‘20-29’, ‘30-39’, ‘40-49’, ‘50-59’, ‘60-69’, ‘70-79’, ‘80-89’,
‘90-99’, ‘100-109’, ‘110-119’, ‘120-129’, ‘130-139’, ‘140-149’, ‘150 and above’]
self.statesLev1 = [‘v_’ + x + ‘l’ + y for x in self.voltageRanges_2 for y in self.loadingPercentRange]
self.states = [‘s1:’ + x + ‘;s2:’ + y + ‘;’ for x in self.statesLev1 for y in self.statesLev1]
# initialise environment
self.env = case9()
# Possible actions to take, combinations of v_ref into combinations of lp_ref
self.actions = ['v_ref:' + str(x) + ';lp_ref:' + str(y) for x in self.env.actionSpace['v_ref_pu'] for y in
self.env.actionSpace['lp_ref']]
# Check if pickle file with current hyperparams exist
self.checkPointName = 'pickles_qlearning\pickled_q_table_lr' + str(learningRate) + 'dr' + str(
decayRate) + 'noe' + str(numOfEpisodes) + 'spe' + str(stepsPerEpisode) + 'e' + str(epsilon) + 'ac' + str(
annealingConstant) + 'aa' + str(annealAfter) + '.pkl'
if os.path.isfile(self.checkPointName) or checkpoint != '':
print('loading data from checkpoint')
# Load Qtable from pickle file
with open(self.checkPointName if os.path.isfile(self.checkPointName) else checkpoint, 'rb') as pickle_file:
data = pickle.load(pickle_file)
self.epsilon = data['e'] if os.path.isfile(self.checkPointName) else epsilon
self.q_table = data['q_table']
self.allRewards = data['allRewards'] if os.path.isfile(self.checkPointName) else []
else: # create new Q_table
self.q_table = pd.DataFrame(0, index=np.arange(len(self.actions)), columns=self.states)
self.epsilon = epsilon
self.allRewards = []
self.numOfEpisodes = numOfEpisodes
self.annealingRate = annealingConstant
self.numOfSteps = stepsPerEpisode
self.learningRate = learningRate
self.decayRate = decayRate
self.annealAfter = annealAfter
def step(self, state, action, reward, next_state, done):
self.memory.add(state, action, reward, next_state, done)
self.timestep += 1
if self.timestep % UPDATE_EVERY == 0:
if len(self.memory) > BATCH_SIZE:
sampled_experiences = self.memory.sample()
self.learn(sampled_experiences)
def learn(self, experiences):
states, actions, rewards, next_states, dones = experiences
action_values = self.fixed_network(next_states).detach()
max_action_values = action_values.max(1)[0].unsqueeze(1)
# If done just use reward, else update Q_target with discounted action values
Q_target = rewards + (GAMMA * max_action_values * (1 - dones))
Q_expected = self.q_network(states).gather(1, actions)
# Calculate loss
loss = F.mse_loss(Q_expected, Q_target)
self.optimizer.zero_grad()
# backward pass
loss.backward()
# update weights
self.optimizer.step()
# Update fixed weights
self.update_fixed_network(self.q_network, self.fixed_network)
def update_fixed_network(self, q_network, fixed_network):
for source_parameters, target_parameters in zip(q_network.parameters(), fixed_network.parameters()):
target_parameters.data.copy_(TAU * source_parameters.data + (1.0 - TAU) * target_parameters.data)
def act(self, state, eps=0.0):
rnd = random.random()
if rnd < eps:
return np.random.randint(self.action_size)
else:
state = torch.from_numpy(state).float().unsqueeze(0).to(device)
# set the network into evaluation mode
self.q_network.eval()
with torch.no_grad():
action_values = self.q_network(state)
# Back to training mode
self.q_network.train()
action = np.argmax(action_values.cpu().data.numpy())
return action
def checkpoint(self, filename):
torch.save(self.q_network.state_dict(), filename)
MAX_EPISODES = 2000
MAX_STEPS = 1000
ENV_SOLVED = 200
PRINT_EVERY = 100
Epsilon schedule
EPS_START = 1.0
EPS_DECAY = 0.999
EPS_MIN = 0.01
EPS_DECAY_RATES = [0.9, 0.99, 0.999, 0.9999]
plt.figure(figsize=(10, 6))
for decay_rate in EPS_DECAY_RATES:
test_eps = EPS_START
eps_list = []
for _ in range(MAX_EPISODES):
test_eps = max(test_eps * decay_rate, EPS_MIN)
eps_list.append(test_eps)
plt.plot(eps_list, label='decay rate: {}'.format(decay_rate))
plt.title(‘Effect of various decay rates’)
plt.legend(loc=‘best’)
plt.xlabel(‘Number of episodes’)
plt.ylabel(‘epsilon’)
plt.show()
Get state and action sizes
state_size = env.observation_space
action_size = env.action_space
print(‘State size: {}, action size: {}’.format(state_size, action_size))
State_size: 8
action_size: 4
dqn_agent = DQNAgent(8, 4, seed=0, numOfEpisodes=50000, stepsPerEpisode=12, learningRate=0.1,
decayRate=0.1, epsilon=1, annealingConstant=0.98, annealAfter=400, checkpoint=’’)
start = time()
scores = []
Maintain a list of last 100 scores
scores_window = deque(maxlen=100)
eps = EPS_START
for episode in range(1, MAX_EPISODES + 1):
state = env.reset
score = 0
for t in range(MAX_STEPS):
action = dqn_agent.act(state, eps)
next_state, reward, done, info = env.step(action)
dqn_agent.step(state, action, reward, next_state, done)
state = next_state
score += reward
if done:
break
eps = max(eps * EPS_DECAY, EPS_MIN)
if episode % PRINT_EVERY == 0:
mean_score = np.mean(scores_window)
print('\r Progress {}/{}, average score:{:.2f}'.format(episode, MAX_EPISODES, mean_score), end="")
if score >= ENV_SOLVED:
mean_score = np.mean(scores_window)
print('\rEnvironment solved in {} episodes, average score: {:.2f}'.format(episode, mean_score), end="")
sys.stdout.flush()
dqn_agent.checkpoint('solved_200.pth')
break
scores_window.append(score)
scores.append(score)
end = time()
print(‘Took {} seconds’.format(end - start))
Progress: 2000 / 2000
average_score: 205.24
dqn_agent.checkpoint(‘solved_200.pth’)
plt.figure(figsize=(10, 6))
plt.plot(scores)
A bit hard to see the above plot, so lets smooth it (red)
plt.plot(pd.Series(scores).rolling(100).mean())
plt.title(‘DQN Training’)
plt.xlabel(’# of episodes’)
plt.ylabel(‘score’)
plt.show()