From 30dda47f95c6533b55d78abe5a354f0cf748b855 Mon Sep 17 00:00:00 2001 From: Vasilis Valatsos Date: Thu, 7 Mar 2024 10:43:14 +0200 Subject: [PATCH] Made the agent advantage accessible --- ml/ppo/agent.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/ml/ppo/agent.py b/ml/ppo/agent.py index 82c5f75..d72f5cb 100644 --- a/ml/ppo/agent.py +++ b/ml/ppo/agent.py @@ -61,7 +61,7 @@ class Agent: state_arr, action_arr, old_probs_arr, vals_arr, reward_arr, dones_arr, batches = self.memory.generate_batches() values = vals_arr - advantage = np.zeros(len(reward_arr), dtype=np.float64) + self.advantage = np.zeros(len(reward_arr), dtype=np.float64) for t in range(len(reward_arr)-1): discount = 1 @@ -71,8 +71,8 @@ class Agent: (reward_arr[k] + self.gamma*values[k+1] * (1-int(dones_arr[k])) - values[k]) discount *= self.gamma * self.gae_lambda - advantage[t] = a_t - advantage = T.tensor(advantage).to(self.actor.device) + self.advantage[t] = a_t + self.advantage = T.tensor(self.advantage).to(self.actor.device) values = T.tensor(values).to(self.actor.device) @@ -90,15 +90,15 @@ class Agent: new_probs = dist.log_prob(actions) prob_ratio = new_probs.exp() / old_probs.exp() - weighted_probs = advantage[batch] * prob_ratio + weighted_probs = self.advantage[batch] * prob_ratio weighted_clipped_probs = T.clamp( - prob_ratio, 1-self.policy_clip, 1+self.policy_clip)*advantage[batch] + prob_ratio, 1-self.policy_clip, 1+self.policy_clip)*self.advantage[batch] self.actor_loss = -T.min(weighted_probs, weighted_clipped_probs).mean() - returns = advantage[batch] + values[batch] + returns = self.advantage[batch] + values[batch] self.critic_loss = (returns - critic_value)**2 self.critic_loss = self.critic_loss.mean()