From 85c1532920e366fdd81972387d66051203cad33b Mon Sep 17 00:00:00 2001
From: Vasilis Valatsos <vasilvalat@gmail.com>
Date: Sat, 9 Dec 2023 13:48:16 +0100
Subject: [PATCH] Updated critic to use leaky ReLu

---
 .DS_Store           | Bin 0 -> 8196 bytes
 agents/.DS_Store    | Bin 0 -> 6148 bytes
 agents/ppo/agent.py |   9 +++++----
 agents/ppo/brain.py |   4 ++--
 4 files changed, 7 insertions(+), 6 deletions(-)
 create mode 100644 .DS_Store
 create mode 100644 agents/.DS_Store

diff --git a/.DS_Store b/.DS_Store
new file mode 100644
index 0000000000000000000000000000000000000000..746009c13fae3ef1a5eacfebd1bcac7ba8812d20
GIT binary patch
literal 8196
zcmeHMO>fgc5S>laI#vbBfl9p~S>hU%P^wlHmo!Zdm4HKo-~cE%iQ8JZc8Q$?6h+Dz
z{sVu3D}Mz33nzH9yRH0@a^L`{*{ybGv-{qx_wCruq(mf|qrn5BO(LqGvaPP8Sx|VM
zYoV0%$OfzcpQukR>KS>|EuhVSQNSo*6fg=H1&jj!g93PGbE#bO-dD4xH3}F7mQn$J
zJ~*gs8`{p5m5&ZIG6jGwV^|jSkq1bQqisXmxw4W9pDKG0nksaPAru|+4#Qy^+Rl|#
zbP|eALa!`zh9czZ;5iabqM@v5jRHo2q5_<|uOnOU(gl6U<nP_1AnNyms0Re!R0pVA
zKV?CWXbrjkDKh<#5{h%!FTca~Ch>*#`?Yrw_;FINe;1{d^6Hvp+19%Cx^osyI*FT%
z;-))%$)BD@QP8{WyS?YWH|bXI97bW{`k^<J1b^TG^6G^j4x&jj8ij*cayx3mwkvkG
zx;dTh?(Ec@?T7VQ&6)1+ZP%RL{oUEju58@9|LC~;HaHKXk8+13^uva3sn1vKC-gG=
zA9sdf5QV20XAaNB`16X|h`X3i>u5{W&w(9+wZ-R<Ra5Y4X@&(x1l>=_#|h-L7e5MU
z56Bbv?-#G8#8;g5UC}4%1M8!o^IrOv!&~|)M!5O{TE@)hRLtbi|2z`*QjxIp$B~fR
zRW24RoRLd>z6;Kl_)bYHWJ7hnqc|E)!0VHjiw2ILG^0WR&|}nK(F!tp(s+v=(-9p~
zo1RjK+BxqWJ!!O!cnsltgdHr5P8mU2#xZ;!P>Y_xZh<$*t|4P6&8W)Qd51N8*<-Eb
zQw8jV-cyq0b5x1jsi;~7lcP1rM=PtFL6)UXA&s*h`JT++Ac84Ifg7&CirU{M=l|{N
zfB(PXDwypV1(sU@QEqiwO=xiWbvG{u=h_bH8&ocgn=30RXk<DLE7Nh<tA7}x?*Pi2
ahPHEMF@o~fe+c037UumgwKvW3D)0-kOLU+B

literal 0
HcmV?d00001

diff --git a/agents/.DS_Store b/agents/.DS_Store
new file mode 100644
index 0000000000000000000000000000000000000000..ede64cb7ce322fd245293854b45ae9d0234745a3
GIT binary patch
literal 6148
zcmeHKL5tHs6n@iI(@=#T6!sznyq2}DOJ&(h>~4j<C~QOzDmB>zH<-<YCf%}>LeBbQ
zy!uQ0FP`*!GgH~7>s3VNl{ep;m-kINUxrMEh{SjvKPKuEQ5VkGa?#vie4TyES|;NF
zg&HHJil+GOQn6KS6aS(D{Oo$_nWdD|_2+kWlF73ulMyO-c|3#f{#1r&%_``M&M1VR
zQ^MmkSYI2gHdaPKa<fW2xVb7?l!f^R-=ZQ*%W63ME?Vu*t=o>{Iz8uoa4r`?6;|_d
z9M0Zy@0FBUw2s5*b($=u{d-3;ufjA>W{Qx;2}<6*N%L4P#&Vv=rQ!zqg6nqOX@6(A
z9PRH9yuByG)xcXGJlz|3ql3|E<#z8sd~|p^{ghqg@-s_;<&X8R?5@Ezd_iF5d>PF0
zOy(bv;cAyoX+kITf}Yb$#78t~sKVUqir=9UN?$_f7)oD|1SXVG+2H+X@X)np6|f3e
z1vV5g@2qZbBWpWW0jq#jK!2rp5Q5(mzQJ0fdUc>tM*v_4X=RA{W05(YLEm7l5m#VB
zQ-PW)%n?IqI`W>W^9|M-HJyYxd<b)AVNNJQ-5uk5s+@$c(Y96rtH7oLZQbqj{y+Ki
z`G1pSdsYFfz<;HH=o|;fV@%2Hty`1hz1D+&fU|L3t?@Snf!c~u%UkgdTp98n2SDFo
Ttq~cR{SZ(x*v2aGR~7gL+_sAG

literal 0
HcmV?d00001

diff --git a/agents/ppo/agent.py b/agents/ppo/agent.py
index ebc7bba..b7215c0 100644
--- a/agents/ppo/agent.py
+++ b/agents/ppo/agent.py
@@ -102,16 +102,17 @@ class Agent:
                 self.critic.optimizer.zero_grad()
                 self.total_loss.backward()
 
+                T.nn.utils.clip_grad_norm_(
+                    self.actor.parameters(), max_norm=2)
+                T.nn.utils.clip_grad_norm_(
+                    self.critic.parameters(), max_norm=2)
+
                 # Calculate the gradient norms for both networks
                 actor_grad_norm = T.nn.utils.clip_grad_norm_(
                     self.actor.parameters(), max_norm=1)
                 critic_grad_norm = T.nn.utils.clip_grad_norm_(
                     self.critic.parameters(), max_norm=1)
 
-                T.nn.utils.clip_grad_norm_(
-                    self.actor.parameters(), max_norm=1)
-                T.nn.utils.clip_grad_norm_(
-                    self.critic.parameters(), max_norm=1)
                 # Log or print the gradient norms
                 print(f"Actor Gradient Norm: {actor_grad_norm}")
                 print(f"Critic Gradient Norm: {critic_grad_norm}")
diff --git a/agents/ppo/brain.py b/agents/ppo/brain.py
index aa0728c..6dc02b9 100644
--- a/agents/ppo/brain.py
+++ b/agents/ppo/brain.py
@@ -96,9 +96,9 @@ class CriticNetwork(nn.Module):
 
         self.critic = nn.Sequential(
             nn.Linear(input_dims, fc1_dims),
-            nn.ReLU(),
+            nn.LeakyReLU(),
             nn.Linear(fc1_dims, fc2_dims),
-            nn.ReLU(),
+            nn.LeakyReLU(),
             nn.Linear(fc2_dims, 1)
         )