first test on self paced

Author: thomasw21

IRL/GradientIRL/__pycache__/gradientIRL.cpython-36.pyc

@@ -66,16 +66,10 @@ def print_jacobian(self):
     def loss2(self, alpha,M):
         return np.dot(alpha, np.dot(M, alpha))
 
-    def loss(self,trajs):
+    def loss(self,w,Ms):
         losses = []
-        for traj in trajs:
-            g = self.expert_policy.grad_log(traj)
-            temp = np.zeros([len(self.expert_policy.get_theta()), len(self.reward.params)])
-            for idx in range(len(self.reward.params)):
-                temp[:,idx] = self.reward.basis_traj(traj, idx) * np.ones(len(temp))
-            jacobian = (g*temp.T).T
-            M = np.dot(jacobian.T, jacobian)
-            losses.append(self.loss2(self.reward.params,M))
+        for M in Ms:
+            losses.append(self.loss2(w,M))
         return np.asarray(losses)
 
 

IRL/GradientIRL/gradientIRL.py

@@ -107,15 +107,13 @@
 # =============================================================================
 reward.import_from_file(write_path_girl)
 
-X = 200
-V = 200
+X = 50
+V = 50
 
 
 
 x = np.linspace(-1.2, 0.6, X)
 v = np.linspace(-0.07, 0.07,V)
-X = len(x)
-V = len(v)
 print(X)
 print(V)
 x, v = np.meshgrid(x, v)
@@ -132,9 +130,6 @@
 # =============================================================================
 #         r[i,j] = reward.basis([xi,vj],0)
 # =============================================================================
-print(x.shape)
-print(v.shape)
-print(r.shape)
 ax.plot_surface(x, v, r.T, cmap=cm.coolwarm,
                        linewidth=0, antialiased=False)
 
@@ -144,8 +139,9 @@
 f_sp = irl.GIRL(reward_sp, policy)
 K0=1
 eps=1 #not working for now
+mu=0.5
 
-girl_self_paced = Self_Paced(f_sp,K0,eps,data)
+girl_self_paced = Self_Paced(f_sp,K0,eps,mu)
 trajs = girl_self_paced.import_data(data)
 alphass = girl_self_paced.fit(trajs)
 
@@ -154,15 +150,15 @@
 
 #plot(alphas)
 
+print(alphass)
+
 reward_sp.set_params(alphass[-1])
 
 reward_sp.export_to_file(write_path_self_paced)
 #reward.import_from_file(write_path)
 
 x = np.linspace(-1.2, 0.6, X)
 v = np.linspace(-0.07, 0.07, V)
-X = len(x)
-V = len(v)
 print(X)
 print(V)
 x, v = np.meshgrid(x, v)

IRL/GradientIRL/main.py

@@ -7,6 +7,7 @@
 import numpy as np
 import scipy.optimize as opt
 from IRL import IRL
+from tqdm import tqdm
 
 #Self paced
 
@@ -18,6 +19,7 @@ def __init__(self,f,K0,eps,mu,model=None,constraint='hard'):
         self.eps=eps
         self.mu = mu
         # params correspond to the ws in the paper.
+        self.model = model
         if self.model is None:
             self.params = self.zero()
         else:
@@ -43,10 +45,12 @@ def fit(self,trajs):
         ws = []
 
         loss = []
-        while((self.v == np.ones(len(trajs))).all()): #find a termination condition perhaps double while (alternative search, and then decrement)
-            
+        while(not (self.v == np.ones(len(trajs))).all()): #find a termination condition perhaps double while (alternative search, and then decrement)
+            print('hey')
             #Alternative search strategy
             while(start == True or not((old_v == self.v).all())):
+                print('ho')
+                start=False
                 #minimising for v
 # =============================================================================
 #                 result_v = opt.minimize(self.objective_v, v0, constraints=self.v_constraints)
@@ -55,35 +59,45 @@ def fit(self,trajs):
 #                     print(result_v)
 #                 self.v = result_v.x
 # =============================================================================
-                
-                losses = self.f.loss(trajs)
-                print(losses)
+                Ms = []
+                for traj in tqdm(trajs):
+                    g = self.f.expert_policy.grad_log(traj)
+                    temp = np.zeros([len(self.f.expert_policy.get_theta()), len(self.f.reward.params)])
+                    for idx in range(len(self.f.reward.params)):
+                        temp[:,idx] = self.f.reward.basis_traj(traj, idx) * np.ones(len(temp))
+                    jacobian = (g*temp.T).T
+                    Ms.append(np.dot(jacobian.T, jacobian))
+                    
+                losses = self.f.loss(self.f.reward.params,Ms)
+                print(np.sum(self.v))
                 #second method use dirac
                 old_v=self.v
                 self.v = np.where(losses < 1/self.K,1,0)
 
                 #minimising for w
-                result_w = opt.minimize(self.objective_w, self.w)
+                print('minimise W')
+                
+                result_w = opt.minimize(self.objective_w, self.w,args=(Ms,))
                 if not result_w.success:
                     print(result_w.message)
                     print(result_w)
                 self.w = result_w.x
                 self.f.reward.set_params(self.w)
 
-            ws.append(self.w)
+                ws.append(self.w)
             self.K=self.mu * self.K
 
         return ws
 
-    def objective_w(self,w,trajs):
-        return(np.dot(self.v, self.f.loss(w,trajs))+self.reg(w) ) #le reste est independant de w donc pas besoin de calculer
+    def objective_w(self,w,Ms):
+        return(np.dot(self.v, self.f.loss(w,Ms))+self.reg(w) ) #le reste est independant de w donc pas besoin de calculer
 
     def objective_v(self,v,trajs):
-        return(np.dot(v,self.f.loss(trajs)) - np.sum(v)/self.K) #think about a way to only calculate objective if v is 1
+        return(np.dot(v,self.f.loss(w,trajs)) - np.sum(v)/self.K) #think about a way to only calculate objective if v is 1
 
     def objective(self,inputs,trajs):
         w,v = inputs
-        return(self.reg(w) + v*self.f.objective(self.w) - np.sum(v)/self.K)
+        return(self.reg(w) + np.dot(v,self.f.loss(self.w,trajs)) - np.sum(v)/self.K)
 
 
 

IRL/SelfPaced.py

@@ -23,7 +23,7 @@ def __init__(self, dx, dv, env):
         # tune sigma according to the discretization
         self.sigma_inv = inv(np.array([[0.5*(self.lx/self.dx)**2, 0.  ],
                                       [0., 0.5*(self.lv/self.dv)**2]])) 
-        self.params = np.zeros(dx * dv)
+        self.params = np.ones(dx * dv)/(dx*dv)
 
     def value(self, state, action):
         r = 0.