First version of q-local-search

bee.py

@@ -1,16 +1,18 @@
 import random
+from rl import QLearning
 
 class Bee :
-    def __init__(self,id,data,locIterations):
+    def __init__(self,id,problem,locIterations):
         self.id=id
-        self.data=data
+        self.data=problem
         self.solution=[]
-        self.fitness=0.0
+        self.fitness= 0.0
         self.locIterations=locIterations
+        self.action = []
 
     def localSearch(self):
         best=self.fitness
-        done=False
+        #done=False
         lista=[j for j, n in enumerate(self.solution) if n == 1]
         indice =lista[0]
 
@@ -44,7 +46,36 @@ def localSearch(self):
                 if (quality<best):
                     self.solution[i]= (self.solution[i] + 1) % 2
                     self.fitness = oldFitness
-
+
+
+    def ql_localSearch(self):
+        state = self.solution
+        action = self.data.ql.get_action(state)
+
+        if not self.data.ql.str_state(state) in self.data.ql.q_table[self.data.ql.nbrUn(state)]:
+            self.data.ql.q_table[self.data.ql.nbrUn(state)][self.data.ql.str_state(state)] = {self.data.ql.str_state(state):{}}
+
+        self.data.ql.learn(state,action,self.data.ql.q_table[self.data.ql.nbrUn(state)][self.data.ql.str_state(state)][str(action)],self.data.ql.get_next_state(state,action))
+        self.fitness = self.data.ql.get_q_value(state,action)
+
+        """state = self.solution
+        best_action = self.data.action_space[0]
+        best_fitness = self.fitness
+
+        for to_flip in self.data.action_space:
+            state[to_flip]= (state[to_flip] + 1) % 2
+            fitness = self.data.evaluate(state)
+            
+            action = self.data.ql.get_action(state)
+
+            if fitness > best_fitness:
+                best_fitness = fitness
+                best_action = to_flip
+                qtable_index = nbrUn(state)
+                qtable_state = self.str_state(state)
+                self.data.ql.q_table[qtable_index][qtable_state] = fitness
+
+        self.fitness = best_fitness"""
 
     def setSolution(self,solution):
         self.solution=solution
@@ -56,4 +87,4 @@ def Rand(self,start, end, num):
         for j in range(num): 
             res.append(random.randint(start, end)) 
 
-        return res
+        return res

fs_data.py

@@ -2,27 +2,36 @@
 from fs_problem import FsProblem
 import pandas as pd
 import os, glob
+from rl import QLearning
 
 class FSData():
 
     def __init__(self):
         #url = "https://raw.githubusercontent.com/jbrownlee/Datasets/master/pima-indians-diabetes.data.csv"
-        path = ".\Benchmarks"
+        path = ".\\Benchmarks"
         self.files = glob.glob(os.path.join(path, "*.csv"))
+
         print(self.files)
+
+
+    def attributs_to_flip(self,dataset):
 
+        return [0,2,7]
+
     def run(self):
 
         for filename in self.files:
-            if(filename=='.\\Benchmarks\\glass.csv'):
+            if(filename=='.\\Benchmarks\\sonar.csv'):
                 print(filename)
                 df=pd.read_csv(filename,header=None)
 
-                df = df.iloc[:, [j for j, c in enumerate(df.columns) if j != 0]]
+                #df = df.iloc[:, [j for j, c in enumerate(df.columns) if j != 0]]
+
+                ql = QLearning(len(df.columns),self.attributs_to_flip(df))
 
-                fsd= FsProblem(df)
+                fsd= FsProblem(df,ql)
                 swarm= Swarm(fsd,4,4,10,10,10)
-            #print(fsd.evaluate([0, 0, 0, 0, 1, 0, 0, 0, 0]))
+                #print(fsd.evaluate([0, 0, 0, 0, 1, 0, 0, 0, 0]))
 
                 #swarm.searchArea()
                 swarm.bso()

fs_problem.py

@@ -4,12 +4,15 @@
 from sklearn.svm import SVC
 from sklearn.preprocessing import StandardScaler
 
+
 class FsProblem :
-    def __init__(self,data):
+    def __init__(self,data,qlearn):
         self.data=data
-        self.nbrAttributs= len(self.data.columns)-1 
-        self.outPuts=self.data.iloc[:,self.nbrAttributs]
-
+        self.nb_attribs= len(self.data.columns)-1 
+        self.outPuts=self.data.iloc[:,self.nb_attribs]
+        self.ql = qlearn
+        self.nb_actions = len(self.ql.actions)
+
 
     def evaluate(self,solution):
         list=[i for i, n in enumerate(solution) if n == 1]
@@ -19,9 +22,9 @@ def evaluate(self,solution):
 
 
         array=df.values
-        nbrAttributs =len(array[0])
+        nb_attribs =len(array[0])
 
-        X = array[:,0:nbrAttributs]
+        X = array[:,0:nb_attribs]
         Y = self.outPuts
         train_X, test_X, train_y, test_y = train_test_split(X, Y, 
                                                     random_state=0,

fs_solution.py

@@ -1,5 +1,7 @@
 from fs_data import FSData
 
+
 if __name__=="__main__":
     instance = FSData()
+
     instance.run()

rl.py

@@ -0,0 +1,70 @@
+import numpy as np
+from collections import defaultdict
+
+class QLearning:
+    def __init__(self,nb_atts,actions):
+        self.actions = actions
+        self.alpha = 0.1 # Facteur d'apprentissage
+        self.gamma = 0.85  
+        self.epsilon = 0.01
+        self.q_table = [ {}  for i in range(nb_atts) ] #defaultdict(lambda : [0.0,0.0,0.0,0.0])
+
+    def get_max_value(self,state,actions_vals):
+        max_val = 0
+
+        if not self.str_state(state) in self.q_table[self.nbrUn(state)]:
+            self.q_table[self.nbrUn(state)][self.str_state(state)] = {self.str_state(state):{}}
+
+        q_s = self.q_table[self.nbrUn(state)][self.str_state(state)]
+        for i in actions_vals:
+
+            if not str(i) in q_s:
+                q_s[str(i)] = 0.0
+
+            if q_s[str(i)] > max_val:
+                max_val = q_s[str(i)]
+        return max_val
+
+
+    def get_q_value(self,state,action):
+        return self.q_table[self.nbrUn(state)][self.str_state(state)][str(action)]
+
+    def get_action(self,state):
+        if np.random.uniform() > self.epsilon :
+            #choisir la meilleure action
+            action_values = self.actions
+            argmax_actions=[] # La meilleure action peut ne pas exister donc on elle est choisie aléatoirement
+            for ac in action_values :
+                if ac == self.get_max_value(state,action_values):
+                    argmax_actions.append(ac)
+            next_action = np.random.choice(argmax_actions) 
+        else :
+             next_action = np.random.choice(self.actions)
+        if self.epsilon > 0 :
+            self.epsilon -= 0.00001 #Décrementer espsilon pour Arreter l'exploration aléatoire qu'on aura un politique optimale
+        if self.epsilon < 0 :
+            self.epsilon = 0
+
+        return next_action
+
+
+    def get_next_state(self,state,action):
+        next_state = state
+        next_state[action] = (next_state[action]+1) % 2
+        return next_state
+
+    def learn(self,current_state,current_action,reward,next_state):
+
+        next_action = self.get_action(next_state)
+        new_q = reward + self.gamma * self.q_table[self.nbrUn(next_state)][self.str_state(next_state)][str(next_action)]
+        self.q_table[self.nbrUn(current_state)][self.str_state(current_state)][str(current_action)] = (1 - self.alpha)*self.q_table[self.nbrUn(current_state)][self.str_state(current_state)][str(current_action)] + self.alpha*new_q
+
+    #@staticmethod
+    def str_state(self,mlist):
+        result = ''
+        for element in mlist:
+            result += str(element)
+        return result
+
+    def nbrUn(self,solution):
+        return len([i for i, n in enumerate(solution) if n == 1])

swarm.py

@@ -3,8 +3,8 @@
 
 
 class Swarm :
-    def __init__(self,data,flip,maxChance,nbrBees,maxIterations,locIterations):
-        self.data=data
+    def __init__(self,problem,flip,maxChance,nbrBees,maxIterations,locIterations):
+        self.data=problem
         self.flip=flip
         self.maxChance=maxChance
         self.nbChance=maxChance
@@ -13,7 +13,7 @@ def __init__(self,data,flip,maxChance,nbrBees,maxIterations,locIterations):
         self.locIterations=locIterations
         self.beeList=[]
         self.refSolution = Bee(-1,self.data,self.locIterations)
-        self.refSolution.setSolution(self.refSolution.Rand(0,1,data.nbrAttributs))
+        self.refSolution.setSolution(self.refSolution.Rand(0,1,self.data.nb_attribs))
         self.bestSolution = self.refSolution
         self.tabou=[]
 
@@ -23,7 +23,7 @@ def searchArea(self):
 
         self.beeList=[]
         while((i<self.nbrBees) and (i < self.flip) ) :
-            print ("first")
+            print ("First method to generate")
 
             solution=self.refSolution.solution
             k=0
@@ -40,13 +40,12 @@ def searchArea(self):
         h=0
 
         while((i<self.nbrBees) and (i< 2*self.flip )):
-            print("second")
-
+            print("Second method to generate")
 
             solution=self.refSolution.solution
             k=0
             while((k<int(len(solution)/self.flip)) and (self.flip*k+h < len(solution))):
-                solution[int(self.data.nbrAttributs/self.flip)*h+k] = ((solution[int(self.data.nbrAttributs/self.flip)*h+k]+1)%2)
+                solution[int(self.data.nb_attribs/self.flip)*h+k] = ((solution[int(self.data.nb_attribs/self.flip)*h+k]+1)%2)
                 k+=1
             newBee=Bee(i,self.data,self.locIterations)
             newBee.solution = copy.deepcopy(solution)
@@ -57,7 +56,7 @@ def searchArea(self):
             i+=1
             h=h+1
         while (i<self.nbrBees):
-            print("alea")
+            print("Random method to generate")
             solution= self.refSolution.solution
             indice = random.randint(0,len(solution)-1)
             solution[indice]=((solution[indice]+1) % 2)
@@ -66,16 +65,13 @@ def searchArea(self):
             self.beeList.append(newBee)
             i+=1
         for bee in (self.beeList):
-            print("here")
+
+            print("Printing all the solutions")
             lista=[j for j, n in enumerate(bee.solution) if n == 1]
             if (len(lista)== 0):
 
-                bee.setSolution(bee.Rand(0,1,self.data.nbrAttributs))
+                bee.setSolution(bee.Rand(0,1,self.data.nb_attribs))
 
-
-
-
-
     def selectRefSol(self):
         self.beeList.sort(key=lambda Bee: Bee.fitness, reverse=True)
         bestQuality=self.beeList[0].fitness
@@ -91,11 +87,12 @@ def selectRefSol(self):
                 self.nbChance-=1
                 if(self.nbChance > 0): return self.bestBeeQuality()
                 else :return self.bestBeeDiversity()
+
     def distanceTabou(self,bee):
-        distanceMin=self.data.nbrAttributs
+        distanceMin=self.data.nb_attribs
         for i in range(len(self.tabou)):
             cpt=0
-            for j in range(self.data.nbrAttributs):
+            for j in range(self.data.nb_attribs):
                 if (bee.solution[j] != self.tabou[i].solution[j]) :
                       cpt +=1
             if (cpt<=1) :
@@ -124,7 +121,7 @@ def bestBeeQuality(self):
             if(pos!=-1) :
                 return self.beeList[pos]
         bee= Bee(-1,self.data,self.locIterations)
-        bee.setSolution(bee.Rand(0,1,self.data.nbrAttributs))
+        bee.setSolution(bee.Rand(0,1,self.data.nb_attribs))
         return bee
 
     def bestBeeDiversity(self):
@@ -134,12 +131,13 @@ def bestBeeDiversity(self):
                 max = self.distanceTabou(self.beeList[i])
         if (max==0):
             bee= Bee(-1,self.data,self.locIterations)
-            bee.setSolution(bee.Rand(0,1,self.data.nbrAttributs))
+            bee.setSolution(bee.Rand(0,1,self.data.nb_attribs))
             return bee
         i=0
         while(i<len(self.beeList) and self.distanceTabou(self.beeList[i])!= max) :
             i+=1
         return self.beeList[i]
+
     def bso(self):
         i=0
         while(i<self.maxIterations):
@@ -158,7 +156,8 @@ def bso(self):
             #La recherche locale
 
             for j in range(self.nbrBees):
-                self.beeList[j].localSearch()
+                #self.beeList[j].localSearch()
+                self.beeList[j].ql_localSearch()
                 #print("fitness de : ")
                 print(self.beeList[j].fitness)
             self.refSolution=self.selectRefSol()