Update frequent itemset mining

Author: llhthinker

Frequent Itemset Mining/apriori.py

@@ -1,10 +1,12 @@
+from datetime import datetime
+
 class Apriori():
 
     def __init__(self, dataset):
         self.dataset = dataset
         self.support_data = {}
         self.freq_itemsets = []
-        self.strong_association_rules = []
+        self.t_num = float(len(self.dataset))
 
 
     def __create_C1(self):
@@ -88,15 +90,16 @@ def __generate_Lk_by_Ck(self, Ck, min_sup):
                         item_count[item] = 1
                     else:
                         item_count[item] += 1
-        t_num = float(len(self.dataset))
+
         for item in item_count:
-            if (item_count[item] / t_num) >= min_sup:
+            if (item_count[item] / self.t_num) >= min_sup:
                 Lk.add(item)
-                self.support_data[item] = item_count[item] / t_num
+                self.support_data[item] = item_count[item] / self.t_num
+
         return Lk
 
 
-    def generate_L(self, k, min_sup):
+    def generate_L(self, min_sup):
         """
         Generate all frequent itemsets.
         Args:
@@ -107,44 +110,41 @@ def generate_L(self, k, min_sup):
             L: The list of Lk.
             support_data: A dictionary. The key is frequent itemset and the value is support.
         """
+        start = datetime.now()
         C1 = self.__create_C1()
+        deltatime = datetime.now() - start
+        create_Ck_time = deltatime.seconds + deltatime.microseconds / 1000000
+        
+        start = datetime.now()
         L1 = self.__generate_Lk_by_Ck(C1, min_sup)
+        deltatime = datetime.now() - start
+        generate_Lk_time = deltatime.seconds + deltatime.microseconds / 1000000
+        
         Lksub1 = L1.copy()
-        print(Lksub1)
         for lk_i in Lksub1:
             self.freq_itemsets.append((lk_i, self.support_data[lk_i]))
-        for i in range(2, k+1):
+        i = 2
+
+        while True:
+            start = datetime.now()
             Ci = self.__create_Ck(Lksub1, i)
+            deltatime = datetime.now() - start
+            create_Ck_time += deltatime.seconds + deltatime.microseconds / 1000000
+
+            start = datetime.now()
             Li = self.__generate_Lk_by_Ck(Ci, min_sup)
+            deltatime = datetime.now() - start
+            generate_Lk_time += deltatime.seconds + deltatime.microseconds / 1000000
+
             Lksub1 = Li.copy()
-            print(Lksub1)
+            
+            if len(Lksub1) == 0:
+                break
             for lk_i in Lksub1:
                 self.freq_itemsets.append((lk_i, self.support_data[lk_i]))
-        return self.freq_itemsets
-
+            i += 1
+        
+        print("Create Ck time (s): ", create_Ck_time)
+        print("Generate Lk time (s): ", generate_Lk_time)
 
-    def generate_big_rules(self, min_conf):
-        """
-        Generate big rules from frequent itemsets.
-        Args:
-            L: The list of Lk.
-            support_data: A dictionary. The key is frequent itemset and the value is support.
-            min_conf: Minimal confidence.
-        Returns:
-            big_rule_list: A list which contains all big rules. Each big rule is represented
-                        as a 3-tuple.
-        """
-        if self.freq_itemsets is None:
-            return
-
-        sub_set_list = []
-        for freq_set, support in self.freq_itemsets:
-            for sub_set in sub_set_list:
-                if sub_set.issubset(freq_set):
-                    conf = support / self.support_data[freq_set - sub_set]
-                    big_rule = (freq_set - sub_set, sub_set, conf)
-                    if conf >= min_conf and big_rule not in self.strong_association_rules:
-                        # print freq_set-sub_set, " => ", sub_set, "conf: ", conf
-                        self.strong_association_rules.append(big_rule)
-            sub_set_list.append(freq_set)
-        return self.strong_association_rules
+        return self.freq_itemsets

Frequent Itemset Mining/fpgrowth.py

@@ -20,13 +20,11 @@ def find_child(self, name):
 
 class FPGrowth():
 
-    def __init__(self, dataset, min_sup=0.0, min_conf=0.0):
+    def __init__(self, dataset, min_sup=0.0):
         self.dataset = dataset
         self.min_sup = min_sup
-        self.min_conf = min_conf
         self.freq_L1 = {}  # 1-频繁项
         self.freq_itemsets = []  # 存储每个频繁项及其对应的计数
-        self.strong_association_rules = []  # 存储强关联规则
 
     def __get_frequency(self, trans_records):
         rect = {}
@@ -38,7 +36,7 @@ def __get_frequency(self, trans_records):
     def build_fptree(self):
         if self.dataset is None:
             return
-        # 依据销售数量创建item序列
+        # 创建item序列
         self.freq_L1 = self.__get_frequency(self.dataset)
         tmp_list = []
         tmp_list.extend(self.freq_L1.keys())
@@ -73,7 +71,7 @@ def __fpgrowth(self, cpb, post_model):
             rule.append(header.name)
             rule.extend(post_model)
             # 表头项+后缀模式  构成一条频繁模式（频繁模式内部也是按照F1排序的），频繁度为表头项的计数
-            temp = (rule, header.count)
+            temp = (rule, header.count / data_num)
             self.freq_itemsets.append(temp)
             # 新的后缀模式：表头项+上一次的后缀模式（注意保持顺序，始终按F1的顺序排列）
             new_post_pattern = []

Frequent Itemset Mining/generate_data.py

@@ -0,0 +1,64 @@
+import os
+import numpy as np
+
+class GenData():
+    def __init__(self, ntrans, tlen, nitems):
+        self.run_apt = "../../IBMGenerator/gen lit "
+        self.target_folder = "./data/"
+        self.ntrans = ntrans
+        self.tlen = tlen
+        self.nitems = nitems
+    
+    def gen_base_data(self):
+        fname = "base_set"
+        run_command = self.run_apt + "-ntrans " + str(self.ntrans)      \
+                    + " -tlen " + str(self.tlen) + " -nitems " + str(self.nitems)       \
+                    + " -fname " + self.target_folder + fname + " -ascii"
+        return os.system(run_command)
+
+    def gen_data_by_ntrans(self):
+        ntrans_range = range(1, 21, 1)
+        sub_folder = "ntrans/"
+        os.system("mkdir " + self.target_folder + sub_folder)
+        for ntrans in ntrans_range:
+            fname = self.target_folder + sub_folder + str(ntrans)
+            print(fname)
+            run_command = self.run_apt + "-ntrans " + str(ntrans)      \
+                    + " -tlen " + str(self.tlen) + " -nitems " + str(self.nitems)       \
+                    + " -fname " + fname + " -ascii"
+            os.system(run_command)
+
+    def gen_data_by_tlen(self):
+        tlen_range = range(1, 21, 1)
+        sub_folder = "tlen/"
+        os.system("mkdir " + self.target_folder + sub_folder)
+        for tlen in tlen_range:
+            fname = self.target_folder + sub_folder + str(tlen)
+            print(fname)
+            run_command = self.run_apt + "-ntrans " + str(self.ntrans)      \
+                    + " -tlen " + str(tlen) + " -nitems " + str(self.nitems)       \
+                    + " -fname " + fname + " -ascii"
+            os.system(run_command)
+
+
+    def gen_data_by_nitems(self):
+        nitems_range = list(np.arange(0.1, 2.1, 0.1))
+        sub_folder = "nitems/"
+        os.system("mkdir " + self.target_folder + sub_folder)
+        for nitems in nitems_range:
+            fname = self.target_folder + sub_folder + str(nitems)
+            print(fname)
+            run_command = self.run_apt + "-ntrans " + str(self.ntrans)      \
+                    + " -tlen " + str(self.tlen) + " -nitems " + str(nitems)       \
+                    + " -fname " + fname + " -ascii"
+            os.system(run_command)
+    
+
+
+if __name__ == "__main__":
+    # base set 5, 10, 1
+    gen_data = GenData(ntrans=5, tlen=10, nitems=1)
+    gen_data.gen_base_data()
+    # gen_data.gen_data_by_ntrans()
+    # gen_data.gen_data_by_tlen()
+    # gen_data.gen_data_by_nitems()

Frequent Itemset Mining/picture/minsup_time.png

@@ -1,6 +1,20 @@
+from datetime import datetime
+import os
+import matplotlib.pyplot as plt
+import numpy as np
+
 from apriori import Apriori
 from fpgrowth import FPGrowth
 
+
+def data_reader(data_file):
+    data_set = []
+    with open(data_file, 'r') as f:
+        for line in f:
+            data_set.append(line.split()[3:])
+    return data_set
+
+
 def load_data_set():
     """
     Load a sample data set (From Data Mining: Concepts and Techniques, 3th Edition)
@@ -12,34 +26,140 @@ def load_data_set():
             ['l1', 'l3'], ['l1', 'l2', 'l3', 'l5'], ['l1', 'l2', 'l3']]
     return data_set
 
-def test_apriori(data_set):
+
+def test_apriori(data_set, min_sup = 0.05):
+    start = datetime.now()
     apriori = Apriori(data_set)
-    freq_itemsets = apriori.generate_L(k=3, min_sup=0.2)
-    big_rules_list = apriori.generate_big_rules(min_conf=0.7)
-    for Lk in freq_itemsets:
-        print(Lk)
-    print()
-    print("Big Rules")
-    for item in big_rules_list:
-        print(item[0], "=>", item[1], "conf: ", item[2])
-
-
-def test_fpgrowth(data_set):
-    min_sup = 0.2
-    min_conf = 0.7
-    t = FPGrowth(data_set, min_sup=min_sup, min_conf=min_conf)
-    t.build_fptree()
-    for i in t.freq_itemsets:
-        print(i)
-    print(t.strong_association_rules)
+    apriori.generate_L(min_sup=min_sup)
+    deltatime = datetime.now() - start
+    print("Apriori over")
+    return deltatime.seconds + deltatime.microseconds / 1000000
+    # print("# of freq itemsets:", len(apriori.freq_itemsets))
+    # print(apriori.freq_itemsets)
+
+
+def test_fpgrowth(data_set, min_sup=0.05):
+    start = datetime.now()
+    fp = FPGrowth(data_set, min_sup=min_sup)
+    fp.build_fptree()
+    deltatime = datetime.now() - start
+    print("FP-Growth over")
+    # print("# of freq itemsets:", len(fp.freq_itemsets))
+
+    return deltatime.seconds + deltatime.microseconds / 1000000
+    
+
+def test_ntrans():
+    data_folder = "./data/ntrans/"
+    ntrans_range = range(1, 21, 1)
+    time_apriori = []
+    time_fpgrowth = []
+    
+    for ntrans in ntrans_range:
+        fname = str(ntrans)+".data"
+        print(fname)
+        data_set = data_reader(data_folder+fname)
+
+        time_apriori.append(test_apriori(data_set))
+
+        time_fpgrowth.append(test_fpgrowth(data_set))
+
+    print(time_apriori)
+    print(time_fpgrowth)
+    plt.plot(ntrans_range, time_apriori, label="Apriori")
+    plt.plot(ntrans_range, time_fpgrowth,label="FP-Growth")
+    plt.xlabel("ntrans (k)")
+    plt.ylabel("time (s)")
+    plt.legend()
+    plt.show()
+
+def test_tlen():
+    data_folder = "./data/tlen/"
+    tlen_range = range(1, 21, 1)
+    time_apriori = []
+    time_fpgrowth = []
+    
+    for tlen in tlen_range:
+        fname = str(tlen)+".data"
+        print(fname)
+        data_set = data_reader(data_folder+fname)
+
+        time_apriori.append(test_apriori(data_set))
+
+        time_fpgrowth.append(test_fpgrowth(data_set))
+
+    print(time_apriori)
+    print(time_fpgrowth)
+    plt.plot(tlen_range, time_apriori, label="Apriori")
+    plt.plot(tlen_range, time_fpgrowth,label="FP-Growth")
+    plt.xlabel("tlen")
+    plt.ylabel("time (s)")
+    plt.legend()
+    plt.show()
+
+
+def test_nitems():
+    data_folder = "./data/nitems/"
+    nitems_range = list(np.arange(0.1, 2.1, 0.1))
+    time_apriori = []
+    time_fpgrowth = []
+    
+    for nitems in nitems_range:
+        fname = str(nitems)+".data"
+        print(fname)
+        data_set = data_reader(data_folder+fname)
+
+        time_apriori.append(test_apriori(data_set))
+
+        time_fpgrowth.append(test_fpgrowth(data_set))
+
+    print(time_apriori)
+    print(time_fpgrowth)
+    plt.plot(nitems_range, time_apriori, label="Apriori")
+    plt.plot(nitems_range, time_fpgrowth,label="FP-Growth")
+    plt.xlabel("nitems (k)")
+    plt.ylabel("time (s)")
+    plt.legend()
+    plt.show()
+
+
+def test_minsup():
+    data_file = "./data/base_set.data"
+    data_set = data_reader(data_file)
+    minsup_range = list(np.arange(0.01, 0.21, 0.01))
+    time_apriori = []
+    time_fpgrowth = []
+    
+    for minsup in minsup_range:
+        time_apriori.append(test_apriori(data_set, min_sup=minsup))
+        time_fpgrowth.append(test_fpgrowth(data_set, min_sup=minsup))
+
+    print(time_apriori)
+    print(time_fpgrowth)
+    plt.plot(minsup_range, time_apriori, label="Apriori")
+    plt.plot(minsup_range, time_fpgrowth,label="FP-Growth")
+    plt.xlabel("minsup")
+    plt.ylabel("time (s)")
+    plt.legend()
+    plt.show()
+
+def test_base():
+    data_file = "./data/base_set.data"
+    data_set = data_reader(data_file)
+    # data_set = load_data_set()
+    # print("Apriori-----------------------")
+    # print("Time (s):", test_apriori(data_set))
+
+    print("FP-Growth-----------------------")
+    print("Time (s):", test_fpgrowth(data_set))
 
 
 if __name__ == "__main__":
     """
     Test
     """
-    data_set = load_data_set()
-    test_apriori(data_set)
-    print("FP-Growth-----------------------")
-    test_fpgrowth(data_set)
-    print(len(data_set))
+    test_base()
+    # test_ntrans()
+    # test_tlen()
+    # test_nitems()
+    # test_minsup()