streamMiningMuhammadAbdullahMulkana.py

# -*- coding: utf-8 -*-
"""GE5-func.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/1h11JSwaAN3j8qt8mocjP9pKlITq6VCz8
"""

! pip install -U scikit-multiflow

#References
# [1] https://scikit-multiflow.readthedocs.io/en/stable/api/generated/skmultiflow.data.HyperplaneGenerator.html 
# [2] https://scikit-multiflow.readthedocs.io/en/stable/user-guide/streams-intro.html
# [3] https://scikit-multiflow.readthedocs.io/en/stable/user-guide/streams-intro.html
# [4] https://scikit-multiflow.readthedocs.io/en/stable/api/generated/skmultiflow.data.FileStream.html
# [5] https://scikit-multiflow.readthedocs.io/en/stable/api/generated/skmultiflow.lazy.KNNClassifier.html 
# [6] https://scikit-multiflow.readthedocs.io/en/stable/api/generated/skmultiflow.trees.HoeffdingTreeClassifier.html
# [7] https://scikit-multiflow.readthedocs.io/en/latest/api/generated/skmultiflow.bayes.NaiveBayes.html 
# [8] https://scikit-multiflow.readthedocs.io/en/stable/api/generated/skmultiflow.meta.DynamicWeightedMajorityClassifier.html#skmultiflow.meta.DynamicWeightedMajorityClassifier

# Note: the interleaved-test-then-train method for online and batch modes is implemented using code given as example in 
# the documentation for KNN [5], HT [6], and NB [7]. The code is modified to make it more generalizable.

import skmultiflow as mf
import numpy as np
import pandas as pd
from skmultiflow.data.file_stream import FileStream
from skmultiflow.transform import WindowedStandardScaler
import matplotlib.pyplot as plt
import time

HG_a = mf.data.HyperplaneGenerator(n_features=10, n_drift_features=2, noise_percentage=0.1) # [1]
x_a, y_a = HG_a.next_sample(20000) # [2]

HG_b = mf.data.HyperplaneGenerator(n_features=10, n_drift_features=2, noise_percentage=0.3) # [1]
x_b, y_b = HG_b.next_sample(20000)  # [2]

HG_c = mf.data.HyperplaneGenerator(n_features=10, n_drift_features=5, noise_percentage=0.1) # [1]
x_c, y_c = HG_c.next_sample(20000)  # [2]

HG_d = mf.data.HyperplaneGenerator(n_features=10, n_drift_features=5, noise_percentage=0.3) # [1]
x_d, y_d = HG_d.next_sample(20000)  # [2]

ds_a = pd.DataFrame(np.hstack((x_a,np.array([y_a]).T))) # [3]
ds_b = pd.DataFrame(np.hstack((x_b,np.array([y_b]).T))) # [3]
ds_c = pd.DataFrame(np.hstack((x_c,np.array([y_c]).T))) # [3]
ds_d = pd.DataFrame(np.hstack((x_d,np.array([y_d]).T))) # [3]

ds_a.to_csv("Hyperplane Dataset 10_2.csv", index=False) # [3]
ds_b.to_csv("Hyperplane Dataset 30_2.csv", index=False) # [3]
ds_c.to_csv("Hyperplane Dataset 10_5.csv", index=False) # [3]
ds_d.to_csv("Hyperplane Dataset 30_5.csv", index=False) # [3]

def model(classifier, batch_size):
  print("Batch Size: {}".format(batch_size))
  start = time.time()
  stream_a = FileStream("Hyperplane Dataset 10_2.csv")# [4]
  stream_b = FileStream("Hyperplane Dataset 30_2.csv")# [4]
  stream_c = FileStream("Hyperplane Dataset 10_5.csv")# [4]
  stream_d = FileStream("Hyperplane Dataset 30_5.csv")# [4]
  streams = [stream_a, stream_b, stream_c, stream_d]
  accuracy = []
  inter_results = np.zeros([4, int(20000/batch_size)])
  for i in range(len(streams)): # Code inside the for loop is inspired by the examples given in [5], [6], and [7]
    stream = streams[i]
    n_samples = 0
    corrects = 0
    dummy = 0
    while stream.has_more_samples():
        X, y = stream.next_sample(batch_size)
        my_pred = classifier.predict(X)
        corrects += sum(y == my_pred)
        classifier = classifier.partial_fit(X, y)
        n_samples += batch_size
        inter_results[i][dummy] = corrects/n_samples
        dummy += 1        
    accuracy.append(corrects/n_samples)
    print("Classifier's performance: {}".format(corrects/n_samples))
  runtime = time.time() - start
  print("Runtime: {}s".format(runtime))
  return accuracy, inter_results, runtime

def ensemble(batch_size):
  print("Batch Size: {}".format(batch_size))
  start = time.time()
  stream_a = FileStream("Hyperplane Dataset 10_2.csv")# [4]
  stream_b = FileStream("Hyperplane Dataset 30_2.csv")# [4]
  stream_c = FileStream("Hyperplane Dataset 10_5.csv")# [4]
  stream_d = FileStream("Hyperplane Dataset 30_5.csv")# [4]
  streams = [stream_a, stream_b, stream_c, stream_d]
  start = time.time()
  accuracy = []
  inter_results = np.zeros([4, int(20000/batch_size)])
  for i in range(len(streams)):# Code inside the for loop is inspired by the examples given in [5], [6], and [7].
    stream = streams[i]
    knn_mv = mf.lazy.KNNClassifier(n_neighbors=141, max_window_size = 2000)
    nb_mv = mf.bayes.NaiveBayes()
    ht_mv = mf.trees.HoeffdingTreeClassifier()
    n_samples = 0
    corrects = 0
    dummy = 0
    while stream.has_more_samples():
        X, y = stream.next_sample(batch_size)
        knn_pred = knn_mv.predict(X)
        ht_pred = ht_mv.predict(X)
        nb_pred = nb_mv.predict(X)
        pred_arr = knn_pred+ht_pred+nb_pred
        pred_arr[pred_arr <2 ] = 0
        pred_arr[pred_arr >= 2] = 1
        corrects += sum(pred_arr == y)
        knn_mv = knn_mv.partial_fit(X, y)
        nb_mv = nb_mv.partial_fit(X, y)
        ht_mv = ht_mv.partial_fit(X, y)

        n_samples += batch_size
        inter_results[i][dummy] = corrects/n_samples
        dummy += 1    
    accuracy.append(corrects/n_samples)
    print("Ensamble Classifier's performance: {}".format(corrects/n_samples))
  runtime = time.time() - start
  print("Runtime: {}s".format(runtime))
  return accuracy, inter_results, runtime

def weighted_ensemble(batch_size, w1, w2, w3):
  print("Batch Size: {}".format(batch_size))
  start = time.time()
  stream_a = FileStream("Hyperplane Dataset 10_2.csv")# [4]
  stream_b = FileStream("Hyperplane Dataset 30_2.csv")# [4]
  stream_c = FileStream("Hyperplane Dataset 10_5.csv")# [4]
  stream_d = FileStream("Hyperplane Dataset 30_5.csv")# [4]
  streams = [stream_a, stream_b, stream_c, stream_d]
  start = time.time()
  accuracy = []
  inter_results = np.zeros([4, int(20000/batch_size)])
  for i in range(len(streams)):# Code inside the for loop is inspired by the examples given in [5], [6], and [7].
    stream = streams[i]
    knn_wmv = mf.lazy.KNNClassifier(n_neighbors=141, max_window_size = 2000)
    nb_wmv = mf.bayes.NaiveBayes()
    ht_wmv = mf.trees.HoeffdingTreeClassifier()
    n_samples = 0
    corrects = 0
    dummy = 0
    while stream.has_more_samples():
        X, y = stream.next_sample(batch_size)
        knn_pred = knn_wmv.predict_proba(X)
        ht_pred = ht_wmv.predict_proba(X)
        nb_pred = nb_wmv.predict_proba(X)

        prob_0 = (w1*knn_pred[:,0]+w2*ht_pred[:,0]+w3*nb_pred[:,0])
        prob_1 = (w1*knn_pred[:,-1]+w2*ht_pred[:,-1]+w3*nb_pred[:,-1])

        pred_arr = prob_0 - prob_1
        pred_arr[pred_arr > 0] = 0
        pred_arr[pred_arr < 0] = 1
        pred_arr[pred_arr == 0] = np.random.randint(0, 1)

        corrects += sum(pred_arr == y)
        knn_wmv = knn_wmv.partial_fit(X, y)
        nb_wmv = nb_wmv.partial_fit(X, y)
        ht_wmv = ht_wmv.partial_fit(X, y)
        n_samples += batch_size
        inter_results[i][dummy] = corrects/n_samples
        dummy += 1    
    accuracy.append(corrects/n_samples)
    print("Ensamble Classifier's performance: {}".format(corrects/n_samples))
  runtime = time.time() - start
  print("Runtime: {}s".format(runtime))
  return accuracy, inter_results, runtime

batch_arr = [1,100,1000]
knn_batch_acc = []
knn_batch_inter = []
knn_batch_time = []
print('KNN Classifier')
for batch_size in batch_arr:
  knn = mf.lazy.KNNClassifier(n_neighbors=141, max_window_size=2000)# [5] 
  knn_acc, knn_inter, rt = model(knn, batch_size)
  knn_batch_acc.append(knn_acc)
  knn_batch_inter.append(knn_inter)
  knn_batch_time.append(rt)

plot_labels = ['10_2', '30_2', '10_5', '30_5']
b1 = np.arange(0,20000)
b2 = np.linspace(0,20000,num=200)
b3 = np.linspace(0,20000,num=20)


for i in range(4):
  plt.plot(b1,knn_batch_inter[0][i], label = 'batch_size = 1')
  plt.plot(b2, knn_batch_inter[1][i], label = 'batch_size = 100')
  plt.plot(b3, knn_batch_inter[2][i], label = 'batch_size = 1000')
  plt.title('KNN plot for dataset ' + plot_labels[i])
  plt.legend()
  plt.show()

batch_arr = [1,100,1000]
ht_batch_acc = []
ht_batch_inter = []
ht_batch_time = []
print('Hoeffding Tree Classifier')
for batch_size in batch_arr:
  ht =  mf.trees.HoeffdingTreeClassifier()# [6]
  ht_acc, ht_inter, rt = model(ht, batch_size)
  ht_batch_acc.append(ht_acc)
  ht_batch_inter.append(ht_inter)
  ht_batch_time.append(rt)

plot_labels = ['10_2', '30_2', '10_5', '30_5']
b1 = np.arange(0,20000)
b2 = np.linspace(0,20000,num=200)
b3 = np.linspace(0,20000,num=20)


for i in range(4):
  plt.plot(b1, ht_batch_inter[0][i], label = 'batch_size = 1')
  plt.plot(b2, ht_batch_inter[1][i], label = 'batch_size = 100')
  plt.plot(b3, ht_batch_inter[2][i], label = 'batch_size = 1000')
  plt.title('HT plot for dataset ' + plot_labels[i])
  plt.legend()
  plt.show()

nb_batch_acc = []
nb_batch_inter = []
nb_batch_time = []
print('Naive Bayes Classifier')
for batch_size in batch_arr:
  nb =  mf.bayes.NaiveBayes()# [7]
  nb_acc, nb_inter, rt = model(nb, batch_size)
  nb_batch_acc.append(nb_acc)
  nb_batch_inter.append(nb_inter)
  nb_batch_time.append(rt)

plot_labels = ['10_2', '30_2', '10_5', '30_5']
b1 = np.arange(0,20000)
b2 = np.linspace(0,20000,num=200)
b3 = np.linspace(0,20000,num=20)


for i in range(4):
  plt.plot(b1, nb_batch_inter[0][i], label = 'batch_size = 1')
  plt.plot(b2, nb_batch_inter[1][i], label = 'batch_size = 100')
  plt.plot(b3, nb_batch_inter[2][i], label = 'batch_size = 1000')
  plt.title('NB plot for dataset ' + plot_labels[i])
  plt.legend()
  plt.show()

"""#Ensemble"""

batch_arr = [1,100,1000]
mv_batch_acc = []
mv_batch_inter = []
mv_batch_time = []
print('Ensemble')
for batch_size in batch_arr:
  mv_acc, mv_inter, rt = ensemble(batch_size)
  mv_batch_acc.append(mv_acc)
  mv_batch_inter.append(mv_inter)
  mv_batch_time.append(rt)

plot_labels = ['10_2', '30_2', '10_5', '30_5']
b1 = np.arange(0,20000)
b2 = np.linspace(0,20000,num=200)
b3 = np.linspace(0,20000,num=20)


for i in range(4):
  plt.plot(b1, mv_batch_inter[0][i], label = 'batch_size = 1')
  plt.plot(b2, mv_batch_inter[1][i], label = 'batch_size = 100')
  plt.plot(b3, mv_batch_inter[2][i], label = 'batch_size = 1000')
  plt.title('Majority Voting plot for dataset ' + plot_labels[i])
  plt.legend()
  plt.show()

batch_arr = [1,100,1000]
wmv_batch_acc = []
wmv_batch_inter = []
wmv_batch_time = []
print('Weighted Ensemble')
for batch_size in batch_arr:
  wmv_acc, wmv_inter, rt = weighted_ensemble(batch_size, 10,2,100)
  wmv_batch_acc.append(wmv_acc)
  wmv_batch_inter.append(wmv_inter)
  wmv_batch_time.append(rt)

plot_labels = ['10_2', '30_2', '10_5', '30_5']
b1 = np.arange(0,20000)
b2 = np.linspace(0,20000,num=200)
b3 = np.linspace(0,20000,num=20)


for i in range(4):
  plt.plot(b1, wmv_batch_inter[0][i], label = 'batch_size = 1')
  plt.plot(b2, wmv_batch_inter[1][i], label = 'batch_size = 100')
  plt.plot(b3, wmv_batch_inter[2][i], label = 'batch_size = 1000')
  plt.title('Majority Voting plot for dataset ' + plot_labels[i])
  plt.legend()
  plt.show()

"""#increasing Accuracy"""

classifier = mf.meta.DynamicWeightedMajorityClassifier()#[8]
batch_size = 1
print("Batch Size: {}".format(batch_size))
start = time.time()
stream_a = FileStream("Hyperplane Dataset 10_2.csv")# [4]
stream_b = FileStream("Hyperplane Dataset 30_2.csv")# [4]
stream_c = FileStream("Hyperplane Dataset 10_5.csv")# [4]
stream_d = FileStream("Hyperplane Dataset 30_5.csv")# [4]
streams = [stream_a, stream_b, stream_c, stream_d]
accuracy = []
inter_results = np.zeros([4, int(20000/batch_size)])
for i in range(len(streams)):# Code inside the for loop is inspired by the examples given in [5], [6], [7], and [8].
  stream = streams[i]
  n_samples = 0
  corrects = 0
  dummy = 0
  while stream.has_more_samples():
      X, y = stream.next_sample(batch_size)
      my_pred = classifier.predict(X)
      corrects += sum(y == my_pred)
      classifier = classifier.partial_fit(X, y)
      n_samples += batch_size
      inter_results[i][dummy] = corrects/n_samples
      dummy += 1        
  accuracy.append(corrects/n_samples)
  print("Classifier's performance: {}".format(corrects/n_samples))
runtime = time.time() - start
print("Runtime: {}s".format(runtime))

knn_mean = np.mean(knn_batch_acc, axis = 1)
ht_mean = np.mean(ht_batch_acc, axis = 1)
nb_mean = np.mean(nb_batch_acc, axis = 1)
mv_mean = np.mean(mv_batch_acc, axis = 1)
wmv_mean = np.mean(wmv_batch_acc, axis = 1)

plt.figure(figsize=(12, 8))
plt.plot(batch_arr, knn_mean, label = 'KNN')
plt.plot(batch_arr,ht_mean, label = 'HT')
plt.plot(batch_arr,nb_mean, label = 'NB')
plt.plot(batch_arr,mv_mean, label = 'MV')
plt.plot(batch_arr,wmv_mean, label = 'WMV')
plt.legend()
# plt.xticks([1, 100, 1000])
plt.title('Mean accuracies vs. batch size')
plt.xlabel('Batch Size')
plt.ylabel('Classification accuracy')
plt.show()