Completed plot pipeline.

Author: gdialektakis

autoencoders/standard_AE.py

@@ -86,7 +86,7 @@ def reconstruct_variables(sess=None, op=None, data=None):
     # run the trained AE for predictions on the test data
     reconstructed_data = sess.run(op, feed_dict={x_input: data})
     print('Reconstructed data shape: {}'.format(reconstructed_data.shape))
-    # We are going to plot the reconstructed data below
+    return reconstructed_data
 
 
 def train(train_model=True, train_data=None, test_data=None):
@@ -143,8 +143,9 @@ def train(train_model=True, train_data=None, test_data=None):
 
             # print("Saved Model Path: {}".format(saved_model_path))
 
-            if recontruct == True:
-                reconstruct_variables(sess=sess, op=decoder_output, data=test_data)
+            if recontruct:
+                reconstructed_data = reconstruct_variables(sess=sess, op=decoder_output, data=test_data)
+                return reconstructed_data
 
         else:
             all_results = os.listdir(results_path)

create_plots.py

@@ -0,0 +1,104 @@
+import os
+import matplotlib.pyplot as plt
+import pandas as pd
+import numpy as np
+
+
+def plot_initial_data(input_data):
+    save_dir = "D:\Desktop\GSoC-ATLAS\initial_data_plots"
+    if not os.path.exists(save_dir):
+        os.makedirs(save_dir)
+
+    variable_list = ['pt_', 'eta_', 'phi_', 'mass_',
+                     'fX', 'fY', 'fZ', 'mJetArea',
+                     'mPileupEnergy', 'mChargedHadronEnergy', 'mNeutralHadronEnergy',
+                     'mPhotonEnergy',
+                     'mElectronEnergy', 'mMuonEnergy', 'mHFHadronEnergy',
+                     'mHFEMEnergy', 'mChargedHadronMultiplicity',
+                     'mNeutralHadronMultiplicity',
+                     'mPhotonMultiplicity', 'mElectronMultiplicity',
+                     'mMuonMultiplicity',
+                     'mHFHadronMultiplicity', 'mHFEMMultiplicity', 'mChargedEmEnergy',
+                     'mChargedMuEnergy', 'mNeutralEmEnergy', 'mChargedMultiplicity',
+                     'mNeutralMultiplicity']
+
+    prefix = 'ak5PFJets_'
+    n_bins = 100
+    save = True  # Option to save figure
+
+    branches = [prefix + 'pt_', prefix + 'eta_', prefix + 'phi_', prefix + 'mass_',
+                prefix + 'fX', prefix + 'fY', prefix + 'fZ', prefix + 'mJetArea',
+                prefix + 'mPileupEnergy', prefix + 'mChargedHadronEnergy', prefix + 'mNeutralHadronEnergy',
+                prefix + 'mPhotonEnergy',
+                prefix + 'mElectronEnergy', prefix + 'mMuonEnergy', prefix + 'mHFHadronEnergy',
+                prefix + 'mHFEMEnergy', prefix + 'mChargedHadronMultiplicity',
+                prefix + 'mNeutralHadronMultiplicity',
+                prefix + 'mPhotonMultiplicity', prefix + 'mElectronMultiplicity',
+                prefix + 'mMuonMultiplicity',
+                prefix + 'mHFHadronMultiplicity', prefix + 'mHFEMMultiplicity', prefix + 'mChargedEmEnergy',
+                prefix + 'mChargedMuEnergy', prefix + 'mNeutralEmEnergy', prefix + 'mChargedMultiplicity',
+                prefix + 'mNeutralMultiplicity']
+
+    for kk in range(0, 28):
+        if branches[kk] == prefix + 'pt_' or branches[kk] == prefix + 'mass_':
+            n_hist_data, bin_edges, _ = plt.hist(input_data[branches[kk]], color='orange', label='Input', alpha=1,
+                                                 bins=n_bins, log=True)
+            plt.xlabel(xlabel=variable_list[kk])
+            plt.ylabel('# of jets')
+        elif branches[kk] == prefix + 'phi_':
+            n_hist_data, bin_edges, _ = plt.hist(input_data[branches[kk]], color='orange', label='Input', alpha=1,
+                                                 bins=50)
+            plt.xlabel(xlabel=variable_list[kk])
+            plt.ylabel('# of jets')
+        else:
+            n_hist_data, bin_edges, _ = plt.hist(input_data[branches[kk]], color='orange', label='Input', alpha=1,
+                                                 bins=n_bins)
+            plt.xlabel(xlabel=variable_list[kk])
+            plt.ylabel('# of jets')
+        plt.show()
+        if save:
+            plt.savefig(os.path.join(save_dir, variable_list[kk] + '.png'))
+
+
+def plot_test_pred_data(test_data, predicted_data):
+    save_dir = "D:\Desktop\GSoC-ATLAS\AE_plots"
+    if not os.path.exists(save_dir):
+        os.makedirs(save_dir)
+
+    variable_list = ['pt_', 'eta_', 'phi_', 'mass_',
+                     'fX', 'fY', 'fZ', 'mJetArea',
+                     'mPileupEnergy', 'mChargedHadronEnergy', 'mNeutralHadronEnergy',
+                     'mPhotonEnergy',
+                     'mElectronEnergy', 'mMuonEnergy', 'mHFHadronEnergy',
+                     'mHFEMEnergy', 'mChargedHadronMultiplicity',
+                     'mNeutralHadronMultiplicity',
+                     'mPhotonMultiplicity', 'mElectronMultiplicity',
+                     'mMuonMultiplicity',
+                     'mHFHadronMultiplicity', 'mHFEMMultiplicity', 'mChargedEmEnergy',
+                     'mChargedMuEnergy', 'mNeutralEmEnergy', 'mChargedMultiplicity',
+                     'mNeutralMultiplicity']
+
+    colors = ['pink', 'green']
+    prefix = 'ak5PFJets_'
+    n_bins = 100
+    save = True  # Option to save figure
+
+    #predicted_data = predicted_data.detach().numpy()
+    test_data = test_data.values
+
+    # plot the input data along with the reconstructed from the AE
+    for kk in np.arange(28):
+        plt.figure()
+        n_hist_data, bin_edges, _ = plt.hist(test_data[:, kk], color=colors[1], label='Input', alpha=1, bins=n_bins)
+        n_hist_pred, _, _ = plt.hist(predicted_data[:, kk], color=colors[0], label='Output', alpha=0.8, bins=bin_edges)
+        plt.suptitle(variable_list[kk])
+        plt.xlabel(xlabel=variable_list[kk])
+        plt.ylabel('Number of jets')
+        # ms.sciy()
+        plt.yscale('log')
+        plt.legend()
+        if save:
+            plt.savefig(os.path.join(save_dir, variable_list[kk] + '.png'))
+
+
+# plot(data_df = pd.read_csv('27D_openCMS_preprocessed_data.csv'))

data_loader.py

@@ -31,7 +31,7 @@ def load_cms_data(filename="open_cms_data.root"):
          prefix + 'm_specific.mChargedEmEnergy', prefix + 'm_specific.mChargedMuEnergy', prefix + 'm_specific.mNeutralEmEnergy',
          prefix + 'm_specific.mChargedMultiplicity', prefix + 'm_specific.mNeutralMultiplicity'],       library="pd")
 
-    prefix2 = 'ak5PFJets.'
+    prefix2 = 'ak5PFJets_'
     # Rename the column names to be shorter
     dataframe.columns = [prefix2 + 'pt_', prefix2 + 'eta_', prefix2 + 'phi_', prefix2 + 'mass_',
                          prefix2 + 'fX', prefix2 + 'fY', prefix2 + 'fZ', prefix2 + 'mJetArea', prefix2 + 'mPileupEnergy',
@@ -45,6 +45,7 @@ def load_cms_data(filename="open_cms_data.root"):
 
 
     print("\nDataframe:")
+    dataframe.sort_values(by=[prefix2 + 'pt_'])
     print(dataframe.head())
     dataframe.to_csv('27D_openCMS_data.csv')
     return dataframe

data_preprocessing.py

@@ -4,11 +4,14 @@
 from sklearn.utils import shuffle
 
 
-def preprocess(filename='27D_openCMS_data.csv'):
-    data_df = pd.read_csv(filename)
+def preprocess(data_df):
+
     data_df = data_df.drop(['entry', 'subentry'], axis=1)
-    # data_df = data_df.astype('float32')
+    # data_df = data_df.sort_values(by=['ak5PFJets_pt_'])
 
+    # filter out jets having pT > 8 TeV
+    data_df = data_df[data_df.ak5PFJets_pt_ < 8000]
+    
     # Standardize our data using Standard Scalar from sklearn
     data_df[data_df.columns] = StandardScaler().fit_transform(data_df)
     print('Normalized data:')
@@ -27,6 +30,8 @@ def preprocess(filename='27D_openCMS_data.csv'):
 
     data_df.to_csv('27D_openCMS_preprocessed_data.csv')
 
-    return train_set, test_set
+    return data_df, train_set, test_set
+
 
+#data_df, train_set, test_set = preprocess(data_df=pd.read_csv('27D_openCMS_data.csv'))
 

main.py

@@ -1,8 +1,13 @@
 from data_preprocessing import preprocess
 from autoencoders import standard_AE
+from create_plots import plot_initial_data, plot_test_pred_data
+import pandas as pd
 from data_loader import load_cms_data
 
 #cms_data_df = load_cms_data(filename="open_cms_data.root")
-train_data, test_data = preprocess()
+data_df = pd.read_csv('27D_openCMS_data.csv')
 
-standard_AE.train(train_data=train_data, test_data=test_data)
+data_df, train_data, test_data = preprocess(data_df)
+#plot_initial_data(data_df)
+reconstructed_data = standard_AE.train(train_data=train_data, test_data=test_data)
+plot_test_pred_data(test_data, reconstructed_data)