Added RMSE and Residual metrics + Code optimization

Author: gdialektakis

autoencoders/autoencoder.py

@@ -64,15 +64,14 @@ def train(self, test_set, epochs):
 
         start = time.perf_counter()  # Starts timer
         # train our autoencoder
-        learn.fit_one_cycle(epochs, 0.014, cbs=[ShowGraphCallback()])
+        learn.fit_one_cycle(epochs, lr_min, cbs=[ShowGraphCallback()])
         end = time.perf_counter()  # Ends timer
         delta_t = end - start
         print('Training took', delta_t, 'seconds')
 
         plt.figure()
         recorder.plot_loss()
         plt.show()
-        print(learn.validate())
 
         data = torch.tensor(test_set.values, dtype=torch.float)
 

create_plots.py

@@ -6,45 +6,71 @@
 #import arviz as az
 
 
-def plot_initial_data(input_data, normalized=False):
+def plot_initial_data(input_data, num_variables, normalized=False):
     input_data = input_data.sort_values(by=['ak5PFJets_pt_'])
 
     if normalized:
         save_dir = "D:\Desktop\GSoC-ATLAS\preprocessed_data_plots"
     else:
         save_dir = "D:\Desktop\GSoC-ATLAS\initial_data_plots"
+
+    prefix = 'ak5PFJets_'
+
+    if num_variables == 24:
+        save_dir = "D:\Desktop\GSoC-ATLAS\preprocessed_data_plots\d24"
+
+        variable_list = ['pt_', 'eta_', 'phi_', 'mass_', 'mJetArea',
+                         'mChargedHadronEnergy', 'mNeutralHadronEnergy',
+                         'mPhotonEnergy',
+                         'mElectronEnergy', 'mMuonEnergy', 'mHFHadronEnergy',
+                         'mHFEMEnergy', 'mChargedHadronMultiplicity',
+                         'mNeutralHadronMultiplicity',
+                         'mPhotonMultiplicity', 'mElectronMultiplicity',
+                         'mMuonMultiplicity',
+                         'mHFHadronMultiplicity', 'mHFEMMultiplicity', 'mChargedEmEnergy',
+                         'mChargedMuEnergy', 'mNeutralEmEnergy', 'mChargedMultiplicity',
+                         'mNeutralMultiplicity']
+
+        branches = [prefix + 'pt_', prefix + 'eta_', prefix + 'phi_', prefix + 'mass_',
+                    prefix + 'mJetArea', prefix + 'mChargedHadronEnergy', prefix + 'mNeutralHadronEnergy',
+                    prefix + 'mPhotonEnergy',
+                    prefix + 'mElectronEnergy', prefix + 'mMuonEnergy', prefix + 'mHFHadronEnergy',
+                    prefix + 'mHFEMEnergy', prefix + 'mChargedHadronMultiplicity',
+                    prefix + 'mNeutralHadronMultiplicity',
+                    prefix + 'mPhotonMultiplicity', prefix + 'mElectronMultiplicity',
+                    prefix + 'mMuonMultiplicity',
+                    prefix + 'mHFHadronMultiplicity', prefix + 'mHFEMMultiplicity', prefix + 'mChargedEmEnergy',
+                    prefix + 'mChargedMuEnergy', prefix + 'mNeutralEmEnergy', prefix + 'mChargedMultiplicity',
+                    prefix + 'mNeutralMultiplicity']
+    else:
+        save_dir = "D:\Desktop\GSoC-ATLAS\preprocessed_data_plots\d19"
+
+        variable_list = ['pt_', 'eta_', 'phi_', 'mass_', 'mJetArea',
+                         'mChargedHadronEnergy', 'mNeutralHadronEnergy',
+                         'mPhotonEnergy', 'mHFHadronEnergy',
+                         'mHFEMEnergy', 'mChargedHadronMultiplicity',
+                         'mNeutralHadronMultiplicity',
+                         'mPhotonMultiplicity', 'mElectronMultiplicity',
+                         'mHFHadronMultiplicity', 'mHFEMMultiplicity', 'mNeutralEmEnergy', 'mChargedMultiplicity',
+                         'mNeutralMultiplicity']
+
+        branches = [prefix + 'pt_', prefix + 'eta_', prefix + 'phi_', prefix + 'mass_',
+                    prefix + 'mJetArea', prefix + 'mChargedHadronEnergy', prefix + 'mNeutralHadronEnergy',
+                    prefix + 'mPhotonEnergy', prefix + 'mHFHadronEnergy',
+                    prefix + 'mHFEMEnergy', prefix + 'mChargedHadronMultiplicity',
+                    prefix + 'mNeutralHadronMultiplicity',
+                    prefix + 'mPhotonMultiplicity', prefix + 'mElectronMultiplicity',
+                    prefix + 'mHFHadronMultiplicity', prefix + 'mHFEMMultiplicity',
+                    prefix + 'mNeutralEmEnergy', prefix + 'mChargedMultiplicity',
+                    prefix + 'mNeutralMultiplicity']
+
     if not os.path.exists(save_dir):
         os.makedirs(save_dir)
 
-    variable_list = ['pt_', 'eta_', 'phi_', 'mass_', 'mJetArea',
-                     'mChargedHadronEnergy', 'mNeutralHadronEnergy',
-                     'mPhotonEnergy',
-                     'mElectronEnergy', 'mMuonEnergy', 'mHFHadronEnergy',
-                     'mHFEMEnergy', 'mChargedHadronMultiplicity',
-                     'mNeutralHadronMultiplicity',
-                     'mPhotonMultiplicity', 'mElectronMultiplicity',
-                     'mMuonMultiplicity',
-                     'mHFHadronMultiplicity', 'mHFEMMultiplicity', 'mChargedEmEnergy',
-                     'mChargedMuEnergy', 'mNeutralEmEnergy', 'mChargedMultiplicity',
-                     'mNeutralMultiplicity']
-
-    prefix = 'ak5PFJets_'
     n_bins = 100
     save = True  # Option to save figure
 
-    branches = [prefix + 'pt_', prefix + 'eta_', prefix + 'phi_', prefix + 'mass_',
-                prefix + 'mJetArea', prefix + 'mChargedHadronEnergy', prefix + 'mNeutralHadronEnergy',
-                prefix + 'mPhotonEnergy',
-                prefix + 'mElectronEnergy', prefix + 'mMuonEnergy', prefix + 'mHFHadronEnergy',
-                prefix + 'mHFEMEnergy', prefix + 'mChargedHadronMultiplicity',
-                prefix + 'mNeutralHadronMultiplicity',
-                prefix + 'mPhotonMultiplicity', prefix + 'mElectronMultiplicity',
-                prefix + 'mMuonMultiplicity',
-                prefix + 'mHFHadronMultiplicity', prefix + 'mHFEMMultiplicity', prefix + 'mChargedEmEnergy',
-                prefix + 'mChargedMuEnergy', prefix + 'mNeutralEmEnergy', prefix + 'mChargedMultiplicity',
-                prefix + 'mNeutralMultiplicity']
-
-    for kk in range(0, 24):
+    for kk in range(0, num_variables):
         if branches[kk] == prefix + 'pt_' or branches[kk] == prefix + 'mass_':
             n_hist_data, bin_edges, _ = plt.hist(input_data[branches[kk]], color='orange', label='Input', alpha=1,
                                                  bins=n_bins, log=True)
@@ -73,12 +99,12 @@ def plot_initial_data(input_data, normalized=False):
 
 
 def plot_test_pred_data(test_data, predicted_data, num_variables, vae=False):
-    if vae:
-        save_dir = "D:\Desktop\GSoC-ATLAS\VAE_plots"
-
 
     if num_variables == 24:
-        save_dir = "D:\Desktop\GSoC-ATLAS\AE_plots\d24"
+        if vae:
+            save_dir = "D:\Desktop\GSoC-ATLAS\VAE_plots\d24"
+        else:
+            save_dir = "D:\Desktop\GSoC-ATLAS\AE_plots\d24"
 
         variable_list = ['pt_', 'eta_', 'phi_', 'mass_', 'mJetArea',
                          'mChargedHadronEnergy', 'mNeutralHadronEnergy',
@@ -92,7 +118,11 @@ def plot_test_pred_data(test_data, predicted_data, num_variables, vae=False):
                          'mChargedMuEnergy', 'mNeutralEmEnergy', 'mChargedMultiplicity',
                          'mNeutralMultiplicity']
     else:
-        save_dir = "D:\Desktop\GSoC-ATLAS\AE_plots\d19"
+        if vae:
+            save_dir = "D:\Desktop\GSoC-ATLAS\VAE_plots\d19"
+        else:
+            save_dir = "D:\Desktop\GSoC-ATLAS\AE_plots\d19"
+
         variable_list = ['pt_', 'eta_', 'phi_', 'mass_', 'mJetArea',
                          'mChargedHadronEnergy', 'mNeutralHadronEnergy',
                          'mPhotonEnergy', 'mHFHadronEnergy',
@@ -106,7 +136,6 @@ def plot_test_pred_data(test_data, predicted_data, num_variables, vae=False):
         os.makedirs(save_dir)
 
     colors = ['pink', 'green']
-    prefix = 'ak5PFJets_'
     n_bins = 100
     save = True  # Option to save figure
 
@@ -127,8 +156,6 @@ def plot_test_pred_data(test_data, predicted_data, num_variables, vae=False):
             plt.savefig(os.path.join(save_dir, variable_list[kk] + '.png'))
 
 
-# plot(data_df = pd.read_csv('27D_openCMS_preprocessed_data.csv'))
-
 def plot_4D_data(test_data, predicted_data):
 
     save_dir = "D:\Desktop\GSoC-ATLAS\AE_4D_plots"

data_processing.py

@@ -9,7 +9,7 @@
 """
 
 
-def preprocess_28D(data_df, num_variables, min_max_all):
+def preprocess_28D(data_df, num_variables, custom_norm):
     data_df = data_df.drop(['entry', 'subentry'], axis=1)
     data_df = data_df.sort_values(by=['ak5PFJets_pt_'])
 
@@ -29,7 +29,7 @@ def preprocess_28D(data_df, num_variables, min_max_all):
     # data_df[data_df.columns] = scaler.fit_transform(data_df)
     min_max_scaler = MinMaxScaler()
 
-    if min_max_all:
+    if not custom_norm:
         # Normalize all variables in the range (0, 1) using MinMax Scaler from sklearn
         data_df[data_df.columns] = min_max_scaler.fit_transform(data_df)
     else:

evaluate.py

@@ -0,0 +1,18 @@
+import numpy as np
+from sklearn.metrics import mean_squared_error
+
+
+def evaluate_model(y_true, y_predicted):
+    # MSE
+    mse = mean_squared_error(y_true, y_predicted)
+    # RMSE
+    rmse = mean_squared_error(y_true, y_predicted, squared=False)
+    # Residuals of the different variables [(output-input)/input]
+    residuals = np.absolute((y_true - y_predicted))
+    residuals = residuals.sum()/residuals.size
+
+    print("---------------------------")
+    print("MSE: {:.6f}".format(mse))
+    print("RMSE: {:.6f}".format(rmse))
+    print("Residual: {:.6f}".format(residuals))
+    print("---------------------------")

main.py

@@ -2,6 +2,7 @@
 import autoencoders.autoencoder as ae
 import autoencoders.variational_autoencoder as vae
 from create_plots import plot_initial_data, plot_test_pred_data, plot_4D_data, plot_residuals, correlation_plots
+from evaluate import evaluate_model
 import pandas as pd
 from data_loader import load_cms_data
 
@@ -10,22 +11,29 @@
     use_vae = False
     # cms_data_df = load_cms_data(filename="open_cms_data.root")
     data_df = pd.read_csv('27D_openCMS_data.csv')
-
+    custom_norm = False
+    num_of_variables = 24
     # Plot the original data
     #plot_initial_data(input_data=data_df)
 
     if openCMS_data:
-        data_df, train_data, test_data, scaler = preprocess_28D(data_df=data_df, num_variables=24, min_max_all=False)
-        data_df = data_df.sort_values(by=['ak5PFJets_pt_'])
+
+        # Preprocess data
+        data_df, train_data, test_data, scaler = preprocess_28D(data_df=data_df, num_variables=num_of_variables, custom_norm=custom_norm)
         # Plot preprocessed data
-        plot_initial_data(input_data=data_df, normalized=True)
+        #plot_initial_data(input_data=data_df, num_variables=num_of_variables, normalized=True)
 
-        # Run the Autoencoder and obtain the reconstructed data
-        standard_ae = ae.Autoencoder(train_data, test_data, num_variables=24)
+        # Initial the Autoencoder
+        standard_ae = ae.Autoencoder(train_data, test_data, num_variables=num_of_variables)
 
+        # Train the standard Autoencoder and obtain the reconstructions
         test_data, reconstructed_data = standard_ae.train(test_data, epochs=30)
 
-        plot_test_pred_data(test_data, reconstructed_data, num_variables=24)
+        # Evaluate the reconstructions of the network based on various metrics
+        evaluate_model(y_true=test_data, y_predicted=reconstructed_data)
+
+        # Plot the reconstructed along with the initial data
+        plot_test_pred_data(test_data, reconstructed_data, num_variables=num_of_variables)
 
     else:
         # Dark machines data