finish training

Author: pro1code1hack

clickhouse/utils.py

@@ -1,6 +1,6 @@
 import base64
 import pickle
-from client import clickhouse_client
+from clickhouse.client import clickhouse_client
 
 
 """

data/load_data.py

@@ -1,6 +1,8 @@
 import pandas as pd
 import numpy as np
 from constants import columns, date_col, agg_cols
+from clickhouse.client import clickhouse_client
+
 import warnings
 warnings.filterwarnings("ignore")
 
@@ -171,14 +173,14 @@ def set_target_columns(df):
 def remove_date_col(df):
     """Drop the date columun"""
     df.drop([date_col],axis=1,inplace=True)
-
+    return df
 
 def get_cols_withohut_pid(df):
     """Return the columns EXCEPT OF PID"""
     cols=df.drop("pid",axis=1).columns
-    return cols
+    return df, cols
 
-def create_target_traffic_by_target_columns(target_columns):    # TODO take the data from the db
+def create_target_traffic_by_target_columns(df, target_columns):
     """Extract the traffic for next hours"""
     next_hrs = []
     for hr in [1,4,8,12,24,72,168]:         # TODO dynamical value, as well dynamical for the database
@@ -188,25 +190,26 @@ def create_target_traffic_by_target_columns(target_columns):    # TODO take the
     return next_hrs
 
 
-#Pre-processing
-df = read_data_csv()
-df = sort_df_by_date_col(date_col, df)
-df = convert_df_to_datetime(df)
-df = filter_df_by_specific_date(df, time_delta_years=1)
-df = filter_df_with_most_frequent_pid(df)
-df = replace_null_values(df)
-df, cat_features = categorize_features(df)
-df = extract_date_components(df, date_col)
-df = add_traffic_table(df)
-df = convert_cat_features_to_dummies(df, cat_features) 
-df = combine_all_pids(df, date_col, agg_cols)
-
-# Setting data fro predictions
-target_columns = set_target_columns(df)
-remove_date_col(df)
-cols = get_cols_withohut_pid(df)
-next_hrs = create_target_traffic_by_target_columns(target_columns)
-
-# Clear N/A
-df= df.dropna()
-
+def pre_process_data():
+    #Pre-processing
+    df = read_data_csv()
+    df = sort_df_by_date_col(date_col, df)
+    df = convert_df_to_datetime(df)
+    df = filter_df_by_specific_date(df, time_delta_years=1)
+    df = filter_df_with_most_frequent_pid(df)
+    df = replace_null_values(df)
+    df, cat_features = categorize_features(df)
+    df = extract_date_components(df, date_col)
+    df = add_traffic_table(df)
+    df = convert_cat_features_to_dummies(df, cat_features) 
+    df = combine_all_pids(df, date_col, agg_cols)
+
+    # Setting data fro predictions
+    target_columns = set_target_columns(df)
+    df = remove_date_col(df)
+    df, cols = get_cols_withohut_pid(df)
+    next_hrs = create_target_traffic_by_target_columns(df, target_columns)
+
+    # Clear N/A
+    df= df.dropna()
+    return df, cat_features, cols, next_hrs

scripts/run_training.py

@@ -3,11 +3,32 @@
 import os
 sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
 
-from data.load_data import df, cols, next_hrs
+from clickhouse.client import clickhouse_client
+from clickhouse.utils import serialize_model
+from data.load_data import pre_process_data 
 from models.train_model import train_model, save_model 
 from datetime import datetime
 
 def train():
-    """Celery task which is called for a model training"""
+    """Celery task which is called for a model training
+        - Gets data from ``load_data`` module
+        - Saves model to ``.pkl`` format
+        - Serialises model to ``base64`` encoding
+        - Inserts data to the DB
+    """
+    timestamp = datetime.now()
+    file_path = f'pickled_model_{timestamp}' 
+
+    df, cat_features, cols, next_hrs = pre_process_data()
     model = train_model(df,cols, next_hrs)
-    save_model(f'pickled_model_{datetime.now()}', model)
+    save_model(file_path, model)
+
+    print("cat_features:", cat_features)
+    print("cols:", cols)
+    print("next_hrs:", next_hrs)
+
+    serialized_model = serialize_model(file_path)
+    training_tmp_data = [(cat_features, cols.to_list(), next_hrs, serialized_model)]
+
+    clickhouse_client.insert_data('training_tmp', training_tmp_data)
+    print("Training has been completed and data is inserted to the database!")