Modifying kubeflow pipeline so anybody can run it on their own projec…

…ts. (GoogleCloudPlatform#170)

* Making component reusable

* Modifying pipeline so anybody can run it on their own project (to publish on AI HUB)

* Reading data from competition site

* Adding comments to code

* taking out hardcoded project

* castinf as timestamp

* casting as timestamp

* instructions to deploy cluster

examples/cloudml-energy-price-forecasting/data_preparation/prepare_data.sql

@@ -6,7 +6,7 @@ WITH
       PARSE_TIMESTAMP("%d/%m/%Y %H:%M", date_utc) AS date_utc,
       price
     FROM
-      `energy-forecasting.Energy.MarketPricePT`
+      Energy.MarketPricePT
   ),
   Distribution_Temp AS (
     SELECT
@@ -18,12 +18,12 @@ WITH
   ),
   Weather_Temp AS (
     SELECT
-      prediction_date,
+      PARSE_TIMESTAMP("%d/%m/%Y %H:%M", prediction_date) AS prediction_date,
       array_concat_agg(ARRAY[temperature, wind_speed_100m, wind_direction_100m, air_density, precipitation,
         wind_gust, radiation, wind_speed, wind_direction, pressure]
         ORDER BY point) AS weather
     FROM
-      `energy-forecasting.Energy.historical_weather`
+      Energy.historical_weather
     GROUP BY prediction_date
   )
 SELECT

examples/cloudml-energy-price-forecasting/data_preparation/weather_mean_std.sql

@@ -32,8 +32,8 @@ FROM
       STDDEV_SAMP(wind_direction) AS wind_direction_s,
       STDDEV_SAMP(pressure) AS pressure_s
     FROM
-      `energy-forecasting.Energy.historical_weather`
+      Energy.historical_weather
     WHERE
-      prediction_date BETWEEN @train_from_date AND @train_to_date
+      PARSE_TIMESTAMP("%d/%m/%Y %H:%M", prediction_date) BETWEEN @train_from_date AND @train_to_date
     GROUP BY point
   ) AS av

examples/cloudml-energy-price-forecasting/kubeflow_pipeline/copy-table.component.yaml

@@ -0,0 +1,12 @@
+version: google.com/cloud/pipelines/component/v1
+name: Copy table
+inputs:
+- {name: dataset, type: str}
+outputs:
+- {name: output}
+implementation:
+  container:
+    image: gcr.io/energy-forecasting/component-copy-table:latest
+    args:
+    - {value: dataset}
+    - {output: output}

examples/cloudml-energy-price-forecasting/kubeflow_pipeline/export-table.component.yaml

@@ -0,0 +1,16 @@
+version: google.com/cloud/pipelines/component/v1
+name: Export table
+inputs:
+- {name: inp, type: str}
+- {name: table, type: str}
+- {name: file, type: str}
+outputs:
+- {name: output}
+implementation:
+  container:
+    image: gcr.io/energy-forecasting/component-export-table:latest
+    args:
+    - {value: inp}
+    - {value: table}
+    - {value: file}
+    - {output: output}

examples/cloudml-energy-price-forecasting/kubeflow_pipeline/run-git-python-script.component.yaml

@@ -0,0 +1,20 @@
+version: google.com/cloud/pipelines/component/v1
+name: Run git python script
+inputs:
+- {name: inp, type: str}
+- {name: code_repo, type: str}
+- {name: code_folder, type: str}
+- {name: script, type: str}
+- {name: script_args, type: str}
+outputs:
+- {name: output}
+implementation:
+  container:
+    image: gcr.io/energy-forecasting/component-run-git-python-script:latest
+    args:
+    - {value: inp}
+    - {value: code_repo}
+    - {value: code_folder}
+    - {value: script}
+    - {value: script_args}
+    - {output: output}

examples/cloudml-energy-price-forecasting/kubeflow_pipeline/train-git-cmle-model.component.yaml

@@ -0,0 +1,36 @@
+version: google.com/cloud/pipelines/component/v1
+name: Train git cmle model
+inputs:
+- {name: tr_inp, type: str}
+- {name: va_inp, type: str}
+- {name: code_repo, type: str}
+- {name: code_folder, type: str}
+- {name: project, type: str}
+- {name: bucket, type: str}
+- {name: package_folder, type: str}
+- {name: cmle_folder, type: str}
+- {name: scale_tier, type: str}
+- {name: python_module, type: str}
+- {name: region, type: str}
+- {name: runtime_version, type: str}
+- {name: cmle_args, type: str}
+outputs:
+- {name: output}
+implementation:
+  container:
+    image: gcr.io/energy-forecasting/component-train-git-cmle-model:latest
+    args:
+    - {value: tr_inp}
+    - {value: va_inp}
+    - {value: code_repo}
+    - {value: code_folder}
+    - {value: project}
+    - {value: bucket}
+    - {value: package_folder}
+    - {value: cmle_folder}
+    - {value: scale_tier}
+    - {value: python_module}
+    - {value: region}
+    - {value: runtime_version}
+    - {value: cmle_args}
+    - {output: output}