test

Author: alexdemeo

.github/workflows/check-contracts-and-assets.yml

@@ -128,6 +128,7 @@ jobs:
             # Can either be specified as a space separated list ('event1.proto event2.proto'), or 
             # a multiline string
             data-asset-options: |
-              --project-root ../../pyspark \
-              --spark-job-entrypoint "job.py --final_output_table pnw_bookings_30_days" \
-              --csv-schema-file ../../pyspark/schemas.csv
+              --source-type pyspark \
+              --project-root ./pyspark \
+              --spark-job-entrypoint job.py \
+              --csv-schema-file ./pyspark/schemas.csv

pyspark/job.py

@@ -1,20 +1,11 @@
-import argparse
+from script import run_job
+
 from pyspark.sql import SparkSession
 from pyspark.sql.functions import *
 from pyspark.sql.types import *
 
-from script import run_job
-
-def parse_arguments(argv=None):
-    ap = argparse.ArgumentParser()
-    ap.add_argument("--final_output_table")
-    return ap.parse_args(argv)
-
-
 if __name__ == "__main__":
-    # Parse args
-    args_main = parse_arguments()
-    final_output_table = args_main.final_output_table
+    final_output_table = "pnw_bookings_30_days"
     print(f"final_output_table: {final_output_table}")
 
     spark = SparkSession.builder.getOrCreate()

pyspark/script.py

@@ -18,6 +18,7 @@ def run_job(spark, final_output_table):
       timestamp(date_trunc(b.confirm_time, 'DD')) AS "booking_date",
       b.earning_adjustment_commission_rate AS "commission_rate",
       b.receipt_payment_type AS "payment_type",
+      b.reward_id AS "reward_id",
       c.*,
     FROM
       bookings_30_days b