Check PySpark

Author: adrianisk

.github/workflows/check-contracts-and-assets.yml

@@ -67,12 +67,34 @@ jobs:
         run: curl -sSL https://install.python-poetry.org | python3 -
         shell: bash
       - name: Install dependencies
-        run: poetry install
         shell: bash
+        run: |
+          # Install root level dependencies
+          poetry install
+          # Create a virtual environment for the PySpark project, install dependencies
+          python3 -m venv "pyspark/.venv"
+          pyspark/.venv/bin/pip install -r pyspark/requirements.txt
+          ls -la pyspark/.venv/bin
       - name: Run migrations from db_migrations directory
         working-directory: ./db_migrations
         run: poetry run alembic upgrade head
         shell: bash
+      - name: Check PySpark Data Assets
+        uses: gabledata/cicd/github-actions/check-data-assets@python_path
+        with:
+            # Provide API key and endpoint secrets
+            gable-api-key: ${{secrets.GABLE_API_KEY_UNSTABLE_SANDBOX}}
+            gable-api-endpoint: ${{secrets.GABLE_API_ENDPOINT_UNSTABLE_SANDBOX}}
+            allow-gable-pre-release: true
+            python-path: pyspark/.venv/bin/python
+            # List of paths to Protobuf files that should be checked with support for glob syntax. 
+            # Can either be specified as a space separated list ('event1.proto event2.proto'), or 
+            # a multiline string
+            data-asset-options: |
+              --source-type pyspark \
+              --project-root ${{ github.workspace }}/pyspark \
+              --csv-schema-file ${{ github.workspace }}/pyspark/schemas.csv \
+              --spark-job-entrypoint 'job.py --final_output_table pnw_bookings_30_days' --debug
       - name: Check Protobuf Data Assets
         uses: gabledata/cicd/github-actions/check-data-assets@latest
         with:

.github/workflows/publish-contracts-and-assets.yml

@@ -111,7 +111,7 @@ jobs:
               --source-type pyspark \
               --project-root ${{ github.workspace }}/pyspark \
               --csv-schema-file ${{ github.workspace }}/pyspark/schemas.csv \
-              --spark-job-entrypoint 'job.py --final_output_table pnw_bookings_30_days' --trace
+              --spark-job-entrypoint 'job.py --final_output_table pnw_bookings_30_days' --debug
       - name: Register Protobuf Data Assets
         uses: gabledata/cicd/github-actions/register-data-assets@latest
         with: