Skip to content

Feature/eventhub handle #1025

Feature/eventhub handle

Feature/eventhub handle #1025

name: Pre-Integration
on:
pull_request:
types:
- opened
- synchronize
- reopened
jobs:
unit_test:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Setup Python
uses: actions/setup-python@v5
with:
python-version: "3.10" # Goes with DBR 14.3
- uses: actions/setup-java@v4
# this workflow needs java so that it can run a local instance of spark
with:
java-version: "11"
distribution: "temurin"
- name: Install the package
run: pip install .
- name: Install test requirements
run: pip install -r requirements_dev.txt
- name: Check code formatting
run: |
black --check .
isort --check .
- name: Check code linting
run: flake8 . --exclude tests
- name: Run Tests
run: python -m pytest tests/local
integration_test:
environment: azure
concurrency: azure-integration
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Setup Python
uses: actions/setup-python@v5
with:
python-version: "3.10" # Goes with DBR 14.3
- name: Build Spetlr Library
shell: pwsh
run: .github/deploy/build.ps1
- name: Log in to azure
shell: pwsh
run: |
az login --service-principal `
-u ${{ secrets.SPN_CLIENT_ID }} `
-p ${{ secrets.SPN_CLIENT_SECRET }} `
--tenant ${{ secrets.SPN_TENANT_ID }}
- name: Setup Terraform
uses: hashicorp/setup-terraform@v3
with:
terraform_wrapper: false
- name: Databricks Metastore Terraform
env:
ARM_ACCESS_KEY: ${{ secrets.BACKEND_STORAGE_ACCESS_KEY }}
ARM_CLIENT_ID: ${{ secrets.SPN_CLIENT_ID }}
ARM_CLIENT_SECRET: ${{ secrets.SPN_CLIENT_SECRET }}
ARM_TENANT_ID: ${{ secrets.SPN_TENANT_ID }}
ARM_SUBSCRIPTION_ID: ${{ secrets.SPN_SUBSCRIPTION_ID }}
TF_VAR_db_account_id: ${{ secrets.DATABRICKS_ACCOUNT_ID }}
run: |
terraform -chdir="./.github/terraform/metastore_and_pemanent" init
terraform -chdir="./.github/terraform/metastore_and_pemanent" apply -auto-approve
- name: Integration Resources Terraform
env:
ARM_CLIENT_ID: ${{ secrets.SPN_CLIENT_ID }}
ARM_CLIENT_SECRET: ${{ secrets.SPN_CLIENT_SECRET }}
ARM_TENANT_ID: ${{ secrets.SPN_TENANT_ID }}
ARM_SUBSCRIPTION_ID: ${{ secrets.SPN_SUBSCRIPTION_ID }}
TF_VAR_db_account_id: ${{ secrets.DATABRICKS_ACCOUNT_ID }}
TF_VAR_uniqueRunId: ${{ github.run_id }}${{ github.run_attempt }}
run: |
terraform -chdir="./.github/terraform/integration_resources" init
terraform -chdir="./.github/terraform/integration_resources" apply -auto-approve
- name: Databricks Integration Terraform
env:
ARM_CLIENT_ID: ${{ secrets.SPN_CLIENT_ID }}
ARM_CLIENT_SECRET: ${{ secrets.SPN_CLIENT_SECRET }}
ARM_TENANT_ID: ${{ secrets.SPN_TENANT_ID }}
ARM_SUBSCRIPTION_ID: ${{ secrets.SPN_SUBSCRIPTION_ID }}
TF_VAR_db_account_id: ${{ secrets.DATABRICKS_ACCOUNT_ID }}
TF_VAR_uniqueRunId: ${{ github.run_id }}${{ github.run_attempt }}
run: |
terraform -chdir="./.github/terraform/integration_databricks" init
terraform -chdir="./.github/terraform/integration_databricks" apply -auto-approve
- name: Install Databricks CLI
shell: bash
run: curl -fsSL https://raw.githubusercontent.com/databricks/setup-cli/main/install.sh | sh
- name: Execute Test Job
shell: pwsh
run: |
. .github/submit/connect.ps1 -uniqueRunId "${{ github.run_id }}${{ github.run_attempt }}"
pip install spetlr-tools
spetlr-test-job submit `
--tests tests/ `
--cluster-file .github/submit/cluster.json `
--requirements-file requirements_test.txt `
--sparklibs-file .github/submit/sparklibs.json `
--out-json test.json `
--pytest-args="-vv" `
--task tests/cluster
spetlr-test-job fetch --runid-json test.json
- name: Destroy Azure resources
if: always()
shell: pwsh
run: |
az login --service-principal `
-u ${{ secrets.SPN_CLIENT_ID }} `
-p ${{ secrets.SPN_CLIENT_SECRET }} `
--tenant ${{ secrets.SPN_TENANT_ID }}
./.github/deploy/cleanup-integration.ps1
- name: Azure logout
if: always()
shell: pwsh
run: az logout