app-cloud-e2e.yml

# Python package
# Create and test a Python package on multiple Python versions.
# Add steps that analyze code, save the dist with the build record, publish to a PyPI-compatible index, and more:
# https://docs.microsoft.com/azure/devops/pipelines/languages/python

trigger:
  tags:
    include:
      - "*"
  branches:
    include:
      - "master"
      - "release/*"
      - "refs/tags/*"

schedules:
  - cron: "0 0 * * *" # At the end of every day
    displayName: Daily midnight testing
    branches:
      include:
        - "master"

pr:
  branches:
    include:
      - "master"
      - "release/*"
  paths:
    include:
      - ".actions/*"
      - ".azure/app-cloud-e2e.yml"
      - "src/lightning/__about__.py"
      - "src/lightning/__init__.py"
      - "src/lightning/__main__.py"
      - "src/lightning/__setup__.py"
      - "src/lightning/__version__.py"
      - "src/lightning/app/**"
      - "src/lightning_app/*"
      - "examples/app/**"
      - "requirements/app/**"
      - "tests/integrations_app/**"
      - "setup.py"
    exclude:
      - "!tests/integrations_app/flagship/**"
      - "requirements/*/docs.txt"
      - "*.md"
      - "**/*.md"

# variables are automatically exported as environment variables so this will override pip's default cache dir
variables:
  - name: pip_cache_dir
    value: $(Pipeline.Workspace)/.pip
  - name: local_id
    value: $(Build.BuildId)
  - name: video_artifact_dir
    value: ./videos

jobs:
  - job: test_e2e
    pool: "azure-cpus"
    container:
      # see all available tags: https://mcr.microsoft.com/en-us/product/playwright/python/tags
      image: mcr.microsoft.com/playwright/python:v1.38.0-focal
      options: "--shm-size=4gb"
    strategy:
      matrix:
        "App: v0_app":
          name: "v0_app"
          dir: "public"
        "App: boring_app":
          name: "boring_app"
          dir: "public"
        "App: template_streamlit_ui":
          name: "template_streamlit_ui"
          dir: "public"
        "App: template_react_ui":
          name: "template_react_ui"
          dir: "public"
        # 'App: template_jupyterlab':  # TODO: clarify where these files lives
        #   name: "template_jupyterlab"
        "App: installation_commands_app":
          name: "installation_commands_app"
          dir: "public"
        "App: drive":
          name: "drive"
          dir: "public"
        "App: payload":
          name: "payload"
          dir: "public"
        "App: commands_and_api":
          name: "commands_and_api"
          dir: "public"
        "App: quick_start":
          name: "quick_start"
          dir: "public"
        "App: idle_timeout":
          name: "idle_timeout"
          dir: "local"
        "App: collect_failures":
          name: "collect_failures"
          dir: "local"
        "App: custom_work_dependencies":
          name: "custom_work_dependencies"
          dir: "local"
    timeoutInMinutes: "15"
    cancelTimeoutInMinutes: "1"
    # values: https://docs.microsoft.com/en-us/azure/devops/pipelines/process/phases?view=azure-devops&tabs=yaml#workspace
    workspace:
      clean: all
    variables:
      FREEZE_REQUIREMENTS: "1"
      HEADLESS: "1"
      PACKAGE_LIGHTNING: "1"
      CLOUD: "1"
      VIDEO_LOCATION: $(video_artifact_dir)
      PR_NUMBER: $(local_id)
      TEST_APP_NAME: $(name)
      TEST_APP_FOLDER: $(dir)
      HAR_LOCATION: "./artifacts/hars"
      SLOW_MO: "50"
      LIGHTNING_DEBUG: "1"
    steps:
      - script: echo '##vso[task.setvariable variable=local_id]$(System.PullRequest.PullRequestNumber)'
        displayName: "Set id for this PR"
        condition: eq(variables['Build.Reason'], 'PullRequest')

      - bash: |
          whoami
          mkdir -p "$(video_artifact_dir)/$(name)"
          printf "local id: $(local_id)\n"
          python --version
          pip --version
          echo "allow fail: ${{ in(variables['name'], 'quick_start', 'template_react_ui') }}"
        displayName: "Info"

      # TODO: we are testing it as `lightning`, so add also version for `lightning_app`
      - bash: |
          pip install -e .[app-dev] \
            -f https://download.pytorch.org/whl/cpu/torch_stable.html
        displayName: "Install Lightning & dependencies"

      - bash: python -m playwright install # --with-deps
        displayName: "Install Playwright system dependencies"

      # The magic happens here it doesn't need to install the quick start dependencies.
      # This test is very important to test the main user story of lightning app.
      # It also e2e tests running on cloud without installing dependencies.
      - bash: |
          git clone https://github.com/Lightning-AI/lightning-quick-start examples/app/quick-start
        # without succeeded this could run even if the job has already failed
        condition: and(succeeded(), eq(variables['name'], 'quick_start'))
        displayName: "Clone Quick start Repo"
      - bash: |
          git clone https://github.com/Lightning-AI/lightning-template-react examples/app/template_react_ui
        # without succeeded this could run even if the job has already failed
        condition: and(succeeded(), eq(variables['name'], 'template_react_ui'))
        displayName: "Clone Template React UI Repo"

      # Replace imports to use `lightning` instead of `lightning_app` since we install lightning only ATM
      - bash: |
          pip install -q -r .actions/requirements.txt
          python .actions/assistant.py copy_replace_imports \
            --source_dir="./examples" --source_import="lightning_app" --target_import="lightning.app"
        displayName: "Adjust examples"

      - bash: pip --version && pip list
        displayName: "List pip dependency"

      - bash: |
          ls -l examples/app/$(TEST_APP_NAME)
          echo ${TEST_FILE}
          python -m pytest ${TEST_FILE}::test_$(TEST_APP_NAME)_example_cloud \
            --timeout=360 --capture=no -v --color=yes
        env:
          TEST_FILE: tests/integrations_app/$(TEST_APP_FOLDER)/test_$(TEST_APP_NAME).py
          #LAI_USER: $(LAI_USER)  # for STAGING
          #LAI_PASS: $(LAI_PASS)  # for STAGING
          LIGHTNING_USER_ID: $(LIGHTNING_USER_ID_PROD)
          LIGHTNING_API_KEY: $(LIGHTNING_API_KEY_PROD)
          LIGHTNING_USERNAME: $(LIGHTNING_USERNAME_PROD)
          LIGHTNING_CLOUD_URL: $(LIGHTNING_CLOUD_URL_PROD)
        # Todo: investigate why these apps are failing
        continueOnError: ${{ in(variables['name'], 'quick_start', 'template_react_ui') }}
        displayName: "Run the tests"

      - task: PublishPipelineArtifact@1
        condition: failed()
        inputs:
          path: "$(video_artifact_dir)/$(name)"
          artifactName: $(name)
          publishLocation: "pipeline"
        displayName: "Publish videos"

      - bash: |
          time python -c "from lightning.app import testing; testing.delete_cloud_lightning_apps()"
        condition: always()
        continueOnError: "true"
        timeoutInMinutes: "3"
        env:
          #LAI_USER: $(LAI_USER)  # for STAGING
          #LAI_PASS: $(LAI_PASS)  # for STAGING
          LIGHTNING_USER_ID: $(LIGHTNING_USER_ID_PROD)
          LIGHTNING_API_KEY: $(LIGHTNING_API_KEY_PROD)
          LIGHTNING_USERNAME: $(LIGHTNING_USERNAME_PROD)
          LIGHTNING_CLOUD_URL: $(LIGHTNING_CLOUD_URL_PROD)
        displayName: "Clean Previous Apps"