Dev Build & Publish #415
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Dev Build & Publish | |
| on: | |
| schedule: | |
| - cron: '0 * * * *' # Every hour | |
| workflow_dispatch: | |
| concurrency: | |
| group: dev-build-publish | |
| cancel-in-progress: false | |
| permissions: | |
| contents: read | |
| actions: read | |
| env: | |
| GCP_REGION: ${{ vars.GCP_REGION }} | |
| GCP_PROJECT_ID: ${{ vars.GCP_PROJECT_ID }} | |
| GCP_REPO_NAME: ${{ vars.GCP_REPO_NAME }} | |
| PROTOC_VERSION: "28.3" | |
| # arduino/setup-protoc@v3.0.0 has no Node.js 24 release yet; remove once a node24 version ships | |
| FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true | |
| jobs: | |
| # --------------------------------------------------------------------------- | |
| # Gate: skip scheduled runs when nothing changed since last success | |
| # --------------------------------------------------------------------------- | |
| check-changes: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 5 | |
| outputs: | |
| has_changes: ${{ steps.check.outputs.has_changes }} | |
| steps: | |
| - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| with: | |
| fetch-depth: 0 | |
| - name: Check for changes since last successful run | |
| id: check | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| run: | | |
| if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then | |
| echo "has_changes=true" >> "$GITHUB_OUTPUT" | |
| echo "Manual trigger — always run" | |
| exit 0 | |
| fi | |
| last_success=$(gh run list \ | |
| --workflow "Dev Build & Publish" \ | |
| --status success \ | |
| --json updatedAt \ | |
| --jq '.[0].updatedAt // empty') | |
| if [ -z "$last_success" ]; then | |
| echo "has_changes=true" >> "$GITHUB_OUTPUT" | |
| echo "No prior successful run found — running" | |
| exit 0 | |
| fi | |
| commit_count=$(git rev-list --count --since="$last_success" HEAD) | |
| if [ "$commit_count" -gt 0 ]; then | |
| echo "has_changes=true" >> "$GITHUB_OUTPUT" | |
| echo "$commit_count new commit(s) since $last_success" | |
| else | |
| echo "has_changes=false" >> "$GITHUB_OUTPUT" | |
| echo "No new commits since $last_success — skipping" | |
| fi | |
| # --------------------------------------------------------------------------- | |
| # Stage 1: Compute dev version | |
| # --------------------------------------------------------------------------- | |
| version: | |
| needs: check-changes | |
| if: needs.check-changes.outputs.has_changes == 'true' | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 5 | |
| outputs: | |
| dev_version: ${{ steps.compute.outputs.dev_version }} | |
| cargo_version: ${{ steps.compute.outputs.cargo_version }} | |
| steps: | |
| - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| - name: Compute dev version | |
| id: compute | |
| run: | | |
| base_version=$(python3 -c " | |
| import tomllib | |
| with open('pyproject.toml', 'rb') as f: | |
| print(tomllib.load(f)['project']['version']) | |
| ") | |
| timestamp=$(date +%s) | |
| short_sha=$(git rev-parse --short HEAD) | |
| dev_version="${base_version}.dev${timestamp}+g${short_sha}" | |
| cargo_version="${base_version}-dev.${timestamp}+g${short_sha}" | |
| echo "dev_version=${dev_version}" >> "$GITHUB_OUTPUT" | |
| echo "cargo_version=${cargo_version}" >> "$GITHUB_OUTPUT" | |
| echo "PEP 440 version (pyproject.toml): ${dev_version}" | |
| echo "SemVer version (Cargo.toml): ${cargo_version}" | |
| echo "${dev_version}" > dev_version.txt | |
| echo "${cargo_version}" > cargo_version.txt | |
| - name: Upload version artifact | |
| uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 | |
| with: | |
| name: dev-version | |
| path: | | |
| dev_version.txt | |
| cargo_version.txt | |
| # --------------------------------------------------------------------------- | |
| # Stage 2: Build wheels (matrix) + sdist | |
| # --------------------------------------------------------------------------- | |
| build-wheels: | |
| needs: version | |
| strategy: | |
| fail-fast: true | |
| matrix: | |
| include: | |
| # Linux glibc x86_64 | |
| - os: ubuntu-latest | |
| target: x86_64-unknown-linux-gnu | |
| manylinux: auto | |
| # Linux glibc aarch64 (native ARM64 runner — avoids ring crate cross-compilation issues) | |
| - os: ubuntu-24.04-arm | |
| target: aarch64-unknown-linux-gnu | |
| manylinux: auto | |
| # Linux musl x86_64 (Alpine Docker images) | |
| - os: ubuntu-latest | |
| target: x86_64-unknown-linux-musl | |
| manylinux: musllinux_1_2 | |
| # Linux musl aarch64 (Alpine on ARM) | |
| - os: ubuntu-24.04-arm | |
| target: aarch64-unknown-linux-musl | |
| manylinux: musllinux_1_2 | |
| # macOS x86_64 (cross-compiled on arm64 runner) | |
| - os: macos-14 | |
| target: x86_64-apple-darwin | |
| # macOS arm64 | |
| - os: macos-14 | |
| target: aarch64-apple-darwin | |
| # Windows x86_64 | |
| - os: windows-latest | |
| target: x86_64-pc-windows-msvc | |
| runs-on: ${{ matrix.os }} | |
| timeout-minutes: 30 | |
| steps: | |
| - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| - name: Download version artifact | |
| uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 | |
| with: | |
| name: dev-version | |
| path: version-artifact | |
| continue-on-error: true | |
| - name: Resolve dev version | |
| shell: bash | |
| run: | | |
| if [ -f version-artifact/dev_version.txt ]; then | |
| DEV_VERSION=$(cat version-artifact/dev_version.txt) | |
| CARGO_VERSION=$(cat version-artifact/cargo_version.txt) | |
| else | |
| DEV_VERSION="${{ needs.version.outputs.dev_version }}" | |
| CARGO_VERSION="${{ needs.version.outputs.cargo_version }}" | |
| fi | |
| echo "DEV_VERSION=${DEV_VERSION}" >> "$GITHUB_ENV" | |
| echo "CARGO_VERSION=${CARGO_VERSION}" >> "$GITHUB_ENV" | |
| - name: Install protoc | |
| if: runner.os != 'Linux' | |
| uses: arduino/setup-protoc@c65c819552d16ad3c9b72d9dfd5ba5237b9c906b # v3.0.0 | |
| with: | |
| version: ${{ env.PROTOC_VERSION }} | |
| repo-token: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Inject dev version | |
| shell: bash | |
| env: | |
| DEV_VERSION: ${{ needs.version.outputs.dev_version }} | |
| CARGO_VERSION: ${{ needs.version.outputs.cargo_version }} | |
| run: | | |
| python3 -c " | |
| import re, os, pathlib | |
| p = pathlib.Path('pyproject.toml') | |
| text = p.read_text() | |
| v = os.environ['DEV_VERSION'] | |
| text = re.sub(r'version\s*=\s*\"[^\"]+\"', f'version = \"{v}\"', text, count=1) | |
| p.write_text(text) | |
| " | |
| python3 -c " | |
| import re, os, pathlib | |
| p = pathlib.Path('rust/Cargo.toml') | |
| text = p.read_text() | |
| v = os.environ['CARGO_VERSION'] | |
| text = re.sub(r'^version\s*=\s*\"[^\"]+\"', f'version = \"{v}\"', text, count=1, flags=re.MULTILINE) | |
| p.write_text(text) | |
| " | |
| cargo generate-lockfile --manifest-path rust/Cargo.toml | |
| grep '^version' pyproject.toml rust/Cargo.toml | |
| - name: Verify version injection | |
| shell: bash | |
| run: | | |
| python3 -c " | |
| import tomllib, os | |
| with open('pyproject.toml', 'rb') as f: | |
| v = tomllib.load(f)['project']['version'] | |
| expected = os.environ['DEV_VERSION'] | |
| assert v == expected, f'Expected {expected}, got {v}' | |
| print(f'Version injection verified: {v}') | |
| " | |
| - name: Build wheels | |
| uses: PyO3/maturin-action@04ac600d27cdf7a9a280dadf7147097c42b757ad # v1.50.1 | |
| with: | |
| target: ${{ matrix.target }} | |
| args: --release --out dist | |
| manylinux: ${{ matrix.manylinux || 'auto' }} | |
| before-script-linux: | | |
| TARGET="${{ matrix.target }}" | |
| PROTOC_VERSION="${{ env.PROTOC_VERSION }}" | |
| case "$TARGET" in | |
| x86_64-*) | |
| PROTOC_ARCH="linux-x86_64" | |
| PROTOC_SHA256="0ad949f04a6a174da83cdcbdb36dee0a4925272a5b6d83f79a6bf9852076d53f" | |
| ;; | |
| aarch64-*) | |
| PROTOC_ARCH="linux-aarch_64" | |
| PROTOC_SHA256="1de522032a8b194002fe35cab86d747848238b5e4de4f99648372079f5b46f9a" | |
| ;; | |
| *) | |
| echo "Unsupported target: $TARGET" >&2 | |
| exit 1 | |
| ;; | |
| esac | |
| PROTOC_ZIP="protoc-${PROTOC_VERSION}-${PROTOC_ARCH}.zip" | |
| curl -fsSLO "https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOC_VERSION}/${PROTOC_ZIP}" | |
| echo "${PROTOC_SHA256} ${PROTOC_ZIP}" | sha256sum -c - | |
| python3 -m zipfile -e "${PROTOC_ZIP}" /usr/local | |
| chmod +x /usr/local/bin/protoc | |
| rm "${PROTOC_ZIP}" | |
| - name: Validate wheel platform tags (x86_64 cross-compile) | |
| if: matrix.target == 'x86_64-apple-darwin' | |
| shell: bash | |
| run: | | |
| echo "Validating wheel platform tags for x86_64 cross-compile..." | |
| bad=0 | |
| for wheel in dist/*.whl; do | |
| echo " $wheel" | |
| if [[ "$wheel" != *"x86_64"* ]]; then | |
| echo "ERROR: expected x86_64 platform tag but got: $wheel" | |
| bad=1 | |
| fi | |
| done | |
| if [ "$bad" -ne 0 ]; then | |
| echo "One or more wheels have incorrect platform tags." | |
| exit 1 | |
| fi | |
| echo "All wheels have correct macosx_*_x86_64 platform tags." | |
| - name: Upload wheels | |
| uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 | |
| with: | |
| name: wheels-${{ matrix.target }} | |
| path: dist/*.whl | |
| build-sdist: | |
| needs: version | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 10 | |
| steps: | |
| - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| - name: Download version artifact | |
| uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 | |
| with: | |
| name: dev-version | |
| path: version-artifact | |
| continue-on-error: true | |
| - name: Resolve dev version | |
| run: | | |
| if [ -f version-artifact/dev_version.txt ]; then | |
| DEV_VERSION=$(cat version-artifact/dev_version.txt) | |
| CARGO_VERSION=$(cat version-artifact/cargo_version.txt) | |
| else | |
| DEV_VERSION="${{ needs.version.outputs.dev_version }}" | |
| CARGO_VERSION="${{ needs.version.outputs.cargo_version }}" | |
| fi | |
| echo "DEV_VERSION=${DEV_VERSION}" >> "$GITHUB_ENV" | |
| echo "CARGO_VERSION=${CARGO_VERSION}" >> "$GITHUB_ENV" | |
| - name: Inject dev version | |
| env: | |
| DEV_VERSION: ${{ needs.version.outputs.dev_version }} | |
| CARGO_VERSION: ${{ needs.version.outputs.cargo_version }} | |
| run: | | |
| python3 -c " | |
| import re, os, pathlib | |
| p = pathlib.Path('pyproject.toml') | |
| text = p.read_text() | |
| v = os.environ['DEV_VERSION'] | |
| text = re.sub(r'version\s*=\s*\"[^\"]+\"', f'version = \"{v}\"', text, count=1) | |
| p.write_text(text) | |
| " | |
| python3 -c " | |
| import re, os, pathlib | |
| p = pathlib.Path('rust/Cargo.toml') | |
| text = p.read_text() | |
| v = os.environ['CARGO_VERSION'] | |
| text = re.sub(r'^version\s*=\s*\"[^\"]+\"', f'version = \"{v}\"', text, count=1, flags=re.MULTILINE) | |
| p.write_text(text) | |
| " | |
| cargo generate-lockfile --manifest-path rust/Cargo.toml | |
| grep '^version' pyproject.toml rust/Cargo.toml | |
| - name: Verify version injection | |
| run: | | |
| python3 -c " | |
| import tomllib, os | |
| with open('pyproject.toml', 'rb') as f: | |
| v = tomllib.load(f)['project']['version'] | |
| expected = os.environ['DEV_VERSION'] | |
| assert v == expected, f'Expected {expected}, got {v}' | |
| print(f'Version injection verified: {v}') | |
| " | |
| - name: Build sdist | |
| uses: PyO3/maturin-action@04ac600d27cdf7a9a280dadf7147097c42b757ad # v1.50.1 | |
| with: | |
| command: sdist | |
| args: --out dist | |
| - name: Upload sdist | |
| uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 | |
| with: | |
| name: sdist | |
| path: dist/*.tar.gz | |
| # --------------------------------------------------------------------------- | |
| # Stage 3: Publish to Artifact Registry | |
| # --------------------------------------------------------------------------- | |
| publish: | |
| needs: [build-wheels, build-sdist] | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 10 | |
| permissions: | |
| contents: read | |
| id-token: write | |
| steps: | |
| - name: Download all artifacts | |
| uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 | |
| with: | |
| path: dist | |
| pattern: '{wheels-*,sdist}' | |
| merge-multiple: true | |
| - name: Validate artifact counts | |
| run: | | |
| echo "Artifacts to publish:" | |
| ls -la dist/ | |
| whl_count=$(ls dist/*.whl 2>/dev/null | wc -l) | |
| sdist_count=$(ls dist/*.tar.gz 2>/dev/null | wc -l) | |
| echo "Wheels: $whl_count, Sdists: $sdist_count" | |
| # 7 platforms × 1 abi3 wheel (cp310-abi3, covers Python 3.10+) = 7 wheels | |
| if [ "$whl_count" -lt 7 ]; then | |
| echo "::error::expected at least 7 wheels but got $whl_count — one or more matrix legs may be missing" | |
| exit 1 | |
| fi | |
| if [ "$sdist_count" -lt 1 ]; then | |
| echo "::error::sdist missing" | |
| exit 1 | |
| fi | |
| - name: Authenticate to Google Cloud | |
| uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # v3.0.0 | |
| with: | |
| workload_identity_provider: ${{ vars.GCP_WORKLOAD_IDENTITY_PROVIDER }} | |
| service_account: ${{ vars.GCP_SERVICE_ACCOUNT }} | |
| - name: Set up Python | |
| uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 | |
| with: | |
| python-version: "3.12" | |
| - name: Install twine | |
| run: pip install twine==6.1.0 | |
| - name: Configure Artifact Registry credentials | |
| run: | | |
| pip install keyring==25.6.0 keyrings.google-artifactregistry-auth==1.1.2 | |
| - name: Validate wheel metadata | |
| run: twine check dist/* | |
| - name: Upload to Artifact Registry | |
| run: | | |
| twine upload \ | |
| --skip-existing \ | |
| --repository-url "https://${GCP_REGION}-python.pkg.dev/${GCP_PROJECT_ID}/${GCP_REPO_NAME}/" \ | |
| dist/* | |
| # --------------------------------------------------------------------------- | |
| # Stage 4: Smoke tests against the just-published wheel (informational only) | |
| # --------------------------------------------------------------------------- | |
| smoke-tests: | |
| name: Smoke tests (informational) | |
| needs: publish | |
| if: success() | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 20 | |
| continue-on-error: true | |
| permissions: | |
| contents: read | |
| env: | |
| PINECONE_API_KEY: ${{ secrets.PINECONE_API_KEY }} | |
| steps: | |
| - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| - name: Set up Python | |
| uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 | |
| with: | |
| python-version: "3.12" | |
| - name: Download linux x86_64 wheel | |
| uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 | |
| with: | |
| name: wheels-x86_64-unknown-linux-gnu | |
| path: dist | |
| - name: Install wheel and test deps | |
| run: | | |
| set -euo pipefail | |
| wheel=$(ls dist/*.whl | head -n1) | |
| echo "Installing ${wheel}" | |
| pip install --upgrade pip | |
| pip install "${wheel}" pytest pytest-asyncio python-dotenv | |
| - name: Run smoke suite | |
| id: smoke | |
| continue-on-error: true | |
| run: | | |
| # The source `pinecone/` directory from actions/checkout shadows the | |
| # wheel-installed package on sys.path (cwd is on the path). The source | |
| # tree has `pinecone/grpc/__init__.py` but not the compiled | |
| # `_grpc.abi3.so` extension, which only ships inside the wheel — so | |
| # any test that constructs a GrpcIndex would crash with | |
| # `ModuleNotFoundError: No module named 'pinecone._grpc'`. Smoke runs | |
| # the published wheel only, so we can safely delete the source. | |
| rm -rf pinecone rust target | |
| pytest tests/smoke/ \ | |
| -v -s -rs \ | |
| --ignore=tests/smoke/test_pod_collections_sync.py \ | |
| --ignore=tests/smoke/test_pod_collections_async.py \ | |
| --junitxml=smoke-results.xml | |
| - name: Append summary | |
| if: always() | |
| run: | | |
| if [ "${{ steps.smoke.outcome }}" = "success" ]; then | |
| echo "## Smoke tests: PASSED" >> "$GITHUB_STEP_SUMMARY" | |
| else | |
| echo "## Smoke tests: FAILED (informational — does not block publish)" >> "$GITHUB_STEP_SUMMARY" | |
| echo "See the 'Run smoke suite' step output and the smoke-results artifact." >> "$GITHUB_STEP_SUMMARY" | |
| fi | |
| - name: Orphan cleanup | |
| if: always() | |
| continue-on-error: true | |
| run: | | |
| python tests/smoke/scripts/cleanup_orphans.py || true | |
| - name: Upload smoke results | |
| if: always() | |
| uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 | |
| with: | |
| name: smoke-results | |
| path: smoke-results.xml |