Skip to content

Release Readiness

Release Readiness #20

name: Release Readiness
# Heavy, optional checks that signal "are we in a state where a real release
# would be safe?". Decoupled from dev-publish so flakes here never block a
# publish, and from release-prod so heavy checks never run mid-release.
#
# A green run within the last 7 days is the manual go/no-go signal for a
# real release.
on:
workflow_dispatch:
inputs:
skip_integration:
description: "Skip the integration-tests job (use while iterating on other jobs; tracked in CI-0019)"
type: boolean
default: false
schedule:
- cron: '0 6 * * 1' # Mondays 06:00 UTC
concurrency:
group: release-readiness
cancel-in-progress: false
permissions:
contents: read
actions: read # resolve-dev-version reads the latest Dev Build & Publish run + its artifact
env:
PROTOC_VERSION: "28.3"
# arduino/setup-protoc@v3.0.0 has no Node.js 24 release yet; remove once a node24 version ships
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true
jobs:
# ---------------------------------------------------------------------------
# Resolve the exact dev wheel version published by the most recent successful
# Dev Build & Publish run. The install-tests and multi-python-install-verify
# jobs pin to this version so they exercise the wheel built from current main,
# not whatever the highest-versioned wheel happens to be on PyPI / AR.
# (Without pinning, a stale prerelease on PyPI like 9.0.0rc1 would shadow
# 8.1.2.dev* dev wheels per PEP 440 ordering.)
# ---------------------------------------------------------------------------
resolve-dev-version:
name: Resolve latest dev wheel version
runs-on: ubuntu-latest
timeout-minutes: 5
outputs:
dev_version: ${{ steps.resolve.outputs.dev_version }}
run_id: ${{ steps.find.outputs.run_id }}
steps:
- name: Find latest Dev Build & Publish run with a dev-version artifact
id: find
env:
GH_TOKEN: ${{ github.token }}
GH_REPO: ${{ github.repository }}
run: |
set -euo pipefail
# Walk recent successful runs and pick the first one that actually
# uploaded a 'dev-version' artifact. The check-changes gate skips the
# version job (and so the upload) when there are no new commits, but
# such runs still complete as success — those must be ignored.
# Use a wide window (100 candidates ≈ ~4 days at hourly cadence) so a
# quiet main (no new commits for >10h) does not exhaust the window
# before finding a real publish run.
mapfile -t candidates < <(gh run list \
--workflow dev-publish.yml \
--branch main \
--status success \
--limit 100 \
--json databaseId \
--jq '.[].databaseId')
run_id=""
for candidate in "${candidates[@]}"; do
if gh api "repos/${GH_REPO}/actions/runs/${candidate}/artifacts" \
--jq '.artifacts[].name' \
| grep -qx "dev-version"; then
run_id="${candidate}"
break
fi
echo "Run ${candidate} has no dev-version artifact; trying older run"
done
if [ -z "$run_id" ]; then
echo "::error::No recent successful 'Dev Build & Publish' run on main has a 'dev-version' artifact"
exit 1
fi
echo "run_id=${run_id}" >> "$GITHUB_OUTPUT"
echo "Found dev-publish run: ${run_id}"
- name: Download dev-version artifact
env:
GH_TOKEN: ${{ github.token }}
GH_REPO: ${{ github.repository }}
run: |
set -euo pipefail
gh run download "${{ steps.find.outputs.run_id }}" \
--name dev-version \
--dir ./dev-version
- name: Resolve dev version
id: resolve
run: |
set -euo pipefail
version=$(tr -d '\n\r' < ./dev-version/dev_version.txt)
if [ -z "$version" ]; then
echo "::error::dev_version.txt is empty in run ${{ steps.find.outputs.run_id }}"
exit 1
fi
echo "dev_version=${version}" >> "$GITHUB_OUTPUT"
echo "Pinning installs to pinecone==${version}"
# ---------------------------------------------------------------------------
# Integration tests against a real Pinecone backend.
# ---------------------------------------------------------------------------
integration-tests:
name: Integration tests (real backend)
# Allow opting out via workflow_dispatch input while iterating on other
# jobs. Schedule and default dispatch still run integration tests.
if: ${{ github.event_name != 'workflow_dispatch' || !inputs.skip_integration }}
runs-on: ubuntu-latest
timeout-minutes: 90
env:
PINECONE_API_KEY: ${{ secrets.PINECONE_API_KEY }}
steps:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- name: Install protoc
uses: arduino/setup-protoc@c65c819552d16ad3c9b72d9dfd5ba5237b9c906b # v3.0.0
with:
version: ${{ env.PROTOC_VERSION }}
repo-token: ${{ secrets.GITHUB_TOKEN }}
- name: Install Rust toolchain
uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable
with:
toolchain: stable
- name: Cache cargo build artifacts
uses: Swatinem/rust-cache@23869a5bd66c73db3c0ac40331f3206eb23791dc # v2.9.1
with:
workspaces: rust
shared-key: integration-tests
save-if: ${{ github.ref == 'refs/heads/main' }}
- name: Install uv
uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0
with:
enable-cache: true
- name: Set up Python
run: uv python install 3.12
- name: Install deps + build Rust extension
run: uv sync --group dev
- name: pytest tests/integration
run: uv run --no-sync pytest tests/integration -m "not preview_integration" -n 6 --dist=loadfile -v --junitxml=integration-results.xml
- name: Upload integration results
if: always()
uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
with:
name: integration-results
path: integration-results.xml
# ---------------------------------------------------------------------------
# Install the latest dev wheel from GCP Artifact Registry and smoke-test it.
# Validates the publish/install path end-to-end (not just that the wheel was
# built — the smoke job in dev-publish already covers that).
# ---------------------------------------------------------------------------
install-tests:
name: Install from Artifact Registry + smoke
needs: resolve-dev-version
runs-on: ubuntu-latest
timeout-minutes: 30
permissions:
contents: read
id-token: write
env:
GCP_REGION: ${{ vars.GCP_REGION }}
GCP_PROJECT_ID: ${{ vars.GCP_PROJECT_ID }}
GCP_REPO_NAME: ${{ vars.GCP_REPO_NAME }}
PINECONE_API_KEY: ${{ secrets.PINECONE_API_KEY }}
DEV_VERSION: ${{ needs.resolve-dev-version.outputs.dev_version }}
steps:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- name: Authenticate to Google Cloud
uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # v3.0.0
with:
workload_identity_provider: ${{ vars.GCP_WORKLOAD_IDENTITY_PROVIDER }}
service_account: ${{ vars.GCP_SERVICE_ACCOUNT }}
- name: Set up Python
uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
with:
python-version: "3.12"
- name: Install Artifact Registry keyring backend
run: |
python -m pip install --upgrade pip
pip install keyring==25.6.0 keyrings.google-artifactregistry-auth==1.1.2
- name: Install pinned dev wheel from Artifact Registry
run: |
set -euo pipefail
INDEX_URL="https://${GCP_REGION}-python.pkg.dev/${GCP_PROJECT_ID}/${GCP_REPO_NAME}/simple/"
echo "Installing pinecone==${DEV_VERSION} from ${INDEX_URL}"
# Pin to the exact version produced by the resolved dev-publish run so
# the smoke suite exercises the wheel built from current main, not a
# higher-versioned PyPI prerelease that pip would otherwise prefer.
# --pre still required because the version specifier itself contains .dev.
pip install --pre --extra-index-url "${INDEX_URL}" \
"pinecone==${DEV_VERSION}" pytest pytest-asyncio python-dotenv
- name: Show installed version
run: pip show pinecone | grep -E '^(Name|Version|Location):'
- name: Verify dependency compatibility (pip check)
run: pip check
# The repo's `pinecone/` source dir would shadow the installed wheel
# because Python prepends CWD to sys.path. The source tree has no
# compiled `_grpc.abi3.so`, so any test that touches the gRPC path
# fails with `ModuleNotFoundError: No module named 'pinecone._grpc'`.
# Move the source aside so `import pinecone` resolves to site-packages.
- name: Move source pinecone/ aside (avoid shadowing installed wheel)
run: mv pinecone _pinecone_source
- name: Run smoke suite
id: smoke
run: |
pytest tests/smoke/ \
-v -s -rs \
--ignore=tests/smoke/test_pod_collections_sync.py \
--ignore=tests/smoke/test_pod_collections_async.py \
--junitxml=install-smoke-results.xml
- name: Orphan cleanup
if: always()
continue-on-error: true
run: python tests/smoke/scripts/cleanup_orphans.py || true
- name: Upload install-smoke results
if: always()
uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
with:
name: install-smoke-results
path: install-smoke-results.xml
# ---------------------------------------------------------------------------
# Multi-Python install verify: exercises the abi3 promise on every supported
# Python version (3.10–3.13). The published wheel is cp310-abi3, so it must
# install and load the Rust extension on each Python we claim to support.
# ---------------------------------------------------------------------------
multi-python-install-verify:
name: Multi-Python install verify (py${{ matrix.python-version }})
needs: resolve-dev-version
runs-on: ubuntu-latest
timeout-minutes: 15
permissions:
contents: read
id-token: write
strategy:
fail-fast: false
matrix:
python-version: ["3.10", "3.11", "3.12", "3.13"]
env:
GCP_REGION: ${{ vars.GCP_REGION }}
GCP_PROJECT_ID: ${{ vars.GCP_PROJECT_ID }}
GCP_REPO_NAME: ${{ vars.GCP_REPO_NAME }}
DEV_VERSION: ${{ needs.resolve-dev-version.outputs.dev_version }}
steps:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- name: Authenticate to Google Cloud
uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # v3.0.0
with:
workload_identity_provider: ${{ vars.GCP_WORKLOAD_IDENTITY_PROVIDER }}
service_account: ${{ vars.GCP_SERVICE_ACCOUNT }}
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
with:
python-version: ${{ matrix.python-version }}
- name: Install AR keyring backend
run: |
python -m pip install --upgrade pip
pip install keyring==25.6.0 keyrings.google-artifactregistry-auth==1.1.2
- name: Install pinned dev wheel from Artifact Registry
run: |
set -euo pipefail
INDEX_URL="https://${GCP_REGION}-python.pkg.dev/${GCP_PROJECT_ID}/${GCP_REPO_NAME}/simple/"
# Pinned to the exact version from the resolved dev-publish run; see
# comment on install-tests for rationale.
pip install --pre --extra-index-url "${INDEX_URL}" "pinecone==${DEV_VERSION}"
- name: Verify dependency compatibility (pip check)
run: pip check
- name: Show installed version + Python
run: |
python -V
pip show pinecone | grep -E '^(Name|Version|Location):'
# The repo's `pinecone/` source dir would shadow the installed wheel
# because Python prepends CWD to sys.path. The source tree has no
# compiled `_grpc.abi3.so`, so the import below fails with
# `ModuleNotFoundError: No module named 'pinecone._grpc'`.
# Move the source aside so `import pinecone` resolves to site-packages.
- name: Move source pinecone/ aside (avoid shadowing installed wheel)
run: mv pinecone _pinecone_source
- name: Verify import + Rust extension on this Python version
run: |
python -c "
import pinecone
print('OK: import pinecone on Python', __import__('sys').version_info[:2])
import pinecone._grpc
print('OK: import pinecone._grpc')
ch = pinecone._grpc.GrpcChannel(
endpoint='https://example.invalid:443',
api_key='test',
api_version='2025-10',
version='0.0.0',
)
print('OK: GrpcChannel constructed:', type(ch).__name__)
"
# ---------------------------------------------------------------------------
# Cross-platform wheel build matrix without injecting dev versions or
# publishing. Independent signal that release-builds work on every target.
# ---------------------------------------------------------------------------
cross-platform-build:
name: Wheel build (${{ matrix.target }})
strategy:
fail-fast: false
matrix:
include:
- os: ubuntu-latest
target: x86_64-unknown-linux-gnu
manylinux: auto
- os: ubuntu-24.04-arm
target: aarch64-unknown-linux-gnu
manylinux: auto
- os: ubuntu-latest
target: x86_64-unknown-linux-musl
manylinux: musllinux_1_2
- os: ubuntu-24.04-arm
target: aarch64-unknown-linux-musl
manylinux: musllinux_1_2
- os: macos-14
target: x86_64-apple-darwin
- os: macos-14
target: aarch64-apple-darwin
- os: windows-latest
target: x86_64-pc-windows-msvc
runs-on: ${{ matrix.os }}
timeout-minutes: 30
steps:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- name: Install protoc
if: runner.os != 'Linux'
uses: arduino/setup-protoc@c65c819552d16ad3c9b72d9dfd5ba5237b9c906b # v3.0.0
with:
version: ${{ env.PROTOC_VERSION }}
repo-token: ${{ secrets.GITHUB_TOKEN }}
- name: Build wheels
uses: PyO3/maturin-action@04ac600d27cdf7a9a280dadf7147097c42b757ad # v1.50.1
with:
target: ${{ matrix.target }}
args: --release --out dist
manylinux: ${{ matrix.manylinux || 'auto' }}
before-script-linux: |
TARGET="${{ matrix.target }}"
PROTOC_VERSION="${{ env.PROTOC_VERSION }}"
case "$TARGET" in
x86_64-*)
PROTOC_ARCH="linux-x86_64"
PROTOC_SHA256="0ad949f04a6a174da83cdcbdb36dee0a4925272a5b6d83f79a6bf9852076d53f"
;;
aarch64-*)
PROTOC_ARCH="linux-aarch_64"
PROTOC_SHA256="1de522032a8b194002fe35cab86d747848238b5e4de4f99648372079f5b46f9a"
;;
*)
echo "Unsupported target: $TARGET" >&2
exit 1
;;
esac
PROTOC_ZIP="protoc-${PROTOC_VERSION}-${PROTOC_ARCH}.zip"
curl -fsSLO "https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOC_VERSION}/${PROTOC_ZIP}"
echo "${PROTOC_SHA256} ${PROTOC_ZIP}" | sha256sum -c -
python3 -m zipfile -e "${PROTOC_ZIP}" /usr/local
chmod +x /usr/local/bin/protoc
rm "${PROTOC_ZIP}"
- name: Validate wheel built
shell: bash
run: |
count=$(ls dist/*.whl 2>/dev/null | wc -l)
echo "Built ${count} wheel(s) for ${{ matrix.target }}"
ls -la dist/
if [ "$count" -lt 1 ]; then
echo "::error::no wheels produced for ${{ matrix.target }}"
exit 1
fi
# abi3audit parses the compiled extension inside the wheel and flags any CPython
# symbols that aren't part of the stable ABI (PEP 384). A cp310-abi3-tagged wheel
# that pulls in a non-stable-ABI symbol imports fine on the build Python but breaks
# at runtime on other 3.x versions. This is the static complement to multi-python
# smoke tests: the smoke tests exercise the wheel; abi3audit proves it is valid by
# inspection. Runs on every OS and reads any wheel format (ELF/Mach-O/PE).
#
# Isolated in its own venv: pip-installing abi3audit into the runner's host
# Python leaves abi3audit + packaging + rich in user site-packages, which then
# makes the verify step's `pip check` fail because the verify Python sees those
# leftover packages but pinecone's install does not pull in their transitive
# deps (urllib3, platformdirs). The venv keeps abi3audit fully out of the
# verify Python's view. See CI-0037 for the original failure (Release Readiness
# run 25457165040).
- name: abi3audit
shell: bash
run: |
set -euo pipefail
VENV_DIR="${RUNNER_TEMP}/abi3audit-venv"
python3 -m venv "$VENV_DIR"
if [ -d "$VENV_DIR/Scripts" ]; then
VBIN="$VENV_DIR/Scripts"
else
VBIN="$VENV_DIR/bin"
fi
"$VBIN/pip" install --quiet abi3audit
"$VBIN/python" -m abi3audit --strict --report dist/*.whl
# auditwheel show inspects ELF binaries inside each manylinux wheel and reports
# the actual minimum glibc version. A wheel tagged manylinux_2_17 but linking
# newer symbols would silently break users on older systems; this catches that
# before we ever publish.
#
# Restricted to glibc: auditwheel resolves library dependencies via the host's
# ldd, so analyzing a musllinux wheel on a glibc runner blows up with
# "ELFError: Magic number does not match" when it follows links into musl libc
# paths that don't exist. Musl ABI is exercised by the in-alpine install step.
#
# Isolated in its own venv: pip-installing auditwheel into the runner's host
# Python leaves auditwheel + packaging in user site-packages, which then makes
# the verify step's `pip check` fail ("auditwheel requires packaging") because
# PEP-668 fallbacks and pip self-upgrade can desync those installs. The venv
# keeps auditwheel fully out of the verify Python's view.
- name: auditwheel show (manylinux only)
if: endsWith(matrix.target, '-unknown-linux-gnu')
shell: bash
run: |
set -euo pipefail
python3 -m venv /tmp/auditwheel-venv
/tmp/auditwheel-venv/bin/pip install --quiet auditwheel
for wheel in dist/*.whl; do
echo "=== auditwheel show: $wheel ==="
/tmp/auditwheel-venv/bin/python -m auditwheel show "$wheel"
done
# Install verification: prove the wheel installs cleanly on its target
# platform and the Rust extension loads + is callable. The GrpcChannel
# constructor is lazy (no network), so this is a fast purely-local check.
#
# Skipped for x86_64-apple-darwin: it's cross-compiled on an arm64 runner
# and can't be exercised in-place. Tracked as a known gap.
- name: Set up Python for install verification
if: matrix.target != 'x86_64-apple-darwin' && !endsWith(matrix.target, '-musl')
uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
with:
python-version: "3.12"
- name: Write install-verification script
if: matrix.target != 'x86_64-apple-darwin'
shell: bash
run: |
mkdir -p .ci
cat > .ci/verify_install.py <<'PY'
"""Smoke-check that the just-installed pinecone wheel works.
Exercises the platform-specific Rust extension by importing
pinecone._grpc and constructing a GrpcChannel (lazy, no network).
"""
import sys
import pinecone # noqa: F401 — top-level import must succeed
print("OK: import pinecone")
import pinecone._grpc # native dylib must load on this platform
print("OK: import pinecone._grpc")
ch = pinecone._grpc.GrpcChannel(
endpoint="https://example.invalid:443",
api_key="test-key",
api_version="2025-10",
version="0.0.0-install-test",
)
print("OK: GrpcChannel constructed:", type(ch).__name__)
sys.exit(0)
PY
- name: Install wheel + run verification (native host)
if: matrix.target != 'x86_64-apple-darwin' && !endsWith(matrix.target, '-musl')
shell: bash
run: |
set -euo pipefail
wheel=$(ls dist/*.whl | head -n1)
echo "Installing ${wheel} on $(uname -sm) with Python $(python -V)"
python -m pip install --upgrade pip
python -m pip install "${wheel}"
python -m pip check
python .ci/verify_install.py
- name: Install wheel + run verification (musl, alpine container)
if: endsWith(matrix.target, '-musl')
shell: bash
run: |
set -euo pipefail
if [[ "${{ matrix.target }}" == aarch64-* ]]; then
IMAGE="arm64v8/python:3.12-alpine"
else
IMAGE="python:3.12-alpine"
fi
echo "Verifying musl wheel inside ${IMAGE}"
docker run --rm \
-v "$PWD/dist:/dist:ro" \
-v "$PWD/.ci/verify_install.py:/verify_install.py:ro" \
"$IMAGE" sh -c '
set -e
ls /dist/*.whl
pip install --quiet /dist/*.whl
pip check
python /verify_install.py
'
- name: Note skipped install verification (x86_64-apple-darwin)
if: matrix.target == 'x86_64-apple-darwin'
run: |
echo "::notice::install verification skipped for x86_64-apple-darwin: cross-compiled on arm64 runner, cannot run in place. Real users on Intel macs are the first to exercise this wheel."
# ---------------------------------------------------------------------------
# Sphinx docs build (warnings are errors).
# ---------------------------------------------------------------------------
doc-build:
name: Sphinx docs build
runs-on: ubuntu-latest
timeout-minutes: 15
steps:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- name: Install protoc
uses: arduino/setup-protoc@c65c819552d16ad3c9b72d9dfd5ba5237b9c906b # v3.0.0
with:
version: ${{ env.PROTOC_VERSION }}
repo-token: ${{ secrets.GITHUB_TOKEN }}
- name: Install Rust toolchain
uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable
with:
toolchain: stable
- name: Cache cargo build artifacts
uses: Swatinem/rust-cache@23869a5bd66c73db3c0ac40331f3206eb23791dc # v2.9.1
with:
workspaces: rust
shared-key: doc-build
save-if: ${{ github.ref == 'refs/heads/main' }}
- name: Install uv
uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0
with:
enable-cache: true
- name: Set up Python
run: uv python install 3.12
- name: Install deps (incl. docs extra) + build Rust extension
run: uv sync --group dev --extra docs
- name: Build HTML docs
run: uv run --no-sync make -C docs html SPHINXOPTS="-W"
- name: Upload docs artifact
if: always()
uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
with:
name: docs-html
path: docs/_build/html
# ---------------------------------------------------------------------------
# Failure notification: open a GitHub issue when any job fails.
# ---------------------------------------------------------------------------
notify-failure:
name: Notify on release-readiness regression
runs-on: ubuntu-latest
needs: [resolve-dev-version, integration-tests, install-tests, multi-python-install-verify, cross-platform-build, doc-build]
if: >-
always() &&
(needs.resolve-dev-version.result == 'failure' ||
needs.integration-tests.result == 'failure' ||
needs.install-tests.result == 'failure' ||
needs.multi-python-install-verify.result == 'failure' ||
needs.cross-platform-build.result == 'failure' ||
needs.doc-build.result == 'failure')
permissions:
issues: write
steps:
- name: Open release-readiness issue
env:
GH_TOKEN: ${{ github.token }}
run: |
run_url="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
body=$(printf '%s\n' \
"## Release Readiness regressed" \
"" \
"**Workflow run:** ${run_url}" \
"**Triggered by:** \`${{ github.event_name }}\`" \
"**Ref:** \`${{ github.ref }}\`" \
"**Commit:** \`${{ github.sha }}\`" \
"" \
"| Job | Result |" \
"|-----|--------|" \
"| resolve-dev-version | ${{ needs.resolve-dev-version.result }} |" \
"| integration-tests | ${{ needs.integration-tests.result }} |" \
"| install-tests | ${{ needs.install-tests.result }} |" \
"| multi-python-install-verify | ${{ needs.multi-python-install-verify.result }} |" \
"| cross-platform-build | ${{ needs.cross-platform-build.result }} |" \
"| doc-build | ${{ needs.doc-build.result }} |" \
"" \
"Release-readiness is the manual go/no-go gate for releases — investigate before cutting one.")
gh issue create \
--repo "${{ github.repository }}" \
--title "[release-readiness] regression — $(date -u +%Y-%m-%d)" \
--body "$body"