From 20a782981498143afe584247f0eacd16fb793de7 Mon Sep 17 00:00:00 2001 From: Ed Morley <501702+edmorley@users.noreply.github.com> Date: Wed, 12 Feb 2025 11:16:08 +0000 Subject: [PATCH 01/26] Improve metrics for builds that fail (#1746) Adds a `failure_detail` field in addition to the existing `failure_reason`, which contains additional context relevant to the failure where available. This will make it easier to find trends in the most frequent user-caused failure modes (eg invalid Python version specifier) so I can then adjust error messages/docs/implementation to improve UX. This context sometimes contain user input, so the values saved to the metadata store now also have additional escaping and validation performed before writing the value (in addition to the existing YAML escaping performed in `bin/report`). GUS-W-17800067. --- CHANGELOG.md | 1 + bin/report | 1 + bin/steps/python | 2 ++ lib/checks.sh | 2 ++ lib/kvstore.sh | 14 +++++++++++++- lib/python_version.sh | 19 ++++++++++++++++--- lib/utils.sh | 6 ++++-- 7 files changed, 39 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c7817d81e..7be1eea19 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## [Unreleased] +- Improved buildpack metrics for builds that fail. ([#1746](https://github.com/heroku/heroku-buildpack-python/pull/1746)) ## [v276] - 2025-02-05 diff --git a/bin/report b/bin/report index f05d67b6c..cc0d1a116 100755 --- a/bin/report +++ b/bin/report @@ -62,6 +62,7 @@ kv_pair_string() { STRING_FIELDS=( cache_status django_collectstatic + failure_detail failure_reason nltk_downloader package_manager diff --git a/bin/steps/python b/bin/steps/python index b345d5cd8..6855448de 100755 --- a/bin/steps/python +++ b/bin/steps/python @@ -23,6 +23,7 @@ if ! curl --output /dev/null --silent --head --fail --retry 3 --retry-connrefuse https://devcenter.heroku.com/articles/python-support#supported-python-versions EOF meta_set "failure_reason" "python-version-not-found" + meta_set "failure_detail" "${python_full_version}" exit 1 fi @@ -42,6 +43,7 @@ else Please try again and to see if the error resolves itself. EOF meta_set "failure_reason" "python-download" + # TODO: Set failure_detail here once refactored. exit 1 fi diff --git a/lib/checks.sh b/lib/checks.sh index 9129244e7..3a43cdad7 100644 --- a/lib/checks.sh +++ b/lib/checks.sh @@ -20,6 +20,7 @@ function checks::ensure_supported_stack() { Upgrade to a newer stack to continue using this buildpack. EOF meta_set "failure_reason" "stack::eol" + meta_set "failure_detail" "${stack}" exit 1 ;; *) @@ -34,6 +35,7 @@ function checks::ensure_supported_stack() { https://devcenter.heroku.com/articles/managing-buildpacks#classic-buildpacks-references EOF meta_set "failure_reason" "stack::unknown" + meta_set "failure_detail" "${stack}" exit 1 ;; esac diff --git a/lib/kvstore.sh b/lib/kvstore.sh index c4a684fea..b5c15aa60 100644 --- a/lib/kvstore.sh +++ b/lib/kvstore.sh @@ -23,8 +23,20 @@ kv_set() { # TODO: Stop ignoring an incorrect number of passed arguments. if [[ $# -eq 3 ]]; then local f="${1}" + local key="${2}" + + # Truncate the value to an arbitrary 100 characters since it will sometimes contain user-provided + # inputs which may be unbounded in size. Ideally individual call sites will perform more aggressive + # truncation themselves based on the expected value size, however this is here as a fallback. + # (Honeycomb supports string fields up to 64KB in size, however, it's not worth filling up the + # metadata store or bloating the payload passed back to Vacuole/submitted to Honeycomb given the + # extra content in those cases is not normally useful.) + local value="${3:0:100}" + # Replace newlines since the data store file format requires that keys don't span multiple lines. + value="${value//$'\n'/ }" + if [[ -f "${f}" ]]; then - echo "${2}=${3}" >>"${f}" + echo "${key}=${value}" >>"${f}" fi fi } diff --git a/lib/python_version.sh b/lib/python_version.sh index eec397496..652c4e478 100644 --- a/lib/python_version.sh +++ b/lib/python_version.sh @@ -104,7 +104,7 @@ function python_version::parse_runtime_txt() { in the correct format. The following file contents were found, which aren't valid: - ${contents} + ${contents:0:100} However, the runtime.txt file is deprecated since it has been replaced by the .python-version file. As such, we @@ -125,6 +125,7 @@ function python_version::parse_runtime_txt() { your app to receive Python security updates. EOF meta_set "failure_reason" "runtime-txt::invalid-version" + meta_set "failure_detail" "${contents:0:50}" exit 1 fi } @@ -174,6 +175,7 @@ function python_version::parse_python_version_file() { your app to receive Python security updates. EOF meta_set "failure_reason" "python-version-file::invalid-version" + meta_set "failure_detail" "${line:0:50}" exit 1 fi ;; @@ -193,9 +195,11 @@ function python_version::parse_python_version_file() { begin with a '#', otherwise it will be treated as a comment. EOF meta_set "failure_reason" "python-version-file::no-version" + meta_set "failure_detail" "${contents:0:50}" exit 1 ;; *) + local first_five_version_lines=("${version_lines[@]:0:5}") output::error <<-EOF Error: Invalid Python version in .python-version. @@ -203,7 +207,7 @@ function python_version::parse_python_version_file() { $( IFS=$'\n' - echo "${version_lines[*]}" + echo "${first_five_version_lines[*]}" ) Update the file so it contains only one Python version. @@ -212,6 +216,10 @@ function python_version::parse_python_version_file() { lines begin with a '#', so that they are ignored. EOF meta_set "failure_reason" "python-version-file::multiple-versions" + meta_set "failure_detail" "$( + IFS=, + echo "${first_five_version_lines[*]}" + )" exit 1 ;; esac @@ -233,17 +241,19 @@ function python_version::read_pipenv_python_version() { fi if ! version=$(jq --raw-output '._meta.requires.python_full_version // ._meta.requires.python_version' "${pipfile_lock_path}" 2>&1); then + local jq_error_message="${version}" output::error <<-EOF Error: Can't parse Pipfile.lock. A Pipfile.lock file was found, however, it couldn't be parsed: - ${version} + ${jq_error_message} This is likely due to it not being valid JSON. Run 'pipenv lock' to regenerate/fix the lockfile. EOF meta_set "failure_reason" "pipfile-lock::invalid-json" + meta_set "failure_detail" "${jq_error_message:0:100}" exit 1 fi @@ -282,6 +292,7 @@ function python_version::read_pipenv_python_version() { https://pipenv.pypa.io/en/stable/specifiers.html#specifying-versions-of-python EOF meta_set "failure_reason" "pipfile-lock::invalid-version" + meta_set "failure_detail" "${version:0:50}" exit 1 fi } @@ -341,6 +352,7 @@ function python_version::resolve_python_version() { EOF fi meta_set "failure_reason" "python-version::eol" + meta_set "failure_detail" "${major}.${minor}" exit 1 fi @@ -388,6 +400,7 @@ function python_version::resolve_python_version() { EOF fi meta_set "failure_reason" "python-version::unknown-major" + meta_set "failure_detail" "${major}.${minor}" exit 1 fi diff --git a/lib/utils.sh b/lib/utils.sh index 2bcae44c2..916886431 100644 --- a/lib/utils.sh +++ b/lib/utils.sh @@ -48,10 +48,12 @@ function utils::bundled_pip_module_path() { } function utils::abort_internal_error() { - local message="${1}" + local message + message="${1} (line $(caller || true))" output::error <<-EOF - Internal error: ${message} (line $(caller || true)). + Internal error: ${message}. EOF meta_set "failure_reason" "internal-error" + meta_set "failure_detail" "${message}" exit 1 } From a840ce3862fa1289ace40965a5f3adbaf17dfa33 Mon Sep 17 00:00:00 2001 From: Ed Morley <501702+edmorley@users.noreply.github.com> Date: Wed, 12 Feb 2025 21:39:16 +0000 Subject: [PATCH 02/26] Link to changelog from runtime.txt deprecation message (#1747) Now that the changelog post exists: https://devcenter.heroku.com/changelog-items/3141 --- bin/compile | 3 ++- spec/hatchet/python_update_warning_spec.rb | 3 ++- spec/hatchet/python_version_spec.rb | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/bin/compile b/bin/compile index 10a2187ce..7fbeda2c1 100755 --- a/bin/compile +++ b/bin/compile @@ -149,7 +149,8 @@ if [[ "${python_version_origin}" == "runtime.txt" ]]; then Warning: The runtime.txt file is deprecated. The runtime.txt file is deprecated since it has been replaced - by the more widely supported .python-version file. + by the more widely supported .python-version file: + https://devcenter.heroku.com/changelog-items/3141 Please delete your runtime.txt file and create a new file named: .python-version diff --git a/spec/hatchet/python_update_warning_spec.rb b/spec/hatchet/python_update_warning_spec.rb index 23365eaba..4f7991dac 100644 --- a/spec/hatchet/python_update_warning_spec.rb +++ b/spec/hatchet/python_update_warning_spec.rb @@ -18,7 +18,8 @@ remote: ! Warning: The runtime.txt file is deprecated. remote: ! remote: ! The runtime.txt file is deprecated since it has been replaced - remote: ! by the more widely supported .python-version file. + remote: ! by the more widely supported .python-version file: + remote: ! https://devcenter.heroku.com/changelog-items/3141 remote: ! remote: ! Please delete your runtime.txt file and create a new file named: remote: ! .python-version diff --git a/spec/hatchet/python_version_spec.rb b/spec/hatchet/python_version_spec.rb index e01771661..4f8742fae 100644 --- a/spec/hatchet/python_version_spec.rb +++ b/spec/hatchet/python_version_spec.rb @@ -389,7 +389,8 @@ remote: ! Warning: The runtime.txt file is deprecated. remote: ! remote: ! The runtime.txt file is deprecated since it has been replaced - remote: ! by the more widely supported .python-version file. + remote: ! by the more widely supported .python-version file: + remote: ! https://devcenter.heroku.com/changelog-items/3141 remote: ! remote: ! Please delete your runtime.txt file and create a new file named: remote: ! .python-version From 4e8ae51d635dcdfe6e9232bb6d945fcd21ba956b Mon Sep 17 00:00:00 2001 From: Ed Morley <501702+edmorley@users.noreply.github.com> Date: Mon, 17 Feb 2025 18:01:40 +0000 Subject: [PATCH 03/26] Improve Python download/installation (#1749) Refactors the Python download/install and outdated version warning steps, improves error/warning messages and improves buildpack metrics. See the changelog entries for more details. Fixes #1701. Closes #1708. GUS-W-8059919. GUS-W-17844538. GUS-W-17844985. GUS-W-17845321. --- CHANGELOG.md | 8 +- bin/compile | 25 ++-- bin/report | 3 + bin/steps/python | 98 --------------- lib/python.sh | 139 +++++++++++++++++++++ lib/python_version.sh | 59 +++++++++ spec/hatchet/pipenv_spec.rb | 19 ++- spec/hatchet/poetry_spec.rb | 19 ++- spec/hatchet/python_update_warning_spec.rb | 38 ++++-- spec/hatchet/python_version_spec.rb | 33 ++++- 10 files changed, 308 insertions(+), 133 deletions(-) delete mode 100755 bin/steps/python create mode 100644 lib/python.sh diff --git a/CHANGELOG.md b/CHANGELOG.md index 7be1eea19..2abd8137e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,13 @@ ## [Unreleased] -- Improved buildpack metrics for builds that fail. ([#1746](https://github.com/heroku/heroku-buildpack-python/pull/1746)) +- Improved the warning message shown when the requested Python version is not the latest patch version. ([#1749](https://github.com/heroku/heroku-buildpack-python/pull/1749)) +- Improved the error message shown when the requested Python patch version isn't available. ([#1749](https://github.com/heroku/heroku-buildpack-python/pull/1749)) +- Improved the error message shown if there was a networking or server related error downloading Python. ([#1749](https://github.com/heroku/heroku-buildpack-python/pull/1749)) +- Adjusted the curl options used when downloading Python to set a maximum download time of 120s to prevent hanging builds in the case of network issues. ([#1749](https://github.com/heroku/heroku-buildpack-python/pull/1749)) +- Refactored the Python download step to avoid an unnecessary version check `HEAD` request to S3 prior to downloading Python or reusing a cached install. ([#1749](https://github.com/heroku/heroku-buildpack-python/pull/1749)) +- Improved buildpack metrics for Python version selection. ([#1749](https://github.com/heroku/heroku-buildpack-python/pull/1749)) +- Improved buildpack metrics for builds that fail. ([#1746](https://github.com/heroku/heroku-buildpack-python/pull/1746) and [#1749](https://github.com/heroku/heroku-buildpack-python/pull/1749)) ## [v276] - 2025-02-05 diff --git a/bin/compile b/bin/compile index 7fbeda2c1..727966b06 100755 --- a/bin/compile +++ b/bin/compile @@ -28,8 +28,9 @@ source "${BUILDPACK_DIR}/lib/output.sh" source "${BUILDPACK_DIR}/lib/package_manager.sh" source "${BUILDPACK_DIR}/lib/pip.sh" source "${BUILDPACK_DIR}/lib/pipenv.sh" -source "${BUILDPACK_DIR}/lib/python_version.sh" source "${BUILDPACK_DIR}/lib/poetry.sh" +source "${BUILDPACK_DIR}/lib/python_version.sh" +source "${BUILDPACK_DIR}/lib/python.sh" compile_start_time=$(nowms) @@ -49,13 +50,6 @@ export PATH=:/usr/local/bin:$PATH # Exported for use in subshells, such as the steps run via sub_env. export BUILD_DIR CACHE_DIR ENV_DIR -# Set the base URL for downloading buildpack assets like Python runtimes. -# The user can provide BUILDPACK_S3_BASE_URL to specify a custom target. -# Note: this is designed for non-Heroku use, as it does not use the user-provided -# environment variable mechanism (the ENV_DIR). -S3_BASE_URL="${BUILDPACK_S3_BASE_URL:-"https://heroku-buildpack-python.s3.us-east-1.amazonaws.com"}" -# This has to be exported since it's used by the geo-libs step which is run in a subshell. - # Common Problem Warnings: # This section creates a temporary file in which to stick the output of `pip install`. # The `warnings` subscript then greps through this for common problems and guides @@ -121,6 +115,7 @@ cached_python_full_version="$(cache::cached_python_full_version "${CACHE_DIR}")" # We use the Bash 4.3+ `nameref` feature to pass back multiple values from this function # without having to hardcode globals. See: https://stackoverflow.com/a/38997681 python_version::read_requested_python_version "${BUILD_DIR}" "${package_manager}" "${cached_python_full_version}" requested_python_version python_version_origin +meta_set "python_version_requested" "${requested_python_version}" meta_set "python_version_reason" "${python_version_origin}" # TODO: More strongly recommend specifying a Python version (eg switch the messaging to @@ -144,6 +139,12 @@ python_major_version="${python_full_version%.*}" meta_set "python_version" "${python_full_version}" meta_set "python_version_major" "${python_major_version}" +if [[ "${requested_python_version}" == "${python_full_version}" ]]; then + meta_set "python_version_pinned" "true" +else + meta_set "python_version_pinned" "false" +fi + if [[ "${python_version_origin}" == "runtime.txt" ]]; then output::warning <<-EOF Warning: The runtime.txt file is deprecated. @@ -170,6 +171,9 @@ if [[ "${python_version_origin}" == "runtime.txt" ]]; then EOF fi +python_version::warn_if_deprecated_major_version "${python_major_version}" "${python_version_origin}" +python_version::warn_if_patch_update_available "${python_full_version}" "${python_major_version}" "${python_version_origin}" + cache::restore "${BUILD_DIR}" "${CACHE_DIR}" "${STACK}" "${cached_python_full_version}" "${python_full_version}" "${package_manager}" # The directory for the .profile.d scripts. @@ -190,10 +194,7 @@ if [[ "$(realpath "${BUILD_DIR}")" != "$(realpath /app)" ]]; then # Note: .heroku/src is copied in later. fi -# Download and install Python using pre-built binaries from S3. -install_python_start_time=$(nowms) -source "${BUILDPACK_DIR}/bin/steps/python" -meta_time "python_install_duration" "${install_python_start_time}" +python::install "${BUILD_DIR}" "${STACK}" "${python_full_version}" "${python_major_version}" "${python_version_origin}" # Install the package manager and related tools. package_manager_install_start_time=$(nowms) diff --git a/bin/report b/bin/report index cc0d1a116..9a3eca067 100755 --- a/bin/report +++ b/bin/report @@ -72,6 +72,7 @@ STRING_FIELDS=( poetry_version python_version_major python_version_reason + python_version_requested python_version setuptools_version wheel_version @@ -81,6 +82,7 @@ STRING_FIELDS=( ALL_OTHER_FIELDS=( cache_restore_duration cache_save_duration + custom_s3_base_url dependencies_install_duration django_collectstatic_duration duplicate_python_buildpack @@ -94,6 +96,7 @@ ALL_OTHER_FIELDS=( pre_compile_hook_duration python_install_duration python_version_outdated + python_version_pinned setup_py_only sqlite_install_duration total_duration diff --git a/bin/steps/python b/bin/steps/python deleted file mode 100755 index 6855448de..000000000 --- a/bin/steps/python +++ /dev/null @@ -1,98 +0,0 @@ -#!/usr/bin/env bash -# shellcheck disable=SC2154 # TODO: Env var is referenced but not assigned. - -set -euo pipefail - -# The Python runtime archive filename is of form: 'python-X.Y.Z-ubuntu-22.04-amd64.tar.zst' -# The Ubuntu version is calculated from `STACK` since it's faster than calling `lsb_release`. -UBUNTU_VERSION="${STACK/heroku-/}.04" -ARCH=$(dpkg --print-architecture) -PYTHON_URL="${S3_BASE_URL}/python-${python_full_version}-ubuntu-${UBUNTU_VERSION}-${ARCH}.tar.zst" - -# The Python version validation earlier will have filtered out most unsupported versions. -# However, the version might still not be found if either: -# 1. It's a Python major version we've deprecated and so is only available on older stacks (i.e: Python 3.8). -# 2. If an exact Python version was requested and the patch version doesn't exist (e.g. 3.12.999). -# 3. The user has pinned to an older buildpack version and the S3 bucket location or layout has changed since. -# TODO: Update this message to be more specific now that Python 3.8 support has been removed, and so (1) can no longer occur. -if ! curl --output /dev/null --silent --head --fail --retry 3 --retry-connrefused --connect-timeout 10 "${PYTHON_URL}"; then - output::error <<-EOF - Error: Python ${python_full_version} isn't available for this stack (${STACK}). - - For a list of the supported Python versions, see: - https://devcenter.heroku.com/articles/python-support#supported-python-versions - EOF - meta_set "failure_reason" "python-version-not-found" - meta_set "failure_detail" "${python_full_version}" - exit 1 -fi - -if [[ -f "${BUILD_DIR}/.heroku/python/bin/python" ]]; then - output::step "Using cached install of Python ${python_full_version}" -else - output::step "Installing Python ${python_full_version}" - mkdir -p "${BUILD_DIR}/.heroku/python" - - if ! curl --silent --show-error --fail --retry 3 --retry-connrefused --connect-timeout 10 "${PYTHON_URL}" | tar --zstd --extract --directory "${BUILD_DIR}/.heroku/python"; then - # The Python version was confirmed to exist previously, so any failure here is due to - # a networking issue or archive/buildpack bug rather than the runtime not existing. - output::error <<-EOF - Error: Failed to download/install Python ${python_full_version}. - - In some cases, this happens due to an unstable network connection. - Please try again and to see if the error resolves itself. - EOF - meta_set "failure_reason" "python-download" - # TODO: Set failure_detail here once refactored. - exit 1 - fi - - hash -r -fi - -function warn_if_patch_update_available() { - local requested_full_version="${1}" - local requested_major_version="${2}" - local latest_patch_version - latest_patch_version="$(python_version::resolve_python_version "${requested_major_version}" "${python_version_origin}")" - # Extract the patch version component of the version strings (ie: the '5' in '3.10.5'). - local requested_patch_number="${requested_full_version##*.}" - local latest_patch_number="${latest_patch_version##*.}" - # TODO: Update this message to suggest using the .python-version major version syntax to stay up to date, - # once runtime.txt is deprecated and sticky-versioning only pins to the major version. - if ((requested_patch_number < latest_patch_number)); then - output::warning <<-EOF - Warning: A Python security update is available! - - Upgrade as soon as possible to: Python ${latest_patch_version} - See: https://devcenter.heroku.com/articles/python-runtimes - EOF - meta_set "python_version_outdated" "true" - else - meta_set "python_version_outdated" "false" - fi -} - -# We wait until now to display outdated Python version warnings, since we only want to show them -# if there weren't any errors with the version to avoid adding noise to the error messages. -# TODO: Move this into lib/ as part of the warnings refactor. -if [[ "${python_major_version}" == "3.9" ]]; then - output::warning <<-EOF - Warning: Support for Python 3.9 is ending soon! - - Python 3.9 will reach its upstream end-of-life in October 2025, - at which point it will no longer receive security updates: - https://devguide.python.org/versions/#supported-versions - - As such, support for Python 3.9 will be removed from this - buildpack on 7th January 2026. - - Upgrade to a newer Python version as soon as possible, by - changing the version in your ${python_version_origin} file. - - For more information, see: - https://devcenter.heroku.com/articles/python-support#supported-python-versions - EOF -fi - -warn_if_patch_update_available "${python_full_version}" "${python_major_version}" diff --git a/lib/python.sh b/lib/python.sh new file mode 100644 index 000000000..4bbeb344a --- /dev/null +++ b/lib/python.sh @@ -0,0 +1,139 @@ +#!/usr/bin/env bash + +# This is technically redundant, since all consumers of this lib will have enabled these, +# however, it helps Shellcheck realise the options under which these functions will run. +set -euo pipefail + +DEFAULT_S3_BASE_URL="https://heroku-buildpack-python.s3.us-east-1.amazonaws.com" + +function python::install() { + local build_dir="${1}" + local stack="${2}" + local python_full_version="${3}" + local python_major_version="${4}" + local python_version_origin="${5}" + + local install_python_start_time + install_python_start_time=$(nowms) + local install_dir="${build_dir}/.heroku/python" + + if [[ -f "${install_dir}/bin/python" ]]; then + output::step "Using cached install of Python ${python_full_version}" + else + output::step "Installing Python ${python_full_version}" + + mkdir -p "${install_dir}" + + # Note: This can't be used via app config vars, since it doesn't reference the value from ENV_DIR. + # TODO: Remove this for parity with the Python CNB, if metrics show it to be unused on Heroku. + if [[ -v BUILDPACK_S3_BASE_URL ]]; then + local s3_base_url="${BUILDPACK_S3_BASE_URL}" + meta_set "custom_s3_base_url" "true" + else + local s3_base_url="${DEFAULT_S3_BASE_URL}" + fi + + # Calculating the Ubuntu version from the stack name saves having to shell out to `lsb_release`. + local ubuntu_version="${stack/heroku-/}.04" + local arch + arch=$(dpkg --print-architecture) + # e.g.: https://heroku-buildpack-python.s3.us-east-1.amazonaws.com/python-3.13.0-ubuntu-24.04-amd64.tar.zst + local python_url="${s3_base_url}/python-${python_full_version}-ubuntu-${ubuntu_version}-${arch}.tar.zst" + + local error_log + error_log=$(mktemp) + + # shellcheck disable=SC2310 # This function is invoked in an 'if' condition so set -e will be disabled. + if ! { + { + # We set max-time for improved UX/metrics for hanging downloads compared to relying + # on the build system timeout. The Python archives are only ~10 MB so take < 1s to + # download on Heroku's build system, however, we use much higher timeouts so that + # the buildpack works in non-Heroku environments that may be far from `us-east-1` + # or have a slower connection. We don't use `--speed-limit` since it gives worse + # error messages when used with retries and piping to tar. + curl \ + --connect-timeout 10 \ + --fail \ + --max-time 120 \ + --retry-max-time 120 \ + --retry 3 \ + --retry-connrefused \ + --show-error \ + --silent \ + "${python_url}" \ + | tar \ + --directory "${install_dir}" \ + --extract \ + --zstd + } \ + |& tee "${error_log}" \ + |& output::indent + }; then + local latest_known_patch_version + latest_known_patch_version="$(python_version::resolve_python_version "${python_major_version}" "${python_version_origin}")" + # Ideally we would inspect the HTTP status code directly instead of grepping, however: + # 1. We want to pipe to tar (since it's faster than performing the download and + # decompression/extraction as separate steps), so can't write to stdout. + # 2. We want to display the original stderr to the user, so can't write to stderr. + # 3. Curl's `--write-out` feature only supports outputting to a file (as opposed to + # stdout/stderr) as of curl v8.3.0, which is newer than the curl on Heroku-20/22. + # This has an integration test run against all stacks, which will mean we will know + # if future versions of curl change the error message string. + # + # We have to check for HTTP 403s too, since S3 will return a 403 instead of a 404 for + # missing files, if the S3 bucket does not have public list permissions enabled. + if [[ "${python_full_version}" != "${latest_known_patch_version}" ]] && grep --quiet "The requested URL returned error: 40[34]" "${error_log}"; then + output::error <<-EOF + Error: The requested Python version isn't available. + + Your app's ${python_version_origin} file specifies a Python version + of ${python_full_version}, however, we couldn't find that version on S3. + + Check that this Python version has been released upstream, + and that the Python buildpack has added support for it: + https://www.python.org/downloads/ + https://github.com/heroku/heroku-buildpack-python/blob/main/CHANGELOG.md + + If it has, make sure that you are using the latest version + of this buildpack, and haven't pinned to an older release: + https://devcenter.heroku.com/articles/managing-buildpacks#view-your-buildpacks + https://devcenter.heroku.com/articles/managing-buildpacks#classic-buildpacks-references + + We also strongly recommend that you do not pin your app to an + exact Python version such as ${python_full_version}, and instead only specify + the major Python version of ${python_major_version} in your ${python_version_origin} file. + This will allow your app to receive the latest available Python + patch version automatically, and prevent this type of error. + EOF + meta_set "failure_reason" "python-version::unknown-patch" + meta_set "failure_detail" "${python_full_version}" + else + output::error <<-EOF + Error: Unable to download/install Python. + + An error occurred while downloading/installing the Python + runtime archive from: + ${python_url} + + In some cases, this happens due to a temporary issue with + the network connection or server. + + First, make sure that you are using the latest version + of this buildpack, and haven't pinned to an older release: + https://devcenter.heroku.com/articles/managing-buildpacks#view-your-buildpacks + https://devcenter.heroku.com/articles/managing-buildpacks#classic-buildpacks-references + + Then try building again to see if the error resolves itself. + EOF + meta_set "failure_reason" "install-python" + # e.g.: 'curl: (6) Could not resolve host: heroku-buildpack-python.s3.us-east-1.amazonaws.com' + meta_set "failure_detail" "$(head --lines=1 "${error_log}" || true)" + fi + + exit 1 + fi + fi + + meta_time "python_install_duration" "${install_python_start_time}" +} diff --git a/lib/python_version.sh b/lib/python_version.sh index 652c4e478..4405b7650 100644 --- a/lib/python_version.sh +++ b/lib/python_version.sh @@ -416,3 +416,62 @@ function python_version::resolve_python_version() { *) utils::abort_internal_error "Unhandled Python major version: ${requested_python_version}" ;; esac } + +function python_version::warn_if_deprecated_major_version() { + local requested_major_version="${1}" + local version_origin="${2}" + + if [[ "${requested_major_version}" == "3.9" ]]; then + output::warning <<-EOF + Warning: Support for Python 3.9 is ending soon! + + Python 3.9 will reach its upstream end-of-life in October 2025, + at which point it will no longer receive security updates: + https://devguide.python.org/versions/#supported-versions + + As such, support for Python 3.9 will be removed from this + buildpack on 7th January 2026. + + Upgrade to a newer Python version as soon as possible, by + changing the version in your ${version_origin} file. + + For more information, see: + https://devcenter.heroku.com/articles/python-support#supported-python-versions + EOF + fi +} + +function python_version::warn_if_patch_update_available() { + local python_full_version="${1}" + local python_major_version="${2}" + local python_version_origin="${3}" + + local latest_known_patch_version + latest_known_patch_version="$(python_version::resolve_python_version "${python_major_version}" "${python_version_origin}")" + # Extract the patch version component of the version strings (ie: the '2' in '3.13.2'). + local requested_patch_number="${python_full_version##*.}" + local latest_patch_number="${latest_known_patch_version##*.}" + + if ((requested_patch_number < latest_patch_number)); then + output::warning <<-EOF + Warning: A Python patch update is available! + + Your app is using Python ${python_full_version}, however, there is a newer + patch release of Python ${python_major_version} available: ${latest_known_patch_version} + + It is important to always use the latest patch version of + Python to keep your app secure. + + Update your ${python_version_origin} file to use the new version. + + We strongly recommend that you do not pin your app to an + exact Python version such as ${python_full_version}, and instead only specify + the major Python version of ${python_major_version} in your ${python_version_origin} file. + This will allow your app to receive the latest available Python + patch version automatically and prevent this warning. + EOF + meta_set "python_version_outdated" "true" + else + meta_set "python_version_outdated" "false" + fi +} diff --git a/spec/hatchet/pipenv_spec.rb b/spec/hatchet/pipenv_spec.rb index 6949e3934..7a657ae82 100644 --- a/spec/hatchet/pipenv_spec.rb +++ b/spec/hatchet/pipenv_spec.rb @@ -78,7 +78,6 @@ expect(clean_output(app.output)).to match(Regexp.new(<<~REGEX)) remote: -----> Python app detected remote: -----> Using Python 3.9.0 specified in Pipfile.lock - remote: -----> Installing Python 3.9.0 remote: remote: ! Warning: Support for Python 3.9 is ending soon! remote: ! @@ -96,11 +95,23 @@ remote: ! https://devcenter.heroku.com/articles/python-support#supported-python-versions remote: remote: - remote: ! Warning: A Python security update is available! + remote: ! Warning: A Python patch update is available! + remote: ! + remote: ! Your app is using Python 3.9.0, however, there is a newer + remote: ! patch release of Python 3.9 available: #{LATEST_PYTHON_3_9} + remote: ! + remote: ! It is important to always use the latest patch version of + remote: ! Python to keep your app secure. remote: ! - remote: ! Upgrade as soon as possible to: Python #{LATEST_PYTHON_3_9} - remote: ! See: https://devcenter.heroku.com/articles/python-runtimes + remote: ! Update your Pipfile.lock file to use the new version. + remote: ! + remote: ! We strongly recommend that you do not pin your app to an + remote: ! exact Python version such as 3.9.0, and instead only specify + remote: ! the major Python version of 3.9 in your Pipfile.lock file. + remote: ! This will allow your app to receive the latest available Python + remote: ! patch version automatically and prevent this warning. remote: + remote: -----> Installing Python 3.9.0 remote: -----> Installing pip #{PIP_VERSION}, setuptools #{SETUPTOOLS_VERSION} and wheel #{WHEEL_VERSION} remote: -----> Installing Pipenv #{PIPENV_VERSION} remote: -----> Installing SQLite3 diff --git a/spec/hatchet/poetry_spec.rb b/spec/hatchet/poetry_spec.rb index e7e85d446..bad760d30 100644 --- a/spec/hatchet/poetry_spec.rb +++ b/spec/hatchet/poetry_spec.rb @@ -168,7 +168,6 @@ expect(clean_output(app.output)).to include(<<~OUTPUT) remote: -----> Python app detected remote: -----> Using Python 3.9.0 specified in .python-version - remote: -----> Installing Python 3.9.0 remote: remote: ! Warning: Support for Python 3.9 is ending soon! remote: ! @@ -186,11 +185,23 @@ remote: ! https://devcenter.heroku.com/articles/python-support#supported-python-versions remote: remote: - remote: ! Warning: A Python security update is available! + remote: ! Warning: A Python patch update is available! + remote: ! + remote: ! Your app is using Python 3.9.0, however, there is a newer + remote: ! patch release of Python 3.9 available: #{LATEST_PYTHON_3_9} + remote: ! + remote: ! It is important to always use the latest patch version of + remote: ! Python to keep your app secure. remote: ! - remote: ! Upgrade as soon as possible to: Python #{LATEST_PYTHON_3_9} - remote: ! See: https://devcenter.heroku.com/articles/python-runtimes + remote: ! Update your .python-version file to use the new version. + remote: ! + remote: ! We strongly recommend that you do not pin your app to an + remote: ! exact Python version such as 3.9.0, and instead only specify + remote: ! the major Python version of 3.9 in your .python-version file. + remote: ! This will allow your app to receive the latest available Python + remote: ! patch version automatically and prevent this warning. remote: + remote: -----> Installing Python 3.9.0 remote: -----> Installing Poetry #{POETRY_VERSION} remote: -----> Installing dependencies using 'poetry sync --only main' remote: Installing dependencies from lock file diff --git a/spec/hatchet/python_update_warning_spec.rb b/spec/hatchet/python_update_warning_spec.rb index 4f7991dac..c1025f415 100644 --- a/spec/hatchet/python_update_warning_spec.rb +++ b/spec/hatchet/python_update_warning_spec.rb @@ -37,7 +37,6 @@ remote: ! In the future support for runtime.txt will be removed and remote: ! this warning will be made an error. remote: - remote: -----> Installing Python 3.9.0 remote: remote: ! Warning: Support for Python 3.9 is ending soon! remote: ! @@ -55,11 +54,23 @@ remote: ! https://devcenter.heroku.com/articles/python-support#supported-python-versions remote: remote: - remote: ! Warning: A Python security update is available! + remote: ! Warning: A Python patch update is available! + remote: ! + remote: ! Your app is using Python 3.9.0, however, there is a newer + remote: ! patch release of Python 3.9 available: #{LATEST_PYTHON_3_9} + remote: ! + remote: ! It is important to always use the latest patch version of + remote: ! Python to keep your app secure. remote: ! - remote: ! Upgrade as soon as possible to: Python #{LATEST_PYTHON_3_9} - remote: ! See: https://devcenter.heroku.com/articles/python-runtimes + remote: ! Update your runtime.txt file to use the new version. + remote: ! + remote: ! We strongly recommend that you do not pin your app to an + remote: ! exact Python version such as 3.9.0, and instead only specify + remote: ! the major Python version of 3.9 in your runtime.txt file. + remote: ! This will allow your app to receive the latest available Python + remote: ! patch version automatically and prevent this warning. remote: + remote: -----> Installing Python 3.9.0 remote: -----> Installing pip #{PIP_VERSION}, setuptools #{SETUPTOOLS_VERSION} and wheel #{WHEEL_VERSION} OUTPUT end @@ -74,13 +85,24 @@ expect(clean_output(app.output)).to include(<<~OUTPUT) remote: -----> Python app detected remote: -----> Using Python 3.10.0 specified in .python-version - remote: -----> Installing Python 3.10.0 remote: - remote: ! Warning: A Python security update is available! + remote: ! Warning: A Python patch update is available! + remote: ! + remote: ! Your app is using Python 3.10.0, however, there is a newer + remote: ! patch release of Python 3.10 available: #{LATEST_PYTHON_3_10} + remote: ! + remote: ! It is important to always use the latest patch version of + remote: ! Python to keep your app secure. remote: ! - remote: ! Upgrade as soon as possible to: Python #{LATEST_PYTHON_3_10} - remote: ! See: https://devcenter.heroku.com/articles/python-runtimes + remote: ! Update your .python-version file to use the new version. + remote: ! + remote: ! We strongly recommend that you do not pin your app to an + remote: ! exact Python version such as 3.10.0, and instead only specify + remote: ! the major Python version of 3.10 in your .python-version file. + remote: ! This will allow your app to receive the latest available Python + remote: ! patch version automatically and prevent this warning. remote: + remote: -----> Installing Python 3.10.0 remote: -----> Installing pip #{PIP_VERSION}, setuptools #{SETUPTOOLS_VERSION} and wheel #{WHEEL_VERSION} OUTPUT end diff --git a/spec/hatchet/python_version_spec.rb b/spec/hatchet/python_version_spec.rb index 4f8742fae..639b67d1c 100644 --- a/spec/hatchet/python_version_spec.rb +++ b/spec/hatchet/python_version_spec.rb @@ -87,7 +87,6 @@ expect(clean_output(app.output)).to include(<<~OUTPUT) remote: -----> Python app detected remote: -----> Using Python 3.9 specified in .python-version - remote: -----> Installing Python #{LATEST_PYTHON_3_9} remote: remote: ! Warning: Support for Python 3.9 is ending soon! remote: ! @@ -104,6 +103,7 @@ remote: ! For more information, see: remote: ! https://devcenter.heroku.com/articles/python-support#supported-python-versions remote: + remote: -----> Installing Python #{LATEST_PYTHON_3_9} remote: -----> Installing pip #{PIP_VERSION}, setuptools #{SETUPTOOLS_VERSION} and wheel #{WHEEL_VERSION} remote: -----> Installing SQLite3 remote: -----> Installing dependencies using 'pip install -r requirements.txt' @@ -293,17 +293,38 @@ it 'aborts the build with a version not available message' do app.deploy do |app| - expect(clean_output(app.output)).to include(<<~OUTPUT) + expect(clean_output(app.output)).to match(Regexp.new(<<~REGEX)) remote: -----> Python app detected remote: -----> Using Python 3.12.999 specified in .python-version + remote: -----> Installing Python 3.12.999 + remote: curl: \\(22\\) The requested URL returned error: 404.* + remote: zstd: /\\*stdin\\*\\\\: unexpected end of file + remote: tar: Child returned status 1 + remote: tar: Error is not recoverable: exiting now remote: - remote: ! Error: Python 3.12.999 isn't available for this stack (#{app.stack}). + remote: ! Error: The requested Python version isn't available. remote: ! - remote: ! For a list of the supported Python versions, see: - remote: ! https://devcenter.heroku.com/articles/python-support#supported-python-versions + remote: ! Your app's .python-version file specifies a Python version + remote: ! of 3.12.999, however, we couldn't find that version on S3. + remote: ! + remote: ! Check that this Python version has been released upstream, + remote: ! and that the Python buildpack has added support for it: + remote: ! https://www.python.org/downloads/ + remote: ! https://github.com/heroku/heroku-buildpack-python/blob/main/CHANGELOG.md + remote: ! + remote: ! If it has, make sure that you are using the latest version + remote: ! of this buildpack, and haven't pinned to an older release: + remote: ! https://devcenter.heroku.com/articles/managing-buildpacks#view-your-buildpacks + remote: ! https://devcenter.heroku.com/articles/managing-buildpacks#classic-buildpacks-references + remote: ! + remote: ! We also strongly recommend that you do not pin your app to an + remote: ! exact Python version such as 3.12.999, and instead only specify + remote: ! the major Python version of 3.12 in your .python-version file. + remote: ! This will allow your app to receive the latest available Python + remote: ! patch version automatically, and prevent this type of error. remote: remote: ! Push rejected, failed to compile Python app. - OUTPUT + REGEX end end end From fc441449edad0a631352ad51cd10f54677a1c01b Mon Sep 17 00:00:00 2001 From: "heroku-linguist[bot]" <136119646+heroku-linguist[bot]@users.noreply.github.com> Date: Mon, 17 Feb 2025 18:16:24 +0000 Subject: [PATCH 04/26] Prepare release v277 (#1750) * Prepare release v277 * Update changelog --------- Co-authored-by: heroku-linguist[bot] <136119646+heroku-linguist[bot]@users.noreply.github.com> Co-authored-by: Ed Morley <501702+edmorley@users.noreply.github.com> --- CHANGELOG.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2abd8137e..0820beac7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,11 +2,15 @@ ## [Unreleased] + +## [v277] - 2025-02-17 + - Improved the warning message shown when the requested Python version is not the latest patch version. ([#1749](https://github.com/heroku/heroku-buildpack-python/pull/1749)) - Improved the error message shown when the requested Python patch version isn't available. ([#1749](https://github.com/heroku/heroku-buildpack-python/pull/1749)) - Improved the error message shown if there was a networking or server related error downloading Python. ([#1749](https://github.com/heroku/heroku-buildpack-python/pull/1749)) - Adjusted the curl options used when downloading Python to set a maximum download time of 120s to prevent hanging builds in the case of network issues. ([#1749](https://github.com/heroku/heroku-buildpack-python/pull/1749)) - Refactored the Python download step to avoid an unnecessary version check `HEAD` request to S3 prior to downloading Python or reusing a cached install. ([#1749](https://github.com/heroku/heroku-buildpack-python/pull/1749)) +- Updated the `runtime.txt` deprecation warning to include a link to the deprecation changelog post. ([#1747](https://github.com/heroku/heroku-buildpack-python/pull/1747)) - Improved buildpack metrics for Python version selection. ([#1749](https://github.com/heroku/heroku-buildpack-python/pull/1749)) - Improved buildpack metrics for builds that fail. ([#1746](https://github.com/heroku/heroku-buildpack-python/pull/1746) and [#1749](https://github.com/heroku/heroku-buildpack-python/pull/1749)) @@ -1164,7 +1168,8 @@ Default Python is now latest 2.7.10. Updated pip and Distribute. - Setuptools updated to v16.0 - pip updated to v7.0.1 -[unreleased]: https://github.com/heroku/heroku-buildpack-python/compare/v276...main +[unreleased]: https://github.com/heroku/heroku-buildpack-python/compare/v277...main +[v277]: https://github.com/heroku/heroku-buildpack-python/compare/v276...v277 [v276]: https://github.com/heroku/heroku-buildpack-python/compare/v275...v276 [v275]: https://github.com/heroku/heroku-buildpack-python/compare/v274...v275 [v274]: https://github.com/heroku/heroku-buildpack-python/compare/v273...v274 From 51193fa1c429a3b0899d1e7c7e08d22382083a31 Mon Sep 17 00:00:00 2001 From: Ed Morley <501702+edmorley@users.noreply.github.com> Date: Wed, 19 Feb 2025 12:29:26 +0000 Subject: [PATCH 05/26] Update changelog links and tests after Git tag archival (#1752) The Git tags for buildpack releases v208 to v265 have been archived (renamed from `vNNN` to `archive/vNNN`) for the reasons in #1699 (similar to the process performed in the past for release v207 and older). As such, the changelog compare URLs need updating, as do the test fixtures that test building an app whose last (cached) build was performed using an older version of the buildpack. Closes #1699. GUS-W-17308840. --- CHANGELOG.md | 118 ++++++++++++++-------------- spec/hatchet/pipenv_spec.rb | 3 +- spec/hatchet/poetry_spec.rb | 6 +- spec/hatchet/python_version_spec.rb | 4 +- spec/hatchet/stack_spec.rb | 12 +-- 5 files changed, 72 insertions(+), 71 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0820beac7..e51af3572 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1180,65 +1180,65 @@ Default Python is now latest 2.7.10. Updated pip and Distribute. [v269]: https://github.com/heroku/heroku-buildpack-python/compare/v268...v269 [v268]: https://github.com/heroku/heroku-buildpack-python/compare/v267...v268 [v267]: https://github.com/heroku/heroku-buildpack-python/compare/v266...v267 -[v266]: https://github.com/heroku/heroku-buildpack-python/compare/v265...v266 -[v265]: https://github.com/heroku/heroku-buildpack-python/compare/v264...v265 -[v264]: https://github.com/heroku/heroku-buildpack-python/compare/v263...v264 -[v263]: https://github.com/heroku/heroku-buildpack-python/compare/v262...v263 -[v262]: https://github.com/heroku/heroku-buildpack-python/compare/v261...v262 -[v261]: https://github.com/heroku/heroku-buildpack-python/compare/v260...v261 -[v260]: https://github.com/heroku/heroku-buildpack-python/compare/v259...v260 -[v259]: https://github.com/heroku/heroku-buildpack-python/compare/v258...v259 -[v258]: https://github.com/heroku/heroku-buildpack-python/compare/v257...v258 -[v257]: https://github.com/heroku/heroku-buildpack-python/compare/v256...v257 -[v256]: https://github.com/heroku/heroku-buildpack-python/compare/v255...v256 -[v255]: https://github.com/heroku/heroku-buildpack-python/compare/v254...v255 -[v254]: https://github.com/heroku/heroku-buildpack-python/compare/v253...v254 -[v253]: https://github.com/heroku/heroku-buildpack-python/compare/v252...v253 -[v252]: https://github.com/heroku/heroku-buildpack-python/compare/v251...v252 -[v251]: https://github.com/heroku/heroku-buildpack-python/compare/v250...v251 -[v250]: https://github.com/heroku/heroku-buildpack-python/compare/v249...v250 -[v249]: https://github.com/heroku/heroku-buildpack-python/compare/v248...v249 -[v248]: https://github.com/heroku/heroku-buildpack-python/compare/v247...v248 -[v247]: https://github.com/heroku/heroku-buildpack-python/compare/v246...v247 -[v246]: https://github.com/heroku/heroku-buildpack-python/compare/v245...v246 -[v245]: https://github.com/heroku/heroku-buildpack-python/compare/v244...v245 -[v244]: https://github.com/heroku/heroku-buildpack-python/compare/v243...v244 -[v243]: https://github.com/heroku/heroku-buildpack-python/compare/v242...v243 -[v242]: https://github.com/heroku/heroku-buildpack-python/compare/v241...v242 -[v241]: https://github.com/heroku/heroku-buildpack-python/compare/v240...v241 -[v240]: https://github.com/heroku/heroku-buildpack-python/compare/v239...v240 -[v239]: https://github.com/heroku/heroku-buildpack-python/compare/v238...v239 -[v238]: https://github.com/heroku/heroku-buildpack-python/compare/v237...v238 -[v237]: https://github.com/heroku/heroku-buildpack-python/compare/v236...v237 -[v236]: https://github.com/heroku/heroku-buildpack-python/compare/v235...v236 -[v235]: https://github.com/heroku/heroku-buildpack-python/compare/v234...v235 -[v234]: https://github.com/heroku/heroku-buildpack-python/compare/v233...v234 -[v233]: https://github.com/heroku/heroku-buildpack-python/compare/v232...v233 -[v232]: https://github.com/heroku/heroku-buildpack-python/compare/v231...v232 -[v231]: https://github.com/heroku/heroku-buildpack-python/compare/v230...v231 -[v230]: https://github.com/heroku/heroku-buildpack-python/compare/v229...v230 -[v229]: https://github.com/heroku/heroku-buildpack-python/compare/v228...v229 -[v228]: https://github.com/heroku/heroku-buildpack-python/compare/v227...v228 -[v227]: https://github.com/heroku/heroku-buildpack-python/compare/v226...v227 -[v226]: https://github.com/heroku/heroku-buildpack-python/compare/v225...v226 -[v225]: https://github.com/heroku/heroku-buildpack-python/compare/v224...v225 -[v224]: https://github.com/heroku/heroku-buildpack-python/compare/v223...v224 -[v223]: https://github.com/heroku/heroku-buildpack-python/compare/v222...v223 -[v222]: https://github.com/heroku/heroku-buildpack-python/compare/v221...v222 -[v221]: https://github.com/heroku/heroku-buildpack-python/compare/v220...v221 -[v220]: https://github.com/heroku/heroku-buildpack-python/compare/v219...v220 -[v219]: https://github.com/heroku/heroku-buildpack-python/compare/v218...v219 -[v218]: https://github.com/heroku/heroku-buildpack-python/compare/v217...v218 -[v217]: https://github.com/heroku/heroku-buildpack-python/compare/v216...v217 -[v216]: https://github.com/heroku/heroku-buildpack-python/compare/v215...v216 -[v215]: https://github.com/heroku/heroku-buildpack-python/compare/v214...v215 -[v214]: https://github.com/heroku/heroku-buildpack-python/compare/v213...v214 -[v213]: https://github.com/heroku/heroku-buildpack-python/compare/v212...v213 -[v212]: https://github.com/heroku/heroku-buildpack-python/compare/v211...v212 -[v211]: https://github.com/heroku/heroku-buildpack-python/compare/v210...v211 -[v210]: https://github.com/heroku/heroku-buildpack-python/compare/v209...v210 -[v209]: https://github.com/heroku/heroku-buildpack-python/compare/v208...v209 -[v208]: https://github.com/heroku/heroku-buildpack-python/compare/archive/v207...v208 +[v266]: https://github.com/heroku/heroku-buildpack-python/compare/archive/v265...v266 +[v265]: https://github.com/heroku/heroku-buildpack-python/compare/archive/v264...archive/v265 +[v264]: https://github.com/heroku/heroku-buildpack-python/compare/archive/v263...archive/v264 +[v263]: https://github.com/heroku/heroku-buildpack-python/compare/archive/v262...archive/v263 +[v262]: https://github.com/heroku/heroku-buildpack-python/compare/archive/v261...archive/v262 +[v261]: https://github.com/heroku/heroku-buildpack-python/compare/archive/v260...archive/v261 +[v260]: https://github.com/heroku/heroku-buildpack-python/compare/archive/v259...archive/v260 +[v259]: https://github.com/heroku/heroku-buildpack-python/compare/archive/v258...archive/v259 +[v258]: https://github.com/heroku/heroku-buildpack-python/compare/archive/v257...archive/v258 +[v257]: https://github.com/heroku/heroku-buildpack-python/compare/archive/v256...archive/v257 +[v256]: https://github.com/heroku/heroku-buildpack-python/compare/archive/v255...archive/v256 +[v255]: https://github.com/heroku/heroku-buildpack-python/compare/archive/v254...archive/v255 +[v254]: https://github.com/heroku/heroku-buildpack-python/compare/archive/v253...archive/v254 +[v253]: https://github.com/heroku/heroku-buildpack-python/compare/archive/v252...archive/v253 +[v252]: https://github.com/heroku/heroku-buildpack-python/compare/archive/v251...archive/v252 +[v251]: https://github.com/heroku/heroku-buildpack-python/compare/archive/v250...archive/v251 +[v250]: https://github.com/heroku/heroku-buildpack-python/compare/archive/v249...archive/v250 +[v249]: https://github.com/heroku/heroku-buildpack-python/compare/archive/v248...archive/v249 +[v248]: https://github.com/heroku/heroku-buildpack-python/compare/archive/v247...archive/v248 +[v247]: https://github.com/heroku/heroku-buildpack-python/compare/archive/v246...archive/v247 +[v246]: https://github.com/heroku/heroku-buildpack-python/compare/archive/v245...archive/v246 +[v245]: https://github.com/heroku/heroku-buildpack-python/compare/archive/v244...archive/v245 +[v244]: https://github.com/heroku/heroku-buildpack-python/compare/archive/v243...archive/v244 +[v243]: https://github.com/heroku/heroku-buildpack-python/compare/archive/v242...archive/v243 +[v242]: https://github.com/heroku/heroku-buildpack-python/compare/archive/v241...archive/v242 +[v241]: https://github.com/heroku/heroku-buildpack-python/compare/archive/v240...archive/v241 +[v240]: https://github.com/heroku/heroku-buildpack-python/compare/archive/v239...archive/v240 +[v239]: https://github.com/heroku/heroku-buildpack-python/compare/archive/v238...archive/v239 +[v238]: https://github.com/heroku/heroku-buildpack-python/compare/archive/v237...archive/v238 +[v237]: https://github.com/heroku/heroku-buildpack-python/compare/archive/v236...archive/v237 +[v236]: https://github.com/heroku/heroku-buildpack-python/compare/archive/v235...archive/v236 +[v235]: https://github.com/heroku/heroku-buildpack-python/compare/archive/v234...archive/v235 +[v234]: https://github.com/heroku/heroku-buildpack-python/compare/archive/v233...archive/v234 +[v233]: https://github.com/heroku/heroku-buildpack-python/compare/archive/v232...archive/v233 +[v232]: https://github.com/heroku/heroku-buildpack-python/compare/archive/v231...archive/v232 +[v231]: https://github.com/heroku/heroku-buildpack-python/compare/archive/v230...archive/v231 +[v230]: https://github.com/heroku/heroku-buildpack-python/compare/archive/v229...archive/v230 +[v229]: https://github.com/heroku/heroku-buildpack-python/compare/archive/v228...archive/v229 +[v228]: https://github.com/heroku/heroku-buildpack-python/compare/archive/v227...archive/v228 +[v227]: https://github.com/heroku/heroku-buildpack-python/compare/archive/v226...archive/v227 +[v226]: https://github.com/heroku/heroku-buildpack-python/compare/archive/v225...archive/v226 +[v225]: https://github.com/heroku/heroku-buildpack-python/compare/archive/v224...archive/v225 +[v224]: https://github.com/heroku/heroku-buildpack-python/compare/archive/v223...archive/v224 +[v223]: https://github.com/heroku/heroku-buildpack-python/compare/archive/v222...archive/v223 +[v222]: https://github.com/heroku/heroku-buildpack-python/compare/archive/v221...archive/v222 +[v221]: https://github.com/heroku/heroku-buildpack-python/compare/archive/v220...archive/v221 +[v220]: https://github.com/heroku/heroku-buildpack-python/compare/archive/v219...archive/v220 +[v219]: https://github.com/heroku/heroku-buildpack-python/compare/archive/v218...archive/v219 +[v218]: https://github.com/heroku/heroku-buildpack-python/compare/archive/v217...archive/v218 +[v217]: https://github.com/heroku/heroku-buildpack-python/compare/archive/v216...archive/v217 +[v216]: https://github.com/heroku/heroku-buildpack-python/compare/archive/v215...archive/v216 +[v215]: https://github.com/heroku/heroku-buildpack-python/compare/archive/v214...archive/v215 +[v214]: https://github.com/heroku/heroku-buildpack-python/compare/archive/v213...archive/v214 +[v213]: https://github.com/heroku/heroku-buildpack-python/compare/archive/v212...archive/v213 +[v212]: https://github.com/heroku/heroku-buildpack-python/compare/archive/v211...archive/v212 +[v211]: https://github.com/heroku/heroku-buildpack-python/compare/archive/v210...archive/v211 +[v210]: https://github.com/heroku/heroku-buildpack-python/compare/archive/v209...archive/v210 +[v209]: https://github.com/heroku/heroku-buildpack-python/compare/archive/v208...archive/v209 +[v208]: https://github.com/heroku/heroku-buildpack-python/compare/archive/v207...archive/v208 [v207]: https://github.com/heroku/heroku-buildpack-python/compare/archive/v206...archive/v207 [v206]: https://github.com/heroku/heroku-buildpack-python/compare/archive/v205...archive/v206 [v205]: https://github.com/heroku/heroku-buildpack-python/compare/archive/v204...archive/v205 diff --git a/spec/hatchet/pipenv_spec.rb b/spec/hatchet/pipenv_spec.rb index 7a657ae82..8d154cfec 100644 --- a/spec/hatchet/pipenv_spec.rb +++ b/spec/hatchet/pipenv_spec.rb @@ -322,7 +322,8 @@ end context 'when the Pipenv and Python versions have changed since the last build' do - let(:buildpacks) { ['https://github.com/heroku/heroku-buildpack-python#v253'] } + # TODO: Bump this buildpack version the next time we update the Pipenv version. + let(:buildpacks) { ['https://github.com/heroku/heroku-buildpack-python#archive/v253'] } let(:app) { Hatchet::Runner.new('spec/fixtures/pipenv_basic', buildpacks:) } it 'clears the cache before installing' do diff --git a/spec/hatchet/poetry_spec.rb b/spec/hatchet/poetry_spec.rb index bad760d30..1fa8ae836 100644 --- a/spec/hatchet/poetry_spec.rb +++ b/spec/hatchet/poetry_spec.rb @@ -62,7 +62,7 @@ end context 'when the Poetry and Python versions have changed since the last build' do - let(:buildpacks) { ['https://github.com/heroku/heroku-buildpack-python#v268'] } + let(:buildpacks) { ['https://github.com/heroku/heroku-buildpack-python#v274'] } let(:app) { Hatchet::Runner.new('spec/fixtures/poetry_basic', buildpacks:) } it 'clears the cache before installing' do @@ -74,8 +74,8 @@ remote: -----> Python app detected remote: -----> Using Python 3.13 specified in .python-version remote: -----> Discarding cache since: - remote: - The Python version has changed from 3.13.0 to #{LATEST_PYTHON_3_13} - remote: - The Poetry version has changed from 1.8.4 to #{POETRY_VERSION} + remote: - The Python version has changed from 3.13.1 to #{LATEST_PYTHON_3_13} + remote: - The Poetry version has changed from 1.8.5 to #{POETRY_VERSION} remote: -----> Installing Python #{LATEST_PYTHON_3_13} remote: -----> Installing Poetry #{POETRY_VERSION} remote: -----> Installing dependencies using 'poetry sync --only main' diff --git a/spec/hatchet/python_version_spec.rb b/spec/hatchet/python_version_spec.rb index 639b67d1c..9e2452e59 100644 --- a/spec/hatchet/python_version_spec.rb +++ b/spec/hatchet/python_version_spec.rb @@ -56,7 +56,7 @@ # - If no Python version is specified, the same major version as the # last build is used (sticky versioning). # - Changes in the pip version are handled correctly. - let(:buildpacks) { ['https://github.com/heroku/heroku-buildpack-python#v257'] } + let(:buildpacks) { ['https://github.com/heroku/heroku-buildpack-python#v267'] } it 'builds with the same Python version as the last build' do app.deploy do |app| @@ -68,7 +68,7 @@ remote: -----> No Python version was specified. Using the same major version as the last build: Python 3.12 remote: To use a different version, see: https://devcenter.heroku.com/articles/python-runtimes remote: -----> Discarding cache since: - remote: - The Python version has changed from 3.12.6 to #{LATEST_PYTHON_3_12} + remote: - The Python version has changed from 3.12.7 to #{LATEST_PYTHON_3_12} remote: - The pip version has changed from 24.0 to #{PIP_VERSION} remote: -----> Installing Python #{LATEST_PYTHON_3_12} remote: -----> Installing pip #{PIP_VERSION}, setuptools #{SETUPTOOLS_VERSION} and wheel #{WHEEL_VERSION} diff --git a/spec/hatchet/stack_spec.rb b/spec/hatchet/stack_spec.rb index 46817cad6..974e6d607 100644 --- a/spec/hatchet/stack_spec.rb +++ b/spec/hatchet/stack_spec.rb @@ -4,12 +4,12 @@ RSpec.describe 'Stack changes' do context 'when the stack is upgraded from Heroku-22 to Heroku-24', stacks: %w[heroku-22] do - # This test performs an initial build using an older buildpack version, followed - # by a build using the current version. This ensures that the current buildpack - # can successfully read the stack metadata written to the build cache in the past. - # The buildpack version chosen is one which had an older default Python version, so - # we can also prove that clearing the cache didn't lose the Python version metadata. - let(:buildpacks) { ['https://github.com/heroku/heroku-buildpack-python#v250'] } + # This test performs an initial build using an older buildpack version, followed by a build + # using the current version. This ensures that the current buildpack can successfully read + # the stack metadata written to the build cache in the past. The buildpack version chosen is + # the oldest to support Heroku-24, and which had an older default Python version so we can + # also prove that clearing the cache didn't lose the sticky Python version metadata. + let(:buildpacks) { ['https://github.com/heroku/heroku-buildpack-python#archive/v250'] } let(:app) { Hatchet::Runner.new('spec/fixtures/python_version_unspecified', buildpacks:) } it 'clears the cache before installing again whilst preserving the sticky Python version' do From ee0a9eb4d21f3d14c7328dda7718e62c0a15c473 Mon Sep 17 00:00:00 2001 From: Ed Morley <501702+edmorley@users.noreply.github.com> Date: Wed, 19 Feb 2025 16:30:16 +0000 Subject: [PATCH 06/26] Change the editable VCS directory location for pip and Pipenv (#1753) When a dependency from a version control system (eg Git) is installed in editable mode, the package manager has to clone the repository somewhere long-lived, that is then referenced by the `.pth` file added to `site-packages`. (When installed in normal non-editable mode, the repo checkouts are instead saved to a temporary directory and deleted after the package is installed.) Until now, the buildpack configured pip and Pipenv to store these repos at `/app/.heroku/src/`, then later copied those files into the build directory and build cache. However, this approach isn't needed with the `.pth` rewriting we have now. In addition, the existing implementation didn't actually restore the cached `src/` directory, so the repos stored in the cache were never re-used on subsequent builds anyway. Now, pip and pipenv are configured to store the repositories at `/.heroku/python/src/`, which means: - The behaviour now matches that when using Poetry. - The repos get cached/restored/invalidated for free, as part of the existing handling of the `.heroku/python/` directory, and we avoid the additional directory copy from `/app` to `/tmp`, both of which help reduce build times. GUS-W-17863838. --- CHANGELOG.md | 1 + bin/compile | 12 ---------- bin/utils | 16 -------------- lib/cache.sh | 14 +----------- lib/pip.sh | 2 +- lib/pipenv.sh | 2 +- spec/hatchet/pip_spec.rb | 44 ++++++++++++++++++------------------- spec/hatchet/pipenv_spec.rb | 44 ++++++++++++++++++------------------- 8 files changed, 48 insertions(+), 87 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e51af3572..bf4cba079 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## [Unreleased] +- Changed the location of repositories for editable VCS dependencies when using pip and Pipenv, to improve build performance and match the behaviour when using Poetry. ([#1753](https://github.com/heroku/heroku-buildpack-python/pull/1753)) ## [v277] - 2025-02-17 diff --git a/bin/compile b/bin/compile index 727966b06..9f2869fc6 100755 --- a/bin/compile +++ b/bin/compile @@ -178,8 +178,6 @@ cache::restore "${BUILD_DIR}" "${CACHE_DIR}" "${STACK}" "${cached_python_full_ve # The directory for the .profile.d scripts. mkdir -p "$(dirname "$PROFILE_PATH")" -# The directory for editable VCS dependencies. -mkdir -p /app/.heroku/src # On Heroku CI, builds happen in `/app`. Otherwise, on the Heroku platform, # they occur in a temp directory. Because Python is not portable, we must create @@ -191,7 +189,6 @@ if [[ "$(realpath "${BUILD_DIR}")" != "$(realpath /app)" ]]; then # python expects to reside in /app, so set up symlinks # we will not remove these later so subsequent buildpacks can still invoke it ln -nsf "$BUILD_DIR/.heroku/python" /app/.heroku/python - # Note: .heroku/src is copied in later. fi python::install "${BUILD_DIR}" "${STACK}" "${python_full_version}" "${python_major_version}" "${python_version_origin}" @@ -250,15 +247,6 @@ nltk_downloader_start_time=$(nowms) sub_env "${BUILDPACK_DIR}/bin/steps/nltk" meta_time "nltk_downloader_duration" "${nltk_downloader_start_time}" -# Support for editable installations. -# In CI, $BUILD_DIR is /app. -# Realpath is used to support use-cases where one of the locations is a symlink to the other. -# shellcheck disable=SC2312 # TODO: Invoke this command separately to avoid masking its return value. -if [[ "$(realpath "${BUILD_DIR}")" != "$(realpath /app)" ]]; then - rm -rf "$BUILD_DIR/.heroku/src" - deep-cp /app/.heroku/src "$BUILD_DIR/.heroku/src" -fi - # Django collectstatic support. # The buildpack automatically runs collectstatic for Django applications. collectstatic_start_time=$(nowms) diff --git a/bin/utils b/bin/utils index 9e662191a..195172a7a 100755 --- a/bin/utils +++ b/bin/utils @@ -8,22 +8,6 @@ shopt -s nullglob source "${BUILDPACK_DIR:?}/vendor/buildpack-stdlib_v8.sh" -# Does some serious copying. -deep-cp() { - declare source="$1" target="$2" - - mkdir -p "$target" - - # cp doesn't like being called without source params, - # so make sure they expand to something first. - # subshell to avoid surprising caller with shopts. - ( - shopt -s nullglob dotglob - set -- "$source"/!(tmp|.|..) - [[ $# == 0 ]] || cp -a "$@" "$target" - ) -} - # Measure the size of the Python installation. measure-size() { { du -s .heroku/python 2>/dev/null || echo 0; } | awk '{print $1}' diff --git a/lib/cache.sh b/lib/cache.sh index ef62d582f..8353a9093 100644 --- a/lib/cache.sh +++ b/lib/cache.sh @@ -129,7 +129,6 @@ function cache::restore() { "${cache_dir}/.heroku/python-poetry" \ "${cache_dir}/.heroku/python-stack" \ "${cache_dir}/.heroku/python-version" \ - "${cache_dir}/.heroku/src" \ "${cache_dir}/.heroku/requirements.txt" meta_set "cache_status" "discarded" @@ -143,17 +142,13 @@ function cache::restore() { # TODO: Compare the performance of moving the directory vs copying files. cp -R "${cache_dir}/.heroku/python" "${build_dir}/.heroku/" &>/dev/null || true - # Editable VCS code repositories, used by pip/pipenv. - if [[ -d "${cache_dir}/.heroku/src" ]]; then - cp -R "${cache_dir}/.heroku/src" "${build_dir}/.heroku/" &>/dev/null || true - fi - meta_set "cache_status" "reused" fi # Remove any legacy cache contents written by older buildpack versions. rm -rf \ "${cache_dir}/.heroku/python-sqlite3-version" \ + "${cache_dir}/.heroku/src" \ "${cache_dir}/.heroku/vendor" meta_time "cache_restore_duration" "${cache_restore_start_time}" @@ -175,13 +170,6 @@ function cache::save() { rm -rf "${cache_dir}/.heroku/python" cp -R "${build_dir}/.heroku/python" "${cache_dir}/.heroku/" - # Editable VCS code repositories, used by pip/pipenv. - rm -rf "${cache_dir}/.heroku/src" - if [[ -d "${build_dir}/.heroku/src" ]]; then - # TODO: Investigate why errors are ignored and ideally stop doing so. - cp -R "${build_dir}/.heroku/src" "${cache_dir}/.heroku/" &>/dev/null || true - fi - # Metadata used by subsequent builds to determine whether the cache can be reused. # These are written/consumed via separate files and not the metadata store for compatibility # with buildpack versions prior to the metadata store existing (which was only added in v252). diff --git a/lib/pip.sh b/lib/pip.sh index 929063fb4..62ebc43ee 100644 --- a/lib/pip.sh +++ b/lib/pip.sh @@ -110,7 +110,7 @@ function pip::install_dependencies() { --no-cache-dir \ --no-input \ --progress-bar off \ - --src='/app/.heroku/src' \ + --src='/app/.heroku/python/src' \ |& tee "${WARNINGS_LOG:?}" \ |& sed --unbuffered --expression '/Requirement already satisfied/d' \ |& output::indent diff --git a/lib/pipenv.sh b/lib/pipenv.sh index 34519eb1b..9e8eb4594 100644 --- a/lib/pipenv.sh +++ b/lib/pipenv.sh @@ -81,7 +81,7 @@ function pipenv::install_dependencies() { # shellcheck disable=SC2310 # This function is invoked in an 'if' condition so set -e will be disabled. if ! { "${pipenv_install_command[@]}" \ - --extra-pip-args='--src=/app/.heroku/src' \ + --extra-pip-args='--src=/app/.heroku/python/src' \ --system \ |& tee "${WARNINGS_LOG:?}" \ |& output::indent diff --git a/spec/hatchet/pip_spec.rb b/spec/hatchet/pip_spec.rb index d4990ae33..64ddb274f 100644 --- a/spec/hatchet/pip_spec.rb +++ b/spec/hatchet/pip_spec.rb @@ -100,21 +100,21 @@ app.deploy do |app| expect(clean_output(app.output)).to match(Regexp.new(<<~REGEX)) remote: -----> Running bin/post_compile hook - remote: easy-install.pth:/app/.heroku/src/gunicorn - remote: easy-install.pth:/tmp/build_.*/packages/local_package_setup_py - remote: __editable___local_package_pyproject_toml_0_0_1_finder.py:/tmp/build_.*/packages/local_package_pyproject_toml/local_package_pyproject_toml'} - remote: gunicorn.egg-link:/app/.heroku/src/gunicorn - remote: local-package-setup-py.egg-link:/tmp/build_.*/packages/local_package_setup_py + remote: easy-install.pth:/tmp/build_.+/.heroku/python/src/gunicorn + remote: easy-install.pth:/tmp/build_.+/packages/local_package_setup_py + remote: __editable___local_package_pyproject_toml_0_0_1_finder.py:/tmp/build_.+/packages/local_package_pyproject_toml/local_package_pyproject_toml'} + remote: gunicorn.egg-link:/tmp/build_.+/.heroku/python/src/gunicorn + remote: local-package-setup-py.egg-link:/tmp/build_.+/packages/local_package_setup_py remote: remote: Running entrypoint for the pyproject.toml-based local package: Hello pyproject.toml! remote: Running entrypoint for the setup.py-based local package: Hello setup.py! remote: Running entrypoint for the VCS package: gunicorn \\(version 20.1.0\\) remote: -----> Inline app detected - remote: easy-install.pth:/app/.heroku/src/gunicorn - remote: easy-install.pth:/tmp/build_.*/packages/local_package_setup_py - remote: __editable___local_package_pyproject_toml_0_0_1_finder.py:/tmp/build_.*/packages/local_package_pyproject_toml/local_package_pyproject_toml'} - remote: gunicorn.egg-link:/app/.heroku/src/gunicorn - remote: local-package-setup-py.egg-link:/tmp/build_.*/packages/local_package_setup_py + remote: easy-install.pth:/tmp/build_.+/.heroku/python/src/gunicorn + remote: easy-install.pth:/tmp/build_.+/packages/local_package_setup_py + remote: __editable___local_package_pyproject_toml_0_0_1_finder.py:/tmp/build_.+/packages/local_package_pyproject_toml/local_package_pyproject_toml'} + remote: gunicorn.egg-link:/tmp/build_.+/.heroku/python/src/gunicorn + remote: local-package-setup-py.egg-link:/tmp/build_.+/packages/local_package_setup_py remote: remote: Running entrypoint for the pyproject.toml-based local package: Hello pyproject.toml! remote: Running entrypoint for the setup.py-based local package: Hello setup.py! @@ -123,10 +123,10 @@ # Test rewritten paths work at runtime. expect(app.run('bin/test-entrypoints.sh')).to include(<<~OUTPUT) - easy-install.pth:/app/.heroku/src/gunicorn + easy-install.pth:/app/.heroku/python/src/gunicorn easy-install.pth:/app/packages/local_package_setup_py __editable___local_package_pyproject_toml_0_0_1_finder.py:/app/packages/local_package_pyproject_toml/local_package_pyproject_toml'} - gunicorn.egg-link:/app/.heroku/src/gunicorn + gunicorn.egg-link:/app/.heroku/python/src/gunicorn local-package-setup-py.egg-link:/app/packages/local_package_setup_py Running entrypoint for the pyproject.toml-based local package: Hello pyproject.toml! @@ -139,21 +139,21 @@ app.push! expect(clean_output(app.output)).to match(Regexp.new(<<~REGEX)) remote: -----> Running bin/post_compile hook - remote: easy-install.pth:/app/.heroku/src/gunicorn - remote: easy-install.pth:/tmp/build_.*/packages/local_package_setup_py - remote: __editable___local_package_pyproject_toml_0_0_1_finder.py:/tmp/build_.*/packages/local_package_pyproject_toml/local_package_pyproject_toml'} - remote: gunicorn.egg-link:/app/.heroku/src/gunicorn - remote: local-package-setup-py.egg-link:/tmp/build_.*/packages/local_package_setup_py + remote: easy-install.pth:/tmp/build_.+/.heroku/python/src/gunicorn + remote: easy-install.pth:/tmp/build_.+/packages/local_package_setup_py + remote: __editable___local_package_pyproject_toml_0_0_1_finder.py:/tmp/build_.+/packages/local_package_pyproject_toml/local_package_pyproject_toml'} + remote: gunicorn.egg-link:/tmp/build_.+/.heroku/python/src/gunicorn + remote: local-package-setup-py.egg-link:/tmp/build_.+/packages/local_package_setup_py remote: remote: Running entrypoint for the pyproject.toml-based local package: Hello pyproject.toml! remote: Running entrypoint for the setup.py-based local package: Hello setup.py! remote: Running entrypoint for the VCS package: gunicorn \\(version 20.1.0\\) remote: -----> Inline app detected - remote: easy-install.pth:/app/.heroku/src/gunicorn - remote: easy-install.pth:/tmp/build_.*/packages/local_package_setup_py - remote: __editable___local_package_pyproject_toml_0_0_1_finder.py:/tmp/build_.*/packages/local_package_pyproject_toml/local_package_pyproject_toml'} - remote: gunicorn.egg-link:/app/.heroku/src/gunicorn - remote: local-package-setup-py.egg-link:/tmp/build_.*/packages/local_package_setup_py + remote: easy-install.pth:/tmp/build_.+/.heroku/python/src/gunicorn + remote: easy-install.pth:/tmp/build_.+/packages/local_package_setup_py + remote: __editable___local_package_pyproject_toml_0_0_1_finder.py:/tmp/build_.+/packages/local_package_pyproject_toml/local_package_pyproject_toml'} + remote: gunicorn.egg-link:/tmp/build_.+/.heroku/python/src/gunicorn + remote: local-package-setup-py.egg-link:/tmp/build_.+/packages/local_package_setup_py remote: remote: Running entrypoint for the pyproject.toml-based local package: Hello pyproject.toml! remote: Running entrypoint for the setup.py-based local package: Hello setup.py! diff --git a/spec/hatchet/pipenv_spec.rb b/spec/hatchet/pipenv_spec.rb index 8d154cfec..18addeaec 100644 --- a/spec/hatchet/pipenv_spec.rb +++ b/spec/hatchet/pipenv_spec.rb @@ -356,21 +356,21 @@ app.deploy do |app| expect(clean_output(app.output)).to match(Regexp.new(<<~REGEX)) remote: -----> Running bin/post_compile hook - remote: easy-install.pth:/app/.heroku/src/gunicorn - remote: easy-install.pth:/tmp/build_.*/packages/local_package_setup_py - remote: __editable___local_package_pyproject_toml_0_0_1_finder.py:/tmp/build_.*/packages/local_package_pyproject_toml/local_package_pyproject_toml'} - remote: gunicorn.egg-link:/app/.heroku/src/gunicorn - remote: local-package-setup-py.egg-link:/tmp/build_.*/packages/local_package_setup_py + remote: easy-install.pth:/tmp/build_.+/.heroku/python/src/gunicorn + remote: easy-install.pth:/tmp/build_.+/packages/local_package_setup_py + remote: __editable___local_package_pyproject_toml_0_0_1_finder.py:/tmp/build_.+/packages/local_package_pyproject_toml/local_package_pyproject_toml'} + remote: gunicorn.egg-link:/tmp/build_.+/.heroku/python/src/gunicorn + remote: local-package-setup-py.egg-link:/tmp/build_.+/packages/local_package_setup_py remote: remote: Running entrypoint for the pyproject.toml-based local package: Hello pyproject.toml! remote: Running entrypoint for the setup.py-based local package: Hello setup.py! remote: Running entrypoint for the VCS package: gunicorn \\(version 20.1.0\\) remote: -----> Inline app detected - remote: easy-install.pth:/app/.heroku/src/gunicorn - remote: easy-install.pth:/tmp/build_.*/packages/local_package_setup_py - remote: __editable___local_package_pyproject_toml_0_0_1_finder.py:/tmp/build_.*/packages/local_package_pyproject_toml/local_package_pyproject_toml'} - remote: gunicorn.egg-link:/app/.heroku/src/gunicorn - remote: local-package-setup-py.egg-link:/tmp/build_.*/packages/local_package_setup_py + remote: easy-install.pth:/tmp/build_.+/.heroku/python/src/gunicorn + remote: easy-install.pth:/tmp/build_.+/packages/local_package_setup_py + remote: __editable___local_package_pyproject_toml_0_0_1_finder.py:/tmp/build_.+/packages/local_package_pyproject_toml/local_package_pyproject_toml'} + remote: gunicorn.egg-link:/tmp/build_.+/.heroku/python/src/gunicorn + remote: local-package-setup-py.egg-link:/tmp/build_.+/packages/local_package_setup_py remote: remote: Running entrypoint for the pyproject.toml-based local package: Hello pyproject.toml! remote: Running entrypoint for the setup.py-based local package: Hello setup.py! @@ -379,10 +379,10 @@ # Test rewritten paths work at runtime. expect(app.run('bin/test-entrypoints.sh')).to include(<<~OUTPUT) - easy-install.pth:/app/.heroku/src/gunicorn + easy-install.pth:/app/.heroku/python/src/gunicorn easy-install.pth:/app/packages/local_package_setup_py __editable___local_package_pyproject_toml_0_0_1_finder.py:/app/packages/local_package_pyproject_toml/local_package_pyproject_toml'} - gunicorn.egg-link:/app/.heroku/src/gunicorn + gunicorn.egg-link:/app/.heroku/python/src/gunicorn local-package-setup-py.egg-link:/app/packages/local_package_setup_py Running entrypoint for the pyproject.toml-based local package: Hello pyproject.toml! @@ -395,21 +395,21 @@ app.push! expect(clean_output(app.output)).to match(Regexp.new(<<~REGEX)) remote: -----> Running bin/post_compile hook - remote: easy-install.pth:/app/.heroku/src/gunicorn - remote: easy-install.pth:/tmp/build_.*/packages/local_package_setup_py - remote: __editable___local_package_pyproject_toml_0_0_1_finder.py:/tmp/build_.*/packages/local_package_pyproject_toml/local_package_pyproject_toml'} - remote: gunicorn.egg-link:/app/.heroku/src/gunicorn - remote: local-package-setup-py.egg-link:/tmp/build_.*/packages/local_package_setup_py + remote: easy-install.pth:/tmp/build_.+/.heroku/python/src/gunicorn + remote: easy-install.pth:/tmp/build_.+/packages/local_package_setup_py + remote: __editable___local_package_pyproject_toml_0_0_1_finder.py:/tmp/build_.+/packages/local_package_pyproject_toml/local_package_pyproject_toml'} + remote: gunicorn.egg-link:/tmp/build_.+/.heroku/python/src/gunicorn + remote: local-package-setup-py.egg-link:/tmp/build_.+/packages/local_package_setup_py remote: remote: Running entrypoint for the pyproject.toml-based local package: Hello pyproject.toml! remote: Running entrypoint for the setup.py-based local package: Hello setup.py! remote: Running entrypoint for the VCS package: gunicorn \\(version 20.1.0\\) remote: -----> Inline app detected - remote: easy-install.pth:/app/.heroku/src/gunicorn - remote: easy-install.pth:/tmp/build_.*/packages/local_package_setup_py - remote: __editable___local_package_pyproject_toml_0_0_1_finder.py:/tmp/build_.*/packages/local_package_pyproject_toml/local_package_pyproject_toml'} - remote: gunicorn.egg-link:/app/.heroku/src/gunicorn - remote: local-package-setup-py.egg-link:/tmp/build_.*/packages/local_package_setup_py + remote: easy-install.pth:/tmp/build_.+/.heroku/python/src/gunicorn + remote: easy-install.pth:/tmp/build_.+/packages/local_package_setup_py + remote: __editable___local_package_pyproject_toml_0_0_1_finder.py:/tmp/build_.+/packages/local_package_pyproject_toml/local_package_pyproject_toml'} + remote: gunicorn.egg-link:/tmp/build_.+/.heroku/python/src/gunicorn + remote: local-package-setup-py.egg-link:/tmp/build_.+/packages/local_package_setup_py remote: remote: Running entrypoint for the pyproject.toml-based local package: Hello pyproject.toml! remote: Running entrypoint for the setup.py-based local package: Hello setup.py! From f2d2bc7c4ad4b9745f441f7bd7a01b002270a73a Mon Sep 17 00:00:00 2001 From: Ed Morley <501702+edmorley@users.noreply.github.com> Date: Thu, 20 Feb 2025 12:13:35 +0000 Subject: [PATCH 07/26] Improve package manager tests (#1754) Backports a number of updates/improvements to the pip/Poetry/Pipenv package manager integration tests found while working on the tests for uv. --- spec/fixtures/pipenv_editable/.python-version | 2 + spec/fixtures/pipenv_editable/Pipfile | 3 +- spec/fixtures/pipenv_editable/Pipfile.lock | 12 +++-- .../pipenv_editable/__init__.py | 0 spec/fixtures/pipenv_editable/pyproject.toml | 8 ++++ .../poetry_editable/bin/test-entrypoints.sh | 2 +- spec/fixtures/poetry_editable/poetry.lock | 46 +++++++++---------- spec/fixtures/poetry_editable/pyproject.toml | 2 +- spec/hatchet/ci_spec.rb | 12 ++--- spec/hatchet/pip_spec.rb | 5 +- spec/hatchet/pipenv_spec.rb | 13 +++++- spec/hatchet/poetry_spec.rb | 45 +++++++++++++----- 12 files changed, 98 insertions(+), 52 deletions(-) create mode 100644 spec/fixtures/pipenv_editable/pipenv_editable/__init__.py create mode 100644 spec/fixtures/pipenv_editable/pyproject.toml diff --git a/spec/fixtures/pipenv_editable/.python-version b/spec/fixtures/pipenv_editable/.python-version index e4fba2183..b81161743 100644 --- a/spec/fixtures/pipenv_editable/.python-version +++ b/spec/fixtures/pipenv_editable/.python-version @@ -1 +1,3 @@ +# Note: This test has to use Python 3.12 until we work around the +# Pipenv editable VCS dependency cache invalidation bug. 3.12 diff --git a/spec/fixtures/pipenv_editable/Pipfile b/spec/fixtures/pipenv_editable/Pipfile index fcbd9c1a0..910d90713 100644 --- a/spec/fixtures/pipenv_editable/Pipfile +++ b/spec/fixtures/pipenv_editable/Pipfile @@ -4,6 +4,7 @@ verify_ssl = true name = "pypi" [packages] +gunicorn = {git = "git+https://github.com/benoitc/gunicorn", ref = "20.1.0", editable = true} local-package-pyproject-toml = {file = "packages/local_package_pyproject_toml", editable = true} local-package-setup-py = {file = "packages/local_package_setup_py", editable = true} -gunicorn = {git = "git+https://github.com/benoitc/gunicorn", ref = "20.1.0", editable = true} +pipenv-editable = {file = ".", editable = true} diff --git a/spec/fixtures/pipenv_editable/Pipfile.lock b/spec/fixtures/pipenv_editable/Pipfile.lock index ae096d558..f6f2acc1b 100644 --- a/spec/fixtures/pipenv_editable/Pipfile.lock +++ b/spec/fixtures/pipenv_editable/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "5de61064f835cc6b28000b3ac1b7b5c38a40829ccc38ff46088e36bb3bd628fa" + "sha256": "ed18332549853701536926a2bae44d9fc142a08c105d4fbf108dbde04b3fedf2" }, "pipfile-spec": 6, "requires": {}, @@ -28,13 +28,17 @@ "editable": true, "file": "packages/local_package_setup_py" }, + "pipenv-editable": { + "editable": true, + "file": "." + }, "setuptools": { "hashes": [ - "sha256:8199222558df7c86216af4f84c30e9b34a61d8ba19366cc914424cdbd28252f6", - "sha256:ce74b49e8f7110f9bf04883b730f4765b774ef3ef28f722cce7c273d253aaf7d" + "sha256:c5afc8f407c626b8313a86e10311dd3f661c6cd9c09d4bf8c15c0e11f9f2b0e6", + "sha256:e3982f444617239225d675215d51f6ba05f845d4eec313da4418fdbb56fb27e3" ], "markers": "python_version >= '3.9'", - "version": "==75.6.0" + "version": "==75.8.0" } }, "develop": {} diff --git a/spec/fixtures/pipenv_editable/pipenv_editable/__init__.py b/spec/fixtures/pipenv_editable/pipenv_editable/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/spec/fixtures/pipenv_editable/pyproject.toml b/spec/fixtures/pipenv_editable/pyproject.toml new file mode 100644 index 000000000..cf7b46f69 --- /dev/null +++ b/spec/fixtures/pipenv_editable/pyproject.toml @@ -0,0 +1,8 @@ +[project] +name = "pipenv-editable" +version = "0.0.0" +requires-python = ">=3.12" + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" diff --git a/spec/fixtures/poetry_editable/bin/test-entrypoints.sh b/spec/fixtures/poetry_editable/bin/test-entrypoints.sh index fc941ed3f..0e66f44bb 100755 --- a/spec/fixtures/poetry_editable/bin/test-entrypoints.sh +++ b/spec/fixtures/poetry_editable/bin/test-entrypoints.sh @@ -4,7 +4,7 @@ set -euo pipefail cd .heroku/python/lib/python*/site-packages/ -# List any path like strings in .pth, and finder files in site-packages. +# List any path like strings in the .pth and finder files in site-packages. grep --extended-regexp --only-matching -- '/\S+' *.pth __editable___*_finder.py | sort echo diff --git a/spec/fixtures/poetry_editable/poetry.lock b/spec/fixtures/poetry_editable/poetry.lock index a26fb9272..30df026b9 100644 --- a/spec/fixtures/poetry_editable/poetry.lock +++ b/spec/fixtures/poetry_editable/poetry.lock @@ -1,35 +1,38 @@ -# This file is automatically @generated by Poetry 1.8.4 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.1.1 and should not be changed by hand. [[package]] name = "gunicorn" -version = "20.1.0" +version = "23.0.0" description = "WSGI HTTP Server for UNIX" optional = false -python-versions = ">=3.5" +python-versions = ">=3.7" +groups = ["main"] files = [] develop = true [package.dependencies] -setuptools = ">=3.0" +packaging = "*" [package.extras] -eventlet = ["eventlet (>=0.24.1)"] +eventlet = ["eventlet (>=0.24.1,!=0.36.0)"] gevent = ["gevent (>=1.4.0)"] setproctitle = ["setproctitle"] +testing = ["coverage", "eventlet", "gevent", "pytest", "pytest-cov"] tornado = ["tornado (>=0.2)"] [package.source] type = "git" url = "https://github.com/benoitc/gunicorn.git" -reference = "20.1.0" -resolved_reference = "61ccfd6c38d477a908e0f376757bbb884438053a" +reference = "HEAD" +resolved_reference = "bacbf8aa5152b94e44aa5d2a94aeaf0318a85248" [[package]] -name = "local_package_pyproject_toml" +name = "local-package-pyproject-toml" version = "0.0.1" description = "" optional = false python-versions = "*" +groups = ["main"] files = [] develop = true @@ -43,6 +46,7 @@ version = "0.0.1" description = "" optional = false python-versions = "*" +groups = ["main"] files = [] develop = true @@ -51,26 +55,18 @@ type = "directory" url = "packages/local_package_setup_py" [[package]] -name = "setuptools" -version = "75.6.0" -description = "Easily download, build, install, upgrade, and uninstall Python packages" +name = "packaging" +version = "24.2" +description = "Core utilities for Python packages" optional = false -python-versions = ">=3.9" +python-versions = ">=3.8" +groups = ["main"] files = [ - {file = "setuptools-75.6.0-py3-none-any.whl", hash = "sha256:ce74b49e8f7110f9bf04883b730f4765b774ef3ef28f722cce7c273d253aaf7d"}, - {file = "setuptools-75.6.0.tar.gz", hash = "sha256:8199222558df7c86216af4f84c30e9b34a61d8ba19366cc914424cdbd28252f6"}, + {file = "packaging-24.2-py3-none-any.whl", hash = "sha256:09abb1bccd265c01f4a3aa3f7a7db064b36514d2cba19a2f694fe6150451a759"}, + {file = "packaging-24.2.tar.gz", hash = "sha256:c228a6dc5e932d346bc5739379109d49e8853dd8223571c7c5b55260edc0b97f"}, ] -[package.extras] -check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1)", "ruff (>=0.7.0)"] -core = ["importlib_metadata (>=6)", "jaraco.collections", "jaraco.functools (>=4)", "jaraco.text (>=3.7)", "more_itertools", "more_itertools (>=8.8)", "packaging", "packaging (>=24.2)", "platformdirs (>=4.2.2)", "tomli (>=2.0.1)", "wheel (>=0.43.0)"] -cover = ["pytest-cov"] -doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "pyproject-hooks (!=1.1)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier", "towncrier (<24.7)"] -enabler = ["pytest-enabler (>=2.2)"] -test = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "ini2toml[lite] (>=0.14)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "jaraco.test (>=5.5)", "packaging (>=24.2)", "pip (>=19.1)", "pyproject-hooks (!=1.1)", "pytest (>=6,!=8.1.*)", "pytest-home (>=0.5)", "pytest-perf", "pytest-subprocess", "pytest-timeout", "pytest-xdist (>=3)", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel (>=0.44.0)"] -type = ["importlib_metadata (>=7.0.2)", "jaraco.develop (>=7.21)", "mypy (>=1.12,<1.14)", "pytest-mypy"] - [metadata] -lock-version = "2.0" +lock-version = "2.1" python-versions = "^3.13" -content-hash = "622d4971a16f64f64de12d8208024658e6d7b9e30e1315e6970110ec5e70975c" +content-hash = "f3fb8b00f781abfc5c963f9a266d16757b9b92cc415866eb28f156b4083fbf48" diff --git a/spec/fixtures/poetry_editable/pyproject.toml b/spec/fixtures/poetry_editable/pyproject.toml index b7659537b..256e2d984 100644 --- a/spec/fixtures/poetry_editable/pyproject.toml +++ b/spec/fixtures/poetry_editable/pyproject.toml @@ -6,7 +6,7 @@ authors = [] [tool.poetry.dependencies] python = "^3.13" -gunicorn = { git = "https://github.com/benoitc/gunicorn.git", tag = "20.1.0", develop = true } +gunicorn = { git = "https://github.com/benoitc/gunicorn.git", develop = true } local-package-pyproject-toml = { path = "packages/local_package_pyproject_toml", develop = true } local-package-setup-py = { path = "packages/local_package_setup_py", develop = true } diff --git a/spec/hatchet/ci_spec.rb b/spec/hatchet/ci_spec.rb index 30126af91..aa4f58dd2 100644 --- a/spec/hatchet/ci_spec.rb +++ b/spec/hatchet/ci_spec.rb @@ -10,7 +10,7 @@ it 'installs both normal and test dependencies and uses cache on subsequent runs' do app.run_ci do |test_run| - expect(test_run.output).to match(Regexp.new(<<~REGEX, Regexp::MULTILINE)) + expect(clean_output(test_run.output)).to match(Regexp.new(<<~REGEX, Regexp::MULTILINE)) -----> Python app detected -----> Using Python #{DEFAULT_PYTHON_MAJOR_VERSION} specified in .python-version -----> Installing Python #{DEFAULT_PYTHON_FULL_VERSION} @@ -62,7 +62,7 @@ REGEX test_run.run_again - expect(test_run.output).to include(<<~OUTPUT) + expect(clean_output(test_run.output)).to include(<<~OUTPUT) -----> Python app detected -----> Using Python #{DEFAULT_PYTHON_MAJOR_VERSION} specified in .python-version -----> Restoring cache @@ -81,7 +81,7 @@ it 'installs both normal and test dependencies and uses cache on subsequent runs' do app.run_ci do |test_run| - expect(test_run.output).to match(Regexp.new(<<~REGEX)) + expect(clean_output(test_run.output)).to match(Regexp.new(<<~REGEX)) -----> Python app detected -----> Using Python #{DEFAULT_PYTHON_MAJOR_VERSION} specified in .python-version -----> Installing Python #{DEFAULT_PYTHON_FULL_VERSION} @@ -134,7 +134,7 @@ REGEX test_run.run_again - expect(test_run.output).to match(Regexp.new(<<~REGEX)) + expect(clean_output(test_run.output)).to match(Regexp.new(<<~REGEX)) -----> Python app detected -----> Using Python #{DEFAULT_PYTHON_MAJOR_VERSION} specified in .python-version -----> Restoring cache @@ -156,7 +156,7 @@ it 'installs both normal and test dependencies and uses cache on subsequent runs' do app.run_ci do |test_run| - expect(test_run.output).to match(Regexp.new(<<~REGEX)) + expect(clean_output(test_run.output)).to match(Regexp.new(<<~REGEX)) -----> Python app detected -----> Using Python #{DEFAULT_PYTHON_MAJOR_VERSION} specified in .python-version -----> Installing Python #{DEFAULT_PYTHON_FULL_VERSION} @@ -217,7 +217,7 @@ REGEX test_run.run_again - expect(test_run.output).to include(<<~OUTPUT) + expect(clean_output(test_run.output)).to include(<<~OUTPUT) -----> Python app detected -----> Using Python #{DEFAULT_PYTHON_MAJOR_VERSION} specified in .python-version -----> Restoring cache diff --git a/spec/hatchet/pip_spec.rb b/spec/hatchet/pip_spec.rb index 64ddb274f..f67a55aa1 100644 --- a/spec/hatchet/pip_spec.rb +++ b/spec/hatchet/pip_spec.rb @@ -91,7 +91,8 @@ end # This test intentionally uses Python 3.12, so that we test rewriting using older globally installed - # setuptools. The Poetry equivalent of this test covers the PEP-517/518 setuptools case. + # setuptools (which causes .egg-link files to be created too). The Pipenv and Poetry equivalents of + # this test covers the PEP-517/518 setuptools case. context 'when requirements.txt contains editable requirements (both VCS and local package)' do let(:buildpacks) { [:default, 'heroku-community/inline'] } let(:app) { Hatchet::Runner.new('spec/fixtures/pip_editable', buildpacks:) } @@ -159,6 +160,8 @@ remote: Running entrypoint for the setup.py-based local package: Hello setup.py! remote: Running entrypoint for the VCS package: gunicorn \\(version 20.1.0\\) REGEX + # Test that the VCS repo checkout was cached correctly. + expect(app.output).to include('Updating /app/.heroku/python/src/gunicorn clone (to revision 20.1.0)') end end end diff --git a/spec/hatchet/pipenv_spec.rb b/spec/hatchet/pipenv_spec.rb index 18addeaec..0ff606900 100644 --- a/spec/hatchet/pipenv_spec.rb +++ b/spec/hatchet/pipenv_spec.rb @@ -47,7 +47,7 @@ remote: typing_extensions 4.12.2 remote: virtualenv .+ remote: - remote: \\ + remote: REGEX app.commit! app.push! @@ -348,6 +348,8 @@ end end + # This test has to use Python 3.12 until we work around the Pipenv editable VCS dependency + # cache invalidation bug when using pyproject.toml / PEP517 based installs. context 'when Pipfile contains editable requirements' do let(:buildpacks) { [:default, 'heroku-community/inline'] } let(:app) { Hatchet::Runner.new('spec/fixtures/pipenv_editable', buildpacks:) } @@ -355,12 +357,15 @@ it 'rewrites .pth, .egg-link and finder paths correctly for hooks, later buildpacks, runtime and cached builds' do app.deploy do |app| expect(clean_output(app.output)).to match(Regexp.new(<<~REGEX)) + remote: -----> Installing dependencies using 'pipenv install --deploy' + remote: Installing dependencies from Pipfile.lock \\(.+\\)... remote: -----> Running bin/post_compile hook remote: easy-install.pth:/tmp/build_.+/.heroku/python/src/gunicorn remote: easy-install.pth:/tmp/build_.+/packages/local_package_setup_py remote: __editable___local_package_pyproject_toml_0_0_1_finder.py:/tmp/build_.+/packages/local_package_pyproject_toml/local_package_pyproject_toml'} remote: gunicorn.egg-link:/tmp/build_.+/.heroku/python/src/gunicorn remote: local-package-setup-py.egg-link:/tmp/build_.+/packages/local_package_setup_py + remote: _pipenv_editable.pth:/tmp/build_.+ remote: remote: Running entrypoint for the pyproject.toml-based local package: Hello pyproject.toml! remote: Running entrypoint for the setup.py-based local package: Hello setup.py! @@ -371,6 +376,7 @@ remote: __editable___local_package_pyproject_toml_0_0_1_finder.py:/tmp/build_.+/packages/local_package_pyproject_toml/local_package_pyproject_toml'} remote: gunicorn.egg-link:/tmp/build_.+/.heroku/python/src/gunicorn remote: local-package-setup-py.egg-link:/tmp/build_.+/packages/local_package_setup_py + remote: _pipenv_editable.pth:/tmp/build_.+ remote: remote: Running entrypoint for the pyproject.toml-based local package: Hello pyproject.toml! remote: Running entrypoint for the setup.py-based local package: Hello setup.py! @@ -384,6 +390,7 @@ __editable___local_package_pyproject_toml_0_0_1_finder.py:/app/packages/local_package_pyproject_toml/local_package_pyproject_toml'} gunicorn.egg-link:/app/.heroku/python/src/gunicorn local-package-setup-py.egg-link:/app/packages/local_package_setup_py + _pipenv_editable.pth:/app Running entrypoint for the pyproject.toml-based local package: Hello pyproject.toml! Running entrypoint for the setup.py-based local package: Hello setup.py! @@ -394,12 +401,15 @@ app.commit! app.push! expect(clean_output(app.output)).to match(Regexp.new(<<~REGEX)) + remote: -----> Installing dependencies using 'pipenv install --deploy' + remote: Installing dependencies from Pipfile.lock \\(.+\\)... remote: -----> Running bin/post_compile hook remote: easy-install.pth:/tmp/build_.+/.heroku/python/src/gunicorn remote: easy-install.pth:/tmp/build_.+/packages/local_package_setup_py remote: __editable___local_package_pyproject_toml_0_0_1_finder.py:/tmp/build_.+/packages/local_package_pyproject_toml/local_package_pyproject_toml'} remote: gunicorn.egg-link:/tmp/build_.+/.heroku/python/src/gunicorn remote: local-package-setup-py.egg-link:/tmp/build_.+/packages/local_package_setup_py + remote: _pipenv_editable.pth:/tmp/build_.+ remote: remote: Running entrypoint for the pyproject.toml-based local package: Hello pyproject.toml! remote: Running entrypoint for the setup.py-based local package: Hello setup.py! @@ -410,6 +420,7 @@ remote: __editable___local_package_pyproject_toml_0_0_1_finder.py:/tmp/build_.+/packages/local_package_pyproject_toml/local_package_pyproject_toml'} remote: gunicorn.egg-link:/tmp/build_.+/.heroku/python/src/gunicorn remote: local-package-setup-py.egg-link:/tmp/build_.+/packages/local_package_setup_py + remote: _pipenv_editable.pth:/tmp/build_.+ remote: remote: Running entrypoint for the pyproject.toml-based local package: Hello pyproject.toml! remote: Running entrypoint for the setup.py-based local package: Hello setup.py! diff --git a/spec/hatchet/poetry_spec.rb b/spec/hatchet/poetry_spec.rb index 1fa8ae836..456a35996 100644 --- a/spec/hatchet/poetry_spec.rb +++ b/spec/hatchet/poetry_spec.rb @@ -94,63 +94,84 @@ let(:buildpacks) { [:default, 'heroku-community/inline'] } let(:app) { Hatchet::Runner.new('spec/fixtures/poetry_editable', buildpacks:) } - it 'rewrites .pth, .egg-link and finder paths correctly for hooks, later buildpacks, runtime and cached builds' do + it 'rewrites .pth and finder paths correctly for hooks, later buildpacks, runtime and cached builds' do app.deploy do |app| expect(clean_output(app.output)).to match(Regexp.new(<<~REGEX)) + remote: -----> Installing dependencies using 'poetry sync --only main' + remote: Installing dependencies from lock file + remote: + remote: Package operations: 4 installs, 0 updates, 0 removals + remote: + remote: - Installing packaging \\(24.2\\) + remote: - Installing gunicorn \\(23.0.0 bacbf8a\\) + remote: - Installing local-package-pyproject-toml \\(0.0.1 /tmp/build_.+/packages/local_package_pyproject_toml\\) + remote: - Installing local-package-setup-py \\(0.0.1 /tmp/build_.+/packages/local_package_setup_py\\) + remote: + remote: Installing the current project: poetry-editable \\(0.0.1\\) remote: -----> Running bin/post_compile hook - remote: __editable___gunicorn_20_1_0_finder.py:/tmp/build_.+/.heroku/python/src/gunicorn/gunicorn'} + remote: __editable___gunicorn_23_0_0_finder.py:/tmp/build_.+/.heroku/python/src/gunicorn/gunicorn'} remote: __editable___local_package_pyproject_toml_0_0_1_finder.py:/tmp/build_.+/packages/local_package_pyproject_toml/local_package_pyproject_toml'} remote: __editable___local_package_setup_py_0_0_1_finder.py:/tmp/build_.+/packages/local_package_setup_py/local_package_setup_py'} remote: poetry_editable.pth:/tmp/build_.+ remote: remote: Running entrypoint for the pyproject.toml-based local package: Hello pyproject.toml! remote: Running entrypoint for the setup.py-based local package: Hello setup.py! - remote: Running entrypoint for the VCS package: gunicorn \\(version 20.1.0\\) + remote: Running entrypoint for the VCS package: gunicorn \\(version 23.0.0\\) remote: -----> Inline app detected - remote: __editable___gunicorn_20_1_0_finder.py:/tmp/build_.+/.heroku/python/src/gunicorn/gunicorn'} + remote: __editable___gunicorn_23_0_0_finder.py:/tmp/build_.+/.heroku/python/src/gunicorn/gunicorn'} remote: __editable___local_package_pyproject_toml_0_0_1_finder.py:/tmp/build_.+/packages/local_package_pyproject_toml/local_package_pyproject_toml'} remote: __editable___local_package_setup_py_0_0_1_finder.py:/tmp/build_.+/packages/local_package_setup_py/local_package_setup_py'} remote: poetry_editable.pth:/tmp/build_.+ remote: remote: Running entrypoint for the pyproject.toml-based local package: Hello pyproject.toml! remote: Running entrypoint for the setup.py-based local package: Hello setup.py! - remote: Running entrypoint for the VCS package: gunicorn \\(version 20.1.0\\) + remote: Running entrypoint for the VCS package: gunicorn \\(version 23.0.0\\) REGEX # Test rewritten paths work at runtime. expect(app.run('bin/test-entrypoints.sh')).to include(<<~OUTPUT) - __editable___gunicorn_20_1_0_finder.py:/app/.heroku/python/src/gunicorn/gunicorn'} + __editable___gunicorn_23_0_0_finder.py:/app/.heroku/python/src/gunicorn/gunicorn'} __editable___local_package_pyproject_toml_0_0_1_finder.py:/app/packages/local_package_pyproject_toml/local_package_pyproject_toml'} __editable___local_package_setup_py_0_0_1_finder.py:/app/packages/local_package_setup_py/local_package_setup_py'} poetry_editable.pth:/app Running entrypoint for the pyproject.toml-based local package: Hello pyproject.toml! Running entrypoint for the setup.py-based local package: Hello setup.py! - Running entrypoint for the VCS package: gunicorn (version 20.1.0) + Running entrypoint for the VCS package: gunicorn (version 23.0.0) OUTPUT # Test that the cached .pth files work correctly. app.commit! app.push! expect(clean_output(app.output)).to match(Regexp.new(<<~REGEX)) + remote: -----> Installing dependencies using 'poetry sync --only main' + remote: Installing dependencies from lock file + remote: + remote: Package operations: 0 installs, 3 updates, 0 removals + remote: + remote: - Updating gunicorn \\(23.0.0 /app/.heroku/python/src/gunicorn -> 23.0.0 bacbf8a\\) + remote: - Updating local-package-pyproject-toml \\(0.0.1 /tmp/build_.+/packages/local_package_pyproject_toml -> 0.0.1 /tmp/build_.+/packages/local_package_pyproject_toml\\) + remote: - Updating local-package-setup-py \\(0.0.1 /tmp/build_.+/packages/local_package_setup_py -> 0.0.1 /tmp/build_.+/packages/local_package_setup_py\\) + remote: + remote: Installing the current project: poetry-editable \\(0.0.1\\) remote: -----> Running bin/post_compile hook - remote: __editable___gunicorn_20_1_0_finder.py:/tmp/build_.+/.heroku/python/src/gunicorn/gunicorn'} + remote: __editable___gunicorn_23_0_0_finder.py:/tmp/build_.+/.heroku/python/src/gunicorn/gunicorn'} remote: __editable___local_package_pyproject_toml_0_0_1_finder.py:/tmp/build_.+/packages/local_package_pyproject_toml/local_package_pyproject_toml'} remote: __editable___local_package_setup_py_0_0_1_finder.py:/tmp/build_.+/packages/local_package_setup_py/local_package_setup_py'} remote: poetry_editable.pth:/tmp/build_.+ remote: remote: Running entrypoint for the pyproject.toml-based local package: Hello pyproject.toml! remote: Running entrypoint for the setup.py-based local package: Hello setup.py! - remote: Running entrypoint for the VCS package: gunicorn \\(version 20.1.0\\) + remote: Running entrypoint for the VCS package: gunicorn \\(version 23.0.0\\) remote: -----> Inline app detected - remote: __editable___gunicorn_20_1_0_finder.py:/tmp/build_.+/.heroku/python/src/gunicorn/gunicorn'} + remote: __editable___gunicorn_23_0_0_finder.py:/tmp/build_.+/.heroku/python/src/gunicorn/gunicorn'} remote: __editable___local_package_pyproject_toml_0_0_1_finder.py:/tmp/build_.+/packages/local_package_pyproject_toml/local_package_pyproject_toml'} remote: __editable___local_package_setup_py_0_0_1_finder.py:/tmp/build_.+/packages/local_package_setup_py/local_package_setup_py'} remote: poetry_editable.pth:/tmp/build_.+ remote: remote: Running entrypoint for the pyproject.toml-based local package: Hello pyproject.toml! remote: Running entrypoint for the setup.py-based local package: Hello setup.py! - remote: Running entrypoint for the VCS package: gunicorn \\(version 20.1.0\\) + remote: Running entrypoint for the VCS package: gunicorn \\(version 23.0.0\\) REGEX end end @@ -160,7 +181,7 @@ # chosen Poetry version also supports our oldest supported Python version. The fixture # also includes a `brotli` directory to test the workaround for an `ensurepip` bug in # older Python versions: https://github.com/heroku/heroku-buildpack-python/issues/1697 - context 'when using the oldest supported Python version' do + context 'when using our oldest supported Python version' do let(:app) { Hatchet::Runner.new('spec/fixtures/poetry_oldest_python') } it 'installs successfully' do From bb282273a9c0fdcf952408fe6a668247427cf009 Mon Sep 17 00:00:00 2001 From: Ed Morley <501702+edmorley@users.noreply.github.com> Date: Fri, 21 Feb 2025 08:30:15 +0000 Subject: [PATCH 08/26] `make run` workflow improvements (#1755) Updates `make run` (used locally during development) to: - Run a second build after the first, which allows for easy testing of cached workflows. This second build uses a different build directory path to the first, to match the Heroku Cedar/classic build system and allow for testing that relocation/path rewriting works as expected. - Exit non-zero if the compile failed (it previously didn't, so we could test `bin/report` for failing builds - but that's now handled via a customisable exit code). GUS-W-17879839. --- .github/workflows/ci.yml | 2 +- Makefile | 23 ++++++++++++++--------- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 57f26122d..56224eeb1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -73,4 +73,4 @@ jobs: - name: Run buildpack using default app fixture run: make run - name: Run buildpack using an app fixture that's expected to fail - run: make run FIXTURE=spec/fixtures/python_version_file_invalid_version/ + run: make run FIXTURE=spec/fixtures/python_version_file_invalid_version/ COMPILE_FAILURE_EXIT_CODE=0 diff --git a/Makefile b/Makefile index 43a6113b3..6ba94fef7 100644 --- a/Makefile +++ b/Makefile @@ -3,6 +3,8 @@ STACK ?= heroku-24 FIXTURE ?= spec/fixtures/python_version_unspecified +# Allow overriding the exit code in CI, so we can test bin/report works for failing builds. +COMPILE_FAILURE_EXIT_CODE ?= 1 # Converts a stack name of `heroku-NN` to its build Docker image tag of `heroku/heroku:NN-build`. STACK_IMAGE_TAG := heroku/$(subst -,:,$(STACK))-build @@ -24,18 +26,21 @@ format: run: @echo "Running buildpack using: STACK=$(STACK) FIXTURE=$(FIXTURE)" @docker run --rm -v $(PWD):/src:ro --tmpfs /app -e "HOME=/app" -e "STACK=$(STACK)" "$(STACK_IMAGE_TAG)" \ - bash -euo pipefail -c '\ - mkdir /tmp/buildpack /tmp/build /tmp/cache /tmp/env; \ + bash -euo pipefail -O dotglob -c '\ + mkdir /tmp/buildpack /tmp/cache /tmp/env; \ cp -r /src/{bin,lib,requirements,vendor} /tmp/buildpack; \ - cp -rT /src/$(FIXTURE) /tmp/build; \ + cp -r /src/$(FIXTURE) /tmp/build_1; \ cd /tmp/buildpack; \ unset $$(printenv | cut -d '=' -f 1 | grep -vE "^(HOME|LANG|PATH|STACK)$$"); \ - echo -e "\n~ Detect:" && ./bin/detect /tmp/build; \ - echo -e "\n~ Compile:" && { ./bin/compile /tmp/build /tmp/cache /tmp/env || COMPILE_FAILED=1; }; \ - echo -e "\n~ Report:" && ./bin/report /tmp/build /tmp/cache /tmp/env; \ - [[ "$${COMPILE_FAILED:-}" == "1" ]] && exit 0; \ - [[ -f /tmp/build/bin/compile ]] && { echo -e "\n~ Compile (Inline Buildpack):" && (source ./export && /tmp/build/bin/compile /tmp/build /tmp/cache /tmp/env); }; \ - echo -e "\n~ Release:" && ./bin/release /tmp/build; \ + echo -en "\n~ Detect: " && ./bin/detect /tmp/build_1; \ + echo -e "\n~ Compile:" && { ./bin/compile /tmp/build_1 /tmp/cache /tmp/env || COMPILE_FAILED=1; }; \ + echo -e "\n~ Report:" && ./bin/report /tmp/build_1 /tmp/cache /tmp/env; \ + [[ "$${COMPILE_FAILED:-}" == "1" ]] && exit $(COMPILE_FAILURE_EXIT_CODE); \ + [[ -f /tmp/build_1/bin/compile ]] && { echo -e "\n~ Compile (Inline Buildpack):" && (source ./export && /tmp/build_1/bin/compile /tmp/build_1 /tmp/cache /tmp/env); }; \ + echo -e "\n~ Release:" && ./bin/release /tmp/build_1; \ + rm -rf /app/* /tmp/buildpack/export /tmp/build_1; \ + cp -r /src/$(FIXTURE) /tmp/build_2; \ + echo -e "\n~ Recompile:" && ./bin/compile /tmp/build_2 /tmp/cache /tmp/env; \ echo -e "\nBuild successful!"; \ ' @echo From b38d4620c382d9645d3cca449f24748acee5e5bd Mon Sep 17 00:00:00 2001 From: Ed Morley <501702+edmorley@users.noreply.github.com> Date: Sat, 22 Feb 2025 08:21:47 +0000 Subject: [PATCH 09/26] Rewrite editable VCS paths at build time too (#1756) Heroku builds occur at a different path to which the app will be run at run-time. As such, we have to perform path rewriting for editable dependencies, so that they work after relocation. The existing rewriting is performed at app boot (see code comment for more details), and works fine with pip and Poetry. However, I discovered that Pipenv doesn't correctly reinstall editable VCS dependencies if they use the new PEP660 style editable interface, which I've reported upstream here: https://github.com/pypa/pipenv/issues/6348 This issue has affected apps using editable VCS dependencies with Pipenv for some time, but until now only at build-time for cached builds. However, after #1753 (which thankfully isn't yet released, due to me catching this as part of updating the tests to exercise the new PEP660 style editable interface) would otherwise affect apps at run-time too. As a workaround, we can perform build time rewriting of paths too, but must do so only for VCS dependencies (see code comment for why). Lastly, the Pipenv bug also requires that we perform explicit cache invalidation for Pipenv apps after the src dir move in #1753. GUS-W-17884520. --- CHANGELOG.md | 1 + bin/compile | 42 ++++++++++++++--- lib/cache.sh | 4 ++ .../pip_editable/bin/test-entrypoints.sh | 2 +- spec/fixtures/pipenv_editable/.python-version | 3 -- spec/fixtures/pipenv_editable/Pipfile | 2 +- spec/fixtures/pipenv_editable/Pipfile.lock | 20 ++++----- .../pipenv_editable/bin/test-entrypoints.sh | 4 +- spec/fixtures/pipenv_editable/pyproject.toml | 2 +- spec/hatchet/pip_spec.rb | 16 +++---- spec/hatchet/pipenv_spec.rb | 45 +++++++------------ spec/hatchet/poetry_spec.rb | 8 ++-- 12 files changed, 85 insertions(+), 64 deletions(-) delete mode 100644 spec/fixtures/pipenv_editable/.python-version diff --git a/CHANGELOG.md b/CHANGELOG.md index bf4cba079..73ffabe25 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## [Unreleased] +- Added build-time rewriting of editable VCS dependency paths (in addition to the existing run-time rewriting), to work around an upstream Pipenv bug with editable VCS dependencies not being reinstalled correctly for cached builds. ([#1756](https://github.com/heroku/heroku-buildpack-python/pull/1756)) - Changed the location of repositories for editable VCS dependencies when using pip and Pipenv, to improve build performance and match the behaviour when using Poetry. ([#1753](https://github.com/heroku/heroku-buildpack-python/pull/1753)) ## [v277] - 2025-02-17 diff --git a/bin/compile b/bin/compile index 9f2869fc6..7a20a8faf 100755 --- a/bin/compile +++ b/bin/compile @@ -280,13 +280,43 @@ if [[ \$HOME != "/app" ]]; then fi EOT -# At runtime, rewrite paths in editable package .egg-link, .pth and finder files from the build time paths -# (such as `/tmp/build_`) back to `/app`. This is not done during the build itself, since later -# buildpacks still need the build time paths. +# When dependencies are installed in editable mode, the package manager/build backend creates `.pth` +# (and related) files in site-packages, which contain absolute path references to the actual location +# of the packages. By default the Heroku build runs from a directory like `/tmp/build_`, which +# changes every build and also differs from the app location at runtime (`/app`). This means any build +# directory paths referenced in .pth and related files will no longer exist at runtime or during cached +# rebuilds, unless we rewrite the paths. +# +# Ideally, we would be able to rewrite all paths to use the `/app/.heroku/python/` symlink trick we use +# when invoking Python, since then the same path would work across the current build, runtime and cached +# rebuilds. However, this trick only works for paths under that directory (since it's not possible to +# symlink `/app` or other directories we don't own), and when apps use path-based editable dependencies +# the paths will be outside of that (such as a subdirectory of the app source, or even the root of the +# build directory). We also can't just rewrite all paths now ready for runtime, since other buildpacks +# might run after this one that make use of the editable dependencies. As such, we have to perform path +# rewriting for path-based editable dependencies at app boot instead. +# +# For VCS editable dependencies, we can use the symlink trick and so configure the repo checkout location +# as `/app/.heroku/python/src/`, which in theory should mean the `.pth` files use that path. However, +# some build backends (such as setuptools' PEP660 implementation) call realpath on it causing the +# `/tmp/build_*` location to be written instead, meaning VCS src paths need to be rewritten regardless. +# +# In addition to ensuring dependencies work for subsequent buildpacks and at runtime, they must also +# work for cached rebuilds. Most package managers will reinstall editable dependencies regardless on +# next install, which means we can avoid having to rewrite paths on cache restore from the old build +# directory to the new location (`/tmp/build_`). However, Pipenv has a bug when using +# PEP660 style editable VCS dependencies where it won't reinstall if it's missing (or in our case, the +# path has changed), which means we must make sure that VCS src paths stored in the cache do use the +# symlink path. See: https://github.com/pypa/pipenv/issues/6348 +# +# As such, we have to perform two rewrites: +# 1. At build time, of just the VCS editable paths (which we can safely change to /app paths early). +# 2. At runtime, to rewrite the remaining path-based editable dependency paths. if [[ "${BUILD_DIR}" != "/app" ]]; then - cat <>"$PROFILE_PATH" -find .heroku/python/lib/python*/site-packages/ -type f -and \( -name '*.egg-link' -or -name '*.pth' -or -name '__editable___*_finder.py' \) -exec sed -i -e 's#${BUILD_DIR}#/app#' {} \+ -EOT + find .heroku/python/lib/python*/site-packages/ -type f -and \( -name '*.egg-link' -or -name '*.pth' -or -name '__editable___*_finder.py' \) -exec sed -i -e "s#${BUILD_DIR}/.heroku/python#/app/.heroku/python#" {} \+ + cat <<-EOT >>"${PROFILE_PATH}" + find .heroku/python/lib/python*/site-packages/ -type f -and \( -name '*.egg-link' -or -name '*.pth' -or -name '__editable___*_finder.py' \) -exec sed -i -e 's#${BUILD_DIR}#/app#' {} \+ + EOT fi # Install sane-default script for $WEB_CONCURRENCY and $FORWARDED_ALLOW_IPS. diff --git a/lib/cache.sh b/lib/cache.sh index 8353a9093..f5055b982 100644 --- a/lib/cache.sh +++ b/lib/cache.sh @@ -102,6 +102,10 @@ function cache::restore() { elif [[ "${cached_pipenv_version}" != "${PIPENV_VERSION:?}" ]]; then cache_invalidation_reasons+=("The Pipenv version has changed from ${cached_pipenv_version} to ${PIPENV_VERSION}") fi + # TODO: Remove this next time the Pipenv version is bumped (since it will trigger cache invalidation of its own) + if [[ -d "${cache_dir}/.heroku/src" ]]; then + cache_invalidation_reasons+=("The editable VCS repository location has changed (and Pipenv doesn't handle this correctly)") + fi ;; poetry) local cached_poetry_version diff --git a/spec/fixtures/pip_editable/bin/test-entrypoints.sh b/spec/fixtures/pip_editable/bin/test-entrypoints.sh index 4c27ba0f4..43afcb553 100755 --- a/spec/fixtures/pip_editable/bin/test-entrypoints.sh +++ b/spec/fixtures/pip_editable/bin/test-entrypoints.sh @@ -4,7 +4,7 @@ set -euo pipefail cd .heroku/python/lib/python*/site-packages/ -# List any path like strings in .egg-link, .pth, and finder files in site-packages. +# List any path like strings in the .egg-link, .pth, and finder files in site-packages. grep --extended-regexp --only-matching -- '/\S+' *.egg-link *.pth __editable___*_finder.py | sort echo diff --git a/spec/fixtures/pipenv_editable/.python-version b/spec/fixtures/pipenv_editable/.python-version deleted file mode 100644 index b81161743..000000000 --- a/spec/fixtures/pipenv_editable/.python-version +++ /dev/null @@ -1,3 +0,0 @@ -# Note: This test has to use Python 3.12 until we work around the -# Pipenv editable VCS dependency cache invalidation bug. -3.12 diff --git a/spec/fixtures/pipenv_editable/Pipfile b/spec/fixtures/pipenv_editable/Pipfile index 910d90713..8bb51ee30 100644 --- a/spec/fixtures/pipenv_editable/Pipfile +++ b/spec/fixtures/pipenv_editable/Pipfile @@ -4,7 +4,7 @@ verify_ssl = true name = "pypi" [packages] -gunicorn = {git = "git+https://github.com/benoitc/gunicorn", ref = "20.1.0", editable = true} +gunicorn = {git = "git+https://github.com/benoitc/gunicorn", editable = true} local-package-pyproject-toml = {file = "packages/local_package_pyproject_toml", editable = true} local-package-setup-py = {file = "packages/local_package_setup_py", editable = true} pipenv-editable = {file = ".", editable = true} diff --git a/spec/fixtures/pipenv_editable/Pipfile.lock b/spec/fixtures/pipenv_editable/Pipfile.lock index f6f2acc1b..8c4b604a3 100644 --- a/spec/fixtures/pipenv_editable/Pipfile.lock +++ b/spec/fixtures/pipenv_editable/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "ed18332549853701536926a2bae44d9fc142a08c105d4fbf108dbde04b3fedf2" + "sha256": "bedd8ea507283c5458c9c2cb1fd55a6e5e69fecba7814ffc96bf25e03feaeabf" }, "pipfile-spec": 6, "requires": {}, @@ -18,7 +18,7 @@ "editable": true, "git": "git+https://github.com/benoitc/gunicorn", "markers": "python_version >= '3.7'", - "ref": "61ccfd6c38d477a908e0f376757bbb884438053a" + "ref": "bacbf8aa5152b94e44aa5d2a94aeaf0318a85248" }, "local-package-pyproject-toml": { "editable": true, @@ -28,17 +28,17 @@ "editable": true, "file": "packages/local_package_setup_py" }, + "packaging": { + "hashes": [ + "sha256:09abb1bccd265c01f4a3aa3f7a7db064b36514d2cba19a2f694fe6150451a759", + "sha256:c228a6dc5e932d346bc5739379109d49e8853dd8223571c7c5b55260edc0b97f" + ], + "markers": "python_version >= '3.8'", + "version": "==24.2" + }, "pipenv-editable": { "editable": true, "file": "." - }, - "setuptools": { - "hashes": [ - "sha256:c5afc8f407c626b8313a86e10311dd3f661c6cd9c09d4bf8c15c0e11f9f2b0e6", - "sha256:e3982f444617239225d675215d51f6ba05f845d4eec313da4418fdbb56fb27e3" - ], - "markers": "python_version >= '3.9'", - "version": "==75.8.0" } }, "develop": {} diff --git a/spec/fixtures/pipenv_editable/bin/test-entrypoints.sh b/spec/fixtures/pipenv_editable/bin/test-entrypoints.sh index 4c27ba0f4..0e66f44bb 100755 --- a/spec/fixtures/pipenv_editable/bin/test-entrypoints.sh +++ b/spec/fixtures/pipenv_editable/bin/test-entrypoints.sh @@ -4,8 +4,8 @@ set -euo pipefail cd .heroku/python/lib/python*/site-packages/ -# List any path like strings in .egg-link, .pth, and finder files in site-packages. -grep --extended-regexp --only-matching -- '/\S+' *.egg-link *.pth __editable___*_finder.py | sort +# List any path like strings in the .pth and finder files in site-packages. +grep --extended-regexp --only-matching -- '/\S+' *.pth __editable___*_finder.py | sort echo echo -n "Running entrypoint for the pyproject.toml-based local package: " diff --git a/spec/fixtures/pipenv_editable/pyproject.toml b/spec/fixtures/pipenv_editable/pyproject.toml index cf7b46f69..2ca4481db 100644 --- a/spec/fixtures/pipenv_editable/pyproject.toml +++ b/spec/fixtures/pipenv_editable/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "pipenv-editable" version = "0.0.0" -requires-python = ">=3.12" +requires-python = ">=3.13" [build-system] requires = ["hatchling"] diff --git a/spec/hatchet/pip_spec.rb b/spec/hatchet/pip_spec.rb index f67a55aa1..f27f713c4 100644 --- a/spec/hatchet/pip_spec.rb +++ b/spec/hatchet/pip_spec.rb @@ -101,20 +101,20 @@ app.deploy do |app| expect(clean_output(app.output)).to match(Regexp.new(<<~REGEX)) remote: -----> Running bin/post_compile hook - remote: easy-install.pth:/tmp/build_.+/.heroku/python/src/gunicorn + remote: easy-install.pth:/app/.heroku/python/src/gunicorn remote: easy-install.pth:/tmp/build_.+/packages/local_package_setup_py remote: __editable___local_package_pyproject_toml_0_0_1_finder.py:/tmp/build_.+/packages/local_package_pyproject_toml/local_package_pyproject_toml'} - remote: gunicorn.egg-link:/tmp/build_.+/.heroku/python/src/gunicorn + remote: gunicorn.egg-link:/app/.heroku/python/src/gunicorn remote: local-package-setup-py.egg-link:/tmp/build_.+/packages/local_package_setup_py remote: remote: Running entrypoint for the pyproject.toml-based local package: Hello pyproject.toml! remote: Running entrypoint for the setup.py-based local package: Hello setup.py! remote: Running entrypoint for the VCS package: gunicorn \\(version 20.1.0\\) remote: -----> Inline app detected - remote: easy-install.pth:/tmp/build_.+/.heroku/python/src/gunicorn + remote: easy-install.pth:/app/.heroku/python/src/gunicorn remote: easy-install.pth:/tmp/build_.+/packages/local_package_setup_py remote: __editable___local_package_pyproject_toml_0_0_1_finder.py:/tmp/build_.+/packages/local_package_pyproject_toml/local_package_pyproject_toml'} - remote: gunicorn.egg-link:/tmp/build_.+/.heroku/python/src/gunicorn + remote: gunicorn.egg-link:/app/.heroku/python/src/gunicorn remote: local-package-setup-py.egg-link:/tmp/build_.+/packages/local_package_setup_py remote: remote: Running entrypoint for the pyproject.toml-based local package: Hello pyproject.toml! @@ -140,20 +140,20 @@ app.push! expect(clean_output(app.output)).to match(Regexp.new(<<~REGEX)) remote: -----> Running bin/post_compile hook - remote: easy-install.pth:/tmp/build_.+/.heroku/python/src/gunicorn + remote: easy-install.pth:/app/.heroku/python/src/gunicorn remote: easy-install.pth:/tmp/build_.+/packages/local_package_setup_py remote: __editable___local_package_pyproject_toml_0_0_1_finder.py:/tmp/build_.+/packages/local_package_pyproject_toml/local_package_pyproject_toml'} - remote: gunicorn.egg-link:/tmp/build_.+/.heroku/python/src/gunicorn + remote: gunicorn.egg-link:/app/.heroku/python/src/gunicorn remote: local-package-setup-py.egg-link:/tmp/build_.+/packages/local_package_setup_py remote: remote: Running entrypoint for the pyproject.toml-based local package: Hello pyproject.toml! remote: Running entrypoint for the setup.py-based local package: Hello setup.py! remote: Running entrypoint for the VCS package: gunicorn \\(version 20.1.0\\) remote: -----> Inline app detected - remote: easy-install.pth:/tmp/build_.+/.heroku/python/src/gunicorn + remote: easy-install.pth:/app/.heroku/python/src/gunicorn remote: easy-install.pth:/tmp/build_.+/packages/local_package_setup_py remote: __editable___local_package_pyproject_toml_0_0_1_finder.py:/tmp/build_.+/packages/local_package_pyproject_toml/local_package_pyproject_toml'} - remote: gunicorn.egg-link:/tmp/build_.+/.heroku/python/src/gunicorn + remote: gunicorn.egg-link:/app/.heroku/python/src/gunicorn remote: local-package-setup-py.egg-link:/tmp/build_.+/packages/local_package_setup_py remote: remote: Running entrypoint for the pyproject.toml-based local package: Hello pyproject.toml! diff --git a/spec/hatchet/pipenv_spec.rb b/spec/hatchet/pipenv_spec.rb index 0ff606900..02f9007fa 100644 --- a/spec/hatchet/pipenv_spec.rb +++ b/spec/hatchet/pipenv_spec.rb @@ -337,6 +337,7 @@ remote: -----> Discarding cache since: remote: - The Python version has changed from 3.12.4 to #{DEFAULT_PYTHON_FULL_VERSION} remote: - The Pipenv version has changed from 2023.12.1 to #{PIPENV_VERSION} + remote: - The editable VCS repository location has changed \\(and Pipenv doesn't handle this correctly\\) remote: -----> Installing Python #{DEFAULT_PYTHON_FULL_VERSION} remote: -----> Installing pip #{PIP_VERSION} remote: -----> Installing Pipenv #{PIPENV_VERSION} @@ -348,53 +349,45 @@ end end - # This test has to use Python 3.12 until we work around the Pipenv editable VCS dependency - # cache invalidation bug when using pyproject.toml / PEP517 based installs. context 'when Pipfile contains editable requirements' do let(:buildpacks) { [:default, 'heroku-community/inline'] } let(:app) { Hatchet::Runner.new('spec/fixtures/pipenv_editable', buildpacks:) } - it 'rewrites .pth, .egg-link and finder paths correctly for hooks, later buildpacks, runtime and cached builds' do + it 'rewrites .pth and finder paths correctly for hooks, later buildpacks, runtime and cached builds' do app.deploy do |app| expect(clean_output(app.output)).to match(Regexp.new(<<~REGEX)) remote: -----> Installing dependencies using 'pipenv install --deploy' remote: Installing dependencies from Pipfile.lock \\(.+\\)... remote: -----> Running bin/post_compile hook - remote: easy-install.pth:/tmp/build_.+/.heroku/python/src/gunicorn - remote: easy-install.pth:/tmp/build_.+/packages/local_package_setup_py + remote: __editable___gunicorn_23_0_0_finder.py:/app/.heroku/python/src/gunicorn/gunicorn'} remote: __editable___local_package_pyproject_toml_0_0_1_finder.py:/tmp/build_.+/packages/local_package_pyproject_toml/local_package_pyproject_toml'} - remote: gunicorn.egg-link:/tmp/build_.+/.heroku/python/src/gunicorn - remote: local-package-setup-py.egg-link:/tmp/build_.+/packages/local_package_setup_py + remote: __editable___local_package_setup_py_0_0_1_finder.py:/tmp/build_.+/packages/local_package_setup_py/local_package_setup_py'} remote: _pipenv_editable.pth:/tmp/build_.+ remote: remote: Running entrypoint for the pyproject.toml-based local package: Hello pyproject.toml! remote: Running entrypoint for the setup.py-based local package: Hello setup.py! - remote: Running entrypoint for the VCS package: gunicorn \\(version 20.1.0\\) + remote: Running entrypoint for the VCS package: gunicorn \\(version 23.0.0\\) remote: -----> Inline app detected - remote: easy-install.pth:/tmp/build_.+/.heroku/python/src/gunicorn - remote: easy-install.pth:/tmp/build_.+/packages/local_package_setup_py + remote: __editable___gunicorn_23_0_0_finder.py:/app/.heroku/python/src/gunicorn/gunicorn'} remote: __editable___local_package_pyproject_toml_0_0_1_finder.py:/tmp/build_.+/packages/local_package_pyproject_toml/local_package_pyproject_toml'} - remote: gunicorn.egg-link:/tmp/build_.+/.heroku/python/src/gunicorn - remote: local-package-setup-py.egg-link:/tmp/build_.+/packages/local_package_setup_py + remote: __editable___local_package_setup_py_0_0_1_finder.py:/tmp/build_.+/packages/local_package_setup_py/local_package_setup_py'} remote: _pipenv_editable.pth:/tmp/build_.+ remote: remote: Running entrypoint for the pyproject.toml-based local package: Hello pyproject.toml! remote: Running entrypoint for the setup.py-based local package: Hello setup.py! - remote: Running entrypoint for the VCS package: gunicorn \\(version 20.1.0\\) + remote: Running entrypoint for the VCS package: gunicorn \\(version 23.0.0\\) REGEX # Test rewritten paths work at runtime. expect(app.run('bin/test-entrypoints.sh')).to include(<<~OUTPUT) - easy-install.pth:/app/.heroku/python/src/gunicorn - easy-install.pth:/app/packages/local_package_setup_py + __editable___gunicorn_23_0_0_finder.py:/app/.heroku/python/src/gunicorn/gunicorn'} __editable___local_package_pyproject_toml_0_0_1_finder.py:/app/packages/local_package_pyproject_toml/local_package_pyproject_toml'} - gunicorn.egg-link:/app/.heroku/python/src/gunicorn - local-package-setup-py.egg-link:/app/packages/local_package_setup_py + __editable___local_package_setup_py_0_0_1_finder.py:/app/packages/local_package_setup_py/local_package_setup_py'} _pipenv_editable.pth:/app Running entrypoint for the pyproject.toml-based local package: Hello pyproject.toml! Running entrypoint for the setup.py-based local package: Hello setup.py! - Running entrypoint for the VCS package: gunicorn (version 20.1.0) + Running entrypoint for the VCS package: gunicorn (version 23.0.0) OUTPUT # Test that the cached .pth files work correctly. @@ -404,27 +397,23 @@ remote: -----> Installing dependencies using 'pipenv install --deploy' remote: Installing dependencies from Pipfile.lock \\(.+\\)... remote: -----> Running bin/post_compile hook - remote: easy-install.pth:/tmp/build_.+/.heroku/python/src/gunicorn - remote: easy-install.pth:/tmp/build_.+/packages/local_package_setup_py + remote: __editable___gunicorn_23_0_0_finder.py:/app/.heroku/python/src/gunicorn/gunicorn'} remote: __editable___local_package_pyproject_toml_0_0_1_finder.py:/tmp/build_.+/packages/local_package_pyproject_toml/local_package_pyproject_toml'} - remote: gunicorn.egg-link:/tmp/build_.+/.heroku/python/src/gunicorn - remote: local-package-setup-py.egg-link:/tmp/build_.+/packages/local_package_setup_py + remote: __editable___local_package_setup_py_0_0_1_finder.py:/tmp/build_.+/packages/local_package_setup_py/local_package_setup_py'} remote: _pipenv_editable.pth:/tmp/build_.+ remote: remote: Running entrypoint for the pyproject.toml-based local package: Hello pyproject.toml! remote: Running entrypoint for the setup.py-based local package: Hello setup.py! - remote: Running entrypoint for the VCS package: gunicorn \\(version 20.1.0\\) + remote: Running entrypoint for the VCS package: gunicorn \\(version 23.0.0\\) remote: -----> Inline app detected - remote: easy-install.pth:/tmp/build_.+/.heroku/python/src/gunicorn - remote: easy-install.pth:/tmp/build_.+/packages/local_package_setup_py + remote: __editable___gunicorn_23_0_0_finder.py:/app/.heroku/python/src/gunicorn/gunicorn'} remote: __editable___local_package_pyproject_toml_0_0_1_finder.py:/tmp/build_.+/packages/local_package_pyproject_toml/local_package_pyproject_toml'} - remote: gunicorn.egg-link:/tmp/build_.+/.heroku/python/src/gunicorn - remote: local-package-setup-py.egg-link:/tmp/build_.+/packages/local_package_setup_py + remote: __editable___local_package_setup_py_0_0_1_finder.py:/tmp/build_.+/packages/local_package_setup_py/local_package_setup_py'} remote: _pipenv_editable.pth:/tmp/build_.+ remote: remote: Running entrypoint for the pyproject.toml-based local package: Hello pyproject.toml! remote: Running entrypoint for the setup.py-based local package: Hello setup.py! - remote: Running entrypoint for the VCS package: gunicorn \\(version 20.1.0\\) + remote: Running entrypoint for the VCS package: gunicorn \\(version 23.0.0\\) REGEX end end diff --git a/spec/hatchet/poetry_spec.rb b/spec/hatchet/poetry_spec.rb index 456a35996..874715f00 100644 --- a/spec/hatchet/poetry_spec.rb +++ b/spec/hatchet/poetry_spec.rb @@ -109,7 +109,7 @@ remote: remote: Installing the current project: poetry-editable \\(0.0.1\\) remote: -----> Running bin/post_compile hook - remote: __editable___gunicorn_23_0_0_finder.py:/tmp/build_.+/.heroku/python/src/gunicorn/gunicorn'} + remote: __editable___gunicorn_23_0_0_finder.py:/app/.heroku/python/src/gunicorn/gunicorn'} remote: __editable___local_package_pyproject_toml_0_0_1_finder.py:/tmp/build_.+/packages/local_package_pyproject_toml/local_package_pyproject_toml'} remote: __editable___local_package_setup_py_0_0_1_finder.py:/tmp/build_.+/packages/local_package_setup_py/local_package_setup_py'} remote: poetry_editable.pth:/tmp/build_.+ @@ -118,7 +118,7 @@ remote: Running entrypoint for the setup.py-based local package: Hello setup.py! remote: Running entrypoint for the VCS package: gunicorn \\(version 23.0.0\\) remote: -----> Inline app detected - remote: __editable___gunicorn_23_0_0_finder.py:/tmp/build_.+/.heroku/python/src/gunicorn/gunicorn'} + remote: __editable___gunicorn_23_0_0_finder.py:/app/.heroku/python/src/gunicorn/gunicorn'} remote: __editable___local_package_pyproject_toml_0_0_1_finder.py:/tmp/build_.+/packages/local_package_pyproject_toml/local_package_pyproject_toml'} remote: __editable___local_package_setup_py_0_0_1_finder.py:/tmp/build_.+/packages/local_package_setup_py/local_package_setup_py'} remote: poetry_editable.pth:/tmp/build_.+ @@ -155,7 +155,7 @@ remote: remote: Installing the current project: poetry-editable \\(0.0.1\\) remote: -----> Running bin/post_compile hook - remote: __editable___gunicorn_23_0_0_finder.py:/tmp/build_.+/.heroku/python/src/gunicorn/gunicorn'} + remote: __editable___gunicorn_23_0_0_finder.py:/app/.heroku/python/src/gunicorn/gunicorn'} remote: __editable___local_package_pyproject_toml_0_0_1_finder.py:/tmp/build_.+/packages/local_package_pyproject_toml/local_package_pyproject_toml'} remote: __editable___local_package_setup_py_0_0_1_finder.py:/tmp/build_.+/packages/local_package_setup_py/local_package_setup_py'} remote: poetry_editable.pth:/tmp/build_.+ @@ -164,7 +164,7 @@ remote: Running entrypoint for the setup.py-based local package: Hello setup.py! remote: Running entrypoint for the VCS package: gunicorn \\(version 23.0.0\\) remote: -----> Inline app detected - remote: __editable___gunicorn_23_0_0_finder.py:/tmp/build_.+/.heroku/python/src/gunicorn/gunicorn'} + remote: __editable___gunicorn_23_0_0_finder.py:/app/.heroku/python/src/gunicorn/gunicorn'} remote: __editable___local_package_pyproject_toml_0_0_1_finder.py:/tmp/build_.+/packages/local_package_pyproject_toml/local_package_pyproject_toml'} remote: __editable___local_package_setup_py_0_0_1_finder.py:/tmp/build_.+/packages/local_package_setup_py/local_package_setup_py'} remote: poetry_editable.pth:/tmp/build_.+ From bf280f4524e53f48aaa031d25c9e789a2bc7160d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 22 Feb 2025 08:39:32 +0000 Subject: [PATCH 10/26] Bump the ruby-dependencies group with 4 updates (#1757) Bumps the ruby-dependencies group with 4 updates: [logger](https://github.com/ruby/logger), [rspec-core](https://github.com/rspec/rspec-core), [rubocop](https://github.com/rubocop/rubocop) and [rubocop-rspec](https://github.com/rubocop/rubocop-rspec). Updates `logger` from 1.6.5 to 1.6.6 - [Release notes](https://github.com/ruby/logger/releases) - [Commits](https://github.com/ruby/logger/compare/v1.6.5...v1.6.6) Updates `rspec-core` from 3.13.2 to 3.13.3 - [Release notes](https://github.com/rspec/rspec-core/releases) - [Changelog](https://github.com/rspec/rspec-core/blob/main/Changelog.md) - [Commits](https://github.com/rspec/rspec-core/commits) Updates `rubocop` from 1.71.1 to 1.72.2 - [Release notes](https://github.com/rubocop/rubocop/releases) - [Changelog](https://github.com/rubocop/rubocop/blob/master/CHANGELOG.md) - [Commits](https://github.com/rubocop/rubocop/compare/v1.71.1...v1.72.2) Updates `rubocop-rspec` from 3.4.0 to 3.5.0 - [Release notes](https://github.com/rubocop/rubocop-rspec/releases) - [Changelog](https://github.com/rubocop/rubocop-rspec/blob/master/CHANGELOG.md) - [Commits](https://github.com/rubocop/rubocop-rspec/compare/v3.4.0...v3.5.0) --- updated-dependencies: - dependency-name: logger dependency-type: direct:development update-type: version-update:semver-patch dependency-group: ruby-dependencies - dependency-name: rspec-core dependency-type: direct:development update-type: version-update:semver-patch dependency-group: ruby-dependencies - dependency-name: rubocop dependency-type: direct:development update-type: version-update:semver-minor dependency-group: ruby-dependencies - dependency-name: rubocop-rspec dependency-type: direct:development update-type: version-update:semver-minor dependency-group: ruby-dependencies ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Gemfile.lock | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index b708a7e83..efa4e8d6b 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -19,16 +19,17 @@ GEM rrrretry (~> 1) thor (~> 1) threaded (~> 0) - json (2.9.1) + json (2.10.1) language_server-protocol (3.17.0.4) - logger (1.6.5) + lint_roller (1.1.0) + logger (1.6.6) moneta (1.0.0) multi_json (1.15.0) parallel (1.26.3) parallel_split_test (0.10.0) parallel (>= 0.5.13) rspec-core (>= 3.9.0) - parser (3.3.7.0) + parser (3.3.7.1) ast (~> 2.4.1) racc platform-api (3.7.0) @@ -40,7 +41,7 @@ GEM rate_throttle_client (0.1.2) regexp_parser (2.10.0) rrrretry (1.0.0) - rspec-core (3.13.2) + rspec-core (3.13.3) rspec-support (~> 3.13.0) rspec-expectations (3.13.3) diff-lcs (>= 1.2.0, < 2.0) @@ -48,9 +49,10 @@ GEM rspec-retry (0.6.2) rspec-core (> 3.3) rspec-support (3.13.2) - rubocop (1.71.1) + rubocop (1.72.2) json (~> 2.3) - language_server-protocol (>= 3.17.0) + language_server-protocol (~> 3.17.0.2) + lint_roller (~> 1.1.0) parallel (~> 1.10) parser (>= 3.3.0.2) rainbow (>= 2.2.2, < 4.0) @@ -60,8 +62,9 @@ GEM unicode-display_width (>= 2.4.0, < 4.0) rubocop-ast (1.38.0) parser (>= 3.3.1.0) - rubocop-rspec (3.4.0) - rubocop (~> 1.61) + rubocop-rspec (3.5.0) + lint_roller (~> 1.1) + rubocop (~> 1.72, >= 1.72.1) ruby-progressbar (1.13.0) thor (1.3.2) threaded (0.0.4) From 53a59ae3cb7f0bd7b55d5bdcc2005989b2d3825b Mon Sep 17 00:00:00 2001 From: "heroku-linguist[bot]" <136119646+heroku-linguist[bot]@users.noreply.github.com> Date: Mon, 24 Feb 2025 09:22:12 +0000 Subject: [PATCH 11/26] Prepare release v278 (#1760) Co-authored-by: heroku-linguist[bot] <136119646+heroku-linguist[bot]@users.noreply.github.com> --- CHANGELOG.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 73ffabe25..3b17a1d9a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,9 @@ ## [Unreleased] + +## [v278] - 2025-02-24 + - Added build-time rewriting of editable VCS dependency paths (in addition to the existing run-time rewriting), to work around an upstream Pipenv bug with editable VCS dependencies not being reinstalled correctly for cached builds. ([#1756](https://github.com/heroku/heroku-buildpack-python/pull/1756)) - Changed the location of repositories for editable VCS dependencies when using pip and Pipenv, to improve build performance and match the behaviour when using Poetry. ([#1753](https://github.com/heroku/heroku-buildpack-python/pull/1753)) @@ -1170,7 +1173,8 @@ Default Python is now latest 2.7.10. Updated pip and Distribute. - Setuptools updated to v16.0 - pip updated to v7.0.1 -[unreleased]: https://github.com/heroku/heroku-buildpack-python/compare/v277...main +[unreleased]: https://github.com/heroku/heroku-buildpack-python/compare/v278...main +[v278]: https://github.com/heroku/heroku-buildpack-python/compare/v277...v278 [v277]: https://github.com/heroku/heroku-buildpack-python/compare/v276...v277 [v276]: https://github.com/heroku/heroku-buildpack-python/compare/v275...v276 [v275]: https://github.com/heroku/heroku-buildpack-python/compare/v274...v275 From 992f51d5c94bdf0de25d651d7bc9ca5e20b65eb3 Mon Sep 17 00:00:00 2001 From: Ed Morley <501702+edmorley@users.noreply.github.com> Date: Mon, 24 Feb 2025 14:35:10 +0000 Subject: [PATCH 12/26] Stop installing pip into Poetry's virtual environment (#1761) This is an alternative approach to installing Poetry that means we can skip installing pip into its virtual environment, but still support the outdated Python versions which bundle older pip (that don't support the `--python` option; see #1687) or that don't correctly isolate the environment when running `ensurepip` (see #1698). Skipping installing pip speeds up the cold cache build for Poetry slightly, and also reduces the build cache size (which will help with the cache save and restore times for warm builds too). The pip installed in the Poetry venv wasn't exposed to apps (since it wasn't on `PATH`) so is safe to remove. GUS-W-17895154. --- CHANGELOG.md | 1 + bin/compile | 2 +- lib/poetry.sh | 31 +++++++++++++++++-------------- 3 files changed, 19 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3b17a1d9a..78d572d74 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## [Unreleased] +- Stopped installing pip into Poetry's virtual environment. ([#1761](https://github.com/heroku/heroku-buildpack-python/pull/1761)) ## [v278] - 2025-02-24 diff --git a/bin/compile b/bin/compile index 7a20a8faf..cda662380 100755 --- a/bin/compile +++ b/bin/compile @@ -205,7 +205,7 @@ case "${package_manager}" in pipenv::install_pipenv ;; poetry) - poetry::install_poetry "${CACHE_DIR}" "${EXPORT_PATH}" + poetry::install_poetry "${python_home}" "${python_major_version}" "${CACHE_DIR}" "${EXPORT_PATH}" ;; *) utils::abort_internal_error "Unhandled package manager: ${package_manager}" diff --git a/lib/poetry.sh b/lib/poetry.sh index a227ae1cd..5bdfdf9c5 100644 --- a/lib/poetry.sh +++ b/lib/poetry.sh @@ -7,8 +7,10 @@ set -euo pipefail POETRY_VERSION=$(utils::get_requirement_version 'poetry') function poetry::install_poetry() { - local cache_dir="${1}" - local export_file="${2}" + local python_home="${1}" + local python_major_version="${2}" + local cache_dir="${3}" + local export_file="${4}" # We store Poetry in the build cache, since we only need it during the build. local poetry_root="${cache_dir}/.heroku/python-poetry" @@ -39,17 +41,12 @@ function poetry::install_poetry() { rm -rf "${poetry_root}" mkdir -p "${poetry_root}" - # We can't use the pip wheel bundled within Python's standard library to install Poetry - # (which would allow us to use `--without-pip` here to skip the pip install), since it - # requires using the `--python` option, which was only added in pip v22.3. And whilst - # all major Python versions we support now bundled a newer pip than that, some apps - # are still using outdated patch releases of those Python versions, whose bundled pip - # can be older (for example Python 3.9.0 ships with pip v20.2.1). Once Python 3.10 EOLs - # we can switch back to the previous approach since Python 3.11.0 ships with pip v22.3. - # Changing the working directory away from the build dir is required to work around an - # `ensurepip` bug in older Python versions, where it doesn't run Python in isolated mode: - # https://github.com/heroku/heroku-buildpack-python/issues/1697 - if ! (cd "${poetry_root}" && python -m venv "${poetry_venv_dir}"); then + # We use the pip wheel bundled within Python's standard library to install Poetry. + # Whilst Poetry does still require pip for some tasks (such as package uninstalls), + # it bundles its own copy for use as a fallback. As such we don't need to install pip + # into the Poetry venv (and in fact, Poetry wouldn't use this install anyway, since + # it only finds an external pip if it exists in the target venv). + if ! python -m venv --without-pip "${poetry_venv_dir}"; then output::error <<-EOF Internal Error: Unable to create virtual environment for Poetry. @@ -62,8 +59,14 @@ function poetry::install_poetry() { exit 1 fi + local bundled_pip_module_path + bundled_pip_module_path="$(utils::bundled_pip_module_path "${python_home}" "${python_major_version}")" + + # We must call the venv Python directly here, rather than relying on pip's `--python` + # option, since `--python` was only added in pip v22.3, so isn't supported by the older + # pip versions bundled with Python 3.9/3.10. if ! { - "${poetry_venv_dir}/bin/pip" \ + "${poetry_venv_dir}/bin/python" "${bundled_pip_module_path}" \ install \ --disable-pip-version-check \ --no-cache-dir \ From ff523c256984f4bb7b5dc694919f65837db3bb3a Mon Sep 17 00:00:00 2001 From: Ed Morley <501702+edmorley@users.noreply.github.com> Date: Mon, 24 Feb 2025 14:38:05 +0000 Subject: [PATCH 13/26] Misc Poetry cleanups (#1762) Ported from the experimental uv branch, since the uv implementation was based on the Poetry one. --- lib/poetry.sh | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/lib/poetry.sh b/lib/poetry.sh index 5bdfdf9c5..695fe78a3 100644 --- a/lib/poetry.sh +++ b/lib/poetry.sh @@ -94,14 +94,20 @@ function poetry::install_poetry() { fi export PATH="${poetry_bin_dir}:${PATH}" - echo "export PATH=\"${poetry_bin_dir}:\${PATH}\"" >>"${export_file}" # Force Poetry to manage the system Python site-packages instead of using venvs. export POETRY_VIRTUALENVS_CREATE="false" - echo 'export POETRY_VIRTUALENVS_CREATE="false"' >>"${export_file}" + + # Set the same env vars in the environment used by later buildpacks. + cat >>"${export_file}" <<-EOF + export PATH="${poetry_bin_dir}:\${PATH}" + export POETRY_VIRTUALENVS_CREATE="false" + EOF } # Note: We cache site-packages since: # - It results in faster builds than only caching Poetry's download/wheel cache. +# - It improves the UX of the build log, since Poetry will display which packages were +# added/removed since the last successful build. # - It's safe to do so, since `poetry sync` fully manages the environment (including # e.g. uninstalling packages when they are removed from the lockfile). # From 25172f45c5747fa3248e7a69c06062e79cba79df Mon Sep 17 00:00:00 2001 From: Ed Morley <501702+edmorley@users.noreply.github.com> Date: Mon, 24 Feb 2025 14:40:42 +0000 Subject: [PATCH 14/26] Migrate to Rubocop's new plugin syntax (#1763) To suppress this Rubocop warning: ``` $ make lint rubocop-rspec extension supports plugin, specify `plugins: rubocop-rspec` instead of `require: rubocop-rspec` in /Users/emorley/src/heroku-buildpack-python/.rubocop.yml. For more information, see https://docs.rubocop.org/rubocop/plugin_migration_guide.html. ``` --- .rubocop.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.rubocop.yml b/.rubocop.yml index 44a322aef..2edf1e039 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -1,4 +1,4 @@ -require: +plugins: - rubocop-rspec AllCops: From 3033b54deff927e3ecc41cbf91469cefb45b26f7 Mon Sep 17 00:00:00 2001 From: Ed Morley <501702+edmorley@users.noreply.github.com> Date: Mon, 24 Feb 2025 15:53:48 +0000 Subject: [PATCH 15/26] Improve package manager bootstrap error messages (#1764) * Any errors are now indented using the same indent helper used for other buildpack subprocesses. * Adds "temporary network issue" verbiage based on that now used for the Python runtime download step. GUS-W-17895970. --- CHANGELOG.md | 1 + lib/pip.sh | 10 +++++++--- lib/pipenv.sh | 11 ++++++++--- lib/poetry.sh | 13 +++++++++---- 4 files changed, 25 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 78d572d74..aa1bcd342 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## [Unreleased] +- Improved the error messages shown if installing pip/Poetry/Pipenv fails. ([#1764](https://github.com/heroku/heroku-buildpack-python/pull/1764)) - Stopped installing pip into Poetry's virtual environment. ([#1761](https://github.com/heroku/heroku-buildpack-python/pull/1761)) ## [v278] - 2025-02-24 diff --git a/lib/pip.sh b/lib/pip.sh index 62ebc43ee..4d05462f5 100644 --- a/lib/pip.sh +++ b/lib/pip.sh @@ -46,6 +46,7 @@ function pip::install_pip_setuptools_wheel() { # app's requirements.txt in the last build). The install will be a no-op if the versions match. output::step "Installing ${packages_display_text}" + # shellcheck disable=SC2310 # This function is invoked in an 'if' condition so set -e will be disabled. if ! { python "${bundled_pip_module_path}" \ install \ @@ -53,15 +54,18 @@ function pip::install_pip_setuptools_wheel() { --no-cache-dir \ --no-input \ --quiet \ - "${packages_to_install[@]}" + "${packages_to_install[@]}" \ + |& output::indent }; then output::error <<-EOF Error: Unable to install pip. + In some cases, this happens due to a temporary issue with + the network connection or Python Package Index (PyPI). + Try building again to see if the error resolves itself. - If that does not help, check the status of PyPI (the Python - package repository service), here: + If that does not help, check the status of PyPI here: https://status.python.org EOF meta_set "failure_reason" "install-package-manager::pip" diff --git a/lib/pipenv.sh b/lib/pipenv.sh index 9e8eb4594..c5c50beb4 100644 --- a/lib/pipenv.sh +++ b/lib/pipenv.sh @@ -18,6 +18,8 @@ function pipenv::install_pipenv() { # TODO: Install Pipenv into a venv so it isn't leaked into the app environment. # TODO: Skip installing Pipenv if its version hasn't changed (once it's installed into a venv). # TODO: Explore viability of making Pipenv only be available during the build, to reduce slug size. + + # shellcheck disable=SC2310 # This function is invoked in an 'if' condition so set -e will be disabled. if ! { pip \ install \ @@ -25,15 +27,18 @@ function pipenv::install_pipenv() { --no-cache-dir \ --no-input \ --quiet \ - "pipenv==${PIPENV_VERSION}" + "pipenv==${PIPENV_VERSION}" \ + |& output::indent }; then output::error <<-EOF Error: Unable to install Pipenv. + In some cases, this happens due to a temporary issue with + the network connection or Python Package Index (PyPI). + Try building again to see if the error resolves itself. - If that does not help, check the status of PyPI (the Python - package repository service), here: + If that does not help, check the status of PyPI here: https://status.python.org EOF meta_set "failure_reason" "install-package-manager::pipenv" diff --git a/lib/poetry.sh b/lib/poetry.sh index 695fe78a3..6b4888eea 100644 --- a/lib/poetry.sh +++ b/lib/poetry.sh @@ -46,7 +46,8 @@ function poetry::install_poetry() { # it bundles its own copy for use as a fallback. As such we don't need to install pip # into the Poetry venv (and in fact, Poetry wouldn't use this install anyway, since # it only finds an external pip if it exists in the target venv). - if ! python -m venv --without-pip "${poetry_venv_dir}"; then + # shellcheck disable=SC2310 # This function is invoked in an 'if' condition so set -e will be disabled. + if ! python -m venv --without-pip "${poetry_venv_dir}" |& output::indent; then output::error <<-EOF Internal Error: Unable to create virtual environment for Poetry. @@ -65,6 +66,7 @@ function poetry::install_poetry() { # We must call the venv Python directly here, rather than relying on pip's `--python` # option, since `--python` was only added in pip v22.3, so isn't supported by the older # pip versions bundled with Python 3.9/3.10. + # shellcheck disable=SC2310 # This function is invoked in an 'if' condition so set -e will be disabled. if ! { "${poetry_venv_dir}/bin/python" "${bundled_pip_module_path}" \ install \ @@ -72,15 +74,18 @@ function poetry::install_poetry() { --no-cache-dir \ --no-input \ --quiet \ - "poetry==${POETRY_VERSION}" + "poetry==${POETRY_VERSION}" \ + |& output::indent }; then output::error <<-EOF Error: Unable to install Poetry. + In some cases, this happens due to a temporary issue with + the network connection or Python Package Index (PyPI). + Try building again to see if the error resolves itself. - If that does not help, check the status of PyPI (the Python - package repository service), here: + If that does not help, check the status of PyPI here: https://status.python.org EOF meta_set "failure_reason" "install-package-manager::poetry" From caa1a19a8b47486eb20e169f0362cd8594110e15 Mon Sep 17 00:00:00 2001 From: Ed Morley <501702+edmorley@users.noreply.github.com> Date: Mon, 24 Feb 2025 17:40:36 +0000 Subject: [PATCH 16/26] Stop filtering out pip's "Requirement already satisfied" log lines (#1765) Since while removing them makes the install logs slightly shorter, it hides what is happening and makes it harder to see what package version a historic build may have been using from the logs alone. For example, when comparing a last successful build to a newly failing build. This now matches the behaviour for our other supported package managers, where we don't filter out install lines relating to already installed/cached packages. As a compromise, we still edit the lines slightly, to remove the redundant site-packages path information, which would otherwise cause each package message to span multiple lines. GUS-W-17897541. --- CHANGELOG.md | 1 + lib/pip.sh | 4 +++- spec/hatchet/ci_spec.rb | 7 +++++-- spec/hatchet/pip_spec.rb | 1 + 4 files changed, 10 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index aa1bcd342..5b2b73c3e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## [Unreleased] +- Stopped filtering out pip's `Requirement already satisfied:` log lines when installing dependencies. ([#1765](https://github.com/heroku/heroku-buildpack-python/pull/1765)) - Improved the error messages shown if installing pip/Poetry/Pipenv fails. ([#1764](https://github.com/heroku/heroku-buildpack-python/pull/1764)) - Stopped installing pip into Poetry's virtual environment. ([#1761](https://github.com/heroku/heroku-buildpack-python/pull/1761)) diff --git a/lib/pip.sh b/lib/pip.sh index 4d05462f5..e56e6d536 100644 --- a/lib/pip.sh +++ b/lib/pip.sh @@ -106,6 +106,8 @@ function pip::install_dependencies() { # We only display the most relevant command args here, to improve the signal to noise ratio. output::step "Installing dependencies using '${pip_install_command[*]}'" + # The sed usage is to reduce the verbosity of output lines like: + # "Requirement already satisfied: typing-extensions==4.12.2 in /app/.heroku/python/lib/python3.13/site-packages (from -r requirements.txt (line 5)) (4.12.2)" # shellcheck disable=SC2310 # This function is invoked in an 'if' condition so set -e will be disabled. if ! { "${pip_install_command[@]}" \ @@ -116,7 +118,7 @@ function pip::install_dependencies() { --progress-bar off \ --src='/app/.heroku/python/src' \ |& tee "${WARNINGS_LOG:?}" \ - |& sed --unbuffered --expression '/Requirement already satisfied/d' \ + |& sed --unbuffered --expression 's# in /app/.heroku/python/lib/python.*/site-packages##' \ |& output::indent }; then # TODO: Overhaul warnings and combine them with error handling. diff --git a/spec/hatchet/ci_spec.rb b/spec/hatchet/ci_spec.rb index aa4f58dd2..5ff6041be 100644 --- a/spec/hatchet/ci_spec.rb +++ b/spec/hatchet/ci_spec.rb @@ -62,16 +62,19 @@ REGEX test_run.run_again - expect(clean_output(test_run.output)).to include(<<~OUTPUT) + expect(clean_output(test_run.output)).to match(Regexp.new(<<~REGEX, Regexp::MULTILINE)) -----> Python app detected -----> Using Python #{DEFAULT_PYTHON_MAJOR_VERSION} specified in .python-version -----> Restoring cache -----> Using cached install of Python #{DEFAULT_PYTHON_FULL_VERSION} -----> Installing pip #{PIP_VERSION} -----> Installing dependencies using 'pip install -r requirements.txt -r requirements-test.txt' + Requirement already satisfied: typing-extensions==.+ + Requirement already satisfied: pytest==.+ + .+ -----> Skipping Django collectstatic since the env var DISABLE_COLLECTSTATIC is set. -----> Running bin/post_compile hook - OUTPUT + REGEX end end end diff --git a/spec/hatchet/pip_spec.rb b/spec/hatchet/pip_spec.rb index f27f713c4..5cc9f8b8e 100644 --- a/spec/hatchet/pip_spec.rb +++ b/spec/hatchet/pip_spec.rb @@ -53,6 +53,7 @@ remote: -----> Using cached install of Python #{DEFAULT_PYTHON_FULL_VERSION} remote: -----> Installing pip #{PIP_VERSION} remote: -----> Installing dependencies using 'pip install -r requirements.txt' + remote: Requirement already satisfied: typing-extensions==4.12.2 (from -r requirements.txt (line 5)) (4.12.2) remote: -----> Inline app detected OUTPUT end From 20cd0b69e571d2a720ba2fd59d383294a5236435 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 25 Feb 2025 22:37:39 +0000 Subject: [PATCH 17/26] Bump poetry from 2.0.1 to 2.1.1 (#1758) * Bump poetry from 2.0.1 to 2.1.1 Bumps [poetry](https://github.com/python-poetry/poetry) from 2.0.1 to 2.1.1. - [Release notes](https://github.com/python-poetry/poetry/releases) - [Changelog](https://github.com/python-poetry/poetry/blob/main/CHANGELOG.md) - [Commits](https://github.com/python-poetry/poetry/compare/2.0.1...2.1.1) --- updated-dependencies: - dependency-name: poetry dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] * Set `POETRY_VIRTUALENVS_USE_POETRY_PYTHON` Since in theory it should force Poetry to not search for other Python versions if the project's `requires-python` (or `tool.poetry` equivalent) doesn't match the Python version we installed. * Add changelog entry * Update one fixture to PEP-621 style `pyproject.toml` So we have fixtures testing both the new and old style config. --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Ed Morley <501702+edmorley@users.noreply.github.com> --- CHANGELOG.md | 1 + lib/poetry.sh | 4 ++++ requirements/poetry.txt | 2 +- spec/fixtures/poetry_basic/poetry.lock | 15 +++++++++++---- spec/fixtures/poetry_basic/pyproject.toml | 14 +++++++++----- spec/hatchet/ci_spec.rb | 2 ++ spec/hatchet/poetry_spec.rb | 5 +++-- 7 files changed, 31 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5b2b73c3e..29c88e5c7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## [Unreleased] +- Updated Poetry from 2.0.1 to 2.1.1. ([#1758](https://github.com/heroku/heroku-buildpack-python/pull/1758)) - Stopped filtering out pip's `Requirement already satisfied:` log lines when installing dependencies. ([#1765](https://github.com/heroku/heroku-buildpack-python/pull/1765)) - Improved the error messages shown if installing pip/Poetry/Pipenv fails. ([#1764](https://github.com/heroku/heroku-buildpack-python/pull/1764)) - Stopped installing pip into Poetry's virtual environment. ([#1761](https://github.com/heroku/heroku-buildpack-python/pull/1761)) diff --git a/lib/poetry.sh b/lib/poetry.sh index 6b4888eea..a453356a1 100644 --- a/lib/poetry.sh +++ b/lib/poetry.sh @@ -101,11 +101,15 @@ function poetry::install_poetry() { export PATH="${poetry_bin_dir}:${PATH}" # Force Poetry to manage the system Python site-packages instead of using venvs. export POETRY_VIRTUALENVS_CREATE="false" + # Force Poetry to use our Python rather than scanning PATH (which might pick system Python). + # Though this currently doesn't work as documented: https://github.com/python-poetry/poetry/issues/10226 + export POETRY_VIRTUALENVS_USE_POETRY_PYTHON="true" # Set the same env vars in the environment used by later buildpacks. cat >>"${export_file}" <<-EOF export PATH="${poetry_bin_dir}:\${PATH}" export POETRY_VIRTUALENVS_CREATE="false" + export POETRY_VIRTUALENVS_USE_POETRY_PYTHON="true" EOF } diff --git a/requirements/poetry.txt b/requirements/poetry.txt index d1c8013e2..f71727796 100644 --- a/requirements/poetry.txt +++ b/requirements/poetry.txt @@ -1 +1 @@ -poetry==2.0.1 +poetry==2.1.1 diff --git a/spec/fixtures/poetry_basic/poetry.lock b/spec/fixtures/poetry_basic/poetry.lock index 7fe5d0794..166f8fea8 100644 --- a/spec/fixtures/poetry_basic/poetry.lock +++ b/spec/fixtures/poetry_basic/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.8.4 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.1.1 and should not be changed by hand. [[package]] name = "colorama" @@ -6,6 +6,8 @@ version = "0.4.6" description = "Cross-platform colored terminal text." optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +groups = ["dev"] +markers = "sys_platform == \"win32\"" files = [ {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, @@ -17,6 +19,7 @@ version = "2.0.0" description = "brain-dead simple config-ini parsing" optional = false python-versions = ">=3.7" +groups = ["dev"] files = [ {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, @@ -28,6 +31,7 @@ version = "24.2" description = "Core utilities for Python packages" optional = false python-versions = ">=3.8" +groups = ["dev"] files = [ {file = "packaging-24.2-py3-none-any.whl", hash = "sha256:09abb1bccd265c01f4a3aa3f7a7db064b36514d2cba19a2f694fe6150451a759"}, {file = "packaging-24.2.tar.gz", hash = "sha256:c228a6dc5e932d346bc5739379109d49e8853dd8223571c7c5b55260edc0b97f"}, @@ -39,6 +43,7 @@ version = "1.5.0" description = "plugin and hook calling mechanisms for python" optional = false python-versions = ">=3.8" +groups = ["dev"] files = [ {file = "pluggy-1.5.0-py3-none-any.whl", hash = "sha256:44e1ad92c8ca002de6377e165f3e0f1be63266ab4d554740532335b9d75ea669"}, {file = "pluggy-1.5.0.tar.gz", hash = "sha256:2cffa88e94fdc978c4c574f15f9e59b7f4201d439195c3715ca9e2486f1d0cf1"}, @@ -54,6 +59,7 @@ version = "8.3.4" description = "pytest: simple powerful testing with Python" optional = false python-versions = ">=3.8" +groups = ["dev"] files = [ {file = "pytest-8.3.4-py3-none-any.whl", hash = "sha256:50e16d954148559c9a74109af1eaf0c945ba2d8f30f0a3d3335edde19788b6f6"}, {file = "pytest-8.3.4.tar.gz", hash = "sha256:965370d062bce11e73868e0335abac31b4d3de0e82f4007408d242b4f8610761"}, @@ -74,12 +80,13 @@ version = "4.12.2" description = "Backported and Experimental Type Hints for Python 3.8+" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "typing_extensions-4.12.2-py3-none-any.whl", hash = "sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d"}, {file = "typing_extensions-4.12.2.tar.gz", hash = "sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8"}, ] [metadata] -lock-version = "2.0" -python-versions = "^3.13" -content-hash = "56ec6342f4a39b402d2b65bd08cfb3af4136a02bb58674df49eac8d317376a7f" +lock-version = "2.1" +python-versions = ">=3.13" +content-hash = "8e82be2391208aa1b8599893f6abec30c0aa5702a846842fd442f50de45481e2" diff --git a/spec/fixtures/poetry_basic/pyproject.toml b/spec/fixtures/poetry_basic/pyproject.toml index 9fc0e8356..85bd48b9d 100644 --- a/spec/fixtures/poetry_basic/pyproject.toml +++ b/spec/fixtures/poetry_basic/pyproject.toml @@ -1,10 +1,14 @@ +[project] +name = "poetry-basic" +version = "0.0.0" +requires-python = ">=3.13" +dependencies = [ + "typing-extensions" +] + [tool.poetry] package-mode = false -[tool.poetry.dependencies] -python = "^3.13" -typing-extensions = "*" - # This group shouldn't be installed due to us passing `--only main`. -[tool.poetry.group.test.dependencies] +[tool.poetry.group.dev.dependencies] pytest = "*" diff --git a/spec/hatchet/ci_spec.rb b/spec/hatchet/ci_spec.rb index 5ff6041be..f11e37a23 100644 --- a/spec/hatchet/ci_spec.rb +++ b/spec/hatchet/ci_spec.rb @@ -189,6 +189,7 @@ PIP_NO_PYTHON_VERSION_WARNING=1 PKG_CONFIG_PATH=/app/.heroku/python/lib/pkg-config POETRY_VIRTUALENVS_CREATE=false + POETRY_VIRTUALENVS_USE_POETRY_PYTHON=true PYTHONUNBUFFERED=1 -----> Inline app detected LANG=en_US.UTF-8 @@ -196,6 +197,7 @@ LIBRARY_PATH=/app/.heroku/python/lib PATH=/app/.heroku/python/bin:/tmp/cache.+/.heroku/python-poetry/bin:/usr/local/bin:/usr/bin:/bin:/app/.sprettur/bin/ POETRY_VIRTUALENVS_CREATE=false + POETRY_VIRTUALENVS_USE_POETRY_PYTHON=true PYTHONHASHSEED=random PYTHONHOME=/app/.heroku/python PYTHONPATH=/app diff --git a/spec/hatchet/poetry_spec.rb b/spec/hatchet/poetry_spec.rb index 874715f00..9d8d6c4ac 100644 --- a/spec/hatchet/poetry_spec.rb +++ b/spec/hatchet/poetry_spec.rb @@ -26,6 +26,7 @@ remote: LIBRARY_PATH=/app/.heroku/python/lib remote: PATH=/app/.heroku/python/bin:/tmp/codon/tmp/cache/.heroku/python-poetry/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin remote: POETRY_VIRTUALENVS_CREATE=false + remote: POETRY_VIRTUALENVS_USE_POETRY_PYTHON=true remote: PYTHONHASHSEED=random remote: PYTHONHOME=/app/.heroku/python remote: PYTHONPATH=/app @@ -62,7 +63,7 @@ end context 'when the Poetry and Python versions have changed since the last build' do - let(:buildpacks) { ['https://github.com/heroku/heroku-buildpack-python#v274'] } + let(:buildpacks) { ['https://github.com/heroku/heroku-buildpack-python#v275'] } let(:app) { Hatchet::Runner.new('spec/fixtures/poetry_basic', buildpacks:) } it 'clears the cache before installing' do @@ -75,7 +76,7 @@ remote: -----> Using Python 3.13 specified in .python-version remote: -----> Discarding cache since: remote: - The Python version has changed from 3.13.1 to #{LATEST_PYTHON_3_13} - remote: - The Poetry version has changed from 1.8.5 to #{POETRY_VERSION} + remote: - The Poetry version has changed from 2.0.1 to #{POETRY_VERSION} remote: -----> Installing Python #{LATEST_PYTHON_3_13} remote: -----> Installing Poetry #{POETRY_VERSION} remote: -----> Installing dependencies using 'poetry sync --only main' From a85e52c7f1cae4262d51f0769e6abc7139df6082 Mon Sep 17 00:00:00 2001 From: "heroku-linguist[bot]" <136119646+heroku-linguist[bot]@users.noreply.github.com> Date: Wed, 26 Feb 2025 08:18:29 +0000 Subject: [PATCH 18/26] Prepare release v279 (#1766) Co-authored-by: heroku-linguist[bot] <136119646+heroku-linguist[bot]@users.noreply.github.com> --- CHANGELOG.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 29c88e5c7..108387e76 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,9 @@ ## [Unreleased] + +## [v279] - 2025-02-26 + - Updated Poetry from 2.0.1 to 2.1.1. ([#1758](https://github.com/heroku/heroku-buildpack-python/pull/1758)) - Stopped filtering out pip's `Requirement already satisfied:` log lines when installing dependencies. ([#1765](https://github.com/heroku/heroku-buildpack-python/pull/1765)) - Improved the error messages shown if installing pip/Poetry/Pipenv fails. ([#1764](https://github.com/heroku/heroku-buildpack-python/pull/1764)) @@ -1177,7 +1180,8 @@ Default Python is now latest 2.7.10. Updated pip and Distribute. - Setuptools updated to v16.0 - pip updated to v7.0.1 -[unreleased]: https://github.com/heroku/heroku-buildpack-python/compare/v278...main +[unreleased]: https://github.com/heroku/heroku-buildpack-python/compare/v279...main +[v279]: https://github.com/heroku/heroku-buildpack-python/compare/v278...v279 [v278]: https://github.com/heroku/heroku-buildpack-python/compare/v277...v278 [v277]: https://github.com/heroku/heroku-buildpack-python/compare/v276...v277 [v276]: https://github.com/heroku/heroku-buildpack-python/compare/v275...v276 From 7c5d82043b6926b8d675f1e9aa04c29edeb1f1ab Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 13 Mar 2025 10:27:04 +0000 Subject: [PATCH 19/26] Bump the ruby-dependencies group across 1 directory with 2 updates (#1770) * Bump the ruby-dependencies group across 1 directory with 2 updates Bumps the ruby-dependencies group with 2 updates in the / directory: [heroku_hatchet](https://github.com/heroku/hatchet) and [rubocop](https://github.com/rubocop/rubocop). Updates `heroku_hatchet` from 8.0.4 to 8.0.5 - [Changelog](https://github.com/heroku/hatchet/blob/main/CHANGELOG.md) - [Commits](https://github.com/heroku/hatchet/compare/v8.0.4...v8.0.5) Updates `rubocop` from 1.72.2 to 1.74.0 - [Release notes](https://github.com/rubocop/rubocop/releases) - [Changelog](https://github.com/rubocop/rubocop/blob/master/CHANGELOG.md) - [Commits](https://github.com/rubocop/rubocop/compare/v1.72.2...v1.74.0) --- updated-dependencies: - dependency-name: heroku_hatchet dependency-type: direct:development update-type: version-update:semver-patch dependency-group: ruby-dependencies - dependency-name: rubocop dependency-type: direct:development update-type: version-update:semver-minor dependency-group: ruby-dependencies ... Signed-off-by: dependabot[bot] * Remove logger workaround + refresh lockfile --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Ed Morley <501702+edmorley@users.noreply.github.com> --- Gemfile | 2 -- Gemfile.lock | 22 +++++++++++----------- 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/Gemfile b/Gemfile index 215de3b19..2abf2b7cb 100644 --- a/Gemfile +++ b/Gemfile @@ -6,8 +6,6 @@ ruby '>= 3.2', '< 3.5' group :test, :development do gem 'heroku_hatchet' - # Work around https://github.com/excon/excon/issues/860 - gem 'logger' gem 'parallel_split_test' gem 'rspec-core' gem 'rspec-expectations' diff --git a/Gemfile.lock b/Gemfile.lock index efa4e8d6b..22b12ea29 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -3,9 +3,10 @@ GEM specs: ast (2.4.2) base64 (0.2.0) - diff-lcs (1.5.1) + diff-lcs (1.6.0) erubis (2.7.0) - excon (0.112.0) + excon (1.2.5) + logger heroics (0.1.3) base64 erubis (~> 2.0) @@ -13,13 +14,13 @@ GEM moneta multi_json (>= 1.9.2) webrick - heroku_hatchet (8.0.4) - excon (~> 0) + heroku_hatchet (8.0.5) + excon (< 2) platform-api (~> 3) rrrretry (~> 1) thor (~> 1) threaded (~> 0) - json (2.10.1) + json (2.10.2) language_server-protocol (3.17.0.4) lint_roller (1.1.0) logger (1.6.6) @@ -32,7 +33,7 @@ GEM parser (3.3.7.1) ast (~> 2.4.1) racc - platform-api (3.7.0) + platform-api (3.8.0) heroics (~> 0.1.1) moneta (~> 1.0.0) rate_throttle_client (~> 0.1.0) @@ -49,7 +50,7 @@ GEM rspec-retry (0.6.2) rspec-core (> 3.3) rspec-support (3.13.2) - rubocop (1.72.2) + rubocop (1.74.0) json (~> 2.3) language_server-protocol (~> 3.17.0.2) lint_roller (~> 1.1.0) @@ -60,7 +61,7 @@ GEM rubocop-ast (>= 1.38.0, < 2.0) ruby-progressbar (~> 1.7) unicode-display_width (>= 2.4.0, < 4.0) - rubocop-ast (1.38.0) + rubocop-ast (1.38.1) parser (>= 3.3.1.0) rubocop-rspec (3.5.0) lint_roller (~> 1.1) @@ -78,7 +79,6 @@ PLATFORMS DEPENDENCIES heroku_hatchet - logger parallel_split_test rspec-core rspec-expectations @@ -87,7 +87,7 @@ DEPENDENCIES rubocop-rspec RUBY VERSION - ruby 3.4.1p0 + ruby 3.4.2p28 BUNDLED WITH - 2.6.2 + 2.6.3 From 49b1639b7be8acc9ca6d171db2d77b0d7659028d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 2 Apr 2025 13:59:14 +0100 Subject: [PATCH 20/26] Bump rubocop from 1.74.0 to 1.75.1 in the ruby-dependencies group (#1771) Bumps the ruby-dependencies group with 1 update: [rubocop](https://github.com/rubocop/rubocop). Updates `rubocop` from 1.74.0 to 1.75.1 - [Release notes](https://github.com/rubocop/rubocop/releases) - [Changelog](https://github.com/rubocop/rubocop/blob/master/CHANGELOG.md) - [Commits](https://github.com/rubocop/rubocop/compare/v1.74.0...v1.75.1) --- updated-dependencies: - dependency-name: rubocop dependency-type: direct:development update-type: version-update:semver-minor dependency-group: ruby-dependencies ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Gemfile.lock | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 22b12ea29..d633d680f 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ GEM remote: https://rubygems.org/ specs: - ast (2.4.2) + ast (2.4.3) base64 (0.2.0) diff-lcs (1.6.0) erubis (2.7.0) @@ -30,13 +30,14 @@ GEM parallel_split_test (0.10.0) parallel (>= 0.5.13) rspec-core (>= 3.9.0) - parser (3.3.7.1) + parser (3.3.7.4) ast (~> 2.4.1) racc platform-api (3.8.0) heroics (~> 0.1.1) moneta (~> 1.0.0) rate_throttle_client (~> 0.1.0) + prism (1.4.0) racc (1.8.1) rainbow (3.1.1) rate_throttle_client (0.1.2) @@ -50,7 +51,7 @@ GEM rspec-retry (0.6.2) rspec-core (> 3.3) rspec-support (3.13.2) - rubocop (1.74.0) + rubocop (1.75.1) json (~> 2.3) language_server-protocol (~> 3.17.0.2) lint_roller (~> 1.1.0) @@ -58,11 +59,12 @@ GEM parser (>= 3.3.0.2) rainbow (>= 2.2.2, < 4.0) regexp_parser (>= 2.9.3, < 3.0) - rubocop-ast (>= 1.38.0, < 2.0) + rubocop-ast (>= 1.43.0, < 2.0) ruby-progressbar (~> 1.7) unicode-display_width (>= 2.4.0, < 4.0) - rubocop-ast (1.38.1) - parser (>= 3.3.1.0) + rubocop-ast (1.43.0) + parser (>= 3.3.7.2) + prism (~> 1.4) rubocop-rspec (3.5.0) lint_roller (~> 1.1) rubocop (~> 1.72, >= 1.72.1) From 41dcf88044d20ff5a7dd68955d740c925ddb8dc9 Mon Sep 17 00:00:00 2001 From: Pablo Temporini Date: Thu, 3 Apr 2025 11:05:11 -0300 Subject: [PATCH 21/26] Update repo metadata (#1773) Signed-off-by: Pablo Temporini --- .github/CODEOWNERS | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 52082f8a2..33be3e324 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,4 +1,6 @@ # Default to requesting pull request reviews from the Heroku Languages team. +#ECCN:Open Source +#GUSINFO:Languages,Heroku Python Platform * @heroku/languages # However, request review from the language owner instead for files that are updated From 51ec7f4b810b7af1aef86cedc884a0e281d408f1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 8 Apr 2025 09:37:41 +0000 Subject: [PATCH 22/26] Bump pip from 24.3.1 to 25.0.1 (#1759) * Bump pip from 24.3.1 to 25.0.1 Bumps [pip](https://github.com/pypa/pip) from 24.3.1 to 25.0.1. - [Changelog](https://github.com/pypa/pip/blob/main/NEWS.rst) - [Commits](https://github.com/pypa/pip/compare/24.3.1...25.0.1) --- updated-dependencies: - dependency-name: pip dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] * Stop setting `PIP_NO_PYTHON_VERSION_WARNING=1` Since that option is a no-op, and now causes a deprecation warning: ``` DEPRECATION: --no-python-version-warning is deprecated. pip 25.1 will enforce this behaviour change. ``` * Add changelog entry --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Ed Morley <501702+edmorley@users.noreply.github.com> --- CHANGELOG.md | 1 + bin/compile | 4 ---- lib/pip.sh | 1 + requirements/pip.txt | 2 +- spec/hatchet/ci_spec.rb | 3 --- spec/hatchet/hooks_spec.rb | 2 -- 6 files changed, 3 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 108387e76..f5f4b496d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## [Unreleased] +- Updated pip from 24.3.1 to 25.0.1. ([#1759](https://github.com/heroku/heroku-buildpack-python/pull/1759)) ## [v279] - 2025-02-26 diff --git a/bin/compile b/bin/compile index cda662380..b75752b91 100755 --- a/bin/compile +++ b/bin/compile @@ -94,10 +94,6 @@ export LIBRARY_PATH="/app/.heroku/python/lib${LIBRARY_PATH:+:${LIBRARY_PATH}}" export LD_LIBRARY_PATH="/app/.heroku/python/lib${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}" export PKG_CONFIG_PATH="/app/.heroku/python/lib/pkg-config${PKG_CONFIG_PATH:+:${PKG_CONFIG_PATH}}" -# Global pip options (https://pip.pypa.io/en/stable/user_guide/#environment-variables). -# Disable pip's warnings about EOL Python since we show our own. -export PIP_NO_PYTHON_VERSION_WARNING=1 - cd "$BUILD_DIR" # Runs a `bin/pre_compile` script if found in the app source, allowing build customisation. diff --git a/lib/pip.sh b/lib/pip.sh index e56e6d536..ca95abfd6 100644 --- a/lib/pip.sh +++ b/lib/pip.sh @@ -106,6 +106,7 @@ function pip::install_dependencies() { # We only display the most relevant command args here, to improve the signal to noise ratio. output::step "Installing dependencies using '${pip_install_command[*]}'" + # TODO: Remove --disable-pip-version-check in favour of exporting PIP_DISABLE_PIP_VERSION_CHECK. # The sed usage is to reduce the verbosity of output lines like: # "Requirement already satisfied: typing-extensions==4.12.2 in /app/.heroku/python/lib/python3.13/site-packages (from -r requirements.txt (line 5)) (4.12.2)" # shellcheck disable=SC2310 # This function is invoked in an 'if' condition so set -e will be disabled. diff --git a/requirements/pip.txt b/requirements/pip.txt index 662f25f21..059862d50 100644 --- a/requirements/pip.txt +++ b/requirements/pip.txt @@ -1 +1 @@ -pip==24.3.1 +pip==25.0.1 diff --git a/spec/hatchet/ci_spec.rb b/spec/hatchet/ci_spec.rb index f11e37a23..50d722c51 100644 --- a/spec/hatchet/ci_spec.rb +++ b/spec/hatchet/ci_spec.rb @@ -30,7 +30,6 @@ LD_LIBRARY_PATH=/app/.heroku/python/lib LIBRARY_PATH=/app/.heroku/python/lib PATH=/app/.heroku/python/bin::/usr/local/bin:/usr/local/bin:/usr/bin:/bin:/app/.sprettur/bin/ - PIP_NO_PYTHON_VERSION_WARNING=1 PKG_CONFIG_PATH=/app/.heroku/python/lib/pkg-config PYTHONUNBUFFERED=1 -----> Inline app detected @@ -105,7 +104,6 @@ LD_LIBRARY_PATH=/app/.heroku/python/lib LIBRARY_PATH=/app/.heroku/python/lib PATH=/app/.heroku/python/bin::/usr/local/bin:/usr/local/bin:/usr/bin:/bin:/app/.sprettur/bin/ - PIP_NO_PYTHON_VERSION_WARNING=1 PKG_CONFIG_PATH=/app/.heroku/python/lib/pkg-config PYTHONUNBUFFERED=1 -----> Inline app detected @@ -186,7 +184,6 @@ LD_LIBRARY_PATH=/app/.heroku/python/lib LIBRARY_PATH=/app/.heroku/python/lib PATH=/tmp/cache.+/.heroku/python-poetry/bin:/app/.heroku/python/bin::/usr/local/bin:/usr/local/bin:/usr/bin:/bin:/app/.sprettur/bin/ - PIP_NO_PYTHON_VERSION_WARNING=1 PKG_CONFIG_PATH=/app/.heroku/python/lib/pkg-config POETRY_VIRTUALENVS_CREATE=false POETRY_VIRTUALENVS_USE_POETRY_PYTHON=true diff --git a/spec/hatchet/hooks_spec.rb b/spec/hatchet/hooks_spec.rb index 1f6af134e..eb20a7e83 100644 --- a/spec/hatchet/hooks_spec.rb +++ b/spec/hatchet/hooks_spec.rb @@ -24,7 +24,6 @@ remote: LD_LIBRARY_PATH=/app/.heroku/python/lib remote: LIBRARY_PATH=/app/.heroku/python/lib remote: PATH=/app/.heroku/python/bin::/usr/local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin - remote: PIP_NO_PYTHON_VERSION_WARNING=1 remote: PKG_CONFIG_PATH=/app/.heroku/python/lib/pkg-config remote: PWD=/tmp/build_ remote: PYTHONUNBUFFERED=1 @@ -48,7 +47,6 @@ remote: LD_LIBRARY_PATH=/app/.heroku/python/lib remote: LIBRARY_PATH=/app/.heroku/python/lib remote: PATH=/app/.heroku/python/bin::/usr/local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin - remote: PIP_NO_PYTHON_VERSION_WARNING=1 remote: PKG_CONFIG_PATH=/app/.heroku/python/lib/pkg-config remote: PWD=/tmp/build_ remote: PYTHONUNBUFFERED=1 From 627847a165c70df114ac733f25a2bf33acf44473 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 8 Apr 2025 11:01:45 +0100 Subject: [PATCH 23/26] Bump poetry from 2.1.1 to 2.1.2 (#1772) * Bump poetry from 2.1.1 to 2.1.2 Bumps [poetry](https://github.com/python-poetry/poetry) from 2.1.1 to 2.1.2. - [Release notes](https://github.com/python-poetry/poetry/releases) - [Changelog](https://github.com/python-poetry/poetry/blob/main/CHANGELOG.md) - [Commits](https://github.com/python-poetry/poetry/compare/2.1.1...2.1.2) --- updated-dependencies: - dependency-name: poetry dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] * Add changelog entry --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Ed Morley <501702+edmorley@users.noreply.github.com> --- CHANGELOG.md | 1 + requirements/poetry.txt | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f5f4b496d..c7e341e82 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ## [Unreleased] - Updated pip from 24.3.1 to 25.0.1. ([#1759](https://github.com/heroku/heroku-buildpack-python/pull/1759)) +- Updated Poetry from 2.1.1 to 2.1.2. ([#1772](https://github.com/heroku/heroku-buildpack-python/pull/1772)) ## [v279] - 2025-02-26 diff --git a/requirements/poetry.txt b/requirements/poetry.txt index f71727796..7fc861c0f 100644 --- a/requirements/poetry.txt +++ b/requirements/poetry.txt @@ -1 +1 @@ -poetry==2.1.1 +poetry==2.1.2 From 76a8152a700f23bf6bda713bc6f6b2be4658819c Mon Sep 17 00:00:00 2001 From: "heroku-linguist[bot]" <136119646+heroku-linguist[bot]@users.noreply.github.com> Date: Tue, 8 Apr 2025 10:05:55 +0000 Subject: [PATCH 24/26] Prepare release v280 (#1774) Co-authored-by: heroku-linguist[bot] <136119646+heroku-linguist[bot]@users.noreply.github.com> --- CHANGELOG.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c7e341e82..f35671877 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,9 @@ ## [Unreleased] + +## [v280] - 2025-04-08 + - Updated pip from 24.3.1 to 25.0.1. ([#1759](https://github.com/heroku/heroku-buildpack-python/pull/1759)) - Updated Poetry from 2.1.1 to 2.1.2. ([#1772](https://github.com/heroku/heroku-buildpack-python/pull/1772)) @@ -1182,7 +1185,8 @@ Default Python is now latest 2.7.10. Updated pip and Distribute. - Setuptools updated to v16.0 - pip updated to v7.0.1 -[unreleased]: https://github.com/heroku/heroku-buildpack-python/compare/v279...main +[unreleased]: https://github.com/heroku/heroku-buildpack-python/compare/v280...main +[v280]: https://github.com/heroku/heroku-buildpack-python/compare/v279...v280 [v279]: https://github.com/heroku/heroku-buildpack-python/compare/v278...v279 [v278]: https://github.com/heroku/heroku-buildpack-python/compare/v277...v278 [v277]: https://github.com/heroku/heroku-buildpack-python/compare/v276...v277 From 3ee5020fb061b3375785db12ddae60c1a187c613 Mon Sep 17 00:00:00 2001 From: Ed Morley <501702+edmorley@users.noreply.github.com> Date: Wed, 9 Apr 2025 00:12:18 +0100 Subject: [PATCH 25/26] Add support for Python 3.13.3, 3.12.10, 3.11.12, 3.10.17 and 3.9.22 (#1775) Release announcement: https://blog.python.org/2025/04/python-3140a7-3133-31210-31112-31017.html Changelogs: https://docs.python.org/release/3.13.3/whatsnew/changelog.html#python-3-13-3-final https://docs.python.org/release/3.12.10/whatsnew/changelog.html#python-3-12-10-final https://docs.python.org/release/3.11.12/whatsnew/changelog.html#python-3-11-12-final https://docs.python.org/release/3.10.17/whatsnew/changelog.html#python-3-10-17-final https://docs.python.org/release/3.9.22/whatsnew/changelog.html#python-3-9-22-final Binary builds: https://github.com/heroku/heroku-buildpack-python/actions/runs/14341087204 https://github.com/heroku/heroku-buildpack-python/actions/runs/14341085641 https://github.com/heroku/heroku-buildpack-python/actions/runs/14341083031 https://github.com/heroku/heroku-buildpack-python/actions/runs/14341080059 https://github.com/heroku/heroku-buildpack-python/actions/runs/14344846015 GUS-W-17604324. GUS-W-17604386. GUS-W-18229136. --- CHANGELOG.md | 5 +++++ lib/python_version.sh | 10 +++++----- spec/spec_helper.rb | 10 +++++----- 3 files changed, 15 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f35671877..946a97a9c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,11 @@ ## [Unreleased] +- The Python 3.13 version alias now resolves to Python 3.13.3. ([#1775](https://github.com/heroku/heroku-buildpack-python/pull/1775)) +- The Python 3.12 version alias now resolves to Python 3.12.10. ([#1775](https://github.com/heroku/heroku-buildpack-python/pull/1775)) +- The Python 3.11 version alias now resolves to Python 3.11.12. ([#1775](https://github.com/heroku/heroku-buildpack-python/pull/1775)) +- The Python 3.10 version alias now resolves to Python 3.10.17. ([#1775](https://github.com/heroku/heroku-buildpack-python/pull/1775)) +- The Python 3.9 version alias now resolves to Python 3.9.22. ([#1775](https://github.com/heroku/heroku-buildpack-python/pull/1775)) ## [v280] - 2025-04-08 diff --git a/lib/python_version.sh b/lib/python_version.sh index 4405b7650..02735b3b5 100644 --- a/lib/python_version.sh +++ b/lib/python_version.sh @@ -4,11 +4,11 @@ # however, it helps Shellcheck realise the options under which these functions will run. set -euo pipefail -LATEST_PYTHON_3_9="3.9.21" -LATEST_PYTHON_3_10="3.10.16" -LATEST_PYTHON_3_11="3.11.11" -LATEST_PYTHON_3_12="3.12.9" -LATEST_PYTHON_3_13="3.13.2" +LATEST_PYTHON_3_9="3.9.22" +LATEST_PYTHON_3_10="3.10.17" +LATEST_PYTHON_3_11="3.11.12" +LATEST_PYTHON_3_12="3.12.10" +LATEST_PYTHON_3_13="3.13.3" OLDEST_SUPPORTED_PYTHON_3_MINOR_VERSION=9 NEWEST_SUPPORTED_PYTHON_3_MINOR_VERSION=13 diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index b8e49eb22..2f561389c 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -9,11 +9,11 @@ FIXTURE_DIR = Pathname.new(__FILE__).parent.join('fixtures') -LATEST_PYTHON_3_9 = '3.9.21' -LATEST_PYTHON_3_10 = '3.10.16' -LATEST_PYTHON_3_11 = '3.11.11' -LATEST_PYTHON_3_12 = '3.12.9' -LATEST_PYTHON_3_13 = '3.13.2' +LATEST_PYTHON_3_9 = '3.9.22' +LATEST_PYTHON_3_10 = '3.10.17' +LATEST_PYTHON_3_11 = '3.11.12' +LATEST_PYTHON_3_12 = '3.12.10' +LATEST_PYTHON_3_13 = '3.13.3' DEFAULT_PYTHON_FULL_VERSION = LATEST_PYTHON_3_13 DEFAULT_PYTHON_MAJOR_VERSION = DEFAULT_PYTHON_FULL_VERSION.gsub(/\.\d+$/, '') From b2966193afbc86d88b0ee694ffbb33cba74fc23c Mon Sep 17 00:00:00 2001 From: "heroku-linguist[bot]" <136119646+heroku-linguist[bot]@users.noreply.github.com> Date: Tue, 8 Apr 2025 23:20:42 +0000 Subject: [PATCH 26/26] Prepare release v281 (#1776) Co-authored-by: heroku-linguist[bot] <136119646+heroku-linguist[bot]@users.noreply.github.com> --- CHANGELOG.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 946a97a9c..d45887f0e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,9 @@ ## [Unreleased] + +## [v281] - 2025-04-08 + - The Python 3.13 version alias now resolves to Python 3.13.3. ([#1775](https://github.com/heroku/heroku-buildpack-python/pull/1775)) - The Python 3.12 version alias now resolves to Python 3.12.10. ([#1775](https://github.com/heroku/heroku-buildpack-python/pull/1775)) - The Python 3.11 version alias now resolves to Python 3.11.12. ([#1775](https://github.com/heroku/heroku-buildpack-python/pull/1775)) @@ -1190,7 +1193,8 @@ Default Python is now latest 2.7.10. Updated pip and Distribute. - Setuptools updated to v16.0 - pip updated to v7.0.1 -[unreleased]: https://github.com/heroku/heroku-buildpack-python/compare/v280...main +[unreleased]: https://github.com/heroku/heroku-buildpack-python/compare/v281...main +[v281]: https://github.com/heroku/heroku-buildpack-python/compare/v280...v281 [v280]: https://github.com/heroku/heroku-buildpack-python/compare/v279...v280 [v279]: https://github.com/heroku/heroku-buildpack-python/compare/v278...v279 [v278]: https://github.com/heroku/heroku-buildpack-python/compare/v277...v278