Improve metrics for builds that fail

Adds a `failure_detail` field in addition to the existing `failure_reason`,
which contains additional context relevant to the failure where available.
This will make it easier to find trends in the most frequent user-caused
failure modes (eg invalid Python version specifier) so I can then adjust
error messages/docs/implementation to improve UX.

This context sometimes contain user input, so the values saved to the
metadata store now also have additional escaping and validation
performed before writing the value (in addition to the existing YAML
escaping performed in `bin/report`).

GUS-W-17800067.

Author: edmorley

CHANGELOG.md

@@ -2,6 +2,7 @@
 
 ## [Unreleased]
 
+- Improved buildpack metrics for builds that fail. ([#1746](https://github.com/heroku/heroku-buildpack-python/pull/1746))
 
 ## [v276] - 2025-02-05
 

bin/report

@@ -62,6 +62,7 @@ kv_pair_string() {
 STRING_FIELDS=(
 	cache_status
 	django_collectstatic
+	failure_detail
 	failure_reason
 	nltk_downloader
 	package_manager

bin/steps/python

@@ -23,6 +23,7 @@ if ! curl --output /dev/null --silent --head --fail --retry 3 --retry-connrefuse
 		https://devcenter.heroku.com/articles/python-support#supported-python-versions
 	EOF
 	meta_set "failure_reason" "python-version-not-found"
+	meta_set "failure_detail" "${python_full_version}"
 	exit 1
 fi
 
@@ -42,6 +43,7 @@ else
 			Please try again and to see if the error resolves itself.
 		EOF
 		meta_set "failure_reason" "python-download"
+		# TODO: Set failure_detail here once refactored.
 		exit 1
 	fi
 

lib/checks.sh

@@ -20,6 +20,7 @@ function checks::ensure_supported_stack() {
 				Upgrade to a newer stack to continue using this buildpack.
 			EOF
 			meta_set "failure_reason" "stack::eol"
+			meta_set "failure_detail" "${stack}"
 			exit 1
 			;;
 		*)
@@ -34,6 +35,7 @@ function checks::ensure_supported_stack() {
 				https://devcenter.heroku.com/articles/managing-buildpacks#classic-buildpacks-references
 			EOF
 			meta_set "failure_reason" "stack::unknown"
+			meta_set "failure_detail" "${stack}"
 			exit 1
 			;;
 	esac

lib/kvstore.sh

@@ -23,8 +23,20 @@ kv_set() {
 	# TODO: Stop ignoring an incorrect number of passed arguments.
 	if [[ $# -eq 3 ]]; then
 		local f="${1}"
+		local key="${2}"
+
+		# Truncate the value to an arbitrary 100 characters since it will sometimes contain user-provided
+		# inputs which may be unbounded in size. Ideally individual call sites will perform more aggressive
+		# truncation themselves based on the expected value size, however this is here as a fallback.
+		# (Honeycomb supports string fields up to 64KB in size, however, it's not worth filling up the
+		# metadata store or bloating the payload passed back to Vacuole/submitted to Honeycomb given the
+		# extra content in those cases is not normally useful.)
+		local value="${3:0:100}"
+		# Replace newlines since the data store file format requires that keys don't span multiple lines.
+		value="${value//$'\n'/ }"
+
 		if [[ -f "${f}" ]]; then
-			echo "${2}=${3}" >>"${f}"
+			echo "${key}=${value}" >>"${f}"
 		fi
 	fi
 }

lib/python_version.sh

@@ -104,7 +104,7 @@ function python_version::parse_runtime_txt() {
 			in the correct format.
 
 			The following file contents were found, which aren't valid:
-			${contents}
+			${contents:0:100}
 
 			However, the runtime.txt file is deprecated since it has
 			been replaced by the .python-version file. As such, we
@@ -125,6 +125,7 @@ function python_version::parse_runtime_txt() {
 			your app to receive Python security updates.
 		EOF
 		meta_set "failure_reason" "runtime-txt::invalid-version"
+		meta_set "failure_detail" "${contents:0:50}"
 		exit 1
 	fi
 }
@@ -174,6 +175,7 @@ function python_version::parse_python_version_file() {
 					your app to receive Python security updates.
 				EOF
 				meta_set "failure_reason" "python-version-file::invalid-version"
+				meta_set "failure_detail" "${line:0:50}"
 				exit 1
 			fi
 			;;
@@ -193,17 +195,19 @@ function python_version::parse_python_version_file() {
 				begin with a '#', otherwise it will be treated as a comment.
 			EOF
 			meta_set "failure_reason" "python-version-file::no-version"
+			meta_set "failure_detail" "${contents:0:50}"
 			exit 1
 			;;
 		*)
+			local first_five_version_lines=("${version_lines[@]:0:5}")
 			output::error <<-EOF
 				Error: Invalid Python version in .python-version.
 
 				Multiple versions were found in your .python-version file:
 
 				$(
 					IFS=$'\n'
-					echo "${version_lines[*]}"
+					echo "${first_five_version_lines[*]}"
 				)
 
 				Update the file so it contains only one Python version.
@@ -212,6 +216,10 @@ function python_version::parse_python_version_file() {
 				lines begin with a '#', so that they are ignored.
 			EOF
 			meta_set "failure_reason" "python-version-file::multiple-versions"
+			meta_set "failure_detail" "$(
+				IFS=,
+				echo "${first_five_version_lines[*]}"
+			)"
 			exit 1
 			;;
 	esac
@@ -233,17 +241,19 @@ function python_version::read_pipenv_python_version() {
 	fi
 
 	if ! version=$(jq --raw-output '._meta.requires.python_full_version // ._meta.requires.python_version' "${pipfile_lock_path}" 2>&1); then
+		local jq_error_message="${version}"
 		output::error <<-EOF
 			Error: Can't parse Pipfile.lock.
 
 			A Pipfile.lock file was found, however, it couldn't be parsed:
-			${version}
+			${jq_error_message}
 
 			This is likely due to it not being valid JSON.
 
 			Run 'pipenv lock' to regenerate/fix the lockfile.
 		EOF
 		meta_set "failure_reason" "pipfile-lock::invalid-json"
+		meta_set "failure_detail" "${jq_error_message:0:100}"
 		exit 1
 	fi
 
@@ -282,6 +292,7 @@ function python_version::read_pipenv_python_version() {
 			https://pipenv.pypa.io/en/stable/specifiers.html#specifying-versions-of-python
 		EOF
 		meta_set "failure_reason" "pipfile-lock::invalid-version"
+		meta_set "failure_detail" "${version:0:50}"
 		exit 1
 	fi
 }
@@ -341,6 +352,7 @@ function python_version::resolve_python_version() {
 			EOF
 		fi
 		meta_set "failure_reason" "python-version::eol"
+		meta_set "failure_detail" "${major}.${minor}"
 		exit 1
 	fi
 
@@ -388,6 +400,7 @@ function python_version::resolve_python_version() {
 			EOF
 		fi
 		meta_set "failure_reason" "python-version::unknown-major"
+		meta_set "failure_detail" "${major}.${minor}"
 		exit 1
 	fi
 

lib/utils.sh

@@ -48,10 +48,12 @@ function utils::bundled_pip_module_path() {
 }
 
 function utils::abort_internal_error() {
-	local message="${1}"
+	local message
+	message="${1} (line $(caller || true))"
 	output::error <<-EOF
-		Internal error: ${message} (line $(caller || true)).
+		Internal error: ${message}.
 	EOF
 	meta_set "failure_reason" "internal-error"
+	meta_set "failure_detail" "${message}"
 	exit 1
 }