feat(airflow/wip): use a newer uv version, and install the correct python version using it

WIP: Still need to fix the build layers, but this appears to work fine (tested the oldest and newest versions via the getting_started guide)

Author: NickLarsenNZ

airflow/Dockerfile

@@ -23,10 +23,10 @@ WORKDIR /tmp/opa-auth-manager
 
 RUN <<EOF
 microdnf update
-microdnf install python${PYTHON_VERSION}-pip
+microdnf install python3-pip
 microdnf clean all
 
-pip${PYTHON_VERSION} install --no-cache-dir uv==${UV_VERSION}
+pip install --no-cache-dir uv==${UV_VERSION}
 
 # This folder is required by the tests to set up an sqlite database
 mkdir /root/airflow
@@ -62,8 +62,7 @@ ARG AIRFLOW_EXTRAS_EXTERNAL_SERVICES=""
 ARG AIRFLOW_EXTRAS_LOCALLY_INSTALLED_SOFTWARE=""
 ARG AIRFLOW_EXTRAS_OTHER=""
 
-RUN microdnf module enable -y nodejs:${NODEJS_VERSION} && \
-    microdnf update && \
+RUN microdnf update && \
     microdnf install \
         cyrus-sasl-devel \
         # Needed for kerberos
@@ -78,10 +77,10 @@ RUN microdnf module enable -y nodejs:${NODEJS_VERSION} && \
         libpq-devel \
         openldap-devel \
         openssl-devel \
-        python${PYTHON_VERSION} \
-        python${PYTHON_VERSION}-devel \
-        python${PYTHON_VERSION}-pip \
-        python${PYTHON_VERSION}-wheel \
+        python3 \
+        python3-devel \
+        python3-pip \
+        # python${PYTHON_VERSION}-wheel \
         # The airflow odbc provider can compile without the development files (headers and libraries) (see https://github.com/stackabletech/docker-images/pull/683)
         unixODBC \
         # Needed for Airflow UI assets
@@ -111,14 +110,16 @@ AIRFLOW_EXTRAS="$AIRFLOW_EXTRAS_CORE,$AIRFLOW_EXTRAS_META,$AIRFLOW_EXTRAS_PROVID
 # Removing duplicates
 AIRFLOW_EXTRAS=$(echo "$AIRFLOW_EXTRAS" | tr ',' '\n' | awk 'NF > 0 {if (!seen[$0]++) print $0}' | tr '\n' ',' | sed 's/,$//')
 
-python${PYTHON_VERSION} -m venv --system-site-packages /stackable/app
-
-source /stackable/app/bin/activate
-
+# TODO: Use uv ${UV_VERSION} from Nexus instead of pip
 # Upgrade pip to the latest version
-# Also install uv to get support for build constraints
+# Also install uv to get support for build constraints and venv for a given python version
 pip install --no-cache-dir --upgrade pip
 pip install --no-cache-dir uv==${UV_VERSION}
+
+uv venv --python "${PYTHON_VERSION}" --system-site-packages /stackable/app
+source /stackable/app/bin/activate
+
+# Another project manager
 uv tool install hatch
 
 cd "$(/stackable/patchable --images-repo-root=src checkout airflow ${PRODUCT_VERSION})"
@@ -200,6 +201,7 @@ ARG TINI_VERSION
 ARG TARGETARCH
 ARG SHARED_STATSD_EXPORTER_VERSION
 ARG STACKABLE_USER_UID
+ARG UV_VERSION
 
 LABEL name="Apache Airflow" \
     maintainer="info@stackable.tech" \
@@ -225,6 +227,9 @@ COPY --from=gitsync-image --chown=${STACKABLE_USER_UID}:0 /git-sync ${HOME}/git-
 
 COPY airflow/licenses /licenses
 
+# TODO (@NickLarsenNZ): Get the image into our registry
+COPY --from=ghcr.io/astral-sh/uv:0.9.10 --chown=${STACKABLE_USER_UID}:0 /uv /uvx /bin/
+
 # Update image and install needed packages
 RUN <<EOF
 microdnf update
@@ -242,8 +247,6 @@ microdnf install \
   openldap-clients \
   openssh-clients \
   openssl-libs \
-  openssl-pkcs11 \
-  python${PYTHON_VERSION} \
   socat \
   unixODBC
 microdnf clean all
@@ -253,6 +256,15 @@ rm -rf /var/cache/yum
 # It is used as an init alternative in the entrypoint
 curl -o /usr/bin/tini "https://repo.stackable.tech/repository/packages/tini/tini-${TINI_VERSION}-${TARGETARCH}"
 
+# There is currently no way to tell `uv` where to write to, and it assumes `$HOME` for the XDG paths.
+export XDG_DATA_HOME=/usr/local/share
+export XDG_CACHE_HOME=/var/cache
+uv venv --python "${PYTHON_VERSION}" --system-site-packages /stackable/app
+chown -R "${STACKABLE_USER_UID}:0" /stackable/app/bin
+source /stackable/app/bin/activate
+# uv python install "${PYTHON_VERSION}" --no-cache -v
+# uv python pin "${PYTHON_VERSION}" --no-cache -v
+
 # fix missing permissions
 chmod a+x /entrypoint.sh
 chmod a+x /run-airflow.sh

airflow/boil-config.toml

@@ -9,10 +9,11 @@ git-sync-version = "v4.4.1"
 s3fs-version = "2024.9.0"
 cyclonedx-bom-version = "6.0.0"
 tini-version = "0.19.0"
-uv-version = "0.7.8"
+uv-version = "0.9.10"
 airflow-extras-other = "async,amazon,celery,cncf.kubernetes,docker,dask,elasticsearch,ftp,grpc,hashicorp,http,ldap,google,google_auth,microsoft.azure,odbc,pandas,postgres,redis,sendgrid,sftp,slack,ssh,statsd,virtualenv,trino"
 opa-auth-manager = "airflow-2"
 nodejs-version = "20"
+nvm-version = "0.40.3"
 
 [versions."2.10.5".local-images]
 "shared/statsd-exporter" = "0.28.0"
@@ -25,10 +26,11 @@ git-sync-version = "v4.4.1"
 s3fs-version = "2024.9.0"
 cyclonedx-bom-version = "6.0.0"
 tini-version = "0.19.0"
-uv-version = "0.7.8"
+uv-version = "0.9.10"
 airflow-extras-other = "async,amazon,celery,cncf.kubernetes,docker,dask,elasticsearch,ftp,grpc,hashicorp,http,ldap,google,google_auth,microsoft.azure,odbc,pandas,postgres,redis,sendgrid,sftp,slack,ssh,statsd,virtualenv,trino"
 opa-auth-manager = "airflow-2"
 nodejs-version = "20"
+nvm-version = "0.40.3"
 
 [versions."3.0.1".local-images]
 "shared/statsd-exporter" = "0.28.0"
@@ -41,10 +43,11 @@ git-sync-version = "v4.4.1"
 s3fs-version = "2024.9.0"
 cyclonedx-bom-version = "6.0.0"
 tini-version = "0.19.0"
-uv-version = "0.7.8"
+uv-version = "0.9.10"
 airflow-extras-other = "async,amazon,celery,cncf-kubernetes,docker,elasticsearch,fab,ftp,grpc,hashicorp,http,ldap,google,microsoft-azure,odbc,pandas,postgres,redis,sendgrid,sftp,slack,ssh,statsd,trino"
 opa-auth-manager = "airflow-3"
 nodejs-version = "20"
+nvm-version = "0.40.3"
 
 [versions."3.0.6".local-images]
 "shared/statsd-exporter" = "0.28.0"
@@ -57,26 +60,27 @@ git-sync-version = "v4.4.1"
 s3fs-version = "2024.9.0"
 cyclonedx-bom-version = "6.0.0"
 tini-version = "0.19.0"
-uv-version = "0.7.8"
+uv-version = "0.9.10"
 
 # Airflow extras are defined in separate lists to make them easier to check against the links below. The lists will be concatenated and duplicates removed in the dockerfile.
 # See https://airflow.apache.org/docs/apache-airflow/3.0.6/extra-packages-ref.html#core-airflow-extras
-airflow-extras-core="async,graphviz,kerberos,otel,sentry,standard,statsd"
+airflow-extras-core = "async,graphviz,kerberos,otel,sentry,standard,statsd"
 
 # See https://airflow.apache.org/docs/apache-airflow/3.0.6/extra-packages-ref.html#meta-airflow-package-extras
-airflow-extras-meta="aiobotocore,cloudpickle,github-enterprise,google-auth,graphviz,ldap,leveldb,pandas,polars,rabbitmq,s3fs,saml,uv"
+airflow-extras-meta = "aiobotocore,cloudpickle,github-enterprise,google-auth,graphviz,ldap,leveldb,pandas,polars,rabbitmq,s3fs,saml,uv"
 
 # See https://airflow.apache.org/docs/apache-airflow/3.0.6/extra-packages-ref.html#apache-software-extras
-airflow-extras-provider-apache="apache-beam,apache-cassandra,apache-drill,apache-druid,apache-flink,apache-hdfs,apache-hive,apache-iceberg,apache-impala,apache-kafka,apache-kylin,apache-livy,apache-pig,apache-pinot"
+airflow-extras-provider-apache = "apache-beam,apache-cassandra,apache-drill,apache-druid,apache-flink,apache-hdfs,apache-hive,apache-iceberg,apache-impala,apache-kafka,apache-kylin,apache-livy,apache-pig,apache-pinot"
 
 # See https://airflow.apache.org/docs/apache-airflow/3.0.6/extra-packages-ref.html#external-services-extras
-airflow-extras-external-services="airbyte,alibaba,apprise,amazon,asana,atlassian-jira,microsoft-azure,cloudant,cohere,databricks,datadog,dbt-cloud,dingding,discord,facebook,github,google,hashicorp,openai,opsgenie,pagerduty,pgvector,pinecone,qdrant,salesforce,sendgrid,segment,slack,snowflake,tableau,tabular,telegram,vertica,weaviate,yandex,ydb,zendesk"
+airflow-extras-external-services = "airbyte,alibaba,apprise,amazon,asana,atlassian-jira,microsoft-azure,cloudant,cohere,databricks,datadog,dbt-cloud,dingding,discord,facebook,github,google,hashicorp,openai,opsgenie,pagerduty,pgvector,pinecone,qdrant,salesforce,sendgrid,segment,slack,snowflake,tableau,tabular,telegram,vertica,weaviate,yandex,ydb,zendesk"
 
 # See https://airflow.apache.org/docs/apache-airflow/3.0.6/extra-packages-ref.html#locally-installed-software-extras
-airflow-extras-locally-installed-software="arangodb,celery,cncf-kubernetes,docker,edge3,elasticsearch,exasol,fab,git,github,influxdb,jenkins,mongo,microsoft-mssql,neo4j,odbc,openfaas,oracle,postgres,presto,redis,samba,singularity,teradata,trino"
+airflow-extras-locally-installed-software = "arangodb,celery,cncf-kubernetes,docker,edge3,elasticsearch,exasol,fab,git,github,influxdb,jenkins,mongo,microsoft-mssql,neo4j,odbc,openfaas,oracle,postgres,presto,redis,samba,singularity,teradata,trino"
 
 # See https://airflow.apache.org/docs/apache-airflow/3.0.6/extra-packages-ref.html#other-extras
-airflow-extras-other="common-compat,common-io,common-messaging,common-sql,ftp,grpc,http,imap,jdbc,microsoft-psrp,microsoft-winrm,openlineage,opensearch,papermill,sftp,smtp,sqlite,ssh"
+airflow-extras-other = "common-compat,common-io,common-messaging,common-sql,ftp,grpc,http,imap,jdbc,microsoft-psrp,microsoft-winrm,openlineage,opensearch,papermill,sftp,smtp,sqlite,ssh"
 
 opa-auth-manager = "airflow-3"
 nodejs-version = "20"
+nvm-version = "0.40.3"