add support for python 3.10

build/__pythonVersions.sh

@@ -1,8 +1,9 @@
 # This file was auto-generated from 'constants.yaml'. Changes may be overridden.
 
-PIP_VERSION='20.2.3'
+PIP_VERSION='21.2.4'
 PYTHON27_VERSION='2.7.18'
-PYTHON36_VERSION='3.6.15'
-PYTHON37_VERSION='3.7.12'
+PYTHON36_VERSION='3.6.14'
+PYTHON37_VERSION='3.7.11'
 PYTHON38_VERSION='3.8.12'
 PYTHON39_VERSION='3.9.7'
+PYTHON310_VERSION='3.10.0'

build/buildPythonSdkByVersion.sh

@@ -0,0 +1,160 @@
+#!/bin/bash
+# --------------------------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT license.
+# --------------------------------------------------------------------------------------------
+
+set -ex
+declare -r REPO_DIR=$( cd $( dirname "$0" ) && cd .. && pwd )
+
+source $REPO_DIR/build/__pythonVersions.sh
+
+pythonVersionGPG=''
+
+version="$1"
+
+buildPythonfromSource()
+{
+    pythonVersion=$PYTHON_VERSION 
+
+    if [ ! -z "$1" ]; then
+       echo "$1"
+       pythonVersion=$1
+    fi
+
+    if [ ! -z "$2" ]; then
+       echo "$2"
+       gpgKey=$2
+    fi
+
+    wget https://www.python.org/ftp/python/${pythonVersion%%[a-z]*}/Python-$pythonVersion.tar.xz -O /python.tar.xz
+    wget https://www.python.org/ftp/python/${pythonVersion%%[a-z]*}/Python-$pythonVersion.tar.xz.asc -O /python.tar.xz.asc
+
+    PYTHON_GET_PIP_URL="https://bootstrap.pypa.io/get-pip.py"
+
+    # for buster and ubuntu we would need following libraries
+        apt-get update && \
+        apt-get upgrade -y && \
+        DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
+            libssl-dev \
+            libncurses5-dev \
+            libsqlite3-dev \
+            libreadline-dev \
+            libbz2-dev \
+            libgdm-dev \
+            libbluetooth-dev \
+            tk-dev \
+            uuid-dev
+
+    # Try getting the keys 5 times at most
+    /tmp/receiveGpgKeys.sh $gpgKey
+
+    gpg --batch --verify /python.tar.xz.asc /python.tar.xz
+    tar -xJf /python.tar.xz --strip-components=1 -C .
+
+    INSTALLATION_PREFIX=/opt/python/$PYTHON_VERSION
+
+    if [ "${PYTHON_VERSION::1}" == "2" ]; then
+        ./configure \
+            --prefix=$INSTALLATION_PREFIX \
+            --build=$(dpkg-architecture --query DEB_BUILD_GNU_TYPE) \
+            --enable-shared \
+            --enable-unicode=ucs4
+    else
+        ./configure \
+            --prefix=$INSTALLATION_PREFIX \
+            --build=$(dpkg-architecture --query DEB_BUILD_GNU_TYPE) \
+            --enable-loadable-sqlite-extensions \
+            --enable-shared \
+            --with-system-expat \
+            --with-system-ffi \
+            --without-ensurepip
+    fi
+
+    make -j $(nproc)
+
+    make install
+
+    rm -rf /usr/src/python
+    find /usr/local -depth \
+        \( \
+            \( -type d -a \( -name test -o -name tests -o -name idle_test \) \) \
+            -o \( -type f -a \( -name '*.pyc' -o -name '*.pyo' -o -name '*.a' \) \) \
+        \) -exec rm -rf '{}' + \
+
+    ldconfig
+    python3 --version
+
+    # make some useful symlinks that are expected to exist
+    cd /usr/local/bin
+    ln -s idle3 idle
+    ln -s pydoc3 pydoc
+    ln -s python3 python
+    ln -s python3-config python-config
+
+    PYTHON_GET_PIP_SHA256="c518250e91a70d7b20cceb15272209a4ded2a0c263ae5776f129e0d9b5674309"
+
+    # Install pip
+    wget "$PYTHON_GET_PIP_URL" -O get-pip.py
+
+    python3 get-pip.py \
+        --trusted-host pypi.python.org \
+        --trusted-host pypi.org \
+        --trusted-host files.pythonhosted.org \
+        --disable-pip-version-check \
+        --no-cache-dir \
+        --no-warn-script-location
+
+    # Currently only for version '2' of Python, the alias 'python' exists in the 'bin'
+    # directory. So to make sure other versions also have this alias, we create the link
+    # explicitly here. This is for the scenarios where a user does 'benv python=3.7' and
+    # expects the alias 'python' to point to '3.7' rather than '2'. In cases where benv is
+    # not passed as an explicit python version, the version '2' is used by default. This is
+    # done in the Dockerfile.
+    pythonBinDir="$INSTALLATION_PREFIX/bin"
+    pythonAliasFile="$pythonBinDir/python"
+    if [ ! -e "$pythonAliasFile" ]; then
+        IFS='.' read -ra SPLIT_VERSION <<< "$PYTHON_VERSION"
+        majorAndMinorParts="${SPLIT_VERSION[0]}.${SPLIT_VERSION[1]}"
+        ln -s $pythonBinDir/python$majorAndMinorParts $pythonBinDir/python
+    fi
+}
+
+getPythonGpgByVersion() {
+    local versionFile="$1"
+    local versionPython="$2"
+
+    while IFS= read -ra VERSION_INFO || [[ -n $VERSION_INFO ]]
+    do
+        # Ignore whitespace and comments
+        if [ -z "$VERSION_INFO" ] || [[ $VERSION_INFO = \#* ]] ; then
+            continue
+        fi
+
+        arg="$(echo -e "${VERSION_INFO}" | sed -e 's/^[[:space:]]*//')"
+        test1=$(echo $arg | cut -d',' -f 1)
+        test2=$(echo $arg | cut -d',' -f 2)
+
+        if [ "$versionPython" == "$test1" ];then
+            pythonVersionGPG="$test2"
+        fi
+    done < "$versionFile"
+}
+
+
+echo
+echo "Building python 3.10 or newer from source code..."
+
+getPythonGpgByVersion "/tmp/versionsToBuild.txt" $version
+
+if  [ "$version" == "2.7" ] || [ "$version" == "3.6" ] || \
+    [ "$version" == "3.7" ] || [ "$version" == "3.8" ] || \
+    [ "$version" == "3.9" ]
+then
+    source /tmp/oryx/images/installPlatform.sh python $version --dir /opt/python/$version --links false
+else
+    buildPythonfromSource $version $pythonVersionGPG
+fi
+
+pip install --upgrade setuptools pip
+pip install gunicorn debugpy

build/constants.yaml

@@ -101,12 +101,13 @@
       file-name-prefix: __
 - name: python-versions
   constants:
-    pip-version: 20.2.3
+    pip-version: 21.2.4
     python27-version: 2.7.18
-    python36-version: 3.6.15
-    python37-version: 3.7.12
+    python36-version: 3.6.14
+    python37-version: 3.7.11
     python38-version: 3.8.12
     python39-version: 3.9.7
+    python310-version: 3.10.0
   outputs:
     - type: csharp
       directory: src/BuildScriptGenerator

images/runtime/python/generateDockerfiles.sh

@@ -17,39 +17,44 @@ declare -r PYTHON_VERSION_PLACEHOLDER="%PYTHON_VERSION%"
 declare -r PYTHON_FULL_VERSION_PLACEHOLDER="%PYTHON_FULL_VERSION%"
 declare -r ORYX_IMAGE_TAG_PLACEHOLDER="%IMAGE_TAG%"
 
+declare -r ORYX_PYTHON_IMAGE_BASE_PLACEHOLDER="%BASE_TAG%"
+
 source "$PYTHON_VERSIONS_PATH"
 
-declare -r PYTHON_BUSTER_VERSION_ARRAY=($PYTHON39_VERSION)
+declare -r PYTHON_BUSTER_VERSION_ARRAY=($PYTHON310_VERSION $PYTHON39_VERSION)
 declare -r PYTHON_STRETCH_VERSION_ARRAY=($PYTHON27_VERSION $PYTHON36_VERSION $PYTHON37_VERSION $PYTHON38_VERSION)
 ImageDebianFlavor="$1"
 echo "python baseimage type: $ImageDebianFlavor"
 
 VERSIONS_DIRECTORY=()
 
 if [ "$ImageDebianFlavor" == "buster" ];then
-	VERSIONS_DIRECTORY=("${PYTHON_BUSTER_VERSION_ARRAY[@]}")
+    VERSIONS_DIRECTORY=("${PYTHON_BUSTER_VERSION_ARRAY[@]}")
 else 
-	VERSIONS_DIRECTORY=("${PYTHON_STRETCH_VERSION_ARRAY[@]}")
+    VERSIONS_DIRECTORY=("${PYTHON_STRETCH_VERSION_ARRAY[@]}")
 fi
 
 
 for VERSION_DIRECTORY in "${VERSIONS_DIRECTORY[@]}"
 do
-	#PYTHON_VERSION=${!PYTHON_VERSION_VAR_NAME}
-	IFS='.' read -ra SPLIT_VERSION <<< "$VERSION_DIRECTORY"
-	#VERSION_DIRECTORY="${SPLIT_VERSION[0]}.${SPLIT_VERSION[1]}"
+    #PYTHON_VERSION=${!PYTHON_VERSION_VAR_NAME}
+    IFS='.' read -ra SPLIT_VERSION <<< "$VERSION_DIRECTORY"
+    #VERSION_DIRECTORY="${SPLIT_VERSION[0]}.${SPLIT_VERSION[1]}"
+
+    # IFS='.' read -ra SPLIT_VERSION <<< "$PYTHON_VERSION"
+    MAJOR_MINOR_VERSION="${SPLIT_VERSION[0]}.${SPLIT_VERSION[1]}"
 
-	# IFS='.' read -ra SPLIT_VERSION <<< "$PYTHON_VERSION"
-	MAJOR_MINOR_VERSION="${SPLIT_VERSION[0]}.${SPLIT_VERSION[1]}"
+    mkdir -p "$DIR/$MAJOR_MINOR_VERSION/"
+    TARGET_DOCKERFILE="$DIR/$MAJOR_MINOR_VERSION/$ImageDebianFlavor.Dockerfile"
+    cp "$DOCKERFILE_TEMPLATE" "$TARGET_DOCKERFILE"
 
-	mkdir -p "$DIR/$MAJOR_MINOR_VERSION/"
-	TARGET_DOCKERFILE="$DIR/$MAJOR_MINOR_VERSION/$ImageDebianFlavor.Dockerfile"
-	cp "$DOCKERFILE_TEMPLATE" "$TARGET_DOCKERFILE"
+    ORYX_PYTHON_IMAGE_BASE_TAG="oryx-run-base-$ImageDebianFlavor"
 
-	# Replace placeholders
-	sed -i "s|$PYTHON_VERSION_PLACEHOLDER|$MAJOR_MINOR_VERSION|g" "$TARGET_DOCKERFILE"
-	sed -i "s|$PYTHON_FULL_VERSION_PLACEHOLDER|$VERSION_DIRECTORY|g" "$TARGET_DOCKERFILE"
-	sed -i "s|$PYTHON_MAJOR_VERSION_PLACEHOLDER|${SPLIT_VERSION[0]}|g" "$TARGET_DOCKERFILE"
-	sed -i "s|$ORYX_IMAGE_TAG_PLACEHOLDER|$PYTHON_BASE_TAG|g" "$TARGET_DOCKERFILE"
+    # Replace placeholders
+    sed -i "s|$PYTHON_VERSION_PLACEHOLDER|$MAJOR_MINOR_VERSION|g" "$TARGET_DOCKERFILE"
+    sed -i "s|$PYTHON_FULL_VERSION_PLACEHOLDER|$VERSION_DIRECTORY|g" "$TARGET_DOCKERFILE"
+    sed -i "s|$PYTHON_MAJOR_VERSION_PLACEHOLDER|${SPLIT_VERSION[0]}|g" "$TARGET_DOCKERFILE"
+    sed -i "s|$ORYX_IMAGE_TAG_PLACEHOLDER|$PYTHON_BASE_TAG|g" "$TARGET_DOCKERFILE"
+    sed -i "s|$ORYX_PYTHON_IMAGE_BASE_PLACEHOLDER|$ORYX_PYTHON_IMAGE_BASE_TAG|g" "$TARGET_DOCKERFILE"
 
-done < <(compgen -A variable | grep 'PYTHON[0-9]\{2,\}_VERSION')
+done < <(compgen -A variable | grep 'PYTHON[0-9]\{2,\}_VERSION')

images/runtime/python/template.Dockerfile

@@ -1,6 +1,7 @@
 ARG DEBIAN_FLAVOR
 # Startup script generator
 FROM golang:1.14-${DEBIAN_FLAVOR} as startupCmdGen
+ARG DEBIAN_FLAVOR
 # Install dep
 RUN go get -u github.com/golang/dep/cmd/dep
 # GOPATH is set to "/go" in the base image
@@ -14,27 +15,59 @@ ENV GIT_COMMIT=${GIT_COMMIT}
 ENV BUILD_NUMBER=${BUILD_NUMBER}
 RUN ./build.sh python /opt/startupcmdgen/startupcmdgen
 
-FROM oryx-run-base-${DEBIAN_FLAVOR} AS main
+FROM %BASE_TAG% as main
 ARG IMAGES_DIR=/tmp/oryx/images
+ARG BUILD_DIR=/tmp/oryx/build
+ENV DEBIAN_FLAVOR=${DEBIAN_FLAVOR}
+
+RUN apt-get update \
+    && apt-get upgrade -y \
+    && apt-get install -y --no-install-recommends \
+        xz-utils \
+    && rm -rf /var/lib/apt/lists/*
+
+ADD images ${IMAGES_DIR}
+ADD build ${BUILD_DIR}
+RUN find ${IMAGES_DIR} -type f -iname "*.sh" -exec chmod +x {} \;
+RUN find ${BUILD_DIR} -type f -iname "*.sh" -exec chmod +x {} \;
+
 ENV PYTHON_VERSION %PYTHON_FULL_VERSION%
 
-RUN ${IMAGES_DIR}/installPlatform.sh python $PYTHON_VERSION --dir /opt/python/$PYTHON_VERSION --links false
+COPY build/__pythonVersions.sh ${BUILD_DIR}
+COPY platforms/__common.sh /tmp/
+COPY platforms/python/prereqs/build.sh /tmp/
+COPY platforms/python/versionsToBuild.txt /tmp/
+COPY images/receiveGpgKeys.sh /tmp/receiveGpgKeys.sh
+
+RUN chmod +x /tmp/receiveGpgKeys.sh
+RUN chmod +x /tmp/build.sh && \
+    apt-get update && \
+    DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
+        build-essential \ 
+        tk-dev \
+        uuid-dev \
+        libgeos-dev
+
+RUN ${BUILD_DIR}/buildPythonSdkByVersion.sh $PYTHON_VERSION $DEBIAN_FLAVOR
+
 RUN set -ex \
  && cd /opt/python/ \
  && ln -s %PYTHON_FULL_VERSION% %PYTHON_VERSION% \
  && ln -s %PYTHON_VERSION% %PYTHON_MAJOR_VERSION% \
  && echo /opt/python/%PYTHON_MAJOR_VERSION%/lib >> /etc/ld.so.conf.d/python.conf \
  && ldconfig \
  && if [ "%PYTHON_MAJOR_VERSION%" = "3" ]; then cd /opt/python/%PYTHON_MAJOR_VERSION%/bin \
- && ln -s idle3 idle \
- && ln -s pydoc3 pydoc \
- && ln -s python3-config python-config; fi
+ && ln -nsf idle3 idle \
+ && ln -nsf pydoc3 pydoc \
+ && ln -nsf python3-config python-config; fi \
+ && rm -rf /var/lib/apt/lists/*
 
 ENV PATH="/opt/python/%PYTHON_MAJOR_VERSION%/bin:${PATH}"
 
 # Bake Application Insights key from pipeline variable into final image
 ARG AI_KEY
 ENV ORYX_AI_INSTRUMENTATION_KEY=${AI_KEY}
+
 RUN ${IMAGES_DIR}/runtime/python/install-dependencies.sh
 RUN pip install --upgrade pip \
     && pip install gunicorn \
@@ -47,5 +80,5 @@ RUN pip install --upgrade pip \
     && apt-get upgrade --assume-yes \
     && rm -rf /var/lib/apt/lists/* \
     && rm -rf /tmp/oryx
-    
-COPY --from=startupCmdGen /opt/startupcmdgen/startupcmdgen /opt/startupcmdgen/startupcmdgen
+
+COPY --from=startupCmdGen /opt/startupcmdgen/startupcmdgen /opt/startupcmdgen/startupcmdgen

platforms/python/buildPython.sh

@@ -18,11 +18,20 @@ mkdir -p "$targetDir"
 
 builtPythonPrereqs=false
 buildPythonPrereqsImage() {
+    debianType=$debianFlavor
+	# stretch is out of support for python, but we still need to build stretch based
+	# binaries because of static sites, they need to move to buster based jamstack image
+	# before we can remove this hack
+    if [ "$debianFlavor" == "stretch" ]; then
+        debianType="focal-scm"
+    fi
+
 	if ! $builtPythonPrereqs; then
 		echo "Building Python pre-requisites image..."
 		echo
 		docker build  \
 			   --build-arg DEBIAN_FLAVOR=$debianFlavor \
+			   --build-arg DEBIAN_HACK_FLAVOR=$debianType \
 			   -f "$pythonPlatformDir/prereqs/Dockerfile"  \
 			   -t "python-build-prereqs" $REPO_DIR
 		builtPythonPrereqs=true
@@ -51,13 +60,16 @@ buildPython() {
 		echo "Building Python version '$version' in a docker image..."
 		echo
 
+        echo "dockerfile is : $pythonPlatformDir/$dockerFile"
 		if [ -z "$dockerFile" ]; then
 			# Use common docker file
 			dockerFile="$pythonPlatformDir/Dockerfile"
 		else
 			dockerFile="$pythonPlatformDir/$dockerFile"
 		fi
 
+		cat $dockerFile
+
 		docker build \
 			-f "$dockerFile" \
 			--build-arg VERSION_TO_BUILD=$version \

platforms/python/prereqs/Dockerfile

@@ -1,8 +1,13 @@
 # Install Python build prerequisites
 ARG DEBIAN_FLAVOR
-FROM buildpack-deps:${DEBIAN_FLAVOR}
+# DEBIAN_HACK_FLAVOR is only needed for python 3.10 or newer
+ARG DEBIAN_HACK_FLAVOR
+FROM buildpack-deps:${DEBIAN_HACK_FLAVOR}
 ARG DEBIAN_FLAVOR
+ARG DEBIAN_HACK_FLAVOR
 ENV DEBIAN_FLAVOR=$DEBIAN_FLAVOR
+ENV DEBIAN_HACK_FLAVOR=$DEBIAN_HACK_FLAVOR
+
 COPY build/__pythonVersions.sh /tmp/
 
 COPY platforms/python/prereqs/build.sh /tmp/