Upgrade docker buildx 0.9.1 and fix Python compilation issues

docker/ci/build-image-multi-arch.sh

@@ -95,6 +95,8 @@ trap cleanup_all TERM INT EXIT
 DIR=`Temp_Folder_Create`
 echo "New workspace $DIR"
 echo -e "\n* Prepare docker buildx"
+docker buildx rm --all-inactive --force
+docker buildx prune --all --force
 docker buildx use default
 docker buildx create --name ${BUILDER_NAME} --use
 docker buildx inspect --bootstrap

docker/ci/dockerfiles/current/build.centos7.opensearch-dashboards.x64.arm64.dockerfile

@@ -68,7 +68,7 @@ ENV PATH=$RUBY_HOME:$RVM_HOME:$PATH
 RUN curl https://www.python.org/ftp/python/3.9.17/Python-3.9.17.tgz | tar xzvf - && \
     cd Python-3.9.17 && \
     ./configure --enable-optimizations && \
-    make altinstall -j $(( `nproc` / 2 ))
+    make altinstall
 
 # Setup Python links
 RUN ln -sfn /usr/local/bin/python3.9 /usr/bin/python3 && \

docker/ci/dockerfiles/current/build.centos7.opensearch.x64.arm64.dockerfile

@@ -77,7 +77,7 @@ ENV PATH=$RUBY_HOME:$RVM_HOME:$PATH
 RUN curl https://www.python.org/ftp/python/3.9.17/Python-3.9.17.tgz | tar xzvf - && \
     cd Python-3.9.17 && \
     ./configure --enable-optimizations && \
-    make altinstall -j $(( `nproc` / 2 ))
+    make altinstall
 
 # Setup Python links
 RUN ln -sfn /usr/local/bin/python3.9 /usr/bin/python3 && \

docker/ci/dockerfiles/current/docker-builder.ubuntu2004.x64.dockerfile

@@ -54,9 +54,14 @@ ENV JAVA_HOME=/opt/java/openjdk-11
 ENV PATH=$PATH:$JAVA_HOME/bin
 
 # Install docker buildx
-# Stays on 0.6.3 as that is the stable version we used since the introduction of OpenSearch Multi-Arch Docker Images
+# 2023-06-20 Upgrade from 0.6.3 to 0.9.1 due to binary translation issues to build arm64 from x64 multi-arch image on Python 3.9
+# https://github.com/docker/buildx/releases/tag/v0.9.1
+# Avoid upgrading to 0.10.0+ due to this change:
+#   Buildx v0.10 enables support for a minimal SLSA Provenance attestation, which requires support for OCI-compliant multi-platform images.
+#   This may introduce issues with registry and runtime support (e.g. Google Cloud Run and Lambda).
+#   You can optionally disable the default provenance attestation functionality using --provenance=false.
 RUN mkdir -p ~/.docker/cli-plugins && \
-    curl -SL https://github.com/docker/buildx/releases/download/v0.6.3/buildx-v0.6.3.linux-amd64 -o ~/.docker/cli-plugins/docker-buildx  && \
+    curl -SL https://github.com/docker/buildx/releases/download/v0.9.1/buildx-v0.9.1.linux-amd64 -o ~/.docker/cli-plugins/docker-buildx  && \
     chmod 775 ~/.docker/cli-plugins/docker-buildx && \
     docker buildx version
 

docker/ci/dockerfiles/current/release.centos7.clients.x64.arm64.dockerfile

@@ -41,11 +41,11 @@ RUN if [[ `uname -m` = 'aarch64' ]]; then mkdir -p aarch64-builds && cd aarch64-
     ln -sfn /lib64/libstdc++.so.6.0.29 /lib64/libstdc++.so.6 && \
     echo "Installing glibc 2.18" && \
     curl -SLO https://ftp.gnu.org/gnu/glibc/glibc-2.18.tar.gz && tar -xzvf glibc-2.18.tar.gz && cd glibc-2.18 && mkdir -p build && cd build && \
-    ../configure --prefix=/usr && make -j $(( `nproc` / 2 )) && make install && cd ../../ && \
+    ../configure --prefix=/usr && make && make install && cd ../../ && \
     echo "Installing libicu 53+" && \
     rpm -e --nodeps libicu && \
     curl -SLO https://github.com/unicode-org/icu/releases/download/release-53-2/icu4c-53_2-src.tgz && tar -xzvf icu4c-53_2-src.tgz && cd icu && \
-    cd source && ./configure --prefix=/usr && make -j $(( `nproc` / 2 )) && make install && \
+    cd source && ./configure --prefix=/usr && make && make install && \
     cd ../../../ && rm -rf aarch64-builds; fi
 
 ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/lib
@@ -75,7 +75,7 @@ RUN chmod -R 777 /dev/shm
 RUN curl https://www.python.org/ftp/python/3.9.17/Python-3.9.17.tgz | tar xzvf - && \
     cd Python-3.9.17 && \
     ./configure --enable-optimizations && \
-    make altinstall -j $(( `nproc` / 2 ))
+    make altinstall
 
 # Setup Python links
 RUN ln -sfn /usr/local/bin/python3.9 /usr/bin/python3 && \
@@ -92,7 +92,7 @@ ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib:/usr/local/lib64:/usr/lib
 RUN yum install -y curl libcurl-devel libfaketime perl-core pcre-devel && yum remove -y openssl-devel && yum clean all && \
     mkdir -p /tmp/openssl && cd /tmp/openssl && \
     curl -sSL -o- https://www.openssl.org/source/openssl-1.1.1g.tar.gz | tar -xz --strip-components 1 && \
-    ./config --prefix=/usr --openssldir=/etc/ssl --libdir=lib shared zlib-dynamic && make -j $(( `nproc` / 2 )) && make install && \
+    ./config --prefix=/usr --openssldir=/etc/ssl --libdir=lib shared zlib-dynamic && make && make install && \
     echo "export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib:/usr/local/lib64:/usr/lib" > /etc/profile.d/openssl.sh && openssl version
 
 # Installing osslsigncode

docker/ci/dockerfiles/current/test.centos7.performance-test.x64.arm64.dockerfile

@@ -38,7 +38,7 @@ RUN yum install -y @development zlib-devel bzip2 bzip2-devel readline-devel sqli
 RUN curl https://www.python.org/ftp/python/3.7.7/Python-3.7.7.tgz | tar xzvf - && \
     cd Python-3.7.7 && \
     ./configure --enable-optimizations && \
-    make altinstall -j $(( `nproc` / 2 ))
+    make altinstall
 
 # Setup Python37 links
 RUN ln -sfn /usr/local/bin/python3.7 /usr/bin/python3 && \

docker/ci/dockerfiles/current/test.rockylinux8.opensearch-dashboards.x64.arm64.dockerfile

@@ -120,7 +120,7 @@ RUN chmod -R 777 /dev/shm
 RUN curl https://www.python.org/ftp/python/3.7.7/Python-3.7.7.tgz | tar xzvf - && \
     cd Python-3.7.7 && \
     ./configure --enable-optimizations && \
-    make altinstall -j $(( `nproc` / 2 ))
+    make altinstall
 
 # Setup Python37 links
 RUN ln -sfn /usr/local/bin/python3.7 /usr/bin/python3 && \

docker/ci/dockerfiles/current/test.rockylinux8.systemd-base.x64.arm64.dockerfile

@@ -131,7 +131,7 @@ RUN chmod -R 777 /dev/shm
 RUN curl https://www.python.org/ftp/python/3.7.7/Python-3.7.7.tgz | tar xzvf - && \
     cd Python-3.7.7 && \
     ./configure --enable-optimizations && \
-    make altinstall -j $(( `nproc` / 2 ))
+    make altinstall
 
 # Setup Python37 links
 RUN ln -sfn /usr/local/bin/python3.7 /usr/bin/python3 && \

docker/release/build-image-multi-arch.sh

@@ -135,6 +135,8 @@ trap cleanup_all TERM INT EXIT
 DIR=`Temp_Folder_Create`
 echo New workspace $DIR
 echo -e "\n* Prepare docker buildx"
+docker buildx rm --all-inactive --force
+docker buildx prune --all --force
 docker buildx use default
 docker buildx create --name $BUILDER_NAME --use
 docker buildx inspect --bootstrap

jenkins/docker/docker-build.jenkinsfile

@@ -24,23 +24,23 @@ pipeline {
         )
     }
     stages {
-        stage('Parameters Check') {
-            steps {
-                script {
-                    if(! DOCKER_BUILD_GIT_REPOSITORY.startsWith('https://github.com/opensearch-project/')) {
-                        currentBuild.result = 'ABORTED'
-                        error('The repository needs to be an opensearch-project repository')
-                    }
-                }
-            }
-        }
+        //stage('Parameters Check') {
+        //    steps {
+        //        script {
+        //            if(! DOCKER_BUILD_GIT_REPOSITORY.startsWith('https://github.com/opensearch-project/')) {
+        //                currentBuild.result = 'ABORTED'
+        //                error('The repository needs to be an opensearch-project repository')
+        //            }
+        //        }
+        //    }
+        //}
         stage('docker-build') {
             agent {
                 docker {
                     label 'Jenkins-Agent-Ubuntu2004-X64-M52xlarge-Docker-Builder'
-                    image 'opensearchstaging/ci-runner:ubuntu2004-x64-docker-buildx0.6.3-qemu5.0-awscli1.22-jdk11-v2'
+                    image 'opensearchstaging/ci-runner:ubuntu2004-x64-docker-buildx0.9.1-qemu5.0-awscli1.22-jdk11-v1.1'
                     args '-u root -v /var/run/docker.sock:/var/run/docker.sock'
-                    registryUrl 'https://public.ecr.aws/'
+                    //registryUrl 'https://public.ecr.aws/'
                     alwaysPull true
                 }
             }