refresh

.github/workflows/cicd.yml

@@ -65,13 +65,14 @@ jobs:
       fail-fast: false
       matrix:
         IMAGES:
-        - "tgagor/centos/stream tgagor/centos/latest tgagor/centos/9 tgagor/centos/stream9"
-        - "tgagor/centos/7 tgagor/centos/8 tgagor/centos/stream8"
+        - "tgagor/centos/latest tgagor/centos/stream"
+        - "tgagor/centos/9 tgagor/centos/stream9"
+        - "tgagor/centos/8 tgagor/centos/centos8 tgagor/centos/stream8"
+        - "tgagor/centos/7 tgagor/centos/centos7"
         #- "corpusops/test/a corpusops/test/b corpusops/test/c"
         # all images that werent explicitly told to be built would be built in the next batches
         # we span them onto N jobs
-        - "zleftover:1/2"
-        - "zleftover:2/2"
+        - "zleftover:1/1"
 on:
   push:
   workflow_dispatch:

.github/workflows/cicd.yml.in

@@ -68,8 +68,7 @@ jobs:
         #- "corpusops/test/a corpusops/test/b corpusops/test/c"
         # all images that werent explicitly told to be built would be built in the next batches
         # we span them onto N jobs
-        - "zleftover:1/2"
-        - "zleftover:2/2"
+        - "zleftover:1/1"
 on:
   push:
   workflow_dispatch:

Dockerfile.args

@@ -3,7 +3,7 @@ ARG NONINTERACTIVE=y
 ARG DEBIAN_FRONTEND=noninteractive
 ARG INSTALL_DEFAULT_LOCALE="fr_FR.UTF-8"
 ARG EXTRA_FILES_LIST=""
-ARG COPS_SYSTEM=${_cops_SYSTEM}
+ARG COPS_SYSTEM=${_cops_SYSTEM-}
 ARG FOREGO_RELEASE=latest
 ARG REMCO_RELEASE=latest
 ARG GOSU_RELEASE=latest

Dockerfile.labels

@@ -1 +1,2 @@
+ARG DOCKER_IMAGES_COMMIT="$DOCKER_IMAGES_COMMIT"
 LABEL com.github.corpusops.docker-images-commit="$DOCKER_IMAGES_COMMIT"

Dockerfile.squash

@@ -0,0 +1,4 @@
+FROM $RSYNC AS squashed-rsync
+FROM $ANCESTOR AS squashed-ancestor
+ARG ROOTFS="/BASE_ROOTFS_TO_COPY_THAT_WONT_COLLIDE_1234567890"
+ARG PATH="${ROOTFS}_rsync/bin:$PATH"

Dockerfile.squashexec

@@ -0,0 +1,4 @@
+SHELL ["busybox",  "sh", "-c"]
+RUN --mount=type=bind,from=final,target=$ROOTFS --mount=type=bind,from=squashed-rsync,target=${ROOTFS}_rsync \
+rsync -Aaz --delete ${ROOTFS}/ / --exclude=/proc --exclude=/sys --exclude=/etc/resolv.conf --exclude=/etc/hosts --exclude=$ROOTFS* --exclude=dev/shm --exclude=dev/pts --exclude=dev/mqueue
+SHELL ["/bin/sh", "-c"]

Dockerfile.squashpre

@@ -0,0 +1,2 @@
+ARG ANCESTOR=${_cops_BASE}:${_cops_VERSION}
+ARG RSYNC=corpusops/rsync

helpers/cops_pkgmgr_install.sh

@@ -36,6 +36,7 @@ SYSTEM_COPS_ROOT=${SYSTEM_COPS_ROOT-$DEFAULT_COPS_ROOT}
 DOCKER_COPS_ROOT=${DOCKER_COPS_ROOT-$SYSTEM_COPS_ROOT}
 COPS_URL=${COPS_URL-$DEFAULT_COPS_URL}
 BASE_PREPROVISION_IMAGES="ubuntu:latest_preprovision"
+BASE_PREPROVISION_IMAGES="$BASE_PREPROVISION_IMAGES corpusops/ubuntu:24.04_preprovision"
 BASE_PREPROVISION_IMAGES="$BASE_PREPROVISION_IMAGES corpusops/ubuntu:22.04_preprovision"
 BASE_PREPROVISION_IMAGES="$BASE_PREPROVISION_IMAGES corpusops/ubuntu:20.04_preprovision"
 BASE_PREPROVISION_IMAGES="$BASE_PREPROVISION_IMAGES corpusops/ubuntu:18.04_preprovision"
@@ -44,6 +45,7 @@ BASE_PREPROVISION_IMAGES="$BASE_PREPROVISION_IMAGES corpusops/ubuntu:14.04_prepr
 BASE_PREPROVISION_IMAGES="$BASE_PREPROVISION_IMAGES corpusops/centos:7_preprovision"
 
 BASE_CORE_IMAGES="$BASE_CORE_IMAGES corpusops/ubuntu:latest"
+BASE_CORE_IMAGES="$BASE_CORE_IMAGES corpusops/ubuntu:24.04"
 BASE_CORE_IMAGES="$BASE_CORE_IMAGES corpusops/ubuntu:22.04"
 BASE_CORE_IMAGES="$BASE_CORE_IMAGES corpusops/ubuntu:20.04"
 BASE_CORE_IMAGES="$BASE_CORE_IMAGES corpusops/ubuntu:18.04"
@@ -55,6 +57,7 @@ EXP_PREPROVISION_IMAGES=""
 EXP_PREPROVISION_IMAGES="$EXP_PREPROVISION_IMAGES archlinux:latest_preprovision"
 EXP_PREPROVISION_IMAGES="$EXP_PREPROVISION_IMAGES debian:latest_preprovision"
 #EXP_PREPROVISION_IMAGES="$EXP_PREPROVISION_IMAGES debian:stretch_preprovision"
+EXP_PREPROVISION_IMAGES="$EXP_PREPROVISION_IMAGES debian:bookworm_preprovision"
 EXP_PREPROVISION_IMAGES="$EXP_PREPROVISION_IMAGES debian:bullseye_preprovision"
 EXP_PREPROVISION_IMAGES="$EXP_PREPROVISION_IMAGES debian:buster_preprovision"
 EXP_PREPROVISION_IMAGES="$EXP_PREPROVISION_IMAGES debian:sid_preprovision"
@@ -64,6 +67,7 @@ EXP_CORE_IMAGES="$EXP_CORE_IMAGES corpusops/debian:latest"
 #EXP_CORE_IMAGES="$EXP_CORE_IMAGES corpusops/debian:stretch"
 EXP_CORE_IMAGES="$EXP_CORE_IMAGES corpusops/debian:bullseye"
 EXP_CORE_IMAGES="$EXP_CORE_IMAGES corpusops/debian:buster"
+EXP_CORE_IMAGES="$EXP_CORE_IMAGES corpusops/debian:bookworm"
 EXP_CORE_IMAGES="$EXP_CORE_IMAGES corpusops/debian:sid"
 EXP_IMAGES="$EXP_PREPROVISION_IMAGES $EXP_CORE_IMAGES"
 # ansible related
@@ -518,8 +522,8 @@ get_python2() {
 get_python3() {
     local py_ver=3
     get_python_ $py_ver \
-        python3.10  python3.9  python3.8  python3.7  python3.6  python3.5  python3.4  \
-        python-3.10 python-3.9 python-3.8 python-3.7 python-3.6 python-3.5 python-3.4 \
+        python3.12  python3.11  python3.10  python3.9  python3.8  python3.7  python3.6  python3.5  python3.4  \
+        python-3.12 python-3.11 python-3.10 python-3.9 python-3.8 python-3.7 python-3.6 python-3.5 python-3.4 \
         python-${py_ver} python${py_ver} python
 }
 has_python_module() {
@@ -538,9 +542,19 @@ pymod_ver() {
 get_setuptools() {
     local py=${1:-python}
     local setuptoolsreq="setuptools"
-    if ( is_python2 $py );then setuptoolsreq="setuptools<=45"; else setuptoolsreq="setuptools<50"; fi
+    local cpyver=$($py -c "import sys;print(sys.version.split()[0])")
+    if ( is_python2 $py );then
+        setuptoolsreq="setuptools<=45"
+    elif ( version_lt $cpyver 3.12.0 );then
+        setuptoolsreq="setuptools<66"
+    else
+        setuptoolsreq="setuptools>=75"
+    fi
     echo "$setuptoolsreq"
 }
+setup_setuptools_requirement() {
+    sed -i -re "s/^setuptools\s*(>|<|=|$)/$(get_setuptools $py)/g" requirements/python_requirements.txt
+}
 install_pip() {
     local py="${1:-python}"
     local DEFAULT_PIP_URL="https://bootstrap.pypa.io/get-pip.py"
@@ -760,8 +774,8 @@ ensure_command() {
 ### archlinux (pacman)
 is_pacman_available() {
     for i in $@;do
-        if ! ( pacman -Si $(i_y) "$i" >/devnull 2>&1 ||\
-                pacman -Sg $(i_y) "$i" >/devnull 2>&1; );then
+        if ! ( pacman -Si $(i_y) "$i" >/dev/null 2>&1 ||\
+                pacman -Sg $(i_y) "$i" >/dev/null 2>&1; );then
             return 1
         fi
     done
@@ -770,7 +784,7 @@ is_pacman_available() {
 
 is_pacman_installed() {
     for i in $@;do
-        if ! ( pacman -Qi $(i_y) "$i" >/devnull 2>&1; ); then
+        if ! ( pacman -Qi $(i_y) "$i" >/dev/null 2>&1; ); then
             return 1
         fi
     done

helpers/up.sh

@@ -1,5 +1,5 @@
 #!/usr/bin/env sh
-set -e
+set -ex
 log() { echo "${@}" >&2; }
 vv() { log "${@}";"${@}"; }
 DO_UPDATE=1
@@ -18,17 +18,19 @@ _cops_SYSTEM=$(system_detect.sh||./system_detect.sh||"$W/system_detect.sh")
 DISTRIB_ID=
 DISTRIB_CODENAME=
 DISTRIB_RELEASE=
-oldubuntu="^(10\.|12\.|13\.|14\.|15\.|16\.|17\.|18\.10|19\.|20\.10|21\.)"
+oldubuntu="^(10\.|12\.|13\.|14\.|15\.|16\.|17\.|18\.10|19\.|20\.10|21\.|22\.10)"
 # oldubuntu="^(10\.|12\.|13\.|14.10|15\.|16.10|17\.04)"
 NOSOCAT=""
 CENTOS_OLDSTABLE=8
+CENTOS_OLDSTABLES="6|7|8"
 OAPTMIRROR="${OAPTMIRROR:-}"
 OYUMMIRROR="${OYUMMIRROR:-}"
 NYUMMIRROR="${NYUMMIRROR:-}"
 OUBUNTUMIRROR="${OUBUNTUMIRROR:-old-releases.ubuntu.com}"
 ODEBIANMIRROR="${ODEBIANMIRROR:-archive.debian.org}"
 NDEBIANMIRROR="${NDEBIANMIRROR:-http.debian.net|httpredir.debian.org|deb.debian.org}"
 NUBUNTUMIRROR="${NUBUNTUMIRROR:-archive.ubuntu.com|security.ubuntu.com}"
+IS_OLD_CENTOS_STABLE=""
 SNCENTOSMIRROR="$(echo "${NCENTOSMIRROR}"|sed -re "s/\|.*//g")"
 SNDEBIANMIRROR="$(echo "${NDEBIANMIRROR}"|sed -re "s/\|.*//g")"
 SNUBUNTUMIRROR="$(echo "${NUBUNTUMIRROR}"|sed -re "s/\|.*//g")"
@@ -58,52 +60,53 @@ elif [ -e /etc/redhat-release ];then
     DISTRIB_RELEASE=$(echo $(head  /etc/issue)|awk '{print tolower($3)}')
 fi
 DISTRIB_MAJOR="$(echo ${DISTRIB_RELEASE}|sed -re "s/\..*//g")"
-if [ "x${DISTRIB_ID}" = "xcentos" ] && ( echo  "${DISTRIB_MAJOR}" | grep -Eq "^(6|7|8)");then
-    sed -i 's/^mirrorlist/#mirrorlist/g;s|^#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-*
-fi
-if ( echo $DISTRIB_ID | grep -E -iq "centos|red|fedora" );then
-    if (echo $DISTRIB_ID|grep -E -iq centos);then
-        if (echo "$DISTRIB_RELEASE"|egrep -q "8|7");then
-            OCENTOSMIRROR="${OCENTOSMIRROR:-mirror.centos.org}"
-            NCENTOSMIRROR="${NCENTOSMIRROR:-vault.centos.org}"
-        elif [ $DISTRIB_RELEASE -le $CENTOS_OLDSTABLE ];then
-            OCENTOSMIRROR="${OCENTOSMIRROR:-vault.centos.org}"
-            NCENTOSMIRROR="${NCENTOSMIRROR:-mirror.centos.org}"
-        else
-            OCENTOSMIRROR="${OCENTOSMIRROR:-mirror.centos.org}"
-            NCENTOSMIRROR="${NCENTOSMIRROR:-vault.centos.org}"
-        fi
-        OYUMMIRROR="${OCENTOSMIRROR}"
-        NYUMMIRROR="${NCENTOSMIRROR}"
+EPEL_RPM_URL="https://dl.fedoraproject.org/pub/epel/epel-release-latest-${DISTRIB_MAJOR}.noarch.rpm"
+if [ "x${DISTRIB_ID}" = "xcentos" ] && ( echo  "${DISTRIB_MAJOR}" | grep -Eq "$CENTOS_OLDSTABLES");then
+    IS_OLD_CENTOS_STABLE="1"
+    EPEL_RPM_URL="https://archives.fedoraproject.org/pub/archive/epel/7/x86_64/Packages/e/epel-release-7-14.noarch.rpm"
+    if (echo "$DISTRIB_RELEASE"|egrep -q "8|7");then
+        OCENTOSMIRROR="${OCENTOSMIRROR:-mirror.centos.org}"
+        NCENTOSMIRROR="${NCENTOSMIRROR:-vault.centos.org}"
+    elif [ $DISTRIB_RELEASE -le $CENTOS_OLDSTABLE ];then
+        OCENTOSMIRROR="${OCENTOSMIRROR:-vault.centos.org}"
+        NCENTOSMIRROR="${NCENTOSMIRROR:-mirror.centos.org}"
+    else
+        OCENTOSMIRROR="${OCENTOSMIRROR:-mirror.centos.org}"
+        NCENTOSMIRROR="${NCENTOSMIRROR:-vault.centos.org}"
     fi
+    OYUMMIRROR="${OCENTOSMIRROR}"
+    NYUMMIRROR="${NCENTOSMIRROR}"
     if [ "$OYUMMIRROR" != "x" ];then
-        sed -i -r -e 's!'$NCENTOSMIRROR'!'$OCENTOSMIRROR'!g' $( find /etc/yum.repos.d -type f; )
+        sed -i -r -e 's!'$OCENTOSMIRROR'!'$NCENTOSMIRROR'!g' $( find /etc/yum.repos.d -type f; )
     fi
+    sed -i "s/^#.*baseurl=http/baseurl=http/g" $( find /etc/yum.repos.d -type f; )
+    sed -i "s/^mirrorlist=http/#mirrorlist=http/g" $( find /etc/yum.repos.d -type f; )
 fi
 if ( grep -q amzn /etc/os-release );then
     yuminstall findutils
     if ( amazon-linux-extras help >/dev/null 2>&1 );then
         amazon-linux-extras install -y epel
     else
-        yum install -y epel-release
-        yum-config-manager --enable epel
+        if ( yum list | grep -q epel );then
+            yum install -y epel-release
+            yum-config-manager --enable epel
+        fi
     fi
 fi
 if [ -e /etc/redhat-release ];then
     if [ -e /etc/fedora-release ];then
         vv yum upgrade -y --nogpg fedora-gpg-keys fedora-repos
     fi
     if [ ! -e /etc/yum.repos.d/epel.repo ];then
-        rpm="epel-release-latest-${DISTRIB_MAJOR}.noarch.rpm"
-        curl -sSLO "https://dl.fedoraproject.org/pub/epel/$rpm"
-        rpm -ivh $(pwd)/$rpm
+        curl -sSLO "$EPEL_RPM_URL"
+        rpm -ivh $(pwd)/$(basename $EPEL_RPM_URL)
     fi
     if ! ( find --version >/dev/null 2>&1);then
         yuminstall findutils
     fi
 fi
-DEBIAN_OLDSTABLE=8
-PG_DEBIAN_OLDSTABLE=9
+DEBIAN_OLDSTABLE=9
+PG_DEBIAN_OLDSTABLE=10
 find /etc -name "*.reactivate" | while read f;do
     mv -fv "$f" "$(basename $f .reactivate)"
 done
@@ -121,6 +124,7 @@ if (echo $DISTRIB_ID | grep -E -iq "debian");then
         if (echo $DISTRIB_RELEASE | grep -E -iq stretch );then  DISTRIB_CODENAME="$DISTRIB_RELEASE";DISTRIB_RELEASE="9" ;fi
         if (echo $DISTRIB_RELEASE | grep -E -iq buster );then   DISTRIB_CODENAME="$DISTRIB_RELEASE";DISTRIB_RELEASE="10";fi
         if (echo $DISTRIB_RELEASE | grep -E -iq bullseye );then DISTRIB_CODENAME="$DISTRIB_RELEASE";DISTRIB_RELEASE="11";fi
+        if (echo $DISTRIB_RELEASE | grep -E -iq bookworm );then DISTRIB_CODENAME="$DISTRIB_RELEASE";DISTRIB_RELEASE="12";fi
     fi
     sed -i -re "s/(old)?oldstable/$DISTRIB_CODENAME/g" $(find /etc/apt/sources.list* -type f)
     NAPTMIRROR="${NDEBIANMIRROR}"
@@ -162,7 +166,7 @@ if ( echo $DISTRIB_ID | grep -E -iq "debian|mint|ubuntu" );then
         sed -i -re "s!sid(/)?!$DISTRIB_CODENAME\1!" $(find /etc/apt/sources.list* -type f)
         OAPTMIRROR="${OAPTMIRROR:-$ODEBIANMIRROR}"
         sed -i -r -e '/-updates|security.debian.org/d' $( find /etc/apt/sources.list* -type f; )
-        if (echo $DISTRIB_ID|grep -E -iq debian) && [ $DISTRIB_RELEASE -eq $DEBIAN_OLDSTABLE ];then
+        if (echo $DISTRIB_ID|grep -E -iq debian) && [ $DISTRIB_RELEASE -eq $DEBIAN_OLDSTABLE ] && [ $DEBIAN_OLDSTABLE -lt 9 ];then
             log "Using debian LTS packages"
             echo "$DEBIAN_LTS_SOURCELIST" >> /etc/apt/sources.list
             rm -rvf /var/lib/apt/*

library/centos/6/Dockerfile

@@ -4,7 +4,7 @@ ARG NONINTERACTIVE=y
 ARG DEBIAN_FRONTEND=noninteractive
 ARG INSTALL_DEFAULT_LOCALE="fr_FR.UTF-8"
 ARG EXTRA_FILES_LIST=""
-ARG COPS_SYSTEM=${_cops_SYSTEM}
+ARG COPS_SYSTEM=${_cops_SYSTEM-}
 ARG FOREGO_RELEASE=latest
 ARG REMCO_RELEASE=latest
 ARG GOSU_RELEASE=latest
@@ -83,4 +83,5 @@ RUN sh -c 'set -ex \
     && ./${_cops_SYSTEM}_clean.sh \
     && ./common_clean.sh \
     && cd / && rm -rf /tmp/corpusopssteroids /var/cache/apk/* /var/lib/apt/lists/*'
+ARG DOCKER_IMAGES_COMMIT="$DOCKER_IMAGES_COMMIT"
 LABEL com.github.corpusops.docker-images-commit="$DOCKER_IMAGES_COMMIT"