better version protections routines

.github/workflows/cicd.yml

@@ -65,24 +65,14 @@ jobs:
       fail-fast: false
       matrix:
         IMAGES:
-        - "library/traefik/alpine library/traefik/1.7-alpine library/traefik/2.0 library/traefik/2.0-alpine library/traefik/v2.0 library/traefik/v2.2 library/traefik/v2.4 library/traefik/2.4"
+        - "library/traefik/latest library/traefik/alpine"
+        - "library/traefik/3 library/traefik/3.1 library/traefik/v3.1 library/traefik/v3"
+        - "library/traefik/v2 library/traefik/2 library/traefik/v2.11 library/traefik/2.11"
         #- "corpusops/test/a corpusops/test/b corpusops/test/c"
         # all images that werent explicitly told to be built would be built in the next batches
         # we span them onto N jobs
-        - "zleftover:1/14"
-        - "zleftover:2/14"
-        - "zleftover:3/14"
-        - "zleftover:4/14"
-        - "zleftover:5/14"
-        - "zleftover:6/14"
-        - "zleftover:7/14"
-        - "zleftover:8/14"
-        - "zleftover:9/14"
-        - "zleftover:10/14"
-        - "zleftover:11/14"
-        - "zleftover:12/14"
-        - "zleftover:13/14"
-        - "zleftover:14/14"
+        - "zleftover:1/2"
+        - "zleftover:2/2"
 on:
   push:
   workflow_dispatch:

.github/workflows/cicd.yml.in

@@ -68,20 +68,8 @@ jobs:
         #- "corpusops/test/a corpusops/test/b corpusops/test/c"
         # all images that werent explicitly told to be built would be built in the next batches
         # we span them onto N jobs
-        - "zleftover:1/14"
-        - "zleftover:2/14"
-        - "zleftover:3/14"
-        - "zleftover:4/14"
-        - "zleftover:5/14"
-        - "zleftover:6/14"
-        - "zleftover:7/14"
-        - "zleftover:8/14"
-        - "zleftover:9/14"
-        - "zleftover:10/14"
-        - "zleftover:11/14"
-        - "zleftover:12/14"
-        - "zleftover:13/14"
-        - "zleftover:14/14"
+        - "zleftover:1/2"
+        - "zleftover:2/2"
 on:
   push:
   workflow_dispatch:

Dockerfile.args

@@ -1,8 +1,9 @@
 ARG DO_UPDATE=y
 ARG NONINTERACTIVE=y
+ARG DEBIAN_FRONTEND=noninteractive
 ARG INSTALL_DEFAULT_LOCALE="fr_FR.UTF-8"
 ARG EXTRA_FILES_LIST=""
-ARG COPS_SYSTEM=${_cops_SYSTEM}
+ARG COPS_SYSTEM=${_cops_SYSTEM-}
 ARG FOREGO_RELEASE=latest
 ARG REMCO_RELEASE=latest
 ARG GOSU_RELEASE=latest

Dockerfile.from

@@ -1 +1 @@
-FROM ${_cops_BASE}:${_cops_VERSION}
+FROM ${_cops_BASE}:${_cops_VERSION} AS final

Dockerfile.labels

@@ -1 +1,2 @@
+ARG DOCKER_IMAGES_COMMIT="$DOCKER_IMAGES_COMMIT"
 LABEL com.github.corpusops.docker-images-commit="$DOCKER_IMAGES_COMMIT"

Dockerfile.squash

@@ -0,0 +1,4 @@
+FROM $RSYNC AS squashed-rsync
+FROM $ANCESTOR AS squashed-ancestor
+ARG ROOTFS="/BASE_ROOTFS_TO_COPY_THAT_WONT_COLLIDE_1234567890"
+ARG PATH="${ROOTFS}_rsync/bin:$PATH"

Dockerfile.squashexec

@@ -0,0 +1,4 @@
+SHELL ["busybox",  "sh", "-c"]
+RUN --mount=type=bind,from=final,target=$ROOTFS --mount=type=bind,from=squashed-rsync,target=${ROOTFS}_rsync \
+rsync -Aaz --delete ${ROOTFS}/ / --exclude=/proc --exclude=/sys --exclude=/etc/resolv.conf --exclude=/etc/hosts --exclude=$ROOTFS* --exclude=dev/shm --exclude=dev/pts --exclude=dev/mqueue
+SHELL ["/bin/sh", "-c"]

Dockerfile.squashpre

@@ -0,0 +1,2 @@
+ARG ANCESTOR=${_cops_BASE}:${_cops_VERSION}
+ARG RSYNC=corpusops/rsync

helpers/cops_pkgmgr_install.sh

@@ -1,7 +1,7 @@
 #!/usr/bin/env sh
 # BEGIN: corpusops common glue
 readlinkf() {
-    if ( uname | egrep -iq "darwin|bsd" );then
+    if ( uname | grep -E -iq "darwin|bsd" );then
         if ( which greadlink 2>&1 >/dev/null );then
             greadlink -f "$@"
         elif ( which perl 2>&1 >/dev/null );then
@@ -36,6 +36,7 @@ SYSTEM_COPS_ROOT=${SYSTEM_COPS_ROOT-$DEFAULT_COPS_ROOT}
 DOCKER_COPS_ROOT=${DOCKER_COPS_ROOT-$SYSTEM_COPS_ROOT}
 COPS_URL=${COPS_URL-$DEFAULT_COPS_URL}
 BASE_PREPROVISION_IMAGES="ubuntu:latest_preprovision"
+BASE_PREPROVISION_IMAGES="$BASE_PREPROVISION_IMAGES corpusops/ubuntu:24.04_preprovision"
 BASE_PREPROVISION_IMAGES="$BASE_PREPROVISION_IMAGES corpusops/ubuntu:22.04_preprovision"
 BASE_PREPROVISION_IMAGES="$BASE_PREPROVISION_IMAGES corpusops/ubuntu:20.04_preprovision"
 BASE_PREPROVISION_IMAGES="$BASE_PREPROVISION_IMAGES corpusops/ubuntu:18.04_preprovision"
@@ -44,6 +45,7 @@ BASE_PREPROVISION_IMAGES="$BASE_PREPROVISION_IMAGES corpusops/ubuntu:14.04_prepr
 BASE_PREPROVISION_IMAGES="$BASE_PREPROVISION_IMAGES corpusops/centos:7_preprovision"
 
 BASE_CORE_IMAGES="$BASE_CORE_IMAGES corpusops/ubuntu:latest"
+BASE_CORE_IMAGES="$BASE_CORE_IMAGES corpusops/ubuntu:24.04"
 BASE_CORE_IMAGES="$BASE_CORE_IMAGES corpusops/ubuntu:22.04"
 BASE_CORE_IMAGES="$BASE_CORE_IMAGES corpusops/ubuntu:20.04"
 BASE_CORE_IMAGES="$BASE_CORE_IMAGES corpusops/ubuntu:18.04"
@@ -54,14 +56,18 @@ BASE_IMAGES="$BASE_PREPROVISION_IMAGES $BASE_CORE_IMAGES"
 EXP_PREPROVISION_IMAGES=""
 EXP_PREPROVISION_IMAGES="$EXP_PREPROVISION_IMAGES archlinux:latest_preprovision"
 EXP_PREPROVISION_IMAGES="$EXP_PREPROVISION_IMAGES debian:latest_preprovision"
-EXP_PREPROVISION_IMAGES="$EXP_PREPROVISION_IMAGES debian:stretch_preprovision"
+#EXP_PREPROVISION_IMAGES="$EXP_PREPROVISION_IMAGES debian:stretch_preprovision"
+EXP_PREPROVISION_IMAGES="$EXP_PREPROVISION_IMAGES debian:bookworm_preprovision"
+EXP_PREPROVISION_IMAGES="$EXP_PREPROVISION_IMAGES debian:bullseye_preprovision"
 EXP_PREPROVISION_IMAGES="$EXP_PREPROVISION_IMAGES debian:buster_preprovision"
 EXP_PREPROVISION_IMAGES="$EXP_PREPROVISION_IMAGES debian:sid_preprovision"
 EXP_CORE_IMAGES=""
 EXP_CORE_IMAGES="$EXP_CORE_IMAGES corpusops/archlinux:latest"
 EXP_CORE_IMAGES="$EXP_CORE_IMAGES corpusops/debian:latest"
-EXP_CORE_IMAGES="$EXP_CORE_IMAGES corpusops/debian:stretch"
+#EXP_CORE_IMAGES="$EXP_CORE_IMAGES corpusops/debian:stretch"
+EXP_CORE_IMAGES="$EXP_CORE_IMAGES corpusops/debian:bullseye"
 EXP_CORE_IMAGES="$EXP_CORE_IMAGES corpusops/debian:buster"
+EXP_CORE_IMAGES="$EXP_CORE_IMAGES corpusops/debian:bookworm"
 EXP_CORE_IMAGES="$EXP_CORE_IMAGES corpusops/debian:sid"
 EXP_IMAGES="$EXP_PREPROVISION_IMAGES $EXP_CORE_IMAGES"
 # ansible related
@@ -77,7 +83,7 @@ LOGGER_NAME=${LOGGER_NAME:-corpusops_build}
 ERROR_MSG="There were errors"
 is_container() {
     if ( grep -q container= /proc/1/environ 2>/dev/null ) \
-       || ( egrep -q 'docker|lxc' /proc/1/cgroup 2>/dev/null ) \
+       || ( grep -E -q 'docker|lxc' /proc/1/cgroup 2>/dev/null ) \
        || [ -e /.dockerenv ];then
            return 0
     fi
@@ -266,12 +272,12 @@ version_lt() { [ "$1" = "$2" ] && return 1 || version_lte $1 $2; }
 version_gte() { [  "$2" = "$(printf "$1\n$2" | sort -V | head -n1)" ]; }
 version_gt() { [ "$1" = "$2" ] && return 1 || version_gte $1 $2; }
 lowcase_distribid() { echo $DISTRIB_ID| awk '{print tolower($0)}'; }
-is_archlinux_like() { echo $DISTRIB_ID | egrep -iq "archlinux|arch"; }
-is_debian_like() { echo $DISTRIB_ID | egrep -iq "debian|ubuntu|mint"; }
-is_suse_like() { echo $DISTRIB_ID | egrep -iq "suse"; }
-is_alpine_like() { echo $DISTRIB_ID | egrep -iq "alpine" || test -e /etc/alpine-release; }
+is_archlinux_like() { echo $DISTRIB_ID | grep -E -iq "archlinux|arch"; }
+is_debian_like() { echo $DISTRIB_ID | grep -E -iq "debian|ubuntu|mint"; }
+is_suse_like() { echo $DISTRIB_ID | grep -E -iq "suse"; }
+is_alpine_like() { echo $DISTRIB_ID | grep -E -iq "alpine" || test -e /etc/alpine-release; }
 is_redhat_like() { echo $DISTRIB_ID \
-        | egrep -iq "((^ol$)|rhel|redhat|red-hat|centos|fedora)"; }
+        | grep -E -iq "((^ol$)|rhel|redhat|red-hat|centos|fedora|amzn)"; }
 set_lang() { locale=${1:-C};export LANG=${locale};export LC_ALL=${locale}; }
 is_darwin () {
     if [ "x${FORCE_DARWIN-}" != "x" ];then return 0;fi
@@ -320,6 +326,10 @@ detect_os() {
         if [ $DISTRIB_MAJOR  -eq 7 ];then DISTRIB_CODENAME="wheezy";fi
         if [ $DISTRIB_MAJOR  -eq 8 ];then DISTRIB_CODENAME="jessie";fi
         if [ $DISTRIB_MAJOR  -eq 9 ];then DISTRIB_CODENAME="stretch";fi
+        if [ $DISTRIB_MAJOR  -eq 10 ];then DISTRIB_CODENAME="buster";fi
+        if [ $DISTRIB_MAJOR  -eq 11 ];then DISTRIB_CODENAME="bullseye";fi
+        if [ $DISTRIB_MAJOR  -eq 12 ];then DISTRIB_CODENAME="bookworm";fi
+        if [ $DISTRIB_MAJOR  -eq 13 ];then DISTRIB_CODENAME="trixie";fi
     elif [ -e /etc/SuSE-brand ] || [ -e /etc/SuSE-release ];then
         for i in /etc/SuSE-brand /etc/SuSE-release;do
             if [ -e $i ];then
@@ -405,8 +415,8 @@ may_sudo() {
 get_ancestor_from_dockerfile() {
     local dockerfile=${1}
     local ancestor=
-    if [ -e "${dockerfile}" ] && egrep -q ^FROM "${dockerfile}"; then
-        ancestor=$(egrep ^FROM "${dockerfile}"\
+    if [ -e "${dockerfile}" ] && grep -E -q ^FROM "${dockerfile}"; then
+        ancestor=$(grep -E ^FROM "${dockerfile}"\
             | head -n1 | awk '{print $2}' | xargs -n1| sort -u )
     fi
     echo ${ancestor}
@@ -463,7 +473,7 @@ upgrade_wd_to_br() {
         if [ "x${test_branch}" != "x${up_branch}" ];then
             warn "Upgrading $wd to branch: $up_branch"
             git fetch --all || die "git fetch in $wd failed"
-            if get_git_branchs | egrep -q "^${up_branch}$";then
+            if get_git_branchs | grep -E -q "^${up_branch}$";then
                 vv git checkout ${up_branch} &&\
                     vv git reset --hard origin/${up_branch}
             else
@@ -496,7 +506,7 @@ get_python_() {
     local py_bins="$@"
     for i in $py_bins;do
         local lpy=$(get_command $i 2>/dev/null)
-        if [ "x$lpy" != "x" ] && ( ${lpy} -V 2>&1| egrep -qi "python $py_ver" );then
+        if [ "x$lpy" != "x" ] && ( ${lpy} -V 2>&1| grep -E -qi "python $py_ver" );then
             selectedpy=${lpy}
             break
         fi
@@ -512,8 +522,8 @@ get_python2() {
 get_python3() {
     local py_ver=3
     get_python_ $py_ver \
-        python3.9  python3.8  python3.7  python3.6  python3.5  python3.4  \
-        python-3.9 python-3.8 python-3.7 python-3.6 python-3.5 python-3.4 \
+        python3.12  python3.11  python3.10  python3.9  python3.8  python3.7  python3.6  python3.5  python3.4  \
+        python-3.12 python-3.11 python-3.10 python-3.9 python-3.8 python-3.7 python-3.6 python-3.5 python-3.4 \
         python-${py_ver} python${py_ver} python
 }
 has_python_module() {
@@ -532,9 +542,19 @@ pymod_ver() {
 get_setuptools() {
     local py=${1:-python}
     local setuptoolsreq="setuptools"
-    if ( is_python2 $py );then setuptoolsreq="setuptools<=45"; else setuptoolsreq="setuptools<50"; fi
+    local cpyver=$($py -c "import sys;print(sys.version.split()[0])")
+    if ( is_python2 $py );then
+        setuptoolsreq="setuptools<=45"
+    elif ( version_lt $cpyver 3.12.0 );then
+        setuptoolsreq="setuptools<66"
+    else
+        setuptoolsreq="setuptools>=75"
+    fi
     echo "$setuptoolsreq"
 }
+setup_setuptools_requirement() {
+    sed -i -re "s/^setuptools\s*(>|<|=|$)/$(get_setuptools $py)/g" requirements/python_requirements.txt
+}
 install_pip() {
     local py="${1:-python}"
     local DEFAULT_PIP_URL="https://bootstrap.pypa.io/get-pip.py"
@@ -754,8 +774,8 @@ ensure_command() {
 ### archlinux (pacman)
 is_pacman_available() {
     for i in $@;do
-        if ! ( pacman -Si $(i_y) "$i" >/devnull 2>&1 ||\
-                pacman -Sg $(i_y) "$i" >/devnull 2>&1; );then
+        if ! ( pacman -Si $(i_y) "$i" >/dev/null 2>&1 ||\
+                pacman -Sg $(i_y) "$i" >/dev/null 2>&1; );then
             return 1
         fi
     done
@@ -764,7 +784,7 @@ is_pacman_available() {
 
 is_pacman_installed() {
     for i in $@;do
-        if ! ( pacman -Qi $(i_y) "$i" >/devnull 2>&1; ); then
+        if ! ( pacman -Qi $(i_y) "$i" >/dev/null 2>&1; ); then
             return 1
         fi
     done
@@ -786,7 +806,7 @@ pacman_install() {
 pacman_setup() {
     ensure_command awk core/gawk
     ensure_command sort core/coreutils
-    ensure_command egrep core/grep
+    ensure_command grep -E core/grep
     ensure_command which core/which
 }
 
@@ -799,7 +819,7 @@ microdnf_repoquery() {
 is_microdnf_available() {
     pkgs="$(microdnf repoquery --available)"
     for i in $@;do
-        if ! ( echo "$pkgs" | egrep -iq "^${i}" ; ); then
+        if ! ( echo "$pkgs" | grep -E -iq "^${i}" ; ); then
             return 1
         fi
     done
@@ -808,7 +828,7 @@ is_microdnf_available() {
 is_microdnf_installed() {
     pkgs="$(microdnf repoquery --installed)"
     for i in $@;do
-        if ! ( echo "$pkgs" | egrep -iq "^${i}" ; ); then
+        if ! ( echo "$pkgs" | grep -E -iq "^${i}" ; ); then
             return 1
         fi
     done
@@ -817,7 +837,7 @@ is_microdnf_installed() {
 microdnf_update() {
     vvv microdnf repoquery $(i_y) --refresh --available --installed >/dev/null
     ret=$?
-    if echo ${ret} | egrep -q '^(0|100)$'; then
+    if echo ${ret} | grep -E -q '^(0|100)$'; then
         return 0
     fi
     return 1
@@ -864,7 +884,7 @@ is_dnf_installed() {
 dnf_update() {
     vvv dnf check-update $(i_y)
     ret=$?
-    if echo ${ret} | egrep -q '^(0|100)$'; then
+    if echo ${ret} | grep -E -q '^(0|100)$'; then
         return 0
     fi
     return 1
@@ -915,7 +935,7 @@ is_yum_installed() {
 yum_update() {
     vvv yum check-update $(i_y)
     ret=$?
-    if echo ${ret} | egrep -q '^(0|100)$'; then
+    if echo ${ret} | grep -E -q '^(0|100)$'; then
         return 0
     fi
     return 1
@@ -967,7 +987,7 @@ rh_setup() {
     ensure_command xargs findutils
     ensure_command awk gawk
     ensure_command sort coreutils
-    ensure_command egrep grep
+    ensure_command grep -E grep
     ensure_command which which
 }
 
@@ -986,7 +1006,7 @@ is_aptget_available() {
 }
 
 is_aptget_installed() {
-    if ! dpkg-query -s ${@} 2>/dev/null|egrep "^Status:"|grep -q installed; then
+    if ! dpkg-query -s ${@} 2>/dev/null|grep -E "^Status:"|grep -q installed; then
         return 1
     fi
 }
@@ -1080,7 +1100,7 @@ is_zypper_available() {
 }
 
 is_zypper_installed() {
-    if ( $(zyppern) info $@|egrep -iq "installed:?\s.*no" ); then
+    if ( $(zyppern) info $@|grep -E -iq "installed:?\s.*no" ); then
         return 1
     fi
     return 0