Merge pull request #33 from CSCfi/auto_base_os

Features

Author: Nortamo

configs/local.yaml

@@ -9,7 +9,7 @@ defaults:
     installation_path: "/LOCAL_TYKKY"
     # if this is not a thing which exist
     # I will do a singularity pull
-    container_src: 'docker://rockylinux:8.9'
+    container_src: auto
     # name of the container image when on disk
     container_image: container.sif 
     sqfs_image: img.sqfs 

configs/lumi.yaml

@@ -12,7 +12,7 @@ defaults:
     composable: true
     # if this is not a thing which exist
     # I will do a singularity pull
-    container_src: 'docker://opensuse/leap:15.5'
+    container_src: auto
     # name of the container image when on disk
     container_image: container.sif 
     sqfs_image: img.sqfs 

configs/mahti.yaml

@@ -12,7 +12,7 @@ defaults:
     composable: true
     # if this is not a thing which exist
     # I will do a singularity pull
-    container_src: 'docker://redhat/ubi8:8.10' 
+    container_src: auto
     # name of the container image when on disk
     container_image: container.sif 
     sqfs_image: img.sqfs 

configs/puhti.yaml

@@ -11,7 +11,7 @@ defaults:
     composable: true
     # if this is not a thing which exist
     # I will do a singularity pull
-    container_src: 'docker://redhat/ubi8:8.6' 
+    container_src: auto 
     # name of the container image when on disk
     container_image: container.sif 
     sqfs_image: img.sqfs 

construct.py

@@ -57,6 +57,15 @@
 if os.getenv("CW_LOG_LEVEL"):
     full_conf["log_level"]=os.getenv("CW_LOG_LEVEL")
 
+if full_conf["container_src"] == "auto":
+    success,container_source = get_docker_image("/etc/os-release")
+    if not success:
+        print_err("Unable to automatically determine base container to use: "+container_source,True)
+        sys.exit(1) 
+    else:
+        print_info("Automatically determined base container to: "+container_source,full_conf["log_level"],2,True)
+    full_conf["container_src"] = f"docker://{container_source}"
+
 if os.getenv("CW_BUILD_TMPDIR"):
     full_conf["build_tmpdir_base"]=os.getenv("CW_BUILD_TMPDIR")
 tmpdir_base=full_conf["build_tmpdir_base"]

cw_common.py

@@ -12,6 +12,7 @@
 colors["GREEN"]='\033[0;32m'
 colors["YELLOW"]='\033[1;33m'
 colors["BLUE"]="\033[1;34m"
+colors["PURPLE"]='\033[0;35m'
 colors["NC"]='\033[0m' # No Color
 
 def print_err(txt,err=False):
@@ -27,6 +28,28 @@ def print_err(txt,err=False):
             print("[ ERROR ] "+txt)
         else:
             print("["+colors["RED"]+" ERROR "+colors["NC"]+"] "+txt)
+
+def print_info(txt,log_level,msg_level,err=False):
+    """Pretty info message, color is disabled if not in a TTY"""
+    if int(log_level) <= msg_level:
+        return
+    if msg_level >= 2:
+        msg="DEBUG"
+        color=colors["PURPLE"]
+    else:
+        msg="INFO"
+        color=colors["BLUE"]
+    if(err):
+        if not sys.stderr.isatty():
+            print(f"[ {msg} ] "+txt,file=sys.stderr)
+        else:
+            print("["+color+f" {msg} "+colors["NC"]+"] "+txt,file=sys.stderr)
+    else:
+        if not sys.stdout.isatty():
+            print(f"[ {msg} ] "+txt,file=sys.stderr)
+        else:
+            print("["+color+f" {msg} "+colors["NC"]+"] "+txt,file=sys.stdout)
+
 def print_warn(txt,err=False):
     if(err):
         if not sys.stderr.isatty():
@@ -65,3 +88,49 @@ def installation_in_PATH():
 def is_installation(base_path):
     markers=["bin","_bin","common.sh"]
     return all( pathlib.Path(base_path+'/../'+m).exists() for m in markers )
+
+# UBI images are namespaced with the major version as part of the name
+# and not just the tag.
+special={}
+special["rhel"]= lambda namespace,version: namespace+version.split('.')[0]
+
+# Get the docker image matching the host OS
+def get_docker_image(release_file):
+    os_release_file = release_file
+    docker_images = {
+        "sles": "opensuse/leap",
+        "rhel": "redhat/ubi",
+        "almalinux": "almalinux",
+        "rocky": "rockylinux",
+        "ubuntu": "ubuntu"
+    }
+
+    try:
+        with open(os_release_file, 'r') as file:
+            lines = file.readlines()
+            os_info = {}
+            for line in lines:
+                # Lazy way to handle empty lines
+                try:
+                    key, value = line.strip().split('=', 1)
+                except:
+                    continue
+                os_info[key] = value.strip('"')
+
+            os_id = os_info.get("ID", "").lower()
+            version_id = os_info.get("VERSION_ID", "").lower()
+
+            if os_id in docker_images:
+                docker_image = docker_images[os_id]
+                if os_id in special:
+                    docker_image = special[os_id](docker_image,version_id)
+                return (True,f"{docker_image}:{version_id}")
+            else:
+                # Guess what the name could be
+                # Will most likely fail for most small distros
+                return (True,f"{os_id}:{version_id}")
+
+    except FileNotFoundError:
+        return (False,"OS release file not found")
+    except Exception as e:
+        return (False,f"An error occurred: {e}")

frontends/containerize

@@ -23,7 +23,7 @@ clean_up () {
     else
         test -f "$_usr_yaml" && rm "$_usr_yaml"
         test -d "$CW_BUILD_TMPDIR" && rm -r "$CW_BUILD_TMPDIR"
-        print_err "Set CW_DEBUG_KEEP_FILES env variable to keep build files"
+        print_err "Set CW_DEBUG_KEEP_FILES env variable to keep build files or set CW_LOG_LEVEL to a higher value for more output, e.g. CW_LOG_LEVEL=3"
     fi
     trap 2
     kill  -- -$$ &>/dev/null
@@ -90,8 +90,10 @@ else
    print_warn "Umask set to $CW_UMASK, check permissions of finished installation" 
 fi
 
-chgrp $(stat -c "%G" $CW_INSTALLATION_PREFIX ) $CW_BUILD_TMPDIR
-chmod g+s $CW_BUILD_TMPDIR
+_install_prefix_group=$(stat -c "%G" $CW_INSTALLATION_PREFIX )
+chgrp $_install_prefix_group $CW_BUILD_TMPDIR || print_warn "Failed to match group ownership of installation directory, attempted group was $_install_prefix_group"
+chmod g+s $CW_BUILD_TMPDIR || print_warn "Failed to set group stickybit for build directory, installation permissions might be incorrect. Please check once the installation is done."
+print_info "Setting group ownership of build directory to $_install_prefix_group" 2 
 
 if [[ "$CW_INSTALLATION_PREFIX" = /* ]]; then
    export _inst_path=$CW_INSTALLATION_PREFIX

generate_wrappers.sh

@@ -216,12 +216,17 @@ if [[ \${_CW_IN_CONTAINER+defined} ]];then
 else" >> _deploy/bin/$target
         if [[ ${CONDA_CMD+defined} ]];then
         echo "
-        if [[ ( -e \$(/usr/bin/dirname \$_O_SOURCE )/../pyvenv.cfg && ! \${CW_FORCE_CONDA_ACTIVATE+defined} ) || \${CW_NO_CONDA_ACTIVATE+defined} ]];then
+        _venvd=\$(/usr/bin/dirname \$_O_SOURCE )
+        test -e \$_venvd/../pyvenv.cfg
+        _v_in_use=\$?
+        if [[ ( \$_v_in_use -eq 0 && ! \${CW_FORCE_CONDA_ACTIVATE+defined} ) || \${CW_NO_CONDA_ACTIVATE+defined} ]];then
         export PATH=\"\$OLD_PATH\"
         $_RUN_CMD $_default_cws exec -a \$_O_SOURCE \$DIR/$target $_cwe  
     else
         export PATH=\"\$OLD_PATH\"
-        $_RUN_CMD  $_cws exec -a \$_O_SOURCE \$DIR/$target $_cwe  
+        _venv_act=\":\"
+        test 0 -eq \$_v_in_use && _venv_act=\"source \$_venvd/activate\" 
+        $_RUN_CMD  $_cws \$_venv_act && exec -a \$_O_SOURCE \$DIR/$target $_cwe  
     fi
 fi
         " >>  _deploy/bin/$target