fixing artifact extraction and removing env from Dockerfile

CHANGELOG.md

@@ -14,6 +14,7 @@ and **Merged pull requests**. Critical items to know are:
 The versions coincide with releases on pip. Only major versions will be released as tags on Github.
 
 ## [0.0.x](https://github.scom/syspack/pakages/tree/main) (0.0.x)
+ - refactoring artifact extraction to allow reuse (0.0.14)
  - Ensuring pakages pushes a generic name too (0.0.13)
  - More control over custom push/pull registries and settings (0.0.12)
  - Added support for building local path or GitHub remote (0.0.11)

Dockerfile

@@ -1,14 +1,37 @@
-ARG ubuntu_version
-FROM ghcr.io/rse-ops/ubuntu:$ubuntu_version
+ARG ubuntu_version=20.04
+FROM ubuntu:$ubuntu_version
+ARG CMAKE=3.20.4
 
 # docker build --build-arg ubuntu_version=20.04 -t ghcr.io/syspack/pakages-ubuntu-20.04 .
 
-# Convert to shallow clone (smaller container)
-RUN rm -rf /opt/spack && \
-   git clone --depth 1 https://github.com/spack/spack /opt/spack
-
 # Build all software with debug flags!
 ENV SPACK_ADD_DEBUG_FLAGS=true
+ENV PATH=/opt/spack/bin:$PATH
+
+# Ubuntu dependencies
+RUN apt-get update && apt-get install -y wget && \
+    wget https://raw.githubusercontent.com/rse-ops/docker-images/main/scripts/ubuntu/apt-install-defaults-plus-args.sh && \
+    chmod +x apt-install-defaults-plus-args.sh && \
+    ./apt-install-defaults-plus-args.sh && \
+    rm apt-install-defaults-plus-args.sh
+
+# Install cmake binary
+RUN curl -s -L https://github.com/Kitware/CMake/releases/download/v$CMAKE/cmake-$CMAKE-linux-x86_64.sh > cmake.sh && \
+    sh cmake.sh --prefix=/usr/local --skip-license && \
+    rm cmake.sh
+
+# And spack deps
+RUN git clone --depth 1 https://github.com/spack/spack /opt/spack && \
+    python3 -m pip install --upgrade pip && \
+    python3 -m pip install clingo && \
+    spack external find && \
+    spack external find python perl binutils git tar xz bzip2 && \
+
+    # build for a generic target
+    spack config add 'packages:all:target:[x86_64]' && \
+
+    # reuse as much as possible, make externals useful
+    spack config add 'concretizer:reuse:true'
 
 # install oras so we don't need to bootstrap
 RUN curl -LO https://github.com/oras-project/oras/releases/download/v0.12.0/oras_0.12.0_linux_amd64.tar.gz && \

pakages/install.py

@@ -4,6 +4,7 @@
 
 import os
 import shutil
+import tarfile
 
 import pakages.oras
 import pakages.sbom
@@ -110,9 +111,10 @@ def prepare_cache(self, registries=None, tag=None):
                     logger.info(f"Extracting archive {artifact}...")
 
                     # Note - for now not signing, since we don't have a consistent key strategy
-                    bd.extract_tarball(request.pkg.spec, artifact, unsigned=True)
-                    spack.hooks.post_install(request.pkg.spec)
-                    spack.store.db.add(request.pkg.spec, spack.store.layout)
+                    # The spack function is a hairball that doesn't respect the provided filename
+                    spec = extract_tarball(request.pkg.spec, artifact)
+                    spack.hooks.post_install(spec)
+                    spack.store.db.add(spec, spack.store.layout)
 
     builder = PakInstaller([(self, kwargs)])
 
@@ -126,3 +128,74 @@ def prepare_cache(self, registries=None, tag=None):
         sbom = pakages.sbom.generate_sbom(self.spec)
         sbom_file = os.path.join(meta_dir, "sbom.json")
         pakages.utils.write_json(sbom, sbom_file)
+
+
+def extract_tarball(spec, filename):
+    """
+    extract binary tarball for given package into install area
+    """
+    # The spec prefix is an easy way to get the directory name
+    extract_dir = os.path.dirname(spec.prefix)
+    from_dir = os.path.dirname(filename)
+    name = os.path.basename(filename)
+    dag_hash = name.split("-")[-1].replace(".spack", "")
+
+    # Assemble the new spec prefix
+    prefix = os.path.join(extract_dir, f"{spec.name}-{spec.version}-{dag_hash}")
+    if os.path.exists(spec.prefix):
+        shutil.rmtree(spec.prefix)
+
+    tmpdir = pakages.utils.get_tmpdir()
+    with tarfile.open(filename, "r") as tar:
+        tar.extractall(tmpdir)
+
+    # Find the json file and .tar.gz
+    spec_file = None
+    targz = None
+    for path in os.listdir(tmpdir):
+        if path.endswith(".json"):
+            spec_file = os.path.join(tmpdir, path)
+        elif path.endswith("gz"):
+            targz = os.path.join(tmpdir, path)
+
+    if not spec_file:
+        shutil.rmtree(tmpdir)
+        shutil.rmtree(from_dir)
+        logger.exit(f"{spec} did not come with a spec json.")
+
+    new_prefix = str(os.path.relpath(prefix, spack.store.layout.root))
+    content = pakages.utils.read_json(spec_file)
+
+    # if the original relative prefix is in the spec file use it
+    buildinfo = content.get("buildinfo", {})
+    relative_prefix = buildinfo.get("relative_prefix", new_prefix)
+
+    extract_tmp = os.path.join(spack.store.layout.root, ".tmp")
+    pakages.utils.mkdirp(extract_tmp)
+    extracted_dir = os.path.join(extract_tmp, relative_prefix.split(os.path.sep)[-1])
+
+    with tarfile.open(targz, "r") as tar:
+        tar.extractall(path=extract_tmp)
+    try:
+        shutil.move(extracted_dir, prefix)
+    except Exception as e:
+        shutil.rmtree(extracted_dir)
+        raise e
+
+    # Clean up
+    os.remove(targz)
+    os.remove(spec_file)
+
+    # Create a dummy spec to do the relocation
+    new_spec = spack.spec.Spec.from_dict({"spec": content["spec"]})
+
+    try:
+        bd.relocate_package(new_spec, False)
+    except Exception as e:
+        shutil.rmtree(spec.prefix)
+        raise e
+    finally:
+        shutil.rmtree(tmpdir)
+        if os.path.exists(filename):
+            os.remove(filename)
+    return new_spec

pakages/oras.py

@@ -77,13 +77,18 @@ def fetch(self, url, save_file):
         """
         # We don't have programmatic access to list, so we just try to pull
         logger.info("Fetching oras {0}".format(url))
-
+        save_dir = os.path.dirname(save_file)
         try:
-            self.oras("pull", url, "-a", "--output", os.path.dirname(save_file))
+            self.oras("pull", url, "-a", "--output", save_dir)
         except:
             logger.info("{0} is not available for pull.".format(url))
             return
 
+        # Files are technically saved with the hash, we just hope spack will use
+        files = os.listdir(save_dir)
+        if len(files) > 0:
+            save_file = os.path.join(save_dir, files[0])
+
         # Return the file if exists
         if os.path.exists(save_file):
             return save_file

pakages/spec.py

@@ -3,6 +3,7 @@
 __license__ = "Apache-2.0"
 
 import spack.spec
+from pakages.logger import logger
 import spack.util.string
 import pakages.repo
 import six
@@ -27,13 +28,16 @@ def parse(string):
 
 
 def parse_specs(packages):
-    """Parse specs from a list of strings, and concretize"""
+    """
+    Parse specs from a list of strings, and concretize
+    """
     if not isinstance(packages, six.string_types):
         packages = " ".join(spack.util.string.quote(packages))
 
     specs = []
     for legacy in spack.spec.SpecParser().parse(packages):
 
+        logger.info(f"Preparing spec for {legacy}")
         # Create a new Pak spec to copy (duplicate) into
         spec = Spec()
         spec._dup(legacy)

pakages/version.py

@@ -2,7 +2,7 @@
 __copyright__ = "Copyright 2021-2022, Vanessa Sochat and Alec Scott"
 __license__ = "Apache-2.0"
 
-__version__ = "0.0.13"
+__version__ = "0.0.14"
 AUTHOR = "Vanessa Sochat, Alec Scott"
 NAME = "pakages"
 PACKAGE_URL = "https://github.com/syspack/pakages"