tweaks to improve install

CHANGELOG.md

@@ -14,6 +14,7 @@ and **Merged pull requests**. Critical items to know are:
 The versions coincide with releases on pip. Only major versions will be released as tags on Github.
 
 ## [0.0.x](https://github.scom/syspack/pakages/tree/main) (0.0.x)
+ - tweaking install process (0.0.15)
  - refactoring artifact extraction to allow reuse (0.0.14)
    - adding exponential backoff and 5 retries for oras push
  - Ensuring pakages pushes a generic name too (0.0.13)

pakages/client.py

@@ -110,10 +110,7 @@ def install(self, packages, registry=None, tag=None):
         for spec in self.iter_specs(packages):
             logger.info("Preparing to install %s" % spec.name)
 
-            # TODO here we actually want to insert preparing GitHub packages build cache
-            # We should match based on a basic set of architectures we know the containers support
-            # E.g., check the platform and spit an error if it's some niche HPC one.
-
+            # Do install - this dumps out matching from build cache first
             spec.package.do_install(
                 force=True,
                 registries=registries,

pakages/install.py

@@ -107,6 +107,13 @@ def prepare_cache(self, registries=None, tag=None):
             if not self.build_requests:
                 return
 
+            # We will update build requests with specs we find
+            # This is a loose matching, we just care about name and version for now
+            requests = {
+                "%s:%s" % (x.pkg.spec.name, x.pkg.spec.version): x
+                for x in self.build_requests
+            }
+
             # If we want to use Github packages API, it requires token with package;read scope
             # https://docs.github.com/en/rest/reference/packages#list-packages-for-an-organization
             for pkg_id, request in self._pakages_tasks.items():
@@ -142,23 +149,41 @@ def prepare_cache(self, registries=None, tag=None):
 
                 # If we have an artifact, extract where needed and tell spack it's installed!
                 if artifact:
-                    logger.info(f"Extracting archive {artifact}...")
-
                     # Note - for now not signing, since we don't have a consistent key strategy
                     # The spack function is a hairball that doesn't respect the provided filename
                     spec = extract_tarball(request.pkg.spec, artifact)
                     if not spec:
                         continue
-                    # Update the build request
-                    #                    request.pkg.spec = spec
-                    #                   request.pkg_id = spack.installer.package_id(spec.package)
-                    #                  request.dependencies = {f"{x.name}-{x.version}-{x._hash}" for x in spec.dependencies()}
-                    #                 self.build_requests[i] = request
+
+                    # Remove the build request if we hit it. Note that this
+                    # might fail if dependencies are still needed (and not hit)
+                    # I haven't tested it yet
+                    spec_id = f"{spec.name}:{spec.version}"
+                    if spec_id in requests:
+
+                        # update the build request
+                        requests[spec_id].pkg.spec = spec
+                        requests[spec_id].pkg_id = spack.installer.package_id(
+                            spec.package
+                        )
+                        requests[spec_id].dependencies = {
+                            f"{x.name}-{x.version}-{x._hash}"
+                            for x in spec.dependencies()
+                        }
+
+                        # And flag the task as installed
+                        task = requests[spec_id]
+                        requests[spec_id].task = spack.installer.STATUS_INSTALLED
+                        self._flag_installed(task.pkg, spec.dependents())
 
                     # And finish this piece of the install
+                    spec.package.installed_from_binary_cache = True
                     spack.hooks.post_install(spec)
                     spack.store.db.add(spec, spack.store.layout)
 
+            # Update build requests
+            self.build_requests = list(requests.values())
+
     builder = PakInstaller([(self, kwargs)])
 
     # Download what we can find from the GitHub cache
@@ -184,16 +209,7 @@ def extract_tarball(spec, filename):
 
     # Anything installed from system
     if spack.store.layout.root not in spec.prefix:
-        extract_dir = spack.store.layout.root
-
-        # ['linux', 'ubuntu20.04', 'x86_64', 'gcc', '9.4.0', 'bzip2', '1.0.8']
-        parts = os.path.basename(filename).split("-")[:-1]
-        prefix = os.path.join(
-            extract_dir,
-            f"{parts[0]}-{parts[1]}-{parts[2]}",
-            f"{parts[3]}-{parts[4]}",
-            f"{spec.name}-{spec.version}-{dag_hash}",
-        )
+        return
     else:
         extract_dir = os.path.dirname(spec.prefix)
         prefix = os.path.join(extract_dir, f"{spec.name}-{spec.version}-{dag_hash}")
@@ -245,6 +261,7 @@ def extract_tarball(spec, filename):
 
     # Create a dummy spec to do the relocation
     new_spec = spack.spec.Spec.from_dict({"spec": content["spec"]})
+    new_spec._prefix = prefix
 
     try:
         bd.relocate_package(new_spec, False)

pakages/oras.py

@@ -57,7 +57,7 @@ def get_manifest_digest(self, uri):
             if digest:
                 return digest.replace("sha256:", "")
 
-    def push(self, uri, push_file, content_type=None, retry=5, sleep=1):
+    def push(self, uri, push_file, content_type=None, retry=3, sleep=1):
         """
         Push an oras artifact to an OCI registry
         """
@@ -91,14 +91,16 @@ def fetch(self, url, save_file):
         Fetch an oras artifact from an OCI registry
         """
         # We don't have programmatic access to list, so we just try to pull
-        logger.info("Fetching oras {0}".format(url))
         save_dir = os.path.dirname(save_file)
         try:
-            self.oras("pull", url, "-a", "--output", save_dir)
+            logger.debug("Trying fetch for oras {0}".format(url))
+            self.oras("pull", url, "-a", "--output", save_dir, output=str, error=str)
         except:
-            logger.info("{0} is not available for pull.".format(url))
+            logger.debug("{0} is not available for pull.".format(url))
             return
 
+        # Just print those that are successful
+        logger.info("Fetched oras {0}".format(url))
         # Files are technically saved with the hash, we just hope spack will use
         files = os.listdir(save_dir)
         if len(files) > 0:

pakages/spec.py

@@ -6,7 +6,6 @@
 from pakages.logger import logger
 import spack.util.string
 import pakages.repo
-import six
 
 
 class Spec(spack.spec.Spec):
@@ -31,16 +30,18 @@ def parse_specs(packages):
     """
     Parse specs from a list of strings, and concretize
     """
-    if not isinstance(packages, six.string_types):
-        packages = " ".join(spack.util.string.quote(packages))
+    if not isinstance(packages, list):
+        packages = list(packages)
 
     specs = []
-    for legacy in spack.spec.SpecParser().parse(packages):
+    for legacy in packages:
 
         logger.info(f"Preparing spec for {legacy}")
+
         # Create a new Pak spec to copy (duplicate) into
+        spack_spec = spack.spec.Spec(legacy)
         spec = Spec()
-        spec._dup(legacy)
+        spec._dup(spack_spec)
 
         # Always set the arch to be general
         spec.architecture = spack.spec.ArchSpec()

pakages/version.py

@@ -2,7 +2,7 @@
 __copyright__ = "Copyright 2021-2022, Vanessa Sochat and Alec Scott"
 __license__ = "Apache-2.0"
 
-__version__ = "0.0.14"
+__version__ = "0.0.15"
 AUTHOR = "Vanessa Sochat, Alec Scott"
 NAME = "pakages"
 PACKAGE_URL = "https://github.com/syspack/pakages"