Implement group=True for group_manifests plugin

atomic_reactor/constants.py

@@ -72,3 +72,4 @@
 PLUGIN_PULP_TAG_KEY = 'pulp_tag'
 PLUGIN_ADD_FILESYSTEM_KEY = 'add_filesystem'
 PLUGIN_FETCH_WORKER_METADATA_KEY = 'fetch_worker_metadata'
+PLUGIN_GROUP_MANIFESTS_KEY = 'group_manifests'

atomic_reactor/plugins/exit_delete_from_registry.py

@@ -184,16 +184,15 @@ def run(self):
             auth = self.setup_secret(registry_noschema, secret_path)
 
             # orchestrator builds use worker_digests
-            if self.handle_worker_digests(worker_digests, registry, insecure,
-                                          auth, deleted_digests):
-                # If we are in orchestrator and found a match, good chance it
-                # will not be in the workflow.push_conf dict.  Just continue.
-                continue
+            orchestrator_delete = self.handle_worker_digests(worker_digests, registry, insecure,
+                                                             auth, deleted_digests)
 
             push_conf_registry = self.find_registry(registry_noschema, self.workflow)
             if not push_conf_registry:
-                self.log.warning("requested deleting image from %s but we haven't pushed there",
-                                 registry_noschema)
+                # only warn if we're not running in the orchestrator
+                if not orchestrator_delete:
+                    self.log.warning("requested deleting image from %s but we haven't pushed there",
+                                     registry_noschema)
                 continue
 
             # worker node and manifests use push_conf_registry

atomic_reactor/plugins/exit_koji_import.py

@@ -16,14 +16,14 @@
 from atomic_reactor.source import GitSource
 from atomic_reactor.plugins.build_orchestrate_build import (get_worker_build_info,
                                                             get_koji_upload_dir)
-from atomic_reactor.plugins.post_fetch_worker_metadata import FetchWorkerMetadataPlugin
 from atomic_reactor.plugins.pre_add_filesystem import AddFilesystemPlugin
 from atomic_reactor.plugins.pre_check_and_set_rebuild import is_rebuild
 from atomic_reactor.constants import (PLUGIN_KOJI_IMPORT_PLUGIN_KEY,
                                       PLUGIN_PULP_PULL_KEY,
-                                      PLUGIN_PULP_SYNC_KEY)
-from atomic_reactor.util import (get_build_json, get_preferred_label,
-                                 df_parser, ImageName)
+                                      PLUGIN_PULP_SYNC_KEY,
+                                      PLUGIN_FETCH_WORKER_METADATA_KEY,
+                                      PLUGIN_GROUP_MANIFESTS_KEY)
+from atomic_reactor.util import (get_build_json, get_preferred_label, df_parser, ImageName)
 from atomic_reactor.koji_util import create_koji_session
 from osbs.conf import Configuration
 from osbs.api import OSBS
@@ -173,6 +173,29 @@ def set_media_types(self, extra, worker_metadatas):
                 extra['image']['media_types'] = json.loads(annotations['media-types'])
                 return
 
+    def set_manifest_list_info(self, extra, worker_metadatas):
+        manifest_list_digests = self.workflow.postbuild_results.get(PLUGIN_GROUP_MANIFESTS_KEY)
+        if manifest_list_digests:
+            index = {}
+            index['tags'] = [image.tag for image in self.workflow.tag_conf.images]
+            repositories = self.workflow.build_result.annotations['repositories']['unique']
+            repo = ImageName.parse(repositories[0]).to_str(registry=False, tag=False)
+            # group_manifests added the registry, so this should be valid
+            registries = self.workflow.push_conf.pulp_registries
+            if not registries:
+                registries = self.workflow.push_conf.all_registries
+            for registry in registries:
+                pullspec = "{0}/{1}@{2}".format(registry.uri, repo,
+                                                manifest_list_digests[0].v2_list)
+                index['pull'] = [pullspec]
+                for tag in index['tags']:
+                    if '-' in tag:  # {version}-{release} only, and only one instance
+                        pullspec = "{0}/{1}:{2}".format(registry.uri, repo, tag)
+                        index['pull'].append(pullspec)
+                        break
+                break
+            extra['image']['index'] = index
+
     def get_build(self, metadata, worker_metadatas):
         start_time = int(atomic_reactor_start_time)
 
@@ -212,8 +235,8 @@ def get_build(self, metadata, worker_metadatas):
                     extra['filesystem_koji_task_id'] = task_id
 
         self.set_help(extra, worker_metadatas)
-
         self.set_media_types(extra, worker_metadatas)
+        self.set_manifest_list_info(extra, worker_metadatas)
 
         build = {
             'name': component,
@@ -237,7 +260,7 @@ def combine_metadata_fragments(self):
 
         metadata_version = 0
 
-        worker_metadatas = self.workflow.postbuild_results.get(FetchWorkerMetadataPlugin.key)
+        worker_metadatas = self.workflow.postbuild_results.get(PLUGIN_FETCH_WORKER_METADATA_KEY)
         build = self.get_build(metadata, worker_metadatas)
         buildroot = self.get_buildroot(worker_metadatas)
         output = self.get_output(worker_metadatas)

atomic_reactor/plugins/post_group_manifests.py

@@ -14,16 +14,20 @@
 from __future__ import unicode_literals
 import requests
 import requests.auth
+from tempfile import NamedTemporaryFile
+import yaml
+from subprocess import check_output, CalledProcessError, STDOUT
 
 from six.moves.urllib.parse import urlparse
 
 from atomic_reactor.plugin import PostBuildPlugin, PluginFailedException
-from atomic_reactor.util import Dockercfg
+from atomic_reactor.util import Dockercfg, get_manifest_digests
+from atomic_reactor.constants import PLUGIN_GROUP_MANIFESTS_KEY
 
 
 class GroupManifestsPlugin(PostBuildPlugin):
-    key = 'group_manifests'
     is_allowed_to_fail = False
+    key = PLUGIN_GROUP_MANIFESTS_KEY
 
     def __init__(self, tasker, workflow, registries, group=True, goarch=None):
         """
@@ -47,6 +51,75 @@ def __init__(self, tasker, workflow, registries, group=True, goarch=None):
         self.registries = registries
         self.worker_registries = {}
 
+    def submit_manifest_list(self, registry, registry_conf, manifest_list_spec):
+        docker_secret_path = registry_conf.get('secret', None)
+        with NamedTemporaryFile(prefix='manifest-list', suffix=".yml", mode='w') as fp:
+            yaml.dump(manifest_list_spec, stream=fp)
+            fp.flush()
+            self.log.debug("Wrote to file %s with config %s", fp.name, docker_secret_path)
+
+            # --docker-cfg may be rendundant here, but it's how the tool should work
+            cmd = ['manifest-tool', '--docker-cfg=%s' % docker_secret_path,
+                   'push', 'from-spec', fp.name]
+            # docker always looks in $HOME for the .dockercfg, so set $HOME to the path
+            try:
+                check_output(cmd, stderr=STDOUT, env={'HOME': docker_secret_path})
+            except CalledProcessError as exc:
+                self.log.error("manifest-tool failed with %s", exc.output)
+                raise
+            self.log.info("Manifest list submitted for %s", registry)
+
+    def get_grouped_manifests(self):
+        grouped_manifests = []
+        for registry, registry_conf in self.registries.items():
+            if registry_conf.get('version') == 'v1':
+                continue
+
+            manifest_list_spec = {}
+            manifest_list_spec['manifests'] = []
+            all_annotations = self.workflow.build_result.annotations['worker-builds']
+            for platform in all_annotations:
+                worker_image = all_annotations[platform]['digests'][0]
+                tag = worker_image['tag']
+                repository = worker_image['repository']
+                arch_entry = {
+                    'image': '{0}/{1}:{2}'.format(registry, repository, tag),
+                    'platform': {
+                        'os': 'linux',
+                        'architecture': self.goarch.get(platform, platform)
+                    }
+                }
+                manifest_list_spec['manifests'].append(arch_entry)
+
+            manifest_list_spec['tags'] = [image.tag for image in self.workflow.tag_conf.images]
+            # use a unique image tag because manifest-tool can't accept a digest that
+            # isn't in the respository yet
+            registry_image = self.workflow.tag_conf.unique_images[0]
+            registry_image.registry = registry
+            manifest_list_spec['image'] = registry_image.to_str()
+            self.log.info("Submitting manifest-list spec %s", manifest_list_spec)
+            self.submit_manifest_list(registry, registry_conf, manifest_list_spec)
+            insecure = registry_conf.get('insecure', False)
+            secret_path = registry_conf.get('secret')
+
+            self.log.debug('attempting get_manifest_digests from %s for %s',
+                           registry, registry_image)
+            manifest_list_digest = get_manifest_digests(registry_image, registry=registry,
+                                                        insecure=insecure,
+                                                        dockercfg_path=secret_path,
+                                                        versions=('v2_list',))
+            if not manifest_list_digest.v2_list:
+                raise PluginFailedException('no manifest list digest for %s', registry)
+            self.log.debug('Digest for registry %s is %s', registry, manifest_list_digest.v2_list)
+            push_conf_registry = self.workflow.push_conf.add_docker_registry(registry,
+                                                                             insecure=insecure)
+            tag = registry_image.to_str(registry=False)
+            push_conf_registry.digests[tag] = manifest_list_digest
+            grouped_manifests.append(manifest_list_digest)
+
+        self.log.info("Manifest lists created and collected for all repositories")
+        return grouped_manifests
+
     def get_worker_manifest(self, worker_data):
         worker_digests = worker_data['digests']
         worker_manifest = []
@@ -123,10 +196,8 @@ def get_worker_manifest(self, worker_data):
 
     def run(self):
         if self.group:
-            raise NotImplementedError('group=True is not supported in group_manifests')
-        grouped_manifests = []
+            return self.get_grouped_manifests()
 
-        valid = False
         all_annotations = self.workflow.build_result.annotations['worker-builds']
         for plat, annotation in all_annotations.items():
             digests = annotation['digests']
@@ -137,12 +208,8 @@ def run(self):
 
         for platform in all_annotations:
             if self.goarch.get(platform, platform) == 'amd64':
-                valid = True
-                grouped_manifests = self.get_worker_manifest(all_annotations[platform])
-                break
+                self.get_worker_manifest(all_annotations[platform])
+                self.log.debug("found an x86_64 platform and grouped its manifest")
+                return []
 
-        if valid:
-            self.log.debug("found an x86_64 platform and grouped its manifest")
-            return grouped_manifests
-        else:
-            raise ValueError('failed to find an x86_64 platform')
+        raise ValueError('failed to find an x86_64 platform')

atomic_reactor/util.py

@@ -619,9 +619,10 @@ def get_credentials(self, docker_registry):
 class ManifestDigest(object):
     """Wrapper for digests for a docker manifest."""
 
-    def __init__(self, v1=None, v2=None):
+    def __init__(self, v1=None, v2=None, v2_list=None):
         self.v1 = v1
         self.v2 = v2
+        self.v2_list = v2_list
 
     @property
     def default(self):
@@ -632,10 +633,12 @@ def default(self):
         In such case, the v1 schema should be used when interacting
         with the registry.
         """
-        return self.v2 or self.v1
+        return self.v2_list or self.v2 or self.v1
 
 
 def get_manifest_media_type(version):
+    if version == 'v2_list':
+        version = 'list.v2'
     return 'application/vnd.docker.distribution.manifest.{}+json'.format(version)