group_manifests: create minimal plugin for group manifests

atomic_reactor/plugins/post_group_manifests.py

@@ -0,0 +1,148 @@
+"""
+Copyright (c) 2017 Red Hat, Inc
+All rights reserved.
+
+This software may be modified and distributed under the terms
+of the BSD license. See the LICENSE file for details.
+
+get the image manifest lists from the worker builders. If possible, group them together
+and return them. if not, return the x86_64/amd64 image manifest instead after re-uploading
+it for all existing image tags.
+"""
+
+
+from __future__ import unicode_literals
+import requests
+import requests.auth
+
+from six.moves.urllib.parse import urlparse
+
+from atomic_reactor.plugin import PostBuildPlugin, PluginFailedException
+from atomic_reactor.util import Dockercfg
+
+
+class GroupManifestsPlugin(PostBuildPlugin):
+    key = 'group_manifests'
+    is_allowed_to_fail = False
+
+    def __init__(self, tasker, workflow, registries, group=True, goarch=None):
+        """
+        constructor
+
+        :param tasker: DockerTasker instance
+        :param workflow: DockerBuildWorkflow instance
+        :param registries: dict, keys are docker registries, values are dicts containing
+                           per-registry parameters.
+                           Params:
+                            * "secret" optional string - path to the secret, which stores
+                              login and password for remote registry
+        :param group: bool, if true, create a manifest list; otherwise only add tags to
+                      amd64 image manifest
+        :param goarch: dict, keys are platform, values are go language platform names
+        """
+        # call parent constructor
+        super(GroupManifestsPlugin, self).__init__(tasker, workflow)
+        self.group = group
+        self.goarch = goarch or {}
+        self.registries = registries
+        self.worker_registries = {}
+
+    def get_worker_manifest(self, worker_data):
+        worker_digests = worker_data['digests']
+        worker_manifest = []
+
+        msg = "worker_registries {0}".format(self.worker_registries)
+        self.log.debug(msg)
+
+        for registry, registry_conf in self.registries.items():
+            if registry_conf.get('version') == 'v1':
+                continue
+
+            if not registry.startswith('http://') and not registry.startswith('https://'):
+                registry = 'https://' + registry
+
+            registry_noschema = urlparse(registry).netloc
+            self.log.debug("evaluating registry %s", registry_noschema)
+
+            insecure = registry_conf.get('insecure', False)
+            auth = None
+            secret_path = registry_conf.get('secret')
+            if secret_path:
+                self.log.debug("registry %s secret %s", registry_noschema, secret_path)
+                dockercfg = Dockercfg(secret_path).get_credentials(registry_noschema)
+                try:
+                    username = dockercfg['username']
+                    password = dockercfg['password']
+                except KeyError:
+                    self.log.error("credentials for registry %s not found in %s",
+                                   registry_noschema, secret_path)
+                else:
+                    self.log.debug("found user %s for registry %s", username, registry_noschema)
+                    auth = requests.auth.HTTPBasicAuth(username, password)
+
+            if registry_noschema in self.worker_registries:
+                self.log.debug("getting manifests from %s", registry_noschema)
+                digest = worker_digests[0]['digest']
+                repo = worker_digests[0]['repository']
+
+                # get a v2 schemav2 response for now
+                v2schema2 = 'application/vnd.docker.distribution.manifest.v2+json'
+                headers = {'accept': v2schema2}
+                kwargs = {'verify': not insecure, 'headers': headers, 'auth': auth}
+
+                url = '{0}/v2/{1}/manifests/{2}'.format(registry, repo, digest)
+                self.log.debug("attempting get from %s", url)
+                response = requests.get(url, **kwargs)
+
+                image_manifest = response.json()
+
+                if image_manifest['schemaVersion'] == '1':
+                    msg = 'invalid schema from {0}'.format(url)
+                    raise PluginFailedException(msg)
+
+                headers = {'Content-Type': v2schema2}
+                kwargs = {'verify': not insecure, 'headers': headers, 'auth': auth}
+
+                for image in self.workflow.tag_conf.images:
+                    image_tag = image.to_str(registry=False).split(':')[1]
+                    url = '{0}/v2/{1}/manifests/{2}'.format(registry, repo, image_tag)
+                    self.log.debug("for image_tag %s, putting at %s", image_tag, url)
+                    response = requests.put(url, json=image_manifest, **kwargs)
+
+                    if not response.ok:
+                        msg = "PUT failed: {0},\n manifest was: {1}".format(response.json(),
+                                                                            image_manifest)
+                        self.log.error(msg)
+                    response.raise_for_status()
+
+                worker_manifest.append(image_manifest)
+                self.log.debug("appending an image_manifest")
+                break
+
+        return worker_manifest
+
+    def run(self):
+        if self.group:
+            raise NotImplementedError('group=True is not supported in group_manifests')
+        grouped_manifests = []
+
+        valid = False
+        all_annotations = self.workflow.build_result.annotations['worker-builds']
+        for plat, annotation in all_annotations.items():
+            digests = annotation['digests']
+            for digest in digests:
+                registry = digest['registry']
+                self.worker_registries.setdefault(registry, [])
+                self.worker_registries[registry].append(registry)
+
+        for platform in all_annotations:
+            if self.goarch.get(platform, platform) == 'amd64':
+                valid = True
+                grouped_manifests = self.get_worker_manifest(all_annotations[platform])
+                break
+
+        if valid:
+            self.log.debug("found an x86_64 platform and grouped its manifest")
+            return grouped_manifests
+        else:
+            raise ValueError('failed to find an x86_64 platform')

tests/plugins/test_group_manifests.py

@@ -0,0 +1,222 @@
+"""
+Copyright (c) 2017 Red Hat, Inc
+All rights reserved.
+
+This software may be modified and distributed under the terms
+of the BSD license. See the LICENSE file for details.
+"""
+
+from __future__ import print_function, unicode_literals
+import pytest
+import json
+import responses
+from copy import deepcopy
+from tempfile import mkdtemp
+import os
+
+from tests.constants import SOURCE, INPUT_IMAGE, MOCK, DOCKER0_REGISTRY
+
+from atomic_reactor.core import DockerTasker
+from atomic_reactor.build import BuildResult
+from atomic_reactor.plugin import PostBuildPluginsRunner, PluginFailedException
+from atomic_reactor.inner import DockerBuildWorkflow, TagConf
+from atomic_reactor.util import ImageName
+from atomic_reactor.plugins.post_group_manifests import GroupManifestsPlugin
+
+if MOCK:
+    from tests.docker_mock import mock_docker
+
+
+DIGEST1 = 'sha256:28b64a8b29fd2723703bb17acf907cd66898440270e536992b937899a4647414'
+DIGEST2 = 'sha256:0000000000000000000000000000000000000000000000000000000000000000'
+
+
+class Y(object):
+    pass
+
+
+class X(object):
+    image_id = INPUT_IMAGE
+    source = Y()
+    source.dockerfile_path = None
+    source.path = None
+    base_image = ImageName(repo="qwe", tag="asd")
+
+
+X86_DIGESTS = [
+    {
+        'digest': 'sha256:worker-build-x86_64-digest',
+        'tag': 'worker-build-x86_64-latest',
+        'registry': DOCKER0_REGISTRY,
+        'repository': 'worker-build-x86_64-repository',
+    },
+]
+X86_ANNOTATIONS = {
+    'build': {
+        'build-name': 'worker-build-x86_64',
+        'cluster-url': 'https://worker_x86_64.com/',
+        'namespace': 'worker_x86_64_namespace'
+    },
+    'digests': X86_DIGESTS,
+    'plugins-metadata': {},
+}
+PPC_DIGESTS = [
+    {
+        'digest': 'sha256:worker-build-ppc64le-digest',
+        'tag': 'worker-build-ppc64le-latest',
+        'registry': 'worker-build-ppc64le-registry',
+        'repository': 'worker-build-ppc64le-repository',
+    },
+]
+PPC_ANNOTATIONS = {
+    'build': {
+        'build-name': 'worker-build-ppc64le',
+        'cluster-url': 'https://worker_ppc64le.com/',
+        'namespace': 'worker_ppc64le_namespace'
+    },
+    'digests': PPC_DIGESTS,
+    'plugins-metadata': {}
+}
+
+BUILD_ANNOTATIONS = {
+        'worker-builds': {
+        },
+        'repositories': {
+            'unique': [
+                'worker-build-ppc64le-unique',
+                'worker-build-x86_64-unique',
+            ],
+            'primary': [
+                'worker-build-ppc64le-primary',
+                'worker-build-x86_64-primary',
+            ],
+        },
+    }
+V1_REGISTRY = "172.17.42.2:5000"
+
+
+def mock_environment(tmpdir, docker_registry=None, primary_images=None,
+                     worker_annotations={}):
+    if MOCK:
+        mock_docker()
+    tasker = DockerTasker()
+    workflow = DockerBuildWorkflow(SOURCE, "test-image")
+    base_image_id = '123456parent-id'
+    setattr(workflow, '_base_image_inspect', {'Id': base_image_id})
+    setattr(workflow, 'builder', X())
+    setattr(workflow.builder, 'image_id', '123456imageid')
+    setattr(workflow.builder, 'base_image', ImageName(repo='Fedora', tag='22'))
+    setattr(workflow.builder, 'source', X())
+    setattr(workflow.builder, 'built_image_info', {'ParentId': base_image_id})
+    setattr(workflow.builder.source, 'dockerfile_path', None)
+    setattr(workflow.builder.source, 'path', None)
+    setattr(workflow, 'tag_conf', TagConf())
+    if primary_images:
+        workflow.tag_conf.add_primary_images(primary_images)
+
+    annotations = deepcopy(BUILD_ANNOTATIONS)
+    if not worker_annotations:
+        worker_annotations = {'ppc64le': PPC_ANNOTATIONS}
+    for worker in worker_annotations:
+        annotations['worker-builds'][worker] = deepcopy(worker_annotations[worker])
+
+    workflow.build_result = BuildResult(image_id='123456', annotations=annotations)
+
+    return tasker, workflow
+
+
+def mock_url_responses(docker_registry, test_images, worker_digests, version='2', respond=True):
+    responses.reset()
+    for worker_digest in worker_digests:
+        digest = worker_digest[0]['digest']
+        repo = worker_digest[0]['repository']
+        for registry in docker_registry:
+            if not registry.startswith('http://') and not registry.startswith('https://'):
+                registry = 'https://' + registry
+            url = '{0}/v2/{1}/manifests/{2}'.format(registry, repo, digest)
+            body = json.dumps({'tag': 'testtag', 'schemaVersion': version})
+            responses.add(responses.GET, url, body=body)
+            if respond:
+                status = 200
+            else:
+                status = 400
+                body = json.dumps({'error': 'INVALID MANIFEST'})
+            for image_tag in test_images:
+                url = '{0}/v2/{1}/manifests/{2}'.format(registry, repo, image_tag.split(':')[1])
+                responses.add(responses.PUT, url, status=status, json=body)
+
+
+class TestGroupManifests(object):
+    def test_group_manifests_unimplemented(self, tmpdir):
+        plugins_conf = [{
+            'name': GroupManifestsPlugin.key,
+            'args': {
+                'registries': {},
+            }
+        }]
+        tasker, workflow = mock_environment(tmpdir)
+
+        runner = PostBuildPluginsRunner(tasker, workflow, plugins_conf)
+        with pytest.raises(PluginFailedException):
+            runner.run()
+
+    @pytest.mark.parametrize('use_secret', [True, False])
+    @pytest.mark.parametrize('version', ['1', '2'])
+    @pytest.mark.parametrize(('goarch', 'worker_annotations', 'valid', 'respond'), [
+        ({}, {}, False, True),
+        ({}, {'x86_64': X86_ANNOTATIONS}, False, True),
+        ({'x86_64': 'amd64'}, {}, False, True),
+        ({'x86_64': 'amd64'}, {'x86_64': X86_ANNOTATIONS}, True, True),
+        ({'ppc64le': 'powerpc', 'x86_64': 'amd64'},
+         {'ppc64le': PPC_ANNOTATIONS, 'x86_64': X86_ANNOTATIONS}, True, True),
+        ({'ppc64le': 'powerpc', 'x86_64': 'amd64'},
+         {'ppc64le': PPC_ANNOTATIONS, 'x86_64': X86_ANNOTATIONS}, True, False),
+    ])
+    @responses.activate  # noqa
+    def test_basic_group_manifests(self, tmpdir, use_secret, goarch,
+                                   worker_annotations, version, valid, respond):
+        if MOCK:
+            mock_docker()
+
+        if version == '1':
+            valid = False
+
+        test_images = ['namespace/httpd:2.4', 'namespace/sshd:2.4']
+        test_results = [{'tag': 'testtag', 'schemaVersion': '2'}]
+
+        registries = {
+            DOCKER0_REGISTRY: {'version': 'v2'},
+            V1_REGISTRY: {'version': 'v1'},
+        }
+        if use_secret:
+            temp_dir = mkdtemp(dir=str(tmpdir))
+            with open(os.path.join(temp_dir, ".dockercfg"), "w+") as dockerconfig:
+                dockerconfig_contents = {
+                    DOCKER0_REGISTRY: {
+                        "username": "user", "password": DOCKER0_REGISTRY
+                    }
+                }
+                dockerconfig.write(json.dumps(dockerconfig_contents))
+                dockerconfig.flush()
+                registries[DOCKER0_REGISTRY]['secret'] = temp_dir
+
+        plugins_conf = [{
+            'name': GroupManifestsPlugin.key,
+            'args': {
+                'registries': registries,
+                'group': False,
+                'goarch': goarch,
+            },
+        }]
+        tasker, workflow = mock_environment(tmpdir, docker_registry=DOCKER0_REGISTRY,
+                                            primary_images=test_images,
+                                            worker_annotations=worker_annotations)
+        mock_url_responses([DOCKER0_REGISTRY], test_images, [X86_DIGESTS], version, respond)
+
+        runner = PostBuildPluginsRunner(tasker, workflow, plugins_conf)
+        if valid and respond:
+            result = runner.run()
+            assert result['group_manifests'] == test_results
+        else:
+            with pytest.raises(PluginFailedException):
+                runner.run()