diff --git a/.bazelci/presubmit.yml b/.bazelci/presubmit.yml index 55f652198..86312cd16 100644 --- a/.bazelci/presubmit.yml +++ b/.bazelci/presubmit.yml @@ -85,7 +85,6 @@ platforms: - "//tests/container:alpine_custom_attr_digest_test" - "//tests/container:alpine_linux_armv6_tar_test_image_tar" - "//tests/container:basic_windows_image_test" - - "//tests/container:basic_windows_image_test_go_join_layers" - "//tests/container:build_tar_test" - "//tests/container:distroless_fixed_id_digest_test" - "//tests/container:distroless_fixed_id_image_digest_test" @@ -102,7 +101,6 @@ platforms: - "//tests/container:test_digest_output1" - "//tests/container:test_digest_output2" - "//tests/container:test_push_digest_output" - - "//tests/contrib:rename_image_go_test" - "//tests/contrib:rename_image_test" - "//tests/contrib:test_compare_ids_test" - "//tests/contrib:test_compare_ids_test_diff_ids_fails_no_regex" diff --git a/container/BUILD b/container/BUILD index 4417ca96f..9a554ded0 100644 --- a/container/BUILD +++ b/container/BUILD @@ -26,51 +26,6 @@ filegroup( visibility = ["//visibility:public"], ) -py_binary( - name = "extract_config", - srcs = ["extract_config.py"], - legacy_create_init = False, - python_version = "PY2", - visibility = ["//visibility:public"], - deps = [ - "@containerregistry", - ], -) - -py_library( - name = "utils", - srcs = [ - "__init__.py", - "utils.py", - ], -) - -py_binary( - name = "join_layers", - srcs = ["join_layers.py"], - legacy_create_init = False, - python_version = "PY2", - visibility = ["//visibility:public"], - deps = [ - ":utils", - "@containerregistry", - "@six", - ], -) - -py_binary( - name = "create_image_config", - srcs = ["create_image_config.py"], - legacy_create_init = False, - python_version = "PY2", - visibility = ["//visibility:public"], - deps = [ - ":utils", - "@containerregistry", - "@six", - ], -) - # TODO(xingao): Flip legacy_create_init to False if possible. py_binary( name = "build_tar", @@ -156,7 +111,6 @@ TEST_TARGETS = [ ":py_image_complex", ":war_image", ":flat", - ":flat_go", ":flatten_with_tarball_base", ] @@ -166,7 +120,7 @@ TEST_DATA = [ ] + [ "//testdata:stamped_bundle_test", "//testdata:stamp_info_file.txt", - "//tests/container:basic_windows_image_go_join_layers.tar", + "//tests/container:basic_windows_image.tar", ] py_test( diff --git a/container/__init__.py b/container/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/container/create_image_config.py b/container/create_image_config.py deleted file mode 100644 index 52c1fda65..000000000 --- a/container/create_image_config.py +++ /dev/null @@ -1,215 +0,0 @@ -# Copyright 2016 The Bazel Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""This package manipulates v2.2 image configuration metadata.""" - -from __future__ import division - -import argparse -import datetime -import json -import sys - -import six - -from container import utils -from containerregistry.transform.v2_2 import metadata as v2_2_metadata - -parser = argparse.ArgumentParser( - description='Manipulate Docker image v2.2 metadata.') - -parser.add_argument('--base', action='store', - help='The parent image.') - -parser.add_argument('--basemanifest', action='store', - help='The parent image manifest.') - -parser.add_argument('--output', action='store', required=True, - help='The output file to generate.') - -parser.add_argument('--manifestoutput', action='store', required=False, - help='The manifest output file to generate.') - -parser.add_argument('--layer', action='append', default=[], - help='Layer sha256 hashes that make up this image') - -parser.add_argument('--entrypoint', action='append', default=[], - help='Override the "Entrypoint" of the previous layer.') - -parser.add_argument('--command', action='append', default=[], - help='Override the "Cmd" of the previous layer.') - -parser.add_argument('--creation_time', action='store', required=False, - help='The creation timestamp. Acceptable formats: ' - 'Integer or floating point seconds since Unix Epoch, RFC ' - '3339 date/time') - -parser.add_argument('--user', action='store', - help='The username to run commands under.') - -parser.add_argument('--labels', action='append', default=[], - help='Augment the "Label" of the previous layer.') - -parser.add_argument('--ports', action='append', default=[], - help='Augment the "ExposedPorts" of the previous layer.') - -parser.add_argument('--volumes', action='append', default=[], - help='Augment the "Volumes" of the previous layer.') - -parser.add_argument('--workdir', action='store', - help='Set the working directory of the layer.') - -parser.add_argument('--env', action='append', default=[], - help='Augment the "Env" of the previous layer.') - -parser.add_argument('--stamp-info-file', action='append', required=False, - help=('A list of files from which to read substitutions ' - 'to make in the provided fields, e.g. {BUILD_USER}')) - -parser.add_argument('--null_entrypoint', action='store', default=False, - help='If True, "Entrypoint" will be set to null.') - -parser.add_argument('--null_cmd', action='store', default=False, - help='If True, "Cmd" will be set to null.') - -parser.add_argument('--operating_system', action='store', default='linux', - choices=['linux', 'windows'], - help=('Operating system to create docker image for, e.g. {linux}')) - -parser.add_argument('--entrypoint_prefix', action='append', default=[], - help='Prefix the "Entrypoint" with the specified arguments.') - -_PROCESSOR_ARCHITECTURE = 'amd64' - -def KeyValueToDict(pair): - """Converts an iterable object of key=value pairs to dictionary.""" - d = dict() - for kv in pair: - (k, v) = kv.split('=', 1) - d[k] = v - return d - - -# See: https://bugs.python.org/issue14364 -def fix_dashdash(l): - return [ - x if x != [] else '--' - for x in l - ] - -def main(): - args = parser.parse_args() - - def Stamp(inp): - """Perform substitutions in the provided value.""" - if not args.stamp_info_file or not inp: - return inp - format_args = {} - for infofile in args.stamp_info_file: - with open(infofile) as info: - for line in info: - line = line.strip('\n') - key, value = line.split(' ', 1) - if key in format_args: - print ('WARNING: Duplicate value for key "%s": ' - 'using "%s"' % (key, value)) - format_args[key] = value - - return inp.format(**format_args) - - base_json = '{}' - if args.base: - with open(args.base, 'r') as r: - base_json = r.read() - data = json.loads(base_json) - - base_manifest_json = '{}' - if args.basemanifest: - with open(args.basemanifest, 'r') as r: - base_manifest_json = r.read() - manifestdata = json.loads(base_manifest_json) - - layers = [] - for layer in args.layer: - layers.append(utils.ExtractValue(layer)) - - labels = KeyValueToDict(args.labels) - for label, value in six.iteritems(labels): - if value.startswith('@'): - with open(value[1:], 'r') as f: - labels[label] = f.read() - elif '{' in value: - labels[label] = Stamp(value) - - creation_time = None - if args.creation_time: - creation_time = Stamp(args.creation_time) - try: - # If creation_time is parsable as a floating point type, assume unix epoch - # timestamp. - parsed_unix_timestamp = float(creation_time) - if parsed_unix_timestamp > 1.0e+11: - # Bazel < 0.12 was bugged and used milliseconds since unix epoch as - # the default. Values > 1e11 are assumed to be unix epoch - # milliseconds. - parsed_unix_timestamp = parsed_unix_timestamp / 1000.0 - - # Construct a RFC 3339 date/time from the Unix epoch. - creation_time = ( - datetime.datetime.utcfromtimestamp( - parsed_unix_timestamp - ).strftime("%Y-%m-%dT%H:%M:%S.%fZ") - ) - except ValueError: - # Otherwise, assume RFC 3339 date/time format. - pass - - output = v2_2_metadata.Override(data, v2_2_metadata.Overrides( - author='Bazel', - created_by='bazel build ...', - layers=layers, - entrypoint=list(map(Stamp, fix_dashdash(args.entrypoint))), - cmd=list(map(Stamp, fix_dashdash(args.command))), - creation_time=creation_time, - user=Stamp(args.user), - labels=labels, env={ - k: Stamp(v) - for (k, v) in six.iteritems(KeyValueToDict(args.env)) - }, - ports=args.ports, volumes=args.volumes, workdir=Stamp(args.workdir)), - architecture=_PROCESSOR_ARCHITECTURE, - operating_system=args.operating_system) - - if ('config' in output and 'Cmd' in output['config'] and - args.null_cmd == "True"): - del (output['config']['Cmd']) - - if ('config' in output and 'Entrypoint' in output['config'] and - args.null_entrypoint == "True"): - del (output['config']['Entrypoint']) - - if args.entrypoint_prefix: - output['config']['Entrypoint'] = (args.entrypoint_prefix + - output['config'].get('Entrypoint', [])) - - with open(args.output, 'w') as fp: - json.dump(output, fp, sort_keys=True) - fp.write('\n') - - if (args.manifestoutput): - with open(args.manifestoutput, 'w') as fp: - json.dump(manifestdata, fp, sort_keys=False) - fp.write('\n') - -if __name__ == '__main__': - main() diff --git a/container/extract_config.py b/container/extract_config.py deleted file mode 100644 index 631120bbe..000000000 --- a/container/extract_config.py +++ /dev/null @@ -1,47 +0,0 @@ -# Copyright 2017 The Bazel Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""A trivial binary to extract the v2.2 config file.""" - -import argparse - -from containerregistry.client.v2_2 import docker_image - -parser = argparse.ArgumentParser( - description='Extract the v2.2 config from a Docker image tarball.') - -parser.add_argument('--tarball', action='store', required=True, - help=('The Docker image tarball from which to ' - 'extract the image name.')) - -parser.add_argument('--output', action='store', required=True, - help='The output file to which we write the config.') - -parser.add_argument('--manifestoutput', action='store', required=True, - help='The output file to which we write the manifest.') - -# Main program to create a docker image. It expect to be run with: -# extract_config --tarball=image.tar \ -# --output=output.config \ -def main(): - args = parser.parse_args() - - with docker_image.FromTarball(args.tarball) as img: - with open(args.output, 'w') as f: - f.write(img.config_file()) - with open(args.manifestoutput, 'w') as f: - f.write(img.manifest()) - - -if __name__ == '__main__': - main() diff --git a/container/flatten.bzl b/container/flatten.bzl index db5841203..fea2a1ad3 100644 --- a/container/flatten.bzl +++ b/container/flatten.bzl @@ -29,34 +29,13 @@ def _impl(ctx): # Leverage our efficient intermediate representation to push. img_args = ctx.actions.args() - img_inputs = [] - if ctx.attr.use_legacy_flattener: - if image.get("legacy"): - img_inputs += [image["legacy"]] - img_args.add(image["legacy"].path, format = "--tarball=%s") - - blobsums = image.get("blobsum", []) - img_args.add_all(blobsums, format_each = "--digest=%s") - img_inputs += blobsums - blobs = image.get("zipped_layer", []) - img_args.add_all(blobs, format_each = "--layer=%s") - img_inputs += blobs - uncompressed_blobs = image.get("unzipped_layer", []) - img_args.add_all(uncompressed_blobs, format_each = "--uncompressed_layer=%s") - img_inputs += uncompressed_blobs - diff_ids = image.get("diff_id", []) - img_args.add_all(diff_ids, format_each = "--diff_id=%s") - img_inputs += diff_ids - img_args.add(image["config"], format = "--config=%s") - img_inputs += [image["config"]] - else: - args, img_inputs = _gen_img_args(ctx, image) - img_args.add_all(args) + args, img_inputs = _gen_img_args(ctx, image) + img_args.add_all(args) img_args.add(ctx.outputs.filesystem, format = "--filesystem=%s") img_args.add(ctx.outputs.metadata, format = "--metadata=%s") ctx.actions.run( - executable = ctx.executable._flattener if ctx.attr.use_legacy_flattener else ctx.executable._go_flattener, + executable = ctx.executable._flattener, arguments = [img_args], inputs = img_inputs, outputs = [ctx.outputs.filesystem, ctx.outputs.metadata], @@ -72,20 +51,7 @@ container_flatten = rule( allow_single_file = [".tar"], mandatory = True, ), - # TODO (smukherj1): Remove once migration in #580 is done. - "use_legacy_flattener": attr.bool( - default = False, - doc = "Use the legacy python flattener to generate the image " + - "filesystem tarball. Uses the Go implementation when set to" + - " false.", - ), "_flattener": attr.label( - default = Label("@containerregistry//:flatten"), - cfg = "host", - executable = True, - allow_files = True, - ), - "_go_flattener": attr.label( default = Label("//container/go/cmd/flattener"), cfg = "host", executable = True, diff --git a/container/image.bzl b/container/image.bzl index 0cd3beb14..34cbdaffe 100644 --- a/container/image.bzl +++ b/container/image.bzl @@ -93,7 +93,7 @@ def _get_base_manifest(ctx, name, base): return layer.get("manifest") return None -def _add_go_args( +def _add_create_image_config_args( ctx, args, inputs, @@ -111,6 +111,9 @@ def _add_go_args( base_config, base_manifest, operating_system): + """ + Add args for the create_image_config Go binary. + """ args.add("-outputConfig", config) args.add("-outputManifest", manifest) @@ -177,79 +180,6 @@ def _add_go_args( def _format_legacy_label(t): return ("--labels=%s=%s" % (t[0], t[1])) -def _add_legacy_args( - ctx, - args, - inputs, - manifest, - config, - labels, - entrypoint, - cmd, - null_cmd, - null_entrypoint, - creation_time, - env, - workdir, - layer_names, - base_config, - base_manifest, - operating_system): - args.add(config, format = "--output=%s") - args.add(manifest, format = "--manifestoutput=%s") - args.add_all(entrypoint, format_each = "--entrypoint=%s") - args.add_all(cmd, format_each = "--command=%s") - args.add_all(ctx.attr.ports, format_each = "--ports=%s") - args.add_all(ctx.attr.volumes, format_each = "--volumes=%s") - args.add("--null_entrypoint=%s" % null_entrypoint) - args.add("--null_cmd=%s" % null_cmd) - - if creation_time: - args.add(creation_time, format = "--creation_time=%s") - elif ctx.attr.stamp: - # If stamping is enabled, and the creation_time is not manually defined, - # default to '{BUILD_TIMESTAMP}'. - args.add("{BUILD_TIMESTAMP}", format = "--creation_time=%s") - - args.add_all(labels.items(), map_each = _format_legacy_label) - - args.add_all(["--env=%s" % "=".join([ - ctx.expand_make_variables("env", key, {}), - ctx.expand_make_variables("env", value, {}), - ]) for key, value in env.items()]) - - if ctx.attr.user: - args.add(ctx.attr.user, format = "--user=%s") - if workdir: - args.add(workdir, format = "--workdir=%s") - - inputs += layer_names - args.add_all(layer_names, format_each = "--layer=@%s") - - if ctx.attr.label_files: - inputs += ctx.files.label_files - - if base_config: - args.add(base_config, format = "--base=%s") - inputs += [base_config] - - if base_manifest: - args.add(base_manifest, format = "--basemanifest=%s") - inputs += [base_manifest] - - if operating_system: - args.add(operating_system, format = "--operating_system=%s") - - if ctx.attr.stamp: - stamp_inputs = [ctx.info_file, ctx.version_file] - args.add_all(stamp_inputs, format_each = "--stamp-info-file=%s") - inputs += stamp_inputs - - if ctx.attr.launcher_args and not ctx.attr.launcher: - fail("launcher_args does nothing when launcher is not specified.", attr = "launcher_args") - if ctx.attr.launcher: - args.add_all(["/" + ctx.file.launcher.basename] + ctx.attr.launcher_args, format_each = "--entrypoint_prefix=%s") - def _image_config( ctx, name, @@ -285,51 +215,28 @@ def _image_config( args = ctx.actions.args() inputs = [] executable = None - if ctx.attr.legacy_create_image_config: - _add_legacy_args( - ctx, - args, - inputs, - manifest, - config, - labels, - entrypoint, - cmd, - null_cmd, - null_entrypoint, - creation_time, - env, - workdir, - layer_names, - base_config, - base_manifest, - operating_system, - ) - executable = ctx.executable.create_image_config - else: - _add_go_args( - ctx, - args, - inputs, - manifest, - config, - labels, - entrypoint, - cmd, - null_cmd, - null_entrypoint, - creation_time, - env, - workdir, - layer_names, - base_config, - base_manifest, - operating_system, - ) - executable = ctx.executable.go_create_image_config + _add_create_image_config_args( + ctx, + args, + inputs, + manifest, + config, + labels, + entrypoint, + cmd, + null_cmd, + null_entrypoint, + creation_time, + env, + workdir, + layer_names, + base_config, + base_manifest, + operating_system, + ) ctx.actions.run( - executable = executable, + executable = ctx.executable.create_image_config, arguments = [args], inputs = inputs, outputs = [config, manifest], @@ -603,7 +510,7 @@ _attrs = dicts.add(_layer.attrs, { "base": attr.label(allow_files = container_filetype), "cmd": attr.string_list(), "create_image_config": attr.label( - default = Label("//container:create_image_config"), + default = Label("//container/go/cmd/create_image_config:create_image_config"), cfg = "host", executable = True, allow_files = True, @@ -611,12 +518,6 @@ _attrs = dicts.add(_layer.attrs, { "creation_time": attr.string(), "docker_run_flags": attr.string(), "entrypoint": attr.string_list(), - "go_create_image_config": attr.label( - default = Label("//container/go/cmd/create_image_config:create_image_config"), - cfg = "host", - executable = True, - allow_files = True, - ), "label_file_strings": attr.string_list(), # Implicit/Undocumented dependencies. "label_files": attr.label_list( @@ -626,10 +527,6 @@ _attrs = dicts.add(_layer.attrs, { "launcher": attr.label(allow_single_file = True), "launcher_args": attr.string_list(default = []), "layers": attr.label_list(providers = [LayerInfo]), - "legacy_create_image_config": attr.bool( - default = False, - doc = ("If set to False, the Go create_image_config binary will be run instead."), - ), "legacy_repository_naming": attr.bool(default = False), "legacy_run_behavior": attr.bool( # TODO(mattmoor): Default this to False. diff --git a/container/image_test.py b/container/image_test.py index dcaaff239..cdbcfc6d1 100644 --- a/container/image_test.py +++ b/container/image_test.py @@ -355,12 +355,6 @@ def test_flattened(self): self.assertTarballContains(tar, [ '.', '/usr', '/usr/bin', '/usr/bin/java', './foo']) - def test_flattened_go(self): - # Test the flattened tarball produced by the Go flattener binary. - with tarfile.open(TestData('flat_go.tar'), mode='r') as tar: - self.assertTarballContains(tar, [ - '.', '/usr', '/usr/bin', '/usr/bin/java', './foo']) - def test_flattened_from_tarball_base(self): # Test the flattened tarball produced by the Go flattener where the # image being flattened derived from an image specified as a tarball. @@ -525,7 +519,7 @@ def test_py_image(self): ]) def test_windows_image_manifest_with_foreign_layers(self): - imgPath = TestRunfilePath("tests/container/basic_windows_image_go_join_layers.tar") + imgPath = TestRunfilePath("tests/container/basic_windows_image.tar") with v2_2_image.FromTarball(imgPath) as img: # Ensure the image manifest in the tarball includes the foreign layer. self.assertIn("https://go.microsoft.com/fwlink/?linkid=873595", diff --git a/container/join_layers.py b/container/join_layers.py deleted file mode 100644 index 80e2f9485..000000000 --- a/container/join_layers.py +++ /dev/null @@ -1,301 +0,0 @@ -# Copyright 2015 The Bazel Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""This tool creates a docker image from a list of layers.""" - -import argparse -import hashlib -import json -import os -import sys -import tarfile - -import six -from six.moves import cStringIO - -from container import utils -from containerregistry.client import docker_name -from containerregistry.client.v2_2 import save as v2_2_save -from containerregistry.client.v2_2 import docker_http -from containerregistry.client.v2_2 import docker_image as v2_2_image - -parser = argparse.ArgumentParser( - description='Link together several layer shards.') - -parser.add_argument('--output', action='store', required=True, - help='The output file, mandatory') - -parser.add_argument('--tags', action='append', required=True, - help=('An associative list of fully qualified tag names ' - 'and the layer they tag. ' - 'e.g. ubuntu=deadbeef,gcr.io/blah/debian=baadf00d')) - -parser.add_argument('--manifests', action='append', required=False, - help=('An associative list of fully qualified tag names ' - 'and the manifest associated' - 'e.g. ubuntu=deadbeef,gcr.io/blah/debian=baadf00d')) - -parser.add_argument('--layer', action='append', required=False, - help=('Each entry is an equivalence class with 4 parts: ' - 'diff_id, blob_sum, unzipped layer, zipped layer.')) - -parser.add_argument('--legacy', action='append', - help='A list of tarballs from which our images may derive.') - -parser.add_argument('--stamp-info-file', action='append', required=False, - help=('If stamping these layers, the list of files from ' - 'which to obtain workspace information')) - -class FromParts(v2_2_image.DockerImage): - """This accesses a more efficient on-disk format than FromTarball. - - FromParts is similar to FromDisk, but leverages the fact that we have both - compressed and uncompressed forms available. - """ - - def __init__(self, config_file, manifest_file, diffid_to_blobsum, - blobsum_to_unzipped, blobsum_to_zipped, blobsum_to_legacy): - self._config = config_file - self._manifest = manifest_file - self._blobsum_to_unzipped = blobsum_to_unzipped - self._blobsum_to_zipped = blobsum_to_zipped - self._blobsum_to_legacy = blobsum_to_legacy - self._diffid_to_blobsum = diffid_to_blobsum - config = json.loads(self._config) - - content = self.config_file().encode('utf-8') - - self._manifest = json.dumps({ - 'schemaVersion': 2, - 'mediaType': docker_http.MANIFEST_SCHEMA2_MIME, - 'config': { - 'mediaType': docker_http.CONFIG_JSON_MIME, - 'size': len(content), - 'digest': 'sha256:' + hashlib.sha256(content).hexdigest() - }, - 'layers': [ - self.diff_id_to_layer_manifest_json(diff_id) - for diff_id in config['rootfs']['diff_ids'] - ] - }, sort_keys=True) - - def manifest(self): - """Override.""" - return self._manifest - - def config_file(self): - """Override.""" - return self._config - - # Could be large, do not memoize - def uncompressed_blob(self, digest): - """Override.""" - - if self.blobsum_to_media_type(digest) == docker_http.FOREIGN_LAYER_MIME: - return bytearray() - elif digest not in self._blobsum_to_unzipped: - return self._blobsum_to_legacy[digest].uncompressed_blob(digest) - - with open(self._blobsum_to_unzipped[digest], 'rb') as reader: - return reader.read() - - # Could be large, do not memoize - def blob(self, digest): - """Override.""" - if digest not in self._blobsum_to_zipped: - return self._blobsum_to_legacy[digest].blob(digest) - with open(self._blobsum_to_zipped[digest], 'rb') as reader: - return reader.read() - - def blob_size(self, digest): - """Override.""" - if self.blobsum_to_media_type(digest) == docker_http.FOREIGN_LAYER_MIME: - return self.blobsum_to_manifest_layer(digest)['size'] - elif digest not in self._blobsum_to_zipped: - return self._blobsum_to_legacy[digest].blob_size(digest) - info = os.stat(self._blobsum_to_zipped[digest]) - return info.st_size - - def diff_id_to_manifest_layer(self, diff_id): - return self.blobsum_to_manifest_layer(self._diffid_to_blobsum[diff_id]) - - def blobsum_to_manifest_layer(self, digest): - if self._manifest: - manifest = json.loads(self._manifest) - if 'layers' in manifest: - for layer in manifest['layers']: - if layer['digest'] == digest: - return layer - return None - - def blobsum_to_media_type(self, digest): - manifest_layer = self.blobsum_to_manifest_layer(digest) - if manifest_layer: - return manifest_layer['mediaType'] - return docker_http.LAYER_MIME - - def diff_id_to_layer_manifest_json(self, diff_id): - manifest_layer = self.diff_id_to_manifest_layer(diff_id) - - if manifest_layer: - return manifest_layer - else: - return { - 'mediaType': docker_http.LAYER_MIME, - 'size': self.blob_size(self._diffid_to_blobsum[diff_id]), - 'digest': self._diffid_to_blobsum[diff_id] - } - - # __enter__ and __exit__ allow use as a context manager. - def __enter__(self): - return self - - def __exit__(self, unused_type, unused_value, unused_traceback): - pass - - -def create_bundle(output, tag_to_config, tag_to_manifest, diffid_to_blobsum, - blobsum_to_unzipped, blobsum_to_zipped, blobsum_to_legacy): - """Creates a Docker image from a list of layers. - - Args: - output: the name of the docker image file to create. - layers: the layers (tar files) to join to the image. - tag_to_layer: a map from docker_name.Tag to the layer id it references. - layer_to_tags: a map from the name of the layer tarball as it appears - in our archives to the list of tags applied to it. - """ - - with tarfile.open(output, 'w') as tar: - def add_file(filename, contents): - info = tarfile.TarInfo(filename) - info.size = len(contents) - tar.addfile(tarinfo=info, fileobj=cStringIO.StringIO(contents)) - - tag_to_image = {} - for (tag, config) in six.iteritems(tag_to_config): - manifest = None - if tag in tag_to_manifest: - manifest = tag_to_manifest[tag] - tag_to_image[tag] = FromParts( - config, manifest, diffid_to_blobsum, - blobsum_to_unzipped, blobsum_to_zipped, blobsum_to_legacy) - - v2_2_save.multi_image_tarball(tag_to_image, tar) - -def create_tag_to_file_content_map(stamp_info, tag_file_pairs): - """ - Creates a Docker image tag to file content map. - - Args: - stamp_info - Tag substitutions to make in the input tags, e.g. {BUILD_USER} - tag_file_pairs - List of input tags and file names - (e.g. ...:image=@bazel-out/...image.0.config) - """ - tag_to_file_content = {} - - if tag_file_pairs: - for entry in tag_file_pairs: - elts = entry.split('=') - if len(elts) != 2: - raise Exception('Expected associative list key=value, got: %s' % entry) - (fq_tag, filename) = elts - - formatted_tag = fq_tag.format(**stamp_info) - tag = docker_name.Tag(formatted_tag, strict=False) - file_contents = utils.ExtractValue(filename) - - # Add the mapping in one direction. - tag_to_file_content[tag] = file_contents - - return tag_to_file_content - -def main(): - args = parser.parse_args() - - tag_to_config = {} - tag_to_manifest = {} - stamp_info = {} - diffid_to_blobsum = {} - blobsum_to_unzipped = {} - blobsum_to_zipped = {} - - if args.stamp_info_file: - for infofile in args.stamp_info_file: - with open(infofile) as info: - for line in info: - line = line.strip("\n") - key, value = line.split(" ", 1) - if key in stamp_info: - print ("WARNING: Duplicate value for workspace status key '%s': " - "using '%s'" % (key, value)) - stamp_info[key] = value - - tag_to_config = create_tag_to_file_content_map(stamp_info, args.tags) - tag_to_manifest = create_tag_to_file_content_map(stamp_info, args.manifests) - - # Do this first so that if there is overlap with the loop below it wins. - blobsum_to_legacy = {} - for tar in args.legacy or []: - with v2_2_image.FromTarball(tar) as legacy_image: - config_file = legacy_image.config_file() - cfg = json.loads(config_file) - fs_layers = list(reversed(legacy_image.fs_layers())) - for i, diff_id in enumerate(cfg['rootfs']['diff_ids']): - blob_sum = fs_layers[i] - diffid_to_blobsum[diff_id] = blob_sum - blobsum_to_legacy[blob_sum] = legacy_image - - if args.layer: - for entry in args.layer: - elts = entry.split('=') - if len(elts) != 4: - raise Exception('Expected associative list key=value, got: %s' % entry) - (diffid_filename, blobsum_filename, - unzipped_filename, zipped_filename) = elts - - diff_id = 'sha256:' + utils.ExtractValue(diffid_filename) - blob_sum = 'sha256:' + utils.ExtractValue(blobsum_filename) - - diffid_to_blobsum[diff_id] = blob_sum - blobsum_to_unzipped[blob_sum] = unzipped_filename - blobsum_to_zipped[blob_sum] = zipped_filename - - # add foreign layers - # - # Windows base images distributed by Microsoft are using foreign layers. - # Foreign layers are not stored in the Docker repository like normal layers. - # Instead they include a list of URLs where the layer can be downloaded. - # This is done because Windows base images are large (2+GB). When someone - # pulls a Windows image, it downloads the foreign layers from those URLs - # instead of requesting the blob from the registry. - # When adding foreign layers through bazel, the actual layer blob is not - # present on the system. Instead the base image manifest is used to - # describe the parent image layers. - for tag, manifest_file in tag_to_manifest.items(): - manifest = json.loads(manifest_file) - if 'layers' in manifest: - config = json.loads(tag_to_config[tag]) - for i, layer in enumerate(manifest['layers']): - diff_id = config['rootfs']['diff_ids'][i] - if layer['mediaType'] == docker_http.FOREIGN_LAYER_MIME: - blob_sum = layer['digest'] - diffid_to_blobsum[diff_id] = blob_sum - - create_bundle( - args.output, tag_to_config, tag_to_manifest, diffid_to_blobsum, - blobsum_to_unzipped, blobsum_to_zipped, blobsum_to_legacy) - - -if __name__ == '__main__': - main() diff --git a/container/layer_tools.bzl b/container/layer_tools.bzl index bdd277e26..b6aec4453 100644 --- a/container/layer_tools.bzl +++ b/container/layer_tools.bzl @@ -123,41 +123,7 @@ def get_from_target(ctx, name, attr_target, file_target = None): target = attr_target.files.to_list()[0] return _extract_layers(ctx, name, target) -def _add_join_layers_py_args(args, inputs, images): - """Add args & inputs needed to call join_layers.py for the given images. - - TODO(smukherj1): Remove this when the migration to go-containerregistry is - complete. - """ - for tag in images: - image = images[tag] - args.add(image["config"], format = "--tags=" + tag + "=@%s") - inputs += [image["config"]] - - if image.get("manifest"): - args.add(image["manifest"], format = "--manifests=" + tag + "=@%s") - inputs += [image["manifest"]] - - for i in range(0, len(image["diff_id"])): - # There's no way to do this with attrs w/o resolving paths here afaik - args.add( - "--layer=" + - "@" + image["diff_id"][i].path + - "=@" + image["blobsum"][i].path + - # No @, not resolved through utils, always filename. - "=" + image["unzipped_layer"][i].path + - "=" + image["zipped_layer"][i].path, - ) - inputs += image["unzipped_layer"] - inputs += image["diff_id"] - inputs += image["zipped_layer"] - inputs += image["blobsum"] - - if image.get("legacy"): - args.add(image["legacy"].path, format = "--legacy=%s") - inputs += [image["legacy"]] - -def _add_join_layers_go_args(args, inputs, images): +def _add_join_layers_args(args, inputs, images): """Add args & inputs needed to call the Go join_layers for the given images """ for tag in images: @@ -207,13 +173,10 @@ def assemble( if stamp: args.add_all([ctx.info_file, ctx.version_file], format_each = "--stamp-info-file=%s") inputs += [ctx.info_file, ctx.version_file] - if ctx.attr.use_legacy_join_layers: - _add_join_layers_py_args(args, inputs, images) - else: - _add_join_layers_go_args(args, inputs, images) + _add_join_layers_args(args, inputs, images) ctx.actions.run( - executable = ctx.executable._join_layers_py if ctx.attr.use_legacy_join_layers else ctx.executable._join_layers_go, + executable = ctx.executable._join_layers, arguments = [args], tools = inputs, outputs = [output], @@ -332,20 +295,9 @@ tools = { default = Label("//container:incremental_load_template"), allow_single_file = True, ), - "use_legacy_join_layers": attr.bool( - default = False, - doc = "Use the legacy python join_layers.py to build the image tarball." + - "Uses the experimental Go implementation when set to false.", - ), - "_join_layers_go": attr.label( + "_join_layers": attr.label( default = Label("//container/go/cmd/join_layers"), cfg = "host", executable = True, ), - "_join_layers_py": attr.label( - default = Label("//container:join_layers"), - cfg = "host", - executable = True, - allow_files = True, - ), } diff --git a/container/utils.py b/container/utils.py deleted file mode 100644 index 0e3d8e406..000000000 --- a/container/utils.py +++ /dev/null @@ -1,29 +0,0 @@ -# Copyright 2015 The Bazel Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""This package contains various functions used when building containers.""" - - -def ExtractValue(value): - """Return the contents of a file point to by value if it starts with an @. - - Args: - value: The possible filename to extract or a string. - - Returns: - The content of the file if value starts with an @, or the passed value. - """ - if value.startswith('@'): - with open(value[1:], 'r') as f: - value = f.read() - return value diff --git a/testdata/BUILD b/testdata/BUILD index 053d1139a..d836d17e2 100644 --- a/testdata/BUILD +++ b/testdata/BUILD @@ -414,12 +414,6 @@ container_flatten( image = ":link_with_files_base", ) -container_flatten( - name = "flat_go", - image = ":link_with_files_base", - use_legacy_flattener = False, -) - container_image( name = "image_with_bar", files = [ @@ -440,7 +434,6 @@ container_image( container_flatten( name = "flatten_with_tarball_base", image = ":image_with_bar_and_baz", - use_legacy_flattener = False, ) docker_push( diff --git a/tests/container/BUILD b/tests/container/BUILD index b6bc47eea..89578c056 100644 --- a/tests/container/BUILD +++ b/tests/container/BUILD @@ -193,14 +193,6 @@ container_image( operating_system = "windows", ) -container_image( - name = "basic_windows_image_go_join_layers", - base = ":import_windows_base_image", - cmd = ["echo bar"], - operating_system = "windows", - use_legacy_join_layers = False, -) - container_test( name = "basic_windows_image_test", configs = ["//tests/container/configs:windows_image.yaml"], @@ -208,13 +200,6 @@ container_test( image = ":basic_windows_image", ) -container_test( - name = "basic_windows_image_test_go_join_layers", - configs = ["//tests/container/configs:windows_image.yaml"], - driver = "tar", - image = ":basic_windows_image_go_join_layers", -) - # The following targets are not imported. # These images are only available and tested on GitHub. # BEGIN_DO_NOT_IMPORT @@ -776,7 +761,6 @@ container_image( "//testdata:bar", "//testdata:foo", ], - use_legacy_join_layers = False, ) container_test( diff --git a/tests/contrib/BUILD b/tests/contrib/BUILD index 781bb4e1b..981614abe 100644 --- a/tests/contrib/BUILD +++ b/tests/contrib/BUILD @@ -40,27 +40,6 @@ rename_image( new_tag = "new_image_tag", ) -sh_test( - name = "rename_image_go_test", - srcs = ["rename_image_test.sh"], - args = [ - "$(location :renamed_image_go.tar)", - "new_image_registry.io/new_image_repo:new_image_tag", - ], - data = [ - ":renamed_image_go.tar", - ], -) - -# Test image renaming using container_bundle using the Go implementation. -container_bundle( - name = "renamed_image_go", - images = { - "new_image_registry.io/new_image_repo:new_image_tag": "@distroless_base//image", - }, - use_legacy_join_layers = False, -) - alias( name = "duplicate1", actual = "@distroless_fixed_id//image",