implement container_layer rule.

README.md

@@ -988,11 +988,163 @@ An executable rule that pushes a Docker image to a Docker registry on `bazel run
   </tbody>
 </table>
 
+<a name="container_layer"></a>
+## container_layer
+
+```python
+container_layer(data_path, directory, files, mode, tars, debs, symlinks, env)
+```
+
+A rule that assembles data into a tarball which can be use as in `layers` attr in `container_image` rule.
+
+<table class="table table-condensed table-bordered table-implicit">
+  <colgroup>
+    <col class="col-param" />
+    <col class="param-description" />
+  </colgroup>
+  <thead>
+    <tr>
+      <th colspan="2">Implicit output targets</th>
+    </tr>
+  </thead>
+  <tbody>
+    <tr>
+      <td><code><i>name</i>-layer.tar</code></td>
+      <td>
+        <code>A tarball of current layer</code>
+        <p>
+            A data tarball corresponding to the layer.
+        </p>
+      </td>
+    </tr>
+  </tbody>
+</table>
+
+<table class="table table-condensed table-bordered table-params">
+  <colgroup>
+    <col class="col-param" />
+    <col class="param-description" />
+  </colgroup>
+  <thead>
+    <tr>
+      <th colspan="2">Attributes</th>
+    </tr>
+  </thead>
+  <tbody>
+    <tr>
+      <td><code>name</code></td>
+      <td>
+        <code>Name, required</code>
+        <p>A unique name for this rule.</p>
+      </td>
+    </tr>
+    <tr>
+      <td><code>data_path</code></td>
+      <td>
+        <code>String, optional</code>
+        <p>Root path of the files.</p>
+        <p>
+          The directory structure from the files is preserved inside the
+          Docker image, but a prefix path determined by `data_path`
+          is removed from the directory structure. This path can
+          be absolute from the workspace root if starting with a `/` or
+          relative to the rule's directory. A relative path may starts with "./"
+          (or be ".") but cannot use go up with "..". By default, the
+          `data_path` attribute is unused, and all files should have no prefix.
+        </p>
+      </td>
+    </tr>
+    <tr>
+      <td><code>directory</code></td>
+      <td>
+        <code>String, optional</code>
+        <p>Target directory.</p>
+        <p>
+          The directory in which to expand the specified files, defaulting to '/'.
+          Only makes sense accompanying one of files/tars/debs.
+        </p>
+      </td>
+    </tr>
+    <tr>
+      <td><code>files</code></td>
+      <td>
+        <code>List of files, optional</code>
+        <p>File to add to the layer.</p>
+        <p>
+          A list of files that should be included in the Docker image.
+        </p>
+      </td>
+    </tr>
+    <tr>
+      <td><code>mode</code></td>
+      <td>
+        <code>String, default to 0555</code>
+        <p>
+          Set the mode of files added by the <code>files</code> attribute.
+        </p>
+      </td>
+    </tr>
+    <tr>
+      <td><code>tars</code></td>
+      <td>
+        <code>List of files, optional</code>
+        <p>Tar file to extract in the layer.</p>
+        <p>
+          A list of tar files whose content should be in the Docker image.
+        </p>
+      </td>
+    </tr>
+    <tr>
+      <td><code>debs</code></td>
+      <td>
+        <code>List of files, optional</code>
+        <p>Debian package to install.</p>
+        <p>
+          A list of debian packages that will be installed in the Docker image.
+        </p>
+      </td>
+    </tr>
+    <tr>
+      <td><code>symlinks</code></td>
+      <td>
+        <code>Dictionary, optional</code>
+        <p>Symlinks to create in the Docker image.</p>
+        <p>
+          <code>
+          symlinks = {
+           "/path/to/link": "/path/to/target",
+           ...
+          },
+          </code>
+        </p>
+      </td>
+    </tr>
+    <tr>
+      <td><code>env</code></td>
+      <td>
+        <code>Dictionary from strings to strings, optional</code>
+        <p><a href="https://docs.docker.com/engine/reference/builder/#env">Dictionary
+               from environment variable names to their values when running the
+               Docker image.</a></p>
+        <p>
+          <code>
+          env = {
+            "FOO": "bar",
+            ...
+          },
+          </code>
+        </p>
+	<p>The values of this field support stamp variables.</p>
+      </td>
+    </tr>
+  </tbody>
+</table>
+
 <a name="container_image"></a>
 ## container_image
 
 ```python
-container_image(name, base, data_path, directory, files, legacy_repository_naming, mode, tars, debs, symlinks, entrypoint, cmd, env, labels, ports, volumes, workdir, repository)
+container_image(name, base, data_path, directory, files, legacy_repository_naming, mode, tars, debs, symlinks, entrypoint, cmd, env, labels, ports, volumes, workdir, layers, repository)
 ```
 
 <table class="table table-condensed table-bordered table-implicit">
@@ -1262,6 +1414,14 @@ container_image(name, base, data_path, directory, files, legacy_repository_namin
 	<p>This field supports stamp variables.</p>
       </td>
     </tr>
+    <tr>
+      <td><code>layers</code></td>
+      <td>
+        <code>Label list, optional</code>
+        <p>List of `container_layer` targets. </p>
+        <p>The data from each `container_layer` will be part of container image, and the environment variable will be available in the image as well.</p>
+      </td>
+    </tr>
     <tr>
       <td><code>repository</code></td>
       <td>

cc/image.bzl

@@ -83,5 +83,5 @@ def cc_image(name, base=None, deps=[], layers=[], binary=None, **kwargs):
     base = this_name
 
   visibility = kwargs.get('visibility', None)
-  app_layer(name=name, base=base, binary=binary, layers=layers,
+  app_layer(name=name, base=base, binary=binary, lang_layers=layers,
             visibility=visibility)

container/BUILD

@@ -84,14 +84,18 @@ TEST_TARGETS = [
     "directory_with_tar_base",
     "files_base",
     "files_with_files_base",
+    "files_in_layer_with_files_base",
     "files_with_tar_base",
     "tar_base",
     "tar_with_files_base",
     "tar_with_tar_base",
+    "tars_in_layer_with_tar_base",
     "docker_tarball_base",
+    "layers_with_docker_tarball_base",
     # TODO(mattmoor): Re-enable once archive is visible
     # "generated_tarball",
     "with_env",
+    "layers_with_env",
     "with_double_env",
     "with_label",
     "with_double_label",
@@ -145,7 +149,7 @@ skylark_library(
     name = "bundle",
     srcs = ["bundle.bzl"],
     deps = [
-        ":layers",
+        ":layer_tools",
         "//skylib:label",
     ],
 )
@@ -166,7 +170,7 @@ skylark_library(
     name = "flatten",
     srcs = ["flatten.bzl"],
     deps = [
-        ":layers",
+        ":layer_tools",
         "//skylib:label",
     ],
 )
@@ -175,7 +179,7 @@ skylark_library(
     name = "image",
     srcs = ["image.bzl"],
     deps = [
-        ":layers",
+        ":layer_tools",
         "//skylib:filetype",
         "//skylib:label",
         "//skylib:path",
@@ -189,7 +193,7 @@ skylark_library(
     name = "import",
     srcs = ["import.bzl"],
     deps = [
-        ":layers",
+        ":layer_tools",
         "//skylib:filetype",
         "//skylib:path",
         "//skylib:zip",
@@ -198,8 +202,8 @@ skylark_library(
 )
 
 skylark_library(
-    name = "layers",
-    srcs = ["layers.bzl"],
+    name = "layer_tools",
+    srcs = ["layer_tools.bzl"],
     deps = ["//skylib:path"],
 )
 
@@ -218,7 +222,7 @@ skylark_library(
     name = "push",
     srcs = ["push.bzl"],
     deps = [
-        ":layers",
+        ":layer_tools",
         "//skylib:path",
     ],
 )

container/bundle.bzl

@@ -18,7 +18,7 @@ load(
     _string_to_label = "string_to_label",
 )
 load(
-    "//container:layers.bzl",
+    "//container:layer_tools.bzl",
     _assemble_image = "assemble",
     _get_layers = "get_from_target",
     _incr_load = "incremental_load",

container/container.bzl

@@ -16,6 +16,7 @@
 load("//container:bundle.bzl", "container_bundle")
 load("//container:flatten.bzl", "container_flatten")
 load("//container:image.bzl", "container_image", "image")
+load("//container:layer.bzl", "container_layer")
 load("//container:import.bzl", "container_import")
 load("//container:load.bzl", "container_load")
 load("//container:pull.bzl", "container_pull")

container/flatten.bzl

@@ -14,7 +14,7 @@
 """A rule to flatten container images."""
 
 load(
-    "//container:layers.bzl",
+    "//container:layer_tools.bzl",
     _get_layers = "get_from_target",
     _layer_tools = "tools",
 )

container/image.bzl

@@ -59,12 +59,17 @@ load(
     _string_to_label = "string_to_label",
 )
 load(
-    "//container:layers.bzl",
+    "//container:layer_tools.bzl",
     _assemble_image = "assemble",
     _get_layers = "get_from_target",
     _incr_load = "incremental_load",
     _layer_tools = "tools",
 )
+load(
+    "//container:layer.bzl",
+    "LayerInfo",
+    _layer = "layer",
+)
 load(
     "//skylib:path.bzl",
     "dirname",
@@ -77,76 +82,15 @@ load(
     _serialize_dict = "dict_to_associative_list",
 )
 
-def magic_path(ctx, f):
-  # Right now the logic this uses is a bit crazy/buggy, so to support
-  # bug-for-bug compatibility in the foo_image rules, expose the logic.
-  # See also: https://github.com/bazelbuild/rules_docker/issues/106
-  # See also: https://groups.google.com/forum/#!topic/bazel-discuss/1lX3aiTZX3Y
-
-  if ctx.attr.data_path:
-    # If data_prefix is specified, then add files relative to that.
-    data_path = _join_path(
-        dirname(ctx.outputs.out.short_path),
-        _canonicalize_path(ctx.attr.data_path))
-    return strip_prefix(f.short_path, data_path)
-  else:
-    # Otherwise, files are added without a directory prefix at all.
-    return f.basename
-
-def _build_layer(ctx, files=None, file_map=None, empty_files=None,
-                 directory=None, symlinks=None, debs=None, tars=None):
-  """Build the current layer for appending it the base layer.
-
-  Args:
-    files: File list, overrides ctx.files.files
-    directory: str, overrides ctx.attr.directory
-    symlinks: str Dict, overrides ctx.attr.symlinks
-  """
-
-  layer = ctx.outputs.layer
-  build_layer = ctx.executable.build_layer
-  args = [
-      "--output=" + layer.path,
-      "--directory=" + directory,
-      "--mode=" + ctx.attr.mode,
-  ]
-
-  args += ["--file=%s=%s" % (f.path, magic_path(ctx, f)) for f in files]
-  args += ["--file=%s=%s" % (f.path, path) for (path, f) in file_map.items()]
-  args += ["--empty_file=%s" % f for f in empty_files or []]
-  args += ["--tar=" + f.path for f in tars]
-  args += ["--deb=" + f.path for f in debs]
-  for k in symlinks:
-    if ':' in k:
-      fail("The source of a symlink cannot contain ':', got: %s" % k)
-  args += ["--link=%s:%s" % (k, symlinks[k])
-           for k in symlinks]
-  arg_file = ctx.new_file(ctx.label.name + ".layer.args")
-  ctx.file_action(arg_file, "\n".join(args))
-
-  ctx.action(
-      executable = build_layer,
-      arguments = ["--flagfile=" + arg_file.path],
-      inputs = files + file_map.values() + tars + debs + [arg_file],
-      outputs = [layer],
-      use_default_shell_env=True,
-      mnemonic="ImageLayer"
-  )
-  return layer, _sha256(ctx, layer)
-
-def _zip_layer(ctx, layer):
-  zipped_layer = _gzip(ctx, layer)
-  return zipped_layer, _sha256(ctx, zipped_layer)
-
 def _get_base_config(ctx):
   if ctx.files.base:
     # The base is the first layer in container_parts if provided.
     l = _get_layers(ctx, ctx.attr.base, ctx.files.base)
     return l.get("config")
 
-def _image_config(ctx, layer_name, entrypoint=None, cmd=None, env=None):
+def _image_config(ctx, layer_names, entrypoint=None, cmd=None, env=None, base_config=None, layer_name=None):
   """Create the configuration for a new container image."""
-  config = ctx.new_file(ctx.label.name + ".config")
+  config = ctx.new_file(ctx.label.name + "." + layer_name + ".config")
 
   label_file_dict = _string_to_label(
       ctx.files.label_files, ctx.attr.label_file_strings)
@@ -182,16 +126,16 @@ def _image_config(ctx, layer_name, entrypoint=None, cmd=None, env=None):
   if ctx.attr.workdir:
     args += ["--workdir=" + ctx.attr.workdir]
 
-  inputs = [layer_name]
-  args += ["--layer=@" + layer_name.path]
+  inputs = layer_names
+  for layer_name in layer_names:
+    args += ["--layer=@" + layer_name.path]
 
   if ctx.attr.label_files:
     inputs += ctx.files.label_files
 
-  base = _get_base_config(ctx)
-  if base:
-    args += ["--base=%s" % base.path]
-    inputs += [base]
+  if base_config:
+    args += ["--base=%s" % base_config.path]
+    inputs += [base_config]
 
   if ctx.attr.stamp:
     stamp_inputs = [ctx.info_file, ctx.version_file]
@@ -218,7 +162,7 @@ def _repository_name(ctx):
 
 def _impl(ctx, files=None, file_map=None, empty_files=None, directory=None,
           entrypoint=None, cmd=None, symlinks=None, output=None, env=None,
-          debs=None, tars=None):
+          layers=None, debs=None, tars=None):
   """Implementation for the container_image rule.
 
   Args:
@@ -232,46 +176,47 @@ def _impl(ctx, files=None, file_map=None, empty_files=None, directory=None,
     symlinks: str Dict, overrides ctx.attr.symlinks
     output: File to use as output for script to load docker image
     env: str Dict, overrides ctx.attr.env
+    layers: label List, overrides ctx.attr.layers
     debs: File list, overrides ctx.files.debs
     tars: File list, overrides ctx.files.tars
   """
-
-  file_map = file_map or {}
-  files = files or ctx.files.files
-  empty_files = empty_files or ctx.attr.empty_files
-  directory = directory or ctx.attr.directory
-  entrypoint = entrypoint or ctx.attr.entrypoint
-  cmd = cmd or ctx.attr.cmd
-  symlinks = symlinks or ctx.attr.symlinks
+  entrypoint=entrypoint or ctx.attr.entrypoint
+  cmd=cmd or ctx.attr.cmd
   output = output or ctx.outputs.executable
-  env = env or ctx.attr.env
-  debs = debs or ctx.files.debs
-  tars = tars or ctx.files.tars
-
-  # Generate the unzipped filesystem layer, and its sha256 (aka diff_id).
-  unzipped_layer, diff_id = _build_layer(ctx, files=files, file_map=file_map,
-                                         empty_files=empty_files,
-                                         directory=directory, symlinks=symlinks,
-                                         debs=debs, tars=tars)
 
-  # Generate the zipped filesystem layer, and its sha256 (aka blob sum)
-  zipped_layer, blob_sum = _zip_layer(ctx, unzipped_layer)
+  # composite a layer from the container_image rule attrs,
+  image_layer = _layer.implementation(ctx=ctx, files=files,
+                                      file_map=file_map,
+                                      empty_files=empty_files,
+                                      directory=directory,
+                                      symlinks=symlinks,
+                                      debs=debs, tars=tars,
+                                      env=env)
 
-  # Generate the new config using the attributes specified and the diff_id
-  config_file, config_digest = _image_config(
-      ctx, diff_id, entrypoint=entrypoint, cmd=cmd, env=env)
-
-  # Construct a temporary name based on the build target.
-  tag_name = _repository_name(ctx) + ":" + ctx.label.name
+  layer_providers= layers or ctx.attr.layers
+  layers = [provider[LayerInfo] for provider in layer_providers] + image_layer
 
   # Get the layers and shas from our base.
   # These are ordered as they'd appear in the v2.2 config,
   # so they grow at the end.
   parent_parts = _get_layers(ctx, ctx.attr.base, ctx.files.base)
-  zipped_layers = parent_parts.get("zipped_layer", []) + [zipped_layer]
-  shas = parent_parts.get("blobsum", []) + [blob_sum]
-  unzipped_layers = parent_parts.get("unzipped_layer", []) + [unzipped_layer]
-  diff_ids = parent_parts.get("diff_id", []) + [diff_id]
+  zipped_layers = parent_parts.get("zipped_layer", []) + [layer.zipped_layer for layer in layers]
+  shas = parent_parts.get("blobsum", [])  + [layer.blob_sum for layer in layers]
+  unzipped_layers = parent_parts.get("unzipped_layer", []) + [layer.unzipped_layer for layer in layers]
+  layer_diff_ids = [layer.diff_id for layer in layers]
+  diff_ids = parent_parts.get("diff_id", []) + layer_diff_ids
+
+  # Get the config for the base layer
+  config_file = _get_base_config(ctx)
+  # Generate the new config layer by layer, using the attributes specified and the diff_id
+  for i, layer in enumerate(layers):
+    config_file, config_digest = _image_config(
+        ctx, [layer_diff_ids[i]],
+        entrypoint=entrypoint, cmd=cmd, env=layer.env,
+        base_config=config_file, layer_name=str(i), )
+
+  # Construct a temporary name based on the build target.
+  tag_name = _repository_name(ctx) + ":" + ctx.label.name
 
   # These are the constituent parts of the Container image, which each
   # rule in the chain must preserve.
@@ -313,13 +258,8 @@ def _impl(ctx, files=None, file_map=None, empty_files=None, directory=None,
                 files = depset([ctx.outputs.layer]),
                 container_parts = container_parts)
 
-_attrs = dict({
+_attrs = dict(_layer.attrs.items() + {
     "base": attr.label(allow_files = container_filetype),
-    "data_path": attr.string(),
-    "directory": attr.string(default = "/"),
-    "tars": attr.label_list(allow_files = tar_filetype),
-    "debs": attr.label_list(allow_files = deb_filetype),
-    "files": attr.label_list(allow_files = True),
     "legacy_repository_naming": attr.bool(default = False),
     # TODO(mattmoor): Default this to False.
     "legacy_run_behavior": attr.bool(default = True),
@@ -329,30 +269,21 @@ _attrs = dict({
     "docker_run_flags": attr.string(
         default = "-i --rm --network=host",
     ),
-    "mode": attr.string(default = "0555"),  # 0555 == a+rx
-    "symlinks": attr.string_dict(),
-    "entrypoint": attr.string_list(),
-    "cmd": attr.string_list(),
     "user": attr.string(),
-    "env": attr.string_dict(),
     "labels": attr.string_dict(),
+    "cmd": attr.string_list(),
+    "entrypoint": attr.string_list(),
     "ports": attr.string_list(),  # Skylark doesn't support int_list...
     "volumes": attr.string_list(),
     "workdir": attr.string(),
+    "layers": attr.label_list(providers = [LayerInfo]),
     "repository": attr.string(default = "bazel"),
     "stamp": attr.bool(default = False),
     # Implicit/Undocumented dependencies.
     "label_files": attr.label_list(
         allow_files = True,
     ),
     "label_file_strings": attr.string_list(),
-    "empty_files": attr.string_list(),
-    "build_layer": attr.label(
-        default = Label("//container:build_tar"),
-        cfg = "host",
-        executable = True,
-        allow_files = True,
-    ),
     "create_image_config": attr.label(
         default = Label("//container:create_image_config"),
         cfg = "host",
@@ -450,6 +381,9 @@ def _validate_command(name, argument):
 #          ...
 #      },
 #
+#      # Other layers built from container_layer rule
+#      layers = [":c-lang-layer", ":java-lang-layer", ...]
+#
 #      # https://docs.docker.com/engine/reference/builder/#entrypoint
 #      entrypoint="...", or
 #      entrypoint=[...],            -- exec form

container/image_test.py

@@ -66,6 +66,14 @@ def test_files_with_file_base(self):
       self.assertEqual(2, len(img.fs_layers()))
       self.assertTopLayerContains(img, ['.', './bar'])
 
+  def test_files_in_layer_with_file_base(self):
+    with TestImage('files_in_layer_with_files_base') as img:
+      self.assertDigest(img, '4b008d8241bdbbe930d72d8f0ee7b61d11561946db0fd52d02dbcb8842b3a958')
+      self.assertEqual(3, len(img.fs_layers()))
+      self.assertLayerNContains(img, 2, ['.', './foo'])
+      self.assertLayerNContains(img, 1, ['.', './baz'])
+      self.assertLayerNContains(img, 0, ['.', './bar'])
+
   def test_tar_base(self):
     with TestImage('tar_base') as img:
       self.assertDigest(img, 'df626b895bc8c7b18e6615ac09ffbd0693268a24a817a94030ff88c37602147e')
@@ -83,6 +91,17 @@ def test_tar_with_tar_base(self):
         './asdf', './usr', './usr/bin',
         './usr/bin/miraclegrow'])
 
+  def test_tars_in_layer_with_tar_base(self):
+    with TestImage('tars_in_layer_with_tar_base') as img:
+      self.assertDigest(img, '80f850359828f763ae544f9b7725f89755f1a28a80738a514735becae60924af')
+      self.assertEqual(3, len(img.fs_layers()))
+      self.assertTopLayerContains(img, [
+        './asdf', './usr', './usr/bin',
+        './usr/bin/miraclegrow'])
+      self.assertLayerNContains(img, 1, ['.', './three', './three/three'])
+      self.assertLayerNContains(img, 2, [
+          './usr', './usr/bin', './usr/bin/unremarkabledeath'])
+
   def test_directory_with_tar_base(self):
     with TestImage('directory_with_tar_base') as img:
       self.assertDigest(img, 'ad11d32eb4b2d3abd01ce599a4200b20cf1c545ce870b174d28fd717c558a58c')
@@ -119,6 +138,14 @@ def test_docker_tarball_base(self):
       self.assertEqual(3, len(img.fs_layers()))
       self.assertTopLayerContains(img, ['.', './foo'])
 
+  def test_layers_with_docker_tarball_base(self):
+    with TestImage('layers_with_docker_tarball_base') as img:
+      self.assertDigest(img, '927b3b98286e16727e2144152efb90f7394b0ad37d16668d40ce22b9e49debf9')
+      self.assertEqual(5, len(img.fs_layers()))
+      self.assertTopLayerContains(img, ['.', './foo'])
+      self.assertLayerNContains(img, 1, ['.', './three', './three/three'])
+      self.assertLayerNContains(img, 2, ['.', './baz'])
+
   def test_base_with_entrypoint(self):
     with TestImage('base_with_entrypoint') as img:
       self.assertDigest(img, '813cb4af1c3f73cc2b5f837a61dca6a62335b87e5cd762e780286ca99f71ac83')
@@ -160,6 +187,12 @@ def test_with_env(self):
       self.assertEqual(2, len(img.fs_layers()))
       self.assertConfigEqual(img, 'Env', ['bar=blah blah blah', 'foo=/asdf'])
 
+  def test_layers_with_env(self):
+    with TestImage('layers_with_env') as img:
+      self.assertDigest(img, 'ecab0f39b4726e69c62747ce4c1662f697060eef23a26db719a47ea379b77d7f')
+      self.assertEqual(3, len(img.fs_layers()))
+      self.assertConfigEqual(img, 'Env', ['PATH=$PATH:/tmp/a:/tmp/b:/tmp/c', 'a=b', 'x=y'])
+
   def test_dummy_repository(self):
     # We allow users to specify an alternate repository name instead of 'bazel/'
     # to prefix their image names.

container/import.bzl

@@ -29,7 +29,7 @@ load(
     _gzip = "gzip",
 )
 load(
-    "//container:layers.bzl",
+    "//container:layer_tools.bzl",
     _assemble_image = "assemble",
     _incr_load = "incremental_load",
     _layer_tools = "tools",

container/layer.bzl

@@ -0,0 +1,187 @@
+# Copyright 2017 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Rule for building a Container layer."""
+
+load(
+    "//skylib:filetype.bzl",
+    container_filetype = "container",
+    deb_filetype = "deb",
+    tar_filetype = "tar",
+)
+load(
+    "@bazel_tools//tools/build_defs/hash:hash.bzl",
+    _hash_tools = "tools",
+    _sha256 = "sha256",
+)
+load(
+    "//skylib:zip.bzl",
+    _gzip = "gzip",
+)
+load(
+    "//container:layer_tools.bzl",
+    _assemble_image = "assemble",
+    _get_layers = "get_from_target",
+    _incr_load = "incremental_load",
+    _layer_tools = "tools",
+)
+load(
+    "//skylib:path.bzl",
+    "dirname",
+    "strip_prefix",
+    _canonicalize_path = "canonicalize",
+    _join_path = "join",
+)
+
+def _magic_path(ctx, f):
+  # Right now the logic this uses is a bit crazy/buggy, so to support
+  # bug-for-bug compatibility in the foo_image rules, expose the logic.
+  # See also: https://github.com/bazelbuild/rules_docker/issues/106
+  # See also: https://groups.google.com/forum/#!topic/bazel-discuss/1lX3aiTZX3Y
+
+  if ctx.attr.data_path:
+    # If data_prefix is specified, then add files relative to that.
+    data_path = _join_path(
+        dirname(ctx.outputs.layer.short_path),
+        _canonicalize_path(ctx.attr.data_path))
+    return strip_prefix(f.short_path, data_path)
+  else:
+    # Otherwise, files are added without a directory prefix at all.
+    return f.basename
+
+def build_layer(ctx, files=None, file_map=None, empty_files=None,
+                 directory=None, symlinks=None, debs=None, tars=None):
+  """Build the current layer for appending it to the base layer"""
+  layer = ctx.outputs.layer
+  build_layer_exec = ctx.executable.build_layer
+  args = [
+      "--output=" + layer.path,
+      "--directory=" + directory,
+      "--mode=" + ctx.attr.mode,
+  ]
+
+  args += ["--file=%s=%s" % (f.path, _magic_path(ctx, f)) for f in files]
+  args += ["--file=%s=%s" % (f.path, path) for (path, f) in file_map.items()]
+  args += ["--empty_file=%s" % f for f in empty_files or []]
+  args += ["--tar=" + f.path for f in tars]
+  args += ["--deb=" + f.path for f in debs]
+  for k in symlinks:
+    if ":" in k:
+      fail("The source of a symlink cannot container ':', got: %s" % k)
+  args += ["--link=%s:%s" % (k, symlinks[k]) for k in symlinks]
+  arg_file = ctx.new_file(ctx.label.name + "-layer.args")
+  ctx.file_action(arg_file, "\n".join(args))
+  ctx.action(
+      executable = build_layer_exec,
+      arguments = ["--flagfile=" + arg_file.path],
+      inputs = files + file_map.values() + tars + debs + [arg_file],
+      outputs = [layer],
+      use_default_shell_env = True,
+      mnemonic="ImageLayer",
+  )
+  return layer, _sha256(ctx, layer)
+
+def zip_layer(ctx, layer):
+  zipped_layer = _gzip(ctx, layer)
+  return zipped_layer, _sha256(ctx, zipped_layer)
+
+# A provider containing information needed in container_image and other rules.
+LayerInfo = provider(fields = [
+    "zipped_layer",
+    "blob_sum",
+    "unzipped_layer",
+    "diff_id",
+    "env",
+])
+
+def _impl(ctx, files=None, file_map=None, empty_files=None, directory=None,
+          symlinks=None, output=None, debs=None, tars=None, env=None):
+  """Implementation for the container_layer rule.
+
+  Args:
+    ctx: The bazel rule context
+    files: File list, overrides ctx.files.files
+    file_map: Dict[str, File], defaults to {}
+    empty_files: str list, overrides ctx.attr.empty_files
+    directory: str, overrides ctx.attr.directory
+    symlinks: str Dict, overrides ctx.attr.symlinks
+    env: str Dict, overrides ctx.attr.env
+    debs: File list, overrides ctx.files.debs
+    tars: File list, overrides ctx.files.tars
+  """
+  file_map = file_map or {}
+  files = files or ctx.files.files
+  empty_files = empty_files or ctx.attr.empty_files
+  directory = directory or ctx.attr.directory
+  symlinks = symlinks or ctx.attr.symlinks
+  debs = debs or ctx.files.debs
+  tars = tars or ctx.files.tars
+
+  # Generate the unzipped filesystem layer, and its sha256 (aka diff_id)
+  unzipped_layer, diff_id = build_layer(ctx, files=files, file_map=file_map,
+                                        empty_files=empty_files,
+                                        directory=directory, symlinks=symlinks,
+                                        debs=debs, tars=tars)
+  # Generate the zipped filesystem layer, and its sha256 (aka blob sum)
+  zipped_layer, blob_sum = zip_layer(ctx, unzipped_layer)
+
+  # Returns constituent parts of the Container layer as provider:
+  # - in container_image rule, we need to use all the following information,
+  #   e.g. zipped_layer etc., to assemble the complete container image.
+  # - in order to expose information from container_layer rule to container_image
+  #   rule, they need to be packaged into a provider, see:
+  #   https://docs.bazel.build/versions/master/skylark/rules.html#providers
+  return [LayerInfo(zipped_layer=zipped_layer,
+                    blob_sum=blob_sum,
+                    unzipped_layer=unzipped_layer,
+                    diff_id=diff_id,
+                    env=env or ctx.attr.env)]
+
+_layer_attrs = dict({
+    "data_path": attr.string(),
+    "directory": attr.string(default = "/"),
+    "files": attr.label_list(allow_files = True),
+    "mode": attr.string(default = "0555"),  # 0555 == a+rx
+    "tars": attr.label_list(allow_files = tar_filetype),
+    "debs": attr.label_list(allow_files = deb_filetype),
+    "symlinks": attr.string_dict(),
+    "env": attr.string_dict(),
+    # Implicit/Undocumented dependencies.
+    "empty_files": attr.string_list(),
+    "build_layer": attr.label(
+        default = Label("//container:build_tar"),
+        cfg = "host",
+        executable = True,
+        allow_files = True,
+    ),
+}.items() + _hash_tools.items() + _layer_tools.items())
+
+_layer_outputs = {
+    "layer": "%{name}-layer.tar",
+}
+
+layer = struct(
+    attrs = _layer_attrs,
+    outputs = _layer_outputs,
+    implementation = _impl,
+)
+
+container_layer_ = rule(
+    attrs = _layer_attrs,
+    executable = False,
+    outputs = _layer_outputs,
+    implementation = _impl,
+)
+
+def container_layer(**kwargs):
+  container_layer_(**kwargs)

container/push.bzl

@@ -22,7 +22,7 @@ load(
     "runfile",
 )
 load(
-    "//container:layers.bzl",
+    "//container:layer_tools.bzl",
     _get_layers = "get_from_target",
     _layer_tools = "tools",
 )

d/image.bzl

@@ -56,5 +56,5 @@ def d_image(name, base=None, deps=[], layers=[], binary=None, **kwargs):
     base = this_name
 
   visibility = kwargs.get('visibility', None)
-  app_layer(name=name, base=base, binary=binary, layers=layers,
+  app_layer(name=name, base=base, binary=binary, lang_layers=layers,
             visibility=visibility)

docker/docker.bzl

@@ -21,6 +21,7 @@ load(
     docker_flatten = "container_flatten",
     docker_image = "container_image",
     docker_import = "container_import",
+    docker_layer = "container_layer",
     docker_load = "container_load",
     docker_pull = "container_pull",
     docker_repositories = "repositories",

go/image.bzl

@@ -87,5 +87,5 @@ def go_image(name, base=None, deps=[], layers=[], binary=None, **kwargs):
     base = this_name
 
   visibility = kwargs.get('visibility', None)
-  app_layer(name=name, base=base, binary=binary, layers=layers,
+  app_layer(name=name, base=base, binary=binary, lang_layers=layers,
             visibility=visibility)

groovy/image.bzl

@@ -68,7 +68,7 @@ def groovy_image(name, base=None, main_class=None,
   visibility = kwargs.get('visibility', None)
   jar_app_layer(name=name, base=base, binary=binary_name,
                  main_class=main_class, jvm_flags=jvm_flags,
-                 deps=deps, layers=layers, visibility=visibility)
+                 deps=deps, jar_layers=layers, visibility=visibility)
 
 def repositories():
   _repositories()

java/image.bzl

@@ -139,7 +139,7 @@ def _jar_app_layer_impl(ctx):
   """Appends the app layer with all remaining runfiles."""
 
   available = depset()
-  for jar in ctx.attr.layers:
+  for jar in ctx.attr.jar_layers:
     available += java_files(jar)
 
   # We compute the set of unavailable stuff by walking deps
@@ -188,7 +188,7 @@ jar_app_layer = rule(
         "binary": attr.label(mandatory = True),
         # The full list of dependencies that have their own layers
         # factored into our base.
-        "layers": attr.label_list(),
+        "jar_layers": attr.label_list(),
         # The rest of the dependencies.
         "deps": attr.label_list(),
         "runtime_deps": attr.label_list(),
@@ -247,7 +247,7 @@ def java_image(name, base=None, main_class=None,
   visibility = kwargs.get('visibility', None)
   jar_app_layer(name=name, base=base, binary=binary_name,
                  main_class=main_class, jvm_flags=jvm_flags,
-                 deps=deps, runtime_deps=runtime_deps, layers=layers,
+                 deps=deps, runtime_deps=runtime_deps, jar_layers=layers,
                  visibility=visibility)
 
 def _war_dep_layer_impl(ctx):
@@ -286,7 +286,7 @@ def _war_app_layer_impl(ctx):
   """Appends the app layer with all remaining runfiles."""
 
   available = depset()
-  for jar in ctx.attr.layers:
+  for jar in ctx.attr.jar_layers:
     available += java_files(jar)
 
   # This is based on rules_appengine's WAR rules.
@@ -307,7 +307,7 @@ _war_app_layer = rule(
         "library": attr.label(mandatory = True),
         # The full list of dependencies that have their own layers
         # factored into our base.
-        "layers": attr.label_list(),
+        "jar_layers": attr.label_list(),
         # The base image on which to overlay the dependency layers.
         "base": attr.label(mandatory = True),
         "entrypoint": attr.string_list(default = []),
@@ -352,5 +352,5 @@ def war_image(name, base=None, deps=[], layers=[], **kwargs):
     base = this_name
 
   visibility = kwargs.get('visibility', None)
-  _war_app_layer(name=name, base=base, library=library_name, layers=layers,
+  _war_app_layer(name=name, base=base, library=library_name, jar_layers=layers,
                  visibility=visibility)

lang/image.bzl

@@ -162,7 +162,7 @@ def _app_layer_impl(ctx, runfiles=None, emptyfiles=None):
   # Compute the set of runfiles that have been made available
   # in our base image, tracking absolute paths.
   available = {}
-  for dep in ctx.attr.layers:
+  for dep in ctx.attr.lang_layers:
     available.update({
         _final_file_path(ctx, f): layer_file_path(ctx, f)
         for f in runfiles(dep)
@@ -229,7 +229,7 @@ app_layer = rule(
         ),
         # The full list of dependencies that have their own layers
         # factored into our base.
-        "layers": attr.label_list(allow_files = True),
+        "lang_layers": attr.label_list(allow_files = True),
         # The base image on which to overlay the dependency layers.
         "base": attr.label(mandatory = True),
         "entrypoint": attr.string_list(default = []),

nodejs/image.bzl

@@ -132,4 +132,4 @@ def nodejs_image(name, base=None, data=[], layers=[],
   app_layer(name=name, base=base, entrypoint=['sh', '-c'],
             # Node.JS hates symlinks.
             agnostic_dep_layout=False,
-            binary=binary_name, layers=layers, visibility=visibility)
+            binary=binary_name, lang_layers=layers, visibility=visibility)

oci/oci.bzl

@@ -20,6 +20,7 @@ load(
     oci_flatten = "container_flatten",
     oci_image = "container_image",
     oci_import = "container_import",
+    oci_layer = "container_layer",
     oci_load = "container_load",
     oci_pull = "container_pull",
 )

python/image.bzl

@@ -82,4 +82,4 @@ def py_image(name, base=None, deps=[], layers=[], **kwargs):
 
   visibility = kwargs.get('visibility', None)
   app_layer(name=name, base=base, entrypoint=['/usr/bin/python'],
-            binary=binary_name, layers=layers, visibility=visibility)
+            binary=binary_name, lang_layers=layers, visibility=visibility)

python3/image.bzl

@@ -82,4 +82,4 @@ def py3_image(name, base=None, deps=[], layers=[], **kwargs):
 
   visibility = kwargs.get('visibility', None)
   app_layer(name=name, base=base, entrypoint=['/usr/bin/python'],
-            binary=binary_name, layers=layers, visibility=visibility)
+            binary=binary_name, lang_layers=layers, visibility=visibility)

rust/image.bzl

@@ -56,5 +56,5 @@ def rust_image(name, base=None, deps=[], layers=[], binary=None, **kwargs):
     base = this_name
 
   visibility = kwargs.get('visibility', None)
-  app_layer(name=name, base=base, binary=binary, layers=layers,
+  app_layer(name=name, base=base, binary=binary, lang_layers=layers,
             visibility=visibility)

scala/image.bzl

@@ -55,7 +55,7 @@ def scala_image(name, base=None, main_class=None,
   visibility = kwargs.get('visibility', None)
   jar_app_layer(name=name, base=base, binary=binary_name,
                  main_class=main_class, jvm_flags=jvm_flags,
-                 deps=deps, runtime_deps=runtime_deps, layers=layers,
+                 deps=deps, runtime_deps=runtime_deps, jar_layers=layers,
                  visibility=visibility)
 
 def repositories():

testdata/BUILD

@@ -19,6 +19,7 @@ load(
     "container_bundle",
     "container_flatten",
     "container_image",
+    "container_layer",
     "container_import",
 )
 load("//testdata:utils.bzl", "generate_deb")
@@ -83,6 +84,20 @@ container_image(
     mode = "0644",
 )
 
+container_image(
+    name = "files_in_layer_with_files_base",
+    base = ":files_base",
+    files = ["bar"],
+    layers = [":files_in_layer"],
+    mode = "0644",
+)
+
+container_layer(
+    name = "files_in_layer",
+    files = ["baz"],
+    mode = "0644",
+)
+
 container_image(
     name = "tar_base",
     tars = ["one.tar"],
@@ -94,6 +109,19 @@ container_image(
     tars = ["two.tar"],
 )
 
+container_image(
+    name = "tars_in_layer_with_tar_base",
+    base = ":tar_base",
+    layers = [":tars_in_layer"],
+    tars = ["two.tar"],
+)
+
+container_layer(
+    name = "tars_in_layer",
+    directory = "/three",
+    tars = ["three.tar"],
+)
+
 container_image(
     name = "directory_with_tar_base",
     base = ":tar_base",
@@ -127,6 +155,17 @@ container_image(
     mode = "0644",
 )
 
+container_image(
+    name = "layers_with_docker_tarball_base",
+    base = "@pause_tar//image",
+    files = ["foo"],
+    layers = [
+        ":files_in_layer",
+        ":tars_in_layer",
+    ],
+    mode = "0644",
+)
+
 # TODO(mattmoor): Test scalar entrypoint
 container_image(
     name = "base_with_entrypoint",
@@ -222,6 +261,24 @@ container_image(
     },
 )
 
+container_image(
+    name = "layers_with_env",
+    base = ":base_with_volume",
+    env = {
+        "PATH": "$PATH:/tmp/b:/tmp/c",
+        "x": "y",
+    },
+    layers = [":env_layer"],
+)
+
+container_layer(
+    name = "env_layer",
+    env = {
+        "PATH": "$PATH:/tmp/a",
+        "a": "b",
+    },
+)
+
 container_image(
     name = "with_label",
     base = ":base_with_volume",

testdata/baz

@@ -0,0 +1 @@
+blah