feat: Make GAPIC Bazel rules production ready (#402)

Shoud fix #400 and #390, plus a bunch of other not-yet-opened issues.

This includes:
1) Fix long initial load time (5+ min). This was caused by python_rules buildling `grpcio` dependency from sources in one core (which was super slow). Switched to using bazel-native `"@com_github_grpc_grpc//src/python/grpcio/grpc:grpcio"` target instead, which is not only much faster, but is also already used in googleapis, so there is no additional cost for reusing it in microgenerator rules.

2) Properly handle `pandoc` dependency (platform-sepcific version of pandoc is properly pulled by bazel itself using toolchains).

3) Add simplistic version of the `py_gapic_assembly_pkg` rule, to make output of microgenerator compatible with `GAPICBazel` class in synthtool.

4) Add `plugin_args` argument for python_gapic_library rule to pass custom argumetns to the plugin (similar to PHP rules).

5) Add compatibility with  `python3.6` runtime (otherwise `python3.7` is minimum because of dependency on `dataclasses` module). Python 3.6 compatibility can be enabled by adding `--define=gapic_gen_python=3.6` command line argument to `bazel build` command. 

6) Add support for Python runtimes installed with `pyenv`. To tell bazel using Python3 installed via pyenv add `--extra_toolchains=@gapic_generator_python//:pyenv3_toolchain` argument to `bazel build` command.

Author: vam-google

BUILD.bazel

@@ -1,21 +1,73 @@
+load("//:gapic_generator_python.bzl", "pandoc_binary", "pandoc_toolchain")
 load("@gapic_generator_python_pip_deps//:requirements.bzl", "requirement")
+load("@bazel_tools//tools/python:toolchain.bzl", "py_runtime_pair")
+
+toolchain_type(
+    name = "pandoc_toolchain_type",
+    visibility = ["//visibility:public"],
+)
+
+pandoc_toolchain(
+    exec_compatible_with = [
+        "@bazel_tools//platforms:linux",
+        "@bazel_tools//platforms:x86_64",
+    ],
+    platform = "linux",
+)
+
+pandoc_toolchain(
+    exec_compatible_with = [
+        "@bazel_tools//platforms:osx",
+        "@bazel_tools//platforms:x86_64",
+    ],
+    platform = "macOS",
+)
+
+pandoc_binary(
+    name = "pandoc_binary",
+)
+
+config_setting(
+    name = "gapic_gen_python_3_6",
+    values = {"define": "gapic_gen_python=3.6"},
+)
+
+py_runtime(
+    name = "pyenv3_runtime",
+    interpreter = ":pyenv3wrapper.sh",
+    python_version="PY3",
+)
+
+py_runtime_pair(
+    name = "pyenv3_runtime_pair",
+    py3_runtime = ":pyenv3_runtime",
+)
+
+toolchain(
+    name = "pyenv3_toolchain",
+    toolchain = ":pyenv3_runtime_pair",
+    toolchain_type = "@bazel_tools//tools/python:toolchain_type",
+)
 
 py_binary(
     name = "gapic_plugin",
     srcs = glob(["gapic/**/*.py"]),
-    data = glob(["gapic/**/*.j2"]),
-    main = "gapic/cli/generate.py",
+    data = [":pandoc_binary"] + glob(["gapic/**/*.j2"]),
+    main = "gapic/cli/generate_with_pandoc.py",
+    python_version = "PY3",
     visibility = ["//visibility:public"],
     deps = [
         "@com_google_protobuf//:protobuf_python",
+        "@com_github_grpc_grpc//src/python/grpcio/grpc:grpcio",
         requirement("click"),
         requirement("google-api-core"),
         requirement("googleapis-common-protos"),
-        requirement("grpcio"),
         requirement("jinja2"),
         requirement("MarkupSafe"),
         requirement("pypandoc"),
         requirement("PyYAML"),
-    ],
-    python_version = "PY3",
+    ] + select({
+        ":gapic_gen_python_3_6": [requirement("dataclasses")],
+        "//conditions:default": [],
+    }),
 )

WORKSPACE

@@ -2,9 +2,11 @@ workspace(name = "gapic_generator_python")
 
 load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
 
-#
-# Import rules_python
-#
+http_archive(
+    name = "bazel_skylib",
+    urls = ["https://github.com/bazelbuild/bazel-skylib/releases/download/0.9.0/bazel_skylib-0.9.0.tar.gz"],
+)
+
 http_archive(
     name = "rules_python",
     strip_prefix = "rules_python-748aa53d7701e71101dfd15d800e100f6ff8e5d1",
@@ -22,14 +24,43 @@ pip_repositories()
 #
 # Import gapic-generator-python specific dependencies
 #
-load("//:repositories.bzl", "gapic_generator_python")
+load("//:repositories.bzl",
+    "gapic_generator_python",
+    "gapic_generator_register_toolchains"
+)
 
 gapic_generator_python()
 
+gapic_generator_register_toolchains()
+
 load("@gapic_generator_python_pip_deps//:requirements.bzl", "pip_install")
 
 pip_install()
 
 load("@com_google_protobuf//:protobuf_deps.bzl", "protobuf_deps")
 
 protobuf_deps()
+
+#
+# Import grpc as a native bazel dependency. This avoids duplication and also
+# speeds up loading phase a lot (otherwise python_rules will be building grpcio
+# from sources in a single-core speed, which takes around 5 minutes on a regular
+# workstation)
+#
+load("@com_github_grpc_grpc//bazel:grpc_deps.bzl", "grpc_deps")
+
+grpc_deps()
+
+load("@upb//bazel:repository_defs.bzl", "bazel_version_repository")
+
+bazel_version_repository(
+    name = "bazel_version",
+)
+
+load("@build_bazel_rules_apple//apple:repositories.bzl", "apple_rules_dependencies")
+
+apple_rules_dependencies()
+
+load("@build_bazel_apple_support//lib:repositories.bzl", "apple_support_dependencies")
+
+apple_support_dependencies()

gapic/cli/generate_with_pandoc.py

@@ -0,0 +1,9 @@
+import os
+
+from gapic.cli import generate
+
+if __name__ == '__main__':
+    os.environ['PYPANDOC_PANDOC'] = os.path.join(
+        os.path.abspath(__file__).rsplit("gapic", 1)[0], "pandoc")
+    os.environ['LC_ALL'] = 'C.UTF-8'
+    generate.generate()

gapic_generator_python.bzl

@@ -0,0 +1,62 @@
+def _pandoc_binary_impl(ctx):
+    toolchain = ctx.toolchains["@gapic_generator_python//:pandoc_toolchain_type"]
+    output = ctx.actions.declare_file(ctx.attr.binary_name)
+
+    script = """
+    cp {input} {output}
+    chmod +x {output}
+    """.format(
+        input = toolchain.pandoc.files.to_list()[0].path,
+        output = output.path,
+    )
+    ctx.actions.run_shell(
+        command = script,
+        inputs = toolchain.pandoc.files,
+        outputs = [output],
+    )
+    return [DefaultInfo(files = depset(direct = [output]), executable = output)]
+
+pandoc_binary = rule(
+    attrs = {
+        "binary_name": attr.string(default = "pandoc")
+    },
+    executable = True,
+    toolchains = ["@gapic_generator_python//:pandoc_toolchain_type"],
+    implementation = _pandoc_binary_impl,
+)
+
+#
+# Toolchains
+#
+def _pandoc_toolchain_info_impl(ctx):
+    return [
+        platform_common.ToolchainInfo(
+            pandoc = ctx.attr.pandoc,
+        ),
+    ]
+
+_pandoc_toolchain_info = rule(
+    attrs = {
+        "pandoc": attr.label(
+            allow_single_file = True,
+            cfg = "host",
+            executable = True,
+        ),
+    },
+    implementation = _pandoc_toolchain_info_impl,
+)
+
+def pandoc_toolchain(platform, exec_compatible_with):
+    toolchain_info_name = "pandoc_toolchain_info_%s" % platform
+    _pandoc_toolchain_info(
+        name = toolchain_info_name,
+        pandoc = "@pandoc_%s//:pandoc" % platform,
+        visibility = ["//visibility:public"],
+    )
+
+    native.toolchain(
+        name = "pandoc_toolchain_%s" % platform,
+        exec_compatible_with = exec_compatible_with,
+        toolchain = toolchain_info_name,
+        toolchain_type = ":pandoc_toolchain_type",
+    )

pyenv3wrapper.sh

@@ -0,0 +1,4 @@
+#!/bin/sh
+
+HOME_DIR=$(getent passwd "$(whoami)" | cut -d: -f6)
+exec "$HOME_DIR/.pyenv/shims/python3" "$@"

repositories.bzl

@@ -1,6 +1,13 @@
 load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
 load("@rules_python//python:pip.bzl", "pip_import")
 
+_PANDOC_BUILD_FILE = """
+filegroup(
+    name = "pandoc",
+    srcs = ["bin/pandoc"],
+    visibility = ["//visibility:public"],
+)"""
+
 def gapic_generator_python():
     _maybe(
         pip_import,
@@ -25,13 +32,42 @@ def gapic_generator_python():
         urls = ["https://github.com/bazelbuild/bazel-skylib/archive/2169ae1c374aab4a09aa90e65efe1a3aad4e279b.tar.gz"],
     )
 
+    _maybe(
+        http_archive,
+        name = "com_github_grpc_grpc",
+        strip_prefix = "grpc-8347f4753568b5b66e49111c60ae2841278d3f33",  # this is 1.25.0 with fixes
+        urls = ["https://github.com/grpc/grpc/archive/8347f4753568b5b66e49111c60ae2841278d3f33.zip"],
+    )
+
+    _maybe(
+        http_archive,
+        name = "pandoc_linux",
+        build_file_content = _PANDOC_BUILD_FILE,
+        strip_prefix = "pandoc-2.2.1",
+        url = "https://github.com/jgm/pandoc/releases/download/2.2.1/pandoc-2.2.1-linux.tar.gz",
+    )
+
+    _maybe(
+        http_archive,
+        name = "pandoc_macOS",
+        build_file_content = _PANDOC_BUILD_FILE,
+        strip_prefix = "pandoc-2.2.1",
+        url = "https://github.com/jgm/pandoc/releases/download/2.2.1/pandoc-2.2.1-macOS.zip",
+    )
+
     _maybe(
         http_archive,
         name = "com_google_api_codegen",
         strip_prefix = "gapic-generator-b32c73219d617f90de70bfa6ff0ea0b0dd638dfe",
         urls = ["https://github.com/googleapis/gapic-generator/archive/b32c73219d617f90de70bfa6ff0ea0b0dd638dfe.zip"],
     )
 
+def gapic_generator_register_toolchains():
+    native.register_toolchains(
+        "@gapic_generator_python//:pandoc_toolchain_linux",
+        "@gapic_generator_python//:pandoc_toolchain_macOS",
+    )
+
 def _maybe(repo_rule, name, strip_repo_prefix = "", **kwargs):
     if not name.startswith(strip_repo_prefix):
         return

requirements.txt

@@ -1,9 +1,9 @@
 click==7.1.2
 google-api-core==1.17.0
 googleapis-common-protos==1.51.0
-grpcio==1.28.1
 jinja2==2.11.2
 MarkupSafe==1.1.1
 protobuf==3.11.3
 pypandoc==1.5
 PyYAML==5.3.1
+dataclasses==0.6

rules_python_gapic/py_gapic.bzl

@@ -14,7 +14,7 @@
 
 load("@com_google_api_codegen//rules_gapic:gapic.bzl", "proto_custom_library")
 
-def py_gapic_library(name, srcs, **kwargs):
+def py_gapic_library(name, srcs, plugin_args = [], **kwargs):
     #    srcjar_target_name = "%s_srcjar" % name
     srcjar_target_name = name
     srcjar_output_suffix = ".srcjar"
@@ -23,7 +23,7 @@ def py_gapic_library(name, srcs, **kwargs):
         name = srcjar_target_name,
         deps = srcs,
         plugin = Label("@gapic_generator_python//:gapic_plugin"),
-        plugin_args = [],
+        plugin_args = plugin_args,
         plugin_file_args = {},
         output_type = "python_gapic",
         output_suffix = srcjar_output_suffix,

rules_python_gapic/py_gapic_pkg.bzl

@@ -0,0 +1,70 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+load("@com_google_api_codegen//rules_gapic:gapic_pkg.bzl", "construct_package_dir_paths")
+
+def _py_gapic_src_pkg_impl(ctx):
+    srcjar_srcs = []
+    for dep in ctx.attr.deps:
+        for f in dep.files.to_list():
+            if f.extension in ("srcjar", "jar", "zip"):
+                srcjar_srcs.append(f)
+
+    paths = construct_package_dir_paths(ctx.attr.package_dir, ctx.outputs.pkg, ctx.label.name)
+
+    script = """
+    mkdir -p {package_dir_path}
+    for srcjar_src in {srcjar_srcs}; do
+        unzip -q -o $srcjar_src -d {package_dir_path}
+    done
+    cd {package_dir_path}/..
+    tar -zchpf {package_dir}/{package_dir}.tar.gz {package_dir}
+    cd -
+    mv {package_dir_path}/{package_dir}.tar.gz {pkg}
+    rm -rf {package_dir_path}
+    """.format(
+        srcjar_srcs = " ".join(["'%s'" % f.path for f in srcjar_srcs]),
+        package_dir_path = paths.package_dir_path,
+        package_dir = paths.package_dir,
+        pkg = ctx.outputs.pkg.path,
+        package_dir_expr = paths.package_dir_expr,
+    )
+
+    ctx.actions.run_shell(
+        inputs = srcjar_srcs,
+        command = script,
+        outputs = [ctx.outputs.pkg],
+    )
+
+_py_gapic_src_pkg = rule(
+    attrs = {
+        "deps": attr.label_list(allow_files = True, mandatory = True),
+        "package_dir": attr.string(mandatory = True),
+    },
+    outputs = {"pkg": "%{name}.tar.gz"},
+    implementation = _py_gapic_src_pkg_impl,
+)
+
+def py_gapic_assembly_pkg(name, deps, assembly_name = None, **kwargs):
+    package_dir = name
+    if assembly_name:
+        package_dir = "%s-%s" % (assembly_name, name)
+    _py_gapic_src_pkg(
+        name = name,
+        deps = deps,
+        package_dir = package_dir,
+        **kwargs
+    )
+
+