Add standalone mode for gapic-generator (googleapis#31)

This is a prerequisite for cc_gapic_library rule (will be submitted as a separate PR).

Most of the pr is `#include` changes. For actual changes please check the following files:
- `standalone.cc/standalone.h`
- `main.cc`
- `gapic_generator_unittest.cc`
- `BUILD.bazel`
- `gax/BUILD.bazel`
- `gax/repositories.bzl`
- `generator/testdata/BUILD.bazel`

Plus various small refactorings.

This includes:
1) "Fix" `#include` statements for external dependencies (`protobuf` and `gtest`): use `< >` instead of `" "`, also shorten path and conform to how it is done in protobuf repository itself (i.e. seems like and idiomatic way of making the imports). This also allows to get rid from `src/` prefix in the imports.
2) Gapic generator main binary can be executed in a plugin and a standalone mode (if command line args are provided, standalone mode is assumed automatically).
3) Standalone mode accepts standalone args according to the stand-alone interface spec:
    `--descriptor` - a list of descriptor sets (delimited with ':' or ';' character) (the spec does not require list, but accepting list is an extension, which still conforms to the spec);
    `--package` - a list (delimited with ':' or ';') which defines the proto packages designated for generation (to distinguish between common protos and actual api-specific protos, both of which must be specified in --descriptor); ability to specify a list is an extension to the spec;
    `--output` - the output folder (also supports outputting to zip archive, if output ends with .zip or .jar, this is an extension to the spec).
4) Refactor `BUILD.bazel` files and package structure.
5) Rename plugin binary to `protoc-gen-cpp_gapic` (to conform to spec and simplify usage as a plugin).
6) Refactor `gapic_generator_unittest.cc` (it will be simplified further, once `cc_gapic_library` is submitted)
7) Move gax-related repositories.bzl portion to its own `gax/repositories.bzl`

Author: vam-google

WORKSPACE

@@ -20,9 +20,3 @@ load(
 )
 
 com_google_gapic_generator_cpp_repositories()
-
-bind(
-    name = "zlib",
-    actual = "@net_zlib//:zlib",
-)
-

gax/backoff_policy_unittest.cc

@@ -16,7 +16,7 @@
 #include <memory>
 #include <random>
 
-#include "googletest/include/gtest/gtest.h"
+#include <gtest/gtest.h>
 
 #include "backoff_policy.h"
 

gax/repositories.bzl

@@ -0,0 +1,26 @@
+# Copyright 2019 Google Inc.  All rights reserved
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+def com_google_gapic_generator_cpp_gax_repositories():
+    _maybe(
+        http_archive,
+        name = "com_github_grpc_grpc",
+        strip_prefix = "grpc-1.18.0",
+        urls = ["https://github.com/grpc/grpc/archive/v1.18.0.tar.gz"]
+    )
+
+
+def _maybe(repo_rule, name, **kwargs):
+    if name not in native.existing_rules():
+        repo_rule(name = name, **kwargs)

gax/retry_policy_unittest.cc

@@ -15,9 +15,9 @@
 #include <chrono>
 #include <memory>
 
-#include "googletest/include/gtest/gtest.h"
 #include "retry_policy.h"
 #include "status.h"
+#include <gtest/gtest.h>
 
 namespace {
 using namespace ::google;

gax/status_or_unittest.cc

@@ -13,8 +13,8 @@
 // limitations under the License.
 
 #include "status_or.h"
-#include "googletest/include/gtest/gtest.h"
 #include "status.h"
+#include <gtest/gtest.h>
 
 #include <memory>
 #include <string>

gax/status_unittest.cc

@@ -13,7 +13,7 @@
 // limitations under the License.
 
 #include "status.h"
-#include "googletest/include/gtest/gtest.h"
+#include <gtest/gtest.h>
 
 #include <sstream>
 #include <string>

generator/BUILD.bazel

@@ -12,30 +12,31 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-package(default_visibility = ["//visibility:public"])
-
 licenses(["notice"])  # Apache 2.0
 
 cc_library(
-    name = "cpp_gapic_generator",
+    name = "gapic_generator",
     srcs = [
         "gapic_generator.cc",
         "internal/client_cc_generator.cc",
         "internal/client_cc_generator.h",
-        "internal/stub_cc_generator.cc",
-        "internal/stub_cc_generator.h",
         "internal/client_header_generator.cc",
         "internal/client_header_generator.h",
-        "internal/stub_header_generator.cc",
-        "internal/stub_header_generator.h",
         "internal/data_model.h",
         "internal/gapic_utils.cc",
         "internal/gapic_utils.h",
         "internal/printer.h",
+        "internal/stub_cc_generator.cc",
+        "internal/stub_cc_generator.h",
+        "internal/stub_header_generator.cc",
+        "internal/stub_header_generator.h",
+        "standalone.cc",
+        "standalone.h",
     ],
     hdrs = [
         "gapic_generator.h",
     ],
+    includes = ["."],
     deps = [
         "@absl//absl/base",
         "@absl//absl/strings",
@@ -44,41 +45,43 @@ cc_library(
     ],
 )
 
+# The protoc plugin/standalone binary. The name (proto-gen-cpp_gapic) is ugly
+# but it is a naming requirement for protoc plugins. To use --cpp_gapic_out
+# command line argument without need to provide additional --plugin argument,
+# the binary must be named proto-gen-cpp_gapic and placed in a PATH.
 cc_binary(
-    name = "cpp_gapic_generator_plugin",
-    srcs = ["plugin.cc"],
-    deps = [
-        ":cpp_gapic_generator",
-    ],
+    name = "protoc-gen-cpp_gapic",
+    srcs = ["main.cc"],
+    includes = ["."],
+    deps = [":gapic_generator"],
 )
 
 cc_test(
     name = "gapic_generator_unittest",
     size = "small",
     srcs = ["gapic_generator_unittest.cc"],
-    data = glob(
-        ["testdata/**"],
-    ) + [
+    data = [
+        "//generator/testdata:library_proto",
+        "//generator/testdata:library_service_baseline",
         "@api_common_protos//google/api:client_proto",
+        "@com_google_protobuf//:descriptor_proto",
     ],
     deps = [
-        "//generator:cpp_gapic_generator",
+        ":gapic_generator",
         "@gtest//:gtest_main",
     ],
 )
 
-generator_unit_tests = [
-    "internal/gapic_utils_unittest.cc",
-]
-
 [cc_test(
     name = "generator_" + test.replace("/", "_").replace(".cc", ""),
     size = "small",
     srcs = [test],
     deps = [
-        "//generator:cpp_gapic_generator",
+        "//generator:gapic_generator",
         "@absl//absl/base",
         "@absl//absl/strings",
         "@gtest//:gtest_main",
     ],
-) for test in generator_unit_tests]
+) for test in [
+    "internal/gapic_utils_unittest.cc",
+]]

generator/gapic_generator.h

@@ -14,12 +14,10 @@
 #ifndef GAPIC_GENERATOR_CPP_GENERATOR_GAPIC_GENERATOR_H_
 #define GAPIC_GENERATOR_CPP_GENERATOR_GAPIC_GENERATOR_H_
 
-#include <memory>
-#include <sstream>
 #include <string>
 
-#include "src/google/protobuf/compiler/code_generator.h"
-#include "src/google/protobuf/descriptor.h"
+#include <google/protobuf/compiler/code_generator.h>
+#include <google/protobuf/descriptor.h>
 
 namespace google {
 namespace api {

generator/gapic_generator_unittest.cc

@@ -17,66 +17,45 @@
 #include <string>
 #include <vector>
 
-#include <dirent.h>
 #include <iostream>
 
-#include "absl/strings/str_cat.h"
+#include <gtest/gtest.h>
 
-#include "src/google/protobuf/compiler/command_line_interface.h"
-#include "src/google/protobuf/io/printer.h"
-#include "src/google/protobuf/io/zero_copy_stream.h"
+#include <gapic_generator.h>
+#include <standalone.h>
 
-#include "googletest/include/gtest/gtest.h"
+namespace google {
+namespace api {
+namespace codegen {
 
-#include "generator/gapic_generator.h"
-
-namespace {
 namespace pb = google::protobuf;
-namespace codegen = google::api::codegen;
 
-inline std::string LoadContent(std::string f) {
+inline std::string LoadContent(std::string const& f) {
   std::ifstream ifs(f);
   EXPECT_TRUE(ifs.good()) << "Could not open " << f;
   return std::string((std::istreambuf_iterator<char>(ifs)),
                      (std::istreambuf_iterator<char>()));
 }
 
-TEST(CppGapicPluginTest, GapicPluginTest) {
-  std::string workspace_dir = ".";
-  std::string external_dir = workspace_dir + "/..";
-  std::string test_dir = workspace_dir + "/generator";
-  std::string test_data_dir = test_dir + "/testdata";
-  std::string gapic_out_dir = workspace_dir;
-
-  DIR* dirp = opendir(".");
-  struct dirent* dp;
-  while ((dp = readdir(dirp)) != NULL) {
-    std::cout << dp->d_name << std::endl;
-  }
-
-  pb::compiler::CommandLineInterface cli;
-  cli.SetInputsAreProtoPathRelative(true);
-
-  codegen::GapicGenerator generator;
-  cli.RegisterGenerator("--cpp_gapic_out", &generator, "");
+TEST(GapicGeneratorBaselineTest, StandaloneTest) {
+  std::string const input_dir("../");
+  std::string const output_dir("./");
+  std::string const data_dir("./generator/testdata/");
 
-  std::string workspace_proto_path = "-I" + workspace_dir;
-  std::string annotations_proto_path =
-      "-I" + external_dir + "/api_common_protos";
-  std::string well_known_types_proto_path =
-      "-I" + external_dir + "/com_google_protobuf";
-  std::string cpp_out = "--cpp_gapic_out=" + gapic_out_dir;
-  std::string library_proto = test_data_dir + "/library.proto";
+  std::vector<std::string> descriptors = {
+      input_dir +
+          "com_google_gapic_generator_cpp/generator/testdata/"
+          "library_proto-descriptor-set.proto.bin",
+      input_dir +
+          "api_common_protos/google/api/client_proto-descriptor-set.proto.bin",
+      input_dir +
+          "com_google_protobuf/descriptor_proto-descriptor-set.proto.bin"};
+  std::string package = "google.example.library.v1";
 
-  char const* argv[] = {"protoc",
-                        workspace_proto_path.c_str(),
-                        annotations_proto_path.c_str(),
-                        well_known_types_proto_path.c_str(),
-                        cpp_out.c_str(),
-                        library_proto.c_str()};
+  GapicGenerator generator;
 
-  EXPECT_EQ(0, cli.Run(sizeof(argv) / sizeof(argv[0]), argv))
-      << "cli.Run failed";
+  int res = StandaloneMain(descriptors, package, output_dir, &generator);
+  EXPECT_EQ(0, res) << "StandaloneMain failed";
 
   std::vector<std::string> files_to_check{
       "google/example/library/v1/library_service.gapic.h",
@@ -85,16 +64,17 @@ TEST(CppGapicPluginTest, GapicPluginTest) {
       "google/example/library/v1/library_service_stub.gapic.cc",
   };
 
-  for (std::string const& f : files_to_check) {
-    std::string actual_file = absl::StrCat(gapic_out_dir, "/", f);
-    std::string expected_file =
-        absl::StrCat(test_data_dir, "/", f, ".baseline");
+  for (auto const& file : files_to_check) {
+    std::string const expected_file(data_dir + file + ".baseline");
+    std::string const actual_file(output_dir + file);
 
-    std::string actual_file_content = LoadContent(actual_file);
-    std::string expected_file_content = LoadContent(expected_file);
+    std::string const& expected_file_content = LoadContent(expected_file);
+    std::string const& actual_file_content = LoadContent(actual_file);
 
     EXPECT_EQ(expected_file_content, actual_file_content);
   }
 }
 
-}  // namespace
+}  // namespace codegen
+}  // namespace api
+}  // namespace google

generator/internal/client_cc_generator.cc

@@ -18,9 +18,9 @@
 
 #include "gapic_utils.h"
 #include "printer.h"
-#include "src/google/protobuf/descriptor.h"
 #include "generator/internal/client_cc_generator.h"
 #include "generator/internal/data_model.h"
+#include <google/protobuf/descriptor.h>
 
 namespace pb = google::protobuf;