feat: add gapic metadata file (#781)

The GAPIC metadata file is used to track code, samples, and test
coverage for every RPC and library method.

.github/CODEOWNERS

@@ -4,5 +4,5 @@
 # For syntax help see:
 # https://help.github.com/en/github/creating-cloning-and-archiving-repositories/about-code-owners#codeowners-syntax
 
-* @googleapis/actools-python @googleapis/yoshi-python @lukesneeringer
-*.yaml @googleapis/actools @googleapis/yoshi-python @googleapis/actools-python @lukesneeringer
+* @googleapis/actools-python @googleapis/yoshi-python
+*.yaml @googleapis/actools @googleapis/yoshi-python @googleapis/actools-python

.github/workflows/tests.yaml

@@ -310,4 +310,4 @@ jobs:
             python -m pip install autopep8
       - name: Check diff
         run: |
-            find gapic tests -name "*.py" | xargs autopep8 --in-place --exit-code
+            find gapic tests -name "*.py" | xargs autopep8 --diff --exit-code

gapic/ads-templates/%namespace/%name/%version/gapic_metadata.json.j2

@@ -0,0 +1 @@
+{# {{ api.gapic_metadata_json(opts) }} #} {# TODO(dovs): This is temporarily commented out pending the addition of a flag #}

gapic/cli/generate.py

@@ -47,10 +47,11 @@ def generate(
     # This generator uses a slightly different mechanism for determining
     # which files to generate; it tracks at package level rather than file
     # level.
-    package = os.path.commonprefix([i.package for i in filter(
-        lambda p: p.name in req.file_to_generate,
-        req.proto_file,
-    )]).rstrip('.')
+    package = os.path.commonprefix([
+        p.package
+        for p in req.proto_file
+        if p.name in req.file_to_generate
+    ]).rstrip('.')
 
     # Build the API model object.
     # This object is a frozen representation of the whole API, and is sent

gapic/schema/api.py

@@ -28,8 +28,10 @@
 
 from google.api_core import exceptions  # type: ignore
 from google.api import resource_pb2  # type: ignore
+from google.gapic.metadata import gapic_metadata_pb2  # type: ignore
 from google.longrunning import operations_pb2  # type: ignore
 from google.protobuf import descriptor_pb2
+from google.protobuf.json_format import MessageToJson
 
 import grpc  # type: ignore
 
@@ -392,6 +394,45 @@ def subpackages(self) -> Mapping[str, 'API']:
                                                       )
         return answer
 
+    def gapic_metadata(self, options: Options) -> gapic_metadata_pb2.GapicMetadata:
+        gm = gapic_metadata_pb2.GapicMetadata(
+            schema="1.0",
+            comment="This file maps proto services/RPCs to the corresponding library clients/methods",
+            language="python",
+            proto_package=self.naming.proto_package,
+            library_package=".".join(
+                self.naming.module_namespace +
+                    (self.naming.versioned_module_name,)
+            ),
+        )
+
+        for service in sorted(self.services.values(), key=lambda s: s.name):
+            service_desc = gm.services.get_or_create(service.name)
+
+            # At least one of "grpc" or "rest" is guaranteed to be present because
+            # of the way that Options instances are created.
+            # This assumes the options are generated by the class method factory.
+            transports = []
+            if "grpc" in options.transport:
+                transports.append(("grpc", service.client_name))
+                transports.append(("grpcAsync", service.async_client_name))
+
+            if "rest" in options.transport:
+                transports.append(("rest", service.client_name))
+
+            methods = sorted(service.methods.values(), key=lambda m: m.name)
+            for tprt, client_name in transports:
+                transport = service_desc.clients.get_or_create(tprt)
+                transport.library_client = client_name
+                for method in methods:
+                    method_desc = transport.rpcs.get_or_create(method.name)
+                    method_desc.methods.append(to_snake_case(method.name))
+
+        return gm
+
+    def gapic_metadata_json(self, options: Options) -> str:
+        return MessageToJson(self.gapic_metadata(options), sort_keys=True)
+
     def requires_package(self, pkg: Tuple[str, ...]) -> bool:
         return any(
             message.ident.package == pkg

gapic/templates/%namespace/%name_%version/gapic_metadata.json.j2

@@ -0,0 +1 @@
+{# {{ api.gapic_metadata_json(opts) }} #} {# TODO(dovs): This is temporarily commented out pending the addition of a flag #}

setup.py

@@ -45,7 +45,7 @@
     install_requires=(
         "click >= 6.7",
         "google-api-core >= 1.17.0",
-        "googleapis-common-protos >= 1.6.0",
+        "googleapis-common-protos >= 1.53.0",
         "grpcio >= 1.24.3",
         "jinja2 >= 2.10",
         "protobuf >= 3.12.0",

tests/unit/schema/test_api.py

@@ -22,8 +22,10 @@
 from google.api import client_pb2
 from google.api import resource_pb2
 from google.api_core import exceptions
+from google.gapic.metadata import gapic_metadata_pb2
 from google.longrunning import operations_pb2
 from google.protobuf import descriptor_pb2
+from google.protobuf.json_format import MessageToJson
 
 from gapic.schema import api
 from gapic.schema import imp
@@ -260,8 +262,8 @@ def test_proto_oneof():
                     name='Bar',
                     fields=(
                         make_field_pb2(name='imported_message', number=1,
-                                   type_name='.google.dep.ImportedMessage',
-                                   oneof_index=0),
+                                       type_name='.google.dep.ImportedMessage',
+                                       oneof_index=0),
                         make_field_pb2(
                             name='primitive', number=2, type=1, oneof_index=0),
                     ),
@@ -1287,3 +1289,224 @@ def test_map_field_name_disambiguation():
     # The same module used in the same place should have the same import alias.
     # Because there's a "mollusc" name used, the import should be disambiguated.
     assert mollusc_ident == mollusc_map_ident == "am_mollusc.Mollusc"
+
+
+def test_gapic_metadata():
+    api_schema = api.API.build(
+        file_descriptors=[
+            descriptor_pb2.FileDescriptorProto(
+                name="cephalopod.proto",
+                package="animalia.mollusca.v1",
+                message_type=[
+                    descriptor_pb2.DescriptorProto(
+                        name="MolluscRequest",
+                    ),
+                    descriptor_pb2.DescriptorProto(
+                        name="Mollusc",
+                    ),
+                ],
+                service=[
+                    descriptor_pb2.ServiceDescriptorProto(
+                        name="Squid",
+                        method=[
+                            descriptor_pb2.MethodDescriptorProto(
+                                name="Ramshorn",
+                                input_type="animalia.mollusca.v1.MolluscRequest",
+                                output_type="animalia.mollusca.v1.Mollusc",
+                            ),
+                            descriptor_pb2.MethodDescriptorProto(
+                                name="Humboldt",
+                                input_type="animalia.mollusca.v1.MolluscRequest",
+                                output_type="animalia.mollusca.v1.Mollusc",
+                            ),
+                            descriptor_pb2.MethodDescriptorProto(
+                                name="Giant",
+                                input_type="animalia.mollusca.v1.MolluscRequest",
+                                output_type="animalia.mollusca.v1.Mollusc",
+                            ),
+                        ],
+                    ),
+                    descriptor_pb2.ServiceDescriptorProto(
+                        name="Octopus",
+                        method=[
+                            descriptor_pb2.MethodDescriptorProto(
+                                name="GiantPacific",
+                                input_type="animalia.mollusca.v1.MolluscRequest",
+                                output_type="animalia.mollusca.v1.Mollusc",
+                            ),
+                            descriptor_pb2.MethodDescriptorProto(
+                                name="BlueSpot",
+                                input_type="animalia.mollusca.v1.MolluscRequest",
+                                output_type="animalia.mollusca.v1.Mollusc",
+                            ),
+                        ]
+                    ),
+                ],
+            )
+        ]
+    )
+
+    opts = Options.build("transport=grpc")
+    expected = gapic_metadata_pb2.GapicMetadata(
+        schema="1.0",
+        comment="This file maps proto services/RPCs to the corresponding library clients/methods",
+        language="python",
+        proto_package="animalia.mollusca.v1",
+        library_package="animalia.mollusca_v1",
+        services={
+            "Octopus": gapic_metadata_pb2.GapicMetadata.ServiceForTransport(
+                clients={
+                    "grpc": gapic_metadata_pb2.GapicMetadata.ServiceAsClient(
+                        library_client="OctopusClient",
+                        rpcs={
+                            "BlueSpot": gapic_metadata_pb2.GapicMetadata.MethodList(methods=["blue_spot"]),
+                            "GiantPacific": gapic_metadata_pb2.GapicMetadata.MethodList(methods=["giant_pacific"]),
+                        },
+                    ),
+                    "grpcAsync": gapic_metadata_pb2.GapicMetadata.ServiceAsClient(
+                        library_client="OctopusAsyncClient",
+                        rpcs={
+                            "BlueSpot": gapic_metadata_pb2.GapicMetadata.MethodList(methods=["blue_spot"]),
+                            "GiantPacific": gapic_metadata_pb2.GapicMetadata.MethodList(methods=["giant_pacific"]),
+                        },
+                    ),
+                }
+            ),
+            "Squid": gapic_metadata_pb2.GapicMetadata.ServiceForTransport(
+                clients={
+                    "grpc": gapic_metadata_pb2.GapicMetadata.ServiceAsClient(
+                        library_client="SquidClient",
+                        rpcs={
+                            "Giant": gapic_metadata_pb2.GapicMetadata.MethodList(methods=["giant"]),
+                            "Humboldt": gapic_metadata_pb2.GapicMetadata.MethodList(methods=["humboldt"]),
+                            "Ramshorn": gapic_metadata_pb2.GapicMetadata.MethodList(methods=["ramshorn"]),
+                        },
+                    ),
+                    "grpcAsync": gapic_metadata_pb2.GapicMetadata.ServiceAsClient(
+                        library_client="SquidAsyncClient",
+                        rpcs={
+                            "Giant": gapic_metadata_pb2.GapicMetadata.MethodList(methods=["giant"]),
+                            "Humboldt": gapic_metadata_pb2.GapicMetadata.MethodList(methods=["humboldt"]),
+                            "Ramshorn": gapic_metadata_pb2.GapicMetadata.MethodList(methods=["ramshorn"]),
+                        },
+                    ),
+                }
+            ),
+        }
+    )
+    actual = api_schema.gapic_metadata(opts)
+    assert expected == actual
+    expected = MessageToJson(expected, sort_keys=True)
+    actual = api_schema.gapic_metadata_json(opts)
+    assert expected == actual
+
+    opts = Options.build("transport=rest")
+    expected = gapic_metadata_pb2.GapicMetadata(
+        schema="1.0",
+        comment="This file maps proto services/RPCs to the corresponding library clients/methods",
+        language="python",
+        proto_package="animalia.mollusca.v1",
+        library_package="animalia.mollusca_v1",
+        services={
+            "Octopus": gapic_metadata_pb2.GapicMetadata.ServiceForTransport(
+                clients={
+                    "rest": gapic_metadata_pb2.GapicMetadata.ServiceAsClient(
+                        library_client="OctopusClient",
+                        rpcs={
+                            "BlueSpot": gapic_metadata_pb2.GapicMetadata.MethodList(methods=["blue_spot"]),
+                            "GiantPacific": gapic_metadata_pb2.GapicMetadata.MethodList(methods=["giant_pacific"]),
+                        },
+                    )
+                }
+            ),
+            "Squid": gapic_metadata_pb2.GapicMetadata.ServiceForTransport(
+                clients={
+                    "rest": gapic_metadata_pb2.GapicMetadata.ServiceAsClient(
+                        library_client="SquidClient",
+                        rpcs={
+                            "Giant": gapic_metadata_pb2.GapicMetadata.MethodList(methods=["giant"]),
+                            "Humboldt": gapic_metadata_pb2.GapicMetadata.MethodList(methods=["humboldt"]),
+                            "Ramshorn": gapic_metadata_pb2.GapicMetadata.MethodList(methods=["ramshorn"]),
+                        },
+                    ),
+
+                }
+            ),
+        }
+    )
+    actual = api_schema.gapic_metadata(opts)
+    assert expected == actual
+    expected = MessageToJson(expected, sort_keys=True)
+    actual = api_schema.gapic_metadata_json(opts)
+    assert expected == actual
+
+    opts = Options.build("transport=rest+grpc")
+    expected = gapic_metadata_pb2.GapicMetadata(
+        schema="1.0",
+        comment="This file maps proto services/RPCs to the corresponding library clients/methods",
+        language="python",
+        proto_package="animalia.mollusca.v1",
+        library_package="animalia.mollusca_v1",
+        services={
+            "Octopus": gapic_metadata_pb2.GapicMetadata.ServiceForTransport(
+                clients={
+                    "grpc": gapic_metadata_pb2.GapicMetadata.ServiceAsClient(
+                        library_client="OctopusClient",
+                        rpcs={
+                            "BlueSpot": gapic_metadata_pb2.GapicMetadata.MethodList(methods=["blue_spot"]),
+                            "GiantPacific": gapic_metadata_pb2.GapicMetadata.MethodList(methods=["giant_pacific"]),
+                        },
+                    ),
+                    "grpcAsync": gapic_metadata_pb2.GapicMetadata.ServiceAsClient(
+                        library_client="OctopusAsyncClient",
+                        rpcs={
+                            "BlueSpot": gapic_metadata_pb2.GapicMetadata.MethodList(methods=["blue_spot"]),
+                            "GiantPacific": gapic_metadata_pb2.GapicMetadata.MethodList(methods=["giant_pacific"]),
+                        },
+                    ),
+                    "rest": gapic_metadata_pb2.GapicMetadata.ServiceAsClient(
+                        library_client="OctopusClient",
+                        rpcs={
+                            "BlueSpot": gapic_metadata_pb2.GapicMetadata.MethodList(methods=["blue_spot"]),
+                            "GiantPacific": gapic_metadata_pb2.GapicMetadata.MethodList(methods=["giant_pacific"]),
+                        },
+                    )
+                }
+            ),
+            "Squid": gapic_metadata_pb2.GapicMetadata.ServiceForTransport(
+                clients={
+                    "grpc": gapic_metadata_pb2.GapicMetadata.ServiceAsClient(
+                        library_client="SquidClient",
+                        rpcs={
+                            "Giant": gapic_metadata_pb2.GapicMetadata.MethodList(methods=["giant"]),
+                            "Humboldt": gapic_metadata_pb2.GapicMetadata.MethodList(methods=["humboldt"]),
+                            "Ramshorn": gapic_metadata_pb2.GapicMetadata.MethodList(methods=["ramshorn"]),
+                        },
+                    ),
+                    "grpcAsync": gapic_metadata_pb2.GapicMetadata.ServiceAsClient(
+                        library_client="SquidAsyncClient",
+                        rpcs={
+                            "Giant": gapic_metadata_pb2.GapicMetadata.MethodList(methods=["giant"]),
+                            "Humboldt": gapic_metadata_pb2.GapicMetadata.MethodList(methods=["humboldt"]),
+                            "Ramshorn": gapic_metadata_pb2.GapicMetadata.MethodList(methods=["ramshorn"]),
+                        },
+                    ),
+                    "rest": gapic_metadata_pb2.GapicMetadata.ServiceAsClient(
+                        library_client="SquidClient",
+                        rpcs={
+                            "Giant": gapic_metadata_pb2.GapicMetadata.MethodList(methods=["giant"]),
+                            "Humboldt": gapic_metadata_pb2.GapicMetadata.MethodList(methods=["humboldt"]),
+                            "Ramshorn": gapic_metadata_pb2.GapicMetadata.MethodList(methods=["ramshorn"]),
+                        },
+                    ),
+
+                }
+            ),
+        }
+    )
+
+    actual = api_schema.gapic_metadata(opts)
+    assert expected == actual
+    expected = MessageToJson(expected, sort_keys=True)
+    actual = api_schema.gapic_metadata_json(opts)
+    assert expected == actual