feat: port REST transport to Ads templates (#1003)

Add REST transport to Ads templates
Add REST transport tests to Ads generated unit tests
Boost google-api-core dependency version

Rewrite of REST transport call stubs to assist testing
Add fragment tests for Ads
Multiple bugfixes

Author: software-dov

.github/sync-repo-settings.yaml

@@ -22,11 +22,19 @@ branchProtectionRules:
     - 'showcase-unit (3.9)'
     - 'showcase-unit (3.9, _alternative_templates)'
     - 'showcase-unit-add-iam-methods'
+    - 'integration'
     - 'style-check'
     - 'unit (3.6)'
     - 'unit (3.7)'
     - 'unit (3.8)'
     - 'unit (3.9)'
+    - 'fragment (3.6)'
+    - 'fragment (3.7)'
+    - 'fragment (3.8)'
+    - 'fragment (3.9)'
+    - 'fragment (3.7, _alternative_templates)'
+    - 'fragment (3.8, _alternative_templates)'
+    - 'fragment (3.9, _alternative_templates)'
   requiredApprovingReviewCount: 1
   requiresCodeOwnerReviews: true
   requiresStrictStatusChecks: true

.github/workflows/tests.yaml

@@ -74,7 +74,7 @@ jobs:
           ./gapic-showcase run &
           cd -
         env:
-          SHOWCASE_VERSION: 0.16.0
+          SHOWCASE_VERSION: 0.18.0
       - name: Install nox.
         run: python -m pip install nox
       - name: Install protoc 3.12.1.
@@ -144,7 +144,7 @@ jobs:
             cd ..
             nox -s ${{ matrix.target }}
         env:
-          SHOWCASE_VERSION: 0.16.0
+          SHOWCASE_VERSION: 0.18.0
   # TODO(yon-mg): add compute unit tests
   showcase-unit:
     strategy:
@@ -183,7 +183,7 @@ jobs:
       - name: Run unit tests.
         run: nox -s showcase_unit${{ matrix.variant }}-${{ matrix.python }}
         env:
-          SHOWCASE_VERSION: 0.16.0
+          SHOWCASE_VERSION: 0.18.0
   showcase-unit-add-iam-methods:
     runs-on: ubuntu-latest
     steps:
@@ -214,7 +214,7 @@ jobs:
       - name: Run unit tests.
         run: nox -s showcase_unit_add_iam_methods
         env:
-          SHOWCASE_VERSION: 0.16.0
+          SHOWCASE_VERSION: 0.18.0
   showcase-mypy:
     runs-on: ubuntu-latest
     strategy:
@@ -248,7 +248,7 @@ jobs:
       - name: Typecheck the generated output.
         run: nox -s showcase_mypy${{ matrix.variant }}
         env:
-          SHOWCASE_VERSION: 0.16.0
+          SHOWCASE_VERSION: 0.18.0
   snippetgen:
     runs-on: ubuntu-latest
     steps:
@@ -299,6 +299,10 @@ jobs:
     strategy:
       matrix:
         python: [3.6, 3.7, 3.8, 3.9]
+        variant: ['', _alternative_templates]
+        exclude:
+          - python: 3.6
+            variant: _alternative_templates
     runs-on: ubuntu-latest
     steps:
       - name: Cancel Previous Runs
@@ -319,7 +323,7 @@ jobs:
         run: |
             python -m pip install nox
       - name: Run fragment tests.
-        run: nox -s fragment-${{ matrix.python }}
+        run: nox -s fragment${{ matrix.variant }}-${{ matrix.python }}
   integration:
     runs-on: ubuntu-latest
     container: gcr.io/gapic-images/googleapis-bazel:20210105

gapic/ads-templates/%namespace/%name/%version/%sub/services/%service/client.py.j2

@@ -7,7 +7,7 @@
 {# This lives within the loop in order to ensure that this template
    is empty if there are no paged methods.
  -#}
-from typing import Any, Callable, Iterable, Sequence, Tuple
+from typing import Any, Callable, Iterable, Sequence, Tuple, Optional, Iterator
 
 {% filter sort_lines %}
 {% for method in service.methods.values() | selectattr('paged_result_field') %}
@@ -70,9 +70,18 @@ class {{ method.name }}Pager:
             self._response = self._method(self._request, metadata=self._metadata)
             yield self._response
 
-    def __iter__(self) -> {{ method.paged_result_field.ident | replace('Sequence', 'Iterable') }}:
+    {% if method.paged_result_field.map %}
+    def __iter__(self) -> Iterator[Tuple[str, {{ method.paged_result_field.type.fields.get('value').ident }}]]:
+        for page in self.pages:
+            yield from page.{{ method.paged_result_field.name}}.items()
+
+    def get(self, key: str) -> Optional[{{ method.paged_result_field.type.fields.get('value').ident }}]:
+        return self._response.{{ method.paged_result_field.name }}.get(key)
+    {% else %}
+    def __iter__(self) -> {{ method.paged_result_field.ident | replace('Sequence', 'Iterator') }}:
         for page in self.pages:
             yield from page.{{ method.paged_result_field.name }}
+    {% endif %}
 
     def __repr__(self) -> str:
         return '{0}<{1!r}>'.format(self.__class__.__name__, self._response)

gapic/ads-templates/%namespace/%name/%version/%sub/services/%service/pagers.py.j2

@@ -6,16 +6,29 @@ from collections import OrderedDict
 from typing import Dict, Type
 
 from .base import {{ service.name }}Transport
+{% if 'grpc' in opts.transport %}
 from .grpc import {{ service.name }}GrpcTransport
-
+{% endif %}
+{% if 'rest' in opts.transport %}
+from .rest import {{ service.name }}RestTransport
+{% endif %}
 
 # Compile a registry of transports.
 _transport_registry = OrderedDict()  # type: Dict[str, Type[{{ service.name }}Transport]]
+{% if 'grpc' in opts.transport %}
 _transport_registry['grpc'] = {{ service.name }}GrpcTransport
-
+{% endif %}
+{% if 'rest' in opts.transport %}
+_transport_registry['rest'] = {{ service.name }}RestTransport
+{% endif %}
 
 __all__ = (
     '{{ service.name }}Transport',
+    {% if 'grpc' in opts.transport %}
     '{{ service.name }}GrpcTransport',
+    {% endif %}
+    {% if 'rest' in opts.transport %}
+    '{{ service.name }}RestTransport',
+    {% endif %}
 )
 {% endblock %}

gapic/ads-templates/%namespace/%name/%version/%sub/services/%service/transports/__init__.py.j2

@@ -3,22 +3,29 @@
 {% block content %}
 
 import abc
-import typing
+from typing import Awaitable, Callable, Dict, Optional, Sequence, Union
 import pkg_resources
 
 import google.auth  # type: ignore
-from google.api_core import gapic_v1
-from google.api_core import retry as retries
+import google.api_core  # type: ignore
+from google.api_core import exceptions as core_exceptions  # type: ignore
+from google.api_core import gapic_v1    # type: ignore
+from google.api_core import retry as retries  # type: ignore
 {% if service.has_lro %}
 from google.api_core import operations_v1
 {% endif %}
 from google.auth import credentials as ga_credentials  # type: ignore
+from google.oauth2 import service_account # type: ignore
 
 {% filter sort_lines %}
 {% for method in service.methods.values() %}
 {{ method.input.ident.python_import }}
 {{ method.output.ident.python_import }}
 {% endfor %}
+{% if opts.add_iam_methods %}
+from google.iam.v1 import iam_policy_pb2  # type: ignore
+from google.iam.v1 import policy_pb2  # type: ignore
+{% endif %}
 {% endfilter %}
 
 try:
@@ -31,7 +38,7 @@ except pkg_resources.DistributionNotFound:
     DEFAULT_CLIENT_INFO = gapic_v1.client_info.ClientInfo()
 
 
-class {{ service.name }}Transport(metaclass=abc.ABCMeta):
+class {{ service.name }}Transport(abc.ABC):
     """Abstract transport class for {{ service.name }}."""
 
     AUTH_SCOPES = (
@@ -40,11 +47,18 @@ class {{ service.name }}Transport(metaclass=abc.ABCMeta):
         {% endfor %}
     )
 
+    DEFAULT_HOST: str = {% if service.host %}'{{ service.host }}'{% else %}{{ '' }}{% endif %}
+
     def __init__(
             self, *,
-            host: str{% if service.host %} = '{{ service.host }}'{% endif %},
+            host: str = DEFAULT_HOST,
             credentials: ga_credentials.Credentials = None,
+            credentials_file: Optional[str] = None,
+            scopes: Optional[Sequence[str]] = None,
+            quota_project_id: Optional[str] = None,
             client_info: gapic_v1.client_info.ClientInfo = DEFAULT_CLIENT_INFO,
+            always_use_jwt_access: Optional[bool] = False,
+            **kwargs,
             ) -> None:
         """Instantiate the transport.
 
@@ -56,30 +70,54 @@ class {{ service.name }}Transport(metaclass=abc.ABCMeta):
                 credentials identify the application to the service; if none
                 are specified, the client will attempt to ascertain the
                 credentials from the environment.
+            credentials_file (Optional[str]): A file with credentials that can
+                be loaded with :func:`google.auth.load_credentials_from_file`.
+                This argument is mutually exclusive with credentials.
+            scopes (Optional[Sequence[str]]): A list of scopes.
+            quota_project_id (Optional[str]): An optional project to use for billing
+                and quota.
             client_info (google.api_core.gapic_v1.client_info.ClientInfo):
                 The client info used to send a user-agent string along with
                 API requests. If ``None``, then default info will be used.
                 Generally, you only need to set this if you're developing
                 your own client library.
+            always_use_jwt_access (Optional[bool]): Whether self signed JWT should
+                be used for service account credentials.
         """
         # Save the hostname. Default to port 443 (HTTPS) if none is specified.
         if ':' not in host:
             host += ':443'
         self._host = host
 
+        scopes_kwargs = {"scopes": scopes, "default_scopes": self.AUTH_SCOPES}
+
+        # Save the scopes.
+        self._scopes = scopes
+
         # If no credentials are provided, then determine the appropriate
         # defaults.
-        if credentials is None:
-            credentials, _ = google.auth.default(scopes=self.AUTH_SCOPES)
+        if credentials and credentials_file:
+            raise core_exceptions.DuplicateCredentialArgs("'credentials_file' and 'credentials' are mutually exclusive")
+
+        if credentials_file is not None:
+            credentials, _ = google.auth.load_credentials_from_file(
+                                credentials_file,
+                                **scopes_kwargs,
+                                quota_project_id=quota_project_id
+                            )
+        elif credentials is None:
+            credentials, _ = google.auth.default(**scopes_kwargs, quota_project_id=quota_project_id)
+
+        # If the credentials are service account credentials, then always try to use self signed JWT.
+        if always_use_jwt_access and isinstance(credentials, service_account.Credentials) and hasattr(service_account.Credentials, "with_always_use_jwt_access"):
+            credentials = credentials.with_always_use_jwt_access(True)
 
         # Save the credentials.
         self._credentials = credentials
 
-        # Lifted into its own function so it can be stubbed out during tests.
-        self._prep_wrapped_messages(client_info)
 
     def _prep_wrapped_messages(self, client_info):
-        # Precomputed wrapped methods
+        # Precompute the wrapped methods.
         self._wrapped_methods = {
             {% for method in service.methods.values() %}
             self.{{ method.name|snake_case }}: gapic_v1.method.wrap_method(
@@ -106,7 +144,7 @@ class {{ service.name }}Transport(metaclass=abc.ABCMeta):
     def close(self):
         """Closes resources associated with the transport.
 
-        .. warning::
+       .. warning::
             Only call this method if the transport is NOT shared
             with other clients - this may cause errors in other clients!
         """
@@ -117,18 +155,53 @@ class {{ service.name }}Transport(metaclass=abc.ABCMeta):
     @property
     def operations_client(self):
         """Return the client designed to process long-running operations."""
-        raise NotImplementedError
+        raise NotImplementedError()
     {% endif %}
     {% for method in service.methods.values() %}
 
     @property
-    def {{ method.name|snake_case }}(self) -> typing.Callable[
+    def {{ method.name|snake_case }}(self) -> Callable[
             [{{ method.input.ident }}],
-            {{ method.output.ident }}]:
-        raise NotImplementedError
+            Union[
+                {{ method.output.ident }},
+                Awaitable[{{ method.output.ident }}]
+            ]]:
+        raise NotImplementedError()
     {% endfor %}
 
 
+    {% if opts.add_iam_methods %}
+    @property
+    def set_iam_policy(
+        self,
+    ) -> Callable[
+        [iam_policy_pb2.SetIamPolicyRequest],
+        Union[policy_pb2.Policy, Awaitable[policy_pb2.Policy]],
+    ]:
+        raise NotImplementedError()
+
+    @property
+    def get_iam_policy(
+        self,
+    ) -> Callable[
+        [iam_policy_pb2.GetIamPolicyRequest],
+        Union[policy_pb2.Policy, Awaitable[policy_pb2.Policy]],
+    ]:
+        raise NotImplementedError()
+
+    @property
+    def test_iam_permissions(
+        self,
+    ) -> Callable[
+        [iam_policy_pb2.TestIamPermissionsRequest],
+        Union[
+            iam_policy_pb2.TestIamPermissionsResponse,
+            Awaitable[iam_policy_pb2.TestIamPermissionsResponse],
+        ],
+    ]:
+        raise NotImplementedError()
+    {% endif %}
+
 __all__ = (
     '{{ service.name }}Transport',
 )