feat: add fragment tests (#1056)

Fragment tests are defined by a small proto file describing an API surface with characteristics such that it is desirable to test the generated surface for correctness or to prevent regressions. As part of a fragment test, the generator is run on a fragment to create a GAPIC library for the fragment. The generated unit tests for the fragment are then executed to test the surface.
googleapis · Nov 1, 2021 · 9d9b33d · 9d9b33d
1 parent 03d8dad
commit 9d9b33d
Show file tree

Hide file tree

Showing 20 changed files with 1,909 additions and 217 deletions.
diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml
@@ -42,39 +42,6 @@ jobs:
         run: python -m pip install nox
       - name: Check type annotations.
         run: nox -s mypy
-  # publish_image:
-  #   runs-on: ubuntu-latest
-  #   container: docker
-  #   steps:
-  #     - uses: actions/checkout@v2
-  #     - setup_remote_docker
-  #     - name: Build Docker image.
-  #       run: docker build . -t gcr.io/gapic-images/gapic-generator-python:latest
-  #     - name: Download curl
-  #       run: apk add --no-cache curl
-  #     - name: Download the GCR credential helper.
-  #       run: |
-  #           curl -fsSL https://github.com/GoogleCloudPlatform/docker-credential-gcr/releases/download/v1.5.0/docker-credential-gcr_linux_amd64-1.5.0.tar.gz \
-  #             | tar xz --to-stdout ./docker-credential-gcr \
-  #             > /usr/bin/docker-credential-gcr && chmod a+x /usr/bin/docker-credential-gcr
-  #     - name: Set up authentication to Google Container Registry.
-  #       run: |
-  #           echo ${GCLOUD_SERVICE_KEY} > ${GOOGLE_APPLICATION_CREDENTIALS}
-  #           docker-credential-gcr configure-docker
-  #     - name: Tag the Docker image and push it to Google Container Registry.
-  #       run: |
-  #           if [ -n "$CIRCLE_TAG" ]; then
-  #             export MAJOR=`echo $CIRCLE_TAG | awk -F '.' '{ print $1; }'`
-  #             export MINOR=`echo $CIRCLE_TAG | awk -F '.' '{ print $2; }'`
-  #             export PATCH=`echo $CIRCLE_TAG | awk -F '.' '{ print $3; }'`
-  #             docker tag gcr.io/gapic-images/gapic-generator-python:latest gcr.io/gapic-images/gapic-generator-python:$MAJOR.$MINOR.$PATCH
-  #             docker tag gcr.io/gapic-images/gapic-generator-python:latest gcr.io/gapic-images/gapic-generator-python:$MAJOR.$MINOR
-  #             docker tag gcr.io/gapic-images/gapic-generator-python:latest gcr.io/gapic-images/gapic-generator-python:$MAJOR
-  #             docker push gcr.io/gapic-images/gapic-generator-python:$MAJOR.$MINOR.$PATCH
-  #             docker push gcr.io/gapic-images/gapic-generator-python:$MAJOR.$MINOR
-  #             docker push gcr.io/gapic-images/gapic-generator-python:$MAJOR
-  #           fi
-  #           docker push gcr.io/gapic-images/gapic-generator-python:latest
   showcase:
     strategy:
       matrix:
@@ -319,6 +286,30 @@ jobs:
             python -m pip install nox
       - name: Run unit tests.
         run: nox -s unit-${{ matrix.python }}
+  fragment:
+    strategy:
+      matrix:
+        python: [3.6, 3.7, 3.8, 3.9]
+    runs-on: ubuntu-latest
+    steps:
+      - name: Cancel Previous Runs
+        uses: styfle/cancel-workflow-action@0.7.0
+        with:
+          access_token: ${{ github.token }}
+      - uses: actions/checkout@v2
+      - name: Set up Python ${{ matrix.python }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python }}
+      - name: Install pandoc
+        run: |
+            sudo apt-get update
+            sudo apt-get install -y pandoc gcc git
+      - name: Install nox.
+        run: |
+            python -m pip install nox
+      - name: Run fragment tests.
+        run: nox -s fragment-${{ matrix.python }}
   integration:
     runs-on: ubuntu-latest
     steps:

diff --git a/WORKSPACE b/WORKSPACE
@@ -13,10 +13,6 @@ http_archive(
     url = "https://github.com/bazelbuild/rules_python/archive/0.1.0.tar.gz",
 )
 
-load("@rules_python//python:pip.bzl", "pip_repositories")
-
-pip_repositories()
-
 #
 # Import gapic-generator-python specific dependencies
 #

diff --git a/gapic/templates/%namespace/%name_%version/%sub/services/%service/client.py.j2 b/gapic/templates/%namespace/%name_%version/%sub/services/%service/client.py.j2
@@ -405,7 +405,13 @@ class {{ service.client_name }}(metaclass={{ service.client_name }}Meta):
             {% endif %}
             {% for key, field in method.flattened_fields.items() if not field.repeated or method.input.ident.package == method.ident.package %}
             if {{ field.name }} is not None:
+                {# Repeated values is a special case, because values can be lists. #}
+                {# In order to not confuse the marshalling logic, extend these fields instead of assigning #}
+                {% if field.ident.ident|string() == "struct_pb2.Value" and field.repeated %}
+                request.{{ key }}.extend({{ field.name }})
+                {% else %}
                 request.{{ key }} = {{ field.name }}
+                {% endif %}{# struct_pb2.Value #}
             {% endfor %}
             {# Map-y fields can be _updated_, however #}
             {% for key, field in method.flattened_fields.items() if field.repeated and method.input.ident.package != method.ident.package %}

diff --git a/gapic/templates/tests/unit/gapic/%name_%version/%sub/test_%service.py.j2 b/gapic/templates/tests/unit/gapic/%name_%version/%sub/test_%service.py.j2
@@ -788,7 +788,15 @@ def test_{{ method.name|snake_case }}_flattened():
         {% elif field.ident|string() == 'duration_pb2.Duration' %}
         assert DurationRule().to_proto(args[0].{{ key }}) == {{ field.mock_value }}
         {% else %}
-        assert args[0].{{ key }} == {{ field.mock_value }}
+        arg = args[0].{{ key }}
+        mock_val = {{ field.mock_value }}
+        {% if field.ident|string() == "struct_pb2.Value" %}
+        from proto.marshal import Marshal
+        from proto.marshal.rules.struct import ValueRule
+        rule = ValueRule(marshal=Marshal(name="Test"))
+        mock_val = rule.to_python(mock_val)
+        {% endif %}{# struct_pb2.Value #}
+        assert arg == mock_val
         {% endif %}
         {% endif %}{% endfor %}
         {% for oneofs in method.flattened_oneof_fields().values() %}
@@ -873,7 +881,15 @@ async def test_{{ method.name|snake_case }}_flattened_async():
         {% elif field.ident|string() == 'duration_pb2.Duration' %}
         assert DurationRule().to_proto(args[0].{{ key }}) == {{ field.mock_value }}
         {% else %}
-        assert args[0].{{ key }} == {{ field.mock_value }}
+        arg = args[0].{{ key }}
+        mock_val = {{ field.mock_value }}
+        {% if field.ident|string() == "struct_pb2.Value" %}
+        from proto.marshal import Marshal
+        from proto.marshal.rules.struct import ValueRule
+        rule = ValueRule(marshal=Marshal(name="Test"))
+        mock_val = rule.to_python(mock_val)
+        {% endif %}{# struct_pb2.Value #}
+        assert arg == mock_val
         {% endif %}
         {% endif %}{% endfor %}
         {% for oneofs in method.flattened_oneof_fields().values() %}

diff --git a/noxfile.py b/noxfile.py
@@ -13,6 +13,7 @@
 # limitations under the License.
 
 from __future__ import absolute_import
+from concurrent.futures import ThreadPoolExecutor
 from pathlib import Path
 import os
 import sys
@@ -29,7 +30,17 @@
 ADS_TEMPLATES = path.join(path.dirname(__file__), "gapic", "ads-templates")
 
 
-@nox.session(python=["3.6", "3.7", "3.8", "3.9", "3.10"])
+ALL_PYTHON = (
+    "3.6",
+    "3.7",
+    "3.8",
+    "3.9",
+)
+
+NEWEST_PYTHON = "3.9"
+
+
+@nox.session(python=ALL_PYTHON)
 def unit(session):
     """Run the unit test suite."""
 
@@ -50,11 +61,89 @@ def unit(session):
                 "--cov-report=term",
                 "--cov-fail-under=100",
                 path.join("tests", "unit"),
-                            ]
+            ]
         ),
     )
 
 
+FRAG_DIR = Path("tests") / "fragments"
+FRAGMENT_FILES = tuple(
+    Path(dirname).relative_to(FRAG_DIR) / f
+    for dirname, _, files in os.walk(FRAG_DIR)
+    for f in files
+    if os.path.splitext(f)[1] == ".proto" and f.startswith("test_")
+)
+
+# Note: this class lives outside 'fragment'
+# so that, if necessary, it can be pickled for a ProcessPoolExecutor
+# A callable class is necessary so that the session can be closed over
+# instead of passed in, which simplifies the invocation via map.
+class FragTester:
+    def __init__(self, session):
+        self.session = session
+
+    def __call__(self, frag):
+        with tempfile.TemporaryDirectory() as tmp_dir:
+            # Generate the fragment GAPIC.
+            outputs = []
+            outputs.append(
+                self.session.run(
+                    "python",
+                    "-m",
+                    "grpc_tools.protoc",
+                    f"--proto_path={str(FRAG_DIR)}",
+                    f"--python_gapic_out={tmp_dir}",
+                    "--python_gapic_opt=transport=grpc+rest",
+                    str(frag),
+                    external=True,
+                    silent=True,
+                )
+            )
+
+            # Install the generated fragment library.
+            # Note: install into the tempdir to prevent issues
+            # with running pip concurrently.
+            self.session.install(tmp_dir, "-e", ".", "-t", tmp_dir, "-qqq")
+
+            # Run the fragment's generated unit tests.
+            # Don't bother parallelizing them: we already parallelize
+            # the fragments, and there usually aren't too many tests per fragment.
+            outputs.append(
+                self.session.run(
+                    "py.test",
+                    "--quiet",
+                    f"--cov-config={str(Path(tmp_dir) / '.coveragerc')}",
+                    "--cov-report=term",
+                    "--cov-fail-under=100",
+                    str(Path(tmp_dir) / "tests" / "unit"),
+                    silent=True,
+                )
+            )
+
+            return "".join(outputs)
+
+
+# TODO(dovs): ads templates
+@nox.session(python=ALL_PYTHON)
+def fragment(session):
+    session.install(
+        "coverage",
+        "pytest",
+        "pytest-cov",
+        "pytest-xdist",
+        "asyncmock",
+        "pytest-asyncio",
+        "grpcio-tools",
+    )
+    session.install("-e", ".")
+
+    with ThreadPoolExecutor() as p:
+        all_outs = p.map(FragTester(session), FRAGMENT_FILES)
+
+    output = "".join(all_outs)
+    session.log(output)
+
+
 # TODO(yon-mg): -add compute context manager that includes rest transport
 #               -add compute unit tests
 #               (to test against temporarily while rest transport is incomplete)
@@ -114,8 +203,7 @@ def showcase_library(
             f"google/showcase/v1beta1/messaging.proto",
         )
         session.run(
-            *cmd_tup,
-            external=True,
+            *cmd_tup, external=True,
         )
 
         # Install the library.
@@ -124,7 +212,7 @@ def showcase_library(
         yield tmp_dir
 
 
-@nox.session(python="3.9")
+@nox.session(python=NEWEST_PYTHON)
 def showcase(
     session,
     templates="DEFAULT",
@@ -136,12 +224,14 @@ def showcase(
     with showcase_library(session, templates=templates, other_opts=other_opts):
         session.install("mock", "pytest", "pytest-asyncio")
         session.run(
-            "py.test", "--quiet", *(session.posargs or [path.join("tests", "system")]),
+            "py.test",
+            "--quiet",
+            *(session.posargs or [path.join("tests", "system")]),
             env=env,
         )
 
 
-@nox.session(python="3.9")
+@nox.session(python=NEWEST_PYTHON)
 def showcase_mtls(
     session,
     templates="DEFAULT",
@@ -161,7 +251,7 @@ def showcase_mtls(
         )
 
 
-@nox.session(python="3.9")
+@nox.session(python=NEWEST_PYTHON)
 def showcase_alternative_templates(session):
     templates = path.join(path.dirname(__file__), "gapic", "ads-templates")
     showcase(
@@ -172,7 +262,7 @@ def showcase_alternative_templates(session):
     )
 
 
-@nox.session(python="3.9")
+@nox.session(python=NEWEST_PYTHON)
 def showcase_mtls_alternative_templates(session):
     templates = path.join(path.dirname(__file__), "gapic", "ads-templates")
     showcase_mtls(
@@ -200,12 +290,12 @@ def run_showcase_unit_tests(session, fail_under=100):
         "--quiet",
         "--cov=google",
         "--cov-append",
-       f"--cov-fail-under={str(fail_under)}",
+        f"--cov-fail-under={str(fail_under)}",
         *(session.posargs or [path.join("tests", "unit")]),
     )
 
 
-@nox.session(python=["3.6", "3.7", "3.8", "3.9"])
+@nox.session(python=ALL_PYTHON)
 def showcase_unit(
     session, templates="DEFAULT", other_opts: typing.Iterable[str] = (),
 ):
@@ -233,14 +323,16 @@ def showcase_unit(
         run_showcase_unit_tests(session, fail_under=100)
 
 
-@nox.session(python=["3.7", "3.8", "3.9"])
+@nox.session(python=ALL_PYTHON[1:])  # Do not test 3.6
 def showcase_unit_alternative_templates(session):
-    with showcase_library(session, templates=ADS_TEMPLATES, other_opts=("old-naming",)) as lib:
+    with showcase_library(
+        session, templates=ADS_TEMPLATES, other_opts=("old-naming",)
+    ) as lib:
         session.chdir(lib)
         run_showcase_unit_tests(session)
 
 
-@nox.session(python=["3.9"])
+@nox.session(python=NEWEST_PYTHON)
 def showcase_unit_add_iam_methods(session):
     with showcase_library(session, other_opts=("add-iam-methods",)) as lib:
         session.chdir(lib)
@@ -257,7 +349,7 @@ def showcase_unit_add_iam_methods(session):
         run_showcase_unit_tests(session, fail_under=100)
 
 
-@nox.session(python="3.9")
+@nox.session(python=NEWEST_PYTHON)
 def showcase_mypy(
     session, templates="DEFAULT", other_opts: typing.Iterable[str] = (),
 ):
@@ -273,12 +365,12 @@ def showcase_mypy(
         session.run("mypy", "--explicit-package-bases", "google")
 
 
-@nox.session(python="3.9")
+@nox.session(python=NEWEST_PYTHON)
 def showcase_mypy_alternative_templates(session):
     showcase_mypy(session, templates=ADS_TEMPLATES, other_opts=("old-naming",))
 
 
-@nox.session(python="3.9")
+@nox.session(python=NEWEST_PYTHON)
 def snippetgen(session):
     # Clone googleapis/api-common-protos which are referenced by the snippet
     # protos
@@ -299,14 +391,10 @@ def snippetgen(session):
 
     session.install("grpcio-tools", "mock", "pytest", "pytest-asyncio")
 
-    session.run(
-        "py.test",
-        "-vv",
-        "tests/snippetgen"
-    )
+    session.run("py.test", "-vv", "tests/snippetgen")
 
 
-@nox.session(python="3.9")
+@nox.session(python=NEWEST_PYTHON)
 def docs(session):
     """Build the docs."""
 
@@ -327,15 +415,10 @@ def docs(session):
     )
 
 
-@nox.session(python=["3.7", "3.8", "3.9"])
+@nox.session(python=NEWEST_PYTHON)
 def mypy(session):
     """Perform typecheck analysis."""
 
-    session.install(
-        "mypy",
-        "types-protobuf",
-        "types-PyYAML",
-        "types-dataclasses"
-    )
+    session.install("mypy", "types-protobuf", "types-PyYAML", "types-dataclasses")
     session.install(".")
     session.run("mypy", "gapic")