build(python): fix docs and docfx builds (#939)

* build(python): fix `docs` and `docfx` builds

Source-Link: googleapis/synthtool@fac8444
Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:5ea6d0ab82c956b50962f91d94e206d3921537ae5fe1549ec5326381d8905cfa

* remove editable install to fix tests

* See googleapis/synthtool#1916

* See googleapis/synthtool#1916

* exclude .nox from coverage

* feat: Introduce compatibility with native namespace packages

* refactor unit test and coverage tests

---------

Co-authored-by: Owl Bot <gcf-owl-bot[bot]@users.noreply.github.com>
Co-authored-by: Anthonios Partheniou <partheniou@google.com>

.coveragerc

@@ -8,6 +8,7 @@ exclude_lines =
     # Re-enable the standard pragma
     pragma: NO COVER
 omit =
+  .nox/*
   */gapic/*.py
   */proto/*.py
   tests/*/*.py

.github/.OwlBot.lock.yaml

@@ -1,4 +1,4 @@
-# Copyright 2023 Google LLC
+# Copyright 2024 Google LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -13,5 +13,5 @@
 # limitations under the License.
 docker:
   image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest
-  digest: sha256:caffe0a9277daeccc4d1de5c9b55ebba0901b57c2f713ec9c876b0d4ec064f61
-# created: 2023-11-08T19:46:45.022803742Z
+  digest: sha256:5ea6d0ab82c956b50962f91d94e206d3921537ae5fe1549ec5326381d8905cfa
+# created: 2024-01-15T16:32:08.142785673Z

.gitignore

@@ -50,3 +50,7 @@ htmlcov
 
 # Built documentation
 docs/_build
+
+# Test logs
+coverage.xml
+*sponge_log.xml

.kokoro/noxfile.py

@@ -89,7 +89,7 @@ def get_pytest_env_vars() -> Dict[str, str]:
 
 # DO NOT EDIT - automatically generated.
 # All versions used to test samples.
-ALL_VERSIONS = ["3.7", "3.8", "3.9", "3.10", "3.11"]
+ALL_VERSIONS = ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"]
 
 # Any default versions that should be ignored.
 IGNORED_VERSIONS = TEST_CONFIG["ignored_versions"]

.kokoro/requirements.txt

@@ -93,30 +93,30 @@ colorlog==6.7.0 \
     # via
     #   gcp-docuploader
     #   nox
-cryptography==41.0.5 \
-    --hash=sha256:0c327cac00f082013c7c9fb6c46b7cc9fa3c288ca702c74773968173bda421bf \
-    --hash=sha256:0d2a6a598847c46e3e321a7aef8af1436f11c27f1254933746304ff014664d84 \
-    --hash=sha256:227ec057cd32a41c6651701abc0328135e472ed450f47c2766f23267b792a88e \
-    --hash=sha256:22892cc830d8b2c89ea60148227631bb96a7da0c1b722f2aac8824b1b7c0b6b8 \
-    --hash=sha256:392cb88b597247177172e02da6b7a63deeff1937fa6fec3bbf902ebd75d97ec7 \
-    --hash=sha256:3be3ca726e1572517d2bef99a818378bbcf7d7799d5372a46c79c29eb8d166c1 \
-    --hash=sha256:573eb7128cbca75f9157dcde974781209463ce56b5804983e11a1c462f0f4e88 \
-    --hash=sha256:580afc7b7216deeb87a098ef0674d6ee34ab55993140838b14c9b83312b37b86 \
-    --hash=sha256:5a70187954ba7292c7876734183e810b728b4f3965fbe571421cb2434d279179 \
-    --hash=sha256:73801ac9736741f220e20435f84ecec75ed70eda90f781a148f1bad546963d81 \
-    --hash=sha256:7d208c21e47940369accfc9e85f0de7693d9a5d843c2509b3846b2db170dfd20 \
-    --hash=sha256:8254962e6ba1f4d2090c44daf50a547cd5f0bf446dc658a8e5f8156cae0d8548 \
-    --hash=sha256:88417bff20162f635f24f849ab182b092697922088b477a7abd6664ddd82291d \
-    --hash=sha256:a48e74dad1fb349f3dc1d449ed88e0017d792997a7ad2ec9587ed17405667e6d \
-    --hash=sha256:b948e09fe5fb18517d99994184854ebd50b57248736fd4c720ad540560174ec5 \
-    --hash=sha256:c707f7afd813478e2019ae32a7c49cd932dd60ab2d2a93e796f68236b7e1fbf1 \
-    --hash=sha256:d38e6031e113b7421db1de0c1b1f7739564a88f1684c6b89234fbf6c11b75147 \
-    --hash=sha256:d3977f0e276f6f5bf245c403156673db103283266601405376f075c849a0b936 \
-    --hash=sha256:da6a0ff8f1016ccc7477e6339e1d50ce5f59b88905585f77193ebd5068f1e797 \
-    --hash=sha256:e270c04f4d9b5671ebcc792b3ba5d4488bf7c42c3c241a3748e2599776f29696 \
-    --hash=sha256:e886098619d3815e0ad5790c973afeee2c0e6e04b4da90b88e6bd06e2a0b1b72 \
-    --hash=sha256:ec3b055ff8f1dce8e6ef28f626e0972981475173d7973d63f271b29c8a2897da \
-    --hash=sha256:fba1e91467c65fe64a82c689dc6cf58151158993b13eb7a7f3f4b7f395636723
+cryptography==41.0.6 \
+    --hash=sha256:068bc551698c234742c40049e46840843f3d98ad7ce265fd2bd4ec0d11306596 \
+    --hash=sha256:0f27acb55a4e77b9be8d550d762b0513ef3fc658cd3eb15110ebbcbd626db12c \
+    --hash=sha256:2132d5865eea673fe6712c2ed5fb4fa49dba10768bb4cc798345748380ee3660 \
+    --hash=sha256:3288acccef021e3c3c10d58933f44e8602cf04dba96d9796d70d537bb2f4bbc4 \
+    --hash=sha256:35f3f288e83c3f6f10752467c48919a7a94b7d88cc00b0668372a0d2ad4f8ead \
+    --hash=sha256:398ae1fc711b5eb78e977daa3cbf47cec20f2c08c5da129b7a296055fbb22aed \
+    --hash=sha256:422e3e31d63743855e43e5a6fcc8b4acab860f560f9321b0ee6269cc7ed70cc3 \
+    --hash=sha256:48783b7e2bef51224020efb61b42704207dde583d7e371ef8fc2a5fb6c0aabc7 \
+    --hash=sha256:4d03186af98b1c01a4eda396b137f29e4e3fb0173e30f885e27acec8823c1b09 \
+    --hash=sha256:5daeb18e7886a358064a68dbcaf441c036cbdb7da52ae744e7b9207b04d3908c \
+    --hash=sha256:60e746b11b937911dc70d164060d28d273e31853bb359e2b2033c9e93e6f3c43 \
+    --hash=sha256:742ae5e9a2310e9dade7932f9576606836ed174da3c7d26bc3d3ab4bd49b9f65 \
+    --hash=sha256:7e00fb556bda398b99b0da289ce7053639d33b572847181d6483ad89835115f6 \
+    --hash=sha256:85abd057699b98fce40b41737afb234fef05c67e116f6f3650782c10862c43da \
+    --hash=sha256:8efb2af8d4ba9dbc9c9dd8f04d19a7abb5b49eab1f3694e7b5a16a5fc2856f5c \
+    --hash=sha256:ae236bb8760c1e55b7a39b6d4d32d2279bc6c7c8500b7d5a13b6fb9fc97be35b \
+    --hash=sha256:afda76d84b053923c27ede5edc1ed7d53e3c9f475ebaf63c68e69f1403c405a8 \
+    --hash=sha256:b27a7fd4229abef715e064269d98a7e2909ebf92eb6912a9603c7e14c181928c \
+    --hash=sha256:b648fe2a45e426aaee684ddca2632f62ec4613ef362f4d681a9a6283d10e079d \
+    --hash=sha256:c5a550dc7a3b50b116323e3d376241829fd326ac47bc195e04eb33a8170902a9 \
+    --hash=sha256:da46e2b5df770070412c46f87bac0849b8d685c5f2679771de277a422c7d0b86 \
+    --hash=sha256:f39812f70fc5c71a15aa3c97b2bbe213c3f2a460b79bd21c40d033bb34a9bf36 \
+    --hash=sha256:ff369dd19e8fe0528b02e8df9f2aeb2479f89b1270d90f96a63500afe9af5cae
     # via
     #   gcp-releasetool
     #   secretstorage
@@ -263,9 +263,9 @@ jeepney==0.8.0 \
     # via
     #   keyring
     #   secretstorage
-jinja2==3.1.2 \
-    --hash=sha256:31351a702a408a9e7595a8fc6150fc3f43bb6bf7e319770cbc0db9df9437e852 \
-    --hash=sha256:6088930bfe239f0e6710546ab9c19c9ef35e29792895fed6e6e31a023a182a61
+jinja2==3.1.3 \
+    --hash=sha256:7d6d50dd97d52cbc355597bd845fabfbac3f551e1f99619e39a35ce8c370b5fa \
+    --hash=sha256:ac8bd6544d4bb2c9792bf3a159e80bba8fda7f07e81bc3aed565432d5925ba90
     # via gcp-releasetool
 keyring==24.2.0 \
     --hash=sha256:4901caaf597bfd3bbd78c9a0c7c4c29fcd8310dab2cffefe749e916b6527acd6 \

.kokoro/samples/python3.12/common.cfg

@@ -0,0 +1,40 @@
+# Format: //devtools/kokoro/config/proto/build.proto
+
+# Build logs will be here
+action {
+  define_artifacts {
+    regex: "**/*sponge_log.xml"
+  }
+}
+
+# Specify which tests to run
+env_vars: {
+    key: "RUN_TESTS_SESSION"
+    value: "py-3.12"
+}
+
+# Declare build specific Cloud project.
+env_vars: {
+    key: "BUILD_SPECIFIC_GCLOUD_PROJECT"
+    value: "python-docs-samples-tests-312"
+}
+
+env_vars: {
+    key: "TRAMPOLINE_BUILD_FILE"
+    value: "github/python-ndb/.kokoro/test-samples.sh"
+}
+
+# Configure the docker image for kokoro-trampoline.
+env_vars: {
+    key: "TRAMPOLINE_IMAGE"
+    value: "gcr.io/cloud-devrel-kokoro-resources/python-samples-testing-docker"
+}
+
+# Download secrets for samples
+gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples"
+
+# Download trampoline resources.
+gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline"
+
+# Use the trampoline script to run in docker.
+build_file: "python-ndb/.kokoro/trampoline_v2.sh"

.kokoro/samples/python3.12/continuous.cfg

@@ -0,0 +1,6 @@
+# Format: //devtools/kokoro/config/proto/build.proto
+
+env_vars: {
+    key: "INSTALL_LIBRARY_FROM_SOURCE"
+    value: "True"
+}

.kokoro/samples/python3.12/periodic-head.cfg

@@ -0,0 +1,11 @@
+# Format: //devtools/kokoro/config/proto/build.proto
+
+env_vars: {
+    key: "INSTALL_LIBRARY_FROM_SOURCE"
+    value: "True"
+}
+
+env_vars: {
+    key: "TRAMPOLINE_BUILD_FILE"
+    value: "github/python-ndb/.kokoro/test-samples-against-head.sh"
+}

.kokoro/samples/python3.12/periodic.cfg

@@ -0,0 +1,6 @@
+# Format: //devtools/kokoro/config/proto/build.proto
+
+env_vars: {
+    key: "INSTALL_LIBRARY_FROM_SOURCE"
+    value: "False"
+}

.kokoro/samples/python3.12/presubmit.cfg

@@ -0,0 +1,6 @@
+# Format: //devtools/kokoro/config/proto/build.proto
+
+env_vars: {
+    key: "INSTALL_LIBRARY_FROM_SOURCE"
+    value: "True"
+}

noxfile.py

@@ -30,52 +30,64 @@
 CURRENT_DIRECTORY = pathlib.Path(__file__).parent.absolute()
 
 BLACK_VERSION = "black==22.3.0"
+UNIT_TEST_STANDARD_DEPENDENCIES = [
+    "mock",
+    "asyncmock",
+    "pytest",
+    "pytest-cov",
+    "google-cloud-testutils",
+    "google-cloud-core",
+]
 
 
 def get_path(*names):
     return os.path.join(NOX_DIR, *names)
 
 
-@nox.session(py=ALL_INTERPRETERS)
-def unit(session):
+def install_unittest_dependencies(session, *constraints):
+    standard_deps = UNIT_TEST_STANDARD_DEPENDENCIES
+    session.install(*standard_deps, *constraints)
+    session.install("-e", ".", *constraints)
+
+
+def default(session):
+    # Install all test dependencies, then install this package in-place.
     constraints_path = str(
         CURRENT_DIRECTORY / "testing" / f"constraints-{session.python}.txt"
     )
-    # Install all dependencies.
-    session.install("pytest", "pytest-cov")
-    session.install("google-cloud-testutils", "-c", constraints_path)
-    session.install("-e", ".", "-c", constraints_path)
-    # This variable is used to skip coverage by Python version
-    session.env["PY_VERSION"] = session.python[0]
+    install_unittest_dependencies(session, "-c", constraints_path)
     # Run py.test against the unit tests.
-    run_args = ["pytest"]
-    if session.posargs:
-        run_args.extend(session.posargs)
-    else:
-        run_args.extend(
-            [
-                "--cov=google.cloud.ndb",
-                "--cov=unit",
-                "--cov-append",
-                "--cov-config",
-                get_path(".coveragerc"),
-                "--cov-report=term-missing",
-            ]
-        )
-    run_args.append(get_path("tests", "unit"))
-    session.run(*run_args)
+    session.run(
+        "py.test",
+        "--quiet",
+        f"--junitxml=unit_{session.python}_sponge_log.xml",
+        "--cov=google",
+        "--cov=tests/unit",
+        "--cov-append",
+        "--cov-config=.coveragerc",
+        "--cov-report=",
+        "--cov-fail-under=0",
+        os.path.join("tests", "unit"),
+        *session.posargs,
+    )
+
 
-    if not session.posargs:
-        session.notify("cover")
+@nox.session(python=ALL_INTERPRETERS)
+def unit(session):
+    """Run the unit test suite."""
+    default(session)
 
 
 @nox.session(py=DEFAULT_INTERPRETER)
 def cover(session):
-    # Install all dependencies.
-    session.install("coverage")
-    # Run coverage report.
-    session.run("coverage", "report", "--fail-under=100", "--show-missing")
-    # Erase cached coverage data.
+    """Run the final coverage report.
+
+    This outputs the coverage report aggregating coverage from the unit
+    test runs (not system test runs), and then erases coverage data.
+    """
+    session.install("coverage", "pytest-cov")
+    session.run("coverage", "report", "--show-missing", "--fail-under=100")
+
     session.run("coverage", "erase")
 
 
@@ -119,9 +131,21 @@ def blacken(session):
 def docs(session):
     """Build the docs for this library."""
 
-    session.install("-e", ".")
+    session.install(".")
     session.install(
-        "Sphinx==4.0.1", "alabaster", "recommonmark", "sphinxcontrib.spelling"
+        # We need to pin to specific versions of the `sphinxcontrib-*` packages
+        # which still support sphinx 4.x.
+        # See https://github.com/googleapis/sphinx-docfx-yaml/issues/344
+        # and https://github.com/googleapis/sphinx-docfx-yaml/issues/345.
+        "sphinxcontrib-applehelp==1.0.4",
+        "sphinxcontrib-devhelp==1.0.2",
+        "sphinxcontrib-htmlhelp==2.0.1",
+        "sphinxcontrib-qthelp==1.0.3",
+        "sphinxcontrib-serializinghtml==1.1.5",
+        "sphinx==4.0.1",
+        "alabaster",
+        "recommonmark",
+        "sphinxcontrib.spelling",
     )
 
     shutil.rmtree(os.path.join("docs", "_build"), ignore_errors=True)
@@ -142,8 +166,19 @@ def docs(session):
 @nox.session(py="3.9")
 def doctest(session):
     # Install all dependencies.
-    session.install("Sphinx==4.0.1")
-    session.install("sphinxcontrib.spelling")
+    session.install(
+        # We need to pin to specific versions of the `sphinxcontrib-*` packages
+        # which still support sphinx 4.x.
+        # See https://github.com/googleapis/sphinx-docfx-yaml/issues/344
+        # and https://github.com/googleapis/sphinx-docfx-yaml/issues/345.
+        "sphinxcontrib-applehelp==1.0.4",
+        "sphinxcontrib-devhelp==1.0.2",
+        "sphinxcontrib-htmlhelp==2.0.1",
+        "sphinxcontrib-qthelp==1.0.3",
+        "sphinxcontrib-serializinghtml==1.1.5",
+        "sphinx==4.0.1",
+        "sphinxcontrib.spelling",
+    )
     session.install(".")
     # Run the script for building docs and running doctests.
     run_args = [
@@ -190,7 +225,7 @@ def system(session):
     session.install("google-cloud-testutils")
     for local_dep in LOCAL_DEPS:
         session.install(local_dep)
-    session.install("-e", ".", "-c", constraints_path)
+    session.install(".", "-c", constraints_path)
 
     # Run py.test against the system tests.
     if system_test_exists: