test: upgrading the ci/cd pipeline to reduce turnover time.

noxfile.py

@@ -18,11 +18,13 @@
 
 from __future__ import absolute_import
 
+from functools import wraps
 import os
 import pathlib
 import re
 import shutil
 import subprocess
+import time
 import warnings
 
 import nox
@@ -76,6 +78,27 @@
 
 CURRENT_DIRECTORY = pathlib.Path(__file__).parent.absolute()
 
+
+def _calculate_duration(func):
+    """This decorator prints the execution time for the decorated function."""
+
+    @wraps(func)
+    def wrapper(*args, **kwargs):
+        start = time.monotonic()
+        result = func(*args, **kwargs)
+        end = time.monotonic()
+        total_seconds = round(end - start)
+        hours = total_seconds // 3600  # Integer division to get hours
+        remaining_seconds = total_seconds % 3600  # Modulo to find remaining seconds
+        minutes = remaining_seconds // 60
+        seconds = remaining_seconds % 60
+        human_time = f"{hours:}:{minutes:0>2}:{seconds:0>2}"
+        print(f"Session ran in {total_seconds} seconds ({human_time})")
+        return result
+
+    return wrapper
+
+
 # 'docfx' is excluded since it only needs to run in 'docs-presubmit'
 nox.options.sessions = [
     "unit",
@@ -92,6 +115,7 @@
 
 
 @nox.session(python=DEFAULT_PYTHON_VERSION)
+@_calculate_duration
 def lint(session):
     """Run linters.
 
@@ -108,6 +132,7 @@ def lint(session):
 
 
 @nox.session(python=DEFAULT_PYTHON_VERSION)
+@_calculate_duration
 def blacken(session):
     """Run black. Format code to uniform standard."""
     session.install(BLACK_VERSION)
@@ -118,6 +143,7 @@ def blacken(session):
 
 
 @nox.session(python=DEFAULT_PYTHON_VERSION)
+@_calculate_duration
 def format(session):
     """
     Run isort to sort imports. Then run black
@@ -138,6 +164,7 @@ def format(session):
 
 
 @nox.session(python=DEFAULT_PYTHON_VERSION)
+@_calculate_duration
 def lint_setup_py(session):
     """Verify that setup.py is valid (including RST check)."""
     session.install("docutils", "pygments")
@@ -199,6 +226,7 @@ def default(session):
 
 
 @nox.session(python=UNIT_TEST_PYTHON_VERSIONS)
+@_calculate_duration
 def unit(session):
     """Run the unit test suite."""
     default(session)
@@ -235,6 +263,7 @@ def install_systemtest_dependencies(session, *constraints):
 
 
 @nox.session(python=SYSTEM_TEST_PYTHON_VERSIONS)
+@_calculate_duration
 def system(session):
     """Run the system test suite."""
     constraints_path = str(
@@ -281,6 +310,7 @@ def system(session):
 
 
 @nox.session(python=DEFAULT_PYTHON_VERSION)
+@_calculate_duration
 def prerelease(session):
     session.install(
         "--extra-index-url",
@@ -334,7 +364,7 @@ def prerelease(session):
         constraints_text = constraints_file.read()
 
     # Ignore leading whitespace and comment lines.
-    deps = [
+    constraints_deps = [
         match.group(1)
         for match in re.finditer(
             r"^\s*(\S+)(?===\S+)", constraints_text, flags=re.MULTILINE
@@ -343,7 +373,7 @@ def prerelease(session):
 
     # We use --no-deps to ensure that pre-release versions aren't overwritten
     # by the version ranges in setup.py.
-    session.install(*deps)
+    session.install(*constraints_deps)
     session.install("--no-deps", "-e", ".[all]")
 
     # Print out prerelease package versions.
@@ -368,6 +398,7 @@ def prerelease(session):
 
 
 @nox.session(python=DEFAULT_PYTHON_VERSION)
+@_calculate_duration
 def cover(session):
     """Run the final coverage report.
 
@@ -390,6 +421,7 @@ def cover(session):
 
 
 @nox.session(python="3.9")
+@_calculate_duration
 def docs(session):
     """Build the docs for this library."""
 
@@ -425,6 +457,7 @@ def docs(session):
 
 
 @nox.session(python="3.10")
+@_calculate_duration
 def docfx(session):
     """Build the docfx yaml files for this library."""
 
@@ -470,92 +503,6 @@ def docfx(session):
     )
 
 
-@nox.session(python=SYSTEM_TEST_PYTHON_VERSIONS)
-def prerelease_deps(session):
-    """Run all tests with prerelease versions of dependencies installed."""
-
-    # Install all dependencies
-    session.install("-e", ".[all, tests, tracing]")
-    unit_deps_all = UNIT_TEST_STANDARD_DEPENDENCIES + UNIT_TEST_EXTERNAL_DEPENDENCIES
-    session.install(*unit_deps_all)
-    system_deps_all = (
-        SYSTEM_TEST_STANDARD_DEPENDENCIES + SYSTEM_TEST_EXTERNAL_DEPENDENCIES
-    )
-    session.install(*system_deps_all)
-
-    # Because we test minimum dependency versions on the minimum Python
-    # version, the first version we test with in the unit tests sessions has a
-    # constraints file containing all dependencies and extras.
-    with open(
-        CURRENT_DIRECTORY
-        / "testing"
-        / f"constraints-{UNIT_TEST_PYTHON_VERSIONS[0]}.txt",
-        encoding="utf-8",
-    ) as constraints_file:
-        constraints_text = constraints_file.read()
-
-    # Ignore leading whitespace and comment lines.
-    constraints_deps = [
-        match.group(1)
-        for match in re.finditer(
-            r"^\s*(\S+)(?===\S+)", constraints_text, flags=re.MULTILINE
-        )
-    ]
-
-    session.install(*constraints_deps)
-
-    prerel_deps = [
-        # "protobuf",
-        # dependency of grpc
-        "six",
-        "googleapis-common-protos",
-        # Exclude version 1.52.0rc1 which has a known issue. See https://github.com/grpc/grpc/issues/32163
-        "grpcio!=1.52.0rc1",
-        "grpcio-status",
-        "google-api-core",
-        "google-auth",
-        "proto-plus",
-        "google-cloud-testutils",
-        # dependencies of google-cloud-testutils"
-        "click",
-    ]
-
-    for dep in prerel_deps:
-        session.install("--pre", "--no-deps", "--upgrade", dep)
-
-    # Remaining dependencies
-    other_deps = [
-        "requests",
-    ]
-    session.install(*other_deps)
-
-    # Print out package versions.
-    session.run("python", "-m", "pip", "freeze")
-
-    session.run("py.test", "tests/unit")
-
-    system_test_path = os.path.join("tests", "system.py")
-    system_test_folder_path = os.path.join("tests", "system")
-
-    # Only run system tests if found.
-    if os.path.exists(system_test_path):
-        session.run(
-            "py.test",
-            "--verbose",
-            f"--junitxml=system_{session.python}_sponge_log.xml",
-            system_test_path,
-            *session.posargs,
-        )
-    if os.path.exists(system_test_folder_path):
-        session.run(
-            "py.test",
-            "--verbose",
-            f"--junitxml=system_{session.python}_sponge_log.xml",
-            system_test_folder_path,
-            *session.posargs,
-        )
-
-
 def install_conda_unittest_dependencies(session, standard_deps, conda_forge_packages):
     """Installs packages from conda forge, pypi, and locally."""
 
@@ -573,6 +520,7 @@ def install_conda_unittest_dependencies(session, standard_deps, conda_forge_pack
 
 
 @nox.session(python=CONDA_TEST_PYTHON_VERSIONS, venv_backend="mamba")
+@_calculate_duration
 def conda_test(session):
     """Run test suite in a conda virtual environment.
 

owlbot.py

@@ -56,9 +56,13 @@
         "docs/multiprocessing.rst",
         "noxfile.py",
         "README.rst",
+
         # exclude .kokoro/build.sh which is customized due to support for conda
         ".kokoro/build.sh",
-    ],
+
+        # exclude this file as we have an alternate prerelease.cfg
+        ".kokoro/presubmit/prerelease-deps.cfg",
+    ]
 )
 
 # ----------------------------------------------------------------------------