Add Black. Template noxfile + flake8.

asset/.flake8

@@ -1,4 +1,5 @@
 [flake8]
+ignore = E203, E266, E501, W503
 exclude =
   # Exclude generated code.
   **/proto/**

asset/synth.py

@@ -18,17 +18,14 @@
 from synthtool import gcp
 
 gapic = gcp.GAPICGenerator()
-
+common = gcp.CommonTemplates()
 versions = ["v1beta1"]
 
-excludes = [
-    'setup.py',
-    'nox*.py',
-    'README.rst',
-    'docs/conf.py',
-    'docs/index.rst',
-]
+excludes = ["setup.py", "nox*.py", "README.rst", "docs/conf.py", "docs/index.rst"]
 
+# ----------------------------------------------------------------------------
+# Generate asset GAPIC layer
+# ----------------------------------------------------------------------------
 for version in versions:
     library = gapic.py_library(
         "asset",
@@ -39,10 +36,6 @@
 
     s.move(library, excludes=excludes)
 
-templated_files = gcp.CommonTemplates().py_library(
-    unit_cov_level=86, cov_level=85)
-s.move(templated_files)
-
 s.replace(
     "google/cloud/asset_v1beta1/proto/assets_pb2.py",
     "from google.iam.v1 import policy_pb2 as",
@@ -85,3 +78,9 @@
     _BORKED_ASSET_DOCSTRING,
     _FIXED_ASSET_DOCSTRING,
 )
+
+# ----------------------------------------------------------------------------
+# Add templated files
+# ----------------------------------------------------------------------------
+templated_files = gcp.CommonTemplates().py_library(unit_cov_level=86, cov_level=85)
+s.move(templated_files)

automl/.coveragerc

@@ -9,6 +9,8 @@ exclude_lines =
     pragma: NO COVER
     # Ignore debug-only repr
     def __repr__
+    # Ignore abstract methods
+    raise NotImplementedError
 omit =
   */gapic/*.py
   */proto/*.py

automl/.flake8

@@ -1,4 +1,5 @@
 [flake8]
+ignore = E203, E266, E501, W503
 exclude =
   # Exclude generated code.
   **/proto/**

automl/noxfile.py

@@ -1,3 +1,5 @@
+# -*- coding: utf-8 -*-
+#
 # Copyright 2018 Google LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -18,31 +20,116 @@
 import nox
 
 
-LOCAL_DEPS = (
-    os.path.join('..', 'api_core'),
-)
+LOCAL_DEPS = (os.path.join("..", "api_core"),)
 
 
 def default(session):
     # Install all test dependencies, then install this package in-place.
-    session.install('pytest', 'mock')
+    session.install("mock", "pytest", "pytest-cov")
     for local_dep in LOCAL_DEPS:
-        session.install('-e', local_dep)
-    session.install('-e', '.')
+        session.install("-e", local_dep)
+    session.install("-e", ".")
 
     # Run py.test against the unit tests.
-    session.run('py.test', '--quiet', os.path.join('tests', 'unit'))
+    session.run(
+        "py.test",
+        "--quiet",
+        "--cov=google.cloud",
+        "--cov=tests.unit",
+        "--cov-append",
+        "--cov-config=.coveragerc",
+        "--cov-report=",
+        "--cov-fail-under=82",
+        os.path.join("tests", "unit"),
+        *session.posargs,
+    )
 
 
-@nox.session(python=['2.7', '3.5', '3.6', '3.7'])
+@nox.session(python=["2.7", "3.5", "3.6", "3.7"])
 def unit(session):
     """Run the unit test suite."""
     default(session)
 
 
-@nox.session(python='3.6')
+@nox.session(python=["2.7", "3.7"])
+def system(session):
+    """Run the system test suite."""
+    system_test_path = os.path.join("tests", "system.py")
+    # Sanity check: Only run tests if the environment variable is set.
+    if not os.environ.get("GOOGLE_APPLICATION_CREDENTIALS", ""):
+        session.skip("Credentials must be set via environment variable")
+    # Sanity check: only run tests if found.
+    if not os.path.exists(system_test_path):
+        session.skip("System tests were not found")
+
+    # Use pre-release gRPC for system tests.
+    session.install("--pre", "grpcio")
+
+    # Install all test dependencies, then install this package into the
+    # virtualenv's dist-packages.
+    session.install("mock", "pytest")
+    for local_dep in LOCAL_DEPS:
+        session.install("-e", local_dep)
+    session.install("-e", "../test_utils/")
+    session.install("-e", ".")
+
+    # Run py.test against the system tests.
+    session.run("py.test", "--quiet", system_test_path, *session.posargs)
+
+
+@nox.session(python="3.7")
+def blacken(session):
+    """Run black.
+
+    Format code to uniform standard.
+    """
+    session.install("black")
+    session.run(
+        "black",
+        "google",
+        "tests",
+        "docs",
+        "--exclude",
+        ".*/proto/.*|.*/gapic/.*|.*/.*_pb2.py",
+    )
+
+
+@nox.session(python="3.7")
+def lint(session):
+    """Run linters.
+
+    Returns a failure if the linters find linting errors or sufficiently
+    serious code quality issues.
+    """
+    session.install("flake8", "black", *LOCAL_DEPS)
+    session.install(".")
+    session.run(
+        "black",
+        "--check",
+        "google",
+        "tests",
+        "docs",
+        "--exclude",
+        ".*/proto/.*|.*/gapic/.*|.*/.*_pb2.py",
+    )
+    session.run("flake8", "google", "tests")
+
+
+@nox.session(python="3.7")
 def lint_setup_py(session):
     """Verify that setup.py is valid (including RST check)."""
-    session.install('docutils', 'pygments')
-    session.run('python', 'setup.py', 'check', '--restructuredtext',
-                '--strict')
+    session.install("docutils", "pygments")
+    session.run("python", "setup.py", "check", "--restructuredtext", "--strict")
+
+
+@nox.session(python="3.7")
+def cover(session):
+    """Run the final coverage report.
+
+    This outputs the coverage report aggregating coverage from the unit
+    test runs (not system test runs), and then erases coverage data.
+    """
+    session.install("coverage", "pytest-cov")
+    session.run("coverage", "report", "--show-missing", "--fail-under=80")
+
+    session.run("coverage", "erase")

automl/synth.py

@@ -18,32 +18,42 @@
 from synthtool import gcp
 
 gapic = gcp.GAPICGenerator()
-
-versions = ['v1beta1']
+common = gcp.CommonTemplates()
+versions = ["v1beta1"]
 
 
+# ----------------------------------------------------------------------------
+# Generate automl GAPIC layer
+# ----------------------------------------------------------------------------
 for version in versions:
-    library = gapic.py_library('automl', version)
-    s.move(library / f'google/cloud/automl_{version}')
-    s.move(library / f'tests/unit/gapic/{version}')
-    s.move(library / f'docs/gapic/{version}')
+    library = gapic.py_library("automl", version)
+    s.move(library / f"google/cloud/automl_{version}")
+    s.move(library / f"tests/unit/gapic/{version}")
+    s.move(library / f"docs/gapic/{version}")
 
 # Use the highest version library to generate import alias.
-s.move(library / 'google/cloud/automl.py')
-
+s.move(library / "google/cloud/automl.py")
 
 # Fixup issues in generated code
 s.replace(
-    '**/gapic/*_client.py',
-    r'metadata_type=operations_pb2.OperationMetadata',
-    r'metadata_type=proto_operations_pb2.OperationMetadata')
+    "**/gapic/*_client.py",
+    r"metadata_type=operations_pb2.OperationMetadata",
+    r"metadata_type=proto_operations_pb2.OperationMetadata",
+)
 
 # Fix spacing/'::' issues in docstrings
 s.replace(
-    'google/cloud/automl_v1beta1/gapic/prediction_service_client.py',
-    '^\s+::',
-    '')
+    "google/cloud/automl_v1beta1/gapic/prediction_service_client.py", "^\s+::", ""
+)
 
-s.replace('google/cloud/automl_v1beta1/gapic/auto_ml_client.py',
-          '^(\s+)(::)\n\n\s+?([^\s])',
-          '    \g<1>\g<2>\n    \g<1>\g<3>')
+s.replace(
+    "google/cloud/automl_v1beta1/gapic/auto_ml_client.py",
+    "^(\s+)(::)\n\n\s+?([^\s])",
+    "    \g<1>\g<2>\n    \g<1>\g<3>",
+)
+
+# ----------------------------------------------------------------------------
+# Add templated files
+# ----------------------------------------------------------------------------
+templated_files = common.py_library(unit_cov_level=82, cov_level=80)
+s.move(templated_files)

bigquery/.coveragerc

@@ -11,3 +11,6 @@ exclude_lines =
     def __repr__
     # Ignore abstract methods
     raise NotImplementedError
+omit =
+  */gapic/*.py
+  */proto/*.py

bigquery/.flake8

@@ -1,4 +1,5 @@
 [flake8]
+ignore = E203, E266, E501, W503
 exclude =
   # Exclude generated code.
   **/proto/**

bigquery_datatransfer/.flake8

@@ -0,0 +1,13 @@
+[flake8]
+ignore = E203, E266, E501, W503
+exclude =
+  # Exclude generated code.
+  **/proto/**
+  **/gapic/**
+  *_pb2.py
+
+  # Standard linting exemptions.
+  __pycache__,
+  .git,
+  *.pyc,
+  conf.py