Add templates for flake8, coveragerc, noxfile, and black. (#6642)

packages/google-cloud-dataproc/.coveragerc

@@ -0,0 +1,18 @@
+[run]
+branch = True
+
+[report]
+fail_under = 100
+show_missing = True
+exclude_lines =
+    # Re-enable the standard pragma
+    pragma: NO COVER
+    # Ignore debug-only repr
+    def __repr__
+    # Ignore abstract methods
+    raise NotImplementedError
+omit =
+  */gapic/*.py
+  */proto/*.py
+  */google-cloud-python/core/*.py
+  */site-packages/*.py

packages/google-cloud-dataproc/.flake8

@@ -0,0 +1,13 @@
+[flake8]
+ignore = E203, E266, E501, W503
+exclude =
+  # Exclude generated code.
+  **/proto/**
+  **/gapic/**
+  *_pb2.py
+
+  # Standard linting exemptions.
+  __pycache__,
+  .git,
+  *.pyc,
+  conf.py

packages/google-cloud-dataproc/noxfile.py

@@ -1,4 +1,6 @@
-# Copyright 2017 Google LLC
+# -*- coding: utf-8 -*-
+#
+# Copyright 2018 Google LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -13,69 +15,126 @@
 # limitations under the License.
 
 from __future__ import absolute_import
-
 import os
 
 import nox
 
 
-LOCAL_DEPS = (
-    os.path.join('..', 'api_core'),
-)
+LOCAL_DEPS = (os.path.join("..", "api_core"), os.path.join("..", "core"))
 
+@nox.session(python="3.7")
+def blacken(session):
+    """Run black.
 
-def default(session):
-    """Default unit test session.
+    Format code to uniform standard.
+    """
+    session.install("black")
+    session.run(
+        "black",
+        "google",
+        "tests",
+        "docs",
+        "--exclude",
+        ".*/proto/.*|.*/gapic/.*|.*/.*_pb2.py",
+    )
 
-    This is intended to be run **without** an interpreter set, so
-    that the current ``python`` (on the ``PATH``) or the version of
-    Python corresponding to the ``nox`` binary on the ``PATH`` can
-    run the tests.
+
+@nox.session(python="3.7")
+def lint(session):
+    """Run linters.
+
+    Returns a failure if the linters find linting errors or sufficiently
+    serious code quality issues.
     """
+    session.install("flake8", "black", *LOCAL_DEPS)
+    session.run(
+        "black",
+        "--check",
+        "google",
+        "tests",
+        "docs",
+        "--exclude",
+        ".*/proto/.*|.*/gapic/.*|.*/.*_pb2.py",
+    )
+    session.run("flake8", "google", "tests")
+
+
+@nox.session(python="3.7")
+def lint_setup_py(session):
+    """Verify that setup.py is valid (including RST check)."""
+    session.install("docutils", "pygments")
+    session.run("python", "setup.py", "check", "--restructuredtext", "--strict")
+
+
+def default(session):
     # Install all test dependencies, then install this package in-place.
-    session.install('mock', 'pytest', 'pytest-cov', *LOCAL_DEPS)
-    session.install('-e', '.')
+    session.install("mock", "pytest", "pytest-cov")
+    for local_dep in LOCAL_DEPS:
+        session.install("-e", local_dep)
+    session.install("-e", ".")
 
     # Run py.test against the unit tests.
     session.run(
-        'py.test',
-        '--quiet',
-        '--cov=google.cloud.dataproc',
-        '--cov=google.cloud.dataproc_v1',
-        '--cov=tests.unit',
-        '--cov-append',
-        '--cov-config=.coveragerc',
-        '--cov-report=',
-        os.path.join('tests', 'unit'),
-        *session.posargs
+        "py.test",
+        "--quiet",
+        "--cov=google.cloud",
+        "--cov=tests.unit",
+        "--cov-append",
+        "--cov-config=.coveragerc",
+        "--cov-report=",
+        "--cov-fail-under=97",
+        os.path.join("tests", "unit"),
+        *session.posargs,
     )
 
 
-@nox.session(python=['2.7', '3.5', '3.6', '3.7'])
+@nox.session(python=["2.7", "3.5", "3.6", "3.7"])
 def unit(session):
     """Run the unit test suite."""
     default(session)
 
 
-@nox.session(python=['2.7', '3.7'])
+@nox.session(python=["2.7", "3.7"])
 def system(session):
     """Run the system test suite."""
+    system_test_path = os.path.join("tests", "system.py")
+    system_test_folder_path = os.path.join("tests", "system")
+    # Sanity check: Only run tests if the environment variable is set.
+    if not os.environ.get("GOOGLE_APPLICATION_CREDENTIALS", ""):
+        session.skip("Credentials must be set via environment variable")
+
+    system_test_exists = os.path.exists(system_test_path)
+    system_test_folder_exists = os.path.exists(system_test_folder_path)
+    # Sanity check: only run tests if found.
+    if not system_test_exists and not system_test_folder_exists:
+        session.skip("System tests were not found")
 
-    if not os.environ.get('GOOGLE_APPLICATION_CREDENTIALS', ''):
-        session.skip('Credentials must be set via environment variable.')
     # Use pre-release gRPC for system tests.
-    session.install('--pre', 'grpcio')
+    session.install("--pre", "grpcio")
 
-    session.install('pytest')
-    session.install('-e', '.')
+    # Install all test dependencies, then install this package into the
+    # virtualenv's dist-packages.
+    session.install("mock", "pytest")
+    for local_dep in LOCAL_DEPS:
+        session.install("-e", local_dep)
+    session.install("-e", "../test_utils/")
+    session.install("-e", ".")
 
-    session.run('py.test', '--quiet', os.path.join('tests', 'system'),
-                *session.posargs)
+    # Run py.test against the system tests.
+    if system_test_exists:
+        session.run("py.test", "--quiet", system_test_path, *session.posargs)
+    if system_test_folder_exists:
+        session.run("py.test", "--quiet", system_test_folder_path, *session.posargs)
 
 
-@nox.session(python='3.6')
-def lint_setup_py(session):
-    """Verify that setup.py is valid (including RST check)."""
-    session.install('docutils', 'pygments')
-    session.run('python', 'setup.py', 'check', '--restructuredtext',
-                '--strict')
+@nox.session(python="3.7")
+def cover(session):
+    """Run the final coverage report.
+
+    This outputs the coverage report aggregating coverage from the unit
+    test runs (not system test runs), and then erases coverage data.
+    """
+    session.install("coverage", "pytest-cov")
+    session.run("coverage", "report", "--show-missing", "--fail-under=100")
+
+    session.run("coverage", "erase")

packages/google-cloud-dataproc/synth.py

@@ -20,26 +20,27 @@
 from synthtool import gcp
 
 gapic = gcp.GAPICGenerator()
+common = gcp.CommonTemplates()
+versions = ["v1beta2", "v1"]
 
-versions = ['v1beta2', 'v1']
-
-
+# ----------------------------------------------------------------------------
+# Generate dataproc GAPIC layer
+# ----------------------------------------------------------------------------
 for version in versions:
-    library = gapic.py_library('dataproc', version)
-    s.move(
-        library,
-        excludes=[
-            'docs/index.rst', 'nox.py', 'README.rst', 'setup.py'])
+    library = gapic.py_library("dataproc", version)
+    s.move(library, excludes=["docs/index.rst", "nox.py", "README.rst", "setup.py"])
 
     s.replace(
-        f'google/cloud/dataproc_{version}/gapic/cluster_controller_client.py',
-        'metadata_type=operations_pb2.ClusterOperationMetadata,',
-        'metadata_type=proto_operations_pb2.ClusterOperationMetadata,')
+        f"google/cloud/dataproc_{version}/gapic/cluster_controller_client.py",
+        "metadata_type=operations_pb2.ClusterOperationMetadata,",
+        "metadata_type=proto_operations_pb2.ClusterOperationMetadata,",
+    )
 
     s.replace(
-        f'google/cloud/dataproc_{version}/gapic/cluster_controller_client.py',
-        '\s+Note:.*\n(.*\n)+?.*types.FieldMask.',
-        f'''
+        f"google/cloud/dataproc_{version}/gapic/cluster_controller_client.py",
+        "\s+<strong>Note:</strong>.*\n(.*\n)+?.*types.FieldMask.",
+        f"""
+
 
                 .. note::
 
@@ -52,7 +53,7 @@
                       secondary worker group
 
                     If a dict is provided, it must be of the same form as the protobuf
-                    message :class:`~google.cloud.dataproc_{version}.types.FieldMask`'''
+                    message :class:`~google.cloud.dataproc_{version}.types.FieldMask`""",
     )
 
     s.replace(
@@ -71,7 +72,14 @@
     )
 
 s.replace(
-    'google/cloud/dataproc_v1beta2/proto/clusters_pb2.py',
-    '# Generated by the protocol buffer compiler.  DO NOT EDIT!',
-    '# -*- coding: utf-8 -*-\n'
-    '# Generated by the protocol buffer compiler.  DO NOT EDIT!')
+    "google/cloud/dataproc_v1beta2/proto/clusters_pb2.py",
+    "# Generated by the protocol buffer compiler.  DO NOT EDIT!",
+    "# -*- coding: utf-8 -*-\n"
+    "# Generated by the protocol buffer compiler.  DO NOT EDIT!",
+)
+
+# ----------------------------------------------------------------------------
+# Add templated files
+# ----------------------------------------------------------------------------
+templated_files = common.py_library(unit_cov_level=97, cov_level=100)
+s.move(templated_files)