chore: add GH Workflow to run systests under emulator (#430)

* chore: add GH Workflow to run systests under emulator (DO NOT MERGE)

* Even if the tests are interrupted.  Also, they seem to need SIGKILL,
rather than SIGTERM.

* ci: run systest_emulated workflow on PRs targeting 'main'

Co-authored-by: Owl Bot <gcf-owl-bot[bot]@users.noreply.github.com>
Co-authored-by: Christopher Wilcox <crwilcox@google.com>

.github/workflows/system_emulated.yml

@@ -0,0 +1,29 @@
+name: "Run systests on emulator"
+on:
+  pull_request:
+    branches:
+      - main
+
+jobs:
+
+  run-systests:
+    runs-on: ubuntu-20.04
+
+    steps:
+
+    - name: Checkout
+      uses: actions/checkout@v2
+
+    - name: Setup Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: '3.8'
+
+    - name: Setup GCloud SDK
+      uses: google-github-actions/setup-gcloud@v0.2.1
+
+    - name: Install / run Nox
+      run: |
+        python -m pip install --upgrade setuptools pip
+        python -m pip install nox
+        nox -s system_emulated

google/cloud/bigtable/client.py

@@ -33,6 +33,7 @@
 
 from google.api_core.gapic_v1 import client_info
 import google.auth
+from google.auth.credentials import AnonymousCredentials
 
 from google.cloud import bigtable_v2
 from google.cloud import bigtable_admin_v2
@@ -67,6 +68,7 @@
 READ_ONLY_SCOPE = "https://www.googleapis.com/auth/bigtable.data.readonly"
 """Scope for reading table data."""
 
+_DEFAULT_BIGTABLE_EMULATOR_CLIENT = "google-cloud-bigtable-emulator"
 _GRPC_CHANNEL_OPTIONS = (
     ("grpc.max_send_message_length", -1),
     ("grpc.max_receive_message_length", -1),
@@ -170,6 +172,12 @@ def __init__(
         self._client_info = client_info
         self._emulator_host = os.getenv(BIGTABLE_EMULATOR)
 
+        if self._emulator_host is not None:
+            if credentials is None:
+                credentials = AnonymousCredentials()
+            if project is None:
+                project = _DEFAULT_BIGTABLE_EMULATOR_CLIENT
+
         if channel is not None:
             warnings.warn(
                 "'channel' is deprecated and no longer used.",

noxfile.py

@@ -119,7 +119,7 @@ def unit(session):
     default(session)
 
 
-@nox.session(python="3.8")
+@nox.session(python=SYSTEM_TEST_PYTHON_VERSIONS)
 def system_emulated(session):
     import subprocess
     import signal
@@ -133,15 +133,17 @@ def system_emulated(session):
     subprocess.call(["gcloud", "components", "install", "beta", "bigtable"])
 
     hostport = "localhost:8789"
+    session.env["BIGTABLE_EMULATOR_HOST"] = hostport
+
     p = subprocess.Popen(
         ["gcloud", "beta", "emulators", "bigtable", "start", "--host-port", hostport]
     )
 
-    session.env["BIGTABLE_EMULATOR_HOST"] = hostport
-    system(session)
-
-    # Stop Emulator
-    os.killpg(os.getpgid(p.pid), signal.SIGTERM)
+    try:
+        system(session)
+    finally:
+        # Stop Emulator
+        os.killpg(os.getpgid(p.pid), signal.SIGKILL)
 
 
 @nox.session(python=SYSTEM_TEST_PYTHON_VERSIONS)

owlbot.py

@@ -105,7 +105,7 @@ def place_before(path, text, *before_text, escape=None):
     s.replace([path], text, replacement)
 
 system_emulated_session = """
-@nox.session(python="3.8")
+@nox.session(python=SYSTEM_TEST_PYTHON_VERSIONS)
 def system_emulated(session):
     import subprocess
     import signal
@@ -119,15 +119,17 @@ def system_emulated(session):
     subprocess.call(["gcloud", "components", "install", "beta", "bigtable"])
 
     hostport = "localhost:8789"
+    session.env["BIGTABLE_EMULATOR_HOST"] = hostport
+
     p = subprocess.Popen(
         ["gcloud", "beta", "emulators", "bigtable", "start", "--host-port", hostport]
     )
 
-    session.env["BIGTABLE_EMULATOR_HOST"] = hostport
-    system(session)
-
-    # Stop Emulator
-    os.killpg(os.getpgid(p.pid), signal.SIGTERM)
+    try:
+        system(session)
+    finally:
+        # Stop Emulator
+        os.killpg(os.getpgid(p.pid), signal.SIGKILL)
 
 """
 

tests/system/conftest.py

@@ -147,12 +147,12 @@ def data_instance_populated(
     in_emulator,
 ):
     instance = admin_client.instance(data_instance_id, labels=instance_labels)
-    cluster = instance.cluster(
-        data_cluster_id, location_id=location_id, serve_nodes=serve_nodes,
-    )
     # Emulator does not support instance admin operations (create / delete).
     # See: https://cloud.google.com/bigtable/docs/emulator
     if not in_emulator:
+        cluster = instance.cluster(
+            data_cluster_id, location_id=location_id, serve_nodes=serve_nodes,
+        )
         operation = instance.create(clusters=[cluster])
         operation.result(timeout=30)
 

tests/unit/test_client.py

@@ -170,22 +170,67 @@ def test_constructor_both_admin_and_read_only(self):
 
     def test_constructor_with_emulator_host(self):
         from google.cloud.environment_vars import BIGTABLE_EMULATOR
+        from google.cloud.bigtable.client import _DEFAULT_BIGTABLE_EMULATOR_CLIENT
         from google.cloud.bigtable.client import _GRPC_CHANNEL_OPTIONS
 
-        credentials = _make_credentials()
         emulator_host = "localhost:8081"
         with mock.patch("os.environ", {BIGTABLE_EMULATOR: emulator_host}):
             with mock.patch("grpc.secure_channel") as factory:
-                client = self._make_one(project=self.PROJECT, credentials=credentials)
+                client = self._make_one()
                 # don't test local_composite_credentials
-                client._local_composite_credentials = lambda: credentials
+                # client._local_composite_credentials = lambda: credentials
+                # channels are formed when needed, so access a client
+                # create a gapic channel
+                client.table_data_client
+
+        self.assertEqual(client._emulator_host, emulator_host)
+        self.assertEqual(client.project, _DEFAULT_BIGTABLE_EMULATOR_CLIENT)
+        factory.assert_called_once_with(
+            emulator_host,
+            mock.ANY,  # test of creds wrapping in '_emulator_host' below
+            options=_GRPC_CHANNEL_OPTIONS,
+        )
+
+    def test_constructor_with_emulator_host_w_project(self):
+        from google.cloud.environment_vars import BIGTABLE_EMULATOR
+        from google.cloud.bigtable.client import _GRPC_CHANNEL_OPTIONS
+
+        emulator_host = "localhost:8081"
+        with mock.patch("os.environ", {BIGTABLE_EMULATOR: emulator_host}):
+            with mock.patch("grpc.secure_channel") as factory:
+                client = self._make_one(project=self.PROJECT)
+                # channels are formed when needed, so access a client
+                # create a gapic channel
+                client.table_data_client
+
+        self.assertEqual(client._emulator_host, emulator_host)
+        self.assertEqual(client.project, self.PROJECT)
+        factory.assert_called_once_with(
+            emulator_host,
+            mock.ANY,  # test of creds wrapping in '_emulator_host' below
+            options=_GRPC_CHANNEL_OPTIONS,
+        )
+
+    def test_constructor_with_emulator_host_w_credentials(self):
+        from google.cloud.environment_vars import BIGTABLE_EMULATOR
+        from google.cloud.bigtable.client import _DEFAULT_BIGTABLE_EMULATOR_CLIENT
+        from google.cloud.bigtable.client import _GRPC_CHANNEL_OPTIONS
+
+        emulator_host = "localhost:8081"
+        credentials = _make_credentials()
+        with mock.patch("os.environ", {BIGTABLE_EMULATOR: emulator_host}):
+            with mock.patch("grpc.secure_channel") as factory:
+                client = self._make_one(credentials=credentials)
                 # channels are formed when needed, so access a client
                 # create a gapic channel
                 client.table_data_client
 
         self.assertEqual(client._emulator_host, emulator_host)
+        self.assertEqual(client.project, _DEFAULT_BIGTABLE_EMULATOR_CLIENT)
         factory.assert_called_once_with(
-            emulator_host, credentials, options=_GRPC_CHANNEL_OPTIONS,
+            emulator_host,
+            mock.ANY,  # test of creds wrapping in '_emulator_host' below
+            options=_GRPC_CHANNEL_OPTIONS,
         )
 
     def test__get_scopes_default(self):