refactor: move system tests into tests/system directory

noxfile.py

@@ -115,9 +115,7 @@ def system(session):
     session.install("ipython", "-c", constraints_path)
 
     # Run py.test against the system tests.
-    session.run(
-        "py.test", "--quiet", os.path.join("tests", "system.py"), *session.posargs
-    )
+    session.run("py.test", "--quiet", os.path.join("tests", "system"), *session.posargs)
 
 
 @nox.session(python=["3.8"])
@@ -181,12 +179,14 @@ def prerelease_deps(session):
     )
     session.install("--pre", "grpcio", "pandas")
     session.install(
+        "freezegun",
+        "google-cloud-storage",
+        "google-cloud-testutils",
+        "IPython",
         "mock",
+        "psutil",
         "pytest",
-        "google-cloud-testutils",
         "pytest-cov",
-        "freezegun",
-        "IPython",
     )
     session.install("-e", ".[all]")
 
@@ -196,7 +196,8 @@ def prerelease_deps(session):
     session.run("python", "-c", "import pyarrow; print(pyarrow.__version__)")
 
     # Run all tests, except a few samples tests which require extra dependencies.
-    session.run("py.test", "tests")
+    session.run("py.test", "tests/unit")
+    session.run("py.test", "tests/system")
     session.run("py.test", "samples/tests")
 
 

tests/system/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 2021 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

tests/system.py → tests/system/test_client.py

@@ -22,13 +22,12 @@
 import json
 import operator
 import os
+import pathlib
 import time
 import unittest
 import uuid
-import re
 
 import psutil
-import pytest
 import pytz
 import pkg_resources
 
@@ -51,13 +50,6 @@
     import pyarrow.types
 except ImportError:  # pragma: NO COVER
     pyarrow = None
-try:
-    import IPython
-    from IPython.utils import io as ipython_io
-    from IPython.testing import tools
-    from IPython.terminal import interactiveshell
-except ImportError:  # pragma: NO COVER
-    IPython = None
 
 from google.api_core.exceptions import PreconditionFailed
 from google.api_core.exceptions import BadRequest
@@ -86,7 +78,7 @@
 
 
 JOB_TIMEOUT = 120  # 2 minutes
-WHERE = os.path.abspath(os.path.dirname(__file__))
+DATA_PATH = pathlib.Path(__file__).parent.parent / "data"
 
 # Common table data used for many tests.
 ROWS = [
@@ -149,10 +141,10 @@ def _make_dataset_id(prefix):
     return "%s%s" % (prefix, unique_resource_id())
 
 
-def _load_json_schema(filename="data/schema.json"):
+def _load_json_schema(filename="schema.json"):
     from google.cloud.bigquery.table import _parse_schema_resource
 
-    json_filename = os.path.join(WHERE, filename)
+    json_filename = DATA_PATH / filename
 
     with open(json_filename, "r") as schema_file:
         return _parse_schema_resource(json.load(schema_file))
@@ -716,7 +708,7 @@ def test_load_table_from_local_avro_file_then_dump_table(self):
         table = Table(table_ref)
         self.to_delete.insert(0, table)
 
-        with open(os.path.join(WHERE, "data", "colors.avro"), "rb") as avrof:
+        with open(DATA_PATH / "colors.avro", "rb") as avrof:
             config = bigquery.LoadJobConfig()
             config.source_format = SourceFormat.AVRO
             config.write_disposition = WriteDisposition.WRITE_TRUNCATE
@@ -1347,7 +1339,7 @@ def test_load_avro_from_uri_then_dump_table(self):
             ("orange", 590),
             ("red", 650),
         ]
-        with open(os.path.join(WHERE, "data", "colors.avro"), "rb") as f:
+        with open(DATA_PATH / "colors.avro", "rb") as f:
             GS_URL = self._write_avro_to_storage(
                 "bq_load_test" + unique_resource_id(), "colors.avro", f
             )
@@ -2707,7 +2699,7 @@ def test_create_table_rows_fetch_nested_schema(self):
 
         to_insert = []
         # Data is in "JSON Lines" format, see http://jsonlines.org/
-        json_filename = os.path.join(WHERE, "data", "characters.jsonl")
+        json_filename = DATA_PATH / "characters.jsonl"
         with open(json_filename) as rows_file:
             for line in rows_file:
                 to_insert.append(json.loads(line))
@@ -2979,47 +2971,6 @@ def temp_dataset(self, dataset_id, location=None):
         return dataset
 
 
-@pytest.mark.skipif(pandas is None, reason="Requires `pandas`")
-@pytest.mark.skipif(IPython is None, reason="Requires `ipython`")
-@pytest.mark.usefixtures("ipython_interactive")
-def test_bigquery_magic():
-    ip = IPython.get_ipython()
-    current_process = psutil.Process()
-    conn_count_start = len(current_process.connections())
-
-    ip.extension_manager.load_extension("google.cloud.bigquery")
-    sql = """
-        SELECT
-            CONCAT(
-            'https://stackoverflow.com/questions/',
-            CAST(id as STRING)) as url,
-            view_count
-        FROM `bigquery-public-data.stackoverflow.posts_questions`
-        WHERE tags like '%google-bigquery%'
-        ORDER BY view_count DESC
-        LIMIT 10
-    """
-    with ipython_io.capture_output() as captured:
-        result = ip.run_cell_magic("bigquery", "--use_rest_api", sql)
-
-    conn_count_end = len(current_process.connections())
-
-    lines = re.split("\n|\r", captured.stdout)
-    # Removes blanks & terminal code (result of display clearing)
-    updates = list(filter(lambda x: bool(x) and x != "\x1b[2K", lines))
-    assert re.match("Executing query with job ID: .*", updates[0])
-    assert all(re.match("Query executing: .*s", line) for line in updates[1:-1])
-    assert re.match("Query complete after .*s", updates[-1])
-    assert isinstance(result, pandas.DataFrame)
-    assert len(result) == 10  # verify row count
-    assert list(result) == ["url", "view_count"]  # verify column names
-
-    # NOTE: For some reason, the number of open sockets is sometimes one *less*
-    # than expected when running system tests on Kokoro, thus using the <= assertion.
-    # That's still fine, however, since the sockets are apparently not leaked.
-    assert conn_count_end <= conn_count_start  # system resources are released
-
-
 def _job_done(instance):
     return instance.state.lower() == "done"
 
@@ -3039,21 +2990,3 @@ def _table_exists(t):
         return True
     except NotFound:
         return False
-
-
-@pytest.fixture(scope="session")
-def ipython():
-    config = tools.default_config()
-    config.TerminalInteractiveShell.simple_prompt = True
-    shell = interactiveshell.TerminalInteractiveShell.instance(config=config)
-    return shell
-
-
-@pytest.fixture()
-def ipython_interactive(request, ipython):
-    """Activate IPython's builtin hooks
-
-    for the duration of the test scope.
-    """
-    with ipython.builtin_trap:
-        yield ipython

tests/system/test_magics.py

@@ -0,0 +1,83 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""System tests for Jupyter/IPython connector."""
+
+import re
+
+import pytest
+import psutil
+
+
+IPython = pytest.importorskip("IPython")
+io = pytest.importorskip("IPython.utils.io")
+pandas = pytest.importorskip("pandas")
+tools = pytest.importorskip("IPython.testing.tools")
+interactiveshell = pytest.importorskip("IPython.terminal.interactiveshell")
+
+
+@pytest.fixture(scope="session")
+def ipython():
+    config = tools.default_config()
+    config.TerminalInteractiveShell.simple_prompt = True
+    shell = interactiveshell.TerminalInteractiveShell.instance(config=config)
+    return shell
+
+
+@pytest.fixture()
+def ipython_interactive(ipython):
+    """Activate IPython's builtin hooks
+
+    for the duration of the test scope.
+    """
+    with ipython.builtin_trap:
+        yield ipython
+
+
+def test_bigquery_magic(ipython_interactive):
+    ip = IPython.get_ipython()
+    current_process = psutil.Process()
+    conn_count_start = len(current_process.connections())
+
+    ip.extension_manager.load_extension("google.cloud.bigquery")
+    sql = """
+        SELECT
+            CONCAT(
+            'https://stackoverflow.com/questions/',
+            CAST(id as STRING)) as url,
+            view_count
+        FROM `bigquery-public-data.stackoverflow.posts_questions`
+        WHERE tags like '%google-bigquery%'
+        ORDER BY view_count DESC
+        LIMIT 10
+    """
+    with io.capture_output() as captured:
+        result = ip.run_cell_magic("bigquery", "--use_rest_api", sql)
+
+    conn_count_end = len(current_process.connections())
+
+    lines = re.split("\n|\r", captured.stdout)
+    # Removes blanks & terminal code (result of display clearing)
+    updates = list(filter(lambda x: bool(x) and x != "\x1b[2K", lines))
+    assert re.match("Executing query with job ID: .*", updates[0])
+    assert all(re.match("Query executing: .*s", line) for line in updates[1:-1])
+    assert re.match("Query complete after .*s", updates[-1])
+    assert isinstance(result, pandas.DataFrame)
+    assert len(result) == 10  # verify row count
+    assert list(result) == ["url", "view_count"]  # verify column names
+
+    # NOTE: For some reason, the number of open sockets is sometimes one *less*
+    # than expected when running system tests on Kokoro, thus using the <= assertion.
+    # That's still fine, however, since the sockets are apparently not leaked.
+    assert conn_count_end <= conn_count_start  # system resources are released