Merge pull request #1832 from tseaver/bigquery-testable_dataset_snippets

Make dataset snippets testable

docs/bigquery-usage.rst

@@ -16,33 +16,15 @@ Authentication / Configuration
   and
   :meth:`from_service_account_p12 <gcloud.bigquery.client.Client.from_service_account_p12>`.
 
-- After setting ``GOOGLE_APPLICATION_CREDENTIALS`` and ``GCLOUD_PROJECT``
-  environment variables, create an instance of
+- After setting :envvar:`GOOGLE_APPLICATION_CREDENTIALS` and
+  :envvar:`GCLOUD_PROJECT` environment variables, create an instance of
   :class:`Client <gcloud.bigquery.client.Client>`.
 
   .. doctest::
 
      >>> from gcloud import bigquery
      >>> client = bigquery.Client()
 
-- Override the credentials inferred from the environment by passing explicit
-  ``credentials`` to one of the alternative ``classmethod`` factories,
-  :meth:`gcloud.bigquery.client.Client.from_service_account_json`:
-
-  .. doctest::
-
-     >>> from gcloud import bigquery
-     >>> client = bigquery.Client.from_service_account_json('/path/to/creds.json')
-
-  or :meth:`gcloud.bigquery.client.Client.from_service_account_p12`:
-
-  .. doctest::
-
-     >>> from gcloud import bigquery
-     >>> client = bigquery.Client.from_service_account_p12(
-     ...     '/path/to/creds.p12', 'jrandom@example.com')
-
-
 Projects
 --------
 
@@ -83,54 +65,35 @@ policies to tables as they are created:
 Dataset operations
 ~~~~~~~~~~~~~~~~~~
 
-Create a new dataset for the client's project:
-
-.. doctest::
-
-   >>> from gcloud import bigquery
-   >>> client = bigquery.Client()
-   >>> dataset = client.dataset('dataset_name')
-   >>> dataset.create()  # API request
-
-Check for the existence of a dataset:
+List datasets for the client's project:
 
-.. doctest::
+.. literalinclude:: bigquery_snippets.py
+   :start-after: [START client_list_datasets]
+   :end-before: [END client_list_datasets]
 
-   >>> from gcloud import bigquery
-   >>> client = bigquery.Client()
-   >>> dataset = client.dataset('dataset_name')
-   >>> dataset.exists()  # API request
-   True
+Create a new dataset for the client's project:
 
-List datasets for the client's project:
+.. literalinclude:: bigquery_snippets.py
+   :start-after: [START dataset_create]
+   :end-before: [END dataset_create]
 
-.. doctest::
+Check for the existence of a dataset:
 
-   >>> from gcloud import bigquery
-   >>> client = bigquery.Client()
-   >>> datasets, next_page_token = client.list_datasets()  # API request
-   >>> [dataset.name for dataset in datasets]
-   ['dataset_name']
+.. literalinclude:: bigquery_snippets.py
+   :start-after: [START dataset_exists]
+   :end-before: [END dataset_exists]
 
 Refresh metadata for a dataset (to pick up changes made by another client):
 
-.. doctest::
-
-   >>> from gcloud import bigquery
-   >>> client = bigquery.Client()
-   >>> dataset = client.dataset('dataset_name')
-   >>> dataset.reload()  # API request
+.. literalinclude:: bigquery_snippets.py
+   :start-after: [START dataset_reload]
+   :end-before: [END dataset_reload]
 
 Patch metadata for a dataset:
 
-.. doctest::
-
-   >>> from gcloud import bigquery
-   >>> client = bigquery.Client()
-   >>> dataset = client.dataset('dataset_name')
-   >>> one_day_ms = 24 * 60 * 60 * 1000
-   >>> dataset.patch(description='Description goes here',
-   ...               default_table_expiration_ms=one_day_ms)  # API request
+.. literalinclude:: bigquery_snippets.py
+   :start-after: [START dataset_patch]
+   :end-before: [END dataset_patch]
 
 Replace the ACL for a dataset, and update all writeable fields:
 
@@ -147,12 +110,9 @@ Replace the ACL for a dataset, and update all writeable fields:
 
 Delete a dataset:
 
-.. doctest::
-
-   >>> from gcloud import bigquery
-   >>> client = bigquery.Client()
-   >>> dataset = client.dataset('dataset_name')
-   >>> dataset.delete()  # API request
+.. literalinclude:: bigquery_snippets.py
+   :start-after: [START dataset_delete]
+   :end-before: [END dataset_delete]
 
 
 Tables

docs/bigquery_snippets.py

@@ -0,0 +1,198 @@
+# Copyright 2016 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Testable usage examples for Google Cloud BigQuery API wrapper
+
+Each example function takes a ``client`` argument (which must be an instance
+of :class:`gcloud.bigquery.client.Client`) and uses it to perform a task with
+the API.
+
+To facilitate running the examples as system tests, each example is also passed
+a ``to_delete`` list;  the function adds to the list any objects created which
+need to be deleted during teardown.
+"""
+
+import time
+
+from gcloud.bigquery.client import Client
+
+
+def snippet(func):
+    """Mark ``func`` as a snippet example function."""
+    func._snippet = True
+    return func
+
+
+def _millis():
+    return time.time() * 1000
+
+
+@snippet
+def client_list_datasets(client, to_delete):  # pylint: disable=unused-argument
+    """List datasets for a project."""
+
+    def do_something_with(sub):  # pylint: disable=unused-argument
+        pass
+
+    # [START client_list_datasets]
+    datasets, token = client.list_datasets()   # API request
+    while True:
+        for dataset in datasets:
+            do_something_with(dataset)
+        if token is None:
+            break
+        datasets, token = client.list_datasets(page_token=token)  # API request
+    # [END client_list_datasets]
+
+
+@snippet
+def dataset_create(client, to_delete):
+    """Create a dataset."""
+    DATASET_NAME = 'dataset_create_%d' % (_millis(),)
+
+    # [START dataset_create]
+    dataset = client.dataset(DATASET_NAME)
+    dataset.create()              # API request
+    # [END dataset_create]
+
+    to_delete.append(dataset)
+
+
+@snippet
+def dataset_exists(client, to_delete):
+    """Test existence of a dataset."""
+    DATASET_NAME = 'dataset_exists_%d' % (_millis(),)
+    dataset = client.dataset(DATASET_NAME)
+    to_delete.append(dataset)
+
+    # [START dataset_exists]
+    assert not dataset.exists()   # API request
+    dataset.create()              # API request
+    assert dataset.exists()       # API request
+    # [END dataset_exists]
+
+
+@snippet
+def dataset_reload(client, to_delete):
+    """Reload a dataset's metadata."""
+    DATASET_NAME = 'dataset_reload_%d' % (_millis(),)
+    ORIGINAL_DESCRIPTION = 'Original description'
+    LOCALLY_CHANGED_DESCRIPTION = 'Locally-changed description'
+    dataset = client.dataset(DATASET_NAME)
+    dataset.description = ORIGINAL_DESCRIPTION
+    dataset.create()
+    to_delete.append(dataset)
+
+    # [START dataset_reload]
+    assert dataset.description == ORIGINAL_DESCRIPTION
+    dataset.description = LOCALLY_CHANGED_DESCRIPTION
+    assert dataset.description == LOCALLY_CHANGED_DESCRIPTION
+    dataset.reload()              # API request
+    assert dataset.description == ORIGINAL_DESCRIPTION
+    # [END dataset_reload]
+
+
+@snippet
+def dataset_patch(client, to_delete):
+    """Patch a dataset's metadata."""
+    DATASET_NAME = 'dataset_patch_%d' % (_millis(),)
+    ORIGINAL_DESCRIPTION = 'Original description'
+    PATCHED_DESCRIPTION = 'Patched description'
+    dataset = client.dataset(DATASET_NAME)
+    dataset.description = ORIGINAL_DESCRIPTION
+    dataset.create()
+    to_delete.append(dataset)
+
+    # [START dataset_patch]
+    ONE_DAY_MS = 24 * 60 * 60 * 1000
+    assert dataset.description == ORIGINAL_DESCRIPTION
+    dataset.patch(
+        description=PATCHED_DESCRIPTION,
+        default_table_expiration_ms=ONE_DAY_MS
+    )      # API request
+    assert dataset.description == PATCHED_DESCRIPTION
+    assert dataset.default_table_expiration_ms == ONE_DAY_MS
+    # [END dataset_patch]
+
+
+@snippet
+def dataset_update(client, to_delete):
+    """Update a dataset's metadata."""
+    DATASET_NAME = 'dataset_update_%d' % (_millis(),)
+    ORIGINAL_DESCRIPTION = 'Original description'
+    UPDATED_DESCRIPTION = 'Updated description'
+    dataset = client.dataset(DATASET_NAME)
+    dataset.description = ORIGINAL_DESCRIPTION
+    dataset.create()
+    to_delete.append(dataset)
+    dataset.reload()
+
+    # [START dataset_update]
+    from gcloud.bigquery import AccessGrant
+    assert dataset.description == ORIGINAL_DESCRIPTION
+    assert dataset.default_table_expiration_ms is None
+    grant = AccessGrant(
+        role='READER', entity_type='domain', entity_id='example.com')
+    assert grant not in dataset.access_grants
+    ONE_DAY_MS = 24 * 60 * 60 * 1000
+    dataset.description = UPDATED_DESCRIPTION
+    dataset.default_table_expiration_ms = ONE_DAY_MS
+    grants = list(dataset.access_grants)
+    grants.append(grant)
+    dataset.access_grants = grants
+    dataset.update()              # API request
+    assert dataset.description == UPDATED_DESCRIPTION
+    assert dataset.default_table_expiration_ms == ONE_DAY_MS
+    assert grant in dataset.access_grants
+    # [END dataset_update]
+
+
+@snippet
+def dataset_delete(client, to_delete):  # pylint: disable=unused-argument
+    """Delete a dataset."""
+    DATASET_NAME = 'dataset_delete_%d' % (_millis(),)
+    dataset = client.dataset(DATASET_NAME)
+    dataset.create()
+
+    # [START dataset_delete]
+    assert dataset.exists()       # API request
+    dataset.delete()
+    assert not dataset.exists()   # API request
+    # [END dataset_delete]
+
+
+def _find_examples():
+    funcs = [obj for obj in globals().values()
+             if getattr(obj, '_snippet', False)]
+    for func in sorted(funcs, key=lambda f: f.func_code.co_firstlineno):
+        yield func
+
+
+def main():
+    client = Client()
+    for example in _find_examples():
+        to_delete = []
+        print('%-25s: %s' % (
+            example.func_name, example.func_doc))
+        try:
+            example(client, to_delete)
+        except AssertionError as e:
+            print('   FAIL: %s' % (e,))
+        except Exception as e:  # pylint: disable=broad-except
+            print('  ERROR: %r' % (e,))
+        for item in to_delete:
+            item.delete()
+
+if __name__ == '__main__':
+    main()

docs/pubsub_snippets.py

@@ -18,7 +18,7 @@
 of :class:`gcloud.pubsub.client.Client`) and uses it to perform a task with
 the API.
 
-To facility running the examples as system tests, each example is also passed
+To facilitate running the examples as system tests, each example is also passed
 a ``to_delete`` list;  the function adds to the list any objects created which
 need to be deleted during teardown.
 """