Cleanup bigtable python examples (#2692)

* Cleanup bigtable python:
Use new row types for mutations
Update bigtable version in requirements
Delete table after tests

* Change bigtable cluster variable to bigtable instance for consistency
Create and delete quickstart table during test

* Fixing step size for metric scaler
Create unique tables for quickstart tests

* Creating fixtures for quickstart tests
Fixing hb quickstart test output

* Fix quickstart extra delete table
Update happybase to use direct row

* Use clearer instance names for tests
Create unique instances for metric scaler tests

* Linting

* remove core dep

Co-authored-by: Leah E. Cole <6719667+leahecole@users.noreply.github.com>

bigtable/hello/main.py

@@ -75,7 +75,7 @@ def main(project_id, instance_id, table_id):
         #
         #     https://cloud.google.com/bigtable/docs/schema-design
         row_key = 'greeting{}'.format(i).encode()
-        row = table.row(row_key)
+        row = table.direct_row(row_key)
         row.set_cell(column_family_id,
                      column,
                      value,

bigtable/hello/main_test.py

@@ -18,16 +18,16 @@
 from main import main
 
 PROJECT = os.environ['GCLOUD_PROJECT']
-BIGTABLE_CLUSTER = os.environ['BIGTABLE_CLUSTER']
-TABLE_NAME_FORMAT = 'hello-bigtable-system-tests-{}'
+BIGTABLE_INSTANCE = os.environ['BIGTABLE_INSTANCE']
+TABLE_NAME_FORMAT = 'hello-world-test-{}'
 TABLE_NAME_RANGE = 10000
 
 
 def test_main(capsys):
     table_name = TABLE_NAME_FORMAT.format(
         random.randrange(TABLE_NAME_RANGE))
 
-    main(PROJECT, BIGTABLE_CLUSTER, table_name)
+    main(PROJECT, BIGTABLE_INSTANCE, table_name)
 
     out, _ = capsys.readouterr()
     assert 'Creating the {} table.'.format(table_name) in out

bigtable/hello/requirements.txt

@@ -1,2 +1,2 @@
-google-cloud-bigtable==1.2.0
+google-cloud-bigtable==1.2.1
 google-cloud-core==1.1.0

bigtable/hello_happybase/main_test.py

@@ -18,8 +18,8 @@
 from main import main
 
 PROJECT = os.environ['GCLOUD_PROJECT']
-BIGTABLE_CLUSTER = os.environ['BIGTABLE_CLUSTER']
-TABLE_NAME_FORMAT = 'hello_happybase-system-tests-{}'
+BIGTABLE_INSTANCE = os.environ['BIGTABLE_INSTANCE']
+TABLE_NAME_FORMAT = 'hello-world-hb-test-{}'
 TABLE_NAME_RANGE = 10000
 
 
@@ -28,7 +28,7 @@ def test_main(capsys):
         random.randrange(TABLE_NAME_RANGE))
     main(
         PROJECT,
-        BIGTABLE_CLUSTER,
+        BIGTABLE_INSTANCE,
         table_name)
 
     out, _ = capsys.readouterr()

bigtable/instanceadmin/requirements.txt

@@ -1 +1 @@
-google-cloud-bigtable==1.2.0
+google-cloud-bigtable==1.2.1

bigtable/metricscaler/metricscaler_test.py

@@ -17,16 +17,25 @@
 import os
 import time
 
+import random
+
+import pytest
 from google.cloud import bigtable
+from google.cloud.bigtable import enums
 from mock import patch
 
 from metricscaler import get_cpu_load
 from metricscaler import main
 from metricscaler import scale_bigtable
 
-# tests assume instance and cluster have the same ID
-BIGTABLE_INSTANCE = os.environ['BIGTABLE_CLUSTER']
+PROJECT = os.environ['GCLOUD_PROJECT']
+BIGTABLE_ZONE = os.environ['BIGTABLE_ZONE']
 SIZE_CHANGE_STEP = 3
+INSTANCE_ID_FORMAT = 'metric-scale-test-{}'
+INSTANCE_ID_RANGE = 10000
+BIGTABLE_INSTANCE = INSTANCE_ID_FORMAT.format(
+    random.randrange(INSTANCE_ID_RANGE))
+
 
 # System tests to verify API calls succeed
 
@@ -35,8 +44,33 @@ def test_get_cpu_load():
     assert float(get_cpu_load()) > 0.0
 
 
-def test_scale_bigtable():
+@pytest.fixture()
+def instance():
+    cluster_id = BIGTABLE_INSTANCE
+
+    client = bigtable.Client(project=PROJECT, admin=True)
+
+    serve_nodes = 3
+    storage_type = enums.StorageType.SSD
+    production = enums.Instance.Type.PRODUCTION
+    labels = {'prod-label': 'prod-label'}
+    instance = client.instance(BIGTABLE_INSTANCE, instance_type=production,
+                               labels=labels)
+
+    if not instance.exists():
+        cluster = instance.cluster(cluster_id, location_id=BIGTABLE_ZONE,
+                                   serve_nodes=serve_nodes,
+                                   default_storage_type=storage_type)
+        instance.create(clusters=[cluster])
+
+    yield
+
+    instance.delete()
+
+
+def test_scale_bigtable(instance):
     bigtable_client = bigtable.Client(admin=True)
+
     instance = bigtable_client.instance(BIGTABLE_INSTANCE)
     instance.reload()
 

bigtable/metricscaler/requirements.txt

@@ -1,2 +1,2 @@
-google-cloud-bigtable==1.2.0
+google-cloud-bigtable==1.2.1
 google-cloud-monitoring==0.34.0

bigtable/quickstart/main_test.py

@@ -13,16 +13,41 @@
 # limitations under the License.
 
 import os
+import random
+import pytest
 
 from main import main
+from google.cloud import bigtable
 
 PROJECT = os.environ['GCLOUD_PROJECT']
-BIGTABLE_CLUSTER = os.environ['BIGTABLE_CLUSTER']
-TABLE_NAME = 'my-table'
+BIGTABLE_INSTANCE = os.environ['BIGTABLE_INSTANCE']
+TABLE_ID_FORMAT = 'quickstart-test-{}'
+TABLE_ID_RANGE = 10000
 
 
-def test_main(capsys):
-    main(PROJECT, BIGTABLE_CLUSTER, TABLE_NAME)
+@pytest.fixture()
+def table():
+    table_id = TABLE_ID_FORMAT.format(
+        random.randrange(TABLE_ID_RANGE))
+    client = bigtable.Client(project=PROJECT, admin=True)
+    instance = client.instance(BIGTABLE_INSTANCE)
+    table = instance.table(table_id)
+    column_family_id = 'cf1'
+    column_families = {column_family_id: None}
+    table.create(column_families=column_families)
+
+    row = table.direct_row("r1")
+    row.set_cell(column_family_id, "c1", "test-value")
+    row.commit()
+
+    yield table_id
+
+    table.delete()
+
+
+def test_main(capsys, table):
+    table_id = table
+    main(PROJECT, BIGTABLE_INSTANCE, table_id)
 
     out, _ = capsys.readouterr()
     assert 'Row key: r1\nData: test-value\n' in out

bigtable/quickstart/requirements.txt

@@ -1,2 +1 @@
-google-cloud-bigtable==1.0.0
-google-cloud-core==1.0.3
+google-cloud-bigtable==1.2.1

bigtable/quickstart_happybase/main.py

@@ -15,7 +15,6 @@
 # limitations under the License.
 # [START bigtable_quickstart_happybase]
 import argparse
-import json
 
 from google.cloud import bigtable
 from google.cloud import happybase
@@ -37,9 +36,10 @@ def main(project_id="project-id", instance_id="instance-id",
 
         key = 'r1'
         row = table.row(key.encode('utf-8'))
-        value = {k.decode("utf-8"): v.decode("utf-8") for k, v in row.items()}
-        print('Row key: {}\nData: {}'.format(key, json.dumps(value, indent=4,
-                                                             sort_keys=True)))
+
+        column = 'cf1:c1'.encode('utf-8')
+        value = row[column].decode('utf-8')
+        print('Row key: {}\nData: {}'.format(key, value))
 
     finally:
         connection.close()

bigtable/quickstart_happybase/main_test.py

@@ -14,15 +14,41 @@
 
 import os
 
+import random
+
+import pytest
+from google.cloud import bigtable
 from main import main
 
 PROJECT = os.environ['GCLOUD_PROJECT']
-BIGTABLE_CLUSTER = os.environ['BIGTABLE_CLUSTER']
-TABLE_NAME = 'my-table'
+BIGTABLE_INSTANCE = os.environ['BIGTABLE_INSTANCE']
+TABLE_ID_FORMAT = 'quickstart-hb-test-{}'
+TABLE_ID_RANGE = 10000
+
+
+@pytest.fixture()
+def table():
+    table_id = TABLE_ID_FORMAT.format(
+        random.randrange(TABLE_ID_RANGE))
+    client = bigtable.Client(project=PROJECT, admin=True)
+    instance = client.instance(BIGTABLE_INSTANCE)
+    table = instance.table(table_id)
+    column_family_id = 'cf1'
+    column_families = {column_family_id: None}
+    table.create(column_families=column_families)
+
+    row = table.direct_row("r1")
+    row.set_cell(column_family_id, "c1", "test-value")
+    row.commit()
+
+    yield table_id
+
+    table.delete()
 
 
-def test_main(capsys):
-    main(PROJECT, BIGTABLE_CLUSTER, TABLE_NAME)
+def test_main(capsys, table):
+    table_id = table
+    main(PROJECT, BIGTABLE_INSTANCE, table_id)
 
     out, _ = capsys.readouterr()
-    assert '"cf1:c1": "test-value"' in out
+    assert 'Row key: r1\nData: test-value\n' in out

bigtable/snippets/writes/requirements.txt

@@ -1 +1 @@
-google-cloud-bigtable==1.2.0
+google-cloud-bigtable==1.2.1

bigtable/snippets/writes/write_batch.py

@@ -26,8 +26,8 @@ def write_batch(project_id, instance_id, table_id):
     timestamp = datetime.datetime.utcnow()
     column_family_id = "stats_summary"
 
-    rows = [table.row("tablet#a0b81f74#20190501"),
-            table.row("tablet#a0b81f74#20190502")]
+    rows = [table.direct_row("tablet#a0b81f74#20190501"),
+            table.direct_row("tablet#a0b81f74#20190502")]
 
     rows[0].set_cell(column_family_id,
                      "connected_wifi",

bigtable/snippets/writes/write_conditionally.py

@@ -33,7 +33,7 @@ def write_conditional(project_id, instance_id, table_id):
         filters=[row_filters.FamilyNameRegexFilter(column_family_id),
                  row_filters.ColumnQualifierRegexFilter('os_build'),
                  row_filters.ValueRegexFilter("PQ2A\\..*")])
-    row = table.row(row_key, filter_=row_filter)
+    row = table.conditional_row(row_key, filter_=row_filter)
     row.set_cell(column_family_id,
                  "os_name",
                  "android",

bigtable/snippets/writes/write_increment.py

@@ -24,7 +24,7 @@ def write_increment(project_id, instance_id, table_id):
     column_family_id = "stats_summary"
 
     row_key = "phone#4c410523#20190501"
-    row = table.row(row_key, append=True)
+    row = table.append_row(row_key)
 
     # Decrement the connected_wifi value by 1.
     row.increment_cell_value(column_family_id, "connected_wifi", -1)

bigtable/snippets/writes/write_simple.py

@@ -27,7 +27,7 @@ def write_simple(project_id, instance_id, table_id):
 
     row_key = "phone#4c410523#20190501"
 
-    row = table.row(row_key)
+    row = table.direct_row(row_key)
     row.set_cell(column_family_id,
                  "connected_cell",
                  1,

bigtable/snippets/writes/writes_test.py

@@ -24,7 +24,7 @@
 from .write_simple import write_simple
 
 PROJECT = os.environ['GCLOUD_PROJECT']
-BIGTABLE_INSTANCE = os.environ['BIGTABLE_CLUSTER']
+BIGTABLE_INSTANCE = os.environ['BIGTABLE_INSTANCE']
 TABLE_ID_PREFIX = 'mobile-time-series-{}'
 
 

bigtable/tableadmin/requirements.txt

@@ -1 +1 @@
-google-cloud-bigtable==1.2.0
+google-cloud-bigtable==1.2.1

bigtable/tableadmin/tableadmin_test.py

@@ -21,16 +21,16 @@
 from tableadmin import run_table_operations
 
 PROJECT = os.environ['GCLOUD_PROJECT']
-BIGTABLE_CLUSTER = os.environ['BIGTABLE_CLUSTER']
-TABLE_NAME_FORMAT = 'hello-bigtable-system-tests-{}'
+BIGTABLE_INSTANCE = os.environ['BIGTABLE_INSTANCE']
+TABLE_NAME_FORMAT = 'tableadmin-test-{}'
 TABLE_NAME_RANGE = 10000
 
 
 def test_run_table_operations(capsys):
     table_name = TABLE_NAME_FORMAT.format(
         random.randrange(TABLE_NAME_RANGE))
 
-    run_table_operations(PROJECT, BIGTABLE_CLUSTER, table_name)
+    run_table_operations(PROJECT, BIGTABLE_INSTANCE, table_name)
     out, _ = capsys.readouterr()
 
     assert 'Creating the ' + table_name + ' table.' in out
@@ -50,13 +50,15 @@ def test_run_table_operations(capsys):
     assert 'Delete a column family cf2...' in out
     assert 'Column family cf2 deleted successfully.' in out
 
+    delete_table(PROJECT, BIGTABLE_INSTANCE, table_name)
+
 
 def test_delete_table(capsys):
     table_name = TABLE_NAME_FORMAT.format(
         random.randrange(TABLE_NAME_RANGE))
-    create_table(PROJECT, BIGTABLE_CLUSTER, table_name)
+    create_table(PROJECT, BIGTABLE_INSTANCE, table_name)
 
-    delete_table(PROJECT, BIGTABLE_CLUSTER, table_name)
+    delete_table(PROJECT, BIGTABLE_INSTANCE, table_name)
     out, _ = capsys.readouterr()
 
     assert 'Table ' + table_name + ' exists.' in out