From 71eeed9c4c8cca31f1071dd231446ea3f1d5a099 Mon Sep 17 00:00:00 2001 From: Danny Hermes Date: Fri, 22 Apr 2016 16:28:43 -0700 Subject: [PATCH] Adding Travis build ID to resource names in system tests. Fixes #1542. --- system_tests/bigquery.py | 22 ++++++++++--------- system_tests/bigtable.py | 7 +++--- system_tests/bigtable_happybase.py | 29 +++++-------------------- system_tests/clear_datastore.py | 6 +++--- system_tests/datastore.py | 7 +++--- system_tests/logging_.py | 17 ++++++++------- system_tests/populate_datastore.py | 8 +++---- system_tests/pubsub.py | 34 ++++++++++++++++-------------- system_tests/run_emulator.py | 4 ++-- system_tests/run_system_test.py | 2 -- system_tests/storage.py | 19 +++++++++-------- system_tests/system_test_utils.py | 14 ++++++++++++ 12 files changed, 84 insertions(+), 85 deletions(-) diff --git a/system_tests/bigquery.py b/system_tests/bigquery.py index dee49786cac19..e587b7585ef18 100644 --- a/system_tests/bigquery.py +++ b/system_tests/bigquery.py @@ -21,8 +21,10 @@ from gcloud.environment_vars import TESTS_PROJECT from gcloud import bigquery +from system_test_utils import unique_resource_id -DATASET_NAME = 'system_tests_%012d' % (1000 * time.time(),) + +DATASET_NAME = 'system_tests' + unique_resource_id() class Config(object): @@ -97,9 +99,9 @@ def test_update_dataset(self): def test_list_datasets(self): datasets_to_create = [ - 'new%d' % (1000 * time.time(),), - 'newer%d' % (1000 * time.time(),), - 'newest%d' % (1000 * time.time(),), + 'new' + unique_resource_id(), + 'newer' + unique_resource_id(), + 'newest' + unique_resource_id(), ] for dataset_name in datasets_to_create: dataset = Config.CLIENT.dataset(dataset_name) @@ -143,9 +145,9 @@ def test_list_tables(self): # Insert some tables to be listed. tables_to_create = [ - 'new%d' % (1000 * time.time(),), - 'newer%d' % (1000 * time.time(),), - 'newest%d' % (1000 * time.time(),), + 'new' + unique_resource_id(), + 'newer' + unique_resource_id(), + 'newest' + unique_resource_id(), ] full_name = bigquery.SchemaField('full_name', 'STRING', mode='REQUIRED') @@ -259,8 +261,8 @@ def test_load_table_from_storage_then_dump_table(self): import csv import tempfile from gcloud.storage import Client as StorageClient - TIMESTAMP = 1000 * time.time() - BUCKET_NAME = 'bq_load_test_%d' % (TIMESTAMP,) + local_id = unique_resource_id() + BUCKET_NAME = 'bq_load_test_' + local_id BLOB_NAME = 'person_ages.csv' GS_URL = 'gs://%s/%s' % (BUCKET_NAME, BLOB_NAME) ROWS = [ @@ -301,7 +303,7 @@ def test_load_table_from_storage_then_dump_table(self): self.to_delete.insert(0, table) job = Config.CLIENT.load_table_from_storage( - 'bq_load_storage_test_%d' % (TIMESTAMP,), table, GS_URL) + 'bq_load_storage_test_' + local_id, table, GS_URL) job.create_disposition = 'CREATE_NEVER' job.skip_leading_rows = 1 job.source_format = 'CSV' diff --git a/system_tests/bigtable.py b/system_tests/bigtable.py index 39d60e0658f6b..5fdd250c3350e 100644 --- a/system_tests/bigtable.py +++ b/system_tests/bigtable.py @@ -32,11 +32,11 @@ from gcloud.bigtable.row_data import PartialRowData from gcloud.environment_vars import TESTS_PROJECT +from system_test_utils import unique_resource_id + -_helpers.PROJECT = TESTS_PROJECT CENTRAL_1C_ZONE = 'us-central1-c' -NOW_MILLIS = int(1000 * time.time()) -CLUSTER_ID = 'gcloud-python-%d' % (NOW_MILLIS,) +CLUSTER_ID = 'gcloud-py' + unique_resource_id('-') TABLE_ID = 'gcloud-python-test-table' COLUMN_FAMILY_ID1 = u'col-fam-id1' COLUMN_FAMILY_ID2 = u'col-fam-id2' @@ -89,6 +89,7 @@ def _operation_wait(operation, max_attempts=5): def setUpModule(): + _helpers.PROJECT = TESTS_PROJECT Config.CLIENT = Client(admin=True) Config.CLUSTER = Config.CLIENT.cluster(CENTRAL_1C_ZONE, CLUSTER_ID, display_name=CLUSTER_ID) diff --git a/system_tests/bigtable_happybase.py b/system_tests/bigtable_happybase.py index a0f3366d7e2d5..90c35a99fe5cc 100644 --- a/system_tests/bigtable_happybase.py +++ b/system_tests/bigtable_happybase.py @@ -15,7 +15,6 @@ import operator import struct -import time import unittest2 @@ -24,13 +23,15 @@ from gcloud.bigtable.happybase.connection import Connection from gcloud.environment_vars import TESTS_PROJECT +from bigtable import _operation_wait +from system_test_utils import unique_resource_id + _PACK_I64 = struct.Struct('>q').pack _FIRST_ELT = operator.itemgetter(0) _helpers.PROJECT = TESTS_PROJECT ZONE = 'us-central1-c' -NOW_MILLIS = int(1000 * time.time()) -CLUSTER_ID = 'gcloud-python-%d' % (NOW_MILLIS,) +CLUSTER_ID = 'gcloud-py' + unique_resource_id('-') TABLE_NAME = 'table-name' ALT_TABLE_NAME = 'other-table' TTL_FOR_TEST = 3 @@ -61,26 +62,6 @@ class Config(object): TABLE = None -def _operation_wait(operation, max_attempts=5): - """Wait until an operation has completed. - - :type operation: :class:`gcloud.bigtable.cluster.Operation` - :param operation: Operation that has not finished. - - :type max_attempts: int - :param max_attempts: (Optional) The maximum number of times to check if - the operation has finished. Defaults to 5. - """ - total_sleep = 0 - while not operation.finished(): - if total_sleep > max_attempts: - return False - time.sleep(1) - total_sleep += 1 - - return True - - def set_connection(): client = client_mod.Client(admin=True) cluster = client.cluster(ZONE, CLUSTER_ID) @@ -646,7 +627,7 @@ def test_put_with_timestamp(self): value1 = 'value1' value2 = 'value2' row1_data = {COL1: value1, COL2: value2} - ts = NOW_MILLIS + ts = 1461367402 # Need to clean-up row1 after. self.rows_to_delete.append(ROW_KEY1) diff --git a/system_tests/clear_datastore.py b/system_tests/clear_datastore.py index 922802d32af20..02467b1b9351e 100644 --- a/system_tests/clear_datastore.py +++ b/system_tests/clear_datastore.py @@ -25,13 +25,13 @@ FETCH_MAX = 20 -ALL_KINDS = [ +ALL_KINDS = ( 'Character', 'Company', 'Kind', 'Person', 'Post', -] +) TRANSACTION_MAX_GROUPS = 5 @@ -98,7 +98,7 @@ def remove_all_entities(client=None): if __name__ == '__main__': print_func('This command will remove all entities for ' 'the following kinds:') - print_func('\n'.join(['- ' + val for val in ALL_KINDS])) + print_func('\n'.join('- ' + val for val in ALL_KINDS)) response = six.moves.input('Is this OK [y/n]? ') if response.lower() == 'y': remove_all_entities() diff --git a/system_tests/datastore.py b/system_tests/datastore.py index 521c727a4c93c..e74f316423fee 100644 --- a/system_tests/datastore.py +++ b/system_tests/datastore.py @@ -14,7 +14,6 @@ import datetime import os -import time import httplib2 import unittest2 @@ -26,11 +25,11 @@ from gcloud.environment_vars import GCD_DATASET from gcloud.environment_vars import TESTS_PROJECT from gcloud.exceptions import Conflict -# This assumes the command is being run via tox hence the -# repository root is the current directory. + import clear_datastore import populate_datastore from system_test_utils import EmulatorCreds +from system_test_utils import unique_resource_id class Config(object): @@ -54,7 +53,7 @@ def clone_client(client): def setUpModule(): emulator_dataset = os.getenv(GCD_DATASET) # Isolated namespace so concurrent test runs don't collide. - test_namespace = 'ns%d' % (1000 * time.time(),) + test_namespace = 'ns' + unique_resource_id() if emulator_dataset is None: _helpers.PROJECT = TESTS_PROJECT Config.CLIENT = datastore.Client(namespace=test_namespace) diff --git a/system_tests/logging_.py b/system_tests/logging_.py index 9e3c02de30e69..0623661ae876d 100644 --- a/system_tests/logging_.py +++ b/system_tests/logging_.py @@ -20,15 +20,17 @@ from gcloud.environment_vars import TESTS_PROJECT from gcloud import logging +from system_test_utils import unique_resource_id -_MILLIS = 1000 * time.time() -DEFAULT_METRIC_NAME = 'system-tests-metric-%d' % (_MILLIS,) -DEFAULT_SINK_NAME = 'system-tests-sink-%d' % (_MILLIS,) + +_RESOURCE_ID = unique_resource_id('-') +DEFAULT_METRIC_NAME = 'system-tests-metric-%d' % (_RESOURCE_ID,) +DEFAULT_SINK_NAME = 'system-tests-sink-%d' % (_RESOURCE_ID,) DEFAULT_FILTER = 'logName:syslog AND severity>=INFO' DEFAULT_DESCRIPTION = 'System testing' -BUCKET_NAME = 'gcloud-python-system-testing-%d' % (_MILLIS,) -DATASET_NAME = 'system_testing_dataset_%d' % (_MILLIS,) -TOPIC_NAME = 'gcloud-python-system-testing-%d' % (_MILLIS,) +BUCKET_NAME = 'gcloud-python-system-testing-%d' % (_RESOURCE_ID,) +DATASET_NAME = 'system_testing_dataset_%d' % (_RESOURCE_ID,) +TOPIC_NAME = 'gcloud-python-system-testing-%d' % (_RESOURCE_ID,) class Config(object): @@ -66,8 +68,7 @@ def tearDown(self): @staticmethod def _logger_name(): - _millis = 1000 * time.time() - return 'system-tests-logger-%d' % (_millis,) + return 'system-tests-logger-' + unique_resource_id('-') def test_log_text(self): TEXT_PAYLOAD = 'System test: test_log_text' diff --git a/system_tests/populate_datastore.py b/system_tests/populate_datastore.py index 6a6679ff521a9..a4e68d1a5e8b4 100644 --- a/system_tests/populate_datastore.py +++ b/system_tests/populate_datastore.py @@ -28,7 +28,7 @@ ANCESTOR = ('Book', 'GoT') RICKARD = ANCESTOR + ('Character', 'Rickard') EDDARD = RICKARD + ('Character', 'Eddard') -KEY_PATHS = [ +KEY_PATHS = ( RICKARD, EDDARD, ANCESTOR + ('Character', 'Catelyn'), @@ -37,8 +37,8 @@ EDDARD + ('Character', 'Robb'), EDDARD + ('Character', 'Bran'), EDDARD + ('Character', 'Jon Snow'), -] -CHARACTERS = [ +) +CHARACTERS = ( { 'name': u'Rickard', 'family': u'Stark', @@ -80,7 +80,7 @@ 'appearances': 32, 'alive': True, }, -] +) def print_func(message): diff --git a/system_tests/pubsub.py b/system_tests/pubsub.py index 9786ca3722399..de8a2161503e7 100644 --- a/system_tests/pubsub.py +++ b/system_tests/pubsub.py @@ -22,10 +22,12 @@ from gcloud.environment_vars import PUBSUB_EMULATOR from gcloud.environment_vars import TESTS_PROJECT from gcloud import pubsub + from system_test_utils import EmulatorCreds +from system_test_utils import unique_resource_id -DEFAULT_TOPIC_NAME = 'subscribe-me%d' % (1000 * time.time(),) +DEFAULT_TOPIC_NAME = 'subscribe-me' + unique_resource_id() class Config(object): @@ -58,7 +60,7 @@ def tearDown(self): doomed.delete() def test_create_topic(self): - topic_name = 'a-new-topic%d' % (1000 * time.time(),) + topic_name = 'a-new-topic' + unique_resource_id() topic = Config.CLIENT.topic(topic_name) self.assertFalse(topic.exists()) topic.create() @@ -68,9 +70,9 @@ def test_create_topic(self): def test_list_topics(self): topics_to_create = [ - 'new%d' % (1000 * time.time(),), - 'newer%d' % (1000 * time.time(),), - 'newest%d' % (1000 * time.time(),), + 'new' + unique_resource_id(), + 'newer' + unique_resource_id(), + 'newest' + unique_resource_id(), ] for topic_name in topics_to_create: topic = Config.CLIENT.topic(topic_name) @@ -89,7 +91,7 @@ def test_create_subscription_defaults(self): self.assertFalse(topic.exists()) topic.create() self.to_delete.append(topic) - SUBSCRIPTION_NAME = 'subscribing-now-%d' % (1000 * time.time(),) + SUBSCRIPTION_NAME = 'subscribing-now-' + unique_resource_id() subscription = topic.subscription(SUBSCRIPTION_NAME) self.assertFalse(subscription.exists()) subscription.create() @@ -103,7 +105,7 @@ def test_create_subscription_w_ack_deadline(self): self.assertFalse(topic.exists()) topic.create() self.to_delete.append(topic) - SUBSCRIPTION_NAME = 'subscribing-now-%d' % (1000 * time.time(),) + SUBSCRIPTION_NAME = 'subscribing-now-' + unique_resource_id() subscription = topic.subscription(SUBSCRIPTION_NAME, ack_deadline=120) self.assertFalse(subscription.exists()) subscription.create() @@ -121,9 +123,9 @@ def test_list_subscriptions(self): empty, _ = topic.list_subscriptions() self.assertEqual(len(empty), 0) subscriptions_to_create = [ - 'new%d' % (1000 * time.time(),), - 'newer%d' % (1000 * time.time(),), - 'newest%d' % (1000 * time.time(),), + 'new' + unique_resource_id(), + 'newer' + unique_resource_id(), + 'newest' + unique_resource_id(), ] for subscription_name in subscriptions_to_create: subscription = topic.subscription(subscription_name) @@ -142,7 +144,7 @@ def test_message_pull_mode_e2e(self): self.assertFalse(topic.exists()) topic.create() self.to_delete.append(topic) - SUBSCRIPTION_NAME = 'subscribing-now-%d' % (1000 * time.time(),) + SUBSCRIPTION_NAME = 'subscribing-now-' + unique_resource_id() subscription = topic.subscription(SUBSCRIPTION_NAME) self.assertFalse(subscription.exists()) subscription.create() @@ -170,7 +172,7 @@ def _by_timestamp(message): self.assertEqual(message2.attributes['extra'], EXTRA_2) def test_topic_iam_policy(self): - topic_name = 'test-topic-iam-policy-topic-%d' % (1000 * time.time(),) + topic_name = 'test-topic-iam-policy-topic-' + unique_resource_id() topic = Config.CLIENT.topic(topic_name) topic.create() count = 5 @@ -185,7 +187,7 @@ def test_topic_iam_policy(self): self.assertEqual(new_policy.viewers, policy.viewers) def test_subscription_iam_policy(self): - topic_name = 'test-sub-iam-policy-topic-%d' % (1000 * time.time(),) + topic_name = 'test-sub-iam-policy-topic-' + unique_resource_id() topic = Config.CLIENT.topic(topic_name) topic.create() count = 5 @@ -194,7 +196,7 @@ def test_subscription_iam_policy(self): count -= 1 self.assertTrue(topic.exists()) self.to_delete.append(topic) - SUB_NAME = 'test-sub-iam-policy-sub-%d' % (1000 * time.time(),) + SUB_NAME = 'test-sub-iam-policy-sub-' + unique_resource_id() subscription = topic.subscription(SUB_NAME) subscription.create() count = 5 @@ -209,8 +211,8 @@ def test_subscription_iam_policy(self): self.assertEqual(new_policy.viewers, policy.viewers) def test_fetch_delete_subscription_w_deleted_topic(self): - TO_DELETE = 'delete-me-%d' % (1000 * time.time(),) - ORPHANED = 'orphaned-%d' % (1000 * time.time(),) + TO_DELETE = 'delete-me-' + unique_resource_id() + ORPHANED = 'orphaned-' + unique_resource_id() topic = Config.CLIENT.topic(TO_DELETE) topic.create() subscription = topic.subscription(ORPHANED) diff --git a/system_tests/run_emulator.py b/system_tests/run_emulator.py index c77f7c2772406..c3ea8fdcc602d 100644 --- a/system_tests/run_emulator.py +++ b/system_tests/run_emulator.py @@ -62,7 +62,7 @@ def get_start_command(package): :rtype: tuple :returns: The arguments to be used, in a tuple. """ - return ('gcloud', 'beta', 'emulators', package, 'start') + return 'gcloud', 'beta', 'emulators', package, 'start' def get_env_init_command(package): @@ -74,7 +74,7 @@ def get_env_init_command(package): :rtype: tuple :returns: The arguments to be used, in a tuple. """ - return ('gcloud', 'beta', 'emulators', package, 'env-init') + return 'gcloud', 'beta', 'emulators', package, 'env-init' def datastore_wait_ready(popen): diff --git a/system_tests/run_system_test.py b/system_tests/run_system_test.py index 6369a9bb72276..fd2d1967f9824 100644 --- a/system_tests/run_system_test.py +++ b/system_tests/run_system_test.py @@ -16,8 +16,6 @@ import sys import unittest2 -# This assumes the command is being run via tox hence the -# repository root is the current directory. import bigquery import bigtable import bigtable_happybase diff --git a/system_tests/storage.py b/system_tests/storage.py index a94233dac8418..d05aa8556ba51 100644 --- a/system_tests/storage.py +++ b/system_tests/storage.py @@ -26,6 +26,8 @@ from gcloud import storage from gcloud.storage._helpers import _base64_md5hash +from system_test_utils import unique_resource_id + HTTP = httplib2.Http() _helpers.PROJECT = TESTS_PROJECT @@ -43,8 +45,7 @@ class Config(object): def setUpModule(): Config.CLIENT = storage.Client() - # %d rounds milliseconds to nearest integer. - bucket_name = 'new%d' % (1000 * time.time(),) + bucket_name = 'new' + unique_resource_id() # In the **very** rare case the bucket name is reserved, this # fails with a ConnectionError. Config.TEST_BUCKET = Config.CLIENT.create_bucket(bucket_name) @@ -65,7 +66,7 @@ def tearDown(self): Config.CLIENT.bucket(bucket_name).delete() def test_create_bucket(self): - new_bucket_name = 'a-new-bucket' + new_bucket_name = 'a-new-bucket' + unique_resource_id() self.assertRaises(exceptions.NotFound, Config.CLIENT.get_bucket, new_bucket_name) created = Config.CLIENT.create_bucket(new_bucket_name) @@ -74,9 +75,9 @@ def test_create_bucket(self): def test_list_buckets(self): buckets_to_create = [ - 'new%d' % (1000 * time.time(),), - 'newer%d' % (1000 * time.time(),), - 'newest%d' % (1000 * time.time(),), + 'new' + unique_resource_id(), + 'newer' + unique_resource_id(), + 'newest' + unique_resource_id(), ] created_buckets = [] for bucket_name in buckets_to_create: @@ -193,7 +194,7 @@ def test_copy_existing_file(self): class TestStorageListFiles(TestStorageFiles): - FILENAMES = ['CloudLogo1', 'CloudLogo2', 'CloudLogo3'] + FILENAMES = ('CloudLogo1', 'CloudLogo2', 'CloudLogo3') @classmethod def setUpClass(cls): @@ -238,14 +239,14 @@ def test_paginate_files(self): class TestStoragePseudoHierarchy(TestStorageFiles): - FILENAMES = [ + FILENAMES = ( 'file01.txt', 'parent/file11.txt', 'parent/child/file21.txt', 'parent/child/file22.txt', 'parent/child/grand/file31.txt', 'parent/child/other/file32.txt', - ] + ) @classmethod def setUpClass(cls): diff --git a/system_tests/system_test_utils.py b/system_tests/system_test_utils.py index d936bd05cd742..4fc9846de3de6 100644 --- a/system_tests/system_test_utils.py +++ b/system_tests/system_test_utils.py @@ -15,6 +15,7 @@ from __future__ import print_function import os import sys +import time from gcloud.environment_vars import CREDENTIALS as TEST_CREDENTIALS from gcloud.environment_vars import TESTS_PROJECT @@ -56,3 +57,16 @@ def check_environ(): if missing: print(ENVIRON_ERROR_MSG % ', '.join(missing), file=sys.stderr) sys.exit(1) + + +def unique_resource_id(delimiter='_'): + """A unique identifier for a resource. + + Intended to help locate resources created in particular + testing environments and at particular times. + """ + build_id = os.getenv('TRAVIS_BUILD_ID', '') + if build_id == '': + return '%d' % (1000 * time.time(),) + else: + return '%d%s%s' % (time.time(), delimiter, build_id)