diff --git a/bigquery/google/cloud/bigquery/_helpers.py b/bigquery/google/cloud/bigquery/_helpers.py index 8cebe9fbec01..44183a3a3f2b 100644 --- a/bigquery/google/cloud/bigquery/_helpers.py +++ b/bigquery/google/cloud/bigquery/_helpers.py @@ -14,6 +14,7 @@ """Shared helper functions for BigQuery API classes.""" +import base64 from collections import OrderedDict import datetime @@ -22,6 +23,7 @@ from google.cloud._helpers import _datetime_to_rfc3339 from google.cloud._helpers import _microseconds_from_datetime from google.cloud._helpers import _RFC3339_NO_FRACTION +from google.cloud._helpers import _time_from_iso8601_time_naive def _not_null(value, field): @@ -47,6 +49,17 @@ def _bool_from_json(value, field): return value.lower() in ['t', 'true', '1'] +def _string_from_json(value, _): + """NOOP string -> string coercion""" + return value + + +def _bytes_from_json(value, field): + """Base64-decode value""" + if _not_null(value, field): + return base64.decodestring(value) + + def _timestamp_from_json(value, field): """Coerce 'value' to a datetime, if set or not nullable.""" if _not_null(value, field): @@ -64,9 +77,17 @@ def _datetime_from_json(value, field): def _date_from_json(value, field): """Coerce 'value' to a datetime date, if set or not nullable""" if _not_null(value, field): + # value will be a string, in YYYY-MM-DD form. return _date_from_iso8601_date(value) +def _time_from_json(value, field): + """Coerce 'value' to a datetime date, if set or not nullable""" + if _not_null(value, field): + # value will be a string, in HH:MM:SS form. + return _time_from_iso8601_time_naive(value) + + def _record_from_json(value, field): """Coerce 'value' to a mapping, if set or not nullable.""" if _not_null(value, field): @@ -82,11 +103,6 @@ def _record_from_json(value, field): return record -def _string_from_json(value, _): - """NOOP string -> string coercion""" - return value - - _CELLDATA_FROM_JSON = { 'INTEGER': _int_from_json, 'INT64': _int_from_json, @@ -94,11 +110,13 @@ def _string_from_json(value, _): 'FLOAT64': _float_from_json, 'BOOLEAN': _bool_from_json, 'BOOL': _bool_from_json, + 'STRING': _string_from_json, + 'BYTES': _bytes_from_json, 'TIMESTAMP': _timestamp_from_json, 'DATETIME': _datetime_from_json, 'DATE': _date_from_json, + 'TIME': _time_from_json, 'RECORD': _record_from_json, - 'STRING': _string_from_json, } @@ -121,6 +139,13 @@ def _bool_to_json(value): return value +def _bytes_to_json(value): + """Coerce 'value' to an JSON-compatible representation.""" + if isinstance(value, bytes): + value = base64.encodestring(value) + return value + + def _timestamp_to_json(value): """Coerce 'value' to an JSON-compatible representation.""" if isinstance(value, datetime.datetime): @@ -142,6 +167,13 @@ def _date_to_json(value): return value +def _time_to_json(value): + """Coerce 'value' to an JSON-compatible representation.""" + if isinstance(value, datetime.time): + value = value.isoformat() + return value + + _SCALAR_VALUE_TO_JSON = { 'INTEGER': _int_to_json, 'INT64': _int_to_json, @@ -149,9 +181,11 @@ def _date_to_json(value): 'FLOAT64': _float_to_json, 'BOOLEAN': _bool_to_json, 'BOOL': _bool_to_json, + 'BYTES': _bytes_to_json, 'TIMESTAMP': _timestamp_to_json, 'DATETIME': _datetime_to_json, 'DATE': _date_to_json, + 'TIME': _time_to_json, } diff --git a/bigquery/unit_tests/test__helpers.py b/bigquery/unit_tests/test__helpers.py index b3ccb1d715f5..76eacc9a04a8 100644 --- a/bigquery/unit_tests/test__helpers.py +++ b/bigquery/unit_tests/test__helpers.py @@ -105,6 +105,44 @@ def test_w_value_other(self): self.assertFalse(coerced) +class Test_string_from_json(unittest.TestCase): + + def _call_fut(self, value, field): + from google.cloud.bigquery._helpers import _string_from_json + return _string_from_json(value, field) + + def test_w_none_nullable(self): + self.assertIsNone(self._call_fut(None, _Field('NULLABLE'))) + + def test_w_none_required(self): + self.assertIsNone(self._call_fut(None, _Field('REQUIRED'))) + + def test_w_string_value(self): + coerced = self._call_fut('Wonderful!', object()) + self.assertEqual(coerced, 'Wonderful!') + + +class Test_bytes_from_json(unittest.TestCase): + + def _call_fut(self, value, field): + from google.cloud.bigquery._helpers import _bytes_from_json + return _bytes_from_json(value, field) + + def test_w_none_nullable(self): + self.assertIsNone(self._call_fut(None, _Field('NULLABLE'))) + + def test_w_none_required(self): + with self.assertRaises(TypeError): + self._call_fut(None, _Field('REQUIRED')) + + def test_w_base64_encoded_value(self): + import base64 + expected = b'Wonderful!' + encoded = base64.encodestring(expected) + coerced = self._call_fut(encoded, object()) + self.assertEqual(coerced, expected) + + class Test_timestamp_from_json(unittest.TestCase): def _call_fut(self, value, field): @@ -177,6 +215,27 @@ def test_w_string_value(self): datetime.date(1987, 9, 22)) +class Test_time_from_json(unittest.TestCase): + + def _call_fut(self, value, field): + from google.cloud.bigquery._helpers import _time_from_json + return _time_from_json(value, field) + + def test_w_none_nullable(self): + self.assertIsNone(self._call_fut(None, _Field('NULLABLE'))) + + def test_w_none_required(self): + with self.assertRaises(TypeError): + self._call_fut(None, _Field('REQUIRED')) + + def test_w_string_value(self): + import datetime + coerced = self._call_fut('12:12:27', object()) + self.assertEqual( + coerced, + datetime.time(12, 12, 27)) + + class Test_record_from_json(unittest.TestCase): def _call_fut(self, value, field): @@ -238,23 +297,6 @@ def test_w_record_subfield(self): self.assertEqual(coerced, expected) -class Test_string_from_json(unittest.TestCase): - - def _call_fut(self, value, field): - from google.cloud.bigquery._helpers import _string_from_json - return _string_from_json(value, field) - - def test_w_none_nullable(self): - self.assertIsNone(self._call_fut(None, _Field('NULLABLE'))) - - def test_w_none_required(self): - self.assertIsNone(self._call_fut(None, _Field('RECORD'))) - - def test_w_string_value(self): - coerced = self._call_fut('Wonderful!', object()) - self.assertEqual(coerced, 'Wonderful!') - - class Test_row_from_json(unittest.TestCase): def _call_fut(self, row, schema): @@ -471,6 +513,23 @@ def test_w_string(self): self.assertEqual(self._call_fut('false'), 'false') +class Test_bytes_to_json(unittest.TestCase): + + def _call_fut(self, value): + from google.cloud.bigquery._helpers import _bytes_to_json + return _bytes_to_json(value) + + def test_w_non_bytes(self): + non_bytes = object() + self.assertIs(self._call_fut(non_bytes), non_bytes) + + def test_w_bytes(self): + import base64 + source = b'source' + expected = base64.encodestring(source) + self.assertEqual(self._call_fut(source), expected) + + class Test_timestamp_to_json(unittest.TestCase): def _call_fut(self, value): @@ -522,6 +581,22 @@ def test_w_datetime(self): self.assertEqual(self._call_fut(when), '2016-12-03') +class Test_time_to_json(unittest.TestCase): + + def _call_fut(self, value): + from google.cloud.bigquery._helpers import _time_to_json + return _time_to_json(value) + + def test_w_string(self): + RFC3339 = '12:13:41' + self.assertEqual(self._call_fut(RFC3339), RFC3339) + + def test_w_datetime(self): + import datetime + when = datetime.time(12, 13, 41) + self.assertEqual(self._call_fut(when), '12:13:41') + + class Test_ConfigurationProperty(unittest.TestCase): @staticmethod diff --git a/core/google/cloud/_helpers.py b/core/google/cloud/_helpers.py index 9b4ec5736cb0..50936915fa33 100644 --- a/core/google/cloud/_helpers.py +++ b/core/google/cloud/_helpers.py @@ -251,6 +251,19 @@ def _date_from_iso8601_date(value): return datetime.datetime.strptime(value, '%Y-%m-%d').date() +def _time_from_iso8601_time_naive(value): + """Convert a zoneless ISO8601 time string to naive datetime time + + :type value: str + :param value: The time string to convert + + :rtype: :class:`datetime.time` + :returns: A datetime time object created from the string + + """ + return datetime.datetime.strptime(value, '%H:%M:%S').time() + + def _rfc3339_to_datetime(dt_str): """Convert a microsecond-precision timetamp to a native datetime. diff --git a/core/unit_tests/test__helpers.py b/core/unit_tests/test__helpers.py index 78391e56ef42..08c27ae556e2 100644 --- a/core/unit_tests/test__helpers.py +++ b/core/unit_tests/test__helpers.py @@ -265,6 +265,18 @@ def test_todays_date(self): self.assertEqual(self._call_fut(TODAY.strftime("%Y-%m-%d")), TODAY) +class Test___time_from_iso8601_time_naive(unittest.TestCase): + + def _call_fut(self, value): + from google.cloud._helpers import _time_from_iso8601_time_naive + return _time_from_iso8601_time_naive(value) + + def test_todays_date(self): + import datetime + WHEN = datetime.time(12, 9, 42) + self.assertEqual(self._call_fut(("12:09:42")), WHEN) + + class Test__rfc3339_to_datetime(unittest.TestCase): def _call_fut(self, dt_str): diff --git a/system_tests/bigquery.py b/system_tests/bigquery.py index bf913d499afb..ff6d3ceff7d9 100644 --- a/system_tests/bigquery.py +++ b/system_tests/bigquery.py @@ -479,12 +479,49 @@ def _job_done(instance): # raise an error, and that the job completed (in the `retry()` # above). - def test_sync_query_w_nested_arrays_and_structs(self): + def test_sync_query_w_standard_sql_types(self): + import datetime + from google.cloud._helpers import UTC + naive = datetime.datetime(2016, 12, 5, 12, 41, 9) + stamp = "%s %s" % (naive.date().isoformat(), naive.time().isoformat()) + zoned = naive.replace(tzinfo=UTC) EXAMPLES = [ { 'sql': 'SELECT 1', 'expected': 1, }, + { + 'sql': 'SELECT 1.3', + 'expected': 1.3, + }, + { + 'sql': 'SELECT TRUE', + 'expected': True, + }, + { + 'sql': 'SELECT "ABC"', + 'expected': 'ABC', + }, + { + 'sql': 'SELECT CAST("foo" AS BYTES)', + 'expected': b'foo', + }, + { + 'sql': 'SELECT TIMESTAMP "%s"' % (stamp,), + 'expected': zoned, + }, + { + 'sql': 'SELECT DATETIME(TIMESTAMP "%s")' % (stamp,), + 'expected': naive, + }, + { + 'sql': 'SELECT DATE(TIMESTAMP "%s")' % (stamp,), + 'expected': naive.date(), + }, + { + 'sql': 'SELECT TIME(TIMESTAMP "%s")' % (stamp,), + 'expected': naive.time(), + }, { 'sql': 'SELECT (1, 2)', 'expected': {'_field_1': 1, '_field_2': 2},