Add max_message_length for larger rows. #2907

daspecster · 2016-12-29T17:18:47Z

system_tests/bigtable.py

+        self.assertEqual(partial_row_data.row_key, ROW_KEY)
+        cell = partial_row_data.cells[COLUMN_FAMILY_ID1]
+        column = cell[COL_NAME1]
+        self.assertIsNotNone(column[0].value)


bigtable/google/cloud/bigtable/client.py

@@ -76,8 +76,15 @@ def _make_data_stub(client):
    :returns: A gRPC stub object.
    """
    if client.emulator_host is None:
+        one_hundred_mb = 100 * 1024 * 1024


core/google/cloud/_helpers.py

@@ -480,22 +480,29 @@ def make_secure_channel(credentials, user_agent, host):
    :type host: str
    :param host: The host for the service.

+    :type extra_options: tuple
+    :param extra_options: Extra gRPC options used when creating the channel.


system_tests/bigtable.py

+        self.assertEqual(partial_row_data.row_key, ROW_KEY)
+        cell = partial_row_data.cells[COLUMN_FAMILY_ID1]
+        column = cell[COL_NAME1]
+        self.assertIsNotNone(column[0].value)


system_tests/bigtable.py

+        row = self._table.row(ROW_KEY)
+        self.rows_to_delete.append(row)
+
+        data = '1' * 10 * 1024 * 1024  # 10MB of 1's.


system_tests/bigtable.py

@@ -356,6 +356,21 @@ def _write_to_row(self, row1=None, row2=None, row3=None, row4=None):
        cell4 = Cell(CELL_VAL4, timestamp4)
        return cell1, cell2, cell3, cell4

+    def test_read_large_row(self):


bigtable/google/cloud/bigtable/client.py

+        # NOTE: 'grpc.max_message_length' will be deprecated in the 1.1 release
+        # of grpcio in favor of 'grpc.max_receive_message_length' and
+        # 'grpc.max_send_message_length'.
+        max_msg_length = (('grpc.max_message_length', one_hundred_mb),


bigtable/google/cloud/bigtable/client.py

@@ -76,8 +76,15 @@ def _make_data_stub(client):
    :returns: A gRPC stub object.
    """
    if client.emulator_host is None:
+        one_hundred_mb = 100 * 1024 * 1024
+        # NOTE: 'grpc.max_message_length' will be deprecated in the 1.1 release


daspecster · 2016-12-29T20:36:39Z

Added Nathaniel's feedback and updated the constant.

bigtable/google/cloud/bigtable/client.py

+        # grpcio 1.1 and later.
+        max_msg_length = (('grpc.max_message_length', _MAX_MSG_LENGTH_100MB),
+                          ('grpc.max_receive_message_length',
+                           _MAX_MSG_LENGTH_100MB))


daspecster · 2016-12-29T21:13:35Z

Reformatted and squashed.

dhermes

General comment: Are you planning to add a system test where two cells combined exceed 10MB but neither does individually?

bigtable/google/cloud/bigtable/client.py

@@ -65,6 +65,13 @@
 READ_ONLY_SCOPE = 'https://www.googleapis.com/auth/bigtable.data.readonly'
 """Scope for reading table data."""

+# NOTE: 'grpc.max_message_length' will no longer be recognized in
+# grpcio 1.1 and later.


system_tests/bigtable.py

+
+        with self.assertRaises(Exception):
+            partial_row_data = self._table.read_row(ROW_KEY)
+


system_tests/bigtable.py

+        row = self._table.row(ROW_KEY)
+        self.rows_to_delete.append(row)
+
+        data = '1' * 101 * 1024 * 1024  # 11MB of 1's.


system_tests/bigtable.py

+
+        data = '1' * 101 * 1024 * 1024  # 11MB of 1's.
+        row.set_cell(COLUMN_FAMILY_ID1, COL_NAME1, data)
+        with self.assertRaises(Exception):


system_tests/bigtable.py

+        self.rows_to_delete.append(row)
+
+        number_of_bytes = 10 * 1024 * 1024
+        data = '1' * number_of_bytes  # 10MB of 1's.


daspecster · 2016-12-30T04:12:29Z

@dhermes and @nathanielmanistaatgoogle
I added the test for multiple cells.

If this looks good, I'll squash and merge.

daspecster · 2017-01-03T17:14:17Z

Squashed.

bigtable/google/cloud/bigtable/client.py

+_MAX_MSG_LENGTH_100MB = 100 * 1024 * 1024
+_GRPC_MAX_LENGTH_OPTIONS = (
+    ('grpc.max_message_length', _MAX_MSG_LENGTH_100MB),
+    ('grpc.max_receive_message_length', _MAX_MSG_LENGTH_100MB)


system_tests/bigtable.py

@@ -30,6 +30,7 @@
 from google.cloud.bigtable.row_data import Cell
 from google.cloud.bigtable.row_data import PartialRowData
 from google.cloud.environment_vars import BIGTABLE_EMULATOR
+from grpc._channel import _Rendezvous


system_tests/bigtable.py

+        cell = partial_row_data.cells[COLUMN_FAMILY_ID1]
+        column = cell[COL_NAME1]
+        value = column[0].value
+        self.assertIsNotNone(value)


system_tests/bigtable.py

+        row.set_cell(COLUMN_FAMILY_ID1, COL_NAME1, data)
+        row.commit()
+        row.set_cell(COLUMN_FAMILY_ID1, COL_NAME1, data)
+        row.commit()


system_tests/bigtable.py

+        row.commit()
+
+        with self.assertRaises(_Rendezvous):
+            self._table.read_row(ROW_KEY)


system_tests/bigtable.py

@@ -356,6 +357,46 @@ def _write_to_row(self, row1=None, row2=None, row3=None, row4=None):
        cell4 = Cell(CELL_VAL4, timestamp4)
        return cell1, cell2, cell3, cell4

+    def test_read_large_cell_limit(self):


system_tests/bigtable.py

-        with self.assertRaises(_Rendezvous):
-            row.commit()
+        self.assertEqual(len(column), 1)
+        self.assertEqual(len(column[0].value), number_of_bytes)


daspecster · 2017-01-04T02:33:19Z

Squashed.

…receive-length Add max_message_length for larger rows.

daspecster added api: bigtable Issues related to the Bigtable API. type: bug Error or flaw in code with unintended results or allowing sub-optimal usage patterns. labels Dec 29, 2016

daspecster assigned tseaver and dhermes Dec 29, 2016

googlebot added the cla: yes This human has signed the Contributor License Agreement. label Dec 29, 2016

daspecster commented Dec 29, 2016

View reviewed changes

system_tests/bigtable.py

self.assertEqual(partial_row_data.row_key, ROW_KEY)

cell = partial_row_data.cells[COLUMN_FAMILY_ID1]

column = cell[COL_NAME1]

self.assertIsNotNone(column[0].value)

This comment was marked as spam.

Sign in to view

This comment was marked as spam.

Sign in to view

dhermes reviewed Dec 29, 2016

View reviewed changes

daspecster changed the title ~~Add max_receive_message_length for larger rows.~~ Add max_message_length for larger rows. Dec 29, 2016

nathanielmanistaatgoogle reviewed Dec 29, 2016

View reviewed changes

dhermes reviewed Dec 29, 2016

View reviewed changes

bigtable/google/cloud/bigtable/client.py

# grpcio 1.1 and later.

max_msg_length = (('grpc.max_message_length', _MAX_MSG_LENGTH_100MB),

('grpc.max_receive_message_length',

_MAX_MSG_LENGTH_100MB))

This comment was marked as spam.

Sign in to view

This comment was marked as spam.

Sign in to view

daspecster force-pushed the increase-bigtable-receive-length branch from 78aa62a to 878fb58 Compare December 29, 2016 21:13

dhermes reviewed Dec 29, 2016

View reviewed changes

daspecster force-pushed the increase-bigtable-receive-length branch from dc49e57 to 11d33bb Compare December 29, 2016 21:39

daspecster force-pushed the increase-bigtable-receive-length branch from 38e46d5 to 8e950a8 Compare January 3, 2017 17:14

dhermes reviewed Jan 3, 2017

View reviewed changes

system_tests/bigtable.py

with self.assertRaises(_Rendezvous):

row.commit()

self.assertEqual(len(column), 1)

self.assertEqual(len(column[0].value), number_of_bytes)

This comment was marked as spam.

Sign in to view

This comment was marked as spam.

Sign in to view

dhermes approved these changes Jan 3, 2017

View reviewed changes

Add max_receive_message_length for larger rows.

85c6bf3

daspecster force-pushed the increase-bigtable-receive-length branch from c5990c1 to 85c6bf3 Compare January 4, 2017 02:33

daspecster merged commit 60f1ada into googleapis:master Jan 4, 2017

daspecster deleted the increase-bigtable-receive-length branch January 4, 2017 03:06

dhermes mentioned this pull request Jan 30, 2017

Cutting google-cloud-core==0.23.0. #2972

Merged

dhermes mentioned this pull request Feb 24, 2017

Upgrading all versions for umbrella release. #3068

Merged

19 tasks

lukesneeringer mentioned this pull request Mar 20, 2017

bigtable table.getRows() method throws error "Received message larger than max" to its callback googleapis/google-cloud-node#2111

Closed

richkadel pushed a commit to richkadel/google-cloud-python that referenced this pull request May 6, 2017

Merge pull request googleapis#2907 from daspecster/increase-bigtable-…

99d6e00

…receive-length Add max_message_length for larger rows.

drigz mentioned this pull request Jul 7, 2017

Can't receive messages larger than 4MiB #3594

Closed

theacodes unassigned dhermes Sep 28, 2018


		with self.assertRaises(Exception):
		partial_row_data = self._table.read_row(ROW_KEY)

Add max_message_length for larger rows. #2907

Add max_message_length for larger rows. #2907

Conversation

daspecster commented Dec 29, 2016

This comment was marked as spam.

This comment was marked as spam.

This comment was marked as spam.

This comment was marked as spam.

This comment was marked as spam.

This comment was marked as spam.

This comment was marked as spam.

This comment was marked as spam.

This comment was marked as spam.

This comment was marked as spam.

daspecster commented Dec 29, 2016

This comment was marked as spam.

This comment was marked as spam.

daspecster commented Dec 29, 2016

dhermes left a comment

Choose a reason for hiding this comment

This comment was marked as spam.

This comment was marked as spam.

This comment was marked as spam.

This comment was marked as spam.

This comment was marked as spam.

This comment was marked as spam.

This comment was marked as spam.

daspecster commented Dec 30, 2016 • edited Loading

daspecster commented Jan 3, 2017

This comment was marked as spam.

This comment was marked as spam.

This comment was marked as spam.

This comment was marked as spam.

This comment was marked as spam.

This comment was marked as spam.

This comment was marked as spam.

This comment was marked as spam.

This comment was marked as spam.

This comment was marked as spam.

This comment was marked as spam.

daspecster commented Jan 4, 2017

daspecster commented Dec 30, 2016 •

edited

Loading