[Tables] Emulator tests and binary serialization (Azure#18829)

* Deserialize to binary

* Updates for emulator support

* Pylint

* Changelog and tests

sdk/tables/azure-data-tables/CHANGELOG.md

@@ -1,5 +1,14 @@
 # Release History
 
+## 12.0.0 (unreleased)
+**Breaking**
+* EdmType.Binary data in entities will now be deserialized as `bytes` in Python 3 and `str` in Python 2, rather than an `EdmProperty` instance. Likewise on serialization, `bytes` in Python 3 and `str` in Python 2 will be interpreted as binary (this is unchanged for Python 3, but breaking for Python 2, where `str` was previously serialized as EdmType.String)
+
+**Fixes**
+* Fixed support for Cosmos emulator endpoint, via URL/credential or connection string.
+* Fixed table name from URL parsing in `TableClient.from_table_url` classmethod.
+* The `account_name` attribute on clients will now be pulled from an `AzureNamedKeyCredential` if used.
+
 ## 12.0.0b7 (2021-05-11)
 **Breaking**
 * The `account_url` parameter in the client constructors has been renamed to `endpoint`.

sdk/tables/azure-data-tables/azure/data/tables/__init__.py

@@ -6,7 +6,7 @@
 from azure.data.tables._models import TableServiceStats
 
 from ._entity import TableEntity, EntityProperty, EdmType
-from ._error import RequestTooLargeError, TableTransactionError
+from ._error import RequestTooLargeError, TableTransactionError, TableErrorCode
 from ._table_shared_access_signature import generate_table_sas, generate_account_sas
 from ._table_client import TableClient
 from ._table_service_client import TableServiceClient
@@ -26,7 +26,6 @@
     TransactionOperation
 )
 from ._version import VERSION
-from ._deserialize import TableErrorCode
 
 __version__ = VERSION
 

sdk/tables/azure-data-tables/azure/data/tables/_base_client.py

@@ -80,7 +80,6 @@ def __init__(
                 account_url = "https://" + account_url
         except AttributeError:
             raise ValueError("Account URL must be a string.")
-        self._cosmos_endpoint = _is_cosmos_endpoint(account_url)
         parsed_url = urlparse(account_url.rstrip("/"))
         if not parsed_url.netloc:
             raise ValueError("Invalid URL: {}".format(account_url))
@@ -94,7 +93,7 @@ def __init__(
         self._location_mode = kwargs.get("location_mode", LocationMode.PRIMARY)
         self._hosts = kwargs.get("_hosts")
         self.scheme = parsed_url.scheme
-        self._cosmos_endpoint = _is_cosmos_endpoint(parsed_url.hostname)
+        self._cosmos_endpoint = _is_cosmos_endpoint(parsed_url)
         if ".core." in parsed_url.netloc or ".cosmos." in parsed_url.netloc:
             account = parsed_url.netloc.split(".table.core.")
             if "cosmos" in parsed_url.netloc:
@@ -114,17 +113,19 @@ def __init__(
         self.credential = credential
         if self.scheme.lower() != "https" and hasattr(self.credential, "get_token"):
             raise ValueError("Token credential is only supported with HTTPS.")
-        if hasattr(self.credential, "account_name"):
-            self.account_name = self.credential.account_name
+        if hasattr(self.credential, "named_key"):
+            self.account_name = self.credential.named_key.name
             secondary_hostname = "{}-secondary.table.{}".format(
-                self.credential.account_name, SERVICE_HOST_BASE
+                self.credential.named_key.name, SERVICE_HOST_BASE
             )
 
         if not self._hosts:
             if len(account) > 1:
                 secondary_hostname = parsed_url.netloc.replace(
                     account[0], account[0] + "-secondary"
-                )
+                ) + parsed_url.path.replace(
+                    account[0], account[0] + "-secondary"
+                ).rstrip("/")
             if kwargs.get("secondary_hostname"):
                 secondary_hostname = kwargs["secondary_hostname"]
             primary_hostname = (parsed_url.netloc + parsed_url.path).rstrip("/")
@@ -346,7 +347,6 @@ def parse_connection_str(conn_str, credential, keyword_args):
             credential = conn_settings.get("sharedaccesssignature")
             # if "sharedaccesssignature" in conn_settings:
             #     credential = AzureSasCredential(conn_settings['sharedaccesssignature'])
-
     primary = conn_settings.get("tableendpoint")
     secondary = conn_settings.get("tablesecondaryendpoint")
     if not primary:

sdk/tables/azure-data-tables/azure/data/tables/_common_conversion.py

@@ -89,12 +89,12 @@ def _sign_string(key, string_to_sign, key_is_base64=True):
 
 
 def _is_cosmos_endpoint(url):
-    if ".table.cosmodb." in url:
+    if ".table.cosmodb." in url.hostname:
         return True
-
-    if ".table.cosmos." in url:
+    if ".table.cosmos." in url.hostname:
+        return True
+    if url.hostname == "localhost" and url.port != 10002:
         return True
-
     return False
 
 

sdk/tables/azure-data-tables/azure/data/tables/_deserialize.py

@@ -4,19 +4,14 @@
 # license information.
 # --------------------------------------------------------------------------
 
-from typing import TYPE_CHECKING
 from uuid import UUID
 import logging
 import datetime
 
-from azure.core.exceptions import ResourceExistsError
+import six
 
 from ._entity import EntityProperty, EdmType, TableEntity
 from ._common_conversion import _decode_base64_to_bytes, TZ_UTC
-from ._error import TableErrorCode
-
-if TYPE_CHECKING:
-    from azure.core.exceptions import AzureError
 
 
 _LOGGER = logging.getLogger(__name__)
@@ -26,18 +21,6 @@
 except ImportError:
     from urllib2 import quote  # type: ignore
 
-if TYPE_CHECKING:
-    from typing import (  # pylint: disable=ungrouped-imports
-        Union,
-        Optional,
-        Any,
-        Iterable,
-        Dict,
-        List,
-        Type,
-        Tuple,
-    )
-
 
 class TablesEntityDatetime(datetime.datetime):
 
@@ -62,29 +45,14 @@ def get_enum_value(value):
         return value
 
 
-def _deserialize_table_creation(response, _, headers):
-    if response.status_code == 204:
-        error_code = TableErrorCode.table_already_exists
-        error = ResourceExistsError(
-            message="Table already exists\nRequestId:{}\nTime:{}\nErrorCode:{}".format(
-                headers["x-ms-request-id"], headers["Date"], error_code
-            ),
-            response=response,
-        )
-        error.error_code = error_code
-        error.additional_info = {}
-        raise error
-    return headers
-
-
 def _from_entity_binary(value):
     # type: (str) -> EntityProperty
-    return EntityProperty(_decode_base64_to_bytes(value), EdmType.BINARY)
+    return _decode_base64_to_bytes(value)
 
 
 def _from_entity_int32(value):
     # type: (str) -> EntityProperty
-    return EntityProperty(int(value), EdmType.INT32)
+    return int(value)
 
 
 def _from_entity_int64(value):
@@ -129,8 +97,9 @@ def _from_entity_guid(value):
 
 def _from_entity_str(value):
     # type: (str) -> EntityProperty
-    return EntityProperty(value, EdmType.STRING)
-
+    if isinstance(six.binary_type):
+        return value.decode('utf-8')
+    return value
 
 _EDM_TYPES = [
     EdmType.BINARY,

sdk/tables/azure-data-tables/azure/data/tables/_table_batch.py

@@ -12,7 +12,7 @@
     Optional
 )
 
-from ._common_conversion import _is_cosmos_endpoint, _transform_patch_to_cosmos_post
+from ._common_conversion import _transform_patch_to_cosmos_post
 from ._models import UpdateMode
 from ._serialize import _get_match_headers, _add_entity_properties
 from ._entity import TableEntity
@@ -43,6 +43,7 @@ def __init__(
         deserializer,  # type: msrest.Deserializer
         config,  # type: AzureTableConfiguration
         table_name,  # type: str
+        is_cosmos_endpoint=False,  # type: bool
         **kwargs  # type: Dict[str, Any]
     ):
         """Create TableClient from a Credential.
@@ -66,6 +67,7 @@ def __init__(
         self._serialize = serializer
         self._deserialize = deserializer
         self._config = config
+        self._is_cosmos_endpoint = is_cosmos_endpoint
         self.table_name = table_name
 
         self._partition_key = kwargs.pop("partition_key", None)
@@ -485,7 +487,7 @@ def _batch_merge_entity(
         request = self._client._client.patch(  # pylint: disable=protected-access
             url, query_parameters, header_parameters, **body_content_kwargs
         )
-        if _is_cosmos_endpoint(url):
+        if self._is_cosmos_endpoint:
             _transform_patch_to_cosmos_post(request)
         self.requests.append(request)
 

sdk/tables/azure-data-tables/azure/data/tables/_table_client.py

@@ -152,6 +152,8 @@ def from_table_url(cls, table_url, credential=None, **kwargs):
             parsed_url.query,
         )
         table_name = unquote(table_path[-1])
+        if table_name.lower().startswith("tables('"):
+            table_name = table_name[8:-2]
         if not table_name:
             raise ValueError(
                 "Invalid URL. Please provide a URL with a valid table name"
@@ -705,6 +707,7 @@ def submit_transaction(
             self._client._deserialize,  # pylint: disable=protected-access
             self._client._config,  # pylint: disable=protected-access
             self.table_name,
+            is_cosmos_endpoint=self._cosmos_endpoint,
             **kwargs
         )
         for operation in operations:

sdk/tables/azure-data-tables/azure/data/tables/aio/_table_batch_async.py

@@ -6,7 +6,7 @@
 from typing import Dict, Any, Optional, Union, TYPE_CHECKING
 import msrest
 
-from .._common_conversion import _is_cosmos_endpoint, _transform_patch_to_cosmos_post
+from .._common_conversion import _transform_patch_to_cosmos_post
 from .._models import UpdateMode
 from .._entity import TableEntity
 from .._table_batch import EntityType
@@ -41,12 +41,14 @@ def __init__(
         deserializer: msrest.Deserializer,
         config: AzureTableConfiguration,
         table_name: str,
+        is_cosmos_endpoint: bool = False,
         **kwargs: Dict[str, Any]
     ) -> None:
         self._client = client
         self._serialize = serializer
         self._deserialize = deserializer
         self._config = config
+        self._is_cosmos_endpoint = is_cosmos_endpoint
         self.table_name = table_name
 
         self._partition_key = kwargs.pop("partition_key", None)
@@ -456,7 +458,7 @@ def _batch_merge_entity(
         request = self._client._client.patch(  # pylint: disable=protected-access
             url, query_parameters, header_parameters, **body_content_kwargs
         )
-        if _is_cosmos_endpoint(url):
+        if self._is_cosmos_endpoint:
             _transform_patch_to_cosmos_post(request)
         self.requests.append(request)
 

sdk/tables/azure-data-tables/azure/data/tables/aio/_table_client_async.py

@@ -149,6 +149,8 @@ def from_table_url(
             parsed_url.query,
         )
         table_name = unquote(table_path[-1])
+        if table_name.lower().startswith("tables('"):
+            table_name = table_name[8:-2]
         if not table_name:
             raise ValueError(
                 "Invalid URL. Please provide a URL with a valid table name"
@@ -689,6 +691,7 @@ async def submit_transaction(
             self._client._deserialize,  # pylint: disable=protected-access
             self._client._config,  # pylint: disable=protected-access
             self.table_name,
+            is_cosmos_endpoint=self._cosmos_endpoint,
             **kwargs
         )
         for operation in operations:

sdk/tables/azure-data-tables/samples/sample_update_upsert_merge_entities.py

@@ -144,22 +144,22 @@ def update_entities(self):
                 insert_entity = table.upsert_entity(mode=UpdateMode.REPLACE, entity=entity1)
                 print("Inserted entity: {}".format(insert_entity))
 
-                created["text"] = "NewMarker"
+                created[u"text"] = u"NewMarker"
                 merged_entity = table.upsert_entity(mode=UpdateMode.MERGE, entity=entity)
                 print("Merged entity: {}".format(merged_entity))
                 # [END upsert_entity]
 
                 # [START update_entity]
                 # Update the entity
-                created["text"] = "NewMarker"
+                created[u"text"] = u"NewMarker"
                 table.update_entity(mode=UpdateMode.REPLACE, entity=created)
 
                 # Get the replaced entity
                 replaced = table.get_entity(partition_key=created["PartitionKey"], row_key=created["RowKey"])
                 print("Replaced entity: {}".format(replaced))
 
                 # Merge the entity
-                replaced["color"] = "Blue"
+                replaced[u"color"] = u"Blue"
                 table.update_entity(mode=UpdateMode.MERGE, entity=replaced)
 
                 # Get the merged entity

sdk/tables/azure-data-tables/tests/_shared/testcase.py

@@ -176,7 +176,7 @@ def _assert_default_entity(self, entity):
         assert entity["large"] == 933311100
         assert entity["Birthday"] == datetime(1973, 10, 4, tzinfo=tzutc())
         assert entity["birthday"] == datetime(1970, 10, 4, tzinfo=tzutc())
-        assert entity["binary"].value == b"binary"
+        assert entity["binary"] == b"binary"
         assert entity["other"] == 20
         assert entity["clsid"] == uuid.UUID("c9da6455-213d-42c9-9a79-3e9149a57833")
         assert entity.metadata["etag"]
@@ -197,7 +197,7 @@ def _assert_default_entity_json_full_metadata(self, entity, headers=None):
         assert entity["large"] == 933311100
         assert entity["Birthday"] == datetime(1973, 10, 4, tzinfo=tzutc())
         assert entity["birthday"] == datetime(1970, 10, 4, tzinfo=tzutc())
-        assert entity["binary"].value == b"binary"
+        assert entity["binary"] == b"binary"
         assert entity["other"] == 20
         assert entity["clsid"] == uuid.UUID("c9da6455-213d-42c9-9a79-3e9149a57833")
         assert entity.metadata["etag"]

sdk/tables/azure-data-tables/tests/test_table.py

@@ -468,13 +468,3 @@ def test_delete_table_invalid_name(self):
 
         assert "Table names must be alphanumeric, cannot begin with a number, and must be between 3-63 characters long.""" in str(
             excinfo)
-
-    def test_azurite_url(self):
-        account_url = "https://127.0.0.1:10002/my_account"
-        tsc = TableServiceClient(account_url, credential=self.credential)
-
-        assert tsc.account_name == "my_account"
-        assert tsc.url == "https://127.0.0.1:10002/my_account"
-        assert tsc._location_mode == "primary"
-        assert tsc.credential.named_key.key == self.credential.named_key.key
-        assert tsc.credential.named_key.name == self.credential.named_key.name

sdk/tables/azure-data-tables/tests/test_table_async.py

@@ -398,13 +398,3 @@ async def test_delete_table_invalid_name(self):
 
         assert "Table names must be alphanumeric, cannot begin with a number, and must be between 3-63 characters long.""" in str(
             excinfo)
-
-    def test_azurite_url(self):
-        account_url = "https://127.0.0.1:10002/my_account"
-        tsc = TableServiceClient(account_url, credential=self.credential)
-
-        assert tsc.account_name == "my_account"
-        assert tsc.url == "https://127.0.0.1:10002/my_account"
-        assert tsc._location_mode == "primary"
-        assert tsc.credential.named_key.key == self.credential.named_key.key
-        assert tsc.credential.named_key.name == self.credential.named_key.name