Add ability to override default serialization (elastic#2018)

* utils: add simplejson based JSON handling

Add an alternative json serialization and deserialization implementation
using simplejson configured to be JSON compliant by converting floats like
nan, +inf and -inf to null .
The idea is to permit users to use a different serialization if they
need a different behaviour.

Refs elastic#1886

* client: fix overriding of transport json serializer

We get a string from config so need to import it.

Author: xrmx

elasticapm/base.py

@@ -155,7 +155,8 @@ def __init__(self, config=None, **inline) -> None:
             "processors": self.load_processors(),
         }
         if config.transport_json_serializer:
-            transport_kwargs["json_serializer"] = config.transport_json_serializer
+            json_serializer_func = import_string(config.transport_json_serializer)
+            transport_kwargs["json_serializer"] = json_serializer_func
 
         self._api_endpoint_url = urllib.parse.urljoin(
             self.config.server_url if self.config.server_url.endswith("/") else self.config.server_url + "/",

elasticapm/utils/json_encoder.py

@@ -31,13 +31,9 @@
 
 import datetime
 import decimal
+import json
 import uuid
 
-try:
-    import json
-except ImportError:
-    import simplejson as json
-
 
 class BetterJSONEncoder(json.JSONEncoder):
     ENCODERS = {

elasticapm/utils/simplejson_encoder.py

@@ -0,0 +1,58 @@
+#  BSD 3-Clause License
+#
+#  Copyright (c) 2012, the Sentry Team, see AUTHORS for more details
+#  Copyright (c) 2019, Elasticsearch BV
+#  All rights reserved.
+#
+#  Redistribution and use in source and binary forms, with or without
+#  modification, are permitted provided that the following conditions are met:
+#
+#  * Redistributions of source code must retain the above copyright notice, this
+#    list of conditions and the following disclaimer.
+#
+#  * Redistributions in binary form must reproduce the above copyright notice,
+#    this list of conditions and the following disclaimer in the documentation
+#    and/or other materials provided with the distribution.
+#
+#  * Neither the name of the copyright holder nor the names of its
+#    contributors may be used to endorse or promote products derived from
+#    this software without specific prior written permission.
+#
+#  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+#  AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+#  IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+#  DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+#  FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+#  DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+#  SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+#  CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+#  OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+
+
+import simplejson as json
+
+from elasticapm.utils.json_encoder import BetterJSONEncoder
+
+
+class BetterSimpleJSONEncoder(json.JSONEncoder):
+    ENCODERS = BetterJSONEncoder.ENCODERS
+
+    def default(self, obj):
+        if type(obj) in self.ENCODERS:
+            return self.ENCODERS[type(obj)](obj)
+        try:
+            return super(BetterSimpleJSONEncoder, self).default(obj)
+        except TypeError:
+            return str(obj)
+
+
+def better_decoder(data):
+    return data
+
+
+def dumps(value, **kwargs):
+    return json.dumps(value, cls=BetterSimpleJSONEncoder, ignore_nan=True, **kwargs)
+
+
+def loads(value, **kwargs):
+    return json.loads(value, object_hook=better_decoder)

tests/client/client_tests.py

@@ -48,6 +48,11 @@
 import elasticapm
 from elasticapm.base import Client
 from elasticapm.conf.constants import ERROR
+
+try:
+    from elasticapm.utils.simplejson_encoder import dumps as simplejson_dumps
+except ImportError:
+    simplejson_dumps = None
 from tests.fixtures import DummyTransport, TempStoreClient
 from tests.utils import assert_any_record_contains
 
@@ -228,6 +233,14 @@ def test_custom_transport(elasticapm_client):
     assert isinstance(elasticapm_client._transport, DummyTransport)
 
 
+@pytest.mark.skipIf(simplejson_dumps is None)
+@pytest.mark.parametrize(
+    "elasticapm_client", [{"transport_json_serializer": "elasticapm.utils.simplejson_encoder.dumps"}], indirect=True
+)
+def test_custom_transport_json_serializer(elasticapm_client):
+    assert elasticapm_client._transport._json_serializer == simplejson_dumps
+
+
 @pytest.mark.parametrize("elasticapm_client", [{"processors": []}], indirect=True)
 def test_empty_processor_list(elasticapm_client):
     assert elasticapm_client.processors == []

tests/requirements/reqs-base.txt

@@ -30,6 +30,7 @@ pytz
 ecs_logging
 structlog
 wrapt>=1.14.1,<1.15.0
+simplejson
 
 pytest-asyncio==0.21.0 ; python_version >= '3.7'
 asynctest==0.13.0 ; python_version >= '3.7'

tests/utils/json_utils/tests.py

@@ -36,6 +36,8 @@
 import decimal
 import uuid
 
+import pytest
+
 from elasticapm.utils import json_encoder as json
 
 
@@ -69,6 +71,11 @@ def test_decimal():
     assert json.dumps(res) == "1.0"
 
 
+@pytest.mark.parametrize("res", [float("nan"), float("+inf"), float("-inf")])
+def test_float_invalid_json(res):
+    assert json.dumps(res) != "null"
+
+
 def test_unsupported():
     res = object()
     assert json.dumps(res).startswith('"<object object at')

tests/utils/json_utils/tests_simplejson.py

@@ -0,0 +1,86 @@
+# -*- coding: utf-8 -*-
+
+#  BSD 3-Clause License
+#
+#  Copyright (c) 2019, Elasticsearch BV
+#  All rights reserved.
+#
+#  Redistribution and use in source and binary forms, with or without
+#  modification, are permitted provided that the following conditions are met:
+#
+#  * Redistributions of source code must retain the above copyright notice, this
+#    list of conditions and the following disclaimer.
+#
+#  * Redistributions in binary form must reproduce the above copyright notice,
+#    this list of conditions and the following disclaimer in the documentation
+#    and/or other materials provided with the distribution.
+#
+#  * Neither the name of the copyright holder nor the names of its
+#    contributors may be used to endorse or promote products derived from
+#    this software without specific prior written permission.
+#
+#  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+#  AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+#  IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+#  DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+#  FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+#  DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+#  SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+#  CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+#  OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+#  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+import datetime
+import decimal
+import uuid
+
+import pytest
+
+simplejson = pytest.importorskip("simplejson")
+
+from elasticapm.utils import simplejson_encoder as json
+
+
+def test_uuid():
+    res = uuid.uuid4()
+    assert json.dumps(res) == '"%s"' % res.hex
+
+
+def test_datetime():
+    res = datetime.datetime(day=1, month=1, year=2011, hour=1, minute=1, second=1)
+    assert json.dumps(res) == '"2011-01-01T01:01:01.000000Z"'
+
+
+def test_set():
+    res = set(["foo", "bar"])
+    assert json.dumps(res) in ('["foo", "bar"]', '["bar", "foo"]')
+
+
+def test_frozenset():
+    res = frozenset(["foo", "bar"])
+    assert json.dumps(res) in ('["foo", "bar"]', '["bar", "foo"]')
+
+
+def test_bytes():
+    res = bytes("foobar", encoding="ascii")
+    assert json.dumps(res) == '"foobar"'
+
+
+def test_decimal():
+    res = decimal.Decimal("1.0")
+    assert json.dumps(res) == "1.0"
+
+
+@pytest.mark.parametrize("res", [float("nan"), float("+inf"), float("-inf")])
+def test_float_invalid_json(res):
+    assert json.dumps(res) == "null"
+
+
+def test_float():
+    res = 1.0
+    assert json.dumps(res) == "1.0"
+
+
+def test_unsupported():
+    res = object()
+    assert json.dumps(res).startswith('"<object object at')