From acd555e5fa65590126accda8bd96c719a3491b1e Mon Sep 17 00:00:00 2001 From: Eyal Ezer Date: Mon, 20 May 2024 12:27:22 -0500 Subject: [PATCH] refactor: Migration of json utilities from core (#28522) Co-authored-by: Eyal Ezer --- .../annotation_layers/annotations/schemas.py | 4 +- superset/charts/data/api.py | 7 +- superset/common/query_object.py | 2 +- superset/dashboards/schemas.py | 4 +- superset/embedded/view.py | 7 +- superset/key_value/utils.py | 2 +- superset/models/core.py | 6 +- superset/models/dashboard.py | 4 +- superset/result_set.py | 4 +- superset/sql_lab.py | 11 +- superset/sqllab/api.py | 11 +- .../sqllab/execution_context_convertor.py | 31 +-- superset/utils/cache.py | 2 +- superset/utils/core.py | 146 +----------- superset/utils/json.py | 211 ++++++++++++++++++ superset/utils/schema.py | 4 +- superset/views/api.py | 6 +- superset/views/base.py | 40 ++-- superset/views/chart/views.py | 6 +- superset/views/core.py | 19 +- superset/views/dashboard/views.py | 7 +- superset/views/sql_lab/views.py | 4 +- superset/views/tags.py | 8 +- superset/viz.py | 10 +- tests/integration_tests/conftest.py | 2 +- tests/integration_tests/core_tests.py | 6 +- tests/integration_tests/sqllab_tests.py | 6 +- tests/integration_tests/utils_tests.py | 12 +- tests/unit_tests/utils/json_tests.py | 114 ++++++++++ tests/unit_tests/utils/test_core.py | 51 ----- 30 files changed, 431 insertions(+), 316 deletions(-) create mode 100644 superset/utils/json.py create mode 100644 tests/unit_tests/utils/json_tests.py diff --git a/superset/annotation_layers/annotations/schemas.py b/superset/annotation_layers/annotations/schemas.py index 7aa553d2a16bb..516241b9e3b44 100644 --- a/superset/annotation_layers/annotations/schemas.py +++ b/superset/annotation_layers/annotations/schemas.py @@ -20,7 +20,7 @@ from marshmallow.validate import Length from superset.exceptions import SupersetException -from superset.utils import core as utils +from superset.utils import json as json_utils openapi_spec_methods_override = { "get": {"get": {"summary": "Get an annotation layer"}}, @@ -51,7 +51,7 @@ def validate_json(value: Union[bytes, bytearray, str]) -> None: try: - utils.validate_json(value) + json_utils.validate_json(value) except SupersetException as ex: raise ValidationError("JSON not valid") from ex diff --git a/superset/charts/data/api.py b/superset/charts/data/api.py index 2e46eb2737a3a..e902d5738a65c 100644 --- a/superset/charts/data/api.py +++ b/superset/charts/data/api.py @@ -21,7 +21,6 @@ import logging from typing import Any, TYPE_CHECKING -import simplejson from flask import current_app, g, make_response, request, Response from flask_appbuilder.api import expose, protect from flask_babel import gettext as _ @@ -47,11 +46,11 @@ from superset.exceptions import QueryObjectValidationError from superset.extensions import event_logger from superset.models.sql_lab import Query +from superset.utils import json as json_utils from superset.utils.core import ( create_zip, DatasourceType, get_user_id, - json_int_dttm_ser, ) from superset.utils.decorators import logs_context from superset.views.base import CsvResponse, generate_download_headers, XlsxResponse @@ -396,9 +395,9 @@ def _process_data(query_data: Any) -> Any: ) if result_format == ChartDataResultFormat.JSON: - response_data = simplejson.dumps( + response_data = json_utils.dumps( {"result": result["queries"]}, - default=json_int_dttm_ser, + default=json_utils.json_int_dttm_ser, ignore_nan=True, ) resp = make_response(response_data, 200) diff --git a/superset/common/query_object.py b/superset/common/query_object.py index b183532ebf936..35b1d2974fcd3 100644 --- a/superset/common/query_object.py +++ b/superset/common/query_object.py @@ -43,10 +43,10 @@ get_column_names, get_metric_names, is_adhoc_metric, - json_int_dttm_ser, QueryObjectFilterClause, ) from superset.utils.hashing import md5_sha_from_dict +from superset.utils.json import json_int_dttm_ser if TYPE_CHECKING: from superset.connectors.sqla.models import BaseDatasource diff --git a/superset/dashboards/schemas.py b/superset/dashboards/schemas.py index 6a6debe397541..c90da33734a9c 100644 --- a/superset/dashboards/schemas.py +++ b/superset/dashboards/schemas.py @@ -24,7 +24,7 @@ from superset import security_manager from superset.exceptions import SupersetException from superset.tags.models import TagType -from superset.utils import core as utils +from superset.utils import json as json_utils get_delete_ids_schema = {"type": "array", "items": {"type": "integer"}} get_export_ids_schema = {"type": "array", "items": {"type": "integer"}} @@ -88,7 +88,7 @@ def validate_json(value: Union[bytes, bytearray, str]) -> None: try: - utils.validate_json(value) + json_utils.validate_json(value) except SupersetException as ex: raise ValidationError("JSON not valid") from ex diff --git a/superset/embedded/view.py b/superset/embedded/view.py index 462c6046faaf0..a260e33b5e553 100644 --- a/superset/embedded/view.py +++ b/superset/embedded/view.py @@ -14,7 +14,6 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -import json from typing import Callable from flask import abort, request @@ -25,7 +24,7 @@ from superset import event_logger, is_feature_enabled from superset.daos.dashboard import EmbeddedDashboardDAO from superset.superset_typing import FlaskResponse -from superset.utils import core as utils +from superset.utils import json as json_utils from superset.views.base import BaseSupersetView, common_bootstrap_payload @@ -87,7 +86,7 @@ def embedded( return self.render_template( "superset/spa.html", entry="embedded", - bootstrap_data=json.dumps( - bootstrap_data, default=utils.pessimistic_json_iso_dttm_ser + bootstrap_data=json_utils.dumps( + bootstrap_data, default=json_utils.pessimistic_json_iso_dttm_ser ), ) diff --git a/superset/key_value/utils.py b/superset/key_value/utils.py index 6b487c278c0d0..1a22cfaa747b3 100644 --- a/superset/key_value/utils.py +++ b/superset/key_value/utils.py @@ -26,7 +26,7 @@ from superset.key_value.exceptions import KeyValueParseKeyError from superset.key_value.types import KeyValueFilter, KeyValueResource -from superset.utils.core import json_dumps_w_dates +from superset.utils.json import json_dumps_w_dates HASHIDS_MIN_LENGTH = 11 diff --git a/superset/models/core.py b/superset/models/core.py index bbf30523420a6..2d7e57929526d 100755 --- a/superset/models/core.py +++ b/superset/models/core.py @@ -76,7 +76,7 @@ from superset.result_set import SupersetResultSet from superset.sql_parse import Table from superset.superset_typing import OAuth2ClientConfig, ResultSetColumnType -from superset.utils import cache as cache_util, core as utils +from superset.utils import cache as cache_util, core as utils, json as json_utils from superset.utils.backports import StrEnum from superset.utils.core import DatasourceName, get_username from superset.utils.oauth2 import get_oauth2_access_token @@ -601,7 +601,7 @@ def column_needs_conversion(df_series: pd.Series) -> bool: for col, coltype in df.dtypes.to_dict().items(): if coltype == numpy.object_ and column_needs_conversion(df[col]): - df[col] = df[col].apply(utils.json_dumps_w_dates) + df[col] = df[col].apply(json_utils.json_dumps_w_dates) return df @property @@ -957,7 +957,7 @@ def get_pk_constraint(self, table: Table) -> dict[str, Any]: def _convert(value: Any) -> Any: try: - return utils.base_json_conv(value) + return json_utils.base_json_conv(value) except TypeError: return None diff --git a/superset/models/dashboard.py b/superset/models/dashboard.py index 991d2d41a46f9..f478583246e21 100644 --- a/superset/models/dashboard.py +++ b/superset/models/dashboard.py @@ -51,7 +51,7 @@ from superset.tasks.thumbnails import cache_dashboard_thumbnail from superset.tasks.utils import get_current_user from superset.thumbnails.digest import get_dashboard_digest -from superset.utils import core as utils +from superset.utils import core as utils, json as json_utils metadata = Model.metadata # pylint: disable=no-member config = app.config @@ -372,7 +372,7 @@ def export_dashboards( # pylint: disable=too-many-locals return json.dumps( {"dashboards": copied_dashboards, "datasources": eager_datasources}, - cls=utils.DashboardEncoder, + cls=json_utils.DashboardEncoder, indent=4, ) diff --git a/superset/result_set.py b/superset/result_set.py index 061656720e4e2..38ca9453a59c4 100644 --- a/superset/result_set.py +++ b/superset/result_set.py @@ -28,7 +28,7 @@ from superset.db_engine_specs import BaseEngineSpec from superset.superset_typing import DbapiDescription, DbapiResult, ResultSetColumnType -from superset.utils import core as utils +from superset.utils import core as utils, json as json_utils from superset.utils.core import GenericDataType logger = logging.getLogger(__name__) @@ -61,7 +61,7 @@ def dedup(l: list[str], suffix: str = "__", case_sensitive: bool = True) -> list def stringify(obj: Any) -> str: - return json.dumps(obj, default=utils.json_iso_dttm_ser) + return json_utils.dumps(obj, default=json_utils.json_iso_dttm_ser) def stringify_values(array: NDArray[Any]) -> NDArray[Any]: diff --git a/superset/sql_lab.py b/superset/sql_lab.py index 3f8c1cc73709c..d2e6680fbb82a 100644 --- a/superset/sql_lab.py +++ b/superset/sql_lab.py @@ -24,7 +24,6 @@ import backoff import msgpack -import simplejson as json from celery.exceptions import SoftTimeLimitExceeded from flask_babel import gettext as __ @@ -59,8 +58,8 @@ ) from superset.sqllab.limiting_factor import LimitingFactor from superset.sqllab.utils import write_ipc_buffer +from superset.utils import json as json_utils from superset.utils.core import ( - json_iso_dttm_ser, override_user, QuerySource, zlib_compress, @@ -349,9 +348,13 @@ def _serialize_payload( ) -> Union[bytes, str]: logger.debug("Serializing to msgpack: %r", use_msgpack) if use_msgpack: - return msgpack.dumps(payload, default=json_iso_dttm_ser, use_bin_type=True) + return msgpack.dumps( + payload, default=json_utils.json_iso_dttm_ser, use_bin_type=True + ) - return json.dumps(payload, default=json_iso_dttm_ser, ignore_nan=True) + return json_utils.dumps( + payload, default=json_utils.json_iso_dttm_ser, ignore_nan=True + ) def _serialize_and_expand_data( diff --git a/superset/sqllab/api.py b/superset/sqllab/api.py index 6d6579c9498f9..df6506a5d2689 100644 --- a/superset/sqllab/api.py +++ b/superset/sqllab/api.py @@ -18,7 +18,6 @@ from typing import Any, cast, Optional from urllib import parse -import simplejson as json from flask import request, Response from flask_appbuilder import permission_name from flask_appbuilder.api import expose, protect, rison, safe @@ -62,7 +61,7 @@ from superset.sqllab.utils import bootstrap_sqllab_data from superset.sqllab.validators import CanAccessQueryValidatorImpl from superset.superset_typing import FlaskResponse -from superset.utils import core as utils +from superset.utils import core as utils, json as json_utils from superset.views.base import CsvResponse, generate_download_headers, json_success from superset.views.base_api import BaseSupersetApi, requires_json, statsd_metrics @@ -132,9 +131,9 @@ def get(self) -> Response: result = bootstrap_sqllab_data(user_id) return json_success( - json.dumps( + json_utils.dumps( {"result": result}, - default=utils.json_iso_dttm_ser, + default=json_utils.json_iso_dttm_ser, ignore_nan=True, ), 200, @@ -343,9 +342,9 @@ def get_results(self, **kwargs: Any) -> FlaskResponse: # Using pessimistic json serialization since some database drivers can return # unserializeable types at times - payload = json.dumps( + payload = json_utils.dumps( result, - default=utils.pessimistic_json_iso_dttm_ser, + default=json_utils.pessimistic_json_iso_dttm_ser, ignore_nan=True, ) return json_success(payload, 200) diff --git a/superset/sqllab/execution_context_convertor.py b/superset/sqllab/execution_context_convertor.py index 06bf3b847462c..9453faa152fc2 100644 --- a/superset/sqllab/execution_context_convertor.py +++ b/superset/sqllab/execution_context_convertor.py @@ -19,11 +19,9 @@ import logging from typing import Any, TYPE_CHECKING -import simplejson as json - -import superset.utils.core as utils from superset.sqllab.command_status import SqlJsonExecutionStatus from superset.sqllab.utils import apply_display_max_row_configuration_if_require +from superset.utils import json as json_utils logger = logging.getLogger(__name__) @@ -52,23 +50,16 @@ def set_payload( def serialize_payload(self) -> str: if self._exc_status == SqlJsonExecutionStatus.HAS_RESULTS: - sql_results = apply_display_max_row_configuration_if_require( - self.payload, self._max_row_in_display_configuration + return json_utils.dumps( + apply_display_max_row_configuration_if_require( + self.payload, self._max_row_in_display_configuration + ), + default=json_utils.pessimistic_json_iso_dttm_ser, + ignore_nan=True, ) - try: - return json.dumps( - sql_results, - default=utils.pessimistic_json_iso_dttm_ser, - ignore_nan=True, - ) - except UnicodeDecodeError: - return json.dumps( - sql_results, - default=utils.pessimistic_json_iso_dttm_ser, - ensure_ascii=False, - ignore_nan=True, - ) - return json.dumps( - {"query": self.payload}, default=utils.json_int_dttm_ser, ignore_nan=True + return json_utils.dumps( + {"query": self.payload}, + default=json_utils.json_int_dttm_ser, + ignore_nan=True, ) diff --git a/superset/utils/cache.py b/superset/utils/cache.py index 00216fc4b1d1f..15f334e128aee 100644 --- a/superset/utils/cache.py +++ b/superset/utils/cache.py @@ -30,8 +30,8 @@ from superset import db from superset.extensions import cache_manager from superset.models.cache import CacheKey -from superset.utils.core import json_int_dttm_ser from superset.utils.hashing import md5_sha_from_dict +from superset.utils.json import json_int_dttm_ser if TYPE_CHECKING: from superset.stats_logger import BaseStatsLogger diff --git a/superset/utils/core.py b/superset/utils/core.py index bf457bf5b3e43..37a11ea7ff592 100644 --- a/superset/utils/core.py +++ b/superset/utils/core.py @@ -21,9 +21,7 @@ import _thread import collections -import decimal import errno -import json import logging import os import platform @@ -40,7 +38,7 @@ from collections.abc import Iterable, Iterator, Sequence from contextlib import closing, contextmanager from dataclasses import dataclass -from datetime import date, datetime, time, timedelta +from datetime import timedelta from email.mime.application import MIMEApplication from email.mime.image import MIMEImage from email.mime.multipart import MIMEMultipart @@ -56,7 +54,6 @@ import markdown as md import nh3 -import numpy as np import pandas as pd import sqlalchemy as sa from cryptography.hazmat.backends import default_backend @@ -65,7 +62,6 @@ from flask_appbuilder import SQLA from flask_appbuilder.security.sqla.models import User from flask_babel import gettext as __ -from flask_babel.speaklater import LazyString from markupsafe import Markup from pandas.api.types import infer_dtype from pandas.core.dtypes.common import is_numeric_dtype @@ -103,7 +99,6 @@ from superset.utils.backports import StrEnum from superset.utils.database import get_example_database from superset.utils.date_parser import parse_human_timedelta -from superset.utils.dates import datetime_to_epoch, EPOCH from superset.utils.hashing import md5_sha_from_dict, md5_sha_from_str if TYPE_CHECKING: @@ -418,136 +413,6 @@ def cast_to_boolean(value: Any) -> bool | None: return False -class DashboardEncoder(json.JSONEncoder): - def __init__(self, *args: Any, **kwargs: Any) -> None: - super().__init__(*args, **kwargs) - self.sort_keys = True - - def default(self, o: Any) -> dict[Any, Any] | str: - if isinstance(o, uuid.UUID): - return str(o) - try: - vals = {k: v for k, v in o.__dict__.items() if k != "_sa_instance_state"} - return {f"__{o.__class__.__name__}__": vals} - except Exception: # pylint: disable=broad-except - if isinstance(o, datetime): - return {"__datetime__": o.replace(microsecond=0).isoformat()} - return json.JSONEncoder(sort_keys=True).default(o) - - -def format_timedelta(time_delta: timedelta) -> str: - """ - Ensures negative time deltas are easily interpreted by humans - - >>> td = timedelta(0) - timedelta(days=1, hours=5,minutes=6) - >>> str(td) - '-2 days, 18:54:00' - >>> format_timedelta(td) - '-1 day, 5:06:00' - """ - if time_delta < timedelta(0): - return "-" + str(abs(time_delta)) - - # Change this to format positive time deltas the way you want - return str(time_delta) - - -def base_json_conv(obj: Any) -> Any: - """ - Tries to convert additional types to JSON compatible forms. - - :param obj: The serializable object - :returns: The JSON compatible form - :raises TypeError: If the object cannot be serialized - :see: https://docs.python.org/3/library/json.html#encoders-and-decoders - """ - - if isinstance(obj, memoryview): - obj = obj.tobytes() - if isinstance(obj, np.int64): - return int(obj) - if isinstance(obj, np.bool_): - return bool(obj) - if isinstance(obj, np.ndarray): - return obj.tolist() - if isinstance(obj, set): - return list(obj) - if isinstance(obj, decimal.Decimal): - return float(obj) - if isinstance(obj, (uuid.UUID, time, LazyString)): - return str(obj) - if isinstance(obj, timedelta): - return format_timedelta(obj) - if isinstance(obj, bytes): - try: - return obj.decode("utf-8") - except Exception: # pylint: disable=broad-except - try: - return obj.decode("utf-16") - except Exception: # pylint: disable=broad-except - return "[bytes]" - - raise TypeError(f"Unserializable object {obj} of type {type(obj)}") - - -def json_iso_dttm_ser(obj: Any, pessimistic: bool = False) -> Any: - """ - A JSON serializer that deals with dates by serializing them to ISO 8601. - - >>> json.dumps({'dttm': datetime(1970, 1, 1)}, default=json_iso_dttm_ser) - '{"dttm": "1970-01-01T00:00:00"}' - - :param obj: The serializable object - :param pessimistic: Whether to be pessimistic regarding serialization - :returns: The JSON compatible form - :raises TypeError: If the non-pessimistic object cannot be serialized - """ - - if isinstance(obj, (datetime, date, pd.Timestamp)): - return obj.isoformat() - - try: - return base_json_conv(obj) - except TypeError as ex: - if pessimistic: - logger.error("Failed to serialize %s", obj) - return f"Unserializable [{type(obj)}]" - raise ex - - -def pessimistic_json_iso_dttm_ser(obj: Any) -> Any: - """Proxy to call json_iso_dttm_ser in a pessimistic way - - If one of object is not serializable to json, it will still succeed""" - return json_iso_dttm_ser(obj, pessimistic=True) - - -def json_int_dttm_ser(obj: Any) -> Any: - """ - A JSON serializer that deals with dates by serializing them to EPOCH. - - >>> json.dumps({'dttm': datetime(1970, 1, 1)}, default=json_int_dttm_ser) - '{"dttm": 0.0}' - - :param obj: The serializable object - :returns: The JSON compatible form - :raises TypeError: If the object cannot be serialized - """ - - if isinstance(obj, (datetime, pd.Timestamp)): - return datetime_to_epoch(obj) - - if isinstance(obj, date): - return (obj - EPOCH.date()).total_seconds() * 1000 - - return base_json_conv(obj) - - -def json_dumps_w_dates(payload: dict[Any, Any], sort_keys: bool = False) -> str: - """Dumps payload to JSON with Datetime objects properly converted""" - return json.dumps(payload, default=json_int_dttm_ser, sort_keys=sort_keys) - - def error_msg_from_exception(ex: Exception) -> str: """Translate exception into error message @@ -691,15 +556,6 @@ def get_datasource_full_name( return ".".join([f"[{part}]" for part in parts if part]) -def validate_json(obj: bytes | bytearray | str) -> None: - if obj: - try: - json.loads(obj) - except Exception as ex: - logger.error("JSON is not valid %s", str(ex), exc_info=True) - raise SupersetException("JSON is not valid") from ex - - class SigalrmTimeout: """ To be used in a ``with`` block and timeout its content. diff --git a/superset/utils/json.py b/superset/utils/json.py new file mode 100644 index 0000000000000..9068a84bb306a --- /dev/null +++ b/superset/utils/json.py @@ -0,0 +1,211 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +import decimal +import json +import logging +import uuid +from datetime import date, datetime, time, timedelta +from typing import Any, Callable, Optional, Union + +import numpy as np +import pandas as pd +import simplejson +from flask_babel.speaklater import LazyString + +from superset.exceptions import SupersetException +from superset.utils.dates import datetime_to_epoch, EPOCH + +logging.getLogger("MARKDOWN").setLevel(logging.INFO) +logger = logging.getLogger(__name__) + + +class DashboardEncoder(json.JSONEncoder): + def __init__(self, *args: Any, **kwargs: Any) -> None: + super().__init__(*args, **kwargs) + self.sort_keys = True + + def default(self, o: Any) -> Union[dict[Any, Any], str]: + if isinstance(o, uuid.UUID): + return str(o) + try: + vals = {k: v for k, v in o.__dict__.items() if k != "_sa_instance_state"} + return {f"__{o.__class__.__name__}__": vals} + except Exception: # pylint: disable=broad-except + if isinstance(o, datetime): + return {"__datetime__": o.replace(microsecond=0).isoformat()} + return json.JSONEncoder(sort_keys=True).default(o) + + +def format_timedelta(time_delta: timedelta) -> str: + """ + Ensures negative time deltas are easily interpreted by humans + + >>> td = timedelta(0) - timedelta(days=1, hours=5,minutes=6) + >>> str(td) + '-2 days, 18:54:00' + >>> format_timedelta(td) + '-1 day, 5:06:00' + """ + if time_delta < timedelta(0): + return "-" + str(abs(time_delta)) + + # Change this to format positive time deltas the way you want + return str(time_delta) + + +def base_json_conv(obj: Any) -> Any: + """ + Tries to convert additional types to JSON compatible forms. + + :param obj: The serializable object + :returns: The JSON compatible form + :raises TypeError: If the object cannot be serialized + :see: https://docs.python.org/3/library/json.html#encoders-and-decoders + """ + + if isinstance(obj, memoryview): + obj = obj.tobytes() + if isinstance(obj, np.int64): + return int(obj) + if isinstance(obj, np.bool_): + return bool(obj) + if isinstance(obj, np.ndarray): + return obj.tolist() + if isinstance(obj, set): + return list(obj) + if isinstance(obj, decimal.Decimal): + return float(obj) + if isinstance(obj, (uuid.UUID, time, LazyString)): + return str(obj) + if isinstance(obj, timedelta): + return format_timedelta(obj) + if isinstance(obj, bytes): + try: + return obj.decode("utf-8") + except Exception: # pylint: disable=broad-except + try: + return obj.decode("utf-16") + except Exception: # pylint: disable=broad-except + return "[bytes]" + + raise TypeError(f"Unserializable object {obj} of type {type(obj)}") + + +def json_iso_dttm_ser(obj: Any, pessimistic: bool = False) -> Any: + """ + A JSON serializer that deals with dates by serializing them to ISO 8601. + + >>> json.dumps({'dttm': datetime(1970, 1, 1)}, default=json_iso_dttm_ser) + '{"dttm": "1970-01-01T00:00:00"}' + + :param obj: The serializable object + :param pessimistic: Whether to be pessimistic regarding serialization + :returns: The JSON compatible form + :raises TypeError: If the non-pessimistic object cannot be serialized + """ + + if isinstance(obj, (datetime, date, pd.Timestamp)): + return obj.isoformat() + + try: + return base_json_conv(obj) + except TypeError as ex: + if pessimistic: + logger.error("Failed to serialize %s", obj) + return f"Unserializable [{type(obj)}]" + raise ex + + +def pessimistic_json_iso_dttm_ser(obj: Any) -> Any: + """Proxy to call json_iso_dttm_ser in a pessimistic way + + If one of object is not serializable to json, it will still succeed""" + return json_iso_dttm_ser(obj, pessimistic=True) + + +def json_int_dttm_ser(obj: Any) -> Any: + """ + A JSON serializer that deals with dates by serializing them to EPOCH. + + >>> json.dumps({'dttm': datetime(1970, 1, 1)}, default=json_int_dttm_ser) + '{"dttm": 0.0}' + + :param obj: The serializable object + :returns: The JSON compatible form + :raises TypeError: If the object cannot be serialized + """ + + if isinstance(obj, (datetime, pd.Timestamp)): + return datetime_to_epoch(obj) + + if isinstance(obj, date): + return (obj - EPOCH.date()).total_seconds() * 1000 + + return base_json_conv(obj) + + +def json_dumps_w_dates(payload: dict[Any, Any], sort_keys: bool = False) -> str: + """Dumps payload to JSON with Datetime objects properly converted""" + return dumps(payload, default=json_int_dttm_ser, sort_keys=sort_keys) + + +def validate_json(obj: Union[bytes, bytearray, str]) -> None: + """ + A JSON Validator that validates an object of bytes, bytes array or string + to be in valid JSON format + + :raises SupersetException: if obj is not serializable to JSON + :param obj: an object that should be parseable to JSON + """ + if obj: + try: + json.loads(obj) + except Exception as ex: + logger.error("JSON is not valid %s", str(ex), exc_info=True) + raise SupersetException("JSON is not valid") from ex + + +def dumps( + obj: Any, + default: Optional[Callable[[Any], Any]] = json_iso_dttm_ser, + ignore_nan: bool = True, + sort_keys: bool = False, +) -> str: + """ + Dumps object to compatible JSON format + + :param obj: The serializable object + :param default: function that should return a serializable version of obj + :param ignore_nan: when set to True nan values will be ignored + :param sort_keys: when set to True keys will be sorted + :returns: String object in the JSON compatible form + """ + + results_string = "" + try: + results_string = simplejson.dumps( + obj, default=default, ignore_nan=ignore_nan, sort_keys=sort_keys + ) + except UnicodeDecodeError: + results_string = simplejson.dumps( # type: ignore[call-overload] + obj, + default=default, + ignore_nan=ignore_nan, + sort_keys=sort_keys, + encoding=None, + ) + return results_string diff --git a/superset/utils/schema.py b/superset/utils/schema.py index c082ae017c211..7fd83d715b887 100644 --- a/superset/utils/schema.py +++ b/superset/utils/schema.py @@ -19,7 +19,7 @@ from marshmallow import validate, ValidationError from superset.exceptions import SupersetException -from superset.utils import core as utils +from superset.utils import json as json_utils class OneOfCaseInsensitive(validate.OneOf): @@ -49,6 +49,6 @@ def validate_json(value: Union[bytes, bytearray, str]) -> None: :param value: an object that should be parseable to JSON """ try: - utils.validate_json(value) + json_utils.validate_json(value) except SupersetException as ex: raise ValidationError("JSON not valid") from ex diff --git a/superset/views/api.py b/superset/views/api.py index d5dd0eca4c5ec..eef2b72bc73ca 100644 --- a/superset/views/api.py +++ b/superset/views/api.py @@ -33,7 +33,7 @@ from superset.legacy import update_time_range from superset.models.slice import Slice from superset.superset_typing import FlaskResponse -from superset.utils import core as utils +from superset.utils import json as json_utils from superset.utils.date_parser import get_since_until from superset.views.base import api, BaseSupersetView, handle_api_exception @@ -73,8 +73,8 @@ def query(self) -> FlaskResponse: query_context.raise_for_access() result = query_context.get_payload() payload_json = result["queries"] - return json.dumps( - payload_json, default=utils.json_int_dttm_ser, ignore_nan=True + return json_utils.dumps( + payload_json, default=json_utils.json_int_dttm_ser, ignore_nan=True ) @event_logger.log_this diff --git a/superset/views/base.py b/superset/views/base.py index 06e330ab64853..04583387bd8cb 100644 --- a/superset/views/base.py +++ b/superset/views/base.py @@ -25,7 +25,6 @@ from importlib.resources import files from typing import Any, Callable, cast -import simplejson as json import yaml from babel import Locale from flask import ( @@ -51,14 +50,13 @@ ) from flask_appbuilder.security.sqla.models import User from flask_appbuilder.widgets import ListWidget -from flask_babel import get_locale, gettext as __, lazy_gettext as _ +from flask_babel import get_locale, gettext as __ from flask_jwt_extended.exceptions import NoAuthorizationError from flask_wtf.csrf import CSRFError from flask_wtf.form import FlaskForm from sqlalchemy import exc from sqlalchemy.orm import Query from werkzeug.exceptions import HTTPException -from wtforms import Form from wtforms.fields.core import Field, UnboundField from superset import ( @@ -85,7 +83,7 @@ from superset.reports.models import ReportRecipientType from superset.superset_typing import FlaskResponse from superset.translations.utils import get_language_pack -from superset.utils import core as utils +from superset.utils import core as utils, json as json_utils from superset.utils.filters import get_dataset_access_filters from .utils import bootstrap_user_data @@ -155,7 +153,9 @@ def json_error_response( payload = payload or {"error": f"{msg}"} return Response( - json.dumps(payload, default=utils.json_iso_dttm_ser, ignore_nan=True), + json_utils.dumps( + payload, default=json_utils.json_iso_dttm_ser, ignore_nan=True + ), status=status, mimetype="application/json", ) @@ -170,7 +170,9 @@ def json_errors_response( payload["errors"] = [dataclasses.asdict(error) for error in errors] return Response( - json.dumps(payload, default=utils.json_iso_dttm_ser, ignore_nan=True), + json_utils.dumps( + payload, default=json_utils.json_iso_dttm_ser, ignore_nan=True + ), status=status, mimetype="application/json; charset=utf-8", ) @@ -292,7 +294,9 @@ class BaseSupersetView(BaseView): @staticmethod def json_response(obj: Any, status: int = 200) -> FlaskResponse: return Response( - json.dumps(obj, default=utils.json_int_dttm_ser, ignore_nan=True), + json_utils.dumps( + obj, default=json_utils.json_int_dttm_ser, ignore_nan=True + ), status=status, mimetype="application/json", ) @@ -308,8 +312,8 @@ def render_app_template( return self.render_template( "superset/spa.html", entry="spa", - bootstrap_data=json.dumps( - payload, default=utils.pessimistic_json_iso_dttm_ser + bootstrap_data=json_utils.dumps( + payload, default=json_utils.pessimistic_json_iso_dttm_ser ), ) @@ -543,9 +547,9 @@ def show_unexpected_exception(ex: Exception) -> FlaskResponse: @superset_app.context_processor def get_common_bootstrap_data() -> dict[str, Any]: def serialize_bootstrap_data() -> str: - return json.dumps( + return json_utils.dumps( {"common": common_bootstrap_payload()}, - default=utils.pessimistic_json_iso_dttm_ser, + default=json_utils.pessimistic_json_iso_dttm_ser, ) return {"bootstrap_data": serialize_bootstrap_data} @@ -626,8 +630,8 @@ def render_app_template(self) -> FlaskResponse: return self.render_template( "superset/spa.html", entry="spa", - bootstrap_data=json.dumps( - payload, default=utils.pessimistic_json_iso_dttm_ser + bootstrap_data=json_utils.dumps( + payload, default=json_utils.pessimistic_json_iso_dttm_ser ), ) @@ -640,16 +644,6 @@ class ListWidgetWithCheckboxes(ListWidget): # pylint: disable=too-few-public-me template = "superset/fab_overrides/list_with_checkboxes.html" -def validate_json(form: Form, field: Field) -> None: # pylint: disable=unused-argument - try: - json.loads(field.data) - except Exception as ex: - logger.exception(ex) - raise Exception( # pylint: disable=broad-exception-raised - _("json isn't valid") - ) from ex - - class YamlExportMixin: # pylint: disable=too-few-public-methods """ Override this if you want a dict response instead, with a certain key. diff --git a/superset/views/chart/views.py b/superset/views/chart/views.py index 00d591587207f..e8ea621902001 100644 --- a/superset/views/chart/views.py +++ b/superset/views/chart/views.py @@ -22,7 +22,7 @@ from superset.constants import MODEL_VIEW_RW_METHOD_PERMISSION_MAP, RouteMethod from superset.models.slice import Slice from superset.superset_typing import FlaskResponse -from superset.utils import core as utils +from superset.utils import json as json_utils from superset.views.base import DeleteMixin, DeprecateModelViewMixin, SupersetModelView from superset.views.chart.mixin import SliceMixin @@ -41,10 +41,10 @@ class SliceModelView( method_permission_name = MODEL_VIEW_RW_METHOD_PERMISSION_MAP def pre_add(self, item: "SliceModelView") -> None: - utils.validate_json(item.params) + json_utils.validate_json(item.params) def pre_update(self, item: "SliceModelView") -> None: - utils.validate_json(item.params) + json_utils.validate_json(item.params) security_manager.raise_for_ownership(item) def pre_delete(self, item: "SliceModelView") -> None: diff --git a/superset/views/core.py b/superset/views/core.py index 35ba3d042889b..83dcbf0a0df2f 100755 --- a/superset/views/core.py +++ b/superset/views/core.py @@ -71,10 +71,9 @@ from superset.models.sql_lab import Query from superset.models.user_attributes import UserAttribute from superset.superset_typing import FlaskResponse -from superset.utils import core as utils +from superset.utils import core as utils, json as json_utils from superset.utils.cache import etag_cache from superset.utils.core import ( - base_json_conv, DatasourceType, get_user_id, ReservedUrlParameters, @@ -576,8 +575,8 @@ def explore( return self.render_template( "superset/basic.html", - bootstrap_data=json.dumps( - bootstrap_data, default=utils.pessimistic_json_iso_dttm_ser + bootstrap_data=json_utils.dumps( + bootstrap_data, default=json_utils.pessimistic_json_iso_dttm_ser ), entry="explore", title=title, @@ -753,7 +752,7 @@ def warm_up_cache(self) -> FlaskResponse: ) return json_success( - json.dumps( + json_utils.dumps( [ { "slice_id" if key == "chart_id" else key: value @@ -765,7 +764,7 @@ def warm_up_cache(self) -> FlaskResponse: } for slc in slices ], - default=base_json_conv, + default=json_utils.base_json_conv, ), ) @@ -814,12 +813,12 @@ def dashboard( "superset/spa.html", entry="spa", title=dashboard.dashboard_title, # dashboard title is always visible - bootstrap_data=json.dumps( + bootstrap_data=json_utils.dumps( { "user": bootstrap_user_data(g.user, include_perms=True), "common": common_bootstrap_payload(), }, - default=utils.pessimistic_json_iso_dttm_ser, + default=json_utils.pessimistic_json_iso_dttm_ser, ), standalone_mode=ReservedUrlParameters.is_standalone_mode(), ) @@ -919,8 +918,8 @@ def welcome(self) -> FlaskResponse: return self.render_template( "superset/spa.html", entry="spa", - bootstrap_data=json.dumps( - payload, default=utils.pessimistic_json_iso_dttm_ser + bootstrap_data=json_utils.dumps( + payload, default=json_utils.pessimistic_json_iso_dttm_ser ), ) diff --git a/superset/views/dashboard/views.py b/superset/views/dashboard/views.py index 3b405fea0af5f..97735b9081b4b 100644 --- a/superset/views/dashboard/views.py +++ b/superset/views/dashboard/views.py @@ -15,7 +15,6 @@ # specific language governing permissions and limitations # under the License. import builtins -import json from typing import Callable, Union from flask import g, redirect, request, Response @@ -35,7 +34,7 @@ from superset.constants import MODEL_VIEW_RW_METHOD_PERMISSION_MAP, RouteMethod from superset.models.dashboard import Dashboard as DashboardModel from superset.superset_typing import FlaskResponse -from superset.utils import core as utils +from superset.utils import json as json_utils from superset.views.base import ( BaseSupersetView, common_bootstrap_payload, @@ -160,8 +159,8 @@ def embedded( return self.render_template( "superset/spa.html", entry="embedded", - bootstrap_data=json.dumps( - bootstrap_data, default=utils.pessimistic_json_iso_dttm_ser + bootstrap_data=json_utils.dumps( + bootstrap_data, default=json_utils.pessimistic_json_iso_dttm_ser ), ) diff --git a/superset/views/sql_lab/views.py b/superset/views/sql_lab/views.py index b481f191b8227..b2c05095914da 100644 --- a/superset/views/sql_lab/views.py +++ b/superset/views/sql_lab/views.py @@ -28,7 +28,7 @@ from superset.constants import MODEL_VIEW_RW_METHOD_PERMISSION_MAP, RouteMethod from superset.models.sql_lab import Query, SavedQuery, TableSchema, TabState from superset.superset_typing import FlaskResponse -from superset.utils import core as utils +from superset.utils import json as json_utils from superset.utils.core import get_user_id from superset.views.base import ( BaseSupersetView, @@ -140,7 +140,7 @@ def get(self, tab_state_id: int) -> FlaskResponse: if tab_state is None: return Response(status=404) return json_success( - json.dumps(tab_state.to_dict(), default=utils.json_iso_dttm_ser) + json.dumps(tab_state.to_dict(), default=json_utils.json_iso_dttm_ser) ) @has_access_api diff --git a/superset/views/tags.py b/superset/views/tags.py index 3b27510ff3ee6..f78b2b5edefda 100644 --- a/superset/views/tags.py +++ b/superset/views/tags.py @@ -17,16 +17,16 @@ import logging -import simplejson as json from flask_appbuilder import expose from flask_appbuilder.hooks import before_request from flask_appbuilder.models.sqla.interface import SQLAInterface from flask_appbuilder.security.decorators import has_access, has_access_api from werkzeug.exceptions import NotFound -from superset import db, is_feature_enabled, utils +from superset import db, is_feature_enabled from superset.superset_typing import FlaskResponse from superset.tags.models import Tag +from superset.utils import json as json_utils from superset.views.base import SupersetModelView from .base import BaseSupersetView, json_success @@ -74,4 +74,6 @@ def tags(self) -> FlaskResponse: } for obj in query ] - return json_success(json.dumps(results, default=utils.core.json_int_dttm_ser)) + return json_success( + json_utils.dumps(results, default=json_utils.json_int_dttm_ser) + ) diff --git a/superset/viz.py b/superset/viz.py index 5a4b323079169..b06bb42186b00 100644 --- a/superset/viz.py +++ b/superset/viz.py @@ -66,7 +66,7 @@ VizData, VizPayload, ) -from superset.utils import core as utils, csv +from superset.utils import core as utils, csv, json as json_utils from superset.utils.cache import set_and_log_cache from superset.utils.core import ( apply_max_row_limit, @@ -440,8 +440,8 @@ def cache_timeout(self) -> int: @deprecated(deprecated_in="3.0") def get_json(self) -> str: - return json.dumps( - self.get_payload(), default=utils.json_int_dttm_ser, ignore_nan=True + return json_utils.dumps( + self.get_payload(), default=json_utils.json_int_dttm_ser, ignore_nan=True ) @deprecated(deprecated_in="3.0") @@ -641,9 +641,9 @@ def get_df_payload( # pylint: disable=too-many-statements @staticmethod @deprecated(deprecated_in="3.0") def json_dumps(query_obj: Any, sort_keys: bool = False) -> str: - return json.dumps( + return json_utils.dumps( query_obj, - default=utils.json_int_dttm_ser, + default=json_utils.json_int_dttm_ser, ignore_nan=True, sort_keys=sort_keys, ) diff --git a/tests/integration_tests/conftest.py b/tests/integration_tests/conftest.py index 185f7d410fc4b..84c579310556b 100644 --- a/tests/integration_tests/conftest.py +++ b/tests/integration_tests/conftest.py @@ -29,8 +29,8 @@ from superset import db, security_manager from superset.extensions import feature_flag_manager -from superset.utils.core import json_dumps_w_dates from superset.utils.database import get_example_database, remove_database +from superset.utils.json import json_dumps_w_dates from tests.integration_tests.test_app import app, login if TYPE_CHECKING: diff --git a/tests/integration_tests/core_tests.py b/tests/integration_tests/core_tests.py index 90873d49b9f9e..ed683c95c5b91 100644 --- a/tests/integration_tests/core_tests.py +++ b/tests/integration_tests/core_tests.py @@ -51,7 +51,7 @@ from superset.models.sql_lab import Query from superset.result_set import SupersetResultSet from superset.sql_parse import Table -from superset.utils import core as utils +from superset.utils import core as utils, json as json_utils from superset.utils.core import backend from superset.utils.database import get_example_database from superset.views.database.views import DatabaseView @@ -502,7 +502,7 @@ def test_dataframe_timezone(self): results = SupersetResultSet(list(data), [["data"]], BaseEngineSpec) df = results.to_pandas_df() data = dataframe.df_to_records(df) - json_str = json.dumps(data, default=utils.pessimistic_json_iso_dttm_ser) + json_str = json.dumps(data, default=json_utils.pessimistic_json_iso_dttm_ser) self.assertDictEqual( data[0], {"data": pd.Timestamp("2017-11-18 21:53:00.219225+0100", tz=tz)} ) @@ -943,7 +943,7 @@ def test_feature_flag_serialization(self): encoded = json.dumps( {"FOO": lambda x: 1, "super": "set"}, - default=utils.pessimistic_json_iso_dttm_ser, + default=json_utils.pessimistic_json_iso_dttm_ser, ) html_string = ( html.escape(encoded, quote=False) diff --git a/tests/integration_tests/sqllab_tests.py b/tests/integration_tests/sqllab_tests.py index 96019c16cc500..8bbf7023a4044 100644 --- a/tests/integration_tests/sqllab_tests.py +++ b/tests/integration_tests/sqllab_tests.py @@ -44,10 +44,8 @@ apply_limit_if_exists, ) from superset.sql_parse import CtasMethod -from superset.utils.core import ( - backend, - datetime_to_epoch, # noqa: F401 -) +from superset.utils.core import backend +from superset.utils.json import datetime_to_epoch # noqa: F401 from superset.utils.database import get_example_database, get_main_database from tests.integration_tests.base_tests import SupersetTestCase diff --git a/tests/integration_tests/utils_tests.py b/tests/integration_tests/utils_tests.py index 67feb27141140..8916d30631066 100644 --- a/tests/integration_tests/utils_tests.py +++ b/tests/integration_tests/utils_tests.py @@ -45,20 +45,16 @@ from superset.models.dashboard import Dashboard # noqa: F401 from superset.models.slice import Slice # noqa: F401 from superset.utils.core import ( - base_json_conv, cast_to_num, convert_legacy_filters_into_adhoc, create_ssl_cert_file, DTTM_ALIAS, extract_dataframe_dtypes, - format_timedelta, GenericDataType, get_form_data_token, as_list, get_email_address_list, get_stacktrace, - json_int_dttm_ser, - json_iso_dttm_ser, merge_extra_filters, merge_extra_form_data, merge_request_params, @@ -66,11 +62,17 @@ parse_ssl_cert, parse_js_uri_path_item, split, - validate_json, zlib_compress, zlib_decompress, DateColumn, ) +from superset.utils.json import ( + base_json_conv, + format_timedelta, + json_int_dttm_ser, + json_iso_dttm_ser, + validate_json, +) from superset.utils.database import get_or_create_db from superset.utils import schema from superset.utils.hashing import md5_sha_from_str diff --git a/tests/unit_tests/utils/json_tests.py b/tests/unit_tests/utils/json_tests.py new file mode 100644 index 0000000000000..a3d0a6d9d9ed9 --- /dev/null +++ b/tests/unit_tests/utils/json_tests.py @@ -0,0 +1,114 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +import datetime +import json +from unittest.mock import MagicMock + +import pytest + +from superset.exceptions import SupersetException +from superset.utils.json import ( + dumps, + json_iso_dttm_ser, + pessimistic_json_iso_dttm_ser, + validate_json, +) + + +def test_json_dumps(): + data = { + "str": "some string", + "int": 123456789, + "float": 0.12345, + "bool": True, + } + json_str = dumps(data, default=pessimistic_json_iso_dttm_ser) + reloaded_data = json.loads(json_str) + assert reloaded_data["str"] == "some string" + assert reloaded_data["int"] == 123456789 + assert reloaded_data["float"] == 0.12345 + assert reloaded_data["bool"] is True + + +def test_json_dumps_encoding(): + data = { + "utf8": b"Hello World", + "utf16": b"\xff\xfeH\x00e\x00l\x00l\x00o\x00 \x00W\x00o\x00r\x00l\x00d\x00", + "bytes": b"\xff", + } + json_str = dumps(data, default=pessimistic_json_iso_dttm_ser) + reloaded_data = json.loads(json_str) + assert reloaded_data["utf8"] == "Hello World" + assert reloaded_data["utf16"] == "Hello World" + assert reloaded_data["bytes"] == "[bytes]" + + +def test_json_iso_dttm_ser(): + data = { + "datetime": datetime.datetime(2021, 1, 1, 0, 0, 0), + "date": datetime.date(2021, 1, 1), + } + json_str = json.dumps(data, default=json_iso_dttm_ser) + reloaded_data = json.loads(json_str) + assert reloaded_data["datetime"] == "2021-01-01T00:00:00" + assert reloaded_data["date"] == "2021-01-01" + + +def test_pessimistic_json_iso_dttm_ser(): + data = { + "datetime": datetime.datetime(2021, 1, 1, 0, 0, 0), + "date": datetime.date(2021, 1, 1), + "UNSERIALIZABLE": MagicMock(), + } + json_str = json.dumps(data, default=pessimistic_json_iso_dttm_ser) + reloaded_data = json.loads(json_str) + assert reloaded_data["datetime"] == "2021-01-01T00:00:00" + assert reloaded_data["date"] == "2021-01-01" + assert ( + reloaded_data["UNSERIALIZABLE"] + == "Unserializable []" + ) + + +def test_pessimistic_json_iso_dttm_ser_nonutf8(): + data = { + "INVALID_UTF8_BYTES": b"\xff", + } + assert isinstance(data["INVALID_UTF8_BYTES"], bytes) + json_str = json.dumps(data, default=pessimistic_json_iso_dttm_ser) + reloaded_data = json.loads(json_str) + assert reloaded_data["INVALID_UTF8_BYTES"] == "[bytes]" + + +def test_pessimistic_json_iso_dttm_ser_utf16(): + data = { + "VALID_UTF16_BYTES": b"\xff\xfeS0\x930k0a0o0\x16NLu", + } + assert isinstance(data["VALID_UTF16_BYTES"], bytes) + json_str = json.dumps(data, default=pessimistic_json_iso_dttm_ser) + reloaded_data = json.loads(json_str) + assert reloaded_data["VALID_UTF16_BYTES"] == "こんにちは世界" + + +def test_validate_json(): + valid = '{"a": 5, "b": [1, 5, ["g", "h"]]}' + assert validate_json(valid) is None + + invalid = '{"a": 5, "b": [1, 5, ["g", "h]]}' + with pytest.raises(SupersetException) as excinfo: + validate_json(invalid) + assert str(excinfo.value) == "JSON is not valid" diff --git a/tests/unit_tests/utils/test_core.py b/tests/unit_tests/utils/test_core.py index c9ace0d3f6820..2ebec87c2a9c2 100644 --- a/tests/unit_tests/utils/test_core.py +++ b/tests/unit_tests/utils/test_core.py @@ -14,8 +14,6 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -import datetime -import json import os from dataclasses import dataclass from typing import Any, Optional @@ -33,10 +31,8 @@ generic_find_fk_constraint_name, get_datasource_full_name, is_test, - json_iso_dttm_ser, normalize_dttm_col, parse_boolean_string, - pessimistic_json_iso_dttm_ser, QueryObjectFilterClause, remove_extra_adhoc_filters, ) @@ -400,50 +396,3 @@ def test_get_datasource_full_name(): get_datasource_full_name("db", "table", "catalog", None) == "[db].[catalog].[table]" ) - - -def test_json_iso_dttm_ser(): - data = { - "datetime": datetime.datetime(2021, 1, 1, 0, 0, 0), - "date": datetime.date(2021, 1, 1), - } - json_str = json.dumps(data, default=json_iso_dttm_ser) - reloaded_data = json.loads(json_str) - assert reloaded_data["datetime"] == "2021-01-01T00:00:00" - assert reloaded_data["date"] == "2021-01-01" - - -def test_pessimistic_json_iso_dttm_ser(): - data = { - "datetime": datetime.datetime(2021, 1, 1, 0, 0, 0), - "date": datetime.date(2021, 1, 1), - "UNSERIALIZABLE": MagicMock(), - } - json_str = json.dumps(data, default=pessimistic_json_iso_dttm_ser) - reloaded_data = json.loads(json_str) - assert reloaded_data["datetime"] == "2021-01-01T00:00:00" - assert reloaded_data["date"] == "2021-01-01" - assert ( - reloaded_data["UNSERIALIZABLE"] - == "Unserializable []" - ) - - -def test_pessimistic_json_iso_dttm_ser_nonutf8(): - data = { - "INVALID_UTF8_BYTES": b"\xff", - } - assert isinstance(data["INVALID_UTF8_BYTES"], bytes) - json_str = json.dumps(data, default=pessimistic_json_iso_dttm_ser) - reloaded_data = json.loads(json_str) - assert reloaded_data["INVALID_UTF8_BYTES"] == "[bytes]" - - -def test_pessimistic_json_iso_dttm_ser_utf16(): - data = { - "VALID_UTF16_BYTES": b"\xff\xfeS0\x930k0a0o0\x16NLu", - } - assert isinstance(data["VALID_UTF16_BYTES"], bytes) - json_str = json.dumps(data, default=pessimistic_json_iso_dttm_ser) - reloaded_data = json.loads(json_str) - assert reloaded_data["VALID_UTF16_BYTES"] == "こんにちは世界"