test: removed unicode_test example from unit tests (apache#11131)

* Removed depemdency to unicode example in tests config.

* Added common methods for creating dashboards for tests.

* Added fixtures to all tests which were using unicode example.

* Added cleanup for unicode_test table

* Removed unnecessary fixture parts of unicode dashboard tests

* Parametrized creating slice for tests

* Moved fixtures for unicode test to separate file and refactored to several methods. Added param types and return types.

* Cleandup after fix

* Changed variable names to more readable

* Added cleanup for dashboards and slices

* Applied unicode fixture to charts api tests

* Update schema variable to dtype in dashboard utils

Co-authored-by: Ville Brofeldt <33317356+villebro@users.noreply.github.com>

* Changed variable schema to dtype in dashboards. Replaced accessing first element with one_or_none

Co-authored-by: Ville Brofeldt <33317356+villebro@users.noreply.github.com>

tests/charts/api_tests.py

@@ -28,6 +28,7 @@
 from sqlalchemy.sql import func
 
 from superset.utils.core import get_example_database
+from tests.fixtures.unicode_dashboard import load_unicode_dashboard_with_slice
 from tests.test_app import app
 from superset.connectors.connector_registry import ConnectorRegistry
 from superset.extensions import db, security_manager
@@ -567,6 +568,7 @@ def test_get_chart_no_data_access(self):
         rv = self.client.get(uri)
         self.assertEqual(rv.status_code, 404)
 
+    @pytest.mark.usefixtures("load_unicode_dashboard_with_slice")
     def test_get_charts(self):
         """
         Chart API: Test get charts
@@ -733,6 +735,7 @@ def test_get_charts_favorite_filter(self):
         assert rv.status_code == 200
         assert len(expected_models) == data["count"]
 
+    @pytest.mark.usefixtures("load_unicode_dashboard_with_slice")
     def test_get_charts_page(self):
         """
         Chart API: Test get charts filter

tests/conftest.py

@@ -45,7 +45,6 @@ def setup_sample_data() -> Any:
         examples.load_energy(sample=True)
         examples.load_world_bank_health_n_pop(sample=True)
         examples.load_birth_names(sample=True)
-        examples.load_unicode_test_data(sample=True)
 
     yield
 
@@ -54,7 +53,6 @@ def setup_sample_data() -> Any:
         engine.execute("DROP TABLE energy_usage")
         engine.execute("DROP TABLE wb_health_population")
         engine.execute("DROP TABLE birth_names")
-        engine.execute("DROP TABLE unicode_test")
 
         # drop sqlachemy tables
 

tests/dashboard_utils.py

@@ -0,0 +1,85 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""Utils to provide dashboards for tests"""
+
+import json
+from typing import Any, Dict
+
+from pandas import DataFrame
+
+from superset import ConnectorRegistry, db
+from superset.connectors.sqla.models import SqlaTable
+from superset.models.core import Database
+from superset.models.dashboard import Dashboard
+from superset.models.slice import Slice
+
+
+def create_table_for_dashboard(
+    df: DataFrame, table_name: str, database: Database, dtype: Dict[str, Any]
+) -> SqlaTable:
+    df.to_sql(
+        table_name,
+        database.get_sqla_engine(),
+        if_exists="replace",
+        chunksize=500,
+        dtype=dtype,
+        index=False,
+        method="multi",
+    )
+
+    table_source = ConnectorRegistry.sources["table"]
+    table = (
+        db.session.query(table_source).filter_by(table_name=table_name).one_or_none()
+    )
+    if not table:
+        table = table_source(table_name=table_name)
+    table.database = database
+    db.session.merge(table)
+    db.session.commit()
+
+    return table
+
+
+def create_slice(
+    title: str, viz_type: str, table: SqlaTable, slices_dict: Dict[str, str]
+) -> Slice:
+    return Slice(
+        slice_name=title,
+        viz_type=viz_type,
+        datasource_type="table",
+        datasource_id=table.id,
+        params=json.dumps(slices_dict, indent=4, sort_keys=True),
+    )
+
+
+def create_dashboard(slug: str, title: str, position: str, slice: Slice) -> Dashboard:
+    dash = db.session.query(Dashboard).filter_by(slug=slug).one_or_none()
+
+    if not dash:
+        dash = Dashboard()
+    dash.dashboard_title = title
+    if position is not None:
+        js = position
+        pos = json.loads(js)
+        dash.position_json = json.dumps(pos, indent=4)
+    dash.slug = slug
+    if slice is not None:
+        dash.slices = [slice]
+    db.session.merge(dash)
+    db.session.commit()
+
+    return dash

tests/databases/api_tests.py

@@ -16,17 +16,28 @@
 # under the License.
 # isort:skip_file
 """Unit tests for Superset"""
+import datetime
 import json
 
+import pandas as pd
 import prison
+import pytest
+import random
+
+from sqlalchemy import String, Date, Float
 from sqlalchemy.sql import func
 
-from superset import db, security_manager
+from superset import db, security_manager, ConnectorRegistry
 from superset.connectors.sqla.models import SqlaTable
 from superset.models.core import Database
 from superset.utils.core import get_example_database, get_main_database
 from tests.base_tests import SupersetTestCase
+from tests.dashboard_utils import (
+    create_table_for_dashboard,
+    create_dashboard,
+)
 from tests.fixtures.certificates import ssl_certificate
+from tests.fixtures.unicode_dashboard import load_unicode_dashboard_with_position
 from tests.test_app import app
 
 
@@ -758,6 +769,7 @@ def test_test_connection_unsafe_uri(self):
         }
         self.assertEqual(response, expected_response)
 
+    @pytest.mark.usefixtures("load_unicode_dashboard_with_position")
     def test_get_database_related_objects(self):
         """
         Database API: Test get chart and dashboard count related to a database

tests/fixtures/unicode_dashboard.py

@@ -0,0 +1,110 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+import pandas as pd
+import pytest
+from pandas import DataFrame
+from sqlalchemy import String
+
+from superset import db
+from superset.connectors.sqla.models import SqlaTable
+from superset.models.dashboard import Dashboard
+from superset.models.slice import Slice
+from superset.utils.core import get_example_database
+from tests.dashboard_utils import (
+    create_dashboard,
+    create_slice,
+    create_table_for_dashboard,
+)
+from tests.test_app import app
+
+
+@pytest.fixture()
+def load_unicode_dashboard_with_slice():
+    table_name = "unicode_test"
+    slice_name = "Unicode Cloud"
+    df = _get_dataframe()
+    with app.app_context():
+        dash = _create_unicode_dashboard(df, table_name, slice_name, None)
+        yield
+
+        _cleanup(dash, slice_name)
+
+
+@pytest.fixture()
+def load_unicode_dashboard_with_position():
+    table_name = "unicode_test"
+    slice_name = "Unicode Cloud"
+    df = _get_dataframe()
+    position = "{}"
+    with app.app_context():
+        dash = _create_unicode_dashboard(df, table_name, slice_name, position)
+        yield
+        _cleanup(dash, slice_name)
+
+
+def _get_dataframe():
+    data = _get_unicode_data()
+    return pd.DataFrame.from_dict(data)
+
+
+def _get_unicode_data():
+    return [
+        {"phrase": "Под"},
+        {"phrase": "řšž"},
+        {"phrase": "視野無限廣"},
+        {"phrase": "微風"},
+        {"phrase": "中国智造"},
+        {"phrase": "æøå"},
+        {"phrase": "ëœéè"},
+        {"phrase": "いろはにほ"},
+    ]
+
+
+def _create_unicode_dashboard(
+    df: DataFrame, table_name: str, slice_title: str, position: str
+) -> Dashboard:
+    database = get_example_database()
+    dtype = {
+        "phrase": String(500),
+    }
+    table = create_table_for_dashboard(df, table_name, database, dtype)
+    table.fetch_metadata()
+
+    if slice_title:
+        slice = _create_and_commit_unicode_slice(table, slice_title)
+
+    return create_dashboard("unicode-test", "Unicode Test", position, slice)
+
+
+def _create_and_commit_unicode_slice(table: SqlaTable, title: str):
+    slice = create_slice(title, "word_cloud", table, {})
+    o = db.session.query(Slice).filter_by(slice_name=slice.slice_name).one_or_none()
+    if o:
+        db.session.delete(o)
+    db.session.add(slice)
+    db.session.commit()
+    return slice
+
+
+def _cleanup(dash: Dashboard, slice_name: str) -> None:
+    engine = get_example_database().get_sqla_engine()
+    engine.execute("DROP TABLE IF EXISTS unicode_test")
+    db.session.delete(dash)
+    if slice_name:
+        slice = db.session.query(Slice).filter_by(slice_name=slice_name).one_or_none()
+        db.session.delete(slice)
+    db.session.commit()

tests/security_tests.py

@@ -15,16 +15,21 @@
 # specific language governing permissions and limitations
 # under the License.
 # isort:skip_file
+import datetime
 import inspect
 import re
 import unittest
 from unittest.mock import Mock, patch
 
+import pandas as pd
 import prison
+import pytest
+import random
+
 from flask import current_app, g
+from sqlalchemy import Float, Date, String
 
-import tests.test_app
-from superset import app, appbuilder, db, security_manager, viz
+from superset import app, appbuilder, db, security_manager, viz, ConnectorRegistry
 from superset.connectors.druid.models import DruidCluster, DruidDatasource
 from superset.connectors.sqla.models import RowLevelSecurityFilter, SqlaTable
 from superset.errors import ErrorLevel, SupersetError, SupersetErrorType
@@ -35,6 +40,12 @@
 from superset.utils.core import get_example_database
 
 from .base_tests import SupersetTestCase
+from .dashboard_utils import (
+    create_table_for_dashboard,
+    create_slice,
+    create_dashboard,
+)
+from .fixtures.unicode_dashboard import load_unicode_dashboard_with_slice
 
 
 def get_perm_tuples(role_name):
@@ -1122,6 +1133,7 @@ def test_rls_filter_doesnt_alter_energy_query(self):
         assert tbl.get_extra_cache_keys(self.query_obj) == []
         assert "value > 1" not in sql
 
+    @pytest.mark.usefixtures("load_unicode_dashboard_with_slice")
     def test_multiple_table_filter_alters_another_tables_query(self):
         g.user = self.get_user(
             username="alpha"

tests/strategy_tests.py

@@ -16,11 +16,20 @@
 # under the License.
 # isort:skip_file
 """Unit tests for Superset cache warmup"""
+import datetime
 import json
 from unittest.mock import MagicMock
 
-import tests.test_app
+from sqlalchemy import String, Date, Float
+
+import pytest
+import pandas as pd
+
+from superset.models.slice import Slice
+from superset.utils.core import get_example_database
+
 from superset import db
+
 from superset.models.core import Log
 from superset.models.tags import get_tag, ObjectTypes, TaggedObject, TagTypes
 from superset.tasks.cache import (
@@ -30,6 +39,8 @@
 )
 
 from .base_tests import SupersetTestCase
+from .dashboard_utils import create_dashboard, create_slice, create_table_for_dashboard
+from .fixtures.unicode_dashboard import load_unicode_dashboard_with_slice
 
 URL_PREFIX = "http://0.0.0.0:8081"
 
@@ -193,6 +204,7 @@ def reset_tag(self, tag):
                 db.session.delete(o)
             db.session.commit()
 
+    @pytest.mark.usefixtures("load_unicode_dashboard_with_slice")
     def test_dashboard_tags(self):
         tag1 = get_tag("tag1", db.session, TagTypes.custom)
         # delete first to make test idempotent