diff --git a/superset/charts/schemas.py b/superset/charts/schemas.py index 80fcb1eea8830..a887ffd13c45d 100644 --- a/superset/charts/schemas.py +++ b/superset/charts/schemas.py @@ -1304,6 +1304,8 @@ class ImportV1ChartSchema(Schema): uuid = fields.UUID(required=True) version = fields.String(required=True) dataset_uuid = fields.UUID(required=True) + is_managed_externally = fields.Boolean(allow_none=True, default=False) + external_url = fields.String(allow_none=True) CHART_SCHEMAS = ( diff --git a/superset/connectors/base/models.py b/superset/connectors/base/models.py index 939a1fc1c7c27..9aacb0dc8c641 100644 --- a/superset/connectors/base/models.py +++ b/superset/connectors/base/models.py @@ -117,6 +117,8 @@ def name(self) -> str: owners: List[User] update_from_object_fields: List[str] + extra_import_fields = ["is_managed_externally", "external_url"] + @property def kind(self) -> DatasourceKind: return DatasourceKind.VIRTUAL if self.sql else DatasourceKind.PHYSICAL diff --git a/superset/dashboards/schemas.py b/superset/dashboards/schemas.py index 71f9bc41261e6..09119db9e1342 100644 --- a/superset/dashboards/schemas.py +++ b/superset/dashboards/schemas.py @@ -305,3 +305,5 @@ class ImportV1DashboardSchema(Schema): position = fields.Dict() metadata = fields.Dict() version = fields.String(required=True) + is_managed_externally = fields.Boolean(allow_none=True, default=False) + external_url = fields.String(allow_none=True) diff --git a/superset/databases/schemas.py b/superset/databases/schemas.py index ee9b4a62d2694..554a0f97cf3df 100644 --- a/superset/databases/schemas.py +++ b/superset/databases/schemas.py @@ -623,6 +623,8 @@ def fix_allow_csv_upload( extra = fields.Nested(ImportV1DatabaseExtraSchema) uuid = fields.UUID(required=True) version = fields.String(required=True) + is_managed_externally = fields.Boolean(allow_none=True, default=False) + external_url = fields.String(allow_none=True) # pylint: disable=no-self-use, unused-argument @validates_schema diff --git a/superset/datasets/schemas.py b/superset/datasets/schemas.py index d85ce6c8cc753..8e4ded92d7057 100644 --- a/superset/datasets/schemas.py +++ b/superset/datasets/schemas.py @@ -216,6 +216,8 @@ def fix_extra(self, data: Dict[str, Any], **kwargs: Any) -> Dict[str, Any]: version = fields.String(required=True) database_uuid = fields.UUID(required=True) data = fields.URL() + is_managed_externally = fields.Boolean(allow_none=True, default=False) + external_url = fields.String(allow_none=True) class DatasetSchema(SQLAlchemyAutoSchema): diff --git a/superset/models/core.py b/superset/models/core.py index 51f0731009950..450ddc965e12f 100755 --- a/superset/models/core.py +++ b/superset/models/core.py @@ -165,7 +165,7 @@ class Database( "allow_file_upload", "extra", ] - extra_import_fields = ["password"] + extra_import_fields = ["password", "is_managed_externally", "external_url"] export_children = ["tables"] def __repr__(self) -> str: diff --git a/superset/models/dashboard.py b/superset/models/dashboard.py index 55bf909b97353..135f36a3fa1b1 100644 --- a/superset/models/dashboard.py +++ b/superset/models/dashboard.py @@ -163,6 +163,7 @@ class Dashboard(Model, AuditMixinNullable, ImportExportMixin): "css", "slug", ] + extra_import_fields = ["is_managed_externally", "external_url"] def __repr__(self) -> str: return f"Dashboard<{self.id or self.slug}>" diff --git a/superset/models/slice.py b/superset/models/slice.py index 9c22efec6d4db..80b11c294f48e 100644 --- a/superset/models/slice.py +++ b/superset/models/slice.py @@ -119,6 +119,7 @@ class Slice( # pylint: disable=too-many-public-methods "cache_timeout", ] export_parent = "table" + extra_import_fields = ["is_managed_externally", "external_url"] def __repr__(self) -> str: return self.slice_name or str(self.id) diff --git a/tests/integration_tests/fixtures/importexport.py b/tests/integration_tests/fixtures/importexport.py index 3ec3d8f0935a4..996c77f12e0f2 100644 --- a/tests/integration_tests/fixtures/importexport.py +++ b/tests/integration_tests/fixtures/importexport.py @@ -449,7 +449,7 @@ "dataset_uuid": "10808100-158b-42c4-842e-f32b99d88dfb", } -dashboard_config = { +dashboard_config: Dict[str, Any] = { "dashboard_title": "Test dash", "description": None, "css": "", diff --git a/tests/unit_tests/charts/commands/__init__.py b/tests/unit_tests/charts/commands/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/tests/unit_tests/charts/commands/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/tests/unit_tests/charts/commands/importers/__init__.py b/tests/unit_tests/charts/commands/importers/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/tests/unit_tests/charts/commands/importers/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/tests/unit_tests/charts/commands/importers/v1/__init__.py b/tests/unit_tests/charts/commands/importers/v1/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/tests/unit_tests/charts/commands/importers/v1/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/tests/unit_tests/charts/commands/importers/v1/import_test.py b/tests/unit_tests/charts/commands/importers/v1/import_test.py new file mode 100644 index 0000000000000..e8687036394cb --- /dev/null +++ b/tests/unit_tests/charts/commands/importers/v1/import_test.py @@ -0,0 +1,69 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# pylint: disable=unused-argument, import-outside-toplevel, unused-import, invalid-name + +import copy + +from sqlalchemy.orm.session import Session + + +def test_import_chart(app_context: None, session: Session) -> None: + """ + Test importing a chart. + """ + from superset.charts.commands.importers.v1.utils import import_chart + from superset.connectors.sqla.models import SqlaTable + from superset.models.core import Database + from superset.models.slice import Slice + from tests.integration_tests.fixtures.importexport import chart_config + + engine = session.get_bind() + Slice.metadata.create_all(engine) # pylint: disable=no-member + + config = copy.deepcopy(chart_config) + config["datasource_id"] = 1 + config["datasource_type"] = "table" + + chart = import_chart(session, config) + assert chart.slice_name == "Deck Path" + assert chart.viz_type == "deck_path" + assert chart.is_managed_externally is False + assert chart.external_url is None + + +def test_import_chart_managed_externally(app_context: None, session: Session) -> None: + """ + Test importing a chart that is managed externally. + """ + from superset.charts.commands.importers.v1.utils import import_chart + from superset.connectors.sqla.models import SqlaTable + from superset.models.core import Database + from superset.models.slice import Slice + from tests.integration_tests.fixtures.importexport import chart_config + + engine = session.get_bind() + Slice.metadata.create_all(engine) # pylint: disable=no-member + + config = copy.deepcopy(chart_config) + config["datasource_id"] = 1 + config["datasource_type"] = "table" + config["is_managed_externally"] = True + config["external_url"] = "https://example.org/my_chart" + + chart = import_chart(session, config) + assert chart.is_managed_externally is True + assert chart.external_url == "https://example.org/my_chart" diff --git a/tests/unit_tests/dashboards/commands/importers/v1/import_test.py b/tests/unit_tests/dashboards/commands/importers/v1/import_test.py new file mode 100644 index 0000000000000..651e5dc10b7ca --- /dev/null +++ b/tests/unit_tests/dashboards/commands/importers/v1/import_test.py @@ -0,0 +1,67 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# pylint: disable=unused-argument, import-outside-toplevel, unused-import, invalid-name + +import copy + +from sqlalchemy.orm.session import Session + + +def test_import_dashboard(app_context: None, session: Session) -> None: + """ + Test importing a dashboard. + """ + from superset.connectors.sqla.models import SqlaTable + from superset.dashboards.commands.importers.v1.utils import import_dashboard + from superset.models.core import Database + from superset.models.slice import Slice + from tests.integration_tests.fixtures.importexport import dashboard_config + + engine = session.get_bind() + Slice.metadata.create_all(engine) # pylint: disable=no-member + + config = copy.deepcopy(dashboard_config) + + dashboard = import_dashboard(session, config) + assert dashboard.dashboard_title == "Test dash" + assert dashboard.description is None + assert dashboard.is_managed_externally is False + assert dashboard.external_url is None + + +def test_import_dashboard_managed_externally( + app_context: None, session: Session +) -> None: + """ + Test importing a dashboard that is managed externally. + """ + from superset.connectors.sqla.models import SqlaTable + from superset.dashboards.commands.importers.v1.utils import import_dashboard + from superset.models.core import Database + from superset.models.slice import Slice + from tests.integration_tests.fixtures.importexport import dashboard_config + + engine = session.get_bind() + Slice.metadata.create_all(engine) # pylint: disable=no-member + + config = copy.deepcopy(dashboard_config) + config["is_managed_externally"] = True + config["external_url"] = "https://example.org/my_dashboard" + + dashboard = import_dashboard(session, config) + assert dashboard.is_managed_externally is True + assert dashboard.external_url == "https://example.org/my_dashboard" diff --git a/tests/unit_tests/databases/__init__.py b/tests/unit_tests/databases/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/tests/unit_tests/databases/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/tests/unit_tests/databases/commands/__init__.py b/tests/unit_tests/databases/commands/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/tests/unit_tests/databases/commands/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/tests/unit_tests/databases/commands/importers/__init__.py b/tests/unit_tests/databases/commands/importers/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/tests/unit_tests/databases/commands/importers/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/tests/unit_tests/databases/commands/importers/v1/__init__.py b/tests/unit_tests/databases/commands/importers/v1/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/tests/unit_tests/databases/commands/importers/v1/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/tests/unit_tests/databases/commands/importers/v1/import_test.py b/tests/unit_tests/databases/commands/importers/v1/import_test.py new file mode 100644 index 0000000000000..622aa27fc3d56 --- /dev/null +++ b/tests/unit_tests/databases/commands/importers/v1/import_test.py @@ -0,0 +1,70 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# pylint: disable=unused-argument, import-outside-toplevel, invalid-name + +import copy + +from sqlalchemy.orm.session import Session + + +def test_import_database(app_context: None, session: Session) -> None: + """ + Test importing a database. + """ + from superset.databases.commands.importers.v1.utils import import_database + from superset.models.core import Database + from tests.integration_tests.fixtures.importexport import database_config + + engine = session.get_bind() + Database.metadata.create_all(engine) # pylint: disable=no-member + + config = copy.deepcopy(database_config) + database = import_database(session, config) + assert database.database_name == "imported_database" + assert database.sqlalchemy_uri == "sqlite:///test.db" + assert database.cache_timeout is None + assert database.expose_in_sqllab is True + assert database.allow_run_async is False + assert database.allow_ctas is True + assert database.allow_cvas is True + assert database.allow_file_upload is True + assert database.extra == "{}" + assert database.uuid == "b8a1ccd3-779d-4ab7-8ad8-9ab119d7fe89" + assert database.is_managed_externally is False + assert database.external_url is None + + +def test_import_database_managed_externally( + app_context: None, session: Session +) -> None: + """ + Test importing a database that is managed externally. + """ + from superset.databases.commands.importers.v1.utils import import_database + from superset.models.core import Database + from tests.integration_tests.fixtures.importexport import database_config + + engine = session.get_bind() + Database.metadata.create_all(engine) # pylint: disable=no-member + + config = copy.deepcopy(database_config) + config["is_managed_externally"] = True + config["external_url"] = "https://example.org/my_database" + + database = import_database(session, config) + assert database.is_managed_externally is True + assert database.external_url == "https://example.org/my_database" diff --git a/tests/unit_tests/datasets/commands/importers/v1/import_test.py b/tests/unit_tests/datasets/commands/importers/v1/import_test.py index 0aa0f67a07690..667584c198cfc 100644 --- a/tests/unit_tests/datasets/commands/importers/v1/import_test.py +++ b/tests/unit_tests/datasets/commands/importers/v1/import_test.py @@ -16,6 +16,7 @@ # under the License. # pylint: disable=import-outside-toplevel, unused-argument, unused-import, invalid-name +import copy import json import uuid from typing import Any, Dict @@ -199,6 +200,7 @@ def test_import_column_extra_is_string(app_context: None, session: Session) -> N "database_uuid": database.uuid, } + # the Marshmallow schema should convert strings to objects schema = ImportV1DatasetSchema() dataset_config = schema.load(yaml_config) dataset_config["database_id"] = database.id @@ -207,3 +209,31 @@ def test_import_column_extra_is_string(app_context: None, session: Session) -> N assert sqla_table.metrics[0].extra == '{"warning_markdown": null}' assert sqla_table.columns[0].extra == '{"certified_by": "User"}' assert sqla_table.extra == '{"warning_markdown": "*WARNING*"}' + + +def test_import_dataset_managed_externally(app_context: None, session: Session) -> None: + """ + Test importing a dataset that is managed externally. + """ + from superset.connectors.sqla.models import SqlaTable, SqlMetric, TableColumn + from superset.datasets.commands.importers.v1.utils import import_dataset + from superset.datasets.schemas import ImportV1DatasetSchema + from superset.models.core import Database + from tests.integration_tests.fixtures.importexport import dataset_config + + engine = session.get_bind() + SqlaTable.metadata.create_all(engine) # pylint: disable=no-member + + database = Database(database_name="my_database", sqlalchemy_uri="sqlite://") + session.add(database) + session.flush() + + dataset_uuid = uuid.uuid4() + config = copy.deepcopy(dataset_config) + config["is_managed_externally"] = True + config["external_url"] = "https://example.org/my_table" + config["database_id"] = database.id + + sqla_table = import_dataset(session, config) + assert sqla_table.is_managed_externally is True + assert sqla_table.external_url == "https://example.org/my_table"