From eac13e3bd38b7f29401444ca77ee243ec4d8f29d Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Tue, 3 Oct 2023 13:08:37 -0500 Subject: [PATCH 1/3] Add meta to SemanticModels (#8754) * WIP * changelog --- .../unreleased/Features-20230929-170945.yaml | 6 +++++ core/dbt/contracts/graph/model_config.py | 4 ++++ core/dbt/contracts/graph/nodes.py | 1 + core/dbt/contracts/graph/unparsed.py | 1 + core/dbt/parser/schema_yaml_readers.py | 7 ++++++ schemas/dbt/manifest/v11.json | 12 ++++++++++ tests/functional/semantic_models/fixtures.py | 4 ++++ .../test_semantic_model_configs.py | 23 +++++++++++++++++++ 8 files changed, 58 insertions(+) create mode 100644 .changes/unreleased/Features-20230929-170945.yaml diff --git a/.changes/unreleased/Features-20230929-170945.yaml b/.changes/unreleased/Features-20230929-170945.yaml new file mode 100644 index 00000000000..3497b2f5246 --- /dev/null +++ b/.changes/unreleased/Features-20230929-170945.yaml @@ -0,0 +1,6 @@ +kind: Features +body: Add meta attribute to SemanticModels +time: 2023-09-29T17:09:45.0354-05:00 +custom: + Author: emmyoop + Issue: "8511" diff --git a/core/dbt/contracts/graph/model_config.py b/core/dbt/contracts/graph/model_config.py index b28091e68c1..f44fee5d50c 100644 --- a/core/dbt/contracts/graph/model_config.py +++ b/core/dbt/contracts/graph/model_config.py @@ -382,6 +382,10 @@ class SemanticModelConfig(BaseConfig): default=None, metadata=CompareBehavior.Exclude.meta(), ) + meta: Dict[str, Any] = field( + default_factory=dict, + metadata=MergeBehavior.Update.meta(), + ) @dataclass diff --git a/core/dbt/contracts/graph/nodes.py b/core/dbt/contracts/graph/nodes.py index 065a60a6d33..57a6043e49d 100644 --- a/core/dbt/contracts/graph/nodes.py +++ b/core/dbt/contracts/graph/nodes.py @@ -1583,6 +1583,7 @@ class SemanticModel(GraphNode): refs: List[RefArgs] = field(default_factory=list) created_at: float = field(default_factory=lambda: time.time()) config: SemanticModelConfig = field(default_factory=SemanticModelConfig) + meta: Dict[str, Any] = field(default_factory=dict) unrendered_config: Dict[str, Any] = field(default_factory=dict) primary_entity: Optional[str] = None group: Optional[str] = None diff --git a/core/dbt/contracts/graph/unparsed.py b/core/dbt/contracts/graph/unparsed.py index d06b13faa03..df8149982f5 100644 --- a/core/dbt/contracts/graph/unparsed.py +++ b/core/dbt/contracts/graph/unparsed.py @@ -712,6 +712,7 @@ class UnparsedSemanticModel(dbtClassMixin): name: str model: str # looks like "ref(...)" config: Dict[str, Any] = field(default_factory=dict) + meta: Dict[str, Any] = field(default_factory=dict) description: Optional[str] = None label: Optional[str] = None defaults: Optional[Defaults] = None diff --git a/core/dbt/parser/schema_yaml_readers.py b/core/dbt/parser/schema_yaml_readers.py index dddad84c6db..2e357bbbe8e 100644 --- a/core/dbt/parser/schema_yaml_readers.py +++ b/core/dbt/parser/schema_yaml_readers.py @@ -549,6 +549,7 @@ def _generate_semantic_model_config( base=False, patch_config_dict=precedence_configs, ) + return config def parse_semantic_model(self, unparsed: UnparsedSemanticModel): @@ -595,8 +596,14 @@ def parse_semantic_model(self, unparsed: UnparsedSemanticModel): config=config, unrendered_config=unrendered_config, group=config.group, + meta=unparsed.meta, ) + # If we have meta in the config, copy to node level, for backwards + # compatibility with earlier node-only config. + if "meta" in config and config["meta"]: + parsed.meta = config["meta"] + ctx = generate_parse_semantic_models( parsed, self.root_project, diff --git a/schemas/dbt/manifest/v11.json b/schemas/dbt/manifest/v11.json index 431641576eb..18047ef5135 100644 --- a/schemas/dbt/manifest/v11.json +++ b/schemas/dbt/manifest/v11.json @@ -5497,6 +5497,12 @@ } ], "default": null + }, + "meta": { + "type": "object", + "propertyNames": { + "type": "string" + } } }, "additionalProperties": true @@ -5635,6 +5641,12 @@ "config": { "$ref": "#/$defs/SemanticModelConfig" }, + "meta": { + "type": "object", + "propertyNames": { + "type": "string" + } + }, "unrendered_config": { "type": "object", "propertyNames": { diff --git a/tests/functional/semantic_models/fixtures.py b/tests/functional/semantic_models/fixtures.py index 3fb65a3a4fb..be5d88a7809 100644 --- a/tests/functional/semantic_models/fixtures.py +++ b/tests/functional/semantic_models/fixtures.py @@ -134,6 +134,9 @@ config: enabled: true group: some_group + meta: + my_meta: 'testing' + my_other_meta: 'testing more' dimensions: - name: favorite_color type: categorical @@ -185,6 +188,7 @@ agg_time_dimension: created_at """ + schema_yml = """models: - name: fct_revenue description: This is the model fct_revenue. It should be able to use doc blocks diff --git a/tests/functional/semantic_models/test_semantic_model_configs.py b/tests/functional/semantic_models/test_semantic_model_configs.py index 38009cddbe0..5938b1040c3 100644 --- a/tests/functional/semantic_models/test_semantic_model_configs.py +++ b/tests/functional/semantic_models/test_semantic_model_configs.py @@ -204,3 +204,26 @@ def test_project_plus_yaml_level(self, project): ).config assert isinstance(config_test_table, SemanticModelConfig) + + +# test setting meta attributes in semantic model config +class TestMetaConfig: + @pytest.fixture(scope="class") + def models(self): + return { + "people.sql": models_people_sql, + "metricflow_time_spine.sql": metricflow_time_spine_sql, + "semantic_models.yml": enabled_semantic_model_people_yml, + "people_metrics.yml": models_people_metrics_yml, + "groups.yml": groups_yml, + } + + def test_meta_config(self, project): + run_dbt(["parse"]) + manifest = get_manifest(project.project_root) + sm_id = "semantic_model.test.semantic_people" + assert sm_id in manifest.semantic_models + sm_node = manifest.semantic_models[sm_id] + meta_expected = {"my_meta": "testing", "my_other_meta": "testing more"} + assert sm_node.meta == meta_expected + assert sm_node.config.meta == meta_expected From c6ff3abecd390788f2d29fd7d5d2338fa1ab8bb2 Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Wed, 4 Oct 2023 13:23:09 -0500 Subject: [PATCH 2/3] remove top level meta attribute (#8766) --- .changes/unreleased/Features-20230929-170945.yaml | 2 +- core/dbt/contracts/graph/nodes.py | 1 - core/dbt/contracts/graph/unparsed.py | 1 - core/dbt/parser/schema_yaml_readers.py | 6 ------ schemas/dbt/manifest/v11.json | 6 ------ .../semantic_models/test_semantic_model_configs.py | 1 - 6 files changed, 1 insertion(+), 16 deletions(-) diff --git a/.changes/unreleased/Features-20230929-170945.yaml b/.changes/unreleased/Features-20230929-170945.yaml index 3497b2f5246..92a2ce484d1 100644 --- a/.changes/unreleased/Features-20230929-170945.yaml +++ b/.changes/unreleased/Features-20230929-170945.yaml @@ -1,5 +1,5 @@ kind: Features -body: Add meta attribute to SemanticModels +body: Add meta attribute to SemanticModels config time: 2023-09-29T17:09:45.0354-05:00 custom: Author: emmyoop diff --git a/core/dbt/contracts/graph/nodes.py b/core/dbt/contracts/graph/nodes.py index 57a6043e49d..065a60a6d33 100644 --- a/core/dbt/contracts/graph/nodes.py +++ b/core/dbt/contracts/graph/nodes.py @@ -1583,7 +1583,6 @@ class SemanticModel(GraphNode): refs: List[RefArgs] = field(default_factory=list) created_at: float = field(default_factory=lambda: time.time()) config: SemanticModelConfig = field(default_factory=SemanticModelConfig) - meta: Dict[str, Any] = field(default_factory=dict) unrendered_config: Dict[str, Any] = field(default_factory=dict) primary_entity: Optional[str] = None group: Optional[str] = None diff --git a/core/dbt/contracts/graph/unparsed.py b/core/dbt/contracts/graph/unparsed.py index df8149982f5..d06b13faa03 100644 --- a/core/dbt/contracts/graph/unparsed.py +++ b/core/dbt/contracts/graph/unparsed.py @@ -712,7 +712,6 @@ class UnparsedSemanticModel(dbtClassMixin): name: str model: str # looks like "ref(...)" config: Dict[str, Any] = field(default_factory=dict) - meta: Dict[str, Any] = field(default_factory=dict) description: Optional[str] = None label: Optional[str] = None defaults: Optional[Defaults] = None diff --git a/core/dbt/parser/schema_yaml_readers.py b/core/dbt/parser/schema_yaml_readers.py index 2e357bbbe8e..167175b4dcd 100644 --- a/core/dbt/parser/schema_yaml_readers.py +++ b/core/dbt/parser/schema_yaml_readers.py @@ -596,14 +596,8 @@ def parse_semantic_model(self, unparsed: UnparsedSemanticModel): config=config, unrendered_config=unrendered_config, group=config.group, - meta=unparsed.meta, ) - # If we have meta in the config, copy to node level, for backwards - # compatibility with earlier node-only config. - if "meta" in config and config["meta"]: - parsed.meta = config["meta"] - ctx = generate_parse_semantic_models( parsed, self.root_project, diff --git a/schemas/dbt/manifest/v11.json b/schemas/dbt/manifest/v11.json index 18047ef5135..aafbab125d3 100644 --- a/schemas/dbt/manifest/v11.json +++ b/schemas/dbt/manifest/v11.json @@ -5641,12 +5641,6 @@ "config": { "$ref": "#/$defs/SemanticModelConfig" }, - "meta": { - "type": "object", - "propertyNames": { - "type": "string" - } - }, "unrendered_config": { "type": "object", "propertyNames": { diff --git a/tests/functional/semantic_models/test_semantic_model_configs.py b/tests/functional/semantic_models/test_semantic_model_configs.py index 5938b1040c3..bd74ad95edd 100644 --- a/tests/functional/semantic_models/test_semantic_model_configs.py +++ b/tests/functional/semantic_models/test_semantic_model_configs.py @@ -225,5 +225,4 @@ def test_meta_config(self, project): assert sm_id in manifest.semantic_models sm_node = manifest.semantic_models[sm_id] meta_expected = {"my_meta": "testing", "my_other_meta": "testing more"} - assert sm_node.meta == meta_expected assert sm_node.config.meta == meta_expected From df791f729cf3ba9d882ea168894cc4318b7193b6 Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Thu, 5 Oct 2023 08:12:28 -0500 Subject: [PATCH 3/3] support doc blocks (#8771) --- .../unreleased/Fixes-20231004-144148.yaml | 6 ++ core/dbt/parser/manifest.py | 4 +- core/dbt/parser/schema_renderer.py | 6 ++ .../docs/test_model_version_docs_blocks.py | 74 +++++++++++++++++++ 4 files changed, 89 insertions(+), 1 deletion(-) create mode 100644 .changes/unreleased/Fixes-20231004-144148.yaml create mode 100644 tests/functional/docs/test_model_version_docs_blocks.py diff --git a/.changes/unreleased/Fixes-20231004-144148.yaml b/.changes/unreleased/Fixes-20231004-144148.yaml new file mode 100644 index 00000000000..9ceeb6c9f3c --- /dev/null +++ b/.changes/unreleased/Fixes-20231004-144148.yaml @@ -0,0 +1,6 @@ +kind: Fixes +body: Support docs blocks on versioned model column descriptions +time: 2023-10-04T14:41:48.843486-05:00 +custom: + Author: emmyoop + Issue: "8540" diff --git a/core/dbt/parser/manifest.py b/core/dbt/parser/manifest.py index fbf2adde746..7a1c4db1d29 100644 --- a/core/dbt/parser/manifest.py +++ b/core/dbt/parser/manifest.py @@ -1117,10 +1117,12 @@ def update_semantic_model(self, semantic_model) -> None: database=refd_node.database, ) - # nodes: node and column descriptions + # nodes: node and column descriptions, version columns descriptions # sources: source and table descriptions, column descriptions # macros: macro argument descriptions # exposures: exposure descriptions + # metrics: metric descriptions + # semantic_models: semantic model descriptions def process_docs(self, config: RuntimeConfig): for node in self.manifest.nodes.values(): if node.created_at < self.started_at: diff --git a/core/dbt/parser/schema_renderer.py b/core/dbt/parser/schema_renderer.py index 66b91fee1b4..0f717515fca 100644 --- a/core/dbt/parser/schema_renderer.py +++ b/core/dbt/parser/schema_renderer.py @@ -34,12 +34,18 @@ def _is_norender_key(self, keypath: Keypath) -> bool: Return True if it's tests or description - those aren't rendered now because they're rendered later in parse_generic_tests or process_docs. """ + # top level descriptions and tests if len(keypath) >= 1 and keypath[0] in ("tests", "description"): return True + # columns descriptions and tests if len(keypath) == 2 and keypath[1] in ("tests", "description"): return True + # versions + if len(keypath) == 5 and keypath[4] == "description": + return True + if ( len(keypath) >= 3 and keypath[0] in ("columns", "dimensions", "measures", "entities") diff --git a/tests/functional/docs/test_model_version_docs_blocks.py b/tests/functional/docs/test_model_version_docs_blocks.py new file mode 100644 index 00000000000..335ef8e8937 --- /dev/null +++ b/tests/functional/docs/test_model_version_docs_blocks.py @@ -0,0 +1,74 @@ +import pytest + +from dbt.tests.util import run_dbt + +model_1 = """ +select 1 as id, 'joe' as first_name +""" + +model_versioned = """ +select 1 as id, 'joe' as first_name +""" + +docs_md = """ +{% docs model_description %} +unversioned model +{% enddocs %} + +{% docs column_id_doc %} +column id for some thing +{% enddocs %} + +{% docs versioned_model_description %} +versioned model +{% enddocs %} + +""" + +schema_yml = """ +models: + - name: model_1 + description: '{{ doc("model_description") }}' + columns: + - name: id + description: '{{ doc("column_id_doc") }}' + + - name: model_versioned + description: '{{ doc("versioned_model_description") }}' + latest_version: 1 + versions: + - v: 1 + config: + alias: my_alias + columns: + - name: id + description: '{{ doc("column_id_doc") }}' + - name: first_name + description: 'plain text' + - v: 2 + columns: + - name: other_id +""" + + +class TestVersionedModelDocsBlock: + @pytest.fixture(scope="class") + def models(self): + return { + "model_1.sql": model_1, + "model_versioned.sql": model_versioned, + "schema.yml": schema_yml, + "docs.md": docs_md, + } + + def test_versioned_doc_ref(self, project): + manifest = run_dbt(["parse"]) + model_1 = manifest.nodes["model.test.model_1"] + model_v1 = manifest.nodes["model.test.model_versioned.v1"] + + assert model_1.description == "unversioned model" + assert model_v1.description == "versioned model" + + assert model_1.columns["id"].description == "column id for some thing" + assert model_v1.columns["id"].description == "column id for some thing" + assert model_v1.columns["first_name"].description == "plain text"