Merge branch 'main' into unit_testing_feature_branch

.changes/unreleased/Features-20230914-074429.yaml

@@ -4,4 +4,4 @@ body: Add support for optional label in semantic_models, measures, dimensions an
 time: 2023-09-14T07:44:29.828199-05:00
 custom:
   Author: emmyoop
-  Issue: "8595"
+  Issue: 8595 8755

.changes/unreleased/Features-20230916-120547.yaml

@@ -0,0 +1,6 @@
+kind: Features
+body: resolve packages with same git repo and unique subdirectory
+time: 2023-09-16T12:05:47.353417149-04:00
+custom:
+  Author: philippeboyd
+  Issue: "5374"

.changes/unreleased/Features-20230918-150855.yaml

@@ -1,4 +1,4 @@
-kind: Features
+kind: Fixes
 body: Support quoted parameter list for MultiOption CLI options.
 time: 2023-09-18T15:08:55.625412-05:00
 custom:

.changes/unreleased/Features-20230919-102148.yaml

@@ -0,0 +1,6 @@
+kind: Features
+body: Add new ResourceReport event to record memory/cpu/io metrics
+time: 2023-09-19T10:21:48.772635-04:00
+custom:
+  Author: peterallenwebb
+  Issue: "8342"

.changes/unreleased/Features-20230922-150754.yaml

@@ -2,5 +2,5 @@ kind: Features
 body: Support `fill_nulls_with` and `join_to_timespine` for metric nodes
 time: 2023-09-22T15:07:54.981752-07:00
 custom:
-  Author: QMalcolm
-  Issue: "8593"
+  Author: QMalcolm emmyoop
+  Issue: 8593 8755

.changes/unreleased/Features-20230926-134728.yaml

@@ -0,0 +1,6 @@
+kind: Features
+body: Raise a warning when a contracted model has a numeric field without scale defined
+time: 2023-09-26T13:47:28.645383-05:00
+custom:
+  Author: emmyoop
+  Issue: "8183"

.changes/unreleased/Features-20230929-154743.yaml

@@ -0,0 +1,6 @@
+kind: Features
+body: Added support for retrieving partial catalog information from a schema
+time: 2023-09-29T15:47:43.612438-04:00
+custom:
+  Author: peterallenwebb
+  Issue: "8521"

.changes/unreleased/Fixes-20230912-153002.yaml

@@ -0,0 +1,6 @@
+kind: Fixes
+body: Disallow cleaning paths outside current working directory
+time: 2023-09-12T15:30:02.089967+01:00
+custom:
+  Author: aranke
+  Issue: "8318"

.changes/unreleased/Fixes-20230913-153924.yaml

@@ -0,0 +1,6 @@
+kind: Fixes
+body: 'update dbt show to include limit in DWH query '
+time: 2023-09-13T15:39:24.591805+01:00
+custom:
+  Author: michelleark
+  Issue: 8496, 8417

.changes/unreleased/Fixes-20230928-184856.yaml

@@ -0,0 +1,6 @@
+kind: Fixes
+body: avoid double-rendering sql_header in dbt show
+time: 2023-09-28T18:48:56.419428+01:00
+custom:
+  Author: michelleark
+  Issue: "8739"

.changes/unreleased/Fixes-20230929-175342.yaml

@@ -0,0 +1,6 @@
+kind: Fixes
+body: Fix tag selection for projects with semantic models
+time: 2023-09-29T17:53:42.960596+02:00
+custom:
+  Author: jtcohen6
+  Issue: "8749"

.github/workflows/docs-issue.yml

@@ -27,7 +27,14 @@ permissions:
 
 jobs:
   open_issues:
-    if: contains( github.event.pull_request.labels.*.name, 'user docs') && github.event.pull_request.merged == true
+    # we only want to run this when the PR has been merged or the label in the labeled event is `user docs`.  Otherwise it runs the
+    # risk of duplicaton of issues being created due to merge and label both triggering this workflow to run and neither having
+    # generating the comment before the other runs.  This lives here instead of the shared workflow because this is where we
+    # decide if it should run or not.
+    if: |
+      (github.event.pull_request.merged == true) &&
+      ((github.event.action == 'closed' && contains( github.event.pull_request.labels.*.name, 'user docs')) ||
+      (github.event.action == 'labeled' && github.event.label.name == 'user docs'))
     uses: dbt-labs/actions/.github/workflows/open-issue-in-repo.yml@main
     with:
         issue_repository: "dbt-labs/docs.getdbt.com"

core/dbt/adapters/base/impl.py

@@ -74,6 +74,7 @@
 from dbt import deprecations
 
 GET_CATALOG_MACRO_NAME = "get_catalog"
+GET_CATALOG_RELATIONS_MACRO_NAME = "get_catalog_relations"
 FRESHNESS_MACRO_NAME = "collect_freshness"
 
 
@@ -161,6 +162,14 @@ def submit(self, compiled_code: str) -> Any:
         raise NotImplementedError("PythonJobHelper submit function is not implemented yet")
 
 
+class AdapterFeature(str, Enum):
+    """Enumeration of optional adapter features which can be probed using BaseAdapter.has_feature()"""
+
+    CatalogByRelations = "CatalogByRelations"
+    """Flags support for retrieving catalog information using a list of relations, rather than always retrieving all
+    the relations in a schema """
+
+
 class BaseAdapter(metaclass=AdapterMeta):
     """The BaseAdapter provides an abstract base class for adapters.
 
@@ -415,6 +424,29 @@ def _get_catalog_schemas(self, manifest: Manifest) -> SchemaSearchMap:
         lowercase strings.
         """
         info_schema_name_map = SchemaSearchMap()
+        relations = self._get_catalog_relations(manifest)
+        for relation in relations:
+            info_schema_name_map.add(relation)
+        # result is a map whose keys are information_schema Relations without
+        # identifiers that have appropriate database prefixes, and whose values
+        # are sets of lowercase schema names that are valid members of those
+        # databases
+        return info_schema_name_map
+
+    def _get_catalog_relations_by_info_schema(
+        self, manifest: Manifest
+    ) -> Dict[InformationSchema, List[BaseRelation]]:
+        relations = self._get_catalog_relations(manifest)
+        relations_by_info_schema: Dict[InformationSchema, List[BaseRelation]] = dict()
+        for relation in relations:
+            info_schema = relation.information_schema_only()
+            if info_schema not in relations_by_info_schema:
+                relations_by_info_schema[info_schema] = []
+            relations_by_info_schema[info_schema].append(relation)
+
+        return relations_by_info_schema
+
+    def _get_catalog_relations(self, manifest: Manifest) -> List[BaseRelation]:
         nodes: Iterator[ResultNode] = chain(
             [
                 node
@@ -423,14 +455,9 @@ def _get_catalog_schemas(self, manifest: Manifest) -> SchemaSearchMap:
             ],
             manifest.sources.values(),
         )
-        for node in nodes:
-            relation = self.Relation.create_from(self.config, node)
-            info_schema_name_map.add(relation)
-        # result is a map whose keys are information_schema Relations without
-        # identifiers that have appropriate database prefixes, and whose values
-        # are sets of lowercase schema names that are valid members of those
-        # databases
-        return info_schema_name_map
+
+        relations = [self.Relation.create_from(self.config, n) for n in nodes]
+        return relations
 
     def _relations_cache_for_schemas(
         self, manifest: Manifest, cache_schemas: Optional[Set[BaseRelation]] = None
@@ -1093,20 +1120,57 @@ def _get_one_catalog(
         results = self._catalog_filter_table(table, manifest)  # type: ignore[arg-type]
         return results
 
+    def _get_one_catalog_by_relations(
+        self,
+        information_schema: InformationSchema,
+        relations: List[BaseRelation],
+        manifest: Manifest,
+    ) -> agate.Table:
+
+        kwargs = {
+            "information_schema": information_schema,
+            "relations": relations,
+        }
+        table = self.execute_macro(
+            GET_CATALOG_RELATIONS_MACRO_NAME,
+            kwargs=kwargs,
+            # pass in the full manifest, so we get any local project
+            # overrides
+            manifest=manifest,
+        )
+
+        results = self._catalog_filter_table(table, manifest)  # type: ignore[arg-type]
+        return results
+
     def get_catalog(self, manifest: Manifest) -> Tuple[agate.Table, List[Exception]]:
-        schema_map = self._get_catalog_schemas(manifest)
 
         with executor(self.config) as tpe:
             futures: List[Future[agate.Table]] = []
-            for info, schemas in schema_map.items():
-                if len(schemas) == 0:
-                    continue
-                name = ".".join([str(info.database), "information_schema"])
-
-                fut = tpe.submit_connected(
-                    self, name, self._get_one_catalog, info, schemas, manifest
-                )
-                futures.append(fut)
+            relation_count = len(self._get_catalog_relations(manifest))
+            if relation_count <= 100 and self.has_feature(AdapterFeature.CatalogByRelations):
+                relations_by_schema = self._get_catalog_relations_by_info_schema(manifest)
+                for info_schema in relations_by_schema:
+                    name = ".".join([str(info_schema.database), "information_schema"])
+                    relations = relations_by_schema[info_schema]
+                    fut = tpe.submit_connected(
+                        self,
+                        name,
+                        self._get_one_catalog_by_relations,
+                        info_schema,
+                        relations,
+                        manifest,
+                    )
+                    futures.append(fut)
+            else:
+                schema_map: SchemaSearchMap = self._get_catalog_schemas(manifest)
+                for info, schemas in schema_map.items():
+                    if len(schemas) == 0:
+                        continue
+                    name = ".".join([str(info.database), "information_schema"])
+                    fut = tpe.submit_connected(
+                        self, name, self._get_one_catalog, info, schemas, manifest
+                    )
+                    futures.append(fut)
 
             catalogs, exceptions = catch_as_completed(futures)
 
@@ -1437,6 +1501,13 @@ def render_model_constraint(cls, constraint: ModelLevelConstraint) -> Optional[s
         else:
             return None
 
+    @classmethod
+    def has_feature(cls, feature: AdapterFeature) -> bool:
+        # The base adapter implementation does not implement any optional
+        # features, so always return false. Adapters which wish to provide
+        # optional features will have to override this function.
+        return False
+
 
 COLUMNS_EQUAL_SQL = """
 with diff_count as (

core/dbt/adapters/base/relation.py

@@ -459,11 +459,11 @@ def add(self, relation: BaseRelation):
         self[key].add(schema)
 
     def search(self) -> Iterator[Tuple[InformationSchema, Optional[str]]]:
-        for information_schema_name, schemas in self.items():
+        for information_schema, schemas in self.items():
             for schema in schemas:
-                yield information_schema_name, schema
+                yield information_schema, schema
 
-    def flatten(self, allow_multiple_databases: bool = False):
+    def flatten(self, allow_multiple_databases: bool = False) -> "SchemaSearchMap":
         new = self.__class__()
 
         # make sure we don't have multiple databases if allow_multiple_databases is set to False

core/dbt/cli/main.py

@@ -170,6 +170,7 @@ def wrapper(*args, **kwargs):
 @p.warn_error
 @p.warn_error_options
 @p.log_format
+@p.show_resource_report
 def cli(ctx, **kwargs):
     """An ELT tool for managing your SQL transformations and data models.
     For more documentation on these commands, visit: docs.getdbt.com
@@ -225,6 +226,7 @@ def build(ctx, **kwargs):
 @cli.command("clean")
 @click.pass_context
 @global_flags
+@p.clean_project_files_only
 @p.profile
 @p.profiles_dir
 @p.project_dir

core/dbt/cli/params.py

@@ -423,6 +423,13 @@
     default=True,
 )
 
+clean_project_files_only = click.option(
+    "--clean-project-files-only / --no-clean-project-files-only",
+    envvar="DBT_CLEAN_PROJECT_FILES_ONLY",
+    help="If disabled, dbt clean will delete all paths specified in clean-paths, even if they're outside the dbt project.",
+    default=True,
+)
+
 show = click.option(
     "--show",
     envvar=None,
@@ -607,3 +614,10 @@ def _version_callback(ctx, _param, value):
     help="Whether or not to write the manifest.json and run_results.json files to the target directory",
     default=True,
 )
+
+show_resource_report = click.option(
+    "--show-resource-report/--no-show-resource-report",
+    default=False,
+    envvar="DBT_SHOW_RESOURCE_REPORT",
+    hidden=True,
+)

core/dbt/cli/requires.py

@@ -9,12 +9,14 @@
 from dbt.cli.flags import Flags
 from dbt.config import RuntimeConfig
 from dbt.config.runtime import load_project, load_profile, UnsetProfile
+from dbt.events.base_types import EventLevel
 from dbt.events.functions import fire_event, LOG_VERSION, set_invocation_id, setup_event_logger
 from dbt.events.types import (
     CommandCompleted,
     MainReportVersion,
     MainReportArgs,
     MainTrackingUserState,
+    ResourceReport,
 )
 from dbt.events.helpers import get_json_string_utcnow
 from dbt.events.types import MainEncounteredError, MainStackTrace
@@ -27,6 +29,7 @@
 
 from click import Context
 from functools import update_wrapper
+import importlib.util
 import time
 import traceback
 
@@ -96,6 +99,28 @@ def wrapper(*args, **kwargs):
             fire_event(MainStackTrace(stack_trace=traceback.format_exc()))
             raise ExceptionExit(e)
         finally:
+            # Fire ResourceReport, but only on systems which support the resource
+            # module. (Skip it on Windows).
+            if importlib.util.find_spec("resource") is not None:
+                import resource
+
+                rusage = resource.getrusage(resource.RUSAGE_SELF)
+                fire_event(
+                    ResourceReport(
+                        command_name=ctx.command.name,
+                        command_success=success,
+                        command_wall_clock_time=time.perf_counter() - start_func,
+                        process_user_time=rusage.ru_utime,
+                        process_kernel_time=rusage.ru_stime,
+                        process_mem_max_rss=rusage.ru_maxrss,
+                        process_in_blocks=rusage.ru_inblock,
+                        process_out_blocks=rusage.ru_oublock,
+                    ),
+                    EventLevel.INFO
+                    if "flags" in ctx.obj and ctx.obj["flags"].SHOW_RESOURCE_REPORT
+                    else None,
+                )
+
             fire_event(
                 CommandCompleted(
                     command=ctx.command_path,

core/dbt/constants.py

@@ -9,6 +9,7 @@
     "https://docs.getdbt.com/docs/package-management#section-specifying-package-versions"
 )
 
+DBT_PROJECT_FILE_NAME = "dbt_project.yml"
 PACKAGES_FILE_NAME = "packages.yml"
 DEPENDENCIES_FILE_NAME = "dependencies.yml"
 MANIFEST_FILE_NAME = "manifest.json"