Updated Readme and improved support for DLT (databrickslabs#45)

## Changes
* Updated Readme
* Minor refactor of the profiler
* Added functions for filtering data sets

### Linked issues
databrickslabs#17
databrickslabs#19


### Tests
<!-- How is this tested? Please see the checklist below and also
describe any other relevant tests -->

- [x] manually tested
- [x] added unit tests
- [ ] added integration tests

src/databricks/labs/dqx/col_functions.py

@@ -21,7 +21,7 @@ def make_condition(condition: Column, message: Column | str, alias: str) -> Colu
     return (F.when(condition, msg_col).otherwise(F.lit(None).cast("string"))).alias(_cleanup_alias_name(alias))
 
 
-def _cleanup_alias_name(col_name: str):
+def _cleanup_alias_name(col_name: str) -> str:
     # avoid issues with structs
     return col_name.replace(".", "_")
 
@@ -116,7 +116,7 @@ def sql_expression(expression: str, msg: str | None = None, name: str | None = N
 
     :param expression: SQL expression
     :param msg: optional message of the `Column` type, automatically generated if None
-    :param name: optional name of the column, automatically generated if None
+    :param name: optional name of the resulting column, automatically generated if None
     :param negate: if the condition should be negated (true) or not. For example, "col is not null" will mark null
     values as "bad". Although sometimes it's easier to specify it other way around "col is null" + negate set to False
     :return: new Column

src/databricks/labs/dqx/engine.py

@@ -169,12 +169,30 @@ def apply_checks_and_split(df: DataFrame, checks: list[DQRule]) -> tuple[DataFra
 
     checked_df = apply_checks(df, checks)
 
-    good_df = checked_df.where(F.col(Columns.ERRORS.value).isNull()).drop(Columns.ERRORS.value, Columns.WARNINGS.value)
-    bad_df = checked_df.where(F.col(Columns.ERRORS.value).isNotNull() | F.col(Columns.WARNINGS.value).isNotNull())
+    good_df = get_valid(checked_df)
+    bad_df = get_invalid(checked_df)
 
     return good_df, bad_df
 
 
+def get_invalid(df: DataFrame) -> DataFrame:
+    """
+    Get records that violate data quality checks.
+    @param df: input DataFrame.
+    @return: dataframe with error and warning rows and corresponding reporting columns.
+    """
+    return df.where(F.col(Columns.ERRORS.value).isNotNull() | F.col(Columns.WARNINGS.value).isNotNull())
+
+
+def get_valid(df: DataFrame) -> DataFrame:
+    """
+    Get records that don't violate data quality checks.
+    @param df: input DataFrame.
+    @return: dataframe with warning rows but no reporting columns.
+    """
+    return df.where(F.col(Columns.ERRORS.value).isNull()).drop(Columns.ERRORS.value, Columns.WARNINGS.value)
+
+
 def build_checks_by_metadata(checks: list[dict], glbs: dict[str, Any] | None = None) -> list[DQRule]:
     """Build checks based on check specification, i.e. function name plus arguments.
 

src/databricks/labs/dqx/profiler/profiler.py

@@ -32,7 +32,7 @@ def do_cast(value: str | None, typ: T.DataType) -> Any | None:
 def get_df_summary_as_dict(df: DataFrame) -> dict[str, Any]:
     """Generate summary for Dataframe & return it as dictionary with column name as a key, and dict of metric/value
 
-    :param df: dataframe to profile
+    :param df: dataframe to _profile
     :return: dict with metrics per column
     """
     sm_dict: dict[str, dict] = {}
@@ -233,7 +233,7 @@ def get_columns_or_fields(cols: list[T.StructField]) -> list[T.StructField]:
 # TODO: split into managebale chunks
 # TODO: how to handle maps, arrays & structs?
 # TODO: return not only DQ rules, but also the profiling results - use named tuple?
-def profile_dataframe(
+def profile(
     df: DataFrame, cols: list[str] | None = None, opts: dict[str, Any] | None = None
 ) -> tuple[dict[str, Any], list[DQProfile]]:
     if opts is None:
@@ -254,12 +254,12 @@ def profile_dataframe(
     max_nulls = opts.get("max_null_ratio", 0)
     trim_strings = opts.get("trim_strings", True)
 
-    profile(df, df_cols, dq_rules, max_nulls, opts, summary_stats, total_count, trim_strings)
+    _profile(df, df_cols, dq_rules, max_nulls, opts, summary_stats, total_count, trim_strings)
 
     return summary_stats, dq_rules
 
 
-def profile(df, df_cols, dq_rules, max_nulls, opts, summary_stats, total_count, trim_strings):
+def _profile(df, df_cols, dq_rules, max_nulls, opts, summary_stats, total_count, trim_strings):
     # TODO: think, how we can do it in fewer passes. Maybe only for specific things, like, min_max, etc.
     for field in get_columns_or_fields(df_cols):
         field_name = field.name

tests/unit/test_profiler.py

@@ -6,7 +6,7 @@
     DQProfile,
     T,
     get_columns_or_fields,
-    profile_dataframe,
+    profile,
 )
 
 
@@ -82,7 +82,8 @@ def test_profiler(spark_session: SparkSession):
         ],
         schema=inp_schema,
     )
-    stats, rules = profile_dataframe(inp_df)
+
+    stats, rules = profile(inp_df)
 
     expected_rules = [
         DQProfile(name="is_not_null", column="t1", description=None, parameters=None),
@@ -114,7 +115,7 @@ def test_profiler(spark_session: SparkSession):
 def test_profiler_empty_df(spark_session: SparkSession):
     test_df = spark_session.createDataFrame([], "data: string")
 
-    stats, rules = profile_dataframe(test_df)
+    actual_summary_stats, actual_dq_rules = profile(test_df)
 
-    assert len(stats.keys()) > 0
-    assert len(rules) == 0
+    assert len(actual_summary_stats.keys()) > 0
+    assert len(actual_dq_rules) == 0