Create-tableau-dashboard-based-on-iasworld-tests

dbt/macros/generate_iasworld_qc_test_view.sql

@@ -0,0 +1,140 @@
+{#-
+    Generate a QC report view that runs a set of boolean "tests" against a
+    base query and returns one row per (record, failing test) pair. We use
+    the resulting views to power iasWorld QC.
+
+    Args:
+        base_query: A SQL SELECT statement (as a string) that returns the
+            identifying columns plus any additional columns referenced by
+            the `condition` or `additional_select_columns` attributes of the
+            `tests` argument. Source tables that don't include a required
+            identifying column (usually `card` or `lline`) should return
+            `CAST(NULL AS <type>) AS <column_name>` for that column to conform
+            to the expected shape. Required identifying columns include:
+                - parid: varchar
+                - taxyr: varchar
+                - card: decimal
+                - lline: decimal
+                - township_code: varchar
+                - class: varchar
+                - who: varchar
+                - wen: varchar
+        tests: A list of dicts, each with keys:
+            - name: A unique, descriptive slug for the test
+            - description: A human-readable description of what the test checks
+            - category: A category slug used to group related tests
+            - condition: A SQL boolean expression, evaluated against
+                `base_query`, that is TRUE when the record passes the test
+                and FALSE when it fails
+            - additional_select_columns (optional): A list of column names
+                from `base_query` to include in the output as a map of column
+                name -> stringified value for records that fail the test
+
+    Returns:
+        A query that selects one row per record per failing test.
+-#}
+{% macro generate_iasworld_qc_test_view(base_query, tests) %}
+    {% do _validate_iasworld_qc_tests(tests, exceptions.raise_compiler_error) %}
+    with
+        base as ({{ base_query }}),
+
+        test_result as (
+            select
+                *,
+                -- noqa: disable=layout.indent
+                {% for test in tests %}
+                    not ({{ test.condition }}) as {{ test.name }}
+                    {{- "," if not loop.last }}
+                {% endfor %}
+            -- noqa: enable=layout.indent
+            from base
+            where
+                taxyr
+                between '{{ var("data_test_iasworld_year_start") }}'
+                and '{{ var("data_test_iasworld_year_end") }}'
+        )
+
+    {% for test in tests %}
+        select
+            parid,
+            taxyr,
+            card,
+            lline,
+            township_code,
+            class,
+            who,
+            wen,
+            '{{ test.name }}' as test_name,
+            '{{ test.description }}' as test_description,
+            '{{ test.category }}' as test_category,
+            {% if test.additional_select_columns -%}
+                map(
+                    array[
+                        {%- for col_name in test.additional_select_columns -%}
+                            '{{ col_name }}'{{ ", " if not loop.last }}
+                        {%- endfor %}
+                    ],
+                    array[
+                        {%- for col_name in test.additional_select_columns -%}
+                            cast({{ col_name }} as varchar) {{- ", " if not loop.last }}
+                        {%- endfor %}
+                    ]
+                ) as additional_columns
+            {%- else -%} cast(null as map(varchar, varchar)) as additional_columns
+            {%- endif %}
+        from test_result
+        where {{ test.name }} {{ "UNION ALL" if not loop.last }}
+    {% endfor %}
+{% endmacro %}
+
+{#-
+    Validate that `tests` is a list of dicts, each containing the keys
+    required by `generate_iasworld_qc_test_view`. Raises a compiler error
+    (via `raise_error_func`) on the first validation failure it finds.
+
+    Args:
+        tests: The `tests` argument passed to `generate_iasworld_qc_test_view`
+        raise_error_func: A function to call with an error message when
+            validation fails. Takes `exceptions.raise_compiler_error` in
+            production, and a mock in unit tests so that the error can be
+            returned for equality comparison instead of raised
+-#}
+{% macro _validate_iasworld_qc_tests(tests, raise_error_func) %}
+    {%- set required_keys = ["name", "description", "category", "condition"] -%}
+    {%- if tests is not iterable or tests is mapping or tests is string -%}
+        {{-
+            return(
+                raise_error_func('"tests" argument must be a list, got: ' ~ tests)
+            )
+        -}}
+    {%- endif -%}
+    {%- for test in tests -%}
+        {%- if test is not mapping -%}
+            {{-
+                return(
+                    raise_error_func(
+                        'Each element of "tests" must be an object/dict,'
+                        ~ " got: "
+                        ~ test
+                    )
+                )
+            -}}
+        {%- endif -%}
+        {%- for key in required_keys -%}
+            {%- if key not in test -%}
+                {{-
+                    return(
+                        raise_error_func(
+                            'Missing required "'
+                            ~ key
+                            ~ '" key in test'
+                            ~ " config: "
+                            ~ test
+                        )
+                    )
+                -}}
+            {%- endif -%}
+        {%- endfor -%}
+    {%- endfor -%}
+    {{ return(none) }}
+{% endmacro %}

dbt/macros/tests/test_all.sql

@@ -6,4 +6,5 @@
     {% do test_generate_alias_name() %}
     {% do test_format_additional_select_columns() %}
     {% do test_insert_hyphens() %}
+    {% do test_generate_iasworld_qc_test_view() %}
 {% endmacro %}

dbt/macros/tests/test_generate_iasworld_qc_test_view.sql

@@ -0,0 +1,66 @@
+{% macro test_generate_iasworld_qc_test_view() %}
+    {% do test_validate_iasworld_qc_tests_not_a_list() %}
+    {% do test_validate_iasworld_qc_tests_element_not_a_dict() %}
+    {% do test_validate_iasworld_qc_tests_missing_required_key() %}
+    {% do test_validate_iasworld_qc_tests_valid() %}
+{% endmacro %}
+
+{% macro test_validate_iasworld_qc_tests_not_a_list() %}
+    {{
+        assert_equals(
+            "test_validate_iasworld_qc_tests_not_a_list",
+            _validate_iasworld_qc_tests({"name": "foo"}, mock_raise_compiler_error),
+            "\"tests\" argument must be a list, got: {'name': 'foo'}",
+        )
+    }}
+{% endmacro %}
+
+{% macro test_validate_iasworld_qc_tests_element_not_a_dict() %}
+    {{
+        assert_equals(
+            "test_validate_iasworld_qc_tests_element_not_a_dict",
+            _validate_iasworld_qc_tests(["foo"], mock_raise_compiler_error),
+            'Each element of "tests" must be an object/dict, got: foo',
+        )
+    }}
+{% endmacro %}
+
+{% macro test_validate_iasworld_qc_tests_missing_required_key() %}
+    {{
+        assert_equals(
+            "test_validate_iasworld_qc_tests_missing_required_key",
+            _validate_iasworld_qc_tests(
+                [
+                    {
+                        "name": "foo",
+                        "description": "bar",
+                        "category": "baz",
+                    }
+                ],
+                mock_raise_compiler_error,
+            ),
+            "Missing required \"condition\" key in test config: {'name':"
+            ~ " 'foo', 'description': 'bar', 'category': 'baz'}",
+        )
+    }}
+{% endmacro %}
+
+{% macro test_validate_iasworld_qc_tests_valid() %}
+    {{
+        assert_equals(
+            "test_validate_iasworld_qc_tests_valid",
+            _validate_iasworld_qc_tests(
+                [
+                    {
+                        "name": "foo",
+                        "description": "bar",
+                        "category": "baz",
+                        "condition": "1 = 1",
+                    }
+                ],
+                mock_raise_compiler_error,
+            ),
+            none,
+        )
+    }}
+{% endmacro %}

dbt/models/qc/docs.md

@@ -202,6 +202,13 @@ classes.
 This view is not currently in use in any QC workflows.
 {% enddocs %}
 
+# vw_report_iasworld_test_pardat
+
+{% docs view_vw_report_iasworld_test_pardat %}
+Aggregates test failures, stemming from iasworld pardat. Each observation is a failure
+based on unique PIN, year, and failing test combinations.
+{% enddocs %}
+
 # vw_report_res_land
 
 {% docs view_vw_report_res_land %}

dbt/models/qc/qc.vw_report_iasworld_test_pardat.sql

@@ -0,0 +1,92 @@
+{%- set tests = [
+    {
+        "name": "iasworld_pardat_adrno_length_lte_5",
+        "description": "adrno should be <= 5 characters long",
+        "category": "column_length",
+        "condition": "length(adrno) <= 5",
+        "additional_select_columns": ["adrno"]
+    },
+    {
+        "name": "iasworld_pardat_class_equals_luc",
+        "description": "class should be the same as luc",
+        "category": "class_mismatch_or_issue",
+        "condition": "class = luc",
+        "additional_select_columns": ["luc"]
+    },
+    {
+        "name": "iasworld_pardat_cur_in_accepted_values",
+        "description": 'cur should be "Y" or "D"',
+        "category": "incorrect_values",
+        "condition": "cur IN ('Y', 'D')",
+        "additional_select_columns": ["cur"]
+    },
+    {
+        "name": "iasworld_pardat_nbhd_matches_legdat_township",
+        "description": "nbhd code first 2 digits should match legdat.user1 (township code)",
+        "category": "relationships",
+        "condition": "SUBSTR(nbhd, 1, 2) = township_code",
+        "additional_select_columns": ["nbhd"]
+    },
+    {
+        "name": "iasworld_pardat_nbhd_matches_spatial_town_nbhd",
+        "description": "nbhd code not valid",
+        "category": "relationships",
+        "condition": "town_nbhd IS NOT NULL OR nbhd LIKE '%999'",
+        "additional_select_columns": ["nbhd"]
+    },
+    {
+        "name": "iasworld_pardat_seq_all_sequential_exist",
+        "description": "seq should be sequential",
+        "category": "incorrect_values",
+        "condition": "seq = prev_seq + 1",
+        "additional_select_columns": ["seq", "prev_seq"]
+    },
+    {
+        "name": "iasworld_pardat_unique_by_parid_taxyr",
+        "description": "pardat should be unique by parid and taxyr",
+        "category": "duplicate_rows",
+        "condition": "num_duplicates = 1",
+        "additional_select_columns": ["num_duplicates"]
+    }
+] -%}
+
+{%- set base_query %}
+    SELECT
+        -- Identifying columns
+        pardat.parid,
+        pardat.taxyr,
+        CAST(NULL AS INTEGER) AS card,
+        CAST(NULL AS INTEGER) AS lline,
+        legdat.user1 AS township_code,
+        pardat.class,
+        pardat.who,
+        pardat.wen,
+        -- Columns to test
+        CAST(pardat.adrno AS VARCHAR) AS adrno,
+        pardat.luc,
+        pardat.cur,
+        pardat.nbhd,
+        nbhd.town_nbhd,
+        pardat.seq,
+        LAG(pardat.seq)
+            OVER (PARTITION BY pardat.parid, pardat.taxyr ORDER BY pardat.seq)
+            AS prev_seq,
+        COUNT(*)
+            OVER (PARTITION BY pardat.parid, pardat.taxyr)
+            AS num_duplicates
+    FROM iasworld.pardat AS pardat
+    LEFT JOIN iasworld.legdat AS legdat
+        ON pardat.parid = legdat.parid
+        AND pardat.taxyr = legdat.taxyr
+        AND legdat.cur = 'Y'
+        AND legdat.deactivat IS NULL
+    LEFT JOIN (
+        SELECT DISTINCT town_nbhd
+        FROM {{ source('spatial', 'neighborhood') }}
+    ) AS nbhd
+        ON pardat.nbhd = nbhd.town_nbhd
+    WHERE pardat.cur = 'Y'
+        AND pardat.deactivat IS NULL
+{% endset %}
+
+{{ generate_iasworld_qc_test_view(base_query, tests) }}

dbt/models/qc/schema.yml

@@ -1,4 +1,7 @@
 models:
+  - name: qc.vw_report_iasworld_test_pardat
+    description: '{{ doc("view_vw_report_iasworld_test_pardat") }}'
+
   - name: qc.vw_change_in_ahsap_values
     description: '{{ doc("view_vw_change_in_ahsap_values") }}'
 

pyproject.toml

@@ -74,3 +74,4 @@ load_macros_from_path = "dbt/macros"
 # lints
 get_s3_dependency_dir = "{% macro get_s3_dependency_dir() %}s3://bucket{% endmacro %}"
 insert_hyphens = "{% macro insert_hyphens(str) %}{% set _ = varargs %}'{{ str }}'{% endmacro %}"
+generate_iasworld_qc_test_view = "{% macro generate_iasworld_qc_test_view(base_query, tests) %}select 1{% endmacro %}"