create identified_users and anonymous_users models (#12)

This creates three new models that will hopefully make it more clear
what an identified user and anonymous user are. Identified users are
informed from actual identify events, where anonymous users are informed
by devices who have not been identified.

integration_tests/dbt_project.yml

@@ -29,11 +29,7 @@ models:
     relation: true
 
 vars:
-  events_table: fullstory_events_integration_tests
-  sessions_identifier: fullstory_sessions_integration_tests
-  users_identifier: fullstory_users_integration_tests
-  dbt_date:time_zone: Etc/UTC
-  fullstory_replay_host: app.fullstory.com
+  fullstory_events_table: fullstory_events_integration_tests
   fullstory_events_types:
     [
       "background",
@@ -62,13 +58,12 @@ vars:
       "pinch_gesture",
       "request",
     ]
+  fullstory_replay_host: app.fullstory.com
 
 seeds:
   dbt_fullstory_integration_tests:
-    +enabled: true
-
+    enabled: true
     fullstory_events_integration_tests:
-      +alias: "{% if target.type == 'snowflake' %}fullstory_events_integration_tests_raw{% endif %}"
       +column_types:
         event_time: datetime
         processed_time: datetime

integration_tests/models/_test_anonymous_users.yml

@@ -0,0 +1,60 @@
+version: 2
+
+models:
+  - name: test_anonymous_users
+    tests:
+      - dbt_utils.expression_is_true:
+          name: last_device_user_agent_is_correct
+          expression: last_device_user_agent = 'good_user_agent'
+          config:
+            where: device_id = 7
+      - dbt_utils.expression_is_true:
+          name: last_device_type_is_correct
+          expression: last_device_type = 'good_device_type'
+          config:
+            where: device_id = 7
+      - dbt_utils.expression_is_true:
+          name: last_device_operating_system_is_correct
+          expression: last_device_operating_system = 'good_operating_system'
+          config:
+            where: device_id = 7
+      - dbt_utils.expression_is_true:
+          name: last_device_browser_is_correct
+          expression: last_device_browser = 'good_browser'
+          config:
+            where: device_id = 7
+      - dbt_utils.expression_is_true:
+          name: last_device_browser_version_is_correct
+          expression: last_device_browser_version = 'good_browser_version'
+          config:
+            where: device_id = 7
+      - dbt_utils.expression_is_true:
+          name: last_geo_ip_address_is_correct
+          expression: last_geo_ip_address = 'good_ip_address'
+          config:
+            where: device_id = 7
+      - dbt_utils.expression_is_true:
+          name: last_geo_region_is_correct
+          expression: last_geo_region = 'good_region'
+          config:
+            where: device_id = 7
+      - dbt_utils.expression_is_true:
+          name: last_geo_city_is_correct
+          expression: last_geo_city = 'good_city'
+          config:
+            where: device_id = 7
+      - dbt_utils.expression_is_true:
+          name: last_geo_lat_long_is_correct
+          expression: last_geo_lat_long = 'good_lat_long'
+          config:
+            where: device_id = 7
+      - dbt_utils.expression_is_true:
+          name: last_event_time_is_correct
+          expression: last_event_time = '2023-01-01T14:00:00'
+          config:
+            where: device_id = 7
+      - dbt_utils.expression_is_true:
+          name: last_event_time_is_correct
+          expression: last_event_id = 'user_last_source_properties_test_3'
+          config:
+            where: device_id = 7

integration_tests/models/_test_identified_users.yml

@@ -0,0 +1,75 @@
+version: 2
+
+models:
+  - name: test_identified_users
+    tests:
+      - dbt_utils.expression_is_true:
+          name: user_email_is_correct
+          expression: user_email = 'good_user_email@fullstory.com'
+          config:
+            where: user_id = 'good_user_id'
+      - dbt_utils.expression_is_true:
+          name: user_display_name_is_correct
+          expression: user_display_name = 'good_display_name'
+          config:
+            where: user_id = 'good_user_id'
+      - dbt_utils.expression_is_true:
+          name: user_properties_are_correct
+          expression: JSON_VALUE(user_properties, '$.good_key') = 'good_value'
+          config:
+            where: user_id = 'good_user_id'
+      - dbt_utils.expression_is_true:
+          name: last_device_user_agent_is_correct
+          expression: last_device_user_agent = 'good_user_agent'
+          config:
+            where: user_id = 'user_test_last_properties_user_id'
+      - dbt_utils.expression_is_true:
+          name: last_device_type_is_correct
+          expression: last_device_type = 'good_device_type'
+          config:
+            where: user_id = 'user_test_last_properties_user_id'
+      - dbt_utils.expression_is_true:
+          name: last_device_operating_system_is_correct
+          expression: last_device_operating_system = 'good_operating_system'
+          config:
+            where: user_id = 'user_test_last_properties_user_id'
+      - dbt_utils.expression_is_true:
+          name: last_device_browser_is_correct
+          expression: last_device_browser = 'good_browser'
+          config:
+            where: user_id = 'user_test_last_properties_user_id'
+      - dbt_utils.expression_is_true:
+          name: last_device_browser_version_is_correct
+          expression: last_device_browser_version = 'good_browser_version'
+          config:
+            where: user_id = 'user_test_last_properties_user_id'
+      - dbt_utils.expression_is_true:
+          name: last_geo_ip_address_is_correct
+          expression: last_geo_ip_address = 'good_ip_address'
+          config:
+            where: user_id = 'user_test_last_properties_user_id'
+      - dbt_utils.expression_is_true:
+          name: last_geo_region_is_correct
+          expression: last_geo_region = 'good_region'
+          config:
+            where: user_id = 'user_test_last_properties_user_id'
+      - dbt_utils.expression_is_true:
+          name: last_geo_city_is_correct
+          expression: last_geo_city = 'good_city'
+          config:
+            where: user_id = 'user_test_last_properties_user_id'
+      - dbt_utils.expression_is_true:
+          name: last_geo_lat_long_is_correct
+          expression: last_geo_lat_long = 'good_lat_long'
+          config:
+            where: user_id = 'user_test_last_properties_user_id'
+      - dbt_utils.expression_is_true:
+          name: last_event_time_is_correct
+          expression: last_event_time = '2023-01-01T14:00:00'
+          config:
+            where: user_id = 'user_test_last_properties_user_id'
+      - dbt_utils.expression_is_true:
+          name: last_event_time_is_correct
+          expression: last_event_id = 'user_last_source_properties_test_3'
+          config:
+            where: user_id = 'user_test_last_properties_user_id'

integration_tests/models/test_anonymous_users.sql

@@ -0,0 +1 @@
+select * from {{ ref('anonymous_users') }}

integration_tests/models/test_identified_users.sql

@@ -0,0 +1 @@
+select * from {{ ref('identified_users') }}

integration_tests/seeds/fullstory_events_integration_tests.csv

@@ -6,9 +6,9 @@ identify_session_test_1,2023-01-01 12:00:00,2023-01-01 12:00:00,2023-01-01 00:00
 identify_session_test_2,2023-01-01 12:01:00,2023-01-01 12:00:00,2023-01-01 00:00:00,2,2,1,identify,"{""user_email"":""bad_user_email@fullstory.com""}",web,{}
 identify_session_test_3,2023-01-01 12:02:00,2023-01-01 12:00:00,2023-01-01 00:00:00,2,2,1,identify,"{""user_display_name"":""bad_user_display_name""}",web,{}
 identify_session_test_4,2023-01-01 12:03:00,2023-01-01 12:00:00,2023-01-01 00:00:00,2,2,1,identify,"{""user_id"":""bad_user_id""}",web,{}
-identify_session_test_6,2023-01-01 13:00:00,2023-01-01 13:00:00,2023-01-01 12:00:00,2,2,1,identify,"{""user_id"":""good_user_id"",""user_display_name"":""good_display_name"",""user_email"":""good_user_email@fullstory.com""}",web,{}
+identify_session_test_6,2023-01-01 13:00:00,2023-01-01 13:00:00,2023-01-01 12:00:00,2,2,1,identify,"{""user_id"":""good_user_id"",""user_display_name"":""good_display_name"",""user_email"":""good_user_email@fullstory.com"",""user_properties"": { ""good_key"":""good_value"" } }",web,{}
 identify_session_test_5,2023-01-01 13:00:00,2023-01-01 12:00:00,2023-01-01 12:00:00,2,2,1,identify,"{""user_id"":""another_bad_user_id"",""user_display_name"":""another_bad_display_name"",""user_email"":""another_bad_user_email@fullstory.com""}",web,{}
-session_event_type_counts_test_1,2023-01-01 13:00:00,2023-01-01 12:00:00,2023-01-01 00:00:00,3,3,1,background,{},web,{}
+session_event_type_counts_test_1,2023-01-01 13:00:00,2023-01-01 12:00:00,2023-01-01 00:00:00,3,3,1,background,{},web,"{""user_agent"":{""raw_user_agent"":""device_3_user_agent""}"
 session_event_type_counts_test_2,2023-01-01 13:00:00,2023-01-01 12:00:00,2023-01-01 00:00:00,3,3,1,background,{},web,{}
 session_event_type_counts_test_3,2023-01-01 13:00:00,2023-01-01 12:00:00,2023-01-01 00:00:00,3,3,1,change,{},web,{}
 session_event_type_counts_test_4,2023-01-01 13:00:00,2023-01-01 12:00:00,2023-01-01 00:00:00,3,3,1,change,{},web,{}

macros/parse_json_into_columns.sql

@@ -1,7 +1,7 @@
 {%- macro parse_json_into_columns(field, columns) -%}
 {%- for column in columns -%}
 
-{%- set inner = json_value(field, column.path, column.array) %}
+{%- set inner = json_value(field, column.path, column.array, column.dtype) %}
 {{ column.prefix -}}
 
 {%- if column.cast_as -%}

models/anonymous_users.sql

@@ -0,0 +1,23 @@
+select
+  devices.device_id,
+  devices.device_user_agent as last_device_user_agent,
+  devices.device_type as last_device_type,
+  devices.device_operating_system as last_device_operating_system,
+  devices.device_browser as last_device_browser,
+  devices.device_browser_version as last_device_browser_version,
+  devices.geo_ip_address as last_geo_ip_address,
+  devices.geo_country as last_geo_country,
+  devices.geo_region as last_geo_region,
+  devices.geo_city as last_geo_city,
+  devices.geo_lat_long as last_geo_lat_long,
+  devices.event_time as last_event_time,
+  devices.event_id as last_event_id,
+from {{ ref('int_devices') }} devices
+where
+  devices.device_id is not null
+  and devices.device_num_desc = 1
+  and devices.source_type != 'server' -- exclude server events, they won't have geo or device values.
+  and devices.device_id not in (
+    select device_id
+    from {{ ref('int_identifies') }}
+  )

models/anonymous_users.yml

@@ -0,0 +1,48 @@
+models:
+  - name: anonymous_users
+    access: public
+    config:
+      alias: "{{ var('fullstory_anonymous_users_model_name', 'anonymous_users') }}"
+      contract:
+        enforce: true
+    description: |-
+      All users who have never been identified.
+    columns:
+      - name: device_id
+        description: '{{ doc("column_last_device_id") }}'
+        constraints:
+          - type: not_null
+        tests:
+          - not_null
+      - name: last_device_user_agent
+        description: '{{ doc("column_last_device_user_agent") }}'
+      - name: last_device_type
+        description: '{{ doc("column_last_device_type") }}'
+      - name: last_device_operating_system
+        description: '{{ doc("column_last_device_operating_system") }}'
+      - name: last_device_browser
+        description: '{{ doc("column_last_device_browser") }}'
+      - name: last_device_browser_version
+        description: '{{ doc("column_last_device_browser_version") }}'
+      - name: last_geo_ip_address
+        description: '{{ doc("column_last_geo_ip_address") }}'
+      - name: last_geo_country
+        description: '{{ doc("column_last_geo_country") }}'
+      - name: last_geo_region
+        description: '{{ doc("column_last_geo_region") }}'
+      - name: last_geo_city
+        description: '{{ doc("column_last_geo_city") }}'
+      - name: last_geo_lat_long
+        description: '{{ doc("column_last_geo_lat_long") }}'
+      - name: last_event_time
+        description: '{{ doc("column_last_event_time") }}'
+        constraints:
+          - type: not_null
+        tests:
+          - not_null
+      - name: last_event_id
+        description: '{{ doc("column_last_event_id") }}'
+        constraints:
+          - type: not_null
+        tests:
+          - not_null