feat(retention): Enable person on events querying for retention

ee/clickhouse/queries/retention/clickhouse_retention.py

@@ -142,13 +142,17 @@ def actors_in_period(self, filter: RetentionFilter, team: Team):
 
 
 def build_returning_event_query(
-    filter: RetentionFilter, team: Team, aggregate_users_by_distinct_id: Optional[bool] = None
+    filter: RetentionFilter,
+    team: Team,
+    aggregate_users_by_distinct_id: Optional[bool] = None,
+    using_person_on_events: bool = False,
 ):
     returning_event_query_templated, returning_event_params = RetentionEventsQuery(
         filter=filter.with_data({"breakdowns": []}),  # Avoid pulling in breakdown values from returning event query
         team=team,
         event_query_type=RetentionQueryType.RETURNING,
         aggregate_users_by_distinct_id=aggregate_users_by_distinct_id,
+        using_person_on_events=using_person_on_events,
     ).get_query()
 
     query = substitute_params(returning_event_query_templated, returning_event_params)
@@ -157,7 +161,10 @@ def build_returning_event_query(
 
 
 def build_target_event_query(
-    filter: RetentionFilter, team: Team, aggregate_users_by_distinct_id: Optional[bool] = None
+    filter: RetentionFilter,
+    team: Team,
+    aggregate_users_by_distinct_id: Optional[bool] = None,
+    using_person_on_events: bool = False,
 ):
     target_event_query_templated, target_event_params = RetentionEventsQuery(
         filter=filter,
@@ -168,6 +175,7 @@ def build_target_event_query(
             else RetentionQueryType.TARGET
         ),
         aggregate_users_by_distinct_id=aggregate_users_by_distinct_id,
+        using_person_on_events=using_person_on_events,
     ).get_query()
 
     query = substitute_params(target_event_query_templated, target_event_params)

ee/clickhouse/queries/retention/retention_actors.py

@@ -125,11 +125,17 @@ def build_actor_activity_query(
     person_ids
     """
     returning_event_query = build_returning_event_query(
-        filter=filter, team=team, aggregate_users_by_distinct_id=aggregate_users_by_distinct_id
+        filter=filter,
+        team=team,
+        aggregate_users_by_distinct_id=aggregate_users_by_distinct_id,
+        using_person_on_events=team.actor_on_events_querying_enabled,
     )
 
     target_event_query = build_target_event_query(
-        filter=filter, team=team, aggregate_users_by_distinct_id=aggregate_users_by_distinct_id
+        filter=filter,
+        team=team,
+        aggregate_users_by_distinct_id=aggregate_users_by_distinct_id,
+        using_person_on_events=team.actor_on_events_querying_enabled,
     )
 
     all_params = {

ee/clickhouse/queries/retention/retention_event_query.py

@@ -14,6 +14,7 @@
 from posthog.models.action.util import Action, format_action_filter
 from posthog.models.filters.retention_filter import RetentionFilter
 from posthog.models.team import Team
+from posthog.models.utils import PersonPropertiesMode
 from posthog.queries.util import format_ch_timestamp, get_trunc_func_ch
 
 
@@ -28,10 +29,14 @@ def __init__(
         event_query_type: RetentionQueryType,
         team: Team,
         aggregate_users_by_distinct_id: Optional[bool] = None,
+        using_person_on_events: bool = False,
     ):
         self._event_query_type = event_query_type
         super().__init__(
-            filter=filter, team=team, override_aggregate_users_by_distinct_id=aggregate_users_by_distinct_id
+            filter=filter,
+            team=team,
+            override_aggregate_users_by_distinct_id=aggregate_users_by_distinct_id,
+            using_person_on_events=using_person_on_events,
         )
 
         self._trunc_func = get_trunc_func_ch(self._filter.period)
@@ -60,7 +65,7 @@ def get_query(self) -> Tuple[str, Dict[str, Any]]:
                     get_aggregation_target_field(
                         self._filter.aggregation_group_type_index,
                         self.EVENT_TABLE_ALIAS,
-                        f"{self.DISTINCT_ID_TABLE_ALIAS}.person_id",
+                        f"{self.DISTINCT_ID_TABLE_ALIAS if not self._using_person_on_events else self.EVENT_TABLE_ALIAS}.person_id",
                     )
                 )
             ]
@@ -77,8 +82,8 @@ def get_query(self) -> Tuple[str, Dict[str, Any]]:
             column = "properties"
 
             if breakdown_type == "person":
-                table = "person"
-                column = "person_props"
+                table = "person" if not self._using_person_on_events else "events"
+                column = "person_props" if not self._using_person_on_events else "person_properties"
 
             breakdown_values_expression = get_single_or_multi_property_string_expr(
                 breakdown=[breakdown["property"] for breakdown in self._filter.breakdowns],
@@ -131,7 +136,12 @@ def get_query(self) -> Tuple[str, Dict[str, Any]]:
         date_query, date_params = self._get_date_filter()
         self.params.update(date_params)
 
-        prop_query, prop_params = self._get_prop_groups(self._filter.property_groups)
+        prop_query, prop_params = self._get_prop_groups(
+            self._filter.property_groups,
+            person_properties_mode=PersonPropertiesMode.DIRECT_ON_EVENTS
+            if self._using_person_on_events
+            else PersonPropertiesMode.USING_PERSON_PROPERTIES_COLUMN,
+        )
 
         self.params.update(prop_params)
 
@@ -179,12 +189,24 @@ def _determine_should_join_distinct_ids(self) -> None:
         else:
             self._should_join_distinct_ids = True
 
+    def _determine_should_join_persons(self) -> None:
+        EnterpriseEventQuery._determine_should_join_persons(self)
+        if self._using_person_on_events:
+            self._should_join_distinct_ids = False
+            self._should_join_persons = False
+
     def _get_entity_query(self, entity: Entity):
         prepend = self._event_query_type
         if entity.type == TREND_FILTER_TYPE_ACTIONS:
             action = Action.objects.get(pk=entity.id)
             action_query, params = format_action_filter(
-                team_id=self._team_id, action=action, prepend=prepend, use_loop=False
+                team_id=self._team_id,
+                action=action,
+                prepend=prepend,
+                use_loop=False,
+                person_properties_mode=PersonPropertiesMode.DIRECT_ON_EVENTS
+                if self._using_person_on_events
+                else PersonPropertiesMode.USING_PERSON_PROPERTIES_COLUMN,
             )
             condition = action_query
         elif entity.type == TREND_FILTER_TYPE_EVENTS:

ee/clickhouse/queries/test/__snapshots__/test_trends.ambr

@@ -110,86 +110,6 @@
   OFFSET 0
   '
 ---
-# name: TestClickhouseTrends.test_breakdown_by_group_props.3
-  '
-
-  SELECT groupArray(day_start) as date,
-         groupArray(count) as data,
-         breakdown_value
-  FROM
-    (SELECT SUM(total) as count,
-            day_start,
-            breakdown_value
-     FROM
-       (SELECT *
-        FROM
-          (SELECT toUInt16(0) AS total,
-                  ticks.day_start as day_start,
-                  breakdown_value
-           FROM
-             (SELECT toStartOfDay(toDateTime('2020-01-12 23:59:59', 'UTC') - number * 86400) as day_start
-              FROM numbers(12)
-              UNION ALL SELECT toStartOfDay(toDateTime('2020-01-01 00:00:00', 'UTC')) as day_start) as ticks
-           CROSS JOIN
-             (SELECT breakdown_value
-              FROM
-                (SELECT ['finance', 'technology'] as breakdown_value) ARRAY
-              JOIN breakdown_value) as sec
-           ORDER BY breakdown_value,
-                    day_start
-           UNION ALL SELECT count(*) as total,
-                            toStartOfDay(timestamp, 'UTC') as day_start,
-                            replaceRegexpAll(JSONExtractRaw(group0_properties, 'industry'), '^"|"$', '') as breakdown_value
-           FROM events e
-           WHERE e.team_id = 2
-             AND event = 'sign up'
-             AND timestamp >= toStartOfDay(toDateTime('2020-01-01 00:00:00'), 'UTC')
-             AND timestamp <= toDateTime('2020-01-12 23:59:59')
-             AND replaceRegexpAll(JSONExtractRaw(group0_properties, 'industry'), '^"|"$', '') in (['finance', 'technology'])
-           GROUP BY day_start,
-                    breakdown_value))
-     GROUP BY day_start,
-              breakdown_value
-     ORDER BY breakdown_value,
-              day_start)
-  GROUP BY breakdown_value
-  ORDER BY breakdown_value
-  '
----
-# name: TestClickhouseTrends.test_breakdown_by_group_props.4
-  '
-
-  SELECT person_id AS actor_id
-  FROM
-    (SELECT e.timestamp as timestamp,
-            pdi.person_id as person_id,
-            e.distinct_id as distinct_id,
-            e.team_id as team_id
-     FROM events e
-     INNER JOIN
-       (SELECT distinct_id,
-               argMax(person_id, version) as person_id
-        FROM person_distinct_id2
-        WHERE team_id = 2
-        GROUP BY distinct_id
-        HAVING argMax(is_deleted, version) = 0) AS pdi ON e.distinct_id = pdi.distinct_id
-     INNER JOIN
-       (SELECT group_key,
-               argMax(group_properties, _timestamp) AS group_properties_0
-        FROM groups
-        WHERE team_id = 2
-          AND group_type_index = 0
-        GROUP BY group_key) groups_0 ON "$group_0" == groups_0.group_key
-     WHERE team_id = 2
-       AND event = 'sign up'
-       AND timestamp >= toDateTime('2020-01-02 00:00:00')
-       AND timestamp <= toDateTime('2020-01-02 23:59:59')
-       AND (has(['technology'], replaceRegexpAll(JSONExtractRaw(group_properties_0, 'industry'), '^"|"$', ''))) )
-  GROUP BY actor_id
-  LIMIT 200
-  OFFSET 0
-  '
----
 # name: TestClickhouseTrends.test_breakdown_by_group_props_with_person_filter
   '
 
@@ -296,72 +216,6 @@
   ORDER BY breakdown_value
   '
 ---
-# name: TestClickhouseTrends.test_breakdown_by_group_props_with_person_filter.2
-  '
-
-  SELECT groupArray(value)
-  FROM
-    (SELECT replaceRegexpAll(JSONExtractRaw(group0_properties, 'industry'), '^"|"$', '') AS value,
-            count(*) as count
-     FROM events e
-     WHERE team_id = 2
-       AND event = 'sign up'
-       AND timestamp >= toDateTime('2020-01-01 00:00:00')
-       AND timestamp <= toDateTime('2020-01-12 23:59:59')
-       AND (has(['value'], replaceRegexpAll(JSONExtractRaw(e.person_properties, 'key'), '^"|"$', '')))
-     GROUP BY value
-     ORDER BY count DESC
-     LIMIT 25
-     OFFSET 0)
-  '
----
-# name: TestClickhouseTrends.test_breakdown_by_group_props_with_person_filter.3
-  '
-
-  SELECT groupArray(day_start) as date,
-         groupArray(count) as data,
-         breakdown_value
-  FROM
-    (SELECT SUM(total) as count,
-            day_start,
-            breakdown_value
-     FROM
-       (SELECT *
-        FROM
-          (SELECT toUInt16(0) AS total,
-                  ticks.day_start as day_start,
-                  breakdown_value
-           FROM
-             (SELECT toStartOfDay(toDateTime('2020-01-12 23:59:59', 'UTC') - number * 86400) as day_start
-              FROM numbers(12)
-              UNION ALL SELECT toStartOfDay(toDateTime('2020-01-01 00:00:00', 'UTC')) as day_start) as ticks
-           CROSS JOIN
-             (SELECT breakdown_value
-              FROM
-                (SELECT ['finance'] as breakdown_value) ARRAY
-              JOIN breakdown_value) as sec
-           ORDER BY breakdown_value,
-                    day_start
-           UNION ALL SELECT count(*) as total,
-                            toStartOfDay(timestamp, 'UTC') as day_start,
-                            replaceRegexpAll(JSONExtractRaw(group0_properties, 'industry'), '^"|"$', '') as breakdown_value
-           FROM events e
-           WHERE e.team_id = 2
-             AND event = 'sign up'
-             AND (has(['value'], replaceRegexpAll(JSONExtractRaw(e.person_properties, 'key'), '^"|"$', '')))
-             AND timestamp >= toStartOfDay(toDateTime('2020-01-01 00:00:00'), 'UTC')
-             AND timestamp <= toDateTime('2020-01-12 23:59:59')
-             AND replaceRegexpAll(JSONExtractRaw(group0_properties, 'industry'), '^"|"$', '') in (['finance'])
-           GROUP BY day_start,
-                    breakdown_value))
-     GROUP BY day_start,
-              breakdown_value
-     ORDER BY breakdown_value,
-              day_start)
-  GROUP BY breakdown_value
-  ORDER BY breakdown_value
-  '
----
 # name: TestClickhouseTrends.test_breakdown_filtering_with_properties_in_new_format
   '
 
@@ -555,36 +409,6 @@
      order by day_start SETTINGS allow_experimental_window_functions = 1) SETTINGS timeout_before_checking_execution_speed = 60
   '
 ---
-# name: TestClickhouseTrends.test_person_property_filtering.1
-  '
-
-  SELECT groupArray(day_start) as date,
-         groupArray(count) as data
-  FROM
-    (SELECT SUM(total) AS count,
-            day_start
-     from
-       (SELECT toUInt16(0) AS total,
-               toStartOfDay(toDateTime('2020-01-04 23:59:59') - toIntervalDay(number), 'UTC') AS day_start
-        FROM numbers(dateDiff('day', toStartOfDay(toDateTime('2019-12-28 00:00:00'), 'UTC'), toDateTime('2020-01-04 23:59:59'), 'UTC'))
-        UNION ALL SELECT toUInt16(0) AS total,
-                         toStartOfDay(toDateTime('2019-12-28 00:00:00'), 'UTC')
-        UNION ALL SELECT count(*) as data,
-                         toStartOfDay(toDateTime(timestamp), 'UTC') as date
-        FROM
-          (SELECT e.timestamp as timestamp,
-                  e.person_id as person_id
-           FROM events e
-           WHERE team_id = 2
-             AND event = 'watched movie'
-             AND timestamp >= toStartOfDay(toDateTime('2019-12-28 00:00:00'), 'UTC')
-             AND timestamp <= toDateTime('2020-01-04 23:59:59')
-             AND (has(['person1'], replaceRegexpAll(JSONExtractRaw(e.person_properties, 'name'), '^"|"$', ''))) )
-        GROUP BY date)
-     group by day_start
-     order by day_start SETTINGS allow_experimental_window_functions = 1) SETTINGS timeout_before_checking_execution_speed = 60
-  '
----
 # name: TestClickhouseTrends.test_person_property_filtering_materialized
   '
 
@@ -628,36 +452,6 @@
      order by day_start SETTINGS allow_experimental_window_functions = 1) SETTINGS timeout_before_checking_execution_speed = 60
   '
 ---
-# name: TestClickhouseTrends.test_person_property_filtering_materialized.1
-  '
-
-  SELECT groupArray(day_start) as date,
-         groupArray(count) as data
-  FROM
-    (SELECT SUM(total) AS count,
-            day_start
-     from
-       (SELECT toUInt16(0) AS total,
-               toStartOfDay(toDateTime('2020-01-04 23:59:59') - toIntervalDay(number), 'UTC') AS day_start
-        FROM numbers(dateDiff('day', toStartOfDay(toDateTime('2019-12-28 00:00:00'), 'UTC'), toDateTime('2020-01-04 23:59:59'), 'UTC'))
-        UNION ALL SELECT toUInt16(0) AS total,
-                         toStartOfDay(toDateTime('2019-12-28 00:00:00'), 'UTC')
-        UNION ALL SELECT count(*) as data,
-                         toStartOfDay(toDateTime(timestamp), 'UTC') as date
-        FROM
-          (SELECT e.timestamp as timestamp,
-                  e.person_id as person_id
-           FROM events e
-           WHERE team_id = 2
-             AND event = 'watched movie'
-             AND timestamp >= toStartOfDay(toDateTime('2019-12-28 00:00:00'), 'UTC')
-             AND timestamp <= toDateTime('2020-01-04 23:59:59')
-             AND (has(['person1'], replaceRegexpAll(JSONExtractRaw(e.person_properties, 'name'), '^"|"$', ''))) )
-        GROUP BY date)
-     group by day_start
-     order by day_start SETTINGS allow_experimental_window_functions = 1) SETTINGS timeout_before_checking_execution_speed = 60
-  '
----
 # name: TestClickhouseTrends.test_timezones
   '