feat: bulk export analytics (#4386) (#4395)

* feat: bulk export analytics (#4386)

* chore: bumped setup.py, resolved errors from rebase (#4386)

Author: Jonah Paten

analytics/analytics_package/analytics/_sheets_utils.py

@@ -104,4 +104,30 @@ def get_change_over_time_df(
     if strftime_format is not None:
         df_combined.index = pd.to_datetime(df_combined.index).strftime(strftime_format)
 
-    return df_combined.reset_index(names=time_dimension["alias"])
+    return df_combined.reset_index(names=time_dimension["alias"])
+
+def get_change_over_time_df_multiple_events(metric, events, time_dimension, **change_over_time_args):
+    """
+    Get a DataFrame with the change over time for the given metrics, renamed to match metric_titles
+    :param metrics: the metrics to be displayed
+    :param events: the events to be used
+    :param time_dimension: the time dimension to be used
+    :param change_over_time_args: any other parameters to be passed to the get_change_over_time_df function, including service params
+    :returns: a datetime with the values of the metrics for each time dimension.
+        Columns are the time dimension alias (as a datetime), metric aliases (as ints), and change metric aliases (as floats)
+    """
+    assert "dimension_filter" not in change_over_time_args
+    return pd.concat(
+        [
+            get_change_over_time_df(
+                [metric],
+                time_dimension,
+                **change_over_time_args,
+                dimension_filter=f"eventName=={event['id']}"
+            ).rename(
+                columns={metric["alias"]: event["alias"], metric["change_alias"]: event["change_alias"]}
+            ).set_index(time_dimension["alias"]) 
+            for event in events
+        ],
+        axis=1
+    )

analytics/analytics_package/analytics/entities.py

@@ -45,17 +45,77 @@
 # Event Names
 # The builtin outbound link click event. Stores the clicked URL in DIMENSION_BUILTIN_URL
 # Triggers under some circumstances where custom click does not, but does not include url fragments in any dimensions
-EVENT_BUILTIN_CLICK = "click"
+EVENT_BUILTIN_CLICK = {
+    "id": "click",
+    "alias": "Builtin Outbound Link Click"
+}
 # The custom outbound link click event. Stores the clicked URL DIMENSION_CUSTOM_URL
 # Includes url fragments, sometimes has a slightly different count to the built in click event
-EVENT_CUSTOM_CLICK = "outbound_link_clicked"
+EVENT_CUSTOM_CLICK = {
+    "id": "outbound_link_clicked",
+    "alias": "Custom Outbound Link Click"
+}
 # The builtin page view event. 
-EVENT_PAGE_VIEW = "page_view"
-EVENT_FILE_DOWNLOADED = "file_downloaded"
-EVENT_ENTITY_SELECTED = "entity_selected"
-EVENT_ENTITY_TABLE_SORTED = "entity_table_sorted"
-EVENT_ENTITY_TABLE_PAGINATED = "entity_table_paginated"
-EVENT_FILTER_SELECTED = "filter_selected"
+EVENT_PAGE_VIEW = {
+    "id": "page_view", 
+    "alias": "Page View"
+}
+
+EVENT_INDEX_BULK_DOWNLOAD_SELECTED = {
+    "id": "index_bulk_download_selected",
+    "alias": "Bulk Download Selected",
+    "change_alias": "Bulk Download Selected Change",
+}
+EVENT_INDEX_BULK_DOWNLOAD_REQUESTED = {
+    "id": "bulk_download_requested",
+    "alias": "Bulk Download Requested",
+    "change_alias": "Bulk Download Requested Change",
+}
+EVENT_INDEX_FILE_MANIFEST_SELECTED = {
+    "id": "index_file_manifest_selected",
+    "alias": "File Manifest Selected",
+    "change_alias": "File Manifest Selected Change",
+}
+EVENT_INDEX_FILE_MANIFEST_REQUESTED = {
+    "id": "index_file_manifest_requested",
+    "alias": "File Manifest Requested",
+    "change_alias": "File Manifest Requested Change",
+}
+EVENT_INDEX_ANALYZE_IN_TERRA_SELECTED = {
+    "id": "index_analyze_in_terra_selected",
+    "alias": "Analyze in Terra Selected",
+    "change_alias": "Analyze in Terra Selected Change",
+}
+EVENT_INDEX_ANALYZE_IN_TERRA_REQUESTED = {
+    "id": "index_analyze_in_terra_requested",
+    "alias": "Analyze in Terra Requested",
+    "change_alias": "Analyze in Terra Requested Change",
+}
+EVENT_FILE_DOWNLOADED = {
+    "id": "file_downloaded",
+    "alias": "File Downloaded",
+    "change_alias": "File Downloaded Change",
+}
+EVENT_ENTITY_SELECTED = {
+    "id": "entity_selected",
+    "alias": "Entity Selected",
+    "change_alias": "Entity Selected Change",
+}
+EVENT_ENTITY_TABLE_SORTED = {
+    "id": "entity_table_sorted",
+    "alias": "Entity Table Sorted",
+    "change_alias": "Entity Table Sorted Change",
+}
+EVENT_ENTITY_TABLE_PAGINATED = {
+    "id": "entity_table_paginated",
+    "alias": "Entity Table Paginated",
+    "change_alias": "Entity Table Paginated Change",
+}
+EVENT_FILTER_SELECTED = {
+    "id": "filter_selected",
+    "alias": "Filter Selected",
+    "change_alias": "Filter Selected Change",
+}
 # DIMENSIONS
 # The path to the page the user is on when the event occurs. Does not include fragments or parameters
 DIMENSION_PAGE_PATH = {

analytics/analytics_package/analytics/sheets_api.py

@@ -226,7 +226,7 @@ def fill_worksheet_with_df(
         # Format worksheet
         # Justify Column Widths
         if "column_widths" not in sheet_formatting_options_filled or sheet_formatting_options_filled["column_widths"]["justify"]:
-            text_widths = df.astype(str).columns.map(
+            text_widths = df.columns.map(
                 lambda column_name: df[column_name].astype(str).str.len().max()
             )
             header_widths = df.columns.str.len()

analytics/analytics_package/analytics/sheets_elements.py

@@ -21,7 +21,7 @@ def get_outbound_links_df(analytics_params, ignore_index=True):
     df_builtin_links =get_data_df_from_fields(
         [METRIC_EVENT_COUNT, METRIC_TOTAL_USERS],
         [DIMENSION_PAGE_PATH, DIMENSION_BUILTIN_URL, DIMENSION_EVENT_NAME],
-        dimension_filter=f"eventName=={EVENT_BUILTIN_CLICK}",
+        dimension_filter=f"eventName=={EVENT_BUILTIN_CLICK['id']}",
         **analytics_params,
     ).groupby(
         [DIMENSION_PAGE_PATH["alias"], DIMENSION_BUILTIN_URL["alias"]]
@@ -30,7 +30,7 @@ def get_outbound_links_df(analytics_params, ignore_index=True):
     df_custom_links = get_data_df_from_fields(
         [METRIC_EVENT_COUNT, METRIC_TOTAL_USERS],
         [DIMENSION_EVENT_NAME, DIMENSION_CUSTOM_URL, DIMENSION_PAGE_PATH], 
-        dimension_filter=f"eventName=={EVENT_CUSTOM_CLICK}",
+        dimension_filter=f"eventName=={EVENT_CUSTOM_CLICK['id']}",
         **analytics_params, 
     ).groupby(
         [DIMENSION_PAGE_PATH["alias"], DIMENSION_CUSTOM_URL["alias"]]
@@ -123,7 +123,7 @@ def get_page_views_df(analytics_params, ignore_index=False):
         [METRIC_EVENT_COUNT, METRIC_TOTAL_USERS, METRIC_PAGE_VIEWS],
         [DIMENSION_PAGE_PATH, DIMENSION_EVENT_NAME],
         **analytics_params,
-        dimension_filter=f"eventName=={EVENT_PAGE_VIEW}",
+        dimension_filter=f"eventName=={EVENT_PAGE_VIEW['id']}",
     )[[DIMENSION_PAGE_PATH["alias"], METRIC_PAGE_VIEWS["alias"], METRIC_TOTAL_USERS["alias"]]].copy()
     if not ignore_index:
         df_response = df_response.set_index(DIMENSION_PAGE_PATH["alias"])
@@ -296,7 +296,7 @@ def get_index_table_download_df(analytics_params, ignore_index=True):
         [METRIC_EVENT_COUNT, METRIC_TOTAL_USERS],
         [DIMENSION_ENTITY_NAME, DIMENSION_RELATED_ENTITY_ID, DIMENSION_RELATED_ENTITY_NAME],
         **analytics_params,
-        dimension_filter=f"eventName=={EVENT_FILE_DOWNLOADED}",
+        dimension_filter=f"eventName=={EVENT_FILE_DOWNLOADED['id']}",
     )
     if not ignore_index:
         df_response = df_response.set_index([DIMENSION_ENTITY_NAME["alias"], DIMENSION_RELATED_ENTITY_ID["alias"], DIMENSION_RELATED_ENTITY_NAME["alias"]])
@@ -343,7 +343,7 @@ def get_index_entity_selected_df(analytics_params, ignore_index=True):
         [METRIC_EVENT_COUNT, METRIC_TOTAL_USERS],
         [DIMENSION_ENTITY_NAME_TAB],
         **analytics_params,
-        dimension_filter=f"eventName=={EVENT_ENTITY_SELECTED}",
+        dimension_filter=f"eventName=={EVENT_ENTITY_SELECTED['id']}",
     )
     if not ignore_index:
         df_response = df_response.set_index([DIMENSION_ENTITY_NAME_TAB["alias"]])
@@ -390,7 +390,7 @@ def get_index_entity_table_sorted_df(analytics_params, ignore_index=True):
         [METRIC_EVENT_COUNT, METRIC_TOTAL_USERS],
         [DIMENSION_ENTITY_NAME_TAB, DIMENSION_COLUMN_NAME, DIMENSION_SORT_DIRECTION],
         **analytics_params,
-        dimension_filter=f"eventName=={EVENT_ENTITY_TABLE_SORTED}",
+        dimension_filter=f"eventName=={EVENT_ENTITY_TABLE_SORTED['id']}",
     )
     if not ignore_index:
         df_response = df_response.set_index([DIMENSION_ENTITY_NAME_TAB["alias"], DIMENSION_COLUMN_NAME["alias"], DIMENSION_SORT_DIRECTION["alias"]])
@@ -438,7 +438,7 @@ def get_index_entity_table_paginated_df(analytics_params, ignore_index=True):
         [METRIC_EVENT_COUNT, METRIC_TOTAL_USERS],
         [DIMENSION_ENTITY_NAME_TAB, DIMENSION_PAGINATION_DIRECTION],
         **analytics_params,
-        dimension_filter=f"eventName=={EVENT_ENTITY_TABLE_PAGINATED}",
+        dimension_filter=f"eventName=={EVENT_ENTITY_TABLE_PAGINATED['id']}",
     )
     if not ignore_index:
         df_response = df_response.set_index([DIMENSION_ENTITY_NAME_TAB["alias"], DIMENSION_PAGINATION_DIRECTION["alias"]])
@@ -487,7 +487,7 @@ def get_index_filter_selected_df(analytics_params, ignore_index=True):
         [METRIC_EVENT_COUNT, METRIC_TOTAL_USERS],
         [DIMENSION_FILTER_NAME, DIMENSION_FILTER_VALUE],
         **analytics_params,
-        dimension_filter=f"eventName=={EVENT_FILTER_SELECTED}",
+        dimension_filter=f"eventName=={EVENT_FILTER_SELECTED['id']}",
     )
     if not ignore_index:
         df_response = df_response.set_index([DIMENSION_FILTER_NAME["alias"], DIMENSION_FILTER_VALUE["alias"]])
@@ -519,3 +519,25 @@ def get_index_filter_selected_change(analytics_params, start_current, end_curren
         sort_results=[METRIC_EVENT_COUNT, METRIC_TOTAL_USERS]
     )
 
+
+def get_event_count_over_time_df(analytics_params, events, additional_data_path=None, additional_data_behavior=None):
+    """
+    Get a DataFrame with pageviews and total active users over time from the Analytics API.
+
+    :param analytics_params: the parameters for the Analytics API, including service params, start dates, and end dates
+    :param additional_data_path: the path to a JSON file with additional data to be added to the DataFrame, defaults to None
+    :param additional_data_behavior: the behavior to use when adding the additional data, as an instance of ADDITIONAL_DATA_BEHAVIOR, defaults to None
+    :return: a DataFrame with the pageviews and total active users over time from the Analytics API. 
+        Columns are the dimension aliases, metrics (as ints), and change metrics (as floats)
+        Dimensions: DIMENSION_YEAR_MONTH (as a datetime)
+        Metrics: METRIC_ACTIVE_USERS, METRIC_PAGE_VIEWS
+    """
+    
+    return get_change_over_time_df_multiple_events(
+        METRIC_EVENT_COUNT,
+        events,
+        DIMENSION_YEAR_MONTH,
+        additional_data_path=additional_data_path,
+        additional_data_behavior=additional_data_behavior,
+        **analytics_params
+    )

analytics/analytics_package/setup.py

@@ -2,7 +2,7 @@
 
 setup(
 	name="analytics",
-	version="4.1.0",
+	version="4.2.0",
 	packages=["analytics"],
 	install_requires=["matplotlib", "pandas", "numpy", "google-auth-oauthlib", "google-api-python-client", "gspread", "gspread-formatting"],
 )