+{"datasets":[{"name":"0ef48946","displayName":"ClassicId_Param","query":"SELECT \n u.usage_metadata.job_id AS job_id\nFROM\n system.billing.usage u\nWHERE\n u.billing_origin_product = 'JOBS'\n AND u.ingestion_date >= :ingestion_date\n AND u.product_features.is_serverless = false\n AND u.workspace_id IN (\n SELECT DISTINCT jt.workspace_id\n FROM system.lakeflow.job_run_timeline jt\n )\nORDER BY\n u.usage_end_time DESC;","parameters":[{"displayName":"ingestion_date","keyword":"ingestion_date","dataType":"DATE","defaultSelection":{"values":{"dataType":"DATE","values":[{"value":"now-30d/d"}]}}}]},{"name":"94f5c979","displayName":"ClassicRun_Param","query":"SELECT \n u.usage_metadata.job_run_id AS job_run_id\nFROM\n system.billing.usage u\nWHERE\n u.billing_origin_product = 'JOBS'\n AND u.usage_metadata.job_id = :job_id\n AND u.workspace_id IN (\n SELECT DISTINCT jt.workspace_id\n FROM system.lakeflow.job_run_timeline jt\n )\nORDER BY\n u.usage_metadata.job_run_id DESC;\n","parameters":[{"displayName":"job_id","keyword":"job_id","dataType":"STRING","defaultSelection":{"values":{"dataType":"STRING","values":[{"value":"785218276376078"}]}}}]},{"name":"89820d4d","displayName":"ServerlessRun_Param","query":"SELECT \n u.usage_metadata.job_run_id AS job_run_id\nFROM\n system.billing.usage u\nWHERE\n u.billing_origin_product = 'JOBS'\n AND u.usage_metadata.job_id = :job_id\n AND u.workspace_id IN (\n SELECT DISTINCT jt.workspace_id\n FROM system.lakeflow.job_run_timeline jt\n )\nORDER BY\n u.usage_metadata.job_run_id DESC;\n","parameters":[{"displayName":"job_id","keyword":"job_id","dataType":"STRING","defaultSelection":{"values":{"dataType":"STRING","values":[{"value":"687993197774248"}]}}}]},{"name":"0bd9b266","displayName":"ServerlessId_Param","query":"SELECT \n u.usage_metadata.job_id AS job_id\nFROM\n system.billing.usage u\nWHERE\n u.billing_origin_product = 'JOBS'\n AND u.ingestion_date >= :ingestion_date\n AND u.product_features.is_serverless = true\n AND u.workspace_id IN (\n SELECT DISTINCT jt.workspace_id\n FROM system.lakeflow.job_run_timeline jt\n )\nORDER BY\n u.usage_end_time DESC;","parameters":[{"displayName":"ingestion_date","keyword":"ingestion_date","dataType":"DATE","defaultSelection":{"values":{"dataType":"DATE","values":[{"value":"now-30d/d"}]}}}]},{"name":"d5ffe2af","displayName":"JoinedRunInfo","query":"WITH classic_base_data AS (\n SELECT \n u.usage_metadata.job_run_id AS job_run_id,\n 'Classic' AS job_type,\n SUM(u.usage_quantity * p.pricing.effective_list.default) AS total_cost,\n (to_unix_timestamp(MAX(jt.period_end_time)) - to_unix_timestamp(MIN(jt.period_start_time))) / 60.0 AS run_duration_minutes,\n MIN(jt.period_start_time) AS run_start_time\n FROM\n system.billing.usage u\n JOIN system.billing.list_prices p \n ON u.sku_name = p.sku_name\n JOIN system.lakeflow.job_run_timeline jt \n ON u.usage_metadata.job_run_id = jt.run_id \n AND u.workspace_id = jt.workspace_id\n WHERE\n u.billing_origin_product = 'JOBS'\n AND u.ingestion_date >= :ingestion_date\n AND p.price_end_time IS NULL\n AND jt.termination_code = 'SUCCESS'\n AND u.usage_metadata.job_id = :classic_job_id\n AND (array_contains(:ClassicRun_Param, 'All') OR array_contains(:ClassicRun_Param, CAST(u.usage_metadata.job_run_id AS STRING)))\n GROUP BY u.usage_metadata.job_run_id\n),\n\nserverless_base_data AS (\n SELECT \n u.usage_metadata.job_run_id AS job_run_id,\n 'Serverless' AS job_type,\n SUM(u.usage_quantity * p.pricing.effective_list.default) AS total_cost,\n (to_unix_timestamp(MAX(jt.period_end_time)) - to_unix_timestamp(MIN(jt.period_start_time))) / 60.0 AS run_duration_minutes,\n MIN(jt.period_start_time) AS run_start_time\n FROM\n system.billing.usage u\n JOIN system.billing.list_prices p \n ON u.sku_name = p.sku_name\n JOIN system.lakeflow.job_run_timeline jt \n ON u.usage_metadata.job_run_id = jt.run_id \n AND u.workspace_id = jt.workspace_id\n WHERE\n u.billing_origin_product = 'JOBS'\n AND u.ingestion_date >= :ingestion_date\n AND p.price_end_time IS NULL\n AND jt.termination_code = 'SUCCESS'\n AND u.usage_metadata.job_id = :serverless_job_id\n AND (array_contains(:ServerlessRun_Param, 'All') OR array_contains(:ServerlessRun_Param, CAST(u.usage_metadata.job_run_id AS STRING)))\n GROUP BY u.usage_metadata.job_run_id\n),\n\nunion_data AS (\n SELECT * FROM classic_base_data\n UNION ALL\n SELECT * FROM serverless_base_data\n),\n\nranked_data AS (\n SELECT\n job_run_id,\n job_type,\n total_cost,\n run_duration_minutes,\n run_start_time,\n ROW_NUMBER() OVER (PARTITION BY job_type ORDER BY run_start_time) AS run_sequence\n FROM union_data\n)\n\nSELECT \n r.job_run_id,\n r.job_type,\n r.run_sequence,\n r.run_duration_minutes,\n r.total_cost\nFROM ranked_data r\nORDER BY r.job_type, r.run_sequence;","parameters":[{"displayName":"ingestion_date","keyword":"ingestion_date","dataType":"DATE","defaultSelection":{"values":{"dataType":"DATE","values":[{"value":"now-30d/d"}]}}},{"displayName":"classic_job_id","keyword":"classic_job_id","dataType":"STRING","defaultSelection":{"values":{"dataType":"STRING","values":[{"value":"785218276376078"}]}}},{"displayName":"serverless_job_id","keyword":"serverless_job_id","dataType":"STRING","defaultSelection":{"values":{"dataType":"STRING","values":[{"value":"687993197774248"}]}}},{"displayName":"classic_runs","keyword":"ClassicRun_Param","dataType":"STRING","complexType":"MULTI","defaultSelection":{"values":{"dataType":"STRING","values":[{"value":"All"}]}}},{"displayName":"serverless_runs","keyword":"ServerlessRun_Param","dataType":"STRING","complexType":"MULTI","defaultSelection":{"values":{"dataType":"STRING","values":[{"value":"All"}]}}}]}],"pages":[{"name":"9d1433ba","displayName":"Job Comparison","layout":[{"widget":{"name":"3aa6323c","textbox_spec":"## Serverless Job Migration Dashboard\nThis dashboard helps evaluate the potential benefits of migrating a job from Classic to Serverless. It provides side-by-side comparisons of job run duration and estimated cost for both workloads. \n\nCost estimates are calculated based on the `system.billing.list_prices` table and reflect approximate pricing using the current list prices. \n\n**Note. This dashboard uses the following system tables:**\n- `system.billing.usage`\n- `system.billing.list_prices`\n- `system.lakeflow.job_run_timeline`\n\n"},"position":{"x":0,"y":0,"width":4,"height":5}},{"widget":{"name":"4f573276","queries":[{"name":"parameter_dashboards/01f045649be915bebfd3f1c9e1275265/datasets/01f04567fb191a96a290f90253a01495_ingestion_date","query":{"datasetName":"0bd9b266","parameters":[{"name":"ingestion_date","keyword":"ingestion_date"}],"disaggregated":false}},{"name":"parameter_dashboards/01f045649be915bebfd3f1c9e1275265/datasets/01f04564a46b108fab030d40006cd1ed_ingestion_date","query":{"datasetName":"0ef48946","parameters":[{"name":"ingestion_date","keyword":"ingestion_date"}],"disaggregated":false}}],"spec":{"version":2,"widgetType":"filter-date-picker","encodings":{"fields":[{"parameterName":"ingestion_date","queryName":"parameter_dashboards/01f045649be915bebfd3f1c9e1275265/datasets/01f04567fb191a96a290f90253a01495_ingestion_date"},{"parameterName":"ingestion_date","queryName":"parameter_dashboards/01f045649be915bebfd3f1c9e1275265/datasets/01f04564a46b108fab030d40006cd1ed_ingestion_date"}]},"frame":{"showTitle":true,"title":"Ingestion Date Cutoff:","showDescription":true,"description":"Sets the ingestion date filter. Only job runs ingested on or after this date will be included in the comparison."}}},"position":{"x":4,"y":2,"width":2,"height":3}},{"widget":{"name":"245a4a57","textbox_spec":"### Classic Jobs\nEnter the source `job_id` you're looking to migrate."},"position":{"x":0,"y":5,"width":3,"height":2}},{"widget":{"name":"ec38898d","textbox_spec":"### Serverless Jobs\nEnter the migrated (serverless) `job_id`. Make sure this job has at least a few succesful runs to allow for meaningful comparison."},"position":{"x":3,"y":5,"width":3,"height":2}},{"widget":{"name":"e559ad49","queries":[{"name":"dashboards/01f045649be915bebfd3f1c9e1275265/datasets/01f04564a46b108fab030d40006cd1ed_job_id","query":{"datasetName":"0ef48946","fields":[{"name":"job_id","expression":"`job_id`"},{"name":"job_id_associativity","expression":"COUNT_IF(`associative_filter_predicate_group`)"}],"disaggregated":false}},{"name":"parameter_dashboards/01f045649be915bebfd3f1c9e1275265/datasets/01f046fdab461621a41d565ddb54e42e_classic_job_id","query":{"datasetName":"d5ffe2af","parameters":[{"name":"classic_job_id","keyword":"classic_job_id"}],"disaggregated":false}},{"name":"parameter_dashboards/01f045649be915bebfd3f1c9e1275265/datasets/01f048811e4a19e4994cb40d12032c11_job_id","query":{"datasetName":"94f5c979","parameters":[{"name":"job_id","keyword":"job_id"}],"disaggregated":false}}],"spec":{"version":2,"widgetType":"filter-single-select","encodings":{"fields":[{"fieldName":"job_id","displayName":"job_id","queryName":"dashboards/01f045649be915bebfd3f1c9e1275265/datasets/01f04564a46b108fab030d40006cd1ed_job_id"},{"parameterName":"classic_job_id","queryName":"parameter_dashboards/01f045649be915bebfd3f1c9e1275265/datasets/01f046fdab461621a41d565ddb54e42e_classic_job_id"},{"parameterName":"job_id","queryName":"parameter_dashboards/01f045649be915bebfd3f1c9e1275265/datasets/01f048811e4a19e4994cb40d12032c11_job_id"}]},"selection":{"defaultSelection":{"operator":{"operator":"OR","args":[]}}},"disallowAll":true,"frame":{"showTitle":true,"title":"Job Id"}}},"position":{"x":0,"y":7,"width":3,"height":1}},{"widget":{"name":"6e9ef33e","queries":[{"name":"dashboards/01f045649be915bebfd3f1c9e1275265/datasets/01f04567fb191a96a290f90253a01495_job_id","query":{"datasetName":"0bd9b266","fields":[{"name":"job_id","expression":"`job_id`"},{"name":"job_id_associativity","expression":"COUNT_IF(`associative_filter_predicate_group`)"}],"disaggregated":false}},{"name":"parameter_dashboards/01f045649be915bebfd3f1c9e1275265/datasets/01f046fdab461621a41d565ddb54e42e_serverless_job_id","query":{"datasetName":"d5ffe2af","parameters":[{"name":"serverless_job_id","keyword":"serverless_job_id"}],"disaggregated":false}},{"name":"parameter_dashboards/01f045649be915bebfd3f1c9e1275265/datasets/01f048811ec31920b2d0364e24616aa2_job_id","query":{"datasetName":"89820d4d","parameters":[{"name":"job_id","keyword":"job_id"}],"disaggregated":false}}],"spec":{"version":2,"widgetType":"filter-single-select","encodings":{"fields":[{"fieldName":"job_id","displayName":"job_id","queryName":"dashboards/01f045649be915bebfd3f1c9e1275265/datasets/01f04567fb191a96a290f90253a01495_job_id"},{"parameterName":"serverless_job_id","queryName":"parameter_dashboards/01f045649be915bebfd3f1c9e1275265/datasets/01f046fdab461621a41d565ddb54e42e_serverless_job_id"},{"parameterName":"job_id","queryName":"parameter_dashboards/01f045649be915bebfd3f1c9e1275265/datasets/01f048811ec31920b2d0364e24616aa2_job_id"}]},"selection":{"defaultSelection":{"operator":{"operator":"OR","args":[]}}},"disallowAll":true,"frame":{"showTitle":true,"title":"Job Id"}}},"position":{"x":3,"y":7,"width":3,"height":1}},{"widget":{"name":"8bf29a35","queries":[{"name":"main_query","query":{"datasetName":"d5ffe2af","fields":[{"name":"job_type","expression":"`job_type`"},{"name":"run_sequence","expression":"`run_sequence`"},{"name":"sum(run_duration_minutes)","expression":"SUM(`run_duration_minutes`)"}],"disaggregated":false}}],"spec":{"version":3,"widgetType":"line","encodings":{"x":{"fieldName":"run_sequence","scale":{"type":"categorical"},"displayName":"Run Number"},"y":{"fieldName":"sum(run_duration_minutes)","scale":{"type":"quantitative","fn":{"type":"symlog"}},"displayName":"Duration (Minutes)"},"color":{"fieldName":"job_type","scale":{"type":"categorical"},"displayName":"Job Type"},"label":{"show":false}},"frame":{"showTitle":true,"title":"Job Run Durations: Classic vs. Serverless","showDescription":true,"description":"Comparison of job run durations (in minutes) across executions in the specified date parameter. The X-axis represents the sequence of executions for each job type ordered by execution start time. "}}},"position":{"x":0,"y":8,"width":6,"height":7}},{"widget":{"name":"99037c53","queries":[{"name":"main_query","query":{"datasetName":"d5ffe2af","fields":[{"name":"job_type","expression":"`job_type`"},{"name":"run_sequence","expression":"`run_sequence`"},{"name":"sum(total_cost)","expression":"SUM(`total_cost`)"}],"disaggregated":false}}],"spec":{"version":3,"widgetType":"line","encodings":{"x":{"fieldName":"run_sequence","scale":{"type":"categorical"},"axis":{"title":"Run Number"},"displayName":"Run Number"},"y":{"fieldName":"sum(total_cost)","scale":{"type":"quantitative","fn":{"type":"symlog"}},"format":{"type":"number-currency","currencyCode":"USD","abbreviation":"compact","decimalPlaces":{"type":"max","places":2}},"axis":{"title":"Cost ($)"},"displayName":"Classic"},"color":{"fieldName":"job_type","scale":{"type":"categorical"},"displayName":"Job Type"},"label":{"show":false}},"frame":{"showTitle":true,"showDescription":true,"title":"Job Run Cost: Classic vs. Serverless","description":"This chart compares the cost of each job run for both Classic and Serverless workloads, ordered by run sequence. Costs are shown in dollars and help highlight relative trends and variability across multiple executions."}}},"position":{"x":0,"y":15,"width":6,"height":7}},{"widget":{"name":"db39c7a0","queries":[{"name":"dashboards/01f045649be915bebfd3f1c9e1275265/datasets/01f048811e4a19e4994cb40d12032c11_job_run_id","query":{"datasetName":"94f5c979","fields":[{"name":"job_run_id","expression":"`job_run_id`"},{"name":"job_run_id_associativity","expression":"COUNT_IF(`associative_filter_predicate_group`)"}],"disaggregated":false}},{"name":"parameter_dashboards/01f045649be915bebfd3f1c9e1275265/datasets/01f046fdab461621a41d565ddb54e42e_ClassicRun_Param","query":{"datasetName":"d5ffe2af","parameters":[{"name":"ClassicRun_Param","keyword":"ClassicRun_Param"}],"disaggregated":false}}],"spec":{"version":2,"widgetType":"filter-multi-select","encodings":{"fields":[{"fieldName":"job_run_id","displayName":"job_run_id","queryName":"dashboards/01f045649be915bebfd3f1c9e1275265/datasets/01f048811e4a19e4994cb40d12032c11_job_run_id"},{"parameterName":"ClassicRun_Param","queryName":"parameter_dashboards/01f045649be915bebfd3f1c9e1275265/datasets/01f046fdab461621a41d565ddb54e42e_ClassicRun_Param"}]},"frame":{"showTitle":true,"title":"Job Runs"}}},"position":{"x":0,"y":8,"width":3,"height":1}},{"widget":{"name":"9f66e6b7","queries":[{"name":"dashboards/01f045649be915bebfd3f1c9e1275265/datasets/01f048811ec31920b2d0364e24616aa2_job_run_id","query":{"datasetName":"89820d4d","fields":[{"name":"job_run_id","expression":"`job_run_id`"},{"name":"job_run_id_associativity","expression":"COUNT_IF(`associative_filter_predicate_group`)"}],"disaggregated":false}},{"name":"parameter_dashboards/01f045649be915bebfd3f1c9e1275265/datasets/01f046fdab461621a41d565ddb54e42e_ServerlessRun_Param","query":{"datasetName":"d5ffe2af","parameters":[{"name":"ServerlessRun_Param","keyword":"ServerlessRun_Param"}],"disaggregated":false}}],"spec":{"version":2,"widgetType":"filter-multi-select","encodings":{"fields":[{"fieldName":"job_run_id","displayName":"job_run_id","queryName":"dashboards/01f045649be915bebfd3f1c9e1275265/datasets/01f048811ec31920b2d0364e24616aa2_job_run_id"},{"parameterName":"ServerlessRun_Param","queryName":"parameter_dashboards/01f045649be915bebfd3f1c9e1275265/datasets/01f046fdab461621a41d565ddb54e42e_ServerlessRun_Param"}]},"frame":{"showTitle":true,"title":"Job Runs"}}},"position":{"x":3,"y":8,"width":3,"height":1}}],"pageType":"PAGE_TYPE_CANVAS"}],"uiSettings":{"theme":{}}}
0 commit comments