1+ #
2+ # Copyright 2018 Google LLC
3+ #
4+ # Licensed under the Apache License, Version 2.0 (the "License");
5+ # you may not use this file except in compliance with the License.
6+ # You may obtain a copy of the License at
7+ #
8+ # https://www.apache.org/licenses/LICENSE-2.0
9+ #
10+ # Unless required by applicable law or agreed to in writing, software
11+ # distributed under the License is distributed on an "AS IS" BASIS,
12+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+ # See the License for the specific language governing permissions and
14+ # limitations under the License.
15+ #
16+
17+ ---
18+ lowercaseOutputName : true
19+ attrNameSnakeCase : true
20+ rules :
21+ # These come from the application driver if it's a streaming application
22+ # Example: default/streaming.driver.com.example.ClassName.StreamingMetrics.streaming.lastCompletedBatch_schedulingDelay
23+ - pattern : metrics<name=(\S+)\.(\S+)\.driver\.(\S+)\.StreamingMetrics\.streaming\.(\S+)><>Value
24+ name : spark_streaming_driver_$4
25+ labels :
26+ app_namespace : " $1"
27+ app_id : " $2"
28+ # These come from the application driver if it's a structured streaming application
29+ # Example: default/sstreaming.driver.spark.streaming.QueryName.inputRate-total
30+ - pattern : metrics<name=(\S+)\.(\S+)\.driver\.spark\.streaming\.(\S+)\.(\S+)><>Value
31+ name : spark_structured_streaming_driver_$4
32+ labels :
33+ app_namespace : " $1"
34+ app_id : " $2"
35+ query_name : " $3"
36+ # These come from the application executors
37+ # Example: default/spark-pi.0.executor.threadpool.activeTasks
38+ - pattern : metrics<name=(\S+)\.(\S+)\.(\S+)\.executor\.(\S+)><>Value
39+ name : spark_executor_$4
40+ type : GAUGE
41+ labels :
42+ app_namespace : " $1"
43+ app_id : " $2"
44+ executor_id : " $3"
45+ # These come from the application driver
46+ # Example: default/spark-pi.driver.DAGScheduler.stage.failedStages
47+ - pattern : metrics<name=(\S+)\.(\S+)\.driver\.(BlockManager|DAGScheduler|jvm)\.(\S+)><>Value
48+ name : spark_driver_$3_$4
49+ type : GAUGE
50+ labels :
51+ app_namespace : " $1"
52+ app_id : " $2"
53+ # These come from the application driver
54+ # Emulate timers for DAGScheduler like messagePRocessingTime
55+ - pattern : metrics<name=(\S+)\.(\S+)\.driver\.DAGScheduler\.(.*)><>Count
56+ name : spark_driver_DAGScheduler_$3_count
57+ type : COUNTER
58+ labels :
59+ app_namespace : " $1"
60+ app_id : " $2"
61+ # HiveExternalCatalog is of type counter
62+ - pattern : metrics<name=(\S+)\.(\S+)\.driver\.HiveExternalCatalog\.(.*)><>Count
63+ name : spark_driver_HiveExternalCatalog_$3_count
64+ type : COUNTER
65+ labels :
66+ app_namespace : " $1"
67+ app_id : " $2"
68+ # These come from the application driver
69+ # Emulate histograms for CodeGenerator
70+ - pattern : metrics<name=(\S+)\.(\S+)\.driver\.CodeGenerator\.(.*)><>Count
71+ name : spark_driver_CodeGenerator_$3_count
72+ type : COUNTER
73+ labels :
74+ app_namespace : " $1"
75+ app_id : " $2"
76+ # These come from the application driver
77+ # Emulate timer (keep only count attribute) plus counters for LiveListenerBus
78+ - pattern : metrics<name=(\S+)\.(\S+)\.driver\.LiveListenerBus\.(.*)><>Count
79+ name : spark_driver_LiveListenerBus_$3_count
80+ type : COUNTER
81+ labels :
82+ app_namespace : " $1"
83+ app_id : " $2"
84+ # Get Gauge type metrics for LiveListenerBus
85+ - pattern : metrics<name=(\S+)\.(\S+)\.driver\.LiveListenerBus\.(.*)><>Value
86+ name : spark_driver_LiveListenerBus_$3
87+ type : GAUGE
88+ labels :
89+ app_namespace : " $1"
90+ app_id : " $2"
91+ # Executors counters
92+ - pattern : metrics<name=(\S+)\.(\S+)\.(.*)\.executor\.(.*)><>Count
93+ name : spark_executor_$4_count
94+ type : COUNTER
95+ labels :
96+ app_namespace : " $1"
97+ app_id : " $2"
98+ executor_id : " $3"
99+ # These come from the application executors
100+ # Example: app-20160809000059-0000.0.jvm.threadpool.activeTasks
101+ - pattern : metrics<name=(\S+)\.(\S+)\.([0-9]+)\.(jvm|NettyBlockTransfer)\.(.*)><>Value
102+ name : spark_executor_$4_$5
103+ type : GAUGE
104+ labels :
105+ app_namespace : " $1"
106+ app_id : " $2"
107+ executor_id : " $3"
108+ - pattern : metrics<name=(\S+)\.(\S+)\.([0-9]+)\.HiveExternalCatalog\.(.*)><>Count
109+ name : spark_executor_HiveExternalCatalog_$4_count
110+ type : COUNTER
111+ labels :
112+ app_namespace : " $1"
113+ app_id : " $2"
114+ executor_id : " $3"
115+ # These come from the application driver
116+ # Emulate histograms for CodeGenerator
117+ - pattern : metrics<name=(\S+)\.(\S+)\.([0-9]+)\.CodeGenerator\.(.*)><>Count
118+ name : spark_executor_CodeGenerator_$4_count
119+ type : COUNTER
120+ labels :
121+ app_namespace : " $1"
122+ app_id : " $2"
123+ executor_id : " $3"
0 commit comments