YQ kqprun added scripts for starting prometheus and connector

ydb/tests/tools/fqrun/flame_graph.sh

@@ -9,4 +9,4 @@ fi
 
 SCRIPT_DIR=$(cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd)
 
-$SCRIPT_DIR/../kqprun/flame_graph.sh ${1:-'30'} ${2:-''} fqrun
+$SCRIPT_DIR/../kqprun/scripts/flame_graph.sh ${1:-'30'} ${2:-''} fqrun

ydb/tests/tools/kqprun/README.md

@@ -8,7 +8,34 @@ For profiling memory allocations build kqprun with ya make flag `-D PROFILE_MEMO
 
 * `flame_graph.sh` - script for collecting flame graphs in svg format, usage:
     ```(bash)
-    ./flame_graph.sh [graph collection time in seconds] [use sudo]
+    ./scripts/flame_graph.sh [graph collection time in seconds] [use sudo]
+    ```
+
+* `start_prometheus.sh` - start prometheus web UI, can be used for counters visualisation (kqprun should be runned with flag `-M <monitoring port>`), usage:
+    ```(bash)
+    ./scripts/start_prometheus.sh <kqprun monitoring port> <web UI port> [config path]
+    ```
+    Prometheus UI available on `http://localhost:<web UI port>/targets`
+
+* `start_grafana.sh` - start grafana other existing prometheus, usage:
+    ```(bash)
+    ./scripts/start_grafana.sh <prometheus port> <web UI port> [additional dashboards dirs]
+    ```
+
+    Command for starting prometheus and grafana:
+    ```(bash)
+    ./scripts/start_prometheus.sh 32000 32001 && ./scripts/start_grafana.sh 32001 32002
+    ```
+    Where 32000 - kqprun monitoring port, graphana UI available on http://localhost:32002 (login: `admin`, password: `admin`)
+
+* `start_connector.sh` - start local FQ connector, usage:
+    ```(bash)
+    ./scripts/start_connector.sh <connector port>
+    ```
+
+* `cleanup_docker.sh` - stop created docker containers, usege:
+    ```(bash)
+    ./scripts/cleanup_docker.sh [name filter]
     ```
 
 ## Examples

ydb/tests/tools/kqprun/configuration/app_config.conf

@@ -55,6 +55,7 @@ FeatureFlags {
   EnableExternalSourceSchemaInference: true
   EnableTempTables: true
   EnableReplaceIfExistsForExternalEntities: true
+  EnableResourcePoolsCounters: true
 }
 
 KQPConfig {

ydb/tests/tools/kqprun/configuration/prometheus_config.yaml

@@ -0,0 +1,75 @@
+global:
+  scrape_interval: 5s
+  evaluation_interval: 5s
+
+scrape_configs:
+  - job_name: "ydb_dynamic"
+    metrics_path: "/counters/counters=ydb/name_label=name/prometheus"
+    static_configs:
+      - targets: ["localhost:${TARGET_PORT}"]
+        labels:
+          container: ydb-dynamic
+    metric_relabel_configs:
+      - source_labels: ["__name__"]
+        target_label: "__name__"
+        replacement: "ydb_$1"
+  - job_name: "utils_dynamic"
+    metrics_path: "/counters/counters=utils/prometheus"
+    static_configs:
+      - targets: ["localhost:${TARGET_PORT}"]
+        labels:
+          container: ydb-dynamic
+    metric_relabel_configs:
+      - source_labels: ["__name__"]
+        target_label: "__name__"
+        replacement: "utils_$1"
+  - job_name: "kqp_dynamic"
+    metrics_path: "/counters/counters=kqp/prometheus"
+    static_configs:
+      - targets: ["localhost:${TARGET_PORT}"]
+        labels:
+          container: ydb-dynamic
+    metric_relabel_configs:
+      - source_labels: ["__name__"]
+        target_label: "__name__"
+        replacement: "kqp_$1"
+  - job_name: "tablets_dynamic"
+    metrics_path: "/counters/counters=tablets/prometheus"
+    static_configs:
+      - targets: ["localhost:${TARGET_PORT}"]
+        labels:
+          container: ydb-dynamic
+    metric_relabel_configs:
+      - source_labels: ["__name__"]
+        target_label: "__name__"
+        replacement: "tablets_$1"
+  - job_name: "proxy_dynamic"
+    metrics_path: "/counters/counters=proxy/prometheus"
+    static_configs:
+      - targets: ["localhost:${TARGET_PORT}"]
+        labels:
+          container: ydb-dynamic
+    metric_relabel_configs:
+      - source_labels: ["__name__"]
+        target_label: "__name__"
+        replacement: "proxy_$1"
+  - job_name: "dsproxynode_dynamic"
+    metrics_path: "/counters/counters=dsproxynode/prometheus"
+    static_configs:
+      - targets: ["localhost:${TARGET_PORT}"]
+        labels:
+          container: ydb-dynamic
+    metric_relabel_configs:
+      - source_labels: ["__name__"]
+        target_label: "__name__"
+        replacement: "dsproxynode_$1"
+  - job_name: "ic_dynamic"
+    metrics_path: "/counters/counters=interconnect/prometheus"
+    static_configs:
+      - targets: ["localhost:${TARGET_PORT}"]
+        labels:
+          container: ydb-dynamic
+    metric_relabel_configs:
+      - source_labels: ["__name__"]
+        target_label: "__name__"
+        replacement: "interconnect_$1"

ydb/tests/tools/kqprun/scripts/cleanup_docker.sh

@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+
+set -eux
+
+if [ $# -gt 1 ]; then
+    echo "Too many arguments"
+    exit -1
+fi
+
+KQPRUN_CONTAINERS=$(docker container ls -q --filter name=${1:-$USER-kqprun-*})
+
+if [ "$KQPRUN_CONTAINERS" ]; then
+    docker container stop $KQPRUN_CONTAINERS
+else
+    echo "Nothing to cleanup"
+fi

ydb/tests/tools/kqprun/scripts/flame_graph.sh

@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+
+set -eux
+
+if [ $# -gt 3 ]; then
+    echo "Too many arguments"
+    exit -1
+fi
+
+SUDO=""
+
+if [ ${2:-''} ]; then
+    SUDO="sudo"
+fi
+
+SCRIPT_DIR=$(cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd)
+
+function cleanup {
+    $SUDO rm $SCRIPT_DIR/profdata
+    rm $SCRIPT_DIR/profdata.txt
+}
+trap cleanup EXIT
+
+TARGER_PID=$(pgrep -u $USER ${3:-'kqprun'})
+
+$SUDO perf record -F 50 --call-graph dwarf -g --proc-map-timeout=10000 --pid $TARGER_PID -v -o $SCRIPT_DIR/profdata -- sleep ${1:-'30'}
+$SUDO perf script -i $SCRIPT_DIR/profdata > $SCRIPT_DIR/profdata.txt
+
+flame_graph_tool="$SCRIPT_DIR/../../../../../contrib/tools/flame-graph/"
+
+${flame_graph_tool}/stackcollapse-perf.pl $SCRIPT_DIR/profdata.txt | ${flame_graph_tool}/flamegraph.pl > ./profdata.svg

ydb/tests/tools/kqprun/scripts/start_connector.sh

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+
+set -eux
+
+if [ $# -le 0 ]; then
+    echo "Please provide connector port (value of Generic::Connector::Endpoint::Port in config)"
+    exit -1
+fi
+
+if [ $# -gt 2 ]; then
+    echo "Too many arguments"
+    exit -1
+fi
+
+SCRIPT_DIR=$(cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd)
+CONTAINER_NAME=$USER-kqprun-fq-connector-go-$1
+
+$SCRIPT_DIR/cleanup_docker.sh $CONTAINER_NAME
+docker pull ghcr.io/ydb-platform/fq-connector-go:latest
+docker run -d --rm --name=$CONTAINER_NAME --network host ghcr.io/ydb-platform/fq-connector-go:latest --connector-port=$1 --metrics-port=$(($1 + 1)) --pprof-port=$(($1 + 2))
+docker container ls --filter name=$CONTAINER_NAME

ydb/tests/tools/kqprun/scripts/start_grafana.sh

@@ -0,0 +1,49 @@
+#!/usr/bin/env bash
+
+set -eux
+
+if [ $# -le 0 ]; then
+    echo "Please provide prometheus port"
+    exit -1
+fi
+
+if [ $# -le 1 ]; then
+    echo "Please provide grafana web ui port"
+    exit -1
+fi
+
+SCRIPT_DIR=$(cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd)
+DASHBOARDS_DIRS="$SCRIPT_DIR/../../../../deploy/helm/ydb-prometheus/dashboards/"
+if [ $# -gt 2 ]; then
+    DASHBOARDS_DIRS="$DASHBOARDS_DIRS $3"
+fi
+
+if [ $# -gt 3 ]; then
+    echo "Too many arguments"
+    exit -1
+fi
+
+CONTAINER_NAME=$USER-kqprun-grafana-$1-$2
+
+$SCRIPT_DIR/cleanup_docker.sh $CONTAINER_NAME
+docker run -d --rm --name=$CONTAINER_NAME --network host -e "GF_SERVER_HTTP_PORT=$2" grafana/grafana-oss
+docker container ls --filter name=$CONTAINER_NAME
+
+echo "Wait grafana is up"
+
+(set -eux; while ! nc -v -z -w 1 localhost $2; do sleep 2; done)
+
+echo "Initialization of datasource and dashboards"
+
+GRAFANA_API="http://admin:admin@localhost:$2/api"
+
+curl -X POST -H "Content-Type: application/json" ${GRAFANA_API}/datasources --data-ascii "{ \"name\": \"prometheus_datasource_$1\", \"type\": \"prometheus\", \"url\": \"http://localhost:$1\", \"access\": \"proxy\" }"
+curl -X POST -H "Content-Type: application/json" ${GRAFANA_API}/folders --data-ascii '{ "uid": "ydb", "title": "YDB" }'
+
+for DASHBOARDS_DIR in $DASHBOARDS_DIRS; do
+    for DASH in "$DASHBOARDS_DIR"/*; do
+        jq '{ folderUid: "ydb", dashboard: . }' < "$DASH" | curl -X POST -H "Content-Type: application/json" ${GRAFANA_API}/dashboards/db -d @-
+    done
+done
+
+echo "Initialization finished successfully"

ydb/tests/tools/kqprun/scripts/start_prometheus.sh

@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+
+set -eux
+
+if [ $# -le 0 ]; then
+    echo "Please provide monitoring port to listen (value of -M argument)"
+    exit -1
+fi
+
+if [ $# -le 1 ]; then
+    echo "Please provide prometheus web ui port"
+    exit -1
+fi
+
+if [ $# -gt 3 ]; then
+    echo "Too many arguments"
+    exit -1
+fi
+
+SCRIPT_DIR=$(cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd)
+
+function cleanup {
+    rm $SCRIPT_DIR/prometheus_config.yaml
+}
+trap cleanup EXIT
+
+sed "s/\${TARGET_PORT}/$1/g" "${3:-$SCRIPT_DIR/../configuration/prometheus_config.yaml}" > $SCRIPT_DIR/prometheus_config.yaml
+
+CONTAINER_NAME=$USER-kqprun-prometheus-$1-$2
+
+$SCRIPT_DIR/cleanup_docker.sh $CONTAINER_NAME
+docker run -d --rm --name=$CONTAINER_NAME --network host -v $SCRIPT_DIR/prometheus_config.yaml:/etc/prometheus/prometheus.yml prom/prometheus --config.file=/etc/prometheus/prometheus.yml --web.listen-address=:$2
+docker container ls --filter name=$CONTAINER_NAME