Introducing monitoring

.env.example

@@ -51,6 +51,10 @@ MBIN_SIDEBAR_SECTIONS_RANDOM_LOCAL_ONLY=false
 MBIN_SIDEBAR_SECTIONS_USERS_LOCAL_ONLY=false
 # Captcha (also enable in admin panel/settings)
 KBIN_CAPTCHA_ENABLED=false
+# Whether requests and messages should be monitored for performance. If enabled this could impact performance.
+MBIN_MONITORING_ENABLED=false
+# Whether the parameter of database queries should be saved. If enabled the spaces used might increase a lot.
+MBIN_MONITORING_QUERY_PARAMETERS_ENABLED=false
 
 ###> meteo-concept/hcaptcha-bundle ###
 HCAPTCHA_SITE_KEY=

.env.example_docker

@@ -56,6 +56,10 @@ MBIN_SIDEBAR_SECTIONS_RANDOM_LOCAL_ONLY=false
 MBIN_SIDEBAR_SECTIONS_USERS_LOCAL_ONLY=false
 # Captcha (also enable in admin panel/settings)
 KBIN_CAPTCHA_ENABLED=false
+# Whether requests and messages should be monitored for performance. If enabled this could impact performance.
+MBIN_MONITORING_ENABLED=false
+# Whether the parameter of database queries should be saved. If enabled the spaces used might increase a lot.
+MBIN_MONITORING_QUERY_PARAMETERS_ENABLED=false
 
 ###> meteo-concept/hcaptcha-bundle ###
 HCAPTCHA_SITE_KEY=

assets/styles/app.scss

@@ -35,6 +35,7 @@
 @use 'components/suggestions';
 @use 'components/login';
 @use 'components/modlog';
+@use 'components/monitoring';
 @use 'components/notification_switch';
 @use 'components/notifications';
 @use 'components/messages';

assets/styles/components/_monitoring.scss

@@ -0,0 +1,73 @@
+.page-admin-monitoring {
+  h1, h2, h3, h4, h5, h6 {
+    margin-top: 0;
+  }
+  .monitoring-twig-render {
+    .children {
+      margin-left: 2rem;
+    }
+  }
+
+  .more {
+    background: var(--kbin-bg);
+    cursor: pointer;
+    text-align: center;
+    width: 100%;
+    margin-top: 1rem;
+
+    // bigger button for touch devices
+    @media (pointer:none), (pointer:coarse) {
+      margin-top: 2rem;
+      padding: 0.5rem;
+    }
+
+    i {
+      padding: .35rem;
+      pointer-events: none;
+    }
+
+    .rounded-edges & {
+      border-radius: var(--kbin-rounded-edges-radius);
+    }
+  }
+
+  input[type=text],
+  input[type=datetime-local],
+  select {
+    padding: 0.65rem;
+    width: 100%;
+  }
+
+  form div {
+    margin-bottom: .25rem;
+  }
+
+  table tr {
+    vertical-align: top;
+    td.query * {
+      margin: 0;
+      overflow: hidden;
+    }
+  }
+
+  .row {
+    display: flex;
+    flex-direction: row;
+
+    .col {
+      flex: 1 1;
+      margin-bottom: 0;
+      padding: 0 .25rem;
+    }
+
+    .col-auto {
+      flex: 0 0 auto;
+      margin-bottom: 0;
+    }
+
+    .btn-col {
+      text-align: right;
+      margin: auto;
+    }
+  }
+}

config/mbin_routes/admin.yaml

@@ -117,6 +117,17 @@ admin_cc:
     path: /admin/cc
     methods: [GET]
 
+admin_monitoring:
+    controller: App\Controller\Admin\AdminMonitoringController::overview
+    path: /admin/monitoring
+    methods: [GET]
+
+admin_monitoring_single_context:
+    controller: App\Controller\Admin\AdminMonitoringController::single
+    defaults: { page: overview }
+    path: /admin/monitoring/{id}/{page}
+    methods: [GET]
+
 admin_dashboard:
     controller: App\Controller\Admin\AdminDashboardController
     path: /admin/{statsPeriod}/{withFederated}

config/services.yaml

@@ -116,6 +116,9 @@ parameters:
     mbin_new_users_need_approval: '%env(bool:default::MBIN_NEW_USERS_NEED_APPROVAL)%'
     mbin_use_federation_allow_list: '%env(bool:default::MBIN_USE_FEDERATION_ALLOW_LIST)%'
 
+    mbin_monitoring_enabled: '%env(bool:default::MBIN_MONITORING_ENABLED)%'
+    mbin_monitoring_query_parameters_enabled: '%env(bool:default::MBIN_MONITORING_QUERY_PARAMETERS_ENABLED)%'
+
 services:
     # default configuration for services in *this* file
     _defaults:
@@ -130,6 +133,8 @@ services:
             $kbinApiItemsPerPage: '%kbin_api_items_per_page%'
             $storageUrl: '%kbin_storage_url%'
             $publicDir: '%kernel.project_dir%/public'
+            $monitoringEnabled: '%mbin_monitoring_enabled%'
+            $monitoringQueryParametersEnabled: '%mbin_monitoring_query_parameters_enabled%'
 
     kbin.s3_client:
         class: Aws\S3\S3Client

migrations/Version20260120175744.php

@@ -0,0 +1,55 @@
+<?php
+
+declare(strict_types=1);
+
+namespace DoctrineMigrations;
+
+use Doctrine\DBAL\Schema\Schema;
+use Doctrine\Migrations\AbstractMigration;
+
+final class Version20260120175744 extends AbstractMigration
+{
+    public function getDescription(): string
+    {
+        return 'Add initial monitoring tables';
+    }
+
+    public function up(Schema $schema): void
+    {
+        $this->addSql('CREATE SEQUENCE monitoring_curl_request_id_seq INCREMENT BY 1 MINVALUE 1 START 1');
+        $this->addSql('CREATE SEQUENCE monitoring_query_id_seq INCREMENT BY 1 MINVALUE 1 START 1');
+        $this->addSql('CREATE SEQUENCE monitoring_twig_render_id_seq INCREMENT BY 1 MINVALUE 1 START 1');
+        $this->addSql('CREATE TABLE monitoring_curl_request (id INT NOT NULL, context_id UUID DEFAULT NULL, url VARCHAR(255) NOT NULL, method VARCHAR(255) NOT NULL, was_successful BOOLEAN NOT NULL, exception VARCHAR(255) DEFAULT NULL, created_at TIMESTAMP(0) WITH TIME ZONE NOT NULL, started_at TIMESTAMP(0) WITHOUT TIME ZONE NOT NULL, started_at_microseconds DOUBLE PRECISION NOT NULL, ended_at TIMESTAMP(0) WITHOUT TIME ZONE NOT NULL, ended_at_microseconds DOUBLE PRECISION NOT NULL, duration_milliseconds DOUBLE PRECISION NOT NULL, PRIMARY KEY(id))');
+        $this->addSql('CREATE INDEX IDX_19A4B8546B00C1CF ON monitoring_curl_request (context_id)');
+        $this->addSql('CREATE TABLE monitoring_execution_context (uuid UUID NOT NULL, execution_type VARCHAR(255) NOT NULL, path VARCHAR(255) NOT NULL, handler VARCHAR(255) NOT NULL, user_type VARCHAR(255) NOT NULL, status_code INT DEFAULT NULL, exception VARCHAR(255) DEFAULT NULL, stacktrace VARCHAR(255) DEFAULT NULL, response_sending_duration_milliseconds DOUBLE PRECISION DEFAULT NULL, query_duration_milliseconds DOUBLE PRECISION NOT NULL, twig_render_duration_milliseconds DOUBLE PRECISION NOT NULL, curl_request_duration_milliseconds DOUBLE PRECISION NOT NULL, created_at TIMESTAMP(0) WITH TIME ZONE NOT NULL, started_at TIMESTAMP(0) WITHOUT TIME ZONE NOT NULL, started_at_microseconds DOUBLE PRECISION NOT NULL, ended_at TIMESTAMP(0) WITHOUT TIME ZONE NOT NULL, ended_at_microseconds DOUBLE PRECISION NOT NULL, duration_milliseconds DOUBLE PRECISION NOT NULL, PRIMARY KEY(uuid))');
+        $this->addSql('CREATE TABLE monitoring_query (id INT NOT NULL, context_id UUID DEFAULT NULL, query_string_id VARCHAR(40) DEFAULT NULL, parameters JSONB DEFAULT NULL, created_at TIMESTAMP(0) WITH TIME ZONE NOT NULL, started_at TIMESTAMP(0) WITHOUT TIME ZONE NOT NULL, started_at_microseconds DOUBLE PRECISION NOT NULL, ended_at TIMESTAMP(0) WITHOUT TIME ZONE NOT NULL, ended_at_microseconds DOUBLE PRECISION NOT NULL, duration_milliseconds DOUBLE PRECISION NOT NULL, PRIMARY KEY(id))');
+        $this->addSql('CREATE INDEX IDX_760D8AF36B00C1CF ON monitoring_query (context_id)');
+        $this->addSql('CREATE INDEX IDX_760D8AF3BCAEFD40 ON monitoring_query (query_string_id)');
+        $this->addSql('CREATE TABLE monitoring_query_string (query_hash VARCHAR(40) NOT NULL, query TEXT NOT NULL, PRIMARY KEY(query_hash))');
+        $this->addSql('CREATE TABLE monitoring_twig_render (id INT NOT NULL, context_id UUID DEFAULT NULL, parent_id INT DEFAULT NULL, short_description TEXT NOT NULL, template_name VARCHAR(255) DEFAULT NULL, name VARCHAR(255) DEFAULT NULL, type VARCHAR(255) DEFAULT NULL, memory_usage INT DEFAULT NULL, peak_memory_usage INT DEFAULT NULL, profiler_duration DOUBLE PRECISION DEFAULT NULL, created_at TIMESTAMP(0) WITH TIME ZONE NOT NULL, started_at TIMESTAMP(0) WITHOUT TIME ZONE NOT NULL, started_at_microseconds DOUBLE PRECISION NOT NULL, ended_at TIMESTAMP(0) WITHOUT TIME ZONE NOT NULL, ended_at_microseconds DOUBLE PRECISION NOT NULL, duration_milliseconds DOUBLE PRECISION NOT NULL, PRIMARY KEY(id))');
+        $this->addSql('CREATE INDEX IDX_55BA2A536B00C1CF ON monitoring_twig_render (context_id)');
+        $this->addSql('CREATE INDEX IDX_55BA2A53727ACA70 ON monitoring_twig_render (parent_id)');
+        $this->addSql('ALTER TABLE monitoring_curl_request ADD CONSTRAINT FK_19A4B8546B00C1CF FOREIGN KEY (context_id) REFERENCES monitoring_execution_context (uuid) ON DELETE CASCADE NOT DEFERRABLE INITIALLY IMMEDIATE');
+        $this->addSql('ALTER TABLE monitoring_query ADD CONSTRAINT FK_760D8AF36B00C1CF FOREIGN KEY (context_id) REFERENCES monitoring_execution_context (uuid) ON DELETE CASCADE NOT DEFERRABLE INITIALLY IMMEDIATE');
+        $this->addSql('ALTER TABLE monitoring_query ADD CONSTRAINT FK_760D8AF3BCAEFD40 FOREIGN KEY (query_string_id) REFERENCES monitoring_query_string (query_hash) ON DELETE CASCADE NOT DEFERRABLE INITIALLY IMMEDIATE');
+        $this->addSql('ALTER TABLE monitoring_twig_render ADD CONSTRAINT FK_55BA2A536B00C1CF FOREIGN KEY (context_id) REFERENCES monitoring_execution_context (uuid) ON DELETE CASCADE NOT DEFERRABLE INITIALLY IMMEDIATE');
+        $this->addSql('ALTER TABLE monitoring_twig_render ADD CONSTRAINT FK_55BA2A53727ACA70 FOREIGN KEY (parent_id) REFERENCES monitoring_twig_render (id) ON DELETE CASCADE NOT DEFERRABLE INITIALLY IMMEDIATE');
+    }
+
+    public function down(Schema $schema): void
+    {
+        $this->addSql('DROP SEQUENCE monitoring_curl_request_id_seq CASCADE');
+        $this->addSql('DROP SEQUENCE monitoring_query_id_seq CASCADE');
+        $this->addSql('DROP SEQUENCE monitoring_twig_render_id_seq CASCADE');
+        $this->addSql('ALTER TABLE monitoring_curl_request DROP CONSTRAINT FK_19A4B8546B00C1CF');
+        $this->addSql('ALTER TABLE monitoring_query DROP CONSTRAINT FK_760D8AF36B00C1CF');
+        $this->addSql('ALTER TABLE monitoring_query DROP CONSTRAINT FK_760D8AF3BCAEFD40');
+        $this->addSql('ALTER TABLE monitoring_twig_render DROP CONSTRAINT FK_55BA2A536B00C1CF');
+        $this->addSql('ALTER TABLE monitoring_twig_render DROP CONSTRAINT FK_55BA2A53727ACA70');
+        $this->addSql('DROP TABLE monitoring_curl_request');
+        $this->addSql('DROP TABLE monitoring_execution_context');
+        $this->addSql('DROP TABLE monitoring_query');
+        $this->addSql('DROP TABLE monitoring_query_string');
+        $this->addSql('DROP TABLE monitoring_twig_render');
+    }
+}

src/Controller/Admin/AdminMonitoringController.php

@@ -0,0 +1,167 @@
+<?php
+
+declare(strict_types=1);
+
+namespace App\Controller\Admin;
+
+use App\Controller\AbstractController;
+use App\DTO\MonitoringExecutionContextFilterDto;
+use App\Form\MonitoringExecutionContextFilterType;
+use App\Repository\MonitoringRepository;
+use Symfony\Component\Form\FormFactoryInterface;
+use Symfony\Component\HttpFoundation\Request;
+use Symfony\Component\HttpFoundation\Response;
+use Symfony\Component\HttpKernel\Attribute\MapQueryParameter;
+use Symfony\Component\Security\Http\Attribute\IsGranted;
+use Symfony\Contracts\Translation\TranslatorInterface;
+use Symfony\UX\Chartjs\Builder\ChartBuilderInterface;
+use Symfony\UX\Chartjs\Model\Chart;
+
+class AdminMonitoringController extends AbstractController
+{
+    public function __construct(
+        private readonly MonitoringRepository $monitoringRepository,
+        private readonly ChartBuilderInterface $chartBuilder,
+        private readonly FormFactoryInterface $formFactory,
+        private readonly TranslatorInterface $translator,
+    ) {
+    }
+
+    #[IsGranted('ROLE_ADMIN')]
+    public function overview(Request $request, #[MapQueryParameter] int $p = 1): Response
+    {
+        $dto = new MonitoringExecutionContextFilterDto();
+        $form = $this->formFactory->createNamed('filter', MonitoringExecutionContextFilterType::class, $dto, ['method' => 'GET']);
+
+        try {
+            $form->handleRequest($request);
+            if ($form->isSubmitted() && $form->isValid()) {
+                $dto = $form->getData();
+            }
+        } catch (\Exception) {
+        }
+
+        $contexts = $this->monitoringRepository->getFilteredContextsPaginated($dto);
+        $contexts->setCurrentPage($p);
+        $contexts->setMaxPerPage(50);
+
+        if (1 === $p) {
+            $chart = $this->getOverViewChart($dto);
+        }
+
+        return $this->render('admin/monitoring/monitoring.html.twig', [
+            'page' => $p,
+            'executionContexts' => $contexts,
+            'chart' => $chart ?? null,
+            'form' => $form,
+        ]);
+    }
+
+    private function getOverViewChart(MonitoringExecutionContextFilterDto $dto): Chart
+    {
+        $chart = $this->chartBuilder->createChart(Chart::TYPE_BAR);
+        $chart->setData($this->getOverviewChartData($dto));
+        $chart->setOptions([
+            'scales' => [
+                'y' => [
+                    'label' => '<%=value%>ms',
+                ],
+            ],
+            'interaction' => [
+                'mode' => 'index',
+                'axis' => 'xy',
+            ],
+            'plugins' => [
+                'tooltip' => [
+                    'enabled' => true,
+                ],
+            ],
+        ]);
+
+        return $chart;
+    }
+
+    public function getOverviewChartData(MonitoringExecutionContextFilterDto $dto): array
+    {
+        $rawData = $this->monitoringRepository->getOverviewRouteCalls($dto);
+        $labels = [];
+        $overallDurationRemaining = [];
+        $queryDurations = [];
+        $twigRenderDuration = [];
+        $curlRequestDuration = [];
+        $sendingDuration = [];
+
+        foreach ($rawData as $data) {
+            $labels[] = $data['path'];
+            $total = round(\floatval($data['total_duration']), 2);
+            $query = round(\floatval($data['query_duration']), 2);
+            $twig = round(\floatval($data['twig_render_duration']), 2);
+            $curl = round(\floatval($data['curl_request_duration']), 2);
+            $sending = round(\floatval($data['response_duration']), 2);
+            $overallDurationRemaining[] = max(0, round($total - $query - $twig - $curl - $sending, 2));
+            $queryDurations[] = $query;
+            $twigRenderDuration[] = $twig;
+            $curlRequestDuration[] = $curl;
+            $sendingDuration[] = $sending;
+        }
+
+        return [
+            'labels' => $labels,
+            'datasets' => [
+                [
+                    'label' => $this->translator->trans('monitoring_duration_overall'),
+                    'data' => $overallDurationRemaining,
+                    'stack' => '1',
+                    'backgroundColor' => 'gray',
+                    'borderRadius' => 5,
+                ],
+                [
+                    'label' => $this->translator->trans('monitoring_duration_query'),
+                    'data' => $queryDurations,
+                    'stack' => '1',
+                    'backgroundColor' => '#a3067c',
+                    'borderRadius' => 5,
+                ],
+                [
+                    'label' => $this->translator->trans('monitoring_duration_twig_render'),
+                    'data' => $twigRenderDuration,
+                    'stack' => '1',
+                    'backgroundColor' => 'green',
+                    'borderRadius' => 5,
+                ],
+                [
+                    'label' => $this->translator->trans('monitoring_duration_curl_request'),
+                    'data' => $curlRequestDuration,
+                    'stack' => '1',
+                    'backgroundColor' => '#07abaf',
+                    'borderRadius' => 5,
+                ],
+                [
+                    'label' => $this->translator->trans('monitoring_duration_sending_response'),
+                    'data' => $sendingDuration,
+                    'stack' => '1',
+                    'backgroundColor' => 'lightgray',
+                    'borderRadius' => 5,
+                ],
+            ],
+        ];
+    }
+
+    #[IsGranted('ROLE_ADMIN')]
+    public function single(string $id, string $page, #[MapQueryParameter] bool $groupSimilar = true, #[MapQueryParameter] bool $formatQuery = false, #[MapQueryParameter] bool $showParameters = false, #[MapQueryParameter] bool $compareToParent = true): Response
+    {
+        $context = $this->monitoringRepository->findOneBy(['uuid' => $id]);
+        if (!$context) {
+            throw $this->createNotFoundException();
+        }
+
+        return $this->render('admin/monitoring/monitoring_single.html.twig', [
+            'context' => $context,
+            'page' => $page,
+            'groupSimilar' => $groupSimilar,
+            'formatQuery' => $formatQuery,
+            'showParameters' => $showParameters,
+            'compareToParent' => $compareToParent,
+        ]);
+    }
+}

src/DTO/GroupedMonitoringQueryDto.php

@@ -0,0 +1,15 @@
+<?php
+
+declare(strict_types=1);
+
+namespace App\DTO;
+
+class GroupedMonitoringQueryDto
+{
+    public string $query;
+    public float $minExecutionTime;
+    public float $maxExecutionTime;
+    public float $meanExecutionTime;
+    public float $totalExecutionTime;
+    public int $count;
+}