[Feature] [monitor] add monitor & jmx (DataLinkDC#2056)

* add monitor & jmx

* add monitor & jmx

* remove metrics model

* add note

* add note

dinky-admin/pom.xml

@@ -221,11 +221,6 @@
             <groupId>org.springframework.boot</groupId>
             <artifactId>spring-boot-starter-aop</artifactId>
         </dependency>
-        <dependency>
-            <groupId>io.prometheus.jmx</groupId>
-            <artifactId>jmx_prometheus_javaagent</artifactId>
-            <version>0.16.1</version>
-        </dependency>
         <dependency>
             <groupId>com.github.jsqlparser</groupId>
             <artifactId>jsqlparser</artifactId>
@@ -302,12 +297,44 @@
         </dependency>
         <dependency>
             <groupId>org.springframework.boot</groupId>
-            <artifactId>spring-boot-actuator</artifactId>
+            <artifactId>spring-boot-starter-data-ldap</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>io.micrometer</groupId>
+            <artifactId>micrometer-registry-prometheus</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>io.prometheus.jmx</groupId>
+            <artifactId>jmx_prometheus_javaagent</artifactId>
+            <version>0.16.1</version>
+            <exclusions>
+                <exclusion>
+                    <groupId>org.yaml</groupId>
+                    <artifactId>snakeyaml</artifactId>
+                </exclusion>
+            </exclusions>
         </dependency>
         <dependency>
             <groupId>org.springframework.boot</groupId>
-            <artifactId>spring-boot-starter-data-ldap</artifactId>
+            <artifactId>spring-boot-starter-actuator</artifactId>
         </dependency>
+        <!-- 获取系统信息 -->
+        <dependency>
+            <groupId>com.github.oshi</groupId>
+            <artifactId>oshi-core</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.paimon</groupId>
+            <artifactId>paimon-bundle</artifactId>
+            <version>0.4.0-incubating</version>
+            <exclusions>
+                <exclusion>
+                    <groupId>org.slf4j</groupId>
+                    <artifactId>slf4j-api</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+
     </dependencies>
 
     <build>

dinky-admin/src/main/java/org/dinky/configure/MetricConfig.java

@@ -0,0 +1,150 @@
+/*
+ *
+ *  Licensed to the Apache Software Foundation (ASF) under one or more
+ *  contributor license agreements.  See the NOTICE file distributed with
+ *  this work for additional information regarding copyright ownership.
+ *  The ASF licenses this file to You under the Apache License, Version 2.0
+ *  (the "License"); you may not use this file except in compliance with
+ *  the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ */
+
+package org.dinky.configure;
+
+import org.dinky.data.annotation.GaugeM;
+import org.dinky.data.metrics.BaseMetrics;
+import org.dinky.data.metrics.Cpu;
+import org.dinky.data.metrics.Jvm;
+import org.dinky.data.metrics.Mem;
+import org.dinky.data.metrics.MetricsTotal;
+import org.dinky.data.vo.MetricsVO;
+import org.dinky.utils.PaimonUtil;
+
+import java.lang.reflect.Field;
+import java.time.LocalDateTime;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Queue;
+import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.SynchronousQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+
+import javax.annotation.PostConstruct;
+import javax.annotation.PreDestroy;
+
+import org.springframework.context.annotation.Configuration;
+import org.springframework.scheduling.annotation.EnableScheduling;
+import org.springframework.scheduling.annotation.Scheduled;
+
+import cn.hutool.core.annotation.AnnotationUtil;
+import cn.hutool.core.lang.Opt;
+import cn.hutool.core.util.ReflectUtil;
+import cn.hutool.json.JSONUtil;
+import io.micrometer.core.instrument.Gauge;
+import io.micrometer.core.instrument.MeterRegistry;
+import lombok.Getter;
+import lombok.RequiredArgsConstructor;
+import lombok.Setter;
+
+@Configuration
+@RequiredArgsConstructor
+@Getter
+@Setter
+@EnableScheduling
+public class MetricConfig {
+
+    private final MeterRegistry registry;
+    private ThreadPoolExecutor writePool =
+            new ThreadPoolExecutor(5, 10, 1, TimeUnit.DAYS, new SynchronousQueue<>());
+    private static final Queue<MetricsVO> metricsQueue = new ConcurrentLinkedQueue<>();
+    public static final int SCHEDULED_RATE = 1000;
+
+    /** Update status per second */
+    @Scheduled(fixedRate = SCHEDULED_RATE)
+    public void scheduled() {
+        updateState();
+    }
+
+    /** Entering the lake every 10 states */
+    @Scheduled(fixedRate = 10000)
+    @PreDestroy
+    public void writeScheduled() {
+        PaimonUtil.writeMetrics(new ArrayList<>(metricsQueue));
+        metricsQueue.clear();
+        //        PaimonUtil.batchReadTable(PaimonUtil.METRICS_IDENTIFIER,Metrics.class,p->
+        // Collections.singletonList(p.greaterThan(0,
+        // Timestamp.fromLocalDateTime(DateUtil.toLocalDateTime(DateUtil.date(1686300257085L))))));
+        PaimonUtil.batchReadTable(PaimonUtil.METRICS_IDENTIFIER, MetricsVO.class);
+    }
+
+    @PostConstruct
+    public void init() {
+        MetricsTotal metricsTotal = MetricsTotal.instance;
+        registerMetrics(metricsTotal.getJvm());
+        registerMetrics(metricsTotal.getCpu());
+        registerMetrics(metricsTotal.getMem());
+    }
+
+    public void updateState() {
+        MetricsTotal metricsTotal = MetricsTotal.instance;
+        LocalDateTime now = LocalDateTime.now();
+
+        metricsTotal.setJvm(Jvm.of());
+        metricsTotal.setCpu(Cpu.of());
+        metricsTotal.setMem(Mem.of());
+
+        MetricsVO metrics = new MetricsVO();
+        metrics.setContent(JSONUtil.toJsonStr(metricsTotal));
+        metrics.setHeartTime(now);
+        metrics.setModel("local");
+        metricsQueue.add(metrics);
+
+        //        writePool.execute(metrics::insert);
+    }
+
+    public void registerMetrics(BaseMetrics baseMetrics) {
+        Field[] baseFields = ReflectUtil.getFields(this.getClass());
+        Field baseField =
+                Arrays.stream(baseFields)
+                        .filter(field -> field.getType().equals(baseMetrics.getClass()))
+                        .findFirst()
+                        .orElse(null);
+        if (baseField == null) {
+            return;
+        }
+        Field[] fields = ReflectUtil.getFields(baseMetrics.getClass());
+        for (Field field : fields) {
+            GaugeM gaugeM = AnnotationUtil.getAnnotation(field, GaugeM.class);
+            Opt.ofNullable(gaugeM)
+                    .ifPresent(
+                            g ->
+                                    Gauge.builder(
+                                                    gaugeM.name(),
+                                                    () ->
+                                                            (Number)
+                                                                    ReflectUtil.getFieldValue(
+                                                                            ReflectUtil
+                                                                                    .getFieldValue(
+                                                                                            this,
+                                                                                            baseField),
+                                                                            field))
+                                            .baseUnit(g.baseUnit())
+                                            .tags(g.tags())
+                                            .description(gaugeM.description())
+                                            .register(registry));
+        }
+    }
+
+    public static Queue<MetricsVO> getMetricsQueue() {
+        return metricsQueue;
+    }
+}

dinky-admin/src/main/java/org/dinky/configure/ThreadPoolConfig.java

@@ -0,0 +1,54 @@
+/*
+ *
+ *  Licensed to the Apache Software Foundation (ASF) under one or more
+ *  contributor license agreements.  See the NOTICE file distributed with
+ *  this work for additional information regarding copyright ownership.
+ *  The ASF licenses this file to You under the Apache License, Version 2.0
+ *  (the "License"); you may not use this file except in compliance with
+ *  the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ */
+
+package org.dinky.configure;
+
+import java.util.concurrent.Executor;
+
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
+
+import cn.hutool.core.thread.BlockPolicy;
+
+@Configuration
+public class ThreadPoolConfig {
+    /**
+     * Use sse to return to monitor the latest data(使用sse返回监控最新数据)
+     *
+     * @return
+     */
+    @Bean
+    public Executor scheduleRefreshMonitorDataExecutor() {
+        ThreadPoolTaskExecutor threadPoolTaskExecutor = new ThreadPoolTaskExecutor();
+        // 核心线程数
+        threadPoolTaskExecutor.setCorePoolSize(10);
+        threadPoolTaskExecutor.setAllowCoreThreadTimeOut(true);
+        // 最大线程数
+        threadPoolTaskExecutor.setMaxPoolSize(200);
+        // 配置队列大小
+        threadPoolTaskExecutor.setQueueCapacity(1000);
+        // 配置线程池前缀
+        threadPoolTaskExecutor.setThreadNamePrefix("SCH-refresh-mt-");
+        // 阻塞策略
+        threadPoolTaskExecutor.setRejectedExecutionHandler(new BlockPolicy());
+        threadPoolTaskExecutor.initialize();
+        return threadPoolTaskExecutor;
+    }
+}

dinky-admin/src/main/java/org/dinky/controller/MonitorController.java

@@ -0,0 +1,65 @@
+/*
+ *
+ *  Licensed to the Apache Software Foundation (ASF) under one or more
+ *  contributor license agreements.  See the NOTICE file distributed with
+ *  this work for additional information regarding copyright ownership.
+ *  The ASF licenses this file to You under the Apache License, Version 2.0
+ *  (the "License"); you may not use this file except in compliance with
+ *  the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ */
+
+package org.dinky.controller;
+
+import org.dinky.data.result.Result;
+import org.dinky.data.vo.MetricsVO;
+import org.dinky.service.MonitorService;
+import org.dinky.sse.SseEmitterUTF8;
+
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+
+import org.springframework.http.MediaType;
+import org.springframework.web.bind.annotation.CrossOrigin;
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RequestParam;
+import org.springframework.web.bind.annotation.RestController;
+import org.springframework.web.servlet.mvc.method.annotation.SseEmitter;
+
+import cn.hutool.core.date.DateUtil;
+import cn.hutool.core.lang.Opt;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+@RestController
+@RequestMapping("/api/monitor")
+@RequiredArgsConstructor
+public class MonitorController {
+    private final MonitorService monitorService;
+
+    @GetMapping("/getSysData")
+    public Result<List<MetricsVO>> getData(@RequestParam Long startTime, Long endTime) {
+        return Result.succeed(
+                monitorService.getData(
+                        DateUtil.date(startTime),
+                        DateUtil.date(Opt.ofNullable(endTime).orElse(DateUtil.date().getTime()))));
+    }
+
+    @GetMapping(value = "/getLastUpdateData", produces = MediaType.TEXT_EVENT_STREAM_VALUE)
+    @CrossOrigin("*")
+    public SseEmitter getLastUpdateData(Long lastTime) {
+        SseEmitter emitter = new SseEmitterUTF8(TimeUnit.MINUTES.toMillis(30));
+        return monitorService.sendLatestData(
+                emitter, DateUtil.date(Opt.ofNullable(lastTime).orElse(DateUtil.date().getTime())));
+    }
+}

dinky-admin/src/main/java/org/dinky/data/annotation/GaugeM.java

@@ -0,0 +1,39 @@
+/*
+ *
+ *  Licensed to the Apache Software Foundation (ASF) under one or more
+ *  contributor license agreements.  See the NOTICE file distributed with
+ *  this work for additional information regarding copyright ownership.
+ *  The ASF licenses this file to You under the Apache License, Version 2.0
+ *  (the "License"); you may not use this file except in compliance with
+ *  the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ */
+
+package org.dinky.data.annotation;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Inherited;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+@Target({ElementType.FIELD})
+@Retention(RetentionPolicy.RUNTIME)
+@Inherited
+public @interface GaugeM {
+    String name();
+
+    String baseUnit() default "bytes";
+
+    String[] tags() default {};
+
+    String description() default "";
+}

dinky-admin/src/main/java/org/dinky/data/metrics/BaseMetrics.java

@@ -0,0 +1,24 @@
+/*
+ *
+ *  Licensed to the Apache Software Foundation (ASF) under one or more
+ *  contributor license agreements.  See the NOTICE file distributed with
+ *  this work for additional information regarding copyright ownership.
+ *  The ASF licenses this file to You under the Apache License, Version 2.0
+ *  (the "License"); you may not use this file except in compliance with
+ *  the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ */
+
+package org.dinky.data.metrics;
+
+import java.io.Serializable;
+
+public class BaseMetrics implements Serializable {}