Merge branch 'dev' into dev

streampark-console/streampark-console-service/src/main/resources/db/data-h2.sql

@@ -82,6 +82,9 @@ insert into `t_menu` values (110302, 110300, 'cluster edit', '/flink/edit_cluste
 
 insert into `t_menu` values (120100, 120000, 'spark.application', '/spark/app', 'spark/app/index', null, null, '0', 1, 2, now(), now());
 insert into `t_menu` values (120200, 120000, 'spark.sparkHome', '/spark/home', 'spark/home/index', null, null, '0', 1, 3, now(), now());
+insert into `t_menu` values (120300, 120000, 'spark.createApplication', '/spark/app/create', 'spark/app/create', 'app:create', '', '0', 0, null, now(), now());
+insert into `t_menu` values (120400, 120000, 'spark.updateApplication', '/spark/app/edit', 'spark/app/edit', 'app:update', '', '0', 0, null, now(), now());
+insert into `t_menu` values (120500, 120000, 'spark.applicationDetail', '/spark/app/detail', 'spark/app/detail', 'app:detail', '', '0', 0, null, now(), now());
 
 insert into `t_menu` values (130100, 130000, 'resource.project', '/resource/project', 'resource/project/View', null, 'github', '0', 1, 2, now(), now());
 insert into `t_menu` values (130200, 130000, 'resource.variable', '/resource/variable', 'resource/variable/View', null, null, '0', 1, 3, now(), now());

streampark-console/streampark-console-webapp/src/api/spark/app.ts

@@ -119,7 +119,7 @@ export function fetchSparkYarn() {
 /**
  * check spark name
  */
-export function fetchCheckSparkName(data: { id?: string; jobName: string }) {
+export function fetchCheckSparkName(data: { id?: string; appName: string }) {
   return defHttp.post<AppExistsStateEnum>({ url: `${apiPrefix}/check/name`, data });
 }
 

streampark-console/streampark-console-webapp/src/api/spark/app.type.ts

@@ -14,138 +14,99 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
+import type { AppStateEnum, OptionStateEnum } from '/@/enums/sparkEnum';
+
 // dashboard
 export interface DashboardResponse {
-  totalTM: number;
-  task: Task;
-  availableSlot: number;
-  totalSlot: number;
-  runningJob: number;
-  tmMemory: number;
-  jmMemory: number;
+  numTasks: number;
+  numStages: number;
+  usedMemory: number;
+  usedVCores: number;
+  numCompletedTasks: number;
+  runningApplication: number;
+  numCompletedStages: number;
 }
 
-interface Task {
-  total: number;
-  created: number;
-  scheduled: number;
-  deploying: number;
-  running: number;
-  finished: number;
-  canceling: number;
-  canceled: number;
-  failed: number;
-  reconciling: number;
-}
 // The list of data
 export interface AppListResponse {
   total: string;
   records: SparkApplication[];
 }
+
 export interface SparkApplication {
+  createTime?: string;
+  modifyTime?: string;
   id?: string;
   teamId?: string;
   jobType?: number;
   appType?: number;
-  versionId?: number;
+  versionId?: string;
   appName?: string;
   executionMode?: number;
   resourceFrom?: number;
-  module?: any;
+  projectId?: number;
+  module?: string;
   mainClass?: string;
   jar?: string;
-  jarCheckSum?: string;
+  jarCheckSum?: number;
   appProperties?: string;
   appArgs?: string;
   appId?: string;
-  yarnQueue?: any;
-
-  projectId?: any;
-  tags?: any;
-  userId?: string;
-  jobName?: string;
-  jobId?: string;
-  clusterId?: string;
-  flinkImage?: string;
+  yarnQueue?: string;
+  yarnQueueName?: string;
+  yarnQueueLabel?: string;
+  k8sMasterUrl?: string;
+  k8sContainerImage?: string;
+  k8sImagePullPolicy?: number;
+  k8sServiceAccount?: number;
   k8sNamespace?: string;
-  state?: number;
-  release?: number;
-  build?: boolean;
+  hadoopUser?: string;
   restartSize?: number;
   restartCount?: number;
-  optionState?: number;
-  alertId?: any;
-  args?: string;
+  state?: AppStateEnum;
   options?: string;
-  hotParams?: string;
-  resolveOrder?: number;
-  dynamicProperties?: string;
+  optionState?: OptionStateEnum;
+  optionTime?: string;
+  userId?: string;
+  description?: string;
   tracking?: number;
-
+  release?: number;
+  build?: boolean;
+  alertId?: number;
   startTime?: string;
   endTime?: string;
-  duration?: string;
-  cpMaxFailureInterval?: any;
-  cpFailureRateInterval?: any;
-  cpFailureAction?: any;
-  totalTM?: any;
-  totalSlot?: any;
-  availableSlot?: any;
-  jmMemory?: number;
-  tmMemory?: number;
-  totalTask?: number;
-  flinkClusterId?: any;
-  description?: string;
-  createTime?: string;
-  optionTime?: string;
-  modifyTime?: string;
-  k8sRestExposedType?: any;
-  k8sPodTemplate?: any;
-  k8sJmPodTemplate?: any;
-  k8sTmPodTemplate?: any;
-  ingressTemplate?: any;
-  defaultModeIngress?: any;
-  k8sHadoopIntegration?: boolean;
-  overview?: any;
-  teamResource?: any;
-  dependency?: any;
-  sqlId?: any;
-  flinkSql?: any;
-  stateArray?: any;
-  jobTypeArray?: any;
+  duration?: number;
+  tags?: string;
+  driverCores?: string;
+  driverMemory?: string;
+  executorCores?: string;
+  executorMemory?: string;
+  executorMaxNums?: string;
+  numTasks?: number;
+  numCompletedTasks?: number;
+  numStages?: number;
+  numCompletedStages?: number;
+  usedMemory?: number;
+  usedVCores?: number;
+  teamResource?: number;
+  dependency?: string;
+  sqlId?: number;
+  sparkSql?: string;
   backUp?: boolean;
   restart?: boolean;
-  userName?: string;
-  nickName?: string;
-  config?: any;
-  configId?: any;
-  flinkVersion?: string;
-  confPath?: any;
-  format?: any;
-  savepointPath?: any;
-  restoreOrTriggerSavepoint?: boolean;
-  drain?: boolean;
-  nativeFormat?: boolean;
-  allowNonRestored?: boolean;
-  socketId?: any;
-  projectName?: any;
-  createTimeFrom?: any;
-  createTimeTo?: any;
-  backUpDescription?: any;
-  teamIdList?: any;
-  teamName?: string;
-  flinkRestUrl?: any;
+  config?: string;
+  configId?: number;
+  sparkVersion?: string;
+  confPath?: string;
+  format?: string;
+  backUpDescription?: string;
+  sparkRestUrl?: string;
   buildStatus?: number;
   appControl?: AppControl;
-  fsOperator?: any;
-  workspace?: any;
-  k8sPodTemplates?: {
-    empty?: boolean;
-  };
+  canBeStart?: boolean;
   streamParkJob?: boolean;
-  hadoopUser?: string;
 }
-
 interface AppControl {
   allowStart: boolean;
   allowStop: boolean;

streampark-console/streampark-console-webapp/src/api/spark/build.ts

@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import type { Result } from '/#/axios';
+import { defHttp } from '/@/utils/http/axios';
+
+const apiPrefix = `/spark/pipe`;
+
+/**
+ * Release application building pipeline.
+ *
+ * @param appId application id
+ * @param forceBuild forced start pipeline or not
+ * @return Whether the pipeline was successfully started
+ */
+export function fetchBuildSparkApp(data: { appId: string; forceBuild: boolean }) {
+  return defHttp.post<Result<boolean>>(
+    { url: `${apiPrefix}/build`, data },
+    { isTransformResponse: false },
+  );
+}
+
+/**
+ * Get application building pipeline progress detail.
+ *
+ * @param appId application id
+ * @return "pipeline" -> pipeline details, "docker" -> docker resolved snapshot
+ */
+
+export function fetchBuildProgressDetail(appId: number) {
+  return defHttp.post<{
+    pipeline: any;
+    docker: any;
+  }>({ url: `${apiPrefix}/detail`, data: { appId } });
+}

streampark-console/streampark-console-webapp/src/api/spark/conf.ts

@@ -14,18 +14,45 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+import type { SparkApplication } from './app.type';
+import type { SparkApplicationConfig } from './conf.type';
 import { defHttp } from '/@/utils/http/axios';
 
 const apiPrefix = '/flink/conf';
 
 export function fetchGetSparkConf(data: { id: string }) {
-  return defHttp.post({ url: `${apiPrefix}/get`, data });
+  return defHttp.post<SparkApplicationConfig>({ url: `${apiPrefix}/get`, data });
 }
-export function handleSparkConfTemplate() {
+
+export function fetchSparkConfTemplate() {
   return defHttp.post<string>({
     url: `${apiPrefix}/template`,
   });
 }
+
+export function fetchSparkConfList(data: Recordable) {
+  return defHttp.post<{
+    total: number;
+    records: SparkApplicationConfig[];
+  }>({
+    url: `${apiPrefix}/list`,
+    data,
+  });
+}
+
+export function fetchSparkConfHistory(data: SparkApplication) {
+  return defHttp.post<SparkApplicationConfig[]>({
+    url: `${apiPrefix}/history`,
+    data,
+  });
+}
+export function fetchSparkConfRemove(data: { id: number }) {
+  return defHttp.post<boolean>({
+    url: `${apiPrefix}/delete`,
+    data,
+  });
+}
+
 export function fetchSysHadoopConf() {
   return defHttp.post({
     url: `${apiPrefix}/sysHadoopConf`,

streampark-console/streampark-console-webapp/src/api/spark/conf.type.ts

@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+export interface SparkApplicationConfig {
+  id?: number;
+  appId?: string;
+  /**
+   * 1)yaml <br>
+   * 2)prop <br>
+   * 3)hocon
+   */
+  format?: 1 | 2 | 3;
+  content?: string;
+  version?: number;
+  latest?: boolean;
+  createTime: string;
+  effective: boolean;
+}

streampark-console/streampark-console-webapp/src/api/spark/sql.ts

@@ -15,16 +15,46 @@
  * limitations under the License.
  */
 
+import type { SparkSql } from './sql.type';
 import { defHttp } from '/@/utils/http/axios';
-const apiPrefix = '/flink/conf';
+const apiPrefix = '/spark/sql';
 
 export function fetchSparkSqlVerify(data: Recordable) {
   return defHttp.post({ url: `${apiPrefix}/verify`, data }, { isTransformResponse: false });
 }
 
+export function fetchSparkSqlList(data: Recordable) {
+  return defHttp.post<{
+    total: number;
+    records: SparkSql[];
+  }>({
+    url: `${apiPrefix}/list`,
+    data,
+  });
+}
+
+export function fetchSparkSqlRemove(data: SparkSql) {
+  return defHttp.post<boolean>({
+    url: `${apiPrefix}/delete`,
+    data,
+  });
+}
+
 export function fetchSparkSql(data: Recordable) {
   return defHttp.post({
     url: `${apiPrefix}/get`,
     data,
   });
 }
+export function fetchSparkSqlHistory(data: Recordable) {
+  return defHttp.post<SparkSql[]>({
+    url: `${apiPrefix}/history`,
+    data,
+  });
+}
+export function fetchSparkSqlComplete(data: Recordable) {
+  return defHttp.post({
+    url: `${apiPrefix}/sqlComplete`,
+    data,
+  });
+}