From eeab42f6964d0792b96c1a546655b8347e15b95a Mon Sep 17 00:00:00 2001 From: coderTomato <570547651@qq.com> Date: Wed, 12 Jan 2022 14:07:16 +0800 Subject: [PATCH 01/10] =?UTF-8?q?monacoEditor=E5=AE=9E=E7=8E=B0=E5=BF=AB?= =?UTF-8?q?=E6=8D=B7=E9=94=AE=E4=BF=9D=E5=AD=98=E3=80=81=E6=A0=A1=E9=AA=8C?= =?UTF-8?q?sql=E3=80=81=E4=BB=A3=E7=A0=81=E6=A0=BC=E5=BC=8F=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../dlink/controller/CatalogueController.java | 2 +- dlink-web/package.json | 1 + .../StudioConsole/StudioExplain/index.tsx | 199 ++++++++++-------- .../components/Studio/StudioEdit/index.tsx | 106 +++++++--- .../components/Studio/StudioMenu/index.tsx | 35 +-- 5 files changed, 186 insertions(+), 157 deletions(-) diff --git a/dlink-admin/src/main/java/com/dlink/controller/CatalogueController.java b/dlink-admin/src/main/java/com/dlink/controller/CatalogueController.java index 5d929a969f..426c9ecc16 100644 --- a/dlink-admin/src/main/java/com/dlink/controller/CatalogueController.java +++ b/dlink-admin/src/main/java/com/dlink/controller/CatalogueController.java @@ -206,7 +206,7 @@ public Result createTask(@RequestBody CatalogueTaskDTO catalogueTaskDTO) throws } /** - * 创建节点和作业 + * 重命名节点和作业 */ @PutMapping("/toRename") public Result toRename(@RequestBody Catalogue catalogue) throws Exception { diff --git a/dlink-web/package.json b/dlink-web/package.json index d2e91391e8..7409cb24c5 100644 --- a/dlink-web/package.json +++ b/dlink-web/package.json @@ -74,6 +74,7 @@ "react-helmet-async": "^1.0.4", "react-highlight-words": "^0.17.0", "react-monaco-editor": "^0.43.0", + "sql-formatter": "^4.0.2", "umi": "^3.5.0", "umi-request": "^1.0.8" }, diff --git a/dlink-web/src/components/Studio/StudioConsole/StudioExplain/index.tsx b/dlink-web/src/components/Studio/StudioConsole/StudioExplain/index.tsx index 9b7e345bfb..81226a80c0 100644 --- a/dlink-web/src/components/Studio/StudioConsole/StudioExplain/index.tsx +++ b/dlink-web/src/components/Studio/StudioConsole/StudioExplain/index.tsx @@ -3,7 +3,8 @@ import {connect} from "umi"; import {Button, Tag, Space, Typography, Modal,} from 'antd'; import {ConsoleSqlOutlined} from "@ant-design/icons"; import ProList from '@ant-design/pro-list'; -import React, {useRef, useState} from "react"; +import {explainSql} from "@/pages/FlinkSqlStudio/service"; +import {useRef, useState, useEffect} from "react"; const {Paragraph} = Typography; @@ -25,17 +26,41 @@ export type StudioExplainProps = { data: Partial; } const StudioExplain = (props: any) => { - + const [explainData, setExplainData] = useState([]); const { - onCancel: handleModalVisible, + onClose, modalVisible, - data, + current, + currentSession, } = props; + useEffect(() => { + let selectsql = null; + if (current.monaco.current) { + let selection = current.monaco.current.editor.getSelection(); + selectsql = current.monaco.current.editor.getModel().getValueInRange(selection); + } + if (selectsql == null || selectsql == '') { + selectsql = current.value; + } + let useSession = !!currentSession.session; + let param = { + ...current.task, + useSession: useSession, + session: currentSession.session, + configJson: JSON.stringify(current.task.config), + statement: selectsql, + }; + const result = explainSql(param); + result.then(res => { + setExplainData(res.datas); + }) + }, []) + const renderFooter = () => { return ( <> - + ); }; @@ -46,100 +71,100 @@ const StudioExplain = (props: any) => { toolBarRender={false} search={{ - filterType: 'light', - }} + filterType: 'light', + }} rowKey="id" - dataSource={data} + dataSource={explainData} pagination={{ - pageSize: 5, - }} + pageSize: 5, + }} showActions="hover" metas={{ - avatar: { - dataIndex: 'index', - search: false, - }, - title: { - dataIndex: 'type', - title: 'type', - render: (_, row) => { - return ( - - - {row.type} - - - ); + avatar: { + dataIndex: 'index', + search: false, }, - }, - description: { - search: false, - render: (_, row) => { - return ( - <> - {row.sql ? - ( - {row.sql} - ) : null - } - {row.explain ? - ( + title: { + dataIndex: 'type', + title: 'type', + render: (_, row) => { + return ( + + + {row.type} + + + ); + }, + }, + description: { + search: false, + render: (_, row) => { + return ( + <> + {row.sql ? + ( + {row.sql} + ) : null + } + {row.explain ? + ( 
                       {row.explain}
- ) : null - } - {row.error ? - ( + ) : null + } + {row.error ? + ( 
                         {row.error}
- ) : null - } - - ) - } - }, - subTitle: { - render: (_, row) => { - return ( - - {row.parseTrue ? - (语法正确) : - (语法有误)} - {row.explainTrue ? - (逻辑正确) : - (逻辑有误)} - {row.explainTime} - - ); + ) : null + } + + ) + } }, - search: false, - }, - }} + subTitle: { + render: (_, row) => { + return ( + + {row.parseTrue ? + (语法正确) : + (语法有误)} + {row.explainTrue ? + (逻辑正确) : + (逻辑有误)} + {row.explainTime} + + ); + }, + search: false, + }, + }} options={{ - search: false, - setting: false - }} - /> - ) - }; - + search: false, + setting: false + }} + /> + ) + }; - return ( - handleModalVisible()} - > - {renderContent()} - - ); - }; + return ( + + {renderContent()} + + ); +}; - export default connect(({Studio}: {Studio: StateType}) => ({ - current: Studio.current, - }))(StudioExplain); +export default connect(({Studio}: { Studio: StateType }) => ({ + current: Studio.current, + currentSession: Studio.currentSession, +}))(StudioExplain); diff --git a/dlink-web/src/components/Studio/StudioEdit/index.tsx b/dlink-web/src/components/Studio/StudioEdit/index.tsx index 8d517c41e3..8c3b257e5f 100644 --- a/dlink-web/src/components/Studio/StudioEdit/index.tsx +++ b/dlink-web/src/components/Studio/StudioEdit/index.tsx @@ -1,14 +1,14 @@ -import React, {useEffect, useImperativeHandle, useRef} from 'react'; +import React, {useEffect, useImperativeHandle, useRef,useState} from 'react'; import * as _monaco from "monaco-editor"; import MonacoEditor from "react-monaco-editor"; -import {BaseDataSourceField, BaseDataSourceHeader, CompletionItem} from "./data"; -import styles from './index.less'; import {StateType} from "@/pages/FlinkSqlStudio/model"; -import {connect} from "umi"; +import {connect,Dispatch} from "umi"; import {DocumentStateType} from "@/pages/Document/model"; import {DocumentTableListItem} from "@/pages/Document/data"; import {parseSqlMetaData} from "@/components/Studio/StudioEvent/Utils"; import {Column, MetaData} from "@/components/Studio/StudioEvent/data"; +import StudioExplain from "@/components/Studio/StudioConsole/StudioExplain"; +import {format} from "sql-formatter"; let provider = { dispose: () => {}, @@ -34,11 +34,11 @@ const FlinkSqlEditor = (props:any) => { }, tabs, fillDocuments, - dispatch, } = props; const editorInstance:any = useRef(); const monacoInstance: any = useRef(); + const [modalVisible, handleModalVisible] = useState(false); const getTabIndex = ():number=>{ for(let i=0;i { const onChangeHandle = (val: string, event: { changes: { text: any }[] }) => { let sqlMetaData = parseSqlMetaData(val); - dispatch({ - type: "Studio/saveSqlMetaData", - payload: { - activeKey:tabs.panes[tabIndex].key, - sqlMetaData, - isModified: true, - }, - }); + props.saveMetaData(sqlMetaData,tabs,tabIndex); onChange(val,event); /*const curWord = event.changes[0].text; if (curWord === ';') { @@ -100,11 +93,7 @@ const FlinkSqlEditor = (props:any) => { return; } cache.current = val;*/ - dispatch({ - type: "Studio/saveSql", - payload: val, - }); - + props.saveSql(val); }; const buildSuggestions = () => { @@ -152,6 +141,16 @@ const FlinkSqlEditor = (props:any) => { const editorDidMountHandle = (editor: any, monaco: any) => { monacoInstance.current = monaco; editorInstance.current = editor; + editor.addCommand(monaco.KeyMod.CtrlCmd|monaco.KeyCode.KEY_1,function (){ + props.saveText(tabs,tabIndex); + }) + + editor.addCommand(monaco.KeyMod.CtrlCmd|monaco.KeyCode.KEY_2,function (){ + handleModalVisible(true); + }) + editor.addCommand(monaco.KeyMod.CtrlCmd|monaco.KeyCode.KEY_3,function (){ + editor.getAction(['editor.action.formatDocument'])._run(); + }) provider.dispose();// 清空提示项 // 提示项设值 provider = monaco.languages.registerCompletionItemProvider('sql', { @@ -163,25 +162,62 @@ const FlinkSqlEditor = (props:any) => { // quickSuggestions: false, // triggerCharacters: ['$', '.', '='], }); + monaco.languages.registerDocumentRangeFormattingEditProvider('sql', { + provideDocumentRangeFormattingEdits(model, range, options) { + var formatted = format(model.getValueInRange(range), { + indent: ' '.repeat(options.tabSize) + }); + return [ + { + range: range, + text: formatted + } + ]; + } + }); editor.focus(); }; -return ( - - - -); -}; + return ( + + + {handleModalVisible(false)}} + visible={modalVisible} + /> + + ) +} + +const mapDispatchToProps = (dispatch:Dispatch)=>({ + saveText:(tabs:any,tabIndex:any)=>dispatch({ + type: "Studio/saveTask", + payload: tabs.panes[tabIndex].task, + }), + saveSql:(val:any)=>dispatch({ + type: "Studio/saveSql", + payload: val, + }), + saveMetaData:(sqlMetaData:any,tabs:any,tabIndex:any)=>dispatch({ + type: "Studio/saveSqlMetaData", + payload: { + activeKey:tabs.panes[tabIndex].key, + sqlMetaData, + isModified: true, + } + }) +}) export default connect(({ Studio,Document }: { Studio: StateType,Document: DocumentStateType }) => ({ current: Studio.current, @@ -189,4 +225,4 @@ export default connect(({ Studio,Document }: { Studio: StateType,Document: Docum tabs: Studio.tabs, monaco: Studio.monaco, fillDocuments: Document.fillDocuments, -}))(FlinkSqlEditor); +}),mapDispatchToProps)(FlinkSqlEditor); diff --git a/dlink-web/src/components/Studio/StudioMenu/index.tsx b/dlink-web/src/components/Studio/StudioMenu/index.tsx index ce8f547775..d94711444c 100644 --- a/dlink-web/src/components/Studio/StudioMenu/index.tsx +++ b/dlink-web/src/components/Studio/StudioMenu/index.tsx @@ -130,36 +130,7 @@ const StudioMenu = (props: any) => { }; const onCheckSql = () => { - let selectsql = null; - if (current.monaco.current) { - let selection = current.monaco.current.editor.getSelection(); - selectsql = current.monaco.current.editor.getModel().getValueInRange(selection); - } - if (selectsql == null || selectsql == '') { - selectsql = current.value; - } - let useSession = !!currentSession.session; - let param = { - ...current.task, - useSession: useSession, - session: currentSession.session, - configJson: JSON.stringify(current.task.config), - statement: selectsql, - }; - const taskKey = (Math.random() * 1000) + ''; - notification.success({ - message: `新任务【${param.jobName}】正在检查`, - description: param.statement.substring(0, 40) + '...', - duration: null, - key: taskKey, - icon: , - }); - const result = explainSql(param); handleModalVisible(true); - result.then(res => { - notification.close(taskKey); - setExplainData(res.datas); - }) }; const onGetStreamGraph=()=>{ @@ -431,12 +402,8 @@ const StudioMenu = (props: any) => { { - handleModalVisible(false); - setExplainData([]); - }} modalVisible={modalVisible} - data={explainData} + onClose={()=>{handleModalVisible(false)}} /> Date: Wed, 12 Jan 2022 17:00:01 +0800 Subject: [PATCH 02/10] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E5=89=8D=E7=AB=AF?= =?UTF-8?q?=E5=BF=AB=E6=8D=B7=E9=94=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dlink-web/src/components/Studio/StudioEdit/index.tsx | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dlink-web/src/components/Studio/StudioEdit/index.tsx b/dlink-web/src/components/Studio/StudioEdit/index.tsx index 8c3b257e5f..309088a337 100644 --- a/dlink-web/src/components/Studio/StudioEdit/index.tsx +++ b/dlink-web/src/components/Studio/StudioEdit/index.tsx @@ -141,14 +141,14 @@ const FlinkSqlEditor = (props:any) => { const editorDidMountHandle = (editor: any, monaco: any) => { monacoInstance.current = monaco; editorInstance.current = editor; - editor.addCommand(monaco.KeyMod.CtrlCmd|monaco.KeyCode.KEY_1,function (){ + editor.addCommand(monaco.KeyMod.Alt|monaco.KeyCode.KEY_1,function (){ props.saveText(tabs,tabIndex); }) - editor.addCommand(monaco.KeyMod.CtrlCmd|monaco.KeyCode.KEY_2,function (){ + editor.addCommand(monaco.KeyMod.Alt|monaco.KeyCode.KEY_2,function (){ handleModalVisible(true); }) - editor.addCommand(monaco.KeyMod.CtrlCmd|monaco.KeyCode.KEY_3,function (){ + editor.addCommand(monaco.KeyMod.Alt|monaco.KeyCode.KEY_3,function (){ editor.getAction(['editor.action.formatDocument'])._run(); }) provider.dispose();// 清空提示项 From a052db280a24becc88e2da245f8e36eda565e780 Mon Sep 17 00:00:00 2001 From: wenmo <32723967+wenmo@users.noreply.github.com> Date: Wed, 12 Jan 2022 20:25:55 +0800 Subject: [PATCH 03/10] =?UTF-8?q?sql=E6=A0=A1=E9=AA=8Cbug=E4=BF=AE?= =?UTF-8?q?=E5=A4=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../components/Studio/StudioConsole/StudioExplain/index.tsx | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/dlink-web/src/components/Studio/StudioConsole/StudioExplain/index.tsx b/dlink-web/src/components/Studio/StudioConsole/StudioExplain/index.tsx index 81226a80c0..a2d30a2983 100644 --- a/dlink-web/src/components/Studio/StudioConsole/StudioExplain/index.tsx +++ b/dlink-web/src/components/Studio/StudioConsole/StudioExplain/index.tsx @@ -35,6 +35,9 @@ const StudioExplain = (props: any) => { } = props; useEffect(() => { + if(!modalVisible){ + return; + } let selectsql = null; if (current.monaco.current) { let selection = current.monaco.current.editor.getSelection(); @@ -55,7 +58,7 @@ const StudioExplain = (props: any) => { result.then(res => { setExplainData(res.datas); }) - }, []) + }, [modalVisible]) const renderFooter = () => { return ( From f9cfbc03ca8172d59232af658ef9dac68ca3e6e6 Mon Sep 17 00:00:00 2001 From: godkaikai <32723967+godkaikai@users.noreply.github.com> Date: Thu, 13 Jan 2022 21:19:07 +0800 Subject: [PATCH 04/10] =?UTF-8?q?=E7=89=87=E6=AE=B5=E6=9C=BA=E5=88=B6bug?= =?UTF-8?q?=E8=A7=A3=E5=86=B3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 1 + .../java/com/dlink/explainer/Explainer.java | 3 ++- .../com/dlink/constant/FlinkSQLConstant.java | 4 ++++ .../java/com/dlink/executor/SqlManager.java | 18 +++++++++++------- dlink-web/src/pages/Welcome.tsx | 9 ++++++--- docs/en-US/guide/quickstart.md | 1 + docs/guide/quickstart.md | 1 + 7 files changed, 26 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 8a92955d7a..f08c0699b5 100644 --- a/README.md +++ b/README.md @@ -49,6 +49,7 @@ Dinky（原 Dlink）： | | | 新增 选中片段执行 | 0.4.0 | | | | 新增 布局拖拽 | 0.4.0 | | | | 新增 SQL导出 | 0.5.0 | +| | | 新增 快捷键保存、校验、美化 | 0.5.0 | | | | 支持 local 模式下 FlinkSQL 提交 | 0.4.0 | | | | 支持 standalone 模式下 FlinkSQL 提交 | 0.4.0 | | | | 支持 yarn session 模式下 FlinkSQL 提交 | 0.4.0 | diff --git a/dlink-core/src/main/java/com/dlink/explainer/Explainer.java b/dlink-core/src/main/java/com/dlink/explainer/Explainer.java index 9f14ac9922..b1c696c746 100644 --- a/dlink-core/src/main/java/com/dlink/explainer/Explainer.java +++ b/dlink-core/src/main/java/com/dlink/explainer/Explainer.java @@ -9,6 +9,7 @@ import com.dlink.interceptor.FlinkInterceptor; import com.dlink.job.JobParam; import com.dlink.job.StatementParam; +import com.dlink.model.SystemConfiguration; import com.dlink.parser.SqlType; import com.dlink.result.ExplainResult; import com.dlink.result.SqlExplainResult; @@ -162,7 +163,7 @@ record = executor.explainSqlRecord(item.getValue()); } } if (inserts.size() > 0) { - String sqlSet = String.join(FlinkSQLConstant.SEPARATOR, inserts); + String sqlSet = String.join(";\r\n ", inserts); try { record.setExplain(executor.explainStatementSet(inserts)); record.setParseTrue(true); diff --git a/dlink-executor/src/main/java/com/dlink/constant/FlinkSQLConstant.java b/dlink-executor/src/main/java/com/dlink/constant/FlinkSQLConstant.java index ac3580ab18..877f2829f9 100644 --- a/dlink-executor/src/main/java/com/dlink/constant/FlinkSQLConstant.java +++ b/dlink-executor/src/main/java/com/dlink/constant/FlinkSQLConstant.java @@ -19,4 +19,8 @@ public interface FlinkSQLConstant { * DML 类型 */ String DML = "DML"; + /** + * 片段 Fragments 标识 + */ + String FRAGMENTS = ":="; } diff --git a/dlink-executor/src/main/java/com/dlink/executor/SqlManager.java b/dlink-executor/src/main/java/com/dlink/executor/SqlManager.java index b60a096e75..51021413dc 100644 --- a/dlink-executor/src/main/java/com/dlink/executor/SqlManager.java +++ b/dlink-executor/src/main/java/com/dlink/executor/SqlManager.java @@ -1,5 +1,8 @@ package com.dlink.executor; +import com.dlink.assertion.Asserts; +import com.dlink.constant.FlinkSQLConstant; +import com.dlink.model.SystemConfiguration; import org.apache.flink.table.api.DataTypes; import org.apache.flink.table.api.ExpressionParserException; import org.apache.flink.table.api.Table; @@ -148,23 +151,24 @@ public boolean checkShowFragments(String sql){ * @throws ExpressionParserException if the name of the variable under the given sql failed. */ public String parseVariable(String statement) { - if (statement == null || "".equals(statement)) { + if (Asserts.isNullString(statement)) { return statement; } - String[] strs = statement.split(";"); + String[] strs = statement.split(SystemConfiguration.getInstances().getSqlSeparator()); StringBuilder sb = new StringBuilder(); for (int i = 0; i < strs.length; i++) { - String str = strs[i].trim(); - if (str.length() == 0) { + String str = strs[i]; + if (str.trim().length() == 0) { continue; } - if (str.contains(":=")) { - String[] strs2 = str.split(":="); + str = strs[i]; + if (str.contains(FlinkSQLConstant.FRAGMENTS)) { + String[] strs2 = str.split(FlinkSQLConstant.FRAGMENTS); if (strs2.length >= 2) { if (strs2[0].length() == 0) { throw new ExpressionParserException("Illegal variable name."); } - String valueString = str.substring(str.indexOf(":=") + 2); + String valueString = str.substring(str.indexOf(FlinkSQLConstant.FRAGMENTS) + 2); this.registerSqlFragment(strs2[0], replaceVariable(valueString)); } else { throw new ExpressionParserException("Illegal variable definition."); diff --git a/dlink-web/src/pages/Welcome.tsx b/dlink-web/src/pages/Welcome.tsx index 2178d406e3..ff6dcd6058 100644 --- a/dlink-web/src/pages/Welcome.tsx +++ b/dlink-web/src/pages/Welcome.tsx @@ -530,7 +530,7 @@ export default (): React.ReactNode => { 修复 set 语法在1.11和1.12的兼容问题
  • - 升级各版本 Flink 依赖至最新版本以解决核弹问题 + 升级 各版本 Flink 依赖至最新版本以解决核弹问题
  • 新增 Yarn 的 Kerboros 验证 @@ -539,10 +539,13 @@ export default (): React.ReactNode => { 新增 ChangLog 和 Table 的查询及自动停止实现
  • - 修改项目名为 Dinky 以及图标 + 修改 项目名为 Dinky 以及图标
  • - 优化血缘分析图 + 优化 血缘分析图 +
    • +
  • + 新增 快捷键保存、校验、美化
    • diff --git a/docs/en-US/guide/quickstart.md b/docs/en-US/guide/quickstart.md index 8d1bd25408..91b23ee59d 100644 --- a/docs/en-US/guide/quickstart.md +++ b/docs/en-US/guide/quickstart.md @@ -87,6 +87,7 @@ Dinky 通过已注册的集群配置来获取对应的 YarnClient 实例。对 | | | 新增 选中片段执行 | 0.4.0 | | | | 新增 布局拖拽 | 0.4.0 | | | | 新增 SQL导出 | 0.5.0 | +| | | 新增 快捷键保存、校验、美化 | 0.5.0 | | | | 支持 local 模式下 FlinkSQL 提交 | 0.4.0 | | | | 支持 standalone 模式下 FlinkSQL 提交 | 0.4.0 | | | | 支持 yarn session 模式下 FlinkSQL 提交 | 0.4.0 | diff --git a/docs/guide/quickstart.md b/docs/guide/quickstart.md index 8d1bd25408..91b23ee59d 100644 --- a/docs/guide/quickstart.md +++ b/docs/guide/quickstart.md @@ -87,6 +87,7 @@ Dinky 通过已注册的集群配置来获取对应的 YarnClient 实例。对 | | | 新增 选中片段执行 | 0.4.0 | | | | 新增 布局拖拽 | 0.4.0 | | | | 新增 SQL导出 | 0.5.0 | +| | | 新增 快捷键保存、校验、美化 | 0.5.0 | | | | 支持 local 模式下 FlinkSQL 提交 | 0.4.0 | | | | 支持 standalone 模式下 FlinkSQL 提交 | 0.4.0 | | | | 支持 yarn session 模式下 FlinkSQL 提交 | 0.4.0 | From 2a2d4f6bf1a294c1806b59068e374c00e431a4da Mon Sep 17 00:00:00 2001 From: wenmo <32723967+wenmo@users.noreply.github.com> Date: Thu, 13 Jan 2022 23:54:25 +0800 Subject: [PATCH 05/10] =?UTF-8?q?UDFJava=20=E8=B0=83=E8=AF=95=E4=B8=8E?= =?UTF-8?q?=E5=8A=A0=E8=BD=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 4 +- .../java/com/dlink/service/TaskService.java | 2 + .../dlink/service/impl/StudioServiceImpl.java | 14 + .../dlink/service/impl/TaskServiceImpl.java | 14 + .../types/extraction/ExtractionUtils.java | 947 +++++++++++++++++ .../types/extraction/ExtractionUtils.java | 947 +++++++++++++++++ .../types/extraction/ExtractionUtils.java | 985 +++++++++++++++++ .../types/extraction/ExtractionUtils.java | 986 ++++++++++++++++++ .../main/java/com/dlink/pool/ClassEntity.java | 42 + .../main/java/com/dlink/pool/ClassPool.java | 61 ++ dlink-core/pom.xml | 5 + .../main/java/com/dlink/job/JobManager.java | 23 + .../dlink/utils/CustomStringJavaCompiler.java | 166 +++ .../main/java/com/dlink/utils/UDFUtil.java | 43 + .../StudioRightTool/StudioUDFInfo/index.less | 9 + .../StudioRightTool/StudioUDFInfo/index.tsx | 56 + .../Studio/StudioRightTool/index.tsx | 11 +- dlink-web/src/pages/Welcome.tsx | 3 + docs/en-US/guide/quickstart.md | 2 +- docs/guide/quickstart.md | 2 +- 20 files changed, 4317 insertions(+), 5 deletions(-) create mode 100644 dlink-client/dlink-client-1.11/src/main/java/org/apache/flink/table/types/extraction/ExtractionUtils.java create mode 100644 dlink-client/dlink-client-1.12/src/main/java/org/apache/flink/table/types/extraction/ExtractionUtils.java create mode 100644 dlink-client/dlink-client-1.13/src/main/java/org/apache/flink/table/types/extraction/ExtractionUtils.java create mode 100644 dlink-client/dlink-client-1.14/src/main/java/org/apache/flink/table/types/extraction/ExtractionUtils.java create mode 100644 dlink-common/src/main/java/com/dlink/pool/ClassEntity.java create mode 100644 dlink-common/src/main/java/com/dlink/pool/ClassPool.java create mode 100644 dlink-core/src/main/java/com/dlink/utils/CustomStringJavaCompiler.java create mode 100644 dlink-core/src/main/java/com/dlink/utils/UDFUtil.java create mode 100644 dlink-web/src/components/Studio/StudioRightTool/StudioUDFInfo/index.less create mode 100644 dlink-web/src/components/Studio/StudioRightTool/StudioUDFInfo/index.tsx diff --git a/README.md b/README.md index f08c0699b5..fc5d5f8fbe 100644 --- a/README.md +++ b/README.md @@ -49,7 +49,7 @@ Dinky（原 Dlink）： | | | 新增 选中片段执行 | 0.4.0 | | | | 新增 布局拖拽 | 0.4.0 | | | | 新增 SQL导出 | 0.5.0 | -| | | 新增 快捷键保存、校验、美化 | 0.5.0 | +| | | 新增 快捷键保存、校验、美化 | 0.5.0 | | | | 支持 local 模式下 FlinkSQL 提交 | 0.4.0 | | | | 支持 standalone 模式下 FlinkSQL 提交 | 0.4.0 | | | | 支持 yarn session 模式下 FlinkSQL 提交 | 0.4.0 | @@ -57,12 +57,12 @@ Dinky（原 Dlink）： | | | 支持 yarn application 模式下 FlinkSQL 提交 | 0.4.0 | | | | 支持 kubernetes session 模式下 FlinkSQL 提交 | 0.5.0 | | | | 支持 kubernetes application 模式下 FlinkSQL 提交 | 0.5.0 | +| | | 支持 UDF Java 方言Local模式在线编写、调试、动态加载 | 0.5.0 | | | Flink 作业 | 支持 yarn application 模式下 Jar 提交 | 0.4.0 | | | | 支持 k8s application 模式下 Jar 提交 | 0.5.0 | | | | 支持 作业 Cancel | 0.4.0 | | | | 支持 作业 SavePoint 的 Cancel、Stop、Trigger | 0.4.0 | | | | 新增 作业自动从 SavePoint 恢复机制（包含最近、最早、指定一次） | 0.4.0 | -| | | 新增 UDF java方言代码的开发 | 0.5.0 | | | Flink 集群 | 支持 查看已注册集群的作业列表与运维 | 0.4.0 | | | | 新增 自动注册 Yarn 创建的集群 | 0.4.0 | | | SQL | 新增 外部数据源的 SQL 校验 | 0.5.0 | diff --git a/dlink-admin/src/main/java/com/dlink/service/TaskService.java b/dlink-admin/src/main/java/com/dlink/service/TaskService.java index fd886e69e4..1954aef5d0 100644 --- a/dlink-admin/src/main/java/com/dlink/service/TaskService.java +++ b/dlink-admin/src/main/java/com/dlink/service/TaskService.java @@ -24,4 +24,6 @@ public interface TaskService extends ISuperService { List listFlinkSQLEnv(); String exportSql(Integer id); + + Task getUDFByClassName(String className); } diff --git a/dlink-admin/src/main/java/com/dlink/service/impl/StudioServiceImpl.java b/dlink-admin/src/main/java/com/dlink/service/impl/StudioServiceImpl.java index 55593c4f2e..32fa612581 100644 --- a/dlink-admin/src/main/java/com/dlink/service/impl/StudioServiceImpl.java +++ b/dlink-admin/src/main/java/com/dlink/service/impl/StudioServiceImpl.java @@ -27,6 +27,7 @@ import com.dlink.session.SessionInfo; import com.dlink.session.SessionPool; import com.dlink.utils.RunTimeUtil; +import com.dlink.utils.UDFUtil; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; @@ -89,6 +90,7 @@ private JobResult executeFlinkSql(StudioExecuteDTO studioExecuteDTO) { if(!config.isUseSession()) { config.setAddress(clusterService.buildEnvironmentAddress(config.isUseRemote(), studioExecuteDTO.getClusterId())); } + initUDF(config,studioExecuteDTO.getStatement()); JobManager jobManager = JobManager.build(config); JobResult jobResult = jobManager.executeSql(studioExecuteDTO.getStatement()); RunTimeUtil.recovery(jobManager); @@ -152,6 +154,7 @@ private List explainFlinkSql(StudioExecuteDTO studioExecuteDTO if(!config.isUseSession()) { config.setAddress(clusterService.buildEnvironmentAddress(config.isUseRemote(), studioExecuteDTO.getClusterId())); } + initUDF(config,studioExecuteDTO.getStatement()); JobManager jobManager = JobManager.buildPlanMode(config); return jobManager.explainSql(studioExecuteDTO.getStatement()).getSqlExplainResults(); } @@ -317,4 +320,15 @@ public boolean savepoint(Integer clusterId, String jobId, String savePointType,S } return false; } + + private void initUDF(JobConfig config,String statement){ + if(!GatewayType.LOCAL.equalsValue(config.getType())){ + return; + } + List udfClassNameList = JobManager.getUDFClassName(statement); + for(String item : udfClassNameList){ + Task task = taskService.getUDFByClassName(item); + JobManager.initUDF(item,task.getStatement()); + } + } } diff --git a/dlink-admin/src/main/java/com/dlink/service/impl/TaskServiceImpl.java b/dlink-admin/src/main/java/com/dlink/service/impl/TaskServiceImpl.java index 8a9d6c293d..ed20a141af 100644 --- a/dlink-admin/src/main/java/com/dlink/service/impl/TaskServiceImpl.java +++ b/dlink-admin/src/main/java/com/dlink/service/impl/TaskServiceImpl.java @@ -15,6 +15,7 @@ import com.dlink.mapper.TaskMapper; import com.dlink.model.*; import com.dlink.service.*; +import com.dlink.utils.CustomStringJavaCompiler; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; @@ -100,6 +101,11 @@ public Task getTaskInfoById(Integer id) { @Override public boolean saveOrUpdateTask(Task task) { + if(Asserts.isNotNullString(task.getDialect()) && Dialect.JAVA.equalsVal(task.getDialect()) + && Asserts.isNotNullString(task.getStatement()) ){ + CustomStringJavaCompiler compiler = new CustomStringJavaCompiler(task.getStatement()); + task.setSavePointPath(compiler.getFullClassName()); + } if (task.getId() != null) { this.updateById(task); if (task.getStatement() != null) { @@ -151,6 +157,14 @@ public String exportSql(Integer id) { } } + @Override + public Task getUDFByClassName(String className) { + Task task = getOne(new QueryWrapper().eq("dialect", "Java").eq("enabled", 1).eq("save_point_path", className)); + Assert.check(task); + task.setStatement(statementService.getById(task.getId()).getStatement()); + return task; + } + private JobConfig buildJobConfig(Task task){ boolean isJarTask = isJarTask(task); if(!isJarTask&&Asserts.isNotNull(task.getEnvId())){ diff --git a/dlink-client/dlink-client-1.11/src/main/java/org/apache/flink/table/types/extraction/ExtractionUtils.java b/dlink-client/dlink-client-1.11/src/main/java/org/apache/flink/table/types/extraction/ExtractionUtils.java new file mode 100644 index 0000000000..b6dfa55f9c --- /dev/null +++ b/dlink-client/dlink-client-1.11/src/main/java/org/apache/flink/table/types/extraction/ExtractionUtils.java @@ -0,0 +1,947 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.types.extraction; + +import com.dlink.pool.ClassPool; +import org.apache.flink.annotation.Internal; +import org.apache.flink.api.common.typeutils.TypeSerializer; +import org.apache.flink.table.api.DataTypes; +import org.apache.flink.table.api.ValidationException; +import org.apache.flink.table.catalog.DataTypeFactory; +import org.apache.flink.table.types.DataType; +import org.apache.flink.table.types.logical.StructuredType; + +import org.apache.flink.shaded.asm7.org.objectweb.asm.ClassReader; +import org.apache.flink.shaded.asm7.org.objectweb.asm.ClassVisitor; +import org.apache.flink.shaded.asm7.org.objectweb.asm.Label; +import org.apache.flink.shaded.asm7.org.objectweb.asm.MethodVisitor; +import org.apache.flink.shaded.asm7.org.objectweb.asm.Opcodes; + +import javax.annotation.Nullable; + +import java.io.IOException; +import java.lang.annotation.Annotation; +import java.lang.reflect.Constructor; +import java.lang.reflect.Executable; +import java.lang.reflect.Field; +import java.lang.reflect.Method; +import java.lang.reflect.Modifier; +import java.lang.reflect.Parameter; +import java.lang.reflect.ParameterizedType; +import java.lang.reflect.Type; +import java.lang.reflect.TypeVariable; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.apache.flink.shaded.asm7.org.objectweb.asm.Type.getConstructorDescriptor; +import static org.apache.flink.shaded.asm7.org.objectweb.asm.Type.getMethodDescriptor; + +/** Utilities for performing reflection tasks. */ +@Internal +public final class ExtractionUtils { + + // -------------------------------------------------------------------------------------------- + // Methods shared across packages + // -------------------------------------------------------------------------------------------- + + /** Collects methods of the given name. */ + public static List collectMethods(Class function, String methodName) { + return Arrays.stream(function.getMethods()) + .filter(method -> method.getName().equals(methodName)) + .sorted(Comparator.comparing(Method::toString)) // for deterministic order + .collect(Collectors.toList()); + } + + /** + * Checks whether a method/constructor can be called with the given argument classes. This + * includes type widening and vararg. {@code null} is a wildcard. + * + *

    E.g., {@code (int.class, int.class)} matches {@code f(Object...), f(int, int), f(Integer, + * Object)} and so forth. + */ + public static boolean isInvokable(Executable executable, Class... classes) { + final int m = executable.getModifiers(); + if (!Modifier.isPublic(m)) { + return false; + } + final int paramCount = executable.getParameterCount(); + final int classCount = classes.length; + // check for enough classes for each parameter + if ((!executable.isVarArgs() && classCount != paramCount) + || (executable.isVarArgs() && classCount < paramCount - 1)) { + return false; + } + int currentClass = 0; + for (int currentParam = 0; currentParam < paramCount; currentParam++) { + final Class param = executable.getParameterTypes()[currentParam]; + // last parameter is a vararg that needs to consume remaining classes + if (currentParam == paramCount - 1 && executable.isVarArgs()) { + final Class paramComponent = + executable.getParameterTypes()[currentParam].getComponentType(); + // we have more than 1 classes left so the vararg needs to consume them all + if (classCount - currentClass > 1) { + while (currentClass < classCount + && ExtractionUtils.isAssignable( + classes[currentClass], paramComponent, true)) { + currentClass++; + } + } else if (currentClass < classCount + && (parameterMatches(classes[currentClass], param) + || parameterMatches(classes[currentClass], paramComponent))) { + currentClass++; + } + } + // entire parameter matches + else if (parameterMatches(classes[currentClass], param)) { + currentClass++; + } + } + // check if all classes have been consumed + return currentClass == classCount; + } + + private static boolean parameterMatches(Class clz, Class param) { + return clz == null || ExtractionUtils.isAssignable(clz, param, true); + } + + /** Creates a method signature string like {@code int eval(Integer, String)}. */ + public static String createMethodSignatureString( + String methodName, Class[] parameters, @Nullable Class returnType) { + final StringBuilder builder = new StringBuilder(); + if (returnType != null) { + builder.append(returnType.getCanonicalName()).append(" "); + } + builder.append(methodName) + .append( + Stream.of(parameters) + .map( + parameter -> { + // in case we don't know the parameter at this location + // (i.e. for accumulators) + if (parameter == null) { + return "_"; + } else { + return parameter.getCanonicalName(); + } + }) + .collect(Collectors.joining(", ", "(", ")"))); + return builder.toString(); + } + + /** + * Validates the characteristics of a class for a {@link StructuredType} such as accessibility. + */ + public static void validateStructuredClass(Class clazz) { + final int m = clazz.getModifiers(); + if (Modifier.isAbstract(m)) { + throw extractionError("Class '%s' must not be abstract.", clazz.getName()); + } + if (!Modifier.isPublic(m)) { + throw extractionError("Class '%s' is not public.", clazz.getName()); + } + if (clazz.getEnclosingClass() != null + && (clazz.getDeclaringClass() == null || !Modifier.isStatic(m))) { + throw extractionError( + "Class '%s' is a not a static, globally accessible class.", clazz.getName()); + } + } + + /** + * Returns the field of a structured type. The logic is as broad as possible to support both + * Java and Scala in different flavors. + */ + public static Field getStructuredField(Class clazz, String fieldName) { + final String normalizedFieldName = fieldName.toUpperCase(); + + final List fields = collectStructuredFields(clazz); + for (Field field : fields) { + if (field.getName().toUpperCase().equals(normalizedFieldName)) { + return field; + } + } + throw extractionError( + "Could not to find a field named '%s' in class '%s' for structured type.", + fieldName, clazz.getName()); + } + + /** + * Checks for a field getter of a structured type. The logic is as broad as possible to support + * both Java and Scala in different flavors. + */ + public static Optional getStructuredFieldGetter(Class clazz, Field field) { + final String normalizedFieldName = field.getName().toUpperCase(); + + final List methods = collectStructuredMethods(clazz); + for (Method method : methods) { + // check name: + // get() + // is() + // () for Scala + final String normalizedMethodName = method.getName().toUpperCase(); + final boolean hasName = + normalizedMethodName.equals("GET" + normalizedFieldName) + || normalizedMethodName.equals("IS" + normalizedFieldName) + || normalizedMethodName.equals(normalizedFieldName); + if (!hasName) { + continue; + } + + // check return type: + // equal to field type + final Type returnType = method.getGenericReturnType(); + final boolean hasReturnType = returnType.equals(field.getGenericType()); + if (!hasReturnType) { + continue; + } + + // check parameters: + // no parameters + final boolean hasNoParameters = method.getParameterCount() == 0; + if (!hasNoParameters) { + continue; + } + + // matching getter found + return Optional.of(method); + } + + // no getter found + return Optional.empty(); + } + + /** + * Checks for a field setters of a structured type. The logic is as broad as possible to support + * both Java and Scala in different flavors. + */ + public static Optional getStructuredFieldSetter(Class clazz, Field field) { + final String normalizedFieldName = field.getName().toUpperCase(); + + final List methods = collectStructuredMethods(clazz); + for (Method method : methods) { + + // check name: + // set(type) + // (type) + // _$eq(type) for Scala + final String normalizedMethodName = method.getName().toUpperCase(); + final boolean hasName = + normalizedMethodName.equals("SET" + normalizedFieldName) + || normalizedMethodName.equals(normalizedFieldName) + || normalizedMethodName.equals(normalizedFieldName + "_$EQ"); + if (!hasName) { + continue; + } + + // check return type: + // void or the declaring class + final Class returnType = method.getReturnType(); + final boolean hasReturnType = returnType == Void.TYPE || returnType == clazz; + if (!hasReturnType) { + continue; + } + + // check parameters: + // one parameter that has the same (or primitive) type of the field + final boolean hasParameter = + method.getParameterCount() == 1 + && (method.getGenericParameterTypes()[0].equals(field.getGenericType()) + || primitiveToWrapper(method.getGenericParameterTypes()[0]) + .equals(field.getGenericType())); + if (!hasParameter) { + continue; + } + + // matching setter found + return Optional.of(method); + } + + // no setter found + return Optional.empty(); + } + + /** + * Checks for an invokable constructor matching the given arguments. + * + * @see #isInvokable(Executable, Class[]) + */ + public static boolean hasInvokableConstructor(Class clazz, Class... classes) { + for (Constructor constructor : clazz.getDeclaredConstructors()) { + if (isInvokable(constructor, classes)) { + return true; + } + } + return false; + } + + /** Checks whether a field is directly readable without a getter. */ + public static boolean isStructuredFieldDirectlyReadable(Field field) { + final int m = field.getModifiers(); + + // field is directly readable + return Modifier.isPublic(m); + } + + /** Checks whether a field is directly writable without a setter or constructor. */ + public static boolean isStructuredFieldDirectlyWritable(Field field) { + final int m = field.getModifiers(); + + // field is immutable + if (Modifier.isFinal(m)) { + return false; + } + + // field is directly writable + return Modifier.isPublic(m); + } + + // -------------------------------------------------------------------------------------------- + // Methods intended for this package + // -------------------------------------------------------------------------------------------- + + /** Helper method for creating consistent exceptions during extraction. */ + static ValidationException extractionError(String message, Object... args) { + return extractionError(null, message, args); + } + + /** Helper method for creating consistent exceptions during extraction. */ + static ValidationException extractionError(Throwable cause, String message, Object... args) { + return new ValidationException(String.format(message, args), cause); + } + + /** + * Collects the partially ordered type hierarchy (i.e. all involved super classes and super + * interfaces) of the given type. + */ + static List collectTypeHierarchy(Type type) { + Type currentType = type; + Class currentClass = toClass(type); + final List typeHierarchy = new ArrayList<>(); + while (currentClass != null) { + // collect type + typeHierarchy.add(currentType); + // collect super interfaces + for (Type genericInterface : currentClass.getGenericInterfaces()) { + final Class interfaceClass = toClass(genericInterface); + if (interfaceClass != null) { + typeHierarchy.addAll(collectTypeHierarchy(genericInterface)); + } + } + currentType = currentClass.getGenericSuperclass(); + currentClass = toClass(currentType); + } + return typeHierarchy; + } + + /** Converts a {@link Type} to {@link Class} if possible, {@code null} otherwise. */ + static @Nullable Class toClass(Type type) { + if (type instanceof Class) { + return (Class) type; + } else if (type instanceof ParameterizedType) { + // this is always a class + return (Class) ((ParameterizedType) type).getRawType(); + } + // unsupported: generic arrays, type variables, wildcard types + return null; + } + + /** Creates a raw data type. */ + @SuppressWarnings({"unchecked", "rawtypes"}) + static DataType createRawType( + DataTypeFactory typeFactory, + @Nullable Class> rawSerializer, + @Nullable Class conversionClass) { + if (rawSerializer != null) { + return DataTypes.RAW( + (Class) createConversionClass(conversionClass), + instantiateRawSerializer(rawSerializer)); + } + return typeFactory.createRawDataType(createConversionClass(conversionClass)); + } + + static Class createConversionClass(@Nullable Class conversionClass) { + if (conversionClass != null) { + return conversionClass; + } + return Object.class; + } + + private static TypeSerializer instantiateRawSerializer( + Class> rawSerializer) { + try { + return rawSerializer.newInstance(); + } catch (Exception e) { + throw extractionError( + e, + "Cannot instantiate type serializer '%s' for RAW type. " + + "Make sure the class is publicly accessible and has a default constructor.", + rawSerializer.getName()); + } + } + + /** Resolves a {@link TypeVariable} using the given type hierarchy if possible. */ + static Type resolveVariable(List typeHierarchy, TypeVariable variable) { + // iterate through hierarchy from top to bottom until type variable gets a non-variable + // assigned + for (int i = typeHierarchy.size() - 1; i >= 0; i--) { + final Type currentType = typeHierarchy.get(i); + + if (currentType instanceof ParameterizedType) { + final Type resolvedType = + resolveVariableInParameterizedType( + variable, (ParameterizedType) currentType); + if (resolvedType instanceof TypeVariable) { + // follow type variables transitively + variable = (TypeVariable) resolvedType; + } else if (resolvedType != null) { + return resolvedType; + } + } + } + // unresolved variable + return variable; + } + + private static @Nullable Type resolveVariableInParameterizedType( + TypeVariable variable, ParameterizedType currentType) { + final Class currentRaw = (Class) currentType.getRawType(); + final TypeVariable[] currentVariables = currentRaw.getTypeParameters(); + // search for matching type variable + for (int paramPos = 0; paramPos < currentVariables.length; paramPos++) { + if (typeVariableEquals(variable, currentVariables[paramPos])) { + return currentType.getActualTypeArguments()[paramPos]; + } + } + return null; + } + + private static boolean typeVariableEquals( + TypeVariable variable, TypeVariable currentVariable) { + return currentVariable.getGenericDeclaration().equals(variable.getGenericDeclaration()) + && currentVariable.getName().equals(variable.getName()); + } + + /** + * Validates if a given type is not already contained in the type hierarchy of a structured + * type. + * + *

    Otherwise this would lead to infinite data type extraction cycles. + */ + static void validateStructuredSelfReference(Type t, List typeHierarchy) { + final Class clazz = toClass(t); + if (clazz != null + && !clazz.isInterface() + && clazz != Object.class + && typeHierarchy.contains(t)) { + throw extractionError( + "Cyclic reference detected for class '%s'. Attributes of structured types must not " + + "(transitively) reference the structured type itself.", + clazz.getName()); + } + } + + /** Returns the fields of a class for a {@link StructuredType}. */ + static List collectStructuredFields(Class clazz) { + final List fields = new ArrayList<>(); + while (clazz != Object.class) { + final Field[] declaredFields = clazz.getDeclaredFields(); + Stream.of(declaredFields) + .filter( + field -> { + final int m = field.getModifiers(); + return !Modifier.isStatic(m) && !Modifier.isTransient(m); + }) + .forEach(fields::add); + clazz = clazz.getSuperclass(); + } + return fields; + } + + /** Validates if a field is properly readable either directly or through a getter. */ + static void validateStructuredFieldReadability(Class clazz, Field field) { + // field is accessible + if (isStructuredFieldDirectlyReadable(field)) { + return; + } + + // field needs a getter + if (!getStructuredFieldGetter(clazz, field).isPresent()) { + throw extractionError( + "Field '%s' of class '%s' is neither publicly accessible nor does it have " + + "a corresponding getter method.", + field.getName(), clazz.getName()); + } + } + + /** + * Checks if a field is mutable or immutable. Returns {@code true} if the field is properly + * mutable. Returns {@code false} if it is properly immutable. + */ + static boolean isStructuredFieldMutable(Class clazz, Field field) { + final int m = field.getModifiers(); + + // field is immutable + if (Modifier.isFinal(m)) { + return false; + } + // field is directly mutable + if (Modifier.isPublic(m)) { + return true; + } + + // field has setters by which it is mutable + if (getStructuredFieldSetter(clazz, field).isPresent()) { + return true; + } + + throw extractionError( + "Field '%s' of class '%s' is mutable but is neither publicly accessible nor does it have " + + "a corresponding setter method.", + field.getName(), clazz.getName()); + } + + /** Returns the boxed type of a primitive type. */ + static Type primitiveToWrapper(Type type) { + if (type instanceof Class) { + return primitiveToWrapper((Class) type); + } + return type; + } + + /** Collects all methods that qualify as methods of a {@link StructuredType}. */ + static List collectStructuredMethods(Class clazz) { + final List methods = new ArrayList<>(); + while (clazz != Object.class) { + final Method[] declaredMethods = clazz.getDeclaredMethods(); + Stream.of(declaredMethods) + .filter( + field -> { + final int m = field.getModifiers(); + return Modifier.isPublic(m) + && !Modifier.isNative(m) + && !Modifier.isAbstract(m); + }) + .forEach(methods::add); + clazz = clazz.getSuperclass(); + } + return methods; + } + + /** + * Collects all annotations of the given type defined in the current class or superclasses. + * Duplicates are ignored. + */ + static Set collectAnnotationsOfClass( + Class annotation, Class annotatedClass) { + final List> classHierarchy = new ArrayList<>(); + Class currentClass = annotatedClass; + while (currentClass != null) { + classHierarchy.add(currentClass); + currentClass = currentClass.getSuperclass(); + } + // convert to top down + Collections.reverse(classHierarchy); + return classHierarchy.stream() + .flatMap(c -> Stream.of(c.getAnnotationsByType(annotation))) + .collect(Collectors.toCollection(LinkedHashSet::new)); + } + + /** + * Collects all annotations of the given type defined in the given method. Duplicates are + * ignored. + */ + static Set collectAnnotationsOfMethod( + Class annotation, Method annotatedMethod) { + return new LinkedHashSet<>(Arrays.asList(annotatedMethod.getAnnotationsByType(annotation))); + } + + // -------------------------------------------------------------------------------------------- + // Parameter Extraction Utilities + // -------------------------------------------------------------------------------------------- + + /** Result of the extraction in {@link #extractAssigningConstructor(Class, List)}. */ + static class AssigningConstructor { + public final Constructor constructor; + public final List parameterNames; + + private AssigningConstructor(Constructor constructor, List parameterNames) { + this.constructor = constructor; + this.parameterNames = parameterNames; + } + } + + /** + * Checks whether the given constructor takes all of the given fields with matching (possibly + * primitive) type and name. An assigning constructor can define the order of fields. + */ + static @Nullable AssigningConstructor extractAssigningConstructor( + Class clazz, List fields) { + AssigningConstructor foundConstructor = null; + for (Constructor constructor : clazz.getDeclaredConstructors()) { + final boolean qualifyingConstructor = + Modifier.isPublic(constructor.getModifiers()) + && constructor.getParameterTypes().length == fields.size(); + if (!qualifyingConstructor) { + continue; + } + final List parameterNames = + extractConstructorParameterNames(constructor, fields); + if (parameterNames != null) { + if (foundConstructor != null) { + throw extractionError( + "Multiple constructors found that assign all fields for class '%s'.", + clazz.getName()); + } + foundConstructor = new AssigningConstructor(constructor, parameterNames); + } + } + return foundConstructor; + } + + /** Extracts the parameter names of a method if possible. */ + static @Nullable List extractMethodParameterNames(Method method) { + return extractExecutableNames(method); + } + + /** + * Extracts ordered parameter names from a constructor that takes all of the given fields with + * matching (possibly primitive) type and name. + */ + private static @Nullable List extractConstructorParameterNames( + Constructor constructor, List fields) { + final Type[] parameterTypes = constructor.getGenericParameterTypes(); + + List parameterNames = extractExecutableNames(constructor); + if (parameterNames == null) { + return null; + } + + final Map fieldMap = + fields.stream().collect(Collectors.toMap(Field::getName, Field::getGenericType)); + + // check that all fields are represented in the parameters of the constructor + for (int i = 0; i < parameterNames.size(); i++) { + final String parameterName = parameterNames.get(i); + final Type fieldType = fieldMap.get(parameterName); // might be null + final Type parameterType = parameterTypes[i]; + // we are tolerant here because frameworks such as Avro accept a boxed type even though + // the field is primitive + if (!primitiveToWrapper(parameterType).equals(primitiveToWrapper(fieldType))) { + return null; + } + } + + return parameterNames; + } + + private static @Nullable List extractExecutableNames(Executable executable) { + final int offset; + if (!Modifier.isStatic(executable.getModifiers())) { + // remove "this" as first parameter + offset = 1; + } else { + offset = 0; + } + // by default parameter names are "arg0, arg1, arg2, ..." if compiler flag is not set + // so we need to extract them manually if possible + List parameterNames = + Stream.of(executable.getParameters()) + .map(Parameter::getName) + .collect(Collectors.toList()); + if (parameterNames.stream().allMatch(n -> n.startsWith("arg"))) { + final ParameterExtractor extractor; + if (executable instanceof Constructor) { + extractor = new ParameterExtractor((Constructor) executable); + } else { + extractor = new ParameterExtractor((Method) executable); + } + getClassReader(executable.getDeclaringClass()).accept(extractor, 0); + + final List extractedNames = extractor.getParameterNames(); + if (extractedNames.size() == 0) { + return null; + } + // remove "this" and additional local variables + // select less names if class file has not the required information + parameterNames = + extractedNames.subList( + offset, + Math.min( + executable.getParameterCount() + offset, + extractedNames.size())); + } + + if (parameterNames.size() != executable.getParameterCount()) { + return null; + } + + return parameterNames; + } + + private static ClassReader getClassReader(Class cls) { + final String className = cls.getName().replaceFirst("^.*\\.", "") + ".class"; + if(ClassPool.exist(cls.getName())){ + return new ClassReader(ClassPool.get(cls.getName()).getClassByte()); + } + try { + return new ClassReader(cls.getResourceAsStream(className)); + } catch (IOException e) { + throw new IllegalStateException("Could not instantiate ClassReader.", e); + } + } + + /** + * Extracts the parameter names and descriptors from a constructor or method. Assuming the + * existence of a local variable table. + * + *

    For example: + * + * 

    {@code
+     * public WC(java.lang.String arg0, long arg1) { //  //(Ljava/lang/String;J)V
+     *   
+     *   
+     *   
+     *   
+     *   
+     * }
+     * }
    + */ + private static class ParameterExtractor extends ClassVisitor { + + private static final int OPCODE = Opcodes.ASM7; + + private final String methodDescriptor; + + private final List parameterNames = new ArrayList<>(); + + ParameterExtractor(Constructor constructor) { + super(OPCODE); + methodDescriptor = getConstructorDescriptor(constructor); + } + + ParameterExtractor(Method method) { + super(OPCODE); + methodDescriptor = getMethodDescriptor(method); + } + + List getParameterNames() { + return parameterNames; + } + + @Override + public MethodVisitor visitMethod( + int access, String name, String descriptor, String signature, String[] exceptions) { + if (descriptor.equals(methodDescriptor)) { + return new MethodVisitor(OPCODE) { + @Override + public void visitLocalVariable( + String name, + String descriptor, + String signature, + Label start, + Label end, + int index) { + parameterNames.add(name); + } + }; + } + return super.visitMethod(access, name, descriptor, signature, exceptions); + } + } + + // -------------------------------------------------------------------------------------------- + // Class Assignment and Boxing + // + // copied from o.a.commons.lang3.ClassUtils (commons-lang3:3.3.2) + // -------------------------------------------------------------------------------------------- + + /** + * Checks if one {@code Class} can be assigned to a variable of another {@code Class}. + * + *

    Unlike the {@link Class#isAssignableFrom(java.lang.Class)} method, this method takes into + * account widenings of primitive classes and {@code null}s. + * + *

    Primitive widenings allow an int to be assigned to a long, float or double. This method + * returns the correct result for these cases. + * + *

    {@code Null} may be assigned to any reference type. This method will return {@code true} + * if {@code null} is passed in and the toClass is non-primitive. + * + *

    Specifically, this method tests whether the type represented by the specified {@code + * Class} parameter can be converted to the type represented by this {@code Class} object via an + * identity conversion widening primitive or widening reference conversion. See The Java Language Specification, + * sections 5.1.1, 5.1.2 and 5.1.4 for details. + * + * @param cls the Class to check, may be null + * @param toClass the Class to try to assign into, returns false if null + * @param autoboxing whether to use implicit autoboxing/unboxing between primitives and wrappers + * @return {@code true} if assignment possible + */ + public static boolean isAssignable( + Class cls, final Class toClass, final boolean autoboxing) { + if (toClass == null) { + return false; + } + // have to check for null, as isAssignableFrom doesn't + if (cls == null) { + return !toClass.isPrimitive(); + } + // autoboxing: + if (autoboxing) { + if (cls.isPrimitive() && !toClass.isPrimitive()) { + cls = primitiveToWrapper(cls); + if (cls == null) { + return false; + } + } + if (toClass.isPrimitive() && !cls.isPrimitive()) { + cls = wrapperToPrimitive(cls); + if (cls == null) { + return false; + } + } + } + if (cls.equals(toClass)) { + return true; + } + if (cls.isPrimitive()) { + if (!toClass.isPrimitive()) { + return false; + } + if (Integer.TYPE.equals(cls)) { + return Long.TYPE.equals(toClass) + || Float.TYPE.equals(toClass) + || Double.TYPE.equals(toClass); + } + if (Long.TYPE.equals(cls)) { + return Float.TYPE.equals(toClass) || Double.TYPE.equals(toClass); + } + if (Boolean.TYPE.equals(cls)) { + return false; + } + if (Double.TYPE.equals(cls)) { + return false; + } + if (Float.TYPE.equals(cls)) { + return Double.TYPE.equals(toClass); + } + if (Character.TYPE.equals(cls)) { + return Integer.TYPE.equals(toClass) + || Long.TYPE.equals(toClass) + || Float.TYPE.equals(toClass) + || Double.TYPE.equals(toClass); + } + if (Short.TYPE.equals(cls)) { + return Integer.TYPE.equals(toClass) + || Long.TYPE.equals(toClass) + || Float.TYPE.equals(toClass) + || Double.TYPE.equals(toClass); + } + if (Byte.TYPE.equals(cls)) { + return Short.TYPE.equals(toClass) + || Integer.TYPE.equals(toClass) + || Long.TYPE.equals(toClass) + || Float.TYPE.equals(toClass) + || Double.TYPE.equals(toClass); + } + // should never get here + return false; + } + return toClass.isAssignableFrom(cls); + } + + /** Maps primitive {@code Class}es to their corresponding wrapper {@code Class}. */ + private static final Map, Class> primitiveWrapperMap = new HashMap<>(); + + static { + primitiveWrapperMap.put(Boolean.TYPE, Boolean.class); + primitiveWrapperMap.put(Byte.TYPE, Byte.class); + primitiveWrapperMap.put(Character.TYPE, Character.class); + primitiveWrapperMap.put(Short.TYPE, Short.class); + primitiveWrapperMap.put(Integer.TYPE, Integer.class); + primitiveWrapperMap.put(Long.TYPE, Long.class); + primitiveWrapperMap.put(Double.TYPE, Double.class); + primitiveWrapperMap.put(Float.TYPE, Float.class); + primitiveWrapperMap.put(Void.TYPE, Void.TYPE); + } + + /** Maps wrapper {@code Class}es to their corresponding primitive types. */ + private static final Map, Class> wrapperPrimitiveMap = new HashMap<>(); + + static { + for (final Class primitiveClass : primitiveWrapperMap.keySet()) { + final Class wrapperClass = primitiveWrapperMap.get(primitiveClass); + if (!primitiveClass.equals(wrapperClass)) { + wrapperPrimitiveMap.put(wrapperClass, primitiveClass); + } + } + } + + /** + * Converts the specified primitive Class object to its corresponding wrapper Class object. + * + *

    NOTE: From v2.2, this method handles {@code Void.TYPE}, returning {@code Void.TYPE}. + * + * @param cls the class to convert, may be null + * @return the wrapper class for {@code cls} or {@code cls} if {@code cls} is not a primitive. + * {@code null} if null input. + * @since 2.1 + */ + public static Class primitiveToWrapper(final Class cls) { + Class convertedClass = cls; + if (cls != null && cls.isPrimitive()) { + convertedClass = primitiveWrapperMap.get(cls); + } + return convertedClass; + } + + /** + * Converts the specified wrapper class to its corresponding primitive class. + * + *

    This method is the counter part of {@code primitiveToWrapper()}. If the passed in class is + * a wrapper class for a primitive type, this primitive type will be returned (e.g. {@code + * Integer.TYPE} for {@code Integer.class}). For other classes, or if the parameter is + * null, the return value is null. + * + * @param cls the class to convert, may be null + * @return the corresponding primitive type if {@code cls} is a wrapper class, null + * otherwise + * @see #primitiveToWrapper(Class) + * @since 2.4 + */ + public static Class wrapperToPrimitive(final Class cls) { + return wrapperPrimitiveMap.get(cls); + } + + // -------------------------------------------------------------------------------------------- + + private ExtractionUtils() { + // no instantiation + } +} diff --git a/dlink-client/dlink-client-1.12/src/main/java/org/apache/flink/table/types/extraction/ExtractionUtils.java b/dlink-client/dlink-client-1.12/src/main/java/org/apache/flink/table/types/extraction/ExtractionUtils.java new file mode 100644 index 0000000000..b6dfa55f9c --- /dev/null +++ b/dlink-client/dlink-client-1.12/src/main/java/org/apache/flink/table/types/extraction/ExtractionUtils.java @@ -0,0 +1,947 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.types.extraction; + +import com.dlink.pool.ClassPool; +import org.apache.flink.annotation.Internal; +import org.apache.flink.api.common.typeutils.TypeSerializer; +import org.apache.flink.table.api.DataTypes; +import org.apache.flink.table.api.ValidationException; +import org.apache.flink.table.catalog.DataTypeFactory; +import org.apache.flink.table.types.DataType; +import org.apache.flink.table.types.logical.StructuredType; + +import org.apache.flink.shaded.asm7.org.objectweb.asm.ClassReader; +import org.apache.flink.shaded.asm7.org.objectweb.asm.ClassVisitor; +import org.apache.flink.shaded.asm7.org.objectweb.asm.Label; +import org.apache.flink.shaded.asm7.org.objectweb.asm.MethodVisitor; +import org.apache.flink.shaded.asm7.org.objectweb.asm.Opcodes; + +import javax.annotation.Nullable; + +import java.io.IOException; +import java.lang.annotation.Annotation; +import java.lang.reflect.Constructor; +import java.lang.reflect.Executable; +import java.lang.reflect.Field; +import java.lang.reflect.Method; +import java.lang.reflect.Modifier; +import java.lang.reflect.Parameter; +import java.lang.reflect.ParameterizedType; +import java.lang.reflect.Type; +import java.lang.reflect.TypeVariable; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.apache.flink.shaded.asm7.org.objectweb.asm.Type.getConstructorDescriptor; +import static org.apache.flink.shaded.asm7.org.objectweb.asm.Type.getMethodDescriptor; + +/** Utilities for performing reflection tasks. */ +@Internal +public final class ExtractionUtils { + + // -------------------------------------------------------------------------------------------- + // Methods shared across packages + // -------------------------------------------------------------------------------------------- + + /** Collects methods of the given name. */ + public static List collectMethods(Class function, String methodName) { + return Arrays.stream(function.getMethods()) + .filter(method -> method.getName().equals(methodName)) + .sorted(Comparator.comparing(Method::toString)) // for deterministic order + .collect(Collectors.toList()); + } + + /** + * Checks whether a method/constructor can be called with the given argument classes. This + * includes type widening and vararg. {@code null} is a wildcard. + * + *

    E.g., {@code (int.class, int.class)} matches {@code f(Object...), f(int, int), f(Integer, + * Object)} and so forth. + */ + public static boolean isInvokable(Executable executable, Class... classes) { + final int m = executable.getModifiers(); + if (!Modifier.isPublic(m)) { + return false; + } + final int paramCount = executable.getParameterCount(); + final int classCount = classes.length; + // check for enough classes for each parameter + if ((!executable.isVarArgs() && classCount != paramCount) + || (executable.isVarArgs() && classCount < paramCount - 1)) { + return false; + } + int currentClass = 0; + for (int currentParam = 0; currentParam < paramCount; currentParam++) { + final Class param = executable.getParameterTypes()[currentParam]; + // last parameter is a vararg that needs to consume remaining classes + if (currentParam == paramCount - 1 && executable.isVarArgs()) { + final Class paramComponent = + executable.getParameterTypes()[currentParam].getComponentType(); + // we have more than 1 classes left so the vararg needs to consume them all + if (classCount - currentClass > 1) { + while (currentClass < classCount + && ExtractionUtils.isAssignable( + classes[currentClass], paramComponent, true)) { + currentClass++; + } + } else if (currentClass < classCount + && (parameterMatches(classes[currentClass], param) + || parameterMatches(classes[currentClass], paramComponent))) { + currentClass++; + } + } + // entire parameter matches + else if (parameterMatches(classes[currentClass], param)) { + currentClass++; + } + } + // check if all classes have been consumed + return currentClass == classCount; + } + + private static boolean parameterMatches(Class clz, Class param) { + return clz == null || ExtractionUtils.isAssignable(clz, param, true); + } + + /** Creates a method signature string like {@code int eval(Integer, String)}. */ + public static String createMethodSignatureString( + String methodName, Class[] parameters, @Nullable Class returnType) { + final StringBuilder builder = new StringBuilder(); + if (returnType != null) { + builder.append(returnType.getCanonicalName()).append(" "); + } + builder.append(methodName) + .append( + Stream.of(parameters) + .map( + parameter -> { + // in case we don't know the parameter at this location + // (i.e. for accumulators) + if (parameter == null) { + return "_"; + } else { + return parameter.getCanonicalName(); + } + }) + .collect(Collectors.joining(", ", "(", ")"))); + return builder.toString(); + } + + /** + * Validates the characteristics of a class for a {@link StructuredType} such as accessibility. + */ + public static void validateStructuredClass(Class clazz) { + final int m = clazz.getModifiers(); + if (Modifier.isAbstract(m)) { + throw extractionError("Class '%s' must not be abstract.", clazz.getName()); + } + if (!Modifier.isPublic(m)) { + throw extractionError("Class '%s' is not public.", clazz.getName()); + } + if (clazz.getEnclosingClass() != null + && (clazz.getDeclaringClass() == null || !Modifier.isStatic(m))) { + throw extractionError( + "Class '%s' is a not a static, globally accessible class.", clazz.getName()); + } + } + + /** + * Returns the field of a structured type. The logic is as broad as possible to support both + * Java and Scala in different flavors. + */ + public static Field getStructuredField(Class clazz, String fieldName) { + final String normalizedFieldName = fieldName.toUpperCase(); + + final List fields = collectStructuredFields(clazz); + for (Field field : fields) { + if (field.getName().toUpperCase().equals(normalizedFieldName)) { + return field; + } + } + throw extractionError( + "Could not to find a field named '%s' in class '%s' for structured type.", + fieldName, clazz.getName()); + } + + /** + * Checks for a field getter of a structured type. The logic is as broad as possible to support + * both Java and Scala in different flavors. + */ + public static Optional getStructuredFieldGetter(Class clazz, Field field) { + final String normalizedFieldName = field.getName().toUpperCase(); + + final List methods = collectStructuredMethods(clazz); + for (Method method : methods) { + // check name: + // get() + // is() + // () for Scala + final String normalizedMethodName = method.getName().toUpperCase(); + final boolean hasName = + normalizedMethodName.equals("GET" + normalizedFieldName) + || normalizedMethodName.equals("IS" + normalizedFieldName) + || normalizedMethodName.equals(normalizedFieldName); + if (!hasName) { + continue; + } + + // check return type: + // equal to field type + final Type returnType = method.getGenericReturnType(); + final boolean hasReturnType = returnType.equals(field.getGenericType()); + if (!hasReturnType) { + continue; + } + + // check parameters: + // no parameters + final boolean hasNoParameters = method.getParameterCount() == 0; + if (!hasNoParameters) { + continue; + } + + // matching getter found + return Optional.of(method); + } + + // no getter found + return Optional.empty(); + } + + /** + * Checks for a field setters of a structured type. The logic is as broad as possible to support + * both Java and Scala in different flavors. + */ + public static Optional getStructuredFieldSetter(Class clazz, Field field) { + final String normalizedFieldName = field.getName().toUpperCase(); + + final List methods = collectStructuredMethods(clazz); + for (Method method : methods) { + + // check name: + // set(type) + // (type) + // _$eq(type) for Scala + final String normalizedMethodName = method.getName().toUpperCase(); + final boolean hasName = + normalizedMethodName.equals("SET" + normalizedFieldName) + || normalizedMethodName.equals(normalizedFieldName) + || normalizedMethodName.equals(normalizedFieldName + "_$EQ"); + if (!hasName) { + continue; + } + + // check return type: + // void or the declaring class + final Class returnType = method.getReturnType(); + final boolean hasReturnType = returnType == Void.TYPE || returnType == clazz; + if (!hasReturnType) { + continue; + } + + // check parameters: + // one parameter that has the same (or primitive) type of the field + final boolean hasParameter = + method.getParameterCount() == 1 + && (method.getGenericParameterTypes()[0].equals(field.getGenericType()) + || primitiveToWrapper(method.getGenericParameterTypes()[0]) + .equals(field.getGenericType())); + if (!hasParameter) { + continue; + } + + // matching setter found + return Optional.of(method); + } + + // no setter found + return Optional.empty(); + } + + /** + * Checks for an invokable constructor matching the given arguments. + * + * @see #isInvokable(Executable, Class[]) + */ + public static boolean hasInvokableConstructor(Class clazz, Class... classes) { + for (Constructor constructor : clazz.getDeclaredConstructors()) { + if (isInvokable(constructor, classes)) { + return true; + } + } + return false; + } + + /** Checks whether a field is directly readable without a getter. */ + public static boolean isStructuredFieldDirectlyReadable(Field field) { + final int m = field.getModifiers(); + + // field is directly readable + return Modifier.isPublic(m); + } + + /** Checks whether a field is directly writable without a setter or constructor. */ + public static boolean isStructuredFieldDirectlyWritable(Field field) { + final int m = field.getModifiers(); + + // field is immutable + if (Modifier.isFinal(m)) { + return false; + } + + // field is directly writable + return Modifier.isPublic(m); + } + + // -------------------------------------------------------------------------------------------- + // Methods intended for this package + // -------------------------------------------------------------------------------------------- + + /** Helper method for creating consistent exceptions during extraction. */ + static ValidationException extractionError(String message, Object... args) { + return extractionError(null, message, args); + } + + /** Helper method for creating consistent exceptions during extraction. */ + static ValidationException extractionError(Throwable cause, String message, Object... args) { + return new ValidationException(String.format(message, args), cause); + } + + /** + * Collects the partially ordered type hierarchy (i.e. all involved super classes and super + * interfaces) of the given type. + */ + static List collectTypeHierarchy(Type type) { + Type currentType = type; + Class currentClass = toClass(type); + final List typeHierarchy = new ArrayList<>(); + while (currentClass != null) { + // collect type + typeHierarchy.add(currentType); + // collect super interfaces + for (Type genericInterface : currentClass.getGenericInterfaces()) { + final Class interfaceClass = toClass(genericInterface); + if (interfaceClass != null) { + typeHierarchy.addAll(collectTypeHierarchy(genericInterface)); + } + } + currentType = currentClass.getGenericSuperclass(); + currentClass = toClass(currentType); + } + return typeHierarchy; + } + + /** Converts a {@link Type} to {@link Class} if possible, {@code null} otherwise. */ + static @Nullable Class toClass(Type type) { + if (type instanceof Class) { + return (Class) type; + } else if (type instanceof ParameterizedType) { + // this is always a class + return (Class) ((ParameterizedType) type).getRawType(); + } + // unsupported: generic arrays, type variables, wildcard types + return null; + } + + /** Creates a raw data type. */ + @SuppressWarnings({"unchecked", "rawtypes"}) + static DataType createRawType( + DataTypeFactory typeFactory, + @Nullable Class> rawSerializer, + @Nullable Class conversionClass) { + if (rawSerializer != null) { + return DataTypes.RAW( + (Class) createConversionClass(conversionClass), + instantiateRawSerializer(rawSerializer)); + } + return typeFactory.createRawDataType(createConversionClass(conversionClass)); + } + + static Class createConversionClass(@Nullable Class conversionClass) { + if (conversionClass != null) { + return conversionClass; + } + return Object.class; + } + + private static TypeSerializer instantiateRawSerializer( + Class> rawSerializer) { + try { + return rawSerializer.newInstance(); + } catch (Exception e) { + throw extractionError( + e, + "Cannot instantiate type serializer '%s' for RAW type. " + + "Make sure the class is publicly accessible and has a default constructor.", + rawSerializer.getName()); + } + } + + /** Resolves a {@link TypeVariable} using the given type hierarchy if possible. */ + static Type resolveVariable(List typeHierarchy, TypeVariable variable) { + // iterate through hierarchy from top to bottom until type variable gets a non-variable + // assigned + for (int i = typeHierarchy.size() - 1; i >= 0; i--) { + final Type currentType = typeHierarchy.get(i); + + if (currentType instanceof ParameterizedType) { + final Type resolvedType = + resolveVariableInParameterizedType( + variable, (ParameterizedType) currentType); + if (resolvedType instanceof TypeVariable) { + // follow type variables transitively + variable = (TypeVariable) resolvedType; + } else if (resolvedType != null) { + return resolvedType; + } + } + } + // unresolved variable + return variable; + } + + private static @Nullable Type resolveVariableInParameterizedType( + TypeVariable variable, ParameterizedType currentType) { + final Class currentRaw = (Class) currentType.getRawType(); + final TypeVariable[] currentVariables = currentRaw.getTypeParameters(); + // search for matching type variable + for (int paramPos = 0; paramPos < currentVariables.length; paramPos++) { + if (typeVariableEquals(variable, currentVariables[paramPos])) { + return currentType.getActualTypeArguments()[paramPos]; + } + } + return null; + } + + private static boolean typeVariableEquals( + TypeVariable variable, TypeVariable currentVariable) { + return currentVariable.getGenericDeclaration().equals(variable.getGenericDeclaration()) + && currentVariable.getName().equals(variable.getName()); + } + + /** + * Validates if a given type is not already contained in the type hierarchy of a structured + * type. + * + *

    Otherwise this would lead to infinite data type extraction cycles. + */ + static void validateStructuredSelfReference(Type t, List typeHierarchy) { + final Class clazz = toClass(t); + if (clazz != null + && !clazz.isInterface() + && clazz != Object.class + && typeHierarchy.contains(t)) { + throw extractionError( + "Cyclic reference detected for class '%s'. Attributes of structured types must not " + + "(transitively) reference the structured type itself.", + clazz.getName()); + } + } + + /** Returns the fields of a class for a {@link StructuredType}. */ + static List collectStructuredFields(Class clazz) { + final List fields = new ArrayList<>(); + while (clazz != Object.class) { + final Field[] declaredFields = clazz.getDeclaredFields(); + Stream.of(declaredFields) + .filter( + field -> { + final int m = field.getModifiers(); + return !Modifier.isStatic(m) && !Modifier.isTransient(m); + }) + .forEach(fields::add); + clazz = clazz.getSuperclass(); + } + return fields; + } + + /** Validates if a field is properly readable either directly or through a getter. */ + static void validateStructuredFieldReadability(Class clazz, Field field) { + // field is accessible + if (isStructuredFieldDirectlyReadable(field)) { + return; + } + + // field needs a getter + if (!getStructuredFieldGetter(clazz, field).isPresent()) { + throw extractionError( + "Field '%s' of class '%s' is neither publicly accessible nor does it have " + + "a corresponding getter method.", + field.getName(), clazz.getName()); + } + } + + /** + * Checks if a field is mutable or immutable. Returns {@code true} if the field is properly + * mutable. Returns {@code false} if it is properly immutable. + */ + static boolean isStructuredFieldMutable(Class clazz, Field field) { + final int m = field.getModifiers(); + + // field is immutable + if (Modifier.isFinal(m)) { + return false; + } + // field is directly mutable + if (Modifier.isPublic(m)) { + return true; + } + + // field has setters by which it is mutable + if (getStructuredFieldSetter(clazz, field).isPresent()) { + return true; + } + + throw extractionError( + "Field '%s' of class '%s' is mutable but is neither publicly accessible nor does it have " + + "a corresponding setter method.", + field.getName(), clazz.getName()); + } + + /** Returns the boxed type of a primitive type. */ + static Type primitiveToWrapper(Type type) { + if (type instanceof Class) { + return primitiveToWrapper((Class) type); + } + return type; + } + + /** Collects all methods that qualify as methods of a {@link StructuredType}. */ + static List collectStructuredMethods(Class clazz) { + final List methods = new ArrayList<>(); + while (clazz != Object.class) { + final Method[] declaredMethods = clazz.getDeclaredMethods(); + Stream.of(declaredMethods) + .filter( + field -> { + final int m = field.getModifiers(); + return Modifier.isPublic(m) + && !Modifier.isNative(m) + && !Modifier.isAbstract(m); + }) + .forEach(methods::add); + clazz = clazz.getSuperclass(); + } + return methods; + } + + /** + * Collects all annotations of the given type defined in the current class or superclasses. + * Duplicates are ignored. + */ + static Set collectAnnotationsOfClass( + Class annotation, Class annotatedClass) { + final List> classHierarchy = new ArrayList<>(); + Class currentClass = annotatedClass; + while (currentClass != null) { + classHierarchy.add(currentClass); + currentClass = currentClass.getSuperclass(); + } + // convert to top down + Collections.reverse(classHierarchy); + return classHierarchy.stream() + .flatMap(c -> Stream.of(c.getAnnotationsByType(annotation))) + .collect(Collectors.toCollection(LinkedHashSet::new)); + } + + /** + * Collects all annotations of the given type defined in the given method. Duplicates are + * ignored. + */ + static Set collectAnnotationsOfMethod( + Class annotation, Method annotatedMethod) { + return new LinkedHashSet<>(Arrays.asList(annotatedMethod.getAnnotationsByType(annotation))); + } + + // -------------------------------------------------------------------------------------------- + // Parameter Extraction Utilities + // -------------------------------------------------------------------------------------------- + + /** Result of the extraction in {@link #extractAssigningConstructor(Class, List)}. */ + static class AssigningConstructor { + public final Constructor constructor; + public final List parameterNames; + + private AssigningConstructor(Constructor constructor, List parameterNames) { + this.constructor = constructor; + this.parameterNames = parameterNames; + } + } + + /** + * Checks whether the given constructor takes all of the given fields with matching (possibly + * primitive) type and name. An assigning constructor can define the order of fields. + */ + static @Nullable AssigningConstructor extractAssigningConstructor( + Class clazz, List fields) { + AssigningConstructor foundConstructor = null; + for (Constructor constructor : clazz.getDeclaredConstructors()) { + final boolean qualifyingConstructor = + Modifier.isPublic(constructor.getModifiers()) + && constructor.getParameterTypes().length == fields.size(); + if (!qualifyingConstructor) { + continue; + } + final List parameterNames = + extractConstructorParameterNames(constructor, fields); + if (parameterNames != null) { + if (foundConstructor != null) { + throw extractionError( + "Multiple constructors found that assign all fields for class '%s'.", + clazz.getName()); + } + foundConstructor = new AssigningConstructor(constructor, parameterNames); + } + } + return foundConstructor; + } + + /** Extracts the parameter names of a method if possible. */ + static @Nullable List extractMethodParameterNames(Method method) { + return extractExecutableNames(method); + } + + /** + * Extracts ordered parameter names from a constructor that takes all of the given fields with + * matching (possibly primitive) type and name. + */ + private static @Nullable List extractConstructorParameterNames( + Constructor constructor, List fields) { + final Type[] parameterTypes = constructor.getGenericParameterTypes(); + + List parameterNames = extractExecutableNames(constructor); + if (parameterNames == null) { + return null; + } + + final Map fieldMap = + fields.stream().collect(Collectors.toMap(Field::getName, Field::getGenericType)); + + // check that all fields are represented in the parameters of the constructor + for (int i = 0; i < parameterNames.size(); i++) { + final String parameterName = parameterNames.get(i); + final Type fieldType = fieldMap.get(parameterName); // might be null + final Type parameterType = parameterTypes[i]; + // we are tolerant here because frameworks such as Avro accept a boxed type even though + // the field is primitive + if (!primitiveToWrapper(parameterType).equals(primitiveToWrapper(fieldType))) { + return null; + } + } + + return parameterNames; + } + + private static @Nullable List extractExecutableNames(Executable executable) { + final int offset; + if (!Modifier.isStatic(executable.getModifiers())) { + // remove "this" as first parameter + offset = 1; + } else { + offset = 0; + } + // by default parameter names are "arg0, arg1, arg2, ..." if compiler flag is not set + // so we need to extract them manually if possible + List parameterNames = + Stream.of(executable.getParameters()) + .map(Parameter::getName) + .collect(Collectors.toList()); + if (parameterNames.stream().allMatch(n -> n.startsWith("arg"))) { + final ParameterExtractor extractor; + if (executable instanceof Constructor) { + extractor = new ParameterExtractor((Constructor) executable); + } else { + extractor = new ParameterExtractor((Method) executable); + } + getClassReader(executable.getDeclaringClass()).accept(extractor, 0); + + final List extractedNames = extractor.getParameterNames(); + if (extractedNames.size() == 0) { + return null; + } + // remove "this" and additional local variables + // select less names if class file has not the required information + parameterNames = + extractedNames.subList( + offset, + Math.min( + executable.getParameterCount() + offset, + extractedNames.size())); + } + + if (parameterNames.size() != executable.getParameterCount()) { + return null; + } + + return parameterNames; + } + + private static ClassReader getClassReader(Class cls) { + final String className = cls.getName().replaceFirst("^.*\\.", "") + ".class"; + if(ClassPool.exist(cls.getName())){ + return new ClassReader(ClassPool.get(cls.getName()).getClassByte()); + } + try { + return new ClassReader(cls.getResourceAsStream(className)); + } catch (IOException e) { + throw new IllegalStateException("Could not instantiate ClassReader.", e); + } + } + + /** + * Extracts the parameter names and descriptors from a constructor or method. Assuming the + * existence of a local variable table. + * + *

    For example: + * + * 

    {@code
+     * public WC(java.lang.String arg0, long arg1) { //  //(Ljava/lang/String;J)V
+     *   
+     *   
+     *   
+     *   
+     *   
+     * }
+     * }
    + */ + private static class ParameterExtractor extends ClassVisitor { + + private static final int OPCODE = Opcodes.ASM7; + + private final String methodDescriptor; + + private final List parameterNames = new ArrayList<>(); + + ParameterExtractor(Constructor constructor) { + super(OPCODE); + methodDescriptor = getConstructorDescriptor(constructor); + } + + ParameterExtractor(Method method) { + super(OPCODE); + methodDescriptor = getMethodDescriptor(method); + } + + List getParameterNames() { + return parameterNames; + } + + @Override + public MethodVisitor visitMethod( + int access, String name, String descriptor, String signature, String[] exceptions) { + if (descriptor.equals(methodDescriptor)) { + return new MethodVisitor(OPCODE) { + @Override + public void visitLocalVariable( + String name, + String descriptor, + String signature, + Label start, + Label end, + int index) { + parameterNames.add(name); + } + }; + } + return super.visitMethod(access, name, descriptor, signature, exceptions); + } + } + + // -------------------------------------------------------------------------------------------- + // Class Assignment and Boxing + // + // copied from o.a.commons.lang3.ClassUtils (commons-lang3:3.3.2) + // -------------------------------------------------------------------------------------------- + + /** + * Checks if one {@code Class} can be assigned to a variable of another {@code Class}. + * + *

    Unlike the {@link Class#isAssignableFrom(java.lang.Class)} method, this method takes into + * account widenings of primitive classes and {@code null}s. + * + *

    Primitive widenings allow an int to be assigned to a long, float or double. This method + * returns the correct result for these cases. + * + *

    {@code Null} may be assigned to any reference type. This method will return {@code true} + * if {@code null} is passed in and the toClass is non-primitive. + * + *

    Specifically, this method tests whether the type represented by the specified {@code + * Class} parameter can be converted to the type represented by this {@code Class} object via an + * identity conversion widening primitive or widening reference conversion. See The Java Language Specification, + * sections 5.1.1, 5.1.2 and 5.1.4 for details. + * + * @param cls the Class to check, may be null + * @param toClass the Class to try to assign into, returns false if null + * @param autoboxing whether to use implicit autoboxing/unboxing between primitives and wrappers + * @return {@code true} if assignment possible + */ + public static boolean isAssignable( + Class cls, final Class toClass, final boolean autoboxing) { + if (toClass == null) { + return false; + } + // have to check for null, as isAssignableFrom doesn't + if (cls == null) { + return !toClass.isPrimitive(); + } + // autoboxing: + if (autoboxing) { + if (cls.isPrimitive() && !toClass.isPrimitive()) { + cls = primitiveToWrapper(cls); + if (cls == null) { + return false; + } + } + if (toClass.isPrimitive() && !cls.isPrimitive()) { + cls = wrapperToPrimitive(cls); + if (cls == null) { + return false; + } + } + } + if (cls.equals(toClass)) { + return true; + } + if (cls.isPrimitive()) { + if (!toClass.isPrimitive()) { + return false; + } + if (Integer.TYPE.equals(cls)) { + return Long.TYPE.equals(toClass) + || Float.TYPE.equals(toClass) + || Double.TYPE.equals(toClass); + } + if (Long.TYPE.equals(cls)) { + return Float.TYPE.equals(toClass) || Double.TYPE.equals(toClass); + } + if (Boolean.TYPE.equals(cls)) { + return false; + } + if (Double.TYPE.equals(cls)) { + return false; + } + if (Float.TYPE.equals(cls)) { + return Double.TYPE.equals(toClass); + } + if (Character.TYPE.equals(cls)) { + return Integer.TYPE.equals(toClass) + || Long.TYPE.equals(toClass) + || Float.TYPE.equals(toClass) + || Double.TYPE.equals(toClass); + } + if (Short.TYPE.equals(cls)) { + return Integer.TYPE.equals(toClass) + || Long.TYPE.equals(toClass) + || Float.TYPE.equals(toClass) + || Double.TYPE.equals(toClass); + } + if (Byte.TYPE.equals(cls)) { + return Short.TYPE.equals(toClass) + || Integer.TYPE.equals(toClass) + || Long.TYPE.equals(toClass) + || Float.TYPE.equals(toClass) + || Double.TYPE.equals(toClass); + } + // should never get here + return false; + } + return toClass.isAssignableFrom(cls); + } + + /** Maps primitive {@code Class}es to their corresponding wrapper {@code Class}. */ + private static final Map, Class> primitiveWrapperMap = new HashMap<>(); + + static { + primitiveWrapperMap.put(Boolean.TYPE, Boolean.class); + primitiveWrapperMap.put(Byte.TYPE, Byte.class); + primitiveWrapperMap.put(Character.TYPE, Character.class); + primitiveWrapperMap.put(Short.TYPE, Short.class); + primitiveWrapperMap.put(Integer.TYPE, Integer.class); + primitiveWrapperMap.put(Long.TYPE, Long.class); + primitiveWrapperMap.put(Double.TYPE, Double.class); + primitiveWrapperMap.put(Float.TYPE, Float.class); + primitiveWrapperMap.put(Void.TYPE, Void.TYPE); + } + + /** Maps wrapper {@code Class}es to their corresponding primitive types. */ + private static final Map, Class> wrapperPrimitiveMap = new HashMap<>(); + + static { + for (final Class primitiveClass : primitiveWrapperMap.keySet()) { + final Class wrapperClass = primitiveWrapperMap.get(primitiveClass); + if (!primitiveClass.equals(wrapperClass)) { + wrapperPrimitiveMap.put(wrapperClass, primitiveClass); + } + } + } + + /** + * Converts the specified primitive Class object to its corresponding wrapper Class object. + * + *

    NOTE: From v2.2, this method handles {@code Void.TYPE}, returning {@code Void.TYPE}. + * + * @param cls the class to convert, may be null + * @return the wrapper class for {@code cls} or {@code cls} if {@code cls} is not a primitive. + * {@code null} if null input. + * @since 2.1 + */ + public static Class primitiveToWrapper(final Class cls) { + Class convertedClass = cls; + if (cls != null && cls.isPrimitive()) { + convertedClass = primitiveWrapperMap.get(cls); + } + return convertedClass; + } + + /** + * Converts the specified wrapper class to its corresponding primitive class. + * + *

    This method is the counter part of {@code primitiveToWrapper()}. If the passed in class is + * a wrapper class for a primitive type, this primitive type will be returned (e.g. {@code + * Integer.TYPE} for {@code Integer.class}). For other classes, or if the parameter is + * null, the return value is null. + * + * @param cls the class to convert, may be null + * @return the corresponding primitive type if {@code cls} is a wrapper class, null + * otherwise + * @see #primitiveToWrapper(Class) + * @since 2.4 + */ + public static Class wrapperToPrimitive(final Class cls) { + return wrapperPrimitiveMap.get(cls); + } + + // -------------------------------------------------------------------------------------------- + + private ExtractionUtils() { + // no instantiation + } +} diff --git a/dlink-client/dlink-client-1.13/src/main/java/org/apache/flink/table/types/extraction/ExtractionUtils.java b/dlink-client/dlink-client-1.13/src/main/java/org/apache/flink/table/types/extraction/ExtractionUtils.java new file mode 100644 index 0000000000..13716e7230 --- /dev/null +++ b/dlink-client/dlink-client-1.13/src/main/java/org/apache/flink/table/types/extraction/ExtractionUtils.java @@ -0,0 +1,985 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.types.extraction; + +import com.dlink.pool.ClassPool; +import org.apache.flink.annotation.Internal; +import org.apache.flink.api.common.typeutils.TypeSerializer; +import org.apache.flink.table.api.DataTypes; +import org.apache.flink.table.api.ValidationException; +import org.apache.flink.table.catalog.DataTypeFactory; +import org.apache.flink.table.types.DataType; +import org.apache.flink.table.types.logical.StructuredType; + +import org.apache.flink.shaded.asm7.org.objectweb.asm.ClassReader; +import org.apache.flink.shaded.asm7.org.objectweb.asm.ClassVisitor; +import org.apache.flink.shaded.asm7.org.objectweb.asm.Label; +import org.apache.flink.shaded.asm7.org.objectweb.asm.MethodVisitor; +import org.apache.flink.shaded.asm7.org.objectweb.asm.Opcodes; + +import javax.annotation.Nullable; + +import java.io.IOException; +import java.lang.annotation.Annotation; +import java.lang.reflect.Constructor; +import java.lang.reflect.Executable; +import java.lang.reflect.Field; +import java.lang.reflect.Method; +import java.lang.reflect.Modifier; +import java.lang.reflect.Parameter; +import java.lang.reflect.ParameterizedType; +import java.lang.reflect.Type; +import java.lang.reflect.TypeVariable; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.function.Function; +import java.util.regex.Pattern; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.apache.flink.shaded.asm7.org.objectweb.asm.Type.getConstructorDescriptor; +import static org.apache.flink.shaded.asm7.org.objectweb.asm.Type.getMethodDescriptor; + +/** Utilities for performing reflection tasks. */ +@Internal +public final class ExtractionUtils { + + // -------------------------------------------------------------------------------------------- + // Methods shared across packages + // -------------------------------------------------------------------------------------------- + + /** Collects methods of the given name. */ + public static List collectMethods(Class function, String methodName) { + return Arrays.stream(function.getMethods()) + .filter(method -> method.getName().equals(methodName)) + .sorted(Comparator.comparing(Method::toString)) // for deterministic order + .collect(Collectors.toList()); + } + + /** + * Checks whether a method/constructor can be called with the given argument classes. This + * includes type widening and vararg. {@code null} is a wildcard. + * + *

    E.g., {@code (int.class, int.class)} matches {@code f(Object...), f(int, int), f(Integer, + * Object)} and so forth. + */ + public static boolean isInvokable(Executable executable, Class... classes) { + final int m = executable.getModifiers(); + if (!Modifier.isPublic(m)) { + return false; + } + final int paramCount = executable.getParameterCount(); + final int classCount = classes.length; + // check for enough classes for each parameter + if ((!executable.isVarArgs() && classCount != paramCount) + || (executable.isVarArgs() && classCount < paramCount - 1)) { + return false; + } + int currentClass = 0; + for (int currentParam = 0; currentParam < paramCount; currentParam++) { + final Class param = executable.getParameterTypes()[currentParam]; + // last parameter is a vararg that needs to consume remaining classes + if (currentParam == paramCount - 1 && executable.isVarArgs()) { + final Class paramComponent = + executable.getParameterTypes()[currentParam].getComponentType(); + // we have more than 1 classes left so the vararg needs to consume them all + if (classCount - currentClass > 1) { + while (currentClass < classCount + && ExtractionUtils.isAssignable( + classes[currentClass], paramComponent, true)) { + currentClass++; + } + } else if (currentClass < classCount + && (parameterMatches(classes[currentClass], param) + || parameterMatches(classes[currentClass], paramComponent))) { + currentClass++; + } + } + // entire parameter matches + else if (parameterMatches(classes[currentClass], param)) { + currentClass++; + } + } + // check if all classes have been consumed + return currentClass == classCount; + } + + private static boolean parameterMatches(Class clz, Class param) { + return clz == null || ExtractionUtils.isAssignable(clz, param, true); + } + + /** Creates a method signature string like {@code int eval(Integer, String)}. */ + public static String createMethodSignatureString( + String methodName, Class[] parameters, @Nullable Class returnType) { + final StringBuilder builder = new StringBuilder(); + if (returnType != null) { + builder.append(returnType.getCanonicalName()).append(" "); + } + builder.append(methodName) + .append( + Stream.of(parameters) + .map( + parameter -> { + // in case we don't know the parameter at this location + // (i.e. for accumulators) + if (parameter == null) { + return "_"; + } else { + return parameter.getCanonicalName(); + } + }) + .collect(Collectors.joining(", ", "(", ")"))); + return builder.toString(); + } + + /** + * Validates the characteristics of a class for a {@link StructuredType} such as accessibility. + */ + public static void validateStructuredClass(Class clazz) { + final int m = clazz.getModifiers(); + if (Modifier.isAbstract(m)) { + throw extractionError("Class '%s' must not be abstract.", clazz.getName()); + } + if (!Modifier.isPublic(m)) { + throw extractionError("Class '%s' is not public.", clazz.getName()); + } + if (clazz.getEnclosingClass() != null + && (clazz.getDeclaringClass() == null || !Modifier.isStatic(m))) { + throw extractionError( + "Class '%s' is a not a static, globally accessible class.", clazz.getName()); + } + } + + /** + * Returns the field of a structured type. The logic is as broad as possible to support both + * Java and Scala in different flavors. + */ + public static Field getStructuredField(Class clazz, String fieldName) { + final String normalizedFieldName = fieldName.toUpperCase(); + + final List fields = collectStructuredFields(clazz); + for (Field field : fields) { + if (field.getName().toUpperCase().equals(normalizedFieldName)) { + return field; + } + } + throw extractionError( + "Could not find a field named '%s' in class '%s' for structured type.", + fieldName, clazz.getName()); + } + + /** + * Checks for a field getter of a structured type. The logic is as broad as possible to support + * both Java and Scala in different flavors. + */ + public static Optional getStructuredFieldGetter(Class clazz, Field field) { + final String normalizedFieldName = normalizeAccessorName(field.getName()); + + final List methods = collectStructuredMethods(clazz); + for (Method method : methods) { + // check name: + // get() + // is() + // () for Scala + final String normalizedMethodName = normalizeAccessorName(method.getName()); + final boolean hasName = + normalizedMethodName.equals("GET" + normalizedFieldName) + || normalizedMethodName.equals("IS" + normalizedFieldName) + || normalizedMethodName.equals(normalizedFieldName); + if (!hasName) { + continue; + } + + // check return type: + // equal to field type + final Type returnType = method.getGenericReturnType(); + final boolean hasReturnType = returnType.equals(field.getGenericType()); + if (!hasReturnType) { + continue; + } + + // check parameters: + // no parameters + final boolean hasNoParameters = method.getParameterCount() == 0; + if (!hasNoParameters) { + continue; + } + + // matching getter found + return Optional.of(method); + } + + // no getter found + return Optional.empty(); + } + + /** + * Checks for a field setters of a structured type. The logic is as broad as possible to support + * both Java and Scala in different flavors. + */ + public static Optional getStructuredFieldSetter(Class clazz, Field field) { + final String normalizedFieldName = normalizeAccessorName(field.getName()); + + final List methods = collectStructuredMethods(clazz); + for (Method method : methods) { + + // check name: + // set(type) + // (type) + // _$eq(type) for Scala + final String normalizedMethodName = normalizeAccessorName(method.getName()); + final boolean hasName = + normalizedMethodName.equals("SET" + normalizedFieldName) + || normalizedMethodName.equals(normalizedFieldName) + || normalizedMethodName.equals(normalizedFieldName + "$EQ"); + if (!hasName) { + continue; + } + + // check return type: + // void or the declaring class + final Class returnType = method.getReturnType(); + final boolean hasReturnType = returnType == Void.TYPE || returnType == clazz; + if (!hasReturnType) { + continue; + } + + // check parameters: + // one parameter that has the same (or primitive) type of the field + final boolean hasParameter = + method.getParameterCount() == 1 + && (method.getGenericParameterTypes()[0].equals(field.getGenericType()) + || primitiveToWrapper(method.getGenericParameterTypes()[0]) + .equals(field.getGenericType())); + if (!hasParameter) { + continue; + } + + // matching setter found + return Optional.of(method); + } + + // no setter found + return Optional.empty(); + } + + private static String normalizeAccessorName(String name) { + return name.toUpperCase().replaceAll(Pattern.quote("_"), ""); + } + + /** + * Checks for an invokable constructor matching the given arguments. + * + * @see #isInvokable(Executable, Class[]) + */ + public static boolean hasInvokableConstructor(Class clazz, Class... classes) { + for (Constructor constructor : clazz.getDeclaredConstructors()) { + if (isInvokable(constructor, classes)) { + return true; + } + } + return false; + } + + /** Checks whether a field is directly readable without a getter. */ + public static boolean isStructuredFieldDirectlyReadable(Field field) { + final int m = field.getModifiers(); + + // field is directly readable + return Modifier.isPublic(m); + } + + /** Checks whether a field is directly writable without a setter or constructor. */ + public static boolean isStructuredFieldDirectlyWritable(Field field) { + final int m = field.getModifiers(); + + // field is immutable + if (Modifier.isFinal(m)) { + return false; + } + + // field is directly writable + return Modifier.isPublic(m); + } + + /** + * A minimal version to extract a generic parameter from a given class. + * + *

    This method should only be used for very specific use cases, in most cases {@link + * DataTypeExtractor#extractFromGeneric(DataTypeFactory, Class, int, Type)} should be more + * appropriate. + */ + public static Optional> extractSimpleGeneric( + Class baseClass, Class clazz, int pos) { + try { + if (clazz.getSuperclass() != baseClass) { + return Optional.empty(); + } + final Type t = + ((ParameterizedType) clazz.getGenericSuperclass()) + .getActualTypeArguments()[pos]; + return Optional.ofNullable(toClass(t)); + } catch (Exception unused) { + return Optional.empty(); + } + } + + // -------------------------------------------------------------------------------------------- + // Methods intended for this package + // -------------------------------------------------------------------------------------------- + + /** Helper method for creating consistent exceptions during extraction. */ + static ValidationException extractionError(String message, Object... args) { + return extractionError(null, message, args); + } + + /** Helper method for creating consistent exceptions during extraction. */ + static ValidationException extractionError(Throwable cause, String message, Object... args) { + return new ValidationException(String.format(message, args), cause); + } + + /** + * Collects the partially ordered type hierarchy (i.e. all involved super classes and super + * interfaces) of the given type. + */ + static List collectTypeHierarchy(Type type) { + Type currentType = type; + Class currentClass = toClass(type); + final List typeHierarchy = new ArrayList<>(); + while (currentClass != null) { + // collect type + typeHierarchy.add(currentType); + // collect super interfaces + for (Type genericInterface : currentClass.getGenericInterfaces()) { + final Class interfaceClass = toClass(genericInterface); + if (interfaceClass != null) { + typeHierarchy.addAll(collectTypeHierarchy(genericInterface)); + } + } + currentType = currentClass.getGenericSuperclass(); + currentClass = toClass(currentType); + } + return typeHierarchy; + } + + /** Converts a {@link Type} to {@link Class} if possible, {@code null} otherwise. */ + static @Nullable Class toClass(Type type) { + if (type instanceof Class) { + return (Class) type; + } else if (type instanceof ParameterizedType) { + // this is always a class + return (Class) ((ParameterizedType) type).getRawType(); + } + // unsupported: generic arrays, type variables, wildcard types + return null; + } + + /** Creates a raw data type. */ + @SuppressWarnings({"unchecked", "rawtypes"}) + static DataType createRawType( + DataTypeFactory typeFactory, + @Nullable Class> rawSerializer, + @Nullable Class conversionClass) { + if (rawSerializer != null) { + return DataTypes.RAW( + (Class) createConversionClass(conversionClass), + instantiateRawSerializer(rawSerializer)); + } + return typeFactory.createRawDataType(createConversionClass(conversionClass)); + } + + static Class createConversionClass(@Nullable Class conversionClass) { + if (conversionClass != null) { + return conversionClass; + } + return Object.class; + } + + private static TypeSerializer instantiateRawSerializer( + Class> rawSerializer) { + try { + return rawSerializer.newInstance(); + } catch (Exception e) { + throw extractionError( + e, + "Cannot instantiate type serializer '%s' for RAW type. " + + "Make sure the class is publicly accessible and has a default constructor.", + rawSerializer.getName()); + } + } + + /** Resolves a {@link TypeVariable} using the given type hierarchy if possible. */ + static Type resolveVariable(List typeHierarchy, TypeVariable variable) { + // iterate through hierarchy from top to bottom until type variable gets a non-variable + // assigned + for (int i = typeHierarchy.size() - 1; i >= 0; i--) { + final Type currentType = typeHierarchy.get(i); + + if (currentType instanceof ParameterizedType) { + final Type resolvedType = + resolveVariableInParameterizedType( + variable, (ParameterizedType) currentType); + if (resolvedType instanceof TypeVariable) { + // follow type variables transitively + variable = (TypeVariable) resolvedType; + } else if (resolvedType != null) { + return resolvedType; + } + } + } + // unresolved variable + return variable; + } + + private static @Nullable Type resolveVariableInParameterizedType( + TypeVariable variable, ParameterizedType currentType) { + final Class currentRaw = (Class) currentType.getRawType(); + final TypeVariable[] currentVariables = currentRaw.getTypeParameters(); + // search for matching type variable + for (int paramPos = 0; paramPos < currentVariables.length; paramPos++) { + if (typeVariableEquals(variable, currentVariables[paramPos])) { + return currentType.getActualTypeArguments()[paramPos]; + } + } + return null; + } + + private static boolean typeVariableEquals( + TypeVariable variable, TypeVariable currentVariable) { + return currentVariable.getGenericDeclaration().equals(variable.getGenericDeclaration()) + && currentVariable.getName().equals(variable.getName()); + } + + /** + * Validates if a given type is not already contained in the type hierarchy of a structured + * type. + * + *

    Otherwise this would lead to infinite data type extraction cycles. + */ + static void validateStructuredSelfReference(Type t, List typeHierarchy) { + final Class clazz = toClass(t); + if (clazz != null + && !clazz.isInterface() + && clazz != Object.class + && typeHierarchy.contains(t)) { + throw extractionError( + "Cyclic reference detected for class '%s'. Attributes of structured types must not " + + "(transitively) reference the structured type itself.", + clazz.getName()); + } + } + + /** Returns the fields of a class for a {@link StructuredType}. */ + static List collectStructuredFields(Class clazz) { + final List fields = new ArrayList<>(); + while (clazz != Object.class) { + final Field[] declaredFields = clazz.getDeclaredFields(); + Stream.of(declaredFields) + .filter( + field -> { + final int m = field.getModifiers(); + return !Modifier.isStatic(m) && !Modifier.isTransient(m); + }) + .forEach(fields::add); + clazz = clazz.getSuperclass(); + } + return fields; + } + + /** Validates if a field is properly readable either directly or through a getter. */ + static void validateStructuredFieldReadability(Class clazz, Field field) { + // field is accessible + if (isStructuredFieldDirectlyReadable(field)) { + return; + } + + // field needs a getter + if (!getStructuredFieldGetter(clazz, field).isPresent()) { + throw extractionError( + "Field '%s' of class '%s' is neither publicly accessible nor does it have " + + "a corresponding getter method.", + field.getName(), clazz.getName()); + } + } + + /** + * Checks if a field is mutable or immutable. Returns {@code true} if the field is properly + * mutable. Returns {@code false} if it is properly immutable. + */ + static boolean isStructuredFieldMutable(Class clazz, Field field) { + final int m = field.getModifiers(); + + // field is immutable + if (Modifier.isFinal(m)) { + return false; + } + // field is directly mutable + if (Modifier.isPublic(m)) { + return true; + } + + // field has setters by which it is mutable + if (getStructuredFieldSetter(clazz, field).isPresent()) { + return true; + } + + throw extractionError( + "Field '%s' of class '%s' is mutable but is neither publicly accessible nor does it have " + + "a corresponding setter method.", + field.getName(), clazz.getName()); + } + + /** Returns the boxed type of a primitive type. */ + static Type primitiveToWrapper(Type type) { + if (type instanceof Class) { + return primitiveToWrapper((Class) type); + } + return type; + } + + /** Collects all methods that qualify as methods of a {@link StructuredType}. */ + static List collectStructuredMethods(Class clazz) { + final List methods = new ArrayList<>(); + while (clazz != Object.class) { + final Method[] declaredMethods = clazz.getDeclaredMethods(); + Stream.of(declaredMethods) + .filter( + field -> { + final int m = field.getModifiers(); + return Modifier.isPublic(m) + && !Modifier.isNative(m) + && !Modifier.isAbstract(m); + }) + .forEach(methods::add); + clazz = clazz.getSuperclass(); + } + return methods; + } + + /** + * Collects all annotations of the given type defined in the current class or superclasses. + * Duplicates are ignored. + */ + static Set collectAnnotationsOfClass( + Class annotation, Class annotatedClass) { + final List> classHierarchy = new ArrayList<>(); + Class currentClass = annotatedClass; + while (currentClass != null) { + classHierarchy.add(currentClass); + currentClass = currentClass.getSuperclass(); + } + // convert to top down + Collections.reverse(classHierarchy); + return classHierarchy.stream() + .flatMap(c -> Stream.of(c.getAnnotationsByType(annotation))) + .collect(Collectors.toCollection(LinkedHashSet::new)); + } + + /** + * Collects all annotations of the given type defined in the given method. Duplicates are + * ignored. + */ + static Set collectAnnotationsOfMethod( + Class annotation, Method annotatedMethod) { + return new LinkedHashSet<>(Arrays.asList(annotatedMethod.getAnnotationsByType(annotation))); + } + + // -------------------------------------------------------------------------------------------- + // Parameter Extraction Utilities + // -------------------------------------------------------------------------------------------- + + /** Result of the extraction in {@link #extractAssigningConstructor(Class, List)}. */ + public static class AssigningConstructor { + public final Constructor constructor; + public final List parameterNames; + + private AssigningConstructor(Constructor constructor, List parameterNames) { + this.constructor = constructor; + this.parameterNames = parameterNames; + } + } + + /** + * Checks whether the given constructor takes all of the given fields with matching (possibly + * primitive) type and name. An assigning constructor can define the order of fields. + */ + public static @Nullable AssigningConstructor extractAssigningConstructor( + Class clazz, List fields) { + AssigningConstructor foundConstructor = null; + for (Constructor constructor : clazz.getDeclaredConstructors()) { + final boolean qualifyingConstructor = + Modifier.isPublic(constructor.getModifiers()) + && constructor.getParameterTypes().length == fields.size(); + if (!qualifyingConstructor) { + continue; + } + final List parameterNames = + extractConstructorParameterNames(constructor, fields); + if (parameterNames != null) { + if (foundConstructor != null) { + throw extractionError( + "Multiple constructors found that assign all fields for class '%s'.", + clazz.getName()); + } + foundConstructor = new AssigningConstructor(constructor, parameterNames); + } + } + return foundConstructor; + } + + /** Extracts the parameter names of a method if possible. */ + static @Nullable List extractMethodParameterNames(Method method) { + return extractExecutableNames(method); + } + + /** + * Extracts ordered parameter names from a constructor that takes all of the given fields with + * matching (possibly primitive and lenient) type and name. + */ + private static @Nullable List extractConstructorParameterNames( + Constructor constructor, List fields) { + final Type[] parameterTypes = constructor.getGenericParameterTypes(); + + List parameterNames = extractExecutableNames(constructor); + if (parameterNames == null) { + return null; + } + + final Map fieldMap = + fields.stream() + .collect( + Collectors.toMap( + f -> normalizeAccessorName(f.getName()), + Function.identity())); + + // check that all fields are represented in the parameters of the constructor + final List fieldNames = new ArrayList<>(); + for (int i = 0; i < parameterNames.size(); i++) { + final String parameterName = normalizeAccessorName(parameterNames.get(i)); + final Field field = fieldMap.get(parameterName); + if (field == null) { + return null; + } + final Type fieldType = field.getGenericType(); + final Type parameterType = parameterTypes[i]; + // we are tolerant here because frameworks such as Avro accept a boxed type even though + // the field is primitive + if (!primitiveToWrapper(parameterType).equals(primitiveToWrapper(fieldType))) { + return null; + } + fieldNames.add(field.getName()); + } + + return fieldNames; + } + + private static @Nullable List extractExecutableNames(Executable executable) { + final int offset; + if (!Modifier.isStatic(executable.getModifiers())) { + // remove "this" as first parameter + offset = 1; + } else { + offset = 0; + } + // by default parameter names are "arg0, arg1, arg2, ..." if compiler flag is not set + // so we need to extract them manually if possible + List parameterNames = + Stream.of(executable.getParameters()) + .map(Parameter::getName) + .collect(Collectors.toList()); + if (parameterNames.stream().allMatch(n -> n.startsWith("arg"))) { + final ParameterExtractor extractor; + if (executable instanceof Constructor) { + extractor = new ParameterExtractor((Constructor) executable); + } else { + extractor = new ParameterExtractor((Method) executable); + } + getClassReader(executable.getDeclaringClass()).accept(extractor, 0); + + final List extractedNames = extractor.getParameterNames(); + if (extractedNames.size() == 0) { + return null; + } + // remove "this" and additional local variables + // select less names if class file has not the required information + parameterNames = + extractedNames.subList( + offset, + Math.min( + executable.getParameterCount() + offset, + extractedNames.size())); + } + + if (parameterNames.size() != executable.getParameterCount()) { + return null; + } + + return parameterNames; + } + + private static ClassReader getClassReader(Class cls) { + final String className = cls.getName().replaceFirst("^.*\\.", "") + ".class"; + if(ClassPool.exist(cls.getName())){ + return new ClassReader(ClassPool.get(cls.getName()).getClassByte()); + } + try { + return new ClassReader(cls.getResourceAsStream(className)); + } catch (IOException e) { + throw new IllegalStateException("Could not instantiate ClassReader.", e); + } + } + + /** + * Extracts the parameter names and descriptors from a constructor or method. Assuming the + * existence of a local variable table. + * + *

    For example: + * + * 

    {@code
+     * public WC(java.lang.String arg0, long arg1) { //  //(Ljava/lang/String;J)V
+     *   
+     *   
+     *   
+     *   
+     *   
+     * }
+     * }
    + */ + private static class ParameterExtractor extends ClassVisitor { + + private static final int OPCODE = Opcodes.ASM7; + + private final String methodDescriptor; + + private final List parameterNames = new ArrayList<>(); + + ParameterExtractor(Constructor constructor) { + super(OPCODE); + methodDescriptor = getConstructorDescriptor(constructor); + } + + ParameterExtractor(Method method) { + super(OPCODE); + methodDescriptor = getMethodDescriptor(method); + } + + List getParameterNames() { + return parameterNames; + } + + @Override + public MethodVisitor visitMethod( + int access, String name, String descriptor, String signature, String[] exceptions) { + if (descriptor.equals(methodDescriptor)) { + return new MethodVisitor(OPCODE) { + @Override + public void visitLocalVariable( + String name, + String descriptor, + String signature, + Label start, + Label end, + int index) { + parameterNames.add(name); + } + }; + } + return super.visitMethod(access, name, descriptor, signature, exceptions); + } + } + + // -------------------------------------------------------------------------------------------- + // Class Assignment and Boxing + // + // copied from o.a.commons.lang3.ClassUtils (commons-lang3:3.3.2) + // -------------------------------------------------------------------------------------------- + + /** + * Checks if one {@code Class} can be assigned to a variable of another {@code Class}. + * + *

    Unlike the {@link Class#isAssignableFrom(java.lang.Class)} method, this method takes into + * account widenings of primitive classes and {@code null}s. + * + *

    Primitive widenings allow an int to be assigned to a long, float or double. This method + * returns the correct result for these cases. + * + *

    {@code Null} may be assigned to any reference type. This method will return {@code true} + * if {@code null} is passed in and the toClass is non-primitive. + * + *

    Specifically, this method tests whether the type represented by the specified {@code + * Class} parameter can be converted to the type represented by this {@code Class} object via an + * identity conversion widening primitive or widening reference conversion. See The Java Language Specification, + * sections 5.1.1, 5.1.2 and 5.1.4 for details. + * + * @param cls the Class to check, may be null + * @param toClass the Class to try to assign into, returns false if null + * @param autoboxing whether to use implicit autoboxing/unboxing between primitives and wrappers + * @return {@code true} if assignment possible + */ + public static boolean isAssignable( + Class cls, final Class toClass, final boolean autoboxing) { + if (toClass == null) { + return false; + } + // have to check for null, as isAssignableFrom doesn't + if (cls == null) { + return !toClass.isPrimitive(); + } + // autoboxing: + if (autoboxing) { + if (cls.isPrimitive() && !toClass.isPrimitive()) { + cls = primitiveToWrapper(cls); + if (cls == null) { + return false; + } + } + if (toClass.isPrimitive() && !cls.isPrimitive()) { + cls = wrapperToPrimitive(cls); + if (cls == null) { + return false; + } + } + } + if (cls.equals(toClass)) { + return true; + } + if (cls.isPrimitive()) { + if (!toClass.isPrimitive()) { + return false; + } + if (Integer.TYPE.equals(cls)) { + return Long.TYPE.equals(toClass) + || Float.TYPE.equals(toClass) + || Double.TYPE.equals(toClass); + } + if (Long.TYPE.equals(cls)) { + return Float.TYPE.equals(toClass) || Double.TYPE.equals(toClass); + } + if (Boolean.TYPE.equals(cls)) { + return false; + } + if (Double.TYPE.equals(cls)) { + return false; + } + if (Float.TYPE.equals(cls)) { + return Double.TYPE.equals(toClass); + } + if (Character.TYPE.equals(cls)) { + return Integer.TYPE.equals(toClass) + || Long.TYPE.equals(toClass) + || Float.TYPE.equals(toClass) + || Double.TYPE.equals(toClass); + } + if (Short.TYPE.equals(cls)) { + return Integer.TYPE.equals(toClass) + || Long.TYPE.equals(toClass) + || Float.TYPE.equals(toClass) + || Double.TYPE.equals(toClass); + } + if (Byte.TYPE.equals(cls)) { + return Short.TYPE.equals(toClass) + || Integer.TYPE.equals(toClass) + || Long.TYPE.equals(toClass) + || Float.TYPE.equals(toClass) + || Double.TYPE.equals(toClass); + } + // should never get here + return false; + } + return toClass.isAssignableFrom(cls); + } + + /** Maps primitive {@code Class}es to their corresponding wrapper {@code Class}. */ + private static final Map, Class> primitiveWrapperMap = new HashMap<>(); + + static { + primitiveWrapperMap.put(Boolean.TYPE, Boolean.class); + primitiveWrapperMap.put(Byte.TYPE, Byte.class); + primitiveWrapperMap.put(Character.TYPE, Character.class); + primitiveWrapperMap.put(Short.TYPE, Short.class); + primitiveWrapperMap.put(Integer.TYPE, Integer.class); + primitiveWrapperMap.put(Long.TYPE, Long.class); + primitiveWrapperMap.put(Double.TYPE, Double.class); + primitiveWrapperMap.put(Float.TYPE, Float.class); + primitiveWrapperMap.put(Void.TYPE, Void.TYPE); + } + + /** Maps wrapper {@code Class}es to their corresponding primitive types. */ + private static final Map, Class> wrapperPrimitiveMap = new HashMap<>(); + + static { + for (final Class primitiveClass : primitiveWrapperMap.keySet()) { + final Class wrapperClass = primitiveWrapperMap.get(primitiveClass); + if (!primitiveClass.equals(wrapperClass)) { + wrapperPrimitiveMap.put(wrapperClass, primitiveClass); + } + } + } + + /** + * Converts the specified primitive Class object to its corresponding wrapper Class object. + * + *

    NOTE: From v2.2, this method handles {@code Void.TYPE}, returning {@code Void.TYPE}. + * + * @param cls the class to convert, may be null + * @return the wrapper class for {@code cls} or {@code cls} if {@code cls} is not a primitive. + * {@code null} if null input. + * @since 2.1 + */ + public static Class primitiveToWrapper(final Class cls) { + Class convertedClass = cls; + if (cls != null && cls.isPrimitive()) { + convertedClass = primitiveWrapperMap.get(cls); + } + return convertedClass; + } + + /** + * Converts the specified wrapper class to its corresponding primitive class. + * + *

    This method is the counter part of {@code primitiveToWrapper()}. If the passed in class is + * a wrapper class for a primitive type, this primitive type will be returned (e.g. {@code + * Integer.TYPE} for {@code Integer.class}). For other classes, or if the parameter is + * null, the return value is null. + * + * @param cls the class to convert, may be null + * @return the corresponding primitive type if {@code cls} is a wrapper class, null + * otherwise + * @see #primitiveToWrapper(Class) + * @since 2.4 + */ + public static Class wrapperToPrimitive(final Class cls) { + return wrapperPrimitiveMap.get(cls); + } + + // -------------------------------------------------------------------------------------------- + + private ExtractionUtils() { + // no instantiation + } +} diff --git a/dlink-client/dlink-client-1.14/src/main/java/org/apache/flink/table/types/extraction/ExtractionUtils.java b/dlink-client/dlink-client-1.14/src/main/java/org/apache/flink/table/types/extraction/ExtractionUtils.java new file mode 100644 index 0000000000..d7b1e6e4f4 --- /dev/null +++ b/dlink-client/dlink-client-1.14/src/main/java/org/apache/flink/table/types/extraction/ExtractionUtils.java @@ -0,0 +1,986 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.types.extraction; + +import com.dlink.pool.ClassPool; +import org.apache.flink.annotation.Internal; +import org.apache.flink.api.common.typeutils.TypeSerializer; +import org.apache.flink.table.api.DataTypes; +import org.apache.flink.table.api.ValidationException; +import org.apache.flink.table.catalog.DataTypeFactory; +import org.apache.flink.table.types.DataType; +import org.apache.flink.table.types.logical.StructuredType; + +import org.apache.flink.shaded.asm7.org.objectweb.asm.ClassReader; +import org.apache.flink.shaded.asm7.org.objectweb.asm.ClassVisitor; +import org.apache.flink.shaded.asm7.org.objectweb.asm.Label; +import org.apache.flink.shaded.asm7.org.objectweb.asm.MethodVisitor; +import org.apache.flink.shaded.asm7.org.objectweb.asm.Opcodes; + +import javax.annotation.Nullable; + +import java.io.IOException; +import java.io.InputStream; +import java.lang.annotation.Annotation; +import java.lang.reflect.Constructor; +import java.lang.reflect.Executable; +import java.lang.reflect.Field; +import java.lang.reflect.Method; +import java.lang.reflect.Modifier; +import java.lang.reflect.Parameter; +import java.lang.reflect.ParameterizedType; +import java.lang.reflect.Type; +import java.lang.reflect.TypeVariable; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.function.Function; +import java.util.regex.Pattern; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.apache.flink.shaded.asm7.org.objectweb.asm.Type.getConstructorDescriptor; +import static org.apache.flink.shaded.asm7.org.objectweb.asm.Type.getMethodDescriptor; + +/** Utilities for performing reflection tasks. */ +@Internal +public final class ExtractionUtils { + + // -------------------------------------------------------------------------------------------- + // Methods shared across packages + // -------------------------------------------------------------------------------------------- + + /** Collects methods of the given name. */ + public static List collectMethods(Class function, String methodName) { + return Arrays.stream(function.getMethods()) + .filter(method -> method.getName().equals(methodName)) + .sorted(Comparator.comparing(Method::toString)) // for deterministic order + .collect(Collectors.toList()); + } + + /** + * Checks whether a method/constructor can be called with the given argument classes. This + * includes type widening and vararg. {@code null} is a wildcard. + * + *

    E.g., {@code (int.class, int.class)} matches {@code f(Object...), f(int, int), f(Integer, + * Object)} and so forth. + */ + public static boolean isInvokable(Executable executable, Class... classes) { + final int m = executable.getModifiers(); + if (!Modifier.isPublic(m)) { + return false; + } + final int paramCount = executable.getParameterCount(); + final int classCount = classes.length; + // check for enough classes for each parameter + if ((!executable.isVarArgs() && classCount != paramCount) + || (executable.isVarArgs() && classCount < paramCount - 1)) { + return false; + } + int currentClass = 0; + for (int currentParam = 0; currentParam < paramCount; currentParam++) { + final Class param = executable.getParameterTypes()[currentParam]; + // last parameter is a vararg that needs to consume remaining classes + if (currentParam == paramCount - 1 && executable.isVarArgs()) { + final Class paramComponent = + executable.getParameterTypes()[currentParam].getComponentType(); + // we have more than 1 classes left so the vararg needs to consume them all + if (classCount - currentClass > 1) { + while (currentClass < classCount + && ExtractionUtils.isAssignable( + classes[currentClass], paramComponent, true)) { + currentClass++; + } + } else if (currentClass < classCount + && (parameterMatches(classes[currentClass], param) + || parameterMatches(classes[currentClass], paramComponent))) { + currentClass++; + } + } + // entire parameter matches + else if (parameterMatches(classes[currentClass], param)) { + currentClass++; + } + } + // check if all classes have been consumed + return currentClass == classCount; + } + + private static boolean parameterMatches(Class clz, Class param) { + return clz == null || ExtractionUtils.isAssignable(clz, param, true); + } + + /** Creates a method signature string like {@code int eval(Integer, String)}. */ + public static String createMethodSignatureString( + String methodName, Class[] parameters, @Nullable Class returnType) { + final StringBuilder builder = new StringBuilder(); + if (returnType != null) { + builder.append(returnType.getCanonicalName()).append(" "); + } + builder.append(methodName) + .append( + Stream.of(parameters) + .map( + parameter -> { + // in case we don't know the parameter at this location + // (i.e. for accumulators) + if (parameter == null) { + return "_"; + } else { + return parameter.getCanonicalName(); + } + }) + .collect(Collectors.joining(", ", "(", ")"))); + return builder.toString(); + } + + /** + * Validates the characteristics of a class for a {@link StructuredType} such as accessibility. + */ + public static void validateStructuredClass(Class clazz) { + final int m = clazz.getModifiers(); + if (Modifier.isAbstract(m)) { + throw extractionError("Class '%s' must not be abstract.", clazz.getName()); + } + if (!Modifier.isPublic(m)) { + throw extractionError("Class '%s' is not public.", clazz.getName()); + } + if (clazz.getEnclosingClass() != null + && (clazz.getDeclaringClass() == null || !Modifier.isStatic(m))) { + throw extractionError( + "Class '%s' is a not a static, globally accessible class.", clazz.getName()); + } + } + + /** + * Returns the field of a structured type. The logic is as broad as possible to support both + * Java and Scala in different flavors. + */ + public static Field getStructuredField(Class clazz, String fieldName) { + final String normalizedFieldName = fieldName.toUpperCase(); + + final List fields = collectStructuredFields(clazz); + for (Field field : fields) { + if (field.getName().toUpperCase().equals(normalizedFieldName)) { + return field; + } + } + throw extractionError( + "Could not find a field named '%s' in class '%s' for structured type.", + fieldName, clazz.getName()); + } + + /** + * Checks for a field getter of a structured type. The logic is as broad as possible to support + * both Java and Scala in different flavors. + */ + public static Optional getStructuredFieldGetter(Class clazz, Field field) { + final String normalizedFieldName = normalizeAccessorName(field.getName()); + + final List methods = collectStructuredMethods(clazz); + for (Method method : methods) { + // check name: + // get() + // is() + // () for Scala + final String normalizedMethodName = normalizeAccessorName(method.getName()); + final boolean hasName = + normalizedMethodName.equals("GET" + normalizedFieldName) + || normalizedMethodName.equals("IS" + normalizedFieldName) + || normalizedMethodName.equals(normalizedFieldName); + if (!hasName) { + continue; + } + + // check return type: + // equal to field type + final Type returnType = method.getGenericReturnType(); + final boolean hasReturnType = returnType.equals(field.getGenericType()); + if (!hasReturnType) { + continue; + } + + // check parameters: + // no parameters + final boolean hasNoParameters = method.getParameterCount() == 0; + if (!hasNoParameters) { + continue; + } + + // matching getter found + return Optional.of(method); + } + + // no getter found + return Optional.empty(); + } + + /** + * Checks for a field setters of a structured type. The logic is as broad as possible to support + * both Java and Scala in different flavors. + */ + public static Optional getStructuredFieldSetter(Class clazz, Field field) { + final String normalizedFieldName = normalizeAccessorName(field.getName()); + + final List methods = collectStructuredMethods(clazz); + for (Method method : methods) { + + // check name: + // set(type) + // (type) + // _$eq(type) for Scala + final String normalizedMethodName = normalizeAccessorName(method.getName()); + final boolean hasName = + normalizedMethodName.equals("SET" + normalizedFieldName) + || normalizedMethodName.equals(normalizedFieldName) + || normalizedMethodName.equals(normalizedFieldName + "$EQ"); + if (!hasName) { + continue; + } + + // check return type: + // void or the declaring class + final Class returnType = method.getReturnType(); + final boolean hasReturnType = returnType == Void.TYPE || returnType == clazz; + if (!hasReturnType) { + continue; + } + + // check parameters: + // one parameter that has the same (or primitive) type of the field + final boolean hasParameter = + method.getParameterCount() == 1 + && (method.getGenericParameterTypes()[0].equals(field.getGenericType()) + || primitiveToWrapper(method.getGenericParameterTypes()[0]) + .equals(field.getGenericType())); + if (!hasParameter) { + continue; + } + + // matching setter found + return Optional.of(method); + } + + // no setter found + return Optional.empty(); + } + + private static String normalizeAccessorName(String name) { + return name.toUpperCase().replaceAll(Pattern.quote("_"), ""); + } + + /** + * Checks for an invokable constructor matching the given arguments. + * + * @see #isInvokable(Executable, Class[]) + */ + public static boolean hasInvokableConstructor(Class clazz, Class... classes) { + for (Constructor constructor : clazz.getDeclaredConstructors()) { + if (isInvokable(constructor, classes)) { + return true; + } + } + return false; + } + + /** Checks whether a field is directly readable without a getter. */ + public static boolean isStructuredFieldDirectlyReadable(Field field) { + final int m = field.getModifiers(); + + // field is directly readable + return Modifier.isPublic(m); + } + + /** Checks whether a field is directly writable without a setter or constructor. */ + public static boolean isStructuredFieldDirectlyWritable(Field field) { + final int m = field.getModifiers(); + + // field is immutable + if (Modifier.isFinal(m)) { + return false; + } + + // field is directly writable + return Modifier.isPublic(m); + } + + /** + * A minimal version to extract a generic parameter from a given class. + * + *

    This method should only be used for very specific use cases, in most cases {@link + * DataTypeExtractor#extractFromGeneric(DataTypeFactory, Class, int, Type)} should be more + * appropriate. + */ + public static Optional> extractSimpleGeneric( + Class baseClass, Class clazz, int pos) { + try { + if (clazz.getSuperclass() != baseClass) { + return Optional.empty(); + } + final Type t = + ((ParameterizedType) clazz.getGenericSuperclass()) + .getActualTypeArguments()[pos]; + return Optional.ofNullable(toClass(t)); + } catch (Exception unused) { + return Optional.empty(); + } + } + + // -------------------------------------------------------------------------------------------- + // Methods intended for this package + // -------------------------------------------------------------------------------------------- + + /** Helper method for creating consistent exceptions during extraction. */ + static ValidationException extractionError(String message, Object... args) { + return extractionError(null, message, args); + } + + /** Helper method for creating consistent exceptions during extraction. */ + static ValidationException extractionError(Throwable cause, String message, Object... args) { + return new ValidationException(String.format(message, args), cause); + } + + /** + * Collects the partially ordered type hierarchy (i.e. all involved super classes and super + * interfaces) of the given type. + */ + static List collectTypeHierarchy(Type type) { + Type currentType = type; + Class currentClass = toClass(type); + final List typeHierarchy = new ArrayList<>(); + while (currentClass != null) { + // collect type + typeHierarchy.add(currentType); + // collect super interfaces + for (Type genericInterface : currentClass.getGenericInterfaces()) { + final Class interfaceClass = toClass(genericInterface); + if (interfaceClass != null) { + typeHierarchy.addAll(collectTypeHierarchy(genericInterface)); + } + } + currentType = currentClass.getGenericSuperclass(); + currentClass = toClass(currentType); + } + return typeHierarchy; + } + + /** Converts a {@link Type} to {@link Class} if possible, {@code null} otherwise. */ + static @Nullable Class toClass(Type type) { + if (type instanceof Class) { + return (Class) type; + } else if (type instanceof ParameterizedType) { + // this is always a class + return (Class) ((ParameterizedType) type).getRawType(); + } + // unsupported: generic arrays, type variables, wildcard types + return null; + } + + /** Creates a raw data type. */ + @SuppressWarnings({"unchecked", "rawtypes"}) + static DataType createRawType( + DataTypeFactory typeFactory, + @Nullable Class> rawSerializer, + @Nullable Class conversionClass) { + if (rawSerializer != null) { + return DataTypes.RAW( + (Class) createConversionClass(conversionClass), + instantiateRawSerializer(rawSerializer)); + } + return typeFactory.createRawDataType(createConversionClass(conversionClass)); + } + + static Class createConversionClass(@Nullable Class conversionClass) { + if (conversionClass != null) { + return conversionClass; + } + return Object.class; + } + + private static TypeSerializer instantiateRawSerializer( + Class> rawSerializer) { + try { + return rawSerializer.newInstance(); + } catch (Exception e) { + throw extractionError( + e, + "Cannot instantiate type serializer '%s' for RAW type. " + + "Make sure the class is publicly accessible and has a default constructor.", + rawSerializer.getName()); + } + } + + /** Resolves a {@link TypeVariable} using the given type hierarchy if possible. */ + static Type resolveVariable(List typeHierarchy, TypeVariable variable) { + // iterate through hierarchy from top to bottom until type variable gets a non-variable + // assigned + for (int i = typeHierarchy.size() - 1; i >= 0; i--) { + final Type currentType = typeHierarchy.get(i); + + if (currentType instanceof ParameterizedType) { + final Type resolvedType = + resolveVariableInParameterizedType( + variable, (ParameterizedType) currentType); + if (resolvedType instanceof TypeVariable) { + // follow type variables transitively + variable = (TypeVariable) resolvedType; + } else if (resolvedType != null) { + return resolvedType; + } + } + } + // unresolved variable + return variable; + } + + private static @Nullable Type resolveVariableInParameterizedType( + TypeVariable variable, ParameterizedType currentType) { + final Class currentRaw = (Class) currentType.getRawType(); + final TypeVariable[] currentVariables = currentRaw.getTypeParameters(); + // search for matching type variable + for (int paramPos = 0; paramPos < currentVariables.length; paramPos++) { + if (typeVariableEquals(variable, currentVariables[paramPos])) { + return currentType.getActualTypeArguments()[paramPos]; + } + } + return null; + } + + private static boolean typeVariableEquals( + TypeVariable variable, TypeVariable currentVariable) { + return currentVariable.getGenericDeclaration().equals(variable.getGenericDeclaration()) + && currentVariable.getName().equals(variable.getName()); + } + + /** + * Validates if a given type is not already contained in the type hierarchy of a structured + * type. + * + *

    Otherwise this would lead to infinite data type extraction cycles. + */ + static void validateStructuredSelfReference(Type t, List typeHierarchy) { + final Class clazz = toClass(t); + if (clazz != null + && !clazz.isInterface() + && clazz != Object.class + && typeHierarchy.contains(t)) { + throw extractionError( + "Cyclic reference detected for class '%s'. Attributes of structured types must not " + + "(transitively) reference the structured type itself.", + clazz.getName()); + } + } + + /** Returns the fields of a class for a {@link StructuredType}. */ + static List collectStructuredFields(Class clazz) { + final List fields = new ArrayList<>(); + while (clazz != Object.class) { + final Field[] declaredFields = clazz.getDeclaredFields(); + Stream.of(declaredFields) + .filter( + field -> { + final int m = field.getModifiers(); + return !Modifier.isStatic(m) && !Modifier.isTransient(m); + }) + .forEach(fields::add); + clazz = clazz.getSuperclass(); + } + return fields; + } + + /** Validates if a field is properly readable either directly or through a getter. */ + static void validateStructuredFieldReadability(Class clazz, Field field) { + // field is accessible + if (isStructuredFieldDirectlyReadable(field)) { + return; + } + + // field needs a getter + if (!getStructuredFieldGetter(clazz, field).isPresent()) { + throw extractionError( + "Field '%s' of class '%s' is neither publicly accessible nor does it have " + + "a corresponding getter method.", + field.getName(), clazz.getName()); + } + } + + /** + * Checks if a field is mutable or immutable. Returns {@code true} if the field is properly + * mutable. Returns {@code false} if it is properly immutable. + */ + static boolean isStructuredFieldMutable(Class clazz, Field field) { + final int m = field.getModifiers(); + + // field is immutable + if (Modifier.isFinal(m)) { + return false; + } + // field is directly mutable + if (Modifier.isPublic(m)) { + return true; + } + + // field has setters by which it is mutable + if (getStructuredFieldSetter(clazz, field).isPresent()) { + return true; + } + + throw extractionError( + "Field '%s' of class '%s' is mutable but is neither publicly accessible nor does it have " + + "a corresponding setter method.", + field.getName(), clazz.getName()); + } + + /** Returns the boxed type of a primitive type. */ + static Type primitiveToWrapper(Type type) { + if (type instanceof Class) { + return primitiveToWrapper((Class) type); + } + return type; + } + + /** Collects all methods that qualify as methods of a {@link StructuredType}. */ + static List collectStructuredMethods(Class clazz) { + final List methods = new ArrayList<>(); + while (clazz != Object.class) { + final Method[] declaredMethods = clazz.getDeclaredMethods(); + Stream.of(declaredMethods) + .filter( + field -> { + final int m = field.getModifiers(); + return Modifier.isPublic(m) + && !Modifier.isNative(m) + && !Modifier.isAbstract(m); + }) + .forEach(methods::add); + clazz = clazz.getSuperclass(); + } + return methods; + } + + /** + * Collects all annotations of the given type defined in the current class or superclasses. + * Duplicates are ignored. + */ + static Set collectAnnotationsOfClass( + Class annotation, Class annotatedClass) { + final List> classHierarchy = new ArrayList<>(); + Class currentClass = annotatedClass; + while (currentClass != null) { + classHierarchy.add(currentClass); + currentClass = currentClass.getSuperclass(); + } + // convert to top down + Collections.reverse(classHierarchy); + return classHierarchy.stream() + .flatMap(c -> Stream.of(c.getAnnotationsByType(annotation))) + .collect(Collectors.toCollection(LinkedHashSet::new)); + } + + /** + * Collects all annotations of the given type defined in the given method. Duplicates are + * ignored. + */ + static Set collectAnnotationsOfMethod( + Class annotation, Method annotatedMethod) { + return new LinkedHashSet<>(Arrays.asList(annotatedMethod.getAnnotationsByType(annotation))); + } + + // -------------------------------------------------------------------------------------------- + // Parameter Extraction Utilities + // -------------------------------------------------------------------------------------------- + + /** Result of the extraction in {@link #extractAssigningConstructor(Class, List)}. */ + public static class AssigningConstructor { + public final Constructor constructor; + public final List parameterNames; + + private AssigningConstructor(Constructor constructor, List parameterNames) { + this.constructor = constructor; + this.parameterNames = parameterNames; + } + } + + /** + * Checks whether the given constructor takes all of the given fields with matching (possibly + * primitive) type and name. An assigning constructor can define the order of fields. + */ + public static @Nullable AssigningConstructor extractAssigningConstructor( + Class clazz, List fields) { + AssigningConstructor foundConstructor = null; + for (Constructor constructor : clazz.getDeclaredConstructors()) { + final boolean qualifyingConstructor = + Modifier.isPublic(constructor.getModifiers()) + && constructor.getParameterTypes().length == fields.size(); + if (!qualifyingConstructor) { + continue; + } + final List parameterNames = + extractConstructorParameterNames(constructor, fields); + if (parameterNames != null) { + if (foundConstructor != null) { + throw extractionError( + "Multiple constructors found that assign all fields for class '%s'.", + clazz.getName()); + } + foundConstructor = new AssigningConstructor(constructor, parameterNames); + } + } + return foundConstructor; + } + + /** Extracts the parameter names of a method if possible. */ + static @Nullable List extractMethodParameterNames(Method method) { + return extractExecutableNames(method); + } + + /** + * Extracts ordered parameter names from a constructor that takes all of the given fields with + * matching (possibly primitive and lenient) type and name. + */ + private static @Nullable List extractConstructorParameterNames( + Constructor constructor, List fields) { + final Type[] parameterTypes = constructor.getGenericParameterTypes(); + + List parameterNames = extractExecutableNames(constructor); + if (parameterNames == null) { + return null; + } + + final Map fieldMap = + fields.stream() + .collect( + Collectors.toMap( + f -> normalizeAccessorName(f.getName()), + Function.identity())); + + // check that all fields are represented in the parameters of the constructor + final List fieldNames = new ArrayList<>(); + for (int i = 0; i < parameterNames.size(); i++) { + final String parameterName = normalizeAccessorName(parameterNames.get(i)); + final Field field = fieldMap.get(parameterName); + if (field == null) { + return null; + } + final Type fieldType = field.getGenericType(); + final Type parameterType = parameterTypes[i]; + // we are tolerant here because frameworks such as Avro accept a boxed type even though + // the field is primitive + if (!primitiveToWrapper(parameterType).equals(primitiveToWrapper(fieldType))) { + return null; + } + fieldNames.add(field.getName()); + } + + return fieldNames; + } + + private static @Nullable List extractExecutableNames(Executable executable) { + final int offset; + if (!Modifier.isStatic(executable.getModifiers())) { + // remove "this" as first parameter + offset = 1; + } else { + offset = 0; + } + // by default parameter names are "arg0, arg1, arg2, ..." if compiler flag is not set + // so we need to extract them manually if possible + List parameterNames = + Stream.of(executable.getParameters()) + .map(Parameter::getName) + .collect(Collectors.toList()); + if (parameterNames.stream().allMatch(n -> n.startsWith("arg"))) { + final ParameterExtractor extractor; + if (executable instanceof Constructor) { + extractor = new ParameterExtractor((Constructor) executable); + } else { + extractor = new ParameterExtractor((Method) executable); + } + getClassReader(executable.getDeclaringClass()).accept(extractor, 0); + + final List extractedNames = extractor.getParameterNames(); + if (extractedNames.size() == 0) { + return null; + } + // remove "this" and additional local variables + // select less names if class file has not the required information + parameterNames = + extractedNames.subList( + offset, + Math.min( + executable.getParameterCount() + offset, + extractedNames.size())); + } + + if (parameterNames.size() != executable.getParameterCount()) { + return null; + } + + return parameterNames; + } + + private static ClassReader getClassReader(Class cls) { + final String className = cls.getName().replaceFirst("^.*\\.", "") + ".class"; + if(ClassPool.exist(cls.getName())){ + return new ClassReader(ClassPool.get(cls.getName()).getClassByte()); + } + try (InputStream i = cls.getResourceAsStream(className)) { + return new ClassReader(i); + } catch (IOException e) { + throw new IllegalStateException("Could not instantiate ClassReader.", e); + } + } + + /** + * Extracts the parameter names and descriptors from a constructor or method. Assuming the + * existence of a local variable table. + * + *

    For example: + * + * 

    {@code
+     * public WC(java.lang.String arg0, long arg1) { //  //(Ljava/lang/String;J)V
+     *   
+     *   
+     *   
+     *   
+     *   
+     * }
+     * }
    + */ + private static class ParameterExtractor extends ClassVisitor { + + private static final int OPCODE = Opcodes.ASM7; + + private final String methodDescriptor; + + private final List parameterNames = new ArrayList<>(); + + ParameterExtractor(Constructor constructor) { + super(OPCODE); + methodDescriptor = getConstructorDescriptor(constructor); + } + + ParameterExtractor(Method method) { + super(OPCODE); + methodDescriptor = getMethodDescriptor(method); + } + + List getParameterNames() { + return parameterNames; + } + + @Override + public MethodVisitor visitMethod( + int access, String name, String descriptor, String signature, String[] exceptions) { + if (descriptor.equals(methodDescriptor)) { + return new MethodVisitor(OPCODE) { + @Override + public void visitLocalVariable( + String name, + String descriptor, + String signature, + Label start, + Label end, + int index) { + parameterNames.add(name); + } + }; + } + return super.visitMethod(access, name, descriptor, signature, exceptions); + } + } + + // -------------------------------------------------------------------------------------------- + // Class Assignment and Boxing + // + // copied from o.a.commons.lang3.ClassUtils (commons-lang3:3.3.2) + // -------------------------------------------------------------------------------------------- + + /** + * Checks if one {@code Class} can be assigned to a variable of another {@code Class}. + * + *

    Unlike the {@link Class#isAssignableFrom(java.lang.Class)} method, this method takes into + * account widenings of primitive classes and {@code null}s. + * + *

    Primitive widenings allow an int to be assigned to a long, float or double. This method + * returns the correct result for these cases. + * + *

    {@code Null} may be assigned to any reference type. This method will return {@code true} + * if {@code null} is passed in and the toClass is non-primitive. + * + *

    Specifically, this method tests whether the type represented by the specified {@code + * Class} parameter can be converted to the type represented by this {@code Class} object via an + * identity conversion widening primitive or widening reference conversion. See The Java Language Specification, + * sections 5.1.1, 5.1.2 and 5.1.4 for details. + * + * @param cls the Class to check, may be null + * @param toClass the Class to try to assign into, returns false if null + * @param autoboxing whether to use implicit autoboxing/unboxing between primitives and wrappers + * @return {@code true} if assignment possible + */ + public static boolean isAssignable( + Class cls, final Class toClass, final boolean autoboxing) { + if (toClass == null) { + return false; + } + // have to check for null, as isAssignableFrom doesn't + if (cls == null) { + return !toClass.isPrimitive(); + } + // autoboxing: + if (autoboxing) { + if (cls.isPrimitive() && !toClass.isPrimitive()) { + cls = primitiveToWrapper(cls); + if (cls == null) { + return false; + } + } + if (toClass.isPrimitive() && !cls.isPrimitive()) { + cls = wrapperToPrimitive(cls); + if (cls == null) { + return false; + } + } + } + if (cls.equals(toClass)) { + return true; + } + if (cls.isPrimitive()) { + if (!toClass.isPrimitive()) { + return false; + } + if (Integer.TYPE.equals(cls)) { + return Long.TYPE.equals(toClass) + || Float.TYPE.equals(toClass) + || Double.TYPE.equals(toClass); + } + if (Long.TYPE.equals(cls)) { + return Float.TYPE.equals(toClass) || Double.TYPE.equals(toClass); + } + if (Boolean.TYPE.equals(cls)) { + return false; + } + if (Double.TYPE.equals(cls)) { + return false; + } + if (Float.TYPE.equals(cls)) { + return Double.TYPE.equals(toClass); + } + if (Character.TYPE.equals(cls)) { + return Integer.TYPE.equals(toClass) + || Long.TYPE.equals(toClass) + || Float.TYPE.equals(toClass) + || Double.TYPE.equals(toClass); + } + if (Short.TYPE.equals(cls)) { + return Integer.TYPE.equals(toClass) + || Long.TYPE.equals(toClass) + || Float.TYPE.equals(toClass) + || Double.TYPE.equals(toClass); + } + if (Byte.TYPE.equals(cls)) { + return Short.TYPE.equals(toClass) + || Integer.TYPE.equals(toClass) + || Long.TYPE.equals(toClass) + || Float.TYPE.equals(toClass) + || Double.TYPE.equals(toClass); + } + // should never get here + return false; + } + return toClass.isAssignableFrom(cls); + } + + /** Maps primitive {@code Class}es to their corresponding wrapper {@code Class}. */ + private static final Map, Class> primitiveWrapperMap = new HashMap<>(); + + static { + primitiveWrapperMap.put(Boolean.TYPE, Boolean.class); + primitiveWrapperMap.put(Byte.TYPE, Byte.class); + primitiveWrapperMap.put(Character.TYPE, Character.class); + primitiveWrapperMap.put(Short.TYPE, Short.class); + primitiveWrapperMap.put(Integer.TYPE, Integer.class); + primitiveWrapperMap.put(Long.TYPE, Long.class); + primitiveWrapperMap.put(Double.TYPE, Double.class); + primitiveWrapperMap.put(Float.TYPE, Float.class); + primitiveWrapperMap.put(Void.TYPE, Void.TYPE); + } + + /** Maps wrapper {@code Class}es to their corresponding primitive types. */ + private static final Map, Class> wrapperPrimitiveMap = new HashMap<>(); + + static { + for (final Class primitiveClass : primitiveWrapperMap.keySet()) { + final Class wrapperClass = primitiveWrapperMap.get(primitiveClass); + if (!primitiveClass.equals(wrapperClass)) { + wrapperPrimitiveMap.put(wrapperClass, primitiveClass); + } + } + } + + /** + * Converts the specified primitive Class object to its corresponding wrapper Class object. + * + *

    NOTE: From v2.2, this method handles {@code Void.TYPE}, returning {@code Void.TYPE}. + * + * @param cls the class to convert, may be null + * @return the wrapper class for {@code cls} or {@code cls} if {@code cls} is not a primitive. + * {@code null} if null input. + * @since 2.1 + */ + public static Class primitiveToWrapper(final Class cls) { + Class convertedClass = cls; + if (cls != null && cls.isPrimitive()) { + convertedClass = primitiveWrapperMap.get(cls); + } + return convertedClass; + } + + /** + * Converts the specified wrapper class to its corresponding primitive class. + * + *

    This method is the counter part of {@code primitiveToWrapper()}. If the passed in class is + * a wrapper class for a primitive type, this primitive type will be returned (e.g. {@code + * Integer.TYPE} for {@code Integer.class}). For other classes, or if the parameter is + * null, the return value is null. + * + * @param cls the class to convert, may be null + * @return the corresponding primitive type if {@code cls} is a wrapper class, null + * otherwise + * @see #primitiveToWrapper(Class) + * @since 2.4 + */ + public static Class wrapperToPrimitive(final Class cls) { + return wrapperPrimitiveMap.get(cls); + } + + // -------------------------------------------------------------------------------------------- + + private ExtractionUtils() { + // no instantiation + } +} diff --git a/dlink-common/src/main/java/com/dlink/pool/ClassEntity.java b/dlink-common/src/main/java/com/dlink/pool/ClassEntity.java new file mode 100644 index 0000000000..e43e9dfa77 --- /dev/null +++ b/dlink-common/src/main/java/com/dlink/pool/ClassEntity.java @@ -0,0 +1,42 @@ +package com.dlink.pool; + +import com.dlink.assertion.Asserts; +import lombok.Getter; +import lombok.Setter; + +/** + * ClassEntity + * + * @author wenmo + * @since 2022/1/12 23:52 + */ +@Getter +@Setter +public class ClassEntity { + private String name; + private String code; + private byte[] classByte; + + public ClassEntity(String name, String code) { + this.name = name; + this.code = code; + } + + public ClassEntity(String name, String code, byte[] classByte) { + this.name = name; + this.code = code; + this.classByte = classByte; + } + + public static ClassEntity build(String name, String code){ + return new ClassEntity(name,code); + } + + public boolean equals(ClassEntity entity) { + if (Asserts.isEquals(name, entity.getName()) && Asserts.isEquals(code, entity.getCode())){ + return true; + }else{ + return false; + } + } +} diff --git a/dlink-common/src/main/java/com/dlink/pool/ClassPool.java b/dlink-common/src/main/java/com/dlink/pool/ClassPool.java new file mode 100644 index 0000000000..7da8f049c6 --- /dev/null +++ b/dlink-common/src/main/java/com/dlink/pool/ClassPool.java @@ -0,0 +1,61 @@ +package com.dlink.pool; + +import java.util.List; +import java.util.Vector; + +/** + * ClassPool + * + * @author wenmo + * @since 2022/1/12 23:52 + */ +public class ClassPool { + + private static volatile List classList = new Vector<>(); + + public static boolean exist(String name) { + for (ClassEntity executorEntity : classList) { + if (executorEntity.getName().equals(name)) { + return true; + } + } + return false; + } + + public static boolean exist(ClassEntity entity) { + for (ClassEntity executorEntity : classList) { + if (executorEntity.equals(entity)) { + return true; + } + } + return false; + } + + public static Integer push(ClassEntity executorEntity){ + if(exist(executorEntity.getName())){ + remove(executorEntity.getName()); + } + classList.add(executorEntity); + return classList.size(); + } + + public static Integer remove(String name) { + int count = classList.size(); + for (int i = 0; i < classList.size(); i++) { + if (name.equals(classList.get(i).getName())) { + classList.remove(i); + break; + } + } + return count - classList.size(); + } + + public static ClassEntity get(String name) { + for (ClassEntity executorEntity : classList) { + if (executorEntity.getName().equals(name)) { + return executorEntity; + } + } + return null; + } +} diff --git a/dlink-core/pom.xml b/dlink-core/pom.xml index f337f286c8..c65cfdad23 100644 --- a/dlink-core/pom.xml +++ b/dlink-core/pom.xml @@ -35,6 +35,11 @@ cn.hutool hutool-all + + org.codehaus.groovy + groovy + 3.0.9 + junit junit diff --git a/dlink-core/src/main/java/com/dlink/job/JobManager.java b/dlink-core/src/main/java/com/dlink/job/JobManager.java index e8ff1cbd46..b57be45959 100644 --- a/dlink-core/src/main/java/com/dlink/job/JobManager.java +++ b/dlink-core/src/main/java/com/dlink/job/JobManager.java @@ -19,6 +19,8 @@ import com.dlink.interceptor.FlinkInterceptor; import com.dlink.model.SystemConfiguration; import com.dlink.parser.SqlType; +import com.dlink.pool.ClassEntity; +import com.dlink.pool.ClassPool; import com.dlink.result.*; import com.dlink.session.ExecutorEntity; import com.dlink.session.SessionConfig; @@ -26,6 +28,7 @@ import com.dlink.session.SessionPool; import com.dlink.trans.Operations; import com.dlink.utils.SqlUtil; +import com.dlink.utils.UDFUtil; import com.fasterxml.jackson.databind.node.ObjectNode; import org.apache.flink.configuration.CoreOptions; import org.apache.flink.configuration.DeploymentOptions; @@ -43,6 +46,8 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; /** * JobManager @@ -491,4 +496,22 @@ public String exportSql(String sql){ sb.append(statement); return sb.toString(); } + + public static List getUDFClassName(String statement){ + Pattern pattern = Pattern.compile("function (.*?)'(.*?)'", Pattern.CASE_INSENSITIVE); + Matcher matcher = pattern.matcher(statement); + List classNameList = new ArrayList<>(); + while (matcher.find()) { + classNameList.add(matcher.group(2)); + } + return classNameList; + } + + public static void initUDF(String className,String code){ + if(ClassPool.exist(ClassEntity.build(className,code))){ + UDFUtil.initClassLoader(className); + }else{ + UDFUtil.buildClass(code); + } + } } diff --git a/dlink-core/src/main/java/com/dlink/utils/CustomStringJavaCompiler.java b/dlink-core/src/main/java/com/dlink/utils/CustomStringJavaCompiler.java new file mode 100644 index 0000000000..5b1540e823 --- /dev/null +++ b/dlink-core/src/main/java/com/dlink/utils/CustomStringJavaCompiler.java @@ -0,0 +1,166 @@ +package com.dlink.utils; + +import javax.tools.*; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.net.URI; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * CustomStringJavaCompiler + * + * @author wenmo + * @since 2021/12/28 22:46 + */ +public class CustomStringJavaCompiler { + //类全名 + private String fullClassName; + private String sourceCode; + //存放编译之后的字节码(key:类全名,value:编译之后输出的字节码) + private Map javaFileObjectMap = new ConcurrentHashMap<>(); + //获取java的编译器 + private JavaCompiler compiler = ToolProvider.getSystemJavaCompiler(); + //存放编译过程中输出的信息 + private DiagnosticCollector diagnosticsCollector = new DiagnosticCollector<>(); + //编译耗时(单位ms) + private long compilerTakeTime; + + public String getFullClassName() { + return fullClassName; + } + + public ByteJavaFileObject getJavaFileObjectMap(String name) { + return javaFileObjectMap.get(name); + } + + public CustomStringJavaCompiler(String sourceCode) { + this.sourceCode = sourceCode; + this.fullClassName = getFullClassName(sourceCode); + } + + /** + * 编译字符串源代码,编译失败在 diagnosticsCollector 中获取提示信息 + * + * @return true:编译成功 false:编译失败 + */ + public boolean compiler() { + long startTime = System.currentTimeMillis(); + //标准的内容管理器,更换成自己的实现，覆盖部分方法 + StandardJavaFileManager standardFileManager = compiler.getStandardFileManager(diagnosticsCollector, null, null); + JavaFileManager javaFileManager = new StringJavaFileManage(standardFileManager); + //构造源代码对象 + JavaFileObject javaFileObject = new StringJavaFileObject(fullClassName, sourceCode); + //获取一个编译任务 + JavaCompiler.CompilationTask task = compiler.getTask(null, javaFileManager, diagnosticsCollector, null, null, Arrays.asList(javaFileObject)); + //设置编译耗时 + compilerTakeTime = System.currentTimeMillis() - startTime; + return task.call(); + } + + /** + * @return 编译信息(错误 警告) + */ + public String getCompilerMessage() { + StringBuilder sb = new StringBuilder(); + List> diagnostics = diagnosticsCollector.getDiagnostics(); + for (Diagnostic diagnostic : diagnostics) { + sb.append(diagnostic.toString()).append("\r\n"); + } + return sb.toString(); + } + + public long getCompilerTakeTime() { + return compilerTakeTime; + } + + /** + * 获取类的全名称 + * + * @param sourceCode 源码 + * @return 类的全名称 + */ + public static String getFullClassName(String sourceCode) { + String className = ""; + Pattern pattern = Pattern.compile("package\\s+\\S+\\s*;"); + Matcher matcher = pattern.matcher(sourceCode); + if (matcher.find()) { + className = matcher.group().replaceFirst("package", "").replace(";", "").trim() + "."; + } + + pattern = Pattern.compile("class\\s+(\\S+)\\s+"); + matcher = pattern.matcher(sourceCode); + if (matcher.find()) { + className += matcher.group(1).trim(); + } + return className; + } + + /** + * 自定义一个字符串的源码对象 + */ + private class StringJavaFileObject extends SimpleJavaFileObject { + //等待编译的源码字段 + private String contents; + + //java源代码 => StringJavaFileObject对象 的时候使用 + public StringJavaFileObject(String className, String contents) { + super(URI.create("string:///" + className.replaceAll("\\.", "/") + Kind.SOURCE.extension), Kind.SOURCE); + this.contents = contents; + } + + //字符串源码会调用该方法 + @Override + public CharSequence getCharContent(boolean ignoreEncodingErrors) throws IOException { + return contents; + } + + } + + /** + * 自定义一个编译之后的字节码对象 + */ + public class ByteJavaFileObject extends SimpleJavaFileObject { + //存放编译后的字节码 + private ByteArrayOutputStream outPutStream; + + public ByteJavaFileObject(String className, Kind kind) { + super(URI.create("string:///" + className.replaceAll("\\.", "/") + Kind.SOURCE.extension), kind); + } + + //StringJavaFileManage 编译之后的字节码输出会调用该方法（把字节码输出到outputStream） + @Override + public OutputStream openOutputStream() { + outPutStream = new ByteArrayOutputStream(); + return outPutStream; + } + + //在类加载器加载的时候需要用到 + public byte[] getCompiledBytes() { + return outPutStream.toByteArray(); + } + } + + /** + * 自定义一个JavaFileManage来控制编译之后字节码的输出位置 + */ + private class StringJavaFileManage extends ForwardingJavaFileManager { + StringJavaFileManage(JavaFileManager fileManager) { + super(fileManager); + } + + //获取输出的文件对象，它表示给定位置处指定类型的指定类。 + @Override + public JavaFileObject getJavaFileForOutput(Location location, String className, JavaFileObject.Kind kind, FileObject sibling) throws IOException { + ByteJavaFileObject javaFileObject = new ByteJavaFileObject(className, kind); + javaFileObjectMap.put(className, javaFileObject); + return javaFileObject; + } + } +} + diff --git a/dlink-core/src/main/java/com/dlink/utils/UDFUtil.java b/dlink-core/src/main/java/com/dlink/utils/UDFUtil.java new file mode 100644 index 0000000000..869b7148b5 --- /dev/null +++ b/dlink-core/src/main/java/com/dlink/utils/UDFUtil.java @@ -0,0 +1,43 @@ +package com.dlink.utils; + +import com.dlink.pool.ClassEntity; +import com.dlink.pool.ClassPool; +import groovy.lang.GroovyClassLoader; +import org.codehaus.groovy.control.CompilerConfiguration; + +/** + * UDFUtil + * + * @author wenmo + * @since 2021/12/27 23:25 + */ +public class UDFUtil { + + public static void buildClass(String code){ + CustomStringJavaCompiler compiler = new CustomStringJavaCompiler(code); + boolean res = compiler.compiler(); + if (res) { + String className = compiler.getFullClassName(); + byte[] compiledBytes = compiler.getJavaFileObjectMap(className).getCompiledBytes(); + ClassPool.push(new ClassEntity(className,code,compiledBytes)); + System.out.println("编译成功"); + System.out.println("compilerTakeTime：" + compiler.getCompilerTakeTime()); + initClassLoader(className); + } else { + System.out.println("编译失败"); + System.out.println(compiler.getCompilerMessage()); + } + } + + public static void initClassLoader(String name){ + ClassEntity classEntity = ClassPool.get(name); + ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader(); + CompilerConfiguration config = new CompilerConfiguration(); + config.setSourceEncoding("UTF-8"); + GroovyClassLoader groovyClassLoader = new GroovyClassLoader(contextClassLoader, config); + groovyClassLoader.setShouldRecompile(true); + groovyClassLoader.defineClass(classEntity.getName(),classEntity.getClassByte()); + Thread.currentThread().setContextClassLoader(groovyClassLoader); +// Class clazz = groovyClassLoader.parseClass(codeSource,"com.dlink.ud.udf.SubstringFunction"); + } +} diff --git a/dlink-web/src/components/Studio/StudioRightTool/StudioUDFInfo/index.less b/dlink-web/src/components/Studio/StudioRightTool/StudioUDFInfo/index.less new file mode 100644 index 0000000000..58a55f65b3 --- /dev/null +++ b/dlink-web/src/components/Studio/StudioRightTool/StudioUDFInfo/index.less @@ -0,0 +1,9 @@ +@import '~antd/es/style/themes/default.less'; + +.form_setting{ + padding-left: 10px; +} + +.form_item{ + margin-bottom: 5px; +} diff --git a/dlink-web/src/components/Studio/StudioRightTool/StudioUDFInfo/index.tsx b/dlink-web/src/components/Studio/StudioRightTool/StudioUDFInfo/index.tsx new file mode 100644 index 0000000000..1486867852 --- /dev/null +++ b/dlink-web/src/components/Studio/StudioRightTool/StudioUDFInfo/index.tsx @@ -0,0 +1,56 @@ +import {connect} from "umi"; +import {StateType} from "@/pages/FlinkSqlStudio/model"; +import {Form, Switch, Row, Col, Tooltip, Button, Input} from "antd"; +import {InfoCircleOutlined,MinusSquareOutlined} from "@ant-design/icons"; +import styles from "./index.less"; +import {useEffect} from "react"; +import {JarStateType} from "@/pages/Jar/model"; +import {Scrollbars} from "react-custom-scrollbars"; + +const StudioUDFInfo = (props: any) => { + + const { current, form, toolHeight} = props; + + useEffect(() => { + form.setFieldsValue(current.task); + }, [current.task]); + + return ( + <> + + +

    + +
    + + + +
    + + + + + + + +
    +     + + ); +}; + +export default connect(({Studio, Jar}: { Studio: StateType, Jar: JarStateType }) => ({ + current: Studio.current, + toolHeight: Studio.toolHeight, +}))(StudioUDFInfo); diff --git a/dlink-web/src/components/Studio/StudioRightTool/index.tsx b/dlink-web/src/components/Studio/StudioRightTool/index.tsx index 0add67eb74..2ecd70ea1d 100644 --- a/dlink-web/src/components/Studio/StudioRightTool/index.tsx +++ b/dlink-web/src/components/Studio/StudioRightTool/index.tsx @@ -8,6 +8,7 @@ import StudioSetting from "./StudioSetting"; import StudioSavePoint from "./StudioSavePoint"; import StudioEnvSetting from "./StudioEnvSetting"; import StudioSqlConfig from "./StudioSqlConfig"; +import StudioUDFInfo from "./StudioUDFInfo"; import {DIALECT, isSql} from "@/components/Studio/conf"; const { TabPane } = Tabs; @@ -26,7 +27,7 @@ const StudioRightTool = (props:any) => { return renderEnvContent(); } if(DIALECT.JAVA === current.task.dialect){ - return undefined; + return renderUDFContent(); } return renderFlinkSqlContent(); }; @@ -47,6 +48,14 @@ const StudioRightTool = (props:any) => { ) }; + const renderUDFContent = () => { + return (<> + UDF信息} key="StudioUDFInfo" > + + + ) + }; + const renderFlinkSqlContent = () => { return (<> 作业配置} key="StudioSetting" > diff --git a/dlink-web/src/pages/Welcome.tsx b/dlink-web/src/pages/Welcome.tsx index ff6dcd6058..802e9e8b82 100644 --- a/dlink-web/src/pages/Welcome.tsx +++ b/dlink-web/src/pages/Welcome.tsx @@ -547,6 +547,9 @@ export default (): React.ReactNode => {
  • 新增 快捷键保存、校验、美化
    • +
  • + 新增 UDF Java方言的Local模式的在线编写、调试、动态加载 +
    • diff --git a/docs/en-US/guide/quickstart.md b/docs/en-US/guide/quickstart.md index 91b23ee59d..6b6ad0c7b8 100644 --- a/docs/en-US/guide/quickstart.md +++ b/docs/en-US/guide/quickstart.md @@ -95,12 +95,12 @@ Dinky 通过已注册的集群配置来获取对应的 YarnClient 实例。对 | | | 支持 yarn application 模式下 FlinkSQL 提交 | 0.4.0 | | | | 支持 kubernetes session 模式下 FlinkSQL 提交 | 0.5.0 | | | | 支持 kubernetes application 模式下 FlinkSQL 提交 | 0.5.0 | +| | | 支持 UDF Java 方言Local模式在线编写、调试、动态加载 | 0.5.0 | | | Flink 作业 | 支持 yarn application 模式下 Jar 提交 | 0.4.0 | | | | 支持 k8s application 模式下 Jar 提交 | 0.5.0 | | | | 支持 作业 Cancel | 0.4.0 | | | | 支持 作业 SavePoint 的 Cancel、Stop、Trigger | 0.4.0 | | | | 新增 作业自动从 SavePoint 恢复机制（包含最近、最早、指定一次） | 0.4.0 | -| | | 新增 UDF java方言代码的开发 | 0.5.0 | | | Flink 集群 | 支持 查看已注册集群的作业列表与运维 | 0.4.0 | | | | 新增 自动注册 Yarn 创建的集群 | 0.4.0 | | | SQL | 新增 外部数据源的 SQL 校验 | 0.5.0 | diff --git a/docs/guide/quickstart.md b/docs/guide/quickstart.md index 91b23ee59d..6b6ad0c7b8 100644 --- a/docs/guide/quickstart.md +++ b/docs/guide/quickstart.md @@ -95,12 +95,12 @@ Dinky 通过已注册的集群配置来获取对应的 YarnClient 实例。对 | | | 支持 yarn application 模式下 FlinkSQL 提交 | 0.4.0 | | | | 支持 kubernetes session 模式下 FlinkSQL 提交 | 0.5.0 | | | | 支持 kubernetes application 模式下 FlinkSQL 提交 | 0.5.0 | +| | | 支持 UDF Java 方言Local模式在线编写、调试、动态加载 | 0.5.0 | | | Flink 作业 | 支持 yarn application 模式下 Jar 提交 | 0.4.0 | | | | 支持 k8s application 模式下 Jar 提交 | 0.5.0 | | | | 支持 作业 Cancel | 0.4.0 | | | | 支持 作业 SavePoint 的 Cancel、Stop、Trigger | 0.4.0 | | | | 新增 作业自动从 SavePoint 恢复机制（包含最近、最早、指定一次） | 0.4.0 | -| | | 新增 UDF java方言代码的开发 | 0.5.0 | | | Flink 集群 | 支持 查看已注册集群的作业列表与运维 | 0.4.0 | | | | 新增 自动注册 Yarn 创建的集群 | 0.4.0 | | | SQL | 新增 外部数据源的 SQL 校验 | 0.5.0 | From 87addd4de144a5050b509e2ff640915bab04ec87 Mon Sep 17 00:00:00 2001 From: zhumingye <934230207@qq.com> Date: Fri, 14 Jan 2022 14:05:48 +0800 Subject: [PATCH 06/10] =?UTF-8?q?fix:=20=20=20=20=E4=BF=AE=E5=A4=8D?= =?UTF-8?q?=E4=BB=BB=E5=8A=A1=E9=87=8D=E5=91=BD=E5=90=8D=E7=9A=84bug=20add?= =?UTF-8?q?:=20=20=20=20=E5=A2=9E=E5=8A=A0SQLStudio=20=E7=9A=84=E5=A4=9Ata?= =?UTF-8?q?b=20=E5=8F=B3=E9=94=AE=E5=85=B3=E9=97=AD=E8=8F=9C=E5=8D=95(?= =?UTF-8?q?=E5=85=B3=E9=97=AD=E5=85=B6=E4=BB=96/=E5=85=B3=E9=97=AD?= =?UTF-8?q?=E6=89=80=E6=9C=89(=E4=BC=9A=E5=B0=86=E8=8D=89=E7=A8=BF?= =?UTF-8?q?=E5=85=B3=E9=97=AD))?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../service/impl/CatalogueServiceImpl.java | 1 + .../components/Studio/StudioTabs/index.tsx | 85 ++- .../pages/FlinkSqlStudio/SqlExport/model.ts | 548 ++++++++++++++++++ 3 files changed, 612 insertions(+), 22 deletions(-) create mode 100644 dlink-web/src/pages/FlinkSqlStudio/SqlExport/model.ts diff --git a/dlink-admin/src/main/java/com/dlink/service/impl/CatalogueServiceImpl.java b/dlink-admin/src/main/java/com/dlink/service/impl/CatalogueServiceImpl.java index 470f487186..b10d45f2ef 100644 --- a/dlink-admin/src/main/java/com/dlink/service/impl/CatalogueServiceImpl.java +++ b/dlink-admin/src/main/java/com/dlink/service/impl/CatalogueServiceImpl.java @@ -88,6 +88,7 @@ public boolean toRename(Catalogue catalogue) { }else{ Task task = new Task(); task.setId(oldCatalogue.getTaskId()); + task.setName(catalogue.getName()); task.setAlias(catalogue.getName()); taskService.updateById(task); this.updateById(catalogue); diff --git a/dlink-web/src/components/Studio/StudioTabs/index.tsx b/dlink-web/src/components/Studio/StudioTabs/index.tsx index 31a67f7f85..73d98bbc42 100644 --- a/dlink-web/src/components/Studio/StudioTabs/index.tsx +++ b/dlink-web/src/components/Studio/StudioTabs/index.tsx @@ -1,24 +1,25 @@ -import {message, Tabs} from 'antd'; +import {message, Tabs, Menu, Dropdown} from 'antd'; import React, {useState} from 'react'; -import {connect} from "umi"; -import {StateType} from "@/pages/FlinkSqlStudio/model"; +import {connect} from 'umi'; +import {StateType} from '@/pages/FlinkSqlStudio/model'; import styles from './index.less'; import StudioEdit from '../StudioEdit'; -import {saveTask} from "@/components/Studio/StudioEvent/DDL"; -import { DIALECT } from '../conf'; +import {saveTask} from '@/components/Studio/StudioEvent/DDL'; +import {DIALECT} from '../conf'; const {TabPane} = Tabs; const EditorTabs = (props: any) => { - const {tabs, dispatch, current, toolHeight,width} = props; + const {tabs, dispatch, current, toolHeight, width} = props; const onChange = (activeKey: any) => { - dispatch&&dispatch({ - type: "Studio/saveToolHeight", - payload: toolHeight-0.0001, + dispatch && + dispatch({ + type: 'Studio/saveToolHeight', + payload: toolHeight - 0.0001, }); dispatch({ - type: "Studio/changeActiveKey", + type: 'Studio/changeActiveKey', payload: activeKey, }); }; @@ -27,9 +28,10 @@ const EditorTabs = (props: any) => { if (action == 'add') { add(); } else if (action == 'remove') { - dispatch&&dispatch({ - type: "Studio/saveToolHeight", - payload: toolHeight-0.0001, + dispatch && + dispatch({ + type: 'Studio/saveToolHeight', + payload: toolHeight - 0.0001, }); if (current.isModified) { saveTask(current, dispatch); @@ -51,7 +53,7 @@ const EditorTabs = (props: any) => { } }); let panes = tabs.panes; - const newPanes = panes.filter(pane => pane.key.toString() != targetKey); + const newPanes = panes.filter((pane) => pane.key.toString() != targetKey); if (newPanes.length && newActiveKey.toString() === targetKey) { if (lastIndex > 0) { newActiveKey = newPanes[lastIndex].key; @@ -60,13 +62,49 @@ const EditorTabs = (props: any) => { } } dispatch({ - type: "Studio/saveTabs", + type: 'Studio/saveTabs', payload: { activeKey: newActiveKey, panes: newPanes, }, }); }; + + const handleClickMenu = (e: any, current) => { + dispatch({ + type: 'Studio/closeTabs', + payload: { + deleteType: e.key, + current + }, + }); + }; + + const menu = (pane) => ( + handleClickMenu(e, pane)}> + + 关闭其他 + + + 关闭所有 + + + ); + + const Tab = (pane: any) => ( + + {pane.key === 0 ? ( + pane.title + ) : ( + + + {pane.title} + + + )} + + ); + return ( { onChange={onChange} activeKey={tabs.activeKey + ''} onEdit={onEdit} - className={styles["edit-tabs"]} + className={styles['edit-tabs']} style={{height: toolHeight}} > - {tabs.panes.map(pane => ( - - + {tabs.panes.map((pane) => ( + + ))} - - ) + ); }; export default connect(({Studio}: { Studio: StateType }) => ({ diff --git a/dlink-web/src/pages/FlinkSqlStudio/SqlExport/model.ts b/dlink-web/src/pages/FlinkSqlStudio/SqlExport/model.ts new file mode 100644 index 0000000000..9dbbfd49ba --- /dev/null +++ b/dlink-web/src/pages/FlinkSqlStudio/SqlExport/model.ts @@ -0,0 +1,548 @@ +import type {Effect, Reducer} from "umi"; +import { + handleAddOrUpdate +} from "@/components/Common/crud"; +import type {SqlMetaData} from "@/components/Studio/StudioEvent/data"; + +export type ClusterType = { + id: number, + name: string, + alias: string, + type: string, + hosts: string, + jobManagerHost: string, + status: number, + note: string, + enabled: boolean, + createTime: Date, + updateTime: Date, +} + +export type ClusterConfigurationType = { + id: number, + name: string, + alias: string, + type: string, + config: any, + available: boolean, + note: string, + enabled: boolean, + createTime: Date, + updateTime: Date, +} + +export type DataBaseType = { + id: number, + name: string, + alias: string, + groupName: string, + type: string, + url: string, + username: string, + password: string, + note: string, + dbVersion: string, + status: boolean, + healthTime: Date, + heartbeatTime: Date, + enabled: boolean, + createTime: Date, + updateTime: Date, +}; + +export type EnvType = { + id?: number, + name?: string, + alias?: string, + fragment?: boolean, +}; + +export type TaskType = { + id?: number, + catalogueId?: number, + name?: string, + alias?: string, + dialect?: string, + type?: string, + checkPoint?: number, + savePointStrategy?: number, + savePointPath?: string, + parallelism?: number, + fragment?: boolean, + statementSet?: boolean, + config?: [], + clusterId?: any, + clusterName?: string, + clusterConfigurationId?: number, + clusterConfigurationName?: string, + databaseId?: number, + databaseName?: string, + jarId?: number, + envId?: number, + note?: string, + enabled?: boolean, + createTime?: Date, + updateTime?: Date, + statement?: string, + session: string; + maxRowNum: number; + jobName: string; + useResult: boolean; + useChangeLog: boolean; + useAutoCancel: boolean; + useSession: boolean; + useRemote: boolean; +}; + +export type ConsoleType = { + result: {}; +} + +export type TabsItemType = { + title: string; + key: number, + value: string; + closable: boolean; + path: string[]; + task?: TaskType; + console: ConsoleType; + monaco?: any; + isModified: boolean; + sqlMetaData?: SqlMetaData; +} + +export type TabsType = { + activeKey: number; + panes?: TabsItemType[]; +} + +export type ConnectorType = { + tablename: string; +} + +export type SessionType = { + session?: string; + sessionConfig?: { + type?: string; + useRemote?: boolean; + clusterId?: number; + clusterName?: string; + address?: string; + } + createUser?: string; + createTime?: string; + connectors: ConnectorType[]; +} + +export type StateType = { + toolHeight?: number; + toolRightWidth?: number; + toolLeftWidth?: number; + cluster?: ClusterType[]; + sessionCluster?: ClusterType[]; + clusterConfiguration?: ClusterConfigurationType[]; + database?: DataBaseType[]; + env?: EnvType[]; + currentSession?: SessionType; + current?: TabsItemType; + sql?: string; + monaco?: any; + currentPath?: string[]; + tabs?: TabsType; + session?: SessionType[]; + result?: {}; + rightClickMenu?: boolean; + refs?: { + history: any; + }; +}; + +export type ModelType = { + namespace: string; + state: StateType; + effects: { + saveTask: Effect; + }; + reducers: { + saveToolHeight: Reducer; + saveToolRightWidth: Reducer; + saveToolLeftWidth: Reducer; + saveSql: Reducer; + saveCurrentPath: Reducer; + saveMonaco: Reducer; + saveSqlMetaData: Reducer; + saveTabs: Reducer; + changeActiveKey: Reducer; + saveTaskData: Reducer; + saveSession: Reducer; + showRightClickMenu: Reducer; + refreshCurrentSession: Reducer; + quitCurrentSession: Reducer; + saveResult: Reducer; + saveCluster: Reducer; + saveSessionCluster: Reducer; + saveClusterConfiguration: Reducer; + saveDataBase: Reducer; + saveEnv: Reducer; + }; +}; + +const Model: ModelType = { + namespace: 'Studio', + state: { + toolHeight: 400, + toolRightWidth: 300, + toolLeftWidth: 300, + cluster: [], + sessionCluster: [], + clusterConfiguration: [], + database: [], + env: [], + currentSession: { + connectors: [], + }, + current: { + title: '草稿', + key: 0, + value: '', + closable: false, + path: ['草稿'], + isModified: false, + task: { + jobName: '草稿', + type: 'local', + checkPoint: 0, + savePointStrategy: 0, + savePointPath: '', + parallelism: 1, + fragment: true, + statementSet: false, + clusterId: 0, + clusterName: "本地环境", + clusterConfigurationId: undefined, + clusterConfigurationName: undefined, + databaseId: undefined, + databaseName: undefined, + jarId: undefined, + envId: undefined, + maxRowNum: 100, + config: [], + session: '', + alias: '草稿', + dialect: 'FlinkSql', + useResult: true, + useChangeLog: false, + useAutoCancel: false, + useSession: false, + useRemote: false, + }, + console: { + result: {}, + }, + monaco: {}, + sqlMetaData: undefined, + }, + sql: '', + monaco: {}, + currentPath: ['草稿'], + tabs: { + activeKey: 0, + panes: [{ + title: '草稿', + key: 0, + value: '', + closable: false, + isModified: false, + path: ['草稿'], + task: { + jobName: '草稿', + type: 'local', + checkPoint: 0, + savePointStrategy: 0, + savePointPath: '', + parallelism: 1, + fragment: true, + statementSet: false, + clusterId: 0, + clusterName: "本地环境", + clusterConfigurationId: undefined, + clusterConfigurationName: undefined, + databaseId: undefined, + databaseName: undefined, + jarId: undefined, + envId: undefined, + session: '', + config: [], + maxRowNum: 100, + alias: '草稿', + dialect: 'FlinkSql', + useResult: true, + useChangeLog: false, + useAutoCancel: false, + useSession: false, + useRemote: false, + }, + console: { + result: {}, + }, + monaco: {}, + sqlMetaData: undefined, + }], + }, + session: [], + result: {}, + rightClickMenu: false, + refs: { + history: {}, + } + }, + + effects: { + * saveTask({payload}, {call, put}) { + const para = payload; + para.configJson = JSON.stringify(payload.config); + yield call(handleAddOrUpdate, 'api/task', para); + yield put({ + type: 'saveTaskData', + payload, + }); + }, + }, + + reducers: { + saveToolHeight(state, {payload}) { + return { + ...state, + toolHeight: payload, + }; + }, saveToolRightWidth(state, {payload}) { + return { + ...state, + toolRightWidth: payload, + }; + }, saveToolLeftWidth(state, {payload}) { + return { + ...state, + toolLeftWidth: payload, + }; + }, + saveSql(state, {payload}) { + const {tabs} = state; + const newCurrent = state.current; + newCurrent.value = payload; + for (let i = 0; i < tabs.panes.length; i++) { + if (tabs.panes[i].key == tabs.activeKey) { + tabs.panes[i].value = payload; + tabs.panes[i].task && (tabs.panes[i].task.statement = payload); + } + } + return { + ...state, + current: { + ...newCurrent + }, + tabs: { + ...tabs + }, + }; + }, + saveCurrentPath(state, {payload}) { + return { + ...state, + currentPath: payload, + }; + }, + saveMonaco(state, {payload}) { + return { + ...state, + monaco: { + ...payload + }, + }; + }, + saveSqlMetaData(state, {payload}) { + const newCurrent = state.current; + const newTabs = state.tabs; + if (newCurrent.key == payload.activeKey) { + newCurrent.sqlMetaData = payload.sqlMetaData; + newCurrent.isModified = payload.isModified; + } + for (let i = 0; i < newTabs.panes.length; i++) { + if (newTabs.panes[i].key == payload.activeKey) { + newTabs.panes[i].sqlMetaData = payload.sqlMetaData; + newTabs.panes[i].isModified = payload.isModified; + break; + } + } + return { + ...state, + current: newCurrent, + tabs: newTabs, + }; + }, + saveTabs(state, {payload}) { + let newCurrent = state.current; + for (let i = 0; i < payload.panes.length; i++) { + if (payload.panes[i].key == payload.activeKey) { + newCurrent = payload.panes[i]; + } + } + return { + ...state, + current: { + ...newCurrent, + isModified: false, + }, + tabs: { + ...payload, + }, + }; + }, + deleteTabByKey(state, {payload}) { + const newTabs = state.tabs; + for (let i = 0; i < newTabs.panes.length; i++) { + if (newTabs.panes[i].key == payload) { + newTabs.panes.splice(i, 1); + break; + } + } + const newCurrent = newTabs.panes[newTabs.panes.length - 1]; + if (newTabs.activeKey == payload) { + newTabs.activeKey = newCurrent.key; + } + return { + ...state, + current: { + ...newCurrent, + }, + tabs: { + ...newTabs, + }, + }; + }, + closeTabs(state, {payload}) { + const {deleteType, current} = payload + const newTabs = state.tabs; + const firstKey = newTabs.panes[0].key + let newCurrent = newTabs.panes[0] + if (deleteType === 'CLOSE_OTHER') { + const keys = [firstKey, current.key]; + newCurrent = {...current} + newTabs.activeKey = current.key + newTabs.panes = newTabs.panes.filter(item => keys.includes(item.key)); + } else { + newTabs.panes = []; + newTabs.activeKey = firstKey + } + + return { + ...state, + current: { + ...newCurrent + }, + tabs: { + ...newTabs, + } + }; + }, + changeActiveKey(state, {payload}) { + const {tabs} = state; + tabs.activeKey = payload; + let newCurrent = state.current; + for (let i = 0; i < tabs.panes.length; i++) { + if (tabs.panes[i].key == tabs.activeKey) { + newCurrent = tabs.panes[i]; + } + } + return { + ...state, + current: { + ...newCurrent, + }, + tabs: { + ...tabs, + }, + currentPath: newCurrent.path, + }; + }, + saveTaskData(state, {payload}) { + const newTabs = state.tabs; + for (let i = 0; i < newTabs.panes.length; i++) { + if (newTabs.panes[i].key == newTabs.activeKey) { + newTabs.panes[i].task = payload; + } + } + return { + ...state, + tabs: { + ...newTabs, + }, + }; + }, + saveSession(state, {payload}) { + return { + ...state, + session: [...payload], + }; + }, + showRightClickMenu(state, {payload}) { + return { + ...state, + rightClickMenu: payload, + }; + }, + refreshCurrentSession(state, {payload}) { + return { + ...state, + currentSession: { + ...state?.currentSession, + ...payload + } + }; + }, + quitCurrentSession(state) { + return { + ...state, + currentSession: { + connectors: [], + } + }; + }, + saveResult(state, {payload}) { + return { + ...state, + result: { + ...payload + }, + }; + }, + saveCluster(state, {payload}) { + return { + ...state, + cluster: payload, + }; + }, saveSessionCluster(state, {payload}) { + return { + ...state, + sessionCluster: payload, + }; + }, saveClusterConfiguration(state, {payload}) { + return { + ...state, + clusterConfiguration: payload, + }; + }, saveDataBase(state, {payload}) { + return { + ...state, + database: payload, + }; + }, saveEnv(state, {payload}) { + return { + ...state, + env: payload, + }; + }, + }, +}; + +export default Model; From 5110bb6915512a7f968a236da7df2f0c4594d5b5 Mon Sep 17 00:00:00 2001 From: aiwenmo <32723967+aiwenmo@users.noreply.github.com> Date: Fri, 14 Jan 2022 16:05:28 +0800 Subject: [PATCH 07/10] Revert "fixbug && add SQLStudio Right Menu" --- .../service/impl/CatalogueServiceImpl.java | 1 - .../components/Studio/StudioTabs/index.tsx | 85 +-- .../pages/FlinkSqlStudio/SqlExport/model.ts | 548 ------------------ 3 files changed, 22 insertions(+), 612 deletions(-) delete mode 100644 dlink-web/src/pages/FlinkSqlStudio/SqlExport/model.ts diff --git a/dlink-admin/src/main/java/com/dlink/service/impl/CatalogueServiceImpl.java b/dlink-admin/src/main/java/com/dlink/service/impl/CatalogueServiceImpl.java index b10d45f2ef..470f487186 100644 --- a/dlink-admin/src/main/java/com/dlink/service/impl/CatalogueServiceImpl.java +++ b/dlink-admin/src/main/java/com/dlink/service/impl/CatalogueServiceImpl.java @@ -88,7 +88,6 @@ public boolean toRename(Catalogue catalogue) { }else{ Task task = new Task(); task.setId(oldCatalogue.getTaskId()); - task.setName(catalogue.getName()); task.setAlias(catalogue.getName()); taskService.updateById(task); this.updateById(catalogue); diff --git a/dlink-web/src/components/Studio/StudioTabs/index.tsx b/dlink-web/src/components/Studio/StudioTabs/index.tsx index 73d98bbc42..31a67f7f85 100644 --- a/dlink-web/src/components/Studio/StudioTabs/index.tsx +++ b/dlink-web/src/components/Studio/StudioTabs/index.tsx @@ -1,25 +1,24 @@ -import {message, Tabs, Menu, Dropdown} from 'antd'; +import {message, Tabs} from 'antd'; import React, {useState} from 'react'; -import {connect} from 'umi'; -import {StateType} from '@/pages/FlinkSqlStudio/model'; +import {connect} from "umi"; +import {StateType} from "@/pages/FlinkSqlStudio/model"; import styles from './index.less'; import StudioEdit from '../StudioEdit'; -import {saveTask} from '@/components/Studio/StudioEvent/DDL'; -import {DIALECT} from '../conf'; +import {saveTask} from "@/components/Studio/StudioEvent/DDL"; +import { DIALECT } from '../conf'; const {TabPane} = Tabs; const EditorTabs = (props: any) => { - const {tabs, dispatch, current, toolHeight, width} = props; + const {tabs, dispatch, current, toolHeight,width} = props; const onChange = (activeKey: any) => { - dispatch && - dispatch({ - type: 'Studio/saveToolHeight', - payload: toolHeight - 0.0001, + dispatch&&dispatch({ + type: "Studio/saveToolHeight", + payload: toolHeight-0.0001, }); dispatch({ - type: 'Studio/changeActiveKey', + type: "Studio/changeActiveKey", payload: activeKey, }); }; @@ -28,10 +27,9 @@ const EditorTabs = (props: any) => { if (action == 'add') { add(); } else if (action == 'remove') { - dispatch && - dispatch({ - type: 'Studio/saveToolHeight', - payload: toolHeight - 0.0001, + dispatch&&dispatch({ + type: "Studio/saveToolHeight", + payload: toolHeight-0.0001, }); if (current.isModified) { saveTask(current, dispatch); @@ -53,7 +51,7 @@ const EditorTabs = (props: any) => { } }); let panes = tabs.panes; - const newPanes = panes.filter((pane) => pane.key.toString() != targetKey); + const newPanes = panes.filter(pane => pane.key.toString() != targetKey); if (newPanes.length && newActiveKey.toString() === targetKey) { if (lastIndex > 0) { newActiveKey = newPanes[lastIndex].key; @@ -62,49 +60,13 @@ const EditorTabs = (props: any) => { } } dispatch({ - type: 'Studio/saveTabs', + type: "Studio/saveTabs", payload: { activeKey: newActiveKey, panes: newPanes, }, }); }; - - const handleClickMenu = (e: any, current) => { - dispatch({ - type: 'Studio/closeTabs', - payload: { - deleteType: e.key, - current - }, - }); - }; - - const menu = (pane) => ( - handleClickMenu(e, pane)}> - - 关闭其他 - - - 关闭所有 - - - ); - - const Tab = (pane: any) => ( - - {pane.key === 0 ? ( - pane.title - ) : ( - - - {pane.title} - - - )} - - ); - return ( { onChange={onChange} activeKey={tabs.activeKey + ''} onEdit={onEdit} - className={styles['edit-tabs']} + className={styles["edit-tabs"]} style={{height: toolHeight}} > - {tabs.panes.map((pane) => ( - - + {tabs.panes.map(pane => ( + + ))} - ); + + ) }; export default connect(({Studio}: { Studio: StateType }) => ({ diff --git a/dlink-web/src/pages/FlinkSqlStudio/SqlExport/model.ts b/dlink-web/src/pages/FlinkSqlStudio/SqlExport/model.ts deleted file mode 100644 index 9dbbfd49ba..0000000000 --- a/dlink-web/src/pages/FlinkSqlStudio/SqlExport/model.ts +++ /dev/null @@ -1,548 +0,0 @@ -import type {Effect, Reducer} from "umi"; -import { - handleAddOrUpdate -} from "@/components/Common/crud"; -import type {SqlMetaData} from "@/components/Studio/StudioEvent/data"; - -export type ClusterType = { - id: number, - name: string, - alias: string, - type: string, - hosts: string, - jobManagerHost: string, - status: number, - note: string, - enabled: boolean, - createTime: Date, - updateTime: Date, -} - -export type ClusterConfigurationType = { - id: number, - name: string, - alias: string, - type: string, - config: any, - available: boolean, - note: string, - enabled: boolean, - createTime: Date, - updateTime: Date, -} - -export type DataBaseType = { - id: number, - name: string, - alias: string, - groupName: string, - type: string, - url: string, - username: string, - password: string, - note: string, - dbVersion: string, - status: boolean, - healthTime: Date, - heartbeatTime: Date, - enabled: boolean, - createTime: Date, - updateTime: Date, -}; - -export type EnvType = { - id?: number, - name?: string, - alias?: string, - fragment?: boolean, -}; - -export type TaskType = { - id?: number, - catalogueId?: number, - name?: string, - alias?: string, - dialect?: string, - type?: string, - checkPoint?: number, - savePointStrategy?: number, - savePointPath?: string, - parallelism?: number, - fragment?: boolean, - statementSet?: boolean, - config?: [], - clusterId?: any, - clusterName?: string, - clusterConfigurationId?: number, - clusterConfigurationName?: string, - databaseId?: number, - databaseName?: string, - jarId?: number, - envId?: number, - note?: string, - enabled?: boolean, - createTime?: Date, - updateTime?: Date, - statement?: string, - session: string; - maxRowNum: number; - jobName: string; - useResult: boolean; - useChangeLog: boolean; - useAutoCancel: boolean; - useSession: boolean; - useRemote: boolean; -}; - -export type ConsoleType = { - result: {}; -} - -export type TabsItemType = { - title: string; - key: number, - value: string; - closable: boolean; - path: string[]; - task?: TaskType; - console: ConsoleType; - monaco?: any; - isModified: boolean; - sqlMetaData?: SqlMetaData; -} - -export type TabsType = { - activeKey: number; - panes?: TabsItemType[]; -} - -export type ConnectorType = { - tablename: string; -} - -export type SessionType = { - session?: string; - sessionConfig?: { - type?: string; - useRemote?: boolean; - clusterId?: number; - clusterName?: string; - address?: string; - } - createUser?: string; - createTime?: string; - connectors: ConnectorType[]; -} - -export type StateType = { - toolHeight?: number; - toolRightWidth?: number; - toolLeftWidth?: number; - cluster?: ClusterType[]; - sessionCluster?: ClusterType[]; - clusterConfiguration?: ClusterConfigurationType[]; - database?: DataBaseType[]; - env?: EnvType[]; - currentSession?: SessionType; - current?: TabsItemType; - sql?: string; - monaco?: any; - currentPath?: string[]; - tabs?: TabsType; - session?: SessionType[]; - result?: {}; - rightClickMenu?: boolean; - refs?: { - history: any; - }; -}; - -export type ModelType = { - namespace: string; - state: StateType; - effects: { - saveTask: Effect; - }; - reducers: { - saveToolHeight: Reducer; - saveToolRightWidth: Reducer; - saveToolLeftWidth: Reducer; - saveSql: Reducer; - saveCurrentPath: Reducer; - saveMonaco: Reducer; - saveSqlMetaData: Reducer; - saveTabs: Reducer; - changeActiveKey: Reducer; - saveTaskData: Reducer; - saveSession: Reducer; - showRightClickMenu: Reducer; - refreshCurrentSession: Reducer; - quitCurrentSession: Reducer; - saveResult: Reducer; - saveCluster: Reducer; - saveSessionCluster: Reducer; - saveClusterConfiguration: Reducer; - saveDataBase: Reducer; - saveEnv: Reducer; - }; -}; - -const Model: ModelType = { - namespace: 'Studio', - state: { - toolHeight: 400, - toolRightWidth: 300, - toolLeftWidth: 300, - cluster: [], - sessionCluster: [], - clusterConfiguration: [], - database: [], - env: [], - currentSession: { - connectors: [], - }, - current: { - title: '草稿', - key: 0, - value: '', - closable: false, - path: ['草稿'], - isModified: false, - task: { - jobName: '草稿', - type: 'local', - checkPoint: 0, - savePointStrategy: 0, - savePointPath: '', - parallelism: 1, - fragment: true, - statementSet: false, - clusterId: 0, - clusterName: "本地环境", - clusterConfigurationId: undefined, - clusterConfigurationName: undefined, - databaseId: undefined, - databaseName: undefined, - jarId: undefined, - envId: undefined, - maxRowNum: 100, - config: [], - session: '', - alias: '草稿', - dialect: 'FlinkSql', - useResult: true, - useChangeLog: false, - useAutoCancel: false, - useSession: false, - useRemote: false, - }, - console: { - result: {}, - }, - monaco: {}, - sqlMetaData: undefined, - }, - sql: '', - monaco: {}, - currentPath: ['草稿'], - tabs: { - activeKey: 0, - panes: [{ - title: '草稿', - key: 0, - value: '', - closable: false, - isModified: false, - path: ['草稿'], - task: { - jobName: '草稿', - type: 'local', - checkPoint: 0, - savePointStrategy: 0, - savePointPath: '', - parallelism: 1, - fragment: true, - statementSet: false, - clusterId: 0, - clusterName: "本地环境", - clusterConfigurationId: undefined, - clusterConfigurationName: undefined, - databaseId: undefined, - databaseName: undefined, - jarId: undefined, - envId: undefined, - session: '', - config: [], - maxRowNum: 100, - alias: '草稿', - dialect: 'FlinkSql', - useResult: true, - useChangeLog: false, - useAutoCancel: false, - useSession: false, - useRemote: false, - }, - console: { - result: {}, - }, - monaco: {}, - sqlMetaData: undefined, - }], - }, - session: [], - result: {}, - rightClickMenu: false, - refs: { - history: {}, - } - }, - - effects: { - * saveTask({payload}, {call, put}) { - const para = payload; - para.configJson = JSON.stringify(payload.config); - yield call(handleAddOrUpdate, 'api/task', para); - yield put({ - type: 'saveTaskData', - payload, - }); - }, - }, - - reducers: { - saveToolHeight(state, {payload}) { - return { - ...state, - toolHeight: payload, - }; - }, saveToolRightWidth(state, {payload}) { - return { - ...state, - toolRightWidth: payload, - }; - }, saveToolLeftWidth(state, {payload}) { - return { - ...state, - toolLeftWidth: payload, - }; - }, - saveSql(state, {payload}) { - const {tabs} = state; - const newCurrent = state.current; - newCurrent.value = payload; - for (let i = 0; i < tabs.panes.length; i++) { - if (tabs.panes[i].key == tabs.activeKey) { - tabs.panes[i].value = payload; - tabs.panes[i].task && (tabs.panes[i].task.statement = payload); - } - } - return { - ...state, - current: { - ...newCurrent - }, - tabs: { - ...tabs - }, - }; - }, - saveCurrentPath(state, {payload}) { - return { - ...state, - currentPath: payload, - }; - }, - saveMonaco(state, {payload}) { - return { - ...state, - monaco: { - ...payload - }, - }; - }, - saveSqlMetaData(state, {payload}) { - const newCurrent = state.current; - const newTabs = state.tabs; - if (newCurrent.key == payload.activeKey) { - newCurrent.sqlMetaData = payload.sqlMetaData; - newCurrent.isModified = payload.isModified; - } - for (let i = 0; i < newTabs.panes.length; i++) { - if (newTabs.panes[i].key == payload.activeKey) { - newTabs.panes[i].sqlMetaData = payload.sqlMetaData; - newTabs.panes[i].isModified = payload.isModified; - break; - } - } - return { - ...state, - current: newCurrent, - tabs: newTabs, - }; - }, - saveTabs(state, {payload}) { - let newCurrent = state.current; - for (let i = 0; i < payload.panes.length; i++) { - if (payload.panes[i].key == payload.activeKey) { - newCurrent = payload.panes[i]; - } - } - return { - ...state, - current: { - ...newCurrent, - isModified: false, - }, - tabs: { - ...payload, - }, - }; - }, - deleteTabByKey(state, {payload}) { - const newTabs = state.tabs; - for (let i = 0; i < newTabs.panes.length; i++) { - if (newTabs.panes[i].key == payload) { - newTabs.panes.splice(i, 1); - break; - } - } - const newCurrent = newTabs.panes[newTabs.panes.length - 1]; - if (newTabs.activeKey == payload) { - newTabs.activeKey = newCurrent.key; - } - return { - ...state, - current: { - ...newCurrent, - }, - tabs: { - ...newTabs, - }, - }; - }, - closeTabs(state, {payload}) { - const {deleteType, current} = payload - const newTabs = state.tabs; - const firstKey = newTabs.panes[0].key - let newCurrent = newTabs.panes[0] - if (deleteType === 'CLOSE_OTHER') { - const keys = [firstKey, current.key]; - newCurrent = {...current} - newTabs.activeKey = current.key - newTabs.panes = newTabs.panes.filter(item => keys.includes(item.key)); - } else { - newTabs.panes = []; - newTabs.activeKey = firstKey - } - - return { - ...state, - current: { - ...newCurrent - }, - tabs: { - ...newTabs, - } - }; - }, - changeActiveKey(state, {payload}) { - const {tabs} = state; - tabs.activeKey = payload; - let newCurrent = state.current; - for (let i = 0; i < tabs.panes.length; i++) { - if (tabs.panes[i].key == tabs.activeKey) { - newCurrent = tabs.panes[i]; - } - } - return { - ...state, - current: { - ...newCurrent, - }, - tabs: { - ...tabs, - }, - currentPath: newCurrent.path, - }; - }, - saveTaskData(state, {payload}) { - const newTabs = state.tabs; - for (let i = 0; i < newTabs.panes.length; i++) { - if (newTabs.panes[i].key == newTabs.activeKey) { - newTabs.panes[i].task = payload; - } - } - return { - ...state, - tabs: { - ...newTabs, - }, - }; - }, - saveSession(state, {payload}) { - return { - ...state, - session: [...payload], - }; - }, - showRightClickMenu(state, {payload}) { - return { - ...state, - rightClickMenu: payload, - }; - }, - refreshCurrentSession(state, {payload}) { - return { - ...state, - currentSession: { - ...state?.currentSession, - ...payload - } - }; - }, - quitCurrentSession(state) { - return { - ...state, - currentSession: { - connectors: [], - } - }; - }, - saveResult(state, {payload}) { - return { - ...state, - result: { - ...payload - }, - }; - }, - saveCluster(state, {payload}) { - return { - ...state, - cluster: payload, - }; - }, saveSessionCluster(state, {payload}) { - return { - ...state, - sessionCluster: payload, - }; - }, saveClusterConfiguration(state, {payload}) { - return { - ...state, - clusterConfiguration: payload, - }; - }, saveDataBase(state, {payload}) { - return { - ...state, - database: payload, - }; - }, saveEnv(state, {payload}) { - return { - ...state, - env: payload, - }; - }, - }, -}; - -export default Model; From c2a9a1cd2d87c5c97c4005575b6d066e35c5fa67 Mon Sep 17 00:00:00 2001 From: aiwenmo <32723967+aiwenmo@users.noreply.github.com> Date: Fri, 14 Jan 2022 16:19:03 +0800 Subject: [PATCH 08/10] Revert "Revert "fixbug && add SQLStudio Right Menu"" --- .../service/impl/CatalogueServiceImpl.java | 1 + .../components/Studio/StudioTabs/index.tsx | 85 ++- .../pages/FlinkSqlStudio/SqlExport/model.ts | 548 ++++++++++++++++++ 3 files changed, 612 insertions(+), 22 deletions(-) create mode 100644 dlink-web/src/pages/FlinkSqlStudio/SqlExport/model.ts diff --git a/dlink-admin/src/main/java/com/dlink/service/impl/CatalogueServiceImpl.java b/dlink-admin/src/main/java/com/dlink/service/impl/CatalogueServiceImpl.java index 470f487186..b10d45f2ef 100644 --- a/dlink-admin/src/main/java/com/dlink/service/impl/CatalogueServiceImpl.java +++ b/dlink-admin/src/main/java/com/dlink/service/impl/CatalogueServiceImpl.java @@ -88,6 +88,7 @@ public boolean toRename(Catalogue catalogue) { }else{ Task task = new Task(); task.setId(oldCatalogue.getTaskId()); + task.setName(catalogue.getName()); task.setAlias(catalogue.getName()); taskService.updateById(task); this.updateById(catalogue); diff --git a/dlink-web/src/components/Studio/StudioTabs/index.tsx b/dlink-web/src/components/Studio/StudioTabs/index.tsx index 31a67f7f85..73d98bbc42 100644 --- a/dlink-web/src/components/Studio/StudioTabs/index.tsx +++ b/dlink-web/src/components/Studio/StudioTabs/index.tsx @@ -1,24 +1,25 @@ -import {message, Tabs} from 'antd'; +import {message, Tabs, Menu, Dropdown} from 'antd'; import React, {useState} from 'react'; -import {connect} from "umi"; -import {StateType} from "@/pages/FlinkSqlStudio/model"; +import {connect} from 'umi'; +import {StateType} from '@/pages/FlinkSqlStudio/model'; import styles from './index.less'; import StudioEdit from '../StudioEdit'; -import {saveTask} from "@/components/Studio/StudioEvent/DDL"; -import { DIALECT } from '../conf'; +import {saveTask} from '@/components/Studio/StudioEvent/DDL'; +import {DIALECT} from '../conf'; const {TabPane} = Tabs; const EditorTabs = (props: any) => { - const {tabs, dispatch, current, toolHeight,width} = props; + const {tabs, dispatch, current, toolHeight, width} = props; const onChange = (activeKey: any) => { - dispatch&&dispatch({ - type: "Studio/saveToolHeight", - payload: toolHeight-0.0001, + dispatch && + dispatch({ + type: 'Studio/saveToolHeight', + payload: toolHeight - 0.0001, }); dispatch({ - type: "Studio/changeActiveKey", + type: 'Studio/changeActiveKey', payload: activeKey, }); }; @@ -27,9 +28,10 @@ const EditorTabs = (props: any) => { if (action == 'add') { add(); } else if (action == 'remove') { - dispatch&&dispatch({ - type: "Studio/saveToolHeight", - payload: toolHeight-0.0001, + dispatch && + dispatch({ + type: 'Studio/saveToolHeight', + payload: toolHeight - 0.0001, }); if (current.isModified) { saveTask(current, dispatch); @@ -51,7 +53,7 @@ const EditorTabs = (props: any) => { } }); let panes = tabs.panes; - const newPanes = panes.filter(pane => pane.key.toString() != targetKey); + const newPanes = panes.filter((pane) => pane.key.toString() != targetKey); if (newPanes.length && newActiveKey.toString() === targetKey) { if (lastIndex > 0) { newActiveKey = newPanes[lastIndex].key; @@ -60,13 +62,49 @@ const EditorTabs = (props: any) => { } } dispatch({ - type: "Studio/saveTabs", + type: 'Studio/saveTabs', payload: { activeKey: newActiveKey, panes: newPanes, }, }); }; + + const handleClickMenu = (e: any, current) => { + dispatch({ + type: 'Studio/closeTabs', + payload: { + deleteType: e.key, + current + }, + }); + }; + + const menu = (pane) => ( + handleClickMenu(e, pane)}> + + 关闭其他 + + + 关闭所有 + + + ); + + const Tab = (pane: any) => ( + + {pane.key === 0 ? ( + pane.title + ) : ( + + + {pane.title} + + + )} + + ); + return ( { onChange={onChange} activeKey={tabs.activeKey + ''} onEdit={onEdit} - className={styles["edit-tabs"]} + className={styles['edit-tabs']} style={{height: toolHeight}} > - {tabs.panes.map(pane => ( - - + {tabs.panes.map((pane) => ( + + ))} - - ) + ); }; export default connect(({Studio}: { Studio: StateType }) => ({ diff --git a/dlink-web/src/pages/FlinkSqlStudio/SqlExport/model.ts b/dlink-web/src/pages/FlinkSqlStudio/SqlExport/model.ts new file mode 100644 index 0000000000..9dbbfd49ba --- /dev/null +++ b/dlink-web/src/pages/FlinkSqlStudio/SqlExport/model.ts @@ -0,0 +1,548 @@ +import type {Effect, Reducer} from "umi"; +import { + handleAddOrUpdate +} from "@/components/Common/crud"; +import type {SqlMetaData} from "@/components/Studio/StudioEvent/data"; + +export type ClusterType = { + id: number, + name: string, + alias: string, + type: string, + hosts: string, + jobManagerHost: string, + status: number, + note: string, + enabled: boolean, + createTime: Date, + updateTime: Date, +} + +export type ClusterConfigurationType = { + id: number, + name: string, + alias: string, + type: string, + config: any, + available: boolean, + note: string, + enabled: boolean, + createTime: Date, + updateTime: Date, +} + +export type DataBaseType = { + id: number, + name: string, + alias: string, + groupName: string, + type: string, + url: string, + username: string, + password: string, + note: string, + dbVersion: string, + status: boolean, + healthTime: Date, + heartbeatTime: Date, + enabled: boolean, + createTime: Date, + updateTime: Date, +}; + +export type EnvType = { + id?: number, + name?: string, + alias?: string, + fragment?: boolean, +}; + +export type TaskType = { + id?: number, + catalogueId?: number, + name?: string, + alias?: string, + dialect?: string, + type?: string, + checkPoint?: number, + savePointStrategy?: number, + savePointPath?: string, + parallelism?: number, + fragment?: boolean, + statementSet?: boolean, + config?: [], + clusterId?: any, + clusterName?: string, + clusterConfigurationId?: number, + clusterConfigurationName?: string, + databaseId?: number, + databaseName?: string, + jarId?: number, + envId?: number, + note?: string, + enabled?: boolean, + createTime?: Date, + updateTime?: Date, + statement?: string, + session: string; + maxRowNum: number; + jobName: string; + useResult: boolean; + useChangeLog: boolean; + useAutoCancel: boolean; + useSession: boolean; + useRemote: boolean; +}; + +export type ConsoleType = { + result: {}; +} + +export type TabsItemType = { + title: string; + key: number, + value: string; + closable: boolean; + path: string[]; + task?: TaskType; + console: ConsoleType; + monaco?: any; + isModified: boolean; + sqlMetaData?: SqlMetaData; +} + +export type TabsType = { + activeKey: number; + panes?: TabsItemType[]; +} + +export type ConnectorType = { + tablename: string; +} + +export type SessionType = { + session?: string; + sessionConfig?: { + type?: string; + useRemote?: boolean; + clusterId?: number; + clusterName?: string; + address?: string; + } + createUser?: string; + createTime?: string; + connectors: ConnectorType[]; +} + +export type StateType = { + toolHeight?: number; + toolRightWidth?: number; + toolLeftWidth?: number; + cluster?: ClusterType[]; + sessionCluster?: ClusterType[]; + clusterConfiguration?: ClusterConfigurationType[]; + database?: DataBaseType[]; + env?: EnvType[]; + currentSession?: SessionType; + current?: TabsItemType; + sql?: string; + monaco?: any; + currentPath?: string[]; + tabs?: TabsType; + session?: SessionType[]; + result?: {}; + rightClickMenu?: boolean; + refs?: { + history: any; + }; +}; + +export type ModelType = { + namespace: string; + state: StateType; + effects: { + saveTask: Effect; + }; + reducers: { + saveToolHeight: Reducer; + saveToolRightWidth: Reducer; + saveToolLeftWidth: Reducer; + saveSql: Reducer; + saveCurrentPath: Reducer; + saveMonaco: Reducer; + saveSqlMetaData: Reducer; + saveTabs: Reducer; + changeActiveKey: Reducer; + saveTaskData: Reducer; + saveSession: Reducer; + showRightClickMenu: Reducer; + refreshCurrentSession: Reducer; + quitCurrentSession: Reducer; + saveResult: Reducer; + saveCluster: Reducer; + saveSessionCluster: Reducer; + saveClusterConfiguration: Reducer; + saveDataBase: Reducer; + saveEnv: Reducer; + }; +}; + +const Model: ModelType = { + namespace: 'Studio', + state: { + toolHeight: 400, + toolRightWidth: 300, + toolLeftWidth: 300, + cluster: [], + sessionCluster: [], + clusterConfiguration: [], + database: [], + env: [], + currentSession: { + connectors: [], + }, + current: { + title: '草稿', + key: 0, + value: '', + closable: false, + path: ['草稿'], + isModified: false, + task: { + jobName: '草稿', + type: 'local', + checkPoint: 0, + savePointStrategy: 0, + savePointPath: '', + parallelism: 1, + fragment: true, + statementSet: false, + clusterId: 0, + clusterName: "本地环境", + clusterConfigurationId: undefined, + clusterConfigurationName: undefined, + databaseId: undefined, + databaseName: undefined, + jarId: undefined, + envId: undefined, + maxRowNum: 100, + config: [], + session: '', + alias: '草稿', + dialect: 'FlinkSql', + useResult: true, + useChangeLog: false, + useAutoCancel: false, + useSession: false, + useRemote: false, + }, + console: { + result: {}, + }, + monaco: {}, + sqlMetaData: undefined, + }, + sql: '', + monaco: {}, + currentPath: ['草稿'], + tabs: { + activeKey: 0, + panes: [{ + title: '草稿', + key: 0, + value: '', + closable: false, + isModified: false, + path: ['草稿'], + task: { + jobName: '草稿', + type: 'local', + checkPoint: 0, + savePointStrategy: 0, + savePointPath: '', + parallelism: 1, + fragment: true, + statementSet: false, + clusterId: 0, + clusterName: "本地环境", + clusterConfigurationId: undefined, + clusterConfigurationName: undefined, + databaseId: undefined, + databaseName: undefined, + jarId: undefined, + envId: undefined, + session: '', + config: [], + maxRowNum: 100, + alias: '草稿', + dialect: 'FlinkSql', + useResult: true, + useChangeLog: false, + useAutoCancel: false, + useSession: false, + useRemote: false, + }, + console: { + result: {}, + }, + monaco: {}, + sqlMetaData: undefined, + }], + }, + session: [], + result: {}, + rightClickMenu: false, + refs: { + history: {}, + } + }, + + effects: { + * saveTask({payload}, {call, put}) { + const para = payload; + para.configJson = JSON.stringify(payload.config); + yield call(handleAddOrUpdate, 'api/task', para); + yield put({ + type: 'saveTaskData', + payload, + }); + }, + }, + + reducers: { + saveToolHeight(state, {payload}) { + return { + ...state, + toolHeight: payload, + }; + }, saveToolRightWidth(state, {payload}) { + return { + ...state, + toolRightWidth: payload, + }; + }, saveToolLeftWidth(state, {payload}) { + return { + ...state, + toolLeftWidth: payload, + }; + }, + saveSql(state, {payload}) { + const {tabs} = state; + const newCurrent = state.current; + newCurrent.value = payload; + for (let i = 0; i < tabs.panes.length; i++) { + if (tabs.panes[i].key == tabs.activeKey) { + tabs.panes[i].value = payload; + tabs.panes[i].task && (tabs.panes[i].task.statement = payload); + } + } + return { + ...state, + current: { + ...newCurrent + }, + tabs: { + ...tabs + }, + }; + }, + saveCurrentPath(state, {payload}) { + return { + ...state, + currentPath: payload, + }; + }, + saveMonaco(state, {payload}) { + return { + ...state, + monaco: { + ...payload + }, + }; + }, + saveSqlMetaData(state, {payload}) { + const newCurrent = state.current; + const newTabs = state.tabs; + if (newCurrent.key == payload.activeKey) { + newCurrent.sqlMetaData = payload.sqlMetaData; + newCurrent.isModified = payload.isModified; + } + for (let i = 0; i < newTabs.panes.length; i++) { + if (newTabs.panes[i].key == payload.activeKey) { + newTabs.panes[i].sqlMetaData = payload.sqlMetaData; + newTabs.panes[i].isModified = payload.isModified; + break; + } + } + return { + ...state, + current: newCurrent, + tabs: newTabs, + }; + }, + saveTabs(state, {payload}) { + let newCurrent = state.current; + for (let i = 0; i < payload.panes.length; i++) { + if (payload.panes[i].key == payload.activeKey) { + newCurrent = payload.panes[i]; + } + } + return { + ...state, + current: { + ...newCurrent, + isModified: false, + }, + tabs: { + ...payload, + }, + }; + }, + deleteTabByKey(state, {payload}) { + const newTabs = state.tabs; + for (let i = 0; i < newTabs.panes.length; i++) { + if (newTabs.panes[i].key == payload) { + newTabs.panes.splice(i, 1); + break; + } + } + const newCurrent = newTabs.panes[newTabs.panes.length - 1]; + if (newTabs.activeKey == payload) { + newTabs.activeKey = newCurrent.key; + } + return { + ...state, + current: { + ...newCurrent, + }, + tabs: { + ...newTabs, + }, + }; + }, + closeTabs(state, {payload}) { + const {deleteType, current} = payload + const newTabs = state.tabs; + const firstKey = newTabs.panes[0].key + let newCurrent = newTabs.panes[0] + if (deleteType === 'CLOSE_OTHER') { + const keys = [firstKey, current.key]; + newCurrent = {...current} + newTabs.activeKey = current.key + newTabs.panes = newTabs.panes.filter(item => keys.includes(item.key)); + } else { + newTabs.panes = []; + newTabs.activeKey = firstKey + } + + return { + ...state, + current: { + ...newCurrent + }, + tabs: { + ...newTabs, + } + }; + }, + changeActiveKey(state, {payload}) { + const {tabs} = state; + tabs.activeKey = payload; + let newCurrent = state.current; + for (let i = 0; i < tabs.panes.length; i++) { + if (tabs.panes[i].key == tabs.activeKey) { + newCurrent = tabs.panes[i]; + } + } + return { + ...state, + current: { + ...newCurrent, + }, + tabs: { + ...tabs, + }, + currentPath: newCurrent.path, + }; + }, + saveTaskData(state, {payload}) { + const newTabs = state.tabs; + for (let i = 0; i < newTabs.panes.length; i++) { + if (newTabs.panes[i].key == newTabs.activeKey) { + newTabs.panes[i].task = payload; + } + } + return { + ...state, + tabs: { + ...newTabs, + }, + }; + }, + saveSession(state, {payload}) { + return { + ...state, + session: [...payload], + }; + }, + showRightClickMenu(state, {payload}) { + return { + ...state, + rightClickMenu: payload, + }; + }, + refreshCurrentSession(state, {payload}) { + return { + ...state, + currentSession: { + ...state?.currentSession, + ...payload + } + }; + }, + quitCurrentSession(state) { + return { + ...state, + currentSession: { + connectors: [], + } + }; + }, + saveResult(state, {payload}) { + return { + ...state, + result: { + ...payload + }, + }; + }, + saveCluster(state, {payload}) { + return { + ...state, + cluster: payload, + }; + }, saveSessionCluster(state, {payload}) { + return { + ...state, + sessionCluster: payload, + }; + }, saveClusterConfiguration(state, {payload}) { + return { + ...state, + clusterConfiguration: payload, + }; + }, saveDataBase(state, {payload}) { + return { + ...state, + database: payload, + }; + }, saveEnv(state, {payload}) { + return { + ...state, + env: payload, + }; + }, + }, +}; + +export default Model; From 9f3e81e274152198d1fd1fc1fa94c710a1d60559 Mon Sep 17 00:00:00 2001 From: godkaikai <32723967+godkaikai@users.noreply.github.com> Date: Fri, 14 Jan 2022 18:41:46 +0800 Subject: [PATCH 09/10] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=9B=BE=E8=A1=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dlink-web/package.json | 1 + .../Chart/LineChartSetting/index.less | 9 + .../Chart/LineChartSetting/index.tsx | 98 ++++ dlink-web/src/components/Chart/index.less | 9 + dlink-web/src/components/Chart/index.tsx | 110 ++++ .../components/Studio/StudioConsole/index.tsx | 16 +- .../components/Studio/StudioMenu/index.tsx | 21 +- dlink-web/src/components/Studio/conf.ts | 20 +- .../pages/FlinkSqlStudio/SqlExport/model.ts | 548 ------------------ dlink-web/src/pages/FlinkSqlStudio/model.ts | 26 + dlink-web/src/pages/Welcome.tsx | 6 + 11 files changed, 304 insertions(+), 560 deletions(-) create mode 100644 dlink-web/src/components/Chart/LineChartSetting/index.less create mode 100644 dlink-web/src/components/Chart/LineChartSetting/index.tsx create mode 100644 dlink-web/src/components/Chart/index.less create mode 100644 dlink-web/src/components/Chart/index.tsx delete mode 100644 dlink-web/src/pages/FlinkSqlStudio/SqlExport/model.ts diff --git a/dlink-web/package.json b/dlink-web/package.json index 7409cb24c5..eb1c42372d 100644 --- a/dlink-web/package.json +++ b/dlink-web/package.json @@ -48,6 +48,7 @@ "dependencies": { "@ant-design/charts": "^1.2.10", "@ant-design/icons": "^4.5.0", + "@ant-design/plots": "^1.0.7", "@ant-design/pro-descriptions": "^1.6.8", "@ant-design/pro-form": "^1.18.3", "@ant-design/pro-layout": "^6.18.0", diff --git a/dlink-web/src/components/Chart/LineChartSetting/index.less b/dlink-web/src/components/Chart/LineChartSetting/index.less new file mode 100644 index 0000000000..58a55f65b3 --- /dev/null +++ b/dlink-web/src/components/Chart/LineChartSetting/index.less @@ -0,0 +1,9 @@ +@import '~antd/es/style/themes/default.less'; + +.form_setting{ + padding-left: 10px; +} + +.form_item{ + margin-bottom: 5px; +} diff --git a/dlink-web/src/components/Chart/LineChartSetting/index.tsx b/dlink-web/src/components/Chart/LineChartSetting/index.tsx new file mode 100644 index 0000000000..879007e433 --- /dev/null +++ b/dlink-web/src/components/Chart/LineChartSetting/index.tsx @@ -0,0 +1,98 @@ +import {Button, Tag,Row, Col,Form,Select, Empty} from "antd"; +import {StateType} from "@/pages/FlinkSqlStudio/model"; +import {connect} from "umi"; +import styles from "./index.less"; +import {FireOutlined, SearchOutlined,RedoOutlined} from '@ant-design/icons'; +import {useState} from "react"; +import React from "react"; + +const {Option} = Select; + +export type LineChartConfig = { + xField: string, + yField: string, + seriesField?: string, + xAxis?: { + type?: string, + }, +}; + +export type LineChartProps = { + onChange: (values: Partial) => void; + data: []; + column: []; +}; + +const LineChartSetting: React.FC = (props) => { + + const {data,column,onChange: handleChange,dispatch} = props; + + const onValuesChange = (change: any, all: any) => { + handleChange(all); + }; + + const getColumnOptions = () => { + const itemList = []; + for (const item of column) { + itemList.push() + } + return itemList; + }; + + return ( + <> +
    + + + + {column&&column.length > 0 ? ( + ):()} + + + + + {column&&column.length > 0 ? ( + ):()} + + + + + + + {column&&column.length > 0 ? ( + ):()} + + + +
    + + ); +}; + +export default connect(({ Studio }: { Studio: StateType }) => ({ + current: Studio.current, + result: Studio.result, +}))(LineChartSetting); diff --git a/dlink-web/src/components/Chart/index.less b/dlink-web/src/components/Chart/index.less new file mode 100644 index 0000000000..58a55f65b3 --- /dev/null +++ b/dlink-web/src/components/Chart/index.less @@ -0,0 +1,9 @@ +@import '~antd/es/style/themes/default.less'; + +.form_setting{ + padding-left: 10px; +} + +.form_item{ + margin-bottom: 5px; +} diff --git a/dlink-web/src/components/Chart/index.tsx b/dlink-web/src/components/Chart/index.tsx new file mode 100644 index 0000000000..9632a50c73 --- /dev/null +++ b/dlink-web/src/components/Chart/index.tsx @@ -0,0 +1,110 @@ +import {Button, Tag,Row, Col,Form,Select, Empty} from "antd"; +import {StateType} from "@/pages/FlinkSqlStudio/model"; +import {connect} from "umi"; +import styles from "./index.less"; +import {FireOutlined, SearchOutlined,RedoOutlined} from '@ant-design/icons'; +import {CHART, isSql} from "@/components/Studio/conf"; +import { Line } from '@ant-design/plots'; +import {useEffect, useState} from "react"; +import LineChartSetting from "./LineChartSetting"; +import {showJobData} from "@/components/Studio/StudioEvent/DQL"; + + +const {Option} = Select; + +const Chart = (props:any) => { + + const {current,result,dispatch} = props; + const [config, setConfig] = useState({}); + const [data, setData] = useState([]); + const [column, setColumn] = useState([]); + const [type, setType] = useState(CHART.LINE); + + useEffect(() => { + toBuild(); + }, [result,current.console.result]); + + const toBuild = () => { + if(isSql(current.task.dialect)){ + setData(current.console.result.result.rowData); + setColumn(current.console.result.result.columns); + }else{ + setData(result.rowData); + setColumn(result.columns); + } + }; + + const toRebuild = () => { + if(!isSql(current.task.diagnosticCodesToIgnore)){ + showJobData(current.console.result.jobId,dispatch); + } + }; + + const onValuesChange = (change: any, all: any) => { + if(change.type){ + setType(change.type); + } + }; + + const renderChartSetting = () => { + switch (type){ + case CHART.LINE: + return { + setConfig(value); + }} />; + default: + return ; + } + }; + + const renderChartContent = () => { + if(column.length==0){ + return ; + } + switch (type){ + case CHART.LINE: + return ; + default: + return ; + } + }; + + return ( +
    + + + {renderChartContent()} + + +
    + + + + + + + + + + +
    + {renderChartSetting()} + +     +
    + ); +}; + +export default connect(({ Studio }: { Studio: StateType }) => ({ + current: Studio.current, + result: Studio.result, +}))(Chart); diff --git a/dlink-web/src/components/Studio/StudioConsole/index.tsx b/dlink-web/src/components/Studio/StudioConsole/index.tsx index 60c74aa603..51f60fa654 100644 --- a/dlink-web/src/components/Studio/StudioConsole/index.tsx +++ b/dlink-web/src/components/Studio/StudioConsole/index.tsx @@ -1,7 +1,7 @@ import {Tabs, Empty} from "antd"; import { CodeOutlined, TableOutlined, RadarChartOutlined, CalendarOutlined, FileSearchOutlined, DesktopOutlined - , FunctionOutlined, ApartmentOutlined + , FunctionOutlined, ApartmentOutlined,BarChartOutlined } from "@ant-design/icons"; import {StateType} from "@/pages/FlinkSqlStudio/model"; import {connect} from "umi"; @@ -13,6 +13,7 @@ import StudioFX from "./StudioFX"; import StudioCA from "./StudioCA"; import StudioProcess from "./StudioProcess"; import {Scrollbars} from 'react-custom-scrollbars'; +import Chart from "@/components/Chart"; const {TabPane} = Tabs; @@ -50,6 +51,19 @@ const StudioConsole = (props: any) => {     + + + 统计 + + } + key="StudioChart" + > + + + + diff --git a/dlink-web/src/components/Studio/StudioMenu/index.tsx b/dlink-web/src/components/Studio/StudioMenu/index.tsx index d94711444c..1b40e0e7be 100644 --- a/dlink-web/src/components/Studio/StudioMenu/index.tsx +++ b/dlink-web/src/components/Studio/StudioMenu/index.tsx @@ -11,14 +11,14 @@ import Button from "antd/es/button/button"; import Breadcrumb from "antd/es/breadcrumb/Breadcrumb"; import {StateType} from "@/pages/FlinkSqlStudio/model"; import {connect} from "umi"; -import {handleAddOrUpdate, postDataArray} from "@/components/Common/crud"; -import {executeSql, explainSql, getJobPlan} from "@/pages/FlinkSqlStudio/service"; +import { postDataArray} from "@/components/Common/crud"; +import {executeSql, getJobPlan} from "@/pages/FlinkSqlStudio/service"; import StudioHelp from "./StudioHelp"; import StudioGraph from "./StudioGraph"; import {showCluster, showTables, saveTask} from "@/components/Studio/StudioEvent/DDL"; import {useEffect, useState} from "react"; import StudioExplain from "../StudioConsole/StudioExplain"; -import {DIALECT, isSql} from "@/components/Studio/conf"; +import {DIALECT, isOnline, isSql} from "@/components/Studio/conf"; import { ModalForm, } from '@ant-design/pro-form'; @@ -37,10 +37,13 @@ const StudioMenu = (props: any) => { const [modalVisible, handleModalVisible] = useState(false); const [exportModalVisible, handleExportModalVisible] = useState(false); const [graphModalVisible, handleGraphModalVisible] = useState(false); - const [explainData, setExplainData] = useState([]); const [graphData, setGraphData] = useState(); const execute = () => { + if(!isSql(current.task.dialect)&&!isOnline(current.task.type)){ + message.warn(`该任务执行模式为【${current.task.type}】，不支持 SQL 查询，请手动保存后使用右侧按钮——作业提交`); + return; + } let selectsql = null; if (current.monaco.current) { let selection = current.monaco.current.editor.getSelection(); @@ -75,7 +78,7 @@ const StudioMenu = (props: any) => { if (res.datas.success) { message.success('执行成功'); } else { - message.success('执行失败'); + message.error('执行失败'); } let newTabs = tabs; for (let i = 0; i < newTabs.panes.length; i++) { @@ -230,8 +233,8 @@ const StudioMenu = (props: any) => { const runMenu = ( - 同步执行 - 异步提交 + SQL 查询 + 提交作业 ); @@ -348,7 +351,7 @@ const StudioMenu = (props: any) => { /> )} {(!current.task.dialect||current.task.dialect === DIALECT.FLINKSQL||isSql( current.task.dialect )) &&( - + - + :undefined} {renderChartSetting()} @@ -104,7 +125,14 @@ const Chart = (props:any) => { ); }; +const mapDispatchToProps = (dispatch: Dispatch)=>({ + saveChart:(chart: any)=>dispatch({ + type: "Studio/saveChart", + payload: chart, + }), +}) + export default connect(({ Studio }: { Studio: StateType }) => ({ current: Studio.current, result: Studio.result, -}))(Chart); +}),mapDispatchToProps)(Chart); diff --git a/dlink-web/src/components/Studio/StudioConsole/StudioHistory/index.tsx b/dlink-web/src/components/Studio/StudioConsole/StudioHistory/index.tsx index a19ba82c00..6824df16a9 100644 --- a/dlink-web/src/components/Studio/StudioConsole/StudioHistory/index.tsx +++ b/dlink-web/src/components/Studio/StudioConsole/StudioHistory/index.tsx @@ -12,6 +12,7 @@ import { import styles from "./index.less"; import {showJobData} from "@/components/Studio/StudioEvent/DQL"; import StudioPreview from "../StudioPreview"; +import {getJobData} from "@/pages/FlinkSqlStudio/service"; const { Title, Paragraph, Text, Link } = Typography; @@ -61,14 +62,19 @@ const StudioHistory = (props: any) => { const [row, setRow] = useState(); const [config,setConfig] = useState(); const [type,setType] = useState(); + const [result,setResult] = useState<{}>(); const showDetail=(row:HistoryItem,type:number)=>{ setRow(row); setModalVisit(true); setType(type); setConfig(JSON.parse(row.config)); - if(type==3){ - showJobData(row.jobId,dispatch) + if(type===3){ + // showJobData(row.jobId,dispatch) + const res = getJobData(row.jobId); + res.then((resd)=>{ + setResult(resd.datas); + }); } }; @@ -341,7 +347,7 @@ const StudioHistory = (props: any) => { - + )} diff --git a/dlink-web/src/components/Studio/StudioConsole/StudioPreview/index.tsx b/dlink-web/src/components/Studio/StudioConsole/StudioPreview/index.tsx index 7183d6a0e3..cd78bd5764 100644 --- a/dlink-web/src/components/Studio/StudioConsole/StudioPreview/index.tsx +++ b/dlink-web/src/components/Studio/StudioConsole/StudioPreview/index.tsx @@ -34,5 +34,5 @@ const StudioPreview = (props:any) => { export default connect(({ Studio }: { Studio: StateType }) => ({ current: Studio.current, - result: Studio.result, + // result: Studio.result, }))(StudioPreview); diff --git a/dlink-web/src/components/Studio/StudioConsole/StudioTable/index.tsx b/dlink-web/src/components/Studio/StudioConsole/StudioTable/index.tsx index b4ce0486cc..abb1f49110 100644 --- a/dlink-web/src/components/Studio/StudioConsole/StudioTable/index.tsx +++ b/dlink-web/src/components/Studio/StudioConsole/StudioTable/index.tsx @@ -32,8 +32,8 @@ const StudioTable = (props:any) => { {current.console.result.jobId && ( {current.console.result.jobId} )} - {result.columns? - + {current.console.result.result&¤t.console.result.result.columns? + :() } ) diff --git a/dlink-web/src/components/Studio/StudioConsole/index.tsx b/dlink-web/src/components/Studio/StudioConsole/index.tsx index 51f60fa654..a4d49afd96 100644 --- a/dlink-web/src/components/Studio/StudioConsole/index.tsx +++ b/dlink-web/src/components/Studio/StudioConsole/index.tsx @@ -55,13 +55,13 @@ const StudioConsole = (props: any) => { tab={ - 统计 + BI } key="StudioChart" > - + { dispatch&&dispatch({ diff --git a/dlink-web/src/components/Studio/StudioTree/index.tsx b/dlink-web/src/components/Studio/StudioTree/index.tsx index df7252ed9b..5f4b6ca732 100644 --- a/dlink-web/src/components/Studio/StudioTree/index.tsx +++ b/dlink-web/src/components/Studio/StudioTree/index.tsx @@ -161,7 +161,8 @@ const StudioTree: React.FC = (props) => { ...result.datas, }, console:{ - result:[], + result: {}, + chart: {}, }, monaco: React.createRef(), }; diff --git a/dlink-web/src/components/Studio/conf.ts b/dlink-web/src/components/Studio/conf.ts index 0e73a285d8..381820cda8 100644 --- a/dlink-web/src/components/Studio/conf.ts +++ b/dlink-web/src/components/Studio/conf.ts @@ -21,7 +21,9 @@ export const DIALECT = { }; export const CHART = { - LINE:'Line', + LINE:'折线图', + BAR:'条形图', + PIE:'饼图', }; export const isSql = (dialect: string)=>{ diff --git a/dlink-web/src/pages/FlinkSqlStudio/model.ts b/dlink-web/src/pages/FlinkSqlStudio/model.ts index 86b9d093c9..ea5635cd0a 100644 --- a/dlink-web/src/pages/FlinkSqlStudio/model.ts +++ b/dlink-web/src/pages/FlinkSqlStudio/model.ts @@ -96,6 +96,7 @@ export type TaskType = { export type ConsoleType = { result: {}; + chart: {}; } export type TabsItemType = { @@ -185,6 +186,7 @@ export type ModelType = { saveClusterConfiguration: Reducer; saveDataBase: Reducer; saveEnv: Reducer; + saveChart: Reducer; }; }; @@ -239,6 +241,7 @@ const Model: ModelType = { }, console: { result: {}, + chart: {}, }, monaco: {}, sqlMetaData: undefined, @@ -285,6 +288,7 @@ const Model: ModelType = { }, console: { result: {}, + chart: {}, }, monaco: {}, sqlMetaData: undefined, @@ -510,11 +514,25 @@ const Model: ModelType = { }; }, saveResult(state, {payload}) { + // return { + // ...state, + // result: { + // ...payload + // }, + // }; + let newTabs = state?.tabs; + let newCurrent = state?.current; + for (let i = 0; i < newTabs.panes.length; i++) { + if (newTabs.panes[i].key === newTabs.activeKey) { + newTabs.panes[i].console.result.result = payload; + newCurrent = newTabs.panes[i]; + break; + } + } return { ...state, - result: { - ...payload - }, + current: newCurrent, + tabs: newTabs, }; }, saveCluster(state, {payload}) { @@ -542,6 +560,21 @@ const Model: ModelType = { ...state, env: payload, }; + },saveChart(state, {payload}) { + let newTabs = state?.tabs; + let newCurrent = state?.current; + for (let i = 0; i < newTabs.panes.length; i++) { + if (newTabs.panes[i].key === newTabs.activeKey) { + newTabs.panes[i].console.chart = payload; + newCurrent = newTabs.panes[i]; + break; + } + } + return { + ...state, + current: newCurrent, + tabs: newTabs, + }; }, }, }; diff --git a/dlink-web/src/pages/Welcome.tsx b/dlink-web/src/pages/Welcome.tsx index 8f358c49ac..213053d7db 100644 --- a/dlink-web/src/pages/Welcome.tsx +++ b/dlink-web/src/pages/Welcome.tsx @@ -554,7 +554,7 @@ export default (): React.ReactNode => { 新增 编辑器选项卡右键关闭其他和关闭所有
  • - 新增 统计选项卡的图标支持 + 新增 BI选项卡的折线图、条形图、饼图
    • diff --git a/docs/en-US/guide/quickstart.md b/docs/en-US/guide/quickstart.md index 6b6ad0c7b8..2172b28fff 100644 --- a/docs/en-US/guide/quickstart.md +++ b/docs/en-US/guide/quickstart.md @@ -105,6 +105,9 @@ Dinky 通过已注册的集群配置来获取对应的 YarnClient 实例。对 | | | 新增 自动注册 Yarn 创建的集群 | 0.4.0 | | | SQL | 新增 外部数据源的 SQL 校验 | 0.5.0 | | | | 新增 外部数据源的 SQL 执行与预览 | 0.5.0 | +| | BI | 新增 折线图的渲染 | 0.5.0 | +| | | 新增 条形图图的渲染 | 0.5.0 | +| | | 新增 饼图的渲染 | 0.5.0 | | | 元数据 | 新增 查询外部数据源的元数据信息 | 0.4.0 | | | 归档 | 新增 执行与提交历史 | 0.4.0 | | 运维中心 | 暂无 | 暂无 | 0.4.0 | diff --git a/docs/guide/quickstart.md b/docs/guide/quickstart.md index 6b6ad0c7b8..2172b28fff 100644 --- a/docs/guide/quickstart.md +++ b/docs/guide/quickstart.md @@ -105,6 +105,9 @@ Dinky 通过已注册的集群配置来获取对应的 YarnClient 实例。对 | | | 新增 自动注册 Yarn 创建的集群 | 0.4.0 | | | SQL | 新增 外部数据源的 SQL 校验 | 0.5.0 | | | | 新增 外部数据源的 SQL 执行与预览 | 0.5.0 | +| | BI | 新增 折线图的渲染 | 0.5.0 | +| | | 新增 条形图图的渲染 | 0.5.0 | +| | | 新增 饼图的渲染 | 0.5.0 | | | 元数据 | 新增 查询外部数据源的元数据信息 | 0.4.0 | | | 归档 | 新增 执行与提交历史 | 0.4.0 | | 运维中心 | 暂无 | 暂无 | 0.4.0 |