finished mssql data dictionary support (#1402)

* finished mssql data dictionary support * fixed test case not pass * 更新功能清单support表格 Co-authored-by: quanbisen <quanbisen@banggood.com>
hhyo · Mar 2, 2022 · e22d704 · e22d704
1 parent 8e2a196
commit e22d704
Show file tree

Hide file tree

Showing 10 changed files with 644 additions and 387 deletions.
diff --git a/README.md b/README.md
@@ -30,10 +30,10 @@ Archery是[archer](https://github.com/jly8866/archer)的分支项目，定位于
 |  | 查询 | 审核 | 执行 | 备份 | 数据字典 | 慢日志 | 会话管理 | 账号管理 | 参数管理 | 数据归档 |
 | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |
 | MySQL | √ | √ | √ | √ | √ | √ | √ | √ | √ | √ |
-| MsSQL | √ | × | √ | × | × | × | × | × | × | × |
+| MsSQL | √ | × | √ | × | √ | × | × | × | × | × |
 | Redis | √ | × | √ | × | × | × | × | × | × | × |
 | PgSQL | √ | × | √ | × | × | × | × | × | × | × |
-| Oracle | √ | √ | √ | √ | × | × | × | × | × | × |
+| Oracle | √ | √ | √ | √ | √ | × | × | × | × | × |
 | MongoDB | √ | √  | √  | × | × | × | × | × | × | × |
 | Phoenix | √ | ×  | √  | × | × | × | × | × | × | × |
 | ODPS | √ | ×  | ×  | × | × | × | × | × | × | × |

diff --git a/common/templates/dictionaryexport.html b/common/templates/dictionaryexport.html
@@ -0,0 +1,37 @@
+<html>
+    <meta charset="utf-8">
+    <title>数据库表结构说明文档</title>
+    <style>
+        body,td,th {font-family:"宋体"; font-size:12px;}
+        table,h1,p{width:960px;margin:0px auto;}
+        table{border-collapse:collapse;border:1px solid #CCC;background:#efefef;}
+        table caption{text-align:left; background-color:#fff; line-height:2em; font-size:14px; font-weight:bold; }
+        table th{text-align:left; font-weight:bold;height:26px; line-height:26px; font-size:12px; border:1px solid #CCC;padding-left:5px;}
+        table td{height:20px; font-size:12px; border:1px solid #CCC;background-color:#fff;padding-left:5px;}
+    </style>
+    {% load format_tags %}
+    <body>
+    <h1 style="text-align:center;">{{ db_name }} 数据字典 (共 {{ tables|length }} 个表)</h1>
+    <p style="text-align:center;margin:20px auto;">生成时间：{{ export_time }}</p>
+    {% for tb in tables %}
+        <table border="1" cellspacing="0" cellpadding="0" align="center">
+            <caption>表名：{{ tb.TABLE_INFO.TABLE_NAME }}</caption>
+            <caption>注释：{{ tb.TABLE_INFO.TABLE_COMMENT }}</caption>
+            <tbody>
+            <tr>
+                {% for key_dict in tb.ENGINE_KEYS %}
+                    <th>{{ key_dict.value }}</th>
+                {% endfor %}
+            </tr>
+            {% for col in tb.COLUMNS %}
+                <tr>
+                    {% for key_dict in tb.ENGINE_KEYS %}
+                        <td>{{ col|key_value:key_dict.key }}</td>
+                    {% endfor %}
+                </tr>
+            {% endfor %}
+            </tbody>
+        </table></br>
+    {% endfor %}
+    </body>
+</html>
diff --git a/sql/data_dictionary.py b/sql/data_dictionary.py
diff --git a/sql/engines/__init__.py b/sql/engines/__init__.py
@@ -101,6 +101,26 @@ def get_all_tables(self, db_name, **kwargs):
         """获取table 列表, 返回一个ResultSet，rows=list"""
         return ResultSet()
 
+    def get_group_tables_by_db(self, db_name, **kwargs):
+        """获取首字符分组的table列表，返回一个dict"""
+        return dict()
+
+    def get_table_meta_data(self, db_name, tb_name, **kwargs):
+        """获取表格元信息"""
+        return dict()
+
+    def get_table_desc_data(self, db_name, tb_name, **kwargs):
+        """获取表格字段信息"""
+        return dict()
+
+    def get_table_index_data(self, db_name, tb_name, **kwargs):
+        """获取表格索引信息"""
+        return dict()
+
+    def get_tables_metas_data(self, db_name, **kwargs):
+        """获取数据库所有表格信息，用作数据字典导出接口"""
+        return list()
+
     def get_all_columns_by_tb(self, db_name, tb_name, **kwargs):
         """获取所有字段, 返回一个ResultSet，rows=list"""
         return ResultSet()

diff --git a/sql/engines/mssql.py b/sql/engines/mssql.py
@@ -41,6 +41,149 @@ def get_all_tables(self, db_name, **kwargs):
         result.rows = tb_list
         return result
 
+    def get_group_tables_by_db(self, db_name):
+        """
+        根据传入的数据库名，获取该库下的表和注释，并按首字符分组，比如 'a': ['account1','apply']
+        :param db_name:
+        :return:
+        """
+        data = {}
+        sql = f"""
+        SELECT t.name AS table_name, 
+            case when td.value is not null then convert(varchar(max),td.value) else '' end AS table_comment
+        FROM    sysobjects t
+        LEFT OUTER JOIN sys.extended_properties td
+        ON      td.major_id = t.id
+        AND     td.minor_id = 0
+        AND     td.name = 'MS_Description'
+        WHERE t.type = 'u' ORDER BY t.name;"""
+        result = self.query(db_name=db_name, sql=sql)
+        for row in result.rows:
+            table_name, table_cmt = row[0], row[1]
+            if table_name[0] not in data:
+                data[table_name[0]] = list()
+            data[table_name[0]].append([table_name, table_cmt])
+        return data
+
+    def get_table_meta_data(self, db_name, tb_name, **kwargs):
+        """数据字典页面使用：获取表格的元信息，返回一个dict{column_list: [], rows: []}"""
+        sql = f"""
+            SELECT space.*,table_comment,index_length,IDENT_CURRENT('{tb_name}') as auto_increment
+            FROM (
+            SELECT 
+                t.NAME AS table_name,
+                t.create_date as create_time,
+                t.modify_date as update_time,
+                p.rows AS table_rows,
+                SUM(a.total_pages) * 8 AS data_total,
+                SUM(a.used_pages) * 8 AS data_length,
+                (SUM(a.total_pages) - SUM(a.used_pages)) * 8 AS data_free
+            FROM 
+                sys.tables t
+            INNER JOIN      
+                sys.indexes i ON t.OBJECT_ID = i.object_id
+            INNER JOIN 
+                sys.partitions p ON i.object_id = p.OBJECT_ID AND i.index_id = p.index_id
+            INNER JOIN 
+                sys.allocation_units a ON p.partition_id = a.container_id
+            WHERE 
+                t.NAME ='{tb_name}'
+                AND t.is_ms_shipped = 0
+                AND i.OBJECT_ID > 255 
+            GROUP BY 
+                t.Name, t.create_date, t.modify_date, p.Rows) 
+            AS space 
+            INNER JOIN (
+            SELECT      t.name AS table_name,
+                        convert(varchar(max),td.value) AS table_comment
+            FROM		sysobjects t
+            LEFT OUTER JOIN sys.extended_properties td
+                ON      td.major_id = t.id
+                AND     td.minor_id = 0
+                AND     td.name = 'MS_Description'
+            WHERE t.type = 'u' and t.name = '{tb_name}') AS comment 
+            ON space.table_name = comment.table_name
+            INNER JOIN (
+            SELECT
+                t.NAME				AS table_name,
+                SUM(page_count * 8) AS index_length
+            FROM sys.dm_db_index_physical_stats(
+                db_id(), object_id('{tb_name}'), NULL, NULL, 'DETAILED') AS s
+            JOIN sys.indexes AS i
+            ON s.[object_id] = i.[object_id] AND s.index_id = i.index_id
+            INNER JOIN      
+                sys.tables t ON t.OBJECT_ID = i.object_id
+            GROUP BY t.NAME
+            ) AS index_size 
+            ON index_size.table_name = space.table_name;
+        """
+        _meta_data = self.query(db_name, sql)
+        return {'column_list': _meta_data.column_list, 'rows': _meta_data.rows[0]}
+
+    def get_table_desc_data(self, db_name, tb_name, **kwargs):
+        """获取表格字段信息"""
+        sql = f"""
+            select COLUMN_NAME 列名, case when ISNUMERIC(CHARACTER_MAXIMUM_LENGTH)=1 
+then DATA_TYPE + '(' + convert(varchar(max), CHARACTER_MAXIMUM_LENGTH) + ')' else DATA_TYPE end 列类型,
+                COLLATION_NAME 列字符集,
+                IS_NULLABLE 是否为空,
+                COLUMN_DEFAULT 默认值
+            from INFORMATION_SCHEMA.columns where TABLE_CATALOG='{db_name}' and TABLE_NAME = '{tb_name}';"""
+        _desc_data = self.query(db_name, sql)
+        return {'column_list': _desc_data.column_list, 'rows': _desc_data.rows}
+
+    def get_table_index_data(self, db_name, tb_name, **kwargs):
+        """获取表格索引信息"""
+        sql = f"""SELECT 
+stuff((select ',' + COL_NAME(t.object_id,t.column_id) from sys.index_columns as t where i.object_id = t.object_id and 
+i.index_id = t.index_id and t.is_included_column = 0 order by key_ordinal for xml path('')),1,1,'') as 列名,
+                i.name AS 索引名,
+                is_unique as 唯一性,is_primary_key as 是否主建
+            FROM sys.indexes AS i  
+            WHERE i.object_id = OBJECT_ID('{tb_name}')
+            group by i.name,i.object_id,i.index_id,is_unique,is_primary_key;"""
+        _index_data = self.query(db_name, sql)
+        return {'column_list': _index_data.column_list, 'rows': _index_data.rows}
+
+    def get_tables_metas_data(self, db_name, **kwargs):
+        """获取数据库所有表格信息，用作数据字典导出接口"""
+        sql = """SELECT t.name AS TABLE_NAME, 
+            case when td.value is not null then convert(varchar(max),td.value) else '' end AS TABLE_COMMENT
+        FROM    sysobjects t
+        LEFT OUTER JOIN sys.extended_properties td
+        ON      td.major_id = t.id
+        AND     td.minor_id = 0
+        AND     td.name = 'MS_Description'
+        WHERE t.type = 'u' ORDER BY t.name;"""
+        result = self.query(db_name=db_name, sql=sql)
+        # query result to dict
+        tbs = []
+        for row in result.rows:
+            tbs.append(dict(zip(result.column_list, row)))
+        table_metas = []
+        for tb in tbs:
+            _meta = dict()
+            engine_keys = [{"key": "COLUMN_NAME", "value": "字段名"}, {"key": "COLUMN_TYPE", "value": "数据类型"},
+                           {"key": "COLLATION_NAME", "value": "列字符集"}, {"key": "IS_NULLABLE", "value": "允许非空"},
+                           {"key": "COLUMN_DEFAULT", "value": "默认值"}]
+            _meta["ENGINE_KEYS"] = engine_keys
+            _meta['TABLE_INFO'] = tb
+            sql_cols = f"""select COLUMN_NAME, case when ISNUMERIC(CHARACTER_MAXIMUM_LENGTH)=1 
+then DATA_TYPE + '(' + convert(varchar(max), CHARACTER_MAXIMUM_LENGTH) + ')' else DATA_TYPE end COLUMN_TYPE,
+                COLLATION_NAME,
+                IS_NULLABLE,
+                COLUMN_DEFAULT
+            from INFORMATION_SCHEMA.columns where TABLE_CATALOG='{db_name}' and TABLE_NAME = '{tb["TABLE_NAME"]}';"""
+            query_result = self.query(db_name=db_name, sql=sql_cols, close_conn=False)
+
+            columns = []
+            # 转换查询结果为dict
+            for row in query_result.rows:
+                columns.append(dict(zip(query_result.column_list, row)))
+            _meta['COLUMNS'] = tuple(columns)
+            table_metas.append(_meta)
+        return table_metas
+
     def get_all_columns_by_tb(self, db_name, tb_name, **kwargs):
         """获取所有字段, 返回一个ResultSet"""
         result = self.describe_table(db_name, tb_name)

diff --git a/sql/engines/mysql.py b/sql/engines/mysql.py
@@ -110,6 +110,112 @@ def get_all_tables(self, db_name, **kwargs):
         result.rows = tb_list
         return result
 
+    def get_group_tables_by_db(self, db_name):
+        # escape
+        db_name = MySQLdb.escape_string(db_name).decode('utf-8')
+        data = {}
+        sql = f"""SELECT TABLE_NAME,
+                            TABLE_COMMENT
+                        FROM
+                            information_schema.TABLES
+                        WHERE
+                            TABLE_SCHEMA='{db_name}';"""
+        result = self.query(db_name=db_name, sql=sql)
+        for row in result.rows:
+            table_name, table_cmt = row[0], row[1]
+            if table_name[0] not in data:
+                data[table_name[0]] = list()
+            data[table_name[0]].append([table_name, table_cmt])
+        return data
+
+    def get_table_meta_data(self, db_name, tb_name, **kwargs):
+        """数据字典页面使用：获取表格的元信息，返回一个dict{column_list: [], rows: []}"""
+        # escape
+        db_name = MySQLdb.escape_string(db_name).decode('utf-8')
+        tb_name = MySQLdb.escape_string(tb_name).decode('utf-8')
+        sql = f"""SELECT
+                        TABLE_NAME as table_name,
+                        ENGINE as engine,
+                        ROW_FORMAT as row_format,
+                        TABLE_ROWS as table_rows,
+                        AVG_ROW_LENGTH as avg_row_length,
+                        round(DATA_LENGTH/1024, 2) as data_length,
+                        MAX_DATA_LENGTH as max_data_length,
+                        round(INDEX_LENGTH/1024, 2) as index_length,
+                        round((DATA_LENGTH + INDEX_LENGTH)/1024, 2) as data_total,
+                        DATA_FREE as data_free,
+                        AUTO_INCREMENT as auto_increment,
+                        TABLE_COLLATION as table_collation,
+                        CREATE_TIME as create_time,
+                        CHECK_TIME as check_time,
+                        UPDATE_TIME as update_time,
+                        TABLE_COMMENT as table_comment
+                    FROM
+                        information_schema.TABLES
+                    WHERE
+                        TABLE_SCHEMA='{db_name}'
+                            AND TABLE_NAME='{tb_name}'"""
+        _meta_data = self.query(db_name, sql)
+        return {'column_list': _meta_data.column_list, 'rows': _meta_data.rows[0]}
+
+    def get_table_desc_data(self, db_name, tb_name, **kwargs):
+        """获取表格字段信息"""
+        sql = f"""SELECT 
+                        COLUMN_NAME as '列名',
+                        COLUMN_TYPE as '列类型',
+                        CHARACTER_SET_NAME as '列字符集',
+                        IS_NULLABLE as '是否为空',
+                        COLUMN_KEY as '索引列',
+                        COLUMN_DEFAULT as '默认值',
+                        EXTRA as '拓展信息',
+                        COLUMN_COMMENT as '列说明'
+                    FROM
+                        information_schema.COLUMNS
+                    WHERE
+                        TABLE_SCHEMA = '{db_name}'
+                            AND TABLE_NAME = '{tb_name}'
+                    ORDER BY ORDINAL_POSITION;"""
+        _desc_data = self.query(db_name, sql)
+        return {'column_list': _desc_data.column_list, 'rows': _desc_data.rows}
+
+    def get_table_index_data(self, db_name, tb_name, **kwargs):
+        """获取表格索引信息"""
+        sql = f"""SELECT
+                        COLUMN_NAME as '列名',
+                        INDEX_NAME as '索引名',
+                        NON_UNIQUE as '唯一性',
+                        SEQ_IN_INDEX as '列序列',
+                        CARDINALITY as '基数',
+                        NULLABLE as '是否为空',
+                        INDEX_TYPE as '索引类型',
+                        COMMENT as '备注'
+                    FROM
+                        information_schema.STATISTICS
+                    WHERE
+                        TABLE_SCHEMA = '{db_name}'
+                    AND TABLE_NAME = '{tb_name}';"""
+        _index_data = self.query(db_name, sql)
+        return {'column_list': _index_data.column_list, 'rows': _index_data.rows}
+
+    def get_tables_metas_data(self, db_name, **kwargs):
+        """获取数据库所有表格信息，用作数据字典导出接口"""
+        sql_tbs = f"SELECT * FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA='{db_name}';"
+        tbs = self.query(sql=sql_tbs, cursorclass=MySQLdb.cursors.DictCursor, close_conn=False).rows
+        table_metas = []
+        for tb in tbs:
+            _meta = dict()
+            engine_keys = [{"key": "COLUMN_NAME", "value": "字段名"}, {"key": "COLUMN_TYPE", "value": "数据类型"},
+                           {"key": "COLUMN_DEFAULT", "value": "默认值"}, {"key": "IS_NULLABLE", "value": "允许非空"},
+                           {"key": "EXTRA", "value": "自动递增"}, {"key": "COLUMN_KEY", "value": "是否主键"},
+                           {"key": "COLUMN_COMMENT", "value": "备注"}]
+            _meta["ENGINE_KEYS"] = engine_keys
+            _meta['TABLE_INFO'] = tb
+            sql_cols = f"""SELECT * FROM INFORMATION_SCHEMA.COLUMNS 
+                            WHERE TABLE_SCHEMA='{tb['TABLE_SCHEMA']}' AND TABLE_NAME='{tb['TABLE_NAME']}';"""
+            _meta['COLUMNS'] = self.query(sql=sql_cols, cursorclass=MySQLdb.cursors.DictCursor, close_conn=False).rows
+            table_metas.append(_meta)
+        return table_metas
+
     def get_all_columns_by_tb(self, db_name, tb_name, **kwargs):
         """获取所有字段, 返回一个ResultSet"""
         sql = f"""SELECT