移除binlog2sql插件支持 (#1636)

移除binlog2sql

README.md

@@ -136,7 +136,6 @@ python manage.py test -v 3
 - MySQL审核/执行/备份 [goInception](https://github.com/hanchuanchuan/goInception)|[inception](https://github.com/hhyo/inception)
 - MySQL索引优化 [SQLAdvisor](https://github.com/Meituan-Dianping/SQLAdvisor)
 - SQL优化/压缩 [SOAR](https://github.com/XiaoMi/soar)
-- Binlog2SQL [binlog2sql](https://github.com/danfengcao/binlog2sql)
 - My2SQL [my2sql](https://github.com/liuhr/my2sql)
 - 表结构同步 [SchemaSync](https://github.com/hhyo/SchemaSync)
 - 慢日志解析展示 [pt-query-digest](https://www.percona.com/doc/percona-toolkit/3.0/pt-query-digest.html)|[aquila_v2](https://github.com/thinkdb/aquila_v2)

common/templates/base.html

@@ -200,11 +200,6 @@
                                         <a href="/archive/">PTArchiver</a>
                                     </li>
                                 {% endif %}
-                                {% if perms.sql.menu_binlog2sql %}
-                                    <li>
-                                        <a href="/binlog2sql/">Binlog2SQL</a>
-                                    </li>
-                                {% endif %}
                                 {% if perms.sql.menu_my2sql %}
                                     <li>
                                         <a href="/my2sql/">My2SQL</a>

common/templates/config.html

@@ -850,17 +850,6 @@ <h4 style="color: darkgrey"><b>其他配置</b></h4>
                                            placeholder="系统首页路径，默认SQL工单页面：sqlworkflow">
                                 </div>
                             </div>
-                            <div class="form-group">
-                                <label for="binlog2sql"
-                                       class="col-sm-4 control-label">BINLOG2SQL</label>
-                                <div class="col-sm-5">
-                                    <input type="text" class="form-control"
-                                           id="binlog2sql"
-                                           key="binlog2sql"
-                                           value="{{ config.binlog2sql }}"
-                                           placeholder="binlog2sql调用路径，类似/opt/binlog2sql.py">
-                                </div>
-                            </div>
                             <div class="form-group">
                                 <label for="my2sql"
                                        class="col-sm-4 control-label">MY2SQL</label>

sql/binlog.py

@@ -15,15 +15,14 @@
 from common.utils.extend_json_encoder import ExtendJSONEncoder
 from sql.engines import get_engine
 
-from sql.plugins.binglog2sql import Binlog2Sql
 from sql.plugins.my2sql import My2SQL
-from sql.notify import notify_for_binlog2sql, notify_for_my2sql
+from sql.notify import notify_for_my2sql
 from .models import Instance
 
 logger = logging.getLogger('default')
 
 
-@permission_required('sql.menu_binlog2sql', raise_exception=True)
+@permission_required('sql.menu_my2sql', raise_exception=True)
 def binlog_list(request):
     """
     获取binlog列表
@@ -81,144 +80,6 @@ def del_binlog(request):
                         content_type='application/json')
 
 
-@permission_required('sql.menu_binlog2sql', raise_exception=True)
-def binlog2sql(request):
-    """
-    通过解析binlog获取SQL
-    :param request:
-    :return:
-    """
-    instance_name = request.POST.get('instance_name')
-    save_sql = True if request.POST.get('save_sql') == 'true' else False
-    instance = Instance.objects.get(instance_name=instance_name)
-    no_pk = True if request.POST.get('no_pk') == 'true' else False
-    flashback = True if request.POST.get('flashback') == 'true' else False
-    back_interval = 0 if request.POST.get('back_interval') == '' else int(request.POST.get('back_interval'))
-    num = 30 if request.POST.get('num') == '' else int(request.POST.get('num'))
-    start_file = request.POST.get('start_file')
-    start_pos = request.POST.get('start_pos') if request.POST.get('start_pos') == '' else int(
-        request.POST.get('start_pos'))
-    end_file = request.POST.get('end_file')
-    end_pos = request.POST.get('end_pos') if request.POST.get('end_pos') == '' else int(request.POST.get('end_pos'))
-    stop_time = request.POST.get('stop_time')
-    start_time = request.POST.get('start_time')
-    only_schemas = request.POST.getlist('only_schemas')
-    only_tables = request.POST.getlist('only_tables[]')
-    only_dml = True if request.POST.get('only_dml') == 'true' else False
-    sql_type = ['INSERT', 'UPDATE', 'DELETE'] if request.POST.getlist('sql_type[]') == [] else request.POST.getlist(
-        'sql_type[]')
-    # 校验sql_type
-    if [i for i in sql_type if i not in ['INSERT', 'UPDATE', 'DELETE']]:
-        return JsonResponse({'status': 1, 'msg': '类型过滤参数不正确', 'data': {}})
-
-    # flashback=True获取DML回滚语句
-    result = {'status': 0, 'msg': 'ok', 'data': ''}
-
-    # 提交给binlog2sql进行解析
-    binlog2sql = Binlog2Sql()
-    # 准备参数
-    instance_password = shlex.quote(f'"{str(instance.password)}"')
-    args = {"conn_options": fr"-h{shlex.quote(str(instance.host))} -u{shlex.quote(str(instance.user))} \
-                -p'{instance_password}' -P{shlex.quote(str(instance.port))} ",
-            "stop_never": False,
-            "no-primary-key": no_pk,
-            "flashback": flashback,
-            "back-interval": back_interval,
-            "start-file": start_file,
-            "start-position": start_pos,
-            "stop-file": end_file,
-            "stop-position": end_pos,
-            "start-datetime": '"'+start_time+'"',
-            "stop-datetime": '"'+stop_time+'"',
-            "databases": ' '.join(only_schemas),
-            "tables": ' '.join(only_tables),
-            "only-dml": only_dml,
-            "sql-type": ' '.join(sql_type),
-            "instance": instance
-            }
-
-    # 参数检查
-    args_check_result = binlog2sql.check_args(args)
-    if args_check_result['status'] == 1:
-        return HttpResponse(json.dumps(args_check_result), content_type='application/json')
-    # 参数转换
-    cmd_args = binlog2sql.generate_args2cmd(args, shell=True)
-    # 执行命令
-    try:
-        p = binlog2sql.execute_cmd(cmd_args, shell=True)
-        # 读取前num行后结束
-        rows = []
-        n = 1
-        for line in iter(p.stdout.readline, ''):
-            if n <= num:
-                n = n + 1
-                row_info = {}
-                try:
-                    row_info['sql'] = line.split('; #')[0] + ";"
-                    row_info['binlog_info'] = line.split('; #')[1].rstrip('\"')
-                except IndexError:
-                    row_info['sql'] = line
-                    row_info['binlog_info'] = None
-                rows.append(row_info)
-            else:
-                break
-        if rows.__len__() == 0:
-            # 判断是否有异常
-            stderr = p.stderr.read()
-            if stderr:
-                result['status'] = 1
-                result['msg'] = stderr
-                return HttpResponse(json.dumps(result), content_type='application/json')
-        # 终止子进程
-        p.kill()
-        result['data'] = rows
-    except Exception as e:
-        logger.error(traceback.format_exc())
-        result['status'] = 1
-        result['msg'] = str(e)
-
-    # 异步保存到文件
-    if save_sql:
-        args.pop('conn_options')
-        async_task(binlog2sql_file, args=args, user=request.user, hook=notify_for_binlog2sql, timeout=-1,
-                   task_name=f'binlog2sql-{time.time()}')
-
-    # 返回查询结果
-    return HttpResponse(json.dumps(result, cls=ExtendJSONEncoder, bigint_as_string=True),
-                        content_type='application/json')
-
-
-def binlog2sql_file(args, user):
-    """
-    用于异步保存binlog解析的文件
-    :param args: 参数
-    :param user: 操作用户对象，用户消息推送
-    :return:
-    """
-    binlog2sql = Binlog2Sql()
-    instance = args.get('instance')
-    instance_password = shlex.quote(f'"{str(instance.password)}"')
-    conn_options = fr"-h{shlex.quote(str(instance.host))} -u{shlex.quote(str(instance.user))} \
-        -p'{instance_password}' -P{shlex.quote(str(instance.port))}"
-    args['conn_options'] = conn_options
-    timestamp = int(time.time())
-    path = os.path.join(settings.BASE_DIR, 'downloads/binlog2sql/')
-    os.makedirs(path, exist_ok=True)
-    if args.get('flashback'):
-        filename = os.path.join(path, f"flashback_{instance.host}_{instance.port}_{timestamp}.sql")
-    else:
-        filename = os.path.join(path, f"{instance.host}_{instance.port}_{timestamp}.sql")
-
-    # 参数转换
-    cmd_args = binlog2sql.generate_args2cmd(args, shell=True)
-    # 执行命令保存到文件
-    with open(filename, 'w') as f:
-        p = binlog2sql.execute_cmd(cmd_args, shell=True)
-        for c in iter(p.stdout.readline, ''):
-            f.write(c)
-    return user, filename
-
-
 @permission_required('sql.menu_my2sql', raise_exception=True)
 def my2sql(request):
     """

sql/fixtures/auth_group.sql

@@ -23,7 +23,7 @@ where codename in ('menu_dashboard','menu_sqlcheck','menu_sqlworkflow','menu_sql
 insert into auth_group_permissions (group_id, permission_id)
 select 3,id
 from auth_permission
-where codename in ('menu_dashboard','menu_sqlcheck','menu_sqlworkflow','menu_sqlanalyze','menu_query','menu_sqlquery','menu_queryapplylist','menu_sqloptimize','menu_sqladvisor','menu_slowquery','menu_instance','menu_instance_list','menu_dbdiagnostic','menu_database','menu_instance_account','menu_param','menu_data_dictionary','menu_tools','menu_archive','menu_binlog2sql','menu_my2sql','menu_schemasync','menu_system','menu_document','menu_openapi','sql_submit','sql_review','sql_execute_for_resource_group','sql_execute','sql_analyze','optimize_sqladvisor','optimize_sqltuning','optimize_soar','query_applypriv','query_mgtpriv','query_review','query_submit','query_all_instances','query_resource_group_instance','process_view','process_kill','tablespace_view','trx_view','trxandlocks_view','instance_account_manage','param_view','param_edit','data_dictionary_export','archive_apply','archive_review','archive_mgt');
+where codename in ('menu_dashboard','menu_sqlcheck','menu_sqlworkflow','menu_sqlanalyze','menu_query','menu_sqlquery','menu_queryapplylist','menu_sqloptimize','menu_sqladvisor','menu_slowquery','menu_instance','menu_instance_list','menu_dbdiagnostic','menu_database','menu_instance_account','menu_param','menu_data_dictionary','menu_tools','menu_archive','menu_my2sql','menu_schemasync','menu_system','menu_document','menu_openapi','sql_submit','sql_review','sql_execute_for_resource_group','sql_execute','sql_analyze','optimize_sqladvisor','optimize_sqltuning','optimize_soar','query_applypriv','query_mgtpriv','query_review','query_submit','query_all_instances','query_resource_group_instance','process_view','process_kill','tablespace_view','trx_view','trxandlocks_view','instance_account_manage','param_view','param_edit','data_dictionary_export','archive_apply','archive_review','archive_mgt');
 
 -- PM
 insert into auth_group_permissions (group_id, permission_id)

sql/models.py

@@ -726,7 +726,6 @@ class Meta:
             ('menu_data_dictionary', '菜单 数据字典'),
             ('menu_tools', '菜单 工具插件'),
             ('menu_archive', '菜单 数据归档'),
-            ('menu_binlog2sql', '菜单 Binlog2SQL'),
             ('menu_my2sql', '菜单 My2SQL'),
             ('menu_schemasync', '菜单 SchemaSync'),
             ('menu_system', '菜单 系统管理'),

sql/notify.py

@@ -275,26 +275,6 @@ def notify_for_execute(workflow):
             __send(msg_title, msg_content, msg_to, msg_cc)
 
 
-def notify_for_binlog2sql(task):
-    """
-    binlog2sql执行结束的通知
-    :param task:
-    :return:
-    """
-    # 判断是否开启消息通知，未开启直接返回
-    if not __notify_cnf_status():
-        return None
-    if task.success:
-        msg_title = '[Archery 通知]Binlog2SQL执行结束'
-        msg_content = f'解析的SQL文件为{task.result[1]}，请到指定目录查看'
-    else:
-        msg_title = '[Archery 通知]Binlog2SQL执行失败'
-        msg_content = f'{task.result}'
-    # 发送
-    msg_to = [task.kwargs['user']]
-    __send(msg_title, msg_content, msg_to)
-
-
 def notify_for_my2sql(task):
     """
     my2sql执行结束的通知

sql/plugins/tests.py

@@ -10,7 +10,6 @@
 from unittest.mock import patch, ANY
 from django.contrib.auth import get_user_model
 
-from sql.plugins.binglog2sql import Binlog2Sql
 from sql.plugins.my2sql import My2SQL
 from sql.plugins.schemasync import SchemaSync
 from sql.plugins.soar import Soar
@@ -172,34 +171,6 @@ def test_schema_sync_generate_args2cmd(self):
         cmd_args = schema_sync.generate_args2cmd(args, True)
         self.assertIsInstance(cmd_args, str)
 
-    def test_binlog2ql_generate_args2cmd(self):
-        """
-        测试binlog2sql参数转换
-        :return:
-        """
-        args = {'conn_options': "-hmysql -uroot -p'123456' -P3306 ",
-                'stop_never': False,
-                'no-primary-key': False,
-                'flashback': True,
-                'back-interval': 0,
-                'start-file': 'mysql-bin.000043',
-                'start-position': 111,
-                'stop-file': '',
-                'stop-position': '',
-                'start-datetime': '',
-                'stop-datetime': '',
-                'databases': 'account_center',
-                'tables': 'ac_apps',
-                'only-dml': True,
-                'sql-type': 'UPDATE'}
-        self.sys_config.set('binlog2sql', '/opt/archery/src/plugins/binlog2sql/binlog2sql.py')
-        self.sys_config.get_all_config()
-        binlog2sql = Binlog2Sql()
-        cmd_args = binlog2sql.generate_args2cmd(args, False)
-        self.assertIsInstance(cmd_args, list)
-        cmd_args = binlog2sql.generate_args2cmd(args, True)
-        self.assertIsInstance(cmd_args, str)
-
     def test_my2sql_generate_args2cmd(self):
         """
         测试my2sql参数转换