[td] SQL block disable create table (mage-ai#2205)

* [WIP][td] SQL block disable create table

* fixed

* fix lint

* support other insert syntax

* add tests

* fixed sql quotes for redshift error

* fix lint

* alert users

mage_ai/data_preparation/models/block/__init__.py

@@ -50,6 +50,7 @@
 import json
 import os
 import pandas as pd
+import re
 import simplejson
 import sys
 import time
@@ -353,16 +354,27 @@ def full_table_name(self) -> str:
         if not self.content:
             return None
 
-        create_statement_partial, _ = extract_and_replace_text_between_strings(
+        statement_partial, _ = extract_and_replace_text_between_strings(
             self.content,
             'create',
             r'\(',
         )
 
-        if not create_statement_partial:
+        if not statement_partial:
+            matches = re.findall(
+                r'insert(?: overwrite)*(?: into)*[\s]+([\w.]+)',
+                self.content,
+                re.IGNORECASE,
+            )
+            if len(matches) >= 1:
+                return matches[len(matches) - 1]
+            else:
+                return None
+
+        if not statement_partial:
             return None
 
-        parts = create_statement_partial[:len(create_statement_partial) - 1].strip().split(' ')
+        parts = statement_partial[:len(statement_partial) - 1].strip().split(' ')
         return parts[-1]
 
     @classmethod

mage_ai/data_preparation/models/block/sql/__init__.py

@@ -18,7 +18,9 @@
 from os import path
 from time import sleep
 from typing import Any, Dict, List
+import re
 
+MAGE_SEMI_COLON = '__MAGE_SEMI_COLON__'
 PREVIEWABLE_BLOCK_TYPES = [
     BlockType.DATA_EXPORTER,
     BlockType.DATA_LOADER,
@@ -82,6 +84,7 @@ def execute_sql_code(
                 loader,
                 block,
                 query_string,
+                configuration=configuration,
                 should_query=should_query,
             )
         else:
@@ -132,6 +135,7 @@ def execute_sql_code(
                     loader,
                     block,
                     query_string,
+                    configuration=configuration,
                     should_query=should_query,
                 )
             else:
@@ -172,6 +176,7 @@ def execute_sql_code(
                     loader,
                     block,
                     query_string,
+                    configuration=configuration,
                     should_query=should_query,
                 )
             else:
@@ -209,6 +214,7 @@ def execute_sql_code(
                     loader,
                     block,
                     query_string,
+                    configuration=configuration,
                     should_query=should_query,
                 )
             else:
@@ -246,6 +252,7 @@ def execute_sql_code(
                     loader,
                     block,
                     query_string,
+                    configuration=configuration,
                     should_query=should_query,
                 )
             else:
@@ -287,6 +294,7 @@ def execute_sql_code(
                     loader,
                     block,
                     query_string,
+                    configuration=configuration,
                     should_query=should_query,
                 )
             else:
@@ -329,6 +337,7 @@ def execute_sql_code(
                     loader,
                     block,
                     query_string,
+                    configuration=configuration,
                     should_query=should_query,
                 )
             else:
@@ -354,34 +363,55 @@ def execute_sql_code(
                     ]
 
 
+def split_query_string(query_string: str) -> List[str]:
+    text_parts = []
+
+    matches = re.finditer(r"'(.*?)'|\"(.*?)\"", query_string, re.IGNORECASE)
+
+    previous_idx = 0
+
+    for idx, match in enumerate(matches):
+        matched_string = match.group()
+        updated_string = re.sub(r';', MAGE_SEMI_COLON, matched_string)
+
+        start_idx, end_idx = match.span()
+
+        previous_chunk = query_string[previous_idx:start_idx]
+        text_parts.append(previous_chunk)
+        text_parts.append(updated_string)
+        previous_idx = end_idx
+
+    text_parts.append(query_string[previous_idx:])
+
+    text_combined = ''.join(text_parts)
+    queries = text_combined.split(';')
+
+    arr = []
+    for query in queries:
+        query = query.strip()
+        if query:
+            lines = query.split('\n')
+            query = '\n'.join(list(filter(lambda x: not x.startswith('--'), lines)))
+            query = query.strip()
+            query = re.sub(MAGE_SEMI_COLON, ';', query)
+            arr.append(query)
+
+    return arr
+
+
 def execute_raw_sql(
     loader,
     block: 'Block',
     query_string: str,
+    configuration: Dict = {},
     should_query: bool = False,
 ) -> List[Any]:
     queries = []
     fetch_query_at_indexes = []
 
-    # create_statement, query_statement = extract_and_replace_text_between_strings(
-    #     query_string,
-    #     'create',
-    #     ';',
-    #     case_sensitive=True,
-    # )
-
-    # if create_statement:
-    #     queries.append(create_statement)
-    #     fetch_query_at_indexes.append(False)
-
-    # queries.append(query_statement)
-    # fetch_query_at_indexes.append(False)
-
-    for query in query_string.split(';'):
-        query = query.strip()
-        if query and not query.startswith('--'):
-            queries.append(query)
-            fetch_query_at_indexes.append(False)
+    for query in split_query_string(query_string):
+        queries.append(query)
+        fetch_query_at_indexes.append(False)
 
     if should_query:
         queries.append(f'SELECT * FROM {block.full_table_name} LIMIT 1000')

mage_ai/frontend/components/settings/Dashboard/constants.ts

@@ -31,7 +31,7 @@ export const SECTIONS = ({ owner, roles }: UserType) => {
   if (!REQUIRE_USER_AUTHENTICATION() || roles <= RoleValueEnum.EDITOR) {
     workspaceItems.push({
       linkProps: {
-        href: '/settings/workspace/sync_data',
+        href: '/settings/workspace/sync-data',
       },
       uuid: SECTION_ITEM_UUID_GIT_SETTINGS,
     });

mage_ai/frontend/interfaces/ChartBlockType.ts

@@ -1,6 +1,6 @@
 export const VARIABLE_NAME_BUCKETS = 'buckets';
 export const VARIABLE_NAME_CHART_STYLE = 'chart_style';
-export const VARIABLE_NAME_GROUP_BY = 'group_by'
+export const VARIABLE_NAME_GROUP_BY = 'group_by';
 export const VARIABLE_NAME_HEIGHT = 'height';
 export const VARIABLE_NAME_INDEX = 'index';
 export const VARIABLE_NAME_LEGEND_LABELS = 'legend_labels';
@@ -13,12 +13,13 @@ export const VARIABLE_NAME_Y = 'y';
 export const VARIABLE_NAME_Y_SORT_ORDER = 'y_sort_order';
 
 export const CONFIG_KEY_DATA_PROVIDER = 'data_provider';
-export const CONFIG_KEY_DATA_PROVIDER_DATABASE = 'data_provider_database'
+export const CONFIG_KEY_DATA_PROVIDER_DATABASE = 'data_provider_database';
 export const CONFIG_KEY_DATA_PROVIDER_PROFILE = 'data_provider_profile';
 export const CONFIG_KEY_DATA_PROVIDER_SCHEMA = 'data_provider_schema';
 export const CONFIG_KEY_DATA_PROVIDER_TABLE = 'data_provider_table';
 export const CONFIG_KEY_DBT_PROJECT_NAME = 'dbt_project_name';
 export const CONFIG_KEY_DBT_PROFILE_TARGET = 'dbt_profile_target';
+export const CONFIG_KEY_DISABLE_CREATE_TABLE = 'disable_create_table';
 export const CONFIG_KEY_EXPORT_WRITE_POLICY = 'export_write_policy';
 export const CONFIG_KEY_LIMIT = 'limit';
 export const CONFIG_KEY_USE_RAW_SQL = 'use_raw_sql';