[CAB-adapter] Include lineitem table refresh in SQL scripts (#350)

microsoft · Oct 29, 2024 · 5db67fc · 5db67fc
1 parent 8104c26
commit 5db67fc
Show file tree

Hide file tree

Showing 2 changed files with 36 additions and 2 deletions.
diff --git a/adapters/cab-converter/sql/spark-3.3.1/build/build.sql b/adapters/cab-converter/sql/spark-3.3.1/build/build.sql
@@ -46,12 +46,13 @@ CREATE
             l_shipmode STRING,
             l_comment STRING
         )
-        TBLPROPERTIES(
+        PARTITIONED BY(months(l_shipdate)) TBLPROPERTIES(
             'primaryKey' = 'l_orderkey,l_linenumber' ${tblproperties_suffix}
         );
 INSERT INTO ${catalog}.${database}${stream_num}.lineitem
 SELECT *
-FROM   ${external_catalog}.${external_database}${stream_num}.lineitem;
+FROM   ${external_catalog}.${external_database}${stream_num}.lineitem
+SORT BY l_shipdate;
 
 DROP
     TABLE

diff --git a/adapters/cab-converter/sql/spark-3.3.1/run/query_23.sql b/adapters/cab-converter/sql/spark-3.3.1/run/query_23.sql
@@ -21,3 +21,36 @@ FROM
     ${catalog}.${database}${stream_num}.orders
 WHERE
     ${param1} <= o_orderkey and o_orderkey < ${param2} and mod(o_orderkey, 32) between ${param3} and ${param4};
+
+INSERT
+    INTO
+        ${catalog}.${database}${stream_num}.lineitem (
+        SELECT
+            l_orderkey + 8,
+            l_partkey,
+            l_suppkey,
+            l_linenumber,
+            l_quantity,
+            l_extendedprice,
+            l_discount,
+            l_tax,
+            l_returnflag,
+            l_linestatus,
+            l_shipdate,
+            l_commitdate,
+            l_receiptdate,
+            l_shipinstruct,
+            l_shipmode,
+            l_comment
+        FROM
+            ${catalog}.${database}${stream_num}.lineitem
+        WHERE
+            ${param1} <= l_orderkey and l_orderkey < ${param2}
+        SORT BY
+            l_shipdate);
+
+DELETE
+FROM
+    ${catalog}.${database}${stream_num}.lineitem
+WHERE
+    ${param1} <= l_orderkey and l_orderkey < ${param2} and mod(l_orderkey, 32) between ${param3} and ${param4};