[CAB-adapter] Include lineitem table refresh in SQL scripts

microsoft · Oct 21, 2024 · 76c0f30 · 76c0f30
1 parent 8104c26
commit 76c0f30
Show file tree

Hide file tree

Showing 2 changed files with 32 additions and 1 deletion.
diff --git a/adapters/cab-converter/sql/spark-3.3.1/build/build.sql b/adapters/cab-converter/sql/spark-3.3.1/build/build.sql
@@ -46,7 +46,7 @@ CREATE
             l_shipmode STRING,
             l_comment STRING
         )
-        TBLPROPERTIES(
+        PARTITIONED BY(months(l_shipdate)) TBLPROPERTIES(
             'primaryKey' = 'l_orderkey,l_linenumber' ${tblproperties_suffix}
         );
 INSERT INTO ${catalog}.${database}${stream_num}.lineitem

diff --git a/adapters/cab-converter/sql/spark-3.3.1/run/query_23.sql b/adapters/cab-converter/sql/spark-3.3.1/run/query_23.sql
@@ -21,3 +21,34 @@ FROM
     ${catalog}.${database}${stream_num}.orders
 WHERE
     ${param1} <= o_orderkey and o_orderkey < ${param2} and mod(o_orderkey, 32) between ${param3} and ${param4};
+
+INSERT
+    INTO
+        ${catalog}.${database}${stream_num}.lineitem (
+        SELECT
+            l_orderkey + 8,
+            l_partkey,
+            l_suppkey,
+            l_linenumber,
+            l_quantity,
+            l_extendedprice,
+            l_discount,
+            l_tax,
+            l_returnflag,
+            l_linestatus,
+            l_shipdate,
+            l_commitdate,
+            l_receiptdate,
+            l_shipinstruct,
+            l_shipmode,
+            l_comment
+        FROM
+            ${catalog}.${database}${stream_num}.lineitem
+        WHERE
+            ${param1} <= l_orderkey and l_orderkey < ${param2});
+
+DELETE
+FROM
+    ${catalog}.${database}${stream_num}.lineitem
+WHERE
+    ${param1} <= l_orderkey and l_orderkey < ${param2} and mod(l_orderkey, 32) between ${param3} and ${param4};