Skip to content

Commit 3a81f48

Browse files
lmanganilmangani
authored
Extend Function Macros (#6)
* Adding tupleDivide MACRO * Add tuple* to examples * Add more Tuple* MACROS * Add more Tuple* MACROS * Create extract_macros.yml * Arithmetic macros * Delete aliases.sql * Fix plus, minus * Test table macro numbers() * numbers() start from 0 * Add generateUUIDv4() * Add url, date, IP macros * Remove dupes * Update chsql_extension.cpp * Extend test suite * Remove duplicate hex function * Remove duplicate unhex * Fix modulo functions * Fix protocol regex * Fix regex url extractions
1 parent 91a397f commit 3a81f48

File tree

4 files changed

+271
-51
lines changed

4 files changed

+271
-51
lines changed

‎.github/workflows/extract_macros.yml‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
name: Extract Macros
2+
3+
on:
4+
release:
5+
published:
6+
7+
jobs:
8+
bench:
9+
runs-on: ubuntu-latest
10+
steps:
11+
- uses: actions/checkout@v4
12+
- name: Extract Macros
13+
run: |
14+
grep -oP '{DEFAULT_SCHEMA, "\K[^"]*|, R"\K[^"]*' ./src/chsql_extension.cpp | awk 'NR%2{printf "%s ", $0; next} 1' | grep -v ^"times_" > /tmp/macros.md
15+
- name: Summary Report
16+
run: |
17+
echo "" >> $GITHUB_STEP_SUMMARY
18+
echo "### Supported Macros" >> $GITHUB_STEP_SUMMARY
19+
cat /tmp/macros.md >> $GITHUB_STEP_SUMMARY
20+
- name: Add Test Report to Release
21+
uses: irongut/EditRelease@v1.2.0
22+
with:
23+
token: ${{ secrets.GITHUB_TOKEN }}
24+
id: ${{ github.event.release.id }}
25+
body: $GITHUB_STEP_SUMMARY
26+
files: "/tmp/macros.md"

‎aliases.sql‎

Lines changed: 0 additions & 47 deletions
This file was deleted.

‎src/chsql_extension.cpp‎

Lines changed: 54 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -27,9 +27,7 @@ namespace duckdb {
2727
// If you do not have parameters, simplify to {nullptr}
2828
// Add the text of your SQL macro as a raw string with the format R"( select 42 )"
2929

30-
3130
static DefaultMacro chsql_macros[] = {
32-
{DEFAULT_SCHEMA, "times_two", {"x", nullptr}, R"(x*2)"},
3331
// -- Type conversion macros
3432
{DEFAULT_SCHEMA, "toString", {"x", nullptr}, R"(CAST(x AS VARCHAR))"},
3533
{DEFAULT_SCHEMA, "toInt8", {"x", nullptr}, R"(CAST(x AS INT8))"},
@@ -69,11 +67,63 @@ static DefaultMacro chsql_macros[] = {
6967
{DEFAULT_SCHEMA, "toFloatOrZero", {"x", nullptr}, R"(CASE WHEN TRY_CAST(x AS DOUBLE) IS NOT NULL THEN CAST(x AS DOUBLE) ELSE 0 END)"},
7068
// -- Arithmetic macros
7169
{DEFAULT_SCHEMA, "intDiv", {"a", "b"}, R"((CAST(a AS BIGINT) // CAST(b AS BIGINT)))"},
70+
{DEFAULT_SCHEMA, "intDivOrNull", {"a", "b"}, R"(TRY_CAST((TRY_CAST(a AS BIGINT) // TRY_CAST(b AS BIGINT)) AS BIGINT))"},
71+
{DEFAULT_SCHEMA, "intDivOZero", {"x", nullptr}, R"(COALESCE((TRY_CAST((TRY_CAST(a AS BIGINT) // TRY_CAST(b AS BIGINT)) AS BIGINT)),0))"},
72+
{DEFAULT_SCHEMA, "plus", {"a", "b"}, R"(add(a, b))"},
73+
{DEFAULT_SCHEMA, "minus", {"a", "b"}, R"(subtract(a, b))"},
74+
{DEFAULT_SCHEMA, "modulo", {"a", "b"}, R"(CAST(a AS BIGINT) % CAST(b AS BIGINT))"},
75+
{DEFAULT_SCHEMA, "moduloOrZero", {"a", "b"}, R"(COALESCE(((TRY_CAST(a AS BIGINT) % TRY_CAST(b AS BIGINT))),0))"},
76+
// -- Tuple macros
77+
{DEFAULT_SCHEMA, "tupleIntDiv", {"a", "b"}, R"(apply(a, (x,i) -> apply(b, x -> CAST(x AS BIGINT))[i] // CAST(x AS BIGINT)))"},
78+
{DEFAULT_SCHEMA, "tupleIntDivByNumber", {"a", "b"}, R"(apply(a, (x) -> CAST(apply(b, x -> CAST(x AS BIGINT))[1] as BIGINT) // CAST(x AS BIGINT)))"},
79+
{DEFAULT_SCHEMA, "tupleDivide", {"a", "b"}, R"(apply(a, (x,i) -> apply(b, x -> CAST(x AS BIGINT))[i] / CAST(x AS BIGINT)))"},
80+
{DEFAULT_SCHEMA, "tupleMultiply", {"a", "b"}, R"(apply(a, (x,i) -> CAST(apply(b, x -> CAST(x AS BIGINT))[i] as BIGINT) * CAST(x AS BIGINT)))"},
81+
{DEFAULT_SCHEMA, "tupleMinus", {"a", "b"}, R"(apply(a, (x,i) -> apply(b, x -> CAST(x AS BIGINT))[i] - CAST(x AS BIGINT)))"},
82+
{DEFAULT_SCHEMA, "tuplePlus", {"a", "b"}, R"(apply(a, (x,i) -> apply(b, x -> CAST(x AS BIGINT))[i] + CAST(x AS BIGINT)))"},
83+
{DEFAULT_SCHEMA, "tupleMultiplyByNumber", {"a", "b"}, R"(apply(a, (x) -> CAST(apply(b, x -> CAST(x AS BIGINT))[1] as BIGINT) * CAST(x AS BIGINT)))"},
84+
{DEFAULT_SCHEMA, "tupleDivideByNumber", {"a", "b"}, R"(apply(a, (x) -> CAST(apply(b, x -> CAST(x AS BIGINT))[1] as BIGINT) / CAST(x AS BIGINT)))"},
85+
{DEFAULT_SCHEMA, "tupleModulo", {"a", "b"}, R"(apply(a, (x) -> CAST(apply(b, x -> CAST(x AS BIGINT))[1] as BIGINT) % CAST(x AS BIGINT)))"},
86+
{DEFAULT_SCHEMA, "tupleModuloByNumber", {"a", "b"}, R"(apply(a, (x) -> CAST(apply(b, x -> CAST(x AS BIGINT))[1] as BIGINT) % CAST(x AS BIGINT)))"},
87+
{DEFAULT_SCHEMA, "tupleConcat", {"a", "b"}, R"(list_concat(a, b))"},
7288
// -- String matching macros
7389
{DEFAULT_SCHEMA, "match", {"string", "token"}, R"(string LIKE token)"},
7490
// -- Array macros
7591
{DEFAULT_SCHEMA, "arrayExists", {"needle", "haystack"}, R"(haystack @> ARRAY[needle])"},
7692
{DEFAULT_SCHEMA, "arrayMap", {"e", "arr"}, R"(array_transform(arr, e -> (e * e)))"},
93+
// Date and Time Functions
94+
{DEFAULT_SCHEMA, "toYear", {"date_expression", nullptr}, R"(EXTRACT(YEAR FROM date_expression))"},
95+
{DEFAULT_SCHEMA, "toMonth", {"date_expression", nullptr}, R"(EXTRACT(MONTH FROM date_expression))"},
96+
{DEFAULT_SCHEMA, "toDayOfMonth", {"date_expression", nullptr}, R"(EXTRACT(DAY FROM date_expression))"},
97+
{DEFAULT_SCHEMA, "toHour", {"date_expression", nullptr}, R"(EXTRACT(HOUR FROM date_expression))"},
98+
{DEFAULT_SCHEMA, "toMinute", {"date_expression", nullptr}, R"(EXTRACT(MINUTE FROM date_expression))"},
99+
{DEFAULT_SCHEMA, "toSecond", {"date_expression", nullptr}, R"(EXTRACT(SECOND FROM date_expression))"},
100+
{DEFAULT_SCHEMA, "toYYYYMM", {"date_expression", nullptr}, R"(DATE_FORMAT(date_expression, '%Y%m'))"},
101+
{DEFAULT_SCHEMA, "toYYYYMMDD", {"date_expression", nullptr}, R"(DATE_FORMAT(date_expression, '%Y%m%d'))"},
102+
{DEFAULT_SCHEMA, "toYYYYMMDDhhmmss", {"date_expression", nullptr}, R"(DATE_FORMAT(date_expression, '%Y%m%d%H%M%S'))"},
103+
{DEFAULT_SCHEMA, "formatDateTime", {"time", "format", "timezone", nullptr}, R"(CASE WHEN timezone IS NULL THEN strftime(time, format) ELSE strftime(time AT TIME ZONE timezone, format) END)"},
104+
// String Functions
105+
{DEFAULT_SCHEMA, "empty", {"str", nullptr}, R"(LENGTH(str) = 0)"},
106+
{DEFAULT_SCHEMA, "notEmpty", {"str", nullptr}, R"(LENGTH(str) > 0)"},
107+
{DEFAULT_SCHEMA, "lengthUTF8", {"str", nullptr}, R"(LENGTH(str))"},
108+
{DEFAULT_SCHEMA, "leftPad", {"str", "length", "pad_str", nullptr}, R"(LPAD(str, length, pad_str))"},
109+
{DEFAULT_SCHEMA, "rightPad", {"str", "length", "pad_str", nullptr}, R"(RPAD(str, length, pad_str))"},
110+
{DEFAULT_SCHEMA, "extractAllGroups", {"text", "pattern", nullptr}, R"(regexp_extract_all(text, pattern))"},
111+
{DEFAULT_SCHEMA, "toFixedString", {"str", "length", nullptr}, R"(RPAD(LEFT(str, length), length, '\0'))"},
112+
{DEFAULT_SCHEMA, "ifNull", {"x", "y", nullptr}, R"(COALESCE(x, y))"},
113+
{DEFAULT_SCHEMA, "arrayJoin", {"arr", nullptr}, R"(UNNEST(arr))"},
114+
{DEFAULT_SCHEMA, "splitByChar", {"separator", "str", nullptr}, R"(string_split(str, separator))"},
115+
// URL Functions
116+
{DEFAULT_SCHEMA, "protocol", {"url", nullptr}, R"(REGEXP_EXTRACT(url, '^(\w+)://', 1))"},
117+
{DEFAULT_SCHEMA, "domain", {"url", nullptr}, R"(REGEXP_EXTRACT(url, '://([^/]+)', 1))"},
118+
{DEFAULT_SCHEMA, "topLevelDomain", {"url", nullptr}, R"(REGEXP_EXTRACT(url, '\.([^./:]+)([:/]|$)', 1))"},
119+
{DEFAULT_SCHEMA, "path", {"url", nullptr}, R"(REGEXP_EXTRACT(url, '://[^/]+(/.*)', 1))"},
120+
// IP Address Functions
121+
{DEFAULT_SCHEMA, "IPv4NumToString", {"num", nullptr}, R"(CONCAT(CAST((num >> 24) & 255 AS VARCHAR), '.', CAST((num >> 16) & 255 AS VARCHAR), '.', CAST((num >> 8) & 255 AS VARCHAR), '.', CAST(num & 255 AS VARCHAR)))"},
122+
{DEFAULT_SCHEMA, "IPv4StringToNum", {"ip", nullptr}, R"(CAST(SPLIT_PART(ip, '.', 1) AS INTEGER) * 256 * 256 * 256 + CAST(SPLIT_PART(ip, '.', 2) AS INTEGER) * 256 * 256 + CAST(SPLIT_PART(ip, '.', 3) AS INTEGER) * 256 + CAST(SPLIT_PART(ip, '.', 4) AS INTEGER))"},
123+
// -- Misc macros
124+
{DEFAULT_SCHEMA, "generateUUIDv4", {nullptr}, R"(toString(uuid()))"},
125+
{DEFAULT_SCHEMA, "parseURL", {"url", "part", nullptr}, R"(CASE part WHEN 'protocol' THEN REGEXP_EXTRACT(url, '^(\w+)://') WHEN 'domain' THEN REGEXP_EXTRACT(url, '://([^/:]+)') WHEN 'port' THEN REGEXP_EXTRACT(url, ':(\d+)') WHEN 'path' THEN REGEXP_EXTRACT(url, '://[^/]+(/.+?)(\?|#|$)') WHEN 'query' THEN REGEXP_EXTRACT(url, '\?([^#]+)') WHEN 'fragment' THEN REGEXP_EXTRACT(url, '#(.+)$') END)"},
126+
{DEFAULT_SCHEMA, "bitCount", {"num", nullptr}, R"(BIT_COUNT(num))"},
77127
{nullptr, nullptr, {nullptr}, nullptr}};
78128

79129
// To add a new table SQL macro, add a new macro to this array!
@@ -92,7 +142,8 @@ static DefaultMacro chsql_macros[] = {
92142

93143
// clang-format off
94144
static const DefaultTableMacro chsql_table_macros[] = {
95-
{DEFAULT_SCHEMA, "times_two_table", {"x", nullptr}, {{"two", "2"}, {nullptr, nullptr}}, R"(SELECT x * two as output_column;)"},
145+
{DEFAULT_SCHEMA, "tableMultiply", {"x", nullptr}, {{"two", "2"}, {nullptr, nullptr}}, R"(SELECT x * two as output_column;)"},
146+
{DEFAULT_SCHEMA, "numbers", {"x", nullptr}, {{"z", "0"}, {nullptr, nullptr}}, R"(SELECT * as number FROM generate_series(z,x-1);)"},
96147
{nullptr, nullptr, {nullptr}, {{nullptr, nullptr}}, nullptr}
97148
};
98149
// clang-format on

‎test/sql/chsql.test‎

Lines changed: 191 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -137,6 +137,87 @@ SELECT toUInt64('123')
137137
----
138138
123
139139

140+
query I
141+
SELECT toInt8OrZero(-1)
142+
----
143+
-1
144+
145+
query I
146+
SELECT toInt16OrZero(-1)
147+
----
148+
-1
149+
150+
query I
151+
SELECT toInt32OrZero(-1)
152+
----
153+
-1
154+
155+
query I
156+
SELECT toInt64OrZero(-1)
157+
----
158+
-1
159+
160+
query I
161+
SELECT toInt128OrZero(-1)
162+
----
163+
-1
164+
165+
query I
166+
SELECT toInt256OrZero(-1)
167+
----
168+
-1
169+
170+
query I
171+
SELECT toInt8OrZero('notanumber')
172+
----
173+
0
174+
175+
query I
176+
SELECT toInt16OrZero('notanumber')
177+
----
178+
0
179+
180+
query I
181+
SELECT toInt32OrZero('notanumber')
182+
----
183+
0
184+
185+
query I
186+
SELECT toInt64OrZero('notanumber')
187+
----
188+
0
189+
190+
query I
191+
SELECT toInt128OrZero('notanumber')
192+
----
193+
0
194+
195+
query I
196+
SELECT toInt256OrZero('notanumber')
197+
----
198+
0
199+
200+
# Unsigned integer conversion macros
201+
query I
202+
SELECT toUInt8('123')
203+
----
204+
123
205+
206+
query I
207+
SELECT toUInt16('123')
208+
----
209+
123
210+
211+
query I
212+
SELECT toUInt32('123')
213+
----
214+
123
215+
216+
query I
217+
SELECT toUInt64('123')
218+
----
219+
123
220+
140221
query I
141222
SELECT toUInt8OrZero(-1)
142223
----
@@ -177,6 +258,7 @@ SELECT toUInt64OrZero('notanumber')
177258
----
178259
0
179260

261+
# Floating-point conversion macros
180262
query I
181263
SELECT toFloat('123.45')
182264
----
@@ -192,7 +274,115 @@ SELECT toFloatOrZero('notanumber')
192274
----
193275
0
194276

277+
# Arithmetic macros
195278
query I
196-
SELECT intDiv(4, 2)
279+
SELECT intDiv(5, 2)
197280
----
198281
2
282+
283+
query I
284+
SELECT intDivOrNull(5, 2)
285+
----
286+
2
287+
288+
query I
289+
SELECT intDivOrNull(5, 0)
290+
----
291+
NULL
292+
293+
query I
294+
SELECT plus(3, 4)
295+
----
296+
7
297+
298+
query I
299+
SELECT minus(7, 3)
300+
----
301+
4
302+
303+
query I
304+
SELECT modulo(10, 3)
305+
----
306+
1
307+
308+
# String matching macros
309+
query I
310+
SELECT match('hello world', '%world%')
311+
----
312+
true
313+
314+
# Array macros
315+
query I
316+
SELECT arrayExists(3, [1, 2, 3, 4, 5])
317+
----
318+
true
319+
320+
# Date and Time Functions
321+
query I
322+
SELECT toYear('2023-05-15'::DATE)
323+
----
324+
2023
325+
326+
query I
327+
SELECT toMonth('2023-05-15'::DATE)
328+
----
329+
5
330+
331+
query I
332+
SELECT toDayOfMonth('2023-05-15'::DATE)
333+
----
334+
15
335+
336+
# String Functions
337+
query I
338+
SELECT empty('')
339+
----
340+
true
341+
342+
query I
343+
SELECT notEmpty('hello')
344+
----
345+
true
346+
347+
query I
348+
SELECT lengthUTF8('hello')
349+
----
350+
5
351+
352+
query I
353+
SELECT leftPad('123', 5, '0')
354+
----
355+
00123
356+
357+
# URL Functions
358+
query I
359+
SELECT protocol('https://example.com/path')
360+
----
361+
https
362+
363+
query I
364+
SELECT domain('https://example.com/path')
365+
----
366+
example.com
367+
368+
# IP Address Functions
369+
query I
370+
SELECT IPv4NumToString(167772161) -- 10.0.0.1
371+
----
372+
10.0.0.1
373+
374+
query I
375+
SELECT IPv4StringToNum('10.0.0.1')
376+
----
377+
167772161
378+
379+
# Misc macros
380+
query I
381+
SELECT hex(255)
382+
----
383+
FF
384+
385+
query I
386+
SELECT bitCount(7) -- Binary: 111
387+
----
388+
3

0 commit comments

Comments
 (0)