Add ORE CLLW and latest SteVec changes to EQL SQL files

justfile

@@ -27,7 +27,7 @@ build:
   #!/usr/bin/env bash
   set -euxo pipefail
 
-  cat sql/database-extensions/postgresql/install.sql sql/dsl-core.sql sql/dsl-config-schema.sql sql/dsl-config-functions.sql sql/dsl-encryptindex.sql > release/cipherstash-encrypt-dsl.sql
+  cat sql/database-extensions/postgresql/install.sql sql/ore-cllw.sql sql/ste-vec.sql sql/dsl-core.sql sql/dsl-config-schema.sql sql/dsl-config-functions.sql sql/dsl-encryptindex.sql > release/cipherstash-encrypt-dsl.sql
 
 
 psql:

release/cipherstash-encrypt-dsl.sql

@@ -315,6 +315,341 @@ CREATE OPERATOR CLASS ore_64_8_v1_btree_ops DEFAULT FOR TYPE ore_64_8_v1 USING b
         OPERATOR 4 >=,
         OPERATOR 5 >,
         FUNCTION 1 compare_ore_64_8_v1(a ore_64_8_v1, b ore_64_8_v1);
+
+---
+--- ORE CLLW types, functions, and operators
+---
+
+-- Represents a ciphertext encrypted with the CLLW ORE scheme
+-- Each output block is 8-bits
+CREATE TYPE ore_cllw_8_v1 AS (
+  bytes bytea
+);
+
+
+CREATE OR REPLACE FUNCTION __compare_inner_ore_cllw_8_v1(a ore_cllw_8_v1, b ore_cllw_8_v1)
+RETURNS int AS $$
+DECLARE
+    len_a INT;
+    x BYTEA;
+    y BYTEA;
+    i INT;
+    differing RECORD;
+BEGIN
+    len_a := LENGTH(a.bytes);
+
+    -- Iterate over each byte and compare them
+    FOR i IN 1..len_a LOOP
+        x := SUBSTRING(a.bytes FROM i FOR 1);
+        y := SUBSTRING(b.bytes FROM i FOR 1);
+
+        -- Check if there's a difference
+        IF x != y THEN
+            differing := (x, y);
+            EXIT;
+        END IF;
+    END LOOP;
+
+    -- If a difference is found, compare the bytes as in Rust logic
+    IF differing IS NOT NULL THEN
+        IF (get_byte(y, 0) + 1) % 256 = get_byte(x, 0) THEN
+            RETURN 1;
+        ELSE
+            RETURN -1;
+        END IF;
+    ELSE
+        RETURN 0;
+    END IF;
+END;
+$$ LANGUAGE plpgsql;
+
+
+CREATE OR REPLACE FUNCTION compare_ore_cllw_8_v1(a ore_cllw_8_v1, b ore_cllw_8_v1)
+RETURNS int AS $$
+DECLARE
+    len_a INT;
+    len_b INT;
+    x BYTEA;
+    y BYTEA;
+    i INT;
+    differing RECORD;
+BEGIN
+    -- Check if the lengths of the two bytea arguments are the same
+    len_a := LENGTH(a.bytes);
+    len_b := LENGTH(b.bytes);
+
+    IF len_a != len_b THEN
+        RAISE EXCEPTION 'Bytea arguments must have the same length';
+    END IF;
+
+    RETURN __compare_inner_ore_cllw_8_v1(a, b);
+END;
+$$ LANGUAGE plpgsql;
+
+CREATE OR REPLACE FUNCTION compare_lex_ore_cllw_8_v1(a ore_cllw_8_v1, b ore_cllw_8_v1)
+RETURNS int AS $$
+DECLARE
+    len_a INT;
+    len_b INT;
+    cmp_result int;
+BEGIN
+    -- Get the lengths of both bytea inputs
+    len_a := LENGTH(a.bytes);
+    len_b := LENGTH(b.bytes);
+
+    -- Handle empty cases
+    IF len_a = 0 AND len_b = 0 THEN
+        RETURN 0;
+    ELSIF len_a = 0 THEN
+        RETURN -1;
+    ELSIF len_b = 0 THEN
+        RETURN 1;
+    END IF;
+
+    -- Use the compare_bytea function to compare byte by byte
+    cmp_result := __compare_inner_ore_cllw_8_v1(a, b);
+
+    -- If the comparison returns 'less' or 'greater', return that result
+    IF cmp_result = -1 THEN
+        RETURN -1;
+    ELSIF cmp_result = 1 THEN
+        RETURN 1;
+    END IF;
+
+    -- If the bytea comparison is 'equal', compare lengths
+    IF len_a < len_b THEN
+        RETURN 1;
+    ELSIF len_a > len_b THEN
+        RETURN -1;
+    ELSE
+        RETURN 0;
+    END IF;
+END;
+$$ LANGUAGE plpgsql;
+
+CREATE OR REPLACE FUNCTION ore_cllw_8_v1_eq(a ore_cllw_8_v1, b ore_cllw_8_v1) RETURNS boolean AS $$
+  SELECT compare_ore_cllw_8_v1(a, b) = 0
+$$ LANGUAGE SQL;
+
+CREATE OR REPLACE FUNCTION ore_cllw_8_v1_neq(a ore_cllw_8_v1, b ore_cllw_8_v1) RETURNS boolean AS $$
+  SELECT compare_ore_cllw_8_v1(a, b) <> 0
+$$ LANGUAGE SQL;
+
+CREATE OR REPLACE FUNCTION ore_cllw_8_v1_lt(a ore_cllw_8_v1, b ore_cllw_8_v1) RETURNS boolean AS $$
+  SELECT compare_ore_cllw_8_v1(a, b) = -1
+$$ LANGUAGE SQL;
+
+CREATE OR REPLACE FUNCTION ore_cllw_8_v1_lte(a ore_cllw_8_v1, b ore_cllw_8_v1) RETURNS boolean AS $$
+  SELECT compare_ore_cllw_8_v1(a, b) != 1
+$$ LANGUAGE SQL;
+
+CREATE OR REPLACE FUNCTION ore_cllw_8_v1_gt(a ore_cllw_8_v1, b ore_cllw_8_v1) RETURNS boolean AS $$
+  SELECT compare_ore_cllw_8_v1(a, b) = 1
+$$ LANGUAGE SQL;
+
+CREATE OR REPLACE FUNCTION ore_cllw_8_v1_gte(a ore_cllw_8_v1, b ore_cllw_8_v1) RETURNS boolean AS $$
+  SELECT compare_ore_cllw_8_v1(a, b) != -1
+$$ LANGUAGE SQL;
+
+CREATE OPERATOR = (
+  PROCEDURE="ore_cllw_8_v1_eq",
+  LEFTARG=ore_cllw_8_v1,
+  RIGHTARG=ore_cllw_8_v1,
+  NEGATOR = <>,
+  RESTRICT = eqsel,
+  JOIN = eqjoinsel,
+  HASHES,
+  MERGES
+);
+
+CREATE OPERATOR <> (
+  PROCEDURE="ore_cllw_8_v1_neq",
+  LEFTARG=ore_cllw_8_v1,
+  RIGHTARG=ore_cllw_8_v1,
+  NEGATOR = =,
+  RESTRICT = eqsel,
+  JOIN = eqjoinsel,
+  HASHES,
+  MERGES
+);
+
+CREATE OPERATOR > (
+  PROCEDURE="ore_cllw_8_v1_gt",
+  LEFTARG=ore_cllw_8_v1,
+  RIGHTARG=ore_cllw_8_v1,
+  NEGATOR = <=,
+  RESTRICT = scalarltsel,
+  JOIN = scalarltjoinsel,
+  HASHES,
+  MERGES
+);
+
+CREATE OPERATOR < (
+  PROCEDURE="ore_cllw_8_v1_lt",
+  LEFTARG=ore_cllw_8_v1,
+  RIGHTARG=ore_cllw_8_v1,
+  NEGATOR = >=,
+  RESTRICT = scalargtsel,
+  JOIN = scalargtjoinsel,
+  HASHES,
+  MERGES
+);
+
+CREATE OPERATOR >= (
+  PROCEDURE="ore_cllw_8_v1_gte",
+  LEFTARG=ore_cllw_8_v1,
+  RIGHTARG=ore_cllw_8_v1,
+  NEGATOR = <,
+  RESTRICT = scalarltsel,
+  JOIN = scalarltjoinsel,
+  HASHES,
+  MERGES
+);
+
+CREATE OPERATOR <= (
+  PROCEDURE="ore_cllw_8_v1_lte",
+  LEFTARG=ore_cllw_8_v1,
+  RIGHTARG=ore_cllw_8_v1,
+  NEGATOR = >,
+  RESTRICT = scalargtsel,
+  JOIN = scalargtjoinsel,
+  HASHES,
+  MERGES
+);
+
+CREATE OPERATOR FAMILY ore_cllw_8_v1_btree_ops USING btree;
+CREATE OPERATOR CLASS ore_cllw_8_v1_btree_ops DEFAULT FOR TYPE ore_cllw_8_v1 USING btree FAMILY ore_cllw_8_v1_btree_ops  AS
+        OPERATOR 1 <,
+        OPERATOR 2 <=,
+        OPERATOR 3 =,
+        OPERATOR 4 >=,
+        OPERATOR 5 >,
+        FUNCTION 1 compare_ore_cllw_8_v1(a ore_cllw_8_v1, b ore_cllw_8_v1);
+
+---
+--- SteVec types, functions, and operators
+---
+
+CREATE TYPE ste_vec_v1_entry AS (
+    tokenized_selector text,
+    term ore_cllw_8_v1,
+    ciphertext text
+);
+
+CREATE TYPE cs_ste_vec_index_v1 AS (
+    entries ste_vec_v1_entry[]
+);
+
+-- Determine if a == b (ignoring ciphertext values)
+CREATE OR REPLACE FUNCTION ste_vec_v1_entry_eq(a ste_vec_v1_entry, b ste_vec_v1_entry)
+RETURNS boolean AS $$
+DECLARE
+    sel_cmp int;
+    term_cmp int;
+BEGIN
+    -- Constant time comparison
+    IF a.tokenized_selector = b.tokenized_selector THEN
+        sel_cmp := 1;
+    ELSE
+        sel_cmp := 0;
+    END IF;
+    IF a.term = b.term THEN
+        term_cmp := 1;
+    ELSE
+        term_cmp := 0;
+    END IF;
+    RETURN (sel_cmp # term_cmp) = 0;
+END;
+$$ LANGUAGE plpgsql;
+
+-- Determine if a contains b (ignoring ciphertext values)
+CREATE OR REPLACE FUNCTION ste_vec_v1_logical_contains(a cs_ste_vec_index_v1, b cs_ste_vec_index_v1)
+RETURNS boolean AS $$
+DECLARE
+    result boolean;
+    intermediate_result boolean;
+BEGIN
+    result := true;
+    IF array_length(b.entries, 1) IS NULL THEN
+        RETURN result;
+    END IF;
+    FOR i IN 1..array_length(b.entries, 1) LOOP
+        intermediate_result := ste_vec_v1_entry_array_contains_entry(a.entries, b.entries[i]);
+        result := result AND intermediate_result;
+    END LOOP;
+    RETURN result;
+END;
+$$ LANGUAGE plpgsql;
+
+-- Determine if a contains b (ignoring ciphertext values)
+CREATE OR REPLACE FUNCTION ste_vec_v1_entry_array_contains_entry(a ste_vec_v1_entry[], b ste_vec_v1_entry)
+RETURNS boolean AS $$
+DECLARE
+    result boolean;
+    intermediate_result boolean;
+BEGIN
+    IF array_length(a, 1) IS NULL THEN
+        RETURN false;
+    END IF;
+
+    result := false;
+    FOR i IN 1..array_length(a, 1) LOOP
+        intermediate_result := a[i].tokenized_selector = b.tokenized_selector AND a[i].term = b.term;
+        result := result OR intermediate_result;
+    END LOOP;
+    RETURN result;
+END;
+$$ LANGUAGE plpgsql;
+
+-- Determine if a is contained by b (ignoring ciphertext values)
+CREATE OR REPLACE FUNCTION ste_vec_v1_logical_is_contained(a cs_ste_vec_index_v1, b cs_ste_vec_index_v1)
+RETURNS boolean AS $$
+BEGIN
+    RETURN ste_vec_v1_logical_contains(b, a);
+END;
+$$ LANGUAGE plpgsql;
+
+
+CREATE OR REPLACE FUNCTION jsonb_to_cs_ste_vec_index_v1(input jsonb)
+RETURNS cs_ste_vec_index_v1 AS $$
+DECLARE
+    vec_entry ste_vec_v1_entry;
+    entry_array ste_vec_v1_entry[];
+    entry_json jsonb;
+    entry_json_array jsonb[];
+    entry_array_length int;
+    i int;
+BEGIN
+    FOR entry_json IN SELECT * FROM jsonb_array_elements(input)
+    LOOP
+        vec_entry := ROW(
+           entry_json->>0,
+           ROW((entry_json->>1)::bytea)::ore_cllw_8_v1,
+           entry_json->>2
+        )::ste_vec_v1_entry;
+        entry_array := array_append(entry_array, vec_entry);
+    END LOOP;
+
+    RETURN ROW(entry_array)::cs_ste_vec_index_v1;
+END;
+$$ LANGUAGE plpgsql;
+
+CREATE CAST (jsonb AS cs_ste_vec_index_v1)
+	WITH FUNCTION jsonb_to_cs_ste_vec_index_v1(jsonb) AS IMPLICIT;
+
+CREATE OPERATOR @> (
+  PROCEDURE="ste_vec_v1_logical_contains",
+  LEFTARG=cs_ste_vec_index_v1,
+  RIGHTARG=cs_ste_vec_index_v1,
+  COMMUTATOR = <@
+);
+
+CREATE OPERATOR <@ (
+  PROCEDURE="ste_vec_v1_logical_is_contained",
+  LEFTARG=cs_ste_vec_index_v1,
+  RIGHTARG=cs_ste_vec_index_v1,
+  COMMUTATOR = @>
+);
 DROP CAST IF EXISTS (text AS ore_64_8_v1_term);
 
 DROP FUNCTION IF EXISTS cs_match_v1;
@@ -338,7 +673,6 @@ DROP DOMAIN IF EXISTS cs_unique_index_v1;
 
 CREATE DOMAIN cs_match_index_v1 AS smallint[];
 CREATE DOMAIN cs_unique_index_v1 AS text;
-CREATE DOMAIN cs_ste_vec_index_v1 AS text[];
 
 -- cs_encrypted_v1 is a column type and cannot be dropped if in use
 DO $$
@@ -354,7 +688,10 @@ CREATE FUNCTION _cs_encrypted_check_kind(val jsonb)
   RETURNS BOOLEAN
 LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE
 BEGIN ATOMIC
-  RETURN (val->>'k' = 'ct' AND val ? 'c') AND NOT val ? 'p';
+  RETURN (
+    (val->>'k' = 'ct' AND val ? 'c') OR
+    (val->>'k' = 'sv' AND val ? 'sv')
+  ) AND NOT val ? 'p';
 END;
 
 CREATE FUNCTION cs_check_encrypted_v1(val jsonb)
@@ -451,7 +788,7 @@ CREATE OR REPLACE FUNCTION cs_ste_vec_v1_v0_0(col jsonb)
   RETURNS cs_ste_vec_index_v1
   LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE
 BEGIN ATOMIC
-	SELECT ARRAY(SELECT jsonb_array_elements(col->'sv'))::cs_ste_vec_index_v1;
+	SELECT (col->'sv')::cs_ste_vec_index_v1;
 END;
 
 CREATE OR REPLACE FUNCTION cs_ste_vec_v1_v0(col jsonb)