Enhance functions for operator-free environments like Supabase

SUPABASE.md

@@ -0,0 +1,70 @@
+# Supabase
+
+
+## No operators, no problems
+
+Supabase does not support custom operators.
+The EQL operator functions can be used in this situation.
+
+PostgreSQL operators are essentially an alias for the equivalent function, so the implementation and behaviour remains the same.
+
+| Operator | Function                                           | Example                                                           |
+| -------- | -------------------------------------------------- | ----------------------------------------------------------------- |
+| `=`      | `eql_v1.eq(eql_v1_encrypted, eql_v1_encrypted)`    | `SELECT * FROM users WHERE eql_v1.eq(encrypted_email, $1)`<br>    |
+| `<>`     | `eql_v1.neq(eql_v1_encrypted, eql_v1_encrypted)`   | `SELECT * FROM users WHERE eql_v1.neq(encrypted_email, $1)`<br>   |
+| `<`      | `eql_v1.lt(eql_v1_encrypted, eql_v1_encrypted)`    | `SELECT * FROM users WHERE eql_v1.lt(encrypted_email, $1)`<br>    |
+| `<=`     | `eql_v1.lte(eql_v1_encrypted, eql_v1_encrypted)`   | `SELECT * FROM users WHERE eql_v1.lte(encrypted_email, $1)`<br>   |
+| `>`      | `eql_v1.gt(eql_v1_encrypted, eql_v1_encrypted)`    | `SELECT * FROM users WHERE eql_v1.gt(encrypted_email, $1)`<br>    |
+| `>=`     | `eql_v1.gte(eql_v1_encrypted, eql_v1_encrypted)`   | `SELECT * FROM users WHERE eql_v1.gte(encrypted_email, $1)`<br>   |
+| `~~`     | `eql_v1.like(eql_v1_encrypted, eql_v1_encrypted)`  | `SELECT * FROM users WHERE eql_v1.like(encrypted_email, $1)`<br>  |
+| `~~*`    | `eql_v1.ilike(eql_v1_encrypted, eql_v1_encrypted)` | `SELECT * FROM users WHERE eql_v1.ilike(encrypted_email, $1)`<br> |
+| `LIKE`   | `eql_v1.like(eql_v1_encrypted, eql_v1_encrypted)`  | `SELECT * FROM users WHERE eql_v1.like(encrypted_email, $1)`<br>  |
+| `ILIKE`  | `eql_v1.ilike(eql_v1_encrypted, eql_v1_encrypted)` | `SELECT * FROM users WHERE eql_v1.ilike(encrypted_email, $1)`<br> |
+
+### Example SQL Statements
+
+#### Equality `=`
+
+
+**Operator**
+```
+SELECT * FROM users WHERE encrypted_email = $1
+```
+
+**Function**
+```
+SELECT * FROM users WHERE eql_v1.eq(encrypted_email, $1)
+```
+
+
+#### Like & ILIKE `~~, ~~*`
+
+
+**Operator**
+```
+SELECT * FROM users WHERE encrypted_email LIKE $1
+```
+
+**Function**
+```
+SELECT * FROM users WHERE eql_v1.like(encrypted_email, $1)
+```
+
+#### Case Sensitivity
+
+The EQL `eql_v1.like` and `eql_v1.ilike` functions are equivalent.
+
+The behaviour of the "match" index term that is used by the encrypted `LIKE` operators is slightly different to default PostgreSQL. Case sensitivity is determined by the index term configuration. A match index term can be configured to enable case sensitive searches. The data is encrypted based on the configuration. The like operation is always the same, and the data is different. The different operators are kept to preserve the semantics of SQL statements in client applications.
+
+
+
+### Order By
+
+Ordering requires wrapping the ordered column in the `eql_v1.order_by` function.
+
+```
+SELECT * FROM users ORDER BY eql_v1.order_by(encrypted_created_at) DESC
+```
+
+PostgreSQL uses operators when handling `ORDER BY` operations. The `eql_v1.order_by` function behaves in
+

src/encryptindex/functions_test.sql

@@ -8,7 +8,7 @@
 TRUNCATE TABLE eql_v1_configuration;
 
 -- Create a table with a plaintext column
--- DROP TABLE IF EXISTS users;
+DROP TABLE IF EXISTS users;
 CREATE TABLE users
 (
     id bigint GENERATED ALWAYS AS IDENTITY,
@@ -63,7 +63,7 @@ $$ LANGUAGE plpgsql;
 TRUNCATE TABLE eql_v1_configuration;
 
 -- Create a table with multiple plaintext columns
--- DROP TABLE IF EXISTS users;
+DROP TABLE IF EXISTS users;
 CREATE TABLE users
 (
     id bigint GENERATED ALWAYS AS IDENTITY,
@@ -119,7 +119,7 @@ $$ LANGUAGE plpgsql;
 -- The schema should be validated first.
 -- Users table does not exist, so should fail.
 -- -----------------------------------------------
--- DROP TABLE IF EXISTS users;
+DROP TABLE IF EXISTS users;
 TRUNCATE TABLE eql_v1_configuration;
 
 
@@ -148,7 +148,7 @@ $$ LANGUAGE plpgsql;
 --
 -- Schema validation is skipped
 -- -----------------------------------------------
--- DROP TABLE IF EXISTS users;
+DROP TABLE IF EXISTS users;
 TRUNCATE TABLE eql_v1_configuration;
 
 DO $$
@@ -194,7 +194,7 @@ INSERT INTO eql_v1_configuration (state, data) VALUES (
 );
 
 -- Create a table with plaintext and encrypted columns
--- DROP TABLE IF EXISTS users;
+DROP TABLE IF EXISTS users;
 CREATE TABLE users
 (
     id bigint GENERATED ALWAYS AS IDENTITY,
@@ -244,7 +244,7 @@ INSERT INTO eql_v1_configuration (state, data) VALUES (
 );
 
 -- Create a table with plaintext and jsonb column
--- DROP TABLE IF EXISTS users;
+DROP TABLE IF EXISTS users;
 CREATE TABLE users
 (
     id bigint GENERATED ALWAYS AS IDENTITY,
@@ -295,7 +295,7 @@ INSERT INTO eql_v1_configuration (state, data) VALUES (
 
 
 -- Create a table with multiple plaintext columns
--- DROP TABLE IF EXISTS users;
+DROP TABLE IF EXISTS users;
 CREATE TABLE users
 (
     id bigint GENERATED ALWAYS AS IDENTITY,

src/operators/order_by.sql

@@ -0,0 +1,72 @@
+-- REQUIRE: src/encrypted/types.sql
+-- REQUIRE: src/ore/types.sql
+-- REQUIRE: src/ore/functions.sql
+-- REQUIRE: src/ore/operators.sql
+-- REQUIRE: src/ore_cllw_u64_8/types.sql
+-- REQUIRE: src/ore_cllw_u64_8/functions.sql
+-- REQUIRE: src/ore_cllw_u64_8/operators.sql
+
+-- order_by function for ordering when operators are not available.
+--
+-- There are multiple index terms that provide equality comparisons
+--   - ore_cllw_u64_8
+--   - ore_cllw_var_8
+--   - ore_64_8_v1
+--
+-- We check these index terms in this order and use the first one that exists for both parameters
+--
+--
+
+-- DROP FUNCTION IF EXISTS eql_v1.order_by(a eql_v1_encrypted, b eql_v1_encrypted);
+
+CREATE FUNCTION eql_v1.order_by(a eql_v1_encrypted)
+  RETURNS eql_v1.ore_64_8_v1
+  IMMUTABLE STRICT PARALLEL SAFE
+AS $$
+  BEGIN
+    BEGIN
+      RETURN eql_v1.ore_64_8_v1(a);
+    EXCEPTION WHEN OTHERS THEN
+      -- PERFORM eql_v1.log('No ore_64_8_v1 index');
+    END;
+
+    RETURN false;
+  END;
+$$ LANGUAGE plpgsql;
+
+-- TODO: make this work
+--       fails with jsonb format issue, which I think is due to the type casting
+--
+CREATE FUNCTION eql_v1.order_by_any(a anyelement)
+  RETURNS anyelement
+  IMMUTABLE STRICT PARALLEL SAFE
+AS $$
+  DECLARE
+    e eql_v1_encrypted;
+    result ALIAS FOR $0;
+  BEGIN
+
+    e := a::eql_v1_encrypted;
+
+    BEGIN
+      result := eql_v1.ore_cllw_u64_8(e);
+    EXCEPTION WHEN OTHERS THEN
+      -- PERFORM eql_v1.log('No ore_cllw_u64_8 index');
+    END;
+
+    BEGIN
+      result := eql_v1.ore_cllw_var_8(e);
+    EXCEPTION WHEN OTHERS THEN
+      -- PERFORM eql_v1.log('No ore_cllw_u64_8 index');
+    END;
+
+    BEGIN
+      result := eql_v1.ore_64_8_v1(e);
+    EXCEPTION WHEN OTHERS THEN
+      -- PERFORM eql_v1.log('No ore_64_8_v1 index');
+    END;
+
+    RETURN result;
+  END;
+$$ LANGUAGE plpgsql;
+

src/operators/order_by_test.sql

@@ -0,0 +1,28 @@
+\set ON_ERROR_STOP on
+
+--
+-- ORE - ORDER BY ore_64_8_v1(eql_v1_encrypted)
+--
+DO $$
+DECLARE
+    e eql_v1_encrypted;
+    ore_term eql_v1_encrypted;
+  BEGIN
+      SELECT ore.e FROM ore WHERE id = 42 INTO ore_term;
+
+      PERFORM assert_count(
+        'ORDER BY eql_v1.order_by(e) DESC',
+        format('SELECT id FROM ore WHERE e < %L ORDER BY eql_v1.order_by(e) DESC', ore_term),
+        41);
+
+      PERFORM assert_result(
+        'ORDER BY eql_v1.order_by(e) DESC returns correct record',
+        format('SELECT id FROM ore WHERE e < %L ORDER BY eql_v1.order_by(e) DESC LIMIT 1', ore_term),
+        '41');
+
+      PERFORM assert_result(
+        'ORDER BY eql_v1.order_by(e) ASC',
+        format('SELECT id FROM ore WHERE e < %L ORDER BY eql_v1.order_by(e) ASC LIMIT 1', ore_term),
+        '1');
+  END;
+$$ LANGUAGE plpgsql;

src/operators/~~.sql

@@ -15,12 +15,23 @@
 
 -- DROP FUNCTION IF EXISTS eql_v1.match(a eql_v1_encrypted, b eql_v1_encrypted);
 
-CREATE FUNCTION eql_v1.match(a eql_v1_encrypted, b eql_v1_encrypted)
+CREATE FUNCTION eql_v1.like(a eql_v1_encrypted, b eql_v1_encrypted)
 RETURNS boolean AS $$
   SELECT eql_v1.match(a) @> eql_v1.match(b);
 $$ LANGUAGE SQL;
 
 
+--
+-- Case sensitivity depends on the index term configuration
+-- Function preserves the SQL semantics
+--
+CREATE FUNCTION eql_v1.ilike(a eql_v1_encrypted, b eql_v1_encrypted)
+RETURNS boolean AS $$
+  SELECT eql_v1.match(a) @> eql_v1.match(b);
+$$ LANGUAGE SQL;
+
+
+
 -- DROP OPERATOR BEFORE FUNCTION
 -- DROP OPERATOR IF EXISTS ~~ (eql_v1_encrypted, eql_v1_encrypted);
 -- DROP OPERATOR IF EXISTS ~~* (eql_v1_encrypted, eql_v1_encrypted);
@@ -31,7 +42,7 @@ CREATE FUNCTION eql_v1."~~"(a eql_v1_encrypted, b eql_v1_encrypted)
   RETURNS boolean
 AS $$
   BEGIN
-    RETURN eql_v1.match(a, b);
+    RETURN eql_v1.like(a, b);
   END;
 $$ LANGUAGE plpgsql;
 
@@ -65,7 +76,7 @@ CREATE FUNCTION eql_v1."~~"(a eql_v1_encrypted, b jsonb)
   RETURNS boolean
 AS $$
   BEGIN
-    RETURN eql_v1.match(a, b::eql_v1_encrypted);
+    RETURN eql_v1.like(a, b::eql_v1_encrypted);
   END;
 $$ LANGUAGE plpgsql;
 
@@ -100,7 +111,7 @@ CREATE FUNCTION eql_v1."~~"(a jsonb, b eql_v1_encrypted)
   RETURNS boolean
 AS $$
   BEGIN
-    RETURN eql_v1.match(a::eql_v1_encrypted, b);
+    RETURN eql_v1.like(a::eql_v1_encrypted, b);
   END;
 $$ LANGUAGE plpgsql;
 

src/operators/~~_test.sql

@@ -87,17 +87,17 @@ DECLARE
       e := create_encrypted_json(i, 'm');
 
       PERFORM assert_result(
-        format('eql_v1.match(eql_v1_encrypted, eql_v1_encrypted)', i),
-        format('SELECT e FROM encrypted WHERE eql_v1.match(e, %L);', e));
+        format('eql_v1.like(eql_v1_encrypted, eql_v1_encrypted)', i),
+        format('SELECT e FROM encrypted WHERE eql_v1.like(e, %L);', e));
 
     end loop;
 
     -- Partial match
     e := create_encrypted_json('m')::jsonb || '{"m": [10, 11]}';
 
     PERFORM assert_result(
-        'eql_v1.match(eql_v1_encrypted, eql_v1_encrypted)',
-        format('SELECT e FROM encrypted WHERE eql_v1.match(e, %L);', e));
+        'eql_v1.like(eql_v1_encrypted, eql_v1_encrypted)',
+        format('SELECT e FROM encrypted WHERE eql_v1.like(e, %L);', e));
 
   END;
 $$ LANGUAGE plpgsql;

src/ore/functions_test.sql

@@ -11,4 +11,31 @@ DO $$
         'SELECT eql_v1.ore_64_8_v1(''{}''::jsonb)');
 
   END;
+$$ LANGUAGE plpgsql;
+
+--
+-- ORE - ORDER BY ore_64_8_v1(eql_v1_encrypted)
+--
+DO $$
+DECLARE
+    e eql_v1_encrypted;
+    ore_term eql_v1_encrypted;
+  BEGIN
+      SELECT ore.e FROM ore WHERE id = 42 INTO ore_term;
+
+      PERFORM assert_count(
+        'ORDER BY eql_v1.ore_64_8_v1(e) DESC',
+        format('SELECT id FROM ore WHERE e < %L ORDER BY eql_v1.ore_64_8_v1(e) DESC', ore_term),
+        41);
+
+      PERFORM assert_result(
+        'ORDER BY eql_v1.ore_64_8_v1(e) DESC returns correct record',
+        format('SELECT id FROM ore WHERE e < %L ORDER BY eql_v1.ore_64_8_v1(e) DESC LIMIT 1', ore_term),
+        '41');
+
+      PERFORM assert_result(
+        'ORDER BY eql_v1.ore_64_8_v1(e) ASC',
+        format('SELECT id FROM ore WHERE e < %L ORDER BY eql_v1.ore_64_8_v1(e) ASC LIMIT 1', ore_term),
+        '1');
+  END;
 $$ LANGUAGE plpgsql;