feat: JSON indexing for EQL V2

Cargo.toml

@@ -9,6 +9,13 @@ edition = "2021"
 [profile.dev]
 incremental = true
 debug = true
+opt-level = 0
+split-debuginfo = "unpacked" # or "unpacked" on macOS
+
+[profile.dev.package.sqltk]
+opt-level = 0
+debug = true
+split-debuginfo = "unpacked" # or "unpacked" on macOS
 
 # [profile.dev.package]# aws-lc-sys.opt-level = 3
 # proc-macro2.opt-level = 3
@@ -17,8 +24,8 @@ debug = true
 # sqlparser.opt-level = 3
 # syn.opt-level = 3
 
-[profile.dev.build-override]
-opt-level = 3
+# [profile.dev.build-override]
+# opt-level = 3
 
 [profile.test]
 incremental = true
@@ -36,6 +43,7 @@ debug = true
 
 [workspace.dependencies]
 sqltk = { version = "0.10.0" }
+cipherstash-client = "0.23.0"
 thiserror = "2.0.9"
 tokio = { version = "1.44.2", features = ["full"] }
 tracing = "0.1"

README.md

@@ -29,17 +29,19 @@
 
 [Read the announcement](https://cipherstash.com/blog/introducing-proxy)
 
+CipherStash Proxy provides transparent, *searchable* encryption for your existing Postgres database.
 
-CipherStash Proxy provides a transparent proxy to your existing Postgres database.
+CipherStash Proxy:
+* Automatically encrypts and decrypts data with zero changes to SQL
+* Supports queries over *encrypted* values:
+   - equality
+   - comparison
+   - ordering
+   - grouping
+* Is written in Rust for high performance and strongly-typed mapping of SQL statements.
+* Manages keys using CipherStash ZeroKMS, offering up to 14x the performance of AWS KMS
 
-Proxy:
-* Automatically encrypts and decrypts the columns you specify
-* Supports most query types over encrypted values
-* Runs in a Docker container
-* Is written in Rust and uses a formal type system for SQL mapping
-* Works with CipherStash ZeroKMS and offers up to 14x the performance of AWS KMS
-
-Behind the scenes, it uses the [Encrypt Query Language](https://github.com/cipherstash/encrypt-query-language/) to index and search encrypted data.
+Behind the scenes, CipherStash Proxy uses the [Encrypt Query Language](https://github.com/cipherstash/encrypt-query-language/) to index and search encrypted data.
 
 ## Table of contents
 
@@ -54,7 +56,7 @@ Behind the scenes, it uses the [Encrypt Query Language](https://github.com/ciphe
 > [!IMPORTANT]
 > **Prerequisites:** Before you start you need to have this software installed:
 >  - [Docker](https://www.docker.com/) — see Docker's [documentation for installing](https://docs.docker.com/get-started/get-docker/)
- 
+
 
 Get up and running in local dev in < 5 minutes:
 

mise.toml

@@ -31,7 +31,7 @@ CS_PROXY__HOST = "proxy"
 # Misc
 DOCKER_CLI_HINTS = "false" # Please don't show us What's Next.
 
-CS_EQL_VERSION = "eql-2.0.4"
+CS_EQL_VERSION = "eql-2.0.6"
 
 [tools]
 "cargo:cargo-binstall" = "latest"

packages/cipherstash-proxy-integration/Cargo.toml

@@ -4,13 +4,22 @@ version = "0.1.0"
 edition = "2021"
 
 [dependencies]
+bytes = "1.10.1"
+cipherstash-client = { workspace = true, features = ["tokio"] }
+cipherstash-config = "0.2.3"
 cipherstash-proxy = { path = "../cipherstash-proxy/" }
 chrono = { version = "0.4.39", features = ["clock"] }
+clap = "4.5.32"
+fake = { version = "4", features = ["chrono", "derive"] }
+
+hex = "0.4.3"
+postgres-types = { version = "0.2.9", features = ["derive"] }
 rand = "0.9"
 recipher = "0.1.3"
 rustls = { version = "0.23.20", default-features = false, features = ["std"] }
 serde = "1.0"
 serde_json = "1.0"
+tap = "1.0.1"
 temp-env = "0.3.6"
 tokio = { workspace = true }
 tokio-postgres = { version = "0.7", features = [
@@ -21,14 +30,5 @@ tokio-postgres-rustls = "0.13.0"
 tokio-rustls = "0.26.0"
 tracing = { workspace = true }
 tracing-subscriber = { workspace = true }
-webpki-roots = "1.0"
-
-[dev-dependencies]
-cipherstash-client = { version = "0.22.0", features = ["tokio"] }
-cipherstash-config = "0.2.3"
-clap = "4.5.32"
-fake = { version = "4", features = ["chrono", "derive"] }
-hex = "0.4.3"
-postgres-types = { version = "0.2.9", features = ["derive"] }
-tap = "1.0.1"
 uuid = { version = "1.11.0", features = ["serde", "v4"] }
+webpki-roots = "1.0"

packages/cipherstash-proxy-integration/src/common.rs

@@ -5,6 +5,7 @@ use rustls::{
     client::danger::ServerCertVerifier, crypto::aws_lc_rs::default_provider,
     pki_types::CertificateDer, ClientConfig,
 };
+use serde_json::Value;
 use std::sync::{Arc, Once};
 use tokio_postgres::{types::ToSql, Client, NoTls};
 use tracing_subscriber::{filter::Directive, EnvFilter, FmtSubscriber};
@@ -199,6 +200,31 @@ pub async fn simple_query_with_null(sql: &str) -> Vec<Option<String>> {
         .collect()
 }
 
+pub async fn insert(sql: &str, params: &[&(dyn ToSql + Sync)]) {
+    let client = connect_with_tls(PROXY).await;
+    client.query(sql, params).await.unwrap();
+}
+
+pub async fn insert_jsonb() -> Value {
+    let id = random_id();
+
+    let encrypted_jsonb = serde_json::json!({
+        "id": id,
+        "string": "hello",
+        "number": 42,
+        "nested": {
+            "number": 1815,
+            "string": "world",
+        }
+    });
+
+    let sql = "INSERT INTO encrypted (id, encrypted_jsonb) VALUES ($1, $2)".to_string();
+
+    insert(&sql, &[&id, &encrypted_jsonb]).await;
+
+    encrypted_jsonb
+}
+
 ///
 /// Configure the client TLS settings.
 /// These are the settings for connecting to the database with TLS.

packages/cipherstash-proxy-integration/src/lib.rs

@@ -18,6 +18,7 @@ mod pipeline;
 mod schema_change;
 mod select;
 mod simple_protocol;
+mod support;
 
 #[macro_export]
 macro_rules! value_for_type {

packages/cipherstash-proxy-integration/src/select/indexing.rs

@@ -0,0 +1,54 @@
+#[cfg(test)]
+mod tests {
+    use crate::common::{
+        connect_with_tls, insert, query_by, random_id, simple_query, trace, PROXY,
+    };
+    use tokio_postgres::types::{FromSql, ToSql};
+    use tracing::info;
+
+    #[derive(Debug, ToSql, FromSql, PartialEq)]
+    #[postgres(name = "domain_type_with_check")]
+    pub struct Domain(String);
+
+    ///
+    /// Tests insertion of custom domain type
+    ///
+    #[tokio::test]
+    async fn select_with_index() {
+        trace();
+
+        // let id = random_id();
+        // let encrypted_val = Domain("ZZ".to_string());
+
+        // CREATE INDEX ON encrypted (e eql_v2.encrypted_operator_class);
+        // SELECT ore.e FROM ore WHERE id = 42 INTO ore_term;
+
+        for n in 1..=10 {
+            let id = random_id();
+
+            let encrypted_text = format!("hello_{}", n);
+
+            let sql = format!("INSERT INTO encrypted (id, encrypted_text) VALUES ($1, $2)");
+            insert(&sql, &[&id, &encrypted_text]).await;
+        }
+
+        let client = connect_with_tls(PROXY).await;
+
+        let sql = "CREATE INDEX ON encrypted (encrypted_text eql_v2.encrypted_operator_class)";
+        let _ = client.simple_query(sql).await;
+
+        // let sql =
+        //     "EXPLAIN ANALYZE SELECT encrypted_text FROM encrypted WHERE encrypted_text <= '{\"hm\": \"abc\"}'::jsonb::eql_v2_encrypted";
+        // let result = simple_query::<String>(sql).await;
+
+        let sql = "EXPLAIN ANALYZE SELECT encrypted_text FROM encrypted WHERE encrypted_text <= $1";
+
+        let encrypted_text = "hello_10".to_string();
+        let result = query_by::<String>(sql, &encrypted_text).await;
+
+        info!("Result: {:?}", result);
+
+        // let expected = vec![encrypted_val];
+        // assert_eq!(expected, result);
+    }
+}

packages/cipherstash-proxy-integration/src/select/jsonb_path_query.rs

@@ -0,0 +1,97 @@
+#[cfg(test)]
+mod tests {
+    use crate::common::{clear, insert_jsonb, query_by, simple_query, trace};
+    use crate::support::assert::assert_expected;
+    use crate::support::json_path::JsonPath;
+    use serde::de::DeserializeOwned;
+    use serde_json::Value;
+
+    async fn select_jsonb<T>(selector: &str, value: T)
+    where
+        T: DeserializeOwned,
+        serde_json::Value: From<T>,
+    {
+        let selector = JsonPath::new(selector);
+        let value = Value::from(value);
+
+        let expected = vec![value];
+
+        let sql = "SELECT jsonb_path_query(encrypted_jsonb, $1) FROM encrypted";
+        let actual = query_by::<Value>(sql, &selector).await;
+
+        assert_expected(&expected, &actual);
+
+        let sql = format!("SELECT jsonb_path_query(encrypted_jsonb, '{selector}') FROM encrypted");
+        let actual = simple_query::<Value>(&sql).await;
+
+        assert_expected(&expected, &actual);
+    }
+
+    #[tokio::test]
+    async fn select_jsonb_path_query_number() {
+        trace();
+
+        clear().await;
+
+        insert_jsonb().await;
+
+        select_jsonb("$.number", 42).await;
+    }
+
+    #[tokio::test]
+    async fn select_jsonb_path_query_string() {
+        trace();
+
+        clear().await;
+
+        insert_jsonb().await;
+
+        select_jsonb("$.nested.string", "world".to_string()).await;
+    }
+
+    #[tokio::test]
+    async fn select_jsonb_path_query_value() {
+        trace();
+
+        clear().await;
+
+        insert_jsonb().await;
+
+        let v = serde_json::json!({
+            "number": 1815,
+            "string": "world",
+        });
+
+        select_jsonb("$.nested", v).await;
+    }
+
+    #[tokio::test]
+    async fn select_jsonb_path_query_with_alias() {
+        trace();
+
+        clear().await;
+
+        insert_jsonb().await;
+
+        let value = serde_json::json!({
+            "number": 1815,
+            "string": "world",
+        });
+
+        let selector = JsonPath::new("$.nested");
+
+        let expected = vec![value];
+
+        let sql = "SELECT jsonb_path_query(encrypted_jsonb, $1) as selected FROM encrypted";
+        let actual = query_by::<Value>(sql, &selector).await;
+
+        assert_expected(&expected, &actual);
+
+        let sql = format!(
+            "SELECT jsonb_path_query(encrypted_jsonb, '{selector}') as selected FROM encrypted"
+        );
+        let actual = simple_query::<Value>(&sql).await;
+
+        assert_expected(&expected, &actual);
+    }
+}

packages/cipherstash-proxy-integration/src/select/mod.rs

@@ -1,4 +1,5 @@
 mod group_by;
+mod jsonb_path_query;
 mod order_by;
 mod order_by_with_null;
 mod pg_catalog;

packages/cipherstash-proxy-integration/src/support/assert.rs

@@ -7,13 +7,3 @@ where
         assert_eq!(e, a);
     }
 }
-
-pub fn assert_expected_as_string<T>(expected: &[T], actual: &[String])
-where
-    T: std::fmt::Display + PartialEq + std::fmt::Debug,
-{
-    assert_eq!(expected.len(), actual.len());
-    for (e, a) in expected.iter().zip(actual) {
-        assert_eq!(e.to_string(), *a);
-    }
-}

packages/cipherstash-proxy-integration/src/support/json_path.rs

@@ -1,3 +1,5 @@
+#![allow(dead_code)]
+
 use bytes::BytesMut;
 use postgres_types::{Format, ToSql, Type};
 use std::{

packages/cipherstash-proxy-integration/src/support/mod.rs

@@ -1,2 +1,4 @@
+#[cfg(test)]
 pub mod assert;
+
 pub mod json_path;

packages/cipherstash-proxy/Cargo.toml

@@ -8,7 +8,7 @@ bigdecimal = { version = "0.4.6", features = ["serde-json"] }
 arc-swap = "1.7.1"
 bytes = { version = "1.9", default-features = false }
 chrono = { version = "0.4.39", features = ["clock"] }
-cipherstash-client = { version = "0.22.1", features = ["tokio"] }
+cipherstash-client = { workspace = true, features = ["tokio"] }
 clap = { version = "4.5.31", features = ["derive", "env"] }
 config = { version = "0.15", features = [
     "async",