feat: allocate FFI output from Rust

.github/workflows/maven.yml

@@ -49,12 +49,7 @@ jobs:
   build_in_docker:
     services:
       kms:
-        image: cosmian/kms:4.3.3
-        env:
-          COSMIAN_SERVER_URL: http://localhost:9998
-          KMS_PUBLIC_PATH: /tmp
-          KMS_PRIVATE_PATH: /tmp
-          KMS_SHARED_PATH: /tmp
+        image: ghcr.io/cosmian/kms:4.4.3
         ports:
           - 9998:9998
       findex_cloud:
@@ -97,6 +92,11 @@ jobs:
       - run: yum -y install java-1.8.0-openjdk maven python3 python3-pip
       - run: python3 scripts/get_native_libraries.py
 
+      - name: Display ldd version
+        run: |
+          ldd --version
+          ldd src/main/resources/linux-x86-64/libcloudproof.so
+
       - name: Build with Maven
         run: mvn compile
         env:
@@ -161,7 +161,8 @@ jobs:
     with:
       branch: develop
       target: wasm32-unknown-unknown
-      kms-version: 4.3.3
+      kms-version: 4.4.3
+      findex-cloud-version: 0.1.0
       copy_fresh_build: false
       copy_regression_files: |
         cp ./cloudproof_java/non_regression_vector.json tests/data/cover_crypt/non_regression/java_non_regression_vector.json
@@ -173,7 +174,7 @@ jobs:
     with:
       branch: develop
       target: x86_64-unknown-linux-gnu
-      kms-version: 4.3.3
+      kms-version: 4.4.3
       copy_fresh_build: false
       copy_regression_files: |
         cp ./cloudproof_java/non_regression_vector.json tests/data/cover_crypt/non_regression/java_non_regression_vector.json

docker-compose.yml

@@ -13,12 +13,7 @@ services:
       - PGDATA=/tmp/postgres2
   kms:
     container_name: kms
-    image: cosmian/kms:4.3.3
-    environment:
-      - KMS_HOSTNAME=0.0.0.0
-      - KMS_PUBLIC_PATH=/tmp
-      - KMS_PRIVATE_PATH=/tmp
-      - KMS_SHARED_PATH=/tmp
+    image: ghcr.io/cosmian/kms:4.4.3
     ports:
       - 9998:9998
     depends_on:
@@ -28,3 +23,8 @@ services:
     image: redis:latest
     ports:
       - 6379:6379
+
+  findex_cloud:
+    image: ghcr.io/cosmian/findex_cloud:0.1.0
+    ports:
+      - 8080:8080

scripts/get_native_libraries.py

@@ -62,6 +62,6 @@ def download_native_libraries(version: str) -> bool:
 
 
 if __name__ == '__main__':
-    ret = download_native_libraries('v2.0.1')
+    ret = download_native_libraries('v2.2.0')
     if ret is False and getenv('GITHUB_ACTIONS'):
-        download_native_libraries('last_build/release/v2.0.1')
+        download_native_libraries('last_build/feature/ffi_allocate_from_rust')

src/main/java/com/cosmian/jna/findex/Findex.java

@@ -9,22 +9,24 @@
 import com.cosmian.jna.findex.ffi.Progress;
 import com.cosmian.jna.findex.ffi.ProgressResults;
 import com.cosmian.jna.findex.ffi.SearchResults;
+import com.cosmian.jna.findex.ffi.UpsertResults;
 import com.cosmian.jna.findex.serde.Leb128Reader;
 import com.cosmian.jna.findex.structs.IndexedValue;
 import com.cosmian.jna.findex.structs.Keyword;
 import com.cosmian.utils.CloudproofException;
 import com.sun.jna.Memory;
+import com.sun.jna.Native;
+import com.sun.jna.Pointer;
 import com.sun.jna.ptr.IntByReference;
 
 public final class Findex extends FindexBase {
 
-    public static void upsert(
-                              byte[] key,
-                              byte[] label,
-                              Map<IndexedValue, Set<Keyword>> additions,
-                              Map<IndexedValue, Set<Keyword>> deletions,
-                              int entryTableNumber,
-                              Database db)
+    public static UpsertResults upsert(byte[] key,
+                                       byte[] label,
+                                       Map<IndexedValue, Set<Keyword>> additions,
+                                       Map<IndexedValue, Set<Keyword>> deletions,
+                                       int entryTableNumber,
+                                       Database db)
         throws CloudproofException {
 
         try (
@@ -33,35 +35,42 @@ public static void upsert(
             keyPointer.write(0, key, 0, key.length);
             labelPointer.write(0, label, 0, label.length);
 
+            // Allocate the amount of memory needed to store a pointer.
+            Memory newKeywordsBuffer = new Memory(8);
+            IntByReference newKeywordsBufferSize = new IntByReference(0);
+
             long start = System.currentTimeMillis();
-            // Indexes creation + insertion/update
-            unwrap(INSTANCE.h_upsert(
-                keyPointer, key.length,
-                labelPointer, label.length,
-                indexedValuesToJson(additions),
-                indexedValuesToJson(deletions),
-                entryTableNumber,
-                db.fetchEntryCallback(),
-                db.upsertEntryCallback(),
-                db.upsertChainCallback()), start);
+            unwrap(INSTANCE.h_upsert(newKeywordsBuffer, newKeywordsBufferSize,
+                                     keyPointer, key.length,
+                                     labelPointer, label.length,
+                                     indexedValuesToJson(additions),
+                                     indexedValuesToJson(deletions),
+                                     entryTableNumber,
+                                     db.fetchEntryCallback(),
+                                     db.upsertEntryCallback(),
+                                     db.upsertChainCallback()),
+                    start);
+
+            byte[] newKeywordsBytes = newKeywordsBuffer.getPointer(0).getByteArray(0, newKeywordsBufferSize.getValue());
+            Native.free(Pointer.nativeValue(newKeywordsBuffer.getPointer(0)));
+            return  new Leb128Reader(newKeywordsBytes).readObject(UpsertResults.class);
         }
     }
 
-    public static void upsert(
-                              byte[] key,
-                              byte[] label,
-                              Map<IndexedValue, Set<Keyword>> additions,
-                              Map<IndexedValue, Set<Keyword>> deletions,
-                              Database db)
+    public static UpsertResults upsert(byte[] key,
+                                       byte[] label,
+                                       Map<IndexedValue, Set<Keyword>> additions,
+                                       Map<IndexedValue, Set<Keyword>> deletions,
+                                       Database db)
         throws CloudproofException {
         // Make entryTableNumber equals to 1 by default
-        upsert(key, label, additions, deletions, 1, db);
+        return upsert(key, label, additions, deletions, 1, db);
     }
 
-    public static void upsert(IndexRequest request)
+    public static UpsertResults upsert(IndexRequest request)
         throws CloudproofException {
-        upsert(request.key, request.label, request.additions, request.deletions, request.entryTableNumber,
-            request.database);
+        return upsert(request.key, request.label, request.additions, request.deletions, request.entryTableNumber,
+                      request.database);
     }
 
     public static SearchResults search(SearchRequest request)
@@ -95,13 +104,6 @@ public static SearchResults search(byte[] key,
                                        Database db,
                                        SearchProgress progressCallback)
         throws CloudproofException {
-        //
-        // Prepare outputs
-        //
-        // start with an arbitration buffer allocation size of 131072 (around 4096
-        // indexedValues)
-        byte[] indexedValuesBuffer = new byte[131072];
-        IntByReference indexedValuesBufferSize = new IntByReference(indexedValuesBuffer.length);
 
         // Findex master keys
         if (key == null) {
@@ -119,37 +121,22 @@ public static SearchResults search(byte[] key,
 
             String wordsJson = keywordsToJson(keyWords);
 
-            // Indexes creation + insertion/update
-            long start = System.currentTimeMillis();
-            int ffiCode = INSTANCE.h_search(
-                indexedValuesBuffer, indexedValuesBufferSize,
-                keyPointer, key.length,
-                labelPointer, label.length,
-                wordsJson,
-                entryTableNumber,
-                wrappedProgress,
-                db.fetchEntryCallback(),
-                db.fetchChainCallback());
-
-            FindexCallbackException.rethrowOnErrorCode(ffiCode, start, System.currentTimeMillis());
-
-            if (ffiCode != 0) {
-                // Retry with correct allocated size
-                indexedValuesBuffer = new byte[indexedValuesBufferSize.getValue()];
-                long startRetry = System.currentTimeMillis();
-                unwrap(INSTANCE.h_search(
-                    indexedValuesBuffer, indexedValuesBufferSize,
-                    keyPointer, key.length,
-                    labelPointer, label.length,
-                    wordsJson,
-                    entryTableNumber,
-                    wrappedProgress,
-                    db.fetchEntryCallback(),
-                    db.fetchChainCallback()), startRetry);
-            }
-
-            byte[] indexedValuesBytes = Arrays.copyOfRange(indexedValuesBuffer, 0, indexedValuesBufferSize.getValue());
+            Memory indexedValuesBuffer = new Memory(8);
+            IntByReference indexedValuesBufferSize = new IntByReference(0);
 
+            long start = System.currentTimeMillis();
+            unwrap(INSTANCE.h_search(indexedValuesBuffer, indexedValuesBufferSize,
+                                     keyPointer, key.length,
+                                     labelPointer, label.length,
+                                     wordsJson,
+                                     entryTableNumber,
+                                     wrappedProgress,
+                                     db.fetchEntryCallback(),
+                                     db.fetchChainCallback()),
+                    start);
+
+            byte[] indexedValuesBytes = indexedValuesBuffer.getPointer(0).getByteArray(0, indexedValuesBufferSize.getValue());
+            Native.free(Pointer.nativeValue(indexedValuesBuffer.getPointer(0)));
             return new Leb128Reader(indexedValuesBytes).readObject(SearchResults.class);
         }
     }