feat(notebooks): port 04_vectorizers.ipynb

- HuggingFace Sentence Transformers (sentence-transformers/all-mpnet-base-v2)
- OpenAI via LangChain4j (text-embedding-ada-002)
- Cohere via LangChain4j (embed-english-v3.0)
- VoyageAI via LangChain4j (voyage-law-2)
- Custom vectorizer implementation
- Vector search demo ("That is a happy cat" → "That is a happy dog")

Author: bsbodden

notebooks/04_vectorizers.ipynb

@@ -11,11 +11,7 @@
     "2. LangChain4j Integration (OpenAI, Cohere, VoyageAI, Azure, etc.)\n",
     "3. Custom vectorizers\n",
     "\n",
-    "Before running this notebook, be sure to:\n",
-    "1. Have Java 17+ installed\n",
-    "2. Have a running Redis Stack instance with RediSearch > 2.4 active\n",
-    "\n",
-    "For example, you can run Redis Stack locally with Docker:\n",
+    "Before running this notebook, be sure to have a running Redis Stack instance. You can start it with Docker:\n",
     "\n",
     "```bash\n",
     "docker run -d -p 6379:6379 -p 8001:8001 redis/redis-stack:latest\n",
@@ -24,38 +20,47 @@
     "This will run Redis on port 6379 and RedisInsight at http://localhost:8001."
    ]
   },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## Setup\n",
-    "\n",
-    "First, add the RedisVL4j JAR and its dependencies to the classpath.\n",
-    "For local development, you can build the project with `./gradlew :core:build` and find the JAR in `core/build/libs/`."
-   ]
-  },
   {
    "cell_type": "code",
    "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
-    "// Add JARs to classpath - adjust paths as needed\n",
-    "%jars /path/to/redisvl4j/core/build/libs/*.jar\n",
-    "\n",
-    "// Import necessary classes\n",
+    "// Load Maven dependencies\n",
+    "%maven redis.clients:jedis:5.2.0\n",
+    "%maven org.slf4j:slf4j-nop:2.0.16\n",
+    "%maven com.fasterxml.jackson.core:jackson-databind:2.18.0\n",
+    "%maven com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.18.0\n",
+    "%maven com.github.f4b6a3:ulid-creator:5.2.3\n",
+    "%maven dev.langchain4j:langchain4j:0.36.2\n",
+    "%maven dev.langchain4j:langchain4j-open-ai:0.36.2\n",
+    "%maven dev.langchain4j:langchain4j-cohere:0.36.2\n",
+    "%maven dev.langchain4j:langchain4j-voyage-ai:0.36.2\n",
+    "%maven com.microsoft.onnxruntime:onnxruntime:1.16.3\n",
+    "%maven com.squareup.okhttp3:okhttp:4.12.0\n",
+    "%maven com.google.code.gson:gson:2.10.1\n",
+    "%maven ai.djl.huggingface:tokenizers:0.30.0\n",
+    "\n",
+    "// Note: RedisVL JAR is in classpath (loaded automatically by Docker container)\n",
+    "\n",
+    "// Import RedisVL classes\n",
     "import com.redis.vl.utils.vectorize.*;\n",
     "import com.redis.vl.index.SearchIndex;\n",
     "import com.redis.vl.schema.IndexSchema;\n",
     "import com.redis.vl.schema.VectorField;\n",
     "import com.redis.vl.query.VectorQuery;\n",
+    "\n",
+    "// Import Redis client\n",
     "import redis.clients.jedis.UnifiedJedis;\n",
-    "import redis.clients.jedis.search.schemafields.VectorField.VectorAlgorithm;\n",
-    "import java.util.List;\n",
-    "import java.util.Map;\n",
-    "import java.util.HashMap;\n",
-    "import java.util.ArrayList;\n",
-    "import java.util.Arrays;"
+    "import redis.clients.jedis.HostAndPort;\n",
+    "\n",
+    "// Import LangChain4J\n",
+    "import dev.langchain4j.model.openai.OpenAiEmbeddingModel;\n",
+    "import dev.langchain4j.model.cohere.CohereEmbeddingModel;\n",
+    "import dev.langchain4j.model.voyageai.VoyageAiEmbeddingModel;\n",
+    "\n",
+    "// Import Java standard libraries\n",
+    "import java.util.*;"
    ]
   },
   {
@@ -104,7 +109,7 @@
    "source": [
     "// Create a vectorizer using HuggingFace Sentence Transformers\n",
     "// This model runs locally - no API key needed!\n",
-    "var hf = new SentenceTransformersVectorizer(\"sentence-transformers/all-mpnet-base-v2\");\n",
+    "BaseVectorizer hf = new SentenceTransformersVectorizer(\"sentence-transformers/all-mpnet-base-v2\");\n",
     "\n",
     "// Embed a single sentence\n",
     "float[] test = hf.embed(\"This is a test sentence.\");\n",
@@ -141,11 +146,9 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "import dev.langchain4j.model.openai.OpenAiEmbeddingModel;\n",
-    "\n",
     "// Get API key from environment\n",
     "String apiKey = System.getenv(\"OPENAI_API_KEY\");\n",
-    "if (apiKey == null) {\n",
+    "if (apiKey == null || apiKey.isEmpty()) {\n",
     "    System.out.println(\"Skipping OpenAI example - OPENAI_API_KEY not set\");\n",
     "} else {\n",
     "    // Create OpenAI embedding model\n",
@@ -155,7 +158,7 @@
     "        .build();\n",
     "    \n",
     "    // Wrap in LangChain4JVectorizer\n",
-    "    var oai = new LangChain4JVectorizer(\"text-embedding-ada-002\", openaiModel);\n",
+    "    BaseVectorizer oai = new LangChain4JVectorizer(\"text-embedding-ada-002\", openaiModel);\n",
     "    \n",
     "    // Embed a sentence\n",
     "    float[] openaiTest = oai.embed(\"This is a test sentence.\");\n",
@@ -185,18 +188,16 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "import dev.langchain4j.model.cohere.CohereEmbeddingModel;\n",
-    "\n",
     "String cohereApiKey = System.getenv(\"COHERE_API_KEY\");\n",
-    "if (cohereApiKey == null) {\n",
+    "if (cohereApiKey == null || cohereApiKey.isEmpty()) {\n",
     "    System.out.println(\"Skipping Cohere example - COHERE_API_KEY not set\");\n",
     "} else {\n",
     "    var cohereModel = CohereEmbeddingModel.builder()\n",
     "        .apiKey(cohereApiKey)\n",
     "        .modelName(\"embed-english-v3.0\")\n",
     "        .build();\n",
     "    \n",
-    "    var co = new LangChain4JVectorizer(\"embed-english-v3.0\", cohereModel);\n",
+    "    BaseVectorizer co = new LangChain4JVectorizer(\"embed-english-v3.0\", cohereModel);\n",
     "    \n",
     "    float[] cohereTest = co.embed(\"This is a test sentence.\");\n",
     "    System.out.println(\"Cohere Vector dimensions: \" + cohereTest.length);\n",
@@ -221,18 +222,16 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "import dev.langchain4j.model.voyageai.VoyageAiEmbeddingModel;\n",
-    "\n",
     "String voyageApiKey = System.getenv(\"VOYAGE_API_KEY\");\n",
-    "if (voyageApiKey == null) {\n",
+    "if (voyageApiKey == null || voyageApiKey.isEmpty()) {\n",
     "    System.out.println(\"Skipping VoyageAI example - VOYAGE_API_KEY not set\");\n",
     "} else {\n",
     "    var voyageModel = VoyageAiEmbeddingModel.builder()\n",
     "        .apiKey(voyageApiKey)\n",
     "        .modelName(\"voyage-law-2\")\n",
     "        .build();\n",
     "    \n",
-    "    var vo = new LangChain4JVectorizer(\"voyage-law-2\", voyageModel);\n",
+    "    BaseVectorizer vo = new LangChain4JVectorizer(\"voyage-law-2\", voyageModel);\n",
     "    \n",
     "    float[] voyageTest = vo.embed(\"This is a test sentence.\");\n",
     "    System.out.println(\"VoyageAI Vector dimensions: \" + voyageTest.length);\n",
@@ -271,13 +270,15 @@
     "    \n",
     "    @Override\n",
     "    protected List<float[]> generateEmbeddingsBatch(List<String> texts, int batchSize) {\n",
-    "        return texts.stream()\n",
-    "            .map(this::generateEmbedding)\n",
-    "            .collect(java.util.stream.Collectors.toList());\n",
+    "        List<float[]> results = new ArrayList<>();\n",
+    "        for (String text : texts) {\n",
+    "            results.add(generateEmbedding(text));\n",
+    "        }\n",
+    "        return results;\n",
     "    }\n",
     "}\n",
     "\n",
-    "var customVectorizer = new CustomVectorizer();\n",
+    "BaseVectorizer customVectorizer = new CustomVectorizer();\n",
     "float[] customEmbed = customVectorizer.embed(\"This is a test sentence.\");\n",
     "System.out.println(\"Custom vectorizer dimensions: \" + customEmbed.length);\n",
     "System.out.println(\"First 10 values: \" + Arrays.toString(Arrays.copyOfRange(customEmbed, 0, 10)));"
@@ -302,23 +303,31 @@
    "outputs": [],
    "source": [
     "// Connect to Redis\n",
-    "var redis = new UnifiedJedis(\"redis://localhost:6379\");\n",
-    "\n",
-    "// Create the schema - matching the Python notebook YAML\n",
-    "var schema = IndexSchema.builder()\n",
-    "    .name(\"vectorizers\")\n",
-    "    .prefix(\"doc\")\n",
-    "    .storageType(IndexSchema.StorageType.HASH)\n",
-    "    .addTextField(\"sentence\", textField -> {})\n",
-    "    .addVectorField(\"embedding\", 768, vectorField ->\n",
-    "        vectorField\n",
-    "            .algorithm(VectorAlgorithm.FLAT)\n",
-    "            .distanceMetric(VectorField.DistanceMetric.COSINE)\n",
-    "            .dataType(VectorField.VectorDataType.FLOAT32))\n",
-    "    .build();\n",
+    "UnifiedJedis jedis = new UnifiedJedis(new HostAndPort(\"redis-stack\", 6379));\n",
+    "\n",
+    "// Create the schema from a Map (matching the Python notebook YAML)\n",
+    "Map<String, Object> schema = Map.of(\n",
+    "    \"index\", Map.of(\n",
+    "        \"name\", \"vectorizers\",\n",
+    "        \"prefix\", \"doc\"\n",
+    "    ),\n",
+    "    \"fields\", List.of(\n",
+    "        Map.of(\"name\", \"sentence\", \"type\", \"text\"),\n",
+    "        Map.of(\n",
+    "            \"name\", \"embedding\",\n",
+    "            \"type\", \"vector\",\n",
+    "            \"attrs\", Map.of(\n",
+    "                \"dims\", 768,\n",
+    "                \"distance_metric\", \"cosine\",\n",
+    "                \"algorithm\", \"flat\",\n",
+    "                \"datatype\", \"float32\"\n",
+    "            )\n",
+    "        )\n",
+    "    )\n",
+    ");\n",
     "\n",
     "// Create the index\n",
-    "var index = new SearchIndex(schema, redis);\n",
+    "SearchIndex index = SearchIndex.fromDict(schema, jedis);\n",
     "index.create(true); // overwrite if exists\n",
     "System.out.println(\"Index created: \" + index.getName());"
    ]
@@ -342,8 +351,9 @@
     "}\n",
     "\n",
     "// Load data into the index\n",
-    "index.load(data);\n",
-    "System.out.println(\"Loaded \" + data.size() + \" documents\");"
+    "List<String> keys = index.load(data);\n",
+    "System.out.println(\"Loaded \" + data.size() + \" documents\");\n",
+    "System.out.println(\"Keys: \" + keys);"
    ]
   },
   {
@@ -356,17 +366,17 @@
     "float[] queryEmbedding = hf.embed(\"That is a happy cat\");\n",
     "\n",
     "// Create and execute a vector query\n",
-    "var query = VectorQuery.builder()\n",
+    "VectorQuery query = VectorQuery.builder()\n",
     "    .vector(queryEmbedding)\n",
     "    .field(\"embedding\")\n",
-    "    .returnFields(List.of(\"sentence\"))\n",
+    "    .returnFields(\"sentence\", \"vector_distance\")\n",
     "    .numResults(3)\n",
     "    .build();\n",
     "\n",
     "List<Map<String, Object>> results = index.query(query);\n",
     "\n",
     "System.out.println(\"\\nSearch results for: 'That is a happy cat'\");\n",
-    "for (var doc : results) {\n",
+    "for (Map<String, Object> doc : results) {\n",
     "    System.out.println(doc.get(\"sentence\") + \" - Distance: \" + doc.get(\"vector_distance\"));\n",
     "}"
    ]
@@ -386,7 +396,8 @@
    "source": [
     "// Cleanup\n",
     "index.delete(true);\n",
-    "System.out.println(\"Index deleted\");"
+    "jedis.close();\n",
+    "System.out.println(\"Index deleted and connection closed\");"
    ]
   },
   {
@@ -412,8 +423,12 @@
    "name": "java"
   },
   "language_info": {
-   "name": "java",
-   "version": "17"
+   "codemirror_mode": "java",
+   "file_extension": ".jshell",
+   "mimetype": "text/x-java-source",
+   "name": "Java",
+   "pygments_lexer": "java",
+   "version": "21+35"
   }
  },
  "nbformat": 4,