[tokenizer] Fixes tokenizer bug (#2843)

Fixes #2840
deepjavalibrary · Apr 26, 2024 · 425d2d9 · 425d2d9
1 parent 6839eb6
commit 425d2d9
Show file tree

Hide file tree

Showing 2 changed files with 7 additions and 1 deletion.
diff --git a/extensions/tokenizers/src/main/java/ai/djl/huggingface/tokenizers/HuggingFaceTokenizer.java b/extensions/tokenizers/src/main/java/ai/djl/huggingface/tokenizers/HuggingFaceTokenizer.java
@@ -787,7 +787,7 @@ public HuggingFaceTokenizer build() throws IOException {
                     return managed(HuggingFaceTokenizer.newInstance(vocab, merges, options));
                 }
                 throw new IOException("tokenizer.json file not found.");
-            } else if (Files.exists(tokenizerPath)) {
+            } else if (!Files.exists(tokenizerPath)) {
                 throw new IOException("Tokenizer file not exits: " + tokenizerPath);
             }
             return managed(HuggingFaceTokenizer.newInstance(tokenizerPath, options));

diff --git a/...ions/tokenizers/src/test/java/ai/djl/huggingface/tokenizers/HuggingFaceTokenizerTest.java b/...ions/tokenizers/src/test/java/ai/djl/huggingface/tokenizers/HuggingFaceTokenizerTest.java
@@ -115,6 +115,12 @@ public void testTokenizer() throws IOException {
             Assert.assertEquals(encodings.length, 2);
             Assert.assertEquals(encodings[0].getIds(), ids);
         }
+
+        Assert.assertThrows(
+                () -> {
+                    Path file = Paths.get("build/tokenizer/non-exists.json");
+                    HuggingFaceTokenizer.builder().optTokenizerPath(file).build();
+                });
     }
 
     @Test