Simplify tokenizer header (#4283)

Summary: The dependency and includes for tokenizer are unnecessarily complicated. This PR deletes some unused and updates some to use/depend on what it really uses. Pull Request resolved: #4283 Test Plan: - test/run_oss_cpp_tests.sh examples/models/llama2/tokenizer/test - test/run_oss_cpp_tests.sh extension/llm/tokenizer/test Reviewed By: larryliu0820 Differential Revision: D59837081 Pulled By: helunwencser fbshipit-source-id: 4c69f01a868a52207725ae39f65fa62ead7b6749
pytorch · Jul 22, 2024 · 844a69f · 844a69f
1 parent 6dbb4dc
commit 844a69f
Show file tree

Hide file tree

Showing 6 changed files with 6 additions and 24 deletions.
diff --git a/extension/llm/tokenizer/bpe_tokenizer.cpp b/extension/llm/tokenizer/bpe_tokenizer.cpp
@@ -8,7 +8,7 @@
 
 #include <executorch/extension/llm/tokenizer/bpe_tokenizer.h>
 
-#include <string>
+#include <cstring>
 
 namespace torch {
 namespace executor {

diff --git a/extension/llm/tokenizer/bpe_tokenizer.h b/extension/llm/tokenizer/bpe_tokenizer.h
@@ -9,7 +9,7 @@
 #pragma once
 
 #include <executorch/extension/llm/tokenizer/tokenizer.h>
-#include <cstdint>
+#include <memory>
 
 namespace torch {
 namespace executor {

diff --git a/extension/llm/tokenizer/targets.bzl b/extension/llm/tokenizer/targets.bzl
@@ -48,8 +48,7 @@ def define_common_targets():
             "bpe_tokenizer.h",
         ],
         exported_deps = [
-            "//executorch/runtime/core/exec_aten:lib",
-            "//executorch/runtime/core/exec_aten/util:scalar_type_util",
+            "//executorch/runtime/core:core",
         ],
         visibility = [
             "@EXECUTORCH_CLIENTS",
@@ -67,8 +66,7 @@ def define_common_targets():
             "base64.h",
         ],
         exported_deps = [
-            "//executorch/runtime/core/exec_aten:lib",
-            "//executorch/runtime/core/exec_aten/util:scalar_type_util",
+            "//executorch/runtime/core:core",
         ],
         visibility = [
             "@EXECUTORCH_CLIENTS",

diff --git a/extension/llm/tokenizer/tiktoken.cpp b/extension/llm/tokenizer/tiktoken.cpp
@@ -27,16 +27,8 @@
 
 #include <executorch/extension/llm/tokenizer/base64.h>
 #include <executorch/extension/llm/tokenizer/tiktoken.h>
-#include <cctype>
-#include <cstdint>
 #include <fstream>
-#include <functional>
 #include <limits>
-#include <memory>
-#include <regex>
-#include <string>
-#include <unordered_set>
-#include <vector>
 
 namespace torch {
 namespace executor {

diff --git a/extension/llm/tokenizer/tiktoken.h b/extension/llm/tokenizer/tiktoken.h
@@ -10,10 +10,8 @@
 
 #include <executorch/extension/llm/tokenizer/tokenizer.h>
 #include <re2/re2.h>
-#include <cstdint>
-#include <functional>
+#include <memory>
 #include <optional>
-#include <regex>
 #include <unordered_map>
 
 namespace torch {

diff --git a/extension/llm/tokenizer/tokenizer.h b/extension/llm/tokenizer/tokenizer.h
@@ -10,17 +10,11 @@
 // model won't work with this class, it needs to go through tokenizer.py first.
 #pragma once
 
-#include <cctype>
-#include <cstdio>
-#include <cstdlib>
-#include <cstring>
-#include <memory>
+#include <cinttypes>
 #include <string>
 #include <vector>
 
 #include <executorch/runtime/core/error.h>
-#include <executorch/runtime/core/exec_aten/exec_aten.h>
-#include <executorch/runtime/core/exec_aten/util/scalar_type_util.h>
 #include <executorch/runtime/core/result.h>
 
 namespace torch {