[tiktoken] hello world

.gitignore

@@ -0,0 +1,42 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# Environments
+.env
+.venv
+
+# Tools
+.mypy_cache
+.coverage
+htmlcov
+
+# General
+.DS_Store
+
+Cargo.lock
+target/

Cargo.toml

@@ -0,0 +1,21 @@
+[package]
+name = "tiktoken"
+version = "0.1.0"
+edition = "2021"
+rust-version = "1.57.0"
+
+[lib]
+name = "_tiktoken"
+crate-type = ["cdylib"]
+
+[dependencies]
+pyo3 = { version = "0.17.3", features = ["extension-module"] }
+
+# tiktoken dependencies
+fancy-regex = "0.10.0"
+regex = "1.7.0"
+rustc-hash = "1.1.0"
+bstr = "1.0.1"
+
+[profile.release]
+incremental = true

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2022 OpenAI, Shantanu Jain
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

MANIFEST.in

@@ -0,0 +1,5 @@
+include *.svg
+include *.toml
+include Makefile
+recursive-include scripts *.py
+recursive-include src *.rs

Makefile

@@ -0,0 +1,49 @@
+PROJECT := tiktoken
+
+.PHONY: default
+default: editable_install
+
+.PHONY: install_rust
+install_rust:
+	which cargo >/dev/null || curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain 1.62
+
+.PHONY: clean
+clean:
+	cargo clean
+	pip uninstall -y $(PROJECT)
+	find . | grep -E '__pycache__|\.pyc' | xargs rm -rf
+	find . | grep -E '\.so' | xargs rm -rf
+	rm -rf dist/ build/
+	rm -rf $(PROJECT).egg-info/
+
+.PHONY: format
+format:
+	@ which black >/dev/null || python3 -m pip install black
+	@ which isort >/dev/null || python3 -m pip install isort
+	cargo fmt -- --config group_imports=StdExternalCrate
+	black --line-length 100 --skip-magic-trailing-comma --quiet .
+	isort --line-length 100 --profile black --quiet .
+
+
+.PHONY: format_check
+format_check:
+	@ which black >/dev/null || python3 -m pip install black
+	@ which isort >/dev/null || python3 -m pip install isort
+	cargo fmt --check -- --config group_imports=StdExternalCrate
+	black --check --line-length 100 --skip-magic-trailing-comma --quiet .
+	isort --check --line-length 100 --profile black --quiet .
+
+.PHONY: lint
+lint:
+	cargo clippy --all -- -D warnings
+	@ which flake8 >/dev/null || python3 -m pip install flake8==5 flake8-bugbear==22.9.11
+	flake8 --ignore=E203,E501,W503,E731 --per-file-ignores="$(PROJECT)/__init__.py:F401 setup.py:E402" --exclude=build .
+
+.PHONY: editable_install
+editable_install:
+	@ if [ -f $(PROJECT).egg-info ]; then \
+		pip install --disable-pip-version-check --progress-bar=off setuptools wheel setuptools-rust ; \
+		pip install --disable-pip-version-check --no-build-isolation -e . ; \
+	else \
+		pip install --disable-pip-version-check --no-deps --no-build-isolation --ignore-installed -e . ; \
+	fi

README.md

@@ -0,0 +1,28 @@
+# ⏳ tiktoken
+
+tiktoken is a fast tokeniser.
+
+```python
+import tiktoken
+enc = tiktoken.get_encoding("gpt2")
+print(enc.encode("hello world"))
+```
+
+The open source version of `tiktoken` can be installed from PyPI:
+```
+pip install tiktoken
+```
+
+The tokeniser API is documented in `tiktoken/core.py`.
+
+
+## Performance
+
+`tiktoken` is between 3-6x faster than huggingface's tokeniser:
+
+![image](./perf.svg)
+
+Performance measured on 1GB of text using the GPT-2 tokeniser, using `GPT2TokenizerFast` from
+`tokenizers==0.13.2` and `transformers==4.24.0`.
+
+