Introduce keras-io documentation for KerasNLP (keras-team#859)

* Add doc structure for KerasNLP section * Add the actual guide text * Updates * Address most review comments * Rename mlm head layer * Download files with keras * Remove preprocessing and reformat api symbols * Move cells * typo fixes * Updates to API docs and landing page * Variable rename * More new language * Update docs to match keras_nlp github * Bump dev version of keras-nlp * Edits to guide and quickstart * Add generated files * Copyedits * Stop pinning pre-release keras-nlp * Regenerate guide * Add an index for keras_nlp guides Co-authored-by: François Chollet <francois.chollet@gmail.com>
satpalsr · May 18, 2022 · 9d6e6b0 · 9d6e6b0
1 parent 28072cb
commit 9d6e6b0
Show file tree

Hide file tree

Showing 15 changed files with 2,169 additions and 3 deletions.
diff --git a/guides/ipynb/keras_nlp/transformer_pretraining.ipynb b/guides/ipynb/keras_nlp/transformer_pretraining.ipynb
diff --git a/guides/keras_nlp/transformer_pretraining.py b/guides/keras_nlp/transformer_pretraining.py
diff --git a/guides/md/keras_nlp/transformer_pretraining.md b/guides/md/keras_nlp/transformer_pretraining.md
diff --git a/requirements.txt b/requirements.txt
@@ -12,3 +12,4 @@ jupyter
 keras-tuner
 pandas
 keras-cv
+keras-nlp
diff --git a/scripts/guides_master.py b/scripts/guides_master.py
@@ -18,6 +18,18 @@
     ]
 }
 
+NLP_GUIDES_MASTER = {
+    'path': 'keras_nlp/',
+    'title': 'KerasNLP',
+    'toc': True,
+    'children': [
+        {
+            'path': 'transformer_pretraining',
+            'title': 'Pretraining a Transformer from scratch with KerasNLP',
+        },
+    ]
+}
+
 KT_GUIDES_MASTER = {
     'path': 'keras_tuner/',
     'title': 'Hyperparameter Tuning',
@@ -120,7 +132,8 @@
         #     'title': 'Hyperparameter optimization',
         # },
         KT_GUIDES_MASTER,
-        CV_GUIDES_MASTER
+        CV_GUIDES_MASTER,
+        NLP_GUIDES_MASTER,
         # TODO: mixed precision
     ]
 }
diff --git a/scripts/master.py b/scripts/master.py
@@ -6,6 +6,7 @@
 from kt_api_master import KT_API_MASTER
 from examples_master import EXAMPLES_MASTER
 from cv_api_master import CV_API_MASTER
+from nlp_api_master import NLP_API_MASTER
 
 MASTER = {
     'path': '/',
@@ -483,7 +484,8 @@
                 },
                 UTILS_MASTER,
                 KT_API_MASTER,
-                CV_API_MASTER
+                CV_API_MASTER,
+                NLP_API_MASTER,
             ]
         },
         EXAMPLES_MASTER,  # The examples section master will be mostly autogenerated.
@@ -507,5 +509,9 @@
             'path': 'keras_cv/',
             'title': 'KerasCV',
         },
+        {
+            'path': 'keras_nlp/',
+            'title': 'KerasNLP',
+        },
     ]
 }
diff --git a/scripts/nlp_api_master.py b/scripts/nlp_api_master.py
@@ -0,0 +1,134 @@
+TOKENIZERS_MASTER = {
+    "path": "tokenizers/",
+    "title": "Tokenizers",
+    "toc": True,
+    "children": [
+        {
+            "path": "tokenizer",
+            "title": "Tokenizer base class",
+            "generate": [
+                "keras_nlp.tokenizers.Tokenizer",
+                "keras_nlp.tokenizers.Tokenizer.tokenize",
+                "keras_nlp.tokenizers.Tokenizer.detokenize",
+                "keras_nlp.tokenizers.Tokenizer.get_vocabulary",
+                "keras_nlp.tokenizers.Tokenizer.vocabulary_size",
+                "keras_nlp.tokenizers.Tokenizer.token_to_id",
+                "keras_nlp.tokenizers.Tokenizer.id_to_token",
+            ],
+        },
+        {
+            "path": "word_piece_tokenizer",
+            "title": "WordPieceTokenizer",
+            "generate": [
+                "keras_nlp.tokenizers.WordPieceTokenizer",
+                "keras_nlp.tokenizers.WordPieceTokenizer.tokenize",
+                "keras_nlp.tokenizers.WordPieceTokenizer.detokenize",
+                "keras_nlp.tokenizers.WordPieceTokenizer.get_vocabulary",
+                "keras_nlp.tokenizers.WordPieceTokenizer.vocabulary_size",
+                "keras_nlp.tokenizers.WordPieceTokenizer.token_to_id",
+                "keras_nlp.tokenizers.WordPieceTokenizer.id_to_token",
+            ],
+        },
+        {
+            "path": "byte_tokenizer",
+            "title": "ByteTokenizer",
+            "generate": [
+                "keras_nlp.tokenizers.ByteTokenizer",
+                "keras_nlp.tokenizers.ByteTokenizer.tokenize",
+                "keras_nlp.tokenizers.ByteTokenizer.detokenize",
+                "keras_nlp.tokenizers.ByteTokenizer.get_vocabulary",
+                "keras_nlp.tokenizers.ByteTokenizer.vocabulary_size",
+                "keras_nlp.tokenizers.ByteTokenizer.token_to_id",
+                "keras_nlp.tokenizers.ByteTokenizer.id_to_token",
+            ],
+        },
+        {
+            "path": "unicode_character_tokenizer",
+            "title": "UnicodeCharacterTokenizer",
+            "generate": [
+                "keras_nlp.tokenizers.UnicodeCharacterTokenizer",
+                "keras_nlp.tokenizers.UnicodeCharacterTokenizer.tokenize",
+                "keras_nlp.tokenizers.UnicodeCharacterTokenizer.detokenize",
+                "keras_nlp.tokenizers.UnicodeCharacterTokenizer.get_vocabulary",
+                "keras_nlp.tokenizers.UnicodeCharacterTokenizer.vocabulary_size",
+                "keras_nlp.tokenizers.UnicodeCharacterTokenizer.token_to_id",
+                "keras_nlp.tokenizers.UnicodeCharacterTokenizer.id_to_token",
+            ],
+        },
+    ],
+}
+
+LAYERS_MASTER = {
+    "path": "layers/",
+    "title": "Layers",
+    "toc": True,
+    "children": [
+        {
+            "path": "transformer_encoder",
+            "title": "TransformerEncoder layer",
+            "generate": [
+                "keras_nlp.layers.TransformerEncoder",
+                "keras_nlp.layers.TransformerEncoder.call",
+            ],
+        },
+        {
+            "path": "transformer_decoder",
+            "title": "TransformerDecoder layer",
+            "generate": [
+                "keras_nlp.layers.TransformerDecoder",
+                "keras_nlp.layers.TransformerDecoder.call",
+            ],
+        },
+        {
+            "path": "fnet_encoder",
+            "title": "FNetEncoder layer",
+            "generate": ["keras_nlp.layers.FNetEncoder"],
+        },
+        {
+            "path": "position_embedding",
+            "title": "PositionEmbedding layer",
+            "generate": ["keras_nlp.layers.PositionEmbedding"],
+        },
+        {
+            "path": "sine_position_encoding",
+            "title": "SinePositionEncoding layer",
+            "generate": ["keras_nlp.layers.SinePositionEncoding"],
+        },
+        {
+            "path": "token_and_position_embedding",
+            "title": "TokenAndPositionEmbedding layer",
+            "generate": ["keras_nlp.layers.TokenAndPositionEmbedding"],
+        },
+        {
+            "path": "mlm_mask_generator",
+            "title": "MLMMaskGenerator layer",
+            "generate": ["keras_nlp.layers.MLMMaskGenerator"],
+        },
+        {
+            "path": "mlm_head",
+            "title": "MLMHead layer",
+            "generate": ["keras_nlp.layers.MLMHead"],
+        },
+    ],
+}
+
+
+METRICS_MASTER = {
+    "path": "metrics/",
+    "title": "Metrics",
+    "toc": True,
+    "children": [
+        {
+            "path": "perplexity",
+            "title": "Perplexity metric",
+            "generate": ["keras_nlp.metrics.Perplexity"],
+        },
+    ],
+}
+
+NLP_API_MASTER = {
+    "path": "keras_nlp/",
+    "title": "KerasNLP",
+    "toc": True,
+    "children": [TOKENIZERS_MASTER, LAYERS_MASTER, METRICS_MASTER],
+}
diff --git a/templates/api/keras_nlp/index.md b/templates/api/keras_nlp/index.md
@@ -0,0 +1,9 @@
+# KerasNLP
+
+The KerasNLP API is a collection of building blocks following standard
+standard Keras interfaces (e.g. layers, metrics).
+
+These building blocks can be composed to quickly and expressively define an NLP
+model or workflow.
+
+{{toc}}
diff --git a/templates/api/keras_nlp/layers/index.md b/templates/api/keras_nlp/layers/index.md
@@ -0,0 +1,8 @@
+# KerasNLP Layers
+
+KerasNLP layers are just `keras.Layer` subclasses with NLP specific use cases.
+
+These layers are building blocks for common NLP model architectures
+(e.g. transformers).
+
+{{toc}}
diff --git a/templates/api/keras_nlp/layers/preprocessing/index.md b/templates/api/keras_nlp/layers/preprocessing/index.md
@@ -0,0 +1,11 @@
+# KerasNLP Preprocessing Layers
+
+KerasNLP preprocessing layers are `keras.Layer` that help with data
+preparation--static transformations of your data that happen before the
+learnable part of your model.
+
+Preprocessing layers for NLP tasks should generally be run inside a
+[tf.data.Dataset.map](https://www.tensorflow.org/api_docs/python/tf/data/Dataset#map)
+during training.
+
+{{toc}}
diff --git a/templates/api/keras_nlp/metrics/index.md b/templates/api/keras_nlp/metrics/index.md
@@ -0,0 +1,5 @@
+# KerasNLP Metrics
+
+KerasNLP metrics are just `keras.Metric` subclasses with NLP specific use cases.
+
+{{toc}}
diff --git a/templates/api/keras_nlp/tokenizers/index.md b/templates/api/keras_nlp/tokenizers/index.md
@@ -0,0 +1,12 @@
+# KerasNLP Tokenizers
+
+Tokenizers convert raw string input into integer input suitable for a
+learnable Keras layers. They can also convert back from predicted integer
+sequences to raw string output.
+
+All tokenizers subclass `keras_nlp.tokenizers.Tokenizer` which in turn
+subclasses `keras.layers.Layer`. Tokenizers should generally be applied inside a
+[tf.data.Dataset.map](https://www.tensorflow.org/api_docs/python/tf/data/Dataset#map)
+during training, and can be included inside a `keras.Model` during inference.
+
+{{toc}}
diff --git a/templates/getting_started/ecosystem.md b/templates/getting_started/ecosystem.md
@@ -9,11 +9,24 @@ It spans a wide range of related initiatives that cover every step of the machin
 
 [KerasTuner Documentation](/keras_tuner/) - [KerasTuner GitHub repository](https://github.com/keras-team/keras-tuner)
 
-
 KerasTuner is an easy-to-use, scalable hyperparameter optimization framework that solves the pain points of hyperparameter search. Easily configure your search space with a define-by-run syntax, then leverage one of the available search algorithms to find the best hyperparameter values for your models. KerasTuner comes with Bayesian Optimization, Hyperband, and Random Search algorithms built-in, and is also designed to be easy for researchers to extend in order to experiment with new search algorithms.
 
 ---
 
+## KerasNLP
+
+[KerasNLP Documentation](/keras_nlp/) - [KerasNLP GitHub repository](https://github.com/keras-team/keras-nlp)
+
+KerasNLP is a simple and powerful API for building Natural Language
+Processing (NLP) models. KerasNLP provides modular building blocks following
+standard Keras interfaces (layers, metrics) that allow you to quickly and
+flexibly iterate on your task. Engineers working in applied NLP can leverage the
+library to assemble training and inference pipelines that are both
+state-of-the-art and production-grade. KerasNLP is maintained directly by the
+Keras team.
+
+---
+
 ## AutoKeras
 
 [AutoKeras Documentation](https://autokeras.com/) - [AutoKeras GitHub repository](https://github.com/keras-team/autokeras)

diff --git a/templates/guides/keras_nlp/index.md b/templates/guides/keras_nlp/index.md
@@ -0,0 +1,7 @@
+# KerasCV
+
+These guides cover the [KerasNLP](/keras_nlp/) library.
+
+## Available guides
+
+{{toc}}
diff --git a/templates/keras_nlp/index.md b/templates/keras_nlp/index.md
@@ -0,0 +1,100 @@
+# KerasNLP
+
+KerasNLP is a simple and powerful API for building Natural Language Processing
+(NLP) models within the Keras ecosystem.
+
+KerasNLP provides modular building blocks following standard Keras interfaces
+(layers, metrics) that allow you to quickly and flexibly iterate on your task.
+Engineers working in applied NLP can leverage the library to assemble training
+and inference pipelines that are both state-of-the-art and production-grade.
+
+KerasNLP can be understood as a horizontal extension of the Keras API:
+components are first-party Keras objects that are too specialized to be
+added to core Keras, but that receive the same level of polish as the rest of
+the Keras API.
+
+KerasNLP is also new and growing! If you are interested in contributing, please
+check out our
+[contributing guide](https://github.com/keras-team/keras-nlp/blob/master/CONTRIBUTING.md).
+
+---
+## Quick links
+
+* [Pretraining guide](/guides/keras_nlp/transformer_pretraining/)
+* [API reference](/api/keras_nlp/)
+* [GitHub repository](https://github.com/keras-team/keras-nlp)
+
+---
+## Installation
+
+KerasNLP requires **Python 3.7+** and **TensorFlow 2.9+**.
+
+Install the latest release:
+
+```
+pip install keras-nlp --upgrade
+```
+
+You can check out release notes and versions on our
+[releases page](https://github.com/keras-team/keras-nlp/releases).
+
+KerasNLP is currently in pre-release (0.y.z) development. Until version 1.0, we
+may break compatibility at any time and APIs should not be considered stable.
+
+---
+## Quick introduction
+
+The following snippet will tokenize some text, build a tiny transformer, and
+train a single batch.
+
+```python
+import keras_nlp
+import tensorflow as tf
+from tensorflow import keras
+
+# Tokenize some inputs with a binary label.
+vocab = ["[UNK]", "the", "qu", "##ick", "br", "##own", "fox", "."]
+sentences = ["The quick brown fox jumped.", "The fox slept."]
+tokenizer = keras_nlp.tokenizers.WordPieceTokenizer(
+    vocabulary=vocab,
+    sequence_length=10,
+)
+x, y = tokenizer(sentences), tf.constant([1, 0])
+
+# Create a tiny transformer.
+inputs = keras.Input(shape=(None,), dtype="int32")
+outputs = keras_nlp.layers.TokenAndPositionEmbedding(
+    vocabulary_size=len(vocab),
+    sequence_length=10,
+    embedding_dim=16,
+)(inputs)
+outputs = keras_nlp.layers.TransformerEncoder(
+    num_heads=4,
+    intermediate_dim=32,
+)(outputs)
+outputs = keras.layers.GlobalAveragePooling1D()(outputs)
+outputs = keras.layers.Dense(1, activation="sigmoid")(outputs)
+model = keras.Model(inputs, outputs)
+
+# Run a single batch of gradient descent.
+model.compile(loss="binary_crossentropy", jit_compile=True)
+model.train_on_batch(x, y)
+```
+
+To see an end-to-end example using KerasNLP, check out our guide on
+[pre-training a transfomer from scratch](/guides/keras_nlp/transformer_pretraining/).
+
+---
+## Citing KerasNLP
+
+If KerasNLP helps your research, we appreciate your citations.
+Here is the BibTeX entry:
+
+```bibtex
+@misc{kerasnlp2022,
+  title={KerasNLP},
+  author={Watson, Matthew, and Qian, Chen, and Zhu, Scott and Chollet, Fran\c{c}ois and others},
+  year={2022},
+  howpublished={\url{https://github.com/keras-team/keras-nlp}},
+}
+```