From fa2064e6b9fb9955c5bccc1e65417f3265e83e05 Mon Sep 17 00:00:00 2001 From: Luke Wood Date: Thu, 7 Oct 2021 21:12:25 -0700 Subject: [PATCH] Adds a keras_recipe on using a keras callback to evaluate a non-tensorflow metric --- .../ipynb/sklearn_metric_callbacks.ipynb | 223 ++++++++++++++++++ .../md/sklearn_metric_callbacks.md | 209 ++++++++++++++++ .../keras_recipes/sklearn_metric_callbacks.py | 141 +++++++++++ 3 files changed, 573 insertions(+) create mode 100644 examples/keras_recipes/ipynb/sklearn_metric_callbacks.ipynb create mode 100644 examples/keras_recipes/md/sklearn_metric_callbacks.md create mode 100644 examples/keras_recipes/sklearn_metric_callbacks.py diff --git a/examples/keras_recipes/ipynb/sklearn_metric_callbacks.ipynb b/examples/keras_recipes/ipynb/sklearn_metric_callbacks.ipynb new file mode 100644 index 00000000000..314f30f2100 --- /dev/null +++ b/examples/keras_recipes/ipynb/sklearn_metric_callbacks.ipynb @@ -0,0 +1,223 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text" + }, + "source": [ + "# Evaluationg and Exporting scikit-learn Metrics in a Keras callback\n", + "\n", + "**Author:** [lukewood](https://lukewood.xyz)
\n", + "**Date created:** 10/07/2021
\n", + "**Last modified:** 10/07/2021
\n", + "**Description:** Example shows how to use Keras callbacks to evaluate and export non-TensorFlow based metrics." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text" + }, + "source": [ + "[Keras callbacks](https://keras.io/api/callbacks/) allow for the execution of arbitrary\n", + "code at various stages of the Keras training process. While Keras offers first class\n", + "support for metric evaluation, Keras [metrics](https://keras.io/api/metrics/) may only\n", + "rely on TensorFlow code internally.\n", + "\n", + "While there are TensorFlow implementations of many metrics online, many metrics are\n", + "implemented using [NumPy](https://numpy.org/) or another numerical computation library.\n", + "By performing metric evaluation inside of a Keras callback, we can leverage any existing\n", + "metric, and ultimately export the result to TensorBoard." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text" + }, + "source": [ + "## Jaccard Score Metric\n", + "This example makes use of a sklearn metric,\n", + "[`sklearn.metrics.jarrard_score`](https://scikit-learn.org/stable/modules/generated/sklear\n", + "n.metrics.jaccard_score.html#sklearn.metrics.jaccard_score), and writes the result to a\n", + "TensorBoard using the `tf.summary` API.\n", + "\n", + "This template can be modified slightly to make it work with any existing sklearn metric." + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "colab_type": "code" + }, + "outputs": [], + "source": [ + "import tensorflow as tf\n", + "from sklearn.metrics import jaccard_score\n", + "import numpy as np\n", + "\n", + "\n", + "class JaccardScoreCallback(tf.keras.callbacks.Callback):\n", + " \"\"\"Computes the jaccard score and logs the results to TensorBoard.\"\"\"\n", + "\n", + " def __init__(self, model_fn, x_test, y_test, summary_writer):\n", + " self.model_fn = model_fn\n", + " self.x_test = x_test\n", + " self.y_test = y_test\n", + " self.summary_writer = summary_writer\n", + " self.keras_metric = tf.keras.metrics.Mean(\"jaccard_score\")\n", + " self.epoch = 0\n", + "\n", + " def on_epoch_end(self, batch, logs=None):\n", + " self.epoch += 1\n", + " self.keras_metric.reset_state()\n", + " predictions = self.model_fn(self.x_test)\n", + " jaccard_value = jaccard_score(\n", + " np.argmax(predictions, axis=-1), self.y_test, average=None\n", + " )\n", + " self.keras_metric.update_state(jaccard_value)\n", + " self._write_metric()\n", + "\n", + " def _write_metric(self):\n", + " with self.summary_writer.as_default():\n", + " tf.summary.scalar(\n", + " self.keras_metric.name,\n", + " self.keras_metric.result().numpy().astype(float),\n", + " step=self.epoch,\n", + " )\n", + " self.summary_writer.flush()\n", + "" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text" + }, + "source": [ + "## Sample Usage\n", + "Let's test our `JaccardScoreCallback` class with a real keras model." + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "colab_type": "code" + }, + "outputs": [], + "source": [ + "import tensorflow.keras as keras\n", + "\n", + "# Model / data parameters\n", + "num_classes = 10\n", + "input_shape = (28, 28, 1)\n", + "\n", + "# the data, split between train and test sets\n", + "(x_train, y_train), (x_test, y_test) = keras.datasets.mnist.load_data()\n", + "\n", + "# Scale images to the [0, 1] range\n", + "x_train = x_train.astype(\"float32\") / 255\n", + "x_test = x_test.astype(\"float32\") / 255\n", + "# Make sure images have shape (28, 28, 1)\n", + "x_train = np.expand_dims(x_train, -1)\n", + "x_test = np.expand_dims(x_test, -1)\n", + "print(\"x_train shape:\", x_train.shape)\n", + "print(x_train.shape[0], \"train samples\")\n", + "print(x_test.shape[0], \"test samples\")\n", + "\n", + "\n", + "# convert class vectors to binary class matrices\n", + "y_train = keras.utils.to_categorical(y_train, num_classes)\n", + "y_test = keras.utils.to_categorical(y_test, num_classes)\n", + "\n", + "import tensorflow.keras.layers as layers\n", + "\n", + "model = keras.Sequential(\n", + " [\n", + " keras.Input(shape=input_shape),\n", + " layers.Conv2D(32, kernel_size=(3, 3), activation=\"relu\"),\n", + " layers.MaxPooling2D(pool_size=(2, 2)),\n", + " layers.Conv2D(64, kernel_size=(3, 3), activation=\"relu\"),\n", + " layers.MaxPooling2D(pool_size=(2, 2)),\n", + " layers.Flatten(),\n", + " layers.Dropout(0.5),\n", + " layers.Dense(num_classes, activation=\"softmax\"),\n", + " ]\n", + ")\n", + "\n", + "model.summary()\n", + "\n", + "batch_size = 128\n", + "epochs = 15\n", + "\n", + "model.compile(loss=\"categorical_crossentropy\", optimizer=\"adam\", metrics=[\"accuracy\"])\n", + "summary_writer = tf.summary.create_file_writer(\"logs/traditional_classifier\")\n", + "callbacks = [\n", + " JaccardScoreCallback(model, x_test, np.argmax(y_test, axis=-1), summary_writer)\n", + "]\n", + "model.fit(\n", + " x_train,\n", + " y_train,\n", + " batch_size=batch_size,\n", + " epochs=epochs,\n", + " validation_split=0.1,\n", + " callbacks=callbacks,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text" + }, + "source": [ + "If you now launch a TensorBoard instance using `tensorboard --logdir=logs`, you will now\n", + "see the jaccard_score metric alongside any other exported metrics!" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text" + }, + "source": [ + "## Conclusion\n", + "Many ML practitioners and researchers rely on metrics that may not yet have a TensorFlow\n", + "implementation. Keras users can still leverage the wide variety of existing metric\n", + "implementations in other frameworks by using a Keras callback. These metrics can be\n", + "exported, viewed and analyzed in the TensorBoard like any other metric." + ] + } + ], + "metadata": { + "colab": { + "collapsed_sections": [], + "name": "sklearn_metric_callbacks", + "private_outputs": false, + "provenance": [], + "toc_visible": true + }, + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.0" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} \ No newline at end of file diff --git a/examples/keras_recipes/md/sklearn_metric_callbacks.md b/examples/keras_recipes/md/sklearn_metric_callbacks.md new file mode 100644 index 00000000000..a930eadf804 --- /dev/null +++ b/examples/keras_recipes/md/sklearn_metric_callbacks.md @@ -0,0 +1,209 @@ +# Evaluationg and Exporting scikit-learn Metrics in a Keras callback + +**Author:** [lukewood](https://lukewood.xyz)
+**Date created:** 10/07/2021
+**Last modified:** 10/07/2021
+**Description:** Example shows how to use Keras callbacks to evaluate and export non-TensorFlow based metrics. + + + [**View in Colab**](https://colab.research.google.com/github/keras-team/keras-io/blob/master/examples/keras_recipes/ipynb/sklearn_metric_callbacks.ipynb) • [**GitHub source**](https://github.com/keras-team/keras-io/blob/master/examples/keras_recipes/sklearn_metric_callbacks.py) + + + +[Keras callbacks](https://keras.io/api/callbacks/) allow for the execution of arbitrary +code at various stages of the Keras training process. While Keras offers first class +support for metric evaluation, Keras [metrics](https://keras.io/api/metrics/) may only +rely on TensorFlow code internally. + +While there are TensorFlow implementations of many metrics online, many metrics are +implemented using [NumPy](https://numpy.org/) or another numerical computation library. +By performing metric evaluation inside of a Keras callback, we can leverage any existing +metric, and ultimately export the result to TensorBoard. + +--- +## Jaccard Score Metric +This example makes use of a sklearn metric, +[`sklearn.metrics.jarrard_score`](https://scikit-learn.org/stable/modules/generated/sklear +n.metrics.jaccard_score.html#sklearn.metrics.jaccard_score), and writes the result to a +TensorBoard using the `tf.summary` API. + +This template can be modified slightly to make it work with any existing sklearn metric. + + +```python +import tensorflow as tf +from sklearn.metrics import jaccard_score +import numpy as np + + +class JaccardScoreCallback(tf.keras.callbacks.Callback): + """Computes the jaccard score and logs the results to TensorBoard.""" + + def __init__(self, model_fn, x_test, y_test, summary_writer): + self.model_fn = model_fn + self.x_test = x_test + self.y_test = y_test + self.summary_writer = summary_writer + self.keras_metric = tf.keras.metrics.Mean("jaccard_score") + self.epoch = 0 + + def on_epoch_end(self, batch, logs=None): + self.epoch += 1 + self.keras_metric.reset_state() + predictions = self.model_fn(self.x_test) + jaccard_value = jaccard_score( + np.argmax(predictions, axis=-1), self.y_test, average=None + ) + self.keras_metric.update_state(jaccard_value) + self._write_metric() + + def _write_metric(self): + with self.summary_writer.as_default(): + tf.summary.scalar( + self.keras_metric.name, + self.keras_metric.result().numpy().astype(float), + step=self.epoch, + ) + self.summary_writer.flush() + +``` + +--- +## Sample Usage +Let's test our `JaccardScoreCallback` class with a real keras model. + + +```python +import tensorflow.keras as keras + +# Model / data parameters +num_classes = 10 +input_shape = (28, 28, 1) + +# the data, split between train and test sets +(x_train, y_train), (x_test, y_test) = keras.datasets.mnist.load_data() + +# Scale images to the [0, 1] range +x_train = x_train.astype("float32") / 255 +x_test = x_test.astype("float32") / 255 +# Make sure images have shape (28, 28, 1) +x_train = np.expand_dims(x_train, -1) +x_test = np.expand_dims(x_test, -1) +print("x_train shape:", x_train.shape) +print(x_train.shape[0], "train samples") +print(x_test.shape[0], "test samples") + + +# convert class vectors to binary class matrices +y_train = keras.utils.to_categorical(y_train, num_classes) +y_test = keras.utils.to_categorical(y_test, num_classes) + +import tensorflow.keras.layers as layers + +model = keras.Sequential( + [ + keras.Input(shape=input_shape), + layers.Conv2D(32, kernel_size=(3, 3), activation="relu"), + layers.MaxPooling2D(pool_size=(2, 2)), + layers.Conv2D(64, kernel_size=(3, 3), activation="relu"), + layers.MaxPooling2D(pool_size=(2, 2)), + layers.Flatten(), + layers.Dropout(0.5), + layers.Dense(num_classes, activation="softmax"), + ] +) + +model.summary() + +batch_size = 128 +epochs = 15 + +model.compile(loss="categorical_crossentropy", optimizer="adam", metrics=["accuracy"]) +summary_writer = tf.summary.create_file_writer("logs/traditional_classifier") +callbacks = [ + JaccardScoreCallback(model, x_test, np.argmax(y_test, axis=-1), summary_writer) +] +model.fit( + x_train, + y_train, + batch_size=batch_size, + epochs=epochs, + validation_split=0.1, + callbacks=callbacks, +) +``` + +
+``` +Downloading data from https://storage.googleapis.com/tensorflow/tf-keras-datasets/mnist.npz +11493376/11490434 [==============================] - 0s 0us/step +x_train shape: (60000, 28, 28, 1) +60000 train samples +10000 test samples +WARNING:tensorflow:Please add `keras.layers.InputLayer` instead of `keras.Input` to Sequential model. `keras.Input` is intended to be used by Functional model. +Model: "sequential" +_________________________________________________________________ +Layer (type) Output Shape Param # +================================================================= +conv2d (Conv2D) (None, 26, 26, 32) 320 +_________________________________________________________________ +max_pooling2d (MaxPooling2D) (None, 13, 13, 32) 0 +_________________________________________________________________ +conv2d_1 (Conv2D) (None, 11, 11, 64) 18496 +_________________________________________________________________ +max_pooling2d_1 (MaxPooling2 (None, 5, 5, 64) 0 +_________________________________________________________________ +flatten (Flatten) (None, 1600) 0 +_________________________________________________________________ +dropout (Dropout) (None, 1600) 0 +_________________________________________________________________ +dense (Dense) (None, 10) 16010 +================================================================= +Total params: 34,826 +Trainable params: 34,826 +Non-trainable params: 0 +_________________________________________________________________ +Epoch 1/15 +422/422 [==============================] - 6s 14ms/step - loss: 0.3802 - accuracy: 0.8849 - val_loss: 0.0906 - val_accuracy: 0.9747 +Epoch 2/15 +422/422 [==============================] - 6s 14ms/step - loss: 0.1131 - accuracy: 0.9659 - val_loss: 0.0591 - val_accuracy: 0.9842 +Epoch 3/15 +422/422 [==============================] - 6s 14ms/step - loss: 0.0842 - accuracy: 0.9740 - val_loss: 0.0504 - val_accuracy: 0.9865 +Epoch 4/15 +422/422 [==============================] - 6s 13ms/step - loss: 0.0705 - accuracy: 0.9778 - val_loss: 0.0428 - val_accuracy: 0.9895 +Epoch 5/15 +422/422 [==============================] - 6s 13ms/step - loss: 0.0617 - accuracy: 0.9813 - val_loss: 0.0397 - val_accuracy: 0.9908 +Epoch 6/15 +422/422 [==============================] - 6s 13ms/step - loss: 0.0550 - accuracy: 0.9828 - val_loss: 0.0378 - val_accuracy: 0.9910 +Epoch 7/15 +422/422 [==============================] - 6s 13ms/step - loss: 0.0512 - accuracy: 0.9841 - val_loss: 0.0348 - val_accuracy: 0.9897 +Epoch 8/15 +422/422 [==============================] - 6s 13ms/step - loss: 0.0467 - accuracy: 0.9856 - val_loss: 0.0326 - val_accuracy: 0.9918 +Epoch 9/15 +422/422 [==============================] - 6s 13ms/step - loss: 0.0422 - accuracy: 0.9864 - val_loss: 0.0315 - val_accuracy: 0.9920 +Epoch 10/15 +422/422 [==============================] - 6s 13ms/step - loss: 0.0410 - accuracy: 0.9868 - val_loss: 0.0302 - val_accuracy: 0.9918 +Epoch 11/15 +422/422 [==============================] - 6s 13ms/step - loss: 0.0366 - accuracy: 0.9884 - val_loss: 0.0311 - val_accuracy: 0.9908 +Epoch 12/15 +422/422 [==============================] - 6s 13ms/step - loss: 0.0362 - accuracy: 0.9879 - val_loss: 0.0304 - val_accuracy: 0.9922 +Epoch 13/15 +422/422 [==============================] - 6s 13ms/step - loss: 0.0354 - accuracy: 0.9885 - val_loss: 0.0294 - val_accuracy: 0.9927 +Epoch 14/15 +422/422 [==============================] - 6s 13ms/step - loss: 0.0325 - accuracy: 0.9894 - val_loss: 0.0322 - val_accuracy: 0.9917 +Epoch 15/15 +422/422 [==============================] - 6s 13ms/step - loss: 0.0305 - accuracy: 0.9899 - val_loss: 0.0277 - val_accuracy: 0.9930 + + + +``` +
+If you now launch a TensorBoard instance using `tensorboard --logdir=logs`, you will now +see the jaccard_score metric alongside any other exported metrics! + +--- +## Conclusion +Many ML practitioners and researchers rely on metrics that may not yet have a TensorFlow +implementation. Keras users can still leverage the wide variety of existing metric +implementations in other frameworks by using a Keras callback. These metrics can be +exported, viewed and analyzed in the TensorBoard like any other metric. diff --git a/examples/keras_recipes/sklearn_metric_callbacks.py b/examples/keras_recipes/sklearn_metric_callbacks.py new file mode 100644 index 00000000000..fd073521089 --- /dev/null +++ b/examples/keras_recipes/sklearn_metric_callbacks.py @@ -0,0 +1,141 @@ +""" +Title: Evaluationg and Exporting scikit-learn Metrics in a Keras callback +Author: [lukewood](https://lukewood.xyz) +Date created: 10/07/2021 +Last modified: 10/07/2021 +Description: Example shows how to use Keras callbacks to evaluate and export non-TensorFlow based metrics. +""" +""" +[Keras callbacks](https://keras.io/api/callbacks/) allow for the execution of arbitrary +code at various stages of the Keras training process. While Keras offers first class +support for metric evaluation, Keras [metrics](https://keras.io/api/metrics/) may only +rely on TensorFlow code internally. + +While there are TensorFlow implementations of many metrics online, many metrics are +implemented using [NumPy](https://numpy.org/) or another numerical computation library. +By performing metric evaluation inside of a Keras callback, we can leverage any existing +metric, and ultimately export the result to TensorBoard. +""" + +""" +## Jaccard Score Metric +This example makes use of a sklearn metric, +[`sklearn.metrics.jarrard_score`](https://scikit-learn.org/stable/modules/generated/sklear +n.metrics.jaccard_score.html#sklearn.metrics.jaccard_score), and writes the result to a +TensorBoard using the `tf.summary` API. + +This template can be modified slightly to make it work with any existing sklearn metric. +""" + +import tensorflow as tf +from sklearn.metrics import jaccard_score +import numpy as np + + +class JaccardScoreCallback(tf.keras.callbacks.Callback): + """Computes the jaccard score and logs the results to TensorBoard.""" + + def __init__(self, model_fn, x_test, y_test, summary_writer): + self.model_fn = model_fn + self.x_test = x_test + self.y_test = y_test + self.summary_writer = summary_writer + self.keras_metric = tf.keras.metrics.Mean("jaccard_score") + self.epoch = 0 + + def on_epoch_end(self, batch, logs=None): + self.epoch += 1 + self.keras_metric.reset_state() + predictions = self.model_fn(self.x_test) + jaccard_value = jaccard_score( + np.argmax(predictions, axis=-1), self.y_test, average=None + ) + self.keras_metric.update_state(jaccard_value) + self._write_metric() + + def _write_metric(self): + with self.summary_writer.as_default(): + tf.summary.scalar( + self.keras_metric.name, + self.keras_metric.result().numpy().astype(float), + step=self.epoch, + ) + self.summary_writer.flush() + + +""" +## Sample Usage +Let's test our `JaccardScoreCallback` class with a real keras model. +""" + +import tensorflow.keras as keras + +# Model / data parameters +num_classes = 10 +input_shape = (28, 28, 1) + +# the data, split between train and test sets +(x_train, y_train), (x_test, y_test) = keras.datasets.mnist.load_data() + +# Scale images to the [0, 1] range +x_train = x_train.astype("float32") / 255 +x_test = x_test.astype("float32") / 255 +# Make sure images have shape (28, 28, 1) +x_train = np.expand_dims(x_train, -1) +x_test = np.expand_dims(x_test, -1) +print("x_train shape:", x_train.shape) +print(x_train.shape[0], "train samples") +print(x_test.shape[0], "test samples") + + +# convert class vectors to binary class matrices +y_train = keras.utils.to_categorical(y_train, num_classes) +y_test = keras.utils.to_categorical(y_test, num_classes) + +import tensorflow.keras.layers as layers + +model = keras.Sequential( + [ + keras.Input(shape=input_shape), + layers.Conv2D(32, kernel_size=(3, 3), activation="relu"), + layers.MaxPooling2D(pool_size=(2, 2)), + layers.Conv2D(64, kernel_size=(3, 3), activation="relu"), + layers.MaxPooling2D(pool_size=(2, 2)), + layers.Flatten(), + layers.Dropout(0.5), + layers.Dense(num_classes, activation="softmax"), + ] +) + +model.summary() + +batch_size = 128 +epochs = 15 + +model.compile(loss="categorical_crossentropy", optimizer="adam", metrics=["accuracy"]) +summary_writer = tf.summary.create_file_writer("logs/traditional_classifier") +callbacks = [ + JaccardScoreCallback(model, x_test, np.argmax(y_test, axis=-1), summary_writer) +] +model.fit( + x_train, + y_train, + batch_size=batch_size, + epochs=epochs, + validation_split=0.1, + callbacks=callbacks, +) + +""" +If you now launch a TensorBoard instance using `tensorboard --logdir=logs`, you will now +see the jaccard_score metric alongside any other exported metrics! + +""" + +""" +## Conclusion +Many ML practitioners and researchers rely on metrics that may not yet have a TensorFlow +implementation. Keras users can still leverage the wide variety of existing metric +implementations in other frameworks by using a Keras callback. These metrics can be +exported, viewed and analyzed in the TensorBoard like any other metric. +"""