Add code for python visualization service

.travis.yml

@@ -73,6 +73,13 @@ matrix:
         - python tests/compiler/main.py
         - $TRAVIS_BUILD_DIR/sdk/python/tests/run_tests.sh
 
+        # Visualization test
+        - cd $TRAVIS_BUILD_DIR/backend/src/apiserver/visualization
+        # Visualization test dependencies
+        - pip3 install -r requirements-test.txt
+        - python3 test_exporter.py
+        - python3 test_server.py
+
         # Test loading all component.yaml definitions
         - $TRAVIS_BUILD_DIR/components/test_load_all_components.sh
 

backend/Dockerfile.visualization

@@ -0,0 +1,31 @@
+# Copyright 2019 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM python:3
+
+RUN curl https://dl.google.com/dl/cloudsdk/release/google-cloud-sdk.tar.gz > /tmp/google-cloud-sdk.tar.gz
+RUN mkdir -p /usr/local/gcloud
+RUN tar -C /usr/local/gcloud -xf /tmp/google-cloud-sdk.tar.gz
+RUN /usr/local/gcloud/google-cloud-sdk/install.sh
+ENV PATH $PATH:/usr/local/gcloud/google-cloud-sdk/bin
+
+WORKDIR /src
+
+COPY src/apiserver/visualization /src
+
+RUN pip3 install -r requirements.txt
+
+RUN ./license.sh third_party_licenses.csv /usr/licenses
+
+ENTRYPOINT [ "python3", "server.py" ]

backend/src/apiserver/visualization/.gitignore

@@ -0,0 +1,5 @@
+*.html
+*.csv
+*.json
+__init__.py
+!third_party_licenses.csv

backend/src/apiserver/visualization/exporter.py

@@ -0,0 +1,119 @@
+"""
+exporter.py provides utility functions for generating NotebookNode objects and
+converting those objects to HTML.
+"""
+
+# Copyright 2019 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+from enum import Enum
+import json
+from pathlib import Path
+from typing import Text
+from jupyter_client import KernelManager
+from nbconvert import HTMLExporter
+from nbconvert.preprocessors import ExecutePreprocessor
+from nbformat import NotebookNode
+from nbformat.v4 import new_code_cell
+
+
+# Visualization Template types:
+# - Basic: Uses the basic.tpl file within the templates directory to generate
+# a visualization that contains no styling and minimal HTML tags. This is ideal
+# for testing as it reduces size of generated visualization. However, for usage
+# with actual visualizations it is not ideal due to its lack of javascript and
+# styling which can limit usability of a visualization.
+# - Full: Uses the full.tpl file within the template directory to generate a
+# visualization that can be viewed as a standalone web page. The full.tpl file
+# utilizes the basic.tpl file for visualizations then wraps that output with
+# additional tags for javascript and style support. This is ideal for generating
+# visualizations that will be displayed via the frontend.
+class TemplateType(Enum):
+    basic = 'basic'
+    full = 'full'
+
+
+class Exporter:
+
+    def __init__(
+        self,
+        timeout: int = 100,
+        template_type: TemplateType = TemplateType.full
+    ):
+        self.timeout = timeout
+        self.template_type = template_type
+        # Create custom KernelManager.
+        # This will circumvent issues where kernel is shutdown after
+        # preprocessing. Due to the shutdown, latency would be introduced
+        # because a kernel must be started per visualization.
+        self.km = KernelManager()
+        self.km.start_kernel()
+        self.ep = ExecutePreprocessor(
+            timeout=self.timeout,
+            kernel_name='python3')
+
+    def __del__(self):
+        self.shutdown_kernel()
+
+    # Takes provided command line arguments and creates a Notebook cell object
+    # with the arguments as variables.
+    #
+    # Returns the generated Notebook cell.
+    @staticmethod
+    def create_cell_from_args(args: argparse.Namespace) -> NotebookNode:
+        variables = ""
+        args = json.loads(args)
+        for key in sorted(args.keys()):
+            # Check type of variable to maintain type when converting from JSON
+            # to notebook cell
+            if args[key] is None or isinstance(args[key], bool):
+                variables += "{} = {}\n".format(key, args[key])
+            else:
+                variables += '{} = "{}"\n'.format(key, args[key])
+
+        return new_code_cell(variables)
+
+    # Reads a python file, then creates a Notebook cell object with the
+    # lines of code from the python file.
+    #
+    # Returns the generated Notebook cell.
+    @staticmethod
+    def create_cell_from_file(filepath: Text) -> NotebookNode:
+        with open(filepath, 'r') as f:
+            code = f.read()
+
+        return new_code_cell(code)
+
+    # Exports a notebook to HTML and generates any required outputs.
+    #
+    # Returns the generated HTML as a string.
+    def generate_html_from_notebook(
+            self,
+            nb: NotebookNode,
+            template_type: TemplateType = TemplateType.full
+    ) -> Text:
+        # HTML generator and exporter object
+        html_exporter = HTMLExporter()
+        template_file = "templates/{}.tpl".format(template_type.value)
+        html_exporter.template_file = str(Path.cwd() / template_file)
+        # Output generator
+        self.ep.preprocess(nb, {"metadata": {"path": Path.cwd()}}, self.km)
+
+        # Export all html and outputs
+        body, _ = html_exporter.from_notebook_node(nb)
+        return body
+
+    def shutdown_kernel(self):
+        self.km.shutdown_kernel()

backend/src/apiserver/visualization/license.sh

@@ -0,0 +1,63 @@
+#!/bin/bash -e
+# Copyright 2019 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+# A script to gather locally-installed python packages, and based on
+# a specified license table (3 columns: name,license_link,license_type) csv file,
+# download and save all license files into specified directory.
+# Usage:
+#   license.sh third_party_licenses.csv /usr/licenses
+
+
+# Get the list of python packages installed locally.
+IFS=$'\n'
+INSTALLED_PACKAGES=($(pip freeze | sed s/=.*//))
+
+
+# Get the list of python packages tracked in the given CSV file.
+REGISTERED_PACKAGES=()
+while IFS=, read -r col1 col2 col3
+do
+  REGISTERED_PACKAGES+=($col1)
+done < $1
+
+# Make sure all locally installed packages are covered.
+DIFF=()
+for i in "${INSTALLED_PACKAGES[@]}"; do
+  skip=
+  for j in "${REGISTERED_PACKAGES[@]}"; do
+    [[ $i == $j ]] && { skip=1; break; }
+  done
+  [[ -n $skip ]] || DIFF+=("$i")
+done
+
+if [ -n "$DIFF" ]; then
+  echo "The following packages are not found for licenses tracking."
+  echo "Please add an entry in $1 for each of them."
+  echo ${DIFF[@]}
+  exit 1
+fi
+
+# Gather license files for each package. For packages with GPL license we mirror the source code.
+mkdir -p $2/source
+while IFS=, read -r col1 col2 col3
+do
+  if [[ " ${INSTALLED_PACKAGES[@]} " =~ " ${col1} " ]]; then
+    wget -O $2/$col1.LICENSE $col2
+    if [[ "${col3}" == *GPL* ]] || [[ "${col3}" =~ ^MPL ]]; then
+      pip install -t "$2/source/${col1}" ${col1}
+    fi
+  fi
+done < $1

backend/src/apiserver/visualization/requirements-test.txt

@@ -0,0 +1,6 @@
+ipykernel==5.1.1
+jupyter_client==5.2.4
+nbconvert==5.5.0
+nbformat==4.4.0
+snapshottest==0.5.1
+tornado==6.0.2

backend/src/apiserver/visualization/requirements.txt

@@ -0,0 +1,13 @@
+bokeh==1.2.0
+gcsfs==0.2.3
+google-api-python-client==1.7.9
+ipykernel==5.1.1
+jupyter_client==5.2.4
+nbconvert==5.5.0
+nbformat==4.4.0
+pandas==0.24.2
+scikit_learn==0.21.2
+tensorflow==1.13.1
+tensorflow-data-validation==0.13.1
+tensorflow-model-analysis==0.13.1
+tornado==6.0.2

backend/src/apiserver/visualization/roc_curve.py

@@ -0,0 +1,61 @@
+# Copyright 2019 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+from pathlib import Path
+from bokeh.layouts import row
+from bokeh.plotting import figure
+from bokeh.io import output_notebook, show
+from bokeh.models import HoverTool
+# gcsfs is required for pandas GCS integration.
+import gcsfs
+import pandas as pd
+from sklearn.metrics import roc_curve
+from tensorflow.python.lib.io import file_io
+
+if is_generated is False:
+    # Create data from specified csv file(s).
+    # The schema file provides column names for the csv file that will be used
+    # to generate the roc curve.
+    schema_file = Path(input_path) / 'schema.json'
+    schema = json.loads(file_io.read_file_to_string(schema_file))
+    names = [x['name'] for x in schema]
+
+    dfs = []
+    files = file_io.get_matching_files(input_path)
+    for f in files:
+        dfs.append(pd.read_csv(f, names=names))
+
+    df = pd.concat(dfs)
+    if target_lambda:
+        df['target'] = df.apply(eval(target_lambda), axis=1)
+    else:
+        df['target'] = df['target'].apply(lambda x: 1 if x == trueclass else 0)
+    fpr, tpr, thresholds = roc_curve(df['target'], df[true_score_column])
+    source = pd.DataFrame({'fpr': fpr, 'tpr': tpr, 'thresholds': thresholds})
+else:
+    # Load data from generated csv file.
+    source = pd.read_csv(input_path, header=None, names=['fpr', 'tpr',
+                                                         'thresholds'])
+
+# Create visualization.
+output_notebook()
+
+p = figure(tools="pan,wheel_zoom,box_zoom,reset,hover,previewsave")
+p.line('fpr', 'tpr', line_width=2, source=source)
+
+hover = p.select(dict(type=HoverTool))
+hover.tooltips = [("Threshold", "@thresholds")]
+
+show(row(p, sizing_mode="scale_width"))