Merged PR 2450: Added embeddings service code.

Added embeddings service code.
We will potentially create a common lib to hold the code shared with SVM, but only if we decide to use both backends.

Related work items: #4441

Author: Pedro Teixeira

.gitignore

@@ -1,101 +1,69 @@
-# Byte-compiled / optimized / DLL files
-__pycache__/
-*.py[cod]
-*$py.class
-
-# C extensions
-*.so
-
-# Distribution / packaging
-.Python
-env/
-build/
-develop-eggs/
-dist/
-downloads/
-eggs/
-.eggs/
-lib/
-lib64/
-parts/
-sdist/
-var/
-wheels/
-*.egg-info/
-.installed.cfg
-*.egg
-
-# PyInstaller
-#  Usually these files are written by a python script from a template
-#  before PyInstaller builds the exe, so as to inject date/other infos into it.
-*.manifest
-*.spec
-
-# Installer logs
-pip-log.txt
-pip-delete-this-directory.txt
+# VS code
+.vscode/
 
-# Unit test / coverage reports
-htmlcov/
-.tox/
-.coverage
-.coverage.*
-.cache
-nosetests.xml
-coverage.xml
-*.cover
-.hypothesis/
+bin/
 
-# Translations
-*.mo
-*.pot
+# ---- PYTHON ----
+# Compiled Python
+*.pyc
+.cache/
+.pytest_cache/
 
-# Django stuff:
-*.log
-local_settings.py
+# Package
+*.egg-info/
+venv/
 
-# Flask stuff:
-instance/
-.webassets-cache
+**/TEST*.xml
 
-# Scrapy stuff:
-.scrapy
+### Intellij ###
+# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and Webstorm
+# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839
 
-# Sphinx documentation
-docs/_build/
+# User-specific stuff:
+**/.idea/workspace.xml
+**/.idea/tasks.xml
+**/.idea/dictionaries
+**/.idea/vcs.xml
+**/.idea/jsLibraryMappings.xml
 
-# PyBuilder
-target/
+# Sensitive or high-churn files:
+**/.idea/dataSources.ids
+**/.idea/dataSources.xml
+**/.idea/dataSources.local.xml
+**/.idea/sqlDataSources.xml
+**/.idea/dynamic.xml
+**/.idea/uiDesigner.xml
+**/.idea/inspectionProfiles/
 
-# Jupyter Notebook
-.ipynb_checkpoints
+# Gradle:
+**/.idea/gradle.xml
+**/.idea/libraries
 
-# pyenv
-.python-version
+# Mongo Explorer plugin:
+**/.idea/mongoSettings.xml
 
-# celery beat schedule file
-celerybeat-schedule
+## File-based project format:
+**/*.iws
 
-# SageMath parsed files
-*.sage.py
+## Plugin-specific files:
 
-# dotenv
-.env
+# IntelliJ
+/out/
 
-# virtualenv
-.venv
-venv/
-ENV/
+# mpeltonen/sbt-idea plugin
+**/.idea_modules/
 
-# Spyder project settings
-.spyderproject
-.spyproject
+# JIRA plugin
+atlassian-ide-plugin.xml
 
-# Rope project settings
-.ropeproject
+# Crashlytics plugin (for Android Studio and IntelliJ)
+com_crashlytics_export_strings.xml
+crashlytics.properties
+crashlytics-build.properties
+fabric.properties
 
-# mkdocs documentation
-/site
+### Intellij Patch ###
+# Comment Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-215987721
 
-# mypy
-.mypy_cache/
+# *.iml
+# modules.xml

src/.flake8

@@ -0,0 +1,3 @@
+[flake8]
+max-line-length=99
+max-complexity=10

src/.style.yapf

@@ -0,0 +1,3 @@
+[style]
+based_on_style = pep8
+column_limit = 79

src/Dockerfile

@@ -0,0 +1,41 @@
+# Base off of the Python image
+FROM python:3.5.2-slim
+LABEL maintainer "Pedro Teixeira <pedrotei@hutoma.com>"
+
+RUN apt-get update \
+ && apt install -y --no-install-recommends \
+  build-essential \
+ && rm -rf /var/lib/apt/lists/*
+
+ RUN pip install --no-cache-dir --upgrade pip
+# Copy common code
+COPY requirements.txt .
+RUN pip install --no-cache-dir --upgrade -r requirements.txt
+
+# Run script to download models for languages we require
+RUN python -m spacy download en
+
+
+# Copy the code
+COPY embedding/ /src/embedding/
+
+#---------------------------
+# switch to non root user
+# define user/group IDs as ARG
+ARG USERID=1000
+ARG GROUPID=1000
+RUN addgroup --system --gid $GROUPID appuser
+RUN adduser --system --uid $USERID --gid $GROUPID appuser
+
+USER appuser
+WORKDIR /home/appuser
+
+# Get the NLTK corpus data
+RUN python -m nltk.downloader stopwords
+
+ENV EMB_SERVER_PORT 9090
+
+
+EXPOSE 9090
+
+CMD ["python3", "/src/embedding/server.py"]

src/embedding/__init__.py

@@ -0,0 +1,80 @@
+"""SVCLASSIFIER chat worker processes"""
+
+import logging
+import numpy
+
+import ai_training.chat_process as ait_c
+from spacy_wrapper import SpacyWrapper
+from text_classifier_class import EmbeddingComparison
+from word2vec_client import Word2VecClient
+from svc_config import SvcConfig
+import aiohttp
+
+MODEL_FILE = "model.pkl"
+
+
+def _get_logger():
+    logger = logging.getLogger('embedding.chat')
+    return logger
+
+
+class EmbeddingChatProcessWorker(ait_c.ChatProcessWorkerABC):
+
+    __spacy_wrapper = None
+
+    def __init__(self, pool, asyncio_loop):
+        super().__init__(pool, asyncio_loop)
+        self.chatter = None
+        self.chat_args = None
+        self.ai = None
+        self.is_ready = False
+        self.logger = _get_logger()
+        self.w2v_client = Word2VecClient(
+            SvcConfig.get_instance().w2v_server_url)
+
+    async def __aenter__(self):
+        return self
+
+    async def __aexit__(self, exc_type, exc, traceb):
+        if self.chatter is not None:
+            self.chatter.__exit__(exc_type, exc, traceb)
+
+    async def start_chat(self, msg: ait_c.WakeChatMessage):
+        """Handle a wake request"""
+
+    async def chat_request(self, msg: ait_c.ChatRequestMessage):
+        """Handle a chat request"""
+        if msg.update_state or self.chatter is None:
+            self.setup_chat_session()
+
+        if self.is_ready:
+            words = msg.question.split(' ')
+            list = []
+            list.append(msg.question)
+            x_tokens_testset = [
+                EmbeddingChatProcessWorker.__spacy_wrapper.tokenizeSpacy(s)
+                for s in list
+            ]
+            try:
+                vecs = await self.w2v_client.get_vectors_for_words(words)
+            except aiohttp.client_exceptions.ClientConnectorError as exc:
+                self.logger.warn(
+                    "Could not receive response from w2v service - {}".format(
+                        exc))
+                return ait_c.ChatResponseMessage(msg, None, 0.0)
+
+            cls = EmbeddingComparison(w2v=vecs)
+            cls.load_model(self.ai_path + "/" + MODEL_FILE)
+            yPred, yProbs = cls.predict(x_tokens_testset)
+            resp = ait_c.ChatResponseMessage(msg, yPred[0], yProbs[0])
+            return resp
+
+        resp = ait_c.ChatResponseMessage(msg, None, 0.0)
+        return resp
+
+    def setup_chat_session(self):
+
+        if EmbeddingChatProcessWorker.__spacy_wrapper is None:
+            EmbeddingChatProcessWorker.__spacy_wrapper = SpacyWrapper()
+
+        self.is_ready = True