Merge branch 'master' into removing-ner-duckling

CHANGELOG.rst

@@ -23,10 +23,13 @@ Changed
   for both entities and intents
 - use cloudpickle version 0.6.1
 - replaced ``yaml`` with ``ruamel.yaml``
+- updated spacy version to 2.0.18
+- updated TensorFlow version to 1.12.0
 
 Removed
 -------
 - ``/config`` endpoint
+- removed pinning of ``msgpack`` and unused package ``python-msgpack``
 
 Fixed
 -----
@@ -35,6 +38,9 @@ Fixed
 - NLU Server can now handle training data mit emojis in it
 - If the ``token_name`` is not given in the endpoint configuration, the default
   value is ``token`` instead of ``None`
+- Updated CORS support for the server.
+  Added the ``Access-Control-Allow-Headers`` and ``Content-Type`` headers for nlu server
+- parsing of emojis which are sent within jsons
 
 [0.13.8] - 2018-11-21
 ^^^^^^^^^^^^^^^^^^^^^

alt_requirements/requirements_bare.txt

@@ -11,8 +11,6 @@ tqdm==4.19.5
 numpy==1.14.5
 simplejson==3.13.2
 cloudpickle==0.6.1
-msgpack-python==0.5.4
 packaging==17.1
 ruamel.yaml==0.15.78
 coloredlogs==9.0
-msgpack==0.5.6

alt_requirements/requirements_spacy_sklearn.txt

@@ -1,7 +1,7 @@
 # Minimum Install Requirements
 -r requirements_bare.txt
 
-spacy==2.0.11
+spacy==2.0.18
 scikit-learn==0.19.1
 scipy==1.1.0
-sklearn-crfsuite==0.3.6
+sklearn-crfsuite==0.3.6

alt_requirements/requirements_tensorflow_sklearn.txt

@@ -2,6 +2,6 @@
 -r requirements_bare.txt
 
 scikit-learn==0.19.1
-tensorflow==1.10.0
+tensorflow==1.12.0
 scipy==1.1.0
 sklearn-crfsuite==0.3.6

docker/Dockerfile_full

@@ -36,7 +36,7 @@ RUN pip install https://github.com/explosion/spacy-models/releases/download/en_c
 
 COPY sample_configs/config_spacy_duckling.yml ${RASA_NLU_HOME}/config.yml
 
-#VOLUME ["/app/projects", "/app/logs", "/app/data"]
+VOLUME ["/app/projects", "/app/logs", "/app/data"]
 
 EXPOSE 5000
 

docker/docker-cloud.yml

@@ -5,3 +5,4 @@ rasanlu:
   volumes:
    - "/rasa-app-data/projects:/app/projects"
    - "/rasa-app-data/logs:/app/logs"
+   - "./rasa-app-data/data:/app/data"

docker/docker-compose.yml

@@ -9,6 +9,7 @@ services:
     volumes:
     - "./rasa-app-data/projects:/app/projects"
     - "./rasa-app-data/logs:/app/logs"
+    - "./rasa-app-data/data:/app/data"
   duckling:
     image: rasa/duckling:latest
     networks: ['rasa-network']

rasa_nlu/server.py

@@ -113,8 +113,20 @@ def decorated(*args, **kwargs):
         if origin:
             if '*' in self.cors_origins:
                 request.setHeader('Access-Control-Allow-Origin', '*')
+                request.setHeader(
+                    'Access-Control-Allow-Headers',
+                    'Content-Type')
+                request.setHeader(
+                    'Access-Control-Allow-Methods',
+                    'POST, GET, OPTIONS, PUT, DELETE')
             elif origin in self.cors_origins:
                 request.setHeader('Access-Control-Allow-Origin', origin)
+                request.setHeader(
+                    'Access-Control-Allow-Headers',
+                    'Content-Type')
+                request.setHeader(
+                    'Access-Control-Allow-Methods',
+                    'POST, GET, OPTIONS, PUT, DELETE')
             else:
                 request.setResponseCode(403)
                 return 'forbidden'

rasa_nlu/utils/__init__.py

@@ -24,6 +24,11 @@
 from future.utils import PY3
 from requests.auth import HTTPBasicAuth
 
+# Regular expression to test if string contains emoji code
+unicode_regex = re.compile(
+    u'[\u231A-\u231B\u2328\u23CF\23E9-\u23F3...\U0001F9C0]',
+    flags=re.UNICODE)
+
 
 def add_logging_option_arguments(parser, default=logging.WARNING):
     """Add options to an argument parser to configure logging levels."""
@@ -247,17 +252,33 @@ def env_var_constructor(loader, node):
 
 
 def read_yaml(content):
+    """Parses yaml from a text.
+
+     Args:
+        content: A text containing yaml content.
+    """
     fix_yaml_loader()
     replace_environment_variables()
 
     yaml_parser = yaml.YAML(typ="safe")
     yaml_parser.version = "1.2"
     yaml_parser.unicode_supplementary = True
 
+    if unicode_regex.match(content):
+        content = (content.encode('utf-8')
+                   .decode('unicode_escape')
+                   .encode("utf-16", 'surrogatepass')
+                   .decode('utf-16'))
+
     return yaml_parser.load(content)
 
 
 def read_yaml_file(filename):
+    """Parses a yaml file.
+
+     Args:
+        filename: The path to the file which should be read.
+    """
     return read_yaml(read_file(filename, "utf-8"))
 
 

setup.py

@@ -14,47 +14,45 @@
     long_description = f.read()
 
 tests_requires = [
-    "pytest",
-    "pytest-pep8",
-    "pytest-cov",
+    "pytest~=3.3",
+    "pytest-pep8~=1.0",
+    "pytest-cov~=2.5",
     "pytest-twisted<1.6",
-    "treq",
+    "treq~=17.8",
     "responses~=0.9.0",
     "httpretty~=0.9.0",
 ]
 
 install_requires = [
-    "pathlib",
-    "cloudpickle",
-    "gevent",
-    "klein",
-    "boto3",
-    "packaging",
-    "typing",
-    "future",
-    "six",
-    "tqdm",
-    "requests",
-    "jsonschema",
+    "cloudpickle~=0.6.1",
+    "gevent~=1.2",
+    "klein~=17.10",
+    "boto3~=1.5",
+    "packaging~=17.1",
+    "typing~=3.6",
+    "future~=0.16.0",
+    "six~=1.11",
+    "tqdm~=4.19",
+    "requests~=2.20",
+    "jsonschema~=2.6",
     "matplotlib~=2.0",
     "numpy>=1.13",
-    "simplejson",
-    "ruamel.yaml~=0.15",
-    "coloredlogs",
-    "msgpack>=0.3.0,<0.6"
+    "simplejson~=3.13",
+    "ruamel.yaml~=0.15.7",
+    "coloredlogs~=9.0",
 ]
 
 extras_requires = {
     'test': tests_requires,
     'spacy': ["scikit-learn<0.20",
-              "sklearn-crfsuite",
-              "scipy",
-              "spacy<=2.0.12,>2.0",
+              "sklearn-crfsuite~=0.3.6",
+              "scipy~=1.1",
+              "spacy<=2.0.18,>2.0",
               ],
     'tensorflow': ["scikit-learn<0.20",
-                   "sklearn-crfsuite",
-                   "scipy",
-                   "tensorflow"
+                   "sklearn-crfsuite~=0.3.6",
+                   "scipy~=1.1",
+                   "tensorflow~=1.12"
                    ],
     'mitie': ["mitie"],
 }

tests/base/test_utils.py

@@ -200,30 +200,41 @@ def test_environment_variable_dict_with_prefix_and_with_postfix():
 def test_emojis_in_yaml():
     test_data = """
     data:
-        - one 😁
+        - one 😁💯 👩🏿‍💻👨🏿‍💻
         - two £
     """
     actual = utils.read_yaml(test_data)
 
-    assert actual["data"][0] == "one 😁"
+    assert actual["data"][0] == "one 😁💯 👩🏿‍💻👨🏿‍💻"
     assert actual["data"][1] == "two £"
 
 
 def test_emojis_in_tmp_file():
     test_data = """
         data:
-            - one 😁
+            - one 😁💯 👩🏿‍💻👨🏿‍💻
             - two £
         """
     test_file = utils.create_temporary_file(test_data)
     with io.open(test_file, mode='r', encoding="utf-8") as f:
         content = f.read()
     actual = utils.read_yaml(content)
 
-    assert actual["data"][0] == "one 😁"
+    assert actual["data"][0] == "one 😁💯 👩🏿‍💻👨🏿‍💻"
     assert actual["data"][1] == "two £"
 
 
+def test_read_emojis_from_json():
+    import json
+    from rasa_nlu.utils import read_yaml
+    d = {"text": "hey 😁💯 👩🏿‍💻👨🏿‍💻🧜‍♂️"}
+    json_string = json.dumps(d, indent=2)
+
+    s = read_yaml(json_string)
+
+    assert s.get('text') == "hey 😁💯 👩🏿‍💻👨🏿‍💻🧜‍♂️"
+
+
 def test_bool_str():
     test_data = """
     one: "yes"