Skip to content

Commit

Permalink
Merge branch 'removing-ner-duckling' of github.com:RasaHQ/rasa_nlu in…
Browse files Browse the repository at this point in the history
…to removing-ner-duckling
  • Loading branch information
EPedrotti committed Jan 21, 2019
2 parents 42740ff + 99e59d0 commit e5703f2
Show file tree
Hide file tree
Showing 12 changed files with 105 additions and 41 deletions.
14 changes: 14 additions & 0 deletions .travis.yml
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,20 @@ jobs:
- cd rasa-website
- git commit --allow-empty -m "trigger nlu docs update"
- git push origin master
- stage: Test starter packs
name: "NLU starter pack"
python: 3.6
script:
- git clone https://github.com/RasaHQ/starter-pack-rasa-nlu.git
- cd starter-pack-rasa-nlu
- python -m pytest tests/test_nlu.py
- stage: Test starter packs
name: "Stack starter pack (NLU only)"
python: 3.6
script:
- git clone https://github.com/RasaHQ/starter-pack-rasa-stack.git
- cd starter-pack-rasa-stack
- python -m pytest tests/test_nlu.py
- stage: deploy
python: 3.6
install: skip
Expand Down
7 changes: 7 additions & 0 deletions CHANGELOG.rst
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ Added
- more documentation on how to run NLU with Docker
- ``analyzer`` parameter to ``intent_featurizer_count_vectors`` featurizer to
configure whether to use word or character n-grams
- Travis script now clones and tests the Rasa NLU starter pack

Changed
-------
Expand All @@ -23,10 +24,13 @@ Changed
for both entities and intents
- use cloudpickle version 0.6.1
- replaced ``yaml`` with ``ruamel.yaml``
- updated spacy version to 2.0.18
- updated TensorFlow version to 1.12.0

Removed
-------
- ``/config`` endpoint
- removed pinning of ``msgpack`` and unused package ``python-msgpack``

Fixed
-----
Expand All @@ -35,6 +39,9 @@ Fixed
- NLU Server can now handle training data mit emojis in it
- If the ``token_name`` is not given in the endpoint configuration, the default
value is ``token`` instead of ``None`
- Updated CORS support for the server.
Added the ``Access-Control-Allow-Headers`` and ``Content-Type`` headers for nlu server
- parsing of emojis which are sent within jsons

[0.13.8] - 2018-11-21
^^^^^^^^^^^^^^^^^^^^^
Expand Down
2 changes: 0 additions & 2 deletions alt_requirements/requirements_bare.txt
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,6 @@ tqdm==4.19.5
numpy==1.14.5
simplejson==3.13.2
cloudpickle==0.6.1
msgpack-python==0.5.4
packaging==17.1
ruamel.yaml==0.15.78
coloredlogs==9.0
msgpack==0.5.6
4 changes: 2 additions & 2 deletions alt_requirements/requirements_spacy_sklearn.txt
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
# Minimum Install Requirements
-r requirements_bare.txt

spacy==2.0.11
spacy==2.0.18
scikit-learn==0.19.1
scipy==1.1.0
sklearn-crfsuite==0.3.6
sklearn-crfsuite==0.3.6
2 changes: 1 addition & 1 deletion alt_requirements/requirements_tensorflow_sklearn.txt
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,6 @@
-r requirements_bare.txt

scikit-learn==0.19.1
tensorflow==1.10.0
tensorflow==1.12.0
scipy==1.1.0
sklearn-crfsuite==0.3.6
2 changes: 1 addition & 1 deletion docker/Dockerfile_full
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ RUN pip install https://github.com/explosion/spacy-models/releases/download/en_c

COPY sample_configs/config_spacy_duckling.yml ${RASA_NLU_HOME}/config.yml

#VOLUME ["/app/projects", "/app/logs", "/app/data"]
VOLUME ["/app/projects", "/app/logs", "/app/data"]

EXPOSE 5000

Expand Down
1 change: 1 addition & 0 deletions docker/docker-cloud.yml
Original file line number Diff line number Diff line change
Expand Up @@ -5,3 +5,4 @@ rasanlu:
volumes:
- "/rasa-app-data/projects:/app/projects"
- "/rasa-app-data/logs:/app/logs"
- "./rasa-app-data/data:/app/data"
1 change: 1 addition & 0 deletions docker/docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ services:
volumes:
- "./rasa-app-data/projects:/app/projects"
- "./rasa-app-data/logs:/app/logs"
- "./rasa-app-data/data:/app/data"
duckling:
image: rasa/duckling:latest
networks: ['rasa-network']
Expand Down
12 changes: 12 additions & 0 deletions rasa_nlu/server.py
Original file line number Diff line number Diff line change
Expand Up @@ -113,8 +113,20 @@ def decorated(*args, **kwargs):
if origin:
if '*' in self.cors_origins:
request.setHeader('Access-Control-Allow-Origin', '*')
request.setHeader(
'Access-Control-Allow-Headers',
'Content-Type')
request.setHeader(
'Access-Control-Allow-Methods',
'POST, GET, OPTIONS, PUT, DELETE')
elif origin in self.cors_origins:
request.setHeader('Access-Control-Allow-Origin', origin)
request.setHeader(
'Access-Control-Allow-Headers',
'Content-Type')
request.setHeader(
'Access-Control-Allow-Methods',
'POST, GET, OPTIONS, PUT, DELETE')
else:
request.setResponseCode(403)
return 'forbidden'
Expand Down
23 changes: 22 additions & 1 deletion rasa_nlu/utils/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -247,17 +247,38 @@ def env_var_constructor(loader, node):


def read_yaml(content):
"""Parses yaml from a text.
Args:
content: A text containing yaml content.
"""
fix_yaml_loader()
replace_environment_variables()

yaml_parser = yaml.YAML(typ="safe")
yaml_parser.version = "1.2"
yaml_parser.unicode_supplementary = True

return yaml_parser.load(content)
try:
return yaml_parser.load(content)
except yaml.scanner.ScannerError as _:
# A `ruamel.yaml.scanner.ScannerError` might happen due to escaped
# unicode sequences that form surrogate pairs. Try converting the input
# to a parsable format based on
# https://stackoverflow.com/a/52187065/3429596.
content = (content.encode('utf-8')
.decode('raw_unicode_escape')
.encode("utf-16", 'surrogatepass')
.decode('utf-16'))
return yaml_parser.load(content)


def read_yaml_file(filename):
"""Parses a yaml file.
Args:
filename: The path to the file which should be read.
"""
return read_yaml(read_file(filename, "utf-8"))


Expand Down
50 changes: 24 additions & 26 deletions setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,47 +14,45 @@
long_description = f.read()

tests_requires = [
"pytest",
"pytest-pep8",
"pytest-cov",
"pytest~=3.3",
"pytest-pep8~=1.0",
"pytest-cov~=2.5",
"pytest-twisted<1.6",
"treq",
"treq~=17.8",
"responses~=0.9.0",
"httpretty~=0.9.0",
]

install_requires = [
"pathlib",
"cloudpickle",
"gevent",
"klein",
"boto3",
"packaging",
"typing",
"future",
"six",
"tqdm",
"requests",
"jsonschema",
"cloudpickle~=0.6.1",
"gevent~=1.2",
"klein~=17.10",
"boto3~=1.5",
"packaging~=17.1",
"typing~=3.6",
"future~=0.16.0",
"six~=1.11",
"tqdm~=4.19",
"requests~=2.20",
"jsonschema~=2.6",
"matplotlib~=2.0",
"numpy>=1.13",
"simplejson",
"ruamel.yaml~=0.15",
"coloredlogs",
"msgpack>=0.3.0,<0.6"
"simplejson~=3.13",
"ruamel.yaml~=0.15.7",
"coloredlogs~=9.0",
]

extras_requires = {
'test': tests_requires,
'spacy': ["scikit-learn<0.20",
"sklearn-crfsuite",
"scipy",
"spacy<=2.0.12,>2.0",
"sklearn-crfsuite~=0.3.6",
"scipy~=1.1",
"spacy<=2.0.18,>2.0",
],
'tensorflow': ["scikit-learn<0.20",
"sklearn-crfsuite",
"scipy",
"tensorflow"
"sklearn-crfsuite~=0.3.6",
"scipy~=1.1",
"tensorflow~=1.12"
],
'mitie': ["mitie"],
}
Expand Down
28 changes: 20 additions & 8 deletions tests/base/test_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -200,28 +200,40 @@ def test_environment_variable_dict_with_prefix_and_with_postfix():
def test_emojis_in_yaml():
test_data = """
data:
- one 😁
- two Β£
- one πŸ˜πŸ’― πŸ‘©πŸΏβ€πŸ’»πŸ‘¨πŸΏβ€πŸ’»
- two Β£ (?u)\\b\\w+\\b f\u00fcr
"""
actual = utils.read_yaml(test_data)

assert actual["data"][0] == "one 😁"
assert actual["data"][1] == "two Β£"
assert actual["data"][0] == "one πŸ˜πŸ’― πŸ‘©πŸΏβ€πŸ’»πŸ‘¨πŸΏβ€πŸ’»"
assert actual["data"][1] == "two Β£ (?u)\\b\\w+\\b fΓΌr"


def test_emojis_in_tmp_file():
test_data = """
data:
- one 😁
- two Β£
- one πŸ˜πŸ’― πŸ‘©πŸΏβ€πŸ’»πŸ‘¨πŸΏβ€πŸ’»
- two Β£ (?u)\\b\\w+\\b f\u00fcr
"""
test_file = utils.create_temporary_file(test_data)
with io.open(test_file, mode='r', encoding="utf-8") as f:
content = f.read()
actual = utils.read_yaml(content)

assert actual["data"][0] == "one 😁"
assert actual["data"][1] == "two Β£"
assert actual["data"][0] == "one πŸ˜πŸ’― πŸ‘©πŸΏβ€πŸ’»πŸ‘¨πŸΏβ€πŸ’»"
assert actual["data"][1] == "two Β£ (?u)\\b\\w+\\b fΓΌr"


def test_read_emojis_from_json():
import json
from rasa_nlu.utils import read_yaml
d = {"text": "hey πŸ˜πŸ’― πŸ‘©πŸΏβ€πŸ’»πŸ‘¨πŸΏβ€πŸ’»πŸ§œβ€β™‚οΈ(?u)\\b\\w+\\b} f\u00fcr"}
json_string = json.dumps(d, indent=2)

s = read_yaml(json_string)

expected = "hey πŸ˜πŸ’― πŸ‘©πŸΏβ€πŸ’»πŸ‘¨πŸΏβ€πŸ’»πŸ§œβ€β™‚οΈ(?u)\\b\\w+\\b} fΓΌr"
assert s.get('text') == expected


def test_bool_str():
Expand Down

0 comments on commit e5703f2

Please sign in to comment.