Fixed unicode handling in builder.py

crossplane/__main__.py

@@ -2,7 +2,6 @@
 # -*- coding: utf-8 -*-
 import os
 import sys
-
 from argparse import ArgumentParser, FileType, RawDescriptionHelpFormatter
 from traceback import format_exception
 
@@ -11,7 +10,7 @@
 from .parser import parse as parse_file
 from .builder import build as build_string, build_files, _enquote, DELIMITERS
 from .errors import NgxParserBaseException
-from .compat import PY2, json, input
+from .compat import json, input
 
 
 def _prompt_yes():
@@ -89,8 +88,6 @@ def build(filename, dirname=None, force=False, indent=4, tabs=False,
             parsed = config['parsed']
             output = build_string(parsed, indent=indent, tabs=tabs, header=header)
             output = output.rstrip() + '\n'
-            if PY2:
-                output = output.encode('utf-8')
             print('# ' + path + '\n' + output)
         return
 

crossplane/builder.py

@@ -1,9 +1,13 @@
 # -*- coding: utf-8 -*-
 import codecs
 import os
+import re
+
+from .compat import PY2
 
 DELIMITERS = ('{', '}', ';')
 EXTERNAL_BUILDERS = {}
+ESCAPE_SEQUENCES_RE = re.compile(r'(\\x[0-9a-f]{2}|\\[0-7]{1,3})')
 
 
 def _escape(string):
@@ -49,10 +53,24 @@ def _needs_quotes(string):
     return char in ('\\', '$') or expanding
 
 
+def _replace_escape_sequences(match):
+    return match.group(1).decode('string-escape')
+
+
 def _enquote(arg):
-    if _needs_quotes(arg):
-        arg = repr(codecs.decode(arg, 'raw_unicode_escape'))
-        arg = arg.replace('\\\\', '\\').lstrip('u')
+    if not _needs_quotes(arg):
+        return arg
+
+    if PY2:
+        arg = codecs.encode(arg, 'utf-8') if isinstance(arg, unicode) else arg
+        arg = codecs.decode(arg, 'raw-unicode-escape')
+        arg = repr(arg).replace('\\\\', '\\').lstrip('u')
+        arg = ESCAPE_SEQUENCES_RE.sub(_replace_escape_sequences, arg)
+        arg = unicode(arg, 'utf-8')
+    else:
+        arg = codecs.decode(arg, 'unicode-internal')
+        arg = repr(arg).replace('\\\\', '\\')
+
     return arg
 
 
@@ -154,7 +172,7 @@ def build_files(payload, dirname=None, indent=4, tabs=False, header=False):
         parsed = config['parsed']
         output = build(parsed, indent=indent, tabs=tabs, header=header)
         output = output.rstrip() + '\n'
-        with codecs.open(path, 'w', encoding='utf-8') as fp:
+        with codecs.open(path, 'w') as fp:
             fp.write(output)
 
 

tests/__init__.py

@@ -33,15 +33,15 @@ def compare_parsed_and_built(conf_dirname, conf_basename, tmpdir, **kwargs):
 
     build1_config = build(original_parsed)
     build1_file = tmpdir.join('build1.conf')
-    build1_file.write(build1_config)
+    build1_file.write_text(build1_config, encoding='utf-8')
     build1_payload = parse(build1_file.strpath, **kwargs)
     build1_parsed = build1_payload['config'][0]['parsed']
 
     assert_equal_payloads(original_parsed, build1_parsed, ignore_keys=['line'])
 
     build2_config = build(build1_parsed)
     build2_file = tmpdir.join('build2.conf')
-    build2_file.write(build2_config)
+    build2_file.write_text(build2_config, encoding='utf-8')
     build2_payload = parse(build2_file.strpath, **kwargs)
     build2_parsed = build2_payload['config'][0]['parsed']
 

tests/configs/russian-text/nginx.conf

@@ -0,0 +1,2 @@
+env 'русский текст';
+events {}

tests/test_build.py

@@ -170,6 +170,20 @@ def test_build_with_comments():
     ])
 
 
+def test_build_with_quoted_unicode():
+    payload = [
+        {
+            "directive": "env",
+            "line": 1,
+            "args": ["русский текст"],
+        }
+    ]
+
+    built = crossplane.build(payload, indent=4, tabs=False)
+
+    assert built == u"env 'русский текст';"
+
+
 def test_build_files_with_missing_status_and_errors(tmpdir):
     assert len(tmpdir.listdir()) == 0
     payload = {
@@ -207,7 +221,7 @@ def test_build_files_with_unicode(tmpdir):
                     {
                         "directive": "user",
                         "line": 1,
-                        "args": [u"\u6e2c\u8a66"],
+                        "args": ["測試"],
                     }
                 ]
             }
@@ -217,7 +231,7 @@ def test_build_files_with_unicode(tmpdir):
     built_files = tmpdir.listdir()
     assert len(built_files) == 1
     assert built_files[0].strpath == os.path.join(tmpdir.strpath, 'nginx.conf')
-    assert built_files[0].read_text('utf-8') == u'user \u6e2c\u8a66;\n'
+    assert built_files[0].read_text('utf-8') == u'user 測試;\n'
 
 
 def test_compare_parsed_and_built_simple(tmpdir):
@@ -234,3 +248,7 @@ def test_compare_parsed_and_built_messy_with_comments(tmpdir):
 
 def test_compare_parsed_and_built_empty_map_values(tmpdir):
     compare_parsed_and_built('empty-value-map', 'nginx.conf', tmpdir)
+
+
+def test_compare_parsed_and_built_russian_text(tmpdir):
+    compare_parsed_and_built('russian-text', 'nginx.conf', tmpdir)