Merge pull request cython#2635 from cython/cy3str

Add new "language_level=3str"

CHANGES.rst

@@ -34,10 +34,12 @@ Features added
 * ``cython.inline()`` supports a direct ``language_level`` keyword argument that
   was previously only available via a directive.
 
-* A new directive ``str_is_str=True`` was added that keeps unprefixed string
-  literals as type 'str' in both Py2 and Py3, and the builtin 'str' type unchanged
-  even when ``language_level=3``  is enabled.  This is meant to help user code to
-  migrate to Python 3 semantics without making support for Python 2.x difficult.
+* A new language level name ``3str`` was added that mostly corresponds to language
+  level 3, but keeps unprefixed string literals as type 'str' in both Py2 and Py3,
+  and the builtin 'str' type unchanged.  This will become the default in the next
+  Cython release and is meant to help user code a) transition more easily to this
+  new default and b) migrate to Python 3 source code semantics without making support
+  for Python 2.x difficult.
 
 * In CPython 3.6 and later, looking up globals in the module dict is almost
   as fast as looking up C globals.
@@ -162,10 +164,13 @@ Other changes
 * Cython now emits a warning when no ``language_level`` (2 or 3) is set explicitly,
   neither as a ``cythonize()`` option nor as a compiler directive.  This is meant
   to prepare the transition of the default language level from currently Py2
-  to Py3, since that is what most new users will expect these days.  The next
-  major release is intended to make that change, so that it will parse all code
-  that does not request a specific language level as Python 3 code. The language
-  level 2 will continue to be supported for an indefinite time.
+  to Py3, since that is what most new users will expect these days.  The future
+  default will, however, not enforce unicode literals, because this has proven a
+  major obstacle in the support for both Python 2.x and 3.x.  The next major
+  release is intended to make this change, so that it will parse all code that
+  does not request a specific language level as Python 3 code, but with ``str``
+  literals.  The language level 2 will continue to be supported for an indefinite
+  time.
 
 * The documentation was restructured, cleaned up and examples are now tested.
   The NumPy tutorial was also rewritten to simplify the running example.

Cython/Compiler/Main.py

@@ -94,23 +94,24 @@ def __init__(self, include_directories, compiler_directives, cpp=False,
 
         if language_level is not None:
             self.set_language_level(language_level)
-        if self.compiler_directives.get('str_is_str') is not None:
-            self.set_str_is_str(self.compiler_directives['str_is_str'])
 
         self.gdb_debug_outputwriter = None
 
-    def set_str_is_str(self, str_is_str):
-        from .Future import unicode_literals
-        if str_is_str:
+    def set_language_level(self, level):
+        from .Future import print_function, unicode_literals, absolute_import, division
+        future_directives = []
+        if level == '3str':
+            future_directives = [print_function, absolute_import, division]
             self.future_directives.discard(unicode_literals)
+            level = 3
         else:
-            self.future_directives.add(unicode_literals)
-
-    def set_language_level(self, level):
+            level = int(level)
+            if level >= 3:
+                future_directives = [print_function, unicode_literals, absolute_import, division]
         self.language_level = level
+        if future_directives:
+            self.future_directives.update(future_directives)
         if level >= 3:
-            from .Future import print_function, unicode_literals, absolute_import, division
-            self.future_directives.update([print_function, unicode_literals, absolute_import, division])
             self.modules['builtins'] = self.modules['__builtin__']
 
     def intern_ustring(self, value, encoding=None):

Cython/Compiler/Options.py

@@ -198,7 +198,6 @@ def get_directive_defaults():
     'iterable_coroutine': False,  # Make async coroutines backwards compatible with the old asyncio yield-from syntax.
     'c_string_type': 'bytes',
     'c_string_encoding': '',
-    'str_is_str': None,  # fall back to 'language_level == 2'
     'type_version_tag': True,  # enables Py_TPFLAGS_HAVE_VERSION_TAG on extension types
     'unraisable_tracebacks': True,
     'old_style_globals': False,
@@ -293,7 +292,7 @@ def normalise_encoding_name(option_name, encoding):
 
 # Override types possibilities above, if needed
 directive_types = {
-    'language_level': int,  # values can be None/2/3, where None == 2+warning
+    'language_level': str,  # values can be None/2/3/'3str', where None == 2+warning
     'auto_pickle': bool,
     'locals': dict,
     'final' : bool,  # final cdef classes and methods
@@ -314,7 +313,6 @@ def normalise_encoding_name(option_name, encoding):
     'freelist': int,
     'c_string_type': one_of('bytes', 'bytearray', 'str', 'unicode'),
     'c_string_encoding': normalise_encoding_name,
-    'str_is_str': bool,
 }
 
 for key, val in _directive_defaults.items():
@@ -349,7 +347,6 @@ def normalise_encoding_name(option_name, encoding):
     # Avoid scope-specific to/from_py_functions for c_string.
     'c_string_type': ('module',),
     'c_string_encoding': ('module',),
-    'str_is_str': ('module',),
     'type_version_tag': ('module', 'cclass'),
     'language_level': ('module',),
     # globals() could conceivably be controlled at a finer granularity,

Cython/Compiler/Parsing.py

@@ -3669,9 +3669,6 @@ def p_compiler_directive_comments(s):
             if 'language_level' in new_directives:
                 # Make sure we apply the language level already to the first token that follows the comments.
                 s.context.set_language_level(new_directives['language_level'])
-            if 'str_is_str' in new_directives:
-                # Make sure we apply 'str_is_str' directive already to the first token that follows the comments.
-                s.context.set_str_is_str(new_directives['str_is_str'])
 
             result.update(new_directives)
 

tests/run/cython3_no_unicode_literals.pyx

@@ -1,4 +1,4 @@
-# cython: language_level=3, binding=True, str_is_str=True
+# cython: language_level=3str, binding=True
 # mode: run
 # tag: python3, str_is_str