Merge python#8

8: Add warnings for strings and operations r=ltratt a=nanjekyejoannah

This PR adds warnings about:
- the unicode type
- the `ur` literal
- the incompatible `basetring type'
- and mixed concat operations for strings

Co-authored-by: Joannah Nanjekye <jnanjekye@python.org>

Author: bors[bot]

Include/warnings.h

@@ -16,6 +16,9 @@ PyAPI_FUNC(int) PyErr_WarnExplicit_WithFix(PyObject *, const char *, const char
 #define PyErr_WarnPy3k(msg, stacklevel) \
   (Py_Py3kWarningFlag ? PyErr_WarnEx(PyExc_DeprecationWarning, msg, stacklevel) : 0)
 
+#define PyErr_WarnPy3k_WithFix(msg, fix, stacklevel) \
+  (Py_Py3kWarningFlag ? PyErr_WarnEx_WithFix(PyExc_DeprecationWarning, msg, fix, stacklevel) : 0)
+
 /* DEPRECATED: Use PyErr_WarnEx() instead. */
 #define PyErr_Warn(category, msg) PyErr_WarnEx(category, msg, 1)
 

Lib/test/string_tests.py

@@ -1081,6 +1081,24 @@ def test_mul(self):
         # but either raises a MemoryError, or succeeds (if you have 54TiB)
         #self.checkraises(OverflowError, 10000*'abc', '__mul__', 2000000000)
 
+    def test_py3x_warnings_isinstance(self):
+        with test_support.check_py3k_warnings(("the basestring type is not supported in 3.x: "
+                                               "import a third party library like six and use"
+                                               "a compatible type like string_types", Py3xWarning)):
+            isinstance(u"fix", basestring)
+            isinstance(b"fix", basestring)
+            isinstance("fix", basestring)
+
+    def test_py3x_warnings_join(self):
+        with test_support.check_py3k_warnings(("mixed bytes, str and unicode operands cannot "
+                                               "be used in string concatenation in Python 3.x: "
+                                               "convert the operand(s) so that they are the same type.", Py3xWarning)):
+            x = 'foo'
+            y = b'foo'
+            z = x + y
+            b = y + x
+            v = x.__add__(y)
+
     def test_join(self):
         # join now works with any sequence type
         # moved here, because the argument order is

Lib/test/test_grammar.py

@@ -1229,6 +1229,27 @@ def test_paren_evaluation(self):
         self.assertFalse((False is 2) is 3)
         self.assertFalse(False is 2 is 3)
 
+    def test_py3x_unicode_warnings_u(self):
+        if sys.py3kwarning:
+            with warnings.catch_warnings(record=True) as w:
+                warnings.filterwarnings('always', category=Py3xWarning)
+                self.assertEqual(u'foo', u'foo')
+                for warning in w:
+                    self.assertTrue(Py3xWarning is w.category)
+                    self.assertEqual(str(w.message), "the unicode type is not supported in 3.x: " \
+                                     "use native strings for compatibility or " \
+                                     "a 'u' or 'b' prefix if a native string is not required")
+
+    def test_py3x_unicode_warnings_ur(self):
+        if sys.py3kwarning:
+            with warnings.catch_warnings(record=True) as w:
+                warnings.filterwarnings('always', category=Py3xWarning)
+                self.assertEqual(ur'foo', u'foo')
+                for warning in w:
+                    self.assertTrue(Py3xWarning is w.category)
+                    self.assertEqual(str(w.message), "the 'ur' prefix in string literals is not supported in 3.x: ", 
+                                     "use a 'u' and two backslashes for a literal backslash")
+
 
 def test_main():
     run_unittest(TokenTests, GrammarTests)

Objects/abstract.c

@@ -2938,6 +2938,14 @@ PyObject_IsInstance(PyObject *inst, PyObject *cls)
 {
     static PyObject *name = NULL;
 
+    if ((PyTypeObject *)cls == &PyBaseString_Type) {
+        if (Py_Py3kWarningFlag &&
+            PyErr_WarnEx_WithFix(PyExc_Py3xWarning,
+                    "the basestring type is not supported in 3.x",
+                    "import a third party library like six and use a compatible type like string_types", 1) < 0) {
+            return 0;
+        }
+    }
     /* Quick test for an exact match */
     if (Py_TYPE(inst) == (PyTypeObject *)cls)
         return 1;

Objects/stringobject.c

@@ -1029,6 +1029,17 @@ string_concat(register PyStringObject *a, register PyObject *bb)
 {
     register Py_ssize_t size;
     register PyStringObject *op;
+    if (a->ob_type != bb->ob_type) {
+        char msgbuf[256];
+        sprintf(msgbuf, "the first string is '%.200s' while the second is '%.200s': "\
+                "mixed bytes, str and unicode operands cannot be used in string concatenation in Python 3.x", 
+                Py_TYPE(a)->tp_name, Py_TYPE(bb)->tp_name);
+        char *fix = "convert the operand(s) so that they are the same type.";
+        if (Py_Py3kWarningFlag &&
+            PyErr_WarnEx_WithFix(PyExc_Py3xWarning, msgbuf, fix, 1) < 0) {
+            return NULL;
+        }
+    }
     if (!PyString_Check(bb)) {
 #ifdef Py_USING_UNICODE
         if (PyUnicode_Check(bb))

Parser/tokenizer.c

@@ -1390,8 +1390,27 @@ tok_get(register struct tok_state *tok, char **p_start, char **p_end)
         case 'u':
         case 'U':
             c = tok_nextc(tok);
-            if (c == 'r' || c == 'R')
+            if (c == 'r' || c == 'R') {
+                if (Py_Py3kWarningFlag) {
+                    if (PyErr_WarnExplicit_WithFix(PyExc_Py3xWarning, "the 'ur' prefix in string literals is not supported in 3.x", 
+                                                   "use a 'u' and two backslashes for a literal backslash", tok->filename, tok->lineno, 
+                                                   NULL, NULL)) {
+                        return NULL;
+                    }
+                }
                 c = tok_nextc(tok);
+            }
+            else {
+                if (Py_Py3kWarningFlag) {
+                    if (PyErr_WarnExplicit_WithFix(PyExc_Py3xWarning, "the unicode type is not supported in 3.x", 
+                                                   "use native strings for compatibility or "
+                                                   "a 'u' or 'b' prefix if a native string is not required", 
+                                                   tok->filename, tok->lineno, NULL, NULL)) {
+                        return NULL;
+                    }
+                }
+
+            }
             if (c == '"' || c == '\'')
                 goto letter_quote;
             break;