Fix conversion of PyUnicodeObject to wstring

On Windows, with Python >= 3.3, PyObject_Length cannot be used to get
the size of the wchar_t string, because it will count the number of
*code points*, but some characters not on the BMP will use two UTF-16
*code units* (surrogate pairs).
This is not a problem on Unix, since wchar_t is 32-bit.

This also fixes a problem where test_builtin_converters.py was not
being run, since the module docstring was not the first statement.

Author: tadeu

src/converter/builtin_converters.cpp

@@ -430,6 +430,17 @@ namespace
       // Remember that this will be used to construct the result object 
       static std::wstring extract(PyObject* intermediate)
       {
+#if defined(_WIN32) && PY_VERSION_HEX >= 0x03030000
+          BOOST_STATIC_ASSERT(sizeof(wchar_t) == 2);
+
+          Py_ssize_t size = 0;
+          wchar_t *buf = PyUnicode_AsWideCharString(intermediate, &size);
+          if (buf == NULL) {
+              boost::python::throw_error_already_set();
+          }
+          std::wstring result(buf, size);
+          PyMem_Free(buf);
+#else
           std::wstring result(::PyObject_Length(intermediate), L' ');
           if (!result.empty())
           {
@@ -444,6 +455,7 @@ namespace
               if (err == -1)
                   throw_error_already_set();
           }
+#endif
           return result;
       }
       static PyTypeObject const* get_pytype() { return &PyUnicode_Type;}

test/test_builtin_converters.py

@@ -1,9 +1,6 @@
 # Copyright David Abrahams 2004. Distributed under the Boost
 # Software License, Version 1.0. (See accompanying
 # file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
-import sys
-if (sys.version_info.major >= 3):
-    long = int
 r"""
 >>> from builtin_converters_ext import *
 
@@ -136,6 +133,9 @@
 >>> print(rewrap_value_wstring(u'yo, wassup?'))
 yo, wassup?
 
+>>> print(rewrap_value_wstring(u'\U0001f4a9'))
+\U0001f4a9
+
    test that overloading on unicode works:
 
 >>> print(rewrap_value_string(u'yo, wassup?'))
@@ -283,6 +283,10 @@
 >>> assert return_null_handle() is None
 """
 
+import sys
+if (sys.version_info.major >= 3):
+    long = int
+
 def run(args = None):
     import sys
     import doctest