Conversion utf8 to utf16 and pretty-printing of Java strings

Added two functions for utf8 to utf16 conversion function depending on
whether we use little or big endian.
Added a function utf16_little_endian_to_ascii to display nicely java
strings as an ascii sequence.

Author: smowton

src/util/unicode.cpp

@@ -7,6 +7,8 @@ Author: Daniel Kroening, kroening@kroening.com
 \*******************************************************************/
 
 #include <cstring>
+#include <locale>
+#include <codecvt>
 
 #include "unicode.h"
 
@@ -253,3 +255,41 @@ const char **narrow_argv(int argc, const wchar_t **argv_wide)
 
   return argv_narrow;
 }
+
+std::wstring utf8_to_utf16_big_endian(const std::string& in)
+{
+  std::wstring_convert<std::codecvt_utf8_utf16<wchar_t> > converter;
+  return converter.from_bytes(in);
+}
+
+std::wstring utf8_to_utf16_little_endian(const std::string& in)
+{
+  const std::codecvt_mode mode=std::codecvt_mode::little_endian;
+
+  // default largest value codecvt_utf8_utf16 reads without error is 0x10ffff
+  // see: http://en.cppreference.com/w/cpp/locale/codecvt_utf8_utf16
+  const unsigned long maxcode=0x10ffff;
+
+  typedef std::codecvt_utf8_utf16<wchar_t, maxcode, mode> codecvt_utf8_utf16t;
+  std::wstring_convert<codecvt_utf8_utf16t> converter;
+  return converter.from_bytes(in);
+}
+
+std::string utf16_little_endian_to_ascii(const std::wstring& in)
+{
+  std::string result;
+  std::locale loc;
+  for(const auto c : in)
+  {
+    if(c<=255 && isprint(c, loc))
+      result+=(unsigned char)c;
+    else
+    {
+      result+="\\u";
+      char hex[5];
+      snprintf(hex, sizeof(hex), "%04x", (wchar_t)c);
+      result+=hex;
+    }
+  }
+  return result;
+}

src/util/unicode.h

@@ -22,6 +22,10 @@ std::wstring widen(const std::string &s);
 std::string utf32_to_utf8(const std::basic_string<unsigned int> &s);
 std::string utf16_to_utf8(const std::basic_string<unsigned short int> &s);
 
+std::wstring utf8_to_utf16_big_endian(const std::string&);
+std::wstring utf8_to_utf16_little_endian(const std::string&);
+std::string utf16_little_endian_to_ascii(const std::wstring& in);
+
 const char **narrow_argv(int argc, const wchar_t **argv_wide);
 
 #endif // CPROVER_UTIL_UNICODE_H