add transcode (#288)

* add transcode (fixes #284)

* fixed method ambig and account for NUL termination of legacy utf16/utf32 functions

* resize rather than slice, to avoid making an unnecessary copy

* add missing T -> T transcode methods

* another transcode test

* whoops

Author: stevengj

README.md

@@ -87,6 +87,9 @@ Currently, the `@compat` macro supports the following syntaxes:
 
 * `unsafe_get` returns the `:value` field of a `Nullable` object without any null-check and has a generic fallback for non-`Nullable` argument. [#18484](https://github.com/JuliaLang/julia/pull/18484) (Also, `isnull` has a generic fallback for non-`Nullable` argument.)
 
+* `transcode` converts between UTF-xx string encodings in Julia 0.5 (as a lightweight
+   alternative to the LegacyStrings package), [#17323](https://github.com/JuliaLang/julia/pull/17323).
+
 ## Renamed functions
 
 * `pointer_to_array` and `pointer_to_string` have been replaced with `unsafe_wrap(Array, ...)` and `unsafe_wrap(String, ...)` respectively.

src/Compat.jl

@@ -1698,4 +1698,30 @@ if VERSION < v"0.6.0-dev.1256"
     Base.take!(io::Base.AbstractIOBuffer) = takebuf_array(io)
 end
 
+# julia #17323
+if VERSION < v"0.5.0-dev+5380"
+    export transcode
+    transcode{T<:Compat.String}(::Type{T}, src::Union{Compat.String,Vector{UInt8}}) = utf8(src)
+    transcode{T<:Compat.String}(::Type{T}, src::Vector{UInt16}) = utf8(utf16(src))
+    transcode{T<:Compat.String}(::Type{T}, src::Vector{UInt32}) = utf8(utf32(src))
+    transcode(::Type{UInt8}, src::Vector{UInt8}) = src
+    transcode(::Type{UInt8}, src) = transcode(Compat.String, src).data
+    function transcode(::Type{UInt16}, src::Union{Compat.String,Vector{UInt8}})
+        d = utf16(utf8(src)).data
+        return resize!(d, length(d)-1) # strip off trailing NUL codeunit
+    end
+    function transcode(::Type{UInt32}, src::Union{Compat.String,Vector{UInt8}})
+        d = utf32(utf8(src)).data
+        return resize!(d, length(d)-1) # strip off trailing NUL codeunit
+    end
+    transcode(::Type{UInt16}, src::Vector{UInt16}) = src
+    transcode(::Type{UInt32}, src::Vector{UInt32}) = src
+    if Cwchar_t == Int32
+        transcode(::Type{Cwchar_t}, src::Vector{Cwchar_t}) = src
+        transcode(::Type{Cwchar_t}, src) = reinterpret(Cwchar_t, transcode(UInt32, src))
+        transcode(::Type{UInt8}, src::Vector{Cwchar_t}) = transcode(UInt8, reinterpret(UInt32, src))
+        transcode(T, src::Vector{Cwchar_t}) = transcode(T, reinterpret(UInt32, src))
+    end
+end
+
 end # module

test/runtests.jl

@@ -1524,3 +1524,13 @@ let io = IOBuffer()
     write(io, "bbb")
     @test String(take!(io)) == "bbb"
 end
+
+let s = "Koala test: 🐨"
+    @test transcode(UInt16, s) == UInt16[75,111,97,108,97,32,116,101,115,116,58,32,55357,56360]
+    @test transcode(UInt32, s) == UInt32[75,111,97,108,97,32,116,101,115,116,58,32,128040]
+    for T in (UInt8,UInt16,UInt32,Cwchar_t)
+        @test transcode(Compat.String, transcode(T, s)) == s
+        @test transcode(UInt8, transcode(T, s)) == s.data
+        @test transcode(T, s) == transcode(T, s.data) == transcode(T, transcode(T, s))
+    end
+end