Fix use correct bytesize of UTF-16 strings

Author: straight-shoota

spec/std/env_spec.cr

@@ -95,4 +95,22 @@ describe "ENV" do
       end
     end
   end
+
+  it "handles unicode" do
+    ENV["TEST_UNICODE_1"] = "bar\u{d7ff}\u{10000}"
+    ENV["TEST_UNICODE_2"] = "\u{1234}"
+    ENV["TEST_UNICODE_1"].should eq "bar\u{d7ff}\u{10000}"
+    ENV["TEST_UNICODE_2"].should eq "\u{1234}"
+
+    values = {} of String => String
+    ENV.each do |key, value|
+      if key.starts_with?("TEST_UNICODE_")
+        values[key] = value
+      end
+    end
+    values.should eq({
+      "TEST_UNICODE_1" => "bar\u{d7ff}\u{10000}",
+      "TEST_UNICODE_2" => "\u{1234}"
+    })
+  end
 end

spec/std/string/utf16_spec.cr

@@ -23,31 +23,50 @@ describe "String UTF16" do
     end
   end
 
-  describe "from_utf16" do
+  describe ".from_utf16" do
     it "in the range U+0000..U+D7FF" do
       input = Slice[0_u16, 0x68_u16, 0x65_u16, 0x6c_u16, 0x6c_u16, 0x6f_u16, 0xd7ff_u16]
       String.from_utf16(input).should eq("\u{0}hello\u{d7ff}")
+      String.from_utf16(input.to_unsafe).should eq({"", input.to_unsafe + 1})
     end
 
     it "in the range U+E000 to U+FFFF" do
       input = Slice[0xe000_u16, 0xffff_u16]
       String.from_utf16(input).should eq("\u{e000}\u{ffff}")
+
+      pointer = Slice[0xe000_u16, 0xffff_u16, 0_u16].to_unsafe
+      String.from_utf16(pointer).should eq({"\u{e000}\u{ffff}", pointer + 3})
     end
 
     it "in the range U+10000..U+10FFFF" do
       input = Slice[0xd800_u16, 0xdc00_u16]
       String.from_utf16(input).should eq("\u{10000}")
+
+      pointer = Slice[0xd800_u16, 0xdc00_u16, 0_u16].to_unsafe
+      String.from_utf16(pointer).should eq({"\u{10000}", pointer + 3})
     end
 
     it "in the range U+D800..U+DFFF" do
       input = Slice[0xdc00_u16, 0xd800_u16]
       String.from_utf16(input).should eq("\u{fffd}\u{fffd}")
+
+      pointer = Slice[0xdc00_u16, 0xd800_u16, 0_u16].to_unsafe
+      String.from_utf16(pointer).should eq({"\u{fffd}\u{fffd}", pointer + 3})
     end
 
     it "handles null bytes" do
       slice = Slice[104_u16, 105_u16, 0_u16, 55296_u16, 56485_u16]
       String.from_utf16(slice).should eq("hi\0000𐂥")
-      String.from_utf16(slice.to_unsafe).should eq("hi")
+      String.from_utf16(slice.to_unsafe).should eq({"hi", slice.to_unsafe + 3})
+    end
+
+    it "with pointer reads multiple strings" do
+      input = Slice[0_u16, 0x68_u16, 0x65_u16, 0x6c_u16, 0x6c_u16, 0x6f_u16, 0xd7ff_u16, 0_u16]
+      pointer = input.to_unsafe
+      string, pointer = String.from_utf16(pointer)
+      string.should eq("")
+      string, pointer = String.from_utf16(pointer)
+      string.should eq("hello\u{d7ff}")
     end
   end
 end

src/crystal/system/win32/env.cr

@@ -54,13 +54,11 @@ module Crystal::System::Env
 
     begin
       while !pointer.value.zero?
-        string = String.from_utf16(pointer)
+        string, pointer = String.from_utf16(pointer)
         key_value = string.split('=', 2)
         key = key_value[0]
         value = key_value[1]? || ""
         yield key, value
-
-        pointer += string.bytesize + 1
       end
     ensure
       LibC.FreeEnvironmentStringsW(orig_pointer)

src/crystal/system/win32/time.cr

@@ -141,13 +141,13 @@ module Crystal::System::Time
 
   # Normalizes the names of the standard and dst zones.
   private def self.normalize_zone_names(info : LibC::TIME_ZONE_INFORMATION) : Tuple(String, String)
-    stdname = String.from_utf16(info.standardName.to_unsafe)
+    stdname = String.from_utf16(info.standardName.to_slice)
 
     if normalized_names = WINDOWS_ZONE_NAMES[stdname]?
       return normalized_names
     end
 
-    dstname = String.from_utf16(info.daylightName.to_unsafe)
+    dstname = String.from_utf16(info.daylightName.to_slice)
 
     if english_name = translate_zone_name(stdname, dstname)
       if normalized_names = WINDOWS_ZONE_NAMES[english_name]?

src/string/utf16.cr

@@ -53,15 +53,7 @@ class String
   # slice = Slice[104_u16, 105_u16, 32_u16, 55296_u16, 56485_u16]
   # String.from_utf16(slice) # => "hi 𐂥"
   # ```
-  #
-  # If *slice* is a pointer, the string ends when a zero value is found.
-  #
-  # ```
-  # slice = Slice[104_u16, 105_u16, 0_u16, 55296_u16, 56485_u16]
-  # String.from_utf16(slice)           # => "hi\0000𐂥"
-  # String.from_utf16(slice.to_unsafe) # => "hi"
-  # ```
-  def self.from_utf16(slice : Slice(UInt16) | Pointer(UInt16)) : String
+  def self.from_utf16(slice : Slice(UInt16)) : String
     bytesize = 0
     size = 0
 
@@ -81,6 +73,41 @@ class String
     end
   end
 
+  # Decodes the given *slice* UTF-16 sequence into a String and returns the
+  # pointer after reading. The string ends when a zero value is found.
+  #
+  # ```
+  # slice = Slice[104_u16, 105_u16, 0_u16, 55296_u16, 56485_u16, 0_u16]
+  # String.from_utf16(slice) # => "hi\0000𐂥"
+  # pointer = slice.to_unsafe
+  # string, pointer = String.from_utf16(pointer) # => "hi"
+  # string, pointer = String.from_utf16(pointer) # => "𐂥"
+  # ```
+  #
+  # Invalid values are encoded using the unicode replacement char with
+  # codepoint `0xfffd`.
+  def self.from_utf16(pointer : Pointer(UInt16)) : {String, Pointer(UInt16)}
+    bytesize = 0
+    size = 0
+
+    each_utf16_char(pointer) do |char|
+      bytesize += char.bytesize
+      size += 1
+    end
+
+    string = String.new(bytesize) do |buffer|
+      pointer = each_utf16_char(pointer) do |char|
+        char.each_byte do |byte|
+          buffer.value = byte
+          buffer += 1
+        end
+      end
+      {bytesize, size}
+    end
+
+    {string, pointer + 1}
+  end
+
   # Yields each decoded char in the given slice.
   private def self.each_utf16_char(slice : Slice(UInt16))
     i = 0
@@ -107,7 +134,7 @@ class String
   end
 
   # Yields each decoded char in the given pointer, stopping at the first null byte.
-  private def self.each_utf16_char(pointer : Pointer(UInt16))
+  private def self.each_utf16_char(pointer : Pointer(UInt16)) : Pointer(UInt16)
     loop do
       byte = pointer.value.to_i
       break if byte == 0
@@ -129,5 +156,7 @@ class String
 
       pointer = pointer + 1
     end
+
+    pointer
   end
 end