Fix for windows time zone name and other Unicode issues (#1375)

* Fix for Unicode characters above U+FFFF.

* Add error handling to time zone conversion.

* Fix scoping error.

* Convert Windows TZ environment variable from CP-1252 to Unicode.

* Don't use a variable length array.

* Fix compiler error.

* Modify readme.

* Fix array lookup.

* Update readme.

* Also fix the fallback daylight saving time string encoding.

* Rephrase readme.

* Add unit test for conversions on Windows.

* Add test for CP1252 conversion and fix conversion function that we have a test for it.

* Remove duplicate code.

* Add missing platform header.

* Add additional characters to locale test.

* Let's go wild, even more characters. This should cover most of what we
need.

* Format code.

* Modified locale test to work with Windows wchar_t.

* Format code.

* Set thread language to English and use GetTimeZoneInformation.

* Format code.

* Add a comment about why we don't use daylight savings time in one spot.

Author: jonsimantov

app/src/heartbeat/heartbeat_storage_desktop.cc

@@ -69,7 +69,7 @@ std::string CreateFilename(const std::string& app_id, const Logger& logger) {
   std::string final_path_utf8 =
       app_dir + "/" + kHeartbeatFilenamePrefix + app_id_without_symbols;
 #if FIREBASE_PLATFORM_WINDOWS
-  std::wstring_convert<std::codecvt_utf8<wchar_t>> final_path_w;
+  std::wstring_convert<std::codecvt_utf8_utf16<wchar_t>> final_path_w;
   return final_path_w.from_bytes(final_path_utf8);
 #else
   return final_path_utf8;

app/src/locale.cc

@@ -44,6 +44,8 @@
 #include <string>
 #include <vector>
 
+#include "app/src/thread.h"
+
 namespace firebase {
 namespace internal {
 
@@ -94,6 +96,27 @@ std::string GetLocale() {
 #endif  // platform selector
 }
 
+#if FIREBASE_PLATFORM_WINDOWS
+std::wstring GetWindowsTimezoneInEnglish(int daylight) {
+  struct TzNames {
+    std::wstring standard;
+    std::wstring daylight;
+  } tz_names;
+  Thread thread(
+      [](TzNames* tz_names_ptr) {
+        SetThreadUILanguage(MAKELANGID(LANG_ENGLISH, SUBLANG_ENGLISH_US));
+        TIME_ZONE_INFORMATION tzi;
+        GetTimeZoneInformation(&tzi);
+        tz_names_ptr->standard = tzi.StandardName;
+        tz_names_ptr->daylight = tzi.DaylightName;
+      },
+      &tz_names);
+  thread.Join();
+
+  return daylight ? tz_names.daylight : tz_names.standard;
+}
+#endif  // FIREBASE_PLATFORM_WINDOWS
+
 // Get the current time zone, e.g. "US/Pacific"
 std::string GetTimezone() {
 #if FIREBASE_PLATFORM_WINDOWS
@@ -103,20 +126,17 @@ std::string GetTimezone() {
                // settings or the TZ variable, as appropriate.
     tz_was_set = true;
   }
-  // Get the standard time zone name.
+
+  // Get the non-daylight time zone, as the IANA conversion below requires the
+  // name of the standard time zone. For example, "Central European Standard
+  // Time" which converts to "Europe/Warsaw" or similar.
+  std::wstring windows_tz_utf16 = GetWindowsTimezoneInEnglish(0);
   std::string windows_tz_utf8;
   {
-    size_t length = 0;  // get the needed string length
-    if (_get_tzname(&length, nullptr, 0, 0) != 0) return "";
-    std::vector<char> namebuf(length);
-    if (_get_tzname(&length, &namebuf[0], length, 0) != 0) return "";
-    windows_tz_utf8 = std::string(&namebuf[0]);
+    std::wstring_convert<std::codecvt_utf8_utf16<wchar_t>, wchar_t> to_utf8;
+    windows_tz_utf8 = to_utf8.to_bytes(windows_tz_utf16);
   }
 
-  // Convert time zone name to wide string
-  std::wstring_convert<std::codecvt_utf8<wchar_t>> to_utf16;
-  std::wstring windows_tz_utf16 = to_utf16.from_bytes(windows_tz_utf8);
-
   std::string locale_name = GetLocale();
   wchar_t iana_time_zone_buffer[128];
   bool got_time_zone = false;
@@ -159,28 +179,47 @@ std::string GetTimezone() {
           windows_tz_utf8.c_str(), u_errorName(error_code), error_code);
     }
   }
+  if (got_time_zone) {
+    // One of the above two succeeded, convert the new time zone name back to
+    // UTF-8.
+    std::wstring iana_tz_utf16(iana_time_zone_buffer);
+    std::wstring_convert<std::codecvt_utf8_utf16<wchar_t>, wchar_t> to_utf8;
+    std::string iana_tz_utf8;
+    try {
+      iana_tz_utf8 = to_utf8.to_bytes(iana_tz_utf16);
+    } catch (std::range_error& ex) {
+      LogError("Failed to convert IANA time zone to UTF-8: %s", ex.what());
+      got_time_zone = false;
+    }
+    if (got_time_zone) {
+      return iana_tz_utf8;
+    }
+  }
   if (!got_time_zone) {
-    // Return the Windows time zone ID as a backup.
-    // In this case, we need to get the correct daylight savings time
-    // setting to get the right name.
+    // Either the IANA time zone couldn't be determined, or couldn't be
+    // converted into UTF-8 for some reason (the std::range_error above).
+    //
+    // In any case, return the Windows time zone ID as a backup. We now need to
+    // get the correct daylight saving time setting to get the right name.
+    //
+    // Also, note as above that _get_tzname() doesn't return a UTF-8 name,
+    // rather CP-1252, so convert it to UTF-8 (via UTF-16) before returning.
+
     int daylight = 0;  // daylight savings time?
-    if (_get_daylight(&daylight) != 0) return windows_tz_utf8;
+    if (_get_daylight(&daylight) != 0) {
+      // Couldn't determine daylight saving time, return the old name.
+      return windows_tz_utf8;
+    }
     if (daylight) {
-      size_t length = 0;  // get the needed string length
-      if (_get_tzname(&length, nullptr, 0, 1) != 0) return windows_tz_utf8;
-      std::vector<char> namebuf(length);
-      if (_get_tzname(&length, &namebuf[0], length, 1) != 0)
-        return windows_tz_utf8;
-      windows_tz_utf8 = std::string(&namebuf[0]);
+      windows_tz_utf16 = GetWindowsTimezoneInEnglish(daylight);
+      {
+        std::wstring_convert<std::codecvt_utf8_utf16<wchar_t>, wchar_t> to_utf8;
+        windows_tz_utf8 = to_utf8.to_bytes(windows_tz_utf16);
+      }
     }
     return windows_tz_utf8;
   }
 
-  std::wstring iana_tz_utf16(iana_time_zone_buffer);
-  std::wstring_convert<std::codecvt_utf8<wchar_t>, wchar_t> to_utf8;
-  std::string iana_tz_utf8 = to_utf8.to_bytes(iana_tz_utf16);
-  return iana_tz_utf8;
-
 #elif FIREBASE_PLATFORM_LINUX
   // Ubuntu: Check /etc/timezone for the full time zone name.
   FILE* tz_file = fopen("/etc/timezone", "r");

app/src/locale.h

@@ -20,6 +20,8 @@
 
 #include <string>
 
+#include "app/src/include/firebase/internal/platform.h"
+
 namespace firebase {
 namespace internal {
 

app/tests/locale_test.cc

@@ -16,6 +16,7 @@
 
 #include "app/src/locale.h"
 
+#include <codecvt>
 #include <string>
 
 #include "app/src/include/firebase/internal/platform.h"
@@ -24,7 +25,7 @@
 #include "gtest/gtest.h"
 
 #if FIREBASE_PLATFORM_WINDOWS
-
+#include <windows.h>
 #else
 #include <clocale>
 #endif  // FIREBASE_PLATFORM_WINDOWS
@@ -52,5 +53,23 @@ TEST_F(LocaleTest, TestGetLocale) {
   EXPECT_NE(loc.find('_'), std::string::npos);
 }
 
+#if FIREBASE_PLATFORM_WINDOWS
+
+TEST_F(LocaleTest, TestTimeZoneNameInEnglish) {
+  LANGID prev_lang = GetThreadUILanguage();
+
+  SetThreadUILanguage(MAKELANGID(LANG_ENGLISH, SUBLANG_ENGLISH_US));
+  std::string timezone_english = GetTimezone();
+
+  SetThreadUILanguage(MAKELANGID(LANG_GERMAN, SUBLANG_GERMAN));
+  std::string timezone_german = GetTimezone();
+
+  EXPECT_EQ(timezone_english, timezone_german);
+
+  SetThreadUILanguage(prev_lang);
+}
+
+#endif  // FIREBASE_PLATFORM_WINDOWS
+
 }  // namespace internal
 }  // namespace firebase

release_build_files/readme.md

@@ -631,6 +631,8 @@ code.
 -   Changes
     - Auth (Android): Fixed an issue where VerifyPhoneNumber's internal
       builder failed to create PhoneAuthOptions with certain compiler settings.
+    - Remote Config (Desktop): Additional fix for handling of non-English time
+      zone names on Windows.
 
 ### 11.2.0
 -   Changes

remote_config/src/desktop/file_manager.cc

@@ -42,7 +42,7 @@ RemoteConfigFileManager::RemoteConfigFileManager(const std::string& filename,
       AppDataDir(app_data_prefix.c_str(), /*should_create=*/true) + "/" +
       filename;
 #if FIREBASE_PLATFORM_WINDOWS
-  std::wstring_convert<std::codecvt_utf8<wchar_t>> utf8_to_wstring;
+  std::wstring_convert<std::codecvt_utf8_utf16<wchar_t>> utf8_to_wstring;
   file_path_ = utf8_to_wstring.from_bytes(file_path);
 #else
   file_path_ = file_path;