fix(core): normalization segment should end on NFC boundary, not NFD 🍒 🏠

core/src/actions_normalize.cpp

@@ -105,7 +105,7 @@ bool km::core::actions_normalize(
     boundary prior to the intersection of the cached_context and the output.
   */
   if(!output.empty()) {
-    while(n > 0 && !km::core::util::has_nfd_boundary_before(output[0])) {
+    while(n > 0 && !km::core::util::has_nfc_boundary_before(output[0])) {
       // The output may interact with the context further in normalization. We
       // need to copy characters back further until we reach a normalization
       // boundary.

core/src/util_normalize.cpp

@@ -208,7 +208,7 @@ bool is_nfd(const std::u32string& str) {
 #endif
 }
 
-bool has_nfd_boundary_before(km_core_usv cp) {
+bool has_nfc_boundary_before(km_core_usv cp) {
 #ifdef __EMSCRIPTEN__
 // it's a negative table. entries in the table mean returning false. non-entries return true.
   for (auto i=0;i<(km_noBoundaryBefore_entries*2);i+=2) {
@@ -221,9 +221,9 @@ bool has_nfd_boundary_before(km_core_usv cp) {
   return true; // fallthrough
 #else
   UErrorCode status = U_ZERO_ERROR;
-  auto nfd = getNFD(status);
-  if (nfd == nullptr) return false;
-  return nfd->hasBoundaryBefore(cp);
+  auto nfc = getNFC(status);
+  if (nfc == nullptr) return false;
+  return nfc->hasBoundaryBefore(cp);
 #endif
 }
 

core/src/util_normalize.hpp

@@ -39,7 +39,7 @@ bool is_nfd(const std::u16string& str);
 bool is_nfd(const std::u32string& str);
 
 /** @return true if cp can interacts with prior chars */
-bool has_nfd_boundary_before(km_core_usv cp);
+bool has_nfc_boundary_before(km_core_usv cp);
 
 /** convenience function, caller owns storage */
 km_core_usv *string_to_usv(const std::u32string& src);

core/src/util_normalize_table_generator.cpp

@@ -28,7 +28,7 @@
 
 
 int
-write_nfd_table() {
+write_nfc_table() {
 #ifndef __EMSCRIPTEN__
   std::cerr << "Note: This is unusual - this generator is usually only run under emscripten!" << std::endl;
 #endif
@@ -49,13 +49,13 @@ write_nfd_table() {
   std::cout << std::endl;
   // we're going to need an NFD normalizer
   UErrorCode status           = U_ZERO_ERROR;
-  const icu::Normalizer2 *nfd = icu::Normalizer2::getNFDInstance(status);
+  const icu::Normalizer2 *nfc = icu::Normalizer2::getNFCInstance(status);
   assert(U_SUCCESS(status));
 
   // collect the raw list of chars that do NOT have a boundary before them.
   std::vector<km_core_usv> noBoundary;
   for (km_core_usv ch = 0; ch < km::core::kmx::Uni_MAX_CODEPOINT; ch++) {
-    bool bb = nfd->hasBoundaryBefore(ch);
+    bool bb = nfc->hasBoundaryBefore(ch);
     assert(!(ch == 0 && !bb)); // assert that we can use U+0000 as a terminator
     if (bb) continue; //only emit nonboundary
     noBoundary.push_back(ch);
@@ -102,6 +102,6 @@ write_nfd_table() {
 
 int
 main(int /*argc*/, const char * /*argv*/[]) {
-  write_nfd_table();
+  write_nfc_table();
   return 0;
 }

core/tests/unit/ldml/unicode.tests.cpp

@@ -198,20 +198,20 @@ void test_has_boundary_before() {
   assert_basic_equal(icu4c_unicode, header_unicode);
   assert_basic_equal(icu4c_icu, header_icu);
 
-  std::cout << std::endl << "Now, let's make sure has_nfd_boundary_before() matches ICU." << std::endl;
+  std::cout << std::endl << "Now, let's make sure has_nfc_boundary_before() matches ICU." << std::endl;
 
   UErrorCode status           = U_ZERO_ERROR;
-  const icu::Normalizer2 *nfd = icu::Normalizer2::getNFDInstance(status);
+  const icu::Normalizer2 *nfc = icu::Normalizer2::getNFCInstance(status);
   UASSERT_SUCCESS(status);
 
   // now, test that hasBoundaryBefore is the same
   for (km_core_usv cp = 0; cp < km::core::kmx::Uni_MAX_CODEPOINT; cp++) {
-    auto km_hbb = km::core::util::has_nfd_boundary_before(cp);
-    auto icu_hbb = nfd->hasBoundaryBefore(cp);
+    auto km_hbb = km::core::util::has_nfc_boundary_before(cp);
+    auto icu_hbb = nfc->hasBoundaryBefore(cp);
 
     if (km_hbb != icu_hbb) {
       std::cerr << "Error: util_normalize_table.h said " << boolstr(km_hbb) << " but ICU said " << boolstr(icu_hbb) << " for "
-                << "has_nfd_boundary_before(0x" << std::hex << cp << std::dec << ")" << std::endl;
+                << "has_nfc_boundary_before(0x" << std::hex << cp << std::dec << ")" << std::endl;
     }
     test_assert(km_hbb == icu_hbb);
   }