fix(core): normalization segment should end on NFC boundary, not NFD

core/src/actions_normalize.cpp

@@ -105,7 +105,7 @@ bool km::core::actions_normalize(
     boundary prior to the intersection of the cached_context and the output.
   */
   if(!output.empty()) {
-    while(n > 0 && !km::core::util::has_nfd_boundary_before(output[0])) {
+    while(n > 0 && !km::core::util::has_nfc_boundary_before(output[0])) {
       // The output may interact with the context further in normalization. We
       // need to copy characters back further until we reach a normalization
       // boundary.

core/src/util_normalize.cpp

@@ -208,7 +208,7 @@ bool is_nfd(const std::u32string& str) {
 #endif
 }
 
-bool has_nfd_boundary_before(km_core_usv cp) {
+bool has_nfc_boundary_before(km_core_usv cp) {
 #ifdef __EMSCRIPTEN__
 // it's a negative table. entries in the table mean returning false. non-entries return true.
   for (auto i=0;i<(km_noBoundaryBefore_entries*2);i+=2) {
@@ -221,9 +221,9 @@ bool has_nfd_boundary_before(km_core_usv cp) {
   return true; // fallthrough
 #else
   UErrorCode status = U_ZERO_ERROR;
-  auto nfd = getNFD(status);
-  if (nfd == nullptr) return false;
-  return nfd->hasBoundaryBefore(cp);
+  auto nfc = getNFC(status);
+  if (nfc == nullptr) return false;
+  return nfc->hasBoundaryBefore(cp);
 #endif
 }
 

core/src/util_normalize.hpp

@@ -39,7 +39,7 @@ bool is_nfd(const std::u16string& str);
 bool is_nfd(const std::u32string& str);
 
 /** @return true if cp can interacts with prior chars */
-bool has_nfd_boundary_before(km_core_usv cp);
+bool has_nfc_boundary_before(km_core_usv cp);
 
 /** convenience function, caller owns storage */
 km_core_usv *string_to_usv(const std::u32string& src);

core/src/util_normalize_table_generator.cpp

@@ -28,7 +28,7 @@
 
 
 int
-write_nfd_table() {
+write_nfc_table() {
 #ifndef __EMSCRIPTEN__
   std::cerr << "Note: This is unusual - this generator is usually only run under emscripten!" << std::endl;
 #endif
@@ -49,13 +49,13 @@ write_nfd_table() {
   std::cout << std::endl;
   // we're going to need an NFD normalizer
   UErrorCode status           = U_ZERO_ERROR;
-  const icu::Normalizer2 *nfd = icu::Normalizer2::getNFDInstance(status);
+  const icu::Normalizer2 *nfc = icu::Normalizer2::getNFCInstance(status);
   assert(U_SUCCESS(status));
 
   // collect the raw list of chars that do NOT have a boundary before them.
   std::vector<km_core_usv> noBoundary;
   for (km_core_usv ch = 0; ch < km::core::kmx::Uni_MAX_CODEPOINT; ch++) {
-    bool bb = nfd->hasBoundaryBefore(ch);
+    bool bb = nfc->hasBoundaryBefore(ch);
     assert(!(ch == 0 && !bb)); // assert that we can use U+0000 as a terminator
     if (bb) continue; //only emit nonboundary
     noBoundary.push_back(ch);
@@ -102,6 +102,6 @@ write_nfd_table() {
 
 int
 main(int /*argc*/, const char * /*argv*/[]) {
-  write_nfd_table();
+  write_nfc_table();
   return 0;
 }

core/tests/unit/kmnkbd/actions_normalize.tests.cpp

@@ -358,6 +358,18 @@ void run_actions_normalize_tests() {
     /* app_context: */                   u"\u0323\u0300\u0302"
   );
 
+  // #15505 - normalization of Bengali characters
+  test_actions_normalize(
+    "Bengali normalization of U+09C7 U+09D7 -> U+09CC",
+    /* app context pre transform: */     u"\u0995\u09C7",
+    /* cached context post transform: */ u"\u0995\u09C7\u09D7",
+    /* cached context post transform: */ nullptr,
+    /* action del, output: */            0, U"\u09D7",
+    // ---- results ----
+    /* action del, output: */            1, U"\u09CC",
+    /* app_context: */                   u"\u0995\u09CC"
+  );
+
   // Modifies the base as well as diacritic
 
   test_actions_normalize(