Improve incremental transliteration

Author: tindzk

README.md

@@ -151,3 +151,4 @@ translit-scala is licensed under the terms of the Apache v2.0 licence.
 
 ## Contributors
 * Tim Nieradzik
+* Darkhan Kubigenov

shared/src/main/scala/translit/Language.scala

@@ -4,21 +4,26 @@ trait Language {
   def latinToCyrillicOfs(text: String,
                          offset: Int,
                          apostrophes: Boolean = true,
-                         incrementalTranslit: Boolean = false): (Int, Char)
+                         incremental: Boolean = false): (Int, Char)
 
   def latinToCyrillic(text: String,
                       apostrophes: Boolean = true,
-                      incrementalTranslit: Boolean = false): String = {
+                      incremental: Boolean = false): String = {
     val result = new StringBuilder(text.length)
     var offset = 0
 
     while (offset < text.length) {
-      val (length, c) = latinToCyrillicOfs(text, offset, apostrophes, incrementalTranslit)
+      val (length, c) = latinToCyrillicOfs(text, offset, apostrophes, incremental)
       if (length < 0) result.setLength(result.length + length)
       result.append(c)
       offset += 1
     }
 
     result.mkString
   }
+
+  def incrementalNgram(ngram: Map[String, Char]): Map[String, Char] =
+    ngram.map { case (prefix, value) =>
+      (latinToCyrillic(prefix.init) + prefix.last, value)
+    }
 }

shared/src/main/scala/translit/Russian.scala

@@ -3,7 +3,6 @@ package translit
 import translit.Helpers._
 
 object Russian extends Language {
-
   val uniGrams = Map(
     'a' -> 'а',
     'b' -> 'б',
@@ -43,8 +42,7 @@ object Russian extends Language {
     "yo" -> 'ё',
     "yu" -> 'ю'
   )
-
-  val biGramsIncremental = getIncrementalNgram(biGrams)
+  val biGramsIncremental = incrementalNgram(biGrams)
 
   val triGrams = Map.empty[String, Char]
   val triGramsIncremental = Map(
@@ -54,11 +52,6 @@ object Russian extends Language {
   val fourGrams = Map(
     "shch" -> 'щ'
   )
-  val fourGramsIncremental = Map.empty[String, Char]
-
-  def getIncrementalNgram(ngram: Map[String, Char]): Map[String, Char] = ngram ++ ngram.map { case (prefix, value) =>
-    (latinToCyrillic(prefix.slice(0, prefix.length - 1), incrementalTranslit = true) + prefix.last, value)
-  }
 
   /**
     * Converts one character starting from `offset`
@@ -70,15 +63,16 @@ object Russian extends Language {
   def latinToCyrillicOfs(text: String,
                          offset: Int,
                          apostrophes: Boolean = true,
-                         incrementalTranslit: Boolean = false): (Int, Char) = {
-    val (biGramsL, triGramsL, fourGramsL) =
-      if (incrementalTranslit) (biGramsIncremental, triGramsIncremental, fourGramsIncremental)
-      else (biGrams, triGrams, fourGrams)
+                         incremental: Boolean = false): (Int, Char) = {
+    val (biGramsL, triGramsL) =
+      if (incremental) (biGramsIncremental, triGramsIncremental)
+      else (biGrams, triGrams)
+
     val ofs = offset + 1
     if (ofs >= 4 &&
-        fourGramsL.contains(text.substring(ofs - 4, ofs).toLowerCase)) {
+        fourGrams.contains(text.substring(ofs - 4, ofs).toLowerCase)) {
       val chars = text.substring(ofs - 4, ofs)
-      val cyrillic = fourGramsL(chars.toLowerCase)
+      val cyrillic = fourGrams(chars.toLowerCase)
       (-2, restoreCaseFirst(chars, cyrillic))
     } else if (ofs >= 3 &&
       triGramsL.contains(text.substring(ofs - 3, ofs).toLowerCase)) {

shared/src/main/scala/translit/Ukrainian.scala

@@ -47,21 +47,20 @@ object Ukrainian extends Language {
     "ts" -> 'ц',
     "zh" -> 'ж'
   )
-
-  val biGramsIncremental = getIncrementalNgram(biGrams)
+  val biGramsIncremental = incrementalNgram(biGrams)
 
   val triGrams = Map(
     "zgh" -> 'г',
   )
-
-  val triGramsIncremental = getIncrementalNgram(triGrams) + ("шцh" -> 'щ')
+  val triGramsIncremental = incrementalNgram(triGrams) ++ Map(
+    "шцh" -> 'щ',
+    "зґh" -> 'г'
+  )
 
   val fourGrams = Map(
     "shch" -> 'щ'
   )
 
-  val fourGramIncremental = Map.empty[String, Char]
-
   val apostrophePatterns = Set(
     ('b', "ya"),
     ('b', "ye"),
@@ -90,10 +89,6 @@ object Ukrainian extends Language {
     ('z', "yi")
   )
 
-  def getIncrementalNgram(ngram: Map[String, Char]): Map[String, Char] = ngram ++ ngram.map { case (prefix, value) =>
-    (latinToCyrillic(prefix.slice(0, prefix.length - 1), incrementalTranslit = true) + prefix.last, value)
-  }
-
   /**
     * Converts one character starting from `offset`
     *
@@ -104,24 +99,25 @@ object Ukrainian extends Language {
   def latinToCyrillicOfs(text: String,
                          offset: Int,
                          apostrophes: Boolean = true,
-                         incrementalTranslit: Boolean = false): (Int, Char) = {
-    val (biGramsL, triGramsL, fourGramsL) =
-      if (incrementalTranslit) (biGramsIncremental, triGramsIncremental, fourGramIncremental)
-      else (biGrams, triGrams, fourGrams)
+                         incremental: Boolean = false): (Int, Char) = {
+    val (biGramsL, triGramsL) =
+      if (incremental) (biGramsIncremental, triGramsIncremental)
+      else (biGrams, triGrams)
     val ofs = offset + 1
     if (ofs >= 4 &&
-      fourGramsL.contains(text.substring(ofs - 4, ofs).toLowerCase)
+      fourGrams.contains(text.substring(ofs - 4, ofs).toLowerCase)
     ) {
       val chars    = text.substring(ofs - 4, ofs)
-      val cyrillic = fourGramsL(chars.toLowerCase)
+      val cyrillic = fourGrams(chars.toLowerCase)
       (-2, restoreCaseFirst(chars, cyrillic))
     } else if (ofs >= 3 &&
       triGramsL.contains(text.substring(ofs - 3, ofs).toLowerCase)
     ) {
       val chars    = text.substring(ofs - 3, ofs)
       val cyrillic = triGramsL(chars.toLowerCase)
-      val newOffset = if (chars == "шцh") -2 else -1
-      (newOffset, restoreCaseAll(chars, cyrillic))
+      if (incremental && chars.equalsIgnoreCase("шцh"))
+        (-2, restoreCaseFirst(chars, cyrillic))
+      else (-1, restoreCaseAll(chars, cyrillic))
     } else if (ofs >= 2 &&
       biGramsL.contains(text.substring(ofs - 2, ofs).toLowerCase)
     ) {

shared/src/test/scala/translit/RussianSpec.scala

@@ -37,20 +37,20 @@ class RussianSpec extends FunSuite {
     "ягненок" -> "yagnenok",
   )
 
-  correctMapping.foreach {
-    case (cyrillic, latin) =>
-      test(s"$latin -> $cyrillic") {
-        assert(
-          Russian.latinToCyrillic(latin) ==
-            cyrillic)
-      }
+  correctMapping.foreach { case (cyrillic, latin) =>
+    test(s"$latin -> $cyrillic") {
+      assert(Russian.latinToCyrillic(latin) == cyrillic)
+    }
   }
 
-  test("Incremental translit") {
-    assert(Russian.latinToCyrillic("peсhkom", incrementalTranslit = true) == "пешком")
-    assert(Russian.latinToCyrillic("зhizn'", incrementalTranslit = true) == "жизнь")
-    assert(Russian.latinToCyrillic("zhizn'", incrementalTranslit = true) == "жизнь")
-    assert(Russian.latinToCyrillic("багазh", incrementalTranslit = true) == "багаж")
-    assert(Russian.latinToCyrillic("шцhetka", incrementalTranslit = true) == "щетка")
+  test("Incremental transliteration") {
+    assert(Russian.latinToCyrillic("зh"    , incremental = true) == "ж")
+    assert(Russian.latinToCyrillic("шцh"   , incremental = true) == "щ")
+    assert(Russian.latinToCyrillic("Шцh"   , incremental = true) == "Щ")
+    assert(Russian.latinToCyrillic("багазh", incremental = true) == "багаж")
+
+    assert(Russian.latinToCyrillic("peshkom" ) == "пешком")
+    assert(Russian.latinToCyrillic("zhizn'"  ) == "жизнь")
+    assert(Russian.latinToCyrillic("shchetka") == "щетка")
   }
 }

shared/src/test/scala/translit/UkrainianSpec.scala

@@ -278,4 +278,11 @@ class UkrainianSpec extends FunSuite {
     assert(Ukrainian.latinToCyrillic("Puzata Xata") == "Пузата Хата")
     assert(Ukrainian.latinToCyrillic("cqwx") == "цщшх")
   }
+
+  test("Incremental transliteration") {
+    assert(Ukrainian.latinToCyrillic("zgh") == "зг")
+    assert(Ukrainian.latinToCyrillic("зґh", incremental = true) == "зг")
+
+    assert(Ukrainian.latinToCyrillic("Шцh", incremental = true) == "Щ")
+  }
 }