Add Android APKs for NeMo CTC models. (#866)

k2-fsa · May 12, 2024 · db85b2c · db85b2c
1 parent 7322f4e
commit db85b2c
Show file tree

Hide file tree

Showing 9 changed files with 210 additions and 3 deletions.
diff --git a/.github/workflows/apk-vad-asr.yaml b/.github/workflows/apk-vad-asr.yaml
@@ -23,8 +23,8 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest]
-        total: ["3"]
-        index: ["0", "1", "2"]
+        total: ["5"]
+        index: ["0", "1", "2", "3", "4"]
 
     steps:
       - uses: actions/checkout@v4

diff --git a/kotlin-api-examples/run.sh b/kotlin-api-examples/run.sh
@@ -63,6 +63,12 @@ function testOnlineAsr() {
     git clone https://huggingface.co/csukuangfj/sherpa-onnx-streaming-zipformer-en-2023-02-21
   fi
 
+  if [ ! -f ./sherpa-onnx-nemo-streaming-fast-conformer-ctc-en-80ms/tokens.txt ]; then
+    curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-nemo-streaming-fast-conformer-ctc-en-80ms.tar.bz2
+    tar xvf sherpa-onnx-nemo-streaming-fast-conformer-ctc-en-80ms.tar.bz2
+    rm sherpa-onnx-nemo-streaming-fast-conformer-ctc-en-80ms.tar.bz2
+  fi
+
   if [ ! -d ./sherpa-onnx-streaming-zipformer-ctc-multi-zh-hans-2023-12-13 ]; then
     curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-streaming-zipformer-ctc-multi-zh-hans-2023-12-13.tar.bz2
     tar xvf sherpa-onnx-streaming-zipformer-ctc-multi-zh-hans-2023-12-13.tar.bz2

diff --git a/kotlin-api-examples/test_online_asr.kt b/kotlin-api-examples/test_online_asr.kt
@@ -4,6 +4,7 @@ fun main() {
   testOnlineAsr("transducer")
   testOnlineAsr("zipformer2-ctc")
   testOnlineAsr("ctc-hlg")
+  testOnlineAsr("nemo-ctc")
 }
 
 fun testOnlineAsr(type: String) {
@@ -42,6 +43,17 @@ fun testOnlineAsr(type: String) {
             debug = false,
         )
       }
+      "nemo-ctc" -> {
+        waveFilename = "./sherpa-onnx-nemo-streaming-fast-conformer-ctc-en-80ms/test_wavs/0.wav"
+        OnlineModelConfig(
+            neMoCtc = OnlineNeMoCtcModelConfig(
+                model = "./sherpa-onnx-nemo-streaming-fast-conformer-ctc-en-80ms/model.onnx",
+            ),
+            tokens = "./sherpa-onnx-nemo-streaming-fast-conformer-ctc-en-80ms/tokens.txt",
+            numThreads = 1,
+            debug = false,
+        )
+      }
       "ctc-hlg" -> {
         waveFilename = "./sherpa-onnx-streaming-zipformer-ctc-small-2024-03-18/test_wavs/1.wav"
         ctcFstDecoderConfig.graph = "./sherpa-onnx-streaming-zipformer-ctc-small-2024-03-18/HLG.fst"

diff --git a/scripts/apk/generate-asr-apk-script.py b/scripts/apk/generate-asr-apk-script.py
@@ -167,6 +167,48 @@ def get_models():
 
             ls -lh
 
+            popd
+            """,
+        ),
+        Model(
+            model_name="sherpa-onnx-nemo-streaming-fast-conformer-ctc-en-80ms",
+            idx=11,
+            lang="en",
+            short_name="nemo_ctc_80ms",
+            cmd="""
+            pushd $model_name
+            rm -rf test_wavs
+
+            ls -lh
+
+            popd
+            """,
+        ),
+        Model(
+            model_name="sherpa-onnx-nemo-streaming-fast-conformer-ctc-en-480ms",
+            idx=12,
+            lang="en",
+            short_name="nemo_ctc_480ms",
+            cmd="""
+            pushd $model_name
+            rm -rf test_wavs
+
+            ls -lh
+
+            popd
+            """,
+        ),
+        Model(
+            model_name="sherpa-onnx-nemo-streaming-fast-conformer-ctc-en-1040ms",
+            idx=13,
+            lang="en",
+            short_name="nemo_ctc_1040ms",
+            cmd="""
+            pushd $model_name
+            rm -rf test_wavs
+
+            ls -lh
+
             popd
             """,
         ),

diff --git a/scripts/apk/generate-vad-asr-apk-script.py b/scripts/apk/generate-vad-asr-apk-script.py
@@ -103,6 +103,66 @@ def get_models():
 
             ls -lh
 
+            popd
+            """,
+        ),
+        Model(
+            model_name="sherpa-onnx-nemo-fast-conformer-ctc-be-de-en-es-fr-hr-it-pl-ru-uk-20k",
+            idx=7,
+            lang="be_de_en_es_fr_hr_it_pl_ru_uk",
+            short_name="fast_conformer_ctc_20k",
+            cmd="""
+            pushd $model_name
+
+            rm -rfv test_wavs
+
+            ls -lh
+
+            popd
+            """,
+        ),
+        Model(
+            model_name="sherpa-onnx-nemo-fast-conformer-ctc-en-24500",
+            idx=8,
+            lang="en",
+            short_name="fast_conformer_ctc_24500",
+            cmd="""
+            pushd $model_name
+
+            rm -rfv test_wavs
+
+            ls -lh
+
+            popd
+            """,
+        ),
+        Model(
+            model_name="sherpa-onnx-nemo-fast-conformer-ctc-en-de-es-fr-14288",
+            idx=9,
+            lang="en_des_es_fr",
+            short_name="fast_conformer_ctc_14288",
+            cmd="""
+            pushd $model_name
+
+            rm -rfv test_wavs
+
+            ls -lh
+
+            popd
+            """,
+        ),
+        Model(
+            model_name="sherpa-onnx-nemo-fast-conformer-ctc-es-1424",
+            idx=10,
+            lang="es",
+            short_name="fast_conformer_ctc_1424",
+            cmd="""
+            pushd $model_name
+
+            rm -rfv test_wavs
+
+            ls -lh
+
             popd
             """,
         ),

diff --git a/sherpa-onnx/csrc/offline-ctc-model.cc b/sherpa-onnx/csrc/offline-ctc-model.cc
@@ -163,7 +163,8 @@ std::unique_ptr<OfflineCtcModel> OfflineCtcModel::Create(
       return std::make_unique<OfflineNemoEncDecCtcModel>(mgr, config);
       break;
     case ModelType::kEncDecHybridRNNTCTCBPEModel:
-      return std::make_unique<OfflineNemoEncDecHybridRNNTCTCBPEModel>(config);
+      return std::make_unique<OfflineNemoEncDecHybridRNNTCTCBPEModel>(mgr,
+                                                                      config);
       break;
     case ModelType::kTdnn:
       return std::make_unique<OfflineTdnnCtcModel>(mgr, config);

diff --git a/sherpa-onnx/jni/online-recognizer.cc b/sherpa-onnx/jni/online-recognizer.cc
@@ -159,6 +159,18 @@ static OnlineRecognizerConfig GetConfig(JNIEnv *env, jobject config) {
   ans.model_config.zipformer2_ctc.model = p;
   env->ReleaseStringUTFChars(s, p);
 
+  // streaming NeMo CTC
+  fid = env->GetFieldID(model_config_cls, "neMoCtc",
+                        "Lcom/k2fsa/sherpa/onnx/OnlineNeMoCtcModelConfig;");
+  jobject nemo_ctc_config = env->GetObjectField(model_config, fid);
+  jclass nemo_ctc_config_cls = env->GetObjectClass(nemo_ctc_config);
+
+  fid = env->GetFieldID(nemo_ctc_config_cls, "model", "Ljava/lang/String;");
+  s = (jstring)env->GetObjectField(nemo_ctc_config, fid);
+  p = env->GetStringUTFChars(s, nullptr);
+  ans.model_config.nemo_ctc.model = p;
+  env->ReleaseStringUTFChars(s, p);
+
   fid = env->GetFieldID(model_config_cls, "tokens", "Ljava/lang/String;");
   s = (jstring)env->GetObjectField(model_config, fid);
   p = env->GetStringUTFChars(s, nullptr);

diff --git a/sherpa-onnx/kotlin-api/OfflineRecognizer.kt b/sherpa-onnx/kotlin-api/OfflineRecognizer.kt
@@ -231,6 +231,45 @@ fun getOfflineModelConfig(type: Int): OfflineModelConfig? {
             )
         }
 
+        7 -> {
+            val modelDir = "sherpa-onnx-nemo-fast-conformer-ctc-be-de-en-es-fr-hr-it-pl-ru-uk-20k"
+            return OfflineModelConfig(
+                nemo = OfflineNemoEncDecCtcModelConfig(
+                    model = "$modelDir/model.onnx",
+                ),
+                tokens = "$modelDir/tokens.txt",
+            )
+        }
+
+        8 -> {
+            val modelDir = "sherpa-onnx-nemo-fast-conformer-ctc-en-24500"
+            return OfflineModelConfig(
+                nemo = OfflineNemoEncDecCtcModelConfig(
+                    model = "$modelDir/model.onnx",
+                ),
+                tokens = "$modelDir/tokens.txt",
+            )
+        }
+
+        9 -> {
+            val modelDir = "sherpa-onnx-nemo-fast-conformer-ctc-en-de-es-fr-14288"
+            return OfflineModelConfig(
+                nemo = OfflineNemoEncDecCtcModelConfig(
+                    model = "$modelDir/model.onnx",
+                ),
+                tokens = "$modelDir/tokens.txt",
+            )
+        }
+
+        10 -> {
+            val modelDir = "sherpa-onnx-nemo-fast-conformer-ctc-es-1424"
+            return OfflineModelConfig(
+                nemo = OfflineNemoEncDecCtcModelConfig(
+                    model = "$modelDir/model.onnx",
+                ),
+                tokens = "$modelDir/tokens.txt",
+            )
+        }
     }
     return null
 }
diff --git a/sherpa-onnx/kotlin-api/OnlineRecognizer.kt b/sherpa-onnx/kotlin-api/OnlineRecognizer.kt
@@ -29,10 +29,15 @@ data class OnlineZipformer2CtcModelConfig(
     var model: String = "",
 )
 
+data class OnlineNeMoCtcModelConfig(
+    var model: String = "",
+)
+
 data class OnlineModelConfig(
     var transducer: OnlineTransducerModelConfig = OnlineTransducerModelConfig(),
     var paraformer: OnlineParaformerModelConfig = OnlineParaformerModelConfig(),
     var zipformer2Ctc: OnlineZipformer2CtcModelConfig = OnlineZipformer2CtcModelConfig(),
+    var neMoCtc: OnlineNeMoCtcModelConfig = OnlineNeMoCtcModelConfig(),
     var tokens: String,
     var numThreads: Int = 1,
     var debug: Boolean = false,
@@ -318,6 +323,36 @@ fun getModelConfig(type: Int): OnlineModelConfig? {
                 modelType = "zipformer",
             )
         }
+
+        11 -> {
+            val modelDir = "sherpa-onnx-nemo-streaming-fast-conformer-ctc-en-80ms"
+            return OnlineModelConfig(
+                neMoCtc = OnlineNeMoCtcModelConfig(
+                    model = "$modelDir/model.onnx",
+                ),
+                tokens = "$modelDir/tokens.txt",
+            )
+        }
+
+        12 -> {
+            val modelDir = "sherpa-onnx-nemo-streaming-fast-conformer-ctc-en-480ms"
+            return OnlineModelConfig(
+                neMoCtc = OnlineNeMoCtcModelConfig(
+                    model = "$modelDir/model.onnx",
+                ),
+                tokens = "$modelDir/tokens.txt",
+            )
+        }
+
+        13 -> {
+            val modelDir = "sherpa-onnx-nemo-streaming-fast-conformer-ctc-en-1040ms"
+            return OnlineModelConfig(
+                neMoCtc = OnlineNeMoCtcModelConfig(
+                    model = "$modelDir/model.onnx",
+                ),
+                tokens = "$modelDir/tokens.txt",
+            )
+        }
     }
     return null
 }