diff --git a/.github/workflows/apk-vad-asr.yaml b/.github/workflows/apk-vad-asr.yaml index dc4d97e42..09e8c73d4 100644 --- a/.github/workflows/apk-vad-asr.yaml +++ b/.github/workflows/apk-vad-asr.yaml @@ -23,8 +23,8 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest] - total: ["3"] - index: ["0", "1", "2"] + total: ["5"] + index: ["0", "1", "2", "3", "4"] steps: - uses: actions/checkout@v4 diff --git a/kotlin-api-examples/run.sh b/kotlin-api-examples/run.sh index 79856bc90..41a37ee32 100755 --- a/kotlin-api-examples/run.sh +++ b/kotlin-api-examples/run.sh @@ -63,6 +63,12 @@ function testOnlineAsr() { git clone https://huggingface.co/csukuangfj/sherpa-onnx-streaming-zipformer-en-2023-02-21 fi + if [ ! -f ./sherpa-onnx-nemo-streaming-fast-conformer-ctc-en-80ms/tokens.txt ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-nemo-streaming-fast-conformer-ctc-en-80ms.tar.bz2 + tar xvf sherpa-onnx-nemo-streaming-fast-conformer-ctc-en-80ms.tar.bz2 + rm sherpa-onnx-nemo-streaming-fast-conformer-ctc-en-80ms.tar.bz2 + fi + if [ ! -d ./sherpa-onnx-streaming-zipformer-ctc-multi-zh-hans-2023-12-13 ]; then curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-streaming-zipformer-ctc-multi-zh-hans-2023-12-13.tar.bz2 tar xvf sherpa-onnx-streaming-zipformer-ctc-multi-zh-hans-2023-12-13.tar.bz2 diff --git a/kotlin-api-examples/test_online_asr.kt b/kotlin-api-examples/test_online_asr.kt index 7376ab61e..ece01de66 100644 --- a/kotlin-api-examples/test_online_asr.kt +++ b/kotlin-api-examples/test_online_asr.kt @@ -4,6 +4,7 @@ fun main() { testOnlineAsr("transducer") testOnlineAsr("zipformer2-ctc") testOnlineAsr("ctc-hlg") + testOnlineAsr("nemo-ctc") } fun testOnlineAsr(type: String) { @@ -42,6 +43,17 @@ fun testOnlineAsr(type: String) { debug = false, ) } + "nemo-ctc" -> { + waveFilename = "./sherpa-onnx-nemo-streaming-fast-conformer-ctc-en-80ms/test_wavs/0.wav" + OnlineModelConfig( + neMoCtc = OnlineNeMoCtcModelConfig( + model = "./sherpa-onnx-nemo-streaming-fast-conformer-ctc-en-80ms/model.onnx", + ), + tokens = "./sherpa-onnx-nemo-streaming-fast-conformer-ctc-en-80ms/tokens.txt", + numThreads = 1, + debug = false, + ) + } "ctc-hlg" -> { waveFilename = "./sherpa-onnx-streaming-zipformer-ctc-small-2024-03-18/test_wavs/1.wav" ctcFstDecoderConfig.graph = "./sherpa-onnx-streaming-zipformer-ctc-small-2024-03-18/HLG.fst" diff --git a/scripts/apk/generate-asr-apk-script.py b/scripts/apk/generate-asr-apk-script.py index f53239edf..845819c5c 100755 --- a/scripts/apk/generate-asr-apk-script.py +++ b/scripts/apk/generate-asr-apk-script.py @@ -167,6 +167,48 @@ def get_models(): ls -lh + popd + """, + ), + Model( + model_name="sherpa-onnx-nemo-streaming-fast-conformer-ctc-en-80ms", + idx=11, + lang="en", + short_name="nemo_ctc_80ms", + cmd=""" + pushd $model_name + rm -rf test_wavs + + ls -lh + + popd + """, + ), + Model( + model_name="sherpa-onnx-nemo-streaming-fast-conformer-ctc-en-480ms", + idx=12, + lang="en", + short_name="nemo_ctc_480ms", + cmd=""" + pushd $model_name + rm -rf test_wavs + + ls -lh + + popd + """, + ), + Model( + model_name="sherpa-onnx-nemo-streaming-fast-conformer-ctc-en-1040ms", + idx=13, + lang="en", + short_name="nemo_ctc_1040ms", + cmd=""" + pushd $model_name + rm -rf test_wavs + + ls -lh + popd """, ), diff --git a/scripts/apk/generate-vad-asr-apk-script.py b/scripts/apk/generate-vad-asr-apk-script.py index 95409b9a7..009c565a7 100755 --- a/scripts/apk/generate-vad-asr-apk-script.py +++ b/scripts/apk/generate-vad-asr-apk-script.py @@ -103,6 +103,66 @@ def get_models(): ls -lh + popd + """, + ), + Model( + model_name="sherpa-onnx-nemo-fast-conformer-ctc-be-de-en-es-fr-hr-it-pl-ru-uk-20k", + idx=7, + lang="be_de_en_es_fr_hr_it_pl_ru_uk", + short_name="fast_conformer_ctc_20k", + cmd=""" + pushd $model_name + + rm -rfv test_wavs + + ls -lh + + popd + """, + ), + Model( + model_name="sherpa-onnx-nemo-fast-conformer-ctc-en-24500", + idx=8, + lang="en", + short_name="fast_conformer_ctc_24500", + cmd=""" + pushd $model_name + + rm -rfv test_wavs + + ls -lh + + popd + """, + ), + Model( + model_name="sherpa-onnx-nemo-fast-conformer-ctc-en-de-es-fr-14288", + idx=9, + lang="en_des_es_fr", + short_name="fast_conformer_ctc_14288", + cmd=""" + pushd $model_name + + rm -rfv test_wavs + + ls -lh + + popd + """, + ), + Model( + model_name="sherpa-onnx-nemo-fast-conformer-ctc-es-1424", + idx=10, + lang="es", + short_name="fast_conformer_ctc_1424", + cmd=""" + pushd $model_name + + rm -rfv test_wavs + + ls -lh + popd """, ), diff --git a/sherpa-onnx/csrc/offline-ctc-model.cc b/sherpa-onnx/csrc/offline-ctc-model.cc index e22027e71..ed18720a3 100644 --- a/sherpa-onnx/csrc/offline-ctc-model.cc +++ b/sherpa-onnx/csrc/offline-ctc-model.cc @@ -163,7 +163,8 @@ std::unique_ptr OfflineCtcModel::Create( return std::make_unique(mgr, config); break; case ModelType::kEncDecHybridRNNTCTCBPEModel: - return std::make_unique(config); + return std::make_unique(mgr, + config); break; case ModelType::kTdnn: return std::make_unique(mgr, config); diff --git a/sherpa-onnx/jni/online-recognizer.cc b/sherpa-onnx/jni/online-recognizer.cc index c88cfd6df..ce0d562f2 100644 --- a/sherpa-onnx/jni/online-recognizer.cc +++ b/sherpa-onnx/jni/online-recognizer.cc @@ -159,6 +159,18 @@ static OnlineRecognizerConfig GetConfig(JNIEnv *env, jobject config) { ans.model_config.zipformer2_ctc.model = p; env->ReleaseStringUTFChars(s, p); + // streaming NeMo CTC + fid = env->GetFieldID(model_config_cls, "neMoCtc", + "Lcom/k2fsa/sherpa/onnx/OnlineNeMoCtcModelConfig;"); + jobject nemo_ctc_config = env->GetObjectField(model_config, fid); + jclass nemo_ctc_config_cls = env->GetObjectClass(nemo_ctc_config); + + fid = env->GetFieldID(nemo_ctc_config_cls, "model", "Ljava/lang/String;"); + s = (jstring)env->GetObjectField(nemo_ctc_config, fid); + p = env->GetStringUTFChars(s, nullptr); + ans.model_config.nemo_ctc.model = p; + env->ReleaseStringUTFChars(s, p); + fid = env->GetFieldID(model_config_cls, "tokens", "Ljava/lang/String;"); s = (jstring)env->GetObjectField(model_config, fid); p = env->GetStringUTFChars(s, nullptr); diff --git a/sherpa-onnx/kotlin-api/OfflineRecognizer.kt b/sherpa-onnx/kotlin-api/OfflineRecognizer.kt index cca5e8757..3b468997b 100644 --- a/sherpa-onnx/kotlin-api/OfflineRecognizer.kt +++ b/sherpa-onnx/kotlin-api/OfflineRecognizer.kt @@ -231,6 +231,45 @@ fun getOfflineModelConfig(type: Int): OfflineModelConfig? { ) } + 7 -> { + val modelDir = "sherpa-onnx-nemo-fast-conformer-ctc-be-de-en-es-fr-hr-it-pl-ru-uk-20k" + return OfflineModelConfig( + nemo = OfflineNemoEncDecCtcModelConfig( + model = "$modelDir/model.onnx", + ), + tokens = "$modelDir/tokens.txt", + ) + } + + 8 -> { + val modelDir = "sherpa-onnx-nemo-fast-conformer-ctc-en-24500" + return OfflineModelConfig( + nemo = OfflineNemoEncDecCtcModelConfig( + model = "$modelDir/model.onnx", + ), + tokens = "$modelDir/tokens.txt", + ) + } + + 9 -> { + val modelDir = "sherpa-onnx-nemo-fast-conformer-ctc-en-de-es-fr-14288" + return OfflineModelConfig( + nemo = OfflineNemoEncDecCtcModelConfig( + model = "$modelDir/model.onnx", + ), + tokens = "$modelDir/tokens.txt", + ) + } + + 10 -> { + val modelDir = "sherpa-onnx-nemo-fast-conformer-ctc-es-1424" + return OfflineModelConfig( + nemo = OfflineNemoEncDecCtcModelConfig( + model = "$modelDir/model.onnx", + ), + tokens = "$modelDir/tokens.txt", + ) + } } return null } diff --git a/sherpa-onnx/kotlin-api/OnlineRecognizer.kt b/sherpa-onnx/kotlin-api/OnlineRecognizer.kt index 894943768..02ad5e5e3 100644 --- a/sherpa-onnx/kotlin-api/OnlineRecognizer.kt +++ b/sherpa-onnx/kotlin-api/OnlineRecognizer.kt @@ -29,10 +29,15 @@ data class OnlineZipformer2CtcModelConfig( var model: String = "", ) +data class OnlineNeMoCtcModelConfig( + var model: String = "", +) + data class OnlineModelConfig( var transducer: OnlineTransducerModelConfig = OnlineTransducerModelConfig(), var paraformer: OnlineParaformerModelConfig = OnlineParaformerModelConfig(), var zipformer2Ctc: OnlineZipformer2CtcModelConfig = OnlineZipformer2CtcModelConfig(), + var neMoCtc: OnlineNeMoCtcModelConfig = OnlineNeMoCtcModelConfig(), var tokens: String, var numThreads: Int = 1, var debug: Boolean = false, @@ -318,6 +323,36 @@ fun getModelConfig(type: Int): OnlineModelConfig? { modelType = "zipformer", ) } + + 11 -> { + val modelDir = "sherpa-onnx-nemo-streaming-fast-conformer-ctc-en-80ms" + return OnlineModelConfig( + neMoCtc = OnlineNeMoCtcModelConfig( + model = "$modelDir/model.onnx", + ), + tokens = "$modelDir/tokens.txt", + ) + } + + 12 -> { + val modelDir = "sherpa-onnx-nemo-streaming-fast-conformer-ctc-en-480ms" + return OnlineModelConfig( + neMoCtc = OnlineNeMoCtcModelConfig( + model = "$modelDir/model.onnx", + ), + tokens = "$modelDir/tokens.txt", + ) + } + + 13 -> { + val modelDir = "sherpa-onnx-nemo-streaming-fast-conformer-ctc-en-1040ms" + return OnlineModelConfig( + neMoCtc = OnlineNeMoCtcModelConfig( + model = "$modelDir/model.onnx", + ), + tokens = "$modelDir/tokens.txt", + ) + } } return null }