Merge pull request #4 from m5stack/dev

Dev

Author: dianjixz

.gitignore

@@ -1,3 +1,4 @@
 .vscode/settings.json
 projects/core135_llm_product_test_ui
-projects/imx678_test
+projects/imx678_test
+projects/test_*

ext_components/ax-samples/SConstruct

@@ -29,14 +29,13 @@ if "CONFIG_AX_SAMPLES_ENABLED" in os.environ:
     LINK_SEARCH_PATH = []
 
     INCLUDE += [
-        os.path.join(env["GIT_REPO_LISTS"]["ax-samples"]["path"], "examples/base"),
-        os.path.join(env["GIT_REPO_LISTS"]["ax-samples"]["path"], "examples/utilities"),
+        os.path.join(env["GIT_REPO_LISTS"]["ax-samples"]["path"], "examples"),
     ]
     if "CONFIG_AX_620E_MSP_ENABLED" in os.environ:
         INCLUDE += [
             os.path.join(
                 env["GIT_REPO_LISTS"]["ax-samples"]["path"],
-                "examples/ax620e/middleware",
+                "examples/ax620e",
             )
         ]
 

projects/llm_framework/SConstruct

@@ -5,7 +5,7 @@ import shutil
 os.environ['SDK_PATH'] = os.path.normpath(str(Path(os.getcwd())/'..'/'..'/'SDK'))
 os.environ['EXT_COMPONENTS_PATH'] = os.path.normpath(str(Path(os.getcwd())/'..'/'..'/'ext_components'))
 
-version = 'v0.0.5'
+version = 'v0.0.7'
 static_lib = 'static_lib'
 update = False
 
@@ -26,4 +26,6 @@ if update:
         exec(f.read())
     down_url = "https://m5stack.oss-cn-shenzhen.aliyuncs.com/resource/linux/llm/static_lib_{}.tar.gz".format(version)
     down_path = check_wget_down(down_url, "static_lib_{}.tar.gz".format(version))
+    if os.path.exists(static_lib):
+        shutil.rmtree(static_lib)
     shutil.move(down_path, static_lib)

projects/llm_framework/include/sherpa/sherpa-onnx/csrc/fast-clustering-config.h

@@ -0,0 +1,39 @@
+// sherpa-onnx/csrc/fast-clustering-config.h
+//
+// Copyright (c)  2024  Xiaomi Corporation
+
+#ifndef SHERPA_ONNX_CSRC_FAST_CLUSTERING_CONFIG_H_
+#define SHERPA_ONNX_CSRC_FAST_CLUSTERING_CONFIG_H_
+
+#include <string>
+
+#include "sherpa-onnx/csrc/parse-options.h"
+
+namespace sherpa_onnx {
+
+struct FastClusteringConfig {
+  // If greater than 0, then threshold is ignored.
+  //
+  // We strongly recommend that you set it if you know the number of clusters
+  // in advance
+  int32_t num_clusters = -1;
+
+  // distance threshold.
+  //
+  // The smaller, the more clusters it will generate.
+  // The larger, the fewer clusters it will generate.
+  float threshold = 0.5;
+
+  FastClusteringConfig() = default;
+
+  FastClusteringConfig(int32_t num_clusters, float threshold)
+      : num_clusters(num_clusters), threshold(threshold) {}
+
+  std::string ToString() const;
+
+  void Register(ParseOptions *po);
+  bool Validate() const;
+};
+
+}  // namespace sherpa_onnx
+#endif  // SHERPA_ONNX_CSRC_FAST_CLUSTERING_CONFIG_H_

projects/llm_framework/include/sherpa/sherpa-onnx/csrc/fast-clustering.h

@@ -0,0 +1,43 @@
+// sherpa-onnx/csrc/fast-clustering.h
+//
+// Copyright (c)  2024  Xiaomi Corporation
+
+#ifndef SHERPA_ONNX_CSRC_FAST_CLUSTERING_H_
+#define SHERPA_ONNX_CSRC_FAST_CLUSTERING_H_
+
+#include <memory>
+#include <vector>
+
+#include "sherpa-onnx/csrc/fast-clustering-config.h"
+
+namespace sherpa_onnx {
+
+class FastClustering {
+ public:
+  explicit FastClustering(const FastClusteringConfig &config);
+  ~FastClustering();
+
+  /**
+   * @param features Pointer to a 2-D feature matrix in row major. Each row
+   *                 is a feature frame. It is changed in-place. We will
+   *                 convert each feature frame to a normalized vector.
+   *                 That is, the L2-norm of each vector will be equal to 1.
+   *                 It uses cosine dissimilarity,
+   *                 which is 1 - (cosine similarity)
+   * @param num_rows Number of feature frames
+   * @param num-cols The feature dimension.
+   *
+   * @return Return a vector of size num_rows. ans[i] contains the label
+   *         for the i-th feature frame, i.e., the i-th row of the feature
+   *         matrix.
+   */
+  std::vector<int32_t> Cluster(float *features, int32_t num_rows,
+                               int32_t num_cols) const;
+
+ private:
+  class Impl;
+  std::unique_ptr<Impl> impl_;
+};
+
+}  // namespace sherpa_onnx
+#endif  // SHERPA_ONNX_CSRC_FAST_CLUSTERING_H_

projects/llm_framework/include/sherpa/sherpa-onnx/csrc/features.h

@@ -57,6 +57,7 @@ struct FeatureExtractorConfig {
   float frame_length_ms = 25.0f;  // in milliseconds.
   bool is_librosa = false;
   bool remove_dc_offset = true;       // Subtract mean of wave before FFT.
+  float preemph_coeff = 0.97f;        // Preemphasis coefficient.
   std::string window_type = "povey";  // e.g. Hamming window
 
   // For models from NeMo

projects/llm_framework/include/sherpa/sherpa-onnx/csrc/fst-utils.h

@@ -0,0 +1,18 @@
+// sherpa-onnx/csrc/fst-utils.h
+//
+// Copyright (c)  2024  Xiaomi Corporation
+
+#ifndef SHERPA_ONNX_CSRC_FST_UTILS_H_
+#define SHERPA_ONNX_CSRC_FST_UTILS_H_
+
+#include <string>
+
+#include "fst/fstlib.h"
+
+namespace sherpa_onnx {
+
+fst::Fst<fst::StdArc> *ReadGraph(const std::string &filename);
+
+}
+
+#endif  // SHERPA_ONNX_CSRC_FST_UTILS_H_

projects/llm_framework/include/sherpa/sherpa-onnx/csrc/hypothesis.h

@@ -51,8 +51,13 @@ struct Hypothesis {
   // LM log prob if any.
   double lm_log_prob = 0;
 
-  // the nn lm score for next token given the current ys
+  // the nn lm score for next token given the current ys,
+  // when using shallow fusion
   CopyableOrtValue nn_lm_scores;
+
+  // cur scored tokens by RNN LM, when rescoring
+  int32_t cur_scored_pos = 0;
+
   // the nn lm states
   std::vector<CopyableOrtValue> nn_lm_states;
 

projects/llm_framework/include/sherpa/sherpa-onnx/csrc/keyword-spotter-transducer-impl.h

@@ -66,15 +66,25 @@ class KeywordSpotterTransducerImpl : public KeywordSpotterImpl {
  public:
   explicit KeywordSpotterTransducerImpl(const KeywordSpotterConfig &config)
       : config_(config),
-        model_(OnlineTransducerModel::Create(config.model_config)),
-        sym_(config.model_config.tokens) {
+        model_(OnlineTransducerModel::Create(config.model_config)) {
+    if (!config.model_config.tokens_buf.empty()) {
+      sym_ = SymbolTable(config.model_config.tokens_buf, false);
+    } else {
+      /// assuming tokens_buf and tokens are guaranteed not being both empty
+      sym_ = SymbolTable(config.model_config.tokens, true);
+    }
+
     if (sym_.Contains("<unk>")) {
       unk_id_ = sym_["<unk>"];
     }
 
     model_->SetFeatureDim(config.feat_config.feature_dim);
 
-    InitKeywords();
+    if (config.keywords_buf.empty()) {
+      InitKeywords();
+    } else {
+      InitKeywordsFromBufStr();
+    }
 
     decoder_ = std::make_unique<TransducerKeywordDecoder>(
         model_.get(), config_.max_active_paths, config_.num_trailing_blanks,
@@ -305,6 +315,12 @@ class KeywordSpotterTransducerImpl : public KeywordSpotterImpl {
   }
 #endif
 
+  void InitKeywordsFromBufStr() {
+    // keywords_buf's content is supposed to be same as the keywords_file's
+    std::istringstream is(config_.keywords_buf);
+    InitKeywords(is);
+  }
+
   void InitOnlineStream(OnlineStream *stream) const {
     auto r = decoder_->GetEmptyResult();
     SHERPA_ONNX_CHECK_EQ(r.hyps.Size(), 1);

projects/llm_framework/include/sherpa/sherpa-onnx/csrc/keyword-spotter.h

@@ -69,6 +69,11 @@ struct KeywordSpotterConfig {
 
   std::string keywords_file;
 
+  /// if keywords_buf is non-empty,
+  /// the keywords will be loaded from the buffer instead of from the
+  /// "keywrods_file"
+  std::string keywords_buf;
+
   KeywordSpotterConfig() = default;
 
   KeywordSpotterConfig(const FeatureExtractorConfig &feat_config,