add streaming websocket server and client (#62)

CMakeLists.txt

@@ -18,6 +18,7 @@ option(BUILD_SHARED_LIBS "Whether to build shared libraries" OFF)
 option(SHERPA_ONNX_ENABLE_PORTAUDIO "Whether to build with portaudio" ON)
 option(SHERPA_ONNX_ENABLE_JNI "Whether to build JNI internface" OFF)
 option(SHERPA_ONNX_ENABLE_C_API "Whether to build C API" ON)
+option(SHERPA_ONNX_ENABLE_WEBSOCKET "Whether to build webscoket server/client" ON)
 
 set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/lib")
 set(CMAKE_LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/lib")
@@ -59,6 +60,8 @@ message(STATUS "SHERPA_ONNX_ENABLE_TESTS ${SHERPA_ONNX_ENABLE_TESTS}")
 message(STATUS "SHERPA_ONNX_ENABLE_CHECK ${SHERPA_ONNX_ENABLE_CHECK}")
 message(STATUS "SHERPA_ONNX_ENABLE_PORTAUDIO ${SHERPA_ONNX_ENABLE_PORTAUDIO}")
 message(STATUS "SHERPA_ONNX_ENABLE_JNI ${SHERPA_ONNX_ENABLE_JNI}")
+message(STATUS "SHERPA_ONNX_ENABLE_C_API ${SHERPA_ONNX_ENABLE_C_API}")
+message(STATUS "SHERPA_ONNX_ENABLE_WEBSOCKET ${SHERPA_ONNX_ENABLE_WEBSOCKET}")
 
 set(CMAKE_CXX_STANDARD 14 CACHE STRING "The C++ version to be used.")
 set(CMAKE_CXX_EXTENSIONS OFF)
@@ -91,6 +94,11 @@ if(SHERPA_ONNX_ENABLE_TESTS)
   include(googletest)
 endif()
 
+if(SHERPA_ONNX_ENABLE_WEBSOCKET)
+  include(websocketpp)
+  include(asio)
+endif()
+
 add_subdirectory(sherpa-onnx)
 
 if(SHERPA_ONNX_ENABLE_C_API)

build-aarch64-linux-gnu.sh

@@ -40,6 +40,10 @@ cmake \
   -DSHERPA_ONNX_ENABLE_TESTS=OFF \
   -DSHERPA_ONNX_ENABLE_PYTHON=OFF \
   -DSHERPA_ONNX_ENABLE_CHECK=OFF \
+  -DSHERPA_ONNX_ENABLE_PORTAUDIO=OFF \
+  -DSHERPA_ONNX_ENABLE_JNI=OFF \
+  -DSHERPA_ONNX_ENABLE_C_API=OFF \
+  -DSHERPA_ONNX_ENABLE_WEBSOCKET=OFF \
   -DCMAKE_TOOLCHAIN_FILE=../toolchains/aarch64-linux-gnu.toolchain.cmake \
   ..
 

build-android-x86-64.sh

@@ -76,6 +76,8 @@ cmake -DCMAKE_TOOLCHAIN_FILE="$ANDROID_NDK/build/cmake/android.toolchain.cmake"
     -DSHERPA_ONNX_ENABLE_JNI=ON \
     -DCMAKE_INSTALL_PREFIX=./install \
     -DANDROID_ABI="x86_64" \
+    -DSHERPA_ONNX_ENABLE_C_API=OFF \
+    -DSHERPA_ONNX_ENABLE_WEBSOCKET=OFF \
     -DANDROID_PLATFORM=android-21 ..
 
 # make VERBOSE=1 -j4

cmake/asio.cmake

@@ -0,0 +1,39 @@
+function(download_asio)
+  include(FetchContent)
+
+  set(asio_URL  "https://github.com/chriskohlhoff/asio/archive/refs/tags/asio-1-24-0.tar.gz")
+  set(asio_HASH "SHA256=cbcaaba0f66722787b1a7c33afe1befb3a012b5af3ad7da7ff0f6b8c9b7a8a5b")
+
+  # If you don't have access to the Internet,
+  # please pre-download asio
+  set(possible_file_locations
+    $ENV{HOME}/Downloads/asio-asio-1-24-0.tar.gz
+    ${PROJECT_SOURCE_DIR}/asio-asio-1-24-0.tar.gz
+    ${PROJECT_BINARY_DIR}/asio-asio-1-24-0.tar.gz
+    /tmp/asio-asio-1-24-0.tar.gz
+    /star-fj/fangjun/download/github/asio-asio-1-24-0.tar.gz
+  )
+
+  foreach(f IN LISTS possible_file_locations)
+    if(EXISTS ${f})
+      set(asio_URL  "file://${f}")
+      break()
+    endif()
+  endforeach()
+
+  FetchContent_Declare(asio
+    URL               ${asio_URL}
+    URL_HASH          ${asio_HASH}
+  )
+
+  FetchContent_GetProperties(asio)
+  if(NOT asio_POPULATED)
+    message(STATUS "Downloading asio ${asio_URL}")
+    FetchContent_Populate(asio)
+  endif()
+  message(STATUS "asio is downloaded to ${asio_SOURCE_DIR}")
+  # add_subdirectory(${asio_SOURCE_DIR} ${asio_BINARY_DIR} EXCLUDE_FROM_ALL)
+  include_directories(${asio_SOURCE_DIR}/asio/include)
+endfunction()
+
+download_asio()

cmake/websocketpp.cmake

@@ -0,0 +1,40 @@
+function(download_websocketpp)
+  include(FetchContent)
+
+  # The latest commit on the develop branch os as 2022-10-22
+  set(websocketpp_URL  "https://github.com/zaphoyd/websocketpp/archive/b9aeec6eaf3d5610503439b4fae3581d9aff08e8.zip")
+  set(websocketpp_HASH "SHA256=1385135ede8191a7fbef9ec8099e3c5a673d48df0c143958216cd1690567f583")
+
+  # If you don't have access to the Internet,
+  # please pre-download websocketpp
+  set(possible_file_locations
+    $ENV{HOME}/Downloads/websocketpp-b9aeec6eaf3d5610503439b4fae3581d9aff08e8.zip
+    ${PROJECT_SOURCE_DIR}/websocketpp-b9aeec6eaf3d5610503439b4fae3581d9aff08e8.zip
+    ${PROJECT_BINARY_DIR}/websocketpp-b9aeec6eaf3d5610503439b4fae3581d9aff08e8.zip
+    /tmp/websocketpp-b9aeec6eaf3d5610503439b4fae3581d9aff08e8.zip
+    /star-fj/fangjun/download/github/websocketpp-b9aeec6eaf3d5610503439b4fae3581d9aff08e8.zip
+  )
+
+  foreach(f IN LISTS possible_file_locations)
+    if(EXISTS ${f})
+      set(websocketpp_URL  "file://${f}")
+      break()
+    endif()
+  endforeach()
+
+  FetchContent_Declare(websocketpp
+    URL               ${websocketpp_URL}
+    URL_HASH          ${websocketpp_HASH}
+  )
+
+  FetchContent_GetProperties(websocketpp)
+  if(NOT websocketpp_POPULATED)
+    message(STATUS "Downloading websocketpp from ${websocketpp_URL}")
+    FetchContent_Populate(websocketpp)
+  endif()
+  message(STATUS "websocketpp is downloaded to ${websocketpp_SOURCE_DIR}")
+  # add_subdirectory(${websocketpp_SOURCE_DIR} ${websocketpp_BINARY_DIR} EXCLUDE_FROM_ALL)
+  include_directories(${websocketpp_SOURCE_DIR})
+endfunction()
+
+download_websocketpp()

sherpa-onnx/csrc/CMakeLists.txt

@@ -4,6 +4,7 @@ set(sources
   cat.cc
   endpoint.cc
   features.cc
+  file-utils.cc
   online-lstm-transducer-model.cc
   online-recognizer.cc
   online-stream.cc
@@ -86,6 +87,32 @@ if(SHERPA_ONNX_ENABLE_PORTAUDIO)
   install(TARGETS sherpa-onnx-microphone DESTINATION bin)
 endif()
 
+if(SHERPA_ONNX_ENABLE_WEBSOCKET)
+  add_definitions(-DASIO_STANDALONE)
+  add_definitions(-D_WEBSOCKETPP_CPP11_STL_)
+
+  add_executable(sherpa-onnx-online-websocket-server
+    online-websocket-server-impl.cc
+    online-websocket-server.cc
+  )
+  target_link_libraries(sherpa-onnx-online-websocket-server sherpa-onnx-core)
+
+
+  add_executable(sherpa-onnx-online-websocket-client
+    online-websocket-client.cc
+  )
+  target_link_libraries(sherpa-onnx-online-websocket-client sherpa-onnx-core)
+
+  if(NOT WIN32)
+    target_link_libraries(sherpa-onnx-online-websocket-server -pthread)
+    target_compile_options(sherpa-onnx-online-websocket-server PRIVATE -Wno-deprecated-declarations)
+
+    target_link_libraries(sherpa-onnx-online-websocket-client -pthread)
+    target_compile_options(sherpa-onnx-online-websocket-client PRIVATE -Wno-deprecated-declarations)
+  endif()
+
+endif()
+
 
 if(SHERPA_ONNX_ENABLE_TESTS)
   set(sherpa_onnx_test_srcs

sherpa-onnx/csrc/CPPLINT.cfg

@@ -0,0 +1 @@
+exclude_files=tee-stream.h

sherpa-onnx/csrc/features.cc

@@ -14,6 +14,15 @@
 
 namespace sherpa_onnx {
 
+void FeatureExtractorConfig::Register(ParseOptions *po) {
+  po->Register("sample-rate", &sampling_rate,
+               "Sampling rate of the input waveform. Must match the one "
+               "expected by the model.");
+
+  po->Register("feat-dim", &feature_dim,
+               "Feature dimension. Must match the one expected by the model.");
+}
+
 std::string FeatureExtractorConfig::ToString() const {
   std::ostringstream os;
 

sherpa-onnx/csrc/features.h

@@ -9,13 +9,17 @@
 #include <string>
 #include <vector>
 
+#include "sherpa-onnx/csrc/parse-options.h"
+
 namespace sherpa_onnx {
 
 struct FeatureExtractorConfig {
   float sampling_rate = 16000;
   int32_t feature_dim = 80;
 
   std::string ToString() const;
+
+  void Register(ParseOptions *po);
 };
 
 class FeatureExtractor {

sherpa-onnx/csrc/file-utils.cc

@@ -0,0 +1,24 @@
+// sherpa-onnx/csrc/file-utils.cc
+//
+// Copyright (c)  2022-2023  Xiaomi Corporation
+
+#include "sherpa-onnx/csrc/file-utils.h"
+
+#include <fstream>
+#include <string>
+
+#include "sherpa-onnx/csrc/log.h"
+
+namespace sherpa_onnx {
+
+bool FileExists(const std::string &filename) {
+  return std::ifstream(filename).good();
+}
+
+void AssertFileExists(const std::string &filename) {
+  if (!FileExists(filename)) {
+    SHERPA_ONNX_LOG(FATAL) << filename << " does not exist!";
+  }
+}
+
+}  // namespace sherpa_onnx

sherpa-onnx/csrc/file-utils.h

@@ -0,0 +1,28 @@
+// sherpa-onnx/csrc/file-utils.h
+//
+// Copyright (c)  2022-2023  Xiaomi Corporation
+
+#ifndef SHERPA_ONNX_CSRC_FILE_UTILS_H_
+#define SHERPA_ONNX_CSRC_FILE_UTILS_H_
+
+#include <fstream>
+#include <string>
+
+namespace sherpa_onnx {
+
+/** Check whether a given path is a file or not
+ *
+ * @param filename Path to check.
+ * @return Return true if the given path is a file; return false otherwise.
+ */
+bool FileExists(const std::string &filename);
+
+/** Abort if the file does not exist.
+ *
+ * @param filename The file to check.
+ */
+void AssertFileExists(const std::string &filename);
+
+}  // namespace sherpa_onnx
+
+#endif  // SHERPA_ONNX_CSRC_FILE_UTILS_H_

sherpa-onnx/csrc/online-recognizer.cc

@@ -12,6 +12,7 @@
 #include <utility>
 #include <vector>
 
+#include "sherpa-onnx/csrc/file-utils.h"
 #include "sherpa-onnx/csrc/online-transducer-decoder.h"
 #include "sherpa-onnx/csrc/online-transducer-greedy-search-decoder.h"
 #include "sherpa-onnx/csrc/online-transducer-model.h"
@@ -31,6 +32,19 @@ static OnlineRecognizerResult Convert(const OnlineTransducerDecoderResult &src,
   return ans;
 }
 
+void OnlineRecognizerConfig::Register(ParseOptions *po) {
+  feat_config.Register(po);
+  model_config.Register(po);
+  endpoint_config.Register(po);
+
+  po->Register("enable-endpoint", &enable_endpoint,
+               "True to enable endpoint detection. False to disable it.");
+}
+
+bool OnlineRecognizerConfig::Validate() const {
+  return model_config.Validate();
+}
+
 std::string OnlineRecognizerConfig::ToString() const {
   std::ostringstream os;
 

sherpa-onnx/csrc/online-recognizer.h

@@ -17,11 +17,15 @@
 #include "sherpa-onnx/csrc/features.h"
 #include "sherpa-onnx/csrc/online-stream.h"
 #include "sherpa-onnx/csrc/online-transducer-model-config.h"
+#include "sherpa-onnx/csrc/parse-options.h"
 
 namespace sherpa_onnx {
 
 struct OnlineRecognizerResult {
   std::string text;
+
+  // TODO(fangjun): Add a method to return a json string
+  std::string ToString() const { return text; }
 };
 
 struct OnlineRecognizerConfig {
@@ -41,6 +45,9 @@ struct OnlineRecognizerConfig {
         endpoint_config(endpoint_config),
         enable_endpoint(enable_endpoint) {}
 
+  void Register(ParseOptions *po);
+  bool Validate() const;
+
   std::string ToString() const;
 };
 

sherpa-onnx/csrc/online-transducer-model-config.cc

@@ -5,8 +5,52 @@
 
 #include <sstream>
 
+#include "sherpa-onnx/csrc/file-utils.h"
+#include "sherpa-onnx/csrc/macros.h"
+
 namespace sherpa_onnx {
 
+void OnlineTransducerModelConfig::Register(ParseOptions *po) {
+  po->Register("encoder", &encoder_filename, "Path to encoder.onnx");
+  po->Register("decoder", &decoder_filename, "Path to decoder.onnx");
+  po->Register("joiner", &joiner_filename, "Path to joiner.onnx");
+  po->Register("tokens", &tokens, "Path to tokens.txt");
+  po->Register("num_threads", &num_threads,
+               "Number of threads to run the neural network");
+
+  po->Register("debug", &debug,
+               "true to print model information while loading it.");
+}
+
+bool OnlineTransducerModelConfig::Validate() const {
+  if (!FileExists(tokens)) {
+    SHERPA_ONNX_LOGE("%s does not exist", tokens.c_str());
+    return false;
+  }
+
+  if (!FileExists(encoder_filename)) {
+    SHERPA_ONNX_LOGE("%s does not exist", encoder_filename.c_str());
+    return false;
+  }
+
+  if (!FileExists(decoder_filename)) {
+    SHERPA_ONNX_LOGE("%s does not exist", decoder_filename.c_str());
+    return false;
+  }
+
+  if (!FileExists(joiner_filename)) {
+    SHERPA_ONNX_LOGE("%s does not exist", joiner_filename.c_str());
+    return false;
+  }
+
+  if (num_threads < 1) {
+    SHERPA_ONNX_LOGE("num_threads should be > 0. Given %d", num_threads);
+    return false;
+  }
+
+  return true;
+}
+
 std::string OnlineTransducerModelConfig::ToString() const {
   std::ostringstream os;
 

sherpa-onnx/csrc/online-transducer-model-config.h

@@ -6,14 +6,16 @@
 
 #include <string>
 
+#include "sherpa-onnx/csrc/parse-options.h"
+
 namespace sherpa_onnx {
 
 struct OnlineTransducerModelConfig {
   std::string encoder_filename;
   std::string decoder_filename;
   std::string joiner_filename;
   std::string tokens;
-  int32_t num_threads;
+  int32_t num_threads = 2;
   bool debug = false;
 
   OnlineTransducerModelConfig() = default;
@@ -29,6 +31,9 @@ struct OnlineTransducerModelConfig {
         num_threads(num_threads),
         debug(debug) {}
 
+  void Register(ParseOptions *po);
+  bool Validate() const;
+
   std::string ToString() const;
 };