diff --git a/.github/workflows/build_docker.yml b/.github/workflows/build_docker.yml index d469efd6ac..2d6f616cb6 100644 --- a/.github/workflows/build_docker.yml +++ b/.github/workflows/build_docker.yml @@ -18,13 +18,18 @@ jobs: include: - image_name: "ubuntu2204" workdir: "tools/docker_for_building/ubuntu/22.04" + platforms: linux/amd64,linux/arm64 - image_name: "ubuntu1804" workdir: "tools/docker_for_building/ubuntu/18.04" + platforms: linux/amd64 runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Build & Publish to Github Container Registry uses: elgohr/Publish-Docker-Github-Action@v5 with: @@ -33,5 +38,6 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} registry: ghcr.io workdir: ${{ matrix.workdir }} + platforms: ${{ matrix.platforms }} snapshot: true tags: "${{ github.ref == 'refs/heads/main' && 'latest,' || '' }}" diff --git a/.github/workflows/build_linux_arm64.yml b/.github/workflows/build_linux_arm64.yml new file mode 100644 index 0000000000..50b73511b1 --- /dev/null +++ b/.github/workflows/build_linux_arm64.yml @@ -0,0 +1,99 @@ +name: Build - linux arm64 + +on: + release: + types: [created] + pull_request: + types: [opened, synchronize, reopened] + paths-ignore: + - "tools/**" + - "docs/**" + - ".vscode/**" + - ".devcontainer/**" + - ".github/**" + - "!.github/workflows/build_linux_arm64.yml" + - "**.md" + +jobs: + build: + runs-on: ubuntu-latest + concurrency: + group: build-linux-arm64-${{ github.head_ref }}-${{ matrix.compiler }}-${{ matrix.build_type }} + cancel-in-progress: true + strategy: + matrix: + compiler: [gcc] + build_type: [release] + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: "0" + submodules: "true" + + - name: Update version + run: | + git config --global --add safe.directory $(pwd) + python3 tools/version/update_version_in_ten_framework.py + python3 tools/version/check_version_in_ten_framework.py + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + with: + platforms: arm64 + + - name: Build + run: | + docker run --rm --platform linux/arm64 \ + -v $(pwd):/${{ github.workspace }} -w ${{ github.workspace }} \ + ghcr.io/ten-framework/ten_building_ubuntu2204 \ + bash -c "\ + export PATH=$(pwd)/core/ten_gn:/usr/local/go/bin:/root/go/bin:/root/.cargo/bin:$PATH && \ + echo $PATH && \ + go env -w GOFLAGS="-buildvcs=false" && \ + go1.20.12 download && \ + rustup default nightly && \ + tgn gen linux arm64 ${{ matrix.build_type }} -- is_clang=${{ matrix.compiler == 'gcc' && 'false' || 'true' }} log_level=1 enable_serialized_actions=true ten_enable_tests=false ten_enable_libwebsockets=false && \ + tgn build linux arm64 ${{ matrix.build_type }} && \ + tree -I 'gen|obj' out \ + " + + - name: Upload tman + uses: actions/upload-artifact@v4 + with: + name: tman-linux-arm64-${{ matrix.compiler }}-${{ matrix.build_type }} + path: out/linux/arm64/ten_manager/bin/tman + + - name: Upload ten_packages + uses: actions/upload-artifact@v4 + with: + name: ten_packages-linux-arm64-${{ matrix.compiler }}-${{ matrix.build_type }} + path: | + out/linux/x64/ten_packages/system/ten_runtime + out/linux/x64/ten_packages/system/ten_runtime_go + out/linux/x64/ten_packages/system/ten_runtime_python + out/linux/arm64/ten_packages/extension/default_extension_cpp + out/linux/arm64/ten_packages/extension/default_extension_go + out/linux/arm64/ten_packages/extension/default_extension_python + out/linux/arm64/ten_packages/extension/py_init_extension_cpp + + - name: Package assets + if: startsWith(github.ref, 'refs/tags/') + run: | + cd out/linux/arm64 + zip -vr tman-linux-arm64-${{ matrix.compiler }}-${{ matrix.build_type }}.zip ten_manager/bin/tman + zip -vr ten_packages-linux-arm64-${{ matrix.compiler }}-${{ matrix.build_type }}.zip \ + ten_packages/system/ten_runtime \ + ten_packages/system/ten_runtime_go \ + ten_packages/system/ten_runtime_python \ + ten_packages/extension/default_extension_cpp \ + ten_packages/extension/default_extension_go \ + ten_packages/extension/default_extension_python \ + ten_packages/extension/py_init_extension_cpp + + - name: Publish to release assets + uses: softprops/action-gh-release@v2 + if: startsWith(github.ref, 'refs/tags/') + with: + files: | + out/linux/arm64/tman-linux-arm64-${{ matrix.compiler }}-${{ matrix.build_type }}.zip + out/linux/arm64/ten_packages-linux-arm64-${{ matrix.compiler }}-${{ matrix.build_type }}.zip diff --git a/.github/workflows/build_linux_ubuntu1804.yml b/.github/workflows/build_linux_ubuntu1804.yml index ccce842a13..15425379f3 100644 --- a/.github/workflows/build_linux_ubuntu1804.yml +++ b/.github/workflows/build_linux_ubuntu1804.yml @@ -3,18 +3,8 @@ name: Build - linux ubuntu1804 on: release: types: [created] - push: - branches: - - "**" - paths-ignore: - - "tools/**" - - "docs/**" - - ".vscode/**" - - ".devcontainer/**" - - ".github/**" - - "!.github/workflows/build_linux_ubuntu1804.yml" - - "**.md" pull_request: + types: [opened, synchronize, reopened] paths-ignore: - "tools/**" - "docs/**" @@ -43,6 +33,12 @@ jobs: fetch-depth: "0" submodules: "true" + - name: Update version + run: | + git config --global --add safe.directory $(pwd) + python3 tools/version/update_version_in_ten_framework.py + python3 tools/version/check_version_in_ten_framework.py + - name: Install cpulimit and taskset run: | apt-get update @@ -54,7 +50,7 @@ jobs: echo $PATH go env -w GOFLAGS="-buildvcs=false" rustup default nightly - tgn gen linux x64 ${{ matrix.build_type }} -- is_clang=false log_level=1 enable_serialized_actions=true ten_enable_integration_tests=false ten_enable_private_tests=false ten_rust_enable_tests=false ten_enable_package_manager=false + tgn gen linux x64 ${{ matrix.build_type }} -- is_clang=false log_level=1 enable_serialized_actions=true ten_enable_integration_tests=false ten_enable_private_tests=false ten_rust_enable_tests=false ten_enable_package_manager=false ten_rust_enable_gen_cargo_config=false tgn build linux x64 ${{ matrix.build_type }} tree -I 'gen|obj' out @@ -113,8 +109,9 @@ jobs: with: name: ten_packages-linux-ubuntu1804-x64-gcc-${{ matrix.build_type }} path: | - out/linux/x64/ten_packages/system - out/linux/x64/ten_packages/extension_group + out/linux/x64/ten_packages/system/ten_runtime + out/linux/x64/ten_packages/system/ten_runtime_go + out/linux/x64/ten_packages/system/ten_runtime_python out/linux/x64/ten_packages/extension/default_extension_cpp out/linux/x64/ten_packages/extension/default_extension_go out/linux/x64/ten_packages/extension/default_extension_python @@ -125,14 +122,15 @@ jobs: run: | cd out/linux/x64 zip -vr ten_packages-linux-ubuntu1804-x64-gcc-${{ matrix.build_type }}.zip \ - ten_packages/system \ - ten_packages/extension_group \ + ten_packages/system/ten_runtime \ + ten_packages/system/ten_runtime_go \ + ten_packages/system/ten_runtime_python \ ten_packages/extension/default_extension_cpp \ ten_packages/extension/default_extension_go \ ten_packages/extension/default_extension_python \ ten_packages/extension/py_init_extension_cpp - - name: Release assets + - name: Publish to release assets uses: softprops/action-gh-release@v1 if: startsWith(github.ref, 'refs/tags/') with: diff --git a/.github/workflows/build_linux_ubuntu2204.yml b/.github/workflows/build_linux_ubuntu2204.yml index 5b33c743ef..fe585e433a 100644 --- a/.github/workflows/build_linux_ubuntu2204.yml +++ b/.github/workflows/build_linux_ubuntu2204.yml @@ -3,18 +3,8 @@ name: Build - linux ubuntu2204 on: release: types: [created] - push: - branches: - - "**" - paths-ignore: - - "tools/**" - - "docs/**" - - ".vscode/**" - - ".devcontainer/**" - - ".github/**" - - "!.github/workflows/build_linux_ubuntu2204.yml" - - "**.md" pull_request: + types: [opened, synchronize, reopened] paths-ignore: - "tools/**" - "docs/**" @@ -42,6 +32,12 @@ jobs: fetch-depth: "0" submodules: "true" + - name: Update version + run: | + git config --global --add safe.directory $(pwd) + python3 tools/version/update_version_in_ten_framework.py + python3 tools/version/check_version_in_ten_framework.py + - name: Install cpulimit and taskset run: | apt-get update @@ -54,7 +50,7 @@ jobs: go env -w GOFLAGS="-buildvcs=false" go1.20.12 download rustup default nightly - tgn gen linux x64 ${{ matrix.build_type }} -- is_clang=${{ matrix.compiler == 'gcc' && 'false' || 'true' }} log_level=1 enable_serialized_actions=true ten_enable_integration_tests=false ten_enable_private_tests=false ten_rust_enable_tests=false ten_package_manager_enable_tests=false + tgn gen linux x64 ${{ matrix.build_type }} -- is_clang=${{ matrix.compiler == 'gcc' && 'false' || 'true' }} log_level=1 enable_serialized_actions=true ten_enable_integration_tests=false ten_enable_private_tests=false ten_rust_enable_tests=false ten_package_manager_enable_tests=false ten_rust_enable_gen_cargo_config=false tgn build linux x64 ${{ matrix.build_type }} tree -I 'gen|obj' out @@ -119,8 +115,9 @@ jobs: with: name: ten_packages-linux-x64-${{ matrix.compiler }}-${{ matrix.build_type }} path: | - out/linux/x64/ten_packages/system - out/linux/x64/ten_packages/extension_group + out/linux/x64/ten_packages/system/ten_runtime + out/linux/x64/ten_packages/system/ten_runtime_go + out/linux/x64/ten_packages/system/ten_runtime_python out/linux/x64/ten_packages/extension/default_extension_cpp out/linux/x64/ten_packages/extension/default_extension_go out/linux/x64/ten_packages/extension/default_extension_python @@ -132,17 +129,42 @@ jobs: cd out/linux/x64 zip -vr tman-linux-x64-${{ matrix.compiler }}-${{ matrix.build_type }}.zip ten_manager/bin/tman zip -vr ten_packages-linux-x64-${{ matrix.compiler }}-${{ matrix.build_type }}.zip \ - ten_packages/system \ - ten_packages/extension_group \ + ten_packages/system/ten_runtime \ + ten_packages/system/ten_runtime_go \ + ten_packages/system/ten_runtime_python \ ten_packages/extension/default_extension_cpp \ ten_packages/extension/default_extension_go \ ten_packages/extension/default_extension_python \ ten_packages/extension/py_init_extension_cpp - - name: Release assets + - name: Publish to release assets uses: softprops/action-gh-release@v2 if: startsWith(github.ref, 'refs/tags/') with: files: | out/linux/x64/tman-linux-x64-${{ matrix.compiler }}-${{ matrix.build_type }}.zip out/linux/x64/ten_packages-linux-x64-${{ matrix.compiler }}-${{ matrix.build_type }}.zip + + - name: Publish release to TEN cloud store + if: ${{ startsWith(github.ref, 'refs/tags/') && matrix.compiler == 'gcc' && matrix.build_type == 'release' }} + run: | + TMAN_BIN=$(pwd)/out/linux/x64/ten_manager/bin/tman + + cd out/linux/x64/ten_packages + ARRAY=( + "system/ten_runtime" + "system/ten_runtime_go" + "system/ten_runtime_python" + "extension/default_extension_cpp" + "extension/default_extension_go" + "extension/default_extension_python" + "extension/py_init_extension_cpp" + ) + + for item in "${ARRAY[@]}"; do + echo $item + cd $item + ${TMAN_BIN} --verbose --user-token ${{ secrets.TEN_CLOUD_STORE }} publish + cd - + done + shell: bash diff --git a/.github/workflows/build_mac.yml b/.github/workflows/build_mac.yml index 222d3cbc4f..1f327f5bf6 100644 --- a/.github/workflows/build_mac.yml +++ b/.github/workflows/build_mac.yml @@ -1,18 +1,8 @@ name: Build - mac on: - push: - branches: - - "**" - paths-ignore: - - "tools/**" - - "docs/**" - - ".vscode/**" - - ".devcontainer/**" - - ".github/**" - - "!.github/workflows/build_mac.yml" - - "**.md" pull_request: + types: [opened, synchronize, reopened] paths-ignore: - "tools/**" - "docs/**" @@ -86,19 +76,19 @@ jobs: fi export PATH=$(pwd)/core/ten_gn:$PATH echo $PATH - tgn gen mac arm64 ${{ matrix.build_type }} -- log_level=1 enable_serialized_actions=true ten_enable_integration_tests=false ten_enable_private_tests=false ten_rust_enable_tests=false + tgn gen mac arm64 ${{ matrix.build_type }} -- log_level=1 enable_serialized_actions=true ten_enable_integration_tests=false ten_enable_private_tests=false ten_rust_enable_tests=false ten_rust_enable_gen_cargo_config=false tgn build mac arm64 ${{ matrix.build_type }} tree -I 'gen|obj' out - # - name: Run Tests (ten_utils_unit_test) - # env: - # ASAN_OPTIONS: detect_stack_use_after_return=1:color=always:unmap_shadow_on_exit=1:abort_on_error=1 - # MALLOC_CHECK_: 3 - # TEN_ENABLE_MEMORY_TRACKING: "true" - # TEN_ENABLE_BACKTRACE_DUMP: "true" - # run: | - # chmod +x out/mac/arm64/tests/standalone/ten_utils_unit_test - # out/mac/arm64/tests/standalone/ten_utils_unit_test + - name: Run Tests (ten_utils_unit_test) + env: + ASAN_OPTIONS: detect_stack_use_after_return=1:color=always:unmap_shadow_on_exit=1:abort_on_error=1 + MALLOC_CHECK_: 3 + TEN_ENABLE_MEMORY_TRACKING: "true" + TEN_ENABLE_BACKTRACE_DUMP: "true" + run: | + chmod +x out/mac/arm64/tests/standalone/ten_utils_unit_test + out/mac/arm64/tests/standalone/ten_utils_unit_test - name: Run Tests (ten_runtime_unit_test) env: @@ -110,15 +100,15 @@ jobs: chmod +x out/mac/arm64/tests/standalone/ten_runtime_unit_test out/mac/arm64/tests/standalone/ten_runtime_unit_test - # - name: Run Tests (ten_runtime_smoke_test) - # env: - # ASAN_OPTIONS: detect_stack_use_after_return=1:color=always:unmap_shadow_on_exit=1:abort_on_error=1 - # MALLOC_CHECK_: 3 - # TEN_ENABLE_MEMORY_TRACKING: "true" - # TEN_ENABLE_BACKTRACE_DUMP: "true" - # run: | - # chmod +x out/mac/arm64/tests/standalone/ten_runtime_smoke_test - # out/mac/arm64/tests/standalone/ten_runtime_smoke_test + - name: Run Tests (ten_runtime_smoke_test) + env: + ASAN_OPTIONS: detect_stack_use_after_return=1:color=always:unmap_shadow_on_exit=1:abort_on_error=1 + MALLOC_CHECK_: 3 + TEN_ENABLE_MEMORY_TRACKING: "true" + TEN_ENABLE_BACKTRACE_DUMP: "true" + run: | + chmod +x out/mac/arm64/tests/standalone/ten_runtime_smoke_test + out/mac/arm64/tests/standalone/ten_runtime_smoke_test build-mac-x64: concurrency: @@ -183,7 +173,7 @@ jobs: fi export PATH=$(pwd)/core/ten_gn:$PATH echo $PATH - tgn gen mac x64 ${{ matrix.build_type }} -- log_level=1 enable_serialized_actions=true ten_enable_integration_tests=false ten_enable_private_tests=false ten_rust_enable_tests=false ten_package_manager_enable_tests=false + tgn gen mac x64 ${{ matrix.build_type }} -- log_level=1 enable_serialized_actions=true ten_enable_integration_tests=false ten_enable_private_tests=false ten_rust_enable_tests=false ten_package_manager_enable_tests=false ten_rust_enable_gen_cargo_config=false tgn build mac x64 ${{ matrix.build_type }} tree -I 'gen|obj' out diff --git a/.github/workflows/build_win.yml b/.github/workflows/build_win.yml index a214b3abdf..d819d5411b 100644 --- a/.github/workflows/build_win.yml +++ b/.github/workflows/build_win.yml @@ -1,18 +1,8 @@ name: Build - win on: - push: - branches: - - "**" - paths-ignore: - - "tools/**" - - "docs/**" - - ".vscode/**" - - ".devcontainer/**" - - ".github/**" - - "!.github/workflows/build_win.yml" - - "**.md" pull_request: + types: [opened, synchronize, reopened] paths-ignore: - "tools/**" - "docs/**" @@ -59,29 +49,26 @@ jobs: - name: Build run: | $ENV:PATH += ";$PWD/core/ten_gn" - tgn gen win x64 ${{ matrix.build_type }} -- vs_version=2022 log_level=1 enable_serialized_actions=true ten_enable_integration_tests=false ten_enable_private_tests=false ten_rust_enable_tests=false ten_package_manager_enable_tests=false + tgn gen win x64 ${{ matrix.build_type }} -- vs_version=2022 log_level=1 enable_serialized_actions=true ten_enable_integration_tests=false ten_enable_private_tests=false ten_rust_enable_tests=false ten_package_manager_enable_tests=false ten_rust_enable_gen_cargo_config=false tgn build win x64 ${{ matrix.build_type }} - name: Run Tests (ten_utils_unit_test) env: - TEN_ENABLE_MEMORY_TRACKING: "true" TEN_ENABLE_BACKTRACE_DUMP: "true" run: | chmod +x out/win/x64/tests/standalone/ten_utils_unit_test out/win/x64/tests/standalone/ten_utils_unit_test - # - name: Run Tests (ten_runtime_unit_test) - # env: - # TEN_ENABLE_MEMORY_TRACKING: "true" - # TEN_ENABLE_BACKTRACE_DUMP: "true" - # run: | - # chmod +x out/win/x64/tests/standalone/ten_runtime_unit_test - # out/win/x64/tests/standalone/ten_runtime_unit_test + - name: Run Tests (ten_runtime_unit_test) + env: + TEN_ENABLE_BACKTRACE_DUMP: "true" + run: | + chmod +x out/win/x64/tests/standalone/ten_runtime_unit_test + out/win/x64/tests/standalone/ten_runtime_unit_test - # - name: Run Tests (ten_runtime_smoke_test) - # env: - # TEN_ENABLE_MEMORY_TRACKING: "true" - # TEN_ENABLE_BACKTRACE_DUMP: "true" - # run: | - # chmod +x out/win/x64/tests/standalone/ten_runtime_smoke_test - # out/win/x64/tests/standalone/ten_runtime_smoke_test + - name: Run Tests (ten_runtime_smoke_test) + env: + TEN_ENABLE_BACKTRACE_DUMP: "true" + run: | + chmod +x out/win/x64/tests/standalone/ten_runtime_smoke_test + out/win/x64/tests/standalone/ten_runtime_smoke_test diff --git a/.gitignore b/.gitignore index 0139d39864..3d1b49daf2 100644 --- a/.gitignore +++ b/.gitignore @@ -75,6 +75,7 @@ package-lock.json core/src/ten_manager/target/ core/src/ten_rust/src/schema/bindings.rs core/src/ten_rust/target/ +/.cargo # private modules packages/private_apps @@ -82,3 +83,6 @@ packages/private_extensions tests/private third_party/private build/private + +# zlib auto-generated files +third_party/zlib/zconf.h.included diff --git a/.vscode/launch.json b/.vscode/launch.json index a8d74cb9fc..78570e4d1c 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -47,7 +47,7 @@ "request": "launch", "program": "${workspaceFolder}/out/linux/x64/tests/standalone/ten_runtime_unit_test", "args": [ - "--gtest_filter=SchemaTest.SchemaStoreValidateProperty" + "--gtest_filter=TenErrorTest.cpp_thread" ], "stopAtEntry": false, "cwd": "${workspaceFolder}/out/linux/x64/", @@ -59,6 +59,14 @@ { "name": "LD_PRELOAD", "value": "/usr/lib/gcc/x86_64-linux-gnu/10/libasan.so" + }, + { + "name": "TEN_ENABLE_MEMORY_TRACKING", + "value": "true" + }, + { + "name": "TEN_ENABLE_BACKTRACE_DUMP", + "value": "true" } ], "externalConsole": false, @@ -371,9 +379,10 @@ "program": "${workspaceFolder}/out/linux/x64/ten_manager/bin/tman", "cwd": "${workspaceFolder}/out/linux/x64/", "args": [ + "--verbose", "dev-server", - "--base-dir=/home/wei/MyData/Temp/ASTRA.ai/agents", - "--port=49484" + "--base-dir=/home/sunxilin/ten_framework_internal_base/ten_framework/out/linux/x64/tests/ten_runtime/integration/python/two_async_extensions_in_different_groups_python/two_async_extensions_in_different_groups_python_app", + "--port=49483" ], }, { @@ -389,6 +398,17 @@ "hello_world", ], }, + { + "name": "tman check_graph (lldb)", + "type": "lldb", + "request": "launch", + "program": "${workspaceFolder}/core/src/ten_manager/target/x86_64-unknown-linux-gnu/debug/tman", + "cwd": "${workspaceFolder}/core/src/ten_manager/", + "args": [ + "check-graph", + "/home/workspace/pcm-pusher" + ] + }, { "name": "ten_rust (rust)", "type": "lldb", diff --git a/.vscode/settings.json b/.vscode/settings.json index fffd5a4929..fa28d7473d 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -7,8 +7,13 @@ "editor.wordWrapColumn": 80 }, "[python]": { - "editor.tabSize": 4 + "editor.tabSize": 4, + "editor.defaultFormatter": "ms-python.black-formatter" }, + "black-formatter.args": [ + "--line-length", + "80" + ], "debug.allowBreakpointsEverywhere": true, "doxdocgen.c.commentPrefix": "/// ", "doxdocgen.c.firstLine": "///", diff --git a/build/common/rust/rust.gni b/build/common/rust/rust.gni index 9decfdd363..291b23139f 100644 --- a/build/common/rust/rust.gni +++ b/build/common/rust/rust.gni @@ -43,12 +43,31 @@ template("rust_target") { _rustflags = invoker._rustflags } - if (is_debug) { - if (ten_rust_enable_asan) { - if (enable_sanitizer) { - _rustflags = "${_rustflags} -Zsanitizer=address" - } + if (!ten_rust_enable_gen_cargo_config && enable_sanitizer && + ten_rust_enable_asan) { + asan_args = [ + "--action", + "print", + "--compiler", + ] + if (is_clang) { + asan_args += [ "clang" ] + } else { + asan_args += [ "gcc" ] } + + asan_args += [ + "--target-os", + target_os, + "--target-arch", + target_cpu, + ] + + flags = exec_script("//build/common/rust/rust_gen_cargo_config.py", + asan_args, + "trim string") + + _rustflags = "${_rustflags} ${flags}" } if (_rustflags != "") { @@ -209,12 +228,31 @@ template("rust_test") { _rustflags = invoker._rustflags } - if (is_debug) { - if (ten_rust_enable_asan) { - if (enable_sanitizer) { - _rustflags = "${_rustflags} -Zsanitizer=address" - } + if (!ten_rust_enable_gen_cargo_config && enable_sanitizer && + ten_rust_enable_asan) { + asan_args = [ + "--action", + "print", + "--compiler", + ] + if (is_clang) { + asan_args += [ "clang" ] + } else { + asan_args += [ "gcc" ] } + + asan_args += [ + "--target-os", + target_os, + "--target-arch", + target_cpu, + ] + + flags = exec_script("//build/common/rust/rust_gen_cargo_config.py", + asan_args, + "trim string") + + _rustflags = "${_rustflags} ${flags}" } if (_rustflags != "") { @@ -353,3 +391,91 @@ template("rust_cbindgen") { outputs = [ _output ] } } + +template("rust_gen_cargo_config") { + assert(defined(invoker.project_root), "project_root is not defined") + + _project_root = rebase_path(invoker.project_root) + _target_name = target_name + _target_path = target_gen_dir + + action("${_target_name}") { + script = "//build/common/rust/rust_gen_cargo_config.py" + _output = "${_target_path}/${_target_name}_gen_cargo_config" + + args = [ + "--project-root", + _project_root, + "--tg-timestamp-proxy-file", + rebase_path(_output), + ] + + args += [ "--compiler" ] + if (is_clang) { + args += [ "clang" ] + } else { + args += [ "gcc" ] + } + + if (is_win) { + if (target_cpu == "x86") { + target = "i686-pc-windows-msvc" + } else if (target_cpu == "x64") { + target = "x86_64-pc-windows-msvc" + } else if (target_cpu == "arm64") { + target = "aarch64-pc-windows-msvc" + } + } else if (is_linux) { + if (target_cpu == "arm64") { + target = "aarch64-unknown-linux-gnu" + } else if (target_cpu == "arm") { + target = "armv7-unknown-linux-gnueabi" + } else if (target_cpu == "x86") { + target = "i686-unknown-linux-gnu" + } else if (target_cpu == "x64") { + target = "x86_64-unknown-linux-gnu" + } + } else if (is_mac) { + if (target_cpu == "arm64") { + target = "aarch64-apple-darwin" + } else if (target_cpu == "x86") { + target = "i686-apple-darwin" + } else if (target_cpu == "x64") { + target = "x86_64-apple-darwin" + } + } + + args += [ + "--target", + target, + "--target-os", + target_os, + "--target-arch", + target_cpu, + ] + + if (enable_sanitizer && ten_rust_enable_gen_cargo_config) { + args += [ + "--action", + "gen", + ] + } else { + # Delete `.cargo/config.toml` file to remove all configurations related to + # Rust + ASan. + args += [ + "--action", + "delete", + ] + } + + forward_variables_from(invoker, + [ + "deps", + "public_deps", + "data_deps", + "public_configs", + ]) + + outputs = [ _output ] + } +} diff --git a/build/common/rust/rust_gen_cargo_config.py b/build/common/rust/rust_gen_cargo_config.py new file mode 100644 index 0000000000..11d8fa7fb9 --- /dev/null +++ b/build/common/rust/rust_gen_cargo_config.py @@ -0,0 +1,211 @@ +# +# Copyright © 2024 Agora +# This file is part of TEN Framework, an open source project. +# Licensed under the Apache License, Version 2.0, with certain conditions. +# Refer to the "LICENSE" file in the root directory for more information. +# +import argparse +import json +import shutil +import sys +import os +from build.scripts import timestamp_proxy + +sys.path.append(os.path.join(os.path.dirname(__file__), "..")) +from scripts import package_asan_lib + + +# The content of the auto generated .cargo/config.toml file is as follows. +# +# - For gcc compiler: +# +# ```toml +# [target.x86_64-unknown-linux-gnu] +# rustflags = ["-C", "linker=gcc", "-Z", "external-clangrt", "-Z", "sanitizer=address", "-l", "asan"] +# +# [build] +# target = "x86_64-unknown-linux-gnu" +# ``` +# +# - For clang compiler: +# +# ```toml +# [target.x86_64-unknown-linux-gnu] +# rustflags = ["-C", "linker=clang", "-Z", "external-clangrt", "-Z", "sanitizer=address", "-C", "link-args=-fsanitize=address"] +# +# [build] +# target = "x86_64-unknown-linux-gnu" +# ``` + +GCC_ASAN_FLAGS = [ + "-C", + "linker=gcc", + "-Z", + "external-clangrt", + "-Z", + "sanitizer=address", + "-l", + "asan", +] + + +# It is better to use lld as the linker in clang. In Rust, you can specify it +# using ["-C", "link-arg=-fuse-ld=lld"]. However, lld needs to be in the PATH. +# On some CI machines, lld is not in the PATH, so we won't use lld here. +CLANG_ASAN_FLAGS = [ + "-C", + "linker=clang", + "-Z", + "external-clangrt", + "-Z", + "sanitizer=address", + "-C", + "link-args=-fsanitize=address", +] + +ASAN_CONFIG = """[target.{build_target}] +rustflags = {asan_flags} + +[build] +target = "{build_target}" +""" + + +class ArgumentInfo(argparse.Namespace): + def __init__(self): + super().__init__() + self.project_root: str + self.build_type: str + self.compiler: str + self.target: str + self.target_os: str + self.target_arch: str + self.tg_timestamp_proxy_file: str | None = None + self.enable_asan: bool + self.action: str + + +# On macOS, only the Clang compiler is available, and Clang's ASan runtime +# is provided only as a dynamic library. However, the linker flag +# '-shared-libasan' is not supported in Cargo with Clang on macOS. Cargo will +# issue a warning and ignore the flag, e.g., 'clang: warning: argument unused +# during compilation: '-shared-libasan''. This could be a bug in Cargo. To +# resolve this, use the following flag instead. +def special_link_args_on_mac(arch: str) -> str: + asan_lib = package_asan_lib.detect_mac_asan_lib(arch) + return f"link-arg=-Wl,{asan_lib}" + + +def gen_cargo_config(args: ArgumentInfo): + if not os.path.exists(args.project_root): + raise Exception(f"Project root {args.project_root} does not exist.") + + # Create .cargo/ folder if not exist. + cargo_dir = os.path.join(args.project_root, ".cargo") + if not os.path.exists(cargo_dir): + os.mkdir(cargo_dir) + + # Check if .cargo/config.toml exists, and remove it if it does. + cargo_config = os.path.join(cargo_dir, "config.toml") + if os.path.exists(cargo_config): + os.remove(cargo_config) + + flags = [] + if args.compiler == "gcc": + flags = GCC_ASAN_FLAGS + else: + flags = CLANG_ASAN_FLAGS + + if args.target_os == "mac": + flags.extend(["-C", special_link_args_on_mac(args.target_arch)]) + + config_content = ASAN_CONFIG.format( + build_target=args.target, asan_flags=json.dumps(flags) + ) + + with open(cargo_config, "w") as f: + f.write(config_content) + + +def delete_cargo_config(root: str): + cargo_dir = os.path.join(root, ".cargo") + if os.path.exists(cargo_dir): + shutil.rmtree(cargo_dir) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + + parser.add_argument( + "--action", type=str, required=True, help="gen|delete|print" + ) + parser.add_argument("--project-root", type=str, required=False) + parser.add_argument("--compiler", type=str, required=True) + parser.add_argument("--target", type=str, required=False) + parser.add_argument("--target-os", type=str, required=True) + parser.add_argument("--target-arch", type=str, required=True) + parser.add_argument( + "--tg-timestamp-proxy-file", type=str, default="", required=False + ) + parser.add_argument( + "--enable-asan", action=argparse.BooleanOptionalAction, default=True + ) + + arg_info = ArgumentInfo() + args = parser.parse_args(namespace=arg_info) + + returncode = 0 + if args.action == "gen": + try: + gen_cargo_config(args) + + # Success to gen cargo config, update the stamp file to represent + # this fact. + timestamp_proxy.touch_timestamp_proxy_file( + args.tg_timestamp_proxy_file + ) + except Exception as exc: + returncode = 1 + timestamp_proxy.remove_timestamp_proxy_file( + args.tg_timestamp_proxy_file + ) + print(exc) + + finally: + sys.exit(-1 if returncode != 0 else 0) + + elif args.action == "delete": + try: + delete_cargo_config(args.project_root) + + # Success to delete cargo config, update the stamp file to represent + # this fact. + timestamp_proxy.touch_timestamp_proxy_file( + args.tg_timestamp_proxy_file + ) + except Exception as exc: + returncode = 1 + timestamp_proxy.remove_timestamp_proxy_file( + args.tg_timestamp_proxy_file + ) + print(exc) + + finally: + sys.exit(-1 if returncode != 0 else 0) + + else: + # action = print + # + # Constructs and prints a space-separated string of the necessary ASan + # flags for Clang, taking into account any macOS-specific handling. + flags = [ + CLANG_ASAN_FLAGS[i] + CLANG_ASAN_FLAGS[i + 1] + for i in range(0, len(CLANG_ASAN_FLAGS) - 1, 2) + ] + + if args.target_os == "mac": + asan_flag = special_link_args_on_mac(args.target_arch) + flags.append(f"-C{asan_flag}") + + print(" ".join(flags)) + sys.exit(0) diff --git a/build/common/scripts/package_asan_lib.py b/build/common/scripts/package_asan_lib.py index a4ee39bdcd..aba618a169 100644 --- a/build/common/scripts/package_asan_lib.py +++ b/build/common/scripts/package_asan_lib.py @@ -52,7 +52,7 @@ def detect_mac_asan_lib(arch: str) -> str: def detect_linux_asan_lib(arch: str) -> str: - if arch == "x64": + if arch in ["x64", "arm64"]: out, _ = subprocess.Popen( "gcc -print-file-name=libasan.so", shell=True, @@ -79,13 +79,21 @@ def detect_linux_asan_lib(arch: str) -> str: # Generally speaking, this function should not need to be called because Clang's # default ASan mechanism is static linking. -def detect_linux_clang_asan_lib(_arch: str) -> str: - out, _ = subprocess.Popen( - "clang -print-file-name=libclang_rt.asan-x86_64.so", - shell=True, - stdout=subprocess.PIPE, - encoding="utf-8", - ).communicate() +def detect_linux_clang_asan_lib(arch: str) -> str: + if arch == "x64": + out, _ = subprocess.Popen( + "clang -print-file-name=libclang_rt.asan-x86_64.so", + shell=True, + stdout=subprocess.PIPE, + encoding="utf-8", + ).communicate() + elif arch == "arm64": + out, _ = subprocess.Popen( + "clang -print-file-name=libclang_rt.asan-aarch64.so", + shell=True, + stdout=subprocess.PIPE, + encoding="utf-8", + ).communicate() libasan_path = out.strip() diff --git a/build/ten_manager/options.gni b/build/ten_manager/options.gni index 539ac3eb57..a92482b9aa 100644 --- a/build/ten_manager/options.gni +++ b/build/ten_manager/options.gni @@ -9,6 +9,6 @@ declare_args() { } declare_args() { - ten_package_manager_enable_tests = - is_linux || is_win || (is_mac && target_cpu == "x64") + ten_package_manager_enable_tests = (is_linux && target_cpu == "x64") || + is_win || (is_mac && target_cpu == "x64") } diff --git a/build/ten_runtime/feature/autotool.py b/build/ten_runtime/feature/autotool.py index 58f3bd2887..db2ab65325 100644 --- a/build/ten_runtime/feature/autotool.py +++ b/build/ten_runtime/feature/autotool.py @@ -149,9 +149,15 @@ def copy_system_deps(self): if sys.platform == "linux": libs.extend( - glob.glob(f"/usr/lib/x86_64-linux-gnu/lib{dep}.so.*") + glob.glob( + f"/usr/lib/{os.uname().machine}-linux-gnu/lib{dep}.so.*" + ) + ) + libs.extend( + glob.glob( + f"/usr/lib/{os.uname().machine}-linux-gnu/lib{dep}.so" + ) ) - libs.extend(glob.glob(f"/usr/lib/x86_64-linux-gnu/lib{dep}.so")) else: print("TODO: Add support for other platforms.") sys.exit(1) diff --git a/build/ten_runtime/feature/cmake.py b/build/ten_runtime/feature/cmake.py index 283102e558..e4f08ac48e 100644 --- a/build/ten_runtime/feature/cmake.py +++ b/build/ten_runtime/feature/cmake.py @@ -192,13 +192,16 @@ def _fill_attributes(self): self.sharedlinkerflags.append("--target=arm-linux-gnueabihf") self.exelinkerflags.append("--target=arm-linux-gnueabihf") elif self.args.target_cpu == "arm64": - self.cflags.append("--target=aarch64-linux-gnu") - self.sharedlinkerflags.append("--target=aarch64-linux-gnu") - self.exelinkerflags.append("--target=aarch64-linux-gnu") - if self.args.use_clang: - self.cflags.append("-fuse-ld=lld") - self.sharedlinkerflags.append("-fuse-ld=lld") - self.exelinkerflags.append("-fuse-ld=lld") + if os.uname().machine in ["arm64", "aarch64"]: + pass + else: + self.cflags.append("--target=aarch64-linux-gnu") + self.sharedlinkerflags.append("--target=aarch64-linux-gnu") + self.exelinkerflags.append("--target=aarch64-linux-gnu") + if self.args.use_clang: + self.cflags.append("-fuse-ld=lld") + self.sharedlinkerflags.append("-fuse-ld=lld") + self.exelinkerflags.append("-fuse-ld=lld") else: raise Exception( "Currently can not build Linux target with CPU arch" diff --git a/build/ten_runtime/feature/install_all.py b/build/ten_runtime/feature/install_all.py index 434cee0311..226a668bf7 100644 --- a/build/ten_runtime/feature/install_all.py +++ b/build/ten_runtime/feature/install_all.py @@ -18,6 +18,7 @@ def __init__(self): self.build_type: str self.config_file: str self.log_level: int + self.assume_yes: bool if __name__ == "__main__": @@ -56,6 +57,7 @@ def __init__(self): parser.add_argument( "--log-level", type=int, required=True, help="specify log level" ) + parser.add_argument("--assume-yes", type=bool, default=False) arg_info = ArgumentInfo() args = parser.parse_args(namespace=arg_info) @@ -74,6 +76,9 @@ def __init__(self): if args.config_file is not None: cmd += ["--config-file=" + args.config_file] + if args.assume_yes: + cmd += ["--yes"] + cmd += ["install"] if args.build_type is not None: diff --git a/build/ten_runtime/feature/install_pkg.py b/build/ten_runtime/feature/install_pkg.py index 61a70d6c78..db59b724d1 100644 --- a/build/ten_runtime/feature/install_pkg.py +++ b/build/ten_runtime/feature/install_pkg.py @@ -23,6 +23,19 @@ def __init__(self): self.depfile_target: str self.log_level: int self.pkg_type: str + self.local_registry_path: str + + +def get_pkgs_from_local_registry( + local_registry_path: str, pkg_type: str, pkg_name: str +) -> list[str]: + # Find directories under // and + # return their names. + pkg_dir = os.path.join(local_registry_path, pkg_type, pkg_name) + if not os.path.exists(pkg_dir): + return [] + + return os.listdir(pkg_dir) def process_possible_published_results(file_paths: list[str]) -> list[str]: @@ -90,6 +103,12 @@ def process_possible_published_results(file_paths: list[str]) -> list[str]: parser.add_argument( "--log-level", type=int, required=True, help="specify log level" ) + parser.add_argument( + "--local-registry-path", + type=str, + required=False, + help="Path to local registry", + ) arg_info = ArgumentInfo() args = parser.parse_args(namespace=arg_info) @@ -125,6 +144,17 @@ def process_possible_published_results(file_paths: list[str]) -> list[str]: if args.config_file is not None: list.append(cmd, "--config-file=" + args.config_file) + if args.local_registry_path is not None: + versions = get_pkgs_from_local_registry( + args.local_registry_path, args.pkg_type, args.src_pkg + ) + + if len(versions) == 1: + # If there is only one version, install using that specific + # version number to avoid installation failures due to it being + # a pre-release version. + args.src_pkg += f"@{versions[0]}" + cmd += [ "install", args.pkg_type, diff --git a/build/ten_runtime/feature/publish.gni b/build/ten_runtime/feature/publish.gni index 845f3c74a1..41502d8261 100644 --- a/build/ten_runtime/feature/publish.gni +++ b/build/ten_runtime/feature/publish.gni @@ -41,6 +41,13 @@ template("ten_package_publish") { args += [ "--no-enable-publish" ] } + args += [ + "--os", + target_os, + "--cpu", + target_cpu, + ] + args += [ "--log-level", "${log_level}", diff --git a/build/ten_runtime/feature/publish.py b/build/ten_runtime/feature/publish.py index 0675a15dc5..89ca096399 100644 --- a/build/ten_runtime/feature/publish.py +++ b/build/ten_runtime/feature/publish.py @@ -10,6 +10,7 @@ import sys from build.scripts import cmd_exec, touch from common.scripts import delete_files +import subprocess class ArgumentInfo(argparse.Namespace): @@ -20,6 +21,8 @@ def __init__(self): self.config_file: str self.log_level: int self.enable_publish: bool + self.os: str + self.cpu: str def extract_publish_path(text: str) -> str | None: @@ -35,8 +38,49 @@ def write_published_results_to_file( file.write(published_results) +def update_manifest( + base_dir: str, os_str: str, cpu_str: str, log_level: int +) -> None: + manifest_path = os.path.join(base_dir, "manifest.json") + + os_arch_pair = f"{os_str}:{cpu_str}" + + script_dir = os.path.dirname(os.path.abspath(__file__)) + update_script_path = os.path.abspath( + os.path.join( + script_dir, + "../../../tools/supports/update_supports_in_manifest_json.py", + ) + ) + + try: + subprocess.run( + [ + "python", + update_script_path, + "--input-file", + manifest_path, + "--output-file", + manifest_path, + "--os-arch-pairs", + os_arch_pair, + "--log-level", + str(log_level), + ], + check=True, + ) + if log_level > 0: + print( + f"Manifest updated successfully with os/cpu: {os_str}/{cpu_str}" + ) + except subprocess.CalledProcessError as e: + print(f"Failed to update manifest.json: {e}") + sys.exit(1) + + if __name__ == "__main__": parser = argparse.ArgumentParser() + parser.add_argument( "--tman-path", type=str, @@ -62,6 +106,8 @@ def write_published_results_to_file( action=argparse.BooleanOptionalAction, default=True, ) + parser.add_argument("--os", type=str, required=True) + parser.add_argument("--cpu", type=str, required=True) arg_info = ArgumentInfo() args = parser.parse_args(namespace=arg_info) @@ -69,6 +115,12 @@ def write_published_results_to_file( if args.enable_publish is False: sys.exit(0) + # The action of adding the `supports` field is currently handled within the + # GitHub CI flow. In the future, if there is a need to perform this in the + # GN flow, we can consider reopening this process. + # + # update_manifest(args.base_dir, args.os, args.cpu, args.log_level) + # Use 'tman publish' to perform the uploading. origin_wd = os.getcwd() diff --git a/build/ten_runtime/feature/test.gni b/build/ten_runtime/feature/test.gni index 0bc0797ca3..ab3e998c13 100644 --- a/build/ten_runtime/feature/test.gni +++ b/build/ten_runtime/feature/test.gni @@ -69,6 +69,11 @@ template("ten_package_test_prepare_app") { "${log_level}", ] + args += [ + "--local-registry-path", + rebase_path("${root_out_dir}/tests/local_registry"), + ] + if (invoker.deps != []) { foreach(dep, invoker.deps) { args += [ @@ -192,6 +197,11 @@ template("ten_package_test_prepare_app") { args += [ "release" ] } + args += [ + "--assume-yes", + "True", + ] + args += [ "--config-file", rebase_path("${root_out_dir}/tests/local_registry/config.json"), @@ -574,6 +584,11 @@ template("ten_package_standalone_pkg") { "${log_level}", ] + args += [ + "--local-registry-path", + rebase_path("${root_out_dir}/tests/local_registry"), + ] + if (invoker.deps != []) { foreach(dep, invoker.deps) { args += [ @@ -639,6 +654,11 @@ template("ten_package_standalone_pkg") { args += [ "release" ] } + args += [ + "--assume-yes", + "True", + ] + args += [ "--config-file", rebase_path("${root_out_dir}/tests/local_registry/config.json"), diff --git a/build/ten_runtime/options.gni b/build/ten_runtime/options.gni index d687200b3f..67c40b9510 100644 --- a/build/ten_runtime/options.gni +++ b/build/ten_runtime/options.gni @@ -24,7 +24,8 @@ declare_args() { ten_enable_go_binding = is_mac || is_linux # TODO: enable it on mac and win - ten_enable_python_binding = is_linux && target_cpu == "x64" + ten_enable_python_binding = + is_linux && (target_cpu == "x64" || target_cpu == "arm64") } # ten_runtime extensions diff --git a/build/ten_rust/options.gni b/build/ten_rust/options.gni index b84655953a..e271b1140a 100644 --- a/build/ten_rust/options.gni +++ b/build/ten_rust/options.gni @@ -11,8 +11,89 @@ declare_args() { # rustc finds the asan runtime library in ~/.rustup/toolchains// # lib/rustlib//lib/librustc-nightly_rt.asan.a ten_rust_enable_asan = is_mac || (is_linux && target_cpu == "x64") + + # The rust projects (i.e., ten_rust and ten_manager) depend on the C static + # library (ex: ten_utils), thus the rust projects must be compiled with asan + # enabled if the C library is compiled with asan. There are two ways to + # enable asan in rust: + # + # 1. Using the RUSTFLAGS environment variable. Ex: + # RUSTFLAGS="-Zsanitizer=address" cargo build. + # 2. Using the cargo config file, i.e., create a `config.toml` file in the + # `.cargo` folder in the source tree of the rust project. Ex, we can + # create a `config.toml` file in `ten_framework/.cargo` folder, and add + # the following content: + # + # [target.x86_64-unknown-linux-gnu] + # rustflags = ["-Z", "sanitizer=address"] + # + # [build] + # target = "x86_64-unknown-linux-gnu" + # + # The second way has a limitation: the cargo config file only effects on the + # builds in the subfolder of the location of the config file. Ex: we add the + # cargo config in `ten_framework` as above. If we run `cargo build` in the + # source folder of ten_manager or ten_rust, the cargo config effects. As + # cargo toolchain supports building rust projects outside of the source + # crate, using the `--manifest-path` option. In other words, we can build + # ten_rust in any folder. The cargo config in the `ten_framework` folder will + # not effect if we build ten_rust in some folder outside of ten_framework + # such as a 'out' folder which is a sibling node of ten_framework. In summary, + # the cargo config effect as follows. + # + # - + # - ten_framework <= works under this folder + # - .cargo + # - out <= not works + # - <= not works + # + # Therefore, the first way is more flexible. That is the default way we use + # in TGN toolchain. + # + # However, the first way can not work well in some cases, ex: develop rust + # in an IDE such as VSCode. In VSCode, when we open a rust file including a + # `main` function or a `test` function, there will be some trigger to run the + # functions, ex: a `Run|Debug` button on the main function, or a `Run Test| + # Debug` button on the test function. The trigger is a task in VSCode, we can + # not customize the task to add the RUSTFLAGS environment variable + # automatically. In addition, if we want to manually run cargo commands in the + # terminal to compile the source code or run some tests during development, + # it is not convenient to add the RUSTFLAGS environment variable every time. + # And the content of the RUSTFLAGS environment variable is different between + # different compilers and platforms. + # + # So we add a new option here, to control whether to auto generate the cargo + # config file under ten_framework if asan is enabled by `enable_sanitizer`. + # The combination of the `enable_sanitizer` and `ten_rust_enable_gen_cargo_config` + # is as follows. + # + # - enable_sanitizer = false + # - ten_rust_enable_gen_cargo_config = true or false + # + # => The asan is disabled, and the cargo config file is not generated. + # + # - enable_sanitizer = true + # - ten_rust_enable_gen_cargo_config = false + # + # => The asan is enabled, which means the C library is compiled with asan. + # => The cargo config is not generated. The rust projects will be compiled + # with asan by setting the RUSTFLAGS environment variable if using tgn. + # => Running the trigger button in the VSCode or manually running `cargo + # build` in the terminal will be FAILED. + # + # - ten_rust_enable_gen_cargo_config = true + # + # => The asan is enabled. + # => The cargo config file is generated in ten_framework. + # => Running the trigger button in the VSCode or manully running `cargo + # build` in the source folder of the rust projects will be success. + # + # Setting this option to true by default to make the development more + # convenient, then developers do not need to care about this option. + ten_rust_enable_gen_cargo_config = true } declare_args() { - ten_rust_enable_tests = is_linux || is_win || (is_mac && target_cpu == "x64") + ten_rust_enable_tests = (is_linux && target_cpu == "x64") || is_win || + (is_mac && target_cpu == "x64") } diff --git a/core/include_internal/ten_runtime/binding/python/ten_env/ten_env.h b/core/include_internal/ten_runtime/binding/python/ten_env/ten_env.h index cdd0c7acea..8e45716db4 100644 --- a/core/include_internal/ten_runtime/binding/python/ten_env/ten_env.h +++ b/core/include_internal/ten_runtime/binding/python/ten_env/ten_env.h @@ -26,6 +26,7 @@ typedef struct ten_py_ten_env_t { // Mark whether the gil state need to be released after 'on_deinit_done'. bool need_to_release_gil_state; + PyThreadState* py_thread_state; } ten_py_ten_env_t; TEN_RUNTIME_PRIVATE_API bool ten_py_ten_env_check_integrity( diff --git a/core/src/ten_manager/BUILD.gn b/core/src/ten_manager/BUILD.gn index bf898b3ee3..bfc344a053 100644 --- a/core/src/ten_manager/BUILD.gn +++ b/core/src/ten_manager/BUILD.gn @@ -13,10 +13,16 @@ declare_args() { use_shared_lib = false } +copy("ten_manager_test_data") { + sources = [ "//core/src/ten_manager/tests/test_data" ] + outputs = [ "${root_out_dir}/tests/standalone/ten_manager/tests/test_data" ] +} + if (ten_enable_package_manager) { if (ten_package_manager_enable_tests) { rust_test("tman_test") { project_path = "//core/src/ten_manager" + integration_test_output_name = "integration_test" clingo_lib_folder = rebase_path("${root_out_dir}/gen/cmake/clingo/install/lib") @@ -37,6 +43,7 @@ if (ten_enable_package_manager) { test_output_dir = "${root_out_dir}/tests/standalone/ten_manager" deps = [ + ":ten_manager_test_data", "//core/src/ten_rust:ten_rust_static_lib", "//third_party/clingo", ] diff --git a/core/src/ten_manager/src/cmd/cmd_check/cmd_check_graph.rs b/core/src/ten_manager/src/cmd/cmd_check/cmd_check_graph.rs new file mode 100644 index 0000000000..29020df052 --- /dev/null +++ b/core/src/ten_manager/src/cmd/cmd_check/cmd_check_graph.rs @@ -0,0 +1,217 @@ +// +// Copyright © 2024 Agora +// This file is part of TEN Framework, an open source project. +// Licensed under the Apache License, Version 2.0, with certain conditions. +// Refer to the "LICENSE" file in the root directory for more information. +// + +use std::{collections::HashMap, fs, path}; + +use anyhow::{Context, Result}; +use clap::{Arg, ArgMatches, Command}; +use console::Emoji; +use ten_rust::pkg_info::{ + default_app_loc, get_all_existed_pkgs_info_of_app, graph::Graph, + property::parse_property_in_folder, PkgInfo, +}; + +use crate::config::TmanConfig; + +#[derive(Debug)] +pub struct CheckGraphCommand { + pub app: Vec, + pub predefined_graph_name: Option, + pub graph: Option, +} + +pub fn create_sub_cmd(_args_cfg: &crate::cmd_line::ArgsCfg) -> Command { + Command::new("graph") + .about( + "Check the predefined graph or start_graph cmd for the primary \ + app. For more detailed usage, run 'graph -h'", + ) + .arg( + Arg::new("APP") + .long("app") + .help( + "The absolute path of the app declared in the graph. By \ + default, the predefined graph will be read from the first \ + one in the list. ", + ) + .required(true), + ) + .arg( + Arg::new("PREDEFINED_GRAPH_NAME") + .long("predefined-graph-name") + .help( + "Specify the predefined graph name only to be checked, \ + otherwise, all graphs will be checked.", + ) + .required(false) + .conflicts_with("GRAPH"), + ) + .arg( + Arg::new("GRAPH") + .long("graph") + .help( + "Specify the json string of a 'start_graph' cmd to be \ + checked. If not specified, the predefined graph in the \ + primary app will be checked.", + ) + .required(false) + .conflicts_with("PREDEFINED_GRAPH_NAME"), + ) +} + +pub fn parse_sub_cmd( + _sub_cmd_args: &ArgMatches, +) -> crate::cmd::cmd_check::cmd_check_graph::CheckGraphCommand { + let cmd = CheckGraphCommand { + app: _sub_cmd_args + .get_many::("APP") + .unwrap_or_default() + .map(|s| s.to_string()) + .collect(), + predefined_graph_name: _sub_cmd_args + .get_one::("PREDEFINED_GRAPH_NAME") + .cloned(), + graph: _sub_cmd_args.get_one::("GRAPH").cloned(), + }; + + cmd +} + +fn validate_cmd_args(command: &CheckGraphCommand) -> Result<()> { + for app in &command.app { + let stat = fs::metadata(app).with_context(|| { + format!("Failed to get metadata of app path [{}].", app) + })?; + if !stat.is_dir() { + return Err(anyhow::anyhow!( + "App path [{}] is not a directory.", + app + )); + } + } + + Ok(()) +} + +fn get_all_pkg_infos( + command: &CheckGraphCommand, +) -> Result>> { + let mut pkgs_info: HashMap> = HashMap::new(); + + let single_app = command.app.len() == 1; + + for app in &command.app { + let app_path = path::Path::new(app); + + // TODO(Liu): Add a limitation in the schema to ensure that the 'uri' in + // the property.json is not 'localhost'. + let app_property = parse_property_in_folder(app_path)?; + let app_pkgs = get_all_existed_pkgs_info_of_app(app_path)?; + + let app_uri = app_property.get_app_uri(); + if !single_app && app_uri.as_str() == default_app_loc() { + return Err(anyhow::anyhow!( + "The app uri should be some string other than 'localhost' when + using in multi-apps graph." + )); + } + + let present_pkg = pkgs_info.insert(app_uri.clone(), app_pkgs); + if present_pkg.is_some() { + return Err(anyhow::anyhow!( + "All apps should have a unique uri, but uri [{}] is duplicated.", + app_uri + )); + } + } + + Ok(pkgs_info) +} + +fn get_graphs_to_be_checked(command: &CheckGraphCommand) -> Result> { + let mut graphs_to_be_checked: Vec = Vec::new(); + + if let Some(graph_str) = &command.graph { + let graph: Graph = serde_json::from_str(graph_str) + .with_context(|| "The graph json string is invalid")?; + graphs_to_be_checked.push(graph); + } else { + let app_path = path::Path::new(&command.app[0]); + let app_property = parse_property_in_folder(app_path)?; + let predefined_graphs = app_property + ._ten + .and_then(|p| p.predefined_graphs) + .ok_or_else(|| { + anyhow::anyhow!( + "No predefined graph is found in the primary app." + ) + })?; + + if let Some(predefined_graph_name) = &command.predefined_graph_name { + let predefined_graph = predefined_graphs + .iter() + .find(|g| g.name == predefined_graph_name.as_str()) + .ok_or_else(|| { + anyhow::anyhow!( + "Predefined graph [{}] is not found.", + predefined_graph_name + ) + })?; + graphs_to_be_checked.push(predefined_graph.graph.clone()); + } else { + for predefined_graph in predefined_graphs { + graphs_to_be_checked.push(predefined_graph.graph.clone()); + } + } + } + + Ok(graphs_to_be_checked) +} + +fn display_error(e: &anyhow::Error) { + e.to_string().lines().for_each(|l| { + println!(" {}", l); + }); +} + +pub async fn execute_cmd( + _tman_config: &TmanConfig, + command_data: CheckGraphCommand, +) -> Result<()> { + validate_cmd_args(&command_data)?; + + let all_pkgs = get_all_pkg_infos(&command_data)?; + let graphs = get_graphs_to_be_checked(&command_data)?; + + let mut err_count = 0; + + for (graph_idx, graph) in graphs.iter().enumerate() { + print!("Checking graph[{}]... ", graph_idx); + + match graph.check(&all_pkgs) { + Ok(_) => println!("{}", Emoji("✅", "Passed")), + Err(e) => { + err_count += 1; + println!("{}. Details:", Emoji("❌", "Failed")); + display_error(&e); + println!(); + } + } + } + + println!("All is done."); + + if err_count > 0 { + Err(anyhow::anyhow!( + "{}/{} graphs failed.", + err_count, + graphs.len() + )) + } else { + Ok(()) + } +} diff --git a/core/src/ten_manager/src/cmd/cmd_check/mod.rs b/core/src/ten_manager/src/cmd/cmd_check/mod.rs new file mode 100644 index 0000000000..0aec165820 --- /dev/null +++ b/core/src/ten_manager/src/cmd/cmd_check/mod.rs @@ -0,0 +1,54 @@ +// +// Copyright © 2024 Agora +// This file is part of TEN Framework, an open source project. +// Licensed under the Apache License, Version 2.0, with certain conditions. +// Refer to the "LICENSE" file in the root directory for more information. +// +pub mod cmd_check_graph; + +use anyhow::Result; +use clap::{ArgMatches, Command}; + +use crate::config::TmanConfig; + +#[derive(Debug)] +pub enum CheckCommandData { + CheckGraph(crate::cmd::cmd_check::cmd_check_graph::CheckGraphCommand), +} + +pub fn create_sub_cmd(args_cfg: &crate::cmd_line::ArgsCfg) -> Command { + Command::new("check") + .about("Check cmd group. For more detailed usage, run 'check -h'") + .subcommand_required(true) + .arg_required_else_help(true) + .subcommand(crate::cmd::cmd_check::cmd_check_graph::create_sub_cmd( + args_cfg, + )) +} + +pub fn parse_sub_cmd(sub_cmd_args: &ArgMatches) -> CheckCommandData { + match sub_cmd_args.subcommand() { + Some(("graph", graph_cmd_args)) => CheckCommandData::CheckGraph( + crate::cmd::cmd_check::cmd_check_graph::parse_sub_cmd( + graph_cmd_args, + ), + ), + + _ => unreachable!("Command not found"), + } +} + +pub async fn execute_cmd( + tman_config: &TmanConfig, + command_data: CheckCommandData, +) -> Result<()> { + match command_data { + CheckCommandData::CheckGraph(cmd) => { + crate::cmd::cmd_check::cmd_check_graph::execute_cmd( + tman_config, + cmd, + ) + .await + } + } +} diff --git a/core/src/ten_manager/src/cmd/cmd_install.rs b/core/src/ten_manager/src/cmd/cmd_install.rs index e840b7372b..edb56ed350 100644 --- a/core/src/ten_manager/src/cmd/cmd_install.rs +++ b/core/src/ten_manager/src/cmd/cmd_install.rs @@ -768,7 +768,7 @@ pub async fn execute_cmd( &all_existing_local_pkgs, ); - if has_conflict { + if has_conflict && !tman_config.assume_yes { // "y" for continuing to install, "n" for stopping. let ans = Confirm::new( "Warning!!! Some local packages will be overwritten, \ diff --git a/core/src/ten_manager/src/cmd/mod.rs b/core/src/ten_manager/src/cmd/mod.rs index 59616039a2..cfa905b388 100644 --- a/core/src/ten_manager/src/cmd/mod.rs +++ b/core/src/ten_manager/src/cmd/mod.rs @@ -4,6 +4,7 @@ // Licensed under the Apache License, Version 2.0, with certain conditions. // Refer to the "LICENSE" file in the root directory for more information. // +pub mod cmd_check; pub mod cmd_delete; pub mod cmd_dev_server; pub mod cmd_install; @@ -22,6 +23,7 @@ pub enum CommandData { Publish(self::cmd_publish::PublishCommand), Delete(self::cmd_delete::DeleteCommand), DevServer(self::cmd_dev_server::DevServerCommand), + Check(self::cmd_check::CheckCommandData), } pub async fn execute_cmd( @@ -47,5 +49,8 @@ pub async fn execute_cmd( CommandData::DevServer(cmd) => { crate::cmd::cmd_dev_server::execute_cmd(tman_config, cmd).await } + CommandData::Check(cmd) => { + crate::cmd::cmd_check::execute_cmd(tman_config, cmd).await + } } } diff --git a/core/src/ten_manager/src/cmd_line.rs b/core/src/ten_manager/src/cmd_line.rs index 3e8da8d3f1..e38d3e38a6 100644 --- a/core/src/ten_manager/src/cmd_line.rs +++ b/core/src/ten_manager/src/cmd_line.rs @@ -88,12 +88,20 @@ fn create_cmd() -> clap::ArgMatches { .help("Enable verbose output") .action(clap::ArgAction::SetTrue), ) + .arg( + Arg::new("ASSUME_YES") + .short('y') + .long("yes") + .help("Automatically answer 'yes' to all prompts") + .action(clap::ArgAction::SetTrue), + ) .subcommand(crate::cmd::cmd_install::create_sub_cmd(&args_cfg)) .subcommand(crate::cmd::cmd_uninstall::create_sub_cmd(&args_cfg)) .subcommand(crate::cmd::cmd_package::create_sub_cmd(&args_cfg)) .subcommand(crate::cmd::cmd_publish::create_sub_cmd(&args_cfg)) .subcommand(crate::cmd::cmd_delete::create_sub_cmd(&args_cfg)) .subcommand(crate::cmd::cmd_dev_server::create_sub_cmd(&args_cfg)) + .subcommand(crate::cmd::cmd_check::create_sub_cmd(&args_cfg)) .get_matches() } @@ -107,6 +115,7 @@ pub fn parse_cmd( tman_config.user_token = matches.get_one::("USER_TOKEN").cloned(); tman_config.mi_mode = matches.get_flag("MI"); tman_config.verbose = matches.get_flag("VERBOSE"); + tman_config.assume_yes = matches.get_flag("ASSUME_YES"); match matches.subcommand() { Some(("install", sub_cmd_args)) => crate::cmd::CommandData::Install( @@ -131,6 +140,9 @@ pub fn parse_cmd( crate::cmd::cmd_dev_server::parse_sub_cmd(sub_cmd_args), ) } + Some(("check", sub_cmd_args)) => crate::cmd::CommandData::Check( + crate::cmd::cmd_check::parse_sub_cmd(sub_cmd_args), + ), _ => unreachable!("Command not found"), } } diff --git a/core/src/ten_manager/src/config.rs b/core/src/ten_manager/src/config.rs index 2303ba88dc..34b09db164 100644 --- a/core/src/ten_manager/src/config.rs +++ b/core/src/ten_manager/src/config.rs @@ -35,6 +35,7 @@ pub struct TmanConfig { pub mi_mode: bool, pub verbose: bool, + pub assume_yes: bool, } // Determine the config file path based on the platform. @@ -94,5 +95,6 @@ pub fn read_config(config_file_path: &Option) -> TmanConfig { user_token: config_file_content.user_token, mi_mode: false, verbose: false, + assume_yes: false, } } diff --git a/core/src/ten_manager/src/dev_server/graphs/connections.rs b/core/src/ten_manager/src/dev_server/graphs/connections.rs index f9edacafc0..31dc6b4b63 100644 --- a/core/src/ten_manager/src/dev_server/graphs/connections.rs +++ b/core/src/ten_manager/src/dev_server/graphs/connections.rs @@ -40,7 +40,7 @@ pub struct DevServerConnection { impl From for DevServerConnection { fn from(conn: GraphConnection) -> Self { DevServerConnection { - app: conn.app, + app: conn.get_app_uri().to_string(), extension_group: conn.extension_group, extension: conn.extension, @@ -94,7 +94,7 @@ pub struct DevServerDestination { impl From for DevServerDestination { fn from(destination: GraphDestination) -> Self { DevServerDestination { - app: destination.app, + app: destination.get_app_uri().to_string(), extension_group: destination.extension_group, extension: destination.extension, } diff --git a/core/src/ten_manager/src/dev_server/graphs/nodes.rs b/core/src/ten_manager/src/dev_server/graphs/nodes.rs index afe5211177..a4647ea7db 100644 --- a/core/src/ten_manager/src/dev_server/graphs/nodes.rs +++ b/core/src/ten_manager/src/dev_server/graphs/nodes.rs @@ -259,7 +259,7 @@ pub async fn get_graph_nodes( addon: extension.addon.clone(), name: extension.name.clone(), extension_group: extension.extension_group.clone().unwrap(), - app: extension.app.clone(), + app: extension.app.as_ref().unwrap().clone(), api: pkg_info.api.as_ref().map(|api| DevServerApi { property: if api.property.is_empty() { None diff --git a/core/src/ten_manager/src/dev_server/messages/compatible.rs b/core/src/ten_manager/src/dev_server/messages/compatible.rs index 72c289f214..f0653142b5 100644 --- a/core/src/ten_manager/src/dev_server/messages/compatible.rs +++ b/core/src/ten_manager/src/dev_server/messages/compatible.rs @@ -50,7 +50,7 @@ pub struct DevServerCompatibleMsg { impl From> for DevServerCompatibleMsg { fn from(compatible: CompatibleExtensionAndMsg) -> Self { DevServerCompatibleMsg { - app: compatible.extension.app.clone(), + app: compatible.extension.app.as_ref().unwrap().clone(), extension_group: compatible .extension .extension_group diff --git a/core/src/ten_manager/src/lib.rs b/core/src/ten_manager/src/lib.rs new file mode 100644 index 0000000000..db8645d738 --- /dev/null +++ b/core/src/ten_manager/src/lib.rs @@ -0,0 +1,58 @@ +// +// Copyright © 2024 Agora +// This file is part of TEN Framework, an open source project. +// Licensed under the Apache License, Version 2.0, with certain conditions. +// Refer to the "LICENSE" file in the root directory for more information. +// + +// The `ten_manager` crate is only a binary crate, and does not need to provide +// a `lib.rs` file. However, we can _NOT_ write integration tests (test cases in +// the `tests` folder) for `ten_manager` if it does not have a `lib.rs` file. In +// other words, `use` directive does not work for binary crates. According to +// the official reference, we can provide this `lib.rs` file to export the +// important functions for integration tests, and the `main.rs` should be a thin +// wrapper around the library crate, to provide the binary functionality. Refer +// to: https://doc.rust-lang.org/book/ch11-03-test-organization.html#integration-tests-for-binary-crates. +// +// The `lib.rs` does not affect the final output of the binary crate, as the +// output name of the library is different from the binary crate. +// +// Because of the existence of this `lib.rs` file, the unit test will be +// compiled with the `lib.rs`, but not `main.rs`, and some common settings such +// as the allocator below should be added to the `lib.rs` file. + +pub mod cmd; +pub mod cmd_line; +pub mod config; +pub mod constants; +mod dep_and_candidate; +mod dev_server; +mod error; +mod fs; +mod install; +mod log; +mod manifest_lock; +mod package_file; +mod package_info; +mod registry; +mod solver; +mod utils; +mod version; + +#[cfg(not(target_os = "windows"))] +use mimalloc::MiMalloc; + +// TODO(Wei): When adding a URL route with variables (e.g., /api/{name}) in +// actix-web, using the default allocator can lead to a memory leak. According +// to the information in the internet, this leak is likely a false positive and +// may be related to the caching mechanism in actix-web. However, if we use +// mimalloc or jemalloc, there won't be any leak. Use this method to avoid the +// issue for now and study it in more detail in the future. +// +// Refer to the following posts: +// https://github.com/hyperium/hyper/issues/1790#issuecomment-2170644852 +// https://github.com/actix/actix-web/issues/1780 +// https://news.ycombinator.com/item?id=21962195 +#[cfg(not(target_os = "windows"))] +#[global_allocator] +static GLOBAL: MiMalloc = MiMalloc; diff --git a/core/src/ten_manager/src/main.rs b/core/src/ten_manager/src/main.rs index 8396b7a9d0..9d823fbbcc 100644 --- a/core/src/ten_manager/src/main.rs +++ b/core/src/ten_manager/src/main.rs @@ -8,48 +8,14 @@ // Enable this when development. // #![allow(dead_code)] -pub mod cmd; -pub mod cmd_line; -pub mod config; -pub mod constants; -mod dep_and_candidate; -mod dev_server; -mod error; -mod fs; -mod install; -mod log; -mod manifest_lock; -mod package_file; -mod package_info; -mod registry; -mod solver; -mod utils; -mod version; - use std::process; use console::Emoji; use tokio::runtime::Runtime; -use config::TmanConfig; - -#[cfg(not(target_os = "windows"))] -use mimalloc::MiMalloc; - -// TODO(Wei): When adding a URL route with variables (e.g., /api/{name}) in -// actix-web, using the default allocator can lead to a memory leak. According -// to the information in the internet, this leak is likely a false positive and -// may be related to the caching mechanism in actix-web. However, if we use -// mimalloc or jemalloc, there won't be any leak. Use this method to avoid the -// issue for now and study it in more detail in the future. -// -// Refer to the following posts: -// https://github.com/hyperium/hyper/issues/1790#issuecomment-2170644852 -// https://github.com/actix/actix-web/issues/1780 -// https://news.ycombinator.com/item?id=21962195 -#[cfg(not(target_os = "windows"))] -#[global_allocator] -static GLOBAL: MiMalloc = MiMalloc; +use ten_manager::cmd; +use ten_manager::cmd_line; +use ten_manager::config::TmanConfig; fn merge(cmd_line: TmanConfig, config_file: TmanConfig) -> TmanConfig { TmanConfig { @@ -59,6 +25,7 @@ fn merge(cmd_line: TmanConfig, config_file: TmanConfig) -> TmanConfig { user_token: cmd_line.user_token.or(config_file.user_token), mi_mode: cmd_line.mi_mode, verbose: cmd_line.verbose, + assume_yes: cmd_line.assume_yes, } } @@ -66,8 +33,9 @@ fn main() { let mut tman_config_from_cmd_line = TmanConfig::default(); let command_data = cmd_line::parse_cmd(&mut tman_config_from_cmd_line); - let tman_config_from_config_file = - crate::config::read_config(&tman_config_from_cmd_line.config_file); + let tman_config_from_config_file = ten_manager::config::read_config( + &tman_config_from_cmd_line.config_file, + ); let tman_config = merge(tman_config_from_cmd_line, tman_config_from_config_file); diff --git a/core/src/ten_manager/src/package_info/predefined_graphs/connection.rs b/core/src/ten_manager/src/package_info/predefined_graphs/connection.rs index bb08be572d..293def6b99 100644 --- a/core/src/ten_manager/src/package_info/predefined_graphs/connection.rs +++ b/core/src/ten_manager/src/package_info/predefined_graphs/connection.rs @@ -14,7 +14,7 @@ use ten_rust::pkg_info::graph::{ impl From for GraphConnection { fn from(dev_server_connection: DevServerConnection) -> Self { GraphConnection { - app: dev_server_connection.app, + app: Some(dev_server_connection.app), extension_group: dev_server_connection.extension_group, extension: dev_server_connection.extension, @@ -56,7 +56,7 @@ impl From for GraphMessageFlow { impl From for GraphDestination { fn from(dev_server_destination: DevServerDestination) -> Self { GraphDestination { - app: dev_server_destination.app, + app: Some(dev_server_destination.app), extension_group: dev_server_destination.extension_group, extension: dev_server_destination.extension, } diff --git a/core/src/ten_manager/src/package_info/predefined_graphs/node.rs b/core/src/ten_manager/src/package_info/predefined_graphs/node.rs index b6ca6cacc6..7dad589b0c 100644 --- a/core/src/ten_manager/src/package_info/predefined_graphs/node.rs +++ b/core/src/ten_manager/src/package_info/predefined_graphs/node.rs @@ -15,7 +15,7 @@ impl From for GraphNode { name: dev_server_extension.name, addon: dev_server_extension.addon, extension_group: Some(dev_server_extension.extension_group.clone()), - app: dev_server_extension.app, + app: Some(dev_server_extension.app), property: dev_server_extension.property, } } diff --git a/core/src/ten_manager/src/version.rs b/core/src/ten_manager/src/version.rs index ca3bf14907..0a81bdc0a2 100644 --- a/core/src/ten_manager/src/version.rs +++ b/core/src/ten_manager/src/version.rs @@ -1 +1 @@ -pub const VERSION: &str = "0.2.0"; \ No newline at end of file +pub const VERSION: &str = "0.3.0-alpha"; diff --git a/core/src/ten_manager/src/version.rs.jinja2 b/core/src/ten_manager/src/version.rs.jinja2 new file mode 100644 index 0000000000..7492e818c5 --- /dev/null +++ b/core/src/ten_manager/src/version.rs.jinja2 @@ -0,0 +1 @@ +pub const VERSION: &str = "{{ VERSION }}"; \ No newline at end of file diff --git a/core/src/ten_manager/tests/cmd_check_graph.rs b/core/src/ten_manager/tests/cmd_check_graph.rs new file mode 100644 index 0000000000..8f87d3daf3 --- /dev/null +++ b/core/src/ten_manager/tests/cmd_check_graph.rs @@ -0,0 +1,127 @@ +// +// Copyright © 2024 Agora +// This file is part of TEN Framework, an open source project. +// Licensed under the Apache License, Version 2.0, with certain conditions. +// Refer to the "LICENSE" file in the root directory for more information. +// +use ten_manager::{ + cmd::cmd_check::cmd_check_graph::CheckGraphCommand, config::TmanConfig, +}; + +#[actix_rt::test] +async fn test_cmd_check_predefined_graph_success() { + let tman_config = TmanConfig::default(); + let command = CheckGraphCommand { + app: vec![ + "tests/test_data/cmd_check_predefined_graph_success".to_string() + ], + graph: None, + predefined_graph_name: None, + }; + + let result = ten_manager::cmd::cmd_check::cmd_check_graph::execute_cmd( + &tman_config, + command, + ) + .await; + assert!(result.is_ok()); +} + +#[actix_rt::test] +async fn test_cmd_check_start_graph_multi_apps() { + let tman_config = TmanConfig::default(); + let command = CheckGraphCommand { + app: vec![ + "tests/test_data/cmd_check_start_graph_multi_apps/app_1" + .to_string(), + "tests/test_data/cmd_check_start_graph_multi_apps/app_2" + .to_string(), + ], + graph: Some( + include_str!( + "test_data/cmd_check_start_graph_multi_apps/start_graph.json" + ) + .to_string(), + ), + predefined_graph_name: None, + }; + + let result = ten_manager::cmd::cmd_check::cmd_check_graph::execute_cmd( + &tman_config, + command, + ) + .await; + + assert!(result.is_err()); + eprintln!("{:?}", result); +} + +#[actix_rt::test] +async fn test_cmd_check_app_in_graph_cannot_be_localhost() { + let tman_config = TmanConfig::default(); + let command = CheckGraphCommand { + app: vec!["tests/test_data/cmd_check_app_in_graph_cannot_be_localhost" + .to_string()], + graph: None, + predefined_graph_name: None, + }; + + let result = ten_manager::cmd::cmd_check::cmd_check_graph::execute_cmd( + &tman_config, + command, + ) + .await; + + assert!(result.is_err()); + eprintln!("{:?}", result); + + let msg = result.err().unwrap().to_string(); + assert!(msg + .contains("the app uri should be some string other than 'localhost'")); +} + +#[actix_rt::test] +async fn test_cmd_check_predefined_graph_only_check_specified() { + let tman_config = TmanConfig::default(); + let command = CheckGraphCommand { + app: vec![ + "tests/test_data/cmd_check_predefined_graph_only_check_specified" + .to_string(), + ], + graph: None, + predefined_graph_name: Some("default".to_string()), + }; + + let result = ten_manager::cmd::cmd_check::cmd_check_graph::execute_cmd( + &tman_config, + command, + ) + .await; + + assert!(result.is_ok()); +} + +#[actix_rt::test] +async fn test_cmd_check_predefined_graph_check_all() { + let tman_config = TmanConfig::default(); + let command = CheckGraphCommand { + app: vec![ + "tests/test_data/cmd_check_predefined_graph_only_check_specified" + .to_string(), + ], + graph: None, + predefined_graph_name: None, + }; + + let result = ten_manager::cmd::cmd_check::cmd_check_graph::execute_cmd( + &tman_config, + command, + ) + .await; + + assert!(result.is_err()); + eprintln!("{:?}", result); + + let msg = result.err().unwrap().to_string(); + assert!(msg.contains("1/2 graphs failed")); +} diff --git a/core/src/ten_manager/tests/integration_test.rs b/core/src/ten_manager/tests/integration_test.rs new file mode 100644 index 0000000000..129ff0602f --- /dev/null +++ b/core/src/ten_manager/tests/integration_test.rs @@ -0,0 +1,18 @@ +// +// Copyright © 2024 Agora +// This file is part of TEN Framework, an open source project. +// Licensed under the Apache License, Version 2.0, with certain conditions. +// Refer to the "LICENSE" file in the root directory for more information. +// + +// Each file in the tests directory is a separate crate, and will be compiled as +// an executable. We need a all-in-one executable to run all the tests, and we +// will copy the all-in-one executable to the out directory. We use this main +// file to achieve this purpose. + +fn main() { + println!("Running integration tests of ten_man..."); +} + +// Those following mods will be compiled in one executable. +mod cmd_check_graph; diff --git a/core/src/ten_manager/tests/test_data/cmd_check_app_in_graph_cannot_be_localhost/manifest.json b/core/src/ten_manager/tests/test_data/cmd_check_app_in_graph_cannot_be_localhost/manifest.json new file mode 100644 index 0000000000..537723c67f --- /dev/null +++ b/core/src/ten_manager/tests/test_data/cmd_check_app_in_graph_cannot_be_localhost/manifest.json @@ -0,0 +1,15 @@ +{ + "type": "app", + "name": "app", + "version": "0.1.0", + "dependencies": [ + { + "type": "extension", + "name": "addon_a", + "version": "0.1.0" + } + ], + "package": { + "include": ["**"] + } +} diff --git a/core/src/ten_manager/tests/test_data/cmd_check_app_in_graph_cannot_be_localhost/property.json b/core/src/ten_manager/tests/test_data/cmd_check_app_in_graph_cannot_be_localhost/property.json new file mode 100644 index 0000000000..db9f0c18c8 --- /dev/null +++ b/core/src/ten_manager/tests/test_data/cmd_check_app_in_graph_cannot_be_localhost/property.json @@ -0,0 +1,19 @@ +{ + "_ten": { + "predefined_graphs": [ + { + "name": "default", + "auto_start": false, + "nodes": [ + { + "type": "extension", + "name": "ext_a", + "addon": "addon_a", + "extension_group": "some_group", + "app": "localhost" + } + ] + } + ] + } +} diff --git a/core/src/ten_manager/tests/test_data/cmd_check_predefined_graph_only_check_specified/manifest.json b/core/src/ten_manager/tests/test_data/cmd_check_predefined_graph_only_check_specified/manifest.json new file mode 100644 index 0000000000..7cb8a2ca0c --- /dev/null +++ b/core/src/ten_manager/tests/test_data/cmd_check_predefined_graph_only_check_specified/manifest.json @@ -0,0 +1,22 @@ +{ + "type": "app", + "name": "check_predefined_graph_success", + "version": "0.1.0", + "dependencies": [ + { + "type": "extension", + "name": "addon_a", + "version": "0.1.0" + }, + { + "type": "extension", + "name": "addon_b", + "version": "0.1.0" + } + ], + "package": { + "include": [ + "**" + ] + } +} \ No newline at end of file diff --git a/core/src/ten_manager/tests/test_data/cmd_check_predefined_graph_only_check_specified/property.json b/core/src/ten_manager/tests/test_data/cmd_check_predefined_graph_only_check_specified/property.json new file mode 100644 index 0000000000..ee386b3d94 --- /dev/null +++ b/core/src/ten_manager/tests/test_data/cmd_check_predefined_graph_only_check_specified/property.json @@ -0,0 +1,76 @@ +{ + "_ten": { + "predefined_graphs": [ + { + "name": "default", + "auto_start": false, + "nodes": [ + { + "type": "extension", + "name": "ext_a", + "addon": "addon_a", + "extension_group": "some_group" + }, + { + "type": "extension", + "name": "ext_b", + "addon": "addon_b", + "extension_group": "some_group" + } + ], + "connections": [ + { + "extension_group": "some_group", + "extension": "ext_a", + "cmd": [ + { + "name": "cmd_1", + "dest": [ + { + "extension_group": "some_group", + "extension": "ext_b" + } + ] + } + ] + } + ] + }, + { + "name": "not_checked", + "auto_start": false, + "nodes": [ + { + "type": "extension", + "name": "ext_a", + "addon": "addon_a", + "extension_group": "some_group" + }, + { + "type": "extension", + "name": "ext_b", + "addon": "addon_b", + "extension_group": "some_group" + } + ], + "connections": [ + { + "extension_group": "some_group", + "extension": "ext_a", + "cmd": [ + { + "name": "cmd_1", + "dest": [ + { + "extension_group": "some_group", + "extension": "ext_c" + } + ] + } + ] + } + ] + } + ] + } +} diff --git a/core/src/ten_manager/tests/test_data/cmd_check_predefined_graph_only_check_specified/ten_packages/extension/addon_a/manifest.json b/core/src/ten_manager/tests/test_data/cmd_check_predefined_graph_only_check_specified/ten_packages/extension/addon_a/manifest.json new file mode 100644 index 0000000000..05434a5648 --- /dev/null +++ b/core/src/ten_manager/tests/test_data/cmd_check_predefined_graph_only_check_specified/ten_packages/extension/addon_a/manifest.json @@ -0,0 +1,18 @@ +{ + "type": "extension", + "name": "addon_a", + "version": "0.1.0", + "dependencies": [ + { + "type": "system", + "name": "ten_runtime_go", + "version": "0.1.0" + } + ], + "package": { + "include": [ + "**" + ] + }, + "api": {} +} \ No newline at end of file diff --git a/core/src/ten_manager/tests/test_data/cmd_check_predefined_graph_only_check_specified/ten_packages/extension/addon_b/manifest.json b/core/src/ten_manager/tests/test_data/cmd_check_predefined_graph_only_check_specified/ten_packages/extension/addon_b/manifest.json new file mode 100644 index 0000000000..9821e29f4d --- /dev/null +++ b/core/src/ten_manager/tests/test_data/cmd_check_predefined_graph_only_check_specified/ten_packages/extension/addon_b/manifest.json @@ -0,0 +1,18 @@ +{ + "type": "extension", + "name": "addon_b", + "version": "0.1.0", + "dependencies": [ + { + "type": "system", + "name": "ten_runtime_go", + "version": "0.1.0" + } + ], + "package": { + "include": [ + "**" + ] + }, + "api": {} +} \ No newline at end of file diff --git a/core/src/ten_manager/tests/test_data/cmd_check_predefined_graph_success/manifest.json b/core/src/ten_manager/tests/test_data/cmd_check_predefined_graph_success/manifest.json new file mode 100644 index 0000000000..7cb8a2ca0c --- /dev/null +++ b/core/src/ten_manager/tests/test_data/cmd_check_predefined_graph_success/manifest.json @@ -0,0 +1,22 @@ +{ + "type": "app", + "name": "check_predefined_graph_success", + "version": "0.1.0", + "dependencies": [ + { + "type": "extension", + "name": "addon_a", + "version": "0.1.0" + }, + { + "type": "extension", + "name": "addon_b", + "version": "0.1.0" + } + ], + "package": { + "include": [ + "**" + ] + } +} \ No newline at end of file diff --git a/core/src/ten_manager/tests/test_data/cmd_check_predefined_graph_success/property.json b/core/src/ten_manager/tests/test_data/cmd_check_predefined_graph_success/property.json new file mode 100644 index 0000000000..31f6bdd67a --- /dev/null +++ b/core/src/ten_manager/tests/test_data/cmd_check_predefined_graph_success/property.json @@ -0,0 +1,41 @@ +{ + "_ten": { + "predefined_graphs": [ + { + "name": "default", + "auto_start": false, + "nodes": [ + { + "type": "extension", + "name": "ext_a", + "addon": "addon_a", + "extension_group": "some_group" + }, + { + "type": "extension", + "name": "ext_b", + "addon": "addon_b", + "extension_group": "some_group" + } + ], + "connections": [ + { + "extension_group": "some_group", + "extension": "ext_a", + "cmd": [ + { + "name": "cmd_1", + "dest": [ + { + "extension_group": "some_group", + "extension": "ext_b" + } + ] + } + ] + } + ] + } + ] + } +} diff --git a/core/src/ten_manager/tests/test_data/cmd_check_predefined_graph_success/ten_packages/extension/addon_a/manifest.json b/core/src/ten_manager/tests/test_data/cmd_check_predefined_graph_success/ten_packages/extension/addon_a/manifest.json new file mode 100644 index 0000000000..05434a5648 --- /dev/null +++ b/core/src/ten_manager/tests/test_data/cmd_check_predefined_graph_success/ten_packages/extension/addon_a/manifest.json @@ -0,0 +1,18 @@ +{ + "type": "extension", + "name": "addon_a", + "version": "0.1.0", + "dependencies": [ + { + "type": "system", + "name": "ten_runtime_go", + "version": "0.1.0" + } + ], + "package": { + "include": [ + "**" + ] + }, + "api": {} +} \ No newline at end of file diff --git a/core/src/ten_manager/tests/test_data/cmd_check_predefined_graph_success/ten_packages/extension/addon_b/manifest.json b/core/src/ten_manager/tests/test_data/cmd_check_predefined_graph_success/ten_packages/extension/addon_b/manifest.json new file mode 100644 index 0000000000..9821e29f4d --- /dev/null +++ b/core/src/ten_manager/tests/test_data/cmd_check_predefined_graph_success/ten_packages/extension/addon_b/manifest.json @@ -0,0 +1,18 @@ +{ + "type": "extension", + "name": "addon_b", + "version": "0.1.0", + "dependencies": [ + { + "type": "system", + "name": "ten_runtime_go", + "version": "0.1.0" + } + ], + "package": { + "include": [ + "**" + ] + }, + "api": {} +} \ No newline at end of file diff --git a/core/src/ten_manager/tests/test_data/cmd_check_start_graph_multi_apps/app_1/manifest.json b/core/src/ten_manager/tests/test_data/cmd_check_start_graph_multi_apps/app_1/manifest.json new file mode 100644 index 0000000000..68cec56dcd --- /dev/null +++ b/core/src/ten_manager/tests/test_data/cmd_check_start_graph_multi_apps/app_1/manifest.json @@ -0,0 +1,15 @@ +{ + "type": "app", + "name": "app_1", + "version": "0.1.0", + "dependencies": [ + { + "type": "extension", + "name": "addon_a", + "version": "0.1.0" + } + ], + "package": { + "include": ["**"] + } +} diff --git a/core/src/ten_manager/tests/test_data/cmd_check_start_graph_multi_apps/app_1/property.json b/core/src/ten_manager/tests/test_data/cmd_check_start_graph_multi_apps/app_1/property.json new file mode 100644 index 0000000000..6ee844f45b --- /dev/null +++ b/core/src/ten_manager/tests/test_data/cmd_check_start_graph_multi_apps/app_1/property.json @@ -0,0 +1,5 @@ +{ + "_ten": { + "uri": "http://localhost:8001" + } +} diff --git a/core/src/ten_manager/tests/test_data/cmd_check_start_graph_multi_apps/app_1/ten_packages/extension/addon_a/manifest.json b/core/src/ten_manager/tests/test_data/cmd_check_start_graph_multi_apps/app_1/ten_packages/extension/addon_a/manifest.json new file mode 100644 index 0000000000..05434a5648 --- /dev/null +++ b/core/src/ten_manager/tests/test_data/cmd_check_start_graph_multi_apps/app_1/ten_packages/extension/addon_a/manifest.json @@ -0,0 +1,18 @@ +{ + "type": "extension", + "name": "addon_a", + "version": "0.1.0", + "dependencies": [ + { + "type": "system", + "name": "ten_runtime_go", + "version": "0.1.0" + } + ], + "package": { + "include": [ + "**" + ] + }, + "api": {} +} \ No newline at end of file diff --git a/core/src/ten_manager/tests/test_data/cmd_check_start_graph_multi_apps/app_2/manifest.json b/core/src/ten_manager/tests/test_data/cmd_check_start_graph_multi_apps/app_2/manifest.json new file mode 100644 index 0000000000..3fbe2455af --- /dev/null +++ b/core/src/ten_manager/tests/test_data/cmd_check_start_graph_multi_apps/app_2/manifest.json @@ -0,0 +1,15 @@ +{ + "type": "app", + "name": "app_2", + "version": "0.1.0", + "dependencies": [ + { + "type": "extension", + "name": "addon_b", + "version": "0.1.0" + } + ], + "package": { + "include": ["**"] + } +} diff --git a/core/src/ten_manager/tests/test_data/cmd_check_start_graph_multi_apps/app_2/property.json b/core/src/ten_manager/tests/test_data/cmd_check_start_graph_multi_apps/app_2/property.json new file mode 100644 index 0000000000..04ae423248 --- /dev/null +++ b/core/src/ten_manager/tests/test_data/cmd_check_start_graph_multi_apps/app_2/property.json @@ -0,0 +1,5 @@ +{ + "_ten": { + "uri": "http://localhost:8002" + } +} diff --git a/core/src/ten_manager/tests/test_data/cmd_check_start_graph_multi_apps/app_2/ten_packages/extension/addon_b/manifest.json b/core/src/ten_manager/tests/test_data/cmd_check_start_graph_multi_apps/app_2/ten_packages/extension/addon_b/manifest.json new file mode 100644 index 0000000000..732d941358 --- /dev/null +++ b/core/src/ten_manager/tests/test_data/cmd_check_start_graph_multi_apps/app_2/ten_packages/extension/addon_b/manifest.json @@ -0,0 +1,27 @@ +{ + "type": "extension", + "name": "addon_b", + "version": "0.1.0", + "dependencies": [ + { + "type": "system", + "name": "ten_runtime_go", + "version": "0.1.0" + } + ], + "package": { + "include": ["**"] + }, + "api": { + "cmd_in": [ + { + "name": "cmd_1", + "property": { + "foo": { + "type": "string" + } + } + } + ] + } +} diff --git a/core/src/ten_manager/tests/test_data/cmd_check_start_graph_multi_apps/start_graph.json b/core/src/ten_manager/tests/test_data/cmd_check_start_graph_multi_apps/start_graph.json new file mode 100644 index 0000000000..538e30f5de --- /dev/null +++ b/core/src/ten_manager/tests/test_data/cmd_check_start_graph_multi_apps/start_graph.json @@ -0,0 +1,39 @@ +{ + "type": "start_graph", + "seq_id": "55", + "nodes": [ + { + "type": "extension", + "name": "ext_a", + "addon": "addon_a", + "extension_group": "some_group", + "app": "http://localhost:8001" + }, + { + "type": "extension", + "name": "ext_b", + "addon": "addon_b", + "extension_group": "some_group", + "app": "http://localhost:8002" + } + ], + "connections": [ + { + "extension_group": "some_group", + "extension": "ext_a", + "app": "http://localhost:8001", + "cmd": [ + { + "name": "cmd_1", + "dest": [ + { + "extension_group": "some_group", + "extension": "ext_b", + "app": "http://localhost:8002" + } + ] + } + ] + } + ] +} diff --git a/core/src/ten_runtime/binding/go/manifest.json b/core/src/ten_runtime/binding/go/manifest.json index 7a3b3e7183..ff95c29194 100644 --- a/core/src/ten_runtime/binding/go/manifest.json +++ b/core/src/ten_runtime/binding/go/manifest.json @@ -1,12 +1,12 @@ { "type": "system", "name": "ten_runtime_go", - "version": "0.1.0", + "version": "0.3.0-alpha", "dependencies": [ { "type": "system", "name": "ten_runtime", - "version": "0.2.0" + "version": "0.3.0-alpha" } ], "package": { diff --git a/core/src/ten_runtime/binding/python/interface/ten/__init__.py b/core/src/ten_runtime/binding/python/interface/ten/__init__.py index cab4f2beb6..ca3eb43b8c 100644 --- a/core/src/ten_runtime/binding/python/interface/ten/__init__.py +++ b/core/src/ten_runtime/binding/python/interface/ten/__init__.py @@ -6,6 +6,8 @@ # from .app import App from .extension import Extension +from .async_extension import AsyncExtension +from .async_ten_env import AsyncTenEnv from .addon import Addon from .decorator import ( register_addon_as_extension, @@ -28,7 +30,9 @@ "register_addon_as_extension_group", "App", "Extension", + "AsyncExtension", "TenEnv", + "AsyncTenEnv", "Cmd", "StatusCode", "VideoFrame", diff --git a/core/src/ten_runtime/binding/python/interface/ten/app.py b/core/src/ten_runtime/binding/python/interface/ten/app.py index 321a928652..7de98704b5 100644 --- a/core/src/ten_runtime/binding/python/interface/ten/app.py +++ b/core/src/ten_runtime/binding/python/interface/ten/app.py @@ -26,3 +26,6 @@ def on_configure(self, ten_env: TenEnv) -> None: def on_init(self, ten_env: TenEnv) -> None: ten_env.on_init_done() + + def on_deinit(self, ten_env: TenEnv) -> None: + ten_env.on_deinit_done() diff --git a/core/src/ten_runtime/binding/python/interface/ten/async_extension.py b/core/src/ten_runtime/binding/python/interface/ten/async_extension.py new file mode 100644 index 0000000000..6f9de65d22 --- /dev/null +++ b/core/src/ten_runtime/binding/python/interface/ten/async_extension.py @@ -0,0 +1,140 @@ +# +# Copyright © 2024 Agora +# This file is part of TEN Framework, an open source project. +# Licensed under the Apache License, Version 2.0, with certain conditions. +# Refer to the "LICENSE" file in the root directory for more information. +# +import asyncio +import threading +from typing import final +from libten_runtime_python import _Extension +from .video_frame import VideoFrame +from .audio_frame import AudioFrame +from .ten_env import TenEnv +from .cmd import Cmd +from .data import Data +from .async_ten_env import AsyncTenEnv + + +class AsyncExtension(_Extension): + def __init__(self, name: str) -> None: + self._ten_stop_event = asyncio.Event() + + def __del__(self) -> None: + self._ten_stop_event.set() + if hasattr(self, "_ten_thread"): + self._ten_thread.join() + + async def _thread_routine(self, ten_env: TenEnv): + self._ten_loop = asyncio.get_running_loop() + self._async_ten_env = AsyncTenEnv( + ten_env, self._ten_loop, self._ten_thread + ) + + await self.on_configure(self._async_ten_env) + + # Suspend the thread until stopEvent is set. + await self._ten_stop_event.wait() + + await self.on_deinit(self._async_ten_env) + + async def _stop_thread(self): + self._ten_stop_event.set() + + @final + def _proxy_on_configure(self, ten_env: TenEnv) -> None: + # We pass the TenEnv object to another Python thread without worrying + # about the thread safety issue of the TenEnv API, because the actual + # execution logic of all TenEnv APIs occurs in the extension thread. + # We only need to ensure that the TenEnv object should remain valid + # while it is being used. The way to achieve this is to ensure that the + # Python thread remains alive until TenEnv.on_deinit_done is called. + self._ten_thread = threading.Thread( + target=asyncio.run, args=(self._thread_routine(ten_env),) + ) + self._ten_thread.start() + + @final + def _proxy_on_init(self, ten_env: TenEnv) -> None: + asyncio.run_coroutine_threadsafe( + self.on_init(self._async_ten_env), self._ten_loop + ) + + @final + def _proxy_on_start(self, ten_env: TenEnv) -> None: + asyncio.run_coroutine_threadsafe( + self.on_start(self._async_ten_env), self._ten_loop + ) + + @final + def _proxy_on_stop(self, ten_env: TenEnv) -> None: + asyncio.run_coroutine_threadsafe( + self.on_stop(self._async_ten_env), self._ten_loop + ) + + @final + def _proxy_on_deinit(self, ten_env: TenEnv) -> None: + asyncio.run_coroutine_threadsafe(self._stop_thread(), self._ten_loop) + + @final + def _proxy_on_cmd(self, ten_env: TenEnv, cmd: Cmd) -> None: + asyncio.run_coroutine_threadsafe( + self.on_cmd(self._async_ten_env, cmd), self._ten_loop + ) + + @final + def _proxy_on_data(self, ten_env: TenEnv, data: Data) -> None: + asyncio.run_coroutine_threadsafe( + self.on_data(self._async_ten_env, data), self._ten_loop + ) + + @final + def _proxy_on_video_frame( + self, ten_env: TenEnv, video_frame: VideoFrame + ) -> None: + asyncio.run_coroutine_threadsafe( + self.on_video_frame(self._async_ten_env, video_frame), + self._ten_loop, + ) + + @final + def _proxy_on_audio_frame( + self, ten_env: TenEnv, audio_frame: AudioFrame + ) -> None: + asyncio.run_coroutine_threadsafe( + self.on_audio_frame(self._async_ten_env, audio_frame), + self._ten_loop, + ) + + # Override these methods in your extension + + async def on_configure(self, async_ten_env: AsyncTenEnv) -> None: + async_ten_env.on_configure_done() + + async def on_init(self, async_ten_env: AsyncTenEnv) -> None: + async_ten_env.on_init_done() + + async def on_start(self, async_ten_env: AsyncTenEnv) -> None: + async_ten_env.on_start_done() + + async def on_stop(self, async_ten_env: AsyncTenEnv) -> None: + async_ten_env.on_stop_done() + + async def on_deinit(self, async_ten_env: AsyncTenEnv) -> None: + async_ten_env.on_deinit_done() + + async def on_cmd(self, async_ten_env: AsyncTenEnv, cmd: Cmd) -> None: + pass + + async def on_data(self, async_ten_env: AsyncTenEnv, data: Data) -> None: + pass + + async def on_video_frame( + self, async_ten_env: AsyncTenEnv, video_frame: VideoFrame + ) -> None: + pass + + async def on_audio_frame( + self, async_ten_env: AsyncTenEnv, audio_frame: AudioFrame + ) -> None: + pass diff --git a/core/src/ten_runtime/binding/python/interface/ten/async_ten_env.py b/core/src/ten_runtime/binding/python/interface/ten/async_ten_env.py new file mode 100644 index 0000000000..84c29563a2 --- /dev/null +++ b/core/src/ten_runtime/binding/python/interface/ten/async_ten_env.py @@ -0,0 +1,61 @@ +# +# Copyright © 2024 Agora +# This file is part of TEN Framework, an open source project. +# Licensed under the Apache License, Version 2.0, with certain conditions. +# Refer to the "LICENSE" file in the root directory for more information. +# +from asyncio import AbstractEventLoop +import asyncio +import threading +from .cmd import Cmd +from .cmd_result import CmdResult +from .ten_env import TenEnv + + +class AsyncTenEnv(TenEnv): + + def __init__( + self, ten_env: TenEnv, loop: AbstractEventLoop, thread: threading.Thread + ) -> None: + self._internal = ten_env._internal + self._ten_loop = loop + self._ten_thread = thread + ten_env._set_release_handler(lambda: self._on_release()) + + def __del__(self) -> None: + pass + + async def send_cmd(self, cmd: Cmd) -> CmdResult: + q = asyncio.Queue(1) + self._internal.send_cmd( + cmd, + lambda ten_env, result: asyncio.run_coroutine_threadsafe( + q.put(result), self._ten_loop + ), # type: ignore + ) + return await q.get() + + async def send_json(self, json_str: str) -> CmdResult: + q = asyncio.Queue(1) + self._internal.send_json( + json_str, + lambda ten_env, result: asyncio.run_coroutine_threadsafe( + q.put(result), self._ten_loop + ), # type: ignore + ) + return await q.get() + + def _deinit_routine(self) -> None: + # Wait for the internal thread to finish. + self._ten_thread.join() + + self._internal.on_deinit_done() + + def _on_release(self) -> None: + if hasattr(self, "_deinit_thread"): + self._deinit_thread.join() + + def on_deinit_done(self) -> None: + # Start the deinit thread to avoid blocking the extension thread. + self._deinit_thread = threading.Thread(target=self._deinit_routine) + self._deinit_thread.start() diff --git a/core/src/ten_runtime/binding/python/interface/ten/extension.py b/core/src/ten_runtime/binding/python/interface/ten/extension.py index d4d0b22752..2a6fbe3ed5 100644 --- a/core/src/ten_runtime/binding/python/interface/ten/extension.py +++ b/core/src/ten_runtime/binding/python/interface/ten/extension.py @@ -33,26 +33,62 @@ def _proxy_on_configure(self, ten_env: TenEnv) -> None: def on_configure(self, ten_env: TenEnv) -> None: ten_env.on_configure_done() + @final + def _proxy_on_init(self, ten_env: TenEnv) -> None: + self.on_init(ten_env) + def on_init(self, ten_env: TenEnv) -> None: ten_env.on_init_done() + @final + def _proxy_on_start(self, ten_env: TenEnv) -> None: + self.on_start(ten_env) + def on_start(self, ten_env: TenEnv) -> None: ten_env.on_start_done() + @final + def _proxy_on_stop(self, ten_env: TenEnv) -> None: + self.on_stop(ten_env) + def on_stop(self, ten_env: TenEnv) -> None: ten_env.on_stop_done() + @final + def _proxy_on_deinit(self, ten_env: TenEnv) -> None: + self.on_deinit(ten_env) + def on_deinit(self, ten_env: TenEnv) -> None: ten_env.on_deinit_done() + @final + def _proxy_on_cmd(self, ten_env: TenEnv, cmd: Cmd) -> None: + self.on_cmd(ten_env, cmd) + def on_cmd(self, ten_env: TenEnv, cmd: Cmd) -> None: pass + @final + def _proxy_on_data(self, ten_env: TenEnv, data: Data) -> None: + self.on_data(ten_env, data) + def on_data(self, ten_env: TenEnv, data: Data) -> None: pass + @final + def _proxy_on_video_frame( + self, ten_env: TenEnv, video_frame: VideoFrame + ) -> None: + self.on_video_frame(ten_env, video_frame) + def on_video_frame(self, ten_env: TenEnv, video_frame: VideoFrame) -> None: pass + @final + def _proxy_on_audio_frame( + self, ten_env: TenEnv, audio_frame: AudioFrame + ) -> None: + self.on_audio_frame(ten_env, audio_frame) + def on_audio_frame(self, ten_env: TenEnv, audio_frame: AudioFrame) -> None: pass diff --git a/core/src/ten_runtime/binding/python/interface/ten/libten_runtime_python.pyi b/core/src/ten_runtime/binding/python/interface/ten/libten_runtime_python.pyi index 9300b0a165..38baeacef2 100644 --- a/core/src/ten_runtime/binding/python/interface/ten/libten_runtime_python.pyi +++ b/core/src/ten_runtime/binding/python/interface/ten/libten_runtime_python.pyi @@ -126,6 +126,10 @@ class _App: self, ten_env: _TenEnv, ) -> None: ... + def on_deinit( + self, + ten_env: _TenEnv, + ) -> None: ... class _Extension: def __init__(self, name: str): ... diff --git a/core/src/ten_runtime/binding/python/interface/ten/ten_env.py b/core/src/ten_runtime/binding/python/interface/ten/ten_env.py index bc15d7fc45..573e1dd10e 100644 --- a/core/src/ten_runtime/binding/python/interface/ten/ten_env.py +++ b/core/src/ten_runtime/binding/python/interface/ten/ten_env.py @@ -30,6 +30,13 @@ def __init__(self, internal_obj: _TenEnv) -> None: def __del__(self) -> None: pass + def _set_release_handler(self, handler: Callable[[], None]) -> None: + self._release_handler = handler + + def _on_release(self) -> None: + if hasattr(self, "_release_handler"): + self._release_handler() + def on_configure_done(self) -> None: from .addon import Addon diff --git a/core/src/ten_runtime/binding/python/manifest.json b/core/src/ten_runtime/binding/python/manifest.json index 9f64d1361b..bfe3299cbe 100644 --- a/core/src/ten_runtime/binding/python/manifest.json +++ b/core/src/ten_runtime/binding/python/manifest.json @@ -1,12 +1,12 @@ { "type": "system", "name": "ten_runtime_python", - "version": "0.1.0", + "version": "0.3.0-alpha", "dependencies": [ { "type": "system", "name": "ten_runtime", - "version": "0.2.0" + "version": "0.3.0-alpha" } ], "package": { diff --git a/core/src/ten_runtime/binding/python/native/app/app.c b/core/src/ten_runtime/binding/python/native/app/app.c index 17586366af..3d5d2ed113 100644 --- a/core/src/ten_runtime/binding/python/native/app/app.c +++ b/core/src/ten_runtime/binding/python/native/app/app.c @@ -74,7 +74,7 @@ static void proxy_on_configure(ten_app_t *app, ten_env_t *ten_env) { // achieving numerical consistency between PyGILState_Ensure and // PyGILState_Release, and only then will the Python thread state be // released. - ten_py_eval_save_thread(); + py_ten_env->py_thread_state = ten_py_eval_save_thread(); } else { // No need to release the GIL. } diff --git a/core/src/ten_runtime/binding/python/native/extension/extension.c b/core/src/ten_runtime/binding/python/native/extension/extension.c index ab40c99ec6..62f28e4a50 100644 --- a/core/src/ten_runtime/binding/python/native/extension/extension.c +++ b/core/src/ten_runtime/binding/python/native/extension/extension.c @@ -82,7 +82,7 @@ static void proxy_on_configure(ten_extension_t *extension, ten_env_t *ten_env) { // We should release the GIL but not destroy the PyThreadState. The // PyThreadState will not be released until the last extension calls // 'on_deinit_done' in the group. - ten_py_eval_save_thread(); + py_ten_env->py_thread_state = ten_py_eval_save_thread(); py_ten_env->need_to_release_gil_state = true; } @@ -112,7 +112,7 @@ static void proxy_on_init(ten_extension_t *extension, ten_env_t *ten_env) { TEN_ASSERT(py_ten_env, "Should not happen."); PyObject *py_res = - PyObject_CallMethod((PyObject *)py_extension, "on_init", "O", + PyObject_CallMethod((PyObject *)py_extension, "_proxy_on_init", "O", ((ten_py_ten_env_t *)py_ten_env)->actual_py_ten_env); Py_XDECREF(py_res); @@ -145,7 +145,7 @@ static void proxy_on_start(ten_extension_t *extension, ten_env_t *ten_env) { TEN_ASSERT(py_ten_env, "Should not happen."); PyObject *py_res = - PyObject_CallMethod((PyObject *)py_extension, "on_start", "O", + PyObject_CallMethod((PyObject *)py_extension, "_proxy_on_start", "O", ((ten_py_ten_env_t *)py_ten_env)->actual_py_ten_env); Py_XDECREF(py_res); @@ -178,7 +178,7 @@ static void proxy_on_stop(ten_extension_t *extension, ten_env_t *ten_env) { TEN_ASSERT(py_ten_env, "Should not happen."); PyObject *py_res = - PyObject_CallMethod((PyObject *)py_extension, "on_stop", "O", + PyObject_CallMethod((PyObject *)py_extension, "_proxy_on_stop", "O", ((ten_py_ten_env_t *)py_ten_env)->actual_py_ten_env); Py_XDECREF(py_res); @@ -211,7 +211,7 @@ static void proxy_on_deinit(ten_extension_t *extension, ten_env_t *ten_env) { TEN_ASSERT(py_ten_env, "Should not happen."); PyObject *py_res = - PyObject_CallMethod((PyObject *)py_extension, "on_deinit", "O", + PyObject_CallMethod((PyObject *)py_extension, "_proxy_on_deinit", "O", ((ten_py_ten_env_t *)py_ten_env)->actual_py_ten_env); Py_XDECREF(py_res); @@ -246,7 +246,7 @@ static void proxy_on_cmd(ten_extension_t *extension, ten_env_t *ten_env, ten_py_cmd_t *py_cmd = ten_py_cmd_wrap(cmd); PyObject *py_res = PyObject_CallMethod( - (PyObject *)py_extension, "on_cmd", "OO", + (PyObject *)py_extension, "_proxy_on_cmd", "OO", ((ten_py_ten_env_t *)py_ten_env)->actual_py_ten_env, py_cmd); Py_XDECREF(py_res); @@ -283,7 +283,7 @@ static void proxy_on_data(ten_extension_t *extension, ten_env_t *ten_env, ten_py_data_t *py_data = ten_py_data_wrap(data); PyObject *py_res = PyObject_CallMethod( - (PyObject *)py_extension, "on_data", "OO", + (PyObject *)py_extension, "_proxy_on_data", "OO", ((ten_py_ten_env_t *)py_ten_env)->actual_py_ten_env, py_data); Py_XDECREF(py_res); @@ -321,7 +321,7 @@ static void proxy_on_audio_frame(ten_extension_t *extension, ten_env_t *ten_env, ten_py_audio_frame_t *py_audio_frame = ten_py_audio_frame_wrap(audio_frame); PyObject *py_res = PyObject_CallMethod( - (PyObject *)py_extension, "on_audio_frame", "OO", + (PyObject *)py_extension, "_proxy_on_audio_frame", "OO", ((ten_py_ten_env_t *)py_ten_env)->actual_py_ten_env, py_audio_frame); Py_XDECREF(py_res); @@ -352,7 +352,7 @@ static void proxy_on_video_frame(ten_extension_t *extension, ten_env_t *ten_env, ten_py_video_frame_t *py_video_frame = ten_py_video_frame_wrap(video_frame); PyObject *py_res = PyObject_CallMethod( - py_extension, "on_video_frame", "OO", + py_extension, "_proxy_on_video_frame", "OO", ((ten_py_ten_env_t *)py_ten_env)->actual_py_ten_env, py_video_frame); Py_XDECREF(py_res); diff --git a/core/src/ten_runtime/binding/python/native/ten_env/ten_env.c b/core/src/ten_runtime/binding/python/native/ten_env/ten_env.c index 40b9f39484..ff8d1615bd 100644 --- a/core/src/ten_runtime/binding/python/native/ten_env/ten_env.c +++ b/core/src/ten_runtime/binding/python/native/ten_env/ten_env.c @@ -95,6 +95,7 @@ ten_py_ten_env_t *ten_py_ten_env_wrap(ten_env_t *ten_env) { py_ten_env->c_ten_env = ten_env; py_ten_env->c_ten_env_proxy = NULL; py_ten_env->need_to_release_gil_state = false; + py_ten_env->py_thread_state = NULL; py_ten_env->actual_py_ten_env = create_actual_py_ten_env_instance(py_ten_env); if (!py_ten_env->actual_py_ten_env) { diff --git a/core/src/ten_runtime/binding/python/native/ten_env/ten_env_on_deinit_done.c b/core/src/ten_runtime/binding/python/native/ten_env/ten_env_on_deinit_done.c index e079e38730..e23bbdadd4 100644 --- a/core/src/ten_runtime/binding/python/native/ten_env/ten_env_on_deinit_done.c +++ b/core/src/ten_runtime/binding/python/native/ten_env/ten_env_on_deinit_done.c @@ -8,10 +8,13 @@ #include "include_internal/ten_runtime/binding/python/common.h" #include "include_internal/ten_runtime/binding/python/common/common.h" +#include "include_internal/ten_runtime/binding/python/common/error.h" #include "include_internal/ten_runtime/binding/python/ten_env/ten_env.h" #include "include_internal/ten_runtime/ten_env/ten_env.h" +#include "pystate.h" #include "ten_runtime/ten_env/internal/on_xxx_done.h" #include "ten_runtime/ten_env_proxy/ten_env_proxy.h" +#include "ten_utils/macro/check.h" #include "ten_utils/macro/mark.h" static void ten_env_proxy_notify_on_deinit_done(ten_env_t *ten_env, @@ -29,6 +32,21 @@ static void ten_env_proxy_notify_on_deinit_done(ten_env_t *ten_env, ten_py_ten_env_t *py_ten_env = user_data; TEN_ASSERT(py_ten_env, "Should not happen."); + // Notify the Python side to do the cleanup. + // + // About to call the Python function, so it's necessary to ensure that the + // GIL has been acquired. + PyGILState_STATE prev_state = ten_py_gil_state_ensure(); + + PyObject *py_res = + PyObject_CallMethod(py_ten_env->actual_py_ten_env, "_on_release", NULL); + Py_XDECREF(py_res); + + bool err_occurred = ten_py_check_and_clear_py_error(); + TEN_ASSERT(!err_occurred, "Should not happen."); + + ten_py_gil_state_release(prev_state); + if (py_ten_env->c_ten_env_proxy) { TEN_ASSERT( ten_env_proxy_get_thread_cnt(py_ten_env->c_ten_env_proxy, NULL) == 1, @@ -49,9 +67,11 @@ static void ten_env_proxy_notify_on_deinit_done(ten_env_t *ten_env, if (py_ten_env->need_to_release_gil_state) { if (!ten_py_is_holding_gil()) { + TEN_ASSERT(py_ten_env->py_thread_state != NULL, "Should not happen."); + // The gil is not held by the current thread, so we have to acquire the // gil before we can release the gil state. - ten_py_eval_restore_thread(NULL); + ten_py_eval_restore_thread(py_ten_env->py_thread_state); // Release the gil state and the gil. ten_py_gil_state_release(PyGILState_UNLOCKED); diff --git a/core/src/ten_runtime/build_template/preserved_metadata.c b/core/src/ten_runtime/build_template/preserved_metadata.c index 3d62db6827..dd516c5a58 100644 --- a/core/src/ten_runtime/build_template/preserved_metadata.c +++ b/core/src/ten_runtime/build_template/preserved_metadata.c @@ -6,7 +6,7 @@ // #include "include_internal/ten_runtime/common/preserved_metadata.h" -static char metadata[] = "version=0.2.0"; +static char metadata[] = "version=0.3.0-alpha"; void ten_preserved_metadata(void) { ((char volatile *)metadata)[0] = metadata[0]; diff --git a/core/src/ten_runtime/manifest.json b/core/src/ten_runtime/manifest.json index 960d375288..308147d2b5 100644 --- a/core/src/ten_runtime/manifest.json +++ b/core/src/ten_runtime/manifest.json @@ -1,7 +1,7 @@ { "type": "system", "name": "ten_runtime", - "version": "0.2.0", + "version": "0.3.0-alpha", "dependencies": [], "package": { "include": [ diff --git a/core/src/ten_rust/BUILD.gn b/core/src/ten_rust/BUILD.gn index bb7b6a6485..0f5ca1d1de 100644 --- a/core/src/ten_rust/BUILD.gn +++ b/core/src/ten_rust/BUILD.gn @@ -80,6 +80,10 @@ group("ten_rust_build_deps") { deps = [ "//core/src/ten_utils:ten_utils_combined_static" ] } +rust_gen_cargo_config("asan_config") { + project_root = "//" +} + group("ten_rust") { deps = [] public_deps = [] diff --git a/core/src/ten_rust/src/pkg_info/graph/check/connections_are_compatible.rs b/core/src/ten_rust/src/pkg_info/graph/check/connections_are_compatible.rs index b77f9d9fcc..f45dab89d0 100644 --- a/core/src/ten_rust/src/pkg_info/graph/check/connections_are_compatible.rs +++ b/core/src/ten_rust/src/pkg_info/graph/check/connections_are_compatible.rs @@ -30,7 +30,7 @@ impl Graph { .find_map(|node| { if node.node_type == PkgType::Extension && node.name.as_str() == extension - && node.app.as_str() == app + && node.get_app_uri() == app { Some(node.addon.as_str()) } else { @@ -56,12 +56,12 @@ impl Graph { let mut errors: Vec = Vec::new(); for dest in dests { let dest_addon = self.get_addon_name_of_extension( - dest.app.as_str(), + dest.get_app_uri(), dest.extension.as_str(), ); let dest_msg_schema = find_msg_schema_from_all_pkgs_info( all_needed_pkgs, - dest.app.as_str(), + dest.get_app_uri(), dest_addon, msg_name, msg_type, @@ -92,12 +92,12 @@ impl Graph { let mut errors: Vec = Vec::new(); for dest in dests { let dest_addon = self.get_addon_name_of_extension( - dest.app.as_str(), + dest.get_app_uri(), dest.extension.as_str(), ); let dest_cmd_schema = find_cmd_schema_from_all_pkgs_info( all_needed_pkgs, - dest.app.as_str(), + dest.get_app_uri(), dest_addon, cmd_name, MsgDirection::In, @@ -126,12 +126,12 @@ impl Graph { if let Some(cmd_flows) = &connection.cmd { for (flow_idx, flow) in cmd_flows.iter().enumerate() { let src_addon = self.get_addon_name_of_extension( - connection.app.as_str(), + connection.get_app_uri(), connection.extension.as_str(), ); let src_cmd_schema = find_cmd_schema_from_all_pkgs_info( all_needed_pkgs, - connection.app.as_str(), + connection.get_app_uri(), src_addon, flow.name.as_str(), MsgDirection::Out, @@ -143,7 +143,7 @@ impl Graph { src_cmd_schema, &flow.dest, ) { - errors.push(format!("- cmd[{}]: \n {}", flow_idx, e)); + errors.push(format!("- cmd[{}]: {}", flow_idx, e)); } } } @@ -151,12 +151,12 @@ impl Graph { if let Some(data_flows) = &connection.data { for (flow_idx, flow) in data_flows.iter().enumerate() { let src_addon = self.get_addon_name_of_extension( - connection.app.as_str(), + connection.get_app_uri(), connection.extension.as_str(), ); let src_msg_schema = find_msg_schema_from_all_pkgs_info( all_needed_pkgs, - connection.app.as_str(), + connection.get_app_uri(), src_addon, flow.name.as_str(), &MsgType::Data, @@ -170,7 +170,7 @@ impl Graph { src_msg_schema, &flow.dest, ) { - errors.push(format!("- data[{}]: \n {}", flow_idx, e)); + errors.push(format!("- data[{}]: {}", flow_idx, e)); } } } @@ -178,12 +178,12 @@ impl Graph { if let Some(video_frame_flows) = &connection.video_frame { for (flow_idx, flow) in video_frame_flows.iter().enumerate() { let src_addon = self.get_addon_name_of_extension( - connection.app.as_str(), + connection.get_app_uri(), connection.extension.as_str(), ); let src_msg_schema = find_msg_schema_from_all_pkgs_info( all_needed_pkgs, - connection.app.as_str(), + connection.get_app_uri(), src_addon, flow.name.as_str(), &MsgType::VideoFrame, @@ -197,10 +197,7 @@ impl Graph { src_msg_schema, &flow.dest, ) { - errors.push(format!( - "- video_frame[{}]: \n {}", - flow_idx, e - )); + errors.push(format!("- video_frame[{}]: {}", flow_idx, e)); } } } @@ -208,12 +205,12 @@ impl Graph { if let Some(audio_frame_flows) = &connection.audio_frame { for (flow_idx, flow) in audio_frame_flows.iter().enumerate() { let src_addon = self.get_addon_name_of_extension( - connection.app.as_str(), + connection.get_app_uri(), connection.extension.as_str(), ); let src_msg_schema = find_msg_schema_from_all_pkgs_info( all_needed_pkgs, - connection.app.as_str(), + connection.get_app_uri(), src_addon, flow.name.as_str(), &MsgType::AudioFrame, @@ -227,10 +224,7 @@ impl Graph { src_msg_schema, &flow.dest, ) { - errors.push(format!( - "- audio_frame[{}]: \n {}", - flow_idx, e - )); + errors.push(format!("- audio_frame[{}]: {}", flow_idx, e)); } } } diff --git a/core/src/ten_rust/src/pkg_info/graph/check/connections_nodes_are_defined.rs b/core/src/ten_rust/src/pkg_info/graph/check/connections_nodes_are_defined.rs index f968049fd8..96f6d2df2e 100644 --- a/core/src/ten_rust/src/pkg_info/graph/check/connections_nodes_are_defined.rs +++ b/core/src/ten_rust/src/pkg_info/graph/check/connections_nodes_are_defined.rs @@ -22,7 +22,9 @@ impl Graph { for dest in &flow.dest { let dest_extension = format!( "{}:{}:{}", - dest.app, dest.extension_group, dest.extension + dest.get_app_uri(), + dest.extension_group, + dest.extension ); if !all_extensions.contains(&dest_extension) { @@ -53,7 +55,7 @@ impl Graph { if node.node_type == PkgType::Extension { let unique_ext_name = format!( "{}:{}:{}", - node.app.as_str(), + node.get_app_uri(), node.extension_group.as_ref().unwrap(), node.name ); @@ -65,7 +67,7 @@ impl Graph { for (conn_idx, connection) in connections.iter().enumerate() { let src_extension = format!( "{}:{}:{}", - connection.app, + connection.get_app_uri(), connection.extension_group, connection.extension ); diff --git a/core/src/ten_rust/src/pkg_info/graph/check/duplicated_nodes.rs b/core/src/ten_rust/src/pkg_info/graph/check/duplicated_nodes.rs index 1c21d665ea..e71c9fd110 100644 --- a/core/src/ten_rust/src/pkg_info/graph/check/duplicated_nodes.rs +++ b/core/src/ten_rust/src/pkg_info/graph/check/duplicated_nodes.rs @@ -19,7 +19,7 @@ impl Graph { PkgType::Extension => { let unique_ext_name = format!( "{}:{}:{}", - node.app.as_str(), + node.get_app_uri(), node.extension_group.as_ref().unwrap(), node.name ); @@ -35,7 +35,7 @@ impl Graph { let ext_group = format!( "{}:{}", - node.app.as_str(), + node.get_app_uri(), node.extension_group.as_ref().unwrap() ); @@ -46,7 +46,7 @@ impl Graph { PkgType::ExtensionGroup => { let unique_ext_group_name = - format!("{}:{}", node.app.as_str(), node.name); + format!("{}:{}", node.get_app_uri(), node.name); if all_extension_groups.contains(&unique_ext_group_name) { return Err(anyhow::anyhow!( diff --git a/core/src/ten_rust/src/pkg_info/graph/check/nodes_are_installed.rs b/core/src/ten_rust/src/pkg_info/graph/check/nodes_are_installed.rs index 7c130cb675..0704df053c 100644 --- a/core/src/ten_rust/src/pkg_info/graph/check/nodes_are_installed.rs +++ b/core/src/ten_rust/src/pkg_info/graph/check/nodes_are_installed.rs @@ -19,15 +19,16 @@ impl Graph { let mut not_installed_pkgs: Vec<(String, PkgType, String)> = Vec::new(); for node in &self.nodes { - if !all_needed_pkgs.contains_key(node.app.as_str()) { + let node_app = node.get_app_uri(); + if !all_needed_pkgs.contains_key(node_app) { not_installed_pkgs.push(( - node.app.clone(), + node_app.to_string(), node.node_type.clone(), node.addon.clone(), )); } - let pkgs_in_app = all_needed_pkgs.get(node.app.as_str()).unwrap(); + let pkgs_in_app = all_needed_pkgs.get(node_app).unwrap(); let found = pkgs_in_app.iter().find(|pkg| { pkg.pkg_identity.pkg_type == node.node_type && pkg.pkg_identity.name == node.addon @@ -35,7 +36,7 @@ impl Graph { }); if found.is_none() { not_installed_pkgs.push(( - node.app.clone(), + node_app.to_string(), node.node_type.clone(), node.addon.clone(), )); diff --git a/core/src/ten_rust/src/pkg_info/graph/mod.rs b/core/src/ten_rust/src/pkg_info/graph/mod.rs index 7e796852a1..29a6fba23f 100644 --- a/core/src/ten_rust/src/pkg_info/graph/mod.rs +++ b/core/src/ten_rust/src/pkg_info/graph/mod.rs @@ -38,16 +38,16 @@ impl FromStr for Graph { impl Graph { pub fn validate_and_complete(&mut self) -> Result<()> { - for node in &mut self.nodes { - node.validate_and_complete()?; + for (idx, node) in &mut self.nodes.iter_mut().enumerate() { + node.validate_and_complete() + .map_err(|e| anyhow::anyhow!("nodes[{}]: {}", idx, e))?; } - for (node_idx, node) in self.nodes.iter().enumerate() { - if node.app.is_empty() { - return Err(anyhow::anyhow!( - "'app' field is missing in nodes[{}].", - node_idx - )); + if let Some(connections) = &mut self.connections { + for (idx, connection) in connections.iter_mut().enumerate() { + connection.validate_and_complete().map_err(|e| { + anyhow::anyhow!("connections[{}].{}", idx, e) + })?; } } @@ -78,9 +78,8 @@ pub struct GraphNode { #[serde(skip_serializing_if = "Option::is_none")] pub extension_group: Option, - // Default is 'localhost'. - #[serde(default = "default_app_loc")] - pub app: String, + #[serde(skip_serializing_if = "is_app_default_loc_or_none")] + pub app: Option, #[serde(skip_serializing_if = "Option::is_none")] pub property: Option, @@ -98,15 +97,30 @@ impl GraphNode { )); } + if let Some(app) = &self.app { + if app.as_str() == default_app_loc() { + return Err(anyhow::anyhow!( + "the app uri should be some string other than 'localhost'" + )); + } + } else { + self.app = Some(default_app_loc().to_string()); + } + Ok(()) } + + pub fn get_app_uri(&self) -> &str { + // The 'app' should be assigned after 'validate_and_complete' is called, + // so it should not be None. + self.app.as_ref().unwrap().as_str() + } } #[derive(Serialize, Deserialize, Debug, Clone)] pub struct GraphConnection { - // If a connection does not specify an app URI, it defaults to localhost. - #[serde(default = "default_app_loc")] - pub app: String, + #[serde(skip_serializing_if = "is_app_default_loc_or_none")] + pub app: Option, pub extension_group: String, pub extension: String, @@ -121,22 +135,108 @@ pub struct GraphConnection { pub video_frame: Option>, } +impl GraphConnection { + fn validate_and_complete(&mut self) -> Result<()> { + if let Some(app) = &self.app { + if app.as_str() == default_app_loc() { + return Err(anyhow::anyhow!( + "the app uri should be some string other than 'localhost'" + )); + } + } else { + self.app = Some(default_app_loc().to_string()); + } + + if let Some(cmd) = &mut self.cmd { + for (idx, cmd_flow) in cmd.iter_mut().enumerate() { + cmd_flow + .validate_and_complete() + .map_err(|e| anyhow::anyhow!("cmd[{}].{}", idx, e))?; + } + } + + if let Some(data) = &mut self.data { + for (idx, data_flow) in data.iter_mut().enumerate() { + data_flow + .validate_and_complete() + .map_err(|e| anyhow::anyhow!("data[{}].{}", idx, e))?; + } + } + + if let Some(audio_frame) = &mut self.audio_frame { + for (idx, audio_flow) in audio_frame.iter_mut().enumerate() { + audio_flow.validate_and_complete().map_err(|e| { + anyhow::anyhow!("audio_frame[{}].{}", idx, e) + })?; + } + } + + if let Some(video_frame) = &mut self.video_frame { + for (idx, video_flow) in video_frame.iter_mut().enumerate() { + video_flow.validate_and_complete().map_err(|e| { + anyhow::anyhow!("video_frame[{}].{}", idx, e) + })?; + } + } + + Ok(()) + } + + pub fn get_app_uri(&self) -> &str { + self.app.as_ref().unwrap().as_str() + } +} + #[derive(Serialize, Deserialize, Debug, Clone)] pub struct GraphMessageFlow { pub name: String, pub dest: Vec, } +impl GraphMessageFlow { + fn validate_and_complete(&mut self) -> Result<()> { + for (idx, dest) in &mut self.dest.iter_mut().enumerate() { + dest.validate_and_complete() + .map_err(|e| anyhow::anyhow!("dest[{}]: {}", idx, e))?; + } + + Ok(()) + } +} + #[derive(Serialize, Deserialize, Debug, Clone)] pub struct GraphDestination { - // If a destination does not specify an app URI, it defaults to localhost. - #[serde(default = "default_app_loc")] - pub app: String, + #[serde(skip_serializing_if = "is_app_default_loc_or_none")] + pub app: Option, pub extension_group: String, pub extension: String, } +impl GraphDestination { + fn validate_and_complete(&mut self) -> Result<()> { + if let Some(app) = &self.app { + if app.as_str() == default_app_loc() { + return Err(anyhow::anyhow!( + "the app uri should be some string other than 'localhost'" + )); + } + } else { + self.app = Some(default_app_loc().to_string()); + } + + Ok(()) + } + + pub fn get_app_uri(&self) -> &str { + self.app.as_ref().unwrap().as_str() + } +} + +pub fn is_app_default_loc_or_none(app: &Option) -> bool { + app.is_none() || app.as_ref().unwrap().as_str() == default_app_loc() +} + #[cfg(test)] mod tests { use std::str::FromStr; @@ -149,9 +249,7 @@ mod tests { fn test_predefined_graph_has_no_extensions() { let property_str = include_str!("test_data_embed/predefined_graph_no_extensions.json"); - let mut property: Property = Property::from_str(property_str).unwrap(); - assert!(property.validate_and_complete().is_ok()); - + let property: Property = Property::from_str(property_str).unwrap(); let ten = property._ten.as_ref().unwrap(); let predefined_graph = ten.predefined_graphs.as_ref().unwrap().first().unwrap(); @@ -166,9 +264,7 @@ mod tests { let property_str = include_str!( "test_data_embed/predefined_graph_has_duplicated_extension.json" ); - let mut property: Property = Property::from_str(property_str).unwrap(); - assert!(property.validate_and_complete().is_ok()); - + let property: Property = Property::from_str(property_str).unwrap(); let ten = property._ten.as_ref().unwrap(); let predefined_graph = ten.predefined_graphs.as_ref().unwrap().first().unwrap(); @@ -184,9 +280,7 @@ mod tests { "test_data_embed/start_graph_cmd_has_duplicated_extension.json" ); - let mut graph: Graph = Graph::from_str(cmd_str).unwrap(); - assert!(graph.validate_and_complete().is_ok()); - + let graph: Graph = Graph::from_str(cmd_str).unwrap(); let result = graph.check_if_nodes_duplicated(); assert!(result.is_err()); println!("Error: {:?}", result.err().unwrap()); @@ -197,9 +291,7 @@ mod tests { let property_str = include_str!( "test_data_embed/predefined_graph_has_duplicated_ext_group.json" ); - let mut property: Property = Property::from_str(property_str).unwrap(); - assert!(property.validate_and_complete().is_ok()); - + let property: Property = Property::from_str(property_str).unwrap(); let ten = property._ten.as_ref().unwrap(); let predefined_graph = ten.predefined_graphs.as_ref().unwrap().first().unwrap(); @@ -215,9 +307,7 @@ mod tests { let property_str = include_str!( "test_data_embed/predefined_graph_unused_ext_group.json" ); - let mut property: Property = Property::from_str(property_str).unwrap(); - assert!(property.validate_and_complete().is_ok()); - + let property: Property = Property::from_str(property_str).unwrap(); let ten = property._ten.as_ref().unwrap(); let predefined_graph = ten.predefined_graphs.as_ref().unwrap().first().unwrap(); @@ -233,9 +323,7 @@ mod tests { let property_str = include_str!( "test_data_embed/predefined_graph_connection_src_not_found.json" ); - let mut property: Property = Property::from_str(property_str).unwrap(); - assert!(property.validate_and_complete().is_ok()); - + let property: Property = Property::from_str(property_str).unwrap(); let ten = property._ten.as_ref().unwrap(); let predefined_graph = ten.predefined_graphs.as_ref().unwrap().first().unwrap(); @@ -252,9 +340,7 @@ mod tests { let property_str = include_str!( "test_data_embed/predefined_graph_connection_dest_not_found.json" ); - let mut property: Property = Property::from_str(property_str).unwrap(); - assert!(property.validate_and_complete().is_ok()); - + let property: Property = Property::from_str(property_str).unwrap(); let ten = property._ten.as_ref().unwrap(); let predefined_graph = ten.predefined_graphs.as_ref().unwrap().first().unwrap(); @@ -265,4 +351,14 @@ mod tests { assert!(result.is_err()); println!("Error: {:?}", result.err().unwrap()); } + + #[test] + fn test_predefined_graph_connection_app_localhost() { + let property_str = include_str!( + "test_data_embed/predefined_graph_connection_app_localhost.json" + ); + let property = Property::from_str(property_str); + assert!(property.is_err()); + println!("Error: {:?}", property.err().unwrap()); + } } diff --git a/core/src/ten_rust/src/pkg_info/graph/test_data_embed/predefined_graph_connection_app_localhost.json b/core/src/ten_rust/src/pkg_info/graph/test_data_embed/predefined_graph_connection_app_localhost.json new file mode 100644 index 0000000000..7871bbaa61 --- /dev/null +++ b/core/src/ten_rust/src/pkg_info/graph/test_data_embed/predefined_graph_connection_app_localhost.json @@ -0,0 +1,41 @@ +{ + "_ten": { + "predefined_graphs": [ + { + "name": "default", + "auto_start": false, + "nodes": [ + { + "type": "extension", + "name": "some_extension", + "addon": "default_extension_go", + "extension_group": "some_group" + }, + { + "type": "extension_group", + "addon": "default_extension_group", + "name": "some_group" + } + ], + "connections": [ + { + "extension": "some_extension", + "extension_group": "producer", + "cmd": [ + { + "name": "hello", + "dest": [ + { + "extension_group": "some_group", + "extension": "some_extension", + "app": "localhost" + } + ] + } + ] + } + ] + } + ] + } +} diff --git a/core/src/ten_rust/src/pkg_info/graph/test_data_embed/predefined_graph_connection_dest_not_found.json b/core/src/ten_rust/src/pkg_info/graph/test_data_embed/predefined_graph_connection_dest_not_found.json index 3e9e467839..cc818f637b 100644 --- a/core/src/ten_rust/src/pkg_info/graph/test_data_embed/predefined_graph_connection_dest_not_found.json +++ b/core/src/ten_rust/src/pkg_info/graph/test_data_embed/predefined_graph_connection_dest_not_found.json @@ -25,7 +25,6 @@ ], "connections": [ { - "app": "localhost", "extension": "some_extension", "extension_group": "some_group", "cmd": [ @@ -33,7 +32,6 @@ "name": "hello", "dest": [ { - "app": "localhost", "extension_group": "some_group", "extension": "some_extension_1" } @@ -43,12 +41,10 @@ "name": "world", "dest": [ { - "app": "localhost", "extension_group": "some_group", "extension": "some_extension_1" }, { - "app": "localhost", "extension_group": "some_group", "extension": "consumer" } @@ -58,7 +54,6 @@ } ] } - ], - "uri": "http://localhost:8001" + ] } -} \ No newline at end of file +} diff --git a/core/src/ten_rust/src/pkg_info/graph/test_data_embed/predefined_graph_connection_src_not_found.json b/core/src/ten_rust/src/pkg_info/graph/test_data_embed/predefined_graph_connection_src_not_found.json index 2ff0e04be0..23bfcf3d3b 100644 --- a/core/src/ten_rust/src/pkg_info/graph/test_data_embed/predefined_graph_connection_src_not_found.json +++ b/core/src/ten_rust/src/pkg_info/graph/test_data_embed/predefined_graph_connection_src_not_found.json @@ -19,7 +19,6 @@ ], "connections": [ { - "app": "localhost", "extension": "some_extension", "extension_group": "producer", "cmd": [ @@ -27,7 +26,6 @@ "name": "hello", "dest": [ { - "app": "localhost", "extension_group": "some_group", "extension": "some_extension" } @@ -37,7 +35,6 @@ } ] } - ], - "uri": "http://localhost:8001" + ] } -} \ No newline at end of file +} diff --git a/core/src/ten_rust/src/pkg_info/graph/test_data_embed/predefined_graph_has_duplicated_ext_group.json b/core/src/ten_rust/src/pkg_info/graph/test_data_embed/predefined_graph_has_duplicated_ext_group.json index fea0d29ecd..9829865b63 100644 --- a/core/src/ten_rust/src/pkg_info/graph/test_data_embed/predefined_graph_has_duplicated_ext_group.json +++ b/core/src/ten_rust/src/pkg_info/graph/test_data_embed/predefined_graph_has_duplicated_ext_group.json @@ -23,7 +23,6 @@ } ] } - ], - "uri": "http://localhost:8001" + ] } -} \ No newline at end of file +} diff --git a/core/src/ten_rust/src/pkg_info/graph/test_data_embed/predefined_graph_has_duplicated_extension.json b/core/src/ten_rust/src/pkg_info/graph/test_data_embed/predefined_graph_has_duplicated_extension.json index 096b8259f1..1b33b890fa 100644 --- a/core/src/ten_rust/src/pkg_info/graph/test_data_embed/predefined_graph_has_duplicated_extension.json +++ b/core/src/ten_rust/src/pkg_info/graph/test_data_embed/predefined_graph_has_duplicated_extension.json @@ -19,7 +19,6 @@ } ] } - ], - "uri": "http://localhost:8001" + ] } -} \ No newline at end of file +} diff --git a/core/src/ten_rust/src/pkg_info/graph/test_data_embed/predefined_graph_no_extensions.json b/core/src/ten_rust/src/pkg_info/graph/test_data_embed/predefined_graph_no_extensions.json index d568e6c089..b9ff5282aa 100644 --- a/core/src/ten_rust/src/pkg_info/graph/test_data_embed/predefined_graph_no_extensions.json +++ b/core/src/ten_rust/src/pkg_info/graph/test_data_embed/predefined_graph_no_extensions.json @@ -12,7 +12,6 @@ } ] } - ], - "uri": "http://localhost:8001" + ] } -} \ No newline at end of file +} diff --git a/core/src/ten_rust/src/pkg_info/graph/test_data_embed/predefined_graph_unused_ext_group.json b/core/src/ten_rust/src/pkg_info/graph/test_data_embed/predefined_graph_unused_ext_group.json index 18d99e5a39..188f7e7aca 100644 --- a/core/src/ten_rust/src/pkg_info/graph/test_data_embed/predefined_graph_unused_ext_group.json +++ b/core/src/ten_rust/src/pkg_info/graph/test_data_embed/predefined_graph_unused_ext_group.json @@ -23,7 +23,6 @@ } ] } - ], - "uri": "http://localhost:8001" + ] } -} \ No newline at end of file +} diff --git a/core/src/ten_rust/src/pkg_info/predefined_graphs/extension.rs b/core/src/ten_rust/src/pkg_info/predefined_graphs/extension.rs index 379c1111cb..d8bcd29013 100644 --- a/core/src/ten_rust/src/pkg_info/predefined_graphs/extension.rs +++ b/core/src/ten_rust/src/pkg_info/predefined_graphs/extension.rs @@ -90,7 +90,7 @@ pub fn get_extension<'a>( .iter() .find(|ext| { ext.node_type == PkgType::Extension - && &ext.app == app + && ext.get_app_uri() == app && ext.extension_group.clone().unwrap_or("".to_string()) == *extension_group && &ext.name == extension diff --git a/core/src/ten_rust/src/pkg_info/property/mod.rs b/core/src/ten_rust/src/pkg_info/property/mod.rs index 8d16fd395a..22b1b4ddf9 100644 --- a/core/src/ten_rust/src/pkg_info/property/mod.rs +++ b/core/src/ten_rust/src/pkg_info/property/mod.rs @@ -21,6 +21,7 @@ use super::{ constants::{PROPERTY_JSON_FILENAME, TEN_FIELD_IN_PROPERTY}, utils::read_file_to_string, }; +use crate::pkg_info::graph::is_app_default_loc_or_none; use crate::{json_schema, pkg_info::default_app_loc}; use predefined_graph::PropertyPredefinedGraph; @@ -74,6 +75,16 @@ impl Property { Ok(()) } + + pub fn get_app_uri(&self) -> String { + if let Some(_ten) = &self._ten { + if let Some(uri) = &_ten.uri { + return uri.clone(); + } + } + + default_app_loc() + } } #[derive(Serialize, Deserialize, Debug, Clone)] @@ -81,7 +92,7 @@ pub struct TenInProperty { #[serde(skip_serializing_if = "Option::is_none")] pub predefined_graphs: Option>, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(skip_serializing_if = "is_app_default_loc_or_none")] pub uri: Option, #[serde(flatten)] @@ -272,4 +283,24 @@ mod tests { assert_eq!(saved_json, original_json); } + + #[test] + fn test_dump_property_without_localhost_app_in_graph() { + let json_data = include_str!("test_data_embed/property.json"); + let property: Property = json_data.parse().unwrap(); + assert!(property._ten.is_some()); + let ten = property._ten.as_ref().unwrap(); + let predefined_graphs = ten.predefined_graphs.as_ref().unwrap(); + let nodes = &predefined_graphs.first().as_ref().unwrap().graph.nodes; + let node = nodes.first().unwrap(); + assert_eq!(node.get_app_uri(), default_app_loc()); + + let dir = tempdir().unwrap(); + let file_path = dir.path().join("property.json"); + property.dump_property_to_file(&file_path).unwrap(); + + let saved_content = fs::read_to_string(file_path).unwrap(); + eprintln!("{}", saved_content); + assert_eq!(saved_content.find(default_app_loc().as_str()), None); + } } diff --git a/core/src/ten_rust/src/pkg_info/property/predefined_graph.rs b/core/src/ten_rust/src/pkg_info/property/predefined_graph.rs index c1bcc82902..f5dd9a63c4 100644 --- a/core/src/ten_rust/src/pkg_info/property/predefined_graph.rs +++ b/core/src/ten_rust/src/pkg_info/property/predefined_graph.rs @@ -47,9 +47,7 @@ mod tests { .unwrap(); assert!(predefined_graphs.len() == 1); - let mut predefined_graph = predefined_graphs.first().unwrap().clone(); - assert!(predefined_graph.validate_and_complete().is_ok()); - + let predefined_graph = predefined_graphs.first().unwrap().clone(); let property_json_value = serde_json::to_value(&predefined_graph).unwrap(); assert_eq!( diff --git a/core/src/ten_utils/BUILD.gn b/core/src/ten_utils/BUILD.gn index bc988d341f..8f2f9c2afc 100644 --- a/core/src/ten_utils/BUILD.gn +++ b/core/src/ten_utils/BUILD.gn @@ -127,5 +127,8 @@ combine_static_library("ten_utils_combined_static") { libuv_static[0], ] - deps = [ ":ten_utils_static" ] + deps = [ + ":ten_utils_static", + "//core/src/ten_rust:asan_config", + ] } diff --git a/core/src/ten_utils/lib/sys/posix/mutex.c b/core/src/ten_utils/lib/sys/posix/mutex.c index 45936e1ba0..783f93e0d8 100644 --- a/core/src/ten_utils/lib/sys/posix/mutex.c +++ b/core/src/ten_utils/lib/sys/posix/mutex.c @@ -64,11 +64,13 @@ ten_mutex_t *ten_mutex_create(void) { } int ten_mutex_lock(ten_mutex_t *mutex) { - TEN_ASSERT(mutex && ten_mutex_check_integrity(mutex), "Invalid argument."); + TEN_ASSERT(mutex, "Invalid argument."); if (!mutex) { return -1; } + TEN_ASSERT(ten_mutex_check_integrity(mutex), "Invalid argument."); + int rc = pthread_mutex_lock(&mutex->mutex); if (rc) { TEN_ASSERT(0, "Should not happen: %d", rc); diff --git a/core/src/ten_utils/sanitizer/memory_check.c b/core/src/ten_utils/sanitizer/memory_check.c index e3c689eb6b..bf90854d8b 100644 --- a/core/src/ten_utils/sanitizer/memory_check.c +++ b/core/src/ten_utils/sanitizer/memory_check.c @@ -52,6 +52,8 @@ static void ten_sanitizer_memory_record_check_enabled(void) { } void ten_sanitizer_memory_record_init(void) { +#if defined(TEN_ENABLE_MEMORY_CHECK) + #if defined(TEN_USE_ASAN) __lsan_disable(); #endif @@ -73,9 +75,15 @@ void ten_sanitizer_memory_record_init(void) { #if defined(TEN_USE_ASAN) __lsan_enable(); #endif + +#else + TEN_LOGI("The memory check is disabled."); +#endif } void ten_sanitizer_memory_record_deinit(void) { +#if defined(TEN_ENABLE_MEMORY_CHECK) + #if defined(TEN_USE_ASAN) __lsan_disable(); #endif @@ -89,6 +97,10 @@ void ten_sanitizer_memory_record_deinit(void) { #if defined(TEN_USE_ASAN) __lsan_enable(); #endif + +#else + TEN_LOGI("The memory check is disabled."); +#endif } static ten_sanitizer_memory_record_t *ten_sanitizer_memory_record_create( diff --git a/core/ten_gn b/core/ten_gn index 258196d871..16810350fc 160000 --- a/core/ten_gn +++ b/core/ten_gn @@ -1 +1 @@ -Subproject commit 258196d871c6cf8665b93b735d812be3662a528c +Subproject commit 16810350fc78948a255ba0bb45d3bc338341a029 diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index 300550651e..471ffea820 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -51,6 +51,7 @@ * [Overview](ten_framework/ten_packages/overview.md) * [App](ten_framework/ten_packages/app.md) * [Extension](ten_framework/ten_packages/extension.md) +* [Python Async Extension](ten_framework/ten_packages/python_async_extension.md) ### TEN Manager @@ -60,3 +61,4 @@ ## Tutorials * [How to debug with logs](tutorials/how_to_debug_with_logs.md) +* [How to run local AI model in Python extension](tutorials/how_to_run_local_model_in_python_extensions.md) diff --git a/docs/ten_agent/getting_started.md b/docs/ten_agent/getting_started.md index 35e2463852..c870d9b110 100644 --- a/docs/ten_agent/getting_started.md +++ b/docs/ten_agent/getting_started.md @@ -106,7 +106,7 @@ Open up a separate terminal window, build the agent and start the server: {% code title=">_ Bash" %} ```bash -docker exec -it astra_agents_dev bash +docker exec -it ten_agent_dev bash make build ``` @@ -126,7 +126,7 @@ make run-server ## **Finish and verify your agent** -You can open [https://localhost:3001](https://localhost:3001/) in browser to use your graph designer. Simultaneously, open another tab at [https://localhost:3000](https://localhost:3000/) to see the customized voice agent up and running. +You can open [localhost:3001](https://localhost:3001/) in browser to use your graph designer. Simultaneously, open another tab at [localhost:3000](https://localhost:3000/) to see the customized voice agent up and running. Now you have the power of the Graph Designer at your fingertips to perform the magic of agent customization yourself. 🎉 diff --git a/docs/ten_framework/ten_packages/python_async_extension.md b/docs/ten_framework/ten_packages/python_async_extension.md new file mode 100644 index 0000000000..28f25db26c --- /dev/null +++ b/docs/ten_framework/ten_packages/python_async_extension.md @@ -0,0 +1,56 @@ +# Python Async Extension + +A **Python async extension** is a modular component that using Python's `asyncio` framework. If you want to wrap existing Python codes that use `asyncio` into a TEN extension, using the Python async extension is the simplest and most convenient option. + +## Example: The Default Python Async Extension + +Here’s how a Python async extension is structured: + +```python +import asyncio +from ten import AsyncExtension, AsyncTenEnv + +class DefaultAsyncExtension(AsyncExtension): + async def on_configure(self, ten_env: AsyncTenEnv) -> None: + # Mock async operation, e.g. network, file I/O. + await asyncio.sleep(0.5) + + ten_env.on_configure_done() + + async def on_init(self, ten_env: AsyncTenEnv) -> None: + # Mock async operation, e.g. network, file I/O. + await asyncio.sleep(0.5) + + ten_env.on_init_done() + + async def on_start(self, ten_env: AsyncTenEnv) -> None: + # Mock async operation, e.g. network, file I/O. + await asyncio.sleep(0.5) + + ten_env.on_start_done() + + async def on_deinit(self, ten_env: AsyncTenEnv) -> None: + # Mock async operation, e.g. network, file I/O. + await asyncio.sleep(0.5) + + ten_env.on_deinit_done() + + async def on_cmd(self, ten_env: AsyncTenEnv, cmd: Cmd) -> None: + cmd_json = cmd.to_json() + ten_env.log_debug(f"DefaultAsyncExtension on_cmd: {cmd_json}") + + # Mock async operation, e.g. network, file I/O. + await asyncio.sleep(0.5) + + # Send a new command to other extensions and wait for the result. The + # result will be returned to the original sender. + new_cmd = Cmd.create("hello") + cmd_result = await ten_env.send_cmd(new_cmd) + ten_env.return_result(cmd_result, cmd) +``` + +Each method simulates a delay using `await asyncio.sleep()`. + +## Conclusion + +TEN's Python async extension provide a powerful way to handle long-running tasks asynchronously. By integrating Python’s `asyncio` framework, the extensions ensure that operations such as network calls or file handling are efficient and non-blocking. This makes TEN a great choice for building scalable, modular applications with asynchronous capabilities. diff --git a/packages/core_apps/default_app_cpp/manifest.json b/packages/core_apps/default_app_cpp/manifest.json index 1446e304fe..03a895779f 100644 --- a/packages/core_apps/default_app_cpp/manifest.json +++ b/packages/core_apps/default_app_cpp/manifest.json @@ -1,12 +1,12 @@ { "type": "app", "name": "default_app_cpp", - "version": "0.1.0", + "version": "0.3.0-alpha", "dependencies": [ { "type": "system", "name": "ten_runtime", - "version": "0.2.0" + "version": "0.3.0-alpha" } ] } \ No newline at end of file diff --git a/packages/core_apps/default_app_cpp/manifest.json.tent b/packages/core_apps/default_app_cpp/manifest.json.tent index 34b4e14716..13d2cb8acd 100644 --- a/packages/core_apps/default_app_cpp/manifest.json.tent +++ b/packages/core_apps/default_app_cpp/manifest.json.tent @@ -1,12 +1,12 @@ { "type": "app", "name": "{{package_name}}", - "version": "0.1.0", + "version": "0.3.0-alpha", "dependencies": [ { "type": "system", "name": "ten_runtime", - "version": "0.2.0" + "version": "0.3.0-alpha" } ] } \ No newline at end of file diff --git a/packages/core_apps/default_app_go/manifest.json b/packages/core_apps/default_app_go/manifest.json index 03ea6710eb..bcece2de52 100644 --- a/packages/core_apps/default_app_go/manifest.json +++ b/packages/core_apps/default_app_go/manifest.json @@ -1,17 +1,17 @@ { "type": "app", "name": "default_app_go", - "version": "0.1.0", + "version": "0.3.0-alpha", "dependencies": [ { "type": "system", "name": "ten_runtime", - "version": "0.2.0" + "version": "0.3.0-alpha" }, { "type": "system", "name": "ten_runtime_go", - "version": "0.1.0" + "version": "0.3.0-alpha" } ] } \ No newline at end of file diff --git a/packages/core_apps/default_app_go/manifest.json.tent b/packages/core_apps/default_app_go/manifest.json.tent index 7dacd887bb..a2ac5bbcf6 100644 --- a/packages/core_apps/default_app_go/manifest.json.tent +++ b/packages/core_apps/default_app_go/manifest.json.tent @@ -1,16 +1,17 @@ { "type": "app", "name": "{{package_name}}", - "version": "0.1.0", + "version": "0.3.0-alpha", "dependencies": [ { "type": "system", "name": "ten_runtime", - "version": "0.2.0" + "version": "0.3.0-alpha" }, { "type": "system", "name": "ten_runtime_go", - "version": "0.1.0" + "version": "0.3.0-alpha" } - ]} \ No newline at end of file + ] +} \ No newline at end of file diff --git a/packages/core_apps/default_app_python/manifest.json b/packages/core_apps/default_app_python/manifest.json index fe8bdbc968..bea6ced7b7 100644 --- a/packages/core_apps/default_app_python/manifest.json +++ b/packages/core_apps/default_app_python/manifest.json @@ -1,17 +1,17 @@ { "type": "app", "name": "default_app_python", - "version": "0.1.0", + "version": "0.3.0-alpha", "dependencies": [ { "type": "system", "name": "ten_runtime", - "version": "0.2.0" + "version": "0.3.0-alpha" }, { "type": "system", "name": "ten_runtime_python", - "version": "0.1.0" + "version": "0.3.0-alpha" } ] } \ No newline at end of file diff --git a/packages/core_apps/default_app_python/manifest.json.tent b/packages/core_apps/default_app_python/manifest.json.tent index f9cbc4e2ba..6565407bba 100644 --- a/packages/core_apps/default_app_python/manifest.json.tent +++ b/packages/core_apps/default_app_python/manifest.json.tent @@ -1,16 +1,17 @@ { "type": "app", "name": "{{package_name}}", - "version": "0.1.0", + "version": "0.3.0-alpha", "dependencies": [ { "type": "system", "name": "ten_runtime", - "version": "0.2.0" + "version": "0.3.0-alpha" }, { "type": "system", "name": "ten_runtime_python", - "version": "0.1.0" + "version": "0.3.0-alpha" } - ]} \ No newline at end of file + ] +} \ No newline at end of file diff --git a/packages/core_extensions/BUILD.gn b/packages/core_extensions/BUILD.gn index 7b14c973d0..66e139aea5 100644 --- a/packages/core_extensions/BUILD.gn +++ b/packages/core_extensions/BUILD.gn @@ -14,6 +14,7 @@ group("core_extensions") { if (ten_enable_python_binding) { deps += [ + "default_async_extension_python", "default_extension_python", "py_init_extension_cpp", ] diff --git a/packages/core_extensions/default_async_extension_python/BUILD.gn b/packages/core_extensions/default_async_extension_python/BUILD.gn new file mode 100644 index 0000000000..a762096779 --- /dev/null +++ b/packages/core_extensions/default_async_extension_python/BUILD.gn @@ -0,0 +1,42 @@ +# +# This file is part of TEN Framework, an open source project. +# Licensed under the Apache License, Version 2.0. +# See the LICENSE file for more information. +# +import("//build/feature/ten_package.gni") +import("//build/ten_runtime/feature/publish.gni") +import("//build/ten_runtime/glob.gni") +import("//build/ten_runtime/options.gni") + +ten_package("default_async_extension_python") { + package_kind = "extension" + + resources = [ + "BUILD_release.gn.tent=>BUILD.gn.tent", + "BUILD_release.gn=>BUILD.gn", + "README.md", + "README.md.tent", + "__init__.py", + "__init__.py.tent", + "addon.py", + "addon.py.tent", + "extension.py", + "extension.py.tent", + "log.py", + "log.py.tent", + "manifest.json", + "manifest.json.tent", + "property.json", + "tests", + ] + + deps = [ "//core/src/ten_runtime" ] +} + +if (ten_enable_package_manager) { + ten_package_publish("upload_default_async_extension_python_to_server") { + base_dir = rebase_path( + "${root_out_dir}/ten_packages/extension/default_async_extension_python") + deps = [ ":default_async_extension_python" ] + } +} diff --git a/packages/core_extensions/default_async_extension_python/BUILD_release.gn b/packages/core_extensions/default_async_extension_python/BUILD_release.gn new file mode 100644 index 0000000000..5053ef7095 --- /dev/null +++ b/packages/core_extensions/default_async_extension_python/BUILD_release.gn @@ -0,0 +1,20 @@ +# +# This file is part of TEN Framework, an open source project. +# Licensed under the Apache License, Version 2.0. +# See the LICENSE file for more information. +# +import("//build/feature/ten_package.gni") + +ten_package("default_async_extension_python") { + package_kind = "extension" + + resources = [ + "__init__.py", + "addon.py", + "extension.py", + "log.py", + "manifest.json", + "property.json", + "tests", + ] +} diff --git a/packages/core_extensions/default_async_extension_python/BUILD_release.gn.tent b/packages/core_extensions/default_async_extension_python/BUILD_release.gn.tent new file mode 100644 index 0000000000..186badc385 --- /dev/null +++ b/packages/core_extensions/default_async_extension_python/BUILD_release.gn.tent @@ -0,0 +1,20 @@ +# +# This file is part of TEN Framework, an open source project. +# Licensed under the Apache License, Version 2.0. +# See the LICENSE file for more information. +# +import("//build/feature/ten_package.gni") + +ten_package("{{package_name}}") { + package_kind = "extension" + + resources = [ + "__init__.py", + "addon.py", + "extension.py", + "log.py", + "manifest.json", + "property.json", + "tests", + ] +} diff --git a/packages/core_extensions/default_async_extension_python/LICENSE b/packages/core_extensions/default_async_extension_python/LICENSE new file mode 100644 index 0000000000..e37872ecc6 --- /dev/null +++ b/packages/core_extensions/default_async_extension_python/LICENSE @@ -0,0 +1,13 @@ +Copyright © 2024 Agora + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/packages/core_extensions/default_async_extension_python/README.md b/packages/core_extensions/default_async_extension_python/README.md new file mode 100644 index 0000000000..2f995d526a --- /dev/null +++ b/packages/core_extensions/default_async_extension_python/README.md @@ -0,0 +1,29 @@ +# default_extension_python + + + +## Features + + + +- xxx feature + +## API + +Refer to `api` definition in [manifest.json] and default values in [property.json](property.json). + + + +## Development + +### Build + + + +### Unit test + + + +## Misc + + diff --git a/packages/core_extensions/default_async_extension_python/README.md.tent b/packages/core_extensions/default_async_extension_python/README.md.tent new file mode 100644 index 0000000000..a327c5694c --- /dev/null +++ b/packages/core_extensions/default_async_extension_python/README.md.tent @@ -0,0 +1,29 @@ +# {{package_name}} + + + +## Features + + + +- xxx feature + +## API + +Refer to `api` definition in [manifest.json] and default values in [property.json](property.json). + + + +## Development + +### Build + + + +### Unit test + + + +## Misc + + diff --git a/packages/core_extensions/default_async_extension_python/__init__.py b/packages/core_extensions/default_async_extension_python/__init__.py new file mode 100644 index 0000000000..2cc80a0ced --- /dev/null +++ b/packages/core_extensions/default_async_extension_python/__init__.py @@ -0,0 +1,9 @@ +# +# This file is part of TEN Framework, an open source project. +# Licensed under the Apache License, Version 2.0. +# See the LICENSE file for more information. +# +from . import addon +from .log import logger + +logger.info("default_async_extension_python extension loaded") diff --git a/packages/core_extensions/default_async_extension_python/__init__.py.tent b/packages/core_extensions/default_async_extension_python/__init__.py.tent new file mode 100644 index 0000000000..9c752bfc8d --- /dev/null +++ b/packages/core_extensions/default_async_extension_python/__init__.py.tent @@ -0,0 +1,9 @@ +# +# This file is part of TEN Framework, an open source project. +# Licensed under the Apache License, Version 2.0. +# See the LICENSE file for more information. +# +from . import addon +from .log import logger + +logger.info("{{package_name}} extension loaded") diff --git a/packages/core_extensions/default_async_extension_python/addon.py b/packages/core_extensions/default_async_extension_python/addon.py new file mode 100644 index 0000000000..d2e813b06b --- /dev/null +++ b/packages/core_extensions/default_async_extension_python/addon.py @@ -0,0 +1,20 @@ +# +# This file is part of TEN Framework, an open source project. +# Licensed under the Apache License, Version 2.0. +# See the LICENSE file for more information. +# +from ten import ( + Addon, + register_addon_as_extension, + TenEnv, +) +from .extension import DefaultAsyncExtension +from .log import logger + + +@register_addon_as_extension("default_async_extension_python") +class DefaultAsyncExtensionAddon(Addon): + + def on_create_instance(self, ten_env: TenEnv, name: str, context) -> None: + logger.info("DefaultAsyncExtensionAddon on_create_instance") + ten_env.on_create_instance_done(DefaultAsyncExtension(name), context) diff --git a/packages/core_extensions/default_async_extension_python/addon.py.tent b/packages/core_extensions/default_async_extension_python/addon.py.tent new file mode 100644 index 0000000000..9c1d3fe19a --- /dev/null +++ b/packages/core_extensions/default_async_extension_python/addon.py.tent @@ -0,0 +1,20 @@ +# +# This file is part of TEN Framework, an open source project. +# Licensed under the Apache License, Version 2.0. +# See the LICENSE file for more information. +# +from ten import ( + Addon, + register_addon_as_extension, + TenEnv, +) +from .extension import {{class_name_prefix}}Extension +from .log import logger + + +@register_addon_as_extension("{{package_name}}") +class {{class_name_prefix}}ExtensionAddon(Addon): + + def on_create_instance(self, ten_env: TenEnv, name: str, context) -> None: + logger.info("{{class_name_prefix}}ExtensionAddon on_create_instance") + ten_env.on_create_instance_done({{class_name_prefix}}Extension(name), context) diff --git a/packages/core_extensions/default_async_extension_python/extension.py b/packages/core_extensions/default_async_extension_python/extension.py new file mode 100644 index 0000000000..64875869d3 --- /dev/null +++ b/packages/core_extensions/default_async_extension_python/extension.py @@ -0,0 +1,73 @@ +# +# This file is part of TEN Framework, an open source project. +# Licensed under the Apache License, Version 2.0. +# See the LICENSE file for more information. +# +from ten import ( + AudioFrame, + VideoFrame, + AsyncExtension, + AsyncTenEnv, + Cmd, + StatusCode, + CmdResult, + Data, +) + + +class DefaultAsyncExtension(AsyncExtension): + async def on_init(self, ten_env: AsyncTenEnv) -> None: + ten_env.log_debug("on_init") + ten_env.on_init_done() + + async def on_start(self, ten_env: AsyncTenEnv) -> None: + ten_env.log_debug("on_start") + + # TODO: read properties, initialize resources + + ten_env.on_start_done() + + async def on_stop(self, ten_env: AsyncTenEnv) -> None: + ten_env.log_debug("on_stop") + + # TODO: clean up resources + + ten_env.on_stop_done() + + async def on_deinit(self, ten_env: AsyncTenEnv) -> None: + ten_env.log_debug("on_deinit") + ten_env.on_deinit_done() + + async def on_cmd(self, ten_env: AsyncTenEnv, cmd: Cmd) -> None: + cmd_name = cmd.get_name() + ten_env.log_debug("on_cmd name {}".format(cmd_name)) + + # TODO: process cmd + + cmd_result = CmdResult.create(StatusCode.OK) + ten_env.return_result(cmd_result, cmd) + + async def on_data(self, ten_env: AsyncTenEnv, data: Data) -> None: + data_name = data.get_name() + ten_env.log_debug("on_data name {}".format(data_name)) + + # TODO: process data + pass + + async def on_audio_frame( + self, ten_env: AsyncTenEnv, audio_frame: AudioFrame + ) -> None: + audio_frame_name = audio_frame.get_name() + ten_env.log_debug("on_audio_frame name {}".format(audio_frame_name)) + + # TODO: process audio frame + pass + + async def on_video_frame( + self, ten_env: AsyncTenEnv, video_frame: VideoFrame + ) -> None: + video_frame_name = video_frame.get_name() + ten_env.log_debug("on_video_frame name {}".format(video_frame_name)) + + # TODO: process video frame + pass diff --git a/packages/core_extensions/default_async_extension_python/extension.py.tent b/packages/core_extensions/default_async_extension_python/extension.py.tent new file mode 100644 index 0000000000..49eb8aebd0 --- /dev/null +++ b/packages/core_extensions/default_async_extension_python/extension.py.tent @@ -0,0 +1,73 @@ +# +# This file is part of TEN Framework, an open source project. +# Licensed under the Apache License, Version 2.0. +# See the LICENSE file for more information. +# +from ten import ( + AudioFrame, + VideoFrame, + AsyncExtension, + AsyncTenEnv, + Cmd, + StatusCode, + CmdResult, + Data, +) + + +class {{class_name_prefix}}Extension(AsyncExtension): + async def on_init(self, ten_env: AsyncTenEnv) -> None: + ten_env.log_debug("on_init") + ten_env.on_init_done() + + async def on_start(self, ten_env: AsyncTenEnv) -> None: + ten_env.log_debug("on_start") + + # TODO: read properties, initialize resources + + ten_env.on_start_done() + + async def on_stop(self, ten_env: AsyncTenEnv) -> None: + ten_env.log_debug("on_stop") + + # TODO: clean up resources + + ten_env.on_stop_done() + + async def on_deinit(self, ten_env: AsyncTenEnv) -> None: + ten_env.log_debug("on_deinit") + ten_env.on_deinit_done() + + async def on_cmd(self, ten_env: AsyncTenEnv, cmd: Cmd) -> None: + cmd_name = cmd.get_name() + ten_env.log_debug("on_cmd name {}".format(cmd_name)) + + # TODO: process cmd + + cmd_result = CmdResult.create(StatusCode.OK) + ten_env.return_result(cmd_result, cmd) + + async def on_data(self, ten_env: AsyncTenEnv, data: Data) -> None: + data_name = data.get_name() + ten_env.log_debug("on_data name {}".format(data_name)) + + # TODO: process data + pass + + async def on_audio_frame( + self, ten_env: AsyncTenEnv, audio_frame: AudioFrame + ) -> None: + audio_frame_name = audio_frame.get_name() + ten_env.log_debug("on_audio_frame name {}".format(audio_frame_name)) + + # TODO: process audio frame + pass + + async def on_video_frame( + self, ten_env: AsyncTenEnv, video_frame: VideoFrame + ) -> None: + video_frame_name = video_frame.get_name() + ten_env.log_debug("on_video_frame name {}".format(video_frame_name)) + + # TODO: process video frame + pass diff --git a/packages/core_extensions/default_async_extension_python/log.py b/packages/core_extensions/default_async_extension_python/log.py new file mode 100644 index 0000000000..8980ea64df --- /dev/null +++ b/packages/core_extensions/default_async_extension_python/log.py @@ -0,0 +1,20 @@ +# +# This file is part of TEN Framework, an open source project. +# Licensed under the Apache License, Version 2.0. +# See the LICENSE file for more information. +# +import logging + +logger = logging.getLogger("default_async_extension_python") +logger.setLevel(logging.INFO) + +formatter_str = ( + "%(asctime)s - %(name)s - %(levelname)s - %(process)d - " + "[%(filename)s:%(lineno)d] - %(message)s" +) +formatter = logging.Formatter(formatter_str) + +console_handler = logging.StreamHandler() +console_handler.setFormatter(formatter) + +logger.addHandler(console_handler) diff --git a/packages/core_extensions/default_async_extension_python/log.py.tent b/packages/core_extensions/default_async_extension_python/log.py.tent new file mode 100644 index 0000000000..5f9bd6b29f --- /dev/null +++ b/packages/core_extensions/default_async_extension_python/log.py.tent @@ -0,0 +1,20 @@ +# +# This file is part of TEN Framework, an open source project. +# Licensed under the Apache License, Version 2.0. +# See the LICENSE file for more information. +# +import logging + +logger = logging.getLogger("{{package_name}}") +logger.setLevel(logging.INFO) + +formatter_str = ( + "%(asctime)s - %(name)s - %(levelname)s - %(process)d - " + "[%(filename)s:%(lineno)d] - %(message)s" +) +formatter = logging.Formatter(formatter_str) + +console_handler = logging.StreamHandler() +console_handler.setFormatter(formatter) + +logger.addHandler(console_handler) diff --git a/packages/core_extensions/default_async_extension_python/manifest.json b/packages/core_extensions/default_async_extension_python/manifest.json new file mode 100644 index 0000000000..0e020f0c5d --- /dev/null +++ b/packages/core_extensions/default_async_extension_python/manifest.json @@ -0,0 +1,24 @@ +{ + "type": "extension", + "name": "default_async_extension_python", + "version": "0.3.0-alpha", + "dependencies": [ + { + "type": "system", + "name": "ten_runtime_python", + "version": "0.3.0-alpha" + } + ], + "package": { + "include": [ + "manifest.json", + "property.json", + "BUILD.gn", + "**.tent", + "**.py", + "README.md", + "tests/**" + ] + }, + "api": {} +} \ No newline at end of file diff --git a/packages/core_extensions/default_async_extension_python/manifest.json.tent b/packages/core_extensions/default_async_extension_python/manifest.json.tent new file mode 100644 index 0000000000..8d62e55140 --- /dev/null +++ b/packages/core_extensions/default_async_extension_python/manifest.json.tent @@ -0,0 +1,24 @@ +{ + "type": "extension", + "name": "{{package_name}}", + "version": "0.3.0-alpha", + "dependencies": [ + { + "type": "system", + "name": "ten_runtime_python", + "version": "0.3.0-alpha" + } + ], + "package": { + "include": [ + "manifest.json", + "property.json", + "BUILD.gn", + "**.tent", + "**.py", + "README.md", + "tests/**" + ] + }, + "api": {} +} \ No newline at end of file diff --git a/packages/core_extensions/default_async_extension_python/property.json b/packages/core_extensions/default_async_extension_python/property.json new file mode 100644 index 0000000000..9e26dfeeb6 --- /dev/null +++ b/packages/core_extensions/default_async_extension_python/property.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/packages/core_extensions/default_async_extension_python/tests/bin/start b/packages/core_extensions/default_async_extension_python/tests/bin/start new file mode 100755 index 0000000000..4723f241bd --- /dev/null +++ b/packages/core_extensions/default_async_extension_python/tests/bin/start @@ -0,0 +1,21 @@ +#!/bin/bash + +set -e + +cd "$(dirname "${BASH_SOURCE[0]}")/../.." + +export PYTHONPATH=.ten/app/ten_packages/system/ten_runtime_python/lib:.ten/app/ten_packages/system/ten_runtime_python/interface + +# If the Python app imports some modules that are compiled with a different +# version of libstdc++ (ex: PyTorch), the Python app may encounter confusing +# errors. To solve this problem, we can preload the correct version of +# libstdc++. +# +# export LD_PRELOAD=/lib/x86_64-linux-gnu/libstdc++.so.6 +# +# Another solution is to make sure the module 'ten_runtime_python' is imported +# _after_ the module that requires another version of libstdc++ is imported. +# +# Refer to https://github.com/pytorch/pytorch/issues/102360?from_wecom=1#issuecomment-1708989096 + +pytest -s tests/ diff --git a/packages/core_extensions/default_async_extension_python/tests/test_basic.py b/packages/core_extensions/default_async_extension_python/tests/test_basic.py new file mode 100644 index 0000000000..c3755f449a --- /dev/null +++ b/packages/core_extensions/default_async_extension_python/tests/test_basic.py @@ -0,0 +1,36 @@ +# +# Copyright © 2024 Agora +# This file is part of TEN Framework, an open source project. +# Licensed under the Apache License, Version 2.0, with certain conditions. +# Refer to the "LICENSE" file in the root directory for more information. +# +from pathlib import Path +from ten import ExtensionTester, TenEnvTester, Cmd, CmdResult, StatusCode + + +class ExtensionTesterBasic(ExtensionTester): + def check_hello(self, ten_env: TenEnvTester, result: CmdResult): + statusCode = result.get_status_code() + print("receive hello_world, status:" + str(statusCode)) + + if statusCode == StatusCode.OK: + ten_env.stop_test() + + def on_start(self, ten_env: TenEnvTester) -> None: + new_cmd = Cmd.create("hello_world") + + print("send hello_world") + ten_env.send_cmd( + new_cmd, + lambda ten_env, result: self.check_hello(ten_env, result), + ) + + print("tester on_start_done") + ten_env.on_start_done() + + +def test_basic(): + tester = ExtensionTesterBasic() + tester.add_addon_base_dir(str(Path(__file__).resolve().parent.parent)) + tester.set_test_mode_single("default_async_extension_python") + tester.run() diff --git a/packages/core_extensions/default_extension_cpp/manifest.json b/packages/core_extensions/default_extension_cpp/manifest.json index 0c37fe8721..bf5e9b7c65 100644 --- a/packages/core_extensions/default_extension_cpp/manifest.json +++ b/packages/core_extensions/default_extension_cpp/manifest.json @@ -1,12 +1,12 @@ { "type": "extension", "name": "default_extension_cpp", - "version": "0.1.0", + "version": "0.3.0-alpha", "dependencies": [ { "type": "system", "name": "ten_runtime", - "version": "0.2.0" + "version": "0.3.0-alpha" } ], "package": { diff --git a/packages/core_extensions/default_extension_cpp/manifest.json.tent b/packages/core_extensions/default_extension_cpp/manifest.json.tent index dc29867cd6..1195bd38c7 100644 --- a/packages/core_extensions/default_extension_cpp/manifest.json.tent +++ b/packages/core_extensions/default_extension_cpp/manifest.json.tent @@ -1,12 +1,12 @@ { "type": "extension", "name": "{{package_name}}_extension_cpp", - "version": "0.1.0", + "version": "0.3.0-alpha", "dependencies": [ { "type": "system", "name": "ten_runtime", - "version": "0.2.0" + "version": "0.3.0-alpha" } ], "package": { diff --git a/packages/core_extensions/default_extension_go/extension.go b/packages/core_extensions/default_extension_go/extension.go index b7971a9709..07b66307c3 100644 --- a/packages/core_extensions/default_extension_go/extension.go +++ b/packages/core_extensions/default_extension_go/extension.go @@ -3,10 +3,6 @@ // Licensed under the Apache License, Version 2.0. // See the LICENSE file for more information. // -// Note that this is just an example extension written in the GO programming -// language, so the package name does not equal to the containing directory -// name. However, it is not common in Go. -// package default_extension_go diff --git a/packages/core_extensions/default_extension_go/manifest.json b/packages/core_extensions/default_extension_go/manifest.json index aaeb717849..b2d17d77f4 100644 --- a/packages/core_extensions/default_extension_go/manifest.json +++ b/packages/core_extensions/default_extension_go/manifest.json @@ -1,12 +1,12 @@ { "type": "extension", "name": "default_extension_go", - "version": "0.1.0", + "version": "0.3.0-alpha", "dependencies": [ { "type": "system", "name": "ten_runtime_go", - "version": "0.1.0" + "version": "0.3.0-alpha" } ], "package": { diff --git a/packages/core_extensions/default_extension_go/manifest.json.tent b/packages/core_extensions/default_extension_go/manifest.json.tent index 8eaa33e846..9f7abf6af0 100644 --- a/packages/core_extensions/default_extension_go/manifest.json.tent +++ b/packages/core_extensions/default_extension_go/manifest.json.tent @@ -1,12 +1,12 @@ { "type": "extension", "name": "{{package_name}}", - "version": "0.1.0", + "version": "0.3.0-alpha", "dependencies": [ { "type": "system", "name": "ten_runtime_go", - "version": "0.1.0" + "version": "0.3.0-alpha" } ], "package": { diff --git a/packages/core_extensions/default_extension_python/manifest.json b/packages/core_extensions/default_extension_python/manifest.json index e9a863ac9e..90a6be051d 100644 --- a/packages/core_extensions/default_extension_python/manifest.json +++ b/packages/core_extensions/default_extension_python/manifest.json @@ -1,12 +1,12 @@ { "type": "extension", "name": "default_extension_python", - "version": "0.1.0", + "version": "0.3.0-alpha", "dependencies": [ { "type": "system", "name": "ten_runtime_python", - "version": "0.1.0" + "version": "0.3.0-alpha" } ], "package": { diff --git a/packages/core_extensions/default_extension_python/manifest.json.tent b/packages/core_extensions/default_extension_python/manifest.json.tent index 7159a3b21a..8d62e55140 100644 --- a/packages/core_extensions/default_extension_python/manifest.json.tent +++ b/packages/core_extensions/default_extension_python/manifest.json.tent @@ -1,12 +1,12 @@ { "type": "extension", "name": "{{package_name}}", - "version": "0.1.0", + "version": "0.3.0-alpha", "dependencies": [ { "type": "system", "name": "ten_runtime_python", - "version": "0.1.0" + "version": "0.3.0-alpha" } ], "package": { diff --git a/packages/core_extensions/py_init_extension_cpp/BUILD.gn b/packages/core_extensions/py_init_extension_cpp/BUILD.gn index bbd03ef4ba..95f332e254 100644 --- a/packages/core_extensions/py_init_extension_cpp/BUILD.gn +++ b/packages/core_extensions/py_init_extension_cpp/BUILD.gn @@ -15,11 +15,6 @@ ten_package("py_init_extension_cpp") { resources = [ "manifest.json", "property.json", - - # We want to pack the source code even in the 'release' package, then users - # could choose to use this extension directly as a 'release' package, or - # develop their own extension based on this. - "src", ] sources = [ "src/main.cc" ] diff --git a/packages/core_extensions/py_init_extension_cpp/manifest.json b/packages/core_extensions/py_init_extension_cpp/manifest.json index 2687a6c31f..1ee53304db 100644 --- a/packages/core_extensions/py_init_extension_cpp/manifest.json +++ b/packages/core_extensions/py_init_extension_cpp/manifest.json @@ -1,17 +1,17 @@ { "type": "extension", "name": "py_init_extension_cpp", - "version": "0.1.0", + "version": "0.3.0-alpha", "dependencies": [ { "type": "system", "name": "ten_runtime", - "version": "0.2.0" + "version": "0.3.0-alpha" }, { "type": "system", "name": "ten_runtime_python", - "version": "0.1.0" + "version": "0.3.0-alpha" } ], "api": {} diff --git a/packages/core_protocols/msgpack/manifest.json b/packages/core_protocols/msgpack/manifest.json index f105b1c3f4..30f8dc2828 100644 --- a/packages/core_protocols/msgpack/manifest.json +++ b/packages/core_protocols/msgpack/manifest.json @@ -1,6 +1,6 @@ { "type": "protocol", "name": "msgpack", - "version": "0.1.0", + "version": "0.3.0-alpha", "dependencies": [] } \ No newline at end of file diff --git a/packages/example_apps/pprof_app_go/manifest.json b/packages/example_apps/pprof_app_go/manifest.json index f6dd5e815d..94ddc69dc8 100644 --- a/packages/example_apps/pprof_app_go/manifest.json +++ b/packages/example_apps/pprof_app_go/manifest.json @@ -6,12 +6,12 @@ { "type": "system", "name": "ten_runtime", - "version": "0.2.0" + "version": "0.3.0-alpha" }, { "type": "system", "name": "ten_runtime_go", - "version": "0.1.0" + "version": "0.3.0-alpha" } ] } \ No newline at end of file diff --git a/packages/example_apps/pprof_app_go/manifest.json.tent b/packages/example_apps/pprof_app_go/manifest.json.tent index 7dacd887bb..2e8d7c87a6 100644 --- a/packages/example_apps/pprof_app_go/manifest.json.tent +++ b/packages/example_apps/pprof_app_go/manifest.json.tent @@ -6,11 +6,12 @@ { "type": "system", "name": "ten_runtime", - "version": "0.2.0" + "version": "0.3.0-alpha" }, { "type": "system", "name": "ten_runtime_go", - "version": "0.1.0" + "version": "0.3.0-alpha" } - ]} \ No newline at end of file + ] +} \ No newline at end of file diff --git a/packages/example_extensions/aio_http_server_python/main.py b/packages/example_extensions/aio_http_server_python/main.py index eade33052d..ac4a441f6e 100644 --- a/packages/example_extensions/aio_http_server_python/main.py +++ b/packages/example_extensions/aio_http_server_python/main.py @@ -3,28 +3,28 @@ # Licensed under the Apache License, Version 2.0. # See the LICENSE file for more information. # +import asyncio import json -import threading from aiohttp import web, web_request, WSMsgType -import asyncio from ten import ( Addon, - Extension, + AsyncExtension, register_addon_as_extension, TenEnv, Cmd, CmdResult, StatusCode, + AsyncTenEnv, ) -class HttpServerExtension(Extension): +class HttpServerExtension(AsyncExtension): async def default_handler(self, request: web_request.Request): - # parse the json body + # Parse the json body. try: data = await request.json() except Exception as e: - print("Error: " + str(e)) + self.ten_env.log_error("Error: " + str(e)) return web.Response(status=400, text="Bad request") cmd_result = None @@ -37,16 +37,23 @@ async def default_handler(self, request: web_request.Request): else: # If the command is a 'close_app' command, send it to the app. if "type" in data["_ten"] and data["_ten"]["type"] == "close_app": - close_app_cmd_json = """{"_ten":{"type":"close_app","dest":[{"app":"localhost"}]}}""" - self.tenEnv.send_json(close_app_cmd_json, None) + close_app_cmd_json = ( + '{"_ten":{"type":"close_app",' + '"dest":[{"app":"localhost"}]}}' + ) + asyncio.create_task(self.ten_env.send_json(close_app_cmd_json)) return web.Response(status=200, text="OK") elif "name" in data["_ten"]: - # send the command to the TEN runtime + # Send the command to the TEN runtime. data["method"] = method data["url"] = url cmd = Cmd.create_from_json(json.dumps(data)) - # send 'test' command to the TEN runtime and wait for the result - cmd_result = await self.send_cmd_async(self.tenEnv, cmd) + + # Send the command to the TEN runtime and wait for the result. + if cmd is None: + return web.Response(status=400, text="Bad request") + + cmd_result = await self.ten_env.send_cmd(cmd) else: return web.Response(status=404, text="Not found") @@ -55,7 +62,7 @@ async def default_handler(self, request: web_request.Request): detail = cmd_result.get_property_string("detail") return web.Response(text=detail) except Exception as e: - print("Error: " + str(e)) + self.ten_env.log_error("Error: " + str(e)) return web.Response(status=500, text="Internal server error") else: return web.Response(status=500, text="Internal server error") @@ -71,7 +78,9 @@ async def default_ws_handler(self, request: web_request.Request): else: await ws.send_str("some websocket message payload") elif msg.type == WSMsgType.ERROR: - print("ws connection closed with exception %s" % ws.exception()) + self.ten_env.log_error( + "ws connection closed with exception %s" % ws.exception() + ) return ws @@ -91,78 +100,41 @@ async def start_server(self, host, port): self.tcpSite = web.TCPSite(runner, host, port) await self.tcpSite.start() - async def __thread_routine(self, ten_env: TenEnv): - print("HttpServerExtension __thread_routine start") - - self.loop = asyncio.get_running_loop() - - await self.start_server("0.0.0.0", self.server_port) - - ten_env.on_start_done() - - # Suspend the thread until stopEvent is set - await self.stopEvent.wait() - - await self.webApp.shutdown() - await self.webApp.cleanup() - - async def stop_thread(self): - self.stopEvent.set() - - async def send_cmd_async(self, ten_env: TenEnv, cmd: Cmd) -> CmdResult: - print("HttpServerExtension send_cmd_async") - q = asyncio.Queue(1) - ten_env.send_cmd( - cmd, - lambda ten_env, result: asyncio.run_coroutine_threadsafe( - q.put(result), self.loop - ), # type: ignore - ) - return await q.get() - def __init__(self, name: str) -> None: super().__init__(name) self.name = name - self.stopEvent = asyncio.Event() - def on_init(self, ten_env: TenEnv) -> None: + async def on_init(self, ten_env: AsyncTenEnv) -> None: + self.ten_env = ten_env ten_env.on_init_done() - def on_start(self, ten_env: TenEnv) -> None: - print("HttpServerExtension on_start") + async def on_start(self, ten_env: AsyncTenEnv) -> None: + ten_env.log_debug("on_start") try: self.server_port = ten_env.get_property_int("server_port") except Exception as e: - print("Could not read 'server_port' from properties." + str(e)) + ten_env.log_error( + "Could not read 'server_port' from properties." + str(e) + ) self.server_port = 8002 - self.tenEnv = ten_env - - self.thread = threading.Thread( - target=asyncio.run, args=(self.__thread_routine(ten_env),) - ) + await self.start_server("0.0.0.0", self.server_port) - # Then 'on_start_done' will be called in the thread - self.thread.start() + ten_env.on_start_done() - def on_deinit(self, ten_env: TenEnv) -> None: - print("HttpServerExtension on_deinit") + async def on_deinit(self, ten_env: AsyncTenEnv) -> None: + ten_env.log_debug("on_deinit") ten_env.on_deinit_done() - def on_cmd(self, ten_env: TenEnv, cmd: Cmd) -> None: - print("HttpServerExtension on_cmd") + async def on_cmd(self, ten_env: AsyncTenEnv, cmd: Cmd) -> None: + ten_env.log_debug("on_cmd") - # Not supported command + # Not supported command. ten_env.return_result(CmdResult.create(StatusCode.ERROR), cmd) - def on_stop(self, ten_env: TenEnv) -> None: - print("HttpServerExtension on_stop") - - if self.thread.is_alive(): - asyncio.run_coroutine_threadsafe(self.stop_thread(), self.loop) - self.thread.join() - + async def on_stop(self, ten_env: AsyncTenEnv) -> None: + ten_env.log_debug("on_stop") ten_env.on_stop_done() diff --git a/packages/example_extensions/aio_http_server_python/manifest.json b/packages/example_extensions/aio_http_server_python/manifest.json index f40c671291..c47ff222db 100644 --- a/packages/example_extensions/aio_http_server_python/manifest.json +++ b/packages/example_extensions/aio_http_server_python/manifest.json @@ -6,7 +6,7 @@ { "type": "system", "name": "ten_runtime_python", - "version": "0.1.0" + "version": "0.3.0-alpha" } ], "api": {} diff --git a/packages/example_extensions/pil_demo_python/manifest.json b/packages/example_extensions/pil_demo_python/manifest.json index 40d5daf14a..597525800c 100644 --- a/packages/example_extensions/pil_demo_python/manifest.json +++ b/packages/example_extensions/pil_demo_python/manifest.json @@ -6,7 +6,7 @@ { "type": "system", "name": "ten_runtime_python", - "version": "0.1.0" + "version": "0.3.0-alpha" } ], "api": {} diff --git a/packages/example_extensions/simple_echo_cpp/manifest.json b/packages/example_extensions/simple_echo_cpp/manifest.json index 898bdd125c..4f1344e69e 100644 --- a/packages/example_extensions/simple_echo_cpp/manifest.json +++ b/packages/example_extensions/simple_echo_cpp/manifest.json @@ -6,7 +6,7 @@ { "type": "system", "name": "ten_runtime", - "version": "0.2.0" + "version": "0.3.0-alpha" } ], "api": {} diff --git a/packages/example_extensions/simple_http_server_cpp/manifest.json b/packages/example_extensions/simple_http_server_cpp/manifest.json index 573c3603a1..f86d5630d3 100644 --- a/packages/example_extensions/simple_http_server_cpp/manifest.json +++ b/packages/example_extensions/simple_http_server_cpp/manifest.json @@ -6,7 +6,7 @@ { "type": "system", "name": "ten_runtime", - "version": "0.2.0" + "version": "0.3.0-alpha" } ] } \ No newline at end of file diff --git a/packages/example_extensions/simple_http_server_go/manifest.json b/packages/example_extensions/simple_http_server_go/manifest.json index 2d8dc5c5ac..f989a54fd4 100644 --- a/packages/example_extensions/simple_http_server_go/manifest.json +++ b/packages/example_extensions/simple_http_server_go/manifest.json @@ -6,7 +6,7 @@ { "type": "system", "name": "ten_runtime_go", - "version": "0.1.0" + "version": "0.3.0-alpha" } ], "api": {} diff --git a/tests/ten_manager/error_context/package_version_not_found/test_app/manifest.json b/tests/ten_manager/error_context/package_version_not_found/test_app/manifest.json index f2659c0a21..f79dc5384c 100644 --- a/tests/ten_manager/error_context/package_version_not_found/test_app/manifest.json +++ b/tests/ten_manager/error_context/package_version_not_found/test_app/manifest.json @@ -6,7 +6,7 @@ { "type": "system", "name": "ten_runtime", - "version": "100.998.999" + "version": "0.3.0-alpha" } ] } \ No newline at end of file diff --git a/tests/ten_runtime/integration/cpp/ffmpeg_basic/ffmpeg_basic_app_source/manifest.json b/tests/ten_runtime/integration/cpp/ffmpeg_basic/ffmpeg_basic_app_source/manifest.json index 97e109cd53..44e7e9280d 100644 --- a/tests/ten_runtime/integration/cpp/ffmpeg_basic/ffmpeg_basic_app_source/manifest.json +++ b/tests/ten_runtime/integration/cpp/ffmpeg_basic/ffmpeg_basic_app_source/manifest.json @@ -3,7 +3,7 @@ { "type": "system", "name": "ten_runtime", - "version": "0.2.0" + "version": "0.3.0-alpha" }, { "type": "extension", @@ -18,12 +18,12 @@ { "type": "extension", "name": "default_extension_cpp", - "version": "0.1.0" + "version": "0.3.0-alpha" }, { "type": "protocol", "name": "msgpack", - "version": "0.1.0" + "version": "0.3.0-alpha" } ] } \ No newline at end of file diff --git a/tests/ten_runtime/integration/cpp/ffmpeg_basic/ffmpeg_basic_app_source/ten_packages/extension/default_extension_cpp/manifest.json b/tests/ten_runtime/integration/cpp/ffmpeg_basic/ffmpeg_basic_app_source/ten_packages/extension/default_extension_cpp/manifest.json index d693eb8ddb..8daafef8b9 100644 --- a/tests/ten_runtime/integration/cpp/ffmpeg_basic/ffmpeg_basic_app_source/ten_packages/extension/default_extension_cpp/manifest.json +++ b/tests/ten_runtime/integration/cpp/ffmpeg_basic/ffmpeg_basic_app_source/ten_packages/extension/default_extension_cpp/manifest.json @@ -1,12 +1,12 @@ { "type": "extension", "name": "default_extension_cpp", - "version": "0.1.0", + "version": "0.3.0-alpha", "dependencies": [ { "type": "system", "name": "ten_runtime", - "version": "0.2.0" + "version": "0.3.0-alpha" } ] } \ No newline at end of file diff --git a/tests/ten_runtime/integration/cpp/ffmpeg_bypass/ffmpeg_bypass_app_source/manifest.json b/tests/ten_runtime/integration/cpp/ffmpeg_bypass/ffmpeg_bypass_app_source/manifest.json index 97e109cd53..44e7e9280d 100644 --- a/tests/ten_runtime/integration/cpp/ffmpeg_bypass/ffmpeg_bypass_app_source/manifest.json +++ b/tests/ten_runtime/integration/cpp/ffmpeg_bypass/ffmpeg_bypass_app_source/manifest.json @@ -3,7 +3,7 @@ { "type": "system", "name": "ten_runtime", - "version": "0.2.0" + "version": "0.3.0-alpha" }, { "type": "extension", @@ -18,12 +18,12 @@ { "type": "extension", "name": "default_extension_cpp", - "version": "0.1.0" + "version": "0.3.0-alpha" }, { "type": "protocol", "name": "msgpack", - "version": "0.1.0" + "version": "0.3.0-alpha" } ] } \ No newline at end of file diff --git a/tests/ten_runtime/integration/cpp/ffmpeg_bypass/ffmpeg_bypass_app_source/ten_packages/extension/default_extension_cpp/manifest.json b/tests/ten_runtime/integration/cpp/ffmpeg_bypass/ffmpeg_bypass_app_source/ten_packages/extension/default_extension_cpp/manifest.json index d693eb8ddb..8daafef8b9 100644 --- a/tests/ten_runtime/integration/cpp/ffmpeg_bypass/ffmpeg_bypass_app_source/ten_packages/extension/default_extension_cpp/manifest.json +++ b/tests/ten_runtime/integration/cpp/ffmpeg_bypass/ffmpeg_bypass_app_source/ten_packages/extension/default_extension_cpp/manifest.json @@ -1,12 +1,12 @@ { "type": "extension", "name": "default_extension_cpp", - "version": "0.1.0", + "version": "0.3.0-alpha", "dependencies": [ { "type": "system", "name": "ten_runtime", - "version": "0.2.0" + "version": "0.3.0-alpha" } ] } \ No newline at end of file diff --git a/tests/ten_runtime/integration/cpp/graph_env_var_1/graph_env_var_1_app_source/manifest.json b/tests/ten_runtime/integration/cpp/graph_env_var_1/graph_env_var_1_app_source/manifest.json index 598a6396a5..758da4f2db 100644 --- a/tests/ten_runtime/integration/cpp/graph_env_var_1/graph_env_var_1_app_source/manifest.json +++ b/tests/ten_runtime/integration/cpp/graph_env_var_1/graph_env_var_1_app_source/manifest.json @@ -3,17 +3,17 @@ { "type": "system", "name": "ten_runtime", - "version": "0.2.0" + "version": "0.3.0-alpha" }, { "type": "extension", "name": "default_extension_cpp", - "version": "0.1.0" + "version": "0.3.0-alpha" }, { "type": "protocol", "name": "msgpack", - "version": "0.1.0" + "version": "0.3.0-alpha" } ] } \ No newline at end of file diff --git a/tests/ten_runtime/integration/cpp/graph_env_var_2/graph_env_var_2_app_source/manifest.json b/tests/ten_runtime/integration/cpp/graph_env_var_2/graph_env_var_2_app_source/manifest.json index 598a6396a5..758da4f2db 100644 --- a/tests/ten_runtime/integration/cpp/graph_env_var_2/graph_env_var_2_app_source/manifest.json +++ b/tests/ten_runtime/integration/cpp/graph_env_var_2/graph_env_var_2_app_source/manifest.json @@ -3,17 +3,17 @@ { "type": "system", "name": "ten_runtime", - "version": "0.2.0" + "version": "0.3.0-alpha" }, { "type": "extension", "name": "default_extension_cpp", - "version": "0.1.0" + "version": "0.3.0-alpha" }, { "type": "protocol", "name": "msgpack", - "version": "0.1.0" + "version": "0.3.0-alpha" } ] } \ No newline at end of file diff --git a/tests/ten_runtime/integration/cpp/graph_env_var_3/graph_env_var_3_app_source/manifest.json b/tests/ten_runtime/integration/cpp/graph_env_var_3/graph_env_var_3_app_source/manifest.json index 598a6396a5..758da4f2db 100644 --- a/tests/ten_runtime/integration/cpp/graph_env_var_3/graph_env_var_3_app_source/manifest.json +++ b/tests/ten_runtime/integration/cpp/graph_env_var_3/graph_env_var_3_app_source/manifest.json @@ -3,17 +3,17 @@ { "type": "system", "name": "ten_runtime", - "version": "0.2.0" + "version": "0.3.0-alpha" }, { "type": "extension", "name": "default_extension_cpp", - "version": "0.1.0" + "version": "0.3.0-alpha" }, { "type": "protocol", "name": "msgpack", - "version": "0.1.0" + "version": "0.3.0-alpha" } ] } \ No newline at end of file diff --git a/tests/ten_runtime/integration/cpp/hello_world/hello_world_app_source/manifest.json b/tests/ten_runtime/integration/cpp/hello_world/hello_world_app_source/manifest.json index f26f4323f2..32229225bb 100644 --- a/tests/ten_runtime/integration/cpp/hello_world/hello_world_app_source/manifest.json +++ b/tests/ten_runtime/integration/cpp/hello_world/hello_world_app_source/manifest.json @@ -3,17 +3,17 @@ { "type": "system", "name": "ten_runtime", - "version": "0.2.0" + "version": "0.3.0-alpha" }, { "type": "extension", "name": "default_extension_cpp", - "version": "0.1.0" + "version": "0.3.0-alpha" }, { "type": "protocol", "name": "msgpack", - "version": "0.1.0" + "version": "0.3.0-alpha" } ], "api": { diff --git a/tests/ten_runtime/integration/cpp/http_basic/restful_app_source/manifest.json b/tests/ten_runtime/integration/cpp/http_basic/restful_app_source/manifest.json index f9c7d5b535..14a6787517 100644 --- a/tests/ten_runtime/integration/cpp/http_basic/restful_app_source/manifest.json +++ b/tests/ten_runtime/integration/cpp/http_basic/restful_app_source/manifest.json @@ -3,7 +3,7 @@ { "type": "system", "name": "ten_runtime", - "version": "0.2.0" + "version": "0.3.0-alpha" }, { "type": "extension", diff --git a/tests/ten_runtime/integration/cpp/large_result/large_result_source/manifest.json b/tests/ten_runtime/integration/cpp/large_result/large_result_source/manifest.json index 78a4a2bba2..81948e98a6 100644 --- a/tests/ten_runtime/integration/cpp/large_result/large_result_source/manifest.json +++ b/tests/ten_runtime/integration/cpp/large_result/large_result_source/manifest.json @@ -3,7 +3,7 @@ { "type": "system", "name": "ten_runtime", - "version": "0.2.0" + "version": "0.3.0-alpha" }, { "type": "extension", @@ -13,7 +13,7 @@ { "type": "extension", "name": "default_extension_cpp", - "version": "0.1.0" + "version": "0.3.0-alpha" } ] } \ No newline at end of file diff --git a/tests/ten_runtime/integration/cpp/large_result/large_result_source/ten_packages/extension/default_extension_cpp/manifest.json b/tests/ten_runtime/integration/cpp/large_result/large_result_source/ten_packages/extension/default_extension_cpp/manifest.json index d693eb8ddb..8daafef8b9 100644 --- a/tests/ten_runtime/integration/cpp/large_result/large_result_source/ten_packages/extension/default_extension_cpp/manifest.json +++ b/tests/ten_runtime/integration/cpp/large_result/large_result_source/ten_packages/extension/default_extension_cpp/manifest.json @@ -1,12 +1,12 @@ { "type": "extension", "name": "default_extension_cpp", - "version": "0.1.0", + "version": "0.3.0-alpha", "dependencies": [ { "type": "system", "name": "ten_runtime", - "version": "0.2.0" + "version": "0.3.0-alpha" } ] } \ No newline at end of file diff --git a/tests/ten_runtime/integration/cpp/standalone_test_cpp/default_extension_cpp/manifest.json b/tests/ten_runtime/integration/cpp/standalone_test_cpp/default_extension_cpp/manifest.json index 0051d2975c..11575e9bc4 100644 --- a/tests/ten_runtime/integration/cpp/standalone_test_cpp/default_extension_cpp/manifest.json +++ b/tests/ten_runtime/integration/cpp/standalone_test_cpp/default_extension_cpp/manifest.json @@ -1,12 +1,12 @@ { "type": "extension", "name": "default_extension_cpp", - "version": "0.1.0", + "version": "0.3.0-alpha", "dependencies": [ { "type": "system", "name": "ten_runtime", - "version": "0.2.0" + "version": "0.3.0-alpha" } ], "package": { diff --git a/tests/ten_runtime/integration/go/access_property_go/BUILD.gn b/tests/ten_runtime/integration/go/access_property_go/BUILD.gn index ecb83ac08f..7ff8b3a997 100644 --- a/tests/ten_runtime/integration/go/access_property_go/BUILD.gn +++ b/tests/ten_runtime/integration/go/access_property_go/BUILD.gn @@ -17,7 +17,7 @@ ten_package_test_prepare_app("access_property_go_app") { "access_property_go_app/property.json", ] - replace_files_after_install_all = [ "access_property_go_app/ten_packages/extension/default_extension_go/default_extension.go" ] + replace_files_after_install_all = [ "access_property_go_app/ten_packages/extension/default_extension_go/extension.go" ] deps = [ "//core/src/ten_manager", diff --git a/tests/ten_runtime/integration/go/access_property_go/access_property_go_app/manifest.json b/tests/ten_runtime/integration/go/access_property_go/access_property_go_app/manifest.json index 6326076425..54015e0959 100644 --- a/tests/ten_runtime/integration/go/access_property_go/access_property_go_app/manifest.json +++ b/tests/ten_runtime/integration/go/access_property_go/access_property_go_app/manifest.json @@ -3,12 +3,12 @@ { "type": "extension", "name": "default_extension_go", - "version": "0.1.0" + "version": "0.3.0-alpha" }, { "type": "protocol", "name": "msgpack", - "version": "0.1.0" + "version": "0.3.0-alpha" } ] } \ No newline at end of file diff --git a/tests/ten_runtime/integration/go/access_property_go/access_property_go_app/ten_packages/extension/default_extension_go/default_extension.go b/tests/ten_runtime/integration/go/access_property_go/access_property_go_app/ten_packages/extension/default_extension_go/extension.go similarity index 95% rename from tests/ten_runtime/integration/go/access_property_go/access_property_go_app/ten_packages/extension/default_extension_go/default_extension.go rename to tests/ten_runtime/integration/go/access_property_go/access_property_go_app/ten_packages/extension/default_extension_go/extension.go index f6fb1ff069..32ff300281 100644 --- a/tests/ten_runtime/integration/go/access_property_go/access_property_go_app/ten_packages/extension/default_extension_go/default_extension.go +++ b/tests/ten_runtime/integration/go/access_property_go/access_property_go_app/ten_packages/extension/default_extension_go/extension.go @@ -1,13 +1,12 @@ +// // Copyright © 2024 Agora // This file is part of TEN Framework, an open source project. // Licensed under the Apache License, Version 2.0, with certain conditions. // Refer to https://github.com/TEN-framework/ten_framework/LICENSE for more // information. // -// Note that this is just an example extension written in the GO programming -// language, so the package name does not equal to the containing directory -// name. However, it is not common in Go. -package defaultextension + +package default_extension_go import ( "ten_framework/ten" diff --git a/tests/ten_runtime/integration/go/close_app_go/client/client.cc b/tests/ten_runtime/integration/go/close_app_go/client/client.cc index 342424e328..b33d13d074 100644 --- a/tests/ten_runtime/integration/go/close_app_go/client/client.cc +++ b/tests/ten_runtime/integration/go/close_app_go/client/client.cc @@ -6,7 +6,6 @@ // #include -#include "include_internal/ten_runtime/binding/cpp/ten.h" #include "ten_utils/macro/mark.h" #include "tests/common/client/cpp/msgpack_tcp.h" @@ -23,18 +22,13 @@ int main(TEN_UNUSED int argc, TEN_UNUSED char **argv) { "app": "msgpack://127.0.0.1:8007/", "graph": "default", "extension_group": "default_extension_group", - "extension": "default_extension" + "extension": "default_extension_go" }] } })"_json); TEN_ASSERT(TEN_STATUS_CODE_OK == resp["_ten"]["status_code"], "Should not happen."); - std::string resp_str = resp["detail"]; - TEN_LOGD("got result: %s", resp_str.c_str()); - TEN_ASSERT(resp_str == std::string("This is default go extension."), - "Should not happen."); - // NOTE the order: client destroy, then connection lost, then nodejs exits delete client; } diff --git a/tests/ten_runtime/integration/go/close_app_go/close_app_go_app/manifest.json b/tests/ten_runtime/integration/go/close_app_go/close_app_go_app/manifest.json index 6326076425..54015e0959 100644 --- a/tests/ten_runtime/integration/go/close_app_go/close_app_go_app/manifest.json +++ b/tests/ten_runtime/integration/go/close_app_go/close_app_go_app/manifest.json @@ -3,12 +3,12 @@ { "type": "extension", "name": "default_extension_go", - "version": "0.1.0" + "version": "0.3.0-alpha" }, { "type": "protocol", "name": "msgpack", - "version": "0.1.0" + "version": "0.3.0-alpha" } ] } \ No newline at end of file diff --git a/tests/ten_runtime/integration/go/close_app_go/close_app_go_app/property.json b/tests/ten_runtime/integration/go/close_app_go/close_app_go_app/property.json index d3212b1fbd..8d63cb8728 100644 --- a/tests/ten_runtime/integration/go/close_app_go/close_app_go_app/property.json +++ b/tests/ten_runtime/integration/go/close_app_go/close_app_go_app/property.json @@ -16,7 +16,7 @@ }, { "type": "extension", - "name": "default_extension", + "name": "default_extension_go", "addon": "default_extension_go", "extension_group": "default_extension_group" } diff --git a/tests/ten_runtime/integration/go/exit_signal/BUILD.gn b/tests/ten_runtime/integration/go/exit_signal/BUILD.gn index 46d46cca03..417fca7ed0 100644 --- a/tests/ten_runtime/integration/go/exit_signal/BUILD.gn +++ b/tests/ten_runtime/integration/go/exit_signal/BUILD.gn @@ -17,7 +17,9 @@ ten_package_test_prepare_app("exit_signal_app") { "exit_signal_app/property.json", ] - replace_files_after_install_all = [ "exit_signal_app/ten_packages/extension/default_extension_go/default_extension.go" ] + replace_files_after_install_all = [ + "exit_signal_app/ten_packages/extension/default_extension_go/extension.go", + ] deps = [ "//core/src/ten_manager", diff --git a/tests/ten_runtime/integration/go/exit_signal/exit_signal_app/manifest.json b/tests/ten_runtime/integration/go/exit_signal/exit_signal_app/manifest.json index af58704cc9..200e96917b 100644 --- a/tests/ten_runtime/integration/go/exit_signal/exit_signal_app/manifest.json +++ b/tests/ten_runtime/integration/go/exit_signal/exit_signal_app/manifest.json @@ -3,7 +3,7 @@ { "type": "extension", "name": "default_extension_go", - "version": "0.1.0" + "version": "0.3.0-alpha" } ] } \ No newline at end of file diff --git a/tests/ten_runtime/integration/go/exit_signal/exit_signal_app/ten_packages/extension/default_extension_go/default_extension.go b/tests/ten_runtime/integration/go/exit_signal/exit_signal_app/ten_packages/extension/default_extension_go/extension.go similarity index 97% rename from tests/ten_runtime/integration/go/exit_signal/exit_signal_app/ten_packages/extension/default_extension_go/default_extension.go rename to tests/ten_runtime/integration/go/exit_signal/exit_signal_app/ten_packages/extension/default_extension_go/extension.go index 0f63fd57d6..461de33ef4 100644 --- a/tests/ten_runtime/integration/go/exit_signal/exit_signal_app/ten_packages/extension/default_extension_go/default_extension.go +++ b/tests/ten_runtime/integration/go/exit_signal/exit_signal_app/ten_packages/extension/default_extension_go/extension.go @@ -1,9 +1,12 @@ +// // Copyright © 2024 Agora // This file is part of TEN Framework, an open source project. // Licensed under the Apache License, Version 2.0, with certain conditions. // Refer to https://github.com/TEN-framework/ten_framework/LICENSE for more // information. -package defaultextension +// + +package default_extension_go import ( "encoding/json" diff --git a/tests/ten_runtime/integration/go/expired_ten_go/BUILD.gn b/tests/ten_runtime/integration/go/expired_ten_go/BUILD.gn index a8524bb0d2..50b3b10df0 100644 --- a/tests/ten_runtime/integration/go/expired_ten_go/BUILD.gn +++ b/tests/ten_runtime/integration/go/expired_ten_go/BUILD.gn @@ -17,7 +17,7 @@ ten_package_test_prepare_app("expired_ten_go_app") { "expired_ten_go_app/property.json", ] - replace_files_after_install_all = [ "expired_ten_go_app/ten_packages/extension/default_extension_go/default_extension.go" ] + replace_files_after_install_all = [ "expired_ten_go_app/ten_packages/extension/default_extension_go/extension.go" ] if (ten_enable_package_manager) { deps = [ diff --git a/tests/ten_runtime/integration/go/expired_ten_go/expired_ten_go_app/manifest.json b/tests/ten_runtime/integration/go/expired_ten_go/expired_ten_go_app/manifest.json index 6326076425..54015e0959 100644 --- a/tests/ten_runtime/integration/go/expired_ten_go/expired_ten_go_app/manifest.json +++ b/tests/ten_runtime/integration/go/expired_ten_go/expired_ten_go_app/manifest.json @@ -3,12 +3,12 @@ { "type": "extension", "name": "default_extension_go", - "version": "0.1.0" + "version": "0.3.0-alpha" }, { "type": "protocol", "name": "msgpack", - "version": "0.1.0" + "version": "0.3.0-alpha" } ] } \ No newline at end of file diff --git a/tests/ten_runtime/integration/go/expired_ten_go/expired_ten_go_app/ten_packages/extension/default_extension_go/default_extension.go b/tests/ten_runtime/integration/go/expired_ten_go/expired_ten_go_app/ten_packages/extension/default_extension_go/extension.go similarity index 90% rename from tests/ten_runtime/integration/go/expired_ten_go/expired_ten_go_app/ten_packages/extension/default_extension_go/default_extension.go rename to tests/ten_runtime/integration/go/expired_ten_go/expired_ten_go_app/ten_packages/extension/default_extension_go/extension.go index be658e4a65..75eecd19dd 100644 --- a/tests/ten_runtime/integration/go/expired_ten_go/expired_ten_go_app/ten_packages/extension/default_extension_go/default_extension.go +++ b/tests/ten_runtime/integration/go/expired_ten_go/expired_ten_go_app/ten_packages/extension/default_extension_go/extension.go @@ -1,13 +1,12 @@ +// // Copyright © 2024 Agora // This file is part of TEN Framework, an open source project. // Licensed under the Apache License, Version 2.0, with certain conditions. // Refer to https://github.com/TEN-framework/ten_framework/LICENSE for more // information. // -// Note that this is just an example extension written in the GO programming -// language, so the package name does not equal to the containing directory -// name. However, it is not common in Go. -package defaultextension + +package default_extension_go import ( "fmt" diff --git a/tests/ten_runtime/integration/go/frequently_cgo_call_go/BUILD.gn b/tests/ten_runtime/integration/go/frequently_cgo_call_go/BUILD.gn index 898310baed..596eea0a8f 100644 --- a/tests/ten_runtime/integration/go/frequently_cgo_call_go/BUILD.gn +++ b/tests/ten_runtime/integration/go/frequently_cgo_call_go/BUILD.gn @@ -17,7 +17,7 @@ ten_package_test_prepare_app("frequently_cgo_call_go_app") { "frequently_cgo_call_go_app/property.json", ] - replace_files_after_install_all = [ "frequently_cgo_call_go_app/ten_packages/extension/default_extension_go/default_extension.go" ] + replace_files_after_install_all = [ "frequently_cgo_call_go_app/ten_packages/extension/default_extension_go/extension.go" ] if (ten_enable_package_manager) { deps = [ diff --git a/tests/ten_runtime/integration/go/frequently_cgo_call_go/frequently_cgo_call_go_app/manifest.json b/tests/ten_runtime/integration/go/frequently_cgo_call_go/frequently_cgo_call_go_app/manifest.json index 6326076425..54015e0959 100644 --- a/tests/ten_runtime/integration/go/frequently_cgo_call_go/frequently_cgo_call_go_app/manifest.json +++ b/tests/ten_runtime/integration/go/frequently_cgo_call_go/frequently_cgo_call_go_app/manifest.json @@ -3,12 +3,12 @@ { "type": "extension", "name": "default_extension_go", - "version": "0.1.0" + "version": "0.3.0-alpha" }, { "type": "protocol", "name": "msgpack", - "version": "0.1.0" + "version": "0.3.0-alpha" } ] } \ No newline at end of file diff --git a/tests/ten_runtime/integration/go/frequently_cgo_call_go/frequently_cgo_call_go_app/ten_packages/extension/default_extension_go/default_extension.go b/tests/ten_runtime/integration/go/frequently_cgo_call_go/frequently_cgo_call_go_app/ten_packages/extension/default_extension_go/extension.go similarity index 95% rename from tests/ten_runtime/integration/go/frequently_cgo_call_go/frequently_cgo_call_go_app/ten_packages/extension/default_extension_go/default_extension.go rename to tests/ten_runtime/integration/go/frequently_cgo_call_go/frequently_cgo_call_go_app/ten_packages/extension/default_extension_go/extension.go index dd363be109..63105821e1 100644 --- a/tests/ten_runtime/integration/go/frequently_cgo_call_go/frequently_cgo_call_go_app/ten_packages/extension/default_extension_go/default_extension.go +++ b/tests/ten_runtime/integration/go/frequently_cgo_call_go/frequently_cgo_call_go_app/ten_packages/extension/default_extension_go/extension.go @@ -5,10 +5,7 @@ // Refer to the "LICENSE" file in the root directory for more information. // -// Package defaultextension is just an example extension written in the GO -// programming language, so the package name does not equal to the containing -// directory name. However, it is not common in Go. -package defaultextension +package default_extension_go import ( "fmt" diff --git a/tests/ten_runtime/integration/go/handle_error_go/BUILD.gn b/tests/ten_runtime/integration/go/handle_error_go/BUILD.gn index c0a649d3e2..7f88fe245b 100644 --- a/tests/ten_runtime/integration/go/handle_error_go/BUILD.gn +++ b/tests/ten_runtime/integration/go/handle_error_go/BUILD.gn @@ -17,7 +17,7 @@ ten_package_test_prepare_app("handle_error_go_app") { "handle_error_go_app/property.json", ] - replace_files_after_install_all = [ "handle_error_go_app/ten_packages/extension/default_extension_go/default_extension.go" ] + replace_files_after_install_all = [ "handle_error_go_app/ten_packages/extension/default_extension_go/extension.go" ] if (ten_enable_package_manager) { deps = [ diff --git a/tests/ten_runtime/integration/go/handle_error_go/handle_error_go_app/manifest.json b/tests/ten_runtime/integration/go/handle_error_go/handle_error_go_app/manifest.json index 6326076425..54015e0959 100644 --- a/tests/ten_runtime/integration/go/handle_error_go/handle_error_go_app/manifest.json +++ b/tests/ten_runtime/integration/go/handle_error_go/handle_error_go_app/manifest.json @@ -3,12 +3,12 @@ { "type": "extension", "name": "default_extension_go", - "version": "0.1.0" + "version": "0.3.0-alpha" }, { "type": "protocol", "name": "msgpack", - "version": "0.1.0" + "version": "0.3.0-alpha" } ] } \ No newline at end of file diff --git a/tests/ten_runtime/integration/go/handle_error_go/handle_error_go_app/ten_packages/extension/default_extension_go/default_extension.go b/tests/ten_runtime/integration/go/handle_error_go/handle_error_go_app/ten_packages/extension/default_extension_go/extension.go similarity index 91% rename from tests/ten_runtime/integration/go/handle_error_go/handle_error_go_app/ten_packages/extension/default_extension_go/default_extension.go rename to tests/ten_runtime/integration/go/handle_error_go/handle_error_go_app/ten_packages/extension/default_extension_go/extension.go index da0d58bd32..f4b9e0a03a 100644 --- a/tests/ten_runtime/integration/go/handle_error_go/handle_error_go_app/ten_packages/extension/default_extension_go/default_extension.go +++ b/tests/ten_runtime/integration/go/handle_error_go/handle_error_go_app/ten_packages/extension/default_extension_go/extension.go @@ -1,13 +1,12 @@ +// // Copyright © 2024 Agora // This file is part of TEN Framework, an open source project. // Licensed under the Apache License, Version 2.0, with certain conditions. // Refer to https://github.com/TEN-framework/ten_framework/LICENSE for more // information. // -// Note that this is just an example extension written in the GO programming -// language, so the package name does not equal to the containing directory -// name. However, it is not common in Go. -package defaultextension + +package default_extension_go import ( "fmt" diff --git a/tests/ten_runtime/integration/go/prepare_to_stop_go/BUILD.gn b/tests/ten_runtime/integration/go/prepare_to_stop_go/BUILD.gn index db7c7077f7..d2612df871 100644 --- a/tests/ten_runtime/integration/go/prepare_to_stop_go/BUILD.gn +++ b/tests/ten_runtime/integration/go/prepare_to_stop_go/BUILD.gn @@ -17,7 +17,7 @@ ten_package_test_prepare_app("prepare_to_stop_go_app") { "prepare_to_stop_go_app/property.json", ] - replace_files_after_install_all = [ "prepare_to_stop_go_app/ten_packages/extension/default_extension_go/default_extension.go" ] + replace_files_after_install_all = [ "prepare_to_stop_go_app/ten_packages/extension/default_extension_go/extension.go" ] if (ten_enable_package_manager) { deps = [ diff --git a/tests/ten_runtime/integration/go/prepare_to_stop_go/prepare_to_stop_go_app/manifest.json b/tests/ten_runtime/integration/go/prepare_to_stop_go/prepare_to_stop_go_app/manifest.json index 6326076425..54015e0959 100644 --- a/tests/ten_runtime/integration/go/prepare_to_stop_go/prepare_to_stop_go_app/manifest.json +++ b/tests/ten_runtime/integration/go/prepare_to_stop_go/prepare_to_stop_go_app/manifest.json @@ -3,12 +3,12 @@ { "type": "extension", "name": "default_extension_go", - "version": "0.1.0" + "version": "0.3.0-alpha" }, { "type": "protocol", "name": "msgpack", - "version": "0.1.0" + "version": "0.3.0-alpha" } ] } \ No newline at end of file diff --git a/tests/ten_runtime/integration/go/prepare_to_stop_go/prepare_to_stop_go_app/ten_packages/extension/default_extension_go/default_extension.go b/tests/ten_runtime/integration/go/prepare_to_stop_go/prepare_to_stop_go_app/ten_packages/extension/default_extension_go/extension.go similarity index 99% rename from tests/ten_runtime/integration/go/prepare_to_stop_go/prepare_to_stop_go_app/ten_packages/extension/default_extension_go/default_extension.go rename to tests/ten_runtime/integration/go/prepare_to_stop_go/prepare_to_stop_go_app/ten_packages/extension/default_extension_go/extension.go index 132bccffa0..46c084e453 100644 --- a/tests/ten_runtime/integration/go/prepare_to_stop_go/prepare_to_stop_go_app/ten_packages/extension/default_extension_go/default_extension.go +++ b/tests/ten_runtime/integration/go/prepare_to_stop_go/prepare_to_stop_go_app/ten_packages/extension/default_extension_go/extension.go @@ -7,7 +7,7 @@ // Note that this is just an example extension written in the GO programming // language, so the package name does not equal to the containing directory // name. However, it is not common in Go. -package defaultextension +package default_extension_go import ( "fmt" diff --git a/tests/ten_runtime/integration/go/return_result_go/BUILD.gn b/tests/ten_runtime/integration/go/return_result_go/BUILD.gn index 7cd389b294..8a79222a7e 100644 --- a/tests/ten_runtime/integration/go/return_result_go/BUILD.gn +++ b/tests/ten_runtime/integration/go/return_result_go/BUILD.gn @@ -17,7 +17,7 @@ ten_package_test_prepare_app("return_result_go_app") { "return_result_go_app/property.json", ] - replace_files_after_install_all = [ "return_result_go_app/ten_packages/extension/default_extension_go/default_extension.go" ] + replace_files_after_install_all = [ "return_result_go_app/ten_packages/extension/default_extension_go/extension.go" ] if (ten_enable_package_manager) { deps = [ diff --git a/tests/ten_runtime/integration/go/return_result_go/return_result_go_app/manifest.json b/tests/ten_runtime/integration/go/return_result_go/return_result_go_app/manifest.json index 6326076425..54015e0959 100644 --- a/tests/ten_runtime/integration/go/return_result_go/return_result_go_app/manifest.json +++ b/tests/ten_runtime/integration/go/return_result_go/return_result_go_app/manifest.json @@ -3,12 +3,12 @@ { "type": "extension", "name": "default_extension_go", - "version": "0.1.0" + "version": "0.3.0-alpha" }, { "type": "protocol", "name": "msgpack", - "version": "0.1.0" + "version": "0.3.0-alpha" } ] } \ No newline at end of file diff --git a/tests/ten_runtime/integration/go/return_result_go/return_result_go_app/ten_packages/extension/default_extension_go/default_extension.go b/tests/ten_runtime/integration/go/return_result_go/return_result_go_app/ten_packages/extension/default_extension_go/extension.go similarity index 95% rename from tests/ten_runtime/integration/go/return_result_go/return_result_go_app/ten_packages/extension/default_extension_go/default_extension.go rename to tests/ten_runtime/integration/go/return_result_go/return_result_go_app/ten_packages/extension/default_extension_go/extension.go index 243334cf9b..f8677c5fa9 100644 --- a/tests/ten_runtime/integration/go/return_result_go/return_result_go_app/ten_packages/extension/default_extension_go/default_extension.go +++ b/tests/ten_runtime/integration/go/return_result_go/return_result_go_app/ten_packages/extension/default_extension_go/extension.go @@ -7,7 +7,7 @@ // Note that this is just an example extension written in the GO programming // language, so the package name does not equal to the containing directory // name. However, it is not common in Go. -package defaultextension +package default_extension_go import ( "fmt" @@ -84,8 +84,8 @@ func (p *extensionB) OnCmd( ) if err != nil { cmdResult, _ := ten.NewCmdResult(ten.StatusCodeError) - cmdResult.SetPropertyString("detail", err.Error()) - tenEnv.ReturnResult(cmdResult, cmd) + cmdResult.SetPropertyString("detail", err.Error()) + tenEnv.ReturnResult(cmdResult, cmd) return } diff --git a/tests/ten_runtime/integration/go/return_value_go/BUILD.gn b/tests/ten_runtime/integration/go/return_value_go/BUILD.gn index 0a3f1a2e01..2e42603264 100644 --- a/tests/ten_runtime/integration/go/return_value_go/BUILD.gn +++ b/tests/ten_runtime/integration/go/return_value_go/BUILD.gn @@ -17,7 +17,7 @@ ten_package_test_prepare_app("return_value_go_app") { "return_value_go_app/property.json", ] - replace_files_after_install_all = [ "return_value_go_app/ten_packages/extension/default_extension_go/default_extension.go" ] + replace_files_after_install_all = [ "return_value_go_app/ten_packages/extension/default_extension_go/extension.go" ] if (ten_enable_package_manager) { deps = [ diff --git a/tests/ten_runtime/integration/go/return_value_go/return_value_go_app/manifest.json b/tests/ten_runtime/integration/go/return_value_go/return_value_go_app/manifest.json index 6326076425..54015e0959 100644 --- a/tests/ten_runtime/integration/go/return_value_go/return_value_go_app/manifest.json +++ b/tests/ten_runtime/integration/go/return_value_go/return_value_go_app/manifest.json @@ -3,12 +3,12 @@ { "type": "extension", "name": "default_extension_go", - "version": "0.1.0" + "version": "0.3.0-alpha" }, { "type": "protocol", "name": "msgpack", - "version": "0.1.0" + "version": "0.3.0-alpha" } ] } \ No newline at end of file diff --git a/tests/ten_runtime/integration/go/return_value_go/return_value_go_app/ten_packages/extension/default_extension_go/default_extension.go b/tests/ten_runtime/integration/go/return_value_go/return_value_go_app/ten_packages/extension/default_extension_go/extension.go similarity index 98% rename from tests/ten_runtime/integration/go/return_value_go/return_value_go_app/ten_packages/extension/default_extension_go/default_extension.go rename to tests/ten_runtime/integration/go/return_value_go/return_value_go_app/ten_packages/extension/default_extension_go/extension.go index 8d6d28e548..9a171e05a2 100644 --- a/tests/ten_runtime/integration/go/return_value_go/return_value_go_app/ten_packages/extension/default_extension_go/default_extension.go +++ b/tests/ten_runtime/integration/go/return_value_go/return_value_go_app/ten_packages/extension/default_extension_go/extension.go @@ -7,7 +7,7 @@ // Note that this is just an example extension written in the GO programming // language, so the package name does not equal to the containing directory // name. However, it is not common in Go. -package defaultextension +package default_extension_go import ( "fmt" diff --git a/tests/ten_runtime/integration/go/send_json_go/BUILD.gn b/tests/ten_runtime/integration/go/send_json_go/BUILD.gn index 10b83fa141..3043f4cd99 100644 --- a/tests/ten_runtime/integration/go/send_json_go/BUILD.gn +++ b/tests/ten_runtime/integration/go/send_json_go/BUILD.gn @@ -17,7 +17,9 @@ ten_package_test_prepare_app("send_json_go_app") { "send_json_go_app/property.json", ] - replace_files_after_install_all = [ "send_json_go_app/ten_packages/extension/default_extension_go/default_extension.go" ] + replace_files_after_install_all = [ + "send_json_go_app/ten_packages/extension/default_extension_go/extension.go", + ] if (ten_enable_package_manager) { deps = [ diff --git a/tests/ten_runtime/integration/go/send_json_go/send_json_go_app/manifest.json b/tests/ten_runtime/integration/go/send_json_go/send_json_go_app/manifest.json index 6326076425..54015e0959 100644 --- a/tests/ten_runtime/integration/go/send_json_go/send_json_go_app/manifest.json +++ b/tests/ten_runtime/integration/go/send_json_go/send_json_go_app/manifest.json @@ -3,12 +3,12 @@ { "type": "extension", "name": "default_extension_go", - "version": "0.1.0" + "version": "0.3.0-alpha" }, { "type": "protocol", "name": "msgpack", - "version": "0.1.0" + "version": "0.3.0-alpha" } ] } \ No newline at end of file diff --git a/tests/ten_runtime/integration/go/send_json_go/send_json_go_app/ten_packages/extension/default_extension_go/default_extension.go b/tests/ten_runtime/integration/go/send_json_go/send_json_go_app/ten_packages/extension/default_extension_go/extension.go similarity index 93% rename from tests/ten_runtime/integration/go/send_json_go/send_json_go_app/ten_packages/extension/default_extension_go/default_extension.go rename to tests/ten_runtime/integration/go/send_json_go/send_json_go_app/ten_packages/extension/default_extension_go/extension.go index f7d2bdbb23..6aeb48a2fc 100644 --- a/tests/ten_runtime/integration/go/send_json_go/send_json_go_app/ten_packages/extension/default_extension_go/default_extension.go +++ b/tests/ten_runtime/integration/go/send_json_go/send_json_go_app/ten_packages/extension/default_extension_go/extension.go @@ -1,13 +1,12 @@ +// // Copyright © 2024 Agora // This file is part of TEN Framework, an open source project. // Licensed under the Apache License, Version 2.0, with certain conditions. // Refer to https://github.com/TEN-framework/ten_framework/LICENSE for more // information. // -// Note that this is just an example extension written in the GO programming -// language, so the package name does not equal to the containing directory -// name. However, it is not common in Go. -package defaultextension + +package default_extension_go import ( "fmt" diff --git a/tests/ten_runtime/integration/go/start_app_sync_go/client/client.cc b/tests/ten_runtime/integration/go/start_app_sync_go/client/client.cc index 7be7752a15..b33d13d074 100644 --- a/tests/ten_runtime/integration/go/start_app_sync_go/client/client.cc +++ b/tests/ten_runtime/integration/go/start_app_sync_go/client/client.cc @@ -22,18 +22,13 @@ int main(TEN_UNUSED int argc, TEN_UNUSED char **argv) { "app": "msgpack://127.0.0.1:8007/", "graph": "default", "extension_group": "default_extension_group", - "extension": "default_extension" + "extension": "default_extension_go" }] } })"_json); TEN_ASSERT(TEN_STATUS_CODE_OK == resp["_ten"]["status_code"], "Should not happen."); - std::string resp_str = resp["detail"]; - TEN_LOGD("got result: %s", resp_str.c_str()); - TEN_ASSERT(resp_str == std::string("This is default go extension."), - "Should not happen."); - // NOTE the order: client destroy, then connection lost, then nodejs exits delete client; } diff --git a/tests/ten_runtime/integration/go/start_app_sync_go/start_app_sync_go_app/main.go b/tests/ten_runtime/integration/go/start_app_sync_go/start_app_sync_go_app/main.go index 757e83cc83..397220a640 100644 --- a/tests/ten_runtime/integration/go/start_app_sync_go/start_app_sync_go_app/main.go +++ b/tests/ten_runtime/integration/go/start_app_sync_go/start_app_sync_go_app/main.go @@ -18,9 +18,19 @@ type defaultApp struct { func (p *defaultApp) OnDeinit(tenEnv ten.TenEnv) { fmt.Println("DefaultApp onDeinit") + value, _ := tenEnv.GetPropertyString("key") + if value != "value" { + panic("failed to get property.") + } + tenEnv.OnDeinitDone() } +func (p *defaultApp) OnInit(tenEnv ten.TenEnv) { + tenEnv.SetPropertyString("key", "value") + tenEnv.OnInitDone() +} + func appRoutine(app ten.App, stopped chan<- struct{}) { app.Run(false) stopped <- struct{}{} diff --git a/tests/ten_runtime/integration/go/start_app_sync_go/start_app_sync_go_app/manifest.json b/tests/ten_runtime/integration/go/start_app_sync_go/start_app_sync_go_app/manifest.json index 6326076425..54015e0959 100644 --- a/tests/ten_runtime/integration/go/start_app_sync_go/start_app_sync_go_app/manifest.json +++ b/tests/ten_runtime/integration/go/start_app_sync_go/start_app_sync_go_app/manifest.json @@ -3,12 +3,12 @@ { "type": "extension", "name": "default_extension_go", - "version": "0.1.0" + "version": "0.3.0-alpha" }, { "type": "protocol", "name": "msgpack", - "version": "0.1.0" + "version": "0.3.0-alpha" } ] } \ No newline at end of file diff --git a/tests/ten_runtime/integration/go/start_app_sync_go/start_app_sync_go_app/property.json b/tests/ten_runtime/integration/go/start_app_sync_go/start_app_sync_go_app/property.json index d50a94ecf8..068bc7abff 100644 --- a/tests/ten_runtime/integration/go/start_app_sync_go/start_app_sync_go_app/property.json +++ b/tests/ten_runtime/integration/go/start_app_sync_go/start_app_sync_go_app/property.json @@ -15,7 +15,7 @@ }, { "type": "extension", - "name": "default_extension", + "name": "default_extension_go", "addon": "default_extension_go", "extension_group": "default_extension_group" } diff --git a/tests/ten_runtime/integration/go/three_extension_cmd_go/BUILD.gn b/tests/ten_runtime/integration/go/three_extension_cmd_go/BUILD.gn index a81540af54..62fce54c6f 100644 --- a/tests/ten_runtime/integration/go/three_extension_cmd_go/BUILD.gn +++ b/tests/ten_runtime/integration/go/three_extension_cmd_go/BUILD.gn @@ -17,7 +17,7 @@ ten_package_test_prepare_app("three_extension_cmd_go_app") { "three_extension_cmd_go_app/property.json", ] - replace_files_after_install_all = [ "three_extension_cmd_go_app/ten_packages/extension/default_extension_go/default_extension.go" ] + replace_files_after_install_all = [ "three_extension_cmd_go_app/ten_packages/extension/default_extension_go/extension.go" ] if (ten_enable_package_manager) { deps = [ diff --git a/tests/ten_runtime/integration/go/three_extension_cmd_go/three_extension_cmd_go_app/manifest.json b/tests/ten_runtime/integration/go/three_extension_cmd_go/three_extension_cmd_go_app/manifest.json index 6326076425..54015e0959 100644 --- a/tests/ten_runtime/integration/go/three_extension_cmd_go/three_extension_cmd_go_app/manifest.json +++ b/tests/ten_runtime/integration/go/three_extension_cmd_go/three_extension_cmd_go_app/manifest.json @@ -3,12 +3,12 @@ { "type": "extension", "name": "default_extension_go", - "version": "0.1.0" + "version": "0.3.0-alpha" }, { "type": "protocol", "name": "msgpack", - "version": "0.1.0" + "version": "0.3.0-alpha" } ] } \ No newline at end of file diff --git a/tests/ten_runtime/integration/go/three_extension_cmd_go/three_extension_cmd_go_app/ten_packages/extension/default_extension_go/default_extension.go b/tests/ten_runtime/integration/go/three_extension_cmd_go/three_extension_cmd_go_app/ten_packages/extension/default_extension_go/extension.go similarity index 99% rename from tests/ten_runtime/integration/go/three_extension_cmd_go/three_extension_cmd_go_app/ten_packages/extension/default_extension_go/default_extension.go rename to tests/ten_runtime/integration/go/three_extension_cmd_go/three_extension_cmd_go_app/ten_packages/extension/default_extension_go/extension.go index 90a24f64d3..940f29fecd 100644 --- a/tests/ten_runtime/integration/go/three_extension_cmd_go/three_extension_cmd_go_app/ten_packages/extension/default_extension_go/default_extension.go +++ b/tests/ten_runtime/integration/go/three_extension_cmd_go/three_extension_cmd_go_app/ten_packages/extension/default_extension_go/extension.go @@ -7,7 +7,7 @@ // Note that this is just an example extension written in the GO programming // language, so the package name does not equal to the containing directory // name. However, it is not common in Go. -package defaultextension +package default_extension_go import ( "encoding/json" diff --git a/tests/ten_runtime/integration/go/transfer_pointer_go/BUILD.gn b/tests/ten_runtime/integration/go/transfer_pointer_go/BUILD.gn index d64f01c285..29406a1447 100644 --- a/tests/ten_runtime/integration/go/transfer_pointer_go/BUILD.gn +++ b/tests/ten_runtime/integration/go/transfer_pointer_go/BUILD.gn @@ -17,7 +17,7 @@ ten_package_test_prepare_app("transfer_pointer_go_app") { "transfer_pointer_go_app/property.json", ] - replace_files_after_install_all = [ "transfer_pointer_go_app/ten_packages/extension/default_extension_go/default_extension.go" ] + replace_files_after_install_all = [ "transfer_pointer_go_app/ten_packages/extension/default_extension_go/extension.go" ] if (ten_enable_package_manager) { deps = [ diff --git a/tests/ten_runtime/integration/go/transfer_pointer_go/transfer_pointer_go_app/manifest.json b/tests/ten_runtime/integration/go/transfer_pointer_go/transfer_pointer_go_app/manifest.json index 6326076425..54015e0959 100644 --- a/tests/ten_runtime/integration/go/transfer_pointer_go/transfer_pointer_go_app/manifest.json +++ b/tests/ten_runtime/integration/go/transfer_pointer_go/transfer_pointer_go_app/manifest.json @@ -3,12 +3,12 @@ { "type": "extension", "name": "default_extension_go", - "version": "0.1.0" + "version": "0.3.0-alpha" }, { "type": "protocol", "name": "msgpack", - "version": "0.1.0" + "version": "0.3.0-alpha" } ] } \ No newline at end of file diff --git a/tests/ten_runtime/integration/go/transfer_pointer_go/transfer_pointer_go_app/ten_packages/extension/default_extension_go/default_extension.go b/tests/ten_runtime/integration/go/transfer_pointer_go/transfer_pointer_go_app/ten_packages/extension/default_extension_go/extension.go similarity index 99% rename from tests/ten_runtime/integration/go/transfer_pointer_go/transfer_pointer_go_app/ten_packages/extension/default_extension_go/default_extension.go rename to tests/ten_runtime/integration/go/transfer_pointer_go/transfer_pointer_go_app/ten_packages/extension/default_extension_go/extension.go index 922416e8bf..537f4d724a 100644 --- a/tests/ten_runtime/integration/go/transfer_pointer_go/transfer_pointer_go_app/ten_packages/extension/default_extension_go/default_extension.go +++ b/tests/ten_runtime/integration/go/transfer_pointer_go/transfer_pointer_go_app/ten_packages/extension/default_extension_go/extension.go @@ -7,7 +7,7 @@ // Note that this is just an example extension written in the GO programming // language, so the package name does not equal to the containing directory // name. However, it is not common in Go. -package defaultextension +package default_extension_go import ( "fmt" diff --git a/tests/ten_runtime/integration/go/two_extension_one_group_cmd_go/BUILD.gn b/tests/ten_runtime/integration/go/two_extension_one_group_cmd_go/BUILD.gn index 3b66efc1c2..1df7a578d0 100644 --- a/tests/ten_runtime/integration/go/two_extension_one_group_cmd_go/BUILD.gn +++ b/tests/ten_runtime/integration/go/two_extension_one_group_cmd_go/BUILD.gn @@ -17,7 +17,7 @@ ten_package_test_prepare_app("two_extension_one_group_cmd_go_app") { "two_extension_one_group_cmd_go_app/property.json", ] - replace_files_after_install_all = [ "two_extension_one_group_cmd_go_app/ten_packages/extension/default_extension_go/default_extension.go" ] + replace_files_after_install_all = [ "two_extension_one_group_cmd_go_app/ten_packages/extension/default_extension_go/extension.go" ] if (ten_enable_package_manager) { deps = [ diff --git a/tests/ten_runtime/integration/go/two_extension_one_group_cmd_go/two_extension_one_group_cmd_go_app/manifest.json b/tests/ten_runtime/integration/go/two_extension_one_group_cmd_go/two_extension_one_group_cmd_go_app/manifest.json index 6326076425..54015e0959 100644 --- a/tests/ten_runtime/integration/go/two_extension_one_group_cmd_go/two_extension_one_group_cmd_go_app/manifest.json +++ b/tests/ten_runtime/integration/go/two_extension_one_group_cmd_go/two_extension_one_group_cmd_go_app/manifest.json @@ -3,12 +3,12 @@ { "type": "extension", "name": "default_extension_go", - "version": "0.1.0" + "version": "0.3.0-alpha" }, { "type": "protocol", "name": "msgpack", - "version": "0.1.0" + "version": "0.3.0-alpha" } ] } \ No newline at end of file diff --git a/tests/ten_runtime/integration/go/two_extension_one_group_cmd_go/two_extension_one_group_cmd_go_app/ten_packages/extension/default_extension_go/default_extension.go b/tests/ten_runtime/integration/go/two_extension_one_group_cmd_go/two_extension_one_group_cmd_go_app/ten_packages/extension/default_extension_go/extension.go similarity index 99% rename from tests/ten_runtime/integration/go/two_extension_one_group_cmd_go/two_extension_one_group_cmd_go_app/ten_packages/extension/default_extension_go/default_extension.go rename to tests/ten_runtime/integration/go/two_extension_one_group_cmd_go/two_extension_one_group_cmd_go_app/ten_packages/extension/default_extension_go/extension.go index 8bd9d623e6..e2f156599a 100644 --- a/tests/ten_runtime/integration/go/two_extension_one_group_cmd_go/two_extension_one_group_cmd_go_app/ten_packages/extension/default_extension_go/default_extension.go +++ b/tests/ten_runtime/integration/go/two_extension_one_group_cmd_go/two_extension_one_group_cmd_go_app/ten_packages/extension/default_extension_go/extension.go @@ -7,7 +7,7 @@ // Note that this is just an example extension written in the GO programming // language, so the package name does not equal to the containing directory // name. However, it is not common in Go. -package defaultextension +package default_extension_go import ( "fmt" diff --git a/tests/ten_runtime/integration/pytest.ini b/tests/ten_runtime/integration/pytest.ini index 83a9a6e542..9135415f10 100644 --- a/tests/ten_runtime/integration/pytest.ini +++ b/tests/ten_runtime/integration/pytest.ini @@ -1,6 +1,7 @@ [pytest] addopts = --ignore=tests/ten_runtime/integration/python/async_io_basic_python/async_io_basic_python_app/ten_packages/extension/default_extension_python/tests/ + --ignore=tests/ten_runtime/integration/python/async_extension_basic_python/async_extension_basic_python_app/ten_packages/extension/default_async_extension_python/tests/ --ignore=tests/ten_runtime/integration/python/cpp_app_multi_process_python/cpp_app_multi_process_python_app_source/out/linux/x64/app/cpp_app_multi_process_python_app_source/ten_packages/extension/default_extension_python/tests/ --ignore=tests/ten_runtime/integration/python/cpp_app_multi_process_python/cpp_app_multi_process_python_app_source/out/linux/x64/ten_packages/extension/default_extension_python/tests/ --ignore=tests/ten_runtime/integration/python/cpp_app_multi_process_python/cpp_app_multi_process_python_app_source/ten_packages/extension/default_extension_python/tests/ @@ -11,6 +12,9 @@ addopts = --ignore=tests/ten_runtime/integration/python/cpp_app_python/cpp_app_python_app_source/ten_packages/extension/default_extension_python/tests/ --ignore=tests/ten_runtime/integration/python/get_set_prop_python/get_set_prop_python_app/ten_packages/extension/default_extension_python/tests/ --ignore=tests/ten_runtime/integration/python/go_app_python/go_app_python_app/ten_packages/extension/default_extension_python/tests/ + --ignore=tests/ten_runtime/integration/python/go_app_cythonize/go_app_cythonize_app/ten_packages/extension/default_extension_python/tests/ + --ignore=tests/ten_runtime/integration/python/go_app_partially_cythonize/go_app_partially_cythonize_app/ten_packages/extension/default_extension_python/tests/ + --ignore=tests/ten_runtime/integration/python/go_app_async_extension_python/go_app_async_extension_python_app/ten_packages/extension/default_extension_python/tests/ --ignore=tests/ten_runtime/integration/python/large_json_python/large_json_python_app/ten_packages/extension/default_extension_python/tests/ --ignore=tests/ten_runtime/integration/python/multi_process_python/multi_process_python_app/ten_packages/extension/default_extension_python/tests/ --ignore=tests/ten_runtime/integration/python/multiple_results_python/multiple_results_python_app/ten_packages/extension/default_extension_python/tests/ @@ -22,4 +26,6 @@ addopts = --ignore=tests/ten_runtime/integration/python/send_recv_image_python/send_recv_image_python_app/ten_packages/extension/default_extension_python/tests/ --ignore=tests/ten_runtime/integration/python/send_recv_pcm_python/send_recv_pcm_python_app/ten_packages/extension/default_extension_python/tests/ --ignore=tests/ten_runtime/integration/python/standalone_test_python/default_extension_python/tests/ + --ignore=tests/ten_runtime/integration/python/two_async_exts_one_group_python/two_async_exts_one_group_python_app/ten_packages/extension/default_extension_python/tests/ + --ignore=tests/ten_runtime/integration/python/two_async_exts_python/two_async_exts_python_app/ten_packages/extension/default_extension_python/tests/ --ignore=tests/ten_runtime/integration/python/unused_addon_python/unused_addon_python_app/ten_packages/extension/default_extension_python/tests/ diff --git a/tests/ten_runtime/integration/python/BUILD.gn b/tests/ten_runtime/integration/python/BUILD.gn index dbbdf82922..5ec10dd537 100644 --- a/tests/ten_runtime/integration/python/BUILD.gn +++ b/tests/ten_runtime/integration/python/BUILD.gn @@ -18,10 +18,12 @@ group("python") { if (ten_enable_python_binding) { deps += [ "aio_http_server_python", + "async_extension_basic_python", "async_io_basic_python", "cpp_app_multi_process_python", "cpp_app_python", "get_set_prop_python", + "go_app_async_extension_python", "large_json_python", "multi_process_python", "multiple_results_python", @@ -33,6 +35,8 @@ group("python") { "send_recv_image_python", "send_recv_pcm_python", "standalone_test_python", + "two_async_exts_one_group_python", + "two_async_exts_python", ] if (ten_enable_python_binding && ten_enable_go_binding) { diff --git a/tests/ten_runtime/integration/python/aio_http_server_python/aio_http_server_python_app/manifest.json b/tests/ten_runtime/integration/python/aio_http_server_python/aio_http_server_python_app/manifest.json index b2d01459f0..32db783821 100644 --- a/tests/ten_runtime/integration/python/aio_http_server_python/aio_http_server_python_app/manifest.json +++ b/tests/ten_runtime/integration/python/aio_http_server_python/aio_http_server_python_app/manifest.json @@ -3,7 +3,7 @@ { "type": "system", "name": "ten_runtime", - "version": "0.2.0" + "version": "0.3.0-alpha" }, { "type": "extension", diff --git a/tests/ten_runtime/integration/python/async_extension_basic_python/BUILD.gn b/tests/ten_runtime/integration/python/async_extension_basic_python/BUILD.gn new file mode 100644 index 0000000000..4828c81873 --- /dev/null +++ b/tests/ten_runtime/integration/python/async_extension_basic_python/BUILD.gn @@ -0,0 +1,50 @@ +# +# Copyright © 2024 Agora +# This file is part of TEN Framework, an open source project. +# Licensed under the Apache License, Version 2.0, with certain conditions. +# Refer to the "LICENSE" file in the root directory for more information. +# +import("//build/ten_runtime/feature/test.gni") +import("//build/ten_runtime/ten.gni") + +ten_package_test_prepare_app("async_extension_basic_python_app") { + src_app = "default_app_python" + src_app_language = "python" + generated_app_src_root_dir_name = "async_extension_basic_python_app" + + replace_files_after_install_app = [ + "async_extension_basic_python_app/manifest.json", + "async_extension_basic_python_app/property.json", + ] + + replace_files_after_install_all = [ "async_extension_basic_python_app/ten_packages/extension/default_async_extension_python/extension.py" ] + + if (ten_enable_package_manager) { + deps = [ + "//core/src/ten_manager", + "//packages/core_apps/default_app_python:upload_default_app_python_to_server", + "//packages/core_extensions/default_async_extension_python:upload_default_async_extension_python_to_server", + "//packages/example_extensions/simple_echo_cpp:upload_simple_echo_cpp_to_server", + "//packages/example_extensions/simple_http_server_cpp:upload_simple_http_server_cpp_to_server", + ] + } +} + +ten_package_test_prepare_auxiliary_resources( + "async_extension_basic_python_test_files") { + resources = [ + "//tests/ten_runtime/integration/common=>common", + "__init__.py", + "test_case.py", + ] + if (enable_sanitizer) { + resources += [ "//tests/ten_runtime/integration/tools/use_asan_lib_marker=>use_asan_lib_marker" ] + } +} + +group("async_extension_basic_python") { + deps = [ + ":async_extension_basic_python_app", + ":async_extension_basic_python_test_files", + ] +} diff --git a/tests/ten_runtime/integration/python/async_extension_basic_python/__init__.py b/tests/ten_runtime/integration/python/async_extension_basic_python/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/ten_runtime/integration/python/async_extension_basic_python/async_extension_basic_python_app/manifest.json b/tests/ten_runtime/integration/python/async_extension_basic_python/async_extension_basic_python_app/manifest.json new file mode 100644 index 0000000000..bbb121767b --- /dev/null +++ b/tests/ten_runtime/integration/python/async_extension_basic_python/async_extension_basic_python_app/manifest.json @@ -0,0 +1,24 @@ +{ + "dependencies": [ + { + "type": "system", + "name": "ten_runtime", + "version": "0.3.0-alpha" + }, + { + "type": "extension", + "name": "simple_http_server_cpp", + "version": "0.1.0" + }, + { + "type": "extension", + "name": "default_async_extension_python", + "version": "0.3.0-alpha" + }, + { + "type": "extension", + "name": "simple_echo_cpp", + "version": "0.1.0" + } + ] +} \ No newline at end of file diff --git a/tests/ten_runtime/integration/python/async_extension_basic_python/async_extension_basic_python_app/property.json b/tests/ten_runtime/integration/python/async_extension_basic_python/async_extension_basic_python_app/property.json new file mode 100644 index 0000000000..97b14a4664 --- /dev/null +++ b/tests/ten_runtime/integration/python/async_extension_basic_python/async_extension_basic_python_app/property.json @@ -0,0 +1,76 @@ +{ + "_ten": { + "log_level": 2, + "predefined_graphs": [ + { + "name": "default", + "auto_start": true, + "nodes": [ + { + "type": "extension_group", + "name": "default_extension_group", + "addon": "default_extension_group" + }, + { + "type": "extension_group", + "name": "test", + "addon": "default_extension_group" + }, + { + "type": "extension", + "name": "simple_http_server_cpp", + "addon": "simple_http_server_cpp", + "extension_group": "default_extension_group", + "property": { + "server_port": 8002 + } + }, + { + "type": "extension", + "name": "default_async_extension_python", + "addon": "default_async_extension_python", + "extension_group": "test" + }, + { + "type": "extension", + "name": "simple_echo_cpp", + "addon": "simple_echo_cpp", + "extension_group": "default_extension_group" + } + ], + "connections": [ + { + "extension_group": "default_extension_group", + "extension": "simple_http_server_cpp", + "cmd": [ + { + "name": "test", + "dest": [ + { + "extension_group": "test", + "extension": "default_async_extension_python" + } + ] + } + ] + }, + { + "extension_group": "test", + "extension": "default_async_extension_python", + "cmd": [ + { + "name": "hello", + "dest": [ + { + "extension_group": "default_extension_group", + "extension": "simple_echo_cpp" + } + ] + } + ] + } + ] + } + ] + } +} \ No newline at end of file diff --git a/tests/ten_runtime/integration/python/async_extension_basic_python/async_extension_basic_python_app/ten_packages/extension/default_async_extension_python/extension.py b/tests/ten_runtime/integration/python/async_extension_basic_python/async_extension_basic_python_app/ten_packages/extension/default_async_extension_python/extension.py new file mode 100644 index 0000000000..c0fcc84202 --- /dev/null +++ b/tests/ten_runtime/integration/python/async_extension_basic_python/async_extension_basic_python_app/ten_packages/extension/default_async_extension_python/extension.py @@ -0,0 +1,54 @@ +# +# Copyright © 2024 Agora +# This file is part of TEN Framework, an open source project. +# Licensed under the Apache License, Version 2.0, with certain conditions. +# Refer to the "LICENSE" file in the root directory for more information. +# +import asyncio +from ten import ( + AsyncExtension, + AsyncTenEnv, + Cmd, +) + + +class DefaultAsyncExtension(AsyncExtension): + def __init__(self, name: str) -> None: + super().__init__(name) + self.name = name + + async def on_configure(self, ten_env: AsyncTenEnv) -> None: + await asyncio.sleep(0.5) + ten_env.on_configure_done() + + async def on_init(self, ten_env: AsyncTenEnv) -> None: + await asyncio.sleep(0.5) + ten_env.on_init_done() + + async def on_start(self, ten_env: AsyncTenEnv) -> None: + await asyncio.sleep(0.5) + ten_env.log_debug("on_start") + ten_env.on_start_done() + + async def on_deinit(self, ten_env: AsyncTenEnv) -> None: + await asyncio.sleep(0.5) + ten_env.on_deinit_done() + + async def on_cmd(self, ten_env: AsyncTenEnv, cmd: Cmd) -> None: + cmd_json = cmd.to_json() + ten_env.log_debug(f"on_cmd: {cmd_json}") + + # Mock async operation, e.g. network, file I/O. + await asyncio.sleep(0.5) + + # Send a new command to other extensions and wait for the result. The + # result will be returned to the original sender. + new_cmd = Cmd.create("hello") + cmd_result = await ten_env.send_cmd(new_cmd) + ten_env.return_result(cmd_result, cmd) + + async def on_stop(self, ten_env: AsyncTenEnv) -> None: + ten_env.log_debug("on_stop") + + await asyncio.sleep(0.5) + ten_env.on_stop_done() diff --git a/tests/ten_runtime/integration/python/async_extension_basic_python/test_case.py b/tests/ten_runtime/integration/python/async_extension_basic_python/test_case.py new file mode 100644 index 0000000000..ea089778ab --- /dev/null +++ b/tests/ten_runtime/integration/python/async_extension_basic_python/test_case.py @@ -0,0 +1,136 @@ +""" +Test async_extension_basic_python. +""" + +import subprocess +import os +import sys +from sys import stdout +from .common import http + + +def http_request(): + return http.post( + "http://127.0.0.1:8002/", + { + "_ten": { + "name": "test", + }, + }, + ) + + +def test_async_extension_basic_python(): + """Test client and app server.""" + base_path = os.path.dirname(os.path.abspath(__file__)) + root_dir = os.path.join(base_path, "../../../../../") + + # Create virtual environment. + venv_dir = os.path.join(base_path, "venv") + subprocess.run([sys.executable, "-m", "venv", venv_dir]) + + my_env = os.environ.copy() + + # Set the required environment variables for the test. + my_env["PYTHONMALLOC"] = "malloc" + my_env["PYTHONDEVMODE"] = "1" + + # Launch virtual environment. + my_env["VIRTUAL_ENV"] = venv_dir + my_env["PATH"] = os.path.join(venv_dir, "bin") + os.pathsep + my_env["PATH"] + + if sys.platform == "win32": + print("test_async_extension_basic_python doesn't support win32") + assert False + elif sys.platform == "darwin": + # client depends on some libraries in the TEN app. + my_env["DYLD_LIBRARY_PATH"] = os.path.join( + base_path, "async_extension_basic_python_app/lib" + ) + else: + # client depends on some libraries in the TEN app. + my_env["LD_LIBRARY_PATH"] = os.path.join( + base_path, "async_extension_basic_python_app/lib" + ) + + app_root_path = os.path.join(base_path, "async_extension_basic_python_app") + + tman_install_cmd = [ + os.path.join(root_dir, "ten_manager/bin/tman"), + "--config-file", + os.path.join(root_dir, "tests/local_registry/config.json"), + "install", + ] + + tman_install_process = subprocess.Popen( + tman_install_cmd, + stdout=stdout, + stderr=subprocess.STDOUT, + env=my_env, + cwd=app_root_path, + ) + tman_install_process.wait() + + bootstrap_cmd = os.path.join( + base_path, "async_extension_basic_python_app/bin/bootstrap" + ) + + bootstrap_process = subprocess.Popen( + bootstrap_cmd, stdout=stdout, stderr=subprocess.STDOUT, env=my_env + ) + bootstrap_process.wait() + + if sys.platform == "linux": + if os.path.exists(os.path.join(base_path, "use_asan_lib_marker")): + libasan_path = os.path.join( + base_path, + "async_extension_basic_python_app/ten_packages/system/ten_runtime/lib/libasan.so", + ) + + if os.path.exists(libasan_path): + my_env["LD_PRELOAD"] = libasan_path + + server_cmd = os.path.join( + base_path, "async_extension_basic_python_app/bin/start" + ) + + server = subprocess.Popen( + server_cmd, + stdout=stdout, + stderr=subprocess.STDOUT, + env=my_env, + cwd=app_root_path, + ) + + is_started = http.is_app_started("127.0.0.1", 8002, 30) + if not is_started: + print( + "The async_extension_basic_python is not started after 30 seconds." + ) + + server.kill() + exit_code = server.wait() + print("The exit code of async_extension_basic_python: ", exit_code) + + assert exit_code == 0 + assert 0 + + return + + try: + resp = http_request() + assert resp != 500 + print(resp) + + finally: + is_stopped = http.stop_app("127.0.0.1", 8002, 30) + if not is_stopped: + print( + "The async_extension_basic_python can not stop after 30 seconds." + ) + server.kill() + + exit_code = server.wait() + print("The exit code of async_extension_basic_python: ", exit_code) + + assert exit_code == 0 diff --git a/tests/ten_runtime/integration/python/async_io_basic_python/async_io_basic_python_app/manifest.json b/tests/ten_runtime/integration/python/async_io_basic_python/async_io_basic_python_app/manifest.json index 7bd9b30f55..6d61860568 100644 --- a/tests/ten_runtime/integration/python/async_io_basic_python/async_io_basic_python_app/manifest.json +++ b/tests/ten_runtime/integration/python/async_io_basic_python/async_io_basic_python_app/manifest.json @@ -3,7 +3,7 @@ { "type": "system", "name": "ten_runtime", - "version": "0.2.0" + "version": "0.3.0-alpha" }, { "type": "extension", @@ -13,7 +13,7 @@ { "type": "extension", "name": "default_extension_python", - "version": "0.1.0" + "version": "0.3.0-alpha" }, { "type": "extension", diff --git a/tests/ten_runtime/integration/python/cpp_app_multi_process_python/cpp_app_multi_process_python_app_source/manifest.json b/tests/ten_runtime/integration/python/cpp_app_multi_process_python/cpp_app_multi_process_python_app_source/manifest.json index e6ab9dc144..4bd4463d21 100644 --- a/tests/ten_runtime/integration/python/cpp_app_multi_process_python/cpp_app_multi_process_python_app_source/manifest.json +++ b/tests/ten_runtime/integration/python/cpp_app_multi_process_python/cpp_app_multi_process_python_app_source/manifest.json @@ -3,12 +3,12 @@ { "type": "system", "name": "ten_runtime", - "version": "0.2.0" + "version": "0.3.0-alpha" }, { "type": "system", "name": "ten_runtime_python", - "version": "0.1.0" + "version": "0.3.0-alpha" }, { "type": "extension", @@ -18,12 +18,12 @@ { "type": "extension", "name": "default_extension_python", - "version": "0.1.0" + "version": "0.3.0-alpha" }, { "type": "extension", "name": "py_init_extension_cpp", - "version": "0.1.0" + "version": "0.3.0-alpha" }, { "type": "extension", diff --git a/tests/ten_runtime/integration/python/cpp_app_python/cpp_app_python_app_source/manifest.json b/tests/ten_runtime/integration/python/cpp_app_python/cpp_app_python_app_source/manifest.json index dfd6c8c414..909b1e5295 100644 --- a/tests/ten_runtime/integration/python/cpp_app_python/cpp_app_python_app_source/manifest.json +++ b/tests/ten_runtime/integration/python/cpp_app_python/cpp_app_python_app_source/manifest.json @@ -3,12 +3,12 @@ { "type": "system", "name": "ten_runtime", - "version": "0.2.0" + "version": "0.3.0-alpha" }, { "type": "system", "name": "ten_runtime_python", - "version": "0.1.0" + "version": "0.3.0-alpha" }, { "type": "extension", @@ -23,12 +23,12 @@ { "type": "extension", "name": "default_extension_python", - "version": "0.1.0" + "version": "0.3.0-alpha" }, { "type": "extension", "name": "py_init_extension_cpp", - "version": "0.1.0" + "version": "0.3.0-alpha" }, { "type": "extension", diff --git a/tests/ten_runtime/integration/python/get_set_prop_python/get_set_prop_python_app/manifest.json b/tests/ten_runtime/integration/python/get_set_prop_python/get_set_prop_python_app/manifest.json index 7bd9b30f55..6d61860568 100644 --- a/tests/ten_runtime/integration/python/get_set_prop_python/get_set_prop_python_app/manifest.json +++ b/tests/ten_runtime/integration/python/get_set_prop_python/get_set_prop_python_app/manifest.json @@ -3,7 +3,7 @@ { "type": "system", "name": "ten_runtime", - "version": "0.2.0" + "version": "0.3.0-alpha" }, { "type": "extension", @@ -13,7 +13,7 @@ { "type": "extension", "name": "default_extension_python", - "version": "0.1.0" + "version": "0.3.0-alpha" }, { "type": "extension", diff --git a/tests/ten_runtime/integration/python/go_app_async_extension_python/BUILD.gn b/tests/ten_runtime/integration/python/go_app_async_extension_python/BUILD.gn new file mode 100644 index 0000000000..462fc2cb4a --- /dev/null +++ b/tests/ten_runtime/integration/python/go_app_async_extension_python/BUILD.gn @@ -0,0 +1,55 @@ +# +# Copyright © 2024 Agora +# This file is part of TEN Framework, an open source project. +# Licensed under the Apache License, Version 2.0, with certain conditions. +# Refer to the "LICENSE" file in the root directory for more information. +# +import("//build/ten_runtime/feature/test.gni") +import("//build/ten_runtime/ten.gni") + +ten_package_test_prepare_app("go_app_async_extension_python_app") { + src_app = "default_app_go" + src_app_language = "go" + generated_app_src_root_dir_name = "go_app_async_extension_python_app" + + replace_files_after_install_app = [ + "go_app_async_extension_python_app/manifest.json", + "go_app_async_extension_python_app/property.json", + ] + + replace_files_after_install_all = [ + "go_app_async_extension_python_app/ten_packages/extension/default_extension_python/extension.py", + "go_app_async_extension_python_app/bin", + ] + + if (ten_enable_package_manager) { + deps = [ + "//core/src/ten_manager", + "//packages/core_apps/default_app_go:upload_default_app_go_to_server", + "//packages/core_extensions/default_extension_go:upload_default_extension_go_to_server", + "//packages/core_extensions/default_extension_python:upload_default_extension_python_to_server", + "//packages/core_extensions/py_init_extension_cpp:upload_py_init_extension_cpp_to_server", + "//packages/example_extensions/pil_demo_python:upload_pil_demo_python_to_server", + "//packages/example_extensions/simple_echo_cpp:upload_simple_echo_cpp_to_server", + "//packages/example_extensions/simple_http_server_cpp:upload_simple_http_server_cpp_to_server", + ] + } +} + +ten_package_test_prepare_auxiliary_resources("go_app_async_extension_python_test_files") { + resources = [ + "//tests/ten_runtime/integration/common=>common", + "__init__.py", + "test_case.py", + ] + if (enable_sanitizer && !is_clang) { + resources += [ "//tests/ten_runtime/integration/tools/use_asan_lib_marker=>use_asan_lib_marker" ] + } +} + +group("go_app_async_extension_python") { + deps = [ + ":go_app_async_extension_python_app", + ":go_app_async_extension_python_test_files", + ] +} diff --git a/tests/ten_runtime/integration/python/go_app_async_extension_python/__init__.py b/tests/ten_runtime/integration/python/go_app_async_extension_python/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/ten_runtime/integration/python/go_app_async_extension_python/go_app_async_extension_python_app/bin/bootstrap b/tests/ten_runtime/integration/python/go_app_async_extension_python/go_app_async_extension_python_app/bin/bootstrap new file mode 100755 index 0000000000..7ae31c9abb --- /dev/null +++ b/tests/ten_runtime/integration/python/go_app_async_extension_python/go_app_async_extension_python_app/bin/bootstrap @@ -0,0 +1,24 @@ +#!/bin/bash + +set -e + +cd "$(dirname "${BASH_SOURCE[0]}")/.." + +# Resolve the dependencies of the Python app and generate the 'merged_requirements.txt' file. +# get the result code of the following python script. +python3 ten_packages/system/ten_runtime_python/tools/deps_resolver.py -i https://pypi.tuna.tsinghua.edu.cn/simple/ +result_code=$? + +# If the Python script is executed failed, exit the script. +if [[ $result_code -ne 0 ]]; then + echo "Error: Failed to resolve the dependencies of the Python app. Cannot proceed to start." + exit $result_code +fi + +if [[ -f "merged_requirements.txt" ]]; then + echo "The 'merged_requirements.txt' file is generated successfully." + # Pip install the dependencies in the 'merged_requirements.txt' file. + pip install -r merged_requirements.txt +else + echo "No 'merged_requirements.txt' file is generated, because there are no dependencies." +fi diff --git a/tests/ten_runtime/integration/python/go_app_async_extension_python/go_app_async_extension_python_app/manifest.json b/tests/ten_runtime/integration/python/go_app_async_extension_python/go_app_async_extension_python_app/manifest.json new file mode 100644 index 0000000000..c32778b6f5 --- /dev/null +++ b/tests/ten_runtime/integration/python/go_app_async_extension_python/go_app_async_extension_python_app/manifest.json @@ -0,0 +1,44 @@ +{ + "dependencies": [ + { + "type": "system", + "name": "ten_runtime", + "version": "0.3.0-alpha" + }, + { + "type": "system", + "name": "ten_runtime_python", + "version": "0.3.0-alpha" + }, + { + "type": "extension", + "name": "default_extension_go", + "version": "0.3.0-alpha" + }, + { + "type": "extension", + "name": "simple_http_server_cpp", + "version": "0.1.0" + }, + { + "type": "extension", + "name": "pil_demo_python", + "version": "0.1.0" + }, + { + "type": "extension", + "name": "default_extension_python", + "version": "0.3.0-alpha" + }, + { + "type": "extension", + "name": "py_init_extension_cpp", + "version": "0.3.0-alpha" + }, + { + "type": "extension", + "name": "simple_echo_cpp", + "version": "0.1.0" + } + ] +} \ No newline at end of file diff --git a/tests/ten_runtime/integration/python/go_app_async_extension_python/go_app_async_extension_python_app/property.json b/tests/ten_runtime/integration/python/go_app_async_extension_python/go_app_async_extension_python_app/property.json new file mode 100644 index 0000000000..3631a072e9 --- /dev/null +++ b/tests/ten_runtime/integration/python/go_app_async_extension_python/go_app_async_extension_python_app/property.json @@ -0,0 +1,92 @@ +{ + "_ten": { + "log_level": 2, + "predefined_graphs": [ + { + "name": "default", + "auto_start": true, + "nodes": [ + { + "type": "extension_group", + "name": "default_extension_group", + "addon": "default_extension_group" + }, + { + "type": "extension", + "name": "simple_http_server_cpp", + "addon": "simple_http_server_cpp", + "extension_group": "default_extension_group", + "property": { + "server_port": 8002 + } + }, + { + "type": "extension", + "name": "default_extension_python", + "addon": "default_extension_python", + "extension_group": "default_extension_group" + }, + { + "type": "extension", + "name": "default_extension_go", + "addon": "default_extension_go", + "extension_group": "default_extension_group" + }, + { + "type": "extension", + "name": "simple_echo_cpp", + "addon": "simple_echo_cpp", + "extension_group": "default_extension_group" + }, + { + "type": "extension", + "name": "pil_demo_python", + "addon": "pil_demo_python", + "extension_group": "default_extension_group" + } + ], + "connections": [ + { + "extension_group": "default_extension_group", + "extension": "simple_http_server_cpp", + "cmd": [ + { + "name": "test", + "dest": [ + { + "extension_group": "default_extension_group", + "extension": "default_extension_python" + } + ] + } + ] + }, + { + "extension_group": "default_extension_group", + "extension": "default_extension_python", + "cmd": [ + { + "name": "hello", + "dest": [ + { + "extension_group": "default_extension_group", + "extension": "simple_echo_cpp" + } + ] + }, + { + "name": "greeting", + "dest": [ + { + "extension_group": "default_extension_group", + "extension": "default_extension_go" + } + ] + } + ] + } + ] + } + ] + } +} \ No newline at end of file diff --git a/tests/ten_runtime/integration/python/go_app_async_extension_python/go_app_async_extension_python_app/ten_packages/extension/default_extension_python/extension.py b/tests/ten_runtime/integration/python/go_app_async_extension_python/go_app_async_extension_python_app/ten_packages/extension/default_extension_python/extension.py new file mode 100644 index 0000000000..f872b2e2a0 --- /dev/null +++ b/tests/ten_runtime/integration/python/go_app_async_extension_python/go_app_async_extension_python_app/ten_packages/extension/default_extension_python/extension.py @@ -0,0 +1,97 @@ +# +# Copyright © 2024 Agora +# This file is part of TEN Framework, an open source project. +# Licensed under the Apache License, Version 2.0, with certain conditions. +# Refer to the "LICENSE" file in the root directory for more information. +# + +# import debugpy +# debugpy.listen(5678) +# debugpy.wait_for_client() + +import asyncio +from ten import ( + AsyncExtension, + AsyncTenEnv, + Cmd, + StatusCode, + CmdResult, +) + + +class DefaultExtension(AsyncExtension): + async def on_configure(self, ten_env: AsyncTenEnv) -> None: + ten_env.log_debug("on_init") + + ten_env.init_property_from_json('{"testKey": "testValue"}') + + await asyncio.sleep(0.5) + + ten_env.on_configure_done() + + async def on_start(self, ten_env: AsyncTenEnv) -> None: + ten_env.log_debug("on_start") + + await asyncio.sleep(0.5) + + ten_env.set_property_from_json("testKey2", '"testValue2"') + testValue = ten_env.get_property_to_json("testKey") + testValue2 = ten_env.get_property_to_json("testKey2") + ten_env.log_info(f"testValue: {testValue}, testValue2: {testValue2}") + + ten_env.on_start_done() + + async def on_stop(self, ten_env: AsyncTenEnv) -> None: + ten_env.log_debug("on_stop") + + await asyncio.sleep(0.5) + + ten_env.on_stop_done() + + async def on_deinit(self, ten_env: AsyncTenEnv) -> None: + ten_env.log_debug("on_deinit") + + await asyncio.sleep(0.5) + ten_env.on_deinit_done() + + async def greeting(self, ten_env: AsyncTenEnv) -> CmdResult: + await asyncio.sleep(1) + + new_cmd = Cmd.create("greeting") + return await ten_env.send_cmd( + new_cmd, + ) + + async def on_cmd(self, ten_env: AsyncTenEnv, cmd: Cmd) -> None: + cmd_json = cmd.to_json() + ten_env.log_debug("on_cmd: " + cmd_json) + + new_cmd = Cmd.create("hello") + new_cmd.set_property_from_json("test", '"testValue2"') + test_value = new_cmd.get_property_to_json("test") + ten_env.log_info(f"on_cmd test_value: {test_value}") + + await asyncio.sleep(0.5) + + result = await ten_env.send_cmd(new_cmd) + + statusCode = result.get_status_code() + detail = result.get_property_string("detail") + ten_env.log_info( + f"check_hello: status: {str(statusCode)}, detail: {detail}" + ) + + greeting_tasks = [self.greeting(ten_env) for _ in range(100)] + + results = await asyncio.gather(*greeting_tasks) + + for result in results: + statusCode = result.get_status_code() + if statusCode != StatusCode.OK: + ten_env.log_fatal(f"check_hello: status: {str(statusCode)}") + + respCmd = CmdResult.create(StatusCode.OK) + respCmd.set_property_string("detail", "received response") + ten_env.log_info("create respCmd") + + ten_env.return_result(respCmd, cmd) diff --git a/tests/ten_runtime/integration/python/go_app_async_extension_python/test_case.py b/tests/ten_runtime/integration/python/go_app_async_extension_python/test_case.py new file mode 100644 index 0000000000..0440d9f454 --- /dev/null +++ b/tests/ten_runtime/integration/python/go_app_async_extension_python/test_case.py @@ -0,0 +1,128 @@ +""" +Test go_app_async_extension_python. +""" + +import subprocess +import os +import sys +from sys import stdout +from .common import http + + +def http_request(): + return http.post( + "http://127.0.0.1:8002/", + { + "_ten": { + "name": "test", + }, + }, + ) + + +def test_go_app_async_extension_python(): + """Test client and app server.""" + base_path = os.path.dirname(os.path.abspath(__file__)) + root_dir = os.path.join(base_path, "../../../../../") + + # Create virtual environment. + venv_dir = os.path.join(base_path, "venv") + subprocess.run([sys.executable, "-m", "venv", venv_dir]) + + my_env = os.environ.copy() + + # Set the required environment variables for the test. + my_env["PYTHONMALLOC"] = "malloc" + my_env["PYTHONDEVMODE"] = "1" + + # Launch virtual environment. + my_env["VIRTUAL_ENV"] = venv_dir + my_env["PATH"] = os.path.join(venv_dir, "bin") + os.pathsep + my_env["PATH"] + + if sys.platform == "win32": + print("test_go_app_async_extension_python doesn't support win32") + assert False + elif sys.platform == "darwin": + # client depends on some libraries in the TEN app. + my_env["DYLD_LIBRARY_PATH"] = os.path.join( + base_path, "go_app_async_extension_python_app/lib" + ) + else: + # client depends on some libraries in the TEN app. + my_env["LD_LIBRARY_PATH"] = os.path.join( + base_path, "go_app_async_extension_python_app/lib" + ) + + app_root_path = os.path.join(base_path, "go_app_async_extension_python_app") + + tman_install_cmd = [ + os.path.join(root_dir, "ten_manager/bin/tman"), + "--config-file", + os.path.join(root_dir, "tests/local_registry/config.json"), + "install", + ] + + tman_install_process = subprocess.Popen( + tman_install_cmd, + stdout=stdout, + stderr=subprocess.STDOUT, + env=my_env, + cwd=app_root_path, + ) + tman_install_process.wait() + + bootstrap_cmd = os.path.join(base_path, "go_app_async_extension_python_app/bin/bootstrap") + + bootstrap_process = subprocess.Popen( + bootstrap_cmd, stdout=stdout, stderr=subprocess.STDOUT, env=my_env + ) + bootstrap_process.wait() + + if sys.platform == "linux": + if os.path.exists(os.path.join(base_path, "use_asan_lib_marker")): + libasan_path = os.path.join( + base_path, + "go_app_async_extension_python_app/ten_packages/system/ten_runtime/lib/libasan.so", + ) + + if os.path.exists(libasan_path): + my_env["LD_PRELOAD"] = libasan_path + + server_cmd = os.path.join(base_path, "go_app_async_extension_python_app/bin/start") + + server = subprocess.Popen( + server_cmd, + stdout=stdout, + stderr=subprocess.STDOUT, + env=my_env, + cwd=app_root_path, + ) + + is_started = http.is_app_started("127.0.0.1", 8002, 30) + if not is_started: + print("The go_app_async_extension_python is not started after 30 seconds.") + + server.kill() + exit_code = server.wait() + print("The exit code of go_app_async_extension_python: ", exit_code) + + assert exit_code == 0 + assert 0 + + return + + try: + resp = http_request() + assert resp != 500 + print(resp) + + finally: + is_stopped = http.stop_app("127.0.0.1", 8002, 30) + if not is_stopped: + print("The go_app_async_extension_python can not stop after 30 seconds.") + server.kill() + + exit_code = server.wait() + print("The exit code of go_app_async_extension_python: ", exit_code) + + assert exit_code == 0 diff --git a/tests/ten_runtime/integration/python/go_app_cythonize/go_app_cythonize_app/manifest.json b/tests/ten_runtime/integration/python/go_app_cythonize/go_app_cythonize_app/manifest.json index 1cc56fa74f..c32778b6f5 100644 --- a/tests/ten_runtime/integration/python/go_app_cythonize/go_app_cythonize_app/manifest.json +++ b/tests/ten_runtime/integration/python/go_app_cythonize/go_app_cythonize_app/manifest.json @@ -3,17 +3,17 @@ { "type": "system", "name": "ten_runtime", - "version": "0.2.0" + "version": "0.3.0-alpha" }, { "type": "system", "name": "ten_runtime_python", - "version": "0.1.0" + "version": "0.3.0-alpha" }, { "type": "extension", "name": "default_extension_go", - "version": "0.1.0" + "version": "0.3.0-alpha" }, { "type": "extension", @@ -28,12 +28,12 @@ { "type": "extension", "name": "default_extension_python", - "version": "0.1.0" + "version": "0.3.0-alpha" }, { "type": "extension", "name": "py_init_extension_cpp", - "version": "0.1.0" + "version": "0.3.0-alpha" }, { "type": "extension", diff --git a/tests/ten_runtime/integration/python/go_app_cythonize/go_app_cythonize_app/ten_packages/extension/default_extension_python/extension.pyx b/tests/ten_runtime/integration/python/go_app_cythonize/go_app_cythonize_app/ten_packages/extension/default_extension_python/extension.pyx index 2b3dab4853..a95e1e6b65 100644 --- a/tests/ten_runtime/integration/python/go_app_cythonize/go_app_cythonize_app/ten_packages/extension/default_extension_python/extension.pyx +++ b/tests/ten_runtime/integration/python/go_app_cythonize/go_app_cythonize_app/ten_packages/extension/default_extension_python/extension.pyx @@ -42,16 +42,13 @@ class DefaultExtension(Extension): self, ten_env: TenEnv, result: CmdResult, receivedCmd: Cmd ): statusCode = result.get_status_code() - detail = result.get_property_string("detail") print( "DefaultExtension check_greeting: status:" + str(statusCode) - + " detail:" - + detail ) respCmd = CmdResult.create(StatusCode.OK) - respCmd.set_property_string("detail", detail + " nbnb") + respCmd.set_property_string("detail", "received response") print("DefaultExtension create respCmd") ten_env.return_result(respCmd, receivedCmd) diff --git a/tests/ten_runtime/integration/python/go_app_cythonize/go_app_cythonize_app/ten_packages/extension/default_extension_python/manifest.json b/tests/ten_runtime/integration/python/go_app_cythonize/go_app_cythonize_app/ten_packages/extension/default_extension_python/manifest.json index aab6d6ff5b..96d41de975 100644 --- a/tests/ten_runtime/integration/python/go_app_cythonize/go_app_cythonize_app/ten_packages/extension/default_extension_python/manifest.json +++ b/tests/ten_runtime/integration/python/go_app_cythonize/go_app_cythonize_app/ten_packages/extension/default_extension_python/manifest.json @@ -1,12 +1,12 @@ { "type": "extension", "name": "default_extension_python", - "version": "0.1.0", + "version": "0.3.0-alpha", "dependencies": [ { "type": "system", "name": "ten_runtime_python", - "version": "0.1.0" + "version": "0.3.0-alpha" } ], "package": { diff --git a/tests/ten_runtime/integration/python/go_app_partially_cythonize/go_app_partially_cythonize_app/manifest.json b/tests/ten_runtime/integration/python/go_app_partially_cythonize/go_app_partially_cythonize_app/manifest.json index 1cc56fa74f..c32778b6f5 100644 --- a/tests/ten_runtime/integration/python/go_app_partially_cythonize/go_app_partially_cythonize_app/manifest.json +++ b/tests/ten_runtime/integration/python/go_app_partially_cythonize/go_app_partially_cythonize_app/manifest.json @@ -3,17 +3,17 @@ { "type": "system", "name": "ten_runtime", - "version": "0.2.0" + "version": "0.3.0-alpha" }, { "type": "system", "name": "ten_runtime_python", - "version": "0.1.0" + "version": "0.3.0-alpha" }, { "type": "extension", "name": "default_extension_go", - "version": "0.1.0" + "version": "0.3.0-alpha" }, { "type": "extension", @@ -28,12 +28,12 @@ { "type": "extension", "name": "default_extension_python", - "version": "0.1.0" + "version": "0.3.0-alpha" }, { "type": "extension", "name": "py_init_extension_cpp", - "version": "0.1.0" + "version": "0.3.0-alpha" }, { "type": "extension", diff --git a/tests/ten_runtime/integration/python/go_app_partially_cythonize/go_app_partially_cythonize_app/ten_packages/extension/default_extension_python/extension.pyx b/tests/ten_runtime/integration/python/go_app_partially_cythonize/go_app_partially_cythonize_app/ten_packages/extension/default_extension_python/extension.pyx index f829f1062b..61443ce859 100644 --- a/tests/ten_runtime/integration/python/go_app_partially_cythonize/go_app_partially_cythonize_app/ten_packages/extension/default_extension_python/extension.pyx +++ b/tests/ten_runtime/integration/python/go_app_partially_cythonize/go_app_partially_cythonize_app/ten_packages/extension/default_extension_python/extension.pyx @@ -42,16 +42,13 @@ class DefaultExtension(Extension): self, ten_env: TenEnv, result: CmdResult, receivedCmd: Cmd ): statusCode = result.get_status_code() - detail = result.get_property_string("detail") print( "DefaultExtension check_greeting: status:" + str(statusCode) - + " detail:" - + detail ) respCmd = CmdResult.create(StatusCode.OK) - respCmd.set_property_string("detail", detail + " nbnb") + respCmd.set_property_string("detail", "received response") print("DefaultExtension create respCmd") ten_env.return_result(respCmd, receivedCmd) diff --git a/tests/ten_runtime/integration/python/go_app_partially_cythonize/go_app_partially_cythonize_app/ten_packages/extension/default_extension_python/manifest.json b/tests/ten_runtime/integration/python/go_app_partially_cythonize/go_app_partially_cythonize_app/ten_packages/extension/default_extension_python/manifest.json index aab6d6ff5b..96d41de975 100644 --- a/tests/ten_runtime/integration/python/go_app_partially_cythonize/go_app_partially_cythonize_app/ten_packages/extension/default_extension_python/manifest.json +++ b/tests/ten_runtime/integration/python/go_app_partially_cythonize/go_app_partially_cythonize_app/ten_packages/extension/default_extension_python/manifest.json @@ -1,12 +1,12 @@ { "type": "extension", "name": "default_extension_python", - "version": "0.1.0", + "version": "0.3.0-alpha", "dependencies": [ { "type": "system", "name": "ten_runtime_python", - "version": "0.1.0" + "version": "0.3.0-alpha" } ], "package": { diff --git a/tests/ten_runtime/integration/python/go_app_python/go_app_python_app/manifest.json b/tests/ten_runtime/integration/python/go_app_python/go_app_python_app/manifest.json index 1cc56fa74f..c32778b6f5 100644 --- a/tests/ten_runtime/integration/python/go_app_python/go_app_python_app/manifest.json +++ b/tests/ten_runtime/integration/python/go_app_python/go_app_python_app/manifest.json @@ -3,17 +3,17 @@ { "type": "system", "name": "ten_runtime", - "version": "0.2.0" + "version": "0.3.0-alpha" }, { "type": "system", "name": "ten_runtime_python", - "version": "0.1.0" + "version": "0.3.0-alpha" }, { "type": "extension", "name": "default_extension_go", - "version": "0.1.0" + "version": "0.3.0-alpha" }, { "type": "extension", @@ -28,12 +28,12 @@ { "type": "extension", "name": "default_extension_python", - "version": "0.1.0" + "version": "0.3.0-alpha" }, { "type": "extension", "name": "py_init_extension_cpp", - "version": "0.1.0" + "version": "0.3.0-alpha" }, { "type": "extension", diff --git a/tests/ten_runtime/integration/python/go_app_python/go_app_python_app/ten_packages/extension/default_extension_python/extension.py b/tests/ten_runtime/integration/python/go_app_python/go_app_python_app/ten_packages/extension/default_extension_python/extension.py index 0d501536de..1df155b5c4 100644 --- a/tests/ten_runtime/integration/python/go_app_python/go_app_python_app/ten_packages/extension/default_extension_python/extension.py +++ b/tests/ten_runtime/integration/python/go_app_python/go_app_python_app/ten_packages/extension/default_extension_python/extension.py @@ -47,13 +47,12 @@ def check_greeting( self, ten_env: TenEnv, result: CmdResult, receivedCmd: Cmd ): statusCode = result.get_status_code() - detail = result.get_property_string("detail") ten_env.log_info( - f"check_greeting: status: {str(statusCode)}, detail: {detail}" + f"check_greeting: status: {str(statusCode)}" ) respCmd = CmdResult.create(StatusCode.OK) - respCmd.set_property_string("detail", detail + " nbnb") + respCmd.set_property_string("detail", "received response") ten_env.log_info("create respCmd") ten_env.return_result(respCmd, receivedCmd) diff --git a/tests/ten_runtime/integration/python/large_json_python/large_json_python_app/manifest.json b/tests/ten_runtime/integration/python/large_json_python/large_json_python_app/manifest.json index 7bd9b30f55..6d61860568 100644 --- a/tests/ten_runtime/integration/python/large_json_python/large_json_python_app/manifest.json +++ b/tests/ten_runtime/integration/python/large_json_python/large_json_python_app/manifest.json @@ -3,7 +3,7 @@ { "type": "system", "name": "ten_runtime", - "version": "0.2.0" + "version": "0.3.0-alpha" }, { "type": "extension", @@ -13,7 +13,7 @@ { "type": "extension", "name": "default_extension_python", - "version": "0.1.0" + "version": "0.3.0-alpha" }, { "type": "extension", diff --git a/tests/ten_runtime/integration/python/multi_process_python/multi_process_python_app/manifest.json b/tests/ten_runtime/integration/python/multi_process_python/multi_process_python_app/manifest.json index 7bd9b30f55..6d61860568 100644 --- a/tests/ten_runtime/integration/python/multi_process_python/multi_process_python_app/manifest.json +++ b/tests/ten_runtime/integration/python/multi_process_python/multi_process_python_app/manifest.json @@ -3,7 +3,7 @@ { "type": "system", "name": "ten_runtime", - "version": "0.2.0" + "version": "0.3.0-alpha" }, { "type": "extension", @@ -13,7 +13,7 @@ { "type": "extension", "name": "default_extension_python", - "version": "0.1.0" + "version": "0.3.0-alpha" }, { "type": "extension", diff --git a/tests/ten_runtime/integration/python/multiple_results_python/multiple_results_python_app/manifest.json b/tests/ten_runtime/integration/python/multiple_results_python/multiple_results_python_app/manifest.json index 1a1a8b1b76..0fd8db6180 100644 --- a/tests/ten_runtime/integration/python/multiple_results_python/multiple_results_python_app/manifest.json +++ b/tests/ten_runtime/integration/python/multiple_results_python/multiple_results_python_app/manifest.json @@ -3,7 +3,7 @@ { "type": "system", "name": "ten_runtime", - "version": "0.2.0" + "version": "0.3.0-alpha" }, { "type": "extension", @@ -13,7 +13,7 @@ { "type": "extension", "name": "default_extension_python", - "version": "0.1.0" + "version": "0.3.0-alpha" } ] } \ No newline at end of file diff --git a/tests/ten_runtime/integration/python/resp_handler_yield_python/resp_handler_yield_python_app/manifest.json b/tests/ten_runtime/integration/python/resp_handler_yield_python/resp_handler_yield_python_app/manifest.json index 7bd9b30f55..6d61860568 100644 --- a/tests/ten_runtime/integration/python/resp_handler_yield_python/resp_handler_yield_python_app/manifest.json +++ b/tests/ten_runtime/integration/python/resp_handler_yield_python/resp_handler_yield_python_app/manifest.json @@ -3,7 +3,7 @@ { "type": "system", "name": "ten_runtime", - "version": "0.2.0" + "version": "0.3.0-alpha" }, { "type": "extension", @@ -13,7 +13,7 @@ { "type": "extension", "name": "default_extension_python", - "version": "0.1.0" + "version": "0.3.0-alpha" }, { "type": "extension", diff --git a/tests/ten_runtime/integration/python/send_cmd_discard_result_python/send_cmd_discard_result_python_app/manifest.json b/tests/ten_runtime/integration/python/send_cmd_discard_result_python/send_cmd_discard_result_python_app/manifest.json index 1a1a8b1b76..0fd8db6180 100644 --- a/tests/ten_runtime/integration/python/send_cmd_discard_result_python/send_cmd_discard_result_python_app/manifest.json +++ b/tests/ten_runtime/integration/python/send_cmd_discard_result_python/send_cmd_discard_result_python_app/manifest.json @@ -3,7 +3,7 @@ { "type": "system", "name": "ten_runtime", - "version": "0.2.0" + "version": "0.3.0-alpha" }, { "type": "extension", @@ -13,7 +13,7 @@ { "type": "extension", "name": "default_extension_python", - "version": "0.1.0" + "version": "0.3.0-alpha" } ] } \ No newline at end of file diff --git a/tests/ten_runtime/integration/python/send_cmd_python/send_cmd_python_app/manifest.json b/tests/ten_runtime/integration/python/send_cmd_python/send_cmd_python_app/manifest.json index 7bd9b30f55..6d61860568 100644 --- a/tests/ten_runtime/integration/python/send_cmd_python/send_cmd_python_app/manifest.json +++ b/tests/ten_runtime/integration/python/send_cmd_python/send_cmd_python_app/manifest.json @@ -3,7 +3,7 @@ { "type": "system", "name": "ten_runtime", - "version": "0.2.0" + "version": "0.3.0-alpha" }, { "type": "extension", @@ -13,7 +13,7 @@ { "type": "extension", "name": "default_extension_python", - "version": "0.1.0" + "version": "0.3.0-alpha" }, { "type": "extension", diff --git a/tests/ten_runtime/integration/python/send_data_python/send_data_python_app/manifest.json b/tests/ten_runtime/integration/python/send_data_python/send_data_python_app/manifest.json index 7bd9b30f55..6d61860568 100644 --- a/tests/ten_runtime/integration/python/send_data_python/send_data_python_app/manifest.json +++ b/tests/ten_runtime/integration/python/send_data_python/send_data_python_app/manifest.json @@ -3,7 +3,7 @@ { "type": "system", "name": "ten_runtime", - "version": "0.2.0" + "version": "0.3.0-alpha" }, { "type": "extension", @@ -13,7 +13,7 @@ { "type": "extension", "name": "default_extension_python", - "version": "0.1.0" + "version": "0.3.0-alpha" }, { "type": "extension", diff --git a/tests/ten_runtime/integration/python/send_json_python/send_json_python_app/manifest.json b/tests/ten_runtime/integration/python/send_json_python/send_json_python_app/manifest.json index 7bd9b30f55..6d61860568 100644 --- a/tests/ten_runtime/integration/python/send_json_python/send_json_python_app/manifest.json +++ b/tests/ten_runtime/integration/python/send_json_python/send_json_python_app/manifest.json @@ -3,7 +3,7 @@ { "type": "system", "name": "ten_runtime", - "version": "0.2.0" + "version": "0.3.0-alpha" }, { "type": "extension", @@ -13,7 +13,7 @@ { "type": "extension", "name": "default_extension_python", - "version": "0.1.0" + "version": "0.3.0-alpha" }, { "type": "extension", diff --git a/tests/ten_runtime/integration/python/send_recv_image_python/send_recv_image_python_app/manifest.json b/tests/ten_runtime/integration/python/send_recv_image_python/send_recv_image_python_app/manifest.json index a69ab2ab9d..9c79b89240 100644 --- a/tests/ten_runtime/integration/python/send_recv_image_python/send_recv_image_python_app/manifest.json +++ b/tests/ten_runtime/integration/python/send_recv_image_python/send_recv_image_python_app/manifest.json @@ -3,7 +3,7 @@ { "type": "system", "name": "ten_runtime", - "version": "0.2.0" + "version": "0.3.0-alpha" }, { "type": "extension", @@ -13,7 +13,7 @@ { "type": "extension", "name": "default_extension_python", - "version": "0.1.0" + "version": "0.3.0-alpha" }, { "type": "extension", diff --git a/tests/ten_runtime/integration/python/send_recv_pcm_python/send_recv_pcm_python_app/manifest.json b/tests/ten_runtime/integration/python/send_recv_pcm_python/send_recv_pcm_python_app/manifest.json index 7bd9b30f55..6d61860568 100644 --- a/tests/ten_runtime/integration/python/send_recv_pcm_python/send_recv_pcm_python_app/manifest.json +++ b/tests/ten_runtime/integration/python/send_recv_pcm_python/send_recv_pcm_python_app/manifest.json @@ -3,7 +3,7 @@ { "type": "system", "name": "ten_runtime", - "version": "0.2.0" + "version": "0.3.0-alpha" }, { "type": "extension", @@ -13,7 +13,7 @@ { "type": "extension", "name": "default_extension_python", - "version": "0.1.0" + "version": "0.3.0-alpha" }, { "type": "extension", diff --git a/tests/ten_runtime/integration/python/standalone_test_python/default_extension_python/manifest.json b/tests/ten_runtime/integration/python/standalone_test_python/default_extension_python/manifest.json index 3692b3e42e..5ffaa0b82a 100644 --- a/tests/ten_runtime/integration/python/standalone_test_python/default_extension_python/manifest.json +++ b/tests/ten_runtime/integration/python/standalone_test_python/default_extension_python/manifest.json @@ -1,12 +1,12 @@ { "type": "extension", "name": "default_extension_python", - "version": "0.1.0", + "version": "0.3.0-alpha", "dependencies": [ { "type": "system", "name": "ten_runtime_python", - "version": "0.1.0" + "version": "0.3.0-alpha" } ], "package": { diff --git a/tests/ten_runtime/integration/python/two_async_exts_one_group_python/BUILD.gn b/tests/ten_runtime/integration/python/two_async_exts_one_group_python/BUILD.gn new file mode 100644 index 0000000000..3918a98ecc --- /dev/null +++ b/tests/ten_runtime/integration/python/two_async_exts_one_group_python/BUILD.gn @@ -0,0 +1,50 @@ +# +# Copyright © 2024 Agora +# This file is part of TEN Framework, an open source project. +# Licensed under the Apache License, Version 2.0, with certain conditions. +# Refer to the "LICENSE" file in the root directory for more information. +# +import("//build/ten_runtime/feature/test.gni") +import("//build/ten_runtime/ten.gni") + +ten_package_test_prepare_app("two_async_exts_one_group_python_app") { + src_app = "default_app_python" + src_app_language = "python" + generated_app_src_root_dir_name = "two_async_exts_one_group_python_app" + + replace_files_after_install_app = [ + "two_async_exts_one_group_python_app/manifest.json", + "two_async_exts_one_group_python_app/property.json", + ] + + replace_files_after_install_all = [ "two_async_exts_one_group_python_app/ten_packages/extension/default_extension_python/extension.py" ] + + if (ten_enable_package_manager) { + deps = [ + "//core/src/ten_manager", + "//packages/core_apps/default_app_python:upload_default_app_python_to_server", + "//packages/core_extensions/default_extension_python:upload_default_extension_python_to_server", + "//packages/example_extensions/simple_echo_cpp:upload_simple_echo_cpp_to_server", + "//packages/example_extensions/simple_http_server_cpp:upload_simple_http_server_cpp_to_server", + ] + } +} + +ten_package_test_prepare_auxiliary_resources( + "two_async_exts_one_group_python_test_files") { + resources = [ + "//tests/ten_runtime/integration/common=>common", + "__init__.py", + "test_case.py", + ] + if (enable_sanitizer) { + resources += [ "//tests/ten_runtime/integration/tools/use_asan_lib_marker=>use_asan_lib_marker" ] + } +} + +group("two_async_exts_one_group_python") { + deps = [ + ":two_async_exts_one_group_python_app", + ":two_async_exts_one_group_python_test_files", + ] +} diff --git a/tests/ten_runtime/integration/python/two_async_exts_one_group_python/__init__.py b/tests/ten_runtime/integration/python/two_async_exts_one_group_python/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/ten_runtime/integration/python/two_async_exts_one_group_python/test_case.py b/tests/ten_runtime/integration/python/two_async_exts_one_group_python/test_case.py new file mode 100644 index 0000000000..61f10f485e --- /dev/null +++ b/tests/ten_runtime/integration/python/two_async_exts_one_group_python/test_case.py @@ -0,0 +1,130 @@ +""" +Test two_async_exts_one_group_python. +""" + +import subprocess +import os +import sys +from sys import stdout +from .common import http + + +def http_request(): + return http.post( + "http://127.0.0.1:8002/", + { + "_ten": { + "name": "test", + }, + }, + ) + + +def test_two_async_exts_one_group_python(): + """Test client and app server.""" + base_path = os.path.dirname(os.path.abspath(__file__)) + root_dir = os.path.join(base_path, "../../../../../") + + # Create virtual environment. + venv_dir = os.path.join(base_path, "venv") + subprocess.run([sys.executable, "-m", "venv", venv_dir]) + + my_env = os.environ.copy() + + # Set the required environment variables for the test. + my_env["PYTHONMALLOC"] = "malloc" + my_env["PYTHONDEVMODE"] = "1" + + # Launch virtual environment. + my_env["VIRTUAL_ENV"] = venv_dir + my_env["PATH"] = os.path.join(venv_dir, "bin") + os.pathsep + my_env["PATH"] + + if sys.platform == "win32": + print("test_two_async_exts_one_group_python doesn't support win32") + assert False + elif sys.platform == "darwin": + # client depends on some libraries in the TEN app. + my_env["DYLD_LIBRARY_PATH"] = os.path.join( + base_path, "two_async_exts_one_group_python_app/lib" + ) + else: + # client depends on some libraries in the TEN app. + my_env["LD_LIBRARY_PATH"] = os.path.join( + base_path, "two_async_exts_one_group_python_app/lib" + ) + + app_root_path = os.path.join(base_path, "two_async_exts_one_group_python_app") + + tman_install_cmd = [ + os.path.join(root_dir, "ten_manager/bin/tman"), + "--config-file", + os.path.join(root_dir, "tests/local_registry/config.json"), + "install", + ] + + tman_install_process = subprocess.Popen( + tman_install_cmd, + stdout=stdout, + stderr=subprocess.STDOUT, + env=my_env, + cwd=app_root_path, + ) + tman_install_process.wait() + + bootstrap_cmd = os.path.join( + base_path, "two_async_exts_one_group_python_app/bin/bootstrap" + ) + + bootstrap_process = subprocess.Popen( + bootstrap_cmd, stdout=stdout, stderr=subprocess.STDOUT, env=my_env + ) + bootstrap_process.wait() + + if sys.platform == "linux": + if os.path.exists(os.path.join(base_path, "use_asan_lib_marker")): + libasan_path = os.path.join( + base_path, + "two_async_exts_one_group_python_app/ten_packages/system/ten_runtime/lib/libasan.so", + ) + + if os.path.exists(libasan_path): + my_env["LD_PRELOAD"] = libasan_path + + server_cmd = os.path.join(base_path, "two_async_exts_one_group_python_app/bin/start") + + server = subprocess.Popen( + server_cmd, + stdout=stdout, + stderr=subprocess.STDOUT, + env=my_env, + cwd=app_root_path, + ) + + is_started = http.is_app_started("127.0.0.1", 8002, 30) + if not is_started: + print("The two_async_exts_one_group_python is not started after 30 seconds.") + + server.kill() + exit_code = server.wait() + print("The exit code of two_async_exts_one_group_python: ", exit_code) + + assert exit_code == 0 + assert 0 + + return + + try: + resp = http_request() + assert resp != 500 + print(resp) + + finally: + is_stopped = http.stop_app("127.0.0.1", 8002, 30) + if not is_stopped: + print("The two_async_exts_one_group_python can not stop after 30 seconds.") + server.kill() + + exit_code = server.wait() + print("The exit code of two_async_exts_one_group_python: ", exit_code) + + assert exit_code == 0 diff --git a/tests/ten_runtime/integration/python/two_async_exts_one_group_python/two_async_exts_one_group_python_app/manifest.json b/tests/ten_runtime/integration/python/two_async_exts_one_group_python/two_async_exts_one_group_python_app/manifest.json new file mode 100644 index 0000000000..6d61860568 --- /dev/null +++ b/tests/ten_runtime/integration/python/two_async_exts_one_group_python/two_async_exts_one_group_python_app/manifest.json @@ -0,0 +1,24 @@ +{ + "dependencies": [ + { + "type": "system", + "name": "ten_runtime", + "version": "0.3.0-alpha" + }, + { + "type": "extension", + "name": "simple_http_server_cpp", + "version": "0.1.0" + }, + { + "type": "extension", + "name": "default_extension_python", + "version": "0.3.0-alpha" + }, + { + "type": "extension", + "name": "simple_echo_cpp", + "version": "0.1.0" + } + ] +} \ No newline at end of file diff --git a/tests/ten_runtime/integration/python/two_async_exts_one_group_python/two_async_exts_one_group_python_app/property.json b/tests/ten_runtime/integration/python/two_async_exts_one_group_python/two_async_exts_one_group_python_app/property.json new file mode 100644 index 0000000000..a02d5f9365 --- /dev/null +++ b/tests/ten_runtime/integration/python/two_async_exts_one_group_python/two_async_exts_one_group_python_app/property.json @@ -0,0 +1,97 @@ +{ + "_ten": { + "log_level": 2, + "predefined_graphs": [ + { + "name": "default", + "auto_start": true, + "nodes": [ + { + "type": "extension_group", + "name": "default_extension_group", + "addon": "default_extension_group" + }, + { + "type": "extension_group", + "name": "test", + "addon": "default_extension_group" + }, + { + "type": "extension", + "name": "simple_http_server_cpp", + "addon": "simple_http_server_cpp", + "extension_group": "default_extension_group", + "property": { + "server_port": 8002 + } + }, + { + "type": "extension", + "name": "default_extension_python", + "addon": "default_extension_python", + "extension_group": "test" + }, + { + "type": "extension", + "name": "default_extension_python2", + "addon": "default_extension_python", + "extension_group": "test" + }, + { + "type": "extension", + "name": "simple_echo_cpp", + "addon": "simple_echo_cpp", + "extension_group": "default_extension_group" + } + ], + "connections": [ + { + "extension_group": "default_extension_group", + "extension": "simple_http_server_cpp", + "cmd": [ + { + "name": "test", + "dest": [ + { + "extension_group": "test", + "extension": "default_extension_python" + } + ] + } + ] + }, + { + "extension_group": "test", + "extension": "default_extension_python", + "cmd": [ + { + "name": "hello", + "dest": [ + { + "extension_group": "test", + "extension": "default_extension_python2" + } + ] + } + ] + }, + { + "extension_group": "test", + "extension": "default_extension_python2", + "cmd": [ + { + "name": "hello", + "dest": [ + { + "extension_group": "default_extension_group", + "extension": "simple_echo_cpp" + } + ] + } + ] + } + ] + } + ] + } +} \ No newline at end of file diff --git a/tests/ten_runtime/integration/python/two_async_exts_one_group_python/two_async_exts_one_group_python_app/ten_packages/extension/default_extension_python/extension.py b/tests/ten_runtime/integration/python/two_async_exts_one_group_python/two_async_exts_one_group_python_app/ten_packages/extension/default_extension_python/extension.py new file mode 100644 index 0000000000..0d2abe1618 --- /dev/null +++ b/tests/ten_runtime/integration/python/two_async_exts_one_group_python/two_async_exts_one_group_python_app/ten_packages/extension/default_extension_python/extension.py @@ -0,0 +1,56 @@ +# +# Copyright © 2024 Agora +# This file is part of TEN Framework, an open source project. +# Licensed under the Apache License, Version 2.0, with certain conditions. +# Refer to the "LICENSE" file in the root directory for more information. +# +import asyncio +from ten import ( + AsyncExtension, + AsyncTenEnv, + Cmd, +) + + +class DefaultExtension(AsyncExtension): + def __init__(self, name: str) -> None: + super().__init__(name) + self.name = name + + async def on_configure(self, ten_env: AsyncTenEnv) -> None: + await asyncio.sleep(0.5) + ten_env.on_configure_done() + + async def on_init(self, ten_env: AsyncTenEnv) -> None: + await asyncio.sleep(0.5) + ten_env.on_init_done() + + async def on_start(self, ten_env: AsyncTenEnv) -> None: + await asyncio.sleep(0.5) + ten_env.log_debug("on_start") + ten_env.on_start_done() + + async def on_deinit(self, ten_env: AsyncTenEnv) -> None: + await asyncio.sleep(0.5) + ten_env.on_deinit_done() + + await asyncio.sleep(1) + + async def on_cmd(self, ten_env: AsyncTenEnv, cmd: Cmd) -> None: + cmd_json = cmd.to_json() + ten_env.log_debug(f"on_cmd: {cmd_json}") + + # Mock async operation, e.g. network, file I/O. + await asyncio.sleep(0.5) + + # Send a new command to other extensions and wait for the result. The + # result will be returned to the original sender. + new_cmd = Cmd.create("hello") + cmd_result = await ten_env.send_cmd(new_cmd) + ten_env.return_result(cmd_result, cmd) + + async def on_stop(self, ten_env: AsyncTenEnv) -> None: + ten_env.log_debug("on_stop") + + await asyncio.sleep(0.5) + ten_env.on_stop_done() diff --git a/tests/ten_runtime/integration/python/two_async_exts_python/BUILD.gn b/tests/ten_runtime/integration/python/two_async_exts_python/BUILD.gn new file mode 100644 index 0000000000..5601b48624 --- /dev/null +++ b/tests/ten_runtime/integration/python/two_async_exts_python/BUILD.gn @@ -0,0 +1,50 @@ +# +# Copyright © 2024 Agora +# This file is part of TEN Framework, an open source project. +# Licensed under the Apache License, Version 2.0, with certain conditions. +# Refer to the "LICENSE" file in the root directory for more information. +# +import("//build/ten_runtime/feature/test.gni") +import("//build/ten_runtime/ten.gni") + +ten_package_test_prepare_app("two_async_exts_python_app") { + src_app = "default_app_python" + src_app_language = "python" + generated_app_src_root_dir_name = "two_async_exts_python_app" + + replace_files_after_install_app = [ + "two_async_exts_python_app/manifest.json", + "two_async_exts_python_app/property.json", + ] + + replace_files_after_install_all = [ "two_async_exts_python_app/ten_packages/extension/default_extension_python/extension.py" ] + + if (ten_enable_package_manager) { + deps = [ + "//core/src/ten_manager", + "//packages/core_apps/default_app_python:upload_default_app_python_to_server", + "//packages/core_extensions/default_extension_python:upload_default_extension_python_to_server", + "//packages/example_extensions/simple_echo_cpp:upload_simple_echo_cpp_to_server", + "//packages/example_extensions/simple_http_server_cpp:upload_simple_http_server_cpp_to_server", + ] + } +} + +ten_package_test_prepare_auxiliary_resources( + "two_async_exts_python_test_files") { + resources = [ + "//tests/ten_runtime/integration/common=>common", + "__init__.py", + "test_case.py", + ] + if (enable_sanitizer) { + resources += [ "//tests/ten_runtime/integration/tools/use_asan_lib_marker=>use_asan_lib_marker" ] + } +} + +group("two_async_exts_python") { + deps = [ + ":two_async_exts_python_app", + ":two_async_exts_python_test_files", + ] +} diff --git a/tests/ten_runtime/integration/python/two_async_exts_python/__init__.py b/tests/ten_runtime/integration/python/two_async_exts_python/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/ten_runtime/integration/python/two_async_exts_python/test_case.py b/tests/ten_runtime/integration/python/two_async_exts_python/test_case.py new file mode 100644 index 0000000000..304dded44b --- /dev/null +++ b/tests/ten_runtime/integration/python/two_async_exts_python/test_case.py @@ -0,0 +1,130 @@ +""" +Test two_async_exts_python. +""" + +import subprocess +import os +import sys +from sys import stdout +from .common import http + + +def http_request(): + return http.post( + "http://127.0.0.1:8002/", + { + "_ten": { + "name": "test", + }, + }, + ) + + +def test_two_async_exts_python(): + """Test client and app server.""" + base_path = os.path.dirname(os.path.abspath(__file__)) + root_dir = os.path.join(base_path, "../../../../../") + + # Create virtual environment. + venv_dir = os.path.join(base_path, "venv") + subprocess.run([sys.executable, "-m", "venv", venv_dir]) + + my_env = os.environ.copy() + + # Set the required environment variables for the test. + my_env["PYTHONMALLOC"] = "malloc" + my_env["PYTHONDEVMODE"] = "1" + + # Launch virtual environment. + my_env["VIRTUAL_ENV"] = venv_dir + my_env["PATH"] = os.path.join(venv_dir, "bin") + os.pathsep + my_env["PATH"] + + if sys.platform == "win32": + print("test_two_async_exts_python doesn't support win32") + assert False + elif sys.platform == "darwin": + # client depends on some libraries in the TEN app. + my_env["DYLD_LIBRARY_PATH"] = os.path.join( + base_path, "two_async_exts_python_app/lib" + ) + else: + # client depends on some libraries in the TEN app. + my_env["LD_LIBRARY_PATH"] = os.path.join( + base_path, "two_async_exts_python_app/lib" + ) + + app_root_path = os.path.join(base_path, "two_async_exts_python_app") + + tman_install_cmd = [ + os.path.join(root_dir, "ten_manager/bin/tman"), + "--config-file", + os.path.join(root_dir, "tests/local_registry/config.json"), + "install", + ] + + tman_install_process = subprocess.Popen( + tman_install_cmd, + stdout=stdout, + stderr=subprocess.STDOUT, + env=my_env, + cwd=app_root_path, + ) + tman_install_process.wait() + + bootstrap_cmd = os.path.join( + base_path, "two_async_exts_python_app/bin/bootstrap" + ) + + bootstrap_process = subprocess.Popen( + bootstrap_cmd, stdout=stdout, stderr=subprocess.STDOUT, env=my_env + ) + bootstrap_process.wait() + + if sys.platform == "linux": + if os.path.exists(os.path.join(base_path, "use_asan_lib_marker")): + libasan_path = os.path.join( + base_path, + "two_async_exts_python_app/ten_packages/system/ten_runtime/lib/libasan.so", + ) + + if os.path.exists(libasan_path): + my_env["LD_PRELOAD"] = libasan_path + + server_cmd = os.path.join(base_path, "two_async_exts_python_app/bin/start") + + server = subprocess.Popen( + server_cmd, + stdout=stdout, + stderr=subprocess.STDOUT, + env=my_env, + cwd=app_root_path, + ) + + is_started = http.is_app_started("127.0.0.1", 8002, 30) + if not is_started: + print("The two_async_exts_python is not started after 30 seconds.") + + server.kill() + exit_code = server.wait() + print("The exit code of two_async_exts_python: ", exit_code) + + assert exit_code == 0 + assert 0 + + return + + try: + resp = http_request() + assert resp != 500 + print(resp) + + finally: + is_stopped = http.stop_app("127.0.0.1", 8002, 30) + if not is_stopped: + print("The two_async_exts_python can not stop after 30 seconds.") + server.kill() + + exit_code = server.wait() + print("The exit code of two_async_exts_python: ", exit_code) + + assert exit_code == 0 diff --git a/tests/ten_runtime/integration/python/two_async_exts_python/two_async_exts_python_app/manifest.json b/tests/ten_runtime/integration/python/two_async_exts_python/two_async_exts_python_app/manifest.json new file mode 100644 index 0000000000..6d61860568 --- /dev/null +++ b/tests/ten_runtime/integration/python/two_async_exts_python/two_async_exts_python_app/manifest.json @@ -0,0 +1,24 @@ +{ + "dependencies": [ + { + "type": "system", + "name": "ten_runtime", + "version": "0.3.0-alpha" + }, + { + "type": "extension", + "name": "simple_http_server_cpp", + "version": "0.1.0" + }, + { + "type": "extension", + "name": "default_extension_python", + "version": "0.3.0-alpha" + }, + { + "type": "extension", + "name": "simple_echo_cpp", + "version": "0.1.0" + } + ] +} \ No newline at end of file diff --git a/tests/ten_runtime/integration/python/two_async_exts_python/two_async_exts_python_app/property.json b/tests/ten_runtime/integration/python/two_async_exts_python/two_async_exts_python_app/property.json new file mode 100644 index 0000000000..12cd22dfc1 --- /dev/null +++ b/tests/ten_runtime/integration/python/two_async_exts_python/two_async_exts_python_app/property.json @@ -0,0 +1,87 @@ +{ + "_ten": { + "log_level": 2, + "predefined_graphs": [ + { + "name": "default", + "auto_start": true, + "nodes": [ + { + "type": "extension", + "name": "simple_http_server_cpp", + "addon": "simple_http_server_cpp", + "extension_group": "default_extension_group", + "property": { + "server_port": 8002 + } + }, + { + "type": "extension", + "name": "default_extension_python", + "addon": "default_extension_python", + "extension_group": "test" + }, + { + "type": "extension", + "name": "default_extension_python2", + "addon": "default_extension_python", + "extension_group": "test2" + }, + { + "type": "extension", + "name": "simple_echo_cpp", + "addon": "simple_echo_cpp", + "extension_group": "default_extension_group" + } + ], + "connections": [ + { + "extension_group": "default_extension_group", + "extension": "simple_http_server_cpp", + "cmd": [ + { + "name": "test", + "dest": [ + { + "extension_group": "test", + "extension": "default_extension_python" + } + ] + } + ] + }, + { + "extension_group": "test", + "extension": "default_extension_python", + "cmd": [ + { + "name": "hello", + "dest": [ + { + "extension_group": "test2", + "extension": "default_extension_python2" + } + ] + } + ] + }, + { + "extension_group": "test2", + "extension": "default_extension_python2", + "cmd": [ + { + "name": "hello", + "dest": [ + { + "extension_group": "default_extension_group", + "extension": "simple_echo_cpp" + } + ] + } + ] + } + ] + } + ] + } +} \ No newline at end of file diff --git a/tests/ten_runtime/integration/python/two_async_exts_python/two_async_exts_python_app/ten_packages/extension/default_extension_python/extension.py b/tests/ten_runtime/integration/python/two_async_exts_python/two_async_exts_python_app/ten_packages/extension/default_extension_python/extension.py new file mode 100644 index 0000000000..0d2abe1618 --- /dev/null +++ b/tests/ten_runtime/integration/python/two_async_exts_python/two_async_exts_python_app/ten_packages/extension/default_extension_python/extension.py @@ -0,0 +1,56 @@ +# +# Copyright © 2024 Agora +# This file is part of TEN Framework, an open source project. +# Licensed under the Apache License, Version 2.0, with certain conditions. +# Refer to the "LICENSE" file in the root directory for more information. +# +import asyncio +from ten import ( + AsyncExtension, + AsyncTenEnv, + Cmd, +) + + +class DefaultExtension(AsyncExtension): + def __init__(self, name: str) -> None: + super().__init__(name) + self.name = name + + async def on_configure(self, ten_env: AsyncTenEnv) -> None: + await asyncio.sleep(0.5) + ten_env.on_configure_done() + + async def on_init(self, ten_env: AsyncTenEnv) -> None: + await asyncio.sleep(0.5) + ten_env.on_init_done() + + async def on_start(self, ten_env: AsyncTenEnv) -> None: + await asyncio.sleep(0.5) + ten_env.log_debug("on_start") + ten_env.on_start_done() + + async def on_deinit(self, ten_env: AsyncTenEnv) -> None: + await asyncio.sleep(0.5) + ten_env.on_deinit_done() + + await asyncio.sleep(1) + + async def on_cmd(self, ten_env: AsyncTenEnv, cmd: Cmd) -> None: + cmd_json = cmd.to_json() + ten_env.log_debug(f"on_cmd: {cmd_json}") + + # Mock async operation, e.g. network, file I/O. + await asyncio.sleep(0.5) + + # Send a new command to other extensions and wait for the result. The + # result will be returned to the original sender. + new_cmd = Cmd.create("hello") + cmd_result = await ten_env.send_cmd(new_cmd) + ten_env.return_result(cmd_result, cmd) + + async def on_stop(self, ten_env: AsyncTenEnv) -> None: + ten_env.log_debug("on_stop") + + await asyncio.sleep(0.5) + ten_env.on_stop_done() diff --git a/tests/ten_runtime/integration/python/unused_addon_python/unused_addon_python_app/manifest.json b/tests/ten_runtime/integration/python/unused_addon_python/unused_addon_python_app/manifest.json index 1cc56fa74f..c32778b6f5 100644 --- a/tests/ten_runtime/integration/python/unused_addon_python/unused_addon_python_app/manifest.json +++ b/tests/ten_runtime/integration/python/unused_addon_python/unused_addon_python_app/manifest.json @@ -3,17 +3,17 @@ { "type": "system", "name": "ten_runtime", - "version": "0.2.0" + "version": "0.3.0-alpha" }, { "type": "system", "name": "ten_runtime_python", - "version": "0.1.0" + "version": "0.3.0-alpha" }, { "type": "extension", "name": "default_extension_go", - "version": "0.1.0" + "version": "0.3.0-alpha" }, { "type": "extension", @@ -28,12 +28,12 @@ { "type": "extension", "name": "default_extension_python", - "version": "0.1.0" + "version": "0.3.0-alpha" }, { "type": "extension", "name": "py_init_extension_cpp", - "version": "0.1.0" + "version": "0.3.0-alpha" }, { "type": "extension", diff --git a/tools/docker_for_building/ubuntu/22.04/Dockerfile b/tools/docker_for_building/ubuntu/22.04/Dockerfile index 2528ba2783..878d8c7bd1 100644 --- a/tools/docker_for_building/ubuntu/22.04/Dockerfile +++ b/tools/docker_for_building/ubuntu/22.04/Dockerfile @@ -22,11 +22,13 @@ RUN apt-get clean && apt-get update && apt-get install -y --no-install-recommend tree \ zip \ unzip \ + jq \ + cpulimit \ + util-linux \ libasan5 \ autoconf \ libtool \ uuid-dev \ - g++-multilib \ libmsgpack-dev \ libmysqlclient-dev \ libmysqlcppconn-dev \ @@ -54,6 +56,11 @@ RUN apt-get clean && apt-get update && apt-get install -y --no-install-recommend python3-pip \ python3-venv +RUN export ARCH=$(dpkg --print-architecture) && \ + if [ ${ARCH} = "amd64" ]; then \ + apt-get install -y --no-install-recommends g++-multilib; \ + fi + # ======================================= # Installing Python deps @@ -84,10 +91,7 @@ RUN wget --no-check-certificate -O - https://apt.llvm.org/llvm-snapshot.gpg.key update-alternatives --install /usr/bin/clang-format clang-format /usr/bin/clang-format-18 100 && \ update-alternatives --install /usr/bin/clang-tidy clang-tidy /usr/bin/clang-tidy-18 100 && \ update-alternatives --install /usr/bin/clang-check clang-check /usr/bin/clang-check-18 100 && \ - update-alternatives --install /usr/bin/clangd clangd /usr/bin/clangd-18 100 && \ - mkdir -p /usr/lib/llvm-18/lib/x86_64-pc-linux-gnu && \ - ln -sf $(clang -print-file-name=libc++.a) /usr/lib/llvm-18/lib/x86_64-pc-linux-gnu && \ - ln -sf $(clang -print-file-name=libclang_rt.asan.so) $(dirname $(clang -print-file-name=libclang_rt.asan.so))/libclang_rt.asan-x86_64.so + update-alternatives --install /usr/bin/clangd clangd /usr/bin/clangd-18 100 # ======================================= # Install golang @@ -97,8 +101,8 @@ ENV PATH="$PATH:/usr/local/go/bin:/root/go/bin" # TEN go binding needs to be compatible with GO 1.20, so we need to install GO # 1.20 to check the compatibility. -RUN curl -OL https://go.dev/dl/go1.22.3.linux-amd64.tar.gz && \ - rm -rf /usr/local/go && tar -C /usr/local -xvf go1.22.3.linux-amd64.tar.gz && rm go1.22.3.linux-amd64.tar.gz && \ +RUN export ARCH=$(dpkg --print-architecture) && curl -OL https://go.dev/dl/go1.22.3.linux-${ARCH}.tar.gz && \ + rm -rf /usr/local/go && tar -C /usr/local -xvf go1.22.3.linux-${ARCH}.tar.gz && rm go1.22.3.linux-${ARCH}.tar.gz && \ go install golang.org/dl/go1.20.12@latest && go1.20.12 download # ======================================= diff --git a/tools/supports/update_supports_in_manifest_json.py b/tools/supports/update_supports_in_manifest_json.py new file mode 100644 index 0000000000..9233da0fee --- /dev/null +++ b/tools/supports/update_supports_in_manifest_json.py @@ -0,0 +1,80 @@ +# +# Copyright © 2024 Agora +# This file is part of TEN Framework, an open source project. +# Licensed under the Apache License, Version 2.0, with certain conditions. +# Refer to the "LICENSE" file in the root directory for more information. +# +import json +import argparse + + +class ArgumentInfo(argparse.Namespace): + def __init__(self): + self.input_file: str + self.output_file: str + self.log_level: int + self.os_arch_pairs: list[str] + + +def update_supports( + input_file: str, + output_file: str, + os_arch_pairs: list[list[str]], + log_level: int, +): + with open(input_file, "r") as file: + data = json.load(file) + + supports = [{"os": pair[0], "arch": pair[1]} for pair in os_arch_pairs] + + if "supports" in data: + del data["supports"] + + data["supports"] = supports + + with open(output_file, "w") as file: + json.dump(data, file, indent=2) + + if log_level > 0: + print(f"Updated {output_file} with new 'supports' field.") + + +def main(): + parser = argparse.ArgumentParser( + description="Add OS:Arch pairs to the supports field of a JSON file." + ) + + parser.add_argument("--input-file", type=str, help="Input JSON file path") + parser.add_argument("--output-file", type=str, help="Output JSON file path") + parser.add_argument( + "--log-level", type=int, required=False, help="specify log level" + ) + parser.add_argument( + "--os-arch-pairs", + metavar="os:arch", + type=str, + nargs="+", + help="OS:Arch pairs (e.g., mac:x64 linux:arm)", + ) + + arg_info = ArgumentInfo() + args = parser.parse_args(namespace=arg_info) + + os_arch_pairs = [pair.split(":") for pair in args.os_arch_pairs] + + for pair in os_arch_pairs: + if ( + len(pair) != 2 + or pair[0] not in ["linux", "mac", "win"] + or pair[1] not in ["x86", "x64", "arm", "arm64"] + ): + print(f"Invalid OS:Arch pair: {':'.join(pair)}") + return + + update_supports( + args.input_file, args.output_file, os_arch_pairs, args.log_level + ) + + +if __name__ == "__main__": + main() diff --git a/tools/version/check_version_in_ten_framework.py b/tools/version/check_version_in_ten_framework.py new file mode 100644 index 0000000000..cc3dc03f48 --- /dev/null +++ b/tools/version/check_version_in_ten_framework.py @@ -0,0 +1,135 @@ +# +# Copyright © 2024 Agora +# This file is part of TEN Framework, an open source project. +# Licensed under the Apache License, Version 2.0, with certain conditions. +# Refer to the "LICENSE" file in the root directory for more information. +# +import json +import os +from common import get_latest_git_tag + + +def check_preserved_metadata_version_of_ten_runtime( + repo_base_dir: str, git_version: str +) -> bool: + c_preserved_metadata_file_src_file = os.path.join( + repo_base_dir, + "core", + "src", + "ten_runtime", + "build_template", + "preserved_metadata.c", + ) + + if os.path.exists(c_preserved_metadata_file_src_file): + with open(c_preserved_metadata_file_src_file, "r") as f: + content = f.read() + + if f"version={git_version}" in content: + return True + + return False + + +def check_version_of_tman(repo_base_dir: str, git_version: str) -> bool: + tman_version_src_file = os.path.join( + repo_base_dir, + "core", + "src", + "ten_manager", + "src", + "version.rs", + ) + + if os.path.exists(tman_version_src_file): + with open(tman_version_src_file, "r") as f: + content = f.read() + + if f'VERSION: &str = "{git_version}"' in content: + return True + + return False + + +def check_version_of_system_packages( + repo_base_dir: str, git_version: str +) -> bool: + manifest_files = [ + # ten_runtime + os.path.join( + repo_base_dir, + "core", + "src", + "ten_runtime", + "manifest.json", + ), + # ten_runtime_go + os.path.join( + repo_base_dir, + "core", + "src", + "ten_runtime", + "binding", + "go", + "manifest.json", + ), + # ten_runtime_python + os.path.join( + repo_base_dir, + "core", + "src", + "ten_runtime", + "binding", + "python", + "manifest.json", + ), + ] + + for manifest_file in manifest_files: + if not os.path.exists(manifest_file): + return False + + with open(manifest_file, "r") as f: + data = json.load(f) + + if data.get("version") != git_version: + return False + + return True + + +def check_various_versions(repo_base_dir: str, git_version: str) -> bool: + if not check_preserved_metadata_version_of_ten_runtime( + repo_base_dir, git_version + ): + print("ten_runtime preserved_metadata version does not match.") + return False + + if not check_version_of_tman(repo_base_dir, git_version): + print("ten_manager version does not match.") + return False + + if not check_version_of_system_packages(repo_base_dir, git_version): + print("ten system package versions do not match.") + return False + + return True + + +if __name__ == "__main__": + repo_base_dir = os.path.abspath( + os.path.join(os.path.dirname(__file__), "..", "..") + ) + + # Change to the correct directory to get the correct git tag. + os.chdir(repo_base_dir) + + # Get the latest Git tag, and optionally strip leading 'v' if present. + git_version = get_latest_git_tag() + git_version = git_version.lstrip("v") + + if check_various_versions(repo_base_dir, git_version): + print("All versions match.") + else: + print("Versions do not match.") + exit(1) diff --git a/tools/version/common.py b/tools/version/common.py new file mode 100644 index 0000000000..65e518ad09 --- /dev/null +++ b/tools/version/common.py @@ -0,0 +1,241 @@ +# +# Copyright © 2024 Agora +# This file is part of TEN Framework, an open source project. +# Licensed under the Apache License, Version 2.0, with certain conditions. +# Refer to the "LICENSE" file in the root directory for more information. +# +import subprocess +import json +import os +from jinja2 import Template + + +class PkgInfo: + def __init__(self, pkg_type, pkg_name): + self.pkg_type = pkg_type + self.pkg_name = pkg_name + + def __eq__(self, value): + return ( + self.pkg_type == value.pkg_type and self.pkg_name == value.pkg_name + ) + + def __hash__(self): + return hash((self.pkg_type, self.pkg_name)) + + +def touch(path): + with open(path, "a"): + try: + os.utime(path, follow_symlinks=False) + except Exception: + try: + # If follow_symlinks parameter is not supported, fall back to + # default behavior + os.utime(path) + except Exception: + exit(1) + + +def get_latest_git_tag() -> str: + result = subprocess.run( + ["git", "describe", "--tags", "--abbrev=0", "--always"], + capture_output=True, + text=True, + ) + if result.returncode != 0: + raise Exception("Failed to execute git command") + return result.stdout.strip() + + +def update_c_preserved_metadata_version( + log_level: int, + year: str, + year_month: str, + version: str, + src_path: str, + template_path: str, +) -> None: + update_needed = False + + if os.path.exists(src_path): + with open(src_path, "r") as file: + content = file.read() + + if f"version={version}" not in content: + update_needed = True + if log_level > 0: + print( + f"Version mismatch found. Updating version in {src_path}." + ) + else: + update_needed = True + + if update_needed: + with open(template_path, "r") as file: + template_content = file.read() + + template = Template(template_content) + + rendered_content = template.render( + VERSION=version, YEAR=year, YEAR_MONTH=year_month + ) + + with open(src_path, "w") as file: + file.write(rendered_content) + else: + if log_level > 0: + print(f"No update needed for {src_path}; versions match.") + + +def update_version_in_manifest_json_file_for_pkgs( + log_level: int, + version: str, + src_path: str, + pkg_infos: list[PkgInfo], +) -> None: + if log_level > 0: + print(f"Checking {src_path} for updates...") + + if not os.path.exists(src_path): + touch(src_path) + + with open(src_path, "r") as file: + data = json.load(file) + + pkg_info = PkgInfo(data.get("type"), data.get("name")) + if pkg_info in pkg_infos and data.get("version") != version: + if log_level > 0: + print(f"Updating version in {src_path}.") + + # Update the version in the JSON data. + data["version"] = version + with open(src_path, "w") as file: + json.dump(data, file, indent=2) + # Notify that the content of the src_path has changed. + touch(src_path) + + +# { +# "type": "", +# "name": "", +# "version": "...", <- update here +# ... +# } +def update_version_in_manifest_json_file( + log_level: int, + version: str, + src_path: str, +) -> None: + if log_level > 0: + print(f"Checking {src_path} for updates...") + + if not os.path.exists(src_path): + touch(src_path) + + with open(src_path, "r") as file: + data = json.load(file) + + if data.get("version") != version: + if log_level > 0: + print(f"Updating version in {src_path}.") + + # Update the version in the JSON data. + data["version"] = version + + with open(src_path, "w") as file: + json.dump(data, file, indent=2) + + # Notify that the content of the src_path has changed. + touch(src_path) + else: + if log_level > 0: + print(f"No update needed for {src_path}; versions match.") + + +# { +# ... +# "dependencies": [ +# { +# "type": "", +# "name": "", +# "version": "...", <- update here +# } +# } +# ... +# } +def update_dependency_version_in_manifest_json_file( + log_level: int, + version: str, + src_path: str, + pkgs: list[PkgInfo], +) -> None: + if log_level > 0: + print(f"Checking {src_path} for updates...") + + if not os.path.exists(src_path): + touch(src_path) + + with open(src_path, "r") as file: + data = json.load(file) + + updated = False + for dependency in data.get("dependencies", []): + dependent_pkg = PkgInfo(dependency.get("type"), dependency.get("name")) + + if dependent_pkg in pkgs and dependency.get("version") != version: + dependency["version"] = version + updated = True + + if updated: + if log_level > 0: + print(f"Updating version in {src_path}.") + + with open(src_path, "w") as file: + json.dump(data, file, indent=2) + + # Notify that the content of the src_path has changed. + touch(src_path) + else: + if log_level > 0: + print(f"No update needed for {src_path}; versions match.") + + +def update_version_source_file_of_tman( + log_level: int, + year: str, + year_month: str, + version: str, + src_path: str, + template_path: str, +) -> None: + update_needed = False + + if os.path.exists(src_path): + with open(src_path, "r") as file: + content = file.read() + + if f'VERSION: &str = "{version}"' not in content: + update_needed = True + if log_level > 0: + print( + f"Version mismatch found. Updating version in {src_path}." + ) + else: + update_needed = True + + if update_needed: + with open(template_path, "r") as file: + template_content = file.read() + + template = Template(template_content) + + rendered_content = template.render( + VERSION=version, YEAR=year, YEAR_MONTH=year_month + ) + + with open(src_path, "w") as file: + file.write(rendered_content) + else: + if log_level > 0: + print(f"No update needed for {src_path}; versions match.") diff --git a/tools/version/update_version_from_git.py b/tools/version/update_version_from_git.py deleted file mode 100644 index 7021359282..0000000000 --- a/tools/version/update_version_from_git.py +++ /dev/null @@ -1,249 +0,0 @@ -# -# Copyright © 2024 Agora -# This file is part of TEN Framework, an open source project. -# Licensed under the Apache License, Version 2.0, with certain conditions. -# Refer to the "LICENSE" file in the root directory for more information. -# -import subprocess -import json -import argparse -import os -import datetime -from jinja2 import Template -from build.scripts import touch - - -class ArgumentInfo(argparse.Namespace): - def __init__(self): - super().__init__() - self.repo_base_dir: str - self.log_level: int - self.pkg_type: str | None = None - self.pkg_name: str | None = None - self.c_preserved_metadata_path: list[tuple[str, str]] = [] - self.c_preserved_metadata_not_in_output_path: list[tuple[str, str]] = [] - self.version_update_manifest_path: list[tuple[str, str]] = [] - self.dependency_version_update_manifest_path: list[tuple[str, str]] = [] - - -def get_latest_git_tag() -> str: - result = subprocess.run( - ["git", "describe", "--tags", "--abbrev=0"], - capture_output=True, - text=True, - ) - if result.returncode != 0: - raise Exception("Failed to execute git command") - return result.stdout.strip() - - -def update_c_preserved_metadata_file( - args: ArgumentInfo, - year: str, - year_month: str, - version: str, - src_path: str, - dest_path: str, -) -> None: - if args.log_level > 0: - print(f"Checking {dest_path} for updates...") - - update_needed = False - - if os.path.exists(dest_path): - with open(dest_path, "r") as file: - content = file.read() - - if f"version={version}" not in content: - update_needed = True - if args.log_level > 0: - print( - f"Version mismatch found. Updating version in {dest_path}." - ) - else: - update_needed = True - - if update_needed: - with open(src_path, "r") as file: - template_content = file.read() - - template = Template(template_content) - - rendered_content = template.render( - VERSION=version, YEAR=year, YEAR_MONTH=year_month - ) - - with open(dest_path, "w") as file: - file.write(rendered_content) - else: - if args.log_level > 0: - print(f"No update needed for {dest_path}; versions match.") - - -# { -# "type": "", -# "name": "", -# "version": "...", <- update here -# ... -# } -def update_version_in_manifest_json_file( - args: ArgumentInfo, - version: str, - src_path: str, - dest_path: str, -) -> None: - if args.log_level > 0: - print(f"Checking {src_path} for updates...") - - if not os.path.exists(dest_path): - touch.touch(dest_path) - - with open(src_path, "r") as file: - data = json.load(file) - - if data.get("version") != version: - if args.log_level > 0: - print(f"Updating version in {src_path}.") - - # Update the version in the JSON data. - data["version"] = version - - with open(src_path, "w") as file: - json.dump(data, file, indent=2) - - # Notify that the content of the src_path has changed. - touch.touch(dest_path) - else: - if args.log_level > 0: - print(f"No update needed for {src_path}; versions match.") - - -# { -# ... -# "dependencies": [ -# { -# "type": "", -# "name": "", -# "version": "...", <- update here -# } -# } -# ... -# } -def update_dependency_manifest_json_file( - args: ArgumentInfo, - version: str, - src_path: str, - dest_path: str, -) -> None: - if args.log_level > 0: - print(f"Checking {src_path} for updates...") - - if not os.path.exists(dest_path): - touch.touch(dest_path) - - with open(src_path, "r") as file: - data = json.load(file) - - updated = False - for dependency in data.get("dependencies", []): - if ( - dependency.get("type") == args.pkg_type - and dependency.get("name") == args.pkg_name - ): - dependency["version"] = version - updated = True - - if updated: - if args.log_level > 0: - print(f"Updating version in {src_path}.") - - with open(src_path, "w") as file: - json.dump(data, file, indent=2) - - # Notify that the content of the src_path has changed. - touch.touch(dest_path) - else: - if args.log_level > 0: - print(f"No update needed for {src_path}; versions match.") - - -def main(args: ArgumentInfo): - now = datetime.datetime.now() - year = now.strftime("%Y") - year_month = now.strftime("%Y-%m") - - # Change to the correct directory to get the correct git tag. - os.chdir(args.repo_base_dir) - - # Get the latest Git tag, and optionally strip leading 'v' if present. - git_version = get_latest_git_tag() - git_version = git_version.lstrip("v") - - if args.c_preserved_metadata_path: - for src, dest in args.c_preserved_metadata_path: - update_c_preserved_metadata_file( - args, year, year_month, git_version, src, dest - ) - - if args.c_preserved_metadata_not_in_output_path: - for src, dest in args.c_preserved_metadata_not_in_output_path: - update_c_preserved_metadata_file( - args, year, year_month, git_version, src, dest - ) - - if args.version_update_manifest_path: - for src, dest in args.version_update_manifest_path: - update_version_in_manifest_json_file(args, git_version, src, dest) - - if args.dependency_version_update_manifest_path: - for src, dest in args.dependency_version_update_manifest_path: - update_dependency_manifest_json_file(args, git_version, src, dest) - - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - - parser.add_argument("--repo-base-dir", type=str, required=True) - parser.add_argument("--log-level", type=int, required=True) - - parser.add_argument("--pkg-type", type=str) - parser.add_argument("--pkg-name", type=str) - - parser.add_argument( - "--c-preserved-metadata-path", - action="append", - nargs=2, - metavar=("src", "dest"), - ) - parser.add_argument( - "--c-preserved-metadata-not-in-output-path", - action="append", - nargs=2, - metavar=("src", "dest"), - ) - parser.add_argument( - "--version-update-manifest-path", - action="append", - nargs=2, - metavar=("src", "dest"), - ) - parser.add_argument( - "--dependency-version-update-manifest-path", - action="append", - nargs=2, - metavar=("src", "dest"), - ) - - arg_info = ArgumentInfo() - args = parser.parse_args(namespace=arg_info) - - if args.dependency_version_update_manifest_path and ( - not args.pkg_type or not args.pkg_name - ): - error_message = ( - "--pkg-type and --pkg-name are required when " - "--dependency-version-update-manifest-path is specified" - ) - parser.error(error_message) - - main(args) diff --git a/tools/version/update_version_in_ten_framework.py b/tools/version/update_version_in_ten_framework.py new file mode 100644 index 0000000000..e95a322d8e --- /dev/null +++ b/tools/version/update_version_in_ten_framework.py @@ -0,0 +1,325 @@ +# +# Copyright © 2024 Agora +# This file is part of TEN Framework, an open source project. +# Licensed under the Apache License, Version 2.0, with certain conditions. +# Refer to the "LICENSE" file in the root directory for more information. +# +import datetime +import json +import os +from common import ( + get_latest_git_tag, + update_c_preserved_metadata_version, + update_version_in_manifest_json_file, + update_dependency_version_in_manifest_json_file, + update_version_source_file_of_tman, + update_version_in_manifest_json_file_for_pkgs, + PkgInfo, +) + +MANIFEST_JSON_FILE = "manifest.json" +MANIFEST_JSON_TENT_FILE = "manifest.json.tent" + + +def __get_pkg_info_from_manifest_file(manifest_file: str) -> PkgInfo: + with open(manifest_file, "r") as f: + manifest = json.load(f) + return PkgInfo(manifest["type"], manifest["name"]) + + +def __collect_manifest_files(directory) -> list[str]: + manifests = [] + + for _, dirs, _ in os.walk(directory, followlinks=True): + for dir in dirs: + if os.path.exists(os.path.join(directory, dir, MANIFEST_JSON_FILE)): + manifests.append( + os.path.join( + directory, + dir, + MANIFEST_JSON_FILE, + ) + ) + break + + return manifests + + +def __collect_manifest_tent_files(directory) -> list[str]: + manifest_templates = [] + + for _, dirs, _ in os.walk(directory, followlinks=True): + for dir in dirs: + if os.path.exists( + os.path.join(directory, dir, MANIFEST_JSON_TENT_FILE) + ): + manifest_templates.append( + os.path.join( + directory, + dir, + MANIFEST_JSON_TENT_FILE, + ) + ) + break + + return manifest_templates + + +def update_c_preserved_metadata_version_of_ten_runtime_binary( + log_level, year, year_month, git_version +): + # Update the version in the C preserved metadata files. + c_preserved_metadata_file_src_file = os.path.join( + repo_base_dir, + "core", + "src", + "ten_runtime", + "build_template", + "preserved_metadata.c", + ) + + c_preserved_metadata_file_template_file = os.path.join( + repo_base_dir, + "core", + "src", + "ten_runtime", + "build_template", + "preserved_metadata.c.jinja2", + ) + + update_c_preserved_metadata_version( + log_level, + year, + year_month, + git_version, + c_preserved_metadata_file_src_file, + c_preserved_metadata_file_template_file, + ) + + +def update_version_of_tman( + log_level: int, + year: str, + year_month: str, + repo_base_dir: str, + git_version: str, +): + tman_version_src_file_path = os.path.join( + repo_base_dir, + "core", + "src", + "ten_manager", + "src", + "version.rs", + ) + + tman_version_template_file_path = os.path.join( + repo_base_dir, + "core", + "src", + "ten_manager", + "src", + "version.rs.jinja2", + ) + + update_version_source_file_of_tman( + log_level, + year, + year_month, + git_version, + tman_version_src_file_path, + tman_version_template_file_path, + ) + + +def collect_and_update_version_of_system_packages( + log_level, repo_base_dir, git_version +) -> list[PkgInfo]: + # Collect manifest files for ten_runtime and all corresponding system + # packages (python & go bindings). + manifest_files = [ + # ten_runtime + os.path.join( + repo_base_dir, + "core", + "src", + "ten_runtime", + MANIFEST_JSON_FILE, + ), + # ten_runtime_go + os.path.join( + repo_base_dir, + "core", + "src", + "ten_runtime", + "binding", + "go", + MANIFEST_JSON_FILE, + ), + # ten_runtime_python + os.path.join( + repo_base_dir, + "core", + "src", + "ten_runtime", + "binding", + "python", + MANIFEST_JSON_FILE, + ), + ] + + pkgInfos = [] + + for manifest_file in manifest_files: + update_version_in_manifest_json_file( + log_level, git_version, manifest_file + ) + pkgInfos.append(__get_pkg_info_from_manifest_file(manifest_file)) + + return pkgInfos + + +def collect_and_update_version_of_core_packages( + log_level, repo_base_dir, git_version +) -> list[PkgInfo]: + # Collect manifest files for all core packages. + core_apps_dir_path = os.path.join( + repo_base_dir, + "packages", + "core_apps", + ) + core_extensions_dir_path = os.path.join( + repo_base_dir, + "packages", + "core_extensions", + ) + core_protocols_dir_path = os.path.join( + repo_base_dir, + "packages", + "core_protocols", + ) + + manifest_files = ( + __collect_manifest_files(core_apps_dir_path) + + __collect_manifest_files(core_extensions_dir_path) + + __collect_manifest_files(core_protocols_dir_path) + ) + + manifest_template_files = ( + __collect_manifest_tent_files(core_apps_dir_path) + + __collect_manifest_tent_files(core_extensions_dir_path) + + __collect_manifest_tent_files(core_protocols_dir_path) + ) + + pkg_infos = [] + + for manifest_file in manifest_files: + update_version_in_manifest_json_file( + log_level, git_version, manifest_file + ) + pkg_infos.append(__get_pkg_info_from_manifest_file(manifest_file)) + + for manifest_template_file in manifest_template_files: + update_version_in_manifest_json_file( + log_level, git_version, manifest_template_file + ) + + # Some core packages are used and overwritten in tests. Update them as well. + manifests_in_tests = [] + + test_dir_path = os.path.join(repo_base_dir, "tests") + for root, _, files in os.walk(test_dir_path, followlinks=True): + for file in files: + if file == MANIFEST_JSON_FILE: + manifests_in_tests.append(os.path.join(root, file)) + + for manifest in manifests_in_tests: + update_version_in_manifest_json_file_for_pkgs( + log_level, git_version, manifest, pkg_infos + ) + + return pkg_infos + + +def update_dependencies_version( + log_level, repo_base_dir, git_version, dependencies +): + # Collect manifest files for system packages. + system_package_dir_path = os.path.join( + repo_base_dir, "core", "src", "ten_runtime", "binding" + ) + + # Collect manifest files for all packages. + packages_dir_path = os.path.join( + repo_base_dir, + "packages", + ) + + # Collect manifest files for testing packages. + test_dir_path = os.path.join(repo_base_dir, "tests") + + manifests = __collect_manifest_files(system_package_dir_path) + + for root, dirs, files in os.walk(packages_dir_path, followlinks=True): + for dir in dirs: + manifests += __collect_manifest_files( + os.path.join(packages_dir_path, dir) + ) + manifests += __collect_manifest_tent_files( + os.path.join(packages_dir_path, dir) + ) + + break + + for root, dirs, files in os.walk(test_dir_path, followlinks=True): + for file in files: + if file == MANIFEST_JSON_FILE: + manifests.append(os.path.join(root, file)) + + for manifest in manifests: + update_dependency_version_in_manifest_json_file( + log_level, + git_version, + manifest, + dependencies, + ) + + +if __name__ == "__main__": + now = datetime.datetime.now() + year = now.strftime("%Y") + year_month = now.strftime("%Y-%m") + + # Get the repo base directory. + repo_base_dir = os.path.abspath( + os.path.join(os.path.dirname(__file__), "..", "..") + ) + + # Change to the correct directory to get the correct git tag. + os.chdir(repo_base_dir) + + # Get the latest Git tag, and optionally strip leading 'v' if present. + git_version = get_latest_git_tag() + git_version = git_version.lstrip("v") + + log_level = 1 + + update_c_preserved_metadata_version_of_ten_runtime_binary( + log_level, year, year_month, git_version + ) + + update_version_of_tman( + log_level, year, year_month, repo_base_dir, git_version + ) + + system_pkgs = collect_and_update_version_of_system_packages( + log_level, repo_base_dir, git_version + ) + + core_pkgs = collect_and_update_version_of_core_packages( + log_level, repo_base_dir, git_version + ) + + update_dependencies_version( + log_level, repo_base_dir, git_version, system_pkgs + core_pkgs + )