From 064b2f4e7f5566c64fbfad6ffdc555f54e714cd6 Mon Sep 17 00:00:00 2001 From: victor <52110451+cs50victor@users.noreply.github.com> Date: Wed, 21 Feb 2024 21:28:11 -0500 Subject: [PATCH] fix: speech to text (#26) * refactor: remove STT encoding to stop empty response * refactor: better TTS struct design * chore: revert CI-RS toolchain * chore: remove unused dependencies * refactor: switch to agents --- .env.example | 2 +- .github/workflows/ci-rs.yml | 5 +- Cargo.lock | 1768 ++++++----------------------------- lkgpt/Cargo.toml | 7 +- lkgpt/src/llm.rs | 13 +- lkgpt/src/main.rs | 229 ++--- lkgpt/src/room_events.rs | 206 ++++ lkgpt/src/server.rs | 11 +- lkgpt/src/stt.rs | 204 ++-- lkgpt/src/tts.rs | 100 +- lkgpt/src/video.rs | 9 +- meet/app/r/[name]/page.tsx | 16 +- meet/app/r/page.tsx | 4 +- 13 files changed, 724 insertions(+), 1850 deletions(-) create mode 100644 lkgpt/src/room_events.rs diff --git a/.env.example b/.env.example index f127084..aeec59b 100644 --- a/.env.example +++ b/.env.example @@ -7,4 +7,4 @@ OPENAI_API_KEY="" OPENAI_ORG_ID="" ELEVENLABS_API_KEY="" -DEEPGRAM_API_KEY="" \ No newline at end of file +DEEPGRAM_API_KEY="" diff --git a/.github/workflows/ci-rs.yml b/.github/workflows/ci-rs.yml index 87fe206..f5eb7dc 100644 --- a/.github/workflows/ci-rs.yml +++ b/.github/workflows/ci-rs.yml @@ -25,7 +25,7 @@ jobs: fail-fast: false matrix: rust-toolchain: - - nightly-2023-09-20 + - nightly steps: - name: Get source code @@ -35,9 +35,8 @@ jobs: uses: brndnmtthws/rust-action@v1 with: toolchain: ${{ matrix.rust-toolchain }} - components: rustfmt, clippy, cargo-bloat + components: rustfmt, clippy enable-sccache: "true" - cargo-packages: "cargo-bloat" - name: Install binaries run: sudo apt-get update && sudo apt-get install -y clang pkg-config libavfilter-dev libavdevice-dev libavcodec-dev libavformat-dev libavutil-dev diff --git a/Cargo.lock b/Cargo.lock index 35985bc..916339a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -18,46 +18,19 @@ version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c71b1793ee61086797f5c80b6efa2b8ffa6d5dd703f118545808a7f2e27f7046" -[[package]] -name = "accesskit" -version = "0.11.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76eb1adf08c5bcaa8490b9851fd53cca27fa9880076f178ea9d29f05196728a8" - [[package]] name = "accesskit" version = "0.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6cb10ed32c63247e4e39a8f42e8e30fb9442fbf7878c8e4a9849e7e381619bea" -[[package]] -name = "accesskit_consumer" -version = "0.15.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04bb4d9e4772fe0d47df57d0d5dbe5d85dd05e2f37ae1ddb6b105e76be58fb00" -dependencies = [ - "accesskit 0.11.2", -] - [[package]] name = "accesskit_consumer" version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8c17cca53c09fbd7288667b22a201274b9becaa27f0b91bf52a526db95de45e6" dependencies = [ - "accesskit 0.12.2", -] - -[[package]] -name = "accesskit_macos" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "134d0acf6acb667c89d3332999b1a5df4edbc8d6113910f392ebb73f2b03bb56" -dependencies = [ - "accesskit 0.11.2", - "accesskit_consumer 0.15.2", - "objc2", - "once_cell", + "accesskit", ] [[package]] @@ -66,61 +39,35 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cd3b6ae1eabbfbced10e840fd3fce8a93ae84f174b3e4ba892ab7bcb42e477a7" dependencies = [ - "accesskit 0.12.2", - "accesskit_consumer 0.16.1", + "accesskit", + "accesskit_consumer", "objc2", "once_cell", ] -[[package]] -name = "accesskit_windows" -version = "0.14.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9eac0a7f2d7cd7a93b938af401d3d8e8b7094217989a7c25c55a953023436e31" -dependencies = [ - "accesskit 0.11.2", - "accesskit_consumer 0.15.2", - "arrayvec", - "once_cell", - "paste", - "windows 0.48.0", -] - [[package]] name = "accesskit_windows" version = "0.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "afcae27ec0974fc7c3b0b318783be89fd1b2e66dd702179fe600166a38ff4a0b" dependencies = [ - "accesskit 0.12.2", - "accesskit_consumer 0.16.1", + "accesskit", + "accesskit_consumer", "once_cell", "paste", "static_assertions", "windows 0.48.0", ] -[[package]] -name = "accesskit_winit" -version = "0.14.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "825d23acee1bd6d25cbaa3ca6ed6e73faf24122a774ec33d52c5c86c6ab423c0" -dependencies = [ - "accesskit 0.11.2", - "accesskit_macos 0.9.0", - "accesskit_windows 0.14.3", - "winit", -] - [[package]] name = "accesskit_winit" version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "88e39fcec2e10971e188730b7a76bab60647dacc973d4591855ebebcadfaa738" dependencies = [ - "accesskit 0.12.2", - "accesskit_macos 0.10.1", - "accesskit_windows 0.15.1", + "accesskit", + "accesskit_macos", + "accesskit_windows", "winit", ] @@ -162,7 +109,7 @@ dependencies = [ "flate2", "futures-core", "h2", - "http 0.2.11", + "http", "httparse", "httpdate", "itoa", @@ -197,7 +144,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d22475596539443685426b6bdadb926ad0ecaefdfc5fb05e5e3441f15463c511" dependencies = [ "bytestring", - "http 0.2.11", + "http", "regex", "serde", "tracing", @@ -706,22 +653,13 @@ version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b" -[[package]] -name = "bevy" -version = "0.11.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91c6d3ec4f89e85294dc97334c5b271ddc301fdf67ac9bb994fe44d9273e6ed7" -dependencies = [ - "bevy_internal 0.11.3", -] - [[package]] name = "bevy" version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e4bc7e09282a82a48d70ade0c4c1154b0fd7882a735a39c66766a5d0f4718ea9" dependencies = [ - "bevy_internal 0.12.1", + "bevy_internal", ] [[package]] @@ -731,21 +669,21 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1c3efee9165f9be9c6a180c6a9bbb4bd39ee0a98a594f4cb7127704e15415cdd" dependencies = [ "bevy-inspector-egui-derive", - "bevy_app 0.12.1", - "bevy_asset 0.12.1", - "bevy_core 0.12.1", - "bevy_core_pipeline 0.12.1", - "bevy_ecs 0.12.1", + "bevy_app", + "bevy_asset", + "bevy_core", + "bevy_core_pipeline", + "bevy_ecs", "bevy_egui", - "bevy_hierarchy 0.12.1", - "bevy_log 0.12.1", - "bevy_math 0.12.1", - "bevy_pbr 0.12.1", - "bevy_reflect 0.12.1", - "bevy_render 0.12.1", - "bevy_time 0.12.1", - "bevy_utils 0.12.1", - "bevy_window 0.12.1", + "bevy_hierarchy", + "bevy_log", + "bevy_math", + "bevy_pbr", + "bevy_reflect", + "bevy_render", + "bevy_time", + "bevy_utils", + "bevy_window", "egui", "image", "once_cell", @@ -764,44 +702,16 @@ dependencies = [ "syn 2.0.48", ] -[[package]] -name = "bevy_a11y" -version = "0.11.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "132c9e35a77c5395951f6d25fa2c52ee92296353426df4f961e60f3ff47e2e42" -dependencies = [ - "accesskit 0.11.2", - "bevy_app 0.11.3", - "bevy_derive 0.11.3", - "bevy_ecs 0.11.3", -] - [[package]] name = "bevy_a11y" version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68080288c932634f6563d3a8299efe0ddc9ea6787539c4c771ba250d089a94f0" dependencies = [ - "accesskit 0.12.2", - "bevy_app 0.12.1", - "bevy_derive 0.12.1", - "bevy_ecs 0.12.1", -] - -[[package]] -name = "bevy_app" -version = "0.11.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f557a7d59e1e16892d7544fc37316506ee598cb5310ef0365125a30783c11531" -dependencies = [ - "bevy_derive 0.11.3", - "bevy_ecs 0.11.3", - "bevy_reflect 0.11.3", - "bevy_tasks 0.11.3", - "bevy_utils 0.11.3", - "downcast-rs", - "wasm-bindgen", - "web-sys", + "accesskit", + "bevy_app", + "bevy_derive", + "bevy_ecs", ] [[package]] @@ -810,44 +720,16 @@ version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d41731817993f92e4363dd3335558e779e290bc71eefc0b5547052b85810907e" dependencies = [ - "bevy_derive 0.12.1", - "bevy_ecs 0.12.1", - "bevy_reflect 0.12.1", - "bevy_tasks 0.12.1", - "bevy_utils 0.12.1", + "bevy_derive", + "bevy_ecs", + "bevy_reflect", + "bevy_tasks", + "bevy_utils", "downcast-rs", "wasm-bindgen", "web-sys", ] -[[package]] -name = "bevy_asset" -version = "0.11.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9714af523da4cdf58c42a317e5ed40349708ad954a18533991fd64c8ae0a6f68" -dependencies = [ - "anyhow", - "async-channel 1.9.0", - "bevy_app 0.11.3", - "bevy_diagnostic 0.11.3", - "bevy_ecs 0.11.3", - "bevy_log 0.11.3", - "bevy_reflect 0.11.3", - "bevy_tasks 0.11.3", - "bevy_utils 0.11.3", - "bevy_winit 0.11.3", - "crossbeam-channel", - "downcast-rs", - "fastrand 1.9.0", - "js-sys", - "parking_lot", - "serde", - "thiserror", - "wasm-bindgen", - "wasm-bindgen-futures", - "web-sys", -] - [[package]] name = "bevy_asset" version = "0.12.1" @@ -857,14 +739,14 @@ dependencies = [ "async-broadcast", "async-fs", "async-lock 2.8.0", - "bevy_app 0.12.1", + "bevy_app", "bevy_asset_macros", - "bevy_ecs 0.12.1", - "bevy_log 0.12.1", - "bevy_reflect 0.12.1", - "bevy_tasks 0.12.1", - "bevy_utils 0.12.1", - "bevy_winit 0.12.1", + "bevy_ecs", + "bevy_log", + "bevy_reflect", + "bevy_tasks", + "bevy_utils", + "bevy_winit", "blake3", "crossbeam-channel", "downcast-rs", @@ -886,134 +768,72 @@ version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f48b9bbe4ec605e4910b5cd1e1a0acbfbe0b80af5f3bcc4489a9fdd1e80058c" dependencies = [ - "bevy_macro_utils 0.12.1", + "bevy_macro_utils", "proc-macro2", "quote", "syn 2.0.48", ] -[[package]] -name = "bevy_core" -version = "0.11.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d5272321be5fcf5ce2fb16023bc825bb10dfcb71611117296537181ce950f48" -dependencies = [ - "bevy_app 0.11.3", - "bevy_ecs 0.11.3", - "bevy_math 0.11.3", - "bevy_reflect 0.11.3", - "bevy_tasks 0.11.3", - "bevy_utils 0.11.3", - "bytemuck", -] - [[package]] name = "bevy_core" version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3daa24502a14839509f02407bc7e48299fe84d260877de23b60662de0f4f4b6c" dependencies = [ - "bevy_app 0.12.1", - "bevy_ecs 0.12.1", - "bevy_math 0.12.1", - "bevy_reflect 0.12.1", - "bevy_tasks 0.12.1", - "bevy_utils 0.12.1", + "bevy_app", + "bevy_ecs", + "bevy_math", + "bevy_reflect", + "bevy_tasks", + "bevy_utils", "bytemuck", ] -[[package]] -name = "bevy_core_pipeline" -version = "0.11.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67382fa9c96ce4f4e5833ed7cedd9886844a8f3284b4a717bd4ac738dcdea0c3" -dependencies = [ - "bevy_app 0.11.3", - "bevy_asset 0.11.3", - "bevy_core 0.11.3", - "bevy_derive 0.11.3", - "bevy_ecs 0.11.3", - "bevy_math 0.11.3", - "bevy_reflect 0.11.3", - "bevy_render 0.11.3", - "bevy_transform 0.11.3", - "bevy_utils 0.11.3", - "bitflags 2.4.1", - "radsort", - "serde", -] - [[package]] name = "bevy_core_pipeline" version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4b77c4fca6e90edbe2e72da7bc9aa7aed7dfdfded0920ae0a0c845f5e11084a" dependencies = [ - "bevy_app 0.12.1", - "bevy_asset 0.12.1", - "bevy_core 0.12.1", - "bevy_derive 0.12.1", - "bevy_ecs 0.12.1", - "bevy_log 0.12.1", - "bevy_math 0.12.1", - "bevy_reflect 0.12.1", - "bevy_render 0.12.1", - "bevy_transform 0.12.1", - "bevy_utils 0.12.1", + "bevy_app", + "bevy_asset", + "bevy_core", + "bevy_derive", + "bevy_ecs", + "bevy_log", + "bevy_math", + "bevy_reflect", + "bevy_render", + "bevy_transform", + "bevy_utils", "bitflags 2.4.1", "radsort", "serde", ] -[[package]] -name = "bevy_derive" -version = "0.11.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a44e4e2784a81430199e4157e02903a987a32127c773985506f020e7d501b62e" -dependencies = [ - "bevy_macro_utils 0.11.3", - "quote", - "syn 2.0.48", -] - [[package]] name = "bevy_derive" version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f484318350462c58ba3942a45a656c1fd6b6e484a6b6b7abc3a787ad1a51e500" dependencies = [ - "bevy_macro_utils 0.12.1", + "bevy_macro_utils", "quote", "syn 2.0.48", ] -[[package]] -name = "bevy_diagnostic" -version = "0.11.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6babb230dc383c98fdfc9603e3a7a2a49e1e2879dbe8291059ef37dca897932e" -dependencies = [ - "bevy_app 0.11.3", - "bevy_core 0.11.3", - "bevy_ecs 0.11.3", - "bevy_log 0.11.3", - "bevy_time 0.11.3", - "bevy_utils 0.11.3", - "sysinfo", -] - [[package]] name = "bevy_diagnostic" version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fa38ca5967d335cc1006a0e0f1a86c350e2f15fd1878449f61d04cd57a7c4060" dependencies = [ - "bevy_app 0.12.1", - "bevy_core 0.12.1", - "bevy_ecs 0.12.1", - "bevy_log 0.12.1", - "bevy_time 0.12.1", - "bevy_utils 0.12.1", + "bevy_app", + "bevy_core", + "bevy_ecs", + "bevy_log", + "bevy_time", + "bevy_utils", "sysinfo", ] @@ -1023,31 +843,10 @@ version = "0.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d13de5afeee849a6f5a9f35eace1a41efe01fc005ad85671c54876909216f2e6" dependencies = [ - "bevy 0.12.1", + "bevy", "interpolation", ] -[[package]] -name = "bevy_ecs" -version = "0.11.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "266144b36df7e834d5198049e037ecdf2a2310a76ce39ed937d1b0a6a2c4e8c6" -dependencies = [ - "async-channel 1.9.0", - "bevy_ecs_macros 0.11.3", - "bevy_ptr 0.11.3", - "bevy_reflect 0.11.3", - "bevy_tasks 0.11.3", - "bevy_utils 0.11.3", - "downcast-rs", - "event-listener 2.5.3", - "fixedbitset", - "rustc-hash", - "serde", - "thiserror", - "thread_local", -] - [[package]] name = "bevy_ecs" version = "0.12.1" @@ -1055,11 +854,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7709fbd22f81fb681534cd913c41e1cd18b17143368743281195d7f024b61aea" dependencies = [ "async-channel 1.9.0", - "bevy_ecs_macros 0.12.1", - "bevy_ptr 0.12.1", - "bevy_reflect 0.12.1", - "bevy_tasks 0.12.1", - "bevy_utils 0.12.1", + "bevy_ecs_macros", + "bevy_ptr", + "bevy_reflect", + "bevy_tasks", + "bevy_utils", "downcast-rs", "event-listener 2.5.3", "fixedbitset", @@ -1069,25 +868,13 @@ dependencies = [ "thread_local", ] -[[package]] -name = "bevy_ecs_macros" -version = "0.11.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7157a9c3be038d5008ee3f114feb6cf6b39c1d3d32ee21a7cacb8f81fccdfa80" -dependencies = [ - "bevy_macro_utils 0.11.3", - "proc-macro2", - "quote", - "syn 2.0.48", -] - [[package]] name = "bevy_ecs_macros" version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a8843aa489f159f25cdcd9fee75cd7d221a7098a71eaa72cb2d6b40ac4e3f1ba" dependencies = [ - "bevy_macro_utils 0.12.1", + "bevy_macro_utils", "proc-macro2", "quote", "syn 2.0.48", @@ -1100,105 +887,29 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c90c01202dbcebc03315a01ea71553b35e1f20b0da6b1cc8c2605344032a3d96" dependencies = [ "arboard", - "bevy 0.12.1", + "bevy", "egui", "thread_local", "webbrowser", ] -[[package]] -name = "bevy_encase_derive" -version = "0.11.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0ac0f55ad6bca1be7b0f35bbd5fc95ed3d31e4e9db158fee8e5327f59006001" -dependencies = [ - "bevy_macro_utils 0.11.3", - "encase_derive_impl", -] - [[package]] name = "bevy_encase_derive" version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5328a3715e933ebbff07d0e99528dc423c4f7a53590ed1ac19a120348b028990" dependencies = [ - "bevy_macro_utils 0.12.1", + "bevy_macro_utils", "encase_derive_impl", ] -[[package]] -name = "bevy_eventlistener" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b089d6bee5e75b1bc2793752ce94c461d4ce581281c15ca44cbd84bb7794073" -dependencies = [ - "bevy_eventlistener_core 0.5.1", - "bevy_eventlistener_derive 0.5.1", -] - -[[package]] -name = "bevy_eventlistener" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db8ee25ddf3d72e8501eb11d99870f7b6a484a111a91f52cef22c3e39302c9b0" -dependencies = [ - "bevy_eventlistener_core 0.6.1", - "bevy_eventlistener_derive 0.6.1", -] - -[[package]] -name = "bevy_eventlistener_core" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4c94b227693f271fd7dad522214f1a371c7856d88cedb6473d4c945e8169103" -dependencies = [ - "bevy_app 0.11.3", - "bevy_ecs 0.11.3", - "bevy_hierarchy 0.11.3", - "bevy_utils 0.11.3", -] - -[[package]] -name = "bevy_eventlistener_core" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "790c6a7ff1f6857a2927becb24b24fe16817bb998903f44342e73c1ed720c369" -dependencies = [ - "bevy_app 0.12.1", - "bevy_ecs 0.12.1", - "bevy_hierarchy 0.12.1", - "bevy_utils 0.12.1", -] - -[[package]] -name = "bevy_eventlistener_derive" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b88fc692e14d483c9aedd74958d1a39f5adbc6f94d97d87a2b330e27fa6a616" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.48", -] - -[[package]] -name = "bevy_eventlistener_derive" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b1150b00713155ef8e9ba879ebbaaaaba43192eaf7fb87555ba1ceb21f71708" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.48", -] - [[package]] name = "bevy_gaussian_splatting" version = "2.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4ad2743463900d9da3df1df14714b9a07beff505caebc20232cbd2468d7d2748" dependencies = [ - "bevy 0.12.1", + "bevy", "bevy-inspector-egui", "bevy_panorbit_camera", "byte-unit", @@ -1215,27 +926,7 @@ dependencies = [ "typenum", "wasm-bindgen", "web-sys", - "wgpu 0.17.2", -] - -[[package]] -name = "bevy_gizmos" -version = "0.11.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e286a3e7276431963f4aa29165ea5429fa7dbbc6d5c5ba0c531e7dd44ecc88a2" -dependencies = [ - "bevy_app 0.11.3", - "bevy_asset 0.11.3", - "bevy_core 0.11.3", - "bevy_core_pipeline 0.11.3", - "bevy_ecs 0.11.3", - "bevy_math 0.11.3", - "bevy_pbr 0.11.3", - "bevy_reflect 0.11.3", - "bevy_render 0.11.3", - "bevy_sprite 0.11.3", - "bevy_transform 0.11.3", - "bevy_utils 0.11.3", + "wgpu", ] [[package]] @@ -1244,32 +935,17 @@ version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "db232274ddca2ae452eb2731b98267b795d133ddd14013121bc7daddde1c7491" dependencies = [ - "bevy_app 0.12.1", - "bevy_asset 0.12.1", - "bevy_core 0.12.1", - "bevy_core_pipeline 0.12.1", - "bevy_ecs 0.12.1", - "bevy_math 0.12.1", - "bevy_reflect 0.12.1", - "bevy_render 0.12.1", - "bevy_sprite 0.12.1", - "bevy_transform 0.12.1", - "bevy_utils 0.12.1", -] - -[[package]] -name = "bevy_hierarchy" -version = "0.11.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "103f8f58416ac6799b8c7f0b418f1fac9eba44fa924df3b0e16b09256b897e3d" -dependencies = [ - "bevy_app 0.11.3", - "bevy_core 0.11.3", - "bevy_ecs 0.11.3", - "bevy_log 0.11.3", - "bevy_reflect 0.11.3", - "bevy_utils 0.11.3", - "smallvec", + "bevy_app", + "bevy_asset", + "bevy_core", + "bevy_core_pipeline", + "bevy_ecs", + "bevy_math", + "bevy_reflect", + "bevy_render", + "bevy_sprite", + "bevy_transform", + "bevy_utils", ] [[package]] @@ -1278,123 +954,61 @@ version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06bd477152ce2ae1430f5e0a4f19216e5785c22fee1ab23788b5982dc59d1a55" dependencies = [ - "bevy_app 0.12.1", - "bevy_core 0.12.1", - "bevy_ecs 0.12.1", - "bevy_log 0.12.1", - "bevy_reflect 0.12.1", - "bevy_utils 0.12.1", + "bevy_app", + "bevy_core", + "bevy_ecs", + "bevy_log", + "bevy_reflect", + "bevy_utils", "smallvec", ] -[[package]] -name = "bevy_input" -version = "0.11.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffbd935401101ac8003f3c3aea70788c65ad03f7a32716a10608bedda7a648bc" -dependencies = [ - "bevy_app 0.11.3", - "bevy_ecs 0.11.3", - "bevy_math 0.11.3", - "bevy_reflect 0.11.3", - "bevy_utils 0.11.3", - "thiserror", -] - [[package]] name = "bevy_input" version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cab9a599189b2a694c182d60cd52219dd9364f9892ff542d87799b8e45d9e6dc" dependencies = [ - "bevy_app 0.12.1", - "bevy_ecs 0.12.1", - "bevy_math 0.12.1", - "bevy_reflect 0.12.1", - "bevy_utils 0.12.1", + "bevy_app", + "bevy_ecs", + "bevy_math", + "bevy_reflect", + "bevy_utils", "thiserror", ] -[[package]] -name = "bevy_internal" -version = "0.11.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0e35a9b2bd29aa784b3cc416bcbf2a298f69f00ca51fd042ea39d9af7fad37e" -dependencies = [ - "bevy_a11y 0.11.3", - "bevy_app 0.11.3", - "bevy_asset 0.11.3", - "bevy_core 0.11.3", - "bevy_core_pipeline 0.11.3", - "bevy_derive 0.11.3", - "bevy_diagnostic 0.11.3", - "bevy_ecs 0.11.3", - "bevy_gizmos 0.11.3", - "bevy_hierarchy 0.11.3", - "bevy_input 0.11.3", - "bevy_log 0.11.3", - "bevy_math 0.11.3", - "bevy_pbr 0.11.3", - "bevy_ptr 0.11.3", - "bevy_reflect 0.11.3", - "bevy_render 0.11.3", - "bevy_scene 0.11.3", - "bevy_sprite 0.11.3", - "bevy_tasks 0.11.3", - "bevy_time 0.11.3", - "bevy_transform 0.11.3", - "bevy_utils 0.11.3", - "bevy_window 0.11.3", -] - [[package]] name = "bevy_internal" version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f124bece9831afd80897815231072d51bfe3ac58c6bb58eca8880963b6d0487c" dependencies = [ - "bevy_a11y 0.12.1", - "bevy_app 0.12.1", - "bevy_asset 0.12.1", - "bevy_core 0.12.1", - "bevy_core_pipeline 0.12.1", - "bevy_derive 0.12.1", - "bevy_diagnostic 0.12.1", - "bevy_ecs 0.12.1", - "bevy_gizmos 0.12.1", - "bevy_hierarchy 0.12.1", - "bevy_input 0.12.1", - "bevy_log 0.12.1", - "bevy_math 0.12.1", - "bevy_ptr 0.12.1", - "bevy_reflect 0.12.1", - "bevy_render 0.12.1", - "bevy_scene 0.12.1", - "bevy_sprite 0.12.1", - "bevy_tasks 0.12.1", + "bevy_a11y", + "bevy_app", + "bevy_asset", + "bevy_core", + "bevy_core_pipeline", + "bevy_derive", + "bevy_diagnostic", + "bevy_ecs", + "bevy_gizmos", + "bevy_hierarchy", + "bevy_input", + "bevy_log", + "bevy_math", + "bevy_ptr", + "bevy_reflect", + "bevy_render", + "bevy_scene", + "bevy_sprite", + "bevy_tasks", "bevy_text", - "bevy_time 0.12.1", - "bevy_transform 0.12.1", + "bevy_time", + "bevy_transform", "bevy_ui", - "bevy_utils 0.12.1", - "bevy_window 0.12.1", - "bevy_winit 0.12.1", -] - -[[package]] -name = "bevy_log" -version = "0.11.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07dcc615ff4f617b06c3f9522fca3c55d56f9644db293318f8ab68fcdea5d4fe" -dependencies = [ - "android_log-sys", - "bevy_app 0.11.3", - "bevy_ecs 0.11.3", - "bevy_utils 0.11.3", - "console_error_panic_hook", - "tracing-log 0.1.4", - "tracing-subscriber", - "tracing-wasm", + "bevy_utils", + "bevy_window", + "bevy_winit", ] [[package]] @@ -1404,27 +1018,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0dc10ba1d225a8477b9e80a1bf797d8a8b8274e83c9b24fb4d9351aec9229755" dependencies = [ "android_log-sys", - "bevy_app 0.12.1", - "bevy_ecs 0.12.1", - "bevy_utils 0.12.1", + "bevy_app", + "bevy_ecs", + "bevy_utils", "console_error_panic_hook", "tracing-log 0.1.4", "tracing-subscriber", "tracing-wasm", ] -[[package]] -name = "bevy_macro_utils" -version = "0.11.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23ddc18d489b4e57832d4958cde7cd2f349f0ad91e5892ac9e2f2ee16546b981" -dependencies = [ - "quote", - "rustc-hash", - "syn 2.0.48", - "toml_edit 0.19.15", -] - [[package]] name = "bevy_macro_utils" version = "0.12.1" @@ -1438,16 +1040,6 @@ dependencies = [ "toml_edit 0.20.7", ] -[[package]] -name = "bevy_math" -version = "0.11.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78286a81fead796dc4b45ab14f4f02fe29a94423d3587bcfef872b2a8e0a474b" -dependencies = [ - "glam", - "serde", -] - [[package]] name = "bevy_math" version = "0.12.1" @@ -1458,15 +1050,6 @@ dependencies = [ "serde", ] -[[package]] -name = "bevy_mikktspace" -version = "0.11.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6cfc2a21ea47970a9b1f0f4735af3256a8f204815bd756110051d10f9d909497" -dependencies = [ - "glam", -] - [[package]] name = "bevy_mikktspace" version = "0.12.1" @@ -1476,368 +1059,58 @@ dependencies = [ "glam", ] -[[package]] -name = "bevy_mod_picking" -version = "0.16.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aebb845900d5da6314f261cb982079266840820032fa29679341a5a901e2c5b9" -dependencies = [ - "bevy_app 0.11.3", - "bevy_core 0.11.3", - "bevy_ecs 0.11.3", - "bevy_eventlistener 0.5.1", - "bevy_math 0.11.3", - "bevy_picking_core 0.16.0", - "bevy_picking_highlight 0.16.0", - "bevy_picking_input 0.16.0", - "bevy_picking_selection 0.16.0", - "bevy_reflect 0.11.3", - "bevy_render 0.11.3", - "bevy_utils 0.11.3", - "bevy_window 0.11.3", -] - -[[package]] -name = "bevy_mod_picking" -version = "0.17.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2392df63bc272d5a546633e0a19ef6d428173e3dd22bad0da0a59c6fddd6f2cf" -dependencies = [ - "bevy_app 0.12.1", - "bevy_core 0.12.1", - "bevy_ecs 0.12.1", - "bevy_eventlistener 0.6.1", - "bevy_math 0.12.1", - "bevy_picking_core 0.17.0", - "bevy_picking_highlight 0.17.0", - "bevy_picking_input 0.17.0", - "bevy_picking_raycast", - "bevy_picking_selection 0.17.0", - "bevy_picking_sprite", - "bevy_picking_ui", - "bevy_reflect 0.12.1", - "bevy_render 0.12.1", - "bevy_text", - "bevy_ui", - "bevy_utils 0.12.1", - "bevy_window 0.12.1", -] - -[[package]] -name = "bevy_mod_raycast" -version = "0.15.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b833e94bd71d2ef27445f3e146b07da4a7922fdb7b464f0f1221521f85a5725a" -dependencies = [ - "bevy 0.11.3", - "crossbeam-channel", -] - -[[package]] -name = "bevy_mod_raycast" -version = "0.16.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b127622dcbfd9e6a1661219af65965625901c2549b434f0c44c4460dd055dce" -dependencies = [ - "bevy_app 0.12.1", - "bevy_asset 0.12.1", - "bevy_derive 0.12.1", - "bevy_ecs 0.12.1", - "bevy_gizmos 0.12.1", - "bevy_math 0.12.1", - "bevy_reflect 0.12.1", - "bevy_render 0.12.1", - "bevy_sprite 0.12.1", - "bevy_transform 0.12.1", - "bevy_utils 0.12.1", - "bevy_window 0.12.1", - "crossbeam-channel", -] - [[package]] name = "bevy_panorbit_camera" version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fd9b28bdcb7d0996bc015051b1bb6b0fabf8047920ebcfc4eefe721c474d0ea4" dependencies = [ - "bevy 0.12.1", + "bevy", "bevy_easings", ] -[[package]] -name = "bevy_pbr" -version = "0.11.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63ca796a619e61cd43a0a3b11fde54644f7f0732a1fba1eef5d406248c6eba85" -dependencies = [ - "bevy_app 0.11.3", - "bevy_asset 0.11.3", - "bevy_core_pipeline 0.11.3", - "bevy_derive 0.11.3", - "bevy_ecs 0.11.3", - "bevy_math 0.11.3", - "bevy_reflect 0.11.3", - "bevy_render 0.11.3", - "bevy_transform 0.11.3", - "bevy_utils 0.11.3", - "bevy_window 0.11.3", - "bitflags 2.4.1", - "bytemuck", - "naga_oil 0.8.2", - "radsort", -] - [[package]] name = "bevy_pbr" version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "520bfd2a898c74f84ea52cfb8eb061f37373ad15e623489d5f75d27ebd6138fe" dependencies = [ - "bevy_app 0.12.1", - "bevy_asset 0.12.1", - "bevy_core_pipeline 0.12.1", - "bevy_derive 0.12.1", - "bevy_ecs 0.12.1", - "bevy_math 0.12.1", - "bevy_reflect 0.12.1", - "bevy_render 0.12.1", - "bevy_transform 0.12.1", - "bevy_utils 0.12.1", - "bevy_window 0.12.1", + "bevy_app", + "bevy_asset", + "bevy_core_pipeline", + "bevy_derive", + "bevy_ecs", + "bevy_math", + "bevy_reflect", + "bevy_render", + "bevy_transform", + "bevy_utils", + "bevy_window", "bitflags 2.4.1", "bytemuck", "fixedbitset", - "naga_oil 0.10.1", + "naga_oil", "radsort", "smallvec", "thread_local", ] -[[package]] -name = "bevy_picking_core" -version = "0.16.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa8194c5c1baf84c4a377223ece3cb7514fc8b9736c0c758efbc188898d19b27" -dependencies = [ - "bevy_app 0.11.3", - "bevy_derive 0.11.3", - "bevy_ecs 0.11.3", - "bevy_eventlistener 0.5.1", - "bevy_math 0.11.3", - "bevy_reflect 0.11.3", - "bevy_render 0.11.3", - "bevy_utils 0.11.3", - "bevy_window 0.11.3", -] - -[[package]] -name = "bevy_picking_core" -version = "0.17.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0799488c4505d68886209afd77ff9c898b634abf7f97fffa8ae37a356cbc8ab2" -dependencies = [ - "bevy_app 0.12.1", - "bevy_derive 0.12.1", - "bevy_ecs 0.12.1", - "bevy_eventlistener 0.6.1", - "bevy_math 0.12.1", - "bevy_reflect 0.12.1", - "bevy_render 0.12.1", - "bevy_utils 0.12.1", - "bevy_window 0.12.1", -] - -[[package]] -name = "bevy_picking_highlight" -version = "0.16.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "603a4e422fa60c5d01e4c32b0293fd3cab06b437a700f0b83f314c010cc20461" -dependencies = [ - "bevy_app 0.11.3", - "bevy_asset 0.11.3", - "bevy_ecs 0.11.3", - "bevy_picking_core 0.16.0", - "bevy_picking_selection 0.16.0", - "bevy_reflect 0.11.3", -] - -[[package]] -name = "bevy_picking_highlight" -version = "0.17.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3cfada25fcbd459b0ecd57a3bb2861820915744bfabadef141a0881a06a7199" -dependencies = [ - "bevy_app 0.12.1", - "bevy_asset 0.12.1", - "bevy_ecs 0.12.1", - "bevy_pbr 0.12.1", - "bevy_picking_core 0.17.0", - "bevy_picking_selection 0.17.0", - "bevy_reflect 0.12.1", - "bevy_render 0.12.1", - "bevy_sprite 0.12.1", -] - -[[package]] -name = "bevy_picking_input" -version = "0.16.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71676712915c8ade223e58bccce29556977f497381497149de435358389d4b5e" -dependencies = [ - "bevy_app 0.11.3", - "bevy_ecs 0.11.3", - "bevy_hierarchy 0.11.3", - "bevy_input 0.11.3", - "bevy_math 0.11.3", - "bevy_picking_core 0.16.0", - "bevy_picking_selection 0.16.0", - "bevy_reflect 0.11.3", - "bevy_render 0.11.3", - "bevy_utils 0.11.3", - "bevy_window 0.11.3", -] - -[[package]] -name = "bevy_picking_input" -version = "0.17.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16a2b375461e3965c2bd4357d53c772eeafa4747812f54be09de373c0c203da4" -dependencies = [ - "bevy_app 0.12.1", - "bevy_ecs 0.12.1", - "bevy_hierarchy 0.12.1", - "bevy_input 0.12.1", - "bevy_math 0.12.1", - "bevy_picking_core 0.17.0", - "bevy_picking_selection 0.17.0", - "bevy_reflect 0.12.1", - "bevy_render 0.12.1", - "bevy_utils 0.12.1", - "bevy_window 0.12.1", -] - -[[package]] -name = "bevy_picking_raycast" -version = "0.17.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a2b76420a1e6d9c0fdd4f35056ffb2cfaa052d102d5b1951f517d9fd36b25ab" -dependencies = [ - "bevy_app 0.12.1", - "bevy_ecs 0.12.1", - "bevy_mod_raycast 0.16.0", - "bevy_picking_core 0.17.0", - "bevy_reflect 0.12.1", - "bevy_render 0.12.1", - "bevy_transform 0.12.1", - "bevy_window 0.12.1", -] - -[[package]] -name = "bevy_picking_selection" -version = "0.16.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5147eecdc5cd7212a1790aaf8b415a134ed422daf64e2633719454a7c7a86deb" -dependencies = [ - "bevy_app 0.11.3", - "bevy_ecs 0.11.3", - "bevy_eventlistener 0.5.1", - "bevy_input 0.11.3", - "bevy_picking_core 0.16.0", - "bevy_reflect 0.11.3", - "bevy_utils 0.11.3", -] - -[[package]] -name = "bevy_picking_selection" -version = "0.17.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "814c784c957653ab0baf77ff5563a90d4150f704ba0a173f9de8abe3037f803c" -dependencies = [ - "bevy_app 0.12.1", - "bevy_ecs 0.12.1", - "bevy_eventlistener 0.6.1", - "bevy_input 0.12.1", - "bevy_picking_core 0.17.0", - "bevy_reflect 0.12.1", - "bevy_utils 0.12.1", -] - -[[package]] -name = "bevy_picking_sprite" -version = "0.17.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cefe1b3172b06f93887f0ed8a01f8796e2d7486fe68a37035e38f59244c88663" -dependencies = [ - "bevy_app 0.12.1", - "bevy_asset 0.12.1", - "bevy_ecs 0.12.1", - "bevy_math 0.12.1", - "bevy_picking_core 0.17.0", - "bevy_render 0.12.1", - "bevy_sprite 0.12.1", - "bevy_transform 0.12.1", - "bevy_window 0.12.1", -] - -[[package]] -name = "bevy_picking_ui" -version = "0.17.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a020364f01021b1faf9cc4de85b592505440d5b6274fb39c9d9a916b7639872b" -dependencies = [ - "bevy_app 0.12.1", - "bevy_ecs 0.12.1", - "bevy_math 0.12.1", - "bevy_picking_core 0.17.0", - "bevy_render 0.12.1", - "bevy_transform 0.12.1", - "bevy_ui", - "bevy_window 0.12.1", -] - -[[package]] -name = "bevy_ptr" -version = "0.11.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72c7586401a46f7d8e436028225c1df5288f2e0082d066b247a82466fea155c6" - [[package]] name = "bevy_ptr" version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c77ec20c8fafcdc196508ef5ccb4f0400a8d193cb61f7b14a36ed9a25ad423cf" -[[package]] -name = "bevy_reflect" -version = "0.11.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0778197a1eb3e095a71417c74b7152ede02975cdc95b5ea4ddc5251ed00a2eb5" -dependencies = [ - "bevy_math 0.11.3", - "bevy_ptr 0.11.3", - "bevy_reflect_derive 0.11.3", - "bevy_utils 0.11.3", - "downcast-rs", - "erased-serde", - "glam", - "once_cell", - "parking_lot", - "serde", - "smallvec", - "smol_str", - "thiserror", -] - [[package]] name = "bevy_reflect" version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d7921f15fc944c9c8ad01d7dbcea6505b8909c6655cd9382bab1407181556038" dependencies = [ - "bevy_math 0.12.1", - "bevy_ptr 0.12.1", - "bevy_reflect_derive 0.12.1", - "bevy_utils 0.12.1", + "bevy_math", + "bevy_ptr", + "bevy_reflect_derive", + "bevy_utils", "downcast-rs", "erased-serde", "glam", @@ -1847,81 +1120,19 @@ dependencies = [ "thiserror", ] -[[package]] -name = "bevy_reflect_derive" -version = "0.11.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "342a4b2d09db22c48607d23ad59a056aff1ee004549050a51d490d375ba29528" -dependencies = [ - "bevy_macro_utils 0.11.3", - "bit-set", - "proc-macro2", - "quote", - "syn 2.0.48", - "uuid", -] - [[package]] name = "bevy_reflect_derive" version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4a8c5475f216e751ef4452a1306b00711f33d2d04d9f149e4c845dfeb6753a0" dependencies = [ - "bevy_macro_utils 0.12.1", + "bevy_macro_utils", "proc-macro2", "quote", "syn 2.0.48", "uuid", ] -[[package]] -name = "bevy_render" -version = "0.11.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39df4824b760928c27afc7b00fb649c7a63c9d76661ab014ff5c86537ee906cb" -dependencies = [ - "anyhow", - "async-channel 1.9.0", - "bevy_app 0.11.3", - "bevy_asset 0.11.3", - "bevy_core 0.11.3", - "bevy_derive 0.11.3", - "bevy_ecs 0.11.3", - "bevy_encase_derive 0.11.3", - "bevy_hierarchy 0.11.3", - "bevy_log 0.11.3", - "bevy_math 0.11.3", - "bevy_mikktspace 0.11.3", - "bevy_reflect 0.11.3", - "bevy_render_macros 0.11.3", - "bevy_tasks 0.11.3", - "bevy_time 0.11.3", - "bevy_transform 0.11.3", - "bevy_utils 0.11.3", - "bevy_window 0.11.3", - "bitflags 2.4.1", - "bytemuck", - "codespan-reporting", - "downcast-rs", - "encase", - "futures-lite 1.13.0", - "hexasphere", - "image", - "js-sys", - "naga 0.12.3", - "naga_oil 0.8.2", - "parking_lot", - "regex", - "serde", - "smallvec", - "thiserror", - "thread_local", - "wasm-bindgen", - "web-sys", - "wgpu 0.16.3", - "wgpu-hal 0.16.2", -] - [[package]] name = "bevy_render" version = "0.12.1" @@ -1929,23 +1140,23 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bdefdd3737125b0d94a6ff20bb70fa8cfe9d7d5dcd72ba4dfe6c5f1d30d9f6e4" dependencies = [ "async-channel 1.9.0", - "bevy_app 0.12.1", - "bevy_asset 0.12.1", - "bevy_core 0.12.1", - "bevy_derive 0.12.1", - "bevy_ecs 0.12.1", - "bevy_encase_derive 0.12.1", - "bevy_hierarchy 0.12.1", - "bevy_log 0.12.1", - "bevy_math 0.12.1", - "bevy_mikktspace 0.12.1", - "bevy_reflect 0.12.1", - "bevy_render_macros 0.12.1", - "bevy_tasks 0.12.1", - "bevy_time 0.12.1", - "bevy_transform 0.12.1", - "bevy_utils 0.12.1", - "bevy_window 0.12.1", + "bevy_app", + "bevy_asset", + "bevy_core", + "bevy_derive", + "bevy_ecs", + "bevy_encase_derive", + "bevy_hierarchy", + "bevy_log", + "bevy_math", + "bevy_mikktspace", + "bevy_reflect", + "bevy_render_macros", + "bevy_tasks", + "bevy_time", + "bevy_transform", + "bevy_utils", + "bevy_window", "bitflags 2.4.1", "bytemuck", "codespan-reporting", @@ -1955,27 +1166,15 @@ dependencies = [ "hexasphere", "image", "js-sys", - "naga 0.13.0", - "naga_oil 0.10.1", + "naga", + "naga_oil", "serde", "smallvec", "thiserror", "thread_local", "wasm-bindgen", "web-sys", - "wgpu 0.17.2", -] - -[[package]] -name = "bevy_render_macros" -version = "0.11.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bd08c740aac73363e32fb45af869b10cec65bcb76fe3e6cd0f8f7eebf4c36c9" -dependencies = [ - "bevy_macro_utils 0.11.3", - "proc-macro2", - "quote", - "syn 2.0.48", + "wgpu", ] [[package]] @@ -1984,97 +1183,50 @@ version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "64d86bfc5a1e7fbeeaec0c4ceab18155530f5506624670965db3415f75826bea" dependencies = [ - "bevy_macro_utils 0.12.1", + "bevy_macro_utils", "proc-macro2", "quote", "syn 2.0.48", ] -[[package]] -name = "bevy_scene" -version = "0.11.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd47e1263506153bef3a8be97fe2d856f206d315668c4f97510ca6cc181d9681" -dependencies = [ - "anyhow", - "bevy_app 0.11.3", - "bevy_asset 0.11.3", - "bevy_derive 0.11.3", - "bevy_ecs 0.11.3", - "bevy_hierarchy 0.11.3", - "bevy_reflect 0.11.3", - "bevy_render 0.11.3", - "bevy_transform 0.11.3", - "bevy_utils 0.11.3", - "ron", - "serde", - "thiserror", - "uuid", -] - [[package]] name = "bevy_scene" version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e7df078b5e406e37c8a1c6ba0d652bf105fde713ce3c3efda7263fe27467eee5" dependencies = [ - "bevy_app 0.12.1", - "bevy_asset 0.12.1", - "bevy_derive 0.12.1", - "bevy_ecs 0.12.1", - "bevy_hierarchy 0.12.1", - "bevy_reflect 0.12.1", - "bevy_render 0.12.1", - "bevy_transform 0.12.1", - "bevy_utils 0.12.1", + "bevy_app", + "bevy_asset", + "bevy_derive", + "bevy_ecs", + "bevy_hierarchy", + "bevy_reflect", + "bevy_render", + "bevy_transform", + "bevy_utils", "ron", "serde", "thiserror", "uuid", ] -[[package]] -name = "bevy_sprite" -version = "0.11.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68a8ca824fad75c6ef74cfbbba0a4ce3ccc435fa23d6bf3f003f260548813397" -dependencies = [ - "bevy_app 0.11.3", - "bevy_asset 0.11.3", - "bevy_core_pipeline 0.11.3", - "bevy_derive 0.11.3", - "bevy_ecs 0.11.3", - "bevy_log 0.11.3", - "bevy_math 0.11.3", - "bevy_reflect 0.11.3", - "bevy_render 0.11.3", - "bevy_transform 0.11.3", - "bevy_utils 0.11.3", - "bitflags 2.4.1", - "bytemuck", - "fixedbitset", - "guillotiere", - "rectangle-pack", - "thiserror", -] - [[package]] name = "bevy_sprite" version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c7cc0c9d946e17e3e0aaa202f182837bc796c4f862b2e5a805134f873f21cf7f" dependencies = [ - "bevy_app 0.12.1", - "bevy_asset 0.12.1", - "bevy_core_pipeline 0.12.1", - "bevy_derive 0.12.1", - "bevy_ecs 0.12.1", - "bevy_log 0.12.1", - "bevy_math 0.12.1", - "bevy_reflect 0.12.1", - "bevy_render 0.12.1", - "bevy_transform 0.12.1", - "bevy_utils 0.12.1", + "bevy_app", + "bevy_asset", + "bevy_core_pipeline", + "bevy_derive", + "bevy_ecs", + "bevy_log", + "bevy_math", + "bevy_reflect", + "bevy_render", + "bevy_transform", + "bevy_utils", "bitflags 2.4.1", "bytemuck", "fixedbitset", @@ -2084,20 +1236,6 @@ dependencies = [ "thiserror", ] -[[package]] -name = "bevy_tasks" -version = "0.11.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c73bbb847c83990d3927005090df52f8ac49332e1643d2ad9aac3cd2974e66bf" -dependencies = [ - "async-channel 1.9.0", - "async-executor", - "async-task", - "concurrent-queue", - "futures-lite 1.13.0", - "wasm-bindgen-futures", -] - [[package]] name = "bevy_tasks" version = "0.12.1" @@ -2119,110 +1257,72 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a9a79d49ca06170d69149949b134c14e8b99ace1444c1ca2cd4743b19d5b055" dependencies = [ "ab_glyph", - "bevy_app 0.12.1", - "bevy_asset 0.12.1", - "bevy_ecs 0.12.1", - "bevy_math 0.12.1", - "bevy_reflect 0.12.1", - "bevy_render 0.12.1", - "bevy_sprite 0.12.1", - "bevy_transform 0.12.1", - "bevy_utils 0.12.1", - "bevy_window 0.12.1", + "bevy_app", + "bevy_asset", + "bevy_ecs", + "bevy_math", + "bevy_reflect", + "bevy_render", + "bevy_sprite", + "bevy_transform", + "bevy_utils", + "bevy_window", "glyph_brush_layout", "serde", "thiserror", ] -[[package]] -name = "bevy_time" -version = "0.11.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d58d6dbae9c8225d8c0e0f04d2c5dbb71d22adc01ecd5ab3cebc364139e4a6d" -dependencies = [ - "bevy_app 0.11.3", - "bevy_ecs 0.11.3", - "bevy_reflect 0.11.3", - "bevy_utils 0.11.3", - "crossbeam-channel", - "thiserror", -] - [[package]] name = "bevy_time" version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6250d76eed3077128b6a3d004f9f198b01107800b9824051e32bb658054e837" dependencies = [ - "bevy_app 0.12.1", - "bevy_ecs 0.12.1", - "bevy_reflect 0.12.1", - "bevy_utils 0.12.1", + "bevy_app", + "bevy_ecs", + "bevy_reflect", + "bevy_utils", "crossbeam-channel", "thiserror", ] -[[package]] -name = "bevy_transform" -version = "0.11.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b9b0ac0149a57cd846cb357a35fc99286f9848e53d4481954608ac9552ed2d4" -dependencies = [ - "bevy_app 0.11.3", - "bevy_ecs 0.11.3", - "bevy_hierarchy 0.11.3", - "bevy_math 0.11.3", - "bevy_reflect 0.11.3", -] - [[package]] name = "bevy_transform" version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d541e0c292edbd96afae816ee680e02247422423ccd5dc635c1e211a20ed64be" dependencies = [ - "bevy_app 0.12.1", - "bevy_ecs 0.12.1", - "bevy_hierarchy 0.12.1", - "bevy_math 0.12.1", - "bevy_reflect 0.12.1", + "bevy_app", + "bevy_ecs", + "bevy_hierarchy", + "bevy_math", + "bevy_reflect", "thiserror", ] -[[package]] -name = "bevy_transform_gizmo" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d277fb359307f77c024539b3afd5fb47e25e00783d8b1dfaef989bc9b67035d" -dependencies = [ - "bevy 0.11.3", - "bevy_mod_picking 0.16.0", - "bevy_mod_raycast 0.15.0", -] - [[package]] name = "bevy_ui" version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d785e3b75dabcb2a8ad0d50933f8f3446d59e512cabc2d2a145e28c2bb8792ba" dependencies = [ - "bevy_a11y 0.12.1", - "bevy_app 0.12.1", - "bevy_asset 0.12.1", - "bevy_core_pipeline 0.12.1", - "bevy_derive 0.12.1", - "bevy_ecs 0.12.1", - "bevy_hierarchy 0.12.1", - "bevy_input 0.12.1", - "bevy_log 0.12.1", - "bevy_math 0.12.1", - "bevy_reflect 0.12.1", - "bevy_render 0.12.1", - "bevy_sprite 0.12.1", + "bevy_a11y", + "bevy_app", + "bevy_asset", + "bevy_core_pipeline", + "bevy_derive", + "bevy_ecs", + "bevy_hierarchy", + "bevy_input", + "bevy_log", + "bevy_math", + "bevy_reflect", + "bevy_render", + "bevy_sprite", "bevy_text", - "bevy_transform 0.12.1", - "bevy_utils 0.12.1", - "bevy_window 0.12.1", + "bevy_transform", + "bevy_utils", + "bevy_window", "bytemuck", "serde", "smallvec", @@ -2230,23 +1330,6 @@ dependencies = [ "thiserror", ] -[[package]] -name = "bevy_utils" -version = "0.11.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08d9484e32434ea84dc548cff246ce0c6f756c1336f5ea03f24ac120a48595c7" -dependencies = [ - "ahash 0.8.7", - "bevy_utils_proc_macros 0.11.3", - "getrandom", - "hashbrown 0.14.3", - "instant", - "petgraph", - "thiserror", - "tracing", - "uuid", -] - [[package]] name = "bevy_utils" version = "0.12.1" @@ -2254,7 +1337,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7915222f4a08ccc782e08d10b751b42e5f9d786e697d0cb3fd09333cb7e8b6ea" dependencies = [ "ahash 0.8.7", - "bevy_utils_proc_macros 0.12.1", + "bevy_utils_proc_macros", "getrandom", "hashbrown 0.14.3", "instant", @@ -2265,17 +1348,6 @@ dependencies = [ "uuid", ] -[[package]] -name = "bevy_utils_proc_macros" -version = "0.11.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5391b242c36f556db01d5891444730c83aa9dd648b6a8fd2b755d22cb3bddb57" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.48", -] - [[package]] name = "bevy_utils_proc_macros" version = "0.12.1" @@ -2293,7 +1365,7 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc7e9b05e9baf4a0ed31e33fc056c0d179d2b6c07a9d63b7e425a847286456b3" dependencies = [ - "bevy 0.12.1", + "bevy", "isahc", "js-sys", "wasm-bindgen", @@ -2301,60 +1373,20 @@ dependencies = [ "web-sys", ] -[[package]] -name = "bevy_window" -version = "0.11.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd584c0da7c4ada6557b09f57f30fb7cff21ccedc641473fc391574b4c9b7944" -dependencies = [ - "bevy_app 0.11.3", - "bevy_ecs 0.11.3", - "bevy_input 0.11.3", - "bevy_math 0.11.3", - "bevy_reflect 0.11.3", - "bevy_utils 0.11.3", - "raw-window-handle", -] - [[package]] name = "bevy_window" version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "41ee72bf7f974000e9b31bb971a89387f1432ba9413f35c4fef59fef49767260" dependencies = [ - "bevy_a11y 0.12.1", - "bevy_app 0.12.1", - "bevy_ecs 0.12.1", - "bevy_input 0.12.1", - "bevy_math 0.12.1", - "bevy_reflect 0.12.1", - "bevy_utils 0.12.1", - "raw-window-handle", -] - -[[package]] -name = "bevy_winit" -version = "0.11.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfdc044abdb95790c20053e6326760f0a2985f0dcd78613d397bf35f16039d53" -dependencies = [ - "accesskit_winit 0.14.4", - "approx", - "bevy_a11y 0.11.3", - "bevy_app 0.11.3", - "bevy_derive 0.11.3", - "bevy_ecs 0.11.3", - "bevy_hierarchy 0.11.3", - "bevy_input 0.11.3", - "bevy_math 0.11.3", - "bevy_tasks 0.11.3", - "bevy_utils 0.11.3", - "bevy_window 0.11.3", - "crossbeam-channel", + "bevy_a11y", + "bevy_app", + "bevy_ecs", + "bevy_input", + "bevy_math", + "bevy_reflect", + "bevy_utils", "raw-window-handle", - "wasm-bindgen", - "web-sys", - "winit", ] [[package]] @@ -2363,18 +1395,18 @@ version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1eb71f287eca9006dda998784c7b931e400ae2cc4c505da315882a8b082f21ad" dependencies = [ - "accesskit_winit 0.15.0", + "accesskit_winit", "approx", - "bevy_a11y 0.12.1", - "bevy_app 0.12.1", - "bevy_derive 0.12.1", - "bevy_ecs 0.12.1", - "bevy_hierarchy 0.12.1", - "bevy_input 0.12.1", - "bevy_math 0.12.1", - "bevy_tasks 0.12.1", - "bevy_utils 0.12.1", - "bevy_window 0.12.1", + "bevy_a11y", + "bevy_app", + "bevy_derive", + "bevy_ecs", + "bevy_hierarchy", + "bevy_input", + "bevy_math", + "bevy_tasks", + "bevy_utils", + "bevy_window", "crossbeam-channel", "raw-window-handle", "wasm-bindgen", @@ -2989,19 +2021,6 @@ dependencies = [ "cfg-if", ] -[[package]] -name = "crossbeam" -version = "0.8.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1137cd7e7fc0fb5d3c5a8678be38ec56e819125d8d7907411fe24ccb943faca8" -dependencies = [ - "crossbeam-channel", - "crossbeam-deque", - "crossbeam-epoch", - "crossbeam-queue", - "crossbeam-utils", -] - [[package]] name = "crossbeam-channel" version = "0.5.11" @@ -3030,15 +2049,6 @@ dependencies = [ "crossbeam-utils", ] -[[package]] -name = "crossbeam-queue" -version = "0.3.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df0346b5d5e76ac2fe4e327c5fd1118d6be7c51dfb18f9b7922923f287471e35" -dependencies = [ - "crossbeam-utils", -] - [[package]] name = "crossbeam-utils" version = "0.8.19" @@ -3136,17 +2146,6 @@ dependencies = [ "syn 2.0.48", ] -[[package]] -name = "d3d12" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8f0de2f5a8e7bd4a9eec0e3c781992a4ce1724f68aec7d7a3715344de8b39da" -dependencies = [ - "bitflags 1.3.2", - "libloading 0.7.4", - "winapi", -] - [[package]] name = "d3d12" version = "0.7.0" @@ -3526,8 +2525,9 @@ dependencies = [ "futures", "futures-util", "getrandom", - "http 0.2.11", + "http", "tokio", + "tokio-rustls", "tokio-tungstenite 0.20.1", "tokio-tungstenite-wasm", "tracing", @@ -3875,16 +2875,6 @@ dependencies = [ "xi-unicode", ] -[[package]] -name = "gpu-alloc" -version = "0.5.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22beaafc29b38204457ea030f6fb7a84c9e4dd1b86e311ba0542533453d87f62" -dependencies = [ - "bitflags 1.3.2", - "gpu-alloc-types 0.2.0", -] - [[package]] name = "gpu-alloc" version = "0.6.0" @@ -3892,16 +2882,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fbcd2dba93594b227a1f57ee09b8b9da8892c34d55aa332e034a228d0fe6a171" dependencies = [ "bitflags 2.4.1", - "gpu-alloc-types 0.3.0", -] - -[[package]] -name = "gpu-alloc-types" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54804d0d6bc9d7f26db4eaec1ad10def69b599315f487d32c334a80d1efe67a5" -dependencies = [ - "bitflags 1.3.2", + "gpu-alloc-types", ] [[package]] @@ -3973,7 +2954,7 @@ dependencies = [ "futures-core", "futures-sink", "futures-util", - "http 0.2.11", + "http", "indexmap 2.1.0", "slab", "tokio", @@ -4090,17 +3071,6 @@ dependencies = [ "itoa", ] -[[package]] -name = "http" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b32afd38673a8016f7c9ae69e5af41a58f81b1d31689040f2f1959594ce194ea" -dependencies = [ - "bytes", - "fnv", - "itoa", -] - [[package]] name = "http-body" version = "0.4.6" @@ -4108,7 +3078,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" dependencies = [ "bytes", - "http 0.2.11", + "http", "pin-project-lite", ] @@ -4141,7 +3111,7 @@ dependencies = [ "futures-core", "futures-util", "h2", - "http 0.2.11", + "http", "http-body", "httparse", "httpdate", @@ -4161,7 +3131,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590" dependencies = [ "futures-util", - "http 0.2.11", + "http", "hyper", "rustls", "tokio", @@ -4313,7 +3283,7 @@ dependencies = [ "encoding_rs", "event-listener 2.5.3", "futures-lite 1.13.0", - "http 0.2.11", + "http", "log", "mime", "once_cell", @@ -4648,19 +3618,15 @@ version = "0.0.1" dependencies = [ "actix-web", "anyhow", - "async-channel 2.1.1", "async-openai", "async-trait", "base64 0.21.7", - "bevy 0.12.1", + "bevy", "bevy_gaussian_splatting", - "bevy_mod_picking 0.17.0", "bevy_panorbit_camera", - "bevy_transform_gizmo", "bevy_web_asset", "bytes", "chrono", - "crossbeam", "crossbeam-channel", "deepgram", "dotenvy", @@ -4679,9 +3645,8 @@ dependencies = [ "serde", "serde_json", "tokio", - "tokio-tungstenite 0.21.0", "url", - "wgpu 0.17.2", + "wgpu", ] [[package]] @@ -4759,20 +3724,6 @@ dependencies = [ "autocfg", ] -[[package]] -name = "metal" -version = "0.24.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de11355d1f6781482d027a3b4d4de7825dcedb197bf573e0596d00008402d060" -dependencies = [ - "bitflags 1.3.2", - "block", - "core-graphics-types", - "foreign-types 0.3.2", - "log", - "objc", -] - [[package]] name = "metal" version = "0.26.0" @@ -4838,27 +3789,6 @@ version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a" -[[package]] -name = "naga" -version = "0.12.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbcc2e0513220fd2b598e6068608d4462db20322c0e77e47f6f488dfcfc279cb" -dependencies = [ - "bit-set", - "bitflags 1.3.2", - "codespan-reporting", - "hexf-parse", - "indexmap 1.9.3", - "log", - "num-traits", - "pp-rs", - "rustc-hash", - "spirv", - "termcolor", - "thiserror", - "unicode-xid", -] - [[package]] name = "naga" version = "0.13.0" @@ -4880,26 +3810,6 @@ dependencies = [ "unicode-xid", ] -[[package]] -name = "naga_oil" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8be942a5c21c58b9b0bf4d9b99db3634ddb7a916f8e1d1d0b71820cc4150e56b" -dependencies = [ - "bit-set", - "codespan-reporting", - "data-encoding", - "indexmap 1.9.3", - "naga 0.12.3", - "once_cell", - "regex", - "regex-syntax 0.6.29", - "rustc-hash", - "thiserror", - "tracing", - "unicode-ident", -] - [[package]] name = "naga_oil" version = "0.10.1" @@ -4910,7 +3820,7 @@ dependencies = [ "codespan-reporting", "data-encoding", "indexmap 1.9.3", - "naga 0.13.0", + "naga", "once_cell", "regex", "regex-syntax 0.7.5", @@ -5932,7 +4842,7 @@ dependencies = [ "futures-core", "futures-util", "h2", - "http 0.2.11", + "http", "http-body", "hyper", "hyper-rustls", @@ -6805,23 +5715,12 @@ dependencies = [ "futures-util", "log", "native-tls", + "rustls", "tokio", "tokio-native-tls", + "tokio-rustls", "tungstenite 0.20.1", -] - -[[package]] -name = "tokio-tungstenite" -version = "0.21.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c83b561d025642014097b66e6c1bb422783339e0909e4429cde4749d1990bc38" -dependencies = [ - "futures-util", - "log", - "native-tls", - "tokio", - "tokio-native-tls", - "tungstenite 0.21.0", + "webpki-roots", ] [[package]] @@ -6832,7 +5731,7 @@ checksum = "5ec8c7cf09b20184f946f114e3d8c0deca34368912c90100812861c14bb63b66" dependencies = [ "futures-channel", "futures-util", - "http 0.2.11", + "http", "httparse", "js-sys", "thiserror", @@ -7016,7 +5915,7 @@ dependencies = [ "base64 0.13.1", "byteorder", "bytes", - "http 0.2.11", + "http", "httparse", "log", "native-tls", @@ -7036,31 +5935,12 @@ dependencies = [ "byteorder", "bytes", "data-encoding", - "http 0.2.11", - "httparse", - "log", - "native-tls", - "rand", - "sha1", - "thiserror", - "url", - "utf-8", -] - -[[package]] -name = "tungstenite" -version = "0.21.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ef1a641ea34f399a848dea702823bbecfb4c486f911735368f1f137cb8257e1" -dependencies = [ - "byteorder", - "bytes", - "data-encoding", - "http 1.0.0", + "http", "httparse", "log", "native-tls", "rand", + "rustls", "sha1", "thiserror", "url", @@ -7334,6 +6214,12 @@ dependencies = [ "web-sys", ] +[[package]] +name = "webpki-roots" +version = "0.25.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1778a42e8b3b90bff8d0f5032bf22250792889a5cdc752aa0020c84abe3aaf10" + [[package]] name = "webrtc-sys" version = "0.3.0" @@ -7366,30 +6252,6 @@ version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9193164d4de03a926d909d3bc7c30543cecb35400c02114792c2cae20d5e2dbb" -[[package]] -name = "wgpu" -version = "0.16.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "480c965c9306872eb6255fa55e4b4953be55a8b64d57e61d7ff840d3dcc051cd" -dependencies = [ - "arrayvec", - "cfg-if", - "js-sys", - "log", - "naga 0.12.3", - "parking_lot", - "profiling", - "raw-window-handle", - "smallvec", - "static_assertions", - "wasm-bindgen", - "wasm-bindgen-futures", - "web-sys", - "wgpu-core 0.16.1", - "wgpu-hal 0.16.2", - "wgpu-types 0.16.1", -] - [[package]] name = "wgpu" version = "0.17.2" @@ -7400,7 +6262,7 @@ dependencies = [ "cfg-if", "js-sys", "log", - "naga 0.13.0", + "naga", "parking_lot", "profiling", "raw-window-handle", @@ -7409,32 +6271,9 @@ dependencies = [ "wasm-bindgen", "wasm-bindgen-futures", "web-sys", - "wgpu-core 0.17.1", - "wgpu-hal 0.17.2", - "wgpu-types 0.17.0", -] - -[[package]] -name = "wgpu-core" -version = "0.16.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f478237b4bf0d5b70a39898a66fa67ca3a007d79f2520485b8b0c3dfc46f8c2" -dependencies = [ - "arrayvec", - "bit-vec", - "bitflags 2.4.1", - "codespan-reporting", - "log", - "naga 0.12.3", - "parking_lot", - "profiling", - "raw-window-handle", - "rustc-hash", - "smallvec", - "thiserror", - "web-sys", - "wgpu-hal 0.16.2", - "wgpu-types 0.16.1", + "wgpu-core", + "wgpu-hal", + "wgpu-types", ] [[package]] @@ -7448,58 +6287,16 @@ dependencies = [ "bitflags 2.4.1", "codespan-reporting", "log", - "naga 0.13.0", - "parking_lot", - "profiling", - "raw-window-handle", - "rustc-hash", - "smallvec", - "thiserror", - "web-sys", - "wgpu-hal 0.17.2", - "wgpu-types 0.17.0", -] - -[[package]] -name = "wgpu-hal" -version = "0.16.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ecb3258078e936deee14fd4e0febe1cfe9bbb5ffef165cb60218d2ee5eb4448" -dependencies = [ - "android_system_properties", - "arrayvec", - "ash", - "bit-set", - "bitflags 2.4.1", - "block", - "core-graphics-types", - "d3d12 0.6.0", - "foreign-types 0.3.2", - "glow", - "gpu-alloc 0.5.4", - "gpu-allocator", - "gpu-descriptor", - "hassle-rs", - "js-sys", - "khronos-egl", - "libc", - "libloading 0.8.1", - "log", - "metal 0.24.0", - "naga 0.12.3", - "objc", + "naga", "parking_lot", "profiling", - "range-alloc", "raw-window-handle", - "renderdoc-sys", "rustc-hash", "smallvec", "thiserror", - "wasm-bindgen", "web-sys", - "wgpu-types 0.16.1", - "winapi", + "wgpu-hal", + "wgpu-types", ] [[package]] @@ -7515,9 +6312,9 @@ dependencies = [ "bitflags 2.4.1", "block", "core-graphics-types", - "d3d12 0.7.0", + "d3d12", "glow", - "gpu-alloc 0.6.0", + "gpu-alloc", "gpu-allocator", "gpu-descriptor", "hassle-rs", @@ -7526,8 +6323,8 @@ dependencies = [ "libc", "libloading 0.8.1", "log", - "metal 0.26.0", - "naga 0.13.0", + "metal", + "naga", "objc", "parking_lot", "profiling", @@ -7539,21 +6336,10 @@ dependencies = [ "thiserror", "wasm-bindgen", "web-sys", - "wgpu-types 0.17.0", + "wgpu-types", "winapi", ] -[[package]] -name = "wgpu-types" -version = "0.16.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0c153280bb108c2979eb5c7391cb18c56642dd3c072e55f52065e13e2a1252a" -dependencies = [ - "bitflags 2.4.1", - "js-sys", - "web-sys", -] - [[package]] name = "wgpu-types" version = "0.17.0" diff --git a/lkgpt/Cargo.toml b/lkgpt/Cargo.toml index 3f34882..a027692 100644 --- a/lkgpt/Cargo.toml +++ b/lkgpt/Cargo.toml @@ -8,7 +8,6 @@ default-run = "lkgpt" [dependencies] actix-web = "4.4.0" anyhow = "1.0.75" -async-channel = "2.1.0" async-openai = "0.18.0" async-trait = "0.1.74" base64 = "0.21.5" @@ -18,17 +17,14 @@ bevy = { version = "0.12.0", default-features = false, features = [ "bevy_render", ] } bevy_gaussian_splatting = { version = "2.0.2", default-features = true } -bevy_mod_picking = "0.17.0" bevy_panorbit_camera = "0.10.0" -bevy_transform_gizmo = "0.9.0" bevy_web_asset = "0.7.0" bytes = "1.5.0" chrono = "0.4.31" -crossbeam = "0.8.2" crossbeam-channel = "0.5.10" deepgram = { git = "https://github.com/cs50victor/deepgram-rust-sdk.git", branch = "live-fork" } dotenvy = "0.15.7" -ezsockets = { version = "0.6.1", features = ["client"] } +ezsockets = { version = "0.6.1", features = ["rustls"] } futures = "0.3.29" futures-intrusive = "0.5.0" image = { version = "0.24.7", default-features = false, features = ["png"] } @@ -45,6 +41,5 @@ rodio = { version = "0.17.3", features = ["symphonia-mp3"] } serde = { version = "1.0.192", features = ["derive"] } serde_json = "1.0.108" tokio = { version = "1.33.0", features = ["full"] } -tokio-tungstenite = { version = "0.21.0", features = ["native-tls"] } url = "2.5.0" wgpu = { version = "0.17.1" } diff --git a/lkgpt/src/llm.rs b/lkgpt/src/llm.rs index de4be58..16d10fd 100644 --- a/lkgpt/src/llm.rs +++ b/lkgpt/src/llm.rs @@ -26,8 +26,8 @@ pub struct LLMChannel { pub text_chat_prefix: &'static str, } -impl FromWorld for LLMChannel { - fn from_world(_world: &mut World) -> Self { +impl Default for LLMChannel { + fn default() -> Self { let open_ai_org_id = std::env::var(OPENAI_ORG_ID).unwrap(); let (tx, rx) = crossbeam_channel::unbounded::(); @@ -58,6 +58,12 @@ impl FromWorld for LLMChannel { } } +impl LLMChannel { + pub fn new() -> Self { + Self::default() + } +} + /// Stream text chunks to gpt as it's being generated. /// Note: if chunks don't end with space or punctuation (" ", ".", "?", "!"), /// the stream will wait for more text. @@ -90,7 +96,8 @@ pub fn run_llm( .build() .unwrap(); - async_runtime.rt.block_on(async { + let rt = async_runtime.rt.clone(); + rt.block_on(async { let mut gpt_resp_stream = llm_channel.client.chat().create_stream(request).await.unwrap(); while let Some(result) = gpt_resp_stream.next().await { diff --git a/lkgpt/src/main.rs b/lkgpt/src/main.rs index 88800b1..2d00626 100644 --- a/lkgpt/src/main.rs +++ b/lkgpt/src/main.rs @@ -2,6 +2,7 @@ mod controls; mod frame_capture; mod llm; +mod room_events; mod server; mod stt; mod tts; @@ -12,9 +13,8 @@ use std::sync::{ Arc, }; -use chrono::Utc; use frame_capture::scene::SceneController; -use image::{ImageBuffer, Rgb, RgbaImage}; +use image::RgbaImage; use livekit::{publication::LocalTrackPublication, webrtc::video_frame::VideoBuffer, Room}; // use actix_web::{middleware, web::Data, App, HttpServer}; use log::info; @@ -32,14 +32,14 @@ use livekit::{ use bevy::{ app::ScheduleRunnerPlugin, core::Name, core_pipeline::tonemapping::Tonemapping, log::LogPlugin, - prelude::*, render::renderer::RenderDevice, time::common_conditions::on_timer, + prelude::*, render::renderer::RenderDevice, tasks::AsyncComputeTaskPool, + time::common_conditions::on_timer, }; use bevy_panorbit_camera::{PanOrbitCamera, PanOrbitCameraPlugin}; use bevy_gaussian_splatting::{GaussianCloud, GaussianSplattingBundle, GaussianSplattingPlugin}; use pollster::FutureExt; -use stt::{receive_and_process_audio, STT}; use futures::StreamExt; use livekit::{ @@ -48,12 +48,13 @@ use livekit::{ DataPacketKind, RoomEvent, }; use log::{error, warn}; +use rodio::cpal::Sample as _; use serde::{Deserialize, Serialize}; -use video::ReceivedVideoFrame; +use stt::STT; use crate::{ - controls::WorldControlChannel, llm::LLMChannel, server::RoomData, stt::AudioInputChannel, - tts::create_tts, video::VideoChannel, + controls::WorldControlChannel, llm::LLMChannel, room_events::handle_room_events, + server::RoomData, tts::TTS, video::VideoChannel, }; pub const LIVEKIT_API_SECRET: &str = "LIVEKIT_API_SECRET"; @@ -71,7 +72,8 @@ pub struct AsyncRuntime { impl FromWorld for AsyncRuntime { fn from_world(_world: &mut World) -> Self { let rt = tokio::runtime::Builder::new_multi_thread().enable_all().build().unwrap(); - AsyncRuntime { rt: std::sync::Arc::new(rt) } + + Self { rt: std::sync::Arc::new(rt) } } } @@ -137,133 +139,11 @@ pub struct AppStateSync { #[derive(Resource)] pub struct LivekitRoom { + #[allow(dead_code)] room: std::sync::Arc, room_events: tokio::sync::mpsc::UnboundedReceiver, } -// SYSTEM -pub fn handle_room_events( - async_runtime: Res, - llm_channel: Res, - audio_channel: Res, - _video_channel: Res, - audio_syncer: ResMut, - mut room_events: ResMut, - single_frame_data: ResMut, -) { - while let Ok(event) = room_events.room_events.try_recv() { - println!("\n\n🤡received room event {:?}", event); - match event { - RoomEvent::TrackSubscribed { track, publication: _, participant: _user } => { - match track { - RemoteTrack::Audio(audio_track) => { - let audio_rtc_track = audio_track.rtc_track(); - let mut audio_stream = NativeAudioStream::new(audio_rtc_track); - let audio_channel_tx = audio_channel.tx.clone(); - let audio_should_stop = audio_syncer.should_stop.clone(); - async_runtime.rt.spawn(async move { - let mut start_time = Utc::now().time(); - let mut ms_audio_buffer: Vec = Vec::new(); - while let Some(frame) = audio_stream.next().await { - if audio_should_stop.load(Ordering::Relaxed) { - continue; - } - ms_audio_buffer.extend_from_slice(&frame.data); - let elapsed = (Utc::now().time() - start_time).num_milliseconds(); - // 10ms of audio - if elapsed >= 10 { - if let Err(e) = audio_channel_tx.send(frame.data.to_vec()) { - log::error!("Couldn't send audio frame to stt {e}"); - }; - start_time = Utc::now().time(); - } - } - }); - }, - RemoteTrack::Video(video_track) => { - let video_rtc_track = video_track.rtc_track(); - let pixel_size = 4; - let mut video_stream = NativeVideoStream::new(video_rtc_track); - - async_runtime.rt.spawn(async move { - // every 10 video frames - let mut i = 0; - while let Some(frame) = video_stream.next().await { - log::error!("🤡received video frame | {:#?}", frame); - // VIDEO FRAME BUFFER (i420_buffer) - let video_frame_buffer = frame.buffer.to_i420(); - let width = video_frame_buffer.width(); - let height = video_frame_buffer.height(); - let rgba_stride = video_frame_buffer.width() * pixel_size; - - let (stride_y, stride_u, stride_v) = video_frame_buffer.strides(); - let (data_y, data_u, data_v) = video_frame_buffer.data(); - - let rgba_buffer = RgbaImage::new(width, height); - let rgba_raw = unsafe { - std::slice::from_raw_parts_mut( - rgba_buffer.as_raw().as_ptr() as *mut u8, - rgba_buffer.len(), - ) - }; - - livekit::webrtc::native::yuv_helper::i420_to_rgba( - data_y, - stride_y, - data_u, - stride_u, - data_v, - stride_v, - rgba_raw, - rgba_stride, - video_frame_buffer.width() as i32, - video_frame_buffer.height() as i32, - ); - - if let Err(e) = rgba_buffer.save(format!("camera/{i}.png")) { - log::error!("Couldn't save video frame {e}"); - }; - i += 1; - } - info!("🤡ended video thread"); - }); - }, - }; - }, - RoomEvent::DataReceived { payload, kind, topic: _, participant: _ } => { - if kind == DataPacketKind::Reliable { - if let Some(payload) = payload.as_ascii() { - let room_text: serde_json::Result = - serde_json::from_str(payload.as_str()); - match room_text { - Ok(room_text) => { - if let Err(e) = - llm_channel.tx.send(format!("[chat]{} ", room_text.message)) - { - error!("Couldn't send the text to gpt {e}") - }; - }, - Err(e) => { - warn!("Couldn't deserialize room text. {e:#?}"); - }, - } - - info!("text from room {:#?}", payload.as_str()); - } - } - }, - // ignoring the participant for now, currently assuming there is only one participant - RoomEvent::TrackMuted { participant: _, publication: _ } => { - audio_syncer.should_stop.store(true, Ordering::Relaxed); - }, - RoomEvent::TrackUnmuted { participant: _, publication: _ } => { - audio_syncer.should_stop.store(false, Ordering::Relaxed); - }, - _ => info!("received room event {:?}", event), - } - } -} - pub struct TracksPublicationData { pub video_src: NativeVideoSource, pub video_pub: LocalTrackPublication, @@ -278,8 +158,8 @@ pub async fn publish_tracks( ) -> Result { let audio_src = NativeAudioSource::new( AudioSourceOptions::default(), - STT::SAMPLE_RATE, - STT::NUM_OF_CHANNELS, + TTS::SAMPLE_RATE, + TTS::NUM_OF_CHANNELS, ); let audio_track = @@ -325,7 +205,7 @@ fn setup_gaussian_cloud( ) { // let remote_file = Some("https://huggingface.co/datasets/cs50victor/splats/resolve/main/train/point_cloud/iteration_7000/point_cloud.gcloud"); // TODO: figure out how to load remote files later - let splat_file = "splats/train/point_cloud/iteration_7000/point_cloud.gcloud"; + let splat_file = "splats/bonsai/point_cloud/iteration_7000/point_cloud.gcloud"; log::info!("loading {}", splat_file); let cloud = asset_server.load(splat_file.to_string()); @@ -340,11 +220,16 @@ fn setup_gaussian_cloud( String::from("main_scene"), ); - commands.spawn((GaussianSplattingBundle { cloud, ..default() }, Name::new("gaussian_cloud"))); + let gs = GaussianSplattingBundle { cloud, ..default() }; + commands.spawn((gs, Name::new("gaussian_cloud"))); commands.spawn(( Camera3dBundle { - transform: Transform::from_translation(Vec3::new(0.0, 1.5, 5.0)), + transform: Transform { + translation: Vec3::new(-0.59989005, -0.88360703, -2.0863006), + rotation: Quat::from_xyzw(-0.97177905, -0.026801618, 0.13693734, -0.1901983), + scale: Vec3::new(1.0, 1.0, 1.0), + }, tonemapping: Tonemapping::None, camera: Camera { target: render_target, ..default() }, ..default() @@ -365,6 +250,7 @@ pub fn sync_bevy_and_server_resources( mut server_state_clone: ResMut, mut set_app_state: ResMut>, scene_controller: Res, + audio_syncer: Res, ) { if !server_state_clone.dirty { let participant_room_name = &(server_state_clone.state.lock().0).clone(); @@ -382,23 +268,53 @@ pub fn sync_bevy_and_server_resources( livekit_room, stream_frame_data, audio_src, + bot_name: _, video_pub: _, audio_pub: _, } = room_data; info!("initializing required bevy resources"); - let tts = async_runtime.rt.block_on(create_tts(audio_src)).unwrap(); - commands.init_resource::(); + let llm_channel = LLMChannel::new(); + let llm_tx = llm_channel.tx.clone(); + let llm_channel_tx = llm_tx.clone(); + + let tts = async_runtime.rt.block_on(TTS::new(audio_src)).unwrap(); + let stt = async_runtime.rt.block_on(STT::new(llm_tx)).unwrap(); + + let video_channel = VideoChannel::new(); + commands.insert_resource(llm_channel); commands.init_resource::(); - commands.init_resource::(); - commands.init_resource::(); + + commands.insert_resource(stt.clone()); + commands.init_resource::(); commands.insert_resource(tts); commands.insert_resource(stream_frame_data); - commands.insert_resource(livekit_room); + // commands.insert_resource(livekit_room); set_app_state.set(AppState::Active); + + let audio_syncer = audio_syncer.should_stop.clone(); + let rt = async_runtime.rt.clone(); + async_runtime.rt.spawn(handle_room_events( + rt, + llm_channel_tx, + stt, + video_channel, + audio_syncer, + livekit_room, + 4, + )); + /* + async_runtime: Res, + llm_channel: Res, + stt_client: ResMut, + _video_channel: Res, + audio_syncer: ResMut, + mut room_events: ResMut, + single_frame_data: ResMut, + */ server_state_clone.dirty = true; }, Err(e) => { @@ -443,7 +359,6 @@ fn main() { let config = AppConfig { width: 1920, height: 1080 }; - // setup frame capture app.insert_resource(frame_capture::scene::SceneController::new(config.width, config.height)); app.insert_resource(ClearColor(Color::rgb_u8(0, 0, 0))); @@ -473,25 +388,19 @@ fn main() { app.init_resource::(); app.add_event::(); - // app.add_systems(Update, move_camera); + app.add_systems(Update, move_camera); app.add_systems(Update, server::shutdown_bevy_remotely); - app.add_systems( - Update, - handle_room_events - .run_if(resource_exists::()) - .run_if(resource_exists::()) - .run_if(resource_exists::()) - .run_if(resource_exists::()), - ); - app.add_systems( - Update, - receive_and_process_audio - .run_if(resource_exists::()) - .run_if(resource_exists::()) - .run_if(resource_exists::()), - ); + // app.add_systems( + // Update, + // room_events::handle_room_events + // .run_if(resource_exists::()) + // .run_if(resource_exists::()) + // .run_if(resource_exists::()) + // .run_if(resource_exists::()), + // ); + app.add_systems( Update, llm::run_llm @@ -512,6 +421,8 @@ fn main() { fn move_camera(mut camera: Query<&mut Transform, With>) { for mut transform in camera.iter_mut() { - transform.translation.x += 5.0; + transform.translation.x += 0.0005; + transform.translation.y += 0.0005; + transform.translation.z += 0.0005; } } diff --git a/lkgpt/src/room_events.rs b/lkgpt/src/room_events.rs new file mode 100644 index 0000000..26bce41 --- /dev/null +++ b/lkgpt/src/room_events.rs @@ -0,0 +1,206 @@ +use std::sync::{ + atomic::{AtomicBool, Ordering}, + Arc, +}; + +use bevy::ecs::system::{Res, ResMut}; +use futures::{future, FutureExt, StreamExt}; +use image::RgbaImage; +use livekit::{ + track::RemoteTrack, + webrtc::{ + audio_stream::native::NativeAudioStream, video_frame::VideoBuffer, + video_stream::native::NativeVideoStream, + }, + DataPacketKind, RoomEvent, +}; +use log::{error, info, warn}; +use rodio::cpal::Sample; +use tokio::runtime::Runtime; + +use crate::{llm, stt::STT, video, AsyncRuntime, AudioSync, LivekitRoom, RoomText}; + +async fn handle_video(mut video_stream: NativeVideoStream, pixel_size: u32) { + // every 10 video frames + let mut i = 0; + info!("📸 handling video"); + while let Some(frame) = video_stream.next().await { + log::error!("🤡received video frame | {:#?}", frame); + // VIDEO FRAME BUFFER (i420_buffer) + let video_frame_buffer = frame.buffer.to_i420(); + let width = video_frame_buffer.width(); + let height = video_frame_buffer.height(); + let rgba_stride = video_frame_buffer.width() * pixel_size; + + let (stride_y, stride_u, stride_v) = video_frame_buffer.strides(); + let (data_y, data_u, data_v) = video_frame_buffer.data(); + + let rgba_buffer = RgbaImage::new(width, height); + let rgba_raw = unsafe { + std::slice::from_raw_parts_mut( + rgba_buffer.as_raw().as_ptr() as *mut u8, + rgba_buffer.len(), + ) + }; + + livekit::webrtc::native::yuv_helper::i420_to_rgba( + data_y, + stride_y, + data_u, + stride_u, + data_v, + stride_v, + rgba_raw, + rgba_stride, + video_frame_buffer.width() as i32, + video_frame_buffer.height() as i32, + ); + + if let Err(e) = rgba_buffer.save(format!("camera/{i}.png")) { + log::error!("Couldn't save video frame {e}"); + }; + i += 1; + } + info!("🤡ended video thread"); +} + +pub async fn handle_room_events( + async_runtime: Arc, + llm_channel_tx: crossbeam_channel::Sender, + stt_client: STT, + _video_channel: video::VideoChannel, + should_stop: Arc, + mut room_events: LivekitRoom, + pixel_size: u32, +) { + while let Some(event) = room_events.room_events.recv().await { + println!("\n\n🤡received room event {:?}", event); + match event { + RoomEvent::TrackSubscribed { track, publication: _, participant: _user } => { + match track { + RemoteTrack::Audio(audio_track) => { + let audio_rtc_track = audio_track.rtc_track(); + let mut audio_stream = NativeAudioStream::new(audio_rtc_track); + let audio_should_stop = should_stop.clone(); + let stt_client = stt_client.clone(); + let rt = async_runtime.clone(); + + std::thread::spawn(move || { + while let Some(frame) = rt.block_on(audio_stream.next()) { + if audio_should_stop.load(Ordering::Relaxed) { + continue; + } + + let audio_buffer = frame + .data + .iter() + .map(|sample| sample.to_sample::()) + .collect::>(); + + if audio_buffer.is_empty() { + warn!("empty audio frame | {:#?}", audio_buffer); + continue; + } + + if let Err(e) = stt_client.send(audio_buffer) { + error!("Couldn't send audio frame to stt {e}"); + }; + } + error!("audio thread ended"); + }); + }, + RemoteTrack::Video(video_track) => { + let video_rtc_track = video_track.rtc_track(); + let mut video_stream = NativeVideoStream::new(video_rtc_track); + let rt = async_runtime.clone(); + + std::thread::spawn(move || { + // every 10 video frames + let mut i = 0; + info!("📸 handling video"); + loop { + while let Some(Some(frame)) = video_stream.next().now_or_never() { + log::error!("🤡received video frame | {:#?}", frame); + // VIDEO FRAME BUFFER (i420_buffer) + let video_frame_buffer = frame.buffer.to_i420(); + let width = video_frame_buffer.width(); + let height = video_frame_buffer.height(); + let rgba_stride = video_frame_buffer.width() * pixel_size; + + let (stride_y, stride_u, stride_v) = + video_frame_buffer.strides(); + let (data_y, data_u, data_v) = video_frame_buffer.data(); + + let rgba_buffer = RgbaImage::new(width, height); + let rgba_raw = unsafe { + std::slice::from_raw_parts_mut( + rgba_buffer.as_raw().as_ptr() as *mut u8, + rgba_buffer.len(), + ) + }; + + livekit::webrtc::native::yuv_helper::i420_to_rgba( + data_y, + stride_y, + data_u, + stride_u, + data_v, + stride_v, + rgba_raw, + rgba_stride, + video_frame_buffer.width() as i32, + video_frame_buffer.height() as i32, + ); + + if let Err(e) = rgba_buffer.save(format!("camera/{i}.png")) { + log::error!("Couldn't save video frame {e}"); + }; + i += 1; + } + } + info!("🤡ended video thread"); + }); + }, + }; + }, + RoomEvent::DataReceived { payload, kind, topic: _, participant: _ } => { + if kind == DataPacketKind::Reliable { + if let Some(payload) = payload.as_ascii() { + let room_text: serde_json::Result = + serde_json::from_str(payload.as_str()); + match room_text { + Ok(room_text) => { + if let Err(e) = + llm_channel_tx.send(format!("[chat]{} ", room_text.message)) + { + error!("Couldn't send the text to gpt {e}") + }; + }, + Err(e) => { + warn!("Couldn't deserialize room text. {e:#?}"); + }, + } + + info!("text from room {:#?}", payload.as_str()); + } + } + }, + // ignoring the participant for now, currently assuming there is only one participant + RoomEvent::TrackMuted { participant: _, publication: _ } => { + should_stop.store(true, Ordering::Relaxed); + }, + RoomEvent::TrackUnmuted { participant: _, publication: _ } => { + should_stop.store(false, Ordering::Relaxed); + }, + // RoomEvent::ActiveSpeakersChanged { speakers } => { + // if speakers.is_empty() { + // should_stop.store(true, Ordering::Relaxed); + // } + // let is_main_participant_muted = speakers.iter().any(|speaker| speaker.name() != "kitt"); + // should_stop.store(is_main_participant_muted, Ordering::Relaxed); + // } + RoomEvent::ConnectionQualityChanged { quality: _, participant: _ } => {}, + _ => info!("received room event {:?}", event), + } + } +} diff --git a/lkgpt/src/server.rs b/lkgpt/src/server.rs index d90783b..726dc4c 100644 --- a/lkgpt/src/server.rs +++ b/lkgpt/src/server.rs @@ -1,4 +1,3 @@ -use image::{ImageBuffer, Rgba}; use livekit::{ publication::LocalTrackPublication, webrtc::{ @@ -90,6 +89,7 @@ pub struct RoomData { pub stream_frame_data: crate::StreamingFrameData, pub audio_src: NativeAudioSource, pub audio_pub: LocalTrackPublication, + pub bot_name: String, } pub async fn setup_and_connect_to_livekit( @@ -130,7 +130,14 @@ pub async fn setup_and_connect_to_livekit( let livekit_room = LivekitRoom { room, room_events }; - Ok(RoomData { livekit_room, stream_frame_data, video_pub, audio_src, audio_pub }) + Ok(RoomData { + livekit_room, + stream_frame_data, + video_pub, + audio_src, + audio_pub, + bot_name: bot_name.to_string(), + }) } mod health_check { diff --git a/lkgpt/src/stt.rs b/lkgpt/src/stt.rs index 610d705..f5e0a2f 100644 --- a/lkgpt/src/stt.rs +++ b/lkgpt/src/stt.rs @@ -1,132 +1,118 @@ -use bevy::ecs::{ - system::{Res, ResMut, Resource}, - world::{FromWorld, World}, +use async_trait::async_trait; +use bevy::ecs::system::Resource; +use bytes::{BufMut, Bytes, BytesMut}; +use deepgram::Deepgram; +use ezsockets::{ + client::ClientCloseMode, Client, ClientConfig, CloseFrame, MessageSignal, MessageStatus, + RawMessage, SocketConfig, WSError, }; -use deepgram::{ - transcription::live::{ - options::{Encoding, Model, Options}, - response::Response, - DeepgramLive, - }, - Deepgram, +use futures::StreamExt; +use livekit::webrtc::audio_stream::native::NativeAudioStream; +use log::{error, info}; +use parking_lot::Mutex; +use serde::{Deserialize, Serialize}; +use serde_json::{json, Map, Value}; +use std::{ + sync::Arc, + time::{Duration, Instant}, }; -use futures::{stream::SplitSink, SinkExt, StreamExt}; +use tokio::sync::mpsc::UnboundedSender; use crate::{AsyncRuntime, DEEPGRAM_API_KEY}; -#[derive(Resource)] -pub struct AudioInputChannel { - pub tx: crossbeam_channel::Sender>, - rx: crossbeam_channel::Receiver>, +#[derive(Clone, Resource)] +pub struct STT { + ws_client: Arc>, } -impl FromWorld for AudioInputChannel { - fn from_world(_: &mut World) -> Self { - let (tx, rx) = crossbeam_channel::unbounded::>(); - Self { tx, rx } - } +struct WSClient { + llm_channel_tx: crossbeam_channel::Sender, } -/// Live Speech To Text using Deepgram's Websocket API -#[allow(clippy::upper_case_acronyms)] -#[derive(Resource)] -pub struct STT { - tx: SplitSink>, - rx: crossbeam_channel::Receiver, -} +#[async_trait] +impl ezsockets::ClientExt for WSClient { + type Call = (); -impl STT { - pub const LATENCY_FRAMES: f32 = (Self::LATENCY_MS / 1_000.0) * Self::SAMPLE_RATE_F32; - // Uses a delay of `LATENCY_MS` milliseconds in case the default input and output streams are not precisely synchronised - pub const LATENCY_MS: f32 = 5000.0; - pub const LATENCY_SAMPLES: u32 = Self::LATENCY_FRAMES as u32 * Self::NUM_OF_CHANNELS; - const MIN_AUDIO_MS_CHUNK: u64 = 25; - pub const NUM_ITERS: usize = 2; - pub const NUM_ITERS_SAVED: usize = 2; - pub const NUM_OF_CHANNELS: u32 = 1; - pub const SAMPLE_RATE: u32 = 44100; - //1600 - pub const SAMPLE_RATE_F32: f32 = Self::SAMPLE_RATE as f32; - pub const SAMPLING_FREQ: f32 = Self::SAMPLE_RATE_F32 / 2.0; -} + async fn on_text(&mut self, text: String) -> Result<(), ezsockets::Error> { + let data: Value = serde_json::from_str(&text)?; + let transcript = data["channel"]["alternatives"][0]["transcript"].clone(); -impl FromWorld for STT { - fn from_world(world: &mut World) -> Self { - let async_rt = world.get_resource::().unwrap(); - - let ws = match async_rt.rt.block_on(async { connect_to_deepgram().await }) { - Ok(ws) => ws, - Err(e) => panic!("Failed to connect to Deepgram: {}", e), - }; - - let (ws_tx, mut socket_stream) = ws.split(); - - // let k = SplitSink>, Message> - let (tx, rx) = crossbeam_channel::unbounded(); - - async_rt.rt.spawn(async move { - while let Some(Ok(resp)) = socket_stream.next().await { - match resp { - Response::Results(result) => { - let txt = result.channel.alternatives[0].transcript.clone(); - if let Err(e) = tx.send(txt) { - log::error!("Failed to send transcribed text: {}", e); - }; - }, - Response::Metadata(metadata) => { - log::info!("STT metadata from deepgram : {:?}", metadata); - }, - } - } - }); - - log::info!("Connected to Deepgram"); - Self { tx: ws_tx, rx } + if transcript != Value::Null { + info!("🎉 from deepgram {transcript}"); + if let Err(e) = self.llm_channel_tx.send(transcript.to_string()) { + error!("Error sending to LLM: {}", e); + }; + } + + Ok(()) } -} -async fn connect_to_deepgram() -> anyhow::Result { - let deepgram_api_key = std::env::var(DEEPGRAM_API_KEY).unwrap(); + async fn on_binary(&mut self, bytes: Vec) -> Result<(), ezsockets::Error> { + info!("received bytes from deepgram: {bytes:?}"); + Ok(()) + } + + async fn on_call(&mut self, call: Self::Call) -> Result<(), ezsockets::Error> { + info!("Deepgram ON CALL: {call:?}"); + let () = call; + Ok(()) + } - let dg_client = Deepgram::new(&deepgram_api_key)?; - let options = Options::builder() - .model(Model::CustomId("nova-2-conversationalai".to_string())) - .encoding_with_channels( - Encoding::Linear16, - STT::SAMPLE_RATE as usize, - STT::NUM_OF_CHANNELS as usize, - ) - .punctuate(true) - .version("latest") - .build(); + async fn on_connect(&mut self) -> Result<(), ezsockets::Error> { + info!("Deepgram CONNECTED 🎉"); + Ok(()) + } - // ("smart_format", "true"), - // ("filler_words", "true"), + async fn on_connect_fail(&mut self, e: WSError) -> Result { + error!("Deepgram CONNECTION FAILED | {e}"); + Ok(ClientCloseMode::Reconnect) + } - let deepgram_socket = dg_client.transcription().live(&options).await?; + async fn on_close( + &mut self, + frame: Option, + ) -> Result { + error!("Deepgram CONNECTION CLOSED | {frame:?}"); + Ok(ClientCloseMode::Reconnect) + } - Ok(deepgram_socket) + async fn on_disconnect(&mut self) -> Result { + error!("Deepgram disconnected"); + Ok(ClientCloseMode::Reconnect) + } } -// SYSTEM -pub fn receive_and_process_audio( - audio_channel: Res, - llm_channel: Res, - mut stt_websocket: ResMut, - async_runtime: Res, -) { - while let Ok(audio_buffer) = audio_channel.rx.try_recv() { - log::info!("receiving audio"); - let sample_bytes = - audio_buffer.into_iter().flat_map(|sample| sample.to_le_bytes()).collect::>(); - - let _ = async_runtime.rt.block_on(async { stt_websocket.tx.send(sample_bytes).await }); +impl STT { + pub async fn new(llm_channel_tx: crossbeam_channel::Sender) -> anyhow::Result { + let deepgram_api_key = std::env::var(DEEPGRAM_API_KEY).unwrap(); + + let config = ClientConfig::new("wss://api.deepgram.com/v1/listen") + .socket_config(SocketConfig { + heartbeat: Duration::from_secs(11), + timeout: Duration::from_secs(30 * 60), // 30 minutes + heartbeat_ping_msg_fn: Arc::new(|_t: Duration| { + RawMessage::Text( + json!({ + "type": "KeepAlive", + }) + .to_string(), + ) + }), + }) + .header("Authorization", &format!("Token {}", deepgram_api_key)) + .query_parameter("model", "nova-2-conversationalai") + .query_parameter("smart_format", "true") + .query_parameter("version", "latest") + .query_parameter("filler_words", "true"); + + let (ws_client, _) = + ezsockets::connect(|_client| WSClient { llm_channel_tx }, config).await; + + Ok(Self { ws_client: Arc::new(ws_client) }) } - while let Ok(message) = stt_websocket.rx.try_recv() { - log::info!("transcribed text: {}", message); - if let Err(e) = llm_channel.tx.send(message) { - log::error!("Failed to send transcribed text: {}", e); - }; + pub fn send(&self, bytes: impl Into>) -> anyhow::Result { + let signal = self.ws_client.binary(bytes)?; + Ok(signal.status()) } } diff --git a/lkgpt/src/tts.rs b/lkgpt/src/tts.rs index c424378..2be67ec 100644 --- a/lkgpt/src/tts.rs +++ b/lkgpt/src/tts.rs @@ -1,4 +1,3 @@ -use anyhow::bail; use async_trait::async_trait; use base64::{ engine::general_purpose::{self}, @@ -13,10 +12,15 @@ use ezsockets::{ use futures::StreamExt; use livekit::webrtc::{audio_frame::AudioFrame, audio_source::native::NativeAudioSource}; use log::{error, info}; -use parking_lot::Mutex; use serde::Serialize; use serde_json::Value; -use std::{sync::Arc, time::Duration}; +use std::{ + sync::{ + atomic::{AtomicBool, Ordering}, + Arc, + }, + time::Duration, +}; use std::io::Cursor; @@ -55,14 +59,18 @@ struct RegularMessage { #[allow(clippy::upper_case_acronyms)] #[derive(Clone, Resource)] pub struct TTS { - ws_client: Option>, - pub started: bool, - eleven_labs_api_key: String, + ws_client: Client, + pub started: Arc, +} + +impl TTS { + pub const NUM_OF_CHANNELS: u32 = 1; + pub const SAMPLE_RATE: u32 = 44100; } struct WSClient { audio_src: NativeAudioSource, - tts_client_ref: Arc>, + tts_ws_started: Arc, } fn decode_base64_audio(base64_audio: &str) -> anyhow::Result> { @@ -83,22 +91,17 @@ impl ezsockets::ClientExt for WSClient { info!("incoming speech from eleven labs"); if base64_audio != Value::Null { - let data = decode_base64_audio(base64_audio.as_str().unwrap())?; - - const FRAME_DURATION: Duration = Duration::from_millis(500); // Write 0.5s of audio at a time - let ms = FRAME_DURATION.as_millis() as u32; + let data = std::borrow::Cow::from(decode_base64_audio(base64_audio.as_str().unwrap())?); let num_channels = self.audio_src.num_channels(); let sample_rate = self.audio_src.sample_rate(); - let num_samples = (sample_rate / 1000 * ms) as usize; - let samples_per_channel = num_samples as u32; + let samples_per_channel = 1_u32; - let audio_frame = - AudioFrame { data: data.into(), num_channels, sample_rate, samples_per_channel }; + let audio_frame = AudioFrame { data, num_channels, sample_rate, samples_per_channel }; self.audio_src.capture_frame(&audio_frame).await?; } else { - error!("received null message from eleven labs: {text:?}"); + error!("received null audio from eleven labs: {text:?}"); } Ok(()) @@ -116,8 +119,7 @@ impl ezsockets::ClientExt for WSClient { } async fn on_connect(&mut self) -> Result<(), ezsockets::Error> { - let mut tts = self.tts_client_ref.lock(); - tts.started = true; + self.tts_ws_started.store(true, Ordering::Relaxed); info!("ELEVEN LABS CONNECTED 🎉"); Ok(()) } @@ -126,7 +128,7 @@ impl ezsockets::ClientExt for WSClient { &mut self, _error: WSError, ) -> Result { - info!("ELEVEN LABS connection FAIL"); + error!("ELEVEN LABS connection FAIL"); Ok(ClientCloseMode::Reconnect) } @@ -134,36 +136,22 @@ impl ezsockets::ClientExt for WSClient { &mut self, _frame: Option, ) -> Result { - info!("ELEVEN LABS connection CLOSE"); - let mut tts = self.tts_client_ref.lock(); - tts.started = false; + error!("ELEVEN LABS connection CLOSE"); + self.tts_ws_started.store(false, Ordering::Relaxed); Ok(ClientCloseMode::Reconnect) } async fn on_disconnect(&mut self) -> Result { - info!("ELEVEN LABS disconnect"); + error!("ELEVEN LABS disconnected"); Ok(ClientCloseMode::Reconnect) } } impl TTS { - pub fn new() -> anyhow::Result { + pub async fn new(audio_src: NativeAudioSource) -> anyhow::Result { let eleven_labs_api_key = std::env::var(ELEVENLABS_API_KEY).unwrap(); + let started = Arc::new(AtomicBool::new(true)); - Ok(Self { ws_client: None, started: false, eleven_labs_api_key }) - } - - pub async fn setup_ws_client(&mut self, audio_src: NativeAudioSource) -> anyhow::Result<()> { - let ws_client = self.connect_ws_client(audio_src).await?; - self.started = true; - self.ws_client = Some(ws_client); - Ok(()) - } - - async fn connect_ws_client( - &self, - audio_src: NativeAudioSource, - ) -> anyhow::Result> { let voice_id = "21m00Tcm4TlvDq8ikWAM"; let model = "eleven_turbo_v2"; @@ -189,10 +177,10 @@ impl TTS { ) }), }) - .header("xi-api-key", &self.eleven_labs_api_key); + .header("xi-api-key", eleven_labs_api_key); let (ws_client, _) = ezsockets::connect( - |_client| WSClient { audio_src, tts_client_ref: Arc::new(Mutex::new(self.clone())) }, + |_client| WSClient { audio_src, tts_ws_started: started.clone() }, config, ) .await; @@ -203,11 +191,12 @@ impl TTS { voice_settings: VoiceSettings { stability: 0.8, similarity_boost: true }, generation_config: GenerationConfig { chunk_length_schedule: [50] }, })?)?; - Ok(ws_client) + + Ok(Self { ws_client, started }) } - pub fn start(&mut self) -> anyhow::Result<()> { - self.started = true; + pub fn restart(&mut self) -> anyhow::Result<()> { + self.started.store(true, Ordering::Relaxed); self.send(" ".to_string())?; Ok(()) } @@ -228,31 +217,12 @@ impl TTS { }; let msg = msg?; - if !self.started { - self.start()?; - } - - if self.ws_client.as_ref().is_none() { - bail!("ws_client is none"); + if !self.started.load(Ordering::Relaxed) { + self.restart()?; } info!("sending to eleven labs {msg}"); - Ok(self.ws_client.as_ref().unwrap().text(msg)?.status()) + Ok(self.ws_client.text(msg)?.status()) } } - -impl Drop for TTS { - fn drop(&mut self) { - info!("DROPPING TTS"); - if let Err(e) = self.send("".to_owned()) { - error!("Error shutting down TTS / Eleven Labs connection | Reason - {e}"); - }; - } -} - -pub async fn create_tts(audio_src: NativeAudioSource) -> anyhow::Result { - let mut tts = TTS::new()?; - tts.setup_ws_client(audio_src).await?; - Ok(tts) -} diff --git a/lkgpt/src/video.rs b/lkgpt/src/video.rs index e62ddee..3b70c88 100644 --- a/lkgpt/src/video.rs +++ b/lkgpt/src/video.rs @@ -16,12 +16,17 @@ pub struct VideoChannel { rx: crossbeam_channel::Receiver>, } -impl FromWorld for VideoChannel { - fn from_world(_: &mut World) -> Self { +impl Default for VideoChannel { + fn default() -> Self { let (tx, rx) = crossbeam_channel::unbounded::>(); Self { tx, rx } } } +impl VideoChannel { + pub fn new() -> Self { + Self::default() + } +} async fn video_stream_handler(mut video: NativeVideoStream) { let mut counter = 0_u8; diff --git a/meet/app/r/[name]/page.tsx b/meet/app/r/[name]/page.tsx index ecb9c7b..7a45f9e 100644 --- a/meet/app/r/[name]/page.tsx +++ b/meet/app/r/[name]/page.tsx @@ -32,12 +32,14 @@ export default function LivekitRoom({ params }: { params: { name: string } }) { } return ( -
+
{roomName && !Array.isArray(roomName) && preJoinChoices ? ( - +
+ +
) : ( -
-

Kitt2

+
+ {/*

Kitt2

*/} console.log('error while setting up prejoin', err)} defaults={{ - username: 'human', + username: 'Pinkman', videoEnabled: false, audioEnabled: false, e2ee: true, diff --git a/meet/app/r/page.tsx b/meet/app/r/page.tsx index d2e38e4..d4a2989 100644 --- a/meet/app/r/page.tsx +++ b/meet/app/r/page.tsx @@ -1,8 +1,8 @@ -import { Button, ButtonLink } from '~/components/ui/button'; +import { ButtonLink } from '~/components/ui/button'; export default function Room() { return ( -
+

Lobby