Add esp-gmf and esp-bsp Integration and Support

.github/workflows/build.yaml

@@ -9,7 +9,7 @@ jobs:
   build:
     strategy:
       matrix:
-        target: [esp32s3, linux]
+        board: [esp-box, esp32s3_korvo_2, esp32s3_others, linux]
 
     runs-on: ubuntu-latest
 
@@ -24,5 +24,5 @@ jobs:
         docker run -v $PWD:/project -w /project -u 0 \
         -e HOME=/tmp -e WIFI_SSID=A -e WIFI_PASSWORD=B -e OPENAI_API_KEY=X \
         espressif/idf:latest \
-        /bin/bash -c 'idf.py --preview set-target ${{ matrix.target }} && idf.py build'
+        /bin/bash -c 'idf.py @boards/${{ matrix.board }}.cfg build'
       shell: bash

.gitignore

@@ -2,3 +2,4 @@ build
 sdkconfig
 sdkconfig.old
 managed_components
+dependencies.lock

.gitmodules

@@ -1,12 +1,6 @@
-[submodule "components/srtp"]
-	path = components/srtp
-	url = https://git@github.com/sepfy/esp_ports
-[submodule "deps/libpeer"]
-	path = deps/libpeer
-	url = https://github.com/sean-der/libpeer
-[submodule "components/esp-libopus"]
-	path = components/esp-libopus
-	url = https://github.com/XasWorks/esp-libopus.git
 [submodule "components/esp-protocols"]
 	path = components/esp-protocols
 	url = https://github.com/espressif/esp-protocols.git
+[submodule "components/peer/libpeer"]
+	path = components/peer/libpeer
+	url = https://github.com/sean-der/libpeer

CMakeLists.txt

@@ -20,8 +20,16 @@ endif()
 add_compile_definitions(OPENAI_API_KEY="$ENV{OPENAI_API_KEY}")
 add_compile_definitions(OPENAI_REALTIMEAPI="https://api.openai.com/v1/realtime?model=gpt-4o-mini-realtime-preview-2024-12-17")
 
-set(COMPONENTS src)
-set(EXTRA_COMPONENT_DIRS "src" "components/srtp" "components/peer" "components/esp-libopus")
+# Select board configuration based on -DBUILD_BOARD
+if(NOT DEFINED BUILD_BOARD)
+    message(FATAL_ERROR "BUILD_BOARD CMake variable is not set. Use idf.py @boards/BOARD.cfg reconfigure")
+endif()
+
+set(ENV{BUILD_BOARD} ${BUILD_BOARD})
+message(STATUS "Building for board: ${BUILD_BOARD}")
+
+set(COMPONENTS src ${BUILD_BOARD})
+set(EXTRA_COMPONENT_DIRS "src")
 
 if(IDF_TARGET STREQUAL linux)
 	add_compile_definitions(LINUX_BUILD=1)

README.md

@@ -33,7 +33,10 @@ Set your Wifi SSID + Password as env variables
 * `export OPENAI_API_KEY=bing`
 
 Build
-* `idf.py build`
+* `idf.py @boards/esp-box.cfg build`
+* `idf.py @boards/esp32s3_korvo_2.cfg build`
+* `idf.py @boards/esp32s3_others.cfg build`
+* `idf.py @boards/linux.cfg build`
 
 If you built for `esp32s3` run the following to flash to the device
 * `sudo -E idf.py flash`

boards/esp-box.cfg

@@ -0,0 +1 @@
+-DSDKCONFIG_DEFAULTS="sdkconfig.defaults.esp-box;sdkconfig.defaults" -DBUILD_BOARD="esp-box" -DIDF_TARGET=esp32s3

boards/esp32s3_korvo_2.cfg

@@ -0,0 +1 @@
+-DSDKCONFIG_DEFAULTS="sdkconfig.defaults.esp32s3_korvo_2;sdkconfig.defaults" -DBUILD_BOARD="esp32s3_korvo_2" -DIDF_TARGET=esp32s3

boards/esp32s3_others.cfg

@@ -0,0 +1 @@
+-DSDKCONFIG_DEFAULTS="sdkconfig.defaults.esp32s3_others;sdkconfig.defaults" -DBUILD_BOARD="others" -DIDF_TARGET=esp32s3

boards/linux.cfg

@@ -0,0 +1 @@
+-DSDKCONFIG_DEFAULTS="sdkconfig.defaults.linux;sdkconfig.defaults" -DBUILD_BOARD="linux" -DIDF_TARGET=linux

components/peer/CMakeLists.txt

@@ -1,4 +1,4 @@
-set(PEER_PROJECT_PATH "../../deps/libpeer")
+set(PEER_PROJECT_PATH "./libpeer")
 file(GLOB CODES "${PEER_PROJECT_PATH}/src/*.c")
 
 idf_component_register(
@@ -8,9 +8,9 @@ idf_component_register(
 )
 
 # Disable KeepAlives
-file(READ ${CMAKE_CURRENT_SOURCE_DIR}/../../deps/libpeer/src/config.h INPUT_CONTENT)
+file(READ ${CMAKE_CURRENT_SOURCE_DIR}/libpeer/src/config.h INPUT_CONTENT)
 string(REPLACE "#define KEEPALIVE_CONNCHECK 10000" "#define KEEPALIVE_CONNCHECK 0" MODIFIED_CONTENT ${INPUT_CONTENT})
-file(WRITE ${CMAKE_CURRENT_SOURCE_DIR}/../../deps/libpeer/src/config.h ${MODIFIED_CONTENT})
+file(WRITE ${CMAKE_CURRENT_SOURCE_DIR}/libpeer/src/config.h ${MODIFIED_CONTENT})
 
 if(NOT IDF_TARGET STREQUAL linux)
   add_definitions("-DESP32 -DCONFIG_USE_LWIP=1 -DCONFIG_AUDIO_BUFFER_SIZE=8096 -DCONFIG_DATA_BUFFER_SIZE=102400 -D__BYTE_ORDER=__LITTLE_ENDIAN")

partitions.csv

@@ -2,5 +2,5 @@
 # Name,   Type, SubType, Offset,  Size, Flags
 nvs,      data, nvs,     0x9000,  0x6000,
 phy_init, data, phy,     0xf000,  0x1000,
-factory,  app,  factory, 0x10000, 0x180000,
+factory,  app,  factory, 0x10000, 0x2M,
 

sdkconfig.defaults.esp-box

@@ -0,0 +1,37 @@
+# ESP Event Loop on Linux
+CONFIG_ESP_EVENT_POST_FROM_ISR=n
+CONFIG_ESP_EVENT_POST_FROM_IRAM_ISR=n
+
+# Disable TLS verification
+# Production needs to include specific cert chain you care about
+CONFIG_ESP_TLS_INSECURE=y
+CONFIG_ESP_TLS_SKIP_SERVER_CERT_VERIFY=y
+
+# Enable DTLS-SRTP
+CONFIG_MBEDTLS_SSL_PROTO_DTLS=y
+
+# Defaults to partitions.csv
+CONFIG_PARTITION_TABLE_CUSTOM=y
+CONFIG_ESPTOOLPY_FLASHSIZE_4MB=y
+
+# Set highest CPU Freq
+CONFIG_ESP_DEFAULT_CPU_FREQ_MHZ_240=y
+
+CONFIG_SPIRAM=y
+CONFIG_SPIRAM_MODE_OCT=y
+CONFIG_SPIRAM_SPEED_80M=y
+
+# Disable Watchdog
+# CONFIG_ESP_INT_WDT is not set
+# CONFIG_ESP_TASK_WDT_EN is not set
+
+# Enable Compiler Optimization
+CONFIG_COMPILER_OPTIMIZATION_PERF=y
+CONFIG_COMPILER_OPTIMIZATION_ASSERTIONS_DISABLE=y
+
+CONFIG_LWIP_TCPIP_RECVMBOX_SIZE=64
+CONFIG_LWIP_MAX_UDP_PCBS=1024
+CONFIG_LWIP_UDP_RECVMBOX_SIZE=64
+
+# CONFIG_ESP_AUDIO_SIMPLE_PLAYER_RESAMPLE_EN is not set
+# CONFIG_CODEC_I2C_BACKWARD_COMPATIBLE is not set

sdkconfig.defaults.esp32s3_korvo_2

@@ -0,0 +1,37 @@
+# ESP Event Loop on Linux
+CONFIG_ESP_EVENT_POST_FROM_ISR=n
+CONFIG_ESP_EVENT_POST_FROM_IRAM_ISR=n
+
+# Disable TLS verification
+# Production needs to include specific cert chain you care about
+CONFIG_ESP_TLS_INSECURE=y
+CONFIG_ESP_TLS_SKIP_SERVER_CERT_VERIFY=y
+
+# Enable DTLS-SRTP
+CONFIG_MBEDTLS_SSL_PROTO_DTLS=y
+
+# Defaults to partitions.csv
+CONFIG_PARTITION_TABLE_CUSTOM=y
+CONFIG_ESPTOOLPY_FLASHSIZE_4MB=y
+
+# Set highest CPU Freq
+CONFIG_ESP_DEFAULT_CPU_FREQ_MHZ_240=y
+
+CONFIG_SPIRAM=y
+CONFIG_SPIRAM_MODE_OCT=y
+CONFIG_SPIRAM_SPEED_80M=y
+
+# Disable Watchdog
+# CONFIG_ESP_INT_WDT is not set
+# CONFIG_ESP_TASK_WDT_EN is not set
+
+# Enable Compiler Optimization
+CONFIG_COMPILER_OPTIMIZATION_PERF=y
+CONFIG_COMPILER_OPTIMIZATION_ASSERTIONS_DISABLE=y
+
+CONFIG_LWIP_TCPIP_RECVMBOX_SIZE=64
+CONFIG_LWIP_MAX_UDP_PCBS=1024
+CONFIG_LWIP_UDP_RECVMBOX_SIZE=64
+
+# CONFIG_ESP_AUDIO_SIMPLE_PLAYER_RESAMPLE_EN is not set
+# CONFIG_CODEC_I2C_BACKWARD_COMPATIBLE is not set

sdkconfig.defaults.esp32s3_others

@@ -0,0 +1,32 @@
+# ESP Event Loop on Linux
+CONFIG_ESP_EVENT_POST_FROM_ISR=n
+CONFIG_ESP_EVENT_POST_FROM_IRAM_ISR=n
+
+# Disable TLS verification
+# Production needs to include specific cert chain you care about
+CONFIG_ESP_TLS_INSECURE=y
+CONFIG_ESP_TLS_SKIP_SERVER_CERT_VERIFY=y
+
+# Enable DTLS-SRTP
+CONFIG_MBEDTLS_SSL_PROTO_DTLS=y
+
+# libpeer requires large stack allocations
+CONFIG_ESP_MAIN_TASK_STACK_SIZE=16384
+
+# Defaults to partitions.csv
+CONFIG_PARTITION_TABLE_CUSTOM=y
+CONFIG_ESPTOOLPY_FLASHSIZE_4MB=y
+
+# Set highest CPU Freq
+CONFIG_ESP_DEFAULT_CPU_FREQ_MHZ_240=y
+
+CONFIG_SPIRAM=y
+CONFIG_SPIRAM_MODE_OCT=y
+
+# Disable Watchdog
+# CONFIG_ESP_INT_WDT is not set
+# CONFIG_ESP_TASK_WDT_EN is not set
+
+# Enable Compiler Optimization
+CONFIG_COMPILER_OPTIMIZATION_PERF=y
+CONFIG_COMPILER_OPTIMIZATION_ASSERTIONS_DISABLE=y

sdkconfig.defaults.linux

@@ -0,0 +1,31 @@
+# ESP Event Loop on Linux
+CONFIG_ESP_EVENT_POST_FROM_ISR=n
+CONFIG_ESP_EVENT_POST_FROM_IRAM_ISR=n
+
+# Disable TLS verification
+# Production needs to include specific cert chain you care about
+CONFIG_ESP_TLS_INSECURE=y
+CONFIG_ESP_TLS_SKIP_SERVER_CERT_VERIFY=y
+
+# Enable DTLS-SRTP
+CONFIG_MBEDTLS_SSL_PROTO_DTLS=y
+
+# libpeer requires large stack allocations
+CONFIG_ESP_MAIN_TASK_STACK_SIZE=16384
+
+# Defaults to partitions.csv
+CONFIG_PARTITION_TABLE_CUSTOM=y
+
+# Set highest CPU Freq
+CONFIG_ESP_DEFAULT_CPU_FREQ_MHZ_240=y
+
+CONFIG_SPIRAM=y
+CONFIG_SPIRAM_MODE_OCT=y
+
+# Disable Watchdog
+# CONFIG_ESP_INT_WDT is not set
+# CONFIG_ESP_TASK_WDT_EN is not set
+
+# Enable Compiler Optimization
+CONFIG_COMPILER_OPTIMIZATION_PERF=y
+CONFIG_COMPILER_OPTIMIZATION_ASSERTIONS_DISABLE=y

src/CMakeLists.txt

@@ -3,20 +3,34 @@ set(COMMON_SRC "webrtc.cpp" "main.cpp" "http.cpp")
 if(IDF_TARGET STREQUAL linux)
 	idf_component_register(
 		SRCS ${COMMON_SRC}
-		REQUIRES peer esp-libopus esp_http_client)
+		REQUIRES peer esp_http_client)
 else()
 	idf_component_register(
-		SRCS ${COMMON_SRC} "wifi.cpp" "media.cpp"
-		REQUIRES driver esp_wifi nvs_flash peer esp_psram esp-libopus esp_http_client)
+		SRCS ${COMMON_SRC} "wifi.cpp" "media.cpp" "media_gmf.cpp"
+		REQUIRES driver esp_wifi nvs_flash peer esp_psram esp_timer esp_http_client)
 endif()
 
 idf_component_get_property(lib peer COMPONENT_LIB)
 target_compile_options(${lib} PRIVATE -Wno-error=restrict)
 target_compile_options(${lib} PRIVATE -Wno-error=stringop-truncation)
 
-idf_component_get_property(lib srtp COMPONENT_LIB)
+idf_component_get_property(lib sepfy__srtp COMPONENT_LIB)
 target_compile_options(${lib} PRIVATE -Wno-error=incompatible-pointer-types)
 
-idf_component_get_property(lib esp-libopus COMPONENT_LIB)
-target_compile_options(${lib} PRIVATE -Wno-error=maybe-uninitialized)
-target_compile_options(${lib} PRIVATE -Wno-error=stringop-overread)
+set(TARGET_COMPONENT "")
+
+idf_build_get_property(build_components BUILD_COMPONENTS)
+foreach(COMPONENT ${build_components})
+	if(COMPONENT MATCHES "gmf_core")
+		set(TARGET_COMPONENT ${COMPONENT})
+		break()
+	endif()
+endforeach()
+
+if(TARGET_COMPONENT STREQUAL "")
+	target_compile_definitions(${COMPONENT_TARGET} PRIVATE "-D CONFIG_OPENAI_WITH_GMF=0")
+else()
+	idf_component_get_property(lib ${TARGET_COMPONENT} COMPONENT_LIB)
+	target_compile_options(${lib} PRIVATE -Wno-error=incompatible-pointer-types)
+	target_compile_definitions(${COMPONENT_TARGET} PRIVATE "-D CONFIG_OPENAI_WITH_GMF=1")
+endif()

src/idf_component.yml

@@ -1,3 +1,56 @@
 dependencies:
   idf:
     version: ">=4.1.0"
+
+  sepfy/srtp:
+    version: "^2.3.0"
+
+  78/esp-opus:
+    version: "^1.0.4"
+    rules:
+    - if: ${BUILD_BOARD} not in ["esp32s3_korvo_2", "esp-box"]
+
+  espressif/esp32_s3_korvo_2:
+    version: "^3.0.0"
+    rules:
+    - if: "${BUILD_BOARD} == esp32s3_korvo_2"
+
+  espressif/esp-box:
+    version: "^3.1.0"
+    rules:
+    - if: "${BUILD_BOARD} == esp-box"
+
+  espressif/gmf_core:
+    path: ./gmf_core
+    git: https://github.com/espressif/esp-gmf.git
+    version: "main"
+    rules:
+    - if: ${BUILD_BOARD} in ["esp32s3_korvo_2", "esp-box"]
+
+  espressif/gmf_io:
+    path: ./gmf_elements/gmf_io
+    git: https://github.com/espressif/esp-gmf.git
+    version: "main"
+    rules:
+    - if: ${BUILD_BOARD} in ["esp32s3_korvo_2", "esp-box"]
+
+  espressif/gmf_misc:
+    path: ./gmf_elements/gmf_misc
+    git: https://github.com/espressif/esp-gmf.git
+    version: "main"
+    rules:
+    - if: ${BUILD_BOARD} in ["esp32s3_korvo_2", "esp-box"]
+
+  espressif/gmf_audio:
+    path: ./gmf_elements/gmf_audio
+    git: https://github.com/espressif/esp-gmf.git
+    version: "main"
+    rules:
+    - if: ${BUILD_BOARD} in ["esp32s3_korvo_2", "esp-box"]
+
+  espressif/esp_audio_simple_player:
+    path: ./examples/esp_audio_simple_player
+    git: https://github.com/espressif/esp-gmf.git
+    version: "main"
+    rules:
+    - if: ${BUILD_BOARD} in ["esp32s3_korvo_2", "esp-box"]

src/main.h

@@ -3,6 +3,10 @@
 #define LOG_TAG "realtimeapi-sdk"
 #define MAX_HTTP_OUTPUT_BUFFER 2048
 
+#ifdef __cplusplus
+extern "C" {
+#endif
+
 void oai_wifi(void);
 void oai_init_audio_capture(void);
 void oai_init_audio_decoder(void);
@@ -11,3 +15,7 @@ void oai_send_audio(PeerConnection *peer_connection);
 void oai_audio_decode(uint8_t *data, size_t size);
 void oai_webrtc();
 void oai_http_request(char *offer, char *answer);
+
+#ifdef __cplusplus
+}
+#endif

src/media.cpp

@@ -1,3 +1,4 @@
+#if (CONFIG_OPENAI_WITH_GMF == 0)
 #include <driver/i2s.h>
 #include <opus.h>
 
@@ -142,3 +143,4 @@ void oai_send_audio(PeerConnection *peer_connection) {
   peer_connection_send_audio(peer_connection, encoder_output_buffer,
                              encoded_size);
 }
+#endif