Merge branch 'doc/vad_programming_guide_v5.4' into 'release/v5.4'

doc: lp_i2s and vad programming guide (v5.4)

See merge request espressif/esp-idf!35108

components/esp_driver_i2s/i2s_private.h

@@ -7,6 +7,7 @@
 #pragma once
 
 #include <sys/lock.h>
+#include <stdatomic.h>
 #include "freertos/FreeRTOS.h"
 #include "freertos/semphr.h"
 #include "freertos/queue.h"
@@ -199,7 +200,7 @@ struct lp_i2s_channel_obj_t {
     i2s_comm_mode_t            mode;           /*!< lp i2s channel communication mode */
     i2s_role_t                 role;           /*!< lp i2s role */
     i2s_dir_t                  dir;            /*!< lp i2s channel direction */
-    i2s_state_t                state;          /*!< lp i2s driver state. Ensuring the driver working in a correct sequence */
+    _Atomic i2s_state_t        state;          /*!< lp i2s driver state. Ensuring the driver working in a correct sequence */
     SemaphoreHandle_t          semphr;         /*!< lp i2s event semphr*/
     lp_i2s_trans_t             trans;          /*!< transaction */
     size_t                     threshold;      /*!< lp i2s threshold*/

components/esp_driver_i2s/include/driver/lp_i2s.h

@@ -55,7 +55,7 @@ esp_err_t lp_i2s_new_channel(const lp_i2s_chan_config_t *chan_cfg, lp_i2s_chan_h
 /**
  * @brief Register LP I2S event callbacks
  *
- * @param[in] handle     LP I2S channel handle
+ * @param[in] chan       LP I2S channel handle
  * @param[in] cbs        Callbacks
  * @param[in] user_data  User data
  *
@@ -64,12 +64,12 @@ esp_err_t lp_i2s_new_channel(const lp_i2s_chan_config_t *chan_cfg, lp_i2s_chan_h
  *        - ESP_ERR_INVALID_ARG:   Invalid argument
  *        - ESP_ERR_INVALID_STATE: Invalid state
  */
-esp_err_t lp_i2s_register_event_callbacks(lp_i2s_chan_handle_t handle, const lp_i2s_evt_cbs_t *cbs, void *user_data);
+esp_err_t lp_i2s_register_event_callbacks(lp_i2s_chan_handle_t chan, const lp_i2s_evt_cbs_t *cbs, void *user_data);
 
 /**
  * @brief Enable LP I2S driver
  *
- * @param[in] handle     LP I2S channel handle
+ * @param[in] chan     LP I2S channel handle
  *
  * @return
  *        - ESP_OK:                On success
@@ -81,7 +81,7 @@ esp_err_t lp_i2s_channel_enable(lp_i2s_chan_handle_t chan);
 /**
  * @brief Read LP I2S received data
  *
- * @param[in] handle      LP I2S channel handle
+ * @param[in] chan        LP I2S channel handle
  * @param[in] trans       LP I2S transaction
  * @param[in] timeout_ms  Timeout in ms, set to `LP_I2S_MAX_DELAY` to wait until read is done
  *
@@ -95,7 +95,7 @@ esp_err_t lp_i2s_channel_read(lp_i2s_chan_handle_t chan, lp_i2s_trans_t *trans,
 /**
  * @brief Read LP I2S received data until certain bytes
  *
- * @param[in] handle      LP I2S channel handle
+ * @param[in] chan        LP I2S channel handle
  * @param[in] trans       LP I2S transaction
  *
  * @return
@@ -108,7 +108,7 @@ esp_err_t lp_i2s_channel_read_until_bytes(lp_i2s_chan_handle_t chan, lp_i2s_tran
 /**
  * @brief Disable LP I2S driver
  *
- * @param[in] handle     LP I2S channel handle
+ * @param[in] chan       LP I2S channel handle
  *
  * @return
  *        - ESP_OK:                On success
@@ -120,13 +120,13 @@ esp_err_t lp_i2s_channel_disable(lp_i2s_chan_handle_t chan);
 /**
  * @brief Delete the LP I2S channel
  *
- * @param[in]   handle      LP I2S channel handler
+ * @param[in]   chan        LP I2S channel handler
  *
  * @return
  *      - ESP_OK                Delete successfully
  *      - ESP_ERR_INVALID_ARG   NULL pointer
  */
-esp_err_t lp_i2s_del_channel(lp_i2s_chan_handle_t handle);
+esp_err_t lp_i2s_del_channel(lp_i2s_chan_handle_t chan);
 
 #ifdef __cplusplus
 }

components/esp_driver_i2s/include/driver/lp_i2s_vad.h

@@ -78,8 +78,9 @@ typedef struct vad_unit_ctx_t *vad_unit_handle_t;
 typedef struct {
     int init_frame_num;               /**< Number of init frames that are used for VAD to denoise, this helps the VAD to decrease the accidental trigger ratio.
                                            Note too big values may lead to voice activity miss */
-    int min_energy_thresh;            ///< Min energy threshold.
-    bool skip_band_energy_thresh;     ///< Skip band energy threshold or not
+    int min_energy_thresh;            ///< Minimum energy threshold, voice activities with energy higher than this value will be detected.
+    bool skip_band_energy_thresh;     /**< Skip band energy threshold or not, the passband energy check determines whether the proportion of passband energy within the total frequency domain meets the required threshold.
+                                           Note in different environments, enabling the passband energy check may reduce false trigger rates but could also increase the rate of missed detections. */
 
     int speak_activity_thresh;        /**< When in speak-activity-listening-state, if number of the detected speak activity is higher than this value, VAD runs into speak-activity-detected-state */
 
@@ -93,6 +94,9 @@ typedef struct {
     int max_speak_activity_thresh;    /**< When in speak-activity-detected-state, if the number of the detected speak activity is higher than this value, VAD runs into speak-activity-listening-state */
 } lp_vad_config_t;
 
+/**
+ * @brief LP VAD Init Configurations
+ */
 typedef struct {
     lp_i2s_chan_handle_t    lp_i2s_chan;  ///< LP I2S channel handle
     lp_vad_config_t         vad_config;   ///< LP VAD config
@@ -115,7 +119,6 @@ esp_err_t lp_i2s_vad_new_unit(lp_vad_t vad_id, const lp_vad_init_config_t *init_
  * @brief Enable LP VAD
  *
  * @param[in] unit          VAD handle
- * @param[in] init_config   Initial configurations
  *
  * @return
  *        - ESP_OK:                 On success
@@ -128,7 +131,6 @@ esp_err_t lp_i2s_vad_enable(vad_unit_handle_t unit);
  * @brief Disable LP VAD
  *
  * @param[in] unit          VAD handle
- * @param[in] init_config   Initial configurations
  *
  * @return
  *        - ESP_OK:                 On success

components/esp_driver_i2s/lp_i2s.c

@@ -18,6 +18,7 @@
 #include "freertos/FreeRTOS.h"
 #include "freertos/semphr.h"
 #include "esp_clk_tree.h"
+#include "esp_memory_utils.h"
 #include "hal/hal_utils.h"
 #include "hal/lp_i2s_hal.h"
 #include "hal/lp_i2s_ll.h"
@@ -134,8 +135,8 @@ esp_err_t lp_i2s_new_channel(const lp_i2s_chan_config_t *chan_cfg, lp_i2s_chan_h
 esp_err_t lp_i2s_channel_enable(lp_i2s_chan_handle_t chan)
 {
     ESP_RETURN_ON_FALSE(chan, ESP_ERR_INVALID_ARG, TAG, "invalid argument: null pointer");
-    ESP_RETURN_ON_FALSE(chan->state < I2S_CHAN_STATE_RUNNING, ESP_ERR_INVALID_STATE, TAG, "the channel is in enabled state already");
-
+    i2s_state_t expected_state = I2S_CHAN_STATE_READY;
+    ESP_RETURN_ON_FALSE(atomic_compare_exchange_strong(&(chan->state), &expected_state, I2S_CHAN_STATE_RUNNING), ESP_ERR_INVALID_STATE, TAG, "the channel isn't enabled");
     lp_i2s_evt_data_t evt_data = {};
     if (chan->cbs.on_request_new_trans) {
         chan->cbs.on_request_new_trans(chan, &evt_data, chan->user_data);
@@ -145,7 +146,6 @@ esp_err_t lp_i2s_channel_enable(lp_i2s_chan_handle_t chan)
         chan->trans = evt_data.trans;
     }
 
-    chan->state = I2S_CHAN_STATE_RUNNING;
     portENTER_CRITICAL(&g_i2s.spinlock);
     lp_i2s_ll_rx_enable_interrupt(chan->ctlr->hal.dev, LP_I2S_LL_EVENT_RX_MEM_THRESHOLD_INT, true);
     lp_i2s_ll_rx_start(chan->ctlr->hal.dev);
@@ -157,7 +157,7 @@ esp_err_t lp_i2s_channel_enable(lp_i2s_chan_handle_t chan)
 esp_err_t lp_i2s_channel_read(lp_i2s_chan_handle_t chan, lp_i2s_trans_t *trans, uint32_t timeout_ms)
 {
     ESP_RETURN_ON_FALSE(chan, ESP_ERR_INVALID_ARG, TAG, "invalid argument: null pointer");
-    ESP_RETURN_ON_FALSE(chan->state == I2S_CHAN_STATE_RUNNING, ESP_ERR_INVALID_STATE, TAG, "the channel isn't enabled");
+    ESP_RETURN_ON_FALSE(atomic_load(&(chan->state)) == I2S_CHAN_STATE_RUNNING, ESP_ERR_INVALID_STATE, TAG, "the channel can't be deleted unless it is disabled");
     ESP_RETURN_ON_FALSE(!chan->cbs.on_request_new_trans, ESP_ERR_INVALID_STATE, TAG, "on_request_new_trans registered, no use of this read API");
 
     TickType_t ticks_to_wait = timeout_ms / portTICK_PERIOD_MS;
@@ -206,8 +206,8 @@ esp_err_t lp_i2s_channel_read_until_bytes(lp_i2s_chan_handle_t chan, lp_i2s_tran
 esp_err_t lp_i2s_channel_disable(lp_i2s_chan_handle_t chan)
 {
     ESP_RETURN_ON_FALSE(chan, ESP_ERR_INVALID_ARG, TAG, "invalid argument: null pointer");
-    ESP_RETURN_ON_FALSE(chan->state  > I2S_CHAN_STATE_READY, ESP_ERR_INVALID_STATE, TAG, "the channel is disabled already");
-    chan->state = I2S_CHAN_STATE_READY;
+    i2s_state_t expected_state = I2S_CHAN_STATE_RUNNING;
+    ESP_RETURN_ON_FALSE(atomic_compare_exchange_strong(&(chan->state), &expected_state, I2S_CHAN_STATE_READY), ESP_ERR_INVALID_STATE, TAG, "the channel isn't enabled");
 
     portENTER_CRITICAL(&g_i2s.spinlock);
     lp_i2s_ll_rx_enable_interrupt(chan->ctlr->hal.dev, LP_I2S_LL_EVENT_RX_MEM_THRESHOLD_INT, false);
@@ -220,7 +220,7 @@ esp_err_t lp_i2s_channel_disable(lp_i2s_chan_handle_t chan)
 esp_err_t lp_i2s_del_channel(lp_i2s_chan_handle_t chan)
 {
     ESP_RETURN_ON_FALSE(chan, ESP_ERR_INVALID_ARG, TAG, "invalid argument: null pointer");
-    ESP_RETURN_ON_FALSE(chan->state < I2S_CHAN_STATE_RUNNING, ESP_ERR_INVALID_STATE, TAG, "the channel can't be deleted unless it is disabled");
+    ESP_RETURN_ON_FALSE(atomic_load(&(chan->state)) == I2S_CHAN_STATE_READY, ESP_ERR_INVALID_STATE, TAG, "the channel can't be deleted unless it is disabled");
 
     int id = chan->ctlr->id;
     portENTER_CRITICAL(&g_i2s.spinlock);
@@ -245,14 +245,21 @@ esp_err_t lp_i2s_del_channel(lp_i2s_chan_handle_t chan)
 _Static_assert(sizeof(lp_i2s_evt_cbs_t) == sizeof(lp_i2s_evt_cbs_internal_t), "Invalid size of lp_i2s_evt_cbs_t structure");
 #endif
 
-esp_err_t lp_i2s_register_event_callbacks(lp_i2s_chan_handle_t handle, const lp_i2s_evt_cbs_t *cbs, void *user_data)
+esp_err_t lp_i2s_register_event_callbacks(lp_i2s_chan_handle_t chan, const lp_i2s_evt_cbs_t *cbs, void *user_data)
 {
-    ESP_RETURN_ON_FALSE(handle && cbs, ESP_ERR_INVALID_ARG, TAG, "invalid argument");
-    ESP_RETURN_ON_FALSE(handle->state < I2S_CHAN_STATE_RUNNING, ESP_ERR_INVALID_STATE, TAG, "the channel is in enabled state already");
+    ESP_RETURN_ON_FALSE(chan && cbs, ESP_ERR_INVALID_ARG, TAG, "invalid argument");
+    ESP_RETURN_ON_FALSE(atomic_load(&(chan->state)) == I2S_CHAN_STATE_READY, ESP_ERR_INVALID_STATE, TAG, "the channel is in enabled state already");
+
+    if (cbs->on_thresh_met) {
+        ESP_RETURN_ON_FALSE(esp_ptr_in_iram(cbs->on_thresh_met), ESP_ERR_INVALID_ARG, TAG, "on_thresh_met callback not in IRAM");
+    }
+    if (cbs->on_request_new_trans) {
+        ESP_RETURN_ON_FALSE(esp_ptr_in_iram(cbs->on_request_new_trans), ESP_ERR_INVALID_ARG, TAG, "on_request_new_trans callback not in IRAM");
+    }
 
-    handle->cbs.on_thresh_met = cbs->on_thresh_met;
-    handle->cbs.on_request_new_trans = cbs->on_request_new_trans;
-    handle->user_data = user_data;
+    chan->cbs.on_thresh_met = cbs->on_thresh_met;
+    chan->cbs.on_request_new_trans = cbs->on_request_new_trans;
+    chan->user_data = user_data;
 
     return ESP_OK;
 }
@@ -279,7 +286,7 @@ static esp_err_t s_i2s_register_channel(lp_i2s_controller_t *ctlr, i2s_dir_t dir
     new_chan->mode = I2S_COMM_MODE_NONE;
     new_chan->role = I2S_ROLE_MASTER;
     new_chan->dir = dir;
-    new_chan->state = I2S_CHAN_STATE_REGISTER;
+    atomic_init(&(new_chan->state), I2S_CHAN_STATE_READY);
     new_chan->ctlr = ctlr;
 
     if (dir == I2S_DIR_RX) {

components/esp_driver_i2s/test_apps/lp_i2s/main/test_lp_i2s.c

@@ -222,15 +222,15 @@ TEST_CASE("test LP I2S read for STD", "[lp_i2s]")
     }
 }
 
-static bool s_lp_i2s_on_thresh_met(lp_i2s_chan_handle_t handle, lp_i2s_evt_data_t *edata, void *user_data)
+static bool IRAM_ATTR s_lp_i2s_on_thresh_met(lp_i2s_chan_handle_t handle, lp_i2s_evt_data_t *edata, void *user_data)
 {
     ESP_DRAM_LOGD(TAG, "edata->trans.received_size: %d", edata->trans.received_size);
     s_data_check(edata->trans.buffer, edata->trans.received_size);
 
     return false;
 }
 
-static bool s_lp_i2s_on_request_new_trans(lp_i2s_chan_handle_t handle, lp_i2s_evt_data_t *edata, void *user_data)
+static bool IRAM_ATTR s_lp_i2s_on_request_new_trans(lp_i2s_chan_handle_t handle, lp_i2s_evt_data_t *edata, void *user_data)
 {
     lp_i2s_trans_t trans = *(lp_i2s_trans_t *)user_data;
     edata->trans.buffer = trans.buffer;

components/esp_hw_support/sleep_modes.c

@@ -1669,8 +1669,27 @@ esp_err_t esp_sleep_enable_timer_wakeup(uint64_t time_in_us)
 #if SOC_LP_VAD_SUPPORTED
 esp_err_t esp_sleep_enable_vad_wakeup(void)
 {
+    esp_err_t ret = ESP_FAIL;
+
+    ret = esp_sleep_pd_config(ESP_PD_DOMAIN_RTC_PERIPH, ESP_PD_OPTION_ON);
+    if (ret != ESP_OK) {
+        ESP_LOGE(TAG, "fail to keep rtc periph power on");
+        return ret;
+    }
+
+    ret = esp_sleep_pd_config(ESP_PD_DOMAIN_XTAL, ESP_PD_OPTION_ON);
+    if (ret != ESP_OK) {
+        ESP_LOGE(TAG, "fail to keep xtal power on");
+        return ret;
+    }
+    ret = esp_sleep_sub_mode_config(ESP_SLEEP_LP_USE_XTAL_MODE, true);
+    if (ret != ESP_OK) {
+        ESP_LOGE(TAG, "fail to set to ESP_SLEEP_LP_USE_XTAL_MODE mode");
+        return ret;
+    }
     s_config.wakeup_triggers |= RTC_LP_VAD_TRIG_EN;
-    return esp_sleep_sub_mode_config(ESP_SLEEP_LP_USE_XTAL_MODE, true);
+
+    return ESP_OK;
 }
 #endif
 

components/esp_hw_support/test_apps/vad_wakeup/README.md

@@ -1,3 +1,4 @@
 | Supported Targets | ESP32-P4 |
 | ----------------- | -------- |
 
+Slave side (LP I2S) should use default (UART0) console, as the USJ isn't working well under sleep modes.

components/esp_hw_support/test_apps/vad_wakeup/main/test_vad_wakeup.c

@@ -98,8 +98,6 @@ static void s_hp_i2s_config(void)
 static void s_lp_vad_config(void)
 {
     ESP_ERROR_CHECK(esp_sleep_enable_vad_wakeup());
-    ESP_ERROR_CHECK(esp_sleep_pd_config(ESP_PD_DOMAIN_RTC_PERIPH, ESP_PD_OPTION_ON));
-    ESP_ERROR_CHECK(esp_sleep_pd_config(ESP_PD_DOMAIN_XTAL, ESP_PD_OPTION_ON));
 
     lp_i2s_chan_handle_t rx_handle = NULL;
     lp_i2s_chan_config_t config = {

components/ulp/lp_core/shared/include/ulp_lp_core_lp_vad_shared.h

@@ -82,7 +82,6 @@ esp_err_t lp_core_lp_vad_init(lp_vad_t vad_id, const lp_core_lp_vad_cfg_t *init_
  * @brief Enable LP VAD
  *
  * @param[in] vad_id        VAD ID
- * @param[in] init_config   Initial configurations
  *
  * @return
  *        - ESP_OK:                 On success
@@ -95,7 +94,6 @@ esp_err_t lp_core_lp_vad_enable(lp_vad_t vad_id);
  * @brief Disable LP VAD
  *
  * @param[in] vad_id        VAD ID
- * @param[in] init_config   Initial configurations
  *
  * @return
  *        - ESP_OK:                 On success

docs/conf_common.py

@@ -174,6 +174,10 @@
 
 I2S_DOCS = ['api-reference/peripherals/i2s.rst']
 
+VAD_DOCS = ['api-reference/peripherals/vad.rst']
+
+LP_I2S_DOCS = ['api-reference/peripherals/lp_i2s.rst']
+
 ISP_DOCS = ['api-reference/peripherals/isp.rst']
 
 DSLP_STUB_DOCS = ['api-guides/deep-sleep-stub.rst']
@@ -286,6 +290,8 @@
                             'SOC_I2C_SUPPORTED':I2C_DOCS,
                             'SOC_GPSPI_SUPPORTED':SPI_DOCS,
                             'SOC_I2S_SUPPORTED':I2S_DOCS,
+                            'SOC_LP_I2S_SUPPORTED':LP_I2S_DOCS,
+                            'SOC_LP_VAD_SUPPORTED':VAD_DOCS,
                             'SOC_ISP_SUPPORTED':ISP_DOCS,
                             'ESP_ROM_SUPPORT_DEEP_SLEEP_WAKEUP_STUB': DSLP_STUB_DOCS,
                             'SOC_ADC_SUPPORTED':ADC_DOCS,

docs/doxygen/Doxyfile_esp32p4

@@ -11,6 +11,7 @@ INPUT += \
     $(PROJECT_PATH)/components/ulp/lp_core/lp_core/include/ulp_lp_core_utils.h \
     $(PROJECT_PATH)/components/ulp/lp_core/lp_core/include/ulp_lp_core_interrupts.h \
     $(PROJECT_PATH)/components/ulp/lp_core/lp_core/include/ulp_lp_core_spi.h \
+    $(PROJECT_PATH)/components/ulp/lp_core/shared/include/ulp_lp_core_lp_vad_shared.h \
     $(PROJECT_PATH)/components/ulp/ulp_common/include/ulp_common.h \
     $(PROJECT_PATH)/components/usb/include/usb/usb_helpers.h \
     $(PROJECT_PATH)/components/usb/include/usb/usb_host.h \
@@ -41,6 +42,10 @@ INPUT += \
     $(PROJECT_PATH)/components/esp_driver_isp/include/driver/isp_gamma.h \
     $(PROJECT_PATH)/components/esp_driver_isp/include/driver/isp_hist.h \
     $(PROJECT_PATH)/components/esp_driver_isp/include/driver/isp_color.h \
+    $(PROJECT_PATH)/components/esp_driver_i2s/include/driver/lp_i2s.h \
+    $(PROJECT_PATH)/components/esp_driver_i2s/include/driver/lp_i2s_std.h \
+    $(PROJECT_PATH)/components/esp_driver_i2s/include/driver/lp_i2s_pdm.h \
+    $(PROJECT_PATH)/components/esp_driver_i2s/include/driver/lp_i2s_vad.h \
     $(PROJECT_PATH)/components/esp_driver_jpeg/include/driver/jpeg_decode.h \
     $(PROJECT_PATH)/components/esp_driver_jpeg/include/driver/jpeg_encode.h \
     $(PROJECT_PATH)/components/esp_driver_ppa/include/driver/ppa.h \

docs/en/api-reference/peripherals/i2s.rst

@@ -10,6 +10,12 @@ Introduction
 
 I2S (Inter-IC Sound) is a synchronous serial communication protocol usually used for transmitting audio data between two digital audio devices.
 
+.. only:: SOC_LP_I2S_SUPPORTED
+
+    .. note::
+
+        For LP I2S documentation, see :doc:`Low Power Inter-IC Sound <./lp_i2s>`.
+
 {IDF_TARGET_NAME} contains {IDF_TARGET_I2S_NUM} I2S peripheral(s). These peripherals can be configured to input and output sample data via the I2S driver.
 
 An I2S bus that communicates in standard or TDM mode consists of the following lines:

docs/en/api-reference/peripherals/index.rst

@@ -41,6 +41,8 @@ Peripherals API
     :SOC_GPSPI_SUPPORTED: spi_master
     :SOC_GPSPI_SUPPORTED: spi_slave
     :SOC_SPI_SUPPORT_SLAVE_HD_VER2: spi_slave_hd
+    :SOC_LP_I2S_SUPPORTED: lp_i2s
+    :SOC_LP_VAD_SUPPORTED: vad
     :SOC_TEMP_SENSOR_SUPPORTED: temp_sensor
     :SOC_TOUCH_SENSOR_SUPPORTED and not esp32p4: touch_pad
     :esp32p4: cap_touch_sens