Merge branch 'contrib/github_pr_11926' into 'master'

Support NVS image generation from CMake (GitHub PR)

Closes IDFGH-10702 and IDFGH-10542

See merge request espressif/esp-idf!25102

components/nvs_flash/project_include.cmake

@@ -0,0 +1,64 @@
+# nvs_create_partition_image
+#
+# Create a NVS image of the specified CSV on the host during build and
+# optionally have the created image flashed using `idf.py flash`
+function(nvs_create_partition_image partition csv)
+    set(options FLASH_IN_PROJECT)
+    set(one VERSION)
+    set(multi DEPENDS)
+    cmake_parse_arguments(arg "${options}" "${one}" "${multi}" "${ARGN}")
+
+    # Default to version 2
+    if(NOT DEFINED arg_VERSION)
+        set(arg_VERSION 2)
+    endif()
+
+    idf_build_get_property(idf_path IDF_PATH)
+    set(nvs_partition_gen_py
+        ${PYTHON}
+        ${idf_path}/components/nvs_flash/nvs_partition_generator/nvs_partition_gen.py
+       )
+
+    get_filename_component(csv_full_path ${csv} ABSOLUTE)
+
+    partition_table_get_partition_info(size "--partition-name ${partition}" "size")
+    partition_table_get_partition_info(offset "--partition-name ${partition}" "offset")
+
+    if("${size}" AND "${offset}")
+        set(image_file ${CMAKE_BINARY_DIR}/${partition}.bin)
+
+        add_custom_command(
+            OUTPUT ${image_file}
+            COMMAND ${nvs_partition_gen_py} generate --version ${arg_VERSION} ${csv_full_path} ${image_file} ${size}
+            MAIN_DEPENDENCY ${csv_full_path}
+            DEPENDS ${arg_DEPENDS}
+            COMMENT "Generating NVS partition image for ${partition} from ${csv}"
+           )
+
+        add_custom_target(nvs_${partition}_bin ALL DEPENDS ${image_file})
+
+        set_property(
+            DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}"
+            APPEND
+            PROPERTY ADDITIONAL_CLEAN_FILES ${image_file}
+           )
+
+        idf_component_get_property(main_args esptool_py FLASH_ARGS)
+        idf_component_get_property(sub_args esptool_py FLASH_SUB_ARGS)
+        esptool_py_flash_target(${partition}-flash "${main_args}" "${sub_args}" ALWAYS_PLAINTEXT)
+        esptool_py_flash_to_partition(${partition}-flash "${partition}" "${image_file}")
+
+        add_dependencies(${partition}-flash nvs_${partition}_bin)
+
+        if(arg_FLASH_IN_PROJECT)
+            esptool_py_flash_to_partition(flash "${partition}" "${image_file}")
+            add_dependencies(flash nvs_${partition}_bin)
+        endif()
+    else()
+        set(message
+            "Failed to create NVS image for partition '${partition}'. "
+            "Check project configuration if using the correct partition table file."
+           )
+        fail_at_build_time(nvs_${partition}_bin "${message}")
+    endif()
+endfunction()

docs/en/api-reference/storage/nvs_flash.rst

@@ -92,6 +92,40 @@ NVS Partition Generator Utility
 
 This utility helps generate NVS partition binary files which can be flashed separately on a dedicated partition via a flashing utility. Key-value pairs to be flashed onto the partition can be provided via a CSV file. For more details, please refer to :doc:`nvs_partition_gen`.
 
+Instead of calling the ``nvs_partition_gen.py`` tool manually, the creation of the partition binary files can also be done directly from CMake using the function ``nvs_create_partition_image``::
+
+    nvs_create_partition_image(<partition> <csv> [FLASH_IN_PROJECT] [DEPENDS  dep dep dep ...])
+
+**Positional Arguments**:
+
+.. list-table::
+    :header-rows: 1
+
+    * - Parameter
+      - Description
+    * - ``partition``
+      - Name of the NVS parition
+    * - ``csv``
+      - Path to CSV file to parse
+
+
+**Optional Arguments**:
+
+.. list-table::
+   :header-rows: 1
+
+   * - Parameter
+     - Description
+   * - ``FLASH_IN_PROJECT``
+     - Name of the NVS parition
+   * - ``DEPENDS``
+     - Specify files on which the command depends
+
+
+If ``FLASH_IN_PROJECT`` is not specified, the image will still be generated, but you will have to flash it manually using ``idf.py <partition>-flash`` (e.g., if your parition name is ``nvs``, then use ``idf.py nvs-flash``).
+
+``nvs_create_partition_image`` must be called from one of the component ``CMakeLists.txt`` files. Currently, only non-encrypted partitions are supported.
+
 Application Example
 -------------------
 

docs/zh_CN/api-reference/storage/nvs_flash.rst

@@ -92,6 +92,40 @@ NVS 分区生成程序
 
 NVS 分区生成程序帮助生成 NVS 分区二进制文件，可使用烧录程序将二进制文件单独烧录至特定分区。烧录至分区上的键值对由 CSV 文件提供，详情请参考 :doc:`nvs_partition_gen`。
 
+可以直接使用函数 ``nvs_create_partition_image`` 通过 CMake 创建分区二进制文件，无需手动调用 ``nvs_partition_gen.py`` 工具::
+
+    nvs_create_partition_image(<partition> <csv> [FLASH_IN_PROJECT] [DEPENDS  dep dep dep ...])
+
+**位置参数**:
+
+.. list-table::
+    :header-rows: 1
+
+    * - 参数
+      - 描述
+    * - ``partition``
+      - NVS 分区名
+    * - ``csv``
+      - 解析的 CSV 文件路径
+
+
+**可选参数**:
+
+.. list-table::
+   :header-rows: 1
+
+   * - 参数
+     - 描述
+   * - ``FLASH_IN_PROJECT``
+     - NVS 分区名
+   * - ``DEPENDS``
+     - 指定命令依赖的文件
+
+
+在没有指定 ``FLASH_IN_PROJECT`` 的情况下，也支持生成分区镜像，不过此时需要使用 ``idf.py <partition>-flash`` 手动进行烧录。举个例子，如果分区名为 ``nvs``，则需使用的命令为 ``idf.py nvs-flash``。
+
+目前，仅支持从组件中的 ``CMakeLists.txt`` 文件调用 ``nvs_create_partition_image``，且此选项仅适用于非加密分区。
+
 应用示例
 -------------------
 

examples/storage/.build-test-rules.yml

@@ -61,6 +61,16 @@ examples/storage/nvs_rw_value_cxx:
       temporary: true
       reason: lack of runners
 
+examples/storage/nvsgen:
+  disable:
+    - if: IDF_TARGET == "esp32c2"
+      temporary: true
+      reason: target esp32c2 is not supported yet
+  disable_test:
+    - if: IDF_TARGET in ["esp32s2", "esp32s3", "esp32c3", "esp32c6", "esp32h2"]
+      temporary: true
+      reason: lack of runners, should be same for every target
+
 examples/storage/partition_api/partition_find:
   disable:
     - if: IDF_TARGET == "esp32c2"

examples/storage/nvsgen/CMakeLists.txt

@@ -0,0 +1,6 @@
+# The following lines of boilerplate have to be in your project's CMakeLists
+# in this exact order for cmake to work correctly
+cmake_minimum_required(VERSION 3.16)
+
+include($ENV{IDF_PATH}/tools/cmake/project.cmake)
+project(nvsgen)

examples/storage/nvsgen/README.md

@@ -0,0 +1,54 @@
+| Supported Targets | ESP32 | ESP32-C3 | ESP32-C6 | ESP32-H2 | ESP32-S2 | ESP32-S3 |
+| ----------------- | ----- | -------- | -------- | -------- | -------- | -------- |
+
+# NVS Partition Image Generation on Build Example
+
+(See the README.md file in the upper level 'examples' directory for more information about examples.)
+
+This example demonstrates how to use the NVS image generation tool [nvs_partition_gen.py](../../../components/nvs_flash/nvs_partition_generator/nvs_partition_gen.py) to automatically create a NVS partition image from the contents of a CSV file during build, with an option of automatically flashing the created image on invocation of `idf.py -p PORT flash`. For more information, see description of `nvs_partition_gen.py` on the ESP-IDF Programming Guide under API Reference > Storage API > NVS Partition Generator Utility.
+
+The following gives an overview of the example:
+
+1. There is a file `nvs_data.csv` from which the NVS partition image will be created.
+
+2. The function `nvs_create_partition_image` is used to specify that a NVS image should be created during build for the `nvs` partition. It is called from [the main component's CMakeLists.txt](./main/CMakeLists.txt). `FLASH_IN_PROJECT` specifies that the created image should be flashed on invocation of `idf.py -p PORT flash` together with app, bootloader, partition table, etc. For both build systems, the image is created on the example's build directory with the output filename `nvs.bin`.
+
+3. Upon invocation of `idf.py -p PORT flash monitor`, application loads and finds there is already a valid and pre-filled NVS partition in the `nvs` partition with values same as those in `nvs_data.csv` file. The application is then able to read those values.
+
+## How to use example
+
+### Build and flash
+
+To run the example, type the following command:
+
+```CMake
+idf.py -p PORT flash monitor
+```
+
+(To exit the serial monitor, type ``Ctrl-]``.)
+
+See the Getting Started Guide for full steps to configure and use ESP-IDF to build projects.
+
+## Example output
+
+Here is the example's console output:
+
+```
+...
+I (357) example: Opening Non-Volatile Storage (NVS) handle
+I (357) example: Done
+I (357) example: Reading values from NVS
+255
+-128
+65535
+4294967295
+-2147483648
+Lorem ipsum dolor sit amet, consectetur adipiscing elit.
+Fusce quis risus justo.
+Suspendisse egestas in nisi sit amet auctor.
+Pellentesque rhoncus dictum sodales.
+In justo erat, viverra at interdum eget, interdum vel dui.
+I (387) example: Reading values from NVS done - all OK
+```
+
+The logic of the example is contained in a [single source file](./main/nvsgen_example_main.c), and it should be relatively simple to match points in its execution with the log outputs above.

examples/storage/nvsgen/main/CMakeLists.txt

@@ -0,0 +1,8 @@
+idf_component_register(SRCS "nvsgen_example_main.c"
+                    INCLUDE_DIRS ".")
+
+# Create a NVS image from the contents of the `nvs_data` CSV file
+# that fits the partition named 'nvs'. FLASH_IN_PROJECT indicates that
+# the generated image should be flashed when the entire project is flashed to
+# the target with 'idf.py -p PORT flash'.
+nvs_create_partition_image(nvs ../nvs_data.csv FLASH_IN_PROJECT)

examples/storage/nvsgen/main/nvsgen_example_main.c

@@ -0,0 +1,89 @@
+/* Non-Volatile Storage (NVS) Image Generation on Build Example
+ *
+ * SPDX-FileCopyrightText: 2023 Espressif Systems (Shanghai) CO LTD
+ *
+ * SPDX-License-Identifier: Unlicense OR CC0-1.0
+ */
+
+#include <stdio.h>
+#include <inttypes.h>
+#include "freertos/FreeRTOS.h"
+#include "freertos/task.h"
+#include "esp_system.h"
+#include "esp_log.h"
+#include "nvs_flash.h"
+#include "nvs.h"
+
+static const char *TAG = "example";
+
+void app_main(void)
+{
+    // Initialize NVS
+    esp_err_t err = nvs_flash_init();
+    if (err == ESP_ERR_NVS_NO_FREE_PAGES || err == ESP_ERR_NVS_NEW_VERSION_FOUND) {
+        // NVS partition was truncated and needs to be erased
+        // Retry nvs_flash_init
+        ESP_ERROR_CHECK(nvs_flash_erase());
+        err = nvs_flash_init();
+    }
+    ESP_ERROR_CHECK(err);
+
+    // Open the pre-filled NVS partition called "nvs"
+    ESP_LOGI(TAG, "Opening Non-Volatile Storage (NVS) handle");
+    nvs_handle_t my_handle;
+    err = nvs_open_from_partition("nvs", "storage", NVS_READONLY, &my_handle);
+    if (err != ESP_OK) {
+        ESP_LOGE(TAG, "Error (%s) opening NVS handle!\n", esp_err_to_name(err));
+        return;
+    }
+    ESP_LOGI(TAG, "The NVS handle successfully opened");
+
+    // Read values
+    ESP_LOGI(TAG, "Reading values from NVS");
+
+    uint8_t u8_val = 0;
+    err = nvs_get_u8(my_handle, "u8_key", &u8_val);
+    ESP_ERROR_CHECK(err);
+    printf("%"PRIu8"\n", u8_val);
+    assert(u8_val == 255);
+
+    int8_t i8_val = 0;
+    err = nvs_get_i8(my_handle, "i8_key", &i8_val);
+    ESP_ERROR_CHECK(err);
+    printf("%"PRIi8"\n", i8_val);
+    assert(i8_val == -128);
+
+    uint16_t u16_val = 0;
+    err = nvs_get_u16(my_handle, "u16_key", &u16_val);
+    ESP_ERROR_CHECK(err);
+    printf("%"PRIu16"\n", u16_val);
+    assert(u16_val == 65535);
+
+    uint32_t u32_val = 0;
+    err = nvs_get_u32(my_handle, "u32_key", &u32_val);
+    ESP_ERROR_CHECK(err);
+    printf("%"PRIu32"\n", u32_val);
+    assert(u32_val == 4294967295);
+
+    int32_t i32_val = 0;
+    err = nvs_get_i32(my_handle, "i32_key", &i32_val);
+    ESP_ERROR_CHECK(err);
+    printf("%"PRIi32"\n", i32_val);
+    assert(i32_val == -2147483648);
+
+    size_t str_len = 0;
+    err = nvs_get_str(my_handle, "str_key", NULL, &str_len);
+    ESP_ERROR_CHECK(err);
+    assert(str_len == 222);
+
+    char* str_val = (char*) malloc(str_len);
+    err = nvs_get_str(my_handle, "str_key", str_val, &str_len);
+    ESP_ERROR_CHECK(err);
+    printf("%s\n", str_val);
+    assert(str_val[0] == 'L');
+    free(str_val);
+
+    // Close
+    nvs_close(my_handle);
+    ESP_LOGI(TAG, "Reading values from NVS done - all OK");
+}

examples/storage/nvsgen/nvs_data.csv

@@ -0,0 +1,13 @@
+# Sample csv file
+key,type,encoding,value
+storage,namespace,,
+u8_key,data,u8,255
+i8_key,data,i8,-128
+u16_key,data,u16,65535
+u32_key,data,u32,4294967295
+i32_key,data,i32,-2147483648
+str_key,data,string,"Lorem ipsum dolor sit amet, consectetur adipiscing elit.
+Fusce quis risus justo.
+Suspendisse egestas in nisi sit amet auctor.
+Pellentesque rhoncus dictum sodales.
+In justo erat, viverra at interdum eget, interdum vel dui."

examples/storage/nvsgen/partitions_example.csv

@@ -0,0 +1,6 @@
+# Name,   Type, SubType, Offset,  Size, Flags
+# Note: if you have increased the bootloader size, make sure to update the offsets to avoid overlap
+# Name,   Type, SubType, Offset,  Size, Flags
+nvs,      data, nvs,     0x9000,  0x6000,
+phy_init, data, phy,     0xf000,  0x1000,
+factory,  app,  factory, 0x10000, 1M,

examples/storage/nvsgen/pytest_nvsgen_example.py

@@ -0,0 +1,11 @@
+# SPDX-FileCopyrightText: 2023 Espressif Systems (Shanghai) CO LTD
+# SPDX-License-Identifier: Unlicense OR CC0-1.0
+
+import pytest
+from pytest_embedded import Dut
+
+
+@pytest.mark.esp32
+def test_nvsgen_example(dut: Dut) -> None:
+    dut.expect('Reading values from NVS', timeout=10)
+    dut.expect('Reading values from NVS done - all OK', timeout=10)

examples/storage/nvsgen/sdkconfig.defaults

@@ -0,0 +1,3 @@
+CONFIG_PARTITION_TABLE_CUSTOM=y
+CONFIG_PARTITION_TABLE_CUSTOM_FILENAME="partitions_example.csv"
+CONFIG_PARTITION_TABLE_FILENAME="partitions_example.csv"