feat(elf2image): add ram-only-header argument

The ram-only-header configuration makes only
the RAM segments visible to the ROM bootloader placing
them at the beginning of the file and altering the
segment count from the image header with the quantity
of these segments, and also writing only their
checksum. This segment placement also may not result
as optimal as the standard way regarding the padding
gap use among the flash segments that could result
in a less fragmented binary.

The image built must then handle the basic hardware
initialization and the flash mapping for code execution
after ROM bootloader boot it.

Signed-off-by: Marek Matej <marek.matej@espressif.com>
Signed-off-by: Almir Okato <almir.okato@espressif.com>

docs/en/esptool/basic-commands.rst

@@ -211,6 +211,10 @@ By default, ``elf2image`` uses the sections in the ELF file to generate each seg
 
     In the above example, the output image file would be called ``my_esp_app.bin``.
 
+    The ``--ram-only-header`` configuration is mainly applicable for use within the Espressif's SIMPLE_BOOT option from 3rd party OSes such as ZephyrOS and NuttX OS.
+    This option makes only the RAM segments visible to the ROM bootloader placing them at the beginning of the file and altering the segment count from the image header with the quantity of these segments, and also writing only their checksum. This segment placement may result in a more fragmented binary because of flash alignment constraints.
+    It is strongly recommended to use this configuration with care, because the image built must then handle the basic hardware initialization and the flash mapping for code execution after ROM bootloader boot it.
+
 .. _image-info:
 
 Output .bin Image Details: image_info
@@ -279,7 +283,7 @@ The output of the command will be in ``raw`` format and gaps between individual
 UF2 Output Format
 ^^^^^^^^^^^^^^^^^
 
-This command will generate a UF2 (`USB Flashing Format <https://github.com/microsoft/uf2>`_) binary. 
+This command will generate a UF2 (`USB Flashing Format <https://github.com/microsoft/uf2>`_) binary.
 This UF2 file can be copied to a USB mass storage device exposed by another ESP running the `ESP USB Bridge <https://github.com/espressif/esp-usb-bridge>`_ project. The bridge MCU will use it to flash the target MCU. This is as simple copying (or "drag-and-dropping") the file to the exposed disk accessed by a file explorer in your machine.
 
 Gaps between the files will be filled with `0x00` bytes.

esptool/__init__.py

@@ -477,6 +477,17 @@ def add_spi_flash_subparsers(parent, allow_keep, auto_detect):
         "must be aligned to. Value 0xFF is used for padding, similar to erase_flash",
         default=None,
     )
+    parser_elf2image.add_argument(
+        "--ram-only-header",
+        help="Order segments of the output so IRAM and DRAM are placed at the "
+        "beginning and force the main header segment number to RAM segments "
+        "quantity. This will make the other segments invisible to the ROM "
+        "loader. Use this argument with care because the ROM loader will load "
+        "only the RAM segments although the other segments being present in "
+        "the output.",
+        action="store_true",
+        default=None,
+    )
 
     add_spi_flash_subparsers(parser_elf2image, allow_keep=False, auto_detect=False)
 

esptool/bin_image.py

@@ -571,7 +571,7 @@ class ESP32FirmwareImage(BaseFirmwareImage):
 
     IROM_ALIGN = 65536
 
-    def __init__(self, load_file=None, append_digest=True):
+    def __init__(self, load_file=None, append_digest=True, ram_only_header=False):
         super(ESP32FirmwareImage, self).__init__()
         self.secure_pad = None
         self.flash_mode = 0
@@ -589,6 +589,7 @@ def __init__(self, load_file=None, append_digest=True):
         self.min_rev = 0
         self.min_rev_full = 0
         self.max_rev_full = 0
+        self.ram_only_header = ram_only_header
 
         self.append_digest = append_digest
 
@@ -701,33 +702,61 @@ def get_alignment_data_needed(segment):
                     pad_len += self.IROM_ALIGN
                 return pad_len
 
-            # try to fit each flash segment on a 64kB aligned boundary
-            # by padding with parts of the non-flash segments...
-            while len(flash_segments) > 0:
-                segment = flash_segments[0]
-                pad_len = get_alignment_data_needed(segment)
-                if pad_len > 0:  # need to pad
-                    if len(ram_segments) > 0 and pad_len > self.SEG_HEADER_LEN:
-                        pad_segment = ram_segments[0].split_image(pad_len)
-                        if len(ram_segments[0].data) == 0:
-                            ram_segments.pop(0)
-                    else:
-                        pad_segment = ImageSegment(0, b"\x00" * pad_len, f.tell())
-                    checksum = self.save_segment(f, pad_segment, checksum)
+            if self.ram_only_header:
+                # write RAM segments first in order to get only RAM segments quantity
+                # and checksum (ROM bootloader will only care for RAM segments and its
+                # correct checksums)
+                for segment in ram_segments:
+                    checksum = self.save_segment(f, segment, checksum)
                     total_segments += 1
-                else:
+                self.append_checksum(f, checksum)
+
+                # reversing to match the same section order from linker script
+                flash_segments.reverse()
+                for segment in flash_segments:
+                    pad_len = get_alignment_data_needed(segment)
+                    while pad_len > 0:
+                        pad_segment = ImageSegment(0, b"\x00" * pad_len, f.tell())
+                        self.save_segment(f, pad_segment)
+                        total_segments += 1
+                        pad_len = get_alignment_data_needed(segment)
                     # write the flash segment
                     assert (
                         f.tell() + 8
                     ) % self.IROM_ALIGN == segment.addr % self.IROM_ALIGN
-                    checksum = self.save_flash_segment(f, segment, checksum)
-                    flash_segments.pop(0)
+                    # save the flash segment but not saving its checksum neither
+                    # saving the number of flash segments, since ROM bootloader
+                    # should "not see" them
+                    self.save_flash_segment(f, segment)
+                    total_segments += 1
+            else:  # not self.ram_only_header
+                # try to fit each flash segment on a 64kB aligned boundary
+                # by padding with parts of the non-flash segments...
+                while len(flash_segments) > 0:
+                    segment = flash_segments[0]
+                    pad_len = get_alignment_data_needed(segment)
+                    if pad_len > 0:  # need to pad
+                        if len(ram_segments) > 0 and pad_len > self.SEG_HEADER_LEN:
+                            pad_segment = ram_segments[0].split_image(pad_len)
+                            if len(ram_segments[0].data) == 0:
+                                ram_segments.pop(0)
+                        else:
+                            pad_segment = ImageSegment(0, b"\x00" * pad_len, f.tell())
+                        checksum = self.save_segment(f, pad_segment, checksum)
+                        total_segments += 1
+                    else:
+                        # write the flash segment
+                        assert (
+                            f.tell() + 8
+                        ) % self.IROM_ALIGN == segment.addr % self.IROM_ALIGN
+                        checksum = self.save_flash_segment(f, segment, checksum)
+                        flash_segments.pop(0)
+                        total_segments += 1
+
+                # flash segments all written, so write any remaining RAM segments
+                for segment in ram_segments:
+                    checksum = self.save_segment(f, segment, checksum)
                     total_segments += 1
-
-            # flash segments all written, so write any remaining RAM segments
-            for segment in ram_segments:
-                checksum = self.save_segment(f, segment, checksum)
-                total_segments += 1
 
             if self.secure_pad:
                 # pad the image so that after signing it will end on a a 64KB boundary.
@@ -759,8 +788,9 @@ def get_alignment_data_needed(segment):
                 checksum = self.save_segment(f, pad_segment, checksum)
                 total_segments += 1
 
-            # done writing segments
-            self.append_checksum(f, checksum)
+            if not self.ram_only_header:
+                # done writing segments
+                self.append_checksum(f, checksum)
             image_length = f.tell()
 
             if self.secure_pad:
@@ -769,7 +799,12 @@ def get_alignment_data_needed(segment):
             # kinda hacky: go back to the initial header and write the new segment count
             # that includes padding segments. This header is not checksummed
             f.seek(1)
-            f.write(bytes([total_segments]))
+            if self.ram_only_header:
+                # Update the header with the RAM segments quantity as it should be
+                # visible by the ROM bootloader
+                f.write(bytes([len(ram_segments)]))
+            else:
+                f.write(bytes([total_segments]))
 
             if self.append_digest:
                 # calculate the SHA256 of the whole file and append it

esptool/cmds.py

@@ -980,6 +980,11 @@ def elf2image(args):
         args.chip = "esp8266"
 
     print("Creating {} image...".format(args.chip))
+    if args.ram_only_header:
+        print(
+            "RAM only visible in the header - only RAM segments are visible to the "
+            "ROM loader!"
+        )
 
     if args.chip != "esp8266":
         image = CHIP_DEFS[args.chip].BOOTLOADER_IMAGE()
@@ -990,6 +995,7 @@ def elf2image(args):
         image.min_rev = args.min_rev
         image.min_rev_full = args.min_rev_full
         image.max_rev_full = args.max_rev_full
+        image.ram_only_header = args.ram_only_header
         image.append_digest = args.append_digest
     elif args.version == "1":  # ESP8266
         image = ESP8266ROMFirmwareImage()

test/test_imagegen.py

@@ -1,6 +1,7 @@
 import hashlib
 import os
 import os.path
+import re
 import struct
 import subprocess
 import sys
@@ -113,7 +114,7 @@ def assertImageContainsSection(self, image, elf, section_name):
                 f" segment(s) in bin image (image segments: {image.segments})"
             )
 
-    def assertImageInfo(self, binpath, chip="esp8266"):
+    def assertImageInfo(self, binpath, chip="esp8266", assert_sha=False):
         """
         Run esptool.py image_info on a binary file,
         assert no red flags about contents.
@@ -126,7 +127,13 @@ def assertImageInfo(self, binpath, chip="esp8266"):
         except subprocess.CalledProcessError as e:
             print(e.output)
             raise
-        assert "invalid" not in output, "Checksum calculation should be valid"
+        assert re.search(
+            r"Checksum: [a-fA-F0-9]{2} \(valid\)", output
+        ), "Checksum calculation should be valid"
+        if assert_sha:
+            assert re.search(
+                r"Validation Hash: [a-fA-F0-9]{64} \(valid\)", output
+            ), "SHA256 should be valid"
         assert (
             "warning" not in output.lower()
         ), "Should be no warnings in image_info output"
@@ -267,7 +274,11 @@ def _test_elf2image(self, elfpath, binpath, extra_args=[]):
         try:
             self.run_elf2image("esp32", elfpath, extra_args=extra_args)
             image = esptool.bin_image.LoadFirmwareImage("esp32", binpath)
-            self.assertImageInfo(binpath, "esp32")
+            self.assertImageInfo(
+                binpath,
+                "esp32",
+                True if "--ram-only-header" not in extra_args else False,
+            )
             return image
         finally:
             try_delete(binpath)
@@ -322,6 +333,13 @@ def test_use_segments(self):
         image = self._test_elf2image(ELF, BIN, ["--use_segments"])
         assert len(image.segments) == 2
 
+    def test_ram_only_header(self):
+        ELF = "esp32-app-template.elf"
+        BIN = "esp32-app-template.bin"
+        # --ram-only-header produces just 2 visible segments in the bin
+        image = self._test_elf2image(ELF, BIN, ["--ram-only-header"])
+        assert len(image.segments) == 2
+
 
 class TestESP8266FlashHeader(BaseTestCase):
     def test_2mb(self):