qemu_x86 transient build errors for mmu_tables.o

[andrewboie]

Describe the bug
Seeing occasional build errors which go away if Make is re-run

[ 87%] Generating mmu_tables.bin, user_mmu_tables.bin
[ 88%] Generating mmu_tables.o
/opt/zephyr-sdk/sysroots/x86_64-pokysdk-linux/usr/bin/i586-zephyr-elf/i586-zephyr-elf-objcopy: 'mmu_tables.bin': No such file
make[3]: *** [zephyr/arch/arch/x86/CMakeFiles/mmu_tables_o.dir/build.make:61: zephyr/arch/arch/x86/mmu_tables.o] Error 1
make[3]: Target 'zephyr/arch/arch/x86/CMakeFiles/mmu_tables_o.dir/build' not remade because of errors.
make[3]: Leaving directory '/projects/zephyr3/sanity-out/qemu_x86/tests/net/route/net.route'

Seems to be a build parallelism issue with the MMU build logic

To reproduce, run sanitycheck -p qemu_x86, for me a few tests always fail, but different ones each time.

[andrewboie]

if(CONFIG_X86_MMU)
  # Use gen_mmu.py and objcopy to generate mmu_tables.o from from the
  # elf file ${ZEPHYR_PREBUILT_EXECUTABLE}, creating the temp files mmu_tables.bin
  # and mmulist.bin along the way.
  #
  # ${ZEPHYR_PREBUILT_EXECUTABLE}.elf -> mmulist.bin -> mmu_tables.bin -> mmu_tables.o
  add_custom_command(
    OUTPUT mmulist.bin
    COMMAND
    ${CMAKE_OBJCOPY}
    -I ${OUTPUT_FORMAT}
    -O binary
    -j mmulist
    $<TARGET_FILE:${ZEPHYR_PREBUILT_EXECUTABLE}>
    mmulist.bin
    DEPENDS ${ZEPHYR_PREBUILT_EXECUTABLE}
    WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
    )
  add_custom_command(
    OUTPUT mmu_tables.bin user_mmu_tables.bin
    COMMAND
    ${PYTHON_EXECUTABLE}
    ${ZEPHYR_BASE}/scripts/gen_mmu_x86.py
    -i mmulist.bin
    -k $<TARGET_FILE:${ZEPHYR_PREBUILT_EXECUTABLE}>
    -o mmu_tables.bin
    -u user_mmu_tables.bin
    $<$<BOOL:${CMAKE_VERBOSE_MAKEFILE}>:-v>
    WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
    DEPENDS mmulist.bin
    DEPENDS ${ZEPHYR_PREBUILT_EXECUTABLE}
    )
  add_custom_command(
    OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/mmu_tables.o
    COMMAND
    ${CMAKE_OBJCOPY}
    -I binary
    -B ${OUTPUT_ARCH}
    -O ${OUTPUT_FORMAT}
    --rename-section .data=.mmu_data
    mmu_tables.bin
    mmu_tables.o
    WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
    DEPENDS mmu_tables.bin
    )

  add_custom_target(  mmu_tables_o DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/mmu_tables.o)
  add_library(        mmu_tables STATIC IMPORTED GLOBAL)
  set_property(TARGET mmu_tables PROPERTY IMPORTED_LOCATION ${CMAKE_CURRENT_BINARY_DIR}/mmu_tables.o)
  add_dependencies(   mmu_tables mmu_tables_o)
  set_property(GLOBAL APPEND PROPERTY GENERATED_KERNEL_OBJECT_FILES mmu_tables)

  if(CONFIG_X86_KPTI)
    add_custom_command(
      OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/user_mmu_tables.o
      COMMAND
      ${CMAKE_OBJCOPY}
      -I binary
      -B ${OUTPUT_ARCH}
      -O ${OUTPUT_FORMAT}
      --rename-section .data=.user_mmu_data
      user_mmu_tables.bin
      user_mmu_tables.o
      WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
      DEPENDS user_mmu_tables.bin
    )

    add_custom_target(  user_mmu_tables_o DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/user_mmu_tables.o)
    add_library(        user_mmu_tables STATIC IMPORTED GLOBAL)
    set_property(TARGET user_mmu_tables PROPERTY IMPORTED_LOCATION ${CMAKE_CURRENT_BINARY_DIR}/user_mmu_tables.o)
    add_dependencies(   user_mmu_tables user_mmu_tables_o)
    set_property(GLOBAL APPEND PROPERTY GENERATED_KERNEL_OBJECT_FILES user_mmu_tables)
  endif()
endif()

[andrewboie]

What is failing is that the rule to generate mmu_tables.o from mmu_tables.bin:

add_custom_command(
    OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/mmu_tables.o
    COMMAND
    ${CMAKE_OBJCOPY}
    -I binary
    -B ${OUTPUT_ARCH}
    -O ${OUTPUT_FORMAT}
    --rename-section .data=.mmu_data
    mmu_tables.bin
    mmu_tables.o
    WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
    DEPENDS mmu_tables.bin
    )

Seems to no longer be properly depending on the rule which generates mmu_tables.bin, it's complaining that the file is missing. It has "DEPENDS mmu_tables.bin" so I don't know what is going on.

The rule that generates this file:

add_custom_command(
    OUTPUT mmu_tables.bin user_mmu_tables.bin
    COMMAND
    ${PYTHON_EXECUTABLE}
    ${ZEPHYR_BASE}/scripts/gen_mmu_x86.py
    -i mmulist.bin
    -k $<TARGET_FILE:${ZEPHYR_PREBUILT_EXECUTABLE}>
    -o mmu_tables.bin
    -u user_mmu_tables.bin
    $<$<BOOL:${CMAKE_VERBOSE_MAKEFILE}>:-v>
    WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
    DEPENDS mmulist.bin
    DEPENDS ${ZEPHYR_PREBUILT_EXECUTABLE}
    )

Note that this rule was recently changed to have two OUTPUT items, since it now emits two binary files.

[andrewboie]

Observed this again today. Notice that in this case, it's generating mmu_tables.o BEFORE mmu_tables.bin, the dependency chain is definitely broken in some way I can't figure out.

[ 88%] Generating mmu_tables.o
[ 89%] Generating mmu_tables.bin, user_mmu_tables.bin
/opt/zephyr-sdk/sysroots/x86_64-pokysdk-linux/usr/bin/i586-zephyr-elf/i586-zephyr-elf-objcopy: 'mmu_tables.bin': No such file
make[3]: *** [zephyr/arch/arch/x86/CMakeFiles/mmu_tables_o.dir/build.make:61: zephyr/arch/arch/x86/mmu_tables.o] Error 1
make[3]: Target 'zephyr/arch/arch/x86/CMakeFiles/mmu_tables_o.dir/build' not remade because of errors.
make[3]: Leaving directory '/projects/zephyr3/sanity-out/qemu_x86/tests/net/neighbor/net.neighbour'
make[2]: *** [CMakeFiles/Makefile2:1161: zephyr/arch/arch/x86/CMakeFiles/mmu_tables_o.dir/all] Error 2