Skip to content

llext-edk: rework, add Python export and more optional information #78727

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -2210,6 +2210,12 @@ add_custom_command(
-DWEST_TOPDIR=${WEST_TOPDIR}
-DZEPHYR_BASE=${ZEPHYR_BASE}
-DCONFIG_LLEXT_EDK_USERSPACE_ONLY=${CONFIG_LLEXT_EDK_USERSPACE_ONLY}
-DARCH=${ARCH}
-DBOARD=${NORMALIZED_BOARD_TARGET}
-DSOC=${SOC_NAME}
-DTOOLCHAIN=${ZEPHYR_TOOLCHAIN_VARIANT}
-DCROSS_COMPILE=${CROSS_COMPILE_TARGET}
-DCOMPILER=${COMPILER}
-P ${ZEPHYR_BASE}/cmake/llext-edk.cmake
DEPENDS ${logical_target_for_zephyr_elf}
COMMAND_EXPAND_LISTS
Expand Down
138 changes: 99 additions & 39 deletions cmake/llext-edk.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,13 @@
# edk directory. This is necessary when building an extension that only
# supports userspace, as the syscall headers are regenerated in the edk
# directory.
# In addition, the following Zephyr internal variables are exported but
# not guaranteed to be present or stable:
# - ARCH, BOARD, SOC: information about the target of the EDK.
# - TOOLCHAIN: the Zephyr toolchain variant used to compile the build.
# - CROSS_COMPILE: the arch-specific prefix used to compile the build
# (when the toolchain defines it).
# - COMPILER: the name of the compiler used to compile the build.

cmake_minimum_required(VERSION 3.20.0)

Expand Down Expand Up @@ -89,9 +96,9 @@ function(relative_dir dir relative_out bindir_out)
endif()
endfunction()

string(REGEX REPLACE "[^a-zA-Z0-9]" "_" llext_edk_name_sane ${llext_edk_name})
string(TOUPPER ${llext_edk_name_sane} llext_edk_name_sane)
set(install_dir_var "${llext_edk_name_sane}_INSTALL_DIR")
string(REGEX REPLACE "[^a-zA-Z0-9]" "_" var_prefix ${llext_edk_name})
string(TOUPPER ${var_prefix} var_prefix)
set(install_dir_var "${var_prefix}_INSTALL_DIR")

separate_arguments(llext_edk_cflags NATIVE_COMMAND ${llext_edk_cflags})

Expand All @@ -106,20 +113,17 @@ foreach(flag ${llext_edk_cflags})
relative_dir(${parent} dest bindir)
cmake_path(RELATIVE_PATH dest BASE_DIRECTORY ${llext_edk} OUTPUT_VARIABLE dest_rel)
if(bindir)
list(APPEND imacros_gen_make "-imacros\$(${install_dir_var})/${dest_rel}/${name}")
list(APPEND imacros_gen_cmake "-imacros\${CMAKE_CURRENT_LIST_DIR}/${dest_rel}/${name}")
list(APPEND imacros_gen "@DASHIMACROS@${dest_rel}/${name}")
else()
list(APPEND imacros_make "-imacros\$(${install_dir_var})/${dest_rel}/${name}")
list(APPEND imacros_cmake "-imacros\${CMAKE_CURRENT_LIST_DIR}/${dest_rel}/${name}")
list(APPEND imacros "@DASHIMACROS@${dest_rel}/${name}")
endif()
else()
list(APPEND new_cflags ${flag})
endif()
endforeach()
set(llext_edk_cflags ${new_cflags})

list(APPEND base_flags_make ${llext_edk_cflags} ${imacros_make})
list(APPEND base_flags_cmake ${llext_edk_cflags} ${imacros_cmake})
list(APPEND base_flags ${llext_edk_cflags} ${imacros})

separate_arguments(include_dirs NATIVE_COMMAND ${INTERFACE_INCLUDE_DIRECTORIES})
file(MAKE_DIRECTORY ${llext_edk_inc})
Expand All @@ -137,54 +141,110 @@ foreach(dir ${include_dirs})

cmake_path(RELATIVE_PATH dest BASE_DIRECTORY ${llext_edk} OUTPUT_VARIABLE dest_rel)
if(bindir)
list(APPEND inc_gen_flags_make "-I\$(${install_dir_var})/${dest_rel}")
list(APPEND inc_gen_flags_cmake "-I\${CMAKE_CURRENT_LIST_DIR}/${dest_rel}")
list(APPEND gen_inc_flags "@DASHI@${dest_rel}")
else()
list(APPEND inc_flags_make "-I\$(${install_dir_var})/${dest_rel}")
list(APPEND inc_flags_cmake "-I\${CMAKE_CURRENT_LIST_DIR}/${dest_rel}")
list(APPEND inc_flags "@DASHI@${dest_rel}")
endif()
list(APPEND all_inc_flags_make "-I\$(${install_dir_var})/${dest_rel}")
list(APPEND all_inc_flags_cmake "-I\${CMAKE_CURRENT_LIST_DIR}/${dest_rel}")
list(APPEND all_inc_flags "@DASHI@${dest_rel}")
endforeach()

list(APPEND all_flags ${base_flags} ${imacros_gen} ${all_inc_flags})

if(CONFIG_LLEXT_EDK_USERSPACE_ONLY)
# Copy syscall headers from edk directory, as they were regenerated there.
file(COPY ${PROJECT_BINARY_DIR}/edk/include/generated/ DESTINATION ${llext_edk_inc}/zephyr/include/generated)
endif()

# Generate flags for Makefile
list(APPEND all_flags_make ${base_flags_make} ${imacros_gen_make} ${all_inc_flags_make})
list(JOIN all_flags_make " " all_flags_str)
file(WRITE ${llext_edk}/Makefile.cflags "LLEXT_CFLAGS = ${all_flags_str}")

list(JOIN all_inc_flags_make " " all_inc_flags_str)
file(APPEND ${llext_edk}/Makefile.cflags "\n\nLLEXT_ALL_INCLUDE_CFLAGS = ${all_inc_flags_str}")

list(JOIN inc_flags_make " " inc_flags_str)
file(APPEND ${llext_edk}/Makefile.cflags "\n\nLLEXT_INCLUDE_CFLAGS = ${inc_flags_str}")

list(JOIN inc_gen_flags_make " " inc_gen_flags_str)
file(APPEND ${llext_edk}/Makefile.cflags "\n\nLLEXT_GENERATED_INCLUDE_CFLAGS = ${inc_gen_flags_str}")
#
# Generate the EDK flags files
#

list(JOIN base_flags_make " " base_flags_str)
file(APPEND ${llext_edk}/Makefile.cflags "\n\nLLEXT_BASE_CFLAGS = ${base_flags_str}")
set(edk_targets MAKEFILE CMAKE PYTHON)
set(edk_file_MAKEFILE ${llext_edk}/Makefile.cflags)
set(edk_file_CMAKE ${llext_edk}/cmake.cflags)
set(edk_file_PYTHON ${llext_edk}/python.cflags)

list(JOIN imacros_gen_make " " imacros_gen_str)
file(APPEND ${llext_edk}/Makefile.cflags "\n\nLLEXT_GENERATED_IMACROS_CFLAGS = ${imacros_gen_str}")
# Escape problematic characters in a string
function(edk_escape target str_in str_out)
string(REPLACE "\\" "\\\\" str_escaped "${str_in}")
string(REPLACE "\"" "\\\"" str_escaped "${str_escaped}")
if(target STREQUAL "PYTHON")
string(REPLACE "{" "\\{" str_escaped "${str_escaped}")
endif()
set(${str_out} "${str_escaped}" PARENT_SCOPE)
endfunction()

# Generate flags for CMake
list(APPEND all_flags_cmake ${base_flags_cmake} ${imacros_gen_cmake} ${all_inc_flags_cmake})
file(WRITE ${llext_edk}/cmake.cflags "set(LLEXT_CFLAGS ${all_flags_cmake})")
# Clear the contents of the requested file
function(edk_write_header target)
file(WRITE ${edk_file_${target}} "")
endfunction()

file(APPEND ${llext_edk}/cmake.cflags "\n\nset(LLEXT_ALL_INCLUDE_CFLAGS ${all_inc_flags_cmake})")
# Mark a section in the file with a single line comment
function(edk_write_comment target comment)
file(APPEND ${edk_file_${target}} "\n# ${comment}\n")
endfunction()

file(APPEND ${llext_edk}/cmake.cflags "\n\nset(LLEXT_INCLUDE_CFLAGS ${inc_flags_cmake})")
# Define a variable in the file
function(edk_write_var target var_name var_value)
if(target STREQUAL "CMAKE")
# CMake: export assignments of the form:
#
# set(var "value1;value2;...")
#
edk_escape(${target} "${exp_var_value}" exp_var_value)
set(DASHIMACROS "-imacros\${CMAKE_CURRENT_LIST_DIR}/")
set(DASHI "-I\${CMAKE_CURRENT_LIST_DIR}/")
string(CONFIGURE "${var_value}" exp_var_value @ONLY)
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Oohh, wish I had found this CONFIGURE trick, I always hated the duplicated variables... Nice!

# The list is otherwise exported verbatim, surrounded by quotes.
file(APPEND ${edk_file_${target}} "set(${var_name} \"${exp_var_value}\")\n")
elseif(target STREQUAL "MAKEFILE")
# Makefile: export assignments of the form:
#
# var = "value1" "value2" ...
#
edk_escape(${target} "${exp_var_value}" exp_var_value)
set(DASHIMACROS "-imacros\$(${install_dir_var})/")
set(DASHI "-I\$(${install_dir_var})/")
string(CONFIGURE "${var_value}" exp_var_value @ONLY)
# Each element of the list is wrapped in quotes and is separated by a space.
list(JOIN exp_var_value "\" \"" exp_var_value_str)
file(APPEND ${edk_file_${target}} "${var_name} = \"${exp_var_value_str}\"\n")
elseif(target STREQUAL "PYTHON")
# Python: export lists of f-strings of the form:
#
# var = [f"value1", f"value2", ...]
#
edk_escape(${target} "${exp_var_value}" exp_var_value)
set(DASHIMACROS "-imacros{${install_dir_var}}/")
set(DASHI "-I{${install_dir_var}}/")
string(CONFIGURE "${var_value}" exp_var_value @ONLY)
# Each element of the list is wrapped in f-quotes and is separated by a
# space and a comma.
list(JOIN exp_var_value "\", f\"" exp_var_value_str)
file(APPEND ${edk_file_${target}} "${var_name} = [f\"${exp_var_value_str}\"]\n")
Comment on lines +224 to +225
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this doesn't look like yaml to me 😉
ref: #78727 (comment)

then a yaml file would have been just as easy / nice to use 😉

Only additional file format I consider acceptable is yaml

remember, yaml can be read directly into Python dict and is much nicer to work with.
We actually have a yaml module in Zephyr CMake now 😄

endif()
endfunction()

file(APPEND ${llext_edk}/cmake.cflags "\n\nset(LLEXT_GENERATED_INCLUDE_CFLAGS ${inc_gen_flags_cmake})")
foreach(target ${edk_targets})
edk_write_header(${target})

file(APPEND ${llext_edk}/cmake.cflags "\n\nset(LLEXT_BASE_CFLAGS ${base_flags_cmake})")
edk_write_comment(${target} "Target and build information")
edk_write_var(${target} "${var_prefix}_ARCH" "${ARCH}")
edk_write_var(${target} "${var_prefix}_BOARD" "${BOARD}")
edk_write_var(${target} "${var_prefix}_SOC" "${SOC}")
edk_write_var(${target} "${var_prefix}_TOOLCHAIN" "${TOOLCHAIN}")
edk_write_var(${target} "${var_prefix}_CROSS_COMPILE" "${CROSS_COMPILE}")
edk_write_var(${target} "${var_prefix}_COMPILER" "${COMPILER}")

file(APPEND ${llext_edk}/cmake.cflags "\n\nset(LLEXT_GENERATED_IMACROS_CFLAGS ${imacros_gen_cmake})")
edk_write_comment(${target} "Compile flags")
edk_write_var(${target} "LLEXT_CFLAGS" "${all_flags}")
edk_write_var(${target} "LLEXT_ALL_INCLUDE_CFLAGS" "${all_inc_flags}")
edk_write_var(${target} "LLEXT_INCLUDE_CFLAGS" "${inc_flags}")
edk_write_var(${target} "LLEXT_GENERATED_INCLUDE_CFLAGS" "${gen_inc_flags}")
edk_write_var(${target} "LLEXT_BASE_CFLAGS" "${base_flags}")
edk_write_var(${target} "LLEXT_GENERATED_IMACROS_CFLAGS" "${imacros_gen}")
endforeach()

# Generate the tarball
file(ARCHIVE_CREATE
Expand Down
69 changes: 62 additions & 7 deletions doc/services/llext/build.rst
Original file line number Diff line number Diff line change
Expand Up @@ -152,15 +152,35 @@ directory. It's a tarball that contains the headers and compile flags needed
to build extensions. The extension developer can then include the headers
and use the compile flags in their build system to build the extension.

EDK definition files
--------------------

The EDK includes several convenience files which define a set of variables that
contain the compile flags needed by the project, as well as other build-related
information, when enabled. The information is currently exported in the
following formats:

- ``Makefile.cflags``, for Makefile-based projects;
- ``cmake.cflags``, for CMake-based projects;
- ``python.cflags``, for interfacing with Python scripts.

Paths to the headers and flags are prefixed by the EDK root directory. This is
automatically obtained for CMake projects from ``CMAKE_CURRENT_LIST_DIR``;
other formats refer to an ``LLEXT_EDK_INSTALL_DIR`` variable, which must be set
by the user with the path where the EDK is installed before including the
generated file.

.. note::
The ``LLEXT_EDK`` prefix in the variable name may be changed with the
:kconfig:option:`CONFIG_LLEXT_EDK_NAME` option.

Compile flags
-------------

The EDK includes the convenience files ``cmake.cflags`` (for CMake-based
projects) and ``Makefile.cflags`` (for Make-based ones), which define a set of
variables that contain the compile flags needed by the project. The full list
of flags needed to build an extension is provided by ``LLEXT_CFLAGS``. Also
provided is a more granular set of flags that can be used in support of
different use cases, such as when building mocks for unit tests:
The full list of flags needed to build an extension is provided by
``LLEXT_CFLAGS``. Also provided is a more granular set of flags that can be
used in support of different use cases, such as when building mocks for unit
tests:

``LLEXT_INCLUDE_CFLAGS``

Expand Down Expand Up @@ -194,6 +214,40 @@ different use cases, such as when building mocks for unit tests:
``LLEXT_ALL_INCLUDE_CFLAGS``, ``LLEXT_GENERATED_IMACROS_CFLAGS`` and
``LLEXT_BASE_CFLAGS``.

Target and build information
----------------------------

The EDK includes information that identifies the target of the current Zephyr
build and the used toolchain. The following variables are currently defined,
which mirror the information available in the Zephyr build system:

``LLEXT_EDK_ARCH``
The architecture for the target used in the Zephyr build.

``LLEXT_EDK_BOARD``
The fully normalized board target used in the Zephyr build.

``LLEXT_EDK_SOC``
The SoC name for the target used in the Zephyr build.

``LLEXT_EDK_TOOLCHAIN``
The toolchain variant used in the Zephyr build.

``LLEXT_EDK_CROSS_COMPILE``
When defined by the toolchain, the cross-compile prefix for the compiler
used in the Zephyr build.

``LLEXT_EDK_COMPILER``
The name of the compiler used in the Zephyr build.

.. important::
The above variables exposie Zephyr internals. The list of variables and
their contents may change in future Zephyr releases.

.. note::
The ``LLEXT_EDK`` prefix in the variable names may be changed with the
:kconfig:option:`CONFIG_LLEXT_EDK_NAME` option.

.. _llext_kconfig_edk:

LLEXT EDK Kconfig options
Expand All @@ -202,7 +256,8 @@ LLEXT EDK Kconfig options
The LLEXT EDK can be configured using the following Kconfig options:

:kconfig:option:`CONFIG_LLEXT_EDK_NAME`
The name of the generated EDK tarball.
The name of the generated EDK tarball. This is also used as the prefix for
several variables defined in the EDK files.

:kconfig:option:`CONFIG_LLEXT_EDK_USERSPACE_ONLY`
If set, the EDK will include headers that do not contain code to route
Expand Down
Loading