Skip to content

Commit bd7569f

Browse files
tejlmandnashif
authored andcommitted
cmake: Extracted Zephyr module processing into python script
Fixes: #14513 This commit move the functionality of extracting zephyr modules into generated CMake and Kconfig include files from CMake into python. This allows other tools, especially CI to re-use the zephyr module functionality. Signed-off-by: Torsten Rasmussen <torsten.rasmussen@nordicsemi.no>
1 parent dfd779c commit bd7569f

File tree

5 files changed

+194
-151
lines changed

5 files changed

+194
-151
lines changed

CMakeLists.txt

Lines changed: 10 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -575,16 +575,17 @@ add_subdirectory(ext)
575575
add_subdirectory(subsys)
576576
add_subdirectory(drivers)
577577

578-
# Add all zephyr modules subdirectories.
579-
if(ZEPHYR_MODULES_NAME)
580-
message("Including module(s): ${ZEPHYR_MODULES_NAME}")
578+
# Include zephyr modules generated CMake file.
579+
if(EXISTS ${CMAKE_BINARY_DIR}/zephyr_modules.txt)
580+
file(STRINGS ${CMAKE_BINARY_DIR}/zephyr_modules.txt ZEPHYR_MODULES_TXT)
581+
582+
foreach(module ${ZEPHYR_MODULES_TXT})
583+
string(REGEX REPLACE "(.*):.*" "\\1" module_name ${module})
584+
string(REGEX REPLACE ".*:(.*)" "\\1" module_path ${module})
585+
message("Including module: ${module_name}")
586+
add_subdirectory(${module_path} ${CMAKE_BINARY_DIR}/${module_name})
587+
endforeach()
581588
endif()
582-
set(index 0)
583-
foreach(module_name ${ZEPHYR_MODULES_NAME})
584-
list(GET ZEPHYR_MODULES_DIR ${index} module_dir)
585-
math(EXPR index "${index} + 1")
586-
add_subdirectory(${module_dir} ${CMAKE_BINARY_DIR}/${module_name})
587-
endforeach()
588589

589590
set(syscall_macros_h ${ZEPHYR_BINARY_DIR}/include/generated/syscall_macros.h)
590591

CODEOWNERS

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -253,6 +253,7 @@
253253
/scripts/series-push-hook.sh @erwango
254254
/scripts/west_commands/ @mbolivar
255255
/scripts/west-commands.yml @mbolivar
256+
/scripts/zephyr_module.py @tejlmand
256257
/subsys/bluetooth/ @sjanc @jhedberg @Vudentz
257258
/subsys/bluetooth/controller/ @carlescufi @cvinayak @thoh-ot
258259
/subsys/fs/ @nashif

cmake/zephyr_module.cmake

Lines changed: 19 additions & 75 deletions
Original file line numberDiff line numberDiff line change
@@ -9,86 +9,30 @@
99
# search for zephyr/module.yml
1010

1111
if(ZEPHYR_MODULES)
12-
set(west_project_list ${ZEPHYR_MODULES})
13-
elseif(WEST)
14-
## Use `west list` to fetch all west handled projects.
15-
execute_process(
16-
COMMAND
17-
${WEST} list --format={posixpath}
18-
OUTPUT_VARIABLE
19-
west_project_output
20-
ERROR_VARIABLE
21-
west_project_error
22-
RESULT_VARIABLE
23-
west_list_result
24-
)
25-
if(NOT ${west_list_result})
26-
string(REGEX REPLACE "[\r\n]+" ";" west_project_list "${west_project_output}")
27-
elseif(NOT ("${west_project_error}" MATCHES
28-
"^Error: .* is not in a west installation\..*"))
29-
message(FATAL_ERROR "${west_project_error}")
30-
endif()
12+
set(ZEPHYR_MODULES_ARG "--modules" ${ZEPHYR_MODULES})
3113
endif()
3214

3315
if(ZEPHYR_EXTRA_MODULES)
34-
list(APPEND west_project_list ${ZEPHYR_EXTRA_MODULES})
16+
set(ZEPHYR_EXTRA_MODULES_ARG "--extra-modules" ${ZEPHYR_EXTRA_MODULES})
3517
endif()
3618

37-
# Clear the Kconfig.modules generated file in case modules has been removed.
38-
# The Kconfig.modules contains a list of additional Kconfig files to be sourced
39-
# based upon <module>/zephyr/module.yml files.
40-
set(KCONFIG_MODULES_FILE ${CMAKE_BINARY_DIR}/Kconfig.modules)
41-
file(WRITE ${KCONFIG_MODULES_FILE}
42-
"# NOTE: THIS FILE IS AUTOGENERATED BY CMAKE\n"
19+
if(WEST OR ZEPHYR_MODULES)
20+
# Zephyr module uses west, so only call it if west is installed or
21+
# ZEPHYR_MODULES was provided as argument to CMake.
22+
execute_process(
23+
COMMAND
24+
${PYTHON_EXECUTABLE} ${ZEPHYR_BASE}/scripts/zephyr_module.py
25+
${ZEPHYR_MODULES_ARG}
26+
${ZEPHYR_EXTRA_MODULES_ARG}
27+
--kconfig-out ${CMAKE_BINARY_DIR}/Kconfig.modules
28+
--cmake-out ${CMAKE_BINARY_DIR}/zephyr_modules.txt
29+
ERROR_VARIABLE
30+
zephyr_module_error_text
31+
RESULT_VARIABLE
32+
zephyr_module_return
4333
)
44-
# For each west managed project, determine if the project is a zephyr module.
45-
foreach(module ${west_project_list})
46-
set(cmake_subdir "zephyr")
47-
if(${ZEPHYR_BASE} STREQUAL ${module})
48-
# Ignore Zephyr project to avoid potential invalid looping
49-
elseif(EXISTS "${module}/zephyr/module.yml")
50-
set(kconfig_osource "osource \"${module}/zephyr/Kconfig\"\n")
51-
get_filename_component(module_name ${module} NAME)
52-
execute_process(
53-
COMMAND
54-
${PYTHON_EXECUTABLE}
55-
${ZEPHYR_BASE}/scripts/yaml_to_cmake.py
56-
-i "${module}/zephyr/module.yml"
57-
-o "${CMAKE_CURRENT_BINARY_DIR}/zephyr_module_${module_name}.txt"
58-
-s "build"
59-
)
60-
file(
61-
STRINGS
62-
"${CMAKE_CURRENT_BINARY_DIR}/zephyr_module_${module_name}.txt"
63-
zephyr_module
64-
)
65-
66-
foreach(key_value ${zephyr_module})
67-
if(${key_value} MATCHES "^cmake=")
68-
string(REGEX REPLACE "^cmake=" "" cmake_subdir ${key_value})
69-
elseif(${key_value} MATCHES "^kconfig=")
70-
string(
71-
REGEX REPLACE
72-
"^kconfig="
73-
"osource \"${module}/"
74-
kconfig_osource
75-
${key_value} "\"\n"
76-
)
77-
endif()
78-
endforeach()
7934

80-
list(APPEND ZEPHYR_MODULES_NAME ${module_name})
81-
list(APPEND ZEPHYR_MODULES_DIR ${module}/${cmake_subdir})
82-
file(APPEND ${KCONFIG_MODULES_FILE} ${kconfig_osource})
83-
elseif(EXISTS "${module}/${cmake_subdir}/CMakeLists.txt")
84-
set(kconfig_osource "osource \"${module}/zephyr/Kconfig\"\n")
85-
get_filename_component(module_name ${module} NAME)
86-
list(APPEND ZEPHYR_MODULES_NAME ${module_name})
87-
list(APPEND ZEPHYR_MODULES_DIR ${module}/${cmake_subdir})
88-
file(APPEND ${KCONFIG_MODULES_FILE} ${kconfig_osource})
89-
else()
90-
# Not a Zephyr module, ignore.
35+
if(${zephyr_module_return})
36+
message(FATAL_ERROR "${zephyr_module_error_text}")
9137
endif()
92-
93-
endforeach()
94-
38+
endif()

scripts/yaml_to_cmake.py

Lines changed: 0 additions & 67 deletions
This file was deleted.

scripts/zephyr_module.py

Lines changed: 164 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,164 @@
1+
#!/usr/bin/env python3
2+
#
3+
# Copyright (c) 2019, Nordic Semiconductor ASA
4+
#
5+
# SPDX-License-Identifier: Apache-2.0
6+
7+
'''Tool for parsing a list of projects to determine if they are Zephyr
8+
projects. If no projects are given then the output from `west list` will be
9+
used as project list.
10+
11+
Include file is generated for Kconfig using --kconfig-out.
12+
A <name>:<path> text file is generated for use with CMake using --cmake-out.
13+
'''
14+
15+
import argparse
16+
import os
17+
import sys
18+
import yaml
19+
import pykwalify.core
20+
import subprocess
21+
import re
22+
23+
24+
METADATA_SCHEMA = '''
25+
## A pykwalify schema for basic validation of the structure of a
26+
## metadata YAML file.
27+
##
28+
# The zephyr/module.yml file is a simple list of key value pairs to be used by
29+
# the build system.
30+
type: map
31+
mapping:
32+
build:
33+
required: true
34+
type: map
35+
mapping:
36+
cmake:
37+
required: false
38+
type: str
39+
kconfig:
40+
required: false
41+
type: str
42+
'''
43+
44+
schema = yaml.safe_load(METADATA_SCHEMA)
45+
46+
47+
def validate_setting(setting, module_path, filename=None):
48+
if setting is not None:
49+
if filename is not None:
50+
checkfile = os.path.join(module_path, setting, filename)
51+
else:
52+
checkfile = os.path.join(module_path, setting)
53+
if not os.path.isfile(checkfile):
54+
return False
55+
return True
56+
57+
58+
def process_module(module, cmake_out=None, kconfig_out=None):
59+
cmake_setting = None
60+
kconfig_setting = None
61+
62+
module_yml = os.path.join(module, 'zephyr/module.yml')
63+
if os.path.isfile(module_yml):
64+
with open(module_yml, 'r') as f:
65+
meta = yaml.safe_load(f.read())
66+
67+
try:
68+
pykwalify.core.Core(source_data=meta, schema_data=schema)\
69+
.validate()
70+
except pykwalify.errors.SchemaError as e:
71+
print('ERROR: Malformed "build" section in file: {}\n{}'
72+
.format(module_yml, e), file=sys.stderr)
73+
sys.exit(1)
74+
75+
section = meta.get('build', dict())
76+
cmake_setting = section.get('cmake', None)
77+
if not validate_setting(cmake_setting, module, 'CMakeLists.txt'):
78+
print('ERROR: "cmake" key in {} has folder value "{}" which '
79+
'does not contain a CMakeLists.txt file.'
80+
.format(module_yml, cmake_setting), file=sys.stderr)
81+
sys.exit(1)
82+
83+
kconfig_setting = section.get('kconfig', None)
84+
if not validate_setting(kconfig_setting, module):
85+
print('ERROR: "kconfig" key in {} has value "{}" which does not '
86+
'point to a valid Kconfig file.'
87+
.format(module_yml, kconfig_setting), file=sys.stderr)
88+
sys.exit(1)
89+
90+
cmake_path = os.path.join(module, cmake_setting or 'zephyr')
91+
cmake_file = os.path.join(cmake_path, 'CMakeLists.txt')
92+
if os.path.isfile(cmake_file) and cmake_out is not None:
93+
cmake_out.write('{}:{}\n'.format(os.path.basename(module),
94+
os.path.abspath(cmake_path)))
95+
96+
kconfig_file = os.path.join(module, kconfig_setting or 'zephyr/Kconfig')
97+
if os.path.isfile(kconfig_file) and kconfig_out is not None:
98+
kconfig_out.write('osource "{}"\n\n'
99+
.format(os.path.abspath(kconfig_file)))
100+
101+
102+
def main():
103+
kconfig_out_file = None
104+
cmake_out_file = None
105+
106+
parser = argparse.ArgumentParser(description='''
107+
Process a list of projects and create Kconfig / CMake include files for
108+
projects which are also a Zephyr module''')
109+
110+
parser.add_argument('--kconfig-out',
111+
help='File to write with resulting KConfig import'
112+
'statements.')
113+
parser.add_argument('--cmake-out',
114+
help='File to write with resulting <name>:<path>'
115+
'values to use for including in CMake')
116+
parser.add_argument('-m', '--modules', nargs='+',
117+
help='List of modules to parse instead of using `west'
118+
'list`')
119+
parser.add_argument('-x', '--extra-modules', nargs='+',
120+
help='List of extra modules to parse')
121+
args = parser.parse_args()
122+
123+
if args.modules is None:
124+
p = subprocess.Popen(['west', 'list', '--format={posixpath}'],
125+
stdout=subprocess.PIPE,
126+
stderr=subprocess.PIPE)
127+
out, err = p.communicate()
128+
if p.returncode == 0:
129+
projects = out.decode(sys.getdefaultencoding()).splitlines()
130+
elif re.match(r'Error: .* is not in a west installation\..*',
131+
err.decode(sys.getdefaultencoding())):
132+
# Only accept the error from bootstrapper in the event we are
133+
# outside a west managed project.
134+
projects = []
135+
else:
136+
# A real error occurred, raise an exception
137+
raise subprocess.CalledProcessError(cmd=p.args,
138+
returncode=p.returncode)
139+
else:
140+
projects = args.modules
141+
142+
if args.extra_modules is not None:
143+
projects += args.extra_modules
144+
145+
if args.kconfig_out:
146+
kconfig_out_file = open(args.kconfig_out, 'w')
147+
148+
if args.cmake_out:
149+
cmake_out_file = open(args.cmake_out, 'w')
150+
151+
try:
152+
for project in projects:
153+
# Avoid including Zephyr base project as module.
154+
if project != os.environ.get('ZEPHYR_BASE'):
155+
process_module(project, cmake_out_file, kconfig_out_file)
156+
finally:
157+
if args.kconfig_out:
158+
kconfig_out_file.close()
159+
if args.cmake_out:
160+
cmake_out_file.close()
161+
162+
163+
if __name__ == "__main__":
164+
main()

0 commit comments

Comments
 (0)