all.sh: consistently generate files before checks, and not before builds

scripts/all-core.sh

@@ -365,6 +365,9 @@ EOF
 # Does not remove generated source files.
 cleanup()
 {
+    # In Mbed TLS, we do in-tree builds, so clean up.
+    # In TF-PSA-Crypto, in-tree builds are technically possible but discouraged
+    # and not done in all.sh so we don't clean up.
     if in_mbedtls_repo; then
         command make clean
     fi
@@ -874,17 +877,6 @@ pre_check_tools () {
     "$@" "${BASH_SOURCE%/*}"/output_env.sh
 }
 
-pre_generate_files() {
-    # since make doesn't have proper dependencies, remove any possibly outdate
-    # file that might be around before generating fresh ones
-    make neat
-    if [ $QUIET -eq 1 ]; then
-        make generated_files >/dev/null
-    else
-        make generated_files
-    fi
-}
-
 pre_load_helpers () {
     # Use a path relative to the currently-sourced file.
     test_script_dir="${BASH_SOURCE%/*}"
@@ -951,6 +943,32 @@ run_component () {
         pre_create_tf_psa_crypto_out_of_source_directory
     fi
 
+    case ${current_component#component_} in
+        # - Many check_xxx components expect generated files to be present,
+        #   and may silently check less if they aren't.
+        # - Build components that cross-compile would fail when trying to
+        #   generate files during the build if they can't an executable
+        #   produced by ${CC}. To keep things simple, pre-generate the files
+        #   for all build-only components, even the ones that don't require it.
+        build_*|tf_psa_crypto_build*|\
+        check_*|tf_psa_crypto_check*)
+            if in_mbedtls_repo; then
+                make generated_files
+            else
+                $FRAMEWORK/scripts/make_generated_files.py
+            fi;;
+        *)
+            # Other build or build-and-test components are supposed to work
+            # whether generated files are already present or not.
+            # Test with the generated files absent, since if this works,
+            # it's likely to work with generated files present as well.
+            if in_mbedtls_repo; then
+                make neat
+            else
+                $FRAMEWORK/scripts/make_generated_files.py --clean
+            fi
+    esac
+
     # Run the component in a subshell, with error trapping and output
     # redirection set up based on the relevant options.
     if [ $KEEP_GOING -eq 1 ]; then
@@ -1021,9 +1039,6 @@ main () {
     pre_print_configuration
     pre_check_tools
     cleanup
-    if in_mbedtls_repo; then
-        pre_generate_files
-    fi
 
     # Run the requested tests.
     for ((error_test_i=1; error_test_i <= error_test; error_test_i++)); do

scripts/all-helpers.sh

@@ -103,6 +103,17 @@ helper_libtestdriver1_adjust_config() {
 # 2. optional: a space-separate list of things to also support.
 # Here "things" are PSA_WANT_ symbols but with PSA_WANT_ removed.
 helper_libtestdriver1_make_drivers() {
+    # Ensure that the configuration-independent platform-independent generated
+    # files are present in the source tree. We specifically need the ones
+    # involved in building the crypto library, because the libtestdriver1
+    # rewritten library/Makefile can't build them (it doesn't have all
+    # paths properly rewritten for that).
+    if in_mbedtls_repo; then
+        make -C library generated_files
+    else
+        $FRAMEWORK/scripts/make_generated_files.py
+    fi
+
     loc_accel_flags=$( echo "$1 ${2-}" | sed 's/[^ ]* */-DLIBTESTDRIVER1_MBEDTLS_PSA_ACCEL_&/g' )
     make CC=$ASAN_CC -C tests libtestdriver1.a CFLAGS=" $ASAN_CFLAGS $loc_accel_flags" LDFLAGS="$ASAN_CFLAGS"
 }

scripts/make_generated_files.py

@@ -9,7 +9,9 @@
 Generate the TF-PSA-Crypto generated files
 """
 import argparse
+import enum
 import filecmp
+import os
 import shutil
 import subprocess
 import sys
@@ -219,21 +221,39 @@ def check_generated_files(generation_scripts: List[GenerationScript], root: Path
             file.unlink()
             bak_file.rename(file)
 
+
+class Action(enum.Enum):
+    CHECK = 0
+    CLEAN = 1
+    GENERATE = 2
+    LIST = 3
+
 def main():
     """
     Main function of this program
     """
     parser = argparse.ArgumentParser()
-
-    parser.add_argument('--list', action='store_true',
-                        default=False, help='List generated files.')
     parser.add_argument('--root', metavar='DIR',
                         help='Root of the tree containing the generated files \
-                              to check (default: Mbed TLS or TF-PSA-Cryto root.)')
-    parser.add_argument('--check', action='store_true',
-                        default=False, help='Check the generated files in root')
+                              (default: Mbed TLS or TF-PSA-Cryto root.)')
+
+    action = parser.add_mutually_exclusive_group()
+    action.add_argument('--check',
+                        dest='action', action='store_const', const=Action.CHECK,
+                        help='Check that the generated files are up to date.')
+    action.add_argument('--clean',
+                        dest='action', action='store_const', const=Action.CLEAN,
+                        help='Remove all generated files')
+    action.add_argument('--generate',
+                        dest='action', action='store_const', const=Action.GENERATE,
+                        help='Generate target-independent file (default mode)')
+    action.add_argument('--list',
+                        dest='action', action='store_const', const=Action.LIST,
+                        help='List generated files and exit')
 
     args = parser.parse_args()
+    if not args.action:
+        args.action = Action.GENERATE
 
     if not build_tree.looks_like_root("."):
         raise RuntimeError("This script must be run from Mbed TLS or TF-PSA-Crypto root.")
@@ -245,13 +265,19 @@ def main():
     else:
         raise Exception("No support for Mbed TLS 3.6")
 
-    if args.list:
+    if args.action == Action.LIST:
         files = get_generated_files(generation_scripts)
         for file in files:
             print(str(file))
-    elif args.check:
+    elif args.action == Action.CLEAN:
+        files = get_generated_files(generation_scripts)
+        for file in files:
+            if os.path.exists(file):
+                os.remove(file)
+    elif args.action == Action.CHECK:
         check_generated_files(generation_scripts, Path(args.root or "."))
     else:
+        assert args.action == Action.GENERATE
         make_generated_files(generation_scripts)
 
 if __name__ == "__main__":