Introduce component groups - let's see if this speeds up CI

tests/scripts/all.sh

@@ -130,6 +130,230 @@ pre_check_environment () {
     fi
 }
 
+pre_define_component_groups () {
+
+    # group00: (very) short list, for testing
+    #group00=(check_test_dependencies)
+    group00=(check_test_dependencies check_files check_names check_test_helpers)
+    group00+=(build_module_alt check_doxy_blocks check_recursion build_tfm)
+
+    # Groups created by running components individually and sorting by time taken.
+    # Some quick components may be missing, as not all components could be run on the system used.
+
+    # group01: 44 components, 291 secs
+    group01=(check_doxy_blocks check_recursion build_dhm_alt check_test_dependencies)
+    group01+=(build_module_alt check_test_helpers build_arm_none_eabi_gcc_arm5vte build_arm_none_eabi_gcc_m0plus)
+    group01+=(check_test_cases build_arm_none_eabi_gcc build_arm_none_eabi_gcc_no_64bit_multiplication)
+    group01+=(build_no_sockets build_arm_none_eabi_gcc_no_udbl_division build_psa_crypto_spm)
+    group01+=(build_psa_alt_headers check_names build_zeroize_checks build_tfm)
+    group01+=(build_no_pk_rsa_alt_support check_doxygen_warnings)
+    group01+=(check_files build_arm_clang_thumb build_crypto_full)
+    group01+=(build_no_ssl_cli build_crypto_default)
+    group01+=(build_psa_config_file build_no_ssl_srv test_ccm_aes_sha256)
+    group01+=(build_mbedtls_config_file build_aes_variations)
+    group01+=(check_changelog)
+    group01+=(build_no_std_function build_psa_accel_alg_sha512 build_psa_accel_key_type_rsa_public_key)
+
+    # group02: 19 components, 293 secs
+    group02=(build_psa_accel_alg_rsa_oaep build_psa_accel_alg_rsa_pss build_psa_accel_alg_sha224)
+    group02+=(build_psa_accel_alg_rsa_pkcs1v15_sign build_psa_accel_alg_md5 build_psa_accel_alg_hkdf)
+    group02+=(build_psa_accel_alg_ripemd160 build_psa_accel_alg_rsa_pkcs1v15_crypt test_full_no_cipher_no_psa_crypto)
+    group02+=(build_psa_accel_alg_sha1 build_psa_accel_key_type_rsa_key_pair build_psa_accel_alg_ecdh)
+    group02+=(build_psa_accel_alg_hmac build_psa_accel_alg_sha256 build_psa_accel_alg_sha384)
+    group02+=(check_generated_files check_python_files build_aes_via_padlock)
+
+    # group03: 10 components, 294 secs
+    group03=(test_crypto_for_psa_service test_aes_only_128_bit_keys)
+    group03+=(test_aes_fewer_tables_and_rom_tables test_aes_only_128_bit_keys_have_builtins)
+    group03+=(test_aes_fewer_tables test_default_no_deprecated)
+    group03+=(test_aes_rom_tables)
+
+    # group04: 6 components, 283 secs
+    group04=(test_have_int64 test_full_no_cipher_with_psa_crypto test_full_block_cipher_legacy_dispatch)
+    group04+=(test_full_no_cipher_with_psa_crypto_config)
+    # group06: 3 components, 237 secs - add into group 4
+    group04+=(test_full_no_deprecated test_crypto_full_md_light_only test_full_no_deprecated_deprecated_warning)
+
+    # group05: 4 components, 285 secs
+    group05=(test_full_no_ccm_star_no_tag test_full_no_bignum test_full_deprecated_warning)
+
+    # group09: these get run on FreeBSD
+    group09=(test_default_out_of_box test_cmake_shared test_clang_opt)
+
+    # The next groups generated by some CI runs, so less precise measurements
+    group10=(test_tfm_config test_tls1_2_ecjpake_compatibility test_tls13_only_psk)
+    group10+=(test_tls13_only_psk_ephemeral test_tfm_config_p256m_driver_accel_ec)
+    group10+=(test_tls13_only_psk_ephemeral_ffdh test_tls1_2_default_stream_cipher_only_use_psa)
+    group10+=(test_tls13_only_psk_all test_make_shared)
+
+    group10+=(test_no_date_time test_when_no_ciphersuites_have_mac test_psa_crypto_rsa_no_genprime)
+    group10+=(test_psa_crypto_client test_tls13_only_ephemeral_ffdh)
+
+    group11=(test_block_cipher_no_decrypt_aesni test_ctr_drbg_aes_128_sha_512)
+
+    group12=(test_small_ssl_dtls_max_buffering test_se_default test_new_psa_want_key_pair_symbol)
+    group12+=(test_tls1_2_default_cbc_legacy_cbc_etm_cipher_only)
+
+    group12+=(test_tls1_2_default_stream_cipher_only test_small_ssl_in_content_len test_small_ssl_out_content_len)
+    group12+=(test_tls1_2_default_cbc_legacy_cipher_only)
+
+    group13=(test_no_ctr_drbg_aes_only_128_bit_keys test_have_int32 test_small_mbedtls_ssl_dtls_max_buffering)
+    group13+=(test_full_no_ccm test_sw_inet_pton)
+
+    group14=(test_psa_ecc_key_pair_no_derive test_psa_external_rng_no_drbg_classic)
+    group14+=(test_psa_external_rng_use_psa_crypto)
+
+    group15=(test_ssl_alloc_buffer_and_mfl test_no_max_fragment_length test_psa_crypto_config_reference_rsa_crypto)
+    group15+=(test_tls1_2_deafult_cbc_legacy_cipher_only_use_psa)
+
+    group15+=(test_tls1_2_default_cbc_legacy_cbc_etm_cipher_only_use_psa)
+    group15+=(test_no_max_fragment_length_small_ssl_out_content_len)
+
+    group17=(test_everest test_everest_curve25519_only)
+
+    group18=(test_no_64bit_multiplication test_no_platform test_no_strings)
+
+    group19=(test_psa_collect_statuses test_platform_calloc_macro)
+
+    group20=(test_no_udbl_division test_min_mpi_window_size)
+
+    group21=(test_full_block_cipher_psa_dispatch test_psa_crypto_config_reference_ffdh)
+
+    group22=(test_psa_crypto_config_accel_ecc_non_weierstrass_curves test_full_cmake_clang)
+    group22+=(test_psa_crypto_config_accel_rsa_crypto)
+
+    group23=(test_psa_crypto_config_accel_hash_keep_builtins test_memory_buffer_allocator_backtrace)
+
+    group24=(test_zeroize test_psa_crypto_config_accel_hash)
+
+    group25=(test_aead_only_ccm test_psa_crypto_config_accel_ecc_weierstrass_curves)
+
+    group26=(test_have_int32_cmake_new_bignum test_psa_crypto_config_accel_ecdsa)
+
+    group27=(test_psa_ecc_key_pair_no_generate test_psa_crypto_config_accel_ecdh)
+
+    group28=(test_psa_crypto_drivers test_aead_chachapoly_disabled)
+
+    group29=(test_ctr_drbg_aes_128_sha_256 test_ctr_drbg_aes_256_sha_256)
+
+    group30=(test_psa_inject_entropy test_rsa_no_crt)
+
+    group31=(test_psa_crypto_config_reference_ecc_no_bignum test_psa_crypto_config_accel_ecc_some_key_types)
+
+    group32=(test_no_hmac_drbg_classic test_no_ctr_drbg_classic)
+
+    group33=(test_psa_crypto_config_accel_aead test_no_hmac_drbg_use_psa)
+
+    group34=(test_psa_crypto_config_reference_ecc_ffdh_no_bignum test_psa_crypto_config_accel_des)
+
+    group35=(test_psa_crypto_config_accel_pake test_no_ctr_drbg_use_psa)
+
+    group36=(test_depends_py_cipher_padding test_memsan_constant_flow_psa)
+
+    group37=(test_psa_crypto_config_accel_ffdh test_memsan_constant_flow)
+
+    group38=(test_depends_py_cipher_padding_psa test_psa_external_rng_no_drbg_use_psa)
+
+    # Only these groups are offered by --list-components and --list-all-components
+    groups=(group01 group02 group03 group04 group05)
+    groups+=(group10 group11 group12 group13 group14)
+    groups+=(group15 group17 group18 group19)
+    groups+=(group20 group21 group22 group23 group24)
+    groups+=(group25 group26 group27 group28 group29)
+    groups+=(group30 group31 group32 group33 group34)
+    groups+=(group35 group36 group37 group38)
+
+    VALID_GROUPS=""     # String version of the list of groups used for validation of command line
+    for group in "${groups[@]}"
+    do
+        VALID_GROUPS="$VALID_GROUPS $group"
+    done
+    VALID_GROUPS="$VALID_GROUPS group00"       # XXX for local testing temporarily
+}
+
+output_components_and_groups () {
+
+    components_str="$@"
+    components=($components_str)        # convert to an array
+
+    # List of slowest 20 or so components, slowest first
+    slowest=(test_full_cmake_gcc_asan_new_bignum
+             test_ref_configs
+             test_m32_o2
+             test_full_cmake_gcc_asan
+             test_psa_crypto_config_accel_ecc_no_ecp_at_all
+             test_psa_crypto_config_accel_ecc_ecp_light_only
+             test_psa_crypto_config_accel_hash_use_psa
+             test_psa_crypto_config_accel_cipher_aead_cmac
+             test_depends_py_kex
+             test_depends_py_curves_psa
+             test_psa_crypto_config_reference_ecc_no_ecp_at_all
+             test_no_use_psa_crypto_full_cmake_asan
+             test_malloc_0_null
+             test_depends_py_curves
+             test_no_x509_info
+             test_psa_crypto_config_reference_ecc_ecp_light_only
+             test_psa_crypto_config_reference_hash_use_psa
+             test_depends_py_kex_psa
+             test_psa_crypto_config_reference_cipher_aead_cmac
+             test_tls13
+             test_depends_py_hashes_psa
+             full_without_ecdhe_ecdsa
+             test_depends_py_hashes
+             test_default_cmake_gcc_asan)
+
+    first=()
+
+    for component in "${slowest[@]}"
+    do
+        # Remove this component - to do this, we need to check each element,
+        # since something like all=( "${all[@]/$c}" ) removes matching prefixes,
+        # and as some components are prefixes of others, this doesn't work for us.
+        for i in "${!components[@]}"; do
+            if [[ ${components[i]} = $component ]]; then
+                unset 'components[i]'
+                first+=($component)
+            fi
+        done
+    done
+
+    for group in "${groups[@]}"
+    do
+        # look at the components in each group
+        ref="$group[@]"
+        for component in "${!ref}"
+        do
+            # Remove this component - to do this, we need to check each element,
+            # since something like all=( "${all[@]/$c}" ) removes matching prefixes,
+            # and as some components are prefixes of others, this doesn't work for us.
+            for i in "${!components[@]}"; do
+                if [[ ${components[i]} = $component ]]; then
+                    unset 'components[i]'
+                fi
+            done
+        done
+    done
+
+    # Output the slowest first, then groups, then the remaining components
+    COMPONENTS_AND_GROUPS=""
+    for component in "${first[@]}"
+    do
+        COMPONENTS_AND_GROUPS="$COMPONENTS_AND_GROUPS $component"
+    done
+    for group in "${groups[@]}"
+    do
+        COMPONENTS_AND_GROUPS="$COMPONENTS_AND_GROUPS $group"
+    done
+    for component in "${components[@]}"
+    do
+        if [[ -n "$component" ]]; then
+            COMPONENTS_AND_GROUPS="$COMPONENTS_AND_GROUPS $component"
+        fi
+    done
+
+    printf '%s\n' $COMPONENTS_AND_GROUPS
+}
+
 pre_initialize_variables () {
     if in_mbedtls_repo; then
         CONFIG_H='include/mbedtls/mbedtls_config.h'
@@ -188,8 +412,9 @@ pre_initialize_variables () {
     : ${GCC_EARLIEST:="gcc-earliest"}
     # if MAKEFLAGS is not set add the -j option to speed up invocations of make
     if [ -z "${MAKEFLAGS+set}" ]; then
-        export MAKEFLAGS="-j$(all_sh_nproc)"
+        export MAKEFLAGS=""     # "-j$(all_sh_nproc)"
     fi
+    echo "MAKEFLAGS is $MAKEFLAGS"
     # if CC is not set, use clang by default (if present) to improve build times
     if [ -z "${CC+set}" ] && (type clang > /dev/null 2>&1); then
         export CC="clang"
@@ -217,7 +442,7 @@ pre_initialize_variables () {
 }
 
 # Test whether the component $1 is included in the command line patterns.
-is_component_included()
+is_component_listed_on_command_line()
 {
     # Temporarily disable wildcard expansion so that $COMMAND_LINE_COMPONENTS
     # only does word splitting.
@@ -230,6 +455,29 @@ is_component_included()
     return 1
 }
 
+# Expand any groups given on the command line. Updates COMMAND_LINE_COMPONENTS.
+expand_command_line_groups()
+{
+    # Temporarily disable wildcard expansion so that $COMMAND_LINE_COMPONENTS
+    # only does word splitting.
+    set -f
+    UPDATED=""
+    for word in $COMMAND_LINE_COMPONENTS; do
+        case $word in
+            group??) ref="$word[@]"
+                    for component in "${!ref}"
+                    do
+                        UPDATED="$UPDATED $component"
+                    done ;;
+
+            *)      UPDATED="$UPDATED $word";;
+        esac
+    done
+    set +f
+    COMMAND_LINE_COMPONENTS="$UPDATED"
+    unset UPDATED
+}
+
 usage()
 {
     cat <<EOF
@@ -427,6 +675,7 @@ pre_parse_command_line () {
     COMMAND_LINE_COMPONENTS=
     all_except=0
     error_test=0
+    list_all_components=0
     list_components=0
     restore_first=0
     no_armcc=
@@ -455,7 +704,7 @@ pre_parse_command_line () {
             --gnutls-serv) shift; GNUTLS_SERV="$1";;
             --help|-h) usage; exit;;
             --keep-going|-k) KEEP_GOING=1;;
-            --list-all-components) printf '%s\n' $ALL_COMPONENTS; exit;;
+            --list-all-components) output_components_and_groups $ALL_COMPONENTS; exit;;
             --list-components) list_components=1;;
             --memory|-m) MEMORY=1;;
             --no-append-outcome) append_outcome=0;;
@@ -494,7 +743,7 @@ pre_parse_command_line () {
     done
 
     if [ $list_components -eq 1 ]; then
-        printf '%s\n' $SUPPORTED_COMPONENTS
+        output_components_and_groups $SUPPORTED_COMPONENTS
         exit
     fi
 
@@ -522,6 +771,9 @@ pre_parse_command_line () {
             case $component in
                 *[*?\[]*) continue;;
             esac
+            case " $VALID_GROUPS " in
+                *" $component "*) continue;;
+            esac
             case " $SUPPORTED_COMPONENTS " in
                 *" $component "*) :;;
                 *)
@@ -535,10 +787,12 @@ pre_parse_command_line () {
         fi
     fi
 
+    expand_command_line_groups
+
     # Build the list of components to run.
     RUN_COMPONENTS=
     for component in $SUPPORTED_COMPONENTS; do
-        if is_component_included "$component"; [ $? -eq $all_except ]; then
+        if is_component_listed_on_command_line "$component"; [ $? -eq $all_except ]; then
             RUN_COMPONENTS="$RUN_COMPONENTS $component"
         fi
     done
@@ -6349,6 +6603,8 @@ run_component () {
 }
 
 # Preliminary setup
+pre_define_component_groups
+
 pre_check_environment
 pre_initialize_variables
 pre_parse_command_line "$@"