wip

Author: pillo79

.github/workflows/package_core.yml

@@ -66,7 +66,7 @@ jobs:
       - build-env
     env:
       CCACHE_IGNOREOPTIONS: -specs=*
-      OUTPUT_ARTIFACT: binaries-${{ matrix.board }}-${{ needs.build-env.outputs.CORE_HASH }}
+      ARTIFACT_TAG: ${{ needs.build-env.outputs.CORE_HASH }}-${{ matrix.board }}
     strategy:
       matrix:
         include:
@@ -90,7 +90,7 @@ jobs:
       - name: Build loader
         shell: bash
         run: |
-          if ! ./extra/build.sh ${{ matrix.board }} 2> >(tee error.log) ; then
+          if ! ./extra/build.sh ${{ matrix.board }} 1> >(tee output.log) 2> >(tee error.log) ; then
             echo "### :x: ${{ matrix.board }} (\`${{ matrix.variant }}\`) build errors" > $GITHUB_STEP_SUMMARY
             echo >> $GITHUB_STEP_SUMMARY
             echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
@@ -99,21 +99,35 @@ jobs:
             exit 1
           fi
 
+          # extract the memory usage table (from the header to the first non-% line)
+          cat output.log | sed -n '/^Memory region/,/^[^%]*$/p' | head -n -1 \
+                         | awk 'BEGIN {split("B KB MB GB", u); for(i in u) m[u[i]]=1024^(i-1)} /:/ {print "[\"" $1 "\"," $2*m[$3] "," $4*m[$5] "]"}' \
+                         | sort | jq -s > firmwares/zephyr-${{ matrix.variant }}.meminfo
+
       - name: Package board binaries
         if: ${{ !cancelled() }}
         run: |
           tar chf - \
             firmwares/*${{ matrix.variant }}* \
             variants/${{ matrix.variant }}/ \
             ${{ (job.status == 'failure') && format('build/{0}/', matrix.variant) }} \
-          | zstd > ${OUTPUT_ARTIFACT}.tar.zstd
+          | zstd > binaries-${ARTIFACT_TAG}.tar.zstd
 
       - name: Archive board binaries
         if: ${{ !cancelled() }}
         uses: actions/upload-artifact@v4
         with:
-          name: ${{ format('{0}{1}', (job.status == 'failure') && 'failed-' || '', env.OUTPUT_ARTIFACT) }}
-          path: ${{ env.OUTPUT_ARTIFACT }}.tar.zstd
+          name: ${{ format('{0}binaries-{1}', (job.status == 'failure') && 'failed-' || '', env.ARTIFACT_TAG) }}
+          path: binaries-${{ env.ARTIFACT_TAG }}.tar.zstd
+
+      - name: Archive board memory report
+        if: ${{ !cancelled() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: mem-report-${{ env.ARTIFACT_TAG }}
+          path: |
+            firmwares/zephyr-${{ matrix.variant }}.meminfo
+            firmwares/zephyr-${{ matrix.variant }}.config
 
   package-core:
     name: Package ${{ matrix.artifact }}
@@ -122,6 +136,7 @@ jobs:
       - build-env
       - build-board
     env:
+      ALL_BOARD_DATA: ${{ needs.build-env.outputs.ALL_BOARD_DATA }}
       CORE_ARTIFACT: ArduinoCore-${{ matrix.artifact }}-${{ needs.build-env.outputs.CORE_HASH }}
       CORE_TAG: ${{ needs.build-env.outputs.CORE_TAG }}
     strategy:
@@ -168,7 +183,6 @@ jobs:
     runs-on: ubuntu-latest
     needs:
       - package-core
-    if: always()
     steps:
       - uses: geekyeggo/delete-artifact@v5.1.0
         with:
@@ -193,8 +207,17 @@ jobs:
       PLAT: arduino:${{ matrix.subarch }}
       FQBN: arduino:${{ matrix.subarch }}:${{ matrix.board }}
       CORE_ARTIFACT: ArduinoCore-${{ matrix.artifact }}-${{ needs.build-env.outputs.CORE_HASH }}
+      ARTIFACT_TAG: ${{ needs.build-env.outputs.CORE_HASH }}-${{ matrix.board }}
     if: ${{ !cancelled() && needs.build-env.result == 'success' }}
     steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          persist-credentials: false
+          sparse-checkout: |
+            extra/ci_test_list.sh
+            extra/artifacts/
+
       - uses: actions/download-artifact@v4
         with:
           name: ${{ env.CORE_ARTIFACT }}
@@ -204,31 +227,33 @@ jobs:
           tar xf ${CORE_ARTIFACT}.tar.bz2 # will create ArduinoCore-zephyr/
           echo "REPORT_FILE=$(echo ${FQBN} | tr ':' '-').json" >> $GITHUB_ENV
 
-      - name: Create Blink sketch
+      - name: Get test sketches
         run: |
-          mkdir Blink/
-          wget -nv https://raw.githubusercontent.com/arduino/arduino-examples/refs/heads/main/examples/01.Basics/Blink/Blink.ino -P Blink/
+          # sets ALL_TESTS and ALL_LIBRARIES env vars
+          extra/ci_test_list.sh ${{ matrix.artifact }} ${{ matrix.variant }}
 
-      - name: Compile Blink for ${{ matrix.board }}
-        uses: pillo79/compile-sketches@main
+      - name: Compile tests for ${{ matrix.board }}
+        uses: pillo79/compile-sketches@next
         with:
           fqbn: ${{ env.FQBN }}
           platforms: |
-            # Use Board Manager to install the latest release of Arduino Zephyr Boards to get the toolchain
-            - name: "arduino:zephyr"
-              source-url: "https://downloads.arduino.cc/packages/package_zephyr_index.json"
+            # Use Board Manager version first, to install the toolchain
+            - name: ${{ env.PLAT }}
             - name: ${{ env.PLAT }}
               source-path: "ArduinoCore-zephyr"
-          sketch-paths: Blink
+          sketch-paths: |
+            ${{ env.ALL_TESTS }}
+          libraries: |
+            ${{ env.ALL_LIBRARIES }}
           cli-compile-flags: |
             - '--build-property'
             - 'compiler.c.extra_flags=-Wno-type-limits -Wno-missing-field-initializers'
             - '--build-property'
             - 'compiler.cpp.extra_flags=-Wno-type-limits -Wno-missing-field-initializers'
           verbose: 'false'
           enable-deltas-report: 'false'
-          enable-warnings-report: 'true'
-          enable-warnings-log: 'true'
+          enable-issues-report: 'true'
+          always-succeed: 'true'
 
       - name: Get job ID
         id: job_id
@@ -247,17 +272,18 @@ jobs:
       - name: Prepare log
         if: ${{ success() || failure() }}
         run: |
+          [ ! -f sketches-reports/${REPORT_FILE} ] && mkdir -p sketches-reports && echo "{}" > sketches-reports/${REPORT_FILE}
           sed -i -e 's!/home/runner/.arduino15/packages/arduino/hardware/zephyr/[^/]*/!!g' sketches-reports/${REPORT_FILE}
-          cat sketches-reports/${REPORT_FILE} | jq -cr ".boards[0].sketches[0] += { job_id: ${{ steps.job_id.outputs.result }} }" > ${REPORT_FILE} && mv ${REPORT_FILE} sketches-reports/
+          cat sketches-reports/${REPORT_FILE} | jq -cr ". += { job_id: ${{ steps.job_id.outputs.result }} }" > ${REPORT_FILE} && mv ${REPORT_FILE} sketches-reports/
 
       - uses: actions/upload-artifact@v4
         if: ${{ success() || failure() }}
         with:
-          name: test-report-${{ needs.build-env.outputs.CORE_TAG }}-${{ matrix.board }}
+          name: test-report-${{ env.ARTIFACT_TAG }}
           path: sketches-reports/*
 
-  collect-logs:
-    name: Collect logs
+  inspect-logs:
+    name: Analyze logs
     runs-on: ubuntu-latest
     needs:
       - build-env
@@ -267,66 +293,38 @@ jobs:
     env:
       ALL_BOARD_DATA: ${{ needs.build-env.outputs.ALL_BOARD_DATA }}
     steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          persist-credentials: false
+
       - uses: actions/download-artifact@v4
         with:
           path: .
-          pattern: test-report-*
+          pattern: "*-report-*"
           merge-multiple: true
 
-      - run: |
-          ARTIFACTS=$(jq -cr 'map(.artifact) | unique | .[]' <<< ${ALL_BOARD_DATA}) # this avoids the 'zephyr' artifact
-          for artifact in $ARTIFACTS ; do
-            echo "### \`$artifact\` test results:" >> "$GITHUB_STEP_SUMMARY"
-            jq -c "map(select(.artifact == \"$artifact\")) | .[]" <<< ${ALL_BOARD_DATA} | while read -r BOARD_DATA; do
-              BOARD=$(echo $BOARD_DATA | jq -cr '.board')
-              VARIANT=$(echo $BOARD_DATA | jq -cr '.variant')
-              SUBARCH=$(echo $BOARD_DATA | jq -cr '.subarch')
-              FQBN="arduino:$SUBARCH:$BOARD"
-              REPORT_FILE="$(echo $FQBN | tr ':' '-').json"
-              if [ ! -f $REPORT_FILE ]; then
-                echo "* :x: $BOARD (\`$VARIANT\`) - No report found?" >> "$GITHUB_STEP_SUMMARY"
-              else
-                REPORT=$(jq -cr '.boards[0].sketches[0]' $REPORT_FILE)
-                JOB_ID=$(echo $REPORT | jq -cr '.job_id')
-                JOB_URL="https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}/job/${JOB_ID}#step:5:2"
-                if ! $(echo $REPORT | jq -cr '.compilation_success') ; then
-                  echo "* :x: [$BOARD]($JOB_URL) (\`$VARIANT\`) - Build failed" >> "$GITHUB_STEP_SUMMARY"
-                else
-                  WARNINGS=$(echo $REPORT | jq -cr '.warnings.current.absolute // 0')
-                  if [ $WARNINGS -eq 0 ]; then
-                    echo "* :white_check_mark: $BOARD (\`$VARIANT\`) - Build successful" >> "$GITHUB_STEP_SUMMARY"
-                  else
-                    echo >> "$GITHUB_STEP_SUMMARY"
-                    echo "<details><summary>&nbsp;&nbsp; :warning: <a href=\"$JOB_URL\">$BOARD</a> (<tt>$VARIANT</tt>) - $WARNINGS warnings:</summary>" >> "$GITHUB_STEP_SUMMARY"
-                    echo >> "$GITHUB_STEP_SUMMARY"
-                    echo "\`\`\`" >> "$GITHUB_STEP_SUMMARY"
-                    echo $REPORT | jq -cr '.warnings_log[]' >> "$GITHUB_STEP_SUMMARY"
-                    echo "\`\`\`" >> "$GITHUB_STEP_SUMMARY"
-                    echo >> "$GITHUB_STEP_SUMMARY"
-                    echo "</details>" >> "$GITHUB_STEP_SUMMARY"
-                    echo >> "$GITHUB_STEP_SUMMARY"
-                  fi
-                fi
-              fi
-            done
-          done
+      # Collect and summarize logs, will fail the step if any test failed
+      - run: extra/ci_inspect_logs.py > $GITHUB_STEP_SUMMARY
 
       - name: Clean up intermediate artifacts
         uses: geekyeggo/delete-artifact@v5.1.0
         with:
-          name: test-report-*
+          name: |
+            mem-report-*
+            test-report-*
           failOnError: false
 
   verify-core:
     runs-on: ubuntu-latest
-    if: cancelled() || contains(needs.*.result, 'failure')
     needs:
       - build-env
       - package-core
-      - test-core
+      - inspect-logs
+    if: ${{ !cancelled() }}
     steps:
-      - name: Notify failure
-        run: exit 1
+      - name: CI run result
+        run: exit ${{ contains(needs.*.result, 'failure') && '1' || '0' }}
 
   publish-core:
     name: Publish core
@@ -335,7 +333,7 @@ jobs:
     needs:
       - build-env
       - package-core
-      - test-core
+      - inspect-logs
     environment: production
     permissions:
       id-token: write
@@ -365,7 +363,7 @@ jobs:
     needs:
       - build-env
       - package-core
-      - test-core
+      - inspect-logs
     env:
       CORE_TAG: ${{ needs.build-env.outputs.CORE_TAG }}
       CORE_HASH: ${{ needs.build-env.outputs.CORE_HASH }}

extra/artifacts/_common.test_setup.sh

@@ -0,0 +1,28 @@
+# This script is sourced from extra/ci_test_list.sh to provide
+# artifact-specific tests for Zephyr CI tests.
+#
+# Two helper functions are provided for easy GitHub queries:
+# - get_branch_tip <repo> <branch> [<path> ...]
+# - get_latest_release <repo> [<path> ...]
+#
+# By default, the whole project will be added to the test suite.
+# When given additional path arguments, the functions will only
+# copy artifacts under the provided paths.
+
+if [ "$ARTIFACT" == "zephyr_contrib" ] ; then
+	# Minimal safety test for Zephyr contrib boards
+	get_branch_tip examples arduino/arduino-examples main \
+		examples/01.Basics/Blink
+else
+	# Get a few core Arduino examples
+	get_branch_tip examples arduino/arduino-examples main \
+		examples/01.Basics/Blink \
+		examples/01.Basics/AnalogReadSerial \
+		examples/04.Communication/SerialPassthrough \
+
+	# Smoke test for C++ features
+	get_latest_release libraries arduino-libraries/Arduino_JSON \
+
+	# Smoke test for SPI API compatibilty
+	# get_branch_tip libraries PaulStoffregen/SerialFlash master
+fi

extra/artifacts/zephyr_contrib.exc

@@ -0,0 +1,3 @@
+libraries/Camera/
+libraries/Storage/
+libraries/Zephyr_SDRAM/

extra/artifacts/zephyr_main.test_setup.sh

@@ -0,0 +1,20 @@
+# This script is sourced from extra/ci_test_list.sh to provide
+# artifact-specific tests for Zephyr CI tests.
+#
+# Two helper functions are provided for easy GitHub queries:
+# - get_branch_tip <repo> <branch> [<path> ...]
+# - get_latest_release <repo> [<path> ...]
+#
+# By default, the whole project will be added to the test suite.
+# When given additional path arguments, the functions will only
+# copy artifacts under the provided paths.
+
+# ArduinoBLE
+get_branch_tip libraries arduino-libraries/ArduinoBLE master \
+	examples/Central/LedControl \
+	examples/Central/Scan \
+	examples/Peripheral/Advertising/EnhancedAdvertising \
+	examples/Peripheral/ButtonLED \
+
+# Arduino_SecureElement
+get_latest_release libraries arduino-libraries/Arduino_SecureElement

extra/artifacts/zephyr_unoq.exc

@@ -0,0 +1,5 @@
+libraries/Camera/
+libraries/Ethernet/
+libraries/Storage/
+libraries/WiFi/
+libraries/Zephyr_SDRAM/

extra/artifacts/zephyr_unoq.only

@@ -0,0 +1,3 @@
+libraries/Arduino_LED_Matrix/
+libraries/Arduino_RouterBridge/
+libraries/Arduino_RPClite/

extra/artifacts/zephyr_unoq.test_setup.sh

@@ -0,0 +1,13 @@
+# This script is sourced from extra/ci_test_list.sh to provide
+# artifact-specific tests for Zephyr CI tests.
+#
+# Two helper functions are provided for easy GitHub queries:
+# - get_branch_tip <repo> <branch> [<path> ...]
+# - get_latest_release <repo> [<path> ...]
+#
+# By default, the whole project will be added to the test suite.
+# When given additional path arguments, the functions will only
+# copy artifacts under the provided paths.
+
+# ArduinoBLE
+get_branch_tip libraries arduino-libraries/ArduinoBLE master

extra/build.sh

@@ -73,6 +73,7 @@ fi
 BUILD_DIR=build/${variant}
 VARIANT_DIR=variants/${variant}
 rm -rf ${BUILD_DIR}
+west spdx --init -d ${BUILD_DIR}
 west build -d ${BUILD_DIR} -b ${target} loader -t llext-edk ${args}
 
 # Extract the generated EDK tarball and copy it to the variant directory