Merge pull request #7631 from MicroDev1/ci

CI: More fixes and refactoring

Author: tannewt

.github/actions/deps/ports/action.yml

@@ -0,0 +1,24 @@
+name: Fetch port deps
+
+inputs:
+  board:
+    required: true
+    type: string
+
+outputs:
+  port:
+    value: ${{ steps.board-to-port.outputs.port }}
+
+runs:
+  using: composite
+  steps:
+    - name: Board to port
+      id: board-to-port
+      run: |
+        PORT=$(find ports/*/boards/ -type d -name ${{ inputs.board }} | sed 's/^ports\///g;s/\/boards.*//g')
+        if [ -z $PORT ]; then (exit 1); else echo >> $GITHUB_OUTPUT "port=$PORT"; fi
+      shell: bash
+
+    - name: Set up espressif port
+      if: steps.board-to-port.outputs.port == 'espressif'
+      uses: ./.github/actions/deps/ports/espressif

.github/workflows/build-boards-custom.yml

@@ -51,20 +51,15 @@ jobs:
       if: inputs.debug || inputs.flags != ''
       run: |
         > custom-build && git add custom-build
-    - name: Get port
-      id: get-port
-      run: |
-        PORT=$(find ports/*/boards/ -type d -name ${{ inputs.board }} | sed 's/^ports\///g;s/\/boards.*//g')
-        if [ -z $PORT ]; then (exit 1); else echo >> $GITHUB_OUTPUT "port=$PORT"; fi
-    - name: Port to platform
-      run: echo >> $GITHUB_ENV "PLATFORM=${{ env[format('PLATFORM_{0}', steps.get-port.outputs.port)] }}"
     - name: Set up python
       uses: actions/setup-python@v4
       with:
         python-version: 3.x
     - name: Set up port
-      if: env.PLATFORM == 'esp'
-      uses: ./.github/actions/deps/ports/espressif
+      id: set-up-port
+      uses: ./.github/actions/deps/ports
+      with:
+        board: ${{ inputs.board }}
     - name: Set up submodules
       id: set-up-submodules
       uses: ./.github/actions/deps/submodules
@@ -75,7 +70,7 @@ jobs:
       uses: ./.github/actions/deps/external
       with:
         action: cache
-        platform: ${{ env.PLATFORM }}
+        platform: ${{ env[format('PLATFORM_{0}', steps.set-up-port.outputs.port)] }}
     - name: Set up mpy-cross
       if: steps.set-up-submodules.outputs.frozen == 'True'
       uses: ./.github/actions/mpy_cross
@@ -96,9 +91,9 @@ jobs:
         mkfs.fat --version || true
     - name: Build board
       run: make -j2 ${{ inputs.flags }} BOARD=${{ inputs.board }} DEBUG=${{ inputs.debug && '1' || '0' }} TRANSLATION=${{ inputs.language }}
-      working-directory: ports/${{ steps.get-port.outputs.port }}
+      working-directory: ports/${{ steps.set-up-port.outputs.port }}
     - name: Upload artifact
       uses: actions/upload-artifact@v3
       with:
         name: ${{ inputs.board }}-${{ inputs.language }}-${{ inputs.version }}${{ inputs.flags != '' && '-custom' || '' }}${{ inputs.debug && '-debug' || '' }}
-        path: ports/${{ steps.get-port.outputs.port }}/build-${{ inputs.board }}/firmware.*
+        path: ports/${{ steps.set-up-port.outputs.port }}/build-${{ inputs.board }}/firmware.*

.github/workflows/build-boards.yml

@@ -38,8 +38,9 @@ jobs:
       with:
         python-version: 3.x
     - name: Set up port
-      if: inputs.platform == 'esp'
-      uses: ./.github/actions/deps/ports/espressif
+      uses: ./.github/actions/deps/ports
+      with:
+        board: ${{ matrix.board }}
     - name: Set up submodules
       id: set-up-submodules
       uses: ./.github/actions/deps/submodules

.github/workflows/build.yml

@@ -22,11 +22,12 @@ jobs:
     outputs:
       build-doc: ${{ steps.set-matrix.outputs.build-doc }}
       build-boards: ${{ steps.set-matrix.outputs.build-boards }}
+      build-windows: ${{ steps.set-matrix.outputs.build-windows }}
       boards-aarch: ${{ steps.set-matrix.outputs.boards-aarch }}
       boards-arm: ${{ steps.set-matrix.outputs.boards-arm }}
+      boards-atmel: ${{ steps.set-matrix.outputs.boards-atmel }}
       boards-esp: ${{ steps.set-matrix.outputs.boards-esp }}
       boards-riscv: ${{ steps.set-matrix.outputs.boards-riscv }}
-      boards-rpi: ${{ steps.set-matrix.outputs.boards-rpi }}
       cp-version: ${{ steps.set-up-submodules.outputs.version }}
     steps:
     - name: Dump GitHub context
@@ -54,6 +55,10 @@ jobs:
       uses: ./.github/actions/deps/external
       with:
         action: cache
+    - name: Set up mpy-cross
+      uses: ./.github/actions/mpy_cross
+      with:
+        download: false
     - name: Get last commit with checks
       id: get-last-commit-with-checks
       if: github.event_name == 'pull_request'
@@ -63,14 +68,10 @@ jobs:
         REPO: ${{ github.repository }}
         PULL: ${{ github.event.number }}
         GITHUB_TOKEN: ${{ github.token }}
-        EXCLUDE_COMMIT: ${{ github.event.after }}
-    - name: Set up mpy-cross
-      uses: ./.github/actions/mpy_cross
-      with:
-        download: false
+        EXCLUDE_COMMIT: ${{ github.event.pull_request.head.sha }}
     - name: Set head sha
       if: github.event_name == 'pull_request'
-      run: echo "HEAD_SHA=$(git show -s --format=%s $GITHUB_SHA | grep -o -P "(?<=Merge ).*(?= into)")" >> $GITHUB_ENV
+      run: echo "HEAD_SHA=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV
     - name: Set base sha
       if: github.event_name == 'pull_request'
       run: |
@@ -218,6 +219,74 @@ jobs:
         [ -z "$TWINE_USERNAME" ] || echo "Uploading dev release to PyPi"
         [ -z "$TWINE_USERNAME" ] || twine upload circuitpython-stubs/dist/*
 
+  build-windows:
+    runs-on: windows-2022
+    needs: scheduler
+    if: ${{ needs.scheduler.outputs.build-windows == 'True' }}
+    env:
+      CP_VERSION: ${{ needs.scheduler.outputs.cp-version }}
+    defaults:
+      run:
+        # We define a custom shell script here, although `msys2.cmd` does neither exist nor is it available in the PATH yet
+        shell: msys2 {0}
+    steps:
+    # We want to change the configuration of the git command that actions/checkout will be using
+    # (since it is not possible to set autocrlf through the action yet, see actions/checkout#226).
+    - run: git config --global core.autocrlf input
+      shell: bash
+    - name: Check python coding (cmd)
+      run: python -c "import sys, locale; print(sys.getdefaultencoding(), locale.getpreferredencoding(False))"
+      shell: cmd
+    # We use a JS Action, which calls the system terminal or other custom terminals directly, if required
+    - uses: msys2/setup-msys2@v2
+      with:
+        install: base-devel git wget unzip gcc python-pip
+    # The goal of this was to test how things worked when the default file encoding (locale.getpreferedencoding())
+    # was not UTF-8. However, msys2 python does use utf-8 as the preferred file encoding, and using actions/setup-python
+    # python3.8 gave a broken build, so we're not really testing what we wanted to test.
+    # However, commandline length limits are being tested so that does some good.
+    - name: Check python coding (msys2)
+      run: |
+        locale -v
+        which python; python --version
+        python -c "import sys, locale; print(sys.getdefaultencoding(), locale.getpreferredencoding(False))"
+        which python3; python3 --version
+        python3 -c "import sys, locale; print(sys.getdefaultencoding(), locale.getpreferredencoding(False))"
+    - name: Install dependencies
+      run: |
+        wget --no-verbose -O gcc-arm.zip https://developer.arm.com/-/media/Files/downloads/gnu-rm/10-2020q4/gcc-arm-none-eabi-10-2020-q4-major-win32.zip
+        unzip -q -d /tmp gcc-arm.zip
+        tar -C /tmp/gcc-arm-none-* -cf - . | tar -C /usr/local -xf -
+        pip install wheel
+        # requirements_dev.txt doesn't install on windows. (with msys2 python)
+        # instead, pick a subset for what we want to do
+        pip install cascadetoml jinja2 typer click intelhex
+        # check that installed packages work....?
+        which python; python --version; python -c "import cascadetoml"
+        which python3; python3 --version; python3 -c "import cascadetoml"
+    - name: Set up repository
+      uses: actions/checkout@v3
+      with:
+        submodules: false
+        fetch-depth: 1
+    - name: Set up submodules
+      uses: ./.github/actions/deps/submodules
+    - name: build mpy-cross
+      run: make -j2 -C mpy-cross
+    - name: build rp2040
+      run: make -j2 -C ports/raspberrypi BOARD=adafruit_feather_rp2040 TRANSLATION=de_DE
+    - name: build samd21
+      run: make -j2 -C ports/atmel-samd BOARD=feather_m0_express TRANSLATION=zh_Latn_pinyin
+    - name: build samd51
+      run: make -j2 -C ports/atmel-samd BOARD=feather_m4_express TRANSLATION=es
+    - name: build nrf
+      run: make -j2 -C ports/nrf BOARD=feather_nrf52840_express TRANSLATION=fr
+    - name: build stm
+      run: make -j2 -C ports/stm BOARD=feather_stm32f405_express TRANSLATION=pt_BR
+    # I gave up trying to do esp builds on windows when I saw
+    # ERROR: Platform MINGW64_NT-10.0-17763-x86_64 appears to be unsupported
+    # https://github.com/espressif/esp-idf/issues/7062
+
   aarch:
     needs: [scheduler, mpy-cross, tests]
     if: ${{ needs.scheduler.outputs.boards-aarch != '[]' }}
@@ -238,6 +307,16 @@ jobs:
       boards: ${{ needs.scheduler.outputs.boards-arm }}
       cp-version: ${{ needs.scheduler.outputs.cp-version }}
 
+  atmel:
+    needs: [scheduler, mpy-cross, tests]
+    if: ${{ needs.scheduler.outputs.boards-atmel != '[]' }}
+    uses: ./.github/workflows/build-boards.yml
+    secrets: inherit
+    with:
+      platform: arm
+      boards: ${{ needs.scheduler.outputs.boards-atmel }}
+      cp-version: ${{ needs.scheduler.outputs.cp-version }}
+
   esp:
     needs: [scheduler, mpy-cross, tests]
     if: ${{ needs.scheduler.outputs.boards-esp != '[]' }}
@@ -257,13 +336,3 @@ jobs:
       platform: riscv
       boards: ${{ needs.scheduler.outputs.boards-riscv }}
       cp-version: ${{ needs.scheduler.outputs.cp-version }}
-
-  rpi:
-    needs: [scheduler, mpy-cross, tests]
-    if: ${{ needs.scheduler.outputs.boards-rpi != '[]' }}
-    uses: ./.github/workflows/build-boards.yml
-    secrets: inherit
-    with:
-      platform: arm
-      boards: ${{ needs.scheduler.outputs.boards-rpi }}
-      cp-version: ${{ needs.scheduler.outputs.cp-version }}

.github/workflows/ports_windows.yml

@@ -172,7 +172,7 @@ def get_bad_check_runs(query_check_runs):
 
     run_types = ["failed", "incomplete"]
 
-    regex_matrix = re.compile("^[^\n ]+ \/ (build|run) \([^\n ]+\)$")
+    regex_matrix = re.compile(r"^\S+ \/ (build|run) \(\S+\)$")
 
     while more_pages:
         check_runs = query_check_runs.fetch()["data"]["node"]

tools/ci_changes_per_commit.py

@@ -80,14 +80,16 @@ def main():
         submodules = ["extmod/ulab", "lib/", "tools/"]
     elif TARGET == "build-doc":
         # used in .readthedocs.yml to generate RTD
-        submodules = ["extmod/ulab", "frozen/"]
+        submodules = ["extmod/ulab"]
+        submodules_tags = ["frozen/"]
     elif TARGET == "mpy-cross" or TARGET == "mpy-cross-mac":
         submodules = ["tools/"]  # for huffman
-    elif TARGET == "windows":
+    elif TARGET == "build-windows":
         # This builds one board from a number of ports so fill out a bunch of submodules
         submodules = ["extmod/ulab", "lib/", "tools/", "ports/", "data/nvm.toml"]
     elif TARGET == "website":
-        submodules = ["tools/adabot/", "frozen/"]
+        submodules = ["tools/adabot/"]
+        submodules_tags = ["frozen/"]
     elif TARGET == "pre-commit":
         submodules = ["extmod/ulab"]
     else: