ci: Add microblaze, nios2 compilers, use JSON for job summaries

[gastmaier]

PR Description

• Updates the container images with nios2 and microblaze compilers;
  use bootlin compilers for cross-compile.
• Replace fail, warn GitHub outputs with single JSON-formatted summary.
• For checkpatch, change strategy for "large-series", that is, squash the history, and check the changes, suppress the commit message errors.

The microblaze and nios2 generate an image with and "empty" devicetree, to be patched, for example, with:

org_repository=
cloudsmith_token=

_read_be32() {
        local bytes=$(dd if="$1" bs=1 skip=$2 count=4 2>/dev/null | xxd -p)
        echo $((16#$bytes))
}

_read_le32() {
        local bytes=$(dd if="$1" bs=1 skip=$2 count=4 2>/dev/null | xxd -p)
        echo $((16#${bytes:6:2}${bytes:4:2}${bytes:2:2}${bytes:0:2}))
}

_write_le32() {
        local value=$1
        # Suppress 'missing hex digit for \x'
        printf '\x%02x\x%02x\x%02x\x%02x' 2>/dev/null \
                $((value & 0xff)) \
                $(((value >> 8) & 0xff)) \
                $(((value >> 16) & 0xff)) \
                $(((value >> 24) & 0xff))
}

_unpack_dtb () {
        echo "  Unpack dtb"

        for path in "$name/dtb/arch"/*; do
                arch=$(echo "$path" | cut -d'/' -f4)
                [[ "$arch" == "microblaze" ]] && continue
                [[ "$arch" == "nios2" ]] && continue

                mkdir -p "dist/$arch/boot"
                cp -a "$path/boot/dts" "dist/$arch/boot"
        done
}
export -f _unpack_dtb

_unpack_kernel () {
        echo "  Unpack $kernel_defconfig"

        mkdir -p "$out/boot/kernel/$kernel_defconfig"
        cp -a "$name/boot/$kernel" "$out/boot/kernel/$kernel_defconfig"

        mkdir -p "$out/lib/modules_set/$kernel_defconfig"
        cp -a "$name/lib/modules/$kernel_release" "$out/lib/modules_set/$kernel_defconfig"
}
export -f _unpack_kernel

_microblaze_patch () {
        echo "patch microblaze"

        local image="dist/microblaze/boot/kernel/adi_mb_defconfig/simpleImage.generic.strip"
        local path="dtb-gcc/dtb/arch/microblaze/boot/dts"
        local dtb_start=$((16#84e5c8))
        local dtb_end=$((16#85f000))
        local dtb_length=$((dtb_end - dtb_start))

        [[ ! -d "$path" ]] && return
        [[ ! -f "$image" ]] && return

        dtb_magic=$(_read_be32 "$image" $dtb_start)
        [ $dtb_magic -ne $((0xd00dfeed)) ] && exit 1

        for file in "$path"/*; do
                filename=$(basename $file)
                filename="${filename%.*}"
                image_out="$(dirname $image)/simpleImage.$filename.strip"

                echo "  $filename"
                cp "$image" "$image_out"
                dd if=/dev/zero status=none \
                        of="$image_out" \
                        bs=1 seek=$dtb_start conv=notrunc count=$dtb_length
                dd if="$file" status=none  \
                        of="$image_out" \
                        bs=1 seek=$dtb_start conv=notrunc
        done

        command rm $image
}

_nios2_patch () {
        echo "patch nios2"

        local image="dist/nios2/boot/kernel/adi_nios2_defconfig/zImage"
        local path="dtb-gcc/dtb/arch/nios2/boot/dts"
        local dtb_start=$((16#543420))
        local vmlinux_gz_offset=$((16#4064))
        local input_len_offset=$((16#4060))

        [[ ! -d "$path" ]] && return
        [[ ! -f "$image" ]] && return

        local tmpdir=$(mktemp -d)
        local size=$(_read_le32 "$image" $input_len_offset)

        dd if="$image" status=none \
                of="$tmpdir/vmlinux.gz" \
                bs=1 skip=$vmlinux_gz_offset count=$size
        gunzip -c "$tmpdir/vmlinux.gz" > "$tmpdir/vmlinux.bin"

        dtb_magic=$(_read_be32 "$tmpdir/vmlinux.bin" $dtb_start)
        [ $dtb_magic -ne $((0xd00dfeed)) ] && return 1

        for file in "$path"/*; do
                filename=$(basename "${file%.*}")
                out_file="$(dirname "$image")/zImage.$filename"

                echo "  $filename"

                cp "$tmpdir/vmlinux.bin" "$tmpdir/vmlinux.0.bin"
                dd if="$file" status=none \
                        of="$tmpdir/vmlinux.0.bin" \
                        bs=1 seek=$dtb_start conv=notrunc status=none
                gzip -n -9 < "$tmpdir/vmlinux.0.bin" > "$tmpdir/vmlinux.0.gz"

                new_size=$(stat -c %s "$tmpdir/vmlinux.0.gz")
                cp "$image" "$out_file"
                _write_le32 $new_size | dd status=none \
                        of="$out_file" \
                        bs=1 seek=$input_len_offset conv=notrunc
                dd if="$tmpdir/vmlinux.0.gz" status=none \
                        of="$out_file" \
                        bs=1 seek=$vmlinux_gz_offset conv=notrunc
        done

        command rm -r "$tmpdir"
        command rm "$image"
}

_get_artifact () {
        local cloudsmith_token="$1"
        local url="$2"

        [[ "$url" == "null" ]] && return
        [[ "$url" =~ $git_sha/adi_ci_defconfig- ]] && return
        [[ "$url" = *-headers ]] && return

        name=$(basename "$url")
        local zip="$name.zip"
        echo "$name"

        local ret="200"
        unzip -t $zip  &>/dev/null && \
                echo "  Cached zip" || \
                ret=$(curl -sL \
                -H "X-Api-Key: $cloudsmith_token" \
                -w "%{http_code}\n" \
                -o $zip \
                $url)

        [[ "$ret" != "200" ]] && return 1

        [[ ! -d $name ]] && \
                unzip -q "$zip" -d $name || \
                echo "  Cached unzip"

        ctx="$name/context.txt"
        [[ ! -f "$ctx" ]] && return
        source "$ctx"
        out="dist/$compiler_arch"

        [[ "$name" == "dtb-gcc" ]] && _unpack_dtb || _unpack_kernel
}
export -f _get_artifact

cs_batch_by_version() {
        git_sha=${1-e8367b3a3994}
        git_sha=${git_sha:0:12}

        local tmpdir=$(mktemp -d)
        command rm -rf dist
        mkdir -p "dist"

        query="version:^$git_sha\$"
        echo "Fetch artifacts for $git_sha"
        ret=$(curl -sL \
                -w "%{http_code}\n" \
                -o "$tmpdir/.raw_urls" \
                -H "X-Api-Key: $cloudsmith_token" \
                "https://api.cloudsmith.io/v1/packages/$org_repository/?query=$query&sort=-date&page_size=100")
        [[ "$ret" != "200" ]] && return 1
        urls=$(cat "$tmpdir/.raw_urls" | jq -r '.[].cdn_url')

        [[ -z "$urls" ]] && { echo "No artifacts found." ; return 1 ; }

        echo "Got $(echo "$urls" | wc -l) urls"

        if command -v parallel &>/dev/null ; then
                echo "$urls" |
                        parallel --jobs 8 \
                                _get_artifact "$cloudsmith_token" {}
        else
                for url in $urls; do
                        _get_artifact $cloudsmith_token $url
                done
        fi

        _microblaze_patch
        _nios2_patch

        command rm -r $tmpdir

        echo "Done! See dist/"
}

PR Type

• Bug fix (a change that fixes an issue)
• New feature (a change that adds new functionality)
• Breaking change (a change that affects other repos or cause CIs to fail)

PR Checklist

• I have conducted a self-review of my own code changes
• I have compiled my changes, including the documentation
• I have tested the changes on the relevant hardware
• I have updated the documentation outside this repo accordingly
• I have provided links for the relevant upstream lore

[pamolloy]

Perhaps ci: Add microblaze and nios2 compiler -> ci: Add compilers for Xilinx and Altera soft-cores?

[pamolloy]

Typo: auxiliar

Perhaps rename to Download FPGA soft-core root filesystems and create empty devicetrees

[gastmaier]

I will go with Prepare kernel auxiliary files
it makes more sense to look into the build.sh for what it is doing, the yaml is a "shim layer" as much as possible

[pamolloy]

What are these empty devicetrees for?

[nunojsa]

I think it's because you need to provide it to build a microblaze simpleImage.

[gastmaier]

Yes. I don't want to upload n_devicestrees * n_defconfig anymore, where n_devicetress is 75 and n_defconfig is 1 (adi_mb_defconfig).

the devicetree is always from (and to) the same byte address and we can assert the magic hex string 0xd00dfeed.
The dtb does not depend on architecture.

The idea is a single dtb archive with all devicetres (3.2MB) and the defconfig archives (10 to 40MB)

then consumers (kuiper) expand, as in the provided script in the pr body.

A time cs_batch_by_version e8367b3a3994 takes 12 seconds, and we reduce the upload time for microblaze by 75 :)

About the other comments: I will rewrite the history to separate the changes better

[pamolloy]

Ideally this and the above change on line 1094 should be in the previous commit

[pamolloy]

Do the ADSP branches need to get updated?

[gastmaier]

It is not breaking change since the old pipe has been kept, but ideally yes so we can clean-up sooner than later.

[pamolloy]

😵

[nunojsa]

Some comments from me which might overlap a bit with what @pamolloy already pointed...

[nunojsa]

This looks a bit unrelated? Same for arm

Also given that I cannot comment on the commit message I'll say it here. You have a typo in "requiser"

[nunojsa]

I think it's because you need to provide it to build a microblaze simpleImage.

[nunojsa]

nit: This could be a precursor patch to add the helper

[nunojsa]

This also does not feel it belongs in this patch

[nunojsa]

I understand what you're doing but the proper way, IMO would be to move arm and aarch64 to bootlin in a different patch and then adding microblaze and nios would really be just about adding those archs