Merge pull request #6838 from uutils/renovate/vmactions-freebsd-vm-1.x #2755
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: GnuTests | |
| # spell-checker:ignore (abbrev/names) CodeCov gnulib GnuTests Swatinem | |
| # spell-checker:ignore (jargon) submodules | |
| # spell-checker:ignore (libs/utils) autopoint chksum gperf lcov libexpect pyinotify shopt texinfo valgrind libattr libcap taiki-e | |
| # spell-checker:ignore (options) Ccodegen Coverflow Cpanic Zpanic | |
| # spell-checker:ignore (people) Dawid Dziurla * dawidd dtolnay | |
| # spell-checker:ignore (vars) FILESET SUBDIRS XPASS | |
| # * note: to run a single test => `REPO/util/run-gnu-test.sh PATH/TO/TEST/SCRIPT` | |
| on: | |
| pull_request: | |
| push: | |
| branches: | |
| - main | |
| permissions: | |
| contents: read | |
| # End the current execution if there is a new changeset in the PR. | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} | |
| jobs: | |
| gnu: | |
| permissions: | |
| actions: read # for dawidd6/action-download-artifact to query and download artifacts | |
| contents: read # for actions/checkout to fetch code | |
| pull-requests: read # for dawidd6/action-download-artifact to query commit hash | |
| name: Run GNU tests | |
| runs-on: ubuntu-24.04 | |
| steps: | |
| - name: Initialize workflow variables | |
| id: vars | |
| shell: bash | |
| run: | | |
| ## VARs setup | |
| outputs() { step_id="${{ github.action }}"; for var in "$@" ; do echo steps.${step_id}.outputs.${var}="${!var}"; echo "${var}=${!var}" >> $GITHUB_OUTPUT; done; } | |
| # * config | |
| path_GNU="gnu" | |
| path_GNU_tests="${path_GNU}/tests" | |
| path_UUTILS="uutils" | |
| path_reference="reference" | |
| outputs path_GNU path_GNU_tests path_reference path_UUTILS | |
| # | |
| repo_default_branch="${{ github.event.repository.default_branch }}" | |
| repo_GNU_ref="v9.5" | |
| repo_reference_branch="${{ github.event.repository.default_branch }}" | |
| outputs repo_default_branch repo_GNU_ref repo_reference_branch | |
| # | |
| SUITE_LOG_FILE="${path_GNU_tests}/test-suite.log" | |
| ROOT_SUITE_LOG_FILE="${path_GNU_tests}/test-suite-root.log" | |
| TEST_LOGS_GLOB="${path_GNU_tests}/**/*.log" ## note: not usable at bash CLI; [why] double globstar not enabled by default b/c MacOS includes only bash v3 which doesn't have double globstar support | |
| TEST_FILESET_PREFIX='test-fileset-IDs.sha1#' | |
| TEST_FILESET_SUFFIX='.txt' | |
| TEST_SUMMARY_FILE='gnu-result.json' | |
| TEST_FULL_SUMMARY_FILE='gnu-full-result.json' | |
| outputs SUITE_LOG_FILE ROOT_SUITE_LOG_FILE TEST_FILESET_PREFIX TEST_FILESET_SUFFIX TEST_LOGS_GLOB TEST_SUMMARY_FILE TEST_FULL_SUMMARY_FILE | |
| - name: Checkout code (uutil) | |
| uses: actions/checkout@v4 | |
| with: | |
| path: '${{ steps.vars.outputs.path_UUTILS }}' | |
| - uses: dtolnay/rust-toolchain@master | |
| with: | |
| toolchain: stable | |
| components: rustfmt | |
| - uses: Swatinem/rust-cache@v2 | |
| with: | |
| workspaces: "./${{ steps.vars.outputs.path_UUTILS }} -> target" | |
| - name: Checkout code (GNU coreutils) | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: 'coreutils/coreutils' | |
| path: '${{ steps.vars.outputs.path_GNU }}' | |
| ref: ${{ steps.vars.outputs.repo_GNU_ref }} | |
| submodules: false | |
| - name: Override submodule URL and initialize submodules | |
| # Use github instead of upstream git server | |
| run: | | |
| git submodule sync --recursive | |
| git config submodule.gnulib.url https://github.com/coreutils/gnulib.git | |
| git submodule update --init --recursive --depth 1 | |
| working-directory: ${{ steps.vars.outputs.path_GNU }} | |
| - name: Retrieve reference artifacts | |
| uses: dawidd6/action-download-artifact@v6 | |
| # ref: <https://github.com/dawidd6/action-download-artifact> | |
| continue-on-error: true ## don't break the build for missing reference artifacts (may be expired or just not generated yet) | |
| with: | |
| workflow: GnuTests.yml | |
| branch: "${{ steps.vars.outputs.repo_reference_branch }}" | |
| # workflow_conclusion: success ## (default); * but, if commit with failed GnuTests is merged into the default branch, future commits will all show regression errors in GnuTests CI until o/w fixed | |
| workflow_conclusion: completed ## continually recalibrates to last commit of default branch with a successful GnuTests (ie, "self-heals" from GnuTest regressions, but needs more supervision for/of regressions) | |
| path: "${{ steps.vars.outputs.path_reference }}" | |
| - name: Install dependencies | |
| shell: bash | |
| run: | | |
| ## Install dependencies | |
| sudo apt-get update | |
| sudo apt-get install -y autoconf autopoint bison texinfo gperf gcc g++ gdb python3-pyinotify jq valgrind libexpect-perl libacl1-dev libattr1-dev libcap-dev libselinux1-dev attr | |
| - name: Add various locales | |
| shell: bash | |
| run: | | |
| ## Add various locales | |
| echo "Before:" | |
| locale -a | |
| ## Some tests fail with 'cannot change locale (en_US.ISO-8859-1): No such file or directory' | |
| ## Some others need a French locale | |
| sudo locale-gen | |
| sudo locale-gen --keep-existing fr_FR | |
| sudo locale-gen --keep-existing fr_FR.UTF-8 | |
| sudo locale-gen --keep-existing sv_SE | |
| sudo locale-gen --keep-existing sv_SE.UTF-8 | |
| sudo locale-gen --keep-existing en_US | |
| sudo locale-gen --keep-existing ru_RU.KOI8-R | |
| sudo update-locale | |
| echo "After:" | |
| locale -a | |
| - name: Build binaries | |
| shell: bash | |
| run: | | |
| ## Build binaries | |
| cd '${{ steps.vars.outputs.path_UUTILS }}' | |
| bash util/build-gnu.sh --release-build | |
| - name: Run GNU tests | |
| shell: bash | |
| run: | | |
| ## Run GNU tests | |
| path_GNU='${{ steps.vars.outputs.path_GNU }}' | |
| path_UUTILS='${{ steps.vars.outputs.path_UUTILS }}' | |
| bash "${path_UUTILS}/util/run-gnu-test.sh" | |
| - name: Run GNU root tests | |
| shell: bash | |
| run: | | |
| ## Run GNU root tests | |
| path_GNU='${{ steps.vars.outputs.path_GNU }}' | |
| path_UUTILS='${{ steps.vars.outputs.path_UUTILS }}' | |
| bash "${path_UUTILS}/util/run-gnu-test.sh" run-root | |
| - name: Extract testing info into JSON | |
| shell: bash | |
| run : | | |
| ## Extract testing info into JSON | |
| path_UUTILS='${{ steps.vars.outputs.path_UUTILS }}' | |
| python ${path_UUTILS}/util/gnu-json-result.py ${{ steps.vars.outputs.path_GNU_tests }} > ${{ steps.vars.outputs.TEST_FULL_SUMMARY_FILE }} | |
| - name: Extract/summarize testing info | |
| id: summary | |
| shell: bash | |
| run: | | |
| ## Extract/summarize testing info | |
| outputs() { step_id="${{ github.action }}"; for var in "$@" ; do echo steps.${step_id}.outputs.${var}="${!var}"; echo "${var}=${!var}" >> $GITHUB_OUTPUT; done; } | |
| # | |
| path_UUTILS='${{ steps.vars.outputs.path_UUTILS }}' | |
| # | |
| SUITE_LOG_FILE='${{ steps.vars.outputs.SUITE_LOG_FILE }}' | |
| ROOT_SUITE_LOG_FILE='${{ steps.vars.outputs.ROOT_SUITE_LOG_FILE }}' | |
| ls -al ${SUITE_LOG_FILE} ${ROOT_SUITE_LOG_FILE} | |
| if test -f "${SUITE_LOG_FILE}" | |
| then | |
| source ${path_UUTILS}/util/analyze-gnu-results.sh ${SUITE_LOG_FILE} ${ROOT_SUITE_LOG_FILE} | |
| if [[ "$TOTAL" -eq 0 || "$TOTAL" -eq 1 ]]; then | |
| echo "::error ::Failed to parse test results from '${SUITE_LOG_FILE}'; failing early" | |
| exit 1 | |
| fi | |
| output="GNU tests summary = TOTAL: $TOTAL / PASS: $PASS / FAIL: $FAIL / ERROR: $ERROR / SKIP: $SKIP" | |
| echo "${output}" | |
| if [[ "$FAIL" -gt 0 || "$ERROR" -gt 0 ]]; then echo "::warning ::${output}" ; fi | |
| jq -n \ | |
| --arg date "$(date --rfc-email)" \ | |
| --arg sha "$GITHUB_SHA" \ | |
| --arg total "$TOTAL" \ | |
| --arg pass "$PASS" \ | |
| --arg skip "$SKIP" \ | |
| --arg fail "$FAIL" \ | |
| --arg xpass "$XPASS" \ | |
| --arg error "$ERROR" \ | |
| '{($date): { sha: $sha, total: $total, pass: $pass, skip: $skip, fail: $fail, xpass: $xpass, error: $error, }}' > '${{ steps.vars.outputs.TEST_SUMMARY_FILE }}' | |
| HASH=$(sha1sum '${{ steps.vars.outputs.TEST_SUMMARY_FILE }}' | cut --delim=" " -f 1) | |
| outputs HASH | |
| else | |
| echo "::error ::Failed to find summary of test results (missing '${SUITE_LOG_FILE}'); failing early" | |
| exit 1 | |
| fi | |
| # Compress logs before upload (fails otherwise) | |
| gzip ${{ steps.vars.outputs.TEST_LOGS_GLOB }} | |
| - name: Reserve SHA1/ID of 'test-summary' | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: "${{ steps.summary.outputs.HASH }}" | |
| path: "${{ steps.vars.outputs.TEST_SUMMARY_FILE }}" | |
| - name: Reserve test results summary | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: test-summary | |
| path: "${{ steps.vars.outputs.TEST_SUMMARY_FILE }}" | |
| - name: Reserve test logs | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: test-logs | |
| path: "${{ steps.vars.outputs.TEST_LOGS_GLOB }}" | |
| - name: Upload full json results | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: gnu-full-result.json | |
| path: ${{ steps.vars.outputs.TEST_FULL_SUMMARY_FILE }} | |
| - name: Compare test failures VS reference | |
| shell: bash | |
| run: | | |
| ## Compare test failures VS reference | |
| have_new_failures="" | |
| REF_LOG_FILE='${{ steps.vars.outputs.path_reference }}/test-logs/test-suite.log' | |
| ROOT_REF_LOG_FILE='${{ steps.vars.outputs.path_reference }}/test-logs/test-suite-root.log' | |
| REF_SUMMARY_FILE='${{ steps.vars.outputs.path_reference }}/test-summary/gnu-result.json' | |
| REPO_DEFAULT_BRANCH='${{ steps.vars.outputs.repo_default_branch }}' | |
| path_UUTILS='${{ steps.vars.outputs.path_UUTILS }}' | |
| # https://github.com/uutils/coreutils/issues/4294 | |
| # https://github.com/uutils/coreutils/issues/4295 | |
| IGNORE_INTERMITTENT="${path_UUTILS}/.github/workflows/ignore-intermittent.txt" | |
| mkdir -p ${{ steps.vars.outputs.path_reference }} | |
| COMMENT_DIR="${{ steps.vars.outputs.path_reference }}/comment" | |
| mkdir -p ${COMMENT_DIR} | |
| echo ${{ github.event.number }} > ${COMMENT_DIR}/NR | |
| COMMENT_LOG="${COMMENT_DIR}/result.txt" | |
| # The comment log might be downloaded from a previous run | |
| # We only want the new changes, so remove it if it exists. | |
| rm -f ${COMMENT_LOG} | |
| touch ${COMMENT_LOG} | |
| compare_tests() { | |
| local new_log_file=$1 | |
| local ref_log_file=$2 | |
| local test_type=$3 # "standard" or "root" | |
| if test -f "${ref_log_file}"; then | |
| echo "Reference ${test_type} test log SHA1/ID: $(sha1sum -- "${ref_log_file}") - ${test_type}" | |
| REF_ERROR=$(sed -n "s/^ERROR: \([[:print:]]\+\).*/\1/p" "${ref_log_file}"| sort) | |
| CURRENT_RUN_ERROR=$(sed -n "s/^ERROR: \([[:print:]]\+\).*/\1/p" "${new_log_file}" | sort) | |
| REF_FAILING=$(sed -n "s/^FAIL: \([[:print:]]\+\).*/\1/p" "${ref_log_file}"| sort) | |
| CURRENT_RUN_FAILING=$(sed -n "s/^FAIL: \([[:print:]]\+\).*/\1/p" "${new_log_file}" | sort) | |
| echo "Detailled information:" | |
| echo "REF_ERROR = ${REF_ERROR}" | |
| echo "CURRENT_RUN_ERROR = ${CURRENT_RUN_ERROR}" | |
| echo "REF_FAILING = ${REF_FAILING}" | |
| echo "CURRENT_RUN_FAILING = ${CURRENT_RUN_FAILING}" | |
| # Compare failing and error tests | |
| for LINE in ${CURRENT_RUN_FAILING} | |
| do | |
| if ! grep -Fxq ${LINE}<<<"${REF_FAILING}" | |
| then | |
| if ! grep ${LINE} ${IGNORE_INTERMITTENT} | |
| then | |
| MSG="GNU test failed: ${LINE}. ${LINE} is passing on '${REPO_DEFAULT_BRANCH}'. Maybe you have to rebase?" | |
| echo "::error ::$MSG" | |
| echo $MSG >> ${COMMENT_LOG} | |
| have_new_failures="true" | |
| else | |
| MSG="Skip an intermittent issue ${LINE} (fails in this run but passes in the 'main' branch)" | |
| echo "::warning ::$MSG" | |
| echo $MSG >> ${COMMENT_LOG} | |
| echo "" | |
| fi | |
| fi | |
| done | |
| for LINE in ${REF_FAILING} | |
| do | |
| if ! grep -Fxq ${LINE}<<<"${CURRENT_RUN_FAILING}" | |
| then | |
| if ! grep ${LINE} ${IGNORE_INTERMITTENT} | |
| then | |
| MSG="Congrats! The gnu test ${LINE} is no longer failing!" | |
| echo "::warning ::$MSG" | |
| echo $MSG >> ${COMMENT_LOG} | |
| else | |
| MSG="Skipping an intermittent issue ${LINE} (passes in this run but fails in the 'main' branch)" | |
| echo "::warning ::$MSG" | |
| echo $MSG >> ${COMMENT_LOG} | |
| echo "" | |
| fi | |
| fi | |
| done | |
| for LINE in ${CURRENT_RUN_ERROR} | |
| do | |
| if ! grep -Fxq ${LINE}<<<"${REF_ERROR}" | |
| then | |
| MSG="GNU test error: ${LINE}. ${LINE} is passing on '${REPO_DEFAULT_BRANCH}'. Maybe you have to rebase?" | |
| echo "::error ::$MSG" | |
| echo $MSG >> ${COMMENT_LOG} | |
| have_new_failures="true" | |
| fi | |
| done | |
| for LINE in ${REF_ERROR} | |
| do | |
| if ! grep -Fxq ${LINE}<<<"${CURRENT_RUN_ERROR}" | |
| then | |
| MSG="Congrats! The gnu test ${LINE} is no longer ERROR!" | |
| echo "::warning ::$MSG" | |
| echo $MSG >> ${COMMENT_LOG} | |
| fi | |
| done | |
| else | |
| echo "::warning ::Skipping ${test_type} test failure comparison; no prior reference test logs are available." | |
| fi | |
| } | |
| # Compare standard tests | |
| compare_tests '${{ steps.vars.outputs.path_GNU_tests }}/test-suite.log' "${REF_LOG_FILE}" "standard" | |
| # Compare root tests | |
| compare_tests '${{ steps.vars.outputs.path_GNU_tests }}/test-suite-root.log' "${ROOT_REF_LOG_FILE}" "root" | |
| if test -n "${have_new_failures}" ; then exit -1 ; fi | |
| - name: Upload comparison log (for GnuComment workflow) | |
| if: success() || failure() # run regardless of prior step success/failure | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: comment | |
| path: ${{ steps.vars.outputs.path_reference }}/comment/ | |
| - name: Compare test summary VS reference | |
| if: success() || failure() # run regardless of prior step success/failure | |
| shell: bash | |
| run: | | |
| ## Compare test summary VS reference | |
| REF_SUMMARY_FILE='${{ steps.vars.outputs.path_reference }}/test-summary/gnu-result.json' | |
| if test -f "${REF_SUMMARY_FILE}"; then | |
| echo "Reference SHA1/ID: $(sha1sum -- "${REF_SUMMARY_FILE}")" | |
| mv "${REF_SUMMARY_FILE}" main-gnu-result.json | |
| python uutils/util/compare_gnu_result.py | |
| else | |
| echo "::warning ::Skipping test summary comparison; no prior reference summary is available." | |
| fi | |
| gnu_coverage: | |
| name: Run GNU tests with coverage | |
| runs-on: ubuntu-24.04 | |
| steps: | |
| - name: Checkout code uutil | |
| uses: actions/checkout@v4 | |
| with: | |
| path: 'uutils' | |
| - name: Checkout GNU coreutils | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: 'coreutils/coreutils' | |
| path: 'gnu' | |
| ref: 'v9.5' | |
| submodules: recursive | |
| - uses: dtolnay/rust-toolchain@master | |
| with: | |
| toolchain: nightly | |
| components: rustfmt | |
| - uses: taiki-e/install-action@grcov | |
| - uses: Swatinem/rust-cache@v2 | |
| with: | |
| workspaces: "./uutils -> target" | |
| - name: Install dependencies | |
| run: | | |
| ## Install dependencies | |
| sudo apt-get update | |
| sudo apt-get install -y autoconf autopoint bison texinfo gperf gcc g++ gdb python3-pyinotify jq valgrind libexpect-perl libacl1-dev libattr1-dev libcap-dev libselinux1-dev attr | |
| - name: Add various locales | |
| run: | | |
| ## Add various locales | |
| echo "Before:" | |
| locale -a | |
| ## Some tests fail with 'cannot change locale (en_US.ISO-8859-1): No such file or directory' | |
| ## Some others need a French locale | |
| sudo locale-gen | |
| sudo locale-gen --keep-existing fr_FR | |
| sudo locale-gen --keep-existing fr_FR.UTF-8 | |
| sudo update-locale | |
| echo "After:" | |
| locale -a | |
| - name: Build binaries | |
| env: | |
| RUSTFLAGS: "-Zprofile -Ccodegen-units=1 -Copt-level=0 -Clink-dead-code -Coverflow-checks=off -Zpanic_abort_tests -Cpanic=abort" | |
| RUSTDOCFLAGS: "-Cpanic=abort" | |
| run: | | |
| ## Build binaries | |
| cd uutils | |
| bash util/build-gnu.sh | |
| - name: Run GNU tests | |
| run: bash uutils/util/run-gnu-test.sh | |
| - name: Generate coverage data (via `grcov`) | |
| id: coverage | |
| run: | | |
| ## Generate coverage data | |
| cd uutils | |
| COVERAGE_REPORT_DIR="target/debug" | |
| COVERAGE_REPORT_FILE="${COVERAGE_REPORT_DIR}/lcov.info" | |
| mkdir -p "${COVERAGE_REPORT_DIR}" | |
| sudo chown -R "$(whoami)" "${COVERAGE_REPORT_DIR}" | |
| # display coverage files | |
| grcov . --output-type files --ignore build.rs --ignore "vendor/*" --ignore "/*" --ignore "[a-zA-Z]:/*" --excl-br-line "^\s*((debug_)?assert(_eq|_ne)?!|#\[derive\()" | sort --unique | |
| # generate coverage report | |
| grcov . --output-type lcov --output-path "${COVERAGE_REPORT_FILE}" --branch --ignore build.rs --ignore "vendor/*" --ignore "/*" --ignore "[a-zA-Z]:/*" --excl-br-line "^\s*((debug_)?assert(_eq|_ne)?!|#\[derive\()" | |
| echo "report=${COVERAGE_REPORT_FILE}" >> $GITHUB_OUTPUT | |
| - name: Upload coverage results (to Codecov.io) | |
| uses: codecov/codecov-action@v4 | |
| with: | |
| token: ${{ secrets.CODECOV_TOKEN }} | |
| file: ${{ steps.coverage.outputs.report }} | |
| flags: gnutests | |
| name: gnutests | |
| working-directory: uutils |