Skip to content

Commit 790ef1e

Browse files
oSoMoNoSoMoN
authored
Run the GNU test suite in CI (fixes #8) (#13)
1 parent 4c1a752 commit 790ef1e

File tree

3 files changed

+196
-0
lines changed

3 files changed

+196
-0
lines changed

‎.github/workflows/ci.yml‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,23 @@ jobs:
4949
- run: rustup component add clippy
5050
- run: cargo clippy -- -D warnings
5151

52+
gnu-testsuite:
53+
name: GNU test suite
54+
runs-on: ubuntu-latest
55+
steps:
56+
- uses: actions/checkout@v4
57+
- uses: dtolnay/rust-toolchain@stable
58+
- run: cargo build --release
59+
# do not fail, the report is merely informative (at least until all tests pass reliably)
60+
- run: ./tests/run-upstream-testsuite.sh release || true
61+
env:
62+
TERM: xterm
63+
- uses: actions/upload-artifact@v4
64+
with:
65+
name: test-results.json
66+
path: tests/test-results.json
67+
- run: ./tests/print-test-results.sh tests/test-results.json
68+
5269
coverage:
5370
name: Code Coverage
5471
runs-on: ${{ matrix.job.os }}

‎tests/print-test-results.sh‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
#!/bin/bash
2+
3+
# Print the test results written to a JSON file by run-upstream-testsuite.sh
4+
# in a markdown format. The printout includes the name of the test, the result,
5+
# the URL to the test script and the contents of stdout and stderr.
6+
# It can be used verbatim as the description when filing an issue for a test
7+
# with an unexpected result.
8+
9+
json="test-results.json"
10+
[[ -n $1 ]] && json="$1"
11+
12+
codeblock () { echo -e "\`\`\`\n$1\n\`\`\`"; }
13+
14+
jq -c '.tests[]' "$json" | while read -r test
15+
do
16+
name=$(echo "$test" | jq -r '.test')
17+
echo "# test: $name"
18+
result=$(echo "$test" | jq -r '.result')
19+
echo "result: $result"
20+
url=$(echo "$test" | jq -r '.url')
21+
echo "url: $url"
22+
if [[ "$result" != "SKIP" ]]
23+
then
24+
stdout=$(echo "$test" | jq -r '.stdout' | base64 -d)
25+
if [[ -n "$stdout" ]]
26+
then
27+
echo "## stdout"
28+
codeblock "$stdout"
29+
fi
30+
stderr=$(echo "$test" | jq -r '.stderr' | base64 -d)
31+
if [[ -n "$stderr" ]]
32+
then
33+
echo "## stderr"
34+
codeblock "$stderr"
35+
fi
36+
fi
37+
echo ""
38+
done

‎tests/run-upstream-testsuite.sh‎

Lines changed: 141 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,141 @@
1+
#!/bin/bash
2+
3+
# Run the GNU upstream test suite for diffutils against a local build of the
4+
# Rust implementation, print out a summary of the test results, and writes a
5+
# JSON file ('test-results.json') containing detailed information about the
6+
# test run.
7+
8+
# The JSON file contains metadata about the test run, and for each test the
9+
# result as well as the contents of stdout, stderr, and of all the files
10+
# written by the test script, if any (excluding subdirectories).
11+
12+
# The script takes a shortcut to fetch only the test suite from the upstream
13+
# repository and carefully avoids running the autotools machinery which is
14+
# time-consuming and resource-intensive, and doesn't offer the option to not
15+
# build the upstream binaries. As a consequence, the environment in which the
16+
# tests are run might not match exactly that used when the upstream tests are
17+
# run through the autotools.
18+
19+
# By default it expects a release build of the diffutils binary, but a
20+
# different build profile can be specified as an argument
21+
# (e.g. 'dev' or 'test').
22+
# Unless overriden by the $TESTS environment variable, all tests in the test
23+
# suite will be run. Tests targeting a command that is not yet implemented
24+
# (e.g. cmp, diff3 or sdiff) are skipped.
25+
26+
scriptpath=$(dirname "$(readlink -f "$0")")
27+
rev=$(git rev-parse HEAD)
28+
29+
# Allow passing a specific profile as parameter (default to "release")
30+
profile="release"
31+
[[ -n $1 ]] && profile="$1"
32+
33+
# Verify that the diffutils binary was built for the requested profile
34+
binary="$scriptpath/../target/$profile/diffutils"
35+
if [[ ! -x "$binary" ]]
36+
then
37+
echo "Missing build for profile $profile"
38+
exit 1
39+
fi
40+
41+
# Work in a temporary directory
42+
tempdir=$(mktemp -d)
43+
cd "$tempdir"
44+
45+
# Check out the upstream test suite
46+
gitserver="https://git.savannah.gnu.org"
47+
testsuite="$gitserver/git/diffutils.git"
48+
echo "Fetching upstream test suite from $testsuite"
49+
git clone -n --depth=1 --filter=tree:0 "$testsuite" &> /dev/null
50+
cd diffutils
51+
git sparse-checkout set --no-cone tests &> /dev/null
52+
git checkout &> /dev/null
53+
upstreamrev=$(git rev-parse HEAD)
54+
55+
# Ensure that calling `diff` invokes the built `diffutils` binary instead of
56+
# the upstream `diff` binary that is most likely installed on the system
57+
mkdir src
58+
cd src
59+
ln -s "$binary" diff
60+
cd ../tests
61+
62+
if [[ -n "$TESTS" ]]
63+
then
64+
tests="$TESTS"
65+
else
66+
# Get a list of all upstream tests (default if $TESTS isn't set)
67+
echo -e '\n\nprinttests:\n\t@echo "${TESTS}"' >> Makefile.am
68+
tests=$(make -f Makefile.am printtests)
69+
fi
70+
total=$(echo "$tests" | wc -w)
71+
echo "Running $total tests"
72+
export LC_ALL=C
73+
export KEEP=yes
74+
exitcode=0
75+
timestamp=$(date -Iseconds)
76+
urlroot="$gitserver/cgit/diffutils.git/tree/tests/"
77+
passed=0
78+
failed=0
79+
skipped=0
80+
normal="$(tput sgr0)"
81+
for test in $tests
82+
do
83+
result="FAIL"
84+
url="$urlroot$test?id=$upstreamrev"
85+
# Run only the tests that invoke `diff`,
86+
# because other binaries aren't implemented yet
87+
if ! grep -E -s -q "(cmp|diff3|sdiff)" "$test"
88+
then
89+
sh "$test" 1> stdout.txt 2> stderr.txt && result="PASS" || exitcode=1
90+
json+="{\"test\":\"$test\",\"result\":\"$result\","
91+
json+="\"url\":\"$url\","
92+
json+="\"stdout\":\"$(base64 -w0 < stdout.txt)\","
93+
json+="\"stderr\":\"$(base64 -w0 < stderr.txt)\","
94+
json+="\"files\":{"
95+
cd gt-$test.*
96+
# Note: this doesn't include the contents of subdirectories,
97+
# but there isn't much value added in doing so
98+
for file in *
99+
do
100+
[[ -f "$file" ]] && json+="\"$file\":\"$(base64 -w0 < "$file")\","
101+
done
102+
json="${json%,}}},"
103+
cd - > /dev/null
104+
[[ "$result" = "PASS" ]] && (( passed++ ))
105+
[[ "$result" = "FAIL" ]] && (( failed++ ))
106+
else
107+
result="SKIP"
108+
(( skipped++ ))
109+
json+="{\"test\":\"$test\",\"url\":\"$url\",\"result\":\"$result\"},"
110+
fi
111+
color=2 # green
112+
[[ "$result" = "FAIL" ]] && color=1 # red
113+
[[ "$result" = "SKIP" ]] && color=3 # yellow
114+
printf " %-40s $(tput setaf $color)$result$(tput sgr0)\n" "$test"
115+
done
116+
echo ""
117+
echo -n "Summary: TOTAL: $total / "
118+
echo -n "$(tput setaf 2)PASS$normal: $passed / "
119+
echo -n "$(tput setaf 1)FAIL$normal: $failed / "
120+
echo "$(tput setaf 3)SKIP$normal: $skipped"
121+
echo ""
122+
123+
json="\"tests\":[${json%,}]"
124+
metadata="\"timestamp\":\"$timestamp\","
125+
metadata+="\"revision\":\"$rev\","
126+
metadata+="\"upstream-revision\":\"$upstreamrev\","
127+
if [[ -n "$GITHUB_ACTIONS" ]]
128+
then
129+
metadata+="\"branch\":\"$GITHUB_REF\","
130+
fi
131+
json="{$metadata $json}"
132+
133+
# Clean up
134+
cd "$scriptpath"
135+
rm -rf "$tempdir"
136+
137+
resultsfile="test-results.json"
138+
echo "$json" | jq > "$resultsfile"
139+
echo "Results written to $scriptpath/$resultsfile"
140+
141+
exit $exitcode

0 commit comments

Comments
 (0)