Add a utility to assist manually testing an exercise

README.md

@@ -92,6 +92,8 @@ Note that:
 - An exercise may contain `.meta/hints.md`.  This is optional and will appear after the normal exercise
   instructions if present.  Rust is different in many ways from other languages.  This is a place where the differences required for Rust are explained.  If it is a large change, you may want to call this out as a comment at the top of `src/lib.rs`, so the user recognises to read this section before starting.
 
+- If the test suite is appreciably sped up by running in release mode, and there is reason to be confident that the example implementation does not contain any overflow errors, consider adding a file `.meta/test-in-release-mode`. This should contain brief comments explaining the situation.
+
 - `README.md` may be [regenerated](https://github.com/exercism/docs/blob/master/maintaining-a-track/regenerating-exercise-readmes.md) from Exercism data. The generator will use the `description.md` from the exercise directory in the [problem-specifications repository](https://github.com/exercism/problem-specifications/tree/master/exercises), then any hints in `.meta/hints.md`, then the [Rust-specific instructions](https://github.com/exercism/rust/blob/master/config/exercise-readme-insert.md). The `## Source` section comes from the `metadata.yml` in the same directory.  Convention is that the description of the source remains text and the link is both name and hyperlink of the markdown link.
 
 - Be sure to add the exercise to an appropriate place in the `config.json` file.  The position in the file determines the order exercises are sent.   Generate a unique UUID for the exercise.  Current difficuly levels in use are 1, 4, 7 and 10.

_test/check-exercises.sh

@@ -1,73 +1,60 @@
 #!/bin/bash
 
+# test for existence and executability of the test-exercise script
+# this depends on that
+if [ ! -f "./bin/test-exercise" ]; then
+   echo "bin/test-exercise does not exist"
+   exit 1
+fi
+if [ ! -x "./bin/test-exercise" ]; then
+   echo "bin/test-exercise does not have its executable bit set"
+   exit 1
+fi
+
 # In DENYWARNINGS mode, do not set -e so that we run all tests.
 # This allows us to see all warnings.
 if [ -z "$DENYWARNINGS" ]; then
     set -e
 fi
 
+# can't benchmark with a stable compiler; to bench, use
+# $ BENCHMARK=1 rustup run nightly _test/check-exercises.sh
 if [ -n "$BENCHMARK" ]; then
     files=exercises/*/benches
 else
     files=exercises/*/tests
 fi
 
-tmp=${TMPDIR:-/tmp/}
-mkdir "${tmp}exercises"
-
-exitcode=0
-
+return_code=0
 # An exercise worth testing is defined here as any top level directory with
 # a 'tests' directory
 for exercise in $files; do
-    # This assumes that exercises are only one directory deep
-    # and that the primary module is named the same as the directory
-    directory=$(dirname "${exercise}");
-
-    workdir=$(mktemp -d "${tmp}${directory}.XXXXXXXXXX")
-
-    cp -R -T $directory $workdir
-
-    # Run in subshell to change workdir without affecting current workdir.
-    (
-        cd $workdir
-
-        cp example.rs src/lib.rs
-
-        # Overwrite empty Cargo.toml if an example specific file exists
-        if [ -f Cargo-example.toml ]; then
-            cp Cargo-example.toml Cargo.toml
-        fi
-
-        # Forcibly strip all "ignore" statements from the testing files
-        for test in tests/*.rs; do
-            sed -i '/\[ignore\]/d' $test
-        done
-
-        # Run benchmarks instead of tests when enabled.
-        if [ -n "$BENCHMARK" ]; then
-            cargo bench
-        elif [ -n "$DENYWARNINGS" ]; then
-            sed -i -e '1i #![deny(warnings)]' src/lib.rs
-
-            # No-run mode so we see no test output.
-            # Quiet mode so we see no compile output
-            # (such as "Compiling"/"Downloading").
-            # Compiler errors will still be shown though.
-            # Both flags are necessary to keep things quiet.
-            cargo test --quiet --no-run
-        else
-            # Run the test and get the status
-            cargo test
-        fi
-    )
-
-    status=$?
-
-    if [ $status -ne 0 ]
-    then
-        exitcode=1
-    fi
+   # This assumes that exercises are only one directory deep
+   # and that the primary module is named the same as the directory
+   directory=$(dirname "${exercise}")
+
+   release=""
+   if [ -z "$BENCHMARK" -a -f "$directory/.meta/test-in-release-mode" ]; then
+      release="--release"
+   fi
+
+   if [ -n "$DENYWARNINGS" ]; then
+      # No-run mode so we see no test output.
+      # Quiet mode so we see no compile output
+      # (such as "Compiling"/"Downloading").
+      # Compiler errors will still be shown though.
+      # Both flags are necessary to keep things quiet.
+      ./bin/test-exercise $directory --quiet --no-run
+      return_code=$(($return_code | $?))
+   else
+      # Run the test and get the status
+      # We use release mode here because, while it somewhat increases
+      # the compile time for all exercises, it substantially improves
+      # the runtime for certain exercises such as alphametics.
+      # Overall this should be an improvement.
+      ./bin/test-exercise $directory $release
+      return_code=$(($return_code | $?))
+   fi
 done
 
-exit $exitcode
+exit $return_code

bin/test-exercise

@@ -0,0 +1,84 @@
+#!/bin/bash
+# Test an exercise
+
+# which exercise are we testing right now?
+# if we were passed an argument, that should be the
+# exercise directory. Otherwise, assume we're in
+# it currently.
+if [ $# -ge 1 ]; then
+   exercise=$1
+   # if this script is called with arguments, it will pass through
+   # any beyond the first to cargo. Note that we can only get a
+   # free default argument if no arguments at all were passed,
+   # so if you are in the exercise directory and want to pass any
+   # arguments to cargo, you need to include the local path first.
+   # I.e. to test in release mode:
+   # $ test-exercise . --release
+   shift 1
+else
+   exercise='.'
+fi
+
+# what cargo command will we use?
+if [ -n "$BENCHMARK" ]; then
+   command="bench"
+else
+   command="test"
+fi
+
+declare -a preserve_files=("src/lib.rs" "Cargo.toml" "Cargo.lock")
+declare -a preserve_dirs=("tests")
+
+# reset instructions
+reset () {
+   for file in ${preserve_files[@]}; do
+      if [ -f "$exercise/$file.orig" ]; then
+         mv -f "$exercise/$file.orig" "$exercise/$file"
+      fi
+   done
+   for dir in ${preserve_dirs[@]}; do
+      if [ -d "$exercise/$dir.orig" ]; then
+         rm -rf "$exercise/$dir"
+         mv "$exercise/$dir.orig" "$exercise/$dir"
+      fi
+   done
+}
+
+# cause the reset to execute when the script exits normally or is killed
+trap reset EXIT INT TERM
+
+# preserve the files and directories we care about
+for file in ${preserve_files[@]}; do
+   if [ -f "$exercise/$file" ]; then
+      cp "$exercise/$file" "$exercise/$file.orig"
+   fi
+done
+for dir in ${preserve_dirs[@]}; do
+   if [ -d "$exercise/$dir" ]; then
+      cp -r "$exercise/$dir" "$exercise/$dir.orig"
+   fi
+done
+
+# Move example files to where Cargo expects them
+cp -f "$exercise/example.rs" "$exercise/src/lib.rs"
+if [ -f "$exercise/Cargo-example.toml" ]; then
+   cp -f "$exercise/Cargo-example.toml" "$exercise/Cargo.toml"
+fi
+
+# If deny warnings, insert a deny warnings compiler directive in the header
+if [ -n "$DENYWARNINGS" ]; then
+   sed -i -e '1i #![deny(warnings)]' "$exercise/src/lib.rs"
+fi
+
+# eliminate #[ignore] lines from tests
+for test in "$exercise/tests/*.rs"; do
+    sed -i '/\[ignore\]/d' $test
+done
+
+# run tests from within exercise directory
+# (use subshell so we auto-reset to current pwd after)
+(
+   cd $exercise
+   # this is the last command; its exit code is what's passed on
+   cargo $command "$@"
+)

exercises/alphametics/.meta/test-in-release-mode

@@ -0,0 +1,2 @@
+Takes a very long time to test in debug mode.
+Example implementation not known to encounter overflow errors.