Introduce lit/FileCheck tests (#3367)

lit and FileCheck are the tools used to run the majority of tests in LLVM. Each
lit test file contains the commands to be run for that test, so lit tests are
much more flexible and can be more precise than our current ad hoc testing
system. FileCheck reads expected test output from comments, so it allows test
output to be written alongside and interspersed with test input, making tests
more readable and precise than in our current system.

This PR adds a new suite to check.py that runs lit tests in the test/lit
directory. A few tests have been ported to demonstrate the features of the new
test runner.

This change is motivated by a need for greater flexibility in testing wasm-split.
See #3359.

Author: tlively

.flake8

@@ -3,4 +3,4 @@ ignore =
   E501, # line too long
   E241, # space after comma (ignored for list in gen-s-parser.py)
   W504  # line break after binary operator
-exclude = ./test/emscripten,./test/spec,./test/wasm-install
+exclude = ./test/emscripten,./test/spec,./test/wasm-install,./test/lit

.github/workflows/ci.yml

@@ -47,6 +47,9 @@ jobs:
         python-version: '3.x'
     - uses: actions/checkout@v1
 
+    - name: install Python dev dependencies
+      run: pip3 install -r requirements-dev.txt
+
     - name: install ninja (linux)
       run: sudo apt-get install ninja-build
       if: matrix.os == 'ubuntu-latest'
@@ -92,7 +95,7 @@ jobs:
         path: out/install
 
     - name: test
-      run: python check.py --binaryen-bin=out/install/bin
+      run: python check.py --binaryen-bin=out/bin
 
   build-clang:
     name: clang
@@ -104,6 +107,8 @@ jobs:
     - uses: actions/checkout@v1
     - name: install ninja
       run: sudo apt-get install ninja-build
+    - name: install Python dev dependencies
+      run: pip3 install -r requirements-dev.txt
     - name: cmake
       run: |
         mkdir -p out
@@ -127,6 +132,8 @@ jobs:
     - uses: actions/checkout@v1
     - name: install ninja
       run: sudo apt-get install ninja-build
+    - name: install Python dev dependencies
+      run: pip3 install -r requirements-dev.txt
     - name: cmake
       run: |
         mkdir -p out
@@ -151,6 +158,8 @@ jobs:
     - uses: actions/checkout@v1
     - name: install ninja
       run: sudo apt-get install ninja-build
+    - name: install Python dev dependencies
+      run: pip3 install -r requirements-dev.txt
     - name: cmake
       run: |
         mkdir -p out
@@ -174,6 +183,8 @@ jobs:
     - uses: actions/checkout@v1
     - name: install ninja
       run: sudo apt-get install ninja-build
+    - name: install Python dev dependencies
+      run: pip3 install -r requirements-dev.txt
     - name: cmake
       run: |
         mkdir -p out

.gitignore

@@ -25,6 +25,7 @@ CMakeFiles
 *.o
 *.obj
 compile_commands.json
+test/lit/lit.site.cfg.py
 
 # files related to bulding in-tree on windows
 /.vs/

CMakeLists.txt

@@ -278,6 +278,9 @@ add_subdirectory(src/support)
 add_subdirectory(src/wasm)
 add_subdirectory(third_party)
 
+# Configure lit tests
+add_subdirectory(test/lit)
+
 # Object files
 set(binaryen_objs
     $<TARGET_OBJECTS:passes>

check.py

@@ -328,6 +328,18 @@ def run_unittest():
         raise Exception("unittest failed")
 
 
+def run_lit():
+    lit_script = os.path.join(shared.options.binaryen_root, 'scripts', 'lit_wrapper.py')
+    lit_cfg = os.path.join(shared.options.binaryen_build, 'test', 'lit')
+    # lit expects to be run as its own executable
+    cmd = [sys.executable, lit_script, lit_cfg, '-vv']
+    result = subprocess.run(cmd)
+    if result.returncode != 0:
+        shared.num_failures += 1
+    if shared.options.abort_on_first_failure and shared.num_failures:
+        raise Exception("lit test failed")
+
+
 TEST_SUITES = OrderedDict([
     ('help-messages', run_help_tests),
     ('wasm-opt', wasm_opt.test_wasm_opt),
@@ -345,6 +357,7 @@ def run_unittest():
     ('unit', run_unittest),
     ('binaryenjs', binaryenjs.test_binaryen_js),
     ('binaryenjs_wasm', binaryenjs.test_binaryen_wasm),
+    ('lit', run_lit),
 ])
 
 

requirements-dev.txt

@@ -4,3 +4,5 @@
 # Install with `pip3 install -r requirements-dev.txt`
 
 flake8==3.7.8
+filecheck==0.0.17
+lit==0.11.0.post1

scripts/lit_wrapper.py

@@ -0,0 +1,22 @@
+#!/usr/bin/env python3
+#
+# Copyright 2020 WebAssembly Community Group participants
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from lit.main import main
+
+# A simple wrapper around `lit`'s main function, since it isn't otherwise
+# exposed.
+if __name__ == '__main__':
+    main()

scripts/not.py

@@ -0,0 +1,30 @@
+#!/usr/bin/env python3
+
+# Copyright 2020 WebAssembly Community Group participants
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys
+import subprocess
+
+
+# Emulate the `not` tool from LLVM's test infrastructure for use with lit and
+# FileCheck. It succeeds if the given subcommand fails and vice versa.
+def main():
+    cmd = sys.argv[1:]
+    result = subprocess.run(cmd)
+    sys.exit(0 if result.returncode != 0 else 1)
+
+
+if __name__ == '__main__':
+    main()

scripts/test/lld.py

@@ -26,8 +26,6 @@ def args_for_finalize(filename):
         ret += ['--side-module']
     if 'standalone-wasm' in filename:
         ret += ['--standalone-wasm']
-    if 'bigint' in filename:
-        ret += ['--bigint']
     return ret
 
 

scripts/test/shared.py

@@ -50,8 +50,9 @@ def parse_args(args):
               ' earlier errors.'))
     parser.add_argument(
         '--binaryen-bin', dest='binaryen_bin', default='',
-        help=('Specifies a path to where the built Binaryen executables reside at.'
-              ' Default: bin/ of current directory (i.e. assume an in-tree build).'
+        help=('Specifies the path to the Binaryen executables in the CMake build'
+              ' directory. Default: bin/ of current directory (i.e. assume an'
+              ' in-tree build).'
               ' If not specified, the environment variable BINARYEN_ROOT= can also'
               ' be used to adjust this.'))
     parser.add_argument(
@@ -129,6 +130,8 @@ def warn(text):
 
 options.binaryen_lib = os.path.normpath(os.path.abspath(options.binaryen_lib))
 
+options.binaryen_build = os.path.dirname(options.binaryen_bin)
+
 # ensure BINARYEN_ROOT is set up
 os.environ['BINARYEN_ROOT'] = os.path.dirname(options.binaryen_bin)