Integrate posixtestsuite for pthread testing

.circleci/config.yml

@@ -123,6 +123,9 @@ commands:
           # Must be absolute path or relative path from working_directory
           at: ~/
       - checkout
+      - run:
+          name: submodule update
+          command: git submodule update --init
       - emsdk-env
       - npm-install
       - run:
@@ -160,6 +163,9 @@ commands:
           # Must be absolute path or relative path from working_directory
           at: ~/
       - checkout
+      - run:
+          name: submodule update
+          command: git submodule update --init
       - emsdk-env
       - npm-install
       - run:
@@ -250,7 +256,7 @@ commands:
             EXTRA_AUTOSTART=$TMPDIR/autostart startx /usr/bin/openbox-session -- $DISPLAY -config ~/.config/X11/xorg.conf -nolisten tcp &
             cat $TMPDIR/fifo > /dev/null # wait until $EXTRA_AUTOSTART is spawned, which indicates the end of Openbox initialization
             rm -r $TMPDIR
-            python3 tests/runner.py browser skip:browser.test_sdl2_mouse skip:browser.test_html5_webgl_create_context skip:browser.test_webgl_offscreen_canvas_in_pthread skip:browser.test_webgl_offscreen_canvas_in_mainthread_after_pthread skip:browser.test_glut_glutget
+            python3 tests/runner.py browser posixtest.test_pthread_create_1_1 skip:browser.test_sdl2_mouse skip:browser.test_html5_webgl_create_context skip:browser.test_webgl_offscreen_canvas_in_pthread skip:browser.test_webgl_offscreen_canvas_in_mainthread_after_pthread skip:browser.test_glut_glutget
             python3 tests/runner.py emrun
             openbox --exit
             wait || true # wait for startx to shutdown cleanly, or not
@@ -261,6 +267,9 @@ commands:
           # Must be absolute path or relative path from working_directory
           at: ~/
       - checkout
+      - run:
+          name: submodule update
+          command: git submodule update --init
       - emsdk-env
       - npm-install
       - run:
@@ -282,7 +291,7 @@ commands:
           command: |
             export EMTEST_BROWSER="/usr/bin/google-chrome $CHROME_FLAGS_BASE $CHROME_FLAGS_HEADLESS $CHROME_FLAGS_WASM $CHROME_FLAGS_NOCACHE"
             # skip test_zzz_zzz_4GB_fail as it OOMs on the current bot
-            python3 tests/runner.py browser skip:browser.test_zzz_zzz_4GB_fail
+            python3 tests/runner.py browser posixtest.test_pthread_create_1_1 skip:browser.test_zzz_zzz_4GB_fail
             python3 tests/runner.py emrun
   test-sockets-chrome:
     description: "Runs emscripten sockets tests under chrome"
@@ -384,7 +393,7 @@ jobs:
     steps:
       - run-tests:
           # also add a few asan tests
-          test_targets: "wasm2 asan.test_embind* asan.test_abort_on_exceptions asan.test_ubsan_full_left_shift_fsanitize_integer"
+          test_targets: "wasm2 asan.test_embind* asan.test_abort_on_exceptions asan.test_ubsan_full_left_shift_fsanitize_integer posixtest.test_pthread_create_1_1"
   test-wasm3:
     executor: bionic
     steps:

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "tests/third_party/posixtestsuite"]
+	path = tests/third_party/posixtestsuite
+	url = https://github.com/juj/posixtestsuite

src/preamble.js

@@ -60,7 +60,7 @@ var ABORT = false;
 // set by exit() and abort().  Passed to 'onExit' handler.
 // NOTE: This is also used as the process return code code in shell environments
 // but only when noExitRuntime is false.
-var EXITSTATUS = 0;
+var EXITSTATUS;
 
 /** @type {function(*, string=)} */
 function assert(condition, text) {

tests/runner.py

@@ -305,6 +305,8 @@ def make_executable(name):
   'asan',
   'lsan',
   'wasm2ss',
+  'posixtest',
+  'posixtest_browser',
 ]
 
 
@@ -1827,7 +1829,7 @@ def flattened_tests(loaded_tests):
 
 
 def suite_for_module(module, tests):
-  suite_supported = module.__name__ in ('test_core', 'test_other')
+  suite_supported = module.__name__ in ('test_core', 'test_other', 'test_posixtest')
   if not EMTEST_SAVE_DIR:
     has_multiple_tests = len(tests) > 1
     has_multiple_cores = parallel_testsuite.num_cores() > 1

tests/test_posixtest.py

@@ -0,0 +1,116 @@
+# Copyright 2020 The Emscripten Authors.  All rights reserved.
+# Emscripten is available under two separate licenses, the MIT license and the
+# University of Illinois/NCSA Open Source License.  Both these licenses can be
+# found in the LICENSE file.
+
+"""Runs the pthreads test from the upstream posixtest suite in:
+   ./tests/third_party/posixtestsuite
+See
+   https://github.com/juj/posixtestsuite
+"""
+
+import glob
+import os
+
+from runner import RunnerCore, path_from_root
+from tools import config
+from tools.shared import EMCC
+import test_posixtest_browser
+
+testsuite_root = path_from_root('tests', 'third_party', 'posixtestsuite')
+
+
+class posixtest(RunnerCore):
+  """Run the suite under node (and in parallel)
+
+  This class get populated dynamically below.
+  """
+  pass
+
+
+def filter_tests(all_tests):
+  pthread_tests = [t for t in all_tests if t.startswith('pthread_')]
+  # filter out some tests we don't support
+  pthread_tests = [t for t in pthread_tests if not t.startswith('pthread_atfork')]
+  pthread_tests = [t for t in pthread_tests if not t.startswith('pthread_sigmask')]
+  return pthread_tests
+
+
+def get_pthread_tests():
+  # For now, we don't require the submodule to exist.  In this case we just report
+  # no tests
+  pthread_test_root = os.path.join(testsuite_root, 'conformance', 'interfaces')
+  if not os.path.exists(pthread_test_root):
+    print('posixtestsuite not found (run git submodule update --init?)')
+    return []
+  pthread_tests = filter_tests(os.listdir(pthread_test_root))
+  pthread_tests = [os.path.join(pthread_test_root, t) for t in pthread_tests]
+  return pthread_tests
+
+
+engine = config.NODE_JS + ['--experimental-wasm-threads', '--experimental-wasm-bulk-memory']
+
+# Mark certain tests as not passing
+disabled = {
+  'test_pthread_create_11_1': 'never returns',
+  'test_pthread_barrier_wait_2_1': 'never returns',
+  'test_pthread_cond_timedwait_2_6': 'never returns',
+  'test_pthread_cond_timedwait_4_3': 'never returns',
+  'test_pthread_attr_setscope_5_1': 'internally skipped (PTS_UNTESTED)',
+  'test_pthread_cond_wait_2_3': 'never returns',
+  'test_pthread_create_5_1': 'never returns',
+  'test_pthread_exit_1_2': 'never returns',
+  'test_pthread_exit_2_2': 'never returns',
+  'test_pthread_exit_3_2': 'never returns',
+  'test_pthread_exit_4_1': 'never returns',
+  'test_pthread_getcpuclockid_1_1': 'never returns',
+  'test_pthread_key_create_1_2': 'never returns',
+  'test_pthread_rwlock_rdlock_1_1': 'fails with "main: Unexpected thread state"',
+  'test_pthread_rwlock_timedrdlock_1_1': 'fails with "main: Unexpected thread state"',
+  'test_pthread_rwlock_timedrdlock_3_1': 'fails with "main: Unexpected thread state"',
+  'test_pthread_rwlock_timedrdlock_5_1': 'fails with "main: Unexpected thread state"',
+  'test_pthread_rwlock_timedwrlock_1_1': 'fails with "main: Unexpected thread state"',
+  'test_pthread_rwlock_timedwrlock_3_1': 'fails with "main: Unexpected thread state"',
+  'test_pthread_rwlock_timedwrlock_5_1': 'fails with "main: Unexpected thread state"',
+  'test_pthread_rwlock_wrlock_1_1': 'fails with "main: Unexpected thread state"',
+  'test_pthread_rwlock_trywrlock_1_1': 'fails with "main: Unexpected thread state"',
+  'test_pthread_spin_destroy_3_1': 'never returns',
+  'test_pthread_spin_init_4_1': 'never returns',
+}
+
+
+def make_test(name, testfile, browser):
+
+  def f(self):
+    if name in disabled:
+      self.skipTest(disabled[name])
+    args = ['-I' + os.path.join(testsuite_root, 'include'),
+            '-Werror',
+            '-Wno-format-security',
+            '-Wno-int-conversion',
+            '-sUSE_PTHREADS',
+            '-sEXIT_RUNTIME',
+            '-sTOTAL_MEMORY=268435456',
+            '-sPTHREAD_POOL_SIZE=40']
+    if browser:
+      # Only are only needed for browser tests of the was btest
+      # injects headers using `-include` flag.
+      args += ['-Wno-macro-redefined', '-D_GNU_SOURCE']
+      self.btest(testfile, args=args, expected='exit:0')
+    else:
+      self.run_process([EMCC, testfile, '-o', 'test.js'] + args)
+      self.run_js('test.js', engine=engine)
+
+  return f
+
+
+for testdir in get_pthread_tests():
+  basename = os.path.basename(testdir)
+  for test_file in glob.glob(os.path.join(testdir, '*.c')):
+    if not os.path.basename(test_file)[0].isdigit():
+      continue
+    test_suffix = os.path.splitext(os.path.basename(test_file))[0]
+    test_suffix = test_suffix.replace('-', '_')
+    test_name = 'test_' + basename + '_' + test_suffix
+    setattr(posixtest, test_name, make_test(test_name, test_file, browser=False))
+    setattr(test_posixtest_browser.posixtest_browser, test_name, make_test(test_name, test_file, browser=True))

tests/test_posixtest_browser.py

@@ -0,0 +1,18 @@
+"""See test_posixtest.py
+
+This class is only in its own file so that we can
+run the non-browser versions in parallel.
+
+Currently only entire modules can be marked as
+parallel.
+"""
+
+from runner import BrowserCore
+
+
+class posixtest_browser(BrowserCore):
+  """Run the suite in the browser (serially)
+
+  This class get populated dynamically below.
+  """
+  pass