Update musl to v1.2.2 (#13006)

The changes here mostly come from running the update_musl.py script
against the following commit:

https://github.com/emscripten-core/musl/tree/merge_v1.2.2

Notes:

- `weak` is defined in musl-internal headers so some of our usage of
  `__attribute__((weak))` was switched to `__weak__` (which is a
  synonym).

- Quite a few changes needed in library_pthread.js due to changes to
  the internal representation of __pthread struct in C.
  - removal of emscripten-specific threadStatus field.  This can
    can be fully replaced with the updated `detach_state` that
    was added to musl (along with enum values for states!) along
    with the existing `cancel` bit.
  - `attr` is no longer stored on `__pthread`; it was only duplicating
    other fields anyway.  We now include `pthread_getattr_np.c` which
    can pull those field out on demand into at attr structure.

Author: sbc100

ChangeLog.md

@@ -18,8 +18,13 @@ to browse the changes between the tags.
 
 See docs/process.md for more on how version tagging works.
 
-2.0.35
+3.0.0
 ------
+- The version of musl libc used by emscripten was upgraded from v1.1.15 to
+  v1.2.2.  There could be some minor size regressions (or gains) due to changes
+  in upstream musl code but we don't expect anything major.  Since this fairly
+  change (at least internally) we are bumping the major version of emscripten
+  to 3. (#13006)
 
 2.0.34 - 11/04/2021
 -------------------

emscripten-version.txt

@@ -1 +1 @@
-2.0.35-git
+3.0.0-git

src/library.js

@@ -594,9 +594,9 @@ LibraryManager.library = {
   __localtime_r: 'localtime_r',
 
   // musl-internal function used to implement both `asctime` and `asctime_r`
-  __asctime__deps: ['mktime'],
-  __asctime__sig: 'iii',
-  __asctime: function(tmPtr, buf) {
+  __asctime_r__deps: ['mktime'],
+  __asctime_r__sig: 'iii',
+  __asctime_r: function(tmPtr, buf) {
     var date = {
       tm_sec: {{{ makeGetValue('tmPtr', C_STRUCTS.tm.tm_sec, 'i32') }}},
       tm_min: {{{ makeGetValue('tmPtr', C_STRUCTS.tm.tm_min, 'i32') }}},
@@ -632,11 +632,11 @@ LibraryManager.library = {
     return ret;
   },
 
-  ctime_r__deps: ['localtime_r', '__asctime', '$withStackSave'],
+  ctime_r__deps: ['localtime_r', '__asctime_r', '$withStackSave'],
   ctime_r__sig: 'iii',
   ctime_r: function(time, buf) {
     return withStackSave(function() {
-      return ___asctime(_localtime_r(time, stackAlloc({{{ C_STRUCTS.tm.__size__ }}})), buf);
+      return ___asctime_r(_localtime_r(time, stackAlloc({{{ C_STRUCTS.tm.__size__ }}})), buf);
     });
   },
 

src/library_pthread.js

@@ -125,6 +125,26 @@ var LibraryPThread = {
     },
 #endif
 
+#if PTHREADS_DEBUG
+    detachStateToString: function(state) {
+      if (state === {{{ cDefine('DT_EXITED') }}}) return 'DT_EXITED';
+      if (state === {{{ cDefine('DT_EXITING') }}}) return 'DT_EXITING';
+      if (state === {{{ cDefine('DT_JOINABLE') }}}) return 'DT_JOINABLE';
+      if (state === {{{ cDefine('DT_DETACHED') }}}) return 'DT_DETACHED';
+      assert(false);
+    },
+#endif
+
+    setDetachState: function(thread, newstate) {
+#if PTHREADS_DEBUG
+      var oldstate = Atomics.load(HEAPU32, (thread + {{{ C_STRUCTS.pthread.detach_state }}}) >> 2);
+      var oldname = PThread.detachStateToString(oldstate);
+      var newname = PThread.detachStateToString(newstate);
+      console.log('thread 0x' + thread.toString(16) + ' state change: ' + oldname + ' -> ' + newname);
+#endif
+      Atomics.store(HEAPU32, (thread + {{{ C_STRUCTS.pthread.detach_state }}}) >> 2, newstate);
+    },
+
     terminateAllThreads: function() {
 #if ASSERTIONS
       assert(!ENVIRONMENT_IS_PTHREAD, 'Internal Error! terminateAllThreads() can only ever be called from main application thread!');
@@ -278,8 +298,8 @@ var LibraryPThread = {
         } else if (cmd === 'detachedExit') {
 #if ASSERTIONS
           assert(worker.pthread);
-          var detached = Atomics.load(HEAPU32, (worker.pthread.threadInfoStruct + {{{ C_STRUCTS.pthread.detached }}}) >> 2);
-          assert(detached);
+          var detach_state = Atomics.load(HEAPU32, (worker.pthread.threadInfoStruct + {{{ C_STRUCTS.pthread.detach_state }}}) >> 2);
+          assert(detach_state == {{{ cDefine('DT_EXITED') }}});
 #endif
           PThread.returnWorkerToPool(worker);
         } else if (cmd === 'cancelDone') {
@@ -570,19 +590,17 @@ var LibraryPThread = {
       worker: worker,
       stackBase: threadParams.stackBase,
       stackSize: threadParams.stackSize,
+      initialState: {{{ cDefine('DT_JOINABLE') }}},
       allocatedOwnStack: threadParams.allocatedOwnStack,
       // Info area for this thread in Emscripten HEAP (shared)
       threadInfoStruct: threadParams.pthread_ptr
     };
     var tis = pthread.threadInfoStruct >> 2;
     // spawnThread is always called with a zero-initialized thread struct so
     // no need to set any valudes to zero here.
-    Atomics.store(HEAPU32, tis + ({{{ C_STRUCTS.pthread.detached }}} >> 2), threadParams.detached);
+    PThread.setDetachState(pthread.threadInfoStruct, threadParams.initialState);
     Atomics.store(HEAPU32, tis + ({{{ C_STRUCTS.pthread.stack_size }}} >> 2), threadParams.stackSize);
     Atomics.store(HEAPU32, tis + ({{{ C_STRUCTS.pthread.stack }}} >> 2), stackHigh);
-    Atomics.store(HEAPU32, tis + ({{{ C_STRUCTS.pthread.attr }}} + 0/*_a_stacksize*/ >> 2), threadParams.stackSize);
-    Atomics.store(HEAPU32, tis + ({{{ C_STRUCTS.pthread.attr }}} + 8/*_a_stackaddr*/ >> 2), stackHigh);
-    Atomics.store(HEAPU32, tis + ({{{ C_STRUCTS.pthread.attr }}} + 12/*_a_detach*/ >> 2), threadParams.detached);
 
 #if PTHREADS_PROFILING
     PThread.createProfilerBlock(pthread.threadInfoStruct);
@@ -772,23 +790,16 @@ var LibraryPThread = {
 
     var stackSize = 0;
     var stackBase = 0;
-    // Default thread attr is PTHREAD_CREATE_JOINABLE, i.e. start as not detached.
-    var detached = 0;
+    // Default thread state is DT_JOINABLE, i.e. start as not detached.
+    var initialState = {{{ cDefine('DT_JOINABLE') }}};
     // When musl creates C11 threads it passes __ATTRP_C11_THREAD (-1) which
     // treat as if it was NULL.
     if (attr && attr != {{{ cDefine('__ATTRP_C11_THREAD') }}}) {
       stackSize = {{{ makeGetValue('attr', 0/*_a_stacksize*/, 'i32') }}};
-      // Musl has a convention that the stack size that is stored to the pthread
-      // attribute structure is always musl's #define DEFAULT_STACK_SIZE
-      // smaller than the actual created stack size. That is, stored stack size
-      // of 0 would mean a stack of DEFAULT_STACK_SIZE in size. All musl
-      // functions hide this impl detail, and offset the size transparently, so
-      // pthread_*() API user does not see this offset when operating with
-      // the pthread API. When reading the structure directly on JS side
-      // however, we need to offset the size manually here.
-      stackSize += {{{ cDefine('DEFAULT_STACK_SIZE') }}};
       stackBase = {{{ makeGetValue('attr', 8/*_a_stackaddr*/, 'i32') }}};
-      detached = {{{ makeGetValue('attr', 12/*_a_detach*/, 'i32') }}} !== 0/*PTHREAD_CREATE_JOINABLE*/;
+      if ({{{ makeGetValue('attr', 12/*_a_detach*/, 'i32') }}}) {
+        initialState = {{{ cDefine('DT_DETACHED') }}};
+      }
     } else {
       // According to
       // http://man7.org/linux/man-pages/man3/pthread_create.3.html, default
@@ -822,7 +833,7 @@ var LibraryPThread = {
       stackBase: stackBase,
       stackSize: stackSize,
       allocatedOwnStack: allocatedOwnStack,
-      detached: detached,
+      initialState: initialState,
       startRoutine: start_routine,
       pthread_ptr: pthread_ptr,
       arg: arg,
@@ -880,14 +891,19 @@ var LibraryPThread = {
     }
     var self = {{{ makeGetValue('thread', C_STRUCTS.pthread.self, 'i32') }}};
     if (self !== thread) {
-      err('pthread_join attempted on thread ' + thread + ', which does not point to a valid thread, or does not exist anymore!');
+      err('pthread_join attempted on thread 0x' + thread.toString(16) + ', which does not point to a valid thread, or does not exist anymore!');
       return {{{ cDefine('ESRCH') }}};
     }
-    var detached = Atomics.load(HEAPU32, (thread + {{{ C_STRUCTS.pthread.detached }}} ) >> 2);
-    if (detached) {
-      err('Attempted to join thread ' + thread + ', which was already detached!');
+    var detach_state = Atomics.load(HEAPU32, (thread + {{{ C_STRUCTS.pthread.detach_state }}}) >> 2);
+    if (detach_state == {{{ cDefine('DT_DETACHED') }}}) {
+      err('Attempted to join thread 0x' + thread.toString(16) + ', which was already detached!');
       return {{{ cDefine('EINVAL') }}}; // The thread is already detached, can no longer join it!
     }
+
+    if (detach_state == {{{ cDefine('DT_EXITED') }}}) {
+      err('Attempted to join thread 0x' + thread.toString(16) + ', which was already joined!');
+      return {{{ cDefine('EINVAL') }}};
+    }
     if (ENVIRONMENT_IS_PTHREAD && _pthread_self() == thread) {
       err('PThread ' + thread + ' is attempting to join to itself!');
       return {{{ cDefine('EDEADLK') }}};
@@ -904,14 +920,14 @@ var LibraryPThread = {
 #endif
 
     for (;;) {
-      var threadStatus = Atomics.load(HEAPU32, (thread + {{{ C_STRUCTS.pthread.threadStatus }}} ) >> 2);
-      if (threadStatus == 1) { // Exited?
+      var detach_state = Atomics.load(HEAPU32, (thread + {{{ C_STRUCTS.pthread.detach_state }}} ) >> 2);
+      if (detach_state == {{{ cDefine('DT_EXITING') }}}) { // Exiting?
         if (status) {
           var result = Atomics.load(HEAPU32, (thread + {{{ C_STRUCTS.pthread.result }}} ) >> 2);
           {{{ makeSetValue('status', 0, 'result', 'i32') }}};
         }
-        // Mark the thread as detached.
-        Atomics.store(HEAPU32, (thread + {{{ C_STRUCTS.pthread.detached }}} ) >> 2, 1);
+        // Mark the thread as exited.
+        PThread.setDetachState(thread, {{{ cDefine('DT_EXITED') }}});
         if (!ENVIRONMENT_IS_PTHREAD) cleanupThread(thread);
         else postMessage({ 'cmd': 'cleanupThread', 'thread': thread });
         return 0;
@@ -924,7 +940,7 @@ var LibraryPThread = {
       // runtime and launched main()), assist pthreads in performing operations
       // that they need to access the Emscripten main runtime for.
       if (!ENVIRONMENT_IS_PTHREAD) _emscripten_main_thread_process_queued_calls();
-      _emscripten_futex_wait(thread + {{{ C_STRUCTS.pthread.threadStatus }}}, threadStatus, ENVIRONMENT_IS_PTHREAD ? 100 : 1);
+      _emscripten_futex_wait(thread + {{{ C_STRUCTS.pthread.detach_state }}}, detach_state, ENVIRONMENT_IS_PTHREAD ? 100 : 1);
     }
   },
 
@@ -977,7 +993,8 @@ var LibraryPThread = {
       err('pthread_cancel attempted on thread ' + thread + ', which does not point to a valid thread, or does not exist anymore!');
       return {{{ cDefine('ESRCH') }}};
     }
-    Atomics.compareExchange(HEAPU32, (thread + {{{ C_STRUCTS.pthread.threadStatus }}} ) >> 2, 0, 2); // Signal the thread that it needs to cancel itself.
+    // Signal the thread that it needs to cancel itself.
+    Atomics.store(HEAPU32, (thread + {{{ C_STRUCTS.pthread.cancel }}}) >> 2, 1);
     if (!ENVIRONMENT_IS_PTHREAD) cancelThread(thread);
     else postMessage({ 'cmd': 'cancelThread', 'thread': thread});
     return 0;

src/struct_info_internal.json

@@ -6,14 +6,19 @@
         "file": "pthread_impl.h",
         "structs": {
             "pthread": [
-              "threadStatus",
               "profilerBlock",
               "self",
-              "detached",
+              "tsd",
+              "detach_state",
               "stack",
               "stack_size",
               "result",
-              "attr"
+              "robust_list",
+              "tid",
+              "cancel",
+              "canceldisable",
+              "cancelasync",
+              "locale"
             ],
             "pthread_attr_t#": [
               "_a_transferredcanvases"
@@ -26,9 +31,12 @@
             ]
         },
         "defines": [
-          "DEFAULT_STACK_SIZE",
           "__ATTRP_C11_THREAD",
-          "EM_THREAD_NAME_MAX"
+          "EM_THREAD_NAME_MAX",
+          "DT_EXITED",
+          "DT_EXITING",
+          "DT_JOINABLE",
+          "DT_DETACHED"
         ]
     },
     {

system/include/compat/sys/random.h

@@ -9,6 +9,8 @@ extern "C" {
 // syscall which is unnecessary indirection for us.
 int getentropy(void *buffer, size_t length);
 
+#include_next <sys/random.h>
+
 #ifdef __cplusplus
 }
 #endif

system/lib/compiler-rt/lib/sanitizer_common/sanitizer_linux.cpp

@@ -104,7 +104,11 @@ extern struct ps_strings *__ps_strings;
 #endif
 
 #if SANITIZER_EMSCRIPTEN
+#define weak __attribute__(__weak__)
+#define hidden __attribute__((__visibility__("hidden")))
 #include <syscall.h>
+#undef weak
+#undef hidden
 #include <emscripten/threading.h>
 #include <math.h>
 #include <wasi/api.h>

system/lib/dlmalloc.c

@@ -867,6 +867,10 @@ extern "C" {
 #ifndef USE_DL_PREFIX
 // XXX Emscripten XXX
 #if defined(__EMSCRIPTEN__)
+void* __libc_malloc(size_t) __attribute__((weak, alias("dlmalloc")));
+void  __libc_free(void*) __attribute__((weak, alias("dlfree")));
+void* __libc_calloc(size_t) __attribute__((weak, alias("dlcalloc")));
+void* __libc_realloc(void*, size_t) __attribute__((weak, alias("dlrealloc")));
 void* malloc(size_t) __attribute__((weak, alias("dlmalloc")));
 void  free(void*) __attribute__((weak, alias("dlfree")));
 void* calloc(size_t, size_t) __attribute__((weak, alias("dlcalloc")));

system/lib/emmalloc.c

@@ -793,6 +793,7 @@ void *emmalloc_memalign(size_t alignment, size_t size)
   return ptr;
 }
 extern __typeof(emmalloc_memalign) emscripten_builtin_memalign __attribute__((alias("emmalloc_memalign")));
+extern __typeof(emmalloc_memalign) __libc_memalign __attribute__((alias("emmalloc_memalign")));
 
 void * EMMALLOC_EXPORT memalign(size_t alignment, size_t size)
 {
@@ -811,6 +812,7 @@ void *emmalloc_malloc(size_t size)
   return emmalloc_memalign(MALLOC_ALIGNMENT, size);
 }
 extern __typeof(emmalloc_malloc) emscripten_builtin_malloc __attribute__((alias("emmalloc_malloc")));
+extern __typeof(emmalloc_malloc) __libc_malloc __attribute__((alias("emmalloc_malloc")));
 
 void * EMMALLOC_EXPORT malloc(size_t size)
 {
@@ -912,6 +914,7 @@ void emmalloc_free(void *ptr)
 #endif
 }
 extern __typeof(emmalloc_free) emscripten_builtin_free __attribute__((alias("emmalloc_free")));
+extern __typeof(emmalloc_free) __libc_free __attribute__((alias("emmalloc_free")));
 
 void EMMALLOC_EXPORT free(void *ptr)
 {
@@ -1132,6 +1135,7 @@ void *emmalloc_realloc(void *ptr, size_t size)
 {
   return emmalloc_aligned_realloc(ptr, MALLOC_ALIGNMENT, size);
 }
+extern __typeof(emmalloc_realloc) __libc_realloc __attribute__((alias("emmalloc_realloc")));
 
 void * EMMALLOC_EXPORT realloc(void *ptr, size_t size)
 {
@@ -1167,6 +1171,7 @@ void *emmalloc_calloc(size_t num, size_t size)
     memset(ptr, 0, bytes);
   return ptr;
 }
+extern __typeof(emmalloc_calloc) __libc_calloc __attribute__((alias("emmalloc_calloc")));
 
 void * EMMALLOC_EXPORT calloc(size_t num, size_t size)
 {

system/lib/libc/README.md

@@ -1,5 +1,9 @@
-This folder contains the musl version of libc at `/musl`. The upstream version can be found at http://www.musl-libc.org/.
-Most of the source comes from musl 1.1.15, with some exceptions listed below.
+This folder contains the musl version of libc at `/musl`. The upstream version
+can be found at http://www.musl-libc.org/.
+
+Most of the source comes from musl v1.2.2, with some exceptions listed below.
+We track these changes from upstream in https://github.com/emscripten-core/musl
+and use a script (`system/lib/update_musl.py`) to pull in updates.
 
 Some changes have been made to the version that was taken from upstream, including:
 

system/lib/libc/compat/aligned_alloc.c

@@ -2,7 +2,7 @@
 
 // Musl has an aligned_alloc routine, but that builds on top of standard malloc(). We are using dlmalloc, so
 // can route to its implementation instead.
-void * __attribute__((weak)) aligned_alloc(size_t alignment, size_t size)
+void * weak aligned_alloc(size_t alignment, size_t size)
 {
   void *ptr;
   if ((alignment % sizeof(void *) != 0) || (size % alignment) != 0)

system/lib/libc/crt1.c

@@ -12,7 +12,7 @@
 #include <stdlib.h>
 #include <wasi/api.h>
 
-__attribute__((weak)) void __wasm_call_ctors(void);
+__attribute__((__weak__)) void __wasm_call_ctors(void);
 
 int __original_main(void);