fix Libc.rand and seed problems (#44432)

Continuation from #43606

 - Replaces thread-unsafe function `rand` with `jl_rand`.
 - Fixes `_ad_hoc_entropy_source` fallback in Random.
 - Uses uv_random for more direct access to quality-randomness (usually
   a syscall rather than a file.)
 - Ensures Array{Bool} are valid when created from RandomDevice.

base/Base.jl

@@ -300,9 +300,6 @@ include("process.jl")
 include("ttyhascolor.jl")
 include("secretbuffer.jl")
 
-# RandomDevice support
-include("randomdevice.jl")
-
 # core math functions
 include("floatfuncs.jl")
 include("math.jl")
@@ -492,8 +489,6 @@ end
 
 if is_primary_base_module
 function __init__()
-    # for the few uses of Libc.rand in Base:
-    Libc.srand()
     # Base library init
     reinit_stdio()
     Multimedia.reinit_displays() # since Multimedia.displays uses stdout as fallback

base/error.jl

@@ -261,7 +261,7 @@ function iterate(ebo::ExponentialBackOff, state= (ebo.n, min(ebo.first_delay, eb
     state[1] < 1 && return nothing
     next_n = state[1]-1
     curr_delay = state[2]
-    next_delay = min(ebo.max_delay, state[2] * ebo.factor * (1.0 - ebo.jitter + (rand(Float64) * 2.0 * ebo.jitter)))
+    next_delay = min(ebo.max_delay, state[2] * ebo.factor * (1.0 - ebo.jitter + (Libc.rand(Float64) * 2.0 * ebo.jitter)))
     (curr_delay, (next_n, next_delay))
 end
 length(ebo::ExponentialBackOff) = ebo.n

base/libc.jl

@@ -255,7 +255,7 @@ time() = ccall(:jl_clock_now, Float64, ())
 
 Get Julia's process ID.
 """
-getpid() = ccall(:jl_getpid, Int32, ())
+getpid() = ccall(:uv_os_getpid, Int32, ())
 
 ## network functions ##
 
@@ -376,31 +376,35 @@ free(p::Cwstring) = free(convert(Ptr{Cwchar_t}, p))
 
 ## Random numbers ##
 
+# Access to very high quality (kernel) randomness
+function getrandom!(A::Union{Array,Base.RefValue})
+    ret = ccall(:uv_random, Cint, (Ptr{Cvoid}, Ptr{Cvoid}, Ptr{Cvoid}, Csize_t,   Cuint, Ptr{Cvoid}),
+                                   C_NULL,     C_NULL,     A,          sizeof(A), 0,     C_NULL)
+    Base.uv_error("getrandom", ret)
+    return A
+end
+_make_uint64_seed() = getrandom!(Base.RefValue{UInt64}())[]
+
 # To limit dependency on rand functionality implemented in the Random module,
-# Libc.rand is used in file.jl, and could be used in error.jl (but it breaks a test)
+# Libc.rand is used in Base (it also is independent from Random.seed, so is
+# only affected by `Libc.srand(seed)` calls)
 """
-    rand([T::Type])
+    rand([T::Type]=UInt32)
 
-Interface to the C `rand()` function. If `T` is provided, generate a value of type `T`
-by composing two calls to `rand()`. `T` can be `UInt32` or `Float64`.
+Generate a random number of type `T`. `T` can be `UInt32` or `Float64`.
 """
-rand() = ccall(:rand, Cint, ())
-@static if Sys.iswindows()
-    # Windows RAND_MAX is 2^15-1
-    rand(::Type{UInt32}) = ((rand() % UInt32) << 17) ⊻ ((rand() % UInt32) << 8) ⊻ (rand() % UInt32)
-else
-    # RAND_MAX is at least 2^15-1 in theory, but we assume 2^16-1
-    # on non-Windows systems (in practice, it's 2^31-1)
-    rand(::Type{UInt32}) = ((rand() % UInt32) << 16) ⊻ (rand() % UInt32)
-end
-rand(::Type{Float64}) = rand(UInt32) * 2.0^-32
+rand() = ccall(:jl_rand, UInt64, ()) % UInt32
+rand(::Type{UInt32}) = rand()
+rand(::Type{Float64}) = rand() * 2.0^-32
 
 """
     srand([seed])
 
-Interface to the C `srand(seed)` function.
+Set a value for the current global `seed`.
 """
-srand(seed=Base._make_uint_seed()) = ccall(:srand, Cvoid, (Cuint,), seed)
+function srand(seed::Integer=_make_uint64_seed())
+    ccall(:jl_srand, Cvoid, (UInt64,), seed % UInt64)
+end
 
 struct Cpasswd
    username::Cstring

src/coverage.cpp

@@ -201,14 +201,14 @@ extern "C" JL_DLLEXPORT void jl_write_coverage_data(const char *output)
     }
     else {
         std::string stm;
-        raw_string_ostream(stm) << "." << jl_getpid() << ".cov";
+        raw_string_ostream(stm) << "." << uv_os_getpid() << ".cov";
         write_log_data(coverageData, stm.c_str());
     }
 }
 
 extern "C" JL_DLLEXPORT void jl_write_malloc_log(void)
 {
     std::string stm;
-    raw_string_ostream(stm) << "." << jl_getpid() << ".mem";
+    raw_string_ostream(stm) << "." << uv_os_getpid() << ".mem";
     write_log_data(mallocData, stm.c_str());
 }

src/gc-debug.c

@@ -467,10 +467,9 @@ static void gc_debug_alloc_init(jl_alloc_num_t *num, const char *name)
         return;
     if (*env == 'r') {
         env++;
-        srand((unsigned)uv_hrtime());
         for (int i = 0;i < 3;i++) {
             while (num->random[i] == 0) {
-                num->random[i] = rand();
+                num->random[i] = jl_rand();
             }
         }
     }

src/init.c

@@ -683,11 +683,12 @@ JL_DLLEXPORT void julia_init(JL_IMAGE_SEARCH rel)
         jl_error("cannot generate code-coverage or track allocation information while generating a .o, .bc, or .s output file");
     }
 
+    jl_init_rand();
     jl_init_runtime_ccall();
-    jl_gc_init();
     jl_init_tasks();
     jl_init_threading();
 
+    jl_gc_init();
     jl_ptls_t ptls = jl_init_threadtls(0);
     // warning: this changes `jl_current_task`, so be careful not to call that from this function
     jl_task_t *ct = jl_init_root_task(ptls, stack_lo, stack_hi);

src/jl_exported_funcs.inc

@@ -201,7 +201,6 @@
     XX(jl_getallocationgranularity) \
     XX(jl_getnameinfo) \
     XX(jl_getpagesize) \
-    XX(jl_getpid) \
     XX(jl_get_ARCH) \
     XX(jl_get_backtrace) \
     XX(jl_get_binding) \

src/jl_uv.c

@@ -640,15 +640,6 @@ JL_DLLEXPORT void jl_exit(int exitcode)
     exit(exitcode);
 }
 
-JL_DLLEXPORT int jl_getpid(void) JL_NOTSAFEPOINT
-{
-#ifdef _OS_WINDOWS_
-    return GetCurrentProcessId();
-#else
-    return getpid();
-#endif
-}
-
 typedef union {
     struct sockaddr in;
     struct sockaddr_in v4;

src/julia.h

@@ -1872,10 +1872,7 @@ typedef struct _jl_task_t {
     jl_value_t *result;
     jl_value_t *logstate;
     jl_function_t *start;
-    uint64_t rngState0; // really rngState[4], but more convenient to split
-    uint64_t rngState1;
-    uint64_t rngState2;
-    uint64_t rngState3;
+    uint64_t rngState[4];
     _Atomic(uint8_t) _state;
     uint8_t sticky; // record whether this Task can be migrated to a new thread
     _Atomic(uint8_t) _isexception; // set if `result` is an exception to throw or that we exited with

src/julia_atomics.h

@@ -149,7 +149,7 @@ bool jl_atomic_cmpswap_explicit(std::atomic<T> *ptr, T *expected, S val, std::me
 {
      return std::atomic_compare_exchange_strong_explicit<T>(ptr, expected, val, order, order);
 }
-#define jl_atomic_cmpswap_relaxed(ptr, val) jl_atomic_cmpswap_explicit(ptr, val, memory_order_relaxed)
+#define jl_atomic_cmpswap_relaxed(ptr, expected, val) jl_atomic_cmpswap_explicit(ptr, expected, val, memory_order_relaxed)
 template<class T, class S>
 T jl_atomic_exchange(std::atomic<T> *ptr, S desired)
 {

src/julia_internal.h

@@ -1113,21 +1113,19 @@ void jl_push_excstack(jl_excstack_t **stack JL_REQUIRE_ROOTED_SLOT JL_ROOTING_AR
 //--------------------------------------------------
 // congruential random number generator
 // for a small amount of thread-local randomness
-// we could just use libc:`rand()`, but we want to ensure this is fast
-STATIC_INLINE void seed_cong(uint64_t *seed)
-{
-    *seed = rand();
-}
-STATIC_INLINE void unbias_cong(uint64_t max, uint64_t *unbias)
+STATIC_INLINE void unbias_cong(uint64_t max, uint64_t *unbias) JL_NOTSAFEPOINT
 {
     *unbias = UINT64_MAX - ((UINT64_MAX % max) + 1);
 }
-STATIC_INLINE uint64_t cong(uint64_t max, uint64_t unbias, uint64_t *seed)
+STATIC_INLINE uint64_t cong(uint64_t max, uint64_t unbias, uint64_t *seed) JL_NOTSAFEPOINT
 {
     while ((*seed = 69069 * (*seed) + 362437) > unbias)
         ;
     return *seed % max;
 }
+JL_DLLEXPORT uint64_t jl_rand(void) JL_NOTSAFEPOINT;
+JL_DLLEXPORT void jl_srand(uint64_t) JL_NOTSAFEPOINT;
+JL_DLLEXPORT void jl_init_rand(void);
 
 JL_DLLEXPORT extern void *jl_libjulia_internal_handle;
 JL_DLLEXPORT extern void *jl_RTLD_DEFAULT_handle;
@@ -1159,7 +1157,6 @@ JL_DLLEXPORT const char *jl_dlfind_win32(const char *name);
 
 // libuv wrappers:
 JL_DLLEXPORT int jl_fs_rename(const char *src_path, const char *dst_path);
-int jl_getpid(void) JL_NOTSAFEPOINT;
 
 #ifdef SEGV_EXCEPTION
 extern JL_DLLEXPORT jl_value_t *jl_segv_exception;

src/runtime_ccall.cpp

@@ -157,7 +157,7 @@ std::string jl_format_filename(StringRef output_pattern)
             }
             switch (c) {
             case 'p':
-                outfile << jl_getpid();
+                outfile << uv_os_getpid();
                 break;
             case 'd':
                 if (got_pwd)

src/signal-handling.c

@@ -345,7 +345,7 @@ JL_DLLEXPORT int jl_profile_init(size_t maxsize, uint64_t delay_nsec)
             profile_round_robin_thread_order[i] = i;
         }
     }
-    seed_cong(&profile_cong_rng_seed);
+    profile_cong_rng_seed = jl_rand();
     unbias_cong(jl_n_threads, &profile_cong_rng_unbias);
     bt_data_prof = (jl_bt_element_t*) calloc(maxsize, sizeof(jl_bt_element_t));
     if (bt_data_prof == NULL && maxsize > 0)

src/signals-unix.c

@@ -897,7 +897,7 @@ static void *signal_listener(void *arg)
                     jl_ptls_t ptls2 = jl_all_tls_states[idx];
                     nrunning += !jl_atomic_load_relaxed(&ptls2->sleep_check_state);
                 }
-                jl_safe_printf("\ncmd: %s %d running %d of %d\n", jl_options.julia_bin ? jl_options.julia_bin : "julia", jl_getpid(), nrunning, jl_n_threads);
+                jl_safe_printf("\ncmd: %s %d running %d of %d\n", jl_options.julia_bin ? jl_options.julia_bin : "julia", uv_os_getpid(), nrunning, jl_n_threads);
 #endif
 
                 jl_safe_printf("\nsignal (%d): %s\n", sig, strsignal(sig));

src/sys.c

@@ -715,6 +715,40 @@ JL_DLLEXPORT size_t jl_maxrss(void)
 #endif
 }
 
+// Simple `rand()` like function, with global seed and added thread-safety
+// (but slow and insecure)
+static _Atomic(uint64_t) g_rngseed;
+JL_DLLEXPORT uint64_t jl_rand(void) JL_NOTSAFEPOINT
+{
+    uint64_t max = UINT64_MAX;
+    uint64_t unbias = UINT64_MAX;
+    uint64_t rngseed0 = jl_atomic_load_relaxed(&g_rngseed);
+    uint64_t rngseed;
+    uint64_t rnd;
+    do {
+        rngseed = rngseed0;
+        rnd = cong(max, unbias, &rngseed);
+    } while (!jl_atomic_cmpswap_relaxed(&g_rngseed, &rngseed0, rngseed));
+    return rnd;
+}
+
+JL_DLLEXPORT void jl_srand(uint64_t rngseed) JL_NOTSAFEPOINT
+{
+    jl_atomic_store_relaxed(&g_rngseed, rngseed);
+}
+
+void jl_init_rand(void) JL_NOTSAFEPOINT
+{
+    uint64_t rngseed;
+    if (uv_random(NULL, NULL, &rngseed, sizeof(rngseed), 0, NULL)) {
+        ios_puts("WARNING: Entropy pool not available to seed RNG; using ad-hoc entropy sources.\n", ios_stderr);
+        rngseed = uv_hrtime();
+        rngseed ^= int64hash(uv_os_getpid());
+    }
+    jl_srand(rngseed);
+    srand(rngseed);
+}
+
 #ifdef __cplusplus
 }
 #endif