Skip to content

Commit

Permalink
fix Libc.rand and seed problems (#44432)
Browse files Browse the repository at this point in the history
Continuation from #43606

 - Replaces thread-unsafe function `rand` with `jl_rand`.
 - Fixes `_ad_hoc_entropy_source` fallback in Random.
 - Uses uv_random for more direct access to quality-randomness (usually
   a syscall rather than a file.)
 - Ensures Array{Bool} are valid when created from RandomDevice.
  • Loading branch information
vtjnash authored Apr 7, 2022
1 parent badad9d commit b4bed71
Show file tree
Hide file tree
Showing 21 changed files with 120 additions and 206 deletions.
5 changes: 0 additions & 5 deletions base/Base.jl
Original file line number Diff line number Diff line change
Expand Up @@ -300,9 +300,6 @@ include("process.jl")
include("ttyhascolor.jl")
include("secretbuffer.jl")

# RandomDevice support
include("randomdevice.jl")

# core math functions
include("floatfuncs.jl")
include("math.jl")
Expand Down Expand Up @@ -492,8 +489,6 @@ end

if is_primary_base_module
function __init__()
# for the few uses of Libc.rand in Base:
Libc.srand()
# Base library init
reinit_stdio()
Multimedia.reinit_displays() # since Multimedia.displays uses stdout as fallback
Expand Down
2 changes: 1 addition & 1 deletion base/error.jl
Original file line number Diff line number Diff line change
Expand Up @@ -261,7 +261,7 @@ function iterate(ebo::ExponentialBackOff, state= (ebo.n, min(ebo.first_delay, eb
state[1] < 1 && return nothing
next_n = state[1]-1
curr_delay = state[2]
next_delay = min(ebo.max_delay, state[2] * ebo.factor * (1.0 - ebo.jitter + (rand(Float64) * 2.0 * ebo.jitter)))
next_delay = min(ebo.max_delay, state[2] * ebo.factor * (1.0 - ebo.jitter + (Libc.rand(Float64) * 2.0 * ebo.jitter)))
(curr_delay, (next_n, next_delay))
end
length(ebo::ExponentialBackOff) = ebo.n
Expand Down
38 changes: 21 additions & 17 deletions base/libc.jl
Original file line number Diff line number Diff line change
Expand Up @@ -255,7 +255,7 @@ time() = ccall(:jl_clock_now, Float64, ())
Get Julia's process ID.
"""
getpid() = ccall(:jl_getpid, Int32, ())
getpid() = ccall(:uv_os_getpid, Int32, ())

## network functions ##

Expand Down Expand Up @@ -376,31 +376,35 @@ free(p::Cwstring) = free(convert(Ptr{Cwchar_t}, p))

## Random numbers ##

# Access to very high quality (kernel) randomness
function getrandom!(A::Union{Array,Base.RefValue})
ret = ccall(:uv_random, Cint, (Ptr{Cvoid}, Ptr{Cvoid}, Ptr{Cvoid}, Csize_t, Cuint, Ptr{Cvoid}),
C_NULL, C_NULL, A, sizeof(A), 0, C_NULL)
Base.uv_error("getrandom", ret)
return A
end
_make_uint64_seed() = getrandom!(Base.RefValue{UInt64}())[]

# To limit dependency on rand functionality implemented in the Random module,
# Libc.rand is used in file.jl, and could be used in error.jl (but it breaks a test)
# Libc.rand is used in Base (it also is independent from Random.seed, so is
# only affected by `Libc.srand(seed)` calls)
"""
rand([T::Type])
rand([T::Type]=UInt32)
Interface to the C `rand()` function. If `T` is provided, generate a value of type `T`
by composing two calls to `rand()`. `T` can be `UInt32` or `Float64`.
Generate a random number of type `T`. `T` can be `UInt32` or `Float64`.
"""
rand() = ccall(:rand, Cint, ())
@static if Sys.iswindows()
# Windows RAND_MAX is 2^15-1
rand(::Type{UInt32}) = ((rand() % UInt32) << 17) ((rand() % UInt32) << 8) (rand() % UInt32)
else
# RAND_MAX is at least 2^15-1 in theory, but we assume 2^16-1
# on non-Windows systems (in practice, it's 2^31-1)
rand(::Type{UInt32}) = ((rand() % UInt32) << 16) (rand() % UInt32)
end
rand(::Type{Float64}) = rand(UInt32) * 2.0^-32
rand() = ccall(:jl_rand, UInt64, ()) % UInt32
rand(::Type{UInt32}) = rand()
rand(::Type{Float64}) = rand() * 2.0^-32

"""
srand([seed])
Interface to the C `srand(seed)` function.
Set a value for the current global `seed`.
"""
srand(seed=Base._make_uint_seed()) = ccall(:srand, Cvoid, (Cuint,), seed)
function srand(seed::Integer=_make_uint64_seed())
ccall(:jl_srand, Cvoid, (UInt64,), seed % UInt64)
end

struct Cpasswd
username::Cstring
Expand Down
77 changes: 0 additions & 77 deletions base/randomdevice.jl

This file was deleted.

4 changes: 2 additions & 2 deletions src/coverage.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -201,14 +201,14 @@ extern "C" JL_DLLEXPORT void jl_write_coverage_data(const char *output)
}
else {
std::string stm;
raw_string_ostream(stm) << "." << jl_getpid() << ".cov";
raw_string_ostream(stm) << "." << uv_os_getpid() << ".cov";
write_log_data(coverageData, stm.c_str());
}
}

extern "C" JL_DLLEXPORT void jl_write_malloc_log(void)
{
std::string stm;
raw_string_ostream(stm) << "." << jl_getpid() << ".mem";
raw_string_ostream(stm) << "." << uv_os_getpid() << ".mem";
write_log_data(mallocData, stm.c_str());
}
3 changes: 1 addition & 2 deletions src/gc-debug.c
Original file line number Diff line number Diff line change
Expand Up @@ -467,10 +467,9 @@ static void gc_debug_alloc_init(jl_alloc_num_t *num, const char *name)
return;
if (*env == 'r') {
env++;
srand((unsigned)uv_hrtime());
for (int i = 0;i < 3;i++) {
while (num->random[i] == 0) {
num->random[i] = rand();
num->random[i] = jl_rand();
}
}
}
Expand Down
3 changes: 2 additions & 1 deletion src/init.c
Original file line number Diff line number Diff line change
Expand Up @@ -683,11 +683,12 @@ JL_DLLEXPORT void julia_init(JL_IMAGE_SEARCH rel)
jl_error("cannot generate code-coverage or track allocation information while generating a .o, .bc, or .s output file");
}

jl_init_rand();
jl_init_runtime_ccall();
jl_gc_init();
jl_init_tasks();
jl_init_threading();

jl_gc_init();
jl_ptls_t ptls = jl_init_threadtls(0);
// warning: this changes `jl_current_task`, so be careful not to call that from this function
jl_task_t *ct = jl_init_root_task(ptls, stack_lo, stack_hi);
Expand Down
1 change: 0 additions & 1 deletion src/jl_exported_funcs.inc
Original file line number Diff line number Diff line change
Expand Up @@ -201,7 +201,6 @@
XX(jl_getallocationgranularity) \
XX(jl_getnameinfo) \
XX(jl_getpagesize) \
XX(jl_getpid) \
XX(jl_get_ARCH) \
XX(jl_get_backtrace) \
XX(jl_get_binding) \
Expand Down
9 changes: 0 additions & 9 deletions src/jl_uv.c
Original file line number Diff line number Diff line change
Expand Up @@ -640,15 +640,6 @@ JL_DLLEXPORT void jl_exit(int exitcode)
exit(exitcode);
}

JL_DLLEXPORT int jl_getpid(void) JL_NOTSAFEPOINT
{
#ifdef _OS_WINDOWS_
return GetCurrentProcessId();
#else
return getpid();
#endif
}

typedef union {
struct sockaddr in;
struct sockaddr_in v4;
Expand Down
5 changes: 1 addition & 4 deletions src/julia.h
Original file line number Diff line number Diff line change
Expand Up @@ -1872,10 +1872,7 @@ typedef struct _jl_task_t {
jl_value_t *result;
jl_value_t *logstate;
jl_function_t *start;
uint64_t rngState0; // really rngState[4], but more convenient to split
uint64_t rngState1;
uint64_t rngState2;
uint64_t rngState3;
uint64_t rngState[4];
_Atomic(uint8_t) _state;
uint8_t sticky; // record whether this Task can be migrated to a new thread
_Atomic(uint8_t) _isexception; // set if `result` is an exception to throw or that we exited with
Expand Down
2 changes: 1 addition & 1 deletion src/julia_atomics.h
Original file line number Diff line number Diff line change
Expand Up @@ -149,7 +149,7 @@ bool jl_atomic_cmpswap_explicit(std::atomic<T> *ptr, T *expected, S val, std::me
{
return std::atomic_compare_exchange_strong_explicit<T>(ptr, expected, val, order, order);
}
#define jl_atomic_cmpswap_relaxed(ptr, val) jl_atomic_cmpswap_explicit(ptr, val, memory_order_relaxed)
#define jl_atomic_cmpswap_relaxed(ptr, expected, val) jl_atomic_cmpswap_explicit(ptr, expected, val, memory_order_relaxed)
template<class T, class S>
T jl_atomic_exchange(std::atomic<T> *ptr, S desired)
{
Expand Down
13 changes: 5 additions & 8 deletions src/julia_internal.h
Original file line number Diff line number Diff line change
Expand Up @@ -1113,21 +1113,19 @@ void jl_push_excstack(jl_excstack_t **stack JL_REQUIRE_ROOTED_SLOT JL_ROOTING_AR
//--------------------------------------------------
// congruential random number generator
// for a small amount of thread-local randomness
// we could just use libc:`rand()`, but we want to ensure this is fast
STATIC_INLINE void seed_cong(uint64_t *seed)
{
*seed = rand();
}
STATIC_INLINE void unbias_cong(uint64_t max, uint64_t *unbias)
STATIC_INLINE void unbias_cong(uint64_t max, uint64_t *unbias) JL_NOTSAFEPOINT
{
*unbias = UINT64_MAX - ((UINT64_MAX % max) + 1);
}
STATIC_INLINE uint64_t cong(uint64_t max, uint64_t unbias, uint64_t *seed)
STATIC_INLINE uint64_t cong(uint64_t max, uint64_t unbias, uint64_t *seed) JL_NOTSAFEPOINT
{
while ((*seed = 69069 * (*seed) + 362437) > unbias)
;
return *seed % max;
}
JL_DLLEXPORT uint64_t jl_rand(void) JL_NOTSAFEPOINT;
JL_DLLEXPORT void jl_srand(uint64_t) JL_NOTSAFEPOINT;
JL_DLLEXPORT void jl_init_rand(void);

JL_DLLEXPORT extern void *jl_libjulia_internal_handle;
JL_DLLEXPORT extern void *jl_RTLD_DEFAULT_handle;
Expand Down Expand Up @@ -1159,7 +1157,6 @@ JL_DLLEXPORT const char *jl_dlfind_win32(const char *name);

// libuv wrappers:
JL_DLLEXPORT int jl_fs_rename(const char *src_path, const char *dst_path);
int jl_getpid(void) JL_NOTSAFEPOINT;

#ifdef SEGV_EXCEPTION
extern JL_DLLEXPORT jl_value_t *jl_segv_exception;
Expand Down
2 changes: 1 addition & 1 deletion src/runtime_ccall.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -157,7 +157,7 @@ std::string jl_format_filename(StringRef output_pattern)
}
switch (c) {
case 'p':
outfile << jl_getpid();
outfile << uv_os_getpid();
break;
case 'd':
if (got_pwd)
Expand Down
2 changes: 1 addition & 1 deletion src/signal-handling.c
Original file line number Diff line number Diff line change
Expand Up @@ -345,7 +345,7 @@ JL_DLLEXPORT int jl_profile_init(size_t maxsize, uint64_t delay_nsec)
profile_round_robin_thread_order[i] = i;
}
}
seed_cong(&profile_cong_rng_seed);
profile_cong_rng_seed = jl_rand();
unbias_cong(jl_n_threads, &profile_cong_rng_unbias);
bt_data_prof = (jl_bt_element_t*) calloc(maxsize, sizeof(jl_bt_element_t));
if (bt_data_prof == NULL && maxsize > 0)
Expand Down
2 changes: 1 addition & 1 deletion src/signals-unix.c
Original file line number Diff line number Diff line change
Expand Up @@ -897,7 +897,7 @@ static void *signal_listener(void *arg)
jl_ptls_t ptls2 = jl_all_tls_states[idx];
nrunning += !jl_atomic_load_relaxed(&ptls2->sleep_check_state);
}
jl_safe_printf("\ncmd: %s %d running %d of %d\n", jl_options.julia_bin ? jl_options.julia_bin : "julia", jl_getpid(), nrunning, jl_n_threads);
jl_safe_printf("\ncmd: %s %d running %d of %d\n", jl_options.julia_bin ? jl_options.julia_bin : "julia", uv_os_getpid(), nrunning, jl_n_threads);
#endif

jl_safe_printf("\nsignal (%d): %s\n", sig, strsignal(sig));
Expand Down
34 changes: 34 additions & 0 deletions src/sys.c
Original file line number Diff line number Diff line change
Expand Up @@ -715,6 +715,40 @@ JL_DLLEXPORT size_t jl_maxrss(void)
#endif
}

// Simple `rand()` like function, with global seed and added thread-safety
// (but slow and insecure)
static _Atomic(uint64_t) g_rngseed;
JL_DLLEXPORT uint64_t jl_rand(void) JL_NOTSAFEPOINT
{
uint64_t max = UINT64_MAX;
uint64_t unbias = UINT64_MAX;
uint64_t rngseed0 = jl_atomic_load_relaxed(&g_rngseed);
uint64_t rngseed;
uint64_t rnd;
do {
rngseed = rngseed0;
rnd = cong(max, unbias, &rngseed);
} while (!jl_atomic_cmpswap_relaxed(&g_rngseed, &rngseed0, rngseed));
return rnd;
}

JL_DLLEXPORT void jl_srand(uint64_t rngseed) JL_NOTSAFEPOINT
{
jl_atomic_store_relaxed(&g_rngseed, rngseed);
}

void jl_init_rand(void) JL_NOTSAFEPOINT
{
uint64_t rngseed;
if (uv_random(NULL, NULL, &rngseed, sizeof(rngseed), 0, NULL)) {
ios_puts("WARNING: Entropy pool not available to seed RNG; using ad-hoc entropy sources.\n", ios_stderr);
rngseed = uv_hrtime();
rngseed ^= int64hash(uv_os_getpid());
}
jl_srand(rngseed);
srand(rngseed);
}

#ifdef __cplusplus
}
#endif
Loading

0 comments on commit b4bed71

Please sign in to comment.