crypto: avoid hang when no algorithm available

src/crypto/crypto_util.cc

@@ -65,6 +65,20 @@ MUST_USE_RESULT CSPRNGResult CSPRNG(void* buffer, size_t length) {
     if (1 == RAND_status())
       if (1 == RAND_bytes(static_cast<unsigned char*>(buffer), length))
         return {true};
+#if OPENSSL_VERSION_MAJOR >= 3
+    const auto code = ERR_peek_last_error();
+    // A misconfigured OpenSSL 3 installation may report 1 from RAND_poll()
+    // and RAND_status() but fail in RAND_bytes() if it cannot look up
+    // a matching algorithm for the CSPRNG.
+    if (ERR_GET_LIB(code) == ERR_LIB_RAND) {
+      const auto reason = ERR_GET_REASON(code);
+      if (reason == RAND_R_ERROR_INSTANTIATING_DRBG ||
+          reason == RAND_R_UNABLE_TO_FETCH_DRBG ||
+          reason == RAND_R_UNABLE_TO_CREATE_DRBG) {
+        return {false};
+      }
+    }
+#endif
   } while (1 == RAND_poll());
 
   return {false};

test/fixtures/openssl3-conf/base_only.cnf

@@ -0,0 +1,12 @@
+nodejs_conf = nodejs_init
+
+[nodejs_init]
+providers = provider_sect
+
+# List of providers to load
+[provider_sect]
+base = base_sect
+
+[base_sect]
+activate = 1
+

test/parallel/test-crypto-no-algorithm.js

@@ -0,0 +1,38 @@
+'use strict';
+
+const common = require('../common');
+if (!common.hasCrypto)
+  common.skip('missing crypto');
+
+if (!common.hasOpenSSL3)
+  common.skip('this test requires OpenSSL 3.x');
+
+const assert = require('node:assert/strict');
+const crypto = require('node:crypto');
+
+if (common.isMainThread) {
+  // TODO(richardlau): Decide if `crypto.setFips` should error if the
+  // provider namd "fips" is not available.
+  crypto.setFips(1);
+  crypto.randomBytes(20, common.mustCall((err) => {
+    // crypto.randomBytes should either succeed or fail but not hang.
+    if (err) {
+      assert.match(err.message, /digital envelope routines::unsupported/);
+      const expected = /random number generator::unable to fetch drbg/;
+      assert(err.opensslErrorStack.some((msg) => expected.test(msg)),
+             `did not find ${expected} in ${err.opensslErrorStack}`);
+    }
+  }));
+}
+
+{
+  // Startup test. Should not hang.
+  const { path } = require('../common/fixtures');
+  const { spawnSync } = require('node:child_process');
+  const baseConf = path('openssl3-conf', 'base_only.cnf');
+  const cp = spawnSync(process.execPath,
+                       [ `--openssl-config=${baseConf}`, '-p', '"hello"' ],
+                       { encoding: 'utf8' });
+  assert(common.nodeProcessAborted(cp.status, cp.signal),
+         `process did not abort, code:${cp.status} signal:${cp.signal}`);
+}