crypto: fix and simplify prime option validation

PR-URL: #37164
Reviewed-By: James M Snell <jasnell@gmail.com>
Reviewed-By: Rich Trott <rtrott@gmail.com>

Author: tniessen

lib/internal/crypto/random.js

@@ -391,14 +391,9 @@ function randomUUID(options) {
   return uuid.latin1Slice(0, 36);
 }
 
-function generatePrime(size, options, callback) {
-  validateUint32(size, 'size', true);
-  if (typeof options === 'function') {
-    callback = options;
-    options = {};
-  }
-  validateCallback(callback);
+function createRandomPrimeJob(type, size, options) {
   validateObject(options, 'options');
+
   const {
     safe = false,
     bigint = false,
@@ -413,7 +408,7 @@ function generatePrime(size, options, callback) {
 
   if (add !== undefined) {
     if (typeof add === 'bigint') {
-      add = Buffer.from(toHexPadded(add), 'hex');
+      add = unsignedBigIntToBuffer(add, 'options.add');
     } else if (!isAnyArrayBuffer(add) && !isArrayBufferView(add)) {
       throw new ERR_INVALID_ARG_TYPE(
         'options.add',
@@ -430,7 +425,7 @@ function generatePrime(size, options, callback) {
 
   if (rem !== undefined) {
     if (typeof rem === 'bigint') {
-      rem = Buffer.from(toHexPadded(rem), 'hex');
+      rem = unsignedBigIntToBuffer(rem, 'options.rem');
     } else if (!isAnyArrayBuffer(rem) && !isArrayBufferView(rem)) {
       throw new ERR_INVALID_ARG_TYPE(
         'options.rem',
@@ -445,7 +440,20 @@ function generatePrime(size, options, callback) {
     }
   }
 
-  const job = new RandomPrimeJob(kCryptoJobAsync, size, safe, add, rem);
+  const job = new RandomPrimeJob(type, size, safe, add, rem);
+  job.result = bigint ? arrayBufferToUnsignedBigInt : (p) => p;
+  return job;
+}
+
+function generatePrime(size, options, callback) {
+  validateUint32(size, 'size', true);
+  if (typeof options === 'function') {
+    callback = options;
+    options = {};
+  }
+  validateCallback(callback);
+
+  const job = createRandomPrimeJob(kCryptoJobAsync, size, options);
   job.ondone = (err, prime) => {
     if (err) {
       callback(err);
@@ -454,79 +462,38 @@ function generatePrime(size, options, callback) {
 
     callback(
       undefined,
-      bigint ?
-        BigInt(`0x${Buffer.from(prime).toString('hex')}`) :
-        prime);
+      job.result(prime));
   };
   job.run();
 }
 
 function generatePrimeSync(size, options = {}) {
   validateUint32(size, 'size', true);
-  validateObject(options, 'options');
-  const {
-    safe = false,
-    bigint = false,
-  } = options;
-  let {
-    add,
-    rem,
-  } = options;
-  validateBoolean(safe, 'options.safe');
-  validateBoolean(bigint, 'options.bigint');
-
-  if (add !== undefined) {
-    if (typeof add === 'bigint') {
-      add = Buffer.from(toHexPadded(add), 'hex');
-    } else if (!isAnyArrayBuffer(add) && !isArrayBufferView(add)) {
-      throw new ERR_INVALID_ARG_TYPE(
-        'options.add',
-        [
-          'ArrayBuffer',
-          'TypedArray',
-          'Buffer',
-          'DataView',
-          'bigint',
-        ],
-        add);
-    }
-  }
-
-  if (rem !== undefined) {
-    if (typeof rem === 'bigint') {
-      rem = Buffer.from(toHexPadded(rem), 'hex');
-    } else if (!isAnyArrayBuffer(rem) && !isArrayBufferView(rem)) {
-      throw new ERR_INVALID_ARG_TYPE(
-        'options.rem',
-        [
-          'ArrayBuffer',
-          'TypedArray',
-          'Buffer',
-          'DataView',
-          'bigint',
-        ],
-        rem);
-    }
-  }
 
-  const job = new RandomPrimeJob(kCryptoJobSync, size, safe, add, rem);
+  const job = createRandomPrimeJob(kCryptoJobSync, size, options);
   const [err, prime] = job.run();
   if (err)
     throw err;
+  return job.result(prime);
+}
 
-  return bigint ?
-    BigInt(`0x${Buffer.from(prime).toString('hex')}`) :
-    prime;
+function arrayBufferToUnsignedBigInt(arrayBuffer) {
+  return BigInt(`0x${Buffer.from(arrayBuffer).toString('hex')}`);
 }
 
-function toHexPadded(bigint) {
+function unsignedBigIntToBuffer(bigint, name) {
+  if (bigint < 0) {
+    throw new ERR_OUT_OF_RANGE(name, '>= 0', bigint);
+  }
+
   const hex = bigint.toString(16);
-  return hex.padStart(hex.length + (hex.length % 2), 0);
+  const padded = hex.padStart(hex.length + (hex.length % 2), 0);
+  return Buffer.from(padded, 'hex');
 }
 
 function checkPrime(candidate, options = {}, callback) {
   if (typeof candidate === 'bigint')
-    candidate = Buffer.from(toHexPadded(candidate), 'hex');
+    candidate = unsignedBigIntToBuffer(candidate, 'candidate');
   if (!isAnyArrayBuffer(candidate) && !isArrayBufferView(candidate)) {
     throw new ERR_INVALID_ARG_TYPE(
       'candidate',
@@ -559,7 +526,7 @@ function checkPrime(candidate, options = {}, callback) {
 
 function checkPrimeSync(candidate, options = {}) {
   if (typeof candidate === 'bigint')
-    candidate = Buffer.from(toHexPadded(candidate), 'hex');
+    candidate = unsignedBigIntToBuffer(candidate, 'candidate');
   if (!isAnyArrayBuffer(candidate) && !isArrayBufferView(candidate)) {
     throw new ERR_INVALID_ARG_TYPE(
       'candidate',

test/parallel/test-crypto-prime.js

@@ -71,6 +71,28 @@ const pCheckPrime = promisify(checkPrime);
   });
 });
 
+{
+  // Negative BigInts should not be converted to 0 silently.
+
+  assert.throws(() => generatePrime(20, { add: -1n }, common.mustNotCall()), {
+    code: 'ERR_OUT_OF_RANGE',
+    message: 'The value of "options.add" is out of range. It must be >= 0. ' +
+             'Received -1n'
+  });
+
+  assert.throws(() => generatePrime(20, { rem: -1n }, common.mustNotCall()), {
+    code: 'ERR_OUT_OF_RANGE',
+    message: 'The value of "options.rem" is out of range. It must be >= 0. ' +
+             'Received -1n'
+  });
+
+  assert.throws(() => checkPrime(-1n, common.mustNotCall()), {
+    code: 'ERR_OUT_OF_RANGE',
+    message: 'The value of "candidate" is out of range. It must be >= 0. ' +
+             'Received -1n'
+  });
+}
+
 generatePrime(80, common.mustSucceed((prime) => {
   assert(checkPrimeSync(prime));
   checkPrime(prime, common.mustSucceed((result) => {