Use enums/More explicit types

.travis.yml

@@ -112,7 +112,8 @@ matrix:
             - gcc-4.9
 
     # clang for x86-64 architecture (64-bit longs and 64-bit pointers)
-    - env: SANITIZER=1 CONV_WARNINGS=1 BUILDOPTIONS='--with-cc=clang-7 --with-m64 --with-valgrind'
+    - env: SANITIZER=1 CONV_WARNINGS=strict BUILDOPTIONS='--with-cc=clang-7 --with-m64 --with-valgrind'
+    - env: SANITIZER=1 CONV_WARNINGS=relaxed BUILDOPTIONS='--with-cc=clang-7 --with-m64 --with-valgrind'
     - env: SANITIZER=1 BUILDOPTIONS='--with-cc=clang-6.0 --with-m64 --with-valgrind'
       addons:
         apt:

bn_deprecated.c

@@ -7,73 +7,73 @@
 /* SPDX-License-Identifier: Unlicense */
 #include <tommath_private.h>
 #ifdef BN_FAST_MP_INVMOD_C
-int fast_mp_invmod(const mp_int *a, const mp_int *b, mp_int *c)
+mp_err fast_mp_invmod(const mp_int *a, const mp_int *b, mp_int *c)
 {
    return s_mp_invmod_fast(a, b, c);
 }
 #endif
 #ifdef BN_FAST_MP_MONTGOMERY_REDUCE_C
-int fast_mp_montgomery_reduce(mp_int *x, const mp_int *n, mp_digit rho)
+mp_err fast_mp_montgomery_reduce(mp_int *x, const mp_int *n, mp_digit rho)
 {
    return s_mp_montgomery_reduce_fast(x, n, rho);
 }
 #endif
 #ifdef BN_FAST_S_MP_MUL_DIGS_C
-int fast_s_mp_mul_digs(const mp_int *a, const mp_int *b, mp_int *c, int digs)
+mp_err fast_s_mp_mul_digs(const mp_int *a, const mp_int *b, mp_int *c, int digs)
 {
    return s_mp_mul_digs_fast(a, b, c, digs);
 }
 #endif
 #ifdef BN_FAST_S_MP_MUL_HIGH_DIGS_C
-int fast_s_mp_mul_high_digs(const mp_int *a, const mp_int *b, mp_int *c, int digs)
+mp_err fast_s_mp_mul_high_digs(const mp_int *a, const mp_int *b, mp_int *c, int digs)
 {
    return s_mp_mul_high_digs_fast(a, b, c, digs);
 }
 #endif
 #ifdef BN_FAST_S_MP_SQR_C
-int fast_s_mp_sqr(const mp_int *a, mp_int *b)
+mp_err fast_s_mp_sqr(const mp_int *a, mp_int *b)
 {
    return s_mp_sqr_fast(a, b);
 }
 #endif
 #ifdef BN_MP_BALANCE_MUL_C
-int mp_balance_mul(const mp_int *a, const mp_int *b, mp_int *c)
+mp_err mp_balance_mul(const mp_int *a, const mp_int *b, mp_int *c)
 {
    return s_mp_balance_mul(a, b, c);
 }
 #endif
 #ifdef BN_MP_EXPTMOD_FAST_C
-int mp_exptmod_fast(const mp_int *G, const mp_int *X, const mp_int *P, mp_int *Y, int redmode)
+mp_err mp_exptmod_fast(const mp_int *G, const mp_int *X, const mp_int *P, mp_int *Y, int redmode)
 {
    return s_mp_exptmod_fast(G, X, P, Y, redmode);
 }
 #endif
 #ifdef BN_MP_INVMOD_SLOW_C
-int mp_invmod_slow(const mp_int *a, const mp_int *b, mp_int *c)
+mp_err mp_invmod_slow(const mp_int *a, const mp_int *b, mp_int *c)
 {
    return s_mp_invmod_slow(a, b, c);
 }
 #endif
 #ifdef BN_MP_KARATSUBA_MUL_C
-int mp_karatsuba_mul(const mp_int *a, const mp_int *b, mp_int *c)
+mp_err mp_karatsuba_mul(const mp_int *a, const mp_int *b, mp_int *c)
 {
    return s_mp_karatsuba_mul(a, b, c);
 }
 #endif
 #ifdef BN_MP_KARATSUBA_SQR_C
-int mp_karatsuba_sqr(const mp_int *a, mp_int *b)
+mp_err mp_karatsuba_sqr(const mp_int *a, mp_int *b)
 {
    return s_mp_karatsuba_sqr(a, b);
 }
 #endif
 #ifdef BN_MP_TOOM_MUL_C
-int mp_toom_mul(const mp_int *a, const mp_int *b, mp_int *c)
+mp_err mp_toom_mul(const mp_int *a, const mp_int *b, mp_int *c)
 {
    return s_mp_toom_mul(a, b, c);
 }
 #endif
 #ifdef BN_MP_TOOM_SQR_C
-int mp_toom_sqr(const mp_int *a, mp_int *b)
+mp_err mp_toom_sqr(const mp_int *a, mp_int *b)
 {
    return s_mp_toom_sqr(a, b);
 }

bn_mp_2expt.c

@@ -8,9 +8,9 @@
  * Simple algorithm which zeroes the int, grows it then just sets one bit
  * as required.
  */
-int mp_2expt(mp_int *a, int b)
+mp_err mp_2expt(mp_int *a, int b)
 {
-   int     res;
+   mp_err    res;
 
    /* zero a as per default */
    mp_zero(a);

bn_mp_abs.c

@@ -7,9 +7,9 @@
  *
  * Simple function copies the input and fixes the sign to positive
  */
-int mp_abs(const mp_int *a, mp_int *b)
+mp_err mp_abs(const mp_int *a, mp_int *b)
 {
-   int     res;
+   mp_err     res;
 
    /* copy a to b */
    if (a != b) {

bn_mp_add.c

@@ -4,9 +4,10 @@
 /* SPDX-License-Identifier: Unlicense */
 
 /* high level addition (handles signs) */
-int mp_add(const mp_int *a, const mp_int *b, mp_int *c)
+mp_err mp_add(const mp_int *a, const mp_int *b, mp_int *c)
 {
-   int     sa, sb, res;
+   mp_sign sa, sb;
+   mp_err res;
 
    /* get sign of both inputs */
    sa = a->sign;

bn_mp_add_d.c

@@ -4,9 +4,10 @@
 /* SPDX-License-Identifier: Unlicense */
 
 /* single digit addition */
-int mp_add_d(const mp_int *a, mp_digit b, mp_int *c)
+mp_err mp_add_d(const mp_int *a, mp_digit b, mp_int *c)
 {
-   int     res, ix, oldused;
+   mp_err     res;
+   int ix, oldused;
    mp_digit *tmpa, *tmpc, mu;
 
    /* grow c as required */

bn_mp_addmod.c

@@ -4,9 +4,9 @@
 /* SPDX-License-Identifier: Unlicense */
 
 /* d = a + b (mod c) */
-int mp_addmod(const mp_int *a, const mp_int *b, const mp_int *c, mp_int *d)
+mp_err mp_addmod(const mp_int *a, const mp_int *b, const mp_int *c, mp_int *d)
 {
-   int     res;
+   mp_err  res;
    mp_int  t;
 
    if ((res = mp_init(&t)) != MP_OKAY) {

bn_mp_and.c

@@ -4,9 +4,10 @@
 /* SPDX-License-Identifier: Unlicense */
 
 /* AND two ints together */
-int mp_and(const mp_int *a, const mp_int *b, mp_int *c)
+mp_err mp_and(const mp_int *a, const mp_int *b, mp_int *c)
 {
-   int     res, ix, px;
+   int ix, px;
+   mp_err res;
    mp_int  t;
    const mp_int *x;
 

bn_mp_cmp.c

@@ -4,7 +4,7 @@
 /* SPDX-License-Identifier: Unlicense */
 
 /* compare two ints (signed)*/
-int mp_cmp(const mp_int *a, const mp_int *b)
+mp_ord mp_cmp(const mp_int *a, const mp_int *b)
 {
    /* compare based on sign */
    if (a->sign != b->sign) {

bn_mp_cmp_d.c

@@ -4,7 +4,7 @@
 /* SPDX-License-Identifier: Unlicense */
 
 /* compare a digit */
-int mp_cmp_d(const mp_int *a, mp_digit b)
+mp_ord mp_cmp_d(const mp_int *a, mp_digit b)
 {
    /* compare based on sign */
    if (a->sign == MP_NEG) {

bn_mp_cmp_mag.c

@@ -4,7 +4,7 @@
 /* SPDX-License-Identifier: Unlicense */
 
 /* compare maginitude of two ints (unsigned) */
-int mp_cmp_mag(const mp_int *a, const mp_int *b)
+mp_ord mp_cmp_mag(const mp_int *a, const mp_int *b)
 {
    int     n;
    mp_digit *tmpa, *tmpb;

bn_mp_complement.c

@@ -4,9 +4,9 @@
 /* SPDX-License-Identifier: Unlicense */
 
 /* b = ~a */
-int mp_complement(const mp_int *a, mp_int *b)
+mp_err mp_complement(const mp_int *a, mp_int *b)
 {
-   int res = mp_neg(a, b);
+   mp_err res = mp_neg(a, b);
    return (res == MP_OKAY) ? mp_sub_d(b, 1uL, b) : res;
 }
 #endif

bn_mp_copy.c

@@ -4,9 +4,10 @@
 /* SPDX-License-Identifier: Unlicense */
 
 /* copy, b = a */
-int mp_copy(const mp_int *a, mp_int *b)
+mp_err mp_copy(const mp_int *a, mp_int *b)
 {
-   int     res, n;
+   int n;
+   mp_err res;
 
    /* if dst == src do nothing */
    if (a == b) {

bn_mp_decr.c

@@ -4,9 +4,9 @@
 /* SPDX-License-Identifier: Unlicense */
 
 /* Decrement "a" by one like "a--". Changes input! */
-int mp_decr(mp_int *a)
+mp_err mp_decr(mp_int *a)
 {
-   int e = MP_OKAY;
+   mp_err e = MP_OKAY;
    if (MP_IS_ZERO(a)) {
       mp_set(a,1uL);
       a->sign = MP_NEG;

bn_mp_div.c

@@ -6,10 +6,11 @@
 #ifdef BN_MP_DIV_SMALL
 
 /* slower bit-bang division... also smaller */
-int mp_div(const mp_int *a, const mp_int *b, mp_int *c, mp_int *d)
+mp_err mp_div(const mp_int *a, const mp_int *b, mp_int *c, mp_int *d)
 {
    mp_int ta, tb, tq, q;
-   int    res, n, n2;
+   int     n, n2;
+   mp_err res;
 
    /* is divisor zero ? */
    if (MP_IS_ZERO(b)) {
@@ -88,10 +89,12 @@ int mp_div(const mp_int *a, const mp_int *b, mp_int *c, mp_int *d)
  * The overall algorithm is as described as
  * 14.20 from HAC but fixed to treat these cases.
 */
-int mp_div(const mp_int *a, const mp_int *b, mp_int *c, mp_int *d)
+mp_err mp_div(const mp_int *a, const mp_int *b, mp_int *c, mp_int *d)
 {
    mp_int  q, x, y, t1, t2;
-   int     res, n, t, i, norm, neg;
+   int     n, t, i, norm;
+   mp_sign neg;
+   mp_err  res;
 
    /* is divisor zero ? */
    if (MP_IS_ZERO(b)) {

bn_mp_div_2.c

@@ -4,9 +4,10 @@
 /* SPDX-License-Identifier: Unlicense */
 
 /* b = a/2 */
-int mp_div_2(const mp_int *a, mp_int *b)
+mp_err mp_div_2(const mp_int *a, mp_int *b)
 {
-   int     x, res, oldused;
+   int     x, oldused;
+   mp_err res;
 
    /* copy */
    if (b->alloc < a->used) {

bn_mp_div_2d.c

@@ -4,10 +4,11 @@
 /* SPDX-License-Identifier: Unlicense */
 
 /* shift right by a certain bit count (store quotient in c, optional remainder in d) */
-int mp_div_2d(const mp_int *a, int b, mp_int *c, mp_int *d)
+mp_err mp_div_2d(const mp_int *a, int b, mp_int *c, mp_int *d)
 {
    mp_digit D, r, rr;
-   int     x, res;
+   int     x;
+   mp_err res;
 
    /* if the shift count is <= 0 then we do no work */
    if (b <= 0) {

bn_mp_div_3.c

@@ -4,12 +4,13 @@
 /* SPDX-License-Identifier: Unlicense */
 
 /* divide by three (based on routine from MPI and the GMP manual) */
-int mp_div_3(const mp_int *a, mp_int *c, mp_digit *d)
+mp_err mp_div_3(const mp_int *a, mp_int *c, mp_digit *d)
 {
    mp_int   q;
    mp_word  w, t;
    mp_digit b;
-   int      res, ix;
+   mp_err   res;
+   int      ix;
 
    /* b = 2**MP_DIGIT_BIT / 3 */
    b = ((mp_word)1 << (mp_word)MP_DIGIT_BIT) / (mp_word)3;

bn_mp_div_d.c

@@ -22,12 +22,13 @@ static int s_is_power_of_two(mp_digit b, int *p)
 }
 
 /* single digit division (based on routine from MPI) */
-int mp_div_d(const mp_int *a, mp_digit b, mp_int *c, mp_digit *d)
+mp_err mp_div_d(const mp_int *a, mp_digit b, mp_int *c, mp_digit *d)
 {
    mp_int  q;
    mp_word w;
    mp_digit t;
-   int     res, ix;
+   mp_err res;
+   int ix;
 
    /* cannot divide by zero */
    if (b == 0u) {

bn_mp_dr_is_modulus.c

@@ -4,24 +4,24 @@
 /* SPDX-License-Identifier: Unlicense */
 
 /* determines if a number is a valid DR modulus */
-int mp_dr_is_modulus(const mp_int *a)
+mp_bool mp_dr_is_modulus(const mp_int *a)
 {
    int ix;
 
    /* must be at least two digits */
    if (a->used < 2) {
-      return 0;
+      return MP_NO;
    }
 
    /* must be of the form b**k - a [a <= b] so all
     * but the first digit must be equal to -1 (mod b).
     */
    for (ix = 1; ix < a->used; ix++) {
       if (a->dp[ix] != MP_MASK) {
-         return 0;
+         return MP_NO;
       }
    }
-   return 1;
+   return MP_YES;
 }
 
 #endif

bn_mp_dr_reduce.c

@@ -17,9 +17,10 @@
  *
  * Input x must be in the range 0 <= x <= (n-1)**2
  */
-int mp_dr_reduce(mp_int *x, const mp_int *n, mp_digit k)
+mp_err mp_dr_reduce(mp_int *x, const mp_int *n, mp_digit k)
 {
-   int      err, i, m;
+   mp_err      err;
+   int i, m;
    mp_word  r;
    mp_digit mu, *tmpx1, *tmpx2;
 
@@ -58,7 +59,7 @@ int mp_dr_reduce(mp_int *x, const mp_int *n, mp_digit k)
    *tmpx1++ = mu;
 
    /* zero words above m */
-   MP_ZERO_DIGITS(tmpx1, x->used - m - 1);
+   MP_ZERO_DIGITS(tmpx1, (x->used - m) - 1);
 
    /* clamp, sub and return */
    mp_clamp(x);

bn_mp_error_to_string.c

@@ -4,7 +4,7 @@
 /* SPDX-License-Identifier: Unlicense */
 
 /* return a char * string for a given code */
-const char *mp_error_to_string(int code)
+const char *mp_error_to_string(mp_err code)
 {
    switch (code) {
    case MP_OKAY:

bn_mp_export.c

@@ -6,10 +6,10 @@
 /* based on gmp's mpz_export.
  * see http://gmplib.org/manual/Integer-Import-and-Export.html
  */
-int mp_export(void *rop, size_t *countp, int order, size_t size,
-              int endian, size_t nails, const mp_int *op)
+mp_err mp_export(void *rop, size_t *countp, int order, size_t size,
+                 int endian, size_t nails, const mp_int *op)
 {
-   int result;
+   mp_err result;
    size_t odd_nails, nail_bytes, i, j, bits, count;
    unsigned char odd_nail_mask;