Exhaustive test improvements + exhaustive schnorrsig tests

sage/gen_exhaustive_groups.sage

@@ -0,0 +1,129 @@
+# Define field size and field
+P = 2^256 - 2^32 - 977
+F = GF(P)
+BETA = F(0x7ae96a2b657c07106e64479eac3434e99cf0497512f58995c1396c28719501ee)
+
+assert(BETA != F(1) and BETA^3 == F(1))
+
+orders_done = set()
+results = {}
+first = True
+for b in range(1, P):
+    # There are only 6 curves (up to isomorphism) of the form y^2=x^3+B. Stop once we have tried all.
+    if len(orders_done) == 6:
+        break
+
+    E = EllipticCurve(F, [0, b])
+    print("Analyzing curve y^2 = x^3 + %i" % b)
+    n = E.order()
+    # Skip curves with an order we've already tried
+    if n in orders_done:
+        print("- Isomorphic to earlier curve")
+        continue
+    orders_done.add(n)
+    # Skip curves isomorphic to the real secp256k1
+    if n.is_pseudoprime():
+        print(" - Isomorphic to secp256k1")
+        continue
+
+    print("- Finding subgroups")
+
+    # Find what prime subgroups exist
+    for f, _ in n.factor():
+        print("- Analyzing subgroup of order %i" % f)
+        # Skip subgroups of order >1000
+        if f < 4 or f > 1000:
+            print("  - Bad size")
+            continue
+
+        # Iterate over X coordinates until we find one that is on the curve, has order f,
+        # and for which curve isomorphism exists that maps it to X coordinate 1.
+        for x in range(1, P):
+            # Skip X coordinates not on the curve, and construct the full point otherwise.
+            if not E.is_x_coord(x):
+                continue
+            G = E.lift_x(F(x))
+
+            print("  - Analyzing (multiples of) point with X=%i" % x)
+
+            # Skip points whose order is not a multiple of f. Project the point to have
+            # order f otherwise.
+            if (G.order() % f):
+                print("    - Bad order")
+                continue
+            G = G * (G.order() // f)
+
+            # Find lambda for endomorphism. Skip if none can be found.
+            lam = None
+            for l in Integers(f)(1).nth_root(3, all=True):
+                if int(l)*G == E(BETA*G[0], G[1]):
+                    lam = int(l)
+                    break
+            if lam is None:
+                print("    - No endomorphism for this subgroup")
+                break
+
+            # Now look for an isomorphism of the curve that gives this point an X
+            # coordinate equal to 1.
+            # If (x,y) is on y^2 = x^3 + b, then (a^2*x, a^3*y) is on y^2 = x^3 + a^6*b.
+            # So look for m=a^2=1/x.
+            m = F(1)/G[0]
+            if not m.is_square():
+                print("    - No curve isomorphism maps it to a point with X=1")
+                continue
+            a = m.sqrt()
+            rb = a^6*b
+            RE = EllipticCurve(F, [0, rb])
+
+            # Use as generator twice the image of G under the above isormorphism.
+            # This means that generator*(1/2 mod f) will have X coordinate 1.
+            RG = RE(1, a^3*G[1]) * 2
+            # And even Y coordinate.
+            if int(RG[1]) % 2:
+                RG = -RG
+            assert(RG.order() == f)
+            assert(lam*RG == RE(BETA*RG[0], RG[1]))
+
+            # We have found curve RE:y^2=x^3+rb with generator RG of order f. Remember it
+            results[f] = {"b": rb, "G": RG, "lambda": lam}
+            print("    - Found solution")
+            break
+
+    print("")
+
+print("")
+print("")
+print("/* To be put in src/group_impl.h: */")
+first = True
+for f in sorted(results.keys()):
+    b = results[f]["b"]
+    G = results[f]["G"]
+    print("#  %s EXHAUSTIVE_TEST_ORDER == %i" % ("if" if first else "elif", f))
+    first = False
+    print("static const secp256k1_ge secp256k1_ge_const_g = SECP256K1_GE_CONST(")
+    print("    0x%08x, 0x%08x, 0x%08x, 0x%08x," % tuple((int(G[0]) >> (32 * (7 - i))) & 0xffffffff for i in range(4)))
+    print("    0x%08x, 0x%08x, 0x%08x, 0x%08x," % tuple((int(G[0]) >> (32 * (7 - i))) & 0xffffffff for i in range(4, 8)))
+    print("    0x%08x, 0x%08x, 0x%08x, 0x%08x," % tuple((int(G[1]) >> (32 * (7 - i))) & 0xffffffff for i in range(4)))
+    print("    0x%08x, 0x%08x, 0x%08x, 0x%08x" % tuple((int(G[1]) >> (32 * (7 - i))) & 0xffffffff for i in range(4, 8)))
+    print(");")
+    print("static const secp256k1_fe secp256k1_fe_const_b = SECP256K1_FE_CONST(")
+    print("    0x%08x, 0x%08x, 0x%08x, 0x%08x," % tuple((int(b) >> (32 * (7 - i))) & 0xffffffff for i in range(4)))
+    print("    0x%08x, 0x%08x, 0x%08x, 0x%08x" % tuple((int(b) >> (32 * (7 - i))) & 0xffffffff for i in range(4, 8)))
+    print(");")
+print("#  else")
+print("#    error No known generator for the specified exhaustive test group order.")
+print("#  endif")
+
+print("")
+print("")
+print("/* To be put in src/scalar_impl.h: */")
+first = True
+for f in sorted(results.keys()):
+    lam = results[f]["lambda"]
+    print("#  %s EXHAUSTIVE_TEST_ORDER == %i" % ("if" if first else "elif", f))
+    first = False
+    print("#    define EXHAUSTIVE_TEST_LAMBDA %i" % lam)
+print("#  else")
+print("#    error No known lambda for the specified exhaustive test group order.")
+print("#  endif")
+print("")

src/group.h

@@ -139,4 +139,15 @@ static void secp256k1_ge_storage_cmov(secp256k1_ge_storage *r, const secp256k1_g
 /** Rescale a jacobian point by b which must be non-zero. Constant-time. */
 static void secp256k1_gej_rescale(secp256k1_gej *r, const secp256k1_fe *b);
 
+/** Determine if a point (which is assumed to be on the curve) is in the correct (sub)group of the curve.
+ *
+ * In normal mode, the used group is secp256k1, which has cofactor=1 meaning that every point on the curve is in the
+ * group, and this function returns always true.
+ *
+ * When compiling in exhaustive test mode, a slightly different curve equation is used, leading to a group with a
+ * (very) small subgroup, and that subgroup is what is used for all cryptographic operations. In that mode, this
+ * function checks whether a point that is on the curve is in fact also in that subgroup.
+ */
+static int secp256k1_ge_is_in_correct_subgroup(const secp256k1_ge* ge);
+
 #endif /* SECP256K1_GROUP_H */

src/group_impl.h

@@ -11,49 +11,38 @@
 #include "field.h"
 #include "group.h"
 
-/* These points can be generated in sage as follows:
+/* These exhaustive group test orders and generators are chosen such that:
+ * - The field size is equal to that of secp256k1, so field code is the same.
+ * - The curve equation is of the form y^2=x^3+B for some constant B.
+ * - The subgroup has a generator 2*P, where P.x=1.
+ * - The subgroup has size less than 1000 to permit exhaustive testing.
+ * - The subgroup admits an endomorphism of the form lambda*(x,y) == (beta*x,y).
  *
- * 0. Setup a worksheet with the following parameters.
- *   b = 4  # whatever CURVE_B will be set to
- *   F = FiniteField (0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFEFFFFFC2F)
- *   C = EllipticCurve ([F (0), F (b)])
- *
- * 1. Determine all the small orders available to you. (If there are
- *    no satisfactory ones, go back and change b.)
- *   print C.order().factor(limit=1000)
- *
- * 2. Choose an order as one of the prime factors listed in the above step.
- *    (You can also multiply some to get a composite order, though the
- *    tests will crash trying to invert scalars during signing.) We take a
- *    random point and scale it to drop its order to the desired value.
- *    There is some probability this won't work; just try again.
- *   order = 199
- *   P = C.random_point()
- *   P = (int(P.order()) / int(order)) * P
- *   assert(P.order() == order)
- *
- * 3. Print the values. You'll need to use a vim macro or something to
- *    split the hex output into 4-byte chunks.
- *   print "%x %x" % P.xy()
+ * These parameters are generated using sage/gen_exhaustive_groups.sage.
  */
 #if defined(EXHAUSTIVE_TEST_ORDER)
-#  if EXHAUSTIVE_TEST_ORDER == 199
+#  if EXHAUSTIVE_TEST_ORDER == 13
 static const secp256k1_ge secp256k1_ge_const_g = SECP256K1_GE_CONST(
-    0xFA7CC9A7, 0x0737F2DB, 0xA749DD39, 0x2B4FB069,
-    0x3B017A7D, 0xA808C2F1, 0xFB12940C, 0x9EA66C18,
-    0x78AC123A, 0x5ED8AEF3, 0x8732BC91, 0x1F3A2868,
-    0x48DF246C, 0x808DAE72, 0xCFE52572, 0x7F0501ED
+    0xc3459c3d, 0x35326167, 0xcd86cce8, 0x07a2417f,
+    0x5b8bd567, 0xde8538ee, 0x0d507b0c, 0xd128f5bb,
+    0x8e467fec, 0xcd30000a, 0x6cc1184e, 0x25d382c2,
+    0xa2f4494e, 0x2fbe9abc, 0x8b64abac, 0xd005fb24
 );
-
-static const int CURVE_B = 4;
-#  elif EXHAUSTIVE_TEST_ORDER == 13
+static const secp256k1_fe secp256k1_fe_const_b = SECP256K1_FE_CONST(
+    0x3d3486b2, 0x159a9ca5, 0xc75638be, 0xb23a69bc,
+    0x946a45ab, 0x24801247, 0xb4ed2b8e, 0x26b6a417
+);
+#  elif EXHAUSTIVE_TEST_ORDER == 199
 static const secp256k1_ge secp256k1_ge_const_g = SECP256K1_GE_CONST(
-    0xedc60018, 0xa51a786b, 0x2ea91f4d, 0x4c9416c0,
-    0x9de54c3b, 0xa1316554, 0x6cf4345c, 0x7277ef15,
-    0x54cb1b6b, 0xdc8c1273, 0x087844ea, 0x43f4603e,
-    0x0eaf9a43, 0xf6effe55, 0x939f806d, 0x37adf8ac
+    0x226e653f, 0xc8df7744, 0x9bacbf12, 0x7d1dcbf9,
+    0x87f05b2a, 0xe7edbd28, 0x1f564575, 0xc48dcf18,
+    0xa13872c2, 0xe933bb17, 0x5d9ffd5b, 0xb5b6e10c,
+    0x57fe3c00, 0xbaaaa15a, 0xe003ec3e, 0x9c269bae
+);
+static const secp256k1_fe secp256k1_fe_const_b = SECP256K1_FE_CONST(
+    0x2cca28fa, 0xfc614b80, 0x2a3db42b, 0x00ba00b1,
+    0xbea8d943, 0xdace9ab2, 0x9536daea, 0x0074defb
 );
-static const int CURVE_B = 2;
 #  else
 #    error No known generator for the specified exhaustive test group order.
 #  endif
@@ -68,7 +57,7 @@ static const secp256k1_ge secp256k1_ge_const_g = SECP256K1_GE_CONST(
     0xFD17B448UL, 0xA6855419UL, 0x9C47D08FUL, 0xFB10D4B8UL
 );
 
-static const int CURVE_B = 7;
+static const secp256k1_fe secp256k1_fe_const_b = SECP256K1_FE_CONST(0, 0, 0, 0, 0, 0, 0, 7);
 #endif
 
 static void secp256k1_ge_set_gej_zinv(secp256k1_ge *r, const secp256k1_gej *a, const secp256k1_fe *zi) {
@@ -219,14 +208,13 @@ static void secp256k1_ge_clear(secp256k1_ge *r) {
 }
 
 static int secp256k1_ge_set_xquad(secp256k1_ge *r, const secp256k1_fe *x) {
-    secp256k1_fe x2, x3, c;
+    secp256k1_fe x2, x3;
     r->x = *x;
     secp256k1_fe_sqr(&x2, x);
     secp256k1_fe_mul(&x3, x, &x2);
     r->infinity = 0;
-    secp256k1_fe_set_int(&c, CURVE_B);
-    secp256k1_fe_add(&c, &x3);
-    return secp256k1_fe_sqrt(&r->y, &c);
+    secp256k1_fe_add(&x3, &secp256k1_fe_const_b);
+    return secp256k1_fe_sqrt(&r->y, &x3);
 }
 
 static int secp256k1_ge_set_xo_var(secp256k1_ge *r, const secp256k1_fe *x, int odd) {
@@ -269,36 +257,15 @@ static int secp256k1_gej_is_infinity(const secp256k1_gej *a) {
     return a->infinity;
 }
 
-static int secp256k1_gej_is_valid_var(const secp256k1_gej *a) {
-    secp256k1_fe y2, x3, z2, z6;
-    if (a->infinity) {
-        return 0;
-    }
-    /** y^2 = x^3 + 7
-     *  (Y/Z^3)^2 = (X/Z^2)^3 + 7
-     *  Y^2 / Z^6 = X^3 / Z^6 + 7
-     *  Y^2 = X^3 + 7*Z^6
-     */
-    secp256k1_fe_sqr(&y2, &a->y);
-    secp256k1_fe_sqr(&x3, &a->x); secp256k1_fe_mul(&x3, &x3, &a->x);
-    secp256k1_fe_sqr(&z2, &a->z);
-    secp256k1_fe_sqr(&z6, &z2); secp256k1_fe_mul(&z6, &z6, &z2);
-    secp256k1_fe_mul_int(&z6, CURVE_B);
-    secp256k1_fe_add(&x3, &z6);
-    secp256k1_fe_normalize_weak(&x3);
-    return secp256k1_fe_equal_var(&y2, &x3);
-}
-
 static int secp256k1_ge_is_valid_var(const secp256k1_ge *a) {
-    secp256k1_fe y2, x3, c;
+    secp256k1_fe y2, x3;
     if (a->infinity) {
         return 0;
     }
     /* y^2 = x^3 + 7 */
     secp256k1_fe_sqr(&y2, &a->y);
     secp256k1_fe_sqr(&x3, &a->x); secp256k1_fe_mul(&x3, &x3, &a->x);
-    secp256k1_fe_set_int(&c, CURVE_B);
-    secp256k1_fe_add(&x3, &c);
+    secp256k1_fe_add(&x3, &secp256k1_fe_const_b);
     secp256k1_fe_normalize_weak(&x3);
     return secp256k1_fe_equal_var(&y2, &x3);
 }
@@ -704,4 +671,25 @@ static int secp256k1_gej_has_quad_y_var(const secp256k1_gej *a) {
     return secp256k1_fe_is_quad_var(&yz);
 }
 
+static int secp256k1_ge_is_in_correct_subgroup(const secp256k1_ge* ge) {
+#ifdef EXHAUSTIVE_TEST_ORDER
+    secp256k1_gej out;
+    int i;
+
+    /* A very simple EC multiplication ladder that avoids a dependecy on ecmult. */
+    secp256k1_gej_set_infinity(&out);
+    for (i = 0; i < 32; ++i) {
+        secp256k1_gej_double_var(&out, &out, NULL);
+        if ((((uint32_t)EXHAUSTIVE_TEST_ORDER) >> (31 - i)) & 1) {
+            secp256k1_gej_add_ge_var(&out, &out, ge, NULL);
+        }
+    }
+    return secp256k1_gej_is_infinity(&out);
+#else
+    (void)ge;
+    /* The real secp256k1 group has cofactor 1, so the subgroup is the entire curve. */
+    return 1;
+#endif
+}
+
 #endif /* SECP256K1_GROUP_IMPL_H */

src/modules/extrakeys/Makefile.am.include

@@ -1,3 +1,4 @@
 include_HEADERS += include/secp256k1_extrakeys.h
 noinst_HEADERS += src/modules/extrakeys/tests_impl.h
+noinst_HEADERS += src/modules/extrakeys/tests_exhaustive_impl.h
 noinst_HEADERS += src/modules/extrakeys/main_impl.h

src/modules/extrakeys/main_impl.h

@@ -33,6 +33,9 @@ int secp256k1_xonly_pubkey_parse(const secp256k1_context* ctx, secp256k1_xonly_p
     if (!secp256k1_ge_set_xo_var(&pk, &x, 0)) {
         return 0;
     }
+    if (!secp256k1_ge_is_in_correct_subgroup(&pk)) {
+        return 0;
+    }
     secp256k1_xonly_pubkey_save(pubkey, &pk);
     return 1;
 }

src/modules/extrakeys/tests_exhaustive_impl.h

@@ -0,0 +1,68 @@
+/**********************************************************************
+ * Copyright (c) 2020 Pieter Wuille                                   *
+ * Distributed under the MIT software license, see the accompanying   *
+ * file COPYING or http://www.opensource.org/licenses/mit-license.php.*
+ **********************************************************************/
+
+#ifndef _SECP256K1_MODULE_EXTRAKEYS_TESTS_EXHAUSTIVE_
+#define _SECP256K1_MODULE_EXTRAKEYS_TESTS_EXHAUSTIVE_
+
+#include "src/modules/extrakeys/main_impl.h"
+#include "include/secp256k1_extrakeys.h"
+
+static void test_exhaustive_extrakeys(const secp256k1_context *ctx, const secp256k1_ge* group) {
+    secp256k1_keypair keypair[EXHAUSTIVE_TEST_ORDER - 1];
+    secp256k1_pubkey pubkey[EXHAUSTIVE_TEST_ORDER - 1];
+    secp256k1_xonly_pubkey xonly_pubkey[EXHAUSTIVE_TEST_ORDER - 1];
+    int parities[EXHAUSTIVE_TEST_ORDER - 1];
+    unsigned char xonly_pubkey_bytes[EXHAUSTIVE_TEST_ORDER - 1][32];
+    int i;
+
+    for (i = 1; i < EXHAUSTIVE_TEST_ORDER; i++) {
+        secp256k1_fe fe;
+        secp256k1_scalar scalar_i;
+        unsigned char buf[33];
+        int parity;
+
+        secp256k1_scalar_set_int(&scalar_i, i);
+        secp256k1_scalar_get_b32(buf, &scalar_i);
+
+        /* Construct pubkey and keypair. */
+        CHECK(secp256k1_keypair_create(ctx, &keypair[i - 1], buf));
+        CHECK(secp256k1_ec_pubkey_create(ctx, &pubkey[i - 1], buf));
+
+        /* Construct serialized xonly_pubkey from keypair. */
+        CHECK(secp256k1_keypair_xonly_pub(ctx, &xonly_pubkey[i - 1], &parities[i - 1], &keypair[i - 1]));
+        CHECK(secp256k1_xonly_pubkey_serialize(ctx, xonly_pubkey_bytes[i - 1], &xonly_pubkey[i - 1]));
+
+        /* Parse the xonly_pubkey back and verify it matches the previously serialized value. */
+        CHECK(secp256k1_xonly_pubkey_parse(ctx, &xonly_pubkey[i - 1], xonly_pubkey_bytes[i - 1]));
+        CHECK(secp256k1_xonly_pubkey_serialize(ctx, buf, &xonly_pubkey[i - 1]));
+        CHECK(memcmp(xonly_pubkey_bytes[i - 1], buf, 32) == 0);
+
+        /* Construct the xonly_pubkey from the pubkey, and verify it matches the same. */
+        CHECK(secp256k1_xonly_pubkey_from_pubkey(ctx, &xonly_pubkey[i - 1], &parity, &pubkey[i - 1]));
+        CHECK(parity == parities[i - 1]);
+        CHECK(secp256k1_xonly_pubkey_serialize(ctx, buf, &xonly_pubkey[i - 1]));
+        CHECK(memcmp(xonly_pubkey_bytes[i - 1], buf, 32) == 0);
+
+        /* Compare the xonly_pubkey bytes against the precomputed group. */
+        secp256k1_fe_set_b32(&fe, xonly_pubkey_bytes[i - 1]);
+        CHECK(secp256k1_fe_equal_var(&fe, &group[i].x));
+
+        /* Check the parity against the precomputed group. */
+        fe = group[i].y;
+        secp256k1_fe_normalize_var(&fe);
+        CHECK(secp256k1_fe_is_odd(&fe) == parities[i - 1]);
+
+        /* Verify that the higher half is identical to the lower half mirrored. */
+        if (i > EXHAUSTIVE_TEST_ORDER / 2) {
+            CHECK(memcmp(xonly_pubkey_bytes[i - 1], xonly_pubkey_bytes[EXHAUSTIVE_TEST_ORDER - i - 1], 32) == 0);
+            CHECK(parities[i - 1] == 1 - parities[EXHAUSTIVE_TEST_ORDER - i - 1]);
+        }
+    }
+
+    /* TODO: keypair/xonly_pubkey tweak tests */
+}
+
+#endif