Make pubkey parsing test whether points are in the correct subgroup

Summary:
This is a partial backport of secp256k1 [[bitcoin-core/secp256k1#808 | PR808]] : bitcoin-core/secp256k1@08d7d89

Depends on D7666

Test Plan:
  ninja check-secp256k1

Reviewers: #bitcoin_abc, Fabien

Reviewed By: #bitcoin_abc, Fabien

Differential Revision: https://reviews.bitcoinabc.org/D7667

Author: sipa

src/secp256k1/src/group.h

@@ -139,4 +139,15 @@ static void secp256k1_ge_storage_cmov(secp256k1_ge_storage *r, const secp256k1_g
 /** Rescale a jacobian point by b which must be non-zero. Constant-time. */
 static void secp256k1_gej_rescale(secp256k1_gej *r, const secp256k1_fe *b);
 
+/** Determine if a point (which is assumed to be on the curve) is in the correct (sub)group of the curve.
+ *
+ * In normal mode, the used group is secp256k1, which has cofactor=1 meaning that every point on the curve is in the
+ * group, and this function returns always true.
+ *
+ * When compiling in exhaustive test mode, a slightly different curve equation is used, leading to a group with a
+ * (very) small subgroup, and that subgroup is what is used for all cryptographic operations. In that mode, this
+ * function checks whether a point that is on the curve is in fact also in that subgroup.
+ */
+static int secp256k1_ge_is_in_correct_subgroup(const secp256k1_ge* ge);
+
 #endif /* SECP256K1_GROUP_H */

src/secp256k1/src/group_impl.h

@@ -671,4 +671,25 @@ static int secp256k1_gej_has_quad_y_var(const secp256k1_gej *a) {
     return secp256k1_fe_is_quad_var(&yz);
 }
 
+static int secp256k1_ge_is_in_correct_subgroup(const secp256k1_ge* ge) {
+#ifdef EXHAUSTIVE_TEST_ORDER
+    secp256k1_gej out;
+    int i;
+
+    /* A very simple EC multiplication ladder that avoids a dependecy on ecmult. */
+    secp256k1_gej_set_infinity(&out);
+    for (i = 0; i < 32; ++i) {
+        secp256k1_gej_double_var(&out, &out, NULL);
+        if ((((uint32_t)EXHAUSTIVE_TEST_ORDER) >> (31 - i)) & 1) {
+            secp256k1_gej_add_ge_var(&out, &out, ge, NULL);
+        }
+    }
+    return secp256k1_gej_is_infinity(&out);
+#else
+    (void)ge;
+    /* The real secp256k1 group has cofactor 1, so the subgroup is the entire curve. */
+    return 1;
+#endif
+}
+
 #endif /* SECP256K1_GROUP_IMPL_H */

src/secp256k1/src/modules/extrakeys/main_impl.h

@@ -33,6 +33,9 @@ int secp256k1_xonly_pubkey_parse(const secp256k1_context* ctx, secp256k1_xonly_p
     if (!secp256k1_ge_set_xo_var(&pk, &x, 0)) {
         return 0;
     }
+    if (!secp256k1_ge_is_in_correct_subgroup(&pk)) {
+        return 0;
+    }
     secp256k1_xonly_pubkey_save(pubkey, &pk);
     return 1;
 }

src/secp256k1/src/secp256k1.c

@@ -280,6 +280,9 @@ int secp256k1_ec_pubkey_parse(const secp256k1_context* ctx, secp256k1_pubkey* pu
     if (!secp256k1_eckey_pubkey_parse(&Q, input, inputlen)) {
         return 0;
     }
+    if (!secp256k1_ge_is_in_correct_subgroup(&Q)) {
+        return 0;
+    }
     secp256k1_pubkey_save(pubkey, &Q);
     secp256k1_ge_clear(&Q);
     return 1;