Merge branch 'main' into rm-statsfile

doc/schemas/app_openapi.json

@@ -263,7 +263,7 @@
   "info": {
     "description": "This CCF sample app implements a simple logging application, securely recording messages at client-specified IDs. It demonstrates most of the features available to CCF apps.",
     "title": "CCF Sample Logging App",
-    "version": "2.4.2"
+    "version": "2.4.3"
   },
   "openapi": "3.0.0",
   "paths": {
@@ -1186,6 +1186,30 @@
         }
       }
     },
+    "/app/log/public/cbor_merkle_proof": {
+      "get": {
+        "operationId": "GetAppLogPublicCborMerkleProof",
+        "responses": {
+          "204": {
+            "description": "Default response description"
+          },
+          "default": {
+            "$ref": "#/components/responses/default"
+          }
+        },
+        "security": [
+          {
+            "jwt": []
+          },
+          {
+            "user_cose_sign1": []
+          }
+        ],
+        "x-ccf-forwarding": {
+          "$ref": "#/components/x-ccf-forwarding/never"
+        }
+      }
+    },
     "/app/log/public/count": {
       "get": {
         "operationId": "GetAppLogPublicCount",

include/ccf/http_consts.h

@@ -36,6 +36,7 @@ namespace ccf
         static constexpr auto GRPC = "application/grpc";
         static constexpr auto COSE = "application/cose";
         static constexpr auto JAVASCRIPT = "text/javascript";
+        static constexpr auto CBOR = "application/cbor";
       }
     }
 

include/ccf/receipt.h

@@ -139,10 +139,10 @@ namespace ccf
 
   enum MerkleProofLabel : int64_t
   {
-    // Values TBD:
+    // Values set in
     // https://github.com/ietf-scitt/draft-birkholz-cose-cometre-ccf-profile
-    MERKLE_PROOF_LEAF_LABEL = 404,
-    MERKLE_PROOF_PATH_LABEL = 405
+    MERKLE_PROOF_LEAF_LABEL = 1,
+    MERKLE_PROOF_PATH_LABEL = 2
   };
   std::optional<std::vector<uint8_t>> describe_merkle_proof_v1(
     const TxReceiptImpl& in);

samples/apps/logging/logging.cpp

@@ -458,7 +458,7 @@ namespace loggingapp
         "recording messages at client-specified IDs. It demonstrates most of "
         "the features available to CCF apps.";
 
-      openapi_info.document_version = "2.4.2";
+      openapi_info.document_version = "2.4.3";
 
       index_per_public_key = std::make_shared<RecordsIndexingStrategy>(
         PUBLIC_RECORDS, context, 10000, 20);
@@ -1928,6 +1928,39 @@ namespace loggingapp
         {ccf::member_cose_sign1_auth_policy})
         .set_auto_schema<void, std::string>()
         .install();
+
+      auto get_cbor_merkle_proof =
+        [this](
+          ccf::endpoints::ReadOnlyEndpointContext& ctx,
+          ccf::historical::StatePtr historical_state) {
+          auto historical_tx = historical_state->store->create_read_only_tx();
+
+          assert(historical_state->receipt);
+          auto cbor_proof =
+            describe_merkle_proof_v1(*historical_state->receipt);
+          if (!cbor_proof.has_value())
+          {
+            ctx.rpc_ctx->set_error(
+              HTTP_STATUS_NOT_FOUND,
+              ccf::errors::ResourceNotFound,
+              "No merkle proof available for this transaction");
+            return;
+          }
+          ctx.rpc_ctx->set_response_status(HTTP_STATUS_OK);
+          ctx.rpc_ctx->set_response_body(std::move(cbor_proof.value()));
+          ctx.rpc_ctx->set_response_header(
+            ccf::http::headers::CONTENT_TYPE,
+            ccf::http::headervalues::contenttype::CBOR);
+        };
+      make_read_only_endpoint(
+        "/log/public/cbor_merkle_proof",
+        HTTP_GET,
+        ccf::historical::read_only_adapter_v4(
+          get_cbor_merkle_proof, context, is_tx_committed),
+        auth_policies)
+        .set_auto_schema<void, void>()
+        .set_forwarding_required(ccf::endpoints::ForwardingRequired::Never)
+        .install();
     }
   };
 }

src/node/historical_queries_adapter.cpp

@@ -209,7 +209,6 @@ namespace ccf
     QCBOREncodeContext ctx;
     QCBOREncode_Init(&ctx, buffer);
 
-    QCBOREncode_BstrWrap(&ctx);
     QCBOREncode_OpenMap(&ctx);
 
     if (!receipt.commit_evidence)
@@ -232,7 +231,6 @@ namespace ccf
     encode_path_cbor(ctx, *receipt.path);
 
     QCBOREncode_CloseMap(&ctx);
-    QCBOREncode_CloseBstrWrap2(&ctx, false, nullptr);
 
     struct q_useful_buf_c result;
     auto qerr = QCBOREncode_Finish(&ctx, &result);
@@ -471,7 +469,7 @@ namespace ccf::historical
         {
           ehandler(
             HistoricalQueryErrorCode::TransactionIdMissing,
-            "Could not extract TX ID",
+            "Could not extract Transaction Id",
             args);
           return;
         }

src/node/test/historical_queries.cpp

@@ -283,7 +283,6 @@ MerkleProofData decode_merkle_proof(const std::vector<uint8_t>& encoded)
   QCBORDecodeContext ctx;
   QCBORDecode_Init(&ctx, buf, QCBOR_DECODE_MODE_NORMAL);
   struct q_useful_buf_c params;
-  QCBORDecode_EnterBstrWrapped(&ctx, QCBOR_TAG_REQUIREMENT_NOT_A_TAG, &params);
   QCBORDecode_EnterMap(&ctx, NULL);
   QCBORDecode_EnterArrayFromMapN(
     &ctx, ccf::MerkleProofLabel::MERKLE_PROOF_LEAF_LABEL);
@@ -328,7 +327,6 @@ MerkleProofData decode_merkle_proof(const std::vector<uint8_t>& encoded)
 
   QCBORDecode_ExitArray(&ctx);
   QCBORDecode_ExitMap(&ctx);
-  QCBORDecode_ExitBstrWrapped(&ctx);
 
   REQUIRE(QCBORDecode_Finish(&ctx) == QCBOR_ERR_NO_MORE_ITEMS);
 

tests/e2e_logging.py

@@ -36,6 +36,7 @@
 import threading
 import copy
 import programmability
+import cbor2
 import e2e_common_endpoints
 
 from loguru import logger as LOG
@@ -909,6 +910,70 @@ def test_genesis_receipt(network, args):
     return network
 
 
+@reqs.description("Read CBOR Merkle Proof")
+def test_cbor_merkle_proof(network, args):
+    primary, _ = network.find_nodes()
+
+    with primary.client("user0") as client:
+        r = client.get("/commit")
+        assert r.status_code == http.HTTPStatus.OK
+        last_txid = TxID.from_str(r.body.json()["transaction_id"])
+
+        for seqno in range(last_txid.seqno, last_txid.seqno - 10, -1):
+            txid = f"{last_txid.view}.{seqno}"
+            LOG.debug(f"Trying to get CBOR Merkle proof for txid {txid}")
+            max_retries = 10
+            found_proof = False
+            for _ in range(max_retries):
+                r = client.get(
+                    "/log/public/cbor_merkle_proof",
+                    headers={infra.clients.CCF_TX_ID_HEADER: txid},
+                    log_capture=[],  # Do not emit raw binary to stdout
+                )
+                if r.status_code == http.HTTPStatus.OK:
+                    cbor_proof = r.body.data()
+                    proof = cbor2.loads(cbor_proof)
+                    assert 1 in proof
+                    leaf = proof[1]
+                    assert len(leaf) == 3
+                    assert isinstance(leaf[0], bytes)  # bstr write_set_digest
+                    assert len(leaf[0]) == 32
+                    assert isinstance(leaf[1], str)  # tstr commit_evidence
+                    assert len(leaf[1]) < 1024
+                    assert isinstance(leaf[2], bytes)  # bstr claims_digest
+                    assert len(leaf[2]) == 32
+                    # path
+                    assert 2 in proof
+                    path = proof[2]
+                    assert isinstance(path, list)
+                    for node in path:
+                        assert isinstance(node, list)
+                        assert len(node) == 2
+                        assert isinstance(node[0], int)
+                        assert node[0] in {0, 1}  # boolean left
+                        assert isinstance(node[1], bytes)  # bstr intermediary digest
+                        assert len(node[1]) == 32
+                    found_proof = True
+                    LOG.debug(f"Checked CBOR Merkle proof for txid {txid}")
+                    break
+                elif r.status_code == http.HTTPStatus.ACCEPTED:
+                    LOG.debug(f"Transaction {txid} accepted, retrying")
+                    time.sleep(0.1)
+                elif r.status_code == http.HTTPStatus.NOT_FOUND:
+                    LOG.debug(f"Transaction {txid} is a signature")
+                    break
+            else:
+                assert (
+                    False
+                ), f"Failed to get receipt for txid {txid} after {max_retries} retries"
+            if found_proof:
+                break
+        else:
+            assert False, "Failed to find a non-signature in the last 10 transactions"
+
+    return network
+
+
 @reqs.description("Read range of historical state")
 @reqs.supports_methods("/app/log/public", "/app/log/public/historical/range")
 def test_historical_query_range(network, args):
@@ -2089,6 +2154,9 @@ def run_main_tests(network, args):
     test_remove(network, args)
     test_clear(network, args)
     test_record_count(network, args)
+    if args.package == "samples/apps/logging/liblogging":
+        test_cbor_merkle_proof(network, args)
+
     # HTTP2 doesn't support forwarding
     if not args.http2:
         test_forwarding_frontends(network, args)