diff --git a/CHANGELOG.md b/CHANGELOG.md index 9a371bffc161..ee64883cbd64 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,9 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ### Added +- Provided API for getting COSE signatures and Merkle proofs (#6477). + - Exposed COSE signature in historical API via `TxReceiptImpl`. + - Introduced `ccf::describe_merkle_proof_v1(receipt)` for Merkle proof construction in CBOR format. - Added COSE signatures over the Merkle root to the KV (#6449). - Signing is done with service key (different from raw signatures, which remain unchanged and are still signed by the node key). - New signature reside in `public:ccf.internal.cose_signatures`. diff --git a/CMakeLists.txt b/CMakeLists.txt index 75eb5d2fd658..53dd25dc6c27 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -826,6 +826,7 @@ if(BUILD_TESTS) ) target_link_libraries( historical_queries_test PRIVATE http_parser.host sss.host ccf_kv.host + ccf_endpoints.host ) # Temporarily disabled flaky test # https://github.com/microsoft/CCF/issues/4403 add_unit_test( indexing_test diff --git a/include/ccf/receipt.h b/include/ccf/receipt.h index 31b13bfa53bf..a7df79888030 100644 --- a/include/ccf/receipt.h +++ b/include/ccf/receipt.h @@ -137,6 +137,16 @@ namespace ccf nlohmann::json describe_receipt_v1(const TxReceiptImpl& receipt); ReceiptPtr describe_receipt_v2(const TxReceiptImpl& receipt); + enum MerkleProofLabel : int64_t + { + // Values TBD: + // https://github.com/ietf-scitt/draft-birkholz-cose-cometre-ccf-profile + MERKLE_PROOF_LEAF_LABEL = 404, + MERKLE_PROOF_PATH_LABEL = 405 + }; + std::optional> describe_merkle_proof_v1( + const TxReceiptImpl& in); + // Manual JSON serializers are specified for these types as they are not // trivial POD structs diff --git a/src/crypto/openssl/cose_sign.cpp b/src/crypto/openssl/cose_sign.cpp index e3719b8a55b0..0097f013e729 100644 --- a/src/crypto/openssl/cose_sign.cpp +++ b/src/crypto/openssl/cose_sign.cpp @@ -69,9 +69,6 @@ namespace QCBOREncodeContext* cbor_encode, const std::vector& protected_headers) { - QCBOREncode_AddTag(cbor_encode, CBOR_TAG_COSE_SIGN1); - QCBOREncode_OpenArray(cbor_encode); - encode_protected_headers(me, cbor_encode, protected_headers); QCBOREncode_OpenMap(cbor_encode); @@ -164,7 +161,8 @@ namespace ccf::crypto std::span payload) { const auto buf_size = estimate_buffer_size(protected_headers, payload); - Q_USEFUL_BUF_MAKE_STACK_UB(signed_cose_buffer, buf_size); + std::vector underlying_buffer(buf_size); + q_useful_buf signed_cose_buffer{underlying_buffer.data(), buf_size}; QCBOREncodeContext cbor_encode; QCBOREncode_Init(&cbor_encode, signed_cose_buffer); @@ -185,6 +183,9 @@ namespace ccf::crypto t_cose_sign1_set_signing_key(&sign_ctx, signing_key, NULL_Q_USEFUL_BUF_C); + QCBOREncode_AddTag(&cbor_encode, CBOR_TAG_COSE_SIGN1); + QCBOREncode_OpenArray(&cbor_encode); + encode_parameters_custom(&sign_ctx, &cbor_encode, protected_headers); // Mark empty payload manually. @@ -216,8 +217,12 @@ namespace ccf::crypto fmt::format("Can't finish QCBOR encoding with error code {}", err)); } - return { - static_cast(signed_cose.ptr), - static_cast(signed_cose.ptr) + signed_cose.len}; + // Memory address is said to match: + // github.com/laurencelundblade/QCBOR/blob/v1.4.1/inc/qcbor/qcbor_encode.h#L2190-L2191 + assert(signed_cose.ptr == underlying_buffer.data()); + + underlying_buffer.resize(signed_cose.len); + underlying_buffer.shrink_to_fit(); + return underlying_buffer; } } diff --git a/src/node/historical_queries.h b/src/node/historical_queries.h index 2cc96a976ec6..5c987ce37f2d 100644 --- a/src/node/historical_queries.h +++ b/src/node/historical_queries.h @@ -74,6 +74,14 @@ namespace ccf::historical return signatures->get(); } + static std::optional get_cose_signature( + const ccf::kv::StorePtr& sig_store) + { + auto tx = sig_store->create_read_only_tx(); + auto signatures = tx.ro(ccf::Tables::COSE_SIGNATURES); + return signatures->get(); + } + static std::optional> get_tree( const ccf::kv::StorePtr& sig_store) { @@ -430,6 +438,7 @@ namespace ccf::historical // Iterate through earlier indices. If this signature covers them // then create a receipt for them const auto sig = get_signature(sig_details->store); + const auto cose_sig = get_cose_signature(sig_details->store); ccf::MerkleTreeHistory tree(get_tree(sig_details->store).value()); // This is either pointing at the sig itself, or the closest larger @@ -453,6 +462,7 @@ namespace ccf::historical details->transaction_id = {sig->view, seqno}; details->receipt = std::make_shared( sig->sig, + cose_sig, proof.get_root(), proof.get_path(), sig->node, @@ -735,10 +745,11 @@ namespace ccf::historical // the receipt _later_ for an already-fetched signature // transaction. const auto sig = get_signature(details->store); + const auto cose_sig = get_cose_signature(details->store); assert(sig.has_value()); details->transaction_id = {sig->view, sig->seqno}; details->receipt = std::make_shared( - sig->sig, sig->root.h, nullptr, sig->node, sig->cert); + sig->sig, cose_sig, sig->root.h, nullptr, sig->node, sig->cert); } auto request_it = requests.begin(); diff --git a/src/node/historical_queries_adapter.cpp b/src/node/historical_queries_adapter.cpp index 260aaabcaa9d..2732c394d307 100644 --- a/src/node/historical_queries_adapter.cpp +++ b/src/node/historical_queries_adapter.cpp @@ -10,6 +10,51 @@ #include "node/rpc/network_identity_subsystem.h" #include "node/tx_receipt_impl.h" +#include + +namespace +{ + void encode_leaf_cbor( + QCBOREncodeContext& ctx, const ccf::TxReceiptImpl& receipt) + { + QCBOREncode_OpenArrayInMapN( + &ctx, ccf::MerkleProofLabel::MERKLE_PROOF_LEAF_LABEL); + + // 1 WSD + const auto& wsd = receipt.write_set_digest->h; + QCBOREncode_AddBytes(&ctx, {wsd.data(), wsd.size()}); + + // 2. CE + const auto& ce = receipt.commit_evidence.value(); + QCBOREncode_AddSZString(&ctx, ce.data()); + + // 3. CD + const auto& cd = receipt.claims_digest.value().h; + QCBOREncode_AddBytes(&ctx, {cd.data(), cd.size()}); + + QCBOREncode_CloseArray(&ctx); + } + + void encode_path_cbor( + QCBOREncodeContext& ctx, const ccf::HistoryTree::Path& path) + { + QCBOREncode_OpenArrayInMapN( + &ctx, ccf::MerkleProofLabel::MERKLE_PROOF_PATH_LABEL); + for (const auto& node : path) + { + const int64_t dir = + (node.direction == ccf::HistoryTree::Path::Direction::PATH_LEFT); + std::vector hash{node.hash}; + + QCBOREncode_OpenArray(&ctx); + QCBOREncode_AddInt64(&ctx, dir); + QCBOREncode_AddBytes(&ctx, {hash.data(), hash.size()}); + QCBOREncode_CloseArray(&ctx); + } + QCBOREncode_CloseArray(&ctx); + } +} + namespace ccf { nlohmann::json describe_receipt_v1(const TxReceiptImpl& receipt) @@ -153,6 +198,58 @@ namespace ccf return receipt; } + + std::optional> describe_merkle_proof_v1( + const TxReceiptImpl& receipt) + { + constexpr size_t buf_size = 2048; + std::vector underlying_buffer(buf_size); + q_useful_buf buffer{underlying_buffer.data(), buf_size}; + + QCBOREncodeContext ctx; + QCBOREncode_Init(&ctx, buffer); + + QCBOREncode_BstrWrap(&ctx); + QCBOREncode_OpenMap(&ctx); + + if (!receipt.commit_evidence) + { + LOG_DEBUG_FMT("Merkle proof is missing commit evidence"); + return std::nullopt; + } + if (!receipt.write_set_digest) + { + LOG_DEBUG_FMT("Merkle proof is missing write set digest"); + return std::nullopt; + } + encode_leaf_cbor(ctx, receipt); + + if (!receipt.path) + { + LOG_DEBUG_FMT("Merkle proof is missing path"); + return std::nullopt; + } + encode_path_cbor(ctx, *receipt.path); + + QCBOREncode_CloseMap(&ctx); + QCBOREncode_CloseBstrWrap2(&ctx, false, nullptr); + + struct q_useful_buf_c result; + auto qerr = QCBOREncode_Finish(&ctx, &result); + if (qerr) + { + LOG_DEBUG_FMT("Failed to encode merkle proof: {}", qerr); + return std::nullopt; + } + + // Memory address is said to match: + // github.com/laurencelundblade/QCBOR/blob/v1.4.1/inc/qcbor/qcbor_encode.h#L2190-L2191 + assert(result.ptr == underlying_buffer.data()); + + underlying_buffer.resize(result.len); + underlying_buffer.shrink_to_fit(); + return underlying_buffer; + } } namespace ccf::historical diff --git a/src/node/snapshot_serdes.h b/src/node/snapshot_serdes.h index c768987043bb..f4ccbc8b2acd 100644 --- a/src/node/snapshot_serdes.h +++ b/src/node/snapshot_serdes.h @@ -161,6 +161,7 @@ namespace ccf cd.set(std::move(claims_digest)); ccf::TxReceiptImpl tx_receipt( sig, + std::nullopt, // cose proof.get_root(), proof.get_path(), node_id, diff --git a/src/node/test/historical_queries.cpp b/src/node/test/historical_queries.cpp index 501ebe1e1a52..109d68240da3 100644 --- a/src/node/test/historical_queries.cpp +++ b/src/node/test/historical_queries.cpp @@ -9,6 +9,7 @@ #include "ccf/crypto/rsa_key_pair.h" #include "ccf/pal/locking.h" +#include "ccf/receipt.h" #include "crypto/openssl/hash.h" #include "ds/messaging.h" #include "ds/test/stub_writer.h" @@ -254,6 +255,86 @@ size_t get_cache_limit_for_entries( }); } +struct MerkleProofData +{ + std::vector write_set_digest; + std::string commit_evidence; + std::vector claims_digest; + std::vector>> path; +}; + +std::vector bstring_to_bytes(QCBORItem& item) +{ + return { + static_cast(item.val.string.ptr), + static_cast(item.val.string.ptr) + item.val.string.len}; +} + +std::string tstring_to_string(QCBORItem& item) +{ + return { + static_cast(item.val.string.ptr), + static_cast(item.val.string.ptr) + item.val.string.len}; +} + +MerkleProofData decode_merkle_proof(const std::vector& encoded) +{ + q_useful_buf_c buf{encoded.data(), encoded.size()}; + QCBORDecodeContext ctx; + QCBORDecode_Init(&ctx, buf, QCBOR_DECODE_MODE_NORMAL); + struct q_useful_buf_c params; + QCBORDecode_EnterBstrWrapped(&ctx, QCBOR_TAG_REQUIREMENT_NOT_A_TAG, ¶ms); + QCBORDecode_EnterMap(&ctx, NULL); + QCBORDecode_EnterArrayFromMapN( + &ctx, ccf::MerkleProofLabel::MERKLE_PROOF_LEAF_LABEL); + QCBORItem item; + MerkleProofData data; + + QCBORDecode_GetNext(&ctx, &item); + REQUIRE(item.uDataType == QCBOR_TYPE_BYTE_STRING); + data.write_set_digest = bstring_to_bytes(item); + + QCBORDecode_GetNext(&ctx, &item); + REQUIRE(item.uDataType == QCBOR_TYPE_TEXT_STRING); + data.commit_evidence = tstring_to_string(item); + + QCBORDecode_GetNext(&ctx, &item); + REQUIRE(item.uDataType == QCBOR_TYPE_BYTE_STRING); + data.claims_digest = bstring_to_bytes(item); + + QCBORDecode_ExitArray(&ctx); + QCBORDecode_EnterArrayFromMapN( + &ctx, ccf::MerkleProofLabel::MERKLE_PROOF_PATH_LABEL); + + for (;;) + { + QCBORDecode_EnterArray(&ctx, &item); + if (QCBORDecode_GetError(&ctx) != QCBOR_SUCCESS) + break; + + std::pair> path_item; + + REQUIRE(QCBORDecode_GetNext(&ctx, &item) == QCBOR_SUCCESS); + REQUIRE(item.uDataType == QCBOR_TYPE_INT64); + path_item.first = item.val.int64; + + REQUIRE(QCBORDecode_GetNext(&ctx, &item) == QCBOR_SUCCESS); + REQUIRE(item.uDataType == QCBOR_TYPE_BYTE_STRING); + path_item.second = bstring_to_bytes(item); + + data.path.push_back(path_item); + QCBORDecode_ExitArray(&ctx); + } + + QCBORDecode_ExitArray(&ctx); + QCBORDecode_ExitMap(&ctx); + QCBORDecode_ExitBstrWrapped(&ctx); + + REQUIRE(QCBORDecode_Finish(&ctx) == QCBOR_ERR_NO_MORE_ITEMS); + + return data; +} + TEST_CASE("StateCache point queries") { auto state = create_and_init_state(); @@ -1855,6 +1936,66 @@ TEST_CASE("Recover historical ledger secrets") ccf::crypto::openssl_sha256_shutdown(); } +TEST_CASE("Valid merkle proof from receipts") +{ + ccf::crypto::openssl_sha256_init(); + auto state = create_and_init_state(); + auto& kv_store = *state.kv_store; + auto sigseq = write_transactions_and_signature(kv_store, 10); + auto ledger = construct_host_ledger(kv_store.get_consensus()); + REQUIRE(ledger.size() == sigseq); + + auto writer = std::make_shared(); + ccf::historical::StateCache cache(kv_store, state.ledger_secrets, writer); + + const auto target = sigseq - 1; + REQUIRE(cache.get_state_at(target, target) == nullptr); + REQUIRE(cache.handle_ledger_entry(target, ledger.at(target))); + REQUIRE(cache.handle_ledger_entry(sigseq, ledger.at(sigseq))); + auto historical_state = cache.get_state_at(target, target); + REQUIRE(historical_state != nullptr); + REQUIRE(historical_state->receipt != nullptr); + + auto proof = ccf::describe_merkle_proof_v1(*historical_state->receipt); + REQUIRE(proof.has_value()); + + auto decoded = decode_merkle_proof(*proof); + + REQUIRE_EQ( + ccf::ds::to_hex(decoded.write_set_digest), + historical_state->receipt->write_set_digest->hex_str()); + REQUIRE_EQ( + decoded.commit_evidence, *historical_state->receipt->commit_evidence); + REQUIRE_EQ( + ccf::ds::to_hex(decoded.claims_digest), + historical_state->receipt->claims_digest.value() + .hex_str()); // HEX as workaround emmpy claims (set flag). + + auto it = decoded.path.begin(); + for (const auto& node : *historical_state->receipt->path) + { + const int64_t dir = + (node.direction == ccf::HistoryTree::Path::Direction::PATH_LEFT); + std::vector hash{node.hash}; + + REQUIRE_EQ(it->first, dir); + REQUIRE_EQ(it->second, hash); + + ++it; + } + + REQUIRE(it == decoded.path.end()); + + historical_state = cache.get_state_at(sigseq, sigseq); + REQUIRE(historical_state != nullptr); + + // We don't provide a merkle proof for the signature itself. + proof = ccf::describe_merkle_proof_v1(*historical_state->receipt); + REQUIRE_FALSE(proof.has_value()); + + ccf::crypto::openssl_sha256_shutdown(); +} + int main(int argc, char** argv) { threading::ThreadMessaging::init(1); diff --git a/src/node/tx_receipt_impl.h b/src/node/tx_receipt_impl.h index c7ad613f84c6..73c83c7f3bd4 100644 --- a/src/node/tx_receipt_impl.h +++ b/src/node/tx_receipt_impl.h @@ -12,6 +12,7 @@ namespace ccf struct TxReceiptImpl { std::vector signature = {}; + std::optional> cose_signature = std::nullopt; HistoryTree::Hash root = {}; std::shared_ptr path = {}; ccf::NodeId node_id = {}; @@ -24,6 +25,7 @@ namespace ccf TxReceiptImpl( const std::vector& signature_, + const std::optional>& cose_signature, const HistoryTree::Hash& root_, std::shared_ptr path_, const NodeId& node_id_, @@ -37,6 +39,7 @@ namespace ccf const std::optional>& service_endorsements_ = std::nullopt) : signature(signature_), + cose_signature(cose_signature), root(root_), path(path_), node_id(node_id_),