COSE back-endorsements for previous service identities

CHANGELOG.md

@@ -5,6 +5,14 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
+## [6.0.0-dev2]
+
+[6.0.0-dev2]: https://github.com/microsoft/CCF/releases/tag/6.0.0-dev2
+
+### Added
+
+- Introduced `ccf::describe_cose_endorsements_v1(receipt)` for COSE-endorsements chain of previous service identities (#6500).
+
 ## [6.0.0-dev1]
 
 [6.0.0-dev1]: https://github.com/microsoft/CCF/releases/tag/6.0.0-dev1

doc/audit/builtin_maps.rst

@@ -509,4 +509,12 @@ Evidence inserted in the ledger by a primary producing a snapshot to establish p
 
 Used to persist submitted shares during a recovery.
 
-While the contents themselves are encrypted, the table is public so as to be accessible by nodes bootstrapping a recovery service.
+While the contents themselves are encrypted, the table is public so as to be accessible by nodes bootstrapping a recovery service.
+
+
+``previous_service_identity_endorsement``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+**Key** Sentinel value 0, represented as a little-endian 64-bit unsigned integer.
+
+**Value** Endorsed COSE sign1 for the interface, represented as a DER-encoded string.

doc/schemas/app_openapi.json

@@ -98,6 +98,17 @@
         ],
         "type": "object"
       },
+      "LoggingGetCoseEndorsements__Out": {
+        "properties": {
+          "endorsements": {
+            "$ref": "#/components/schemas/base64string_array"
+          }
+        },
+        "required": [
+          "endorsements"
+        ],
+        "type": "object"
+      },
       "LoggingGetHistoricalRange__Entry": {
         "properties": {
           "id": {
@@ -215,6 +226,16 @@
         ],
         "type": "string"
       },
+      "base64string": {
+        "format": "base64",
+        "type": "string"
+      },
+      "base64string_array": {
+        "items": {
+          "$ref": "#/components/schemas/base64string"
+        },
+        "type": "array"
+      },
       "boolean": {
         "type": "boolean"
       },
@@ -1210,6 +1231,37 @@
         }
       }
     },
+    "/app/log/public/cose_endorsements": {
+      "get": {
+        "operationId": "GetAppLogPublicCoseEndorsements",
+        "responses": {
+          "200": {
+            "content": {
+              "application/json": {
+                "schema": {
+                  "$ref": "#/components/schemas/LoggingGetCoseEndorsements__Out"
+                }
+              }
+            },
+            "description": "Default response description"
+          },
+          "default": {
+            "$ref": "#/components/responses/default"
+          }
+        },
+        "security": [
+          {
+            "jwt": []
+          },
+          {
+            "user_cose_sign1": []
+          }
+        ],
+        "x-ccf-forwarding": {
+          "$ref": "#/components/x-ccf-forwarding/never"
+        }
+      }
+    },
     "/app/log/public/count": {
       "get": {
         "operationId": "GetAppLogPublicCount",

include/ccf/crypto/cose_verifier.h

@@ -27,4 +27,12 @@ namespace ccf::crypto
   COSEVerifierUniquePtr make_cose_verifier_from_cert(
     const std::vector<uint8_t>& cert);
   COSEVerifierUniquePtr make_cose_verifier_from_key(const Pem& public_key);
+
+  struct COSEEndorsementValidity
+  {
+    std::string from_txid{};
+    std::string to_txid{};
+  };
+  COSEEndorsementValidity extract_cose_endorsement_validity(
+    std::span<const uint8_t> cose_msg);
 }

include/ccf/historical_queries_utils.h

@@ -25,4 +25,19 @@ namespace ccf::historical
     AbstractStateCache& state_cache,
     std::shared_ptr<NetworkIdentitySubsystemInterface>
       network_identity_subsystem);
+
+  // Modifies the receipt stored in state to include historical service
+  // endorsements, where required. If the state talks about a different service
+  // identity, which is known to be a predecessor of this service (via disaster
+  // recoveries), then an endorsement chain of all service identities preceding
+  // the current one will be created. This may need to use the state_cache to
+  // request additional historical entries to construct this endorsement, and
+  // may read from the current/latest state via tx. Returns true if the
+  // operation is complete, though it may still have failed to produce an
+  // endorsement. Returns false if additional entries have been requested, in
+  // which case the caller should retry later.
+  bool populate_cose_service_endorsements(
+    ccf::kv::ReadOnlyTx& tx,
+    ccf::historical::StatePtr& state,
+    AbstractStateCache& state_cache);
 }

include/ccf/receipt.h

@@ -147,6 +147,11 @@ namespace ccf
   std::optional<std::vector<uint8_t>> describe_merkle_proof_v1(
     const TxReceiptImpl& in);
 
+  using SerialisedCoseEndorsement = std::vector<uint8_t>;
+  using SerialisedCoseEndorsements = std::vector<SerialisedCoseEndorsement>;
+  std::optional<SerialisedCoseEndorsements> describe_cose_endorsements_v1(
+    const TxReceiptImpl& in);
+
   // Manual JSON serializers are specified for these types as they are not
   // trivial POD structs
 

python/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "ccf"
-version = "6.0.0-dev1"
+version = "6.0.0-dev2"
 authors = [
   { name="CCF Team", email="[email protected]" },
 ]

python/src/ccf/cose.py

@@ -174,16 +174,14 @@ def create_cose_sign1_finish(
     return msg.encode(sign=False)
 
 
-def validate_cose_sign1(payload: bytes, cert_pem: Pem, cose_sign1: bytes):
-    cert = load_pem_x509_certificate(cert_pem.encode("ascii"), default_backend())
-    if not isinstance(cert.public_key(), EllipticCurvePublicKey):
-        raise NotImplementedError("unsupported key type")
-
-    key = cert.public_key()
-    cose_key = from_cryptography_eckey_obj(key)
+def validate_cose_sign1(pubkey, cose_sign1, payload=None):
+    cose_key = from_cryptography_eckey_obj(pubkey)
     msg = Sign1Message.decode(cose_sign1)
     msg.key = cose_key
-    msg.payload = payload
+
+    if payload:
+        # Detached payload
+        msg.payload = payload
 
     if not msg.verify_signature():
         raise ValueError("signature is invalid")

python/src/ccf/ledger.py

@@ -571,8 +571,11 @@ def _verify_root_signature(self, tx_info: TxBundleInfo):
 
     def _verify_root_cose_signature(self, root, cose_sign1):
         try:
+            cert = load_pem_x509_certificate(
+                self.service_cert.encode("ascii"), default_backend()
+            )
             validate_cose_sign1(
-                payload=root, cert_pem=self.service_cert, cose_sign1=cose_sign1
+                cose_sign1=cose_sign1, pubkey=cert.public_key(), payload=root
             )
         except Exception as exc:
             raise InvalidRootCoseSignatureException(

samples/apps/logging/logging.cpp

@@ -1961,6 +1961,47 @@ namespace loggingapp
         .set_auto_schema<void, void>()
         .set_forwarding_required(ccf::endpoints::ForwardingRequired::Never)
         .install();
+
+      auto get_cose_endorsements =
+        [this](
+          ccf::endpoints::ReadOnlyEndpointContext& ctx,
+          ccf::historical::StatePtr historical_state) {
+          auto historical_tx = historical_state->store->create_read_only_tx();
+
+          assert(historical_state->receipt);
+          auto endorsements =
+            describe_cose_endorsements_v1(*historical_state->receipt);
+          if (!endorsements.has_value())
+          {
+            ctx.rpc_ctx->set_error(
+              HTTP_STATUS_NOT_FOUND,
+              ccf::errors::ResourceNotFound,
+              "No COSE endorsements available for this transaction");
+            return;
+          }
+          LoggingGetCoseEndorsements::Out response{
+            .endorsements = ccf::SerialisedCoseEndorsements{}};
+          for (const auto& endorsement : *endorsements)
+          {
+            response.endorsements->push_back(endorsement);
+          }
+
+          nlohmann::json j_response = response;
+          ctx.rpc_ctx->set_response_status(HTTP_STATUS_OK);
+          ctx.rpc_ctx->set_response_header(
+            ccf::http::headers::CONTENT_TYPE,
+            ccf::http::headervalues::contenttype::JSON);
+          ctx.rpc_ctx->set_response_body(j_response.dump());
+        };
+      make_read_only_endpoint(
+        "/log/public/cose_endorsements",
+        HTTP_GET,
+        ccf::historical::read_only_adapter_v4(
+          get_cose_endorsements, context, is_tx_committed),
+        auth_policies)
+        .set_auto_schema<void, LoggingGetCoseEndorsements::Out>()
+        .set_forwarding_required(ccf::endpoints::ForwardingRequired::Never)
+        .install();
     }
   };
 }

samples/apps/logging/logging_schema.h

@@ -98,6 +98,16 @@ namespace loggingapp
   DECLARE_JSON_OPTIONAL_FIELDS_WITH_RENAMES(
     LoggingGetHistoricalRange::Out, next_link, "@nextLink");
 
+  struct LoggingGetCoseEndorsements
+  {
+    struct Out
+    {
+      std::optional<ccf::SerialisedCoseEndorsements> endorsements;
+    };
+  };
+  DECLARE_JSON_TYPE(LoggingGetCoseEndorsements::Out);
+  DECLARE_JSON_REQUIRED_FIELDS(LoggingGetCoseEndorsements::Out, endorsements);
+
   // Public record/get
   // Manual schemas, verified then parsed in handler
   static const std::string j_record_public_in = R"!!!(

src/crypto/openssl/cose_sign.cpp

@@ -79,7 +79,8 @@ namespace
 
 namespace ccf::crypto
 {
-  std::optional<int> key_to_cose_alg_id(ccf::crypto::PublicKey_OpenSSL& key)
+  std::optional<int> key_to_cose_alg_id(
+    const ccf::crypto::PublicKey_OpenSSL& key)
   {
     const auto cid = key.get_curve_id();
     switch (cid)
@@ -155,10 +156,25 @@ namespace ccf::crypto
       args_size);
   }
 
+  COSEParametersFactory cose_params_string_bytes(
+    const std::string& key, const std::vector<uint8_t>& value)
+  {
+    const size_t args_size = key.size() + value.size() +
+      extra_size_for_seq_tag + extra_size_for_seq_tag;
+    q_useful_buf_c buf{value.data(), value.size()};
+    return COSEParametersFactory(
+      [=](QCBOREncodeContext* ctx) {
+        QCBOREncode_AddSZString(ctx, key.data());
+        QCBOREncode_AddBytes(ctx, buf);
+      },
+      args_size);
+  }
+
   std::vector<uint8_t> cose_sign1(
-    KeyPair_OpenSSL& key,
+    const KeyPair_OpenSSL& key,
     const std::vector<COSEParametersFactory>& protected_headers,
-    std::span<const uint8_t> payload)
+    std::span<const uint8_t> payload,
+    bool detached_payload)
   {
     const auto buf_size = estimate_buffer_size(protected_headers, payload);
     std::vector<uint8_t> underlying_buffer(buf_size);
@@ -188,8 +204,16 @@ namespace ccf::crypto
 
     encode_parameters_custom(&sign_ctx, &cbor_encode, protected_headers);
 
-    // Mark empty payload manually.
-    QCBOREncode_AddNULL(&cbor_encode);
+    if (detached_payload)
+    {
+      // Mark empty payload explicitly.
+      QCBOREncode_AddNULL(&cbor_encode);
+    }
+    else
+    {
+      UsefulBufC payload_buffer{payload.data(), payload.size()};
+      QCBOREncode_AddBytes(&cbor_encode, payload_buffer);
+    }
 
     // If payload is empty - we still want to sign. Putting NULL_Q_USEFUL_BUF_C,
     // however, makes t_cose think that the payload is included into the

src/crypto/openssl/cose_sign.h

@@ -19,7 +19,11 @@ namespace ccf::crypto
   // Standardised: verifiable data structure
   static constexpr int64_t COSE_PHEADER_KEY_VDS = 395;
   // CCF-specific: last signed TxID
-  static constexpr const char* COSE_PHEADER_KEY_TXID = "ccf.txid";
+  static const std::string COSE_PHEADER_KEY_TXID = "ccf.txid";
+  // CCF-specific: first TX in the range.
+  static const std::string COSE_PHEADER_KEY_RANGE_BEGIN = "ccf.epoch.begin";
+  // CCF-specific: last TX included in the range.
+  static const std::string COSE_PHEADER_KEY_RANGE_END = "ccf.epoch.end";
 
   class COSEParametersFactory
   {
@@ -59,12 +63,16 @@ namespace ccf::crypto
   COSEParametersFactory cose_params_int_bytes(
     int64_t key, const std::vector<uint8_t>& value);
 
+  COSEParametersFactory cose_params_string_bytes(
+    const std::string& key, const std::vector<uint8_t>& value);
+
   struct COSESignError : public std::runtime_error
   {
     COSESignError(const std::string& msg) : std::runtime_error(msg) {}
   };
 
-  std::optional<int> key_to_cose_alg_id(ccf::crypto::PublicKey_OpenSSL& key);
+  std::optional<int> key_to_cose_alg_id(
+    const ccf::crypto::PublicKey_OpenSSL& key);
 
   /* Sign a cose_sign1 payload with custom protected headers as strings, where
        - key: integer label to be assigned in a COSE value
@@ -74,7 +82,8 @@ namespace ccf::crypto
     https://www.iana.org/assignments/cose/cose.xhtml#header-parameters.
    */
   std::vector<uint8_t> cose_sign1(
-    KeyPair_OpenSSL& key,
+    const KeyPair_OpenSSL& key,
     const std::vector<COSEParametersFactory>& protected_headers,
-    std::span<const uint8_t> payload);
+    std::span<const uint8_t> payload,
+    bool detached_payload = true);
 }