Merge branch 'main' of github.com:microsoft/CCF into loop_until_sync

eddyashton · Feb 15, 2024 · dff789c · dff789c
2 parents f021f69 + 74326ef
commit dff789c
Show file tree

Hide file tree

Showing 46 changed files with 631 additions and 350 deletions.
diff --git a/.azure-pipelines-gh-pages.yml b/.azure-pipelines-gh-pages.yml
@@ -11,7 +11,7 @@ jobs:
     variables:
       Codeql.SkipTaskAutoInjection: true
       skipComponentGovernanceDetection: true
-    container: ccfmsrc.azurecr.io/ccf/ci:25-01-2024-virtual-clang15
+    container: ccfmsrc.azurecr.io/ccf/ci:12-02-2024-virtual-clang15
     pool:
       vmImage: ubuntu-20.04
 

diff --git a/.azure-pipelines-release.yml b/.azure-pipelines-release.yml
@@ -8,15 +8,15 @@ pr: none
 resources:
   containers:
     - container: virtual
-      image: ccfmsrc.azurecr.io/ccf/ci:25-01-2024-virtual-clang15
+      image: ccfmsrc.azurecr.io/ccf/ci:12-02-2024-virtual-clang15
       options: --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE -v /lib/modules:/lib/modules:ro
 
     - container: snp
-      image: ccfmsrc.azurecr.io/ccf/ci:25-01-2024-snp-clang15
+      image: ccfmsrc.azurecr.io/ccf/ci:12-02-2024-snp-clang15
       options: --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE -v /lib/modules:/lib/modules:ro
 
     - container: sgx
-      image: ccfmsrc.azurecr.io/ccf/ci:25-01-2024-sgx
+      image: ccfmsrc.azurecr.io/ccf/ci:12-02-2024-sgx
       options: --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --device /dev/sgx_enclave:/dev/sgx_enclave --device /dev/sgx_provision:/dev/sgx_provision -v /dev/sgx:/dev/sgx -v /lib/modules:/lib/modules:ro
 
 variables:

diff --git a/.azure-pipelines-templates/deploy_aci.yml b/.azure-pipelines-templates/deploy_aci.yml
@@ -51,7 +51,7 @@ jobs:
       - script: |
           set -ex
           docker login -u $ACR_TOKEN_NAME -p $ACR_CI_PUSH_TOKEN_PASSWORD $ACR_REGISTRY
-          docker pull $ACR_REGISTRY/ccf/ci:25-01-2024-snp-clang15
+          docker pull $ACR_REGISTRY/ccf/ci:12-02-2024-snp-clang15
           docker build -f docker/ccf_ci_built . --build-arg="base=$BASE_IMAGE" --build-arg="platform=snp" -t $ACR_REGISTRY/ccf/ci:pr-`git rev-parse HEAD`
           docker push $ACR_REGISTRY/ccf/ci:pr-`git rev-parse HEAD`
         name: build_ci_image
@@ -60,7 +60,7 @@ jobs:
           ACR_TOKEN_NAME: ci-push-token
           ACR_CI_PUSH_TOKEN_PASSWORD: $(ACR_CI_PUSH_TOKEN_PASSWORD)
           ACR_REGISTRY: ccfmsrc.azurecr.io
-          BASE_IMAGE: ccfmsrc.azurecr.io/ccf/ci:25-01-2024-snp-clang15
+          BASE_IMAGE: ccfmsrc.azurecr.io/ccf/ci:12-02-2024-snp-clang15
 
       - script: |
           set -ex

diff --git a/.azure-pipelines-templates/trace_validation.yml b/.azure-pipelines-templates/trace_validation.yml
@@ -10,11 +10,13 @@ steps:
       set -ex
       set -o pipefail
       cd tla/
-      parallel 'JSON={} java -XX:+UseParallelGC -Dtlc2.tool.impl.Tool.cdot=true -Dtlc2.tool.queue.IStateQueue=StateDeque -cp tla2tools.jar:CommunityModules-deps.jar tlc2.TLC -lncheck final -checkpoint 0 -dump dot,constrained,colorize,actionlabels {}.dot consensus/Traceccfraft.tla' ::: $(ls ../build/*.ndjson | grep -v _deprecated)
+      mkdir traces
+      cp ../build/*.ndjson traces/
+      parallel 'set -o pipefail && echo {} && JVM_OPTIONS=-Dtlc2.tool.queue.IStateQueue=StateDeque JSON={} ./tlc.sh -dump dot,constrained,colorize,actionlabels {}.dot -dumpTrace tla {}.trace.tla -dumpTrace json {}.trace.json consensus/Traceccfraft.tla | ./last_line.sh {}.trace.json' ::: $(ls traces/*.ndjson)
     displayName: "Run trace validation"
 
   - task: PublishPipelineArtifact@1
     inputs:
       artifactName: "Trace Validation Output"
-      targetPath: tla
+      targetPath: tla/traces
     condition: or(failed(), canceled())
diff --git a/.azure-pipelines.yml b/.azure-pipelines.yml
@@ -30,15 +30,15 @@ schedules:
 resources:
   containers:
     - container: virtual
-      image: ccfmsrc.azurecr.io/ccf/ci:25-01-2024-virtual-clang15
+      image: ccfmsrc.azurecr.io/ccf/ci:12-02-2024-virtual-clang15
       options: --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE -v /lib/modules:/lib/modules:ro
 
     - container: snp
-      image: ccfmsrc.azurecr.io/ccf/ci:25-01-2024-snp-clang15
+      image: ccfmsrc.azurecr.io/ccf/ci:12-02-2024-snp-clang15
       options: --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE -v /lib/modules:/lib/modules:ro
 
     - container: sgx
-      image: ccfmsrc.azurecr.io/ccf/ci:25-01-2024-sgx
+      image: ccfmsrc.azurecr.io/ccf/ci:12-02-2024-sgx
       options: --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --device /dev/sgx_enclave:/dev/sgx_enclave --device /dev/sgx_provision:/dev/sgx_provision -v /dev/sgx:/dev/sgx -v /lib/modules:/lib/modules:ro
 
 variables:

diff --git a/.azure_pipelines_snp.yml b/.azure_pipelines_snp.yml
@@ -32,7 +32,7 @@ schedules:
 resources:
   containers:
     - container: virtual
-      image: ccfmsrc.azurecr.io/ccf/ci:25-01-2024-virtual-clang15
+      image: ccfmsrc.azurecr.io/ccf/ci:12-02-2024-virtual-clang15
       options: --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE -v /lib/modules:/lib/modules:ro
 
 jobs:

diff --git a/.daily.yml b/.daily.yml
@@ -27,15 +27,15 @@ schedules:
 resources:
   containers:
     - container: virtual
-      image: ccfmsrc.azurecr.io/ccf/ci:25-01-2024-virtual-clang15
+      image: ccfmsrc.azurecr.io/ccf/ci:12-02-2024-virtual-clang15
       options: --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE
 
     - container: snp
-      image: ccfmsrc.azurecr.io/ccf/ci:25-01-2024-snp-clang15
+      image: ccfmsrc.azurecr.io/ccf/ci:12-02-2024-snp-clang15
       options: --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE -v /lib/modules:/lib/modules:ro
 
     - container: sgx
-      image: ccfmsrc.azurecr.io/ccf/ci:25-01-2024-sgx
+      image: ccfmsrc.azurecr.io/ccf/ci:12-02-2024-sgx
       options: --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --device /dev/sgx_enclave:/dev/sgx_enclave --device /dev/sgx_provision:/dev/sgx_provision -v /dev/sgx:/dev/sgx
 
 jobs:

diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json
@@ -1,6 +1,6 @@
 {
   "name": "CCF Development Environment",
-  "image": "ccfmsrc.azurecr.io/ccf/ci:25-01-2024-virtual-clang15",
+  "image": "ccfmsrc.azurecr.io/ccf/ci:12-02-2024-virtual-clang15",
   "runArgs": [],
   "extensions": [
     "eamodio.gitlens",

diff --git a/.github/workflows/ci-checks.yml b/.github/workflows/ci-checks.yml
@@ -11,7 +11,7 @@ permissions: read-all
 jobs:
   checks:
     runs-on: ubuntu-latest
-    container: ccfmsrc.azurecr.io/ccf/ci:25-01-2024-virtual-clang15
+    container: ccfmsrc.azurecr.io/ccf/ci:12-02-2024-virtual-clang15
 
     steps:
       - run: git config --global --add safe.directory "$GITHUB_WORKSPACE"

diff --git a/.github/workflows/tlaplus.yml b/.github/workflows/tlaplus.yml
@@ -16,7 +16,7 @@ jobs:
     name: Model Checking - Consistency
     runs-on: [self-hosted, 1ES.Pool=gha-virtual-ccf-sub]
     container:
-      image: ccfmsrc.azurecr.io/ccf/ci:25-01-2024-virtual-clang15
+      image: ccfmsrc.azurecr.io/ccf/ci:12-02-2024-virtual-clang15
 
     steps:
       - uses: actions/checkout@v3

diff --git a/.multi-thread.yml b/.multi-thread.yml
@@ -20,7 +20,7 @@ pr:
 resources:
   containers:
     - container: virtual
-      image: ccfmsrc.azurecr.io/ccf/ci:25-01-2024-virtual-clang15
+      image: ccfmsrc.azurecr.io/ccf/ci:12-02-2024-virtual-clang15
       options: --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE -v /lib/modules:/lib/modules:ro
 
 jobs:

diff --git a/.stress.yml b/.stress.yml
@@ -24,7 +24,7 @@ schedules:
 resources:
   containers:
     - container: sgx
-      image: ccfmsrc.azurecr.io/ccf/ci:25-01-2024-sgx
+      image: ccfmsrc.azurecr.io/ccf/ci:12-02-2024-sgx
       options: --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --device /dev/sgx_enclave:/dev/sgx_enclave --device /dev/sgx_provision:/dev/sgx_provision -v /dev/sgx:/dev/sgx
 
 jobs:

diff --git a/docker/ccf_ci_built b/docker/ccf_ci_built
@@ -4,7 +4,7 @@
 
 # Latest image as of this change
 ARG platform=sgx
-ARG base=ccfmsrc.azurecr.io/ccf/ci:25-01-2024-snp-clang-15
+ARG base=ccfmsrc.azurecr.io/ccf/ci:12-02-2024-snp-clang-15
 FROM ${base}
 
 # SSH. Note that this could (should) be done in the base ccf_ci image instead

diff --git a/include/ccf/crypto/entropy.h b/include/ccf/crypto/entropy.h
@@ -303,6 +303,5 @@ namespace crypto
   using EntropyPtr = std::shared_ptr<Entropy>;
   static EntropyPtr intel_drng_ptr;
 
-  /** Create a default Entropy object */
-  EntropyPtr create_entropy();
+  EntropyPtr get_entropy();
 }
diff --git a/include/ccf/crypto/symmetric_key.h b/include/ccf/crypto/symmetric_key.h
@@ -44,7 +44,7 @@ namespace crypto
       return GCM_SIZE_TAG + IV_SIZE;
     }
 
-    void set_random_iv(EntropyPtr entropy = crypto::create_entropy())
+    void set_random_iv(EntropyPtr entropy = crypto::get_entropy())
     {
       iv = entropy->random(IV_SIZE);
     }

diff --git a/scripts/azure_deployment/arm_aci.py b/scripts/azure_deployment/arm_aci.py
@@ -133,7 +133,7 @@ def parse_aci_args(parser: ArgumentParser) -> Namespace:
         "--aci-image",
         help="The name of the image to deploy in the ACI",
         type=str,
-        default="ccfmsrc.azurecr.io/ccf/ci:25-01-2024-snp",
+        default="ccfmsrc.azurecr.io/ccf/ci:12-02-2024-snp",
     )
     parser.add_argument(
         "--aci-type",

diff --git a/src/consensus/aft/raft.h b/src/consensus/aft/raft.h
@@ -296,16 +296,29 @@ namespace aft
       return state->membership_state == kv::MembershipState::Retired;
     }
 
-    void set_retired_committed(ccf::SeqNo seqno) override
-    {
-      state->retirement_phase = kv::RetirementPhase::RetiredCommitted;
-      CCF_ASSERT_FMT(
-        state->retired_committed_idx == state->commit_idx,
-        "Retired "
-        "committed index {} does not match current commit index {}",
-        state->retired_committed_idx.value_or(0),
-        state->commit_idx);
-      state->retired_committed_idx = seqno;
+    void set_retired_committed(
+      ccf::SeqNo seqno, const std::vector<kv::NodeId>& node_ids) override
+    {
+      for (auto& node_id : node_ids)
+      {
+        if (id() == node_id)
+        {
+          CCF_ASSERT(
+            state->membership_state == kv::MembershipState::Retired,
+            "Node is not retired, cannot become retired committed");
+          CCF_ASSERT(
+            state->retirement_phase == kv::RetirementPhase::Completed,
+            "Node is not retired, cannot become retired committed");
+          CCF_ASSERT_FMT(
+            state->retired_committed_idx == state->commit_idx,
+            "Retired "
+            "committed index {} does not match current commit index {}",
+            state->retired_committed_idx.value_or(0),
+            state->commit_idx);
+          state->retirement_phase = kv::RetirementPhase::RetiredCommitted;
+          state->retired_committed_idx = seqno;
+        }
+      }
     }
 
     Index last_committable_index() const
@@ -1981,54 +1994,58 @@ namespace aft
       }
       else if (phase == kv::RetirementPhase::Completed)
       {
-        leader_id.reset();
-        state->leadership_state = kv::LeadershipState::None;
-        ProposeRequestVote prv{.term = state->current_view};
-
-        std::optional<ccf::NodeId> successor = std::nullopt;
-        Index max_match_idx = 0;
-        kv::ReconfigurationId reconf_id_of_max_match = 0;
-
-        // Pick the node that has the highest match_idx, and break
-        // ties by looking at the highest reconfiguration id they are
-        // part of. This can lead to nudging a node that is
-        // about to retire too, but that node will then nudge
-        // a successor, and that seems preferable to nudging a node that
-        // risks not being eligible if reconfiguration id is prioritised.
-        // Alternatively, we could pick the node with the higest match idx
-        // in the latest config, provided that match idx at least as high as a
-        // majority. That would make them both eligible and unlikely to retire
-        // soon.
-        for (auto& [node, node_state] : all_other_nodes)
+        if (state->leadership_state == kv::LeadershipState::Leader)
         {
-          if (node_state.match_idx >= max_match_idx)
+          ProposeRequestVote prv{.term = state->current_view};
+
+          std::optional<ccf::NodeId> successor = std::nullopt;
+          Index max_match_idx = 0;
+          kv::ReconfigurationId reconf_id_of_max_match = 0;
+
+          // Pick the node that has the highest match_idx, and break
+          // ties by looking at the highest reconfiguration id they are
+          // part of. This can lead to nudging a node that is
+          // about to retire too, but that node will then nudge
+          // a successor, and that seems preferable to nudging a node that
+          // risks not being eligible if reconfiguration id is prioritised.
+          // Alternatively, we could pick the node with the higest match idx
+          // in the latest config, provided that match idx at least as high as a
+          // majority. That would make them both eligible and unlikely to retire
+          // soon.
+          for (auto& [node, node_state] : all_other_nodes)
           {
-            kv::ReconfigurationId latest_reconf_id = 0;
-            auto conf = configurations.rbegin();
-            while (conf != configurations.rend())
+            if (node_state.match_idx >= max_match_idx)
             {
-              if (conf->nodes.find(node) != conf->nodes.end())
+              kv::ReconfigurationId latest_reconf_id = 0;
+              auto conf = configurations.rbegin();
+              while (conf != configurations.rend())
               {
-                latest_reconf_id = conf->idx;
-                break;
+                if (conf->nodes.find(node) != conf->nodes.end())
+                {
+                  latest_reconf_id = conf->idx;
+                  break;
+                }
+                conf++;
+              }
+              if (!(node_state.match_idx == max_match_idx &&
+                    latest_reconf_id < reconf_id_of_max_match))
+              {
+                reconf_id_of_max_match = latest_reconf_id;
+                successor = node;
+                max_match_idx = node_state.match_idx;
               }
-              conf++;
-            }
-            if (!(node_state.match_idx == max_match_idx &&
-                  latest_reconf_id < reconf_id_of_max_match))
-            {
-              reconf_id_of_max_match = latest_reconf_id;
-              successor = node;
-              max_match_idx = node_state.match_idx;
             }
           }
+          if (successor.has_value())
+          {
+            RAFT_INFO_FMT("Node retired, nudging {}", successor.value());
+            channels->send_authenticated(
+              successor.value(), ccf::NodeMsgType::consensus_msg, prv);
+          }
         }
-        if (successor.has_value())
-        {
-          RAFT_INFO_FMT("Node retired, nudging {}", successor.value());
-          channels->send_authenticated(
-            successor.value(), ccf::NodeMsgType::consensus_msg, prv);
-        }
+
+        leader_id.reset();
+        state->leadership_state = kv::LeadershipState::None;
       }
 
       state->membership_state = kv::MembershipState::Retired;

diff --git a/src/consensus/aft/test/driver.h b/src/consensus/aft/test/driver.h
@@ -198,7 +198,9 @@ class RaftDriver
       std::make_shared<aft::State>(node_id),
       nullptr);
     kv->set_set_retired_committed_hook(
-      [&raft](aft::Index idx) { raft->set_retired_committed(idx); });
+      [raft](aft::Index idx, const std::vector<kv::NodeId>& node_ids) {
+        raft->set_retired_committed(idx, node_ids);
+      });
     raft->start_ticking();
 
     if (_nodes.find(node_id) != _nodes.end())

diff --git a/src/consensus/aft/test/logging_stub.h b/src/consensus/aft/test/logging_stub.h
@@ -329,17 +329,19 @@ namespace aft
     }
   };
 
+  using RCHook = std::function<void(Index, const std::vector<kv::NodeId>&)>;
+
   class LoggingStubStore
   {
   protected:
     ccf::NodeId _id;
-    std::function<void(Index)> set_retired_committed_hook;
+    RCHook set_retired_committed_hook;
 
   public:
     LoggingStubStore(ccf::NodeId id) : _id(id) {}
 
     virtual void set_set_retired_committed_hook(
-      std::function<void(Index)> set_retired_committed_hook_)
+      RCHook set_retired_committed_hook_)
     {
       set_retired_committed_hook = set_retired_committed_hook_;
     }
@@ -480,16 +482,12 @@ namespace aft
       {
         if (version <= i)
         {
-          std::cout << "Retired committed configuration: "
-                    << configuration.dump() << std::endl;
-          if (configuration.find(_id) != configuration.end())
+          std::vector<kv::NodeId> retired_committed_node_ids;
+          for (auto& [node_id, _] : configuration.items())
           {
-            std::cout << "Node id: " << _id
-                      << " is in the retired committed configuration, calling "
-                         "set_retired_committed hook"
-                      << std::endl;
-            set_retired_committed_hook(i);
+            retired_committed_node_ids.push_back(node_id);
           }
+          set_retired_committed_hook(i, retired_committed_node_ids);
         }
         else
         {

diff --git a/src/crypto/entropy.cpp b/src/crypto/entropy.cpp
@@ -7,7 +7,7 @@
 
 namespace crypto
 {
-  EntropyPtr create_entropy()
+  EntropyPtr get_entropy()
   {
     if (use_drng)
     {