Add custom overlays for external messages (#949)

* Private overlay for external messages * Improve ext msg overlays * Manage from validator console * Bypass out queue size limit for high-priority messages * Shuffle messages in get_external_messages * Cleanup mempool when creating validator group * Improve private overlays for externals 1. Allow using validator adnl ids in addition to fullnode ids 2. Set priority per sender, not per overlay 3. Require the same overlay name for all nodes 4. Enable lz4 in private block overlay * Fix typo, add debug logs * Enable lz4 in private block overlay by config Change proto_version for lz4 in catchain overlays to 4 * Add logs for broadcasts in fullnode --------- Co-authored-by: SpyCheese <[email protected]>
ton-blockchain · Apr 1, 2024 · 0434ead · 0434ead
1 parent 46ca0e6
commit 0434ead
Show file tree

Hide file tree

Showing 32 changed files with 840 additions and 224 deletions.
diff --git a/catchain/catchain-receiver.cpp b/catchain/catchain-receiver.cpp
@@ -520,7 +520,8 @@ void CatChainReceiverImpl::start_up() {
   }
   td::actor::send_closure(overlay_manager_, &overlay::Overlays::create_private_overlay,
                           get_source(local_idx_)->get_adnl_id(), overlay_full_id_.clone(), std::move(ids),
-                          make_callback(), overlay::OverlayPrivacyRules{0, 0, std::move(root_keys)});
+                          make_callback(), overlay::OverlayPrivacyRules{0, 0, std::move(root_keys)},
+                          R"({ "type": "catchain" })");
 
   CHECK(root_block_);
 

diff --git a/create-hardfork/create-hardfork.cpp b/create-hardfork/create-hardfork.cpp
@@ -213,7 +213,7 @@ class HardforkCreator : public td::actor::Actor {
         ton::validator::ValidatorManagerHardforkFactory::create(opts, shard_, shard_top_block_id_, db_root_);
     for (auto &msg : ext_msgs_) {
       td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManager::new_external_message,
-                              std::move(msg));
+                              std::move(msg), 0);
     }
     for (auto &topmsg : top_shard_descrs_) {
       td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManager::new_shard_block, ton::BlockIdExt{},

diff --git a/overlay/overlay-manager.cpp b/overlay/overlay-manager.cpp
@@ -108,11 +108,12 @@ void OverlayManager::create_public_overlay_ex(adnl::AdnlNodeIdShort local_id, Ov
 
 void OverlayManager::create_private_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id,
                                             std::vector<adnl::AdnlNodeIdShort> nodes,
-                                            std::unique_ptr<Callback> callback, OverlayPrivacyRules rules) {
+                                            std::unique_ptr<Callback> callback, OverlayPrivacyRules rules,
+                                            std::string scope) {
   auto id = overlay_id.compute_short_id();
   register_overlay(local_id, id,
                    Overlay::create(keyring_, adnl_, actor_id(this), dht_node_, local_id, std::move(overlay_id),
-                                   std::move(nodes), std::move(callback), std::move(rules)));
+                                   std::move(nodes), std::move(callback), std::move(rules), std::move(scope)));
 }
 
 void OverlayManager::receive_message(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, td::BufferSlice data) {

diff --git a/overlay/overlay-manager.h b/overlay/overlay-manager.h
@@ -57,7 +57,7 @@ class OverlayManager : public Overlays {
                                  OverlayOptions opts) override;
   void create_private_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id,
                               std::vector<adnl::AdnlNodeIdShort> nodes, std::unique_ptr<Callback> callback,
-                              OverlayPrivacyRules rules) override;
+                              OverlayPrivacyRules rules, std::string scope) override;
   void delete_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdShort overlay_id) override;
   void send_query(adnl::AdnlNodeIdShort dst, adnl::AdnlNodeIdShort src, OverlayIdShort overlay_id, std::string name,
                   td::Promise<td::BufferSlice> promise, td::Timestamp timeout, td::BufferSlice query) override {

diff --git a/overlay/overlay.cpp b/overlay/overlay.cpp
@@ -49,10 +49,11 @@ td::actor::ActorOwn<Overlay> Overlay::create(td::actor::ActorId<keyring::Keyring
                                              td::actor::ActorId<OverlayManager> manager,
                                              td::actor::ActorId<dht::Dht> dht_node, adnl::AdnlNodeIdShort local_id,
                                              OverlayIdFull overlay_id, std::vector<adnl::AdnlNodeIdShort> nodes,
-                                             std::unique_ptr<Overlays::Callback> callback, OverlayPrivacyRules rules) {
-  auto R =
-      td::actor::create_actor<OverlayImpl>("overlay", keyring, adnl, manager, dht_node, local_id, std::move(overlay_id),
-                                           false, std::move(nodes), std::move(callback), std::move(rules));
+                                             std::unique_ptr<Overlays::Callback> callback, OverlayPrivacyRules rules,
+                                             std::string scope) {
+  auto R = td::actor::create_actor<OverlayImpl>("overlay", keyring, adnl, manager, dht_node, local_id,
+                                                std::move(overlay_id), false, std::move(nodes), std::move(callback),
+                                                std::move(rules), std::move(scope));
   return td::actor::ActorOwn<Overlay>(std::move(R));
 }
 

diff --git a/overlay/overlay.h b/overlay/overlay.h
@@ -48,7 +48,8 @@ class Overlay : public td::actor::Actor {
                                              td::actor::ActorId<OverlayManager> manager,
                                              td::actor::ActorId<dht::Dht> dht_node, adnl::AdnlNodeIdShort local_id,
                                              OverlayIdFull overlay_id, std::vector<adnl::AdnlNodeIdShort> nodes,
-                                             std::unique_ptr<Overlays::Callback> callback, OverlayPrivacyRules rules);
+                                             std::unique_ptr<Overlays::Callback> callback, OverlayPrivacyRules rules,
+                                             std::string scope);
 
   virtual void update_dht_node(td::actor::ActorId<dht::Dht> dht) = 0;
 

diff --git a/overlay/overlays.h b/overlay/overlays.h
@@ -205,7 +205,7 @@ class Overlays : public td::actor::Actor {
                                          td::string scope, OverlayOptions opts) = 0;
   virtual void create_private_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id,
                                       std::vector<adnl::AdnlNodeIdShort> nodes, std::unique_ptr<Callback> callback,
-                                      OverlayPrivacyRules rules) = 0;
+                                      OverlayPrivacyRules rules, std::string scope) = 0;
   virtual void delete_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdShort overlay_id) = 0;
 
   virtual void send_query(adnl::AdnlNodeIdShort dst, adnl::AdnlNodeIdShort src, OverlayIdShort overlay_id,

diff --git a/test/test-ton-collator.cpp b/test/test-ton-collator.cpp
@@ -300,7 +300,7 @@ class TestNode : public td::actor::Actor {
                                                                              shard_top_block_id_, db_root_);
     for (auto &msg : ext_msgs_) {
       td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManager::new_external_message,
-                              std::move(msg));
+                              std::move(msg), 0);
     }
     for (auto &topmsg : top_shard_descrs_) {
       td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManager::new_shard_block, ton::BlockIdExt{},

diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl
@@ -400,6 +400,7 @@ tonNode.newShardBlockBroadcast block:tonNode.newShardBlock = tonNode.Broadcast;
 tonNode.shardPublicOverlayId workchain:int shard:long zero_state_file_hash:int256 = tonNode.ShardPublicOverlayId;
 
 tonNode.privateBlockOverlayId zero_state_file_hash:int256 nodes:(vector int256) = tonNode.PrivateBlockOverlayId;
+tonNode.privateExtMsgsOverlayId zero_state_file_hash:int256 name:string nodes:(vector int256) = tonNode.PrivateExtMsgsOverlayId;
 
 tonNode.keyBlocks blocks:(vector tonNode.blockIdExt) incomplete:Bool error:Bool = tonNode.KeyBlocks;
 
@@ -594,6 +595,10 @@ engine.validator.config out_port:int addrs:(vector engine.Addr) adnl:(vector eng
         liteservers:(vector engine.liteServer) control:(vector engine.controlInterface)
         gc:engine.gc = engine.validator.Config;
 
+engine.validator.privateExtMsgOverlayNode adnl_id:int256 sender:Bool sender_priority:int = engine.validator.PrivateExtMsgOverlayNode;
+engine.validator.privateExtMsgOverlay name:string nodes:(vector engine.validator.privateExtMsgOverlayNode) = engine.validator.PrivateExtMsgOverlay;
+engine.validator.privateExtMsgOverlaysConfig overlays:(vector engine.validator.privateExtMsgOverlay) = engine.validator.PrivateExtMsgOverlaysConfig;
+
 ---functions---
 ---types---
 
@@ -697,6 +702,10 @@ engine.validator.getPerfTimerStats name:string = engine.validator.PerfTimerStats
 engine.validator.getShardOutQueueSize flags:# block_id:tonNode.blockId dest_wc:flags.0?int dest_shard:flags.0?long = engine.validator.ShardOutQueueSize;
 engine.validator.setExtMessagesBroadcastDisabled disabled:Bool = engine.validator.Success;
 
+engine.validator.addPrivateExtMsgOverlay overlay:engine.validator.privateExtMsgOverlay = engine.validator.Success;
+engine.validator.delPrivateExtMsgOverlay name:string = engine.validator.Success;
+engine.validator.showPrivateExtMsgOverlays = engine.validator.PrivateExtMsgOverlaysConfig;
+
 ---types---
 
 storage.pong = storage.Pong;

diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo
diff --git a/validator-engine-console/validator-engine-console-query.cpp b/validator-engine-console/validator-engine-console-query.cpp
@@ -33,6 +33,8 @@
 #include "td/utils/filesystem.h"
 #include "overlay/overlays.h"
 #include "ton/ton-tl.hpp"
+#include "td/utils/JsonBuilder.h"
+#include "auto/tl/ton_api_json.h"
 
 #include <cctype>
 #include <fstream>
@@ -1107,3 +1109,73 @@ td::Status SetExtMessagesBroadcastDisabledQuery::receive(td::BufferSlice data) {
   td::TerminalIO::out() << "success\n";
   return td::Status::OK();
 }
+
+td::Status AddPrivateExtMsgOverlayQuery::run() {
+  TRY_RESULT_ASSIGN(file_name_, tokenizer_.get_token<std::string>());
+  TRY_STATUS(tokenizer_.check_endl());
+  return td::Status::OK();
+}
+
+td::Status AddPrivateExtMsgOverlayQuery::send() {
+  TRY_RESULT(data, td::read_file(file_name_));
+  TRY_RESULT(json, td::json_decode(data.as_slice()));
+  auto overlay = ton::create_tl_object<ton::ton_api::engine_validator_privateExtMsgOverlay>();
+  TRY_STATUS(ton::ton_api::from_json(*overlay, json.get_object()));
+  auto b = ton::create_serialize_tl_object<ton::ton_api::engine_validator_addPrivateExtMsgOverlay>(std::move(overlay));
+  td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise());
+  return td::Status::OK();
+}
+
+td::Status AddPrivateExtMsgOverlayQuery::receive(td::BufferSlice data) {
+  TRY_RESULT_PREFIX(f, ton::fetch_tl_object<ton::ton_api::engine_validator_success>(data.as_slice(), true),
+                    "received incorrect answer: ");
+  td::TerminalIO::out() << "success\n";
+  return td::Status::OK();
+}
+
+td::Status DelPrivateExtMsgOverlayQuery::run() {
+  TRY_RESULT_ASSIGN(name_, tokenizer_.get_token<std::string>());
+  TRY_STATUS(tokenizer_.check_endl());
+  return td::Status::OK();
+}
+
+td::Status DelPrivateExtMsgOverlayQuery::send() {
+  auto b = ton::create_serialize_tl_object<ton::ton_api::engine_validator_delPrivateExtMsgOverlay>(name_);
+  td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise());
+  return td::Status::OK();
+}
+
+td::Status DelPrivateExtMsgOverlayQuery::receive(td::BufferSlice data) {
+  TRY_RESULT_PREFIX(f, ton::fetch_tl_object<ton::ton_api::engine_validator_success>(data.as_slice(), true),
+                    "received incorrect answer: ");
+  td::TerminalIO::out() << "success\n";
+  return td::Status::OK();
+}
+
+td::Status ShowPrivateExtMsgOverlaysQuery::run() {
+  TRY_STATUS(tokenizer_.check_endl());
+  return td::Status::OK();
+}
+
+td::Status ShowPrivateExtMsgOverlaysQuery::send() {
+  auto b = ton::create_serialize_tl_object<ton::ton_api::engine_validator_showPrivateExtMsgOverlays>();
+  td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise());
+  return td::Status::OK();
+}
+
+td::Status ShowPrivateExtMsgOverlaysQuery::receive(td::BufferSlice data) {
+  TRY_RESULT_PREFIX(
+      f, ton::fetch_tl_object<ton::ton_api::engine_validator_privateExtMsgOverlaysConfig>(data.as_slice(), true),
+      "received incorrect answer: ");
+  td::TerminalIO::out() << f->overlays_.size() << " private overlays:\n\n";
+  for (const auto &overlay : f->overlays_) {
+    td::TerminalIO::out() << "Overlay \"" << overlay->name_ << "\": " << overlay->nodes_.size() << " nodes\n";
+    for (const auto &node : overlay->nodes_) {
+      td::TerminalIO::out() << "  " << node->adnl_id_
+                            << (node->sender_ ? (PSTRING() << " (sender, p=" << node->sender_priority_ << ")") : "")
+                            << "\n";
+    }
+    td::TerminalIO::out() << "\n";
+  }
+  return td::Status::OK();
+}
diff --git a/validator-engine-console/validator-engine-console-query.h b/validator-engine-console/validator-engine-console-query.h
@@ -1144,3 +1144,67 @@ class SetExtMessagesBroadcastDisabledQuery : public Query {
  private:
   bool value;
 };
+
+class AddPrivateExtMsgOverlayQuery : public Query {
+ public:
+  AddPrivateExtMsgOverlayQuery(td::actor::ActorId<ValidatorEngineConsole> console, Tokenizer tokenizer)
+      : Query(console, std::move(tokenizer)) {
+  }
+  td::Status run() override;
+  td::Status send() override;
+  td::Status receive(td::BufferSlice data) override;
+  static std::string get_name() {
+    return "addprivateextmsgoverlay";
+  }
+  static std::string get_help() {
+    return "addprivateextmsgoverlay <filename>\tadd private overlay for external messages with config from file "
+           "<filename>";
+  }
+  std::string name() const override {
+    return get_name();
+  }
+
+ private:
+  std::string file_name_;
+};
+
+class DelPrivateExtMsgOverlayQuery : public Query {
+ public:
+  DelPrivateExtMsgOverlayQuery(td::actor::ActorId<ValidatorEngineConsole> console, Tokenizer tokenizer)
+      : Query(console, std::move(tokenizer)) {
+  }
+  td::Status run() override;
+  td::Status send() override;
+  td::Status receive(td::BufferSlice data) override;
+  static std::string get_name() {
+    return "delprivateextmsgoverlay";
+  }
+  static std::string get_help() {
+    return "delprivateextmsgoverlay <name>\tdelete private overlay for external messages with name <name>";
+  }
+  std::string name() const override {
+    return get_name();
+  }
+
+ private:
+  std::string name_;
+};
+
+class ShowPrivateExtMsgOverlaysQuery : public Query {
+ public:
+  ShowPrivateExtMsgOverlaysQuery(td::actor::ActorId<ValidatorEngineConsole> console, Tokenizer tokenizer)
+      : Query(console, std::move(tokenizer)) {
+  }
+  td::Status run() override;
+  td::Status send() override;
+  td::Status receive(td::BufferSlice data) override;
+  static std::string get_name() {
+    return "showprivateextmsgoverlays";
+  }
+  static std::string get_help() {
+    return "showprivateextmsgoverlays\tshow all private overlay for external messages";
+  }
+  std::string name() const override {
+    return get_name();
+  }
+};
diff --git a/validator-engine-console/validator-engine-console.cpp b/validator-engine-console/validator-engine-console.cpp
@@ -143,6 +143,9 @@ void ValidatorEngineConsole::run() {
   add_query_runner(std::make_unique<QueryRunnerImpl<GetPerfTimerStatsJsonQuery>>());
   add_query_runner(std::make_unique<QueryRunnerImpl<GetShardOutQueueSizeQuery>>());
   add_query_runner(std::make_unique<QueryRunnerImpl<SetExtMessagesBroadcastDisabledQuery>>());
+  add_query_runner(std::make_unique<QueryRunnerImpl<AddPrivateExtMsgOverlayQuery>>());
+  add_query_runner(std::make_unique<QueryRunnerImpl<DelPrivateExtMsgOverlayQuery>>());
+  add_query_runner(std::make_unique<QueryRunnerImpl<ShowPrivateExtMsgOverlaysQuery>>());
 }
 
 bool ValidatorEngineConsole::envelope_send_query(td::BufferSlice query, td::Promise<td::BufferSlice> promise) {