added support for config change proposals

- added some fift scripts for the config change proposal voting - added validator-engine-console support for the config change proposal voting - additional sanity checks in catchain - unsafe slow catchain resync method
Ricchkdbanks · Mar 30, 2020 · 4dd5eea · 4dd5eea
1 parent a31f8d4
commit 4dd5eea
Show file tree

Hide file tree

Showing 35 changed files with 753 additions and 144 deletions.
diff --git a/catchain/catchain-receiver-interface.h b/catchain/catchain-receiver-interface.h
@@ -60,7 +60,8 @@ class CatChainReceiverInterface : public td::actor::Actor {
                                                                td::actor::ActorId<adnl::Adnl> adnl,
                                                                td::actor::ActorId<overlay::Overlays> overlay_manager,
                                                                std::vector<CatChainNode> ids, PublicKeyHash local_id,
-                                                               CatChainSessionId unique_hash, std::string db_root);
+                                                               CatChainSessionId unique_hash, std::string db_root,
+                                                               bool allow_unsafe_self_blocks_resync);
 
   virtual ~CatChainReceiverInterface() = default;
 };

diff --git a/catchain/catchain-receiver-source.h b/catchain/catchain-receiver-source.h
@@ -53,6 +53,9 @@ class CatChainReceiverSource {
   virtual void block_received(CatChainBlockHeight height) = 0;
   virtual void block_delivered(CatChainBlockHeight height) = 0;
 
+  virtual bool has_unreceived() const = 0;
+  virtual bool has_undelivered() const = 0;
+
   virtual td::Status validate_dep_sync(tl_object_ptr<ton_api::catchain_block_dep> &dep) = 0;
   virtual void on_new_block(CatChainReceivedBlock *block) = 0;
   virtual void on_found_fork_proof(td::Slice fork) = 0;

diff --git a/catchain/catchain-receiver-source.hpp b/catchain/catchain-receiver-source.hpp
@@ -78,6 +78,19 @@ class CatChainReceiverSourceImpl : public CatChainReceiverSource {
   CatChainBlockHeight received_height() const override {
     return received_height_;
   }
+  bool has_unreceived() const override {
+    if (blamed()) {
+      return true;
+    }
+    if (!blocks_.size()) {
+      return false;
+    }
+    CHECK(blocks_.rbegin()->second->get_height() >= received_height_);
+    return blocks_.rbegin()->second->get_height() > received_height_;
+  }
+  bool has_undelivered() const override {
+    return delivered_height_ < received_height_;
+  }
   CatChainReceivedBlock *get_block(CatChainBlockHeight height) const override;
 
   td::Status validate_dep_sync(tl_object_ptr<ton_api::catchain_block_dep> &dep) override;

diff --git a/catchain/catchain-receiver.cpp b/catchain/catchain-receiver.cpp
@@ -83,6 +83,16 @@ void CatChainReceiverImpl::receive_block(adnl::AdnlNodeIdShort src, tl_object_pt
     return;
   }
 
+  if (block->src_ == static_cast<td::int32>(local_idx_)) {
+    if (!allow_unsafe_self_blocks_resync_ || started_) {
+      LOG(FATAL) << this << ": received unknown SELF block from " << src
+                 << " (unsafe=" << allow_unsafe_self_blocks_resync_ << ")";
+    } else {
+      LOG(ERROR) << this << ": received unknown SELF block from " << src << ". UPDATING LOCAL DATABASE. UNSAFE";
+      initial_sync_complete_at_ = td::Timestamp::in(300.0);
+    }
+  }
+
   auto raw_data = serialize_tl_object(block, true, payload.as_slice());
   create_block(std::move(block), td::SharedSlice{payload.as_slice()});
 
@@ -420,14 +430,16 @@ CatChainReceiverImpl::CatChainReceiverImpl(std::unique_ptr<Callback> callback, C
                                            td::actor::ActorId<adnl::Adnl> adnl,
                                            td::actor::ActorId<overlay::Overlays> overlay_manager,
                                            std::vector<CatChainNode> ids, PublicKeyHash local_id,
-                                           CatChainSessionId unique_hash, std::string db_root)
+                                           CatChainSessionId unique_hash, std::string db_root,
+                                           bool allow_unsafe_self_blocks_resync)
     : callback_(std::move(callback))
     , opts_(std::move(opts))
     , keyring_(keyring)
     , adnl_(adnl)
     , overlay_manager_(overlay_manager)
     , local_id_(local_id)
-    , db_root_(db_root) {
+    , db_root_(db_root)
+    , allow_unsafe_self_blocks_resync_(allow_unsafe_self_blocks_resync) {
   std::vector<td::Bits256> short_ids;
   local_idx_ = static_cast<td::uint32>(ids.size());
   for (auto &id : ids) {
@@ -589,19 +601,20 @@ void CatChainReceiverImpl::read_db() {
 
   next_rotate_ = td::Timestamp::in(60 + td::Random::fast(0, 60));
   next_sync_ = td::Timestamp::in(0.001 * td::Random::fast(0, 60));
+  initial_sync_complete_at_ = td::Timestamp::in(allow_unsafe_self_blocks_resync_ ? 300.0 : 5.0);
   alarm_timestamp().relax(next_rotate_);
   alarm_timestamp().relax(next_sync_);
-
-  callback_->start();
+  alarm_timestamp().relax(initial_sync_complete_at_);
 }
 
 td::actor::ActorOwn<CatChainReceiverInterface> CatChainReceiverInterface::create(
     std::unique_ptr<Callback> callback, CatChainOptions opts, td::actor::ActorId<keyring::Keyring> keyring,
     td::actor::ActorId<adnl::Adnl> adnl, td::actor::ActorId<overlay::Overlays> overlay_manager,
-    std::vector<CatChainNode> ids, PublicKeyHash local_id, CatChainSessionId unique_hash, std::string db_root) {
+    std::vector<CatChainNode> ids, PublicKeyHash local_id, CatChainSessionId unique_hash, std::string db_root,
+    bool allow_unsafe_self_blocks_resync) {
   auto A = td::actor::create_actor<CatChainReceiverImpl>("catchainreceiver", std::move(callback), std::move(opts),
                                                          keyring, adnl, overlay_manager, std::move(ids), local_id,
-                                                         unique_hash, db_root);
+                                                         unique_hash, db_root, allow_unsafe_self_blocks_resync);
   return std::move(A);
 }
 
@@ -906,6 +919,59 @@ void CatChainReceiverImpl::choose_neighbours() {
   neighbours_ = std::move(n);
 }
 
+bool CatChainReceiverImpl::unsafe_start_up_check_completed() {
+  auto S = get_source(local_idx_);
+  CHECK(!S->blamed());
+  if (S->has_unreceived() || S->has_undelivered()) {
+    LOG(INFO) << "catchain: has_unreceived=" << S->has_unreceived() << " has_undelivered=" << S->has_undelivered();
+    run_scheduler();
+    initial_sync_complete_at_ = td::Timestamp::in(60.0);
+    return false;
+  }
+  auto h = S->delivered_height();
+  if (h == 0) {
+    CHECK(last_sent_block_->get_height() == 0);
+    CHECK(!unsafe_root_block_writing_);
+    return true;
+  }
+  if (last_sent_block_->get_height() == h) {
+    CHECK(!unsafe_root_block_writing_);
+    return true;
+  }
+  if (unsafe_root_block_writing_) {
+    initial_sync_complete_at_ = td::Timestamp::in(5.0);
+    LOG(INFO) << "catchain: writing=true";
+    return false;
+  }
+
+  unsafe_root_block_writing_ = true;
+  auto B = S->get_block(h);
+  CHECK(B != nullptr);
+  CHECK(B->delivered());
+  CHECK(B->in_db());
+
+  auto id = B->get_hash();
+
+  td::BufferSlice raw_data{32};
+  raw_data.as_slice().copy_from(as_slice(id));
+
+  auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), block = B](td::Result<td::Unit> R) mutable {
+    R.ensure();
+    td::actor::send_closure(SelfId, &CatChainReceiverImpl::written_unsafe_root_block, block);
+  });
+
+  db_.set(CatChainBlockHash::zero(), std::move(raw_data), std::move(P), 0);
+  initial_sync_complete_at_ = td::Timestamp::in(5.0);
+  LOG(INFO) << "catchain: need update root";
+  return false;
+}
+
+void CatChainReceiverImpl::written_unsafe_root_block(CatChainReceivedBlock *block) {
+  CHECK(last_sent_block_->get_height() < block->get_height());
+  last_sent_block_ = block;
+  unsafe_root_block_writing_ = false;
+}
+
 void CatChainReceiverImpl::alarm() {
   alarm_timestamp() = td::Timestamp::never();
   if (next_sync_ && next_sync_.is_in_past()) {
@@ -923,8 +989,22 @@ void CatChainReceiverImpl::alarm() {
     next_rotate_ = td::Timestamp::in(td::Random::fast(60.0, 120.0));
     choose_neighbours();
   }
+  if (!started_ && read_db_ && initial_sync_complete_at_ && initial_sync_complete_at_.is_in_past()) {
+    bool allow = false;
+    if (allow_unsafe_self_blocks_resync_) {
+      allow = unsafe_start_up_check_completed();
+    } else {
+      allow = true;
+    }
+    if (allow) {
+      initial_sync_complete_at_ = td::Timestamp::never();
+      started_ = true;
+      callback_->start();
+    }
+  }
   alarm_timestamp().relax(next_rotate_);
   alarm_timestamp().relax(next_sync_);
+  alarm_timestamp().relax(initial_sync_complete_at_);
 }
 
 void CatChainReceiverImpl::send_fec_broadcast(td::BufferSlice data) {

diff --git a/catchain/catchain-receiver.hpp b/catchain/catchain-receiver.hpp
@@ -134,14 +134,17 @@ class CatChainReceiverImpl : public CatChainReceiver {
 
   void block_written_to_db(CatChainBlockHash hash);
 
+  bool unsafe_start_up_check_completed();
+  void written_unsafe_root_block(CatChainReceivedBlock *block);
+
   void destroy() override;
 
   CatChainReceivedBlock *get_block(CatChainBlockHash hash) const;
 
   CatChainReceiverImpl(std::unique_ptr<Callback> callback, CatChainOptions opts,
                        td::actor::ActorId<keyring::Keyring> keyring, td::actor::ActorId<adnl::Adnl> adnl,
                        td::actor::ActorId<overlay::Overlays>, std::vector<CatChainNode> ids, PublicKeyHash local_id,
-                       CatChainBlockHash unique_hash, std::string db_root);
+                       CatChainBlockHash unique_hash, std::string db_root, bool allow_unsafe_self_blocks_resync);
 
  private:
   std::unique_ptr<overlay::Overlays::Callback> make_callback() {
@@ -222,6 +225,10 @@ class CatChainReceiverImpl : public CatChainReceiver {
   DbType db_;
 
   bool intentional_fork_ = false;
+  td::Timestamp initial_sync_complete_at_{td::Timestamp::never()};
+  bool allow_unsafe_self_blocks_resync_{false};
+  bool unsafe_root_block_writing_{false};
+  bool started_{false};
 
   std::list<CatChainReceivedBlock *> to_run_;
 };

diff --git a/catchain/catchain.cpp b/catchain/catchain.cpp
@@ -222,8 +222,9 @@ void CatChainImpl::on_receiver_started() {
 CatChainImpl::CatChainImpl(std::unique_ptr<Callback> callback, CatChainOptions opts,
                            td::actor::ActorId<keyring::Keyring> keyring, td::actor::ActorId<adnl::Adnl> adnl,
                            td::actor::ActorId<overlay::Overlays> overlay_manager, std::vector<CatChainNode> ids,
-                           PublicKeyHash local_id, CatChainSessionId unique_hash, std::string db_root)
-    : opts_(std::move(opts)), db_root_(db_root) {
+                           PublicKeyHash local_id, CatChainSessionId unique_hash, std::string db_root,
+                           bool allow_unsafe_self_blocks_resync)
+    : opts_(std::move(opts)), db_root_(db_root), allow_unsafe_self_blocks_resync_(allow_unsafe_self_blocks_resync) {
   callback_ = std::move(callback);
   sources_.resize(ids.size());
   unique_hash_ = unique_hash;
@@ -280,9 +281,9 @@ void CatChainImpl::start_up() {
 
   auto cb = std::make_unique<ChainCb>(actor_id(this));
 
-  receiver_ =
-      CatChainReceiverInterface::create(std::move(cb), opts_, args_->keyring, args_->adnl, args_->overlay_manager,
-                                        std::move(args_->ids), args_->local_id, args_->unique_hash, db_root_);
+  receiver_ = CatChainReceiverInterface::create(std::move(cb), opts_, args_->keyring, args_->adnl,
+                                                args_->overlay_manager, std::move(args_->ids), args_->local_id,
+                                                args_->unique_hash, db_root_, allow_unsafe_self_blocks_resync_);
   args_ = nullptr;
   //alarm_timestamp() = td::Timestamp::in(opts_.idle_timeout);
 }
@@ -292,9 +293,11 @@ td::actor::ActorOwn<CatChain> CatChain::create(std::unique_ptr<Callback> callbac
                                                td::actor::ActorId<adnl::Adnl> adnl,
                                                td::actor::ActorId<overlay::Overlays> overlay_manager,
                                                std::vector<CatChainNode> ids, PublicKeyHash local_id,
-                                               CatChainSessionId unique_hash, std::string db_root) {
+                                               CatChainSessionId unique_hash, std::string db_root,
+                                               bool allow_unsafe_self_blocks_resync) {
   return td::actor::create_actor<CatChainImpl>("catchain", std::move(callback), std::move(opts), keyring, adnl,
-                                               overlay_manager, std::move(ids), local_id, unique_hash, db_root);
+                                               overlay_manager, std::move(ids), local_id, unique_hash, db_root,
+                                               allow_unsafe_self_blocks_resync);
 }
 
 CatChainBlock *CatChainImpl::get_block(CatChainBlockHash hash) const {

diff --git a/catchain/catchain.h b/catchain/catchain.h
@@ -102,7 +102,8 @@ class CatChain : public td::actor::Actor {
                                               td::actor::ActorId<adnl::Adnl> adnl,
                                               td::actor::ActorId<overlay::Overlays> overlay_manager,
                                               std::vector<CatChainNode> ids, PublicKeyHash local_id,
-                                              CatChainSessionId unique_hash, std::string db_root);
+                                              CatChainSessionId unique_hash, std::string db_root,
+                                              bool allow_unsafe_self_blocks_resync);
   virtual ~CatChain() = default;
 };
 

diff --git a/catchain/catchain.hpp b/catchain/catchain.hpp
@@ -51,6 +51,7 @@ class CatChainImpl : public CatChain {
   bool receiver_started_ = false;
 
   std::string db_root_;
+  bool allow_unsafe_self_blocks_resync_;
 
   void send_process();
   void send_preprocess(CatChainBlock *block);
@@ -118,7 +119,7 @@ class CatChainImpl : public CatChain {
   CatChainImpl(std::unique_ptr<Callback> callback, CatChainOptions opts, td::actor::ActorId<keyring::Keyring> keyring,
                td::actor::ActorId<adnl::Adnl> adnl, td::actor::ActorId<overlay::Overlays> overlay_manager,
                std::vector<CatChainNode> ids, PublicKeyHash local_id, CatChainSessionId unique_hash,
-               std::string db_root);
+               std::string db_root, bool allow_unsafe_self_blocks_resync);
 
   void alarm() override;
   void start_up() override;

diff --git a/crypto/fift/words.cpp b/crypto/fift/words.cpp
@@ -932,6 +932,32 @@ void interpret_concat_builders(vm::Stack& stack) {
   stack.push_builder(std::move(cb1));
 }
 
+void interpret_cell_datasize(vm::Stack& stack, int mode) {
+  auto bound = (mode & 4 ? stack.pop_int() : td::make_refint(1 << 22));
+  Ref<vm::Cell> cell;
+  Ref<vm::CellSlice> cs;
+  if (mode & 2) {
+    cs = stack.pop_cellslice();
+  } else {
+    cell = stack.pop_maybe_cell();
+  }
+  if (!bound->is_valid() || bound->sgn() < 0) {
+    throw IntError{"finite non-negative integer expected"};
+  }
+  vm::VmStorageStat stat{bound->unsigned_fits_bits(63) ? bound->to_long() : (1ULL << 63) - 1};
+  bool ok = (mode & 2 ? stat.add_storage(cs.write()) : stat.add_storage(std::move(cell)));
+  if (ok) {
+    stack.push_smallint(stat.cells);
+    stack.push_smallint(stat.bits);
+    stack.push_smallint(stat.refs);
+  } else if (!(mode & 1)) {
+    throw IntError{"scanned too many cells"};
+  }
+  if (mode & 1) {
+    stack.push_bool(ok);
+  }
+}
+
 void interpret_slice_bitrefs(vm::Stack& stack, int mode) {
   auto cs = stack.pop_cellslice();
   if (mode & 1) {
@@ -2755,6 +2781,8 @@ void init_words_common(Dictionary& d) {
   d.def_stack_word("sbits ", std::bind(interpret_slice_bitrefs, _1, 1));
   d.def_stack_word("srefs ", std::bind(interpret_slice_bitrefs, _1, 2));
   d.def_stack_word("sbitrefs ", std::bind(interpret_slice_bitrefs, _1, 3));
+  d.def_stack_word("totalcsize ", std::bind(interpret_cell_datasize, _1, 0));
+  d.def_stack_word("totalssize ", std::bind(interpret_cell_datasize, _1, 2));
   // boc manipulation
   d.def_stack_word("B>boc ", interpret_boc_deserialize);
   d.def_stack_word("boc>B ", interpret_boc_serialize);