tonlib: big update

catchain/catchain-receiver.cpp

@@ -588,7 +588,7 @@ void CatChainReceiverImpl::read_db() {
   read_db_ = true;
 
   next_rotate_ = td::Timestamp::in(60 + td::Random::fast(0, 60));
-  next_sync_ = td::Timestamp::in(0.01 * td::Random::fast(0, 60));
+  next_sync_ = td::Timestamp::in(0.001 * td::Random::fast(0, 60));
   alarm_timestamp().relax(next_rotate_);
   alarm_timestamp().relax(next_sync_);
 
@@ -627,7 +627,7 @@ void CatChainReceiverImpl::receive_query_from_overlay(adnl::AdnlNodeIdShort src,
     promise.set_error(td::Status::Error(ErrorCode::notready, "db not read"));
     return;
   }
-  td::PerfWarningTimer t{"catchain query process", 0.001};
+  td::PerfWarningTimer t{"catchain query process", 0.05};
   auto F = fetch_tl_object<ton_api::Function>(data.clone(), true);
   if (F.is_error()) {
     callback_->on_custom_query(get_source_by_adnl_id(src)->get_hash(), std::move(data), std::move(promise));
@@ -909,7 +909,7 @@ void CatChainReceiverImpl::choose_neighbours() {
 void CatChainReceiverImpl::alarm() {
   alarm_timestamp() = td::Timestamp::never();
   if (next_sync_ && next_sync_.is_in_past()) {
-    next_sync_ = td::Timestamp::in(td::Random::fast(2.0, 3.0));
+    next_sync_ = td::Timestamp::in(td::Random::fast(0.1, 0.2));
     for (auto i = 0; i < 3; i++) {
       auto S = get_source(td::Random::fast(0, get_sources_cnt() - 1));
       CHECK(S != nullptr);

crypto/CMakeLists.txt

@@ -315,6 +315,8 @@ if (NOT CMAKE_CROSSCOMPILING)
   GenFif(DEST smartcont/config-code.fif SOURCE smartcont/config-code.fc)
   GenFif(DEST smartcont/wallet-code.fif SOURCE smartcont/wallet-code.fc)
   GenFif(DEST smartcont/simple-wallet-code.fif SOURCE smartcont/simple-wallet-code.fc)
+  GenFif(DEST smartcont/highload-wallet-code.fif SOURCE smartcont/highload-wallet-code.fc)
+  GenFif(DEST smartcont/highload-wallet-v2-code.fif SOURCE smartcont/highload-wallet-v2-code.fc)
   GenFif(DEST smartcont/elector-code.fif SOURCE smartcont/elector-code.fc)
 endif()
 
@@ -333,3 +335,6 @@ target_link_libraries(dump-block PUBLIC ton_crypto fift-lib ton_block)
 if (WINGETOPT_FOUND)
   target_link_libraries_system(dump-block wingetopt)
 endif()
+
+install(TARGETS fift func RUNTIME DESTINATION bin)
+install(DIRECTORY fift/lib/ DESTINATION lib/fift)

crypto/parser/srcread.cpp

@@ -169,6 +169,11 @@ bool SourceReader::load_line() {
     error("line too long");
     return false;
   }
+  if (len && cur_line.back() == '\r') {
+    // CP/M line breaks support
+    cur_line.pop_back();
+    --len;
+  }
   loc.text = cur_line;
   cur_line_len = (int)len;
   loc.line_pos = 0;

crypto/smartcont/highload-wallet-code.fc

@@ -0,0 +1,41 @@
+;; Heavy-duty wallet for mass transfers (e.g., for cryptocurrency exchanges)
+;; accepts orders for up to 254 internal messages (transfers) in one external message
+
+() recv_internal(slice in_msg) impure {
+  ;; do nothing for internal messages
+}
+
+() recv_external(slice in_msg) impure {
+  var signature = in_msg~load_bits(512);
+  var cs = in_msg;
+  var (subwallet_id, valid_until, msg_seqno) = (cs~load_uint(32), cs~load_uint(32), cs~load_uint(32));
+  throw_if(35, valid_until <= now());
+  var ds = get_data().begin_parse();
+  var (stored_seqno, stored_subwallet, public_key) = (ds~load_uint(32), ds~load_uint(32), ds~load_uint(256));
+  ds.end_parse();
+  throw_unless(33, msg_seqno == stored_seqno);
+  throw_unless(34, subwallet_id == stored_subwallet);
+  throw_unless(35, check_signature(slice_hash(in_msg), signature, public_key));
+  var dict = cs~load_dict();
+  cs.end_parse();
+  accept_message();
+  int i = -1;
+  do {
+    (i, var cs, var f) = dict.idict_get_next?(16, i);
+    if (f) {
+      var mode = cs~load_uint(8);
+      send_raw_message(cs~load_ref(), mode);
+    }
+  } until (~ f);
+  set_data(begin_cell()
+    .store_uint(stored_seqno + 1, 32)
+    .store_uint(stored_subwallet, 32)
+    .store_uint(public_key, 256)
+    .end_cell());
+}
+
+;; Get methods
+
+int seqno() method_id {
+  return get_data().begin_parse().preload_uint(32);
+}

crypto/smartcont/highload-wallet-v2-code.fc

@@ -0,0 +1,65 @@
+;; Heavy-duty wallet for mass transfers (e.g., for cryptocurrency exchanges)
+;; accepts orders for up to 254 internal messages (transfers) in one external message
+;; this version does not use seqno for replay protection; instead, it remembers all recent query_ids
+;; in this way several external messages with different query_id can be sent in parallel
+
+() recv_internal(slice in_msg) impure {
+  ;; do nothing for internal messages
+}
+
+() recv_external(slice in_msg) impure {
+  var signature = in_msg~load_bits(512);
+  var cs = in_msg;
+  var (subwallet_id, query_id) = (cs~load_uint(32), cs~load_uint(64));
+  var bound = (now() << 32);
+  throw_if(35, query_id < bound);
+  var ds = get_data().begin_parse();
+  var (stored_subwallet, last_cleaned, public_key, old_queries) = (ds~load_uint(32), ds~load_uint(64), ds~load_uint(256), ds~load_dict());
+  ds.end_parse();
+  (_, var found?) = old_queries.udict_get?(64, query_id);
+  throw_if(32, found?);
+  throw_unless(34, subwallet_id == stored_subwallet);
+  throw_unless(35, check_signature(slice_hash(in_msg), signature, public_key));
+  var dict = cs~load_dict();
+  cs.end_parse();
+  accept_message();
+  int i = -1;
+  do {
+    (i, var cs, var f) = dict.idict_get_next?(16, i);
+    if (f) {
+      var mode = cs~load_uint(8);
+      send_raw_message(cs~load_ref(), mode);
+    }
+  } until (~ f);
+  bound -= (64 << 32);   ;; clean up records expired more than 64 seconds ago
+  old_queries~udict_set_builder(64, query_id, begin_cell());
+  var queries = old_queries;
+  do {
+    var (old_queries', i, _, f) = old_queries.udict_delete_get_min(64);
+    f~touch();
+    if (f) {
+      f = (i < bound);
+    }
+    if (f) {
+      old_queries = old_queries';
+      last_cleaned = i;
+    }
+  } until (~ f);
+  set_data(begin_cell()
+    .store_uint(stored_subwallet, 32)
+    .store_uint(last_cleaned, 64)
+    .store_uint(public_key, 256)
+    .store_dict(old_queries)
+    .end_cell());
+}
+
+;; Get methods
+
+;; returns -1 for processed queries, 0 for unprocessed, 1 for unknown (forgotten)
+int processed?(int query_id) method_id {
+  var ds = get_data().begin_parse();
+  var (_, last_cleaned, _, old_queries) = (ds~load_uint(32), ds~load_uint(64), ds~load_uint(256), ds~load_dict());
+  ds.end_parse();
+  (_, var found) = old_queries.udict_get?(64, query_id);
+  return found ? true : - (query_id <= last_cleaned);
+}

crypto/smartcont/highload-wallet.fif

@@ -0,0 +1,68 @@
+#!/usr/bin/env fift -s
+"TonUtil.fif" include
+
+{ ."usage: " @' $0 type ." <filename-base> <subwallet-id> <seqno> <order-file> [<savefile>]" cr
+  ."Creates a request with up to 254 orders loaded from <order-file> to high-load (sub)wallet created by new-highload-wallet.fif, with private key loaded from file <filename-base>.pk "
+  ."and address from <filename-base><subwallet-id>.addr, and saves it into <savefile>.boc ('wallet-query.boc' by default)" cr 
+  ."<order-file> is a text file with lines `SEND <dest-addr> <amount>`" cr 1 halt
+} : usage
+$# dup 4 < swap 5 > or ' usage if
+
+$1 =: file-base
+$2 parse-int dup 32 fits ' usage ifnot =: subwallet-id   // parse subwallet-id
+{ subwallet-id (.) $+ } : +subwallet
+$3 parse-int =: seqno
+$4 =: order-file
+def? $5 { @' $5 } { "wallet-query" } cond constant savefile
+3 constant send-mode  // mode for SENDRAWMSG: +1 - sender pays fees, +2 - ignore errors
+60 constant timeout   // external message expires in 60 seconds
+
+file-base +subwallet +".addr" load-address
+2dup 2constant wallet_addr
+."Source wallet address = " 2dup .addr cr 6 .Addr cr
+file-base +".pk" load-keypair nip constant wallet_pk
+
+variable orders   dictnew orders !
+variable order#   order# 0!
+// c --
+{ <s order# @ dup 254 >= abort"more than 254 orders"
+  orders @ 16 udict!+ not abort"cannot add order to dictionary"
+  orders ! order# 1+!
+} : add-order
+// b body -- b'
+{ tuck <s 2dup s-fits? not rot over 1 i, -rot
+  { drop swap ref, } { s, nip } cond
+} : append-msg-body
+// ng wc addr bounce body -- c
+{ <b b{01} s, rot 1 i, b{000100} s, 2swap addr, rot Gram,
+  0 9 64 32 + + 1+ u, swap append-msg-body b>
+} : create-int-msg
+// ng wc addr bnc --
+{ ."Transferring " 3 roll .GR ."to account " 
+  -rot 2dup 4 pick 7 + .Addr ." = " .addr ." bounce=" . cr
+} : .transfer
+// addr$ ng -- c
+{ swap parse-smc-addr  // ng wc addr bnc 
+  2over 2over .transfer
+  <b 0 32 u, b> create-int-msg
+} : create-simple-transfer
+// c m -- c'
+{ <b swap 8 u, swap ref, b> } : create-order
+
+// addr$ ng --
+{ create-simple-transfer send-mode create-order add-order } : send
+{ bl word bl word $>GR send } : SEND
+
+// parse order file
+order-file include
+
+// create external message
+<b subwallet-id 32 i, now timeout + 32 u, seqno 32 u, orders @ dict, b>
+dup ."signing message: " <s csr. cr
+dup hash wallet_pk ed25519_sign_uint
+<b b{1000100} s, wallet_addr addr, 0 Gram, b{00} s,
+   swap B, swap <s s, b>
+dup ."resulting external message: " <s csr. cr
+2 boc+>B dup Bx. cr
+savefile +".boc" tuck B>file
+."(Saved to file " type .")" cr

crypto/smartcont/new-highload-wallet.fif

@@ -0,0 +1,44 @@
+#!/usr/bin/env fift -s
+"TonUtil.fif" include
+"Asm.fif" include
+
+{ ."usage: " @' $0 type ." <workchain-id> <subwallet-id> [<filename-base>]" cr
+  ."Creates a new high-load wallet in the specified workchain, with the controlling private key saved to or loaded from <filename-base>.pk "
+  ."('new-wallet.pk' by default)" cr 
+  ."<subwallet-id> is the 32-bit identifier of this subwallet among all controlled by the same private key" cr 1 halt
+} : usage
+$# 2- -2 and ' usage if
+
+$1 parse-workchain-id =: wc    // set workchain id from command line argument
+$2 parse-int dup =: subwallet-id   // parse subwallet-id
+32 fits ' usage ifnot
+{ subwallet-id (.) $+ } : +subwallet
+def? $3 { @' $3 } { "new-wallet" } cond constant file-base
+
+."Creating new high-load wallet in workchain " wc . 
+."with subwallet id " subwallet-id . cr
+
+// Create new high-load wallet; source code included from `highload-wallet-code.fif`
+"highload-wallet-code.fif" include
+// code
+<b 0 32 u, subwallet-id 32 i,
+   file-base +".pk" load-generate-keypair
+   constant wallet_pk
+   B, 
+b> // data
+null // no libraries
+<b b{0011} s, 3 roll ref, rot ref, swap dict, b>  // create StateInit
+dup ."StateInit: " <s csr. cr
+dup hash wc swap 2dup 2constant wallet_addr
+."new wallet address = " 2dup .addr cr
+2dup file-base +subwallet +".addr" save-address-verbose
+."Non-bounceable address (for init): " 2dup 7 .Addr cr
+."Bounceable address (for later access): " 6 .Addr cr
+<b subwallet-id 32 i, -1 32 i, 0 32 u, false 1 i, b>
+dup ."signing message: " <s csr. cr
+dup hash wallet_pk ed25519_sign_uint rot
+<b b{1000100} s, wallet_addr addr, b{000010} s, swap <s s, b{0} s, swap B, swap <s s, b>
+dup ."External message for initialization is " <s csr. cr
+2 boc+>B dup Bx. cr
+file-base +subwallet +"-query.boc" tuck B>file
+."(Saved wallet creating query to file " type .")" cr

crypto/smartcont/stdlib.fc

@@ -86,6 +86,8 @@ cell dict_set_builder(cell dict, int key_len, slice index, builder value) asm(va
 (cell, (slice, slice, int)) ~dict::delete_get_min(cell dict, int key_len) asm(-> 0 2 1 3) "DICTREMMIN" "NULLSWAPIFNOT" "NULLSWAPIFNOT";
 (int, slice, int) udict_get_next?(cell dict, int key_len, int pivot) asm(pivot dict key_len -> 1 0 2) "DICTUGETNEXT" "NULLSWAPIFNOT" "NULLSWAPIFNOT";
 (int, slice, int) udict_get_prev?(cell dict, int key_len, int pivot) asm(pivot dict key_len -> 1 0 2) "DICTUGETPREV" "NULLSWAPIFNOT" "NULLSWAPIFNOT";
+(int, slice, int) idict_get_next?(cell dict, int key_len, int pivot) asm(pivot dict key_len -> 1 0 2) "DICTIGETNEXT" "NULLSWAPIFNOT" "NULLSWAPIFNOT";
+(int, slice, int) idict_get_prev?(cell dict, int key_len, int pivot) asm(pivot dict key_len -> 1 0 2) "DICTIGETPREV" "NULLSWAPIFNOT" "NULLSWAPIFNOT";
 cell new_dict() asm "NEWDICT";
 int dict_empty?(cell c) asm "DICTEMPTY";
 

tl/generate/scheme/tonlib_api.tl

@@ -16,8 +16,12 @@ vector {t:Type} # [ t ] = Vector t;
 error code:int32 message:string = Error;
 ok = Ok;
 
+keyStoreTypeDirectory directory:string = KeyStoreType;
+keyStoreTypeInMemory = KeyStoreType;
+
 config config:string blockchain_name:string use_callbacks_for_network:Bool ignore_cache:Bool = Config;
-options config:config keystore_directory:string = Options;
+
+options config:config keystore_type:KeyStoreType = Options;
 
 key public_key:string secret:secureBytes = Key;
 inputKey key:key local_password:secureBytes = InputKey;
@@ -37,7 +41,7 @@ raw.initialAccountState code:bytes data:bytes = raw.InitialAccountState;
 raw.accountState balance:int64 code:bytes data:bytes last_transaction_id:internal.transactionId sync_utime:int53 = raw.AccountState;
 raw.message source:string destination:string value:int64 message:bytes = raw.Message;
 raw.transaction utime:int53 data:bytes transaction_id:internal.transactionId fee:int64 in_msg:raw.message out_msgs:vector<raw.message> = raw.Transaction;
-raw.transactions transactions:vector<raw.Transaction> previous_transaction_id:internal.transactionId = raw.Transactions;
+raw.transactions transactions:vector<raw.transaction> previous_transaction_id:internal.transactionId = raw.Transactions;
 
 testWallet.initialAccountState public_key:string = testWallet.InitialAccountState;
 testWallet.accountState balance:int64 seqno:int32 last_transaction_id:internal.transactionId sync_utime:int53 = testWallet.AccountState;

tonlib/CMakeLists.txt

@@ -5,13 +5,15 @@ if (NOT OPENSSL_FOUND)
 endif()
 
 set(TONLIB_SOURCE
+  tonlib/CellString.cpp
   tonlib/Client.cpp
   tonlib/Config.cpp
   tonlib/ExtClient.cpp
   tonlib/ExtClientLazy.cpp
   tonlib/ExtClientOutbound.cpp
   tonlib/GenericAccount.cpp
   tonlib/KeyStorage.cpp
+  tonlib/KeyValue.cpp
   tonlib/LastBlock.cpp
   tonlib/LastBlockStorage.cpp
   tonlib/Logging.cpp
@@ -21,13 +23,15 @@ set(TONLIB_SOURCE
   tonlib/utils.cpp
   tonlib/Wallet.cpp
 
+  tonlib/CellString.h
   tonlib/Client.h
   tonlib/Config.h
   tonlib/ExtClient.h
   tonlib/ExtClientLazy.h
   tonlib/ExtClientOutbound.h
   tonlib/GenericAccount.h
   tonlib/KeyStorage.h
+  tonlib/KeyValue.h
   tonlib/LastBlock.h
   tonlib/LastBlockStorage.h
   tonlib/Logging.h

tonlib/TonlibConfig.cmake

@@ -0,0 +1,3 @@
+include(CMakeFindDependencyMacro)
+#TODO: write all external dependencies
+include("${CMAKE_CURRENT_LIST_DIR}/TonlibTargets.cmake")