Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Optimize block unmarshalling #236

Merged
merged 3 commits into from
Feb 2, 2024
Merged

Optimize block unmarshalling #236

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
05d7e5b
Improve block tests
aleksej-paschenko Feb 2, 2024
208974f
Speedup block unmarshalling
aleksej-paschenko Feb 2, 2024
94afbc5
Optimize counting keys on hashmap
aleksej-paschenko Feb 2, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
42 changes: 35 additions & 7 deletions tlb/block.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -349,13 +349,41 @@ func (m *ValueFlow) UnmarshalTLB(c *boc.Cell, decoder *Decoder) error {
// created_by:bits256
// custom:(Maybe ^McBlockExtra) = BlockExtra;
type BlockExtra struct {
Magic Magic `tlb:"block_extra#4a33f6fd"`
InMsgDescr HashmapAugE[Bits256, InMsg, ImportFees] `tlb:"^"` // tlb.Any `tlb:"^"`
OutMsgDescr HashmapAugE[Bits256, OutMsg, CurrencyCollection] `tlb:"^"` // tlb.Any `tlb:"^"`
AccountBlocks HashmapAugE[Bits256, AccountBlock, CurrencyCollection] `tlb:"^"` // tlb.Any `tlb:"^"` //
RandSeed Bits256
CreatedBy Bits256
Custom Maybe[Ref[McBlockExtra]]
Magic Magic `tlb:"block_extra#4a33f6fd"`
InMsgDescrCell boc.Cell `tlb:"^"`
OutMsgDescrCell boc.Cell `tlb:"^"`
AccountBlocks HashmapAugE[Bits256, AccountBlock, CurrencyCollection] `tlb:"^"`
RandSeed Bits256
CreatedBy Bits256
Custom Maybe[Ref[McBlockExtra]]
}

func (extra *BlockExtra) InMsgDescrLength() (int, error) {
cell := boc.Cell(extra.InMsgDescrCell)
cell.ResetCounters()
return hashmapAugExtraCountLeafs[Bits256](&cell)
}

func (extra *BlockExtra) InMsgDescr() (HashmapAugE[Bits256, InMsg, ImportFees], error) {
var hashmap HashmapAugE[Bits256, InMsg, ImportFees]
if err := Unmarshal(&extra.InMsgDescrCell, &hashmap); err != nil {
return HashmapAugE[Bits256, InMsg, ImportFees]{}, err
}
return hashmap, nil
}

func (extra *BlockExtra) OutMsgDescrLength() (int, error) {
cell := boc.Cell(extra.OutMsgDescrCell)
cell.ResetCounters()
return hashmapAugExtraCountLeafs[Bits256](&cell)
}

func (extra *BlockExtra) OutMsgDescr() (HashmapAugE[Bits256, OutMsg, CurrencyCollection], error) {
var hashmap HashmapAugE[Bits256, OutMsg, CurrencyCollection]
if err := Unmarshal(&extra.OutMsgDescrCell, &hashmap); err != nil {
return HashmapAugE[Bits256, OutMsg, CurrencyCollection]{}, err
}
return hashmap, nil
}

// masterchain_block_extra#cca5
Expand Down
78 changes: 31 additions & 47 deletions tlb/block_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,15 +6,14 @@ import (
"encoding/json"
"os"
"path"
"reflect"
"sort"
"testing"

"github.com/tonkeeper/tongo/boc"
)

func Test_tlb_Unmarshal(t *testing.T) {
type transaction struct {
type Transaction struct {
AccountAddr string
Lt uint64
PrevTransHash string
Expand All @@ -24,8 +23,15 @@ func Test_tlb_Unmarshal(t *testing.T) {
OrigStatus AccountStatus
EndStatus AccountStatus
}
type accountBlock struct {
Transactions map[uint64]transaction
type AccountBlock struct {
Transactions map[uint64]Transaction
}
type BlockContent struct {
Accounts map[string]*AccountBlock
TxHashes []string
ValueFlow ValueFlow
InMsgDescrLength int
OutMsgDescrLength int
}
testCases := []struct {
name string
Expand Down Expand Up @@ -60,17 +66,17 @@ func Test_tlb_Unmarshal(t *testing.T) {
if err != nil {
t.Errorf("Unmarshal() failed: %v", err)
}
accounts := map[string]*accountBlock{}
accounts := map[string]*AccountBlock{}
var txHashes []string
for _, account := range block.Extra.AccountBlocks.Values() {
accBlock, ok := accounts[hex.EncodeToString(account.AccountAddr[:])]
if !ok {
accBlock = &accountBlock{Transactions: map[uint64]transaction{}}
accBlock = &AccountBlock{Transactions: map[uint64]Transaction{}}
accounts[hex.EncodeToString(account.AccountAddr[:])] = accBlock
}
for _, txRef := range account.Transactions.Values() {
tx := txRef.Value
accBlock.Transactions[txRef.Value.Lt] = transaction{
accBlock.Transactions[txRef.Value.Lt] = Transaction{
AccountAddr: hex.EncodeToString(tx.AccountAddr[:]),
Lt: tx.Lt,
PrevTransHash: hex.EncodeToString(tx.PrevTransHash[:]),
Expand All @@ -86,7 +92,22 @@ func Test_tlb_Unmarshal(t *testing.T) {
sort.Slice(txHashes, func(i, j int) bool {
return txHashes[i] < txHashes[j]
})
bs, err := json.MarshalIndent(accounts, " ", " ")
inMsgLength, err := block.Extra.InMsgDescrLength()
if err != nil {
t.Errorf("InMsgDescrLength() failed: %v", err)
}
outMsgLength, err := block.Extra.OutMsgDescrLength()
if err != nil {
t.Errorf("InMsgDescrLength() failed: %v", err)
}
blk := BlockContent{
Accounts: accounts,
TxHashes: txHashes,
ValueFlow: block.ValueFlow,
InMsgDescrLength: inMsgLength,
OutMsgDescrLength: outMsgLength,
}
bs, err := json.MarshalIndent(blk, " ", " ")
if err != nil {
t.Errorf("json.MarshalIndent() failed: %v", err)
}
Expand All @@ -99,45 +120,8 @@ func Test_tlb_Unmarshal(t *testing.T) {
if err != nil {
t.Errorf("ReadFile() failed: %v", err)
}
expectedAccounts := map[string]*accountBlock{}
if err := json.Unmarshal(content, &expectedAccounts); err != nil {
t.Errorf("json.Unmarshal() failed: %v", err)
}
if !reflect.DeepEqual(accounts, expectedAccounts) {
t.Errorf("expectedAccounts differs from accounts")
}
bs, err = json.MarshalIndent(block.ValueFlow, " ", " ")
if err != nil {
t.Fatalf("MarshalIndent() failed: %v", err)
}
valueFlowOutput := path.Join(tc.folder, "value-flow.output.json")
if err := os.WriteFile(valueFlowOutput, bs, 0644); err != nil {
t.Fatalf("WriteFile() failed: %v", err)
}
expectedValueFlowFilename := path.Join(tc.folder, "value-flow.expected.json")
data, err = os.ReadFile(expectedValueFlowFilename)
if err != nil {
t.Fatalf("ReadFile() failed: %v", err)
}
if bytes.Compare(bytes.Trim(bs, " \n"), bytes.Trim(data, " \n")) != 0 {
t.Errorf("ValueFlows differ")
}
// compare hashes
bs, err = json.MarshalIndent(txHashes, " ", " ")
if err != nil {
t.Fatalf("MarshalIndent() failed: %v", err)
}
hashesOutputFilename := path.Join(tc.folder, "tx-hashes.output.json")
if err := os.WriteFile(hashesOutputFilename, bs, 0644); err != nil {
t.Fatalf("WriteFile() failed: %v", err)
}
expectedHashesFilename := path.Join(tc.folder, "tx-hashes.expected.json")
data, err = os.ReadFile(expectedHashesFilename)
if err != nil {
t.Fatalf("ReadFile() failed: %v", err)
}
if bytes.Compare(bytes.Trim(bs, " \n"), bytes.Trim(data, " \n")) != 0 {
t.Errorf("tx hashes differ")
if bytes.Compare(bytes.Trim(content, " \n"), bytes.Trim(bs, " \n")) != 0 {
t.Errorf("block content mismatch")
}
})
}
Expand Down
87 changes: 87 additions & 0 deletions tlb/hashmap.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package tlb

import (
"encoding/json"
"errors"
"fmt"
"strings"

Expand Down Expand Up @@ -121,6 +122,55 @@ func (h *Hashmap[keyT, T]) UnmarshalTLB(c *boc.Cell, decoder *Decoder) error {
return nil
}

func hashmapAugExtraCountLeafs[keyT fixedSize](c *boc.Cell) (int, error) {
maybe, err := c.ReadBit()
if err != nil {
return 0, err
}
if !maybe {
return 0, nil
}
ref, err := c.NextRef()
if err != nil {
return 0, err
}
var s keyT
keySize := s.FixedSize()
return countLeafs(keySize, keySize, ref)
}

func countLeafs(keySize, leftKeySize int, c *boc.Cell) (int, error) {
if c.CellType() == boc.PrunedBranchCell {
return 0, errors.New("can't count leafs for hashmap with pruned branch cell")
}
size, err := loadLabelSize(leftKeySize, c)
if err != nil {
return 0, err
}
if keySize-leftKeySize+size < keySize {
// 0 bit branch
left, err := c.NextRef()
if err != nil {
return 0, err
}
leftCount, err := countLeafs(keySize, leftKeySize-(1+size), left)
if err != nil {
return 0, err
}
// 1 bit branch
right, err := c.NextRef()
if err != nil {
return 0, err
}
rightCount, err := countLeafs(keySize, leftKeySize-(1+size), right)
if err != nil {
return 0, err
}
return leftCount + rightCount, nil
}
return 1, nil
}

func (h *Hashmap[keyT, T]) mapInner(keySize, leftKeySize int, c *boc.Cell, keyPrefix *boc.BitString, decoder *Decoder) error {
var err error
var size int
Expand Down Expand Up @@ -456,6 +506,43 @@ func (h HashmapAugE[keyT, T1, T2]) Keys() []keyT {
return h.m.keys
}

func loadLabelSize(size int, c *boc.Cell) (int, error) {
first, err := c.ReadBit()
if err != nil {
return 0, err
}
// hml_short$0
if !first {
// Unary, while 1, add to ln
ln, err := c.ReadUnary()
if err != nil {
return 0, err
}
return int(ln), nil
}
second, err := c.ReadBit()
if err != nil {
return 0, err
}
// hml_long$10
if !second {
ln, err := c.ReadLimUint(size)
if err != nil {
return 0, err
}
return int(ln), nil
}
// hml_same$11
_, err = c.ReadBit()
if err != nil {
return 0, err
}
ln, err := c.ReadLimUint(size)
if err != nil {
return 0, err
}
return int(ln), nil
}
func loadLabel(size int, c *boc.Cell, key *boc.BitString) (int, *boc.BitString, error) {
first, err := c.ReadBit()
if err != nil {
Expand Down
Loading
Loading