From a8f2741117d3944587225cfdb44259656c4d8397 Mon Sep 17 00:00:00 2001 From: Ho Vei Date: Thu, 6 Apr 2023 14:49:00 +0800 Subject: [PATCH 1/9] new deletion proof --- trie/zk_trie.go | 52 +++++------------ trie/zk_trie_proof_test.go | 10 ++-- trie/zktrie_deletionproof.go | 109 +++++++++++++++++++++++++++++++++++ 3 files changed, 131 insertions(+), 40 deletions(-) create mode 100644 trie/zktrie_deletionproof.go diff --git a/trie/zk_trie.go b/trie/zk_trie.go index 449b2aa8922e..627d3ee582ed 100644 --- a/trie/zk_trie.go +++ b/trie/zk_trie.go @@ -174,49 +174,29 @@ func (t *ZkTrie) NodeIterator(start []byte) NodeIterator { // nodes of the longest existing prefix of the key (at least the root node), ending // with the node that proves the absence of the key. func (t *ZkTrie) Prove(key []byte, fromLevel uint, proofDb ethdb.KeyValueWriter) error { - // omit sibling, which is not required for proving only - _, err := t.ProveWithDeletion(key, fromLevel, proofDb) - return err -} - -// ProveWithDeletion is the implement of Prove, it also return possible sibling node -// (if there is, i.e. the node of key exist and is not the only node in trie) -// so witness generator can predict the final state root after deletion of this key -// the returned sibling node has no key along with it for witness generator must decode -// the node for its purpose -func (t *ZkTrie) ProveWithDeletion(key []byte, fromLevel uint, proofDb ethdb.KeyValueWriter) (sibling []byte, err error) { - err = t.ZkTrie.ProveWithDeletion(key, fromLevel, - func(n *zktrie.Node) error { - nodeHash, err := n.NodeHash() - if err != nil { - return err - } + err := t.ZkTrie.Prove(key, fromLevel, func(n *zktrie.Node) error { + nodeHash, err := n.NodeHash() + if err != nil { + return err + } - if n.Type == zktrie.NodeTypeLeaf { - preImage := t.GetKey(n.NodeKey.Bytes()) - if len(preImage) > 0 { - n.KeyPreimage = &zkt.Byte32{} - copy(n.KeyPreimage[:], preImage) - //return fmt.Errorf("key preimage not found for [%x] ref %x", n.NodeKey.Bytes(), k.Bytes()) - } - } - return proofDb.Put(nodeHash[:], n.Value()) - }, - func(_ *zktrie.Node, n *zktrie.Node) { - // the sibling for each leaf should be unique except for EmptyNode - if n != nil && n.Type != zktrie.NodeTypeEmpty { - sibling = n.Value() + if n.Type == zktrie.NodeTypeLeaf { + preImage := t.GetKey(n.NodeKey.Bytes()) + if len(preImage) > 0 { + n.KeyPreimage = &zkt.Byte32{} + copy(n.KeyPreimage[:], preImage) + //return fmt.Errorf("key preimage not found for [%x] ref %x", n.NodeKey.Bytes(), k.Bytes()) } - }, - ) + } + return proofDb.Put(nodeHash[:], n.Value()) + }) if err != nil { - return + return err } // we put this special kv pair in db so we can distinguish the type and // make suitable Proof - err = proofDb.Put(magicHash, zktrie.ProofMagicBytes()) - return + return proofDb.Put(magicHash, zktrie.ProofMagicBytes()) } // VerifyProof checks merkle proofs. The given proof must contain the value for diff --git a/trie/zk_trie_proof_test.go b/trie/zk_trie_proof_test.go index c3652b7eed02..b1020ef68b64 100644 --- a/trie/zk_trie_proof_test.go +++ b/trie/zk_trie_proof_test.go @@ -217,11 +217,13 @@ func TestProofWithDeletion(t *testing.T) { s_key1, err := zkt.ToSecureKeyBytes(key1) assert.NoError(t, err) - sibling1, err := tr.ProveWithDeletion(s_key1.Bytes(), 0, proof) + delTracer := tr.NewDeletionTracer() + + sibling1, err := delTracer.ProveWithDeletion(s_key1.Bytes(), proof) assert.NoError(t, err) nd, err := tr.TryGet(key2) assert.NoError(t, err) - l := len(sibling1) + l := len(sibling1[0]) // a hacking to grep the value part directly from the encoded leaf node, // notice the sibling of key `k*32`` is just the leaf of key `m*32` assert.Equal(t, sibling1[l-33:l-1], nd) @@ -229,8 +231,8 @@ func TestProofWithDeletion(t *testing.T) { s_key2, err := zkt.ToSecureKeyBytes(bytes.Repeat([]byte("x"), 32)) assert.NoError(t, err) - sibling2, err := tr.ProveWithDeletion(s_key2.Bytes(), 0, proof) + sibling2, err := delTracer.ProveWithDeletion(s_key2.Bytes(), proof) assert.NoError(t, err) - assert.Nil(t, sibling2) + assert.Equal(t, 0, len(sibling2)) } diff --git a/trie/zktrie_deletionproof.go b/trie/zktrie_deletionproof.go new file mode 100644 index 000000000000..ce7a54f8cf00 --- /dev/null +++ b/trie/zktrie_deletionproof.go @@ -0,0 +1,109 @@ +package trie + +import ( + "bytes" + + zktrie "github.com/scroll-tech/zktrie/trie" + zkt "github.com/scroll-tech/zktrie/types" + + "github.com/scroll-tech/go-ethereum/ethdb" +) + +// Pick Node from its hash directly from database, notice it has different +// interface with the function of same name in `trie` +func (t *ZkTrie) TryGetNode(nodeHash *zkt.Hash) (*zktrie.Node, error) { + if bytes.Equal(nodeHash[:], zkt.HashZero[:]) { + return zktrie.NewEmptyNode(), nil + } + nBytes, err := t.db.Get(nodeHash[:]) + if err == zktrie.ErrKeyNotFound { + return nil, zktrie.ErrKeyNotFound + } else if err != nil { + return nil, err + } + return zktrie.NewNodeFromBytes(nBytes) +} + +type deletionProofTracer struct { + *ZkTrie + deletionTracer map[zkt.Hash]struct{} +} + +func (t *ZkTrie) NewDeletionTracer() *deletionProofTracer { + return &deletionProofTracer{ + ZkTrie: t, + deletionTracer: map[zkt.Hash]struct{}{zkt.HashZero: {}}, + } +} + +// ProveWithDeletion is the implement of Prove, it also return possible sibling node +// (if there is, i.e. the node of key exist and is not the only node in trie) +// so witness generator can predict the final state root after deletion of this key +// the returned sibling node has no key along with it for witness generator must decode +// the node for its purpose +func (t *deletionProofTracer) ProveWithDeletion(key []byte, proofDb ethdb.KeyValueWriter) (siblings [][]byte, err error) { + var mptPath []*zktrie.Node + err = t.ZkTrie.ProveWithDeletion(key, 0, + func(n *zktrie.Node) error { + nodeHash, err := n.NodeHash() + if err != nil { + return err + } + + if n.Type == zktrie.NodeTypeLeaf { + preImage := t.GetKey(n.NodeKey.Bytes()) + if len(preImage) > 0 { + n.KeyPreimage = &zkt.Byte32{} + copy(n.KeyPreimage[:], preImage) + } + } else if n.Type == zktrie.NodeTypeParent { + mptPath = append(mptPath, n) + } + + return proofDb.Put(nodeHash[:], n.Value()) + }, + func(delNode *zktrie.Node, n *zktrie.Node) { + nodeHash, _ := delNode.NodeHash() + t.deletionTracer[*nodeHash] = struct{}{} + + // the sibling for each leaf should be unique except for EmptyNode + if n != nil && n.Type != zktrie.NodeTypeEmpty { + siblings = append(siblings, n.Value()) + } + }, + ) + if err != nil { + return + } + + // now handle mptpath reversively + for i := len(mptPath); i > 0; i-- { + n := mptPath[i-1] + _, deletedL := t.deletionTracer[*n.ChildL] + _, deletedR := t.deletionTracer[*n.ChildR] + if deletedL && deletedR { + nodeHash, _ := n.NodeHash() + t.deletionTracer[*nodeHash] = struct{}{} + } else if i != len(mptPath) { + var siblingHash *zkt.Hash + if deletedL { + siblingHash = n.ChildR + } else if deletedR { + siblingHash = n.ChildL + } + if siblingHash != nil { + var sibling *zktrie.Node + sibling, err = t.TryGetNode(siblingHash) + if err != nil { + return + } + siblings = append(siblings, sibling.Value()) + } + } + } + + // we put this special kv pair in db so we can distinguish the type and + // make suitable Proof + err = proofDb.Put(magicHash, zktrie.ProofMagicBytes()) + return +} From a413434e44511f9a0a723aeebc71632c369e4f52 Mon Sep 17 00:00:00 2001 From: Ho Vei Date: Thu, 6 Apr 2023 23:42:28 +0800 Subject: [PATCH 2/9] complete tracer and test --- trie/zk_trie_proof_test.go | 10 +++-- trie/zktrie_deletionproof.go | 74 +++++++++++++++++++++++------------- 2 files changed, 53 insertions(+), 31 deletions(-) diff --git a/trie/zk_trie_proof_test.go b/trie/zk_trie_proof_test.go index b1020ef68b64..877015ebed94 100644 --- a/trie/zk_trie_proof_test.go +++ b/trie/zk_trie_proof_test.go @@ -219,20 +219,22 @@ func TestProofWithDeletion(t *testing.T) { delTracer := tr.NewDeletionTracer() - sibling1, err := delTracer.ProveWithDeletion(s_key1.Bytes(), proof) + err = delTracer.ProveWithDeletion(s_key1.Bytes(), proof) assert.NoError(t, err) nd, err := tr.TryGet(key2) assert.NoError(t, err) + sibling1 := delTracer.GetProofs() + assert.Equal(t, 1, len(sibling1)) l := len(sibling1[0]) // a hacking to grep the value part directly from the encoded leaf node, // notice the sibling of key `k*32`` is just the leaf of key `m*32` - assert.Equal(t, sibling1[l-33:l-1], nd) + assert.Equal(t, sibling1[0][l-33:l-1], nd) s_key2, err := zkt.ToSecureKeyBytes(bytes.Repeat([]byte("x"), 32)) assert.NoError(t, err) - sibling2, err := delTracer.ProveWithDeletion(s_key2.Bytes(), proof) + err = delTracer.ProveWithDeletion(s_key2.Bytes(), proof) assert.NoError(t, err) - assert.Equal(t, 0, len(sibling2)) + assert.Equal(t, len(sibling1), len(delTracer.GetProofs())) } diff --git a/trie/zktrie_deletionproof.go b/trie/zktrie_deletionproof.go index ce7a54f8cf00..5925098e8c6a 100644 --- a/trie/zktrie_deletionproof.go +++ b/trie/zktrie_deletionproof.go @@ -27,23 +27,35 @@ func (t *ZkTrie) TryGetNode(nodeHash *zkt.Hash) (*zktrie.Node, error) { type deletionProofTracer struct { *ZkTrie deletionTracer map[zkt.Hash]struct{} + proofs map[zkt.Hash][]byte } +// NewDeletionTracer create a deletion tracer object func (t *ZkTrie) NewDeletionTracer() *deletionProofTracer { return &deletionProofTracer{ ZkTrie: t, deletionTracer: map[zkt.Hash]struct{}{zkt.HashZero: {}}, + proofs: make(map[zkt.Hash][]byte), } } -// ProveWithDeletion is the implement of Prove, it also return possible sibling node -// (if there is, i.e. the node of key exist and is not the only node in trie) -// so witness generator can predict the final state root after deletion of this key -// the returned sibling node has no key along with it for witness generator must decode -// the node for its purpose -func (t *deletionProofTracer) ProveWithDeletion(key []byte, proofDb ethdb.KeyValueWriter) (siblings [][]byte, err error) { +// GetProofs collect the proofs +func (t *deletionProofTracer) GetProofs() (ret [][]byte) { + for _, bt := range t.proofs { + ret = append(ret, bt) + } + return +} + +// ProveWithDeletion act the same as Prove, while also trace the possible sibling node +// from a series deletion records, the collected deletion proofs being collect +// enabling witness generator to predict the final state root after executing any deletion +// in the traced series, no matter of the deletion occurs in any position of the mpt ops +// Note the collected sibling node has no key along with it since witness generator would +// always decode the node for its purpose +func (t *deletionProofTracer) ProveWithDeletion(key []byte, proofDb ethdb.KeyValueWriter) error { var mptPath []*zktrie.Node - err = t.ZkTrie.ProveWithDeletion(key, 0, + err := t.ZkTrie.ProveWithDeletion(key, 0, func(n *zktrie.Node) error { nodeHash, err := n.NodeHash() if err != nil { @@ -65,15 +77,22 @@ func (t *deletionProofTracer) ProveWithDeletion(key []byte, proofDb ethdb.KeyVal func(delNode *zktrie.Node, n *zktrie.Node) { nodeHash, _ := delNode.NodeHash() t.deletionTracer[*nodeHash] = struct{}{} - // the sibling for each leaf should be unique except for EmptyNode if n != nil && n.Type != zktrie.NodeTypeEmpty { - siblings = append(siblings, n.Value()) + nodeHash, _ := n.NodeHash() + t.proofs[*nodeHash] = n.Value() } + }, ) if err != nil { - return + return err + } + // we put this special kv pair in db so we can distinguish the type and + // make suitable Proof + err = proofDb.Put(magicHash, zktrie.ProofMagicBytes()) + if err != nil { + return err } // now handle mptpath reversively @@ -84,26 +103,27 @@ func (t *deletionProofTracer) ProveWithDeletion(key []byte, proofDb ethdb.KeyVal if deletedL && deletedR { nodeHash, _ := n.NodeHash() t.deletionTracer[*nodeHash] = struct{}{} - } else if i != len(mptPath) { - var siblingHash *zkt.Hash - if deletedL { - siblingHash = n.ChildR - } else if deletedR { - siblingHash = n.ChildL - } - if siblingHash != nil { - var sibling *zktrie.Node - sibling, err = t.TryGetNode(siblingHash) - if err != nil { - return + } else { + if i != len(mptPath) { + var siblingHash *zkt.Hash + if deletedL { + siblingHash = n.ChildR + } else if deletedR { + siblingHash = n.ChildL + } + if siblingHash != nil { + sibling, err := t.TryGetNode(siblingHash) + if err != nil { + return err + } + if sibling.Type != zktrie.NodeTypeEmpty { + t.proofs[*siblingHash] = sibling.Value() + } } - siblings = append(siblings, sibling.Value()) } + return nil } } - // we put this special kv pair in db so we can distinguish the type and - // make suitable Proof - err = proofDb.Put(magicHash, zktrie.ProofMagicBytes()) - return + return nil } From 3ca688c2d183992008cd2cc72612bed45a39d656 Mon Sep 17 00:00:00 2001 From: Ho Vei Date: Fri, 7 Apr 2023 14:42:00 +0800 Subject: [PATCH 3/9] extend proof for parallel tracing --- trie/zk_trie_proof_test.go | 61 +++++++++++--- trie/zktrie_deletionproof.go | 156 +++++++++++++++++++++-------------- 2 files changed, 144 insertions(+), 73 deletions(-) diff --git a/trie/zk_trie_proof_test.go b/trie/zk_trie_proof_test.go index 877015ebed94..073aad14a867 100644 --- a/trie/zk_trie_proof_test.go +++ b/trie/zk_trie_proof_test.go @@ -196,7 +196,7 @@ func randomZktrie(t *testing.T, n int) (*ZkTrie, map[string]*kv) { return tr, vals } -// Tests that new "proof with deletion" feature +// Tests that new "proof trace" feature func TestProofWithDeletion(t *testing.T) { tr, _ := NewZkTrie(common.Hash{}, NewZktrieDatabase((memorydb.New()))) mt := &zkTrieImplTestWrapper{tr.Tree()} @@ -217,24 +217,65 @@ func TestProofWithDeletion(t *testing.T) { s_key1, err := zkt.ToSecureKeyBytes(key1) assert.NoError(t, err) - delTracer := tr.NewDeletionTracer() + proofTracer := tr.NewProofTracer() - err = delTracer.ProveWithDeletion(s_key1.Bytes(), proof) + err = proofTracer.Prove(s_key1.Bytes(), 0, proof) assert.NoError(t, err) nd, err := tr.TryGet(key2) assert.NoError(t, err) - sibling1 := delTracer.GetProofs() - assert.Equal(t, 1, len(sibling1)) - l := len(sibling1[0]) + + s_key2, err := zkt.ToSecureKeyBytes(bytes.Repeat([]byte("x"), 32)) + assert.NoError(t, err) + + err = proofTracer.Prove(s_key2.Bytes(), 0, proof) + assert.NoError(t, err) + // assert.Equal(t, len(sibling1), len(delTracer.GetProofs())) + + siblings, err := proofTracer.GetDeletionProofs() + assert.NoError(t, err) + assert.Equal(t, 0, len(siblings)) + + proofTracer.MarkDeletion(s_key1.Bytes()) + siblings, err = proofTracer.GetDeletionProofs() + assert.NoError(t, err) + assert.Equal(t, 1, len(siblings)) + l := len(siblings[0]) // a hacking to grep the value part directly from the encoded leaf node, // notice the sibling of key `k*32`` is just the leaf of key `m*32` - assert.Equal(t, sibling1[0][l-33:l-1], nd) + assert.Equal(t, siblings[0][l-33:l-1], nd) - s_key2, err := zkt.ToSecureKeyBytes(bytes.Repeat([]byte("x"), 32)) + // no effect + proofTracer.MarkDeletion(s_key2.Bytes()) + siblings, err = proofTracer.GetDeletionProofs() + assert.NoError(t, err) + assert.Equal(t, 1, len(siblings)) + + key3 := bytes.Repeat([]byte("x"), 32) + err = mt.UpdateWord( + zkt.NewByte32FromBytesPaddingZero(key3), + zkt.NewByte32FromBytesPaddingZero(bytes.Repeat([]byte("z"), 32)), + ) + + proofTracer = tr.NewProofTracer() + err = proofTracer.Prove(s_key1.Bytes(), 0, proof) + assert.NoError(t, err) + err = proofTracer.Prove(s_key2.Bytes(), 0, proof) + assert.NoError(t, err) + + proofTracer.MarkDeletion(s_key1.Bytes()) + siblings, err = proofTracer.GetDeletionProofs() assert.NoError(t, err) + assert.Equal(t, 1, len(siblings)) - err = delTracer.ProveWithDeletion(s_key2.Bytes(), proof) + proofTracer.MarkDeletion(s_key2.Bytes()) + siblings, err = proofTracer.GetDeletionProofs() assert.NoError(t, err) - assert.Equal(t, len(sibling1), len(delTracer.GetProofs())) + assert.Equal(t, 2, len(siblings)) + // one of the siblings is just leaf for key2, while + // another one must be a middle node + match1 := bytes.Equal(siblings[0][l-33:l-1], nd) + match2 := bytes.Equal(siblings[1][l-33:l-1], nd) + assert.True(t, match1 || match2) + assert.False(t, match1 && match2) } diff --git a/trie/zktrie_deletionproof.go b/trie/zktrie_deletionproof.go index 5925098e8c6a..68dd1755e438 100644 --- a/trie/zktrie_deletionproof.go +++ b/trie/zktrie_deletionproof.go @@ -24,38 +24,107 @@ func (t *ZkTrie) TryGetNode(nodeHash *zkt.Hash) (*zktrie.Node, error) { return zktrie.NewNodeFromBytes(nBytes) } -type deletionProofTracer struct { +type proofTracer struct { *ZkTrie deletionTracer map[zkt.Hash]struct{} - proofs map[zkt.Hash][]byte + rawPaths map[string][]*zktrie.Node } -// NewDeletionTracer create a deletion tracer object -func (t *ZkTrie) NewDeletionTracer() *deletionProofTracer { - return &deletionProofTracer{ - ZkTrie: t, +// NewProofTracer create a proof tracer object +func (t *ZkTrie) NewProofTracer() *proofTracer { + return &proofTracer{ + ZkTrie: t, + // always consider 0 is "deleted" deletionTracer: map[zkt.Hash]struct{}{zkt.HashZero: {}}, - proofs: make(map[zkt.Hash][]byte), + rawPaths: make(map[string][]*zktrie.Node), } } -// GetProofs collect the proofs -func (t *deletionProofTracer) GetProofs() (ret [][]byte) { - for _, bt := range t.proofs { - ret = append(ret, bt) +// Merge merge the input tracer into current and return current tracer +func (t *proofTracer) Merge(another *proofTracer) *proofTracer { + + // sanity checking + if !bytes.Equal(t.Hash().Bytes(), another.Hash().Bytes()) { + panic("can not merge two proof tracer base on different trie") + } + + for k := range another.deletionTracer { + t.deletionTracer[k] = struct{}{} + } + + for k, v := range another.rawPaths { + t.rawPaths[k] = v } - return + + return t } -// ProveWithDeletion act the same as Prove, while also trace the possible sibling node -// from a series deletion records, the collected deletion proofs being collect -// enabling witness generator to predict the final state root after executing any deletion -// in the traced series, no matter of the deletion occurs in any position of the mpt ops +// GetDeletionProofs generate current deletionTracer and collect deletion proofs +// which is possible to be used from all rawPaths, which enabling witness generator +// to predict the final state root after executing any deletion +// along any of the rawpath, no matter of the deletion occurs in any position of the mpt ops // Note the collected sibling node has no key along with it since witness generator would // always decode the node for its purpose -func (t *deletionProofTracer) ProveWithDeletion(key []byte, proofDb ethdb.KeyValueWriter) error { +func (t *proofTracer) GetDeletionProofs() ([][]byte, error) { + + var ret [][]byte + + // check each path: reversively, skip the final leaf node + for _, path := range t.rawPaths { + + checkPath := path[:len(path)-1] + for i := len(checkPath); i > 0; i-- { + n := checkPath[i-1] + _, deletedL := t.deletionTracer[*n.ChildL] + _, deletedR := t.deletionTracer[*n.ChildR] + if deletedL && deletedR { + nodeHash, _ := n.NodeHash() + t.deletionTracer[*nodeHash] = struct{}{} + } else { + var siblingHash *zkt.Hash + if deletedL { + siblingHash = n.ChildR + } else if deletedR { + siblingHash = n.ChildL + } + if siblingHash != nil { + sibling, err := t.TryGetNode(siblingHash) + if err != nil { + return nil, err + } + if sibling.Type != zktrie.NodeTypeEmpty { + ret = append(ret, sibling.Value()) + } + } + break + } + } + + } + + return ret, nil + +} + +// MarkDeletion mark a key has been involved into deletion +func (t *proofTracer) MarkDeletion(key []byte) { + if path, existed := t.rawPaths[string(key)]; existed { + // sanity check + leafNode := path[len(path)-1] + if leafNode.Type != zktrie.NodeTypeLeaf { + panic("all path recorded in proofTrace should be ended with leafNode") + } + + nodeHash, _ := leafNode.NodeHash() + t.deletionTracer[*nodeHash] = struct{}{} + } +} + +// Prove act the same as zktrie.Prove, while also collect the raw path +// for collecting deletion proofs in a post-work +func (t *proofTracer) Prove(key []byte, fromLevel uint, proofDb ethdb.KeyValueWriter) error { var mptPath []*zktrie.Node - err := t.ZkTrie.ProveWithDeletion(key, 0, + err := t.ZkTrie.ProveWithDeletion(key, fromLevel, func(n *zktrie.Node) error { nodeHash, err := n.NodeHash() if err != nil { @@ -74,15 +143,11 @@ func (t *deletionProofTracer) ProveWithDeletion(key []byte, proofDb ethdb.KeyVal return proofDb.Put(nodeHash[:], n.Value()) }, - func(delNode *zktrie.Node, n *zktrie.Node) { - nodeHash, _ := delNode.NodeHash() - t.deletionTracer[*nodeHash] = struct{}{} - // the sibling for each leaf should be unique except for EmptyNode - if n != nil && n.Type != zktrie.NodeTypeEmpty { - nodeHash, _ := n.NodeHash() - t.proofs[*nodeHash] = n.Value() - } - + func(n *zktrie.Node, _ *zktrie.Node) { + // only "hit" path (i.e. the leaf node corresponding the input key can be found) + // would be add into tracer + mptPath = append(mptPath, n) + t.rawPaths[string(key)] = mptPath }, ) if err != nil { @@ -90,40 +155,5 @@ func (t *deletionProofTracer) ProveWithDeletion(key []byte, proofDb ethdb.KeyVal } // we put this special kv pair in db so we can distinguish the type and // make suitable Proof - err = proofDb.Put(magicHash, zktrie.ProofMagicBytes()) - if err != nil { - return err - } - - // now handle mptpath reversively - for i := len(mptPath); i > 0; i-- { - n := mptPath[i-1] - _, deletedL := t.deletionTracer[*n.ChildL] - _, deletedR := t.deletionTracer[*n.ChildR] - if deletedL && deletedR { - nodeHash, _ := n.NodeHash() - t.deletionTracer[*nodeHash] = struct{}{} - } else { - if i != len(mptPath) { - var siblingHash *zkt.Hash - if deletedL { - siblingHash = n.ChildR - } else if deletedR { - siblingHash = n.ChildL - } - if siblingHash != nil { - sibling, err := t.TryGetNode(siblingHash) - if err != nil { - return err - } - if sibling.Type != zktrie.NodeTypeEmpty { - t.proofs[*siblingHash] = sibling.Value() - } - } - } - return nil - } - } - - return nil + return proofDb.Put(magicHash, zktrie.ProofMagicBytes()) } From f7bb624ce1b380652d2efcf08b994e45e29cf2b2 Mon Sep 17 00:00:00 2001 From: Ho Vei Date: Fri, 7 Apr 2023 18:47:16 +0800 Subject: [PATCH 4/9] integrating into blocktrace --- core/state/state_prove.go | 85 +++++++++++++++++++++++++++++++++++ core/state/statedb.go | 47 +------------------ eth/tracers/api_blocktrace.go | 49 ++++++++++++++++++-- trie/zktrie_deletionproof.go | 14 +++--- 4 files changed, 139 insertions(+), 56 deletions(-) create mode 100644 core/state/state_prove.go diff --git a/core/state/state_prove.go b/core/state/state_prove.go new file mode 100644 index 000000000000..15b5f65c8cce --- /dev/null +++ b/core/state/state_prove.go @@ -0,0 +1,85 @@ +package state + +import ( + "errors" + "fmt" + + zktrie "github.com/scroll-tech/go-ethereum/trie" + zkt "github.com/scroll-tech/zktrie/types" + + "github.com/scroll-tech/go-ethereum/common" + "github.com/scroll-tech/go-ethereum/crypto" + "github.com/scroll-tech/go-ethereum/ethdb" +) + +type TrieProve interface { + Prove(key []byte, fromLevel uint, proofDb ethdb.KeyValueWriter) error +} + +type ZktrieProofTracer struct { + *zktrie.ProofTracer +} + +// MarkDeletion overwrite the underlayer method with secure key +func (t ZktrieProofTracer) MarkDeletion(key common.Hash) { + key_s, _ := zkt.ToSecureKeyBytes(key.Bytes()) + t.ProofTracer.MarkDeletion(key_s.Bytes()) +} + +// Merge overwrite underlayer method with proper argument +func (t ZktrieProofTracer) Merge(another ZktrieProofTracer) { + t.ProofTracer.Merge(another.ProofTracer) +} + +func (t ZktrieProofTracer) Avaliable() bool { + return t.ProofTracer != nil +} + +// NewProofTracer is not in Db interface and used explictily for reading proof in storage trie (not updated by the dirty value) +func (s *StateDB) NewProofTracer(trieS Trie) ZktrieProofTracer { + if s.IsZktrie() { + zkTrie := trieS.(*zktrie.ZkTrie) + if zkTrie == nil { + panic("unexpected trie type for zktrie") + } + return ZktrieProofTracer{zkTrie.NewProofTracer()} + } + return ZktrieProofTracer{} +} + +// GetStorageTrieForProof is not in Db interface and used explictily for reading proof in storage trie (not updated by the dirty value) +func (s *StateDB) GetStorageTrieForProof(addr common.Address) (Trie, error) { + + // try the trie in stateObject first, else we would create one + stateObject := s.getStateObject(addr) + if stateObject == nil { + return nil, errors.New("storage trie for requested address does not exist") + } + + trie := stateObject.trie + var err error + if trie == nil { + // use a new, temporary trie + trie, err = s.db.OpenStorageTrie(stateObject.addrHash, stateObject.data.Root) + if err != nil { + return nil, fmt.Errorf("can't create storage trie on root %s: %v ", stateObject.data.Root, err) + } + } + + return trie, nil +} + +// GetSecureTrieProof handle any interface with Prove (should be a Trie in most case) and +// deliver the proof in bytes +func (s *StateDB) GetSecureTrieProof(trieProve TrieProve, key common.Hash) ([][]byte, error) { + + var proof proofList + var err error + if s.IsZktrie() { + key_s, _ := zkt.ToSecureKeyBytes(key.Bytes()) + err = trieProve.Prove(key_s.Bytes(), 0, &proof) + } else { + err = trieProve.Prove(crypto.Keccak256(key.Bytes()), 0, &proof) + } + return proof, err +} diff --git a/core/state/statedb.go b/core/state/statedb.go index 8dc4cc8abff2..61898bd14011 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -350,56 +350,13 @@ func (s *StateDB) GetRootHash() common.Hash { return s.trie.Hash() } -// StorageTrieProof is not in Db interface and used explictily for reading proof in storage trie (not the dirty value) -// For zktrie it also provide required data for predict the deletion, else it just fallback to GetStorageProof -func (s *StateDB) GetStorageTrieProof(a common.Address, key common.Hash) ([][]byte, []byte, error) { - - // try the trie in stateObject first, else we would create one - stateObject := s.getStateObject(a) - if stateObject == nil { - return nil, nil, errors.New("storage trie for requested address does not exist") - } - - trieS := stateObject.trie - var err error - if trieS == nil { - // use a new, temporary trie - trieS, err = s.db.OpenStorageTrie(stateObject.addrHash, stateObject.data.Root) - if err != nil { - return nil, nil, fmt.Errorf("can't create storage trie on root %s: %v ", stateObject.data.Root, err) - } - } - - var proof proofList - var sibling []byte - if s.IsZktrie() { - zkTrie := trieS.(*trie.ZkTrie) - if zkTrie == nil { - panic("unexpected trie type for zktrie") - } - key_s, _ := zkt.ToSecureKeyBytes(key.Bytes()) - sibling, err = zkTrie.ProveWithDeletion(key_s.Bytes(), 0, &proof) - } else { - err = trieS.Prove(crypto.Keccak256(key.Bytes()), 0, &proof) - } - return proof, sibling, err -} - // GetStorageProof returns the Merkle proof for given storage slot. func (s *StateDB) GetStorageProof(a common.Address, key common.Hash) ([][]byte, error) { - var proof proofList trie := s.StorageTrie(a) if trie == nil { - return proof, errors.New("storage trie for requested address does not exist") + return nil, errors.New("storage trie for requested address does not exist") } - var err error - if s.IsZktrie() { - key_s, _ := zkt.ToSecureKeyBytes(key.Bytes()) - err = trie.Prove(key_s.Bytes(), 0, &proof) - } else { - err = trie.Prove(crypto.Keccak256(key.Bytes()), 0, &proof) - } - return proof, err + return s.GetSecureTrieProof(trie, key) } // GetCommittedState retrieves a value from the given account's committed storage trie. diff --git a/eth/tracers/api_blocktrace.go b/eth/tracers/api_blocktrace.go index 18dac4539633..2998bbccbaef 100644 --- a/eth/tracers/api_blocktrace.go +++ b/eth/tracers/api_blocktrace.go @@ -1,6 +1,7 @@ package tracers import ( + "bytes" "context" "errors" "fmt" @@ -42,6 +43,8 @@ type traceEnv struct { // this lock is used to protect StorageTrace's read and write mutual exclusion. sMu sync.Mutex *types.StorageTrace + // zktrie tracer is used for zktrie storage to build additional deletion proof + zkTrieTracer map[string]state.ZktrieProofTracer executionResults []*types.ExecutionResult } @@ -119,6 +122,7 @@ func (api *API) createTraceEnv(ctx context.Context, config *TraceConfig, block * Proofs: make(map[string][]hexutil.Bytes), StorageProofs: make(map[string]map[string][]hexutil.Bytes), }, + zkTrieTracer: make(map[string]state.ZktrieProofTracer), executionResults: make([]*types.ExecutionResult, block.Transactions().Len()), } @@ -189,6 +193,18 @@ func (api *API) getBlockTrace(block *types.Block, env *traceEnv) (*types.BlockTr close(jobs) pend.Wait() + // after all tx has been traced, collect "deletion proof" for zktrie + for _, tracer := range env.zkTrieTracer { + delProofs, err := tracer.GetDeletionProofs() + if err != nil { + log.Error("deletion proof failure", "error", err) + } else { + for _, proof := range delProofs { + env.DeletionProofs = append(env.DeletionProofs, proof) + } + } + } + // If execution failed in between, abort select { case err := <-errCh: @@ -299,22 +315,47 @@ func (api *API) getTxResult(env *traceEnv, state *state.StateDB, index int, bloc proofStorages := tracer.UpdatedStorages() for addr, keys := range proofStorages { - for key := range keys { + env.sMu.Lock() + trie, err := state.GetStorageTrieForProof(addr) + if err != nil { + // but we still continue to next address + log.Error("Storage trie not available", "error", err, "address", addr) + env.sMu.Unlock() + continue + } + zktrieTracer := state.NewProofTracer(trie) + env.sMu.Unlock() + + for key, values := range keys { addrStr := addr.String() keyStr := key.String() + isDelete := bytes.Equal(values.Bytes(), common.Hash{}.Bytes()) env.sMu.Lock() m, existed := env.StorageProofs[addrStr] if !existed { m = make(map[string][]hexutil.Bytes) env.StorageProofs[addrStr] = m + if zktrieTracer.Avaliable() { + env.zkTrieTracer[addrStr] = zktrieTracer + } } else if _, existed := m[keyStr]; existed { + // still need to touch tracer for deletion + if isDelete && zktrieTracer.Avaliable() { + env.zkTrieTracer[addrStr].MarkDeletion(key) + } env.sMu.Unlock() continue } env.sMu.Unlock() - proof, sibling, err := state.GetStorageTrieProof(addr, key) + var proof [][]byte + var err error + if zktrieTracer.Avaliable() { + proof, err = state.GetSecureTrieProof(zktrieTracer, key) + } else { + proof, err = state.GetSecureTrieProof(trie, key) + } if err != nil { log.Error("Storage proof not available", "error", err, "address", addrStr, "key", keyStr) // but we still mark the proofs map with nil array @@ -325,8 +366,8 @@ func (api *API) getTxResult(env *traceEnv, state *state.StateDB, index int, bloc } env.sMu.Lock() m[keyStr] = wrappedProof - if sibling != nil { - env.DeletionProofs = append(env.DeletionProofs, sibling) + if zktrieTracer.Avaliable() { + env.zkTrieTracer[addrStr].Merge(zktrieTracer) } env.sMu.Unlock() } diff --git a/trie/zktrie_deletionproof.go b/trie/zktrie_deletionproof.go index 68dd1755e438..5cb6dd97ee38 100644 --- a/trie/zktrie_deletionproof.go +++ b/trie/zktrie_deletionproof.go @@ -24,15 +24,15 @@ func (t *ZkTrie) TryGetNode(nodeHash *zkt.Hash) (*zktrie.Node, error) { return zktrie.NewNodeFromBytes(nBytes) } -type proofTracer struct { +type ProofTracer struct { *ZkTrie deletionTracer map[zkt.Hash]struct{} rawPaths map[string][]*zktrie.Node } // NewProofTracer create a proof tracer object -func (t *ZkTrie) NewProofTracer() *proofTracer { - return &proofTracer{ +func (t *ZkTrie) NewProofTracer() *ProofTracer { + return &ProofTracer{ ZkTrie: t, // always consider 0 is "deleted" deletionTracer: map[zkt.Hash]struct{}{zkt.HashZero: {}}, @@ -41,7 +41,7 @@ func (t *ZkTrie) NewProofTracer() *proofTracer { } // Merge merge the input tracer into current and return current tracer -func (t *proofTracer) Merge(another *proofTracer) *proofTracer { +func (t *ProofTracer) Merge(another *ProofTracer) *ProofTracer { // sanity checking if !bytes.Equal(t.Hash().Bytes(), another.Hash().Bytes()) { @@ -65,7 +65,7 @@ func (t *proofTracer) Merge(another *proofTracer) *proofTracer { // along any of the rawpath, no matter of the deletion occurs in any position of the mpt ops // Note the collected sibling node has no key along with it since witness generator would // always decode the node for its purpose -func (t *proofTracer) GetDeletionProofs() ([][]byte, error) { +func (t *ProofTracer) GetDeletionProofs() ([][]byte, error) { var ret [][]byte @@ -107,7 +107,7 @@ func (t *proofTracer) GetDeletionProofs() ([][]byte, error) { } // MarkDeletion mark a key has been involved into deletion -func (t *proofTracer) MarkDeletion(key []byte) { +func (t *ProofTracer) MarkDeletion(key []byte) { if path, existed := t.rawPaths[string(key)]; existed { // sanity check leafNode := path[len(path)-1] @@ -122,7 +122,7 @@ func (t *proofTracer) MarkDeletion(key []byte) { // Prove act the same as zktrie.Prove, while also collect the raw path // for collecting deletion proofs in a post-work -func (t *proofTracer) Prove(key []byte, fromLevel uint, proofDb ethdb.KeyValueWriter) error { +func (t *ProofTracer) Prove(key []byte, fromLevel uint, proofDb ethdb.KeyValueWriter) error { var mptPath []*zktrie.Node err := t.ZkTrie.ProveWithDeletion(key, fromLevel, func(n *zktrie.Node) error { From c1c2c00df1901408ed2a229f7299351f28a6fa96 Mon Sep 17 00:00:00 2001 From: Ho Vei Date: Fri, 7 Apr 2023 19:26:18 +0800 Subject: [PATCH 5/9] lint --- core/state/state_prove.go | 5 +++-- eth/tracers/api_blocktrace.go | 8 ++++---- trie/zk_trie_proof_test.go | 1 + 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/core/state/state_prove.go b/core/state/state_prove.go index 15b5f65c8cce..5016494db4d6 100644 --- a/core/state/state_prove.go +++ b/core/state/state_prove.go @@ -4,9 +4,10 @@ import ( "errors" "fmt" - zktrie "github.com/scroll-tech/go-ethereum/trie" zkt "github.com/scroll-tech/zktrie/types" + zktrie "github.com/scroll-tech/go-ethereum/trie" + "github.com/scroll-tech/go-ethereum/common" "github.com/scroll-tech/go-ethereum/crypto" "github.com/scroll-tech/go-ethereum/ethdb" @@ -31,7 +32,7 @@ func (t ZktrieProofTracer) Merge(another ZktrieProofTracer) { t.ProofTracer.Merge(another.ProofTracer) } -func (t ZktrieProofTracer) Avaliable() bool { +func (t ZktrieProofTracer) Available() bool { return t.ProofTracer != nil } diff --git a/eth/tracers/api_blocktrace.go b/eth/tracers/api_blocktrace.go index 2998bbccbaef..2ea45694961d 100644 --- a/eth/tracers/api_blocktrace.go +++ b/eth/tracers/api_blocktrace.go @@ -336,12 +336,12 @@ func (api *API) getTxResult(env *traceEnv, state *state.StateDB, index int, bloc if !existed { m = make(map[string][]hexutil.Bytes) env.StorageProofs[addrStr] = m - if zktrieTracer.Avaliable() { + if zktrieTracer.Available() { env.zkTrieTracer[addrStr] = zktrieTracer } } else if _, existed := m[keyStr]; existed { // still need to touch tracer for deletion - if isDelete && zktrieTracer.Avaliable() { + if isDelete && zktrieTracer.Available() { env.zkTrieTracer[addrStr].MarkDeletion(key) } env.sMu.Unlock() @@ -351,7 +351,7 @@ func (api *API) getTxResult(env *traceEnv, state *state.StateDB, index int, bloc var proof [][]byte var err error - if zktrieTracer.Avaliable() { + if zktrieTracer.Available() { proof, err = state.GetSecureTrieProof(zktrieTracer, key) } else { proof, err = state.GetSecureTrieProof(trie, key) @@ -366,7 +366,7 @@ func (api *API) getTxResult(env *traceEnv, state *state.StateDB, index int, bloc } env.sMu.Lock() m[keyStr] = wrappedProof - if zktrieTracer.Avaliable() { + if zktrieTracer.Available() { env.zkTrieTracer[addrStr].Merge(zktrieTracer) } env.sMu.Unlock() diff --git a/trie/zk_trie_proof_test.go b/trie/zk_trie_proof_test.go index 073aad14a867..0109e9be859e 100644 --- a/trie/zk_trie_proof_test.go +++ b/trie/zk_trie_proof_test.go @@ -255,6 +255,7 @@ func TestProofWithDeletion(t *testing.T) { zkt.NewByte32FromBytesPaddingZero(key3), zkt.NewByte32FromBytesPaddingZero(bytes.Repeat([]byte("z"), 32)), ) + assert.NoError(t, err) proofTracer = tr.NewProofTracer() err = proofTracer.Prove(s_key1.Bytes(), 0, proof) From e649e1b6c6ee570815dded480d0318bc1c98c6a7 Mon Sep 17 00:00:00 2001 From: Ho Vei Date: Mon, 10 Apr 2023 12:58:14 +0800 Subject: [PATCH 6/9] deduplication of deletion proofs --- trie/zktrie_deletionproof.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/trie/zktrie_deletionproof.go b/trie/zktrie_deletionproof.go index 5cb6dd97ee38..ebb8419e7adb 100644 --- a/trie/zktrie_deletionproof.go +++ b/trie/zktrie_deletionproof.go @@ -67,7 +67,7 @@ func (t *ProofTracer) Merge(another *ProofTracer) *ProofTracer { // always decode the node for its purpose func (t *ProofTracer) GetDeletionProofs() ([][]byte, error) { - var ret [][]byte + retMap := map[zkt.Hash][]byte{} // check each path: reversively, skip the final leaf node for _, path := range t.rawPaths { @@ -93,13 +93,17 @@ func (t *ProofTracer) GetDeletionProofs() ([][]byte, error) { return nil, err } if sibling.Type != zktrie.NodeTypeEmpty { - ret = append(ret, sibling.Value()) + retMap[*siblingHash] = sibling.Value() } } break } } + } + var ret [][]byte + for _, bt := range retMap { + ret = append(ret, bt) } return ret, nil From 8973c8e37ce60c5aa9d490c4fcc36d4243061ec9 Mon Sep 17 00:00:00 2001 From: Ho Vei Date: Wed, 12 Apr 2023 10:29:49 +0800 Subject: [PATCH 7/9] fix an issue on marking deletion --- eth/tracers/api_blocktrace.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/eth/tracers/api_blocktrace.go b/eth/tracers/api_blocktrace.go index 2ea45694961d..47bf7ea971c4 100644 --- a/eth/tracers/api_blocktrace.go +++ b/eth/tracers/api_blocktrace.go @@ -367,6 +367,9 @@ func (api *API) getTxResult(env *traceEnv, state *state.StateDB, index int, bloc env.sMu.Lock() m[keyStr] = wrappedProof if zktrieTracer.Available() { + if isDelete { + zktrieTracer.MarkDeletion(key) + } env.zkTrieTracer[addrStr].Merge(zktrieTracer) } env.sMu.Unlock() From cd5b5547e01120cf27898db1eace00898a1e4a2c Mon Sep 17 00:00:00 2001 From: Ho Vei Date: Mon, 24 Apr 2023 21:33:36 +0800 Subject: [PATCH 8/9] fixs since last review --- core/state/state_prove.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/core/state/state_prove.go b/core/state/state_prove.go index 5016494db4d6..95c54988dc18 100644 --- a/core/state/state_prove.go +++ b/core/state/state_prove.go @@ -1,7 +1,6 @@ package state import ( - "errors" "fmt" zkt "github.com/scroll-tech/zktrie/types" @@ -54,7 +53,10 @@ func (s *StateDB) GetStorageTrieForProof(addr common.Address) (Trie, error) { // try the trie in stateObject first, else we would create one stateObject := s.getStateObject(addr) if stateObject == nil { - return nil, errors.New("storage trie for requested address does not exist") + // still return a empty trie + addrHash := crypto.Keccak256Hash(addr[:]) + dummy_trie, _ := s.db.OpenStorageTrie(addrHash, common.Hash{}) + return dummy_trie, nil } trie := stateObject.trie From 78b66e6d6836a2d10e5589b876394f49470197bb Mon Sep 17 00:00:00 2001 From: HAOYUatHZ <37070449+HAOYUatHZ@users.noreply.github.com> Date: Mon, 8 May 2023 15:25:15 +0800 Subject: [PATCH 9/9] Update version.go --- params/version.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/params/version.go b/params/version.go index 6496343e6576..a22915db1ed8 100644 --- a/params/version.go +++ b/params/version.go @@ -24,7 +24,7 @@ import ( const ( VersionMajor = 3 // Major version component of the current release VersionMinor = 1 // Minor version component of the current release - VersionPatch = 10 // Patch version component of the current release + VersionPatch = 11 // Patch version component of the current release VersionMeta = "alpha" // Version metadata to append to the version string )