WIP DAB interface improvement

dataavailability/dataavailability.go

@@ -7,12 +7,11 @@ import (
 
 	"github.com/0xPolygonHermez/zkevm-node/etherman/types"
 	"github.com/0xPolygonHermez/zkevm-node/log"
-	"github.com/0xPolygonHermez/zkevm-node/state"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/crypto"
 )
 
-const unexpectedHashTemplate = "missmatch on transaction data for batch num %d. Expected hash %s, actual hash: %s"
+const unexpectedHashTemplate = "mismatch on transaction data for batch num %d. Expected hash %s, actual hash: %s"
 
 // DataAvailability implements an abstract data availability integration
 type DataAvailability struct {
@@ -60,57 +59,81 @@ func (d *DataAvailability) PostSequence(ctx context.Context, sequences []types.S
 // 1. From local DB
 // 2. From Sequencer
 // 3. From DA backend
-func (d *DataAvailability) GetBatchL2Data(batchNum uint64, expectedTransactionsHash common.Hash) ([]byte, error) {
-	found := true
-	transactionsData, err := d.state.GetBatchL2DataByNumber(d.ctx, batchNum, nil)
-	if err != nil {
-		if err == state.ErrNotFound {
-			found = false
+func (d *DataAvailability) GetBatchL2Data(batchNums []uint64, batchHashes []common.Hash, dataAvailabilityMessage []byte) ([][]byte, error) {
+	if len(batchNums) != len(batchHashes) {
+		return nil, fmt.Errorf("invalid L2 batch data retrieval arguments, %d != %d", len(batchNums), len(batchHashes))
+	}
+
+	data, err := d.localData(batchNums, batchHashes)
+	if err == nil {
+		return data, nil
+	}
+
+	if !d.isTrustedSequencer {
+		data, err = d.trustedSequencerData(batchNums, batchHashes)
+		if err != nil {
+			log.Warnf("trusted sequencer failed to return data for batches %v: %s", batchNums, err.Error())
 		} else {
-			return nil, fmt.Errorf("failed to get batch data from state for batch num %d: %w", batchNum, err)
+			return data, nil
 		}
 	}
-	actualTransactionsHash := crypto.Keccak256Hash(transactionsData)
-	if !found || expectedTransactionsHash != actualTransactionsHash {
-		if found {
-			log.Warnf(unexpectedHashTemplate, batchNum, expectedTransactionsHash, actualTransactionsHash)
-		}
 
-		if !d.isTrustedSequencer {
-			log.Info("trying to get data from trusted sequencer")
-			data, err := d.getDataFromTrustedSequencer(batchNum, expectedTransactionsHash)
-			if err != nil {
-				log.Warn("failed to get data from trusted sequencer: %w", err)
-			} else {
-				return data, nil
-			}
-		}
+	return d.backend.GetSequence(d.ctx, batchHashes, dataAvailabilityMessage)
+}
 
-		log.Info("trying to get data from the data availability backend")
-		data, err := d.backend.GetBatchL2Data(batchNum, expectedTransactionsHash)
-		if err != nil {
-			log.Error("failed to get data from the data availability backend: %w", err)
-			if d.isTrustedSequencer {
-				return nil, fmt.Errorf("data not found on the local DB nor on any data committee member")
-			} else {
-				return nil, fmt.Errorf("data not found on the local DB, nor from the trusted sequencer nor on any data committee member")
-			}
+// localData retrieves batches from local database and returns an error unless all are found
+func (d *DataAvailability) localData(numbers []uint64, hashes []common.Hash) ([][]byte, error) {
+	data, err := d.state.GetBatchL2DataByNumbers(d.ctx, numbers, nil)
+	if err != nil {
+		return nil, err
+	}
+	var batches [][]byte
+	for i := 0; i < len(numbers); i++ {
+		batchNumber := numbers[i]
+		expectedHash := hashes[i]
+		batchData, ok := data[batchNumber]
+		if !ok {
+			return nil, fmt.Errorf("missing batch %v", batchNumber)
+		}
+		actualHash := crypto.Keccak256Hash(batchData)
+		if actualHash != expectedHash {
+			err = fmt.Errorf(unexpectedHashTemplate, batchNumber, expectedHash, actualHash)
+			log.Warnf("wrong local data for hash: %s", err.Error())
+			return nil, err
+		} else {
+			batches = append(batches, batchData)
 		}
-		return data, nil
 	}
-	return transactionsData, nil
+	return batches, nil
 }
 
-func (d *DataAvailability) getDataFromTrustedSequencer(batchNum uint64, expectedTransactionsHash common.Hash) ([]byte, error) {
-	b, err := d.zkEVMClient.BatchByNumber(d.ctx, new(big.Int).SetUint64(batchNum))
+// trustedSequencerData retrieved batch data from the trusted sequencer and returns an error unless all are found
+func (d *DataAvailability) trustedSequencerData(batchNums []uint64, expectedHashes []common.Hash) ([][]byte, error) {
+	if len(batchNums) != len(expectedHashes) {
+		return nil, fmt.Errorf("invalid arguments, len of batch numbers does not equal length of expected hashes: %d != %d",
+			len(batchNums), len(expectedHashes))
+	}
+	var nums []*big.Int
+	for _, n := range batchNums {
+		nums = append(nums, new(big.Int).SetUint64(n))
+	}
+	batchData, err := d.zkEVMClient.BatchesByNumbers(d.ctx, nums)
 	if err != nil {
-		return nil, fmt.Errorf("failed to get batch num %d from trusted sequencer: %w", batchNum, err)
+		return nil, err
 	}
-	actualTransactionsHash := crypto.Keccak256Hash(b.BatchL2Data)
-	if expectedTransactionsHash != actualTransactionsHash {
-		return nil, fmt.Errorf(
-			unexpectedHashTemplate, batchNum, expectedTransactionsHash, actualTransactionsHash,
-		)
+	if len(batchData) != len(batchNums) {
+		return nil, fmt.Errorf("missing batch data, expected %d, got %d", len(batchNums), len(batchData))
+	}
+	var result [][]byte
+	for i := 0; i < len(batchNums); i++ {
+		number := batchNums[i]
+		batch := batchData[i]
+		expectedTransactionsHash := expectedHashes[i]
+		actualTransactionsHash := crypto.Keccak256Hash(batch.BatchL2Data)
+		if expectedTransactionsHash != actualTransactionsHash {
+			return nil, fmt.Errorf(unexpectedHashTemplate, number, expectedTransactionsHash, actualTransactionsHash)
+		}
+		result = append(result, batch.BatchL2Data)
 	}
-	return b.BatchL2Data, nil
+	return result, nil
 }

dataavailability/datacommittee/datacommittee.go

@@ -20,7 +20,7 @@ import (
 	"golang.org/x/net/context"
 )
 
-const unexpectedHashTemplate = "missmatch on transaction data for batch num %d. Expected hash %s, actual hash: %s"
+const unexpectedHashTemplate = "missmatch on transaction data. Expected hash %s, actual hash: %s"
 
 // DataCommitteeMember represents a member of the Data Committee
 type DataCommitteeMember struct {
@@ -87,8 +87,22 @@ func (d *DataCommitteeBackend) Init() error {
 	return nil
 }
 
+// GetSequence gets backend data one hash at a time. This should be optimized on the DAC side to get them all at once.
+func (d *DataCommitteeBackend) GetSequence(ctx context.Context, hashes []common.Hash, dataAvailabilityMessage []byte) ([][]byte, error) {
+	// TODO: optimize this on the DAC side by implementing a multi batch retrieve api
+	var batchData [][]byte
+	for _, h := range hashes {
+		data, err := d.GetBatchL2Data(h)
+		if err != nil {
+			return nil, err
+		}
+		batchData = append(batchData, data)
+	}
+	return batchData, nil
+}
+
 // GetBatchL2Data returns the data from the DAC. It checks that it matches with the expected hash
-func (d *DataCommitteeBackend) GetBatchL2Data(batchNum uint64, hash common.Hash) ([]byte, error) {
+func (d *DataCommitteeBackend) GetBatchL2Data(hash common.Hash) ([]byte, error) {
 	intialMember := d.selectedCommitteeMember
 	found := false
 	for !found && intialMember != -1 {
@@ -110,7 +124,7 @@ func (d *DataCommitteeBackend) GetBatchL2Data(batchNum uint64, hash common.Hash)
 		actualTransactionsHash := crypto.Keccak256Hash(data)
 		if actualTransactionsHash != hash {
 			unexpectedHash := fmt.Errorf(
-				unexpectedHashTemplate, batchNum, hash, actualTransactionsHash,
+				unexpectedHashTemplate, hash, actualTransactionsHash,
 			)
 			log.Warnf(
 				"error getting data from DAC node %s at %s: %s",

dataavailability/interfaces.go

@@ -10,34 +10,49 @@ import (
 	"github.com/jackc/pgx/v4"
 )
 
+// DABackender is an interface for components that store and retrieve batch data
+type DABackender interface {
+	SequenceRetriever
+	SequenceSender
+	// Init initializes the DABackend
+	Init() error
+}
+
+// SequenceSender is used to send provided sequence of batches
+type SequenceSender interface {
+	// PostSequence sends the sequence data to the data availability backend, and returns the dataAvailabilityMessage
+	// as expected by the contract
+	PostSequence(ctx context.Context, batchesData [][]byte) ([]byte, error)
+}
+
+// SequenceRetriever is used to retrieve batch data
+type SequenceRetriever interface {
+	// GetSequence retrieves the sequence data from the data availability backend
+	GetSequence(ctx context.Context, batchHashes []common.Hash, dataAvailabilityMessage []byte) ([][]byte, error)
+}
+
+// === Internal interfaces ===
+
 type stateInterface interface {
 	GetBatchL2DataByNumber(ctx context.Context, batchNumber uint64, dbTx pgx.Tx) ([]byte, error)
+	GetBatchL2DataByNumbers(ctx context.Context, batchNumbers []uint64, dbTx pgx.Tx) (map[uint64][]byte, error)
 	GetBatchByNumber(ctx context.Context, batchNumber uint64, dbTx pgx.Tx) (*state.Batch, error)
 }
 
 // BatchDataProvider is used to retrieve batch data
 type BatchDataProvider interface {
 	// GetBatchL2Data retrieve the data of a batch from the DA backend. The returned data must be the pre-image of the hash
-	GetBatchL2Data(batchNum uint64, hash common.Hash) ([]byte, error)
+	GetBatchL2Data(batchNum []uint64, batchHashes []common.Hash, dataAvailabilityMessage []byte) ([][]byte, error)
 }
 
-// SequenceSender is used to send provided sequence of batches
-type SequenceSender interface {
-	// PostSequence sends the sequence data to the data availability backend, and returns the dataAvailabilityMessage
-	// as expected by the contract
-	PostSequence(ctx context.Context, batchesData [][]byte) ([]byte, error)
-}
-
-// DABackender is the interface needed to implement in order to
-// integrate a DA service
-type DABackender interface {
+// DataManager is an interface for components that send and retrieve batch data
+type DataManager interface {
 	BatchDataProvider
 	SequenceSender
-	// Init initializes the DABackend
-	Init() error
 }
 
 // ZKEVMClientTrustedBatchesGetter contains the methods required to interact with zkEVM-RPC
 type ZKEVMClientTrustedBatchesGetter interface {
 	BatchByNumber(ctx context.Context, number *big.Int) (*types.Batch, error)
+	BatchesByNumbers(ctx context.Context, numbers []*big.Int) ([]*types.BatchData, error)
 }

etherman/etherman.go

@@ -1264,9 +1264,10 @@ func decodeSequences(txData []byte, lastBatchNumber uint64, sequencer common.Add
 	if err != nil {
 		return nil, err
 	}
-	var sequences []polygonzkevm.PolygonRollupBaseEtrogBatchData
+
 	switch method.Name {
 	case "rollup": // TODO: put correct value
+		var sequences []polygonzkevm.PolygonRollupBaseEtrogBatchData
 		err = json.Unmarshal(bytedata, &sequences)
 		if err != nil {
 			return nil, err
@@ -1295,18 +1296,29 @@ func decodeSequences(txData []byte, lastBatchNumber uint64, sequencer common.Add
 			return nil, err
 		}
 		coinbase := (data[1]).(common.Address)
+		dataAvailabilityMessage := (data[2]).([]byte)
 		sequencedBatches := make([]SequencedBatch, len(sequencesValidium))
-		for i, seq := range sequencesValidium {
+		var batchNums []uint64
+		var hashes []common.Hash
+		for i, validiumData := range sequencesValidium {
 			bn := lastBatchNumber - uint64(len(sequencesValidium)-(i+1))
-			batchL2Data, err := da.GetBatchL2Data(bn, sequencesValidium[i].TransactionsHash)
-			if err != nil {
-				return nil, err
+			// ForcedBatches are skipped here. They are not currently enabled for validium, but once they are supported,
+			// their data must come from the state.forced_batch table
+			if validiumData.ForcedTimestamp == 0 {
+				batchNums = append(batchNums, bn)
+				hashes = append(hashes, validiumData.TransactionsHash)
 			}
+		}
+		batchL2Data, err := da.GetBatchL2Data(batchNums, hashes, dataAvailabilityMessage)
+		if err != nil {
+			return nil, err
+		}
+		for i, bn := range batchNums {
 			s := polygonzkevm.PolygonRollupBaseEtrogBatchData{
-				Transactions:         batchL2Data, // TODO: get data from DA
-				ForcedGlobalExitRoot: seq.ForcedGlobalExitRoot,
-				ForcedTimestamp:      seq.ForcedTimestamp,
-				ForcedBlockHashL1:    seq.ForcedBlockHashL1,
+				Transactions:         batchL2Data[i],
+				ForcedGlobalExitRoot: sequencesValidium[i].ForcedGlobalExitRoot,
+				ForcedTimestamp:      sequencesValidium[i].ForcedTimestamp,
+				ForcedBlockHashL1:    sequencesValidium[i].ForcedBlockHashL1,
 			}
 			sequencedBatches[i] = SequencedBatch{
 				BatchNumber:                     bn,
@@ -1318,7 +1330,6 @@ func decodeSequences(txData []byte, lastBatchNumber uint64, sequencer common.Add
 				PolygonRollupBaseEtrogBatchData: &s,
 			}
 		}
-
 		return sequencedBatches, nil
 	default:
 		return nil, fmt.Errorf("unexpected method called in sequence batches transaction: %s", method.RawName)

etherman/etherman_test.go

@@ -164,9 +164,12 @@ func TestSequencedBatchesEvent(t *testing.T) {
 	}, polygonzkevm.PolygonValidiumEtrogValidiumBatchData{
 		TransactionsHash: txsHash,
 	})
-	da.Mock.On("GetBatchL2Data", uint64(2), txsHash).Return(data, nil)
-	da.Mock.On("GetBatchL2Data", uint64(3), txsHash).Return(data, nil)
-	_, err = etherman.ZkEVM.SequenceBatchesValidium(auth, sequences, auth.From, []byte{})
+	batchNums := []uint64{2, 3}
+	batchHashes := []common.Hash{txsHash, txsHash}
+	batchData := [][]byte{data, data}
+	daMessage, _ := hex.DecodeString("0x123456789123456789")
+	da.Mock.On("GetBatchL2Data", batchNums, batchHashes, daMessage).Return(batchData, nil)
+	_, err = etherman.ZkEVM.SequenceBatchesValidium(auth, sequences, auth.From, daMessage)
 	require.NoError(t, err)
 
 	// Mine the tx in a block
@@ -204,9 +207,10 @@ func TestVerifyBatchEvent(t *testing.T) {
 	tx := polygonzkevm.PolygonValidiumEtrogValidiumBatchData{
 		TransactionsHash: crypto.Keccak256Hash(common.Hex2Bytes(rawTxs)),
 	}
-	_, err = etherman.ZkEVM.SequenceBatchesValidium(auth, []polygonzkevm.PolygonValidiumEtrogValidiumBatchData{tx}, auth.From, nil)
+	daMessage, _ := hex.DecodeString("0x1234")
+	_, err = etherman.ZkEVM.SequenceBatchesValidium(auth, []polygonzkevm.PolygonValidiumEtrogValidiumBatchData{tx}, auth.From, daMessage)
 	require.NoError(t, err)
-	da.Mock.On("GetBatchL2Data", uint64(2), crypto.Keccak256Hash(common.Hex2Bytes(rawTxs))).Return(common.Hex2Bytes(rawTxs), nil)
+	da.Mock.On("GetBatchL2Data", []uint64{2}, []common.Hash{crypto.Keccak256Hash(common.Hex2Bytes(rawTxs))}, daMessage).Return([][]byte{common.Hex2Bytes(rawTxs)}, nil)
 
 	// Mine the tx in a block
 	ethBackend.Commit()
@@ -317,9 +321,10 @@ func TestSendSequences(t *testing.T) {
 	sequence := ethmanTypes.Sequence{
 		BatchL2Data: batchL2Data,
 	}
-	tx, err := etherman.sequenceBatches(*auth, []ethmanTypes.Sequence{sequence}, auth.From, []byte{})
+	daMessage, _ := hex.DecodeString("0x1234")
+	tx, err := etherman.sequenceBatches(*auth, []ethmanTypes.Sequence{sequence}, auth.From, daMessage)
 	require.NoError(t, err)
-	da.Mock.On("GetBatchL2Data", uint64(2), crypto.Keccak256Hash(batchL2Data)).Return(batchL2Data, nil)
+	da.Mock.On("GetBatchL2Data", []uint64{2}, []common.Hash{crypto.Keccak256Hash(batchL2Data)}, daMessage).Return([][]byte{batchL2Data}, nil)
 	log.Debug("TX: ", tx.Hash())
 	ethBackend.Commit()
 

etherman/interfaces.go

@@ -3,5 +3,5 @@ package etherman
 import "github.com/ethereum/go-ethereum/common"
 
 type dataAvailabilityProvider interface {
-	GetBatchL2Data(batchNum uint64, hash common.Hash) ([]byte, error)
+	GetBatchL2Data(batchNum []uint64, hash []common.Hash, dataAvailabilityMessage []byte) ([][]byte, error)
 }