WIP DAB interface improvement

dataavailability/dataavailability.go

@@ -7,12 +7,11 @@
 
 	"github.com/0xPolygonHermez/zkevm-node/etherman/types"
 	"github.com/0xPolygonHermez/zkevm-node/log"
-	"github.com/0xPolygonHermez/zkevm-node/state"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/crypto"
 )
 
-const unexpectedHashTemplate = "missmatch on transaction data for batch num %d. Expected hash %s, actual hash: %s"
+const unexpectedHashTemplate = "mismatch on transaction data for batch num %d. Expected hash %s, actual hash: %s"
 
 // DataAvailability implements an abstract data availability integration
 type DataAvailability struct {
@@ -60,48 +59,72 @@
 // 1. From local DB
 // 2. From Sequencer
 // 3. From DA backend
-func (d *DataAvailability) GetBatchL2Data(batchNum uint64, expectedTransactionsHash common.Hash) ([]byte, error) {
-	found := true
-	transactionsData, err := d.state.GetBatchL2DataByNumber(d.ctx, batchNum, nil)
+func (d *DataAvailability) GetBatchL2Data(batchNums []uint64, batchHashes []common.Hash, dataAvailabilityMessage []byte) ([][]byte, error) {
+	if len(batchNums) != len(batchHashes) {
+		return nil, fmt.Errorf("invalid L2 batch data retrieval arguments, %d != %d", len(batchNums), len(batchHashes))
+	}
+
+	data, err := d.localData(batchNums, batchHashes)
 	if err != nil {
-		if err == state.ErrNotFound {
-			found = false
+		log.Warnf("error retrieving local data for batches %v: %s", batchNums, err.Error())
+	} else {
+		return data, nil
+	}
+
+	if !d.isTrustedSequencer {
+		data, err = d.trustedSeqData(batchNums, batchHashes)
+		if err != nil {
+			log.Warnf("error retrieving trusted sequencer data for batches %v: %s", batchNums, err.Error())
 		} else {
-			return nil, fmt.Errorf("failed to get batch data from state for batch num %d: %w", batchNum, err)
+			return data, nil
 		}
 	}
-	actualTransactionsHash := crypto.Keccak256Hash(transactionsData)
-	if !found || expectedTransactionsHash != actualTransactionsHash {
-		if found {
-			log.Warnf(unexpectedHashTemplate, batchNum, expectedTransactionsHash, actualTransactionsHash)
-		}
 
-		if !d.isTrustedSequencer {
-			log.Info("trying to get data from trusted sequencer")
-			data, err := d.getDataFromTrustedSequencer(batchNum, expectedTransactionsHash)
-			if err != nil {
-				log.Warn("failed to get data from trusted sequencer: %w", err)
-			} else {
-				return data, nil
-			}
+	return d.backend.GetSequence(d.ctx, batchHashes, dataAvailabilityMessage)
+}
+
+func (d *DataAvailability) localData(numbers []uint64, hashes []common.Hash) ([][]byte, error) {
+	data, err := d.state.GetBatchL2DataByNumbers(d.ctx, numbers, nil)
+	if err != nil {
+		return nil, err
+	}
+	var batches [][]byte
+	for i := 0; i < len(numbers); i++ {
+		batchNumber := numbers[i]
+		expectedHash := hashes[i]
+		batchData, ok := data[batchNumber]
+		if !ok {
+			return nil, fmt.Errorf("could not get data locally for batch numbers %v", numbers)
 		}
+		actualHash := crypto.Keccak256Hash(batchData)
+		if actualHash != expectedHash {
+			log.Warnf(unexpectedHashTemplate, batchNumber, expectedHash, actualHash)
+		} else {
+			batches = append(batches, batchData)
+		}
+	}
+	return batches, nil
+}
 
-		log.Info("trying to get data from the data availability backend")
-		data, err := d.backend.GetBatchL2Data(batchNum, expectedTransactionsHash)
-		if err != nil {
-			log.Error("failed to get data from the data availability backend: %w", err)
-			if d.isTrustedSequencer {
-				return nil, fmt.Errorf("data not found on the local DB nor on any data committee member")
-			} else {
-				return nil, fmt.Errorf("data not found on the local DB, nor from the trusted sequencer nor on any data committee member")
-			}
+func (d *DataAvailability) trustedSeqData(numbers []uint64, hashes []common.Hash) ([][]byte, error) {
+	data, err := d.getBatchesDataFromTrustedSequencer(numbers, hashes)
+	if err != nil {
+		return nil, err
+	}
+	var batches [][]byte
+	for i := 0; i < len(numbers); i++ {
+		batchNumber := numbers[i]
+		// hash has already been checked
+		batchData, ok := data[batchNumber]
+		if !ok {
+			continue
 		}
-		return data, nil
+		batches[i] = batchData
 	}
-	return transactionsData, nil
+	return batches, nil
 }
 
 func (d *DataAvailability) getDataFromTrustedSequencer(batchNum uint64, expectedTransactionsHash common.Hash) ([]byte, error) {
 	b, err := d.zkEVMClient.BatchByNumber(d.ctx, new(big.Int).SetUint64(batchNum))
 	if err != nil {
 		return nil, fmt.Errorf("failed to get batch num %d from trusted sequencer: %w", batchNum, err)
@@ -114,3 +137,34 @@
 	}
 	return b.BatchL2Data, nil
 }
+
+func (d *DataAvailability) getBatchesDataFromTrustedSequencer(batchNums []uint64, expectedHashes []common.Hash) (map[uint64][]byte, error) {
+	if len(batchNums) != len(expectedHashes) {
+		return nil, fmt.Errorf("invalid arguments, len of batch numbers does not equal length of expected hashes: %d != %d",
+			len(batchNums), len(expectedHashes))
+	}
+	var nums []*big.Int
+	for _, n := range batchNums {
+		nums = append(nums, new(big.Int).SetUint64(n))
+	}
+	batchData, err := d.zkEVMClient.BatchesByNumbers(d.ctx, nums)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get batches %v data from trusted sequencer: %w", batchNums, err)
+	}
+	result := make(map[uint64][]byte)
+	for i := 0; i < len(batchNums); i++ {
+		number := batchNums[i]
+		batch := batchData[i]
+		if batch.Empty {
+			continue
+		}
+		expectedTransactionsHash := expectedHashes[i]
+		actualTransactionsHash := crypto.Keccak256Hash(batch.BatchL2Data)
+		if expectedTransactionsHash != actualTransactionsHash {
+			log.Warnf(unexpectedHashTemplate, number, expectedTransactionsHash, actualTransactionsHash)
+			continue
+		}
+		result[number] = batch.BatchL2Data
+	}
+	return result, nil
+}

dataavailability/datacommittee/datacommittee.go

@@ -20,7 +20,7 @@
 	"golang.org/x/net/context"
 )
 
-const unexpectedHashTemplate = "missmatch on transaction data for batch num %d. Expected hash %s, actual hash: %s"
+const unexpectedHashTemplate = "missmatch on transaction data. Expected hash %s, actual hash: %s"
 
 // DataCommitteeMember represents a member of the Data Committee
 type DataCommitteeMember struct {
@@ -87,8 +87,22 @@
 	return nil
 }
 
+func (d *DataCommitteeBackend) GetSequence(ctx context.Context, hashes []common.Hash, dataAvailabilityMessage []byte) ([][]byte, error) {
+	// TODO: optimize this on the DAC side by implementing a multi batch retrieve api
+	// FIXME: how to use dataAvailabilityMessage ?
+	var batchData [][]byte
+	for _, h := range hashes {
+		data, err := d.GetBatchL2Data(h)
+		if err != nil {
+			return nil, err
+		}
+		batchData = append(batchData, data)
+	}
+	return batchData, nil
+}
+
 // GetBatchL2Data returns the data from the DAC. It checks that it matches with the expected hash
-func (d *DataCommitteeBackend) GetBatchL2Data(batchNum uint64, hash common.Hash) ([]byte, error) {
+func (d *DataCommitteeBackend) GetBatchL2Data(hash common.Hash) ([]byte, error) {
 	intialMember := d.selectedCommitteeMember
 	found := false
 	for !found && intialMember != -1 {
@@ -110,7 +124,7 @@
 		actualTransactionsHash := crypto.Keccak256Hash(data)
 		if actualTransactionsHash != hash {
 			unexpectedHash := fmt.Errorf(
-				unexpectedHashTemplate, batchNum, hash, actualTransactionsHash,
+				unexpectedHashTemplate, hash, actualTransactionsHash,
 			)
 			log.Warnf(
 				"error getting data from DAC node %s at %s: %s",

dataavailability/interfaces.go

@@ -12,13 +12,14 @@
 
 type stateInterface interface {
 	GetBatchL2DataByNumber(ctx context.Context, batchNumber uint64, dbTx pgx.Tx) ([]byte, error)
+	GetBatchL2DataByNumbers(ctx context.Context, batchNumbers []uint64, dbTx pgx.Tx) (map[uint64][]byte, error)
 	GetBatchByNumber(ctx context.Context, batchNumber uint64, dbTx pgx.Tx) (*state.Batch, error)
 }
 
 // BatchDataProvider is used to retrieve batch data
 type BatchDataProvider interface {
 	// GetBatchL2Data retrieve the data of a batch from the DA backend. The returned data must be the pre-image of the hash
-	GetBatchL2Data(batchNum uint64, hash common.Hash) ([]byte, error)
+	GetBatchL2Data(batchNum []uint64, batchHashes []common.Hash, dataAvailabilityMessage []byte) ([][]byte, error)
 }
 
 // SequenceSender is used to send provided sequence of batches
@@ -28,16 +29,24 @@
 	PostSequence(ctx context.Context, batchesData [][]byte) ([]byte, error)
 }
 
-// DABackender is the interface needed to implement in order to
-// integrate a DA service
-type DABackender interface {
+type SequenceRetriever interface {
+	GetSequence(ctx context.Context, batchHashes []common.Hash, dataAvailabilityMessage []byte) ([][]byte, error)
+}
+
+type DataManager interface {
 	BatchDataProvider
 	SequenceSender
+}
+
+type DABackender interface {
+	SequenceRetriever
+	SequenceSender
 	// Init initializes the DABackend
 	Init() error
 }
 
 // ZKEVMClientTrustedBatchesGetter contains the methods required to interact with zkEVM-RPC
 type ZKEVMClientTrustedBatchesGetter interface {
 	BatchByNumber(ctx context.Context, number *big.Int) (*types.Batch, error)
+	BatchesByNumbers(ctx context.Context, numbers []*big.Int) ([]*types.BatchData, error)
 }

etherman/etherman.go

@@ -1264,9 +1264,10 @@ func decodeSequences(txData []byte, lastBatchNumber uint64, sequencer common.Add
 	if err != nil {
 		return nil, err
 	}
-	var sequences []polygonzkevm.PolygonRollupBaseEtrogBatchData
+
 	switch method.Name {
 	case "rollup": // TODO: put correct value
+		var sequences []polygonzkevm.PolygonRollupBaseEtrogBatchData
 		err = json.Unmarshal(bytedata, &sequences)
 		if err != nil {
 			return nil, err
@@ -1295,18 +1296,25 @@ func decodeSequences(txData []byte, lastBatchNumber uint64, sequencer common.Add
 			return nil, err
 		}
 		coinbase := (data[1]).(common.Address)
+		dataAvailabilityMessage := (data[2]).([]byte) // TODO: is this right???
 		sequencedBatches := make([]SequencedBatch, len(sequencesValidium))
-		for i, seq := range sequencesValidium {
+		var batchNums []uint64
+		var hashes []common.Hash
+		for i, _ := range sequencesValidium {
 			bn := lastBatchNumber - uint64(len(sequencesValidium)-(i+1))
-			batchL2Data, err := da.GetBatchL2Data(bn, sequencesValidium[i].TransactionsHash)
-			if err != nil {
-				return nil, err
-			}
+			batchNums = append(batchNums, bn)
+			hashes = append(hashes, sequencesValidium[i].TransactionsHash)
+		}
+		batchL2Data, err := da.GetBatchL2Data(batchNums, hashes, dataAvailabilityMessage)
+		if err != nil {
+			return nil, err
+		}
+		for i, bn := range batchNums {
 			s := polygonzkevm.PolygonRollupBaseEtrogBatchData{
-				Transactions:         batchL2Data, // TODO: get data from DA
-				ForcedGlobalExitRoot: seq.ForcedGlobalExitRoot,
-				ForcedTimestamp:      seq.ForcedTimestamp,
-				ForcedBlockHashL1:    seq.ForcedBlockHashL1,
+				Transactions:         batchL2Data[i],
+				ForcedGlobalExitRoot: sequencesValidium[i].ForcedGlobalExitRoot,
+				ForcedTimestamp:      sequencesValidium[i].ForcedTimestamp,
+				ForcedBlockHashL1:    sequencesValidium[i].ForcedBlockHashL1,
 			}
 			sequencedBatches[i] = SequencedBatch{
 				BatchNumber:                     bn,
@@ -1318,7 +1326,6 @@ func decodeSequences(txData []byte, lastBatchNumber uint64, sequencer common.Add
 				PolygonRollupBaseEtrogBatchData: &s,
 			}
 		}
-
 		return sequencedBatches, nil
 	default:
 		return nil, fmt.Errorf("unexpected method called in sequence batches transaction: %s", method.RawName)

etherman/interfaces.go

@@ -3,5 +3,5 @@ package etherman
 import "github.com/ethereum/go-ethereum/common"
 
 type dataAvailabilityProvider interface {
-	GetBatchL2Data(batchNum uint64, hash common.Hash) ([]byte, error)
+	GetBatchL2Data(batchNum []uint64, hash []common.Hash, dataAvailabilityMessage []byte) ([][]byte, error)
 }

jsonrpc/client/zkevm.go

@@ -58,6 +58,38 @@
 	return result, nil
 }
 
+func (c *Client) BatchesByNumbers(_ context.Context, numbers []*big.Int) ([]*types.BatchData, error) {
+	var list []types.BatchNumber
+	for _, n := range numbers {
+		list = append(list, types.BatchNumber(n.Int64()))
+	}
+	if len(list) == 0 {
+		list = append(list, types.LatestBatchNumber)
+	}
+
+	var batchNumbers []string
+	for _, n := range list {
+		batchNumbers = append(batchNumbers, n.StringOrHex())
+	}
+
+	response, err := JSONRPCCall(c.url, "zkevm_getBatchDataByNumbers", batchNumbers, true)
+	if err != nil {
+		return nil, err
+	}
+
+	if response.Error != nil {
+		return nil, response.Error.RPCError()
+	}
+
+	var result *types.BatchDataResult
+	err = json.Unmarshal(response.Result, &result)
+	if err != nil {
+		return nil, err
+	}
+
+	return result.Data, nil
+}
+
 // ExitRootsByGER returns the exit roots accordingly to the provided Global Exit Root
 func (c *Client) ExitRootsByGER(ctx context.Context, globalExitRoot common.Hash) (*types.ExitRoots, error) {
 	response, err := JSONRPCCall(c.url, "zkevm_getExitRootsByGER", globalExitRoot.String())