Merge branch 'main' into 1208-improve-unhealthy-status-management

.github/workflows/e2e-tests.yml

@@ -17,38 +17,12 @@ jobs:
     uses: dymensionxyz/rollapp-wasm/.github/workflows/build_image_with_dymint.yml@main
     with:
       commit_hash: ${{ github.sha }}
-
-  combine-artifacts:
-    needs:
-      - build-rollapp-evm
-      - build-rollapp-wasm
-    runs-on: ubuntu-latest
-    steps:
-      - name: Download RollApp EVM artifact
-        uses: actions/download-artifact@v3
-        with:
-          name: rollapp-evm
-          path: /tmp
-
-      - name: Download RollApp WASM artifact
-        uses: actions/download-artifact@v3
-        with:
-          name: rollapp-wasm
-          path: /tmp
-
-      - name: Combine Images into e2e.tar
-        run: |
-          tar -cf /tmp/e2e.tar rollapp-evm.tar rollapp-wasm.tar
-
-      - name: Upload artifact
-        uses: actions/upload-artifact@v3
-        with:
-          name: e2e
-          path: /tmp/e2e.tar
 
   e2e-tests:
-    needs: combine-artifacts
-    uses: dymensionxyz/e2e-tests/.github/workflows/e2e-test-workflow-call.yml@main
+    needs: 
+      - build-rollapp-evm
+      - build-rollapp-wasm
+    uses: dymensionxyz/e2e-tests/.github/workflows/e2e-test-dymint-workflow-call.yml@main
     with:
       rollapp_evm_ci: "e2e"
       rollapp_wasm_ci: "e2e"

block/executor.go

@@ -164,10 +164,8 @@ func (e *Executor) CreateBlock(
 			ProposerAddress: e.localAddress,
 		},
 		Data: types.Data{
-			Txs:                    toDymintTxs(mempoolTxs),
-			IntermediateStateRoots: types.IntermediateStateRoots{RawRootsList: nil},
-			Evidence:               types.EvidenceData{Evidence: nil},
-			ConsensusMessages:      protoutils.FromProtoMsgSliceToAnySlice(e.consensusMsgQueue.Get()...),
+			Txs:               toDymintTxs(mempoolTxs),
+			ConsensusMessages: protoutils.FromProtoMsgSliceToAnySlice(e.consensusMsgQueue.Get()...),
 		},
 		LastCommit: *lastCommit,
 	}

config/config.go

@@ -230,6 +230,8 @@ type DBConfig struct {
 	SyncWrites bool `mapstructure:"sync_writes"`
 	// InMemory sets the database to run in-memory, without touching the disk.
 	InMemory bool `mapstructure:"in_memory"`
+	// if zero, default is used
+	BadgerCompactors int `mapstructure:"badger_num_compactors"`
 }
 
 func (dbc DBConfig) Validate() error {

config/toml.go

@@ -150,6 +150,8 @@ sync_writes = {{ .DBConfig.SyncWrites }}
 # When true, the database will run in-memory only (FOR EXPERIMENTAL USE ONLY)
 in_memory = {{ .DBConfig.InMemory }}
 
+# When zero/empty, uses default
+badger_num_compactors = {{ .DBConfig.BadgerCompactors }}
 
 #######################################################
 ###       Instrumentation Configuration Options     ###

da/celestia/celestia_test.go

@@ -324,21 +324,16 @@ func getRandomBlock(height uint64, nTxs int) *types.Block {
 		},
 		Data: types.Data{
 			Txs: make(types.Txs, nTxs),
-			IntermediateStateRoots: types.IntermediateStateRoots{
-				RawRootsList: make([][]byte, nTxs),
-			},
 		},
 	}
 	copy(block.Header.AppHash[:], getRandomBytes(32))
 
 	for i := 0; i < nTxs; i++ {
 		block.Data.Txs[i] = getRandomTx()
-		block.Data.IntermediateStateRoots.RawRootsList[i] = getRandomBytes(32)
 	}
 
 	if nTxs == 0 {
 		block.Data.Txs = nil
-		block.Data.IntermediateStateRoots.RawRootsList = nil
 	}
 
 	return block

da/da_test.go

@@ -214,22 +214,17 @@ func getRandomBlock(height uint64, nTxs int) *types.Block {
 		},
 		Data: types.Data{
 			Txs: make(types.Txs, nTxs),
-			IntermediateStateRoots: types.IntermediateStateRoots{
-				RawRootsList: make([][]byte, nTxs),
-			},
 		},
 	}
 	copy(block.Header.AppHash[:], getRandomBytes(32))
 
 	for i := 0; i < nTxs; i++ {
 		block.Data.Txs[i] = getRandomTx()
-		block.Data.IntermediateStateRoots.RawRootsList[i] = getRandomBytes(32)
 	}
 
 	// TODO(tzdybal): see https://github.com/dymensionxyz/dymint/issues/143
 	if nTxs == 0 {
 		block.Data.Txs = nil
-		block.Data.IntermediateStateRoots.RawRootsList = nil
 	}
 
 	return block

node/node.go

@@ -108,7 +108,8 @@ func NewNode(
 		dstore = datastore.NewMapDatastore()
 	} else {
 		// TODO(omritoptx): Move dymint to const
-		baseKV = store.NewKVStore(conf.RootDir, conf.DBPath, "dymint", conf.DBConfig.SyncWrites, logger)
+		baseKV = store.NewKVStore(conf.RootDir, conf.DBPath, "dymint",
+			store.BadgerOpts{SyncWrites: conf.DBConfig.SyncWrites, NumCompactors: conf.DBConfig.BadgerCompactors}, logger)
 		path := filepath.Join(store.Rootify(conf.RootDir, conf.DBPath), "blocksync")
 		var err error
 		dstore, err = leveldb.NewDatastore(path, &leveldb.Options{})

proto/types/dymint/dymint.proto

@@ -79,8 +79,7 @@ message Commit {
 
 message Data {
 	repeated bytes txs = 1;
-	repeated bytes intermediate_state_roots = 2;
-	repeated tendermint.abci.Evidence evidence = 3;
+	reserved 2,3;
 	repeated google.protobuf.Any consensus_messages = 4;
 }
 

rpc/client/client_test.go

@@ -934,22 +934,17 @@ func getRandomBlock(height uint64, nTxs int) *types.Block {
 		},
 		Data: types.Data{
 			Txs: make(types.Txs, nTxs),
-			IntermediateStateRoots: types.IntermediateStateRoots{
-				RawRootsList: make([][]byte, nTxs),
-			},
 		},
 	}
 	copy(block.Header.AppHash[:], getRandomBytes(32))
 
 	for i := 0; i < nTxs; i++ {
 		block.Data.Txs[i] = getRandomTx()
-		block.Data.IntermediateStateRoots.RawRootsList[i] = getRandomBytes(32)
 	}
 
 	// TODO(tzdybal): see https://github.com/dymensionxyz/dymint/issues/143
 	if nTxs == 0 {
 		block.Data.Txs = nil
-		block.Data.IntermediateStateRoots.RawRootsList = nil
 	}
 
 	tmprotoLC, err := tmtypes.CommitFromProto(&tmproto.Commit{})

store/badger.go

@@ -43,9 +43,13 @@ func NewDefaultInMemoryKVStore() KV {
 	}
 }
 
-func NewKVStore(rootDir, dbPath, dbName string, syncWrites bool, logger types.Logger) KV {
+func NewKVStore(rootDir, dbPath, dbName string,
+	badgerOpts BadgerOpts,
+
+	logger types.Logger,
+) KV {
 	path := filepath.Join(Rootify(rootDir, dbPath), dbName)
-	opts := memoryEfficientBadgerConfig(path, syncWrites)
+	opts := memoryEfficientBadgerConfig(path, badgerOpts)
 	db, err := badger.Open(*opts)
 	if err != nil {
 		panic(err)
@@ -60,7 +64,7 @@ func NewKVStore(rootDir, dbPath, dbName string, syncWrites bool, logger types.Lo
 
 // NewDefaultKVStore creates instance of default key-value store.
 func NewDefaultKVStore(rootDir, dbPath, dbName string) KV {
-	return NewKVStore(rootDir, dbPath, dbName, true, log.NewNopLogger())
+	return NewKVStore(rootDir, dbPath, dbName, BadgerOpts{SyncWrites: true}, log.NewNopLogger())
 }
 
 // Rootify is helper function to make config creation independent of root dir
@@ -228,13 +232,18 @@ func (i *BadgerIterator) Discard() {
 	i.txn.Discard()
 }
 
+type BadgerOpts struct {
+	SyncWrites    bool
+	NumCompactors int
+}
+
 // memoryEfficientBadgerConfig sets badger configuration parameters to reduce memory usage, specially during compactions to avoid memory spikes that causes OOM.
 // based on https://github.com/celestiaorg/celestia-node/issues/2905
-func memoryEfficientBadgerConfig(path string, syncWrites bool) *badger.Options {
+func memoryEfficientBadgerConfig(path string, o BadgerOpts) *badger.Options {
 	opts := badger.DefaultOptions(path) // this must be copied
 	// SyncWrites is a configuration option in Badger that determines whether writes are immediately synced to disk or no.
 	// If set to true it writes to the write-ahead log (value log) are synced to disk before being applied to the LSM tree.
-	opts.SyncWrites = syncWrites
+	opts.SyncWrites = o.SyncWrites
 	// default 64mib => 0 - disable block cache
 	// BlockCacheSize specifies how much data cache should hold in memory.
 	// It improves lookup performance but increases memory consumption.
@@ -254,7 +263,10 @@ func memoryEfficientBadgerConfig(path string, syncWrites bool) *badger.Options {
 	// default 15 => 5 - this prevents memory growth on CPU constraint systems by blocking all writers
 	opts.NumLevelZeroTablesStall = 5
 	// reducing number compactors, makes it slower but reduces memory usage during compaction
-	opts.NumCompactors = 2
+	opts.NumCompactors = o.NumCompactors
+	if opts.NumCompactors == 0 {
+		opts.NumCompactors = 2 // default
+	}
 	// makes sure badger is always compacted on shutdown
 	opts.CompactL0OnClose = true
 

testutil/types.go

@@ -88,9 +88,7 @@ func generateBlock(height uint64, proposerHash []byte, lastHeaderHash [32]byte)
 			ConsensusMessagesHash: types.ConsMessagesHash(nil),
 		},
 		Data: types.Data{
-			Txs:                    nil,
-			IntermediateStateRoots: types.IntermediateStateRoots{RawRootsList: [][]byte{{0x1}}},
-			Evidence:               types.EvidenceData{Evidence: nil},
+			Txs: nil,
 		},
 		LastCommit: types.Commit{
 			Height:     8,
@@ -115,14 +113,10 @@ func GenerateBlocksWithTxs(startHeight uint64, num uint64, proposerKey crypto.Pr
 
 		block.Data = types.Data{
 			Txs: make(types.Txs, nTxs),
-			IntermediateStateRoots: types.IntermediateStateRoots{
-				RawRootsList: make([][]byte, nTxs),
-			},
 		}
 
 		for i := 0; i < nTxs; i++ {
 			block.Data.Txs[i] = GetRandomTx()
-			block.Data.IntermediateStateRoots.RawRootsList[i] = GetRandomBytes(32)
 		}
 
 		signature, err := generateSignature(proposerKey, &block.Header)
@@ -394,15 +388,11 @@ func GetRandomBlock(height uint64, nTxs int) *types.Block {
 		},
 		Data: types.Data{
 			Txs: make(types.Txs, nTxs),
-			IntermediateStateRoots: types.IntermediateStateRoots{
-				RawRootsList: make([][]byte, nTxs),
-			},
 		},
 	}
 
 	for i := 0; i < nTxs; i++ {
 		block.Data.Txs[i] = GetRandomTx()
-		block.Data.IntermediateStateRoots.RawRootsList[i] = GetRandomBytes(32)
 	}
 
 	return block

types/block.go

@@ -89,10 +89,8 @@ var (
 
 // Data defines Dymint block data.
 type Data struct {
-	Txs                    Txs
-	IntermediateStateRoots IntermediateStateRoots
-	Evidence               EvidenceData
-	ConsensusMessages      []*proto.Any
+	Txs               Txs
+	ConsensusMessages []*proto.Any
 }
 
 // EvidenceData defines how evidence is stored in block.