Database Split (0xPolygonHermez#1155)

alagbe003 · Sep 15, 2022 · cfd4c11 · cfd4c11
1 parent 75a5cb6
commit cfd4c11
Show file tree

Hide file tree

Showing 31 changed files with 704 additions and 357 deletions.
diff --git a/Makefile b/Makefile
@@ -4,20 +4,24 @@ DOCKERCOMPOSEAPPAGG := zkevm-aggregator
 DOCKERCOMPOSEAPPRPC := zkevm-json-rpc
 DOCKERCOMPOSEAPPSYNC := zkevm-sync
 DOCKERCOMPOSEAPPBROADCAST := zkevm-broadcast
-DOCKERCOMPOSEDB := zkevm-db
+DOCKERCOMPOSESTATEDB := zkevm-state-db
+DOCKERCOMPOSEPOOLDB := zkevm-pool-db
+DOCKERCOMPOSERPCDB := zkevm-rpc-db
 DOCKERCOMPOSENETWORK := zkevm-mock-l1-network
 DOCKERCOMPOSEEXPLORERL1 := zkevm-explorer-l1
 DOCKERCOMPOSEEXPLORERL1DB := zkevm-explorer-l1-db
-DOCKERCOMPOSEEXPLORER := zkevm-explorer
-DOCKERCOMPOSEEXPLORERDB := zkevm-explorer-db
+DOCKERCOMPOSEEXPLORERL2 := zkevm-explorer-l2
+DOCKERCOMPOSEEXPLORERL2DB := zkevm-explorer-l2-db
 DOCKERCOMPOSEEXPLORERRPC := zkevm-explorer-json-rpc
 DOCKERCOMPOSEZKPROVER := zkevm-prover
 DOCKERCOMPOSEZKPROVERMOCK := zkprover-mock
 DOCKERCOMPOSEPERMISSIONLESSDB := zkevm-permissionless-db
 DOCKERCOMPOSEPERMISSIONLESSNODE := zkevm-permissionless-node
 DOCKERCOMPOSENODEAPPROVE := zkevm-approve
 
-RUNDB := $(DOCKERCOMPOSE) up -d $(DOCKERCOMPOSEDB)
+RUNSTATEDB := $(DOCKERCOMPOSE) up -d $(DOCKERCOMPOSESTATEDB)
+RUNPOOLDB := $(DOCKERCOMPOSE) up -d $(DOCKERCOMPOSEPOOLDB)
+RUNRPCDB := $(DOCKERCOMPOSE) up -d $(DOCKERCOMPOSERPCDB)
 RUNSEQUENCER := $(DOCKERCOMPOSE) up -d $(DOCKERCOMPOSEAPPSEQ)
 RUNAGGREGATOR := $(DOCKERCOMPOSE) up -d $(DOCKERCOMPOSEAPPAGG)
 RUNJSONRPC := $(DOCKERCOMPOSE) up -d $(DOCKERCOMPOSEAPPRPC)
@@ -27,8 +31,8 @@ RUNBROADCAST := $(DOCKERCOMPOSE) up -d $(DOCKERCOMPOSEAPPBROADCAST)
 RUNL1NETWORK := $(DOCKERCOMPOSE) up -d $(DOCKERCOMPOSENETWORK)
 RUNEXPLORERL1 := $(DOCKERCOMPOSE) up -d $(DOCKERCOMPOSEEXPLORERL1)
 RUNEXPLORERL1DB := $(DOCKERCOMPOSE) up -d $(DOCKERCOMPOSEEXPLORERL1DB)
-RUNEXPLORER := $(DOCKERCOMPOSE) up -d $(DOCKERCOMPOSEEXPLORER)
-RUNEXPLORERDB := $(DOCKERCOMPOSE) up -d $(DOCKERCOMPOSEEXPLORERDB)
+RUNEXPLORERL2 := $(DOCKERCOMPOSE) up -d $(DOCKERCOMPOSEEXPLORERL2)
+RUNEXPLORERL2DB := $(DOCKERCOMPOSE) up -d $(DOCKERCOMPOSEEXPLORERL2DB)
 RUNEXPLORERJSONRPC := $(DOCKERCOMPOSE) up -d $(DOCKERCOMPOSEEXPLORERRPC)
 RUNZKPROVER := $(DOCKERCOMPOSE) up -d $(DOCKERCOMPOSEZKPROVER)
 RUNZKPROVERMOCK := $(DOCKERCOMPOSE) up -d $(DOCKERCOMPOSEZKPROVERMOCK)
@@ -40,7 +44,9 @@ RUNAPPROVE := $(DOCKERCOMPOSE) up -d $(DOCKERCOMPOSENODEAPPROVE)
 
 RUN := $(DOCKERCOMPOSE) up -d
 
-STOPDB := $(DOCKERCOMPOSE) stop $(DOCKERCOMPOSEDB) && $(DOCKERCOMPOSE) rm -f $(DOCKERCOMPOSEDB)
+STOPSTATEDB := $(DOCKERCOMPOSE) stop $(DOCKERCOMPOSESTATEDB) && $(DOCKERCOMPOSE) rm -f $(DOCKERCOMPOSESTATEDB)
+STOPPOOLDB := $(DOCKERCOMPOSE) stop $(DOCKERCOMPOSEPOOLDB) && $(DOCKERCOMPOSE) rm -f $(DOCKERCOMPOSEPOOLDB)
+STOPRPCDB := $(DOCKERCOMPOSE) stop $(DOCKERCOMPOSERPCDB) && $(DOCKERCOMPOSE) rm -f $(DOCKERCOMPOSERPCDB)
 STOPSEQUENCER := $(DOCKERCOMPOSE) stop $(DOCKERCOMPOSEAPPSEQ) && $(DOCKERCOMPOSE) rm -f $(DOCKERCOMPOSEAPPSEQ)
 STOPAGGREGATOR := $(DOCKERCOMPOSE) stop $(DOCKERCOMPOSEAPPAGG) && $(DOCKERCOMPOSE) rm -f $(DOCKERCOMPOSEAPPAGG)
 STOPJSONRPC := $(DOCKERCOMPOSE) stop $(DOCKERCOMPOSEAPPRPC) && $(DOCKERCOMPOSE) rm -f $(DOCKERCOMPOSEAPPRPC)
@@ -50,8 +56,8 @@ STOPBROADCAST := $(DOCKERCOMPOSE) stop $(DOCKERCOMPOSEAPPBROADCAST) && $(DOCKERC
 STOPNETWORK := $(DOCKERCOMPOSE) stop $(DOCKERCOMPOSENETWORK) && $(DOCKERCOMPOSE) rm -f $(DOCKERCOMPOSENETWORK)
 STOPEXPLORERL1 := $(DOCKERCOMPOSE) stop $(DOCKERCOMPOSEEXPLORERL1) && $(DOCKERCOMPOSE) rm -f $(DOCKERCOMPOSEEXPLORERL1)
 STOPEXPLORERL1DB := $(DOCKERCOMPOSE) stop $(DOCKERCOMPOSEEXPLORERL1DB) && $(DOCKERCOMPOSE) rm -f $(DOCKERCOMPOSEEXPLORERL1DB)
-STOPEXPLORER := $(DOCKERCOMPOSE) stop $(DOCKERCOMPOSEEXPLORER) && $(DOCKERCOMPOSE) rm -f $(DOCKERCOMPOSEEXPLORER)
-STOPEXPLORERDB := $(DOCKERCOMPOSE) stop $(DOCKERCOMPOSEEXPLORERDB) && $(DOCKERCOMPOSE) rm -f $(DOCKERCOMPOSEEXPLORERDB)
+STOPEXPLORERL2 := $(DOCKERCOMPOSE) stop $(DOCKERCOMPOSEEXPLORERL2) && $(DOCKERCOMPOSE) rm -f $(DOCKERCOMPOSEEXPLORERL2)
+STOPEXPLORERL2DB := $(DOCKERCOMPOSE) stop $(DOCKERCOMPOSEEXPLORERL2DB) && $(DOCKERCOMPOSE) rm -f $(DOCKERCOMPOSEEXPLORERL2DB)
 STOPEXPLORERRPC := $(DOCKERCOMPOSE) stop $(DOCKERCOMPOSEEXPLORERRPC) && $(DOCKERCOMPOSE) rm -f $(DOCKERCOMPOSEEXPLORERRPC)
 STOPZKPROVER := $(DOCKERCOMPOSE) stop $(DOCKERCOMPOSEZKPROVER) && $(DOCKERCOMPOSE) rm -f $(DOCKERCOMPOSEZKPROVER)
 STOPZKPROVERMOCK := $(DOCKERCOMPOSE) stop $(DOCKERCOMPOSEZKPROVERMOCK) && $(DOCKERCOMPOSE) rm -f $(DOCKERCOMPOSEZKPROVERMOCK)
@@ -93,58 +99,82 @@ build-docker-nc: ## Builds a docker image with the node binary - but without bui
 .PHONY: test
 test: compile-scs ## Runs only short tests without checking race conditions
 	export CONFIG_MODE="test"	
-	$(STOPDB)
+	$(STOPSTATEDB)
+	$(STOPPOOLDB)
+	$(STOPRPCDB)
 	$(STOPZKPROVER)
-	$(RUNDB); sleep 5
+	$(RUNSTATEDB)
+	$(RUNPOOLDB)
+	$(RUNRPCDB)
 	$(RUNZKPROVER); sleep 5
-	trap '$(STOPDB) && $(STOPZKPROVER)' EXIT; go test -short -race -p 1 ./...
+	trap '$(STOPSTATEDB) && $(STOPPOOLDB) && $(STOPRPCDB) && $(STOPZKPROVER)' EXIT; go test -short -race -p 1 ./...
 
 .PHONY: test-full
 test-full: build-docker compile-scs ## Runs all tests checking race conditions
 	export CONFIG_MODE="test"
-	$(STOPDB)
+	$(STOPSTATEDB)
+	$(STOPPOOLDB)
+	$(STOPRPCDB)
 	$(STOPZKPROVER)
-	$(RUNDB); sleep 7
+	$(RUNSTATEDB)
+	$(RUNPOOLDB)
+	$(RUNRPCDB)
 	$(RUNZKPROVER); sleep 5
 	$(RUNZKPROVERMOCK)
-	trap '$(STOPDB) && $(STOPZKPROVER) && $(STOPZKPROVERMOCK)' EXIT; MallocNanoZone=0 go test -race -v -p 1 -timeout 1200s `go list ./... | grep -v \/ci\/e2e-group`
+	trap '$(STOPSTATEDB) && $(STOPPOOLDB) && $(STOPRPCDB) && $(STOPZKPROVER) && $(STOPZKPROVERMOCK)' EXIT; MallocNanoZone=0 go test -race -v -p 1 -timeout 1200s `go list ./... | grep -v \/ci\/e2e-group`
 
 .PHONY: test-full-non-e2e
 test-full-non-e2e: build-docker compile-scs ## Runs non-e2e tests checking race conditions
 	export CONFIG_MODE="test"	
-	$(STOPDB)
+	$(STOPSTATEDB)
+	$(STOPPOOLDB)
+	$(STOPRPCDB)
 	$(STOPZKPROVER)
-	$(RUNDB); sleep 7
-	$(RUNZKPROVER)
+	$(RUNSTATEDB)
+	$(RUNPOOLDB)
+	$(RUNRPCDB)
+	$(RUNZKPROVER); sleep 5
 	$(RUNZKPROVERMOCK)
 	sleep 15
 	docker logs $(DOCKERCOMPOSEZKPROVER)
-	trap '$(STOPDB) && $(STOPZKPROVER) && $(STOPZKPROVERMOCK)' EXIT; MallocNanoZone=0 go test -short -race -p 1 -timeout 60s ./...
+	trap '$(STOPSTATEDB) && $(STOPPOOLDB) && $(STOPRPCDB) && $(STOPZKPROVER) && $(STOPZKPROVERMOCK)' EXIT; MallocNanoZone=0 go test -short -race -p 1 -timeout 60s ./...
 
 .PHONY: test-e2e-group-1
 test-e2e-group-1: build-docker compile-scs ## Runs group 1 e2e tests checking race conditions
 	export CONFIG_MODE="test"	
-	$(STOPDB)
-	$(RUNDB); sleep 7
-	trap '$(STOPDB)' EXIT; MallocNanoZone=0 go test -race -v -p 1 -timeout 600s ./ci/e2e-group1/...
+	$(STOPSTATEDB)
+	$(STOPPOOLDB)
+	$(STOPRPCDB)
+	$(RUNSTATEDB)
+	$(RUNPOOLDB)
+	$(RUNRPCDB); sleep 5
+	trap '$(STOPSTATEDB) && $(STOPPOOLDB) && $(STOPRPCDB)' EXIT; MallocNanoZone=0 go test -race -v -p 1 -timeout 600s ./ci/e2e-group1/...
 
 .PHONY: test-e2e-group-2
 test-e2e-group-2: build-docker compile-scs ## Runs group 2 e2e tests checking race conditions
 	export CONFIG_MODE="test"	
-	$(STOPDB)
+	$(STOPSTATEDB)
+	$(STOPPOOLDB)
+	$(STOPRPCDB)
 	$(STOPZKPROVER)
-	$(RUNDB); sleep 7
+	$(RUNSTATEDB)
+	$(RUNPOOLDB)
+	$(RUNRPCDB); sleep 5
 	CONFIG_MODE="test" $(RUNZKPROVER)
 	docker ps -a
 	docker logs $(DOCKERCOMPOSEZKPROVER)
-	trap '$(STOPDB) && $(STOPZKPROVER)' EXIT; MallocNanoZone=0 go test -race -v -p 1 -timeout 600s ./ci/e2e-group2/...
+	trap '$(STOPSTATEDB) && $(STOPPOOLDB) && $(STOPRPCDB) && $(STOPZKPROVER)' EXIT; MallocNanoZone=0 go test -race -v -p 1 -timeout 600s ./ci/e2e-group2/...
 
 .PHONY: test-e2e-group-3
 test-e2e-group-3: build-docker compile-scs ## Runs group 3 e2e tests checking race conditions
 	export CONFIG_MODE="test"	
-	$(STOPDB)
-	$(RUNDB); sleep 7
-	trap '$(STOPDB)' EXIT; MallocNanoZone=0 go test -race -v -p 1 -timeout 600s ./ci/e2e-group3/...
+	$(STOPSTATEDB)
+	$(STOPPOOLDB)
+	$(STOPRPCDB)
+	$(RUNSTATEDB)
+	$(RUNPOOLDB)
+	$(RUNRPCDB); sleep 5
+	trap '$(STOPSTATEDB) && $(STOPPOOLDB) && $(STOPRPCDB)' EXIT; MallocNanoZone=0 go test -race -v -p 1 -timeout 600s ./ci/e2e-group3/...
 
 .PHONY: install-linter
 install-linter: ## Installs the linter
@@ -162,11 +192,15 @@ validate: lint build test-full ## lint, build, unit and e2e tests
 
 .PHONY: run-db
 run-db: ## Runs the node database
-	$(RUNDB)
+	$(RUNSTATEDB)
+	$(RUNPOOLDB)
+	$(RUNRPCDB)
 
 .PHONY: stop-db
 stop-db: ## Stops the node database
-	$(STOPDB)
+	$(STOPRPCDB)
+	$(STOPPOOLDB)
+	$(STOPSTATEDB)
 
 .PHONY: run-node
 run-node: ## Runs the node
@@ -209,30 +243,34 @@ stop-zkprover-mock: ## Stops zkprover-mock
 .PHONY: run-explorer
 run-explorer: ## Runs the explorer
 	$(RUNEXPLORERL1DB)
-	$(RUNEXPLORERDB)
+	$(RUNEXPLORERL2DB)
 	$(RUNEXPLORERJSONRPC)
 	$(RUNEXPLORERL1)
-	$(RUNEXPLORER)
+	$(RUNEXPLORERL2)
 
 .PHONY: stop-explorer
 stop-explorer: ## Stops the explorer
-	$(STOPEXPLORER)
+	$(STOPEXPLORERL2)
 	$(STOPEXPLORERL1)
 	$(STOPEXPLORERRPC)
-	$(STOPEXPLORERDB)
+	$(STOPEXPLORERL2DB)
 	$(STOPEXPLORERL1DB)
 
 .PHONY: run-explorer-db
 run-explorer-db: ## Runs the explorer database
-	$(RUNEXPLORERDB)
+	$(RUNEXPLORERL1DB)
+	$(RUNEXPLORERL2DB)
 
 .PHONY: stop-explorer-db
 stop-explorer-db: ## Stops the explorer database
-	$(STOPEXPLORERDB)
+	$(STOPEXPLORERL2DB)
+	$(STOPEXPLORERL1DB)
 
 .PHONY: run
 run: ## Runs all the services
-	$(RUNDB)
+	$(RUNSTATEDB)
+	$(RUNPOOLDB)
+	$(RUNRPCDB)
 	$(RUNL1NETWORK)
 	sleep 2
 	$(RUNZKPROVER)

diff --git a/cmd/approve.go b/cmd/approve.go
@@ -37,7 +37,9 @@ func approveTokens(ctx *cli.Context) error {
 
 	setupLog(c.Log)
 
-	runMigrations(c.Database)
+	runStateMigrations(c.StateDB)
+	runPoolMigrations(c.PoolDB)
+	runRPCMigrations(c.RPC.DB)
 
 	// Check if it is already registered
 	etherman, err := newEtherman(*c)

diff --git a/cmd/dumpstate.go b/cmd/dumpstate.go
@@ -111,11 +111,11 @@ func dumpState(ctx *cli.Context) error {
 	}
 
 	// Connect to SQL
-	sqlDB, err := db.NewSQLDB(c.Database)
+	stateSqlDB, err := db.NewSQLDB(c.StateDB)
 	if err != nil {
 		return err
 	}
-	stateDB := state.NewPostgresStorage(sqlDB)
+	stateDB := state.NewPostgresStorage(stateSqlDB)
 
 	dump := dumpedState{
 		Description: description,

diff --git a/cmd/run.go b/cmd/run.go
@@ -43,20 +43,15 @@ func start(cliCtx *cli.Context) error {
 		return err
 	}
 	setupLog(c.Log)
-	runMigrations(c.Database)
-
-	sqlDB, err := db.NewSQLDB(c.Database)
+	runStateMigrations(c.StateDB)
+	stateSqlDB, err := db.NewSQLDB(c.StateDB)
 	if err != nil {
 		log.Fatal(err)
 	}
+
 	ctx := context.Background()
 
-	st := newState(ctx, c, sqlDB)
-
-	poolDb, err := pgpoolstorage.NewPostgresPoolStorage(c.Database)
-	if err != nil {
-		log.Fatal(err)
-	}
+	st := newState(ctx, c, stateSqlDB)
 
 	var (
 		grpcClientConns []*grpc.ClientConn
@@ -74,8 +69,6 @@ func start(cliCtx *cli.Context) error {
 		}
 	}
 
-	npool := pool.NewPool(poolDb, st, c.NetworkConfig.L2GlobalExitRootManagerAddr)
-	gpe := createGasPriceEstimator(c.GasPriceEstimator, st, npool)
 	ethTxManager := ethtxmanager.New(c.EthTxManager, etherman)
 	proverClient, proverConn := newProverClient(c.Prover)
 	for _, item := range cliCtx.StringSlice(config.FlagComponents) {
@@ -85,15 +78,19 @@ func start(cliCtx *cli.Context) error {
 			go runAggregator(ctx, c.Aggregator, etherman, ethTxManager, proverClient, st)
 		case SEQUENCER:
 			log.Info("Running sequencer")
-			seq := createSequencer(*c, npool, st, etherman, ethTxManager)
+			poolInstance := createPool(c.PoolDB, c.NetworkConfig, st)
+			seq := createSequencer(*c, poolInstance, st, etherman, ethTxManager)
 			go seq.Start(ctx)
 		case RPC:
 			log.Info("Running JSON-RPC server")
+			runRPCMigrations(c.RPC.DB)
+			poolInstance := createPool(c.PoolDB, c.NetworkConfig, st)
+			gpe := createGasPriceEstimator(c.GasPriceEstimator, st, poolInstance)
 			apis := map[string]bool{}
 			for _, a := range cliCtx.StringSlice(config.FlagHTTPAPI) {
 				apis[a] = true
 			}
-			go runJSONRPCServer(*c, npool, st, gpe, apis)
+			go runJSONRPCServer(*c, poolInstance, st, gpe, apis)
 		case SYNCHRONIZER:
 			log.Info("Running synchronizer")
 			go runSynchronizer(*c, etherman, st)
@@ -114,8 +111,20 @@ func setupLog(c log.Config) {
 	log.Init(c)
 }
 
-func runMigrations(c db.Config) {
-	err := db.RunMigrationsUp(c)
+func runStateMigrations(c db.Config) {
+	runMigrations(c, db.StateMigrationName)
+}
+
+func runPoolMigrations(c db.Config) {
+	runMigrations(c, db.PoolMigrationName)
+}
+
+func runRPCMigrations(c db.Config) {
+	runMigrations(c, db.RPCMigrationName)
+}
+
+func runMigrations(c db.Config, name string) {
+	err := db.RunMigrationsUp(c, name)
 	if err != nil {
 		log.Fatal(err)
 	}
@@ -144,7 +153,7 @@ func runSynchronizer(cfg config.Config, etherman *etherman.Client, st *state.Sta
 }
 
 func runJSONRPCServer(c config.Config, pool *pool.Pool, st *state.State, gpe gasPriceEstimator, apis map[string]bool) {
-	storage, err := jsonrpc.NewPostgresStorage(c.Database)
+	storage, err := jsonrpc.NewPostgresStorage(c.RPC.DB)
 	if err != nil {
 		log.Fatal(err)
 	}
@@ -288,3 +297,13 @@ func newState(ctx context.Context, c *config.Config, sqlDB *pgxpool.Pool) *state
 	st := state.NewState(stateCfg, stateDb, executorClient, stateTree)
 	return st
 }
+
+func createPool(poolDBConfig db.Config, networkConfig config.NetworkConfig, st *state.State) *pool.Pool {
+	runPoolMigrations(poolDBConfig)
+	poolStorage, err := pgpoolstorage.NewPostgresPoolStorage(poolDBConfig)
+	if err != nil {
+		log.Fatal(err)
+	}
+	poolInstance := pool.NewPool(poolStorage, st, networkConfig.L2GlobalExitRootManagerAddr)
+	return poolInstance
+}
diff --git a/config/config.debug.toml b/config/config.debug.toml
@@ -4,15 +4,24 @@ IsTrustedSequencer = true
 Level = "debug"
 Outputs = ["stdout"]
 
-[Database]
-User = "test_user"
-Password = "test_password"
-Name = "test_db"
+[StateDB]
+User = "state_user"
+Password = "state_password"
+Name = "state_db"
 Host = "localhost"
 Port = "5432"
 EnableLog = false
 MaxConns = 10
 
+[PoolDB]
+User = "pool_user"
+Password = "pool_password"
+Name = "pool_db"
+Host = "localhost"
+Port = "5433"
+EnableLog = false
+MaxConns = 10
+
 [Etherman]
 URL = "http://localhost:8545"
 PrivateKeyPath = "../test/test.keystore"
@@ -31,6 +40,14 @@ MaxRequestsPerIPAndSecond = 10000
 SequencerNodeURI = ""
 BroadcastURI = "127.0.0.1:61090"
 DefaultSenderAddress = "0x1111111111111111111111111111111111111111"
+	[RPC.DB]
+		User = "rpc_user"
+		Password = "rpc_password"
+		Name = "rpc_db"
+		Host = "localhost"
+		Port = "5434"
+		EnableLog = false
+		MaxConns = 10
 
 [Synchronizer]
 SyncInterval = "5s"