Merge pull request #2231 from ledgerwatch/stable-2020-06-05

2020.06.05

.dockerignore

@@ -6,6 +6,6 @@
 
 build
 tests/testdata
+tests/.git
 cmd/prometheus
 
-.git

.github/workflows/ci.yml

@@ -15,8 +15,7 @@ jobs:
 
     steps:
       - uses: actions/checkout@v2
-        with:
-          submodules: recursive
+      - run: git submodule update --init --recursive --force
       - uses: actions/setup-go@v2
         with:
           go-version: 1.16.x
@@ -57,25 +56,20 @@ jobs:
 
     steps:
       - uses: actions/checkout@v2
-        with:
-          submodules: recursive
+      - run: git submodule update --init --recursive --force
       - uses: actions/setup-go@v2
         with:
           go-version: 1.16.x
       - run: choco upgrade mingw cmake -y --no-progress
       - name: Build
         run: |
-          cd ethdb/mdbx/dist
-          cmake -G "MinGW Makefiles" . -D MDBX_BUILD_SHARED_LIBRARY:BOOL=ON -D MDBX_WITHOUT_MSVC_CRT:BOOOL=OFF -D CMAKE_C_FLAGS:STRING="-Wno-unused-variable -Wno-unused-parameter -Wno-unused-function" -D CMAKE_CXX_FLAGS:STRING="-Wno-unused-variable -Wno-unused-parameter -Wno-unused-function"
-          cmake --build .
-          cp libmdbx.dll (Join-Path $env:SystemRoot system32)
-          go mod download
-          go mod vendor
+          .\wmake.ps1
+          make test
+        shell: powershell
 
   docker:
     runs-on: ubuntu-18.04
     steps:
       - uses: actions/checkout@v2
-        with:
-          submodules: recursive
+      - run: git submodule update --init --recursive --force
       - run: docker build .

.gitignore

@@ -67,3 +67,4 @@ docker-compose.dev.yml
 *.tmp
 
 /ethdb/*.fail
+

.gitmodules

@@ -1,3 +1,6 @@
 [submodule "tests"]
 	path = tests/testdata
 	url = https://github.com/ethereum/tests
+[submodule "libmdbx"]
+	path = libmdbx
+	url = https://github.com/erthink/libmdbx

Makefile

@@ -5,7 +5,7 @@ GIT_COMMIT ?= $(shell git rev-list -1 HEAD)
 GIT_BRANCH ?= $(shell git rev-parse --abbrev-ref HEAD)
 GIT_TAG    ?= $(shell git describe --tags)
 GOBUILD = env GO111MODULE=on go build -trimpath -ldflags "-X github.com/ledgerwatch/erigon/params.GitCommit=${GIT_COMMIT} -X github.com/ledgerwatch/erigon/params.GitBranch=${GIT_BRANCH} -X github.com/ledgerwatch/params.GitTag=${GIT_TAG}"
-GO_DBG_BUILD = env CGO_CFLAGS='-O0 -g -DMDBX_BUILD_FLAGS_CONFIG="config.h"' GODEBUG=cgocheck=2 go build -trimpath -tags=debug -ldflags "-X github.com/ledgerwatch/erigon/params.GitCommit=${GIT_COMMIT} -X github.com/ledgerwatch/erigon/params.GitBranch=${GIT_BRANCH} -X github.com/ledgerwatch/erigon/params.GitTag=${GIT_TAG}" -gcflags=all="-N -l"  # see delve docs
+GO_DBG_BUILD = go build -trimpath -tags=debug -ldflags "-X github.com/ledgerwatch/erigon/params.GitCommit=${GIT_COMMIT} -X github.com/ledgerwatch/erigon/params.GitBranch=${GIT_BRANCH} -X github.com/ledgerwatch/erigon/params.GitTag=${GIT_TAG}" -gcflags=all="-N -l"  # see delve docs
 
 GO_MAJOR_VERSION = $(shell go version | cut -c 14- | cut -d' ' -f1 | cut -d'.' -f1)
 GO_MINOR_VERSION = $(shell go version | cut -c 14- | cut -d' ' -f1 | cut -d'.' -f2)
@@ -34,12 +34,12 @@ docker-compose:
 	docker-compose up
 
 # debug build allows see C stack traces, run it with GOTRACEBACK=crash. You don't need debug build for C pit for profiling. To profile C code use SETCGOTRCKEBACK=1
-dbg: mdbx-dbg
+dbg:
 	$(GO_DBG_BUILD) -o $(GOBIN)/ ./cmd/...
 
 geth: erigon
 
-erigon: go-version mdbx
+erigon: go-version
 	@echo "Building Erigon"
 	rm -f $(GOBIN)/tg # Remove old binary to prevent confusion where users still use it because of the scripts
 	$(GOBUILD) -o $(GOBIN)/erigon ./cmd/erigon
@@ -108,41 +108,25 @@ tracker:
 	@echo "Done building."
 	@echo "Run \"$(GOBIN)/tracker\" to run snapshots tracker."
 
-db-tools: mdbx
-	@echo "Building bb-tools"
-
-	cd ethdb/mdbx/dist/ && MDBX_BUILD_TIMESTAMP=unknown make tools
-	cp ethdb/mdbx/dist/mdbx_chk $(GOBIN)
-	cp ethdb/mdbx/dist/mdbx_copy $(GOBIN)
-	cp ethdb/mdbx/dist/mdbx_dump $(GOBIN)
-	cp ethdb/mdbx/dist/mdbx_drop $(GOBIN)
-	cp ethdb/mdbx/dist/mdbx_load $(GOBIN)
-	cp ethdb/mdbx/dist/mdbx_stat $(GOBIN)
+db-tools:
+	@echo "Building db-tools"
+	cc --version
+	cd libmdbx && ls -la && MDBX_BUILD_TIMESTAMP=unknown make tools
+	cp libmdbx/mdbx_chk $(GOBIN)
+	cp libmdbx/mdbx_copy $(GOBIN)
+	cp libmdbx/mdbx_dump $(GOBIN)
+	cp libmdbx/mdbx_drop $(GOBIN)
+	cp libmdbx/mdbx_load $(GOBIN)
+	cp libmdbx/mdbx_stat $(GOBIN)
 	@echo "Run \"$(GOBIN)/mdbx_stat -h\" to get info about mdbx db file."
 
-mdbx:
-	@echo "Building mdbx"
-	@cd ethdb/mdbx/dist/ \
-		&& make clean && MDBX_BUILD_TIMESTAMP=unknown make config.h \
-		&& echo '#define MDBX_DEBUG 0' >> config.h \
-		&& echo '#define MDBX_FORCE_ASSERTIONS 0' >> config.h \
-        && CFLAGS_EXTRA="-Wno-deprecated-declarations" make mdbx-static.o
-
-mdbx-dbg:
-	@echo "Building mdbx"
-	@cd ethdb/mdbx/dist/ \
-		&& make clean && make config.h \
-		&& echo '#define MDBX_DEBUG 1' >> config.h \
-		&& echo '#define MDBX_FORCE_ASSERTIONS 1' >> config.h \
-        && CFLAGS_EXTRA="-Wno-deprecated-declarations" CFLAGS='-O0 -g -Wall -Werror -Wextra -Wpedantic -ffunction-sections -fPIC -fvisibility=hidden -std=gnu11 -pthread -Wno-error=attributes' make mdbx-static.o
-
-test: mdbx
+test:
 	$(GOTEST) --timeout 30m
 
 lint:
 	@./build/bin/golangci-lint run --config ./.golangci.yml
 
-lintci: mdbx
+lintci:
 	@echo "--> Running linter for code"
 	@./build/bin/golangci-lint run --config ./.golangci.yml
 
@@ -153,7 +137,7 @@ lintci-deps:
 clean:
 	env GO111MODULE=on go clean -cache
 	rm -fr build/*
-	cd ethdb/mdbx/dist/ && make clean
+	cd libmdbx/ && make clean
 
 # The devtools target installs tools required for 'go generate'.
 # You need to put $GOBIN (or $GOPATH/bin) in your PATH to use 'go generate'.

README.md

@@ -94,18 +94,19 @@ Support only remote-miners.
 
 Windows users may run erigon in 3 possible ways:
 
-* Build executable binaries natively for Windows using provided `win-build.ps1` PowerShell script which has to be run with local Administrator privileges.
-  The script creates `libmdbx.dll` (MDBX is current default database for Erigon) and copies it into Windows's `system32` folder (generally `C:\Windows\system32`).
+* Build executable binaries natively for Windows using provided `wmake.ps1` PowerShell script.
+  Usage syntax is the same as `make` command so you have to run `.\wmake.ps1 [-target] <targetname>`. Example: `.\wmake.ps1 erigon` builds erigon executable.
+  All binaries are placed in `.\build\bin\` subfolder.
   There are some requirements for a successful native build on windows :
   * [Git](https://git-scm.com/downloads) for Windows must be installed. If you're cloning this repository is very likely you already have it
   * [GO Programming Language](https://golang.org/dl/) must be installed. Minimum required version is 1.16
-  * [Chocolatey package manager](https://chocolatey.org/) for Windows must be installed. By Chocolatey you need to install the following components : `cmake`, `make`, `mingw` by `choco install cmake make mingw`.
+  * If you need to build MDBX tools (i.e. `.\wmake.ps1 db-tools`) then [Chocolatey package manager](https://chocolatey.org/) for Windows must be installed. By Chocolatey you need to install the following components : `cmake`, `make`, `mingw` by `choco install cmake make mingw`.
 
   **Important note about Anti-Viruses**
   During MinGW's compiler detection phase some temporary executables are generated to test compiler capabilities. It's been reported some anti-virus programs detect
   those files as possibly infected by `Win64/Kryptic.CIS` trojan horse (or a variant of it). Although those are false positives we have no control over 100+ vendors of
   security products for Windows and their respective detection algorythms and we understand this might make your experience with Windows builds uncomfortable. To
-  workaround the issue you might either set exlusions for your antivirus specifically for `ethdb\mdbx\dist\CMakeFiles` folder or you can run erigon on Docker or WSL
+  workaround the issue you might either set exlusions for your antivirus specifically for `build\bin\mdbx\CMakeFiles` sub-folder of the cloned repo or you can run erigon using the following other two options
 
 * Use Docker :  see [docker-compose.yml](./docker-compose.yml)
 

accounts/abi/bind/backends/simulated.go

@@ -42,6 +42,7 @@ import (
 	"github.com/ledgerwatch/erigon/core/vm"
 	"github.com/ledgerwatch/erigon/eth/filters"
 	"github.com/ledgerwatch/erigon/ethdb"
+	"github.com/ledgerwatch/erigon/ethdb/kv"
 	"github.com/ledgerwatch/erigon/event"
 	"github.com/ledgerwatch/erigon/log"
 	"github.com/ledgerwatch/erigon/params"
@@ -103,7 +104,7 @@ func NewSimulatedBackendWithConfig(alloc core.GenesisAlloc, config *params.Chain
 			return h
 		},
 	}
-	backend.checkTEVM = ethdb.GetCheckTEVM(ethdb.NewObjectDatabase(m.DB))
+	backend.checkTEVM = ethdb.GetCheckTEVM(kv.NewObjectDatabase(m.DB))
 	backend.events = filters.NewEventSystem(&filterBackend{m.DB, backend})
 	backend.emptyPendingBlock()
 	return backend
@@ -165,7 +166,7 @@ func (b *SimulatedBackend) emptyPendingBlock() {
 	b.pendingReceipts = chain.Receipts[0]
 	b.pendingHeader = chain.Headers[0]
 	b.gasPool = new(core.GasPool).AddGas(b.pendingHeader.GasLimit)
-	b.pendingReader = state.NewPlainStateReader(ethdb.NewObjectDatabase(b.m.DB))
+	b.pendingReader = state.NewPlainStateReader(kv.NewObjectDatabase(b.m.DB))
 	b.pendingState = state.New(b.pendingReader)
 }
 

accounts/abi/bind/backends/simulated_test.go

@@ -29,6 +29,7 @@ import (
 	"github.com/holiman/uint256"
 	"github.com/ledgerwatch/erigon/core/rawdb"
 	"github.com/ledgerwatch/erigon/ethdb"
+	"github.com/ledgerwatch/erigon/ethdb/kv"
 
 	ethereum "github.com/ledgerwatch/erigon"
 	"github.com/ledgerwatch/erigon/accounts/abi"
@@ -147,7 +148,7 @@ func TestNewSimulatedBackend(t *testing.T) {
 	}); err != nil {
 		t.Fatal(err)
 	}
-	statedb := state.New(state.NewPlainDBState(ethdb.NewRoTxDb(tx), num))
+	statedb := state.New(state.NewPlainDBState(kv.NewRoTxDb(tx), num))
 	bal := statedb.GetBalance(testAddr)
 	if !bal.Eq(expectedBal) {
 		t.Errorf("expected balance for test address not received. expected: %v actual: %v", expectedBal, bal)

cmd/cons/commands/root.go

@@ -7,6 +7,7 @@ import (
 	"github.com/ledgerwatch/erigon/cmd/utils"
 	"github.com/ledgerwatch/erigon/common/paths"
 	"github.com/ledgerwatch/erigon/ethdb"
+	"github.com/ledgerwatch/erigon/ethdb/kv"
 	"github.com/ledgerwatch/erigon/internal/debug"
 	"github.com/spf13/cobra"
 )
@@ -66,7 +67,7 @@ func openDatabase(path string) ethdb.RwKV {
 }
 
 func openKV(path string, exclusive bool) ethdb.RwKV {
-	opts := ethdb.NewMDBX().Path(path)
+	opts := kv.NewMDBX().Path(path)
 	if exclusive {
 		opts = opts.Exclusive()
 	}

cmd/devp2p/crawl.go

@@ -104,7 +104,7 @@ loop:
 }
 
 func (c *crawler) runIterator(done chan<- enode.Iterator, it enode.Iterator) {
-	defer func() { debug.LogPanic(nil, true, recover()) }()
+	defer debug.LogPanic()
 	defer func() { done <- it }()
 	for it.Next() {
 		select {

cmd/devp2p/dnscmd.go

@@ -29,7 +29,7 @@ import (
 	"github.com/ledgerwatch/erigon/crypto"
 	"github.com/ledgerwatch/erigon/p2p/dnsdisc"
 	"github.com/ledgerwatch/erigon/p2p/enode"
-	cli "github.com/urfave/cli"
+	"github.com/urfave/cli"
 )
 
 var (
@@ -335,7 +335,7 @@ func writeTreeMetadata(directory string, def *dnsDefinition) {
 	if err != nil {
 		exit(err)
 	}
-	if err := os.Mkdir(directory, 0744); err != nil && !os.IsExist(err) {
+	if err := os.MkdirAll(directory, 0744); err != nil && !os.IsExist(err) {
 		exit(err)
 	}
 	metaFile, _ := treeDefinitionFiles(directory)

cmd/erigon/main.go

@@ -15,8 +15,7 @@ import (
 )
 
 func main() {
-	// catch panics from main thread and logs stack trace into a file
-	defer func() { debug.LogPanic(nil, true, recover()) }()
+	defer debug.LogPanic()
 	// creating a erigon-api app with all defaults
 	app := erigoncli.MakeApp(runErigon, erigoncli.DefaultFlags)
 	if err := app.Run(os.Args); err != nil {

cmd/evm/internal/t8ntool/execution.go

@@ -31,6 +31,7 @@ import (
 	"github.com/ledgerwatch/erigon/core/vm"
 	"github.com/ledgerwatch/erigon/crypto"
 	"github.com/ledgerwatch/erigon/ethdb"
+	"github.com/ledgerwatch/erigon/ethdb/kv"
 	"github.com/ledgerwatch/erigon/log"
 	"github.com/ledgerwatch/erigon/params"
 	"github.com/ledgerwatch/erigon/rlp"
@@ -106,7 +107,7 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig,
 		return h
 	}
 	var (
-		db          = ethdb.NewMemDatabase()
+		db          = kv.NewMemDatabase()
 		ibs         = MakePreState(context.Background(), db, pre.Pre)
 		signer      = types.MakeSigner(chainConfig, pre.Env.Number)
 		gaspool     = new(core.GasPool)

cmd/evm/runner.go

@@ -30,6 +30,7 @@ import (
 	"time"
 
 	"github.com/holiman/uint256"
+	"github.com/ledgerwatch/erigon/ethdb/kv"
 	"github.com/urfave/cli"
 
 	"github.com/ledgerwatch/erigon/cmd/evm/internal/compiler"
@@ -39,7 +40,6 @@ import (
 	"github.com/ledgerwatch/erigon/core/state"
 	"github.com/ledgerwatch/erigon/core/vm"
 	"github.com/ledgerwatch/erigon/core/vm/runtime"
-	"github.com/ledgerwatch/erigon/ethdb"
 	"github.com/ledgerwatch/erigon/log"
 	"github.com/ledgerwatch/erigon/params"
 )
@@ -135,7 +135,7 @@ func runCmd(ctx *cli.Context) error {
 	} else {
 		debugLogger = vm.NewStructLogger(logconfig)
 	}
-	db := ethdb.NewMemDatabase()
+	db := kv.NewMemDatabase()
 	if ctx.GlobalString(GenesisFlag.Name) != "" {
 		gen := readGenesis(ctx.GlobalString(GenesisFlag.Name))
 		_, _, err := gen.Commit(db, false)

cmd/evm/staterunner.go

@@ -27,7 +27,7 @@ import (
 	"github.com/ledgerwatch/erigon/common"
 	"github.com/ledgerwatch/erigon/core/state"
 	"github.com/ledgerwatch/erigon/core/vm"
-	"github.com/ledgerwatch/erigon/ethdb"
+	"github.com/ledgerwatch/erigon/ethdb/kv"
 	"github.com/ledgerwatch/erigon/log"
 	"github.com/ledgerwatch/erigon/tests"
 	"github.com/ledgerwatch/erigon/turbo/trie"
@@ -97,7 +97,7 @@ func stateTestCmd(ctx *cli.Context) error {
 		Debug:  ctx.GlobalBool(DebugFlag.Name) || ctx.GlobalBool(MachineFlag.Name),
 	}
 	results := make([]StatetestResult, 0, len(tests))
-	db := ethdb.NewMemKV()
+	db := kv.NewMemKV()
 	defer db.Close()
 
 	tx, txErr := db.BeginRw(context.Background())

cmd/hack/db/lmdb.go

@@ -17,6 +17,7 @@ import (
 	"github.com/ledgerwatch/erigon/common/dbutils"
 	"github.com/ledgerwatch/erigon/common/debug"
 	"github.com/ledgerwatch/erigon/ethdb"
+	kv2 "github.com/ledgerwatch/erigon/ethdb/kv"
 	"github.com/ledgerwatch/erigon/log"
 )
 
@@ -291,7 +292,7 @@ func launchReader(kv ethdb.RwKV, tx ethdb.Tx, expectVal string, startCh chan str
 	}
 	// Wait for the signal to start reading
 	go func() {
-		defer func() { debug.LogPanic(nil, true, recover()) }()
+		defer debug.LogPanic()
 		defer tx1.Rollback()
 		<-startCh
 		c, err := tx1.Cursor("t")
@@ -337,20 +338,20 @@ func defragSteps(filename string, bucketsCfg dbutils.BucketsCfg, generateFs ...f
 		return fmt.Errorf("creating temp dir for db visualisation: %w", err)
 	}
 	defer os.RemoveAll(dir)
-	var kv ethdb.RwKV
-	kv, err = ethdb.NewMDBX().Path(dir).WithBucketsConfig(func(dbutils.BucketsCfg) dbutils.BucketsCfg {
+	var db ethdb.RwKV
+	db, err = kv2.NewMDBX().Path(dir).WithBucketsConfig(func(dbutils.BucketsCfg) dbutils.BucketsCfg {
 		return bucketsCfg
 	}).Open()
 	if err != nil {
 		return fmt.Errorf("opening database: %w", err)
 	}
-	defer kv.Close()
+	defer db.Close()
 	for gi, generateF := range generateFs {
 		var display bool
-		if err = kv.Update(context.Background(), func(tx ethdb.RwTx) error {
+		if err = db.Update(context.Background(), func(tx ethdb.RwTx) error {
 			var err1 error
 			//nolint:scopelint
-			display, err1 = generateF(kv, tx)
+			display, err1 = generateF(db, tx)
 			return err1
 		}); err != nil {
 			return fmt.Errorf("generating data in temp db - function %d, file: %s: %w", gi, filename, err)

cmd/hack/flow/flow.go

@@ -73,7 +73,7 @@ func worker(code []byte) {
 	start := time.Now()
 
 	go func() {
-		defer func() { debug.LogPanic(nil, true, recover()) }()
+		defer debug.LogPanic()
 		cfg, _ := vm.GenCfg(code, maxAnlyCounterLimit, maxStackLen, maxStackCount, &metrics)
 		if cfg.Metrics.Valid {
 			proof := cfg.GenerateProof()
@@ -90,7 +90,7 @@ func worker(code []byte) {
 	oom := make(chan int, 1)
 
 	go func() {
-		defer func() { debug.LogPanic(nil, true, recover()) }()
+		defer debug.LogPanic()
 		for {
 			var m runtime.MemStats
 			runtime.ReadMemStats(&m)
@@ -180,7 +180,7 @@ func batchServer() {
 
 	for i := 0; i < numWorkers; i++ {
 		go func(id int) {
-			defer func() { debug.LogPanic(nil, true, recover()) }()
+			defer debug.LogPanic()
 			for job := range jobs {
 				enc := hex.EncodeToString(job.code)
 				cmd := exec.Command("./build/bin/hack",