Addressing issue #41 - maxSyncConcurrency environment variable

.gitignore

@@ -1,3 +1,5 @@
 rosetta-bitcoin
 bitcoin-data
 cli-data
+.DS_Store
+rosetta-sdk-go

Dockerfile.dev

@@ -0,0 +1,95 @@
+# Copyright 2020 Coinbase, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Build bitcoind
+FROM ubuntu:18.04 as bitcoind-builder
+
+RUN mkdir -p /app \
+  && chown -R nobody:nogroup /app
+WORKDIR /app
+
+# Copy the local sdk to your container
+COPY ./rosetta-sdk-go/* /app/
+
+# Source: https://github.com/bitcoin/bitcoin/blob/master/doc/build-unix.md#ubuntu--debian
+RUN apt-get update && apt-get install -y make gcc g++ autoconf autotools-dev bsdmainutils build-essential git libboost-all-dev \
+  libcurl4-openssl-dev libdb++-dev libevent-dev libssl-dev libtool pkg-config python python-pip libzmq3-dev wget
+
+# VERSION: Bitcoin Core 0.20.1
+RUN git clone https://github.com/bitcoin/bitcoin \
+  && cd bitcoin \
+  && git checkout 7ff64311bee570874c4f0dfa18f518552188df08
+
+RUN cd bitcoin \
+  && ./autogen.sh \
+  && ./configure --disable-tests --without-miniupnpc --without-gui --with-incompatible-bdb --disable-hardening --disable-zmq --disable-bench --disable-wallet \
+  && make
+
+RUN mv bitcoin/src/bitcoind /app/bitcoind \
+  && rm -rf bitcoin
+
+# Build Rosetta Server Components
+FROM ubuntu:18.04 as rosetta-builder
+
+RUN mkdir -p /app \
+  && chown -R nobody:nogroup /app
+WORKDIR /app
+
+RUN apt-get update && apt-get install -y curl make gcc g++
+ENV GOLANG_VERSION 1.15.5
+ENV GOLANG_DOWNLOAD_SHA256 9a58494e8da722c3aef248c9227b0e9c528c7318309827780f16220998180a0d
+ENV GOLANG_DOWNLOAD_URL https://golang.org/dl/go$GOLANG_VERSION.linux-amd64.tar.gz
+
+RUN curl -fsSL "$GOLANG_DOWNLOAD_URL" -o golang.tar.gz \
+  && echo "$GOLANG_DOWNLOAD_SHA256  golang.tar.gz" | sha256sum -c - \
+  && tar -C /usr/local -xzf golang.tar.gz \
+  && rm golang.tar.gz
+
+ENV GOPATH /go
+ENV PATH $GOPATH/bin:/usr/local/go/bin:$PATH
+RUN mkdir -p "$GOPATH/src" "$GOPATH/bin" && chmod -R 777 "$GOPATH"
+
+# Use native remote build context to build in any directory
+COPY . src
+RUN cd src \
+  && go build \
+  && cd .. \
+  && mv src/rosetta-bitcoin /app/rosetta-bitcoin \
+  && mv src/assets/* /app \
+  && rm -rf src
+
+## Build Final Image
+FROM ubuntu:18.04
+
+RUN apt-get update && \
+  apt-get install --no-install-recommends -y libevent-dev libboost-system-dev libboost-filesystem-dev libboost-test-dev libboost-thread-dev && \
+  apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+
+RUN mkdir -p /app \
+  && chown -R nobody:nogroup /app \
+  && mkdir -p /data \
+  && chown -R nobody:nogroup /data
+
+WORKDIR /app
+
+# Copy binary from bitcoind-builder
+COPY --from=bitcoind-builder /app/bitcoind /app/bitcoind
+
+# Copy binary from rosetta-builder
+COPY --from=rosetta-builder /app/* /app/
+
+# Set permissions for everything added to /app
+RUN chmod -R 755 /app/*
+
+CMD ["/app/rosetta-bitcoin"]

Makefile

@@ -25,22 +25,25 @@ build:
 build-local:
 	docker build -t rosetta-bitcoin:latest .
 
+build-local-dev:
+	docker build -f Dockerfile.dev -t rosetta-bitcoin:latest .
+
 build-release:
 	# make sure to always set version with vX.X.X
 	docker build -t rosetta-bitcoin:$(version) .;
 	docker save rosetta-bitcoin:$(version) | gzip > rosetta-bitcoin-$(version).tar.gz;
 
 run-mainnet-online:
-	docker run -d --rm --ulimit "nofile=${NOFILE}:${NOFILE}" -v "${PWD}/bitcoin-data:/data" -e "MODE=ONLINE" -e "NETWORK=MAINNET" -e "PORT=8080" -p 8080:8080 -p 8333:8333 rosetta-bitcoin:latest
+	docker run -d --rm --ulimit "nofile=${NOFILE}:${NOFILE}" -v "${PWD}/bitcoin-data:/data" -e "MAXSYNC=256" -e "MODE=ONLINE" -e "NETWORK=MAINNET" -e "PORT=8080" -p 8080:8080 -p 8333:8333 rosetta-bitcoin:latest
 
 run-mainnet-offline:
-	docker run -d --rm -e "MODE=OFFLINE" -e "NETWORK=MAINNET" -e "PORT=8081" -p 8081:8081 rosetta-bitcoin:latest
+	docker run -d --rm -e "MAXSYNC=256" -e "MODE=OFFLINE" -e "NETWORK=MAINNET" -e "PORT=8081" -p 8081:8081 rosetta-bitcoin:latest
 
 run-testnet-online:
-	docker run -d --rm --ulimit "nofile=${NOFILE}:${NOFILE}" -v "${PWD}/bitcoin-data:/data" -e "MODE=ONLINE" -e "NETWORK=TESTNET" -e "PORT=8080" -p 8080:8080 -p 18333:18333 rosetta-bitcoin:latest
+	docker run -d --rm --ulimit "nofile=${NOFILE}:${NOFILE}" -v "${PWD}/bitcoin-data:/data" -e "MAXSYNC=256" -e "MODE=ONLINE" -e "NETWORK=TESTNET" -e "PORT=8080" -p 8080:8080 -p 18333:18333 rosetta-bitcoin:latest
 
 run-testnet-offline:
-	docker run -d --rm -e "MODE=OFFLINE" -e "NETWORK=TESTNET" -e "PORT=8081" -p 8081:8081 rosetta-bitcoin:latest
+	docker run -d --rm -e "MAXSYNC=256" -e "MODE=OFFLINE" -e "NETWORK=TESTNET" -e "PORT=8081" -p 8081:8081 rosetta-bitcoin:latest
 
 train:
 	./zstd-train.sh $(network) transaction $(data-directory)

README.md

@@ -52,7 +52,16 @@ After cloning this repository, run:
 ```text
 make build-local
 ```
+#### From Source and Developing Locally
+After cloning this repository, run:
+```text
+make build-local-dev
+```
+Note: The purpose of this command is to build rosetta-bitcoin using a local sdk - this means you must have a local copy of rosetta-sdk-go in the same folder as the Dockefile and you must add the following to the bottom of your go.mod
 
+```text
+replace github.com/coinbase/rosetta-sdk-go v0.6.5 => ./rosetta-sdk-go
+```
 ### Run
 Running the following commands will start a Docker container in
 [detached mode](https://docs.docker.com/engine/reference/run/#detached--d) with
@@ -61,25 +70,25 @@ at port `8080`.
 
 #### Mainnet:Online
 ```text
-docker run -d --rm --ulimit "nofile=100000:100000" -v "$(pwd)/bitcoin-data:/data" -e "MODE=ONLINE" -e "NETWORK=MAINNET" -e "PORT=8080" -p 8080:8080 -p 8333:8333 rosetta-bitcoin:latest
+docker run -d --rm --ulimit "nofile=100000:100000" -v "$(pwd)/bitcoin-data:/data" -e "MAXSYNC=256" -e "MODE=ONLINE" -e "NETWORK=MAINNET" -e "PORT=8080" -p 8080:8080 -p 8333:8333 rosetta-bitcoin:latest
 ```
 _If you cloned the repository, you can run `make run-mainnet-online`._
 
 #### Mainnet:Offline
 ```text
-docker run -d --rm -e "MODE=OFFLINE" -e "NETWORK=MAINNET" -e "PORT=8081" -p 8081:8081 rosetta-bitcoin:latest
+docker run -d --rm -e "MAXSYNC=256" -e "MODE=OFFLINE" -e "NETWORK=MAINNET" -e "PORT=8081" -p 8081:8081 rosetta-bitcoin:latest
 ```
 _If you cloned the repository, you can run `make run-mainnet-offline`._
 
 #### Testnet:Online
 ```text
-docker run -d --rm --ulimit "nofile=100000:100000" -v "$(pwd)/bitcoin-data:/data" -e "MODE=ONLINE" -e "NETWORK=TESTNET" -e "PORT=8080" -p 8080:8080 -p 18333:18333 rosetta-bitcoin:latest
+docker run -d --rm --ulimit "nofile=100000:100000" -v "$(pwd)/bitcoin-data:/data" -e "MAXSYNC=256" -e "MODE=ONLINE" -e "NETWORK=TESTNET" -e "PORT=8080" -p 8080:8080 -p 18333:18333 rosetta-bitcoin:latest
 ```
 _If you cloned the repository, you can run `make run-testnet-online`._
 
 #### Testnet:Offline
 ```text
-docker run -d --rm -e "MODE=OFFLINE" -e "NETWORK=TESTNET" -e "PORT=8081" -p 8081:8081 rosetta-bitcoin:latest
+docker run -d --rm -e "MAXSYNC=256" -e "MODE=OFFLINE" -e "NETWORK=TESTNET" -e "PORT=8081" -p 8081:8081 rosetta-bitcoin:latest
 ```
 _If you cloned the repository, you can run `make run-testnet-offline`._
 

configuration/configuration.go

@@ -17,6 +17,7 @@ package configuration
 import (
 	"errors"
 	"fmt"
+	"github.com/coinbase/rosetta-sdk-go/syncer"
 	"os"
 	"path"
 	"strconv"
@@ -90,6 +91,10 @@ const (
 	// to determine mode.
 	ModeEnv = "MODE"
 
+	// MaxSyncConcurrency is an environment variable
+	// used to cap the syncer concurrency
+	MaxSyncConcurrency = "MAXSYNC"
+
 	// NetworkEnv is the environment variable
 	// read to determine network.
 	NetworkEnv = "NETWORK"
@@ -122,6 +127,7 @@ type Configuration struct {
 	IndexerPath            string
 	BitcoindPath           string
 	Compressors            []*encoder.CompressorEntry
+	MaxSyncConcurrency     int64
 }
 
 // LoadConfiguration attempts to create a new Configuration
@@ -134,6 +140,26 @@ func LoadConfiguration(baseDirectory string) (*Configuration, error) {
 		MinHeight: minPruneHeight,
 	}
 
+	defaultMaxSync := false
+	maxSyncValue := os.Getenv(MaxSyncConcurrency)
+	if len(maxSyncValue) == 0 {
+		defaultMaxSync = true
+	}
+
+	parsedValue, err := strconv.ParseInt(maxSyncValue, 10, 64)
+	if !(err == nil || defaultMaxSync) {
+		return nil, fmt.Errorf("%w: unable to parse maxsync %s", err, maxSyncValue)
+	}
+
+	switch {
+	case defaultMaxSync:
+		config.MaxSyncConcurrency = syncer.DefaultMaxConcurrency
+	case parsedValue <= 0:
+		return nil, errors.New("syncer concurrency must be greater than zero")
+	default:
+		config.MaxSyncConcurrency = parsedValue
+	}
+
 	modeValue := Mode(os.Getenv(ModeEnv))
 	switch modeValue {
 	case Online:

configuration/configuration_test.go

@@ -30,9 +30,10 @@ import (
 
 func TestLoadConfiguration(t *testing.T) {
 	tests := map[string]struct {
-		Mode    string
-		Network string
-		Port    string
+		Mode               string
+		Network            string
+		Port               string
+		MaxSyncConcurrency string
 
 		cfg *Configuration
 		err error
@@ -44,15 +45,17 @@ func TestLoadConfiguration(t *testing.T) {
 			Mode: string(Online),
 			err:  errors.New("NETWORK must be populated"),
 		},
-		"only mode and network set": {
-			Mode:    string(Online),
-			Network: Mainnet,
-			err:     errors.New("PORT must be populated"),
+		"port not set": {
+			Mode:               string(Online),
+			Network:            Mainnet,
+			MaxSyncConcurrency: "256",
+			err:                errors.New("PORT must be populated"),
 		},
 		"all set (mainnet)": {
-			Mode:    string(Online),
-			Network: Mainnet,
-			Port:    "1000",
+			Mode:               string(Online),
+			Network:            Mainnet,
+			Port:               "1000",
+			MaxSyncConcurrency: "256",
 			cfg: &Configuration{
 				Mode: Online,
 				Network: &types.NetworkIdentifier{
@@ -63,6 +66,7 @@ func TestLoadConfiguration(t *testing.T) {
 				Currency:               bitcoin.MainnetCurrency,
 				GenesisBlockIdentifier: bitcoin.MainnetGenesisBlockIdentifier,
 				Port:                   1000,
+				MaxSyncConcurrency:     256,
 				RPCPort:                mainnetRPCPort,
 				ConfigPath:             mainnetConfigPath,
 				Pruning: &PruningConfiguration{
@@ -79,9 +83,41 @@ func TestLoadConfiguration(t *testing.T) {
 			},
 		},
 		"all set (testnet)": {
-			Mode:    string(Online),
-			Network: Testnet,
-			Port:    "1000",
+			Mode:               string(Online),
+			Network:            Testnet,
+			Port:               "1000",
+			MaxSyncConcurrency: "256",
+			cfg: &Configuration{
+				Mode: Online,
+				Network: &types.NetworkIdentifier{
+					Network:    bitcoin.TestnetNetwork,
+					Blockchain: bitcoin.Blockchain,
+				},
+				Params:                 bitcoin.TestnetParams,
+				Currency:               bitcoin.TestnetCurrency,
+				GenesisBlockIdentifier: bitcoin.TestnetGenesisBlockIdentifier,
+				Port:                   1000,
+				MaxSyncConcurrency:     256,
+				RPCPort:                testnetRPCPort,
+				ConfigPath:             testnetConfigPath,
+				Pruning: &PruningConfiguration{
+					Frequency: pruneFrequency,
+					Depth:     pruneDepth,
+					MinHeight: minPruneHeight,
+				},
+				Compressors: []*encoder.CompressorEntry{
+					{
+						Namespace:      transactionNamespace,
+						DictionaryPath: testnetTransactionDictionary,
+					},
+				},
+			},
+		},
+		"default max sync set": {
+			Mode:               string(Online),
+			Network:            Testnet,
+			Port:               "1000",
+			MaxSyncConcurrency: "",
 			cfg: &Configuration{
 				Mode: Online,
 				Network: &types.NetworkIdentifier{
@@ -92,6 +128,7 @@ func TestLoadConfiguration(t *testing.T) {
 				Currency:               bitcoin.TestnetCurrency,
 				GenesisBlockIdentifier: bitcoin.TestnetGenesisBlockIdentifier,
 				Port:                   1000,
+				MaxSyncConcurrency:     256,
 				RPCPort:                testnetRPCPort,
 				ConfigPath:             testnetConfigPath,
 				Pruning: &PruningConfiguration{
@@ -107,6 +144,20 @@ func TestLoadConfiguration(t *testing.T) {
 				},
 			},
 		},
+		"maxsync negative mainnet": {
+			Mode:               string(Online),
+			Network:            Mainnet,
+			Port:               "1000",
+			MaxSyncConcurrency: "-2",
+			err:                errors.New("syncer concurrency must be greater than zero"),
+		},
+		"maxsync zero testnet": {
+			Mode:               string(Online),
+			Network:            Testnet,
+			Port:               "1000",
+			MaxSyncConcurrency: "0",
+			err:                errors.New("syncer concurrency must be greater than zero"),
+		},
 		"invalid mode": {
 			Mode:    "bad mode",
 			Network: Testnet,
@@ -136,6 +187,7 @@ func TestLoadConfiguration(t *testing.T) {
 			os.Setenv(ModeEnv, test.Mode)
 			os.Setenv(NetworkEnv, test.Network)
 			os.Setenv(PortEnv, test.Port)
+			os.Setenv(MaxSyncConcurrency, test.MaxSyncConcurrency)
 
 			cfg, err := LoadConfiguration(newDir)
 			if test.err != nil {

go.mod

@@ -13,4 +13,4 @@ require (
 	golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208
 	golang.org/x/tools v0.0.0-20200904185747-39188db58858 // indirect
 	honnef.co/go/tools v0.0.1-2020.1.5 // indirect
-)
+)