add build scripts, add work-around for poor order-book sync between c…

…lient and server, mm bot rework of basic strategy to make it competitive, relax requirements for POL reserved/locked for fees, get rid of depth chart, other UI cleanup

Makefile

@@ -0,0 +1,34 @@
+build:
+	./build.sh
+
+a:
+	./a.sh
+
+l:
+	./l.sh
+
+server_build:
+	./build_server.sh
+
+server_run:
+	./run_server.sh
+
+build_linux:
+	./build_linux.sh
+
+test:
+	./run_tests.sh
+
+format:
+	gofmt -w -s $$(find . -type f -name '*.go' -not -path "./pkg/proto/*" -not -name "*.gen.go" -not -path "*/mock/*")
+	goimports -w $$(find . -type f -name '*.go' -not -path "./pkg/proto/*" -not -name "*.gen.go" -not -path "*/mock/*")
+
+#comment:
+#	$GOPATH/bin/commentwrap -fix -docflow_limit=90 /Users/norwnd/crypto-integrations/applications/outgoing-transactions/domain/outgoing_transaction.go
+#    $GOPATH/bin/commentwrap -fix -docflow_limit=90 TODO
+
+server_deps_up:
+	docker-compose -f server/docker/deps-compose.yml up -d --build
+
+server_deps_down:
+	docker-compose -f server/docker/deps-compose.yml down

a.sh

@@ -0,0 +1,13 @@
+set -e
+
+#!/usr/bin/env bash
+
+# mainnet
+(./client/cmd/bisonw/bisonw --webaddr=127.0.0.1:3333 --log=trace)
+# mainnet with dedicated DB:
+#(./client/cmd/bisonw/bisonw --db=/Users/norwnd/d-e-x-c-db/db_mainnet --webaddr=127.0.0.1:3333 --log=trace)
+# testnet
+#(./client/cmd/bisonw/bisonw --db=/Users/norwnd/d-e-x-c-db/db_mainnet --webaddr=127.0.0.1:3333 --log=trace --testnet)
+
+# with disaster backups:
+#(./client/cmd/bisonw/bisonw --webaddr=127.0.0.1:3333 --log=trace --skynetapikey=I8JMMBPUNMEHGCV056IG1V8OEEIB0CJUTG97S0J3IQNJJ4CSGH0G --skynetapiurl='https://web3portal.com' --testnet)

build.sh

@@ -0,0 +1,20 @@
+set -e
+
+#!/usr/bin/env bash
+
+pushd client/webserver/site
+npm clean-install
+npm run build
+popd
+
+(cd client/cmd/bisonw/ && CGO_ENABLED=1 GO111MODULE=on go build -tags lgpl)
+#
+# Mac-specific issue with older Golang versions (resolved with `codesign`), see discussions
+# here for details: https://github.com/golang/go/issues/63997
+#(cd client/cmd/bisonw/ && CGO_ENABLED=1 GO111MODULE=on go build -tags lgpl && codesign -s - -f ./bisonw)
+#
+# to specify OS and Architecture
+#(cd client/cmd/bisonw/ && CGO_ENABLED=1 GO111MODULE=on GOOS=darwin GOARCH=arm64 go build -tags lgpl)
+#
+# -race build
+#(cd client/cmd/bisonw/ && CGO_ENABLED=1 GO111MODULE=on go build -race -tags lgpl)

build_linux.sh

@@ -0,0 +1,19 @@
+set -e
+
+#!/usr/bin/env bash
+
+pushd client/webserver/site
+npm clean-install
+npm run build
+popd
+
+#(cd client/cmd/bisonw/ && CGO_ENABLED=1 GO111MODULE=on GOOS=linux GOARCH=amd64 go build -tags lgpl)
+(cd client/cmd/bisonw/ && CGO_ENABLED=1 GO111MODULE=on go build -tags lgpl)
+#
+# Note, this is -race build, to be used for testing only!
+#(cd client/cmd/bisonw/ && CGO_ENABLED=1 GO111MODULE=on go build -race -tags lgpl)
+# TODO, previously was CGO_ENABLED=0 ?
+
+# Run Bison binary with:
+# - (./client/cmd/bisonw/bisonw --db=/home/t/dcrdex-old-decrediton/db --webaddr=127.0.0.1:5758 --log=trace)
+# - (./client/cmd/bisonw/bisonw --db=/home/t/dcrdex-anon/db --webaddr=127.0.0.1:5758 --log=trace)

build_server.sh

@@ -0,0 +1,6 @@
+set -e
+
+#!/usr/bin/env bash
+
+#(cd server/cmd/dcrdex/ && CGO_ENABLED=0 GO111MODULE=on go build -tags lgpl -race)
+(cd server/cmd/dcrdex/ && CGO_ENABLED=0 GO111MODULE=on go build -tags lgpl)

client/asset/eth/eth.go

@@ -1461,6 +1461,10 @@ func (w *assetWallet) maxOrder(lotSize uint64, maxFeeRate uint64, ver uint32,
 	refundCost := g.Refund * maxFeeRate
 	oneFee := g.oneGas * maxFeeRate
 	feeReservesPerLot := oneFee + refundCost
+	// this doesn't work because server additionally validates at his own discretion whether
+	// a client has enough of base assets to fund his order:
+	//// statistically, 1-lot orders are rare - hence we can increase allowed lots here by x20
+	//feeReservesPerLot := (oneFee + refundCost) / 20 // integer division is fine here
 	var lots uint64
 	if feeWallet == nil {
 		lots = balance.Available / (lotSize + feeReservesPerLot)
@@ -1482,6 +1486,7 @@ func (w *assetWallet) maxOrder(lotSize uint64, maxFeeRate uint64, ver uint32,
 			FeeReservesPerLot: feeReservesPerLot,
 		}, nil
 	}
+
 	return w.estimateSwap(lots, lotSize, maxFeeRate, ver, feeReservesPerLot)
 }
 
@@ -1663,13 +1668,23 @@ func (w *ETHWallet) FundOrder(ord *asset.Order) (asset.Coins, []dex.Bytes, uint6
 			dex.BipIDSymbol(w.assetID), ord.MaxFeeRate, w.gasFeeLimit())
 	}
 
-	g, err := w.initGasEstimate(int(ord.MaxSwapCount), ord.Version,
-		ord.RedeemVersion, ord.RedeemAssetID)
+	g, err := w.initGasEstimate(int(ord.MaxSwapCount), ord.Version, ord.RedeemVersion, ord.RedeemAssetID)
 	if err != nil {
 		return nil, nil, 0, fmt.Errorf("error estimating swap gas: %v", err)
 	}
 
-	ethToLock := ord.MaxFeeRate*g.Swap*ord.MaxSwapCount + ord.Value
+	// statistically, we can shave off an order of magnitude here in funds locked
+	// for blockchain fees to process orders with large number of lots, this is safe
+	// to do for maker, but might be somewhat risky if we are taker,
+	// this doesn't fully work because server additionally validates from his own
+	// perspective whether a client has enough of base assets to fund his order
+	expectedSwapCnt := ord.MaxSwapCount
+	if expectedSwapCnt > 20 {
+		expectedSwapCnt = expectedSwapCnt / 20
+	} else if expectedSwapCnt > 10 {
+		expectedSwapCnt = expectedSwapCnt / 10
+	}
+	ethToLock := ord.MaxFeeRate*g.Swap*expectedSwapCnt + ord.Value
 	// Note: In a future refactor, we could lock the redemption funds here too
 	// and signal to the user so that they don't call `RedeemN`. This has the
 	// same net effect, but avoids a lockFunds -> unlockFunds for us and likely
@@ -1721,7 +1736,18 @@ func (w *TokenWallet) FundOrder(ord *asset.Order) (asset.Coins, []dex.Bytes, uin
 		return nil, nil, 0, fmt.Errorf("error estimating swap gas: %v", err)
 	}
 
-	ethToLock := ord.MaxFeeRate * g.Swap * ord.MaxSwapCount
+	// statistically, we can shave off an order of magnitude here in funds locked
+	// for blockchain fees to process orders with large number of lots, this is safe
+	// to do for maker, but might be somewhat risky if we are taker,
+	// this doesn't fully work because server additionally validates from his own
+	// perspective whether a client has enough of base assets to fund his order
+	expectedSwapCnt := ord.MaxSwapCount
+	if expectedSwapCnt > 20 {
+		expectedSwapCnt = expectedSwapCnt / 20
+	} else if expectedSwapCnt > 10 {
+		expectedSwapCnt = expectedSwapCnt / 10
+	}
+	ethToLock := ord.MaxFeeRate * g.Swap * expectedSwapCnt
 	var success bool
 	if err = w.lockFunds(ord.Value, initiationReserve); err != nil {
 		return nil, nil, 0, fmt.Errorf("error locking token funds: %v", err)

client/core/bookie.go

@@ -4,6 +4,7 @@
 package core
 
 import (
+	"encoding/hex"
 	"errors"
 	"fmt"
 	"sync"
@@ -374,6 +375,7 @@ func (b *bookie) book() *OrderBook {
 // be supplied.
 func (b *bookie) minifyOrder(oid dex.Bytes, trade *msgjson.TradeNote, epoch uint64) *MiniOrder {
 	return &MiniOrder{
+		ID:        hex.EncodeToString(oid),
 		Qty:       float64(trade.Quantity) / float64(b.baseUnits.Conventional.ConversionFactor),
 		QtyAtomic: trade.Quantity,
 		Rate:      calc.ConventionalRate(trade.Rate, b.baseUnits, b.quoteUnits),
@@ -393,11 +395,14 @@ func (dc *dexConnection) bookie(marketID string) *bookie {
 
 func (dc *dexConnection) midGap(base, quote uint32) (midGap uint64, err error) {
 	marketID := marketName(base, quote)
+	return dc.midGapMkt(marketID)
+}
+
+func (dc *dexConnection) midGapMkt(marketID string) (midGap uint64, err error) {
 	booky := dc.bookie(marketID)
 	if booky == nil {
 		return 0, fmt.Errorf("no bookie found for market %s", marketID)
 	}
-
 	return booky.MidGap()
 }
 
@@ -412,9 +417,20 @@ func (dc *dexConnection) syncBook(base, quote uint32) (*orderbook.OrderBook, Boo
 	dc.booksMtx.Lock()
 	defer dc.booksMtx.Unlock()
 
+	// Note, there are a bunch of issues with the way order server-originating book
+	// updates are handled, for details see: https://github.com/norwnd/dcrdex/pulls
+	// in particular this negatively affects mm bot operations, as a work-around
+	// we can remedy this for the most part by forcing full order book re-sync every
+	// once in a while (instead of using cached book every 5th call we'll re-populate
+	// it).
+
+	// TODO
+	dc.obSyncReqCnt = 1
+	//dc.obSyncReqCnt++
+
 	mktID := marketName(base, quote)
 	booky, found := dc.books[mktID]
-	if !found {
+	if !found || (dc.obSyncReqCnt%5 == 0) {
 		// Make sure the market exists.
 		if dc.marketConfig(mktID) == nil {
 			return nil, nil, fmt.Errorf("unknown market %s", mktID)

client/core/core.go

@@ -164,6 +164,9 @@ type dexConnection struct {
 
 	booksMtx sync.RWMutex
 	books    map[string]*bookie
+	// obSyncReqCnt counts order book sync requests issued to server
+	// through this dexConnection
+	obSyncReqCnt uint64
 
 	// tradeMtx is used to synchronize access to the trades map.
 	tradeMtx sync.RWMutex
@@ -6146,6 +6149,45 @@ func (c *Core) createTradeRequest(wallets *walletSet, coins asset.Coins, redeemS
 	}, nil
 }
 
+func validateTradeRate(sell bool, rate uint64, market string, dc *dexConnection) error {
+	if rate == 0 {
+		return newError(orderParamsErr, "zero rate is invalid")
+	}
+
+	// sanity check we are placing a trade that doesn't significantly diverge from the price
+	// on Bison market (10% seems like a worrisome divergence we don't want to permit)
+	bisonRate, err := dc.midGapMkt(market)
+	if err != nil {
+		return newError(walletErr, fmt.Sprintf("couldn't fetch mid-gap rate: %v", err))
+	}
+	// Note, we can't use "spot price" here because its value reflects the price of last
+	// trade which might have happened hours/days ago - and hence is too stale to rely on.
+	//bisonRate := dc.coreMarket(market).SpotPrice.Rate
+	if bisonRate == 0 {
+		return newError(walletErr, fmt.Sprintf("couldn't determine Bison rate "+
+			"for market: %s", market))
+	}
+	if math.Abs(float64(rate)-float64(bisonRate)) > (0.10 * float64(bisonRate)) {
+		return newError(orderParamsErr, fmt.Sprintf("trying to place trade with rate %d "+
+			"that's diverging from Bison rate %d for more than 10 percent", rate, bisonRate))
+	}
+
+	// additionally, prevent placing limit-orders that might result into slippage of 1% or more
+	if sell {
+		if float64(rate) < (float64(bisonRate) - 0.01*float64(bisonRate)) {
+			return newError(orderParamsErr, fmt.Sprintf("trying to place trade with rate %d "+
+				"that'd result into slippage of more than 1 percent (Bison rate = %d)", rate, bisonRate))
+		}
+	} else {
+		if float64(rate) > (float64(bisonRate) + 0.01*float64(bisonRate)) {
+			return newError(orderParamsErr, fmt.Sprintf("trying to place trade with rate %d "+
+				"that'd result into slippage of more than 1 percent (due to Bison rate = %d)", rate, bisonRate))
+		}
+	}
+
+	return nil
+}
+
 // prepareTradeRequest prepares a trade request.
 func (c *Core) prepareTradeRequest(pw []byte, form *TradeForm) (*tradeRequest, error) {
 	wallets, assetConfigs, dc, mktConf, err := c.prepareForTradeRequestPrep(pw, form.Base, form.Quote, form.Host, form.Sell)
@@ -6158,8 +6200,11 @@ func (c *Core) prepareTradeRequest(pw []byte, form *TradeForm) (*tradeRequest, e
 
 	rate, qty := form.Rate, form.Qty
 	if form.IsLimit {
-		if rate == 0 {
-			return nil, newError(orderParamsErr, "zero-rate order not allowed")
+		if qty == 0 {
+			return nil, newError(orderParamsErr, "zero quantity order not allowed")
+		}
+		if err := validateTradeRate(form.Sell, rate, mktID, dc); err != nil {
+			return nil, err
 		}
 		if minRate := dc.minimumMarketRate(assetConfigs.quoteAsset, mktConf.LotSize); rate < minRate {
 			return nil, newError(orderParamsErr, "order's rate is lower than market's minimum rate. %d < %d", rate, minRate)
@@ -6278,14 +6323,15 @@ func (c *Core) prepareMultiTradeRequests(pw []byte, form *MultiTradeForm) ([]*tr
 		return nil, err
 	}
 	fromWallet, toWallet := wallets.fromWallet, wallets.toWallet
+	mktID := marketName(form.Base, form.Quote)
 
 	for _, trade := range form.Placements {
-		if trade.Rate == 0 {
-			return nil, newError(orderParamsErr, "zero rate is invalid")
-		}
 		if trade.Qty == 0 {
 			return nil, newError(orderParamsErr, "zero quantity is invalid")
 		}
+		if err := validateTradeRate(form.Sell, trade.Rate, mktID, dc); err != nil {
+			return nil, err
+		}
 	}
 
 	redeemAddresses := make([]string, 0, len(form.Placements))
@@ -7209,7 +7255,7 @@ func (c *Core) initialize() error {
 // connectAccount makes a connection to the DEX for the given account. If a
 // non-nil dexConnection is returned from newDEXConnection, it was inserted into
 // the conns map even if the connection attempt failed (connected == false), and
-// the connect retry / keepalive loop is active. The intial connection attempt
+// the connect retry / keepalive loop is active. The initial connection attempt
 // or keepalive loop will not run if acct is disabled.
 func (c *Core) connectAccount(acct *db.AccountInfo) (dc *dexConnection, connected bool) {
 	host, err := addrHost(acct.Host)
@@ -8154,7 +8200,7 @@ func (c *Core) newDEXConnection(acctInfo *db.AccountInfo, flag connectDEXFlag) (
 
 	wsCfg := comms.WsCfg{
 		URL:      wsURL.String(),
-		PingWait: 20 * time.Second, // larger than server's pingPeriod (server/comms/server.go)
+		PingWait: 50 * time.Second, // larger than server's pingPeriod (server/comms/server.go)
 		Cert:     acctInfo.Cert,
 		Logger:   c.log.SubLogger(wsURL.String()),
 	}

client/core/types.go

@@ -726,6 +726,7 @@ func newDisplayIDFromSymbols(base, quote string) string {
 // MiniOrder is minimal information about an order in a market's order book.
 // Replaced MiniOrder, which had a float Qty in conventional units.
 type MiniOrder struct {
+	ID        string  `json:"id"`
 	Qty       float64 `json:"qty"`
 	QtyAtomic uint64  `json:"qtyAtomic"`
 	Rate      float64 `json:"rate"`

client/mm/exchange_adaptor.go

@@ -968,6 +968,7 @@ func (u *unifiedExchangeAdaptor) placeMultiTrade(placements []*dexOrderInfo, sel
 
 	for i, res := range results {
 		if res.Error != nil {
+			u.log.Errorf("incomplete multi-trade, couldn't make a placement: %s", res.Error)
 			continue
 		}