Merge pull request #533 from threefoldtech/dev_proxy_optimization

optimize database queries

grid-proxy/Makefile

@@ -1,6 +1,7 @@
 .DEFAULT_GOAL := help
 PQ_HOST = $(shell docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' postgres)
 PQ_CONTAINER = postgres
+count = 3
 
 install-swag: 
 	@go install github.com/swaggo/swag/cmd/[email protected];
@@ -41,7 +42,7 @@ db-stop: ## Stop the database container if running
 	@if [ ! "$(shell docker ps | grep '$(PQ_CONTAINER)' )" = "" ]; then \
 		docker stop postgres; \
 	fi
-
+db-refill: db-stop db-start sleep db-fill
 server-start: ## Start the proxy server			(Args: `m=<MNEMONICS>`)
 	@go run cmds/proxy_server/main.go \
 		-no-cert \
@@ -61,22 +62,26 @@ sleep:
 test-queries: ## Run all queries tests
 	@cd tests/queries/ &&\
 	go test -v \
+		-parallel 20 \
 		--seed 13 \
 		--postgres-host $(PQ_HOST) \
 		--postgres-db tfgrid-graphql \
 		--postgres-password postgres \
 		--postgres-user postgres \
-		--endpoint http://localhost:8080
+		--endpoint http://localhost:8080 \
+		-count $(count) 
 
 test-query: ## Run specific test query 			(Args: `t=TestName`).
 	@cd tests/queries/ &&\
 	go test -v \
+		-parallel 10 \
 		--seed 13 \
 		--postgres-host $(PQ_HOST) \
 		--postgres-db tfgrid-graphql \
 		--postgres-password postgres \
 		--postgres-user postgres \
 		--endpoint http://localhost:8080 \
+		-count $(count) \
 		-run $(t)
 
 test-unit: ## Run only unit tests
@@ -124,3 +129,13 @@ spelling:
 staticcheck:
 	@echo "Running $@"
 	@$(shell go env GOPATH)/bin/staticcheck -- ./...
+
+bench:
+	@cd tests/queries/ &&\
+	go test -v -bench Bench -run notests -count 5\
+		--seed 13 \
+		--postgres-host $(PQ_HOST) \
+		--postgres-db tfgrid-graphql \
+		--postgres-password postgres \
+		--postgres-user postgres \
+		--endpoint http://localhost:8080

grid-proxy/cmds/proxy_server/main.go

@@ -20,6 +20,7 @@ import (
 	logging "github.com/threefoldtech/tfgrid-sdk-go/grid-proxy/pkg"
 	rmb "github.com/threefoldtech/tfgrid-sdk-go/rmb-sdk-go"
 	"github.com/threefoldtech/tfgrid-sdk-go/rmb-sdk-go/peer"
+	"gorm.io/gorm/logger"
 )
 
 const (
@@ -39,6 +40,7 @@ type flags struct {
 	postgresDB               string
 	postgresUser             string
 	postgresPassword         string
+	sqlLogLevel              int
 	address                  string
 	version                  bool
 	nocert                   bool
@@ -65,6 +67,7 @@ func main() {
 	flag.StringVar(&f.postgresDB, "postgres-db", "", "postgres database")
 	flag.StringVar(&f.postgresUser, "postgres-user", "", "postgres username")
 	flag.StringVar(&f.postgresPassword, "postgres-password", "", "postgres password")
+	flag.IntVar(&f.sqlLogLevel, "sql-log-level", 2, "sql logger level")
 	flag.BoolVar(&f.version, "v", false, "shows the package version")
 	flag.BoolVar(&f.nocert, "no-cert", false, "start the server without certificate")
 	flag.StringVar(&f.domain, "domain", "", "domain on which the server will be served")
@@ -108,18 +111,22 @@ func main() {
 		log.Fatal().Err(err).Msg("failed to create relay client")
 	}
 
-	db, err := db.NewPostgresDatabase(f.postgresHost, f.postgresPort, f.postgresUser, f.postgresPassword, f.postgresDB, f.maxPoolOpenConnections)
+	db, err := db.NewPostgresDatabase(f.postgresHost, f.postgresPort, f.postgresUser, f.postgresPassword, f.postgresDB, f.maxPoolOpenConnections, logger.LogLevel(f.sqlLogLevel))
 	if err != nil {
 		log.Fatal().Err(err).Msg("couldn't get postgres client")
 	}
 
-	dbClient := explorer.DBClient{DB: db}
+	if err := db.Initialize(); err != nil {
+		log.Fatal().Err(err).Msg("failed to initialize database")
+	}
+
+	dbClient := explorer.DBClient{DB: &db}
 
 	indexer, err := gpuindexer.NewNodeGPUIndexer(
 		ctx,
 		f.relayURL,
 		f.mnemonics,
-		subManager, db,
+		subManager, &db,
 		f.indexerCheckIntervalMins,
 		f.indexerBatchSize,
 		f.indexerResultWorkers,

grid-proxy/internal/explorer/converters.go

@@ -52,14 +52,14 @@ func nodeFromDBNode(info db.Node) types.Node {
 		InDedicatedFarm:   info.FarmDedicated,
 		CertificationType: info.Certification,
 		RentContractID:    uint(info.RentContractID),
-		RentedByTwinID:    uint(info.RentedByTwinID),
+		RentedByTwinID:    uint(info.Renter),
 		SerialNumber:      info.SerialNumber,
 		Power:             types.NodePower(info.Power),
 		NumGPU:            info.NumGPU,
 		ExtraFee:          info.ExtraFee,
 	}
 	node.Status = nodestatus.DecideNodeStatus(node.Power, node.UpdatedAt)
-	node.Dedicated = info.FarmDedicated || !info.HasNodeContract || info.RentContractID != 0
+	node.Dedicated = info.FarmDedicated || info.NodeContractsCount == 0 || info.Renter != 0
 	return node
 }
 
@@ -124,14 +124,14 @@ func nodeWithNestedCapacityFromDBNode(info db.Node) types.NodeWithNestedCapacity
 		InDedicatedFarm:   info.FarmDedicated,
 		CertificationType: info.Certification,
 		RentContractID:    uint(info.RentContractID),
-		RentedByTwinID:    uint(info.RentedByTwinID),
+		RentedByTwinID:    uint(info.Renter),
 		SerialNumber:      info.SerialNumber,
 		Power:             types.NodePower(info.Power),
 		NumGPU:            info.NumGPU,
 		ExtraFee:          info.ExtraFee,
 	}
 	node.Status = nodestatus.DecideNodeStatus(node.Power, node.UpdatedAt)
-	node.Dedicated = info.FarmDedicated || !info.HasNodeContract || info.RentContractID != 0
+	node.Dedicated = info.FarmDedicated || info.NodeContractsCount == 0 || info.Renter != 0
 	return node
 }