diff --git a/grid-proxy/Makefile b/grid-proxy/Makefile index e774cf98e..1bedce54d 100644 --- a/grid-proxy/Makefile +++ b/grid-proxy/Makefile @@ -1,6 +1,7 @@ .DEFAULT_GOAL := help PQ_HOST = $(shell docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' postgres) PQ_CONTAINER = postgres +count = 3 install-swag: @go install github.com/swaggo/swag/cmd/swag@v1.8.12; @@ -41,7 +42,7 @@ db-stop: ## Stop the database container if running @if [ ! "$(shell docker ps | grep '$(PQ_CONTAINER)' )" = "" ]; then \ docker stop postgres; \ fi - +db-refill: db-stop db-start sleep db-fill server-start: ## Start the proxy server (Args: `m=`) @go run cmds/proxy_server/main.go \ -no-cert \ @@ -61,22 +62,26 @@ sleep: test-queries: ## Run all queries tests @cd tests/queries/ &&\ go test -v \ + -parallel 20 \ --seed 13 \ --postgres-host $(PQ_HOST) \ --postgres-db tfgrid-graphql \ --postgres-password postgres \ --postgres-user postgres \ - --endpoint http://localhost:8080 + --endpoint http://localhost:8080 \ + -count $(count) test-query: ## Run specific test query (Args: `t=TestName`). @cd tests/queries/ &&\ go test -v \ + -parallel 10 \ --seed 13 \ --postgres-host $(PQ_HOST) \ --postgres-db tfgrid-graphql \ --postgres-password postgres \ --postgres-user postgres \ --endpoint http://localhost:8080 \ + -count $(count) \ -run $(t) test-unit: ## Run only unit tests @@ -124,3 +129,13 @@ spelling: staticcheck: @echo "Running $@" @$(shell go env GOPATH)/bin/staticcheck -- ./... + +bench: + @cd tests/queries/ &&\ + go test -v -bench Bench -run notests -count 5\ + --seed 13 \ + --postgres-host $(PQ_HOST) \ + --postgres-db tfgrid-graphql \ + --postgres-password postgres \ + --postgres-user postgres \ + --endpoint http://localhost:8080 \ No newline at end of file diff --git a/grid-proxy/cmds/proxy_server/main.go b/grid-proxy/cmds/proxy_server/main.go index f1bb8208c..aa98bceba 100644 --- a/grid-proxy/cmds/proxy_server/main.go +++ b/grid-proxy/cmds/proxy_server/main.go @@ -20,6 +20,7 @@ import ( logging "github.com/threefoldtech/tfgrid-sdk-go/grid-proxy/pkg" rmb "github.com/threefoldtech/tfgrid-sdk-go/rmb-sdk-go" "github.com/threefoldtech/tfgrid-sdk-go/rmb-sdk-go/peer" + "gorm.io/gorm/logger" ) const ( @@ -39,6 +40,7 @@ type flags struct { postgresDB string postgresUser string postgresPassword string + sqlLogLevel int address string version bool nocert bool @@ -65,6 +67,7 @@ func main() { flag.StringVar(&f.postgresDB, "postgres-db", "", "postgres database") flag.StringVar(&f.postgresUser, "postgres-user", "", "postgres username") flag.StringVar(&f.postgresPassword, "postgres-password", "", "postgres password") + flag.IntVar(&f.sqlLogLevel, "sql-log-level", 2, "sql logger level") flag.BoolVar(&f.version, "v", false, "shows the package version") flag.BoolVar(&f.nocert, "no-cert", false, "start the server without certificate") flag.StringVar(&f.domain, "domain", "", "domain on which the server will be served") @@ -108,18 +111,22 @@ func main() { log.Fatal().Err(err).Msg("failed to create relay client") } - db, err := db.NewPostgresDatabase(f.postgresHost, f.postgresPort, f.postgresUser, f.postgresPassword, f.postgresDB, f.maxPoolOpenConnections) + db, err := db.NewPostgresDatabase(f.postgresHost, f.postgresPort, f.postgresUser, f.postgresPassword, f.postgresDB, f.maxPoolOpenConnections, logger.LogLevel(f.sqlLogLevel)) if err != nil { log.Fatal().Err(err).Msg("couldn't get postgres client") } - dbClient := explorer.DBClient{DB: db} + if err := db.Initialize(); err != nil { + log.Fatal().Err(err).Msg("failed to initialize database") + } + + dbClient := explorer.DBClient{DB: &db} indexer, err := gpuindexer.NewNodeGPUIndexer( ctx, f.relayURL, f.mnemonics, - subManager, db, + subManager, &db, f.indexerCheckIntervalMins, f.indexerBatchSize, f.indexerResultWorkers, diff --git a/grid-proxy/internal/explorer/converters.go b/grid-proxy/internal/explorer/converters.go index d2c804754..acafeac3a 100644 --- a/grid-proxy/internal/explorer/converters.go +++ b/grid-proxy/internal/explorer/converters.go @@ -52,14 +52,14 @@ func nodeFromDBNode(info db.Node) types.Node { InDedicatedFarm: info.FarmDedicated, CertificationType: info.Certification, RentContractID: uint(info.RentContractID), - RentedByTwinID: uint(info.RentedByTwinID), + RentedByTwinID: uint(info.Renter), SerialNumber: info.SerialNumber, Power: types.NodePower(info.Power), NumGPU: info.NumGPU, ExtraFee: info.ExtraFee, } node.Status = nodestatus.DecideNodeStatus(node.Power, node.UpdatedAt) - node.Dedicated = info.FarmDedicated || !info.HasNodeContract || info.RentContractID != 0 + node.Dedicated = info.FarmDedicated || info.NodeContractsCount == 0 || info.Renter != 0 return node } @@ -124,14 +124,14 @@ func nodeWithNestedCapacityFromDBNode(info db.Node) types.NodeWithNestedCapacity InDedicatedFarm: info.FarmDedicated, CertificationType: info.Certification, RentContractID: uint(info.RentContractID), - RentedByTwinID: uint(info.RentedByTwinID), + RentedByTwinID: uint(info.Renter), SerialNumber: info.SerialNumber, Power: types.NodePower(info.Power), NumGPU: info.NumGPU, ExtraFee: info.ExtraFee, } node.Status = nodestatus.DecideNodeStatus(node.Power, node.UpdatedAt) - node.Dedicated = info.FarmDedicated || !info.HasNodeContract || info.RentContractID != 0 + node.Dedicated = info.FarmDedicated || info.NodeContractsCount == 0 || info.Renter != 0 return node } diff --git a/grid-proxy/internal/explorer/db/postgres.go b/grid-proxy/internal/explorer/db/postgres.go index 9d5db79db..8df35cbbc 100644 --- a/grid-proxy/internal/explorer/db/postgres.go +++ b/grid-proxy/internal/explorer/db/postgres.go @@ -16,6 +16,8 @@ import ( "gorm.io/gorm/clause" "gorm.io/gorm/logger" + _ "embed" + "github.com/pkg/errors" "github.com/rs/zerolog/log" ) @@ -26,85 +28,13 @@ var ( // ErrFarmNotFound farm not found ErrFarmNotFound = errors.New("farm not found") //ErrViewNotFound - ErrNodeResourcesViewNotFound = errors.New("ERROR: relation \"nodes_resources_view\" does not exist (SQLSTATE 42P01)") + ErrResourcesCacheTableNotFound = errors.New("ERROR: relation \"resources_cache\" does not exist (SQLSTATE 42P01)") // ErrContractNotFound contract not found ErrContractNotFound = errors.New("contract not found") ) -const ( - setupPostgresql = ` - CREATE OR REPLACE VIEW nodes_resources_view AS SELECT - node.node_id, - COALESCE(sum(contract_resources.cru), 0) as used_cru, - COALESCE(sum(contract_resources.mru), 0) + GREATEST(CAST((node_resources_total.mru / 10) AS bigint), 2147483648) as used_mru, - COALESCE(sum(contract_resources.hru), 0) as used_hru, - COALESCE(sum(contract_resources.sru), 0) + 21474836480 as used_sru, - node_resources_total.mru - COALESCE(sum(contract_resources.mru), 0) - GREATEST(CAST((node_resources_total.mru / 10) AS bigint), 2147483648) as free_mru, - node_resources_total.hru - COALESCE(sum(contract_resources.hru), 0) as free_hru, - node_resources_total.sru - COALESCE(sum(contract_resources.sru), 0) - 21474836480 as free_sru, - COALESCE(node_resources_total.cru, 0) as total_cru, - COALESCE(node_resources_total.mru, 0) as total_mru, - COALESCE(node_resources_total.hru, 0) as total_hru, - COALESCE(node_resources_total.sru, 0) as total_sru, - COALESCE(COUNT(DISTINCT state), 0) as states - FROM contract_resources - JOIN node_contract as node_contract - ON node_contract.resources_used_id = contract_resources.id AND node_contract.state IN ('Created', 'GracePeriod') - RIGHT JOIN node as node - ON node.node_id = node_contract.node_id - JOIN node_resources_total AS node_resources_total - ON node_resources_total.node_id = node.id - GROUP BY node.node_id, node_resources_total.mru, node_resources_total.sru, node_resources_total.hru, node_resources_total.cru; - - DROP FUNCTION IF EXISTS node_resources(query_node_id INTEGER); - CREATE OR REPLACE function node_resources(query_node_id INTEGER) - returns table (node_id INTEGER, used_cru NUMERIC, used_mru NUMERIC, used_hru NUMERIC, used_sru NUMERIC, free_mru NUMERIC, free_hru NUMERIC, free_sru NUMERIC, total_cru NUMERIC, total_mru NUMERIC, total_hru NUMERIC, total_sru NUMERIC, states BIGINT) - as - $body$ - SELECT - node.node_id, - COALESCE(sum(contract_resources.cru), 0) as used_cru, - COALESCE(sum(contract_resources.mru), 0) + GREATEST(CAST((node_resources_total.mru / 10) AS bigint), 2147483648) as used_mru, - COALESCE(sum(contract_resources.hru), 0) as used_hru, - COALESCE(sum(contract_resources.sru), 0) + 21474836480 as used_sru, - node_resources_total.mru - COALESCE(sum(contract_resources.mru), 0) - GREATEST(CAST((node_resources_total.mru / 10) AS bigint), 2147483648) as free_mru, - node_resources_total.hru - COALESCE(sum(contract_resources.hru), 0) as free_hru, - node_resources_total.sru - COALESCE(sum(contract_resources.sru), 0) - 21474836480 as free_sru, - COALESCE(node_resources_total.cru, 0) as total_cru, - COALESCE(node_resources_total.mru, 0) as total_mru, - COALESCE(node_resources_total.hru, 0) as total_hru, - COALESCE(node_resources_total.sru, 0) as total_sru, - COALESCE(COUNT(DISTINCT state), 0) as states - FROM contract_resources - JOIN node_contract as node_contract - ON node_contract.resources_used_id = contract_resources.id AND node_contract.state IN ('Created', 'GracePeriod') - RIGHT JOIN node as node - ON node.node_id = node_contract.node_id - JOIN node_resources_total AS node_resources_total - ON node_resources_total.node_id = node.id - WHERE node.node_id = query_node_id - GROUP BY node.node_id, node_resources_total.mru, node_resources_total.sru, node_resources_total.hru, node_resources_total.cru; - $body$ - language sql; - - DROP FUNCTION IF EXISTS convert_to_decimal(v_input text); - CREATE OR REPLACE FUNCTION convert_to_decimal(v_input text) - RETURNS DECIMAL AS $$ - DECLARE v_dec_value DECIMAL DEFAULT NULL; - BEGIN - BEGIN - v_dec_value := v_input::DECIMAL; - EXCEPTION WHEN OTHERS THEN - RAISE NOTICE 'Invalid decimal value: "%". Returning NULL.', v_input; - RETURN NULL; - END; - RETURN v_dec_value; - END; - $$ LANGUAGE plpgsql; - - DROP TRIGGER IF EXISTS node_added ON node; - ` -) +//go:embed setup.sql +var setupFile string // PostgresDatabase postgres db client type PostgresDatabase struct { @@ -117,32 +47,30 @@ func (d *PostgresDatabase) GetConnectionString() string { } // NewPostgresDatabase returns a new postgres db client -func NewPostgresDatabase(host string, port int, user, password, dbname string, maxConns int) (Database, error) { +func NewPostgresDatabase(host string, port int, user, password, dbname string, maxConns int, logLevel logger.LogLevel) (PostgresDatabase, error) { connString := fmt.Sprintf("host=%s port=%d user=%s "+ "password=%s dbname=%s sslmode=disable", host, port, user, password, dbname) - gormDB, err := gorm.Open(postgres.Open(connString), &gorm.Config{}) + gormDB, err := gorm.Open(postgres.Open(connString), &gorm.Config{ + Logger: logger.Default.LogMode(logLevel), + }) if err != nil { - return nil, errors.Wrap(err, "failed to create orm wrapper around db") + return PostgresDatabase{}, errors.Wrap(err, "failed to create orm wrapper around db") } sql, err := gormDB.DB() if err != nil { - return nil, errors.Wrap(err, "failed to configure DB connection") + return PostgresDatabase{}, errors.Wrap(err, "failed to configure DB connection") } - sql.SetMaxIdleConns(3) - sql.SetMaxOpenConns(maxConns) - err = gormDB.AutoMigrate(&NodeGPU{}) if err != nil { - return nil, errors.Wrap(err, "failed to auto migrate DB") + return PostgresDatabase{}, errors.Wrap(err, "failed to migrate node_gpu table") } + sql.SetMaxIdleConns(3) + sql.SetMaxOpenConns(maxConns) res := PostgresDatabase{gormDB, connString} - if err := res.initialize(); err != nil { - return nil, errors.Wrap(err, "failed to setup tables") - } - return &res, nil + return res, nil } // Close the db connection @@ -154,9 +82,8 @@ func (d *PostgresDatabase) Close() error { return db.Close() } -func (d *PostgresDatabase) initialize() error { - res := d.gormDB.Exec(setupPostgresql) - return res.Error +func (d *PostgresDatabase) Initialize() error { + return d.gormDB.Exec(setupFile).Error } // GetStats returns aggregate info about the grid @@ -213,11 +140,7 @@ func (d *PostgresDatabase) GetStats(ctx context.Context, filter types.StatsFilte } query := d.gormDB.WithContext(ctx). Table("node"). - Joins( - `RIGHT JOIN public_config - ON node.id = public_config.node_id - `, - ) + Joins("RIGHT JOIN public_config ON node.id = public_config.node_id") if res := query.Where(condition).Where("COALESCE(public_config.ipv4, '') != '' OR COALESCE(public_config.ipv6, '') != ''").Count(&stats.AccessNodes); res.Error != nil { return stats, errors.Wrap(res.Error, "couldn't get access node count") @@ -277,10 +200,8 @@ func (np *NodePower) Scan(value interface{}) error { // GetNode returns node info func (d *PostgresDatabase) GetNode(ctx context.Context, nodeID uint32) (Node, error) { - q := d.nodeTableQuery() - q = q.WithContext(ctx) + q := d.nodeTableQuery(ctx, types.NodeFilter{}, &gorm.DB{}) q = q.Where("node.node_id = ?", nodeID) - q = q.Session(&gorm.Session{Logger: logger.Default.LogMode(logger.Silent)}) var node Node res := q.Scan(&node) if d.shouldRetry(res.Error) { @@ -297,8 +218,7 @@ func (d *PostgresDatabase) GetNode(ctx context.Context, nodeID uint32) (Node, er // GetFarm return farm info func (d *PostgresDatabase) GetFarm(ctx context.Context, farmID uint32) (Farm, error) { - q := d.farmTableQuery() - q = q.WithContext(ctx) + q := d.gormDB.WithContext(ctx).Table("farm") q = q.Where("farm.farm_id = ?", farmID) var farm Farm if res := q.Scan(&farm); res.Error != nil { @@ -336,84 +256,8 @@ func printQuery(query string, args ...interface{}) { fmt.Printf("node query: %s", query) } -func (d *PostgresDatabase) farmTableQuery() *gorm.DB { - return d.gormDB. - Table("farm"). - Select( - "farm.id", - "farm.farm_id", - "farm.name", - "farm.twin_id", - "farm.pricing_policy_id", - "farm.certification", - "farm.stellar_address", - "farm.dedicated_farm as dedicated", - "COALESCE(public_ips.public_ips, '[]') as public_ips", - "bool_or(node.rent_contract_id != 0)", - ). - Joins( - `left join( - SELECT - node.node_id, - node.twin_id, - node.farm_id, - node.power, - node.updated_at, - node.certification, - node.country, - nodes_resources_view.free_mru, - nodes_resources_view.free_hru, - nodes_resources_view.free_sru, - nodes_resources_view.total_cru, - COALESCE(rent_contract.contract_id, 0) rent_contract_id, - COALESCE(rent_contract.twin_id, 0) renter, - COALESCE(node_gpu.id, '') gpu_id - FROM node - LEFT JOIN nodes_resources_view ON node.node_id = nodes_resources_view.node_id - LEFT JOIN rent_contract ON node.node_id = rent_contract.node_id AND rent_contract.state IN ('Created', 'GracePeriod') - LEFT JOIN node_gpu ON node.twin_id = node_gpu.node_twin_id - ) node on node.farm_id = farm.farm_id`, - ). - Joins(`left join( - SELECT - p1.farm_id, - COUNT(p1.id) total_ips, - COUNT(CASE WHEN p2.contract_id = 0 THEN 1 END) free_ips - FROM public_ip p1 - LEFT JOIN public_ip p2 ON p1.id = p2.id - GROUP BY p1.farm_id - ) public_ip_count on public_ip_count.farm_id = farm.id`). - Joins(`left join ( - select - farm_id, - jsonb_agg(jsonb_build_object('id', id, 'ip', ip, 'contract_id', contract_id, 'gateway', gateway)) as public_ips - from public_ip - GROUP BY farm_id - ) public_ips on public_ips.farm_id = farm.id`). - Joins( - "LEFT JOIN country ON node.country = country.name", - ). - Group( - `farm.id, - farm.farm_id, - farm.name, - farm.twin_id, - farm.pricing_policy_id, - farm.certification, - farm.stellar_address, - farm.dedicated_farm, - COALESCE(public_ips.public_ips, '[]')`, - ) -} - -func (d *PostgresDatabase) nodeTableQuery() *gorm.DB { - subquery := d.gormDB.Table("node_contract"). - Select("DISTINCT ON (node_id) node_id, contract_id"). - Where("state IN ('Created', 'GracePeriod')") - - nodeGPUQuery := `(SELECT count(node_gpu.id) FROM node_gpu WHERE node_gpu.node_twin_id = node.twin_id) as num_gpu` - - return d.gormDB. +func (d *PostgresDatabase) nodeTableQuery(ctx context.Context, filter types.NodeFilter, nodeGpuSubquery *gorm.DB) *gorm.DB { + q := d.gormDB.WithContext(ctx). Table("node"). Select( "node.id", @@ -426,15 +270,15 @@ func (d *PostgresDatabase) nodeTableQuery() *gorm.DB { "node.uptime", "node.created", "node.farming_policy_id", - "updated_at", - "nodes_resources_view.total_cru", - "nodes_resources_view.total_sru", - "nodes_resources_view.total_hru", - "nodes_resources_view.total_mru", - "nodes_resources_view.used_cru", - "nodes_resources_view.used_sru", - "nodes_resources_view.used_hru", - "nodes_resources_view.used_mru", + "node.updated_at", + "resources_cache.total_cru", + "resources_cache.total_sru", + "resources_cache.total_hru", + "resources_cache.total_mru", + "resources_cache.used_cru", + "resources_cache.used_sru", + "resources_cache.used_hru", + "resources_cache.used_mru", "public_config.domain", "public_config.gw4", "public_config.gw6", @@ -442,43 +286,232 @@ func (d *PostgresDatabase) nodeTableQuery() *gorm.DB { "public_config.ipv6", "node.certification", "farm.dedicated_farm as farm_dedicated", - "rent_contract.contract_id as rent_contract_id", - "rent_contract.twin_id as rented_by_twin_id", + "resources_cache.rent_contract_id as rent_contract_id", + "resources_cache.renter", "node.serial_number", "convert_to_decimal(location.longitude) as longitude", "convert_to_decimal(location.latitude) as latitude", "node.power", "node.extra_fee", - "CASE WHEN node_contract.contract_id IS NOT NULL THEN 1 ELSE 0 END AS has_node_contract", - nodeGPUQuery, - ). - Joins( - "LEFT JOIN nodes_resources_view ON node.node_id = nodes_resources_view.node_id", - ). - Joins( - "LEFT JOIN public_config ON node.id = public_config.node_id", - ). - Joins( - "LEFT JOIN rent_contract ON rent_contract.state IN ('Created', 'GracePeriod') AND rent_contract.node_id = node.node_id", + "resources_cache.node_contracts_count", + "resources_cache.node_gpu_count AS num_gpu", ). - Joins( - "LEFT JOIN (?) AS node_contract ON node_contract.node_id = node.node_id", subquery, - ). - Joins( - "LEFT JOIN farm ON node.farm_id = farm.farm_id", - ). - Joins( - "LEFT JOIN location ON node.location_id = location.id", + Joins(` + LEFT JOIN resources_cache ON node.node_id = resources_cache.node_id + LEFT JOIN public_ips_cache ON public_ips_cache.farm_id = node.farm_id + LEFT JOIN public_config ON node.id = public_config.node_id + LEFT JOIN farm ON node.farm_id = farm.farm_id + LEFT JOIN location ON node.location_id = location.id + `) + + if filter.HasGPU != nil || filter.GpuDeviceName != nil || + filter.GpuVendorName != nil || filter.GpuVendorID != nil || + filter.GpuDeviceID != nil || filter.GpuAvailable != nil { + q.Joins( + `RIGHT JOIN (?) AS gpu ON gpu.node_twin_id = node.twin_id`, nodeGpuSubquery, + ) + } + + return q +} + +func (d *PostgresDatabase) farmTableQuery(ctx context.Context, filter types.FarmFilter, nodeQuery *gorm.DB) *gorm.DB { + q := d.gormDB.WithContext(ctx). + Table("farm"). + Select( + "farm.id", + "farm.farm_id", + "farm.name", + "farm.twin_id", + "farm.pricing_policy_id", + "farm.certification", + "farm.stellar_address", + "farm.dedicated_farm as dedicated", + "COALESCE(public_ips_cache.ips, '[]') as public_ips", ). Joins( - "LEFT JOIN country ON country.name = node.country", + "LEFT JOIN public_ips_cache ON public_ips_cache.farm_id = farm.farm_id", ) + + if filter.NodeAvailableFor != nil || filter.NodeFreeHRU != nil || + filter.NodeCertified != nil || filter.NodeFreeMRU != nil || + filter.NodeFreeSRU != nil || filter.NodeHasGPU != nil || + filter.NodeRentedBy != nil || filter.NodeStatus != nil || + filter.NodeTotalCRU != nil || filter.Country != nil || + filter.Region != nil { + q.Joins(`RIGHT JOIN (?) AS resources_cache on resources_cache.farm_id = farm.farm_id`, nodeQuery). + Group(` + farm.id, + farm.farm_id, + farm.name, + farm.twin_id, + farm.pricing_policy_id, + farm.certification, + farm.stellar_address, + farm.dedicated_farm, + COALESCE(public_ips_cache.ips, '[]') + `) + } + + return q +} + +// GetFarms return farms filtered and paginated +func (d *PostgresDatabase) GetFarms(ctx context.Context, filter types.FarmFilter, limit types.Limit) ([]Farm, uint, error) { + nodeQuery := d.gormDB.Table("resources_cache"). + Select("resources_cache.farm_id", "renter"). + Joins("LEFT JOIN node ON node.node_id = resources_cache.node_id"). + Group(`resources_cache.farm_id, renter`) + + if filter.NodeFreeMRU != nil { + nodeQuery = nodeQuery.Where("resources_cache.free_mru >= ?", *filter.NodeFreeMRU) + } + if filter.NodeFreeHRU != nil { + nodeQuery = nodeQuery.Where("resources_cache.free_hru >= ?", *filter.NodeFreeHRU) + } + if filter.NodeFreeSRU != nil { + nodeQuery = nodeQuery.Where("resources_cache.free_sru >= ?", *filter.NodeFreeSRU) + } + if filter.NodeTotalCRU != nil { + nodeQuery = nodeQuery.Where("resources_cache.total_cru >= ?", *filter.NodeTotalCRU) + } + + if filter.NodeHasGPU != nil { + nodeQuery = nodeQuery.Where("(resources_cache.node_gpu_count > 0) = ?", *filter.NodeHasGPU) + } + + if filter.NodeRentedBy != nil { + nodeQuery = nodeQuery.Where("COALESCE(resources_cache.renter, 0) = ?", *filter.NodeRentedBy) + } + + if filter.Country != nil { + nodeQuery = nodeQuery.Where("LOWER(resources_cache.country) = LOWER(?)", *filter.Country) + } + + if filter.Region != nil { + nodeQuery = nodeQuery.Where("LOWER(resources_cache.region) = LOWER(?)", *filter.Region) + } + + if filter.NodeStatus != nil { + condition := nodestatus.DecideNodeStatusCondition(*filter.NodeStatus) + nodeQuery = nodeQuery.Where(condition) + } + + if filter.NodeCertified != nil { + nodeQuery = nodeQuery.Where("(node.certification = 'Certified') = ?", *filter.NodeCertified) + } + + q := d.farmTableQuery(ctx, filter, nodeQuery) + + if filter.NodeAvailableFor != nil { + q = q.Where("COALESCE(resources_cache.renter, 0) = ? OR (resources_cache.renter IS NULL AND farm.dedicated_farm = false)", *filter.NodeAvailableFor) + } + + if filter.FreeIPs != nil { + q = q.Where("COALESCE(public_ips_cache.free_ips, 0) >= ?", *filter.FreeIPs) + } + if filter.TotalIPs != nil { + q = q.Where("COALESCE(public_ips_cache.total_ips, 0) >= ?", *filter.TotalIPs) + } + if filter.StellarAddress != nil { + q = q.Where("COALESCE(farm.stellar_address, '') = ?", *filter.StellarAddress) + } + if filter.PricingPolicyID != nil { + q = q.Where("farm.pricing_policy_id = ?", *filter.PricingPolicyID) + } + if filter.FarmID != nil { + q = q.Where("farm.farm_id = ?", *filter.FarmID) + } + if filter.TwinID != nil { + q = q.Where("farm.twin_id = ?", *filter.TwinID) + } + if filter.Name != nil { + q = q.Where("LOWER(farm.name) = LOWER(?)", *filter.Name) + } + + if filter.NameContains != nil { + escaped := strings.Replace(*filter.NameContains, "%", "\\%", -1) + escaped = strings.Replace(escaped, "_", "\\_", -1) + q = q.Where("farm.name ILIKE ?", fmt.Sprintf("%%%s%%", escaped)) + } + + if filter.CertificationType != nil { + q = q.Where("farm.certification = ?", *filter.CertificationType) + } + + if filter.Dedicated != nil { + q = q.Where("farm.dedicated_farm = ?", *filter.Dedicated) + } + var count int64 + if limit.Randomize || limit.RetCount { + if res := q.Count(&count); res.Error != nil { + return nil, 0, errors.Wrap(res.Error, "couldn't get farm count") + } + } + + // Sorting + if limit.Randomize { + q = q.Order("random()") + } else { + if filter.NodeAvailableFor != nil { + q = q.Order("(bool_or(resources_cache.renter IS NOT NULL)) DESC") + } + if limit.SortBy != "" { + order := types.SortOrderAsc + if strings.EqualFold(string(limit.SortOrder), string(types.SortOrderDesc)) { + order = types.SortOrderDesc + } + q = q.Order(fmt.Sprintf("%s %s", limit.SortBy, order)) + } else { + q = q.Order("farm.farm_id") + } + } + + // Pagination + q = q.Limit(int(limit.Size)).Offset(int(limit.Page-1) * int(limit.Size)) + + var farms []Farm + if res := q.Scan(&farms); res.Error != nil { + return farms, uint(count), errors.Wrap(res.Error, "failed to scan returned farm from database") + } + return farms, uint(count), nil } // GetNodes returns nodes filtered and paginated func (d *PostgresDatabase) GetNodes(ctx context.Context, filter types.NodeFilter, limit types.Limit) ([]Node, uint, error) { - q := d.nodeTableQuery() - q = q.WithContext(ctx) + /* + used distinct selecting to avoid duplicated node after the join. + - postgres apply WHERE before DISTINCT so filters will still filter on the whole data. + - we don't return any gpu info on the node object so no worries of losing the data because DISTINCT. + */ + nodeGpuSubquery := d.gormDB.Table("node_gpu"). + Select("DISTINCT ON (node_twin_id) node_twin_id") + + if filter.HasGPU != nil { + nodeGpuSubquery = nodeGpuSubquery.Where("(COALESCE(node_gpu.id, '') != '') = ?", *filter.HasGPU) + } + + if filter.GpuDeviceName != nil { + nodeGpuSubquery = nodeGpuSubquery.Where("COALESCE(node_gpu.device, '') ILIKE '%' || ? || '%'", *filter.GpuDeviceName) + } + + if filter.GpuVendorName != nil { + nodeGpuSubquery = nodeGpuSubquery.Where("COALESCE(node_gpu.vendor, '') ILIKE '%' || ? || '%'", *filter.GpuVendorName) + } + + if filter.GpuVendorID != nil { + nodeGpuSubquery = nodeGpuSubquery.Where("COALESCE(node_gpu.id, '') ILIKE '%' || ? || '%'", *filter.GpuVendorID) + } + + if filter.GpuDeviceID != nil { + nodeGpuSubquery = nodeGpuSubquery.Where("COALESCE(node_gpu.id, '') ILIKE '%' || ? || '%'", *filter.GpuDeviceID) + } + + if filter.GpuAvailable != nil { + nodeGpuSubquery = nodeGpuSubquery.Where("(COALESCE(node_gpu.contract, 0) = 0) = ?", *filter.GpuAvailable) + } + + q := d.nodeTableQuery(ctx, filter, nodeGpuSubquery) condition := "TRUE" if filter.Status != nil { @@ -488,25 +521,25 @@ func (d *PostgresDatabase) GetNodes(ctx context.Context, filter types.NodeFilter q = q.Where(condition) if filter.FreeMRU != nil { - q = q.Where("nodes_resources_view.free_mru >= ?", *filter.FreeMRU) + q = q.Where("resources_cache.free_mru >= ?", *filter.FreeMRU) } if filter.FreeHRU != nil { - q = q.Where("nodes_resources_view.free_hru >= ?", *filter.FreeHRU) + q = q.Where("resources_cache.free_hru >= ?", *filter.FreeHRU) } if filter.FreeSRU != nil { - q = q.Where("nodes_resources_view.free_sru >= ?", *filter.FreeSRU) + q = q.Where("resources_cache.free_sru >= ?", *filter.FreeSRU) } if filter.TotalCRU != nil { - q = q.Where("nodes_resources_view.total_cru >= ?", *filter.TotalCRU) + q = q.Where("resources_cache.total_cru >= ?", *filter.TotalCRU) } if filter.TotalHRU != nil { - q = q.Where("nodes_resources_view.total_hru >= ?", *filter.TotalHRU) + q = q.Where("resources_cache.total_hru >= ?", *filter.TotalHRU) } if filter.TotalMRU != nil { - q = q.Where("nodes_resources_view.total_mru >= ?", *filter.TotalMRU) + q = q.Where("resources_cache.total_mru >= ?", *filter.TotalMRU) } if filter.TotalSRU != nil { - q = q.Where("nodes_resources_view.total_sru >= ?", *filter.TotalSRU) + q = q.Where("resources_cache.total_sru >= ?", *filter.TotalSRU) } if filter.Country != nil { q = q.Where("LOWER(node.country) = LOWER(?)", *filter.Country) @@ -521,7 +554,7 @@ func (d *PostgresDatabase) GetNodes(ctx context.Context, filter types.NodeFilter q = q.Where("node.city ILIKE '%' || ? || '%'", *filter.CityContains) } if filter.Region != nil { - q = q.Where("LOWER(country.subregion) = LOWER(?)", *filter.Region) + q = q.Where("LOWER(resources_cache.region) = LOWER(?)", *filter.Region) } if filter.NodeID != nil { q = q.Where("node.node_id = ?", *filter.NodeID) @@ -539,16 +572,16 @@ func (d *PostgresDatabase) GetNodes(ctx context.Context, filter types.NodeFilter q = q.Where("farm.name ILIKE '%' || ? || '%'", *filter.FarmNameContains) } if filter.FreeIPs != nil { - q = q.Where("(SELECT count(id) from public_ip WHERE public_ip.farm_id = farm.id AND public_ip.contract_id = 0) >= ?", *filter.FreeIPs) + q = q.Where("COALESCE(public_ips_cache.free_ips, 0) >= ?", *filter.FreeIPs) } if filter.IPv4 != nil { - q = q.Where("(COALESCE(public_config.ipv4, '') = '') != ?", *filter.IPv4) + q = q.Where("(COALESCE(public_config.ipv4, '') != '') = ?", *filter.IPv4) } if filter.IPv6 != nil { - q = q.Where("(COALESCE(public_config.ipv6, '') = '') != ?", *filter.IPv6) + q = q.Where("(COALESCE(public_config.ipv6, '') != '') = ?", *filter.IPv6) } if filter.Domain != nil { - q = q.Where("(COALESCE(public_config.domain, '') = '') != ?", *filter.Domain) + q = q.Where("(COALESCE(public_config.domain, '') != '') = ?", *filter.Domain) } if filter.CertificationType != nil { q = q.Where("node.certification ILIKE ?", *filter.CertificationType) @@ -559,67 +592,24 @@ func (d *PostgresDatabase) GetNodes(ctx context.Context, filter types.NodeFilter q = q.Where(`farm.dedicated_farm = ?`, *filter.InDedicatedFarm) } if filter.Dedicated != nil { - q = q.Where(`? = (farm.dedicated_farm = true OR COALESCE(node_contract.contract_id, 0) = 0 OR COALESCE(rent_contract.contract_id, 0) != 0)`, *filter.Dedicated) + q = q.Where(`? = (farm.dedicated_farm = true OR resources_cache.node_contracts_count = 0 OR resources_cache.renter is not null)`, *filter.Dedicated) } if filter.Rentable != nil { - q = q.Where(`? = ((farm.dedicated_farm = true OR COALESCE(node_contract.contract_id, 0) = 0) AND COALESCE(rent_contract.contract_id, 0) = 0)`, *filter.Rentable) + q = q.Where(`? = ((farm.dedicated_farm = true OR resources_cache.node_contracts_count = 0) AND resources_cache.renter is null)`, *filter.Rentable) } if filter.AvailableFor != nil { - q = q.Where(`COALESCE(rent_contract.twin_id, 0) = ? OR (COALESCE(rent_contract.twin_id, 0) = 0 AND farm.dedicated_farm = false)`, *filter.AvailableFor) + q = q.Where(`COALESCE(resources_cache.renter, 0) = ? OR (resources_cache.renter is null AND farm.dedicated_farm = false)`, *filter.AvailableFor) } if filter.RentedBy != nil { - q = q.Where(`COALESCE(rent_contract.twin_id, 0) = ?`, *filter.RentedBy) + q = q.Where(`COALESCE(resources_cache.renter, 0) = ?`, *filter.RentedBy) } if filter.Rented != nil { - q = q.Where(`? = (COALESCE(rent_contract.contract_id, 0) != 0)`, *filter.Rented) - } - if filter.CertificationType != nil { - q = q.Where("node.certification ILIKE ?", *filter.CertificationType) + q = q.Where(`? = (resources_cache.renter is not null)`, *filter.Rented) } if filter.OwnedBy != nil { q = q.Where(`COALESCE(farm.twin_id, 0) = ?`, *filter.OwnedBy) } - /* - used distinct selecting to avoid duplicated node after the join. - - postgres apply WHERE before DISTINCT so filters will still filter on the whole data. - - we don't return any gpu info on the node object so no worries of losing the data because DISTINCT. - */ - nodeGpuSubquery := d.gormDB.Table("node_gpu"). - Select("DISTINCT ON (node_twin_id) node_twin_id") - - if filter.HasGPU != nil { - nodeGpuSubquery = nodeGpuSubquery.Where("(COALESCE(node_gpu.id, '') != '') = ?", *filter.HasGPU) - } - - if filter.GpuDeviceName != nil { - nodeGpuSubquery = nodeGpuSubquery.Where("COALESCE(node_gpu.device, '') ILIKE '%' || ? || '%'", *filter.GpuDeviceName) - } - - if filter.GpuVendorName != nil { - nodeGpuSubquery = nodeGpuSubquery.Where("COALESCE(node_gpu.vendor, '') ILIKE '%' || ? || '%'", *filter.GpuVendorName) - } - - if filter.GpuVendorID != nil { - nodeGpuSubquery = nodeGpuSubquery.Where("COALESCE(node_gpu.id, '') ILIKE '%' || ? || '%'", *filter.GpuVendorID) - } - - if filter.GpuDeviceID != nil { - nodeGpuSubquery = nodeGpuSubquery.Where("COALESCE(node_gpu.id, '') ILIKE '%' || ? || '%'", *filter.GpuDeviceID) - } - - if filter.GpuAvailable != nil { - nodeGpuSubquery = nodeGpuSubquery.Where("(COALESCE(node_gpu.contract, 0) = 0) = ?", *filter.GpuAvailable) - } - - if filter.HasGPU != nil || filter.GpuDeviceName != nil || filter.GpuVendorName != nil || filter.GpuVendorID != nil || - filter.GpuDeviceID != nil || filter.GpuAvailable != nil { - - q.Joins( - `INNER JOIN (?) AS gpu ON gpu.node_twin_id = node.twin_id`, nodeGpuSubquery, - ) - } - var count int64 if limit.Randomize || limit.RetCount { q = q.Session(&gorm.Session{}) @@ -637,7 +627,7 @@ func (d *PostgresDatabase) GetNodes(ctx context.Context, filter types.NodeFilter q = q.Order("random()") } else { if filter.AvailableFor != nil { - q = q.Order("(case when rent_contract is not null then 1 else 2 end)") + q = q.Order("(case when resources_cache.renter is not null then 1 else 2 end)") } if limit.SortBy != "" { @@ -666,8 +656,8 @@ func (d *PostgresDatabase) GetNodes(ctx context.Context, filter types.NodeFilter } func (d *PostgresDatabase) shouldRetry(resError error) bool { - if resError != nil && resError.Error() == ErrNodeResourcesViewNotFound.Error() { - if err := d.initialize(); err != nil { + if resError != nil && resError.Error() == ErrResourcesCacheTableNotFound.Error() { + if err := d.Initialize(); err != nil { log.Logger.Err(err).Msg("failed to reinitialize database") } else { return true @@ -676,118 +666,6 @@ func (d *PostgresDatabase) shouldRetry(resError error) bool { return false } -// GetFarms return farms filtered and paginated -func (d *PostgresDatabase) GetFarms(ctx context.Context, filter types.FarmFilter, limit types.Limit) ([]Farm, uint, error) { - q := d.farmTableQuery() - q = q.WithContext(ctx) - if filter.NodeFreeMRU != nil { - q = q.Where("node.free_mru >= ?", *filter.NodeFreeMRU) - } - if filter.NodeFreeHRU != nil { - q = q.Where("node.free_hru >= ?", *filter.NodeFreeHRU) - } - if filter.NodeFreeSRU != nil { - q = q.Where("node.free_sru >= ?", *filter.NodeFreeSRU) - } - if filter.NodeTotalCRU != nil { - q = q.Where("node.total_cru >= ?", *filter.NodeTotalCRU) - } - - if filter.NodeAvailableFor != nil { - q = q.Where("node.renter = ? OR (node.renter = 0 AND farm.dedicated_farm = false)", *filter.NodeAvailableFor) - q = q.Order("CASE WHEN bool_or(node.rent_contract_id != 0) = TRUE THEN 1 ELSE 2 END") - } - - if filter.NodeHasGPU != nil { - q = q.Where("(node.gpu_id != '') = ?", *filter.NodeHasGPU) - } - - if filter.NodeRentedBy != nil { - q = q.Where("node.renter = ?", *filter.NodeRentedBy) - } - - if filter.Country != nil { - q = q.Where("LOWER(node.country) = LOWER(?)", *filter.Country) - } - - if filter.Region != nil { - q = q.Where("LOWER(country.subregion) = LOWER(?)", *filter.Region) - } - - if filter.NodeStatus != nil { - condition := nodestatus.DecideNodeStatusCondition(*filter.NodeStatus) - q = q.Where(condition) - } - - if filter.NodeCertified != nil { - q = q.Where("(node.certification = 'Certified') = ?", *filter.NodeCertified) - } - - if filter.FreeIPs != nil { - q = q.Where("public_ip_count.free_ips >= ?", *filter.FreeIPs) - } - if filter.TotalIPs != nil { - q = q.Where("public_ip_count.total_ips >= ?", *filter.TotalIPs) - } - if filter.StellarAddress != nil { - q = q.Where("farm.stellar_address = ?", *filter.StellarAddress) - } - if filter.PricingPolicyID != nil { - q = q.Where("farm.pricing_policy_id = ?", *filter.PricingPolicyID) - } - if filter.FarmID != nil { - q = q.Where("farm.farm_id = ?", *filter.FarmID) - } - if filter.TwinID != nil { - q = q.Where("farm.twin_id = ?", *filter.TwinID) - } - if filter.Name != nil { - q = q.Where("LOWER(farm.name) = LOWER(?)", *filter.Name) - } - - if filter.NameContains != nil { - escaped := strings.Replace(*filter.NameContains, "%", "\\%", -1) - escaped = strings.Replace(escaped, "_", "\\_", -1) - q = q.Where("farm.name ILIKE ?", fmt.Sprintf("%%%s%%", escaped)) - } - - if filter.CertificationType != nil { - q = q.Where("farm.certification = ?", *filter.CertificationType) - } - - if filter.Dedicated != nil { - q = q.Where("farm.dedicated_farm = ?", *filter.Dedicated) - } - var count int64 - if limit.Randomize || limit.RetCount { - if res := q.Count(&count); res.Error != nil { - return nil, 0, errors.Wrap(res.Error, "couldn't get farm count") - } - } - - // Sorting - if limit.Randomize { - q = q.Order("random()") - } else if limit.SortBy != "" { - order := types.SortOrderAsc - if strings.EqualFold(string(limit.SortOrder), string(types.SortOrderDesc)) { - order = types.SortOrderDesc - } - q = q.Order(fmt.Sprintf("%s %s", limit.SortBy, order)) - } else { - q = q.Order("farm.farm_id") - } - - // Pagination - q = q.Limit(int(limit.Size)).Offset(int(limit.Page-1) * int(limit.Size)) - - var farms []Farm - if res := q.Scan(&farms); res.Error != nil { - return farms, uint(count), errors.Wrap(res.Error, "failed to scan returned farm from database") - } - return farms, uint(count), nil -} - // GetTwins returns twins filtered and paginated func (d *PostgresDatabase) GetTwins(ctx context.Context, filter types.TwinFilter, limit types.Limit) ([]types.Twin, uint, error) { q := d.gormDB.WithContext(ctx). @@ -845,10 +723,10 @@ func (d *PostgresDatabase) contractTableQuery() *gorm.DB { contractTablesQuery := `( SELECT contract_id, twin_id, state, created_at, '' AS name, node_id, deployment_data, deployment_hash, number_of_public_i_ps, 'node' AS type FROM node_contract - UNION + UNION ALL SELECT contract_id, twin_id, state, created_at, '' AS name, node_id, '', '', 0, 'rent' AS type FROM rent_contract - UNION + UNION ALL SELECT contract_id, twin_id, state, created_at, name, 0, '', '', 0, 'name' AS type FROM name_contract ) contracts` diff --git a/grid-proxy/internal/explorer/db/setup.sql b/grid-proxy/internal/explorer/db/setup.sql new file mode 100644 index 000000000..4d259f7aa --- /dev/null +++ b/grid-proxy/internal/explorer/db/setup.sql @@ -0,0 +1,480 @@ +BEGIN; + +---- +-- Helper functions +---- +DROP FUNCTION IF EXISTS convert_to_decimal(v_input text); + +CREATE OR REPLACE FUNCTION convert_to_decimal(v_input TEXT) RETURNS DECIMAL AS +$$ +DECLARE v_dec_value DECIMAL DEFAULT NULL; +BEGIN + BEGIN + v_dec_value := v_input:: DECIMAL; + EXCEPTION + WHEN OTHERS THEN + RAISE NOTICE 'Invalid decimal value: "%". Returning NULL.', v_input; + RETURN NULL; + END; +RETURN v_dec_value; +END; +$$ LANGUAGE plpgsql; + + +---- +-- Clean old triggers +---- +DROP TRIGGER IF EXISTS node_added ON node; + +---- +-- Resources cache table +---- +DROP VIEW IF EXISTS resources_cache_view; + +CREATE OR REPLACE VIEW resources_cache_view AS +SELECT + node.node_id as node_id, + node.farm_id as farm_id, + COALESCE(node_resources_total.hru, 0) as total_hru, + COALESCE(node_resources_total.mru, 0) as total_mru, + COALESCE(node_resources_total.sru, 0) as total_sru, + COALESCE(node_resources_total.cru, 0) as total_cru, + COALESCE(node_resources_total.hru, 0) - COALESCE(sum(contract_resources.hru), 0) as free_hru, + COALESCE(node_resources_total.mru, 0) - COALESCE(sum(contract_resources.mru), 0) - GREATEST(CAST((node_resources_total.mru / 10) AS bigint), 2147483648) as free_mru, + COALESCE(node_resources_total.sru, 0) - COALESCE(sum(contract_resources.sru), 0) - 21474836480 as free_sru, + COALESCE(sum(contract_resources.hru), 0) as used_hru, + COALESCE(sum(contract_resources.mru), 0) + GREATEST(CAST( (node_resources_total.mru / 10) AS bigint), 2147483648 ) as used_mru, + COALESCE(sum(contract_resources.sru), 0) + 21474836480 as used_sru, + COALESCE(sum(contract_resources.cru), 0) as used_cru, + rent_contract.twin_id as renter, + rent_contract.contract_id as rent_contract_id, + count(node_contract.contract_id) as node_contracts_count, + COALESCE(node_gpu.node_gpu_count, 0) as node_gpu_count, + node.country as country, + country.subregion as region +FROM node + LEFT JOIN node_contract ON node.node_id = node_contract.node_id AND node_contract.state IN ('Created', 'GracePeriod') + LEFT JOIN contract_resources ON node_contract.resources_used_id = contract_resources.id + LEFT JOIN node_resources_total AS node_resources_total ON node_resources_total.node_id = node.id + LEFT JOIN rent_contract on node.node_id = rent_contract.node_id AND rent_contract.state IN ('Created', 'GracePeriod') + LEFT JOIN( + SELECT + node_twin_id, + COUNT(id) as node_gpu_count + FROM node_gpu + GROUP BY + node_twin_id + ) AS node_gpu ON node.twin_id = node_gpu.node_twin_id + LEFT JOIN country ON LOWER(node.country) = LOWER(country.name) +GROUP BY + node.node_id, + node_resources_total.mru, + node_resources_total.sru, + node_resources_total.hru, + node_resources_total.cru, + node.farm_id, + rent_contract.contract_id, + rent_contract.twin_id, + COALESCE(node_gpu.node_gpu_count, 0), + node.country, + country.subregion; + +DROP TABLE IF EXISTS resources_cache; +CREATE TABLE IF NOT EXISTS resources_cache( + node_id INTEGER PRIMARY KEY, + farm_id INTEGER NOT NULL, + total_hru NUMERIC NOT NULL, + total_mru NUMERIC NOT NULL, + total_sru NUMERIC NOT NULL, + total_cru NUMERIC NOT NULL, + free_hru NUMERIC NOT NULL, + free_mru NUMERIC NOT NULL, + free_sru NUMERIC NOT NULL, + used_hru NUMERIC NOT NULL, + used_mru NUMERIC NOT NULL, + used_sru NUMERIC NOT NULL, + used_cru NUMERIC NOT NULL, + renter INTEGER, + rent_contract_id INTEGER, + node_contracts_count INTEGER NOT NULL, + node_gpu_count INTEGER NOT NULL, + country TEXT, + region TEXT + ); + +INSERT INTO resources_cache +SELECT * +FROM resources_cache_view; + +---- +-- PublicIpsCache table +---- +DROP TABLE IF EXISTS public_ips_cache; +CREATE TABLE public_ips_cache( + farm_id INTEGER PRIMARY KEY, + free_ips INTEGER NOT NULL, + total_ips INTEGER NOT NULL, + ips jsonb +); + +INSERT INTO public_ips_cache + SELECT + farm.farm_id, + COALESCE(public_ip_agg.free_ips, 0), + COALESCE(public_ip_agg.total_ips, 0), + COALESCE(public_ip_agg.ips, '[]') +FROM farm + LEFT JOIN( + SELECT + p1.farm_id, + COUNT(p1.id) total_ips, + COUNT(CASE WHEN p2.contract_id = 0 THEN 1 END) free_ips, + jsonb_agg(jsonb_build_object('id', p1.id, 'ip', p1.ip, 'contract_id', p1.contract_id, 'gateway', p1.gateway)) as ips + FROM public_ip AS p1 + LEFT JOIN public_ip p2 ON p1.id = p2.id + GROUP BY + p1.farm_id + ) public_ip_agg on public_ip_agg.farm_id = farm.id; + +---- +-- Create Indices +---- +CREATE EXTENSION IF NOT EXISTS pg_trgm; + +CREATE EXTENSION IF NOT EXISTS btree_gin; + +CREATE INDEX IF NOT EXISTS idx_node_id ON public.node(node_id); +CREATE INDEX IF NOT EXISTS idx_twin_id ON public.twin(twin_id); +CREATE INDEX IF NOT EXISTS idx_farm_id ON public.farm(farm_id); +CREATE INDEX IF NOT EXISTS idx_node_contract_id ON public.node_contract USING gin(id); +CREATE INDEX IF NOT EXISTS idx_name_contract_id ON public.name_contract USING gin(id); +CREATE INDEX IF NOT EXISTS idx_rent_contract_id ON public.rent_contract USING gin(id); + + +CREATE INDEX IF NOT EXISTS idx_resources_cache_farm_id ON resources_cache (farm_id); +CREATE INDEX IF NOT EXISTS idx_resources_cache_node_id ON resources_cache(node_id); +CREATE INDEX IF NOT EXISTS idx_public_ips_cache_farm_id ON public_ips_cache(farm_id); + +CREATE INDEX IF NOT EXISTS idx_location_id ON location USING gin(id); +CREATE INDEX IF NOT EXISTS idx_public_config_node_id ON public_config USING gin(node_id); + +---- +--create triggers +---- + +/* + Node Trigger: + - Insert node record > Insert new resources_cache record + - Update node country > update resources_cache country/region +*/ +CREATE OR REPLACE FUNCTION reflect_node_changes() RETURNS TRIGGER AS +$$ +BEGIN + IF (TG_OP = 'UPDATE') THEN + BEGIN + UPDATE resources_cache + SET + country = NEW.country, + region = ( + SELECT subregion FROM country WHERE LOWER(country.name) = LOWER(NEW.country) + ) + WHERE + resources_cache.node_id = NEW.node_id; + EXCEPTION + WHEN OTHERS THEN + RAISE NOTICE 'Error updating resources_cache: %', SQLERRM; + END; + + ELSIF (TG_OP = 'INSERT') THEN + BEGIN + INSERT INTO resources_cache + SELECT * + FROM resources_cache_view + WHERE resources_cache_view.node_id = NEW.node_id; + EXCEPTION + WHEN OTHERS THEN + RAISE NOTICE 'Error inserting resources_cache: %', SQLERRM; + END; + + ELSIF (TG_OP = 'DELETE') THEN + BEGIN + DELETE FROM resources_cache WHERE node_id = OLD.node_id; + EXCEPTION + WHEN OTHERS THEN + RAISE NOTICE 'Error deleting node from resources_cache: %', SQLERRM; + END; + END IF; + + RETURN NULL; +END; +$$ LANGUAGE plpgsql; + +CREATE OR REPLACE TRIGGER tg_node + AFTER INSERT OR DELETE OR UPDATE OF country + ON node + FOR EACH ROW EXECUTE PROCEDURE reflect_node_changes(); + +/* + Total resources trigger + - Insert/Update node_resources_total > Update equivalent resources_cache record. + */ +CREATE OR REPLACE FUNCTION reflect_total_resources_changes() RETURNS TRIGGER AS +$$ +BEGIN + BEGIN + UPDATE resources_cache + SET + total_cru = NEW.cru, + total_mru = NEW.mru, + total_sru = NEW.sru, + total_hru = NEW.hru, + free_mru = free_mru + GREATEST(CAST((OLD.mru / 10) AS bigint), 2147483648) - + GREATEST(CAST((NEW.mru / 10) AS bigint), 2147483648) + (NEW.mru-COALESCE(OLD.mru, 0)), + free_hru = free_hru + (NEW.hru-COALESCE(OLD.hru, 0)), + free_sru = free_sru + (NEW.sru-COALESCE(OLD.sru, 0)), + used_mru = used_mru - GREATEST(CAST((OLD.mru / 10) AS bigint), 2147483648) + + GREATEST(CAST((NEW.mru / 10) AS bigint), 2147483648) + WHERE + resources_cache.node_id = ( + SELECT node.node_id FROM node WHERE node.id = New.node_id + ); + EXCEPTION + WHEN OTHERS THEN + RAISE NOTICE 'Error reflecting total_resources changes %', SQLERRM; + END; +RETURN NULL; +END; +$$ LANGUAGE plpgsql; + +CREATE OR REPLACE TRIGGER tg_node_resources_total + AFTER INSERT OR UPDATE + ON node_resources_total FOR EACH ROW + EXECUTE PROCEDURE reflect_total_resources_changes(); + + +/* + Contract resources + - Insert/Update contract_resources report > update resources_cache used/free fields + */ + +CREATE OR REPLACE FUNCTION reflect_contract_resources_changes() RETURNS TRIGGER AS +$$ +BEGIN + BEGIN + UPDATE resources_cache + SET used_cru = used_cru + (NEW.cru - COALESCE(OLD.cru, 0)), + used_mru = used_mru + (NEW.mru - COALESCE(OLD.mru, 0)), + used_sru = used_sru + (NEW.sru - COALESCE(OLD.sru, 0)), + used_hru = used_hru + (NEW.hru - COALESCE(OLD.hru, 0)), + free_mru = free_mru - (NEW.mru - COALESCE(OLD.mru, 0)), + free_hru = free_hru - (NEW.hru - COALESCE(OLD.hru, 0)), + free_sru = free_sru - (NEW.sru - COALESCE(OLD.sru, 0)) + WHERE + -- (SELECT state from node_contract where id = NEW.contract_id) != 'Deleted' AND + resources_cache.node_id = ( + SELECT node_id FROM node_contract WHERE node_contract.id = NEW.contract_id + ); + EXCEPTION + WHEN OTHERS THEN + RAISE NOTICE 'Error reflecting contract_resources changes %', SQLERRM; + END; +RETURN NULL; +END; +$$ LANGUAGE plpgsql; + +CREATE OR REPLACE TRIGGER tg_contract_resources + AFTER INSERT OR UPDATE ON contract_resources FOR EACH ROW + EXECUTE PROCEDURE reflect_contract_resources_changes(); + +/* + Node contract trigger + - Insert new contract > increment resources_cache node_contracts_count + - Update contract state to 'Deleted' > decrement used an increment free fields on resources_cache +*/ +CREATE OR REPLACE FUNCTION reflect_node_contract_changes() RETURNS TRIGGER AS +$$ +BEGIN + IF (TG_OP = 'UPDATE' AND NEW.state = 'Deleted') THEN + BEGIN + UPDATE resources_cache + SET (used_cru, used_mru, used_sru, used_hru, free_mru, free_sru, free_hru, node_contracts_count) = + ( + SELECT + resources_cache.used_cru - cru, + resources_cache.used_mru - mru, + resources_cache.used_sru - sru, + resources_cache.used_hru - hru, + resources_cache.free_mru + mru, + resources_cache.free_sru + sru, + resources_cache.free_hru + hru, + resources_cache.node_contracts_count - 1 + FROM resources_cache + LEFT JOIN contract_resources ON contract_resources.contract_id = NEW.id + WHERE resources_cache.node_id = NEW.node_id + ) WHERE resources_cache.node_id = NEW.node_id; + EXCEPTION + WHEN OTHERS THEN + RAISE NOTICE 'Error reflecting node_contract updates %', SQLERRM; + END; + + ELSIF (TG_OP = 'INSERT') THEN + BEGIN + UPDATE resources_cache + SET node_contracts_count = node_contracts_count + 1 + WHERE resources_cache.node_id = NEW.node_id; + EXCEPTION + WHEN OTHERS THEN + RAISE NOTICE 'Error incrementing node_contracts_count %', SQLERRM; + END; + END IF; +RETURN NULL; +END; +$$ LANGUAGE plpgsql; + +CREATE OR REPLACE TRIGGER tg_node_contract + AFTER INSERT OR UPDATE OF state ON node_contract FOR EACH ROW + EXECUTE PROCEDURE reflect_node_contract_changes(); + +/* + Rent contract trigger + - Insert new rent contract > Update resources_cache renter/rent_contract_id + - Update (state to 'Deleted') > nullify resources_cache renter/rent_contract_id +*/ + +CREATE OR REPLACE FUNCTION reflect_rent_contract_changes() RETURNS TRIGGER AS +$$ +BEGIN + IF (TG_OP = 'UPDATE' AND NEW.state = 'Deleted') THEN + BEGIN + UPDATE resources_cache + SET renter = NULL, + rent_contract_id = NULL + WHERE + resources_cache.node_id = NEW.node_id; + EXCEPTION + WHEN OTHERS THEN + RAISE NOTICE 'Error removing resources_cache rent fields %', SQLERRM; + END; + ELSIF (TG_OP = 'INSERT') THEN + BEGIN + UPDATE resources_cache + SET renter = NEW.twin_id, + rent_contract_id = NEW.contract_id + WHERE + resources_cache.node_id = NEW.node_id; + EXCEPTION + WHEN OTHERS THEN + RAISE NOTICE 'Error reflecting rent_contract changes %', SQLERRM; + END; + END IF; +RETURN NULL; +END; +$$ LANGUAGE plpgsql; + +CREATE OR REPLACE TRIGGER tg_rent_contract + AFTER INSERT OR UPDATE OF state ON rent_contract FOR EACH ROW + EXECUTE PROCEDURE reflect_rent_contract_changes(); + +/* + Public ips trigger + - Insert new ip > increment free/total ips + re-aggregate ips object + - Deleted > decrement total, decrement free ips (if it was used) + re-aggregate ips object + - Update > increment/decrement free ips based on usage + re-aggregate ips object +*/ +CREATE OR REPLACE FUNCTION reflect_public_ip_changes() RETURNS TRIGGER AS +$$ +BEGIN + + BEGIN + UPDATE public_ips_cache + SET free_ips = free_ips + ( + CASE + -- handles insertion/update by freeing ip + WHEN TG_OP != 'DELETE' AND NEW.contract_id = 0 + THEN 1 + -- handles deletion/update by reserving ip + WHEN TG_OP != 'INSERT' AND OLD.contract_id = 0 + THEN -1 + -- handles delete reserved ips + ELSE 0 + END + ), + + total_ips = total_ips + ( + CASE + WHEN TG_OP = 'INSERT' + THEN 1 + WHEn TG_OP = 'DELETE' + THEN -1 + ELSE 0 + END + ), + + ips = ( + SELECT jsonb_agg( + jsonb_build_object( + 'id', + public_ip.id, + 'ip', + public_ip.ip, + 'contract_id', + public_ip.contract_id, + 'gateway', + public_ip.gateway + ) + ) + -- old/new farm_id are the same + from public_ip where farm_id = COALESCE(NEW.farm_id, OLD.farm_id) + ) + WHERE + public_ips_cache.farm_id = ( + SELECT farm_id FROM farm WHERE farm.id = COALESCE(NEW.farm_id, OLD.farm_id) + ); + EXCEPTION + WHEN OTHERS THEN + RAISE NOTICE 'Error reflect public_ips changes %s', SQLERRM; + END; + +RETURN NULL; +END; +$$ LANGUAGE plpgsql; + +CREATE OR REPLACE TRIGGER tg_public_ip + AFTER INSERT OR DELETE OR UPDATE OF contract_id ON public_ip FOR EACH ROW + EXECUTE PROCEDURE reflect_public_ip_changes(); + + +CREATE OR REPLACE FUNCTION reflect_farm_changes() RETURNS TRIGGER AS +$$ +BEGIN + IF TG_OP = 'INSERT' THEN + BEGIN + INSERT INTO public_ips_cache VALUES( + NEW.farm_id, + 0, + 0, + '[]' + ); + EXCEPTION + WHEN OTHERS THEN + RAISE NOTICE 'Error inserting public_ips_cache record %s', SQLERRM; + END; + + ELSIF (TG_OP = 'DELETE') THEN + BEGIN + DELETE FROM public_ips_cache WHERE public_ips_cache.farm_id = OLD.farm_id; + EXCEPTION + WHEN OTHERS THEN + RAISE NOTICE 'Error deleting public_ips_cache record %s', SQLERRM; + END; + END IF; + +RETURN NULL; +END; +$$ LANGUAGE plpgsql; + +CREATE OR REPLACE TRIGGER tg_farm + AFTER INSERT OR DELETE ON farm FOR EACH ROW + EXECUTE PROCEDURE reflect_farm_changes(); + +COMMIT; \ No newline at end of file diff --git a/grid-proxy/internal/explorer/db/types.go b/grid-proxy/internal/explorer/db/types.go index a2404a7cb..4dfb529ba 100644 --- a/grid-proxy/internal/explorer/db/types.go +++ b/grid-proxy/internal/explorer/db/types.go @@ -41,41 +41,41 @@ type DBContract struct { // Node data about a node which is calculated from the chain type Node struct { - ID string - NodeID int64 - FarmID int64 - TwinID int64 - Country string - GridVersion int64 - City string - Uptime int64 - Created int64 - FarmingPolicyID int64 - UpdatedAt int64 - TotalCru int64 - TotalMru int64 - TotalSru int64 - TotalHru int64 - UsedCru int64 - UsedMru int64 - UsedSru int64 - UsedHru int64 - Domain string - Gw4 string - Gw6 string - Ipv4 string - Ipv6 string - Certification string - FarmDedicated bool `gorm:"farm_dedicated"` - RentContractID int64 - RentedByTwinID int64 - SerialNumber string - Longitude *float64 - Latitude *float64 - Power NodePower `gorm:"type:jsonb"` - NumGPU int `gorm:"num_gpu"` - ExtraFee uint64 - HasNodeContract bool `gorm:"has_node_contract"` + ID string + NodeID int64 + FarmID int64 + TwinID int64 + Country string + GridVersion int64 + City string + Uptime int64 + Created int64 + FarmingPolicyID int64 + UpdatedAt int64 + TotalCru int64 + TotalMru int64 + TotalSru int64 + TotalHru int64 + UsedCru int64 + UsedMru int64 + UsedSru int64 + UsedHru int64 + Domain string + Gw4 string + Gw6 string + Ipv4 string + Ipv6 string + Certification string + FarmDedicated bool `gorm:"farm_dedicated"` + RentContractID int64 + Renter int64 + SerialNumber string + Longitude *float64 + Latitude *float64 + Power NodePower `gorm:"type:jsonb"` + NumGPU int `gorm:"num_gpu"` + ExtraFee uint64 + NodeContractsCount uint64 `gorm:"node_contracts_count"` } // NodePower struct is the farmerbot report for node status diff --git a/grid-proxy/pkg/nodestatus/nodestatus.go b/grid-proxy/pkg/nodestatus/nodestatus.go index 695eec9e4..d567de2c2 100644 --- a/grid-proxy/pkg/nodestatus/nodestatus.go +++ b/grid-proxy/pkg/nodestatus/nodestatus.go @@ -18,7 +18,7 @@ const ( func DecideNodeStatusCondition(status string) string { condition := "TRUE" - nilPower := "node.power IS NULL" + nilPower := "node.power->> 'state' = '' AND node.power->> 'target' = ''" poweredOn := "node.power->> 'state' = 'Up' AND node.power->> 'target' = 'Up'" poweredOff := "node.power->> 'state' = 'Down' AND node.power->> 'target' = 'Down'" diff --git a/grid-proxy/tests/queries/bench_test.go b/grid-proxy/tests/queries/bench_test.go new file mode 100644 index 000000000..e5c431099 --- /dev/null +++ b/grid-proxy/tests/queries/bench_test.go @@ -0,0 +1,81 @@ +package test + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + proxytypes "github.com/threefoldtech/tfgrid-sdk-go/grid-proxy/pkg/types" +) + +func BenchmarkNodes(b *testing.B) { + for i := 0; i < b.N; i++ { + b.StopTimer() + agg := calcNodesAggregates(&data) + l := proxytypes.Limit{ + Size: 999999999999, + Page: 1, + RetCount: true, + } + f, err := randomNodeFilter(&agg) + require.NoError(b, err) + + b.StartTimer() + _, _, err = DBClient.GetNodes(context.Background(), f, l) + require.NoError(b, err) + } +} + +func BenchmarkFarms(b *testing.B) { + for i := 0; i < b.N; i++ { + b.StopTimer() + agg := calcFarmsAggregates(&data) + l := proxytypes.Limit{ + Size: 999999999999, + Page: 1, + RetCount: true, + } + f, err := randomFarmsFilter(&agg) + require.NoError(b, err) + + b.StartTimer() + _, _, err = DBClient.GetFarms(context.Background(), f, l) + require.NoError(b, err) + } +} + +func BenchmarkContracts(b *testing.B) { + for i := 0; i < b.N; i++ { + b.StopTimer() + agg := calcContractsAggregates(&data) + l := proxytypes.Limit{ + Size: 999999999999, + Page: 1, + RetCount: true, + } + f, err := randomContractsFilter(&agg) + require.NoError(b, err) + + b.StartTimer() + _, _, err = DBClient.GetContracts(context.Background(), f, l) + require.NoError(b, err) + } +} + +func BenchmarkTwins(b *testing.B) { + for i := 0; i < b.N; i++ { + b.StopTimer() + agg := calcTwinsAggregates(&data) + l := proxytypes.Limit{ + Size: 999999999999, + Page: 1, + RetCount: true, + } + f, err := randomTwinsFilter(&agg) + require.NoError(b, err) + + b.StartTimer() + _, _, err = DBClient.GetTwins(context.Background(), f, l) + require.NoError(b, err) + } +} diff --git a/grid-proxy/tests/queries/conn_test.go b/grid-proxy/tests/queries/conn_test.go index 629fb473a..2197fa6e2 100644 --- a/grid-proxy/tests/queries/conn_test.go +++ b/grid-proxy/tests/queries/conn_test.go @@ -9,14 +9,10 @@ import ( "github.com/rs/zerolog/log" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - db "github.com/threefoldtech/tfgrid-sdk-go/grid-proxy/internal/explorer/db" "github.com/threefoldtech/tfgrid-sdk-go/grid-proxy/pkg/types" ) func TestDBManyOpenConnections(t *testing.T) { - p, err := db.NewPostgresDatabase(POSTGRES_HOST, POSTGRES_PORT, POSTGRES_USER, POSTGRES_PASSSWORD, POSTGRES_DB, 80) - require.NoError(t, err) gotQueriesCnt := atomic.Int32{} wg := sync.WaitGroup{} @@ -29,7 +25,7 @@ func TestDBManyOpenConnections(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), time.Second*30) defer cancel() - _, _, err := p.GetTwins(ctx, types.TwinFilter{}, types.Limit{Size: 100}) + _, _, err := DBClient.GetTwins(ctx, types.TwinFilter{}, types.Limit{Size: 100}) if err != nil { log.Err(err).Msg("twin query failed") return diff --git a/grid-proxy/tests/queries/contract_test.go b/grid-proxy/tests/queries/contract_test.go index 4a2b153da..546451f0e 100644 --- a/grid-proxy/tests/queries/contract_test.go +++ b/grid-proxy/tests/queries/contract_test.go @@ -54,28 +54,43 @@ var contractFilterRandomValueGenerator = map[string]func(agg ContractsAggregate) return &agg.States[rand.Intn(len(agg.States))] }, "Name": func(agg ContractsAggregate) interface{} { - return &agg.Names[rand.Intn(len(agg.Names))] + c := agg.Names[rand.Intn(len(agg.Names))] + if len(c) == 0 { + return nil + } + return &c }, "NumberOfPublicIps": func(agg ContractsAggregate) interface{} { return rndref(0, agg.maxNumberOfPublicIPs) }, "DeploymentData": func(agg ContractsAggregate) interface{} { - return &agg.DeploymentsData[rand.Intn(len(agg.DeploymentsData))] + c := agg.DeploymentsData[rand.Intn(len(agg.DeploymentsData))] + if len(c) == 0 { + return nil + } + return &c }, "DeploymentHash": func(agg ContractsAggregate) interface{} { - return &agg.DeploymentHashes[rand.Intn(len(agg.DeploymentHashes))] + c := agg.DeploymentHashes[rand.Intn(len(agg.DeploymentHashes))] + if len(c) == 0 { + return nil + } + return &c }, } func TestContracts(t *testing.T) { + t.Parallel() t.Run("contracts pagination test", func(t *testing.T) { + t.Parallel() + node := "node" f := proxytypes.ContractFilter{ Type: &node, } l := proxytypes.Limit{ - Size: 5, + Size: 100, Page: 1, RetCount: true, } @@ -99,6 +114,8 @@ func TestContracts(t *testing.T) { }) t.Run("contracts stress test", func(t *testing.T) { + t.Parallel() + agg := calcContractsAggregates(&data) for i := 0; i < CONTRACTS_TESTS; i++ { l := proxytypes.Limit{ @@ -124,19 +141,28 @@ func TestContracts(t *testing.T) { } func TestContract(t *testing.T) { + t.Parallel() t.Run("single contract test", func(t *testing.T) { + t.Parallel() + contractID := rand.Intn(CONTRACTS_TESTS) - want, err := mockClient.Contract(context.Background(), uint32(contractID)) - require.NoError(t, err) + want, errGot := mockClient.Contract(context.Background(), uint32(contractID)) - got, err := gridProxyClient.Contract(context.Background(), uint32(contractID)) - require.NoError(t, err) + got, errWant := gridProxyClient.Contract(context.Background(), uint32(contractID)) + + if errGot != nil && errWant != nil { + require.True(t, errors.As(errWant, &errGot)) + } else { + require.True(t, errWant == errGot) + } require.True(t, reflect.DeepEqual(want, got), fmt.Sprintf("wanted: %+v\n got: %+v", want, got)) }) t.Run("contract not found test", func(t *testing.T) { + t.Parallel() + contractID := 1000000000000 _, err := gridProxyClient.Contract(context.Background(), uint32(contractID)) assert.Equal(t, err.Error(), ErrContractNotFound.Error()) @@ -145,6 +171,8 @@ func TestContract(t *testing.T) { func TestBills(t *testing.T) { t.Run("contract bills test", func(t *testing.T) { + t.Parallel() + contractID := rand.Intn(CONTRACTS_TESTS) l := proxytypes.Limit{ @@ -174,6 +202,8 @@ func TestBills(t *testing.T) { // TestContractsFilter iterates over all ContractFilter fields, and for each one generates a random value, then runs a test between the mock client and the gridproxy client func TestContractsFilter(t *testing.T) { + t.Parallel() + f := proxytypes.ContractFilter{} fp := &f v := reflect.ValueOf(fp).Elem() @@ -190,6 +220,10 @@ func TestContractsFilter(t *testing.T) { require.True(t, ok, "Filter field %s has no random value generator", v.Type().Field(i).Name) randomFieldValue := generator(agg) + if randomFieldValue == nil { + continue + } + if v.Field(i).Type().Kind() != reflect.Slice { v.Field(i).Set(reflect.New(v.Field(i).Type().Elem())) } @@ -281,6 +315,10 @@ func randomContractsFilter(agg *ContractsAggregate) (proxytypes.ContractFilter, } randomFieldValue := contractFilterRandomValueGenerator[v.Type().Field(i).Name](*agg) + if randomFieldValue == nil { + continue + } + if v.Field(i).Type().Kind() != reflect.Slice { v.Field(i).Set(reflect.New(v.Field(i).Type().Elem())) } diff --git a/grid-proxy/tests/queries/counters_test.go b/grid-proxy/tests/queries/counters_test.go index b09b06f46..8e3686ce1 100644 --- a/grid-proxy/tests/queries/counters_test.go +++ b/grid-proxy/tests/queries/counters_test.go @@ -21,6 +21,7 @@ var statsFilterRandomValues = map[string]func() interface{}{ } func TestStats(t *testing.T) { + t.Parallel() t.Run("stats up test", func(t *testing.T) { f := proxytypes.StatsFilter{ Status: &STATUS_UP, @@ -48,6 +49,8 @@ func TestStats(t *testing.T) { } func TestStatsFilter(t *testing.T) { + t.Parallel() + f := proxytypes.StatsFilter{} fp := &f v := reflect.ValueOf(fp).Elem() diff --git a/grid-proxy/tests/queries/farm_test.go b/grid-proxy/tests/queries/farm_test.go index 587dc138e..7f91f90d2 100644 --- a/grid-proxy/tests/queries/farm_test.go +++ b/grid-proxy/tests/queries/farm_test.go @@ -27,7 +27,12 @@ var farmFilterRandomValueGenerator = map[string]func(agg FarmsAggregate) interfa return rndref(0, agg.maxTotalIPs) }, "StellarAddress": func(agg FarmsAggregate) interface{} { - return &agg.stellarAddresses[rand.Intn(len(agg.stellarAddresses))] + c := agg.stellarAddresses[rand.Intn(len(agg.stellarAddresses))] + if len(c) == 0 { + return nil + } + + return &c }, "PricingPolicyID": func(agg FarmsAggregate) interface{} { return &agg.pricingPolicyIDs[rand.Intn(len(agg.pricingPolicyIDs))] @@ -40,24 +45,42 @@ var farmFilterRandomValueGenerator = map[string]func(agg FarmsAggregate) interfa }, "Name": func(agg FarmsAggregate) interface{} { name := changeCase(agg.farmNames[rand.Intn(len(agg.farmNames))]) + if len(name) == 0 { + return nil + } + return &name }, "Country": func(_ FarmsAggregate) interface{} { aggNode := calcNodesAggregates(&data) country := changeCase(aggNode.countries[rand.Intn(len(aggNode.countries))]) + if len(country) == 0 { + return nil + } + return &country }, "Region": func(agg FarmsAggregate) interface{} { - region := changeCase(agg.regions[rand.Intn(len(agg.regions))]) - return ®ion + c := changeCase(agg.regions[rand.Intn(len(agg.regions))]) + if len(c) == 0 { + return nil + } + + return &c }, "NameContains": func(agg FarmsAggregate) interface{} { c := agg.farmNames[rand.Intn(len(agg.farmNames))] - a, b := rand.Intn(len(c)), rand.Intn(len(c)) + runesList := []rune(c) + a, b := rand.Intn(len(runesList)), rand.Intn(len(runesList)) if a > b { a, b = b, a } - c = c[a : b+1] + runesList = runesList[a : b+1] + c = string(runesList) + if len(c) == 0 { + return nil + } + return &c }, "CertificationType": func(agg FarmsAggregate) interface{} { @@ -131,13 +154,16 @@ type FarmsAggregate struct { } func TestFarm(t *testing.T) { + t.Parallel() t.Run("farms pagination test", func(t *testing.T) { + t.Parallel() + one := uint64(1) f := proxytypes.FarmFilter{ TotalIPs: &one, } l := proxytypes.Limit{ - Size: 5, + Size: 100, Page: 1, RetCount: true, } @@ -160,6 +186,8 @@ func TestFarm(t *testing.T) { }) t.Run("farms stress test", func(t *testing.T) { + t.Parallel() + agg := calcFarmsAggregates(&data) for i := 0; i < FARM_TESTS; i++ { l := proxytypes.Limit{ @@ -183,6 +211,8 @@ func TestFarm(t *testing.T) { } }) t.Run("farms list node free hru", func(t *testing.T) { + t.Parallel() + aggNode := calcNodesAggregates(&data) l := proxytypes.Limit{ Size: 999999999999, @@ -205,6 +235,8 @@ func TestFarm(t *testing.T) { require.True(t, reflect.DeepEqual(want, got), fmt.Sprintf("Used Filter:\n%s", SerializeFilter(f)), fmt.Sprintf("Difference:\n%s", cmp.Diff(want, got))) }) t.Run("farms list node free hru, mru", func(t *testing.T) { + t.Parallel() + aggNode := calcNodesAggregates(&data) l := proxytypes.Limit{ @@ -233,6 +265,8 @@ func TestFarm(t *testing.T) { // TestFarmFilter iterates over all FarmFilter fields, and for each one generates a random value, then runs a test between the mock client and the gridproxy client func TestFarmFilter(t *testing.T) { + t.Parallel() + f := proxytypes.FarmFilter{} fp := &f v := reflect.ValueOf(fp).Elem() @@ -249,6 +283,10 @@ func TestFarmFilter(t *testing.T) { require.True(t, ok, "Filter field %s has no random value generator", v.Type().Field(i).Name) randomFieldValue := generator(agg) + if randomFieldValue == nil { + continue + } + if v.Field(i).Type().Kind() != reflect.Slice { v.Field(i).Set(reflect.New(v.Field(i).Type().Elem())) } @@ -337,6 +375,10 @@ func randomFarmsFilter(agg *FarmsAggregate) (proxytypes.FarmFilter, error) { } randomFieldValue := generate(*agg) + if randomFieldValue == nil { + continue + } + if v.Field(i).Type().Kind() != reflect.Slice { v.Field(i).Set(reflect.New(v.Field(i).Type().Elem())) } diff --git a/grid-proxy/tests/queries/main_test.go b/grid-proxy/tests/queries/main_test.go index 9d3af7036..6e6e4fb13 100644 --- a/grid-proxy/tests/queries/main_test.go +++ b/grid-proxy/tests/queries/main_test.go @@ -12,8 +12,12 @@ import ( _ "github.com/lib/pq" "github.com/pkg/errors" + "github.com/threefoldtech/tfgrid-sdk-go/grid-proxy/internal/explorer/db" + proxyDB "github.com/threefoldtech/tfgrid-sdk-go/grid-proxy/internal/explorer/db" proxyclient "github.com/threefoldtech/tfgrid-sdk-go/grid-proxy/pkg/client" mock "github.com/threefoldtech/tfgrid-sdk-go/grid-proxy/tests/queries/mock_client" + "github.com/threefoldtech/tfgrid-sdk-go/grid-proxy/tools/db/crafter" + "gorm.io/gorm/logger" ) var ( @@ -30,6 +34,7 @@ var ( mockClient proxyclient.Client data mock.DBData gridProxyClient proxyclient.Client + DBClient db.Database ) func parseCmdline() { @@ -44,6 +49,8 @@ func parseCmdline() { } func TestMain(m *testing.M) { + var exitCode int + parseCmdline() if SEED != 0 { rand.New(rand.NewSource(int64(SEED))) @@ -58,14 +65,126 @@ func TestMain(m *testing.M) { } defer db.Close() + // proxy client + gridProxyClient = proxyclient.NewClient(ENDPOINT) + + // mock client + dbClient, err := proxyDB.NewPostgresDatabase(POSTGRES_HOST, POSTGRES_PORT, POSTGRES_USER, POSTGRES_PASSSWORD, POSTGRES_DB, 80, logger.Error) + if err != nil { + panic(err) + } + DBClient = &dbClient + + // load mock client data, err = mock.Load(db) if err != nil { panic(err) } + // mockClient = mock.NewGridProxyMockClient(data) + err = modifyDataToFireTriggers(db, data) + if err != nil { + panic(err) + } + data, err = mock.Load(db) + if err != nil { + panic(err) + } mockClient = mock.NewGridProxyMockClient(data) - gridProxyClient = proxyclient.NewClient(ENDPOINT) - exitcode := m.Run() - os.Exit(exitcode) + exitCode = m.Run() + os.Exit(exitCode) +} + +func modifyDataToFireTriggers(db *sql.DB, data mock.DBData) error { + twinStart := len(data.Twins) + 1 + farmStart := len(data.Farms) + 1 + nodeStart := len(data.Nodes) + 1 + contractStart := len(data.NodeContracts) + len(data.RentContracts) + len(data.NameContracts) + 1 + billStart := data.BillReports + 1 + publicIpStart := len(data.PublicIPs) + 1 + + const ( + NodeCount = 600 + FarmCount = 100 + TwinCount = 600 + 100 + 600 // nodes + farms + normal users + PublicIPCount = 10 + NodeContractCount = 50 + NameContractCount = 10 + RentContractCount = 1 + ) + + generator := crafter.NewCrafter(db, + SEED, + NodeCount, + FarmCount, + TwinCount, + PublicIPCount, + NodeContractCount, + NameContractCount, + RentContractCount, + uint(nodeStart), + uint(farmStart), + uint(twinStart), + uint(contractStart), + uint(billStart), + uint(publicIpStart)) + + // insertion + if err := generator.GenerateTwins(); err != nil { + return fmt.Errorf("failed to generate twins: %w", err) + } + + if err := generator.GenerateFarms(); err != nil { + return fmt.Errorf("failed to generate farms: %w", err) + } + + if err := generator.GenerateNodes(); err != nil { + return fmt.Errorf("failed to generate nodes: %w", err) + } + + // rentCount is 1 because the generate method have .1 percent of 10 farms to be dedicated + if err := generator.GenerateContracts(); err != nil { + return fmt.Errorf("failed to generate contracts: %w", err) + } + + if err := generator.GeneratePublicIPs(); err != nil { + return fmt.Errorf("failed to generate public ips: %w", err) + } + + // updates + if err := generator.UpdateNodeCountry(); err != nil { + return fmt.Errorf("failed to update node country: %w", err) + } + + if err := generator.UpdateNodeTotalResources(); err != nil { + return fmt.Errorf("failed to update node total resources: %w", err) + } + + if err := generator.UpdateContractResources(); err != nil { + return fmt.Errorf("failed to update contract resources: %w", err) + } + + if err := generator.UpdateNodeContractState(); err != nil { + return fmt.Errorf("failed to update node node contract: %w", err) + } + + if err := generator.UpdateRentContract(); err != nil { + return fmt.Errorf("failed to update rent contract: %w", err) + } + + if err := generator.UpdatePublicIps(); err != nil { + return fmt.Errorf("failed to update public ips: %w", err) + } + + // // deletions + if err := generator.DeleteNodes(); err != nil { + return fmt.Errorf("failed to delete node: %w", err) + } + + if err := generator.DeletePublicIps(); err != nil { + return fmt.Errorf("failed to delete node: %w", err) + } + + return nil } diff --git a/grid-proxy/tests/queries/mock_client/contracts.go b/grid-proxy/tests/queries/mock_client/contracts.go index 2cbd47fd5..27432230f 100644 --- a/grid-proxy/tests/queries/mock_client/contracts.go +++ b/grid-proxy/tests/queries/mock_client/contracts.go @@ -2,6 +2,7 @@ package mock import ( "context" + "fmt" "sort" "github.com/threefoldtech/tfgrid-sdk-go/grid-proxy/pkg/types" @@ -245,6 +246,10 @@ func (g *GridProxyMockClient) Contract(ctx context.Context, contractID uint32) ( }, err } + if !ok { + return res, fmt.Errorf("contract not found") + } + return res, err } diff --git a/grid-proxy/tests/queries/mock_client/farms.go b/grid-proxy/tests/queries/mock_client/farms.go index 323ad2e0b..5409c5a2c 100644 --- a/grid-proxy/tests/queries/mock_client/farms.go +++ b/grid-proxy/tests/queries/mock_client/farms.go @@ -32,13 +32,17 @@ func (g *GridProxyMockClient) Farms(ctx context.Context, filter types.FarmFilter for _, farm := range g.data.Farms { if farm.satisfies(filter, &g.data) { + ips := []types.PublicIP{} + if pubIPs, ok := publicIPs[farm.FarmID]; ok { + ips = pubIPs + } res = append(res, types.Farm{ Name: farm.Name, FarmID: int(farm.FarmID), TwinID: int(farm.TwinID), PricingPolicyID: int(farm.PricingPolicyID), StellarAddress: farm.StellarAddress, - PublicIps: publicIPs[farm.FarmID], + PublicIps: ips, Dedicated: farm.DedicatedFarm, CertificationType: farm.Certification, }) @@ -47,8 +51,8 @@ func (g *GridProxyMockClient) Farms(ctx context.Context, filter types.FarmFilter if filter.NodeAvailableFor != nil { sort.Slice(res, func(i, j int) bool { - f1 := g.data.FarmHasRentedNode[uint64(res[i].FarmID)] - f2 := g.data.FarmHasRentedNode[uint64(res[j].FarmID)] + f1 := g.data.FarmHasRentedNode[uint64(res[i].FarmID)][*filter.NodeAvailableFor] + f2 := g.data.FarmHasRentedNode[uint64(res[j].FarmID)][*filter.NodeAvailableFor] lessFarmID := res[i].FarmID < res[j].FarmID return f1 && !f2 || f1 && f2 && lessFarmID || !f1 && !f2 && lessFarmID @@ -105,8 +109,15 @@ func (f *Farm) satisfies(filter types.FarmFilter, data *DBData) bool { return false } - if !f.satisfyFarmNodesFilter(data, filter) { - return false + if filter.NodeAvailableFor != nil || filter.NodeCertified != nil || + filter.NodeFreeHRU != nil || filter.NodeFreeMRU != nil || + filter.NodeFreeSRU != nil || filter.NodeHasGPU != nil || + filter.NodeRentedBy != nil || filter.NodeStatus != nil || + filter.Country != nil || filter.Region != nil || + filter.NodeTotalCRU != nil { + if !f.satisfyFarmNodesFilter(data, filter) { + return false + } } return true @@ -120,16 +131,16 @@ func (f *Farm) satisfyFarmNodesFilter(data *DBData, filter types.FarmFilter) boo total := data.NodeTotalResources[node.NodeID] used := data.NodeUsedResources[node.NodeID] - free := calcFreeResources(total, used) - if filter.NodeFreeHRU != nil && free.HRU < *filter.NodeFreeHRU { + free := CalcFreeResources(total, used) + if filter.NodeFreeHRU != nil && int64(free.HRU) < int64(*filter.NodeFreeHRU) { continue } - if filter.NodeFreeMRU != nil && free.MRU < *filter.NodeFreeMRU { + if filter.NodeFreeMRU != nil && int64(free.MRU) < int64(*filter.NodeFreeMRU) { continue } - if filter.NodeFreeSRU != nil && free.SRU < *filter.NodeFreeSRU { + if filter.NodeFreeSRU != nil && int64(free.SRU) < int64(*filter.NodeFreeSRU) { continue } @@ -167,7 +178,7 @@ func (f *Farm) satisfyFarmNodesFilter(data *DBData, filter types.FarmFilter) boo continue } - if filter.Region != nil && !strings.EqualFold(*filter.Region, data.Regions[node.Country]) { + if filter.Region != nil && !strings.EqualFold(*filter.Region, data.Regions[strings.ToLower(node.Country)]) { continue } diff --git a/grid-proxy/tests/queries/mock_client/loader.go b/grid-proxy/tests/queries/mock_client/loader.go index 04184d25f..6cf87ae67 100644 --- a/grid-proxy/tests/queries/mock_client/loader.go +++ b/grid-proxy/tests/queries/mock_client/loader.go @@ -3,6 +3,7 @@ package mock import ( "database/sql" "math" + "strings" "github.com/threefoldtech/zos/pkg/gridtypes" ) @@ -17,7 +18,7 @@ type DBData struct { NodeUsedResources map[uint64]NodeResourcesTotal NodeRentedBy map[uint64]uint64 NodeRentContractID map[uint64]uint64 - FarmHasRentedNode map[uint64]bool + FarmHasRentedNode map[uint64]map[uint64]bool Nodes map[uint64]Node NodeTotalResources map[uint64]NodeResourcesTotal @@ -29,10 +30,12 @@ type DBData struct { RentContracts map[uint64]RentContract NameContracts map[uint64]NameContract Billings map[uint64][]ContractBillReport + BillReports uint32 ContractResources map[string]ContractResources NonDeletedContracts map[uint64][]uint64 GPUs map[uint64][]NodeGPU Regions map[string]string + Locations map[string]Location DB *sql.DB } @@ -132,7 +135,7 @@ func calcRentInfo(data *DBData) error { data.NodeRentedBy[contract.NodeID] = contract.TwinID data.NodeRentContractID[contract.NodeID] = contract.ContractID farmID := data.Nodes[contract.NodeID].FarmID - data.FarmHasRentedNode[farmID] = true + data.FarmHasRentedNode[farmID][contract.TwinID] = true } return nil } @@ -212,6 +215,7 @@ func loadFarms(db *sql.DB, data *DBData) error { } data.Farms[farm.FarmID] = farm data.FarmIDMap[farm.ID] = farm.FarmID + data.FarmHasRentedNode[farm.FarmID] = map[uint64]bool{} } return nil } @@ -475,6 +479,7 @@ func loadContractBillingReports(db *sql.DB, data *DBData) error { return err } data.Billings[contractBillReport.ContractID] = append(data.Billings[contractBillReport.ContractID], contractBillReport) + data.BillReports++ } return nil } @@ -511,11 +516,42 @@ func loadCountries(db *sql.DB, data *DBData) error { ); err != nil { return err } - data.Regions[country.Name] = country.Subregion + data.Regions[strings.ToLower(country.Name)] = country.Subregion } return nil } + +func loadLocations(db *sql.DB, data *DBData) error { + rows, err := db.Query(` + SELECT + COALESCE(id, ''), + CASE WHEN longitude = '' THEN NULL ELSE longitude END, + CASE WHEN latitude = '' THEN NULL ELSE latitude END + FROM + location; + `) + + if err != nil { + return err + } + + for rows.Next() { + var location Location + if err := rows.Scan( + &location.ID, + &location.Longitude, + &location.Latitude, + ); err != nil { + return err + } + + data.Locations[location.ID] = location + } + + return nil +} + func loadNodeGPUs(db *sql.DB, data *DBData) error { rows, err := db.Query(` SELECT @@ -562,13 +598,15 @@ func Load(db *sql.DB) (DBData, error) { NodeRentedBy: make(map[uint64]uint64), NodeRentContractID: make(map[uint64]uint64), Billings: make(map[uint64][]ContractBillReport), + BillReports: 0, ContractResources: make(map[string]ContractResources), NodeTotalResources: make(map[uint64]NodeResourcesTotal), NodeUsedResources: make(map[uint64]NodeResourcesTotal), NonDeletedContracts: make(map[uint64][]uint64), GPUs: make(map[uint64][]NodeGPU), - FarmHasRentedNode: make(map[uint64]bool), + FarmHasRentedNode: make(map[uint64]map[uint64]bool), Regions: make(map[string]string), + Locations: make(map[string]Location), DB: db, } if err := loadNodes(db, &data); err != nil { @@ -610,6 +648,9 @@ func Load(db *sql.DB) (DBData, error) { if err := loadCountries(db, &data); err != nil { return data, err } + if err := loadLocations(db, &data); err != nil { + return data, err + } if err := calcNodesUsedResources(&data); err != nil { return data, err } diff --git a/grid-proxy/tests/queries/mock_client/nodes.go b/grid-proxy/tests/queries/mock_client/nodes.go index 82a403771..c7d064333 100644 --- a/grid-proxy/tests/queries/mock_client/nodes.go +++ b/grid-proxy/tests/queries/mock_client/nodes.go @@ -2,6 +2,7 @@ package mock import ( "context" + "fmt" "sort" "strings" @@ -59,8 +60,10 @@ func (g *GridProxyMockClient) Nodes(ctx context.Context, filter types.NodeFilter SRU: gridtypes.Unit(g.data.NodeUsedResources[node.NodeID].SRU), }, Location: types.Location{ - Country: node.Country, - City: node.City, + Country: node.Country, + City: node.City, + Longitude: g.data.Locations[node.LocationID].Longitude, + Latitude: g.data.Locations[node.LocationID].Latitude, }, PublicConfig: types.PublicConfig{ Domain: g.data.PublicConfigs[node.NodeID].Domain, @@ -81,7 +84,8 @@ func (g *GridProxyMockClient) Nodes(ctx context.Context, filter types.NodeFilter State: node.Power.State, Target: node.Power.Target, }, - NumGPU: numGPU, + NumGPU: numGPU, + ExtraFee: node.ExtraFee, }) } } @@ -103,7 +107,11 @@ func (g *GridProxyMockClient) Nodes(ctx context.Context, filter types.NodeFilter } func (g *GridProxyMockClient) Node(ctx context.Context, nodeID uint32) (res types.NodeWithNestedCapacity, err error) { - node := g.data.Nodes[uint64(nodeID)] + node, ok := g.data.Nodes[uint64(nodeID)] + if !ok { + return res, fmt.Errorf("node not found") + } + numGPU := len(g.data.GPUs[node.TwinID]) nodePower := types.NodePower{ @@ -137,8 +145,10 @@ func (g *GridProxyMockClient) Node(ctx context.Context, nodeID uint32) (res type }, }, Location: types.Location{ - Country: node.Country, - City: node.City, + Country: node.Country, + City: node.City, + Longitude: g.data.Locations[node.LocationID].Longitude, + Latitude: g.data.Locations[node.LocationID].Latitude, }, PublicConfig: types.PublicConfig{ Domain: g.data.PublicConfigs[node.NodeID].Domain, @@ -166,7 +176,11 @@ func (g *GridProxyMockClient) Node(ctx context.Context, nodeID uint32) (res type } func (g *GridProxyMockClient) NodeStatus(ctx context.Context, nodeID uint32) (res types.NodeStatus, err error) { - node := g.data.Nodes[uint64(nodeID)] + node, ok := g.data.Nodes[uint64(nodeID)] + if !ok { + return res, fmt.Errorf("node not found") + } + nodePower := types.NodePower{ State: node.Power.State, Target: node.Power.Target, @@ -183,21 +197,21 @@ func (n *Node) satisfies(f types.NodeFilter, data *DBData) bool { total := data.NodeTotalResources[n.NodeID] used := data.NodeUsedResources[n.NodeID] - free := calcFreeResources(total, used) + free := CalcFreeResources(total, used) if f.Status != nil && *f.Status != nodestatus.DecideNodeStatus(nodePower, int64(n.UpdatedAt)) { return false } - if f.FreeMRU != nil && *f.FreeMRU > free.MRU { + if f.FreeMRU != nil && int64(*f.FreeMRU) > int64(free.MRU) { return false } - if f.FreeHRU != nil && *f.FreeHRU > free.HRU { + if f.FreeHRU != nil && int64(*f.FreeHRU) > int64(free.HRU) { return false } - if f.FreeSRU != nil && *f.FreeSRU > free.SRU { + if f.FreeSRU != nil && int64(*f.FreeSRU) > int64(free.SRU) { return false } @@ -221,7 +235,7 @@ func (n *Node) satisfies(f types.NodeFilter, data *DBData) bool { return false } - if f.Region != nil && !strings.EqualFold(*f.Region, data.Regions[n.Country]) { + if f.Region != nil && !strings.EqualFold(*f.Region, data.Regions[strings.ToLower(n.Country)]) { return false } diff --git a/grid-proxy/tests/queries/mock_client/types.go b/grid-proxy/tests/queries/mock_client/types.go index e673d1215..606c67968 100644 --- a/grid-proxy/tests/queries/mock_client/types.go +++ b/grid-proxy/tests/queries/mock_client/types.go @@ -49,6 +49,7 @@ type Node struct { Power NodePower `gorm:"type:jsonb"` HasGPU bool ExtraFee uint64 + Dedicated bool } type NodePower struct { @@ -158,3 +159,9 @@ type Country struct { Lat string Long string } + +type Location struct { + ID string + Longitude *float64 + Latitude *float64 +} diff --git a/grid-proxy/tests/queries/mock_client/utils.go b/grid-proxy/tests/queries/mock_client/utils.go index aff1efa51..411f1a367 100644 --- a/grid-proxy/tests/queries/mock_client/utils.go +++ b/grid-proxy/tests/queries/mock_client/utils.go @@ -10,16 +10,7 @@ type Result interface { types.Contract | types.Farm | types.Node | types.Twin } -func calcFreeResources(total NodeResourcesTotal, used NodeResourcesTotal) NodeResourcesTotal { - if total.MRU < used.MRU { - panic("total mru is less than mru") - } - if total.HRU < used.HRU { - panic("total hru is less than hru") - } - if total.SRU < used.SRU { - panic("total sru is less than sru") - } +func CalcFreeResources(total NodeResourcesTotal, used NodeResourcesTotal) NodeResourcesTotal { return NodeResourcesTotal{ HRU: total.HRU - used.HRU, SRU: total.SRU - used.SRU, diff --git a/grid-proxy/tests/queries/node_test.go b/grid-proxy/tests/queries/node_test.go index 86f63a57e..11084a467 100644 --- a/grid-proxy/tests/queries/node_test.go +++ b/grid-proxy/tests/queries/node_test.go @@ -102,45 +102,83 @@ var nodeFilterRandomValueGenerator = map[string]func(agg NodesAggregate) interfa }, "Country": func(agg NodesAggregate) interface{} { country := changeCase(agg.countries[rand.Intn(len(agg.countries))]) + if len(country) == 0 { + return nil + } + return &country }, "Region": func(agg NodesAggregate) interface{} { region := changeCase(agg.regions[rand.Intn(len(agg.regions))]) + if len(region) == 0 { + return nil + } + return ®ion }, "CountryContains": func(agg NodesAggregate) interface{} { c := agg.countries[rand.Intn(len(agg.countries))] - a, b := rand.Intn(len(c)), rand.Intn(len(c)) + if len(c) == 0 { + return nil + } + + runesList := []rune(c) + a, b := rand.Intn(len(runesList)), rand.Intn(len(runesList)) if a > b { a, b = b, a } - c = c[a : b+1] + runesList = runesList[a : b+1] + c = string(runesList) + if len(c) == 0 { + return nil + } + return &c }, "City": func(agg NodesAggregate) interface{} { city := changeCase(agg.cities[rand.Intn(len(agg.cities))]) + if len(city) == 0 { + return nil + } + return &city }, "CityContains": func(agg NodesAggregate) interface{} { c := agg.cities[rand.Intn(len(agg.cities))] - a, b := rand.Intn(len(c)), rand.Intn(len(c)) + if len(c) == 0 { + return nil + } + + runesList := []rune(c) + a, b := rand.Intn(len(runesList)), rand.Intn(len(runesList)) if a > b { a, b = b, a } - c = c[a : b+1] + runesList = runesList[a : b+1] + c = string(runesList) return &c }, "FarmName": func(agg NodesAggregate) interface{} { name := changeCase(agg.farmNames[rand.Intn(len(agg.farmNames))]) + if len(name) == 0 { + return nil + } + return &name }, "FarmNameContains": func(agg NodesAggregate) interface{} { c := agg.farmNames[rand.Intn(len(agg.farmNames))] - a, b := rand.Intn(len(c)), rand.Intn(len(c)) + if len(c) == 0 { + return nil + } + + runesList := []rune(c) + a, b := rand.Intn(len(runesList)), rand.Intn(len(runesList)) if a > b { a, b = b, a } - c = c[a : b+1] + runesList = runesList[a : b+1] + c = string(runesList) return &c }, "FarmIDs": func(agg NodesAggregate) interface{} { @@ -270,6 +308,7 @@ var nodeFilterRandomValueGenerator = map[string]func(agg NodesAggregate) interfa } func TestNode(t *testing.T) { + t.Parallel() t.Run("node pagination test", func(t *testing.T) { nodePaginationCheck(t, mockClient, gridProxyClient) }) @@ -279,6 +318,8 @@ func TestNode(t *testing.T) { }) t.Run("node up test", func(t *testing.T) { + t.Parallel() + f := types.NodeFilter{ Status: &STATUS_UP, } @@ -301,13 +342,18 @@ func TestNode(t *testing.T) { }) t.Run("node status test", func(t *testing.T) { + t.Parallel() + for i := 1; i <= NODE_COUNT; i++ { if flip(.3) { - want, err := mockClient.NodeStatus(context.Background(), uint32(i)) - require.NoError(t, err) + want, errWant := mockClient.NodeStatus(context.Background(), uint32(i)) + got, errGot := gridProxyClient.NodeStatus(context.Background(), uint32(i)) - got, err := gridProxyClient.NodeStatus(context.Background(), uint32(i)) - require.NoError(t, err) + if errGot != nil && errWant != nil { + require.True(t, errors.As(errWant, &errGot), fmt.Sprintf("errors should match: want error %s, got error %s", errWant, errGot)) + } else { + require.True(t, errWant == errGot) + } require.True(t, reflect.DeepEqual(want, got), fmt.Sprintf("Difference:\n%s", cmp.Diff(want, got))) } @@ -315,6 +361,8 @@ func TestNode(t *testing.T) { }) t.Run("node stress test", func(t *testing.T) { + t.Parallel() + agg := calcNodesAggregates(&data) for i := 0; i < NODE_TESTS; i++ { l := types.Limit{ @@ -338,26 +386,16 @@ func TestNode(t *testing.T) { }) t.Run("node not found test", func(t *testing.T) { + t.Parallel() + nodeID := 1000000000 _, err := gridProxyClient.Node(context.Background(), uint32(nodeID)) assert.Equal(t, err.Error(), ErrNodeNotFound.Error()) }) - t.Run("nodes test without resources view", func(t *testing.T) { - db := data.DB - _, err := db.Exec("drop view nodes_resources_view ;") - assert.NoError(t, err) - - singleNodeCheck(t, mockClient, gridProxyClient) - assert.NoError(t, err) - - _, err = db.Exec("drop view nodes_resources_view ;") - assert.NoError(t, err) - - nodePaginationCheck(t, mockClient, gridProxyClient) - }) - t.Run("nodes test certification_type filter", func(t *testing.T) { + t.Parallel() + certType := "Diy" nodes, _, err := gridProxyClient.Nodes(context.Background(), types.NodeFilter{CertificationType: &certType}, types.DefaultLimit()) require.NoError(t, err) @@ -373,6 +411,8 @@ func TestNode(t *testing.T) { }) t.Run("nodes test has_gpu filter", func(t *testing.T) { + t.Parallel() + l := proxytypes.DefaultLimit() hasGPU := true f := proxytypes.NodeFilter{ @@ -389,6 +429,8 @@ func TestNode(t *testing.T) { }) t.Run("nodes test gpu vendor, device name filter", func(t *testing.T) { + t.Parallel() + device := "navi" vendor := "advanced" nodes, _, err := gridProxyClient.Nodes(context.Background(), types.NodeFilter{GpuDeviceName: &device, GpuVendorName: &vendor}, types.DefaultLimit()) @@ -401,6 +443,8 @@ func TestNode(t *testing.T) { }) t.Run("nodes test gpu vendor, device id filter", func(t *testing.T) { + t.Parallel() + device := "744c" vendor := "1002" nodes, _, err := gridProxyClient.Nodes(context.Background(), types.NodeFilter{GpuDeviceID: &device, GpuVendorID: &vendor}, types.DefaultLimit()) @@ -413,6 +457,8 @@ func TestNode(t *testing.T) { }) t.Run("nodes test gpu available", func(t *testing.T) { + t.Parallel() + available := false nodes, _, err := gridProxyClient.Nodes(context.Background(), types.NodeFilter{GpuAvailable: &available}, types.DefaultLimit()) assert.NoError(t, err) @@ -426,6 +472,8 @@ func TestNode(t *testing.T) { // TestNodeFilter iterates over all NodeFilter fields, and for each one generates a random value, then runs a test between the mock client and the gridproxy client func TestNodeFilter(t *testing.T) { + t.Parallel() + f := types.NodeFilter{} fp := &f v := reflect.ValueOf(fp).Elem() @@ -442,10 +490,14 @@ func TestNodeFilter(t *testing.T) { require.True(t, ok, "Filter field %s has no random value generator", v.Type().Field(i).Name) randomFieldValue := generator(agg) + if randomFieldValue == nil { + continue + } if v.Field(i).Type().Kind() != reflect.Slice { v.Field(i).Set(reflect.New(v.Field(i).Type().Elem())) } + v.Field(i).Set(reflect.ValueOf(randomFieldValue)) want, wantCount, err := mockClient.Nodes(context.Background(), f, l) @@ -463,12 +515,17 @@ func TestNodeFilter(t *testing.T) { } func singleNodeCheck(t *testing.T, localClient proxyclient.Client, proxyClient proxyclient.Client) { + t.Parallel() nodeID := rand.Intn(NODE_COUNT) - want, err := mockClient.Node(context.Background(), uint32(nodeID)) - require.NoError(t, err) + want, errWant := mockClient.Node(context.Background(), uint32(nodeID)) - got, err := gridProxyClient.Node(context.Background(), uint32(nodeID)) - require.NoError(t, err) + got, errGot := gridProxyClient.Node(context.Background(), uint32(nodeID)) + + if errGot != nil && errWant != nil { + require.True(t, errors.As(errWant, &errGot)) + } else { + require.True(t, errWant == errGot) + } require.True(t, reflect.DeepEqual(want, got), fmt.Sprintf("Difference:\n%s", cmp.Diff(want, got))) } @@ -478,7 +535,7 @@ func nodePaginationCheck(t *testing.T, localClient proxyclient.Client, proxyClie Status: &STATUS_DOWN, } l := types.Limit{ - Size: 5, + Size: 100, Page: 1, RetCount: true, } @@ -515,6 +572,10 @@ func randomNodeFilter(agg *NodesAggregate) (types.NodeFilter, error) { if v.Field(i).Type().Kind() != reflect.Slice { v.Field(i).Set(reflect.New(v.Field(i).Type().Elem())) } + if randomFieldValue == nil { + continue + } + v.Field(i).Set(reflect.ValueOf(randomFieldValue)) } } @@ -529,13 +590,27 @@ func calcNodesAggregates(data *mock.DBData) (res NodesAggregate) { cities[node.City] = struct{}{} countries[node.Country] = struct{}{} total := data.NodeTotalResources[node.NodeID] - free := calcFreeResources(total, data.NodeUsedResources[node.NodeID]) - res.maxFreeHRU = max(res.maxFreeHRU, free.HRU) - res.maxFreeSRU = max(res.maxFreeSRU, free.SRU) - res.maxFreeMRU = max(res.maxFreeMRU, free.MRU) - res.freeMRUs = append(res.freeMRUs, free.MRU) - res.freeSRUs = append(res.freeSRUs, free.SRU) - res.freeHRUs = append(res.freeHRUs, free.HRU) + free := mock.CalcFreeResources(total, data.NodeUsedResources[node.NodeID]) + freeHRU := free.HRU + if int64(freeHRU) < 0 { + freeHRU = 0 + } + freeMRU := free.MRU + if int64(freeMRU) < 0 { + freeMRU = 0 + } + + freeSRU := free.SRU + if int64(freeSRU) < 0 { + freeSRU = 0 + } + + res.maxFreeHRU = max(res.maxFreeHRU, freeHRU) + res.maxFreeSRU = max(res.maxFreeSRU, freeSRU) + res.maxFreeMRU = max(res.maxFreeMRU, freeMRU) + res.freeMRUs = append(res.freeMRUs, freeMRU) + res.freeSRUs = append(res.freeSRUs, freeSRU) + res.freeHRUs = append(res.freeHRUs, freeHRU) res.maxTotalMRU = max(res.maxTotalMRU, total.MRU) res.totalMRUs = append(res.totalMRUs, total.MRU) diff --git a/grid-proxy/tests/queries/twin_test.go b/grid-proxy/tests/queries/twin_test.go index a70dea93b..9420f503d 100644 --- a/grid-proxy/tests/queries/twin_test.go +++ b/grid-proxy/tests/queries/twin_test.go @@ -32,21 +32,37 @@ var twinFilterRandomValueGenerator = map[string]func(agg TwinsAggregate) interfa return &agg.twinIDs[rand.Intn(len(agg.twinIDs))] }, "AccountID": func(agg TwinsAggregate) interface{} { - return &agg.accountIDs[rand.Intn(len(agg.accountIDs))] + c := agg.accountIDs[rand.Intn(len(agg.accountIDs))] + if len(c) == 0 { + return nil + } + return &c }, "Relay": func(agg TwinsAggregate) interface{} { - return &agg.relays[rand.Intn(len(agg.relays))] + c := agg.relays[rand.Intn(len(agg.relays))] + if len(c) == 0 { + return nil + } + return &c }, "PublicKey": func(agg TwinsAggregate) interface{} { - return &agg.publicKeys[rand.Intn(len(agg.publicKeys))] + c := agg.publicKeys[rand.Intn(len(agg.publicKeys))] + if len(c) == 0 { + return nil + } + return &c }, } func TestTwins(t *testing.T) { + t.Parallel() + t.Run("twins pagination test", func(t *testing.T) { + t.Parallel() + f := proxytypes.TwinFilter{} l := proxytypes.Limit{ - Size: 5, + Size: 100, Page: 1, RetCount: true, } @@ -69,6 +85,8 @@ func TestTwins(t *testing.T) { }) t.Run("twins stress test", func(t *testing.T) { + t.Parallel() + agg := calcTwinsAggregates(&data) for i := 0; i < TWINS_TESTS; i++ { l := proxytypes.Limit{ @@ -94,6 +112,8 @@ func TestTwins(t *testing.T) { // TestTwinFilter iterates over all TwinFilter fields, and for each one generates a random value, then runs a test between the mock client and the gridproxy client func TestTwinFilter(t *testing.T) { + t.Parallel() + f := proxytypes.TwinFilter{} fp := &f v := reflect.ValueOf(fp).Elem() @@ -110,6 +130,9 @@ func TestTwinFilter(t *testing.T) { require.True(t, ok, "Filter field %s has no random value generator", v.Type().Field(i).Name) randomFieldValue := generator(agg) + if randomFieldValue == nil { + continue + } if v.Field(i).Type().Kind() != reflect.Slice { v.Field(i).Set(reflect.New(v.Field(i).Type().Elem())) @@ -143,6 +166,10 @@ func randomTwinsFilter(agg *TwinsAggregate) (proxytypes.TwinFilter, error) { } randomFieldValue := twinFilterRandomValueGenerator[v.Type().Field(i).Name](*agg) + if randomFieldValue == nil { + continue + } + if v.Field(i).Type().Kind() != reflect.Slice { v.Field(i).Set(reflect.New(v.Field(i).Type().Elem())) } diff --git a/grid-proxy/tests/queries/utils.go b/grid-proxy/tests/queries/utils.go index 9f87bb567..423d3079e 100644 --- a/grid-proxy/tests/queries/utils.go +++ b/grid-proxy/tests/queries/utils.go @@ -4,33 +4,15 @@ import ( "fmt" "math/rand" "reflect" - "strings" + "unicode" "github.com/threefoldtech/tfgrid-sdk-go/grid-proxy/pkg/types" - mock "github.com/threefoldtech/tfgrid-sdk-go/grid-proxy/tests/queries/mock_client" ) type Filter interface { types.ContractFilter | types.NodeFilter | types.FarmFilter | types.TwinFilter | types.StatsFilter } -func calcFreeResources(total mock.NodeResourcesTotal, used mock.NodeResourcesTotal) mock.NodeResourcesTotal { - if total.MRU < used.MRU { - panic("total mru is less than mru") - } - if total.HRU < used.HRU { - panic("total hru is less than hru") - } - if total.SRU < used.SRU { - panic("total sru is less than sru") - } - return mock.NodeResourcesTotal{ - HRU: total.HRU - used.HRU, - SRU: total.SRU - used.SRU, - MRU: total.MRU - used.MRU, - } -} - func flip(success float32) bool { return rand.Float32() < success } @@ -55,8 +37,14 @@ func min(a, b uint64) uint64 { } func changeCase(s string) string { - idx := rand.Intn(len(s)) - return strings.Replace(s, string(s[idx]), strings.ToUpper(string(s[idx])), 1) + if len(s) == 0 { + return s + } + + runesList := []rune(s) + idx := rand.Intn(len(runesList)) + runesList[idx] = unicode.ToUpper(runesList[idx]) + return string(runesList) } func SerializeFilter[F Filter](f F) string { diff --git a/grid-proxy/tools/db/README.md b/grid-proxy/tools/db/README.md index 59ef37586..570c686a1 100644 --- a/grid-proxy/tools/db/README.md +++ b/grid-proxy/tools/db/README.md @@ -1,30 +1,24 @@ -# DB for testing -## Run postgresql container - - ```bash - docker run --rm --name postgres \ - -e POSTGRES_USER=postgres \ - -e POSTGRES_PASSWORD=postgres \ - -e POSTGRES_DB=tfgrid-graphql \ - -p 5432:5432 -d postgres - ``` - -## Create the DB -you can either Generate a db with relevant schema to test things locally quickly, or load a previously taken DB dump file: - -### Method 1: Generate a db with relevant schema using the db helper tool: - - ```bash - cd tools/db/ && go run . \ - --postgres-host 127.0.0.1 \ - --postgres-db tfgrid-graphql \ - --postgres-password postgres \ - --postgres-user postgres \ - --reset \ - ``` - -### Method 2: Fill the DB from a Production db dump file, for example if you have `dump.sql` file, you can run: - - ```bash - psql -h 127.0.0.1 -U postgres -d tfgrid-graphql < dump.sql - ``` +# Data Preparation for Testing + +Starting the postgres container + +```bash +make db-start +``` + +Populate the database using one of the two available methods: +Generate a mock database: + +`./schema.sql` is a database schema taken from graphql processor on dev net, it creates all the needed tables for the database. + +the `data-crafter` have a various of methods to create/update/delete the main tables on the database. it can be customized by the params passed to the `NewCrafter`. + +```bash +make db-fill +``` + +Or use a data dump file (contains both schema and data) + +```bash +make db-dump p= +``` diff --git a/grid-proxy/tools/db/crafter/deleter.go b/grid-proxy/tools/db/crafter/deleter.go new file mode 100644 index 000000000..a813825b6 --- /dev/null +++ b/grid-proxy/tools/db/crafter/deleter.go @@ -0,0 +1,45 @@ +package crafter + +import ( + "fmt" +) + +// deletions +func (g *Crafter) DeleteNodes() error { + // delete node contracts on this node + // free public ips that are assigned to the deleted contracts + // delete rent contracts on this node + // delete node + deleteCount := r.Intn(10) + 1 + query := "" + + for i := 0; i < deleteCount; i++ { + nodeID := int(g.NodeCount) - i + + query += fmt.Sprintf("UPDATE public_ip SET contract_id = 0 WHERE contract_id IN (SELECT contract_id FROM node_contract WHERE node_id = %d);", nodeID) + query += fmt.Sprintf("UPDATE node_contract SET state = 'Deleted' WHERE node_id = %d;", nodeID) + query += fmt.Sprintf("UPDATE rent_contract set state = 'Deleted' WHERE node_id = %d;", nodeID) + query += fmt.Sprintf("DELETE FROM node_resources_total WHERE node_id = (SELECT id FROM node WHERE node_id = %d);", nodeID) + query += fmt.Sprintf("DELETE FROM public_config WHERE node_id = (SELECT id FROM node WHERE node_id = %d);", nodeID) + query += fmt.Sprintf("DELETE FROM node WHERE node_id = %d;", nodeID) + } + + fmt.Println("nodes deleted") + + _, err := g.db.Exec(query) + return err +} + +func (g *Crafter) DeletePublicIps() error { + maxDeleteCount := r.Intn(10) + 1 + query := fmt.Sprintf("DELETE FROM public_ip WHERE id in (SELECT id FROM public_ip WHERE contract_id = 0 LIMIT %d);", maxDeleteCount) + + _, err := g.db.Exec(query) + if err != nil { + return fmt.Errorf("failed to delete public ips: %w", err) + } + + fmt.Println("public ips deleted") + + return nil +} diff --git a/grid-proxy/tools/db/crafter/generator.go b/grid-proxy/tools/db/crafter/generator.go new file mode 100644 index 000000000..62985db63 --- /dev/null +++ b/grid-proxy/tools/db/crafter/generator.go @@ -0,0 +1,838 @@ +package crafter + +import ( + "fmt" + "strings" + "time" + + "github.com/google/uuid" + "github.com/threefoldtech/zos/pkg/gridtypes" +) + +const deleted = "Deleted" +const created = "Created" +const gracePeriod = "GracePeriod" + +func (c *Crafter) GenerateTwins() error { + start := c.TwinStart + end := c.TwinCount + c.TwinStart + + var twins []string + + for i := uint64(start); i < uint64(end); i++ { + twin := twin{ + id: fmt.Sprintf("twin-%d", i), + account_id: fmt.Sprintf("account-id-%d", i), + relay: fmt.Sprintf("relay-%d", i), + public_key: fmt.Sprintf("public-key-%d", i), + twin_id: i, + grid_version: 3, + } + tuple, err := objectToTupleString(twin) + if err != nil { + return fmt.Errorf("failed to convert twin object to tuple string: %w", err) + } + twins = append(twins, tuple) + } + + if err := c.insertTuples(twin{}, twins); err != nil { + return fmt.Errorf("failed to insert twins: %w", err) + } + fmt.Printf("twins generated [%d : %d[\n", start, end) + + return nil +} + +func (c *Crafter) GenerateFarms() error { + start := c.FarmStart + end := c.FarmCount + c.FarmStart + farmTwinsStart := c.TwinStart + c.FarmStart + + var farms []string + for i := uint64(start); i < uint64(end); i++ { + farm := farm{ + id: fmt.Sprintf("farm-%d", i), + farm_id: i, + name: fmt.Sprintf("farm-name-%d", i), + certification: "Diy", + dedicated_farm: flip(.1), + twin_id: uint64(farmTwinsStart) + i, + pricing_policy_id: 1, + grid_version: 3, + stellar_address: "", + } + + if farm.dedicated_farm { + c.dedicatedFarms[farm.farm_id] = struct{}{} + } + + farmTuple, err := objectToTupleString(farm) + if err != nil { + return fmt.Errorf("failed to convert farm object to tuple string: %w", err) + } + farms = append(farms, farmTuple) + } + + if err := c.insertTuples(farm{}, farms); err != nil { + return fmt.Errorf("failed to insert farms: %w", err) + } + fmt.Printf("farms generated [%d : %d[\n", start, end) + + return nil +} + +func (c *Crafter) GenerateNodes() error { + start := c.NodeStart + end := c.NodeStart + c.NodeCount + nodeTwinsStart := c.TwinStart + (c.FarmStart + c.FarmCount) + + powerState := []string{"Up", "Down"} + var locations []string + var nodes []string + var totalResources []string + var publicConfigs []string + for i := uint64(start); i < uint64(end); i++ { + mru, err := rnd(4, 256) + if err != nil { + return fmt.Errorf("failed to generate random mru: %w", err) + } + mru *= 1024 * 1024 * 1024 + + hru, err := rnd(100, 30*1024) + if err != nil { + return fmt.Errorf("failed to generate random hru: %w", err) + } + hru *= 1024 * 1024 * 1024 // 100GB -> 30TB + + sru, err := rnd(200, 30*1024) + if err != nil { + return fmt.Errorf("failed to generate random sru: %w", err) + } + sru *= 1024 * 1024 * 1024 // 100GB -> 30TB + + cru, err := rnd(4, 128) + if err != nil { + return fmt.Errorf("failed to generate random cru: %w", err) + } + + up := flip(nodeUpRatio) + periodFromLatestUpdate, err := rnd(60*40*3, 60*60*24*30*12) + if err != nil { + return fmt.Errorf("failed to generate random period from latest update: %w", err) + } + updatedAt := time.Now().Unix() - int64(periodFromLatestUpdate) + + if up { + periodFromLatestUpdate, err = rnd(0, 60*40*1) + if err != nil { + return fmt.Errorf("failed to generate period from latest update: %w", err) + } + updatedAt = time.Now().Unix() - int64(periodFromLatestUpdate) + } + + c.nodesMRU[i] = mru - max(2*uint64(gridtypes.Gigabyte), mru/10) + c.nodesSRU[i] = sru - 100*uint64(gridtypes.Gigabyte) + c.nodesHRU[i] = hru + c.nodeUP[i] = up + + // location latitude and longitue needs to be castable to decimal + // if not, the convert_to_decimal function will raise a notice + // reporting the incident, which downgrades performance + locationId := fmt.Sprintf("location-%d", uint64(start)+i) + location := location{ + id: locationId, + longitude: fmt.Sprintf("%d", i), + latitude: fmt.Sprintf("%d", i), + } + + countryIndex := r.Intn(len(countries)) + cityIndex := r.Intn(len(cities[countries[countryIndex]])) + farmId := r.Intn(int(c.FarmCount)) + int(c.FarmStart) + node := node{ + id: fmt.Sprintf("node-%d", i), + location_id: locationId, + node_id: i, + farm_id: uint64(farmId), + twin_id: uint64(nodeTwinsStart) + i, + country: countries[countryIndex], + city: cities[countries[countryIndex]][cityIndex], + uptime: 1000, + updated_at: uint64(updatedAt), + created: uint64(time.Now().Unix()), + created_at: uint64(time.Now().Unix()), + farming_policy_id: 1, + grid_version: 3, + certification: "Diy", + secure: false, + virtualized: false, + serial_number: "", + power: nodePower{ + State: powerState[r.Intn(len(powerState))], + Target: powerState[r.Intn(len(powerState))], + }, + extra_fee: 0, + dedicated: false, + } + + total_resources := node_resources_total{ + id: fmt.Sprintf("total-resources-%d", i), + hru: hru, + sru: sru, + cru: cru, + mru: mru, + node_id: fmt.Sprintf("node-%d", i), + } + + if _, ok := c.dedicatedFarms[node.farm_id]; ok { + c.availableRentNodes[i] = struct{}{} + c.availableRentNodesList = append(c.availableRentNodesList, i) + } + + locationTuple, err := objectToTupleString(location) + if err != nil { + return fmt.Errorf("failed to convert location object to tuple string: %w", err) + } + locations = append(locations, locationTuple) + + nodeTuple, err := objectToTupleString(node) + if err != nil { + return fmt.Errorf("failed to convert node object to tuple string: %w", err) + } + nodes = append(nodes, nodeTuple) + + totalResourcesTuple, err := objectToTupleString(total_resources) + if err != nil { + return fmt.Errorf("failed to convert total resources object to tuple string: %w", err) + } + totalResources = append(totalResources, totalResourcesTuple) + + if flip(.1) { + publicConfig := public_config{ + id: fmt.Sprintf("public-config-%d", i), + ipv4: "185.16.5.2/24", + gw4: "185.16.5.2", + ipv6: "::1/64", + gw6: "::1", + domain: "hamada.com", + node_id: fmt.Sprintf("node-%d", i), + } + publicConfigTuple, err := objectToTupleString(publicConfig) + if err != nil { + return fmt.Errorf("failed to convert public config object to tuple string: %w", err) + } + publicConfigs = append(publicConfigs, publicConfigTuple) + + } + } + + if err := c.insertTuples(location{}, locations); err != nil { + return fmt.Errorf("failed to insert locations: %w", err) + } + + if err := c.insertTuples(node{}, nodes); err != nil { + return fmt.Errorf("failed to isnert nodes: %w", err) + } + + if err := c.insertTuples(node_resources_total{}, totalResources); err != nil { + return fmt.Errorf("failed to insert node resources total: %w", err) + } + + if err := c.insertTuples(public_config{}, publicConfigs); err != nil { + return fmt.Errorf("failed to insert public configs: %w", err) + } + fmt.Printf("nodes generated [%d : %d[\n", start, end) + + return nil +} + +func (c *Crafter) GenerateContracts() error { + var billReports []string + + rentContractsBillReports, nodeContractIDStart, err := c.GenerateRentContracts(int(c.BillStart), int(c.ContractStart), int(c.RentContractCount)) + if err != nil { + return fmt.Errorf("failed to generate rent contracts: %w", err) + } + billReports = append(billReports, rentContractsBillReports...) + + nodeContractsBillReports, nameContractIDStart, err := c.generateNodeContracts(len(billReports)+int(c.BillStart), nodeContractIDStart, int(c.NodeContractCount), int(c.NodeStart), int(c.NodeCount)) + if err != nil { + return fmt.Errorf("failed to generate node contracts: %w", err) + } + billReports = append(billReports, nodeContractsBillReports...) + + nameContractsBillReports, _, err := c.GenerateNameContracts(len(billReports)+int(c.BillStart), nameContractIDStart, int(c.NameContractCount)) + if err != nil { + return fmt.Errorf("failed to generate name contracts: %w", err) + } + billReports = append(billReports, nameContractsBillReports...) + + if err := c.insertTuples(contract_bill_report{}, billReports); err != nil { + return fmt.Errorf("failed to generate contract bill reports: %w", err) + } + return nil +} + +func (c *Crafter) generateNodeContracts(billsStartID, contractsStartID, contractCount, nodeStart, nodeSize int) ([]string, int, error) { + end := contractsStartID + contractCount + start := contractsStartID + + var contracts []string + var contractResources []string + var billingReports []string + toDelete := "" + toGracePeriod := "" + contractToResource := map[uint64]string{} + for i := start; i < end; i++ { + nodeID := uint64(r.Intn(nodeSize) + nodeStart) + state := deleted + if c.nodeUP[nodeID] { + if flip(0.9) { + state = created + } else if flip(0.4) { + state = gracePeriod + } + } + + if state == deleted { + if len(toDelete) != 0 { + toDelete += ", " + } + toDelete += fmt.Sprint(i) + } + + if state == gracePeriod { + if len(toGracePeriod) != 0 { + toGracePeriod += ", " + } + toGracePeriod += fmt.Sprint(i) + } + + if state != deleted && (minContractHRU > c.nodesHRU[nodeID] || minContractMRU > c.nodesMRU[nodeID] || minContractSRU > c.nodesSRU[nodeID]) { + i-- + continue + } + + twinID, err := rnd(1100, 3100) + if err != nil { + return nil, i, fmt.Errorf("failed to generate random twin id: %w", err) + } + + if renter, ok := c.renter[nodeID]; ok { + twinID = renter + } + + if _, ok := c.availableRentNodes[nodeID]; ok { + i-- + continue + } + + contract := node_contract{ + id: fmt.Sprintf("node-contract-%d", i), + twin_id: twinID, + contract_id: uint64(i), + state: created, + created_at: uint64(time.Now().Unix()), + node_id: nodeID, + deployment_data: fmt.Sprintf("deployment-data-%d", i), + deployment_hash: fmt.Sprintf("deployment-hash-%d", i), + number_of_public_i_ps: 0, + grid_version: 3, + resources_used_id: "", + } + + cru, err := rnd(minContractCRU, maxContractCRU) + if err != nil { + return nil, i, fmt.Errorf("failed to generate random cru: %w", err) + } + + hru, err := rnd(minContractHRU, min(maxContractHRU, c.nodesHRU[nodeID])) + if err != nil { + return nil, i, fmt.Errorf("failed to generate random hru: %w", err) + } + + sru, err := rnd(minContractSRU, min(maxContractSRU, c.nodesSRU[nodeID])) + if err != nil { + return nil, i, fmt.Errorf("failed to generate random sru: %w", err) + } + + mru, err := rnd(minContractMRU, min(maxContractMRU, c.nodesMRU[nodeID])) + if err != nil { + return nil, i, fmt.Errorf("failed to generate random mru: %w", err) + } + + contract_resources := contract_resources{ + id: fmt.Sprintf("contract-resources-%d", i), + hru: hru, + sru: sru, + cru: cru, + mru: mru, + contract_id: fmt.Sprintf("node-contract-%d", i), + } + contractToResource[contract.contract_id] = contract_resources.id + + if state != deleted { + c.nodesHRU[nodeID] -= hru + c.nodesSRU[nodeID] -= sru + c.nodesMRU[nodeID] -= mru + c.createdNodeContracts = append(c.createdNodeContracts, uint64(i)) + } + + contractTuple, err := objectToTupleString(contract) + if err != nil { + return nil, i, fmt.Errorf("failed to convert contract object to tuple string: %w", err) + } + contracts = append(contracts, contractTuple) + + contractResourcesTuple, err := objectToTupleString(contract_resources) + if err != nil { + return nil, i, fmt.Errorf("failed to convert contract resources object to tuple string: %w", err) + } + contractResources = append(contractResources, contractResourcesTuple) + + billings, err := rnd(0, 10) + if err != nil { + return nil, i, fmt.Errorf("failed to generate random billing count: %w", err) + } + + amountBilled, err := rnd(0, 100000) + if err != nil { + return nil, i, fmt.Errorf("failed to generate random amount billed: %w", err) + } + for j := uint64(0); j < billings; j++ { + billing := contract_bill_report{ + id: fmt.Sprintf("contract-bill-report-%d", billsStartID), + contract_id: uint64(i), + discount_received: "Default", + amount_billed: amountBilled, + timestamp: uint64(time.Now().UnixNano()), + } + billsStartID++ + + billTuple, err := objectToTupleString(billing) + if err != nil { + return nil, i, fmt.Errorf("failed to convert contract bill report object to tuple string: %w", err) + } + billingReports = append(billingReports, billTuple) + } + } + + if err := c.insertTuples(node_contract{}, contracts); err != nil { + return nil, end, fmt.Errorf("failed to insert node contracts: %w", err) + } + + if err := c.insertTuples(contract_resources{}, contractResources); err != nil { + return nil, end, fmt.Errorf("failed to insert contract resources: %w", err) + } + + if err := c.updateNodeContractResourceID(contractToResource); err != nil { + return nil, end, fmt.Errorf("failed to update node contract resources id: %w", err) + } + + if len(toDelete) > 0 { + if _, err := c.db.Exec(fmt.Sprintf("UPDATE node_contract SET state = '%s' WHERE contract_id IN (%s)", deleted, toDelete)); err != nil { + return nil, 0, fmt.Errorf("failed to update node_contract state to deleted: %w", err) + } + } + + if len(toGracePeriod) > 0 { + if _, err := c.db.Exec(fmt.Sprintf("UPDATE node_contract SET state = '%s' WHERE contract_id IN (%s)", gracePeriod, toGracePeriod)); err != nil { + return nil, 0, fmt.Errorf("failed to update node_contract state to grace period: %w", err) + } + } + + fmt.Printf("node contracts generated [%d : %d[\n", start, end) + + return billingReports, end, nil +} + +func (c *Crafter) updateNodeContractResourceID(contractToResource map[uint64]string) error { + query := "" + for contractID, ResourceID := range contractToResource { + query += fmt.Sprintf("UPDATE node_contract SET resources_used_id = '%s' WHERE contract_id = %d;", ResourceID, contractID) + } + + if _, err := c.db.Exec(query); err != nil { + return fmt.Errorf("failed to update node contract resource id: %w", err) + } + return nil +} + +func (c *Crafter) GenerateNameContracts(billsStartID, contractsStartID, contractCount int) ([]string, int, error) { + end := contractsStartID + contractCount + start := contractsStartID + var contracts []string + var billReports []string + toDelete := "" + toGracePeriod := "" + + for i := start; i < end; i++ { + // WATCH: + nodeID, err := rnd(1, uint64(c.NodeCount)) + if err != nil { + return nil, i, fmt.Errorf("failed to generate random node id: %w", err) + } + + state := deleted + if c.nodeUP[nodeID] { + if flip(0.9) { + state = created + } else if flip(0.5) { + state = gracePeriod + } + } + + if state == deleted { + if len(toDelete) != 0 { + toDelete += ", " + } + toDelete += fmt.Sprint(i) + } + + if state == gracePeriod { + if len(toGracePeriod) != 0 { + toGracePeriod += ", " + } + toGracePeriod += fmt.Sprint(i) + } + + twinID, err := rnd(1100, 3100) + if err != nil { + return nil, i, fmt.Errorf("failed to generate random twin id: %w", err) + } + + if renter, ok := c.renter[nodeID]; ok { + twinID = renter + } + + if _, ok := c.availableRentNodes[nodeID]; ok { + i-- + continue + } + + contract := name_contract{ + id: fmt.Sprintf("name-contract-%d", i), + twin_id: twinID, + contract_id: uint64(i), + state: state, + created_at: uint64(time.Now().Unix()), + grid_version: 3, + name: uuid.NewString(), + } + + contractTuple, err := objectToTupleString(contract) + if err != nil { + return nil, i, fmt.Errorf("failed to convert contract object to tuple string: %w", err) + } + contracts = append(contracts, contractTuple) + + billings, err := rnd(0, 10) + if err != nil { + return nil, i, fmt.Errorf("failed to generate random billings count: %w", err) + } + amountBilled, err := rnd(0, 100000) + if err != nil { + return nil, i, fmt.Errorf("failed to generate random amount billed: %w", err) + } + + for j := uint64(0); j < billings; j++ { + billing := contract_bill_report{ + id: fmt.Sprintf("contract-bill-report-%d", billsStartID), + contract_id: uint64(i), + discount_received: "Default", + amount_billed: amountBilled, + timestamp: uint64(time.Now().UnixNano()), + } + billsStartID++ + + billTuple, err := objectToTupleString(billing) + if err != nil { + return nil, i, fmt.Errorf("failed to convert contract bill report object to tuple string: %w", err) + } + billReports = append(billReports, billTuple) + } + } + + if err := c.insertTuples(name_contract{}, contracts); err != nil { + return nil, end, fmt.Errorf("failed to insert name contracts: %w", err) + } + + if len(toDelete) > 0 { + if _, err := c.db.Exec(fmt.Sprintf("UPDATE rent_contract SET state = '%s' WHERE contract_id IN (%s)", deleted, toDelete)); err != nil { + return nil, 0, fmt.Errorf("failed to update rent_contract state to deleted: %w", err) + } + } + + if len(toGracePeriod) > 0 { + if _, err := c.db.Exec(fmt.Sprintf("UPDATE rent_contract SET state = '%s' WHERE contract_id IN (%s)", gracePeriod, toGracePeriod)); err != nil { + return nil, 0, fmt.Errorf("failed to update rent_contract state to grace period: %w", err) + } + } + + fmt.Printf("name contracts generated [%d : %d[\n", start, end) + + return billReports, end, nil +} + +func (c *Crafter) GenerateRentContracts(billsStart, contractStart, rentConCount int) ([]string, int, error) { + end := contractStart + rentConCount + start := contractStart + + var contracts []string + var billReports []string + toDelete := "" + toGracePeriod := "" + for i := start; i < end; i++ { + nl, nodeID, err := popRandom(c.availableRentNodesList) + if err != nil { + return nil, i, fmt.Errorf("failed to select random element from the given slice: %w", err) + } + + c.availableRentNodesList = nl + delete(c.availableRentNodes, nodeID) + state := deleted + if c.nodeUP[nodeID] { + if flip(0.9) { + state = created + } else if flip(0.5) { + state = gracePeriod + } + } + + if state == deleted { + if len(toDelete) != 0 { + toDelete += ", " + } + toDelete += fmt.Sprint(i) + } + + if state == gracePeriod { + if len(toGracePeriod) != 0 { + toGracePeriod += ", " + } + toGracePeriod += fmt.Sprint(i) + } + + twinID, err := rnd(1100, 3100) + if err != nil { + return nil, i, fmt.Errorf("failed to generate a random twin id: %w", err) + } + + contract := rent_contract{ + id: fmt.Sprintf("rent-contract-%d", i), + twin_id: twinID, + contract_id: uint64(i), + state: created, + created_at: uint64(time.Now().Unix()), + node_id: nodeID, + grid_version: 3, + } + + if state != deleted { + c.renter[nodeID] = contract.twin_id + } + + contractTuple, err := objectToTupleString(contract) + if err != nil { + return nil, i, fmt.Errorf("failed to convert contract object to tuple string: %w", err) + } + contracts = append(contracts, contractTuple) + + billings, err := rnd(0, 10) + if err != nil { + return nil, i, fmt.Errorf("failed to generate billings count: %w", err) + } + + amountBilled, err := rnd(0, 100000) + if err != nil { + return nil, i, fmt.Errorf("failed to generate amount billed: %w", err) + } + + for j := uint64(0); j < billings; j++ { + billing := contract_bill_report{ + id: fmt.Sprintf("contract-bill-report-%d", billsStart), + contract_id: uint64(i), + discount_received: "Default", + amount_billed: amountBilled, + timestamp: uint64(time.Now().UnixNano()), + } + + billsStart++ + + billTuple, err := objectToTupleString(billing) + if err != nil { + return nil, i, fmt.Errorf("failed to convert contract bill report object to tuple string: %w", err) + } + billReports = append(billReports, billTuple) + + } + } + + if err := c.insertTuples(rent_contract{}, contracts); err != nil { + return nil, end, fmt.Errorf("failed to insert rent contracts: %w", err) + } + + if len(toDelete) > 0 { + if _, err := c.db.Exec(fmt.Sprintf("UPDATE rent_contract SET state = '%s' WHERE contract_id IN (%s)", deleted, toDelete)); err != nil { + return nil, 0, fmt.Errorf("failed to update rent_contract state to deleted: %w", err) + } + } + + if len(toGracePeriod) > 0 { + if _, err := c.db.Exec(fmt.Sprintf("UPDATE rent_contract SET state = '%s' WHERE contract_id IN (%s)", gracePeriod, toGracePeriod)); err != nil { + return nil, 0, fmt.Errorf("failed to update rent_contract state to grace period: %w", err) + } + } + + fmt.Printf("rent contracts generated [%d : %d[\n", start, end) + + return billReports, end, nil +} + +func (c *Crafter) GeneratePublicIPs() error { + start := c.PublicIPStart + end := c.PublicIPCount + c.PublicIPStart + + var publicIPs []string + var nodeContracts []uint64 + reservedIPs := map[string]uint64{} + for i := uint64(start); i < uint64(end); i++ { + contract_id := uint64(0) + if flip(usedPublicIPsRatio) { + idx, err := rnd(0, uint64(len(c.createdNodeContracts))-1) + if err != nil { + return fmt.Errorf("failed to generate random index: %w", err) + } + contract_id = c.createdNodeContracts[idx] + } + + ip := randomIPv4() + + farmID := r.Int63n(int64(c.FarmCount)) + int64(c.FarmStart) + + public_ip := public_ip{ + id: fmt.Sprintf("public-ip-%d", i), + gateway: ip.String(), + ip: IPv4Subnet(ip).String(), + contract_id: 0, + farm_id: fmt.Sprintf("farm-%d", farmID), + } + + if contract_id != 0 { + reservedIPs[public_ip.id] = contract_id + } + + publicIpTuple, err := objectToTupleString(public_ip) + if err != nil { + return fmt.Errorf("failed to convert public ip object to tuple string: %w", err) + } + publicIPs = append(publicIPs, publicIpTuple) + nodeContracts = append(nodeContracts, contract_id) + } + + if err := c.insertTuples(public_ip{}, publicIPs); err != nil { + return fmt.Errorf("failed to insert public ips: %w", err) + } + + if err := c.updateNodeContractPublicIPs(nodeContracts); err != nil { + return fmt.Errorf("failed to update contract public ips: %w", err) + } + + for id, contractID := range reservedIPs { + if _, err := c.db.Exec(fmt.Sprintf("UPDATE public_ip SET contract_id = %d WHERE id = '%s'", contractID, id)); err != nil { + return fmt.Errorf("failed to reserve ip %s: %w", id, err) + } + } + + fmt.Printf("public IPs generated [%d : %d[\n", start, end) + + return nil +} + +func (c *Crafter) updateNodeContractPublicIPs(nodeContracts []uint64) error { + + if len(nodeContracts) != 0 { + var IDs []string + for _, contractID := range nodeContracts { + IDs = append(IDs, fmt.Sprintf("%d", contractID)) + + } + + query := "UPDATE node_contract set number_of_public_i_ps = number_of_public_i_ps + 1 WHERE contract_id IN (" + query += strings.Join(IDs, ",") + ");" + if _, err := c.db.Exec(query); err != nil { + return fmt.Errorf("failed to update node contracts public ips: %w", err) + } + } + return nil +} + +func (c *Crafter) GenerateNodeGPUs() error { + var GPUs []string + vendors := []string{"NVIDIA Corporation", "AMD", "Intel Corporation"} + devices := []string{"GeForce RTX 3080", "Radeon RX 6800 XT", "Intel Iris Xe MAX"} + + nodeTwinsStart := c.TwinStart + (c.FarmStart + c.FarmCount) + nodeWithGpuNum := 10 + + for i := 1; i <= nodeWithGpuNum; i++ { + gpuNum := len(vendors) - 1 + for j := 0; j <= gpuNum; j++ { + g := node_gpu{ + // WATCH + node_twin_id: uint64(nodeTwinsStart + uint(i)), + vendor: vendors[j], + device: devices[j], + contract: i % 2, + id: fmt.Sprintf("node-gpu-%d-%d", nodeTwinsStart+uint(i), j), + } + gpuTuple, err := objectToTupleString(g) + if err != nil { + return fmt.Errorf("failed to convert gpu object to tuple string: %w", err) + } + GPUs = append(GPUs, gpuTuple) + } + } + + if err := c.insertTuples(node_gpu{}, GPUs); err != nil { + return fmt.Errorf("failed to insert node gpu: %w", err) + } + + fmt.Println("node GPUs generated") + + return nil +} + +func (c *Crafter) GenerateCountries() error { + var countriesValues []string + + // depends on nodeStart to not duplicate the value of country.id + start := c.NodeStart + + index := start + for countryName, region := range regions { + index++ + country := country{ + id: fmt.Sprintf("country-%d", index), + country_id: uint64(index), + name: countryName, + code: countriesCodes[countryName], + region: "unknown", + subregion: region, + lat: fmt.Sprintf("%d", 0), + long: fmt.Sprintf("%d", 0), + } + + countryTuple, err := objectToTupleString(country) + if err != nil { + return fmt.Errorf("failed to convert country object to tuple string: %w", err) + } + countriesValues = append(countriesValues, countryTuple) + } + + if err := c.insertTuples(country{}, countriesValues); err != nil { + return fmt.Errorf("failed to insert country: %w", err) + } + fmt.Println("countries generated") + + return nil +} diff --git a/grid-proxy/tools/db/crafter/modifier.go b/grid-proxy/tools/db/crafter/modifier.go new file mode 100644 index 000000000..0f58717be --- /dev/null +++ b/grid-proxy/tools/db/crafter/modifier.go @@ -0,0 +1,115 @@ +package crafter + +import ( + "fmt" +) + +func (g *Crafter) UpdateNodeCountry() error { + updatesCount := 10 + query := "" + + for i := 0; i < updatesCount; i++ { + // WATCH + nodeId := r.Intn(int(g.NodeCount)) + 1 + country := countries[r.Intn(len(countries))] + query += fmt.Sprintf("UPDATE node SET country = '%s' WHERE node_id = %d;", country, nodeId) + } + + _, err := g.db.Exec(query) + fmt.Println("node country updated") + return err +} + +func (g *Crafter) UpdateNodeTotalResources() error { + updatesCount := 10 + scaling := 1 * 1024 * 1024 * 1024 + query := "" + for i := 0; i < updatesCount; i++ { + // WATCH + nodeId := r.Intn(int(g.NodeCount)) + 1 + + cru := 10 + hru := g.nodesHRU[uint64(nodeId)] + uint64(scaling) + mru := g.nodesMRU[uint64(nodeId)] + uint64(scaling) + sru := g.nodesSRU[uint64(nodeId)] + uint64(scaling) + + query += fmt.Sprintf("UPDATE node_resources_total SET cru = %d, hru = %d, mru = %d, sru = %d WHERE node_id = 'node-%d';", cru, hru, mru, sru, nodeId) + } + + _, err := g.db.Exec(query) + fmt.Println("node total resources updated") + return err +} + +func (g *Crafter) UpdateContractResources() error { + updatesCount := 10 + query := "" + for i := 0; i < updatesCount; i++ { + // WATCH + contractId := r.Intn(int(g.NodeContractCount)) + 1 + + cru := minContractCRU + hru := minContractHRU + sru := minContractSRU + mru := minContractMRU + + query += fmt.Sprintf("UPDATE contract_resources SET cru = %d, hru = %d, mru = %d, sru = %d WHERE contract_id = 'node-contract-%d';", cru, hru, mru, sru, contractId) + } + + _, err := g.db.Exec(query) + fmt.Println("contract resources updated") + return err +} + +func (g *Crafter) UpdateNodeContractState() error { + updatesCount := 10 + query := "" + states := []string{"Deleted", "GracePeriod"} + + for i := 0; i < updatesCount; i++ { + contractId := g.createdNodeContracts[r.Intn(len(g.createdNodeContracts))] + state := states[r.Intn(2)] + query += fmt.Sprintf("UPDATE node_contract SET state = '%s' WHERE contract_id = %d AND state != 'Deleted';", state, contractId) + } + + _, err := g.db.Exec(query) + fmt.Println("node contract state updated") + return err +} + +func (g *Crafter) UpdateRentContract() error { + updatesCount := 10 + query := "" + states := []string{"Deleted", "GracePeriod"} + + for i := 0; i < updatesCount; i++ { + // WATCH + contractId := r.Intn(int(g.RentContractCount)) + 1 + state := states[r.Intn(2)] + query += fmt.Sprintf("UPDATE rent_contract SET state = '%s' WHERE contract_id = %d;", state, contractId) + } + + _, err := g.db.Exec(query) + fmt.Println("rent contracts updated") + return err +} + +func (g *Crafter) UpdatePublicIps() error { + updatesCount := 10 + query := "" + + for i := 0; i < updatesCount; i++ { + idx := r.Intn(len(g.createdNodeContracts)) + contractID := g.createdNodeContracts[idx] + // WATCH + publicIPID := r.Intn(int(g.PublicIPCount)) + + query += fmt.Sprintf("UPDATE public_ip SET contract_id = (CASE WHEN contract_id = 0 THEN %d ELSE 0 END) WHERE id = 'public-ip-%d';", contractID, publicIPID) + query += fmt.Sprintf("UPDATE node_contract SET number_of_public_i_ps = (SELECT COUNT(id) FROM public_ip WHERE contract_id = %d) WHERE contract_id = %d;", contractID, contractID) + + } + + _, err := g.db.Exec(query) + fmt.Println("public ip contract_id update") + return err +} diff --git a/grid-proxy/tools/db/crafter/types.go b/grid-proxy/tools/db/crafter/types.go new file mode 100644 index 000000000..0cdf0230e --- /dev/null +++ b/grid-proxy/tools/db/crafter/types.go @@ -0,0 +1,266 @@ +package crafter + +import ( + "database/sql" + "math/rand" +) + +const ( + contractCreatedRatio = .1 // from devnet + usedPublicIPsRatio = .9 + nodeUpRatio = .5 + maxContractHRU = 1024 * 1024 * 1024 * 300 + maxContractSRU = 1024 * 1024 * 1024 * 300 + maxContractMRU = 1024 * 1024 * 1024 * 16 + maxContractCRU = 16 + minContractHRU = 0 + minContractSRU = 1024 * 1024 * 256 + minContractMRU = 1024 * 1024 * 256 + minContractCRU = 1 +) + +var ( + r *rand.Rand + + countries = []string{"Belgium", "United States", "Egypt", "United Kingdom"} + regions = map[string]string{ + "Belgium": "Europe", + "United States": "Americas", + "Egypt": "Africa", + "United Kingdom": "Europe", + } + countriesCodes = map[string]string{ + "Belgium": "BG", + "United States": "US", + "Egypt": "EG", + "United Kingdom": "UK", + } + cities = map[string][]string{ + "Belgium": {"Brussels", "Antwerp", "Ghent", "Charleroi"}, + "United States": {"New York", "Chicago", "Los Angeles", "San Francisco"}, + "Egypt": {"Cairo", "Giza", "October", "Nasr City"}, + "United Kingdom": {"London", "Liverpool", "Manchester", "Cambridge"}, + } +) + +type Crafter struct { + db *sql.DB + + nodesMRU map[uint64]uint64 + nodesSRU map[uint64]uint64 + nodesHRU map[uint64]uint64 + nodeUP map[uint64]bool + createdNodeContracts []uint64 + dedicatedFarms map[uint64]struct{} + availableRentNodes map[uint64]struct{} + availableRentNodesList []uint64 + renter map[uint64]uint64 + + NodeCount uint + FarmCount uint + PublicIPCount uint + TwinCount uint + NodeContractCount uint + RentContractCount uint + NameContractCount uint + + NodeStart uint + FarmStart uint + TwinStart uint + ContractStart uint + BillStart uint + PublicIPStart uint +} + +func NewCrafter(db *sql.DB, + seed int, + nodeCount, + farmCount, + twinCount, + ipCount, + nodeContractCount, + nameContractCount, + rentContractCount, + nodeStart, + farmStart, + twinStart, + contractStart, + billStart, + publicIPStart uint) Crafter { + + r = rand.New(rand.NewSource(int64(seed))) + + return Crafter{ + db: db, + + nodesMRU: make(map[uint64]uint64), + nodesSRU: make(map[uint64]uint64), + nodesHRU: make(map[uint64]uint64), + nodeUP: make(map[uint64]bool), + createdNodeContracts: make([]uint64, 0), + dedicatedFarms: make(map[uint64]struct{}), + availableRentNodes: make(map[uint64]struct{}), + availableRentNodesList: make([]uint64, 0), + renter: make(map[uint64]uint64), + + TwinCount: twinCount, + FarmCount: farmCount, + NodeCount: nodeCount, + PublicIPCount: ipCount, + NodeContractCount: nodeContractCount, + RentContractCount: rentContractCount, + NameContractCount: nameContractCount, + + NodeStart: nodeStart, + FarmStart: farmStart, + TwinStart: twinStart, + ContractStart: contractStart, + BillStart: billStart, + PublicIPStart: publicIPStart, + } +} + +type contract_resources struct { + id string + hru uint64 + sru uint64 + cru uint64 + mru uint64 + contract_id string +} +type farm struct { + id string + grid_version uint64 + farm_id uint64 + name string + twin_id uint64 + pricing_policy_id uint64 + certification string + stellar_address string + dedicated_farm bool +} + +type node struct { + id string + grid_version uint64 + node_id uint64 + farm_id uint64 + twin_id uint64 + country string + city string + uptime uint64 + created uint64 + farming_policy_id uint64 + certification string + secure bool + virtualized bool + serial_number string + created_at uint64 + updated_at uint64 + location_id string + power nodePower `gorm:"type:jsonb"` + extra_fee uint64 + dedicated bool +} + +type nodePower struct { + State string `json:"state"` + Target string `json:"target"` +} +type twin struct { + id string + grid_version uint64 + twin_id uint64 + account_id string + relay string + public_key string +} + +type public_ip struct { + id string + gateway string + ip string + contract_id uint64 + farm_id string +} +type node_contract struct { + id string + grid_version uint64 + contract_id uint64 + twin_id uint64 + node_id uint64 + deployment_data string + deployment_hash string + number_of_public_i_ps uint64 + state string + created_at uint64 + resources_used_id string +} +type node_resources_total struct { + id string + hru uint64 + sru uint64 + cru uint64 + mru uint64 + node_id string +} +type public_config struct { + id string + ipv4 string + ipv6 string + gw4 string + gw6 string + domain string + node_id string +} +type rent_contract struct { + id string + grid_version uint64 + contract_id uint64 + twin_id uint64 + node_id uint64 + state string + created_at uint64 +} +type location struct { + id string + longitude string + latitude string +} + +type contract_bill_report struct { + id string + contract_id uint64 + discount_received string + amount_billed uint64 + timestamp uint64 +} + +type name_contract struct { + id string + grid_version uint64 + contract_id uint64 + twin_id uint64 + name string + state string + created_at uint64 +} + +type node_gpu struct { + node_twin_id uint64 + id string + vendor string + device string + contract int +} + +type country struct { + id string + country_id uint64 + code string + name string + region string + subregion string + lat string + long string +} diff --git a/grid-proxy/tools/db/utils.go b/grid-proxy/tools/db/crafter/utils.go similarity index 78% rename from grid-proxy/tools/db/utils.go rename to grid-proxy/tools/db/crafter/utils.go index 0a846607e..e08b9d228 100644 --- a/grid-proxy/tools/db/utils.go +++ b/grid-proxy/tools/db/crafter/utils.go @@ -1,4 +1,4 @@ -package main +package crafter import ( "encoding/json" @@ -92,7 +92,8 @@ func objectToTupleString(v interface{}) (string, error) { if err != nil { return "", fmt.Errorf("failed to marshal the power map to JSON: %w", err) } - v = fmt.Sprintf("'%s'", string(powerJSON)) + escapedJSON := strings.ReplaceAll(string(powerJSON), "'", "''") + v = fmt.Sprintf("'%s'", escapedJSON) } vals = fmt.Sprintf("%s, %s", vals, v) } @@ -100,6 +101,30 @@ func objectToTupleString(v interface{}) (string, error) { return fmt.Sprintf("%s)", vals), nil } +func (g *Crafter) insertTuples(tupleObj interface{}, tuples []string) error { + + if len(tuples) != 0 { + query := "INSERT INTO " + reflect.Indirect(reflect.ValueOf(tupleObj)).Type().Name() + " (" + objType := reflect.TypeOf(tupleObj) + for i := 0; i < objType.NumField(); i++ { + if i != 0 { + query += ", " + } + query += objType.Field(i).Name + } + + query += ") VALUES " + + query += strings.Join(tuples, ",") + query += ";" + if _, err := g.db.Exec(query); err != nil { + return fmt.Errorf("failed to insert tuples: %w", err) + } + + } + return nil +} + // popRandom selects a random element from the given slice, func popRandom(l []uint64) ([]uint64, uint64, error) { idx, err := rnd(0, uint64(len(l)-1)) diff --git a/grid-proxy/tools/db/utils_test.go b/grid-proxy/tools/db/crafter/utils_test.go similarity index 99% rename from grid-proxy/tools/db/utils_test.go rename to grid-proxy/tools/db/crafter/utils_test.go index acd1c904d..23ab67eb6 100644 --- a/grid-proxy/tools/db/utils_test.go +++ b/grid-proxy/tools/db/crafter/utils_test.go @@ -1,4 +1,4 @@ -package main +package crafter import ( "fmt" diff --git a/grid-proxy/tools/db/db.go b/grid-proxy/tools/db/db.go index 3abbc035c..f688c1cbb 100644 --- a/grid-proxy/tools/db/db.go +++ b/grid-proxy/tools/db/db.go @@ -4,7 +4,6 @@ import ( "database/sql" "flag" "fmt" - "math/rand" // used by the orm @@ -12,10 +11,6 @@ import ( "github.com/pkg/errors" ) -var ( - r *rand.Rand -) - type flags struct { postgresHost string postgresPort int @@ -41,67 +36,33 @@ func parseCmdline() flags { func main() { f := parseCmdline() - r = rand.New(rand.NewSource(int64(f.seed))) - psqlInfo := fmt.Sprintf("host=%s port=%d user=%s "+ - "password=%s dbname=%s sslmode=disable", + psqlInfo := fmt.Sprintf("host=%s port=%d user=%s password=%s dbname=%s sslmode=disable", f.postgresHost, f.postgresPort, f.postgresUser, f.postgresPassword, f.postgresDB) db, err := sql.Open("postgres", psqlInfo) if err != nil { panic(errors.Wrap(err, "failed to open db")) } defer db.Close() + if f.reset { - if _, err := db.Exec( - ` - DROP TABLE IF EXISTS account CASCADE; - DROP TABLE IF EXISTS burn_transaction CASCADE; - DROP TABLE IF EXISTS city CASCADE; - DROP TABLE IF EXISTS contract_bill_report CASCADE; - DROP TABLE IF EXISTS contract_resources CASCADE; - DROP TABLE IF EXISTS country CASCADE; - DROP TABLE IF EXISTS entity CASCADE; - DROP TABLE IF EXISTS entity_proof CASCADE; - DROP TABLE IF EXISTS farm CASCADE; - DROP TABLE IF EXISTS farming_policy CASCADE; - DROP TABLE IF EXISTS historical_balance CASCADE; - DROP TABLE IF EXISTS interfaces CASCADE; - DROP TABLE IF EXISTS location CASCADE; - DROP TABLE IF EXISTS migrations CASCADE; - DROP TABLE IF EXISTS mint_transaction CASCADE; - DROP TABLE IF EXISTS name_contract CASCADE; - DROP TABLE IF EXISTS node CASCADE; - DROP TABLE IF EXISTS node_contract CASCADE; - DROP TABLE IF EXISTS node_resources_free CASCADE; - DROP TABLE IF EXISTS node_resources_total CASCADE; - DROP TABLE IF EXISTS node_resources_used CASCADE; - DROP TABLE IF EXISTS nru_consumption CASCADE; - DROP TABLE IF EXISTS pricing_policy CASCADE; - DROP TABLE IF EXISTS public_config CASCADE; - DROP TABLE IF EXISTS public_ip CASCADE; - DROP TABLE IF EXISTS refund_transaction CASCADE; - DROP TABLE IF EXISTS rent_contract CASCADE; - DROP TABLE IF EXISTS transfer CASCADE; - DROP TABLE IF EXISTS twin CASCADE; - DROP TABLE IF EXISTS typeorm_metadata CASCADE; - DROP TABLE IF EXISTS uptime_event CASCADE; - DROP SCHEMA IF EXISTS substrate_threefold_status CASCADE; - DROP TABLE IF EXISTS node_gpu CASCADE; - - `); err != nil { + if err := reset(db); err != nil { panic(err) } } + if err := initSchema(db); err != nil { panic(err) } + // it looks like a useless block but everything breaks when it's removed _, err = db.Query("SELECT current_database();") if err != nil { panic(err) } // ---- - if err := generateData(db); err != nil { + + if err := generateData(db, f.seed); err != nil { panic(err) } } diff --git a/grid-proxy/tools/db/docker.sh b/grid-proxy/tools/db/docker.sh deleted file mode 100644 index 05a7907cf..000000000 --- a/grid-proxy/tools/db/docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/bash - diff --git a/grid-proxy/tools/db/generate.go b/grid-proxy/tools/db/generate.go index 18b70a2db..bea47f153 100644 --- a/grid-proxy/tools/db/generate.go +++ b/grid-proxy/tools/db/generate.go @@ -4,74 +4,60 @@ import ( "database/sql" "fmt" "os" - "reflect" - "strings" - "time" - "github.com/google/uuid" - "github.com/threefoldtech/zos/pkg/gridtypes" -) - -var ( - nodesMRU = make(map[uint64]uint64) - nodesSRU = make(map[uint64]uint64) - nodesHRU = make(map[uint64]uint64) - nodeUP = make(map[uint64]bool) - createdNodeContracts = make([]uint64, 0) - dedicatedFarms = make(map[uint64]struct{}) - availableRentNodes = make(map[uint64]struct{}) - availableRentNodesList = make([]uint64, 0) - renter = make(map[uint64]uint64) + "github.com/threefoldtech/tfgrid-sdk-go/grid-proxy/tools/db/crafter" ) const ( - contractCreatedRatio = .1 // from devnet - usedPublicIPsRatio = .9 - nodeUpRatio = .5 - nodeCount = 1000 - farmCount = 100 - normalUsers = 2000 - publicIPCount = 1000 - twinCount = nodeCount + farmCount + normalUsers - nodeContractCount = 3000 - rentContractCount = 100 - nameContractCount = 300 - - maxContractHRU = 1024 * 1024 * 1024 * 300 - maxContractSRU = 1024 * 1024 * 1024 * 300 - maxContractMRU = 1024 * 1024 * 1024 * 16 - maxContractCRU = 16 - minContractHRU = 0 - minContractSRU = 1024 * 1024 * 256 - minContractMRU = 1024 * 1024 * 256 - minContractCRU = 1 -) - -var ( - countries = []string{"Belgium", "United States", "Egypt", "United Kingdom"} - regions = map[string]string{ - "Belgium": "Europe", - "United States": "Americas", - "Egypt": "Africa", - "United Kingdom": "Europe", - } - countriesCodes = map[string]string{ - "Belgium": "BG", - "United States": "US", - "Egypt": "EG", - "United Kingdom": "UK", - } - cities = map[string][]string{ - "Belgium": {"Brussels", "Antwerp", "Ghent", "Charleroi"}, - "United States": {"New York", "Chicago", "Los Angeles", "San Francisco"}, - "Egypt": {"Cairo", "Giza", "October", "Nasr City"}, - "United Kingdom": {"London", "Liverpool", "Manchester", "Cambridge"}, - } + NodeCount = 6000 + FarmCount = 600 + TwinCount = 6000 + 600 + 6000 // nodes + farms + normal users + PublicIPCount = 1000 + NodeContractCount = 9000 + RentContractCount = 100 + NameContractCount = 300 ) -const deleted = "Deleted" -const created = "Created" -const gracePeriod = "GracePeriod" +func reset(db *sql.DB) error { + _, err := db.Exec( + ` + DROP TABLE IF EXISTS account CASCADE; + DROP TABLE IF EXISTS burn_transaction CASCADE; + DROP TABLE IF EXISTS city CASCADE; + DROP TABLE IF EXISTS contract_bill_report CASCADE; + DROP TABLE IF EXISTS contract_resources CASCADE; + DROP TABLE IF EXISTS country CASCADE; + DROP TABLE IF EXISTS entity CASCADE; + DROP TABLE IF EXISTS entity_proof CASCADE; + DROP TABLE IF EXISTS farm CASCADE; + DROP TABLE IF EXISTS farming_policy CASCADE; + DROP TABLE IF EXISTS historical_balance CASCADE; + DROP TABLE IF EXISTS interfaces CASCADE; + DROP TABLE IF EXISTS location CASCADE; + DROP TABLE IF EXISTS migrations CASCADE; + DROP TABLE IF EXISTS mint_transaction CASCADE; + DROP TABLE IF EXISTS name_contract CASCADE; + DROP TABLE IF EXISTS node CASCADE; + DROP TABLE IF EXISTS node_contract CASCADE; + DROP TABLE IF EXISTS node_resources_free CASCADE; + DROP TABLE IF EXISTS node_resources_total CASCADE; + DROP TABLE IF EXISTS node_resources_used CASCADE; + DROP TABLE IF EXISTS nru_consumption CASCADE; + DROP TABLE IF EXISTS pricing_policy CASCADE; + DROP TABLE IF EXISTS public_config CASCADE; + DROP TABLE IF EXISTS public_ip CASCADE; + DROP TABLE IF EXISTS refund_transaction CASCADE; + DROP TABLE IF EXISTS rent_contract CASCADE; + DROP TABLE IF EXISTS transfer CASCADE; + DROP TABLE IF EXISTS twin CASCADE; + DROP TABLE IF EXISTS typeorm_metadata CASCADE; + DROP TABLE IF EXISTS uptime_event CASCADE; + DROP SCHEMA IF EXISTS substrate_threefold_status CASCADE; + DROP TABLE IF EXISTS node_gpu CASCADE; + + `) + return err +} func initSchema(db *sql.DB) error { schema, err := os.ReadFile("./schema.sql") @@ -85,751 +71,48 @@ func initSchema(db *sql.DB) error { return nil } -func generateTwins(db *sql.DB) error { - var twins []string - - for i := uint64(1); i <= twinCount; i++ { - twin := twin{ - id: fmt.Sprintf("twin-%d", i), - account_id: fmt.Sprintf("account-id-%d", i), - relay: fmt.Sprintf("relay-%d", i), - public_key: fmt.Sprintf("public-key-%d", i), - twin_id: i, - grid_version: 3, - } - tuple, err := objectToTupleString(twin) - if err != nil { - return fmt.Errorf("failed to convert twin object to tuple string: %w", err) - } - twins = append(twins, tuple) - } - - if err := insertTuples(db, twin{}, twins); err != nil { - return fmt.Errorf("failed to insert twins: %w", err) - } - fmt.Println("twins generated") - - return nil -} - -func generatePublicIPs(db *sql.DB) error { - var publicIPs []string - var nodeContracts []uint64 - - for i := uint64(1); i <= publicIPCount; i++ { - contract_id := uint64(0) - if flip(usedPublicIPsRatio) { - idx, err := rnd(0, uint64(len(createdNodeContracts))-1) - if err != nil { - return fmt.Errorf("failed to generate random index: %w", err) - } - contract_id = createdNodeContracts[idx] - } - ip := randomIPv4() - farmID, err := rnd(1, farmCount) - if err != nil { - return fmt.Errorf("failed to generate random farm id: %w", err) - } - - public_ip := public_ip{ - id: fmt.Sprintf("public-ip-%d", i), - gateway: ip.String(), - ip: IPv4Subnet(ip).String(), - contract_id: contract_id, - farm_id: fmt.Sprintf("farm-%d", farmID), - } - publicIpTuple, err := objectToTupleString(public_ip) - if err != nil { - return fmt.Errorf("failed to convert public ip object to tuple string: %w", err) - } - publicIPs = append(publicIPs, publicIpTuple) - nodeContracts = append(nodeContracts, contract_id) - } - - if err := insertTuples(db, public_ip{}, publicIPs); err != nil { - return fmt.Errorf("failed to insert public ips: %w", err) - } - - if err := updateNodeContractPublicIPs(db, nodeContracts); err != nil { - return fmt.Errorf("failed to update contract public ips: %w", err) - } - - fmt.Println("public IPs generated") - - return nil -} - -func generateFarms(db *sql.DB) error { - var farms []string - - for i := uint64(1); i <= farmCount; i++ { - farm := farm{ - id: fmt.Sprintf("farm-%d", i), - farm_id: i, - name: fmt.Sprintf("farm-name-%d", i), - certification: "Diy", - dedicated_farm: flip(.1), - twin_id: i, - pricing_policy_id: 1, - grid_version: 3, - stellar_address: "", - } - - if farm.dedicated_farm { - dedicatedFarms[farm.farm_id] = struct{}{} - } - - farmTuple, err := objectToTupleString(farm) - if err != nil { - return fmt.Errorf("failed to convert farm object to tuple string: %w", err) - } - farms = append(farms, farmTuple) - } - - if err := insertTuples(db, farm{}, farms); err != nil { - return fmt.Errorf("failed to insert farms: %w", err) - } - fmt.Println("farms generated") - - return nil -} - -func generateCountries(db *sql.DB) error { - var countriesValues []string - index := 0 - for countryName, region := range regions { - index++ - country := country{ - id: fmt.Sprintf("country-%d", index), - country_id: uint64(index), - name: countryName, - code: countriesCodes[countryName], - region: "unknown", - subregion: region, - lat: fmt.Sprintf("%d", 0), - long: fmt.Sprintf("%d", 0), - } - - countryTuple, err := objectToTupleString(country) - if err != nil { - return fmt.Errorf("failed to convert country object to tuple string: %w", err) - } - countriesValues = append(countriesValues, countryTuple) - } - - if err := insertTuples(db, country{}, countriesValues); err != nil { - return fmt.Errorf("failed to insert country: %w", err) - } - fmt.Println("countries generated") - - return nil -} - -func generateNodeContracts(db *sql.DB, billsStartID, contractsStartID int) ([]string, int, error) { - var contracts []string - var contractResources []string - var billingReports []string - - for i := uint64(1); i <= nodeContractCount; i++ { - nodeID, err := rnd(1, nodeCount) - if err != nil { - return nil, contractsStartID, fmt.Errorf("failed to generate random node id: %w", err) - } - state := deleted - - if nodeUP[nodeID] { - if flip(contractCreatedRatio) { - state = created - } else if flip(0.5) { - state = gracePeriod - } - } - - if state != deleted && (minContractHRU > nodesHRU[nodeID] || minContractMRU > nodesMRU[nodeID] || minContractSRU > nodesSRU[nodeID]) { - i-- - continue - } - - twinID, err := rnd(1100, 3100) - if err != nil { - return nil, contractsStartID, fmt.Errorf("failed to generate random twin id: %w", err) - } - - if renter, ok := renter[nodeID]; ok { - twinID = renter - } - - if _, ok := availableRentNodes[nodeID]; ok { - i-- - continue - } - - contract := node_contract{ - id: fmt.Sprintf("node-contract-%d", contractsStartID), - twin_id: twinID, - contract_id: uint64(contractsStartID), - state: state, - created_at: uint64(time.Now().Unix()), - node_id: nodeID, - deployment_data: fmt.Sprintf("deployment-data-%d", contractsStartID), - deployment_hash: fmt.Sprintf("deployment-hash-%d", contractsStartID), - number_of_public_i_ps: 0, - grid_version: 3, - resources_used_id: "", - } - - cru, err := rnd(minContractCRU, maxContractCRU) - if err != nil { - return nil, contractsStartID, fmt.Errorf("failed to generate random cru: %w", err) - } - - hru, err := rnd(minContractHRU, min(maxContractHRU, nodesHRU[nodeID])) - if err != nil { - return nil, contractsStartID, fmt.Errorf("failed to generate random hru: %w", err) - } - - sru, err := rnd(minContractSRU, min(maxContractSRU, nodesSRU[nodeID])) - if err != nil { - return nil, contractsStartID, fmt.Errorf("failed to generate random sru: %w", err) - } - - mru, err := rnd(minContractMRU, min(maxContractMRU, nodesMRU[nodeID])) - if err != nil { - return nil, contractsStartID, fmt.Errorf("failed to generate random mru: %w", err) - } - - contract_resources := contract_resources{ - id: fmt.Sprintf("contract-resources-%d", contractsStartID), - hru: hru, - sru: sru, - cru: cru, - mru: mru, - contract_id: fmt.Sprintf("node-contract-%d", contractsStartID), - } - if contract.state != deleted { - nodesHRU[nodeID] -= hru - nodesSRU[nodeID] -= sru - nodesMRU[nodeID] -= mru - createdNodeContracts = append(createdNodeContracts, uint64(contractsStartID)) - } - - contractTuple, err := objectToTupleString(contract) - if err != nil { - return nil, contractsStartID, fmt.Errorf("failed to convert contract object to tuple string: %w", err) - } - contracts = append(contracts, contractTuple) - - contractResourcesTuple, err := objectToTupleString(contract_resources) - if err != nil { - return nil, contractsStartID, fmt.Errorf("failed to convert contract resources object to tuple string: %w", err) - } - contractResources = append(contractResources, contractResourcesTuple) - - billings, err := rnd(0, 10) - if err != nil { - return nil, contractsStartID, fmt.Errorf("failed to generate random billing count: %w", err) - } - - amountBilled, err := rnd(0, 100000) - if err != nil { - return nil, contractsStartID, fmt.Errorf("failed to generate random amount billed: %w", err) - } - for j := uint64(0); j < billings; j++ { - billing := contract_bill_report{ - id: fmt.Sprintf("contract-bill-report-%d", billsStartID), - contract_id: uint64(contractsStartID), - discount_received: "Default", - amount_billed: amountBilled, - timestamp: uint64(time.Now().UnixNano()), - } - billsStartID++ - - billTuple, err := objectToTupleString(billing) - if err != nil { - return nil, contractsStartID, fmt.Errorf("failed to convert contract bill report object to tuple string: %w", err) - } - billingReports = append(billingReports, billTuple) - } - contractsStartID++ - } - - if err := insertTuples(db, node_contract{}, contracts); err != nil { - return nil, contractsStartID, fmt.Errorf("failed to insert node contracts: %w", err) - } - - if err := insertTuples(db, contract_resources{}, contractResources); err != nil { - return nil, contractsStartID, fmt.Errorf("failed to insert contract resources: %w", err) - } - - if err := updateNodeContractResourceID(db, contractsStartID-nodeContractCount, contractsStartID); err != nil { - return nil, contractsStartID, fmt.Errorf("failed to update node contract resources id: %w", err) - } - - fmt.Println("node contracts generated") - - return billingReports, contractsStartID, nil -} - -func generateNameContracts(db *sql.DB, billsStartID, contractsStartID int) ([]string, int, error) { - var contracts []string - var billReports []string - for i := uint64(1); i <= nameContractCount; i++ { - nodeID, err := rnd(1, nodeCount) - if err != nil { - return nil, contractsStartID, fmt.Errorf("failed to generate random node id: %w", err) - } - - state := deleted - if nodeUP[nodeID] { - if flip(contractCreatedRatio) { - state = created - } else if flip(0.5) { - state = gracePeriod - } - } - - twinID, err := rnd(1100, 3100) - if err != nil { - return nil, contractsStartID, fmt.Errorf("failed to generate random twin id: %w", err) - } - - if renter, ok := renter[nodeID]; ok { - twinID = renter - } - - if _, ok := availableRentNodes[nodeID]; ok { - i-- - continue - } - - contract := name_contract{ - id: fmt.Sprintf("name-contract-%d", contractsStartID), - twin_id: twinID, - contract_id: uint64(contractsStartID), - state: state, - created_at: uint64(time.Now().Unix()), - grid_version: 3, - name: uuid.NewString(), - } - - contractTuple, err := objectToTupleString(contract) - if err != nil { - return nil, contractsStartID, fmt.Errorf("failed to convert contract object to tuple string: %w", err) - } - contracts = append(contracts, contractTuple) - - billings, err := rnd(0, 10) - if err != nil { - return nil, contractsStartID, fmt.Errorf("failed to generate random billings count: %w", err) - } - amountBilled, err := rnd(0, 100000) - if err != nil { - return nil, contractsStartID, fmt.Errorf("failed to generate random amount billed: %w", err) - } - - for j := uint64(0); j < billings; j++ { - billing := contract_bill_report{ - id: fmt.Sprintf("contract-bill-report-%d", billsStartID), - contract_id: uint64(contractsStartID), - discount_received: "Default", - amount_billed: amountBilled, - timestamp: uint64(time.Now().UnixNano()), - } - billsStartID++ - - billTuple, err := objectToTupleString(billing) - if err != nil { - return nil, contractsStartID, fmt.Errorf("failed to convert contract bill report object to tuple string: %w", err) - } - billReports = append(billReports, billTuple) - } - contractsStartID++ - } - - if err := insertTuples(db, name_contract{}, contracts); err != nil { - return nil, contractsStartID, fmt.Errorf("failed to insert name contracts: %w", err) - } - - fmt.Println("name contracts generated") - - return billReports, contractsStartID, nil -} -func generateRentContracts(db *sql.DB, billsStartID, contractsStartID int) ([]string, int, error) { - var contracts []string - var billReports []string - for i := uint64(1); i <= rentContractCount; i++ { - nl, nodeID, err := popRandom(availableRentNodesList) - if err != nil { - return nil, contractsStartID, fmt.Errorf("failed to select random element from the gives slice: %w", err) - } - - availableRentNodesList = nl - delete(availableRentNodes, nodeID) - state := deleted - if nodeUP[nodeID] { - if flip(0.9) { - state = created - } else if flip(0.5) { - state = gracePeriod - } - } - - twinID, err := rnd(1100, 3100) - if err != nil { - return nil, contractsStartID, fmt.Errorf("failed to generate a random twin id: %w", err) - } - - contract := rent_contract{ - id: fmt.Sprintf("rent-contract-%d", contractsStartID), - twin_id: twinID, - contract_id: uint64(contractsStartID), - state: state, - created_at: uint64(time.Now().Unix()), - node_id: nodeID, - grid_version: 3, - } - - if state != deleted { - renter[nodeID] = contract.twin_id - } - - contractTuple, err := objectToTupleString(contract) - if err != nil { - return nil, contractsStartID, fmt.Errorf("failed to convert contract object to tuple string: %w", err) - } - contracts = append(contracts, contractTuple) - - billings, err := rnd(0, 10) - if err != nil { - return nil, contractsStartID, fmt.Errorf("failed to generate billings count: %w", err) - } - - amountBilled, err := rnd(0, 100000) - if err != nil { - return nil, contractsStartID, fmt.Errorf("failed to generate amount billed: %w", err) - } - - for j := uint64(0); j < billings; j++ { - billing := contract_bill_report{ - id: fmt.Sprintf("contract-bill-report-%d", billsStartID), - contract_id: uint64(contractsStartID), - discount_received: "Default", - amount_billed: amountBilled, - timestamp: uint64(time.Now().UnixNano()), - } - - billsStartID++ - - billTuple, err := objectToTupleString(billing) - if err != nil { - return nil, contractsStartID, fmt.Errorf("failed to convert contract bill report object to tuple string: %w", err) - } - billReports = append(billReports, billTuple) - - } - contractsStartID++ - } - - if err := insertTuples(db, rent_contract{}, contracts); err != nil { - return nil, contractsStartID, fmt.Errorf("failed to insert rent contracts: %w", err) - } - - fmt.Println("rent contracts generated") - - return billReports, contractsStartID, nil -} - -func generateNodes(db *sql.DB) error { - powerState := []string{"Up", "Down"} - var locations []string - var nodes []string - var totalResources []string - var publicConfigs []string - for i := uint64(1); i <= nodeCount; i++ { - mru, err := rnd(4, 256) - if err != nil { - return fmt.Errorf("failed to generate random mru: %w", err) - } - mru *= 1024 * 1024 * 1024 - - hru, err := rnd(100, 30*1024) - if err != nil { - return fmt.Errorf("failed to generate random hru: %w", err) - } - hru *= 1024 * 1024 * 1024 // 100GB -> 30TB - - sru, err := rnd(200, 30*1024) - if err != nil { - return fmt.Errorf("failed to generate random sru: %w", err) - } - sru *= 1024 * 1024 * 1024 // 100GB -> 30TB - - cru, err := rnd(4, 128) - if err != nil { - return fmt.Errorf("failed to generate random cru: %w", err) - } - - up := flip(nodeUpRatio) - periodFromLatestUpdate, err := rnd(60*40*3, 60*60*24*30*12) - if err != nil { - return fmt.Errorf("failed to generate random period from latest update: %w", err) - } - updatedAt := time.Now().Unix() - int64(periodFromLatestUpdate) - - if up { - periodFromLatestUpdate, err = rnd(0, 60*40*1) - if err != nil { - return fmt.Errorf("failed to generate period from latest update: %w", err) - } - updatedAt = time.Now().Unix() - int64(periodFromLatestUpdate) - } - - nodesMRU[i] = mru - max(2*uint64(gridtypes.Gigabyte), mru/10) - nodesSRU[i] = sru - 100*uint64(gridtypes.Gigabyte) - nodesHRU[i] = hru - nodeUP[i] = up - - location := location{ - id: fmt.Sprintf("location-%d", i), - longitude: fmt.Sprintf("location--long-%d", i), - latitude: fmt.Sprintf("location-lat-%d", i), - } - - countryIndex := r.Intn(len(countries)) - cityIndex := r.Intn(len(cities[countries[countryIndex]])) - node := node{ - id: fmt.Sprintf("node-%d", i), - location_id: fmt.Sprintf("location-%d", i), - node_id: i, - farm_id: i%100 + 1, - twin_id: i + 100 + 1, - country: countries[countryIndex], - city: cities[countries[countryIndex]][cityIndex], - uptime: 1000, - updated_at: uint64(updatedAt), - created: uint64(time.Now().Unix()), - created_at: uint64(time.Now().Unix()), - farming_policy_id: 1, - grid_version: 3, - certification: "Diy", - secure: false, - virtualized: false, - serial_number: "", - power: nodePower{ - State: powerState[r.Intn(len(powerState))], - Target: powerState[r.Intn(len(powerState))], - }, - extra_fee: 0, - } - - total_resources := node_resources_total{ - id: fmt.Sprintf("total-resources-%d", i), - hru: hru, - sru: sru, - cru: cru, - mru: mru, - node_id: fmt.Sprintf("node-%d", i), - } - - if _, ok := dedicatedFarms[node.farm_id]; ok { - availableRentNodes[i] = struct{}{} - availableRentNodesList = append(availableRentNodesList, i) - } - - locationTuple, err := objectToTupleString(location) - if err != nil { - return fmt.Errorf("failed to convert location object to tuple string: %w", err) - } - locations = append(locations, locationTuple) - - nodeTuple, err := objectToTupleString(node) - if err != nil { - return fmt.Errorf("failed to convert node object to tuple string: %w", err) - } - nodes = append(nodes, nodeTuple) - - totalResourcesTuple, err := objectToTupleString(total_resources) - if err != nil { - return fmt.Errorf("failed to convert total resources object to tuple string: %w", err) - } - totalResources = append(totalResources, totalResourcesTuple) - - if flip(.1) { - publicConfig := public_config{ - id: fmt.Sprintf("public-config-%d", i), - ipv4: "185.16.5.2/24", - gw4: "185.16.5.2", - ipv6: "::1/64", - gw6: "::1", - domain: "hamada.com", - node_id: fmt.Sprintf("node-%d", i), - } - publicConfigTuple, err := objectToTupleString(publicConfig) - if err != nil { - return fmt.Errorf("failed to convert public config object to tuple string: %w", err) - } - publicConfigs = append(publicConfigs, publicConfigTuple) - - } - } - - if err := insertTuples(db, location{}, locations); err != nil { - return fmt.Errorf("failed to insert locations: %w", err) - } - - if err := insertTuples(db, node{}, nodes); err != nil { - return fmt.Errorf("failed to isnert nodes: %w", err) - } - - if err := insertTuples(db, node_resources_total{}, totalResources); err != nil { - return fmt.Errorf("failed to insert node resources total: %w", err) - } - - if err := insertTuples(db, public_config{}, publicConfigs); err != nil { - return fmt.Errorf("failed to insert public configs: %w", err) - } - fmt.Println("nodes generated") - - return nil -} - -func generateNodeGPUs(db *sql.DB) error { - var GPUs []string - vendors := []string{"NVIDIA Corporation", "AMD", "Intel Corporation"} - devices := []string{"GeForce RTX 3080", "Radeon RX 6800 XT", "Intel Iris Xe MAX"} - - for i := 0; i <= 10; i++ { - gpuNum := len(vendors) - 1 - for j := 0; j <= gpuNum; j++ { - g := node_gpu{ - node_twin_id: uint64(i + 100), - vendor: vendors[j], - device: devices[j], - contract: i % 2, - id: fmt.Sprintf("0000:0e:00.0/1002/744c/%d", j), - } - gpuTuple, err := objectToTupleString(g) - if err != nil { - return fmt.Errorf("failed to convert gpu object to tuple string: %w", err) - } - GPUs = append(GPUs, gpuTuple) - } - } - - if err := insertTuples(db, node_gpu{}, GPUs); err != nil { - return fmt.Errorf("failed to insert node gpu: %w", err) - } - - fmt.Println("node GPUs generated") - - return nil -} - -func generateContracts(db *sql.DB) error { - rentContractIDStart := 1 - - var billReports []string - - rentContractsBillReports, nodeContractIDStart, err := generateRentContracts(db, 1, rentContractIDStart) - if err != nil { - return fmt.Errorf("failed to generate rent contracts: %w", err) - } - billReports = append(billReports, rentContractsBillReports...) - - nodeContractsBillReports, nameContractIDStart, err := generateNodeContracts(db, len(billReports)+1, nodeContractIDStart) - if err != nil { - return fmt.Errorf("failed to generate node contracts: %w", err) - } - billReports = append(billReports, nodeContractsBillReports...) - - nameContractsBillReports, _, err := generateNameContracts(db, len(billReports)+1, nameContractIDStart) - if err != nil { - return fmt.Errorf("failed to generate name contracts: %w", err) - } - billReports = append(billReports, nameContractsBillReports...) - - if err := insertTuples(db, contract_bill_report{}, billReports); err != nil { - return fmt.Errorf("failed to generate contract bill reports: %w", err) - } - return nil -} - -func insertTuples(db *sql.DB, tupleObj interface{}, tuples []string) error { - - if len(tuples) != 0 { - query := "INSERT INTO " + reflect.Indirect(reflect.ValueOf(tupleObj)).Type().Name() + " (" - objType := reflect.TypeOf(tupleObj) - for i := 0; i < objType.NumField(); i++ { - if i != 0 { - query += ", " - } - query += objType.Field(i).Name - } - - query += ") VALUES " - - query += strings.Join(tuples, ",") - query += ";" - if _, err := db.Exec(query); err != nil { - return fmt.Errorf("failed to insert tuples: %w", err) - } - - } - return nil -} - -func updateNodeContractPublicIPs(db *sql.DB, nodeContracts []uint64) error { - - if len(nodeContracts) != 0 { - var IDs []string - for _, contractID := range nodeContracts { - IDs = append(IDs, fmt.Sprintf("%d", contractID)) - - } - - query := "UPDATE node_contract set number_of_public_i_ps = number_of_public_i_ps + 1 WHERE contract_id IN (" - query += strings.Join(IDs, ",") + ");" - if _, err := db.Exec(query); err != nil { - return fmt.Errorf("failed to update node contracts public ips: %w", err) - } - } - return nil -} - -func updateNodeContractResourceID(db *sql.DB, min, max int) error { - query := fmt.Sprintf(`UPDATE node_contract SET resources_used_id = CONCAT('contract-resources-',split_part(id, '-', -1)) - WHERE CAST(split_part(id, '-', -1) AS INTEGER) BETWEEN %d AND %d;`, min, max) - if _, err := db.Exec(query); err != nil { - return fmt.Errorf("failed to update node contract resource id: %w", err) - } - return nil -} -func generateData(db *sql.DB) error { - if err := generateTwins(db); err != nil { - return fmt.Errorf("failed to genrate twins: %w", err) - } - - if err := generateFarms(db); err != nil { +func generateData(db *sql.DB, seed int) error { + generator := crafter.NewCrafter(db, + seed, + NodeCount, + FarmCount, + TwinCount, + PublicIPCount, + NodeContractCount, + NameContractCount, + RentContractCount, + 1, + 1, + 1, + 1, + 1, + 1) + + if err := generator.GenerateTwins(); err != nil { + return fmt.Errorf("failed to generate twins: %w", err) + } + + if err := generator.GenerateFarms(); err != nil { return fmt.Errorf("failed to generate farms: %w", err) } - if err := generateNodes(db); err != nil { + if err := generator.GenerateNodes(); err != nil { return fmt.Errorf("failed to generate nodes: %w", err) } - if err := generateContracts(db); err != nil { + if err := generator.GenerateContracts(); err != nil { return fmt.Errorf("failed to generate contracts: %w", err) } - if err := generatePublicIPs(db); err != nil { + if err := generator.GeneratePublicIPs(); err != nil { return fmt.Errorf("failed to generate public ips: %w", err) } - if err := generateNodeGPUs(db); err != nil { + if err := generator.GenerateNodeGPUs(); err != nil { return fmt.Errorf("failed to generate node gpus: %w", err) } - if err := generateCountries(db); err != nil { + if err := generator.GenerateCountries(); err != nil { return fmt.Errorf("failed to generate countries: %w", err) } return nil diff --git a/grid-proxy/tools/db/schema.sql b/grid-proxy/tools/db/schema.sql index 8e31a7112..1ca4fc54e 100644 --- a/grid-proxy/tools/db/schema.sql +++ b/grid-proxy/tools/db/schema.sql @@ -2,8 +2,8 @@ -- PostgreSQL database dump -- --- Dumped from database version 15.3 (Debian 15.3-1.pgdg110+1) --- Dumped by pg_dump version 15.3 (Ubuntu 15.3-0ubuntu0.23.04.1) +-- Dumped from database version 16.1 (Debian 16.1-1.pgdg120+1) +-- Dumped by pg_dump version 16.1 (Debian 16.1-1.pgdg120+1) SET statement_timeout = 0; SET lock_timeout = 0; @@ -17,32 +17,18 @@ SET client_min_messages = warning; SET row_security = off; -- --- Name: substrate_threefold_status; Type: SCHEMA; Schema: -; Owner: postgres +-- Name: squid_processor; Type: SCHEMA; Schema: -; Owner: postgres -- -CREATE SCHEMA substrate_threefold_status; +CREATE SCHEMA squid_processor; -ALTER SCHEMA substrate_threefold_status OWNER TO postgres; - +ALTER SCHEMA squid_processor OWNER TO postgres; SET default_tablespace = ''; SET default_table_access_method = heap; --- --- Name: account; Type: TABLE; Schema: public; Owner: postgres --- - -CREATE TABLE public.account ( - id character varying NOT NULL, - wallet text NOT NULL, - balance numeric NOT NULL -); - - -ALTER TABLE public.account OWNER TO postgres; - -- -- Name: burn_transaction; Type: TABLE; Schema: public; Owner: postgres -- @@ -96,7 +82,7 @@ CREATE TABLE public.contract_resources ( sru numeric NOT NULL, cru numeric NOT NULL, mru numeric NOT NULL, - contract_id character varying NOT NULL + contract_id character varying ); @@ -145,7 +131,7 @@ CREATE TABLE public.entity_proof ( id character varying NOT NULL, entity_id integer NOT NULL, signature text NOT NULL, - twin_rel_id character varying NOT NULL + twin_rel_id character varying ); @@ -195,20 +181,6 @@ CREATE TABLE public.farming_policy ( ALTER TABLE public.farming_policy OWNER TO postgres; --- --- Name: historical_balance; Type: TABLE; Schema: public; Owner: postgres --- - -CREATE TABLE public.historical_balance ( - id character varying NOT NULL, - balance numeric NOT NULL, - "timestamp" numeric NOT NULL, - account_id character varying NOT NULL -); - - -ALTER TABLE public.historical_balance OWNER TO postgres; - -- -- Name: interfaces; Type: TABLE; Schema: public; Owner: postgres -- @@ -218,7 +190,7 @@ CREATE TABLE public.interfaces ( name text NOT NULL, mac text NOT NULL, ips text NOT NULL, - node_id character varying NOT NULL + node_id character varying ); @@ -263,7 +235,7 @@ CREATE SEQUENCE public.migrations_id_seq CACHE 1; -ALTER TABLE public.migrations_id_seq OWNER TO postgres; +ALTER SEQUENCE public.migrations_id_seq OWNER TO postgres; -- -- Name: migrations_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: postgres @@ -297,7 +269,8 @@ CREATE TABLE public.name_contract ( twin_id integer NOT NULL, name text NOT NULL, created_at numeric NOT NULL, - state character varying(11) NOT NULL + state character varying(11) NOT NULL, + solution_provider_id integer ); @@ -323,10 +296,11 @@ CREATE TABLE public.node ( serial_number text, created_at numeric NOT NULL, updated_at numeric NOT NULL, - location_id character varying NOT NULL, + location_id character varying, certification character varying(9), connection_price integer, power jsonb, + dedicated boolean NOT NULL, extra_fee numeric ); @@ -348,27 +322,13 @@ CREATE TABLE public.node_contract ( number_of_public_i_ps integer NOT NULL, created_at numeric NOT NULL, resources_used_id character varying, - state character varying(11) NOT NULL + state character varying(11) NOT NULL, + solution_provider_id integer ); ALTER TABLE public.node_contract OWNER TO postgres; --- --- Name: node_gpu; Type: TABLE; Schema: public; Owner: postgres --- - -CREATE TABLE public.node_gpu ( - node_twin_id bigint NOT NULL, - id text NOT NULL, - vendor text, - device text, - contract bigint -); - - -ALTER TABLE public.node_gpu OWNER TO postgres; - -- -- Name: node_resources_free; Type: TABLE; Schema: public; Owner: postgres -- @@ -417,33 +377,6 @@ CREATE TABLE public.node_resources_used ( ALTER TABLE public.node_resources_used OWNER TO postgres; --- --- Name: nodes_resources_view; Type: VIEW; Schema: public; Owner: postgres --- - -CREATE VIEW public.nodes_resources_view AS - SELECT node.node_id, - COALESCE(sum(contract_resources.cru), (0)::numeric) AS used_cru, - (COALESCE(sum(contract_resources.mru), (0)::numeric) + (GREATEST(((node_resources_total.mru / (10)::numeric))::bigint, '2147483648'::bigint))::numeric) AS used_mru, - COALESCE(sum(contract_resources.hru), (0)::numeric) AS used_hru, - (COALESCE(sum(contract_resources.sru), (0)::numeric) + ('21474836480'::bigint)::numeric) AS used_sru, - ((node_resources_total.mru - COALESCE(sum(contract_resources.mru), (0)::numeric)) - (GREATEST(((node_resources_total.mru / (10)::numeric))::bigint, '2147483648'::bigint))::numeric) AS free_mru, - (node_resources_total.hru - COALESCE(sum(contract_resources.hru), (0)::numeric)) AS free_hru, - ((node_resources_total.sru - COALESCE(sum(contract_resources.sru), (0)::numeric)) - ('21474836480'::bigint)::numeric) AS free_sru, - COALESCE(node_resources_total.cru, (0)::numeric) AS total_cru, - COALESCE(node_resources_total.mru, (0)::numeric) AS total_mru, - COALESCE(node_resources_total.hru, (0)::numeric) AS total_hru, - COALESCE(node_resources_total.sru, (0)::numeric) AS total_sru, - COALESCE(count(DISTINCT node_contract.state), (0)::bigint) AS states - FROM (((public.contract_resources - JOIN public.node_contract node_contract ON ((((node_contract.resources_used_id)::text = (contract_resources.id)::text) AND ((node_contract.state)::text = ANY ((ARRAY['Created'::character varying, 'GracePeriod'::character varying])::text[]))))) - RIGHT JOIN public.node node ON ((node.node_id = node_contract.node_id))) - JOIN public.node_resources_total node_resources_total ON (((node_resources_total.node_id)::text = (node.id)::text))) - GROUP BY node.node_id, node_resources_total.mru, node_resources_total.sru, node_resources_total.hru, node_resources_total.cru; - - -ALTER TABLE public.nodes_resources_view OWNER TO postgres; - -- -- Name: nru_consumption; Type: TABLE; Schema: public; Owner: postgres -- @@ -506,11 +439,23 @@ CREATE TABLE public.public_ip ( gateway text NOT NULL, ip text NOT NULL, contract_id numeric NOT NULL, - farm_id character varying NOT NULL + farm_id character varying ); +ALTER TABLE public.public_ip OWNER TO postgres; +-- +-- Name: node_gpu; Type: TABLE; Schema: public; Owner: postgres +-- -ALTER TABLE public.public_ip OWNER TO postgres; +CREATE TABLE IF NOT EXISTS public.node_gpu ( + id text NOT NULL, + node_twin_id bigint NOT NULL, + vendor text, + device text, + contract bigint +); + +ALTER TABLE public.node_gpu OWNER TO postgres; -- -- Name: refund_transaction; Type: TABLE; Schema: public; Owner: postgres @@ -538,26 +483,65 @@ CREATE TABLE public.rent_contract ( twin_id integer NOT NULL, node_id integer NOT NULL, created_at numeric NOT NULL, - state character varying(11) NOT NULL + state character varying(11) NOT NULL, + solution_provider_id integer ); ALTER TABLE public.rent_contract OWNER TO postgres; -- --- Name: transfer; Type: TABLE; Schema: public; Owner: postgres +-- Name: service_contract; Type: TABLE; Schema: public; Owner: postgres -- -CREATE TABLE public.transfer ( +CREATE TABLE public.service_contract ( id character varying NOT NULL, - "from" text NOT NULL, - "to" text NOT NULL, - amount numeric NOT NULL, - "timestamp" numeric NOT NULL + service_contract_id numeric NOT NULL, + service_twin_id integer NOT NULL, + consumer_twin_id integer NOT NULL, + base_fee numeric NOT NULL, + variable_fee numeric NOT NULL, + metadata text NOT NULL, + accepted_by_service boolean NOT NULL, + accepted_by_consumer boolean NOT NULL, + last_bill numeric NOT NULL, + state character varying(14) NOT NULL ); -ALTER TABLE public.transfer OWNER TO postgres; +ALTER TABLE public.service_contract OWNER TO postgres; + +-- +-- Name: service_contract_bill; Type: TABLE; Schema: public; Owner: postgres +-- + +CREATE TABLE public.service_contract_bill ( + id character varying NOT NULL, + service_contract_id numeric NOT NULL, + variable_amount numeric NOT NULL, + "window" numeric NOT NULL, + metadata text, + amount numeric NOT NULL +); + + +ALTER TABLE public.service_contract_bill OWNER TO postgres; + +-- +-- Name: solution_provider; Type: TABLE; Schema: public; Owner: postgres +-- + +CREATE TABLE public.solution_provider ( + id character varying NOT NULL, + solution_provider_id numeric NOT NULL, + description text NOT NULL, + link text NOT NULL, + approved boolean NOT NULL, + providers jsonb +); + + +ALTER TABLE public.solution_provider OWNER TO postgres; -- -- Name: twin; Type: TABLE; Schema: public; Owner: postgres @@ -575,22 +559,6 @@ CREATE TABLE public.twin ( ALTER TABLE public.twin OWNER TO postgres; --- --- Name: typeorm_metadata; Type: TABLE; Schema: public; Owner: postgres --- - -CREATE TABLE public.typeorm_metadata ( - type character varying NOT NULL, - database character varying, - schema character varying, - "table" character varying, - name character varying, - value text -); - - -ALTER TABLE public.typeorm_metadata OWNER TO postgres; - -- -- Name: uptime_event; Type: TABLE; Schema: public; Owner: postgres -- @@ -606,16 +574,16 @@ CREATE TABLE public.uptime_event ( ALTER TABLE public.uptime_event OWNER TO postgres; -- --- Name: status; Type: TABLE; Schema: substrate_threefold_status; Owner: postgres +-- Name: status; Type: TABLE; Schema: squid_processor; Owner: postgres -- -CREATE TABLE substrate_threefold_status.status ( +CREATE TABLE squid_processor.status ( id integer NOT NULL, height integer NOT NULL ); -ALTER TABLE substrate_threefold_status.status OWNER TO postgres; +ALTER TABLE squid_processor.status OWNER TO postgres; -- -- Name: migrations id; Type: DEFAULT; Schema: public; Owner: postgres @@ -656,6 +624,14 @@ ALTER TABLE ONLY public.mint_transaction ADD CONSTRAINT "PK_19f4328320501dfd14e2bae0855" PRIMARY KEY (id); +-- +-- Name: service_contract_bill PK_1fd26292c0913e974b774342fa7; Type: CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.service_contract_bill + ADD CONSTRAINT "PK_1fd26292c0913e974b774342fa7" PRIMARY KEY (id); + + -- -- Name: burn_transaction PK_20ec76c5c56dd6b47dec5f0aaa8; Type: CONSTRAINT; Schema: public; Owner: postgres -- @@ -688,14 +664,6 @@ ALTER TABLE ONLY public.entity ADD CONSTRAINT "PK_50a7741b415bc585fcf9c984332" PRIMARY KEY (id); --- --- Name: account PK_54115ee388cdb6d86bb4bf5b2ea; Type: CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.account - ADD CONSTRAINT "PK_54115ee388cdb6d86bb4bf5b2ea" PRIMARY KEY (id); - - -- -- Name: contract_resources PK_557de19994fcca90916e8c6582f; Type: CONSTRAINT; Schema: public; Owner: postgres -- @@ -720,14 +688,6 @@ ALTER TABLE ONLY public.farming_policy ADD CONSTRAINT "PK_5d2ec9534104f44e4d989c4e82f" PRIMARY KEY (id); --- --- Name: historical_balance PK_74ac29ad0bdffb6d1281a1e17e8; Type: CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.historical_balance - ADD CONSTRAINT "PK_74ac29ad0bdffb6d1281a1e17e8" PRIMARY KEY (id); - - -- -- Name: refund_transaction PK_74ffc5427c595968dd777f71bf4; Type: CONSTRAINT; Schema: public; Owner: postgres -- @@ -848,6 +808,14 @@ ALTER TABLE ONLY public.nru_consumption ADD CONSTRAINT "PK_ca7956fb8fcdb7198737387d9a8" PRIMARY KEY (id); +-- +-- Name: solution_provider PK_dbb1dd40ae8f70dc9bbe2ce6347; Type: CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.solution_provider + ADD CONSTRAINT "PK_dbb1dd40ae8f70dc9bbe2ce6347" PRIMARY KEY (id); + + -- -- Name: public_ip PK_f170b0b519632730f41d2ef78f4; Type: CONSTRAINT; Schema: public; Owner: postgres -- @@ -857,11 +825,11 @@ ALTER TABLE ONLY public.public_ip -- --- Name: transfer PK_fd9ddbdd49a17afcbe014401295; Type: CONSTRAINT; Schema: public; Owner: postgres +-- Name: service_contract PK_ff58318f8230b8053067edd0343; Type: CONSTRAINT; Schema: public; Owner: postgres -- -ALTER TABLE ONLY public.transfer - ADD CONSTRAINT "PK_fd9ddbdd49a17afcbe014401295" PRIMARY KEY (id); +ALTER TABLE ONLY public.service_contract + ADD CONSTRAINT "PK_ff58318f8230b8053067edd0343" PRIMARY KEY (id); -- @@ -901,7 +869,7 @@ ALTER TABLE ONLY public.node_resources_total -- ALTER TABLE ONLY public.node_gpu - ADD CONSTRAINT node_gpu_pkey PRIMARY KEY (node_twin_id, id); + ADD CONSTRAINT node_gpu_pkey PRIMARY KEY (id); @@ -909,7 +877,7 @@ ALTER TABLE ONLY public.node_gpu -- Name: status status_pkey; Type: CONSTRAINT; Schema: substrate_threefold_status; Owner: postgres -- -ALTER TABLE ONLY substrate_threefold_status.status +ALTER TABLE ONLY squid_processor.status ADD CONSTRAINT status_pkey PRIMARY KEY (id); @@ -920,13 +888,6 @@ ALTER TABLE ONLY substrate_threefold_status.status CREATE INDEX "IDX_23937641f28c607f061dab4694" ON public.interfaces USING btree (node_id); --- --- Name: IDX_383ff006e4b59db91d32cb891e; Type: INDEX; Schema: public; Owner: postgres --- - -CREATE INDEX "IDX_383ff006e4b59db91d32cb891e" ON public.historical_balance USING btree (account_id); - - -- -- Name: IDX_3d9cbf30c68b79a801e1d5c9b4; Type: INDEX; Schema: public; Owner: postgres -- @@ -998,14 +959,6 @@ ALTER TABLE ONLY public.interfaces ADD CONSTRAINT "FK_23937641f28c607f061dab4694b" FOREIGN KEY (node_id) REFERENCES public.node(id); --- --- Name: historical_balance FK_383ff006e4b59db91d32cb891e9; Type: FK CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.historical_balance - ADD CONSTRAINT "FK_383ff006e4b59db91d32cb891e9" FOREIGN KEY (account_id) REFERENCES public.account(id); - - -- -- Name: entity_proof FK_3d9cbf30c68b79a801e1d5c9b41; Type: FK CONSTRAINT; Schema: public; Owner: postgres -- diff --git a/grid-proxy/tools/db/test.go b/grid-proxy/tools/db/test.go deleted file mode 100644 index 0a296ff41..000000000 --- a/grid-proxy/tools/db/test.go +++ /dev/null @@ -1,79 +0,0 @@ -package main - -// func unusedNecessaryColumns() string { -// return ` -// INSERT INTO location VALUES ('1', '2', '3'); -// ` -// } -// func cleanup() string { -// return ` -// DELETE FROM node_resources_total WHERE id = '1'; -// DELETE FROM node WHERE id = '112'; -// DELETE FROM location WHERE id = '1'; -// DELETE FROM farm WHERE id = '112'; -// UPDATE node_contract SET resources_used_id = NULL WHERE id = '112'; -// DELETE FROM contract_resources WHERE id = '112'; -// DELETE FROM node_contract WHERE id = '112'; -// ` -// } -// func test(db *sql.DB) error { -// node := node{ -// id: "112", -// location_id: "1", -// node_id: 112, -// } -// total_resources := node_resources_total{ -// id: "1", -// hru: 1, -// sru: 2, -// cru: 3, -// mru: 5, -// node_id: "112", -// } -// farm := farm{ -// id: "112", -// farm_id: 112, -// name: "345", -// certification_type: "123", -// } -// contract_resources := contract_resources{ -// id: "112", -// hru: 1, -// sru: 2, -// cru: 3, -// mru: 4, -// contract_id: "112", -// } -// node_contract := node_contract{ -// id: "112", -// contract_id: 112, -// twin_id: 112, -// node_id: 112, -// resources_used_id: "", -// deployment_data: "123", -// deployment_hash: "123", -// state: "Created", -// } -// if _, err := db.Exec(unusedNecessaryColumns()); err != nil { -// return err -// } -// if _, err := db.Exec(insertQuery(&node)); err != nil { -// return err -// } -// if _, err := db.Exec(insertQuery(&total_resources)); err != nil { -// return err -// } -// if _, err := db.Exec(insertQuery(&farm)); err != nil { -// return err -// } -// if _, err := db.Exec(insertQuery(&node_contract)); err != nil { -// return err -// } -// if _, err := db.Exec(insertQuery(&contract_resources)); err != nil { -// return err -// } -// if _, err := db.Exec(setContractResource("112", "112")); err != nil { -// return err -// } -// return nil -// } diff --git a/grid-proxy/tools/db/types.go b/grid-proxy/tools/db/types.go deleted file mode 100644 index b98da210c..000000000 --- a/grid-proxy/tools/db/types.go +++ /dev/null @@ -1,145 +0,0 @@ -package main - -type contract_resources struct { - id string - hru uint64 - sru uint64 - cru uint64 - mru uint64 - contract_id string -} -type farm struct { - id string - grid_version uint64 - farm_id uint64 - name string - twin_id uint64 - pricing_policy_id uint64 - certification string - stellar_address string - dedicated_farm bool -} - -type node struct { - id string - grid_version uint64 - node_id uint64 - farm_id uint64 - twin_id uint64 - country string - city string - uptime uint64 - created uint64 - farming_policy_id uint64 - certification string - secure bool - virtualized bool - serial_number string - created_at uint64 - updated_at uint64 - location_id string - power nodePower `gorm:"type:jsonb"` - extra_fee uint64 -} - -type nodePower struct { - State string `json:"state"` - Target string `json:"target"` -} -type twin struct { - id string - grid_version uint64 - twin_id uint64 - account_id string - relay string - public_key string -} - -type public_ip struct { - id string - gateway string - ip string - contract_id uint64 - farm_id string -} -type node_contract struct { - id string - grid_version uint64 - contract_id uint64 - twin_id uint64 - node_id uint64 - deployment_data string - deployment_hash string - number_of_public_i_ps uint64 - state string - created_at uint64 - resources_used_id string -} -type node_resources_total struct { - id string - hru uint64 - sru uint64 - cru uint64 - mru uint64 - node_id string -} -type public_config struct { - id string - ipv4 string - ipv6 string - gw4 string - gw6 string - domain string - node_id string -} -type rent_contract struct { - id string - grid_version uint64 - contract_id uint64 - twin_id uint64 - node_id uint64 - state string - created_at uint64 -} -type location struct { - id string - longitude string - latitude string -} - -type contract_bill_report struct { - id string - contract_id uint64 - discount_received string - amount_billed uint64 - timestamp uint64 -} - -type name_contract struct { - id string - grid_version uint64 - contract_id uint64 - twin_id uint64 - name string - state string - created_at uint64 -} - -type node_gpu struct { - node_twin_id uint64 - id string - vendor string - device string - contract int -} - -type country struct { - id string - country_id uint64 - code string - name string - region string - subregion string - lat string - long string -}