From 7f16f484c2ee58088e713bef603edfa7b2e2910e Mon Sep 17 00:00:00 2001 From: Omar Abdulaziz Date: Sun, 10 Dec 2023 14:43:35 +0200 Subject: [PATCH 01/39] add bench tests Co-authored-by: Mario Bassem --- grid-proxy/Makefile | 10 ++++ grid-proxy/tests/queries/bench_test.go | 81 ++++++++++++++++++++++++++ grid-proxy/tests/queries/main_test.go | 7 +++ 3 files changed, 98 insertions(+) create mode 100644 grid-proxy/tests/queries/bench_test.go diff --git a/grid-proxy/Makefile b/grid-proxy/Makefile index e774cf98e..7ed767520 100644 --- a/grid-proxy/Makefile +++ b/grid-proxy/Makefile @@ -124,3 +124,13 @@ spelling: staticcheck: @echo "Running $@" @$(shell go env GOPATH)/bin/staticcheck -- ./... + +bench: + @cd tests/queries/ &&\ + go test -v -bench Bench -run notests -count 5\ + --seed 13 \ + --postgres-host $(PQ_HOST) \ + --postgres-db tfgrid-graphql \ + --postgres-password postgres \ + --postgres-user postgres \ + --endpoint http://localhost:8080 \ No newline at end of file diff --git a/grid-proxy/tests/queries/bench_test.go b/grid-proxy/tests/queries/bench_test.go new file mode 100644 index 000000000..e5c431099 --- /dev/null +++ b/grid-proxy/tests/queries/bench_test.go @@ -0,0 +1,81 @@ +package test + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + proxytypes "github.com/threefoldtech/tfgrid-sdk-go/grid-proxy/pkg/types" +) + +func BenchmarkNodes(b *testing.B) { + for i := 0; i < b.N; i++ { + b.StopTimer() + agg := calcNodesAggregates(&data) + l := proxytypes.Limit{ + Size: 999999999999, + Page: 1, + RetCount: true, + } + f, err := randomNodeFilter(&agg) + require.NoError(b, err) + + b.StartTimer() + _, _, err = DBClient.GetNodes(context.Background(), f, l) + require.NoError(b, err) + } +} + +func BenchmarkFarms(b *testing.B) { + for i := 0; i < b.N; i++ { + b.StopTimer() + agg := calcFarmsAggregates(&data) + l := proxytypes.Limit{ + Size: 999999999999, + Page: 1, + RetCount: true, + } + f, err := randomFarmsFilter(&agg) + require.NoError(b, err) + + b.StartTimer() + _, _, err = DBClient.GetFarms(context.Background(), f, l) + require.NoError(b, err) + } +} + +func BenchmarkContracts(b *testing.B) { + for i := 0; i < b.N; i++ { + b.StopTimer() + agg := calcContractsAggregates(&data) + l := proxytypes.Limit{ + Size: 999999999999, + Page: 1, + RetCount: true, + } + f, err := randomContractsFilter(&agg) + require.NoError(b, err) + + b.StartTimer() + _, _, err = DBClient.GetContracts(context.Background(), f, l) + require.NoError(b, err) + } +} + +func BenchmarkTwins(b *testing.B) { + for i := 0; i < b.N; i++ { + b.StopTimer() + agg := calcTwinsAggregates(&data) + l := proxytypes.Limit{ + Size: 999999999999, + Page: 1, + RetCount: true, + } + f, err := randomTwinsFilter(&agg) + require.NoError(b, err) + + b.StartTimer() + _, _, err = DBClient.GetTwins(context.Background(), f, l) + require.NoError(b, err) + } +} diff --git a/grid-proxy/tests/queries/main_test.go b/grid-proxy/tests/queries/main_test.go index 9d3af7036..49b90b102 100644 --- a/grid-proxy/tests/queries/main_test.go +++ b/grid-proxy/tests/queries/main_test.go @@ -12,6 +12,8 @@ import ( _ "github.com/lib/pq" "github.com/pkg/errors" + "github.com/threefoldtech/tfgrid-sdk-go/grid-proxy/internal/explorer/db" + proxyDB "github.com/threefoldtech/tfgrid-sdk-go/grid-proxy/internal/explorer/db" proxyclient "github.com/threefoldtech/tfgrid-sdk-go/grid-proxy/pkg/client" mock "github.com/threefoldtech/tfgrid-sdk-go/grid-proxy/tests/queries/mock_client" ) @@ -30,6 +32,7 @@ var ( mockClient proxyclient.Client data mock.DBData gridProxyClient proxyclient.Client + DBClient db.Database ) func parseCmdline() { @@ -62,6 +65,10 @@ func TestMain(m *testing.M) { if err != nil { panic(err) } + DBClient, err = proxyDB.NewPostgresDatabase(POSTGRES_HOST, POSTGRES_PORT, POSTGRES_USER, POSTGRES_PASSSWORD, POSTGRES_DB, 80) + if err != nil { + panic(err) + } mockClient = mock.NewGridProxyMockClient(data) gridProxyClient = proxyclient.NewClient(ENDPOINT) From 4480a86d5257d7acc483968cd47a4610e32fdf01 Mon Sep 17 00:00:00 2001 From: Omar Abdulaziz Date: Sun, 10 Dec 2023 16:40:25 +0200 Subject: [PATCH 02/39] wip: add startup materialized views --- grid-proxy/internal/explorer/db/postgres.go | 127 ++++++++++++++++++-- 1 file changed, 117 insertions(+), 10 deletions(-) diff --git a/grid-proxy/internal/explorer/db/postgres.go b/grid-proxy/internal/explorer/db/postgres.go index e7a3845eb..5b08166db 100644 --- a/grid-proxy/internal/explorer/db/postgres.go +++ b/grid-proxy/internal/explorer/db/postgres.go @@ -31,8 +31,109 @@ var ( ErrContractNotFound = errors.New("contract not found") ) +var views = []string{ + // Node view needed for join on farm + ` + CREATE MATERIALIZED VIEW node_materialized_view AS + SELECT + node.node_id, + node.twin_id, + node.farm_id, + node.power, + node.updated_at, + node.certification, + node.country, + nodes_resources_view.free_mru, + nodes_resources_view.free_hru, + nodes_resources_view.free_sru, + COALESCE(rent_contract.contract_id, 0) as rent_contract_id, + COALESCE(rent_contract.twin_id, 0) as renter, + COALESCE(node_gpu.id, '') as gpu_id + FROM + node + LEFT JOIN nodes_resources_view ON node.node_id = nodes_resources_view.node_id + LEFT JOIN rent_contract ON node.node_id = rent_contract.node_id AND rent_contract.state IN ('Created', 'GracePeriod') + LEFT JOIN node_gpu ON node.twin_id = node_gpu.node_twin_id; + + -- define a refresher function + CREATE OR REPLACE FUNCTION refresh_node_materialized_view() + RETURNS TRIGGER AS $$ + BEGIN + REFRESH MATERIALIZED VIEW node_materialized_view; + RETURN NULL; + END; + $$ LANGUAGE plpgsql; + + -- trigger the refresh on each node update + CREATE TRIGGER refresh_node_materialized_trigger + AFTER UPDATE ON node + FOR EACH ROW + EXECUTE FUNCTION refresh_node_materialized_view(); + + -- trigger a backup refresh each 6 hours + SELECT cron.schedule('0 */6 * * *', 'SELECT refresh_node_materialized_view()'); + `, + // Farm view + ` + CREATE MATERIALIZED VIEW farm_materialized_view AS + SELECT + farm.id, + farm.farm_id, + farm.name, + farm.twin_id, + farm.pricing_policy_id, + farm.certification, + farm.stellar_address, + farm.dedicated_farm as dedicated, + COALESCE(public_ips.public_ips, '[]') as public_ips, + bool_or(node_materialized_view.rent_contract_id != 0) as has_rent_contract + FROM + farm + + LEFT JOIN node_materialized_view ON node_materialized_view.farm_id = farm.farm_id + LEFT JOIN country ON node_materialized_view.country = country.name + + LEFT JOIN ( + SELECT + p1.farm_id, + COUNT(p1.id) total_ips, + COUNT(CASE WHEN p2.contract_id = 0 THEN 1 END) free_ips + FROM + public_ip p1 + LEFT JOIN public_ip p2 ON p1.id = p2.id + GROUP BY p1.farm_id + ) public_ip_count on public_ip_count.farm_id = farm.id + + LEFT JOIN ( + SELECT + farm_id, + jsonb_agg(jsonb_build_object('id', id, 'ip', ip, 'contract_id', contract_id, 'gateway', gateway)) as public_ips + FROM + public_ip + GROUP BY farm_id + ) public_ips on public_ips.farm_id = farm.id + + + GROUP BY + farm.id, + farm.farm_id, + farm.name, + farm.twin_id, + farm.pricing_policy_id, + farm.certification, + farm.stellar_address, + farm.dedicated_farm, + COALESCE(public_ips.public_ips, '[]'); + `, +} + const ( setupPostgresql = ` + CREATE INDEX IF NOT EXISTS idx_node_id ON public.node(node_id); + CREATE INDEX IF NOT EXISTS idx_twin_id ON public.twin(twin_id); + CREATE INDEX IF NOT EXISTS idx_farm_id ON public.farm(farm_id); + CREATE INDEX IF NOT EXISTS idx_contract_id ON public.node_contract(contract_id); + CREATE OR REPLACE VIEW nodes_resources_view AS SELECT node.node_id, COALESCE(sum(contract_resources.cru), 0) as used_cru, @@ -121,7 +222,9 @@ func NewPostgresDatabase(host string, port int, user, password, dbname string, m connString := fmt.Sprintf("host=%s port=%d user=%s "+ "password=%s dbname=%s sslmode=disable", host, port, user, password, dbname) - gormDB, err := gorm.Open(postgres.Open(connString), &gorm.Config{}) + gormDB, err := gorm.Open(postgres.Open(connString), &gorm.Config{ + Logger: logger.Default.LogMode(logger.Info), + }) if err != nil { return nil, errors.Wrap(err, "failed to create orm wrapper around db") } @@ -155,8 +258,12 @@ func (d *PostgresDatabase) Close() error { } func (d *PostgresDatabase) initialize() error { - res := d.gormDB.Exec(setupPostgresql) - return res.Error + err := d.gormDB.Exec(setupPostgresql).Error + for _, view := range views { + err = d.gormDB.Exec(view).Error + } + + return err } // GetStats returns aggregate info about the grid @@ -382,13 +489,13 @@ func (d *PostgresDatabase) farmTableQuery() *gorm.DB { LEFT JOIN public_ip p2 ON p1.id = p2.id GROUP BY p1.farm_id ) public_ip_count on public_ip_count.farm_id = farm.id`). - Joins(`left join ( - select - farm_id, - jsonb_agg(jsonb_build_object('id', id, 'ip', ip, 'contract_id', contract_id, 'gateway', gateway)) as public_ips - from public_ip - GROUP BY farm_id - ) public_ips on public_ips.farm_id = farm.id`). + // Joins(`left join ( + // select + // farm_id, + // jsonb_agg(jsonb_build_object('id', id, 'ip', ip, 'contract_id', contract_id, 'gateway', gateway)) as public_ips + // from public_ip + // GROUP BY farm_id + // ) public_ips on public_ips.farm_id = farm.id`). Joins( "LEFT JOIN country ON node.country = country.name", ). From 184c8910fa9635ab6874e478aed5fb2990be2eac Mon Sep 17 00:00:00 2001 From: Omar Abdulaziz Date: Mon, 11 Dec 2023 18:36:33 +0200 Subject: [PATCH 03/39] wip: filter node related fields first before join node table to farm --- grid-proxy/internal/explorer/db/postgres.go | 144 ++++++++++---------- grid-proxy/tools/db/generate.go | 8 +- 2 files changed, 78 insertions(+), 74 deletions(-) diff --git a/grid-proxy/internal/explorer/db/postgres.go b/grid-proxy/internal/explorer/db/postgres.go index 5b08166db..a3ab35fdd 100644 --- a/grid-proxy/internal/explorer/db/postgres.go +++ b/grid-proxy/internal/explorer/db/postgres.go @@ -32,7 +32,6 @@ var ( ) var views = []string{ - // Node view needed for join on farm ` CREATE MATERIALIZED VIEW node_materialized_view AS SELECT @@ -73,7 +72,6 @@ var views = []string{ -- trigger a backup refresh each 6 hours SELECT cron.schedule('0 */6 * * *', 'SELECT refresh_node_materialized_view()'); `, - // Farm view ` CREATE MATERIALIZED VIEW farm_materialized_view AS SELECT @@ -259,9 +257,10 @@ func (d *PostgresDatabase) Close() error { func (d *PostgresDatabase) initialize() error { err := d.gormDB.Exec(setupPostgresql).Error - for _, view := range views { - err = d.gormDB.Exec(view).Error - } + // concatenate in one string before exec + // for _, view := range views { + // err = d.gormDB.Exec(view).Error + // } return err } @@ -320,11 +319,7 @@ func (d *PostgresDatabase) GetStats(ctx context.Context, filter types.StatsFilte } query := d.gormDB.WithContext(ctx). Table("node"). - Joins( - `RIGHT JOIN public_config - ON node.id = public_config.node_id - `, - ) + Joins("RIGHT JOIN public_config ON node.id = public_config.node_id") if res := query.Where(condition).Where("COALESCE(public_config.ipv4, '') != '' OR COALESCE(public_config.ipv6, '') != ''").Count(&stats.AccessNodes); res.Error != nil { return stats, errors.Wrap(res.Error, "couldn't get access node count") @@ -404,7 +399,9 @@ func (d *PostgresDatabase) GetNode(ctx context.Context, nodeID uint32) (Node, er // GetFarm return farm info func (d *PostgresDatabase) GetFarm(ctx context.Context, farmID uint32) (Farm, error) { - q := d.farmTableQuery() + //TODO: make a quick table here without needs to joins + // q := d.farmTableQuery() + q := d.gormDB q = q.WithContext(ctx) q = q.Where("farm.farm_id = ?", farmID) var farm Farm @@ -443,7 +440,7 @@ func printQuery(query string, args ...interface{}) { fmt.Printf("node query: %s", query) } -func (d *PostgresDatabase) farmTableQuery() *gorm.DB { +func (d *PostgresDatabase) farmTableQuery(nodeQuery *gorm.DB) *gorm.DB { return d.gormDB. Table("farm"). Select( @@ -456,30 +453,9 @@ func (d *PostgresDatabase) farmTableQuery() *gorm.DB { "farm.stellar_address", "farm.dedicated_farm as dedicated", "COALESCE(public_ips.public_ips, '[]') as public_ips", - "bool_or(node.rent_contract_id != 0)", - ). - Joins( - `left join( - SELECT - node.node_id, - node.twin_id, - node.farm_id, - node.power, - node.updated_at, - node.certification, - node.country, - nodes_resources_view.free_mru, - nodes_resources_view.free_hru, - nodes_resources_view.free_sru, - COALESCE(rent_contract.contract_id, 0) rent_contract_id, - COALESCE(rent_contract.twin_id, 0) renter, - COALESCE(node_gpu.id, '') gpu_id - FROM node - LEFT JOIN nodes_resources_view ON node.node_id = nodes_resources_view.node_id - LEFT JOIN rent_contract ON node.node_id = rent_contract.node_id AND rent_contract.state IN ('Created', 'GracePeriod') - LEFT JOIN node_gpu ON node.twin_id = node_gpu.node_twin_id - ) node on node.farm_id = farm.farm_id`, + "node.rented", ). + Joins("LEFT JOIN (?) AS node ON farm.farm_id = node.farm_id", nodeQuery). Joins(`left join( SELECT p1.farm_id, @@ -489,26 +465,26 @@ func (d *PostgresDatabase) farmTableQuery() *gorm.DB { LEFT JOIN public_ip p2 ON p1.id = p2.id GROUP BY p1.farm_id ) public_ip_count on public_ip_count.farm_id = farm.id`). - // Joins(`left join ( - // select - // farm_id, - // jsonb_agg(jsonb_build_object('id', id, 'ip', ip, 'contract_id', contract_id, 'gateway', gateway)) as public_ips - // from public_ip - // GROUP BY farm_id - // ) public_ips on public_ips.farm_id = farm.id`). + Joins(`left join ( + select + farm_id, + jsonb_agg(jsonb_build_object('id', id, 'ip', ip, 'contract_id', contract_id, 'gateway', gateway)) as public_ips + from public_ip + GROUP BY farm_id + ) public_ips on public_ips.farm_id = farm.id`). Joins( "LEFT JOIN country ON node.country = country.name", ). Group( `farm.id, - farm.farm_id, - farm.name, - farm.twin_id, - farm.pricing_policy_id, - farm.certification, - farm.stellar_address, - farm.dedicated_farm, - COALESCE(public_ips.public_ips, '[]')`, + farm.farm_id, + farm.name, + farm.twin_id, + farm.pricing_policy_id, + farm.certification, + farm.stellar_address, + farm.dedicated_farm, + COALESCE(public_ips.public_ips, '[]')`, ) } @@ -784,48 +760,71 @@ func (d *PostgresDatabase) shouldRetry(resError error) bool { // GetFarms return farms filtered and paginated func (d *PostgresDatabase) GetFarms(ctx context.Context, filter types.FarmFilter, limit types.Limit) ([]Farm, uint, error) { - q := d.farmTableQuery() - q = q.WithContext(ctx) + // filter the nodes first + nodeQuery := d.gormDB.Table("node"). + Select( + "node.farm_id", + "country", + "certification", + "rent_contract.twin_id as renter", + // check using the aggregated function + "CASE WHEN rent_contract.contract_id IS NOT NULL AND rent_contract.contract_id <> 0 THEN true ELSE false END AS rented", + "COALESCE(node_gpu.id, '') as gpu_id", + ).Joins( + "LEFT JOIN rent_contract ON rent_contract.state IN ('Created', 'GracePeriod') AND rent_contract.node_id = node.node_id", + "LEFT JOIN nodes_resources_view ON nodes_resources_view.node_id = node.node_id", + "LEFT JOIN country ON country.name = node.country", + "LEFT JOIN node_gpu ON node.twin_id = node_gpu.node_twin_id", + ).Group( + `node.farm_id, + rent_contract.twin_id, + node_gpu.id, + node.certification, + node.country`, + ) + if filter.NodeFreeMRU != nil { - q = q.Where("node.free_mru >= ?", *filter.NodeFreeMRU) + nodeQuery = nodeQuery.Where("nodes_resources_view.free_mru >= ?", *filter.NodeFreeMRU) } if filter.NodeFreeHRU != nil { - q = q.Where("node.free_hru >= ?", *filter.NodeFreeHRU) + nodeQuery = nodeQuery.Where("nodes_resources_view.free_hru >= ?", *filter.NodeFreeHRU) } if filter.NodeFreeSRU != nil { - q = q.Where("node.free_sru >= ?", *filter.NodeFreeSRU) + nodeQuery = nodeQuery.Where("nodes_resources_view.free_sru >= ?", *filter.NodeFreeSRU) } if filter.NodeAvailableFor != nil { - q = q.Where("node.renter = ? OR (node.renter = 0 AND farm.dedicated_farm = false)", *filter.NodeAvailableFor) - q = q.Order("CASE WHEN bool_or(node.rent_contract_id != 0) = TRUE THEN 1 ELSE 2 END") + nodeQuery = nodeQuery.Where("node.renter = ? OR (node.renter = 0 AND farm.dedicated_farm = false)", *filter.NodeAvailableFor) + // nodeQuery = nodeQuery.Order("CASE WHEN bool_or(node.rent_contract_id != 0) = TRUE THEN 1 ELSE 2 END") } if filter.NodeHasGPU != nil { - q = q.Where("(node.gpu_id != '') = ?", *filter.NodeHasGPU) + nodeQuery = nodeQuery.Where("(node.gpu_id != '') = ?", *filter.NodeHasGPU) } if filter.NodeRentedBy != nil { - q = q.Where("node.renter = ?", *filter.NodeRentedBy) + nodeQuery = nodeQuery.Where("node.renter = ?", *filter.NodeRentedBy) } if filter.Country != nil { - q = q.Where("LOWER(node.country) = LOWER(?)", *filter.Country) + nodeQuery = nodeQuery.Where("LOWER(node.country) = LOWER(?)", *filter.Country) } if filter.Region != nil { - q = q.Where("LOWER(country.subregion) = LOWER(?)", *filter.Region) + nodeQuery = nodeQuery.Where("LOWER(country.subregion) = LOWER(?)", *filter.Region) } if filter.NodeStatus != nil { condition := nodestatus.DecideNodeStatusCondition(*filter.NodeStatus) - q = q.Where(condition) + nodeQuery = nodeQuery.Where(condition) } if filter.NodeCertified != nil { - q = q.Where("(node.certification = 'Certified') = ?", *filter.NodeCertified) + nodeQuery = nodeQuery.Where("(node.certification = 'Certified') = ?", *filter.NodeCertified) } + q := d.farmTableQuery(nodeQuery) + if filter.FreeIPs != nil { q = q.Where("public_ip_count.free_ips >= ?", *filter.FreeIPs) } @@ -862,13 +861,14 @@ func (d *PostgresDatabase) GetFarms(ctx context.Context, filter types.FarmFilter q = q.Where("farm.dedicated_farm = ?", *filter.Dedicated) } var count int64 - if limit.Randomize || limit.RetCount { - if res := q.Count(&count); res.Error != nil { - return nil, 0, errors.Wrap(res.Error, "couldn't get farm count") - } - } + // if limit.Randomize || limit.RetCount { + // if res := q.Count(&count); res.Error != nil { + // return nil, 0, errors.Wrap(res.Error, "couldn't get farm count") + // } + // } // Sorting + //TODO: sort with available for if limit.Randomize { q = q.Order("random()") } else if limit.SortBy != "" { @@ -885,8 +885,12 @@ func (d *PostgresDatabase) GetFarms(ctx context.Context, filter types.FarmFilter q = q.Limit(int(limit.Size)).Offset(int(limit.Page-1) * int(limit.Size)) var farms []Farm - if res := q.Scan(&farms); res.Error != nil { - return farms, uint(count), errors.Wrap(res.Error, "failed to scan returned farm from database") + err := q.Scan(&farms).Error + if d.shouldRetry(err) { + err = q.Scan(&farms).Error + } + if err != nil { + return farms, 0, errors.Wrap(err, "failed to scan returned farm from database") } return farms, uint(count), nil } diff --git a/grid-proxy/tools/db/generate.go b/grid-proxy/tools/db/generate.go index 18b70a2db..183476808 100644 --- a/grid-proxy/tools/db/generate.go +++ b/grid-proxy/tools/db/generate.go @@ -28,12 +28,12 @@ const ( contractCreatedRatio = .1 // from devnet usedPublicIPsRatio = .9 nodeUpRatio = .5 - nodeCount = 1000 - farmCount = 100 - normalUsers = 2000 + nodeCount = 6000 + farmCount = 600 + normalUsers = 6000 publicIPCount = 1000 twinCount = nodeCount + farmCount + normalUsers - nodeContractCount = 3000 + nodeContractCount = 9000 rentContractCount = 100 nameContractCount = 300 From cae02bb4178efef7fd476f225fde9d72fe22ec22 Mon Sep 17 00:00:00 2001 From: Omar Abdulaziz Date: Tue, 12 Dec 2023 10:49:16 +0200 Subject: [PATCH 04/39] make the retCount default value is false, cause it will double the request time --- grid-proxy/pkg/types/limit.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/grid-proxy/pkg/types/limit.go b/grid-proxy/pkg/types/limit.go index f69016c93..64b9fff28 100644 --- a/grid-proxy/pkg/types/limit.go +++ b/grid-proxy/pkg/types/limit.go @@ -84,7 +84,7 @@ func DefaultLimit() Limit { return Limit{ Size: 50, Page: 1, - RetCount: true, + RetCount: false, Randomize: false, SortBy: "", SortOrder: "", From 07cf15d1cf321a445a75b54179ad527c8d7ee25c Mon Sep 17 00:00:00 2001 From: mario Date: Sun, 17 Dec 2023 12:54:17 +0200 Subject: [PATCH 05/39] wip: improve farms endpoint query Signed-off-by: mario --- grid-proxy/internal/explorer/db/farms.go | 326 ++++++++++++++++++ grid-proxy/internal/explorer/db/postgres.go | 281 +-------------- grid-proxy/internal/explorer/db/view.sql | 85 +++++ grid-proxy/tests/queries/farm_test.go | 10 +- grid-proxy/tests/queries/mock_client/farms.go | 15 +- 5 files changed, 431 insertions(+), 286 deletions(-) create mode 100644 grid-proxy/internal/explorer/db/farms.go create mode 100644 grid-proxy/internal/explorer/db/view.sql diff --git a/grid-proxy/internal/explorer/db/farms.go b/grid-proxy/internal/explorer/db/farms.go new file mode 100644 index 000000000..e1348c4d5 --- /dev/null +++ b/grid-proxy/internal/explorer/db/farms.go @@ -0,0 +1,326 @@ +package db + +import ( + "context" + "fmt" + "strings" + + "github.com/pkg/errors" + "github.com/threefoldtech/tfgrid-sdk-go/grid-proxy/pkg/nodestatus" + "github.com/threefoldtech/tfgrid-sdk-go/grid-proxy/pkg/types" + "gorm.io/gorm" +) + +type farmQuery struct { + d *gorm.DB + q *gorm.DB + nodeQuery *gorm.DB + + hasPublicIPCount bool // to indicate if public ip count subquery was joined + hasNodes bool // to indicate if nodes subquery was joined + hasNodesResources bool // to indicate if nodes_resources_view was joined + hasNodeGPU bool + hasRentContract bool + hasCountry bool +} + +func newFarmsQuery(d *gorm.DB) farmQuery { + return farmQuery{ + d: d, + q: d.Table("farm"). + Select(` + farm.farm_id, + farm.name, + farm.twin_id, + farm.pricing_policy_id, + farm.certification, + farm.stellar_address, + farm.dedicated_farm as dedicated, + COALESCE(public_ips.public_ips, '[]') as public_ips + `). + Joins(` + LEFT JOIN ( + SELECT + farm_id, + jsonb_agg(jsonb_build_object('id', id, 'ip', ip, 'contract_id', contract_id, 'gateway', gateway)) AS public_ips + FROM public_ip + GROUP BY farm_id + ) public_ips ON public_ips.farm_id = farm.id + `). + Group(` + farm.id, + farm.farm_id, + farm.name, + farm.twin_id, + farm.pricing_policy_id, + farm.certification, + farm.stellar_address, + farm.dedicated_farm, + COALESCE(public_ips.public_ips, '[]')`), + } +} + +func (f *farmQuery) joinPublicIPCount() { + if !f.hasPublicIPCount { + f.hasPublicIPCount = true + + f.q = f.q.Joins(` + LEFT JOIN( + SELECT + p1.farm_id, + COUNT(p1.id) total_ips, + COUNT(CASE WHEN p2.contract_id = 0 THEN 1 END) free_ips + FROM public_ip p1 + LEFT JOIN public_ip p2 ON p1.id = p2.id + GROUP BY p1.farm_id + ) public_ip_count ON public_ip_count.farm_id = farm.id + `) + } +} + +func (f *farmQuery) createNodesSubquery() { + if f.nodeQuery == nil { + f.nodeQuery = f.d.Table("node").Select("node.farm_id") + } +} + +func (f *farmQuery) joinNodesResourcesView() { + f.createNodesSubquery() + + if !f.hasNodesResources { + f.hasNodesResources = true + f.nodeQuery = f.nodeQuery.Joins("LEFT JOIN nodes_resources_view ON nodes_resources_view.node_id = node.node_id") + } +} + +func (f *farmQuery) joinGPUTable() { + f.createNodesSubquery() + + if !f.hasNodeGPU { + f.hasNodeGPU = true + + f.nodeQuery = f.nodeQuery.Joins("LEFT JOIN node_gpu ON node.twin_id = node_gpu.node_twin_id"). + Group("node.farm_id") + if f.hasRentContract { + f.nodeQuery = f.nodeQuery.Group(`node.farm_id, rent_contract.twin_id`) + } + } +} + +func (f *farmQuery) joinRentContractTable() { + f.createNodesSubquery() + + if !f.hasRentContract { + f.hasRentContract = true + f.nodeQuery = f.nodeQuery.Joins("LEFT JOIN rent_contract ON rent_contract.state IN ('Created', 'GracePeriod') AND rent_contract.node_id = node.node_id") + } +} + +func (f *farmQuery) joinCountryTable() { + f.createNodesSubquery() + + if !f.hasCountry { + f.hasCountry = true + f.nodeQuery = f.nodeQuery.Joins("LEFT JOIN country ON country.name = node.country") + } +} + +func (f *farmQuery) addNodeResourcesFilter(str string, args ...interface{}) { + // ensure node subquery is joined + f.joinNodesResourcesView() + + // add where clause + f.nodeQuery = f.nodeQuery.Where(str, args...) +} + +func (f *farmQuery) addGPUFilter(str string) { + f.joinGPUTable() + + f.nodeQuery = f.nodeQuery.Where(str) +} + +func (f *farmQuery) addNodeFilter(str string, args ...interface{}) { + f.createNodesSubquery() + + f.nodeQuery = f.nodeQuery.Where(str, args...) +} + +func (f *farmQuery) addNodeAvailabilityFilter(str string, args ...interface{}) { + f.joinRentContractTable() + + f.nodeQuery = f.nodeQuery.Select( + "node.farm_id", + "COALESCE(rent_contract.twin_id, 0) as renter", + ) + + if f.hasNodeGPU { + f.nodeQuery = f.nodeQuery.Group(`node.farm_id, rent_contract.twin_id`) + } + + f.q = f.q.Where(str, args...) +} + +func (f *farmQuery) addCountryFilter(str string, args ...interface{}) { + f.joinCountryTable() + + f.nodeQuery = f.nodeQuery.Where(str, args...) +} + +func (f *farmQuery) addPublicIPCountFilter(str string, args ...interface{}) { + // ensure public ip table is joined + f.joinPublicIPCount() + + // add where clause + f.q = f.q.Where(str, args...) +} + +func (f *farmQuery) addFarmFilter(str string, args ...interface{}) { + f.q = f.q.Where(str, args...) +} + +func (f *farmQuery) getQuery() *gorm.DB { + if f.nodeQuery != nil { + f.q = f.q.Joins("RIGHT JOIN (?) AS node ON farm.farm_id = node.farm_id", f.nodeQuery) + } + + if f.hasRentContract { + f.q = f.q.Select(` + farm.farm_id, + farm.name, + farm.twin_id, + farm.pricing_policy_id, + farm.certification, + farm.stellar_address, + farm.dedicated_farm as dedicated, + COALESCE(public_ips.public_ips, '[]') as public_ips, + bool_or(node.renter != 0) + `) + } + + return f.q +} + +// GetFarms return farms filtered and paginated +func (d *PostgresDatabase) GetFarms(ctx context.Context, filter types.FarmFilter, limit types.Limit) ([]Farm, uint, error) { + fq := newFarmsQuery(d.gormDB) + + if filter.NodeFreeMRU != nil { + fq.addNodeResourcesFilter("nodes_resources_view.free_mru >= ?", *filter.NodeFreeMRU) + } + + if filter.NodeFreeHRU != nil { + fq.addNodeResourcesFilter("nodes_resources_view.free_hru >= ?", *filter.NodeFreeHRU) + } + + if filter.NodeFreeSRU != nil { + fq.addNodeResourcesFilter("nodes_resources_view.free_sru >= ?", *filter.NodeFreeSRU) + } + + if filter.NodeHasGPU != nil { + if *filter.NodeHasGPU == true { + fq.addGPUFilter("node_gpu.node_twin_id IS NOT NULL") + } else { + fq.addGPUFilter("node_gpu.node_twin_id IS NULL") + } + } + + if filter.NodeRentedBy != nil { + fq.addNodeAvailabilityFilter("renter = ?", *filter.NodeRentedBy) + } + + if filter.Country != nil { + fq.addCountryFilter("node.country ILIKE ?", *filter.Country) + } + + if filter.Region != nil { + fq.addCountryFilter("country.subregion ILIKE ?", *filter.Region) + } + + if filter.NodeStatus != nil { + condition := nodestatus.DecideNodeStatusCondition(*filter.NodeStatus) + fq.addNodeFilter(condition) + } + + if filter.NodeCertified != nil { + fq.addNodeFilter("(node.certification = 'Certified') = ?", *filter.NodeCertified) + } + + if filter.NodeAvailableFor != nil { + fq.addNodeAvailabilityFilter("renter = ? OR (renter = 0 AND farm.dedicated_farm = false)", *filter.NodeAvailableFor) + } + + if filter.FreeIPs != nil { + fq.addPublicIPCountFilter("COALESCE(public_ip_count.free_ips, 0) >= ?", *filter.FreeIPs) + } + if filter.TotalIPs != nil { + fq.addPublicIPCountFilter("COALESCE(public_ip_count.total_ips, 0) >= ?", *filter.TotalIPs) + } + if filter.StellarAddress != nil { + fq.addFarmFilter("farm.stellar_address = ?", *filter.StellarAddress) + } + if filter.PricingPolicyID != nil { + fq.addFarmFilter("farm.pricing_policy_id = ?", *filter.PricingPolicyID) + } + if filter.FarmID != nil { + fq.addFarmFilter("farm.farm_id = ?", *filter.FarmID) + } + if filter.TwinID != nil { + fq.addFarmFilter("farm.twin_id = ?", *filter.TwinID) + } + if filter.Name != nil { + fq.addFarmFilter("farm.name ILIKE ?", *filter.Name) + } + + if filter.NameContains != nil { + escaped := strings.Replace(*filter.NameContains, "%", "\\%", -1) + escaped = strings.Replace(escaped, "_", "\\_", -1) + fq.addFarmFilter("farm.name ILIKE ?", fmt.Sprintf("%%%s%%", escaped)) + } + + if filter.CertificationType != nil { + fq.addFarmFilter("farm.certification = ?", *filter.CertificationType) + } + + if filter.Dedicated != nil { + fq.addFarmFilter("farm.dedicated_farm = ?", *filter.Dedicated) + } + + q := fq.getQuery() + + var count int64 + if limit.Randomize || limit.RetCount { + if res := q.Count(&count); res.Error != nil { + return nil, 0, errors.Wrap(res.Error, "couldn't get farm count") + } + } + + if limit.Randomize { + q = q.Order("random()") + } else { + if filter.NodeAvailableFor != nil { + q = q.Order("bool_or(node.renter != 0) DESC") + } + + if limit.SortBy != "" { + order := types.SortOrderAsc + if strings.EqualFold(string(limit.SortOrder), string(types.SortOrderDesc)) { + order = types.SortOrderDesc + } + q = q.Order(fmt.Sprintf("%s %s", limit.SortBy, order)) + } else { + q = q.Order("farm.farm_id") + } + } + // Pagination + q = q.Limit(int(limit.Size)).Offset(int(limit.Page-1) * int(limit.Size)) + + var farms []Farm + err := q.Scan(&farms).Error + if d.shouldRetry(err) { + err = q.Scan(&farms).Error + } + if err != nil { + return farms, 0, errors.Wrap(err, "failed to scan returned farm from database") + } + return farms, uint(count), nil +} diff --git a/grid-proxy/internal/explorer/db/postgres.go b/grid-proxy/internal/explorer/db/postgres.go index a3ab35fdd..ee2ca1a85 100644 --- a/grid-proxy/internal/explorer/db/postgres.go +++ b/grid-proxy/internal/explorer/db/postgres.go @@ -16,6 +16,8 @@ import ( "gorm.io/gorm/clause" "gorm.io/gorm/logger" + _ "embed" + "github.com/pkg/errors" "github.com/rs/zerolog/log" ) @@ -31,100 +33,6 @@ var ( ErrContractNotFound = errors.New("contract not found") ) -var views = []string{ - ` - CREATE MATERIALIZED VIEW node_materialized_view AS - SELECT - node.node_id, - node.twin_id, - node.farm_id, - node.power, - node.updated_at, - node.certification, - node.country, - nodes_resources_view.free_mru, - nodes_resources_view.free_hru, - nodes_resources_view.free_sru, - COALESCE(rent_contract.contract_id, 0) as rent_contract_id, - COALESCE(rent_contract.twin_id, 0) as renter, - COALESCE(node_gpu.id, '') as gpu_id - FROM - node - LEFT JOIN nodes_resources_view ON node.node_id = nodes_resources_view.node_id - LEFT JOIN rent_contract ON node.node_id = rent_contract.node_id AND rent_contract.state IN ('Created', 'GracePeriod') - LEFT JOIN node_gpu ON node.twin_id = node_gpu.node_twin_id; - - -- define a refresher function - CREATE OR REPLACE FUNCTION refresh_node_materialized_view() - RETURNS TRIGGER AS $$ - BEGIN - REFRESH MATERIALIZED VIEW node_materialized_view; - RETURN NULL; - END; - $$ LANGUAGE plpgsql; - - -- trigger the refresh on each node update - CREATE TRIGGER refresh_node_materialized_trigger - AFTER UPDATE ON node - FOR EACH ROW - EXECUTE FUNCTION refresh_node_materialized_view(); - - -- trigger a backup refresh each 6 hours - SELECT cron.schedule('0 */6 * * *', 'SELECT refresh_node_materialized_view()'); - `, - ` - CREATE MATERIALIZED VIEW farm_materialized_view AS - SELECT - farm.id, - farm.farm_id, - farm.name, - farm.twin_id, - farm.pricing_policy_id, - farm.certification, - farm.stellar_address, - farm.dedicated_farm as dedicated, - COALESCE(public_ips.public_ips, '[]') as public_ips, - bool_or(node_materialized_view.rent_contract_id != 0) as has_rent_contract - FROM - farm - - LEFT JOIN node_materialized_view ON node_materialized_view.farm_id = farm.farm_id - LEFT JOIN country ON node_materialized_view.country = country.name - - LEFT JOIN ( - SELECT - p1.farm_id, - COUNT(p1.id) total_ips, - COUNT(CASE WHEN p2.contract_id = 0 THEN 1 END) free_ips - FROM - public_ip p1 - LEFT JOIN public_ip p2 ON p1.id = p2.id - GROUP BY p1.farm_id - ) public_ip_count on public_ip_count.farm_id = farm.id - - LEFT JOIN ( - SELECT - farm_id, - jsonb_agg(jsonb_build_object('id', id, 'ip', ip, 'contract_id', contract_id, 'gateway', gateway)) as public_ips - FROM - public_ip - GROUP BY farm_id - ) public_ips on public_ips.farm_id = farm.id - - - GROUP BY - farm.id, - farm.farm_id, - farm.name, - farm.twin_id, - farm.pricing_policy_id, - farm.certification, - farm.stellar_address, - farm.dedicated_farm, - COALESCE(public_ips.public_ips, '[]'); - `, -} - const ( setupPostgresql = ` CREATE INDEX IF NOT EXISTS idx_node_id ON public.node(node_id); @@ -440,54 +348,6 @@ func printQuery(query string, args ...interface{}) { fmt.Printf("node query: %s", query) } -func (d *PostgresDatabase) farmTableQuery(nodeQuery *gorm.DB) *gorm.DB { - return d.gormDB. - Table("farm"). - Select( - "farm.id", - "farm.farm_id", - "farm.name", - "farm.twin_id", - "farm.pricing_policy_id", - "farm.certification", - "farm.stellar_address", - "farm.dedicated_farm as dedicated", - "COALESCE(public_ips.public_ips, '[]') as public_ips", - "node.rented", - ). - Joins("LEFT JOIN (?) AS node ON farm.farm_id = node.farm_id", nodeQuery). - Joins(`left join( - SELECT - p1.farm_id, - COUNT(p1.id) total_ips, - COUNT(CASE WHEN p2.contract_id = 0 THEN 1 END) free_ips - FROM public_ip p1 - LEFT JOIN public_ip p2 ON p1.id = p2.id - GROUP BY p1.farm_id - ) public_ip_count on public_ip_count.farm_id = farm.id`). - Joins(`left join ( - select - farm_id, - jsonb_agg(jsonb_build_object('id', id, 'ip', ip, 'contract_id', contract_id, 'gateway', gateway)) as public_ips - from public_ip - GROUP BY farm_id - ) public_ips on public_ips.farm_id = farm.id`). - Joins( - "LEFT JOIN country ON node.country = country.name", - ). - Group( - `farm.id, - farm.farm_id, - farm.name, - farm.twin_id, - farm.pricing_policy_id, - farm.certification, - farm.stellar_address, - farm.dedicated_farm, - COALESCE(public_ips.public_ips, '[]')`, - ) -} - func (d *PostgresDatabase) nodeTableQuery() *gorm.DB { subquery := d.gormDB.Table("node_contract"). Select("DISTINCT ON (node_id) node_id, contract_id"). @@ -758,143 +618,6 @@ func (d *PostgresDatabase) shouldRetry(resError error) bool { return false } -// GetFarms return farms filtered and paginated -func (d *PostgresDatabase) GetFarms(ctx context.Context, filter types.FarmFilter, limit types.Limit) ([]Farm, uint, error) { - // filter the nodes first - nodeQuery := d.gormDB.Table("node"). - Select( - "node.farm_id", - "country", - "certification", - "rent_contract.twin_id as renter", - // check using the aggregated function - "CASE WHEN rent_contract.contract_id IS NOT NULL AND rent_contract.contract_id <> 0 THEN true ELSE false END AS rented", - "COALESCE(node_gpu.id, '') as gpu_id", - ).Joins( - "LEFT JOIN rent_contract ON rent_contract.state IN ('Created', 'GracePeriod') AND rent_contract.node_id = node.node_id", - "LEFT JOIN nodes_resources_view ON nodes_resources_view.node_id = node.node_id", - "LEFT JOIN country ON country.name = node.country", - "LEFT JOIN node_gpu ON node.twin_id = node_gpu.node_twin_id", - ).Group( - `node.farm_id, - rent_contract.twin_id, - node_gpu.id, - node.certification, - node.country`, - ) - - if filter.NodeFreeMRU != nil { - nodeQuery = nodeQuery.Where("nodes_resources_view.free_mru >= ?", *filter.NodeFreeMRU) - } - if filter.NodeFreeHRU != nil { - nodeQuery = nodeQuery.Where("nodes_resources_view.free_hru >= ?", *filter.NodeFreeHRU) - } - if filter.NodeFreeSRU != nil { - nodeQuery = nodeQuery.Where("nodes_resources_view.free_sru >= ?", *filter.NodeFreeSRU) - } - - if filter.NodeAvailableFor != nil { - nodeQuery = nodeQuery.Where("node.renter = ? OR (node.renter = 0 AND farm.dedicated_farm = false)", *filter.NodeAvailableFor) - // nodeQuery = nodeQuery.Order("CASE WHEN bool_or(node.rent_contract_id != 0) = TRUE THEN 1 ELSE 2 END") - } - - if filter.NodeHasGPU != nil { - nodeQuery = nodeQuery.Where("(node.gpu_id != '') = ?", *filter.NodeHasGPU) - } - - if filter.NodeRentedBy != nil { - nodeQuery = nodeQuery.Where("node.renter = ?", *filter.NodeRentedBy) - } - - if filter.Country != nil { - nodeQuery = nodeQuery.Where("LOWER(node.country) = LOWER(?)", *filter.Country) - } - - if filter.Region != nil { - nodeQuery = nodeQuery.Where("LOWER(country.subregion) = LOWER(?)", *filter.Region) - } - - if filter.NodeStatus != nil { - condition := nodestatus.DecideNodeStatusCondition(*filter.NodeStatus) - nodeQuery = nodeQuery.Where(condition) - } - - if filter.NodeCertified != nil { - nodeQuery = nodeQuery.Where("(node.certification = 'Certified') = ?", *filter.NodeCertified) - } - - q := d.farmTableQuery(nodeQuery) - - if filter.FreeIPs != nil { - q = q.Where("public_ip_count.free_ips >= ?", *filter.FreeIPs) - } - if filter.TotalIPs != nil { - q = q.Where("public_ip_count.total_ips >= ?", *filter.TotalIPs) - } - if filter.StellarAddress != nil { - q = q.Where("farm.stellar_address = ?", *filter.StellarAddress) - } - if filter.PricingPolicyID != nil { - q = q.Where("farm.pricing_policy_id = ?", *filter.PricingPolicyID) - } - if filter.FarmID != nil { - q = q.Where("farm.farm_id = ?", *filter.FarmID) - } - if filter.TwinID != nil { - q = q.Where("farm.twin_id = ?", *filter.TwinID) - } - if filter.Name != nil { - q = q.Where("LOWER(farm.name) = LOWER(?)", *filter.Name) - } - - if filter.NameContains != nil { - escaped := strings.Replace(*filter.NameContains, "%", "\\%", -1) - escaped = strings.Replace(escaped, "_", "\\_", -1) - q = q.Where("farm.name ILIKE ?", fmt.Sprintf("%%%s%%", escaped)) - } - - if filter.CertificationType != nil { - q = q.Where("farm.certification = ?", *filter.CertificationType) - } - - if filter.Dedicated != nil { - q = q.Where("farm.dedicated_farm = ?", *filter.Dedicated) - } - var count int64 - // if limit.Randomize || limit.RetCount { - // if res := q.Count(&count); res.Error != nil { - // return nil, 0, errors.Wrap(res.Error, "couldn't get farm count") - // } - // } - - // Sorting - //TODO: sort with available for - if limit.Randomize { - q = q.Order("random()") - } else if limit.SortBy != "" { - order := types.SortOrderAsc - if strings.EqualFold(string(limit.SortOrder), string(types.SortOrderDesc)) { - order = types.SortOrderDesc - } - q = q.Order(fmt.Sprintf("%s %s", limit.SortBy, order)) - } else { - q = q.Order("farm.farm_id") - } - - // Pagination - q = q.Limit(int(limit.Size)).Offset(int(limit.Page-1) * int(limit.Size)) - - var farms []Farm - err := q.Scan(&farms).Error - if d.shouldRetry(err) { - err = q.Scan(&farms).Error - } - if err != nil { - return farms, 0, errors.Wrap(err, "failed to scan returned farm from database") - } - return farms, uint(count), nil -} - // GetTwins returns twins filtered and paginated func (d *PostgresDatabase) GetTwins(ctx context.Context, filter types.TwinFilter, limit types.Limit) ([]types.Twin, uint, error) { q := d.gormDB.WithContext(ctx). diff --git a/grid-proxy/internal/explorer/db/view.sql b/grid-proxy/internal/explorer/db/view.sql new file mode 100644 index 000000000..45f9fe101 --- /dev/null +++ b/grid-proxy/internal/explorer/db/view.sql @@ -0,0 +1,85 @@ +CREATE MATERIALIZED VIEW node_materialized_view AS +SELECT node.node_id, + node.twin_id, + node.farm_id, + node.power, + node.updated_at, + node.certification, + node.country, + nodes_resources_view.free_mru, + nodes_resources_view.free_hru, + nodes_resources_view.free_sru, + COALESCE(rent_contract.contract_id, 0) as rent_contract_id, + COALESCE(rent_contract.twin_id, 0) as renter, + COALESCE(node_gpu.id, '') as gpu_id +FROM node + LEFT JOIN nodes_resources_view ON node.node_id = nodes_resources_view.node_id + LEFT JOIN rent_contract ON node.node_id = rent_contract.node_id + AND rent_contract.state IN ('Created', 'GracePeriod') + LEFT JOIN node_gpu ON node.twin_id = node_gpu.node_twin_id; +-- define a refresher function +CREATE OR REPLACE FUNCTION refresh_node_materialized_view() RETURNS TRIGGER AS $$ BEGIN REFRESH MATERIALIZED VIEW node_materialized_view; +RETURN NULL; +END; +$$ LANGUAGE plpgsql; +-- trigger the refresh on each node update +CREATE TRIGGER refresh_node_materialized_trigger +AFTER +UPDATE ON node FOR EACH ROW EXECUTE FUNCTION refresh_node_materialized_view(); +-- trigger a backup refresh each 6 hours +SELECT cron.schedule( + '0 */6 * * *', + 'SELECT refresh_node_materialized_view()' + ); +CREATE MATERIALIZED VIEW farm_materialized_view AS +SELECT farm.id, + farm.farm_id, + farm.name, + farm.twin_id, + farm.pricing_policy_id, + farm.certification, + farm.stellar_address, + farm.dedicated_farm as dedicated, + COALESCE(public_ips.public_ips, '[]') as public_ips, + bool_or(node_materialized_view.rent_contract_id != 0) as has_rent_contract +FROM farm + LEFT JOIN node_materialized_view ON node_materialized_view.farm_id = farm.farm_id + LEFT JOIN country ON node_materialized_view.country = country.name + LEFT JOIN ( + SELECT p1.farm_id, + COUNT(p1.id) total_ips, + COUNT( + CASE + WHEN p2.contract_id = 0 THEN 1 + END + ) free_ips + FROM public_ip p1 + LEFT JOIN public_ip p2 ON p1.id = p2.id + GROUP BY p1.farm_id + ) public_ip_count on public_ip_count.farm_id = farm.id + LEFT JOIN ( + SELECT farm_id, + jsonb_agg( + jsonb_build_object( + 'id', + id, + 'ip', + ip, + 'contract_id', + contract_id, + 'gateway', + gateway + ) + ) as public_ips + FROM public_ip + GROUP BY farm_id + ) public_ips on public_ips.farm_id = farm.id +GROUP BY farm.id, + farm.farm_id, + farm.name, + farm.twin_id, + farm.pricing_policy_id, + farm.certification, + farm.stellar_address, + farm.dedicated_farm, + COALESCE(public_ips.public_ips, '[]'); \ No newline at end of file diff --git a/grid-proxy/tests/queries/farm_test.go b/grid-proxy/tests/queries/farm_test.go index 4fca43088..b4d2ad4a2 100644 --- a/grid-proxy/tests/queries/farm_test.go +++ b/grid-proxy/tests/queries/farm_test.go @@ -139,10 +139,10 @@ func TestFarm(t *testing.T) { for ; ; l.Page++ { want, wantCount, err := mockClient.Farms(context.Background(), f, l) require.NoError(t, err) - + // log.Printf("want: %+v", want) got, gotCount, err := gridProxyClient.Farms(context.Background(), f, l) require.NoError(t, err) - + // log.Printf("got: %+v", got) assert.Equal(t, wantCount, gotCount) sortPublicIPs(want, got) @@ -167,9 +167,10 @@ func TestFarm(t *testing.T) { want, wantCount, err := mockClient.Farms(context.Background(), f, l) require.NoError(t, err) - + // log.Printf("want: %+v", want) got, gotCount, err := gridProxyClient.Farms(context.Background(), f, l) require.NoError(t, err) + // log.Printf("got: %+v", got) assert.Equal(t, wantCount, gotCount) @@ -190,9 +191,10 @@ func TestFarm(t *testing.T) { want, wantCount, err := mockClient.Farms(context.Background(), f, l) require.NoError(t, err) - + // log.Printf("want: %+v", want) got, gotCount, err := gridProxyClient.Farms(context.Background(), f, l) require.NoError(t, err) + // log.Printf("got: %+v", got) assert.Equal(t, wantCount, gotCount) diff --git a/grid-proxy/tests/queries/mock_client/farms.go b/grid-proxy/tests/queries/mock_client/farms.go index 4940b2873..0b5b052e7 100644 --- a/grid-proxy/tests/queries/mock_client/farms.go +++ b/grid-proxy/tests/queries/mock_client/farms.go @@ -32,13 +32,17 @@ func (g *GridProxyMockClient) Farms(ctx context.Context, filter types.FarmFilter for _, farm := range g.data.Farms { if farm.satisfies(filter, &g.data) { + ips := []types.PublicIP{} + if pubIPs, ok := publicIPs[farm.FarmID]; ok { + ips = pubIPs + } res = append(res, types.Farm{ Name: farm.Name, FarmID: int(farm.FarmID), TwinID: int(farm.TwinID), PricingPolicyID: int(farm.PricingPolicyID), StellarAddress: farm.StellarAddress, - PublicIps: publicIPs[farm.FarmID], + PublicIps: ips, Dedicated: farm.DedicatedFarm, CertificationType: farm.Certification, }) @@ -105,8 +109,13 @@ func (f *Farm) satisfies(filter types.FarmFilter, data *DBData) bool { return false } - if !f.satisfyFarmNodesFilter(data, filter) { - return false + if filter.NodeAvailableFor != nil || filter.NodeCertified != nil || filter.NodeFreeHRU != nil || + filter.NodeFreeMRU != nil || filter.NodeFreeSRU != nil || filter.NodeHasGPU != nil || + filter.NodeRentedBy != nil || filter.NodeStatus != nil || filter.Country != nil || filter.Region != nil { + + if !f.satisfyFarmNodesFilter(data, filter) { + return false + } } return true From 5006869152ef5ebc8d48723549f2795a9f4cadd8 Mon Sep 17 00:00:00 2001 From: mario Date: Mon, 18 Dec 2023 18:31:51 +0200 Subject: [PATCH 06/39] wip: add resources_cache table Co-authored-by: omarabdulaziz --- grid-proxy/internal/explorer/converters.go | 8 +- grid-proxy/internal/explorer/db/farms.go | 326 ------------------ grid-proxy/internal/explorer/db/new_table.go | 88 +++++ grid-proxy/internal/explorer/db/postgres.go | 245 ++++++++++--- grid-proxy/internal/explorer/db/types.go | 70 ++-- grid-proxy/pkg/nodestatus/nodestatus.go | 2 +- grid-proxy/pkg/types/farms.go | 1 + grid-proxy/tests/queries/farm_test.go | 5 + grid-proxy/tests/queries/mock_client/farms.go | 14 +- grid-proxy/tools/db/generate.go | 7 +- 10 files changed, 355 insertions(+), 411 deletions(-) delete mode 100644 grid-proxy/internal/explorer/db/farms.go create mode 100644 grid-proxy/internal/explorer/db/new_table.go diff --git a/grid-proxy/internal/explorer/converters.go b/grid-proxy/internal/explorer/converters.go index d2c804754..acafeac3a 100644 --- a/grid-proxy/internal/explorer/converters.go +++ b/grid-proxy/internal/explorer/converters.go @@ -52,14 +52,14 @@ func nodeFromDBNode(info db.Node) types.Node { InDedicatedFarm: info.FarmDedicated, CertificationType: info.Certification, RentContractID: uint(info.RentContractID), - RentedByTwinID: uint(info.RentedByTwinID), + RentedByTwinID: uint(info.Renter), SerialNumber: info.SerialNumber, Power: types.NodePower(info.Power), NumGPU: info.NumGPU, ExtraFee: info.ExtraFee, } node.Status = nodestatus.DecideNodeStatus(node.Power, node.UpdatedAt) - node.Dedicated = info.FarmDedicated || !info.HasNodeContract || info.RentContractID != 0 + node.Dedicated = info.FarmDedicated || info.NodeContractsCount == 0 || info.Renter != 0 return node } @@ -124,14 +124,14 @@ func nodeWithNestedCapacityFromDBNode(info db.Node) types.NodeWithNestedCapacity InDedicatedFarm: info.FarmDedicated, CertificationType: info.Certification, RentContractID: uint(info.RentContractID), - RentedByTwinID: uint(info.RentedByTwinID), + RentedByTwinID: uint(info.Renter), SerialNumber: info.SerialNumber, Power: types.NodePower(info.Power), NumGPU: info.NumGPU, ExtraFee: info.ExtraFee, } node.Status = nodestatus.DecideNodeStatus(node.Power, node.UpdatedAt) - node.Dedicated = info.FarmDedicated || !info.HasNodeContract || info.RentContractID != 0 + node.Dedicated = info.FarmDedicated || info.NodeContractsCount == 0 || info.Renter != 0 return node } diff --git a/grid-proxy/internal/explorer/db/farms.go b/grid-proxy/internal/explorer/db/farms.go deleted file mode 100644 index e1348c4d5..000000000 --- a/grid-proxy/internal/explorer/db/farms.go +++ /dev/null @@ -1,326 +0,0 @@ -package db - -import ( - "context" - "fmt" - "strings" - - "github.com/pkg/errors" - "github.com/threefoldtech/tfgrid-sdk-go/grid-proxy/pkg/nodestatus" - "github.com/threefoldtech/tfgrid-sdk-go/grid-proxy/pkg/types" - "gorm.io/gorm" -) - -type farmQuery struct { - d *gorm.DB - q *gorm.DB - nodeQuery *gorm.DB - - hasPublicIPCount bool // to indicate if public ip count subquery was joined - hasNodes bool // to indicate if nodes subquery was joined - hasNodesResources bool // to indicate if nodes_resources_view was joined - hasNodeGPU bool - hasRentContract bool - hasCountry bool -} - -func newFarmsQuery(d *gorm.DB) farmQuery { - return farmQuery{ - d: d, - q: d.Table("farm"). - Select(` - farm.farm_id, - farm.name, - farm.twin_id, - farm.pricing_policy_id, - farm.certification, - farm.stellar_address, - farm.dedicated_farm as dedicated, - COALESCE(public_ips.public_ips, '[]') as public_ips - `). - Joins(` - LEFT JOIN ( - SELECT - farm_id, - jsonb_agg(jsonb_build_object('id', id, 'ip', ip, 'contract_id', contract_id, 'gateway', gateway)) AS public_ips - FROM public_ip - GROUP BY farm_id - ) public_ips ON public_ips.farm_id = farm.id - `). - Group(` - farm.id, - farm.farm_id, - farm.name, - farm.twin_id, - farm.pricing_policy_id, - farm.certification, - farm.stellar_address, - farm.dedicated_farm, - COALESCE(public_ips.public_ips, '[]')`), - } -} - -func (f *farmQuery) joinPublicIPCount() { - if !f.hasPublicIPCount { - f.hasPublicIPCount = true - - f.q = f.q.Joins(` - LEFT JOIN( - SELECT - p1.farm_id, - COUNT(p1.id) total_ips, - COUNT(CASE WHEN p2.contract_id = 0 THEN 1 END) free_ips - FROM public_ip p1 - LEFT JOIN public_ip p2 ON p1.id = p2.id - GROUP BY p1.farm_id - ) public_ip_count ON public_ip_count.farm_id = farm.id - `) - } -} - -func (f *farmQuery) createNodesSubquery() { - if f.nodeQuery == nil { - f.nodeQuery = f.d.Table("node").Select("node.farm_id") - } -} - -func (f *farmQuery) joinNodesResourcesView() { - f.createNodesSubquery() - - if !f.hasNodesResources { - f.hasNodesResources = true - f.nodeQuery = f.nodeQuery.Joins("LEFT JOIN nodes_resources_view ON nodes_resources_view.node_id = node.node_id") - } -} - -func (f *farmQuery) joinGPUTable() { - f.createNodesSubquery() - - if !f.hasNodeGPU { - f.hasNodeGPU = true - - f.nodeQuery = f.nodeQuery.Joins("LEFT JOIN node_gpu ON node.twin_id = node_gpu.node_twin_id"). - Group("node.farm_id") - if f.hasRentContract { - f.nodeQuery = f.nodeQuery.Group(`node.farm_id, rent_contract.twin_id`) - } - } -} - -func (f *farmQuery) joinRentContractTable() { - f.createNodesSubquery() - - if !f.hasRentContract { - f.hasRentContract = true - f.nodeQuery = f.nodeQuery.Joins("LEFT JOIN rent_contract ON rent_contract.state IN ('Created', 'GracePeriod') AND rent_contract.node_id = node.node_id") - } -} - -func (f *farmQuery) joinCountryTable() { - f.createNodesSubquery() - - if !f.hasCountry { - f.hasCountry = true - f.nodeQuery = f.nodeQuery.Joins("LEFT JOIN country ON country.name = node.country") - } -} - -func (f *farmQuery) addNodeResourcesFilter(str string, args ...interface{}) { - // ensure node subquery is joined - f.joinNodesResourcesView() - - // add where clause - f.nodeQuery = f.nodeQuery.Where(str, args...) -} - -func (f *farmQuery) addGPUFilter(str string) { - f.joinGPUTable() - - f.nodeQuery = f.nodeQuery.Where(str) -} - -func (f *farmQuery) addNodeFilter(str string, args ...interface{}) { - f.createNodesSubquery() - - f.nodeQuery = f.nodeQuery.Where(str, args...) -} - -func (f *farmQuery) addNodeAvailabilityFilter(str string, args ...interface{}) { - f.joinRentContractTable() - - f.nodeQuery = f.nodeQuery.Select( - "node.farm_id", - "COALESCE(rent_contract.twin_id, 0) as renter", - ) - - if f.hasNodeGPU { - f.nodeQuery = f.nodeQuery.Group(`node.farm_id, rent_contract.twin_id`) - } - - f.q = f.q.Where(str, args...) -} - -func (f *farmQuery) addCountryFilter(str string, args ...interface{}) { - f.joinCountryTable() - - f.nodeQuery = f.nodeQuery.Where(str, args...) -} - -func (f *farmQuery) addPublicIPCountFilter(str string, args ...interface{}) { - // ensure public ip table is joined - f.joinPublicIPCount() - - // add where clause - f.q = f.q.Where(str, args...) -} - -func (f *farmQuery) addFarmFilter(str string, args ...interface{}) { - f.q = f.q.Where(str, args...) -} - -func (f *farmQuery) getQuery() *gorm.DB { - if f.nodeQuery != nil { - f.q = f.q.Joins("RIGHT JOIN (?) AS node ON farm.farm_id = node.farm_id", f.nodeQuery) - } - - if f.hasRentContract { - f.q = f.q.Select(` - farm.farm_id, - farm.name, - farm.twin_id, - farm.pricing_policy_id, - farm.certification, - farm.stellar_address, - farm.dedicated_farm as dedicated, - COALESCE(public_ips.public_ips, '[]') as public_ips, - bool_or(node.renter != 0) - `) - } - - return f.q -} - -// GetFarms return farms filtered and paginated -func (d *PostgresDatabase) GetFarms(ctx context.Context, filter types.FarmFilter, limit types.Limit) ([]Farm, uint, error) { - fq := newFarmsQuery(d.gormDB) - - if filter.NodeFreeMRU != nil { - fq.addNodeResourcesFilter("nodes_resources_view.free_mru >= ?", *filter.NodeFreeMRU) - } - - if filter.NodeFreeHRU != nil { - fq.addNodeResourcesFilter("nodes_resources_view.free_hru >= ?", *filter.NodeFreeHRU) - } - - if filter.NodeFreeSRU != nil { - fq.addNodeResourcesFilter("nodes_resources_view.free_sru >= ?", *filter.NodeFreeSRU) - } - - if filter.NodeHasGPU != nil { - if *filter.NodeHasGPU == true { - fq.addGPUFilter("node_gpu.node_twin_id IS NOT NULL") - } else { - fq.addGPUFilter("node_gpu.node_twin_id IS NULL") - } - } - - if filter.NodeRentedBy != nil { - fq.addNodeAvailabilityFilter("renter = ?", *filter.NodeRentedBy) - } - - if filter.Country != nil { - fq.addCountryFilter("node.country ILIKE ?", *filter.Country) - } - - if filter.Region != nil { - fq.addCountryFilter("country.subregion ILIKE ?", *filter.Region) - } - - if filter.NodeStatus != nil { - condition := nodestatus.DecideNodeStatusCondition(*filter.NodeStatus) - fq.addNodeFilter(condition) - } - - if filter.NodeCertified != nil { - fq.addNodeFilter("(node.certification = 'Certified') = ?", *filter.NodeCertified) - } - - if filter.NodeAvailableFor != nil { - fq.addNodeAvailabilityFilter("renter = ? OR (renter = 0 AND farm.dedicated_farm = false)", *filter.NodeAvailableFor) - } - - if filter.FreeIPs != nil { - fq.addPublicIPCountFilter("COALESCE(public_ip_count.free_ips, 0) >= ?", *filter.FreeIPs) - } - if filter.TotalIPs != nil { - fq.addPublicIPCountFilter("COALESCE(public_ip_count.total_ips, 0) >= ?", *filter.TotalIPs) - } - if filter.StellarAddress != nil { - fq.addFarmFilter("farm.stellar_address = ?", *filter.StellarAddress) - } - if filter.PricingPolicyID != nil { - fq.addFarmFilter("farm.pricing_policy_id = ?", *filter.PricingPolicyID) - } - if filter.FarmID != nil { - fq.addFarmFilter("farm.farm_id = ?", *filter.FarmID) - } - if filter.TwinID != nil { - fq.addFarmFilter("farm.twin_id = ?", *filter.TwinID) - } - if filter.Name != nil { - fq.addFarmFilter("farm.name ILIKE ?", *filter.Name) - } - - if filter.NameContains != nil { - escaped := strings.Replace(*filter.NameContains, "%", "\\%", -1) - escaped = strings.Replace(escaped, "_", "\\_", -1) - fq.addFarmFilter("farm.name ILIKE ?", fmt.Sprintf("%%%s%%", escaped)) - } - - if filter.CertificationType != nil { - fq.addFarmFilter("farm.certification = ?", *filter.CertificationType) - } - - if filter.Dedicated != nil { - fq.addFarmFilter("farm.dedicated_farm = ?", *filter.Dedicated) - } - - q := fq.getQuery() - - var count int64 - if limit.Randomize || limit.RetCount { - if res := q.Count(&count); res.Error != nil { - return nil, 0, errors.Wrap(res.Error, "couldn't get farm count") - } - } - - if limit.Randomize { - q = q.Order("random()") - } else { - if filter.NodeAvailableFor != nil { - q = q.Order("bool_or(node.renter != 0) DESC") - } - - if limit.SortBy != "" { - order := types.SortOrderAsc - if strings.EqualFold(string(limit.SortOrder), string(types.SortOrderDesc)) { - order = types.SortOrderDesc - } - q = q.Order(fmt.Sprintf("%s %s", limit.SortBy, order)) - } else { - q = q.Order("farm.farm_id") - } - } - // Pagination - q = q.Limit(int(limit.Size)).Offset(int(limit.Page-1) * int(limit.Size)) - - var farms []Farm - err := q.Scan(&farms).Error - if d.shouldRetry(err) { - err = q.Scan(&farms).Error - } - if err != nil { - return farms, 0, errors.Wrap(err, "failed to scan returned farm from database") - } - return farms, uint(count), nil -} diff --git a/grid-proxy/internal/explorer/db/new_table.go b/grid-proxy/internal/explorer/db/new_table.go new file mode 100644 index 000000000..3cc29c52d --- /dev/null +++ b/grid-proxy/internal/explorer/db/new_table.go @@ -0,0 +1,88 @@ +package db + +/* + node: + free_hru, free_mru, free_sru, total_hru, total_mru, total_sru, total_cru, + + total_resources + contract_resources -> used node resources + renter -> to know who rented it, available for + node_contracts -> to know whether it's rentable + nodeid, farmid + + triggers: + - trigger on node table (insert) + - trigger on total resources (insert/update) + - trigger on contract resources (insert/update) + - trigger on rent contract (insert/update) + - trigger on node contract (insert/update) + + triggers need to be in the same transaction with table creation +*/ + +var resourcesCache = ` + drop table if exists resources_cache; + CREATE TABLE IF NOT EXISTS resources_cache( + node_id INTEGER PRIMARY KEY, + farm_id INTEGER NOT NULL, + total_hru NUMERIC NOT NULL, + total_mru NUMERIC NOT NULL, + total_sru NUMERIC NOT NULL, + total_cru NUMERIC NOT NULL, + free_hru NUMERIC NOT NULL, + free_mru NUMERIC NOT NULL, + free_sru NUMERIC NOT NULL, + used_hru NUMERIC NOT NULL, + used_mru NUMERIC NOT NULL, + used_sru NUMERIC NOT NULL, + used_cru NUMERIC NOT NULL, + renter INTEGER, + rent_contract_id INTEGER, + node_contracts_count INTEGER NOT NULL, + node_gpu_count INTEGER NOT NULL + ); + + INSERT INTO resources_cache + SELECT * + FROM ( + SELECT node.node_id as node_id, + node.farm_id as farm_id, + COALESCE(node_resources_total.hru, 0) as total_hru, + COALESCE(node_resources_total.mru, 0) as total_mru, + COALESCE(node_resources_total.sru, 0) as total_sru, + COALESCE(node_resources_total.cru, 0) as total_cru, + node_resources_total.hru - COALESCE(sum(contract_resources.hru), 0) as free_hru, + node_resources_total.mru - COALESCE(sum(contract_resources.mru), 0) - GREATEST( + CAST((node_resources_total.mru / 10) AS bigint), + 2147483648 + ) as free_mru, + node_resources_total.sru - COALESCE(sum(contract_resources.sru), 0) - 21474836480 as free_sru, + COALESCE(sum(contract_resources.hru), 0) as used_hru, + COALESCE(sum(contract_resources.mru), 0) + GREATEST( + CAST((node_resources_total.mru / 10) AS bigint), + 2147483648 + ) as used_mru, + COALESCE(sum(contract_resources.sru), 0) + 21474836480 as used_sru, + COALESCE(sum(contract_resources.cru), 0) as used_cru, + rent_contract.twin_id as renter, + rent_contract.contract_id as rent_contract_id, + count(node_contract.contract_id) as node_contract_count, + count(node_gpu.id) as node_gpu_count + FROM contract_resources + JOIN node_contract as node_contract ON node_contract.resources_used_id = contract_resources.id + AND node_contract.state IN ('Created', 'GracePeriod') + RIGHT JOIN node as node ON node.node_id = node_contract.node_id + JOIN node_resources_total AS node_resources_total ON node_resources_total.node_id = node.id + LEFT JOIN rent_contract on node.node_id = rent_contract.node_id + AND rent_contract.state IN ('Created', 'GracePeriod') + LEFT JOIN node_gpu ON node.twin_id = node_gpu.node_twin_id + GROUP BY node.node_id, + node_resources_total.mru, + node_resources_total.sru, + node_resources_total.hru, + node_resources_total.cru, + node.farm_id, + rent_contract.contract_id, + rent_contract.twin_id + ) as node_resources; +` diff --git a/grid-proxy/internal/explorer/db/postgres.go b/grid-proxy/internal/explorer/db/postgres.go index ee2ca1a85..ab7cabda3 100644 --- a/grid-proxy/internal/explorer/db/postgres.go +++ b/grid-proxy/internal/explorer/db/postgres.go @@ -52,8 +52,7 @@ const ( COALESCE(node_resources_total.cru, 0) as total_cru, COALESCE(node_resources_total.mru, 0) as total_mru, COALESCE(node_resources_total.hru, 0) as total_hru, - COALESCE(node_resources_total.sru, 0) as total_sru, - COALESCE(COUNT(DISTINCT state), 0) as states + COALESCE(node_resources_total.sru, 0) as total_sru FROM contract_resources JOIN node_contract as node_contract ON node_contract.resources_used_id = contract_resources.id AND node_contract.state IN ('Created', 'GracePeriod') @@ -65,7 +64,7 @@ const ( DROP FUNCTION IF EXISTS node_resources(query_node_id INTEGER); CREATE OR REPLACE function node_resources(query_node_id INTEGER) - returns table (node_id INTEGER, used_cru NUMERIC, used_mru NUMERIC, used_hru NUMERIC, used_sru NUMERIC, free_mru NUMERIC, free_hru NUMERIC, free_sru NUMERIC, total_cru NUMERIC, total_mru NUMERIC, total_hru NUMERIC, total_sru NUMERIC, states BIGINT) + returns table (node_id INTEGER, used_cru NUMERIC, used_mru NUMERIC, used_hru NUMERIC, used_sru NUMERIC, free_mru NUMERIC, free_hru NUMERIC, free_sru NUMERIC, total_cru NUMERIC, total_mru NUMERIC, total_hru NUMERIC, total_sru NUMERIC) as $body$ SELECT @@ -80,8 +79,7 @@ const ( COALESCE(node_resources_total.cru, 0) as total_cru, COALESCE(node_resources_total.mru, 0) as total_mru, COALESCE(node_resources_total.hru, 0) as total_hru, - COALESCE(node_resources_total.sru, 0) as total_sru, - COALESCE(COUNT(DISTINCT state), 0) as states + COALESCE(node_resources_total.sru, 0) as total_sru FROM contract_resources JOIN node_contract as node_contract ON node_contract.resources_used_id = contract_resources.id AND node_contract.state IN ('Created', 'GracePeriod') @@ -349,9 +347,9 @@ func printQuery(query string, args ...interface{}) { } func (d *PostgresDatabase) nodeTableQuery() *gorm.DB { - subquery := d.gormDB.Table("node_contract"). - Select("DISTINCT ON (node_id) node_id, contract_id"). - Where("state IN ('Created', 'GracePeriod')") + // subquery := d.gormDB.Table("node_contract"). + // Select("DISTINCT ON (node_id) node_id, contract_id"). + // Where("state IN ('Created', 'GracePeriod')") nodeGPUQuery := `(SELECT count(node_gpu.id) FROM node_gpu WHERE node_gpu.node_twin_id = node.twin_id) as num_gpu` @@ -369,14 +367,14 @@ func (d *PostgresDatabase) nodeTableQuery() *gorm.DB { "node.created", "node.farming_policy_id", "updated_at", - "nodes_resources_view.total_cru", - "nodes_resources_view.total_sru", - "nodes_resources_view.total_hru", - "nodes_resources_view.total_mru", - "nodes_resources_view.used_cru", - "nodes_resources_view.used_sru", - "nodes_resources_view.used_hru", - "nodes_resources_view.used_mru", + "resources_cache.total_cru", + "resources_cache.total_sru", + "resources_cache.total_hru", + "resources_cache.total_mru", + "resources_cache.used_cru", + "resources_cache.used_sru", + "resources_cache.used_hru", + "resources_cache.used_mru", "public_config.domain", "public_config.gw4", "public_config.gw6", @@ -384,28 +382,23 @@ func (d *PostgresDatabase) nodeTableQuery() *gorm.DB { "public_config.ipv6", "node.certification", "farm.dedicated_farm as farm_dedicated", - "rent_contract.contract_id as rent_contract_id", - "rent_contract.twin_id as rented_by_twin_id", + "resources_cache.rent_contract_id as rent_contract_id", + "resources_cache.renter", "node.serial_number", "convert_to_decimal(location.longitude) as longitude", "convert_to_decimal(location.latitude) as latitude", "node.power", "node.extra_fee", - "CASE WHEN node_contract.contract_id IS NOT NULL THEN 1 ELSE 0 END AS has_node_contract", + "resources_cache.node_contracts_count", + // "CASE WHEN node_contract.contract_id IS NOT NULL THEN 1 ELSE 0 END AS has_node_contract", nodeGPUQuery, ). Joins( - "LEFT JOIN nodes_resources_view ON node.node_id = nodes_resources_view.node_id", + "LEFT JOIN resources_cache ON node.node_id = resources_cache.node_id", ). Joins( "LEFT JOIN public_config ON node.id = public_config.node_id", ). - Joins( - "LEFT JOIN rent_contract ON rent_contract.state IN ('Created', 'GracePeriod') AND rent_contract.node_id = node.node_id", - ). - Joins( - "LEFT JOIN (?) AS node_contract ON node_contract.node_id = node.node_id", subquery, - ). Joins( "LEFT JOIN farm ON node.farm_id = farm.farm_id", ). @@ -417,6 +410,180 @@ func (d *PostgresDatabase) nodeTableQuery() *gorm.DB { ) } +func (d *PostgresDatabase) farmTableQuery(filter types.FarmFilter) *gorm.DB { + q := d.gormDB. + Table("farm"). + Select( + "farm.id", + "farm.farm_id", + "farm.name", + "farm.twin_id", + "farm.pricing_policy_id", + "farm.certification", + "farm.stellar_address", + "farm.dedicated_farm as dedicated", + "COALESCE(public_ips.public_ips, '[]') as public_ips", + ). + Joins(`left join( + SELECT + p1.farm_id, + COUNT(p1.id) total_ips, + COUNT(CASE WHEN p2.contract_id = 0 THEN 1 END) free_ips + FROM public_ip p1 + LEFT JOIN public_ip p2 ON p1.id = p2.id + GROUP BY p1.farm_id + ) public_ip_count on public_ip_count.farm_id = farm.id`). + Joins(`left join ( + select + farm_id, + jsonb_agg(jsonb_build_object('id', id, 'ip', ip, 'contract_id', contract_id, 'gateway', gateway)) as public_ips + from public_ip + GROUP BY farm_id + ) public_ips on public_ips.farm_id = farm.id`). + Group( + `farm.id, + farm.farm_id, + farm.name, + farm.twin_id, + farm.pricing_policy_id, + farm.certification, + farm.stellar_address, + farm.dedicated_farm, + COALESCE(public_ips.public_ips, '[]')`, + ) + if filter.NodeAvailableFor != nil || filter.NodeFreeHRU != nil || + filter.NodeCertified != nil || filter.NodeFreeMRU != nil || + filter.NodeFreeSRU != nil || filter.NodeHasGPU != nil || + filter.NodeRentedBy != nil || filter.NodeStatus != nil || + filter.NodeTotalCRU != nil || filter.Country != nil || + filter.Region != nil { + q.Joins(` + RIGHT JOIN resources_cache ON resources_cache.farm_id = farm.farm_id + LEFT JOIN node ON node.node_id = resources_cache.node_id + LEFT JOIN country ON node.country = country.name + `) + } + + return q +} + +// GetFarms return farms filtered and paginated +func (d *PostgresDatabase) GetFarms(ctx context.Context, filter types.FarmFilter, limit types.Limit) ([]Farm, uint, error) { + q := d.farmTableQuery(filter) + q = q.WithContext(ctx) + + if filter.NodeFreeMRU != nil { + q = q.Where("resources_cache.free_mru >= ?", *filter.NodeFreeMRU) + } + if filter.NodeFreeHRU != nil { + q = q.Where("resources_cache.free_hru >= ?", *filter.NodeFreeHRU) + } + if filter.NodeFreeSRU != nil { + q = q.Where("resources_cache.free_sru >= ?", *filter.NodeFreeSRU) + } + if filter.NodeTotalCRU != nil { + q = q.Where("resources_cache.total_cru >= ?", *filter.NodeTotalCRU) + } + + if filter.NodeAvailableFor != nil { + q = q.Where("COALESCE(resources_cache.renter, 0) = ? OR (resources_cache.renter IS NULL AND farm.dedicated_farm = false)", *filter.NodeAvailableFor) + } + + if filter.NodeHasGPU != nil { + q = q.Where("(resources_cache.node_gpu_count > 0) = ?", *filter.NodeHasGPU) + } + + if filter.NodeRentedBy != nil { + q = q.Where("COALESCE(resources_cache.renter, 0) = ?", *filter.NodeRentedBy) + } + + if filter.Country != nil { + q = q.Where("LOWER(node.country) = LOWER(?)", *filter.Country) + } + + if filter.Region != nil { + q = q.Where("LOWER(country.subregion) = LOWER(?)", *filter.Region) + } + + if filter.NodeStatus != nil { + condition := nodestatus.DecideNodeStatusCondition(*filter.NodeStatus) + q = q.Where(condition) + } + + if filter.NodeCertified != nil { + q = q.Where("(node.certification = 'Certified') = ?", *filter.NodeCertified) + } + + if filter.FreeIPs != nil { + q = q.Where("COALESCE(public_ip_count.free_ips, 0) >= ?", *filter.FreeIPs) + } + if filter.TotalIPs != nil { + q = q.Where("COALESCE(public_ip_count.total_ips, 0) >= ?", *filter.TotalIPs) + } + if filter.StellarAddress != nil { + q = q.Where("farm.stellar_address = ?", *filter.StellarAddress) + } + if filter.PricingPolicyID != nil { + q = q.Where("farm.pricing_policy_id = ?", *filter.PricingPolicyID) + } + if filter.FarmID != nil { + q = q.Where("farm.farm_id = ?", *filter.FarmID) + } + if filter.TwinID != nil { + q = q.Where("farm.twin_id = ?", *filter.TwinID) + } + if filter.Name != nil { + q = q.Where("LOWER(farm.name) = LOWER(?)", *filter.Name) + } + + if filter.NameContains != nil { + escaped := strings.Replace(*filter.NameContains, "%", "\\%", -1) + escaped = strings.Replace(escaped, "_", "\\_", -1) + q = q.Where("farm.name ILIKE ?", fmt.Sprintf("%%%s%%", escaped)) + } + + if filter.CertificationType != nil { + q = q.Where("farm.certification = ?", *filter.CertificationType) + } + + if filter.Dedicated != nil { + q = q.Where("farm.dedicated_farm = ?", *filter.Dedicated) + } + var count int64 + if limit.Randomize || limit.RetCount { + if res := q.Count(&count); res.Error != nil { + return nil, 0, errors.Wrap(res.Error, "couldn't get farm count") + } + } + + // Sorting + if limit.Randomize { + q = q.Order("random()") + } else { + if filter.NodeAvailableFor != nil { + q = q.Order("(case when bool_or(resources_cache.renter is not null) then 1 else 2 end)") + } + if limit.SortBy != "" { + order := types.SortOrderAsc + if strings.EqualFold(string(limit.SortOrder), string(types.SortOrderDesc)) { + order = types.SortOrderDesc + } + q = q.Order(fmt.Sprintf("%s %s", limit.SortBy, order)) + } else { + q = q.Order("farm.farm_id") + } + } + + // Pagination + q = q.Limit(int(limit.Size)).Offset(int(limit.Page-1) * int(limit.Size)) + + var farms []Farm + if res := q.Scan(&farms); res.Error != nil { + return farms, uint(count), errors.Wrap(res.Error, "failed to scan returned farm from database") + } + return farms, uint(count), nil +} + // GetNodes returns nodes filtered and paginated func (d *PostgresDatabase) GetNodes(ctx context.Context, filter types.NodeFilter, limit types.Limit) ([]Node, uint, error) { q := d.nodeTableQuery() @@ -430,25 +597,25 @@ func (d *PostgresDatabase) GetNodes(ctx context.Context, filter types.NodeFilter q = q.Where(condition) if filter.FreeMRU != nil { - q = q.Where("nodes_resources_view.free_mru >= ?", *filter.FreeMRU) + q = q.Where("resources_cache.free_mru >= ?", *filter.FreeMRU) } if filter.FreeHRU != nil { - q = q.Where("nodes_resources_view.free_hru >= ?", *filter.FreeHRU) + q = q.Where("resources_cache.free_hru >= ?", *filter.FreeHRU) } if filter.FreeSRU != nil { - q = q.Where("nodes_resources_view.free_sru >= ?", *filter.FreeSRU) + q = q.Where("resources_cache.free_sru >= ?", *filter.FreeSRU) } if filter.TotalCRU != nil { - q = q.Where("nodes_resources_view.total_cru >= ?", *filter.TotalCRU) + q = q.Where("resources_cache.total_cru >= ?", *filter.TotalCRU) } if filter.TotalHRU != nil { - q = q.Where("nodes_resources_view.total_hru >= ?", *filter.TotalHRU) + q = q.Where("resources_cache.total_hru >= ?", *filter.TotalHRU) } if filter.TotalMRU != nil { - q = q.Where("nodes_resources_view.total_mru >= ?", *filter.TotalMRU) + q = q.Where("resources_cache.total_mru >= ?", *filter.TotalMRU) } if filter.TotalSRU != nil { - q = q.Where("nodes_resources_view.total_sru >= ?", *filter.TotalSRU) + q = q.Where("resources_cache.total_sru >= ?", *filter.TotalSRU) } if filter.Country != nil { q = q.Where("LOWER(node.country) = LOWER(?)", *filter.Country) @@ -501,19 +668,19 @@ func (d *PostgresDatabase) GetNodes(ctx context.Context, filter types.NodeFilter q = q.Where(`farm.dedicated_farm = ?`, *filter.InDedicatedFarm) } if filter.Dedicated != nil { - q = q.Where(`? = (farm.dedicated_farm = true OR COALESCE(node_contract.contract_id, 0) = 0 OR COALESCE(rent_contract.contract_id, 0) != 0)`, *filter.Dedicated) + q = q.Where(`? = (farm.dedicated_farm = true OR resources_cache.node_contracts_count = 0 OR resources_cache.renter is not null)`, *filter.Dedicated) } if filter.Rentable != nil { - q = q.Where(`? = ((farm.dedicated_farm = true OR COALESCE(node_contract.contract_id, 0) = 0) AND COALESCE(rent_contract.contract_id, 0) = 0)`, *filter.Rentable) + q = q.Where(`? = ((farm.dedicated_farm = true OR resources_cache.node_contracts_count = 0) AND resources_cache.renter is null)`, *filter.Rentable) } if filter.AvailableFor != nil { - q = q.Where(`COALESCE(rent_contract.twin_id, 0) = ? OR (COALESCE(rent_contract.twin_id, 0) = 0 AND farm.dedicated_farm = false)`, *filter.AvailableFor) + q = q.Where(`COALESCE(resources_cache.renter, 0) = ? OR (resources_cache.renter is null AND farm.dedicated_farm = false)`, *filter.AvailableFor) } if filter.RentedBy != nil { - q = q.Where(`COALESCE(rent_contract.twin_id, 0) = ?`, *filter.RentedBy) + q = q.Where(`COALESCE(resources_cache.renter, 0) = ?`, *filter.RentedBy) } if filter.Rented != nil { - q = q.Where(`? = (COALESCE(rent_contract.contract_id, 0) != 0)`, *filter.Rented) + q = q.Where(`? = (resources_cache.renter is not null)`, *filter.Rented) } if filter.CertificationType != nil { q = q.Where("node.certification ILIKE ?", *filter.CertificationType) @@ -579,7 +746,7 @@ func (d *PostgresDatabase) GetNodes(ctx context.Context, filter types.NodeFilter q = q.Order("random()") } else { if filter.AvailableFor != nil { - q = q.Order("(case when rent_contract is not null then 1 else 2 end)") + q = q.Order("(case when resources_cache.renter is not null then 1 else 2 end)") } if limit.SortBy != "" { diff --git a/grid-proxy/internal/explorer/db/types.go b/grid-proxy/internal/explorer/db/types.go index a2404a7cb..4dfb529ba 100644 --- a/grid-proxy/internal/explorer/db/types.go +++ b/grid-proxy/internal/explorer/db/types.go @@ -41,41 +41,41 @@ type DBContract struct { // Node data about a node which is calculated from the chain type Node struct { - ID string - NodeID int64 - FarmID int64 - TwinID int64 - Country string - GridVersion int64 - City string - Uptime int64 - Created int64 - FarmingPolicyID int64 - UpdatedAt int64 - TotalCru int64 - TotalMru int64 - TotalSru int64 - TotalHru int64 - UsedCru int64 - UsedMru int64 - UsedSru int64 - UsedHru int64 - Domain string - Gw4 string - Gw6 string - Ipv4 string - Ipv6 string - Certification string - FarmDedicated bool `gorm:"farm_dedicated"` - RentContractID int64 - RentedByTwinID int64 - SerialNumber string - Longitude *float64 - Latitude *float64 - Power NodePower `gorm:"type:jsonb"` - NumGPU int `gorm:"num_gpu"` - ExtraFee uint64 - HasNodeContract bool `gorm:"has_node_contract"` + ID string + NodeID int64 + FarmID int64 + TwinID int64 + Country string + GridVersion int64 + City string + Uptime int64 + Created int64 + FarmingPolicyID int64 + UpdatedAt int64 + TotalCru int64 + TotalMru int64 + TotalSru int64 + TotalHru int64 + UsedCru int64 + UsedMru int64 + UsedSru int64 + UsedHru int64 + Domain string + Gw4 string + Gw6 string + Ipv4 string + Ipv6 string + Certification string + FarmDedicated bool `gorm:"farm_dedicated"` + RentContractID int64 + Renter int64 + SerialNumber string + Longitude *float64 + Latitude *float64 + Power NodePower `gorm:"type:jsonb"` + NumGPU int `gorm:"num_gpu"` + ExtraFee uint64 + NodeContractsCount uint64 `gorm:"node_contracts_count"` } // NodePower struct is the farmerbot report for node status diff --git a/grid-proxy/pkg/nodestatus/nodestatus.go b/grid-proxy/pkg/nodestatus/nodestatus.go index 695eec9e4..d567de2c2 100644 --- a/grid-proxy/pkg/nodestatus/nodestatus.go +++ b/grid-proxy/pkg/nodestatus/nodestatus.go @@ -18,7 +18,7 @@ const ( func DecideNodeStatusCondition(status string) string { condition := "TRUE" - nilPower := "node.power IS NULL" + nilPower := "node.power->> 'state' = '' AND node.power->> 'target' = ''" poweredOn := "node.power->> 'state' = 'Up' AND node.power->> 'target' = 'Up'" poweredOff := "node.power->> 'state' = 'Down' AND node.power->> 'target' = 'Down'" diff --git a/grid-proxy/pkg/types/farms.go b/grid-proxy/pkg/types/farms.go index 1337d7326..0f4c3266f 100644 --- a/grid-proxy/pkg/types/farms.go +++ b/grid-proxy/pkg/types/farms.go @@ -36,6 +36,7 @@ type FarmFilter struct { NodeFreeMRU *uint64 `schema:"node_free_mru,omitempty"` NodeFreeHRU *uint64 `schema:"node_free_hru,omitempty"` NodeFreeSRU *uint64 `schema:"node_free_sru,omitempty"` + NodeTotalCRU *uint64 `schema:"node_total_cru,omitempty"` NodeStatus *string `schema:"node_status,omitempty"` NodeRentedBy *uint64 `schema:"node_rented_by,omitempty"` NodeAvailableFor *uint64 `schema:"node_available_for,omitempty"` diff --git a/grid-proxy/tests/queries/farm_test.go b/grid-proxy/tests/queries/farm_test.go index b4d2ad4a2..d16209c56 100644 --- a/grid-proxy/tests/queries/farm_test.go +++ b/grid-proxy/tests/queries/farm_test.go @@ -85,6 +85,11 @@ var farmFilterRandomValueGenerator = map[string]func(agg FarmsAggregate) interfa sru := uint64(rand.Int63n(int64(aggNode.maxFreeSRU))) return &sru }, + "NodeTotalCRU": func(agg FarmsAggregate) interface{} { + aggNode := calcNodesAggregates(&data) + cru := uint64(rand.Int63n(int64(aggNode.maxTotalCRU))) + return &cru + }, "NodeStatus": func(agg FarmsAggregate) interface{} { nodeStatuses := []string{"up", "down", "standby"} return &nodeStatuses[rand.Intn(len(nodeStatuses))] diff --git a/grid-proxy/tests/queries/mock_client/farms.go b/grid-proxy/tests/queries/mock_client/farms.go index 0b5b052e7..734d8ceef 100644 --- a/grid-proxy/tests/queries/mock_client/farms.go +++ b/grid-proxy/tests/queries/mock_client/farms.go @@ -109,10 +109,12 @@ func (f *Farm) satisfies(filter types.FarmFilter, data *DBData) bool { return false } - if filter.NodeAvailableFor != nil || filter.NodeCertified != nil || filter.NodeFreeHRU != nil || - filter.NodeFreeMRU != nil || filter.NodeFreeSRU != nil || filter.NodeHasGPU != nil || - filter.NodeRentedBy != nil || filter.NodeStatus != nil || filter.Country != nil || filter.Region != nil { - + if filter.NodeAvailableFor != nil || filter.NodeCertified != nil || + filter.NodeFreeHRU != nil || filter.NodeFreeMRU != nil || + filter.NodeFreeSRU != nil || filter.NodeHasGPU != nil || + filter.NodeRentedBy != nil || filter.NodeStatus != nil || + filter.Country != nil || filter.Region != nil || + filter.NodeTotalCRU != nil { if !f.satisfyFarmNodesFilter(data, filter) { return false } @@ -142,6 +144,10 @@ func (f *Farm) satisfyFarmNodesFilter(data *DBData, filter types.FarmFilter) boo continue } + if filter.NodeTotalCRU != nil && total.CRU < *filter.NodeTotalCRU { + continue + } + if filter.NodeAvailableFor != nil && ((data.NodeRentedBy[node.NodeID] != 0 && data.NodeRentedBy[node.NodeID] != *filter.NodeAvailableFor) || (data.NodeRentedBy[node.NodeID] != *filter.NodeAvailableFor && data.Farms[node.FarmID].DedicatedFarm)) { continue diff --git a/grid-proxy/tools/db/generate.go b/grid-proxy/tools/db/generate.go index 183476808..a04b497b3 100644 --- a/grid-proxy/tools/db/generate.go +++ b/grid-proxy/tools/db/generate.go @@ -589,10 +589,13 @@ func generateNodes(db *sql.DB) error { nodesHRU[i] = hru nodeUP[i] = up + // location latitue and longitue needs to be castable to decimal + // if not, the convert_to_decimal function will raise a notice + // reporting the incident, which downgrades performance location := location{ id: fmt.Sprintf("location-%d", i), - longitude: fmt.Sprintf("location--long-%d", i), - latitude: fmt.Sprintf("location-lat-%d", i), + longitude: fmt.Sprintf("%d", i), + latitude: fmt.Sprintf("%d", i), } countryIndex := r.Intn(len(countries)) From 72fb5f5b9eaaebe3c9f62ee63873b4c5a393d3c4 Mon Sep 17 00:00:00 2001 From: Omar Abdulaziz Date: Tue, 19 Dec 2023 16:56:02 +0200 Subject: [PATCH 07/39] wip: add public_ip_cache table Co-authored-by: Mario Bassem --- grid-proxy/Makefile | 1 + grid-proxy/cmds/proxy_server/main.go | 5 +- grid-proxy/internal/explorer/db/new_table.go | 88 --------- grid-proxy/internal/explorer/db/postgres.go | 189 ++++--------------- grid-proxy/internal/explorer/db/setup.sql | 156 +++++++++++++++ grid-proxy/tests/queries/conn_test.go | 3 +- grid-proxy/tests/queries/main_test.go | 3 +- 7 files changed, 203 insertions(+), 242 deletions(-) delete mode 100644 grid-proxy/internal/explorer/db/new_table.go create mode 100644 grid-proxy/internal/explorer/db/setup.sql diff --git a/grid-proxy/Makefile b/grid-proxy/Makefile index 7ed767520..6bf92d692 100644 --- a/grid-proxy/Makefile +++ b/grid-proxy/Makefile @@ -51,6 +51,7 @@ server-start: ## Start the proxy server (Args: `m=`) --postgres-db tfgrid-graphql \ --postgres-password postgres \ --postgres-user postgres \ + --sql-log-level 4 \ --mnemonics "$(m)" ; all-start: db-stop db-start sleep db-fill server-start ## Full start of the database and the server (Args: `m=`) diff --git a/grid-proxy/cmds/proxy_server/main.go b/grid-proxy/cmds/proxy_server/main.go index cf3ebc39b..ce26cc923 100644 --- a/grid-proxy/cmds/proxy_server/main.go +++ b/grid-proxy/cmds/proxy_server/main.go @@ -20,6 +20,7 @@ import ( logging "github.com/threefoldtech/tfgrid-sdk-go/grid-proxy/pkg" rmb "github.com/threefoldtech/tfgrid-sdk-go/rmb-sdk-go" "github.com/threefoldtech/tfgrid-sdk-go/rmb-sdk-go/peer" + "gorm.io/gorm/logger" ) const ( @@ -39,6 +40,7 @@ type flags struct { postgresDB string postgresUser string postgresPassword string + sqlLogLevel int address string version bool nocert bool @@ -65,6 +67,7 @@ func main() { flag.StringVar(&f.postgresDB, "postgres-db", "", "postgres database") flag.StringVar(&f.postgresUser, "postgres-user", "", "postgres username") flag.StringVar(&f.postgresPassword, "postgres-password", "", "postgres password") + flag.IntVar(&f.sqlLogLevel, "sql-log-level", 2, "sql logger level") flag.BoolVar(&f.version, "v", false, "shows the package version") flag.BoolVar(&f.nocert, "no-cert", false, "start the server without certificate") flag.StringVar(&f.domain, "domain", "", "domain on which the server will be served") @@ -113,7 +116,7 @@ func main() { log.Fatal().Err(err).Msg("failed to create relay client") } - db, err := db.NewPostgresDatabase(f.postgresHost, f.postgresPort, f.postgresUser, f.postgresPassword, f.postgresDB, f.maxPoolOpenConnections) + db, err := db.NewPostgresDatabase(f.postgresHost, f.postgresPort, f.postgresUser, f.postgresPassword, f.postgresDB, f.maxPoolOpenConnections, logger.LogLevel(f.sqlLogLevel)) if err != nil { log.Fatal().Err(err).Msg("couldn't get postgres client") } diff --git a/grid-proxy/internal/explorer/db/new_table.go b/grid-proxy/internal/explorer/db/new_table.go deleted file mode 100644 index 3cc29c52d..000000000 --- a/grid-proxy/internal/explorer/db/new_table.go +++ /dev/null @@ -1,88 +0,0 @@ -package db - -/* - node: - free_hru, free_mru, free_sru, total_hru, total_mru, total_sru, total_cru, - - total_resources - contract_resources -> used node resources - renter -> to know who rented it, available for - node_contracts -> to know whether it's rentable - nodeid, farmid - - triggers: - - trigger on node table (insert) - - trigger on total resources (insert/update) - - trigger on contract resources (insert/update) - - trigger on rent contract (insert/update) - - trigger on node contract (insert/update) - - triggers need to be in the same transaction with table creation -*/ - -var resourcesCache = ` - drop table if exists resources_cache; - CREATE TABLE IF NOT EXISTS resources_cache( - node_id INTEGER PRIMARY KEY, - farm_id INTEGER NOT NULL, - total_hru NUMERIC NOT NULL, - total_mru NUMERIC NOT NULL, - total_sru NUMERIC NOT NULL, - total_cru NUMERIC NOT NULL, - free_hru NUMERIC NOT NULL, - free_mru NUMERIC NOT NULL, - free_sru NUMERIC NOT NULL, - used_hru NUMERIC NOT NULL, - used_mru NUMERIC NOT NULL, - used_sru NUMERIC NOT NULL, - used_cru NUMERIC NOT NULL, - renter INTEGER, - rent_contract_id INTEGER, - node_contracts_count INTEGER NOT NULL, - node_gpu_count INTEGER NOT NULL - ); - - INSERT INTO resources_cache - SELECT * - FROM ( - SELECT node.node_id as node_id, - node.farm_id as farm_id, - COALESCE(node_resources_total.hru, 0) as total_hru, - COALESCE(node_resources_total.mru, 0) as total_mru, - COALESCE(node_resources_total.sru, 0) as total_sru, - COALESCE(node_resources_total.cru, 0) as total_cru, - node_resources_total.hru - COALESCE(sum(contract_resources.hru), 0) as free_hru, - node_resources_total.mru - COALESCE(sum(contract_resources.mru), 0) - GREATEST( - CAST((node_resources_total.mru / 10) AS bigint), - 2147483648 - ) as free_mru, - node_resources_total.sru - COALESCE(sum(contract_resources.sru), 0) - 21474836480 as free_sru, - COALESCE(sum(contract_resources.hru), 0) as used_hru, - COALESCE(sum(contract_resources.mru), 0) + GREATEST( - CAST((node_resources_total.mru / 10) AS bigint), - 2147483648 - ) as used_mru, - COALESCE(sum(contract_resources.sru), 0) + 21474836480 as used_sru, - COALESCE(sum(contract_resources.cru), 0) as used_cru, - rent_contract.twin_id as renter, - rent_contract.contract_id as rent_contract_id, - count(node_contract.contract_id) as node_contract_count, - count(node_gpu.id) as node_gpu_count - FROM contract_resources - JOIN node_contract as node_contract ON node_contract.resources_used_id = contract_resources.id - AND node_contract.state IN ('Created', 'GracePeriod') - RIGHT JOIN node as node ON node.node_id = node_contract.node_id - JOIN node_resources_total AS node_resources_total ON node_resources_total.node_id = node.id - LEFT JOIN rent_contract on node.node_id = rent_contract.node_id - AND rent_contract.state IN ('Created', 'GracePeriod') - LEFT JOIN node_gpu ON node.twin_id = node_gpu.node_twin_id - GROUP BY node.node_id, - node_resources_total.mru, - node_resources_total.sru, - node_resources_total.hru, - node_resources_total.cru, - node.farm_id, - rent_contract.contract_id, - rent_contract.twin_id - ) as node_resources; -` diff --git a/grid-proxy/internal/explorer/db/postgres.go b/grid-proxy/internal/explorer/db/postgres.go index ab7cabda3..608ac1756 100644 --- a/grid-proxy/internal/explorer/db/postgres.go +++ b/grid-proxy/internal/explorer/db/postgres.go @@ -28,88 +28,13 @@ var ( // ErrFarmNotFound farm not found ErrFarmNotFound = errors.New("farm not found") //ErrViewNotFound - ErrNodeResourcesViewNotFound = errors.New("ERROR: relation \"nodes_resources_view\" does not exist (SQLSTATE 42P01)") + ErrResourcesCacheTableNotFound = errors.New("ERROR: relation \"resources_cache\" does not exist (SQLSTATE 42P01)") // ErrContractNotFound contract not found ErrContractNotFound = errors.New("contract not found") ) -const ( - setupPostgresql = ` - CREATE INDEX IF NOT EXISTS idx_node_id ON public.node(node_id); - CREATE INDEX IF NOT EXISTS idx_twin_id ON public.twin(twin_id); - CREATE INDEX IF NOT EXISTS idx_farm_id ON public.farm(farm_id); - CREATE INDEX IF NOT EXISTS idx_contract_id ON public.node_contract(contract_id); - - CREATE OR REPLACE VIEW nodes_resources_view AS SELECT - node.node_id, - COALESCE(sum(contract_resources.cru), 0) as used_cru, - COALESCE(sum(contract_resources.mru), 0) + GREATEST(CAST((node_resources_total.mru / 10) AS bigint), 2147483648) as used_mru, - COALESCE(sum(contract_resources.hru), 0) as used_hru, - COALESCE(sum(contract_resources.sru), 0) + 21474836480 as used_sru, - node_resources_total.mru - COALESCE(sum(contract_resources.mru), 0) - GREATEST(CAST((node_resources_total.mru / 10) AS bigint), 2147483648) as free_mru, - node_resources_total.hru - COALESCE(sum(contract_resources.hru), 0) as free_hru, - node_resources_total.sru - COALESCE(sum(contract_resources.sru), 0) - 21474836480 as free_sru, - COALESCE(node_resources_total.cru, 0) as total_cru, - COALESCE(node_resources_total.mru, 0) as total_mru, - COALESCE(node_resources_total.hru, 0) as total_hru, - COALESCE(node_resources_total.sru, 0) as total_sru - FROM contract_resources - JOIN node_contract as node_contract - ON node_contract.resources_used_id = contract_resources.id AND node_contract.state IN ('Created', 'GracePeriod') - RIGHT JOIN node as node - ON node.node_id = node_contract.node_id - JOIN node_resources_total AS node_resources_total - ON node_resources_total.node_id = node.id - GROUP BY node.node_id, node_resources_total.mru, node_resources_total.sru, node_resources_total.hru, node_resources_total.cru; - - DROP FUNCTION IF EXISTS node_resources(query_node_id INTEGER); - CREATE OR REPLACE function node_resources(query_node_id INTEGER) - returns table (node_id INTEGER, used_cru NUMERIC, used_mru NUMERIC, used_hru NUMERIC, used_sru NUMERIC, free_mru NUMERIC, free_hru NUMERIC, free_sru NUMERIC, total_cru NUMERIC, total_mru NUMERIC, total_hru NUMERIC, total_sru NUMERIC) - as - $body$ - SELECT - node.node_id, - COALESCE(sum(contract_resources.cru), 0) as used_cru, - COALESCE(sum(contract_resources.mru), 0) + GREATEST(CAST((node_resources_total.mru / 10) AS bigint), 2147483648) as used_mru, - COALESCE(sum(contract_resources.hru), 0) as used_hru, - COALESCE(sum(contract_resources.sru), 0) + 21474836480 as used_sru, - node_resources_total.mru - COALESCE(sum(contract_resources.mru), 0) - GREATEST(CAST((node_resources_total.mru / 10) AS bigint), 2147483648) as free_mru, - node_resources_total.hru - COALESCE(sum(contract_resources.hru), 0) as free_hru, - node_resources_total.sru - COALESCE(sum(contract_resources.sru), 0) - 21474836480 as free_sru, - COALESCE(node_resources_total.cru, 0) as total_cru, - COALESCE(node_resources_total.mru, 0) as total_mru, - COALESCE(node_resources_total.hru, 0) as total_hru, - COALESCE(node_resources_total.sru, 0) as total_sru - FROM contract_resources - JOIN node_contract as node_contract - ON node_contract.resources_used_id = contract_resources.id AND node_contract.state IN ('Created', 'GracePeriod') - RIGHT JOIN node as node - ON node.node_id = node_contract.node_id - JOIN node_resources_total AS node_resources_total - ON node_resources_total.node_id = node.id - WHERE node.node_id = query_node_id - GROUP BY node.node_id, node_resources_total.mru, node_resources_total.sru, node_resources_total.hru, node_resources_total.cru; - $body$ - language sql; - - DROP FUNCTION IF EXISTS convert_to_decimal(v_input text); - CREATE OR REPLACE FUNCTION convert_to_decimal(v_input text) - RETURNS DECIMAL AS $$ - DECLARE v_dec_value DECIMAL DEFAULT NULL; - BEGIN - BEGIN - v_dec_value := v_input::DECIMAL; - EXCEPTION WHEN OTHERS THEN - RAISE NOTICE 'Invalid decimal value: "%". Returning NULL.', v_input; - RETURN NULL; - END; - RETURN v_dec_value; - END; - $$ LANGUAGE plpgsql; - - DROP TRIGGER IF EXISTS node_added ON node; - ` -) +//go:embed setup.sql +var setupFile string // PostgresDatabase postgres db client type PostgresDatabase struct { @@ -122,12 +47,12 @@ func (d *PostgresDatabase) GetConnectionString() string { } // NewPostgresDatabase returns a new postgres db client -func NewPostgresDatabase(host string, port int, user, password, dbname string, maxConns int) (Database, error) { +func NewPostgresDatabase(host string, port int, user, password, dbname string, maxConns int, logLevel logger.LogLevel) (Database, error) { connString := fmt.Sprintf("host=%s port=%d user=%s "+ "password=%s dbname=%s sslmode=disable", host, port, user, password, dbname) gormDB, err := gorm.Open(postgres.Open(connString), &gorm.Config{ - Logger: logger.Default.LogMode(logger.Info), + Logger: logger.Default.LogMode(logLevel), }) if err != nil { return nil, errors.Wrap(err, "failed to create orm wrapper around db") @@ -162,13 +87,7 @@ func (d *PostgresDatabase) Close() error { } func (d *PostgresDatabase) initialize() error { - err := d.gormDB.Exec(setupPostgresql).Error - // concatenate in one string before exec - // for _, view := range views { - // err = d.gormDB.Exec(view).Error - // } - - return err + return d.gormDB.Exec(setupFile).Error } // GetStats returns aggregate info about the grid @@ -305,9 +224,7 @@ func (d *PostgresDatabase) GetNode(ctx context.Context, nodeID uint32) (Node, er // GetFarm return farm info func (d *PostgresDatabase) GetFarm(ctx context.Context, farmID uint32) (Farm, error) { - //TODO: make a quick table here without needs to joins - // q := d.farmTableQuery() - q := d.gormDB + q := d.farmTableQuery(types.FarmFilter{}) q = q.WithContext(ctx) q = q.Where("farm.farm_id = ?", farmID) var farm Farm @@ -347,12 +264,6 @@ func printQuery(query string, args ...interface{}) { } func (d *PostgresDatabase) nodeTableQuery() *gorm.DB { - // subquery := d.gormDB.Table("node_contract"). - // Select("DISTINCT ON (node_id) node_id, contract_id"). - // Where("state IN ('Created', 'GracePeriod')") - - nodeGPUQuery := `(SELECT count(node_gpu.id) FROM node_gpu WHERE node_gpu.node_twin_id = node.twin_id) as num_gpu` - return d.gormDB. Table("node"). Select( @@ -366,7 +277,7 @@ func (d *PostgresDatabase) nodeTableQuery() *gorm.DB { "node.uptime", "node.created", "node.farming_policy_id", - "updated_at", + "node.updated_at", "resources_cache.total_cru", "resources_cache.total_sru", "resources_cache.total_hru", @@ -390,23 +301,14 @@ func (d *PostgresDatabase) nodeTableQuery() *gorm.DB { "node.power", "node.extra_fee", "resources_cache.node_contracts_count", - // "CASE WHEN node_contract.contract_id IS NOT NULL THEN 1 ELSE 0 END AS has_node_contract", - nodeGPUQuery, - ). - Joins( - "LEFT JOIN resources_cache ON node.node_id = resources_cache.node_id", - ). - Joins( - "LEFT JOIN public_config ON node.id = public_config.node_id", + "resources_cache.node_gpu_count AS num_gpu", ). Joins( - "LEFT JOIN farm ON node.farm_id = farm.farm_id", - ). - Joins( - "LEFT JOIN location ON node.location_id = location.id", - ). - Joins( - "LEFT JOIN country ON country.name = node.country", + `LEFT JOIN resources_cache ON node.node_id = resources_cache.node_id + LEFT JOIN public_config ON node.id = public_config.node_id + LEFT JOIN farm ON node.farm_id = farm.farm_id + LEFT JOIN location ON node.location_id = location.id + LEFT JOIN public_ips_cache ON public_ips_cache.farm_id = node.farm_id`, ) } @@ -422,46 +324,31 @@ func (d *PostgresDatabase) farmTableQuery(filter types.FarmFilter) *gorm.DB { "farm.certification", "farm.stellar_address", "farm.dedicated_farm as dedicated", - "COALESCE(public_ips.public_ips, '[]') as public_ips", + "public_ips_cache.ips as public_ips", ). - Joins(`left join( - SELECT - p1.farm_id, - COUNT(p1.id) total_ips, - COUNT(CASE WHEN p2.contract_id = 0 THEN 1 END) free_ips - FROM public_ip p1 - LEFT JOIN public_ip p2 ON p1.id = p2.id - GROUP BY p1.farm_id - ) public_ip_count on public_ip_count.farm_id = farm.id`). - Joins(`left join ( - select - farm_id, - jsonb_agg(jsonb_build_object('id', id, 'ip', ip, 'contract_id', contract_id, 'gateway', gateway)) as public_ips - from public_ip - GROUP BY farm_id - ) public_ips on public_ips.farm_id = farm.id`). - Group( - `farm.id, - farm.farm_id, - farm.name, - farm.twin_id, - farm.pricing_policy_id, - farm.certification, - farm.stellar_address, - farm.dedicated_farm, - COALESCE(public_ips.public_ips, '[]')`, + Joins( + "LEFT JOIN public_ips_cache ON public_ips_cache.farm_id = farm.farm_id", ) + if filter.NodeAvailableFor != nil || filter.NodeFreeHRU != nil || filter.NodeCertified != nil || filter.NodeFreeMRU != nil || filter.NodeFreeSRU != nil || filter.NodeHasGPU != nil || filter.NodeRentedBy != nil || filter.NodeStatus != nil || filter.NodeTotalCRU != nil || filter.Country != nil || filter.Region != nil { - q.Joins(` - RIGHT JOIN resources_cache ON resources_cache.farm_id = farm.farm_id - LEFT JOIN node ON node.node_id = resources_cache.node_id - LEFT JOIN country ON node.country = country.name - `) + q. + Joins(`RIGHT JOIN resources_cache ON resources_cache.farm_id = farm.farm_id + LEFT JOIN node ON node.farm_id = farm.farm_id`). + Group(` + farm.id, + farm.farm_id, + farm.name, + farm.twin_id, + farm.pricing_policy_id, + farm.certification, + farm.stellar_address, + farm.dedicated_farm, + public_ips_cache.ips`) } return q @@ -498,11 +385,11 @@ func (d *PostgresDatabase) GetFarms(ctx context.Context, filter types.FarmFilter } if filter.Country != nil { - q = q.Where("LOWER(node.country) = LOWER(?)", *filter.Country) + q = q.Where("LOWER(resources_cache.country) = LOWER(?)", *filter.Country) } if filter.Region != nil { - q = q.Where("LOWER(country.subregion) = LOWER(?)", *filter.Region) + q = q.Where("LOWER(resources_cache.region) = LOWER(?)", *filter.Region) } if filter.NodeStatus != nil { @@ -515,10 +402,10 @@ func (d *PostgresDatabase) GetFarms(ctx context.Context, filter types.FarmFilter } if filter.FreeIPs != nil { - q = q.Where("COALESCE(public_ip_count.free_ips, 0) >= ?", *filter.FreeIPs) + q = q.Where("public_ips_cache.free_ips >= ?", *filter.FreeIPs) } if filter.TotalIPs != nil { - q = q.Where("COALESCE(public_ip_count.total_ips, 0) >= ?", *filter.TotalIPs) + q = q.Where("public_ips_cache.total_ips >= ?", *filter.TotalIPs) } if filter.StellarAddress != nil { q = q.Where("farm.stellar_address = ?", *filter.StellarAddress) @@ -630,7 +517,7 @@ func (d *PostgresDatabase) GetNodes(ctx context.Context, filter types.NodeFilter q = q.Where("node.city ILIKE '%' || ? || '%'", *filter.CityContains) } if filter.Region != nil { - q = q.Where("LOWER(country.subregion) = LOWER(?)", *filter.Region) + q = q.Where("LOWER(resources_cache.subregion) = LOWER(?)", *filter.Region) } if filter.NodeID != nil { q = q.Where("node.node_id = ?", *filter.NodeID) @@ -648,7 +535,7 @@ func (d *PostgresDatabase) GetNodes(ctx context.Context, filter types.NodeFilter q = q.Where("farm.name ILIKE '%' || ? || '%'", *filter.FarmNameContains) } if filter.FreeIPs != nil { - q = q.Where("(SELECT count(id) from public_ip WHERE public_ip.farm_id = farm.id AND public_ip.contract_id = 0) >= ?", *filter.FreeIPs) + q = q.Where("public_ips_cache.free_ips >= ?", *filter.FreeIPs) } if filter.IPv4 != nil { q = q.Where("(COALESCE(public_config.ipv4, '') = '') != ?", *filter.IPv4) @@ -775,7 +662,7 @@ func (d *PostgresDatabase) GetNodes(ctx context.Context, filter types.NodeFilter } func (d *PostgresDatabase) shouldRetry(resError error) bool { - if resError != nil && resError.Error() == ErrNodeResourcesViewNotFound.Error() { + if resError != nil && resError.Error() == ErrResourcesCacheTableNotFound.Error() { if err := d.initialize(); err != nil { log.Logger.Err(err).Msg("failed to reinitialize database") } else { diff --git a/grid-proxy/internal/explorer/db/setup.sql b/grid-proxy/internal/explorer/db/setup.sql new file mode 100644 index 000000000..9aa545009 --- /dev/null +++ b/grid-proxy/internal/explorer/db/setup.sql @@ -0,0 +1,156 @@ +/* + node: + free_hru, free_mru, free_sru, total_hru, total_mru, total_sru, total_cru, + + total_resources + contract_resources -> used node resources + renter -> to know who rented it, available for + node_contracts -> to know whether it's rentable + nodeid, farmid + + triggers: + - trigger on node table (insert) + - trigger on total resources (insert/update) + - trigger on contract resources (insert/update) + - trigger on rent contract (insert/update) + - trigger on node contract (insert/update) + + triggers need to be in the same transaction with table creation + */ + + +-- Helper functions + DROP FUNCTION IF EXISTS convert_to_decimal(v_input text); + CREATE OR REPLACE FUNCTION convert_to_decimal(v_input text) + RETURNS DECIMAL AS $$ + DECLARE v_dec_value DECIMAL DEFAULT NULL; + BEGIN + BEGIN + v_dec_value := v_input::DECIMAL; + EXCEPTION WHEN OTHERS THEN + RAISE NOTICE 'Invalid decimal value: "%". Returning NULL.', v_input; + RETURN NULL; + END; + RETURN v_dec_value; + END; + $$ LANGUAGE plpgsql; + +-- Clean old triggers + DROP TRIGGER IF EXISTS node_added ON node; + +-- Resources cache table +DROP TABLE IF EXISTS resources_cache; + +CREATE TABLE IF NOT EXISTS resources_cache( + node_id INTEGER PRIMARY KEY, + farm_id INTEGER NOT NULL, + total_hru NUMERIC NOT NULL, + total_mru NUMERIC NOT NULL, + total_sru NUMERIC NOT NULL, + total_cru NUMERIC NOT NULL, + free_hru NUMERIC NOT NULL, + free_mru NUMERIC NOT NULL, + free_sru NUMERIC NOT NULL, + used_hru NUMERIC NOT NULL, + used_mru NUMERIC NOT NULL, + used_sru NUMERIC NOT NULL, + used_cru NUMERIC NOT NULL, + renter INTEGER, + rent_contract_id INTEGER, + node_contracts_count INTEGER NOT NULL, + node_gpu_count INTEGER NOT NULL, + country TEXT, + region TEXT +); + +INSERT INTO resources_cache +SELECT * +FROM ( + SELECT + node.node_id as node_id, + node.farm_id as farm_id, + COALESCE(node_resources_total.hru, 0) as total_hru, + COALESCE(node_resources_total.mru, 0) as total_mru, + COALESCE(node_resources_total.sru, 0) as total_sru, + COALESCE(node_resources_total.cru, 0) as total_cru, + node_resources_total.hru - COALESCE(sum(contract_resources.hru), 0) as free_hru, + node_resources_total.mru - COALESCE(sum(contract_resources.mru), 0) - GREATEST( + CAST((node_resources_total.mru / 10) AS bigint), + 2147483648 + ) as free_mru, + node_resources_total.sru - COALESCE(sum(contract_resources.sru), 0) - 21474836480 as free_sru, + COALESCE(sum(contract_resources.hru), 0) as used_hru, + COALESCE(sum(contract_resources.mru), 0) + GREATEST( + CAST((node_resources_total.mru / 10) AS bigint), + 2147483648 + ) as used_mru, + COALESCE(sum(contract_resources.sru), 0) + 21474836480 as used_sru, + COALESCE(sum(contract_resources.cru), 0) as used_cru, + rent_contract.twin_id as renter, + rent_contract.contract_id as rent_contract_id, + count(node_contract.contract_id) as node_contract_count, + count(node_gpu.id) as node_gpu_count, + country.name as country, + country.subregion as region + + FROM contract_resources + JOIN node_contract as node_contract ON node_contract.resources_used_id = contract_resources.id + AND node_contract.state IN ('Created', 'GracePeriod') + + RIGHT JOIN node as node ON node.node_id = node_contract.node_id + + JOIN node_resources_total AS node_resources_total ON node_resources_total.node_id = node.id + + LEFT JOIN rent_contract on node.node_id = rent_contract.node_id + AND rent_contract.state IN ('Created', 'GracePeriod') + + LEFT JOIN node_gpu ON node.twin_id = node_gpu.node_twin_id + + LEFT JOIN country ON node.country = country.name + + GROUP BY node.node_id, + node_resources_total.mru, + node_resources_total.sru, + node_resources_total.hru, + node_resources_total.cru, + node.farm_id, + rent_contract.contract_id, + rent_contract.twin_id, + country.name, + country.subregion + ) as node_resources; + + +-- PublicIpsCache table +DROP TABLE IF EXISTS public_ips_cache; +CREATE TABLE IF NOT EXISTS public_ips_cache( + farm_id INTEGER PRIMARY KEY, + free_ips INTEGER NOT NULL, + total_ips INTEGER NOT NULL, + ips jsonb); +INSERT INTO public_ips_cache +SELECT + farm.farm_id, + COALESCE(public_ip_agg.free_ips, 0), + COALESCE(public_ip_agg.total_ips, 0), + COALESCE(public_ip_agg.ips, '[]') +FROM farm +LEFT JOIN( + SELECT + p1.farm_id, + COUNT(p1.id) total_ips, + COUNT(CASE WHEN p2.contract_id = 0 THEN 1 END) free_ips, + jsonb_agg(jsonb_build_object('id', p1.id, 'ip', p1.ip, 'contract_id', p1.contract_id, 'gateway', p1.gateway)) as ips + FROM public_ip p1 + LEFT JOIN public_ip p2 ON p1.id = p2.id + GROUP BY p1.farm_id +) public_ip_agg on public_ip_agg.farm_id = farm.id; + + +-- Create Indices + CREATE INDEX IF NOT EXISTS idx_node_id ON public.node(node_id); + CREATE INDEX IF NOT EXISTS idx_twin_id ON public.twin(twin_id); + CREATE INDEX IF NOT EXISTS idx_farm_id ON public.farm(farm_id); + CREATE INDEX IF NOT EXISTS idx_contract_id ON public.node_contract(contract_id); + + diff --git a/grid-proxy/tests/queries/conn_test.go b/grid-proxy/tests/queries/conn_test.go index 629fb473a..516fc7984 100644 --- a/grid-proxy/tests/queries/conn_test.go +++ b/grid-proxy/tests/queries/conn_test.go @@ -12,10 +12,11 @@ import ( "github.com/stretchr/testify/require" db "github.com/threefoldtech/tfgrid-sdk-go/grid-proxy/internal/explorer/db" "github.com/threefoldtech/tfgrid-sdk-go/grid-proxy/pkg/types" + "gorm.io/gorm/logger" ) func TestDBManyOpenConnections(t *testing.T) { - p, err := db.NewPostgresDatabase(POSTGRES_HOST, POSTGRES_PORT, POSTGRES_USER, POSTGRES_PASSSWORD, POSTGRES_DB, 80) + p, err := db.NewPostgresDatabase(POSTGRES_HOST, POSTGRES_PORT, POSTGRES_USER, POSTGRES_PASSSWORD, POSTGRES_DB, 80, logger.Error) require.NoError(t, err) gotQueriesCnt := atomic.Int32{} diff --git a/grid-proxy/tests/queries/main_test.go b/grid-proxy/tests/queries/main_test.go index 49b90b102..f0146bdd5 100644 --- a/grid-proxy/tests/queries/main_test.go +++ b/grid-proxy/tests/queries/main_test.go @@ -16,6 +16,7 @@ import ( proxyDB "github.com/threefoldtech/tfgrid-sdk-go/grid-proxy/internal/explorer/db" proxyclient "github.com/threefoldtech/tfgrid-sdk-go/grid-proxy/pkg/client" mock "github.com/threefoldtech/tfgrid-sdk-go/grid-proxy/tests/queries/mock_client" + "gorm.io/gorm/logger" ) var ( @@ -65,7 +66,7 @@ func TestMain(m *testing.M) { if err != nil { panic(err) } - DBClient, err = proxyDB.NewPostgresDatabase(POSTGRES_HOST, POSTGRES_PORT, POSTGRES_USER, POSTGRES_PASSSWORD, POSTGRES_DB, 80) + DBClient, err = proxyDB.NewPostgresDatabase(POSTGRES_HOST, POSTGRES_PORT, POSTGRES_USER, POSTGRES_PASSSWORD, POSTGRES_DB, 80, logger.Error) if err != nil { panic(err) } From fefd59564970a93379bcd99782ad30f230474e4f Mon Sep 17 00:00:00 2001 From: mario Date: Tue, 19 Dec 2023 18:56:40 +0200 Subject: [PATCH 08/39] fixes in farm query joins Signed-off-by: mario --- grid-proxy/internal/explorer/db/postgres.go | 47 +++++++++++---------- 1 file changed, 25 insertions(+), 22 deletions(-) diff --git a/grid-proxy/internal/explorer/db/postgres.go b/grid-proxy/internal/explorer/db/postgres.go index 608ac1756..57884844c 100644 --- a/grid-proxy/internal/explorer/db/postgres.go +++ b/grid-proxy/internal/explorer/db/postgres.go @@ -224,8 +224,7 @@ func (d *PostgresDatabase) GetNode(ctx context.Context, nodeID uint32) (Node, er // GetFarm return farm info func (d *PostgresDatabase) GetFarm(ctx context.Context, farmID uint32) (Farm, error) { - q := d.farmTableQuery(types.FarmFilter{}) - q = q.WithContext(ctx) + q := d.gormDB.WithContext(ctx).Table("farm") q = q.Where("farm.farm_id = ?", farmID) var farm Farm if res := q.Scan(&farm); res.Error != nil { @@ -312,7 +311,7 @@ func (d *PostgresDatabase) nodeTableQuery() *gorm.DB { ) } -func (d *PostgresDatabase) farmTableQuery(filter types.FarmFilter) *gorm.DB { +func (d *PostgresDatabase) farmTableQuery(filter types.FarmFilter, nodeQuery *gorm.DB) *gorm.DB { q := d.gormDB. Table("farm"). Select( @@ -337,8 +336,7 @@ func (d *PostgresDatabase) farmTableQuery(filter types.FarmFilter) *gorm.DB { filter.NodeTotalCRU != nil || filter.Country != nil || filter.Region != nil { q. - Joins(`RIGHT JOIN resources_cache ON resources_cache.farm_id = farm.farm_id - LEFT JOIN node ON node.farm_id = farm.farm_id`). + Joins(`RIGHT JOIN (?) AS resources_cache on resources_cache.farm_id = farm.farm_id`, nodeQuery). Group(` farm.id, farm.farm_id, @@ -356,49 +354,54 @@ func (d *PostgresDatabase) farmTableQuery(filter types.FarmFilter) *gorm.DB { // GetFarms return farms filtered and paginated func (d *PostgresDatabase) GetFarms(ctx context.Context, filter types.FarmFilter, limit types.Limit) ([]Farm, uint, error) { - q := d.farmTableQuery(filter) - q = q.WithContext(ctx) + q := d.gormDB.WithContext(ctx) + nodeQuery := d.gormDB.Table("resources_cache"). + Select("resources_cache.farm_id", "renter"). + Joins("LEFT JOIN node ON node.node_id = resources_cache.node_id"). + Group(`resources_cache.farm_id, renter`) if filter.NodeFreeMRU != nil { - q = q.Where("resources_cache.free_mru >= ?", *filter.NodeFreeMRU) + nodeQuery = nodeQuery.Where("resources_cache.free_mru >= ?", *filter.NodeFreeMRU) } if filter.NodeFreeHRU != nil { - q = q.Where("resources_cache.free_hru >= ?", *filter.NodeFreeHRU) + nodeQuery = nodeQuery.Where("resources_cache.free_hru >= ?", *filter.NodeFreeHRU) } if filter.NodeFreeSRU != nil { - q = q.Where("resources_cache.free_sru >= ?", *filter.NodeFreeSRU) + nodeQuery = nodeQuery.Where("resources_cache.free_sru >= ?", *filter.NodeFreeSRU) } if filter.NodeTotalCRU != nil { - q = q.Where("resources_cache.total_cru >= ?", *filter.NodeTotalCRU) - } - - if filter.NodeAvailableFor != nil { - q = q.Where("COALESCE(resources_cache.renter, 0) = ? OR (resources_cache.renter IS NULL AND farm.dedicated_farm = false)", *filter.NodeAvailableFor) + nodeQuery = nodeQuery.Where("resources_cache.total_cru >= ?", *filter.NodeTotalCRU) } if filter.NodeHasGPU != nil { - q = q.Where("(resources_cache.node_gpu_count > 0) = ?", *filter.NodeHasGPU) + nodeQuery = nodeQuery.Where("(resources_cache.node_gpu_count > 0) = ?", *filter.NodeHasGPU) } if filter.NodeRentedBy != nil { - q = q.Where("COALESCE(resources_cache.renter, 0) = ?", *filter.NodeRentedBy) + nodeQuery = nodeQuery.Where("COALESCE(resources_cache.renter, 0) = ?", *filter.NodeRentedBy) } if filter.Country != nil { - q = q.Where("LOWER(resources_cache.country) = LOWER(?)", *filter.Country) + nodeQuery = nodeQuery.Where("LOWER(resources_cache.country) = LOWER(?)", *filter.Country) } if filter.Region != nil { - q = q.Where("LOWER(resources_cache.region) = LOWER(?)", *filter.Region) + nodeQuery = nodeQuery.Where("LOWER(resources_cache.region) = LOWER(?)", *filter.Region) } if filter.NodeStatus != nil { condition := nodestatus.DecideNodeStatusCondition(*filter.NodeStatus) - q = q.Where(condition) + nodeQuery = nodeQuery.Where(condition) } if filter.NodeCertified != nil { - q = q.Where("(node.certification = 'Certified') = ?", *filter.NodeCertified) + nodeQuery = nodeQuery.Where("(node.certification = 'Certified') = ?", *filter.NodeCertified) + } + + q = d.farmTableQuery(filter, nodeQuery) + + if filter.NodeAvailableFor != nil { + q = q.Where("COALESCE(resources_cache.renter, 0) = ? OR (resources_cache.renter IS NULL AND farm.dedicated_farm = false)", *filter.NodeAvailableFor) } if filter.FreeIPs != nil { @@ -448,7 +451,7 @@ func (d *PostgresDatabase) GetFarms(ctx context.Context, filter types.FarmFilter q = q.Order("random()") } else { if filter.NodeAvailableFor != nil { - q = q.Order("(case when bool_or(resources_cache.renter is not null) then 1 else 2 end)") + q = q.Order("(bool_or(resources_cache.renter IS NOT NULL)) DESC") } if limit.SortBy != "" { order := types.SortOrderAsc From 22bf872249f157fdc8d826d0dce2f706f9e9cc91 Mon Sep 17 00:00:00 2001 From: Omar Abdulaziz Date: Tue, 19 Dec 2023 22:20:38 +0200 Subject: [PATCH 09/39] fix nil location on node --- grid-proxy/internal/explorer/db/postgres.go | 123 +++++++++--------- grid-proxy/internal/explorer/db/setup.sql | 102 +++++++-------- .../tests/queries/mock_client/loader.go | 36 +++++ grid-proxy/tests/queries/mock_client/nodes.go | 12 +- grid-proxy/tests/queries/mock_client/types.go | 6 + grid-proxy/tests/queries/node_test.go | 14 -- 6 files changed, 158 insertions(+), 135 deletions(-) diff --git a/grid-proxy/internal/explorer/db/postgres.go b/grid-proxy/internal/explorer/db/postgres.go index 57884844c..32f67fe57 100644 --- a/grid-proxy/internal/explorer/db/postgres.go +++ b/grid-proxy/internal/explorer/db/postgres.go @@ -204,10 +204,9 @@ func (np *NodePower) Scan(value interface{}) error { // GetNode returns node info func (d *PostgresDatabase) GetNode(ctx context.Context, nodeID uint32) (Node, error) { - q := d.nodeTableQuery() + q := d.nodeTableQuery(types.NodeFilter{}, &gorm.DB{}) q = q.WithContext(ctx) q = q.Where("node.node_id = ?", nodeID) - q = q.Session(&gorm.Session{Logger: logger.Default.LogMode(logger.Silent)}) var node Node res := q.Scan(&node) if d.shouldRetry(res.Error) { @@ -262,8 +261,8 @@ func printQuery(query string, args ...interface{}) { fmt.Printf("node query: %s", query) } -func (d *PostgresDatabase) nodeTableQuery() *gorm.DB { - return d.gormDB. +func (d *PostgresDatabase) nodeTableQuery(filter types.NodeFilter, nodeGpuSubquery *gorm.DB) *gorm.DB { + q := d.gormDB. Table("node"). Select( "node.id", @@ -302,13 +301,23 @@ func (d *PostgresDatabase) nodeTableQuery() *gorm.DB { "resources_cache.node_contracts_count", "resources_cache.node_gpu_count AS num_gpu", ). - Joins( - `LEFT JOIN resources_cache ON node.node_id = resources_cache.node_id + Joins(` + LEFT JOIN resources_cache ON node.node_id = resources_cache.node_id + LEFT JOIN public_ips_cache ON public_ips_cache.farm_id = node.farm_id LEFT JOIN public_config ON node.id = public_config.node_id LEFT JOIN farm ON node.farm_id = farm.farm_id LEFT JOIN location ON node.location_id = location.id - LEFT JOIN public_ips_cache ON public_ips_cache.farm_id = node.farm_id`, + `) + + if filter.HasGPU != nil || filter.GpuDeviceName != nil || + filter.GpuVendorName != nil || filter.GpuVendorID != nil || + filter.GpuDeviceID != nil || filter.GpuAvailable != nil { + q.Joins( + `RIGHT JOIN (?) AS gpu ON gpu.node_twin_id = node.twin_id`, nodeGpuSubquery, ) + } + + return q } func (d *PostgresDatabase) farmTableQuery(filter types.FarmFilter, nodeQuery *gorm.DB) *gorm.DB { @@ -335,18 +344,18 @@ func (d *PostgresDatabase) farmTableQuery(filter types.FarmFilter, nodeQuery *go filter.NodeRentedBy != nil || filter.NodeStatus != nil || filter.NodeTotalCRU != nil || filter.Country != nil || filter.Region != nil { - q. - Joins(`RIGHT JOIN (?) AS resources_cache on resources_cache.farm_id = farm.farm_id`, nodeQuery). + q.Joins(`RIGHT JOIN (?) AS resources_cache on resources_cache.farm_id = farm.farm_id`, nodeQuery). Group(` - farm.id, - farm.farm_id, - farm.name, - farm.twin_id, - farm.pricing_policy_id, - farm.certification, - farm.stellar_address, - farm.dedicated_farm, - public_ips_cache.ips`) + farm.id, + farm.farm_id, + farm.name, + farm.twin_id, + farm.pricing_policy_id, + farm.certification, + farm.stellar_address, + farm.dedicated_farm, + public_ips_cache.ips + `) } return q @@ -476,8 +485,40 @@ func (d *PostgresDatabase) GetFarms(ctx context.Context, filter types.FarmFilter // GetNodes returns nodes filtered and paginated func (d *PostgresDatabase) GetNodes(ctx context.Context, filter types.NodeFilter, limit types.Limit) ([]Node, uint, error) { - q := d.nodeTableQuery() - q = q.WithContext(ctx) + q := d.gormDB.WithContext(ctx) + /* + used distinct selecting to avoid duplicated node after the join. + - postgres apply WHERE before DISTINCT so filters will still filter on the whole data. + - we don't return any gpu info on the node object so no worries of losing the data because DISTINCT. + */ + nodeGpuSubquery := d.gormDB.Table("node_gpu"). + Select("DISTINCT ON (node_twin_id) node_twin_id") + + if filter.HasGPU != nil { + nodeGpuSubquery = nodeGpuSubquery.Where("(COALESCE(node_gpu.id, '') != '') = ?", *filter.HasGPU) + } + + if filter.GpuDeviceName != nil { + nodeGpuSubquery = nodeGpuSubquery.Where("COALESCE(node_gpu.device, '') ILIKE '%' || ? || '%'", *filter.GpuDeviceName) + } + + if filter.GpuVendorName != nil { + nodeGpuSubquery = nodeGpuSubquery.Where("COALESCE(node_gpu.vendor, '') ILIKE '%' || ? || '%'", *filter.GpuVendorName) + } + + if filter.GpuVendorID != nil { + nodeGpuSubquery = nodeGpuSubquery.Where("COALESCE(node_gpu.id, '') ILIKE '%' || ? || '%'", *filter.GpuVendorID) + } + + if filter.GpuDeviceID != nil { + nodeGpuSubquery = nodeGpuSubquery.Where("COALESCE(node_gpu.id, '') ILIKE '%' || ? || '%'", *filter.GpuDeviceID) + } + + if filter.GpuAvailable != nil { + nodeGpuSubquery = nodeGpuSubquery.Where("(COALESCE(node_gpu.contract, 0) = 0) = ?", *filter.GpuAvailable) + } + + q = d.nodeTableQuery(filter, nodeGpuSubquery) condition := "TRUE" if filter.Status != nil { @@ -520,7 +561,7 @@ func (d *PostgresDatabase) GetNodes(ctx context.Context, filter types.NodeFilter q = q.Where("node.city ILIKE '%' || ? || '%'", *filter.CityContains) } if filter.Region != nil { - q = q.Where("LOWER(resources_cache.subregion) = LOWER(?)", *filter.Region) + q = q.Where("LOWER(resources_cache.region) = LOWER(?)", *filter.Region) } if filter.NodeID != nil { q = q.Where("node.node_id = ?", *filter.NodeID) @@ -579,46 +620,6 @@ func (d *PostgresDatabase) GetNodes(ctx context.Context, filter types.NodeFilter q = q.Where(`COALESCE(farm.twin_id, 0) = ?`, *filter.OwnedBy) } - /* - used distinct selecting to avoid duplicated node after the join. - - postgres apply WHERE before DISTINCT so filters will still filter on the whole data. - - we don't return any gpu info on the node object so no worries of losing the data because DISTINCT. - */ - nodeGpuSubquery := d.gormDB.Table("node_gpu"). - Select("DISTINCT ON (node_twin_id) node_twin_id") - - if filter.HasGPU != nil { - nodeGpuSubquery = nodeGpuSubquery.Where("(COALESCE(node_gpu.id, '') != '') = ?", *filter.HasGPU) - } - - if filter.GpuDeviceName != nil { - nodeGpuSubquery = nodeGpuSubquery.Where("COALESCE(node_gpu.device, '') ILIKE '%' || ? || '%'", *filter.GpuDeviceName) - } - - if filter.GpuVendorName != nil { - nodeGpuSubquery = nodeGpuSubquery.Where("COALESCE(node_gpu.vendor, '') ILIKE '%' || ? || '%'", *filter.GpuVendorName) - } - - if filter.GpuVendorID != nil { - nodeGpuSubquery = nodeGpuSubquery.Where("COALESCE(node_gpu.id, '') ILIKE '%' || ? || '%'", *filter.GpuVendorID) - } - - if filter.GpuDeviceID != nil { - nodeGpuSubquery = nodeGpuSubquery.Where("COALESCE(node_gpu.id, '') ILIKE '%' || ? || '%'", *filter.GpuDeviceID) - } - - if filter.GpuAvailable != nil { - nodeGpuSubquery = nodeGpuSubquery.Where("(COALESCE(node_gpu.contract, 0) = 0) = ?", *filter.GpuAvailable) - } - - if filter.HasGPU != nil || filter.GpuDeviceName != nil || filter.GpuVendorName != nil || filter.GpuVendorID != nil || - filter.GpuDeviceID != nil || filter.GpuAvailable != nil { - - q.Joins( - `INNER JOIN (?) AS gpu ON gpu.node_twin_id = node.twin_id`, nodeGpuSubquery, - ) - } - var count int64 if limit.Randomize || limit.RetCount { q = q.Session(&gorm.Session{}) diff --git a/grid-proxy/internal/explorer/db/setup.sql b/grid-proxy/internal/explorer/db/setup.sql index 9aa545009..e67b38146 100644 --- a/grid-proxy/internal/explorer/db/setup.sql +++ b/grid-proxy/internal/explorer/db/setup.sql @@ -64,62 +64,52 @@ CREATE TABLE IF NOT EXISTS resources_cache( ); INSERT INTO resources_cache -SELECT * -FROM ( - SELECT - node.node_id as node_id, - node.farm_id as farm_id, - COALESCE(node_resources_total.hru, 0) as total_hru, - COALESCE(node_resources_total.mru, 0) as total_mru, - COALESCE(node_resources_total.sru, 0) as total_sru, - COALESCE(node_resources_total.cru, 0) as total_cru, - node_resources_total.hru - COALESCE(sum(contract_resources.hru), 0) as free_hru, - node_resources_total.mru - COALESCE(sum(contract_resources.mru), 0) - GREATEST( - CAST((node_resources_total.mru / 10) AS bigint), - 2147483648 - ) as free_mru, - node_resources_total.sru - COALESCE(sum(contract_resources.sru), 0) - 21474836480 as free_sru, - COALESCE(sum(contract_resources.hru), 0) as used_hru, - COALESCE(sum(contract_resources.mru), 0) + GREATEST( - CAST((node_resources_total.mru / 10) AS bigint), - 2147483648 - ) as used_mru, - COALESCE(sum(contract_resources.sru), 0) + 21474836480 as used_sru, - COALESCE(sum(contract_resources.cru), 0) as used_cru, - rent_contract.twin_id as renter, - rent_contract.contract_id as rent_contract_id, - count(node_contract.contract_id) as node_contract_count, - count(node_gpu.id) as node_gpu_count, - country.name as country, - country.subregion as region - - FROM contract_resources - JOIN node_contract as node_contract ON node_contract.resources_used_id = contract_resources.id - AND node_contract.state IN ('Created', 'GracePeriod') - - RIGHT JOIN node as node ON node.node_id = node_contract.node_id - - JOIN node_resources_total AS node_resources_total ON node_resources_total.node_id = node.id - - LEFT JOIN rent_contract on node.node_id = rent_contract.node_id - AND rent_contract.state IN ('Created', 'GracePeriod') - - LEFT JOIN node_gpu ON node.twin_id = node_gpu.node_twin_id - - LEFT JOIN country ON node.country = country.name - - GROUP BY node.node_id, - node_resources_total.mru, - node_resources_total.sru, - node_resources_total.hru, - node_resources_total.cru, - node.farm_id, - rent_contract.contract_id, - rent_contract.twin_id, - country.name, - country.subregion - ) as node_resources; - +SELECT + node.node_id as node_id, + node.farm_id as farm_id, + COALESCE(node_resources_total.hru, 0) as total_hru, + COALESCE(node_resources_total.mru, 0) as total_mru, + COALESCE(node_resources_total.sru, 0) as total_sru, + COALESCE(node_resources_total.cru, 0) as total_cru, + node_resources_total.hru - COALESCE(sum(contract_resources.hru), 0) as free_hru, + node_resources_total.mru - COALESCE(sum(contract_resources.mru), 0) - GREATEST(CAST((node_resources_total.mru / 10) AS bigint), 2147483648) as free_mru, + node_resources_total.sru - COALESCE(sum(contract_resources.sru), 0) - 21474836480 as free_sru, + COALESCE(sum(contract_resources.hru), 0) as used_hru, + COALESCE(sum(contract_resources.mru), 0) + GREATEST(CAST((node_resources_total.mru / 10) AS bigint), 2147483648) as used_mru, + COALESCE(sum(contract_resources.sru), 0) + 21474836480 as used_sru, + COALESCE(sum(contract_resources.cru), 0) as used_cru, + rent_contract.twin_id as renter, + rent_contract.contract_id as rent_contract_id, + count(node_contract.contract_id) as node_contract_count, + count(node_gpu.id) as node_gpu_count, + country.name as country, + country.subregion as region + +FROM contract_resources + JOIN node_contract as node_contract ON node_contract.resources_used_id = contract_resources.id + AND node_contract.state IN ('Created', 'GracePeriod') + + RIGHT JOIN node as node ON node.node_id = node_contract.node_id + + JOIN node_resources_total AS node_resources_total ON node_resources_total.node_id = node.id + + LEFT JOIN rent_contract on node.node_id = rent_contract.node_id + AND rent_contract.state IN ('Created', 'GracePeriod') + + LEFT JOIN node_gpu ON node.twin_id = node_gpu.node_twin_id + + LEFT JOIN country ON node.country = country.name + +GROUP BY node.node_id, + node_resources_total.mru, + node_resources_total.sru, + node_resources_total.hru, + node_resources_total.cru, + node.farm_id, + rent_contract.contract_id, + rent_contract.twin_id, + country.name, + country.subregion; -- PublicIpsCache table DROP TABLE IF EXISTS public_ips_cache; diff --git a/grid-proxy/tests/queries/mock_client/loader.go b/grid-proxy/tests/queries/mock_client/loader.go index 04184d25f..daab00bd4 100644 --- a/grid-proxy/tests/queries/mock_client/loader.go +++ b/grid-proxy/tests/queries/mock_client/loader.go @@ -33,6 +33,7 @@ type DBData struct { NonDeletedContracts map[uint64][]uint64 GPUs map[uint64][]NodeGPU Regions map[string]string + Locations map[string]Location DB *sql.DB } @@ -516,6 +517,37 @@ func loadCountries(db *sql.DB, data *DBData) error { return nil } + +func loadLocations(db *sql.DB, data *DBData) error { + rows, err := db.Query(` + SELECT + COALESCE(id, ''), + COALESCE(longitude, ''), + COALESCE(latitude, '') + FROM + location; + `) + + if err != nil { + return err + } + + for rows.Next() { + var location Location + if err := rows.Scan( + &location.ID, + &location.Longitude, + &location.Latitude, + ); err != nil { + return err + } + + data.Locations[location.ID] = location + } + + return nil +} + func loadNodeGPUs(db *sql.DB, data *DBData) error { rows, err := db.Query(` SELECT @@ -569,6 +601,7 @@ func Load(db *sql.DB) (DBData, error) { GPUs: make(map[uint64][]NodeGPU), FarmHasRentedNode: make(map[uint64]bool), Regions: make(map[string]string), + Locations: make(map[string]Location), DB: db, } if err := loadNodes(db, &data); err != nil { @@ -610,6 +643,9 @@ func Load(db *sql.DB) (DBData, error) { if err := loadCountries(db, &data); err != nil { return data, err } + if err := loadLocations(db, &data); err != nil { + return data, err + } if err := calcNodesUsedResources(&data); err != nil { return data, err } diff --git a/grid-proxy/tests/queries/mock_client/nodes.go b/grid-proxy/tests/queries/mock_client/nodes.go index 82a403771..682fce4d2 100644 --- a/grid-proxy/tests/queries/mock_client/nodes.go +++ b/grid-proxy/tests/queries/mock_client/nodes.go @@ -59,8 +59,10 @@ func (g *GridProxyMockClient) Nodes(ctx context.Context, filter types.NodeFilter SRU: gridtypes.Unit(g.data.NodeUsedResources[node.NodeID].SRU), }, Location: types.Location{ - Country: node.Country, - City: node.City, + Country: node.Country, + City: node.City, + Longitude: g.data.Locations[node.LocationID].Longitude, + Latitude: g.data.Locations[node.LocationID].Latitude, }, PublicConfig: types.PublicConfig{ Domain: g.data.PublicConfigs[node.NodeID].Domain, @@ -137,8 +139,10 @@ func (g *GridProxyMockClient) Node(ctx context.Context, nodeID uint32) (res type }, }, Location: types.Location{ - Country: node.Country, - City: node.City, + Country: node.Country, + City: node.City, + Longitude: g.data.Locations[node.LocationID].Longitude, + Latitude: g.data.Locations[node.LocationID].Latitude, }, PublicConfig: types.PublicConfig{ Domain: g.data.PublicConfigs[node.NodeID].Domain, diff --git a/grid-proxy/tests/queries/mock_client/types.go b/grid-proxy/tests/queries/mock_client/types.go index e673d1215..57f8d758d 100644 --- a/grid-proxy/tests/queries/mock_client/types.go +++ b/grid-proxy/tests/queries/mock_client/types.go @@ -158,3 +158,9 @@ type Country struct { Lat string Long string } + +type Location struct { + ID string + Longitude *float64 + Latitude *float64 +} diff --git a/grid-proxy/tests/queries/node_test.go b/grid-proxy/tests/queries/node_test.go index 86f63a57e..fac602e02 100644 --- a/grid-proxy/tests/queries/node_test.go +++ b/grid-proxy/tests/queries/node_test.go @@ -343,20 +343,6 @@ func TestNode(t *testing.T) { assert.Equal(t, err.Error(), ErrNodeNotFound.Error()) }) - t.Run("nodes test without resources view", func(t *testing.T) { - db := data.DB - _, err := db.Exec("drop view nodes_resources_view ;") - assert.NoError(t, err) - - singleNodeCheck(t, mockClient, gridProxyClient) - assert.NoError(t, err) - - _, err = db.Exec("drop view nodes_resources_view ;") - assert.NoError(t, err) - - nodePaginationCheck(t, mockClient, gridProxyClient) - }) - t.Run("nodes test certification_type filter", func(t *testing.T) { certType := "Diy" nodes, _, err := gridProxyClient.Nodes(context.Background(), types.NodeFilter{CertificationType: &certType}, types.DefaultLimit()) From 9f3fd030c80a38a7fe137926ce890f9d2d8d26d1 Mon Sep 17 00:00:00 2001 From: mario Date: Wed, 20 Dec 2023 02:43:27 +0200 Subject: [PATCH 10/39] update server setup.sql Signed-off-by: mario --- grid-proxy/internal/explorer/db/setup.sql | 130 ++++++++++++---------- 1 file changed, 74 insertions(+), 56 deletions(-) diff --git a/grid-proxy/internal/explorer/db/setup.sql b/grid-proxy/internal/explorer/db/setup.sql index e67b38146..876a3f94f 100644 --- a/grid-proxy/internal/explorer/db/setup.sql +++ b/grid-proxy/internal/explorer/db/setup.sql @@ -17,30 +17,29 @@ triggers need to be in the same transaction with table creation */ - - +---- -- Helper functions - DROP FUNCTION IF EXISTS convert_to_decimal(v_input text); - CREATE OR REPLACE FUNCTION convert_to_decimal(v_input text) - RETURNS DECIMAL AS $$ - DECLARE v_dec_value DECIMAL DEFAULT NULL; - BEGIN - BEGIN - v_dec_value := v_input::DECIMAL; - EXCEPTION WHEN OTHERS THEN - RAISE NOTICE 'Invalid decimal value: "%". Returning NULL.', v_input; - RETURN NULL; - END; - RETURN v_dec_value; - END; - $$ LANGUAGE plpgsql; - +---- +DROP FUNCTION IF EXISTS convert_to_decimal(v_input text); +CREATE OR REPLACE FUNCTION convert_to_decimal(v_input text) RETURNS DECIMAL AS $$ +DECLARE v_dec_value DECIMAL DEFAULT NULL; +BEGIN BEGIN v_dec_value := v_input::DECIMAL; +EXCEPTION +WHEN OTHERS THEN RAISE NOTICE 'Invalid decimal value: "%". Returning NULL.', +v_input; +RETURN NULL; +END; +RETURN v_dec_value; +END; +$$ LANGUAGE plpgsql; +---- -- Clean old triggers - DROP TRIGGER IF EXISTS node_added ON node; - +---- +DROP TRIGGER IF EXISTS node_added ON node; +---- -- Resources cache table +---- DROP TABLE IF EXISTS resources_cache; - CREATE TABLE IF NOT EXISTS resources_cache( node_id INTEGER PRIMARY KEY, farm_id INTEGER NOT NULL, @@ -62,44 +61,46 @@ CREATE TABLE IF NOT EXISTS resources_cache( country TEXT, region TEXT ); - INSERT INTO resources_cache -SELECT - node.node_id as node_id, +SELECT node.node_id as node_id, node.farm_id as farm_id, COALESCE(node_resources_total.hru, 0) as total_hru, COALESCE(node_resources_total.mru, 0) as total_mru, COALESCE(node_resources_total.sru, 0) as total_sru, COALESCE(node_resources_total.cru, 0) as total_cru, node_resources_total.hru - COALESCE(sum(contract_resources.hru), 0) as free_hru, - node_resources_total.mru - COALESCE(sum(contract_resources.mru), 0) - GREATEST(CAST((node_resources_total.mru / 10) AS bigint), 2147483648) as free_mru, + node_resources_total.mru - COALESCE(sum(contract_resources.mru), 0) - GREATEST( + CAST((node_resources_total.mru / 10) AS bigint), + 2147483648 + ) as free_mru, node_resources_total.sru - COALESCE(sum(contract_resources.sru), 0) - 21474836480 as free_sru, COALESCE(sum(contract_resources.hru), 0) as used_hru, - COALESCE(sum(contract_resources.mru), 0) + GREATEST(CAST((node_resources_total.mru / 10) AS bigint), 2147483648) as used_mru, + COALESCE(sum(contract_resources.mru), 0) + GREATEST( + CAST((node_resources_total.mru / 10) AS bigint), + 2147483648 + ) as used_mru, COALESCE(sum(contract_resources.sru), 0) + 21474836480 as used_sru, COALESCE(sum(contract_resources.cru), 0) as used_cru, rent_contract.twin_id as renter, rent_contract.contract_id as rent_contract_id, count(node_contract.contract_id) as node_contract_count, - count(node_gpu.id) as node_gpu_count, + COALESCE(node_gpu.node_gpu_count, 0) as node_gpu_count, country.name as country, country.subregion as region - FROM contract_resources JOIN node_contract as node_contract ON node_contract.resources_used_id = contract_resources.id AND node_contract.state IN ('Created', 'GracePeriod') - RIGHT JOIN node as node ON node.node_id = node_contract.node_id - JOIN node_resources_total AS node_resources_total ON node_resources_total.node_id = node.id - LEFT JOIN rent_contract on node.node_id = rent_contract.node_id AND rent_contract.state IN ('Created', 'GracePeriod') - - LEFT JOIN node_gpu ON node.twin_id = node_gpu.node_twin_id - + Left JOIN( + SELECT node_twin_id, + COUNT(id) as node_gpu_count + FROM node_gpu + GROUP BY node_twin_id + ) AS node_gpu ON node.twin_id = node_gpu.node_twin_id LEFT JOIN country ON node.country = country.name - GROUP BY node.node_id, node_resources_total.mru, node_resources_total.sru, @@ -108,39 +109,56 @@ GROUP BY node.node_id, node.farm_id, rent_contract.contract_id, rent_contract.twin_id, + COALESCE(node_gpu.node_gpu_count, 0), country.name, country.subregion; - +---- -- PublicIpsCache table +---- DROP TABLE IF EXISTS public_ips_cache; CREATE TABLE IF NOT EXISTS public_ips_cache( farm_id INTEGER PRIMARY KEY, free_ips INTEGER NOT NULL, total_ips INTEGER NOT NULL, - ips jsonb); + ips jsonb +); INSERT INTO public_ips_cache -SELECT - farm.farm_id, +SELECT farm.farm_id, COALESCE(public_ip_agg.free_ips, 0), COALESCE(public_ip_agg.total_ips, 0), COALESCE(public_ip_agg.ips, '[]') FROM farm -LEFT JOIN( - SELECT - p1.farm_id, - COUNT(p1.id) total_ips, - COUNT(CASE WHEN p2.contract_id = 0 THEN 1 END) free_ips, - jsonb_agg(jsonb_build_object('id', p1.id, 'ip', p1.ip, 'contract_id', p1.contract_id, 'gateway', p1.gateway)) as ips - FROM public_ip p1 - LEFT JOIN public_ip p2 ON p1.id = p2.id - GROUP BY p1.farm_id -) public_ip_agg on public_ip_agg.farm_id = farm.id; - - + LEFT JOIN( + SELECT p1.farm_id, + COUNT(p1.id) total_ips, + COUNT( + CASE + WHEN p2.contract_id = 0 THEN 1 + END + ) free_ips, + jsonb_agg( + jsonb_build_object( + 'id', + p1.id, + 'ip', + p1.ip, + 'contract_id', + p1.contract_id, + 'gateway', + p1.gateway + ) + ) as ips + FROM public_ip p1 + LEFT JOIN public_ip p2 ON p1.id = p2.id + GROUP BY p1.farm_id + ) public_ip_agg on public_ip_agg.farm_id = farm.id; -- Create Indices - CREATE INDEX IF NOT EXISTS idx_node_id ON public.node(node_id); - CREATE INDEX IF NOT EXISTS idx_twin_id ON public.twin(twin_id); - CREATE INDEX IF NOT EXISTS idx_farm_id ON public.farm(farm_id); - CREATE INDEX IF NOT EXISTS idx_contract_id ON public.node_contract(contract_id); - - +CREATE INDEX IF NOT EXISTS idx_node_id ON public.node(node_id); +CREATE INDEX IF NOT EXISTS idx_twin_id ON public.twin(twin_id); +CREATE INDEX IF NOT EXISTS idx_farm_id ON public.farm(farm_id); +CREATE INDEX IF NOT EXISTS idx_contract_id ON public.node_contract(contract_id); +CREATE INDEX IF NOT EXISTS resources_cache_farm_id ON resources_cache (farm_id); +CREATE INDEX IF NOT EXISTS location_id ON location USING gin(id); +CREATE INDEX IF NOT EXISTS resources_cache_node_id ON resources_cache(node_id); +CREATE INDEX IF NOT EXISTS public_ips_cache_farm_id ON public_ips_cache(farm_id); +CREATE INDEX IF NOT EXISTS public_config_node_id ON public_config USING gin(node_id); \ No newline at end of file From 4f271d5d207202e5281bc6dcd0f833f24b38abb8 Mon Sep 17 00:00:00 2001 From: mario Date: Wed, 20 Dec 2023 02:44:13 +0200 Subject: [PATCH 11/39] fixes in talble joins and where conditions Signed-off-by: mario --- grid-proxy/internal/explorer/db/postgres.go | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/grid-proxy/internal/explorer/db/postgres.go b/grid-proxy/internal/explorer/db/postgres.go index 32f67fe57..f7103825b 100644 --- a/grid-proxy/internal/explorer/db/postgres.go +++ b/grid-proxy/internal/explorer/db/postgres.go @@ -313,7 +313,7 @@ func (d *PostgresDatabase) nodeTableQuery(filter types.NodeFilter, nodeGpuSubque filter.GpuVendorName != nil || filter.GpuVendorID != nil || filter.GpuDeviceID != nil || filter.GpuAvailable != nil { q.Joins( - `RIGHT JOIN (?) AS gpu ON gpu.node_twin_id = node.twin_id`, nodeGpuSubquery, + `INNER JOIN (?) AS gpu ON gpu.node_twin_id = node.twin_id`, nodeGpuSubquery, ) } @@ -582,13 +582,13 @@ func (d *PostgresDatabase) GetNodes(ctx context.Context, filter types.NodeFilter q = q.Where("public_ips_cache.free_ips >= ?", *filter.FreeIPs) } if filter.IPv4 != nil { - q = q.Where("(COALESCE(public_config.ipv4, '') = '') != ?", *filter.IPv4) + q = q.Where("(public_config.ipv4 IS NULL) != ?", *filter.IPv4) } if filter.IPv6 != nil { - q = q.Where("(COALESCE(public_config.ipv6, '') = '') != ?", *filter.IPv6) + q = q.Where("(public_config.ipv6 IS NULL) != ?", *filter.IPv6) } if filter.Domain != nil { - q = q.Where("(COALESCE(public_config.domain, '') = '') != ?", *filter.Domain) + q = q.Where("(public_config.domain IS NULL) != ?", *filter.Domain) } if filter.CertificationType != nil { q = q.Where("node.certification ILIKE ?", *filter.CertificationType) @@ -613,9 +613,6 @@ func (d *PostgresDatabase) GetNodes(ctx context.Context, filter types.NodeFilter if filter.Rented != nil { q = q.Where(`? = (resources_cache.renter is not null)`, *filter.Rented) } - if filter.CertificationType != nil { - q = q.Where("node.certification ILIKE ?", *filter.CertificationType) - } if filter.OwnedBy != nil { q = q.Where(`COALESCE(farm.twin_id, 0) = ?`, *filter.OwnedBy) } From 15ea8f69410d688d75ce5dee6de8897647b32dfa Mon Sep 17 00:00:00 2001 From: mario Date: Wed, 20 Dec 2023 03:44:47 +0200 Subject: [PATCH 12/39] wip: add triggers Signed-off-by: mario --- grid-proxy/internal/explorer/db/setup.sql | 177 +++++++++++++++++----- 1 file changed, 137 insertions(+), 40 deletions(-) diff --git a/grid-proxy/internal/explorer/db/setup.sql b/grid-proxy/internal/explorer/db/setup.sql index 876a3f94f..8e520ee54 100644 --- a/grid-proxy/internal/explorer/db/setup.sql +++ b/grid-proxy/internal/explorer/db/setup.sql @@ -9,7 +9,7 @@ nodeid, farmid triggers: - - trigger on node table (insert) + - trigger on node table (insert/update) - trigger on total resources (insert/update) - trigger on contract resources (insert/update) - trigger on rent contract (insert/update) @@ -17,52 +17,48 @@ triggers need to be in the same transaction with table creation */ +BEGIN; + ---- -- Helper functions ---- DROP FUNCTION IF EXISTS convert_to_decimal(v_input text); -CREATE OR REPLACE FUNCTION convert_to_decimal(v_input text) RETURNS DECIMAL AS $$ -DECLARE v_dec_value DECIMAL DEFAULT NULL; -BEGIN BEGIN v_dec_value := v_input::DECIMAL; + +CREATE +OR REPLACE FUNCTION convert_to_decimal(v_input text) RETURNS DECIMAL AS $ $ DECLARE v_dec_value DECIMAL DEFAULT NULL; + +BEGIN BEGIN v_dec_value := v_input :: DECIMAL; + EXCEPTION WHEN OTHERS THEN RAISE NOTICE 'Invalid decimal value: "%". Returning NULL.', v_input; + RETURN NULL; + END; + RETURN v_dec_value; + END; -$$ LANGUAGE plpgsql; + +$ $ LANGUAGE plpgsql; + ---- -- Clean old triggers ---- DROP TRIGGER IF EXISTS node_added ON node; + ---- -- Resources cache table ---- DROP TABLE IF EXISTS resources_cache; -CREATE TABLE IF NOT EXISTS resources_cache( - node_id INTEGER PRIMARY KEY, - farm_id INTEGER NOT NULL, - total_hru NUMERIC NOT NULL, - total_mru NUMERIC NOT NULL, - total_sru NUMERIC NOT NULL, - total_cru NUMERIC NOT NULL, - free_hru NUMERIC NOT NULL, - free_mru NUMERIC NOT NULL, - free_sru NUMERIC NOT NULL, - used_hru NUMERIC NOT NULL, - used_mru NUMERIC NOT NULL, - used_sru NUMERIC NOT NULL, - used_cru NUMERIC NOT NULL, - renter INTEGER, - rent_contract_id INTEGER, - node_contracts_count INTEGER NOT NULL, - node_gpu_count INTEGER NOT NULL, - country TEXT, - region TEXT -); -INSERT INTO resources_cache -SELECT node.node_id as node_id, + +DROP VIEW IF EXISTS resources_cache_view; + +CREATE +OR REPLACE VIEW resources_cache_view AS +SELECT + node.node_id as node_id, node.farm_id as farm_id, COALESCE(node_resources_total.hru, 0) as total_hru, COALESCE(node_resources_total.mru, 0) as total_mru, @@ -87,7 +83,8 @@ SELECT node.node_id as node_id, COALESCE(node_gpu.node_gpu_count, 0) as node_gpu_count, country.name as country, country.subregion as region -FROM contract_resources +FROM + contract_resources JOIN node_contract as node_contract ON node_contract.resources_used_id = contract_resources.id AND node_contract.state IN ('Created', 'GracePeriod') RIGHT JOIN node as node ON node.node_id = node_contract.node_id @@ -95,13 +92,17 @@ FROM contract_resources LEFT JOIN rent_contract on node.node_id = rent_contract.node_id AND rent_contract.state IN ('Created', 'GracePeriod') Left JOIN( - SELECT node_twin_id, + SELECT + node_twin_id, COUNT(id) as node_gpu_count - FROM node_gpu - GROUP BY node_twin_id + FROM + node_gpu + GROUP BY + node_twin_id ) AS node_gpu ON node.twin_id = node_gpu.node_twin_id LEFT JOIN country ON node.country = country.name -GROUP BY node.node_id, +GROUP BY + node.node_id, node_resources_total.mru, node_resources_total.sru, node_resources_total.hru, @@ -112,24 +113,60 @@ GROUP BY node.node_id, COALESCE(node_gpu.node_gpu_count, 0), country.name, country.subregion; + +CREATE TABLE IF NOT EXISTS resources_cache( + node_id INTEGER PRIMARY KEY, + farm_id INTEGER NOT NULL, + total_hru NUMERIC NOT NULL, + total_mru NUMERIC NOT NULL, + total_sru NUMERIC NOT NULL, + total_cru NUMERIC NOT NULL, + free_hru NUMERIC NOT NULL, + free_mru NUMERIC NOT NULL, + free_sru NUMERIC NOT NULL, + used_hru NUMERIC NOT NULL, + used_mru NUMERIC NOT NULL, + used_sru NUMERIC NOT NULL, + used_cru NUMERIC NOT NULL, + renter INTEGER, + rent_contract_id INTEGER, + node_contracts_count INTEGER NOT NULL, + node_gpu_count INTEGER NOT NULL, + country TEXT, + region TEXT +); + +INSERT INTO + resources_cache +SELECT + * +FROM + resources_cache_view; + ---- -- PublicIpsCache table ---- DROP TABLE IF EXISTS public_ips_cache; + CREATE TABLE IF NOT EXISTS public_ips_cache( farm_id INTEGER PRIMARY KEY, free_ips INTEGER NOT NULL, total_ips INTEGER NOT NULL, ips jsonb ); -INSERT INTO public_ips_cache -SELECT farm.farm_id, + +INSERT INTO + public_ips_cache +SELECT + farm.farm_id, COALESCE(public_ip_agg.free_ips, 0), COALESCE(public_ip_agg.total_ips, 0), COALESCE(public_ip_agg.ips, '[]') -FROM farm +FROM + farm LEFT JOIN( - SELECT p1.farm_id, + SELECT + p1.farm_id, COUNT(p1.id) total_ips, COUNT( CASE @@ -148,17 +185,77 @@ FROM farm p1.gateway ) ) as ips - FROM public_ip p1 + FROM + public_ip p1 LEFT JOIN public_ip p2 ON p1.id = p2.id - GROUP BY p1.farm_id + GROUP BY + p1.farm_id ) public_ip_agg on public_ip_agg.farm_id = farm.id; + +---- -- Create Indices +---- +CREATE EXTENSION pg_trgm; + +CREATE EXTENSION btree_gin; + CREATE INDEX IF NOT EXISTS idx_node_id ON public.node(node_id); + CREATE INDEX IF NOT EXISTS idx_twin_id ON public.twin(twin_id); + CREATE INDEX IF NOT EXISTS idx_farm_id ON public.farm(farm_id); + CREATE INDEX IF NOT EXISTS idx_contract_id ON public.node_contract(contract_id); + CREATE INDEX IF NOT EXISTS resources_cache_farm_id ON resources_cache (farm_id); + CREATE INDEX IF NOT EXISTS location_id ON location USING gin(id); + CREATE INDEX IF NOT EXISTS resources_cache_node_id ON resources_cache(node_id); + CREATE INDEX IF NOT EXISTS public_ips_cache_farm_id ON public_ips_cache(farm_id); -CREATE INDEX IF NOT EXISTS public_config_node_id ON public_config USING gin(node_id); \ No newline at end of file + +CREATE INDEX IF NOT EXISTS public_config_node_id ON public_config USING gin(node_id); + +---- +--create triggers +---- +CREATE OR REPLACE FUNCTION node_upsert() RETURNS TRIGGER AS $$ + BEGIN + + IF (TG_OP = 'UPDATE') THEN + UPDATE resources_cache SET country = NEW.country + WHERE resources_cache.node_id = NEW.node_id; + ELSIF (TG_OP = 'INSERT') THEN + INSERT INTO + resources_cache SELECT * FROM resources_cache_view; + + END IF; + END; + +$$ LANGUAGE plpgsql; + +CREATE TRIGGER node_trigger +AFTER INSERT OR UPDATE + ON node FOR EACH ROW EXECUTE PROCEDURE node_upsert(); + +CREATE OR REPLACE FUNCTION total_resources_upsert() RETURNS TRIGGER AS $$ + BEGIN + + UPDATE resources_cache + SET total_cru = NEW.total_cru, + total_mru = NEW.total_mru, + total_sru = New.total_sru, + total_hru = NEW.total_hru + WHERE resources_cache.id = NEW.node_id; + + END; + +$$ LANGUAGE plpgsql; + +CREATE TRIGGER total_resources_trigger +AFTER INSERT OR UPDATE + ON total_resources FOR EACH ROW EXECUTE PROCEDURE total_resources_trigger(); + + +COMMIT; \ No newline at end of file From c880fe35f4866babe9b5b74b859ec7d428bc1ed2 Mon Sep 17 00:00:00 2001 From: mario Date: Wed, 20 Dec 2023 12:07:06 +0200 Subject: [PATCH 13/39] update gpu generation to have node twin ids that exist Co-authored-by: omarabdulaziz --- grid-proxy/internal/explorer/db/postgres.go | 2 +- grid-proxy/tools/db/generate.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/grid-proxy/internal/explorer/db/postgres.go b/grid-proxy/internal/explorer/db/postgres.go index f7103825b..ee8493860 100644 --- a/grid-proxy/internal/explorer/db/postgres.go +++ b/grid-proxy/internal/explorer/db/postgres.go @@ -313,7 +313,7 @@ func (d *PostgresDatabase) nodeTableQuery(filter types.NodeFilter, nodeGpuSubque filter.GpuVendorName != nil || filter.GpuVendorID != nil || filter.GpuDeviceID != nil || filter.GpuAvailable != nil { q.Joins( - `INNER JOIN (?) AS gpu ON gpu.node_twin_id = node.twin_id`, nodeGpuSubquery, + `RIGHT JOIN (?) AS gpu ON gpu.node_twin_id = node.twin_id`, nodeGpuSubquery, ) } diff --git a/grid-proxy/tools/db/generate.go b/grid-proxy/tools/db/generate.go index a04b497b3..37b859743 100644 --- a/grid-proxy/tools/db/generate.go +++ b/grid-proxy/tools/db/generate.go @@ -705,7 +705,7 @@ func generateNodeGPUs(db *sql.DB) error { gpuNum := len(vendors) - 1 for j := 0; j <= gpuNum; j++ { g := node_gpu{ - node_twin_id: uint64(i + 100), + node_twin_id: uint64(i + 100 + 2), // node twin ids start from 102 vendor: vendors[j], device: devices[j], contract: i % 2, From 75c07d724be833c34ee8844d5ac995f5e24e4921 Mon Sep 17 00:00:00 2001 From: mario Date: Wed, 20 Dec 2023 17:56:35 +0200 Subject: [PATCH 14/39] add triggers to update cache tables Co-authored-by: omarabulaziz --- grid-proxy/internal/explorer/db/setup.sql | 468 +++++++++++++++++----- 1 file changed, 357 insertions(+), 111 deletions(-) diff --git a/grid-proxy/internal/explorer/db/setup.sql b/grid-proxy/internal/explorer/db/setup.sql index 8e520ee54..0b998a862 100644 --- a/grid-proxy/internal/explorer/db/setup.sql +++ b/grid-proxy/internal/explorer/db/setup.sql @@ -24,24 +24,19 @@ BEGIN; ---- DROP FUNCTION IF EXISTS convert_to_decimal(v_input text); -CREATE -OR REPLACE FUNCTION convert_to_decimal(v_input text) RETURNS DECIMAL AS $ $ DECLARE v_dec_value DECIMAL DEFAULT NULL; +CREATE OR REPLACE FUNCTION CONVERT_TO_DECIMAL(V_INPUT +TEXT) RETURNS DECIMAL AS $$ + DECLARE v_dec_value DECIMAL DEFAULT NULL; + BEGIN BEGIN v_dec_value := v_input:: DECIMAL; + EXCEPTION + WHEN OTHERS THEN RAISE NOTICE 'Invalid decimal value: "%". Returning NULL.', + v_input; + RETURN NULL; + END; + RETURN v_dec_value; + END; + $$ LANGUAGE plpgsql; -BEGIN BEGIN v_dec_value := v_input :: DECIMAL; - -EXCEPTION -WHEN OTHERS THEN RAISE NOTICE 'Invalid decimal value: "%". Returning NULL.', -v_input; - -RETURN NULL; - -END; - -RETURN v_dec_value; - -END; - -$ $ LANGUAGE plpgsql; ---- -- Clean old triggers @@ -55,8 +50,7 @@ DROP TABLE IF EXISTS resources_cache; DROP VIEW IF EXISTS resources_cache_view; -CREATE -OR REPLACE VIEW resources_cache_view AS +CREATE OR REPLACE VIEW resources_cache_view AS SELECT node.node_id as node_id, node.farm_id as farm_id, @@ -64,39 +58,58 @@ SELECT COALESCE(node_resources_total.mru, 0) as total_mru, COALESCE(node_resources_total.sru, 0) as total_sru, COALESCE(node_resources_total.cru, 0) as total_cru, - node_resources_total.hru - COALESCE(sum(contract_resources.hru), 0) as free_hru, - node_resources_total.mru - COALESCE(sum(contract_resources.mru), 0) - GREATEST( - CAST((node_resources_total.mru / 10) AS bigint), + node_resources_total.hru - COALESCE( + sum(contract_resources.hru), + 0 + ) as free_hru, + node_resources_total.mru - COALESCE( + sum(contract_resources.mru), + 0 + ) - GREATEST( + CAST( (node_resources_total.mru / 10) AS bigint + ), 2147483648 ) as free_mru, - node_resources_total.sru - COALESCE(sum(contract_resources.sru), 0) - 21474836480 as free_sru, - COALESCE(sum(contract_resources.hru), 0) as used_hru, - COALESCE(sum(contract_resources.mru), 0) + GREATEST( - CAST((node_resources_total.mru / 10) AS bigint), + node_resources_total.sru - COALESCE( + sum(contract_resources.sru), + 0 + ) - 21474836480 as free_sru, + COALESCE( + sum(contract_resources.hru), + 0 + ) as used_hru, + COALESCE( + sum(contract_resources.mru), + 0 + ) + GREATEST( + CAST( (node_resources_total.mru / 10) AS bigint + ), 2147483648 ) as used_mru, - COALESCE(sum(contract_resources.sru), 0) + 21474836480 as used_sru, - COALESCE(sum(contract_resources.cru), 0) as used_cru, + COALESCE( + sum(contract_resources.sru), + 0 + ) + 21474836480 as used_sru, + COALESCE( + sum(contract_resources.cru), + 0 + ) as used_cru, rent_contract.twin_id as renter, rent_contract.contract_id as rent_contract_id, count(node_contract.contract_id) as node_contract_count, COALESCE(node_gpu.node_gpu_count, 0) as node_gpu_count, country.name as country, country.subregion as region -FROM - contract_resources - JOIN node_contract as node_contract ON node_contract.resources_used_id = contract_resources.id - AND node_contract.state IN ('Created', 'GracePeriod') +FROM contract_resources + JOIN node_contract as node_contract ON node_contract.resources_used_id = contract_resources.id AND node_contract.state IN ('Created', 'GracePeriod') RIGHT JOIN node as node ON node.node_id = node_contract.node_id JOIN node_resources_total AS node_resources_total ON node_resources_total.node_id = node.id - LEFT JOIN rent_contract on node.node_id = rent_contract.node_id - AND rent_contract.state IN ('Created', 'GracePeriod') + LEFT JOIN rent_contract on node.node_id = rent_contract.node_id AND rent_contract.state IN ('Created', 'GracePeriod') Left JOIN( SELECT node_twin_id, COUNT(id) as node_gpu_count - FROM - node_gpu + FROM node_gpu GROUP BY node_twin_id ) AS node_gpu ON node.twin_id = node_gpu.node_twin_id @@ -114,56 +127,51 @@ GROUP BY country.name, country.subregion; -CREATE TABLE IF NOT EXISTS resources_cache( - node_id INTEGER PRIMARY KEY, - farm_id INTEGER NOT NULL, - total_hru NUMERIC NOT NULL, - total_mru NUMERIC NOT NULL, - total_sru NUMERIC NOT NULL, - total_cru NUMERIC NOT NULL, - free_hru NUMERIC NOT NULL, - free_mru NUMERIC NOT NULL, - free_sru NUMERIC NOT NULL, - used_hru NUMERIC NOT NULL, - used_mru NUMERIC NOT NULL, - used_sru NUMERIC NOT NULL, - used_cru NUMERIC NOT NULL, - renter INTEGER, - rent_contract_id INTEGER, - node_contracts_count INTEGER NOT NULL, - node_gpu_count INTEGER NOT NULL, - country TEXT, - region TEXT -); - -INSERT INTO - resources_cache -SELECT - * -FROM - resources_cache_view; +CREATE TABLE + IF NOT EXISTS resources_cache( + node_id INTEGER PRIMARY KEY, + farm_id INTEGER NOT NULL, + total_hru NUMERIC NOT NULL, + total_mru NUMERIC NOT NULL, + total_sru NUMERIC NOT NULL, + total_cru NUMERIC NOT NULL, + free_hru NUMERIC NOT NULL, + free_mru NUMERIC NOT NULL, + free_sru NUMERIC NOT NULL, + used_hru NUMERIC NOT NULL, + used_mru NUMERIC NOT NULL, + used_sru NUMERIC NOT NULL, + used_cru NUMERIC NOT NULL, + renter INTEGER, + rent_contract_id INTEGER, + node_contracts_count INTEGER NOT NULL, + node_gpu_count INTEGER NOT NULL, + country TEXT, + region TEXT + ); + +INSERT INTO resources_cache SELECT * FROM resources_cache_view; ---- -- PublicIpsCache table ---- DROP TABLE IF EXISTS public_ips_cache; -CREATE TABLE IF NOT EXISTS public_ips_cache( - farm_id INTEGER PRIMARY KEY, - free_ips INTEGER NOT NULL, - total_ips INTEGER NOT NULL, - ips jsonb -); +CREATE TABLE + IF NOT EXISTS public_ips_cache( + farm_id INTEGER PRIMARY KEY, + free_ips INTEGER NOT NULL, + total_ips INTEGER NOT NULL, + ips jsonb + ); -INSERT INTO - public_ips_cache +INSERT INTO public_ips_cache SELECT farm.farm_id, COALESCE(public_ip_agg.free_ips, 0), COALESCE(public_ip_agg.total_ips, 0), COALESCE(public_ip_agg.ips, '[]') -FROM - farm +FROM farm LEFT JOIN( SELECT p1.farm_id, @@ -185,19 +193,18 @@ FROM p1.gateway ) ) as ips - FROM - public_ip p1 + FROM public_ip p1 LEFT JOIN public_ip p2 ON p1.id = p2.id GROUP BY p1.farm_id ) public_ip_agg on public_ip_agg.farm_id = farm.id; ---- --- Create Indices +-- Create Indices ---- -CREATE EXTENSION pg_trgm; +CREATE EXTENSION IF NOT EXISTS pg_trgm; -CREATE EXTENSION btree_gin; +CREATE EXTENSION IF NOT EXISTS btree_gin; CREATE INDEX IF NOT EXISTS idx_node_id ON public.node(node_id); @@ -205,57 +212,296 @@ CREATE INDEX IF NOT EXISTS idx_twin_id ON public.twin(twin_id); CREATE INDEX IF NOT EXISTS idx_farm_id ON public.farm(farm_id); -CREATE INDEX IF NOT EXISTS idx_contract_id ON public.node_contract(contract_id); +CREATE INDEX + IF NOT EXISTS idx_contract_id ON public.node_contract(contract_id); -CREATE INDEX IF NOT EXISTS resources_cache_farm_id ON resources_cache (farm_id); +CREATE INDEX + IF NOT EXISTS resources_cache_farm_id ON resources_cache (farm_id); CREATE INDEX IF NOT EXISTS location_id ON location USING gin(id); -CREATE INDEX IF NOT EXISTS resources_cache_node_id ON resources_cache(node_id); +CREATE INDEX + IF NOT EXISTS resources_cache_node_id ON resources_cache(node_id); -CREATE INDEX IF NOT EXISTS public_ips_cache_farm_id ON public_ips_cache(farm_id); +CREATE INDEX + IF NOT EXISTS public_ips_cache_farm_id ON public_ips_cache(farm_id); -CREATE INDEX IF NOT EXISTS public_config_node_id ON public_config USING gin(node_id); +CREATE INDEX + IF NOT EXISTS public_config_node_id ON public_config USING gin(node_id); ---- --create triggers ---- -CREATE OR REPLACE FUNCTION node_upsert() RETURNS TRIGGER AS $$ - BEGIN - IF (TG_OP = 'UPDATE') THEN - UPDATE resources_cache SET country = NEW.country - WHERE resources_cache.node_id = NEW.node_id; - ELSIF (TG_OP = 'INSERT') THEN - INSERT INTO - resources_cache SELECT * FROM resources_cache_view; +/* + node trigger + */ - END IF; - END; +CREATE OR REPLACE FUNCTION node_upsert() RETURNS TRIGGER AS + $$ + BEGIN + IF (TG_OP = 'UPDATE') THEN + UPDATE resources_cache + SET country = NEW.country, + region = ( + Select subregion from country where country.name = NEW.country + ) + WHERE + resources_cache.node_id = NEW.node_id; + + ELSIF (TG_OP = 'INSERT') THEN + INSERT INTO + resources_cache + SELECT * + FROM resources_cache_view WHERE resources_cache_view.node_id = NEW.node_id; + END IF; + END; + $$ LANGUAGE plpgsql; + + +CREATE OR REPLACE TRIGGER node_trigger + AFTER INSERT OR UPDATE OF country + ON node + FOR EACH ROW EXECUTE PROCEDURE node_upsert(); -$$ LANGUAGE plpgsql; -CREATE TRIGGER node_trigger -AFTER INSERT OR UPDATE - ON node FOR EACH ROW EXECUTE PROCEDURE node_upsert(); +/* + total resources trigger + */ +CREATE OR REPLACE FUNCTION node_resources_total_upsert() +RETURNS TRIGGER AS $$ + BEGIN + UPDATE resources_cache + SET + total_cru = NEW.cru, + total_mru = NEW.mru, + total_sru = NEW.sru, + total_hru = NEW.hru, + free_mru = free_mru + (NEW.mru-OLD.mru), + free_hru = free_hru + (NEW.hru-OLD.hru), + free_sru = free_sru + (NEW.sru-OLD.sru) + WHERE + resources_cache.id = NEW.node_id; + END; + $$ LANGUAGE plpgsql; -CREATE OR REPLACE FUNCTION total_resources_upsert() RETURNS TRIGGER AS $$ - BEGIN - UPDATE resources_cache - SET total_cru = NEW.total_cru, - total_mru = NEW.total_mru, - total_sru = New.total_sru, - total_hru = NEW.total_hru - WHERE resources_cache.id = NEW.node_id; +/* + trigger works only after updates because new records in total resources + must also have new node records, which is monitored +*/ +CREATE OR REPLACE TRIGGER node_resources_total_trigger AFTER + UPDATE + ON node_resources_total FOR EACH ROW + EXECUTE + PROCEDURE node_resources_total_upsert(); + - END; +/* + contract resources + */ +CREATE OR REPLACE FUNCTION contract_resources_upsert() +RETURNS TRIGGER AS + $$ + BEGIN + IF (TG_OP = 'UPDATE') THEN + UPDATE resources_cache + SET used_cru = used_cru + (NEW.cru - OLD.cru), + used_mru = used_mru + (NEW.mru - OLD.mru), + used_sru = used_sru + (NEW.sru - OLD.sru), + used_hru = used_hru + (NEW.hru - OLD.hru), + free_mru = free_mru - (NEW.mru - OLD.mru), + free_hru = free_hru - (NEW.hru - OLD.hru), + free_sru = free_sru - (NEW.sru - OLD.sru) + WHERE + resources_cache.node_id = ( + Select node.node_id from node + left join node_contract on node.node_id = node_contract.node_id + left join contract_resources on contract_resources.contract_id = node_contract.id + where contract_resources.contract_id = NEW.contract_id + ); + ELSIF (TG_OP = 'INSERT') THEN + UPDATE resources_cache + SET used_cru = used_cru + NEW.cru, + used_mru = used_mru + NEW.mru, + used_sru = used_sru + NEW.sru, + used_hru = used_hru + NEW.hru, + free_mru = free_mru - NEW.mru, + free_hru = free_hru - NEW.hru, + free_sru = free_sru - NEW.sru + WHERE + resources_cache.node_id = ( + Select node.node_id from node + left join node_contract on node.node_id = node_contract.node_id + left join contract_resources on contract_resources.contract_id = node_contract.id + where contract_resources.contract_id = NEW.contract_id + ); + END IF; + END; + $$ LANGUAGE plpgsql; + + +CREATE OR REPLACE TRIGGER contract_resources_trigger AFTER + INSERT OR UPDATE ON contract_resources + FOR EACH ROW EXECUTE PROCEDURE contract_resources_upsert(); -$$ LANGUAGE plpgsql; +/* + node_contract_trigger +*/ +CREATE OR REPLACE FUNCTION node_contract_upsert() RETURNS TRIGGER AS + $$ + BEGIN + IF (TG_OP = 'UPDATE' AND NEW.state = 'Deleted') THEN + UPDATE resources_cache + SET (used_cru, used_mru, used_sru, used_hru, free_mru, free_sru, free_hru, node_contracts_count) = + ( + select resources_cache.used_cru - cru, + resources_cache.used_mru - mru, + resources_cache.used_sru - sru, + resources_cache.used_hru - hru, + resources_cache.free_mru + mru, + resources_cache.free_sru + sru, + resources_cache.free_hru + hru, + resources_cache.node_contract_count - 1 + from resources_cache + left join node_contract on resources_cache.node_id = node_contract.node_id + left join contract_resources on node_contract.id = contract_resources.contract_id + where resources_cache.node_id = NEW.node_id and node_contract.contract_id = NEW.contract_id + ) where resources_cache.node_id = NEW.node_id; + + ELSIF (TG_OP = 'INSERT') THEN + UPDATE resources_cache + SET node_contracts_count = node_contracts_count + 1 + WHERE resources_cache.node_id = NEW.node_id; + END IF; + END; + $$ LANGUAGE plpgsql; + + +CREATE OR REPLACE TRIGGER node_contract_trigger + AFTER INSERT OR UPDATE OF state + ON node_contract + FOR EACH ROW EXECUTE PROCEDURE node_contract_upsert(); + + +CREATE OR REPLACE FUNCTION rent_contract_upsert() RETURNS TRIGGER AS + $$ + BEGIN + IF (TG_OP = 'UPDATE' AND NEW.state = 'Deleted') THEN + UPDATE resources_cache + SET renter = NULL, + rent_contract_id = NULL + WHERE + resources_cache.node_id = NEW.node_id; + + ELSIF (TG_OP = 'INSERT') THEN + UPDATE resources_cache + SET renter = NEW.twin_id, + rent_contract_id = NEW.contract_id + WHERE + resources_cache.node_id = NEW.node_id; + + END IF; + END; + $$ LANGUAGE plpgsql; + + +CREATE OR REPLACE TRIGGER rent_contract_trigger + AFTER INSERT OR UPDATE OF state + ON rent_contract + FOR EACH ROW EXECUTE PROCEDURE rent_contract_upsert(); -CREATE TRIGGER total_resources_trigger -AFTER INSERT OR UPDATE - ON total_resources FOR EACH ROW EXECUTE PROCEDURE total_resources_trigger(); +/* + public ips trigger +-- */ +CREATE OR REPLACE FUNCTION public_ip_upsert() RETURNS TRIGGER AS + $$ + BEGIN + IF (TG_OP = 'UPDATE') THEN + UPDATE public_ips_cache + SET free_ips = free_ips + (CASE WHEN NEW.contract_id = 0 THEN 1 ELSE -1 END), + ips = ( + select jsonb_agg( + jsonb_build_object( + 'id', + public_ip.id, + 'ip', + public_ip.ip, + 'contract_id', + public_ip.contract_id, + 'gateway', + public_ip.gateway + ) + ) + from public_ip where farm_id = NEW.farm_id + ) + WHERE + public_ips_cache.farm_id = ( + SELECT farm.farm_id from public_ip + LEFT JOIN farm ON farm.id = public_ip.farm_id + WHERE public_ip.id = NEW.id + ); + + ELSIF (TG_OP = 'INSERT') THEN + UPDATE public_ips_cache + SET free_ips = free_ips + 1, + total_ips = total_ips + 1, + ips = ( + select jsonb_agg( + jsonb_build_object( + 'id', + public_ip.id, + 'ip', + public_ip.ip, + 'contract_id', + public_ip.contract_id, + 'gateway', + public_ip.gateway + ) + ) + from public_ip where farm_id = NEW.farm_id + ) + WHERE + public_ips_cache.farm_id = ( + SELECT farm.farm_id from public_ip + LEFT JOIN farm ON farm.id = public_ip.farm_id + WHERE public_ip.id = NEW.id + ); + + ELSIF (TG_OP = 'DELETE') THEN + UPDATE public_ips_cache + SET free_ips = free_ips - (CASE WHEN OLD.contract_id = 0 THEN 1 ELSE 0 END), + total_ips = total_ips - 1, + ips = ( + select jsonb_agg( + jsonb_build_object( + 'id', + public_ip.id, + 'ip', + public_ip.ip, + 'contract_id', + public_ip.contract_id, + 'gateway', + public_ip.gateway + ) + ) + from public_ip where farm_id = OLD.farm_id + ) + WHERE + public_ips_cache.farm_id = ( + SELECT farm.farm_id from public_ip + LEFT JOIN farm ON farm.id = public_ip.farm_id + WHERE public_ip.id = OLD.id + ); + END IF; + END; + $$ LANGUAGE plpgsql; + + +CREATE OR REPLACE TRIGGER public_ip_trigger + AFTER INSERT OR DELETE OR UPDATE OF contract_id + ON public_ip + FOR EACH ROW EXECUTE PROCEDURE public_ip_upsert(); COMMIT; \ No newline at end of file From 943ad263344935f1c1f36fbd0da93cc565ecbfb6 Mon Sep 17 00:00:00 2001 From: mario Date: Wed, 20 Dec 2023 17:58:07 +0200 Subject: [PATCH 15/39] seperate db initialization from construction Co-authored-by: omarabdulasis --- grid-proxy/cmds/proxy_server/main.go | 8 ++++++-- grid-proxy/internal/explorer/db/postgres.go | 17 +++++++---------- grid-proxy/tests/queries/conn_test.go | 7 +------ grid-proxy/tests/queries/main_test.go | 9 ++++++++- 4 files changed, 22 insertions(+), 19 deletions(-) diff --git a/grid-proxy/cmds/proxy_server/main.go b/grid-proxy/cmds/proxy_server/main.go index ce26cc923..ce595d20d 100644 --- a/grid-proxy/cmds/proxy_server/main.go +++ b/grid-proxy/cmds/proxy_server/main.go @@ -121,13 +121,17 @@ func main() { log.Fatal().Err(err).Msg("couldn't get postgres client") } - dbClient := explorer.DBClient{DB: db} + if err := db.Initialize(); err != nil { + log.Fatal().Err(err).Msg("failed to initialize database") + } + + dbClient := explorer.DBClient{DB: &db} indexer, err := gpuindexer.NewNodeGPUIndexer( ctx, f.relayURL, f.mnemonics, - sub, db, + sub, &db, f.indexerCheckIntervalMins, f.indexerBatchSize, f.indexerResultWorkers, diff --git a/grid-proxy/internal/explorer/db/postgres.go b/grid-proxy/internal/explorer/db/postgres.go index ee8493860..6e405e613 100644 --- a/grid-proxy/internal/explorer/db/postgres.go +++ b/grid-proxy/internal/explorer/db/postgres.go @@ -47,7 +47,7 @@ func (d *PostgresDatabase) GetConnectionString() string { } // NewPostgresDatabase returns a new postgres db client -func NewPostgresDatabase(host string, port int, user, password, dbname string, maxConns int, logLevel logger.LogLevel) (Database, error) { +func NewPostgresDatabase(host string, port int, user, password, dbname string, maxConns int, logLevel logger.LogLevel) (PostgresDatabase, error) { connString := fmt.Sprintf("host=%s port=%d user=%s "+ "password=%s dbname=%s sslmode=disable", host, port, user, password, dbname) @@ -55,11 +55,11 @@ func NewPostgresDatabase(host string, port int, user, password, dbname string, m Logger: logger.Default.LogMode(logLevel), }) if err != nil { - return nil, errors.Wrap(err, "failed to create orm wrapper around db") + return PostgresDatabase{}, errors.Wrap(err, "failed to create orm wrapper around db") } sql, err := gormDB.DB() if err != nil { - return nil, errors.Wrap(err, "failed to configure DB connection") + return PostgresDatabase{}, errors.Wrap(err, "failed to configure DB connection") } sql.SetMaxIdleConns(3) @@ -67,14 +67,11 @@ func NewPostgresDatabase(host string, port int, user, password, dbname string, m err = gormDB.AutoMigrate(&NodeGPU{}) if err != nil { - return nil, errors.Wrap(err, "failed to auto migrate DB") + return PostgresDatabase{}, errors.Wrap(err, "failed to auto migrate DB") } res := PostgresDatabase{gormDB, connString} - if err := res.initialize(); err != nil { - return nil, errors.Wrap(err, "failed to setup tables") - } - return &res, nil + return res, nil } // Close the db connection @@ -86,7 +83,7 @@ func (d *PostgresDatabase) Close() error { return db.Close() } -func (d *PostgresDatabase) initialize() error { +func (d *PostgresDatabase) Initialize() error { return d.gormDB.Exec(setupFile).Error } @@ -664,7 +661,7 @@ func (d *PostgresDatabase) GetNodes(ctx context.Context, filter types.NodeFilter func (d *PostgresDatabase) shouldRetry(resError error) bool { if resError != nil && resError.Error() == ErrResourcesCacheTableNotFound.Error() { - if err := d.initialize(); err != nil { + if err := d.Initialize(); err != nil { log.Logger.Err(err).Msg("failed to reinitialize database") } else { return true diff --git a/grid-proxy/tests/queries/conn_test.go b/grid-proxy/tests/queries/conn_test.go index 516fc7984..2197fa6e2 100644 --- a/grid-proxy/tests/queries/conn_test.go +++ b/grid-proxy/tests/queries/conn_test.go @@ -9,15 +9,10 @@ import ( "github.com/rs/zerolog/log" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - db "github.com/threefoldtech/tfgrid-sdk-go/grid-proxy/internal/explorer/db" "github.com/threefoldtech/tfgrid-sdk-go/grid-proxy/pkg/types" - "gorm.io/gorm/logger" ) func TestDBManyOpenConnections(t *testing.T) { - p, err := db.NewPostgresDatabase(POSTGRES_HOST, POSTGRES_PORT, POSTGRES_USER, POSTGRES_PASSSWORD, POSTGRES_DB, 80, logger.Error) - require.NoError(t, err) gotQueriesCnt := atomic.Int32{} wg := sync.WaitGroup{} @@ -30,7 +25,7 @@ func TestDBManyOpenConnections(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), time.Second*30) defer cancel() - _, _, err := p.GetTwins(ctx, types.TwinFilter{}, types.Limit{Size: 100}) + _, _, err := DBClient.GetTwins(ctx, types.TwinFilter{}, types.Limit{Size: 100}) if err != nil { log.Err(err).Msg("twin query failed") return diff --git a/grid-proxy/tests/queries/main_test.go b/grid-proxy/tests/queries/main_test.go index f0146bdd5..89711d57e 100644 --- a/grid-proxy/tests/queries/main_test.go +++ b/grid-proxy/tests/queries/main_test.go @@ -66,14 +66,21 @@ func TestMain(m *testing.M) { if err != nil { panic(err) } - DBClient, err = proxyDB.NewPostgresDatabase(POSTGRES_HOST, POSTGRES_PORT, POSTGRES_USER, POSTGRES_PASSSWORD, POSTGRES_DB, 80, logger.Error) + dbClient, err := proxyDB.NewPostgresDatabase(POSTGRES_HOST, POSTGRES_PORT, POSTGRES_USER, POSTGRES_PASSSWORD, POSTGRES_DB, 80, logger.Error) if err != nil { panic(err) } + DBClient = &dbClient mockClient = mock.NewGridProxyMockClient(data) gridProxyClient = proxyclient.NewClient(ENDPOINT) exitcode := m.Run() + if exitcode == 0 { + exitcode = m.Run() + } + // trigger + // load + // m.Run() os.Exit(exitcode) } From 5fe3f52efe52c8b59e066f824b07efba9aac5b8d Mon Sep 17 00:00:00 2001 From: Omar Abdulaziz Date: Wed, 20 Dec 2023 23:33:38 +0200 Subject: [PATCH 16/39] fix the triggers in setup.sql Co-authored-by: Mario Wassef --- grid-proxy/internal/explorer/db/setup.sql | 523 +++++++++++----------- 1 file changed, 252 insertions(+), 271 deletions(-) diff --git a/grid-proxy/internal/explorer/db/setup.sql b/grid-proxy/internal/explorer/db/setup.sql index 0b998a862..fd579e9c1 100644 --- a/grid-proxy/internal/explorer/db/setup.sql +++ b/grid-proxy/internal/explorer/db/setup.sql @@ -58,22 +58,9 @@ SELECT COALESCE(node_resources_total.mru, 0) as total_mru, COALESCE(node_resources_total.sru, 0) as total_sru, COALESCE(node_resources_total.cru, 0) as total_cru, - node_resources_total.hru - COALESCE( - sum(contract_resources.hru), - 0 - ) as free_hru, - node_resources_total.mru - COALESCE( - sum(contract_resources.mru), - 0 - ) - GREATEST( - CAST( (node_resources_total.mru / 10) AS bigint - ), - 2147483648 - ) as free_mru, - node_resources_total.sru - COALESCE( - sum(contract_resources.sru), - 0 - ) - 21474836480 as free_sru, + COALESCE(node_resources_total.hru, 0) - COALESCE(sum(contract_resources.hru), 0) as free_hru, + COALESCE(node_resources_total.mru, 0) - COALESCE(sum(contract_resources.mru), 0) - GREATEST(CAST((node_resources_total.mru / 10) AS bigint), 2147483648) as free_mru, + COALESCE(node_resources_total.sru, 0) - COALESCE(sum(contract_resources.sru), 0) - 21474836480 as free_sru, COALESCE( sum(contract_resources.hru), 0 @@ -100,12 +87,12 @@ SELECT COALESCE(node_gpu.node_gpu_count, 0) as node_gpu_count, country.name as country, country.subregion as region -FROM contract_resources - JOIN node_contract as node_contract ON node_contract.resources_used_id = contract_resources.id AND node_contract.state IN ('Created', 'GracePeriod') - RIGHT JOIN node as node ON node.node_id = node_contract.node_id - JOIN node_resources_total AS node_resources_total ON node_resources_total.node_id = node.id +FROM node + LEFT JOIN node_contract ON node.node_id = node_contract.node_id + LEFT JOIN contract_resources ON node_contract.resources_used_id = contract_resources.id AND node_contract.state IN ('Created', 'GracePeriod') + LEFT JOIN node_resources_total AS node_resources_total ON node_resources_total.node_id = node.id LEFT JOIN rent_contract on node.node_id = rent_contract.node_id AND rent_contract.state IN ('Created', 'GracePeriod') - Left JOIN( + LEFT JOIN( SELECT node_twin_id, COUNT(id) as node_gpu_count @@ -127,30 +114,31 @@ GROUP BY country.name, country.subregion; -CREATE TABLE - IF NOT EXISTS resources_cache( - node_id INTEGER PRIMARY KEY, - farm_id INTEGER NOT NULL, - total_hru NUMERIC NOT NULL, - total_mru NUMERIC NOT NULL, - total_sru NUMERIC NOT NULL, - total_cru NUMERIC NOT NULL, - free_hru NUMERIC NOT NULL, - free_mru NUMERIC NOT NULL, - free_sru NUMERIC NOT NULL, - used_hru NUMERIC NOT NULL, - used_mru NUMERIC NOT NULL, - used_sru NUMERIC NOT NULL, - used_cru NUMERIC NOT NULL, - renter INTEGER, - rent_contract_id INTEGER, - node_contracts_count INTEGER NOT NULL, - node_gpu_count INTEGER NOT NULL, - country TEXT, - region TEXT +CREATE TABLE IF NOT EXISTS resources_cache( + node_id INTEGER PRIMARY KEY, + farm_id INTEGER NOT NULL, + total_hru NUMERIC NOT NULL, + total_mru NUMERIC NOT NULL, + total_sru NUMERIC NOT NULL, + total_cru NUMERIC NOT NULL, + free_hru NUMERIC NOT NULL, + free_mru NUMERIC NOT NULL, + free_sru NUMERIC NOT NULL, + used_hru NUMERIC NOT NULL, + used_mru NUMERIC NOT NULL, + used_sru NUMERIC NOT NULL, + used_cru NUMERIC NOT NULL, + renter INTEGER, + rent_contract_id INTEGER, + node_contracts_count INTEGER NOT NULL, + node_gpu_count INTEGER NOT NULL, + country TEXT, + region TEXT ); -INSERT INTO resources_cache SELECT * FROM resources_cache_view; +INSERT INTO resources_cache +SELECT * +FROM resources_cache_view; ---- -- PublicIpsCache table @@ -233,275 +221,268 @@ CREATE INDEX --create triggers ---- -/* - node trigger - */ +/* + Node Trigger: + - Insert node record > Insert new resources_cache record + - Update node country > update resources_cache country/region +*/ CREATE OR REPLACE FUNCTION node_upsert() RETURNS TRIGGER AS - $$ +$$ +BEGIN + IF (TG_OP = 'UPDATE') THEN BEGIN - IF (TG_OP = 'UPDATE') THEN - UPDATE resources_cache - SET country = NEW.country, - region = ( - Select subregion from country where country.name = NEW.country - ) - WHERE - resources_cache.node_id = NEW.node_id; - - ELSIF (TG_OP = 'INSERT') THEN - INSERT INTO - resources_cache - SELECT * - FROM resources_cache_view WHERE resources_cache_view.node_id = NEW.node_id; - END IF; + UPDATE resources_cache + SET + country = NEW.country, + region = ( + SELECT subregion FROM country WHERE country.name = NEW.country + ) + WHERE + resources_cache.node_id = NEW.node_id; + EXCEPTION + WHEN OTHERS THEN + RAISE NOTICE 'Error updating resources_cache: %', SQLERRM; END; - $$ LANGUAGE plpgsql; + ELSIF (TG_OP = 'INSERT') THEN + BEGIN + INSERT INTO resources_cache + SELECT * + FROM resources_cache_view + WHERE resources_cache_view.node_id = 99995; + EXCEPTION + WHEN OTHERS THEN + RAISE NOTICE 'Error inserting resources_cache: %', SQLERRM; + END; + END IF; + + RETURN NULL; +END; +$$ LANGUAGE plpgsql; CREATE OR REPLACE TRIGGER node_trigger AFTER INSERT OR UPDATE OF country - ON node + ON node FOR EACH ROW EXECUTE PROCEDURE node_upsert(); - /* - total resources trigger + Total resources trigger + - Insert/Update node_resources_total > Update equivalent resources_cache record. */ -CREATE OR REPLACE FUNCTION node_resources_total_upsert() -RETURNS TRIGGER AS $$ - BEGIN +CREATE OR REPLACE FUNCTION node_resources_total_upsert() RETURNS TRIGGER AS +$$ +BEGIN + BEGIN UPDATE resources_cache SET total_cru = NEW.cru, total_mru = NEW.mru, total_sru = NEW.sru, total_hru = NEW.hru, - free_mru = free_mru + (NEW.mru-OLD.mru), - free_hru = free_hru + (NEW.hru-OLD.hru), - free_sru = free_sru + (NEW.sru-OLD.sru) + free_mru = free_mru + (NEW.mru-COALESCE(OLD.mru, 0)), + free_hru = free_hru + (NEW.hru-COALESCE(OLD.hru, 0)), + free_sru = free_sru + (NEW.sru-COALESCE(OLD.sru, 0)) WHERE - resources_cache.id = NEW.node_id; - END; - $$ LANGUAGE plpgsql; + resources_cache.node_id = ( + SELECT node.node_id FROM node WHERE node.id = New.node_id + ); + EXCEPTION + WHEN OTHERS THEN + RAISE NOTICE 'Error reflecting total_resources changes %', SQLERRM; + END; +RETURN NULL; +END; +$$ LANGUAGE plpgsql; + +CREATE OR REPLACE TRIGGER node_resources_total_trigger + AFTER INSERT OR UPDATE + ON node_resources_total FOR EACH ROW + EXECUTE PROCEDURE node_resources_total_upsert(); /* - trigger works only after updates because new records in total resources - must also have new node records, which is monitored -*/ -CREATE OR REPLACE TRIGGER node_resources_total_trigger AFTER - UPDATE - ON node_resources_total FOR EACH ROW - EXECUTE - PROCEDURE node_resources_total_upsert(); - - -/* - contract resources + Contract resources + - Insert/Update contract_resources report > update resources_cache used/free fields */ -CREATE OR REPLACE FUNCTION contract_resources_upsert() -RETURNS TRIGGER AS - $$ - BEGIN - IF (TG_OP = 'UPDATE') THEN - UPDATE resources_cache - SET used_cru = used_cru + (NEW.cru - OLD.cru), - used_mru = used_mru + (NEW.mru - OLD.mru), - used_sru = used_sru + (NEW.sru - OLD.sru), - used_hru = used_hru + (NEW.hru - OLD.hru), - free_mru = free_mru - (NEW.mru - OLD.mru), - free_hru = free_hru - (NEW.hru - OLD.hru), - free_sru = free_sru - (NEW.sru - OLD.sru) - WHERE - resources_cache.node_id = ( - Select node.node_id from node - left join node_contract on node.node_id = node_contract.node_id - left join contract_resources on contract_resources.contract_id = node_contract.id - where contract_resources.contract_id = NEW.contract_id - ); - ELSIF (TG_OP = 'INSERT') THEN - UPDATE resources_cache - SET used_cru = used_cru + NEW.cru, - used_mru = used_mru + NEW.mru, - used_sru = used_sru + NEW.sru, - used_hru = used_hru + NEW.hru, - free_mru = free_mru - NEW.mru, - free_hru = free_hru - NEW.hru, - free_sru = free_sru - NEW.sru - WHERE - resources_cache.node_id = ( - Select node.node_id from node - left join node_contract on node.node_id = node_contract.node_id - left join contract_resources on contract_resources.contract_id = node_contract.id - where contract_resources.contract_id = NEW.contract_id - ); - END IF; - END; - $$ LANGUAGE plpgsql; - -CREATE OR REPLACE TRIGGER contract_resources_trigger AFTER - INSERT OR UPDATE ON contract_resources - FOR EACH ROW EXECUTE PROCEDURE contract_resources_upsert(); +CREATE OR REPLACE FUNCTION contract_resources_upsert() RETURNS TRIGGER AS +$$ +BEGIN + BEGIN + UPDATE resources_cache + SET used_cru = used_cru + (NEW.cru - COALESCE(OLD.cru, 0)), + used_mru = used_mru + (NEW.mru - COALESCE(OLD.mru, 0)), + used_sru = used_sru + (NEW.sru - COALESCE(OLD.sru, 0)), + used_hru = used_hru + (NEW.hru - COALESCE(OLD.hru, 0)), + free_mru = free_mru - (NEW.mru - COALESCE(OLD.mru, 0)), + free_hru = free_hru - (NEW.hru - COALESCE(OLD.hru, 0)), + free_sru = free_sru - (NEW.sru - COALESCE(OLD.sru, 0)) + WHERE + resources_cache.node_id = ( + SELECT node_id FROM node_contract WHERE node_contract.id = NEW.contract_id + ); + EXCEPTION + WHEN OTHERS THEN + RAISE NOTICE 'Error reflecting contract_resources changes %', SQLERRM; + END; +RETURN NULL; +END; +$$ LANGUAGE plpgsql; + +CREATE OR REPLACE TRIGGER contract_resources_trigger + AFTER INSERT OR UPDATE ON contract_resources FOR EACH ROW + EXECUTE PROCEDURE contract_resources_upsert(); /* - node_contract_trigger + Node contract trigger + - Insert new contract > increment resources_cache node_contract_count + - Update contract state to 'Deleted' > decrement used an increment free fields on resources_cache */ CREATE OR REPLACE FUNCTION node_contract_upsert() RETURNS TRIGGER AS - $$ +$$ +BEGIN + IF (TG_OP = 'UPDATE' AND NEW.state = 'Deleted') THEN BEGIN - IF (TG_OP = 'UPDATE' AND NEW.state = 'Deleted') THEN - UPDATE resources_cache - SET (used_cru, used_mru, used_sru, used_hru, free_mru, free_sru, free_hru, node_contracts_count) = - ( - select resources_cache.used_cru - cru, - resources_cache.used_mru - mru, - resources_cache.used_sru - sru, - resources_cache.used_hru - hru, - resources_cache.free_mru + mru, - resources_cache.free_sru + sru, - resources_cache.free_hru + hru, - resources_cache.node_contract_count - 1 - from resources_cache - left join node_contract on resources_cache.node_id = node_contract.node_id - left join contract_resources on node_contract.id = contract_resources.contract_id - where resources_cache.node_id = NEW.node_id and node_contract.contract_id = NEW.contract_id - ) where resources_cache.node_id = NEW.node_id; - - ELSIF (TG_OP = 'INSERT') THEN - UPDATE resources_cache - SET node_contracts_count = node_contracts_count + 1 - WHERE resources_cache.node_id = NEW.node_id; - END IF; - END; - $$ LANGUAGE plpgsql; - + UPDATE resources_cache + SET (used_cru, used_mru, used_sru, used_hru, free_mru, free_sru, free_hru, node_contracts_count) = + ( + SELECT + resources_cache.used_cru - cru, + resources_cache.used_mru - mru, + resources_cache.used_sru - sru, + resources_cache.used_hru - hru, + resources_cache.free_mru + mru, + resources_cache.free_sru + sru, + resources_cache.free_hru + hru, + resources_cache.node_contracts_count - 1 + FROM resources_cache + LEFT JOIN contract_resources ON contract_resources.contract_id = NEW.id + WHERE resources_cache.node_id = NEW.node_id + ) WHERE resources_cache.node_id = NEW.node_id; + EXCEPTION + WHEN OTHERS THEN + RAISE NOTICE 'Error reflecting node_contract updates %', SQLERRM; + END; + + ELSIF (TG_OP = 'INSERT') THEN + BEGIN + UPDATE resources_cache + SET node_contracts_count = node_contracts_count + 1 + WHERE resources_cache.node_id = NEW.node_id; + EXCEPTION + WHEN OTHERS THEN + RAISE NOTICE 'Error incrementing node_contract_count %', SQLERRM; + END; + END IF; +RETURN NULL; +END; +$$ LANGUAGE plpgsql; CREATE OR REPLACE TRIGGER node_contract_trigger - AFTER INSERT OR UPDATE OF state - ON node_contract - FOR EACH ROW EXECUTE PROCEDURE node_contract_upsert(); + AFTER INSERT OR UPDATE OF state ON node_contract FOR EACH ROW + EXECUTE PROCEDURE node_contract_upsert(); +/* + Rent contract trigger + - Insert new rent contract > Update resources_cache renter/rent_contract_id + - Update (state to 'Deleted') > nullify resources_cache renter/rent_contract_id +*/ CREATE OR REPLACE FUNCTION rent_contract_upsert() RETURNS TRIGGER AS - $$ +$$ +BEGIN + IF (TG_OP = 'UPDATE' AND NEW.state = 'Deleted') THEN BEGIN - IF (TG_OP = 'UPDATE' AND NEW.state = 'Deleted') THEN - UPDATE resources_cache - SET renter = NULL, - rent_contract_id = NULL - WHERE - resources_cache.node_id = NEW.node_id; - - ELSIF (TG_OP = 'INSERT') THEN - UPDATE resources_cache - SET renter = NEW.twin_id, - rent_contract_id = NEW.contract_id - WHERE - resources_cache.node_id = NEW.node_id; - - END IF; - END; - $$ LANGUAGE plpgsql; - + UPDATE resources_cache + SET renter = NULL, + rent_contract_id = NULL + WHERE + resources_cache.node_id = NEW.node_id; + EXCEPTION + WHEN OTHERS THEN + RAISE NOTICE 'Error removing resources_cache rent fields %', SQLERRM; + END; + ELSIF (TG_OP = 'INSERT') THEN + BEGIN + UPDATE resources_cache + SET renter = NEW.twin_id, + rent_contract_id = NEW.contract_id + WHERE + resources_cache.node_id = NEW.node_id; + EXCEPTION + WHEN OTHERS THEN + RAISE NOTICE 'Error reflecting rent_contract changes %', SQLERRM; + END; + END IF; +RETURN NULL; +END; +$$ LANGUAGE plpgsql; CREATE OR REPLACE TRIGGER rent_contract_trigger - AFTER INSERT OR UPDATE OF state - ON rent_contract - FOR EACH ROW EXECUTE PROCEDURE rent_contract_upsert(); + AFTER INSERT OR UPDATE OF state ON rent_contract FOR EACH ROW + EXECUTE PROCEDURE rent_contract_upsert(); /* - public ips trigger --- */ + Public ips trigger + - Insert new ip > increment free/total ips + re-aggregate ips object + - Deleted > decrement total, decrement free ips (if it was used) + re-aggregate ips object + - Update > increment/decrement free ips based on usage + re-aggregate ips object +*/ CREATE OR REPLACE FUNCTION public_ip_upsert() RETURNS TRIGGER AS - $$ - BEGIN - IF (TG_OP = 'UPDATE') THEN - UPDATE public_ips_cache - SET free_ips = free_ips + (CASE WHEN NEW.contract_id = 0 THEN 1 ELSE -1 END), - ips = ( - select jsonb_agg( - jsonb_build_object( - 'id', - public_ip.id, - 'ip', - public_ip.ip, - 'contract_id', - public_ip.contract_id, - 'gateway', - public_ip.gateway - ) - ) - from public_ip where farm_id = NEW.farm_id - ) - WHERE - public_ips_cache.farm_id = ( - SELECT farm.farm_id from public_ip - LEFT JOIN farm ON farm.id = public_ip.farm_id - WHERE public_ip.id = NEW.id - ); - - ELSIF (TG_OP = 'INSERT') THEN - UPDATE public_ips_cache - SET free_ips = free_ips + 1, - total_ips = total_ips + 1, - ips = ( - select jsonb_agg( - jsonb_build_object( - 'id', - public_ip.id, - 'ip', - public_ip.ip, - 'contract_id', - public_ip.contract_id, - 'gateway', - public_ip.gateway - ) - ) - from public_ip where farm_id = NEW.farm_id - ) - WHERE - public_ips_cache.farm_id = ( - SELECT farm.farm_id from public_ip - LEFT JOIN farm ON farm.id = public_ip.farm_id - WHERE public_ip.id = NEW.id - ); - - ELSIF (TG_OP = 'DELETE') THEN - UPDATE public_ips_cache - SET free_ips = free_ips - (CASE WHEN OLD.contract_id = 0 THEN 1 ELSE 0 END), - total_ips = total_ips - 1, - ips = ( - select jsonb_agg( - jsonb_build_object( - 'id', - public_ip.id, - 'ip', - public_ip.ip, - 'contract_id', - public_ip.contract_id, - 'gateway', - public_ip.gateway - ) - ) - from public_ip where farm_id = OLD.farm_id - ) - WHERE - public_ips_cache.farm_id = ( - SELECT farm.farm_id from public_ip - LEFT JOIN farm ON farm.id = public_ip.farm_id - WHERE public_ip.id = OLD.id - ); - END IF; - END; - $$ LANGUAGE plpgsql; - +$$ +BEGIN + BEGIN + UPDATE public_ips_cache + SET free_ips = free_ips + ( + CASE + -- handles insertion/update by freeing ip + WHEN TG_OP != 'DELETE' AND NEW.contract_id = 0 + THEN 1 + -- handles deletion/update by reserving ip + WHEN COALESCE(OLD.contract_id, 0) = 0 + THEN -1 + -- handles delete reserved ips + ELSE 0 + END ), + total_ips = total_ips + ( + CASE + WHEN TG_OP = 'INSERT' + THEN 1 + WHEn TG_OP = 'DELETE' + THEN -1 + ELSE 0 + END ), + ips = ( + select jsonb_agg( + jsonb_build_object( + 'id', + public_ip.id, + 'ip', + public_ip.ip, + 'contract_id', + public_ip.contract_id, + 'gateway', + public_ip.gateway + ) + ) + -- old/new farm_id are the same + from public_ip where farm_id = COALESCE(NEW.farm_id, OLD.farm_id) + ) + WHERE + public_ips_cache.farm_id = ( + SELECT farm_id FROM farm WHERE farm.id = COALESCE(NEW.farm_id, OLD.farm_id) + ); + EXCEPTION + WHEN OTHERS THEN + RAISE NOTICE 'Error reflect public_ips changes %s', SQLERRM; + END; +RETURN NULL; +END; +$$ LANGUAGE plpgsql; CREATE OR REPLACE TRIGGER public_ip_trigger - AFTER INSERT OR DELETE OR UPDATE OF contract_id - ON public_ip - FOR EACH ROW EXECUTE PROCEDURE public_ip_upsert(); - + AFTER INSERT OR DELETE OR UPDATE OF contract_id ON public_ip FOR EACH ROW + EXECUTE PROCEDURE public_ip_upsert(); COMMIT; \ No newline at end of file From ba8bed7189678d6fd52bd17e6eaa99beabc30042 Mon Sep 17 00:00:00 2001 From: Omar Abdulaziz Date: Thu, 21 Dec 2023 00:05:48 +0200 Subject: [PATCH 17/39] wip: add some modifications on db to hit the triggers. Co-authored-by: Mario Wassef --- grid-proxy/tests/queries/main_test.go | 25 ++- grid-proxy/tests/queries/modifiers.sql | 236 +++++++++++++++++++++++++ 2 files changed, 256 insertions(+), 5 deletions(-) create mode 100644 grid-proxy/tests/queries/modifiers.sql diff --git a/grid-proxy/tests/queries/main_test.go b/grid-proxy/tests/queries/main_test.go index 89711d57e..dc0b9d6b2 100644 --- a/grid-proxy/tests/queries/main_test.go +++ b/grid-proxy/tests/queries/main_test.go @@ -17,6 +17,8 @@ import ( proxyclient "github.com/threefoldtech/tfgrid-sdk-go/grid-proxy/pkg/client" mock "github.com/threefoldtech/tfgrid-sdk-go/grid-proxy/tests/queries/mock_client" "gorm.io/gorm/logger" + + _ "embed" ) var ( @@ -36,6 +38,9 @@ var ( DBClient db.Database ) +//go:embed modifiers.sql +var modifiersFile string + func parseCmdline() { flag.StringVar(&POSTGRES_HOST, "postgres-host", "", "postgres host") flag.IntVar(&POSTGRES_PORT, "postgres-port", 5432, "postgres port") @@ -76,11 +81,21 @@ func TestMain(m *testing.M) { gridProxyClient = proxyclient.NewClient(ENDPOINT) exitcode := m.Run() - if exitcode == 0 { - exitcode = m.Run() + if exitcode != 0 { + os.Exit(exitcode) } - // trigger - // load - // m.Run() + + _, err = db.Exec(modifiersFile) + if err != nil { + panic(err) + } + + data, err = mock.Load(db) + if err != nil { + panic(err) + } + mockClient = mock.NewGridProxyMockClient(data) + + exitcode = m.Run() os.Exit(exitcode) } diff --git a/grid-proxy/tests/queries/modifiers.sql b/grid-proxy/tests/queries/modifiers.sql new file mode 100644 index 000000000..0f09eee5c --- /dev/null +++ b/grid-proxy/tests/queries/modifiers.sql @@ -0,0 +1,236 @@ +-- node modifiers +INSERT INTO node ( + id, + grid_version, + node_id, + farm_id, + twin_id, + country, + city, + uptime, + created, + farming_policy_id, + secure, + virtualized, + serial_number, + created_at, + updated_at, + location_id, + certification, + connection_price, + power, + extra_fee + ) +VALUES ( + 'node-id-999999', + 3, + 999999, + 1, + 999999, + 'Egypt', + 'Cairo', + 1651769370, + 1651769370, + 1, + false, + false, + '', + 1651769370, + 1651769370, + 'location-1', + 'not', + 0, + '{}', + 0 + ); +UPDATE node +SET country = 'Belgium' +WHERE node_id = 999999; +INSERT INTO node_resources_total (id, hru, sru, cru, mru, node_id) +VALUES ( + 'total-resources-999999', + 10000000000000, + 10000000000000, + 1000, + 10000000000000, + 'node-id-999999' + ); +UPDATE node_resources_total +SET cru = 2000, + hru = 20000000000000, + mru = 20000000000000, + sru = 20000000000000 +WHERE node_id = 'node-id-999999'; +-- capacity modifiers +INSERT INTO node_contract ( + id, + grid_version, + contract_id, + twin_id, + node_id, + deployment_data, + deployment_hash, + number_of_public_i_ps, + created_at, + resources_used_id, + state + ) +VALUES ( + 'node-contract-999999', + 1, + 999999, + 99, + 999999, + 'deployment_data:text', + 'deployment_hash:text', + 0, + 1600000000, + NULL, + 'Created' + ); +INSERT INTO node_contract ( + id, + grid_version, + contract_id, + twin_id, + node_id, + deployment_data, + deployment_hash, + number_of_public_i_ps, + created_at, + resources_used_id, + state + ) +VALUES ( + 'node-contract-999998', + 1, + 999998, + 99, + 999999, + 'deployment_data:text', + 'deployment_hash:text', + 0, + 1600000000, + NULL, + 'Created' + ); +INSERT INTO node_contract ( + id, + grid_version, + contract_id, + twin_id, + node_id, + deployment_data, + deployment_hash, + number_of_public_i_ps, + created_at, + resources_used_id, + state + ) +VALUES ( + 'node-contract-999995', + 1, + 999995, + 99, + 999999, + 'deployment_data:text', + 'deployment_hash:text', + 0, + 1600000000, + NULL, + 'Created' + ); +INSERT INTO contract_resources (id, hru, sru, cru, mru, contract_id) +VALUES ( + 'contract-resources-999999', + 1, + 1, + 1, + 1, + 'node-contract-999999' + ); +INSERT INTO contract_resources (id, hru, sru, cru, mru, contract_id) +VALUES ( + 'contract-resources-999998', + 1, + 1, + 1, + 1, + 'node-contract-999998' + ); +UPDATE contract_resources +SET cru = 1 +where contract_id = 'node-contract-999999'; +update node_contract +SET state = 'Deleted' +where contract_id = 999999; +-- renting modifiers +INSERT INTO rent_contract ( + id, + grid_version, + contract_id, + twin_id, + node_id, + created_at, + state + ) +VALUES ( + 'rent-contract-999997', + 1, + 999997, + 99, + 999999, + 15000000, + 'Created' + ); +Update rent_contract +set state = 'Deleted' +where contract_id = 999997; +INSERT INTO rent_contract ( + id, + grid_version, + contract_id, + twin_id, + node_id, + created_at, + state + ) +VALUES ( + 'rent-contract-999996', + 1, + 999996, + 99, + 999999, + 15000000, + 'Created' + ); +-- ips modifiers +INSERT INTO public_ip (id, gateway, ip, contract_id, farm_id) +VALUES ( + 'public-ip-999999', + 'gateway:text', + 'ip:text', + 0, + 'farm-1' + ); +INSERT INTO public_ip (id, gateway, ip, contract_id, farm_id) +VALUES ( + 'public-ip-999998', + 'gateway:text', + 'ip:text', + 0, + 'farm-1' + ); +update public_ip +set contract_id = 999998 +where id = 'public-ip-999999'; +update public_ip +set contract_id = 999995 +where id = 'public-ip-999998'; +update public_ip +set contract_id = 0 +where id = 'public-ip-999999'; +Delete from public_ip +where id = 'public-ip-999999'; +Delete from public_ip +where id = 'public-ip-999998'; \ No newline at end of file From 1e038978c8120d649a8258f5c1867e69c6982d96 Mon Sep 17 00:00:00 2001 From: mario Date: Thu, 21 Dec 2023 02:29:16 +0200 Subject: [PATCH 18/39] fix typos in setup.sql Co-authored-by: omarabulasis --- grid-proxy/internal/explorer/db/setup.sql | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/grid-proxy/internal/explorer/db/setup.sql b/grid-proxy/internal/explorer/db/setup.sql index fd579e9c1..3a59b22c0 100644 --- a/grid-proxy/internal/explorer/db/setup.sql +++ b/grid-proxy/internal/explorer/db/setup.sql @@ -88,8 +88,8 @@ SELECT country.name as country, country.subregion as region FROM node - LEFT JOIN node_contract ON node.node_id = node_contract.node_id - LEFT JOIN contract_resources ON node_contract.resources_used_id = contract_resources.id AND node_contract.state IN ('Created', 'GracePeriod') + LEFT JOIN node_contract ON node.node_id = node_contract.node_id AND node_contract.state IN ('Created', 'GracePeriod') + LEFT JOIN contract_resources ON node_contract.resources_used_id = contract_resources.id LEFT JOIN node_resources_total AS node_resources_total ON node_resources_total.node_id = node.id LEFT JOIN rent_contract on node.node_id = rent_contract.node_id AND rent_contract.state IN ('Created', 'GracePeriod') LEFT JOIN( @@ -250,7 +250,7 @@ BEGIN INSERT INTO resources_cache SELECT * FROM resources_cache_view - WHERE resources_cache_view.node_id = 99995; + WHERE resources_cache_view.node_id = NEW.node_id; EXCEPTION WHEN OTHERS THEN RAISE NOTICE 'Error inserting resources_cache: %', SQLERRM; @@ -280,9 +280,12 @@ BEGIN total_mru = NEW.mru, total_sru = NEW.sru, total_hru = NEW.hru, - free_mru = free_mru + (NEW.mru-COALESCE(OLD.mru, 0)), + free_mru = free_mru + GREATEST(CAST((OLD.mru / 10) AS bigint), 2147483648) - + GREATEST(CAST((NEW.mru / 10) AS bigint), 2147483648) + (NEW.mru-COALESCE(OLD.mru, 0)), free_hru = free_hru + (NEW.hru-COALESCE(OLD.hru, 0)), - free_sru = free_sru + (NEW.sru-COALESCE(OLD.sru, 0)) + free_sru = free_sru + (NEW.sru-COALESCE(OLD.sru, 0)), + used_mru = used_mru - GREATEST(CAST((OLD.mru / 10) AS bigint), 2147483648) + + GREATEST(CAST((NEW.mru / 10) AS bigint), 2147483648) WHERE resources_cache.node_id = ( SELECT node.node_id FROM node WHERE node.id = New.node_id From d847869763e4fe9bcc5e71b2689132615a16d061 Mon Sep 17 00:00:00 2001 From: mario Date: Thu, 21 Dec 2023 02:30:43 +0200 Subject: [PATCH 19/39] update node contract resources used id after insertion Co-authored-by: omarabulasis --- grid-proxy/tests/queries/modifiers.sql | 2 ++ 1 file changed, 2 insertions(+) diff --git a/grid-proxy/tests/queries/modifiers.sql b/grid-proxy/tests/queries/modifiers.sql index 0f09eee5c..930f27cbb 100644 --- a/grid-proxy/tests/queries/modifiers.sql +++ b/grid-proxy/tests/queries/modifiers.sql @@ -158,6 +158,8 @@ VALUES ( 1, 'node-contract-999998' ); +update node_contract set resources_used_id = 'contract-resources-999999' where contract_id = 999999; +update node_contract set resources_used_id = 'contract-resources-999998' where contract_id = 999998; UPDATE contract_resources SET cru = 1 where contract_id = 'node-contract-999999'; From 25f188fe6ffebd52420756bae435a27390a09f4a Mon Sep 17 00:00:00 2001 From: mario Date: Thu, 21 Dec 2023 11:49:14 +0200 Subject: [PATCH 20/39] fix farm sorting with available for flag in mock client Co-authored-by: omarabdulasis --- grid-proxy/tests/queries/mock_client/farms.go | 4 ++-- grid-proxy/tests/queries/mock_client/loader.go | 7 ++++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/grid-proxy/tests/queries/mock_client/farms.go b/grid-proxy/tests/queries/mock_client/farms.go index 734d8ceef..79ed57f34 100644 --- a/grid-proxy/tests/queries/mock_client/farms.go +++ b/grid-proxy/tests/queries/mock_client/farms.go @@ -51,8 +51,8 @@ func (g *GridProxyMockClient) Farms(ctx context.Context, filter types.FarmFilter if filter.NodeAvailableFor != nil { sort.Slice(res, func(i, j int) bool { - f1 := g.data.FarmHasRentedNode[uint64(res[i].FarmID)] - f2 := g.data.FarmHasRentedNode[uint64(res[j].FarmID)] + f1 := g.data.FarmHasRentedNode[uint64(res[i].FarmID)][*filter.NodeAvailableFor] + f2 := g.data.FarmHasRentedNode[uint64(res[j].FarmID)][*filter.NodeAvailableFor] lessFarmID := res[i].FarmID < res[j].FarmID return f1 && !f2 || f1 && f2 && lessFarmID || !f1 && !f2 && lessFarmID diff --git a/grid-proxy/tests/queries/mock_client/loader.go b/grid-proxy/tests/queries/mock_client/loader.go index daab00bd4..30ecf388d 100644 --- a/grid-proxy/tests/queries/mock_client/loader.go +++ b/grid-proxy/tests/queries/mock_client/loader.go @@ -17,7 +17,7 @@ type DBData struct { NodeUsedResources map[uint64]NodeResourcesTotal NodeRentedBy map[uint64]uint64 NodeRentContractID map[uint64]uint64 - FarmHasRentedNode map[uint64]bool + FarmHasRentedNode map[uint64]map[uint64]bool Nodes map[uint64]Node NodeTotalResources map[uint64]NodeResourcesTotal @@ -133,7 +133,7 @@ func calcRentInfo(data *DBData) error { data.NodeRentedBy[contract.NodeID] = contract.TwinID data.NodeRentContractID[contract.NodeID] = contract.ContractID farmID := data.Nodes[contract.NodeID].FarmID - data.FarmHasRentedNode[farmID] = true + data.FarmHasRentedNode[farmID][contract.TwinID] = true } return nil } @@ -213,6 +213,7 @@ func loadFarms(db *sql.DB, data *DBData) error { } data.Farms[farm.FarmID] = farm data.FarmIDMap[farm.ID] = farm.FarmID + data.FarmHasRentedNode[farm.FarmID] = map[uint64]bool{} } return nil } @@ -599,7 +600,7 @@ func Load(db *sql.DB) (DBData, error) { NodeUsedResources: make(map[uint64]NodeResourcesTotal), NonDeletedContracts: make(map[uint64][]uint64), GPUs: make(map[uint64][]NodeGPU), - FarmHasRentedNode: make(map[uint64]bool), + FarmHasRentedNode: make(map[uint64]map[uint64]bool), Regions: make(map[string]string), Locations: make(map[string]Location), DB: db, From 38a2939161603e9a496c17e99c1e7f6dffb46c2a Mon Sep 17 00:00:00 2001 From: mario Date: Thu, 21 Dec 2023 11:50:32 +0200 Subject: [PATCH 21/39] remove commented lines in tests Co-authored-by: omarabdulasis --- grid-proxy/tests/queries/farm_test.go | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/grid-proxy/tests/queries/farm_test.go b/grid-proxy/tests/queries/farm_test.go index d16209c56..587dc138e 100644 --- a/grid-proxy/tests/queries/farm_test.go +++ b/grid-proxy/tests/queries/farm_test.go @@ -144,10 +144,10 @@ func TestFarm(t *testing.T) { for ; ; l.Page++ { want, wantCount, err := mockClient.Farms(context.Background(), f, l) require.NoError(t, err) - // log.Printf("want: %+v", want) + got, gotCount, err := gridProxyClient.Farms(context.Background(), f, l) require.NoError(t, err) - // log.Printf("got: %+v", got) + assert.Equal(t, wantCount, gotCount) sortPublicIPs(want, got) @@ -172,10 +172,9 @@ func TestFarm(t *testing.T) { want, wantCount, err := mockClient.Farms(context.Background(), f, l) require.NoError(t, err) - // log.Printf("want: %+v", want) + got, gotCount, err := gridProxyClient.Farms(context.Background(), f, l) require.NoError(t, err) - // log.Printf("got: %+v", got) assert.Equal(t, wantCount, gotCount) @@ -196,10 +195,9 @@ func TestFarm(t *testing.T) { want, wantCount, err := mockClient.Farms(context.Background(), f, l) require.NoError(t, err) - // log.Printf("want: %+v", want) + got, gotCount, err := gridProxyClient.Farms(context.Background(), f, l) require.NoError(t, err) - // log.Printf("got: %+v", got) assert.Equal(t, wantCount, gotCount) From b76f4aa319c36ff09abc8cd90db66fbc1d248ee8 Mon Sep 17 00:00:00 2001 From: mario Date: Thu, 21 Dec 2023 12:59:34 +0200 Subject: [PATCH 22/39] parallelize tests and make test run count default to 3 Co-authored-by: omarabdulasis --- grid-proxy/Makefile | 7 ++++++- grid-proxy/tests/queries/contract_test.go | 16 ++++++++++++++- grid-proxy/tests/queries/counters_test.go | 3 +++ grid-proxy/tests/queries/farm_test.go | 13 +++++++++++- grid-proxy/tests/queries/node_test.go | 24 ++++++++++++++++++++++- grid-proxy/tests/queries/twin_test.go | 10 +++++++++- 6 files changed, 68 insertions(+), 5 deletions(-) diff --git a/grid-proxy/Makefile b/grid-proxy/Makefile index 6bf92d692..e6ac938be 100644 --- a/grid-proxy/Makefile +++ b/grid-proxy/Makefile @@ -1,6 +1,7 @@ .DEFAULT_GOAL := help PQ_HOST = $(shell docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' postgres) PQ_CONTAINER = postgres +count = 3 install-swag: @go install github.com/swaggo/swag/cmd/swag@v1.8.12; @@ -62,22 +63,26 @@ sleep: test-queries: ## Run all queries tests @cd tests/queries/ &&\ go test -v \ + -parallel 20 \ --seed 13 \ --postgres-host $(PQ_HOST) \ --postgres-db tfgrid-graphql \ --postgres-password postgres \ --postgres-user postgres \ - --endpoint http://localhost:8080 + --endpoint http://localhost:8080 \ + -count $(count) test-query: ## Run specific test query (Args: `t=TestName`). @cd tests/queries/ &&\ go test -v \ + -parallel 10 \ --seed 13 \ --postgres-host $(PQ_HOST) \ --postgres-db tfgrid-graphql \ --postgres-password postgres \ --postgres-user postgres \ --endpoint http://localhost:8080 \ + -count $(count) \ -run $(t) test-unit: ## Run only unit tests diff --git a/grid-proxy/tests/queries/contract_test.go b/grid-proxy/tests/queries/contract_test.go index 4a2b153da..66784b7db 100644 --- a/grid-proxy/tests/queries/contract_test.go +++ b/grid-proxy/tests/queries/contract_test.go @@ -68,14 +68,17 @@ var contractFilterRandomValueGenerator = map[string]func(agg ContractsAggregate) } func TestContracts(t *testing.T) { + t.Parallel() t.Run("contracts pagination test", func(t *testing.T) { + t.Parallel() + node := "node" f := proxytypes.ContractFilter{ Type: &node, } l := proxytypes.Limit{ - Size: 5, + Size: 100, Page: 1, RetCount: true, } @@ -99,6 +102,8 @@ func TestContracts(t *testing.T) { }) t.Run("contracts stress test", func(t *testing.T) { + t.Parallel() + agg := calcContractsAggregates(&data) for i := 0; i < CONTRACTS_TESTS; i++ { l := proxytypes.Limit{ @@ -124,7 +129,10 @@ func TestContracts(t *testing.T) { } func TestContract(t *testing.T) { + t.Parallel() t.Run("single contract test", func(t *testing.T) { + t.Parallel() + contractID := rand.Intn(CONTRACTS_TESTS) want, err := mockClient.Contract(context.Background(), uint32(contractID)) @@ -137,6 +145,8 @@ func TestContract(t *testing.T) { }) t.Run("contract not found test", func(t *testing.T) { + t.Parallel() + contractID := 1000000000000 _, err := gridProxyClient.Contract(context.Background(), uint32(contractID)) assert.Equal(t, err.Error(), ErrContractNotFound.Error()) @@ -145,6 +155,8 @@ func TestContract(t *testing.T) { func TestBills(t *testing.T) { t.Run("contract bills test", func(t *testing.T) { + t.Parallel() + contractID := rand.Intn(CONTRACTS_TESTS) l := proxytypes.Limit{ @@ -174,6 +186,8 @@ func TestBills(t *testing.T) { // TestContractsFilter iterates over all ContractFilter fields, and for each one generates a random value, then runs a test between the mock client and the gridproxy client func TestContractsFilter(t *testing.T) { + t.Parallel() + f := proxytypes.ContractFilter{} fp := &f v := reflect.ValueOf(fp).Elem() diff --git a/grid-proxy/tests/queries/counters_test.go b/grid-proxy/tests/queries/counters_test.go index b09b06f46..8e3686ce1 100644 --- a/grid-proxy/tests/queries/counters_test.go +++ b/grid-proxy/tests/queries/counters_test.go @@ -21,6 +21,7 @@ var statsFilterRandomValues = map[string]func() interface{}{ } func TestStats(t *testing.T) { + t.Parallel() t.Run("stats up test", func(t *testing.T) { f := proxytypes.StatsFilter{ Status: &STATUS_UP, @@ -48,6 +49,8 @@ func TestStats(t *testing.T) { } func TestStatsFilter(t *testing.T) { + t.Parallel() + f := proxytypes.StatsFilter{} fp := &f v := reflect.ValueOf(fp).Elem() diff --git a/grid-proxy/tests/queries/farm_test.go b/grid-proxy/tests/queries/farm_test.go index 587dc138e..2f7e30183 100644 --- a/grid-proxy/tests/queries/farm_test.go +++ b/grid-proxy/tests/queries/farm_test.go @@ -131,13 +131,16 @@ type FarmsAggregate struct { } func TestFarm(t *testing.T) { + t.Parallel() t.Run("farms pagination test", func(t *testing.T) { + t.Parallel() + one := uint64(1) f := proxytypes.FarmFilter{ TotalIPs: &one, } l := proxytypes.Limit{ - Size: 5, + Size: 100, Page: 1, RetCount: true, } @@ -160,6 +163,8 @@ func TestFarm(t *testing.T) { }) t.Run("farms stress test", func(t *testing.T) { + t.Parallel() + agg := calcFarmsAggregates(&data) for i := 0; i < FARM_TESTS; i++ { l := proxytypes.Limit{ @@ -183,6 +188,8 @@ func TestFarm(t *testing.T) { } }) t.Run("farms list node free hru", func(t *testing.T) { + t.Parallel() + aggNode := calcNodesAggregates(&data) l := proxytypes.Limit{ Size: 999999999999, @@ -205,6 +212,8 @@ func TestFarm(t *testing.T) { require.True(t, reflect.DeepEqual(want, got), fmt.Sprintf("Used Filter:\n%s", SerializeFilter(f)), fmt.Sprintf("Difference:\n%s", cmp.Diff(want, got))) }) t.Run("farms list node free hru, mru", func(t *testing.T) { + t.Parallel() + aggNode := calcNodesAggregates(&data) l := proxytypes.Limit{ @@ -233,6 +242,8 @@ func TestFarm(t *testing.T) { // TestFarmFilter iterates over all FarmFilter fields, and for each one generates a random value, then runs a test between the mock client and the gridproxy client func TestFarmFilter(t *testing.T) { + t.Parallel() + f := proxytypes.FarmFilter{} fp := &f v := reflect.ValueOf(fp).Elem() diff --git a/grid-proxy/tests/queries/node_test.go b/grid-proxy/tests/queries/node_test.go index fac602e02..66a8925a6 100644 --- a/grid-proxy/tests/queries/node_test.go +++ b/grid-proxy/tests/queries/node_test.go @@ -270,6 +270,7 @@ var nodeFilterRandomValueGenerator = map[string]func(agg NodesAggregate) interfa } func TestNode(t *testing.T) { + t.Parallel() t.Run("node pagination test", func(t *testing.T) { nodePaginationCheck(t, mockClient, gridProxyClient) }) @@ -279,6 +280,8 @@ func TestNode(t *testing.T) { }) t.Run("node up test", func(t *testing.T) { + t.Parallel() + f := types.NodeFilter{ Status: &STATUS_UP, } @@ -301,6 +304,8 @@ func TestNode(t *testing.T) { }) t.Run("node status test", func(t *testing.T) { + t.Parallel() + for i := 1; i <= NODE_COUNT; i++ { if flip(.3) { want, err := mockClient.NodeStatus(context.Background(), uint32(i)) @@ -315,6 +320,8 @@ func TestNode(t *testing.T) { }) t.Run("node stress test", func(t *testing.T) { + t.Parallel() + agg := calcNodesAggregates(&data) for i := 0; i < NODE_TESTS; i++ { l := types.Limit{ @@ -338,12 +345,16 @@ func TestNode(t *testing.T) { }) t.Run("node not found test", func(t *testing.T) { + t.Parallel() + nodeID := 1000000000 _, err := gridProxyClient.Node(context.Background(), uint32(nodeID)) assert.Equal(t, err.Error(), ErrNodeNotFound.Error()) }) t.Run("nodes test certification_type filter", func(t *testing.T) { + t.Parallel() + certType := "Diy" nodes, _, err := gridProxyClient.Nodes(context.Background(), types.NodeFilter{CertificationType: &certType}, types.DefaultLimit()) require.NoError(t, err) @@ -359,6 +370,8 @@ func TestNode(t *testing.T) { }) t.Run("nodes test has_gpu filter", func(t *testing.T) { + t.Parallel() + l := proxytypes.DefaultLimit() hasGPU := true f := proxytypes.NodeFilter{ @@ -375,6 +388,8 @@ func TestNode(t *testing.T) { }) t.Run("nodes test gpu vendor, device name filter", func(t *testing.T) { + t.Parallel() + device := "navi" vendor := "advanced" nodes, _, err := gridProxyClient.Nodes(context.Background(), types.NodeFilter{GpuDeviceName: &device, GpuVendorName: &vendor}, types.DefaultLimit()) @@ -387,6 +402,8 @@ func TestNode(t *testing.T) { }) t.Run("nodes test gpu vendor, device id filter", func(t *testing.T) { + t.Parallel() + device := "744c" vendor := "1002" nodes, _, err := gridProxyClient.Nodes(context.Background(), types.NodeFilter{GpuDeviceID: &device, GpuVendorID: &vendor}, types.DefaultLimit()) @@ -399,6 +416,8 @@ func TestNode(t *testing.T) { }) t.Run("nodes test gpu available", func(t *testing.T) { + t.Parallel() + available := false nodes, _, err := gridProxyClient.Nodes(context.Background(), types.NodeFilter{GpuAvailable: &available}, types.DefaultLimit()) assert.NoError(t, err) @@ -412,6 +431,8 @@ func TestNode(t *testing.T) { // TestNodeFilter iterates over all NodeFilter fields, and for each one generates a random value, then runs a test between the mock client and the gridproxy client func TestNodeFilter(t *testing.T) { + t.Parallel() + f := types.NodeFilter{} fp := &f v := reflect.ValueOf(fp).Elem() @@ -449,6 +470,7 @@ func TestNodeFilter(t *testing.T) { } func singleNodeCheck(t *testing.T, localClient proxyclient.Client, proxyClient proxyclient.Client) { + t.Parallel() nodeID := rand.Intn(NODE_COUNT) want, err := mockClient.Node(context.Background(), uint32(nodeID)) require.NoError(t, err) @@ -464,7 +486,7 @@ func nodePaginationCheck(t *testing.T, localClient proxyclient.Client, proxyClie Status: &STATUS_DOWN, } l := types.Limit{ - Size: 5, + Size: 100, Page: 1, RetCount: true, } diff --git a/grid-proxy/tests/queries/twin_test.go b/grid-proxy/tests/queries/twin_test.go index a70dea93b..0a0ba3d70 100644 --- a/grid-proxy/tests/queries/twin_test.go +++ b/grid-proxy/tests/queries/twin_test.go @@ -43,10 +43,14 @@ var twinFilterRandomValueGenerator = map[string]func(agg TwinsAggregate) interfa } func TestTwins(t *testing.T) { + t.Parallel() + t.Run("twins pagination test", func(t *testing.T) { + t.Parallel() + f := proxytypes.TwinFilter{} l := proxytypes.Limit{ - Size: 5, + Size: 100, Page: 1, RetCount: true, } @@ -69,6 +73,8 @@ func TestTwins(t *testing.T) { }) t.Run("twins stress test", func(t *testing.T) { + t.Parallel() + agg := calcTwinsAggregates(&data) for i := 0; i < TWINS_TESTS; i++ { l := proxytypes.Limit{ @@ -94,6 +100,8 @@ func TestTwins(t *testing.T) { // TestTwinFilter iterates over all TwinFilter fields, and for each one generates a random value, then runs a test between the mock client and the gridproxy client func TestTwinFilter(t *testing.T) { + t.Parallel() + f := proxytypes.TwinFilter{} fp := &f v := reflect.ValueOf(fp).Elem() From db92beb44388010f34e52f2122b40018eb2d9eae Mon Sep 17 00:00:00 2001 From: mario Date: Thu, 21 Dec 2023 13:25:32 +0200 Subject: [PATCH 23/39] fix linter issues in gridproxy Co-authored-by: omarabdulasis --- grid-proxy/internal/explorer/db/postgres.go | 17 +++++++---------- grid-proxy/tools/db/generate.go | 6 +++--- 2 files changed, 10 insertions(+), 13 deletions(-) diff --git a/grid-proxy/internal/explorer/db/postgres.go b/grid-proxy/internal/explorer/db/postgres.go index 6e405e613..14d3a3804 100644 --- a/grid-proxy/internal/explorer/db/postgres.go +++ b/grid-proxy/internal/explorer/db/postgres.go @@ -201,8 +201,7 @@ func (np *NodePower) Scan(value interface{}) error { // GetNode returns node info func (d *PostgresDatabase) GetNode(ctx context.Context, nodeID uint32) (Node, error) { - q := d.nodeTableQuery(types.NodeFilter{}, &gorm.DB{}) - q = q.WithContext(ctx) + q := d.nodeTableQuery(ctx, types.NodeFilter{}, &gorm.DB{}) q = q.Where("node.node_id = ?", nodeID) var node Node res := q.Scan(&node) @@ -258,8 +257,8 @@ func printQuery(query string, args ...interface{}) { fmt.Printf("node query: %s", query) } -func (d *PostgresDatabase) nodeTableQuery(filter types.NodeFilter, nodeGpuSubquery *gorm.DB) *gorm.DB { - q := d.gormDB. +func (d *PostgresDatabase) nodeTableQuery(ctx context.Context, filter types.NodeFilter, nodeGpuSubquery *gorm.DB) *gorm.DB { + q := d.gormDB.WithContext(ctx). Table("node"). Select( "node.id", @@ -317,8 +316,8 @@ func (d *PostgresDatabase) nodeTableQuery(filter types.NodeFilter, nodeGpuSubque return q } -func (d *PostgresDatabase) farmTableQuery(filter types.FarmFilter, nodeQuery *gorm.DB) *gorm.DB { - q := d.gormDB. +func (d *PostgresDatabase) farmTableQuery(ctx context.Context, filter types.FarmFilter, nodeQuery *gorm.DB) *gorm.DB { + q := d.gormDB.WithContext(ctx). Table("farm"). Select( "farm.id", @@ -360,7 +359,6 @@ func (d *PostgresDatabase) farmTableQuery(filter types.FarmFilter, nodeQuery *go // GetFarms return farms filtered and paginated func (d *PostgresDatabase) GetFarms(ctx context.Context, filter types.FarmFilter, limit types.Limit) ([]Farm, uint, error) { - q := d.gormDB.WithContext(ctx) nodeQuery := d.gormDB.Table("resources_cache"). Select("resources_cache.farm_id", "renter"). Joins("LEFT JOIN node ON node.node_id = resources_cache.node_id"). @@ -404,7 +402,7 @@ func (d *PostgresDatabase) GetFarms(ctx context.Context, filter types.FarmFilter nodeQuery = nodeQuery.Where("(node.certification = 'Certified') = ?", *filter.NodeCertified) } - q = d.farmTableQuery(filter, nodeQuery) + q := d.farmTableQuery(ctx, filter, nodeQuery) if filter.NodeAvailableFor != nil { q = q.Where("COALESCE(resources_cache.renter, 0) = ? OR (resources_cache.renter IS NULL AND farm.dedicated_farm = false)", *filter.NodeAvailableFor) @@ -482,7 +480,6 @@ func (d *PostgresDatabase) GetFarms(ctx context.Context, filter types.FarmFilter // GetNodes returns nodes filtered and paginated func (d *PostgresDatabase) GetNodes(ctx context.Context, filter types.NodeFilter, limit types.Limit) ([]Node, uint, error) { - q := d.gormDB.WithContext(ctx) /* used distinct selecting to avoid duplicated node after the join. - postgres apply WHERE before DISTINCT so filters will still filter on the whole data. @@ -515,7 +512,7 @@ func (d *PostgresDatabase) GetNodes(ctx context.Context, filter types.NodeFilter nodeGpuSubquery = nodeGpuSubquery.Where("(COALESCE(node_gpu.contract, 0) = 0) = ?", *filter.GpuAvailable) } - q = d.nodeTableQuery(filter, nodeGpuSubquery) + q := d.nodeTableQuery(ctx, filter, nodeGpuSubquery) condition := "TRUE" if filter.Status != nil { diff --git a/grid-proxy/tools/db/generate.go b/grid-proxy/tools/db/generate.go index 37b859743..9d8e063df 100644 --- a/grid-proxy/tools/db/generate.go +++ b/grid-proxy/tools/db/generate.go @@ -589,7 +589,7 @@ func generateNodes(db *sql.DB) error { nodesHRU[i] = hru nodeUP[i] = up - // location latitue and longitue needs to be castable to decimal + // location latitude and longitue needs to be castable to decimal // if not, the convert_to_decimal function will raise a notice // reporting the incident, which downgrades performance location := location{ @@ -604,8 +604,8 @@ func generateNodes(db *sql.DB) error { id: fmt.Sprintf("node-%d", i), location_id: fmt.Sprintf("location-%d", i), node_id: i, - farm_id: i%100 + 1, - twin_id: i + 100 + 1, + farm_id: i%farmCount + 1, + twin_id: i + farmCount + 1, country: countries[countryIndex], city: cities[countries[countryIndex]][cityIndex], uptime: 1000, From 116dcaac9e76978dafb51feda10efc100bb7a2f4 Mon Sep 17 00:00:00 2001 From: mario Date: Thu, 21 Dec 2023 14:02:10 +0200 Subject: [PATCH 24/39] revert farm number change in gridproxy tests Co-authored-by: omarabdulasis --- grid-proxy/tools/db/generate.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/grid-proxy/tools/db/generate.go b/grid-proxy/tools/db/generate.go index 9d8e063df..f89fd7831 100644 --- a/grid-proxy/tools/db/generate.go +++ b/grid-proxy/tools/db/generate.go @@ -604,8 +604,8 @@ func generateNodes(db *sql.DB) error { id: fmt.Sprintf("node-%d", i), location_id: fmt.Sprintf("location-%d", i), node_id: i, - farm_id: i%farmCount + 1, - twin_id: i + farmCount + 1, + farm_id: i%100 + 1, + twin_id: i + 100 + 1, country: countries[countryIndex], city: cities[countries[countryIndex]][cityIndex], uptime: 1000, From 53d6916da2961fb29601c288e070a70ef07db024 Mon Sep 17 00:00:00 2001 From: mario Date: Thu, 21 Dec 2023 15:43:04 +0200 Subject: [PATCH 25/39] handle node delete case in node trigger Co-authored-by: omarabdulasis --- grid-proxy/internal/explorer/db/setup.sql | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/grid-proxy/internal/explorer/db/setup.sql b/grid-proxy/internal/explorer/db/setup.sql index 3a59b22c0..7f3b2e926 100644 --- a/grid-proxy/internal/explorer/db/setup.sql +++ b/grid-proxy/internal/explorer/db/setup.sql @@ -255,6 +255,14 @@ BEGIN WHEN OTHERS THEN RAISE NOTICE 'Error inserting resources_cache: %', SQLERRM; END; + + ELSIF (TG_OP = 'DELETE') THEN + BEGIN + DELETE FROM resources_cache WHERE node_id = OLD.node_id; + EXCEPTION + WHEN OTHERS THEN + RAISE NOTICE 'Error deleting node from resources_cache: %', SQLERRM; + END; END IF; RETURN NULL; @@ -262,7 +270,7 @@ END; $$ LANGUAGE plpgsql; CREATE OR REPLACE TRIGGER node_trigger - AFTER INSERT OR UPDATE OF country + AFTER INSERT OR DELETE OR UPDATE OF country ON node FOR EACH ROW EXECUTE PROCEDURE node_upsert(); From 7e791d01e970d7426c48442650140474315e7ed9 Mon Sep 17 00:00:00 2001 From: mario Date: Thu, 21 Dec 2023 15:43:46 +0200 Subject: [PATCH 26/39] wip: modularize data generation Co-authored-by: omarabdulasis --- grid-proxy/tests/queries/main_test.go | 26 + grid-proxy/tools/db/db.go | 3 +- grid-proxy/tools/db/generate.go | 1561 +++++++++---------- grid-proxy/tools/db/modifiers/generators.go | 714 +++++++++ grid-proxy/tools/db/modifiers/modifiers.go | 5 + grid-proxy/tools/db/modifiers/types.go | 246 +++ grid-proxy/tools/db/modifiers/utils.go | 139 ++ 7 files changed, 1897 insertions(+), 797 deletions(-) create mode 100644 grid-proxy/tools/db/modifiers/generators.go create mode 100644 grid-proxy/tools/db/modifiers/modifiers.go create mode 100644 grid-proxy/tools/db/modifiers/types.go create mode 100644 grid-proxy/tools/db/modifiers/utils.go diff --git a/grid-proxy/tests/queries/main_test.go b/grid-proxy/tests/queries/main_test.go index dc0b9d6b2..7191c1ec6 100644 --- a/grid-proxy/tests/queries/main_test.go +++ b/grid-proxy/tests/queries/main_test.go @@ -97,5 +97,31 @@ func TestMain(m *testing.M) { mockClient = mock.NewGridProxyMockClient(data) exitcode = m.Run() + + // cleanup modified data os.Exit(exitcode) } + +func modifyDataToFireTriggers(d *sql.DB) { + /* + - insert nodes - y + - should be on new/old farms + - should be on new/old locations + - insert node total resources - y + - insert node contracts - y + - insert contract resources - y + - insert rent contracts - y + - insert public ips - y + + - update node country - y + - update node total resources - y + - update contract_resources - y + - update node contract state - y + - update rent contract state - y + - update public ip contract id - y + + - delete node - y + - delete public ip - y + */ + // modifiers.GenerateTwins(db) +} diff --git a/grid-proxy/tools/db/db.go b/grid-proxy/tools/db/db.go index 3abbc035c..d96fcb9f6 100644 --- a/grid-proxy/tools/db/db.go +++ b/grid-proxy/tools/db/db.go @@ -41,7 +41,6 @@ func parseCmdline() flags { func main() { f := parseCmdline() - r = rand.New(rand.NewSource(int64(f.seed))) psqlInfo := fmt.Sprintf("host=%s port=%d user=%s "+ "password=%s dbname=%s sslmode=disable", @@ -101,7 +100,7 @@ func main() { panic(err) } // ---- - if err := generateData(db); err != nil { + if err := generateData(db, f.seed); err != nil { panic(err) } } diff --git a/grid-proxy/tools/db/generate.go b/grid-proxy/tools/db/generate.go index f89fd7831..540293209 100644 --- a/grid-proxy/tools/db/generate.go +++ b/grid-proxy/tools/db/generate.go @@ -4,74 +4,70 @@ import ( "database/sql" "fmt" "os" - "reflect" - "strings" - "time" - "github.com/google/uuid" - "github.com/threefoldtech/zos/pkg/gridtypes" + "github.com/threefoldtech/tfgrid-sdk-go/grid-proxy/tools/db/modifiers" ) -var ( - nodesMRU = make(map[uint64]uint64) - nodesSRU = make(map[uint64]uint64) - nodesHRU = make(map[uint64]uint64) - nodeUP = make(map[uint64]bool) - createdNodeContracts = make([]uint64, 0) - dedicatedFarms = make(map[uint64]struct{}) - availableRentNodes = make(map[uint64]struct{}) - availableRentNodesList = make([]uint64, 0) - renter = make(map[uint64]uint64) -) - -const ( - contractCreatedRatio = .1 // from devnet - usedPublicIPsRatio = .9 - nodeUpRatio = .5 - nodeCount = 6000 - farmCount = 600 - normalUsers = 6000 - publicIPCount = 1000 - twinCount = nodeCount + farmCount + normalUsers - nodeContractCount = 9000 - rentContractCount = 100 - nameContractCount = 300 - - maxContractHRU = 1024 * 1024 * 1024 * 300 - maxContractSRU = 1024 * 1024 * 1024 * 300 - maxContractMRU = 1024 * 1024 * 1024 * 16 - maxContractCRU = 16 - minContractHRU = 0 - minContractSRU = 1024 * 1024 * 256 - minContractMRU = 1024 * 1024 * 256 - minContractCRU = 1 -) - -var ( - countries = []string{"Belgium", "United States", "Egypt", "United Kingdom"} - regions = map[string]string{ - "Belgium": "Europe", - "United States": "Americas", - "Egypt": "Africa", - "United Kingdom": "Europe", - } - countriesCodes = map[string]string{ - "Belgium": "BG", - "United States": "US", - "Egypt": "EG", - "United Kingdom": "UK", - } - cities = map[string][]string{ - "Belgium": {"Brussels", "Antwerp", "Ghent", "Charleroi"}, - "United States": {"New York", "Chicago", "Los Angeles", "San Francisco"}, - "Egypt": {"Cairo", "Giza", "October", "Nasr City"}, - "United Kingdom": {"London", "Liverpool", "Manchester", "Cambridge"}, - } -) - -const deleted = "Deleted" -const created = "Created" -const gracePeriod = "GracePeriod" +// var ( +// nodesMRU = make(map[uint64]uint64) +// nodesSRU = make(map[uint64]uint64) +// nodesHRU = make(map[uint64]uint64) +// nodeUP = make(map[uint64]bool) +// createdNodeContracts = make([]uint64, 0) +// dedicatedFarms = make(map[uint64]struct{}) +// availableRentNodes = make(map[uint64]struct{}) +// availableRentNodesList = make([]uint64, 0) +// renter = make(map[uint64]uint64) +// ) + +// const ( +// contractCreatedRatio = .1 // from devnet +// usedPublicIPsRatio = .9 +// nodeUpRatio = .5 +// nodeCount = 6000 +// farmCount = 600 +// normalUsers = 6000 +// publicIPCount = 1000 +// twinCount = nodeCount + farmCount + normalUsers +// nodeContractCount = 9000 +// rentContractCount = 100 +// nameContractCount = 300 + +// maxContractHRU = 1024 * 1024 * 1024 * 300 +// maxContractSRU = 1024 * 1024 * 1024 * 300 +// maxContractMRU = 1024 * 1024 * 1024 * 16 +// maxContractCRU = 16 +// minContractHRU = 0 +// minContractSRU = 1024 * 1024 * 256 +// minContractMRU = 1024 * 1024 * 256 +// minContractCRU = 1 +// ) + +// var ( +// countries = []string{"Belgium", "United States", "Egypt", "United Kingdom"} +// regions = map[string]string{ +// "Belgium": "Europe", +// "United States": "Americas", +// "Egypt": "Africa", +// "United Kingdom": "Europe", +// } +// countriesCodes = map[string]string{ +// "Belgium": "BG", +// "United States": "US", +// "Egypt": "EG", +// "United Kingdom": "UK", +// } +// cities = map[string][]string{ +// "Belgium": {"Brussels", "Antwerp", "Ghent", "Charleroi"}, +// "United States": {"New York", "Chicago", "Los Angeles", "San Francisco"}, +// "Egypt": {"Cairo", "Giza", "October", "Nasr City"}, +// "United Kingdom": {"London", "Liverpool", "Manchester", "Cambridge"}, +// } +// ) + +// const deleted = "Deleted" +// const created = "Created" +// const gracePeriod = "GracePeriod" func initSchema(db *sql.DB) error { schema, err := os.ReadFile("./schema.sql") @@ -85,754 +81,729 @@ func initSchema(db *sql.DB) error { return nil } -func generateTwins(db *sql.DB) error { - var twins []string - - for i := uint64(1); i <= twinCount; i++ { - twin := twin{ - id: fmt.Sprintf("twin-%d", i), - account_id: fmt.Sprintf("account-id-%d", i), - relay: fmt.Sprintf("relay-%d", i), - public_key: fmt.Sprintf("public-key-%d", i), - twin_id: i, - grid_version: 3, - } - tuple, err := objectToTupleString(twin) - if err != nil { - return fmt.Errorf("failed to convert twin object to tuple string: %w", err) - } - twins = append(twins, tuple) - } - - if err := insertTuples(db, twin{}, twins); err != nil { - return fmt.Errorf("failed to insert twins: %w", err) - } - fmt.Println("twins generated") - - return nil -} - -func generatePublicIPs(db *sql.DB) error { - var publicIPs []string - var nodeContracts []uint64 - - for i := uint64(1); i <= publicIPCount; i++ { - contract_id := uint64(0) - if flip(usedPublicIPsRatio) { - idx, err := rnd(0, uint64(len(createdNodeContracts))-1) - if err != nil { - return fmt.Errorf("failed to generate random index: %w", err) - } - contract_id = createdNodeContracts[idx] - } - ip := randomIPv4() - farmID, err := rnd(1, farmCount) - if err != nil { - return fmt.Errorf("failed to generate random farm id: %w", err) - } - - public_ip := public_ip{ - id: fmt.Sprintf("public-ip-%d", i), - gateway: ip.String(), - ip: IPv4Subnet(ip).String(), - contract_id: contract_id, - farm_id: fmt.Sprintf("farm-%d", farmID), - } - publicIpTuple, err := objectToTupleString(public_ip) - if err != nil { - return fmt.Errorf("failed to convert public ip object to tuple string: %w", err) - } - publicIPs = append(publicIPs, publicIpTuple) - nodeContracts = append(nodeContracts, contract_id) - } - - if err := insertTuples(db, public_ip{}, publicIPs); err != nil { - return fmt.Errorf("failed to insert public ips: %w", err) - } - - if err := updateNodeContractPublicIPs(db, nodeContracts); err != nil { - return fmt.Errorf("failed to update contract public ips: %w", err) - } - - fmt.Println("public IPs generated") - - return nil -} - -func generateFarms(db *sql.DB) error { - var farms []string - - for i := uint64(1); i <= farmCount; i++ { - farm := farm{ - id: fmt.Sprintf("farm-%d", i), - farm_id: i, - name: fmt.Sprintf("farm-name-%d", i), - certification: "Diy", - dedicated_farm: flip(.1), - twin_id: i, - pricing_policy_id: 1, - grid_version: 3, - stellar_address: "", - } - - if farm.dedicated_farm { - dedicatedFarms[farm.farm_id] = struct{}{} - } - - farmTuple, err := objectToTupleString(farm) - if err != nil { - return fmt.Errorf("failed to convert farm object to tuple string: %w", err) - } - farms = append(farms, farmTuple) - } - - if err := insertTuples(db, farm{}, farms); err != nil { - return fmt.Errorf("failed to insert farms: %w", err) - } - fmt.Println("farms generated") - - return nil -} - -func generateCountries(db *sql.DB) error { - var countriesValues []string - index := 0 - for countryName, region := range regions { - index++ - country := country{ - id: fmt.Sprintf("country-%d", index), - country_id: uint64(index), - name: countryName, - code: countriesCodes[countryName], - region: "unknown", - subregion: region, - lat: fmt.Sprintf("%d", 0), - long: fmt.Sprintf("%d", 0), - } - - countryTuple, err := objectToTupleString(country) - if err != nil { - return fmt.Errorf("failed to convert country object to tuple string: %w", err) - } - countriesValues = append(countriesValues, countryTuple) - } - - if err := insertTuples(db, country{}, countriesValues); err != nil { - return fmt.Errorf("failed to insert country: %w", err) - } - fmt.Println("countries generated") - - return nil -} - -func generateNodeContracts(db *sql.DB, billsStartID, contractsStartID int) ([]string, int, error) { - var contracts []string - var contractResources []string - var billingReports []string - - for i := uint64(1); i <= nodeContractCount; i++ { - nodeID, err := rnd(1, nodeCount) - if err != nil { - return nil, contractsStartID, fmt.Errorf("failed to generate random node id: %w", err) - } - state := deleted - - if nodeUP[nodeID] { - if flip(contractCreatedRatio) { - state = created - } else if flip(0.5) { - state = gracePeriod - } - } - - if state != deleted && (minContractHRU > nodesHRU[nodeID] || minContractMRU > nodesMRU[nodeID] || minContractSRU > nodesSRU[nodeID]) { - i-- - continue - } - - twinID, err := rnd(1100, 3100) - if err != nil { - return nil, contractsStartID, fmt.Errorf("failed to generate random twin id: %w", err) - } - - if renter, ok := renter[nodeID]; ok { - twinID = renter - } - - if _, ok := availableRentNodes[nodeID]; ok { - i-- - continue - } - - contract := node_contract{ - id: fmt.Sprintf("node-contract-%d", contractsStartID), - twin_id: twinID, - contract_id: uint64(contractsStartID), - state: state, - created_at: uint64(time.Now().Unix()), - node_id: nodeID, - deployment_data: fmt.Sprintf("deployment-data-%d", contractsStartID), - deployment_hash: fmt.Sprintf("deployment-hash-%d", contractsStartID), - number_of_public_i_ps: 0, - grid_version: 3, - resources_used_id: "", - } - - cru, err := rnd(minContractCRU, maxContractCRU) - if err != nil { - return nil, contractsStartID, fmt.Errorf("failed to generate random cru: %w", err) - } - - hru, err := rnd(minContractHRU, min(maxContractHRU, nodesHRU[nodeID])) - if err != nil { - return nil, contractsStartID, fmt.Errorf("failed to generate random hru: %w", err) - } - - sru, err := rnd(minContractSRU, min(maxContractSRU, nodesSRU[nodeID])) - if err != nil { - return nil, contractsStartID, fmt.Errorf("failed to generate random sru: %w", err) - } - - mru, err := rnd(minContractMRU, min(maxContractMRU, nodesMRU[nodeID])) - if err != nil { - return nil, contractsStartID, fmt.Errorf("failed to generate random mru: %w", err) - } - - contract_resources := contract_resources{ - id: fmt.Sprintf("contract-resources-%d", contractsStartID), - hru: hru, - sru: sru, - cru: cru, - mru: mru, - contract_id: fmt.Sprintf("node-contract-%d", contractsStartID), - } - if contract.state != deleted { - nodesHRU[nodeID] -= hru - nodesSRU[nodeID] -= sru - nodesMRU[nodeID] -= mru - createdNodeContracts = append(createdNodeContracts, uint64(contractsStartID)) - } - - contractTuple, err := objectToTupleString(contract) - if err != nil { - return nil, contractsStartID, fmt.Errorf("failed to convert contract object to tuple string: %w", err) - } - contracts = append(contracts, contractTuple) - - contractResourcesTuple, err := objectToTupleString(contract_resources) - if err != nil { - return nil, contractsStartID, fmt.Errorf("failed to convert contract resources object to tuple string: %w", err) - } - contractResources = append(contractResources, contractResourcesTuple) - - billings, err := rnd(0, 10) - if err != nil { - return nil, contractsStartID, fmt.Errorf("failed to generate random billing count: %w", err) - } - - amountBilled, err := rnd(0, 100000) - if err != nil { - return nil, contractsStartID, fmt.Errorf("failed to generate random amount billed: %w", err) - } - for j := uint64(0); j < billings; j++ { - billing := contract_bill_report{ - id: fmt.Sprintf("contract-bill-report-%d", billsStartID), - contract_id: uint64(contractsStartID), - discount_received: "Default", - amount_billed: amountBilled, - timestamp: uint64(time.Now().UnixNano()), - } - billsStartID++ - - billTuple, err := objectToTupleString(billing) - if err != nil { - return nil, contractsStartID, fmt.Errorf("failed to convert contract bill report object to tuple string: %w", err) - } - billingReports = append(billingReports, billTuple) - } - contractsStartID++ - } - - if err := insertTuples(db, node_contract{}, contracts); err != nil { - return nil, contractsStartID, fmt.Errorf("failed to insert node contracts: %w", err) - } - - if err := insertTuples(db, contract_resources{}, contractResources); err != nil { - return nil, contractsStartID, fmt.Errorf("failed to insert contract resources: %w", err) - } - - if err := updateNodeContractResourceID(db, contractsStartID-nodeContractCount, contractsStartID); err != nil { - return nil, contractsStartID, fmt.Errorf("failed to update node contract resources id: %w", err) - } - - fmt.Println("node contracts generated") - - return billingReports, contractsStartID, nil -} - -func generateNameContracts(db *sql.DB, billsStartID, contractsStartID int) ([]string, int, error) { - var contracts []string - var billReports []string - for i := uint64(1); i <= nameContractCount; i++ { - nodeID, err := rnd(1, nodeCount) - if err != nil { - return nil, contractsStartID, fmt.Errorf("failed to generate random node id: %w", err) - } - - state := deleted - if nodeUP[nodeID] { - if flip(contractCreatedRatio) { - state = created - } else if flip(0.5) { - state = gracePeriod - } - } - - twinID, err := rnd(1100, 3100) - if err != nil { - return nil, contractsStartID, fmt.Errorf("failed to generate random twin id: %w", err) - } - - if renter, ok := renter[nodeID]; ok { - twinID = renter - } - - if _, ok := availableRentNodes[nodeID]; ok { - i-- - continue - } - - contract := name_contract{ - id: fmt.Sprintf("name-contract-%d", contractsStartID), - twin_id: twinID, - contract_id: uint64(contractsStartID), - state: state, - created_at: uint64(time.Now().Unix()), - grid_version: 3, - name: uuid.NewString(), - } - - contractTuple, err := objectToTupleString(contract) - if err != nil { - return nil, contractsStartID, fmt.Errorf("failed to convert contract object to tuple string: %w", err) - } - contracts = append(contracts, contractTuple) - - billings, err := rnd(0, 10) - if err != nil { - return nil, contractsStartID, fmt.Errorf("failed to generate random billings count: %w", err) - } - amountBilled, err := rnd(0, 100000) - if err != nil { - return nil, contractsStartID, fmt.Errorf("failed to generate random amount billed: %w", err) - } - - for j := uint64(0); j < billings; j++ { - billing := contract_bill_report{ - id: fmt.Sprintf("contract-bill-report-%d", billsStartID), - contract_id: uint64(contractsStartID), - discount_received: "Default", - amount_billed: amountBilled, - timestamp: uint64(time.Now().UnixNano()), - } - billsStartID++ - - billTuple, err := objectToTupleString(billing) - if err != nil { - return nil, contractsStartID, fmt.Errorf("failed to convert contract bill report object to tuple string: %w", err) - } - billReports = append(billReports, billTuple) - } - contractsStartID++ - } - - if err := insertTuples(db, name_contract{}, contracts); err != nil { - return nil, contractsStartID, fmt.Errorf("failed to insert name contracts: %w", err) - } - - fmt.Println("name contracts generated") - - return billReports, contractsStartID, nil -} -func generateRentContracts(db *sql.DB, billsStartID, contractsStartID int) ([]string, int, error) { - var contracts []string - var billReports []string - for i := uint64(1); i <= rentContractCount; i++ { - nl, nodeID, err := popRandom(availableRentNodesList) - if err != nil { - return nil, contractsStartID, fmt.Errorf("failed to select random element from the gives slice: %w", err) - } - - availableRentNodesList = nl - delete(availableRentNodes, nodeID) - state := deleted - if nodeUP[nodeID] { - if flip(0.9) { - state = created - } else if flip(0.5) { - state = gracePeriod - } - } - - twinID, err := rnd(1100, 3100) - if err != nil { - return nil, contractsStartID, fmt.Errorf("failed to generate a random twin id: %w", err) - } - - contract := rent_contract{ - id: fmt.Sprintf("rent-contract-%d", contractsStartID), - twin_id: twinID, - contract_id: uint64(contractsStartID), - state: state, - created_at: uint64(time.Now().Unix()), - node_id: nodeID, - grid_version: 3, - } - - if state != deleted { - renter[nodeID] = contract.twin_id - } - - contractTuple, err := objectToTupleString(contract) - if err != nil { - return nil, contractsStartID, fmt.Errorf("failed to convert contract object to tuple string: %w", err) - } - contracts = append(contracts, contractTuple) - - billings, err := rnd(0, 10) - if err != nil { - return nil, contractsStartID, fmt.Errorf("failed to generate billings count: %w", err) - } - - amountBilled, err := rnd(0, 100000) - if err != nil { - return nil, contractsStartID, fmt.Errorf("failed to generate amount billed: %w", err) - } - - for j := uint64(0); j < billings; j++ { - billing := contract_bill_report{ - id: fmt.Sprintf("contract-bill-report-%d", billsStartID), - contract_id: uint64(contractsStartID), - discount_received: "Default", - amount_billed: amountBilled, - timestamp: uint64(time.Now().UnixNano()), - } - - billsStartID++ - - billTuple, err := objectToTupleString(billing) - if err != nil { - return nil, contractsStartID, fmt.Errorf("failed to convert contract bill report object to tuple string: %w", err) - } - billReports = append(billReports, billTuple) - - } - contractsStartID++ - } - - if err := insertTuples(db, rent_contract{}, contracts); err != nil { - return nil, contractsStartID, fmt.Errorf("failed to insert rent contracts: %w", err) - } - - fmt.Println("rent contracts generated") - - return billReports, contractsStartID, nil -} - -func generateNodes(db *sql.DB) error { - powerState := []string{"Up", "Down"} - var locations []string - var nodes []string - var totalResources []string - var publicConfigs []string - for i := uint64(1); i <= nodeCount; i++ { - mru, err := rnd(4, 256) - if err != nil { - return fmt.Errorf("failed to generate random mru: %w", err) - } - mru *= 1024 * 1024 * 1024 - - hru, err := rnd(100, 30*1024) - if err != nil { - return fmt.Errorf("failed to generate random hru: %w", err) - } - hru *= 1024 * 1024 * 1024 // 100GB -> 30TB - - sru, err := rnd(200, 30*1024) - if err != nil { - return fmt.Errorf("failed to generate random sru: %w", err) - } - sru *= 1024 * 1024 * 1024 // 100GB -> 30TB - - cru, err := rnd(4, 128) - if err != nil { - return fmt.Errorf("failed to generate random cru: %w", err) - } - - up := flip(nodeUpRatio) - periodFromLatestUpdate, err := rnd(60*40*3, 60*60*24*30*12) - if err != nil { - return fmt.Errorf("failed to generate random period from latest update: %w", err) - } - updatedAt := time.Now().Unix() - int64(periodFromLatestUpdate) - - if up { - periodFromLatestUpdate, err = rnd(0, 60*40*1) - if err != nil { - return fmt.Errorf("failed to generate period from latest update: %w", err) - } - updatedAt = time.Now().Unix() - int64(periodFromLatestUpdate) - } - - nodesMRU[i] = mru - max(2*uint64(gridtypes.Gigabyte), mru/10) - nodesSRU[i] = sru - 100*uint64(gridtypes.Gigabyte) - nodesHRU[i] = hru - nodeUP[i] = up - - // location latitude and longitue needs to be castable to decimal - // if not, the convert_to_decimal function will raise a notice - // reporting the incident, which downgrades performance - location := location{ - id: fmt.Sprintf("location-%d", i), - longitude: fmt.Sprintf("%d", i), - latitude: fmt.Sprintf("%d", i), - } - - countryIndex := r.Intn(len(countries)) - cityIndex := r.Intn(len(cities[countries[countryIndex]])) - node := node{ - id: fmt.Sprintf("node-%d", i), - location_id: fmt.Sprintf("location-%d", i), - node_id: i, - farm_id: i%100 + 1, - twin_id: i + 100 + 1, - country: countries[countryIndex], - city: cities[countries[countryIndex]][cityIndex], - uptime: 1000, - updated_at: uint64(updatedAt), - created: uint64(time.Now().Unix()), - created_at: uint64(time.Now().Unix()), - farming_policy_id: 1, - grid_version: 3, - certification: "Diy", - secure: false, - virtualized: false, - serial_number: "", - power: nodePower{ - State: powerState[r.Intn(len(powerState))], - Target: powerState[r.Intn(len(powerState))], - }, - extra_fee: 0, - } - - total_resources := node_resources_total{ - id: fmt.Sprintf("total-resources-%d", i), - hru: hru, - sru: sru, - cru: cru, - mru: mru, - node_id: fmt.Sprintf("node-%d", i), - } - - if _, ok := dedicatedFarms[node.farm_id]; ok { - availableRentNodes[i] = struct{}{} - availableRentNodesList = append(availableRentNodesList, i) - } - - locationTuple, err := objectToTupleString(location) - if err != nil { - return fmt.Errorf("failed to convert location object to tuple string: %w", err) - } - locations = append(locations, locationTuple) - - nodeTuple, err := objectToTupleString(node) - if err != nil { - return fmt.Errorf("failed to convert node object to tuple string: %w", err) - } - nodes = append(nodes, nodeTuple) - - totalResourcesTuple, err := objectToTupleString(total_resources) - if err != nil { - return fmt.Errorf("failed to convert total resources object to tuple string: %w", err) - } - totalResources = append(totalResources, totalResourcesTuple) - - if flip(.1) { - publicConfig := public_config{ - id: fmt.Sprintf("public-config-%d", i), - ipv4: "185.16.5.2/24", - gw4: "185.16.5.2", - ipv6: "::1/64", - gw6: "::1", - domain: "hamada.com", - node_id: fmt.Sprintf("node-%d", i), - } - publicConfigTuple, err := objectToTupleString(publicConfig) - if err != nil { - return fmt.Errorf("failed to convert public config object to tuple string: %w", err) - } - publicConfigs = append(publicConfigs, publicConfigTuple) - - } - } - - if err := insertTuples(db, location{}, locations); err != nil { - return fmt.Errorf("failed to insert locations: %w", err) - } - - if err := insertTuples(db, node{}, nodes); err != nil { - return fmt.Errorf("failed to isnert nodes: %w", err) - } - - if err := insertTuples(db, node_resources_total{}, totalResources); err != nil { - return fmt.Errorf("failed to insert node resources total: %w", err) - } - - if err := insertTuples(db, public_config{}, publicConfigs); err != nil { - return fmt.Errorf("failed to insert public configs: %w", err) - } - fmt.Println("nodes generated") - - return nil -} - -func generateNodeGPUs(db *sql.DB) error { - var GPUs []string - vendors := []string{"NVIDIA Corporation", "AMD", "Intel Corporation"} - devices := []string{"GeForce RTX 3080", "Radeon RX 6800 XT", "Intel Iris Xe MAX"} - - for i := 0; i <= 10; i++ { - gpuNum := len(vendors) - 1 - for j := 0; j <= gpuNum; j++ { - g := node_gpu{ - node_twin_id: uint64(i + 100 + 2), // node twin ids start from 102 - vendor: vendors[j], - device: devices[j], - contract: i % 2, - id: fmt.Sprintf("0000:0e:00.0/1002/744c/%d", j), - } - gpuTuple, err := objectToTupleString(g) - if err != nil { - return fmt.Errorf("failed to convert gpu object to tuple string: %w", err) - } - GPUs = append(GPUs, gpuTuple) - } - } - - if err := insertTuples(db, node_gpu{}, GPUs); err != nil { - return fmt.Errorf("failed to insert node gpu: %w", err) - } - - fmt.Println("node GPUs generated") - - return nil -} - -func generateContracts(db *sql.DB) error { - rentContractIDStart := 1 - - var billReports []string - - rentContractsBillReports, nodeContractIDStart, err := generateRentContracts(db, 1, rentContractIDStart) - if err != nil { - return fmt.Errorf("failed to generate rent contracts: %w", err) - } - billReports = append(billReports, rentContractsBillReports...) - - nodeContractsBillReports, nameContractIDStart, err := generateNodeContracts(db, len(billReports)+1, nodeContractIDStart) - if err != nil { - return fmt.Errorf("failed to generate node contracts: %w", err) - } - billReports = append(billReports, nodeContractsBillReports...) - - nameContractsBillReports, _, err := generateNameContracts(db, len(billReports)+1, nameContractIDStart) - if err != nil { - return fmt.Errorf("failed to generate name contracts: %w", err) - } - billReports = append(billReports, nameContractsBillReports...) - - if err := insertTuples(db, contract_bill_report{}, billReports); err != nil { - return fmt.Errorf("failed to generate contract bill reports: %w", err) - } - return nil -} - -func insertTuples(db *sql.DB, tupleObj interface{}, tuples []string) error { - - if len(tuples) != 0 { - query := "INSERT INTO " + reflect.Indirect(reflect.ValueOf(tupleObj)).Type().Name() + " (" - objType := reflect.TypeOf(tupleObj) - for i := 0; i < objType.NumField(); i++ { - if i != 0 { - query += ", " - } - query += objType.Field(i).Name - } - - query += ") VALUES " - - query += strings.Join(tuples, ",") - query += ";" - if _, err := db.Exec(query); err != nil { - return fmt.Errorf("failed to insert tuples: %w", err) - } - - } - return nil -} - -func updateNodeContractPublicIPs(db *sql.DB, nodeContracts []uint64) error { - - if len(nodeContracts) != 0 { - var IDs []string - for _, contractID := range nodeContracts { - IDs = append(IDs, fmt.Sprintf("%d", contractID)) - - } - - query := "UPDATE node_contract set number_of_public_i_ps = number_of_public_i_ps + 1 WHERE contract_id IN (" - query += strings.Join(IDs, ",") + ");" - if _, err := db.Exec(query); err != nil { - return fmt.Errorf("failed to update node contracts public ips: %w", err) - } - } - return nil -} - -func updateNodeContractResourceID(db *sql.DB, min, max int) error { - query := fmt.Sprintf(`UPDATE node_contract SET resources_used_id = CONCAT('contract-resources-',split_part(id, '-', -1)) - WHERE CAST(split_part(id, '-', -1) AS INTEGER) BETWEEN %d AND %d;`, min, max) - if _, err := db.Exec(query); err != nil { - return fmt.Errorf("failed to update node contract resource id: %w", err) - } - return nil -} -func generateData(db *sql.DB) error { - if err := generateTwins(db); err != nil { +// func generatePublicIPs(db *sql.DB) error { +// var publicIPs []string +// var nodeContracts []uint64 + +// for i := uint64(1); i <= publicIPCount; i++ { +// contract_id := uint64(0) +// if flip(usedPublicIPsRatio) { +// idx, err := rnd(0, uint64(len(createdNodeContracts))-1) +// if err != nil { +// return fmt.Errorf("failed to generate random index: %w", err) +// } +// contract_id = createdNodeContracts[idx] +// } +// ip := randomIPv4() +// farmID, err := rnd(1, farmCount) +// if err != nil { +// return fmt.Errorf("failed to generate random farm id: %w", err) +// } + +// public_ip := public_ip{ +// id: fmt.Sprintf("public-ip-%d", i), +// gateway: ip.String(), +// ip: IPv4Subnet(ip).String(), +// contract_id: contract_id, +// farm_id: fmt.Sprintf("farm-%d", farmID), +// } +// publicIpTuple, err := objectToTupleString(public_ip) +// if err != nil { +// return fmt.Errorf("failed to convert public ip object to tuple string: %w", err) +// } +// publicIPs = append(publicIPs, publicIpTuple) +// nodeContracts = append(nodeContracts, contract_id) +// } + +// if err := insertTuples(db, public_ip{}, publicIPs); err != nil { +// return fmt.Errorf("failed to insert public ips: %w", err) +// } + +// if err := updateNodeContractPublicIPs(db, nodeContracts); err != nil { +// return fmt.Errorf("failed to update contract public ips: %w", err) +// } + +// fmt.Println("public IPs generated") + +// return nil +// } + +// func generateFarms(db *sql.DB) error { +// var farms []string + +// for i := uint64(1); i <= farmCount; i++ { +// farm := farm{ +// id: fmt.Sprintf("farm-%d", i), +// farm_id: i, +// name: fmt.Sprintf("farm-name-%d", i), +// certification: "Diy", +// dedicated_farm: flip(.1), +// twin_id: i, +// pricing_policy_id: 1, +// grid_version: 3, +// stellar_address: "", +// } + +// if farm.dedicated_farm { +// dedicatedFarms[farm.farm_id] = struct{}{} +// } + +// farmTuple, err := objectToTupleString(farm) +// if err != nil { +// return fmt.Errorf("failed to convert farm object to tuple string: %w", err) +// } +// farms = append(farms, farmTuple) +// } + +// if err := insertTuples(db, farm{}, farms); err != nil { +// return fmt.Errorf("failed to insert farms: %w", err) +// } +// fmt.Println("farms generated") + +// return nil +// } + +// func generateCountries(db *sql.DB) error { +// var countriesValues []string +// index := 0 +// for countryName, region := range regions { +// index++ +// country := country{ +// id: fmt.Sprintf("country-%d", index), +// country_id: uint64(index), +// name: countryName, +// code: countriesCodes[countryName], +// region: "unknown", +// subregion: region, +// lat: fmt.Sprintf("%d", 0), +// long: fmt.Sprintf("%d", 0), +// } + +// countryTuple, err := objectToTupleString(country) +// if err != nil { +// return fmt.Errorf("failed to convert country object to tuple string: %w", err) +// } +// countriesValues = append(countriesValues, countryTuple) +// } + +// if err := insertTuples(db, country{}, countriesValues); err != nil { +// return fmt.Errorf("failed to insert country: %w", err) +// } +// fmt.Println("countries generated") + +// return nil +// } + +// func generateNodeContracts(db *sql.DB, billsStartID, contractsStartID int) ([]string, int, error) { +// var contracts []string +// var contractResources []string +// var billingReports []string + +// for i := uint64(1); i <= nodeContractCount; i++ { +// nodeID, err := rnd(1, nodeCount) +// if err != nil { +// return nil, contractsStartID, fmt.Errorf("failed to generate random node id: %w", err) +// } +// state := deleted + +// if nodeUP[nodeID] { +// if flip(contractCreatedRatio) { +// state = created +// } else if flip(0.5) { +// state = gracePeriod +// } +// } + +// if state != deleted && (minContractHRU > nodesHRU[nodeID] || minContractMRU > nodesMRU[nodeID] || minContractSRU > nodesSRU[nodeID]) { +// i-- +// continue +// } + +// twinID, err := rnd(1100, 3100) +// if err != nil { +// return nil, contractsStartID, fmt.Errorf("failed to generate random twin id: %w", err) +// } + +// if renter, ok := renter[nodeID]; ok { +// twinID = renter +// } + +// if _, ok := availableRentNodes[nodeID]; ok { +// i-- +// continue +// } + +// contract := node_contract{ +// id: fmt.Sprintf("node-contract-%d", contractsStartID), +// twin_id: twinID, +// contract_id: uint64(contractsStartID), +// state: state, +// created_at: uint64(time.Now().Unix()), +// node_id: nodeID, +// deployment_data: fmt.Sprintf("deployment-data-%d", contractsStartID), +// deployment_hash: fmt.Sprintf("deployment-hash-%d", contractsStartID), +// number_of_public_i_ps: 0, +// grid_version: 3, +// resources_used_id: "", +// } + +// cru, err := rnd(minContractCRU, maxContractCRU) +// if err != nil { +// return nil, contractsStartID, fmt.Errorf("failed to generate random cru: %w", err) +// } + +// hru, err := rnd(minContractHRU, min(maxContractHRU, nodesHRU[nodeID])) +// if err != nil { +// return nil, contractsStartID, fmt.Errorf("failed to generate random hru: %w", err) +// } + +// sru, err := rnd(minContractSRU, min(maxContractSRU, nodesSRU[nodeID])) +// if err != nil { +// return nil, contractsStartID, fmt.Errorf("failed to generate random sru: %w", err) +// } + +// mru, err := rnd(minContractMRU, min(maxContractMRU, nodesMRU[nodeID])) +// if err != nil { +// return nil, contractsStartID, fmt.Errorf("failed to generate random mru: %w", err) +// } + +// contract_resources := contract_resources{ +// id: fmt.Sprintf("contract-resources-%d", contractsStartID), +// hru: hru, +// sru: sru, +// cru: cru, +// mru: mru, +// contract_id: fmt.Sprintf("node-contract-%d", contractsStartID), +// } +// if contract.state != deleted { +// nodesHRU[nodeID] -= hru +// nodesSRU[nodeID] -= sru +// nodesMRU[nodeID] -= mru +// createdNodeContracts = append(createdNodeContracts, uint64(contractsStartID)) +// } + +// contractTuple, err := objectToTupleString(contract) +// if err != nil { +// return nil, contractsStartID, fmt.Errorf("failed to convert contract object to tuple string: %w", err) +// } +// contracts = append(contracts, contractTuple) + +// contractResourcesTuple, err := objectToTupleString(contract_resources) +// if err != nil { +// return nil, contractsStartID, fmt.Errorf("failed to convert contract resources object to tuple string: %w", err) +// } +// contractResources = append(contractResources, contractResourcesTuple) + +// billings, err := rnd(0, 10) +// if err != nil { +// return nil, contractsStartID, fmt.Errorf("failed to generate random billing count: %w", err) +// } + +// amountBilled, err := rnd(0, 100000) +// if err != nil { +// return nil, contractsStartID, fmt.Errorf("failed to generate random amount billed: %w", err) +// } +// for j := uint64(0); j < billings; j++ { +// billing := contract_bill_report{ +// id: fmt.Sprintf("contract-bill-report-%d", billsStartID), +// contract_id: uint64(contractsStartID), +// discount_received: "Default", +// amount_billed: amountBilled, +// timestamp: uint64(time.Now().UnixNano()), +// } +// billsStartID++ + +// billTuple, err := objectToTupleString(billing) +// if err != nil { +// return nil, contractsStartID, fmt.Errorf("failed to convert contract bill report object to tuple string: %w", err) +// } +// billingReports = append(billingReports, billTuple) +// } +// contractsStartID++ +// } + +// if err := insertTuples(db, node_contract{}, contracts); err != nil { +// return nil, contractsStartID, fmt.Errorf("failed to insert node contracts: %w", err) +// } + +// if err := insertTuples(db, contract_resources{}, contractResources); err != nil { +// return nil, contractsStartID, fmt.Errorf("failed to insert contract resources: %w", err) +// } + +// if err := updateNodeContractResourceID(db, contractsStartID-nodeContractCount, contractsStartID); err != nil { +// return nil, contractsStartID, fmt.Errorf("failed to update node contract resources id: %w", err) +// } + +// fmt.Println("node contracts generated") + +// return billingReports, contractsStartID, nil +// } + +// func generateNameContracts(db *sql.DB, billsStartID, contractsStartID int) ([]string, int, error) { +// var contracts []string +// var billReports []string +// for i := uint64(1); i <= nameContractCount; i++ { +// nodeID, err := rnd(1, nodeCount) +// if err != nil { +// return nil, contractsStartID, fmt.Errorf("failed to generate random node id: %w", err) +// } + +// state := deleted +// if nodeUP[nodeID] { +// if flip(contractCreatedRatio) { +// state = created +// } else if flip(0.5) { +// state = gracePeriod +// } +// } + +// twinID, err := rnd(1100, 3100) +// if err != nil { +// return nil, contractsStartID, fmt.Errorf("failed to generate random twin id: %w", err) +// } + +// if renter, ok := renter[nodeID]; ok { +// twinID = renter +// } + +// if _, ok := availableRentNodes[nodeID]; ok { +// i-- +// continue +// } + +// contract := name_contract{ +// id: fmt.Sprintf("name-contract-%d", contractsStartID), +// twin_id: twinID, +// contract_id: uint64(contractsStartID), +// state: state, +// created_at: uint64(time.Now().Unix()), +// grid_version: 3, +// name: uuid.NewString(), +// } + +// contractTuple, err := objectToTupleString(contract) +// if err != nil { +// return nil, contractsStartID, fmt.Errorf("failed to convert contract object to tuple string: %w", err) +// } +// contracts = append(contracts, contractTuple) + +// billings, err := rnd(0, 10) +// if err != nil { +// return nil, contractsStartID, fmt.Errorf("failed to generate random billings count: %w", err) +// } +// amountBilled, err := rnd(0, 100000) +// if err != nil { +// return nil, contractsStartID, fmt.Errorf("failed to generate random amount billed: %w", err) +// } + +// for j := uint64(0); j < billings; j++ { +// billing := contract_bill_report{ +// id: fmt.Sprintf("contract-bill-report-%d", billsStartID), +// contract_id: uint64(contractsStartID), +// discount_received: "Default", +// amount_billed: amountBilled, +// timestamp: uint64(time.Now().UnixNano()), +// } +// billsStartID++ + +// billTuple, err := objectToTupleString(billing) +// if err != nil { +// return nil, contractsStartID, fmt.Errorf("failed to convert contract bill report object to tuple string: %w", err) +// } +// billReports = append(billReports, billTuple) +// } +// contractsStartID++ +// } + +// if err := insertTuples(db, name_contract{}, contracts); err != nil { +// return nil, contractsStartID, fmt.Errorf("failed to insert name contracts: %w", err) +// } + +// fmt.Println("name contracts generated") + +// return billReports, contractsStartID, nil +// } +// func generateRentContracts(db *sql.DB, billsStartID, contractsStartID int) ([]string, int, error) { +// var contracts []string +// var billReports []string +// for i := uint64(1); i <= rentContractCount; i++ { +// nl, nodeID, err := popRandom(availableRentNodesList) +// if err != nil { +// return nil, contractsStartID, fmt.Errorf("failed to select random element from the gives slice: %w", err) +// } + +// availableRentNodesList = nl +// delete(availableRentNodes, nodeID) +// state := deleted +// if nodeUP[nodeID] { +// if flip(0.9) { +// state = created +// } else if flip(0.5) { +// state = gracePeriod +// } +// } + +// twinID, err := rnd(1100, 3100) +// if err != nil { +// return nil, contractsStartID, fmt.Errorf("failed to generate a random twin id: %w", err) +// } + +// contract := rent_contract{ +// id: fmt.Sprintf("rent-contract-%d", contractsStartID), +// twin_id: twinID, +// contract_id: uint64(contractsStartID), +// state: state, +// created_at: uint64(time.Now().Unix()), +// node_id: nodeID, +// grid_version: 3, +// } + +// if state != deleted { +// renter[nodeID] = contract.twin_id +// } + +// contractTuple, err := objectToTupleString(contract) +// if err != nil { +// return nil, contractsStartID, fmt.Errorf("failed to convert contract object to tuple string: %w", err) +// } +// contracts = append(contracts, contractTuple) + +// billings, err := rnd(0, 10) +// if err != nil { +// return nil, contractsStartID, fmt.Errorf("failed to generate billings count: %w", err) +// } + +// amountBilled, err := rnd(0, 100000) +// if err != nil { +// return nil, contractsStartID, fmt.Errorf("failed to generate amount billed: %w", err) +// } + +// for j := uint64(0); j < billings; j++ { +// billing := contract_bill_report{ +// id: fmt.Sprintf("contract-bill-report-%d", billsStartID), +// contract_id: uint64(contractsStartID), +// discount_received: "Default", +// amount_billed: amountBilled, +// timestamp: uint64(time.Now().UnixNano()), +// } + +// billsStartID++ + +// billTuple, err := objectToTupleString(billing) +// if err != nil { +// return nil, contractsStartID, fmt.Errorf("failed to convert contract bill report object to tuple string: %w", err) +// } +// billReports = append(billReports, billTuple) + +// } +// contractsStartID++ +// } + +// if err := insertTuples(db, rent_contract{}, contracts); err != nil { +// return nil, contractsStartID, fmt.Errorf("failed to insert rent contracts: %w", err) +// } + +// fmt.Println("rent contracts generated") + +// return billReports, contractsStartID, nil +// } + +// func generateNodes(db *sql.DB) error { +// powerState := []string{"Up", "Down"} +// var locations []string +// var nodes []string +// var totalResources []string +// var publicConfigs []string +// for i := uint64(1); i <= nodeCount; i++ { +// mru, err := rnd(4, 256) +// if err != nil { +// return fmt.Errorf("failed to generate random mru: %w", err) +// } +// mru *= 1024 * 1024 * 1024 + +// hru, err := rnd(100, 30*1024) +// if err != nil { +// return fmt.Errorf("failed to generate random hru: %w", err) +// } +// hru *= 1024 * 1024 * 1024 // 100GB -> 30TB + +// sru, err := rnd(200, 30*1024) +// if err != nil { +// return fmt.Errorf("failed to generate random sru: %w", err) +// } +// sru *= 1024 * 1024 * 1024 // 100GB -> 30TB + +// cru, err := rnd(4, 128) +// if err != nil { +// return fmt.Errorf("failed to generate random cru: %w", err) +// } + +// up := flip(nodeUpRatio) +// periodFromLatestUpdate, err := rnd(60*40*3, 60*60*24*30*12) +// if err != nil { +// return fmt.Errorf("failed to generate random period from latest update: %w", err) +// } +// updatedAt := time.Now().Unix() - int64(periodFromLatestUpdate) + +// if up { +// periodFromLatestUpdate, err = rnd(0, 60*40*1) +// if err != nil { +// return fmt.Errorf("failed to generate period from latest update: %w", err) +// } +// updatedAt = time.Now().Unix() - int64(periodFromLatestUpdate) +// } + +// nodesMRU[i] = mru - max(2*uint64(gridtypes.Gigabyte), mru/10) +// nodesSRU[i] = sru - 100*uint64(gridtypes.Gigabyte) +// nodesHRU[i] = hru +// nodeUP[i] = up + +// // location latitude and longitue needs to be castable to decimal +// // if not, the convert_to_decimal function will raise a notice +// // reporting the incident, which downgrades performance +// location := location{ +// id: fmt.Sprintf("location-%d", i), +// longitude: fmt.Sprintf("%d", i), +// latitude: fmt.Sprintf("%d", i), +// } + +// countryIndex := r.Intn(len(countries)) +// cityIndex := r.Intn(len(cities[countries[countryIndex]])) +// node := node{ +// id: fmt.Sprintf("node-%d", i), +// location_id: fmt.Sprintf("location-%d", i), +// node_id: i, +// farm_id: i%100 + 1, +// twin_id: i + 100 + 1, +// country: countries[countryIndex], +// city: cities[countries[countryIndex]][cityIndex], +// uptime: 1000, +// updated_at: uint64(updatedAt), +// created: uint64(time.Now().Unix()), +// created_at: uint64(time.Now().Unix()), +// farming_policy_id: 1, +// grid_version: 3, +// certification: "Diy", +// secure: false, +// virtualized: false, +// serial_number: "", +// power: nodePower{ +// State: powerState[r.Intn(len(powerState))], +// Target: powerState[r.Intn(len(powerState))], +// }, +// extra_fee: 0, +// } + +// total_resources := node_resources_total{ +// id: fmt.Sprintf("total-resources-%d", i), +// hru: hru, +// sru: sru, +// cru: cru, +// mru: mru, +// node_id: fmt.Sprintf("node-%d", i), +// } + +// if _, ok := dedicatedFarms[node.farm_id]; ok { +// availableRentNodes[i] = struct{}{} +// availableRentNodesList = append(availableRentNodesList, i) +// } + +// locationTuple, err := objectToTupleString(location) +// if err != nil { +// return fmt.Errorf("failed to convert location object to tuple string: %w", err) +// } +// locations = append(locations, locationTuple) + +// nodeTuple, err := objectToTupleString(node) +// if err != nil { +// return fmt.Errorf("failed to convert node object to tuple string: %w", err) +// } +// nodes = append(nodes, nodeTuple) + +// totalResourcesTuple, err := objectToTupleString(total_resources) +// if err != nil { +// return fmt.Errorf("failed to convert total resources object to tuple string: %w", err) +// } +// totalResources = append(totalResources, totalResourcesTuple) + +// if flip(.1) { +// publicConfig := public_config{ +// id: fmt.Sprintf("public-config-%d", i), +// ipv4: "185.16.5.2/24", +// gw4: "185.16.5.2", +// ipv6: "::1/64", +// gw6: "::1", +// domain: "hamada.com", +// node_id: fmt.Sprintf("node-%d", i), +// } +// publicConfigTuple, err := objectToTupleString(publicConfig) +// if err != nil { +// return fmt.Errorf("failed to convert public config object to tuple string: %w", err) +// } +// publicConfigs = append(publicConfigs, publicConfigTuple) + +// } +// } + +// if err := insertTuples(db, location{}, locations); err != nil { +// return fmt.Errorf("failed to insert locations: %w", err) +// } + +// if err := insertTuples(db, node{}, nodes); err != nil { +// return fmt.Errorf("failed to isnert nodes: %w", err) +// } + +// if err := insertTuples(db, node_resources_total{}, totalResources); err != nil { +// return fmt.Errorf("failed to insert node resources total: %w", err) +// } + +// if err := insertTuples(db, public_config{}, publicConfigs); err != nil { +// return fmt.Errorf("failed to insert public configs: %w", err) +// } +// fmt.Println("nodes generated") + +// return nil +// } + +// func generateNodeGPUs(db *sql.DB) error { +// var GPUs []string +// vendors := []string{"NVIDIA Corporation", "AMD", "Intel Corporation"} +// devices := []string{"GeForce RTX 3080", "Radeon RX 6800 XT", "Intel Iris Xe MAX"} + +// for i := 0; i <= 10; i++ { +// gpuNum := len(vendors) - 1 +// for j := 0; j <= gpuNum; j++ { +// g := node_gpu{ +// node_twin_id: uint64(i + 100 + 2), // node twin ids start from 102 +// vendor: vendors[j], +// device: devices[j], +// contract: i % 2, +// id: fmt.Sprintf("0000:0e:00.0/1002/744c/%d", j), +// } +// gpuTuple, err := objectToTupleString(g) +// if err != nil { +// return fmt.Errorf("failed to convert gpu object to tuple string: %w", err) +// } +// GPUs = append(GPUs, gpuTuple) +// } +// } + +// if err := insertTuples(db, node_gpu{}, GPUs); err != nil { +// return fmt.Errorf("failed to insert node gpu: %w", err) +// } + +// fmt.Println("node GPUs generated") + +// return nil +// } + +// func generateContracts(db *sql.DB) error { +// rentContractIDStart := 1 + +// var billReports []string + +// rentContractsBillReports, nodeContractIDStart, err := generateRentContracts(db, 1, rentContractIDStart) +// if err != nil { +// return fmt.Errorf("failed to generate rent contracts: %w", err) +// } +// billReports = append(billReports, rentContractsBillReports...) + +// nodeContractsBillReports, nameContractIDStart, err := generateNodeContracts(db, len(billReports)+1, nodeContractIDStart) +// if err != nil { +// return fmt.Errorf("failed to generate node contracts: %w", err) +// } +// billReports = append(billReports, nodeContractsBillReports...) + +// nameContractsBillReports, _, err := generateNameContracts(db, len(billReports)+1, nameContractIDStart) +// if err != nil { +// return fmt.Errorf("failed to generate name contracts: %w", err) +// } +// billReports = append(billReports, nameContractsBillReports...) + +// if err := insertTuples(db, contract_bill_report{}, billReports); err != nil { +// return fmt.Errorf("failed to generate contract bill reports: %w", err) +// } +// return nil +// } + +// func insertTuples(db *sql.DB, tupleObj interface{}, tuples []string) error { + +// if len(tuples) != 0 { +// query := "INSERT INTO " + reflect.Indirect(reflect.ValueOf(tupleObj)).Type().Name() + " (" +// objType := reflect.TypeOf(tupleObj) +// for i := 0; i < objType.NumField(); i++ { +// if i != 0 { +// query += ", " +// } +// query += objType.Field(i).Name +// } + +// query += ") VALUES " + +// query += strings.Join(tuples, ",") +// query += ";" +// if _, err := db.Exec(query); err != nil { +// return fmt.Errorf("failed to insert tuples: %w", err) +// } + +// } +// return nil +// } + +// func updateNodeContractPublicIPs(db *sql.DB, nodeContracts []uint64) error { + +// if len(nodeContracts) != 0 { +// var IDs []string +// for _, contractID := range nodeContracts { +// IDs = append(IDs, fmt.Sprintf("%d", contractID)) + +// } + +// query := "UPDATE node_contract set number_of_public_i_ps = number_of_public_i_ps + 1 WHERE contract_id IN (" +// query += strings.Join(IDs, ",") + ");" +// if _, err := db.Exec(query); err != nil { +// return fmt.Errorf("failed to update node contracts public ips: %w", err) +// } +// } +// return nil +// } + +// func updateNodeContractResourceID(db *sql.DB, min, max int) error { +// query := fmt.Sprintf(`UPDATE node_contract SET resources_used_id = CONCAT('contract-resources-',split_part(id, '-', -1)) +// WHERE CAST(split_part(id, '-', -1) AS INTEGER) BETWEEN %d AND %d;`, min, max) +// if _, err := db.Exec(query); err != nil { +// return fmt.Errorf("failed to update node contract resource id: %w", err) +// } +// return nil +// } +func generateData(db *sql.DB, seed int) error { + generator := modifiers.NewGenerator(db, seed) + + if err := generator.GenerateTwins(); err != nil { return fmt.Errorf("failed to genrate twins: %w", err) } - if err := generateFarms(db); err != nil { + if err := generator.GenerateFarms(); err != nil { return fmt.Errorf("failed to generate farms: %w", err) } - if err := generateNodes(db); err != nil { + if err := generator.GenerateNodes(); err != nil { return fmt.Errorf("failed to generate nodes: %w", err) } - if err := generateContracts(db); err != nil { + if err := generator.GenerateContracts(); err != nil { return fmt.Errorf("failed to generate contracts: %w", err) } - if err := generatePublicIPs(db); err != nil { + if err := generator.GeneratePublicIPs(); err != nil { return fmt.Errorf("failed to generate public ips: %w", err) } - if err := generateNodeGPUs(db); err != nil { + if err := generator.GenerateNodeGPUs(); err != nil { return fmt.Errorf("failed to generate node gpus: %w", err) } - if err := generateCountries(db); err != nil { + if err := generator.GenerateCountries(); err != nil { return fmt.Errorf("failed to generate countries: %w", err) } return nil diff --git a/grid-proxy/tools/db/modifiers/generators.go b/grid-proxy/tools/db/modifiers/generators.go new file mode 100644 index 000000000..9124c698c --- /dev/null +++ b/grid-proxy/tools/db/modifiers/generators.go @@ -0,0 +1,714 @@ +package modifiers + +import ( + "fmt" + "strings" + "time" + + "github.com/google/uuid" + "github.com/threefoldtech/zos/pkg/gridtypes" +) + +const deleted = "Deleted" +const created = "Created" +const gracePeriod = "GracePeriod" + +func (g *Generator) GenerateTwins() error { + var twins []string + + for i := uint64(1); i <= uint64(g.TwinCount); i++ { + twin := twin{ + id: fmt.Sprintf("twin-%d", i), + account_id: fmt.Sprintf("account-id-%d", i), + relay: fmt.Sprintf("relay-%d", i), + public_key: fmt.Sprintf("public-key-%d", i), + twin_id: i, + grid_version: 3, + } + tuple, err := objectToTupleString(twin) + if err != nil { + return fmt.Errorf("failed to convert twin object to tuple string: %w", err) + } + twins = append(twins, tuple) + } + + if err := g.insertTuples(twin{}, twins); err != nil { + return fmt.Errorf("failed to insert twins: %w", err) + } + fmt.Println("twins generated") + + return nil +} + +func (g *Generator) GenerateFarms() error { + var farms []string + + for i := uint64(1); i <= uint64(g.FarmCount); i++ { + farm := farm{ + id: fmt.Sprintf("farm-%d", i), + farm_id: i, + name: fmt.Sprintf("farm-name-%d", i), + certification: "Diy", + dedicated_farm: flip(.1), + twin_id: i, + pricing_policy_id: 1, + grid_version: 3, + stellar_address: "", + } + + if farm.dedicated_farm { + g.dedicatedFarms[farm.farm_id] = struct{}{} + } + + farmTuple, err := objectToTupleString(farm) + if err != nil { + return fmt.Errorf("failed to convert farm object to tuple string: %w", err) + } + farms = append(farms, farmTuple) + } + + if err := g.insertTuples(farm{}, farms); err != nil { + return fmt.Errorf("failed to insert farms: %w", err) + } + fmt.Println("farms generated") + + return nil +} + +func (g *Generator) GenerateNodes() error { + powerState := []string{"Up", "Down"} + var locations []string + var nodes []string + var totalResources []string + var publicConfigs []string + for i := uint64(1); i <= uint64(g.NodeCount); i++ { + mru, err := rnd(4, 256) + if err != nil { + return fmt.Errorf("failed to generate random mru: %w", err) + } + mru *= 1024 * 1024 * 1024 + + hru, err := rnd(100, 30*1024) + if err != nil { + return fmt.Errorf("failed to generate random hru: %w", err) + } + hru *= 1024 * 1024 * 1024 // 100GB -> 30TB + + sru, err := rnd(200, 30*1024) + if err != nil { + return fmt.Errorf("failed to generate random sru: %w", err) + } + sru *= 1024 * 1024 * 1024 // 100GB -> 30TB + + cru, err := rnd(4, 128) + if err != nil { + return fmt.Errorf("failed to generate random cru: %w", err) + } + + up := flip(g.nodeUpRatio) + periodFromLatestUpdate, err := rnd(60*40*3, 60*60*24*30*12) + if err != nil { + return fmt.Errorf("failed to generate random period from latest update: %w", err) + } + updatedAt := time.Now().Unix() - int64(periodFromLatestUpdate) + + if up { + periodFromLatestUpdate, err = rnd(0, 60*40*1) + if err != nil { + return fmt.Errorf("failed to generate period from latest update: %w", err) + } + updatedAt = time.Now().Unix() - int64(periodFromLatestUpdate) + } + + g.nodesMRU[i] = mru - max(2*uint64(gridtypes.Gigabyte), mru/10) + g.nodesSRU[i] = sru - 100*uint64(gridtypes.Gigabyte) + g.nodesHRU[i] = hru + g.nodeUP[i] = up + + // location latitude and longitue needs to be castable to decimal + // if not, the convert_to_decimal function will raise a notice + // reporting the incident, which downgrades performance + location := location{ + id: fmt.Sprintf("location-%d", i), + longitude: fmt.Sprintf("%d", i), + latitude: fmt.Sprintf("%d", i), + } + + countryIndex := r.Intn(len(g.countries)) + cityIndex := r.Intn(len(g.cities[g.countries[countryIndex]])) + node := node{ + id: fmt.Sprintf("node-%d", i), + location_id: fmt.Sprintf("location-%d", i), + node_id: i, + farm_id: i%100 + 1, + twin_id: i + 100 + 1, + country: g.countries[countryIndex], + city: g.cities[g.countries[countryIndex]][cityIndex], + uptime: 1000, + updated_at: uint64(updatedAt), + created: uint64(time.Now().Unix()), + created_at: uint64(time.Now().Unix()), + farming_policy_id: 1, + grid_version: 3, + certification: "Diy", + secure: false, + virtualized: false, + serial_number: "", + power: nodePower{ + State: powerState[r.Intn(len(powerState))], + Target: powerState[r.Intn(len(powerState))], + }, + extra_fee: 0, + } + + total_resources := node_resources_total{ + id: fmt.Sprintf("total-resources-%d", i), + hru: hru, + sru: sru, + cru: cru, + mru: mru, + node_id: fmt.Sprintf("node-%d", i), + } + + if _, ok := g.dedicatedFarms[node.farm_id]; ok { + g.availableRentNodes[i] = struct{}{} + g.availableRentNodesList = append(g.availableRentNodesList, i) + } + + locationTuple, err := objectToTupleString(location) + if err != nil { + return fmt.Errorf("failed to convert location object to tuple string: %w", err) + } + locations = append(locations, locationTuple) + + nodeTuple, err := objectToTupleString(node) + if err != nil { + return fmt.Errorf("failed to convert node object to tuple string: %w", err) + } + nodes = append(nodes, nodeTuple) + + totalResourcesTuple, err := objectToTupleString(total_resources) + if err != nil { + return fmt.Errorf("failed to convert total resources object to tuple string: %w", err) + } + totalResources = append(totalResources, totalResourcesTuple) + + if flip(.1) { + publicConfig := public_config{ + id: fmt.Sprintf("public-config-%d", i), + ipv4: "185.16.5.2/24", + gw4: "185.16.5.2", + ipv6: "::1/64", + gw6: "::1", + domain: "hamada.com", + node_id: fmt.Sprintf("node-%d", i), + } + publicConfigTuple, err := objectToTupleString(publicConfig) + if err != nil { + return fmt.Errorf("failed to convert public config object to tuple string: %w", err) + } + publicConfigs = append(publicConfigs, publicConfigTuple) + + } + } + + if err := g.insertTuples(location{}, locations); err != nil { + return fmt.Errorf("failed to insert locations: %w", err) + } + + if err := g.insertTuples(node{}, nodes); err != nil { + return fmt.Errorf("failed to isnert nodes: %w", err) + } + + if err := g.insertTuples(node_resources_total{}, totalResources); err != nil { + return fmt.Errorf("failed to insert node resources total: %w", err) + } + + if err := g.insertTuples(public_config{}, publicConfigs); err != nil { + return fmt.Errorf("failed to insert public configs: %w", err) + } + fmt.Println("nodes generated") + + return nil +} + +func (g *Generator) GenerateContracts() error { + rentContractIDStart := 1 + + var billReports []string + + rentContractsBillReports, nodeContractIDStart, err := g.GenerateRentContracts(1, rentContractIDStart) + if err != nil { + return fmt.Errorf("failed to generate rent contracts: %w", err) + } + billReports = append(billReports, rentContractsBillReports...) + + nodeContractsBillReports, nameContractIDStart, err := g.generateNodeContracts(len(billReports)+1, nodeContractIDStart) + if err != nil { + return fmt.Errorf("failed to generate node contracts: %w", err) + } + billReports = append(billReports, nodeContractsBillReports...) + + nameContractsBillReports, _, err := g.GenerateNameContracts(len(billReports)+1, nameContractIDStart) + if err != nil { + return fmt.Errorf("failed to generate name contracts: %w", err) + } + billReports = append(billReports, nameContractsBillReports...) + + if err := g.insertTuples(contract_bill_report{}, billReports); err != nil { + return fmt.Errorf("failed to generate contract bill reports: %w", err) + } + return nil +} + +func (g *Generator) generateNodeContracts(billsStartID, contractsStartID int) ([]string, int, error) { + var contracts []string + var contractResources []string + var billingReports []string + + for i := uint64(1); i <= uint64(g.NodeContractCount); i++ { + nodeID, err := rnd(1, uint64(g.NodeCount)) + if err != nil { + return nil, contractsStartID, fmt.Errorf("failed to generate random node id: %w", err) + } + state := deleted + + if g.nodeUP[nodeID] { + if flip(g.contractCreatedRatio) { + state = created + } else if flip(0.5) { + state = gracePeriod + } + } + + if state != deleted && (g.minContractHRU > g.nodesHRU[nodeID] || g.minContractMRU > g.nodesMRU[nodeID] || g.minContractSRU > g.nodesSRU[nodeID]) { + i-- + continue + } + + twinID, err := rnd(1100, 3100) + if err != nil { + return nil, contractsStartID, fmt.Errorf("failed to generate random twin id: %w", err) + } + + if renter, ok := g.renter[nodeID]; ok { + twinID = renter + } + + if _, ok := g.availableRentNodes[nodeID]; ok { + i-- + continue + } + + contract := node_contract{ + id: fmt.Sprintf("node-contract-%d", contractsStartID), + twin_id: twinID, + contract_id: uint64(contractsStartID), + state: state, + created_at: uint64(time.Now().Unix()), + node_id: nodeID, + deployment_data: fmt.Sprintf("deployment-data-%d", contractsStartID), + deployment_hash: fmt.Sprintf("deployment-hash-%d", contractsStartID), + number_of_public_i_ps: 0, + grid_version: 3, + resources_used_id: "", + } + + cru, err := rnd(g.minContractCRU, g.maxContractCRU) + if err != nil { + return nil, contractsStartID, fmt.Errorf("failed to generate random cru: %w", err) + } + + hru, err := rnd(g.minContractHRU, min(g.maxContractHRU, g.nodesHRU[nodeID])) + if err != nil { + return nil, contractsStartID, fmt.Errorf("failed to generate random hru: %w", err) + } + + sru, err := rnd(g.minContractSRU, min(g.maxContractSRU, g.nodesSRU[nodeID])) + if err != nil { + return nil, contractsStartID, fmt.Errorf("failed to generate random sru: %w", err) + } + + mru, err := rnd(g.minContractMRU, min(g.maxContractMRU, g.nodesMRU[nodeID])) + if err != nil { + return nil, contractsStartID, fmt.Errorf("failed to generate random mru: %w", err) + } + + contract_resources := contract_resources{ + id: fmt.Sprintf("contract-resources-%d", contractsStartID), + hru: hru, + sru: sru, + cru: cru, + mru: mru, + contract_id: fmt.Sprintf("node-contract-%d", contractsStartID), + } + if contract.state != deleted { + g.nodesHRU[nodeID] -= hru + g.nodesSRU[nodeID] -= sru + g.nodesMRU[nodeID] -= mru + g.createdNodeContracts = append(g.createdNodeContracts, uint64(contractsStartID)) + } + + contractTuple, err := objectToTupleString(contract) + if err != nil { + return nil, contractsStartID, fmt.Errorf("failed to convert contract object to tuple string: %w", err) + } + contracts = append(contracts, contractTuple) + + contractResourcesTuple, err := objectToTupleString(contract_resources) + if err != nil { + return nil, contractsStartID, fmt.Errorf("failed to convert contract resources object to tuple string: %w", err) + } + contractResources = append(contractResources, contractResourcesTuple) + + billings, err := rnd(0, 10) + if err != nil { + return nil, contractsStartID, fmt.Errorf("failed to generate random billing count: %w", err) + } + + amountBilled, err := rnd(0, 100000) + if err != nil { + return nil, contractsStartID, fmt.Errorf("failed to generate random amount billed: %w", err) + } + for j := uint64(0); j < billings; j++ { + billing := contract_bill_report{ + id: fmt.Sprintf("contract-bill-report-%d", billsStartID), + contract_id: uint64(contractsStartID), + discount_received: "Default", + amount_billed: amountBilled, + timestamp: uint64(time.Now().UnixNano()), + } + billsStartID++ + + billTuple, err := objectToTupleString(billing) + if err != nil { + return nil, contractsStartID, fmt.Errorf("failed to convert contract bill report object to tuple string: %w", err) + } + billingReports = append(billingReports, billTuple) + } + contractsStartID++ + } + + if err := g.insertTuples(node_contract{}, contracts); err != nil { + return nil, contractsStartID, fmt.Errorf("failed to insert node contracts: %w", err) + } + + if err := g.insertTuples(contract_resources{}, contractResources); err != nil { + return nil, contractsStartID, fmt.Errorf("failed to insert contract resources: %w", err) + } + + if err := g.updateNodeContractResourceID(contractsStartID-int(g.NodeContractCount), contractsStartID); err != nil { + return nil, contractsStartID, fmt.Errorf("failed to update node contract resources id: %w", err) + } + + fmt.Println("node contracts generated") + + return billingReports, contractsStartID, nil +} + +func (g *Generator) updateNodeContractResourceID(min, max int) error { + query := fmt.Sprintf(`UPDATE node_contract SET resources_used_id = CONCAT('contract-resources-',split_part(id, '-', -1)) + WHERE CAST(split_part(id, '-', -1) AS INTEGER) BETWEEN %d AND %d;`, min, max) + if _, err := g.db.Exec(query); err != nil { + return fmt.Errorf("failed to update node contract resource id: %w", err) + } + return nil +} + +func (g *Generator) GenerateNameContracts(billsStartID, contractsStartID int) ([]string, int, error) { + var contracts []string + var billReports []string + for i := uint64(1); i <= uint64(g.NameContractCount); i++ { + nodeID, err := rnd(1, uint64(g.NodeCount)) + if err != nil { + return nil, contractsStartID, fmt.Errorf("failed to generate random node id: %w", err) + } + + state := deleted + if g.nodeUP[nodeID] { + if flip(g.contractCreatedRatio) { + state = created + } else if flip(0.5) { + state = gracePeriod + } + } + + twinID, err := rnd(1100, 3100) + if err != nil { + return nil, contractsStartID, fmt.Errorf("failed to generate random twin id: %w", err) + } + + if renter, ok := g.renter[nodeID]; ok { + twinID = renter + } + + if _, ok := g.availableRentNodes[nodeID]; ok { + i-- + continue + } + + contract := name_contract{ + id: fmt.Sprintf("name-contract-%d", contractsStartID), + twin_id: twinID, + contract_id: uint64(contractsStartID), + state: state, + created_at: uint64(time.Now().Unix()), + grid_version: 3, + name: uuid.NewString(), + } + + contractTuple, err := objectToTupleString(contract) + if err != nil { + return nil, contractsStartID, fmt.Errorf("failed to convert contract object to tuple string: %w", err) + } + contracts = append(contracts, contractTuple) + + billings, err := rnd(0, 10) + if err != nil { + return nil, contractsStartID, fmt.Errorf("failed to generate random billings count: %w", err) + } + amountBilled, err := rnd(0, 100000) + if err != nil { + return nil, contractsStartID, fmt.Errorf("failed to generate random amount billed: %w", err) + } + + for j := uint64(0); j < billings; j++ { + billing := contract_bill_report{ + id: fmt.Sprintf("contract-bill-report-%d", billsStartID), + contract_id: uint64(contractsStartID), + discount_received: "Default", + amount_billed: amountBilled, + timestamp: uint64(time.Now().UnixNano()), + } + billsStartID++ + + billTuple, err := objectToTupleString(billing) + if err != nil { + return nil, contractsStartID, fmt.Errorf("failed to convert contract bill report object to tuple string: %w", err) + } + billReports = append(billReports, billTuple) + } + contractsStartID++ + } + + if err := g.insertTuples(name_contract{}, contracts); err != nil { + return nil, contractsStartID, fmt.Errorf("failed to insert name contracts: %w", err) + } + + fmt.Println("name contracts generated") + + return billReports, contractsStartID, nil +} + +func (g *Generator) GenerateRentContracts(billsStartID, contractsStartID int) ([]string, int, error) { + var contracts []string + var billReports []string + for i := uint64(1); i <= uint64(g.RentContractCount); i++ { + nl, nodeID, err := popRandom(g.availableRentNodesList) + if err != nil { + return nil, contractsStartID, fmt.Errorf("failed to select random element from the gives slice: %w", err) + } + + g.availableRentNodesList = nl + delete(g.availableRentNodes, nodeID) + state := deleted + if g.nodeUP[nodeID] { + if flip(0.9) { + state = created + } else if flip(0.5) { + state = gracePeriod + } + } + + twinID, err := rnd(1100, 3100) + if err != nil { + return nil, contractsStartID, fmt.Errorf("failed to generate a random twin id: %w", err) + } + + contract := rent_contract{ + id: fmt.Sprintf("rent-contract-%d", contractsStartID), + twin_id: twinID, + contract_id: uint64(contractsStartID), + state: state, + created_at: uint64(time.Now().Unix()), + node_id: nodeID, + grid_version: 3, + } + + if state != deleted { + g.renter[nodeID] = contract.twin_id + } + + contractTuple, err := objectToTupleString(contract) + if err != nil { + return nil, contractsStartID, fmt.Errorf("failed to convert contract object to tuple string: %w", err) + } + contracts = append(contracts, contractTuple) + + billings, err := rnd(0, 10) + if err != nil { + return nil, contractsStartID, fmt.Errorf("failed to generate billings count: %w", err) + } + + amountBilled, err := rnd(0, 100000) + if err != nil { + return nil, contractsStartID, fmt.Errorf("failed to generate amount billed: %w", err) + } + + for j := uint64(0); j < billings; j++ { + billing := contract_bill_report{ + id: fmt.Sprintf("contract-bill-report-%d", billsStartID), + contract_id: uint64(contractsStartID), + discount_received: "Default", + amount_billed: amountBilled, + timestamp: uint64(time.Now().UnixNano()), + } + + billsStartID++ + + billTuple, err := objectToTupleString(billing) + if err != nil { + return nil, contractsStartID, fmt.Errorf("failed to convert contract bill report object to tuple string: %w", err) + } + billReports = append(billReports, billTuple) + + } + contractsStartID++ + } + + if err := g.insertTuples(rent_contract{}, contracts); err != nil { + return nil, contractsStartID, fmt.Errorf("failed to insert rent contracts: %w", err) + } + + fmt.Println("rent contracts generated") + + return billReports, contractsStartID, nil +} + +func (g *Generator) GeneratePublicIPs() error { + var publicIPs []string + var nodeContracts []uint64 + + for i := uint64(1); i <= uint64(g.PublicIPCount); i++ { + contract_id := uint64(0) + if flip(g.usedPublicIPsRatio) { + idx, err := rnd(0, uint64(len(g.createdNodeContracts))-1) + if err != nil { + return fmt.Errorf("failed to generate random index: %w", err) + } + contract_id = g.createdNodeContracts[idx] + } + ip := randomIPv4() + farmID, err := rnd(1, uint64(g.FarmCount)) + if err != nil { + return fmt.Errorf("failed to generate random farm id: %w", err) + } + + public_ip := public_ip{ + id: fmt.Sprintf("public-ip-%d", i), + gateway: ip.String(), + ip: IPv4Subnet(ip).String(), + contract_id: contract_id, + farm_id: fmt.Sprintf("farm-%d", farmID), + } + publicIpTuple, err := objectToTupleString(public_ip) + if err != nil { + return fmt.Errorf("failed to convert public ip object to tuple string: %w", err) + } + publicIPs = append(publicIPs, publicIpTuple) + nodeContracts = append(nodeContracts, contract_id) + } + + if err := g.insertTuples(public_ip{}, publicIPs); err != nil { + return fmt.Errorf("failed to insert public ips: %w", err) + } + + if err := g.updateNodeContractPublicIPs(nodeContracts); err != nil { + return fmt.Errorf("failed to update contract public ips: %w", err) + } + + fmt.Println("public IPs generated") + + return nil +} + +func (g *Generator) updateNodeContractPublicIPs(nodeContracts []uint64) error { + + if len(nodeContracts) != 0 { + var IDs []string + for _, contractID := range nodeContracts { + IDs = append(IDs, fmt.Sprintf("%d", contractID)) + + } + + query := "UPDATE node_contract set number_of_public_i_ps = number_of_public_i_ps + 1 WHERE contract_id IN (" + query += strings.Join(IDs, ",") + ");" + if _, err := g.db.Exec(query); err != nil { + return fmt.Errorf("failed to update node contracts public ips: %w", err) + } + } + return nil +} + +func (g *Generator) GenerateNodeGPUs() error { + var GPUs []string + vendors := []string{"NVIDIA Corporation", "AMD", "Intel Corporation"} + devices := []string{"GeForce RTX 3080", "Radeon RX 6800 XT", "Intel Iris Xe MAX"} + + for i := 0; i <= 10; i++ { + gpuNum := len(vendors) - 1 + for j := 0; j <= gpuNum; j++ { + g := node_gpu{ + node_twin_id: uint64(i + 100 + 2), // node twin ids start from 102 + vendor: vendors[j], + device: devices[j], + contract: i % 2, + id: fmt.Sprintf("0000:0e:00.0/1002/744c/%d", j), + } + gpuTuple, err := objectToTupleString(g) + if err != nil { + return fmt.Errorf("failed to convert gpu object to tuple string: %w", err) + } + GPUs = append(GPUs, gpuTuple) + } + } + + if err := g.insertTuples(node_gpu{}, GPUs); err != nil { + return fmt.Errorf("failed to insert node gpu: %w", err) + } + + fmt.Println("node GPUs generated") + + return nil +} + +func (g *Generator) GenerateCountries() error { + var countriesValues []string + index := 0 + for countryName, region := range g.regions { + index++ + country := country{ + id: fmt.Sprintf("country-%d", index), + country_id: uint64(index), + name: countryName, + code: g.countriesCodes[countryName], + region: "unknown", + subregion: region, + lat: fmt.Sprintf("%d", 0), + long: fmt.Sprintf("%d", 0), + } + + countryTuple, err := objectToTupleString(country) + if err != nil { + return fmt.Errorf("failed to convert country object to tuple string: %w", err) + } + countriesValues = append(countriesValues, countryTuple) + } + + if err := g.insertTuples(country{}, countriesValues); err != nil { + return fmt.Errorf("failed to insert country: %w", err) + } + fmt.Println("countries generated") + + return nil +} diff --git a/grid-proxy/tools/db/modifiers/modifiers.go b/grid-proxy/tools/db/modifiers/modifiers.go new file mode 100644 index 000000000..f378f3565 --- /dev/null +++ b/grid-proxy/tools/db/modifiers/modifiers.go @@ -0,0 +1,5 @@ +package modifiers + +func Modify() error { + return nil +} diff --git a/grid-proxy/tools/db/modifiers/types.go b/grid-proxy/tools/db/modifiers/types.go new file mode 100644 index 000000000..2ee98aaa0 --- /dev/null +++ b/grid-proxy/tools/db/modifiers/types.go @@ -0,0 +1,246 @@ +package modifiers + +import ( + "database/sql" + "math/rand" +) + +type Generator struct { + db *sql.DB + + NodeCount uint32 + FarmCount uint32 + TwinCount uint32 + NodeContractCount uint32 + RentContractCount uint32 + NameContractCount uint32 + PublicIPCount uint32 + NormalUsersCount uint32 + + maxContractHRU uint64 + maxContractSRU uint64 + maxContractMRU uint64 + maxContractCRU uint64 + minContractHRU uint64 + minContractSRU uint64 + minContractMRU uint64 + minContractCRU uint64 + + contractCreatedRatio float32 + usedPublicIPsRatio float32 + nodeUpRatio float32 + + nodesMRU map[uint64]uint64 + nodesSRU map[uint64]uint64 + nodesHRU map[uint64]uint64 + nodeUP map[uint64]bool + createdNodeContracts []uint64 + dedicatedFarms map[uint64]struct{} + availableRentNodes map[uint64]struct{} + availableRentNodesList []uint64 + renter map[uint64]uint64 + countries []string + regions map[string]string + countriesCodes map[string]string + cities map[string][]string +} + +func NewGenerator(db *sql.DB, seed int) Generator { + r = rand.New(rand.NewSource(int64(seed))) + + return Generator{ + db: db, + contractCreatedRatio: .1, // from devnet + usedPublicIPsRatio: .9, + nodeUpRatio: .5, + NodeCount: 6000, + FarmCount: 600, + NormalUsersCount: 6000, + PublicIPCount: 1000, + TwinCount: 6000 + 600 + 6000, // nodes + farms + normal users + NodeContractCount: 9000, + RentContractCount: 100, + NameContractCount: 300, + + maxContractHRU: 1024 * 1024 * 1024 * 300, + maxContractSRU: 1024 * 1024 * 1024 * 300, + maxContractMRU: 1024 * 1024 * 1024 * 16, + maxContractCRU: 16, + minContractHRU: 0, + minContractSRU: 1024 * 1024 * 256, + minContractMRU: 1024 * 1024 * 256, + minContractCRU: 1, + nodesMRU: make(map[uint64]uint64), + nodesSRU: make(map[uint64]uint64), + nodesHRU: make(map[uint64]uint64), + nodeUP: make(map[uint64]bool), + createdNodeContracts: make([]uint64, 0), + dedicatedFarms: make(map[uint64]struct{}), + availableRentNodes: make(map[uint64]struct{}), + availableRentNodesList: make([]uint64, 0), + renter: make(map[uint64]uint64), + countries: []string{"Belgium", "United States", "Egypt", "United Kingdom"}, + regions: map[string]string{ + "Belgium": "Europe", + "United States": "Americas", + "Egypt": "Africa", + "United Kingdom": "Europe", + }, + countriesCodes: map[string]string{ + "Belgium": "BG", + "United States": "US", + "Egypt": "EG", + "United Kingdom": "UK", + }, + cities: map[string][]string{ + "Belgium": {"Brussels", "Antwerp", "Ghent", "Charleroi"}, + "United States": {"New York", "Chicago", "Los Angeles", "San Francisco"}, + "Egypt": {"Cairo", "Giza", "October", "Nasr City"}, + "United Kingdom": {"London", "Liverpool", "Manchester", "Cambridge"}, + }, + } +} + +type contract_resources struct { + id string + hru uint64 + sru uint64 + cru uint64 + mru uint64 + contract_id string +} +type farm struct { + id string + grid_version uint64 + farm_id uint64 + name string + twin_id uint64 + pricing_policy_id uint64 + certification string + stellar_address string + dedicated_farm bool +} + +type node struct { + id string + grid_version uint64 + node_id uint64 + farm_id uint64 + twin_id uint64 + country string + city string + uptime uint64 + created uint64 + farming_policy_id uint64 + certification string + secure bool + virtualized bool + serial_number string + created_at uint64 + updated_at uint64 + location_id string + power nodePower `gorm:"type:jsonb"` + extra_fee uint64 +} + +type nodePower struct { + State string `json:"state"` + Target string `json:"target"` +} +type twin struct { + id string + grid_version uint64 + twin_id uint64 + account_id string + relay string + public_key string +} + +type public_ip struct { + id string + gateway string + ip string + contract_id uint64 + farm_id string +} +type node_contract struct { + id string + grid_version uint64 + contract_id uint64 + twin_id uint64 + node_id uint64 + deployment_data string + deployment_hash string + number_of_public_i_ps uint64 + state string + created_at uint64 + resources_used_id string +} +type node_resources_total struct { + id string + hru uint64 + sru uint64 + cru uint64 + mru uint64 + node_id string +} +type public_config struct { + id string + ipv4 string + ipv6 string + gw4 string + gw6 string + domain string + node_id string +} +type rent_contract struct { + id string + grid_version uint64 + contract_id uint64 + twin_id uint64 + node_id uint64 + state string + created_at uint64 +} +type location struct { + id string + longitude string + latitude string +} + +type contract_bill_report struct { + id string + contract_id uint64 + discount_received string + amount_billed uint64 + timestamp uint64 +} + +type name_contract struct { + id string + grid_version uint64 + contract_id uint64 + twin_id uint64 + name string + state string + created_at uint64 +} + +type node_gpu struct { + node_twin_id uint64 + id string + vendor string + device string + contract int +} + +type country struct { + id string + country_id uint64 + code string + name string + region string + subregion string + lat string + long string +} diff --git a/grid-proxy/tools/db/modifiers/utils.go b/grid-proxy/tools/db/modifiers/utils.go new file mode 100644 index 000000000..b8b1be2f3 --- /dev/null +++ b/grid-proxy/tools/db/modifiers/utils.go @@ -0,0 +1,139 @@ +package modifiers + +import ( + "encoding/json" + "fmt" + "math/rand" + "net" + "reflect" + "strings" +) + +const null = "NULL" + +var r *rand.Rand + +// rnd gets a random number between min and max +func rnd(min, max uint64) (uint64, error) { + if max-min+1 <= 0 { + return 0, fmt.Errorf("min (%d) cannot be greater than max (%d)", min, max) + } + randomNumber := r.Uint64()%(max-min+1) + min + return randomNumber, nil +} + +// flip simulates a coin flip with a given success probability. +func flip(success float32) bool { + return r.Float32() < success +} + +// randomIPv4 gets a random IPv4 +func randomIPv4() net.IP { + ip := make([]byte, 4) + r.Read(ip) + return net.IP(ip) +} + +// IPv4Subnet gets the ipv4 subnet given the ip +func IPv4Subnet(ip net.IP) *net.IPNet { + return &net.IPNet{ + IP: ip, + Mask: net.CIDRMask(24, 32), + } +} + +// min gets min between 2 numbers +func min(a, b uint64) uint64 { + if a < b { + return a + } + return b +} + +// max gets max between 2 numbers +func max(a, b uint64) uint64 { + if a > b { + return a + } + return b +} + +// objectToTupleString converts a object into a string representation suitable for sql query +func objectToTupleString(v interface{}) (string, error) { + vals := "(" + val := reflect.ValueOf(v) + for i := 0; i < val.NumField(); i++ { + if i == 0 { + v := fmt.Sprint(val.Field(i)) + if v == "" { + v = null + } + if v != null && val.Field(i).Type().Name() == "string" { + v = fmt.Sprintf(`'%s'`, v) + } + vals = fmt.Sprintf("%s%s", vals, v) + } else { + v := fmt.Sprint(val.Field(i)) + if v == "" { + v = null + } + if v != null && val.Field(i).Type().Name() == "string" { + v = fmt.Sprintf(`'%s'`, v) + } + if v != null && val.Field(i).Type().Name() == "nodePower" { + // Construct the nodePower object + val2 := val.Field(i) + power := make(map[string]string) + for j := 0; j < val2.NumField(); j++ { + fieldName := strings.ToLower(val2.Type().Field(j).Name) + fieldValue := val2.Field(j).String() + power[fieldName] = fieldValue + } + + // Marshal the power map to JSON and wrap it in quotes + powerJSON, err := json.Marshal(power) + if err != nil { + return "", fmt.Errorf("failed to marshal the power map to JSON: %w", err) + } + v = fmt.Sprintf("'%s'", string(powerJSON)) + } + vals = fmt.Sprintf("%s, %s", vals, v) + } + } + return fmt.Sprintf("%s)", vals), nil +} + +func (g *Generator) insertTuples(tupleObj interface{}, tuples []string) error { + + if len(tuples) != 0 { + query := "INSERT INTO " + reflect.Indirect(reflect.ValueOf(tupleObj)).Type().Name() + " (" + objType := reflect.TypeOf(tupleObj) + for i := 0; i < objType.NumField(); i++ { + if i != 0 { + query += ", " + } + query += objType.Field(i).Name + } + + query += ") VALUES " + + query += strings.Join(tuples, ",") + query += ";" + if _, err := g.db.Exec(query); err != nil { + return fmt.Errorf("failed to insert tuples: %w", err) + } + + } + return nil +} + +// popRandom selects a random element from the given slice, +func popRandom(l []uint64) ([]uint64, uint64, error) { + idx, err := rnd(0, uint64(len(l)-1)) + if err != nil { + return nil, 0, err + } + e := l[idx] + l[idx], l[len(l)-1] = l[len(l)-1], l[idx] + return l[:len(l)-1], e, nil +} From b69e2620c414be750e38bcafcaf504b107307bcd Mon Sep 17 00:00:00 2001 From: Omar Abdulaziz Date: Thu, 21 Dec 2023 19:05:29 +0200 Subject: [PATCH 27/39] wip: add modifiers methods on tests Co-authored-by: Mario Wassef --- grid-proxy/tests/queries/main_test.go | 91 ++++-- grid-proxy/tools/db/modifiers/modifiers.go | 111 ++++++- grid-proxy/tools/db/modifiers/types.go | 2 + grid-proxy/tools/db/modifiers/utils.go | 3 - .../tools/db/{ => modifiers}/utils_test.go | 2 +- grid-proxy/tools/db/types.go | 272 +++++++++--------- grid-proxy/tools/db/utils.go | 200 ++++++------- 7 files changed, 424 insertions(+), 257 deletions(-) rename grid-proxy/tools/db/{ => modifiers}/utils_test.go (98%) diff --git a/grid-proxy/tests/queries/main_test.go b/grid-proxy/tests/queries/main_test.go index 7191c1ec6..fc0c60955 100644 --- a/grid-proxy/tests/queries/main_test.go +++ b/grid-proxy/tests/queries/main_test.go @@ -16,9 +16,9 @@ import ( proxyDB "github.com/threefoldtech/tfgrid-sdk-go/grid-proxy/internal/explorer/db" proxyclient "github.com/threefoldtech/tfgrid-sdk-go/grid-proxy/pkg/client" mock "github.com/threefoldtech/tfgrid-sdk-go/grid-proxy/tests/queries/mock_client" + "github.com/threefoldtech/tfgrid-sdk-go/grid-proxy/tools/db/modifiers" "gorm.io/gorm/logger" - - _ "embed" + // _ "embed" ) var ( @@ -38,8 +38,8 @@ var ( DBClient db.Database ) -//go:embed modifiers.sql -var modifiersFile string +// //go:embed modifiers.sql +// var modifiersFile string func parseCmdline() { flag.StringVar(&POSTGRES_HOST, "postgres-host", "", "postgres host") @@ -85,24 +85,24 @@ func TestMain(m *testing.M) { os.Exit(exitcode) } - _, err = db.Exec(modifiersFile) - if err != nil { - panic(err) - } + // err = modifyDataToFireTriggers(db) + // if err != nil { + // panic(err) + // } - data, err = mock.Load(db) - if err != nil { - panic(err) - } - mockClient = mock.NewGridProxyMockClient(data) + // data, err = mock.Load(db) + // if err != nil { + // panic(err) + // } + // mockClient = mock.NewGridProxyMockClient(data) - exitcode = m.Run() + // exitcode = m.Run() // cleanup modified data - os.Exit(exitcode) + // os.Exit(exitcode) } -func modifyDataToFireTriggers(d *sql.DB) { +func modifyDataToFireTriggers(db *sql.DB) error { /* - insert nodes - y - should be on new/old farms @@ -124,4 +124,63 @@ func modifyDataToFireTriggers(d *sql.DB) { - delete public ip - y */ // modifiers.GenerateTwins(db) + + generator := modifiers.NewGenerator(db, SEED) + + // insertion + // if err := generator.GenerateTwins(); err != nil { + // return fmt.Errorf("failed to genrate twins: %w", err) + // } + + // if err := generator.GenerateFarms(); err != nil { + // return fmt.Errorf("failed to generate farms: %w", err) + // } + + // if err := generator.GenerateNodes(); err != nil { + // return fmt.Errorf("failed to generate nodes: %w", err) + // } + + // if err := generator.GenerateContracts(); err != nil { + // return fmt.Errorf("failed to generate contracts: %w", err) + // } + + // if err := generator.GeneratePublicIPs(); err != nil { + // return fmt.Errorf("failed to generate public ips: %w", err) + // } + + // updates + if err := generator.UpdateNodeCountry(); err != nil { + return fmt.Errorf("failed to update node country: %w", err) + } + + if err := generator.UpdateNodeTotalResources(); err != nil { + return fmt.Errorf("failed to update node total resources: %w", err) + } + + if err := generator.UpdateContractResources(); err != nil { + return fmt.Errorf("failed to update contract resources: %w", err) + } + + if err := generator.UpdateNodeContractState(); err != nil { + return fmt.Errorf("failed to update node node contract: %w", err) + } + + if err := generator.UpdateRentContract(); err != nil { + return fmt.Errorf("failed to update rent contract: %w", err) + } + + if err := generator.UpdatePublicIps(); err != nil { + return fmt.Errorf("failed to update public ips: %w", err) + } + + // deletions + if err := generator.DeleteNode(); err != nil { + return fmt.Errorf("failed to delete node: %w", err) + } + + if err := generator.DeletePublicIps(); err != nil { + return fmt.Errorf("failed to delete node: %w", err) + } + + return nil } diff --git a/grid-proxy/tools/db/modifiers/modifiers.go b/grid-proxy/tools/db/modifiers/modifiers.go index f378f3565..bff7fdb93 100644 --- a/grid-proxy/tools/db/modifiers/modifiers.go +++ b/grid-proxy/tools/db/modifiers/modifiers.go @@ -1,5 +1,114 @@ package modifiers -func Modify() error { +import ( + "fmt" + + "github.com/rs/zerolog/log" +) + +func (g *Generator) UpdateNodeCountry() error { + updatesCount := 10 + query := "" + + for i := 0; i < updatesCount; i++ { + nodeId := r.Intn(int(g.NodeCount)) + 1 + country := g.countries[r.Intn(len(g.countries))] + query += fmt.Sprintf("UPDATE node SET country = '%s' WHERE node_id = %d;", country, nodeId) + } + + log.Debug().Str("query", query).Msg("update node country") + + _, err := g.db.Exec(query) + return err +} + +func (g *Generator) UpdateNodeTotalResources() error { + updatesCount := 10 + padding := 1 * 1024 * 1024 * 1024 + query := "" + for i := 0; i < updatesCount; i++ { + nodeId := r.Intn(int(g.NodeCount)) + 1 + + cru := 10 + hru := g.nodesHRU[uint64(nodeId)] + uint64(padding) + mru := g.nodesMRU[uint64(nodeId)] + uint64(padding) + sru := g.nodesSRU[uint64(nodeId)] + uint64(padding) + + query += fmt.Sprintf("UPDATE node_resources_total SET cru = %d, hru = %d, mru = %d, sru = %d WHERE node_id = 'node-%d';", cru, hru, mru, sru, nodeId) + } + + log.Debug().Str("query", query).Msg("update node country") + + _, err := g.db.Exec(query) + return err +} + +func (g *Generator) UpdateContractResources() error { + updatesCount := 10 + query := "" + for i := 0; i < updatesCount; i++ { + contractId := r.Intn(int(g.NodeContractCount)) + 1 + + cru := g.minContractCRU + hru := g.minContractHRU + sru := g.minContractSRU + mru := g.minContractMRU + + query += fmt.Sprintf("UPDATE contract_resources SET cru = %d, hru = %d, mru = %d, sru = %d WHERE contract_id = 'node-contract-%d';", cru, hru, mru, sru, contractId) + } + + log.Debug().Str("query", query).Msg("update node country") + + _, err := g.db.Exec(query) + return err +} + +func (g *Generator) UpdateNodeContractState() error { + updatesCount := 10 + query := "" + states := []string{"Deleted", "GracePeriod"} + + for i := 0; i < updatesCount; i++ { + contractId := g.createdNodeContracts[r.Intn(len(g.createdNodeContracts))] + state := states[r.Intn(2)] + query += fmt.Sprintf("UPDATE node_contract SET state = '%s' WHERE contract_id = %d;", state, contractId) + } + + log.Debug().Str("query", query).Msg("update node country") + + _, err := g.db.Exec(query) + return err +} + +func (g *Generator) UpdateRentContract() error { + updatesCount := 10 + query := "" + states := []string{"Deleted", "GracePeriod"} + + for i := 0; i < updatesCount; i++ { + contractId := r.Intn(int(g.RentContractCount)) + 1 + state := states[r.Intn(2)] + query += fmt.Sprintf("UPDATE rent_contract SET state = '%s' WHERE contract_id = %d;", state, contractId) + } + + log.Debug().Str("query", query).Msg("update node country") + + _, err := g.db.Exec(query) + return err +} + +func (g *Generator) UpdatePublicIps() error { + + return nil +} + +// deletions +func (g *Generator) DeleteNode() error { + + return nil +} + +func (g *Generator) DeletePublicIps() error { + return nil } diff --git a/grid-proxy/tools/db/modifiers/types.go b/grid-proxy/tools/db/modifiers/types.go index 2ee98aaa0..f472fded1 100644 --- a/grid-proxy/tools/db/modifiers/types.go +++ b/grid-proxy/tools/db/modifiers/types.go @@ -5,6 +5,8 @@ import ( "math/rand" ) +var r *rand.Rand + type Generator struct { db *sql.DB diff --git a/grid-proxy/tools/db/modifiers/utils.go b/grid-proxy/tools/db/modifiers/utils.go index b8b1be2f3..efc978d07 100644 --- a/grid-proxy/tools/db/modifiers/utils.go +++ b/grid-proxy/tools/db/modifiers/utils.go @@ -3,7 +3,6 @@ package modifiers import ( "encoding/json" "fmt" - "math/rand" "net" "reflect" "strings" @@ -11,8 +10,6 @@ import ( const null = "NULL" -var r *rand.Rand - // rnd gets a random number between min and max func rnd(min, max uint64) (uint64, error) { if max-min+1 <= 0 { diff --git a/grid-proxy/tools/db/utils_test.go b/grid-proxy/tools/db/modifiers/utils_test.go similarity index 98% rename from grid-proxy/tools/db/utils_test.go rename to grid-proxy/tools/db/modifiers/utils_test.go index acd1c904d..058729b5c 100644 --- a/grid-proxy/tools/db/utils_test.go +++ b/grid-proxy/tools/db/modifiers/utils_test.go @@ -1,4 +1,4 @@ -package main +package modifiers import ( "fmt" diff --git a/grid-proxy/tools/db/types.go b/grid-proxy/tools/db/types.go index b98da210c..6db8fb9c7 100644 --- a/grid-proxy/tools/db/types.go +++ b/grid-proxy/tools/db/types.go @@ -1,145 +1,145 @@ package main -type contract_resources struct { - id string - hru uint64 - sru uint64 - cru uint64 - mru uint64 - contract_id string -} -type farm struct { - id string - grid_version uint64 - farm_id uint64 - name string - twin_id uint64 - pricing_policy_id uint64 - certification string - stellar_address string - dedicated_farm bool -} +// type contract_resources struct { +// id string +// hru uint64 +// sru uint64 +// cru uint64 +// mru uint64 +// contract_id string +// } +// type farm struct { +// id string +// grid_version uint64 +// farm_id uint64 +// name string +// twin_id uint64 +// pricing_policy_id uint64 +// certification string +// stellar_address string +// dedicated_farm bool +// } -type node struct { - id string - grid_version uint64 - node_id uint64 - farm_id uint64 - twin_id uint64 - country string - city string - uptime uint64 - created uint64 - farming_policy_id uint64 - certification string - secure bool - virtualized bool - serial_number string - created_at uint64 - updated_at uint64 - location_id string - power nodePower `gorm:"type:jsonb"` - extra_fee uint64 -} +// type node struct { +// id string +// grid_version uint64 +// node_id uint64 +// farm_id uint64 +// twin_id uint64 +// country string +// city string +// uptime uint64 +// created uint64 +// farming_policy_id uint64 +// certification string +// secure bool +// virtualized bool +// serial_number string +// created_at uint64 +// updated_at uint64 +// location_id string +// power nodePower `gorm:"type:jsonb"` +// extra_fee uint64 +// } -type nodePower struct { - State string `json:"state"` - Target string `json:"target"` -} -type twin struct { - id string - grid_version uint64 - twin_id uint64 - account_id string - relay string - public_key string -} +// type nodePower struct { +// State string `json:"state"` +// Target string `json:"target"` +// } +// type twin struct { +// id string +// grid_version uint64 +// twin_id uint64 +// account_id string +// relay string +// public_key string +// } -type public_ip struct { - id string - gateway string - ip string - contract_id uint64 - farm_id string -} -type node_contract struct { - id string - grid_version uint64 - contract_id uint64 - twin_id uint64 - node_id uint64 - deployment_data string - deployment_hash string - number_of_public_i_ps uint64 - state string - created_at uint64 - resources_used_id string -} -type node_resources_total struct { - id string - hru uint64 - sru uint64 - cru uint64 - mru uint64 - node_id string -} -type public_config struct { - id string - ipv4 string - ipv6 string - gw4 string - gw6 string - domain string - node_id string -} -type rent_contract struct { - id string - grid_version uint64 - contract_id uint64 - twin_id uint64 - node_id uint64 - state string - created_at uint64 -} -type location struct { - id string - longitude string - latitude string -} +// type public_ip struct { +// id string +// gateway string +// ip string +// contract_id uint64 +// farm_id string +// } +// type node_contract struct { +// id string +// grid_version uint64 +// contract_id uint64 +// twin_id uint64 +// node_id uint64 +// deployment_data string +// deployment_hash string +// number_of_public_i_ps uint64 +// state string +// created_at uint64 +// resources_used_id string +// } +// type node_resources_total struct { +// id string +// hru uint64 +// sru uint64 +// cru uint64 +// mru uint64 +// node_id string +// } +// type public_config struct { +// id string +// ipv4 string +// ipv6 string +// gw4 string +// gw6 string +// domain string +// node_id string +// } +// type rent_contract struct { +// id string +// grid_version uint64 +// contract_id uint64 +// twin_id uint64 +// node_id uint64 +// state string +// created_at uint64 +// } +// type location struct { +// id string +// longitude string +// latitude string +// } -type contract_bill_report struct { - id string - contract_id uint64 - discount_received string - amount_billed uint64 - timestamp uint64 -} +// type contract_bill_report struct { +// id string +// contract_id uint64 +// discount_received string +// amount_billed uint64 +// timestamp uint64 +// } -type name_contract struct { - id string - grid_version uint64 - contract_id uint64 - twin_id uint64 - name string - state string - created_at uint64 -} +// type name_contract struct { +// id string +// grid_version uint64 +// contract_id uint64 +// twin_id uint64 +// name string +// state string +// created_at uint64 +// } -type node_gpu struct { - node_twin_id uint64 - id string - vendor string - device string - contract int -} +// type node_gpu struct { +// node_twin_id uint64 +// id string +// vendor string +// device string +// contract int +// } -type country struct { - id string - country_id uint64 - code string - name string - region string - subregion string - lat string - long string -} +// type country struct { +// id string +// country_id uint64 +// code string +// name string +// region string +// subregion string +// lat string +// long string +// } diff --git a/grid-proxy/tools/db/utils.go b/grid-proxy/tools/db/utils.go index 0a846607e..646a66fe6 100644 --- a/grid-proxy/tools/db/utils.go +++ b/grid-proxy/tools/db/utils.go @@ -1,112 +1,112 @@ package main -import ( - "encoding/json" - "fmt" - "net" - "reflect" - "strings" -) +// import ( +// "encoding/json" +// "fmt" +// "net" +// "reflect" +// "strings" +// ) -const null = "NULL" +// const null = "NULL" -// rnd gets a random number between min and max -func rnd(min, max uint64) (uint64, error) { - if max-min+1 <= 0 { - return 0, fmt.Errorf("min (%d) cannot be greater than max (%d)", min, max) - } - randomNumber := r.Uint64()%(max-min+1) + min - return randomNumber, nil -} +// // rnd gets a random number between min and max +// func rnd(min, max uint64) (uint64, error) { +// if max-min+1 <= 0 { +// return 0, fmt.Errorf("min (%d) cannot be greater than max (%d)", min, max) +// } +// randomNumber := r.Uint64()%(max-min+1) + min +// return randomNumber, nil +// } -// flip simulates a coin flip with a given success probability. -func flip(success float32) bool { - return r.Float32() < success -} +// // flip simulates a coin flip with a given success probability. +// func flip(success float32) bool { +// return r.Float32() < success +// } -// randomIPv4 gets a random IPv4 -func randomIPv4() net.IP { - ip := make([]byte, 4) - r.Read(ip) - return net.IP(ip) -} +// // randomIPv4 gets a random IPv4 +// func randomIPv4() net.IP { +// ip := make([]byte, 4) +// r.Read(ip) +// return net.IP(ip) +// } -// IPv4Subnet gets the ipv4 subnet given the ip -func IPv4Subnet(ip net.IP) *net.IPNet { - return &net.IPNet{ - IP: ip, - Mask: net.CIDRMask(24, 32), - } -} +// // IPv4Subnet gets the ipv4 subnet given the ip +// func IPv4Subnet(ip net.IP) *net.IPNet { +// return &net.IPNet{ +// IP: ip, +// Mask: net.CIDRMask(24, 32), +// } +// } -// min gets min between 2 numbers -func min(a, b uint64) uint64 { - if a < b { - return a - } - return b -} +// // min gets min between 2 numbers +// func min(a, b uint64) uint64 { +// if a < b { +// return a +// } +// return b +// } -// max gets max between 2 numbers -func max(a, b uint64) uint64 { - if a > b { - return a - } - return b -} +// // max gets max between 2 numbers +// func max(a, b uint64) uint64 { +// if a > b { +// return a +// } +// return b +// } -// objectToTupleString converts a object into a string representation suitable for sql query -func objectToTupleString(v interface{}) (string, error) { - vals := "(" - val := reflect.ValueOf(v) - for i := 0; i < val.NumField(); i++ { - if i == 0 { - v := fmt.Sprint(val.Field(i)) - if v == "" { - v = null - } - if v != null && val.Field(i).Type().Name() == "string" { - v = fmt.Sprintf(`'%s'`, v) - } - vals = fmt.Sprintf("%s%s", vals, v) - } else { - v := fmt.Sprint(val.Field(i)) - if v == "" { - v = null - } - if v != null && val.Field(i).Type().Name() == "string" { - v = fmt.Sprintf(`'%s'`, v) - } - if v != null && val.Field(i).Type().Name() == "nodePower" { - // Construct the nodePower object - val2 := val.Field(i) - power := make(map[string]string) - for j := 0; j < val2.NumField(); j++ { - fieldName := strings.ToLower(val2.Type().Field(j).Name) - fieldValue := val2.Field(j).String() - power[fieldName] = fieldValue - } +// // objectToTupleString converts a object into a string representation suitable for sql query +// func objectToTupleString(v interface{}) (string, error) { +// vals := "(" +// val := reflect.ValueOf(v) +// for i := 0; i < val.NumField(); i++ { +// if i == 0 { +// v := fmt.Sprint(val.Field(i)) +// if v == "" { +// v = null +// } +// if v != null && val.Field(i).Type().Name() == "string" { +// v = fmt.Sprintf(`'%s'`, v) +// } +// vals = fmt.Sprintf("%s%s", vals, v) +// } else { +// v := fmt.Sprint(val.Field(i)) +// if v == "" { +// v = null +// } +// if v != null && val.Field(i).Type().Name() == "string" { +// v = fmt.Sprintf(`'%s'`, v) +// } +// if v != null && val.Field(i).Type().Name() == "nodePower" { +// // Construct the nodePower object +// val2 := val.Field(i) +// power := make(map[string]string) +// for j := 0; j < val2.NumField(); j++ { +// fieldName := strings.ToLower(val2.Type().Field(j).Name) +// fieldValue := val2.Field(j).String() +// power[fieldName] = fieldValue +// } - // Marshal the power map to JSON and wrap it in quotes - powerJSON, err := json.Marshal(power) - if err != nil { - return "", fmt.Errorf("failed to marshal the power map to JSON: %w", err) - } - v = fmt.Sprintf("'%s'", string(powerJSON)) - } - vals = fmt.Sprintf("%s, %s", vals, v) - } - } - return fmt.Sprintf("%s)", vals), nil -} +// // Marshal the power map to JSON and wrap it in quotes +// powerJSON, err := json.Marshal(power) +// if err != nil { +// return "", fmt.Errorf("failed to marshal the power map to JSON: %w", err) +// } +// v = fmt.Sprintf("'%s'", string(powerJSON)) +// } +// vals = fmt.Sprintf("%s, %s", vals, v) +// } +// } +// return fmt.Sprintf("%s)", vals), nil +// } -// popRandom selects a random element from the given slice, -func popRandom(l []uint64) ([]uint64, uint64, error) { - idx, err := rnd(0, uint64(len(l)-1)) - if err != nil { - return nil, 0, err - } - e := l[idx] - l[idx], l[len(l)-1] = l[len(l)-1], l[idx] - return l[:len(l)-1], e, nil -} +// // popRandom selects a random element from the given slice, +// func popRandom(l []uint64) ([]uint64, uint64, error) { +// idx, err := rnd(0, uint64(len(l)-1)) +// if err != nil { +// return nil, 0, err +// } +// e := l[idx] +// l[idx], l[len(l)-1] = l[len(l)-1], l[idx] +// return l[:len(l)-1], e, nil +// } From 88a99be42cde5aef500c0070f33aa93fd6900141 Mon Sep 17 00:00:00 2001 From: mario Date: Sun, 24 Dec 2023 13:30:25 +0200 Subject: [PATCH 28/39] wip: add data modifiers Co-authored-by: omarabdulasis --- grid-proxy/tools/db/modifiers/modifiers.go | 54 ++++++++++++++++++++-- 1 file changed, 51 insertions(+), 3 deletions(-) diff --git a/grid-proxy/tools/db/modifiers/modifiers.go b/grid-proxy/tools/db/modifiers/modifiers.go index bff7fdb93..c7e9098a3 100644 --- a/grid-proxy/tools/db/modifiers/modifiers.go +++ b/grid-proxy/tools/db/modifiers/modifiers.go @@ -98,17 +98,65 @@ func (g *Generator) UpdateRentContract() error { } func (g *Generator) UpdatePublicIps() error { + updatesCount := 10 + query := "" - return nil + for i := 0; i < updatesCount; i++ { + idx := r.Intn(len(g.createdNodeContracts)) + contractID := g.createdNodeContracts[idx] + publicIPID := r.Intn(int(g.PublicIPCount)) + + query += fmt.Sprintf("UPDATE public_ip SET contract_id = (CASE WHEN contract_id = 0 THEN %d ELSE 0 END) WHERE id = 'public-ip-%d';", contractID, publicIPID) + } + + log.Debug().Str("query", query).Msg("update public ip contract_id") + + _, err := g.db.Exec(query) + return err } // deletions -func (g *Generator) DeleteNode() error { +func (g *Generator) DeleteNodes() error { + // delete node contracts on this node + // free public ips that are assigned to the deleted contracts + // delete rent contracts on this node + // delete node + updatesCount := r.Intn(10) + 1 + query := "" - return nil + for i := 0; i < updatesCount; i++ { + nodeID := r.Intn(int(g.NodeCount)) + 1 + g.NodeCount-- + + query += fmt.Sprintf("UPDATE public_ip SET contract_id = 0 WHERE contract_id IN (SELECT contract_id FROM node_contract WHERE node_id = %d);", nodeID) + query += fmt.Sprintf("UPDATE node_contract SET state = 'Deleted' WHERE node_id = %d;", nodeID) + query += fmt.Sprintf("UPDATE rent_contract set state = 'Deleted' WHERE node_id = %d;", nodeID) + query += fmt.Sprintf("DELETE FROM node_resources_total WHERE node_id = (SELECT id FROM node WHERE node_id = %d);", nodeID) + query += fmt.Sprintf("DELETE FROM node WHERE node_id = %d;", nodeID) + } + + log.Debug().Str("query", query).Msg("delete nodes") + + _, err := g.db.Exec(query) + return err } func (g *Generator) DeletePublicIps() error { + maxDeleteCount := r.Intn(10) + 1 + query := fmt.Sprintf("DELETE FROM public_ip WHERE id in (SELECT id FROM public_ip WHERE contract_id = 0 LIMIT %d);", maxDeleteCount) + + res, err := g.db.Exec(query) + if err != nil { + return fmt.Errorf("failed to delete public ips: %w", err) + } + + rowsAffercted, err := res.RowsAffected() + if err != nil { + return fmt.Errorf("failed to get rows affected by public ips delete: %w", err) + } + g.PublicIPCount -= uint32(rowsAffercted) + + log.Debug().Str("query", query).Msg("delete public ips") return nil } From d4dd88e7c6053c5f18db19033b4c4ea691e5dceb Mon Sep 17 00:00:00 2001 From: Omar Abdulaziz Date: Sun, 24 Dec 2023 20:22:42 +0200 Subject: [PATCH 29/39] wip: working on modifiers - add farm insertion/deletion trigger - update generation functions Co-authored-by: Mario Wassef --- grid-proxy/internal/explorer/db/setup.sql | 30 + grid-proxy/tests/queries/main_test.go | 119 +-- .../tests/queries/mock_client/loader.go | 3 + grid-proxy/tools/db/generate.go | 768 +----------------- grid-proxy/tools/db/modifiers/generators.go | 189 +++-- grid-proxy/tools/db/modifiers/modifiers.go | 36 +- grid-proxy/tools/db/modifiers/types.go | 112 +-- 7 files changed, 257 insertions(+), 1000 deletions(-) diff --git a/grid-proxy/internal/explorer/db/setup.sql b/grid-proxy/internal/explorer/db/setup.sql index 7f3b2e926..f081d86a6 100644 --- a/grid-proxy/internal/explorer/db/setup.sql +++ b/grid-proxy/internal/explorer/db/setup.sql @@ -443,6 +443,22 @@ CREATE OR REPLACE TRIGGER rent_contract_trigger CREATE OR REPLACE FUNCTION public_ip_upsert() RETURNS TRIGGER AS $$ BEGIN + + BEGIN + IF TG_OP = 'INSERT' AND NEW.farm_id NOT IN (select farm_id from public_ips_cache) THEN + INSERT INTO public_ips_cache VALUES( + NEW.farm_id, + 0, + 0, + '[]' + ); + END IF; + EXCEPTION + WHEN OTHERS THEN + RAISE NOTICE 'Error inserting public_ips_cache record %s', SQLERRM; + END; + + BEGIN UPDATE public_ips_cache SET free_ips = free_ips + ( @@ -456,6 +472,7 @@ BEGIN -- handles delete reserved ips ELSE 0 END ), + total_ips = total_ips + ( CASE WHEN TG_OP = 'INSERT' @@ -464,6 +481,7 @@ BEGIN THEN -1 ELSE 0 END ), + ips = ( select jsonb_agg( jsonb_build_object( @@ -488,6 +506,18 @@ BEGIN WHEN OTHERS THEN RAISE NOTICE 'Error reflect public_ips changes %s', SQLERRM; END; + + BEGIN + IF TG_OP = 'DELETE' THEN + -- delete if exists + DELETE FROM public_ips_cache WHERE public_ips_cache.total_ips = 0 AND + public_ips_cache.farm_id = (select farm_id from farm where farm.id = OLD.farm_id); + END IF; + EXCEPTION + WHEN OTHERS THEN + RAISE NOTICE 'Error deleting public_ips_cache record %s', SQLERRM; + END; + RETURN NULL; END; $$ LANGUAGE plpgsql; diff --git a/grid-proxy/tests/queries/main_test.go b/grid-proxy/tests/queries/main_test.go index fc0c60955..8f008dfc6 100644 --- a/grid-proxy/tests/queries/main_test.go +++ b/grid-proxy/tests/queries/main_test.go @@ -80,29 +80,27 @@ func TestMain(m *testing.M) { mockClient = mock.NewGridProxyMockClient(data) gridProxyClient = proxyclient.NewClient(ENDPOINT) - exitcode := m.Run() - if exitcode != 0 { - os.Exit(exitcode) - } - - // err = modifyDataToFireTriggers(db) - // if err != nil { - // panic(err) + // exitcode := m.Run() + // if exitcode != 0 { + // os.Exit(exitcode) // } - // data, err = mock.Load(db) - // if err != nil { - // panic(err) - // } - // mockClient = mock.NewGridProxyMockClient(data) + err = modifyDataToFireTriggers(db, data) + if err != nil { + panic(err) + } - // exitcode = m.Run() + data, err = mock.Load(db) + if err != nil { + panic(err) + } + mockClient = mock.NewGridProxyMockClient(data) - // cleanup modified data - // os.Exit(exitcode) + exitcode := m.Run() + os.Exit(exitcode) } -func modifyDataToFireTriggers(db *sql.DB) error { +func modifyDataToFireTriggers(db *sql.DB, data mock.DBData) error { /* - insert nodes - y - should be on new/old farms @@ -127,60 +125,65 @@ func modifyDataToFireTriggers(db *sql.DB) error { generator := modifiers.NewGenerator(db, SEED) + twinStart := len(data.Twins) + 1 + farmStart := len(data.Farms) + 1 + nodeStart := len(data.Nodes) + 1 + contractStart := len(data.NodeContracts) + len(data.RentContracts) + len(data.NameContracts) + 1 + billStart := data.BillReports + 1 + publicIpStart := len(data.PublicIPs) + 1 + size := 10 + // insertion - // if err := generator.GenerateTwins(); err != nil { - // return fmt.Errorf("failed to genrate twins: %w", err) - // } + if err := generator.GenerateFarms(farmStart, 100, twinStart); err != nil { + return fmt.Errorf("failed to generate farms: %w", err) + } - // if err := generator.GenerateFarms(); err != nil { - // return fmt.Errorf("failed to generate farms: %w", err) - // } + if err := generator.GenerateNodes(nodeStart, 600, farmStart, 100, twinStart); err != nil { + return fmt.Errorf("failed to generate nodes: %w", err) + } - // if err := generator.GenerateNodes(); err != nil { - // return fmt.Errorf("failed to generate nodes: %w", err) - // } + // rentCount is 1 because the generate method have .1 percent of 10 farms to be dedicated + if err := generator.GenerateContracts(int(billStart), contractStart, 50, size, 1, nodeStart, 600); err != nil { + return fmt.Errorf("failed to generate contracts: %w", err) + } - // if err := generator.GenerateContracts(); err != nil { - // return fmt.Errorf("failed to generate contracts: %w", err) - // } + if err := generator.GeneratePublicIPs(publicIpStart, size); err != nil { + return fmt.Errorf("failed to generate public ips: %w", err) + } - // if err := generator.GeneratePublicIPs(); err != nil { - // return fmt.Errorf("failed to generate public ips: %w", err) + // // updates + // if err := generator.UpdateNodeCountry(); err != nil { + // return fmt.Errorf("failed to update node country: %w", err) // } - // updates - if err := generator.UpdateNodeCountry(); err != nil { - return fmt.Errorf("failed to update node country: %w", err) - } - - if err := generator.UpdateNodeTotalResources(); err != nil { - return fmt.Errorf("failed to update node total resources: %w", err) - } + // if err := generator.UpdateNodeTotalResources(); err != nil { + // return fmt.Errorf("failed to update node total resources: %w", err) + // } - if err := generator.UpdateContractResources(); err != nil { - return fmt.Errorf("failed to update contract resources: %w", err) - } + // if err := generator.UpdateContractResources(); err != nil { + // return fmt.Errorf("failed to update contract resources: %w", err) + // } - if err := generator.UpdateNodeContractState(); err != nil { - return fmt.Errorf("failed to update node node contract: %w", err) - } + // if err := generator.UpdateNodeContractState(); err != nil { + // return fmt.Errorf("failed to update node node contract: %w", err) + // } - if err := generator.UpdateRentContract(); err != nil { - return fmt.Errorf("failed to update rent contract: %w", err) - } + // if err := generator.UpdateRentContract(); err != nil { + // return fmt.Errorf("failed to update rent contract: %w", err) + // } - if err := generator.UpdatePublicIps(); err != nil { - return fmt.Errorf("failed to update public ips: %w", err) - } + // if err := generator.UpdatePublicIps(); err != nil { + // return fmt.Errorf("failed to update public ips: %w", err) + // } - // deletions - if err := generator.DeleteNode(); err != nil { - return fmt.Errorf("failed to delete node: %w", err) - } + // // deletions + // if err := generator.DeleteNodes(); err != nil { + // return fmt.Errorf("failed to delete node: %w", err) + // } - if err := generator.DeletePublicIps(); err != nil { - return fmt.Errorf("failed to delete node: %w", err) - } + // if err := generator.DeletePublicIps(); err != nil { + // return fmt.Errorf("failed to delete node: %w", err) + // } return nil } diff --git a/grid-proxy/tests/queries/mock_client/loader.go b/grid-proxy/tests/queries/mock_client/loader.go index 30ecf388d..4702172d2 100644 --- a/grid-proxy/tests/queries/mock_client/loader.go +++ b/grid-proxy/tests/queries/mock_client/loader.go @@ -29,6 +29,7 @@ type DBData struct { RentContracts map[uint64]RentContract NameContracts map[uint64]NameContract Billings map[uint64][]ContractBillReport + BillReports uint32 ContractResources map[string]ContractResources NonDeletedContracts map[uint64][]uint64 GPUs map[uint64][]NodeGPU @@ -477,6 +478,7 @@ func loadContractBillingReports(db *sql.DB, data *DBData) error { return err } data.Billings[contractBillReport.ContractID] = append(data.Billings[contractBillReport.ContractID], contractBillReport) + data.BillReports++ } return nil } @@ -595,6 +597,7 @@ func Load(db *sql.DB) (DBData, error) { NodeRentedBy: make(map[uint64]uint64), NodeRentContractID: make(map[uint64]uint64), Billings: make(map[uint64][]ContractBillReport), + BillReports: 0, ContractResources: make(map[string]ContractResources), NodeTotalResources: make(map[uint64]NodeResourcesTotal), NodeUsedResources: make(map[uint64]NodeResourcesTotal), diff --git a/grid-proxy/tools/db/generate.go b/grid-proxy/tools/db/generate.go index 540293209..6200e9e10 100644 --- a/grid-proxy/tools/db/generate.go +++ b/grid-proxy/tools/db/generate.go @@ -8,67 +8,6 @@ import ( "github.com/threefoldtech/tfgrid-sdk-go/grid-proxy/tools/db/modifiers" ) -// var ( -// nodesMRU = make(map[uint64]uint64) -// nodesSRU = make(map[uint64]uint64) -// nodesHRU = make(map[uint64]uint64) -// nodeUP = make(map[uint64]bool) -// createdNodeContracts = make([]uint64, 0) -// dedicatedFarms = make(map[uint64]struct{}) -// availableRentNodes = make(map[uint64]struct{}) -// availableRentNodesList = make([]uint64, 0) -// renter = make(map[uint64]uint64) -// ) - -// const ( -// contractCreatedRatio = .1 // from devnet -// usedPublicIPsRatio = .9 -// nodeUpRatio = .5 -// nodeCount = 6000 -// farmCount = 600 -// normalUsers = 6000 -// publicIPCount = 1000 -// twinCount = nodeCount + farmCount + normalUsers -// nodeContractCount = 9000 -// rentContractCount = 100 -// nameContractCount = 300 - -// maxContractHRU = 1024 * 1024 * 1024 * 300 -// maxContractSRU = 1024 * 1024 * 1024 * 300 -// maxContractMRU = 1024 * 1024 * 1024 * 16 -// maxContractCRU = 16 -// minContractHRU = 0 -// minContractSRU = 1024 * 1024 * 256 -// minContractMRU = 1024 * 1024 * 256 -// minContractCRU = 1 -// ) - -// var ( -// countries = []string{"Belgium", "United States", "Egypt", "United Kingdom"} -// regions = map[string]string{ -// "Belgium": "Europe", -// "United States": "Americas", -// "Egypt": "Africa", -// "United Kingdom": "Europe", -// } -// countriesCodes = map[string]string{ -// "Belgium": "BG", -// "United States": "US", -// "Egypt": "EG", -// "United Kingdom": "UK", -// } -// cities = map[string][]string{ -// "Belgium": {"Brussels", "Antwerp", "Ghent", "Charleroi"}, -// "United States": {"New York", "Chicago", "Los Angeles", "San Francisco"}, -// "Egypt": {"Cairo", "Giza", "October", "Nasr City"}, -// "United Kingdom": {"London", "Liverpool", "Manchester", "Cambridge"}, -// } -// ) - -// const deleted = "Deleted" -// const created = "Created" -// const gracePeriod = "GracePeriod" - func initSchema(db *sql.DB) error { schema, err := os.ReadFile("./schema.sql") if err != nil { @@ -81,721 +20,26 @@ func initSchema(db *sql.DB) error { return nil } -// func generatePublicIPs(db *sql.DB) error { -// var publicIPs []string -// var nodeContracts []uint64 - -// for i := uint64(1); i <= publicIPCount; i++ { -// contract_id := uint64(0) -// if flip(usedPublicIPsRatio) { -// idx, err := rnd(0, uint64(len(createdNodeContracts))-1) -// if err != nil { -// return fmt.Errorf("failed to generate random index: %w", err) -// } -// contract_id = createdNodeContracts[idx] -// } -// ip := randomIPv4() -// farmID, err := rnd(1, farmCount) -// if err != nil { -// return fmt.Errorf("failed to generate random farm id: %w", err) -// } - -// public_ip := public_ip{ -// id: fmt.Sprintf("public-ip-%d", i), -// gateway: ip.String(), -// ip: IPv4Subnet(ip).String(), -// contract_id: contract_id, -// farm_id: fmt.Sprintf("farm-%d", farmID), -// } -// publicIpTuple, err := objectToTupleString(public_ip) -// if err != nil { -// return fmt.Errorf("failed to convert public ip object to tuple string: %w", err) -// } -// publicIPs = append(publicIPs, publicIpTuple) -// nodeContracts = append(nodeContracts, contract_id) -// } - -// if err := insertTuples(db, public_ip{}, publicIPs); err != nil { -// return fmt.Errorf("failed to insert public ips: %w", err) -// } - -// if err := updateNodeContractPublicIPs(db, nodeContracts); err != nil { -// return fmt.Errorf("failed to update contract public ips: %w", err) -// } - -// fmt.Println("public IPs generated") - -// return nil -// } - -// func generateFarms(db *sql.DB) error { -// var farms []string - -// for i := uint64(1); i <= farmCount; i++ { -// farm := farm{ -// id: fmt.Sprintf("farm-%d", i), -// farm_id: i, -// name: fmt.Sprintf("farm-name-%d", i), -// certification: "Diy", -// dedicated_farm: flip(.1), -// twin_id: i, -// pricing_policy_id: 1, -// grid_version: 3, -// stellar_address: "", -// } - -// if farm.dedicated_farm { -// dedicatedFarms[farm.farm_id] = struct{}{} -// } - -// farmTuple, err := objectToTupleString(farm) -// if err != nil { -// return fmt.Errorf("failed to convert farm object to tuple string: %w", err) -// } -// farms = append(farms, farmTuple) -// } - -// if err := insertTuples(db, farm{}, farms); err != nil { -// return fmt.Errorf("failed to insert farms: %w", err) -// } -// fmt.Println("farms generated") - -// return nil -// } - -// func generateCountries(db *sql.DB) error { -// var countriesValues []string -// index := 0 -// for countryName, region := range regions { -// index++ -// country := country{ -// id: fmt.Sprintf("country-%d", index), -// country_id: uint64(index), -// name: countryName, -// code: countriesCodes[countryName], -// region: "unknown", -// subregion: region, -// lat: fmt.Sprintf("%d", 0), -// long: fmt.Sprintf("%d", 0), -// } - -// countryTuple, err := objectToTupleString(country) -// if err != nil { -// return fmt.Errorf("failed to convert country object to tuple string: %w", err) -// } -// countriesValues = append(countriesValues, countryTuple) -// } - -// if err := insertTuples(db, country{}, countriesValues); err != nil { -// return fmt.Errorf("failed to insert country: %w", err) -// } -// fmt.Println("countries generated") - -// return nil -// } - -// func generateNodeContracts(db *sql.DB, billsStartID, contractsStartID int) ([]string, int, error) { -// var contracts []string -// var contractResources []string -// var billingReports []string - -// for i := uint64(1); i <= nodeContractCount; i++ { -// nodeID, err := rnd(1, nodeCount) -// if err != nil { -// return nil, contractsStartID, fmt.Errorf("failed to generate random node id: %w", err) -// } -// state := deleted - -// if nodeUP[nodeID] { -// if flip(contractCreatedRatio) { -// state = created -// } else if flip(0.5) { -// state = gracePeriod -// } -// } - -// if state != deleted && (minContractHRU > nodesHRU[nodeID] || minContractMRU > nodesMRU[nodeID] || minContractSRU > nodesSRU[nodeID]) { -// i-- -// continue -// } - -// twinID, err := rnd(1100, 3100) -// if err != nil { -// return nil, contractsStartID, fmt.Errorf("failed to generate random twin id: %w", err) -// } - -// if renter, ok := renter[nodeID]; ok { -// twinID = renter -// } - -// if _, ok := availableRentNodes[nodeID]; ok { -// i-- -// continue -// } - -// contract := node_contract{ -// id: fmt.Sprintf("node-contract-%d", contractsStartID), -// twin_id: twinID, -// contract_id: uint64(contractsStartID), -// state: state, -// created_at: uint64(time.Now().Unix()), -// node_id: nodeID, -// deployment_data: fmt.Sprintf("deployment-data-%d", contractsStartID), -// deployment_hash: fmt.Sprintf("deployment-hash-%d", contractsStartID), -// number_of_public_i_ps: 0, -// grid_version: 3, -// resources_used_id: "", -// } - -// cru, err := rnd(minContractCRU, maxContractCRU) -// if err != nil { -// return nil, contractsStartID, fmt.Errorf("failed to generate random cru: %w", err) -// } - -// hru, err := rnd(minContractHRU, min(maxContractHRU, nodesHRU[nodeID])) -// if err != nil { -// return nil, contractsStartID, fmt.Errorf("failed to generate random hru: %w", err) -// } - -// sru, err := rnd(minContractSRU, min(maxContractSRU, nodesSRU[nodeID])) -// if err != nil { -// return nil, contractsStartID, fmt.Errorf("failed to generate random sru: %w", err) -// } - -// mru, err := rnd(minContractMRU, min(maxContractMRU, nodesMRU[nodeID])) -// if err != nil { -// return nil, contractsStartID, fmt.Errorf("failed to generate random mru: %w", err) -// } - -// contract_resources := contract_resources{ -// id: fmt.Sprintf("contract-resources-%d", contractsStartID), -// hru: hru, -// sru: sru, -// cru: cru, -// mru: mru, -// contract_id: fmt.Sprintf("node-contract-%d", contractsStartID), -// } -// if contract.state != deleted { -// nodesHRU[nodeID] -= hru -// nodesSRU[nodeID] -= sru -// nodesMRU[nodeID] -= mru -// createdNodeContracts = append(createdNodeContracts, uint64(contractsStartID)) -// } - -// contractTuple, err := objectToTupleString(contract) -// if err != nil { -// return nil, contractsStartID, fmt.Errorf("failed to convert contract object to tuple string: %w", err) -// } -// contracts = append(contracts, contractTuple) - -// contractResourcesTuple, err := objectToTupleString(contract_resources) -// if err != nil { -// return nil, contractsStartID, fmt.Errorf("failed to convert contract resources object to tuple string: %w", err) -// } -// contractResources = append(contractResources, contractResourcesTuple) - -// billings, err := rnd(0, 10) -// if err != nil { -// return nil, contractsStartID, fmt.Errorf("failed to generate random billing count: %w", err) -// } - -// amountBilled, err := rnd(0, 100000) -// if err != nil { -// return nil, contractsStartID, fmt.Errorf("failed to generate random amount billed: %w", err) -// } -// for j := uint64(0); j < billings; j++ { -// billing := contract_bill_report{ -// id: fmt.Sprintf("contract-bill-report-%d", billsStartID), -// contract_id: uint64(contractsStartID), -// discount_received: "Default", -// amount_billed: amountBilled, -// timestamp: uint64(time.Now().UnixNano()), -// } -// billsStartID++ - -// billTuple, err := objectToTupleString(billing) -// if err != nil { -// return nil, contractsStartID, fmt.Errorf("failed to convert contract bill report object to tuple string: %w", err) -// } -// billingReports = append(billingReports, billTuple) -// } -// contractsStartID++ -// } - -// if err := insertTuples(db, node_contract{}, contracts); err != nil { -// return nil, contractsStartID, fmt.Errorf("failed to insert node contracts: %w", err) -// } - -// if err := insertTuples(db, contract_resources{}, contractResources); err != nil { -// return nil, contractsStartID, fmt.Errorf("failed to insert contract resources: %w", err) -// } - -// if err := updateNodeContractResourceID(db, contractsStartID-nodeContractCount, contractsStartID); err != nil { -// return nil, contractsStartID, fmt.Errorf("failed to update node contract resources id: %w", err) -// } - -// fmt.Println("node contracts generated") - -// return billingReports, contractsStartID, nil -// } - -// func generateNameContracts(db *sql.DB, billsStartID, contractsStartID int) ([]string, int, error) { -// var contracts []string -// var billReports []string -// for i := uint64(1); i <= nameContractCount; i++ { -// nodeID, err := rnd(1, nodeCount) -// if err != nil { -// return nil, contractsStartID, fmt.Errorf("failed to generate random node id: %w", err) -// } - -// state := deleted -// if nodeUP[nodeID] { -// if flip(contractCreatedRatio) { -// state = created -// } else if flip(0.5) { -// state = gracePeriod -// } -// } - -// twinID, err := rnd(1100, 3100) -// if err != nil { -// return nil, contractsStartID, fmt.Errorf("failed to generate random twin id: %w", err) -// } - -// if renter, ok := renter[nodeID]; ok { -// twinID = renter -// } - -// if _, ok := availableRentNodes[nodeID]; ok { -// i-- -// continue -// } - -// contract := name_contract{ -// id: fmt.Sprintf("name-contract-%d", contractsStartID), -// twin_id: twinID, -// contract_id: uint64(contractsStartID), -// state: state, -// created_at: uint64(time.Now().Unix()), -// grid_version: 3, -// name: uuid.NewString(), -// } - -// contractTuple, err := objectToTupleString(contract) -// if err != nil { -// return nil, contractsStartID, fmt.Errorf("failed to convert contract object to tuple string: %w", err) -// } -// contracts = append(contracts, contractTuple) - -// billings, err := rnd(0, 10) -// if err != nil { -// return nil, contractsStartID, fmt.Errorf("failed to generate random billings count: %w", err) -// } -// amountBilled, err := rnd(0, 100000) -// if err != nil { -// return nil, contractsStartID, fmt.Errorf("failed to generate random amount billed: %w", err) -// } - -// for j := uint64(0); j < billings; j++ { -// billing := contract_bill_report{ -// id: fmt.Sprintf("contract-bill-report-%d", billsStartID), -// contract_id: uint64(contractsStartID), -// discount_received: "Default", -// amount_billed: amountBilled, -// timestamp: uint64(time.Now().UnixNano()), -// } -// billsStartID++ - -// billTuple, err := objectToTupleString(billing) -// if err != nil { -// return nil, contractsStartID, fmt.Errorf("failed to convert contract bill report object to tuple string: %w", err) -// } -// billReports = append(billReports, billTuple) -// } -// contractsStartID++ -// } - -// if err := insertTuples(db, name_contract{}, contracts); err != nil { -// return nil, contractsStartID, fmt.Errorf("failed to insert name contracts: %w", err) -// } - -// fmt.Println("name contracts generated") - -// return billReports, contractsStartID, nil -// } -// func generateRentContracts(db *sql.DB, billsStartID, contractsStartID int) ([]string, int, error) { -// var contracts []string -// var billReports []string -// for i := uint64(1); i <= rentContractCount; i++ { -// nl, nodeID, err := popRandom(availableRentNodesList) -// if err != nil { -// return nil, contractsStartID, fmt.Errorf("failed to select random element from the gives slice: %w", err) -// } - -// availableRentNodesList = nl -// delete(availableRentNodes, nodeID) -// state := deleted -// if nodeUP[nodeID] { -// if flip(0.9) { -// state = created -// } else if flip(0.5) { -// state = gracePeriod -// } -// } - -// twinID, err := rnd(1100, 3100) -// if err != nil { -// return nil, contractsStartID, fmt.Errorf("failed to generate a random twin id: %w", err) -// } - -// contract := rent_contract{ -// id: fmt.Sprintf("rent-contract-%d", contractsStartID), -// twin_id: twinID, -// contract_id: uint64(contractsStartID), -// state: state, -// created_at: uint64(time.Now().Unix()), -// node_id: nodeID, -// grid_version: 3, -// } - -// if state != deleted { -// renter[nodeID] = contract.twin_id -// } - -// contractTuple, err := objectToTupleString(contract) -// if err != nil { -// return nil, contractsStartID, fmt.Errorf("failed to convert contract object to tuple string: %w", err) -// } -// contracts = append(contracts, contractTuple) - -// billings, err := rnd(0, 10) -// if err != nil { -// return nil, contractsStartID, fmt.Errorf("failed to generate billings count: %w", err) -// } - -// amountBilled, err := rnd(0, 100000) -// if err != nil { -// return nil, contractsStartID, fmt.Errorf("failed to generate amount billed: %w", err) -// } - -// for j := uint64(0); j < billings; j++ { -// billing := contract_bill_report{ -// id: fmt.Sprintf("contract-bill-report-%d", billsStartID), -// contract_id: uint64(contractsStartID), -// discount_received: "Default", -// amount_billed: amountBilled, -// timestamp: uint64(time.Now().UnixNano()), -// } - -// billsStartID++ - -// billTuple, err := objectToTupleString(billing) -// if err != nil { -// return nil, contractsStartID, fmt.Errorf("failed to convert contract bill report object to tuple string: %w", err) -// } -// billReports = append(billReports, billTuple) - -// } -// contractsStartID++ -// } - -// if err := insertTuples(db, rent_contract{}, contracts); err != nil { -// return nil, contractsStartID, fmt.Errorf("failed to insert rent contracts: %w", err) -// } - -// fmt.Println("rent contracts generated") - -// return billReports, contractsStartID, nil -// } - -// func generateNodes(db *sql.DB) error { -// powerState := []string{"Up", "Down"} -// var locations []string -// var nodes []string -// var totalResources []string -// var publicConfigs []string -// for i := uint64(1); i <= nodeCount; i++ { -// mru, err := rnd(4, 256) -// if err != nil { -// return fmt.Errorf("failed to generate random mru: %w", err) -// } -// mru *= 1024 * 1024 * 1024 - -// hru, err := rnd(100, 30*1024) -// if err != nil { -// return fmt.Errorf("failed to generate random hru: %w", err) -// } -// hru *= 1024 * 1024 * 1024 // 100GB -> 30TB - -// sru, err := rnd(200, 30*1024) -// if err != nil { -// return fmt.Errorf("failed to generate random sru: %w", err) -// } -// sru *= 1024 * 1024 * 1024 // 100GB -> 30TB - -// cru, err := rnd(4, 128) -// if err != nil { -// return fmt.Errorf("failed to generate random cru: %w", err) -// } - -// up := flip(nodeUpRatio) -// periodFromLatestUpdate, err := rnd(60*40*3, 60*60*24*30*12) -// if err != nil { -// return fmt.Errorf("failed to generate random period from latest update: %w", err) -// } -// updatedAt := time.Now().Unix() - int64(periodFromLatestUpdate) - -// if up { -// periodFromLatestUpdate, err = rnd(0, 60*40*1) -// if err != nil { -// return fmt.Errorf("failed to generate period from latest update: %w", err) -// } -// updatedAt = time.Now().Unix() - int64(periodFromLatestUpdate) -// } - -// nodesMRU[i] = mru - max(2*uint64(gridtypes.Gigabyte), mru/10) -// nodesSRU[i] = sru - 100*uint64(gridtypes.Gigabyte) -// nodesHRU[i] = hru -// nodeUP[i] = up - -// // location latitude and longitue needs to be castable to decimal -// // if not, the convert_to_decimal function will raise a notice -// // reporting the incident, which downgrades performance -// location := location{ -// id: fmt.Sprintf("location-%d", i), -// longitude: fmt.Sprintf("%d", i), -// latitude: fmt.Sprintf("%d", i), -// } - -// countryIndex := r.Intn(len(countries)) -// cityIndex := r.Intn(len(cities[countries[countryIndex]])) -// node := node{ -// id: fmt.Sprintf("node-%d", i), -// location_id: fmt.Sprintf("location-%d", i), -// node_id: i, -// farm_id: i%100 + 1, -// twin_id: i + 100 + 1, -// country: countries[countryIndex], -// city: cities[countries[countryIndex]][cityIndex], -// uptime: 1000, -// updated_at: uint64(updatedAt), -// created: uint64(time.Now().Unix()), -// created_at: uint64(time.Now().Unix()), -// farming_policy_id: 1, -// grid_version: 3, -// certification: "Diy", -// secure: false, -// virtualized: false, -// serial_number: "", -// power: nodePower{ -// State: powerState[r.Intn(len(powerState))], -// Target: powerState[r.Intn(len(powerState))], -// }, -// extra_fee: 0, -// } - -// total_resources := node_resources_total{ -// id: fmt.Sprintf("total-resources-%d", i), -// hru: hru, -// sru: sru, -// cru: cru, -// mru: mru, -// node_id: fmt.Sprintf("node-%d", i), -// } - -// if _, ok := dedicatedFarms[node.farm_id]; ok { -// availableRentNodes[i] = struct{}{} -// availableRentNodesList = append(availableRentNodesList, i) -// } - -// locationTuple, err := objectToTupleString(location) -// if err != nil { -// return fmt.Errorf("failed to convert location object to tuple string: %w", err) -// } -// locations = append(locations, locationTuple) - -// nodeTuple, err := objectToTupleString(node) -// if err != nil { -// return fmt.Errorf("failed to convert node object to tuple string: %w", err) -// } -// nodes = append(nodes, nodeTuple) - -// totalResourcesTuple, err := objectToTupleString(total_resources) -// if err != nil { -// return fmt.Errorf("failed to convert total resources object to tuple string: %w", err) -// } -// totalResources = append(totalResources, totalResourcesTuple) - -// if flip(.1) { -// publicConfig := public_config{ -// id: fmt.Sprintf("public-config-%d", i), -// ipv4: "185.16.5.2/24", -// gw4: "185.16.5.2", -// ipv6: "::1/64", -// gw6: "::1", -// domain: "hamada.com", -// node_id: fmt.Sprintf("node-%d", i), -// } -// publicConfigTuple, err := objectToTupleString(publicConfig) -// if err != nil { -// return fmt.Errorf("failed to convert public config object to tuple string: %w", err) -// } -// publicConfigs = append(publicConfigs, publicConfigTuple) - -// } -// } - -// if err := insertTuples(db, location{}, locations); err != nil { -// return fmt.Errorf("failed to insert locations: %w", err) -// } - -// if err := insertTuples(db, node{}, nodes); err != nil { -// return fmt.Errorf("failed to isnert nodes: %w", err) -// } - -// if err := insertTuples(db, node_resources_total{}, totalResources); err != nil { -// return fmt.Errorf("failed to insert node resources total: %w", err) -// } - -// if err := insertTuples(db, public_config{}, publicConfigs); err != nil { -// return fmt.Errorf("failed to insert public configs: %w", err) -// } -// fmt.Println("nodes generated") - -// return nil -// } - -// func generateNodeGPUs(db *sql.DB) error { -// var GPUs []string -// vendors := []string{"NVIDIA Corporation", "AMD", "Intel Corporation"} -// devices := []string{"GeForce RTX 3080", "Radeon RX 6800 XT", "Intel Iris Xe MAX"} - -// for i := 0; i <= 10; i++ { -// gpuNum := len(vendors) - 1 -// for j := 0; j <= gpuNum; j++ { -// g := node_gpu{ -// node_twin_id: uint64(i + 100 + 2), // node twin ids start from 102 -// vendor: vendors[j], -// device: devices[j], -// contract: i % 2, -// id: fmt.Sprintf("0000:0e:00.0/1002/744c/%d", j), -// } -// gpuTuple, err := objectToTupleString(g) -// if err != nil { -// return fmt.Errorf("failed to convert gpu object to tuple string: %w", err) -// } -// GPUs = append(GPUs, gpuTuple) -// } -// } - -// if err := insertTuples(db, node_gpu{}, GPUs); err != nil { -// return fmt.Errorf("failed to insert node gpu: %w", err) -// } - -// fmt.Println("node GPUs generated") - -// return nil -// } - -// func generateContracts(db *sql.DB) error { -// rentContractIDStart := 1 - -// var billReports []string - -// rentContractsBillReports, nodeContractIDStart, err := generateRentContracts(db, 1, rentContractIDStart) -// if err != nil { -// return fmt.Errorf("failed to generate rent contracts: %w", err) -// } -// billReports = append(billReports, rentContractsBillReports...) - -// nodeContractsBillReports, nameContractIDStart, err := generateNodeContracts(db, len(billReports)+1, nodeContractIDStart) -// if err != nil { -// return fmt.Errorf("failed to generate node contracts: %w", err) -// } -// billReports = append(billReports, nodeContractsBillReports...) - -// nameContractsBillReports, _, err := generateNameContracts(db, len(billReports)+1, nameContractIDStart) -// if err != nil { -// return fmt.Errorf("failed to generate name contracts: %w", err) -// } -// billReports = append(billReports, nameContractsBillReports...) - -// if err := insertTuples(db, contract_bill_report{}, billReports); err != nil { -// return fmt.Errorf("failed to generate contract bill reports: %w", err) -// } -// return nil -// } - -// func insertTuples(db *sql.DB, tupleObj interface{}, tuples []string) error { - -// if len(tuples) != 0 { -// query := "INSERT INTO " + reflect.Indirect(reflect.ValueOf(tupleObj)).Type().Name() + " (" -// objType := reflect.TypeOf(tupleObj) -// for i := 0; i < objType.NumField(); i++ { -// if i != 0 { -// query += ", " -// } -// query += objType.Field(i).Name -// } - -// query += ") VALUES " - -// query += strings.Join(tuples, ",") -// query += ";" -// if _, err := db.Exec(query); err != nil { -// return fmt.Errorf("failed to insert tuples: %w", err) -// } - -// } -// return nil -// } - -// func updateNodeContractPublicIPs(db *sql.DB, nodeContracts []uint64) error { - -// if len(nodeContracts) != 0 { -// var IDs []string -// for _, contractID := range nodeContracts { -// IDs = append(IDs, fmt.Sprintf("%d", contractID)) - -// } - -// query := "UPDATE node_contract set number_of_public_i_ps = number_of_public_i_ps + 1 WHERE contract_id IN (" -// query += strings.Join(IDs, ",") + ");" -// if _, err := db.Exec(query); err != nil { -// return fmt.Errorf("failed to update node contracts public ips: %w", err) -// } -// } -// return nil -// } - -// func updateNodeContractResourceID(db *sql.DB, min, max int) error { -// query := fmt.Sprintf(`UPDATE node_contract SET resources_used_id = CONCAT('contract-resources-',split_part(id, '-', -1)) -// WHERE CAST(split_part(id, '-', -1) AS INTEGER) BETWEEN %d AND %d;`, min, max) -// if _, err := db.Exec(query); err != nil { -// return fmt.Errorf("failed to update node contract resource id: %w", err) -// } -// return nil -// } func generateData(db *sql.DB, seed int) error { generator := modifiers.NewGenerator(db, seed) - if err := generator.GenerateTwins(); err != nil { - return fmt.Errorf("failed to genrate twins: %w", err) + if err := generator.GenerateTwins(1, modifiers.TwinCount); err != nil { + return fmt.Errorf("failed to generate twins: %w", err) } - if err := generator.GenerateFarms(); err != nil { + if err := generator.GenerateFarms(1, modifiers.FarmCount, 1); err != nil { return fmt.Errorf("failed to generate farms: %w", err) } - if err := generator.GenerateNodes(); err != nil { + if err := generator.GenerateNodes(1, modifiers.NodeCount, 1, modifiers.FarmCount, 1); err != nil { return fmt.Errorf("failed to generate nodes: %w", err) } - if err := generator.GenerateContracts(); err != nil { + if err := generator.GenerateContracts(1, 1, modifiers.NodeContractCount, modifiers.NameContractCount, modifiers.RentContractCount, 1, modifiers.NodeCount); err != nil { return fmt.Errorf("failed to generate contracts: %w", err) } - if err := generator.GeneratePublicIPs(); err != nil { + if err := generator.GeneratePublicIPs(1, modifiers.PublicIPCount); err != nil { return fmt.Errorf("failed to generate public ips: %w", err) } diff --git a/grid-proxy/tools/db/modifiers/generators.go b/grid-proxy/tools/db/modifiers/generators.go index 9124c698c..eaaf74235 100644 --- a/grid-proxy/tools/db/modifiers/generators.go +++ b/grid-proxy/tools/db/modifiers/generators.go @@ -13,10 +13,10 @@ const deleted = "Deleted" const created = "Created" const gracePeriod = "GracePeriod" -func (g *Generator) GenerateTwins() error { +func (g *Generator) GenerateTwins(start, size int) error { var twins []string - for i := uint64(1); i <= uint64(g.TwinCount); i++ { + for i := uint64(start); i < uint64(start+size); i++ { twin := twin{ id: fmt.Sprintf("twin-%d", i), account_id: fmt.Sprintf("account-id-%d", i), @@ -40,17 +40,17 @@ func (g *Generator) GenerateTwins() error { return nil } -func (g *Generator) GenerateFarms() error { +func (g *Generator) GenerateFarms(start, size, twinStart int) error { var farms []string - for i := uint64(1); i <= uint64(g.FarmCount); i++ { + for i := uint64(start); i < uint64(start+size); i++ { farm := farm{ id: fmt.Sprintf("farm-%d", i), farm_id: i, name: fmt.Sprintf("farm-name-%d", i), certification: "Diy", dedicated_farm: flip(.1), - twin_id: i, + twin_id: uint64(twinStart) + (i - uint64(start)), pricing_policy_id: 1, grid_version: 3, stellar_address: "", @@ -75,13 +75,13 @@ func (g *Generator) GenerateFarms() error { return nil } -func (g *Generator) GenerateNodes() error { +func (g *Generator) GenerateNodes(start, size, farmStart, farmSize, twinStart int) error { powerState := []string{"Up", "Down"} var locations []string var nodes []string var totalResources []string var publicConfigs []string - for i := uint64(1); i <= uint64(g.NodeCount); i++ { + for i := uint64(start); i < uint64(start+size); i++ { mru, err := rnd(4, 256) if err != nil { return fmt.Errorf("failed to generate random mru: %w", err) @@ -105,7 +105,7 @@ func (g *Generator) GenerateNodes() error { return fmt.Errorf("failed to generate random cru: %w", err) } - up := flip(g.nodeUpRatio) + up := flip(nodeUpRatio) periodFromLatestUpdate, err := rnd(60*40*3, 60*60*24*30*12) if err != nil { return fmt.Errorf("failed to generate random period from latest update: %w", err) @@ -134,16 +134,18 @@ func (g *Generator) GenerateNodes() error { latitude: fmt.Sprintf("%d", i), } - countryIndex := r.Intn(len(g.countries)) - cityIndex := r.Intn(len(g.cities[g.countries[countryIndex]])) + countryIndex := r.Intn(len(countries)) + cityIndex := r.Intn(len(cities[countries[countryIndex]])) + farmId := r.Intn((farmStart+farmSize)-farmStart) + farmStart + twinId := twinStart + farmSize + (int(i) - start) node := node{ id: fmt.Sprintf("node-%d", i), location_id: fmt.Sprintf("location-%d", i), node_id: i, - farm_id: i%100 + 1, - twin_id: i + 100 + 1, - country: g.countries[countryIndex], - city: g.cities[g.countries[countryIndex]][cityIndex], + farm_id: uint64(farmId), + twin_id: uint64(twinId), + country: countries[countryIndex], + city: cities[countries[countryIndex]][cityIndex], uptime: 1000, updated_at: uint64(updatedAt), created: uint64(time.Now().Unix()), @@ -232,24 +234,22 @@ func (g *Generator) GenerateNodes() error { return nil } -func (g *Generator) GenerateContracts() error { - rentContractIDStart := 1 - +func (g *Generator) GenerateContracts(billStart, contractStart, nodeConCount, nameConCount, rentConCount, nodeStart, nodeSize int) error { var billReports []string - rentContractsBillReports, nodeContractIDStart, err := g.GenerateRentContracts(1, rentContractIDStart) + rentContractsBillReports, nodeContractIDStart, err := g.GenerateRentContracts(billStart, contractStart, rentConCount) if err != nil { return fmt.Errorf("failed to generate rent contracts: %w", err) } billReports = append(billReports, rentContractsBillReports...) - nodeContractsBillReports, nameContractIDStart, err := g.generateNodeContracts(len(billReports)+1, nodeContractIDStart) + nodeContractsBillReports, nameContractIDStart, err := g.generateNodeContracts(len(billReports)+billStart, nodeContractIDStart, nodeConCount, nodeStart, nodeSize) if err != nil { return fmt.Errorf("failed to generate node contracts: %w", err) } billReports = append(billReports, nodeContractsBillReports...) - nameContractsBillReports, _, err := g.GenerateNameContracts(len(billReports)+1, nameContractIDStart) + nameContractsBillReports, _, err := g.GenerateNameContracts(len(billReports)+billStart, nameContractIDStart, nameConCount) if err != nil { return fmt.Errorf("failed to generate name contracts: %w", err) } @@ -261,34 +261,33 @@ func (g *Generator) GenerateContracts() error { return nil } -func (g *Generator) generateNodeContracts(billsStartID, contractsStartID int) ([]string, int, error) { +func (g *Generator) generateNodeContracts(billsStartID, contractsStartID, contractCount, nodeStart, nodeSize int) ([]string, int, error) { + end := contractsStartID + contractCount + var contracts []string var contractResources []string var billingReports []string - for i := uint64(1); i <= uint64(g.NodeContractCount); i++ { - nodeID, err := rnd(1, uint64(g.NodeCount)) - if err != nil { - return nil, contractsStartID, fmt.Errorf("failed to generate random node id: %w", err) - } + for i := contractsStartID; i < end; i++ { + nodeID := uint64(r.Intn((nodeStart+nodeSize)-nodeStart) + nodeStart) state := deleted if g.nodeUP[nodeID] { - if flip(g.contractCreatedRatio) { + if flip(contractCreatedRatio) { state = created } else if flip(0.5) { state = gracePeriod } } - if state != deleted && (g.minContractHRU > g.nodesHRU[nodeID] || g.minContractMRU > g.nodesMRU[nodeID] || g.minContractSRU > g.nodesSRU[nodeID]) { + if state != deleted && (minContractHRU > g.nodesHRU[nodeID] || minContractMRU > g.nodesMRU[nodeID] || minContractSRU > g.nodesSRU[nodeID]) { i-- continue } twinID, err := rnd(1100, 3100) if err != nil { - return nil, contractsStartID, fmt.Errorf("failed to generate random twin id: %w", err) + return nil, i, fmt.Errorf("failed to generate random twin id: %w", err) } if renter, ok := g.renter[nodeID]; ok { @@ -301,79 +300,79 @@ func (g *Generator) generateNodeContracts(billsStartID, contractsStartID int) ([ } contract := node_contract{ - id: fmt.Sprintf("node-contract-%d", contractsStartID), + id: fmt.Sprintf("node-contract-%d", i), twin_id: twinID, - contract_id: uint64(contractsStartID), + contract_id: uint64(i), state: state, created_at: uint64(time.Now().Unix()), node_id: nodeID, - deployment_data: fmt.Sprintf("deployment-data-%d", contractsStartID), - deployment_hash: fmt.Sprintf("deployment-hash-%d", contractsStartID), + deployment_data: fmt.Sprintf("deployment-data-%d", i), + deployment_hash: fmt.Sprintf("deployment-hash-%d", i), number_of_public_i_ps: 0, grid_version: 3, resources_used_id: "", } - cru, err := rnd(g.minContractCRU, g.maxContractCRU) + cru, err := rnd(minContractCRU, maxContractCRU) if err != nil { - return nil, contractsStartID, fmt.Errorf("failed to generate random cru: %w", err) + return nil, i, fmt.Errorf("failed to generate random cru: %w", err) } - hru, err := rnd(g.minContractHRU, min(g.maxContractHRU, g.nodesHRU[nodeID])) + hru, err := rnd(minContractHRU, min(maxContractHRU, g.nodesHRU[nodeID])) if err != nil { - return nil, contractsStartID, fmt.Errorf("failed to generate random hru: %w", err) + return nil, i, fmt.Errorf("failed to generate random hru: %w", err) } - sru, err := rnd(g.minContractSRU, min(g.maxContractSRU, g.nodesSRU[nodeID])) + sru, err := rnd(minContractSRU, min(maxContractSRU, g.nodesSRU[nodeID])) if err != nil { - return nil, contractsStartID, fmt.Errorf("failed to generate random sru: %w", err) + return nil, i, fmt.Errorf("failed to generate random sru: %w", err) } - mru, err := rnd(g.minContractMRU, min(g.maxContractMRU, g.nodesMRU[nodeID])) + mru, err := rnd(minContractMRU, min(maxContractMRU, g.nodesMRU[nodeID])) if err != nil { - return nil, contractsStartID, fmt.Errorf("failed to generate random mru: %w", err) + return nil, i, fmt.Errorf("failed to generate random mru: %w", err) } contract_resources := contract_resources{ - id: fmt.Sprintf("contract-resources-%d", contractsStartID), + id: fmt.Sprintf("contract-resources-%d", i), hru: hru, sru: sru, cru: cru, mru: mru, - contract_id: fmt.Sprintf("node-contract-%d", contractsStartID), + contract_id: fmt.Sprintf("node-contract-%d", i), } if contract.state != deleted { g.nodesHRU[nodeID] -= hru g.nodesSRU[nodeID] -= sru g.nodesMRU[nodeID] -= mru - g.createdNodeContracts = append(g.createdNodeContracts, uint64(contractsStartID)) + g.createdNodeContracts = append(g.createdNodeContracts, uint64(i)) } contractTuple, err := objectToTupleString(contract) if err != nil { - return nil, contractsStartID, fmt.Errorf("failed to convert contract object to tuple string: %w", err) + return nil, i, fmt.Errorf("failed to convert contract object to tuple string: %w", err) } contracts = append(contracts, contractTuple) contractResourcesTuple, err := objectToTupleString(contract_resources) if err != nil { - return nil, contractsStartID, fmt.Errorf("failed to convert contract resources object to tuple string: %w", err) + return nil, i, fmt.Errorf("failed to convert contract resources object to tuple string: %w", err) } contractResources = append(contractResources, contractResourcesTuple) billings, err := rnd(0, 10) if err != nil { - return nil, contractsStartID, fmt.Errorf("failed to generate random billing count: %w", err) + return nil, i, fmt.Errorf("failed to generate random billing count: %w", err) } amountBilled, err := rnd(0, 100000) if err != nil { - return nil, contractsStartID, fmt.Errorf("failed to generate random amount billed: %w", err) + return nil, i, fmt.Errorf("failed to generate random amount billed: %w", err) } for j := uint64(0); j < billings; j++ { billing := contract_bill_report{ id: fmt.Sprintf("contract-bill-report-%d", billsStartID), - contract_id: uint64(contractsStartID), + contract_id: uint64(i), discount_received: "Default", amount_billed: amountBilled, timestamp: uint64(time.Now().UnixNano()), @@ -382,28 +381,27 @@ func (g *Generator) generateNodeContracts(billsStartID, contractsStartID int) ([ billTuple, err := objectToTupleString(billing) if err != nil { - return nil, contractsStartID, fmt.Errorf("failed to convert contract bill report object to tuple string: %w", err) + return nil, i, fmt.Errorf("failed to convert contract bill report object to tuple string: %w", err) } billingReports = append(billingReports, billTuple) } - contractsStartID++ } if err := g.insertTuples(node_contract{}, contracts); err != nil { - return nil, contractsStartID, fmt.Errorf("failed to insert node contracts: %w", err) + return nil, end, fmt.Errorf("failed to insert node contracts: %w", err) } if err := g.insertTuples(contract_resources{}, contractResources); err != nil { - return nil, contractsStartID, fmt.Errorf("failed to insert contract resources: %w", err) + return nil, end, fmt.Errorf("failed to insert contract resources: %w", err) } - if err := g.updateNodeContractResourceID(contractsStartID-int(g.NodeContractCount), contractsStartID); err != nil { - return nil, contractsStartID, fmt.Errorf("failed to update node contract resources id: %w", err) + if err := g.updateNodeContractResourceID(contractsStartID, end); err != nil { + return nil, end, fmt.Errorf("failed to update node contract resources id: %w", err) } fmt.Println("node contracts generated") - return billingReports, contractsStartID, nil + return billingReports, end, nil } func (g *Generator) updateNodeContractResourceID(min, max int) error { @@ -415,18 +413,19 @@ func (g *Generator) updateNodeContractResourceID(min, max int) error { return nil } -func (g *Generator) GenerateNameContracts(billsStartID, contractsStartID int) ([]string, int, error) { +func (g *Generator) GenerateNameContracts(billsStartID, contractsStartID, contractCount int) ([]string, int, error) { + end := contractsStartID + contractCount var contracts []string var billReports []string - for i := uint64(1); i <= uint64(g.NameContractCount); i++ { - nodeID, err := rnd(1, uint64(g.NodeCount)) + for i := contractsStartID; i < end; i++ { + nodeID, err := rnd(1, uint64(NodeCount)) if err != nil { - return nil, contractsStartID, fmt.Errorf("failed to generate random node id: %w", err) + return nil, i, fmt.Errorf("failed to generate random node id: %w", err) } state := deleted if g.nodeUP[nodeID] { - if flip(g.contractCreatedRatio) { + if flip(contractCreatedRatio) { state = created } else if flip(0.5) { state = gracePeriod @@ -435,7 +434,7 @@ func (g *Generator) GenerateNameContracts(billsStartID, contractsStartID int) ([ twinID, err := rnd(1100, 3100) if err != nil { - return nil, contractsStartID, fmt.Errorf("failed to generate random twin id: %w", err) + return nil, i, fmt.Errorf("failed to generate random twin id: %w", err) } if renter, ok := g.renter[nodeID]; ok { @@ -448,9 +447,9 @@ func (g *Generator) GenerateNameContracts(billsStartID, contractsStartID int) ([ } contract := name_contract{ - id: fmt.Sprintf("name-contract-%d", contractsStartID), + id: fmt.Sprintf("name-contract-%d", i), twin_id: twinID, - contract_id: uint64(contractsStartID), + contract_id: uint64(i), state: state, created_at: uint64(time.Now().Unix()), grid_version: 3, @@ -459,23 +458,23 @@ func (g *Generator) GenerateNameContracts(billsStartID, contractsStartID int) ([ contractTuple, err := objectToTupleString(contract) if err != nil { - return nil, contractsStartID, fmt.Errorf("failed to convert contract object to tuple string: %w", err) + return nil, i, fmt.Errorf("failed to convert contract object to tuple string: %w", err) } contracts = append(contracts, contractTuple) billings, err := rnd(0, 10) if err != nil { - return nil, contractsStartID, fmt.Errorf("failed to generate random billings count: %w", err) + return nil, i, fmt.Errorf("failed to generate random billings count: %w", err) } amountBilled, err := rnd(0, 100000) if err != nil { - return nil, contractsStartID, fmt.Errorf("failed to generate random amount billed: %w", err) + return nil, i, fmt.Errorf("failed to generate random amount billed: %w", err) } for j := uint64(0); j < billings; j++ { billing := contract_bill_report{ id: fmt.Sprintf("contract-bill-report-%d", billsStartID), - contract_id: uint64(contractsStartID), + contract_id: uint64(i), discount_received: "Default", amount_billed: amountBilled, timestamp: uint64(time.Now().UnixNano()), @@ -484,29 +483,30 @@ func (g *Generator) GenerateNameContracts(billsStartID, contractsStartID int) ([ billTuple, err := objectToTupleString(billing) if err != nil { - return nil, contractsStartID, fmt.Errorf("failed to convert contract bill report object to tuple string: %w", err) + return nil, i, fmt.Errorf("failed to convert contract bill report object to tuple string: %w", err) } billReports = append(billReports, billTuple) } - contractsStartID++ } if err := g.insertTuples(name_contract{}, contracts); err != nil { - return nil, contractsStartID, fmt.Errorf("failed to insert name contracts: %w", err) + return nil, end, fmt.Errorf("failed to insert name contracts: %w", err) } fmt.Println("name contracts generated") - return billReports, contractsStartID, nil + return billReports, end, nil } -func (g *Generator) GenerateRentContracts(billsStartID, contractsStartID int) ([]string, int, error) { +func (g *Generator) GenerateRentContracts(billsStart, contractStart, rentConCount int) ([]string, int, error) { + end := contractStart + rentConCount + var contracts []string var billReports []string - for i := uint64(1); i <= uint64(g.RentContractCount); i++ { + for i := contractStart; i < end; i++ { nl, nodeID, err := popRandom(g.availableRentNodesList) if err != nil { - return nil, contractsStartID, fmt.Errorf("failed to select random element from the gives slice: %w", err) + return nil, i, fmt.Errorf("failed to select random element from the given slice: %w", err) } g.availableRentNodesList = nl @@ -522,13 +522,13 @@ func (g *Generator) GenerateRentContracts(billsStartID, contractsStartID int) ([ twinID, err := rnd(1100, 3100) if err != nil { - return nil, contractsStartID, fmt.Errorf("failed to generate a random twin id: %w", err) + return nil, i, fmt.Errorf("failed to generate a random twin id: %w", err) } contract := rent_contract{ - id: fmt.Sprintf("rent-contract-%d", contractsStartID), + id: fmt.Sprintf("rent-contract-%d", i), twin_id: twinID, - contract_id: uint64(contractsStartID), + contract_id: uint64(i), state: state, created_at: uint64(time.Now().Unix()), node_id: nodeID, @@ -541,57 +541,56 @@ func (g *Generator) GenerateRentContracts(billsStartID, contractsStartID int) ([ contractTuple, err := objectToTupleString(contract) if err != nil { - return nil, contractsStartID, fmt.Errorf("failed to convert contract object to tuple string: %w", err) + return nil, i, fmt.Errorf("failed to convert contract object to tuple string: %w", err) } contracts = append(contracts, contractTuple) billings, err := rnd(0, 10) if err != nil { - return nil, contractsStartID, fmt.Errorf("failed to generate billings count: %w", err) + return nil, i, fmt.Errorf("failed to generate billings count: %w", err) } amountBilled, err := rnd(0, 100000) if err != nil { - return nil, contractsStartID, fmt.Errorf("failed to generate amount billed: %w", err) + return nil, i, fmt.Errorf("failed to generate amount billed: %w", err) } for j := uint64(0); j < billings; j++ { billing := contract_bill_report{ - id: fmt.Sprintf("contract-bill-report-%d", billsStartID), - contract_id: uint64(contractsStartID), + id: fmt.Sprintf("contract-bill-report-%d", billsStart), + contract_id: uint64(i), discount_received: "Default", amount_billed: amountBilled, timestamp: uint64(time.Now().UnixNano()), } - billsStartID++ + billsStart++ billTuple, err := objectToTupleString(billing) if err != nil { - return nil, contractsStartID, fmt.Errorf("failed to convert contract bill report object to tuple string: %w", err) + return nil, i, fmt.Errorf("failed to convert contract bill report object to tuple string: %w", err) } billReports = append(billReports, billTuple) } - contractsStartID++ } if err := g.insertTuples(rent_contract{}, contracts); err != nil { - return nil, contractsStartID, fmt.Errorf("failed to insert rent contracts: %w", err) + return nil, end, fmt.Errorf("failed to insert rent contracts: %w", err) } fmt.Println("rent contracts generated") - return billReports, contractsStartID, nil + return billReports, end, nil } -func (g *Generator) GeneratePublicIPs() error { +func (g *Generator) GeneratePublicIPs(start, size int) error { var publicIPs []string var nodeContracts []uint64 - for i := uint64(1); i <= uint64(g.PublicIPCount); i++ { + for i := uint64(start); i < uint64(start+size); i++ { contract_id := uint64(0) - if flip(g.usedPublicIPsRatio) { + if flip(usedPublicIPsRatio) { idx, err := rnd(0, uint64(len(g.createdNodeContracts))-1) if err != nil { return fmt.Errorf("failed to generate random index: %w", err) @@ -599,7 +598,7 @@ func (g *Generator) GeneratePublicIPs() error { contract_id = g.createdNodeContracts[idx] } ip := randomIPv4() - farmID, err := rnd(1, uint64(g.FarmCount)) + farmID, err := rnd(1, uint64(FarmCount)) if err != nil { return fmt.Errorf("failed to generate random farm id: %w", err) } @@ -659,7 +658,7 @@ func (g *Generator) GenerateNodeGPUs() error { gpuNum := len(vendors) - 1 for j := 0; j <= gpuNum; j++ { g := node_gpu{ - node_twin_id: uint64(i + 100 + 2), // node twin ids start from 102 + node_twin_id: uint64(i + FarmCount + 2), // node twin ids start from 102 vendor: vendors[j], device: devices[j], contract: i % 2, @@ -685,13 +684,13 @@ func (g *Generator) GenerateNodeGPUs() error { func (g *Generator) GenerateCountries() error { var countriesValues []string index := 0 - for countryName, region := range g.regions { + for countryName, region := range regions { index++ country := country{ id: fmt.Sprintf("country-%d", index), country_id: uint64(index), name: countryName, - code: g.countriesCodes[countryName], + code: countriesCodes[countryName], region: "unknown", subregion: region, lat: fmt.Sprintf("%d", 0), diff --git a/grid-proxy/tools/db/modifiers/modifiers.go b/grid-proxy/tools/db/modifiers/modifiers.go index c7e9098a3..5708225f8 100644 --- a/grid-proxy/tools/db/modifiers/modifiers.go +++ b/grid-proxy/tools/db/modifiers/modifiers.go @@ -11,8 +11,8 @@ func (g *Generator) UpdateNodeCountry() error { query := "" for i := 0; i < updatesCount; i++ { - nodeId := r.Intn(int(g.NodeCount)) + 1 - country := g.countries[r.Intn(len(g.countries))] + nodeId := r.Intn(int(NodeCount)) + 1 + country := countries[r.Intn(len(countries))] query += fmt.Sprintf("UPDATE node SET country = '%s' WHERE node_id = %d;", country, nodeId) } @@ -27,7 +27,7 @@ func (g *Generator) UpdateNodeTotalResources() error { padding := 1 * 1024 * 1024 * 1024 query := "" for i := 0; i < updatesCount; i++ { - nodeId := r.Intn(int(g.NodeCount)) + 1 + nodeId := r.Intn(int(NodeCount)) + 1 cru := 10 hru := g.nodesHRU[uint64(nodeId)] + uint64(padding) @@ -47,12 +47,12 @@ func (g *Generator) UpdateContractResources() error { updatesCount := 10 query := "" for i := 0; i < updatesCount; i++ { - contractId := r.Intn(int(g.NodeContractCount)) + 1 + contractId := r.Intn(int(NodeContractCount)) + 1 - cru := g.minContractCRU - hru := g.minContractHRU - sru := g.minContractSRU - mru := g.minContractMRU + cru := minContractCRU + hru := minContractHRU + sru := minContractSRU + mru := minContractMRU query += fmt.Sprintf("UPDATE contract_resources SET cru = %d, hru = %d, mru = %d, sru = %d WHERE contract_id = 'node-contract-%d';", cru, hru, mru, sru, contractId) } @@ -86,7 +86,7 @@ func (g *Generator) UpdateRentContract() error { states := []string{"Deleted", "GracePeriod"} for i := 0; i < updatesCount; i++ { - contractId := r.Intn(int(g.RentContractCount)) + 1 + contractId := r.Intn(int(RentContractCount)) + 1 state := states[r.Intn(2)] query += fmt.Sprintf("UPDATE rent_contract SET state = '%s' WHERE contract_id = %d;", state, contractId) } @@ -104,7 +104,7 @@ func (g *Generator) UpdatePublicIps() error { for i := 0; i < updatesCount; i++ { idx := r.Intn(len(g.createdNodeContracts)) contractID := g.createdNodeContracts[idx] - publicIPID := r.Intn(int(g.PublicIPCount)) + publicIPID := r.Intn(int(PublicIPCount)) query += fmt.Sprintf("UPDATE public_ip SET contract_id = (CASE WHEN contract_id = 0 THEN %d ELSE 0 END) WHERE id = 'public-ip-%d';", contractID, publicIPID) } @@ -125,8 +125,8 @@ func (g *Generator) DeleteNodes() error { query := "" for i := 0; i < updatesCount; i++ { - nodeID := r.Intn(int(g.NodeCount)) + 1 - g.NodeCount-- + nodeID := r.Intn(int(NodeCount)) + 1 + // NodeCount-- query += fmt.Sprintf("UPDATE public_ip SET contract_id = 0 WHERE contract_id IN (SELECT contract_id FROM node_contract WHERE node_id = %d);", nodeID) query += fmt.Sprintf("UPDATE node_contract SET state = 'Deleted' WHERE node_id = %d;", nodeID) @@ -145,16 +145,16 @@ func (g *Generator) DeletePublicIps() error { maxDeleteCount := r.Intn(10) + 1 query := fmt.Sprintf("DELETE FROM public_ip WHERE id in (SELECT id FROM public_ip WHERE contract_id = 0 LIMIT %d);", maxDeleteCount) - res, err := g.db.Exec(query) + _, err := g.db.Exec(query) if err != nil { return fmt.Errorf("failed to delete public ips: %w", err) } - rowsAffercted, err := res.RowsAffected() - if err != nil { - return fmt.Errorf("failed to get rows affected by public ips delete: %w", err) - } - g.PublicIPCount -= uint32(rowsAffercted) + // rowsAffercted, err := res.RowsAffected() + // if err != nil { + // return fmt.Errorf("failed to get rows affected by public ips delete: %w", err) + // } + // PublicIPCount -= uint32(rowsAffercted) log.Debug().Str("query", query).Msg("delete public ips") diff --git a/grid-proxy/tools/db/modifiers/types.go b/grid-proxy/tools/db/modifiers/types.go index f472fded1..0428abd85 100644 --- a/grid-proxy/tools/db/modifiers/types.go +++ b/grid-proxy/tools/db/modifiers/types.go @@ -7,31 +7,52 @@ import ( var r *rand.Rand -type Generator struct { - db *sql.DB - - NodeCount uint32 - FarmCount uint32 - TwinCount uint32 - NodeContractCount uint32 - RentContractCount uint32 - NameContractCount uint32 - PublicIPCount uint32 - NormalUsersCount uint32 - - maxContractHRU uint64 - maxContractSRU uint64 - maxContractMRU uint64 - maxContractCRU uint64 - minContractHRU uint64 - minContractSRU uint64 - minContractMRU uint64 - minContractCRU uint64 +const ( + contractCreatedRatio = .1 // from devnet + usedPublicIPsRatio = .9 + nodeUpRatio = .5 + NodeCount = 6000 + FarmCount = 600 + NormalUsersCount = 6000 + PublicIPCount = 1000 + TwinCount = 6000 + 600 + 6000 // nodes + farms + normal users + NodeContractCount = 9000 + RentContractCount = 100 + NameContractCount = 300 + maxContractHRU = 1024 * 1024 * 1024 * 300 + maxContractSRU = 1024 * 1024 * 1024 * 300 + maxContractMRU = 1024 * 1024 * 1024 * 16 + maxContractCRU = 16 + minContractHRU = 0 + minContractSRU = 1024 * 1024 * 256 + minContractMRU = 1024 * 1024 * 256 + minContractCRU = 1 +) - contractCreatedRatio float32 - usedPublicIPsRatio float32 - nodeUpRatio float32 +var ( + countries = []string{"Belgium", "United States", "Egypt", "United Kingdom"} + regions = map[string]string{ + "Belgium": "Europe", + "United States": "Americas", + "Egypt": "Africa", + "United Kingdom": "Europe", + } + countriesCodes = map[string]string{ + "Belgium": "BG", + "United States": "US", + "Egypt": "EG", + "United Kingdom": "UK", + } + cities = map[string][]string{ + "Belgium": {"Brussels", "Antwerp", "Ghent", "Charleroi"}, + "United States": {"New York", "Chicago", "Los Angeles", "San Francisco"}, + "Egypt": {"Cairo", "Giza", "October", "Nasr City"}, + "United Kingdom": {"London", "Liverpool", "Manchester", "Cambridge"}, + } +) +type Generator struct { + db *sql.DB nodesMRU map[uint64]uint64 nodesSRU map[uint64]uint64 nodesHRU map[uint64]uint64 @@ -41,37 +62,13 @@ type Generator struct { availableRentNodes map[uint64]struct{} availableRentNodesList []uint64 renter map[uint64]uint64 - countries []string - regions map[string]string - countriesCodes map[string]string - cities map[string][]string } func NewGenerator(db *sql.DB, seed int) Generator { r = rand.New(rand.NewSource(int64(seed))) return Generator{ - db: db, - contractCreatedRatio: .1, // from devnet - usedPublicIPsRatio: .9, - nodeUpRatio: .5, - NodeCount: 6000, - FarmCount: 600, - NormalUsersCount: 6000, - PublicIPCount: 1000, - TwinCount: 6000 + 600 + 6000, // nodes + farms + normal users - NodeContractCount: 9000, - RentContractCount: 100, - NameContractCount: 300, - - maxContractHRU: 1024 * 1024 * 1024 * 300, - maxContractSRU: 1024 * 1024 * 1024 * 300, - maxContractMRU: 1024 * 1024 * 1024 * 16, - maxContractCRU: 16, - minContractHRU: 0, - minContractSRU: 1024 * 1024 * 256, - minContractMRU: 1024 * 1024 * 256, - minContractCRU: 1, + db: db, nodesMRU: make(map[uint64]uint64), nodesSRU: make(map[uint64]uint64), nodesHRU: make(map[uint64]uint64), @@ -81,25 +78,6 @@ func NewGenerator(db *sql.DB, seed int) Generator { availableRentNodes: make(map[uint64]struct{}), availableRentNodesList: make([]uint64, 0), renter: make(map[uint64]uint64), - countries: []string{"Belgium", "United States", "Egypt", "United Kingdom"}, - regions: map[string]string{ - "Belgium": "Europe", - "United States": "Americas", - "Egypt": "Africa", - "United Kingdom": "Europe", - }, - countriesCodes: map[string]string{ - "Belgium": "BG", - "United States": "US", - "Egypt": "EG", - "United Kingdom": "UK", - }, - cities: map[string][]string{ - "Belgium": {"Brussels", "Antwerp", "Ghent", "Charleroi"}, - "United States": {"New York", "Chicago", "Los Angeles", "San Francisco"}, - "Egypt": {"Cairo", "Giza", "October", "Nasr City"}, - "United Kingdom": {"London", "Liverpool", "Manchester", "Cambridge"}, - }, } } From 985960b6fbcaf9bcef256ee5e9ec8d94e3c0f389 Mon Sep 17 00:00:00 2001 From: mario Date: Mon, 25 Dec 2023 14:35:14 +0200 Subject: [PATCH 30/39] wip: add update/delete modifiers and fixes in setup.sql Co-authored-by: omarabdulasis --- grid-proxy/internal/explorer/db/postgres.go | 10 +- grid-proxy/internal/explorer/db/setup.sql | 74 ++++++----- grid-proxy/tests/queries/main_test.go | 58 +++++---- grid-proxy/tests/queries/mock_client/farms.go | 6 +- grid-proxy/tests/queries/mock_client/nodes.go | 6 +- grid-proxy/tests/queries/mock_client/utils.go | 24 ++-- grid-proxy/tests/queries/node_test.go | 26 +++- grid-proxy/tests/queries/utils.go | 24 ++-- grid-proxy/tools/db/generate.go | 2 +- grid-proxy/tools/db/modifiers/generators.go | 120 ++++++++++++++++-- grid-proxy/tools/db/modifiers/modifiers.go | 28 ++-- 11 files changed, 253 insertions(+), 125 deletions(-) diff --git a/grid-proxy/internal/explorer/db/postgres.go b/grid-proxy/internal/explorer/db/postgres.go index 14d3a3804..f5f503d9e 100644 --- a/grid-proxy/internal/explorer/db/postgres.go +++ b/grid-proxy/internal/explorer/db/postgres.go @@ -328,7 +328,7 @@ func (d *PostgresDatabase) farmTableQuery(ctx context.Context, filter types.Farm "farm.certification", "farm.stellar_address", "farm.dedicated_farm as dedicated", - "public_ips_cache.ips as public_ips", + "COALESCE(public_ips_cache.ips, '[]') as public_ips", ). Joins( "LEFT JOIN public_ips_cache ON public_ips_cache.farm_id = farm.farm_id", @@ -350,7 +350,7 @@ func (d *PostgresDatabase) farmTableQuery(ctx context.Context, filter types.Farm farm.certification, farm.stellar_address, farm.dedicated_farm, - public_ips_cache.ips + COALESCE(public_ips_cache.ips, '[]') `) } @@ -409,10 +409,10 @@ func (d *PostgresDatabase) GetFarms(ctx context.Context, filter types.FarmFilter } if filter.FreeIPs != nil { - q = q.Where("public_ips_cache.free_ips >= ?", *filter.FreeIPs) + q = q.Where("COALESCE(public_ips_cache.free_ips, 0) >= ?", *filter.FreeIPs) } if filter.TotalIPs != nil { - q = q.Where("public_ips_cache.total_ips >= ?", *filter.TotalIPs) + q = q.Where("COALESCE(public_ips_cache.total_ips, 0) >= ?", *filter.TotalIPs) } if filter.StellarAddress != nil { q = q.Where("farm.stellar_address = ?", *filter.StellarAddress) @@ -573,7 +573,7 @@ func (d *PostgresDatabase) GetNodes(ctx context.Context, filter types.NodeFilter q = q.Where("farm.name ILIKE '%' || ? || '%'", *filter.FarmNameContains) } if filter.FreeIPs != nil { - q = q.Where("public_ips_cache.free_ips >= ?", *filter.FreeIPs) + q = q.Where("COALESCE(public_ips_cache.free_ips, 0) >= ?", *filter.FreeIPs) } if filter.IPv4 != nil { q = q.Where("(public_config.ipv4 IS NULL) != ?", *filter.IPv4) diff --git a/grid-proxy/internal/explorer/db/setup.sql b/grid-proxy/internal/explorer/db/setup.sql index f081d86a6..ba7a9c8d5 100644 --- a/grid-proxy/internal/explorer/db/setup.sql +++ b/grid-proxy/internal/explorer/db/setup.sql @@ -83,7 +83,7 @@ SELECT ) as used_cru, rent_contract.twin_id as renter, rent_contract.contract_id as rent_contract_id, - count(node_contract.contract_id) as node_contract_count, + count(node_contract.contract_id) as node_contracts_count, COALESCE(node_gpu.node_gpu_count, 0) as node_gpu_count, country.name as country, country.subregion as region @@ -330,6 +330,7 @@ BEGIN free_hru = free_hru - (NEW.hru - COALESCE(OLD.hru, 0)), free_sru = free_sru - (NEW.sru - COALESCE(OLD.sru, 0)) WHERE + (SELECT state from node_contract where id = NEW.contract_id) != 'Deleted' AND resources_cache.node_id = ( SELECT node_id FROM node_contract WHERE node_contract.id = NEW.contract_id ); @@ -347,7 +348,7 @@ CREATE OR REPLACE TRIGGER contract_resources_trigger /* Node contract trigger - - Insert new contract > increment resources_cache node_contract_count + - Insert new contract > increment resources_cache node_contracts_count - Update contract state to 'Deleted' > decrement used an increment free fields on resources_cache */ CREATE OR REPLACE FUNCTION node_contract_upsert() RETURNS TRIGGER AS @@ -368,8 +369,8 @@ BEGIN resources_cache.free_hru + hru, resources_cache.node_contracts_count - 1 FROM resources_cache - LEFT JOIN contract_resources ON contract_resources.contract_id = NEW.id - WHERE resources_cache.node_id = NEW.node_id + LEFT JOIN contract_resources ON contract_resources.contract_id = NEW.id + WHERE resources_cache.node_id = NEW.node_id ) WHERE resources_cache.node_id = NEW.node_id; EXCEPTION WHEN OTHERS THEN @@ -383,7 +384,7 @@ BEGIN WHERE resources_cache.node_id = NEW.node_id; EXCEPTION WHEN OTHERS THEN - RAISE NOTICE 'Error incrementing node_contract_count %', SQLERRM; + RAISE NOTICE 'Error incrementing node_contracts_count %', SQLERRM; END; END IF; RETURN NULL; @@ -443,21 +444,6 @@ CREATE OR REPLACE TRIGGER rent_contract_trigger CREATE OR REPLACE FUNCTION public_ip_upsert() RETURNS TRIGGER AS $$ BEGIN - - BEGIN - IF TG_OP = 'INSERT' AND NEW.farm_id NOT IN (select farm_id from public_ips_cache) THEN - INSERT INTO public_ips_cache VALUES( - NEW.farm_id, - 0, - 0, - '[]' - ); - END IF; - EXCEPTION - WHEN OTHERS THEN - RAISE NOTICE 'Error inserting public_ips_cache record %s', SQLERRM; - END; - BEGIN UPDATE public_ips_cache @@ -467,7 +453,7 @@ BEGIN WHEN TG_OP != 'DELETE' AND NEW.contract_id = 0 THEN 1 -- handles deletion/update by reserving ip - WHEN COALESCE(OLD.contract_id, 0) = 0 + WHEN TG_OP != 'INSERT' AND OLD.contract_id = 0 THEN -1 -- handles delete reserved ips ELSE 0 @@ -507,17 +493,6 @@ BEGIN RAISE NOTICE 'Error reflect public_ips changes %s', SQLERRM; END; - BEGIN - IF TG_OP = 'DELETE' THEN - -- delete if exists - DELETE FROM public_ips_cache WHERE public_ips_cache.total_ips = 0 AND - public_ips_cache.farm_id = (select farm_id from farm where farm.id = OLD.farm_id); - END IF; - EXCEPTION - WHEN OTHERS THEN - RAISE NOTICE 'Error deleting public_ips_cache record %s', SQLERRM; - END; - RETURN NULL; END; $$ LANGUAGE plpgsql; @@ -526,4 +501,39 @@ CREATE OR REPLACE TRIGGER public_ip_trigger AFTER INSERT OR DELETE OR UPDATE OF contract_id ON public_ip FOR EACH ROW EXECUTE PROCEDURE public_ip_upsert(); + +CREATE OR REPLACE FUNCTION farm_insert_delete() RETURNS TRIGGER AS +$$ +BEGIN + + IF TG_OP = 'INSERT' THEN + BEGIN + INSERT INTO public_ips_cache VALUES( + NEW.farm_id, + 0, + 0, + '[]' + ); + EXCEPTION + WHEN OTHERS THEN + RAISE NOTICE 'Error inserting public_ips_cache record %s', SQLERRM; + END; + + ELSIF (TG_OP = 'DELETE') THEN + BEGIN + DELETE FROM public_ips_cache WHERE public_ips_cache.farm_id = OLD.farm_id; + EXCEPTION + WHEN OTHERS THEN + RAISE NOTICE 'Error deleting public_ips_cache record %s', SQLERRM; + END; + END IF; + +RETURN NULL; +END; +$$ LANGUAGE plpgsql; + +CREATE OR REPLACE TRIGGER farms_trigger + AFTER INSERT OR DELETE ON farm FOR EACH ROW + EXECUTE PROCEDURE farm_insert_delete(); + COMMIT; \ No newline at end of file diff --git a/grid-proxy/tests/queries/main_test.go b/grid-proxy/tests/queries/main_test.go index 8f008dfc6..5f8115fbb 100644 --- a/grid-proxy/tests/queries/main_test.go +++ b/grid-proxy/tests/queries/main_test.go @@ -127,63 +127,65 @@ func modifyDataToFireTriggers(db *sql.DB, data mock.DBData) error { twinStart := len(data.Twins) + 1 farmStart := len(data.Farms) + 1 + farmSize := 100 nodeStart := len(data.Nodes) + 1 + nodeSize := 600 contractStart := len(data.NodeContracts) + len(data.RentContracts) + len(data.NameContracts) + 1 billStart := data.BillReports + 1 publicIpStart := len(data.PublicIPs) + 1 size := 10 // insertion - if err := generator.GenerateFarms(farmStart, 100, twinStart); err != nil { + if err := generator.GenerateFarms(farmStart, farmSize, twinStart); err != nil { return fmt.Errorf("failed to generate farms: %w", err) } - if err := generator.GenerateNodes(nodeStart, 600, farmStart, 100, twinStart); err != nil { + if err := generator.GenerateNodes(nodeStart, nodeSize, farmStart, farmSize, twinStart); err != nil { return fmt.Errorf("failed to generate nodes: %w", err) } // rentCount is 1 because the generate method have .1 percent of 10 farms to be dedicated - if err := generator.GenerateContracts(int(billStart), contractStart, 50, size, 1, nodeStart, 600); err != nil { + if err := generator.GenerateContracts(int(billStart), contractStart, 50, size, 1, nodeStart, nodeSize); err != nil { return fmt.Errorf("failed to generate contracts: %w", err) } - if err := generator.GeneratePublicIPs(publicIpStart, size); err != nil { + if err := generator.GeneratePublicIPs(publicIpStart, size, farmStart, farmSize); err != nil { return fmt.Errorf("failed to generate public ips: %w", err) } // // updates - // if err := generator.UpdateNodeCountry(); err != nil { - // return fmt.Errorf("failed to update node country: %w", err) - // } + if err := generator.UpdateNodeCountry(); err != nil { + return fmt.Errorf("failed to update node country: %w", err) + } - // if err := generator.UpdateNodeTotalResources(); err != nil { - // return fmt.Errorf("failed to update node total resources: %w", err) - // } + if err := generator.UpdateNodeTotalResources(); err != nil { + return fmt.Errorf("failed to update node total resources: %w", err) + } - // if err := generator.UpdateContractResources(); err != nil { - // return fmt.Errorf("failed to update contract resources: %w", err) - // } + if err := generator.UpdateContractResources(); err != nil { + return fmt.Errorf("failed to update contract resources: %w", err) + } - // if err := generator.UpdateNodeContractState(); err != nil { - // return fmt.Errorf("failed to update node node contract: %w", err) - // } + if err := generator.UpdateNodeContractState(); err != nil { + return fmt.Errorf("failed to update node node contract: %w", err) + } - // if err := generator.UpdateRentContract(); err != nil { - // return fmt.Errorf("failed to update rent contract: %w", err) - // } + if err := generator.UpdateRentContract(); err != nil { + return fmt.Errorf("failed to update rent contract: %w", err) + } - // if err := generator.UpdatePublicIps(); err != nil { - // return fmt.Errorf("failed to update public ips: %w", err) - // } + if err := generator.UpdatePublicIps(); err != nil { + return fmt.Errorf("failed to update public ips: %w", err) + } // // deletions - // if err := generator.DeleteNodes(); err != nil { - // return fmt.Errorf("failed to delete node: %w", err) - // } + if err := generator.DeleteNodes(len(data.Nodes) + nodeSize); err != nil { + return fmt.Errorf("failed to delete node: %w", err) + } - // if err := generator.DeletePublicIps(); err != nil { - // return fmt.Errorf("failed to delete node: %w", err) - // } + if err := generator.DeletePublicIps(); err != nil { + return fmt.Errorf("failed to delete node: %w", err) + } return nil } diff --git a/grid-proxy/tests/queries/mock_client/farms.go b/grid-proxy/tests/queries/mock_client/farms.go index 79ed57f34..4e9e19925 100644 --- a/grid-proxy/tests/queries/mock_client/farms.go +++ b/grid-proxy/tests/queries/mock_client/farms.go @@ -132,15 +132,15 @@ func (f *Farm) satisfyFarmNodesFilter(data *DBData, filter types.FarmFilter) boo total := data.NodeTotalResources[node.NodeID] used := data.NodeUsedResources[node.NodeID] free := calcFreeResources(total, used) - if filter.NodeFreeHRU != nil && free.HRU < *filter.NodeFreeHRU { + if filter.NodeFreeHRU != nil && int64(free.HRU) < int64(*filter.NodeFreeHRU) { continue } - if filter.NodeFreeMRU != nil && free.MRU < *filter.NodeFreeMRU { + if filter.NodeFreeMRU != nil && int64(free.MRU) < int64(*filter.NodeFreeMRU) { continue } - if filter.NodeFreeSRU != nil && free.SRU < *filter.NodeFreeSRU { + if filter.NodeFreeSRU != nil && int64(free.SRU) < int64(*filter.NodeFreeSRU) { continue } diff --git a/grid-proxy/tests/queries/mock_client/nodes.go b/grid-proxy/tests/queries/mock_client/nodes.go index 682fce4d2..ea62f6df2 100644 --- a/grid-proxy/tests/queries/mock_client/nodes.go +++ b/grid-proxy/tests/queries/mock_client/nodes.go @@ -193,15 +193,15 @@ func (n *Node) satisfies(f types.NodeFilter, data *DBData) bool { return false } - if f.FreeMRU != nil && *f.FreeMRU > free.MRU { + if f.FreeMRU != nil && int64(*f.FreeMRU) > int64(free.MRU) { return false } - if f.FreeHRU != nil && *f.FreeHRU > free.HRU { + if f.FreeHRU != nil && int64(*f.FreeHRU) > int64(free.HRU) { return false } - if f.FreeSRU != nil && *f.FreeSRU > free.SRU { + if f.FreeSRU != nil && int64(*f.FreeSRU) > int64(free.SRU) { return false } diff --git a/grid-proxy/tests/queries/mock_client/utils.go b/grid-proxy/tests/queries/mock_client/utils.go index aff1efa51..1a4b7352f 100644 --- a/grid-proxy/tests/queries/mock_client/utils.go +++ b/grid-proxy/tests/queries/mock_client/utils.go @@ -11,19 +11,25 @@ type Result interface { } func calcFreeResources(total NodeResourcesTotal, used NodeResourcesTotal) NodeResourcesTotal { - if total.MRU < used.MRU { - panic("total mru is less than mru") + mru := total.MRU - used.MRU + if mru < 0 { + mru = 0 } - if total.HRU < used.HRU { - panic("total hru is less than hru") + + hru := total.HRU - used.HRU + if hru < 0 { + hru = 0 } - if total.SRU < used.SRU { - panic("total sru is less than sru") + + sru := total.SRU - used.SRU + if sru < 0 { + sru = 0 } + return NodeResourcesTotal{ - HRU: total.HRU - used.HRU, - SRU: total.SRU - used.SRU, - MRU: total.MRU - used.MRU, + HRU: hru, + SRU: sru, + MRU: mru, } } diff --git a/grid-proxy/tests/queries/node_test.go b/grid-proxy/tests/queries/node_test.go index 66a8925a6..000394217 100644 --- a/grid-proxy/tests/queries/node_test.go +++ b/grid-proxy/tests/queries/node_test.go @@ -538,12 +538,26 @@ func calcNodesAggregates(data *mock.DBData) (res NodesAggregate) { countries[node.Country] = struct{}{} total := data.NodeTotalResources[node.NodeID] free := calcFreeResources(total, data.NodeUsedResources[node.NodeID]) - res.maxFreeHRU = max(res.maxFreeHRU, free.HRU) - res.maxFreeSRU = max(res.maxFreeSRU, free.SRU) - res.maxFreeMRU = max(res.maxFreeMRU, free.MRU) - res.freeMRUs = append(res.freeMRUs, free.MRU) - res.freeSRUs = append(res.freeSRUs, free.SRU) - res.freeHRUs = append(res.freeHRUs, free.HRU) + freeHRU := free.HRU + if int64(freeHRU) < 0 { + freeHRU = 0 + } + freeMRU := free.MRU + if int64(freeMRU) < 0 { + freeMRU = 0 + } + + freeSRU := free.SRU + if int64(freeSRU) < 0 { + freeSRU = 0 + } + + res.maxFreeHRU = max(res.maxFreeHRU, freeHRU) + res.maxFreeSRU = max(res.maxFreeSRU, freeSRU) + res.maxFreeMRU = max(res.maxFreeMRU, freeMRU) + res.freeMRUs = append(res.freeMRUs, freeMRU) + res.freeSRUs = append(res.freeSRUs, freeSRU) + res.freeHRUs = append(res.freeHRUs, freeHRU) res.maxTotalMRU = max(res.maxTotalMRU, total.MRU) res.totalMRUs = append(res.totalMRUs, total.MRU) diff --git a/grid-proxy/tests/queries/utils.go b/grid-proxy/tests/queries/utils.go index 9f87bb567..15105701c 100644 --- a/grid-proxy/tests/queries/utils.go +++ b/grid-proxy/tests/queries/utils.go @@ -15,19 +15,25 @@ type Filter interface { } func calcFreeResources(total mock.NodeResourcesTotal, used mock.NodeResourcesTotal) mock.NodeResourcesTotal { - if total.MRU < used.MRU { - panic("total mru is less than mru") + mru := total.MRU - used.MRU + if mru < 0 { + mru = 0 } - if total.HRU < used.HRU { - panic("total hru is less than hru") + + hru := total.HRU - used.HRU + if hru < 0 { + hru = 0 } - if total.SRU < used.SRU { - panic("total sru is less than sru") + + sru := total.SRU - used.SRU + if sru < 0 { + sru = 0 } + return mock.NodeResourcesTotal{ - HRU: total.HRU - used.HRU, - SRU: total.SRU - used.SRU, - MRU: total.MRU - used.MRU, + HRU: hru, + SRU: sru, + MRU: mru, } } diff --git a/grid-proxy/tools/db/generate.go b/grid-proxy/tools/db/generate.go index 6200e9e10..5d87ebcd3 100644 --- a/grid-proxy/tools/db/generate.go +++ b/grid-proxy/tools/db/generate.go @@ -39,7 +39,7 @@ func generateData(db *sql.DB, seed int) error { return fmt.Errorf("failed to generate contracts: %w", err) } - if err := generator.GeneratePublicIPs(1, modifiers.PublicIPCount); err != nil { + if err := generator.GeneratePublicIPs(1, modifiers.PublicIPCount, 1, modifiers.FarmCount); err != nil { return fmt.Errorf("failed to generate public ips: %w", err) } diff --git a/grid-proxy/tools/db/modifiers/generators.go b/grid-proxy/tools/db/modifiers/generators.go index eaaf74235..47bfa1906 100644 --- a/grid-proxy/tools/db/modifiers/generators.go +++ b/grid-proxy/tools/db/modifiers/generators.go @@ -267,19 +267,34 @@ func (g *Generator) generateNodeContracts(billsStartID, contractsStartID, contra var contracts []string var contractResources []string var billingReports []string + toDelete := "" + toGracePeriod := "" for i := contractsStartID; i < end; i++ { nodeID := uint64(r.Intn((nodeStart+nodeSize)-nodeStart) + nodeStart) state := deleted - if g.nodeUP[nodeID] { - if flip(contractCreatedRatio) { + if flip(0.9) { state = created } else if flip(0.5) { state = gracePeriod } } + if state == deleted { + if len(toDelete) != 0 { + toDelete += ", " + } + toDelete += fmt.Sprint(i) + } + + if state == gracePeriod { + if len(toGracePeriod) != 0 { + toGracePeriod += ", " + } + toGracePeriod += fmt.Sprint(i) + } + if state != deleted && (minContractHRU > g.nodesHRU[nodeID] || minContractMRU > g.nodesMRU[nodeID] || minContractSRU > g.nodesSRU[nodeID]) { i-- continue @@ -303,7 +318,7 @@ func (g *Generator) generateNodeContracts(billsStartID, contractsStartID, contra id: fmt.Sprintf("node-contract-%d", i), twin_id: twinID, contract_id: uint64(i), - state: state, + state: created, created_at: uint64(time.Now().Unix()), node_id: nodeID, deployment_data: fmt.Sprintf("deployment-data-%d", i), @@ -341,7 +356,7 @@ func (g *Generator) generateNodeContracts(billsStartID, contractsStartID, contra mru: mru, contract_id: fmt.Sprintf("node-contract-%d", i), } - if contract.state != deleted { + if state != deleted { g.nodesHRU[nodeID] -= hru g.nodesSRU[nodeID] -= sru g.nodesMRU[nodeID] -= mru @@ -399,6 +414,18 @@ func (g *Generator) generateNodeContracts(billsStartID, contractsStartID, contra return nil, end, fmt.Errorf("failed to update node contract resources id: %w", err) } + if len(toDelete) > 0 { + if _, err := g.db.Exec(fmt.Sprintf("UPDATE node_contract SET state = '%s' WHERE contract_id IN (%s)", deleted, toDelete)); err != nil { + return nil, 0, fmt.Errorf("failed to update node_contract state to deleted: %w", err) + } + } + + if len(toGracePeriod) > 0 { + if _, err := g.db.Exec(fmt.Sprintf("UPDATE node_contract SET state = '%s' WHERE contract_id IN (%s)", gracePeriod, toGracePeriod)); err != nil { + return nil, 0, fmt.Errorf("failed to update node_contract state to grace period: %w", err) + } + } + fmt.Println("node contracts generated") return billingReports, end, nil @@ -417,6 +444,9 @@ func (g *Generator) GenerateNameContracts(billsStartID, contractsStartID, contra end := contractsStartID + contractCount var contracts []string var billReports []string + toDelete := "" + toGracePeriod := "" + for i := contractsStartID; i < end; i++ { nodeID, err := rnd(1, uint64(NodeCount)) if err != nil { @@ -425,13 +455,27 @@ func (g *Generator) GenerateNameContracts(billsStartID, contractsStartID, contra state := deleted if g.nodeUP[nodeID] { - if flip(contractCreatedRatio) { + if flip(0.9) { state = created } else if flip(0.5) { state = gracePeriod } } + if state == deleted { + if len(toDelete) != 0 { + toDelete += ", " + } + toDelete += fmt.Sprint(i) + } + + if state == gracePeriod { + if len(toGracePeriod) != 0 { + toGracePeriod += ", " + } + toGracePeriod += fmt.Sprint(i) + } + twinID, err := rnd(1100, 3100) if err != nil { return nil, i, fmt.Errorf("failed to generate random twin id: %w", err) @@ -493,6 +537,18 @@ func (g *Generator) GenerateNameContracts(billsStartID, contractsStartID, contra return nil, end, fmt.Errorf("failed to insert name contracts: %w", err) } + if len(toDelete) > 0 { + if _, err := g.db.Exec(fmt.Sprintf("UPDATE rent_contract SET state = '%s' WHERE contract_id IN (%s)", deleted, toDelete)); err != nil { + return nil, 0, fmt.Errorf("failed to update rent_contract state to deleted: %w", err) + } + } + + if len(toGracePeriod) > 0 { + if _, err := g.db.Exec(fmt.Sprintf("UPDATE rent_contract SET state = '%s' WHERE contract_id IN (%s)", gracePeriod, toGracePeriod)); err != nil { + return nil, 0, fmt.Errorf("failed to update rent_contract state to grace period: %w", err) + } + } + fmt.Println("name contracts generated") return billReports, end, nil @@ -503,6 +559,8 @@ func (g *Generator) GenerateRentContracts(billsStart, contractStart, rentConCoun var contracts []string var billReports []string + toDelete := "" + toGracePeriod := "" for i := contractStart; i < end; i++ { nl, nodeID, err := popRandom(g.availableRentNodesList) if err != nil { @@ -520,6 +578,20 @@ func (g *Generator) GenerateRentContracts(billsStart, contractStart, rentConCoun } } + if state == deleted { + if len(toDelete) != 0 { + toDelete += ", " + } + toDelete += fmt.Sprint(i) + } + + if state == gracePeriod { + if len(toGracePeriod) != 0 { + toGracePeriod += ", " + } + toGracePeriod += fmt.Sprint(i) + } + twinID, err := rnd(1100, 3100) if err != nil { return nil, i, fmt.Errorf("failed to generate a random twin id: %w", err) @@ -529,7 +601,7 @@ func (g *Generator) GenerateRentContracts(billsStart, contractStart, rentConCoun id: fmt.Sprintf("rent-contract-%d", i), twin_id: twinID, contract_id: uint64(i), - state: state, + state: created, created_at: uint64(time.Now().Unix()), node_id: nodeID, grid_version: 3, @@ -579,15 +651,27 @@ func (g *Generator) GenerateRentContracts(billsStart, contractStart, rentConCoun return nil, end, fmt.Errorf("failed to insert rent contracts: %w", err) } + if len(toDelete) > 0 { + if _, err := g.db.Exec(fmt.Sprintf("UPDATE rent_contract SET state = '%s' WHERE contract_id IN (%s)", deleted, toDelete)); err != nil { + return nil, 0, fmt.Errorf("failed to update rent_contract state to deleted: %w", err) + } + } + + if len(toGracePeriod) > 0 { + if _, err := g.db.Exec(fmt.Sprintf("UPDATE rent_contract SET state = '%s' WHERE contract_id IN (%s)", gracePeriod, toGracePeriod)); err != nil { + return nil, 0, fmt.Errorf("failed to update rent_contract state to grace period: %w", err) + } + } + fmt.Println("rent contracts generated") return billReports, end, nil } -func (g *Generator) GeneratePublicIPs(start, size int) error { +func (g *Generator) GeneratePublicIPs(start, size, farmStart, farmSize int) error { var publicIPs []string var nodeContracts []uint64 - + reservedIPs := map[string]uint64{} for i := uint64(start); i < uint64(start+size); i++ { contract_id := uint64(0) if flip(usedPublicIPsRatio) { @@ -597,19 +681,23 @@ func (g *Generator) GeneratePublicIPs(start, size int) error { } contract_id = g.createdNodeContracts[idx] } + ip := randomIPv4() - farmID, err := rnd(1, uint64(FarmCount)) - if err != nil { - return fmt.Errorf("failed to generate random farm id: %w", err) - } + + farmID := r.Int63n(int64(farmSize)) + int64(farmStart) public_ip := public_ip{ id: fmt.Sprintf("public-ip-%d", i), gateway: ip.String(), ip: IPv4Subnet(ip).String(), - contract_id: contract_id, + contract_id: 0, farm_id: fmt.Sprintf("farm-%d", farmID), } + + if contract_id != 0 { + reservedIPs[public_ip.id] = contract_id + } + publicIpTuple, err := objectToTupleString(public_ip) if err != nil { return fmt.Errorf("failed to convert public ip object to tuple string: %w", err) @@ -626,6 +714,12 @@ func (g *Generator) GeneratePublicIPs(start, size int) error { return fmt.Errorf("failed to update contract public ips: %w", err) } + for id, contractID := range reservedIPs { + if _, err := g.db.Exec(fmt.Sprintf("UPDATE public_ip SET contract_id = %d WHERE id = '%s'", contractID, id)); err != nil { + return fmt.Errorf("failed to reserve ip %s: %w", id, err) + } + } + fmt.Println("public IPs generated") return nil diff --git a/grid-proxy/tools/db/modifiers/modifiers.go b/grid-proxy/tools/db/modifiers/modifiers.go index 5708225f8..549a3db7b 100644 --- a/grid-proxy/tools/db/modifiers/modifiers.go +++ b/grid-proxy/tools/db/modifiers/modifiers.go @@ -37,7 +37,7 @@ func (g *Generator) UpdateNodeTotalResources() error { query += fmt.Sprintf("UPDATE node_resources_total SET cru = %d, hru = %d, mru = %d, sru = %d WHERE node_id = 'node-%d';", cru, hru, mru, sru, nodeId) } - log.Debug().Str("query", query).Msg("update node country") + log.Debug().Str("query", query).Msg("update node total resources") _, err := g.db.Exec(query) return err @@ -57,7 +57,7 @@ func (g *Generator) UpdateContractResources() error { query += fmt.Sprintf("UPDATE contract_resources SET cru = %d, hru = %d, mru = %d, sru = %d WHERE contract_id = 'node-contract-%d';", cru, hru, mru, sru, contractId) } - log.Debug().Str("query", query).Msg("update node country") + log.Debug().Str("query", query).Msg("update contract resources") _, err := g.db.Exec(query) return err @@ -71,10 +71,10 @@ func (g *Generator) UpdateNodeContractState() error { for i := 0; i < updatesCount; i++ { contractId := g.createdNodeContracts[r.Intn(len(g.createdNodeContracts))] state := states[r.Intn(2)] - query += fmt.Sprintf("UPDATE node_contract SET state = '%s' WHERE contract_id = %d;", state, contractId) + query += fmt.Sprintf("UPDATE node_contract SET state = '%s' WHERE contract_id = %d AND state != 'Deleted';", state, contractId) } - log.Debug().Str("query", query).Msg("update node country") + log.Debug().Str("query", query).Msg("update node contract state") _, err := g.db.Exec(query) return err @@ -91,7 +91,7 @@ func (g *Generator) UpdateRentContract() error { query += fmt.Sprintf("UPDATE rent_contract SET state = '%s' WHERE contract_id = %d;", state, contractId) } - log.Debug().Str("query", query).Msg("update node country") + log.Debug().Str("query", query).Msg("update rent contracts") _, err := g.db.Exec(query) return err @@ -107,6 +107,8 @@ func (g *Generator) UpdatePublicIps() error { publicIPID := r.Intn(int(PublicIPCount)) query += fmt.Sprintf("UPDATE public_ip SET contract_id = (CASE WHEN contract_id = 0 THEN %d ELSE 0 END) WHERE id = 'public-ip-%d';", contractID, publicIPID) + query += fmt.Sprintf("UPDATE node_contract SET number_of_public_i_ps = (SELECT COUNT(id) FROM public_ip WHERE contract_id = %d) WHERE contract_id = %d;", contractID, contractID) + } log.Debug().Str("query", query).Msg("update public ip contract_id") @@ -116,22 +118,22 @@ func (g *Generator) UpdatePublicIps() error { } // deletions -func (g *Generator) DeleteNodes() error { +func (g *Generator) DeleteNodes(nodesCount int) error { // delete node contracts on this node // free public ips that are assigned to the deleted contracts // delete rent contracts on this node // delete node - updatesCount := r.Intn(10) + 1 + deleteCount := r.Intn(10) + 1 query := "" - for i := 0; i < updatesCount; i++ { - nodeID := r.Intn(int(NodeCount)) + 1 - // NodeCount-- + for i := 0; i < deleteCount; i++ { + nodeID := nodesCount - i query += fmt.Sprintf("UPDATE public_ip SET contract_id = 0 WHERE contract_id IN (SELECT contract_id FROM node_contract WHERE node_id = %d);", nodeID) query += fmt.Sprintf("UPDATE node_contract SET state = 'Deleted' WHERE node_id = %d;", nodeID) query += fmt.Sprintf("UPDATE rent_contract set state = 'Deleted' WHERE node_id = %d;", nodeID) query += fmt.Sprintf("DELETE FROM node_resources_total WHERE node_id = (SELECT id FROM node WHERE node_id = %d);", nodeID) + query += fmt.Sprintf("DELETE FROM public_config WHERE node_id = (SELECT id FROM node WHERE node_id = %d);", nodeID) query += fmt.Sprintf("DELETE FROM node WHERE node_id = %d;", nodeID) } @@ -150,12 +152,6 @@ func (g *Generator) DeletePublicIps() error { return fmt.Errorf("failed to delete public ips: %w", err) } - // rowsAffercted, err := res.RowsAffected() - // if err != nil { - // return fmt.Errorf("failed to get rows affected by public ips delete: %w", err) - // } - // PublicIPCount -= uint32(rowsAffercted) - log.Debug().Str("query", query).Msg("delete public ips") return nil From 1a9a4c24335f3afbd8442ab8b322ec5f4ac3ff96 Mon Sep 17 00:00:00 2001 From: mario Date: Mon, 25 Dec 2023 18:18:32 +0200 Subject: [PATCH 31/39] wip: fixing tests against devnet Co-authored-by: omarabdulasis --- grid-proxy/internal/explorer/db/postgres.go | 7 +------ grid-proxy/internal/explorer/db/setup.sql | 11 ++++++++++- grid-proxy/tests/queries/mock_client/loader.go | 4 ++-- grid-proxy/tests/queries/mock_client/types.go | 1 + grid-proxy/tests/queries/utils.go | 4 ++++ grid-proxy/tools/db/modifiers/generators.go | 15 ++++++++++----- grid-proxy/tools/db/modifiers/types.go | 1 + 7 files changed, 29 insertions(+), 14 deletions(-) diff --git a/grid-proxy/internal/explorer/db/postgres.go b/grid-proxy/internal/explorer/db/postgres.go index f5f503d9e..dd28d8691 100644 --- a/grid-proxy/internal/explorer/db/postgres.go +++ b/grid-proxy/internal/explorer/db/postgres.go @@ -65,11 +65,6 @@ func NewPostgresDatabase(host string, port int, user, password, dbname string, m sql.SetMaxIdleConns(3) sql.SetMaxOpenConns(maxConns) - err = gormDB.AutoMigrate(&NodeGPU{}) - if err != nil { - return PostgresDatabase{}, errors.Wrap(err, "failed to auto migrate DB") - } - res := PostgresDatabase{gormDB, connString} return res, nil } @@ -415,7 +410,7 @@ func (d *PostgresDatabase) GetFarms(ctx context.Context, filter types.FarmFilter q = q.Where("COALESCE(public_ips_cache.total_ips, 0) >= ?", *filter.TotalIPs) } if filter.StellarAddress != nil { - q = q.Where("farm.stellar_address = ?", *filter.StellarAddress) + q = q.Where("COALESCE(farm.stellar_address, '') = ?", *filter.StellarAddress) } if filter.PricingPolicyID != nil { q = q.Where("farm.pricing_policy_id = ?", *filter.PricingPolicyID) diff --git a/grid-proxy/internal/explorer/db/setup.sql b/grid-proxy/internal/explorer/db/setup.sql index ba7a9c8d5..6cde6cf57 100644 --- a/grid-proxy/internal/explorer/db/setup.sql +++ b/grid-proxy/internal/explorer/db/setup.sql @@ -43,6 +43,15 @@ TEXT) RETURNS DECIMAL AS $$ ---- DROP TRIGGER IF EXISTS node_added ON node; +CREATE TABLE IF NOT EXISTS node_gpu ( + id text PRIMARY KEY, + node_twin_id bigint NOT NULL, + vendor text, + device text, + contract bigint +); + + ---- -- Resources cache table ---- @@ -100,7 +109,7 @@ FROM node GROUP BY node_twin_id ) AS node_gpu ON node.twin_id = node_gpu.node_twin_id - LEFT JOIN country ON node.country = country.name + LEFT JOIN country ON LOWER(node.country) = LOWER(country.name) GROUP BY node.node_id, node_resources_total.mru, diff --git a/grid-proxy/tests/queries/mock_client/loader.go b/grid-proxy/tests/queries/mock_client/loader.go index 4702172d2..5eefb9ede 100644 --- a/grid-proxy/tests/queries/mock_client/loader.go +++ b/grid-proxy/tests/queries/mock_client/loader.go @@ -525,8 +525,8 @@ func loadLocations(db *sql.DB, data *DBData) error { rows, err := db.Query(` SELECT COALESCE(id, ''), - COALESCE(longitude, ''), - COALESCE(latitude, '') + case when longitude = '' then '0' else longitude end, + case when latitude = '' then '0' else latitude end FROM location; `) diff --git a/grid-proxy/tests/queries/mock_client/types.go b/grid-proxy/tests/queries/mock_client/types.go index 57f8d758d..606c67968 100644 --- a/grid-proxy/tests/queries/mock_client/types.go +++ b/grid-proxy/tests/queries/mock_client/types.go @@ -49,6 +49,7 @@ type Node struct { Power NodePower `gorm:"type:jsonb"` HasGPU bool ExtraFee uint64 + Dedicated bool } type NodePower struct { diff --git a/grid-proxy/tests/queries/utils.go b/grid-proxy/tests/queries/utils.go index 15105701c..94f4f1737 100644 --- a/grid-proxy/tests/queries/utils.go +++ b/grid-proxy/tests/queries/utils.go @@ -61,6 +61,10 @@ func min(a, b uint64) uint64 { } func changeCase(s string) string { + if len(s) == 0 { + return s + } + idx := rand.Intn(len(s)) return strings.Replace(s, string(s[idx]), strings.ToUpper(string(s[idx])), 1) } diff --git a/grid-proxy/tools/db/modifiers/generators.go b/grid-proxy/tools/db/modifiers/generators.go index 47bfa1906..d637607a3 100644 --- a/grid-proxy/tools/db/modifiers/generators.go +++ b/grid-proxy/tools/db/modifiers/generators.go @@ -269,7 +269,7 @@ func (g *Generator) generateNodeContracts(billsStartID, contractsStartID, contra var billingReports []string toDelete := "" toGracePeriod := "" - + contractToResource := map[uint64]string{} for i := contractsStartID; i < end; i++ { nodeID := uint64(r.Intn((nodeStart+nodeSize)-nodeStart) + nodeStart) state := deleted @@ -356,6 +356,8 @@ func (g *Generator) generateNodeContracts(billsStartID, contractsStartID, contra mru: mru, contract_id: fmt.Sprintf("node-contract-%d", i), } + contractToResource[contract.contract_id] = contract_resources.id + if state != deleted { g.nodesHRU[nodeID] -= hru g.nodesSRU[nodeID] -= sru @@ -410,7 +412,7 @@ func (g *Generator) generateNodeContracts(billsStartID, contractsStartID, contra return nil, end, fmt.Errorf("failed to insert contract resources: %w", err) } - if err := g.updateNodeContractResourceID(contractsStartID, end); err != nil { + if err := g.updateNodeContractResourceID(contractToResource); err != nil { return nil, end, fmt.Errorf("failed to update node contract resources id: %w", err) } @@ -431,9 +433,12 @@ func (g *Generator) generateNodeContracts(billsStartID, contractsStartID, contra return billingReports, end, nil } -func (g *Generator) updateNodeContractResourceID(min, max int) error { - query := fmt.Sprintf(`UPDATE node_contract SET resources_used_id = CONCAT('contract-resources-',split_part(id, '-', -1)) - WHERE CAST(split_part(id, '-', -1) AS INTEGER) BETWEEN %d AND %d;`, min, max) +func (g *Generator) updateNodeContractResourceID(contractToResource map[uint64]string) error { + query := "" + for contractID, ResourceID := range contractToResource { + query += fmt.Sprintf("UPDATE node_contract SET resources_used_id = '%s' WHERE contract_id = %d;", ResourceID, contractID) + } + if _, err := g.db.Exec(query); err != nil { return fmt.Errorf("failed to update node contract resource id: %w", err) } diff --git a/grid-proxy/tools/db/modifiers/types.go b/grid-proxy/tools/db/modifiers/types.go index 0428abd85..d62702bfb 100644 --- a/grid-proxy/tools/db/modifiers/types.go +++ b/grid-proxy/tools/db/modifiers/types.go @@ -121,6 +121,7 @@ type node struct { location_id string power nodePower `gorm:"type:jsonb"` extra_fee uint64 + dedicated bool } type nodePower struct { From 475354de5a0e317c59f4c150b1f3f0317fa83ad3 Mon Sep 17 00:00:00 2001 From: mario Date: Tue, 26 Dec 2023 12:14:49 +0200 Subject: [PATCH 32/39] wip: fixing tests against devnet Co-authored-by: omarabdulasis --- grid-proxy/internal/explorer/db/postgres.go | 10 +-- grid-proxy/tests/queries/contract_test.go | 26 +++++- grid-proxy/tests/queries/farm_test.go | 41 ++++++++-- .../tests/queries/mock_client/loader.go | 4 +- grid-proxy/tests/queries/mock_client/nodes.go | 16 +++- grid-proxy/tests/queries/node_test.go | 81 +++++++++++++++---- grid-proxy/tests/queries/twin_test.go | 25 +++++- grid-proxy/tests/queries/utils.go | 8 +- 8 files changed, 173 insertions(+), 38 deletions(-) diff --git a/grid-proxy/internal/explorer/db/postgres.go b/grid-proxy/internal/explorer/db/postgres.go index dd28d8691..938d11f20 100644 --- a/grid-proxy/internal/explorer/db/postgres.go +++ b/grid-proxy/internal/explorer/db/postgres.go @@ -571,13 +571,13 @@ func (d *PostgresDatabase) GetNodes(ctx context.Context, filter types.NodeFilter q = q.Where("COALESCE(public_ips_cache.free_ips, 0) >= ?", *filter.FreeIPs) } if filter.IPv4 != nil { - q = q.Where("(public_config.ipv4 IS NULL) != ?", *filter.IPv4) + q = q.Where("(COALESCE(public_config.ipv4, '') != '') = ?", *filter.IPv4) } if filter.IPv6 != nil { - q = q.Where("(public_config.ipv6 IS NULL) != ?", *filter.IPv6) + q = q.Where("(COALESCE(public_config.ipv6, '') != '') = ?", *filter.IPv6) } if filter.Domain != nil { - q = q.Where("(public_config.domain IS NULL) != ?", *filter.Domain) + q = q.Where("(COALESCE(public_config.domain, '') != '') = ?", *filter.Domain) } if filter.CertificationType != nil { q = q.Where("node.certification ILIKE ?", *filter.CertificationType) @@ -719,10 +719,10 @@ func (d *PostgresDatabase) contractTableQuery() *gorm.DB { contractTablesQuery := `( SELECT contract_id, twin_id, state, created_at, '' AS name, node_id, deployment_data, deployment_hash, number_of_public_i_ps, 'node' AS type FROM node_contract - UNION + UNION ALL SELECT contract_id, twin_id, state, created_at, '' AS name, node_id, '', '', 0, 'rent' AS type FROM rent_contract - UNION + UNION ALL SELECT contract_id, twin_id, state, created_at, name, 0, '', '', 0, 'name' AS type FROM name_contract ) contracts` diff --git a/grid-proxy/tests/queries/contract_test.go b/grid-proxy/tests/queries/contract_test.go index 66784b7db..65a1df254 100644 --- a/grid-proxy/tests/queries/contract_test.go +++ b/grid-proxy/tests/queries/contract_test.go @@ -54,16 +54,28 @@ var contractFilterRandomValueGenerator = map[string]func(agg ContractsAggregate) return &agg.States[rand.Intn(len(agg.States))] }, "Name": func(agg ContractsAggregate) interface{} { - return &agg.Names[rand.Intn(len(agg.Names))] + c := agg.Names[rand.Intn(len(agg.Names))] + if len(c) == 0 { + return nil + } + return &c }, "NumberOfPublicIps": func(agg ContractsAggregate) interface{} { return rndref(0, agg.maxNumberOfPublicIPs) }, "DeploymentData": func(agg ContractsAggregate) interface{} { - return &agg.DeploymentsData[rand.Intn(len(agg.DeploymentsData))] + c := agg.DeploymentsData[rand.Intn(len(agg.DeploymentsData))] + if len(c) == 0 { + return nil + } + return &c }, "DeploymentHash": func(agg ContractsAggregate) interface{} { - return &agg.DeploymentHashes[rand.Intn(len(agg.DeploymentHashes))] + c := agg.DeploymentHashes[rand.Intn(len(agg.DeploymentHashes))] + if len(c) == 0 { + return nil + } + return &c }, } @@ -204,6 +216,10 @@ func TestContractsFilter(t *testing.T) { require.True(t, ok, "Filter field %s has no random value generator", v.Type().Field(i).Name) randomFieldValue := generator(agg) + if randomFieldValue == nil { + continue + } + if v.Field(i).Type().Kind() != reflect.Slice { v.Field(i).Set(reflect.New(v.Field(i).Type().Elem())) } @@ -295,6 +311,10 @@ func randomContractsFilter(agg *ContractsAggregate) (proxytypes.ContractFilter, } randomFieldValue := contractFilterRandomValueGenerator[v.Type().Field(i).Name](*agg) + if randomFieldValue == nil { + continue + } + if v.Field(i).Type().Kind() != reflect.Slice { v.Field(i).Set(reflect.New(v.Field(i).Type().Elem())) } diff --git a/grid-proxy/tests/queries/farm_test.go b/grid-proxy/tests/queries/farm_test.go index 2f7e30183..7f91f90d2 100644 --- a/grid-proxy/tests/queries/farm_test.go +++ b/grid-proxy/tests/queries/farm_test.go @@ -27,7 +27,12 @@ var farmFilterRandomValueGenerator = map[string]func(agg FarmsAggregate) interfa return rndref(0, agg.maxTotalIPs) }, "StellarAddress": func(agg FarmsAggregate) interface{} { - return &agg.stellarAddresses[rand.Intn(len(agg.stellarAddresses))] + c := agg.stellarAddresses[rand.Intn(len(agg.stellarAddresses))] + if len(c) == 0 { + return nil + } + + return &c }, "PricingPolicyID": func(agg FarmsAggregate) interface{} { return &agg.pricingPolicyIDs[rand.Intn(len(agg.pricingPolicyIDs))] @@ -40,24 +45,42 @@ var farmFilterRandomValueGenerator = map[string]func(agg FarmsAggregate) interfa }, "Name": func(agg FarmsAggregate) interface{} { name := changeCase(agg.farmNames[rand.Intn(len(agg.farmNames))]) + if len(name) == 0 { + return nil + } + return &name }, "Country": func(_ FarmsAggregate) interface{} { aggNode := calcNodesAggregates(&data) country := changeCase(aggNode.countries[rand.Intn(len(aggNode.countries))]) + if len(country) == 0 { + return nil + } + return &country }, "Region": func(agg FarmsAggregate) interface{} { - region := changeCase(agg.regions[rand.Intn(len(agg.regions))]) - return ®ion + c := changeCase(agg.regions[rand.Intn(len(agg.regions))]) + if len(c) == 0 { + return nil + } + + return &c }, "NameContains": func(agg FarmsAggregate) interface{} { c := agg.farmNames[rand.Intn(len(agg.farmNames))] - a, b := rand.Intn(len(c)), rand.Intn(len(c)) + runesList := []rune(c) + a, b := rand.Intn(len(runesList)), rand.Intn(len(runesList)) if a > b { a, b = b, a } - c = c[a : b+1] + runesList = runesList[a : b+1] + c = string(runesList) + if len(c) == 0 { + return nil + } + return &c }, "CertificationType": func(agg FarmsAggregate) interface{} { @@ -260,6 +283,10 @@ func TestFarmFilter(t *testing.T) { require.True(t, ok, "Filter field %s has no random value generator", v.Type().Field(i).Name) randomFieldValue := generator(agg) + if randomFieldValue == nil { + continue + } + if v.Field(i).Type().Kind() != reflect.Slice { v.Field(i).Set(reflect.New(v.Field(i).Type().Elem())) } @@ -348,6 +375,10 @@ func randomFarmsFilter(agg *FarmsAggregate) (proxytypes.FarmFilter, error) { } randomFieldValue := generate(*agg) + if randomFieldValue == nil { + continue + } + if v.Field(i).Type().Kind() != reflect.Slice { v.Field(i).Set(reflect.New(v.Field(i).Type().Elem())) } diff --git a/grid-proxy/tests/queries/mock_client/loader.go b/grid-proxy/tests/queries/mock_client/loader.go index 5eefb9ede..de6344080 100644 --- a/grid-proxy/tests/queries/mock_client/loader.go +++ b/grid-proxy/tests/queries/mock_client/loader.go @@ -525,8 +525,8 @@ func loadLocations(db *sql.DB, data *DBData) error { rows, err := db.Query(` SELECT COALESCE(id, ''), - case when longitude = '' then '0' else longitude end, - case when latitude = '' then '0' else latitude end + CASE WHEN longitude = '' THEN NULL ELSE longitude END, + CASE WHEN latitude = '' THEN NULL ELSE latitude END FROM location; `) diff --git a/grid-proxy/tests/queries/mock_client/nodes.go b/grid-proxy/tests/queries/mock_client/nodes.go index ea62f6df2..2028f48eb 100644 --- a/grid-proxy/tests/queries/mock_client/nodes.go +++ b/grid-proxy/tests/queries/mock_client/nodes.go @@ -2,6 +2,7 @@ package mock import ( "context" + "fmt" "sort" "strings" @@ -83,7 +84,8 @@ func (g *GridProxyMockClient) Nodes(ctx context.Context, filter types.NodeFilter State: node.Power.State, Target: node.Power.Target, }, - NumGPU: numGPU, + NumGPU: numGPU, + ExtraFee: node.ExtraFee, }) } } @@ -105,7 +107,11 @@ func (g *GridProxyMockClient) Nodes(ctx context.Context, filter types.NodeFilter } func (g *GridProxyMockClient) Node(ctx context.Context, nodeID uint32) (res types.NodeWithNestedCapacity, err error) { - node := g.data.Nodes[uint64(nodeID)] + node, ok := g.data.Nodes[uint64(nodeID)] + if !ok { + return res, fmt.Errorf("node not found") + } + numGPU := len(g.data.GPUs[node.TwinID]) nodePower := types.NodePower{ @@ -170,7 +176,11 @@ func (g *GridProxyMockClient) Node(ctx context.Context, nodeID uint32) (res type } func (g *GridProxyMockClient) NodeStatus(ctx context.Context, nodeID uint32) (res types.NodeStatus, err error) { - node := g.data.Nodes[uint64(nodeID)] + node, ok := g.data.Nodes[uint64(nodeID)] + if !ok { + return res, fmt.Errorf("node not found") + } + nodePower := types.NodePower{ State: node.Power.State, Target: node.Power.Target, diff --git a/grid-proxy/tests/queries/node_test.go b/grid-proxy/tests/queries/node_test.go index 000394217..a4423c69e 100644 --- a/grid-proxy/tests/queries/node_test.go +++ b/grid-proxy/tests/queries/node_test.go @@ -102,45 +102,83 @@ var nodeFilterRandomValueGenerator = map[string]func(agg NodesAggregate) interfa }, "Country": func(agg NodesAggregate) interface{} { country := changeCase(agg.countries[rand.Intn(len(agg.countries))]) + if len(country) == 0 { + return nil + } + return &country }, "Region": func(agg NodesAggregate) interface{} { region := changeCase(agg.regions[rand.Intn(len(agg.regions))]) + if len(region) == 0 { + return nil + } + return ®ion }, "CountryContains": func(agg NodesAggregate) interface{} { c := agg.countries[rand.Intn(len(agg.countries))] - a, b := rand.Intn(len(c)), rand.Intn(len(c)) + if len(c) == 0 { + return nil + } + + runesList := []rune(c) + a, b := rand.Intn(len(runesList)), rand.Intn(len(runesList)) if a > b { a, b = b, a } - c = c[a : b+1] + runesList = runesList[a : b+1] + c = string(runesList) + if len(c) == 0 { + return nil + } + return &c }, "City": func(agg NodesAggregate) interface{} { city := changeCase(agg.cities[rand.Intn(len(agg.cities))]) + if len(city) == 0 { + return nil + } + return &city }, "CityContains": func(agg NodesAggregate) interface{} { c := agg.cities[rand.Intn(len(agg.cities))] - a, b := rand.Intn(len(c)), rand.Intn(len(c)) + if len(c) == 0 { + return nil + } + + runesList := []rune(c) + a, b := rand.Intn(len(runesList)), rand.Intn(len(runesList)) if a > b { a, b = b, a } - c = c[a : b+1] + runesList = runesList[a : b+1] + c = string(runesList) return &c }, "FarmName": func(agg NodesAggregate) interface{} { name := changeCase(agg.farmNames[rand.Intn(len(agg.farmNames))]) + if len(name) == 0 { + return nil + } + return &name }, "FarmNameContains": func(agg NodesAggregate) interface{} { c := agg.farmNames[rand.Intn(len(agg.farmNames))] - a, b := rand.Intn(len(c)), rand.Intn(len(c)) + if len(c) == 0 { + return nil + } + + runesList := []rune(c) + a, b := rand.Intn(len(runesList)), rand.Intn(len(runesList)) if a > b { a, b = b, a } - c = c[a : b+1] + runesList = runesList[a : b+1] + c = string(runesList) return &c }, "FarmIDs": func(agg NodesAggregate) interface{} { @@ -308,11 +346,14 @@ func TestNode(t *testing.T) { for i := 1; i <= NODE_COUNT; i++ { if flip(.3) { - want, err := mockClient.NodeStatus(context.Background(), uint32(i)) - require.NoError(t, err) + want, errWant := mockClient.NodeStatus(context.Background(), uint32(i)) + got, errGot := gridProxyClient.NodeStatus(context.Background(), uint32(i)) - got, err := gridProxyClient.NodeStatus(context.Background(), uint32(i)) - require.NoError(t, err) + if errGot != nil && errWant != nil { + require.True(t, errors.As(errWant, &errGot), fmt.Sprintf("errors should match: want error %s, got error %s", errWant, errGot)) + } else { + require.True(t, errWant == errGot) + } require.True(t, reflect.DeepEqual(want, got), fmt.Sprintf("Difference:\n%s", cmp.Diff(want, got))) } @@ -449,10 +490,14 @@ func TestNodeFilter(t *testing.T) { require.True(t, ok, "Filter field %s has no random value generator", v.Type().Field(i).Name) randomFieldValue := generator(agg) + if randomFieldValue == nil { + continue + } if v.Field(i).Type().Kind() != reflect.Slice { v.Field(i).Set(reflect.New(v.Field(i).Type().Elem())) } + v.Field(i).Set(reflect.ValueOf(randomFieldValue)) want, wantCount, err := mockClient.Nodes(context.Background(), f, l) @@ -472,11 +517,15 @@ func TestNodeFilter(t *testing.T) { func singleNodeCheck(t *testing.T, localClient proxyclient.Client, proxyClient proxyclient.Client) { t.Parallel() nodeID := rand.Intn(NODE_COUNT) - want, err := mockClient.Node(context.Background(), uint32(nodeID)) - require.NoError(t, err) + want, errWant := mockClient.Node(context.Background(), uint32(nodeID)) + + got, errGot := gridProxyClient.Node(context.Background(), uint32(nodeID)) - got, err := gridProxyClient.Node(context.Background(), uint32(nodeID)) - require.NoError(t, err) + if errGot != nil && errWant != nil { + require.True(t, errors.As(errWant, &errGot)) + } else { + require.True(t, errWant == errGot) + } require.True(t, reflect.DeepEqual(want, got), fmt.Sprintf("Difference:\n%s", cmp.Diff(want, got))) } @@ -523,6 +572,10 @@ func randomNodeFilter(agg *NodesAggregate) (types.NodeFilter, error) { if v.Field(i).Type().Kind() != reflect.Slice { v.Field(i).Set(reflect.New(v.Field(i).Type().Elem())) } + if randomFieldValue == nil { + continue + } + v.Field(i).Set(reflect.ValueOf(randomFieldValue)) } } diff --git a/grid-proxy/tests/queries/twin_test.go b/grid-proxy/tests/queries/twin_test.go index 0a0ba3d70..9420f503d 100644 --- a/grid-proxy/tests/queries/twin_test.go +++ b/grid-proxy/tests/queries/twin_test.go @@ -32,13 +32,25 @@ var twinFilterRandomValueGenerator = map[string]func(agg TwinsAggregate) interfa return &agg.twinIDs[rand.Intn(len(agg.twinIDs))] }, "AccountID": func(agg TwinsAggregate) interface{} { - return &agg.accountIDs[rand.Intn(len(agg.accountIDs))] + c := agg.accountIDs[rand.Intn(len(agg.accountIDs))] + if len(c) == 0 { + return nil + } + return &c }, "Relay": func(agg TwinsAggregate) interface{} { - return &agg.relays[rand.Intn(len(agg.relays))] + c := agg.relays[rand.Intn(len(agg.relays))] + if len(c) == 0 { + return nil + } + return &c }, "PublicKey": func(agg TwinsAggregate) interface{} { - return &agg.publicKeys[rand.Intn(len(agg.publicKeys))] + c := agg.publicKeys[rand.Intn(len(agg.publicKeys))] + if len(c) == 0 { + return nil + } + return &c }, } @@ -118,6 +130,9 @@ func TestTwinFilter(t *testing.T) { require.True(t, ok, "Filter field %s has no random value generator", v.Type().Field(i).Name) randomFieldValue := generator(agg) + if randomFieldValue == nil { + continue + } if v.Field(i).Type().Kind() != reflect.Slice { v.Field(i).Set(reflect.New(v.Field(i).Type().Elem())) @@ -151,6 +166,10 @@ func randomTwinsFilter(agg *TwinsAggregate) (proxytypes.TwinFilter, error) { } randomFieldValue := twinFilterRandomValueGenerator[v.Type().Field(i).Name](*agg) + if randomFieldValue == nil { + continue + } + if v.Field(i).Type().Kind() != reflect.Slice { v.Field(i).Set(reflect.New(v.Field(i).Type().Elem())) } diff --git a/grid-proxy/tests/queries/utils.go b/grid-proxy/tests/queries/utils.go index 94f4f1737..aa1e06fec 100644 --- a/grid-proxy/tests/queries/utils.go +++ b/grid-proxy/tests/queries/utils.go @@ -4,7 +4,7 @@ import ( "fmt" "math/rand" "reflect" - "strings" + "unicode" "github.com/threefoldtech/tfgrid-sdk-go/grid-proxy/pkg/types" mock "github.com/threefoldtech/tfgrid-sdk-go/grid-proxy/tests/queries/mock_client" @@ -65,8 +65,10 @@ func changeCase(s string) string { return s } - idx := rand.Intn(len(s)) - return strings.Replace(s, string(s[idx]), strings.ToUpper(string(s[idx])), 1) + runesList := []rune(s) + idx := rand.Intn(len(runesList)) + runesList[idx] = unicode.ToUpper(runesList[idx]) + return string(runesList) } func SerializeFilter[F Filter](f F) string { From f1ce1ace5b38a0409f4d030b34481bc8809c83fa Mon Sep 17 00:00:00 2001 From: Omar Abdulaziz Date: Tue, 26 Dec 2023 13:25:46 +0200 Subject: [PATCH 33/39] - fix testing against devnet db dump - update schema with recently generated one from processor-db Co-authored-by: Mario Wassef --- grid-proxy/internal/explorer/db/postgres.go | 4 + grid-proxy/internal/explorer/db/setup.sql | 13 +- grid-proxy/tests/queries/main_test.go | 18 +- grid-proxy/tests/queries/mock_client/farms.go | 2 +- .../tests/queries/mock_client/loader.go | 3 +- grid-proxy/tests/queries/mock_client/nodes.go | 2 +- grid-proxy/tools/db/schema.sql | 245 +++++++----------- 7 files changed, 118 insertions(+), 169 deletions(-) diff --git a/grid-proxy/internal/explorer/db/postgres.go b/grid-proxy/internal/explorer/db/postgres.go index 938d11f20..8df35cbbc 100644 --- a/grid-proxy/internal/explorer/db/postgres.go +++ b/grid-proxy/internal/explorer/db/postgres.go @@ -62,6 +62,10 @@ func NewPostgresDatabase(host string, port int, user, password, dbname string, m return PostgresDatabase{}, errors.Wrap(err, "failed to configure DB connection") } + err = gormDB.AutoMigrate(&NodeGPU{}) + if err != nil { + return PostgresDatabase{}, errors.Wrap(err, "failed to migrate node_gpu table") + } sql.SetMaxIdleConns(3) sql.SetMaxOpenConns(maxConns) diff --git a/grid-proxy/internal/explorer/db/setup.sql b/grid-proxy/internal/explorer/db/setup.sql index 6cde6cf57..07e6f4bb7 100644 --- a/grid-proxy/internal/explorer/db/setup.sql +++ b/grid-proxy/internal/explorer/db/setup.sql @@ -43,15 +43,6 @@ TEXT) RETURNS DECIMAL AS $$ ---- DROP TRIGGER IF EXISTS node_added ON node; -CREATE TABLE IF NOT EXISTS node_gpu ( - id text PRIMARY KEY, - node_twin_id bigint NOT NULL, - vendor text, - device text, - contract bigint -); - - ---- -- Resources cache table ---- @@ -94,7 +85,7 @@ SELECT rent_contract.contract_id as rent_contract_id, count(node_contract.contract_id) as node_contracts_count, COALESCE(node_gpu.node_gpu_count, 0) as node_gpu_count, - country.name as country, + node.country as country, country.subregion as region FROM node LEFT JOIN node_contract ON node.node_id = node_contract.node_id AND node_contract.state IN ('Created', 'GracePeriod') @@ -120,7 +111,7 @@ GROUP BY rent_contract.contract_id, rent_contract.twin_id, COALESCE(node_gpu.node_gpu_count, 0), - country.name, + node.country, country.subregion; CREATE TABLE IF NOT EXISTS resources_cache( diff --git a/grid-proxy/tests/queries/main_test.go b/grid-proxy/tests/queries/main_test.go index 5f8115fbb..fa283520d 100644 --- a/grid-proxy/tests/queries/main_test.go +++ b/grid-proxy/tests/queries/main_test.go @@ -85,16 +85,16 @@ func TestMain(m *testing.M) { // os.Exit(exitcode) // } - err = modifyDataToFireTriggers(db, data) - if err != nil { - panic(err) - } + // err = modifyDataToFireTriggers(db, data) + // if err != nil { + // panic(err) + // } - data, err = mock.Load(db) - if err != nil { - panic(err) - } - mockClient = mock.NewGridProxyMockClient(data) + // data, err = mock.Load(db) + // if err != nil { + // panic(err) + // } + // mockClient = mock.NewGridProxyMockClient(data) exitcode := m.Run() os.Exit(exitcode) diff --git a/grid-proxy/tests/queries/mock_client/farms.go b/grid-proxy/tests/queries/mock_client/farms.go index 4e9e19925..f61e07265 100644 --- a/grid-proxy/tests/queries/mock_client/farms.go +++ b/grid-proxy/tests/queries/mock_client/farms.go @@ -178,7 +178,7 @@ func (f *Farm) satisfyFarmNodesFilter(data *DBData, filter types.FarmFilter) boo continue } - if filter.Region != nil && !strings.EqualFold(*filter.Region, data.Regions[node.Country]) { + if filter.Region != nil && !strings.EqualFold(*filter.Region, data.Regions[strings.ToLower(node.Country)]) { continue } diff --git a/grid-proxy/tests/queries/mock_client/loader.go b/grid-proxy/tests/queries/mock_client/loader.go index de6344080..6cf87ae67 100644 --- a/grid-proxy/tests/queries/mock_client/loader.go +++ b/grid-proxy/tests/queries/mock_client/loader.go @@ -3,6 +3,7 @@ package mock import ( "database/sql" "math" + "strings" "github.com/threefoldtech/zos/pkg/gridtypes" ) @@ -515,7 +516,7 @@ func loadCountries(db *sql.DB, data *DBData) error { ); err != nil { return err } - data.Regions[country.Name] = country.Subregion + data.Regions[strings.ToLower(country.Name)] = country.Subregion } return nil diff --git a/grid-proxy/tests/queries/mock_client/nodes.go b/grid-proxy/tests/queries/mock_client/nodes.go index 2028f48eb..9b0b10ba6 100644 --- a/grid-proxy/tests/queries/mock_client/nodes.go +++ b/grid-proxy/tests/queries/mock_client/nodes.go @@ -235,7 +235,7 @@ func (n *Node) satisfies(f types.NodeFilter, data *DBData) bool { return false } - if f.Region != nil && !strings.EqualFold(*f.Region, data.Regions[n.Country]) { + if f.Region != nil && !strings.EqualFold(*f.Region, data.Regions[strings.ToLower(n.Country)]) { return false } diff --git a/grid-proxy/tools/db/schema.sql b/grid-proxy/tools/db/schema.sql index 8e31a7112..816bf31fe 100644 --- a/grid-proxy/tools/db/schema.sql +++ b/grid-proxy/tools/db/schema.sql @@ -2,8 +2,8 @@ -- PostgreSQL database dump -- --- Dumped from database version 15.3 (Debian 15.3-1.pgdg110+1) --- Dumped by pg_dump version 15.3 (Ubuntu 15.3-0ubuntu0.23.04.1) +-- Dumped from database version 16.1 (Debian 16.1-1.pgdg120+1) +-- Dumped by pg_dump version 16.1 (Debian 16.1-1.pgdg120+1) SET statement_timeout = 0; SET lock_timeout = 0; @@ -17,32 +17,18 @@ SET client_min_messages = warning; SET row_security = off; -- --- Name: substrate_threefold_status; Type: SCHEMA; Schema: -; Owner: postgres +-- Name: squid_processor; Type: SCHEMA; Schema: -; Owner: postgres -- -CREATE SCHEMA substrate_threefold_status; +CREATE SCHEMA squid_processor; -ALTER SCHEMA substrate_threefold_status OWNER TO postgres; - +ALTER SCHEMA squid_processor OWNER TO postgres; SET default_tablespace = ''; SET default_table_access_method = heap; --- --- Name: account; Type: TABLE; Schema: public; Owner: postgres --- - -CREATE TABLE public.account ( - id character varying NOT NULL, - wallet text NOT NULL, - balance numeric NOT NULL -); - - -ALTER TABLE public.account OWNER TO postgres; - -- -- Name: burn_transaction; Type: TABLE; Schema: public; Owner: postgres -- @@ -96,7 +82,7 @@ CREATE TABLE public.contract_resources ( sru numeric NOT NULL, cru numeric NOT NULL, mru numeric NOT NULL, - contract_id character varying NOT NULL + contract_id character varying ); @@ -145,7 +131,7 @@ CREATE TABLE public.entity_proof ( id character varying NOT NULL, entity_id integer NOT NULL, signature text NOT NULL, - twin_rel_id character varying NOT NULL + twin_rel_id character varying ); @@ -195,20 +181,6 @@ CREATE TABLE public.farming_policy ( ALTER TABLE public.farming_policy OWNER TO postgres; --- --- Name: historical_balance; Type: TABLE; Schema: public; Owner: postgres --- - -CREATE TABLE public.historical_balance ( - id character varying NOT NULL, - balance numeric NOT NULL, - "timestamp" numeric NOT NULL, - account_id character varying NOT NULL -); - - -ALTER TABLE public.historical_balance OWNER TO postgres; - -- -- Name: interfaces; Type: TABLE; Schema: public; Owner: postgres -- @@ -218,7 +190,7 @@ CREATE TABLE public.interfaces ( name text NOT NULL, mac text NOT NULL, ips text NOT NULL, - node_id character varying NOT NULL + node_id character varying ); @@ -263,7 +235,7 @@ CREATE SEQUENCE public.migrations_id_seq CACHE 1; -ALTER TABLE public.migrations_id_seq OWNER TO postgres; +ALTER SEQUENCE public.migrations_id_seq OWNER TO postgres; -- -- Name: migrations_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: postgres @@ -297,7 +269,8 @@ CREATE TABLE public.name_contract ( twin_id integer NOT NULL, name text NOT NULL, created_at numeric NOT NULL, - state character varying(11) NOT NULL + state character varying(11) NOT NULL, + solution_provider_id integer ); @@ -323,10 +296,11 @@ CREATE TABLE public.node ( serial_number text, created_at numeric NOT NULL, updated_at numeric NOT NULL, - location_id character varying NOT NULL, + location_id character varying, certification character varying(9), connection_price integer, power jsonb, + dedicated boolean NOT NULL, extra_fee numeric ); @@ -348,27 +322,13 @@ CREATE TABLE public.node_contract ( number_of_public_i_ps integer NOT NULL, created_at numeric NOT NULL, resources_used_id character varying, - state character varying(11) NOT NULL + state character varying(11) NOT NULL, + solution_provider_id integer ); ALTER TABLE public.node_contract OWNER TO postgres; --- --- Name: node_gpu; Type: TABLE; Schema: public; Owner: postgres --- - -CREATE TABLE public.node_gpu ( - node_twin_id bigint NOT NULL, - id text NOT NULL, - vendor text, - device text, - contract bigint -); - - -ALTER TABLE public.node_gpu OWNER TO postgres; - -- -- Name: node_resources_free; Type: TABLE; Schema: public; Owner: postgres -- @@ -417,33 +377,6 @@ CREATE TABLE public.node_resources_used ( ALTER TABLE public.node_resources_used OWNER TO postgres; --- --- Name: nodes_resources_view; Type: VIEW; Schema: public; Owner: postgres --- - -CREATE VIEW public.nodes_resources_view AS - SELECT node.node_id, - COALESCE(sum(contract_resources.cru), (0)::numeric) AS used_cru, - (COALESCE(sum(contract_resources.mru), (0)::numeric) + (GREATEST(((node_resources_total.mru / (10)::numeric))::bigint, '2147483648'::bigint))::numeric) AS used_mru, - COALESCE(sum(contract_resources.hru), (0)::numeric) AS used_hru, - (COALESCE(sum(contract_resources.sru), (0)::numeric) + ('21474836480'::bigint)::numeric) AS used_sru, - ((node_resources_total.mru - COALESCE(sum(contract_resources.mru), (0)::numeric)) - (GREATEST(((node_resources_total.mru / (10)::numeric))::bigint, '2147483648'::bigint))::numeric) AS free_mru, - (node_resources_total.hru - COALESCE(sum(contract_resources.hru), (0)::numeric)) AS free_hru, - ((node_resources_total.sru - COALESCE(sum(contract_resources.sru), (0)::numeric)) - ('21474836480'::bigint)::numeric) AS free_sru, - COALESCE(node_resources_total.cru, (0)::numeric) AS total_cru, - COALESCE(node_resources_total.mru, (0)::numeric) AS total_mru, - COALESCE(node_resources_total.hru, (0)::numeric) AS total_hru, - COALESCE(node_resources_total.sru, (0)::numeric) AS total_sru, - COALESCE(count(DISTINCT node_contract.state), (0)::bigint) AS states - FROM (((public.contract_resources - JOIN public.node_contract node_contract ON ((((node_contract.resources_used_id)::text = (contract_resources.id)::text) AND ((node_contract.state)::text = ANY ((ARRAY['Created'::character varying, 'GracePeriod'::character varying])::text[]))))) - RIGHT JOIN public.node node ON ((node.node_id = node_contract.node_id))) - JOIN public.node_resources_total node_resources_total ON (((node_resources_total.node_id)::text = (node.id)::text))) - GROUP BY node.node_id, node_resources_total.mru, node_resources_total.sru, node_resources_total.hru, node_resources_total.cru; - - -ALTER TABLE public.nodes_resources_view OWNER TO postgres; - -- -- Name: nru_consumption; Type: TABLE; Schema: public; Owner: postgres -- @@ -506,11 +439,23 @@ CREATE TABLE public.public_ip ( gateway text NOT NULL, ip text NOT NULL, contract_id numeric NOT NULL, - farm_id character varying NOT NULL + farm_id character varying ); +ALTER TABLE public.public_ip OWNER TO postgres; +-- +-- Name: node_gpu; Type: TABLE; Schema: public; Owner: postgres +-- -ALTER TABLE public.public_ip OWNER TO postgres; +CREATE TABLE IF NOT EXISTS public.node_gpu ( + id text NOT NULL, + node_twin_id bigint NOT NULL, + vendor text, + device text, + contract bigint +); + +ALTER TABLE public.node_gpu OWNER TO postgres; -- -- Name: refund_transaction; Type: TABLE; Schema: public; Owner: postgres @@ -538,26 +483,65 @@ CREATE TABLE public.rent_contract ( twin_id integer NOT NULL, node_id integer NOT NULL, created_at numeric NOT NULL, - state character varying(11) NOT NULL + state character varying(11) NOT NULL, + solution_provider_id integer ); ALTER TABLE public.rent_contract OWNER TO postgres; -- --- Name: transfer; Type: TABLE; Schema: public; Owner: postgres +-- Name: service_contract; Type: TABLE; Schema: public; Owner: postgres -- -CREATE TABLE public.transfer ( +CREATE TABLE public.service_contract ( id character varying NOT NULL, - "from" text NOT NULL, - "to" text NOT NULL, - amount numeric NOT NULL, - "timestamp" numeric NOT NULL + service_contract_id numeric NOT NULL, + service_twin_id integer NOT NULL, + consumer_twin_id integer NOT NULL, + base_fee numeric NOT NULL, + variable_fee numeric NOT NULL, + metadata text NOT NULL, + accepted_by_service boolean NOT NULL, + accepted_by_consumer boolean NOT NULL, + last_bill numeric NOT NULL, + state character varying(14) NOT NULL ); -ALTER TABLE public.transfer OWNER TO postgres; +ALTER TABLE public.service_contract OWNER TO postgres; + +-- +-- Name: service_contract_bill; Type: TABLE; Schema: public; Owner: postgres +-- + +CREATE TABLE public.service_contract_bill ( + id character varying NOT NULL, + service_contract_id numeric NOT NULL, + variable_amount numeric NOT NULL, + "window" numeric NOT NULL, + metadata text, + amount numeric NOT NULL +); + + +ALTER TABLE public.service_contract_bill OWNER TO postgres; + +-- +-- Name: solution_provider; Type: TABLE; Schema: public; Owner: postgres +-- + +CREATE TABLE public.solution_provider ( + id character varying NOT NULL, + solution_provider_id numeric NOT NULL, + description text NOT NULL, + link text NOT NULL, + approved boolean NOT NULL, + providers jsonb +); + + +ALTER TABLE public.solution_provider OWNER TO postgres; -- -- Name: twin; Type: TABLE; Schema: public; Owner: postgres @@ -575,22 +559,6 @@ CREATE TABLE public.twin ( ALTER TABLE public.twin OWNER TO postgres; --- --- Name: typeorm_metadata; Type: TABLE; Schema: public; Owner: postgres --- - -CREATE TABLE public.typeorm_metadata ( - type character varying NOT NULL, - database character varying, - schema character varying, - "table" character varying, - name character varying, - value text -); - - -ALTER TABLE public.typeorm_metadata OWNER TO postgres; - -- -- Name: uptime_event; Type: TABLE; Schema: public; Owner: postgres -- @@ -606,16 +574,16 @@ CREATE TABLE public.uptime_event ( ALTER TABLE public.uptime_event OWNER TO postgres; -- --- Name: status; Type: TABLE; Schema: substrate_threefold_status; Owner: postgres +-- Name: status; Type: TABLE; Schema: squid_processor; Owner: postgres -- -CREATE TABLE substrate_threefold_status.status ( +CREATE TABLE squid_processor.status ( id integer NOT NULL, height integer NOT NULL ); -ALTER TABLE substrate_threefold_status.status OWNER TO postgres; +ALTER TABLE squid_processor.status OWNER TO postgres; -- -- Name: migrations id; Type: DEFAULT; Schema: public; Owner: postgres @@ -656,6 +624,14 @@ ALTER TABLE ONLY public.mint_transaction ADD CONSTRAINT "PK_19f4328320501dfd14e2bae0855" PRIMARY KEY (id); +-- +-- Name: service_contract_bill PK_1fd26292c0913e974b774342fa7; Type: CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.service_contract_bill + ADD CONSTRAINT "PK_1fd26292c0913e974b774342fa7" PRIMARY KEY (id); + + -- -- Name: burn_transaction PK_20ec76c5c56dd6b47dec5f0aaa8; Type: CONSTRAINT; Schema: public; Owner: postgres -- @@ -688,14 +664,6 @@ ALTER TABLE ONLY public.entity ADD CONSTRAINT "PK_50a7741b415bc585fcf9c984332" PRIMARY KEY (id); --- --- Name: account PK_54115ee388cdb6d86bb4bf5b2ea; Type: CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.account - ADD CONSTRAINT "PK_54115ee388cdb6d86bb4bf5b2ea" PRIMARY KEY (id); - - -- -- Name: contract_resources PK_557de19994fcca90916e8c6582f; Type: CONSTRAINT; Schema: public; Owner: postgres -- @@ -720,14 +688,6 @@ ALTER TABLE ONLY public.farming_policy ADD CONSTRAINT "PK_5d2ec9534104f44e4d989c4e82f" PRIMARY KEY (id); --- --- Name: historical_balance PK_74ac29ad0bdffb6d1281a1e17e8; Type: CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.historical_balance - ADD CONSTRAINT "PK_74ac29ad0bdffb6d1281a1e17e8" PRIMARY KEY (id); - - -- -- Name: refund_transaction PK_74ffc5427c595968dd777f71bf4; Type: CONSTRAINT; Schema: public; Owner: postgres -- @@ -848,6 +808,14 @@ ALTER TABLE ONLY public.nru_consumption ADD CONSTRAINT "PK_ca7956fb8fcdb7198737387d9a8" PRIMARY KEY (id); +-- +-- Name: solution_provider PK_dbb1dd40ae8f70dc9bbe2ce6347; Type: CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.solution_provider + ADD CONSTRAINT "PK_dbb1dd40ae8f70dc9bbe2ce6347" PRIMARY KEY (id); + + -- -- Name: public_ip PK_f170b0b519632730f41d2ef78f4; Type: CONSTRAINT; Schema: public; Owner: postgres -- @@ -857,11 +825,11 @@ ALTER TABLE ONLY public.public_ip -- --- Name: transfer PK_fd9ddbdd49a17afcbe014401295; Type: CONSTRAINT; Schema: public; Owner: postgres +-- Name: service_contract PK_ff58318f8230b8053067edd0343; Type: CONSTRAINT; Schema: public; Owner: postgres -- -ALTER TABLE ONLY public.transfer - ADD CONSTRAINT "PK_fd9ddbdd49a17afcbe014401295" PRIMARY KEY (id); +ALTER TABLE ONLY public.service_contract + ADD CONSTRAINT "PK_ff58318f8230b8053067edd0343" PRIMARY KEY (id); -- @@ -909,7 +877,7 @@ ALTER TABLE ONLY public.node_gpu -- Name: status status_pkey; Type: CONSTRAINT; Schema: substrate_threefold_status; Owner: postgres -- -ALTER TABLE ONLY substrate_threefold_status.status +ALTER TABLE ONLY squid_processor.status ADD CONSTRAINT status_pkey PRIMARY KEY (id); @@ -920,13 +888,6 @@ ALTER TABLE ONLY substrate_threefold_status.status CREATE INDEX "IDX_23937641f28c607f061dab4694" ON public.interfaces USING btree (node_id); --- --- Name: IDX_383ff006e4b59db91d32cb891e; Type: INDEX; Schema: public; Owner: postgres --- - -CREATE INDEX "IDX_383ff006e4b59db91d32cb891e" ON public.historical_balance USING btree (account_id); - - -- -- Name: IDX_3d9cbf30c68b79a801e1d5c9b4; Type: INDEX; Schema: public; Owner: postgres -- @@ -998,14 +959,6 @@ ALTER TABLE ONLY public.interfaces ADD CONSTRAINT "FK_23937641f28c607f061dab4694b" FOREIGN KEY (node_id) REFERENCES public.node(id); --- --- Name: historical_balance FK_383ff006e4b59db91d32cb891e9; Type: FK CONSTRAINT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.historical_balance - ADD CONSTRAINT "FK_383ff006e4b59db91d32cb891e9" FOREIGN KEY (account_id) REFERENCES public.account(id); - - -- -- Name: entity_proof FK_3d9cbf30c68b79a801e1d5c9b41; Type: FK CONSTRAINT; Schema: public; Owner: postgres -- From 3cfd407384263eeb375f2ad49867d960ffe9cdb8 Mon Sep 17 00:00:00 2001 From: Omar Abdulaziz Date: Wed, 27 Dec 2023 14:33:28 +0200 Subject: [PATCH 34/39] refactoring crafter module to be customizable Co-authored-by: Mario Wassef --- grid-proxy/Makefile | 3 +- grid-proxy/internal/explorer/db/setup.sql | 241 +++++++----------- grid-proxy/internal/explorer/db/view.sql | 85 ------ grid-proxy/tests/queries/main_test.go | 119 +++++---- grid-proxy/tests/queries/modifiers.sql | 238 ----------------- grid-proxy/tools/db/README.md | 36 +-- grid-proxy/tools/db/crafter/deleter.go | 45 ++++ .../generators.go => crafter/generator.go} | 224 +++++++++------- grid-proxy/tools/db/crafter/modifier.go | 115 +++++++++ .../tools/db/{modifiers => crafter}/types.go | 71 ++++-- .../tools/db/{modifiers => crafter}/utils.go | 7 +- .../db/{modifiers => crafter}/utils_test.go | 2 +- grid-proxy/tools/db/db.go | 50 +--- grid-proxy/tools/db/docker.sh | 2 - grid-proxy/tools/db/generate.go | 79 +++++- grid-proxy/tools/db/modifiers/modifiers.go | 158 ------------ grid-proxy/tools/db/schema.sql | 2 +- grid-proxy/tools/db/test.go | 79 ------ grid-proxy/tools/db/types.go | 145 ----------- grid-proxy/tools/db/utils.go | 112 -------- 20 files changed, 589 insertions(+), 1224 deletions(-) delete mode 100644 grid-proxy/internal/explorer/db/view.sql delete mode 100644 grid-proxy/tests/queries/modifiers.sql create mode 100644 grid-proxy/tools/db/crafter/deleter.go rename grid-proxy/tools/db/{modifiers/generators.go => crafter/generator.go} (74%) create mode 100644 grid-proxy/tools/db/crafter/modifier.go rename grid-proxy/tools/db/{modifiers => crafter}/types.go (82%) rename grid-proxy/tools/db/{modifiers => crafter}/utils.go (93%) rename grid-proxy/tools/db/{modifiers => crafter}/utils_test.go (98%) delete mode 100644 grid-proxy/tools/db/docker.sh delete mode 100644 grid-proxy/tools/db/modifiers/modifiers.go delete mode 100644 grid-proxy/tools/db/test.go delete mode 100644 grid-proxy/tools/db/types.go delete mode 100644 grid-proxy/tools/db/utils.go diff --git a/grid-proxy/Makefile b/grid-proxy/Makefile index e6ac938be..1bedce54d 100644 --- a/grid-proxy/Makefile +++ b/grid-proxy/Makefile @@ -42,7 +42,7 @@ db-stop: ## Stop the database container if running @if [ ! "$(shell docker ps | grep '$(PQ_CONTAINER)' )" = "" ]; then \ docker stop postgres; \ fi - +db-refill: db-stop db-start sleep db-fill server-start: ## Start the proxy server (Args: `m=`) @go run cmds/proxy_server/main.go \ -no-cert \ @@ -52,7 +52,6 @@ server-start: ## Start the proxy server (Args: `m=`) --postgres-db tfgrid-graphql \ --postgres-password postgres \ --postgres-user postgres \ - --sql-log-level 4 \ --mnemonics "$(m)" ; all-start: db-stop db-start sleep db-fill server-start ## Full start of the database and the server (Args: `m=`) diff --git a/grid-proxy/internal/explorer/db/setup.sql b/grid-proxy/internal/explorer/db/setup.sql index 07e6f4bb7..4d259f7aa 100644 --- a/grid-proxy/internal/explorer/db/setup.sql +++ b/grid-proxy/internal/explorer/db/setup.sql @@ -1,22 +1,3 @@ -/* - node: - free_hru, free_mru, free_sru, total_hru, total_mru, total_sru, total_cru, - - total_resources - contract_resources -> used node resources - renter -> to know who rented it, available for - node_contracts -> to know whether it's rentable - nodeid, farmid - - triggers: - - trigger on node table (insert/update) - - trigger on total resources (insert/update) - - trigger on contract resources (insert/update) - - trigger on rent contract (insert/update) - - trigger on node contract (insert/update) - - triggers need to be in the same transaction with table creation - */ BEGIN; ---- @@ -24,18 +5,20 @@ BEGIN; ---- DROP FUNCTION IF EXISTS convert_to_decimal(v_input text); -CREATE OR REPLACE FUNCTION CONVERT_TO_DECIMAL(V_INPUT -TEXT) RETURNS DECIMAL AS $$ - DECLARE v_dec_value DECIMAL DEFAULT NULL; - BEGIN BEGIN v_dec_value := v_input:: DECIMAL; - EXCEPTION - WHEN OTHERS THEN RAISE NOTICE 'Invalid decimal value: "%". Returning NULL.', - v_input; - RETURN NULL; - END; - RETURN v_dec_value; +CREATE OR REPLACE FUNCTION convert_to_decimal(v_input TEXT) RETURNS DECIMAL AS +$$ +DECLARE v_dec_value DECIMAL DEFAULT NULL; +BEGIN + BEGIN + v_dec_value := v_input:: DECIMAL; + EXCEPTION + WHEN OTHERS THEN + RAISE NOTICE 'Invalid decimal value: "%". Returning NULL.', v_input; + RETURN NULL; END; - $$ LANGUAGE plpgsql; +RETURN v_dec_value; +END; +$$ LANGUAGE plpgsql; ---- @@ -46,8 +29,6 @@ DROP TRIGGER IF EXISTS node_added ON node; ---- -- Resources cache table ---- -DROP TABLE IF EXISTS resources_cache; - DROP VIEW IF EXISTS resources_cache_view; CREATE OR REPLACE VIEW resources_cache_view AS @@ -61,26 +42,10 @@ SELECT COALESCE(node_resources_total.hru, 0) - COALESCE(sum(contract_resources.hru), 0) as free_hru, COALESCE(node_resources_total.mru, 0) - COALESCE(sum(contract_resources.mru), 0) - GREATEST(CAST((node_resources_total.mru / 10) AS bigint), 2147483648) as free_mru, COALESCE(node_resources_total.sru, 0) - COALESCE(sum(contract_resources.sru), 0) - 21474836480 as free_sru, - COALESCE( - sum(contract_resources.hru), - 0 - ) as used_hru, - COALESCE( - sum(contract_resources.mru), - 0 - ) + GREATEST( - CAST( (node_resources_total.mru / 10) AS bigint - ), - 2147483648 - ) as used_mru, - COALESCE( - sum(contract_resources.sru), - 0 - ) + 21474836480 as used_sru, - COALESCE( - sum(contract_resources.cru), - 0 - ) as used_cru, + COALESCE(sum(contract_resources.hru), 0) as used_hru, + COALESCE(sum(contract_resources.mru), 0) + GREATEST(CAST( (node_resources_total.mru / 10) AS bigint), 2147483648 ) as used_mru, + COALESCE(sum(contract_resources.sru), 0) + 21474836480 as used_sru, + COALESCE(sum(contract_resources.cru), 0) as used_cru, rent_contract.twin_id as renter, rent_contract.contract_id as rent_contract_id, count(node_contract.contract_id) as node_contracts_count, @@ -114,6 +79,7 @@ GROUP BY node.country, country.subregion; +DROP TABLE IF EXISTS resources_cache; CREATE TABLE IF NOT EXISTS resources_cache( node_id INTEGER PRIMARY KEY, farm_id INTEGER NOT NULL, @@ -144,44 +110,27 @@ FROM resources_cache_view; -- PublicIpsCache table ---- DROP TABLE IF EXISTS public_ips_cache; - -CREATE TABLE - IF NOT EXISTS public_ips_cache( - farm_id INTEGER PRIMARY KEY, - free_ips INTEGER NOT NULL, - total_ips INTEGER NOT NULL, - ips jsonb - ); +CREATE TABLE public_ips_cache( + farm_id INTEGER PRIMARY KEY, + free_ips INTEGER NOT NULL, + total_ips INTEGER NOT NULL, + ips jsonb +); INSERT INTO public_ips_cache -SELECT - farm.farm_id, - COALESCE(public_ip_agg.free_ips, 0), - COALESCE(public_ip_agg.total_ips, 0), - COALESCE(public_ip_agg.ips, '[]') + SELECT + farm.farm_id, + COALESCE(public_ip_agg.free_ips, 0), + COALESCE(public_ip_agg.total_ips, 0), + COALESCE(public_ip_agg.ips, '[]') FROM farm LEFT JOIN( SELECT p1.farm_id, COUNT(p1.id) total_ips, - COUNT( - CASE - WHEN p2.contract_id = 0 THEN 1 - END - ) free_ips, - jsonb_agg( - jsonb_build_object( - 'id', - p1.id, - 'ip', - p1.ip, - 'contract_id', - p1.contract_id, - 'gateway', - p1.gateway - ) - ) as ips - FROM public_ip p1 + COUNT(CASE WHEN p2.contract_id = 0 THEN 1 END) free_ips, + jsonb_agg(jsonb_build_object('id', p1.id, 'ip', p1.ip, 'contract_id', p1.contract_id, 'gateway', p1.gateway)) as ips + FROM public_ip AS p1 LEFT JOIN public_ip p2 ON p1.id = p2.id GROUP BY p1.farm_id @@ -195,48 +144,39 @@ CREATE EXTENSION IF NOT EXISTS pg_trgm; CREATE EXTENSION IF NOT EXISTS btree_gin; CREATE INDEX IF NOT EXISTS idx_node_id ON public.node(node_id); - CREATE INDEX IF NOT EXISTS idx_twin_id ON public.twin(twin_id); - CREATE INDEX IF NOT EXISTS idx_farm_id ON public.farm(farm_id); +CREATE INDEX IF NOT EXISTS idx_node_contract_id ON public.node_contract USING gin(id); +CREATE INDEX IF NOT EXISTS idx_name_contract_id ON public.name_contract USING gin(id); +CREATE INDEX IF NOT EXISTS idx_rent_contract_id ON public.rent_contract USING gin(id); -CREATE INDEX - IF NOT EXISTS idx_contract_id ON public.node_contract(contract_id); - -CREATE INDEX - IF NOT EXISTS resources_cache_farm_id ON resources_cache (farm_id); - -CREATE INDEX IF NOT EXISTS location_id ON location USING gin(id); -CREATE INDEX - IF NOT EXISTS resources_cache_node_id ON resources_cache(node_id); +CREATE INDEX IF NOT EXISTS idx_resources_cache_farm_id ON resources_cache (farm_id); +CREATE INDEX IF NOT EXISTS idx_resources_cache_node_id ON resources_cache(node_id); +CREATE INDEX IF NOT EXISTS idx_public_ips_cache_farm_id ON public_ips_cache(farm_id); -CREATE INDEX - IF NOT EXISTS public_ips_cache_farm_id ON public_ips_cache(farm_id); - -CREATE INDEX - IF NOT EXISTS public_config_node_id ON public_config USING gin(node_id); +CREATE INDEX IF NOT EXISTS idx_location_id ON location USING gin(id); +CREATE INDEX IF NOT EXISTS idx_public_config_node_id ON public_config USING gin(node_id); ---- --create triggers ---- - /* Node Trigger: - Insert node record > Insert new resources_cache record - Update node country > update resources_cache country/region */ -CREATE OR REPLACE FUNCTION node_upsert() RETURNS TRIGGER AS +CREATE OR REPLACE FUNCTION reflect_node_changes() RETURNS TRIGGER AS $$ BEGIN IF (TG_OP = 'UPDATE') THEN BEGIN UPDATE resources_cache - SET + SET country = NEW.country, region = ( - SELECT subregion FROM country WHERE country.name = NEW.country + SELECT subregion FROM country WHERE LOWER(country.name) = LOWER(NEW.country) ) WHERE resources_cache.node_id = NEW.node_id; @@ -269,16 +209,16 @@ BEGIN END; $$ LANGUAGE plpgsql; -CREATE OR REPLACE TRIGGER node_trigger +CREATE OR REPLACE TRIGGER tg_node AFTER INSERT OR DELETE OR UPDATE OF country - ON node - FOR EACH ROW EXECUTE PROCEDURE node_upsert(); + ON node + FOR EACH ROW EXECUTE PROCEDURE reflect_node_changes(); /* Total resources trigger - Insert/Update node_resources_total > Update equivalent resources_cache record. */ -CREATE OR REPLACE FUNCTION node_resources_total_upsert() RETURNS TRIGGER AS +CREATE OR REPLACE FUNCTION reflect_total_resources_changes() RETURNS TRIGGER AS $$ BEGIN BEGIN @@ -306,10 +246,10 @@ RETURN NULL; END; $$ LANGUAGE plpgsql; -CREATE OR REPLACE TRIGGER node_resources_total_trigger +CREATE OR REPLACE TRIGGER tg_node_resources_total AFTER INSERT OR UPDATE ON node_resources_total FOR EACH ROW - EXECUTE PROCEDURE node_resources_total_upsert(); + EXECUTE PROCEDURE reflect_total_resources_changes(); /* @@ -317,7 +257,7 @@ CREATE OR REPLACE TRIGGER node_resources_total_trigger - Insert/Update contract_resources report > update resources_cache used/free fields */ -CREATE OR REPLACE FUNCTION contract_resources_upsert() RETURNS TRIGGER AS +CREATE OR REPLACE FUNCTION reflect_contract_resources_changes() RETURNS TRIGGER AS $$ BEGIN BEGIN @@ -330,7 +270,7 @@ BEGIN free_hru = free_hru - (NEW.hru - COALESCE(OLD.hru, 0)), free_sru = free_sru - (NEW.sru - COALESCE(OLD.sru, 0)) WHERE - (SELECT state from node_contract where id = NEW.contract_id) != 'Deleted' AND + -- (SELECT state from node_contract where id = NEW.contract_id) != 'Deleted' AND resources_cache.node_id = ( SELECT node_id FROM node_contract WHERE node_contract.id = NEW.contract_id ); @@ -342,16 +282,16 @@ RETURN NULL; END; $$ LANGUAGE plpgsql; -CREATE OR REPLACE TRIGGER contract_resources_trigger +CREATE OR REPLACE TRIGGER tg_contract_resources AFTER INSERT OR UPDATE ON contract_resources FOR EACH ROW - EXECUTE PROCEDURE contract_resources_upsert(); + EXECUTE PROCEDURE reflect_contract_resources_changes(); /* Node contract trigger - Insert new contract > increment resources_cache node_contracts_count - Update contract state to 'Deleted' > decrement used an increment free fields on resources_cache */ -CREATE OR REPLACE FUNCTION node_contract_upsert() RETURNS TRIGGER AS +CREATE OR REPLACE FUNCTION reflect_node_contract_changes() RETURNS TRIGGER AS $$ BEGIN IF (TG_OP = 'UPDATE' AND NEW.state = 'Deleted') THEN @@ -391,9 +331,9 @@ RETURN NULL; END; $$ LANGUAGE plpgsql; -CREATE OR REPLACE TRIGGER node_contract_trigger +CREATE OR REPLACE TRIGGER tg_node_contract AFTER INSERT OR UPDATE OF state ON node_contract FOR EACH ROW - EXECUTE PROCEDURE node_contract_upsert(); + EXECUTE PROCEDURE reflect_node_contract_changes(); /* Rent contract trigger @@ -401,7 +341,7 @@ CREATE OR REPLACE TRIGGER node_contract_trigger - Update (state to 'Deleted') > nullify resources_cache renter/rent_contract_id */ -CREATE OR REPLACE FUNCTION rent_contract_upsert() RETURNS TRIGGER AS +CREATE OR REPLACE FUNCTION reflect_rent_contract_changes() RETURNS TRIGGER AS $$ BEGIN IF (TG_OP = 'UPDATE' AND NEW.state = 'Deleted') THEN @@ -431,9 +371,9 @@ RETURN NULL; END; $$ LANGUAGE plpgsql; -CREATE OR REPLACE TRIGGER rent_contract_trigger +CREATE OR REPLACE TRIGGER tg_rent_contract AFTER INSERT OR UPDATE OF state ON rent_contract FOR EACH ROW - EXECUTE PROCEDURE rent_contract_upsert(); + EXECUTE PROCEDURE reflect_rent_contract_changes(); /* Public ips trigger @@ -441,35 +381,37 @@ CREATE OR REPLACE TRIGGER rent_contract_trigger - Deleted > decrement total, decrement free ips (if it was used) + re-aggregate ips object - Update > increment/decrement free ips based on usage + re-aggregate ips object */ -CREATE OR REPLACE FUNCTION public_ip_upsert() RETURNS TRIGGER AS +CREATE OR REPLACE FUNCTION reflect_public_ip_changes() RETURNS TRIGGER AS $$ BEGIN BEGIN UPDATE public_ips_cache SET free_ips = free_ips + ( - CASE - -- handles insertion/update by freeing ip - WHEN TG_OP != 'DELETE' AND NEW.contract_id = 0 - THEN 1 - -- handles deletion/update by reserving ip - WHEN TG_OP != 'INSERT' AND OLD.contract_id = 0 - THEN -1 - -- handles delete reserved ips - ELSE 0 - END ), + CASE + -- handles insertion/update by freeing ip + WHEN TG_OP != 'DELETE' AND NEW.contract_id = 0 + THEN 1 + -- handles deletion/update by reserving ip + WHEN TG_OP != 'INSERT' AND OLD.contract_id = 0 + THEN -1 + -- handles delete reserved ips + ELSE 0 + END + ), total_ips = total_ips + ( - CASE - WHEN TG_OP = 'INSERT' - THEN 1 - WHEn TG_OP = 'DELETE' - THEN -1 - ELSE 0 - END ), + CASE + WHEN TG_OP = 'INSERT' + THEN 1 + WHEn TG_OP = 'DELETE' + THEN -1 + ELSE 0 + END + ), ips = ( - select jsonb_agg( + SELECT jsonb_agg( jsonb_build_object( 'id', public_ip.id, @@ -497,23 +439,22 @@ RETURN NULL; END; $$ LANGUAGE plpgsql; -CREATE OR REPLACE TRIGGER public_ip_trigger +CREATE OR REPLACE TRIGGER tg_public_ip AFTER INSERT OR DELETE OR UPDATE OF contract_id ON public_ip FOR EACH ROW - EXECUTE PROCEDURE public_ip_upsert(); + EXECUTE PROCEDURE reflect_public_ip_changes(); -CREATE OR REPLACE FUNCTION farm_insert_delete() RETURNS TRIGGER AS +CREATE OR REPLACE FUNCTION reflect_farm_changes() RETURNS TRIGGER AS $$ BEGIN - IF TG_OP = 'INSERT' THEN BEGIN - INSERT INTO public_ips_cache VALUES( - NEW.farm_id, - 0, - 0, - '[]' - ); + INSERT INTO public_ips_cache VALUES( + NEW.farm_id, + 0, + 0, + '[]' + ); EXCEPTION WHEN OTHERS THEN RAISE NOTICE 'Error inserting public_ips_cache record %s', SQLERRM; @@ -532,8 +473,8 @@ RETURN NULL; END; $$ LANGUAGE plpgsql; -CREATE OR REPLACE TRIGGER farms_trigger +CREATE OR REPLACE TRIGGER tg_farm AFTER INSERT OR DELETE ON farm FOR EACH ROW - EXECUTE PROCEDURE farm_insert_delete(); + EXECUTE PROCEDURE reflect_farm_changes(); COMMIT; \ No newline at end of file diff --git a/grid-proxy/internal/explorer/db/view.sql b/grid-proxy/internal/explorer/db/view.sql deleted file mode 100644 index 45f9fe101..000000000 --- a/grid-proxy/internal/explorer/db/view.sql +++ /dev/null @@ -1,85 +0,0 @@ -CREATE MATERIALIZED VIEW node_materialized_view AS -SELECT node.node_id, - node.twin_id, - node.farm_id, - node.power, - node.updated_at, - node.certification, - node.country, - nodes_resources_view.free_mru, - nodes_resources_view.free_hru, - nodes_resources_view.free_sru, - COALESCE(rent_contract.contract_id, 0) as rent_contract_id, - COALESCE(rent_contract.twin_id, 0) as renter, - COALESCE(node_gpu.id, '') as gpu_id -FROM node - LEFT JOIN nodes_resources_view ON node.node_id = nodes_resources_view.node_id - LEFT JOIN rent_contract ON node.node_id = rent_contract.node_id - AND rent_contract.state IN ('Created', 'GracePeriod') - LEFT JOIN node_gpu ON node.twin_id = node_gpu.node_twin_id; --- define a refresher function -CREATE OR REPLACE FUNCTION refresh_node_materialized_view() RETURNS TRIGGER AS $$ BEGIN REFRESH MATERIALIZED VIEW node_materialized_view; -RETURN NULL; -END; -$$ LANGUAGE plpgsql; --- trigger the refresh on each node update -CREATE TRIGGER refresh_node_materialized_trigger -AFTER -UPDATE ON node FOR EACH ROW EXECUTE FUNCTION refresh_node_materialized_view(); --- trigger a backup refresh each 6 hours -SELECT cron.schedule( - '0 */6 * * *', - 'SELECT refresh_node_materialized_view()' - ); -CREATE MATERIALIZED VIEW farm_materialized_view AS -SELECT farm.id, - farm.farm_id, - farm.name, - farm.twin_id, - farm.pricing_policy_id, - farm.certification, - farm.stellar_address, - farm.dedicated_farm as dedicated, - COALESCE(public_ips.public_ips, '[]') as public_ips, - bool_or(node_materialized_view.rent_contract_id != 0) as has_rent_contract -FROM farm - LEFT JOIN node_materialized_view ON node_materialized_view.farm_id = farm.farm_id - LEFT JOIN country ON node_materialized_view.country = country.name - LEFT JOIN ( - SELECT p1.farm_id, - COUNT(p1.id) total_ips, - COUNT( - CASE - WHEN p2.contract_id = 0 THEN 1 - END - ) free_ips - FROM public_ip p1 - LEFT JOIN public_ip p2 ON p1.id = p2.id - GROUP BY p1.farm_id - ) public_ip_count on public_ip_count.farm_id = farm.id - LEFT JOIN ( - SELECT farm_id, - jsonb_agg( - jsonb_build_object( - 'id', - id, - 'ip', - ip, - 'contract_id', - contract_id, - 'gateway', - gateway - ) - ) as public_ips - FROM public_ip - GROUP BY farm_id - ) public_ips on public_ips.farm_id = farm.id -GROUP BY farm.id, - farm.farm_id, - farm.name, - farm.twin_id, - farm.pricing_policy_id, - farm.certification, - farm.stellar_address, - farm.dedicated_farm, - COALESCE(public_ips.public_ips, '[]'); \ No newline at end of file diff --git a/grid-proxy/tests/queries/main_test.go b/grid-proxy/tests/queries/main_test.go index fa283520d..6e6e4fb13 100644 --- a/grid-proxy/tests/queries/main_test.go +++ b/grid-proxy/tests/queries/main_test.go @@ -16,9 +16,8 @@ import ( proxyDB "github.com/threefoldtech/tfgrid-sdk-go/grid-proxy/internal/explorer/db" proxyclient "github.com/threefoldtech/tfgrid-sdk-go/grid-proxy/pkg/client" mock "github.com/threefoldtech/tfgrid-sdk-go/grid-proxy/tests/queries/mock_client" - "github.com/threefoldtech/tfgrid-sdk-go/grid-proxy/tools/db/modifiers" + "github.com/threefoldtech/tfgrid-sdk-go/grid-proxy/tools/db/crafter" "gorm.io/gorm/logger" - // _ "embed" ) var ( @@ -38,9 +37,6 @@ var ( DBClient db.Database ) -// //go:embed modifiers.sql -// var modifiersFile string - func parseCmdline() { flag.StringVar(&POSTGRES_HOST, "postgres-host", "", "postgres host") flag.IntVar(&POSTGRES_PORT, "postgres-port", 5432, "postgres port") @@ -53,6 +49,8 @@ func parseCmdline() { } func TestMain(m *testing.M) { + var exitCode int + parseCmdline() if SEED != 0 { rand.New(rand.NewSource(int64(SEED))) @@ -67,93 +65,94 @@ func TestMain(m *testing.M) { } defer db.Close() - data, err = mock.Load(db) + // proxy client + gridProxyClient = proxyclient.NewClient(ENDPOINT) + + // mock client + dbClient, err := proxyDB.NewPostgresDatabase(POSTGRES_HOST, POSTGRES_PORT, POSTGRES_USER, POSTGRES_PASSSWORD, POSTGRES_DB, 80, logger.Error) if err != nil { panic(err) } - dbClient, err := proxyDB.NewPostgresDatabase(POSTGRES_HOST, POSTGRES_PORT, POSTGRES_USER, POSTGRES_PASSSWORD, POSTGRES_DB, 80, logger.Error) + DBClient = &dbClient + + // load mock client + data, err = mock.Load(db) if err != nil { panic(err) } - DBClient = &dbClient + // mockClient = mock.NewGridProxyMockClient(data) + err = modifyDataToFireTriggers(db, data) + if err != nil { + panic(err) + } + data, err = mock.Load(db) + if err != nil { + panic(err) + } mockClient = mock.NewGridProxyMockClient(data) - gridProxyClient = proxyclient.NewClient(ENDPOINT) - - // exitcode := m.Run() - // if exitcode != 0 { - // os.Exit(exitcode) - // } - // err = modifyDataToFireTriggers(db, data) - // if err != nil { - // panic(err) - // } - - // data, err = mock.Load(db) - // if err != nil { - // panic(err) - // } - // mockClient = mock.NewGridProxyMockClient(data) - - exitcode := m.Run() - os.Exit(exitcode) + exitCode = m.Run() + os.Exit(exitCode) } func modifyDataToFireTriggers(db *sql.DB, data mock.DBData) error { - /* - - insert nodes - y - - should be on new/old farms - - should be on new/old locations - - insert node total resources - y - - insert node contracts - y - - insert contract resources - y - - insert rent contracts - y - - insert public ips - y - - - update node country - y - - update node total resources - y - - update contract_resources - y - - update node contract state - y - - update rent contract state - y - - update public ip contract id - y - - - delete node - y - - delete public ip - y - */ - // modifiers.GenerateTwins(db) - - generator := modifiers.NewGenerator(db, SEED) - twinStart := len(data.Twins) + 1 farmStart := len(data.Farms) + 1 - farmSize := 100 nodeStart := len(data.Nodes) + 1 - nodeSize := 600 contractStart := len(data.NodeContracts) + len(data.RentContracts) + len(data.NameContracts) + 1 billStart := data.BillReports + 1 publicIpStart := len(data.PublicIPs) + 1 - size := 10 + + const ( + NodeCount = 600 + FarmCount = 100 + TwinCount = 600 + 100 + 600 // nodes + farms + normal users + PublicIPCount = 10 + NodeContractCount = 50 + NameContractCount = 10 + RentContractCount = 1 + ) + + generator := crafter.NewCrafter(db, + SEED, + NodeCount, + FarmCount, + TwinCount, + PublicIPCount, + NodeContractCount, + NameContractCount, + RentContractCount, + uint(nodeStart), + uint(farmStart), + uint(twinStart), + uint(contractStart), + uint(billStart), + uint(publicIpStart)) // insertion - if err := generator.GenerateFarms(farmStart, farmSize, twinStart); err != nil { + if err := generator.GenerateTwins(); err != nil { + return fmt.Errorf("failed to generate twins: %w", err) + } + + if err := generator.GenerateFarms(); err != nil { return fmt.Errorf("failed to generate farms: %w", err) } - if err := generator.GenerateNodes(nodeStart, nodeSize, farmStart, farmSize, twinStart); err != nil { + if err := generator.GenerateNodes(); err != nil { return fmt.Errorf("failed to generate nodes: %w", err) } // rentCount is 1 because the generate method have .1 percent of 10 farms to be dedicated - if err := generator.GenerateContracts(int(billStart), contractStart, 50, size, 1, nodeStart, nodeSize); err != nil { + if err := generator.GenerateContracts(); err != nil { return fmt.Errorf("failed to generate contracts: %w", err) } - if err := generator.GeneratePublicIPs(publicIpStart, size, farmStart, farmSize); err != nil { + if err := generator.GeneratePublicIPs(); err != nil { return fmt.Errorf("failed to generate public ips: %w", err) } - // // updates + // updates if err := generator.UpdateNodeCountry(); err != nil { return fmt.Errorf("failed to update node country: %w", err) } @@ -179,7 +178,7 @@ func modifyDataToFireTriggers(db *sql.DB, data mock.DBData) error { } // // deletions - if err := generator.DeleteNodes(len(data.Nodes) + nodeSize); err != nil { + if err := generator.DeleteNodes(); err != nil { return fmt.Errorf("failed to delete node: %w", err) } diff --git a/grid-proxy/tests/queries/modifiers.sql b/grid-proxy/tests/queries/modifiers.sql deleted file mode 100644 index 930f27cbb..000000000 --- a/grid-proxy/tests/queries/modifiers.sql +++ /dev/null @@ -1,238 +0,0 @@ --- node modifiers -INSERT INTO node ( - id, - grid_version, - node_id, - farm_id, - twin_id, - country, - city, - uptime, - created, - farming_policy_id, - secure, - virtualized, - serial_number, - created_at, - updated_at, - location_id, - certification, - connection_price, - power, - extra_fee - ) -VALUES ( - 'node-id-999999', - 3, - 999999, - 1, - 999999, - 'Egypt', - 'Cairo', - 1651769370, - 1651769370, - 1, - false, - false, - '', - 1651769370, - 1651769370, - 'location-1', - 'not', - 0, - '{}', - 0 - ); -UPDATE node -SET country = 'Belgium' -WHERE node_id = 999999; -INSERT INTO node_resources_total (id, hru, sru, cru, mru, node_id) -VALUES ( - 'total-resources-999999', - 10000000000000, - 10000000000000, - 1000, - 10000000000000, - 'node-id-999999' - ); -UPDATE node_resources_total -SET cru = 2000, - hru = 20000000000000, - mru = 20000000000000, - sru = 20000000000000 -WHERE node_id = 'node-id-999999'; --- capacity modifiers -INSERT INTO node_contract ( - id, - grid_version, - contract_id, - twin_id, - node_id, - deployment_data, - deployment_hash, - number_of_public_i_ps, - created_at, - resources_used_id, - state - ) -VALUES ( - 'node-contract-999999', - 1, - 999999, - 99, - 999999, - 'deployment_data:text', - 'deployment_hash:text', - 0, - 1600000000, - NULL, - 'Created' - ); -INSERT INTO node_contract ( - id, - grid_version, - contract_id, - twin_id, - node_id, - deployment_data, - deployment_hash, - number_of_public_i_ps, - created_at, - resources_used_id, - state - ) -VALUES ( - 'node-contract-999998', - 1, - 999998, - 99, - 999999, - 'deployment_data:text', - 'deployment_hash:text', - 0, - 1600000000, - NULL, - 'Created' - ); -INSERT INTO node_contract ( - id, - grid_version, - contract_id, - twin_id, - node_id, - deployment_data, - deployment_hash, - number_of_public_i_ps, - created_at, - resources_used_id, - state - ) -VALUES ( - 'node-contract-999995', - 1, - 999995, - 99, - 999999, - 'deployment_data:text', - 'deployment_hash:text', - 0, - 1600000000, - NULL, - 'Created' - ); -INSERT INTO contract_resources (id, hru, sru, cru, mru, contract_id) -VALUES ( - 'contract-resources-999999', - 1, - 1, - 1, - 1, - 'node-contract-999999' - ); -INSERT INTO contract_resources (id, hru, sru, cru, mru, contract_id) -VALUES ( - 'contract-resources-999998', - 1, - 1, - 1, - 1, - 'node-contract-999998' - ); -update node_contract set resources_used_id = 'contract-resources-999999' where contract_id = 999999; -update node_contract set resources_used_id = 'contract-resources-999998' where contract_id = 999998; -UPDATE contract_resources -SET cru = 1 -where contract_id = 'node-contract-999999'; -update node_contract -SET state = 'Deleted' -where contract_id = 999999; --- renting modifiers -INSERT INTO rent_contract ( - id, - grid_version, - contract_id, - twin_id, - node_id, - created_at, - state - ) -VALUES ( - 'rent-contract-999997', - 1, - 999997, - 99, - 999999, - 15000000, - 'Created' - ); -Update rent_contract -set state = 'Deleted' -where contract_id = 999997; -INSERT INTO rent_contract ( - id, - grid_version, - contract_id, - twin_id, - node_id, - created_at, - state - ) -VALUES ( - 'rent-contract-999996', - 1, - 999996, - 99, - 999999, - 15000000, - 'Created' - ); --- ips modifiers -INSERT INTO public_ip (id, gateway, ip, contract_id, farm_id) -VALUES ( - 'public-ip-999999', - 'gateway:text', - 'ip:text', - 0, - 'farm-1' - ); -INSERT INTO public_ip (id, gateway, ip, contract_id, farm_id) -VALUES ( - 'public-ip-999998', - 'gateway:text', - 'ip:text', - 0, - 'farm-1' - ); -update public_ip -set contract_id = 999998 -where id = 'public-ip-999999'; -update public_ip -set contract_id = 999995 -where id = 'public-ip-999998'; -update public_ip -set contract_id = 0 -where id = 'public-ip-999999'; -Delete from public_ip -where id = 'public-ip-999999'; -Delete from public_ip -where id = 'public-ip-999998'; \ No newline at end of file diff --git a/grid-proxy/tools/db/README.md b/grid-proxy/tools/db/README.md index 59ef37586..a073f41d8 100644 --- a/grid-proxy/tools/db/README.md +++ b/grid-proxy/tools/db/README.md @@ -1,30 +1,22 @@ -# DB for testing -## Run postgresql container - +## Data Preparation for Testing + +- Starting the postgres container + ```bash - docker run --rm --name postgres \ - -e POSTGRES_USER=postgres \ - -e POSTGRES_PASSWORD=postgres \ - -e POSTGRES_DB=tfgrid-graphql \ - -p 5432:5432 -d postgres + make db-start ``` -## Create the DB -you can either Generate a db with relevant schema to test things locally quickly, or load a previously taken DB dump file: - -### Method 1: Generate a db with relevant schema using the db helper tool: +- Populate the database using one of the two available methods: - ```bash - cd tools/db/ && go run . \ - --postgres-host 127.0.0.1 \ - --postgres-db tfgrid-graphql \ - --postgres-password postgres \ - --postgres-user postgres \ - --reset \ - ``` + - Generate a mock database -### Method 2: Fill the DB from a Production db dump file, for example if you have `dump.sql` file, you can run: + - `./schema.sql` is a database schema taken from graphql processor on dev net, it creates all the needed tables for the database. + - the `data-crafter` have a various of methods to create/update/delete the main tables on the database. it can be customized by the params passed to the `NewCrafter`. + ```bash + make db-fill + ``` + - Use a data dump file (contains both schema and data) ```bash - psql -h 127.0.0.1 -U postgres -d tfgrid-graphql < dump.sql + make db-dump p= ``` diff --git a/grid-proxy/tools/db/crafter/deleter.go b/grid-proxy/tools/db/crafter/deleter.go new file mode 100644 index 000000000..a813825b6 --- /dev/null +++ b/grid-proxy/tools/db/crafter/deleter.go @@ -0,0 +1,45 @@ +package crafter + +import ( + "fmt" +) + +// deletions +func (g *Crafter) DeleteNodes() error { + // delete node contracts on this node + // free public ips that are assigned to the deleted contracts + // delete rent contracts on this node + // delete node + deleteCount := r.Intn(10) + 1 + query := "" + + for i := 0; i < deleteCount; i++ { + nodeID := int(g.NodeCount) - i + + query += fmt.Sprintf("UPDATE public_ip SET contract_id = 0 WHERE contract_id IN (SELECT contract_id FROM node_contract WHERE node_id = %d);", nodeID) + query += fmt.Sprintf("UPDATE node_contract SET state = 'Deleted' WHERE node_id = %d;", nodeID) + query += fmt.Sprintf("UPDATE rent_contract set state = 'Deleted' WHERE node_id = %d;", nodeID) + query += fmt.Sprintf("DELETE FROM node_resources_total WHERE node_id = (SELECT id FROM node WHERE node_id = %d);", nodeID) + query += fmt.Sprintf("DELETE FROM public_config WHERE node_id = (SELECT id FROM node WHERE node_id = %d);", nodeID) + query += fmt.Sprintf("DELETE FROM node WHERE node_id = %d;", nodeID) + } + + fmt.Println("nodes deleted") + + _, err := g.db.Exec(query) + return err +} + +func (g *Crafter) DeletePublicIps() error { + maxDeleteCount := r.Intn(10) + 1 + query := fmt.Sprintf("DELETE FROM public_ip WHERE id in (SELECT id FROM public_ip WHERE contract_id = 0 LIMIT %d);", maxDeleteCount) + + _, err := g.db.Exec(query) + if err != nil { + return fmt.Errorf("failed to delete public ips: %w", err) + } + + fmt.Println("public ips deleted") + + return nil +} diff --git a/grid-proxy/tools/db/modifiers/generators.go b/grid-proxy/tools/db/crafter/generator.go similarity index 74% rename from grid-proxy/tools/db/modifiers/generators.go rename to grid-proxy/tools/db/crafter/generator.go index d637607a3..62985db63 100644 --- a/grid-proxy/tools/db/modifiers/generators.go +++ b/grid-proxy/tools/db/crafter/generator.go @@ -1,4 +1,4 @@ -package modifiers +package crafter import ( "fmt" @@ -13,10 +13,13 @@ const deleted = "Deleted" const created = "Created" const gracePeriod = "GracePeriod" -func (g *Generator) GenerateTwins(start, size int) error { +func (c *Crafter) GenerateTwins() error { + start := c.TwinStart + end := c.TwinCount + c.TwinStart + var twins []string - for i := uint64(start); i < uint64(start+size); i++ { + for i := uint64(start); i < uint64(end); i++ { twin := twin{ id: fmt.Sprintf("twin-%d", i), account_id: fmt.Sprintf("account-id-%d", i), @@ -32,32 +35,35 @@ func (g *Generator) GenerateTwins(start, size int) error { twins = append(twins, tuple) } - if err := g.insertTuples(twin{}, twins); err != nil { + if err := c.insertTuples(twin{}, twins); err != nil { return fmt.Errorf("failed to insert twins: %w", err) } - fmt.Println("twins generated") + fmt.Printf("twins generated [%d : %d[\n", start, end) return nil } -func (g *Generator) GenerateFarms(start, size, twinStart int) error { - var farms []string +func (c *Crafter) GenerateFarms() error { + start := c.FarmStart + end := c.FarmCount + c.FarmStart + farmTwinsStart := c.TwinStart + c.FarmStart - for i := uint64(start); i < uint64(start+size); i++ { + var farms []string + for i := uint64(start); i < uint64(end); i++ { farm := farm{ id: fmt.Sprintf("farm-%d", i), farm_id: i, name: fmt.Sprintf("farm-name-%d", i), certification: "Diy", dedicated_farm: flip(.1), - twin_id: uint64(twinStart) + (i - uint64(start)), + twin_id: uint64(farmTwinsStart) + i, pricing_policy_id: 1, grid_version: 3, stellar_address: "", } if farm.dedicated_farm { - g.dedicatedFarms[farm.farm_id] = struct{}{} + c.dedicatedFarms[farm.farm_id] = struct{}{} } farmTuple, err := objectToTupleString(farm) @@ -67,21 +73,25 @@ func (g *Generator) GenerateFarms(start, size, twinStart int) error { farms = append(farms, farmTuple) } - if err := g.insertTuples(farm{}, farms); err != nil { + if err := c.insertTuples(farm{}, farms); err != nil { return fmt.Errorf("failed to insert farms: %w", err) } - fmt.Println("farms generated") + fmt.Printf("farms generated [%d : %d[\n", start, end) return nil } -func (g *Generator) GenerateNodes(start, size, farmStart, farmSize, twinStart int) error { +func (c *Crafter) GenerateNodes() error { + start := c.NodeStart + end := c.NodeStart + c.NodeCount + nodeTwinsStart := c.TwinStart + (c.FarmStart + c.FarmCount) + powerState := []string{"Up", "Down"} var locations []string var nodes []string var totalResources []string var publicConfigs []string - for i := uint64(start); i < uint64(start+size); i++ { + for i := uint64(start); i < uint64(end); i++ { mru, err := rnd(4, 256) if err != nil { return fmt.Errorf("failed to generate random mru: %w", err) @@ -120,30 +130,30 @@ func (g *Generator) GenerateNodes(start, size, farmStart, farmSize, twinStart in updatedAt = time.Now().Unix() - int64(periodFromLatestUpdate) } - g.nodesMRU[i] = mru - max(2*uint64(gridtypes.Gigabyte), mru/10) - g.nodesSRU[i] = sru - 100*uint64(gridtypes.Gigabyte) - g.nodesHRU[i] = hru - g.nodeUP[i] = up + c.nodesMRU[i] = mru - max(2*uint64(gridtypes.Gigabyte), mru/10) + c.nodesSRU[i] = sru - 100*uint64(gridtypes.Gigabyte) + c.nodesHRU[i] = hru + c.nodeUP[i] = up // location latitude and longitue needs to be castable to decimal // if not, the convert_to_decimal function will raise a notice // reporting the incident, which downgrades performance + locationId := fmt.Sprintf("location-%d", uint64(start)+i) location := location{ - id: fmt.Sprintf("location-%d", i), + id: locationId, longitude: fmt.Sprintf("%d", i), latitude: fmt.Sprintf("%d", i), } countryIndex := r.Intn(len(countries)) cityIndex := r.Intn(len(cities[countries[countryIndex]])) - farmId := r.Intn((farmStart+farmSize)-farmStart) + farmStart - twinId := twinStart + farmSize + (int(i) - start) + farmId := r.Intn(int(c.FarmCount)) + int(c.FarmStart) node := node{ id: fmt.Sprintf("node-%d", i), - location_id: fmt.Sprintf("location-%d", i), + location_id: locationId, node_id: i, farm_id: uint64(farmId), - twin_id: uint64(twinId), + twin_id: uint64(nodeTwinsStart) + i, country: countries[countryIndex], city: cities[countries[countryIndex]][cityIndex], uptime: 1000, @@ -161,6 +171,7 @@ func (g *Generator) GenerateNodes(start, size, farmStart, farmSize, twinStart in Target: powerState[r.Intn(len(powerState))], }, extra_fee: 0, + dedicated: false, } total_resources := node_resources_total{ @@ -172,9 +183,9 @@ func (g *Generator) GenerateNodes(start, size, farmStart, farmSize, twinStart in node_id: fmt.Sprintf("node-%d", i), } - if _, ok := g.dedicatedFarms[node.farm_id]; ok { - g.availableRentNodes[i] = struct{}{} - g.availableRentNodesList = append(g.availableRentNodesList, i) + if _, ok := c.dedicatedFarms[node.farm_id]; ok { + c.availableRentNodes[i] = struct{}{} + c.availableRentNodesList = append(c.availableRentNodesList, i) } locationTuple, err := objectToTupleString(location) @@ -214,55 +225,56 @@ func (g *Generator) GenerateNodes(start, size, farmStart, farmSize, twinStart in } } - if err := g.insertTuples(location{}, locations); err != nil { + if err := c.insertTuples(location{}, locations); err != nil { return fmt.Errorf("failed to insert locations: %w", err) } - if err := g.insertTuples(node{}, nodes); err != nil { + if err := c.insertTuples(node{}, nodes); err != nil { return fmt.Errorf("failed to isnert nodes: %w", err) } - if err := g.insertTuples(node_resources_total{}, totalResources); err != nil { + if err := c.insertTuples(node_resources_total{}, totalResources); err != nil { return fmt.Errorf("failed to insert node resources total: %w", err) } - if err := g.insertTuples(public_config{}, publicConfigs); err != nil { + if err := c.insertTuples(public_config{}, publicConfigs); err != nil { return fmt.Errorf("failed to insert public configs: %w", err) } - fmt.Println("nodes generated") + fmt.Printf("nodes generated [%d : %d[\n", start, end) return nil } -func (g *Generator) GenerateContracts(billStart, contractStart, nodeConCount, nameConCount, rentConCount, nodeStart, nodeSize int) error { +func (c *Crafter) GenerateContracts() error { var billReports []string - rentContractsBillReports, nodeContractIDStart, err := g.GenerateRentContracts(billStart, contractStart, rentConCount) + rentContractsBillReports, nodeContractIDStart, err := c.GenerateRentContracts(int(c.BillStart), int(c.ContractStart), int(c.RentContractCount)) if err != nil { return fmt.Errorf("failed to generate rent contracts: %w", err) } billReports = append(billReports, rentContractsBillReports...) - nodeContractsBillReports, nameContractIDStart, err := g.generateNodeContracts(len(billReports)+billStart, nodeContractIDStart, nodeConCount, nodeStart, nodeSize) + nodeContractsBillReports, nameContractIDStart, err := c.generateNodeContracts(len(billReports)+int(c.BillStart), nodeContractIDStart, int(c.NodeContractCount), int(c.NodeStart), int(c.NodeCount)) if err != nil { return fmt.Errorf("failed to generate node contracts: %w", err) } billReports = append(billReports, nodeContractsBillReports...) - nameContractsBillReports, _, err := g.GenerateNameContracts(len(billReports)+billStart, nameContractIDStart, nameConCount) + nameContractsBillReports, _, err := c.GenerateNameContracts(len(billReports)+int(c.BillStart), nameContractIDStart, int(c.NameContractCount)) if err != nil { return fmt.Errorf("failed to generate name contracts: %w", err) } billReports = append(billReports, nameContractsBillReports...) - if err := g.insertTuples(contract_bill_report{}, billReports); err != nil { + if err := c.insertTuples(contract_bill_report{}, billReports); err != nil { return fmt.Errorf("failed to generate contract bill reports: %w", err) } return nil } -func (g *Generator) generateNodeContracts(billsStartID, contractsStartID, contractCount, nodeStart, nodeSize int) ([]string, int, error) { +func (c *Crafter) generateNodeContracts(billsStartID, contractsStartID, contractCount, nodeStart, nodeSize int) ([]string, int, error) { end := contractsStartID + contractCount + start := contractsStartID var contracts []string var contractResources []string @@ -270,13 +282,13 @@ func (g *Generator) generateNodeContracts(billsStartID, contractsStartID, contra toDelete := "" toGracePeriod := "" contractToResource := map[uint64]string{} - for i := contractsStartID; i < end; i++ { - nodeID := uint64(r.Intn((nodeStart+nodeSize)-nodeStart) + nodeStart) + for i := start; i < end; i++ { + nodeID := uint64(r.Intn(nodeSize) + nodeStart) state := deleted - if g.nodeUP[nodeID] { + if c.nodeUP[nodeID] { if flip(0.9) { state = created - } else if flip(0.5) { + } else if flip(0.4) { state = gracePeriod } } @@ -295,7 +307,7 @@ func (g *Generator) generateNodeContracts(billsStartID, contractsStartID, contra toGracePeriod += fmt.Sprint(i) } - if state != deleted && (minContractHRU > g.nodesHRU[nodeID] || minContractMRU > g.nodesMRU[nodeID] || minContractSRU > g.nodesSRU[nodeID]) { + if state != deleted && (minContractHRU > c.nodesHRU[nodeID] || minContractMRU > c.nodesMRU[nodeID] || minContractSRU > c.nodesSRU[nodeID]) { i-- continue } @@ -305,11 +317,11 @@ func (g *Generator) generateNodeContracts(billsStartID, contractsStartID, contra return nil, i, fmt.Errorf("failed to generate random twin id: %w", err) } - if renter, ok := g.renter[nodeID]; ok { + if renter, ok := c.renter[nodeID]; ok { twinID = renter } - if _, ok := g.availableRentNodes[nodeID]; ok { + if _, ok := c.availableRentNodes[nodeID]; ok { i-- continue } @@ -333,17 +345,17 @@ func (g *Generator) generateNodeContracts(billsStartID, contractsStartID, contra return nil, i, fmt.Errorf("failed to generate random cru: %w", err) } - hru, err := rnd(minContractHRU, min(maxContractHRU, g.nodesHRU[nodeID])) + hru, err := rnd(minContractHRU, min(maxContractHRU, c.nodesHRU[nodeID])) if err != nil { return nil, i, fmt.Errorf("failed to generate random hru: %w", err) } - sru, err := rnd(minContractSRU, min(maxContractSRU, g.nodesSRU[nodeID])) + sru, err := rnd(minContractSRU, min(maxContractSRU, c.nodesSRU[nodeID])) if err != nil { return nil, i, fmt.Errorf("failed to generate random sru: %w", err) } - mru, err := rnd(minContractMRU, min(maxContractMRU, g.nodesMRU[nodeID])) + mru, err := rnd(minContractMRU, min(maxContractMRU, c.nodesMRU[nodeID])) if err != nil { return nil, i, fmt.Errorf("failed to generate random mru: %w", err) } @@ -359,10 +371,10 @@ func (g *Generator) generateNodeContracts(billsStartID, contractsStartID, contra contractToResource[contract.contract_id] = contract_resources.id if state != deleted { - g.nodesHRU[nodeID] -= hru - g.nodesSRU[nodeID] -= sru - g.nodesMRU[nodeID] -= mru - g.createdNodeContracts = append(g.createdNodeContracts, uint64(i)) + c.nodesHRU[nodeID] -= hru + c.nodesSRU[nodeID] -= sru + c.nodesMRU[nodeID] -= mru + c.createdNodeContracts = append(c.createdNodeContracts, uint64(i)) } contractTuple, err := objectToTupleString(contract) @@ -404,62 +416,64 @@ func (g *Generator) generateNodeContracts(billsStartID, contractsStartID, contra } } - if err := g.insertTuples(node_contract{}, contracts); err != nil { + if err := c.insertTuples(node_contract{}, contracts); err != nil { return nil, end, fmt.Errorf("failed to insert node contracts: %w", err) } - if err := g.insertTuples(contract_resources{}, contractResources); err != nil { + if err := c.insertTuples(contract_resources{}, contractResources); err != nil { return nil, end, fmt.Errorf("failed to insert contract resources: %w", err) } - if err := g.updateNodeContractResourceID(contractToResource); err != nil { + if err := c.updateNodeContractResourceID(contractToResource); err != nil { return nil, end, fmt.Errorf("failed to update node contract resources id: %w", err) } if len(toDelete) > 0 { - if _, err := g.db.Exec(fmt.Sprintf("UPDATE node_contract SET state = '%s' WHERE contract_id IN (%s)", deleted, toDelete)); err != nil { + if _, err := c.db.Exec(fmt.Sprintf("UPDATE node_contract SET state = '%s' WHERE contract_id IN (%s)", deleted, toDelete)); err != nil { return nil, 0, fmt.Errorf("failed to update node_contract state to deleted: %w", err) } } if len(toGracePeriod) > 0 { - if _, err := g.db.Exec(fmt.Sprintf("UPDATE node_contract SET state = '%s' WHERE contract_id IN (%s)", gracePeriod, toGracePeriod)); err != nil { + if _, err := c.db.Exec(fmt.Sprintf("UPDATE node_contract SET state = '%s' WHERE contract_id IN (%s)", gracePeriod, toGracePeriod)); err != nil { return nil, 0, fmt.Errorf("failed to update node_contract state to grace period: %w", err) } } - fmt.Println("node contracts generated") + fmt.Printf("node contracts generated [%d : %d[\n", start, end) return billingReports, end, nil } -func (g *Generator) updateNodeContractResourceID(contractToResource map[uint64]string) error { +func (c *Crafter) updateNodeContractResourceID(contractToResource map[uint64]string) error { query := "" for contractID, ResourceID := range contractToResource { query += fmt.Sprintf("UPDATE node_contract SET resources_used_id = '%s' WHERE contract_id = %d;", ResourceID, contractID) } - if _, err := g.db.Exec(query); err != nil { + if _, err := c.db.Exec(query); err != nil { return fmt.Errorf("failed to update node contract resource id: %w", err) } return nil } -func (g *Generator) GenerateNameContracts(billsStartID, contractsStartID, contractCount int) ([]string, int, error) { +func (c *Crafter) GenerateNameContracts(billsStartID, contractsStartID, contractCount int) ([]string, int, error) { end := contractsStartID + contractCount + start := contractsStartID var contracts []string var billReports []string toDelete := "" toGracePeriod := "" - for i := contractsStartID; i < end; i++ { - nodeID, err := rnd(1, uint64(NodeCount)) + for i := start; i < end; i++ { + // WATCH: + nodeID, err := rnd(1, uint64(c.NodeCount)) if err != nil { return nil, i, fmt.Errorf("failed to generate random node id: %w", err) } state := deleted - if g.nodeUP[nodeID] { + if c.nodeUP[nodeID] { if flip(0.9) { state = created } else if flip(0.5) { @@ -486,11 +500,11 @@ func (g *Generator) GenerateNameContracts(billsStartID, contractsStartID, contra return nil, i, fmt.Errorf("failed to generate random twin id: %w", err) } - if renter, ok := g.renter[nodeID]; ok { + if renter, ok := c.renter[nodeID]; ok { twinID = renter } - if _, ok := g.availableRentNodes[nodeID]; ok { + if _, ok := c.availableRentNodes[nodeID]; ok { i-- continue } @@ -538,44 +552,45 @@ func (g *Generator) GenerateNameContracts(billsStartID, contractsStartID, contra } } - if err := g.insertTuples(name_contract{}, contracts); err != nil { + if err := c.insertTuples(name_contract{}, contracts); err != nil { return nil, end, fmt.Errorf("failed to insert name contracts: %w", err) } if len(toDelete) > 0 { - if _, err := g.db.Exec(fmt.Sprintf("UPDATE rent_contract SET state = '%s' WHERE contract_id IN (%s)", deleted, toDelete)); err != nil { + if _, err := c.db.Exec(fmt.Sprintf("UPDATE rent_contract SET state = '%s' WHERE contract_id IN (%s)", deleted, toDelete)); err != nil { return nil, 0, fmt.Errorf("failed to update rent_contract state to deleted: %w", err) } } if len(toGracePeriod) > 0 { - if _, err := g.db.Exec(fmt.Sprintf("UPDATE rent_contract SET state = '%s' WHERE contract_id IN (%s)", gracePeriod, toGracePeriod)); err != nil { + if _, err := c.db.Exec(fmt.Sprintf("UPDATE rent_contract SET state = '%s' WHERE contract_id IN (%s)", gracePeriod, toGracePeriod)); err != nil { return nil, 0, fmt.Errorf("failed to update rent_contract state to grace period: %w", err) } } - fmt.Println("name contracts generated") + fmt.Printf("name contracts generated [%d : %d[\n", start, end) return billReports, end, nil } -func (g *Generator) GenerateRentContracts(billsStart, contractStart, rentConCount int) ([]string, int, error) { +func (c *Crafter) GenerateRentContracts(billsStart, contractStart, rentConCount int) ([]string, int, error) { end := contractStart + rentConCount + start := contractStart var contracts []string var billReports []string toDelete := "" toGracePeriod := "" - for i := contractStart; i < end; i++ { - nl, nodeID, err := popRandom(g.availableRentNodesList) + for i := start; i < end; i++ { + nl, nodeID, err := popRandom(c.availableRentNodesList) if err != nil { return nil, i, fmt.Errorf("failed to select random element from the given slice: %w", err) } - g.availableRentNodesList = nl - delete(g.availableRentNodes, nodeID) + c.availableRentNodesList = nl + delete(c.availableRentNodes, nodeID) state := deleted - if g.nodeUP[nodeID] { + if c.nodeUP[nodeID] { if flip(0.9) { state = created } else if flip(0.5) { @@ -613,7 +628,7 @@ func (g *Generator) GenerateRentContracts(billsStart, contractStart, rentConCoun } if state != deleted { - g.renter[nodeID] = contract.twin_id + c.renter[nodeID] = contract.twin_id } contractTuple, err := objectToTupleString(contract) @@ -652,44 +667,47 @@ func (g *Generator) GenerateRentContracts(billsStart, contractStart, rentConCoun } } - if err := g.insertTuples(rent_contract{}, contracts); err != nil { + if err := c.insertTuples(rent_contract{}, contracts); err != nil { return nil, end, fmt.Errorf("failed to insert rent contracts: %w", err) } if len(toDelete) > 0 { - if _, err := g.db.Exec(fmt.Sprintf("UPDATE rent_contract SET state = '%s' WHERE contract_id IN (%s)", deleted, toDelete)); err != nil { + if _, err := c.db.Exec(fmt.Sprintf("UPDATE rent_contract SET state = '%s' WHERE contract_id IN (%s)", deleted, toDelete)); err != nil { return nil, 0, fmt.Errorf("failed to update rent_contract state to deleted: %w", err) } } if len(toGracePeriod) > 0 { - if _, err := g.db.Exec(fmt.Sprintf("UPDATE rent_contract SET state = '%s' WHERE contract_id IN (%s)", gracePeriod, toGracePeriod)); err != nil { + if _, err := c.db.Exec(fmt.Sprintf("UPDATE rent_contract SET state = '%s' WHERE contract_id IN (%s)", gracePeriod, toGracePeriod)); err != nil { return nil, 0, fmt.Errorf("failed to update rent_contract state to grace period: %w", err) } } - fmt.Println("rent contracts generated") + fmt.Printf("rent contracts generated [%d : %d[\n", start, end) return billReports, end, nil } -func (g *Generator) GeneratePublicIPs(start, size, farmStart, farmSize int) error { +func (c *Crafter) GeneratePublicIPs() error { + start := c.PublicIPStart + end := c.PublicIPCount + c.PublicIPStart + var publicIPs []string var nodeContracts []uint64 reservedIPs := map[string]uint64{} - for i := uint64(start); i < uint64(start+size); i++ { + for i := uint64(start); i < uint64(end); i++ { contract_id := uint64(0) if flip(usedPublicIPsRatio) { - idx, err := rnd(0, uint64(len(g.createdNodeContracts))-1) + idx, err := rnd(0, uint64(len(c.createdNodeContracts))-1) if err != nil { return fmt.Errorf("failed to generate random index: %w", err) } - contract_id = g.createdNodeContracts[idx] + contract_id = c.createdNodeContracts[idx] } ip := randomIPv4() - farmID := r.Int63n(int64(farmSize)) + int64(farmStart) + farmID := r.Int63n(int64(c.FarmCount)) + int64(c.FarmStart) public_ip := public_ip{ id: fmt.Sprintf("public-ip-%d", i), @@ -711,26 +729,26 @@ func (g *Generator) GeneratePublicIPs(start, size, farmStart, farmSize int) erro nodeContracts = append(nodeContracts, contract_id) } - if err := g.insertTuples(public_ip{}, publicIPs); err != nil { + if err := c.insertTuples(public_ip{}, publicIPs); err != nil { return fmt.Errorf("failed to insert public ips: %w", err) } - if err := g.updateNodeContractPublicIPs(nodeContracts); err != nil { + if err := c.updateNodeContractPublicIPs(nodeContracts); err != nil { return fmt.Errorf("failed to update contract public ips: %w", err) } for id, contractID := range reservedIPs { - if _, err := g.db.Exec(fmt.Sprintf("UPDATE public_ip SET contract_id = %d WHERE id = '%s'", contractID, id)); err != nil { + if _, err := c.db.Exec(fmt.Sprintf("UPDATE public_ip SET contract_id = %d WHERE id = '%s'", contractID, id)); err != nil { return fmt.Errorf("failed to reserve ip %s: %w", id, err) } } - fmt.Println("public IPs generated") + fmt.Printf("public IPs generated [%d : %d[\n", start, end) return nil } -func (g *Generator) updateNodeContractPublicIPs(nodeContracts []uint64) error { +func (c *Crafter) updateNodeContractPublicIPs(nodeContracts []uint64) error { if len(nodeContracts) != 0 { var IDs []string @@ -741,27 +759,31 @@ func (g *Generator) updateNodeContractPublicIPs(nodeContracts []uint64) error { query := "UPDATE node_contract set number_of_public_i_ps = number_of_public_i_ps + 1 WHERE contract_id IN (" query += strings.Join(IDs, ",") + ");" - if _, err := g.db.Exec(query); err != nil { + if _, err := c.db.Exec(query); err != nil { return fmt.Errorf("failed to update node contracts public ips: %w", err) } } return nil } -func (g *Generator) GenerateNodeGPUs() error { +func (c *Crafter) GenerateNodeGPUs() error { var GPUs []string vendors := []string{"NVIDIA Corporation", "AMD", "Intel Corporation"} devices := []string{"GeForce RTX 3080", "Radeon RX 6800 XT", "Intel Iris Xe MAX"} - for i := 0; i <= 10; i++ { + nodeTwinsStart := c.TwinStart + (c.FarmStart + c.FarmCount) + nodeWithGpuNum := 10 + + for i := 1; i <= nodeWithGpuNum; i++ { gpuNum := len(vendors) - 1 for j := 0; j <= gpuNum; j++ { g := node_gpu{ - node_twin_id: uint64(i + FarmCount + 2), // node twin ids start from 102 + // WATCH + node_twin_id: uint64(nodeTwinsStart + uint(i)), vendor: vendors[j], device: devices[j], contract: i % 2, - id: fmt.Sprintf("0000:0e:00.0/1002/744c/%d", j), + id: fmt.Sprintf("node-gpu-%d-%d", nodeTwinsStart+uint(i), j), } gpuTuple, err := objectToTupleString(g) if err != nil { @@ -771,7 +793,7 @@ func (g *Generator) GenerateNodeGPUs() error { } } - if err := g.insertTuples(node_gpu{}, GPUs); err != nil { + if err := c.insertTuples(node_gpu{}, GPUs); err != nil { return fmt.Errorf("failed to insert node gpu: %w", err) } @@ -780,9 +802,13 @@ func (g *Generator) GenerateNodeGPUs() error { return nil } -func (g *Generator) GenerateCountries() error { +func (c *Crafter) GenerateCountries() error { var countriesValues []string - index := 0 + + // depends on nodeStart to not duplicate the value of country.id + start := c.NodeStart + + index := start for countryName, region := range regions { index++ country := country{ @@ -803,7 +829,7 @@ func (g *Generator) GenerateCountries() error { countriesValues = append(countriesValues, countryTuple) } - if err := g.insertTuples(country{}, countriesValues); err != nil { + if err := c.insertTuples(country{}, countriesValues); err != nil { return fmt.Errorf("failed to insert country: %w", err) } fmt.Println("countries generated") diff --git a/grid-proxy/tools/db/crafter/modifier.go b/grid-proxy/tools/db/crafter/modifier.go new file mode 100644 index 000000000..0f58717be --- /dev/null +++ b/grid-proxy/tools/db/crafter/modifier.go @@ -0,0 +1,115 @@ +package crafter + +import ( + "fmt" +) + +func (g *Crafter) UpdateNodeCountry() error { + updatesCount := 10 + query := "" + + for i := 0; i < updatesCount; i++ { + // WATCH + nodeId := r.Intn(int(g.NodeCount)) + 1 + country := countries[r.Intn(len(countries))] + query += fmt.Sprintf("UPDATE node SET country = '%s' WHERE node_id = %d;", country, nodeId) + } + + _, err := g.db.Exec(query) + fmt.Println("node country updated") + return err +} + +func (g *Crafter) UpdateNodeTotalResources() error { + updatesCount := 10 + scaling := 1 * 1024 * 1024 * 1024 + query := "" + for i := 0; i < updatesCount; i++ { + // WATCH + nodeId := r.Intn(int(g.NodeCount)) + 1 + + cru := 10 + hru := g.nodesHRU[uint64(nodeId)] + uint64(scaling) + mru := g.nodesMRU[uint64(nodeId)] + uint64(scaling) + sru := g.nodesSRU[uint64(nodeId)] + uint64(scaling) + + query += fmt.Sprintf("UPDATE node_resources_total SET cru = %d, hru = %d, mru = %d, sru = %d WHERE node_id = 'node-%d';", cru, hru, mru, sru, nodeId) + } + + _, err := g.db.Exec(query) + fmt.Println("node total resources updated") + return err +} + +func (g *Crafter) UpdateContractResources() error { + updatesCount := 10 + query := "" + for i := 0; i < updatesCount; i++ { + // WATCH + contractId := r.Intn(int(g.NodeContractCount)) + 1 + + cru := minContractCRU + hru := minContractHRU + sru := minContractSRU + mru := minContractMRU + + query += fmt.Sprintf("UPDATE contract_resources SET cru = %d, hru = %d, mru = %d, sru = %d WHERE contract_id = 'node-contract-%d';", cru, hru, mru, sru, contractId) + } + + _, err := g.db.Exec(query) + fmt.Println("contract resources updated") + return err +} + +func (g *Crafter) UpdateNodeContractState() error { + updatesCount := 10 + query := "" + states := []string{"Deleted", "GracePeriod"} + + for i := 0; i < updatesCount; i++ { + contractId := g.createdNodeContracts[r.Intn(len(g.createdNodeContracts))] + state := states[r.Intn(2)] + query += fmt.Sprintf("UPDATE node_contract SET state = '%s' WHERE contract_id = %d AND state != 'Deleted';", state, contractId) + } + + _, err := g.db.Exec(query) + fmt.Println("node contract state updated") + return err +} + +func (g *Crafter) UpdateRentContract() error { + updatesCount := 10 + query := "" + states := []string{"Deleted", "GracePeriod"} + + for i := 0; i < updatesCount; i++ { + // WATCH + contractId := r.Intn(int(g.RentContractCount)) + 1 + state := states[r.Intn(2)] + query += fmt.Sprintf("UPDATE rent_contract SET state = '%s' WHERE contract_id = %d;", state, contractId) + } + + _, err := g.db.Exec(query) + fmt.Println("rent contracts updated") + return err +} + +func (g *Crafter) UpdatePublicIps() error { + updatesCount := 10 + query := "" + + for i := 0; i < updatesCount; i++ { + idx := r.Intn(len(g.createdNodeContracts)) + contractID := g.createdNodeContracts[idx] + // WATCH + publicIPID := r.Intn(int(g.PublicIPCount)) + + query += fmt.Sprintf("UPDATE public_ip SET contract_id = (CASE WHEN contract_id = 0 THEN %d ELSE 0 END) WHERE id = 'public-ip-%d';", contractID, publicIPID) + query += fmt.Sprintf("UPDATE node_contract SET number_of_public_i_ps = (SELECT COUNT(id) FROM public_ip WHERE contract_id = %d) WHERE contract_id = %d;", contractID, contractID) + + } + + _, err := g.db.Exec(query) + fmt.Println("public ip contract_id update") + return err +} diff --git a/grid-proxy/tools/db/modifiers/types.go b/grid-proxy/tools/db/crafter/types.go similarity index 82% rename from grid-proxy/tools/db/modifiers/types.go rename to grid-proxy/tools/db/crafter/types.go index d62702bfb..0cdf0230e 100644 --- a/grid-proxy/tools/db/modifiers/types.go +++ b/grid-proxy/tools/db/crafter/types.go @@ -1,24 +1,14 @@ -package modifiers +package crafter import ( "database/sql" "math/rand" ) -var r *rand.Rand - const ( contractCreatedRatio = .1 // from devnet usedPublicIPsRatio = .9 nodeUpRatio = .5 - NodeCount = 6000 - FarmCount = 600 - NormalUsersCount = 6000 - PublicIPCount = 1000 - TwinCount = 6000 + 600 + 6000 // nodes + farms + normal users - NodeContractCount = 9000 - RentContractCount = 100 - NameContractCount = 300 maxContractHRU = 1024 * 1024 * 1024 * 300 maxContractSRU = 1024 * 1024 * 1024 * 300 maxContractMRU = 1024 * 1024 * 1024 * 16 @@ -30,6 +20,8 @@ const ( ) var ( + r *rand.Rand + countries = []string{"Belgium", "United States", "Egypt", "United Kingdom"} regions = map[string]string{ "Belgium": "Europe", @@ -51,8 +43,9 @@ var ( } ) -type Generator struct { - db *sql.DB +type Crafter struct { + db *sql.DB + nodesMRU map[uint64]uint64 nodesSRU map[uint64]uint64 nodesHRU map[uint64]uint64 @@ -62,13 +55,44 @@ type Generator struct { availableRentNodes map[uint64]struct{} availableRentNodesList []uint64 renter map[uint64]uint64 + + NodeCount uint + FarmCount uint + PublicIPCount uint + TwinCount uint + NodeContractCount uint + RentContractCount uint + NameContractCount uint + + NodeStart uint + FarmStart uint + TwinStart uint + ContractStart uint + BillStart uint + PublicIPStart uint } -func NewGenerator(db *sql.DB, seed int) Generator { +func NewCrafter(db *sql.DB, + seed int, + nodeCount, + farmCount, + twinCount, + ipCount, + nodeContractCount, + nameContractCount, + rentContractCount, + nodeStart, + farmStart, + twinStart, + contractStart, + billStart, + publicIPStart uint) Crafter { + r = rand.New(rand.NewSource(int64(seed))) - return Generator{ - db: db, + return Crafter{ + db: db, + nodesMRU: make(map[uint64]uint64), nodesSRU: make(map[uint64]uint64), nodesHRU: make(map[uint64]uint64), @@ -78,6 +102,21 @@ func NewGenerator(db *sql.DB, seed int) Generator { availableRentNodes: make(map[uint64]struct{}), availableRentNodesList: make([]uint64, 0), renter: make(map[uint64]uint64), + + TwinCount: twinCount, + FarmCount: farmCount, + NodeCount: nodeCount, + PublicIPCount: ipCount, + NodeContractCount: nodeContractCount, + RentContractCount: rentContractCount, + NameContractCount: nameContractCount, + + NodeStart: nodeStart, + FarmStart: farmStart, + TwinStart: twinStart, + ContractStart: contractStart, + BillStart: billStart, + PublicIPStart: publicIPStart, } } diff --git a/grid-proxy/tools/db/modifiers/utils.go b/grid-proxy/tools/db/crafter/utils.go similarity index 93% rename from grid-proxy/tools/db/modifiers/utils.go rename to grid-proxy/tools/db/crafter/utils.go index efc978d07..e08b9d228 100644 --- a/grid-proxy/tools/db/modifiers/utils.go +++ b/grid-proxy/tools/db/crafter/utils.go @@ -1,4 +1,4 @@ -package modifiers +package crafter import ( "encoding/json" @@ -92,7 +92,8 @@ func objectToTupleString(v interface{}) (string, error) { if err != nil { return "", fmt.Errorf("failed to marshal the power map to JSON: %w", err) } - v = fmt.Sprintf("'%s'", string(powerJSON)) + escapedJSON := strings.ReplaceAll(string(powerJSON), "'", "''") + v = fmt.Sprintf("'%s'", escapedJSON) } vals = fmt.Sprintf("%s, %s", vals, v) } @@ -100,7 +101,7 @@ func objectToTupleString(v interface{}) (string, error) { return fmt.Sprintf("%s)", vals), nil } -func (g *Generator) insertTuples(tupleObj interface{}, tuples []string) error { +func (g *Crafter) insertTuples(tupleObj interface{}, tuples []string) error { if len(tuples) != 0 { query := "INSERT INTO " + reflect.Indirect(reflect.ValueOf(tupleObj)).Type().Name() + " (" diff --git a/grid-proxy/tools/db/modifiers/utils_test.go b/grid-proxy/tools/db/crafter/utils_test.go similarity index 98% rename from grid-proxy/tools/db/modifiers/utils_test.go rename to grid-proxy/tools/db/crafter/utils_test.go index 058729b5c..23ab67eb6 100644 --- a/grid-proxy/tools/db/modifiers/utils_test.go +++ b/grid-proxy/tools/db/crafter/utils_test.go @@ -1,4 +1,4 @@ -package modifiers +package crafter import ( "fmt" diff --git a/grid-proxy/tools/db/db.go b/grid-proxy/tools/db/db.go index d96fcb9f6..f688c1cbb 100644 --- a/grid-proxy/tools/db/db.go +++ b/grid-proxy/tools/db/db.go @@ -4,7 +4,6 @@ import ( "database/sql" "flag" "fmt" - "math/rand" // used by the orm @@ -12,10 +11,6 @@ import ( "github.com/pkg/errors" ) -var ( - r *rand.Rand -) - type flags struct { postgresHost string postgresPort int @@ -42,64 +37,31 @@ func parseCmdline() flags { func main() { f := parseCmdline() - psqlInfo := fmt.Sprintf("host=%s port=%d user=%s "+ - "password=%s dbname=%s sslmode=disable", + psqlInfo := fmt.Sprintf("host=%s port=%d user=%s password=%s dbname=%s sslmode=disable", f.postgresHost, f.postgresPort, f.postgresUser, f.postgresPassword, f.postgresDB) db, err := sql.Open("postgres", psqlInfo) if err != nil { panic(errors.Wrap(err, "failed to open db")) } defer db.Close() + if f.reset { - if _, err := db.Exec( - ` - DROP TABLE IF EXISTS account CASCADE; - DROP TABLE IF EXISTS burn_transaction CASCADE; - DROP TABLE IF EXISTS city CASCADE; - DROP TABLE IF EXISTS contract_bill_report CASCADE; - DROP TABLE IF EXISTS contract_resources CASCADE; - DROP TABLE IF EXISTS country CASCADE; - DROP TABLE IF EXISTS entity CASCADE; - DROP TABLE IF EXISTS entity_proof CASCADE; - DROP TABLE IF EXISTS farm CASCADE; - DROP TABLE IF EXISTS farming_policy CASCADE; - DROP TABLE IF EXISTS historical_balance CASCADE; - DROP TABLE IF EXISTS interfaces CASCADE; - DROP TABLE IF EXISTS location CASCADE; - DROP TABLE IF EXISTS migrations CASCADE; - DROP TABLE IF EXISTS mint_transaction CASCADE; - DROP TABLE IF EXISTS name_contract CASCADE; - DROP TABLE IF EXISTS node CASCADE; - DROP TABLE IF EXISTS node_contract CASCADE; - DROP TABLE IF EXISTS node_resources_free CASCADE; - DROP TABLE IF EXISTS node_resources_total CASCADE; - DROP TABLE IF EXISTS node_resources_used CASCADE; - DROP TABLE IF EXISTS nru_consumption CASCADE; - DROP TABLE IF EXISTS pricing_policy CASCADE; - DROP TABLE IF EXISTS public_config CASCADE; - DROP TABLE IF EXISTS public_ip CASCADE; - DROP TABLE IF EXISTS refund_transaction CASCADE; - DROP TABLE IF EXISTS rent_contract CASCADE; - DROP TABLE IF EXISTS transfer CASCADE; - DROP TABLE IF EXISTS twin CASCADE; - DROP TABLE IF EXISTS typeorm_metadata CASCADE; - DROP TABLE IF EXISTS uptime_event CASCADE; - DROP SCHEMA IF EXISTS substrate_threefold_status CASCADE; - DROP TABLE IF EXISTS node_gpu CASCADE; - - `); err != nil { + if err := reset(db); err != nil { panic(err) } } + if err := initSchema(db); err != nil { panic(err) } + // it looks like a useless block but everything breaks when it's removed _, err = db.Query("SELECT current_database();") if err != nil { panic(err) } // ---- + if err := generateData(db, f.seed); err != nil { panic(err) } diff --git a/grid-proxy/tools/db/docker.sh b/grid-proxy/tools/db/docker.sh deleted file mode 100644 index 05a7907cf..000000000 --- a/grid-proxy/tools/db/docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/bash - diff --git a/grid-proxy/tools/db/generate.go b/grid-proxy/tools/db/generate.go index 5d87ebcd3..bea47f153 100644 --- a/grid-proxy/tools/db/generate.go +++ b/grid-proxy/tools/db/generate.go @@ -5,9 +5,60 @@ import ( "fmt" "os" - "github.com/threefoldtech/tfgrid-sdk-go/grid-proxy/tools/db/modifiers" + "github.com/threefoldtech/tfgrid-sdk-go/grid-proxy/tools/db/crafter" ) +const ( + NodeCount = 6000 + FarmCount = 600 + TwinCount = 6000 + 600 + 6000 // nodes + farms + normal users + PublicIPCount = 1000 + NodeContractCount = 9000 + RentContractCount = 100 + NameContractCount = 300 +) + +func reset(db *sql.DB) error { + _, err := db.Exec( + ` + DROP TABLE IF EXISTS account CASCADE; + DROP TABLE IF EXISTS burn_transaction CASCADE; + DROP TABLE IF EXISTS city CASCADE; + DROP TABLE IF EXISTS contract_bill_report CASCADE; + DROP TABLE IF EXISTS contract_resources CASCADE; + DROP TABLE IF EXISTS country CASCADE; + DROP TABLE IF EXISTS entity CASCADE; + DROP TABLE IF EXISTS entity_proof CASCADE; + DROP TABLE IF EXISTS farm CASCADE; + DROP TABLE IF EXISTS farming_policy CASCADE; + DROP TABLE IF EXISTS historical_balance CASCADE; + DROP TABLE IF EXISTS interfaces CASCADE; + DROP TABLE IF EXISTS location CASCADE; + DROP TABLE IF EXISTS migrations CASCADE; + DROP TABLE IF EXISTS mint_transaction CASCADE; + DROP TABLE IF EXISTS name_contract CASCADE; + DROP TABLE IF EXISTS node CASCADE; + DROP TABLE IF EXISTS node_contract CASCADE; + DROP TABLE IF EXISTS node_resources_free CASCADE; + DROP TABLE IF EXISTS node_resources_total CASCADE; + DROP TABLE IF EXISTS node_resources_used CASCADE; + DROP TABLE IF EXISTS nru_consumption CASCADE; + DROP TABLE IF EXISTS pricing_policy CASCADE; + DROP TABLE IF EXISTS public_config CASCADE; + DROP TABLE IF EXISTS public_ip CASCADE; + DROP TABLE IF EXISTS refund_transaction CASCADE; + DROP TABLE IF EXISTS rent_contract CASCADE; + DROP TABLE IF EXISTS transfer CASCADE; + DROP TABLE IF EXISTS twin CASCADE; + DROP TABLE IF EXISTS typeorm_metadata CASCADE; + DROP TABLE IF EXISTS uptime_event CASCADE; + DROP SCHEMA IF EXISTS substrate_threefold_status CASCADE; + DROP TABLE IF EXISTS node_gpu CASCADE; + + `) + return err +} + func initSchema(db *sql.DB) error { schema, err := os.ReadFile("./schema.sql") if err != nil { @@ -21,25 +72,39 @@ func initSchema(db *sql.DB) error { } func generateData(db *sql.DB, seed int) error { - generator := modifiers.NewGenerator(db, seed) + generator := crafter.NewCrafter(db, + seed, + NodeCount, + FarmCount, + TwinCount, + PublicIPCount, + NodeContractCount, + NameContractCount, + RentContractCount, + 1, + 1, + 1, + 1, + 1, + 1) - if err := generator.GenerateTwins(1, modifiers.TwinCount); err != nil { + if err := generator.GenerateTwins(); err != nil { return fmt.Errorf("failed to generate twins: %w", err) } - if err := generator.GenerateFarms(1, modifiers.FarmCount, 1); err != nil { + if err := generator.GenerateFarms(); err != nil { return fmt.Errorf("failed to generate farms: %w", err) } - if err := generator.GenerateNodes(1, modifiers.NodeCount, 1, modifiers.FarmCount, 1); err != nil { + if err := generator.GenerateNodes(); err != nil { return fmt.Errorf("failed to generate nodes: %w", err) } - if err := generator.GenerateContracts(1, 1, modifiers.NodeContractCount, modifiers.NameContractCount, modifiers.RentContractCount, 1, modifiers.NodeCount); err != nil { + if err := generator.GenerateContracts(); err != nil { return fmt.Errorf("failed to generate contracts: %w", err) } - if err := generator.GeneratePublicIPs(1, modifiers.PublicIPCount, 1, modifiers.FarmCount); err != nil { + if err := generator.GeneratePublicIPs(); err != nil { return fmt.Errorf("failed to generate public ips: %w", err) } diff --git a/grid-proxy/tools/db/modifiers/modifiers.go b/grid-proxy/tools/db/modifiers/modifiers.go deleted file mode 100644 index 549a3db7b..000000000 --- a/grid-proxy/tools/db/modifiers/modifiers.go +++ /dev/null @@ -1,158 +0,0 @@ -package modifiers - -import ( - "fmt" - - "github.com/rs/zerolog/log" -) - -func (g *Generator) UpdateNodeCountry() error { - updatesCount := 10 - query := "" - - for i := 0; i < updatesCount; i++ { - nodeId := r.Intn(int(NodeCount)) + 1 - country := countries[r.Intn(len(countries))] - query += fmt.Sprintf("UPDATE node SET country = '%s' WHERE node_id = %d;", country, nodeId) - } - - log.Debug().Str("query", query).Msg("update node country") - - _, err := g.db.Exec(query) - return err -} - -func (g *Generator) UpdateNodeTotalResources() error { - updatesCount := 10 - padding := 1 * 1024 * 1024 * 1024 - query := "" - for i := 0; i < updatesCount; i++ { - nodeId := r.Intn(int(NodeCount)) + 1 - - cru := 10 - hru := g.nodesHRU[uint64(nodeId)] + uint64(padding) - mru := g.nodesMRU[uint64(nodeId)] + uint64(padding) - sru := g.nodesSRU[uint64(nodeId)] + uint64(padding) - - query += fmt.Sprintf("UPDATE node_resources_total SET cru = %d, hru = %d, mru = %d, sru = %d WHERE node_id = 'node-%d';", cru, hru, mru, sru, nodeId) - } - - log.Debug().Str("query", query).Msg("update node total resources") - - _, err := g.db.Exec(query) - return err -} - -func (g *Generator) UpdateContractResources() error { - updatesCount := 10 - query := "" - for i := 0; i < updatesCount; i++ { - contractId := r.Intn(int(NodeContractCount)) + 1 - - cru := minContractCRU - hru := minContractHRU - sru := minContractSRU - mru := minContractMRU - - query += fmt.Sprintf("UPDATE contract_resources SET cru = %d, hru = %d, mru = %d, sru = %d WHERE contract_id = 'node-contract-%d';", cru, hru, mru, sru, contractId) - } - - log.Debug().Str("query", query).Msg("update contract resources") - - _, err := g.db.Exec(query) - return err -} - -func (g *Generator) UpdateNodeContractState() error { - updatesCount := 10 - query := "" - states := []string{"Deleted", "GracePeriod"} - - for i := 0; i < updatesCount; i++ { - contractId := g.createdNodeContracts[r.Intn(len(g.createdNodeContracts))] - state := states[r.Intn(2)] - query += fmt.Sprintf("UPDATE node_contract SET state = '%s' WHERE contract_id = %d AND state != 'Deleted';", state, contractId) - } - - log.Debug().Str("query", query).Msg("update node contract state") - - _, err := g.db.Exec(query) - return err -} - -func (g *Generator) UpdateRentContract() error { - updatesCount := 10 - query := "" - states := []string{"Deleted", "GracePeriod"} - - for i := 0; i < updatesCount; i++ { - contractId := r.Intn(int(RentContractCount)) + 1 - state := states[r.Intn(2)] - query += fmt.Sprintf("UPDATE rent_contract SET state = '%s' WHERE contract_id = %d;", state, contractId) - } - - log.Debug().Str("query", query).Msg("update rent contracts") - - _, err := g.db.Exec(query) - return err -} - -func (g *Generator) UpdatePublicIps() error { - updatesCount := 10 - query := "" - - for i := 0; i < updatesCount; i++ { - idx := r.Intn(len(g.createdNodeContracts)) - contractID := g.createdNodeContracts[idx] - publicIPID := r.Intn(int(PublicIPCount)) - - query += fmt.Sprintf("UPDATE public_ip SET contract_id = (CASE WHEN contract_id = 0 THEN %d ELSE 0 END) WHERE id = 'public-ip-%d';", contractID, publicIPID) - query += fmt.Sprintf("UPDATE node_contract SET number_of_public_i_ps = (SELECT COUNT(id) FROM public_ip WHERE contract_id = %d) WHERE contract_id = %d;", contractID, contractID) - - } - - log.Debug().Str("query", query).Msg("update public ip contract_id") - - _, err := g.db.Exec(query) - return err -} - -// deletions -func (g *Generator) DeleteNodes(nodesCount int) error { - // delete node contracts on this node - // free public ips that are assigned to the deleted contracts - // delete rent contracts on this node - // delete node - deleteCount := r.Intn(10) + 1 - query := "" - - for i := 0; i < deleteCount; i++ { - nodeID := nodesCount - i - - query += fmt.Sprintf("UPDATE public_ip SET contract_id = 0 WHERE contract_id IN (SELECT contract_id FROM node_contract WHERE node_id = %d);", nodeID) - query += fmt.Sprintf("UPDATE node_contract SET state = 'Deleted' WHERE node_id = %d;", nodeID) - query += fmt.Sprintf("UPDATE rent_contract set state = 'Deleted' WHERE node_id = %d;", nodeID) - query += fmt.Sprintf("DELETE FROM node_resources_total WHERE node_id = (SELECT id FROM node WHERE node_id = %d);", nodeID) - query += fmt.Sprintf("DELETE FROM public_config WHERE node_id = (SELECT id FROM node WHERE node_id = %d);", nodeID) - query += fmt.Sprintf("DELETE FROM node WHERE node_id = %d;", nodeID) - } - - log.Debug().Str("query", query).Msg("delete nodes") - - _, err := g.db.Exec(query) - return err -} - -func (g *Generator) DeletePublicIps() error { - maxDeleteCount := r.Intn(10) + 1 - query := fmt.Sprintf("DELETE FROM public_ip WHERE id in (SELECT id FROM public_ip WHERE contract_id = 0 LIMIT %d);", maxDeleteCount) - - _, err := g.db.Exec(query) - if err != nil { - return fmt.Errorf("failed to delete public ips: %w", err) - } - - log.Debug().Str("query", query).Msg("delete public ips") - - return nil -} diff --git a/grid-proxy/tools/db/schema.sql b/grid-proxy/tools/db/schema.sql index 816bf31fe..1ca4fc54e 100644 --- a/grid-proxy/tools/db/schema.sql +++ b/grid-proxy/tools/db/schema.sql @@ -869,7 +869,7 @@ ALTER TABLE ONLY public.node_resources_total -- ALTER TABLE ONLY public.node_gpu - ADD CONSTRAINT node_gpu_pkey PRIMARY KEY (node_twin_id, id); + ADD CONSTRAINT node_gpu_pkey PRIMARY KEY (id); diff --git a/grid-proxy/tools/db/test.go b/grid-proxy/tools/db/test.go deleted file mode 100644 index 0a296ff41..000000000 --- a/grid-proxy/tools/db/test.go +++ /dev/null @@ -1,79 +0,0 @@ -package main - -// func unusedNecessaryColumns() string { -// return ` -// INSERT INTO location VALUES ('1', '2', '3'); -// ` -// } -// func cleanup() string { -// return ` -// DELETE FROM node_resources_total WHERE id = '1'; -// DELETE FROM node WHERE id = '112'; -// DELETE FROM location WHERE id = '1'; -// DELETE FROM farm WHERE id = '112'; -// UPDATE node_contract SET resources_used_id = NULL WHERE id = '112'; -// DELETE FROM contract_resources WHERE id = '112'; -// DELETE FROM node_contract WHERE id = '112'; -// ` -// } -// func test(db *sql.DB) error { -// node := node{ -// id: "112", -// location_id: "1", -// node_id: 112, -// } -// total_resources := node_resources_total{ -// id: "1", -// hru: 1, -// sru: 2, -// cru: 3, -// mru: 5, -// node_id: "112", -// } -// farm := farm{ -// id: "112", -// farm_id: 112, -// name: "345", -// certification_type: "123", -// } -// contract_resources := contract_resources{ -// id: "112", -// hru: 1, -// sru: 2, -// cru: 3, -// mru: 4, -// contract_id: "112", -// } -// node_contract := node_contract{ -// id: "112", -// contract_id: 112, -// twin_id: 112, -// node_id: 112, -// resources_used_id: "", -// deployment_data: "123", -// deployment_hash: "123", -// state: "Created", -// } -// if _, err := db.Exec(unusedNecessaryColumns()); err != nil { -// return err -// } -// if _, err := db.Exec(insertQuery(&node)); err != nil { -// return err -// } -// if _, err := db.Exec(insertQuery(&total_resources)); err != nil { -// return err -// } -// if _, err := db.Exec(insertQuery(&farm)); err != nil { -// return err -// } -// if _, err := db.Exec(insertQuery(&node_contract)); err != nil { -// return err -// } -// if _, err := db.Exec(insertQuery(&contract_resources)); err != nil { -// return err -// } -// if _, err := db.Exec(setContractResource("112", "112")); err != nil { -// return err -// } -// return nil -// } diff --git a/grid-proxy/tools/db/types.go b/grid-proxy/tools/db/types.go deleted file mode 100644 index 6db8fb9c7..000000000 --- a/grid-proxy/tools/db/types.go +++ /dev/null @@ -1,145 +0,0 @@ -package main - -// type contract_resources struct { -// id string -// hru uint64 -// sru uint64 -// cru uint64 -// mru uint64 -// contract_id string -// } -// type farm struct { -// id string -// grid_version uint64 -// farm_id uint64 -// name string -// twin_id uint64 -// pricing_policy_id uint64 -// certification string -// stellar_address string -// dedicated_farm bool -// } - -// type node struct { -// id string -// grid_version uint64 -// node_id uint64 -// farm_id uint64 -// twin_id uint64 -// country string -// city string -// uptime uint64 -// created uint64 -// farming_policy_id uint64 -// certification string -// secure bool -// virtualized bool -// serial_number string -// created_at uint64 -// updated_at uint64 -// location_id string -// power nodePower `gorm:"type:jsonb"` -// extra_fee uint64 -// } - -// type nodePower struct { -// State string `json:"state"` -// Target string `json:"target"` -// } -// type twin struct { -// id string -// grid_version uint64 -// twin_id uint64 -// account_id string -// relay string -// public_key string -// } - -// type public_ip struct { -// id string -// gateway string -// ip string -// contract_id uint64 -// farm_id string -// } -// type node_contract struct { -// id string -// grid_version uint64 -// contract_id uint64 -// twin_id uint64 -// node_id uint64 -// deployment_data string -// deployment_hash string -// number_of_public_i_ps uint64 -// state string -// created_at uint64 -// resources_used_id string -// } -// type node_resources_total struct { -// id string -// hru uint64 -// sru uint64 -// cru uint64 -// mru uint64 -// node_id string -// } -// type public_config struct { -// id string -// ipv4 string -// ipv6 string -// gw4 string -// gw6 string -// domain string -// node_id string -// } -// type rent_contract struct { -// id string -// grid_version uint64 -// contract_id uint64 -// twin_id uint64 -// node_id uint64 -// state string -// created_at uint64 -// } -// type location struct { -// id string -// longitude string -// latitude string -// } - -// type contract_bill_report struct { -// id string -// contract_id uint64 -// discount_received string -// amount_billed uint64 -// timestamp uint64 -// } - -// type name_contract struct { -// id string -// grid_version uint64 -// contract_id uint64 -// twin_id uint64 -// name string -// state string -// created_at uint64 -// } - -// type node_gpu struct { -// node_twin_id uint64 -// id string -// vendor string -// device string -// contract int -// } - -// type country struct { -// id string -// country_id uint64 -// code string -// name string -// region string -// subregion string -// lat string -// long string -// } diff --git a/grid-proxy/tools/db/utils.go b/grid-proxy/tools/db/utils.go deleted file mode 100644 index 646a66fe6..000000000 --- a/grid-proxy/tools/db/utils.go +++ /dev/null @@ -1,112 +0,0 @@ -package main - -// import ( -// "encoding/json" -// "fmt" -// "net" -// "reflect" -// "strings" -// ) - -// const null = "NULL" - -// // rnd gets a random number between min and max -// func rnd(min, max uint64) (uint64, error) { -// if max-min+1 <= 0 { -// return 0, fmt.Errorf("min (%d) cannot be greater than max (%d)", min, max) -// } -// randomNumber := r.Uint64()%(max-min+1) + min -// return randomNumber, nil -// } - -// // flip simulates a coin flip with a given success probability. -// func flip(success float32) bool { -// return r.Float32() < success -// } - -// // randomIPv4 gets a random IPv4 -// func randomIPv4() net.IP { -// ip := make([]byte, 4) -// r.Read(ip) -// return net.IP(ip) -// } - -// // IPv4Subnet gets the ipv4 subnet given the ip -// func IPv4Subnet(ip net.IP) *net.IPNet { -// return &net.IPNet{ -// IP: ip, -// Mask: net.CIDRMask(24, 32), -// } -// } - -// // min gets min between 2 numbers -// func min(a, b uint64) uint64 { -// if a < b { -// return a -// } -// return b -// } - -// // max gets max between 2 numbers -// func max(a, b uint64) uint64 { -// if a > b { -// return a -// } -// return b -// } - -// // objectToTupleString converts a object into a string representation suitable for sql query -// func objectToTupleString(v interface{}) (string, error) { -// vals := "(" -// val := reflect.ValueOf(v) -// for i := 0; i < val.NumField(); i++ { -// if i == 0 { -// v := fmt.Sprint(val.Field(i)) -// if v == "" { -// v = null -// } -// if v != null && val.Field(i).Type().Name() == "string" { -// v = fmt.Sprintf(`'%s'`, v) -// } -// vals = fmt.Sprintf("%s%s", vals, v) -// } else { -// v := fmt.Sprint(val.Field(i)) -// if v == "" { -// v = null -// } -// if v != null && val.Field(i).Type().Name() == "string" { -// v = fmt.Sprintf(`'%s'`, v) -// } -// if v != null && val.Field(i).Type().Name() == "nodePower" { -// // Construct the nodePower object -// val2 := val.Field(i) -// power := make(map[string]string) -// for j := 0; j < val2.NumField(); j++ { -// fieldName := strings.ToLower(val2.Type().Field(j).Name) -// fieldValue := val2.Field(j).String() -// power[fieldName] = fieldValue -// } - -// // Marshal the power map to JSON and wrap it in quotes -// powerJSON, err := json.Marshal(power) -// if err != nil { -// return "", fmt.Errorf("failed to marshal the power map to JSON: %w", err) -// } -// v = fmt.Sprintf("'%s'", string(powerJSON)) -// } -// vals = fmt.Sprintf("%s, %s", vals, v) -// } -// } -// return fmt.Sprintf("%s)", vals), nil -// } - -// // popRandom selects a random element from the given slice, -// func popRandom(l []uint64) ([]uint64, uint64, error) { -// idx, err := rnd(0, uint64(len(l)-1)) -// if err != nil { -// return nil, 0, err -// } -// e := l[idx] -// l[idx], l[len(l)-1] = l[len(l)-1], l[idx] -// return l[:len(l)-1], e, nil -// } From a7177fff7f83d4703e43eed3843585f4950deb8e Mon Sep 17 00:00:00 2001 From: Omar Abdulaziz Date: Wed, 27 Dec 2023 15:11:51 +0200 Subject: [PATCH 35/39] remove unnecessary check Co-authored-by: Mario Wassef --- grid-proxy/tests/queries/mock_client/farms.go | 2 +- grid-proxy/tests/queries/mock_client/nodes.go | 2 +- grid-proxy/tests/queries/mock_client/utils.go | 23 ++++-------------- grid-proxy/tests/queries/node_test.go | 2 +- grid-proxy/tests/queries/utils.go | 24 ------------------- 5 files changed, 7 insertions(+), 46 deletions(-) diff --git a/grid-proxy/tests/queries/mock_client/farms.go b/grid-proxy/tests/queries/mock_client/farms.go index f61e07265..5409c5a2c 100644 --- a/grid-proxy/tests/queries/mock_client/farms.go +++ b/grid-proxy/tests/queries/mock_client/farms.go @@ -131,7 +131,7 @@ func (f *Farm) satisfyFarmNodesFilter(data *DBData, filter types.FarmFilter) boo total := data.NodeTotalResources[node.NodeID] used := data.NodeUsedResources[node.NodeID] - free := calcFreeResources(total, used) + free := CalcFreeResources(total, used) if filter.NodeFreeHRU != nil && int64(free.HRU) < int64(*filter.NodeFreeHRU) { continue } diff --git a/grid-proxy/tests/queries/mock_client/nodes.go b/grid-proxy/tests/queries/mock_client/nodes.go index 9b0b10ba6..c7d064333 100644 --- a/grid-proxy/tests/queries/mock_client/nodes.go +++ b/grid-proxy/tests/queries/mock_client/nodes.go @@ -197,7 +197,7 @@ func (n *Node) satisfies(f types.NodeFilter, data *DBData) bool { total := data.NodeTotalResources[n.NodeID] used := data.NodeUsedResources[n.NodeID] - free := calcFreeResources(total, used) + free := CalcFreeResources(total, used) if f.Status != nil && *f.Status != nodestatus.DecideNodeStatus(nodePower, int64(n.UpdatedAt)) { return false diff --git a/grid-proxy/tests/queries/mock_client/utils.go b/grid-proxy/tests/queries/mock_client/utils.go index 1a4b7352f..411f1a367 100644 --- a/grid-proxy/tests/queries/mock_client/utils.go +++ b/grid-proxy/tests/queries/mock_client/utils.go @@ -10,26 +10,11 @@ type Result interface { types.Contract | types.Farm | types.Node | types.Twin } -func calcFreeResources(total NodeResourcesTotal, used NodeResourcesTotal) NodeResourcesTotal { - mru := total.MRU - used.MRU - if mru < 0 { - mru = 0 - } - - hru := total.HRU - used.HRU - if hru < 0 { - hru = 0 - } - - sru := total.SRU - used.SRU - if sru < 0 { - sru = 0 - } - +func CalcFreeResources(total NodeResourcesTotal, used NodeResourcesTotal) NodeResourcesTotal { return NodeResourcesTotal{ - HRU: hru, - SRU: sru, - MRU: mru, + HRU: total.HRU - used.HRU, + SRU: total.SRU - used.SRU, + MRU: total.MRU - used.MRU, } } diff --git a/grid-proxy/tests/queries/node_test.go b/grid-proxy/tests/queries/node_test.go index a4423c69e..11084a467 100644 --- a/grid-proxy/tests/queries/node_test.go +++ b/grid-proxy/tests/queries/node_test.go @@ -590,7 +590,7 @@ func calcNodesAggregates(data *mock.DBData) (res NodesAggregate) { cities[node.City] = struct{}{} countries[node.Country] = struct{}{} total := data.NodeTotalResources[node.NodeID] - free := calcFreeResources(total, data.NodeUsedResources[node.NodeID]) + free := mock.CalcFreeResources(total, data.NodeUsedResources[node.NodeID]) freeHRU := free.HRU if int64(freeHRU) < 0 { freeHRU = 0 diff --git a/grid-proxy/tests/queries/utils.go b/grid-proxy/tests/queries/utils.go index aa1e06fec..423d3079e 100644 --- a/grid-proxy/tests/queries/utils.go +++ b/grid-proxy/tests/queries/utils.go @@ -7,36 +7,12 @@ import ( "unicode" "github.com/threefoldtech/tfgrid-sdk-go/grid-proxy/pkg/types" - mock "github.com/threefoldtech/tfgrid-sdk-go/grid-proxy/tests/queries/mock_client" ) type Filter interface { types.ContractFilter | types.NodeFilter | types.FarmFilter | types.TwinFilter | types.StatsFilter } -func calcFreeResources(total mock.NodeResourcesTotal, used mock.NodeResourcesTotal) mock.NodeResourcesTotal { - mru := total.MRU - used.MRU - if mru < 0 { - mru = 0 - } - - hru := total.HRU - used.HRU - if hru < 0 { - hru = 0 - } - - sru := total.SRU - used.SRU - if sru < 0 { - sru = 0 - } - - return mock.NodeResourcesTotal{ - HRU: hru, - SRU: sru, - MRU: mru, - } -} - func flip(success float32) bool { return rand.Float32() < success } From 51de63e7f85d5a456795b27b9ce22f8f53dd4d23 Mon Sep 17 00:00:00 2001 From: Omar Abdulaziz Date: Wed, 27 Dec 2023 15:31:32 +0200 Subject: [PATCH 36/39] reformat readme file --- grid-proxy/tools/db/README.md | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/grid-proxy/tools/db/README.md b/grid-proxy/tools/db/README.md index a073f41d8..8fd557f03 100644 --- a/grid-proxy/tools/db/README.md +++ b/grid-proxy/tools/db/README.md @@ -1,22 +1,22 @@ ## Data Preparation for Testing -- Starting the postgres container +Starting the postgres container - ```bash - make db-start - ``` +```bash +make db-start +``` -- Populate the database using one of the two available methods: +Populate the database using one of the two available methods: +Generate a mock database - - Generate a mock database +- `./schema.sql` is a database schema taken from graphql processor on dev net, it creates all the needed tables for the database. +- the `data-crafter` have a various of methods to create/update/delete the main tables on the database. it can be customized by the params passed to the `NewCrafter`. + ```bash + make db-fill + ``` - - `./schema.sql` is a database schema taken from graphql processor on dev net, it creates all the needed tables for the database. - - the `data-crafter` have a various of methods to create/update/delete the main tables on the database. it can be customized by the params passed to the `NewCrafter`. - ```bash - make db-fill - ``` +Or use a data dump file (contains both schema and data) - - Use a data dump file (contains both schema and data) - ```bash - make db-dump p= - ``` +```bash +make db-dump p= +``` From b6a59a68ad361b7a3157dbfa75673a1e15daae3a Mon Sep 17 00:00:00 2001 From: Omar Abdulaziz Date: Wed, 27 Dec 2023 15:54:07 +0200 Subject: [PATCH 37/39] handles contract not found --- grid-proxy/tests/queries/contract_test.go | 12 ++++++++---- grid-proxy/tests/queries/mock_client/contracts.go | 5 +++++ grid-proxy/tools/db/README.md | 7 +++---- 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/grid-proxy/tests/queries/contract_test.go b/grid-proxy/tests/queries/contract_test.go index 65a1df254..546451f0e 100644 --- a/grid-proxy/tests/queries/contract_test.go +++ b/grid-proxy/tests/queries/contract_test.go @@ -147,11 +147,15 @@ func TestContract(t *testing.T) { contractID := rand.Intn(CONTRACTS_TESTS) - want, err := mockClient.Contract(context.Background(), uint32(contractID)) - require.NoError(t, err) + want, errGot := mockClient.Contract(context.Background(), uint32(contractID)) - got, err := gridProxyClient.Contract(context.Background(), uint32(contractID)) - require.NoError(t, err) + got, errWant := gridProxyClient.Contract(context.Background(), uint32(contractID)) + + if errGot != nil && errWant != nil { + require.True(t, errors.As(errWant, &errGot)) + } else { + require.True(t, errWant == errGot) + } require.True(t, reflect.DeepEqual(want, got), fmt.Sprintf("wanted: %+v\n got: %+v", want, got)) }) diff --git a/grid-proxy/tests/queries/mock_client/contracts.go b/grid-proxy/tests/queries/mock_client/contracts.go index 2cbd47fd5..27432230f 100644 --- a/grid-proxy/tests/queries/mock_client/contracts.go +++ b/grid-proxy/tests/queries/mock_client/contracts.go @@ -2,6 +2,7 @@ package mock import ( "context" + "fmt" "sort" "github.com/threefoldtech/tfgrid-sdk-go/grid-proxy/pkg/types" @@ -245,6 +246,10 @@ func (g *GridProxyMockClient) Contract(ctx context.Context, contractID uint32) ( }, err } + if !ok { + return res, fmt.Errorf("contract not found") + } + return res, err } diff --git a/grid-proxy/tools/db/README.md b/grid-proxy/tools/db/README.md index 8fd557f03..b77775949 100644 --- a/grid-proxy/tools/db/README.md +++ b/grid-proxy/tools/db/README.md @@ -7,10 +7,9 @@ make db-start ``` Populate the database using one of the two available methods: -Generate a mock database - -- `./schema.sql` is a database schema taken from graphql processor on dev net, it creates all the needed tables for the database. -- the `data-crafter` have a various of methods to create/update/delete the main tables on the database. it can be customized by the params passed to the `NewCrafter`. +Generate a mock database: + - `./schema.sql` is a database schema taken from graphql processor on dev net, it creates all the needed tables for the database. + - the `data-crafter` have a various of methods to create/update/delete the main tables on the database. it can be customized by the params passed to the `NewCrafter`. ```bash make db-fill ``` From e7087d7e8ec8892a4c3f1bf999ea0403647823b0 Mon Sep 17 00:00:00 2001 From: Omar Abdulaziz Date: Wed, 27 Dec 2023 15:59:37 +0200 Subject: [PATCH 38/39] reformat readme file --- grid-proxy/tools/db/README.md | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/grid-proxy/tools/db/README.md b/grid-proxy/tools/db/README.md index b77775949..efd206288 100644 --- a/grid-proxy/tools/db/README.md +++ b/grid-proxy/tools/db/README.md @@ -1,4 +1,4 @@ -## Data Preparation for Testing +# Data Preparation for Testing Starting the postgres container @@ -8,11 +8,13 @@ make db-start Populate the database using one of the two available methods: Generate a mock database: - - `./schema.sql` is a database schema taken from graphql processor on dev net, it creates all the needed tables for the database. - - the `data-crafter` have a various of methods to create/update/delete the main tables on the database. it can be customized by the params passed to the `NewCrafter`. - ```bash - make db-fill - ``` + +- `./schema.sql` is a database schema taken from graphql processor on dev net, it creates all the needed tables for the database. +- the `data-crafter` have a various of methods to create/update/delete the main tables on the database. it can be customized by the params passed to the `NewCrafter`. + +```bash +make db-fill +``` Or use a data dump file (contains both schema and data) From 7bd69c7bfb7c16008f43fa8f48b53c93ebb2a76d Mon Sep 17 00:00:00 2001 From: Omar Abdulaziz Date: Wed, 27 Dec 2023 16:07:21 +0200 Subject: [PATCH 39/39] fix markdown lint --- grid-proxy/tools/db/README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/grid-proxy/tools/db/README.md b/grid-proxy/tools/db/README.md index efd206288..570c686a1 100644 --- a/grid-proxy/tools/db/README.md +++ b/grid-proxy/tools/db/README.md @@ -9,8 +9,9 @@ make db-start Populate the database using one of the two available methods: Generate a mock database: -- `./schema.sql` is a database schema taken from graphql processor on dev net, it creates all the needed tables for the database. -- the `data-crafter` have a various of methods to create/update/delete the main tables on the database. it can be customized by the params passed to the `NewCrafter`. +`./schema.sql` is a database schema taken from graphql processor on dev net, it creates all the needed tables for the database. + +the `data-crafter` have a various of methods to create/update/delete the main tables on the database. it can be customized by the params passed to the `NewCrafter`. ```bash make db-fill