gridproxy: add standby node status (#126)

* add power filed to node struct & add standby node status

* fix linter

* fixing linter

* add new `power` field and `standby` status to the generated data. to fix tests

* fix linting

* hot fixes

* add column type to node struct

* fixes for codacy

grid-proxy/cmds/proxy_server/main.go

@@ -16,7 +16,7 @@ import (
 	"github.com/threefoldtech/tfgrid-sdk-go/grid-proxy/internal/explorer"
 	"github.com/threefoldtech/tfgrid-sdk-go/grid-proxy/internal/explorer/db"
 	logging "github.com/threefoldtech/tfgrid-sdk-go/grid-proxy/pkg"
-	"github.com/threefoldtech/tfgrid-sdk-go/rmb-sdk-go"
+	rmb "github.com/threefoldtech/tfgrid-sdk-go/rmb-sdk-go"
 	"github.com/threefoldtech/tfgrid-sdk-go/rmb-sdk-go/direct"
 )
 

grid-proxy/internal/explorer/converters.go

@@ -10,6 +10,22 @@ import (
 	"github.com/threefoldtech/zos/pkg/gridtypes"
 )
 
+const (
+	nodeUpInterval = -3 * time.Hour
+)
+
+func decideNodeStatus(power types.NodePower, updatedAt int64) string {
+	if power.Target == "Down" { // off or powering off
+		return "standby"
+	} else if power.Target == "Up" && power.State == "Down" { // powering on
+		return "down"
+	} else if updatedAt >= time.Now().Add(nodeUpInterval).Unix() {
+		return "up"
+	} else {
+		return "down"
+	}
+}
+
 func nodeFromDBNode(info db.Node) types.Node {
 	node := types.Node{
 		ID:              info.ID,
@@ -53,12 +69,9 @@ func nodeFromDBNode(info db.Node) types.Node {
 		RentContractID:    uint(info.RentContractID),
 		RentedByTwinID:    uint(info.RentedByTwinID),
 		SerialNumber:      info.SerialNumber,
+		Power:             types.NodePower(info.Power),
 	}
-	if node.UpdatedAt >= time.Now().Add(-3*time.Hour).Unix() {
-		node.Status = "up"
-	} else {
-		node.Status = "down"
-	}
+	node.Status = decideNodeStatus(node.Power, node.UpdatedAt)
 	return node
 }
 
@@ -125,12 +138,9 @@ func nodeWithNestedCapacityFromDBNode(info db.Node) types.NodeWithNestedCapacity
 		RentContractID:    uint(info.RentContractID),
 		RentedByTwinID:    uint(info.RentedByTwinID),
 		SerialNumber:      info.SerialNumber,
+		Power:             types.NodePower(info.Power),
 	}
-	if node.UpdatedAt >= time.Now().Add(-3*time.Hour).Unix() {
-		node.Status = "up"
-	} else {
-		node.Status = "down"
-	}
+	node.Status = decideNodeStatus(node.Power, node.UpdatedAt)
 	return node
 }
 

grid-proxy/internal/explorer/db/postgres.go

@@ -1,6 +1,7 @@
 package db
 
 import (
+	"encoding/json"
 	"fmt"
 	"math/rand"
 	"strings"
@@ -141,6 +142,23 @@ func (d *PostgresDatabase) initialize() error {
 	return res.Error
 }
 
+func decideNodeStatusCondition(status string) string {
+	condition := "TRUE"
+	nodeUpInterval := time.Now().Unix() - nodeStateFactor*int64(reportInterval.Seconds())
+
+	if status == "up" {
+		condition = fmt.Sprintf(`node.updated_at >= %d`, nodeUpInterval)
+	} else if status == "down" {
+		condition = fmt.Sprintf(`node.updated_at < %d 
+				OR node.updated_at IS NULL
+				OR node.power->> 'target' = 'Up' AND node.power->> 'state' = 'Down'`, nodeUpInterval)
+	} else if status == "standby" {
+		condition = `node.power->> 'target' = 'Down'`
+	}
+
+	return condition
+}
+
 // GetCounters returns aggregate info about the grid
 func (d *PostgresDatabase) GetCounters(filter types.StatsFilter) (types.Counters, error) {
 	var counters types.Counters
@@ -169,12 +187,7 @@ func (d *PostgresDatabase) GetCounters(filter types.StatsFilter) (types.Counters
 
 	condition := "TRUE"
 	if filter.Status != nil {
-		nodeUpInterval := time.Now().Unix() - nodeStateFactor*int64(reportInterval.Seconds())
-		if *filter.Status == "up" {
-			condition = fmt.Sprintf(`node.updated_at >= %d`, nodeUpInterval)
-		} else if *filter.Status == "down" {
-			condition = fmt.Sprintf(`node.updated_at < %d`, nodeUpInterval)
-		}
+		condition = decideNodeStatusCondition(*filter.Status)
 	}
 
 	if res := d.gormDB.
@@ -225,6 +238,17 @@ func (d *PostgresDatabase) GetCounters(filter types.StatsFilter) (types.Counters
 	return counters, nil
 }
 
+// Scan is a custom decoder for jsonb filed. executed while scanning the node.
+func (np *NodePower) Scan(value interface{}) error {
+	if value == nil {
+		return nil
+	}
+	if data, ok := value.([]byte); ok {
+		return json.Unmarshal(data, np)
+	}
+	return fmt.Errorf("failed to unmarshal NodePower")
+}
+
 // GetNode returns node info
 func (d *PostgresDatabase) GetNode(nodeID uint32) (Node, error) {
 	q := d.nodeTableQuery()
@@ -342,6 +366,7 @@ func (d *PostgresDatabase) nodeTableQuery() *gorm.DB {
 			"node.serial_number",
 			"convert_to_decimal(location.longitude) as longitude",
 			"convert_to_decimal(location.latitude) as latitude",
+			"node.power",
 		).
 		Joins(
 			"LEFT JOIN nodes_resources_view ON node.node_id = nodes_resources_view.node_id",
@@ -364,15 +389,14 @@ func (d *PostgresDatabase) nodeTableQuery() *gorm.DB {
 func (d *PostgresDatabase) GetNodes(filter types.NodeFilter, limit types.Limit) ([]Node, uint, error) {
 	q := d.nodeTableQuery()
 	q = q.Session(&gorm.Session{Logger: logger.Default.LogMode(logger.Silent)})
+
+	condition := "TRUE"
 	if filter.Status != nil {
-		// TODO: this shouldn't be in db
-		threshold := time.Now().Unix() - nodeStateFactor*int64(reportInterval.Seconds())
-		if *filter.Status == "down" {
-			q = q.Where("node.updated_at < ? OR node.updated_at IS NULL", threshold)
-		} else {
-			q = q.Where("node.updated_at >= ?", threshold)
-		}
+		condition = decideNodeStatusCondition(*filter.Status)
 	}
+
+	q = q.Where(condition)
+
 	if filter.FreeMRU != nil {
 		q = q.Where("nodes_resources_view.free_mru >= ?", *filter.FreeMRU)
 	}

grid-proxy/internal/explorer/db/types.go

@@ -63,6 +63,13 @@ type Node struct {
 	SerialNumber    string
 	Longitude       *float64
 	Latitude        *float64
+	Power           NodePower `gorm:"type:jsonb"`
+}
+
+// NodePower struct is the farmerbot report for node status
+type NodePower struct {
+	State  string `json:"state"`
+	Target string `json:"target"`
 }
 
 // Farm data about a farm which is calculated from the chain

grid-proxy/internal/explorer/models.go

@@ -3,11 +3,11 @@ package explorer
 import (
 	"encoding/json"
 
-	"github.com/patrickmn/go-cache"
+	cache "github.com/patrickmn/go-cache"
 	"github.com/pkg/errors"
 	"github.com/threefoldtech/tfgrid-sdk-go/grid-proxy/internal/explorer/db"
 	"github.com/threefoldtech/tfgrid-sdk-go/grid-proxy/pkg/types"
-	"github.com/threefoldtech/tfgrid-sdk-go/rmb-sdk-go"
+	rmb "github.com/threefoldtech/tfgrid-sdk-go/rmb-sdk-go"
 )
 
 // ErrNodeNotFound creates new error type to define node existence or server problem

grid-proxy/internal/explorer/server.go

@@ -8,7 +8,7 @@ import (
 	"time"
 
 	"github.com/gorilla/mux"
-	"github.com/patrickmn/go-cache"
+	cache "github.com/patrickmn/go-cache"
 	"github.com/rs/zerolog/log"
 	httpSwagger "github.com/swaggo/http-swagger"
 
@@ -17,7 +17,7 @@ import (
 	"github.com/threefoldtech/tfgrid-sdk-go/grid-proxy/internal/explorer/db"
 	"github.com/threefoldtech/tfgrid-sdk-go/grid-proxy/internal/explorer/mw"
 	"github.com/threefoldtech/tfgrid-sdk-go/grid-proxy/pkg/types"
-	"github.com/threefoldtech/tfgrid-sdk-go/rmb-sdk-go"
+	rmb "github.com/threefoldtech/tfgrid-sdk-go/rmb-sdk-go"
 )
 
 const (

grid-proxy/pkg/client/retrying_grid_client.go

@@ -4,7 +4,7 @@ import (
 	"log"
 	"time"
 
-	"github.com/cenkalti/backoff/v3"
+	backoff "github.com/cenkalti/backoff/v3"
 	"github.com/threefoldtech/tfgrid-sdk-go/grid-proxy/pkg/types"
 )
 

grid-proxy/pkg/types/types.go

@@ -156,6 +156,12 @@ type Location struct {
 	Latitude  *float64 `json:"latitude"`
 }
 
+// NodePower struct is the farmerbot report for node status
+type NodePower struct {
+	State  string `json:"state"`
+	Target string `json:"target"`
+}
+
 // Node is a struct holding the data for a Node for the nodes view
 type Node struct {
 	ID                string       `json:"id"`
@@ -179,6 +185,7 @@ type Node struct {
 	RentContractID    uint         `json:"rentContractId"`
 	RentedByTwinID    uint         `json:"rentedByTwinId"`
 	SerialNumber      string       `json:"serialNumber"`
+	Power             NodePower    `json:"power"`
 }
 
 // CapacityResult is the NodeData capacity results to unmarshal json in it
@@ -209,6 +216,7 @@ type NodeWithNestedCapacity struct {
 	RentContractID    uint           `json:"rentContractId"`
 	RentedByTwinID    uint           `json:"rentedByTwinId"`
 	SerialNumber      string         `json:"serialNumber"`
+	Power             NodePower      `json:"power"`
 }
 
 type Twin struct {

grid-proxy/tests/queries/loader.go

@@ -50,7 +50,8 @@ func loadNodes(db *sql.DB, data *DBData) error {
 		COALESCE(serial_number, ''),
 		COALESCE(created_at, 0),
 		COALESCE(updated_at, 0),
-		COALESCE(location_id, '')
+		COALESCE(location_id, ''),
+		power
 	FROM
 		node;`)
 	if err != nil {
@@ -76,6 +77,7 @@ func loadNodes(db *sql.DB, data *DBData) error {
 			&node.created_at,
 			&node.updated_at,
 			&node.location_id,
+			&node.power,
 		); err != nil {
 			return err
 		}

grid-proxy/tests/queries/local_client.go

@@ -3,12 +3,17 @@ package main
 import (
 	"sort"
 	"strings"
+	"time"
 
 	proxyclient "github.com/threefoldtech/tfgrid-sdk-go/grid-proxy/pkg/client"
 	proxytypes "github.com/threefoldtech/tfgrid-sdk-go/grid-proxy/pkg/types"
 	"github.com/threefoldtech/zos/pkg/gridtypes"
 )
 
+const (
+	nodeUpInterval = -3 * time.Hour
+)
+
 // GridProxyClientimpl client that returns data directly from the db
 type GridProxyClientimpl struct {
 	data DBData
@@ -25,6 +30,18 @@ func (g *GridProxyClientimpl) Ping() error {
 	return nil
 }
 
+func decideNodeStatus(power nodePower, updatedAt uint64) string {
+	if power.Target == "Down" { // off or powering off
+		return "standby"
+	} else if power.Target == "Up" && power.State == "Down" { // powering on
+		return "down"
+	} else if int64(updatedAt) >= time.Now().Add(nodeUpInterval).Unix() {
+		return "up"
+	} else {
+		return "down"
+	}
+}
+
 // Nodes returns nodes with the given filters and pagination parameters
 func (g *GridProxyClientimpl) Nodes(filter proxytypes.NodeFilter, limit proxytypes.Limit) (res []proxytypes.Node, totalCount int, err error) {
 	if limit.Page == 0 {
@@ -35,10 +52,7 @@ func (g *GridProxyClientimpl) Nodes(filter proxytypes.NodeFilter, limit proxytyp
 	}
 	for _, node := range g.data.nodes {
 		if nodeSatisfies(&g.data, node, filter) {
-			status := STATUS_DOWN
-			if isUp(node.updated_at) {
-				status = STATUS_UP
-			}
+			status := decideNodeStatus(node.power, node.updated_at)
 			res = append(res, proxytypes.Node{
 				ID:              node.id,
 				NodeID:          int(node.node_id),
@@ -80,6 +94,10 @@ func (g *GridProxyClientimpl) Nodes(filter proxytypes.NodeFilter, limit proxytyp
 				RentedByTwinID:    uint(g.data.nodeRentedBy[node.node_id]),
 				RentContractID:    uint(g.data.nodeRentContractID[node.node_id]),
 				SerialNumber:      node.serial_number,
+				Power: proxytypes.NodePower{
+					State:  node.power.State,
+					Target: node.power.Target,
+				},
 			})
 		}
 	}
@@ -277,10 +295,7 @@ func (g *GridProxyClientimpl) Twins(filter proxytypes.TwinFilter, limit proxytyp
 }
 func (g *GridProxyClientimpl) Node(nodeID uint32) (res proxytypes.NodeWithNestedCapacity, err error) {
 	node := g.data.nodes[uint64(nodeID)]
-	status := STATUS_DOWN
-	if isUp(node.updated_at) {
-		status = STATUS_UP
-	}
+	status := decideNodeStatus(node.power, node.updated_at)
 	res = proxytypes.NodeWithNestedCapacity{
 		ID:              node.id,
 		NodeID:          int(node.node_id),
@@ -324,16 +339,17 @@ func (g *GridProxyClientimpl) Node(nodeID uint32) (res proxytypes.NodeWithNested
 		RentedByTwinID:    uint(g.data.nodeRentedBy[node.node_id]),
 		RentContractID:    uint(g.data.nodeRentContractID[node.node_id]),
 		SerialNumber:      node.serial_number,
+		Power: proxytypes.NodePower{
+			State:  node.power.State,
+			Target: node.power.Target,
+		},
 	}
 	return
 }
 
 func (g *GridProxyClientimpl) NodeStatus(nodeID uint32) (res proxytypes.NodeStatus, err error) {
 	node := g.data.nodes[uint64(nodeID)]
-	res.Status = STATUS_DOWN
-	if isUp(node.updated_at) {
-		res.Status = STATUS_UP
-	}
+	res.Status = decideNodeStatus(node.power, node.updated_at)
 	return
 }
 

grid-proxy/tests/queries/types.go

@@ -1,6 +1,11 @@
 // nolint
 package main
 
+import (
+	"encoding/json"
+	"fmt"
+)
+
 // TODO: the one in tools/db/types.go is unexported but it's the same file
 
 var (
@@ -53,7 +58,25 @@ type node struct {
 	created_at        uint64
 	updated_at        uint64
 	location_id       string
+	power             nodePower `gorm:"type:jsonb"`
+}
+
+type nodePower struct {
+	State  string `json:"state"`
+	Target string `json:"target"`
 }
+
+// Scan is a custom decoder for jsonb filed. executed while scanning the node.
+func (np *nodePower) Scan(value interface{}) error {
+	if value == nil {
+		return nil
+	}
+	if data, ok := value.([]byte); ok {
+		return json.Unmarshal(data, np)
+	}
+	return fmt.Errorf("failed to unmarshal NodePower")
+}
+
 type twin struct {
 	id           string
 	grid_version uint64