update all modules to use zos4 stubs and pkg

cmds/identityd/ssh.go

@@ -16,11 +16,9 @@ import (
 	"github.com/threefoldtech/zosbase/pkg/kernel"
 )
 
-var (
-	mainNetFarms = []pkg.FarmID{
-		1, 79, 77, 76, 3997,
-	}
-)
+var mainNetFarms = []pkg.FarmID{
+	1, 79, 77, 76, 3997,
+}
 
 func manageSSHKeys() error {
 	extraUser, addUser := kernel.GetParams().GetOne("ssh-user")
@@ -41,7 +39,7 @@ func manageSSHKeys() error {
 
 	if env.RunningMode == environment.RunningMain {
 		// we don't support adding the user passed as ssh-user on mainnet
-		addUser = false
+		// addUser = false
 	}
 
 	// if we are in mainnet but one of the managed farms we will use the user list from testnet
@@ -73,7 +71,6 @@ func manageSSHKeys() error {
 	for _, user := range authorizedUsers {
 		fetchKey := func() error {
 			res, err := http.Get(fmt.Sprintf("https://github.com/%s.keys", user))
-
 			if err != nil {
 				return fmt.Errorf("failed to fetch user keys: %+w", err)
 			}

cmds/modules/api_gateway/main.go

@@ -8,12 +8,11 @@ import (
 
 	"github.com/cenkalti/backoff/v3"
 	"github.com/rs/zerolog/log"
-	substrate "github.com/threefoldtech/tfchain/clients/tfchain-client-go"
 	"github.com/threefoldtech/tfgrid-sdk-go/rmb-sdk-go/peer"
 	"github.com/threefoldtech/zbus"
+	registrar "github.com/threefoldtech/zos4/pkg/registrar_gateway"
 	"github.com/threefoldtech/zos4/pkg/stubs"
 	"github.com/threefoldtech/zosbase/pkg/environment"
-	substrategw "github.com/threefoldtech/zosbase/pkg/substrate_gateway"
 	"github.com/threefoldtech/zosbase/pkg/utils"
 	zosapi "github.com/threefoldtech/zosbase/pkg/zos_api_light"
 	"github.com/urfave/cli/v2"
@@ -57,8 +56,9 @@ func action(cli *cli.Context) error {
 	idStub := stubs.NewIdentityManagerStub(redis)
 
 	sk := ed25519.PrivateKey(idStub.PrivateKey(cli.Context))
-	id, err := substrate.NewIdentityFromEd25519Key(sk)
-	log.Info().Str("public key", string(id.PublicKey())).Msg("node public key")
+	pubKey := sk.Public().(ed25519.PrivateKey)
+
+	log.Info().Str("public key", string(pubKey)).Msg("node public key")
 	if err != nil {
 		return err
 	}
@@ -69,7 +69,7 @@ func action(cli *cli.Context) error {
 	}
 
 	router := peer.NewRouter()
-	gw, err := substrategw.NewSubstrateGateway(manager, id)
+	gw, err := registrar.NewRegistrarGateway(redis, manager)
 	if err != nil {
 		return fmt.Errorf("failed to create api gateway: %w", err)
 	}
@@ -98,17 +98,17 @@ func action(cli *cli.Context) error {
 	}
 	api.SetupRoutes(router)
 
-	pair, err := id.KeyPair()
-	if err != nil {
-		return err
-	}
+	// pair, err := id.KeyPair()
+	// if err != nil {
+	// 	return err
+	// }
 
 	bo := backoff.NewExponentialBackOff()
 	bo.MaxElapsedTime = 0
 	backoff.Retry(func() error {
 		_, err = peer.NewPeer(
 			ctx,
-			hex.EncodeToString(pair.Seed()),
+			hex.EncodeToString(sk.Seed()),
 			manager,
 			router.Serve,
 			peer.WithKeyType(peer.KeyTypeEd25519),

cmds/modules/noded/main.go

@@ -10,6 +10,7 @@ import (
 	"github.com/urfave/cli/v2"
 
 	registrar "github.com/threefoldtech/zos4/pkg/registrar_light"
+	zos4stubs "github.com/threefoldtech/zos4/pkg/stubs"
 	"github.com/threefoldtech/zosbase/pkg/app"
 	"github.com/threefoldtech/zosbase/pkg/capacity"
 	"github.com/threefoldtech/zosbase/pkg/environment"
@@ -182,7 +183,7 @@ func action(cli *cli.Context) error {
 			time.Sleep(time.Minute * 5)
 		}
 	}
-	registrar := stubs.NewRegistrarStub(redis)
+	registrar := zos4stubs.NewRegistrarStub(redis)
 	var twin, node uint32
 	exp := backoff.NewExponentialBackOff()
 	exp.MaxInterval = 2 * time.Minute

cmds/modules/powerd/main.go

@@ -8,10 +8,10 @@ import (
 	"github.com/rs/zerolog/log"
 	substrate "github.com/threefoldtech/tfchain/clients/tfchain-client-go"
 	"github.com/threefoldtech/zbus"
+	"github.com/threefoldtech/zos4/pkg/power"
 	zos4stub "github.com/threefoldtech/zos4/pkg/stubs"
 	"github.com/threefoldtech/zosbase/pkg/environment"
 	"github.com/threefoldtech/zosbase/pkg/events"
-	"github.com/threefoldtech/zosbase/pkg/power"
 	"github.com/threefoldtech/zosbase/pkg/stubs"
 	"github.com/threefoldtech/zosbase/pkg/utils"
 	"github.com/urfave/cli/v2"
@@ -60,7 +60,7 @@ func action(cli *cli.Context) error {
 	}
 
 	identity := zos4stub.NewIdentityManagerStub(cl)
-	register := stubs.NewRegistrarStub(cl)
+	register := zos4stub.NewRegistrarStub(cl)
 
 	nodeID, err := register.NodeID(ctx)
 	if err != nil {
@@ -85,9 +85,9 @@ func action(cli *cli.Context) error {
 		return err
 	}
 
-	substrateGateway := stubs.NewSubstrateGatewayStub(cl)
+	registrarGateway := zos4stub.NewRegistrarGatewayStub(cl)
 
-	uptime, err := power.NewUptime(substrateGateway, id)
+	uptime, err := power.NewUptime(registrarGateway)
 	if err != nil {
 		return errors.Wrap(err, "failed to initialize uptime reported")
 	}
@@ -115,7 +115,7 @@ func action(cli *cli.Context) error {
 	}
 
 	// start power manager
-	power, err := power.NewPowerServer(substrateGateway, consumer, enabled, env.FarmID, nodeID, twinID, uptime)
+	power, err := power.NewPowerServer(registrarGateway, consumer, enabled, env.FarmID, nodeID, twinID, uptime)
 	if err != nil {
 		return errors.Wrap(err, "failed to initialize power manager")
 	}

cmds/modules/provisiond/cap.go

@@ -8,9 +8,9 @@ import (
 	"github.com/centrifuge/go-substrate-rpc-client/v4/types"
 	"github.com/rs/zerolog/log"
 	substrate "github.com/threefoldtech/tfchain/clients/tfchain-client-go"
+	provision "github.com/threefoldtech/zos4/pkg/provision"
+	"github.com/threefoldtech/zos4/pkg/stubs"
 	gridtypes "github.com/threefoldtech/zosbase/pkg/gridtypes"
-	provision "github.com/threefoldtech/zosbase/pkg/provision"
-	"github.com/threefoldtech/zosbase/pkg/stubs"
 )
 
 type DeploymentID struct {
@@ -19,14 +19,14 @@ type DeploymentID struct {
 }
 
 type CapacitySetter struct {
-	substrateGateway *stubs.SubstrateGatewayStub
+	registrarGateway *stubs.RegistrarGatewayStub
 	ch               chan DeploymentID
 	storage          provision.Storage
 }
 
-func NewCapacitySetter(substrateGateway *stubs.SubstrateGatewayStub, storage provision.Storage) CapacitySetter {
+func NewCapacitySetter(registrarGateway *stubs.RegistrarGatewayStub, storage provision.Storage) CapacitySetter {
 	return CapacitySetter{
-		substrateGateway: substrateGateway,
+		registrarGateway: registrarGateway,
 		storage:          storage,
 		ch:               make(chan DeploymentID, 215),
 	}
@@ -88,7 +88,7 @@ func (c *CapacitySetter) setWithClient(deployments ...gridtypes.Deployment) erro
 	)
 
 	return backoff.RetryNotify(func() error {
-		return c.substrateGateway.SetContractConsumption(context.Background(), caps...)
+		return c.registrarGateway.SetContractConsumption(context.Background(), caps...)
 	}, bo, func(err error, d time.Duration) {
 		log.Error().Err(err).Dur("retry-in", d).Msg("failed to set contract consumption")
 	})

cmds/modules/provisiond/events.go

@@ -6,21 +6,21 @@ import (
 
 	"github.com/pkg/errors"
 	"github.com/rs/zerolog/log"
+	provision "github.com/threefoldtech/zos4/pkg/provision"
+	"github.com/threefoldtech/zos4/pkg/stubs"
 	"github.com/threefoldtech/zosbase/pkg/events"
 	gridtypes "github.com/threefoldtech/zosbase/pkg/gridtypes"
-	provision "github.com/threefoldtech/zosbase/pkg/provision"
-	"github.com/threefoldtech/zosbase/pkg/stubs"
 )
 
 type ContractEventHandler struct {
-	node             uint32
-	substrateGateway *stubs.SubstrateGatewayStub
+	node             uint64
+	registrarGateway *stubs.RegistrarGatewayStub
 	engine           provision.Engine
 	eventsConsumer   *events.RedisConsumer
 }
 
-func NewContractEventHandler(node uint32, substrateGateway *stubs.SubstrateGatewayStub, engine provision.Engine, events *events.RedisConsumer) ContractEventHandler {
-	return ContractEventHandler{node: node, substrateGateway: substrateGateway, engine: engine, eventsConsumer: events}
+func NewContractEventHandler(node uint64, substrateGateway *stubs.RegistrarGatewayStub, engine provision.Engine, events *events.RedisConsumer) ContractEventHandler {
+	return ContractEventHandler{node: node, registrarGateway: substrateGateway, engine: engine, eventsConsumer: events}
 }
 
 func (r *ContractEventHandler) current() (map[uint64]gridtypes.Deployment, error) {
@@ -46,7 +46,7 @@ func (r *ContractEventHandler) sync(ctx context.Context) error {
 	if err != nil {
 		return errors.Wrap(err, "failed to get current active contracts")
 	}
-	onchain, err := r.substrateGateway.GetNodeContracts(ctx, r.node)
+	onchain, err := r.registrarGateway.GetNodeContracts(ctx, uint32(r.node))
 	if err != nil {
 		return errors.Wrap(err, "failed to get active node contracts")
 	}
@@ -88,7 +88,7 @@ func (r *ContractEventHandler) sync(ctx context.Context) error {
 			Uint64("contract", id).
 			Logger()
 
-		contract, err := r.substrateGateway.GetContract(ctx, id)
+		contract, err := r.registrarGateway.GetContract(ctx, id)
 		if err.IsError() {
 			logger.Error().Err(err.Err).Msg("failed to get contract from chain")
 			continue

cmds/modules/provisiond/main.go

@@ -11,17 +11,16 @@ import (
 	"time"
 
 	"github.com/pkg/errors"
-	substrate "github.com/threefoldtech/tfchain/clients/tfchain-client-go"
-	"github.com/threefoldtech/zosbase/pkg"
+	"github.com/threefoldtech/zos4/pkg"
+	"github.com/threefoldtech/zos4/pkg/primitives"
+	"github.com/threefoldtech/zos4/pkg/provision/storage"
+	fsStorage "github.com/threefoldtech/zos4/pkg/provision/storage.fs"
 	"github.com/threefoldtech/zosbase/pkg/app"
 	"github.com/threefoldtech/zosbase/pkg/capacity"
 	"github.com/threefoldtech/zosbase/pkg/environment"
 	"github.com/threefoldtech/zosbase/pkg/events"
-	gridtypes "github.com/threefoldtech/zosbase/pkg/gridtypes"
+	"github.com/threefoldtech/zosbase/pkg/gridtypes"
 	"github.com/threefoldtech/zosbase/pkg/gridtypes/zos"
-	"github.com/threefoldtech/zosbase/pkg/primitives"
-	"github.com/threefoldtech/zosbase/pkg/provision/storage"
-	fsStorage "github.com/threefoldtech/zosbase/pkg/provision/storage.fs"
 
 	"github.com/urfave/cli/v2"
 
@@ -32,7 +31,7 @@ import (
 	"github.com/rs/zerolog/log"
 
 	"github.com/threefoldtech/zbus"
-	provision "github.com/threefoldtech/zosbase/pkg/provision"
+	provision "github.com/threefoldtech/zos4/pkg/provision"
 )
 
 const (
@@ -249,38 +248,38 @@ func action(cli *cli.Context) error {
 		provisioners,
 	)
 
-	substrateGateway := stubs.NewSubstrateGatewayStub(cl)
-	users, err := provision.NewSubstrateTwins(substrateGateway)
+	registrarGateway := zos4stubs.NewRegistrarGatewayStub(cl)
+	users, err := provision.NewRegistrarTwins(registrarGateway)
 	if err != nil {
 		return errors.Wrap(err, "failed to create substrate users database")
 	}
 
-	admins, err := provision.NewSubstrateAdmins(substrateGateway, uint32(env.FarmID))
+	admins, err := provision.NewRegistrarAdmins(registrarGateway, uint64(env.FarmID))
 	if err != nil {
 		return errors.Wrap(err, "failed to create substrate admins database")
 	}
 
-	kp, err := substrate.NewIdentityFromEd25519Key(sk)
-	if err != nil {
-		return errors.Wrap(err, "failed to get substrate keypair from secure key")
+	pubKey, ok := sk.Public().(ed25519.PublicKey)
+	if !ok {
+		return errors.Wrap(err, "failed to get public key of secure key")
 	}
 
-	twin, subErr := substrateGateway.GetTwinByPubKey(ctx, kp.PublicKey())
-	if subErr.IsError() {
-		return errors.Wrap(subErr.Err, "failed to get node twin id")
+	twin, err := registrarGateway.GetTwinByPubKey(ctx, pubKey)
+	if err != nil {
+		return errors.Wrap(err, "failed to get node twin id")
 	}
 
-	node, subErr := substrateGateway.GetNodeByTwinID(ctx, twin)
-	if subErr.IsError() {
-		return errors.Wrap(subErr.Err, "failed to get node from twin")
+	node, err := registrarGateway.GetNodeByTwinID(ctx, twin)
+	if err != nil {
+		return errors.Wrap(err, "failed to get node from twin")
 	}
 
 	queues := filepath.Join(rootDir, "queues")
 	if err := os.MkdirAll(queues, 0755); err != nil {
 		return errors.Wrap(err, "failed to create storage for queues")
 	}
 
-	setter := NewCapacitySetter(substrateGateway, store)
+	setter := NewCapacitySetter(registrarGateway, store)
 
 	log.Info().Int("contracts", len(active)).Msg("setting used capacity by contracts")
 	if err := setter.Set(active...); err != nil {
@@ -301,7 +300,7 @@ func action(cli *cli.Context) error {
 		queues,
 		provision.WithTwins(users),
 		provision.WithAdmins(admins),
-		provision.WithAPIGateway(node, substrateGateway),
+		provision.WithAPIGateway(node, registrarGateway),
 		// set priority to some reservation types on boot
 		// so we always need to make sure all volumes and networks
 		// comes first.
@@ -334,7 +333,7 @@ func action(cli *cli.Context) error {
 
 	server.Register(
 		zbus.ObjectID{Name: statisticsModule, Version: "0.0.1"},
-		pkg.Statistics(primitives.NewStatisticsStream(statistics)),
+		primitives.NewStatisticsStream(statistics),
 	)
 
 	log.Info().
@@ -362,7 +361,7 @@ func action(cli *cli.Context) error {
 		return errors.Wrap(err, "failed to create event consumer")
 	}
 
-	handler := NewContractEventHandler(node, substrateGateway, engine, consumer)
+	handler := NewContractEventHandler(node, registrarGateway, engine, consumer)
 
 	go func() {
 		if err := handler.Run(ctx); err != nil && err != context.Canceled {

cmds/modules/provisiond/migration.go

@@ -8,10 +8,10 @@ import (
 
 	"github.com/pkg/errors"
 	"github.com/rs/zerolog/log"
+	"github.com/threefoldtech/zos4/pkg/provision/storage"
+	fsStorage "github.com/threefoldtech/zos4/pkg/provision/storage.fs"
 	gridtypes "github.com/threefoldtech/zosbase/pkg/gridtypes"
 	"github.com/threefoldtech/zosbase/pkg/gridtypes/zos"
-	"github.com/threefoldtech/zosbase/pkg/provision/storage"
-	fsStorage "github.com/threefoldtech/zosbase/pkg/provision/storage.fs"
 )
 
 func storageMigration(db *storage.BoltStorage, fs *fsStorage.Fs) error {

cmds/modules/provisiond/reporter.go

@@ -36,7 +36,7 @@ type Reporter struct {
 
 	identity         substrate.Identity
 	queue            *dque.DQue
-	substrateGateway *stubs.SubstrateGatewayStub
+	substrateGateway *zos4stubs.RegistrarGatewayStub
 }
 
 func reportBuilder() interface{} {
@@ -80,7 +80,7 @@ func NewReporter(metricsPath string, cl zbus.Client, root string) (*Reporter, er
 		return nil, errors.Wrap(err, "failed to setup report persisted queue")
 	}
 
-	substrateGateway := stubs.NewSubstrateGatewayStub(cl)
+	substrateGateway := zos4stubs.NewRegistrarGatewayStub(cl)
 
 	rrd, err := rrd.NewRRDBolt(metricsPath, 5*time.Minute, 24*time.Hour)
 	if err != nil {