From eae1b0f6bfc687c87c70b1093f4c9f45386099df Mon Sep 17 00:00:00 2001 From: AbdelrahmanElawady <60783742+AbdelrahmanElawady@users.noreply.github.com> Date: Tue, 30 Jan 2024 12:37:38 +0200 Subject: [PATCH] Add json flag for json input and output (#692) * Add json flag for json input and output * Add output flag * Remove false config check * Add error check * Remove error check --- mass-deployer/README.md | 2 +- mass-deployer/cmd/cancel.go | 4 +- mass-deployer/cmd/deploy.go | 11 ++- mass-deployer/example/conf.json | 43 ++++++++++++ mass-deployer/internal/parser/parser.go | 10 ++- mass-deployer/internal/parser/parser_test.go | 47 ++++++++----- mass-deployer/pkg/mass-deployer/deployer.go | 69 +++++++++---------- mass-deployer/pkg/mass-deployer/types.go | 71 +++++++++++--------- 8 files changed, 166 insertions(+), 91 deletions(-) create mode 100644 mass-deployer/example/conf.json diff --git a/mass-deployer/README.md b/mass-deployer/README.md index 42fa91340..7a7fc0029 100644 --- a/mass-deployer/README.md +++ b/mass-deployer/README.md @@ -8,7 +8,7 @@ tfrobot is tool designed to automate mass deployment of groups of VMs on ThreeFo - **Mass Deployment:** Deploy groups of vms on ThreeFold Grid simultaneously. - **Mass Cancelation:** cancel all vms on ThreeFold Grid defined in configuration file simultaneously. -- **Customizable Configurations:** Define Node groups, VMs groups and other configurations through YAML file. +- **Customizable Configurations:** Define Node groups, VMs groups and other configurations through YAML or JSON file. ## Download diff --git a/mass-deployer/cmd/cancel.go b/mass-deployer/cmd/cancel.go index e63f4506a..1e6d350dd 100644 --- a/mass-deployer/cmd/cancel.go +++ b/mass-deployer/cmd/cancel.go @@ -3,6 +3,7 @@ package cmd import ( "os" + "path/filepath" "github.com/rs/zerolog/log" "github.com/spf13/cobra" @@ -24,8 +25,9 @@ var cancelCmd = &cobra.Command{ log.Fatal().Err(err).Msgf("failed to open config file: %s", configPath) } defer configFile.Close() + jsonFmt := filepath.Ext(configPath) == ".json" - cfg, err := parser.ParseConfig(configFile) + cfg, err := parser.ParseConfig(configFile, jsonFmt) if err != nil { log.Fatal().Err(err).Msgf("failed to parse config file: %s", configPath) } diff --git a/mass-deployer/cmd/deploy.go b/mass-deployer/cmd/deploy.go index 291d4367b..f594dba44 100644 --- a/mass-deployer/cmd/deploy.go +++ b/mass-deployer/cmd/deploy.go @@ -4,6 +4,7 @@ package cmd import ( "context" "os" + "path/filepath" "github.com/rs/zerolog/log" "github.com/spf13/cobra" @@ -19,20 +20,25 @@ var deployCmd = &cobra.Command{ if err != nil || configPath == "" { log.Fatal().Err(err).Msg("error in config file") } + output, err := cmd.Flags().GetString("output") + if err != nil { + log.Fatal().Err(err).Msg("error in output file") + } configFile, err := os.Open(configPath) if err != nil { log.Fatal().Err(err).Msgf("failed to open config file: %s", configPath) } defer configFile.Close() + jsonFmt := filepath.Ext(configPath) == ".json" - cfg, err := parser.ParseConfig(configFile) + cfg, err := parser.ParseConfig(configFile, jsonFmt) if err != nil { log.Fatal().Err(err).Msgf("failed to parse config file: %s", configPath) } ctx := context.Background() - err = deployer.RunDeployer(cfg, ctx) + err = deployer.RunDeployer(cfg, ctx, output) if err != nil { log.Fatal().Err(err).Msg("failed to run the deployer") } @@ -41,5 +47,6 @@ var deployCmd = &cobra.Command{ func init() { deployCmd.Flags().StringP("config", "c", "", "path to config file") + deployCmd.Flags().StringP("output", "o", "", "output file") rootCmd.AddCommand(deployCmd) } diff --git a/mass-deployer/example/conf.json b/mass-deployer/example/conf.json new file mode 100644 index 000000000..9cd643e75 --- /dev/null +++ b/mass-deployer/example/conf.json @@ -0,0 +1,43 @@ +{ + "node_groups": [ + { + "name": "group_a", + "nodes_count": 3, + "free_cpu": 2, + "free_mru": 16384, + "free_ssd": 100, + "free_hdd": 50, + "dedicated": false, + "public_ip4": false, + "public_ip6": false, + "certified": false, + "region": "europe" + } + ], + "vms": [ + { + "name": "examplevm123", + "vms_count": 1, + "node_group": "group_a", + "cpu": 1, + "mem": 256, + "ssd": [ + { + "size": 15, + "mount_point": "/mnt/ssd" + } + ], + "public_ip4": false, + "public_ip6": false, + "flist": "https://hub.grid.tf/tf-official-apps/base:latest.flist", + "entry_point": "/sbin/zinit init", + "root_size": 0, + "ssh_key": "example1" + } + ], + "ssh_keys": { + "example1": "ssh_key1" + }, + "mnemonic": "example-mnemonic", + "network": "dev" +} \ No newline at end of file diff --git a/mass-deployer/internal/parser/parser.go b/mass-deployer/internal/parser/parser.go index 2de4940d3..eaf1f7cc9 100644 --- a/mass-deployer/internal/parser/parser.go +++ b/mass-deployer/internal/parser/parser.go @@ -1,6 +1,7 @@ package parser import ( + "encoding/json" "fmt" "io" "os" @@ -17,7 +18,7 @@ const ( networkKey = "NETWORK" ) -func ParseConfig(file io.Reader) (deployer.Config, error) { +func ParseConfig(file io.Reader, jsonFmt bool) (deployer.Config, error) { conf := deployer.Config{} nodeGroupsNames := []string{} @@ -25,8 +26,11 @@ func ParseConfig(file io.Reader) (deployer.Config, error) { if err != nil { return deployer.Config{}, fmt.Errorf("failed to read the config file %+w", err) } - - err = yaml.Unmarshal(configFile, &conf) + if jsonFmt { + err = json.Unmarshal(configFile, &conf) + } else { + err = yaml.Unmarshal(configFile, &conf) + } if err != nil { return deployer.Config{}, err } diff --git a/mass-deployer/internal/parser/parser_test.go b/mass-deployer/internal/parser/parser_test.go index 02ce340f6..570010d02 100644 --- a/mass-deployer/internal/parser/parser_test.go +++ b/mass-deployer/internal/parser/parser_test.go @@ -1,6 +1,7 @@ package parser import ( + "encoding/json" "os" "strings" "testing" @@ -57,7 +58,7 @@ func TestParseConfig(t *testing.T) { } ` configFile := strings.NewReader(conf) - _, err := ParseConfig(configFile) + _, err := ParseConfig(configFile, false) assert.Error(t, err) }) @@ -70,7 +71,7 @@ func TestParseConfig(t *testing.T) { configFile := strings.NewReader(string(data)) - _, err = ParseConfig(configFile) + _, err = ParseConfig(configFile, false) assert.Error(t, err) }) @@ -83,7 +84,7 @@ func TestParseConfig(t *testing.T) { configFile := strings.NewReader(string(data)) - _, err = ParseConfig(configFile) + _, err = ParseConfig(configFile, false) assert.Error(t, err) }) @@ -96,7 +97,7 @@ func TestParseConfig(t *testing.T) { configFile := strings.NewReader(string(data)) - _, err = ParseConfig(configFile) + _, err = ParseConfig(configFile, false) assert.Error(t, err) }) @@ -109,7 +110,7 @@ func TestParseConfig(t *testing.T) { configFile := strings.NewReader(string(data)) - _, err = ParseConfig(configFile) + _, err = ParseConfig(configFile, false) assert.Error(t, err) }) @@ -122,7 +123,7 @@ func TestParseConfig(t *testing.T) { configFile := strings.NewReader(string(data)) - _, err = ParseConfig(configFile) + _, err = ParseConfig(configFile, false) assert.Error(t, err) }) @@ -137,7 +138,7 @@ func TestParseConfig(t *testing.T) { configFile := strings.NewReader(string(data)) - _, err = ParseConfig(configFile) + _, err = ParseConfig(configFile, false) assert.Error(t, err) }) @@ -152,7 +153,7 @@ func TestParseConfig(t *testing.T) { configFile := strings.NewReader(string(data)) - _, err = ParseConfig(configFile) + _, err = ParseConfig(configFile, false) assert.Error(t, err) }) @@ -167,7 +168,7 @@ func TestParseConfig(t *testing.T) { configFile := strings.NewReader(string(data)) - _, err = ParseConfig(configFile) + _, err = ParseConfig(configFile, false) assert.Error(t, err) }) @@ -182,7 +183,7 @@ func TestParseConfig(t *testing.T) { configFile := strings.NewReader(string(data)) - _, err = ParseConfig(configFile) + _, err = ParseConfig(configFile, false) assert.Error(t, err) }) @@ -197,7 +198,7 @@ func TestParseConfig(t *testing.T) { configFile := strings.NewReader(string(data)) - _, err = ParseConfig(configFile) + _, err = ParseConfig(configFile, false) assert.Error(t, err) }) @@ -212,7 +213,7 @@ func TestParseConfig(t *testing.T) { configFile := strings.NewReader(string(data)) - _, err = ParseConfig(configFile) + _, err = ParseConfig(configFile, false) assert.Error(t, err) }) @@ -227,7 +228,7 @@ func TestParseConfig(t *testing.T) { configFile := strings.NewReader(string(data)) - _, err = ParseConfig(configFile) + _, err = ParseConfig(configFile, false) assert.Error(t, err) }) @@ -242,7 +243,7 @@ func TestParseConfig(t *testing.T) { configFile := strings.NewReader(string(data)) - _, err = ParseConfig(configFile) + _, err = ParseConfig(configFile, false) assert.Error(t, err) }) @@ -257,7 +258,7 @@ func TestParseConfig(t *testing.T) { configFile := strings.NewReader(string(data)) - _, err = ParseConfig(configFile) + _, err = ParseConfig(configFile, false) assert.Error(t, err) }) @@ -272,7 +273,7 @@ func TestParseConfig(t *testing.T) { configFile := strings.NewReader(string(data)) - _, err = ParseConfig(configFile) + _, err = ParseConfig(configFile, false) assert.Error(t, err) }) @@ -284,7 +285,19 @@ func TestParseConfig(t *testing.T) { configFile := strings.NewReader(string(data)) - _, err = ParseConfig(configFile) + _, err = ParseConfig(configFile, false) assert.NoError(t, err) }) + t.Run("valid json config", func(t *testing.T) { + conf := confStruct + + data, err := json.Marshal(conf) + assert.NoError(t, err) + + configFile := strings.NewReader(string(data)) + + parsedConf, err := ParseConfig(configFile, true) + assert.NoError(t, err) + assert.Equal(t, conf, parsedConf) + }) } diff --git a/mass-deployer/pkg/mass-deployer/deployer.go b/mass-deployer/pkg/mass-deployer/deployer.go index fc448bde5..6cbf7acdc 100644 --- a/mass-deployer/pkg/mass-deployer/deployer.go +++ b/mass-deployer/pkg/mass-deployer/deployer.go @@ -2,9 +2,11 @@ package deployer import ( "context" + "encoding/json" "fmt" "net" "os" + "path/filepath" "sync" "time" @@ -17,8 +19,8 @@ import ( "github.com/threefoldtech/zos/pkg/gridtypes" ) -func RunDeployer(cfg Config, ctx context.Context) error { - passedGroups := map[string][]string{} +func RunDeployer(cfg Config, ctx context.Context, output string) error { + passedGroups := map[string][]vmOutput{} failedGroups := map[string]error{} tfPluginClient, err := setup(cfg) @@ -54,36 +56,43 @@ func RunDeployer(cfg Config, ctx context.Context) error { log.Info().Msgf("deployment took %s", time.Since(deploymentStart)) - if len(passedGroups) > 0 { - fmt.Println("ok:") + outData := struct { + OK map[string][]vmOutput `json:"ok"` + Error map[string]error `json:"error"` + }{ + OK: passedGroups, + Error: failedGroups, } - for group, info := range passedGroups { - fmt.Printf("%s:\n", group) - for _, s := range info { - fmt.Println(s) - } + var out []byte + if filepath.Ext(output) == ".json" { + out, err = json.MarshalIndent(outData, "", " ") + } else { + out, err = yaml.Marshal(outData) } - - if len(failedGroups) > 0 { - fmt.Fprintln(os.Stderr, "error:") + if err != nil { + return err + } + fmt.Println(string(out)) + if output == "" { + return nil } - for group, err := range failedGroups { - fmt.Fprintf(os.Stderr, "%s: %v\n", group, err) + if err := os.WriteFile(output, out, 0644); err != nil { + return err } return nil } -func deployNodeGroup(tfPluginClient deployer.TFPluginClient, ctx context.Context, nodeGroup NodesGroup, vms []Vms, sshKeys map[string]string) ([]string, error) { +func deployNodeGroup(tfPluginClient deployer.TFPluginClient, ctx context.Context, nodeGroup NodesGroup, vms []Vms, sshKeys map[string]string) ([]vmOutput, error) { nodesIDs, err := filterNodes(tfPluginClient, nodeGroup, ctx) if err != nil { - return []string{}, err + return []vmOutput{}, err } groupsDeployments := parseGroupVMs(vms, nodeGroup.Name, nodesIDs, sshKeys) info, err := massDeploy(tfPluginClient, ctx, groupsDeployments) if err != nil { - return []string{}, err + return []vmOutput{}, err } return info, nil @@ -100,17 +109,17 @@ func parseGroupVMs(vms []Vms, nodeGroup string, nodesIDs []int, sshKeys map[stri return buildDeployments(vmsOfNodeGroup, nodeGroup, nodesIDs, sshKeys) } -func massDeploy(tfPluginClient deployer.TFPluginClient, ctx context.Context, deployments groupDeploymentsInfo) ([]string, error) { +func massDeploy(tfPluginClient deployer.TFPluginClient, ctx context.Context, deployments groupDeploymentsInfo) ([]vmOutput, error) { err := tfPluginClient.NetworkDeployer.BatchDeploy(ctx, deployments.networkDeployments) if err != nil { cancelContractsOfFailedDeployments(tfPluginClient, deployments.networkDeployments, []*workloads.Deployment{}) - return []string{}, err + return []vmOutput{}, err } err = tfPluginClient.DeploymentDeployer.BatchDeploy(ctx, deployments.vmDeployments) if err != nil { cancelContractsOfFailedDeployments(tfPluginClient, deployments.networkDeployments, deployments.vmDeployments) - return []string{}, err + return []vmOutput{}, err } vmsInfo := loadDeploymentsInfo(tfPluginClient, deployments.deploymentsInfo) @@ -188,8 +197,8 @@ func cancelContractsOfFailedDeployments(tfPluginClient deployer.TFPluginClient, } } -func loadDeploymentsInfo(tfPluginClient deployer.TFPluginClient, deployments []vmDeploymentInfo) []string { - vmsInfo := []string{} +func loadDeploymentsInfo(tfPluginClient deployer.TFPluginClient, deployments []vmDeploymentInfo) []vmOutput { + vmsInfo := []vmOutput{} var lock sync.Mutex var wg sync.WaitGroup @@ -205,23 +214,11 @@ func loadDeploymentsInfo(tfPluginClient deployer.TFPluginClient, deployments []v return } - vmInfo := struct { - Name string - PublicIP4 string - PublicIP6 string - YggIP string - IP string - Mounts []workloads.Mount - }{vm.Name, vm.ComputedIP, vm.ComputedIP6, vm.YggIP, vm.IP, vm.Mounts} - - groupInfo, err := yaml.Marshal(vmInfo) - if err != nil { - log.Debug().Err(err).Msg("failed to marshal json") - } + vmInfo := vmOutput{vm.Name, vm.ComputedIP, vm.ComputedIP6, vm.YggIP, vm.IP, vm.Mounts} lock.Lock() defer lock.Unlock() - vmsInfo = append(vmsInfo, string(groupInfo)) + vmsInfo = append(vmsInfo, vmInfo) }(info) } diff --git a/mass-deployer/pkg/mass-deployer/types.go b/mass-deployer/pkg/mass-deployer/types.go index 9bf60f113..e7bedf20b 100644 --- a/mass-deployer/pkg/mass-deployer/types.go +++ b/mass-deployer/pkg/mass-deployer/types.go @@ -5,46 +5,46 @@ import "github.com/threefoldtech/tfgrid-sdk-go/grid-client/workloads" // type config contains configuration used to deploy multible groups of vms in batches // **note: please make sure to run validator (validator.Validate(conf))** type Config struct { - NodeGroups []NodesGroup `yaml:"node_groups" validate:"nonzero"` - Vms []Vms `yaml:"vms" validate:"nonzero"` - SSHKeys map[string]string `yaml:"ssh_keys" validate:"nonzero"` - Mnemonic string `yaml:"mnemonic" validate:"nonzero"` - Network string `yaml:"network" validate:"nonzero"` + NodeGroups []NodesGroup `yaml:"node_groups" validate:"nonzero" json:"node_groups"` + Vms []Vms `yaml:"vms" validate:"nonzero" json:"vms"` + SSHKeys map[string]string `yaml:"ssh_keys" validate:"nonzero" json:"ssh_keys"` + Mnemonic string `yaml:"mnemonic" validate:"nonzero" json:"mnemonic"` + Network string `yaml:"network" validate:"nonzero" json:"network"` } type NodesGroup struct { - Name string `yaml:"name" validate:"nonzero"` - NodesCount uint64 `yaml:"nodes_count" validate:"nonzero"` - FreeCPU uint64 `yaml:"free_cpu" validate:"nonzero"` - FreeMRU uint64 `yaml:"free_mru" validate:"nonzero"` - FreeSRU uint64 `yaml:"free_ssd"` - FreeHRU uint64 `yaml:"free_hdd"` - Dedicated bool `yaml:"dedicated"` - PublicIP4 bool `yaml:"public_ip4"` - PublicIP6 bool `yaml:"public_ip6"` - Certified bool `yaml:"certified"` - Regions string `yaml:"regions"` + Name string `yaml:"name" validate:"nonzero" json:"name"` + NodesCount uint64 `yaml:"nodes_count" validate:"nonzero" json:"nodes_count"` + FreeCPU uint64 `yaml:"free_cpu" validate:"nonzero" json:"free_cpu"` + FreeMRU uint64 `yaml:"free_mru" validate:"nonzero" json:"free_mru"` + FreeSRU uint64 `yaml:"free_ssd" json:"free_ssd"` + FreeHRU uint64 `yaml:"free_hdd" json:"free_hdd"` + Dedicated bool `yaml:"dedicated" json:"dedicated"` + PublicIP4 bool `yaml:"public_ip4" json:"pubip4"` + PublicIP6 bool `yaml:"public_ip6" json:"pubip6"` + Certified bool `yaml:"certified" json:"certified"` + Regions string `yaml:"regions" json:"regions"` } type Vms struct { - Name string `yaml:"name" validate:"nonzero"` - Count uint64 `yaml:"vms_count" validate:"nonzero"` - Nodegroup string `yaml:"node_group" validate:"nonzero"` - FreeCPU uint64 `yaml:"cpu" validate:"nonzero,max=32"` - FreeMRU uint64 `yaml:"mem" validate:"nonzero,min=256,max=262144"` // min: 256MB, max: 256 GB - SSDDisks []Disk `yaml:"ssd"` - PublicIP4 bool `yaml:"public_ip4"` - PublicIP6 bool `yaml:"public_ip6"` - Planetary bool `yaml:"planetary"` - Flist string `yaml:"flist" validate:"nonzero"` - Rootsize uint64 `yaml:"root_size" validate:"max=10240"` // max 10 TB - Entrypoint string `yaml:"entry_point" validate:"nonzero"` - SSHKey string `yaml:"ssh_key" validate:"nonzero"` + Name string `yaml:"name" validate:"nonzero" json:"name"` + Count uint64 `yaml:"vms_count" validate:"nonzero" json:"vms_count"` + Nodegroup string `yaml:"node_group" validate:"nonzero" json:"node_group"` + FreeCPU uint64 `yaml:"cpu" validate:"nonzero,max=32" json:"cpu"` + FreeMRU uint64 `yaml:"mem" validate:"nonzero,min=256,max=262144" json:"mem"` // min: 256MB, max: 256 GB + SSDDisks []Disk `yaml:"ssd" json:"ssd"` + PublicIP4 bool `yaml:"public_ip4" json:"public_ip4"` + PublicIP6 bool `yaml:"public_ip6" json:"public_ip6"` + Planetary bool `yaml:"planetary" json:"planetary"` + Flist string `yaml:"flist" validate:"nonzero" json:"flist"` + Rootsize uint64 `yaml:"root_size" validate:"max=10240" json:"root_size"` // max 10 TB + Entrypoint string `yaml:"entry_point" validate:"nonzero" json:"entry_point"` + SSHKey string `yaml:"ssh_key" validate:"nonzero" json:"ssh_key"` } type Disk struct { - Size uint64 `yaml:"size" validate:"nonzero,min=15"` // min 15 GB - Mount string `yaml:"mount_point" validate:"nonzero"` + Size uint64 `yaml:"size" validate:"nonzero,min=15" json:"size"` // min 15 GB + Mount string `yaml:"mount_point" validate:"nonzero" json:"mount_point"` } type groupDeploymentsInfo struct { @@ -58,3 +58,12 @@ type vmDeploymentInfo struct { vmName string deploymentName string } + +type vmOutput struct { + Name string + PublicIP4 string + PublicIP6 string + YggIP string + IP string + Mounts []workloads.Mount +}