Skip to content

Commit

Permalink
Command interface improvements
Browse files Browse the repository at this point in the history 
* Introduced temporal solution for Column container. Greenmask has a few transformers with parameters that are lists of objects that point out on the columns of the table. For correct `list-transformers` and `show-transformer` command print the IsColumnContainer property is introduced. This later will rise the case for "Container" parameters introspection.
* Reduced output in list-transformers command. Now it prints only the transformer name, description, and column parameters
* Added `show-transformer` command that prints details about transformer found by name provided
* Fixed list-dump, list-transformers, and restore commands exit code on error
  • Loading branch information
@wwoytenko
wwoytenko committed Feb 9, 2024
1 parent 9e4afa1 commit ea951e8
Show file tree
Hide file tree
Showing 9 changed files with 287 additions and 107 deletions.
4 changes: 2 additions & 2 deletions cmd/greenmask/cmd/list_dump/list_dump.go → cmd/greenmask/cmd/list_dumps/list_dumps.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.

package list_dump
package list_dumps

import (
"context"
Expand Down Expand Up @@ -43,7 +43,7 @@ var (
}

if err := listDumps(); err != nil {
log.Err(err).Msg("")
log.Fatal().Err(err).Msg("")
}
},
}
Expand Down
121 changes: 59 additions & 62 deletions cmd/greenmask/cmd/list_transformers/list_transformers.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,9 +20,9 @@ import (
"fmt"
"os"
"slices"
"strconv"
"strings"

"github.com/greenmaskio/greenmask/pkg/toolkit"
"github.com/olekukonko/tablewriter"
"github.com/rs/zerolog/log"
"github.com/spf13/cobra"
Expand All @@ -42,8 +42,8 @@ var (
log.Err(err).Msg("")
}

if err := run(args); err != nil {
log.Err(err).Msg("")
if err := run(); err != nil {
log.Fatal().Err(err).Msg("")
}
},
}
Expand All @@ -56,19 +56,35 @@ const (
TextFormatName = "text"
)

func run(transformerNames []string) error {
const anyTypesValue = "any"

type parameter struct {
Name string `json:"name,omitempty"`
SupportedTypes []string `json:"supported_types,omitempty"`
}

type jsonResponse struct {
Name string `json:"name,omitempty"`
Description string `json:"description,omitempty"`
Parameters []*parameter `json:"parameters,omitempty"`
}

func run() error {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
err := custom.BootstrapCustomTransformers(ctx, utils.DefaultTransformerRegistry, Config.CustomTransformers)
if err != nil {
return fmt.Errorf("error registering custom transformer: %w", err)
}

// TODO: Consider about listing format. The transformer can have one and more columns as an input
// and

switch format {
case JsonFormatName:
err = listTransformersJson(utils.DefaultTransformerRegistry, transformerNames)
err = listTransformersJson(utils.DefaultTransformerRegistry)
case TextFormatName:
err = listTransformersText(utils.DefaultTransformerRegistry, transformerNames)
err = listTransformersText(utils.DefaultTransformerRegistry)
default:
return fmt.Errorf(`unknown format %s`, format)
}
Expand All @@ -79,92 +95,73 @@ func run(transformerNames []string) error {
return nil
}

func listTransformersJson(registry *utils.TransformerRegistry, transformerNames []string) error {
var transformers []*utils.Definition

if len(transformerNames) > 0 {
func listTransformersJson(registry *utils.TransformerRegistry) error {
var transformers []*jsonResponse

for _, name := range transformerNames {
def, ok := registry.M[name]
if ok {
transformers = append(transformers, def)
} else {
return fmt.Errorf("unknown transformer name \"%s\"", name)
for _, def := range registry.M {
var params []*parameter
for _, p := range def.Parameters {
if !p.IsColumn && !p.IsColumnContainer {
continue
}
supportedTypes := getColumnTypes(p)
params = append(params, &parameter{Name: p.Name, SupportedTypes: supportedTypes})
}

} else {
for _, def := range registry.M {
transformers = append(transformers, def)
}
transformers = append(transformers, &jsonResponse{
Name: def.Properties.Name,
Description: def.Properties.Description,
Parameters: params,
})
}

slices.SortFunc(transformers, func(a, b *jsonResponse) int {
return strings.Compare(a.Name, b.Name)
})

if err := json.NewEncoder(os.Stdout).Encode(transformers); err != nil {
return err
}
return nil
}

func listTransformersText(registry *utils.TransformerRegistry, transformerNames []string) error {
func listTransformersText(registry *utils.TransformerRegistry) error {

var data [][]string
table := tablewriter.NewWriter(os.Stdout)
var names []string
if len(transformerNames) > 0 {
for _, name := range transformerNames {
_, ok := registry.M[name]
if ok {
names = append(names, name)
} else {
return fmt.Errorf("unknown transformer name \"%s\"", name)
}
}

} else {
for name := range registry.M {
names = append(names, name)
}
slices.Sort(names)
for name := range registry.M {
names = append(names, name)
}

slices.Sort(names)
table.SetHeader([]string{"name", "description", "column parameter name", "supported types"})
for _, name := range names {
def := registry.M[name]
data = append(data, []string{def.Properties.Name, "description", def.Properties.Description, "", "", ""})
//allowedTypes := getAllowedTypesList(def)
for _, p := range def.Parameters {
data = append(data, []string{def.Properties.Name, "parameters", p.Name, "description", p.Description, ""})
data = append(data, []string{def.Properties.Name, "parameters", p.Name, "required", strconv.FormatBool(p.Required), ""})
if p.DefaultValue != nil {
data = append(data, []string{def.Properties.Name, "parameters", p.Name, "default", string(p.DefaultValue), ""})
}
if p.LinkParameter != "" {
data = append(data, []string{def.Properties.Name, "parameters", p.Name, "linked_parameter", p.LinkParameter, ""})
}
if p.CastDbType != "" {
data = append(data, []string{def.Properties.Name, "parameters", p.Name, "cast_to_db_type", p.CastDbType, ""})
if !p.IsColumn && !p.IsColumnContainer {
continue
}
if p.ColumnProperties != nil {
if len(p.ColumnProperties.AllowedTypes) > 0 {
allowedTypes := strings.Join(p.ColumnProperties.AllowedTypes, ", ")
data = append(data, []string{def.Properties.Name, "parameters", p.Name, "column_properties", "allowed_types", allowedTypes})
}
isAffected := strconv.FormatBool(p.ColumnProperties.Affected)
data = append(data, []string{def.Properties.Name, "parameters", p.Name, "column_properties", "is_affected", isAffected})
skipOriginalData := strconv.FormatBool(p.ColumnProperties.SkipOriginalData)
data = append(data, []string{def.Properties.Name, "parameters", p.Name, "column_properties", "skip_original_data", skipOriginalData})
skipOnNull := strconv.FormatBool(p.ColumnProperties.SkipOnNull)
data = append(data, []string{def.Properties.Name, "parameters", p.Name, "column_properties", "skip_on_null", skipOnNull})
}

supportedTypes := getColumnTypes(p)
data = append(data, []string{def.Properties.Name, def.Properties.Description, p.Name, strings.Join(supportedTypes, ", ")})
}
}

table.AppendBulk(data)
table.SetRowLine(true)
table.SetAutoMergeCellsByColumnIndex([]int{0, 1, 2, 3})
table.SetAutoMergeCellsByColumnIndex([]int{0, 1})
table.Render()

return nil
}

func getColumnTypes(p *toolkit.Parameter) []string {
if p.ColumnProperties != nil && len(p.ColumnProperties.AllowedTypes) > 0 {
return p.ColumnProperties.AllowedTypes
}
return []string{anyTypesValue}
}

func init() {
Cmd.Flags().StringVarP(&format, "format", "f", TextFormatName, "output format [text|json]")
}
84 changes: 48 additions & 36 deletions cmd/greenmask/cmd/restore/restore.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import (
"path"
"slices"

"github.com/greenmaskio/greenmask/internal/storages"
"github.com/rs/zerolog/log"
"github.com/spf13/cobra"
"github.com/spf13/viper"
Expand All @@ -36,7 +37,6 @@ var (
Args: cobra.ExactArgs(1),
Short: "restore dump with ID or the latest to the target database",
Run: func(cmd *cobra.Command, args []string) {
var dumpId string

if err := logger.SetLogLevel(Config.Log.Level, Config.Log.Format); err != nil {
log.Fatal().Err(err).Msg("fatal")
Expand All @@ -49,41 +49,9 @@ var (
log.Fatal().Err(err).Msg("fatal")
}

if args[0] == "latest" {
var backupNames []string

_, dirs, err := st.ListDir(ctx)
if err != nil {
log.Fatal().Err(err).Msg("cannot walk through directory")
}
for _, dir := range dirs {
exists, err := dir.Exists(ctx, "metadata.json")
if err != nil {
log.Fatal().Err(err).Msg("cannot check file existence")
}
if exists {
backupNames = append(backupNames, dir.Dirname())
}
}

slices.SortFunc(
backupNames, func(a, b string) int {
if a > b {
return -1
}
return 1
},
)
dumpId = backupNames[0]
} else {
dumpId = args[0]
exists, err := st.Exists(ctx, path.Join(dumpId, "metadata.json"))
if err != nil {
log.Fatal().Err(err).Msg("cannot check file existence")
}
if !exists {
log.Fatal().Err(err).Msg("choose another dump is failed")
}
dumpId, err := getDumpId(ctx, st, args[0])
if err != nil {
log.Fatal().Err(err).Msg("")
}

st = st.SubStorage(dumpId, true)
Expand All @@ -104,6 +72,50 @@ var (
Config = pgDomains.NewConfig()
)

func getDumpId(ctx context.Context, st storages.Storager, dumpId string) (string, error) {
if dumpId == "latest" {
var backupNames []string

_, dirs, err := st.ListDir(ctx)
if err != nil {
log.Fatal().Err(err).Msg("cannot walk through directory")
}
for _, dir := range dirs {
exists, err := dir.Exists(ctx, "metadata.json")
if err != nil {
log.Fatal().Err(err).Msg("cannot check file existence")
}
if exists {
backupNames = append(backupNames, dir.Dirname())
}
}

slices.SortFunc(
backupNames, func(a, b string) int {
if a > b {
return -1
}
return 1
},
)
dumpId = backupNames[0]
} else {
exists, err := st.Exists(ctx, path.Join(dumpId, "metadata.json"))
if err != nil {
log.Fatal().
Err(err).
Msg("cannot check file existence")
}
if !exists {
log.Fatal().
Err(err).
Str("DumpId", dumpId).
Msg("dump with provided id is not found")
}
}
return dumpId, nil
}

// TODO: Option that currently does not implemented:
// * data-only
// * exit-on-error
Expand Down
6 changes: 4 additions & 2 deletions cmd/greenmask/cmd/root.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,10 +26,11 @@ import (

"github.com/greenmaskio/greenmask/cmd/greenmask/cmd/delete_backup"
"github.com/greenmaskio/greenmask/cmd/greenmask/cmd/dump"
"github.com/greenmaskio/greenmask/cmd/greenmask/cmd/list_dump"
"github.com/greenmaskio/greenmask/cmd/greenmask/cmd/list_dumps"
"github.com/greenmaskio/greenmask/cmd/greenmask/cmd/list_transformers"
"github.com/greenmaskio/greenmask/cmd/greenmask/cmd/restore"
"github.com/greenmaskio/greenmask/cmd/greenmask/cmd/show_dump"
"github.com/greenmaskio/greenmask/cmd/greenmask/cmd/show_transformer"
"github.com/greenmaskio/greenmask/cmd/greenmask/cmd/validate"
pgDomains "github.com/greenmaskio/greenmask/internal/domains"
configUtils "github.com/greenmaskio/greenmask/internal/utils/config"
Expand Down Expand Up @@ -90,12 +91,13 @@ func init() {
)

RootCmd.AddCommand(dump.Cmd)
RootCmd.AddCommand(list_dump.Cmd)
RootCmd.AddCommand(list_dumps.Cmd)
RootCmd.AddCommand(restore.Cmd)
RootCmd.AddCommand(delete_backup.Cmd)
RootCmd.AddCommand(show_dump.Cmd)
RootCmd.AddCommand(list_transformers.Cmd)
RootCmd.AddCommand(validate.Cmd)
RootCmd.AddCommand(show_transformer.Cmd)

if err := RootCmd.MarkPersistentFlagRequired("config"); err != nil {
log.Fatal().Err(err).Msg("")
Expand Down
Loading

0 comments on commit ea951e8

Please sign in to comment.