From 49055935ded6594bccbc3bf1e04d2da107b9702d Mon Sep 17 00:00:00 2001 From: Vadim Voitenko Date: Mon, 12 Feb 2024 00:19:50 +0200 Subject: [PATCH 01/10] validate command new features and refactoring * Moved command line interface internals into `internal/cmd` * Implemented JsonDocument and JsonPrinter --- internal/db/postgres/{ => cmd}/dump.go | 28 ++-- internal/db/postgres/{ => cmd}/restore.go | 2 +- internal/db/postgres/{ => cmd}/show_dump.go | 2 +- internal/db/postgres/{ => cmd}/validate.go | 147 ++++++++++++------ .../cmd/validate_utils/json_document.go | 85 ++++++++++ .../cmd/validate_utils/json_printer.go | 54 +++++++ .../db/postgres/cmd/validate_utils/printer.go | 1 + .../cmd/validate_utils/text_document.go | 1 + .../cmd/validate_utils/text_printer.go | 1 + .../db/postgres/cmd/validate_utils/utils.go | 61 ++++++++ .../{dump => dump_objects}/entry_producer.go | 2 +- .../{dump => dump_objects}/large_object.go | 2 +- .../{dump => dump_objects}/sequence.go | 2 +- .../postgres/{dump => dump_objects}/table.go | 2 +- 14 files changed, 324 insertions(+), 66 deletions(-) rename internal/db/postgres/{ => cmd}/dump.go (97%) rename internal/db/postgres/{ => cmd}/restore.go (99%) rename internal/db/postgres/{ => cmd}/show_dump.go (99%) rename internal/db/postgres/{ => cmd}/validate.go (84%) create mode 100644 internal/db/postgres/cmd/validate_utils/json_document.go create mode 100644 internal/db/postgres/cmd/validate_utils/json_printer.go create mode 100644 internal/db/postgres/cmd/validate_utils/printer.go create mode 100644 internal/db/postgres/cmd/validate_utils/text_document.go create mode 100644 internal/db/postgres/cmd/validate_utils/text_printer.go create mode 100644 internal/db/postgres/cmd/validate_utils/utils.go rename internal/db/postgres/{dump => dump_objects}/entry_producer.go (97%) rename internal/db/postgres/{dump => dump_objects}/large_object.go (99%) rename internal/db/postgres/{dump => dump_objects}/sequence.go (98%) rename internal/db/postgres/{dump => dump_objects}/table.go (99%) diff --git a/internal/db/postgres/dump.go b/internal/db/postgres/cmd/dump.go similarity index 97% rename from internal/db/postgres/dump.go rename to internal/db/postgres/cmd/dump.go index 720a41e5..2499584f 100644 --- a/internal/db/postgres/dump.go +++ b/internal/db/postgres/cmd/dump.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package postgres +package cmd import ( "bytes" @@ -29,7 +29,7 @@ import ( "golang.org/x/sync/errgroup" runtimeContext "github.com/greenmaskio/greenmask/internal/db/postgres/context" - "github.com/greenmaskio/greenmask/internal/db/postgres/dump" + "github.com/greenmaskio/greenmask/internal/db/postgres/dump_objects" "github.com/greenmaskio/greenmask/internal/db/postgres/dumpers" "github.com/greenmaskio/greenmask/internal/db/postgres/pgdump" storageDto "github.com/greenmaskio/greenmask/internal/db/postgres/storage" @@ -59,7 +59,7 @@ type Dump struct { dumpedObjectSizes map[int32]storageDto.ObjectSizeStat tocFileSize int64 version int - blobs *dump.Blobs + blobs *dump_objects.Blobs // validate shows that dump worker must be in validation mode validate bool } @@ -221,7 +221,7 @@ func (d *Dump) schemaOnlyDump(ctx context.Context, tx pgx.Tx) error { func (d *Dump) dataDump(ctx context.Context) error { // TODO: You should use pointer to dumpers.DumpTask instead tasks := make(chan dumpers.DumpTask, d.pgDumpOptions.Jobs) - result := make(chan dump.Entry, d.pgDumpOptions.Jobs) + result := make(chan dump_objects.Entry, d.pgDumpOptions.Jobs) log.Debug().Msgf("planned %d workers", d.pgDumpOptions.Jobs) eg, gtx := errgroup.WithContext(ctx) @@ -256,11 +256,11 @@ func (d *Dump) dataDump(ctx context.Context) error { dumpObj.SetDumpId(d.dumpIdSequence) var task dumpers.DumpTask switch v := dumpObj.(type) { - case *dump.Table: + case *dump_objects.Table: task = dumpers.NewTableDumper(v, d.validate, d.config.Validate.Diff) - case *dump.Sequence: + case *dump_objects.Sequence: task = dumpers.NewSequenceDumper(v) - case *dump.Blobs: + case *dump_objects.Blobs: d.blobs = v task = dumpers.NewLargeObjectDumper(v) default: @@ -280,7 +280,7 @@ func (d *Dump) dataDump(ctx context.Context) error { func() error { var tables, sequences, largeObjects []*toc.Entry for { - var entry dump.Entry + var entry dump_objects.Entry var ok bool select { case <-gtx.Done(): @@ -299,15 +299,15 @@ func (d *Dump) dataDump(ctx context.Context) error { return fmt.Errorf("error producing toc entry: %w", err) } switch v := entry.(type) { - case *dump.Table: + case *dump_objects.Table: d.dumpedObjectSizes[e.DumpId] = storageDto.ObjectSizeStat{ Original: v.OriginalSize, Compressed: v.CompressedSize, } tables = append(tables, e) - case *dump.Sequence: + case *dump_objects.Sequence: sequences = append(sequences, e) - case *dump.Blobs: + case *dump_objects.Blobs: d.dumpedObjectSizes[e.DumpId] = storageDto.ObjectSizeStat{ Original: v.OriginalSize, Compressed: v.CompressedSize, @@ -506,7 +506,7 @@ func (d *Dump) getWorkerTransaction(ctx context.Context) (*pgx.Conn, pgx.Tx, err } func (d *Dump) dumpWorker( - ctx context.Context, tasks <-chan dumpers.DumpTask, result chan<- dump.Entry, id int, + ctx context.Context, tasks <-chan dumpers.DumpTask, result chan<- dump_objects.Entry, id int, ) error { conn, tx, err := d.getWorkerTransaction(ctx) @@ -574,7 +574,7 @@ func (d *Dump) dumpWorker( } func (d *Dump) validateDumpWorker( - ctx context.Context, tasks <-chan dumpers.DumpTask, result chan<- dump.Entry, id int, + ctx context.Context, tasks <-chan dumpers.DumpTask, result chan<- dump_objects.Entry, id int, ) error { for { @@ -601,7 +601,7 @@ func (d *Dump) validateDumpWorker( Str("ObjectName", task.DebugInfo()). Msgf("dumping started") - entry, err := func() (dump.Entry, error) { + entry, err := func() (dump_objects.Entry, error) { conn, tx, err := d.getWorkerTransaction(ctx) if err != nil { diff --git a/internal/db/postgres/restore.go b/internal/db/postgres/cmd/restore.go similarity index 99% rename from internal/db/postgres/restore.go rename to internal/db/postgres/cmd/restore.go index 85a37127..daa2e899 100644 --- a/internal/db/postgres/restore.go +++ b/internal/db/postgres/cmd/restore.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package postgres +package cmd import ( "bufio" diff --git a/internal/db/postgres/show_dump.go b/internal/db/postgres/cmd/show_dump.go similarity index 99% rename from internal/db/postgres/show_dump.go rename to internal/db/postgres/cmd/show_dump.go index 05f73926..c5370a45 100644 --- a/internal/db/postgres/show_dump.go +++ b/internal/db/postgres/cmd/show_dump.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package postgres +package cmd import ( "context" diff --git a/internal/db/postgres/validate.go b/internal/db/postgres/cmd/validate.go similarity index 84% rename from internal/db/postgres/validate.go rename to internal/db/postgres/cmd/validate.go index 2fb3efec..f8a13ede 100644 --- a/internal/db/postgres/validate.go +++ b/internal/db/postgres/cmd/validate.go @@ -1,4 +1,4 @@ -package postgres +package cmd import ( "bufio" @@ -14,11 +14,12 @@ import ( "strings" "time" + "github.com/greenmaskio/greenmask/internal/db/postgres/cmd/validate_utils" "github.com/olekukonko/tablewriter" "github.com/rs/zerolog/log" runtimeContext "github.com/greenmaskio/greenmask/internal/db/postgres/context" - "github.com/greenmaskio/greenmask/internal/db/postgres/dump" + "github.com/greenmaskio/greenmask/internal/db/postgres/dump_objects" "github.com/greenmaskio/greenmask/internal/db/postgres/pgcopy" "github.com/greenmaskio/greenmask/internal/db/postgres/toc" "github.com/greenmaskio/greenmask/internal/db/postgres/transformers/custom" @@ -34,15 +35,16 @@ import ( const nullStringValue = "NULL" const ( - horizontalFormatName = "horizontal" - verticalFormatName = "vertical" + horizontalTableFormatName = "horizontal" + verticalTableFormatName = "vertical" maxWrapLength = 64 -) -var endOfFileSeq = []byte(`\.`) + jsonFormat string = "json" + textFormat string = "json" +) -type PrintSettings struct { +type printSettings struct { OriginalColors []tablewriter.Colors TransformedColors []tablewriter.Colors HeaderColors []tablewriter.Colors @@ -181,10 +183,10 @@ func (v *Validate) Run(ctx context.Context) error { return nil } - var tablesWithTransformers []dump.Entry + var tablesWithTransformers []dump_objects.Entry for _, item := range v.context.DataSectionObjects { - if t, ok := item.(*dump.Table); ok && len(t.Transformers) > 0 { + if t, ok := item.(*dump_objects.Table); ok && len(t.Transformers) > 0 { t.ValidateLimitedRecords = v.config.Validate.RowsLimit tablesWithTransformers = append(tablesWithTransformers, t) } @@ -196,14 +198,14 @@ func (v *Validate) Run(ctx context.Context) error { } for _, e := range v.dataEntries { - idx := slices.IndexFunc(v.context.DataSectionObjects, func(entry dump.Entry) bool { - t := entry.(*dump.Table) + idx := slices.IndexFunc(v.context.DataSectionObjects, func(entry dump_objects.Entry) bool { + t := entry.(*dump_objects.Table) return t.DumpId == e.DumpId }) - t := v.context.DataSectionObjects[idx].(*dump.Table) + t := v.context.DataSectionObjects[idx].(*dump_objects.Table) - if err = v.prettyPrintTable(ctx, t); err != nil { + if err = v.printText(ctx, t); err != nil { return fmt.Errorf("error pretty printing table \"%s\".\"%s\": %w", t.Table.Schema, t.Table.Name, err) } } @@ -215,7 +217,7 @@ func (v *Validate) getVerticalRowColors(affectedColumns map[int]struct{}, column var colors []tablewriter.Colors var isEqual bool if v.config.Validate.Diff { - isEqual = valuesEqual(original, transformed) + isEqual = validate_utils.ValuesEqual(original, transformed) colors = make([]tablewriter.Colors, 4) } else { colors = make([]tablewriter.Colors, 3) @@ -247,7 +249,7 @@ func (v *Validate) getVerticalRowColors(affectedColumns map[int]struct{}, column return colors, isEqual } -func (v *Validate) getAffectedColumns(t *dump.Table) map[int]struct{} { +func (v *Validate) getAffectedColumns(t *dump_objects.Table) map[int]struct{} { affectedColumns := make(map[int]struct{}) for _, tr := range t.Transformers { ac := tr.GetAffectedColumns() @@ -258,7 +260,7 @@ func (v *Validate) getAffectedColumns(t *dump.Table) map[int]struct{} { return affectedColumns } -func (v *Validate) getHorizontalSettings(t *dump.Table) *PrintSettings { +func (v *Validate) getHorizontalSettings(t *dump_objects.Table) *printSettings { affectedColumns := v.getAffectedColumns(t) originalColumnsColors := make([]tablewriter.Colors, len(t.Columns)) @@ -283,7 +285,7 @@ func (v *Validate) getHorizontalSettings(t *dump.Table) *PrintSettings { transformedColumnsColors = slices.Insert(transformedColumnsColors, 0, tablewriter.Colors{}) columnsAlignments = slices.Insert(columnsAlignments, 0, tablewriter.ALIGN_LEFT) - return &PrintSettings{ + return &printSettings{ OriginalColors: originalColumnsColors, TransformedColors: transformedColumnsColors, HeaderColors: headerColors, @@ -292,7 +294,7 @@ func (v *Validate) getHorizontalSettings(t *dump.Table) *PrintSettings { } -func (v *Validate) printHorizontally(ctx context.Context, t *dump.Table) error { +func (v *Validate) printHorizontally(ctx context.Context, t *dump_objects.Table) error { settings := v.getHorizontalSettings(t) prettyWriter := tablewriter.NewWriter(os.Stdout) @@ -326,8 +328,8 @@ func (v *Validate) printHorizontally(ctx context.Context, t *dump.Table) error { return fmt.Errorf("unable to read line: %w", err) } - // Handle end of dump file seq - if lineIsEndOfData(line) { + // Handle end of dump_objects file seq + if validate_utils.LineIsEndOfData(line) { break } @@ -374,7 +376,7 @@ func (v *Validate) printHorizontally(ctx context.Context, t *dump.Table) error { return nil } -func (v *Validate) printHorizontallyWithDiff(ctx context.Context, t *dump.Table) error { +func (v *Validate) printHorizontallyWithDiff(ctx context.Context, t *dump_objects.Table) error { settings := v.getHorizontalSettings(t) prettyWriter := tablewriter.NewWriter(os.Stdout) @@ -411,8 +413,8 @@ func (v *Validate) printHorizontallyWithDiff(ctx context.Context, t *dump.Table) } return fmt.Errorf("unable to read line: %w", err) } - // Handle end of dump file seq - if lineIsEndOfData(originalLine) { + // Handle end of dump_objects file seq + if validate_utils.LineIsEndOfData(originalLine) { break } @@ -435,8 +437,8 @@ func (v *Validate) printHorizontallyWithDiff(ctx context.Context, t *dump.Table) } return fmt.Errorf("unable to read line: %w", err) } - // Handle end of dump file seq - if lineIsEndOfData(transformedLine) { + // Handle end of dump_objects file seq + if validate_utils.LineIsEndOfData(transformedLine) { break } if err = transformedRow.Decode(transformedLine); err != nil { @@ -472,7 +474,7 @@ func (v *Validate) printHorizontallyWithDiff(ctx context.Context, t *dump.Table) if idx == 2 { log.Debug().Msg("") } - if !valuesEqual(originalValue, transformedValue) { + if !validate_utils.ValuesEqual(originalValue, transformedValue) { originalRecordColors[idx] = tablewriter.Colors{tablewriter.FgHiGreenColor} transformedRecordColors[idx] = tablewriter.Colors{tablewriter.FgHiRedColor} realAffectedColumns[idx] = struct{}{} @@ -518,7 +520,7 @@ func (v *Validate) printHorizontallyWithDiff(ctx context.Context, t *dump.Table) return nil } -func (v *Validate) getVerticalHeaderColors(t *dump.Table, affectedColumns map[int]struct{}) []tablewriter.Colors { +func (v *Validate) getVerticalHeaderColors(t *dump_objects.Table, affectedColumns map[int]struct{}) []tablewriter.Colors { headerColors := make([]tablewriter.Colors, len(t.Columns)) for idx := range t.Columns { if _, ok := affectedColumns[idx]; ok { @@ -532,7 +534,7 @@ func (v *Validate) getVerticalHeaderColors(t *dump.Table, affectedColumns map[in return headerColors } -func (v *Validate) printVertically(ctx context.Context, t *dump.Table) error { +func (v *Validate) printVertically(ctx context.Context, t *dump_objects.Table) error { var recordSize = 3 if v.config.Validate.Diff { @@ -587,8 +589,8 @@ func (v *Validate) printVertically(ctx context.Context, t *dump.Table) error { } return fmt.Errorf("unable to read line: %w", err) } - // Handle end of dump file seq - if lineIsEndOfData(originalLine) { + // Handle end of dump_objects file seq + if validate_utils.LineIsEndOfData(originalLine) { break } @@ -611,8 +613,8 @@ func (v *Validate) printVertically(ctx context.Context, t *dump.Table) error { } return fmt.Errorf("unable to read line: %w", err) } - // Handle end of dump file seq - if lineIsEndOfData(transformedLine) { + // Handle end of dump_objects file seq + if validate_utils.LineIsEndOfData(transformedLine) { break } if err = transformedRow.Decode(transformedLine); err != nil { @@ -673,26 +675,79 @@ func (v *Validate) printVertically(ctx context.Context, t *dump.Table) error { return nil } -func (v *Validate) prettyPrintTable(ctx context.Context, t *dump.Table) error { - switch v.config.Validate.Format { - case horizontalFormatName: +func (v *Validate) print(ctx context.Context, t *dump_objects.Table, format string, withDiff bool) error { + row := *pgcopy.NewRow(len(t.Columns)) + tableData, err := v.st.GetObject(ctx, fmt.Sprintf("%d.dat.gz", t.DumpId)) + if err != nil { + log.Err(err).Msg("") + } + defer tableData.Close() + gz, err := gzip.NewReader(tableData) + if err != nil { + return fmt.Errorf("cannot create gzip reader: %w", err) + } + defer gz.Close() + r := bufio.NewReader(gz) + + var lineNum = 1 + realAffectedColumns := v.getAffectedColumns(t) + diffValues := make([][]*toolkit.RawValue, len(t.Columns)) + for idx := range t.Columns { + diffValues[idx] = make([]*toolkit.RawValue, 2) + } + for { + line, err := reader.ReadLine(r) + if err != nil { + if errors.Is(err, io.EOF) { + break + } + return fmt.Errorf("unable to read line: %w", err) + } + + // Handle end of dump_objects file seq + if validate_utils.LineIsEndOfData(line) { + break + } + + if err = row.Decode(line); err != nil { + return fmt.Errorf("error decoding copy line: %w", err) + } + record := make([]string, len(t.Columns)) + for idx, c := range t.Columns { + value, err := row.GetColumn(idx) + if err != nil { + return fmt.Errorf("unable to get column \"%s\" value: %w", c.Name, err) + } + if value.IsNull { + record[idx] = nullStringValue + } else { + record[idx] = stringsUtils.WrapString(string(value.Data), maxWrapLength) + } + } + + record = slices.Insert(record, 0, fmt.Sprintf("%d", lineNum)) + + lineNum++ + } + + header := make([]string, len(t.Columns)) + for idx, c := range t.Columns { + header[idx] = c.Name + } + return nil +} + +func (v *Validate) printText(ctx context.Context, t *dump_objects.Table) error { + switch v.config.Validate.TableFormat { + case horizontalTableFormatName: if v.config.Validate.Diff { return v.printHorizontallyWithDiff(ctx, t) } else { return v.printHorizontally(ctx, t) } - case verticalFormatName: + case verticalTableFormatName: return v.printVertically(ctx, t) default: - return fmt.Errorf("unknwon data format \"%s\"", v.config.Validate.Format) + return fmt.Errorf("unknwon data format \"%s\"", v.config.Validate.TableFormat) } - -} - -func lineIsEndOfData(line []byte) bool { - return len(endOfFileSeq) == len(line) && line[0] == '\\' && line[1] == '.' -} - -func valuesEqual(a *toolkit.RawValue, b *toolkit.RawValue) bool { - return a.IsNull == b.IsNull && slices.Equal(a.Data, b.Data) } diff --git a/internal/db/postgres/cmd/validate_utils/json_document.go b/internal/db/postgres/cmd/validate_utils/json_document.go new file mode 100644 index 00000000..c27e2d15 --- /dev/null +++ b/internal/db/postgres/cmd/validate_utils/json_document.go @@ -0,0 +1,85 @@ +package validate_utils + +import ( + "encoding/json" + "fmt" + + "github.com/greenmaskio/greenmask/internal/db/postgres/dump_objects" + "github.com/greenmaskio/greenmask/internal/db/postgres/pgcopy" + "github.com/greenmaskio/greenmask/pkg/toolkit" +) + +type Documenter interface { + Append(original, transformed *pgcopy.Row) error + Data() ([]byte, error) +} + +type values struct { + Original string `json:"original,omitempty"` + Transformed string `json:"transformed,omitempty"` + Changed bool `json:"changed,omitempty"` + Implicit bool `json:"implicit,omitempty"` +} + +type jsonDocument struct { + Schema string `json:"schema"` + Name string `json:"name"` + PrimaryKeyColumns []string `json:"primary_key_columns,omitempty"` + Records []jsonRecord `json:"records,omitempty"` +} + +type jsonRecord map[string]*values + +type JsonDocument struct { + result *jsonDocument + table *dump_objects.Table + withDiff bool + expectedAffectedColumns map[int]struct{} + pkColumns map[int]*toolkit.Column +} + +func NewJsonDocument(table *dump_objects.Table, withDiff bool) *JsonDocument { + pkColumns := getPrimaryKeyConstraintColumns(table) + expectedAffectedColumns := getAffectedColumns(table) + var pkColumnsList []string + for _, c := range pkColumns { + pkColumnsList = append(pkColumnsList, c.Name) + } + + return &JsonDocument{ + result: &jsonDocument{ + PrimaryKeyColumns: pkColumnsList, + Records: make([]jsonRecord, 0), + }, + withDiff: withDiff, + table: table, + pkColumns: pkColumns, + expectedAffectedColumns: expectedAffectedColumns, + } +} + +func (jc *JsonDocument) Append(original, transformed *pgcopy.Row) error { + r := make(jsonRecord) + for idx, c := range jc.table.Columns { + originalRawValue, err := original.GetColumn(idx) + if err != nil { + return fmt.Errorf("error getting column from original record: %w", err) + } + transformedRawValue, err := transformed.GetColumn(idx) + if err != nil { + return fmt.Errorf("error getting column from transformed record: %w", err) + } + + r[c.Name] = &values{ + Original: getStringFromRawValue(originalRawValue), + Transformed: getStringFromRawValue(transformedRawValue), + Changed: ValuesEqual(originalRawValue, transformedRawValue), + } + jc.result.Records = append(jc.result.Records) + } + return nil +} + +func (jc *JsonDocument) Data() ([]byte, error) { + return json.Marshal(jc.result) +} diff --git a/internal/db/postgres/cmd/validate_utils/json_printer.go b/internal/db/postgres/cmd/validate_utils/json_printer.go new file mode 100644 index 00000000..b56e5ed6 --- /dev/null +++ b/internal/db/postgres/cmd/validate_utils/json_printer.go @@ -0,0 +1,54 @@ +package validate_utils + +import ( + "fmt" + "io" +) + +type Printer interface { + PrintDocument(w io.Writer, d Documenter) error + Complete(w io.Writer) error +} + +type JsonPrinter struct { + firstLinePrinted bool +} + +func NewJsonPrinter() *JsonPrinter { + return &JsonPrinter{} +} + +func (jp *JsonPrinter) PrintDocument(w io.Writer, d Documenter) error { + if !jp.firstLinePrinted { + _, err := w.Write([]byte("[")) + if err != nil { + return fmt.Errorf("error printing intial line: %w", err) + } + } + _, err := w.Write([]byte(",")) + if err != nil { + return fmt.Errorf("error printing list separator: %w", err) + } + data, err := d.Data() + if err != nil { + return fmt.Errorf("error getting document data: %w", err) + } + _, err = w.Write(data) + if err != nil { + return fmt.Errorf("error printing document data: %w", err) + } + return nil +} + +func (jp *JsonPrinter) Complete(w io.Writer) error { + // Print the list closing bracket "]" + seq := []byte("]") + if !jp.firstLinePrinted { + // If there was not any document that we print empty document + seq = []byte("[]") + } + if _, err := w.Write(seq); err != nil { + return err + } + return nil +} diff --git a/internal/db/postgres/cmd/validate_utils/printer.go b/internal/db/postgres/cmd/validate_utils/printer.go new file mode 100644 index 00000000..147e532c --- /dev/null +++ b/internal/db/postgres/cmd/validate_utils/printer.go @@ -0,0 +1 @@ +package validate_utils diff --git a/internal/db/postgres/cmd/validate_utils/text_document.go b/internal/db/postgres/cmd/validate_utils/text_document.go new file mode 100644 index 00000000..147e532c --- /dev/null +++ b/internal/db/postgres/cmd/validate_utils/text_document.go @@ -0,0 +1 @@ +package validate_utils diff --git a/internal/db/postgres/cmd/validate_utils/text_printer.go b/internal/db/postgres/cmd/validate_utils/text_printer.go new file mode 100644 index 00000000..147e532c --- /dev/null +++ b/internal/db/postgres/cmd/validate_utils/text_printer.go @@ -0,0 +1 @@ +package validate_utils diff --git a/internal/db/postgres/cmd/validate_utils/utils.go b/internal/db/postgres/cmd/validate_utils/utils.go new file mode 100644 index 00000000..efad645c --- /dev/null +++ b/internal/db/postgres/cmd/validate_utils/utils.go @@ -0,0 +1,61 @@ +package validate_utils + +import ( + "slices" + + "github.com/greenmaskio/greenmask/internal/db/postgres/dump_objects" + "github.com/greenmaskio/greenmask/pkg/toolkit" +) + +var endOfFileSeq = []byte(`\.`) + +const nullStringValue = "NULL" + +func getAffectedColumns(t *dump_objects.Table) map[int]struct{} { + affectedColumns := make(map[int]struct{}) + for _, tr := range t.Transformers { + ac := tr.GetAffectedColumns() + for idx := range ac { + affectedColumns[idx] = struct{}{} + } + } + return affectedColumns +} + +func LineIsEndOfData(line []byte) bool { + return len(endOfFileSeq) == len(line) && line[0] == '\\' && line[1] == '.' +} + +func ValuesEqual(a *toolkit.RawValue, b *toolkit.RawValue) bool { + return a.IsNull == b.IsNull && slices.Equal(a.Data, b.Data) +} + +func getPrimaryKeyConstraintColumns(t *dump_objects.Table) map[int]*toolkit.Column { + idx := slices.IndexFunc(t.Constraints, func(constraint toolkit.Constraint) bool { + return constraint.Type() == toolkit.PkConstraintType + }) + if idx == -1 { + return nil + } + pk := t.Constraints[idx].(*toolkit.PrimaryKey) + + columns := make(map[int]*toolkit.Column, len(pk.Columns)) + + for _, attNum := range pk.Columns { + columnIdx := slices.IndexFunc(t.Columns, func(column *toolkit.Column) bool { + return column.Num == attNum + }) + if columnIdx == -1 { + panic("unable to find column by attnum") + } + columns[columnIdx] = t.Columns[columnIdx] + } + return columns +} + +func getStringFromRawValue(v *toolkit.RawValue) string { + if v.IsNull { + return nullStringValue + } + return string(v.Data) +} diff --git a/internal/db/postgres/dump/entry_producer.go b/internal/db/postgres/dump_objects/entry_producer.go similarity index 97% rename from internal/db/postgres/dump/entry_producer.go rename to internal/db/postgres/dump_objects/entry_producer.go index 6ce19014..2f7dce13 100644 --- a/internal/db/postgres/dump/entry_producer.go +++ b/internal/db/postgres/dump_objects/entry_producer.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package dump +package dump_objects import "github.com/greenmaskio/greenmask/internal/db/postgres/toc" diff --git a/internal/db/postgres/dump/large_object.go b/internal/db/postgres/dump_objects/large_object.go similarity index 99% rename from internal/db/postgres/dump/large_object.go rename to internal/db/postgres/dump_objects/large_object.go index 938f53e8..6f7ccd35 100644 --- a/internal/db/postgres/dump/large_object.go +++ b/internal/db/postgres/dump_objects/large_object.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package dump +package dump_objects import ( "fmt" diff --git a/internal/db/postgres/dump/sequence.go b/internal/db/postgres/dump_objects/sequence.go similarity index 98% rename from internal/db/postgres/dump/sequence.go rename to internal/db/postgres/dump_objects/sequence.go index d61eeb80..57c5ecd6 100644 --- a/internal/db/postgres/dump/sequence.go +++ b/internal/db/postgres/dump_objects/sequence.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package dump +package dump_objects import ( "fmt" diff --git a/internal/db/postgres/dump/table.go b/internal/db/postgres/dump_objects/table.go similarity index 99% rename from internal/db/postgres/dump/table.go rename to internal/db/postgres/dump_objects/table.go index de1358a9..638432c8 100644 --- a/internal/db/postgres/dump/table.go +++ b/internal/db/postgres/dump_objects/table.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package dump +package dump_objects import ( "errors" From b51f65c58759a152554025eff6eb7ea639bf550a Mon Sep 17 00:00:00 2001 From: Vadim Voitenko Date: Thu, 15 Feb 2024 18:46:46 +0200 Subject: [PATCH 02/10] Saved changes. Implemented base json document for diff print --- internal/db/postgres/cmd/show_dump.go | 4 +- .../cmd/validate_utils/json_document.go | 97 +++++++++++++++---- .../cmd/validate_utils/json_printer.go | 4 +- .../db/postgres/cmd/validate_utils/utils.go | 8 +- internal/db/postgres/context/context.go | 6 +- internal/domains/config.go | 1 + 6 files changed, 90 insertions(+), 30 deletions(-) diff --git a/internal/db/postgres/cmd/show_dump.go b/internal/db/postgres/cmd/show_dump.go index c5370a45..a28aff3e 100644 --- a/internal/db/postgres/cmd/show_dump.go +++ b/internal/db/postgres/cmd/show_dump.go @@ -38,12 +38,12 @@ const ( ) var templateString = `; -; Archive created at {{ .Header.CreationDate.Format "2006-01-02 15:04:05 UTC" }} +; Archive created at {{ .Header.CreationDate.TableFormat "2006-01-02 15:04:05 UTC" }} ; dbname: {{ .Header.DbName }} ; TOC Entries: {{ .Header.TocEntriesCount }} ; Compression: {{ .Header.Compression }} ; Dump Version: {{ .Header.DumpVersion }} -; Format: DIRECTORY +; TableFormat: DIRECTORY ; Integer: {{ .Header.Integer }} bytes ; Offset: {{ .Header.Offset }} bytes ; Dumped from database version: {{ .Header.DumpedFrom }} diff --git a/internal/db/postgres/cmd/validate_utils/json_document.go b/internal/db/postgres/cmd/validate_utils/json_document.go index c27e2d15..cc4c0a10 100644 --- a/internal/db/postgres/cmd/validate_utils/json_document.go +++ b/internal/db/postgres/cmd/validate_utils/json_document.go @@ -3,25 +3,22 @@ package validate_utils import ( "encoding/json" "fmt" + "maps" "github.com/greenmaskio/greenmask/internal/db/postgres/dump_objects" "github.com/greenmaskio/greenmask/internal/db/postgres/pgcopy" "github.com/greenmaskio/greenmask/pkg/toolkit" ) -type Documenter interface { - Append(original, transformed *pgcopy.Row) error - Data() ([]byte, error) -} - type values struct { + ColNum int `json:"-"` Original string `json:"original,omitempty"` Transformed string `json:"transformed,omitempty"` Changed bool `json:"changed,omitempty"` Implicit bool `json:"implicit,omitempty"` } -type jsonDocument struct { +type JsonDocumentResult struct { Schema string `json:"schema"` Name string `json:"name"` PrimaryKeyColumns []string `json:"primary_key_columns,omitempty"` @@ -31,14 +28,16 @@ type jsonDocument struct { type jsonRecord map[string]*values type JsonDocument struct { - result *jsonDocument - table *dump_objects.Table - withDiff bool - expectedAffectedColumns map[int]struct{} - pkColumns map[int]*toolkit.Column + result *JsonDocumentResult + table *dump_objects.Table + withDiff bool + expectedAffectedColumns map[string]struct{} + unexpectedAffectedColumns map[string]struct{} + pkColumns map[int]*toolkit.Column + onlyTransformed bool } -func NewJsonDocument(table *dump_objects.Table, withDiff bool) *JsonDocument { +func NewJsonDocument(table *dump_objects.Table, withDiff bool, onlyTransformed bool) *JsonDocument { pkColumns := getPrimaryKeyConstraintColumns(table) expectedAffectedColumns := getAffectedColumns(table) var pkColumnsList []string @@ -47,14 +46,16 @@ func NewJsonDocument(table *dump_objects.Table, withDiff bool) *JsonDocument { } return &JsonDocument{ - result: &jsonDocument{ + result: &JsonDocumentResult{ PrimaryKeyColumns: pkColumnsList, Records: make([]jsonRecord, 0), }, - withDiff: withDiff, - table: table, - pkColumns: pkColumns, - expectedAffectedColumns: expectedAffectedColumns, + withDiff: withDiff, + table: table, + pkColumns: pkColumns, + expectedAffectedColumns: expectedAffectedColumns, + unexpectedAffectedColumns: make(map[string]struct{}), + onlyTransformed: onlyTransformed, } } @@ -70,16 +71,74 @@ func (jc *JsonDocument) Append(original, transformed *pgcopy.Row) error { return fmt.Errorf("error getting column from transformed record: %w", err) } + changed := ValuesEqual(originalRawValue, transformedRawValue) + var implicit bool + if _, ok := jc.expectedAffectedColumns[c.Name]; changed && !ok { + implicit = false + jc.unexpectedAffectedColumns[c.Name] = struct{}{} + } + r[c.Name] = &values{ Original: getStringFromRawValue(originalRawValue), Transformed: getStringFromRawValue(transformedRawValue), - Changed: ValuesEqual(originalRawValue, transformedRawValue), + Changed: changed, + Implicit: implicit, + ColNum: idx, } jc.result.Records = append(jc.result.Records) } return nil } -func (jc *JsonDocument) Data() ([]byte, error) { +func (jc *JsonDocument) GetAffectedColumns() map[string]struct{} { + if jc.onlyTransformed { + columnsToPrint := maps.Clone(jc.expectedAffectedColumns) + maps.Copy(columnsToPrint, jc.unexpectedAffectedColumns) + for _, colName := range jc.result.PrimaryKeyColumns { + columnsToPrint[colName] = struct{}{} + } + return columnsToPrint + } + + columnsToPrint := make(map[string]struct{}, len(jc.table.Columns)) + for _, c := range jc.table.Columns { + columnsToPrint[c.Name] = struct{}{} + } + return columnsToPrint +} + +func (jc *JsonDocument) GetRecords() *JsonDocumentResult { + if jc.onlyTransformed { + jc.filterColumns() + } + return jc.result +} + +func (jc *JsonDocument) Marshal() ([]byte, error) { + // TODO: + // 1. Return all columns if requested + // 2. Return only transformed data. + // 2.1 Analyze affected columns + unexpectedly affected + // 2.2 Return all affected columns with data + primary key + if jc.onlyTransformed { + jc.filterColumns() + } return json.Marshal(jc.result) } + +func (jc *JsonDocument) filterColumns() { + // Determine list of the affected columns + columnsToPrint := jc.GetAffectedColumns() + columnsToDelete := make(map[string]struct{}) + for _, c := range jc.table.Columns { + if _, ok := columnsToPrint[c.Name]; !ok { + columnsToDelete[c.Name] = struct{}{} + } + } + + for _, r := range jc.result.Records { + for name := range columnsToDelete { + delete(r, name) + } + } +} diff --git a/internal/db/postgres/cmd/validate_utils/json_printer.go b/internal/db/postgres/cmd/validate_utils/json_printer.go index b56e5ed6..5c293725 100644 --- a/internal/db/postgres/cmd/validate_utils/json_printer.go +++ b/internal/db/postgres/cmd/validate_utils/json_printer.go @@ -6,7 +6,7 @@ import ( ) type Printer interface { - PrintDocument(w io.Writer, d Documenter) error + Print(w io.Writer, d Documenter) error Complete(w io.Writer) error } @@ -18,7 +18,7 @@ func NewJsonPrinter() *JsonPrinter { return &JsonPrinter{} } -func (jp *JsonPrinter) PrintDocument(w io.Writer, d Documenter) error { +func (jp *JsonPrinter) Print(w io.Writer, d Documenter) error { if !jp.firstLinePrinted { _, err := w.Write([]byte("[")) if err != nil { diff --git a/internal/db/postgres/cmd/validate_utils/utils.go b/internal/db/postgres/cmd/validate_utils/utils.go index efad645c..06b64c40 100644 --- a/internal/db/postgres/cmd/validate_utils/utils.go +++ b/internal/db/postgres/cmd/validate_utils/utils.go @@ -11,12 +11,12 @@ var endOfFileSeq = []byte(`\.`) const nullStringValue = "NULL" -func getAffectedColumns(t *dump_objects.Table) map[int]struct{} { - affectedColumns := make(map[int]struct{}) +func getAffectedColumns(t *dump_objects.Table) map[string]struct{} { + affectedColumns := make(map[string]struct{}) for _, tr := range t.Transformers { ac := tr.GetAffectedColumns() - for idx := range ac { - affectedColumns[idx] = struct{}{} + for _, name := range ac { + affectedColumns[name] = struct{}{} } } return affectedColumns diff --git a/internal/db/postgres/context/context.go b/internal/db/postgres/context/context.go index 9a5364f7..7f72bdc0 100644 --- a/internal/db/postgres/context/context.go +++ b/internal/db/postgres/context/context.go @@ -21,7 +21,7 @@ import ( "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgtype" - "github.com/greenmaskio/greenmask/internal/db/postgres/dump" + "github.com/greenmaskio/greenmask/internal/db/postgres/dump_objects" "github.com/greenmaskio/greenmask/internal/db/postgres/pgdump" transformersUtils "github.com/greenmaskio/greenmask/internal/db/postgres/transformers/utils" "github.com/greenmaskio/greenmask/internal/domains" @@ -31,11 +31,11 @@ import ( // RuntimeContext - describes current runtime behaviour according to the config and schema objects type RuntimeContext struct { // Tables - map of build tables with toolkit that was wrapped into dump.Entry - Tables map[toolkit.Oid]*dump.Table + Tables map[toolkit.Oid]*dump_objects.Table // Types - list of custom types that are used in DB schema Types []*toolkit.Type // DataSectionObjects - list of objects to dump in data-section. There are sequences, tables and large objects - DataSectionObjects []dump.Entry + DataSectionObjects []dump_objects.Entry // Warnings - list of occurred ValidationWarning during validation and config building Warnings toolkit.ValidationWarnings // Registry - registry of all the registered transformers definition diff --git a/internal/domains/config.go b/internal/domains/config.go index ec6160b6..5acc53d2 100644 --- a/internal/domains/config.go +++ b/internal/domains/config.go @@ -59,6 +59,7 @@ type Validate struct { Diff bool `mapstructure:"diff" yaml:"diff" json:"diff,omitempty"` RowsLimit uint64 `mapstructure:"rows_limit" yaml:"rows_limit" json:"rows_limit,omitempty"` ResolvedWarnings []string `mapstructure:"resolved_warnings" yaml:"resolved_warnings" json:"resolved_warnings,omitempty"` + TableFormat string `mapstructure:"table_format" yaml:"table_format" json:"table_format,omitempty"` Format string `mapstructure:"format" yaml:"format" json:"format,omitempty"` } From bea7dca511a4bf49f3a107f63c23e7efe9138040 Mon Sep 17 00:00:00 2001 From: Vadim Voitenko Date: Fri, 16 Feb 2024 21:17:52 +0200 Subject: [PATCH 03/10] Saved changes. Added json_document.go tests --- cmd/greenmask/cmd/validate/validate.go | 1 - .../cmd/validate_utils/json_document.go | 28 +-- .../cmd/validate_utils/json_document_test.go | 183 ++++++++++++++++++ .../cmd/validate_utils/json_printer.go | 99 +++++----- internal/db/postgres/dumpers/dumpers.go | 4 +- pkg/toolkit/constraints.go | 25 +++ 6 files changed, 273 insertions(+), 67 deletions(-) create mode 100644 internal/db/postgres/cmd/validate_utils/json_document_test.go diff --git a/cmd/greenmask/cmd/validate/validate.go b/cmd/greenmask/cmd/validate/validate.go index 8ee42c27..ea84574b 100644 --- a/cmd/greenmask/cmd/validate/validate.go +++ b/cmd/greenmask/cmd/validate/validate.go @@ -21,7 +21,6 @@ import ( "github.com/spf13/cobra" "github.com/spf13/viper" - "github.com/greenmaskio/greenmask/internal/db/postgres" "github.com/greenmaskio/greenmask/internal/db/postgres/transformers/utils" "github.com/greenmaskio/greenmask/internal/domains" "github.com/greenmaskio/greenmask/internal/utils/logger" diff --git a/internal/db/postgres/cmd/validate_utils/json_document.go b/internal/db/postgres/cmd/validate_utils/json_document.go index cc4c0a10..7ee6b300 100644 --- a/internal/db/postgres/cmd/validate_utils/json_document.go +++ b/internal/db/postgres/cmd/validate_utils/json_document.go @@ -14,8 +14,8 @@ type values struct { ColNum int `json:"-"` Original string `json:"original,omitempty"` Transformed string `json:"transformed,omitempty"` - Changed bool `json:"changed,omitempty"` - Implicit bool `json:"implicit,omitempty"` + Equal bool `json:"equal,omitempty"` + Expected bool `json:"implicit,omitempty"` } type JsonDocumentResult struct { @@ -71,26 +71,30 @@ func (jc *JsonDocument) Append(original, transformed *pgcopy.Row) error { return fmt.Errorf("error getting column from transformed record: %w", err) } - changed := ValuesEqual(originalRawValue, transformedRawValue) - var implicit bool - if _, ok := jc.expectedAffectedColumns[c.Name]; changed && !ok { - implicit = false + equal := ValuesEqual(originalRawValue, transformedRawValue) + expected := true + if _, ok := jc.expectedAffectedColumns[c.Name]; !equal && !ok { + expected = false jc.unexpectedAffectedColumns[c.Name] = struct{}{} } r[c.Name] = &values{ Original: getStringFromRawValue(originalRawValue), Transformed: getStringFromRawValue(transformedRawValue), - Changed: changed, - Implicit: implicit, + Equal: equal, + Expected: expected, ColNum: idx, } - jc.result.Records = append(jc.result.Records) } + jc.result.Records = append(jc.result.Records, r) return nil } -func (jc *JsonDocument) GetAffectedColumns() map[string]struct{} { +func (jc *JsonDocument) GetImplicitlyChangedColumns() map[string]struct{} { + return jc.unexpectedAffectedColumns +} + +func (jc *JsonDocument) GetColumnsToPrint() map[string]struct{} { if jc.onlyTransformed { columnsToPrint := maps.Clone(jc.expectedAffectedColumns) maps.Copy(columnsToPrint, jc.unexpectedAffectedColumns) @@ -107,7 +111,7 @@ func (jc *JsonDocument) GetAffectedColumns() map[string]struct{} { return columnsToPrint } -func (jc *JsonDocument) GetRecords() *JsonDocumentResult { +func (jc *JsonDocument) Get() *JsonDocumentResult { if jc.onlyTransformed { jc.filterColumns() } @@ -128,7 +132,7 @@ func (jc *JsonDocument) Marshal() ([]byte, error) { func (jc *JsonDocument) filterColumns() { // Determine list of the affected columns - columnsToPrint := jc.GetAffectedColumns() + columnsToPrint := jc.GetColumnsToPrint() columnsToDelete := make(map[string]struct{}) for _, c := range jc.table.Columns { if _, ok := columnsToPrint[c.Name]; !ok { diff --git a/internal/db/postgres/cmd/validate_utils/json_document_test.go b/internal/db/postgres/cmd/validate_utils/json_document_test.go new file mode 100644 index 00000000..9d2ea970 --- /dev/null +++ b/internal/db/postgres/cmd/validate_utils/json_document_test.go @@ -0,0 +1,183 @@ +package validate_utils + +import ( + "context" + "encoding/json" + "testing" + + "github.com/greenmaskio/greenmask/internal/db/postgres/dump_objects" + "github.com/greenmaskio/greenmask/internal/db/postgres/pgcopy" + "github.com/greenmaskio/greenmask/internal/db/postgres/transformers/utils" + "github.com/greenmaskio/greenmask/pkg/toolkit" + "github.com/rs/zerolog/log" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type testTransformer struct{} + +func (tt *testTransformer) Init(ctx context.Context) error { + return nil +} + +func (tt *testTransformer) Done(ctx context.Context) error { + return nil +} + +func (tt *testTransformer) Transform(ctx context.Context, r *toolkit.Record) (*toolkit.Record, error) { + return nil, nil +} + +func (tt *testTransformer) GetAffectedColumns() map[int]string { + return map[int]string{ + 1: "name", + } +} + +func TestJsonDocument_GetAffectedColumns(t *testing.T) { + tab, _, _ := getTableAndRows() + jd := NewJsonDocument(tab, true, true) + colsToPrint := jd.GetColumnsToPrint() + _, ok := colsToPrint["departmentid"] + assert.True(t, ok) + _, ok = colsToPrint["name"] + assert.True(t, ok) +} + +func TestJsonDocument_GetRecords(t *testing.T) { + tab, originalRecs, transformedRecs := getTableAndRows() + log.Warn().Any("1", t) + log.Warn().Any("1", originalRecs) + log.Warn().Any("1", transformedRecs) + + original := pgcopy.NewRow(4) + transformed := pgcopy.NewRow(4) + + jd := NewJsonDocument(tab, true, true) + + for idx := range originalRecs { + err := original.Decode(originalRecs[idx]) + require.NoErrorf(t, err, "error at %d line", idx) + err = transformed.Decode(transformedRecs[idx]) + require.NoErrorf(t, err, "error at %d line", idx) + err = jd.Append(original, transformed) + require.NoErrorf(t, err, "error at %d line", idx) + } + columnsToPrint := jd.GetColumnsToPrint() + assert.Equal(t, columnsToPrint, map[string]struct{}{"departmentid": {}, "name": {}, "modifieddate": {}}) + + columnsImplicitlyChanged := jd.GetImplicitlyChangedColumns() + assert.Equal(t, columnsImplicitlyChanged, map[string]struct{}{"modifieddate": {}}) + + result := jd.Get() + require.Len(t, result.Records, 6) + + //log.Warn().Any("a", records) + + //driver, err := toolkit.NewDriver(table, nil, nil) + //if err != nil { + // panic(err.Error()) + //} + //row := pgcopy.NewRow(1) + //_ = row.Decode([]byte(value)) + //r := toolkit.NewRecord( + // driver, + //) + //r.SetRow(row) +} + +func getTableAndRows() (table *dump_objects.Table, original, transformed [][]byte) { + + tableDef := ` + { + "schema": "humanresources", + "name": "department", + "oid": 16526, + "columns": [ + { + "name": "departmentid", + "type_name": "integer", + "type_oid": 23, + "num": 1, + "not_null": true, + "length": -1 + }, + { + "name": "name", + "type_name": "\"Name\"", + "type_oid": 16426, + "num": 2, + "not_null": true, + "length": -1 + }, + { + "name": "groupname", + "type_name": "\"Name\"", + "type_oid": 16426, + "num": 3, + "not_null": true, + "length": -1 + }, + { + "name": "modifieddate", + "type_name": "timestamp without time zone", + "type_oid": 1114, + "num": 4, + "not_null": true, + "length": -1 + } + ] + } + ` + + colnstraintDef := ` + { + "schema": "humanresources", + "name": "PK_Department_DepartmentID", + "oid": 17387, + "columns": [ + 1 + ], + "definition": "PRIMARY KEY (departmentid)" + } + ` + + original = [][]byte{ + []byte("1\tEngineering\tResearch and Development\t2008-04-30 00:00:00"), + []byte("2\tTool Design\tResearch and Development\t2008-04-30 00:00:00"), + []byte("3\tSales\tSales and Marketing\t2008-04-30 00:00:00"), + []byte("4\tMarketing\tSales and Marketing\t2008-04-30 00:00:00"), + []byte("5\tPurchasing\tInventory Management\t2008-04-30 00:00:00"), + []byte("6\tResearch and Development\tResearch and Development\t2008-04-30 00:00:00"), + } + + transformed = [][]byte{ + []byte("1\ttes1\tResearch and Development\t2008-04-30 00:00:00"), + []byte("2\ttes2\tResearch and Development\t2008-04-30 00:00:00"), + []byte("3\ttes3\tSales and Marketing\t2008-04-30 00:00:00"), + []byte("4\ttes4\tSales and Marketing\t\\N"), + []byte("5\ttes5\tInventory Management\t2008-04-30 00:00:00"), + []byte("6\ttes6 and Development\tResearch and Development\t2008-04-30 00:00:00"), + } + + c := &toolkit.PrimaryKey{} + err := json.Unmarshal([]byte(colnstraintDef), c) + if err != nil { + panic(err) + } + + t := &toolkit.Table{ + Constraints: []toolkit.Constraint{c}, + } + err = json.Unmarshal([]byte(tableDef), t) + if err != nil { + panic(err) + } + + table = &dump_objects.Table{ + Table: t, + Transformers: []utils.Transformer{&testTransformer{}}, + } + + return table, original, transformed +} diff --git a/internal/db/postgres/cmd/validate_utils/json_printer.go b/internal/db/postgres/cmd/validate_utils/json_printer.go index 5c293725..e2128859 100644 --- a/internal/db/postgres/cmd/validate_utils/json_printer.go +++ b/internal/db/postgres/cmd/validate_utils/json_printer.go @@ -1,54 +1,49 @@ package validate_utils -import ( - "fmt" - "io" -) - -type Printer interface { - Print(w io.Writer, d Documenter) error - Complete(w io.Writer) error -} - -type JsonPrinter struct { - firstLinePrinted bool -} - -func NewJsonPrinter() *JsonPrinter { - return &JsonPrinter{} -} - -func (jp *JsonPrinter) Print(w io.Writer, d Documenter) error { - if !jp.firstLinePrinted { - _, err := w.Write([]byte("[")) - if err != nil { - return fmt.Errorf("error printing intial line: %w", err) - } - } - _, err := w.Write([]byte(",")) - if err != nil { - return fmt.Errorf("error printing list separator: %w", err) - } - data, err := d.Data() - if err != nil { - return fmt.Errorf("error getting document data: %w", err) - } - _, err = w.Write(data) - if err != nil { - return fmt.Errorf("error printing document data: %w", err) - } - return nil -} - -func (jp *JsonPrinter) Complete(w io.Writer) error { - // Print the list closing bracket "]" - seq := []byte("]") - if !jp.firstLinePrinted { - // If there was not any document that we print empty document - seq = []byte("[]") - } - if _, err := w.Write(seq); err != nil { - return err - } - return nil -} +//type Printer interface { +// Print(w io.Writer, d Documenter) error +// Complete(w io.Writer) error +//} +// +//type JsonPrinter struct { +// firstLinePrinted bool +//} +// +//func NewJsonPrinter() *JsonPrinter { +// return &JsonPrinter{} +//} +// +//func (jp *JsonPrinter) Print(w io.Writer, d Documenter) error { +// if !jp.firstLinePrinted { +// _, err := w.Write([]byte("[")) +// if err != nil { +// return fmt.Errorf("error printing intial line: %w", err) +// } +// } +// _, err := w.Write([]byte(",")) +// if err != nil { +// return fmt.Errorf("error printing list separator: %w", err) +// } +// data, err := d.Data() +// if err != nil { +// return fmt.Errorf("error getting document data: %w", err) +// } +// _, err = w.Write(data) +// if err != nil { +// return fmt.Errorf("error printing document data: %w", err) +// } +// return nil +//} +// +//func (jp *JsonPrinter) Complete(w io.Writer) error { +// // Print the list closing bracket "]" +// seq := []byte("]") +// if !jp.firstLinePrinted { +// // If there was not any document that we print empty document +// seq = []byte("[]") +// } +// if _, err := w.Write(seq); err != nil { +// return err +// } +// return nil +//} diff --git a/internal/db/postgres/dumpers/dumpers.go b/internal/db/postgres/dumpers/dumpers.go index 09d213f1..8b70e9c6 100644 --- a/internal/db/postgres/dumpers/dumpers.go +++ b/internal/db/postgres/dumpers/dumpers.go @@ -19,11 +19,11 @@ import ( "github.com/jackc/pgx/v5" - "github.com/greenmaskio/greenmask/internal/db/postgres/dump" + "github.com/greenmaskio/greenmask/internal/db/postgres/dump_objects" "github.com/greenmaskio/greenmask/internal/storages" ) type DumpTask interface { - Execute(ctx context.Context, tx pgx.Tx, st storages.Storager) (dump.Entry, error) + Execute(ctx context.Context, tx pgx.Tx, st storages.Storager) (dump_objects.Entry, error) DebugInfo() string } diff --git a/pkg/toolkit/constraints.go b/pkg/toolkit/constraints.go index dab2837e..0790b445 100644 --- a/pkg/toolkit/constraints.go +++ b/pkg/toolkit/constraints.go @@ -32,6 +32,7 @@ const ( type Constraint interface { IsAffected(column *Column, columnProperties *ColumnProperties) (w ValidationWarnings) + Type() string } type DefaultConstraintDefinition struct { @@ -74,6 +75,10 @@ func (c *Check) IsAffected(column *Column, columnProperties *ColumnProperties) ( return } +func (c *Check) Type() string { + return CheckConstraintType +} + type Exclusion DefaultConstraintDefinition func NewExclusion(schema, name, definition string, oid Oid, columns []AttNum) *Exclusion { @@ -101,6 +106,10 @@ func (e *Exclusion) IsAffected(column *Column, columnProperties *ColumnPropertie return w } +func (e *Exclusion) Type() string { + return ExclusionConstraintType +} + // LinkedTable - table that involved into constraint, required for ForeignKey and PrimaryKeyReferences type LinkedTable struct { // Schema - table schema name @@ -147,6 +156,10 @@ func (fk *ForeignKey) IsAffected(column *Column, columnProperties *ColumnPropert return w } +func (fk *ForeignKey) Type() string { + return FkConstraintType +} + type PrimaryKey struct { DefaultConstraintDefinition References []*LinkedTable @@ -195,6 +208,10 @@ func (pk *PrimaryKey) IsAffected(column *Column, columnProperties *ColumnPropert return w } +func (pk *PrimaryKey) Type() string { + return PkConstraintType +} + type Unique DefaultConstraintDefinition func NewUnique(schema, name, definition string, oid Oid, columns []AttNum) *Unique { @@ -222,6 +239,10 @@ func (u *Unique) IsAffected(column *Column, columnProperties *ColumnProperties) return w } +func (u *Unique) Type() string { + return UniqueConstraintType +} + type TriggerConstraint DefaultConstraintDefinition func NewTriggerConstraint(schema, name, definition string, oid Oid, columns []AttNum) *TriggerConstraint { @@ -248,3 +269,7 @@ func (tc *TriggerConstraint) IsAffected(column *Column, columnProperties *Column } return w } + +func (tc *TriggerConstraint) Type() string { + return TriggerConstraintType +} From 917dec629189ba901a439481a090ff58fc459c50 Mon Sep 17 00:00:00 2001 From: Vadim Voitenko Date: Sat, 17 Feb 2024 21:29:39 +0200 Subject: [PATCH 04/10] Saved changes --- cmd/greenmask/cmd/dump/dump.go | 4 +- cmd/greenmask/cmd/restore/restore.go | 4 +- cmd/greenmask/cmd/show_dump/show_dump.go | 4 +- cmd/greenmask/cmd/validate/validate.go | 3 +- .../cmd/validate_utils/json_document.go | 18 +++++++++ .../cmd/validate_utils/json_document_test.go | 7 ++-- .../cmd/validate_utils/json_printer.go | 2 +- internal/db/postgres/context/pg_catalog.go | 40 +++++++++---------- internal/db/postgres/context/table.go | 16 ++++---- internal/db/postgres/dumpers/large_object.go | 8 ++-- .../postgres/dumpers/plain_dump_pipeline.go | 2 +- internal/db/postgres/dumpers/sequence.go | 8 ++-- internal/db/postgres/dumpers/table.go | 2 +- .../dumpers/transformation_pipeline.go | 2 +- .../postgres/dumpers/transformation_window.go | 2 +- .../postgres/dumpers/validation_pipeline.go | 3 +- internal/domains/config.go | 1 + .../postgres/cmd => old_validate}/validate.go | 9 +++-- 18 files changed, 79 insertions(+), 56 deletions(-) rename {internal/db/postgres/cmd => old_validate}/validate.go (99%) diff --git a/cmd/greenmask/cmd/dump/dump.go b/cmd/greenmask/cmd/dump/dump.go index 6c4782f2..bd8a4dcf 100644 --- a/cmd/greenmask/cmd/dump/dump.go +++ b/cmd/greenmask/cmd/dump/dump.go @@ -24,7 +24,7 @@ import ( "github.com/spf13/cobra" "github.com/spf13/viper" - "github.com/greenmaskio/greenmask/internal/db/postgres" + cmdInternals "github.com/greenmaskio/greenmask/internal/db/postgres/cmd" "github.com/greenmaskio/greenmask/internal/db/postgres/transformers/utils" pgDomains "github.com/greenmaskio/greenmask/internal/domains" "github.com/greenmaskio/greenmask/internal/storages/builder" @@ -52,7 +52,7 @@ var ( log.Fatal().Msg("common.tmp_dir cannot be empty") } - dump := postgres.NewDump(Config, st, utils.DefaultTransformerRegistry) + dump := cmdInternals.NewDump(Config, st, utils.DefaultTransformerRegistry) if err := dump.Run(ctx); err != nil { log.Fatal().Err(err).Msg("cannot make a backup") diff --git a/cmd/greenmask/cmd/restore/restore.go b/cmd/greenmask/cmd/restore/restore.go index 9b7b7541..c74c6d5e 100644 --- a/cmd/greenmask/cmd/restore/restore.go +++ b/cmd/greenmask/cmd/restore/restore.go @@ -24,7 +24,7 @@ import ( "github.com/spf13/cobra" "github.com/spf13/viper" - "github.com/greenmaskio/greenmask/internal/db/postgres" + cmdInternals "github.com/greenmaskio/greenmask/internal/db/postgres/cmd" pgDomains "github.com/greenmaskio/greenmask/internal/domains" "github.com/greenmaskio/greenmask/internal/storages/builder" "github.com/greenmaskio/greenmask/internal/utils/logger" @@ -88,7 +88,7 @@ var ( st = st.SubStorage(dumpId, true) - restore := postgres.NewRestore( + restore := cmdInternals.NewRestore( Config.Common.PgBinPath, st, &Config.Restore.PgRestoreOptions, Config.Restore.Scripts, Config.Common.TempDirectory, ) diff --git a/cmd/greenmask/cmd/show_dump/show_dump.go b/cmd/greenmask/cmd/show_dump/show_dump.go index c5f3bf49..fc76f4cd 100644 --- a/cmd/greenmask/cmd/show_dump/show_dump.go +++ b/cmd/greenmask/cmd/show_dump/show_dump.go @@ -22,7 +22,7 @@ import ( "github.com/rs/zerolog/log" "github.com/spf13/cobra" - "github.com/greenmaskio/greenmask/internal/db/postgres" + cmdInternals "github.com/greenmaskio/greenmask/internal/db/postgres/cmd" pgDomains "github.com/greenmaskio/greenmask/internal/domains" "github.com/greenmaskio/greenmask/internal/storages/builder" "github.com/greenmaskio/greenmask/internal/utils/logger" @@ -87,7 +87,7 @@ var ( } } - if err := postgres.ShowDump(ctx, st, dumpId, format); err != nil { + if err := cmdInternals.ShowDump(ctx, st, dumpId, format); err != nil { log.Fatal().Err(err).Msg("") } }, diff --git a/cmd/greenmask/cmd/validate/validate.go b/cmd/greenmask/cmd/validate/validate.go index ea84574b..87cc6e81 100644 --- a/cmd/greenmask/cmd/validate/validate.go +++ b/cmd/greenmask/cmd/validate/validate.go @@ -21,6 +21,7 @@ import ( "github.com/spf13/cobra" "github.com/spf13/viper" + cmd2 "github.com/greenmaskio/greenmask/internal/db/postgres/cmd" "github.com/greenmaskio/greenmask/internal/db/postgres/transformers/utils" "github.com/greenmaskio/greenmask/internal/domains" "github.com/greenmaskio/greenmask/internal/utils/logger" @@ -46,7 +47,7 @@ var ( ctx, cancel := context.WithCancel(context.Background()) defer cancel() - validate, err := postgres.NewValidate(Config, utils.DefaultTransformerRegistry) + validate, err := cmd2.NewValidateV2(Config, utils.DefaultTransformerRegistry) if err != nil { log.Fatal().Err(err).Msg("") } diff --git a/internal/db/postgres/cmd/validate_utils/json_document.go b/internal/db/postgres/cmd/validate_utils/json_document.go index 7ee6b300..061062f6 100644 --- a/internal/db/postgres/cmd/validate_utils/json_document.go +++ b/internal/db/postgres/cmd/validate_utils/json_document.go @@ -3,6 +3,7 @@ package validate_utils import ( "encoding/json" "fmt" + "io" "maps" "github.com/greenmaskio/greenmask/internal/db/postgres/dump_objects" @@ -10,6 +11,11 @@ import ( "github.com/greenmaskio/greenmask/pkg/toolkit" ) +type Documenter interface { + Print(w io.Writer) error + Append(original, transformed *pgcopy.Row) error +} + type values struct { ColNum int `json:"-"` Original string `json:"original,omitempty"` @@ -22,6 +28,8 @@ type JsonDocumentResult struct { Schema string `json:"schema"` Name string `json:"name"` PrimaryKeyColumns []string `json:"primary_key_columns,omitempty"` + WithDiff bool `json:"with_diff,omitempty"` + OnlyTransformed bool `json:"only_changed,omitempty"` Records []jsonRecord `json:"records,omitempty"` } @@ -48,6 +56,8 @@ func NewJsonDocument(table *dump_objects.Table, withDiff bool, onlyTransformed b return &JsonDocument{ result: &JsonDocumentResult{ PrimaryKeyColumns: pkColumnsList, + WithDiff: withDiff, + OnlyTransformed: onlyTransformed, Records: make([]jsonRecord, 0), }, withDiff: withDiff, @@ -90,6 +100,14 @@ func (jc *JsonDocument) Append(original, transformed *pgcopy.Row) error { return nil } +func (jc *JsonDocument) Print(w io.Writer) error { + res := jc.Get() + if err := json.NewEncoder(w).Encode(res); err != nil { + return err + } + return nil +} + func (jc *JsonDocument) GetImplicitlyChangedColumns() map[string]struct{} { return jc.unexpectedAffectedColumns } diff --git a/internal/db/postgres/cmd/validate_utils/json_document_test.go b/internal/db/postgres/cmd/validate_utils/json_document_test.go index 9d2ea970..ca0fb042 100644 --- a/internal/db/postgres/cmd/validate_utils/json_document_test.go +++ b/internal/db/postgres/cmd/validate_utils/json_document_test.go @@ -5,13 +5,14 @@ import ( "encoding/json" "testing" + "github.com/rs/zerolog/log" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/greenmaskio/greenmask/internal/db/postgres/dump_objects" "github.com/greenmaskio/greenmask/internal/db/postgres/pgcopy" "github.com/greenmaskio/greenmask/internal/db/postgres/transformers/utils" "github.com/greenmaskio/greenmask/pkg/toolkit" - "github.com/rs/zerolog/log" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" ) type testTransformer struct{} diff --git a/internal/db/postgres/cmd/validate_utils/json_printer.go b/internal/db/postgres/cmd/validate_utils/json_printer.go index e2128859..d783fa94 100644 --- a/internal/db/postgres/cmd/validate_utils/json_printer.go +++ b/internal/db/postgres/cmd/validate_utils/json_printer.go @@ -1,6 +1,6 @@ package validate_utils -//type Printer interface { +//type Documenter interface { // Print(w io.Writer, d Documenter) error // Complete(w io.Writer) error //} diff --git a/internal/db/postgres/context/pg_catalog.go b/internal/db/postgres/context/pg_catalog.go index c32f1469..67af4a8d 100644 --- a/internal/db/postgres/context/pg_catalog.go +++ b/internal/db/postgres/context/pg_catalog.go @@ -22,7 +22,7 @@ import ( "github.com/jackc/pgx/v5" - "github.com/greenmaskio/greenmask/internal/db/postgres/dump" + "github.com/greenmaskio/greenmask/internal/db/postgres/dump_objects" "github.com/greenmaskio/greenmask/internal/db/postgres/pgdump" "github.com/greenmaskio/greenmask/internal/db/postgres/toc" "github.com/greenmaskio/greenmask/pkg/toolkit" @@ -36,8 +36,8 @@ const ( // TODO: Rewrite it using gotemplate func getDumpObjects( - ctx context.Context, tx pgx.Tx, options *pgdump.Options, config map[toolkit.Oid]*dump.Table, -) ([]dump.Entry, error) { + ctx context.Context, tx pgx.Tx, options *pgdump.Options, config map[toolkit.Oid]*dump_objects.Table, +) ([]dump_objects.Entry, error) { // Building relation search query using regexp adaptation rules and pre-defined query templates // TODO: Refactor it to gotemplate @@ -55,9 +55,9 @@ func getDumpObjects( defer tableSearchRows.Close() // Generate table objects - //sequences := make([]*dump.Sequence, 0) - //tables := make([]*dump.Table, 0) - var dataObjects []dump.Entry + //sequences := make([]*dump_objects.Sequence, 0) + //tables := make([]*dump_objects.Table, 0) + var dataObjects []dump_objects.Entry defer tableSearchRows.Close() for tableSearchRows.Next() { var oid toc.Oid @@ -73,12 +73,12 @@ func getDumpObjects( if err != nil { return nil, fmt.Errorf("unnable scan data: %w", err) } - var table *dump.Table + var table *dump_objects.Table switch relKind { case 'S': // Building sequence objects - dataObjects = append(dataObjects, &dump.Sequence{ + dataObjects = append(dataObjects, &dump_objects.Sequence{ Name: name, Schema: schemaName, Oid: oid, @@ -101,7 +101,7 @@ func getDumpObjects( } else { // If table is not found - create new table object and collect all the columns - table = &dump.Table{ + table = &dump_objects.Table{ Table: &toolkit.Table{ Name: name, Schema: schemaName, @@ -130,7 +130,7 @@ func getDumpObjects( // Assigning columns for each table for _, obj := range dataObjects { switch v := obj.(type) { - case *dump.Table: + case *dump_objects.Table: columns, err := getColumnsConfig(ctx, tx, v.Oid) if err != nil { return nil, fmt.Errorf("unable to collect table columns: %w", err) @@ -150,14 +150,14 @@ func getDumpObjects( } // Getting list of the all large objects - var largeObjects []*dump.LargeObject + var largeObjects []*dump_objects.LargeObject loListRows, err := tx.Query(ctx, LargeObjectsListQuery) if err != nil { return nil, fmt.Errorf("error executing LargeObjectsListQuery: %w", err) } defer loListRows.Close() for loListRows.Next() { - lo := &dump.LargeObject{TableOid: tableOid} + lo := &dump_objects.LargeObject{TableOid: tableOid} if err = loListRows.Scan(&lo.Oid, &lo.Owner, &lo.Comment); err != nil { return nil, fmt.Errorf("error scanning LargeObjectsListQuery: %w", err) } @@ -168,20 +168,20 @@ func getDumpObjects( for _, lo := range largeObjects { // Getting default ACL - defaultACL := &dump.ACL{} + defaultACL := &dump_objects.ACL{} row = tx.QueryRow(ctx, LargeObjectGetDefaultAclQuery, lo.Oid) if err = row.Scan(&defaultACL.Value); err != nil { return nil, fmt.Errorf("error scanning LargeObjectGetDefaultAclQuery: %w", err) } // Getting default ACL items - var defaultACLItems []*dump.ACLItem + var defaultACLItems []*dump_objects.ACLItem loDescribeDefaultAclRows, err := tx.Query(ctx, LargeObjectDescribeAclItemQuery, defaultACL.Value) if err != nil { return nil, fmt.Errorf("error quering LargeObjectDescribeAclItemQuery: %w", err) } defer loDescribeDefaultAclRows.Close() for loDescribeDefaultAclRows.Next() { - item := &dump.ACLItem{} + item := &dump_objects.ACLItem{} if err = loDescribeDefaultAclRows.Scan(&item.Grantor, &item.Grantee, &item.PrivilegeType, &item.Grantable); err != nil { return nil, fmt.Errorf("error scanning LargeObjectDescribeAclItemQuery: %w", err) } @@ -191,14 +191,14 @@ func getDumpObjects( lo.DefaultACL = defaultACL // Getting ACL - var acls []*dump.ACL + var acls []*dump_objects.ACL loAclRows, err := tx.Query(ctx, LargeObjectGetAclQuery, lo.Oid) if err != nil { return nil, fmt.Errorf("error quering LargeObjectGetAclQuery: %w", err) } defer loAclRows.Close() for loAclRows.Next() { - a := &dump.ACL{} + a := &dump_objects.ACL{} if err = loAclRows.Scan(&a.Value); err != nil { return nil, fmt.Errorf("error scanning LargeObjectGetAclQuery: %w", err) } @@ -207,14 +207,14 @@ func getDumpObjects( // Getting ACL items for _, a := range acls { - var aclItems []*dump.ACLItem + var aclItems []*dump_objects.ACLItem loDescribeAclRows, err := tx.Query(ctx, LargeObjectDescribeAclItemQuery, a.Value) if err != nil { return nil, fmt.Errorf("error quering LargeObjectDescribeAclItemQuery: %w", err) } defer loDescribeAclRows.Close() for loDescribeAclRows.Next() { - item := &dump.ACLItem{} + item := &dump_objects.ACLItem{} if err = loDescribeAclRows.Scan(&item.Grantor, &item.Grantee, &item.PrivilegeType, &item.Grantable); err != nil { return nil, fmt.Errorf("error scanning LargeObjectDescribeAclItemQuery: %w", err) } @@ -227,7 +227,7 @@ func getDumpObjects( } if len(largeObjects) > 0 { - dataObjects = append(dataObjects, &dump.Blobs{ + dataObjects = append(dataObjects, &dump_objects.Blobs{ LargeObjects: largeObjects, }) } diff --git a/internal/db/postgres/context/table.go b/internal/db/postgres/context/table.go index 562552ca..c8a3fad1 100644 --- a/internal/db/postgres/context/table.go +++ b/internal/db/postgres/context/table.go @@ -24,7 +24,7 @@ import ( "github.com/jackc/pgx/v5/pgtype" "github.com/rs/zerolog/log" - "github.com/greenmaskio/greenmask/internal/db/postgres/dump" + "github.com/greenmaskio/greenmask/internal/db/postgres/dump_objects" transformersUtils "github.com/greenmaskio/greenmask/internal/db/postgres/transformers/utils" "github.com/greenmaskio/greenmask/internal/domains" "github.com/greenmaskio/greenmask/pkg/toolkit" @@ -37,8 +37,8 @@ func validateAndBuildTablesConfig( ctx context.Context, tx pgx.Tx, typeMap *pgtype.Map, cfg []*domains.Table, registry *transformersUtils.TransformerRegistry, version int, types []*toolkit.Type, -) (map[toolkit.Oid]*dump.Table, toolkit.ValidationWarnings, error) { - result := make(map[toolkit.Oid]*dump.Table, len(cfg)) +) (map[toolkit.Oid]*dump_objects.Table, toolkit.ValidationWarnings, error) { + result := make(map[toolkit.Oid]*dump_objects.Table, len(cfg)) var warnings toolkit.ValidationWarnings for _, tableCfg := range cfg { @@ -100,12 +100,12 @@ func validateAndBuildTablesConfig( return result, warnings, nil } -func getTable(ctx context.Context, tx pgx.Tx, t *domains.Table) ([]*dump.Table, toolkit.ValidationWarnings, error) { - table := &dump.Table{ +func getTable(ctx context.Context, tx pgx.Tx, t *domains.Table) ([]*dump_objects.Table, toolkit.ValidationWarnings, error) { + table := &dump_objects.Table{ Table: &toolkit.Table{}, } var warnings toolkit.ValidationWarnings - var tables []*dump.Table + var tables []*dump_objects.Table row := tx.QueryRow(ctx, TableSearchQuery, t.Schema, t.Name) err := row.Scan(&table.Oid, &table.Schema, &table.Name, &table.Owner, &table.RelKind, @@ -137,7 +137,7 @@ func getTable(ctx context.Context, tx pgx.Tx, t *domains.Table) ([]*dump.Table, Str("TableName", table.Name). Msg("table is partitioned: gathering all partitions and creating dumping tasks") // Get list of inherited tables - var parts []*dump.Table + var parts []*dump_objects.Table rows, err := tx.Query(ctx, TableGetChildPatsQuery, table.Oid) if err != nil { @@ -146,7 +146,7 @@ func getTable(ctx context.Context, tx pgx.Tx, t *domains.Table) ([]*dump.Table, defer rows.Close() for rows.Next() { - pt := &dump.Table{ + pt := &dump_objects.Table{ Table: &toolkit.Table{}, RootPtSchema: table.Schema, RootPtName: table.Name, diff --git a/internal/db/postgres/dumpers/large_object.go b/internal/db/postgres/dumpers/large_object.go index 5d33c6bd..3645cc5c 100644 --- a/internal/db/postgres/dumpers/large_object.go +++ b/internal/db/postgres/dumpers/large_object.go @@ -26,7 +26,7 @@ import ( "github.com/rs/zerolog/log" "golang.org/x/sync/errgroup" - "github.com/greenmaskio/greenmask/internal/db/postgres/dump" + "github.com/greenmaskio/greenmask/internal/db/postgres/dump_objects" "github.com/greenmaskio/greenmask/internal/storages" "github.com/greenmaskio/greenmask/internal/utils/countwriter" ) @@ -34,18 +34,18 @@ import ( const loBufSize = 1024 * 1024 type BlobsDumper struct { - Blobs *dump.Blobs + Blobs *dump_objects.Blobs OriginalSize int64 CompressedSize int64 } -func NewLargeObjectDumper(blobs *dump.Blobs) *BlobsDumper { +func NewLargeObjectDumper(blobs *dump_objects.Blobs) *BlobsDumper { return &BlobsDumper{ Blobs: blobs, } } -func (lod *BlobsDumper) Execute(ctx context.Context, tx pgx.Tx, st storages.Storager) (dump.Entry, error) { +func (lod *BlobsDumper) Execute(ctx context.Context, tx pgx.Tx, st storages.Storager) (dump_objects.Entry, error) { for _, lo := range lod.Blobs.LargeObjects { eg, gtx := errgroup.WithContext(ctx) diff --git a/internal/db/postgres/dumpers/plain_dump_pipeline.go b/internal/db/postgres/dumpers/plain_dump_pipeline.go index 0452c11c..50c32c4b 100644 --- a/internal/db/postgres/dumpers/plain_dump_pipeline.go +++ b/internal/db/postgres/dumpers/plain_dump_pipeline.go @@ -19,7 +19,7 @@ import ( "fmt" "io" - "github.com/greenmaskio/greenmask/internal/db/postgres/dump" + dump "github.com/greenmaskio/greenmask/internal/db/postgres/dump_objects" "github.com/greenmaskio/greenmask/internal/db/postgres/pgcopy" ) diff --git a/internal/db/postgres/dumpers/sequence.go b/internal/db/postgres/dumpers/sequence.go index 6f5383ee..13c90440 100644 --- a/internal/db/postgres/dumpers/sequence.go +++ b/internal/db/postgres/dumpers/sequence.go @@ -20,21 +20,21 @@ import ( "github.com/jackc/pgx/v5" - "github.com/greenmaskio/greenmask/internal/db/postgres/dump" + "github.com/greenmaskio/greenmask/internal/db/postgres/dump_objects" "github.com/greenmaskio/greenmask/internal/storages" ) type SequenceDumper struct { - sequence *dump.Sequence + sequence *dump_objects.Sequence } -func NewSequenceDumper(sequence *dump.Sequence) *SequenceDumper { +func NewSequenceDumper(sequence *dump_objects.Sequence) *SequenceDumper { return &SequenceDumper{ sequence: sequence, } } -func (sd *SequenceDumper) Execute(ctx context.Context, tx pgx.Tx, st storages.Storager) (dump.Entry, error) { +func (sd *SequenceDumper) Execute(ctx context.Context, tx pgx.Tx, st storages.Storager) (dump_objects.Entry, error) { return sd.sequence, nil } diff --git a/internal/db/postgres/dumpers/table.go b/internal/db/postgres/dumpers/table.go index 5dd69233..56dbba77 100644 --- a/internal/db/postgres/dumpers/table.go +++ b/internal/db/postgres/dumpers/table.go @@ -24,7 +24,7 @@ import ( "github.com/rs/zerolog/log" "golang.org/x/sync/errgroup" - "github.com/greenmaskio/greenmask/internal/db/postgres/dump" + dump "github.com/greenmaskio/greenmask/internal/db/postgres/dump_objects" "github.com/greenmaskio/greenmask/internal/storages" "github.com/greenmaskio/greenmask/internal/utils/countwriter" ) diff --git a/internal/db/postgres/dumpers/transformation_pipeline.go b/internal/db/postgres/dumpers/transformation_pipeline.go index 71c0db4c..c791b095 100644 --- a/internal/db/postgres/dumpers/transformation_pipeline.go +++ b/internal/db/postgres/dumpers/transformation_pipeline.go @@ -23,7 +23,7 @@ import ( "github.com/rs/zerolog/log" "golang.org/x/sync/errgroup" - "github.com/greenmaskio/greenmask/internal/db/postgres/dump" + dump "github.com/greenmaskio/greenmask/internal/db/postgres/dump_objects" "github.com/greenmaskio/greenmask/internal/db/postgres/pgcopy" "github.com/greenmaskio/greenmask/internal/db/postgres/transformers" "github.com/greenmaskio/greenmask/internal/db/postgres/transformers/utils" diff --git a/internal/db/postgres/dumpers/transformation_window.go b/internal/db/postgres/dumpers/transformation_window.go index 40485bf6..bfd6e240 100644 --- a/internal/db/postgres/dumpers/transformation_window.go +++ b/internal/db/postgres/dumpers/transformation_window.go @@ -20,7 +20,7 @@ import ( "golang.org/x/sync/errgroup" - "github.com/greenmaskio/greenmask/internal/db/postgres/dump" + dump "github.com/greenmaskio/greenmask/internal/db/postgres/dump_objects" "github.com/greenmaskio/greenmask/internal/db/postgres/transformers/utils" "github.com/greenmaskio/greenmask/pkg/toolkit" ) diff --git a/internal/db/postgres/dumpers/validation_pipeline.go b/internal/db/postgres/dumpers/validation_pipeline.go index 6bc6f702..112a717c 100644 --- a/internal/db/postgres/dumpers/validation_pipeline.go +++ b/internal/db/postgres/dumpers/validation_pipeline.go @@ -5,8 +5,9 @@ import ( "fmt" "io" - "github.com/greenmaskio/greenmask/internal/db/postgres/dump" "golang.org/x/sync/errgroup" + + dump "github.com/greenmaskio/greenmask/internal/db/postgres/dump_objects" ) type ValidationPipeline struct { diff --git a/internal/domains/config.go b/internal/domains/config.go index 5acc53d2..b606c7c7 100644 --- a/internal/domains/config.go +++ b/internal/domains/config.go @@ -61,6 +61,7 @@ type Validate struct { ResolvedWarnings []string `mapstructure:"resolved_warnings" yaml:"resolved_warnings" json:"resolved_warnings,omitempty"` TableFormat string `mapstructure:"table_format" yaml:"table_format" json:"table_format,omitempty"` Format string `mapstructure:"format" yaml:"format" json:"format,omitempty"` + OnlyTransformed bool `mapstructure:"only_transformed" yaml:"only_transformed" json:"only_transformed,omitempty"` } type Common struct { diff --git a/internal/db/postgres/cmd/validate.go b/old_validate/validate.go similarity index 99% rename from internal/db/postgres/cmd/validate.go rename to old_validate/validate.go index f8a13ede..17083100 100644 --- a/internal/db/postgres/cmd/validate.go +++ b/old_validate/validate.go @@ -1,4 +1,4 @@ -package cmd +package old_validate import ( "bufio" @@ -14,10 +14,11 @@ import ( "strings" "time" - "github.com/greenmaskio/greenmask/internal/db/postgres/cmd/validate_utils" "github.com/olekukonko/tablewriter" "github.com/rs/zerolog/log" + "github.com/greenmaskio/greenmask/internal/db/postgres/cmd" + "github.com/greenmaskio/greenmask/internal/db/postgres/cmd/validate_utils" runtimeContext "github.com/greenmaskio/greenmask/internal/db/postgres/context" "github.com/greenmaskio/greenmask/internal/db/postgres/dump_objects" "github.com/greenmaskio/greenmask/internal/db/postgres/pgcopy" @@ -52,7 +53,7 @@ type printSettings struct { } type Validate struct { - *Dump + *cmd.Dump tmpDir string } @@ -65,7 +66,7 @@ func NewValidate(cfg *domains.Config, registry *utils.TransformerRegistry) (*Val tmpDir := strconv.FormatInt(time.Now().UnixMilli(), 10) st = st.SubStorage(tmpDir, true) - d := NewDump(cfg, st, registry) + d := cmd.NewDump(cfg, st, registry) d.dumpIdSequence = toc.NewDumpSequence(0) d.validate = true return &Validate{ From 9fde753c32b8b9d622a607cb98dda1e3ff710da6 Mon Sep 17 00:00:00 2001 From: Vadim Voitenko Date: Sat, 17 Feb 2024 22:36:33 +0200 Subject: [PATCH 05/10] Saved changes --- cmd/greenmask/cmd/validate/validate.go | 20 ++- .../cmd/validate_utils/json_document.go | 125 ++++++++++++++---- .../cmd/validate_utils/json_document_test.go | 2 +- 3 files changed, 120 insertions(+), 27 deletions(-) diff --git a/cmd/greenmask/cmd/validate/validate.go b/cmd/greenmask/cmd/validate/validate.go index 87cc6e81..43c0f320 100644 --- a/cmd/greenmask/cmd/validate/validate.go +++ b/cmd/greenmask/cmd/validate/validate.go @@ -98,13 +98,31 @@ func init() { log.Fatal().Err(err).Msg("fatal") } + tableFormatFlagName := "table-format" + Cmd.Flags().String( + tableFormatFlagName, "horizontal", "format of table output. possible values [horizontal|vertical]", + ) + flag = Cmd.Flags().Lookup(tableFormatFlagName) + if err := viper.BindPFlag("validate.table_format", flag); err != nil { + log.Fatal().Err(err).Msg("fatal") + } + formatFlagName := "format" Cmd.Flags().String( - formatFlagName, "horizontal", "format of table output. possible values [horizontal|vertical]", + formatFlagName, "json", "format of output. possible values [text|json]", ) flag = Cmd.Flags().Lookup(formatFlagName) if err := viper.BindPFlag("validate.format", flag); err != nil { log.Fatal().Err(err).Msg("fatal") } + onlyTransformedFlagName := "only-transformed" + Cmd.Flags().Bool( + onlyTransformedFlagName, false, "include into diff result only transformed columns and primary key value", + ) + flag = Cmd.Flags().Lookup(onlyTransformedFlagName) + if err := viper.BindPFlag("validate.only_transformed", flag); err != nil { + log.Fatal().Err(err).Msg("fatal") + } + } diff --git a/internal/db/postgres/cmd/validate_utils/json_document.go b/internal/db/postgres/cmd/validate_utils/json_document.go index 061062f6..6a1a102b 100644 --- a/internal/db/postgres/cmd/validate_utils/json_document.go +++ b/internal/db/postgres/cmd/validate_utils/json_document.go @@ -16,24 +16,42 @@ type Documenter interface { Append(original, transformed *pgcopy.Row) error } -type values struct { +type valueWithDiff struct { ColNum int `json:"-"` - Original string `json:"original,omitempty"` - Transformed string `json:"transformed,omitempty"` - Equal bool `json:"equal,omitempty"` - Expected bool `json:"implicit,omitempty"` + Original string `json:"original"` + Transformed string `json:"transformed"` + Equal bool `json:"equal"` + Expected bool `json:"implicit"` } type JsonDocumentResult struct { - Schema string `json:"schema"` - Name string `json:"name"` - PrimaryKeyColumns []string `json:"primary_key_columns,omitempty"` - WithDiff bool `json:"with_diff,omitempty"` - OnlyTransformed bool `json:"only_changed,omitempty"` - Records []jsonRecord `json:"records,omitempty"` + Schema string + Name string + PrimaryKeyColumns []string + WithDiff bool + OnlyTransformed bool + RecordsWithDiff []jsonRecordWithDiff + RecordsPlain []jsonRecordPlain } -type jsonRecord map[string]*values +type jsonDocumentResponseWithDiff struct { + Schema string `json:"schema"` + Name string `json:"name"` + PrimaryKeyColumns []string `json:"primary_key_columns"` + WithDiff bool `json:"with_diff"` + OnlyTransformed bool `json:"only_transformed"` + Records []jsonRecordWithDiff `json:"records"` +} + +type jsonDocumentResponsePlain struct { + Schema string `json:"schema"` + Name string `json:"name"` + Records []jsonRecordPlain `json:"records"` +} + +type jsonRecordWithDiff map[string]*valueWithDiff + +type jsonRecordPlain map[string]string type JsonDocument struct { result *JsonDocumentResult @@ -55,10 +73,12 @@ func NewJsonDocument(table *dump_objects.Table, withDiff bool, onlyTransformed b return &JsonDocument{ result: &JsonDocumentResult{ + Schema: table.Schema, + Name: table.Name, PrimaryKeyColumns: pkColumnsList, WithDiff: withDiff, OnlyTransformed: onlyTransformed, - Records: make([]jsonRecord, 0), + RecordsWithDiff: make([]jsonRecordWithDiff, 0), }, withDiff: withDiff, table: table, @@ -69,40 +89,95 @@ func NewJsonDocument(table *dump_objects.Table, withDiff bool, onlyTransformed b } } -func (jc *JsonDocument) Append(original, transformed *pgcopy.Row) error { - r := make(jsonRecord) +func (jc *JsonDocument) appendWithDiff(original, transformed *pgcopy.Row) error { + r := make(jsonRecordWithDiff) for idx, c := range jc.table.Columns { originalRawValue, err := original.GetColumn(idx) if err != nil { return fmt.Errorf("error getting column from original record: %w", err) } + + var originalValue, transformedValue string + + originalValue = getStringFromRawValue(originalRawValue) + + equal := true + expected := true + transformedRawValue, err := transformed.GetColumn(idx) if err != nil { return fmt.Errorf("error getting column from transformed record: %w", err) } - - equal := ValuesEqual(originalRawValue, transformedRawValue) - expected := true + transformedValue = getStringFromRawValue(transformedRawValue) + equal = ValuesEqual(originalRawValue, transformedRawValue) if _, ok := jc.expectedAffectedColumns[c.Name]; !equal && !ok { expected = false jc.unexpectedAffectedColumns[c.Name] = struct{}{} } - r[c.Name] = &values{ - Original: getStringFromRawValue(originalRawValue), - Transformed: getStringFromRawValue(transformedRawValue), + r[c.Name] = &valueWithDiff{ + Original: originalValue, + Transformed: transformedValue, Equal: equal, Expected: expected, ColNum: idx, } } - jc.result.Records = append(jc.result.Records, r) + jc.result.RecordsWithDiff = append(jc.result.RecordsWithDiff, r) + return nil +} + +func (jc *JsonDocument) appendPlain(original *pgcopy.Row) error { + r := make(jsonRecordPlain) + for idx, c := range jc.table.Columns { + originalRawValue, err := original.GetColumn(idx) + if err != nil { + return fmt.Errorf("error getting column from original record: %w", err) + } + + r[c.Name] = getStringFromRawValue(originalRawValue) + } + jc.result.RecordsPlain = append(jc.result.RecordsPlain, r) + return nil +} + +func (jc *JsonDocument) Append(original, transformed *pgcopy.Row) error { + if jc.withDiff { + if err := jc.appendWithDiff(original, transformed); err != nil { + return fmt.Errorf("error appending data with diff: %w", err) + } + } else { + if err := jc.appendPlain(original); err != nil { + return fmt.Errorf("error appending data without diff: %w", err) + } + } return nil } func (jc *JsonDocument) Print(w io.Writer) error { - res := jc.Get() - if err := json.NewEncoder(w).Encode(res); err != nil { + result := jc.Get() + + if result.WithDiff { + response := &jsonDocumentResponseWithDiff{ + Schema: result.Schema, + Name: result.Name, + PrimaryKeyColumns: result.PrimaryKeyColumns, + WithDiff: result.WithDiff, + OnlyTransformed: result.OnlyTransformed, + Records: result.RecordsWithDiff, + } + if err := json.NewEncoder(w).Encode(response); err != nil { + return err + } + return nil + } + + response := &jsonDocumentResponsePlain{ + Schema: result.Schema, + Name: result.Name, + Records: result.RecordsPlain, + } + if err := json.NewEncoder(w).Encode(response); err != nil { return err } return nil @@ -158,7 +233,7 @@ func (jc *JsonDocument) filterColumns() { } } - for _, r := range jc.result.Records { + for _, r := range jc.result.RecordsWithDiff { for name := range columnsToDelete { delete(r, name) } diff --git a/internal/db/postgres/cmd/validate_utils/json_document_test.go b/internal/db/postgres/cmd/validate_utils/json_document_test.go index ca0fb042..62825846 100644 --- a/internal/db/postgres/cmd/validate_utils/json_document_test.go +++ b/internal/db/postgres/cmd/validate_utils/json_document_test.go @@ -71,7 +71,7 @@ func TestJsonDocument_GetRecords(t *testing.T) { assert.Equal(t, columnsImplicitlyChanged, map[string]struct{}{"modifieddate": {}}) result := jd.Get() - require.Len(t, result.Records, 6) + require.Len(t, result.RecordsWithDiff, 6) //log.Warn().Any("a", records) From 69a66b7835bc22ef5cecdaf7011c2491dbd0385e Mon Sep 17 00:00:00 2001 From: Vadim Voitenko Date: Mon, 19 Feb 2024 23:19:09 +0200 Subject: [PATCH 06/10] saved changes --- .../cmd/validate_utils/json_document.go | 11 +- .../cmd/validate_utils/json_document_test.go | 2 +- .../cmd/validate_utils/json_printer.go | 49 ----- .../db/postgres/cmd/validate_utils/printer.go | 1 - .../cmd/validate_utils/text_document.go | 187 ++++++++++++++++++ .../cmd/validate_utils/text_printer.go | 1 - 6 files changed, 196 insertions(+), 55 deletions(-) delete mode 100644 internal/db/postgres/cmd/validate_utils/json_printer.go delete mode 100644 internal/db/postgres/cmd/validate_utils/printer.go delete mode 100644 internal/db/postgres/cmd/validate_utils/text_printer.go diff --git a/internal/db/postgres/cmd/validate_utils/json_document.go b/internal/db/postgres/cmd/validate_utils/json_document.go index 6a1a102b..df303030 100644 --- a/internal/db/postgres/cmd/validate_utils/json_document.go +++ b/internal/db/postgres/cmd/validate_utils/json_document.go @@ -183,14 +183,19 @@ func (jc *JsonDocument) Print(w io.Writer) error { return nil } -func (jc *JsonDocument) GetImplicitlyChangedColumns() map[string]struct{} { +func (jc *JsonDocument) GetUnexpectedlyChangedColumns() map[string]struct{} { return jc.unexpectedAffectedColumns } +func (jc *JsonDocument) GetAffectedColumns() map[string]struct{} { + affectedColumns := maps.Clone(jc.expectedAffectedColumns) + maps.Copy(affectedColumns, jc.unexpectedAffectedColumns) + return affectedColumns +} + func (jc *JsonDocument) GetColumnsToPrint() map[string]struct{} { if jc.onlyTransformed { - columnsToPrint := maps.Clone(jc.expectedAffectedColumns) - maps.Copy(columnsToPrint, jc.unexpectedAffectedColumns) + columnsToPrint := jc.GetAffectedColumns() for _, colName := range jc.result.PrimaryKeyColumns { columnsToPrint[colName] = struct{}{} } diff --git a/internal/db/postgres/cmd/validate_utils/json_document_test.go b/internal/db/postgres/cmd/validate_utils/json_document_test.go index 62825846..4524fab1 100644 --- a/internal/db/postgres/cmd/validate_utils/json_document_test.go +++ b/internal/db/postgres/cmd/validate_utils/json_document_test.go @@ -67,7 +67,7 @@ func TestJsonDocument_GetRecords(t *testing.T) { columnsToPrint := jd.GetColumnsToPrint() assert.Equal(t, columnsToPrint, map[string]struct{}{"departmentid": {}, "name": {}, "modifieddate": {}}) - columnsImplicitlyChanged := jd.GetImplicitlyChangedColumns() + columnsImplicitlyChanged := jd.GetUnexpectedlyChangedColumns() assert.Equal(t, columnsImplicitlyChanged, map[string]struct{}{"modifieddate": {}}) result := jd.Get() diff --git a/internal/db/postgres/cmd/validate_utils/json_printer.go b/internal/db/postgres/cmd/validate_utils/json_printer.go deleted file mode 100644 index d783fa94..00000000 --- a/internal/db/postgres/cmd/validate_utils/json_printer.go +++ /dev/null @@ -1,49 +0,0 @@ -package validate_utils - -//type Documenter interface { -// Print(w io.Writer, d Documenter) error -// Complete(w io.Writer) error -//} -// -//type JsonPrinter struct { -// firstLinePrinted bool -//} -// -//func NewJsonPrinter() *JsonPrinter { -// return &JsonPrinter{} -//} -// -//func (jp *JsonPrinter) Print(w io.Writer, d Documenter) error { -// if !jp.firstLinePrinted { -// _, err := w.Write([]byte("[")) -// if err != nil { -// return fmt.Errorf("error printing intial line: %w", err) -// } -// } -// _, err := w.Write([]byte(",")) -// if err != nil { -// return fmt.Errorf("error printing list separator: %w", err) -// } -// data, err := d.Data() -// if err != nil { -// return fmt.Errorf("error getting document data: %w", err) -// } -// _, err = w.Write(data) -// if err != nil { -// return fmt.Errorf("error printing document data: %w", err) -// } -// return nil -//} -// -//func (jp *JsonPrinter) Complete(w io.Writer) error { -// // Print the list closing bracket "]" -// seq := []byte("]") -// if !jp.firstLinePrinted { -// // If there was not any document that we print empty document -// seq = []byte("[]") -// } -// if _, err := w.Write(seq); err != nil { -// return err -// } -// return nil -//} diff --git a/internal/db/postgres/cmd/validate_utils/printer.go b/internal/db/postgres/cmd/validate_utils/printer.go deleted file mode 100644 index 147e532c..00000000 --- a/internal/db/postgres/cmd/validate_utils/printer.go +++ /dev/null @@ -1 +0,0 @@ -package validate_utils diff --git a/internal/db/postgres/cmd/validate_utils/text_document.go b/internal/db/postgres/cmd/validate_utils/text_document.go index 147e532c..d7fd1fd2 100644 --- a/internal/db/postgres/cmd/validate_utils/text_document.go +++ b/internal/db/postgres/cmd/validate_utils/text_document.go @@ -1 +1,188 @@ package validate_utils + +import ( + "fmt" + "io" + "os" + "slices" + + "github.com/olekukonko/tablewriter" + + "github.com/greenmaskio/greenmask/internal/db/postgres/dump_objects" + stringsUtils "github.com/greenmaskio/greenmask/internal/utils/strings" + "github.com/greenmaskio/greenmask/pkg/toolkit" +) + +const ( + horizontalTableFormatName = "horizontal" + verticalTableFormatName = "vertical" +) + +const maxWrapLength = 64 + +type printSettings struct { + OriginalColors []tablewriter.Colors + TransformedColors []tablewriter.Colors + HeaderColors []tablewriter.Colors + ColumnsAlignments []int +} + +type TextDocument struct { + *JsonDocument +} + +func (td *TextDocument) Print(w io.Writer) error { + panic("implement me") +} + +func (td *TextDocument) printWithDiffVertical(w io.Writer) error { + panic("implement me") +} + +func (td *TextDocument) getColumnsIdxsUnexpected() []int { + var res []int + colToPrint := td.GetUnexpectedlyChangedColumns() + for colName := range colToPrint { + idx := slices.IndexFunc(td.table.Columns, func(column *toolkit.Column) bool { + return column.Name == colName + }) + if idx != -1 { + panic("expected column to be found in the table column list") + } + res = append(res, idx) + } + slices.Sort(res) + return res +} + +func (td *TextDocument) getColumnsIdxsToPrint() []int { + var res []int + colToPrint := td.GetColumnsToPrint() + for colName := range colToPrint { + idx := slices.IndexFunc(td.table.Columns, func(column *toolkit.Column) bool { + return column.Name == colName + }) + if idx != -1 { + panic("expected column to be found in the table column list") + } + res = append(res, idx) + } + slices.Sort(res) + return res +} + +func (td *TextDocument) getVerticalHorizontalColors(t *dump_objects.Table, affectedColumns map[int]struct{}) []tablewriter.Colors { + headerColors := make([]tablewriter.Colors, len(t.Columns)) + for idx := range t.Columns { + if _, ok := affectedColumns[idx]; ok { + headerColors[idx] = []int{tablewriter.BgRedColor} + } else { + headerColors[idx] = []int{} + } + } + // Adding formatting setting for LineNum + headerColors = slices.Insert(headerColors, 0, tablewriter.Colors{}) + return headerColors +} + +func (td *TextDocument) printWithDiffHorizontal(w io.Writer) error { + settings := td.getHorizontalSettings() + prettyWriter := tablewriter.NewWriter(w) + prettyWriter.SetColumnAlignment(settings.ColumnsAlignments) + + result := td.JsonDocument.Get() + colIdxsToPrint := td.getColumnsIdxsToPrint() + + for lineIdx, res := range result.RecordsWithDiff { + originalRecord := make([]string, len(td.table.Columns)) + transformedRecord := make([]string, len(td.table.Columns)) + originalRecordColors := make([]tablewriter.Colors, len(td.table.Columns)) + transformedRecordColors := make([]tablewriter.Colors, len(td.table.Columns)) + for _, colIdx := range colIdxsToPrint { + colName := td.table.Columns[colIdx].Name + colValue := res[colName] + originalRecord[colIdx] = stringsUtils.WrapString(colValue.Original, maxWrapLength) + transformedRecord[colIdx] = stringsUtils.WrapString(colValue.Transformed, maxWrapLength) + + if !colValue.Equal { + originalRecordColors[colIdx] = tablewriter.Colors{tablewriter.FgHiGreenColor} + transformedRecordColors[colIdx] = tablewriter.Colors{tablewriter.FgHiRedColor} + } else { + originalRecordColors[colIdx] = []int{} + transformedRecordColors[colIdx] = []int{} + } + + originalRecordColors = slices.Insert(originalRecordColors, 0, tablewriter.Colors{}) + transformedRecordColors = slices.Insert(transformedRecordColors, 0, tablewriter.Colors{}) + originalRecord = slices.Insert(originalRecord, 0, fmt.Sprintf("%d", lineIdx)) + transformedRecord = slices.Insert(transformedRecord, 0, fmt.Sprintf("%d", lineIdx)) + prettyWriter.Rich(originalRecord, originalRecordColors) + prettyWriter.Rich(transformedRecord, transformedRecordColors) + } + } + + unexpectedlyChanged := td.GetUnexpectedlyChangedColumns() + header := make([]string, len(colIdxsToPrint)) + for tableColIdx, colIdx := range colIdxsToPrint { + c := td.table.Columns[colIdx] + if _, ok := unexpectedlyChanged[c.Name]; ok { + header[tableColIdx] = fmt.Sprintf("%s (!!!)", c.Name) + } else { + header[tableColIdx] = c.Name + } + } + header = slices.Insert(header, 0, "%LineNum%") + headerColors := td.getVerticalHorizontalColors(t, realAffectedColumns) + + os.Stdout.Write([]byte(fmt.Sprintf("\n\n\t\"%s\".\"%s\"\n", t.Schema, t.Name))) + prettyWriter.SetHeader(header) + prettyWriter.SetRowLine(true) + prettyWriter.SetAutoMergeCellsByColumnIndex([]int{0}) + prettyWriter.SetAutoWrapText(true) + prettyWriter.SetHeaderLine(true) + prettyWriter.SetHeaderColor(headerColors...) + + prettyWriter.Render() + return nil +} + +func (td *TextDocument) printPlainHorizontal(w io.Writer) error { + panic("implement me") +} + +func (td *TextDocument) printPlainVertical(w io.Writer) error { + panic("implement me") +} + +func (td *TextDocument) getHorizontalSettings() *printSettings { + affectedColumns := td.JsonDocument.GetColumnsToPrint() + + originalColumnsColors := make([]tablewriter.Colors, len(td.JsonDocument.table.Columns)) + transformedColumnsColors := make([]tablewriter.Colors, len(td.JsonDocument.table.Columns)) + headerColors := make([]tablewriter.Colors, len(td.JsonDocument.table.Columns)) + columnsAlignments := make([]int, len(td.JsonDocument.table.Columns)) + for idx, c := range td.JsonDocument.table.Columns { + if _, ok := affectedColumns[c.Name]; ok { + originalColumnsColors[idx] = []int{tablewriter.FgHiGreenColor} + transformedColumnsColors[idx] = []int{tablewriter.FgHiRedColor} + headerColors[idx] = []int{tablewriter.BgRedColor} + } else { + originalColumnsColors[idx] = []int{} + headerColors[idx] = []int{} + transformedColumnsColors[idx] = []int{} + } + columnsAlignments[idx] = tablewriter.ALIGN_LEFT + } + // Adding formatting setting for LineNum + originalColumnsColors = slices.Insert(originalColumnsColors, 0, tablewriter.Colors{}) + headerColors = slices.Insert(headerColors, 0, tablewriter.Colors{}) + transformedColumnsColors = slices.Insert(transformedColumnsColors, 0, tablewriter.Colors{}) + columnsAlignments = slices.Insert(columnsAlignments, 0, tablewriter.ALIGN_LEFT) + + return &printSettings{ + OriginalColors: originalColumnsColors, + TransformedColors: transformedColumnsColors, + HeaderColors: headerColors, + ColumnsAlignments: columnsAlignments, + } +} diff --git a/internal/db/postgres/cmd/validate_utils/text_printer.go b/internal/db/postgres/cmd/validate_utils/text_printer.go deleted file mode 100644 index 147e532c..00000000 --- a/internal/db/postgres/cmd/validate_utils/text_printer.go +++ /dev/null @@ -1 +0,0 @@ -package validate_utils From 32da6a6bd7b1c8fb6a82ae9554f7a66e63acf8bb Mon Sep 17 00:00:00 2001 From: Vadim Voitenko Date: Tue, 20 Feb 2024 11:44:47 +0200 Subject: [PATCH 07/10] Lost validate_v2 file --- internal/db/postgres/cmd/validate_v2.go | 375 ++++++++++++++++++++++++ 1 file changed, 375 insertions(+) create mode 100644 internal/db/postgres/cmd/validate_v2.go diff --git a/internal/db/postgres/cmd/validate_v2.go b/internal/db/postgres/cmd/validate_v2.go new file mode 100644 index 00000000..eb5d1b4f --- /dev/null +++ b/internal/db/postgres/cmd/validate_v2.go @@ -0,0 +1,375 @@ +package cmd + +import ( + "bufio" + "compress/gzip" + "context" + "errors" + "fmt" + "io" + "os" + "path" + "slices" + "strconv" + "strings" + "time" + + "github.com/rs/zerolog/log" + + "github.com/greenmaskio/greenmask/internal/db/postgres/cmd/validate_utils" + runtimeContext "github.com/greenmaskio/greenmask/internal/db/postgres/context" + "github.com/greenmaskio/greenmask/internal/db/postgres/dump_objects" + "github.com/greenmaskio/greenmask/internal/db/postgres/pgcopy" + "github.com/greenmaskio/greenmask/internal/db/postgres/toc" + "github.com/greenmaskio/greenmask/internal/db/postgres/transformers/custom" + "github.com/greenmaskio/greenmask/internal/db/postgres/transformers/utils" + "github.com/greenmaskio/greenmask/internal/domains" + "github.com/greenmaskio/greenmask/internal/storages" + "github.com/greenmaskio/greenmask/internal/storages/directory" + "github.com/greenmaskio/greenmask/internal/utils/reader" + //"github.com/greenmaskio/greenmask/old_validate" + "github.com/greenmaskio/greenmask/pkg/toolkit" +) + +const ( + jsonFormat string = "json" + textFormat string = "text" +) + +type closeFunc func() + +type ValidateV2 struct { + *Dump + tmpDir string +} + +func NewValidateV2(cfg *domains.Config, registry *utils.TransformerRegistry) (*ValidateV2, error) { + var st storages.Storager + st, err := directory.NewStorage(&directory.Config{Path: cfg.Common.TempDirectory}) + if err != nil { + return nil, fmt.Errorf("error initializing storage") + } + tmpDir := strconv.FormatInt(time.Now().UnixMilli(), 10) + st = st.SubStorage(tmpDir, true) + + d := NewDump(cfg, st, registry) + d.dumpIdSequence = toc.NewDumpSequence(0) + d.validate = true + return &ValidateV2{ + Dump: d, + tmpDir: path.Join(cfg.Common.TempDirectory, tmpDir), + }, nil +} + +func (v *ValidateV2) Run(ctx context.Context) error { + + defer func() { + // Deleting temp dir after closing it + if err := os.RemoveAll(v.tmpDir); err != nil { + log.Warn().Err(err).Msgf("unable to delete temp directory") + } + }() + if err := custom.BootstrapCustomTransformers(ctx, v.registry, v.config.CustomTransformers); err != nil { + return fmt.Errorf("error bootstraping custom transformers: %w", err) + } + + dsn, err := v.pgDumpOptions.GetPgDSN() + if err != nil { + return fmt.Errorf("cannot build connection string: %w", err) + } + + conn, err := v.connect(ctx, dsn) + if err != nil { + return err + } + defer func() { + if err := conn.Close(ctx); err != nil { + log.Warn().Err(err) + } + }() + + tx, err := v.startMainTx(ctx, conn) + if err != nil { + return fmt.Errorf("cannot prepare backup transaction: %w", err) + } + defer func() { + if err := tx.Rollback(ctx); err != nil { + log.Warn().Err(err) + } + }() + + if err = v.gatherPgFacts(ctx, tx); err != nil { + return fmt.Errorf("error gathering facts: %w", err) + } + + // Get list of tables to validate + tablesToValidate, err := v.getTablesToValidate() + if err != nil { + return err + } + v.config.Dump.Transformation = tablesToValidate + + v.context, err = runtimeContext.NewRuntimeContext(ctx, tx, v.config.Dump.Transformation, v.registry, + v.pgDumpOptions, v.version) + if err != nil { + return fmt.Errorf("unable to build runtime context: %w", err) + } + + if err = v.printValidationWarnings(); err != nil { + return err + } + + if !v.config.Validate.Data { + return nil + } + + if err = v.dumpTables(ctx); err != nil { + return err + } + + if err = v.print(ctx); err != nil { + return err + } + + return nil +} + +func (v *ValidateV2) print(ctx context.Context) error { + for _, e := range v.dataEntries { + idx := slices.IndexFunc(v.context.DataSectionObjects, func(entry dump_objects.Entry) bool { + t := entry.(*dump_objects.Table) + return t.DumpId == e.DumpId + }) + + t := v.context.DataSectionObjects[idx].(*dump_objects.Table) + doc, err := v.createDocument(ctx, t) + if err != nil { + return fmt.Errorf("unable to create validation document: %w", err) + } + + if err = doc.Print(os.Stdout); err != nil { + return fmt.Errorf("unable to print validation document: %w", err) + } + } + return nil +} + +func (v *ValidateV2) getDocument(table *dump_objects.Table) validate_utils.Documenter { + if v.config.Validate.Format == jsonFormat { + return validate_utils.NewJsonDocument(table, v.config.Validate.Diff, v.config.Validate.OnlyTransformed) + } + panic("IMPLEMENT TEXT PRINTER") + return nil +} + +func (v *ValidateV2) getReader(ctx context.Context, table *dump_objects.Table) (closeFunc, *bufio.Reader, error) { + tableData, err := v.st.GetObject(ctx, fmt.Sprintf("%d.dat.gz", table.DumpId)) + if err != nil { + return nil, nil, fmt.Errorf("unable to get object from storage: %w", err) + } + + gz, err := gzip.NewReader(tableData) + if err != nil { + tableData.Close() + return nil, nil, fmt.Errorf("cannot create gzip reader: %w", err) + } + + f := func() { + if err := tableData.Close(); err != nil { + log.Warn().Err(err).Msg("caused error when closing reader object") + } + if err := gz.Close(); err != nil { + log.Warn().Err(err).Msg("caused error when closing gzip reader") + } + } + + return f, bufio.NewReader(gz), nil +} + +func (v *ValidateV2) readRecords(r *bufio.Reader, t *dump_objects.Table) (original, transformed *pgcopy.Row, err error) { + var originalLine, transformedLine []byte + var originalRow, transformedRow *pgcopy.Row + + if v.config.Validate.Diff { + + originalRow = pgcopy.NewRow(len(t.Columns)) + transformedRow = pgcopy.NewRow(len(t.Columns)) + + originalLine, err = reader.ReadLine(r) + if err != nil { + if errors.Is(err, io.EOF) { + return nil, nil, err + } + return nil, nil, fmt.Errorf("unable to read line: %w", err) + } + // Handle end of dump_objects file seq + if validate_utils.LineIsEndOfData(originalLine) { + return nil, nil, io.EOF + } + + transformedLine, err = reader.ReadLine(r) + if err != nil { + return nil, nil, fmt.Errorf("unable to read line: %w", err) + } + + if err = originalRow.Decode(originalLine); err != nil { + return nil, nil, fmt.Errorf("error decoding copy line: %w", err) + } + if err = transformedRow.Decode(transformedLine); err != nil { + return nil, nil, fmt.Errorf("error decoding copy line: %w", err) + } + } else { + originalRow = pgcopy.NewRow(len(t.Columns)) + + originalLine, err = reader.ReadLine(r) + if err != nil { + if errors.Is(err, io.EOF) { + return nil, nil, err + } + return nil, nil, fmt.Errorf("unable to read line: %w", err) + } + // Handle end of dump_objects file seq + if validate_utils.LineIsEndOfData(originalLine) { + return nil, nil, io.EOF + } + + if err = originalRow.Decode(originalLine); err != nil { + return nil, nil, fmt.Errorf("error decoding copy line: %w", err) + } + } + return originalRow, transformedRow, nil +} + +func (v *ValidateV2) createDocument(ctx context.Context, t *dump_objects.Table) (validate_utils.Documenter, error) { + doc := v.getDocument(t) + + closeReader, r, err := v.getReader(ctx, t) + if err != nil { + return nil, err + } + defer closeReader() + + var line int + for { + + original, transformed, err := v.readRecords(r, t) + if err != nil { + if errors.Is(err, io.EOF) { + break + } + return nil, err + } + if err := doc.Append(original, transformed); err != nil { + return nil, fmt.Errorf("unable to append line %d to document: %w", line, err) + } + + line++ + } + + return doc, nil +} + +func (v *ValidateV2) dumpTables(ctx context.Context) error { + var tablesWithTransformers []dump_objects.Entry + for _, item := range v.context.DataSectionObjects { + + if t, ok := item.(*dump_objects.Table); ok && len(t.Transformers) > 0 { + t.ValidateLimitedRecords = v.config.Validate.RowsLimit + tablesWithTransformers = append(tablesWithTransformers, t) + } + } + v.context.DataSectionObjects = tablesWithTransformers + + if err := v.dataDump(ctx); err != nil { + return fmt.Errorf("data stage dumping error: %w", err) + } + return nil +} + +func (v *ValidateV2) printValidationWarnings() error { + // TODO: Implement warnings hook, such as logging and HTTP sender + for _, w := range v.context.Warnings { + w.MakeHash() + if idx := slices.Index(v.config.Validate.ResolvedWarnings, w.Hash); idx != -1 { + log.Debug().Str("hash", w.Hash).Msg("resolved warning has been excluded") + if w.Severity == toolkit.ErrorValidationSeverity { + return fmt.Errorf("warning with hash %s cannot be excluded because it is an error", w.Hash) + } + continue + } + + if w.Severity == toolkit.ErrorValidationSeverity { + log.Error().Any("ValidationWarning", w).Msg("") + } else { + log.Warn().Any("ValidationWarning", w).Msg("") + } + } + if v.context.IsFatal() { + return fmt.Errorf("fatal validation error") + } + return nil +} + +func (v *ValidateV2) getTablesToValidate() ([]*domains.Table, error) { + var tablesToValidate []*domains.Table + for _, tv := range v.config.Validate.Tables { + + schemaName, tableName, err := parseTableName(tv) + if err != nil { + return nil, err + } + + foundTable, err := findTableBySchemaAndName(v.config.Dump.Transformation, schemaName, tableName) + if err != nil { + return nil, err + } + + if foundTable != nil { + tablesToValidate = append(tablesToValidate, foundTable) + } + } + + if len(tablesToValidate) == 0 { + return v.config.Dump.Transformation, nil + } + + return tablesToValidate, nil +} + +func findTableBySchemaAndName(Transformations []*domains.Table, schemaName, tableName string) (*domains.Table, error) { + var foundTable *domains.Table + for _, t := range Transformations { + if t.Schema == schemaName && t.Name == tableName { + foundTable = t + break + } + if schemaName == "" && t.Name == tableName { + if foundTable != nil { + return nil, fmt.Errorf("wrong \"validate_table\" value: unable uniqually identify table \"%s\": sepcify schema name", tableName) + } + foundTable = t + } + } + if foundTable == nil { + errMsg := fmt.Sprintf("table %s is not found in transformation config", tableName) + if schemaName != "" && tableName != "" { + errMsg = fmt.Sprintf("table %s.%s is not found in transformation config", schemaName, tableName) + } + return nil, fmt.Errorf("unable to find table from \"validate_table\" parameter: %s", errMsg) + } + return foundTable, nil +} + +func parseTableName(name string) (tableName string, schemaName string, err error) { + parts := strings.Split(name, ".") + + if len(parts) > 2 { + return "", "", fmt.Errorf("wrong \"validate_table\" format \"%s\": value has %d coma symbols (.)", name, len(parts)) + } else if len(parts) == 2 { + schemaName = parts[0] + tableName = parts[1] + } else { + tableName = parts[0] + } + return +} From 88a0dea6e4814e154dea5707699a05bd36c81745 Mon Sep 17 00:00:00 2001 From: Vadim Voitenko Date: Tue, 20 Feb 2024 12:19:19 +0200 Subject: [PATCH 08/10] Implemented text horizontal printer for only changed columns --- cmd/greenmask/cmd/validate/validate.go | 18 +-- .../cmd/validate_utils/text_document.go | 135 ++++++++++++------ internal/db/postgres/cmd/validate_v2.go | 11 +- 3 files changed, 106 insertions(+), 58 deletions(-) diff --git a/cmd/greenmask/cmd/validate/validate.go b/cmd/greenmask/cmd/validate/validate.go index 43c0f320..dd9854d6 100644 --- a/cmd/greenmask/cmd/validate/validate.go +++ b/cmd/greenmask/cmd/validate/validate.go @@ -98,27 +98,27 @@ func init() { log.Fatal().Err(err).Msg("fatal") } - tableFormatFlagName := "table-format" + formatFlagName := "format" Cmd.Flags().String( - tableFormatFlagName, "horizontal", "format of table output. possible values [horizontal|vertical]", + formatFlagName, "text", "format of output. possible values [text|json]", ) - flag = Cmd.Flags().Lookup(tableFormatFlagName) - if err := viper.BindPFlag("validate.table_format", flag); err != nil { + flag = Cmd.Flags().Lookup(formatFlagName) + if err := viper.BindPFlag("validate.format", flag); err != nil { log.Fatal().Err(err).Msg("fatal") } - formatFlagName := "format" + tableFormatFlagName := "table-format" Cmd.Flags().String( - formatFlagName, "json", "format of output. possible values [text|json]", + tableFormatFlagName, "vertical", "format of table output. possible values [vertical|horizontal]", ) - flag = Cmd.Flags().Lookup(formatFlagName) - if err := viper.BindPFlag("validate.format", flag); err != nil { + flag = Cmd.Flags().Lookup(tableFormatFlagName) + if err := viper.BindPFlag("validate.table_format", flag); err != nil { log.Fatal().Err(err).Msg("fatal") } onlyTransformedFlagName := "only-transformed" Cmd.Flags().Bool( - onlyTransformedFlagName, false, "include into diff result only transformed columns and primary key value", + onlyTransformedFlagName, false, "print only transformed column and primary key", ) flag = Cmd.Flags().Lookup(onlyTransformedFlagName) if err := viper.BindPFlag("validate.only_transformed", flag); err != nil { diff --git a/internal/db/postgres/cmd/validate_utils/text_document.go b/internal/db/postgres/cmd/validate_utils/text_document.go index d7fd1fd2..4f122a65 100644 --- a/internal/db/postgres/cmd/validate_utils/text_document.go +++ b/internal/db/postgres/cmd/validate_utils/text_document.go @@ -29,10 +29,34 @@ type printSettings struct { type TextDocument struct { *JsonDocument + tableFormat string +} + +func NewTextDocument(table *dump_objects.Table, withDiff bool, onlyTransformed bool, tableFormat string) *TextDocument { + jd := NewJsonDocument(table, withDiff, onlyTransformed) + if tableFormat != horizontalTableFormatName && tableFormat != verticalTableFormatName { + panic(fmt.Sprintf("unknown table format %s", tableFormat)) + } + return &TextDocument{ + JsonDocument: jd, + tableFormat: tableFormat, + } } func (td *TextDocument) Print(w io.Writer) error { - panic("implement me") + switch td.tableFormat { + case verticalTableFormatName: + if td.withDiff { + return td.printWithDiffVertical(w) + } + return td.printPlainVertical(w) + case horizontalTableFormatName: + if td.withDiff { + return td.printWithDiffHorizontal(w) + } + return td.printPlainHorizontal(w) + } + return nil } func (td *TextDocument) printWithDiffVertical(w io.Writer) error { @@ -55,6 +79,21 @@ func (td *TextDocument) getColumnsIdxsUnexpected() []int { return res } +func (td *TextDocument) getAffectedColumns() map[int]struct{} { + res := make(map[int]struct{}) + colToPrint := td.GetAffectedColumns() + for colName := range colToPrint { + idx := slices.IndexFunc(td.table.Columns, func(column *toolkit.Column) bool { + return column.Name == colName + }) + if idx == -1 { + panic("expected column to be found in the table column list") + } + res[idx] = struct{}{} + } + return res +} + func (td *TextDocument) getColumnsIdxsToPrint() []int { var res []int colToPrint := td.GetColumnsToPrint() @@ -62,7 +101,7 @@ func (td *TextDocument) getColumnsIdxsToPrint() []int { idx := slices.IndexFunc(td.table.Columns, func(column *toolkit.Column) bool { return column.Name == colName }) - if idx != -1 { + if idx == -1 { panic("expected column to be found in the table column list") } res = append(res, idx) @@ -71,13 +110,16 @@ func (td *TextDocument) getColumnsIdxsToPrint() []int { return res } -func (td *TextDocument) getVerticalHorizontalColors(t *dump_objects.Table, affectedColumns map[int]struct{}) []tablewriter.Colors { - headerColors := make([]tablewriter.Colors, len(t.Columns)) - for idx := range t.Columns { - if _, ok := affectedColumns[idx]; ok { - headerColors[idx] = []int{tablewriter.BgRedColor} +func (td *TextDocument) getVerticalHorizontalColors() []tablewriter.Colors { + columnsToPrint := td.getColumnsIdxsToPrint() + affectedColumns := td.getAffectedColumns() + + headerColors := make([]tablewriter.Colors, len(columnsToPrint)) + for tableColIdx, colIdx := range columnsToPrint { + if _, ok := affectedColumns[colIdx]; ok { + headerColors[tableColIdx] = []int{tablewriter.BgRedColor} } else { - headerColors[idx] = []int{} + headerColors[tableColIdx] = []int{} } } // Adding formatting setting for LineNum @@ -94,47 +136,47 @@ func (td *TextDocument) printWithDiffHorizontal(w io.Writer) error { colIdxsToPrint := td.getColumnsIdxsToPrint() for lineIdx, res := range result.RecordsWithDiff { - originalRecord := make([]string, len(td.table.Columns)) - transformedRecord := make([]string, len(td.table.Columns)) - originalRecordColors := make([]tablewriter.Colors, len(td.table.Columns)) - transformedRecordColors := make([]tablewriter.Colors, len(td.table.Columns)) - for _, colIdx := range colIdxsToPrint { + originalRecord := make([]string, len(colIdxsToPrint)) + transformedRecord := make([]string, len(colIdxsToPrint)) + originalRecordColors := make([]tablewriter.Colors, len(colIdxsToPrint)) + transformedRecordColors := make([]tablewriter.Colors, len(colIdxsToPrint)) + for tableColIdx, colIdx := range colIdxsToPrint { colName := td.table.Columns[colIdx].Name colValue := res[colName] - originalRecord[colIdx] = stringsUtils.WrapString(colValue.Original, maxWrapLength) - transformedRecord[colIdx] = stringsUtils.WrapString(colValue.Transformed, maxWrapLength) + originalRecord[tableColIdx] = stringsUtils.WrapString(colValue.Original, maxWrapLength) + transformedRecord[tableColIdx] = stringsUtils.WrapString(colValue.Transformed, maxWrapLength) + originalRecordColors[tableColIdx] = []int{} + transformedRecordColors[tableColIdx] = []int{} if !colValue.Equal { - originalRecordColors[colIdx] = tablewriter.Colors{tablewriter.FgHiGreenColor} - transformedRecordColors[colIdx] = tablewriter.Colors{tablewriter.FgHiRedColor} - } else { - originalRecordColors[colIdx] = []int{} - transformedRecordColors[colIdx] = []int{} + originalRecordColors[tableColIdx] = tablewriter.Colors{tablewriter.FgHiGreenColor} + transformedRecordColors[tableColIdx] = tablewriter.Colors{tablewriter.FgHiRedColor} } - - originalRecordColors = slices.Insert(originalRecordColors, 0, tablewriter.Colors{}) - transformedRecordColors = slices.Insert(transformedRecordColors, 0, tablewriter.Colors{}) - originalRecord = slices.Insert(originalRecord, 0, fmt.Sprintf("%d", lineIdx)) - transformedRecord = slices.Insert(transformedRecord, 0, fmt.Sprintf("%d", lineIdx)) - prettyWriter.Rich(originalRecord, originalRecordColors) - prettyWriter.Rich(transformedRecord, transformedRecordColors) } + + // Adding Line number columns + originalRecordColors = slices.Insert(originalRecordColors, 0, tablewriter.Colors{}) + transformedRecordColors = slices.Insert(transformedRecordColors, 0, tablewriter.Colors{}) + originalRecord = slices.Insert(originalRecord, 0, fmt.Sprintf("%d", lineIdx)) + transformedRecord = slices.Insert(transformedRecord, 0, fmt.Sprintf("%d", lineIdx)) + + prettyWriter.Rich(originalRecord, originalRecordColors) + prettyWriter.Rich(transformedRecord, transformedRecordColors) } unexpectedlyChanged := td.GetUnexpectedlyChangedColumns() header := make([]string, len(colIdxsToPrint)) for tableColIdx, colIdx := range colIdxsToPrint { c := td.table.Columns[colIdx] + header[tableColIdx] = c.Name if _, ok := unexpectedlyChanged[c.Name]; ok { header[tableColIdx] = fmt.Sprintf("%s (!!!)", c.Name) - } else { - header[tableColIdx] = c.Name } } header = slices.Insert(header, 0, "%LineNum%") - headerColors := td.getVerticalHorizontalColors(t, realAffectedColumns) + headerColors := td.getVerticalHorizontalColors() - os.Stdout.Write([]byte(fmt.Sprintf("\n\n\t\"%s\".\"%s\"\n", t.Schema, t.Name))) + os.Stdout.Write([]byte(fmt.Sprintf("\n\n\t\"%s\".\"%s\"\n", td.table.Schema, td.table.Name))) prettyWriter.SetHeader(header) prettyWriter.SetRowLine(true) prettyWriter.SetAutoMergeCellsByColumnIndex([]int{0}) @@ -155,23 +197,24 @@ func (td *TextDocument) printPlainVertical(w io.Writer) error { } func (td *TextDocument) getHorizontalSettings() *printSettings { - affectedColumns := td.JsonDocument.GetColumnsToPrint() - - originalColumnsColors := make([]tablewriter.Colors, len(td.JsonDocument.table.Columns)) - transformedColumnsColors := make([]tablewriter.Colors, len(td.JsonDocument.table.Columns)) - headerColors := make([]tablewriter.Colors, len(td.JsonDocument.table.Columns)) - columnsAlignments := make([]int, len(td.JsonDocument.table.Columns)) - for idx, c := range td.JsonDocument.table.Columns { - if _, ok := affectedColumns[c.Name]; ok { - originalColumnsColors[idx] = []int{tablewriter.FgHiGreenColor} - transformedColumnsColors[idx] = []int{tablewriter.FgHiRedColor} - headerColors[idx] = []int{tablewriter.BgRedColor} + columnsToPrint := td.getColumnsIdxsToPrint() + affectedColumns := td.getAffectedColumns() + + originalColumnsColors := make([]tablewriter.Colors, len(columnsToPrint)) + transformedColumnsColors := make([]tablewriter.Colors, len(columnsToPrint)) + headerColors := make([]tablewriter.Colors, len(columnsToPrint)) + columnsAlignments := make([]int, len(columnsToPrint)) + for tableColIdx, colIdx := range columnsToPrint { + if _, ok := affectedColumns[colIdx]; ok { + originalColumnsColors[tableColIdx] = []int{tablewriter.FgHiGreenColor} + transformedColumnsColors[tableColIdx] = []int{tablewriter.FgHiRedColor} + headerColors[tableColIdx] = []int{tablewriter.BgRedColor} } else { - originalColumnsColors[idx] = []int{} - headerColors[idx] = []int{} - transformedColumnsColors[idx] = []int{} + originalColumnsColors[tableColIdx] = []int{} + transformedColumnsColors[tableColIdx] = []int{} + headerColors[tableColIdx] = []int{} } - columnsAlignments[idx] = tablewriter.ALIGN_LEFT + columnsAlignments[tableColIdx] = tablewriter.ALIGN_LEFT } // Adding formatting setting for LineNum originalColumnsColors = slices.Insert(originalColumnsColors, 0, tablewriter.Colors{}) diff --git a/internal/db/postgres/cmd/validate_v2.go b/internal/db/postgres/cmd/validate_v2.go index eb5d1b4f..fe2b605d 100644 --- a/internal/db/postgres/cmd/validate_v2.go +++ b/internal/db/postgres/cmd/validate_v2.go @@ -155,11 +155,16 @@ func (v *ValidateV2) print(ctx context.Context) error { } func (v *ValidateV2) getDocument(table *dump_objects.Table) validate_utils.Documenter { - if v.config.Validate.Format == jsonFormat { + switch v.config.Validate.Format { + case jsonFormat: return validate_utils.NewJsonDocument(table, v.config.Validate.Diff, v.config.Validate.OnlyTransformed) + case textFormat: + return validate_utils.NewTextDocument( + table, v.config.Validate.Diff, v.config.Validate.OnlyTransformed, v.config.Validate.TableFormat, + ) + default: + panic(fmt.Sprintf("unknown format %s", v.config.Validate.Format)) } - panic("IMPLEMENT TEXT PRINTER") - return nil } func (v *ValidateV2) getReader(ctx context.Context, table *dump_objects.Table) (closeFunc, *bufio.Reader, error) { From f2bfc72bdccc28086ca68c7a0a258deddcaf6acf Mon Sep 17 00:00:00 2001 From: Vadim Voitenko Date: Wed, 21 Feb 2024 17:22:28 +0200 Subject: [PATCH 09/10] Remastered Validate command --- cmd/greenmask/cmd/validate/validate.go | 89 ++- internal/db/postgres/cmd/dump.go | 2 +- .../cmd/{validate_v2.go => validate.go} | 107 ++- .../cmd/validate_utils/json_document.go | 73 +- .../cmd/validate_utils/text_document.go | 163 +++- internal/db/postgres/dumpers/table.go | 16 +- .../postgres/dumpers/validation_pipeline.go | 12 +- internal/domains/config.go | 3 +- old_validate/validate.go | 754 ------------------ 9 files changed, 298 insertions(+), 921 deletions(-) rename internal/db/postgres/cmd/{validate_v2.go => validate.go} (77%) delete mode 100644 old_validate/validate.go diff --git a/cmd/greenmask/cmd/validate/validate.go b/cmd/greenmask/cmd/validate/validate.go index dd9854d6..2b9fbcd2 100644 --- a/cmd/greenmask/cmd/validate/validate.go +++ b/cmd/greenmask/cmd/validate/validate.go @@ -21,7 +21,7 @@ import ( "github.com/spf13/cobra" "github.com/spf13/viper" - cmd2 "github.com/greenmaskio/greenmask/internal/db/postgres/cmd" + cmdInternals "github.com/greenmaskio/greenmask/internal/db/postgres/cmd" "github.com/greenmaskio/greenmask/internal/db/postgres/transformers/utils" "github.com/greenmaskio/greenmask/internal/domains" "github.com/greenmaskio/greenmask/internal/utils/logger" @@ -31,40 +31,56 @@ var ( Cmd = &cobra.Command{ Use: "validate", Short: "perform validation procedure and data diff of transformation", - Run: func(cmd *cobra.Command, args []string) { - if err := logger.SetLogLevel(Config.Log.Level, Config.Log.Format); err != nil { - log.Err(err).Msg("") - } + Run: run, + } + Config = domains.NewConfig() +) + +func run(cmd *cobra.Command, args []string) { + if err := logger.SetLogLevel(Config.Log.Level, Config.Log.Format); err != nil { + log.Err(err).Msg("") + } - if Config.Common.TempDirectory == "" { - log.Fatal().Msg("common.tmp_dir cannot be empty") - } + if Config.Common.TempDirectory == "" { + log.Fatal().Msg("common.tmp_dir cannot be empty") + } - if Config.Validate.RowsLimit == 0 { - log.Fatal().Msgf("--rows-limit must be greater than 0 got %d", Config.Validate.RowsLimit) - } + if Config.Validate.RowsLimit <= 0 { + log.Fatal(). + Msgf("--rows-limit must be greater than 0 got %d", Config.Validate.RowsLimit) + } - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + if Config.Validate.Format != cmdInternals.JsonFormat && + Config.Validate.Format != cmdInternals.TextFormat { + log.Fatal(). + Str("RequestedFormat", Config.Validate.Format). + Msg("unknown --format value") + } - validate, err := cmd2.NewValidateV2(Config, utils.DefaultTransformerRegistry) - if err != nil { - log.Fatal().Err(err).Msg("") - } + if Config.Validate.TableFormat != cmdInternals.VerticalTableFormat && + Config.Validate.TableFormat != cmdInternals.HorizontalTableFormat { + log.Fatal(). + Str("RequestedTableFormat", Config.Validate.TableFormat). + Msg("unknown --table-format value") + } - if err := validate.Run(ctx); err != nil { - log.Fatal().Err(err).Msg("") - } + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() - }, + validate, err := cmdInternals.NewValidate(Config, utils.DefaultTransformerRegistry) + if err != nil { + log.Fatal().Err(err).Msg("") } - Config = domains.NewConfig() -) + + if err := validate.Run(ctx); err != nil { + log.Fatal().Err(err).Msg("") + } +} func init() { tableFlagName := "table" Cmd.Flags().StringSlice( - tableFlagName, nil, "check tables dump only for specific tables", + tableFlagName, nil, "Check tables dump only for specific tables", ) flag := Cmd.Flags().Lookup(tableFlagName) if err := viper.BindPFlag("validate.tables", flag); err != nil { @@ -73,7 +89,7 @@ func init() { dataFlagName := "data" Cmd.Flags().Bool( - dataFlagName, false, "perform test dump for --rows-limit rows and print it pretty", + dataFlagName, false, "Perform test dump for --rows-limit rows and print it pretty", ) flag = Cmd.Flags().Lookup(dataFlagName) if err := viper.BindPFlag("validate.data", flag); err != nil { @@ -82,7 +98,7 @@ func init() { rowsLimitFlagName := "rows-limit" Cmd.Flags().Uint64( - rowsLimitFlagName, 10, "check tables dump only for specific tables", + rowsLimitFlagName, 10, "Check tables dump only for specific tables", ) flag = Cmd.Flags().Lookup(rowsLimitFlagName) if err := viper.BindPFlag("validate.rows_limit", flag); err != nil { @@ -91,7 +107,7 @@ func init() { diffFlagName := "diff" Cmd.Flags().Bool( - diffFlagName, false, "find difference between original and transformed data", + diffFlagName, false, "Find difference between original and transformed data", ) flag = Cmd.Flags().Lookup(diffFlagName) if err := viper.BindPFlag("validate.diff", flag); err != nil { @@ -100,7 +116,7 @@ func init() { formatFlagName := "format" Cmd.Flags().String( - formatFlagName, "text", "format of output. possible values [text|json]", + formatFlagName, "text", "Format of output. possible values [text|json]", ) flag = Cmd.Flags().Lookup(formatFlagName) if err := viper.BindPFlag("validate.format", flag); err != nil { @@ -109,19 +125,28 @@ func init() { tableFormatFlagName := "table-format" Cmd.Flags().String( - tableFormatFlagName, "vertical", "format of table output. possible values [vertical|horizontal]", + tableFormatFlagName, cmdInternals.VerticalTableFormat, "Format of table output (only for --format=text). Possible values [vertical|horizontal]", ) flag = Cmd.Flags().Lookup(tableFormatFlagName) if err := viper.BindPFlag("validate.table_format", flag); err != nil { log.Fatal().Err(err).Msg("fatal") } - onlyTransformedFlagName := "only-transformed" + onlyTransformedFlagName := "transformed-only" Cmd.Flags().Bool( - onlyTransformedFlagName, false, "print only transformed column and primary key", + onlyTransformedFlagName, false, "Print only transformed column and primary key", ) flag = Cmd.Flags().Lookup(onlyTransformedFlagName) - if err := viper.BindPFlag("validate.only_transformed", flag); err != nil { + if err := viper.BindPFlag("validate.transformed_only", flag); err != nil { + log.Fatal().Err(err).Msg("fatal") + } + + warningsFlagName := "warnings" + Cmd.Flags().Bool( + warningsFlagName, false, "Print warnings", + ) + flag = Cmd.Flags().Lookup(warningsFlagName) + if err := viper.BindPFlag("validate.warnings", flag); err != nil { log.Fatal().Err(err).Msg("fatal") } diff --git a/internal/db/postgres/cmd/dump.go b/internal/db/postgres/cmd/dump.go index 2499584f..1e0ad292 100644 --- a/internal/db/postgres/cmd/dump.go +++ b/internal/db/postgres/cmd/dump.go @@ -257,7 +257,7 @@ func (d *Dump) dataDump(ctx context.Context) error { var task dumpers.DumpTask switch v := dumpObj.(type) { case *dump_objects.Table: - task = dumpers.NewTableDumper(v, d.validate, d.config.Validate.Diff) + task = dumpers.NewTableDumper(v, d.validate) case *dump_objects.Sequence: task = dumpers.NewSequenceDumper(v) case *dump_objects.Blobs: diff --git a/internal/db/postgres/cmd/validate_v2.go b/internal/db/postgres/cmd/validate.go similarity index 77% rename from internal/db/postgres/cmd/validate_v2.go rename to internal/db/postgres/cmd/validate.go index fe2b605d..55eab12b 100644 --- a/internal/db/postgres/cmd/validate_v2.go +++ b/internal/db/postgres/cmd/validate.go @@ -27,23 +27,27 @@ import ( "github.com/greenmaskio/greenmask/internal/storages" "github.com/greenmaskio/greenmask/internal/storages/directory" "github.com/greenmaskio/greenmask/internal/utils/reader" - //"github.com/greenmaskio/greenmask/old_validate" "github.com/greenmaskio/greenmask/pkg/toolkit" ) const ( - jsonFormat string = "json" - textFormat string = "text" + JsonFormat string = "json" + TextFormat string = "text" +) + +const ( + VerticalTableFormat = "vertical" + HorizontalTableFormat = "horizontal" ) type closeFunc func() -type ValidateV2 struct { +type Validate struct { *Dump tmpDir string } -func NewValidateV2(cfg *domains.Config, registry *utils.TransformerRegistry) (*ValidateV2, error) { +func NewValidate(cfg *domains.Config, registry *utils.TransformerRegistry) (*Validate, error) { var st storages.Storager st, err := directory.NewStorage(&directory.Config{Path: cfg.Common.TempDirectory}) if err != nil { @@ -55,13 +59,13 @@ func NewValidateV2(cfg *domains.Config, registry *utils.TransformerRegistry) (*V d := NewDump(cfg, st, registry) d.dumpIdSequence = toc.NewDumpSequence(0) d.validate = true - return &ValidateV2{ + return &Validate{ Dump: d, tmpDir: path.Join(cfg.Common.TempDirectory, tmpDir), }, nil } -func (v *ValidateV2) Run(ctx context.Context) error { +func (v *Validate) Run(ctx context.Context) error { defer func() { // Deleting temp dir after closing it @@ -134,7 +138,7 @@ func (v *ValidateV2) Run(ctx context.Context) error { return nil } -func (v *ValidateV2) print(ctx context.Context) error { +func (v *Validate) print(ctx context.Context) error { for _, e := range v.dataEntries { idx := slices.IndexFunc(v.context.DataSectionObjects, func(entry dump_objects.Entry) bool { t := entry.(*dump_objects.Table) @@ -154,11 +158,11 @@ func (v *ValidateV2) print(ctx context.Context) error { return nil } -func (v *ValidateV2) getDocument(table *dump_objects.Table) validate_utils.Documenter { +func (v *Validate) getDocument(table *dump_objects.Table) validate_utils.Documenter { switch v.config.Validate.Format { - case jsonFormat: + case JsonFormat: return validate_utils.NewJsonDocument(table, v.config.Validate.Diff, v.config.Validate.OnlyTransformed) - case textFormat: + case TextFormat: return validate_utils.NewTextDocument( table, v.config.Validate.Diff, v.config.Validate.OnlyTransformed, v.config.Validate.TableFormat, ) @@ -167,7 +171,7 @@ func (v *ValidateV2) getDocument(table *dump_objects.Table) validate_utils.Docum } } -func (v *ValidateV2) getReader(ctx context.Context, table *dump_objects.Table) (closeFunc, *bufio.Reader, error) { +func (v *Validate) getReader(ctx context.Context, table *dump_objects.Table) (closeFunc, *bufio.Reader, error) { tableData, err := v.st.GetObject(ctx, fmt.Sprintf("%d.dat.gz", table.DumpId)) if err != nil { return nil, nil, fmt.Errorf("unable to get object from storage: %w", err) @@ -191,61 +195,40 @@ func (v *ValidateV2) getReader(ctx context.Context, table *dump_objects.Table) ( return f, bufio.NewReader(gz), nil } -func (v *ValidateV2) readRecords(r *bufio.Reader, t *dump_objects.Table) (original, transformed *pgcopy.Row, err error) { +func (v *Validate) readRecords(r *bufio.Reader, t *dump_objects.Table) (original, transformed *pgcopy.Row, err error) { var originalLine, transformedLine []byte var originalRow, transformedRow *pgcopy.Row - if v.config.Validate.Diff { - - originalRow = pgcopy.NewRow(len(t.Columns)) - transformedRow = pgcopy.NewRow(len(t.Columns)) - - originalLine, err = reader.ReadLine(r) - if err != nil { - if errors.Is(err, io.EOF) { - return nil, nil, err - } - return nil, nil, fmt.Errorf("unable to read line: %w", err) - } - // Handle end of dump_objects file seq - if validate_utils.LineIsEndOfData(originalLine) { - return nil, nil, io.EOF - } - - transformedLine, err = reader.ReadLine(r) - if err != nil { - return nil, nil, fmt.Errorf("unable to read line: %w", err) - } + originalRow = pgcopy.NewRow(len(t.Columns)) + transformedRow = pgcopy.NewRow(len(t.Columns)) - if err = originalRow.Decode(originalLine); err != nil { - return nil, nil, fmt.Errorf("error decoding copy line: %w", err) - } - if err = transformedRow.Decode(transformedLine); err != nil { - return nil, nil, fmt.Errorf("error decoding copy line: %w", err) + originalLine, err = reader.ReadLine(r) + if err != nil { + if errors.Is(err, io.EOF) { + return nil, nil, err } - } else { - originalRow = pgcopy.NewRow(len(t.Columns)) + return nil, nil, fmt.Errorf("unable to read line: %w", err) + } + // Handle end of dump_objects file seq + if validate_utils.LineIsEndOfData(originalLine) { + return nil, nil, io.EOF + } - originalLine, err = reader.ReadLine(r) - if err != nil { - if errors.Is(err, io.EOF) { - return nil, nil, err - } - return nil, nil, fmt.Errorf("unable to read line: %w", err) - } - // Handle end of dump_objects file seq - if validate_utils.LineIsEndOfData(originalLine) { - return nil, nil, io.EOF - } + transformedLine, err = reader.ReadLine(r) + if err != nil { + return nil, nil, fmt.Errorf("unable to read line: %w", err) + } - if err = originalRow.Decode(originalLine); err != nil { - return nil, nil, fmt.Errorf("error decoding copy line: %w", err) - } + if err = originalRow.Decode(originalLine); err != nil { + return nil, nil, fmt.Errorf("error decoding copy line: %w", err) + } + if err = transformedRow.Decode(transformedLine); err != nil { + return nil, nil, fmt.Errorf("error decoding copy line: %w", err) } return originalRow, transformedRow, nil } -func (v *ValidateV2) createDocument(ctx context.Context, t *dump_objects.Table) (validate_utils.Documenter, error) { +func (v *Validate) createDocument(ctx context.Context, t *dump_objects.Table) (validate_utils.Documenter, error) { doc := v.getDocument(t) closeReader, r, err := v.getReader(ctx, t) @@ -274,7 +257,7 @@ func (v *ValidateV2) createDocument(ctx context.Context, t *dump_objects.Table) return doc, nil } -func (v *ValidateV2) dumpTables(ctx context.Context) error { +func (v *Validate) dumpTables(ctx context.Context) error { var tablesWithTransformers []dump_objects.Entry for _, item := range v.context.DataSectionObjects { @@ -291,7 +274,7 @@ func (v *ValidateV2) dumpTables(ctx context.Context) error { return nil } -func (v *ValidateV2) printValidationWarnings() error { +func (v *Validate) printValidationWarnings() error { // TODO: Implement warnings hook, such as logging and HTTP sender for _, w := range v.context.Warnings { w.MakeHash() @@ -304,9 +287,13 @@ func (v *ValidateV2) printValidationWarnings() error { } if w.Severity == toolkit.ErrorValidationSeverity { + // The warnings with error severity must be printed anyway log.Error().Any("ValidationWarning", w).Msg("") } else { - log.Warn().Any("ValidationWarning", w).Msg("") + // Print warnings with severity level lower than ErrorValidationSeverity only if requested + if v.config.Validate.Warnings { + log.Warn().Any("ValidationWarning", w).Msg("") + } } } if v.context.IsFatal() { @@ -315,7 +302,7 @@ func (v *ValidateV2) printValidationWarnings() error { return nil } -func (v *ValidateV2) getTablesToValidate() ([]*domains.Table, error) { +func (v *Validate) getTablesToValidate() ([]*domains.Table, error) { var tablesToValidate []*domains.Table for _, tv := range v.config.Validate.Tables { diff --git a/internal/db/postgres/cmd/validate_utils/json_document.go b/internal/db/postgres/cmd/validate_utils/json_document.go index df303030..99d11c35 100644 --- a/internal/db/postgres/cmd/validate_utils/json_document.go +++ b/internal/db/postgres/cmd/validate_utils/json_document.go @@ -39,14 +39,17 @@ type jsonDocumentResponseWithDiff struct { Name string `json:"name"` PrimaryKeyColumns []string `json:"primary_key_columns"` WithDiff bool `json:"with_diff"` - OnlyTransformed bool `json:"only_transformed"` + TransformedOnly bool `json:"transformed_only"` Records []jsonRecordWithDiff `json:"records"` } type jsonDocumentResponsePlain struct { - Schema string `json:"schema"` - Name string `json:"name"` - Records []jsonRecordPlain `json:"records"` + Schema string `json:"schema"` + Name string `json:"name"` + PrimaryKeyColumns []string `json:"primary_key_columns"` + WithDiff bool `json:"with_diff"` + TransformedOnly bool `json:"transformed_only"` + Records []jsonRecordPlain `json:"records"` } type jsonRecordWithDiff map[string]*valueWithDiff @@ -89,7 +92,7 @@ func NewJsonDocument(table *dump_objects.Table, withDiff bool, onlyTransformed b } } -func (jc *JsonDocument) appendWithDiff(original, transformed *pgcopy.Row) error { +func (jc *JsonDocument) Append(original, transformed *pgcopy.Row) error { r := make(jsonRecordWithDiff) for idx, c := range jc.table.Columns { originalRawValue, err := original.GetColumn(idx) @@ -127,33 +130,6 @@ func (jc *JsonDocument) appendWithDiff(original, transformed *pgcopy.Row) error return nil } -func (jc *JsonDocument) appendPlain(original *pgcopy.Row) error { - r := make(jsonRecordPlain) - for idx, c := range jc.table.Columns { - originalRawValue, err := original.GetColumn(idx) - if err != nil { - return fmt.Errorf("error getting column from original record: %w", err) - } - - r[c.Name] = getStringFromRawValue(originalRawValue) - } - jc.result.RecordsPlain = append(jc.result.RecordsPlain, r) - return nil -} - -func (jc *JsonDocument) Append(original, transformed *pgcopy.Row) error { - if jc.withDiff { - if err := jc.appendWithDiff(original, transformed); err != nil { - return fmt.Errorf("error appending data with diff: %w", err) - } - } else { - if err := jc.appendPlain(original); err != nil { - return fmt.Errorf("error appending data without diff: %w", err) - } - } - return nil -} - func (jc *JsonDocument) Print(w io.Writer) error { result := jc.Get() @@ -163,7 +139,7 @@ func (jc *JsonDocument) Print(w io.Writer) error { Name: result.Name, PrimaryKeyColumns: result.PrimaryKeyColumns, WithDiff: result.WithDiff, - OnlyTransformed: result.OnlyTransformed, + TransformedOnly: result.OnlyTransformed, Records: result.RecordsWithDiff, } if err := json.NewEncoder(w).Encode(response); err != nil { @@ -172,10 +148,23 @@ func (jc *JsonDocument) Print(w io.Writer) error { return nil } + records := make([]jsonRecordPlain, len(result.RecordsWithDiff)) + + for idx := range records { + record := make(map[string]string, len(result.RecordsWithDiff[idx])) + for name, value := range result.RecordsWithDiff[idx] { + record[name] = value.Transformed + } + records = append(records, record) + } + response := &jsonDocumentResponsePlain{ - Schema: result.Schema, - Name: result.Name, - Records: result.RecordsPlain, + Schema: result.Schema, + Name: result.Name, + PrimaryKeyColumns: result.PrimaryKeyColumns, + WithDiff: result.WithDiff, + TransformedOnly: result.OnlyTransformed, + Records: records, } if err := json.NewEncoder(w).Encode(response); err != nil { return err @@ -216,18 +205,6 @@ func (jc *JsonDocument) Get() *JsonDocumentResult { return jc.result } -func (jc *JsonDocument) Marshal() ([]byte, error) { - // TODO: - // 1. Return all columns if requested - // 2. Return only transformed data. - // 2.1 Analyze affected columns + unexpectedly affected - // 2.2 Return all affected columns with data + primary key - if jc.onlyTransformed { - jc.filterColumns() - } - return json.Marshal(jc.result) -} - func (jc *JsonDocument) filterColumns() { // Determine list of the affected columns columnsToPrint := jc.GetColumnsToPrint() diff --git a/internal/db/postgres/cmd/validate_utils/text_document.go b/internal/db/postgres/cmd/validate_utils/text_document.go index 4f122a65..07fca604 100644 --- a/internal/db/postgres/cmd/validate_utils/text_document.go +++ b/internal/db/postgres/cmd/validate_utils/text_document.go @@ -46,10 +46,7 @@ func NewTextDocument(table *dump_objects.Table, withDiff bool, onlyTransformed b func (td *TextDocument) Print(w io.Writer) error { switch td.tableFormat { case verticalTableFormatName: - if td.withDiff { - return td.printWithDiffVertical(w) - } - return td.printPlainVertical(w) + return td.printVertical(w) case horizontalTableFormatName: if td.withDiff { return td.printWithDiffHorizontal(w) @@ -59,8 +56,97 @@ func (td *TextDocument) Print(w io.Writer) error { return nil } -func (td *TextDocument) printWithDiffVertical(w io.Writer) error { - panic("implement me") +func (td *TextDocument) getVerticalRowColors(affectedColumns map[int]struct{}, value *valueWithDiff) []tablewriter.Colors { + var colors []tablewriter.Colors + + colors = make([]tablewriter.Colors, 4) + if !td.withDiff { + colors = make([]tablewriter.Colors, 3) + } + colors[0] = tablewriter.Colors{} + + colors[1] = tablewriter.Colors{} + if !value.Equal { + colors[1] = tablewriter.Colors{tablewriter.BgRedColor} + } + + if td.withDiff { + colors[2] = tablewriter.Colors{} + colors[3] = tablewriter.Colors{} + if !value.Equal { + colors[2] = tablewriter.Colors{tablewriter.FgHiGreenColor} + colors[3] = tablewriter.Colors{tablewriter.FgHiRedColor} + } + } else { + colors[2] = tablewriter.Colors{} + if !value.Equal { + colors[2] = tablewriter.Colors{tablewriter.FgHiRedColor} + } + } + return colors +} + +func (td *TextDocument) printVertical(w io.Writer) error { + + recordSize := 3 + if td.withDiff { + recordSize = 4 + } + + headerColorSetting := []int{tablewriter.Bold} + alignmentSettings := tablewriter.ALIGN_LEFT + headerColors := make([]tablewriter.Colors, recordSize) + columnAlignments := make([]int, recordSize) + for idx := range headerColors { + headerColors[idx] = headerColorSetting + columnAlignments[idx] = alignmentSettings + } + + prettyWriter := tablewriter.NewWriter(os.Stdout) + prettyWriter.SetAutoMergeCellsByColumnIndex([]int{0}) + prettyWriter.SetColumnAlignment(columnAlignments) + prettyWriter.SetAutoWrapText(true) + prettyWriter.SetHeaderLine(true) + prettyWriter.SetRowLine(true) + + header := []string{"%LineNum%", "Column", "Value"} + if td.withDiff { + header = []string{"%LineNum%", "Column", "OriginalValue", "TransformedValue"} + } + prettyWriter.Rich(header, headerColors) + affectedColumns := td.getAffectedColumns() + + result := td.JsonDocument.Get() + colIdxsToPrint := td.getColumnsIdxsToPrint() + + for lineIdx, res := range result.RecordsWithDiff { + for _, colIdx := range colIdxsToPrint { + record := make([]string, recordSize) + record[0] = fmt.Sprintf("%d", lineIdx) + colName := td.table.Columns[colIdx].Name + colValue := res[colName] + record[1] = colName + if td.withDiff { + record[2] = colValue.Original + record[3] = colValue.Transformed + } else { + record[2] = colValue.Transformed + } + + colors := td.getVerticalRowColors(affectedColumns, colValue) + if !colValue.Expected { + record[1] = fmt.Sprintf("%s (!!!)", colName) + } + prettyWriter.Rich(record, colors) + } + } + + if err := td.writeTableTitle(w); err != nil { + return err + } + prettyWriter.Render() + + return nil } func (td *TextDocument) getColumnsIdxsUnexpected() []int { @@ -176,7 +262,9 @@ func (td *TextDocument) printWithDiffHorizontal(w io.Writer) error { header = slices.Insert(header, 0, "%LineNum%") headerColors := td.getVerticalHorizontalColors() - os.Stdout.Write([]byte(fmt.Sprintf("\n\n\t\"%s\".\"%s\"\n", td.table.Schema, td.table.Name))) + if err := td.writeTableTitle(w); err != nil { + return err + } prettyWriter.SetHeader(header) prettyWriter.SetRowLine(true) prettyWriter.SetAutoMergeCellsByColumnIndex([]int{0}) @@ -188,8 +276,67 @@ func (td *TextDocument) printWithDiffHorizontal(w io.Writer) error { return nil } +func (td *TextDocument) writeTableTitle(w io.Writer) error { + _, err := w.Write([]byte(fmt.Sprintf("\n\n\t\"%s\".\"%s\"\n", td.table.Schema, td.table.Name))) + if err != nil { + return fmt.Errorf("error writing title: %w", err) + } + return nil +} + func (td *TextDocument) printPlainHorizontal(w io.Writer) error { - panic("implement me") + settings := td.getHorizontalSettings() + prettyWriter := tablewriter.NewWriter(w) + prettyWriter.SetColumnAlignment(settings.ColumnsAlignments) + + result := td.JsonDocument.Get() + colIdxsToPrint := td.getColumnsIdxsToPrint() + + for lineIdx, res := range result.RecordsWithDiff { + transformedRecord := make([]string, len(colIdxsToPrint)) + transformedRecordColors := make([]tablewriter.Colors, len(colIdxsToPrint)) + for tableColIdx, colIdx := range colIdxsToPrint { + colName := td.table.Columns[colIdx].Name + colValue := res[colName] + transformedRecord[tableColIdx] = stringsUtils.WrapString(colValue.Transformed, maxWrapLength) + + transformedRecordColors[tableColIdx] = []int{} + if !colValue.Equal { + transformedRecordColors[tableColIdx] = tablewriter.Colors{tablewriter.FgHiRedColor} + } + } + + // Adding Line number columns + transformedRecordColors = slices.Insert(transformedRecordColors, 0, tablewriter.Colors{}) + transformedRecord = slices.Insert(transformedRecord, 0, fmt.Sprintf("%d", lineIdx)) + + prettyWriter.Rich(transformedRecord, transformedRecordColors) + } + + unexpectedlyChanged := td.GetUnexpectedlyChangedColumns() + header := make([]string, len(colIdxsToPrint)) + for tableColIdx, colIdx := range colIdxsToPrint { + c := td.table.Columns[colIdx] + header[tableColIdx] = c.Name + if _, ok := unexpectedlyChanged[c.Name]; ok { + header[tableColIdx] = fmt.Sprintf("%s (!!!)", c.Name) + } + } + header = slices.Insert(header, 0, "%LineNum%") + headerColors := td.getVerticalHorizontalColors() + + if err := td.writeTableTitle(w); err != nil { + return err + } + prettyWriter.SetHeader(header) + prettyWriter.SetRowLine(true) + prettyWriter.SetAutoMergeCellsByColumnIndex([]int{0}) + prettyWriter.SetAutoWrapText(true) + prettyWriter.SetHeaderLine(true) + prettyWriter.SetHeaderColor(headerColors...) + + prettyWriter.Render() + return nil } func (td *TextDocument) printPlainVertical(w io.Writer) error { diff --git a/internal/db/postgres/dumpers/table.go b/internal/db/postgres/dumpers/table.go index 56dbba77..2ba74092 100644 --- a/internal/db/postgres/dumpers/table.go +++ b/internal/db/postgres/dumpers/table.go @@ -30,17 +30,15 @@ import ( ) type TableDumper struct { - table *dump.Table - recordNum uint64 - validate bool - validateWithOriginal bool + table *dump.Table + recordNum uint64 + validate bool } -func NewTableDumper(table *dump.Table, validate bool, validateWithOriginal bool) *TableDumper { +func NewTableDumper(table *dump.Table, validate bool) *TableDumper { return &TableDumper{ - table: table, - validate: validate, - validateWithOriginal: validateWithOriginal, + table: table, + validate: validate, } } @@ -73,7 +71,7 @@ func (td *TableDumper) Execute(ctx context.Context, tx pgx.Tx, st storages.Stora var err error if len(td.table.Transformers) > 0 { if td.validate { - pipeline, err = NewValidationPipeline(gtx, eg, td.table, w, td.validateWithOriginal) + pipeline, err = NewValidationPipeline(gtx, eg, td.table, w) if err != nil { return fmt.Errorf("cannot initialize validation pipeline: %w", err) } diff --git a/internal/db/postgres/dumpers/validation_pipeline.go b/internal/db/postgres/dumpers/validation_pipeline.go index 112a717c..f92bdbcd 100644 --- a/internal/db/postgres/dumpers/validation_pipeline.go +++ b/internal/db/postgres/dumpers/validation_pipeline.go @@ -12,26 +12,22 @@ import ( type ValidationPipeline struct { *TransformationPipeline - withOriginal bool } -func NewValidationPipeline(ctx context.Context, eg *errgroup.Group, table *dump.Table, w io.Writer, withOriginal bool) (*ValidationPipeline, error) { +func NewValidationPipeline(ctx context.Context, eg *errgroup.Group, table *dump.Table, w io.Writer) (*ValidationPipeline, error) { tpp, err := NewTransformationPipeline(ctx, eg, table, w) if err != nil { return nil, err } return &ValidationPipeline{ TransformationPipeline: tpp, - withOriginal: withOriginal, }, err } func (vp *ValidationPipeline) Dump(ctx context.Context, data []byte) (err error) { - if vp.withOriginal { - _, err = vp.w.Write(data) - if err != nil { - return NewDumpError(vp.table.Schema, vp.table.Name, vp.line, fmt.Errorf("error writing original dumped data: %w", err)) - } + _, err = vp.w.Write(data) + if err != nil { + return NewDumpError(vp.table.Schema, vp.table.Name, vp.line, fmt.Errorf("error writing original dumped data: %w", err)) } return vp.TransformationPipeline.Dump(ctx, data) diff --git a/internal/domains/config.go b/internal/domains/config.go index b606c7c7..c179c457 100644 --- a/internal/domains/config.go +++ b/internal/domains/config.go @@ -61,7 +61,8 @@ type Validate struct { ResolvedWarnings []string `mapstructure:"resolved_warnings" yaml:"resolved_warnings" json:"resolved_warnings,omitempty"` TableFormat string `mapstructure:"table_format" yaml:"table_format" json:"table_format,omitempty"` Format string `mapstructure:"format" yaml:"format" json:"format,omitempty"` - OnlyTransformed bool `mapstructure:"only_transformed" yaml:"only_transformed" json:"only_transformed,omitempty"` + OnlyTransformed bool `mapstructure:"transformed_only" yaml:"transformed_only" json:"transformed_only,omitempty"` + Warnings bool `mapstructure:"warnings" yaml:"warnings" json:"warnings,omitempty"` } type Common struct { diff --git a/old_validate/validate.go b/old_validate/validate.go deleted file mode 100644 index 17083100..00000000 --- a/old_validate/validate.go +++ /dev/null @@ -1,754 +0,0 @@ -package old_validate - -import ( - "bufio" - "compress/gzip" - "context" - "errors" - "fmt" - "io" - "os" - "path" - "slices" - "strconv" - "strings" - "time" - - "github.com/olekukonko/tablewriter" - "github.com/rs/zerolog/log" - - "github.com/greenmaskio/greenmask/internal/db/postgres/cmd" - "github.com/greenmaskio/greenmask/internal/db/postgres/cmd/validate_utils" - runtimeContext "github.com/greenmaskio/greenmask/internal/db/postgres/context" - "github.com/greenmaskio/greenmask/internal/db/postgres/dump_objects" - "github.com/greenmaskio/greenmask/internal/db/postgres/pgcopy" - "github.com/greenmaskio/greenmask/internal/db/postgres/toc" - "github.com/greenmaskio/greenmask/internal/db/postgres/transformers/custom" - "github.com/greenmaskio/greenmask/internal/db/postgres/transformers/utils" - "github.com/greenmaskio/greenmask/internal/domains" - "github.com/greenmaskio/greenmask/internal/storages" - "github.com/greenmaskio/greenmask/internal/storages/directory" - "github.com/greenmaskio/greenmask/internal/utils/reader" - stringsUtils "github.com/greenmaskio/greenmask/internal/utils/strings" - "github.com/greenmaskio/greenmask/pkg/toolkit" -) - -const nullStringValue = "NULL" - -const ( - horizontalTableFormatName = "horizontal" - verticalTableFormatName = "vertical" - - maxWrapLength = 64 - - jsonFormat string = "json" - textFormat string = "json" -) - -type printSettings struct { - OriginalColors []tablewriter.Colors - TransformedColors []tablewriter.Colors - HeaderColors []tablewriter.Colors - ColumnsAlignments []int -} - -type Validate struct { - *cmd.Dump - tmpDir string -} - -func NewValidate(cfg *domains.Config, registry *utils.TransformerRegistry) (*Validate, error) { - var st storages.Storager - st, err := directory.NewStorage(&directory.Config{Path: cfg.Common.TempDirectory}) - if err != nil { - return nil, fmt.Errorf("error initializing storage") - } - tmpDir := strconv.FormatInt(time.Now().UnixMilli(), 10) - st = st.SubStorage(tmpDir, true) - - d := cmd.NewDump(cfg, st, registry) - d.dumpIdSequence = toc.NewDumpSequence(0) - d.validate = true - return &Validate{ - Dump: d, - tmpDir: path.Join(cfg.Common.TempDirectory, tmpDir), - }, nil -} - -func (v *Validate) Run(ctx context.Context) error { - - defer func() { - // Deleting temp dir after closing it - if err := os.RemoveAll(v.tmpDir); err != nil { - log.Warn().Err(err).Msgf("unable to delete temp directory") - } - }() - if err := custom.BootstrapCustomTransformers(ctx, v.registry, v.config.CustomTransformers); err != nil { - return fmt.Errorf("error bootstraping custom transformers: %w", err) - } - - dsn, err := v.pgDumpOptions.GetPgDSN() - if err != nil { - return fmt.Errorf("cannot build connection string: %w", err) - } - - conn, err := v.connect(ctx, dsn) - if err != nil { - return err - } - defer func() { - if err := conn.Close(ctx); err != nil { - log.Warn().Err(err) - } - }() - - tx, err := v.startMainTx(ctx, conn) - if err != nil { - return fmt.Errorf("cannot prepare backup transaction: %w", err) - } - defer func() { - if err := tx.Rollback(ctx); err != nil { - log.Warn().Err(err) - } - }() - - if err = v.gatherPgFacts(ctx, tx); err != nil { - return fmt.Errorf("error gathering facts: %w", err) - } - - // filter tables that must be validated if empty validate all - var tablesToValidate []*domains.Table - for _, tv := range v.config.Validate.Tables { - var schemaName, tableName string - parts := strings.Split(tv, ".") - if len(parts) > 2 { - return fmt.Errorf("wrong \"validate_table\" format \"%s\": value has %d coma symbols (.)", tv, len(parts)) - } else if len(parts) == 2 { - schemaName = parts[0] - tableName = parts[1] - } else { - tableName = parts[0] - } - - var foundTable *domains.Table - for _, t := range v.config.Dump.Transformation { - if t.Schema == schemaName && t.Name == tableName { - foundTable = t - break - } - if schemaName == "" && t.Name == tableName { - if foundTable != nil { - return fmt.Errorf("wrong \"validate_table\" value: unable uniqually identify table \"%s\": sepcify schema name", tv) - } - foundTable = t - } - } - if foundTable != nil { - tablesToValidate = append(tablesToValidate, foundTable) - } else { - return fmt.Errorf("unable to find table from \"validate_table\" parameter: table %s is not found in transformation config", tv) - } - } - - if len(tablesToValidate) > 0 { - v.config.Dump.Transformation = tablesToValidate - } - - v.context, err = runtimeContext.NewRuntimeContext(ctx, tx, v.config.Dump.Transformation, v.registry, - v.pgDumpOptions, v.version) - if err != nil { - return fmt.Errorf("unable to build runtime context: %w", err) - } - // TODO: Implement warnings hook, such as logging and HTTP sender - for _, w := range v.context.Warnings { - w.MakeHash() - if idx := slices.Index(v.config.Validate.ResolvedWarnings, w.Hash); idx != -1 { - log.Debug().Str("hash", w.Hash).Msg("resolved warning has been excluded") - if w.Severity == toolkit.ErrorValidationSeverity { - return fmt.Errorf("warning with hash %s cannot be excluded because it is an error", w.Hash) - } - continue - } - - if w.Severity == toolkit.ErrorValidationSeverity { - log.Error().Any("ValidationWarning", w).Msg("") - } else { - log.Warn().Any("ValidationWarning", w).Msg("") - } - } - if v.context.IsFatal() { - return fmt.Errorf("fatal validation error") - } - - if !v.config.Validate.Data { - return nil - } - - var tablesWithTransformers []dump_objects.Entry - for _, item := range v.context.DataSectionObjects { - - if t, ok := item.(*dump_objects.Table); ok && len(t.Transformers) > 0 { - t.ValidateLimitedRecords = v.config.Validate.RowsLimit - tablesWithTransformers = append(tablesWithTransformers, t) - } - } - v.context.DataSectionObjects = tablesWithTransformers - - if err = v.dataDump(ctx); err != nil { - return fmt.Errorf("data stage dumping error: %w", err) - } - - for _, e := range v.dataEntries { - idx := slices.IndexFunc(v.context.DataSectionObjects, func(entry dump_objects.Entry) bool { - t := entry.(*dump_objects.Table) - return t.DumpId == e.DumpId - }) - - t := v.context.DataSectionObjects[idx].(*dump_objects.Table) - - if err = v.printText(ctx, t); err != nil { - return fmt.Errorf("error pretty printing table \"%s\".\"%s\": %w", t.Table.Schema, t.Table.Name, err) - } - } - - return nil -} - -func (v *Validate) getVerticalRowColors(affectedColumns map[int]struct{}, columnIdx int, original, transformed *toolkit.RawValue) ([]tablewriter.Colors, bool) { - var colors []tablewriter.Colors - var isEqual bool - if v.config.Validate.Diff { - isEqual = validate_utils.ValuesEqual(original, transformed) - colors = make([]tablewriter.Colors, 4) - } else { - colors = make([]tablewriter.Colors, 3) - } - colors[0] = tablewriter.Colors{} - _, affected := affectedColumns[columnIdx] - if affected || (v.config.Validate.Diff && !isEqual) { - colors[1] = tablewriter.Colors{tablewriter.BgRedColor} - } else { - colors[1] = tablewriter.Colors{} - } - - if v.config.Validate.Diff { - if !isEqual { - colors[2] = tablewriter.Colors{tablewriter.FgHiGreenColor} - colors[3] = tablewriter.Colors{tablewriter.FgHiRedColor} - } else { - colors[2] = tablewriter.Colors{} - colors[3] = tablewriter.Colors{} - } - } else { - if affected { - colors[2] = tablewriter.Colors{tablewriter.FgHiRedColor} - } else { - colors[2] = tablewriter.Colors{} - } - - } - return colors, isEqual -} - -func (v *Validate) getAffectedColumns(t *dump_objects.Table) map[int]struct{} { - affectedColumns := make(map[int]struct{}) - for _, tr := range t.Transformers { - ac := tr.GetAffectedColumns() - for idx := range ac { - affectedColumns[idx] = struct{}{} - } - } - return affectedColumns -} - -func (v *Validate) getHorizontalSettings(t *dump_objects.Table) *printSettings { - affectedColumns := v.getAffectedColumns(t) - - originalColumnsColors := make([]tablewriter.Colors, len(t.Columns)) - transformedColumnsColors := make([]tablewriter.Colors, len(t.Columns)) - headerColors := make([]tablewriter.Colors, len(t.Columns)) - columnsAlignments := make([]int, len(t.Columns)) - for idx := range t.Columns { - if _, ok := affectedColumns[idx]; ok { - originalColumnsColors[idx] = []int{tablewriter.FgHiGreenColor} - transformedColumnsColors[idx] = []int{tablewriter.FgHiRedColor} - headerColors[idx] = []int{tablewriter.BgRedColor} - } else { - originalColumnsColors[idx] = []int{} - headerColors[idx] = []int{} - transformedColumnsColors[idx] = []int{} - } - columnsAlignments[idx] = tablewriter.ALIGN_LEFT - } - // Adding formatting setting for LineNum - originalColumnsColors = slices.Insert(originalColumnsColors, 0, tablewriter.Colors{}) - headerColors = slices.Insert(headerColors, 0, tablewriter.Colors{}) - transformedColumnsColors = slices.Insert(transformedColumnsColors, 0, tablewriter.Colors{}) - columnsAlignments = slices.Insert(columnsAlignments, 0, tablewriter.ALIGN_LEFT) - - return &printSettings{ - OriginalColors: originalColumnsColors, - TransformedColors: transformedColumnsColors, - HeaderColors: headerColors, - ColumnsAlignments: columnsAlignments, - } - -} - -func (v *Validate) printHorizontally(ctx context.Context, t *dump_objects.Table) error { - settings := v.getHorizontalSettings(t) - - prettyWriter := tablewriter.NewWriter(os.Stdout) - prettyWriter.SetColumnAlignment(settings.ColumnsAlignments) - - row := *pgcopy.NewRow(len(t.Columns)) - tableData, err := v.st.GetObject(ctx, fmt.Sprintf("%d.dat.gz", t.DumpId)) - if err != nil { - log.Err(err).Msg("") - } - defer tableData.Close() - gz, err := gzip.NewReader(tableData) - if err != nil { - return fmt.Errorf("cannot create gzip reader: %w", err) - } - defer gz.Close() - r := bufio.NewReader(gz) - - var lineNum = 1 - realAffectedColumns := v.getAffectedColumns(t) - diffValues := make([][]*toolkit.RawValue, len(t.Columns)) - for idx := range t.Columns { - diffValues[idx] = make([]*toolkit.RawValue, 2) - } - for { - line, err := reader.ReadLine(r) - if err != nil { - if errors.Is(err, io.EOF) { - break - } - return fmt.Errorf("unable to read line: %w", err) - } - - // Handle end of dump_objects file seq - if validate_utils.LineIsEndOfData(line) { - break - } - - if err = row.Decode(line); err != nil { - return fmt.Errorf("error decoding copy line: %w", err) - } - record := make([]string, len(t.Columns)) - - for idx, c := range t.Columns { - value, err := row.GetColumn(idx) - if err != nil { - return fmt.Errorf("unable to get column \"%s\" value: %w", c.Name, err) - } - if value.IsNull { - record[idx] = nullStringValue - } else { - record[idx] = stringsUtils.WrapString(string(value.Data), maxWrapLength) - } - } - - record = slices.Insert(record, 0, fmt.Sprintf("%d", lineNum)) - - colors := settings.TransformedColors - prettyWriter.Rich(record, colors) - lineNum++ - } - - header := make([]string, len(t.Columns)) - for idx, c := range t.Columns { - header[idx] = c.Name - } - header = slices.Insert(header, 0, "%LineNum%") - headerColors := v.getVerticalHeaderColors(t, realAffectedColumns) - - os.Stdout.Write([]byte(fmt.Sprintf("\n\n\t\"%s\".\"%s\"\n", t.Schema, t.Name))) - prettyWriter.SetHeader(header) - prettyWriter.SetRowLine(true) - prettyWriter.SetAutoMergeCellsByColumnIndex([]int{0}) - prettyWriter.SetAutoWrapText(true) - prettyWriter.SetHeaderLine(true) - prettyWriter.SetHeaderColor(headerColors...) - - prettyWriter.Render() - return nil -} - -func (v *Validate) printHorizontallyWithDiff(ctx context.Context, t *dump_objects.Table) error { - settings := v.getHorizontalSettings(t) - - prettyWriter := tablewriter.NewWriter(os.Stdout) - prettyWriter.SetColumnAlignment(settings.ColumnsAlignments) - - tableData, err := v.st.GetObject(ctx, fmt.Sprintf("%d.dat.gz", t.DumpId)) - if err != nil { - log.Err(err).Msg("") - } - defer tableData.Close() - gz, err := gzip.NewReader(tableData) - if err != nil { - return fmt.Errorf("cannot create gzip reader: %w", err) - } - defer gz.Close() - r := bufio.NewReader(gz) - - realAffectedColumns := v.getAffectedColumns(t) - affectedColumns := v.getAffectedColumns(t) - originalRow := pgcopy.NewRow(len(t.Columns)) - transformedRow := pgcopy.NewRow(len(t.Columns)) - var lineNum = 1 - - for { - - var originalLine, transformedLine []byte - var originalValue, transformedValue *toolkit.RawValue - - if v.config.Validate.Diff { - originalLine, err = reader.ReadLine(r) - if err != nil { - if errors.Is(err, io.EOF) { - break - } - return fmt.Errorf("unable to read line: %w", err) - } - // Handle end of dump_objects file seq - if validate_utils.LineIsEndOfData(originalLine) { - break - } - - transformedLine, err = reader.ReadLine(r) - if err != nil { - return fmt.Errorf("unable to read line: %w", err) - } - - if err = originalRow.Decode(originalLine); err != nil { - return fmt.Errorf("error decoding copy line: %w", err) - } - if err = transformedRow.Decode(transformedLine); err != nil { - return fmt.Errorf("error decoding copy line: %w", err) - } - } else { - transformedLine, err = reader.ReadLine(r) - if err != nil { - if errors.Is(err, io.EOF) { - break - } - return fmt.Errorf("unable to read line: %w", err) - } - // Handle end of dump_objects file seq - if validate_utils.LineIsEndOfData(transformedLine) { - break - } - if err = transformedRow.Decode(transformedLine); err != nil { - return fmt.Errorf("error decoding copy line: %w", err) - } - } - - originalRecord := make([]string, len(t.Columns)) - transformedRecord := make([]string, len(t.Columns)) - originalRecordColors := make([]tablewriter.Colors, len(t.Columns)) - transformedRecordColors := make([]tablewriter.Colors, len(t.Columns)) - for idx := range t.Columns { - originalValue, err = originalRow.GetColumn(idx) - if err != nil { - return err - } - if originalValue.IsNull { - originalRecord[idx] = nullStringValue - } else { - originalRecord[idx] = stringsUtils.WrapString(string(originalValue.Data), maxWrapLength) - } - - transformedValue, err = transformedRow.GetColumn(idx) - if err != nil { - return err - } - if transformedValue.IsNull { - transformedRecord[idx] = nullStringValue - } else { - transformedRecord[idx] = stringsUtils.WrapString(string(transformedValue.Data), maxWrapLength) - } - - if idx == 2 { - log.Debug().Msg("") - } - if !validate_utils.ValuesEqual(originalValue, transformedValue) { - originalRecordColors[idx] = tablewriter.Colors{tablewriter.FgHiGreenColor} - transformedRecordColors[idx] = tablewriter.Colors{tablewriter.FgHiRedColor} - realAffectedColumns[idx] = struct{}{} - } else { - originalRecordColors[idx] = []int{} - transformedRecordColors[idx] = []int{} - } - } - - originalRecordColors = slices.Insert(originalRecordColors, 0, tablewriter.Colors{}) - transformedRecordColors = slices.Insert(transformedRecordColors, 0, tablewriter.Colors{}) - originalRecord = slices.Insert(originalRecord, 0, fmt.Sprintf("%d", lineNum)) - transformedRecord = slices.Insert(transformedRecord, 0, fmt.Sprintf("%d", lineNum)) - prettyWriter.Rich(originalRecord, originalRecordColors) - prettyWriter.Rich(transformedRecord, transformedRecordColors) - - lineNum++ - } - - header := make([]string, len(t.Columns)) - for idx, c := range t.Columns { - _, expected := affectedColumns[idx] - _, notExpectedButChanged := realAffectedColumns[idx] - if !expected && notExpectedButChanged { - header[idx] = fmt.Sprintf("%s (!!!)", c.Name) - } else { - header[idx] = c.Name - } - - } - header = slices.Insert(header, 0, "%LineNum%") - headerColors := v.getVerticalHeaderColors(t, realAffectedColumns) - - os.Stdout.Write([]byte(fmt.Sprintf("\n\n\t\"%s\".\"%s\"\n", t.Schema, t.Name))) - prettyWriter.SetHeader(header) - prettyWriter.SetRowLine(true) - prettyWriter.SetAutoMergeCellsByColumnIndex([]int{0}) - prettyWriter.SetAutoWrapText(true) - prettyWriter.SetHeaderLine(true) - prettyWriter.SetHeaderColor(headerColors...) - - prettyWriter.Render() - return nil -} - -func (v *Validate) getVerticalHeaderColors(t *dump_objects.Table, affectedColumns map[int]struct{}) []tablewriter.Colors { - headerColors := make([]tablewriter.Colors, len(t.Columns)) - for idx := range t.Columns { - if _, ok := affectedColumns[idx]; ok { - headerColors[idx] = []int{tablewriter.BgRedColor} - } else { - headerColors[idx] = []int{} - } - } - // Adding formatting setting for LineNum - headerColors = slices.Insert(headerColors, 0, tablewriter.Colors{}) - return headerColors -} - -func (v *Validate) printVertically(ctx context.Context, t *dump_objects.Table) error { - - var recordSize = 3 - if v.config.Validate.Diff { - recordSize = 4 - } - headerColorSetting := []int{tablewriter.Bold} - alignmentSettings := tablewriter.ALIGN_LEFT - headerColors := make([]tablewriter.Colors, recordSize) - columnAlignments := make([]int, recordSize) - for idx := range headerColors { - headerColors[idx] = headerColorSetting - columnAlignments[idx] = alignmentSettings - } - - prettyWriter := tablewriter.NewWriter(os.Stdout) - prettyWriter.SetAutoMergeCellsByColumnIndex([]int{0}) - prettyWriter.SetColumnAlignment(columnAlignments) - prettyWriter.SetAutoWrapText(true) - prettyWriter.SetHeaderLine(true) - prettyWriter.SetRowLine(true) - header := []string{"%LineNum%", "Column", "OriginalValue", "TransformedValue"} - if !v.config.Validate.Diff { - header = []string{"%LineNum%", "Column", "TransformedValue"} - } - - affectedColumns := v.getAffectedColumns(t) - - originalRow := pgcopy.NewRow(len(t.Columns)) - transformedRow := pgcopy.NewRow(len(t.Columns)) - tableData, err := v.st.GetObject(ctx, fmt.Sprintf("%d.dat.gz", t.DumpId)) - if err != nil { - log.Err(err).Msg("") - } - defer tableData.Close() - gz, err := gzip.NewReader(tableData) - if err != nil { - return fmt.Errorf("cannot create gzip reader: %w", err) - } - defer gz.Close() - r := bufio.NewReader(gz) - - var lineNum = 1 - for { - var originalLine, transformedLine []byte - var originalValue, transformedValue *toolkit.RawValue - - if v.config.Validate.Diff { - originalLine, err = reader.ReadLine(r) - if err != nil { - if errors.Is(err, io.EOF) { - break - } - return fmt.Errorf("unable to read line: %w", err) - } - // Handle end of dump_objects file seq - if validate_utils.LineIsEndOfData(originalLine) { - break - } - - transformedLine, err = reader.ReadLine(r) - if err != nil { - return fmt.Errorf("unable to read line: %w", err) - } - - if err = originalRow.Decode(originalLine); err != nil { - return fmt.Errorf("error decoding copy line: %w", err) - } - if err = transformedRow.Decode(transformedLine); err != nil { - return fmt.Errorf("error decoding copy line: %w", err) - } - } else { - transformedLine, err = reader.ReadLine(r) - if err != nil { - if errors.Is(err, io.EOF) { - break - } - return fmt.Errorf("unable to read line: %w", err) - } - // Handle end of dump_objects file seq - if validate_utils.LineIsEndOfData(transformedLine) { - break - } - if err = transformedRow.Decode(transformedLine); err != nil { - return fmt.Errorf("error decoding copy line: %w", err) - } - } - - prettyWriter.Rich(header, headerColors) - for idx, c := range t.Columns { - record := make([]string, recordSize) - record[0] = fmt.Sprintf("%d", lineNum) - record[1] = c.Name - - if v.config.Validate.Diff { - originalValue, err = originalRow.GetColumn(idx) - if err != nil { - return err - } - if originalValue.IsNull { - record[2] = nullStringValue - } else { - record[2] = stringsUtils.WrapString(string(originalValue.Data), maxWrapLength) - } - - transformedValue, err = transformedRow.GetColumn(idx) - if err != nil { - return err - } - if transformedValue.IsNull { - record[3] = nullStringValue - } else { - record[3] = stringsUtils.WrapString(string(transformedValue.Data), maxWrapLength) - } - } else { - transformedValue, err = transformedRow.GetColumn(idx) - if err != nil { - return err - } - if transformedValue.IsNull { - record[2] = nullStringValue - } else { - record[2] = stringsUtils.WrapString(string(transformedValue.Data), maxWrapLength) - } - } - - colors, isEqual := v.getVerticalRowColors(affectedColumns, idx, originalValue, transformedValue) - _, affected := affectedColumns[idx] - if v.config.Validate.Diff && !isEqual && !affected { - record[1] = fmt.Sprintf("%s (!!!)", c.Name) - } - prettyWriter.Rich(record, colors) - } - lineNum++ - } - os.Stdout.Write([]byte(fmt.Sprintf("\n\n\t\"%s\".\"%s\"\n", t.Schema, t.Name))) - prettyWriter.Render() - - return nil -} - -func (v *Validate) print(ctx context.Context, t *dump_objects.Table, format string, withDiff bool) error { - row := *pgcopy.NewRow(len(t.Columns)) - tableData, err := v.st.GetObject(ctx, fmt.Sprintf("%d.dat.gz", t.DumpId)) - if err != nil { - log.Err(err).Msg("") - } - defer tableData.Close() - gz, err := gzip.NewReader(tableData) - if err != nil { - return fmt.Errorf("cannot create gzip reader: %w", err) - } - defer gz.Close() - r := bufio.NewReader(gz) - - var lineNum = 1 - realAffectedColumns := v.getAffectedColumns(t) - diffValues := make([][]*toolkit.RawValue, len(t.Columns)) - for idx := range t.Columns { - diffValues[idx] = make([]*toolkit.RawValue, 2) - } - for { - line, err := reader.ReadLine(r) - if err != nil { - if errors.Is(err, io.EOF) { - break - } - return fmt.Errorf("unable to read line: %w", err) - } - - // Handle end of dump_objects file seq - if validate_utils.LineIsEndOfData(line) { - break - } - - if err = row.Decode(line); err != nil { - return fmt.Errorf("error decoding copy line: %w", err) - } - record := make([]string, len(t.Columns)) - for idx, c := range t.Columns { - value, err := row.GetColumn(idx) - if err != nil { - return fmt.Errorf("unable to get column \"%s\" value: %w", c.Name, err) - } - if value.IsNull { - record[idx] = nullStringValue - } else { - record[idx] = stringsUtils.WrapString(string(value.Data), maxWrapLength) - } - } - - record = slices.Insert(record, 0, fmt.Sprintf("%d", lineNum)) - - lineNum++ - } - - header := make([]string, len(t.Columns)) - for idx, c := range t.Columns { - header[idx] = c.Name - } - return nil -} - -func (v *Validate) printText(ctx context.Context, t *dump_objects.Table) error { - switch v.config.Validate.TableFormat { - case horizontalTableFormatName: - if v.config.Validate.Diff { - return v.printHorizontallyWithDiff(ctx, t) - } else { - return v.printHorizontally(ctx, t) - } - case verticalTableFormatName: - return v.printVertically(ctx, t) - default: - return fmt.Errorf("unknwon data format \"%s\"", v.config.Validate.TableFormat) - } -} From 516c8299f8a499d47d44037907a52594e21c2d95 Mon Sep 17 00:00:00 2001 From: Vadim Voitenko Date: Thu, 7 Mar 2024 19:08:20 +0200 Subject: [PATCH 10/10] Renamed package according to the Go convention --- internal/db/postgres/cmd/dump.go | 26 +++++++------- internal/db/postgres/cmd/validate.go | 20 +++++------ .../cmd/validate_utils/json_document.go | 6 ++-- .../cmd/validate_utils/json_document_test.go | 6 ++-- .../cmd/validate_utils/text_document.go | 4 +-- .../db/postgres/cmd/validate_utils/utils.go | 6 ++-- internal/db/postgres/context/context.go | 6 ++-- internal/db/postgres/context/pg_catalog.go | 36 +++++++++---------- internal/db/postgres/context/table.go | 16 ++++----- internal/db/postgres/dumpers/dumpers.go | 4 +-- internal/db/postgres/dumpers/large_object.go | 8 ++--- .../postgres/dumpers/plain_dump_pipeline.go | 6 ++-- internal/db/postgres/dumpers/sequence.go | 8 ++--- internal/db/postgres/dumpers/table.go | 8 ++--- .../dumpers/transformation_pipeline.go | 6 ++-- .../postgres/dumpers/transformation_window.go | 4 +-- .../postgres/dumpers/validation_pipeline.go | 4 +-- .../entry_producer.go | 2 +- .../{dump_objects => entries}/large_object.go | 2 +- .../{dump_objects => entries}/sequence.go | 2 +- .../{dump_objects => entries}/table.go | 2 +- 21 files changed, 91 insertions(+), 91 deletions(-) rename internal/db/postgres/{dump_objects => entries}/entry_producer.go (97%) rename internal/db/postgres/{dump_objects => entries}/large_object.go (99%) rename internal/db/postgres/{dump_objects => entries}/sequence.go (98%) rename internal/db/postgres/{dump_objects => entries}/table.go (99%) diff --git a/internal/db/postgres/cmd/dump.go b/internal/db/postgres/cmd/dump.go index 1e0ad292..96687cd5 100644 --- a/internal/db/postgres/cmd/dump.go +++ b/internal/db/postgres/cmd/dump.go @@ -29,8 +29,8 @@ import ( "golang.org/x/sync/errgroup" runtimeContext "github.com/greenmaskio/greenmask/internal/db/postgres/context" - "github.com/greenmaskio/greenmask/internal/db/postgres/dump_objects" "github.com/greenmaskio/greenmask/internal/db/postgres/dumpers" + "github.com/greenmaskio/greenmask/internal/db/postgres/entries" "github.com/greenmaskio/greenmask/internal/db/postgres/pgdump" storageDto "github.com/greenmaskio/greenmask/internal/db/postgres/storage" "github.com/greenmaskio/greenmask/internal/db/postgres/toc" @@ -59,7 +59,7 @@ type Dump struct { dumpedObjectSizes map[int32]storageDto.ObjectSizeStat tocFileSize int64 version int - blobs *dump_objects.Blobs + blobs *entries.Blobs // validate shows that dump worker must be in validation mode validate bool } @@ -221,7 +221,7 @@ func (d *Dump) schemaOnlyDump(ctx context.Context, tx pgx.Tx) error { func (d *Dump) dataDump(ctx context.Context) error { // TODO: You should use pointer to dumpers.DumpTask instead tasks := make(chan dumpers.DumpTask, d.pgDumpOptions.Jobs) - result := make(chan dump_objects.Entry, d.pgDumpOptions.Jobs) + result := make(chan entries.Entry, d.pgDumpOptions.Jobs) log.Debug().Msgf("planned %d workers", d.pgDumpOptions.Jobs) eg, gtx := errgroup.WithContext(ctx) @@ -256,11 +256,11 @@ func (d *Dump) dataDump(ctx context.Context) error { dumpObj.SetDumpId(d.dumpIdSequence) var task dumpers.DumpTask switch v := dumpObj.(type) { - case *dump_objects.Table: + case *entries.Table: task = dumpers.NewTableDumper(v, d.validate) - case *dump_objects.Sequence: + case *entries.Sequence: task = dumpers.NewSequenceDumper(v) - case *dump_objects.Blobs: + case *entries.Blobs: d.blobs = v task = dumpers.NewLargeObjectDumper(v) default: @@ -280,7 +280,7 @@ func (d *Dump) dataDump(ctx context.Context) error { func() error { var tables, sequences, largeObjects []*toc.Entry for { - var entry dump_objects.Entry + var entry entries.Entry var ok bool select { case <-gtx.Done(): @@ -299,15 +299,15 @@ func (d *Dump) dataDump(ctx context.Context) error { return fmt.Errorf("error producing toc entry: %w", err) } switch v := entry.(type) { - case *dump_objects.Table: + case *entries.Table: d.dumpedObjectSizes[e.DumpId] = storageDto.ObjectSizeStat{ Original: v.OriginalSize, Compressed: v.CompressedSize, } tables = append(tables, e) - case *dump_objects.Sequence: + case *entries.Sequence: sequences = append(sequences, e) - case *dump_objects.Blobs: + case *entries.Blobs: d.dumpedObjectSizes[e.DumpId] = storageDto.ObjectSizeStat{ Original: v.OriginalSize, Compressed: v.CompressedSize, @@ -506,7 +506,7 @@ func (d *Dump) getWorkerTransaction(ctx context.Context) (*pgx.Conn, pgx.Tx, err } func (d *Dump) dumpWorker( - ctx context.Context, tasks <-chan dumpers.DumpTask, result chan<- dump_objects.Entry, id int, + ctx context.Context, tasks <-chan dumpers.DumpTask, result chan<- entries.Entry, id int, ) error { conn, tx, err := d.getWorkerTransaction(ctx) @@ -574,7 +574,7 @@ func (d *Dump) dumpWorker( } func (d *Dump) validateDumpWorker( - ctx context.Context, tasks <-chan dumpers.DumpTask, result chan<- dump_objects.Entry, id int, + ctx context.Context, tasks <-chan dumpers.DumpTask, result chan<- entries.Entry, id int, ) error { for { @@ -601,7 +601,7 @@ func (d *Dump) validateDumpWorker( Str("ObjectName", task.DebugInfo()). Msgf("dumping started") - entry, err := func() (dump_objects.Entry, error) { + entry, err := func() (entries.Entry, error) { conn, tx, err := d.getWorkerTransaction(ctx) if err != nil { diff --git a/internal/db/postgres/cmd/validate.go b/internal/db/postgres/cmd/validate.go index 55eab12b..84178d8d 100644 --- a/internal/db/postgres/cmd/validate.go +++ b/internal/db/postgres/cmd/validate.go @@ -18,7 +18,7 @@ import ( "github.com/greenmaskio/greenmask/internal/db/postgres/cmd/validate_utils" runtimeContext "github.com/greenmaskio/greenmask/internal/db/postgres/context" - "github.com/greenmaskio/greenmask/internal/db/postgres/dump_objects" + "github.com/greenmaskio/greenmask/internal/db/postgres/entries" "github.com/greenmaskio/greenmask/internal/db/postgres/pgcopy" "github.com/greenmaskio/greenmask/internal/db/postgres/toc" "github.com/greenmaskio/greenmask/internal/db/postgres/transformers/custom" @@ -140,12 +140,12 @@ func (v *Validate) Run(ctx context.Context) error { func (v *Validate) print(ctx context.Context) error { for _, e := range v.dataEntries { - idx := slices.IndexFunc(v.context.DataSectionObjects, func(entry dump_objects.Entry) bool { - t := entry.(*dump_objects.Table) + idx := slices.IndexFunc(v.context.DataSectionObjects, func(entry entries.Entry) bool { + t := entry.(*entries.Table) return t.DumpId == e.DumpId }) - t := v.context.DataSectionObjects[idx].(*dump_objects.Table) + t := v.context.DataSectionObjects[idx].(*entries.Table) doc, err := v.createDocument(ctx, t) if err != nil { return fmt.Errorf("unable to create validation document: %w", err) @@ -158,7 +158,7 @@ func (v *Validate) print(ctx context.Context) error { return nil } -func (v *Validate) getDocument(table *dump_objects.Table) validate_utils.Documenter { +func (v *Validate) getDocument(table *entries.Table) validate_utils.Documenter { switch v.config.Validate.Format { case JsonFormat: return validate_utils.NewJsonDocument(table, v.config.Validate.Diff, v.config.Validate.OnlyTransformed) @@ -171,7 +171,7 @@ func (v *Validate) getDocument(table *dump_objects.Table) validate_utils.Documen } } -func (v *Validate) getReader(ctx context.Context, table *dump_objects.Table) (closeFunc, *bufio.Reader, error) { +func (v *Validate) getReader(ctx context.Context, table *entries.Table) (closeFunc, *bufio.Reader, error) { tableData, err := v.st.GetObject(ctx, fmt.Sprintf("%d.dat.gz", table.DumpId)) if err != nil { return nil, nil, fmt.Errorf("unable to get object from storage: %w", err) @@ -195,7 +195,7 @@ func (v *Validate) getReader(ctx context.Context, table *dump_objects.Table) (cl return f, bufio.NewReader(gz), nil } -func (v *Validate) readRecords(r *bufio.Reader, t *dump_objects.Table) (original, transformed *pgcopy.Row, err error) { +func (v *Validate) readRecords(r *bufio.Reader, t *entries.Table) (original, transformed *pgcopy.Row, err error) { var originalLine, transformedLine []byte var originalRow, transformedRow *pgcopy.Row @@ -228,7 +228,7 @@ func (v *Validate) readRecords(r *bufio.Reader, t *dump_objects.Table) (original return originalRow, transformedRow, nil } -func (v *Validate) createDocument(ctx context.Context, t *dump_objects.Table) (validate_utils.Documenter, error) { +func (v *Validate) createDocument(ctx context.Context, t *entries.Table) (validate_utils.Documenter, error) { doc := v.getDocument(t) closeReader, r, err := v.getReader(ctx, t) @@ -258,10 +258,10 @@ func (v *Validate) createDocument(ctx context.Context, t *dump_objects.Table) (v } func (v *Validate) dumpTables(ctx context.Context) error { - var tablesWithTransformers []dump_objects.Entry + var tablesWithTransformers []entries.Entry for _, item := range v.context.DataSectionObjects { - if t, ok := item.(*dump_objects.Table); ok && len(t.Transformers) > 0 { + if t, ok := item.(*entries.Table); ok && len(t.Transformers) > 0 { t.ValidateLimitedRecords = v.config.Validate.RowsLimit tablesWithTransformers = append(tablesWithTransformers, t) } diff --git a/internal/db/postgres/cmd/validate_utils/json_document.go b/internal/db/postgres/cmd/validate_utils/json_document.go index 99d11c35..26561678 100644 --- a/internal/db/postgres/cmd/validate_utils/json_document.go +++ b/internal/db/postgres/cmd/validate_utils/json_document.go @@ -6,7 +6,7 @@ import ( "io" "maps" - "github.com/greenmaskio/greenmask/internal/db/postgres/dump_objects" + "github.com/greenmaskio/greenmask/internal/db/postgres/entries" "github.com/greenmaskio/greenmask/internal/db/postgres/pgcopy" "github.com/greenmaskio/greenmask/pkg/toolkit" ) @@ -58,7 +58,7 @@ type jsonRecordPlain map[string]string type JsonDocument struct { result *JsonDocumentResult - table *dump_objects.Table + table *entries.Table withDiff bool expectedAffectedColumns map[string]struct{} unexpectedAffectedColumns map[string]struct{} @@ -66,7 +66,7 @@ type JsonDocument struct { onlyTransformed bool } -func NewJsonDocument(table *dump_objects.Table, withDiff bool, onlyTransformed bool) *JsonDocument { +func NewJsonDocument(table *entries.Table, withDiff bool, onlyTransformed bool) *JsonDocument { pkColumns := getPrimaryKeyConstraintColumns(table) expectedAffectedColumns := getAffectedColumns(table) var pkColumnsList []string diff --git a/internal/db/postgres/cmd/validate_utils/json_document_test.go b/internal/db/postgres/cmd/validate_utils/json_document_test.go index 4524fab1..987e4954 100644 --- a/internal/db/postgres/cmd/validate_utils/json_document_test.go +++ b/internal/db/postgres/cmd/validate_utils/json_document_test.go @@ -9,7 +9,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/greenmaskio/greenmask/internal/db/postgres/dump_objects" + "github.com/greenmaskio/greenmask/internal/db/postgres/entries" "github.com/greenmaskio/greenmask/internal/db/postgres/pgcopy" "github.com/greenmaskio/greenmask/internal/db/postgres/transformers/utils" "github.com/greenmaskio/greenmask/pkg/toolkit" @@ -87,7 +87,7 @@ func TestJsonDocument_GetRecords(t *testing.T) { //r.SetRow(row) } -func getTableAndRows() (table *dump_objects.Table, original, transformed [][]byte) { +func getTableAndRows() (table *entries.Table, original, transformed [][]byte) { tableDef := ` { @@ -175,7 +175,7 @@ func getTableAndRows() (table *dump_objects.Table, original, transformed [][]byt panic(err) } - table = &dump_objects.Table{ + table = &entries.Table{ Table: t, Transformers: []utils.Transformer{&testTransformer{}}, } diff --git a/internal/db/postgres/cmd/validate_utils/text_document.go b/internal/db/postgres/cmd/validate_utils/text_document.go index 07fca604..7e4a2428 100644 --- a/internal/db/postgres/cmd/validate_utils/text_document.go +++ b/internal/db/postgres/cmd/validate_utils/text_document.go @@ -8,7 +8,7 @@ import ( "github.com/olekukonko/tablewriter" - "github.com/greenmaskio/greenmask/internal/db/postgres/dump_objects" + "github.com/greenmaskio/greenmask/internal/db/postgres/entries" stringsUtils "github.com/greenmaskio/greenmask/internal/utils/strings" "github.com/greenmaskio/greenmask/pkg/toolkit" ) @@ -32,7 +32,7 @@ type TextDocument struct { tableFormat string } -func NewTextDocument(table *dump_objects.Table, withDiff bool, onlyTransformed bool, tableFormat string) *TextDocument { +func NewTextDocument(table *entries.Table, withDiff bool, onlyTransformed bool, tableFormat string) *TextDocument { jd := NewJsonDocument(table, withDiff, onlyTransformed) if tableFormat != horizontalTableFormatName && tableFormat != verticalTableFormatName { panic(fmt.Sprintf("unknown table format %s", tableFormat)) diff --git a/internal/db/postgres/cmd/validate_utils/utils.go b/internal/db/postgres/cmd/validate_utils/utils.go index 06b64c40..f153263e 100644 --- a/internal/db/postgres/cmd/validate_utils/utils.go +++ b/internal/db/postgres/cmd/validate_utils/utils.go @@ -3,7 +3,7 @@ package validate_utils import ( "slices" - "github.com/greenmaskio/greenmask/internal/db/postgres/dump_objects" + "github.com/greenmaskio/greenmask/internal/db/postgres/entries" "github.com/greenmaskio/greenmask/pkg/toolkit" ) @@ -11,7 +11,7 @@ var endOfFileSeq = []byte(`\.`) const nullStringValue = "NULL" -func getAffectedColumns(t *dump_objects.Table) map[string]struct{} { +func getAffectedColumns(t *entries.Table) map[string]struct{} { affectedColumns := make(map[string]struct{}) for _, tr := range t.Transformers { ac := tr.GetAffectedColumns() @@ -30,7 +30,7 @@ func ValuesEqual(a *toolkit.RawValue, b *toolkit.RawValue) bool { return a.IsNull == b.IsNull && slices.Equal(a.Data, b.Data) } -func getPrimaryKeyConstraintColumns(t *dump_objects.Table) map[int]*toolkit.Column { +func getPrimaryKeyConstraintColumns(t *entries.Table) map[int]*toolkit.Column { idx := slices.IndexFunc(t.Constraints, func(constraint toolkit.Constraint) bool { return constraint.Type() == toolkit.PkConstraintType }) diff --git a/internal/db/postgres/context/context.go b/internal/db/postgres/context/context.go index 7f72bdc0..23af9fa5 100644 --- a/internal/db/postgres/context/context.go +++ b/internal/db/postgres/context/context.go @@ -21,7 +21,7 @@ import ( "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgtype" - "github.com/greenmaskio/greenmask/internal/db/postgres/dump_objects" + "github.com/greenmaskio/greenmask/internal/db/postgres/entries" "github.com/greenmaskio/greenmask/internal/db/postgres/pgdump" transformersUtils "github.com/greenmaskio/greenmask/internal/db/postgres/transformers/utils" "github.com/greenmaskio/greenmask/internal/domains" @@ -31,11 +31,11 @@ import ( // RuntimeContext - describes current runtime behaviour according to the config and schema objects type RuntimeContext struct { // Tables - map of build tables with toolkit that was wrapped into dump.Entry - Tables map[toolkit.Oid]*dump_objects.Table + Tables map[toolkit.Oid]*entries.Table // Types - list of custom types that are used in DB schema Types []*toolkit.Type // DataSectionObjects - list of objects to dump in data-section. There are sequences, tables and large objects - DataSectionObjects []dump_objects.Entry + DataSectionObjects []entries.Entry // Warnings - list of occurred ValidationWarning during validation and config building Warnings toolkit.ValidationWarnings // Registry - registry of all the registered transformers definition diff --git a/internal/db/postgres/context/pg_catalog.go b/internal/db/postgres/context/pg_catalog.go index 67af4a8d..1f5a2b58 100644 --- a/internal/db/postgres/context/pg_catalog.go +++ b/internal/db/postgres/context/pg_catalog.go @@ -22,7 +22,7 @@ import ( "github.com/jackc/pgx/v5" - "github.com/greenmaskio/greenmask/internal/db/postgres/dump_objects" + "github.com/greenmaskio/greenmask/internal/db/postgres/entries" "github.com/greenmaskio/greenmask/internal/db/postgres/pgdump" "github.com/greenmaskio/greenmask/internal/db/postgres/toc" "github.com/greenmaskio/greenmask/pkg/toolkit" @@ -36,8 +36,8 @@ const ( // TODO: Rewrite it using gotemplate func getDumpObjects( - ctx context.Context, tx pgx.Tx, options *pgdump.Options, config map[toolkit.Oid]*dump_objects.Table, -) ([]dump_objects.Entry, error) { + ctx context.Context, tx pgx.Tx, options *pgdump.Options, config map[toolkit.Oid]*entries.Table, +) ([]entries.Entry, error) { // Building relation search query using regexp adaptation rules and pre-defined query templates // TODO: Refactor it to gotemplate @@ -57,7 +57,7 @@ func getDumpObjects( // Generate table objects //sequences := make([]*dump_objects.Sequence, 0) //tables := make([]*dump_objects.Table, 0) - var dataObjects []dump_objects.Entry + var dataObjects []entries.Entry defer tableSearchRows.Close() for tableSearchRows.Next() { var oid toc.Oid @@ -73,12 +73,12 @@ func getDumpObjects( if err != nil { return nil, fmt.Errorf("unnable scan data: %w", err) } - var table *dump_objects.Table + var table *entries.Table switch relKind { case 'S': // Building sequence objects - dataObjects = append(dataObjects, &dump_objects.Sequence{ + dataObjects = append(dataObjects, &entries.Sequence{ Name: name, Schema: schemaName, Oid: oid, @@ -101,7 +101,7 @@ func getDumpObjects( } else { // If table is not found - create new table object and collect all the columns - table = &dump_objects.Table{ + table = &entries.Table{ Table: &toolkit.Table{ Name: name, Schema: schemaName, @@ -130,7 +130,7 @@ func getDumpObjects( // Assigning columns for each table for _, obj := range dataObjects { switch v := obj.(type) { - case *dump_objects.Table: + case *entries.Table: columns, err := getColumnsConfig(ctx, tx, v.Oid) if err != nil { return nil, fmt.Errorf("unable to collect table columns: %w", err) @@ -150,14 +150,14 @@ func getDumpObjects( } // Getting list of the all large objects - var largeObjects []*dump_objects.LargeObject + var largeObjects []*entries.LargeObject loListRows, err := tx.Query(ctx, LargeObjectsListQuery) if err != nil { return nil, fmt.Errorf("error executing LargeObjectsListQuery: %w", err) } defer loListRows.Close() for loListRows.Next() { - lo := &dump_objects.LargeObject{TableOid: tableOid} + lo := &entries.LargeObject{TableOid: tableOid} if err = loListRows.Scan(&lo.Oid, &lo.Owner, &lo.Comment); err != nil { return nil, fmt.Errorf("error scanning LargeObjectsListQuery: %w", err) } @@ -168,20 +168,20 @@ func getDumpObjects( for _, lo := range largeObjects { // Getting default ACL - defaultACL := &dump_objects.ACL{} + defaultACL := &entries.ACL{} row = tx.QueryRow(ctx, LargeObjectGetDefaultAclQuery, lo.Oid) if err = row.Scan(&defaultACL.Value); err != nil { return nil, fmt.Errorf("error scanning LargeObjectGetDefaultAclQuery: %w", err) } // Getting default ACL items - var defaultACLItems []*dump_objects.ACLItem + var defaultACLItems []*entries.ACLItem loDescribeDefaultAclRows, err := tx.Query(ctx, LargeObjectDescribeAclItemQuery, defaultACL.Value) if err != nil { return nil, fmt.Errorf("error quering LargeObjectDescribeAclItemQuery: %w", err) } defer loDescribeDefaultAclRows.Close() for loDescribeDefaultAclRows.Next() { - item := &dump_objects.ACLItem{} + item := &entries.ACLItem{} if err = loDescribeDefaultAclRows.Scan(&item.Grantor, &item.Grantee, &item.PrivilegeType, &item.Grantable); err != nil { return nil, fmt.Errorf("error scanning LargeObjectDescribeAclItemQuery: %w", err) } @@ -191,14 +191,14 @@ func getDumpObjects( lo.DefaultACL = defaultACL // Getting ACL - var acls []*dump_objects.ACL + var acls []*entries.ACL loAclRows, err := tx.Query(ctx, LargeObjectGetAclQuery, lo.Oid) if err != nil { return nil, fmt.Errorf("error quering LargeObjectGetAclQuery: %w", err) } defer loAclRows.Close() for loAclRows.Next() { - a := &dump_objects.ACL{} + a := &entries.ACL{} if err = loAclRows.Scan(&a.Value); err != nil { return nil, fmt.Errorf("error scanning LargeObjectGetAclQuery: %w", err) } @@ -207,14 +207,14 @@ func getDumpObjects( // Getting ACL items for _, a := range acls { - var aclItems []*dump_objects.ACLItem + var aclItems []*entries.ACLItem loDescribeAclRows, err := tx.Query(ctx, LargeObjectDescribeAclItemQuery, a.Value) if err != nil { return nil, fmt.Errorf("error quering LargeObjectDescribeAclItemQuery: %w", err) } defer loDescribeAclRows.Close() for loDescribeAclRows.Next() { - item := &dump_objects.ACLItem{} + item := &entries.ACLItem{} if err = loDescribeAclRows.Scan(&item.Grantor, &item.Grantee, &item.PrivilegeType, &item.Grantable); err != nil { return nil, fmt.Errorf("error scanning LargeObjectDescribeAclItemQuery: %w", err) } @@ -227,7 +227,7 @@ func getDumpObjects( } if len(largeObjects) > 0 { - dataObjects = append(dataObjects, &dump_objects.Blobs{ + dataObjects = append(dataObjects, &entries.Blobs{ LargeObjects: largeObjects, }) } diff --git a/internal/db/postgres/context/table.go b/internal/db/postgres/context/table.go index c8a3fad1..aa7f5be1 100644 --- a/internal/db/postgres/context/table.go +++ b/internal/db/postgres/context/table.go @@ -24,7 +24,7 @@ import ( "github.com/jackc/pgx/v5/pgtype" "github.com/rs/zerolog/log" - "github.com/greenmaskio/greenmask/internal/db/postgres/dump_objects" + "github.com/greenmaskio/greenmask/internal/db/postgres/entries" transformersUtils "github.com/greenmaskio/greenmask/internal/db/postgres/transformers/utils" "github.com/greenmaskio/greenmask/internal/domains" "github.com/greenmaskio/greenmask/pkg/toolkit" @@ -37,8 +37,8 @@ func validateAndBuildTablesConfig( ctx context.Context, tx pgx.Tx, typeMap *pgtype.Map, cfg []*domains.Table, registry *transformersUtils.TransformerRegistry, version int, types []*toolkit.Type, -) (map[toolkit.Oid]*dump_objects.Table, toolkit.ValidationWarnings, error) { - result := make(map[toolkit.Oid]*dump_objects.Table, len(cfg)) +) (map[toolkit.Oid]*entries.Table, toolkit.ValidationWarnings, error) { + result := make(map[toolkit.Oid]*entries.Table, len(cfg)) var warnings toolkit.ValidationWarnings for _, tableCfg := range cfg { @@ -100,12 +100,12 @@ func validateAndBuildTablesConfig( return result, warnings, nil } -func getTable(ctx context.Context, tx pgx.Tx, t *domains.Table) ([]*dump_objects.Table, toolkit.ValidationWarnings, error) { - table := &dump_objects.Table{ +func getTable(ctx context.Context, tx pgx.Tx, t *domains.Table) ([]*entries.Table, toolkit.ValidationWarnings, error) { + table := &entries.Table{ Table: &toolkit.Table{}, } var warnings toolkit.ValidationWarnings - var tables []*dump_objects.Table + var tables []*entries.Table row := tx.QueryRow(ctx, TableSearchQuery, t.Schema, t.Name) err := row.Scan(&table.Oid, &table.Schema, &table.Name, &table.Owner, &table.RelKind, @@ -137,7 +137,7 @@ func getTable(ctx context.Context, tx pgx.Tx, t *domains.Table) ([]*dump_objects Str("TableName", table.Name). Msg("table is partitioned: gathering all partitions and creating dumping tasks") // Get list of inherited tables - var parts []*dump_objects.Table + var parts []*entries.Table rows, err := tx.Query(ctx, TableGetChildPatsQuery, table.Oid) if err != nil { @@ -146,7 +146,7 @@ func getTable(ctx context.Context, tx pgx.Tx, t *domains.Table) ([]*dump_objects defer rows.Close() for rows.Next() { - pt := &dump_objects.Table{ + pt := &entries.Table{ Table: &toolkit.Table{}, RootPtSchema: table.Schema, RootPtName: table.Name, diff --git a/internal/db/postgres/dumpers/dumpers.go b/internal/db/postgres/dumpers/dumpers.go index 8b70e9c6..fcdf16e4 100644 --- a/internal/db/postgres/dumpers/dumpers.go +++ b/internal/db/postgres/dumpers/dumpers.go @@ -19,11 +19,11 @@ import ( "github.com/jackc/pgx/v5" - "github.com/greenmaskio/greenmask/internal/db/postgres/dump_objects" + "github.com/greenmaskio/greenmask/internal/db/postgres/entries" "github.com/greenmaskio/greenmask/internal/storages" ) type DumpTask interface { - Execute(ctx context.Context, tx pgx.Tx, st storages.Storager) (dump_objects.Entry, error) + Execute(ctx context.Context, tx pgx.Tx, st storages.Storager) (entries.Entry, error) DebugInfo() string } diff --git a/internal/db/postgres/dumpers/large_object.go b/internal/db/postgres/dumpers/large_object.go index 3645cc5c..edaa8f80 100644 --- a/internal/db/postgres/dumpers/large_object.go +++ b/internal/db/postgres/dumpers/large_object.go @@ -26,7 +26,7 @@ import ( "github.com/rs/zerolog/log" "golang.org/x/sync/errgroup" - "github.com/greenmaskio/greenmask/internal/db/postgres/dump_objects" + "github.com/greenmaskio/greenmask/internal/db/postgres/entries" "github.com/greenmaskio/greenmask/internal/storages" "github.com/greenmaskio/greenmask/internal/utils/countwriter" ) @@ -34,18 +34,18 @@ import ( const loBufSize = 1024 * 1024 type BlobsDumper struct { - Blobs *dump_objects.Blobs + Blobs *entries.Blobs OriginalSize int64 CompressedSize int64 } -func NewLargeObjectDumper(blobs *dump_objects.Blobs) *BlobsDumper { +func NewLargeObjectDumper(blobs *entries.Blobs) *BlobsDumper { return &BlobsDumper{ Blobs: blobs, } } -func (lod *BlobsDumper) Execute(ctx context.Context, tx pgx.Tx, st storages.Storager) (dump_objects.Entry, error) { +func (lod *BlobsDumper) Execute(ctx context.Context, tx pgx.Tx, st storages.Storager) (entries.Entry, error) { for _, lo := range lod.Blobs.LargeObjects { eg, gtx := errgroup.WithContext(ctx) diff --git a/internal/db/postgres/dumpers/plain_dump_pipeline.go b/internal/db/postgres/dumpers/plain_dump_pipeline.go index 50c32c4b..c7da2e5a 100644 --- a/internal/db/postgres/dumpers/plain_dump_pipeline.go +++ b/internal/db/postgres/dumpers/plain_dump_pipeline.go @@ -19,17 +19,17 @@ import ( "fmt" "io" - dump "github.com/greenmaskio/greenmask/internal/db/postgres/dump_objects" + "github.com/greenmaskio/greenmask/internal/db/postgres/entries" "github.com/greenmaskio/greenmask/internal/db/postgres/pgcopy" ) type PlainDumpPipeline struct { w io.Writer line uint64 - table *dump.Table + table *entries.Table } -func NewPlainDumpPipeline(table *dump.Table, w io.Writer) *PlainDumpPipeline { +func NewPlainDumpPipeline(table *entries.Table, w io.Writer) *PlainDumpPipeline { return &PlainDumpPipeline{ table: table, w: w, diff --git a/internal/db/postgres/dumpers/sequence.go b/internal/db/postgres/dumpers/sequence.go index 13c90440..436e8c16 100644 --- a/internal/db/postgres/dumpers/sequence.go +++ b/internal/db/postgres/dumpers/sequence.go @@ -20,21 +20,21 @@ import ( "github.com/jackc/pgx/v5" - "github.com/greenmaskio/greenmask/internal/db/postgres/dump_objects" + "github.com/greenmaskio/greenmask/internal/db/postgres/entries" "github.com/greenmaskio/greenmask/internal/storages" ) type SequenceDumper struct { - sequence *dump_objects.Sequence + sequence *entries.Sequence } -func NewSequenceDumper(sequence *dump_objects.Sequence) *SequenceDumper { +func NewSequenceDumper(sequence *entries.Sequence) *SequenceDumper { return &SequenceDumper{ sequence: sequence, } } -func (sd *SequenceDumper) Execute(ctx context.Context, tx pgx.Tx, st storages.Storager) (dump_objects.Entry, error) { +func (sd *SequenceDumper) Execute(ctx context.Context, tx pgx.Tx, st storages.Storager) (entries.Entry, error) { return sd.sequence, nil } diff --git a/internal/db/postgres/dumpers/table.go b/internal/db/postgres/dumpers/table.go index 2ba74092..4227ea4f 100644 --- a/internal/db/postgres/dumpers/table.go +++ b/internal/db/postgres/dumpers/table.go @@ -24,25 +24,25 @@ import ( "github.com/rs/zerolog/log" "golang.org/x/sync/errgroup" - dump "github.com/greenmaskio/greenmask/internal/db/postgres/dump_objects" + "github.com/greenmaskio/greenmask/internal/db/postgres/entries" "github.com/greenmaskio/greenmask/internal/storages" "github.com/greenmaskio/greenmask/internal/utils/countwriter" ) type TableDumper struct { - table *dump.Table + table *entries.Table recordNum uint64 validate bool } -func NewTableDumper(table *dump.Table, validate bool) *TableDumper { +func NewTableDumper(table *entries.Table, validate bool) *TableDumper { return &TableDumper{ table: table, validate: validate, } } -func (td *TableDumper) Execute(ctx context.Context, tx pgx.Tx, st storages.Storager) (dump.Entry, error) { +func (td *TableDumper) Execute(ctx context.Context, tx pgx.Tx, st storages.Storager) (entries.Entry, error) { w, r := countwriter.NewGzipPipe() diff --git a/internal/db/postgres/dumpers/transformation_pipeline.go b/internal/db/postgres/dumpers/transformation_pipeline.go index c791b095..0e6c8912 100644 --- a/internal/db/postgres/dumpers/transformation_pipeline.go +++ b/internal/db/postgres/dumpers/transformation_pipeline.go @@ -23,7 +23,7 @@ import ( "github.com/rs/zerolog/log" "golang.org/x/sync/errgroup" - dump "github.com/greenmaskio/greenmask/internal/db/postgres/dump_objects" + "github.com/greenmaskio/greenmask/internal/db/postgres/entries" "github.com/greenmaskio/greenmask/internal/db/postgres/pgcopy" "github.com/greenmaskio/greenmask/internal/db/postgres/transformers" "github.com/greenmaskio/greenmask/internal/db/postgres/transformers/utils" @@ -35,7 +35,7 @@ var endOfLineSeq = []byte("\n") type TransformationFunc func(ctx context.Context, r *toolkit.Record) (*toolkit.Record, error) type TransformationPipeline struct { - table *dump.Table + table *entries.Table //buf *bytes.Buffer w io.Writer line uint64 @@ -46,7 +46,7 @@ type TransformationPipeline struct { record *toolkit.Record } -func NewTransformationPipeline(ctx context.Context, eg *errgroup.Group, table *dump.Table, w io.Writer) (*TransformationPipeline, error) { +func NewTransformationPipeline(ctx context.Context, eg *errgroup.Group, table *entries.Table, w io.Writer) (*TransformationPipeline, error) { var tws []*TransformationWindow var isAsync bool diff --git a/internal/db/postgres/dumpers/transformation_window.go b/internal/db/postgres/dumpers/transformation_window.go index bfd6e240..eb1a8a4e 100644 --- a/internal/db/postgres/dumpers/transformation_window.go +++ b/internal/db/postgres/dumpers/transformation_window.go @@ -20,7 +20,7 @@ import ( "golang.org/x/sync/errgroup" - dump "github.com/greenmaskio/greenmask/internal/db/postgres/dump_objects" + "github.com/greenmaskio/greenmask/internal/db/postgres/entries" "github.com/greenmaskio/greenmask/internal/db/postgres/transformers/utils" "github.com/greenmaskio/greenmask/pkg/toolkit" ) @@ -47,7 +47,7 @@ func NewTransformationWindow(ctx context.Context, eg *errgroup.Group) *Transform } } -func (tw *TransformationWindow) TryAdd(table *dump.Table, t utils.Transformer) bool { +func (tw *TransformationWindow) TryAdd(table *entries.Table, t utils.Transformer) bool { affectedColumn := t.GetAffectedColumns() if len(affectedColumn) == 0 { diff --git a/internal/db/postgres/dumpers/validation_pipeline.go b/internal/db/postgres/dumpers/validation_pipeline.go index f92bdbcd..69498aa9 100644 --- a/internal/db/postgres/dumpers/validation_pipeline.go +++ b/internal/db/postgres/dumpers/validation_pipeline.go @@ -7,14 +7,14 @@ import ( "golang.org/x/sync/errgroup" - dump "github.com/greenmaskio/greenmask/internal/db/postgres/dump_objects" + "github.com/greenmaskio/greenmask/internal/db/postgres/entries" ) type ValidationPipeline struct { *TransformationPipeline } -func NewValidationPipeline(ctx context.Context, eg *errgroup.Group, table *dump.Table, w io.Writer) (*ValidationPipeline, error) { +func NewValidationPipeline(ctx context.Context, eg *errgroup.Group, table *entries.Table, w io.Writer) (*ValidationPipeline, error) { tpp, err := NewTransformationPipeline(ctx, eg, table, w) if err != nil { return nil, err diff --git a/internal/db/postgres/dump_objects/entry_producer.go b/internal/db/postgres/entries/entry_producer.go similarity index 97% rename from internal/db/postgres/dump_objects/entry_producer.go rename to internal/db/postgres/entries/entry_producer.go index 2f7dce13..e8d71622 100644 --- a/internal/db/postgres/dump_objects/entry_producer.go +++ b/internal/db/postgres/entries/entry_producer.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package dump_objects +package entries import "github.com/greenmaskio/greenmask/internal/db/postgres/toc" diff --git a/internal/db/postgres/dump_objects/large_object.go b/internal/db/postgres/entries/large_object.go similarity index 99% rename from internal/db/postgres/dump_objects/large_object.go rename to internal/db/postgres/entries/large_object.go index 6f7ccd35..8e48b081 100644 --- a/internal/db/postgres/dump_objects/large_object.go +++ b/internal/db/postgres/entries/large_object.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package dump_objects +package entries import ( "fmt" diff --git a/internal/db/postgres/dump_objects/sequence.go b/internal/db/postgres/entries/sequence.go similarity index 98% rename from internal/db/postgres/dump_objects/sequence.go rename to internal/db/postgres/entries/sequence.go index 57c5ecd6..99180a76 100644 --- a/internal/db/postgres/dump_objects/sequence.go +++ b/internal/db/postgres/entries/sequence.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package dump_objects +package entries import ( "fmt" diff --git a/internal/db/postgres/dump_objects/table.go b/internal/db/postgres/entries/table.go similarity index 99% rename from internal/db/postgres/dump_objects/table.go rename to internal/db/postgres/entries/table.go index 638432c8..6a25f9b4 100644 --- a/internal/db/postgres/dump_objects/table.go +++ b/internal/db/postgres/entries/table.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package dump_objects +package entries import ( "errors"