From 53c97624da0798882a6ca93c0445738812194d8e Mon Sep 17 00:00:00 2001 From: Lars T Hansen Date: Sun, 24 Nov 2024 20:01:41 +0100 Subject: [PATCH] Fix #615 - Test cases, tweaks, better nodefaults handling --- code/sonalyze/cmd/parse/parse.go | 25 +- code/sonalyze/cmd/profile/print.go | 37 +- code/sonalyze/table/data.go | 26 +- code/sonalyze/table/data_test.go | 49 +++ code/sonalyze/table/format.go | 8 +- code/sonalyze/table/reflect.go | 80 ++--- code/sonalyze/table/reflect_test.go | 319 ++++++++++++++++++ code/tests/relative/sonalyze/profile-print.sh | 12 +- code/tests/sonalyze/format/nodefault.sh | 4 +- code/tests/sonalyze/parse/parse.sh | 2 +- 10 files changed, 456 insertions(+), 106 deletions(-) create mode 100644 code/sonalyze/table/data_test.go diff --git a/code/sonalyze/cmd/parse/parse.go b/code/sonalyze/cmd/parse/parse.go index ff47e10b..b9c7d742 100644 --- a/code/sonalyze/cmd/parse/parse.go +++ b/code/sonalyze/cmd/parse/parse.go @@ -226,9 +226,6 @@ type SFS = SimpleFormatSpec type AFS = SimpleFormatSpecWithAttr type ZFA = SynthesizedFormatSpecWithAttr -// TODO: IMPROVEME: The defaulted fields here follow the Rust code. Now that it's trivial to do so, -// we should consider adding more. -// // TODO: IMPROVEME: The use of utc for "localtime" is a bug that comes from the Rust code. var parseFormatters = DefineTableFromMap( @@ -242,8 +239,8 @@ var parseFormatters = DefineTableFromMap( "MemtotalKB": SFS{"Installed main memory", ""}, "memtotal": ZFA{"Installed main memory (GB)", "MemtotalKB", FmtDivideBy1M}, "User": SFS{"Username of process owner", "user"}, - "Pid": AFS{"Process ID", "pid", FmtDefaultable}, - "Ppid": AFS{"Process parent ID", "ppid", FmtDefaultable}, + "Pid": SFS{"Process ID", "pid"}, + "Ppid": SFS{"Process parent ID", "ppid"}, "Job": SFS{"Job ID", "job"}, "Cmd": SFS{"Command name", "cmd"}, "CpuPct": SFS{"cpu% reading (CONSULT DOCUMENTATION)", "cpu_pct"}, @@ -251,16 +248,16 @@ var parseFormatters = DefineTableFromMap( "mem_gb": ZFA{"Virtual memory reading", "CpuKB", FmtDivideBy1M}, "RssAnonKB": SFS{"RssAnon reading", ""}, "res_gb": ZFA{"RssAnon reading", "RssAnonKB", FmtDivideBy1M}, - "Gpus": AFS{"GPU set (`none`,`unknown`,list)", "gpus", FmtDefaultable}, - "GpuPct": AFS{"GPU utilization reading", "gpu_pct", FmtDefaultable}, - "GpuMemPct": AFS{"GPU memory percentage reading", "gpumem_pct", FmtDefaultable}, - "GpuKB": AFS{"GPU memory utilization reading", "gpukib", FmtDefaultable}, - "gpumem_gb": ZFA{"GPU memory utilization reading", "GpuKB", FmtDivideBy1M | FmtDefaultable}, - "GpuFail": AFS{"GPU status flag (0=ok, 1=error state)", "gpu_status", FmtDefaultable}, - "CpuTimeSec": AFS{"CPU time since last reading (seconds, CONSULT DOCUMENTATION)", "cputime_sec", FmtDefaultable}, - "Rolledup": AFS{"Number of rolled-up processes, minus 1", "rolledup", FmtDefaultable}, + "Gpus": SFS{"GPU set (`none`,`unknown`,list)", "gpus"}, + "GpuPct": SFS{"GPU utilization reading", "gpu_pct"}, + "GpuMemPct": SFS{"GPU memory percentage reading", "gpumem_pct"}, + "GpuKB": SFS{"GPU memory utilization reading", "gpukib"}, + "gpumem_gb": ZFA{"GPU memory utilization reading", "GpuKB", FmtDivideBy1M}, + "GpuFail": SFS{"GPU status flag (0=ok, 1=error state)", "gpu_status"}, + "CpuTimeSec": SFS{"CPU time since last reading (seconds, CONSULT DOCUMENTATION)", "cputime_sec"}, + "Rolledup": SFS{"Number of rolled-up processes, minus 1", "rolledup"}, "Flags": SFS{"Bit vector of flags, UTSL", ""}, - "CpuUtilPct": AFS{"CPU utilization since last reading (percent, CONSULT DOCUMENTATION)", "cpu_util_pct", FmtDefaultable}, + "CpuUtilPct": SFS{"CPU utilization since last reading (percent, CONSULT DOCUMENTATION)", "cpu_util_pct"}, }, ) diff --git a/code/sonalyze/cmd/profile/print.go b/code/sonalyze/cmd/profile/print.go index a4f1fb88..4f34be5f 100644 --- a/code/sonalyze/cmd/profile/print.go +++ b/code/sonalyze/cmd/profile/print.go @@ -71,29 +71,22 @@ func (pc *ProfileCommand) printProfile( m *profData, processes sonarlog.SampleStreams, ) error { - if hasRolledup { - // Add the "nproc" / "NumProcs" field if it is required and the fields are still the - // defaults. This is pretty dumb (but compatible with the Rust code); it gets even dumber - // with two versions of the default fields string. - currFields := strings.Join( - uslices.Map( - pc.PrintFields, - func(fs FieldSpec) string { return fs.Name }, - ), - ",", + // Add the "nproc" / "NumProcs" field if it is required (we can't do it until rollup has + // happened) and the fields are still the defaults. This is not quite compatible with older + // sonalyze: here we have default fields only if no fields were specified (defaults were + // applied), while in older code a field list identical to the default would be taken as having + // default fields too. The new logic is better. + // + // Anyway, this hack depends on nothing interesting having happened to pc.Fmt or pc.PrintFields + // after the initial parsing. + hasDefaultFields := pc.Fmt == "" + if hasRolledup && hasDefaultFields { + pc.PrintFields, _, _ = ParseFormatSpec( + profileDefaultFieldsWithNproc, + "", + profileFormatters, + profileAliases, ) - var newFields string - if currFields == v0ProfileDefaultFields { - newFields = v0ProfileDefaultFieldsWithNproc - } else if currFields == v1ProfileDefaultFields { - newFields = v1ProfileDefaultFieldsWithNproc - } - if newFields != "" { - pc.PrintFields = uslices.Map( - strings.Split(newFields, ","), - func(name string) FieldSpec { return FieldSpec{Name: name} }, - ) - } } if pc.PrintOpts.Csv || pc.PrintOpts.Awk { diff --git a/code/sonalyze/table/data.go b/code/sonalyze/table/data.go index ac43fced..93b0b167 100644 --- a/code/sonalyze/table/data.go +++ b/code/sonalyze/table/data.go @@ -43,8 +43,6 @@ func (val IntOrEmpty) String() string { return strconv.FormatInt(int64(val), 10) } -// Formatters that take a context value - func FormatDurationValue(secs int64, ctx PrintMods) string { if (ctx & PrintModSec) != 0 { return fmt.Sprint(secs) @@ -52,18 +50,6 @@ func FormatDurationValue(secs int64, ctx PrintMods) string { return FormatDurationDHM(secs, ctx) } -func FormatDateTimeValue(timestamp int64, ctx PrintMods) string { - // Note, it is part of the API that PrintModSec takes precedence over PrintModIso (this - // simplifies various other paths). - if (ctx & PrintModSec) != 0 { - return fmt.Sprint(timestamp) - } - if (ctx & PrintModIso) != 0 { - return FormatIsoUtc(timestamp) - } - return FormatYyyyMmDdHhMmUtc(timestamp) -} - // The DurationValue had two different formats in older code: %2dd%2dh%2dm and %dd%2dh%2dm. The // embedded spaces made things line up properly in fixed-format outputs of jobs, and most scripts // would likely use "duration/sec" etc instead, but the variability in the leading space is weird @@ -91,6 +77,18 @@ func FormatDurationDHM(durationSec int64, ctx PrintMods) string { return fmt.Sprintf("%dd%dh%dm", days, hours, minutes) } +func FormatDateTimeValue(timestamp int64, ctx PrintMods) string { + // Note, it is part of the API that PrintModSec takes precedence over PrintModIso (this + // simplifies various other paths). + if (ctx & PrintModSec) != 0 { + return fmt.Sprint(timestamp) + } + if (ctx & PrintModIso) != 0 { + return FormatIsoUtc(timestamp) + } + return FormatYyyyMmDdHhMmUtc(timestamp) +} + func FormatYyyyMmDdHhMmUtc(t int64) string { return time.Unix(t, 0).UTC().Format("2006-01-02 15:04") } diff --git a/code/sonalyze/table/data_test.go b/code/sonalyze/table/data_test.go new file mode 100644 index 00000000..71245626 --- /dev/null +++ b/code/sonalyze/table/data_test.go @@ -0,0 +1,49 @@ +package table + +import ( + "fmt" + "testing" +) + +const ( + now = 1732518173 // 2024-11-25T07:02:53Z + dur = 3600*33 + 60*6 + 38 // 1d 9h 7m, rounded up +) + +func TestDataFormatting(t *testing.T) { + if s := DateValue(now).String(); s != "2024-11-25" { + t.Fatalf("DateValue %s", s) + } + if s := TimeValue(now).String(); s != "07:02" { + t.Fatalf("TimeValue %s", s) + } + + if s := IntOrEmpty(7).String(); s != "7" { + t.Fatalf("IntOrEmpty %s", s) + } + if s := IntOrEmpty(0).String(); s != "" { + t.Fatalf("IntOrEmpty %s", s) + } + + if s := FormatDurationValue(dur, 0); s != "1d9h7m" { + t.Fatalf("Duration %s", s) + } + if s := FormatDurationValue(dur, PrintModFixed); s != " 1d 9h 7m" { + t.Fatalf("Duration %s", s) + } + if s := FormatDurationValue(dur, PrintModSec); s != fmt.Sprint(dur) { + t.Fatalf("Duration %s", s) + } + + if s := FormatDateTimeValue(now, 0); s != "2024-11-25 07:02" { + t.Fatalf("DateTimeValue %s", s) + } + if s := FormatDateTimeValue(now, PrintModSec|PrintModIso); s != fmt.Sprint(now) { + t.Fatalf("DateTimeValue %s", s) + } + if s := FormatDateTimeValue(now, PrintModIso); s != "2024-11-25T07:02:53Z" { + t.Fatalf("DateTimeValue %s", s) + } + + // For the other types, the formatters are all embedded in the reflection code. +} diff --git a/code/sonalyze/table/format.go b/code/sonalyze/table/format.go index 111f553c..07f39b9b 100644 --- a/code/sonalyze/table/format.go +++ b/code/sonalyze/table/format.go @@ -492,11 +492,9 @@ func formatAwk(unbufOut io.Writer, fields []FieldSpec, opts *FormatOptions, cols sep := "" for col := range fields { val := cols[col][row] - if !(opts.NoDefaults && val == "*skip*") { - line.WriteString(sep) - line.WriteString(strings.ReplaceAll(val, " ", "_")) - sep = " " - } + line.WriteString(sep) + line.WriteString(strings.ReplaceAll(val, " ", "_")) + sep = " " } if opts.Tag != "" { line.WriteString(sep) diff --git a/code/sonalyze/table/reflect.go b/code/sonalyze/table/reflect.go index 8614e64b..d2fdb4e0 100644 --- a/code/sonalyze/table/reflect.go +++ b/code/sonalyze/table/reflect.go @@ -8,8 +8,6 @@ // allowlist. // // Both will descend into embedded fields. -// -// TODO: At the moment they do not handle circular structures, but they could (and should). package table @@ -39,12 +37,9 @@ var ( // Given a struct type, DefineTableFromTags constructs a map from field names to a formatter for // each field. Fields are excluded if they appear in isExcluded or have no `desc` annotation. // -// A field may have an `alias` annotation in addition to its name. The alias is treated just as the -// name. Aliases are a consequence of older code using "convenient" names for fields while we want -// to move to a world where fields are named in a transparent and uniform way. Clients can ask for -// the real name or the alias. Default lists in client code can refer to whatever fields they want. -// The `alias` annotation is a comma-separated list of alias names. Fields are excluded if any of -// their aliases appear in isExcluded. +// A field may have an `alias` annotation in addition to its name. The `alias` annotation is a +// comma-separated list of alias names. Fields are excluded if any of their aliases appear in +// isExcluded. // // There must be no duplicates in the union of field names and aliases, or in the set of aliases. // @@ -95,8 +90,8 @@ func DefineTableFromTags( // // The field values must be one of the *FormatSpec types below. // -// SimpleFormatSpecWithAttr uses an attribute to specify a simple formatting rule, that in the case -// of DefineTableFromTags can be expressed through a type. +// SimpleFormatSpecWithAttr uses an attribute to specify a simple formatting rule that in the case +// of DefineTableFromTags would be expressed through a type. // // SynthesizedFormatSpecWithAttr uses an attribute to specify a simple formatting rule for a // synthesized output field computed from a real field. @@ -106,21 +101,17 @@ type SimpleFormatSpec struct { Aliases string } -// `FmtDefaultable` indicates that the field has a default value to skip if nodefaults is set. -// Numbers, Ustr, string, and GpuSet are defaultable. -// // `FmtCeil` and `FmtDivideBy1M` apply simple numeric transformations. (There could be more.) // -// For `Fmt` see Typename at top of file - these attributes request that values be -// formatted as for those types. +// For `Fmt` see Typename in data.go - these attributes request that values be formatted +// as for those types. const ( - FmtDefaultable = (1 << iota) - FmtCeil // type must be floating, take ceil, print as integer - FmtDivideBy1M // type must be integer, integer divide by 1024*1024 - FmtDateTimeValue // type must be int64 - FmtIsoDateTimeValue // type must be int64 - FmtDurationValue // type must be int64 + FmtCeil = (1 << iota) // type must be floating, take ceil, print as integer + FmtDivideBy1M // type must be integer, integer divide by 1024*1024 + FmtDateTimeValue // type must be int64 + FmtIsoDateTimeValue // type must be int64 + FmtDurationValue // type must be int64 ) type SimpleFormatSpecWithAttr struct { @@ -132,8 +123,7 @@ type SimpleFormatSpecWithAttr struct { type SynthesizedFormatSpecWithAttr struct { Desc string RealName string - // TODO: Should have aliases probably - Attr int + Attr int } func DefineTableFromMap( @@ -294,7 +284,7 @@ func reflectTypeFormatter(ix int, attrs int, ty reflect.Type) func(any, PrintMod case ty == dateTimeValueTy || ty == dateTimeValueOrBlankTy: return func(d any, ctx PrintMods) string { val := reflect.Indirect(reflect.ValueOf(d)).Field(ix).Int() - if (attrs&FmtDefaultable) != 0 && (ctx&PrintModNoDefaults) != 0 && val == 0 { + if (ctx&PrintModNoDefaults) != 0 && val == 0 { return "*skip*" } if val == 0 && ty == dateTimeValueOrBlankTy { @@ -313,7 +303,7 @@ func reflectTypeFormatter(ix int, attrs int, ty reflect.Type) func(any, PrintMod case ty == durationValueTy: return func(d any, ctx PrintMods) string { val := reflect.Indirect(reflect.ValueOf(d)).Field(ix).Int() - if (attrs&FmtDefaultable) != 0 && (ctx&PrintModNoDefaults) != 0 && val == 0 { + if (ctx&PrintModNoDefaults) != 0 && val == 0 { return "*skip*" } return FormatDurationValue(val, ctx) @@ -323,7 +313,7 @@ func reflectTypeFormatter(ix int, attrs int, ty reflect.Type) func(any, PrintMod // GpuSet is uint32 val := Ustr(reflect.Indirect(reflect.ValueOf(d)).Field(ix).Uint()) set := gpuset.GpuSet(val) - if (attrs&FmtDefaultable) != 0 && (ctx&PrintModNoDefaults) != 0 && set.IsEmpty() { + if (ctx&PrintModNoDefaults) != 0 && set.IsEmpty() { return "*skip*" } return set.String() @@ -332,7 +322,7 @@ func reflectTypeFormatter(ix int, attrs int, ty reflect.Type) func(any, PrintMod return func(d any, ctx PrintMods) string { // Ustr is uint32 val := Ustr(reflect.Indirect(reflect.ValueOf(d)).Field(ix).Uint()) - if (attrs&FmtDefaultable) != 0 && (ctx&PrintModNoDefaults) != 0 && val == UstrEmpty { + if (ctx&PrintModNoDefaults) != 0 && val == UstrEmpty { return "*skip*" } s := val.String() @@ -354,7 +344,7 @@ func reflectTypeFormatter(ix int, attrs int, ty reflect.Type) func(any, PrintMod return func(d any, ctx PrintMods) string { vals := reflect.Indirect(reflect.ValueOf(d)).Field(ix) lim := vals.Len() - if (attrs&FmtDefaultable) != 0 && (ctx&PrintModNoDefaults) != 0 && lim == 0 { + if (ctx&PrintModNoDefaults) != 0 && lim == 0 { return "*skip*" } ss := make([]string, lim) @@ -365,14 +355,14 @@ func reflectTypeFormatter(ix int, attrs int, ty reflect.Type) func(any, PrintMod return strings.Join(ss, ",") } default: - panic("NYI") + panic("NYI - non-string slice") } case ty.Implements(stringerTy): // If it implements fmt.Stringer then use it return func(d any, ctx PrintMods) string { val := reflect.Indirect(reflect.ValueOf(d)).Field(ix) s := val.MethodByName("String").Call(nil)[0].String() - if (attrs&FmtDefaultable) != 0 && (ctx&PrintModNoDefaults) != 0 && s == "" { + if (ctx&PrintModNoDefaults) != 0 && s == "" { return "*skip*" } return s @@ -383,7 +373,7 @@ func reflectTypeFormatter(ix int, attrs int, ty reflect.Type) func(any, PrintMod case reflect.Bool: return func(d any, ctx PrintMods) string { val := reflect.Indirect(reflect.ValueOf(d)).Field(ix).Bool() - if (attrs&FmtDefaultable) != 0 && (ctx&PrintModNoDefaults) != 0 && !val { + if (ctx&PrintModNoDefaults) != 0 && !val { return "*skip*" } // These are backwards compatible values. @@ -395,7 +385,12 @@ func reflectTypeFormatter(ix int, attrs int, ty reflect.Type) func(any, PrintMod case reflect.Int64: return func(d any, ctx PrintMods) string { val := reflect.Indirect(reflect.ValueOf(d)).Field(ix).Int() - if (attrs&FmtDefaultable) != 0 && (ctx&PrintModNoDefaults) != 0 && val == 0 { + // Scale before default-checking. This will scale date values too, but that's + // considered a user error. + if (attrs & FmtDivideBy1M) != 0 { + val /= 1024 * 1024 + } + if (ctx&PrintModNoDefaults) != 0 && val == 0 { return "*skip*" } if (attrs & FmtDateTimeValue) != 0 { @@ -407,31 +402,30 @@ func reflectTypeFormatter(ix int, attrs int, ty reflect.Type) func(any, PrintMod if (attrs & FmtDurationValue) != 0 { return FormatDurationValue(val, ctx) } - if (attrs & FmtDivideBy1M) != 0 { - val /= 1024 * 1024 - } return strconv.FormatInt(val, 10) } case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32: return func(d any, ctx PrintMods) string { val := reflect.Indirect(reflect.ValueOf(d)).Field(ix).Int() - if (attrs&FmtDefaultable) != 0 && (ctx&PrintModNoDefaults) != 0 && val == 0 { - return "*skip*" - } + // Scale before default-checking. if (attrs & FmtDivideBy1M) != 0 { val /= 1024 * 1024 } + if (ctx&PrintModNoDefaults) != 0 && val == 0 { + return "*skip*" + } return strconv.FormatInt(val, 10) } case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: return func(d any, ctx PrintMods) string { val := reflect.Indirect(reflect.ValueOf(d)).Field(ix).Uint() - if (attrs&FmtDefaultable) != 0 && (ctx&PrintModNoDefaults) != 0 && val == 0 { - return "*skip*" - } + // Scale before default-checking. if (attrs & FmtDivideBy1M) != 0 { val /= 1024 * 1024 } + if (ctx&PrintModNoDefaults) != 0 && val == 0 { + return "*skip*" + } return strconv.FormatUint(val, 10) } case reflect.Float32, reflect.Float64: @@ -440,7 +434,7 @@ func reflectTypeFormatter(ix int, attrs int, ty reflect.Type) func(any, PrintMod if (attrs & FmtCeil) != 0 { val = math.Ceil(val) } - if (attrs&FmtDefaultable) != 0 && (ctx&PrintModNoDefaults) != 0 && val == 0 { + if (ctx&PrintModNoDefaults) != 0 && val == 0 { return "*skip*" } prec := 64 @@ -452,7 +446,7 @@ func reflectTypeFormatter(ix int, attrs int, ty reflect.Type) func(any, PrintMod case reflect.String: return func(d any, ctx PrintMods) string { val := reflect.Indirect(reflect.ValueOf(d)).Field(ix).String() - if (attrs&FmtDefaultable) != 0 && (ctx&PrintModNoDefaults) != 0 && val == "" { + if (ctx&PrintModNoDefaults) != 0 && val == "" { return "*skip*" } return val diff --git a/code/sonalyze/table/reflect_test.go b/code/sonalyze/table/reflect_test.go index 12559c94..549627e4 100644 --- a/code/sonalyze/table/reflect_test.go +++ b/code/sonalyze/table/reflect_test.go @@ -1,12 +1,18 @@ package table import ( + "fmt" "reflect" "testing" + "go-utils/gpuset" uslices "go-utils/slices" + + . "sonalyze/common" ) +// Structure traversal tests + type S1 struct { x int `desc:"x field" alias:"xx"` *T1 @@ -88,3 +94,316 @@ func TestFormatting3(t *testing.T) { t.Fatal(ss) } } + +// Field formatter tests. Also see data_test.go. + +// now and dur are defined in data_test.go + +func TestReflectDateTimeValue(t *testing.T) { + type s0 struct { + V DateTimeValue + } + s0f := reflectTypeFormatter(0, 0, reflect.TypeFor[DateTimeValue]()) + if s := s0f(s0{now}, 0); s != "2024-11-25 07:02" { + t.Fatalf("DateTimeValue %s", s) + } + if s := s0f(s0{0}, PrintModNoDefaults); s != "*skip*" { + t.Fatalf("DateTimeValue %s", s) + } +} + +func TestReflectDateTimeValueOrBlank(t *testing.T) { + type s1 struct { + V DateTimeValueOrBlank + } + s1f := reflectTypeFormatter(0, 0, reflect.TypeFor[DateTimeValueOrBlank]()) + if s := s1f(s1{now}, 0); s != "2024-11-25 07:02" { + t.Fatalf("DateTimeValueOrBlank %s", s) + } + if s := s1f(s1{0}, 0); s != " " { + t.Fatalf("DateTimeValueOrBlank %s", s) + } + if s := s1f(s1{0}, PrintModNoDefaults); s != "*skip*" { + t.Fatalf("DateTimeValueOrBlank %s", s) + } +} + +func TestReflectIsoDateTimeOrUnknown(t *testing.T) { + type s2 struct { + V IsoDateTimeOrUnknown + } + s2f := reflectTypeFormatter(0, 0, reflect.TypeFor[IsoDateTimeOrUnknown]()) + if s := s2f(s2{now}, 0); s != "2024-11-25T07:02:53Z" { + t.Fatalf("IsoDateTimeOrUnknown %s", s) + } + if s := s2f(s2{now}, PrintModSec); s != fmt.Sprint(now) { + t.Fatalf("IsoDateTimeOrUnknown %s", s) + } + if s := s2f(s2{0}, PrintModSec); s != "Unknown" { // "Unknown" wins over "/sec" + t.Fatalf("IsoDateTimeOrUnknown %s", s) + } + if s := s2f(s2{0}, PrintModNoDefaults); s != "Unknown" { // "Unknown" wins over "nodefaults" + t.Fatalf("IsoDateTimeOrUnknown %s", s) + } +} + +func TestReflectDurationValue(t *testing.T) { + type s3 struct { + V DurationValue + } + s3f := reflectTypeFormatter(0, 0, reflect.TypeFor[DurationValue]()) + if s := s3f(s3{dur}, 0); s != "1d9h7m" { + t.Fatalf("DurationValue %s", s) + } + if s := s3f(s3{dur}, PrintModSec); s != fmt.Sprint(dur) { + t.Fatalf("DurationValue %s", s) + } + if s := s3f(s3{0}, PrintModNoDefaults); s != "*skip*" { + t.Fatalf("DurationValue %s", s) + } +} + +func TestReflectGpuSet(t *testing.T) { + type s4 struct { + V gpuset.GpuSet + } + s13, err := gpuset.NewGpuSet("1,3") + if err != nil { + t.Fatal(err) + } + s4f := reflectTypeFormatter(0, 0, reflect.TypeFor[gpuset.GpuSet]()) + if s := s4f(s4{s13}, 0); s != "1,3" { + t.Fatalf("GpuSet %s", s) + } + if s := s4f(s4{gpuset.EmptyGpuSet()}, PrintModNoDefaults); s != "*skip*" { + t.Fatalf("GpuSet %s", s) + } +} + +func TestReflectUstrMax30(t *testing.T) { + type st struct { + V UstrMax30 + } + sf := reflectTypeFormatter(0, 0, reflect.TypeFor[UstrMax30]()) + d := st{UstrMax30(StringToUstr("supercallifragilisticexpialidocious"))} + if s := sf(d, 0); s != "supercallifragilisticexpialidocious" { + t.Fatalf("UstrMax30 %s", s) + } + if s := sf(d, PrintModFixed); s != "supercallifragilisticexpialido" { + t.Fatalf("UstrMax30 %s", s) + } + if s := sf(st{UstrMax30(UstrEmpty)}, PrintModNoDefaults); s != "*skip*" { + t.Fatalf("UstrMax30 %s", s) + } +} + +func TestReflectSlice(t *testing.T) { + type st struct { + V []string + } + sf := reflectTypeFormatter(0, 0, reflect.TypeFor[[]string]()) + if s := sf(st{[]string{"ho", "hi", "z", "a"}}, 0); s != "a,hi,ho,z" { + t.Fatalf("[]string %s", s) + } + if s := sf(st{[]string{}}, 0); s != "" { + t.Fatalf("[]string %s", s) + } + if s := sf(st{[]string{}}, PrintModNoDefaults); s != "*skip*" { + t.Fatalf("[]string %s", s) + } +} + +type tstringer int + +func (t tstringer) String() string { + if t < 0 { + return "" + } + return fmt.Sprintf("+%d+", int(t)) +} + +func TestReflectStringer(t *testing.T) { + type st struct { + V tstringer + } + sf := reflectTypeFormatter(0, 0, reflect.TypeFor[tstringer]()) + if s := sf(st{33}, PrintModNoDefaults); s != "+33+" { + t.Fatalf("stringer %s", s) + } + if s := sf(st{-1}, PrintModNoDefaults); s != "*skip*" { + t.Fatalf("stringer %s", s) + } +} + +func TestReflectBool(t *testing.T) { + type st struct { + V bool + } + sf := reflectTypeFormatter(0, 0, reflect.TypeFor[bool]()) + if s := sf(st{true}, 0); s != "yes" { + t.Fatalf("bool %s", s) + } + if s := sf(st{false}, 0); s != "no" { + t.Fatalf("bool %s", s) + } + if s := sf(st{false}, PrintModNoDefaults); s != "*skip*" { + t.Fatalf("bool %s", s) + } +} + +func TestReflectInt64(t *testing.T) { + type st struct { + V int64 + } + + // Plain type, plain printing + sf := reflectTypeFormatter(0, 0, reflect.TypeFor[int64]()) + if s := sf(st{37}, 0); s != "37" { + t.Fatalf("int64 %s", s) + } + if s := sf(st{0}, PrintModNoDefaults); s != "*skip*" { + t.Fatalf("int64 %s", s) + } + + // For the following, NoDefaults takes precedence over the formatting directive + + // Scaled + sf = reflectTypeFormatter(0, FmtDivideBy1M, reflect.TypeFor[int64]()) + if s := sf(st{123456789}, 0); s != "117" { + t.Fatalf("int64 scaled %s", s) + } + if s := sf(st{0}, PrintModNoDefaults); s != "*skip*" { + t.Fatalf("int64 scaled %s", s) + } + + // As DateTimeValue + sf = reflectTypeFormatter(0, FmtDateTimeValue, reflect.TypeFor[int64]()) + if s := sf(st{now}, 0); s != "2024-11-25 07:02" { + t.Fatalf("int64 datetime %s", s) + } + if s := sf(st{0}, PrintModNoDefaults); s != "*skip*" { + t.Fatalf("int64 datetime %s", s) + } + + // As IsoDateTimeValue + sf = reflectTypeFormatter(0, FmtIsoDateTimeValue, reflect.TypeFor[int64]()) + if s := sf(st{now}, 0); s != "2024-11-25T07:02:53Z" { + t.Fatalf("int64 isodatetime %s", s) + } + if s := sf(st{0}, PrintModNoDefaults); s != "*skip*" { + t.Fatalf("int64 isodatetime %s", s) + } + + // As Duration + sf = reflectTypeFormatter(0, FmtDurationValue, reflect.TypeFor[int64]()) + if s := sf(st{dur}, 0); s != "1d9h7m" { + t.Fatalf("int64 duration %s", s) + } + if s := sf(st{0}, PrintModNoDefaults); s != "*skip*" { + t.Fatalf("int64 duration %s", s) + } +} + +// int, int8, int16, int32, uint, uint8, uint16, uint32, uint64 all have the same logic, which is +// the same as the base case logic of int64. The only complication is the scaling values. +// +// Notably, the default value test applies *before* scaling. This is probably not desirable. +// Contrast the float case below, where the ceiling operation is performed first and we then check +// for the default value. + +func testReflectIntish[T int | int8 | int16 | int32 | int64 | uint | uint8 | uint16 | uint32 | uint64]( + t *testing.T, + val, scaled T, +) { + type st struct { + V T + } + + // Plain printing + sf := reflectTypeFormatter(0, 0, reflect.TypeFor[T]()) + if s := sf(st{val}, 0); s != fmt.Sprint(val) { + t.Fatalf("intish %s", s) + } + if s := sf(st{0}, PrintModNoDefaults); s != "*skip*" { + t.Fatalf("intish %s", s) + } + + // Scaled. + sf = reflectTypeFormatter(0, FmtDivideBy1M, reflect.TypeFor[T]()) + if s := sf(st{val}, 0); s != fmt.Sprint(scaled) { + t.Fatalf("intish scaled %s", s) + } + if s := sf(st{0}, PrintModNoDefaults); s != "*skip*" { + t.Fatalf("intish scaled %s", s) + } + // If scaling produces zero then the NoDefault filtering kicks in + if s := sf(st{127}, PrintModNoDefaults); s != "*skip*" { + t.Fatalf("intish scaled %s", s) + } +} + +func TestReflectIntish(t *testing.T) { + testReflectIntish[int](t, 123456789, 117) + testReflectIntish[int32](t, 123456789, 117) + testReflectIntish[uint](t, 123456789, 117) + testReflectIntish[uint32](t, 123456789, 117) + testReflectIntish[uint64](t, 123456789, 117) + testReflectIntish[int8](t, 123, 0) + testReflectIntish[int16](t, 1234, 0) + testReflectIntish[uint8](t, 123, 0) + testReflectIntish[uint16](t, 1234, 0) + // Re-test int64 to ensure the logic is the same as for the others + testReflectIntish[int64](t, 123456789, 117) +} + +// float32 and float64 have the same logic. The rounding operations are applied before default +// testing, cf the opposite logic for ints above. + +func testReflectFloatish[T float32 | float64](t *testing.T, val, ceiling T) { + type st struct { + V T + } + + sf := reflectTypeFormatter(0, 0, reflect.TypeFor[T]()) + if s := sf(st{val}, 0); s != fmt.Sprint(val) { + t.Fatalf("floatish %s", s) + } + if s := sf(st{0}, PrintModNoDefaults); s != "*skip*" { + t.Fatalf("floatish %s", s) + } + + sf = reflectTypeFormatter(0, FmtCeil, reflect.TypeFor[T]()) + if val >= 0 { + if s := sf(st{val}, 0); s != fmt.Sprint(ceiling) { + t.Fatalf("floatish %s", s) + } + } else { + if s := sf(st{val}, PrintModNoDefaults); s != "*skip*" { + t.Fatalf("floatish %s", s) + } + } +} + +func TestReflectFloatish(t *testing.T) { + testReflectFloatish[float32](t, 13.75, 14) + testReflectFloatish[float32](t, -0.5, 0) + testReflectFloatish[float64](t, 13.75, 14) + testReflectFloatish[float64](t, -0.5, 0) +} + +func TestReflectString(t *testing.T) { + type st struct { + V string + } + + sf := reflectTypeFormatter(0, 0, reflect.TypeFor[string]()) + if s := sf(st{"hi there"}, 0); s != "hi there" { + t.Fatalf("string %s", s) + } + if s := sf(st{"hi there"}, PrintModNoDefaults); s != "hi there" { + t.Fatalf("string %s", s) + } + if s := sf(st{""}, PrintModNoDefaults); s != "*skip*" { + t.Fatalf("string %s", s) + } +} diff --git a/code/tests/relative/sonalyze/profile-print.sh b/code/tests/relative/sonalyze/profile-print.sh index 3623cca9..a1ecaa57 100755 --- a/code/tests/relative/sonalyze/profile-print.sh +++ b/code/tests/relative/sonalyze/profile-print.sh @@ -16,17 +16,19 @@ source test-settings # We want this to pick something with more than one process, so do our best. It might find nothing. job=$($OLD_SONALYZE jobs -data-dir "$DATA_PATH" -config-file "$CONFIG" -f 2d -t 1d -min-runtime 1h -u - -fmt awk,job,cmd | grep , | head -n 1 | awk '{ print $1 }') -# No 'default' alias in the old code, so expand it +# No 'default' alias in the old code, so expand it. Here I add nproc because I don't want to +# test the nproc-insertion logic, as it is different in the new code. echo "Format old: fixed,default" -$OLD_SONALYZE profile -data-dir "$DATA_PATH" -f 20d -config-file "$CONFIG" -job $job -t 1d -fmt fixed,time,cpu,mem,gpu,gpumem,cmd > old-output.txt -$NEW_SONALYZE profile -data-dir "$DATA_PATH" -f 20d -config-file "$CONFIG" -job $job -t 1d -fmt fixed,default > new-output.txt +$OLD_SONALYZE profile -data-dir "$DATA_PATH" -f 20d -config-file "$CONFIG" -job $job -t 1d -fmt fixed,time,cpu,mem,gpu,gpumem,cmd,nproc > old-output.txt +$NEW_SONALYZE profile -data-dir "$DATA_PATH" -f 20d -config-file "$CONFIG" -job $job -t 1d -fmt fixed,default,nproc > new-output.txt diff -b old-output.txt new-output.txt rm -f old-output.txt new-output.txt # v0 and v1 default (new names) should print the same. It's broken that this is 'mem' and not 'res' but we're not going to rock that boat. +# Again add nproc to avoid testing the nproc-insertion logic. echo "Format old vs v1default: fixed,default" -$OLD_SONALYZE profile -data-dir "$DATA_PATH" -f 20d -config-file "$CONFIG" -job $job -t 1d -fmt fixed,noheader,time,cpu,mem,gpu,gpumem,cmd > old-output.txt -$NEW_SONALYZE profile -data-dir "$DATA_PATH" -f 20d -config-file "$CONFIG" -job $job -t 1d -fmt fixed,noheader,v1default > new-output.txt +$OLD_SONALYZE profile -data-dir "$DATA_PATH" -f 20d -config-file "$CONFIG" -job $job -t 1d -fmt fixed,noheader,time,cpu,mem,gpu,gpumem,cmd,nproc > old-output.txt +$NEW_SONALYZE profile -data-dir "$DATA_PATH" -f 20d -config-file "$CONFIG" -job $job -t 1d -fmt fixed,noheader,v1default,nproc > new-output.txt diff -b old-output.txt new-output.txt rm -f old-output.txt new-output.txt diff --git a/code/tests/sonalyze/format/nodefault.sh b/code/tests/sonalyze/format/nodefault.sh index 0f87a481..13a6ad93 100644 --- a/code/tests/sonalyze/format/nodefault.sh +++ b/code/tests/sonalyze/format/nodefault.sh @@ -1,9 +1,9 @@ output=$($SONALYZE parse --fmt=csvnamed,all,nodefaults -- nodefault.csv) CHECK format_csv_nodefault \ - 'version=0.7.0,localtime=2023-10-04 07:40,host=ml4.hpc.uio.no,cores=64,memtotal=0,user=einarvid,job=1269178,cmd=python3,cpu_pct=1714.2,mem_gb=261,res_gb=0,cputime_sec=10192,rolledup=69' \ + 'version=0.7.0,localtime=2023-10-04 07:40,host=ml4.hpc.uio.no,cores=64,user=einarvid,job=1269178,cmd=python3,cpu_pct=1714.2,mem_gb=261,cputime_sec=10192,rolledup=69' \ "$output" output=$($SONALYZE parse --fmt=json,all,nodefaults -- nodefault.csv) CHECK format_csv_nodefault \ - '[{"version":"0.7.0","localtime":"2023-10-04 07:40","host":"ml4.hpc.uio.no","cores":"64","memtotal":"0","user":"einarvid","job":"1269178","cmd":"python3","cpu_pct":"1714.2","mem_gb":"261","res_gb":"0","cputime_sec":"10192","rolledup":"69"}]' \ + '[{"version":"0.7.0","localtime":"2023-10-04 07:40","host":"ml4.hpc.uio.no","cores":"64","user":"einarvid","job":"1269178","cmd":"python3","cpu_pct":"1714.2","mem_gb":"261","cputime_sec":"10192","rolledup":"69"}]' \ "$output" diff --git a/code/tests/sonalyze/parse/parse.sh b/code/tests/sonalyze/parse/parse.sh index 4faf9528..90593a12 100644 --- a/code/tests/sonalyze/parse/parse.sh +++ b/code/tests/sonalyze/parse/parse.sh @@ -8,4 +8,4 @@ output=$($SONALYZE parse --fmt=json,all -- empty_input.csv) CHECK parse_json_empty "[]" "$output" output=$($SONALYZE parse --fmt=json,all,nodefaults -- parse.csv) -CHECK parse_json '[{"version":"0.7.0","localtime":"2023-10-04 07:40","host":"ml4.hpc.uio.no","cores":"64","memtotal":"0","user":"einarvid","job":"1269178","cmd":"python3","cpu_pct":"1714.2","mem_gb":"261","res_gb":"0","cputime_sec":"10192","rolledup":"69"}]' "$output" +CHECK parse_json '[{"version":"0.7.0","localtime":"2023-10-04 07:40","host":"ml4.hpc.uio.no","cores":"64","user":"einarvid","job":"1269178","cmd":"python3","cpu_pct":"1714.2","mem_gb":"261","cputime_sec":"10192","rolledup":"69"}]' "$output"