From 1323a3b1b83d0fcd07d8a9f553f554eda4be6d0f Mon Sep 17 00:00:00 2001 From: Lars T Hansen Date: Mon, 18 Nov 2024 13:08:07 +0100 Subject: [PATCH] Fix #654 - Upgrade to go 1.22 --- code/go-utils/go.mod | 2 +- code/heatmap/go.mod | 2 +- code/jsoncheck/go.mod | 2 +- code/make-cluster-config/go.mod | 2 +- code/naicreport/go.mod | 2 +- code/netsink/go.mod | 2 +- code/numdiff/go.mod | 2 +- code/sacctd/go.mod | 2 +- code/slurminfo/go.mod | 2 +- code/sonalyze/clusters/clusters.go | 3 +- code/sonalyze/command/reflect.go | 46 +++++---------- code/sonalyze/command/reflect_test.go | 3 +- code/sonalyze/configs/configs.go | 3 +- code/sonalyze/go.mod | 2 +- code/sonalyze/load/print.go | 6 +- code/sonalyze/metadata/metadata.go | 6 +- code/sonalyze/nodes/nodes.go | 3 +- code/sonalyze/parse/parse.go | 17 +++++- code/sonalyze/profile/print.go | 6 +- code/sonalyze/sacct/print.go | 6 +- code/sonalyze/sonarlog/postprocess.go | 7 ++- code/sonalyze/sonarlog/sort.go | 81 --------------------------- code/sonalyze/uptime/perform.go | 15 +++-- code/sonalyze/uptime/print.go | 6 +- code/sonard/go.mod | 2 +- 25 files changed, 65 insertions(+), 165 deletions(-) delete mode 100644 code/sonalyze/sonarlog/sort.go diff --git a/code/go-utils/go.mod b/code/go-utils/go.mod index c34a304d..d8be83b5 100644 --- a/code/go-utils/go.mod +++ b/code/go-utils/go.mod @@ -1,6 +1,6 @@ module jobanalyzer_go_utils -go 1.21.10 +go 1.22.1 require go-utils v0.0.0-00010101000000-000000000000 replace go-utils => ../go-utils diff --git a/code/heatmap/go.mod b/code/heatmap/go.mod index 90b3bfef..4a4a01cd 100644 --- a/code/heatmap/go.mod +++ b/code/heatmap/go.mod @@ -1,3 +1,3 @@ module heatmap -go 1.21.11 +go 1.22.1 diff --git a/code/jsoncheck/go.mod b/code/jsoncheck/go.mod index 36100a41..22015f5d 100644 --- a/code/jsoncheck/go.mod +++ b/code/jsoncheck/go.mod @@ -1,3 +1,3 @@ module jsoncheck -go 1.21.10 +go 1.22.1 diff --git a/code/make-cluster-config/go.mod b/code/make-cluster-config/go.mod index 6a948b4e..2b315b1f 100644 --- a/code/make-cluster-config/go.mod +++ b/code/make-cluster-config/go.mod @@ -1,6 +1,6 @@ module make-cluster-config -go 1.21.10 +go 1.22.1 require go-utils v0.0.0-00010101000000-000000000000 replace go-utils => ../go-utils diff --git a/code/naicreport/go.mod b/code/naicreport/go.mod index 47f7010b..88052786 100644 --- a/code/naicreport/go.mod +++ b/code/naicreport/go.mod @@ -1,6 +1,6 @@ module naicreport -go 1.21.10 +go 1.22.1 require go-utils v0.0.0-00010101000000-000000000000 replace go-utils => ../go-utils diff --git a/code/netsink/go.mod b/code/netsink/go.mod index 89b5ad76..1d9bf4b7 100644 --- a/code/netsink/go.mod +++ b/code/netsink/go.mod @@ -1,6 +1,6 @@ module netsink -go 1.21.10 +go 1.22.1 require go-utils v0.0.0-00010101000000-000000000000 replace go-utils => ../go-utils diff --git a/code/numdiff/go.mod b/code/numdiff/go.mod index ba3cda31..546705b5 100644 --- a/code/numdiff/go.mod +++ b/code/numdiff/go.mod @@ -1,3 +1,3 @@ module numdiff -go 1.21.10 +go 1.22.1 diff --git a/code/sacctd/go.mod b/code/sacctd/go.mod index fbd7c085..e47780aa 100644 --- a/code/sacctd/go.mod +++ b/code/sacctd/go.mod @@ -1,3 +1,3 @@ module sacctd -go 1.21.11 +go 1.22.1 diff --git a/code/slurminfo/go.mod b/code/slurminfo/go.mod index 6c7b76d9..7e5a769f 100644 --- a/code/slurminfo/go.mod +++ b/code/slurminfo/go.mod @@ -1,6 +1,6 @@ module slurminfo -go 1.21.10 +go 1.22.1 require go-utils v0.0.0-00010101000000-000000000000 replace go-utils => ../go-utils diff --git a/code/sonalyze/clusters/clusters.go b/code/sonalyze/clusters/clusters.go index a3dd1f9e..4f55570c 100644 --- a/code/sonalyze/clusters/clusters.go +++ b/code/sonalyze/clusters/clusters.go @@ -142,8 +142,7 @@ type SFS = SimpleFormatSpec // MT: Constant after initialization; immutable var clustersFormatters = DefineTableFromMap( - // TODO: Go 1.22, reflect.TypeFor[db.ClusterEntry] - reflect.TypeOf((*db.ClusterEntry)(nil)).Elem(), + reflect.TypeFor[db.ClusterEntry](), map[string]any{ "Name": SFS{"Cluster name", "cluster"}, "Description": SFS{"Human-consumable cluster summary", "desc"}, diff --git a/code/sonalyze/command/reflect.go b/code/sonalyze/command/reflect.go index f6d43990..814c412b 100644 --- a/code/sonalyze/command/reflect.go +++ b/code/sonalyze/command/reflect.go @@ -61,27 +61,13 @@ func (val IntOrEmpty) String() string { // Type representations of some of those, for when we need the types. var ( - // TODO: When we can adopt Go 1.22: valTy := reflect.TypeFor[TypeName]() - dummyDateTimeValue DateTimeValue - dateTimeValueTy = reflect.TypeOf(dummyDateTimeValue) - - dummyDateTimeValueOrBlank DateTimeValueOrBlank - dateTimeValueOrBlankTy = reflect.TypeOf(dummyDateTimeValueOrBlank) - - dummyIsoDateTimeOrUnknown IsoDateTimeOrUnknown - isoDateTimeOrUnknownTy = reflect.TypeOf(dummyIsoDateTimeOrUnknown) - - dummyDurationValue DurationValue - durationValueTy = reflect.TypeOf(dummyDurationValue) - - dummyUstrMax30Value UstrMax30 - ustrMax30Ty = reflect.TypeOf(dummyUstrMax30Value) - - dummyGpuSetValue gpuset.GpuSet - gpuSetTy = reflect.TypeOf(dummyGpuSetValue) - - // See the Example for reflect.TypeOf in the Go documentation. - stringerTy = reflect.TypeOf((*fmt.Stringer)(nil)).Elem() + dateTimeValueTy = reflect.TypeFor[DateTimeValue]() + dateTimeValueOrBlankTy = reflect.TypeFor[DateTimeValueOrBlank]() + isoDateTimeOrUnknownTy = reflect.TypeFor[IsoDateTimeOrUnknown]() + durationValueTy = reflect.TypeFor[DurationValue]() + ustrMax30Ty = reflect.TypeFor[UstrMax30]() + gpuSetTy = reflect.TypeFor[gpuset.GpuSet]() + stringerTy = reflect.TypeFor[fmt.Stringer]() ) // Given a struct type, DefineTableFromTags constructs a map from field names to a formatter for @@ -278,9 +264,7 @@ func reflectStructFormatters( panic("Struct type required") } for i, lim := 0, structTy.NumField(); i < lim; i++ { - // TODO: once we move to Go 1.22: no temp binding - ix := i - fld := structTy.Field(ix) + fld := structTy.Field(i) if fld.Anonymous { // Trace through embedded field. The formatting function will receive the outer // structure (or pointer to it), but the formatter generator code operates on the inner @@ -300,20 +284,18 @@ func reflectStructFormatters( subFormatters := make(map[string]Formatter) reflectStructFormatters(fldTy, subFormatters, admissible, synthesizable) for name, fmt := range subFormatters { - // TODO: once we move to Go 1.22: no temp binding - theFmt := fmt.Fmt f := Formatter{ Help: fmt.Help, } if mustTakeAddress { f.Fmt = func(d any, mods PrintMods) string { - val := reflect.Indirect(reflect.ValueOf(d)).Field(ix).Addr() - return theFmt(val.Interface(), mods) + val := reflect.Indirect(reflect.ValueOf(d)).Field(i).Addr() + return fmt.Fmt(val.Interface(), mods) } } else { f.Fmt = func(d any, mods PrintMods) string { - val := reflect.Indirect(reflect.ValueOf(d)).Field(ix) - return theFmt(val.Interface(), mods) + val := reflect.Indirect(reflect.ValueOf(d)).Field(i) + return fmt.Fmt(val.Interface(), mods) } } formatters[name] = f @@ -322,7 +304,7 @@ func reflectStructFormatters( if ok, name, desc, aliases, attrs := admissible(fld); ok { f := Formatter{ Help: desc, - Fmt: reflectTypeFormatter(ix, attrs, fld.Type), + Fmt: reflectTypeFormatter(i, attrs, fld.Type), } formatters[name] = f for _, a := range aliases { @@ -338,7 +320,7 @@ func reflectStructFormatters( // we'll require synthesizable (or earlier code) to panic if that case occurs. f := Formatter{ Help: desc, - Fmt: reflectTypeFormatter(ix, attrs, fld.Type), + Fmt: reflectTypeFormatter(i, attrs, fld.Type), } formatters[name] = f } diff --git a/code/sonalyze/command/reflect_test.go b/code/sonalyze/command/reflect_test.go index 261ae62c..ae79481c 100644 --- a/code/sonalyze/command/reflect_test.go +++ b/code/sonalyze/command/reflect_test.go @@ -68,8 +68,7 @@ type SFS = SimpleFormatSpec func TestFormatting3(t *testing.T) { v1 := S2{x: 10, T2: T2{y: 20, U2: U2{z: 30}}} fs := DefineTableFromMap( - // TODO: Go 1.22, reflect.TypeFor[S2] - reflect.TypeOf((*S2)(nil)).Elem(), + reflect.TypeFor[S2](), map[string]any{ "x": SFS{"x field", "xx"}, "y": SFS{"y field", "yy"}, diff --git a/code/sonalyze/configs/configs.go b/code/sonalyze/configs/configs.go index 53aeefee..726dcf4b 100644 --- a/code/sonalyze/configs/configs.go +++ b/code/sonalyze/configs/configs.go @@ -162,8 +162,7 @@ type SFS = SimpleFormatSpec // MT: Constant after initialization; immutable var configsFormatters = DefineTableFromMap( - // TODO: Go 1.22, reflect.TypeFor[config.NodeConfigRecord] - reflect.TypeOf((*config.NodeConfigRecord)(nil)).Elem(), + reflect.TypeFor[config.NodeConfigRecord](), map[string]any{ "Timestamp": SFS{"Full ISO timestamp of when the reading was taken", "timestamp"}, "Hostname": SFS{"Name that host is known by on the cluster", "host"}, diff --git a/code/sonalyze/go.mod b/code/sonalyze/go.mod index cd2786cb..86d9c4f9 100644 --- a/code/sonalyze/go.mod +++ b/code/sonalyze/go.mod @@ -1,6 +1,6 @@ module sonalyze -go 1.21.10 +go 1.22.1 require go-utils v0.0.0-00010101000000-000000000000 replace go-utils => ../go-utils diff --git a/code/sonalyze/load/print.go b/code/sonalyze/load/print.go index 8184cab7..1c4a1bc0 100644 --- a/code/sonalyze/load/print.go +++ b/code/sonalyze/load/print.go @@ -175,8 +175,4 @@ var loadAliases = map[string][]string{ } // MT: Constant after initialization; immutable -var loadFormatters = DefineTableFromTags( - // TODO: Go 1.22, reflect.TypeFor[ReportRecord] - reflect.TypeOf((*ReportRecord)(nil)).Elem(), - nil, -) +var loadFormatters = DefineTableFromTags(reflect.TypeFor[ReportRecord](), nil) diff --git a/code/sonalyze/metadata/metadata.go b/code/sonalyze/metadata/metadata.go index d442a842..1c86cf4e 100644 --- a/code/sonalyze/metadata/metadata.go +++ b/code/sonalyze/metadata/metadata.go @@ -197,8 +197,4 @@ var metadataAliases = map[string][]string{ } // MT: Constant after initialization; immutable -var metadataFormatters = DefineTableFromTags( - // TODO: Go 1.22, reflect.TypeFor[metadataItem] - reflect.TypeOf((*metadataItem)(nil)).Elem(), - nil, -) +var metadataFormatters = DefineTableFromTags(reflect.TypeFor[metadataItem](), nil) diff --git a/code/sonalyze/nodes/nodes.go b/code/sonalyze/nodes/nodes.go index 9f32fb8f..133bb136 100644 --- a/code/sonalyze/nodes/nodes.go +++ b/code/sonalyze/nodes/nodes.go @@ -241,8 +241,7 @@ type SFS = SimpleFormatSpec // MT: Constant after initialization; immutable var nodesFormatters = DefineTableFromMap( - // TODO: Go 1.22, reflect.TypeFor[config.NodeConfigRecord] - reflect.TypeOf((*config.NodeConfigRecord)(nil)).Elem(), + reflect.TypeFor[config.NodeConfigRecord](), map[string]any{ "Timestamp": SFS{"Full ISO timestamp of when the reading was taken", "timestamp"}, "Hostname": SFS{"Name that host is known by on the cluster", "host"}, diff --git a/code/sonalyze/parse/parse.go b/code/sonalyze/parse/parse.go index 5cb1095c..260bbab3 100644 --- a/code/sonalyze/parse/parse.go +++ b/code/sonalyze/parse/parse.go @@ -1,12 +1,13 @@ package parse import ( + "cmp" "errors" "flag" "fmt" "io" "reflect" - "sort" + "slices" "strings" "go-utils/config" @@ -131,7 +132,19 @@ func (pc *ParseCommand) Perform( if mergedSamples != nil { // All elements that are part of the InputStreamKey must be part of the sort key here. - sort.Stable(sonarlog.HostTimeJobCmdSortableSampleStreams(mergedSamples)) + slices.SortStableFunc(mergedSamples, func(a, b *sonarlog.SampleStream) int { + c := cmp.Compare((*a)[0].Host.String(), (*b)[0].Host.String()) + if c == 0 { + c = cmp.Compare((*a)[0].Timestamp, (*b)[0].Timestamp) + if c == 0 { + c = cmp.Compare((*a)[0].Job, (*b)[0].Job) + if c == 0 { + c = cmp.Compare((*a)[0].Cmd.String(), (*b)[0].Cmd.String()) + } + } + } + return c + }) for _, stream := range mergedSamples { fmt.Fprintln(out, "*") FormatData( diff --git a/code/sonalyze/profile/print.go b/code/sonalyze/profile/print.go index 348124d2..8074c62b 100644 --- a/code/sonalyze/profile/print.go +++ b/code/sonalyze/profile/print.go @@ -529,11 +529,7 @@ var profileAliases = map[string][]string{ } // MT: Constant after initialization; immutable -var profileFormatters = DefineTableFromTags( - // TODO: Go 1.22, reflect.TypeFor[fixedLine] - reflect.TypeOf((*fixedLine)(nil)).Elem(), - nil, -) +var profileFormatters = DefineTableFromTags(reflect.TypeFor[fixedLine](), nil) func formatTime(t int64) string { return FormatYyyyMmDdHhMmUtc(t) diff --git a/code/sonalyze/sacct/print.go b/code/sonalyze/sacct/print.go index 6e446c70..55d51d50 100644 --- a/code/sonalyze/sacct/print.go +++ b/code/sonalyze/sacct/print.go @@ -120,8 +120,4 @@ var sacctAliases = map[string][]string{ } // MT: Constant after initialization; immutable -var sacctFormatters = DefineTableFromTags( - // TODO: Go 1.22, reflect.TypeFor[SacctRegular] - reflect.TypeOf((*SacctRegular)(nil)).Elem(), - nil, -) +var sacctFormatters = DefineTableFromTags(reflect.TypeFor[SacctRegular](), nil) diff --git a/code/sonalyze/sonarlog/postprocess.go b/code/sonalyze/sonarlog/postprocess.go index 1b1d375d..6a77365a 100644 --- a/code/sonalyze/sonarlog/postprocess.go +++ b/code/sonalyze/sonarlog/postprocess.go @@ -3,14 +3,15 @@ package sonarlog import ( + "cmp" "errors" "maps" "slices" - "sort" "strconv" "strings" "go-utils/config" + . "sonalyze/common" "sonalyze/db" ) @@ -136,7 +137,9 @@ func createInputStreams( // same data file are kept in order, in corner cases (esp around roundtripping with `parse`) // it's surprising if they are not in order. for _, stream := range streams { - sort.Stable(TimeSortableSampleStream(*stream)) + slices.SortStableFunc(*stream, func (a, b Sample) int { + return cmp.Compare(a.Timestamp, b.Timestamp) + }) } // Remove duplicate timestamps. These may appear due to system effects, notably, sonar log diff --git a/code/sonalyze/sonarlog/sort.go b/code/sonalyze/sonarlog/sort.go deleted file mode 100644 index 510a0720..00000000 --- a/code/sonalyze/sonarlog/sort.go +++ /dev/null @@ -1,81 +0,0 @@ -// Various sorting types for use with the old-timey sort.Sort function. -// -// TODO: Now that we have moved to Go 1.21 we can get rid of all of this with slices.SortFunc. - -package sonarlog - -// Sort SampleStream by time - -type TimeSortableSampleStream SampleStream - -func (t TimeSortableSampleStream) Len() int { return len(t) } - -func (t TimeSortableSampleStream) Swap(i, j int) { - t[i], t[j] = t[j], t[i] -} - -func (t TimeSortableSampleStream) Less(i, j int) bool { - return t[i].Timestamp < t[j].Timestamp -} - -// Sort SampleStream by host (primary) and time (secondary) - -type HostTimeSortableSampleStream SampleStream - -func (ss HostTimeSortableSampleStream) Len() int { - return len(ss) -} - -func (ss HostTimeSortableSampleStream) Swap(i, j int) { - ss[i], ss[j] = ss[j], ss[i] -} - -func (ss HostTimeSortableSampleStream) Less(i, j int) bool { - if ss[i].Host != ss[j].Host { - return ss[i].Host.String() < ss[j].Host.String() - } - return ss[i].Timestamp < ss[j].Timestamp -} - -// Sort SampleStreams by host, time, job, and command (in that order) - -type HostTimeJobCmdSortableSampleStreams SampleStreams - -func (sss HostTimeJobCmdSortableSampleStreams) Len() int { - return len(sss) -} - -func (sss HostTimeJobCmdSortableSampleStreams) Swap(i, j int) { - sss[i], sss[j] = sss[j], sss[i] -} - -func (sss HostTimeJobCmdSortableSampleStreams) Less(i, j int) bool { - if (*sss[i])[0].Host == (*sss[j])[0].Host { - if (*sss[i])[0].Timestamp == (*sss[j])[0].Timestamp { - if (*sss[i])[0].Job == (*sss[j])[0].Job { - return (*sss[i])[0].Cmd.String() < (*sss[j])[0].Cmd.String() - } - return (*sss[i])[0].Job < (*sss[j])[0].Job - } else { - return (*sss[i])[0].Timestamp < (*sss[j])[0].Timestamp - } - } else { - return (*sss[i])[0].Host.String() < (*sss[j])[0].Host.String() - } -} - -// Sort SampleStreams by host - -type HostSortableSampleStreams SampleStreams - -func (sss HostSortableSampleStreams) Len() int { - return len(sss) -} - -func (sss HostSortableSampleStreams) Swap(i, j int) { - sss[i], sss[j] = sss[j], sss[i] -} - -func (sss HostSortableSampleStreams) Less(i, j int) bool { - return (*sss[i])[0].Host.String() < (*sss[j])[0].Host.String() -} diff --git a/code/sonalyze/uptime/perform.go b/code/sonalyze/uptime/perform.go index 5c55015a..d550833b 100644 --- a/code/sonalyze/uptime/perform.go +++ b/code/sonalyze/uptime/perform.go @@ -41,13 +41,15 @@ package uptime import ( + "cmp" "io" - "sort" + "slices" "go-utils/config" "go-utils/hostglob" "go-utils/maps" - "go-utils/slices" + uslices "go-utils/slices" + . "sonalyze/command" . "sonalyze/common" "sonalyze/db" @@ -96,7 +98,7 @@ func (uc *UptimeCommand) Perform( hostGlobber *hostglob.HostGlobber, _ *db.SampleFilter, ) error { - samples := slices.CatenateP(maps.Values(streams)) + samples := uslices.CatenateP(maps.Values(streams)) if uc.Verbose { Log.Infof("%d streams", len(streams)) Log.Infof("%d records after hack", len(samples)) @@ -117,7 +119,12 @@ func (uc *UptimeCommand) computeReports( reports := make([]*UptimeLine, 0) fromIncl, toIncl := uc.InterpretFromToWithBounds(bounds) - sort.Stable(sonarlog.HostTimeSortableSampleStream(samples)) + slices.SortStableFunc(samples, func(a, b sonarlog.Sample) int { + if a.Host != b.Host { + return cmp.Compare(a.Host.String(), b.Host.String()) + } + return cmp.Compare(a.Timestamp, b.Timestamp) + }) uc.computeAlwaysDown(&reports, samples, cfg, hostGlobber, fromIncl, toIncl) diff --git a/code/sonalyze/uptime/print.go b/code/sonalyze/uptime/print.go index f121f0cb..8720a19d 100644 --- a/code/sonalyze/uptime/print.go +++ b/code/sonalyze/uptime/print.go @@ -68,8 +68,4 @@ var uptimeAliases = map[string][]string{ } // MT: Constant after initialization; immutable -var uptimeFormatters = DefineTableFromTags( - // TODO: Go 1.22, reflect.TypeFor[UptimeLine] - reflect.TypeOf((*UptimeLine)(nil)).Elem(), - nil, -) +var uptimeFormatters = DefineTableFromTags(reflect.TypeFor[UptimeLine](), nil) diff --git a/code/sonard/go.mod b/code/sonard/go.mod index f5035f92..0717442f 100644 --- a/code/sonard/go.mod +++ b/code/sonard/go.mod @@ -1,6 +1,6 @@ module sonard -go 1.21.10 +go 1.22.1 require go-utils v0.0.0-00010101000000-000000000000 replace go-utils => ../go-utils