No bug: Uplift: consistent naming of Hostname / Hosts fields

code/generate-table/generate-table.go

@@ -85,14 +85,14 @@ func main() {
 // operators apply; if a type is "GpuSet" then some kind of set operators apply (TBD).
 
 type typeInfo struct {
-	helpName    string			// default is the name as given
-	formatter   string			// default is Format<Typename>
+	helpName  string // default is the name as given
+	formatter string // default is Format<Typename>
 }
 
 var knownTypes = map[string]typeInfo{
 	"[]string": typeInfo{
-		helpName:    "string list",
-		formatter:   "FormatStrings",
+		helpName:  "string list",
+		formatter: "FormatStrings",
 	},
 	"F64Ceil": typeInfo{
 		helpName: "int",
@@ -110,8 +110,12 @@ var knownTypes = map[string]typeInfo{
 	"Ustr":                 typeInfo{helpName: "string"},
 	"UstrMax30":            typeInfo{helpName: "string"},
 	"gpuset.GpuSet": typeInfo{
-		helpName:    "GpuSet",
-		formatter:   "FormatGpuSet",
+		helpName:  "GpuSet",
+		formatter: "FormatGpuSet",
+	},
+	"*Hostnames": typeInfo{
+		helpName:  "Hostnames",
+		formatter: "FormatHostnames",
 	},
 }
 
@@ -159,9 +163,9 @@ import (
 var (
 	_ = cmp.Compare(0,0)
 	_ fmt.Formatter
-    _ = io.SeekStart
+	_ = io.SeekStart
 	_ = UstrEmpty
-    _ gpuset.GpuSet
+	_ gpuset.GpuSet
 )
 `)
 	fieldList := fieldSection(block.TableName, &block.Fields)

code/go-utils/gpuset/gpuset.go

@@ -65,7 +65,7 @@ func (this GpuSet) HasSubset(that GpuSet, proper bool) bool {
 	if this == unknown || that == unknown {
 		return false
 	}
-	return this & that == that && (!proper || this != that)
+	return this&that == that && (!proper || this != that)
 }
 
 func Adjoin(s GpuSet, xs ...uint32) (GpuSet, error) {

code/sonalyze/application/parse_format_test.go

@@ -30,7 +30,7 @@ func TestParseOldFieldNames(t *testing.T) {
 }
 
 func TestParseNewFieldNames(t *testing.T) {
-	fields := "Timestamp,Host,Cores,MemtotalKB,User,Pid,Ppid,Job,Cmd,CpuPct,CpuKB,RssAnonKB," +
+	fields := "Timestamp,Hostname,Cores,MemtotalKB,User,Pid,Ppid,Job,Cmd,CpuPct,CpuKB,RssAnonKB," +
 		"Gpus,GpuPct,GpuMemPct,GpuKB,GpuFail,CpuTimeSec,Rolledup,Flags,Version"
 	lines := strings.Split(mockitParse(t, fields), "\n")
 
@@ -39,7 +39,7 @@ func TestParseNewFieldNames(t *testing.T) {
 	}
 
 	// The next line should be the lowest timestamped record, but in the order of fields
-	expect := "Timestamp=2024-10-31 00:00,Host=ml6.hpc.uio.no,Cores=64,MemtotalKB=263419260," +
+	expect := "Timestamp=2024-10-31 00:00,Hostname=ml6.hpc.uio.no,Cores=64,MemtotalKB=263419260," +
 		"User=testuser,Pid=2811127,Ppid=1234,Job=1999327,Cmd=testprog.cuda,CpuPct=96.8,CpuKB=9361016," +
 		"RssAnonKB=476264,Gpus=5,GpuPct=85,GpuMemPct=16,GpuKB=581632,GpuFail=3,CpuTimeSec=1454," +
 		"Rolledup=4,Flags=0,Version=0.9.0"

code/sonalyze/cmd/config.go

@@ -26,7 +26,7 @@ func EnsureConfigForInputStreams(
 	// Remove streams for which we have no config data.
 	bad := make(map[sonarlog.InputStreamKey]bool)
 	for key, stream := range streams {
-		hn := (*stream)[0].Host.String()
+		hn := (*stream)[0].Hostname.String()
 		if cfg.LookupHost(hn) == nil {
 			bad[key] = true
 			Log.Infof("Warning: Missing host configuration for %s", hn)

code/sonalyze/cmd/gpus/gpus.go

@@ -143,7 +143,7 @@ func (gc *GpuCommand) Perform(stdin io.Reader, stdout, stderr io.Writer) error {
 				if gc.Gpu != -1 && i == gc.Gpu {
 					var r ReportLine
 					r.Timestamp = DateTimeValue(d.Time)
-					r.Hostname = s.Host
+					r.Hostname = s.Hostname
 					r.Gpu = i
 					r.PerGpuDatum = &gpu
 					reports = append(reports, &r)

code/sonalyze/cmd/jobs/jobs-table.go

@@ -219,11 +219,11 @@ var jobsFormatters = map[string]Formatter[*jobSummary]{
 		},
 		Help: "(string) The commands invoking the processes of the job",
 	},
-	"Host": {
+	"Hosts": {
 		Fmt: func(d *jobSummary, ctx PrintMods) string {
-			return FormatString((d.Host), ctx)
+			return FormatHostnames((d.Hosts), ctx)
 		},
-		Help: "(string) List of the host name(s) running the job (first elements of FQDNs, compressed)",
+		Help: "(Hostnames) List of the host name(s) running the job",
 	},
 	"Now": {
 		Fmt: func(d *jobSummary, ctx PrintMods) string {
@@ -462,7 +462,8 @@ func init() {
 	DefAlias(jobsFormatters, "Gpus", "gpus")
 	DefAlias(jobsFormatters, "GpuFail", "gpufail")
 	DefAlias(jobsFormatters, "Cmd", "cmd")
-	DefAlias(jobsFormatters, "Host", "host")
+	DefAlias(jobsFormatters, "Hosts", "host")
+	DefAlias(jobsFormatters, "Hosts", "hosts")
 	DefAlias(jobsFormatters, "Now", "now")
 	DefAlias(jobsFormatters, "Classification", "classification")
 	DefAlias(jobsFormatters, "CpuTime", "cputime")
@@ -517,8 +518,8 @@ var jobsAliases = map[string][]string{
 	"gpumem":                 []string{"gpumem-avg", "gpumem-peak"},
 	"rgpumem":                []string{"rgpumem-avg", "rgpumem-peak"},
 	"sgpumem":                []string{"sgpumem-avg", "sgpumem-peak"},
-	"All":                    []string{"JobAndMark", "Job", "User", "Duration", "Duration/sec", "Start", "Start/sec", "End", "End/sec", "CpuAvgPct", "CpuPeakPct", "RelativeCpuAvgPct", "RelativeCpuPeakPct", "MemAvgGB", "MemPeakGB", "RelativeMemAvgPct", "RelativeMemPeakPct", "ResidentMemAvgGB", "ResidentMemPeakGB", "RelativeResidentMemAvgPct", "RelativeResidentMemPeakPct", "GpuAvgPct", "GpuPeakPct", "RelativeGpuAvgPct", "RelativeGpuPeakPct", "OccupiedRelativeGpuAvgPct", "OccupiedRelativeGpuPeakPct", "GpuMemAvgGB", "GpuMemPeakGB", "RelativeGpuMemAvgPct", "RelativeGpuMemPeakPct", "OccupiedRelativeGpuMemAvgPct", "OccupiedRelativeGpuMemPeakPct", "Gpus", "GpuFail", "Cmd", "Host", "Now", "Now/sec", "Classification", "CpuTime/sec", "CpuTime", "GpuTime/sec", "GpuTime", "SomeGpu", "NoGpu", "Running", "Completed", "Zombie", "Primordial", "BornLater"},
-	"Std":                    []string{"JobAndMark", "User", "Duration", "Host"},
+	"All":                    []string{"JobAndMark", "Job", "User", "Duration", "Duration/sec", "Start", "Start/sec", "End", "End/sec", "CpuAvgPct", "CpuPeakPct", "RelativeCpuAvgPct", "RelativeCpuPeakPct", "MemAvgGB", "MemPeakGB", "RelativeMemAvgPct", "RelativeMemPeakPct", "ResidentMemAvgGB", "ResidentMemPeakGB", "RelativeResidentMemAvgPct", "RelativeResidentMemPeakPct", "GpuAvgPct", "GpuPeakPct", "RelativeGpuAvgPct", "RelativeGpuPeakPct", "OccupiedRelativeGpuAvgPct", "OccupiedRelativeGpuPeakPct", "GpuMemAvgGB", "GpuMemPeakGB", "RelativeGpuMemAvgPct", "RelativeGpuMemPeakPct", "OccupiedRelativeGpuMemAvgPct", "OccupiedRelativeGpuMemPeakPct", "Gpus", "GpuFail", "Cmd", "Hosts", "Now", "Now/sec", "Classification", "CpuTime/sec", "CpuTime", "GpuTime/sec", "GpuTime", "SomeGpu", "NoGpu", "Running", "Completed", "Zombie", "Primordial", "BornLater"},
+	"Std":                    []string{"JobAndMark", "User", "Duration", "Hosts"},
 	"Cpu":                    []string{"CpuAvgPct", "CpuPeakPct"},
 	"RelativeCpu":            []string{"RelativeCpuAvgPct", "RelativeCpuPeakPct"},
 	"Mem":                    []string{"MemAvgGB", "MemPeakGB"},

code/sonalyze/cmd/jobs/perform.go

@@ -11,7 +11,6 @@ import (
 	"go-utils/config"
 	"go-utils/gpuset"
 	"go-utils/hostglob"
-	umaps "go-utils/maps"
 	"go-utils/sonalyze"
 
 	. "sonalyze/cmd"
@@ -98,7 +97,7 @@ type jobAggregate struct {
 	computed [numF64Fields]float64
 	IsZombie bool
 	Cmd      string
-	Host     string
+	Hosts    *Hostnames
 }
 
 func (jc *JobsCommand) NeedsBounds() bool {
@@ -183,16 +182,16 @@ func (jc *JobsCommand) aggregateAndFilterJobs(
 	}
 	var (
 		needCmd        = false
-		needHost       = false
+		needHosts      = false
 		needJobAndMark = false
 		needSacctInfo  = slurmFilter != nil
 	)
 	for _, f := range jc.PrintFields {
 		switch f.Name {
 		case "cmd", "Cmd":
 			needCmd = true
-		case "host", "Host":
-			needHost = true
+		case "host", "hosts", "Hosts":
+			needHosts = true
 		case "jobm", "JobAndMark":
 			needJobAndMark = true
 		case "Submit", "JobName", "State", "Account", "Layout", "Reservation",
@@ -207,13 +206,13 @@ func (jc *JobsCommand) aggregateAndFilterJobs(
 	discarded := 0
 	for _, job := range jobs {
 		if uint(len(*job)) >= minSamples {
-			host := (*job)[0].Host
+			host := (*job)[0].Hostname
 			jobId := (*job)[0].Job
 			user := (*job)[0].User
 			first := (*job)[0].Timestamp
 			last := (*job)[len(*job)-1].Timestamp
 			duration := last - first
-			aggregate := jc.aggregateJob(cfg, host, *job, needCmd, needHost, jc.Zombie)
+			aggregate := jc.aggregateJob(cfg, host, *job, needCmd, needHosts, jc.Zombie)
 			aggregate.computed[kDuration] = float64(duration)
 			usesGpu := !aggregate.Gpus.IsEmpty()
 			flags := 0
@@ -436,7 +435,7 @@ func (jc *JobsCommand) aggregateJob(
 	cfg *config.ClusterConfig,
 	host Ustr,
 	job sonarlog.SampleStream,
-	needCmd, needHost, needZombie bool,
+	needCmd, needHosts, needZombie bool,
 ) jobAggregate {
 	gpus := gpuset.EmptyGpuSet()
 	var (
@@ -531,39 +530,19 @@ func (jc *JobsCommand) aggregateJob(
 		}
 	}
 
-	hostnames := ""
-	if needHost {
-		// TODO: It's not clear any more why len(hosts) would ever be other than 1, and why this
-		// processing is needed at all.  This could be very old code that is no longer relevant.
-		// The Go code just copies the Rust code here.
-		//
-		// Names are assumed to be compressed as the set of jobs is always the output of some
-		// merge process that will compress when appropriate.  (If they are not compressed for
-		// reasons having to do with how the merge was done, and we don't compress them here,
-		// then there may be substantial redundancy in the output: "c1-10.fox, c1-11.fox", etc,
-		// instead of the desirable "c1-[10,11].fox", but that should not currently be an issue
-		// for `jobs`.)  Therefore there is no compression here.  But even the uniq'ing, sorting
-		// and joining may be redundant.
-		hosts := make(map[string]bool)
+	var hosts *Hostnames
+	if needHosts {
+		hosts = NewHostnames()
 		for _, s := range job {
-			var name string
-			if jc.PrintOpts.Fixed {
-				name, _, _ = strings.Cut(s.Host.String(), ".")
-			} else {
-				name = s.Host.String()
-			}
-			hosts[name] = true
+			hosts.Add(s.Hostname.String())
 		}
-		keys := umaps.Keys(hosts)
-		slices.Sort(keys)
-		hostnames = strings.Join(keys, ", ")
 	}
 	n := float64(len(job))
 	a := jobAggregate{
 		Gpus:     gpus,
 		GpuFail:  int(gpuFail),
 		Cmd:      cmd,
-		Host:     hostnames,
+		Hosts:    hosts,
 		IsZombie: isZombie,
 	}
 	a.computed[kCpuPctAvg] = cpuPctAvg / n

code/sonalyze/cmd/jobs/print.go

@@ -70,7 +70,7 @@ FIELDS *jobSummary
   Gpus               gpuset.GpuSet desc:"GPU device numbers used by the job, 'none' if none or 'unknown' in error states" alias:"gpus"
   GpuFail            int           desc:"Flag indicating GPU status (0=Ok, 1=Failing)" alias:"gpufail"
   Cmd                string        desc:"The commands invoking the processes of the job" alias:"cmd"
-  Host               string        desc:"List of the host name(s) running the job (first elements of FQDNs, compressed)" alias:"host"
+  Hosts              *Hostnames    desc:"List of the host name(s) running the job" alias:"host,hosts"
   Now                DateTimeValue desc:"The current time" alias:"now"
   Classification     int           desc:"Bit vector of live-at-start (2) and live-at-end (1) flags" alias:"classification"
   CpuTime            DurationValue desc:"Total CPU time of the job across all cores" alias:"cputime"
@@ -166,9 +166,9 @@ ALIASES
               GpuAvgPct,GpuPeakPct,RelativeGpuAvgPct,RelativeGpuPeakPct,OccupiedRelativeGpuAvgPct,\
               OccupiedRelativeGpuPeakPct,GpuMemAvgGB,GpuMemPeakGB,RelativeGpuMemAvgPct,\
               RelativeGpuMemPeakPct,OccupiedRelativeGpuMemAvgPct,OccupiedRelativeGpuMemPeakPct,Gpus,GpuFail,\
-              Cmd,Host,Now,Now/sec,Classification,CpuTime/sec,CpuTime,GpuTime/sec,GpuTime,\
+              Cmd,Hosts,Now,Now/sec,Classification,CpuTime/sec,CpuTime,GpuTime/sec,GpuTime,\
               SomeGpu,NoGpu,Running,Completed,Zombie,Primordial,BornLater
-  Std         JobAndMark,User,Duration,Host
+  Std         JobAndMark,User,Duration,Hosts
   Cpu         CpuAvgPct,CpuPeakPct
   RelativeCpu RelativeCpuAvgPct,RelativeCpuPeakPct
   Mem         MemAvgGB,MemPeakGB
@@ -229,6 +229,7 @@ func (jc *JobsCommand) printJobSummaries(out io.Writer, summaries []*jobSummary)
 	}
 
 	summaries = slices.DeleteFunc(summaries, func(s *jobSummary) bool { return !s.selected })
+
 	FormatData(
 		out,
 		jc.PrintFields,

code/sonalyze/cmd/load/perform.go

@@ -73,7 +73,7 @@ func (lc *LoadCommand) Perform(
 		if cfg != nil {
 			for _, stream := range mergedStreams {
 				// probe is non-nil by previous construction
-				probe := cfg.LookupHost((*stream)[0].Host.String())
+				probe := cfg.LookupHost((*stream)[0].Hostname.String())
 				if theConf.Description != "" {
 					theConf.Description += "|||" // JSON-compatible separator
 				}
@@ -102,7 +102,7 @@ func (lc *LoadCommand) Perform(
 	// Generate data to be printed
 	reports := make([]LoadReport, 0)
 	for _, stream := range mergedStreams {
-		hostname := (*stream)[0].Host.String()
+		hostname := (*stream)[0].Hostname.String()
 		conf := mergedConf
 		if conf == nil && cfg != nil {
 			conf = cfg.LookupHost(hostname)
@@ -142,13 +142,13 @@ func (lc *LoadCommand) insertMissingRecords(ss *sonarlog.SampleStream, fromIncl,
 	default:
 		panic("Unexpected case")
 	}
-	host := (*ss)[0].Host
+	host := (*ss)[0].Hostname
 	t := trunc(fromIncl)
 	result := make(sonarlog.SampleStream, 0)
 
 	for _, s := range *ss {
 		for t < s.Timestamp {
-			newS := sonarlog.Sample{Sample: &db.Sample{Timestamp: t, Host: host}}
+			newS := sonarlog.Sample{Sample: &db.Sample{Timestamp: t, Hostname: host}}
 			result = append(result, newS)
 			t = step(t)
 		}
@@ -157,7 +157,7 @@ func (lc *LoadCommand) insertMissingRecords(ss *sonarlog.SampleStream, fromIncl,
 	}
 	ending := trunc(toIncl)
 	for t <= ending {
-		newS := sonarlog.Sample{Sample: &db.Sample{Timestamp: t, Host: host}}
+		newS := sonarlog.Sample{Sample: &db.Sample{Timestamp: t, Hostname: host}}
 		result = append(result, newS)
 		t = step(t)
 	}
@@ -208,7 +208,7 @@ func generateReport(
 			GpuGB:               int(d.GpuKB / (1024 * 1024)),
 			RelativeGpuMem:      relativeGpuMem,
 			Gpus:                d.Gpus,
-			Hostname:            d.Host,
+			Hostname:            d.Hostname,
 		})
 	}
 	return

code/sonalyze/cmd/parse/parse-table.go

@@ -41,9 +41,9 @@ var parseFormatters = map[string]Formatter[sonarlog.Sample]{
 		},
 		Help: "(IsoDateTimeValue) Timestamp of record",
 	},
-	"Host": {
+	"Hostname": {
 		Fmt: func(d sonarlog.Sample, ctx PrintMods) string {
-			return FormatUstr((d.Host), ctx)
+			return FormatUstr((d.Hostname), ctx)
 		},
 		Help: "(string) Host name (FQDN)",
 	},
@@ -191,7 +191,7 @@ func init() {
 	DefAlias(parseFormatters, "Version", "version")
 	DefAlias(parseFormatters, "Version", "v")
 	DefAlias(parseFormatters, "Timestamp", "localtime")
-	DefAlias(parseFormatters, "Host", "host")
+	DefAlias(parseFormatters, "Hostname", "host")
 	DefAlias(parseFormatters, "Cores", "cores")
 	DefAlias(parseFormatters, "User", "user")
 	DefAlias(parseFormatters, "Pid", "pid")
@@ -241,7 +241,7 @@ var parseAliases = map[string][]string{
 	"default":   []string{"job", "user", "cmd"},
 	"Default":   []string{"Job", "User", "Cmd"},
 	"all":       []string{"version", "localtime", "host", "cores", "memtotal", "user", "pid", "job", "cmd", "cpu_pct", "mem_gb", "res_gb", "gpus", "gpu_pct", "gpumem_pct", "gpumem_gb", "gpu_status", "cputime_sec", "rolledup", "cpu_util_pct"},
-	"All":       []string{"Version", "Timestamp", "Host", "Cores", "MemtotalKB", "User", "Pid", "Ppid", "Job", "Cmd", "CpuPct", "CpuKB", "RssAnonKB", "Gpus", "GpuPct", "GpuMemPct", "GpuKB", "GpuFail", "CpuTimeSec", "Rolledup", "CpuUtilPct"},
+	"All":       []string{"Version", "Timestamp", "Hostname", "Cores", "MemtotalKB", "User", "Pid", "Ppid", "Job", "Cmd", "CpuPct", "CpuKB", "RssAnonKB", "Gpus", "GpuPct", "GpuMemPct", "GpuKB", "GpuFail", "CpuTimeSec", "Rolledup", "CpuUtilPct"},
 	"roundtrip": []string{"v", "time", "host", "cores", "user", "job", "pid", "cmd", "cpu%", "cpukib", "gpus", "gpu%", "gpumem%", "gpukib", "gpufail", "cputime_sec", "rolledup"},
 }
 

code/sonalyze/cmd/parse/parse.go

@@ -34,7 +34,7 @@ FIELDS sonarlog.Sample
  Version    Ustr                desc:"Semver string (MAJOR.MINOR.BUGFIX)" alias:"version,v"
  Timestamp  DateTimeValue       desc:"Timestamp of record " alias:"localtime"
  time       IsoDateTimeValue    desc:"Timestamp of record" field:"Timestamp"
- Host       Ustr                desc:"Host name (FQDN)" alias:"host"
+ Hostname   Ustr                desc:"Host name (FQDN)" alias:"host"
  Cores      uint32              desc:"Total number of cores (including hyperthreads)" alias:"cores"
  MemtotalKB uint64              desc:"Installed main memory"
  memtotal   U64Div1M            desc:"Installed main memory (GB)" field:"MemtotalKB"
@@ -82,7 +82,7 @@ ALIASES
   default   job,user,cmd
   Default   Job,User,Cmd
   all       version,localtime,host,cores,memtotal,user,pid,job,cmd,cpu_pct,mem_gb,res_gb,gpus,gpu_pct,gpumem_pct,gpumem_gb,gpu_status,cputime_sec,rolledup,cpu_util_pct
-  All       Version,Timestamp,Host,Cores,MemtotalKB,User,Pid,Ppid,Job,Cmd,CpuPct,CpuKB,RssAnonKB,Gpus,GpuPct,GpuMemPct,GpuKB,GpuFail,CpuTimeSec,Rolledup,CpuUtilPct
+  All       Version,Timestamp,Hostname,Cores,MemtotalKB,User,Pid,Ppid,Job,Cmd,CpuPct,CpuKB,RssAnonKB,Gpus,GpuPct,GpuMemPct,GpuKB,GpuFail,CpuTimeSec,Rolledup,CpuUtilPct
   roundtrip v,time,host,cores,user,job,pid,cmd,cpu%,cpukib,gpus,gpu%,gpumem%,gpukib,gpufail,cputime_sec,rolledup
 
 DEFAULTS default
@@ -199,7 +199,7 @@ func (pc *ParseCommand) Perform(
 	if mergedSamples != nil {
 		// All elements that are part of the InputStreamKey must be part of the sort key here.
 		slices.SortStableFunc(mergedSamples, func(a, b *sonarlog.SampleStream) int {
-			c := cmp.Compare((*a)[0].Host.String(), (*b)[0].Host.String())
+			c := cmp.Compare((*a)[0].Hostname.String(), (*b)[0].Hostname.String())
 			if c == 0 {
 				c = cmp.Compare((*a)[0].Timestamp, (*b)[0].Timestamp)
 				if c == 0 {