From a06e5dec2c9c3b099e909a0524d102a8207a49a5 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Fri, 15 Nov 2024 16:59:12 -0800 Subject: [PATCH] important fixes for pool avg / max computation --- axon/pool.go | 9 ++-- axon/pool.goal | 9 ++-- examples/ra25/ra25.go | 105 +++++++++++++++++++++++++++++++++++----- examples/ra25/ra25.goal | 105 +++++++++++++++++++++++++++++++++++----- 4 files changed, 196 insertions(+), 32 deletions(-) diff --git a/axon/pool.go b/axon/pool.go index 2231a4d80..3779878ec 100644 --- a/axon/pool.go +++ b/axon/pool.go @@ -204,8 +204,8 @@ func PoolAvgMaxUpdate(pi, di, ni uint32) { // PoolAvgMaxCalcVar does Calc on Cycle level, and re-inits, for given Var func PoolAvgMaxCalcVar(vr AvgMaxVars, pi, di uint32) { floatFromInt := float32(1.0) / float32(uint32(1)<<20) - vim := AvgMaxIntVarIndex(vr, Max) - sum := PoolsInt.Value(int(vim), int(pi), int(di)) + vis := AvgMaxIntVarIndex(vr, Avg) + sum := PoolsInt.Value(int(vis), int(pi), int(di)) if sum < 0 { //gosl:end log.Println("PoolAvgMaxCalc overflow in Sum", "pi:", pi, "di:", di, "sum:", sum) @@ -213,13 +213,16 @@ func PoolAvgMaxCalcVar(vr AvgMaxVars, pi, di uint32) { sum = int32(uint32(1) << 20) } Pools.Set(float32(sum)*floatFromInt, int(AvgMaxVarIndex(vr, AMCycle, Avg)), int(pi), int(di)) - mx := PoolsInt.Value(int(AvgMaxIntVarIndex(vr, Max)), int(pi), int(di)) + PoolsInt.Set(0, int(vis), int(pi), int(di)) + vim := AvgMaxIntVarIndex(vr, Max) + mx := PoolsInt.Value(int(vim), int(pi), int(di)) if mx < 0 { //gosl:end log.Println("PoolAvgMaxCalc overflow in Max", "pi:", pi, "di:", di, "max:", mx) //gosl:start mx = int32(uint32(1) << 20) } + PoolsInt.Set(0, int(vim), int(pi), int(di)) Pools.Set(float32(mx)*floatFromInt, int(AvgMaxVarIndex(vr, AMCycle, Max)), int(pi), int(di)) } diff --git a/axon/pool.goal b/axon/pool.goal index 90c1be4bd..723e261a7 100644 --- a/axon/pool.goal +++ b/axon/pool.goal @@ -202,8 +202,8 @@ func PoolAvgMaxUpdate(pi, di, ni uint32) { // PoolAvgMaxCalcVar does Calc on Cycle level, and re-inits, for given Var func PoolAvgMaxCalcVar(vr AvgMaxVars, pi, di uint32) { floatFromInt := float32(1.0) / float32(uint32(1) << 20) - vim := AvgMaxIntVarIndex(vr, Max) - sum := PoolsInt[vim, pi, di] + vis := AvgMaxIntVarIndex(vr, Avg) + sum := PoolsInt[vis, pi, di] if sum < 0 { //gosl:end log.Println("PoolAvgMaxCalc overflow in Sum", "pi:", pi, "di:", di, "sum:", sum) @@ -211,13 +211,16 @@ func PoolAvgMaxCalcVar(vr AvgMaxVars, pi, di uint32) { sum = int32(uint32(1) << 20) } Pools[AvgMaxVarIndex(vr, AMCycle, Avg), pi, di] = float32(sum) * floatFromInt - mx := PoolsInt[AvgMaxIntVarIndex(vr, Max), pi, di] + PoolsInt[vis, pi, di] = 0 + vim := AvgMaxIntVarIndex(vr, Max) + mx := PoolsInt[vim, pi, di] if mx < 0 { //gosl:end log.Println("PoolAvgMaxCalc overflow in Max", "pi:", pi, "di:", di, "max:", mx) //gosl:start mx = int32(uint32(1) << 20) } + PoolsInt[vim, pi, di] = 0 Pools[AvgMaxVarIndex(vr, AMCycle, Max), pi, di] = float32(mx) * floatFromInt } diff --git a/examples/ra25/ra25.go b/examples/ra25/ra25.go index 57e4008e4..aeb4795bc 100644 --- a/examples/ra25/ra25.go +++ b/examples/ra25/ra25.go @@ -13,7 +13,6 @@ package main //go:generate core generate -add-types import ( - "fmt" "log" "os" @@ -115,7 +114,7 @@ type RunConfig struct { GPU bool `default:"false"` // number of data-parallel items to process in parallel per trial -- works (and is significantly faster) for both CPU and GPU. Results in an effective mini-batch of learning. - NData int `default:"1" min:"1"` + NData int `default:"2" min:"1"` // number of parallel threads for CPU computation -- 0 = use default NThreads int `default:"0"` @@ -594,19 +593,19 @@ func (ss *Sim) RunStats(lmode Modes, ltime Times, lphase StatsPhase) { } func (ss *Sim) InitStats() { - fmt.Println("init") for mode, st := range ss.Loops.Stacks { for _, tm := range st.Order { ctm := tm.(Times) if ctm == Cycle { continue } - fmt.Println("calling stats at:", mode, tm) ss.RunStats(mode.(Modes), ctm, Start) } } } +// ConfigStats handles configures functions to do all stats computation +// in the datafs system. func (ss *Sim) ConfigStats() { ss.Stats, _ = ss.Root.Mkdir("Stats") ss.Current, _ = ss.Stats.Mkdir("Current") @@ -617,17 +616,16 @@ func (ss *Sim) ConfigStats() { if ctm == Cycle { continue } - fmt.Println("adding stat for:", cmd, ctm) ss.AddStat(func(lmode Modes, ltime Times, lphase StatsPhase) { if lmode != cmd || ltime > ctm { // don't record counter for time above it return } name := tm.String() // name of stat = time + ndata := ss.Config.Run.NData modeDir := ss.Stats.RecycleDir(lmode.String()) timeDir := modeDir.RecycleDir(ltime.String()) tsr := datafs.Value[int](timeDir, name) if lphase == Start { - fmt.Println(lmode, ltime) tsr.SetNumRows(0) if ps := plot.GetStylersFrom(tsr); ps == nil { ps.Add(func(s *plot.Style) { @@ -638,18 +636,27 @@ func (ss *Sim) ConfigStats() { return } ctv := ss.Loops.Stacks[cmd].Loops[ctm].Counter.Cur - datafs.Scalar[int](ss.Current, name).SetInt1D(ctv, 0) - tsr.AppendRowInt(ctv) + if ltime == Trial { + for di := range ndata { + datafs.Value[int](ss.Current, name, ndata).SetInt1D(ctv, di) + tsr.AppendRowInt(ctv) + if ctm == Trial { + ctv++ + } + } + } else { + datafs.Scalar[int](ss.Current, name).SetInt1D(ctv, 0) + tsr.AppendRowInt(ctv) + } }) } } - // note: it is essential to only have 1 per func - // so generic names can be used for everything. ss.AddStat(func(lmode Modes, ltime Times, lphase StatsPhase) { name := "UnitErr" modeDir := ss.Stats.RecycleDir(lmode.String()) timeDir := modeDir.RecycleDir(ltime.String()) tsr := datafs.Value[float64](timeDir, name) + ndata := ss.Config.Run.NData if lphase == Start { tsr.SetNumRows(0) if ps := plot.GetStylersFrom(tsr); ps == nil { @@ -665,9 +672,8 @@ func (ss *Sim) ConfigStats() { case Trial: out := ss.Net.LayerByName("Output") stat := out.PctUnitErr(ss.Net.Context()) - for di := 0; di < ss.Config.Run.NData; di++ { - // todo: current needs di tensor - datafs.Scalar[float64](ss.Current, name).SetFloat(stat[di], 0) + for di := range ndata { + datafs.Value[float64](ss.Current, name, ndata).SetFloat1D(stat[di], di) tsr.AppendRowFloat(stat[di]) } case Epoch: @@ -680,6 +686,79 @@ func (ss *Sim) ConfigStats() { tsr.AppendRow(stat) } }) + ss.AddStat(func(lmode Modes, ltime Times, lphase StatsPhase) { + name := "Err" + modeDir := ss.Stats.RecycleDir(lmode.String()) + timeDir := modeDir.RecycleDir(ltime.String()) + tsr := datafs.Value[float64](timeDir, name) + ndata := ss.Config.Run.NData + if lphase == Start { + tsr.SetNumRows(0) + if ps := plot.GetStylersFrom(tsr); ps == nil { + ps.Add(func(s *plot.Style) { + s.Range.SetMin(0).SetMax(1) + s.On = true + }) + plot.SetStylersTo(tsr, ps) + } + return + } + switch ltime { + case Trial: + for di := range ndata { + uniterr := datafs.Value[float64](ss.Current, "UnitErr", ndata).Float1D(di) + er := 1.0 + if uniterr == 0 { + er = 0 + } + datafs.Value[float64](ss.Current, name, ndata).SetFloat(er, di) + tsr.AppendRowFloat(er) + } + case Epoch: + subd := modeDir.RecycleDir(Trial.String()) + stat := stats.StatMean.Call(subd.Value(name)) + tsr.AppendRow(stat) + case Run: + subd := modeDir.RecycleDir(Epoch.String()) + stat := stats.StatMean.Call(subd.Value(name)) + tsr.AppendRow(stat) + } + }) + ss.AddStat(func(lmode Modes, ltime Times, lphase StatsPhase) { + name := "CorSim" + modeDir := ss.Stats.RecycleDir(lmode.String()) + timeDir := modeDir.RecycleDir(ltime.String()) + tsr := datafs.Value[float64](timeDir, name) + ndata := ss.Config.Run.NData + if lphase == Start { + tsr.SetNumRows(0) + if ps := plot.GetStylersFrom(tsr); ps == nil { + ps.Add(func(s *plot.Style) { + s.Range.SetMin(0).SetMax(1) + s.On = true + }) + plot.SetStylersTo(tsr, ps) + } + return + } + switch ltime { + case Trial: + out := ss.Net.LayerByName("Output") + for di := range ndata { + stat := float64(axon.LayerStates.Value(int(axon.LayerPhaseDiff), int(out.Index), int(di))) + datafs.Value[float64](ss.Current, name, ndata).SetFloat1D(stat, di) + tsr.AppendRowFloat(stat) + } + case Epoch: + subd := modeDir.RecycleDir(Trial.String()) + stat := stats.StatMean.Call(subd.Value(name)) + tsr.AppendRow(stat) + case Run: + subd := modeDir.RecycleDir(Epoch.String()) + stat := stats.StatMean.Call(subd.Value(name)) + tsr.AppendRow(stat) + } + }) } // StatCounters returns counters string to show at bottom of netview. diff --git a/examples/ra25/ra25.goal b/examples/ra25/ra25.goal index 86ba6a48d..de26ce5a5 100644 --- a/examples/ra25/ra25.goal +++ b/examples/ra25/ra25.goal @@ -11,7 +11,6 @@ package main //go:generate core generate -add-types import ( - "fmt" "log" "os" @@ -113,7 +112,7 @@ type RunConfig struct { GPU bool `default:"false"` // number of data-parallel items to process in parallel per trial -- works (and is significantly faster) for both CPU and GPU. Results in an effective mini-batch of learning. - NData int `default:"1" min:"1"` + NData int `default:"2" min:"1"` // number of parallel threads for CPU computation -- 0 = use default NThreads int `default:"0"` @@ -592,19 +591,19 @@ func (ss *Sim) RunStats(lmode Modes, ltime Times, lphase StatsPhase) { } func (ss *Sim) InitStats() { - fmt.Println("init") for mode, st := range ss.Loops.Stacks { for _, tm := range st.Order { ctm := tm.(Times) if ctm == Cycle { continue } - fmt.Println("calling stats at:", mode, tm) ss.RunStats(mode.(Modes), ctm, Start) } } } +// ConfigStats handles configures functions to do all stats computation +// in the datafs system. func (ss *Sim) ConfigStats() { ss.Stats, _ = ss.Root.Mkdir("Stats") ss.Current, _ = ss.Stats.Mkdir("Current") @@ -615,17 +614,16 @@ func (ss *Sim) ConfigStats() { if ctm == Cycle { continue } - fmt.Println("adding stat for:", cmd, ctm) ss.AddStat(func(lmode Modes, ltime Times, lphase StatsPhase) { if lmode != cmd || ltime > ctm { // don't record counter for time above it return } name := tm.String() // name of stat = time + ndata := ss.Config.Run.NData modeDir := ss.Stats.RecycleDir(lmode.String()) timeDir := modeDir.RecycleDir(ltime.String()) tsr := datafs.Value[int](timeDir, name) if lphase == Start { - fmt.Println(lmode, ltime) tsr.SetNumRows(0) if ps := plot.GetStylersFrom(tsr); ps == nil { ps.Add(func(s *plot.Style) { @@ -636,18 +634,27 @@ func (ss *Sim) ConfigStats() { return } ctv := ss.Loops.Stacks[cmd].Loops[ctm].Counter.Cur - datafs.Scalar[int](ss.Current, name).SetInt1D(ctv, 0) - tsr.AppendRowInt(ctv) + if ltime == Trial { + for di := range ndata { + datafs.Value[int](ss.Current, name, ndata).SetInt1D(ctv, di) + tsr.AppendRowInt(ctv) + if ctm == Trial { + ctv++ + } + } + } else { + datafs.Scalar[int](ss.Current, name).SetInt1D(ctv, 0) + tsr.AppendRowInt(ctv) + } }) } } - // note: it is essential to only have 1 per func - // so generic names can be used for everything. ss.AddStat(func(lmode Modes, ltime Times, lphase StatsPhase) { name := "UnitErr" modeDir := ss.Stats.RecycleDir(lmode.String()) timeDir := modeDir.RecycleDir(ltime.String()) tsr := datafs.Value[float64](timeDir, name) + ndata := ss.Config.Run.NData if lphase == Start { tsr.SetNumRows(0) if ps := plot.GetStylersFrom(tsr); ps == nil { @@ -663,9 +670,8 @@ func (ss *Sim) ConfigStats() { case Trial: out := ss.Net.LayerByName("Output") stat := out.PctUnitErr(ss.Net.Context()) - for di := 0; di < ss.Config.Run.NData; di++ { - // todo: current needs di tensor - datafs.Scalar[float64](ss.Current, name).SetFloat(stat[di], 0) + for di := range ndata { + datafs.Value[float64](ss.Current, name, ndata).SetFloat1D(stat[di], di) tsr.AppendRowFloat(stat[di]) } case Epoch: @@ -678,6 +684,79 @@ func (ss *Sim) ConfigStats() { tsr.AppendRow(stat) } }) + ss.AddStat(func(lmode Modes, ltime Times, lphase StatsPhase) { + name := "Err" + modeDir := ss.Stats.RecycleDir(lmode.String()) + timeDir := modeDir.RecycleDir(ltime.String()) + tsr := datafs.Value[float64](timeDir, name) + ndata := ss.Config.Run.NData + if lphase == Start { + tsr.SetNumRows(0) + if ps := plot.GetStylersFrom(tsr); ps == nil { + ps.Add(func(s *plot.Style) { + s.Range.SetMin(0).SetMax(1) + s.On = true + }) + plot.SetStylersTo(tsr, ps) + } + return + } + switch ltime { + case Trial: + for di := range ndata { + uniterr := datafs.Value[float64](ss.Current, "UnitErr", ndata).Float1D(di) + er := 1.0 + if uniterr == 0 { + er = 0 + } + datafs.Value[float64](ss.Current, name, ndata).SetFloat(er, di) + tsr.AppendRowFloat(er) + } + case Epoch: + subd := modeDir.RecycleDir(Trial.String()) + stat := stats.StatMean.Call(subd.Value(name)) + tsr.AppendRow(stat) + case Run: + subd := modeDir.RecycleDir(Epoch.String()) + stat := stats.StatMean.Call(subd.Value(name)) + tsr.AppendRow(stat) + } + }) + ss.AddStat(func(lmode Modes, ltime Times, lphase StatsPhase) { + name := "CorSim" + modeDir := ss.Stats.RecycleDir(lmode.String()) + timeDir := modeDir.RecycleDir(ltime.String()) + tsr := datafs.Value[float64](timeDir, name) + ndata := ss.Config.Run.NData + if lphase == Start { + tsr.SetNumRows(0) + if ps := plot.GetStylersFrom(tsr); ps == nil { + ps.Add(func(s *plot.Style) { + s.Range.SetMin(0).SetMax(1) + s.On = true + }) + plot.SetStylersTo(tsr, ps) + } + return + } + switch ltime { + case Trial: + out := ss.Net.LayerByName("Output") + for di := range ndata { + stat := float64(axon.LayerStates[axon.LayerPhaseDiff, out.Index, di]) + datafs.Value[float64](ss.Current, name, ndata).SetFloat1D(stat, di) + tsr.AppendRowFloat(stat) + } + case Epoch: + subd := modeDir.RecycleDir(Trial.String()) + stat := stats.StatMean.Call(subd.Value(name)) + tsr.AppendRow(stat) + case Run: + subd := modeDir.RecycleDir(Epoch.String()) + stat := stats.StatMean.Call(subd.Value(name)) + tsr.AppendRow(stat) + } + }) } // StatCounters returns counters string to show at bottom of netview.