progress on gpu

emer · Nov 22, 2024 · 6645131 · 6645131
1 parent fc10f4e
commit 6645131
Show file tree

Hide file tree

Showing 10 changed files with 105 additions and 39 deletions.
diff --git a/axon/act-net.go b/axon/act-net.go
diff --git a/axon/act-net.goal b/axon/act-net.goal
@@ -81,6 +81,7 @@ func (nt *Network) ApplyExts() {
 	ctx := nt.Context()
 	nd := int(nix.NNeurons * ctx.NData)
 	RunApplyExtsNeuron(nd)
+	// note: not completed
 }
 
 // MinusPhase does updating after end of minus phase.
@@ -91,6 +92,7 @@ func (nt *Network) MinusPhase() {
 	pd := int(nix.NPools * ctx.NData)
 	RunMinusPhasePool(pd)
 	RunMinusPhaseNeuron(nd)
+	RunDoneLayersNeurons()
 	nt.MinusPhasePost()
 	ToGPULayersNeurons()
 	// todo:
@@ -115,6 +117,7 @@ func (nt *Network) PlusPhaseStart() {
 	ctx := nt.Context()
 	nd := int(nix.NNeurons * ctx.NData)
 	RunPlusPhaseStartNeuron(nd)
+	RunDone()
 }
 
 // PlusPhase does updating after end of plus phase
@@ -125,6 +128,7 @@ func (nt *Network) PlusPhase() {
 	pd := int(nix.NPools * ctx.NData)
 	RunPlusPhasePool(pd)
 	RunPlusPhaseNeuron(nd)
+	RunDoneLayersNeurons()
 	nt.PlusPhasePost()
 	ToGPULayersNeurons()
 	// todo:

diff --git a/axon/basic_test.go b/axon/basic_test.go
diff --git a/axon/basic_test.goal b/axon/basic_test.goal
@@ -395,9 +395,9 @@ func TestNetActShort(t *testing.T) {
 }
 
 func TestGPUAct(t *testing.T) {
-	// if os.Getenv("TEST_GPU") != "true" {
-	// 	t.Skip("Set TEST_GPU env var to run GPU tests")
-	// }
+	if os.Getenv("TEST_GPU") != "true" {
+		t.Skip("Set TEST_GPU env var to run GPU tests")
+	}
 	NetActTestShort(t, Tol6, true)
 }
 
@@ -522,7 +522,7 @@ func NetActTest(t *testing.T, tol float32, gpu bool) {
 		testNet.PlusPhase()
 	}
 
-	// testNet.GPU.Destroy()
+	GPURelease()
 }
 
 // NetActTestShort runs an activation test on the network and checks
@@ -936,21 +936,13 @@ func NetTestLearn(t *testing.T, tol float32, gpu bool) {
 		}
 
 		testNet.DWt()
-		if gpu {
-			// testNet.GPU.SyncSynapsesFromGPU()
-			// testNet.GPU.SyncSynCaFromGPU()
-		}
 
 		didx := pi
 
 		hiddwt[didx] = hidLay.RecvPaths[0].SynValue("DWt", pi, pi)
 		outdwt[didx] = outLay.RecvPaths[0].SynValue("DWt", pi, pi)
 
 		testNet.WtFromDWt()
-		if gpu {
-			// testNet.GPU.SyncSynapsesFromGPU()
-			// testNet.GPU.SyncSynCaFromGPU()
-		}
 
 		hidwt[didx] = hidLay.RecvPaths[0].SynValue("Wt", pi, pi)
 		outwt[didx] = outLay.RecvPaths[0].SynValue("Wt", pi, pi)
@@ -966,7 +958,7 @@ func NetTestLearn(t *testing.T, tol float32, gpu bool) {
 	CompareFloats(tol, hidwt, hidWts, "hidWts", t)
 	CompareFloats(tol, outwt, outWts, "outWts", t)
 
-	// testNet.GPU.Destroy()
+	GPURelease()
 }
 
 func TestNetRLRate(t *testing.T) {

diff --git a/axon/learn-net.go b/axon/learn-net.go
diff --git a/axon/learn-net.goal b/axon/learn-net.goal
@@ -4,27 +4,52 @@
 
 package axon
 
-// DWt computes the weight change (learning) based on current running-average activation values
+// DWt computes the weight change (learning) based on current
+// running-average activation values. Copies synapses back from GPU,
+// for case where viewing the synapses.
 func (nt *Network) DWt() {
 	nix := nt.NetIxs()
 	ctx := nt.Context()
 	sd := int(nix.NSyns * ctx.NData)
 	RunDWtSyn(sd)
 	RunDWtFromDiSyn(int(nix.NSyns))
+	RunDoneSynapsesTrace()
 }
 
-// WtFromDWt updates the weights from delta-weight changes.
+// WtFromDWt updates the weights from delta-weight changes,
+// after having done DWt previously.
 // Also does ctx.SlowInc() and calls SlowAdapt at SlowInterval
 func (nt *Network) WtFromDWt() {
 	nix := nt.NetIxs()
 	ctx := nt.Context()
 	RunDWtSubMeanPath(int(nix.NPaths))
 	RunWtFromDWtSyn(int(nix.NSyns))
+	RunDoneSynapses()
 	if ctx.SlowInc() {
 		nt.SlowAdapt()
 	}
 }
 
+
+// DWtToWt computes the weight change (learning) based on current
+// running-average activation values, and then WtFromDWt, syncing
+// back only the synapses (not SynapseTraces).
+// This should be used when not viewing the weights.
+func (nt *Network) DWtToWt() {
+	nix := nt.NetIxs()
+	ctx := nt.Context()
+	sd := int(nix.NSyns * ctx.NData)
+	RunDWtSyn(sd)
+	RunDWtFromDiSyn(int(nix.NSyns))
+	RunDWtSubMeanPath(int(nix.NPaths))
+	RunWtFromDWtSyn(int(nix.NSyns))
+	RunDoneSynapses()
+	if ctx.SlowInc() {
+		nt.SlowAdapt()
+		ToGPUSynapses()
+	}
+}
+
 // SlowAdapt is the layer-level slow adaptation functions: Synaptic scaling,
 // and adapting inhibition
 func (nt *Network) SlowAdapt() {

diff --git a/axon/looper.go b/axon/looper.go
@@ -44,8 +44,8 @@ func LooperStandard(ls *looper.Stacks, net *Network, viewFunc func(mode enums.En
 	for mode, st := range ls.Stacks {
 		cycLoop := st.Loops[cycle]
 		cycLoop.OnStart.Add("Cycle", func() {
-			nCycles := 10
-			getNeurons := false
+			nCycles := fastNCycles
+			getNeurons := true // todo: need back for phases..
 			if ls.ModeStack().StepLevel.Int64() == cycle.Int64() {
 				nCycles = 1
 				getNeurons = true
@@ -71,14 +71,13 @@ func LooperStandard(ls *looper.Stacks, net *Network, viewFunc func(mode enums.En
 		})
 		if mode.Int64() == trainMode.Int64() {
 			trlLoop.OnEnd.Add("UpdateWeights", func() {
-				net.DWt() // todo: need to get synapses here, not after
 				if view := viewFunc(mode); view != nil && view.IsViewingSynapse() {
-					//TODO:
-					// net.GPU.SyncSynapsesFromGPU()
-					// net.GPU.SyncSynCaFromGPU() // note: only time we call this
+					net.DWt()         // todo: need to get synapses here, not after
 					view.RecordSyns() // note: critical to update weights here so DWt is visible
+					net.WtFromDWt()
+				} else {
+					net.DWtToWt()
 				}
-				net.WtFromDWt()
 			})
 		}
 	}

diff --git a/axon/network.go b/axon/network.go
diff --git a/axon/network.goal b/axon/network.goal
@@ -970,6 +970,17 @@ func RunDoneLayersNeurons() {
 	RunDone(CtxVar, GlobalScalarsVar, GlobalVectorsVar, LayerStatesVar, PoolsVar, PoolsIntVar, NeuronsVar, NeuronAvgsVar)
 }
 
+// RunDoneSynapses finishes running and copies the Synapse state back.
+func RunDoneSynapses() {
+	RunDone(SynapsesVar)
+}
+
+// RunDoneSynapses finishes running and copies the Synapse state back,
+// including SynapseTraces, for visualization.
+func RunDoneSynapsesTrace() {
+	RunDone(SynapsesVar, SynapseTracesVar)
+}
+
 // BuildPathGBuf builds the PathGBuf, PathGSyns,
 // based on the MaxDelay values in the PathParams,
 // which should have been configured by this point.

diff --git a/examples/ra25/ra25.go b/examples/ra25/ra25.go
@@ -109,7 +109,7 @@ type RunConfig struct {
 	GPU bool `default:"false"`
 
 	// number of data-parallel items to process in parallel per trial -- works (and is significantly faster) for both CPU and GPU.  Results in an effective mini-batch of learning.
-	NData int `default:"2" min:"1"`
+	NData int `default:"16" min:"1"`
 
 	// number of parallel threads for CPU computation -- 0 = use default
 	NThreads int `default:"0"`
@@ -253,6 +253,10 @@ func (ss *Sim) New() {
 
 // ConfigAll configures all the elements using the standard functions
 func (ss *Sim) ConfigAll() {
+	if ss.Config.Run.GPU {
+		axon.GPUInit()
+		axon.UseGPU = true
+	}
 	// ss.ConfigPats()
 	ss.OpenPats()
 	ss.ConfigEnv()
@@ -399,7 +403,7 @@ func (ss *Sim) ConfigLoops() {
 		AddLevelIncr(Trial, trls, ss.Config.Run.NData).
 		AddLevel(Cycle, 200)
 
-	axon.LooperStandard(ls, ss.Net, ss.NetViewUpdater, 10, 150, 199, Cycle, Trial, Train)
+	axon.LooperStandard(ls, ss.Net, ss.NetViewUpdater, 50, 150, 199, Cycle, Trial, Train)
 
 	ls.Stacks[Train].OnInit.Add("Init", func() { ss.Init() })