Added testing framework

br/firesim-overlay/etc/init.d/S99run

@@ -5,7 +5,7 @@ SYSLOGD_ARGS=-n
 KLOGD_ARGS=-n
 
 start() {
-    /firesim.sh
+    echo "FIRESIM RUN START" && /firesim.sh && echo "FIRESIM RUN END"
 }
 
 case "$1" in

sw_manager.py

@@ -239,7 +239,9 @@ def handleBuild(args, cfgs):
                 imgList.append(jCfg['img'])
 
     # The order isn't critical here, we should have defined the dependencies correctly in loader 
-    doit.doit_cmd.DoitMain(loader).run(binList + imgList)
+    ret = doit.doit_cmd.DoitMain(loader).run(binList + imgList)
+    if ret != 0:
+        raise RuntimeError("Error while building workload")
 
 # Returns a command string to luanch the given config in spike. Must be called with shell=True.
 def getSpikeCmd(config, initramfs=False):
@@ -287,7 +289,8 @@ def handleLaunch(args, cfgs):
         # Run the base image
         config = cfgs[args.config_file]
 
-    runResDir = os.path.join(res_dir, getRunName(), config['name'])
+    baseResDir = os.path.join(res_dir, getRunName())
+    runResDir = os.path.join(baseResDir, config['name'])
     uartLog = os.path.join(runResDir, "uartlog")
     os.makedirs(runResDir)
 
@@ -305,6 +308,10 @@ def handleLaunch(args, cfgs):
         outputSpec = [ FileSpec(src=f, dst=runResDir + "/") for f in config['outputs']] 
         copyImgFiles(config['img'], outputSpec, direction='out')
 
+    if 'post_run_hook' in config:
+        log.info("Running post_run_hook script: " + config['post_run_hook'])
+        run(config['post_run_hook'] + " " + baseResDir, cwd=config['workdir'], shell=True)
+
     log.info("Run output available in: " + os.path.dirname(runResDir))
 
 # Now build linux/bbl
@@ -325,8 +332,10 @@ def makeBin(config, initramfs=False):
         else: 
             run(['make', 'ARCH=riscv', 'vmlinux', jlevel], cwd=config['linux-src'])
 
-        if not os.path.exists('riscv-pk/build'):
-            os.mkdir('riscv-pk/build')
+        # BBL doesn't seem to detect changes in its configuration and won't rebuild if the payload path changes
+        if os.path.exists('riscv-pk/build'):
+            shutil.rmtree('riscv-pk/build')
+        os.mkdir('riscv-pk/build')
 
         run(['../configure', '--host=riscv64-unknown-elf',
             '--with-payload=' + os.path.join(config['linux-src'], 'vmlinux')], cwd='riscv-pk/build')

test/fed-run.json

@@ -3,6 +3,7 @@
   "base" : "fedora-base.json",
   "run" : "run.sh",
   "testing" : {
-    "refDir" : "refOutput"
+    "refDir" : "refOutput",
+    "strip" : true
   }
 }

test/fed-run/refOutput/fed-run/uartlog

@@ -1 +1 @@
-[   75.478773] firesim.sh[216]: Global : run
+Global : run

test/fed-smoke0.json

@@ -2,19 +2,25 @@
   "name" : "fed-smoke0",
   "base" : "fedora-base.json",
   "overlay" : "overlay",
-  "guest-init" : "guest-init.sh",
+  "guest-init" : "init.sh",
   "run" : "run.sh",
+  "testing" : {
+    "refDir" : "refOutput",
+    "strip" : true
+  },
   "jobs" : [
     {
       "name" : "j0",
       "files" : [ ["j0_output", "/root/"] ],
-      "guest-init" : "guest-init0.sh",
+      "outputs" : [ "/root/j0_output" ],
+      "guest-init" : "init0.sh",
       "run" : "run0.sh"
     },
     {
       "name" : "j1",
       "files" : [ ["j1_output", "/root/"] ],
-      "guest-init" : "guest-init1.sh",
+      "outputs" : [ "/root/j1_output" ],
+      "guest-init" : "init1.sh",
       "run" : "run1.sh"
     }
   ]

test/fed-smoke0/refOutput/fed-smoke0-j0/j0_output

@@ -0,0 +1,3 @@
+j0 : file
+j0 : init
+j0 : run

test/fed-smoke0/refOutput/fed-smoke0-j0/uartlog

@@ -0,0 +1,8 @@
+J0 Running:
+Global output:
+Global : file
+Global : init
+Job output:
+j0 : file
+j0 : init
+j0 : run

test/fed-smoke0/refOutput/fed-smoke0-j1/j1_output

@@ -0,0 +1,3 @@
+j1 : file
+j1 : init
+j1 : run

test/fed-smoke0/refOutput/fed-smoke0-j1/uartlog

@@ -0,0 +1,8 @@
+J1 Running:
+Global output:
+Global : file
+Global : init
+Job output:
+j1 : file
+j1 : init
+j1 : run

test/flist.json

@@ -2,5 +2,9 @@
   "name" : "flist",
   "base" : "br-base.json",
   "files" : [["runOutput", "/root/"]],
-  "command" : "cat /root/runOutput"
+  "command" : "cat /root/runOutput",
+  "testing" : {
+    "refDir" : "refOutput",
+    "strip" : true
+  }
 }

test/flist/refOutput/flist/uartlog

@@ -0,0 +1 @@
+Global : file

test/guest-init.json

@@ -2,5 +2,9 @@
   "name" : "guest-init",
   "base" : "br-base.json",
   "guest-init" : "init.sh",
-  "command" : "echo 'Global : command' >> /root/runOutput && cat /root/runOutput"
+  "command" : "echo 'Global : command' >> /root/runOutput && cat /root/runOutput",
+  "testing" : {
+    "refDir" : "refOutput",
+    "strip" : true
+  }
 }

test/guest-init/refOutput/guest-init/uartlog

@@ -0,0 +1,2 @@
+Global : init
+Global : command

test/hard.json

@@ -1,5 +1,10 @@
 {
   "name" : "hard",
   "bin" : "hard-bin",
-  "img" : "hard.img"
+  "img" : "hard.img",
+  "host-init" : "copy-files.sh",
+  "testing" : {
+    "refDir" : "refOutput",
+    "strip" : true
+  }
 }

test/hard/README.md

@@ -2,10 +2,11 @@ This is a hard-coded img/bin test. You would typically do this while actively
 developing a workload or goofing around with something temporarirly. You should typically
 not hard-code img and bin paths to avoid creating unreproducible images.
 
-*WARNING*: you should NOT include any other options in a hard-coded config like this. It's possible that strange errors could occur or your image/binary could be overwritten if you specify other options than: "name", "img", and "bin".
+*WARNING*: you should NOT include any other options in a hard-coded config like
+this. It's possible that strange errors could occur or your image/binary could
+be overwritten if you specify other options than: "name", "img", and "bin".
 
 # How the test works
 You should first manually build command.json (this choice was arbitrary). After
 that you can build hard.json and the host-init will copy over the images. The
 output will be identical to that of command.json.
-

test/hard/refOutput/hard/uartlog

@@ -0,0 +1 @@
+Global: command

test/host-init.json

@@ -1,7 +1,11 @@
 {
-  "name" : "host-guest-init",
+  "name" : "host-init",
   "base" : "br-base.json",
   "files" : [["runOutput", "/root/"]],
-  "host-guest-init" : "host-guest-init.sh",
-  "command" : "cat /root/runOutput"
+  "host-init" : "host-init.sh",
+  "command" : "cat /root/runOutput",
+  "testing" : {
+    "refDir" : "refOutput",
+    "strip" : true
+  }
 }

test/host-init/.gitignore

@@ -0,0 +1 @@
+runOutput

test/host-init/host-init.sh

@@ -1,2 +1,2 @@
 #!/bin/bash
-echo "Global : host-init" >> runOutput
+echo "Global : host-init" > runOutput

test/host-init/refOutput/host-init/uartlog

@@ -0,0 +1 @@
+Global : host-init

test/host-init/runOutput

@@ -1,3 +1 @@
-
-
-Global : file
+Global : host-init

test/jobs.json

@@ -1,6 +1,10 @@
 {
   "name" : "jobs",
   "base" : "br-base.json",
+  "testing" : {
+    "refDir" : "refOutput",
+    "strip" : true
+  },
   "jobs" : [
     {
       "name" : "j0",

test/jobs/refOutput/jobs-j0/uartlog

@@ -0,0 +1 @@
+j0 : run

test/jobs/refOutput/jobs-j1/uartlog

@@ -0,0 +1 @@
+j1 : run

test/linux-src.json

@@ -3,5 +3,9 @@
   "base" : "br-base.json",
   "host-init" : "copy-src.sh",
   "linux-src" : "riscv-linux",
-  "command" : "echo Global: command" 
+  "command" : "echo Global : command",
+  "post_run_hook" : "./check-output.sh linux-src/uartlog",
+  "testing" : {
+    "refDir" : "refOutput"
+  }
 }

test/linux-src/README.md

@@ -0,0 +1,16 @@
+Test support for alternate linux source directories. In this case, we copy the
+normal linux source tree and add a simple patch that includes an extra printk
+during shutdown. A more typical usage might be to have a custom fork of linux
+that you submodule in your workload directory.
+
+= How the unit test works =
+printk's get timestamps, which means the normal uartlog stripping and grepping
+doesn't really play nice with this non-deterministic output. Instead, we
+include a post\_run\_hook that greps for the correct outputs and touches a
+"SUCCESS" file if it works. It touches "FAILURE" if it doesn't work (although
+we don't actually check for that). The refOutput dir just checks for the
+existence of this file.
+
+It's likely that more complex workloads will create non-deterministic output
+that is complex to validate. Adding an empty "SUCCESS" file is the typical way
+to deal with that. It could be created during run-time or in a post\_run\_hook.

test/linux-src/check-output.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+# Usage: ./check-output.sh UARTLOG BASE_DIR
+#   - UARTLOG: path to uartlog to check
+#   - BASE_DIR: cd here before running
+#
+# Will check for a fixed pattern in the uartlog of a run. Will touch a file
+# called "SUCCESS" if the output matches, "FAILURE" otherwise. This is intended
+# to be used as a post_run_hook to the sw_manager or firesim commands (hence
+# the slightly odd arguments).
+
+uartlog=$1
+baseDir=$2
+
+cd $baseDir
+grep -q "Global : command" $uartlog
+commandRes=$?
+
+grep -q "Global : linux-src" $uartlog
+linuxRes=$?
+
+if [ $commandRes == 0 ] && [ $linuxRes == 0 ]; then
+  touch SUCCESS
+else
+  touch FAILURE
+fi

test/overlay.json

@@ -2,5 +2,9 @@
   "name" : "overlay",
   "base" : "br-base.json",
   "overlay" : "overlay",
-  "command" : "cat /root/runOutput"
+  "command" : "cat /root/runOutput",
+  "testing" : {
+    "refDir" : "refOutput",
+    "strip" : true
+  }
 }

test/overlay/refOutput/overlay/uartlog

@@ -0,0 +1 @@
+Global : overlay

test/post_run_hook.json

@@ -0,0 +1,9 @@
+{
+  "name" : "post_run_hook",
+  "base" : "br-base.json",
+  "command" : "echo Global: command",
+  "post_run_hook" : "./post_run_hook.sh",
+  "testing" : {
+      "refDir" : "refOutput"
+  }
+}

test/post_run_hook/post_run_hook.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+outDir=$1
+
+cd $outDir
+echo "Global : post_run_hook" > post_run_hook/runOutput

test/post_run_hook/refOutput/post_run_hook/runOutput

@@ -0,0 +1 @@
+Global : post_run_hook

test/run.json

@@ -1,5 +1,9 @@
 {
   "name" : "run",
   "base" : "br-base.json",
-  "run" : "run.sh"
+  "run" : "run.sh",
+  "testing" : {
+    "refDir" : "refOutput",
+    "strip" : true
+  }
 }

test/run/refOutput/run/uartlog

@@ -0,0 +1 @@
+Global : run

test/run/run.sh

@@ -1,2 +1,3 @@
 #!/bin/bash
 echo "Global : run"
+poweroff

test/smoke0-workdir/refOutput/smoke0/uartlog

@@ -0,0 +1,3 @@
+Global : file
+Global : init
+Global : command

test/smoke0-workdir/runOutput

@@ -1,3 +1 @@
-
-
 Global : file

test/smoke0.json

@@ -4,5 +4,9 @@
   "base" : "br-base.json",
   "files" : [ [ "runOutput", "/root/" ] ],
   "guest-init" : "init.sh",
-  "command" : "echo Global : command >> /root/runOutput; cat /root/runOutput"
+  "command" : "echo Global : command >> /root/runOutput; cat /root/runOutput",
+  "testing" : {
+    "refDir" : "refOutput",
+    "strip" : true
+  }
 }

test/smoke1.json

@@ -3,6 +3,10 @@
   "workdir" : "smoke1",
   "base" : "br-base.json",
   "overlay" : "overlay",
-  "guest-init" : "guest-init.sh",
-  "run" : "run.sh"
+  "guest-init" : "init.sh",
+  "run" : "run.sh",
+  "testing" : {
+    "refDir" : "refOutput",
+    "strip" : true
+  }
 }

test/smoke1/refOutput/smoke1/uartlog

@@ -0,0 +1,3 @@
+Global : overlay
+Global : init
+Global : run

test/smoke2.json

@@ -5,18 +5,22 @@
   "guest-init" : "init.sh",
   "outputs" : [ "/root/runOutput" ],
   "run" : "run.sh",
+  "testing" : {
+    "refDir" : "refOutput",
+    "strip" : true
+  },
   "jobs" : [
     {
       "name" : "j0",
       "files" : [ ["j0_output", "/root/"] ],
       "guest-init" : "init0.sh",
-      "outputs" : [ "/root/j0_output" ],
+      "outputs" : [ "/root/j0_output", "/root/runOutput" ],
       "run" : "run0.sh"
     },
     {
       "name" : "j1",
       "files" : [ ["j1_output", "/root/"] ],
-      "outputs" : [ "/root/j1_output" ],
+      "outputs" : [ "/root/j1_output", "/root/runOutput" ],
       "guest-init" : "init1.sh",
       "run" : "run1.sh"
     }

test/smoke2/refOutput/smoke2-j0/j0_output

@@ -0,0 +1,3 @@
+j0 : file
+j0 : init
+j0 : run

test/smoke2/refOutput/smoke2-j0/runOutput

@@ -0,0 +1,2 @@
+Global : file
+Global : init

test/smoke2/refOutput/smoke2-j0/uartlog

@@ -0,0 +1,5 @@
+Global : file
+Global : init
+j0 : file
+j0 : init
+j0 : run

test/smoke2/refOutput/smoke2-j1/j1_output

@@ -0,0 +1,3 @@
+j1 : file
+j1 : init
+j1 : run