Fix initramfs on fedora. More tests (including timeout tests and more

robust full_test.sh).

.gitignore

@@ -1,6 +1,7 @@
 *.img
 *-bin
 *.swp
+test.log
 fedora/Fedora-Developer-Rawhide-20181015.n.0-sda.raw
 fedora/Fedora-Developer-Rawhide-20181015.n.0-sda.raw.xz
 fedora/rootfs.cpio

full_test.sh

@@ -2,6 +2,8 @@
 # Enable extended globbing
 shopt -s extglob
 
+SUITE_PASS=true
+
 # Reset the test log
 echo "" > test.log
 
@@ -11,28 +13,67 @@ echo "Running clean test" | tee -a test.log
 ./test/clean/test.py >> test.log 
 if [ $? != 0 ]; then
   echo "Failure" | tee -a test.log
-  exit 1
+  SUITE_PASS=false
 fi
 
 echo "Running incremental test" | tee -a test.log
 ./test/incremental/test.py >> test.log
 if [ $? != 0 ]; then
   echo "Failure" | tee -a test.log
+  SUITE_PASS=false
   exit 1
 fi
 
+echo "Running build timeout test (should timeout):" | tee -a test.log
+./sw_manager.py test test/timeout-build.json | grep "timeout while building"
+if [ $? != 0 ]; then
+  echo "Failure" | tee -a test.log
+  SUITE_PASS=false
+else
+  echo "Success" | tee -a test.log
+fi
+
+echo "Running build timeout test (should timeout):" | tee -a test.log
+./sw_manager.py test test/timeout-run.json | grep "timeout while running"
+if [ $? != 0 ]; then
+  echo "Failure" | tee -a test.log
+  SUITE_PASS=false
+else
+  echo "Success" | tee -a test.log
+fi
+
 # Run the bulk tests (all work with the 'test' command)
 # Note the funny extended globbing, these are just lists of tests that
 # shouldn't be tested (e.g. we exclude the base configs and some specialized
 # tests)
 echo "Running regular tests" | tee -a test.log
-BULK_EXCLUDE="(br-base|fedora-base|incremental|clean)"
+BULK_EXCLUDE="(br-base|fedora-base|incremental|clean|timeout-build|timeout-run)"
+./sw_manager.py clean test/!$BULK_EXCLUDE.json | tee -a test.log
 ./sw_manager.py test test/!$BULK_EXCLUDE.json | tee -a test.log
+if [ $? != 0 ]; then
+  echo "Failure" | tee -a test.log
+  SUITE_PASS=false
+else
+  echo "Success" | tee -a test.log
+fi
 
 # Run the initramfs versions on spike, we exclude a few tests that don't make
 # sense to use with initramfs and/or spike (e.g. bare-metal)
 echo "Running initramfs capable tests on spike" | tee -a test.log
-IS_EXCLUDE="(hard|bare|br-base|fedora-base|incremental|clean)"
+IS_EXCLUDE="(hard|bare|br-base|fedora-base|incremental|clean|timeout-build|timeout-run)"
+./sw_manager.py -i clean test/!$IS_EXCLUDE.json | tee -a test.log
 ./sw_manager.py -i test -s test/!$IS_EXCLUDE.json | tee -a test.log
+if [ $? != 0 ]; then
+  echo "Failure" | tee -a test.log
+  SUITE_PASS=false
+else
+  echo "Success" | tee -a test.log
+fi
 
-echo "Full Test Success" | tee -a test.log
+if [ $SUITE_PASS = false ]; then
+  echo "Some tests failed" | tee -a test.log
+  exit 1
+else
+  echo "Full Test Success" | tee -a test.log
+  exit 0
+fi

sw_manager.py

@@ -4,13 +4,16 @@
 import os
 import logging
 import wlutil
+import contextlib
 
-# from wlutil.test import cmpOutput
-# print(cmpOutput("/data/repos/firesim/sw/firesim-software/runOutput/fed-run-test-2018-12-17--05-02-00-5WB1RF8OYAZUWBY3/", "/data/repos/firesim/sw/firesim-software/test/fed-run/refOutput/"))
-# sys.exit()
 if 'RISCV' not in os.environ:
     sys.exit("Please source firesim/sourceme-manager-f1.sh first\n")
 
+# Delete a file but don't throw an exception if it doesn't exist
+def deleteSafe(pth):
+    with contextlib.suppress(FileNotFoundError):
+        os.remove(pth)
+
 def main():
     parser = argparse.ArgumentParser(
         description="Build and run (in spike or qemu) boot code and disk images for firesim")
@@ -95,29 +98,33 @@ def main():
             wlutil.launchWorkload(cfgPath, cfgs, args.job, args.spike)
         elif args.command == "test":
             skipCount = 0
+            failCount = 0
             log.info("Running: " + cfgPath)
             res = wlutil.testWorkload(cfgPath, cfgs, args.verbose, spike=args.spike)
             if res is wlutil.testResult.failure:
+                print("Test Failed")
                 suitePass = False
+                failCount += 1
             elif res is wlutil.testResult.skip:
+                print("Test Skipped")
                 skipCount += 1
+            else:
+                print("Test Passed")
             log.info("")
         elif args.command == 'clean':
-            try:
+            # with contextlib.suppress(FileNotFoundError):
                 if 'bin' in targetCfg:
-                    os.remove(targetCfg['bin'])
-                    os.remove(targetCfg['bin'] + '-initramfs')
+                    deleteSafe(targetCfg['bin'])
+                    deleteSafe(targetCfg['bin'] + '-initramfs')
                 if 'img' in targetCfg:
-                    os.remove(targetCfg['img'])
+                    deleteSafe(targetCfg['img'])
                 if 'jobs' in targetCfg:
                     for jCfg in targetCfg['jobs'].values():
                         if 'bin' in jCfg:
-                            os.remove(jCfg['bin'])
-                            os.remove(jCfg['bin'] + '-initramfs')
+                            deleteSafe(jCfg['bin'])
+                            deleteSafe(jCfg['bin'] + '-initramfs')
                         if 'img' in jCfg:
-                            os.remove(jCfg['img'])
-            except FileNotFoundError:
-                pass
+                            deleteSafe(jCfg['img'])
         else:
             log.error("No subcommand specified")
             sys.exit(1)

test/fed-smoke0.json

@@ -5,6 +5,8 @@
   "guest-init" : "init.sh",
   "run" : "run.sh",
   "testing" : {
+    "buildTimeout" : 1200,
+    "runTimeout" : 600,
     "refDir" : "refOutput",
     "strip" : true
   },

test/fedora-base/linux-config

@@ -1382,7 +1382,7 @@ CONFIG_NLS_DEFAULT="iso8859-1"
 # Kernel hacking
 #
 CONFIG_CMDLINE_BOOL=y
-CONFIG_CMDLINE="console=hvc0 root=/dev/generic-blkdev rw"
+CONFIG_CMDLINE="root=/dev/generic-blkdev rw"
 # CONFIG_CMDLINE_OVERRIDE is not set
 CONFIG_EARLY_PRINTK=y
 

test/smoke0-workdir/init.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
-echo "Global : init" >> /root/runOutput
-cat /root/runOutput
+echo "Global : init" >> /root/testFile
+cat /root/testFile
 
 sync
 poweroff -f

test/smoke0.json

@@ -2,9 +2,9 @@
   "name" : "smoke0",
   "workdir" : "smoke0-workdir",
   "base" : "br-base.json",
-  "files" : [ [ "runOutput", "/root/" ] ],
+  "files" : [ [ "testFile", "/root/" ] ],
   "guest-init" : "init.sh",
-  "command" : "echo Global : command >> /root/runOutput; cat /root/runOutput",
+  "command" : "echo Global : command > /root/runOutput; cat /root/testFile; cat /root/runOutput",
   "testing" : {
     "refDir" : "refOutput",
     "strip" : true

test/spike/refOutput/spike/uartlog

@@ -1,3 +1,2 @@
 Global : spike
 Hello World
-Global : spike

test/timeout-build.json

@@ -0,0 +1,10 @@
+{
+  "name" : "timeout-build",
+  "base" : "br-base.json",
+  "workdir" : "command",
+  "command" : "echo Global: command",
+  "testing" : {
+      "buildTimeout" : 1,
+      "refDir" : "refOutput"
+  }
+}

test/timeout-run.json

@@ -0,0 +1,10 @@
+{
+  "name" : "timeout-run",
+  "base" : "br-base.json",
+  "workdir" : "command",
+  "command" : "echo Global: command",
+  "testing" : {
+      "runTimeout" : 1,
+      "refDir" : "refOutput"
+  }
+}

wlutil/init

@@ -0,0 +1 @@
+/sbin/init

wlutil/launch.py

@@ -1,6 +1,10 @@
 import logging
 from .wlutil import *
 
+# The amount of memory to use when launching
+launch_mem = "16384"
+launch_cores = "4"
+
 # Returns a command string to luanch the given config in spike. Must be called with shell=True.
 def getSpikeCmd(config, initramfs=False):
     if 'spike' in config:
@@ -9,9 +13,9 @@ def getSpikeCmd(config, initramfs=False):
         spikeBin = 'spike'
 
     if initramfs:
-        return spikeBin + ' -p4 -m4096 ' + config['bin'] + '-initramfs'
+        return spikeBin + ' -p' + launch_cores + ' -m' + launch_mem + " " + config['bin'] + '-initramfs'
     elif 'img' not in config:
-        return spikeBin + ' -p4 -m4096 ' + config['bin']
+        return spikeBin + ' -p' + launch_cores + ' -m' + launch_mem + " " + config['bin']
     else:
         raise ValueError("Spike does not support disk-based configurations")
 
@@ -26,9 +30,9 @@ def getQemuCmd(config, initramfs=False):
 
     cmd = ['qemu-system-riscv64',
            '-nographic',
-           '-smp', '4',
+           '-smp', launch_cores,
            '-machine', 'virt',
-           '-m', '4G',
+           '-m', launch_mem,
            '-kernel', exe,
            '-object', 'rng-random,filename=/dev/urandom,id=rng0',
            '-device', 'virtio-rng-device,rng=rng0',

wlutil/test.py

@@ -19,6 +19,12 @@
 
 testResult = Enum('testResult', ['success', 'failure', 'skip'])
 
+# Default timeouts (in seconds)
+defBuildTimeout = 900 # 15 min (if there's lots of jobs, init scripts, and/or fedora)
+defRunTimeout = 600 # 5 min
+# defBuildTimeout = 2 # 15 min (if there's lots of jobs, init scripts, and/or fedora)
+# defRunTimeout = 600 # 5 min
+
 # Compares two runOutput directories. Returns None if they match or a message
 # describing the difference if they don't.
 #   - Directory structures are compared directly (same folders in the same
@@ -67,10 +73,6 @@ def cmpOutput(testDir, refDir, strip=False):
 
     return None
 
-# Default timeouts (in seconds)
-defBuildTimeout = 900 # 15 min (if there's lots of jobs, init scripts, and/or fedora)
-defRunTimeout = 300 # 5 min
-
 # adapted from https://stackoverflow.com/questions/4675728/redirect-stdout-to-a-file-in-python/22434262#22434262
 def fileno(file_or_fd):
     fd = getattr(file_or_fd, 'fileno', lambda: file_or_fd)()
@@ -149,7 +151,7 @@ def stripBrUart(lines):
 def stripUartlog(config, outputPath):
     outDir = pathlib.Path(outputPath)
     for uartPath in outDir.glob("**/uartlog"):
-        with open(str(uartPath), 'r') as uFile:
+        with open(str(uartPath), 'r', errors='ignore') as uFile:
             uartlog = uFile.readlines()
 
         if 'distro' in config:
@@ -215,20 +217,24 @@ def testWorkload(cfgName, cfgs, verbose=False, spike=False):
 
     except TimeoutError as e:
         suitePass = False
-        if e.args[0] == "handleBuild":
+        if e.args[0] == "buildWorkload":
             log.info("Test " + os.path.basename(cfgName) + " failure: timeout while building")
-        elif e.args[0] == "handleLaunch":
+        elif e.args[0] == "launchWorkload":
             log.info("Test " + os.path.basename(cfgName) + " failure: timeout while running")
+        else:
+            log.error("Internal tester error: timeout from unrecognized function: " + e.args[0])
 
         log.info("Output available in " + testPath)
         return testResult.failure
 
     except ChildProcessError as e:
         suitePass = False
-        if e.args[0] == "handleBuild":
+        if e.args[0] == "buildWorkload":
             log.info("Test " + os.path.basename(cfgName) + " failure: Exception while building")
-        elif e.args[0] == "handleLaunch":
+        elif e.args[0] == "launchWorkload":
             log.info("Test " + os.path.basename(cfgName) + " failure: Exception while running")
+        else:
+            log.error("Internal tester error: exception in unknown function: " + e.args[0])
 
         log.info("Output available in " + testPath)
         return testResult.failure

wlutil/wlutil.py

@@ -6,8 +6,9 @@
 import string
 import sys
 import collections
-import pathlib as pth
+import shutil
 
+wlutil_dir = os.path.normpath(os.path.dirname(__file__))
 root_dir = os.getcwd()
 image_dir = os.path.join(root_dir, "images")
 linux_dir = os.path.join(root_dir, "riscv-linux")
@@ -110,10 +111,15 @@ def toCpio(config, src, dst):
 
     run(['sudo', 'mount', '-o', 'loop', src, mnt])
     try:
+        # Fedora needs a special init in order to boot from initramfs
         run("sudo find -print0 | sudo cpio --owner root:root --null -ov --format=newc > " + dst, shell=True, cwd=mnt)
     finally:
         run(['sudo', 'umount', mnt])
 
+    # fedora needs a special init to work
+    if config['distro'] == 'fedora':
+        sp.call("cat " + os.path.join(wlutil_dir, "fedora-initramfs-append.cpio") + " >> " + dst, shell=True)
+
 # Apply the overlay directory "overlay" to the filesystem image "img"
 # Note that all paths must be absolute
 def applyOverlay(img, overlay):
@@ -154,5 +160,3 @@ def copyImgFiles(img, files, direction):
         # run(['guestunmount', mnt])
         # run(['fusermount', '-u', mnt])
         run(['sudo', 'umount', mnt])
-
-