Skip to content

Writing test results to JSON files

Richard R. Drake edited this page May 11, 2023 · 1 revision

After vvtest completes a testing sequence, all the results can be dumped to a JSON file for later analysis. The command line option is --json <filename>, and note that it can be used after vvtest finishes by adding -i, such as vvtest -i --json <filename>.

The structure of the JSON file is described in the following pseudo-format:

{
    "vvtest": {
        "curdir": str,
        "startdate": str,
        "command": str,
        "vvtestdir": str,
        "vvplatform": str,  # the vvtest platform name
        "compiler": (None, str),  # None or a string
        "rundir": str,
        "starttime": (int,float),  # a number
        "endtime": (int,float),
        "enddate": (None, str),
        "duration": float,
        "returncode": int,
        "onopts": str,
        "offopts": str,
        "testargs": str,
        "machine": {
            "platform": str,
            "system": str,
            "nodename": str,
            "release": str,
            "version": str,
            "arch": str,
        },
        "python": {
            "executable": str,
            "version": str,
            "version info": list,
        },
        "environment": dict,
        "tests": {
            "tests": int,  # number of test cases
            "pass": int,
            "notdone": int,
            "notrun": int,
            "diff": int,
            "fail": int,
            "timeout": int,
            "cases": [
                {
                    "name": str,
                    "case": str,
                    "id": str,
                    "root": str,
                    "path": str,
                    "keywords": [str,],  # a list of strings
                    "parameters": dict,  # must be { "name":value, ...}
                    "starttime": (int,float),
                    "endtime": (int,float),
                    "returncode": (None,int),
                    "result": str,
                    "timeout": int,
                    "resources": {
                        "processors": (None, int),  # num processors
                        "processor ids": (None, [int,]),  # a list of int
                        "total processors": (None, int),
                        "devices": (None, int),  # num devices
                        "device ids": (None, [int,]),  # None or a list of int
                        "total devices": (None, int),
                    },
                    "command": (None, str),
                    "log": (None, str),  # None or a b64 encoded zlib compressed string
                },
            ],
        },
    }
}

Note that the tests will be listed in the "cases" entry.