add spion benchmark suite

.editorconfig

@@ -0,0 +1,14 @@
+# This file is for unifying the coding style for different editors and IDEs
+# editorconfig.org
+
+root = true
+
+[*]
+end_of_line = lf
+charset = utf-8
+trim_trailing_whitespace = true
+insert_final_newline = true
+
+[**.js]
+indent_style = space
+indent_size = 4

.gitignore

@@ -1,6 +1,7 @@
 node_modules/*
 todo.txt
 npm-debug.log
+benchmark/async-compare/node_modules/*
 benchmark/promises-benchmark/node_modules/*
 benchmark/promises-benchmark/fixtures/*
 benchmark/perf-promises/node_modules/*

.npmignore

@@ -0,0 +1,15 @@
+node_modules/*
+todo.txt
+npm-debug.log
+benchmark/*
+benchmark/async-compare/node_modules/*
+benchmark/promises-benchmark/node_modules/*
+benchmark/promises-benchmark/fixtures/*
+benchmark/perf-promises/node_modules/*
+async
+sync
+mixed
+bench.json
+js/bluebird.min.js
+js/bluebird_sync.js
+js/bluebird_sync.min.js

Gruntfile.js

@@ -158,29 +158,29 @@ module.exports = function( grunt ) {
     grunt.loadNpmTasks('grunt-contrib-concat');
     grunt.loadNpmTasks('grunt-bump');
 
-    function runIndependent( file, cb ) {
-    var fs = require("fs");
-    var sys = require('sys');
-    var spawn = require('child_process').spawn;
-    var node = spawn('node', ["./"+file]);
-    node.stdout.on('data', function( data ) {
-        process.stdout.write(data);
-    });
+    function runIndependentTest( file, cb ) {
+        var fs = require("fs");
+        var sys = require('sys');
+        var spawn = require('child_process').spawn;
+        var node = spawn('node', ["./"+file]);
+        node.stdout.on('data', function( data ) {
+            process.stdout.write(data);
+        });
 
-    node.stderr.on('data', function( data ) {
-        process.stderr.write(data);
-    });
+        node.stderr.on('data', function( data ) {
+            process.stderr.write(data);
+        });
 
-    function exit( code ) {
-        if( code !== 0 ) {
-            cb(new Error("process didn't exit normally"));
-        }
-        else {
-            cb(null);
+        function exit( code ) {
+            if( code !== 0 ) {
+                throw new Error("process didn't exit normally");
+            }
+            else {
+                cb(null);
+            }
         }
-    }
 
-    node.on('exit', exit );
+        node.on('exit', exit );
     }
 
     function fixStrict( code ) {
@@ -309,7 +309,7 @@ module.exports = function( grunt ) {
                     var suite = mochas[i].suite;
                     if( suite.suites.length === 0 &&
                         suite.tests.length === 0 ) {
-                        runIndependent(mochas[i].files[0], function(err) {
+                        runIndependentTest(mochas[i].files[0], function(err) {
                             if( err ) throw err;
                             setTimeout(function(){
                                 runner( mochas, i + 1 );
@@ -347,10 +347,17 @@ module.exports = function( grunt ) {
                 done();
             }
             else {
-                grunt.log.writeln("Running benchmark " + files[i] );
-                require(files[i])(function(){
+                var file = files[i];
+                grunt.log.writeln("Running benchmark " + file );
+                if( file.indexOf( "matcha" ) !== -1 ) {
+                    grunt.log.writeln("Run matcha benchmarks with match");
                     runner(files, i + 1 );
-                });
+                }
+                else {
+                    require(file)(function(){
+                        runner(files, i + 1 );
+                    });
+                }
             }
         })(files, 0);
     }

benchmark/async-compare/README.md

@@ -0,0 +1,116 @@
+# async-compare
+
+This project aims compare various node.js async patterns by their
+
+- complexity (number of necessary tokens)
+- performance when executing in parallel (time and memory)
+- debuggability 
+
+The resulting analysis is available at 
+[this blog post](http://spion.github.io/posts/analysis-generators-and-other-async-patterns-node.html)
+
+## example problem
+
+The problem is directly extracted from a DoxBee project. Its a typical if 
+somewhat complex CRUD method executed when a user uploads a new document
+to the database. It involves multiple queries to the database, a couple of 
+selects, some inserts and one update. Lots of mixed sync/async action.
+
+## files
+
+Example solutions for all patterns are located in the `examples` directory
+
+Non-js sorce files begin with `src-` (they're not checked for performance)
+
+Compiled files are prefixed with `dst-` (they're not checked for complexity)
+
+All other files are checked for both performance and complexity
+
+## complexity
+
+Complexity is measured by the number of tokens in the source code found by
+Esprima's lexer (comments excluded)
+
+Run `node complexity.js` to get complexity reports for all files.
+
+
+## fakes.js
+
+Wrappers can be added in `lib/fakes.js`
+
+For examples, look at the promise and thunk wrappers for query methods.
+
+Things that are specific to the upload function are not allowed here.
+
+
+## performance
+
+All external methods are mocked with setTimeout, to simulate waiting for I/O 
+operations.
+
+Performance is measured by performance.js
+
+    node performance.js --n <parallel> --t <miliseconds> ./examples/*.js --harmony
+
+where `n` is the number of parallel executions of the method, while `t` is the
+time each simulated I/O operation should take, and `--harmony` enables
+all features hidden behind the v8 flag.
+
+There is an optional parameter `--file <file>` which will only test a single
+file and report any encountered errors in detail:
+
+    node --harmony performance.js --n 10000 --t 10 --file ./examples/genny.js
+
+Also, this variant doesn't spawn a new process so which means additional
+(v8) options can be passed to node.
+
+If you omit `--n`, tests will be made with 100, 500, 1000 and 2000 parallel
+requests and a giant JSON report (suitable for charts) will be generated.
+
+    node performance.js --t 1 ./examples/*.js --harmony
+
+If you omit `--n` *and* replace `--t` with `--dt`, I/O time `t` will grow with 
+`n` by the formula `t = n * dt`
+
+    node performance.js --dt 0.1 ./examples/*.js --harmony
+
+Execution time and peak memory usage are reported.
+
+
+## debuggability
+
+
+`debuggability.js` measures the distance between the function that creates the 
+error and the actual error in the stack trace. Reports "-" at places where
+the stack trace is completely missing the original file.
+
+To check all examples for async errors:
+
+```
+node debuggability.js --harmony --error 
+```
+
+and for exceptions:
+
+```
+node debuggability.js --harmony --throw
+```
+
+and finally for exceptions inside async calls (most things can't handle this):
+
+
+```
+node debuggability.js --harmony --athrow
+```
+
+
+## misc 
+
+These are factors potentially important for collaboration which could
+be added as points to arrive at a final score:
+
+- does it require native modules (-2)
+- does it require code transformation (-2) 
+- will it eventually become available without code transformation (+1)
+
+

benchmark/async-compare/complexity.js

@@ -0,0 +1,42 @@
+
+var fs = require('fs');
+var table = require('text-table');
+
+var stats = module.exports = function stats() {
+    return fs.readdirSync(__dirname + '/examples').filter(function(f){ 
+       return !/^dst-/.test(f);
+    }).map(function(f) {
+        var file = fs.readFileSync('./examples/'+f).toString();
+        file = file.replace(/function\s*\*/g, 'function')
+            .replace(/yield/g, 'void');
+        try {
+            var tree = require('esprima').parse(file, {
+                tolerant: true,
+                tokens: true
+            });
+        } catch (e) {
+            console.log("In file", f, ":");
+            console.log(e);
+        }
+        return {name: f, tokens: tree.tokens.length}
+    });
+}
+
+var s = stats();
+
+var mintokens = s.reduce(function(acc, f) { 
+    return Math.min(acc, f.tokens);
+}, Number.POSITIVE_INFINITY);
+
+s = s.sort(function(s1, s2) {
+    return s1.tokens - s2.tokens;
+});
+
+s.forEach(function(f) {
+    f.complexity = f.tokens / mintokens;
+});
+
+console.log(table([['name', 'tokens', 'complexity']].concat(
+    s.map(function(f) { return [f.name, f.tokens, f.complexity.toFixed(2)] })
+), {align: ['l','r','r']}));
+

benchmark/async-compare/debuggability.js

@@ -0,0 +1,112 @@
+Error.stackTraceLimit = Infinity;
+
+var args = require('optimist').argv;
+
+global.longStackSupport = require('q').longStackSupport = true;
+
+var perf = module.exports = function(args, done) {
+    global.asyncTime = args.t || 1;
+    global.testThrow = args.throw;
+    global.testThrowAsync = args.athrow;
+    global.testError = args.error;
+
+    var fn = require(args.file);
+    fn('a','b','c', done);
+}
+
+
+if (args.file) {
+    perf(args, function(err) {
+        if (err) { 
+            //throw err; 
+            console.log(err);
+            // for browser-compatibility, stratifiedjs reports errors
+            // on __oni_stack (or toString()), rather than 'stack'.
+            console.error(err.__oni_stack ? err.__oni_stack.map(function(x) { return x.join(':') }).join('\n') : new Error(err.stack).stack);
+        }
+    });
+} else {
+    var cp    = require('child_process')
+    var async = require('async');
+    var fs    = require('fs');    
+    var dir = __dirname + '/examples';
+
+    var table = require('text-table');
+
+
+    var allfiles = fs.readdirSync(dir);
+
+    var files = allfiles.filter(function(f) {
+        return !/^src-/.test(f);
+    });
+
+    var sources = allfiles.filter(function(f) {
+        return !/^dst-/.test(f);
+    });
+
+    var sourceOf = function(f) {
+        var parts = f.split('-');
+        var name = parts[1];
+        return sources.filter(function(s) {
+            return s.indexOf(name) >= 0;
+        })[0] || f;      
+    }
+
+    async.mapSeries(files, function(f, done) {
+        console.error("testing", f);
+
+        var argsFork = [__filename, 
+            '--file', dir + '/' + f];
+        if (args.error) argsFork.push('--error')
+        if (args.throw) argsFork.push('--throw');
+        if (args.athrow) argsFork.push('--athrow');
+
+        if (args.harmony) argsFork.unshift('--harmony');
+
+        var p = cp.spawn(process.execPath, argsFork);
+
+
+        var lineNumber = fs.readFileSync(dir + '/' + sourceOf(f), 'utf8')
+            .split('\n')
+            .map(function(l, k) { 
+                return { 
+                    contained: l.indexOf('FileVersion.insert') >= 0, 
+                    line: k + 2
+                };
+            }).filter(function(l) { return l.contained; })[0].line;
+
+        var r = { file: f, data: [], line: lineNumber };
+
+        p.stderr.pipe(process.stderr);
+        p.stderr.on('data', function(d) { r.data.push(d.toString()); });
+        p.stderr.on('end', function(code) {
+            r.data = r.data.join('').split('\n').filter(function(line) {
+                // match lines reporting either compiled or source files: 
+                return line.match('examples/' + f) || line.match('examples/' + sourceOf(f))
+            }).map(function(l) { 
+                return {content: l, 
+                    line: l.split(':')[1],
+                    distance: Math.abs(l.split(':')[1] - r.line)};
+            }).sort(function(l1, l2) {
+                return l1.distance - l2.distance;
+            })[0];
+            done(null, r);
+        });
+    }, function(err, res) {
+        console.log("");
+        console.log("error reporting");
+        console.log("");
+        res = res.sort(function(r1, r2) {
+            return parseFloat(r1.data ? r1.data.distance : Infinity)
+                - parseFloat(r2.data ? r2.data.distance : Infinity)
+        });
+        res = res.map(function(r) { 
+            return [r.file, r.line, 
+                r.data ? r.data.line : '-',
+                r.data ? r.data.distance : '-'];
+                //r.data ? 'yes ' + r.data.content :'no'];
+        })
+        res = [['file', 'actual-line', 'rep-line', 'distance']].concat(res)
+        console.log(table(res, {align: ['l','r','r','r']}));
+    });
+}