From 504e09123ef637c35b016f2092b5ec7b8ade4199 Mon Sep 17 00:00:00 2001
From: Frederic Charette <fcharette@shutterstock.com>
Date: Thu, 28 Feb 2019 14:38:15 -0500
Subject: [PATCH 1/5] workerized app for bench

---
 tests/profiling/dao.js      | 29 +++++++++++++
 tests/profiling/index.js    | 82 ++++++++++++-------------------------
 tests/profiling/settings.js |  9 ++--
 tests/profiling/testApp.js  | 68 ++++++++++++++++++++++++++++++
 4 files changed, 129 insertions(+), 59 deletions(-)
 create mode 100644 tests/profiling/dao.js
 create mode 100644 tests/profiling/testApp.js

diff --git a/tests/profiling/dao.js b/tests/profiling/dao.js
new file mode 100644
index 0000000..25c7a52
--- /dev/null
+++ b/tests/profiling/dao.js
@@ -0,0 +1,29 @@
+function simulateNetwork() {
+    let tick = 0;
+    while(true) {
+        if (++tick > 0xffff) break;
+    }
+    return true;
+}
+
+function calculateResponseTime(numItems) {
+    return Math.round(30 + numItems * 0.5);
+}
+
+function getAssets(ids, { language }) {
+    return new Promise((resolve, reject) => {
+        simulateNetwork();
+        setTimeout(() => resolve(ids.map(id => ({ id, language }))), calculateResponseTime(ids.length));
+    });
+}
+
+function getErroredRequest(ids, { language }) {
+    return new Promise((resolve, reject) => {
+      throw new Error('Something went wrong');
+    });
+}
+
+module.exports = {
+    getAssets,
+    getErroredRequest,
+};
diff --git a/tests/profiling/index.js b/tests/profiling/index.js
index 3b6bd26..24c990b 100644
--- a/tests/profiling/index.js
+++ b/tests/profiling/index.js
@@ -6,75 +6,47 @@
 /* Requires ------------------------------------------------------------------*/
 
 const crypto = require('crypto');
-const {getAssets, getErroredRequest} = require('../integration/utils/dao.js');
 const settings = require('./settings');
-const HA = require('../../src/index.js');
+const { Worker } = require('worker_threads');
 
 /* Init ----------------------------------------------------------------------*/
 
 // Setup
-const store = HA(settings.setup);
+const app = new Worker('./testApp.js');
 const now = Date.now();
 
-// Suite
-const suite = {
-  sampleRange: 2,
-  completed: 0,
-  cacheHits: 0,
-  sum: 0,
-  timeouts: 0,
-  batches: 0,
-  startHeap: process.memoryUsage().rss,
-};
-
-store.on('query', () => { suite.batches++; });
-store.on('cacheHit', () => { suite.cacheHits++; });
-
 async function hitStore() {
   if (Date.now() - now <  settings.test.testDuration) {
-    setTimeout(hitStore, settings.test.requestDelay);
-    let finished = false;
-    setTimeout(() => {
-      if (finished === false) suite.timeouts++;
-    }, 500);
+    process.nextTick(hitStore);
+    
     // Simulate normal z-distribution
-    suite.sampleRange = (Math.round(Math.random()*3) === 0) ? 1:2;
-    const randomError = (Math.round(Math.random()*10) === 0);
-    const id = crypto.randomBytes(suite.sampleRange).toString('hex');
+    let sampleRange = (Math.round(Math.random()*3) === 0) ? 1:2;
+    const id = crypto.randomBytes(sampleRange).toString('hex');
     const language = settings.test.languages[Math.floor(Math.random()*settings.test.languages.length)];
-    const before = Date.now();
-    if (randomError) store.config.resolver = getErroredRequest;
-    else store.config.resolver = getAssets;
-    store.get(id, { language }, crypto.randomBytes(8).toString('hex'))
-      .then((result) => {
-        if (!result || result.id !== id || result.language !== language) {
-          console.log(result, 'does not match', id, language);
-          throw new Error('integrity test failed');
-        }
-        suite.sum += (Date.now() - before);
-        finished = true;
-        suite.completed++;
-      }, () => {})
-      .catch((err) => { console.log(err); process.exit(1)} );
+    app.postMessage({ id, language });
   }
   else {
-    console.log(`
-      ${suite.completed} completed requests
-      ${suite.cacheHits} cache hits
-      ${JSON.stringify(await store.size())}
-      ${suite.timeouts} timed out
-      avg response time ${(suite.sum / suite.completed).toFixed(3)}
-      ${suite.batches} queries sent
-      ${((process.memoryUsage().rss - suite.startHeap) / 1024).toFixed(2)} Kbytes allocated
-    `);
-    for (const expectation in settings.assert) {
-      if (suite[expectation] < settings.assert[expectation][0] || suite[expectation] > settings.assert[expectation][1]) {
-        console.log(expectation, 'did not match expectation', settings.assert[expectation]);
-        throw new Error('performance test failed');
+    app.postMessage({ event: 'finish' });
+    app.on('message', (suite) => {
+      console.log(`
+        ${suite.completed} completed requests
+        ${suite.cacheHits} cache hits
+        ${JSON.stringify(await store.size())}
+        ${suite.timeouts} timed out
+        avg response time ${(suite.sum / suite.completed).toFixed(3)}
+        ${suite.batches} queries sent
+        ${suite.avgBatchSize} items per queries on average
+        ${((process.memoryUsage().rss - suite.startHeap) / 1024).toFixed(2)} Kbytes allocated
+      `);
+
+      for (const expectation in settings.assert) {
+        if (suite[expectation] <= settings.assert[expectation][0] || suite[expectation] >= settings.assert[expectation][1]) {
+          console.log(expectation, 'did not match expectation', settings.assert[expectation]);
+          throw new Error('performance test failed');
+        }
       }
-    }
-    
-    process.exit(0);
+      process.exit(0);
+    });
   }
 }
 
diff --git a/tests/profiling/settings.js b/tests/profiling/settings.js
index 95deb72..ed5cb96 100644
--- a/tests/profiling/settings.js
+++ b/tests/profiling/settings.js
@@ -1,16 +1,16 @@
-const {getAssets} = require('../integration/utils/dao.js');
+const {getAssets} = require('./dao.js');
 
 module.exports = {
     test: {
-        testDuration: 60000,
+        testDuration: 6000,
         requestDelay: 0,
-        languages: ['fr', 'en', 'pr', 'it', 'ge']
+        languages: ['fr'/*, 'en', 'ge'*/],
     },
     setup: {
         resolver: getAssets,
         uniqueParams: ['language'],
         cache: { limit: 60000, steps: 5, base: 5000 },
-        batch: { tick: 2, limit: 50 },
+        batch: { tick: 10, max: 50 },
         retry: { base: 1, step: 2 },
     },
     assert: {
@@ -19,5 +19,6 @@ module.exports = {
         timeouts: [500, 4000],
         batches: [20000, 60000],
         rss: [30000, 60000],
+        avgBatchSize: [5, 50],
     },
 }
\ No newline at end of file
diff --git a/tests/profiling/testApp.js b/tests/profiling/testApp.js
new file mode 100644
index 0000000..feb581d
--- /dev/null
+++ b/tests/profiling/testApp.js
@@ -0,0 +1,68 @@
+/**
+ * Test app worker - allows us to saturate the request generator without impacting the app
+ */
+
+/* Requires ------------------------------------------------------------------*/
+
+const { parentPort } = require('worker_threads');
+const settings = require('./settings');
+const HA = require('../../src/index.js');
+
+/* Local variables -----------------------------------------------------------*/
+
+let handbreak = false;
+
+const suite = {
+    completed: 0,
+    cacheHits: 0,
+    sum: 0,
+    timeouts: 0,
+    batches: 0,
+    avgBatchSize: 0,
+    startHeap: process.memoryUsage().rss,
+};
+
+const store = HA(settings.setup);
+
+/* Methods -------------------------------------------------------------------*/
+
+function handleRequest(id, language) {
+    const randomError = (Math.round(Math.random()*10) === 0);
+    let finished = false;
+    const before = Date.now();
+    setTimeout(() => {
+      if (finished === false) suite.timeouts++;
+    }, 500);
+    if (randomError) store.config.resolver = getErroredRequest;
+    else store.config.resolver = getAssets;
+    store.get(id, { language }, crypto.randomBytes(8).toString('hex'))
+    .then((result) => {
+        finished = true;
+        if (handbreak) return;
+        if (!result || result.id !== id || result.language !== language) {
+        console.log(result, 'does not match', id, language);
+        throw new Error('integrity test failed');
+        }
+        suite.sum += (Date.now() - before);
+        suite.completed++;
+    }, () => {})
+    .catch((err) => { console.log(err); process.exit(1)} );
+}
+
+store.on('query', batch => { suite.batches++; suite.avgBatchSize += batch.ids.length; });
+store.on('cacheHit', () => { suite.cacheHits++; });
+
+//End
+function complete() {
+    handbreak = true;
+    suite.avgBatchSize = Math.round(suite.avgBatchSize / suite.batches);
+    
+    parentPort.postMessage(suite);
+}
+
+/* Init ----------------------------------------------------------------------*/
+
+parentPort.on('message', (msg) => {
+    if (msg.event && msg.event === 'finish') complete();
+    else handleRequest(msg.id, msg.language);
+});

From 4ef9457d33c035eed7b8ac3e94a54992f8ce50af Mon Sep 17 00:00:00 2001
From: Frederic Charette <fcharette@shutterstock.com>
Date: Thu, 28 Feb 2019 16:10:54 -0500
Subject: [PATCH 2/5] fixed bugs, typos

---
 tests/integration/batch.js  |  2 +-
 tests/profiling/index.js    | 54 +++++++++++++++++++------------------
 tests/profiling/settings.js | 16 +++++------
 tests/profiling/testApp.js  | 14 +++++-----
 4 files changed, 44 insertions(+), 42 deletions(-)

diff --git a/tests/integration/batch.js b/tests/integration/batch.js
index 8609d7f..900bcb1 100644
--- a/tests/integration/batch.js
+++ b/tests/integration/batch.js
@@ -108,7 +108,7 @@ describe('Batching', () => {
         });
     });
 
-    it('should properly bucket very large requests (optimal back size)', () => {
+    it('should properly bucket very large requests (optimal batch size)', () => {
       testStore.config.batch = { max: 6, tick: 1 };
       testStore.get(['foo2', 'bar2', 'abc2', 'def2', 'ghi2']);
       return testStore.get(['foo', 'bar', 'abc', 'def', 'ghi'], { language: 'en' })
diff --git a/tests/profiling/index.js b/tests/profiling/index.js
index 24c990b..a84290b 100644
--- a/tests/profiling/index.js
+++ b/tests/profiling/index.js
@@ -7,48 +7,50 @@
 
 const crypto = require('crypto');
 const settings = require('./settings');
-const { Worker } = require('worker_threads');
+const {fork} = require('child_process');
+const path = require('path');
 
 /* Init ----------------------------------------------------------------------*/
 
 // Setup
-const app = new Worker('./testApp.js');
-const now = Date.now();
+const app = fork(path.resolve(__dirname, './testApp.js'));
+const startTime = Date.now();
 
 async function hitStore() {
-  if (Date.now() - now <  settings.test.testDuration) {
+  if (Date.now() - startTime <  settings.test.testDuration) {
     process.nextTick(hitStore);
     
     // Simulate normal z-distribution
     let sampleRange = (Math.round(Math.random()*3) === 0) ? 1:2;
     const id = crypto.randomBytes(sampleRange).toString('hex');
     const language = settings.test.languages[Math.floor(Math.random()*settings.test.languages.length)];
-    app.postMessage({ id, language });
+    app.send({ id, language });
   }
   else {
-    app.postMessage({ event: 'finish' });
-    app.on('message', (suite) => {
-      console.log(`
-        ${suite.completed} completed requests
-        ${suite.cacheHits} cache hits
-        ${JSON.stringify(await store.size())}
-        ${suite.timeouts} timed out
-        avg response time ${(suite.sum / suite.completed).toFixed(3)}
-        ${suite.batches} queries sent
-        ${suite.avgBatchSize} items per queries on average
-        ${((process.memoryUsage().rss - suite.startHeap) / 1024).toFixed(2)} Kbytes allocated
-      `);
-
-      for (const expectation in settings.assert) {
-        if (suite[expectation] <= settings.assert[expectation][0] || suite[expectation] >= settings.assert[expectation][1]) {
-          console.log(expectation, 'did not match expectation', settings.assert[expectation]);
-          throw new Error('performance test failed');
-        }
-      }
-      process.exit(0);
-    });
+    app.send('finish');
   }
 }
 
+app.on('message', async (suite) => {
+  console.log(`
+    ${suite.completed} completed requests
+    ${suite.cacheHits} cache hits
+    ${JSON.stringify(suite.size)} in memory
+    ${suite.timeouts} timed out
+    avg response time ${(suite.sum / suite.completed).toFixed(3)}
+    ${suite.batches} queries sent
+    ${suite.avgBatchSize} items per queries on average
+    ${(suite.startHeap / 1024).toFixed(2)} Kbytes allocated
+  `);
+
+  for (const expectation in settings.assert) {
+    if (suite[expectation] <= settings.assert[expectation][0] || suite[expectation] >= settings.assert[expectation][1]) {
+      console.log(expectation, 'did not match expectation', settings.assert[expectation]);
+      throw new Error('performance test failed');
+    }
+  }
+  process.exit(0);
+});
+
 hitStore();
 
diff --git a/tests/profiling/settings.js b/tests/profiling/settings.js
index ed5cb96..d8b8f0b 100644
--- a/tests/profiling/settings.js
+++ b/tests/profiling/settings.js
@@ -2,9 +2,9 @@ const {getAssets} = require('./dao.js');
 
 module.exports = {
     test: {
-        testDuration: 6000,
+        testDuration: 1000,
         requestDelay: 0,
-        languages: ['fr'/*, 'en', 'ge'*/],
+        languages: ['fr', 'en', 'ge', 'it', 'pr'],
     },
     setup: {
         resolver: getAssets,
@@ -14,11 +14,11 @@ module.exports = {
         retry: { base: 1, step: 2 },
     },
     assert: {
-        completed: [30000, 100000],
-        cacheHits: [1000, 4000],
-        timeouts: [500, 4000],
-        batches: [20000, 60000],
-        rss: [30000, 60000],
-        avgBatchSize: [5, 50],
+        completed: [100000, 200000],
+        cacheHits: [40000, 70000],
+        timeouts: [500, 8000],
+        batches: [500, 4000],
+        rss: [90000, 200000],
+        avgBatchSize: [30, 50],
     },
 }
\ No newline at end of file
diff --git a/tests/profiling/testApp.js b/tests/profiling/testApp.js
index feb581d..919b07a 100644
--- a/tests/profiling/testApp.js
+++ b/tests/profiling/testApp.js
@@ -4,9 +4,10 @@
 
 /* Requires ------------------------------------------------------------------*/
 
-const { parentPort } = require('worker_threads');
 const settings = require('./settings');
 const HA = require('../../src/index.js');
+const {getAssets,getErroredRequest} = require('./dao');
+const crypto = require('crypto');
 
 /* Local variables -----------------------------------------------------------*/
 
@@ -53,16 +54,15 @@ store.on('query', batch => { suite.batches++; suite.avgBatchSize += batch.ids.le
 store.on('cacheHit', () => { suite.cacheHits++; });
 
 //End
-function complete() {
+async function complete() {
     handbreak = true;
     suite.avgBatchSize = Math.round(suite.avgBatchSize / suite.batches);
+    suite.size = await store.size();
+    suite.startHeap = process.memoryUsage().rss - suite.startHeap;
     
-    parentPort.postMessage(suite);
+    process.send(suite);
 }
 
 /* Init ----------------------------------------------------------------------*/
 
-parentPort.on('message', (msg) => {
-    if (msg.event && msg.event === 'finish') complete();
-    else handleRequest(msg.id, msg.language);
-});
+process.on('message', (msg) => msg === 'finish' ? complete() : handleRequest(msg.id, msg.language));

From 21b8c5dc309d839ac87523bc13a4a4bc3dd95f71 Mon Sep 17 00:00:00 2001
From: Frederic Charette <fcharette@shutterstock.com>
Date: Fri, 22 Mar 2019 15:59:30 -0400
Subject: [PATCH 3/5] merged next

---
 tests/profiling/index.js | 27 ---------------------------
 1 file changed, 27 deletions(-)

diff --git a/tests/profiling/index.js b/tests/profiling/index.js
index 03b7de1..a84290b 100644
--- a/tests/profiling/index.js
+++ b/tests/profiling/index.js
@@ -13,7 +13,6 @@ const path = require('path');
 /* Init ----------------------------------------------------------------------*/
 
 // Setup
-<<<<<<< HEAD
 const app = fork(path.resolve(__dirname, './testApp.js'));
 const startTime = Date.now();
 
@@ -21,32 +20,6 @@ async function hitStore() {
   if (Date.now() - startTime <  settings.test.testDuration) {
     process.nextTick(hitStore);
     
-=======
-const store = HA(settings.setup);
-const startTime = Date.now();
-
-// Suite
-const suite = {
-  sampleRange: 2,
-  completed: 0,
-  cacheHits: 0,
-  sum: 0,
-  timeouts: 0,
-  batches: 0,
-  startHeap: process.memoryUsage().rss,
-};
-
-store.on('query', () => { suite.batches++; });
-store.on('cacheHit', () => { suite.cacheHits++; });
-
-async function hitStore() {
-  if (Date.now() - startTime <  settings.test.testDuration) {
-    setTimeout(hitStore, settings.test.requestDelay);
-    let finished = false;
-    setTimeout(() => {
-      if (finished === false) suite.timeouts++;
-    }, 500);
->>>>>>> dbb8bdae167b20efe994bc765b4a92ba38186cc8
     // Simulate normal z-distribution
     let sampleRange = (Math.round(Math.random()*3) === 0) ? 1:2;
     const id = crypto.randomBytes(sampleRange).toString('hex');

From 725a0b1cbdd63939ce20f032cc0508a58227ee52 Mon Sep 17 00:00:00 2001
From: Frederic Charette <fcharette@shutterstock.com>
Date: Fri, 22 Mar 2019 16:06:42 -0400
Subject: [PATCH 4/5] reduced cacheHit requirements

---
 tests/profiling/settings.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/profiling/settings.js b/tests/profiling/settings.js
index d8b8f0b..aca4c9b 100644
--- a/tests/profiling/settings.js
+++ b/tests/profiling/settings.js
@@ -15,7 +15,7 @@ module.exports = {
     },
     assert: {
         completed: [100000, 200000],
-        cacheHits: [40000, 70000],
+        cacheHits: [25000, 70000],
         timeouts: [500, 8000],
         batches: [500, 4000],
         rss: [90000, 200000],

From 861220eaedaa3eb5c9b787782b818f2d416051f2 Mon Sep 17 00:00:00 2001
From: Frederic Charette <fcharette@shutterstock.com>
Date: Fri, 22 Mar 2019 16:13:39 -0400
Subject: [PATCH 5/5] fixed hanging travis build

---
 tests/profiling/index.js | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tests/profiling/index.js b/tests/profiling/index.js
index a84290b..d2dbaba 100644
--- a/tests/profiling/index.js
+++ b/tests/profiling/index.js
@@ -44,9 +44,9 @@ app.on('message', async (suite) => {
   `);
 
   for (const expectation in settings.assert) {
-    if (suite[expectation] <= settings.assert[expectation][0] || suite[expectation] >= settings.assert[expectation][1]) {
-      console.log(expectation, 'did not match expectation', settings.assert[expectation]);
-      throw new Error('performance test failed');
+    if (suite[expectation] < settings.assert[expectation][0] || suite[expectation] > settings.assert[expectation][1]) {
+      console.error(new Error(`Performance test failed: ${expectation} did not match expectation ${settings.assert[expectation]}`));
+      process.exit(1);
     }
   }
   process.exit(0);