From eebc0902909323475fb6ea18880acbe4ac25408f Mon Sep 17 00:00:00 2001 From: Mathieu St-Vincent Date: Tue, 2 Jun 2020 16:27:49 -0400 Subject: [PATCH 1/5] Added missing array types on HAStore.get ids --- src/index.d.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/index.d.ts b/src/index.d.ts index dc65e67..d9cd889 100644 --- a/src/index.d.ts +++ b/src/index.d.ts @@ -36,7 +36,7 @@ export interface HAStoreConfig { } export interface HAStore extends EventEmitter { - get(ids: string | number, params?: Params, context?: Serializable): Promise + get(ids: string | number | Array, params?: Params, context?: Serializable): Promise set(items: Serializable, ids: string[] | number[], params?: Params): Promise clear(ids: RequestIds, params?: Params): void size(): { contexts: number, queries: number, records: number } From accca1af408d2e80d967e8e85036a82689dd9d56 Mon Sep 17 00:00:00 2001 From: Mathieu St-Vincent <6082025+mats852@users.noreply.github.com> Date: Sat, 5 Sep 2020 23:23:25 -0400 Subject: [PATCH 2/5] Fixed resolver name in README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 84ec348..273a3cc 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,7 @@ const itemStore = store({ }); // Define your resolver -function getItem(ids, params, contexts) { +function getItems(ids, params, contexts) { // Ids will be a list of all the unique requested items // Params will be the parameters for the request, which must be declared in the `uniqueParams` config of the store // Contexts will be the list of originating context information From 6717087f4f97ea4b6fe37d988f08c1048f6be10e Mon Sep 17 00:00:00 2001 From: Frederic Charette Date: Thu, 1 Oct 2020 12:37:46 -0400 Subject: [PATCH 3/5] v2.0.6 (#104) * Update package.json * Update README.md * Update LICENSE --- LICENSE | 2 +- README.md | 4 ++-- package.json | 11 ++--------- 3 files changed, 5 insertions(+), 12 deletions(-) diff --git a/LICENSE b/LICENSE index a2c0742..c6746cb 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ -Copyright 2019 Frederic Charette +Copyright 2020 Frederic Charette Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/README.md b/README.md index 273a3cc..fbc733d 100644 --- a/README.md +++ b/README.md @@ -70,7 +70,7 @@ resolver | true | - | The method to wrap, and how to interpret the returned data responseParser | false | (system) | The method that format the results from the resolver into an indexed collection. Accepts indexed collections or arrays of objects with an `id` property. Uses the format `` uniqueParams | false | `[]` | The list of parameters that, when passed, generate unique results. Ex: 'language', 'view', 'fields', 'country'. These will generate different combinations of cache keys. store | false | `null` | A custom store for the data, like [ha-store-redis](https://github.com/fed135/ha-redis-adapter). -cache | false | 
{
  limit: 5000,
  ttl: 300000
}
| Caching options for the data - `limit` - the maximum number of records, and `ttl` - time to live for a record. +cache | false | 
{
  limit: 5000,
  ttl: 300000
}
| Caching options for the data - `limit` - the maximum number of records, and `ttl` - time to live for a record in milliseconds. batch | false | 
{
  tick: 50,
  max: 100
}
| Batching options for the requests *All options are in (ms) @@ -112,5 +112,5 @@ I am always looking for more maintainers, as well. ## License -[Apache 2.0](LICENSE) (c) 2019 Frederic Charette +[Apache 2.0](LICENSE) (c) 2020 Frederic Charette diff --git a/package.json b/package.json index e218242..2aaf651 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "ha-store", - "version": "2.0.5", + "version": "2.0.6", "description": "Efficient data fetching", "main": "src/index.js", "scripts": { @@ -18,20 +18,13 @@ }, "keywords": [ "store", - "high", "availability", - "network", "optimize", "throughput", - "retry", - "request", "cache", "service", "batch", - "micro", - "latency", "congestion", - "control", "tlru" ], "bugs": { @@ -54,6 +47,6 @@ ], "typings": "./src/index.d.ts", "dependencies": { - "lru-native2": "^1.2.0" + "lru-native2": "^1.2.2" } } From 4ff195164d97c6ab4ba382c3c4df5ce23eef114f Mon Sep 17 00:00:00 2001 From: Frederic Charette Date: Wed, 26 May 2021 15:31:42 -0400 Subject: [PATCH 4/5] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index fbc733d..f0a7684 100644 --- a/README.md +++ b/README.md @@ -112,5 +112,5 @@ I am always looking for more maintainers, as well. ## License -[Apache 2.0](LICENSE) (c) 2020 Frederic Charette +[Apache 2.0](LICENSE) (c) 2021 Frederic Charette From 97c78d180b9049bc4657eeb5b8c8b055933475e3 Mon Sep 17 00:00:00 2001 From: Frederic Charette Date: Mon, 12 Jul 2021 17:32:29 -0400 Subject: [PATCH 5/5] V3.0.0 (#106) * Split get interface for many, using Promise.allSettled, removed binds, removed parsers, comments * fixed linting, unit tests * fixed tests * deprecated node 12, fixed bench in github actions * deprecated node 12 * fixed github action bench * fixed node version support tag in readme * Update src/index.js Co-authored-by: Mathieu St-Vincent <6082025+mats852@users.noreply.github.com> * Update src/index.js Co-authored-by: Mathieu St-Vincent <6082025+mats852@users.noreply.github.com> * Update src/queries.js Co-authored-by: Mathieu St-Vincent <6082025+mats852@users.noreply.github.com> * added extra test for mixed responses Co-authored-by: Frederic Charette Co-authored-by: Mathieu St-Vincent <6082025+mats852@users.noreply.github.com> --- .eslintrc | 8 +- .github/workflows/pr-validation.yml | 30 +++++ .travis.yml | 16 --- LICENSE | 2 +- README.md | 22 ++-- package.json | 20 +-- src/index.d.ts | 38 +++--- src/index.js | 72 ++++------- src/options.js | 12 +- src/queries.js | 183 ++++++++++++++-------------- src/store.js | 14 --- src/utils.js | 59 +-------- tests/integration/batch.js | 165 +++++++++++++++---------- tests/integration/cache.js | 56 ++++----- tests/integration/utils/dao.js | 34 ++++-- tests/profiling/dao.js | 36 +++--- tests/profiling/index.js | 6 +- tests/profiling/settings.js | 40 +++--- tests/profiling/worker.js | 84 +++++-------- tests/unit/index.spec.js | 28 ++--- tests/unit/options.spec.js | 12 +- tests/unit/utils.spec.js | 15 --- 22 files changed, 439 insertions(+), 513 deletions(-) create mode 100644 .github/workflows/pr-validation.yml delete mode 100644 .travis.yml diff --git a/.eslintrc b/.eslintrc index a01fe60..53131aa 100644 --- a/.eslintrc +++ b/.eslintrc @@ -1,6 +1,7 @@ { + "extends": "eslint:recommended", "parserOptions": { - "ecmaVersion": 9 + "ecmaVersion": 11 }, "rules": { "arrow-body-style": "off", @@ -62,5 +63,10 @@ ], "linebreak-style": "off", "no-lonely-if": "off" + }, + "env": { + "node": true, + "mocha": true, + "es6": true } } diff --git a/.github/workflows/pr-validation.yml b/.github/workflows/pr-validation.yml new file mode 100644 index 0000000..b7c58db --- /dev/null +++ b/.github/workflows/pr-validation.yml @@ -0,0 +1,30 @@ +name: pr-validation + +on: + pull_request: + branches: + - '*' + +jobs: + build: + + runs-on: ubuntu-latest + + strategy: + matrix: + node-version: [14.x, 16.x] + + steps: + - uses: actions/checkout@v1 + - name: Use Node.js ${{ matrix.node-version }} + uses: actions/setup-node@v1 + with: + node-version: ${{ matrix.node-version }} + - name: npm install, build, and test + run: | + npm install + npm run lint + npm run test + wget --no-check-certificate --content-disposition https://gist.githubusercontent.com/fed135/56282783a4c13d87a7f89c178b5d08d7/raw/bacd6f814475726cee37a5a591cb313dd1542d2b/sample.txt && npm run bench + env: + CI: true diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 51e123e..0000000 --- a/.travis.yml +++ /dev/null @@ -1,16 +0,0 @@ -language: node_js -node_js: -- '10' -- '8' -sudo: false -script: npm test -jobs: - include: - - stage: npm release - node_js: '10' - deploy: - provider: npm - email: fredericcharette@gmail.com - api_key: - secure: LeWowKy1A0KIOz3Igq8WNoDxqrhguENWMtNUPSvdf2Un9PA8CDho3f7+SkaN7l74kFQxdKZGRNyC2D9r354wq3RSgGS2xUgKwv1nv/r7FR/oqlAfc8kNXVzf7IKItw9aUJbUuhCRi4qNdtrfcdUGmB8oNJ6NxlcQA2BigLGzmxO1zho0byqdKjmnrcAMYYRN8+hbn+YCnRuSpQP6fsh8oEhm4YWH19yk6l3rJfEjBD0Mv8GcGjxVCzEUmGov+cCIcjECka5UqoVrkoB4q6vC8ZpwR7h5aPS8yoX69+wKrj2MGdCz096d2pTXfwu170C+Ass5yRBa0CY5hfMhL5UwOf6u4tHF4VhaQ561oQo+XINKC7JP7LRnag79ToHJgThOUNBj2OOBCtZ60MqtqVJzOd5bx0wXF643MZfTZkVq3Kmy5Dnr90SrHeboN6qksw4fDBpp01sgWLnXT/ZuuC509PVyt74H/7lyrSkJxK7o4xzFihsqoaIXsPAqbtZvSS4mrJ1zO/LI1LYDDmlrxQ5sXcehCkPYylcgVyJMLZWWXMLU162ym66BsljwNjLzoVt5XOsLjJ7C6inY90hlgWpBJn35iUAP2Q5ZNLuJMUygmCaT8sH0fN7COaDCPoC0oHA3gEvPmd2NyhfkLuxNdQeiq7Y72K/ahX4poNCXVrbXntI= - if: branch = master diff --git a/LICENSE b/LICENSE index c6746cb..b7c7db1 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ -Copyright 2020 Frederic Charette +Copyright 2021 Frederic Charette Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/README.md b/README.md index f0a7684..e67d873 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@
[![ha-store](https://img.shields.io/npm/v/ha-store.svg)](https://www.npmjs.com/package/ha-store) -[![Node](https://img.shields.io/badge/node->%3D8.0-blue.svg)](https://nodejs.org) +[![Node](https://img.shields.io/badge/node->%3D14.0-blue.svg)](https://nodejs.org) [![Build Status](https://travis-ci.org/fed135/ha-store.svg?branch=master)](https://travis-ci.org/fed135/ha-store) [![Dependencies Status](https://david-dm.org/fed135/ha-store.svg)](https://david-dm.org/fed135/ha-store) @@ -36,29 +36,28 @@ Learn how you can improve your app's performance, design and resiliancy [here](h const store = require('ha-store'); const itemStore = store({ resolver: getItems, - uniqueParams: ['language'] + delimiter: ['language'] }); // Define your resolver function getItems(ids, params, contexts) { // Ids will be a list of all the unique requested items - // Params will be the parameters for the request, which must be declared in the `uniqueParams` config of the store + // Params will be the parameters for the request, which must be declared in the `delimiter` config of the store // Contexts will be the list of originating context information // Now perform some exensive network call or database lookup... - // Then, respond with your data formatted into one of these two formats: - // a) [ { id: '123', language: 'fr', name: 'fred' } ] - // b) { '123': { language: 'fr', name: 'fred' } } + // Then, respond with your data formatted into this formats: + // { '123': { language: 'fr', name: 'fred' } } } // Now to use your store -itemStore.get('123', { language: 'fr' }, { some: 'context' }) +itemStore.get('123', { language: 'fr' }, { requestId: '123' }) .then(item => /* The item you requested */); // You can even ask for more than one item at a time -itemStore.get(['123', '456'], { language: 'en' }, { another: 'context' }) - .then(items => /* All the items you requested */); +itemStore.getMany(['123', '456'], { language: 'en' }, { requestId: '123' }) + .then(items => /* All the items you requested, in Promise.allSettled fashion */); ``` @@ -67,11 +66,10 @@ itemStore.get(['123', '456'], { language: 'en' }, { another: 'context' }) Name | Required | Default | Description --- | --- | --- | --- resolver | true | - | The method to wrap, and how to interpret the returned data. Uses the format `` -responseParser | false | (system) | The method that format the results from the resolver into an indexed collection. Accepts indexed collections or arrays of objects with an `id` property. Uses the format `` -uniqueParams | false | `[]` | The list of parameters that, when passed, generate unique results. Ex: 'language', 'view', 'fields', 'country'. These will generate different combinations of cache keys. +delimiter | false | `[]` | The list of parameters that, when passed, generate unique results. Ex: 'language', 'view', 'fields', 'country'. These will generate different combinations of cache keys. store | false | `null` | A custom store for the data, like [ha-store-redis](https://github.com/fed135/ha-redis-adapter). cache | false | 
{
  limit: 5000,
  ttl: 300000
}
| Caching options for the data - `limit` - the maximum number of records, and `ttl` - time to live for a record in milliseconds. -batch | false | 
{
  tick: 50,
  max: 100
}
| Batching options for the requests +batch | false | 
{
  delay: 50,
  limit: 100
}
| Batching options for the requests *All options are in (ms) diff --git a/package.json b/package.json index 2aaf651..9dbefdd 100644 --- a/package.json +++ b/package.json @@ -1,16 +1,18 @@ { "name": "ha-store", - "version": "2.0.6", + "version": "3.0.0", "description": "Efficient data fetching", "main": "src/index.js", "scripts": { + "lint": "eslint .", + "lint:fix": "yarn lint --fix", "test": "npm run test:unit && npm run test:integration", "test:unit": "mocha ./tests/unit --exit", "test:integration": "mocha ./tests/integration --exit", "bench": "node ./tests/profiling/index.js" }, "engines": { - "node": ">=8.0.0" + "node": ">=14.0.0" }, "repository": { "type": "git", @@ -33,12 +35,12 @@ "author": "frederic charette ", "license": "Apache-2.0", "devDependencies": { - "chai": "^4.2.0", - "heapdump": "^0.3.12", - "mocha": "^6.0.0", - "sinon": "^7.2.0", - "split2": "^3.1.1", - "ha-store-redis": "^2.0.1" + "chai": "^4.3.0", + "eslint": "^7.29.0", + "ha-store-redis": "^2.0.1", + "mocha": "^9.0.0", + "sinon": "^11.1.0", + "split2": "^3.2.0" }, "contributors": [ "frederic charette ", @@ -47,6 +49,6 @@ ], "typings": "./src/index.d.ts", "dependencies": { - "lru-native2": "^1.2.2" + "lru-native2": "^1.2.5" } } diff --git a/src/index.d.ts b/src/index.d.ts index d9cd889..16a6e57 100644 --- a/src/index.d.ts +++ b/src/index.d.ts @@ -4,43 +4,39 @@ type Params = { [key: string]: string } -type RequestIds = string | number | string[] | number[] -type Serializable = string | number | boolean | { [key: string]: Serializable } | Array - export interface HAExternalStore { - get: (key: string) => Promise - getMulti: (recordKey: (contextKey: string) => string, keys: RequestIds) => Promise - set: (recordKey: (contextKey: string) => string, keys: RequestIds, values: Serializable) => Promise - clear: (key?: string) => boolean - size: () => number + get(key: string): Promise + getMulti(recordKey: (contextKey: string) => string, keys: RequestIds): Promise + set(recordKey: (contextKey: string) => string, keys: RequestIds, values: DataType): boolean + clear(key?: string): boolean + size(): number connection?: any } export interface HAStoreConfig { - resolver(ids: RequestIds, params?: Params, context?: Serializable): Promise - uniqueParams?: string[] - responseParser?( - response: Serializable, - requestedIds: string[] | number[], - params?: Params - ): object + resolver(ids: string[], params?: Params): Promise<{ [id: string]: Response }> + resolver(ids: string[], params?: Params, context?: Context): Promise<{ [id: string]: Response }> + delimiter?: string[] cache?: { limit?: number ttl?: number } batch?: { - tick?: number - max?: number + delay?: number + limit?: number }, store?: HAExternalStore } export interface HAStore extends EventEmitter { - get(ids: string | number | Array, params?: Params, context?: Serializable): Promise - set(items: Serializable, ids: string[] | number[], params?: Params): Promise - clear(ids: RequestIds, params?: Params): void + get(id: string, params?: Params): Promise + get(id: string, params?: Params, context?: Context): Promise + getMany(id: string[], params?: Params): Promise<{status: string, value: Response}[]> + getMany(id: string[], params?: Params, context?: Context): Promise<{status: string, value: Response}[]> + set(items: { [id: string]: any }, ids: string[], params?: Params): boolean + clear(ids: string[], params?: Params): void size(): { contexts: number, queries: number, records: number } - getKey(id: string | number, params?: Params): string + getStorageKey(id: string, params?: Params): string } export default function batcher(config: HAStoreConfig, emitter?: EventEmitter): HAStore diff --git a/src/index.js b/src/index.js index ea1e546..54be2d5 100644 --- a/src/index.js +++ b/src/index.js @@ -1,11 +1,3 @@ -/** - * Batcher index - */ - -'use strict'; - -/* Requires ------------------------------------------------------------------*/ - const queue = require('./queries.js'); const {contextKey, recordKey, contextRecordKey} = require('./utils.js'); const EventEmitter = require('events').EventEmitter; @@ -17,18 +9,16 @@ class HaStore extends EventEmitter { constructor(initialConfig, emitter) { super(); - // Parameter validation if (typeof initialConfig.resolver !== 'function') { throw new Error(`config.resolver [${initialConfig.resolver}] is not a function`); } - if (emitter && !emitter.emit) { + if (!emitter?.emit) { throw new Error(`${emitter} is not an EventEmitter`); } this.config = hydrateConfig(initialConfig); - // Local variables if (this.setMaxListeners) { this.setMaxListeners(Infinity); } @@ -38,57 +28,43 @@ class HaStore extends EventEmitter { this.queue = queue( this.config, this, - this.store, + this.store ); } - /** - * Gets a list of records from source - * @param {string|number|array} ids The id of the record to fetch - * @param {object} params (Optional)The Request parameters - * @returns {Promise} The eventual single record - */ - async get(ids, params = {}, agg = null) { + get(id, params = {}, agg = null) { + if (params === null) params = {}; + const key = contextKey(this.config.delimiter, params); + return this.queue.getHandles(key, [id], params, agg) + .then(handles => handles[0]); + } + + getMany(ids, params = {}, agg = null) { if (params === null) params = {}; - const requestIds = (Array.isArray(ids)) ? ids : [ids]; - const key = contextKey(this.config.uniqueParams, params); - const handles = await this.queue.getHandles(key, requestIds, params, agg); - return Promise.all(handles) - .then(response => (!Array.isArray(ids)) ? response[0] : response); + const key = contextKey(this.config.delimiter, params); + return this.queue.getHandles(key, ids, params, agg) + .then((handles) => Promise.allSettled(handles) + .then((outcomes) => ids.reduce((handles, id, index) => { + handles[id] = outcomes[index]; + return handles; + }, {}))); } - /** - * Inserts results into cache manually - * @param {*} items Raw results from a data-source to load into cache - * @param {array} ids The id(s) to extract from the raw dataset - * @param {object} params (Optional)The Request parameters - * @returns {Promise} The eventual single record - */ set(items, ids, params = {}) { if (!Array.isArray(ids) || ids.length === 0) throw new Error('Missing required argument id list in batcher #set. '); - const key = contextKey(this.config.uniqueParams, params); + const key = contextKey(this.config.delimiter, params); return this.store.set(contextRecordKey(key), ids, items); } - /** - * Clears one or more recors from temp store - * @param {string|number|array} ids The id(s) to clear - * @param {object} params (Optional) The Request parameters - * @returns {boolean} The result of the clearing - */ clear(ids, params) { if (this.store === null) return true; if (Array.isArray(ids)) { return ids.map(id => this.clear(id, params)); } - return this.store.clear(this.getKey(ids, params)); + return this.store.clear(this.getStorageKey(ids, params)); } - /** - * Returns the amount of records and contexts in memory - * @returns {object} - */ size() { return { ...this.queue.size(), @@ -96,14 +72,8 @@ class HaStore extends EventEmitter { }; } - /** - * Returns a record key - * @param {string|number} id The id of the item - * @param {object} params The parameters for the request - * @returns {string} The record key - */ - getKey(id, params) { - return recordKey(contextKey(this.config.uniqueParams, params), id); + getStorageKey(id, params) { + return recordKey(contextKey(this.config.delimiter, params), id); } } diff --git a/src/options.js b/src/options.js index fad82e2..b6706c0 100644 --- a/src/options.js +++ b/src/options.js @@ -1,19 +1,11 @@ -/** - * Options - */ - -'use strict'; - -/* Requires ------------------------------------------------------------------*/ - const store = require('./store.js'); /* Local variables -----------------------------------------------------------*/ const defaultConfig = { batch: { - tick: 50, - max: 100, + delay: 50, + limit: 100, }, cache: { limit: 5000, diff --git a/src/queries.js b/src/queries.js index 0e34f97..17131e1 100644 --- a/src/queries.js +++ b/src/queries.js @@ -1,109 +1,108 @@ -const {basicParser, contextRecordKey, deferred} = require('./utils'); +const {contextRecordKey, deferred} = require('./utils'); function queriesStore(config, emitter, targetStore) { - const queries = {}; + const queries = {}; - async function getHandles(key, ids, params, context) { - if (!(key in queries)) queries[key] = []; + async function getHandles(key, ids, params, context) { + if (!(key in queries)) queries[key] = []; - let numCoalesced = 0; - let numCached = 0; - let numMisses = 0; + let numCoalesced = 0; + let numCached = 0; + let numMisses = 0; - const handles = []; - for (let i = 0; i < ids.length; i++) { - handles[i] = undefined; + const handles = []; + for (let i = 0; i < ids.length; i++) { + handles[i] = undefined; - const existingQuery = queries[key].find(query => query.handles.has(ids[i])); - if (existingQuery) { - numCoalesced++; - handles[i] = existingQuery.handles.get(ids[i]).promise; - } - } - - if (config.cache !== null) { - const cacheResult = await targetStore.getMulti(contextRecordKey(key), ids.map((id, i) => handles[i] === undefined ? id : undefined)); - for (let i = 0; i < cacheResult.length; i++) { - if (cacheResult[i] !== undefined) { - numCached++; - handles[i] = cacheResult[i]; - } - } - } - - for (let i = 0; i < ids.length; i++) { - if (handles[i] === undefined) { - numMisses++; - handles[i] = assignQuery(key, ids[i], params, context); - } - } - - if (numCached > 0) emitter.emit('cacheHit', { key, found: numCached }); - if (numMisses > 0) emitter.emit('cacheMiss', { key, found: numMisses }); - if (numCoalesced > 0) emitter.emit('coalescedHit', { key, found: numCoalesced }); - - return handles; - } - - function assignQuery(key, id, params, context) { - const sizeLimit = config.batch && config.batch.max || 1; - const query = queries[key].find(q => q.size < sizeLimit && q.state === 0) || createQuery(key, params); - query.size++; - if (!query.handles.has(id)) query.handles.set(id, deferred()); - if (query.contexts.indexOf(context) == -1) query.contexts.push(context); - - if (query.size >= sizeLimit) runQuery(query); - return query.handles.get(id).promise; + const existingQuery = queries[key].find(query => query.handles.has(ids[i])); + if (existingQuery) { + numCoalesced++; + handles[i] = existingQuery.handles.get(ids[i]).promise; + } } - function createQuery(key, params) { - const query = { uid: Math.random().toString(36), key, params, handles: new Map(), state: 0, timer: null, contexts: [], size: 0 }; - queries[key].push(query); - - query.timer = setTimeout(() => runQuery(query), config.batch && config.batch.tick || 0); - return query; - } - - function deleteQuery(key, uid) { - const index = (queries[key] || []).findIndex(query => query.uid === uid); - if (index > -1) queries[key].splice(index, 1); - if (queries[key].length === 0) delete queries[key]; - } - - async function runQuery(query) { - query.state = 1; - clearTimeout(query.timer); - emitter.emit('query', query); - config.resolver(Array.from(query.handles.keys()), query.params, query.contexts) - .then(handleQuerySuccess.bind(null, query), handleQueryError.bind(null, query)); - } - - function handleQueryError(query, error) { - query.state = 2; - emitter.emit('queryFailed', { key: query.key, uid: query.uid, size: query.size, params: query.params, error }); - Array.from(query.handles.values()).forEach(handle => handle.reject(error)); - deleteQuery(query.key, query.uid); + if (config.cache !== null) { + const cacheResult = await targetStore.getMulti(contextRecordKey(key), ids.map((id, i) => handles[i] === undefined ? id : undefined)); + for (let i = 0; i < cacheResult.length; i++) { + if (cacheResult[i] !== undefined) { + numCached++; + handles[i] = cacheResult[i]; + } + } } - function handleQuerySuccess(query, rawResponse) { - query.state = 2; - emitter.emit('querySuccess', { key: query.key, uid: query.uid, size: query.size, params: query.params }); - const ids = Array.from(query.handles.keys()); - const entries = basicParser(rawResponse, ids, query.params); - ids.forEach(id => query.handles.get(id).resolve(entries[id])); - if (config.cache !== null) targetStore.set(contextRecordKey(query.key), ids, entries); - deleteQuery(query.key, query.uid); + for (let i = 0; i < ids.length; i++) { + if (handles[i] === undefined) { + numMisses++; + handles[i] = assignQuery(key, ids[i], params, context); + } } - function size() { - const contexts = Object.keys(queries); - return { - contexts: contexts.length, - queries: contexts.reduce((acc, curr) => acc + queries[curr].length, 0), - } + if (numCached > 0) emitter.emit('cacheHit', { key, found: numCached }); + if (numMisses > 0) emitter.emit('cacheMiss', { key, found: numMisses }); + if (numCoalesced > 0) emitter.emit('coalescedHit', { key, found: numCoalesced }); + + return handles; + } + + function assignQuery(key, id, params, context) { + const sizeLimit = config.batch?.limit || 1; + const query = queries[key].find(q => q.size < sizeLimit && q.state === 0) || createQuery(key, params); + query.size++; + if (!query.handles.has(id)) query.handles.set(id, deferred()); + if (query.contexts.indexOf(context) == -1) query.contexts.push(context); + + if (query.size >= sizeLimit) runQuery(query); + return query.handles.get(id).promise; + } + + function createQuery(key, params) { + const query = { uid: Math.random().toString(36), key, params, handles: new Map(), state: 0, timer: null, contexts: [], size: 0 }; + queries[key].push(query); + + query.timer = setTimeout(() => runQuery(query), config.batch?.delay || 0); + return query; + } + + function deleteQuery(key, uid) { + const index = (queries[key] || []).findIndex(query => query.uid === uid); + if (index > -1) queries[key].splice(index, 1); + if (queries[key].length === 0) delete queries[key]; + } + + function runQuery(query) { + query.state = 1; + clearTimeout(query.timer); + emitter.emit('query', query); + config.resolver(Array.from(query.handles.keys()), query.params, query.contexts) + .then((entries) => handleQuerySuccess(query, entries), (error) => handleQueryError(query, error)); + } + + function handleQueryError(query, error) { + query.state = 2; + emitter.emit('queryFailed', { key: query.key, uid: query.uid, size: query.size, params: query.params, error }); + Array.from(query.handles.values()).forEach(handle => handle.reject(error)); + deleteQuery(query.key, query.uid); + } + + function handleQuerySuccess(query, entries) { + query.state = 2; + emitter.emit('querySuccess', { key: query.key, uid: query.uid, size: query.size, params: query.params }); + const ids = Array.from(query.handles.keys()); + ids.forEach(id => query.handles.get(id).resolve(entries?.[id])); + if (config.cache !== null) targetStore.set(contextRecordKey(query.key), ids, entries || {}); + deleteQuery(query.key, query.uid); + } + + function size() { + const contexts = Object.keys(queries); + return { + contexts: contexts.length, + queries: contexts.reduce((acc, curr) => acc + queries[curr].length, 0), } + } - return { getHandles, size }; + return { getHandles, size }; } module.exports = queriesStore; diff --git a/src/store.js b/src/store.js index 0262e5c..9d24a62 100644 --- a/src/store.js +++ b/src/store.js @@ -1,11 +1,3 @@ -/** - * Record Store - */ - -'use strict'; - -/* Requires ------------------------------------------------------------------*/ - const lruNative = require('lru-native2'); /* Methods -------------------------------------------------------------------*/ @@ -28,12 +20,6 @@ function localStore(config) { }); } - /** - * Performs a query that returns a single entities to be cached - * @param {object} opts The options for the dao - * @param {string} method The dao method to call - * @returns {undefined} - */ function set(recordKey, keys, values) { for (let i = 0; i < keys.length; i++) { if (values[keys[i]] !== undefined && values[keys[i]] !== null) { diff --git a/src/utils.js b/src/utils.js index bc77842..efad69c 100644 --- a/src/utils.js +++ b/src/utils.js @@ -1,60 +1,3 @@ -/** - * Utilities - */ - -'use strict'; - -/* Methods -------------------------------------------------------------------*/ - -/** - * Parses the results from the data-source query - * @param {*} results The raw results - * @param {array} ids The list of ids to look for in the response - * @param {*} params The original parameters of the query - * @returns {object} The indexed result set found - */ -function basicParser(results, ids, params = {}) { - if (results === null || results === undefined) return {}; - ids = ids.map(id => `${id}`); - return Array.isArray(results) - ? arrayParser(results, ids, params) - : objectParser(results, ids, params); -} - -/** - * Parses the results from the data-source query - * @param {array} results The raw results - * @param {array} ids The list of ids to look for in the response - * @param {*} params The original parameters of the query - * @returns {object} The indexed result set found - */ -function arrayParser(results, ids, params = {}) { - return results.reduce((acc, curr) => { - if (!curr || !curr.id) return acc; - if (ids.includes(`${curr.id}`)) { - acc[curr.id] = curr; - } - return acc; - }, {}); -} - -/** - * Parses the results from the data-source query - * @param {object} results The raw results - * @param {array} ids The list of ids to look for in the response - * @param {*} params The original parameters of the query - * @returns {object} The indexed result set found - */ -function objectParser(results, ids, params = {}) { - const acc = {}; - for (let key in results) { - if (ids.includes(key) && results[key] !== null && results[key] !== undefined) { - acc[key] = results[key]; - } - } - return acc; -} - function deferred() { let resolve; let reject; @@ -74,4 +17,4 @@ const contextRecordKey = key => id => recordKey(key, id); /* Exports -------------------------------------------------------------------*/ -module.exports = { deferred, basicParser, contextKey, recordKey, contextRecordKey }; +module.exports = { deferred, contextKey, recordKey, contextRecordKey }; diff --git a/tests/integration/batch.js b/tests/integration/batch.js index 59856a5..c105567 100644 --- a/tests/integration/batch.js +++ b/tests/integration/batch.js @@ -22,7 +22,7 @@ describe('Batching', () => { beforeEach(() => { mockSource = sinon.mock(dao); testStore = store({ - uniqueParams: ['language'], + delimiter: ['language'], resolver: dao.getAssets, }); }); @@ -30,7 +30,7 @@ describe('Batching', () => { it('should batch single calls', () => { return Promise.all([ testStore.get('foo'), - testStore.get('abc') + testStore.get('abc'), ]) .then((result) => { expect(result).to.deep.equal([{ id: 'foo', language: undefined }, { id: 'abc', language: undefined }]); @@ -41,9 +41,9 @@ describe('Batching', () => { }); it('should batch multi calls', () => { - return testStore.get(['abc', 'foo']) + return testStore.getMany(['abc', 'foo']) .then((result) => { - expect(result).to.deep.equal([{ id: 'abc', language: undefined }, { id: 'foo', language: undefined }]); + expect(result).to.deep.equal({ abc: { status: 'fulfilled', value: { id: 'abc', language: undefined } }, foo: { status: 'fulfilled', value: { id: 'foo', language: undefined } } }); mockSource.expects('getAssets') .once() .withArgs(['foo', 'abc']); @@ -52,11 +52,11 @@ describe('Batching', () => { it('should batch mixed calls', () => { return Promise.all([ - testStore.get(['foo', 'bar']), - testStore.get('abc') + testStore.getMany(['foo', 'bar']), + testStore.get('abc'), ]) .then((result) => { - expect(result).to.deep.equal([[{ id: 'foo', language: undefined }, { id: 'bar', language: undefined }], { id: 'abc', language: undefined }]); + expect(result).to.deep.equal([{ bar: { status: 'fulfilled', value: { id: 'bar', language: undefined } }, foo: { status: 'fulfilled', value: { id: 'foo', language: undefined } } }, { id: 'abc', language: undefined }]); mockSource.expects('getAssets') .once() .withArgs(['foo', 'bar', 'abc']); @@ -65,11 +65,11 @@ describe('Batching', () => { it('should mix unique params matches', () => { return Promise.all([ - testStore.get(['foo', 'bar'], { language: 'fr' }), - testStore.get('abc', { language: 'fr' }) + testStore.getMany(['foo', 'bar'], { language: 'fr' }), + testStore.get('abc', { language: 'fr' }), ]) .then((result) => { - expect(result).to.deep.equal([[{ id: 'foo', language: 'fr' }, { id: 'bar', language: 'fr' }], { id: 'abc', language: 'fr' }]); + expect(result).to.deep.equal([{ bar: { status: 'fulfilled', value: { id: 'bar', language: 'fr' } }, foo: { status: 'fulfilled', value: { id: 'foo', language: 'fr' } } }, { id: 'abc', language: 'fr' }]); mockSource.expects('getAssets') .once().withArgs(['foo', 'bar', 'abc'], { language: 'fr' }); }); @@ -77,11 +77,11 @@ describe('Batching', () => { it('should not mix unique params mismatches', () => { return Promise.all([ - testStore.get(['foo', 'bar'], { language: 'fr' }), - testStore.get('abc', { language: 'en' }) + testStore.getMany(['foo', 'bar'], { language: 'fr' }), + testStore.get('abc', { language: 'en' }), ]) .then((result) => { - expect(result).to.deep.equal([[{ id: 'foo', language: 'fr' }, { id: 'bar', language: 'fr' }], { id: 'abc', language: 'en' }]); + expect(result).to.deep.equal([{ bar: { status: 'fulfilled', value: { id: 'bar', language: 'fr' } }, foo: { status: 'fulfilled', value: { id: 'foo', language: 'fr' } } }, { id: 'abc', language: 'en' }]); mockSource.expects('getAssets') .once().withArgs(['abc'], { language: 'en' }) .once().withArgs(['foo', 'bar'], { language: 'fr' }); @@ -91,7 +91,7 @@ describe('Batching', () => { it('should coalesce duplicate entries', () => { return Promise.all([ testStore.get('foo', { language: 'fr' }), - testStore.get('foo', { language: 'fr' }) + testStore.get('foo', { language: 'fr' }), ]) .then((result) => { expect(result).to.deep.equal([{ id: 'foo', language: 'fr' }, { id: 'foo', language: 'fr' }]); @@ -103,7 +103,7 @@ describe('Batching', () => { it('should maintain id ordering with numeric ids', () => { return Promise.all([ testStore.get(2, { language: 'fr' }), - testStore.get(1, { language: 'fr' }) + testStore.get(1, { language: 'fr' }), ]) .then((result) => { expect(result).to.deep.equal([{ id: 2, language: 'fr' }, { id: 1, language: 'fr' }]); @@ -113,16 +113,17 @@ describe('Batching', () => { }); it('should properly bucket large requests', () => { - testStore.config.batch = { max: 2, tick: 1 }; - return testStore.get(['foo', 'bar', 'abc', 'def', 'ghi'], { language: 'en' }) + testStore.config.batch = { limit: 2, delay: 1 }; + return testStore.getMany(['foo', 'bar', 'abc', 'def', 'ghi'], { language: 'en' }) .then((result) => { - expect(result).to.deep.equal([ - { id: 'foo', language: 'en' }, - { id: 'bar', language: 'en' }, - { id: 'abc', language: 'en' }, - { id: 'def', language: 'en' }, - { id: 'ghi', language: 'en' }, - ]); + expect(result).to.deep.equal({ + foo: { status: 'fulfilled', value: { id: 'foo', language: 'en' } }, + bar: { status: 'fulfilled', value: { id: 'bar', language: 'en' } }, + abc: { status: 'fulfilled', value: { id: 'abc', language: 'en' } }, + def: { status: 'fulfilled', value: { id: 'def', language: 'en' } }, + ghi: { status: 'fulfilled', value: { id: 'ghi', language: 'en' } }, + }); + mockSource.expects('getAssets') .once().withArgs(['foo', 'bar'], { language: 'en' }) .once().withArgs(['abc', 'def'], { language: 'en' }) @@ -131,25 +132,25 @@ describe('Batching', () => { }); it('should properly bucket very large requests (optimal batch size)', () => { - testStore.config.batch = { max: 6, tick: 1 }; + testStore.config.batch = { limit: 6, delay: 1 }; return Promise.all([ - testStore.get(['foo2', 'bar2', 'abc2', 'def2', 'ghi2']), - testStore.get(['foo', 'bar', 'abc', 'def', 'ghi'], { language: 'en' }) + testStore.getMany(['foo2', 'bar2', 'abc2', 'def2', 'ghi2']), + testStore.getMany(['foo', 'bar', 'abc', 'def', 'ghi'], { language: 'en' }), ]) .then((result) => { - expect(result).to.deep.equal([[ - { id: 'foo2', language: undefined }, - { id: 'bar2', language: undefined }, - { id: 'abc2', language: undefined }, - { id: 'def2', language: undefined }, - { id: 'ghi2', language: undefined }, - ],[ - { id: 'foo', language: 'en' }, - { id: 'bar', language: 'en' }, - { id: 'abc', language: 'en' }, - { id: 'def', language: 'en' }, - { id: 'ghi', language: 'en' }, - ]]); + expect(result).to.deep.equal([{ + foo2: { status: 'fulfilled', value: { id: 'foo2', language: undefined } }, + bar2: { status: 'fulfilled', value: { id: 'bar2', language: undefined } }, + abc2: { status: 'fulfilled', value: { id: 'abc2', language: undefined } }, + def2: { status: 'fulfilled', value: { id: 'def2', language: undefined } }, + ghi2: { status: 'fulfilled', value: { id: 'ghi2', language: undefined } }, + },{ + foo: { status: 'fulfilled', value: { id: 'foo', language: 'en' } }, + bar: { status: 'fulfilled', value: { id: 'bar', language: 'en' } }, + abc: { status: 'fulfilled', value: { id: 'abc', language: 'en' } }, + def: { status: 'fulfilled', value: { id: 'def', language: 'en' } }, + ghi: { status: 'fulfilled', value: { id: 'ghi', language: 'en' } }, + }]); mockSource.expects('getAssets') .once().withArgs(['foo2', 'bar2', 'abc2', 'def2', 'ghi2', 'foo']) .once().withArgs(['bar', 'abc', 'def', 'ghi'], { language: 'en' }) @@ -159,7 +160,7 @@ describe('Batching', () => { it('should accumulate batch data', () => { return Promise.all([ testStore.get('foo', null, '1234567890'), - testStore.get('foo', null, '2345678901') + testStore.get('foo', null, '2345678901'), ]) .then((result) => { expect(result).to.deep.equal([{ id: 'foo', language: undefined }, { id: 'foo', language: undefined }]); @@ -173,7 +174,7 @@ describe('Batching', () => { testStore.config.batch = null; return Promise.all([ testStore.get('foo'), - testStore.get('abc', null, '1234567890') + testStore.get('abc', null, '1234567890'), ]) .then((result) => { expect(result).to.deep.equal([{ id: 'foo', language: undefined }, { id: 'abc', language: undefined }]); @@ -187,7 +188,7 @@ describe('Batching', () => { testStore.config.batch = null; return Promise.all([ testStore.get('foo'), - testStore.get('abc') + testStore.get('abc'), ]) .then((result) => { expect(result).to.deep.equal([{ id: 'foo', language: undefined }, { id: 'abc', language: undefined }]); @@ -211,7 +212,7 @@ describe('Batching', () => { beforeEach(() => { mockSource = sinon.mock(dao); testStore = store({ - uniqueParams: ['language'], + delimiter: ['language'], resolver: dao.getEmptyGroup, }); }); @@ -219,7 +220,7 @@ describe('Batching', () => { it('should batch single calls', () => { return Promise.all([ testStore.get('foo'), - testStore.get('abc') + testStore.get('abc'), ]) .then((result) => { expect(result).to.deep.equal([undefined, undefined]); @@ -230,9 +231,9 @@ describe('Batching', () => { }); it('should batch multi calls', () => { - return testStore.get(['abc', 'foo']) + return testStore.getMany(['abc', 'foo']) .then((result) => { - expect(result).to.deep.equal([undefined, undefined]); + expect(result).to.deep.equal({ abc: { status: 'fulfilled', value: undefined }, foo: { status: 'fulfilled', value: undefined } }); mockSource.expects('getEmptyGroup') .once() .withArgs(['foo', 'abc']); @@ -241,11 +242,11 @@ describe('Batching', () => { it('should batch mixed calls', () => { return Promise.all([ - testStore.get(['foo', 'bar']), - testStore.get('abc') + testStore.getMany(['foo', 'bar']), + testStore.get('abc'), ]) .then((result) => { - expect(result).to.deep.equal([[undefined, undefined], undefined]); + expect(result).to.deep.equal([{ bar: { status: 'fulfilled', value: undefined }, foo: { status: 'fulfilled', value: undefined } }, undefined]); mockSource.expects('getEmptyGroup') .once() .withArgs(['foo', 'bar', 'abc']); @@ -254,11 +255,11 @@ describe('Batching', () => { it('should mix unique params matches', () => { return Promise.all([ - testStore.get(['foo', 'bar'], { language: 'fr' }), - testStore.get('abc', { language: 'fr' }) + testStore.getMany(['foo', 'bar'], { language: 'fr' }), + testStore.get('abc', { language: 'fr' }), ]) .then((result) => { - expect(result).to.deep.equal([[undefined, undefined], undefined]); + expect(result).to.deep.equal([{ bar: { status: 'fulfilled', value: undefined }, foo: { status: 'fulfilled', value: undefined } }, undefined]); mockSource.expects('getEmptyGroup') .once().withArgs(['foo', 'bar', 'abc'], { language: 'fr' }); }); @@ -266,11 +267,11 @@ describe('Batching', () => { it('should not mix unique params mismatches', () => { return Promise.all([ - testStore.get(['foo', 'bar'], { language: 'fr' }), - testStore.get('abc', { language: 'en' }) + testStore.getMany(['foo', 'bar'], { language: 'fr' }), + testStore.get('abc', { language: 'en' }), ]) .then((result) => { - expect(result).to.deep.equal([[undefined, undefined], undefined]); + expect(result).to.deep.equal([{ bar: { status: 'fulfilled', value: undefined }, foo: { status: 'fulfilled', value: undefined } }, undefined]); mockSource.expects('getEmptyGroup') .once().withArgs(['abc'], { language: 'en' }) .once().withArgs(['foo', 'bar'], { language: 'fr' }); @@ -281,7 +282,7 @@ describe('Batching', () => { testStore.config.batch = null; return Promise.all([ testStore.get('foo'), - testStore.get('abc') + testStore.get('abc'), ]) .then((result) => { expect(result).to.deep.equal([undefined,undefined]); @@ -302,15 +303,16 @@ describe('Batching', () => { beforeEach(() => { mockSource = sinon.mock(dao); testStore = store({ - uniqueParams: ['language'], + delimiter: ['language'], resolver: dao.getPartialGroup, }); }); it('should return the valid results mixed calls', () => { - return testStore.get(['abc', 'foo', 'bar']) + testStore.config.batch = { limit: 6, delay: 1 }; + return testStore.getMany(['abc', 'foo', 'bar']) .then((result) => { - expect(result).to.deep.equal([{ id: 'abc', language: undefined }, undefined, undefined]); + expect(result).to.deep.equal({ abc: { status: 'fulfilled', value: { id: 'abc', language: undefined } }, foo: { status: 'fulfilled', value: undefined }, bar: { status: 'fulfilled', value: undefined } }); mockSource.expects('getPartialGroup') .once() .withArgs(['foo', 'bar', 'abc']); @@ -328,7 +330,7 @@ describe('Batching', () => { beforeEach(() => { mockSource = sinon.mock(dao); testStore = store({ - uniqueParams: ['language'], + delimiter: ['language'], resolver: dao.getFailedRequest, }); }); @@ -343,7 +345,7 @@ describe('Batching', () => { }); it('should properly reject on multi request', () => { - return testStore.get(['abc', 'foo'], { language: 'en' }) + return testStore.getMany(['abc', 'foo'], { language: 'en' }) .then(null, (error) => { expect(error).to.deep.equal({ error: 'Something went wrong' }); mockSource.expects('getFailedRequest') @@ -373,7 +375,7 @@ describe('Batching', () => { beforeEach(() => { mockSource = sinon.mock(dao); testStore = store({ - uniqueParams: ['language'], + delimiter: ['language'], resolver: dao.getErroredRequest, }); }); @@ -388,8 +390,15 @@ describe('Batching', () => { }); it('should properly reject on multi request', () => { - return testStore.get(['abc', 'foo'], { language: 'en' }) - .then(null, (error) => { + return testStore.getMany(['abc', 'foo'], { language: 'en' }) + .then((result) => { + //expect(result).to.deep.include({ abc: { status: 'rejected', reason: new Error('Something went wrong') }, foo: { status: 'rejected', reason: new Error('Something went wrong') } }); + expect(result.abc.status).to.equal('rejected'); + expect(result.abc.reason).to.be.instanceOf(Error).with.property('message', 'Something went wrong'); + expect(result.foo.status).to.equal('rejected'); + expect(result.foo.reason).to.be.instanceOf(Error).with.property('message', 'Something went wrong'); + }, + (error) => { expect(error).to.be.instanceOf(Error).with.property('message', 'Something went wrong'); mockSource.expects('getErroredRequest') .once().withArgs(['abc', 'foo'], { language: 'en' }); @@ -407,4 +416,30 @@ describe('Batching', () => { }); }); }); + + describe('Mixed multi requests', () => { + let testStore; + let mockSource; + afterEach(() => { + testStore = null; + mockSource.restore(); + }); + beforeEach(() => { + mockSource = sinon.mock(dao); + testStore = store({ + delimiter: ['language'], + resolver: dao.getFailOnFoo, + batch: { limit: 1 }, + }); + }); + + it('should properly return a mix of valid items and errors', () => { + return testStore.getMany(['abc', 'foo'], { language: 'en' }) + .then((result) => { + expect(result.abc).to.deep.equal({ status: 'fulfilled', value: { id: 'abc', language: 'en' } }); + expect(result.foo.status).to.equal('rejected'); + expect(result.foo.reason).to.be.instanceOf(Error).with.property('message', 'Something went wrong'); + }); + }); + }); }); \ No newline at end of file diff --git a/tests/integration/cache.js b/tests/integration/cache.js index 8b34bca..74999c0 100644 --- a/tests/integration/cache.js +++ b/tests/integration/cache.js @@ -22,7 +22,7 @@ describe('Caching', () => { beforeEach(() => { mockSource = sinon.mock(dao); testStore = store({ - uniqueParams: ['language'], + delimiter: ['language'], resolver: dao.getAssets, }); }); @@ -39,10 +39,10 @@ describe('Caching', () => { }); it('should cache multi values', () => { - testStore.get(['abc', 'foo']) - return testStore.get(['abc', 'foo']) + testStore.getMany(['abc', 'foo']) + return testStore.getMany(['abc', 'foo']) .then((result) => { - expect(result).to.deep.equal([{ id: 'abc', language: undefined }, { id: 'foo', language: undefined }]); + expect(result).to.deep.equal({ abc: { status: 'fulfilled', value: { id: 'abc', language: undefined } }, foo: { status: 'fulfilled', value: { id: 'foo', language: undefined } } }); mockSource.expects('getAssets') .exactly(1) .withArgs(['foo', 'abc']); @@ -63,10 +63,10 @@ describe('Caching', () => { it('should cache multi values without batching', () => { testStore.config.batch = null; - testStore.get(['abc', 'foo']) - return testStore.get(['abc', 'foo']) + testStore.getMany(['abc', 'foo']) + return testStore.getMany(['abc', 'foo']) .then((result) => { - expect(result).to.deep.equal([{ id: 'abc', language: undefined }, { id: 'foo', language: undefined }]); + expect(result).to.deep.equal({ abc: { status: 'fulfilled', value: { id: 'abc', language: undefined } }, foo: { status: 'fulfilled', value: { id: 'foo', language: undefined } } }); mockSource.expects('getAssets') .exactly(1) .withArgs(['foo', 'abc']); @@ -131,7 +131,7 @@ describe('Caching', () => { beforeEach(() => { mockSource = sinon.mock(dao); testStore = store({ - uniqueParams: ['language'], + delimiter: ['language'], resolver: dao.getAssets, cache: null, }); @@ -149,10 +149,10 @@ describe('Caching', () => { }); it('should cache multi values', () => { - testStore.get(['abc', 'foo']) - return testStore.get(['abc', 'foo']) + testStore.getMany(['abc', 'foo']) + return testStore.getMany(['abc', 'foo']) .then((result) => { - expect(result).to.deep.equal([{ id: 'abc', language: undefined }, { id: 'foo', language: undefined }]); + expect(result).to.deep.equal({ abc: { status: 'fulfilled', value: { id: 'abc', language: undefined } }, foo: { status: 'fulfilled', value: { id: 'foo', language: undefined } } }); mockSource.expects('getAssets') .exactly(1) .withArgs(['foo', 'abc']); @@ -170,7 +170,7 @@ describe('Caching', () => { beforeEach(() => { mockSource = sinon.mock(dao); testStore = store({ - uniqueParams: ['language'], + delimiter: ['language'], resolver: dao.getAssets, batch: null, }); @@ -188,10 +188,10 @@ describe('Caching', () => { }); it('should cache multi values', () => { - testStore.get(['abc', 'foo']) - return testStore.get(['abc', 'foo']) + testStore.getMany(['abc', 'foo']) + return testStore.getMany(['abc', 'foo']) .then((result) => { - expect(result).to.deep.equal([{ id: 'abc', language: undefined }, { id: 'foo', language: undefined }]); + expect(result).to.deep.equal({ abc: { status: 'fulfilled', value: { id: 'abc', language: undefined } }, foo: { status: 'fulfilled', value: { id: 'foo', language: undefined } } }); mockSource.expects('getAssets') .exactly(1) .withArgs(['foo', 'abc']); @@ -209,7 +209,7 @@ describe('Caching', () => { beforeEach(() => { mockSource = sinon.mock(dao); testStore = store({ - uniqueParams: ['language'], + delimiter: ['language'], resolver: dao.getAssets, cache: null, batch: null, @@ -228,10 +228,10 @@ describe('Caching', () => { }); it('should cache multi values', () => { - testStore.get(['abc', 'foo']) - return testStore.get(['abc', 'foo']) + testStore.getMany(['abc', 'foo']) + return testStore.getMany(['abc', 'foo']) .then((result) => { - expect(result).to.deep.equal([{ id: 'abc', language: undefined }, { id: 'foo', language: undefined }]); + expect(result).to.deep.equal({ abc: { status: 'fulfilled', value: { id: 'abc', language: undefined } }, foo: { status: 'fulfilled', value: { id: 'foo', language: undefined } } }); mockSource.expects('getAssets') .exactly(1) .withArgs(['foo', 'abc']); @@ -249,7 +249,7 @@ describe('Caching', () => { beforeEach(() => { mockSource = sinon.mock(dao); testStore = store({ - uniqueParams: ['language'], + delimiter: ['language'], resolver: dao.getEmptyGroup, }); }); @@ -266,9 +266,9 @@ describe('Caching', () => { }); it('should batch empty multi values', () => { - return testStore.get(['abc', 'foo']) + return testStore.getMany(['abc', 'foo']) .then((result) => { - expect(result).to.deep.equal([undefined, undefined]); + expect(result).to.deep.equal({ abc: { status: 'fulfilled', value: undefined }, foo: { status: 'fulfilled', value: undefined } }); mockSource.expects('getEmptyGroup') .once() .withArgs(['foo', 'abc']); @@ -298,15 +298,15 @@ describe('Caching', () => { beforeEach(() => { mockSource = sinon.mock(dao); testStore = store({ - uniqueParams: ['language'], + delimiter: ['language'], resolver: dao.getPartialGroup, }); }); it('should cache all the results on mixed responses', () => { - return testStore.get(['abc', 'foo', 'bar']) + return testStore.getMany(['abc', 'foo', 'bar']) .then((result) => { - expect(result).to.deep.equal([{ id: 'abc', language: undefined }, undefined, undefined]); + expect(result).to.deep.equal({ abc: { status: 'fulfilled', value: { id: 'abc', language: undefined } }, foo: { status: 'fulfilled', value: undefined }, bar: { status: 'fulfilled', value: undefined } }); mockSource.expects('getPartialGroup') .once() .withArgs(['foo', 'bar', 'abc']); @@ -336,7 +336,7 @@ describe('Caching', () => { beforeEach(() => { mockSource = sinon.mock(dao); testStore = store({ - uniqueParams: ['language'], + delimiter: ['language'], resolver: dao.getFailedRequest, }); }); @@ -351,7 +351,7 @@ describe('Caching', () => { }); it('should not cache failed multi requests', () => { - return testStore.get(['abc', 'foo'], { language: 'en' }) + return testStore.getMany(['abc', 'foo'], { language: 'en' }) .then(null, (error) => { expect(error).to.deep.equal({ error: 'Something went wrong' }); mockSource.expects('getFailedRequest') @@ -381,7 +381,7 @@ describe('Caching', () => { beforeEach(() => { mockSource = sinon.mock(dao); testStore = store({ - uniqueParams: ['language'], + delimiter: ['language'], resolver: dao.getErroredRequest, }); }); diff --git a/tests/integration/utils/dao.js b/tests/integration/utils/dao.js index 6f906e2..f15e69f 100644 --- a/tests/integration/utils/dao.js +++ b/tests/integration/utils/dao.js @@ -1,36 +1,47 @@ function getAssets(ids, { language }) { - return new Promise((resolve, reject) => { - setTimeout(() => resolve(ids.map(id => ({ id, language }))), (ids.length > 1) ? 130 : 100); + return new Promise((resolve) => { + setTimeout(() => resolve(ids.reduce((acc, id) => { + acc[id] = { id, language }; + return acc; + }, {})), (ids.length > 1) ? 130 : 100); }); } -function getEmptyGroup(ids, { language }) { - return new Promise((resolve, reject) => { +function getEmptyGroup() { + return new Promise((resolve) => { setTimeout(() => resolve([]), 10); }); } function getPartialGroup(ids, { language }) { - return new Promise((resolve, reject) => { - setTimeout(() => resolve([{ id: ids[0], language }]), 5); + return new Promise((resolve) => { + setTimeout(() => resolve({ [ids[0]]: { id: ids[0], language }}), 5); }); } -function getErroredRequest(ids, { language }) { - return new Promise((resolve, reject) => { +function getFailOnFoo(ids, params) { + if (ids[0] === 'foo') return getErroredRequest(); + return getAssets(ids, params); +} + +function getErroredRequest() { + return new Promise(() => { throw new Error('Something went wrong'); }); } -function getFailedRequest(ids, { language }) { +function getFailedRequest() { return new Promise((resolve, reject) => { setTimeout(() => reject({ error: 'Something went wrong' }), 10); }); } function getSlowRequest(ids, { language }) { - return new Promise((resolve, reject) => { - setTimeout(() => resolve(ids.map(id => ({ id, language }))), 1000); + return new Promise((resolve) => { + setTimeout(() => resolve(ids.reduce((acc, id) => { + acc[id] = { id, language }; + return acc; + }, {})), 1000); }); } @@ -40,5 +51,6 @@ module.exports = { getPartialGroup, getErroredRequest, getFailedRequest, + getFailOnFoo, getSlowRequest, }; diff --git a/tests/profiling/dao.js b/tests/profiling/dao.js index 25c7a52..a7f31c9 100644 --- a/tests/profiling/dao.js +++ b/tests/profiling/dao.js @@ -1,29 +1,33 @@ function simulateNetwork() { - let tick = 0; - while(true) { - if (++tick > 0xffff) break; - } - return true; + let tick = 0; + while(true) { // eslint-disable-line + if (++tick > 0xffff) break; + } + return true; } function calculateResponseTime(numItems) { - return Math.round(30 + numItems * 0.5); + return Math.round(30 + numItems * 0.5); } function getAssets(ids, { language }) { - return new Promise((resolve, reject) => { - simulateNetwork(); - setTimeout(() => resolve(ids.map(id => ({ id, language }))), calculateResponseTime(ids.length)); - }); + return new Promise((resolve) => { + simulateNetwork(); + setTimeout(() => resolve(ids.reduce((acc, id) => { + acc[id] = { id, language }; + return acc; + }, {}) + ), calculateResponseTime(ids.length)); + }); } -function getErroredRequest(ids, { language }) { - return new Promise((resolve, reject) => { - throw new Error('Something went wrong'); - }); +function getErroredRequest() { + return new Promise(() => { + throw new Error('Something went wrong'); + }); } module.exports = { - getAssets, - getErroredRequest, + getAssets, + getErroredRequest, }; diff --git a/tests/profiling/index.js b/tests/profiling/index.js index 757cb84..d5f51f6 100644 --- a/tests/profiling/index.js +++ b/tests/profiling/index.js @@ -14,7 +14,7 @@ const split2 = require('split2'); /* Init ----------------------------------------------------------------------*/ // Setup -const app = fork(path.resolve(__dirname, './worker.js')/*, { execArgv: ['--prof']}*/); +const app = fork(path.resolve(__dirname, './worker.js') /*{ execArgv: ['--inspect=10245']}*/); const stream = fs.createReadStream(path.resolve(settings.test.sampleFile), 'utf-8').pipe(split2()); app.on('message', async (suite) => { @@ -36,7 +36,9 @@ app.on('message', async (suite) => { process.exit(1); } } - process.exit(0); + //process.exit(0); + + }); stream.on('data', (chunk) => { diff --git a/tests/profiling/settings.js b/tests/profiling/settings.js index 553d7ad..c52ad17 100644 --- a/tests/profiling/settings.js +++ b/tests/profiling/settings.js @@ -2,24 +2,24 @@ const {getAssets} = require('./dao.js'); //const redisStore = require('ha-store-redis'); module.exports = { - test: { - sampleFile: './sample.txt', - }, - setup: { - resolver: getAssets, - // store: redisStore('footage-api-ha-cache', { host: '0.0.0.0', port: 6379 }), - uniqueParams: ['language'], - cache: { limit: 5000, ttl: 300000 }, - batch: { tick: 10, max: 50 }, - retry: { base: 1, step: 2 }, - }, - assert: { - completed: [300000, 300000], - coalescedHit: [8000, 15000], - cacheHits: [35000, 45000], - timeouts: [0, 0], - batches: [4800, 5300], - rss: [50000, 80000], - avgBatchSize: [45, 50], - }, + test: { + sampleFile: './sample.txt', + }, + setup: { + resolver: getAssets, + // store: redisStore('footage-api-ha-cache', { host: '0.0.0.0', port: 6379 }), + delimiter: ['language'], + cache: { limit: 5000, ttl: 300000 }, + batch: { delay: 10, limit: 50 }, + retry: { base: 1, step: 2 }, + }, + assert: { + completed: [300000, 300000], + coalescedHit: [8000, 50000], + cacheHits: [35000, 50000], + timeouts: [0, 0], + batches: [4800, 5300], + rss: [50000, 80000], + avgBatchSize: [45, 50], + }, } \ No newline at end of file diff --git a/tests/profiling/worker.js b/tests/profiling/worker.js index 4725e50..53e264c 100644 --- a/tests/profiling/worker.js +++ b/tests/profiling/worker.js @@ -7,46 +7,42 @@ const settings = require('./settings'); const HA = require('../../src/index.js'); const crypto = require('crypto'); -const v8 = require('v8'); -const heapdump = require('heapdump'); /* Local variables -----------------------------------------------------------*/ const suite = { - completed: 0, - cacheHits: 0, - coalescedHit: 0, - sum: 0, - timeouts: 0, - batches: 0, - avgBatchSize: 0, - startHeap: process.memoryUsage().rss, + completed: 0, + cacheHits: 0, + coalescedHit: 0, + sum: 0, + timeouts: 0, + batches: 0, + avgBatchSize: 0, + startHeap: process.memoryUsage().rss, }; const store = HA(settings.setup); -// console.log(v8.getHeapSpaceStatistics().map(parseMemorySpace)); - /* Methods -------------------------------------------------------------------*/ function handleRequest(id, language) { - let finished = false; - const before = Date.now(); - const timeout = setTimeout(() => { - if (finished === false) { - suite.timeouts++; - console.log(`request timed out: { id: ${id}, lang: ${language}}`); - } - }, 500); - store.get(id, { language }, crypto.randomBytes(8).toString('hex')) + let finished = false; + const before = Date.now(); + const timeout = setTimeout(() => { + if (finished === false) { + suite.timeouts++; + console.log(`request timed out: { id: ${id}, lang: ${language}}`); + } + }, 500); + store.get(id, { language }, crypto.randomBytes(8).toString('hex')) .then((result) => { - clearTimeout(timeout); - finished = true; - if (!result || result.id !== id || result.language !== language) { - throw new Error(`Integrity test failed: ${result} does not match {${id} ${language}}`); - } - suite.sum += (Date.now() - before); - suite.completed++; + clearTimeout(timeout); + finished = true; + if (!result || result.id !== id || result.language !== language) { + throw new Error(`Integrity test failed: ${result} does not match {${id} ${language}}`); + } + suite.sum += (Date.now() - before); + suite.completed++; }, () => {}) .catch((err) => { console.log(err); process.exit(1)} ); } @@ -55,31 +51,17 @@ store.on('query', batch => { suite.batches++; suite.avgBatchSize += batch.size; store.on('cacheHit', evt => { suite.cacheHits+=evt.found; }); store.on('coalescedHit', evt => { suite.coalescedHit+=evt.found; }); -function roundMi(value) { - return Math.round((value / 1024 / 1024) * 1000) / 1000; -} - -function parseMemorySpace(space) { - return `${space.space_name} = ${roundMi(space.space_used_size)}/${roundMi(space.space_size)}`; -} - //End function complete() { - // give a chance to in-flight requests to complete - setTimeout(async () => { - suite.avgBatchSize = Math.round(suite.avgBatchSize / suite.batches); - suite.size = await store.size(); - suite.startHeap = process.memoryUsage().rss - suite.startHeap; - - // console.log(v8.getHeapSpaceStatistics().map(parseMemorySpace)); - - /*heapdump.writeSnapshot((err, filename) => { - console.log('Heap dump written to', filename) - });*/ - - process.send(suite); - process.exit(0); - }, 1000); + // give a chance to in-flight requests to complete + setTimeout(async () => { + suite.avgBatchSize = Math.round(suite.avgBatchSize / suite.batches); + suite.size = await store.size(); + suite.startHeap = process.memoryUsage().rss - suite.startHeap; + + process.send(suite); + process.exit(0); + }, 1000); } /* Init ----------------------------------------------------------------------*/ diff --git a/tests/unit/index.spec.js b/tests/unit/index.spec.js index bb0cefa..57f8e2a 100644 --- a/tests/unit/index.spec.js +++ b/tests/unit/index.spec.js @@ -14,7 +14,7 @@ function checkForPublicProperties(store) { expect(store.get).to.not.be.undefined; expect(store.set).to.not.be.undefined; expect(store.clear).to.not.be.undefined; - expect(store.getKey).to.not.be.undefined; + expect(store.getStorageKey).to.not.be.undefined; expect(store.config).to.not.be.undefined; expect(store.queue).to.not.be.undefined; } @@ -32,7 +32,7 @@ describe('index', () => { const test = root({ resolver: () => { }, - uniqueParams: ['a', 'b', 'c'], + delimiter: ['a', 'b', 'c'], cache: null, batch: null, }); @@ -42,7 +42,7 @@ describe('index', () => { it('should produce a batcher with all the default config when called with true requirements', () => { const test = root({ resolver: noop, - uniqueParams: ['a', 'b', 'c'], + delimiter: ['a', 'b', 'c'], cache: true, batch: true, }); @@ -52,9 +52,9 @@ describe('index', () => { it('should produce a batcher with all the merged config when called with custom requirements', () => { const test = root({ resolver: noop, - uniqueParams: ['a', 'b', 'c'], + delimiter: ['a', 'b', 'c'], cache: {base: 2}, - batch: {max: 12}, + batch: {limit: 12}, }); checkForPublicProperties(test); }); @@ -143,7 +143,7 @@ describe('index', () => { it('should handle multi record clear queries with params', () => { const test = root({ resolver: noop, - uniqueParams: ['foo'], + delimiter: ['foo'], }); const params = {foo: 'bar'}; const storeMock = sinon.mock(test.store); @@ -168,25 +168,25 @@ describe('index', () => { }); }); - describe('#getKey', () => { + describe('#getStorageKey', () => { it('should return a record key when given an id', () => { const test = root({resolver: noop}); - expect(test.getKey('123abc')).to.be.equal('::123abc'); + expect(test.getStorageKey('123abc')).to.be.equal('::123abc'); }); it('should return a record key when given an id and segregators', () => { - const test = root({resolver: noop, uniqueParams: ['language']}); - expect(test.getKey('123abc')).to.be.equal('language=undefined::123abc'); + const test = root({resolver: noop, delimiter: ['language']}); + expect(test.getStorageKey('123abc')).to.be.equal('language=undefined::123abc'); }); it('should return a record key when given an id, segregators and params', () => { - const test = root({resolver: noop, uniqueParams: ['language']}); - expect(test.getKey('123abc', {language:'fr'})).to.be.equal('language="fr"::123abc'); + const test = root({resolver: noop, delimiter: ['language']}); + expect(test.getStorageKey('123abc', {language:'fr'})).to.be.equal('language="fr"::123abc'); }); it('should return a record key when given an id, segregators and multiple params', () => { - const test = root({resolver: noop, uniqueParams: ['language', 'country']}); - expect(test.getKey('123abc', {language:'fr',country:'FR'})).to.be.equal('language="fr";country="FR"::123abc'); + const test = root({resolver: noop, delimiter: ['language', 'country']}); + expect(test.getStorageKey('123abc', {language:'fr',country:'FR'})).to.be.equal('language="fr";country="FR"::123abc'); }); }); }); diff --git a/tests/unit/options.spec.js b/tests/unit/options.spec.js index 9161cf2..b367144 100644 --- a/tests/unit/options.spec.js +++ b/tests/unit/options.spec.js @@ -13,7 +13,7 @@ const store = require('../../src/store'); describe('options', () => { const defaultConfig = { - "batch": {"tick": 50, "max": 100}, + "batch": {"delay": 50, "limit": 100}, "cache": {"limit": 5000, "ttl": 300000}, "store": store, }; @@ -22,28 +22,28 @@ describe('options', () => { it('should produce a batcher with all the default config when called with true requirements', () => { const test = batcher({ resolver: noop, - uniqueParams: ['a', 'b', 'c'], + delimiter: ['a', 'b', 'c'], cache: true, batch: true, }); expect(test.config).to.deep.contain({ cache: {limit: 5000, ttl: 300000}, - batch: {max: 100, tick: 50}, + batch: {limit: 100, delay: 50}, }); }); it('should produce a batcher with all the merged config when called with custom requirements', () => { const test = batcher({ resolver: noop, - uniqueParams: ['a', 'b', 'c'], + delimiter: ['a', 'b', 'c'], cache: {limit: 2}, - batch: {max: 12}, + batch: {limit: 12}, }); expect(test.config).to.deep.contain({ cache: {limit: 2, ttl: 300000}, - batch: {max: 12, tick: 50}, + batch: {limit: 12, delay: 50}, }); }); diff --git a/tests/unit/utils.spec.js b/tests/unit/utils.spec.js index ecc2dee..75053d5 100644 --- a/tests/unit/utils.spec.js +++ b/tests/unit/utils.spec.js @@ -10,21 +10,6 @@ const expect = require('chai').expect; /* Tests ---------------------------------------------------------------------*/ describe('utils', () => { - - describe('#basicParser', () => { - it('should return properly formatted results', () => { - expect(utils.basicParser({ '123': { id: 123 } }, ['123'])).to.deep.equal({ '123': { id: 123 }}); - expect(utils.basicParser({ '123': { id: 123 } }, [123])).to.deep.equal({ '123': { id: 123 }}); - expect(utils.basicParser([{ id: 123 }], [123])).to.deep.equal({ '123': { id: 123 }}); - expect(utils.basicParser([{ id: '123' }], [123])).to.deep.equal({ '123': { id: '123' }}); - }); - - it('should trim out unwanted results', () => { - expect(utils.basicParser({ '123': { id: 123 }, '456': { id: 456 } }, [123])).to.deep.equal({ '123': { id: 123 }}); - expect(utils.basicParser([{ id: '123' }, { id: '456'}], [123])).to.deep.equal({ '123': { id: '123' }}); - }); - }); - describe('#contextKey', () => { it('should print a context key based on empty params', () => { expect(utils.contextKey(['foo'], {})).to.equal('foo=undefined');