diff --git a/.buildkite/Dockerfile b/.buildkite/Dockerfile index 5608747b6..6d44e2211 100644 --- a/.buildkite/Dockerfile +++ b/.buildkite/Dockerfile @@ -10,5 +10,7 @@ RUN apt-get clean -y && \ WORKDIR /usr/src/app -COPY . . +COPY package.json . RUN npm install --production=false + +COPY . . diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 32b37b6c6..1dca14548 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -6,7 +6,7 @@ steps: env: NODE_VERSION: "{{ matrix.nodejs }}" TEST_SUITE: "{{ matrix.suite }}" - STACK_VERSION: 8.10.3-SNAPSHOT + STACK_VERSION: 8.12.0-SNAPSHOT matrix: setup: suite: diff --git a/.dockerignore b/.dockerignore index 54eb2a95a..e34f9ff27 100644 --- a/.dockerignore +++ b/.dockerignore @@ -3,3 +3,5 @@ npm-debug.log test/benchmarks elasticsearch .git +lib +junit-output diff --git a/.github/workflows/npm-publish.yml b/.github/workflows/npm-publish.yml new file mode 100644 index 000000000..73a7d36c2 --- /dev/null +++ b/.github/workflows/npm-publish.yml @@ -0,0 +1,27 @@ +name: Publish Package to npm +on: + workflow_dispatch: + inputs: + branch: + description: 'Git branch to build and publish' + required: true +jobs: + build: + runs-on: ubuntu-latest + permissions: + contents: read + id-token: write + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.event.inputs.branch }} + - uses: actions/setup-node@v3 + with: + node-version: '20.x' + registry-url: 'https://registry.npmjs.org' + - run: npm install -g npm + - run: npm install + - run: npm test + - run: npm publish --provenance --access public + env: + NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} diff --git a/catalog-info.yaml b/catalog-info.yaml index b8bbd36ff..80c943cd8 100644 --- a/catalog-info.yaml +++ b/catalog-info.yaml @@ -42,9 +42,9 @@ spec: main_semi_daily: branch: 'main' cronline: '0 */12 * * *' - 8_9_semi_daily: - branch: '8.9' + 8_12_semi_daily: + branch: '8.12' cronline: '0 */12 * * *' - 8_8_daily: - branch: '8.8' + 8_11_daily: + branch: '8.11' cronline: '@daily' diff --git a/docs/advanced-config.asciidoc b/docs/advanced-config.asciidoc index 638aeada4..b3c9388a4 100644 --- a/docs/advanced-config.asciidoc +++ b/docs/advanced-config.asciidoc @@ -91,6 +91,95 @@ const client = new Client({ }) ---- +[discrete] +[[redaction]] +==== Redaction of potentially sensitive data + +When the client raises an `Error` that originated at the HTTP layer, like a `ConnectionError` or `TimeoutError`, a `meta` object is often attached to the error object that includes metadata useful for debugging, like request and response information. Because this can include potentially sensitive data, like authentication secrets in an `Authorization` header, the client takes measures to redact common sources of sensitive data when this metadata is attached and serialized. + +If your configuration requires extra headers or other configurations that may include sensitive data, you may want to adjust these settings to account for that. + +By default, the `redaction` option is set to `{ type: 'replace' }`, which recursively searches for sensitive key names, case insensitive, and replaces their values with the string `[redacted]`. + +[source,js] +---- +const { Client } = require('@elastic/elasticsearch') + +const client = new Client({ + cloud: { id: '' }, + auth: { apiKey: 'base64EncodedKey' }, +}) + +try { + await client.indices.create({ index: 'my_index' }) +} catch (err) { + console.log(err.meta.meta.request.options.headers.authorization) // prints "[redacted]" +} +---- + +If you would like to redact additional properties, you can include additional key names to search and replace: + +[source,js] +---- +const { Client } = require('@elastic/elasticsearch') + +const client = new Client({ + cloud: { id: '' }, + auth: { apiKey: 'base64EncodedKey' }, + headers: { 'X-My-Secret-Password': 'shhh it's a secret!' }, + redaction: { + type: "replace", + additionalKeys: ["x-my-secret-password"] + } +}) + +try { + await client.indices.create({ index: 'my_index' }) +} catch (err) { + console.log(err.meta.meta.request.options.headers['X-My-Secret-Password']) // prints "[redacted]" +} +---- + +Alternatively, if you know you're not going to use the metadata at all, setting the redaction type to `remove` will remove all optional sources of potentially sensitive data entirely, or replacing them with `null` for required properties. + +[source,js] +---- +const { Client } = require('@elastic/elasticsearch') + +const client = new Client({ + cloud: { id: '' }, + auth: { apiKey: 'base64EncodedKey' }, + redaction: { type: "remove" } +}) + +try { + await client.indices.create({ index: 'my_index' }) +} catch (err) { + console.log(err.meta.meta.request.options.headers) // undefined +} +---- + +Finally, if you prefer to turn off redaction altogether, perhaps while debugging on a local developer environment, you can set the redaction type to `off`. This will revert the client to pre-8.11.0 behavior, where basic redaction is only performed during common serialization methods like `console.log` and `JSON.stringify`. + +WARNING: Setting `redaction.type` to `off` is not recommended in production environments. + +[source,js] +---- +const { Client } = require('@elastic/elasticsearch') + +const client = new Client({ + cloud: { id: '' }, + auth: { apiKey: 'base64EncodedKey' }, + redaction: { type: "off" } +}) + +try { + await client.indices.create({ index: 'my_index' }) +} catch (err) { + console.log(err.meta.meta.request.options.headers.authorization) // the actual header value will be logged +} +---- + [discrete] ==== Migrate to v8 diff --git a/docs/changelog.asciidoc b/docs/changelog.asciidoc index b82c397da..0daeee5d7 100644 --- a/docs/changelog.asciidoc +++ b/docs/changelog.asciidoc @@ -1,6 +1,59 @@ [[changelog-client]] == Release notes +[discrete] +=== 8.12.2 + +[discrete] +==== Fixes + +[discrete] +===== Upgrade transport to 8.4.1 https://github.com/elastic/elasticsearch-js/pull/2137[#2137] + +Upgrades `@elastic/transport` to 8.4.1 to resolve https://github.com/elastic/elastic-transport-js/pull/83[a bug] where arrays in error diagnostics were unintentionally transformed into objects. + +[discrete] +=== 8.12.1 + +[discrete] +==== Fixes + +[discrete] +===== Fix hang in bulk helper semaphore https://github.com/elastic/elasticsearch-js/pull/2027[#2027] + +The failing state could be reached when a server's response times are slower than flushInterval. + +[discrete] +=== 8.12.0 + +[discrete] +=== Features + +[discrete] +===== Support for Elasticsearch `v8.12.0` + +You can find all the API changes +https://www.elastic.co/guide/en/elasticsearch/reference/8.12/release-notes-8.12.0.html[here]. + +[discrete] +=== 8.11.0 + +[discrete] +=== Features + +[discrete] +===== Support for Elasticsearch `v8.11.0` + +You can find all the API changes +https://www.elastic.co/guide/en/elasticsearch/reference/8.11/release-notes-8.11.0.html[here]. + +[discrete] +===== Enhanced support for redacting potentially sensitive data https://github.com/elastic/elasticsearch-js/pull/2095[#2095] + +`@elastic/transport` https://github.com/elastic/elastic-transport-js/releases/tag/v8.4.0[version 8.4.0] introduces enhanced measures for ensuring that request metadata attached to some `Error` objects is redacted. This functionality is primarily to address custom logging solutions that don't use common serialization methods like `JSON.stringify`, `console.log`, or `util.inspect`, which were already accounted for. + +See <> for more information. + [discrete] === 8.10.0 @@ -342,6 +395,9 @@ The client API leaks HTTP-related notions in many places, and removing them woul This could be a rather big breaking change, so a double solution could be used during the 8.x lifecycle. (accepting body keys without them being wrapped in the body as well as the current solution). +To convert code from 7.x, you need to remove the `body` parameter in all the endpoints request. +For instance, this is an example for the `search` endpoint: + [source,js] ---- // from @@ -380,6 +436,12 @@ If you weren't extending the internals of the client, this won't be a breaking c The client API leaks HTTP-related notions in many places, and removing them would definitely improve the DX. The client will expose a new request-specific option to still get the full response details. +The new behaviour returns the `body` value directly as response. +If you want to have the 7.x response format, you need to add `meta : true` in the request. +This will return all the HTTP meta information, including the `body`. + +For instance, this is an example for the `search` endpoint: + [source,js] ---- // from diff --git a/docs/connecting.asciidoc b/docs/connecting.asciidoc index 77c2e7d11..15007ceb3 100644 --- a/docs/connecting.asciidoc +++ b/docs/connecting.asciidoc @@ -12,6 +12,7 @@ This page contains the information you need to connect and use the Client with * <> * <> * <> +* <> * <> [[authentication]] @@ -691,6 +692,20 @@ const client = new Client({ }) ---- +[discrete] +[[close-connections]] +=== Closing a client's connections + +If you would like to close all open connections being managed by an instance of the client, use the `close()` function: + +[source,js] +---- +const client = new Client({ + node: 'http://localhost:9200' +}); +client.close(); +---- + [discrete] [[product-check]] === Automatic product check diff --git a/docs/doc_examples/36b86b97feedcf5632824eefc251d6ed.asciidoc b/docs/doc_examples/36b86b97feedcf5632824eefc251d6ed.asciidoc new file mode 100644 index 000000000..408ce2f71 --- /dev/null +++ b/docs/doc_examples/36b86b97feedcf5632824eefc251d6ed.asciidoc @@ -0,0 +1,12 @@ +[source,js] +---- +const response = await client.search({ + index: 'books', + query: { + match: { + name: 'brave' + } + } +}) +console.log(response) +---- diff --git a/docs/doc_examples/8575c966b004fb124c7afd6bb5827b50.asciidoc b/docs/doc_examples/8575c966b004fb124c7afd6bb5827b50.asciidoc new file mode 100644 index 000000000..d99bd96dc --- /dev/null +++ b/docs/doc_examples/8575c966b004fb124c7afd6bb5827b50.asciidoc @@ -0,0 +1,13 @@ +[source,js] +---- +const response = await client.index({ + index: 'books', + document: { + name: 'Snow Crash', + author: 'Neal Stephenson', + release_date: '1992-06-01', + page_count: 470, + } +}) +console.log(response) +---- diff --git a/docs/doc_examples/bcc75fc01b45e482638c65b8fbdf09fa.asciidoc b/docs/doc_examples/bcc75fc01b45e482638c65b8fbdf09fa.asciidoc new file mode 100644 index 000000000..1708d0956 --- /dev/null +++ b/docs/doc_examples/bcc75fc01b45e482638c65b8fbdf09fa.asciidoc @@ -0,0 +1,7 @@ +[source,js] +---- +const response = await client.search({ + index: 'books' +}) +console.log(response) +---- diff --git a/docs/doc_examples/d04f0c8c44e8b4fb55f2e7d9d05977e7.asciidoc b/docs/doc_examples/d04f0c8c44e8b4fb55f2e7d9d05977e7.asciidoc new file mode 100644 index 000000000..e5ce437b2 --- /dev/null +++ b/docs/doc_examples/d04f0c8c44e8b4fb55f2e7d9d05977e7.asciidoc @@ -0,0 +1,43 @@ +[source,js] +---- +const response = await client.bulk({ + operations: [ + { index: { _index: 'books' } }, + { + name: 'Revelation Space', + author: 'Alastair Reynolds', + release_date: '2000-03-15', + page_count: 585, + }, + { index: { _index: 'books' } }, + { + name: '1984', + author: 'George Orwell', + release_date: '1985-06-01', + page_count: 328, + }, + { index: { _index: 'books' } }, + { + name: 'Fahrenheit 451', + author: 'Ray Bradbury', + release_date: '1953-10-15', + page_count: 227, + }, + { index: { _index: 'books' } }, + { + name: 'Brave New World', + author: 'Aldous Huxley', + release_date: '1932-06-01', + page_count: 268, + }, + { index: { _index: 'books' } }, + { + name: 'The Handmaids Tale', + author: 'Margaret Atwood', + release_date: '1985-06-01', + page_count: 311, + } + ] +}) +console.log(response) +---- diff --git a/package.json b/package.json index a772f791e..0ecd601ca 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "@elastic/elasticsearch", - "version": "8.10.3", - "versionCanary": "8.10.3-canary.1", + "version": "8.12.0", + "versionCanary": "8.12.0-canary.0", "description": "The official Elasticsearch client for Node.js", "main": "index.js", "types": "index.d.ts", @@ -83,7 +83,7 @@ "zx": "^7.2.2" }, "dependencies": { - "@elastic/transport": "^8.3.4", + "@elastic/transport": "^8.4.1", "tslib": "^2.4.0" }, "tap": { diff --git a/src/client.ts b/src/client.ts index 09118d58c..50ba4942f 100644 --- a/src/client.ts +++ b/src/client.ts @@ -43,6 +43,7 @@ import { BearerAuth, Context } from '@elastic/transport/lib/types' +import { RedactionOptions } from '@elastic/transport/lib/Transport' import BaseConnection, { prepareHeaders } from '@elastic/transport/lib/connection/BaseConnection' import SniffingTransport from './sniffingTransport' import Helpers from './helpers' @@ -113,6 +114,7 @@ export interface ClientOptions { caFingerprint?: string maxResponseSize?: number maxCompressedResponseSize?: number + redaction?: RedactionOptions } export default class Client extends API { @@ -186,7 +188,11 @@ export default class Client extends API { proxy: null, enableMetaHeader: true, maxResponseSize: null, - maxCompressedResponseSize: null + maxCompressedResponseSize: null, + redaction: { + type: 'replace', + additionalKeys: [] + } }, opts) if (options.caFingerprint != null && isHttpConnection(opts.node ?? opts.nodes)) { @@ -259,7 +265,8 @@ export default class Client extends API { jsonContentType: 'application/vnd.elasticsearch+json; compatible-with=8', ndjsonContentType: 'application/vnd.elasticsearch+x-ndjson; compatible-with=8', accept: 'application/vnd.elasticsearch+json; compatible-with=8,text/plain' - } + }, + redaction: options.redaction }) this.helpers = new Helpers({ diff --git a/src/helpers.ts b/src/helpers.ts index 0bd1b1c5c..fbf4ff334 100644 --- a/src/helpers.ts +++ b/src/helpers.ts @@ -196,8 +196,11 @@ export default class Helpers { await sleep(wait) } assert(response !== undefined, 'The response is undefined, please file a bug report') + + const { redaction = { type: 'replace' } } = options + const errorOptions = { redaction } if (response.statusCode === 429) { - throw new ResponseError(response) + throw new ResponseError(response, errorOptions) } let scroll_id = response.body._scroll_id @@ -237,7 +240,7 @@ export default class Helpers { await sleep(wait) } if (response.statusCode === 429) { - throw new ResponseError(response) + throw new ResponseError(response, errorOptions) } } @@ -289,6 +292,9 @@ export default class Helpers { } = options reqOptions.meta = true + const { redaction = { type: 'replace' } } = reqOptions + const errorOptions = { redaction } + let stopReading = false let stopError: Error | null = null let timeoutRef = null @@ -502,7 +508,7 @@ export default class Helpers { // @ts-expect-error addDocumentsGetter(result) if (response.status != null && response.status >= 400) { - callbacks[i](new ResponseError(result), result) + callbacks[i](new ResponseError(result, errorOptions), result) } else { callbacks[i](null, result) } @@ -618,7 +624,7 @@ export default class Helpers { let chunkBytes = 0 timeoutRef = setTimeout(onFlushTimeout, flushInterval) // eslint-disable-line - // @ts-expect-error datasoruce is an iterable + // @ts-expect-error datasource is an iterable for await (const chunk of datasource) { if (shouldAbort) break timeoutRef.refresh() @@ -650,15 +656,16 @@ export default class Helpers { if (chunkBytes >= flushBytes) { stats.bytes += chunkBytes - const send = await semaphore() - send(bulkBody.slice()) + const bulkBodyCopy = bulkBody.slice() bulkBody.length = 0 chunkBytes = 0 + const send = await semaphore() + send(bulkBodyCopy) } } clearTimeout(timeoutRef) - // In some cases the previos http call does not have finished, + // In some cases the previous http call has not finished, // or we didn't reach the flush bytes threshold, so we force one last operation. if (!shouldAbort && chunkBytes > 0) { const send = await semaphore() @@ -702,8 +709,8 @@ export default class Helpers { // to guarantee that no more than the number of operations // allowed to run at the same time are executed. // It returns a semaphore function which resolves in the next tick - // if we didn't reach the maximim concurrency yet, otherwise it returns - // a promise that resolves as soon as one of the running request has finshed. + // if we didn't reach the maximum concurrency yet, otherwise it returns + // a promise that resolves as soon as one of the running requests has finished. // The semaphore function resolves a send function, which will be used // to send the actual bulk request. // It also returns a finish function, which returns a promise that is resolved diff --git a/test/integration/index.js b/test/integration/index.js index defdb400f..b07ddd2d7 100644 --- a/test/integration/index.js +++ b/test/integration/index.js @@ -317,7 +317,7 @@ async function start ({ client, isXPack }) { if (name === 'setup' || name === 'teardown') continue if (options.test && !name.endsWith(options.test)) continue - const junitTestCase = junitTestSuite.testcase(name, `node_${process.version}/${cleanPath}`) + const junitTestCase = junitTestSuite.testcase(name, `node_${process.version}: ${cleanPath}`) stats.total += 1 if (shouldSkip(isXPack, file, name)) { @@ -336,6 +336,7 @@ async function start ({ client, isXPack }) { junitTestSuite.end() junitTestSuites.end() generateJunitXmlReport(junit, isXPack ? 'platinum' : 'free') + err.meta = JSON.stringify(err.meta ?? {}, null, 2) console.error(err) if (options.bail) { @@ -374,6 +375,7 @@ async function start ({ client, isXPack }) { - Total: ${stats.total} - Skip: ${stats.skip} - Pass: ${stats.pass} + - Fail: ${stats.total - (stats.pass + stats.skip)} - Assertions: ${stats.assertions} `) } diff --git a/test/integration/test-runner.js b/test/integration/test-runner.js index 64570945a..ce80da43e 100644 --- a/test/integration/test-runner.js +++ b/test/integration/test-runner.js @@ -593,13 +593,14 @@ function build (opts = {}) { const key = Object.keys(action.match)[0] match( // in some cases, the yaml refers to the body with an empty string - key === '$body' || key === '' + key.split('.')[0] === '$body' || key === '' ? response : delve(response, fillStashedValues(key)), - key === '$body' + key.split('.')[0] === '$body' ? action.match[key] : fillStashedValues(action.match)[key], - action.match + action.match, + response ) } @@ -608,7 +609,8 @@ function build (opts = {}) { const key = Object.keys(action.lt)[0] lt( delve(response, fillStashedValues(key)), - fillStashedValues(action.lt)[key] + fillStashedValues(action.lt)[key], + response ) } @@ -617,7 +619,8 @@ function build (opts = {}) { const key = Object.keys(action.gt)[0] gt( delve(response, fillStashedValues(key)), - fillStashedValues(action.gt)[key] + fillStashedValues(action.gt)[key], + response ) } @@ -626,7 +629,8 @@ function build (opts = {}) { const key = Object.keys(action.lte)[0] lte( delve(response, fillStashedValues(key)), - fillStashedValues(action.lte)[key] + fillStashedValues(action.lte)[key], + response ) } @@ -635,7 +639,8 @@ function build (opts = {}) { const key = Object.keys(action.gte)[0] gte( delve(response, fillStashedValues(key)), - fillStashedValues(action.gte)[key] + fillStashedValues(action.gte)[key], + response ) } @@ -648,7 +653,8 @@ function build (opts = {}) { : delve(response, fillStashedValues(key)), key === '$body' ? action.length[key] - : fillStashedValues(action.length)[key] + : fillStashedValues(action.length)[key], + response ) } @@ -657,7 +663,8 @@ function build (opts = {}) { const isTrue = fillStashedValues(action.is_true) is_true( delve(response, isTrue), - isTrue + isTrue, + response ) } @@ -666,7 +673,8 @@ function build (opts = {}) { const isFalse = fillStashedValues(action.is_false) is_false( delve(response, isFalse), - isFalse + isFalse, + response ) } } @@ -679,46 +687,67 @@ function build (opts = {}) { * Asserts that the given value is truthy * @param {any} the value to check * @param {string} an optional message + * @param {any} debugging metadata to attach to any assertion errors * @returns {TestRunner} */ -function is_true (val, msg) { - assert.ok(val, `expect truthy value: ${msg} - value: ${JSON.stringify(val)}`) +function is_true (val, msg, response) { + try { + assert.ok((typeof val === 'string' && val.toLowerCase() === 'true') || val, `expect truthy value: ${msg} - value: ${JSON.stringify(val)}`) + } catch (err) { + err.response = JSON.stringify(response) + throw err + } } /** * Asserts that the given value is falsey * @param {any} the value to check * @param {string} an optional message + * @param {any} debugging metadata to attach to any assertion errors * @returns {TestRunner} */ -function is_false (val, msg) { - assert.ok(!val, `expect falsey value: ${msg} - value: ${JSON.stringify(val)}`) +function is_false (val, msg, response) { + try { + assert.ok((typeof val === 'string' && val.toLowerCase() === 'false') || !val, `expect falsey value: ${msg} - value: ${JSON.stringify(val)}`) + } catch (err) { + err.response = JSON.stringify(response) + throw err + } } /** * Asserts that two values are the same * @param {any} the first value * @param {any} the second value + * @param {any} debugging metadata to attach to any assertion errors * @returns {TestRunner} */ -function match (val1, val2, action) { - // both values are objects - if (typeof val1 === 'object' && typeof val2 === 'object') { - assert.deepEqual(val1, val2, typeof action === 'object' ? JSON.stringify(action) : action) - // the first value is the body as string and the second a pattern string - } else if ( - typeof val1 === 'string' && typeof val2 === 'string' && - val2.startsWith('/') && (val2.endsWith('/\n') || val2.endsWith('/')) - ) { - const regStr = val2 - .replace(/(^|[^\\])#.*/g, '$1') - .replace(/(^|[^\\])\s+/g, '$1') - .slice(1, -1) - // 'm' adds the support for multiline regex - assert.match(val1, new RegExp(regStr, 'm'), `should match pattern provided: ${val2}, but got: ${val1}`) - // everything else - } else { - assert.equal(val1, val2, `should be equal: ${val1} - ${val2}, action: ${JSON.stringify(action)}`) +function match (val1, val2, action, response) { + try { + // both values are objects + if (typeof val1 === 'object' && typeof val2 === 'object') { + assert.deepEqual(val1, val2, typeof action === 'object' ? JSON.stringify(action) : action) + // the first value is the body as string and the second a pattern string + } else if ( + typeof val1 === 'string' && typeof val2 === 'string' && + val2.startsWith('/') && (val2.endsWith('/\n') || val2.endsWith('/')) + ) { + const regStr = val2 + .replace(/(^|[^\\])#.*/g, '$1') + .replace(/(^|[^\\])\s+/g, '$1') + .slice(1, -1) + // 'm' adds the support for multiline regex + assert.match(val1, new RegExp(regStr, 'm'), `should match pattern provided: ${val2}, but got: ${val1}: ${JSON.stringify(action)}`) + } else if (typeof val1 === 'string' && typeof val2 === 'string') { + // string comparison + assert.include(val1, val2, `should include pattern provided: ${val2}, but got: ${val1}: ${JSON.stringify(action)}`) + } else { + // everything else + assert.equal(val1, val2, `should be equal: ${val1} - ${val2}, action: ${JSON.stringify(action)}`) + } + } catch (err) { + err.response = JSON.stringify(response) + throw err } } @@ -727,11 +756,17 @@ function match (val1, val2, action) { * It also verifies that the two values are numbers * @param {any} the first value * @param {any} the second value + * @param {any} debugging metadata to attach to any assertion errors * @returns {TestRunner} */ -function lt (val1, val2) { - ;[val1, val2] = getNumbers(val1, val2) - assert.ok(val1 < val2) +function lt (val1, val2, response) { + try { + ;[val1, val2] = getNumbers(val1, val2) + assert.ok(val1 < val2) + } catch (err) { + err.response = JSON.stringify(response) + throw err + } } /** @@ -739,11 +774,17 @@ function lt (val1, val2) { * It also verifies that the two values are numbers * @param {any} the first value * @param {any} the second value + * @param {any} debugging metadata to attach to any assertion errors * @returns {TestRunner} */ -function gt (val1, val2) { - ;[val1, val2] = getNumbers(val1, val2) - assert.ok(val1 > val2) +function gt (val1, val2, response) { + try { + ;[val1, val2] = getNumbers(val1, val2) + assert.ok(val1 > val2) + } catch (err) { + err.response = JSON.stringify(response) + throw err + } } /** @@ -751,11 +792,17 @@ function gt (val1, val2) { * It also verifies that the two values are numbers * @param {any} the first value * @param {any} the second value + * @param {any} debugging metadata to attach to any assertion errors * @returns {TestRunner} */ -function lte (val1, val2) { - ;[val1, val2] = getNumbers(val1, val2) - assert.ok(val1 <= val2) +function lte (val1, val2, response) { + try { + ;[val1, val2] = getNumbers(val1, val2) + assert.ok(val1 <= val2) + } catch (err) { + err.response = JSON.stringify(response) + throw err + } } /** @@ -763,26 +810,38 @@ function lte (val1, val2) { * It also verifies that the two values are numbers * @param {any} the first value * @param {any} the second value + * @param {any} debugging metadata to attach to any assertion errors * @returns {TestRunner} */ -function gte (val1, val2) { - ;[val1, val2] = getNumbers(val1, val2) - assert.ok(val1 >= val2) +function gte (val1, val2, response) { + try { + ;[val1, val2] = getNumbers(val1, val2) + assert.ok(val1 >= val2) + } catch (err) { + err.response = JSON.stringify(response) + throw err + } } /** * Asserts that the given value has the specified length * @param {string|object|array} the object to check * @param {number} the expected length + * @param {any} debugging metadata to attach to any assertion errors * @returns {TestRunner} */ -function length (val, len) { - if (typeof val === 'string' || Array.isArray(val)) { - assert.equal(val.length, len) - } else if (typeof val === 'object' && val !== null) { - assert.equal(Object.keys(val).length, len) - } else { - assert.fail(`length: the given value is invalid: ${val}`) +function length (val, len, response) { + try { + if (typeof val === 'string' || Array.isArray(val)) { + assert.equal(val.length, len) + } else if (typeof val === 'object' && val !== null) { + assert.equal(Object.keys(val).length, len) + } else { + assert.fail(`length: the given value is invalid: ${val}`) + } + } catch (err) { + err.response = JSON.stringify(response) + throw err } } @@ -813,6 +872,10 @@ function length (val, len) { */ function parseDo (action) { action = JSON.parse(JSON.stringify(action)) + + if (typeof action === 'string') action = {[action]: {}} + if (Array.isArray(action)) action = action[0] + return Object.keys(action).reduce((acc, val) => { switch (val) { case 'catch': diff --git a/test/unit/helpers/bulk.test.ts b/test/unit/helpers/bulk.test.ts index 2c3229ce9..62c297ebf 100644 --- a/test/unit/helpers/bulk.test.ts +++ b/test/unit/helpers/bulk.test.ts @@ -23,9 +23,11 @@ import { createReadStream } from 'fs' import * as http from 'http' import { join } from 'path' import split from 'split2' +import { Readable } from 'stream' import { test } from 'tap' import { Client, errors } from '../../../' import { buildServer, connection } from '../../utils' +const { sleep } = require('../../integration/helper') let clientVersion: string = require('../../../package.json').version // eslint-disable-line if (clientVersion.includes('-')) { @@ -1594,3 +1596,150 @@ test('Flush interval', t => { t.end() }) + +test(`flush timeout does not lock process when flushInterval is less than server timeout`, async t => { + const flushInterval = 500 + + async function handler (req: http.IncomingMessage, res: http.ServerResponse) { + setTimeout(() => { + res.writeHead(200, { 'content-type': 'application/json' }) + res.end(JSON.stringify({ errors: false, items: [{}] })) + }, 1000) + } + + const [{ port }, server] = await buildServer(handler) + const client = new Client({ node: `http://localhost:${port}` }) + + async function * generator () { + const data = dataset.slice() + for (const doc of data) { + await sleep(flushInterval) + yield doc + } + } + + const result = await client.helpers.bulk({ + datasource: Readable.from(generator()), + flushBytes: 1, + flushInterval: flushInterval, + concurrency: 1, + onDocument (_) { + return { + index: { _index: 'test' } + } + }, + onDrop (_) { + t.fail('This should never be called') + } + }) + + t.type(result.time, 'number') + t.type(result.bytes, 'number') + t.match(result, { + total: 3, + successful: 3, + retry: 0, + failed: 0, + aborted: false + }) + + server.stop() +}) + +test(`flush timeout does not lock process when flushInterval is greater than server timeout`, async t => { + const flushInterval = 500 + + async function handler (req: http.IncomingMessage, res: http.ServerResponse) { + setTimeout(() => { + res.writeHead(200, { 'content-type': 'application/json' }) + res.end(JSON.stringify({ errors: false, items: [{}] })) + }, 250) + } + + const [{ port }, server] = await buildServer(handler) + const client = new Client({ node: `http://localhost:${port}` }) + + async function * generator () { + const data = dataset.slice() + for (const doc of data) { + await sleep(flushInterval) + yield doc + } + } + + const result = await client.helpers.bulk({ + datasource: Readable.from(generator()), + flushBytes: 1, + flushInterval: flushInterval, + concurrency: 1, + onDocument (_) { + return { + index: { _index: 'test' } + } + }, + onDrop (_) { + t.fail('This should never be called') + } + }) + + t.type(result.time, 'number') + t.type(result.bytes, 'number') + t.match(result, { + total: 3, + successful: 3, + retry: 0, + failed: 0, + aborted: false + }) + + server.stop() +}) + +test(`flush timeout does not lock process when flushInterval is equal to server timeout`, async t => { + const flushInterval = 500 + + async function handler (req: http.IncomingMessage, res: http.ServerResponse) { + setTimeout(() => { + res.writeHead(200, { 'content-type': 'application/json' }) + res.end(JSON.stringify({ errors: false, items: [{}] })) + }, flushInterval) + } + + const [{ port }, server] = await buildServer(handler) + const client = new Client({ node: `http://localhost:${port}` }) + + async function * generator () { + const data = dataset.slice() + for (const doc of data) { + await sleep(flushInterval) + yield doc + } + } + + const result = await client.helpers.bulk({ + datasource: Readable.from(generator()), + flushBytes: 1, + flushInterval: flushInterval, + concurrency: 1, + onDocument (_) { + return { + index: { _index: 'test' } + } + }, + onDrop (_) { + t.fail('This should never be called') + } + }) + + t.type(result.time, 'number') + t.type(result.bytes, 'number') + t.match(result, { + total: 3, + successful: 3, + retry: 0, + failed: 0, + aborted: false + }) + + server.stop() +})