From c2c417a9fd84e7f77a84b0cfa079bdde40473574 Mon Sep 17 00:00:00 2001
From: Josh Mock <joshua.mock@elastic.co>
Date: Tue, 12 Dec 2023 15:40:47 -0600
Subject: [PATCH 01/11] Bump transport to 8.4.0 (#2095)

* Support for transport 8.4.0 redaction functionality

* Docs for `redaction` options
---
 docs/advanced-config.asciidoc | 88 +++++++++++++++++++++++++++++++++++
 package.json                  |  2 +-
 src/client.ts                 | 11 ++++-
 src/helpers.ts                | 12 +++--
 4 files changed, 107 insertions(+), 6 deletions(-)

diff --git a/docs/advanced-config.asciidoc b/docs/advanced-config.asciidoc
index 638aeada4..167061dce 100644
--- a/docs/advanced-config.asciidoc
+++ b/docs/advanced-config.asciidoc
@@ -91,6 +91,94 @@ const client = new Client({
 })
 ----
 
+[discrete]
+==== Redaction of potentially sensitive data
+
+When the client raises an `Error` that originated at the HTTP layer, like a `ConnectionError` or `TimeoutError`, a `meta` object is often attached to the error object that includes metadata useful for debugging, like request and response information. Because this can include potentially sensitive data, like authentication secrets in an `Authorization` header, the client takes measures to redact common sources of sensitive data when this metadata is attached and serialized.
+
+If your configuration requires extra headers or other configurations that may include sensitive data, you may want to adjust these settings to account for that.
+
+By default, the `redaction` option is set to `{ type: 'replace' }`, which recursively searches for sensitive key names, case insensitive, and replaces their values with the string `[redacted]`.
+
+[source,js]
+----
+const { Client } = require('@elastic/elasticsearch')
+
+const client = new Client({
+  cloud: { id: '<cloud-id>' },
+  auth: { apiKey: 'base64EncodedKey' },
+})
+
+try {
+  await client.indices.create({ index: 'my_index' })
+} catch (err) {
+  console.log(err.meta.meta.request.options.headers.authorization) // prints "[redacted]"
+}
+----
+
+If you would like to redact additional properties, you can include additional key names to search and replace:
+
+[source,js]
+----
+const { Client } = require('@elastic/elasticsearch')
+
+const client = new Client({
+  cloud: { id: '<cloud-id>' },
+  auth: { apiKey: 'base64EncodedKey' },
+  headers: { 'X-My-Secret-Password': 'shhh it's a secret!' },
+  redaction: {
+    type: "replace",
+    additionalKeys: ["x-my-secret-password"]
+  }
+})
+
+try {
+  await client.indices.create({ index: 'my_index' })
+} catch (err) {
+  console.log(err.meta.meta.request.options.headers['X-My-Secret-Password']) // prints "[redacted]"
+}
+----
+
+Alternatively, if you know you're not going to use the metadata at all, setting the redaction type to `remove` will remove all optional sources of potentially sensitive data entirely, or replacing them with `null` for required properties.
+
+[source,js]
+----
+const { Client } = require('@elastic/elasticsearch')
+
+const client = new Client({
+  cloud: { id: '<cloud-id>' },
+  auth: { apiKey: 'base64EncodedKey' },
+  redaction: { type: "remove" }
+})
+
+try {
+  await client.indices.create({ index: 'my_index' })
+} catch (err) {
+  console.log(err.meta.meta.request.options.headers) // undefined
+}
+----
+
+Finally, if you prefer to turn off redaction altogether, perhaps while debugging on a local developer environment, you can set the redaction type to `off`. This will revert the client to pre-8.11.0 behavior, where basic redaction is only performed during common serialization methods like `console.log` and `JSON.stringify`.
+
+WARNING: Setting `redaction.type` to `off` is not recommended in production environments.
+
+[source,js]
+----
+const { Client } = require('@elastic/elasticsearch')
+
+const client = new Client({
+  cloud: { id: '<cloud-id>' },
+  auth: { apiKey: 'base64EncodedKey' },
+  redaction: { type: "off" }
+})
+
+try {
+  await client.indices.create({ index: 'my_index' })
+} catch (err) {
+  console.log(err.meta.meta.request.options.headers.authorization) // the actual header value will be logged
+}
+----
+
 [discrete]
 ==== Migrate to v8
 
diff --git a/package.json b/package.json
index a772f791e..56332e190 100644
--- a/package.json
+++ b/package.json
@@ -83,7 +83,7 @@
     "zx": "^7.2.2"
   },
   "dependencies": {
-    "@elastic/transport": "^8.3.4",
+    "@elastic/transport": "^8.4.0",
     "tslib": "^2.4.0"
   },
   "tap": {
diff --git a/src/client.ts b/src/client.ts
index 09118d58c..50ba4942f 100644
--- a/src/client.ts
+++ b/src/client.ts
@@ -43,6 +43,7 @@ import {
   BearerAuth,
   Context
 } from '@elastic/transport/lib/types'
+import { RedactionOptions } from '@elastic/transport/lib/Transport'
 import BaseConnection, { prepareHeaders } from '@elastic/transport/lib/connection/BaseConnection'
 import SniffingTransport from './sniffingTransport'
 import Helpers from './helpers'
@@ -113,6 +114,7 @@ export interface ClientOptions {
   caFingerprint?: string
   maxResponseSize?: number
   maxCompressedResponseSize?: number
+  redaction?: RedactionOptions
 }
 
 export default class Client extends API {
@@ -186,7 +188,11 @@ export default class Client extends API {
       proxy: null,
       enableMetaHeader: true,
       maxResponseSize: null,
-      maxCompressedResponseSize: null
+      maxCompressedResponseSize: null,
+      redaction: {
+        type: 'replace',
+        additionalKeys: []
+      }
     }, opts)
 
     if (options.caFingerprint != null && isHttpConnection(opts.node ?? opts.nodes)) {
@@ -259,7 +265,8 @@ export default class Client extends API {
         jsonContentType: 'application/vnd.elasticsearch+json; compatible-with=8',
         ndjsonContentType: 'application/vnd.elasticsearch+x-ndjson; compatible-with=8',
         accept: 'application/vnd.elasticsearch+json; compatible-with=8,text/plain'
-      }
+      },
+      redaction: options.redaction
     })
 
     this.helpers = new Helpers({
diff --git a/src/helpers.ts b/src/helpers.ts
index 0bd1b1c5c..efad8b49b 100644
--- a/src/helpers.ts
+++ b/src/helpers.ts
@@ -196,8 +196,11 @@ export default class Helpers {
       await sleep(wait)
     }
     assert(response !== undefined, 'The response is undefined, please file a bug report')
+
+    const { redaction = { type: 'replace' } } = options
+    const errorOptions = { redaction }
     if (response.statusCode === 429) {
-      throw new ResponseError(response)
+      throw new ResponseError(response, errorOptions)
     }
 
     let scroll_id = response.body._scroll_id
@@ -237,7 +240,7 @@ export default class Helpers {
         await sleep(wait)
       }
       if (response.statusCode === 429) {
-        throw new ResponseError(response)
+        throw new ResponseError(response, errorOptions)
       }
     }
 
@@ -289,6 +292,9 @@ export default class Helpers {
     } = options
     reqOptions.meta = true
 
+    const { redaction = { type: 'replace' } } = reqOptions
+    const errorOptions = { redaction }
+
     let stopReading = false
     let stopError: Error | null = null
     let timeoutRef = null
@@ -502,7 +508,7 @@ export default class Helpers {
               // @ts-expect-error
               addDocumentsGetter(result)
               if (response.status != null && response.status >= 400) {
-                callbacks[i](new ResponseError(result), result)
+                callbacks[i](new ResponseError(result, errorOptions), result)
               } else {
                 callbacks[i](null, result)
               }

From 1fb789862dd36e85c211bf1d32aa90c3454c5dc2 Mon Sep 17 00:00:00 2001
From: Josh Mock <joshua.mock@elastic.co>
Date: Tue, 12 Dec 2023 16:06:03 -0600
Subject: [PATCH 02/11] 8.11.0 changelog (#2097)

* Changelog for 8.11.0

* Add redaction docs link to changelog
---
 docs/advanced-config.asciidoc |  1 +
 docs/changelog.asciidoc       | 19 +++++++++++++++++++
 package.json                  |  4 ++--
 3 files changed, 22 insertions(+), 2 deletions(-)

diff --git a/docs/advanced-config.asciidoc b/docs/advanced-config.asciidoc
index 167061dce..b3c9388a4 100644
--- a/docs/advanced-config.asciidoc
+++ b/docs/advanced-config.asciidoc
@@ -92,6 +92,7 @@ const client = new Client({
 ----
 
 [discrete]
+[[redaction]]
 ==== Redaction of potentially sensitive data
 
 When the client raises an `Error` that originated at the HTTP layer, like a `ConnectionError` or `TimeoutError`, a `meta` object is often attached to the error object that includes metadata useful for debugging, like request and response information. Because this can include potentially sensitive data, like authentication secrets in an `Authorization` header, the client takes measures to redact common sources of sensitive data when this metadata is attached and serialized.
diff --git a/docs/changelog.asciidoc b/docs/changelog.asciidoc
index b82c397da..0d79214b2 100644
--- a/docs/changelog.asciidoc
+++ b/docs/changelog.asciidoc
@@ -1,6 +1,25 @@
 [[changelog-client]]
 == Release notes
 
+[discrete]
+=== 8.11.0
+
+[discrete]
+=== Features
+
+[discrete]
+===== Support for Elasticsearch `v8.11.0`
+
+You can find all the API changes
+https://www.elastic.co/guide/en/elasticsearch/reference/8.11/release-notes-8.11.0.html[here].
+
+[discrete]
+===== Enhanced support for redacting potentially sensitive data https://github.com/elastic/elasticsearch-js/pull/2095[#2095]
+
+`@elastic/transport` https://github.com/elastic/elastic-transport-js/releases/tag/v8.4.0[version 8.4.0] introduces enhanced measures for ensuring that request metadata attached to some `Error` objects is redacted. This functionality is primarily to address custom logging solutions that don't use common serialization methods like `JSON.stringify`, `console.log`, or `util.inspect`, which were already accounted for.
+
+See <<redaction>> for more information.
+
 [discrete]
 === 8.10.0
 
diff --git a/package.json b/package.json
index 56332e190..add6446fa 100644
--- a/package.json
+++ b/package.json
@@ -1,7 +1,7 @@
 {
   "name": "@elastic/elasticsearch",
-  "version": "8.10.3",
-  "versionCanary": "8.10.3-canary.1",
+  "version": "8.11.0",
+  "versionCanary": "8.11.0-canary.0",
   "description": "The official Elasticsearch client for Node.js",
   "main": "index.js",
   "types": "index.d.ts",

From 51323e769dbaedc8c584ca7ad6cbdf3a4722234b Mon Sep 17 00:00:00 2001
From: Josh Mock <joshua.mock@elastic.co>
Date: Wed, 13 Dec 2023 11:20:04 -0600
Subject: [PATCH 03/11] Github action for publishing to npm with provenance
 metadata (#2103)

---
 .github/workflows/npm-publish.yml | 27 +++++++++++++++++++++++++++
 1 file changed, 27 insertions(+)
 create mode 100644 .github/workflows/npm-publish.yml

diff --git a/.github/workflows/npm-publish.yml b/.github/workflows/npm-publish.yml
new file mode 100644
index 000000000..73a7d36c2
--- /dev/null
+++ b/.github/workflows/npm-publish.yml
@@ -0,0 +1,27 @@
+name: Publish Package to npm
+on:
+  workflow_dispatch:
+    inputs:
+      branch:
+        description: 'Git branch to build and publish'
+        required: true
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      id-token: write
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.inputs.branch }}
+      - uses: actions/setup-node@v3
+        with:
+          node-version: '20.x'
+          registry-url: 'https://registry.npmjs.org'
+      - run: npm install -g npm
+      - run: npm install
+      - run: npm test
+      - run: npm publish --provenance --access public
+        env:
+          NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}

From d3f22f1e14d292849d55eaea356d6f28ea7abfec Mon Sep 17 00:00:00 2001
From: Josh Mock <joshua.mock@elastic.co>
Date: Thu, 14 Dec 2023 09:46:04 -0600
Subject: [PATCH 04/11] Add doc for closing connections (#2104)

---
 docs/connecting.asciidoc | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/docs/connecting.asciidoc b/docs/connecting.asciidoc
index 77c2e7d11..15007ceb3 100644
--- a/docs/connecting.asciidoc
+++ b/docs/connecting.asciidoc
@@ -12,6 +12,7 @@ This page contains the information you need to connect and use the Client with
 * <<client-connect-proxy, Connecting through a proxy>>
 * <<client-error-handling, Handling errors>>
 * <<keep-alive, Keep-alive connections>>
+* <<close-connections, Closing a client's connections>>
 * <<product-check, Automatic product check>>
 
 [[authentication]]
@@ -691,6 +692,20 @@ const client = new Client({
 })
 ----
 
+[discrete]
+[[close-connections]]
+=== Closing a client's connections
+
+If you would like to close all open connections being managed by an instance of the client, use the `close()` function:
+
+[source,js]
+----
+const client = new Client({
+  node: 'http://localhost:9200'
+});
+client.close();
+----
+
 [discrete]
 [[product-check]]
 === Automatic product check

From 4aaf49b6ea86a0906dc7768b4d227f10b6bd8198 Mon Sep 17 00:00:00 2001
From: Josh Mock <joshua.mock@elastic.co>
Date: Thu, 14 Dec 2023 16:35:37 -0600
Subject: [PATCH 05/11] Integration test improvements (#2109)

* Improvements to integrations

Borrowed largely from https://github.com/elastic/elasticsearch-serverless-js/pull/38

* Bump all the things to 8.12.0

* Split Dockerfile copy into two layers

* Fix test cron names
---
 .buildkite/Dockerfile           |   4 +-
 .buildkite/pipeline.yml         |   2 +-
 .dockerignore                   |   2 +
 catalog-info.yaml               |   8 +-
 package.json                    |   6 +-
 test/integration/index.js       |   4 +-
 test/integration/test-runner.js | 165 ++++++++++++++++++++++----------
 7 files changed, 130 insertions(+), 61 deletions(-)

diff --git a/.buildkite/Dockerfile b/.buildkite/Dockerfile
index 5608747b6..6d44e2211 100644
--- a/.buildkite/Dockerfile
+++ b/.buildkite/Dockerfile
@@ -10,5 +10,7 @@ RUN apt-get clean -y && \
 
 WORKDIR /usr/src/app
 
-COPY . .
+COPY package.json .
 RUN npm install --production=false
+
+COPY . .
diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index 32b37b6c6..1dca14548 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -6,7 +6,7 @@ steps:
     env:
       NODE_VERSION: "{{ matrix.nodejs }}"
       TEST_SUITE: "{{ matrix.suite }}"
-      STACK_VERSION: 8.10.3-SNAPSHOT
+      STACK_VERSION: 8.12.0-SNAPSHOT
     matrix:
       setup:
         suite:
diff --git a/.dockerignore b/.dockerignore
index 54eb2a95a..e34f9ff27 100644
--- a/.dockerignore
+++ b/.dockerignore
@@ -3,3 +3,5 @@ npm-debug.log
 test/benchmarks
 elasticsearch
 .git
+lib
+junit-output
diff --git a/catalog-info.yaml b/catalog-info.yaml
index b8bbd36ff..80c943cd8 100644
--- a/catalog-info.yaml
+++ b/catalog-info.yaml
@@ -42,9 +42,9 @@ spec:
         main_semi_daily:
           branch: 'main'
           cronline: '0 */12 * * *'
-        8_9_semi_daily:
-          branch: '8.9'
+        8_12_semi_daily:
+          branch: '8.12'
           cronline: '0 */12 * * *'
-        8_8_daily:
-          branch: '8.8'
+        8_11_daily:
+          branch: '8.11'
           cronline: '@daily'
diff --git a/package.json b/package.json
index add6446fa..7e3d637b0 100644
--- a/package.json
+++ b/package.json
@@ -1,7 +1,7 @@
 {
   "name": "@elastic/elasticsearch",
-  "version": "8.11.0",
-  "versionCanary": "8.11.0-canary.0",
+  "version": "8.12.0",
+  "versionCanary": "8.12.0-canary.0",
   "description": "The official Elasticsearch client for Node.js",
   "main": "index.js",
   "types": "index.d.ts",
@@ -93,4 +93,4 @@
     "coverage": false,
     "check-coverage": false
   }
-}
+}
\ No newline at end of file
diff --git a/test/integration/index.js b/test/integration/index.js
index defdb400f..b07ddd2d7 100644
--- a/test/integration/index.js
+++ b/test/integration/index.js
@@ -317,7 +317,7 @@ async function start ({ client, isXPack }) {
         if (name === 'setup' || name === 'teardown') continue
         if (options.test && !name.endsWith(options.test)) continue
 
-        const junitTestCase = junitTestSuite.testcase(name, `node_${process.version}/${cleanPath}`)
+        const junitTestCase = junitTestSuite.testcase(name, `node_${process.version}: ${cleanPath}`)
 
         stats.total += 1
         if (shouldSkip(isXPack, file, name)) {
@@ -336,6 +336,7 @@ async function start ({ client, isXPack }) {
           junitTestSuite.end()
           junitTestSuites.end()
           generateJunitXmlReport(junit, isXPack ? 'platinum' : 'free')
+          err.meta = JSON.stringify(err.meta ?? {}, null, 2)
           console.error(err)
 
           if (options.bail) {
@@ -374,6 +375,7 @@ async function start ({ client, isXPack }) {
   - Total: ${stats.total}
   - Skip: ${stats.skip}
   - Pass: ${stats.pass}
+  - Fail: ${stats.total - (stats.pass + stats.skip)}
   - Assertions: ${stats.assertions}
   `)
 }
diff --git a/test/integration/test-runner.js b/test/integration/test-runner.js
index 64570945a..ce80da43e 100644
--- a/test/integration/test-runner.js
+++ b/test/integration/test-runner.js
@@ -593,13 +593,14 @@ function build (opts = {}) {
         const key = Object.keys(action.match)[0]
         match(
           // in some cases, the yaml refers to the body with an empty string
-          key === '$body' || key === ''
+          key.split('.')[0] === '$body' || key === ''
             ? response
             : delve(response, fillStashedValues(key)),
-          key === '$body'
+          key.split('.')[0] === '$body'
             ? action.match[key]
             : fillStashedValues(action.match)[key],
-          action.match
+          action.match,
+          response
         )
       }
 
@@ -608,7 +609,8 @@ function build (opts = {}) {
         const key = Object.keys(action.lt)[0]
         lt(
           delve(response, fillStashedValues(key)),
-          fillStashedValues(action.lt)[key]
+          fillStashedValues(action.lt)[key],
+          response
         )
       }
 
@@ -617,7 +619,8 @@ function build (opts = {}) {
         const key = Object.keys(action.gt)[0]
         gt(
           delve(response, fillStashedValues(key)),
-          fillStashedValues(action.gt)[key]
+          fillStashedValues(action.gt)[key],
+          response
         )
       }
 
@@ -626,7 +629,8 @@ function build (opts = {}) {
         const key = Object.keys(action.lte)[0]
         lte(
           delve(response, fillStashedValues(key)),
-          fillStashedValues(action.lte)[key]
+          fillStashedValues(action.lte)[key],
+          response
         )
       }
 
@@ -635,7 +639,8 @@ function build (opts = {}) {
         const key = Object.keys(action.gte)[0]
         gte(
           delve(response, fillStashedValues(key)),
-          fillStashedValues(action.gte)[key]
+          fillStashedValues(action.gte)[key],
+          response
         )
       }
 
@@ -648,7 +653,8 @@ function build (opts = {}) {
             : delve(response, fillStashedValues(key)),
           key === '$body'
             ? action.length[key]
-            : fillStashedValues(action.length)[key]
+            : fillStashedValues(action.length)[key],
+          response
         )
       }
 
@@ -657,7 +663,8 @@ function build (opts = {}) {
         const isTrue = fillStashedValues(action.is_true)
         is_true(
           delve(response, isTrue),
-          isTrue
+          isTrue,
+          response
         )
       }
 
@@ -666,7 +673,8 @@ function build (opts = {}) {
         const isFalse = fillStashedValues(action.is_false)
         is_false(
           delve(response, isFalse),
-          isFalse
+          isFalse,
+          response
         )
       }
     }
@@ -679,46 +687,67 @@ function build (opts = {}) {
  * Asserts that the given value is truthy
  * @param {any} the value to check
  * @param {string} an optional message
+ * @param {any} debugging metadata to attach to any assertion errors
  * @returns {TestRunner}
  */
-function is_true (val, msg) {
-  assert.ok(val, `expect truthy value: ${msg} - value: ${JSON.stringify(val)}`)
+function is_true (val, msg, response) {
+  try {
+    assert.ok((typeof val === 'string' && val.toLowerCase() === 'true') || val, `expect truthy value: ${msg} - value: ${JSON.stringify(val)}`)
+  } catch (err) {
+    err.response = JSON.stringify(response)
+    throw err
+  }
 }
 
 /**
  * Asserts that the given value is falsey
  * @param {any} the value to check
  * @param {string} an optional message
+ * @param {any} debugging metadata to attach to any assertion errors
  * @returns {TestRunner}
  */
-function is_false (val, msg) {
-  assert.ok(!val, `expect falsey value: ${msg} - value: ${JSON.stringify(val)}`)
+function is_false (val, msg, response) {
+  try {
+    assert.ok((typeof val === 'string' && val.toLowerCase() === 'false') || !val, `expect falsey value: ${msg} - value: ${JSON.stringify(val)}`)
+  } catch (err) {
+    err.response = JSON.stringify(response)
+    throw err
+  }
 }
 
 /**
  * Asserts that two values are the same
  * @param {any} the first value
  * @param {any} the second value
+ * @param {any} debugging metadata to attach to any assertion errors
  * @returns {TestRunner}
  */
-function match (val1, val2, action) {
-  // both values are objects
-  if (typeof val1 === 'object' && typeof val2 === 'object') {
-    assert.deepEqual(val1, val2, typeof action === 'object' ? JSON.stringify(action) : action)
-  // the first value is the body as string and the second a pattern string
-  } else if (
-    typeof val1 === 'string' && typeof val2 === 'string' &&
-    val2.startsWith('/') && (val2.endsWith('/\n') || val2.endsWith('/'))
-  ) {
-    const regStr = val2
-      .replace(/(^|[^\\])#.*/g, '$1')
-      .replace(/(^|[^\\])\s+/g, '$1')
-      .slice(1, -1)
-    // 'm' adds the support for multiline regex
-    assert.match(val1, new RegExp(regStr, 'm'), `should match pattern provided: ${val2}, but got: ${val1}`)
-  // everything else
-  } else {
-    assert.equal(val1, val2, `should be equal: ${val1} - ${val2}, action: ${JSON.stringify(action)}`)
+function match (val1, val2, action, response) {
+  try {
+    // both values are objects
+    if (typeof val1 === 'object' && typeof val2 === 'object') {
+      assert.deepEqual(val1, val2, typeof action === 'object' ? JSON.stringify(action) : action)
+    // the first value is the body as string and the second a pattern string
+    } else if (
+      typeof val1 === 'string' && typeof val2 === 'string' &&
+      val2.startsWith('/') && (val2.endsWith('/\n') || val2.endsWith('/'))
+    ) {
+      const regStr = val2
+        .replace(/(^|[^\\])#.*/g, '$1')
+        .replace(/(^|[^\\])\s+/g, '$1')
+        .slice(1, -1)
+      // 'm' adds the support for multiline regex
+      assert.match(val1, new RegExp(regStr, 'm'), `should match pattern provided: ${val2}, but got: ${val1}: ${JSON.stringify(action)}`)
+    } else if (typeof val1 === 'string' && typeof val2 === 'string') {
+      // string comparison
+      assert.include(val1, val2, `should include pattern provided: ${val2}, but got: ${val1}: ${JSON.stringify(action)}`)
+    } else {
+      // everything else
+      assert.equal(val1, val2, `should be equal: ${val1} - ${val2}, action: ${JSON.stringify(action)}`)
+    }
+  } catch (err) {
+    err.response = JSON.stringify(response)
+    throw err
   }
 }
 
@@ -727,11 +756,17 @@ function match (val1, val2, action) {
  * It also verifies that the two values are numbers
  * @param {any} the first value
  * @param {any} the second value
+ * @param {any} debugging metadata to attach to any assertion errors
  * @returns {TestRunner}
  */
-function lt (val1, val2) {
-  ;[val1, val2] = getNumbers(val1, val2)
-  assert.ok(val1 < val2)
+function lt (val1, val2, response) {
+  try {
+    ;[val1, val2] = getNumbers(val1, val2)
+    assert.ok(val1 < val2)
+  } catch (err) {
+    err.response = JSON.stringify(response)
+    throw err
+  }
 }
 
 /**
@@ -739,11 +774,17 @@ function lt (val1, val2) {
  * It also verifies that the two values are numbers
  * @param {any} the first value
  * @param {any} the second value
+ * @param {any} debugging metadata to attach to any assertion errors
  * @returns {TestRunner}
  */
-function gt (val1, val2) {
-  ;[val1, val2] = getNumbers(val1, val2)
-  assert.ok(val1 > val2)
+function gt (val1, val2, response) {
+  try {
+    ;[val1, val2] = getNumbers(val1, val2)
+    assert.ok(val1 > val2)
+  } catch (err) {
+    err.response = JSON.stringify(response)
+    throw err
+  }
 }
 
 /**
@@ -751,11 +792,17 @@ function gt (val1, val2) {
  * It also verifies that the two values are numbers
  * @param {any} the first value
  * @param {any} the second value
+ * @param {any} debugging metadata to attach to any assertion errors
  * @returns {TestRunner}
  */
-function lte (val1, val2) {
-  ;[val1, val2] = getNumbers(val1, val2)
-  assert.ok(val1 <= val2)
+function lte (val1, val2, response) {
+  try {
+    ;[val1, val2] = getNumbers(val1, val2)
+    assert.ok(val1 <= val2)
+  } catch (err) {
+    err.response = JSON.stringify(response)
+    throw err
+  }
 }
 
 /**
@@ -763,26 +810,38 @@ function lte (val1, val2) {
  * It also verifies that the two values are numbers
  * @param {any} the first value
  * @param {any} the second value
+ * @param {any} debugging metadata to attach to any assertion errors
  * @returns {TestRunner}
 */
-function gte (val1, val2) {
-  ;[val1, val2] = getNumbers(val1, val2)
-  assert.ok(val1 >= val2)
+function gte (val1, val2, response) {
+  try {
+    ;[val1, val2] = getNumbers(val1, val2)
+    assert.ok(val1 >= val2)
+  } catch (err) {
+    err.response = JSON.stringify(response)
+    throw err
+  }
 }
 
 /**
  * Asserts that the given value has the specified length
  * @param {string|object|array} the object to check
  * @param {number} the expected length
+ * @param {any} debugging metadata to attach to any assertion errors
  * @returns {TestRunner}
  */
-function length (val, len) {
-  if (typeof val === 'string' || Array.isArray(val)) {
-    assert.equal(val.length, len)
-  } else if (typeof val === 'object' && val !== null) {
-    assert.equal(Object.keys(val).length, len)
-  } else {
-    assert.fail(`length: the given value is invalid: ${val}`)
+function length (val, len, response) {
+  try {
+    if (typeof val === 'string' || Array.isArray(val)) {
+      assert.equal(val.length, len)
+    } else if (typeof val === 'object' && val !== null) {
+      assert.equal(Object.keys(val).length, len)
+    } else {
+      assert.fail(`length: the given value is invalid: ${val}`)
+    }
+  } catch (err) {
+    err.response = JSON.stringify(response)
+    throw err
   }
 }
 
@@ -813,6 +872,10 @@ function length (val, len) {
  */
 function parseDo (action) {
   action = JSON.parse(JSON.stringify(action))
+
+  if (typeof action === 'string') action = {[action]: {}}
+  if (Array.isArray(action)) action = action[0]
+
   return Object.keys(action).reduce((acc, val) => {
     switch (val) {
       case 'catch':

From 5413eb5f35d0f3cbda483b5800475d9a3da59038 Mon Sep 17 00:00:00 2001
From: Josh Mock <joshua.mock@elastic.co>
Date: Thu, 14 Dec 2023 17:19:20 -0600
Subject: [PATCH 06/11] Add missing snippets (#2113)

For https://github.com/elastic/clients-team/issues/728
---
 .../36b86b97feedcf5632824eefc251d6ed.asciidoc | 12 ++++++
 .../8575c966b004fb124c7afd6bb5827b50.asciidoc | 13 ++++++
 .../bcc75fc01b45e482638c65b8fbdf09fa.asciidoc |  7 +++
 .../d04f0c8c44e8b4fb55f2e7d9d05977e7.asciidoc | 43 +++++++++++++++++++
 4 files changed, 75 insertions(+)
 create mode 100644 docs/doc_examples/36b86b97feedcf5632824eefc251d6ed.asciidoc
 create mode 100644 docs/doc_examples/8575c966b004fb124c7afd6bb5827b50.asciidoc
 create mode 100644 docs/doc_examples/bcc75fc01b45e482638c65b8fbdf09fa.asciidoc
 create mode 100644 docs/doc_examples/d04f0c8c44e8b4fb55f2e7d9d05977e7.asciidoc

diff --git a/docs/doc_examples/36b86b97feedcf5632824eefc251d6ed.asciidoc b/docs/doc_examples/36b86b97feedcf5632824eefc251d6ed.asciidoc
new file mode 100644
index 000000000..408ce2f71
--- /dev/null
+++ b/docs/doc_examples/36b86b97feedcf5632824eefc251d6ed.asciidoc
@@ -0,0 +1,12 @@
+[source,js]
+----
+const response = await client.search({
+  index: 'books',
+  query: {
+    match: {
+      name: 'brave'
+    }
+  }
+})
+console.log(response)
+----
diff --git a/docs/doc_examples/8575c966b004fb124c7afd6bb5827b50.asciidoc b/docs/doc_examples/8575c966b004fb124c7afd6bb5827b50.asciidoc
new file mode 100644
index 000000000..d99bd96dc
--- /dev/null
+++ b/docs/doc_examples/8575c966b004fb124c7afd6bb5827b50.asciidoc
@@ -0,0 +1,13 @@
+[source,js]
+----
+const response = await client.index({
+  index: 'books',
+  document: {
+    name: 'Snow Crash',
+    author: 'Neal Stephenson',
+    release_date: '1992-06-01',
+    page_count: 470,
+  }
+})
+console.log(response)
+----
diff --git a/docs/doc_examples/bcc75fc01b45e482638c65b8fbdf09fa.asciidoc b/docs/doc_examples/bcc75fc01b45e482638c65b8fbdf09fa.asciidoc
new file mode 100644
index 000000000..1708d0956
--- /dev/null
+++ b/docs/doc_examples/bcc75fc01b45e482638c65b8fbdf09fa.asciidoc
@@ -0,0 +1,7 @@
+[source,js]
+----
+const response = await client.search({
+  index: 'books'
+})
+console.log(response)
+----
diff --git a/docs/doc_examples/d04f0c8c44e8b4fb55f2e7d9d05977e7.asciidoc b/docs/doc_examples/d04f0c8c44e8b4fb55f2e7d9d05977e7.asciidoc
new file mode 100644
index 000000000..e5ce437b2
--- /dev/null
+++ b/docs/doc_examples/d04f0c8c44e8b4fb55f2e7d9d05977e7.asciidoc
@@ -0,0 +1,43 @@
+[source,js]
+----
+const response = await client.bulk({
+  operations: [
+    { index: { _index: 'books' } },
+    {
+      name: 'Revelation Space',
+      author: 'Alastair Reynolds',
+      release_date: '2000-03-15',
+      page_count: 585,
+    },
+    { index: { _index: 'books' } },
+    {
+      name: '1984',
+      author: 'George Orwell',
+      release_date: '1985-06-01',
+      page_count: 328,
+    },
+    { index: { _index: 'books' } },
+    {
+      name: 'Fahrenheit 451',
+      author: 'Ray Bradbury',
+      release_date: '1953-10-15',
+      page_count: 227,
+    },
+    { index: { _index: 'books' } },
+    {
+      name: 'Brave New World',
+      author: 'Aldous Huxley',
+      release_date: '1932-06-01',
+      page_count: 268,
+    },
+    { index: { _index: 'books' } },
+    {
+      name: 'The Handmaids Tale',
+      author: 'Margaret Atwood',
+      release_date: '1985-06-01',
+      page_count: 311,
+    }
+  ]
+})
+console.log(response)
+----

From 6eabf37097c5dc32a618e47fdc0c968377ced314 Mon Sep 17 00:00:00 2001
From: Enrico Zimuel <e.zimuel@gmail.com>
Date: Thu, 4 Jan 2024 13:13:26 +0100
Subject: [PATCH 07/11] Improved the body BC break description in
 request/response for 8.x documentation (#2117)

* Improved the body bc break in 8.x documentation

* Removed just in the sentence
---
 docs/changelog.asciidoc | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/docs/changelog.asciidoc b/docs/changelog.asciidoc
index 0d79214b2..1b7e68c5a 100644
--- a/docs/changelog.asciidoc
+++ b/docs/changelog.asciidoc
@@ -361,6 +361,9 @@ The client API leaks HTTP-related notions in many places, and removing them woul
 
 This could be a rather big breaking change, so a double solution could be used during the 8.x lifecycle. (accepting body keys without them being wrapped in the body as well as the current solution).
 
+To convert code from 7.x, you need to remove the `body` parameter in all the endpoints request.
+For instance, this is an example for the `search` endpoint:
+
 [source,js]
 ----
 // from
@@ -399,6 +402,12 @@ If you weren't extending the internals of the client, this won't be a breaking c
 The client API leaks HTTP-related notions in many places, and removing them would definitely improve the DX.
 The client will expose a new request-specific option to still get the full response details.
 
+The new behaviour returns the `body` value directly as response.
+If you want to have the 7.x response format, you need to add `meta : true` in the request.
+This will return all the HTTP meta information, including the `body`.
+
+For instance, this is an example for the `search` endpoint:
+
 [source,js]
 ----
 // from

From 57ee5cf6c257289557e7ed9e15da66995aae4240 Mon Sep 17 00:00:00 2001
From: Quentin Pradet <quentin.pradet@elastic.co>
Date: Wed, 31 Jan 2024 13:37:05 +0400
Subject: [PATCH 08/11] 8.12.0 changelog (#2125)

---
 docs/changelog.asciidoc | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/docs/changelog.asciidoc b/docs/changelog.asciidoc
index 1b7e68c5a..4dbf11907 100644
--- a/docs/changelog.asciidoc
+++ b/docs/changelog.asciidoc
@@ -1,6 +1,18 @@
 [[changelog-client]]
 == Release notes
 
+[discrete]
+=== 8.12.0
+
+[discrete]
+=== Features
+
+[discrete]
+===== Support for Elasticsearch `v8.12.0`
+
+You can find all the API changes
+https://www.elastic.co/guide/en/elasticsearch/reference/8.12/release-notes-8.12.0.html[here].
+
 [discrete]
 === 8.11.0
 

From 1607a0d3f78a6e60c44d62c5326dc36dedb972ea Mon Sep 17 00:00:00 2001
From: Josh Mock <joshua.mock@elastic.co>
Date: Mon, 5 Feb 2024 23:58:21 -0600
Subject: [PATCH 09/11] Fix hang in bulk helper semaphore when server responses
 are slower than flushInterval (#2027)

* Set version to 8.10.1

* Add tests for bulk helper with various flush and server timeouts

* Copy and empty bulkBody when flushBytes is reached

Before it was waiting until after semaphore resolved, then sending with
a reference to bulkBody. If flushInterval is reached after `await
semaphore()` but before `send(bulkBody)`, onFlushTimeout is "stealing"
bulkBody so that there is nothing left in bulkBody for the flushBytes
block to send, causing an indefinite hang for a promise that does not
resolve.

* comment typo fixes

---------

Co-authored-by: Quentin Pradet <quentin.pradet@elastic.co>
---
 src/helpers.ts                 |  13 +--
 test/unit/helpers/bulk.test.ts | 149 +++++++++++++++++++++++++++++++++
 2 files changed, 156 insertions(+), 6 deletions(-)

diff --git a/src/helpers.ts b/src/helpers.ts
index efad8b49b..fbf4ff334 100644
--- a/src/helpers.ts
+++ b/src/helpers.ts
@@ -624,7 +624,7 @@ export default class Helpers {
       let chunkBytes = 0
       timeoutRef = setTimeout(onFlushTimeout, flushInterval) // eslint-disable-line
 
-      // @ts-expect-error datasoruce is an iterable
+      // @ts-expect-error datasource is an iterable
       for await (const chunk of datasource) {
         if (shouldAbort) break
         timeoutRef.refresh()
@@ -656,15 +656,16 @@ export default class Helpers {
 
         if (chunkBytes >= flushBytes) {
           stats.bytes += chunkBytes
-          const send = await semaphore()
-          send(bulkBody.slice())
+          const bulkBodyCopy = bulkBody.slice()
           bulkBody.length = 0
           chunkBytes = 0
+          const send = await semaphore()
+          send(bulkBodyCopy)
         }
       }
 
       clearTimeout(timeoutRef)
-      // In some cases the previos http call does not have finished,
+      // In some cases the previous http call has not finished,
       // or we didn't reach the flush bytes threshold, so we force one last operation.
       if (!shouldAbort && chunkBytes > 0) {
         const send = await semaphore()
@@ -708,8 +709,8 @@ export default class Helpers {
     // to guarantee that no more than the number of operations
     // allowed to run at the same time are executed.
     // It returns a semaphore function which resolves in the next tick
-    // if we didn't reach the maximim concurrency yet, otherwise it returns
-    // a promise that resolves as soon as one of the running request has finshed.
+    // if we didn't reach the maximum concurrency yet, otherwise it returns
+    // a promise that resolves as soon as one of the running requests has finished.
     // The semaphore function resolves a send function, which will be used
     // to send the actual bulk request.
     // It also returns a finish function, which returns a promise that is resolved
diff --git a/test/unit/helpers/bulk.test.ts b/test/unit/helpers/bulk.test.ts
index 2c3229ce9..62c297ebf 100644
--- a/test/unit/helpers/bulk.test.ts
+++ b/test/unit/helpers/bulk.test.ts
@@ -23,9 +23,11 @@ import { createReadStream } from 'fs'
 import * as http from 'http'
 import { join } from 'path'
 import split from 'split2'
+import { Readable } from 'stream'
 import { test } from 'tap'
 import { Client, errors } from '../../../'
 import { buildServer, connection } from '../../utils'
+const { sleep } = require('../../integration/helper')
 
 let clientVersion: string = require('../../../package.json').version // eslint-disable-line
 if (clientVersion.includes('-')) {
@@ -1594,3 +1596,150 @@ test('Flush interval', t => {
 
   t.end()
 })
+
+test(`flush timeout does not lock process when flushInterval is less than server timeout`, async t => {
+  const flushInterval = 500
+
+  async function handler (req: http.IncomingMessage, res: http.ServerResponse) {
+    setTimeout(() => {
+      res.writeHead(200, { 'content-type': 'application/json' })
+      res.end(JSON.stringify({ errors: false, items: [{}] }))
+    }, 1000)
+  }
+
+  const [{ port }, server] = await buildServer(handler)
+  const client = new Client({ node: `http://localhost:${port}` })
+
+  async function * generator () {
+    const data = dataset.slice()
+    for (const doc of data) {
+      await sleep(flushInterval)
+      yield doc
+    }
+  }
+
+  const result = await client.helpers.bulk({
+    datasource: Readable.from(generator()),
+    flushBytes: 1,
+    flushInterval: flushInterval,
+    concurrency: 1,
+    onDocument (_) {
+      return {
+        index: { _index: 'test' }
+      }
+    },
+    onDrop (_) {
+      t.fail('This should never be called')
+    }
+  })
+
+  t.type(result.time, 'number')
+  t.type(result.bytes, 'number')
+  t.match(result, {
+    total: 3,
+    successful: 3,
+    retry: 0,
+    failed: 0,
+    aborted: false
+  })
+
+  server.stop()
+})
+
+test(`flush timeout does not lock process when flushInterval is greater than server timeout`, async t => {
+  const flushInterval = 500
+
+  async function handler (req: http.IncomingMessage, res: http.ServerResponse) {
+    setTimeout(() => {
+      res.writeHead(200, { 'content-type': 'application/json' })
+      res.end(JSON.stringify({ errors: false, items: [{}] }))
+    }, 250)
+  }
+
+  const [{ port }, server] = await buildServer(handler)
+  const client = new Client({ node: `http://localhost:${port}` })
+
+  async function * generator () {
+    const data = dataset.slice()
+    for (const doc of data) {
+      await sleep(flushInterval)
+      yield doc
+    }
+  }
+
+  const result = await client.helpers.bulk({
+    datasource: Readable.from(generator()),
+    flushBytes: 1,
+    flushInterval: flushInterval,
+    concurrency: 1,
+    onDocument (_) {
+      return {
+        index: { _index: 'test' }
+      }
+    },
+    onDrop (_) {
+      t.fail('This should never be called')
+    }
+  })
+
+  t.type(result.time, 'number')
+  t.type(result.bytes, 'number')
+  t.match(result, {
+    total: 3,
+    successful: 3,
+    retry: 0,
+    failed: 0,
+    aborted: false
+  })
+
+  server.stop()
+})
+
+test(`flush timeout does not lock process when flushInterval is equal to server timeout`, async t => {
+  const flushInterval = 500
+
+  async function handler (req: http.IncomingMessage, res: http.ServerResponse) {
+    setTimeout(() => {
+      res.writeHead(200, { 'content-type': 'application/json' })
+      res.end(JSON.stringify({ errors: false, items: [{}] }))
+    }, flushInterval)
+  }
+
+  const [{ port }, server] = await buildServer(handler)
+  const client = new Client({ node: `http://localhost:${port}` })
+
+  async function * generator () {
+    const data = dataset.slice()
+    for (const doc of data) {
+      await sleep(flushInterval)
+      yield doc
+    }
+  }
+
+  const result = await client.helpers.bulk({
+    datasource: Readable.from(generator()),
+    flushBytes: 1,
+    flushInterval: flushInterval,
+    concurrency: 1,
+    onDocument (_) {
+      return {
+        index: { _index: 'test' }
+      }
+    },
+    onDrop (_) {
+      t.fail('This should never be called')
+    }
+  })
+
+  t.type(result.time, 'number')
+  t.type(result.bytes, 'number')
+  t.match(result, {
+    total: 3,
+    successful: 3,
+    retry: 0,
+    failed: 0,
+    aborted: false
+  })
+
+  server.stop()
+})

From 8df91fce7cd757a25ca98e50cb24422748b15ef1 Mon Sep 17 00:00:00 2001
From: Josh Mock <joshua.mock@elastic.co>
Date: Fri, 23 Feb 2024 13:18:01 -0600
Subject: [PATCH 10/11] Upgrade transport to 8.4.1 (#2137)

---
 package.json | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/package.json b/package.json
index 7e3d637b0..0ecd601ca 100644
--- a/package.json
+++ b/package.json
@@ -83,7 +83,7 @@
     "zx": "^7.2.2"
   },
   "dependencies": {
-    "@elastic/transport": "^8.4.0",
+    "@elastic/transport": "^8.4.1",
     "tslib": "^2.4.0"
   },
   "tap": {
@@ -93,4 +93,4 @@
     "coverage": false,
     "check-coverage": false
   }
-}
\ No newline at end of file
+}

From 1d8da99d5bb9c282e7ecd500620c45024fafccfc Mon Sep 17 00:00:00 2001
From: Josh Mock <joshua.mock@elastic.co>
Date: Fri, 23 Feb 2024 14:01:55 -0600
Subject: [PATCH 11/11] Update changelog for 8.12.2 (#2139)

* Backport changelog for 8.12.1

* Add changelog for 8.12.2
---
 docs/changelog.asciidoc | 22 ++++++++++++++++++++++
 1 file changed, 22 insertions(+)

diff --git a/docs/changelog.asciidoc b/docs/changelog.asciidoc
index 4dbf11907..0daeee5d7 100644
--- a/docs/changelog.asciidoc
+++ b/docs/changelog.asciidoc
@@ -1,6 +1,28 @@
 [[changelog-client]]
 == Release notes
 
+[discrete]
+=== 8.12.2
+
+[discrete]
+==== Fixes
+
+[discrete]
+===== Upgrade transport to 8.4.1 https://github.com/elastic/elasticsearch-js/pull/2137[#2137]
+
+Upgrades `@elastic/transport` to 8.4.1 to resolve https://github.com/elastic/elastic-transport-js/pull/83[a bug] where arrays in error diagnostics were unintentionally transformed into objects.
+
+[discrete]
+=== 8.12.1
+
+[discrete]
+==== Fixes
+
+[discrete]
+===== Fix hang in bulk helper semaphore https://github.com/elastic/elasticsearch-js/pull/2027[#2027]
+
+The failing state could be reached when a server's response times are slower than flushInterval.
+
 [discrete]
 === 8.12.0