Switch 8.11 clients to only get patch updates to transport (#2215)

* Backport changelog from 8.10

* Pin transport to ~8.4.0

.buildkite/Dockerfile

@@ -10,5 +10,7 @@ RUN apt-get clean -y && \
 
 WORKDIR /usr/src/app
 
-COPY . .
+COPY package.json .
 RUN npm install --production=false
+
+COPY . .

.buildkite/pipeline.yml

@@ -6,7 +6,7 @@ steps:
     env:
       NODE_VERSION: "{{ matrix.nodejs }}"
       TEST_SUITE: "{{ matrix.suite }}"
-      STACK_VERSION: 8.11.0-SNAPSHOT
+      STACK_VERSION: 8.11.0
     matrix:
       setup:
         suite:

.dockerignore

@@ -3,3 +3,5 @@ npm-debug.log
 test/benchmarks
 elasticsearch
 .git
+lib
+junit-output

catalog-info.yaml

@@ -42,9 +42,9 @@ spec:
         main_semi_daily:
           branch: 'main'
           cronline: '0 */12 * * *'
-        8_11_semi_daily:
-          branch: '8.11'
+        8_12_semi_daily:
+          branch: '8.12'
           cronline: '0 */12 * * *'
-        8_10_daily:
-          branch: '8.10'
+        8_11_daily:
+          branch: '8.11'
           cronline: '@daily'

docs/changelog.asciidoc

@@ -1,11 +1,22 @@
 [[changelog-client]]
 == Release notes
 
+[discrete]
+=== 8.11.1
+
+[discrete]
+==== Fixes
+
+[discrete]
+===== Bump @elastic/transport to `~8.4.0`
+
+Switching from `^8.4.0` to `~8.4.0` ensures 8.11 client users are not required to update to Node.js v18+, which is a new requirement set by `@elastic/transport` v8.5.0. See https://github.com/elastic/elastic-transport-js/issues/91[elastic/elastic-transport-js#91] for details.
+
 [discrete]
 === 8.11.0
 
 [discrete]
-=== Features
+==== Features
 
 [discrete]
 ===== Support for Elasticsearch `v8.11.0`
@@ -20,11 +31,22 @@ https://www.elastic.co/guide/en/elasticsearch/reference/8.11/release-notes-8.11.
 
 See <<redaction>> for more information.
 
+[discrete]
+=== 8.10.1
+
+[discrete]
+==== Fixes
+
+[discrete]
+===== Bump @elastic/transport to `~8.3.4`
+
+Switching from `^8.3.4` to `~8.3.4` ensures 8.10 client users are not required to update to Node.js v18+, which is a new requirement set by `@elastic/transport` v8.5.0. See https://github.com/elastic/elastic-transport-js/issues/91[elastic/elastic-transport-js#91] for details.
+
 [discrete]
 === 8.10.0
 
 [discrete]
-=== Features
+==== Features
 
 [discrete]
 ===== Support for Elasticsearch `v8.10.0`
@@ -32,6 +54,17 @@ See <<redaction>> for more information.
 You can find all the API changes
 https://www.elastic.co/guide/en/elasticsearch/reference/8.10/release-notes-8.10.0.html[here].
 
+[discrete]
+=== 8.9.2
+
+[discrete]
+==== Fixes
+
+[discrete]
+===== Bump @elastic/transport to `~8.3.4`
+
+Switching from `^8.3.4` to `~8.3.4` ensures 8.9 client users are not required to update to Node.js v18+, which is a new requirement set by `@elastic/transport` v8.5.0. See https://github.com/elastic/elastic-transport-js/issues/91[elastic/elastic-transport-js#91] for details.
+
 [discrete]
 === 8.9.1
 
@@ -68,6 +101,17 @@ In the https://www.elastic.co/guide/en/elasticsearch/client/javascript-api/curre
 
 The `user-agent` header the client used to connect to Elasticsearch was using a non-standard format that has been improved.
 
+[discrete]
+=== 8.8.2
+
+[discrete]
+==== Fixes
+
+[discrete]
+===== Bump @elastic/transport to `~8.3.2`
+
+Switching from `^8.3.2` to `~8.3.2` ensures 8.8 client users are not required to update to Node.js v18+, which is a new requirement set by `@elastic/transport` v8.5.0. See https://github.com/elastic/elastic-transport-js/issues/91[elastic/elastic-transport-js#91] for details.
+
 [discrete]
 === 8.8.1
 
@@ -113,6 +157,17 @@ https://www.elastic.co/guide/en/elasticsearch/reference/8.8/release-notes-8.8.0.
 
 Prior releases contained a bug where type declarations for legacy types that include a `body` key were not actually importing the type that includes the `body` key.
 
+[discrete]
+=== 8.7.3
+
+[discrete]
+==== Fixes
+
+[discrete]
+===== Bump @elastic/transport to `~8.3.1`
+
+Switching from `^8.3.1` to `~8.3.1` ensures 8.7 client users are not required to update to Node.js v18+, which is a new requirement set by `@elastic/transport` v8.5.0. See https://github.com/elastic/elastic-transport-js/issues/91[elastic/elastic-transport-js#91] for details.
+
 [discrete]
 === 8.7.0
 
@@ -122,6 +177,17 @@ Prior releases contained a bug where type declarations for legacy types that inc
 You can find all the API changes
 https://www.elastic.co/guide/en/elasticsearch/reference/8.7/release-notes-8.7.0.html[here].
 
+[discrete]
+=== 8.6.1
+
+[discrete]
+==== Fixes
+
+[discrete]
+===== Bump @elastic/transport to `~8.3.1`
+
+Switching from `^8.3.1` to `~8.3.1` ensures 8.6 client users are not required to update to Node.js v18+, which is a new requirement set by `@elastic/transport` v8.5.0. See https://github.com/elastic/elastic-transport-js/issues/91[elastic/elastic-transport-js#91] for details.
+
 [discrete]
 === 8.6.0
 
@@ -361,6 +427,9 @@ The client API leaks HTTP-related notions in many places, and removing them woul
 
 This could be a rather big breaking change, so a double solution could be used during the 8.x lifecycle. (accepting body keys without them being wrapped in the body as well as the current solution).
 
+To convert code from 7.x, you need to remove the `body` parameter in all the endpoints request.
+For instance, this is an example for the `search` endpoint:
+
 [source,js]
 ----
 // from
@@ -399,6 +468,12 @@ If you weren't extending the internals of the client, this won't be a breaking c
 The client API leaks HTTP-related notions in many places, and removing them would definitely improve the DX.
 The client will expose a new request-specific option to still get the full response details.
 
+The new behaviour returns the `body` value directly as response.
+If you want to have the 7.x response format, you need to add `meta : true` in the request.
+This will return all the HTTP meta information, including the `body`.
+
+For instance, this is an example for the `search` endpoint:
+
 [source,js]
 ----
 // from

docs/connecting.asciidoc

@@ -12,6 +12,7 @@ This page contains the information you need to connect and use the Client with
 * <<client-connect-proxy, Connecting through a proxy>>
 * <<client-error-handling, Handling errors>>
 * <<keep-alive, Keep-alive connections>>
+* <<close-connections, Closing a client's connections>>
 * <<product-check, Automatic product check>>
 
 [[authentication]]
@@ -691,6 +692,20 @@ const client = new Client({
 })
 ----
 
+[discrete]
+[[close-connections]]
+=== Closing a client's connections
+
+If you would like to close all open connections being managed by an instance of the client, use the `close()` function:
+
+[source,js]
+----
+const client = new Client({
+  node: 'http://localhost:9200'
+});
+client.close();
+----
+
 [discrete]
 [[product-check]]
 === Automatic product check

docs/doc_examples/36b86b97feedcf5632824eefc251d6ed.asciidoc

@@ -0,0 +1,12 @@
+[source,js]
+----
+const response = await client.search({
+  index: 'books',
+  query: {
+    match: {
+      name: 'brave'
+    }
+  }
+})
+console.log(response)
+----

docs/doc_examples/8575c966b004fb124c7afd6bb5827b50.asciidoc

@@ -0,0 +1,13 @@
+[source,js]
+----
+const response = await client.index({
+  index: 'books',
+  document: {
+    name: 'Snow Crash',
+    author: 'Neal Stephenson',
+    release_date: '1992-06-01',
+    page_count: 470,
+  }
+})
+console.log(response)
+----

docs/doc_examples/bcc75fc01b45e482638c65b8fbdf09fa.asciidoc

@@ -0,0 +1,7 @@
+[source,js]
+----
+const response = await client.search({
+  index: 'books'
+})
+console.log(response)
+----

docs/doc_examples/d04f0c8c44e8b4fb55f2e7d9d05977e7.asciidoc

@@ -0,0 +1,43 @@
+[source,js]
+----
+const response = await client.bulk({
+  operations: [
+    { index: { _index: 'books' } },
+    {
+      name: 'Revelation Space',
+      author: 'Alastair Reynolds',
+      release_date: '2000-03-15',
+      page_count: 585,
+    },
+    { index: { _index: 'books' } },
+    {
+      name: '1984',
+      author: 'George Orwell',
+      release_date: '1985-06-01',
+      page_count: 328,
+    },
+    { index: { _index: 'books' } },
+    {
+      name: 'Fahrenheit 451',
+      author: 'Ray Bradbury',
+      release_date: '1953-10-15',
+      page_count: 227,
+    },
+    { index: { _index: 'books' } },
+    {
+      name: 'Brave New World',
+      author: 'Aldous Huxley',
+      release_date: '1932-06-01',
+      page_count: 268,
+    },
+    { index: { _index: 'books' } },
+    {
+      name: 'The Handmaids Tale',
+      author: 'Margaret Atwood',
+      release_date: '1985-06-01',
+      page_count: 311,
+    }
+  ]
+})
+console.log(response)
+----

docs/reference.asciidoc

@@ -8950,7 +8950,7 @@ client.synonyms.putSynonymRule({ set_id, rule_id, synonyms })
 * *Request (object):*
 ** *`set_id` (string)*: The id of the synonym set to be updated with the synonym rule
 ** *`rule_id` (string)*: The id of the synonym rule to be updated or created
-** *`synonyms` (string[])*
+** *`synonyms` (string)*
 
 [discrete]
 === tasks

package.json

@@ -1,7 +1,7 @@
 {
   "name": "@elastic/elasticsearch",
-  "version": "8.11.0",
-  "versionCanary": "8.11.0-canary.1",
+  "version": "8.11.1",
+  "versionCanary": "8.11.1-canary.0",
   "description": "The official Elasticsearch client for Node.js",
   "main": "index.js",
   "types": "index.d.ts",
@@ -82,7 +82,7 @@
     "zx": "^7.2.2"
   },
   "dependencies": {
-    "@elastic/transport": "^8.4.0",
+    "@elastic/transport": "~8.4.0",
     "tslib": "^2.4.0"
   },
   "tap": {

src/api/types.ts

@@ -15682,11 +15682,9 @@ export interface SecurityCreatedStatus {
 }
 
 export interface SecurityFieldRule {
-  username?: Name
+  username?: Names
   dn?: Names
   groups?: Names
-  metadata?: any
-  realm?: SecurityRealm
 }
 
 export interface SecurityFieldSecurity {
@@ -15716,10 +15714,6 @@ export interface SecurityManageUserPrivileges {
   applications: string[]
 }
 
-export interface SecurityRealm {
-  name: Name
-}
-
 export interface SecurityRealmInfo {
   name: Name
   type: string
@@ -17184,7 +17178,7 @@ export interface SynonymsPutSynonymResponse {
 export interface SynonymsPutSynonymRuleRequest extends RequestBase {
   set_id: Id
   rule_id: Id
-  synonyms: SynonymsSynonymString[]
+  synonyms: SynonymsSynonymString
 }
 
 export type SynonymsPutSynonymRuleResponse = SynonymsSynonymsUpdateResult

src/api/typesWithBodyKey.ts

@@ -15993,11 +15993,9 @@ export interface SecurityCreatedStatus {
 }
 
 export interface SecurityFieldRule {
-  username?: Name
+  username?: Names
   dn?: Names
   groups?: Names
-  metadata?: any
-  realm?: SecurityRealm
 }
 
 export interface SecurityFieldSecurity {
@@ -16027,10 +16025,6 @@ export interface SecurityManageUserPrivileges {
   applications: string[]
 }
 
-export interface SecurityRealm {
-  name: Name
-}
-
 export interface SecurityRealmInfo {
   name: Name
   type: string
@@ -17592,7 +17586,7 @@ export interface SynonymsPutSynonymRuleRequest extends RequestBase {
   rule_id: Id
   /** @deprecated The use of the 'body' key has been deprecated, move the nested keys to the top level object. */
   body?: {
-    synonyms: SynonymsSynonymString[]
+    synonyms: SynonymsSynonymString
   }
 }
 

test/integration/index.js

@@ -317,7 +317,7 @@ async function start ({ client, isXPack }) {
         if (name === 'setup' || name === 'teardown') continue
         if (options.test && !name.endsWith(options.test)) continue
 
-        const junitTestCase = junitTestSuite.testcase(name, `node_${process.version}/${cleanPath}`)
+        const junitTestCase = junitTestSuite.testcase(name, `node_${process.version}: ${cleanPath}`)
 
         stats.total += 1
         if (shouldSkip(isXPack, file, name)) {
@@ -336,6 +336,7 @@ async function start ({ client, isXPack }) {
           junitTestSuite.end()
           junitTestSuites.end()
           generateJunitXmlReport(junit, isXPack ? 'platinum' : 'free')
+          err.meta = JSON.stringify(err.meta ?? {}, null, 2)
           console.error(err)
 
           if (options.bail) {
@@ -374,6 +375,7 @@ async function start ({ client, isXPack }) {
   - Total: ${stats.total}
   - Skip: ${stats.skip}
   - Pass: ${stats.pass}
+  - Fail: ${stats.total - (stats.pass + stats.skip)}
   - Assertions: ${stats.assertions}
   `)
 }

test/integration/test-runner.js

@@ -593,13 +593,14 @@ function build (opts = {}) {
         const key = Object.keys(action.match)[0]
         match(
           // in some cases, the yaml refers to the body with an empty string
-          key === '$body' || key === ''
+          key.split('.')[0] === '$body' || key === ''
             ? response
             : delve(response, fillStashedValues(key)),
-          key === '$body'
+          key.split('.')[0] === '$body'
             ? action.match[key]
             : fillStashedValues(action.match)[key],
-          action.match
+          action.match,
+          response
         )
       }
 
@@ -608,7 +609,8 @@ function build (opts = {}) {
         const key = Object.keys(action.lt)[0]
         lt(
           delve(response, fillStashedValues(key)),
-          fillStashedValues(action.lt)[key]
+          fillStashedValues(action.lt)[key],
+          response
         )
       }
 
@@ -617,7 +619,8 @@ function build (opts = {}) {
         const key = Object.keys(action.gt)[0]
         gt(
           delve(response, fillStashedValues(key)),
-          fillStashedValues(action.gt)[key]
+          fillStashedValues(action.gt)[key],
+          response
         )
       }
 
@@ -626,7 +629,8 @@ function build (opts = {}) {
         const key = Object.keys(action.lte)[0]
         lte(
           delve(response, fillStashedValues(key)),
-          fillStashedValues(action.lte)[key]
+          fillStashedValues(action.lte)[key],
+          response
         )
       }
 
@@ -635,7 +639,8 @@ function build (opts = {}) {
         const key = Object.keys(action.gte)[0]
         gte(
           delve(response, fillStashedValues(key)),
-          fillStashedValues(action.gte)[key]
+          fillStashedValues(action.gte)[key],
+          response
         )
       }
 
@@ -648,7 +653,8 @@ function build (opts = {}) {
             : delve(response, fillStashedValues(key)),
           key === '$body'
             ? action.length[key]
-            : fillStashedValues(action.length)[key]
+            : fillStashedValues(action.length)[key],
+          response
         )
       }
 
@@ -657,7 +663,8 @@ function build (opts = {}) {
         const isTrue = fillStashedValues(action.is_true)
         is_true(
           delve(response, isTrue),
-          isTrue
+          isTrue,
+          response
         )
       }
 
@@ -666,7 +673,8 @@ function build (opts = {}) {
         const isFalse = fillStashedValues(action.is_false)
         is_false(
           delve(response, isFalse),
-          isFalse
+          isFalse,
+          response
         )
       }
     }
@@ -679,46 +687,67 @@ function build (opts = {}) {
  * Asserts that the given value is truthy
  * @param {any} the value to check
  * @param {string} an optional message
+ * @param {any} debugging metadata to attach to any assertion errors
  * @returns {TestRunner}
  */
-function is_true (val, msg) {
-  assert.ok(val, `expect truthy value: ${msg} - value: ${JSON.stringify(val)}`)
+function is_true (val, msg, response) {
+  try {
+    assert.ok((typeof val === 'string' && val.toLowerCase() === 'true') || val, `expect truthy value: ${msg} - value: ${JSON.stringify(val)}`)
+  } catch (err) {
+    err.response = JSON.stringify(response)
+    throw err
+  }
 }
 
 /**
  * Asserts that the given value is falsey
  * @param {any} the value to check
  * @param {string} an optional message
+ * @param {any} debugging metadata to attach to any assertion errors
  * @returns {TestRunner}
  */
-function is_false (val, msg) {
-  assert.ok(!val, `expect falsey value: ${msg} - value: ${JSON.stringify(val)}`)
+function is_false (val, msg, response) {
+  try {
+    assert.ok((typeof val === 'string' && val.toLowerCase() === 'false') || !val, `expect falsey value: ${msg} - value: ${JSON.stringify(val)}`)
+  } catch (err) {
+    err.response = JSON.stringify(response)
+    throw err
+  }
 }
 
 /**
  * Asserts that two values are the same
  * @param {any} the first value
  * @param {any} the second value
+ * @param {any} debugging metadata to attach to any assertion errors
  * @returns {TestRunner}
  */
-function match (val1, val2, action) {
-  // both values are objects
-  if (typeof val1 === 'object' && typeof val2 === 'object') {
-    assert.deepEqual(val1, val2, typeof action === 'object' ? JSON.stringify(action) : action)
-  // the first value is the body as string and the second a pattern string
-  } else if (
-    typeof val1 === 'string' && typeof val2 === 'string' &&
-    val2.startsWith('/') && (val2.endsWith('/\n') || val2.endsWith('/'))
-  ) {
-    const regStr = val2
-      .replace(/(^|[^\\])#.*/g, '$1')
-      .replace(/(^|[^\\])\s+/g, '$1')
-      .slice(1, -1)
-    // 'm' adds the support for multiline regex
-    assert.match(val1, new RegExp(regStr, 'm'), `should match pattern provided: ${val2}, but got: ${val1}`)
-  // everything else
-  } else {
-    assert.equal(val1, val2, `should be equal: ${val1} - ${val2}, action: ${JSON.stringify(action)}`)
+function match (val1, val2, action, response) {
+  try {
+    // both values are objects
+    if (typeof val1 === 'object' && typeof val2 === 'object') {
+      assert.deepEqual(val1, val2, typeof action === 'object' ? JSON.stringify(action) : action)
+    // the first value is the body as string and the second a pattern string
+    } else if (
+      typeof val1 === 'string' && typeof val2 === 'string' &&
+      val2.startsWith('/') && (val2.endsWith('/\n') || val2.endsWith('/'))
+    ) {
+      const regStr = val2
+        .replace(/(^|[^\\])#.*/g, '$1')
+        .replace(/(^|[^\\])\s+/g, '$1')
+        .slice(1, -1)
+      // 'm' adds the support for multiline regex
+      assert.match(val1, new RegExp(regStr, 'm'), `should match pattern provided: ${val2}, but got: ${val1}: ${JSON.stringify(action)}`)
+    } else if (typeof val1 === 'string' && typeof val2 === 'string') {
+      // string comparison
+      assert.include(val1, val2, `should include pattern provided: ${val2}, but got: ${val1}: ${JSON.stringify(action)}`)
+    } else {
+      // everything else
+      assert.equal(val1, val2, `should be equal: ${val1} - ${val2}, action: ${JSON.stringify(action)}`)
+    }
+  } catch (err) {
+    err.response = JSON.stringify(response)
+    throw err
   }
 }
 
@@ -727,11 +756,17 @@ function match (val1, val2, action) {
  * It also verifies that the two values are numbers
  * @param {any} the first value
  * @param {any} the second value
+ * @param {any} debugging metadata to attach to any assertion errors
  * @returns {TestRunner}
  */
-function lt (val1, val2) {
-  ;[val1, val2] = getNumbers(val1, val2)
-  assert.ok(val1 < val2)
+function lt (val1, val2, response) {
+  try {
+    ;[val1, val2] = getNumbers(val1, val2)
+    assert.ok(val1 < val2)
+  } catch (err) {
+    err.response = JSON.stringify(response)
+    throw err
+  }
 }
 
 /**
@@ -739,11 +774,17 @@ function lt (val1, val2) {
  * It also verifies that the two values are numbers
  * @param {any} the first value
  * @param {any} the second value
+ * @param {any} debugging metadata to attach to any assertion errors
  * @returns {TestRunner}
  */
-function gt (val1, val2) {
-  ;[val1, val2] = getNumbers(val1, val2)
-  assert.ok(val1 > val2)
+function gt (val1, val2, response) {
+  try {
+    ;[val1, val2] = getNumbers(val1, val2)
+    assert.ok(val1 > val2)
+  } catch (err) {
+    err.response = JSON.stringify(response)
+    throw err
+  }
 }
 
 /**
@@ -751,11 +792,17 @@ function gt (val1, val2) {
  * It also verifies that the two values are numbers
  * @param {any} the first value
  * @param {any} the second value
+ * @param {any} debugging metadata to attach to any assertion errors
  * @returns {TestRunner}
  */
-function lte (val1, val2) {
-  ;[val1, val2] = getNumbers(val1, val2)
-  assert.ok(val1 <= val2)
+function lte (val1, val2, response) {
+  try {
+    ;[val1, val2] = getNumbers(val1, val2)
+    assert.ok(val1 <= val2)
+  } catch (err) {
+    err.response = JSON.stringify(response)
+    throw err
+  }
 }
 
 /**
@@ -763,26 +810,38 @@ function lte (val1, val2) {
  * It also verifies that the two values are numbers
  * @param {any} the first value
  * @param {any} the second value
+ * @param {any} debugging metadata to attach to any assertion errors
  * @returns {TestRunner}
 */
-function gte (val1, val2) {
-  ;[val1, val2] = getNumbers(val1, val2)
-  assert.ok(val1 >= val2)
+function gte (val1, val2, response) {
+  try {
+    ;[val1, val2] = getNumbers(val1, val2)
+    assert.ok(val1 >= val2)
+  } catch (err) {
+    err.response = JSON.stringify(response)
+    throw err
+  }
 }
 
 /**
  * Asserts that the given value has the specified length
  * @param {string|object|array} the object to check
  * @param {number} the expected length
+ * @param {any} debugging metadata to attach to any assertion errors
  * @returns {TestRunner}
  */
-function length (val, len) {
-  if (typeof val === 'string' || Array.isArray(val)) {
-    assert.equal(val.length, len)
-  } else if (typeof val === 'object' && val !== null) {
-    assert.equal(Object.keys(val).length, len)
-  } else {
-    assert.fail(`length: the given value is invalid: ${val}`)
+function length (val, len, response) {
+  try {
+    if (typeof val === 'string' || Array.isArray(val)) {
+      assert.equal(val.length, len)
+    } else if (typeof val === 'object' && val !== null) {
+      assert.equal(Object.keys(val).length, len)
+    } else {
+      assert.fail(`length: the given value is invalid: ${val}`)
+    }
+  } catch (err) {
+    err.response = JSON.stringify(response)
+    throw err
   }
 }
 
@@ -813,6 +872,10 @@ function length (val, len) {
  */
 function parseDo (action) {
   action = JSON.parse(JSON.stringify(action))
+
+  if (typeof action === 'string') action = {[action]: {}}
+  if (Array.isArray(action)) action = action[0]
+
   return Object.keys(action).reduce((acc, val) => {
     switch (val) {
       case 'catch':