From 62b2d78b15188375a799adf452001e78dfa07848 Mon Sep 17 00:00:00 2001
From: "github-actions[bot]"
 <41898282+github-actions[bot]@users.noreply.github.com>
Date: Mon, 12 Aug 2024 13:22:07 -0500
Subject: [PATCH] [Backport 8.15] Generate documentation example snippets
 (#2334)

* Update docs example generation script

* Add docs examples generation to codegen job

(cherry picked from commit 77e2f613f234467f3cfdfe0337406e2968fd4abb)

Co-authored-by: Josh Mock <joshua.mock@elastic.co>
---
 .buildkite/make.mjs               |   7 ++
 package.json                      |   1 +
 scripts/generate-docs-examples.js | 201 +++++++++++-------------------
 3 files changed, 83 insertions(+), 126 deletions(-)

diff --git a/.buildkite/make.mjs b/.buildkite/make.mjs
index 0f3e724fd..3026b61f3 100644
--- a/.buildkite/make.mjs
+++ b/.buildkite/make.mjs
@@ -125,6 +125,13 @@ async function codegen (args) {
   await $`cp -R ${join(import.meta.url, '..', '..', 'elastic-client-generator-js', 'output')}/* ${join(import.meta.url, '..', 'src', 'api')}`
   await $`mv ${join(import.meta.url, '..', 'src', 'api', 'reference.asciidoc')} ${join(import.meta.url, '..', 'docs', 'reference.asciidoc')}`
   await $`npm run build`
+
+  // run docs example generation
+  if (version === 'main') {
+    await $`node ./scripts/generate-docs-examples.js`
+  } else {
+    await $`node ./scripts/generate-docs-examples.js ${version.split('.').slice(0, 2).join('.')}`
+  }
 }
 
 function onError (err) {
diff --git a/package.json b/package.json
index ed81cc6d4..a2f5293f7 100644
--- a/package.json
+++ b/package.json
@@ -50,6 +50,7 @@
     "node": ">=18"
   },
   "devDependencies": {
+    "@elastic/request-converter": "^8.16.0",
     "@sinonjs/fake-timers": "github:sinonjs/fake-timers#0bfffc1",
     "@types/debug": "^4.1.7",
     "@types/ms": "^0.7.31",
diff --git a/scripts/generate-docs-examples.js b/scripts/generate-docs-examples.js
index 02a04420a..a9c229095 100644
--- a/scripts/generate-docs-examples.js
+++ b/scripts/generate-docs-examples.js
@@ -17,147 +17,96 @@
  * under the License.
  */
 
-'use strict'
-
-/**
- * To run this generator you must have the
- * `alternatives_report.spec.json` placed in the root of this project.
- * To get the `alternatives_report.spec.json` you must run the script
- * to parse the original `alternatives_report.json`, which is not yet public
- * and lives in github.com/elastic/clients-team/tree/master/scripts/docs-json-generator
- *
- * This script will remove the content of the `docs/doc_examples` folder and generate
- * all the files present in the `enabledFiles` list below.
- * You can run it with the following command:
- *
- * ```bash
- *   $ node scripts/generate-docs-examples.js
- * ```
- */
-
 const { join } = require('path')
-const { writeFileSync } = require('fs')
+const { writeFile } = require('fs/promises')
+const fetch = require('node-fetch')
 const rimraf = require('rimraf')
-const standard = require('standard')
-const dedent = require('dedent')
+const ora = require('ora')
+const { convertRequests } = require('@elastic/request-converter')
+const minimist = require('minimist')
 
 const docsExamplesDir = join('docs', 'doc_examples')
 
-const enabledFiles = [
-  'docs/delete.asciidoc',
-  'docs/get.asciidoc',
-  'docs/index_.asciidoc',
-  'getting-started.asciidoc',
-  'query-dsl/query-string-query.asciidoc',
-  'query-dsl.asciidoc',
-  'search/request-body.asciidoc',
-  'setup/install/check-running.asciidoc',
-  'mapping.asciidoc',
-  'query-dsl/query_filter_context.asciidoc',
-  'query-dsl/bool-query.asciidoc',
-  'query-dsl/match-query.asciidoc',
-  'indices/create-index.asciidoc',
-  'docs/index_.asciidoc',
-  'aggregations/bucket/terms-aggregation.asciidoc',
-  'query-dsl/range-query.asciidoc',
-  'search/search.asciidoc',
-  'query-dsl/multi-match-query.asciidoc',
-  'docs/bulk.asciidoc',
-  'indices/delete-index.asciidoc',
-  'indices/put-mapping.asciidoc',
-  'query-dsl/match-all-query.asciidoc',
-  'query-dsl/term-query.asciidoc',
-  'docs/update.asciidoc',
-  'docs/reindex.asciidoc',
-  'indices/templates.asciidoc',
-  'query-dsl/exists-query.asciidoc',
-  'query-dsl/terms-query.asciidoc',
-  'query-dsl/wildcard-query.asciidoc',
-  'mapping/types/nested.asciidoc',
-  'mapping/params/format.asciidoc',
-  'docs/delete-by-query.asciidoc',
-  'search/request/sort.asciidoc',
-  'query-dsl/function-score-query.asciidoc',
-  'query-dsl/nested-query.asciidoc',
-  'query-dsl/regexp-query.asciidoc',
-  'mapping/types/array.asciidoc',
-  'mapping/types/date.asciidoc',
-  'mapping/types/keyword.asciidoc',
-  'mapping/params/fielddata.asciidoc',
-  'cluster/health.asciidoc',
-  'docs/bulk.asciidoc',
-  'indices/aliases.asciidoc',
-  'indices/update-settings.asciidoc',
-  'search/request/from-size.asciidoc',
-  'search/count.asciidoc',
-  'setup/logging-config.asciidoc',
-  'search/request/from-size.asciidoc',
-  'query-dsl/match-phrase-query.asciidoc',
-  'aggregations/metrics/valuecount-aggregation.asciidoc',
-  'aggregations/bucket/datehistogram-aggregation.asciidoc',
-  'aggregations/bucket/filter-aggregation.asciidoc',
-  'mapping/types/numeric.asciidoc',
-  'mapping/fields/id-field.asciidoc',
-  'mapping/params/multi-fields.asciidoc',
-  'api-conventions.asciidoc',
-  'cat/indices.asciidoc',
-  'docs/update-by-query.asciidoc',
-  'indices/get-index.asciidoc',
-  'indices/get-mapping.asciidoc',
-  'search.asciidoc',
-  'search/request/scroll.asciidoc',
-  'search/suggesters.asciidoc'
-]
+const log = ora('Generating example snippets')
+
+const failures = {}
+
+async function getAlternativesReport (version = 'master') {
+  const reportUrl = `https://raw.githubusercontent.com/elastic/built-docs/master/raw/en/elasticsearch/reference/${version}/alternatives_report.json`
+  const response = await fetch(reportUrl)
+  if (!response.ok) {
+    log.fail(`unexpected response ${response.statusText}`)
+    process.exit(1)
+  }
+  return await response.json()
+}
+
+async function makeSnippet (example) {
+  const { source, digest } = example
+  const fileName = `${digest}.asciidoc`
+  const filePath = join(docsExamplesDir, fileName)
+
+  try {
+    const code = await convertRequests(source, 'javascript', {
+      complete: false,
+      printResponse: true
+    })
+    await writeFile(filePath, asciidocWrapper(code), 'utf8')
+  } catch (err) {
+    failures[digest] = err.message
+  }
+}
+
+async function generate (version) {
+  log.start()
 
-function generate () {
   rimraf.sync(join(docsExamplesDir, '*'))
-  const examples = require(join(__dirname, '..', 'alternatives_report.spec.json'))
+
+  log.text = `Downloading alternatives report for version ${version}`
+  const examples = await getAlternativesReport(version)
+
+  let counter = 1
   for (const example of examples) {
-    if (example.lang !== 'console') continue
-    if (!enabledFiles.includes(example.source_location.file)) continue
+    log.text = `${counter++}/${examples.length}: ${example.digest}`
 
-    const asciidoc = generateAsciidoc(example.parsed_source)
-    writeFileSync(
-      join(docsExamplesDir, `${example.digest}.asciidoc`),
-      asciidoc,
-      'utf8'
-    )
+    // skip over bad request definitions
+    if (example.source.startsWith('{') || example.source.endsWith('...')) {
+      failures[example.digest] = 'Incomplete request syntax'
+      continue
+    }
+
+    await makeSnippet(example)
   }
 }
 
-function generateAsciidoc (source) {
-  let asciidoc = '// This file is autogenerated, DO NOT EDIT\n'
-  asciidoc += '// Use `node scripts/generate-docs-examples.js` to generate the docs examples\n\n'
-  let code = 'async function run (client) {\n// START\n'
+function asciidocWrapper (source) {
+  return `// This file is autogenerated, DO NOT EDIT
+// Use \`node scripts/generate-docs-examples.js\` to generate the docs examples
 
-  for (let i = 0; i < source.length; i++) {
-    const { api, query, params, body } = source[i]
-    const apiArguments = Object.assign({}, params, query, body ? { body } : body)
-    const serializedApiArguments = Object.keys(apiArguments).length > 0
-      ? JSON.stringify(apiArguments, null, 2)
-      : ''
-    code += `const response${getResponsePostfix(i)} = await client.${api.replace(/_([a-z])/g, g => g[1].toUpperCase())}(${serializedApiArguments})
-console.log(response${getResponsePostfix(i)})
-\n`
-  }
-
-  code += '// END\n}'
-  const { results } = standard.lintTextSync(code, { fix: true })
-  code = results[0].output
-  code = code.slice(code.indexOf('// START\n') + 9, code.indexOf('\n\n// END'))
-
-  asciidoc += `[source, js]
+[source, js]
 ----
-${dedent(code)}
+${source.trim()}
 ----
-
 `
-  return asciidoc
-
-  function getResponsePostfix (i) {
-    if (source.length === 1) return ''
-    return String(i)
-  }
 }
 
-generate()
+const options = minimist(process.argv.slice(2), {
+  string: ['version'],
+  default: {
+    version: 'master'
+  }
+})
+
+generate(options.version)
+  .then(() => log.succeed('done!'))
+  .catch(err => log.fail(err.message))
+  .finally(() => {
+    const keys = Object.keys(failures)
+    if (keys.length > 0) {
+      let message = 'Some examples failed to generate:\n\n'
+      for (const key of keys) {
+        message += `${key}: ${failures[key]}\n`
+      }
+      console.error(message)
+    }
+  })