Migrate integration tests to built JS files (#2750) (#2780)

scripts/download-artifacts.js

@@ -17,162 +17,102 @@
  * under the License.
  */
 
-'use strict'
-
 const { join } = require('path')
-const minimist = require('minimist')
 const stream = require('stream')
 const { promisify } = require('util')
 const { createWriteStream, promises } = require('fs')
-const rimraf = require('rimraf')
+const { rimraf } = require('rimraf')
 const fetch = require('node-fetch')
 const crossZip = require('cross-zip')
 const ora = require('ora')
 
-const { mkdir, writeFile } = promises
+const { mkdir, cp } = promises
 const pipeline = promisify(stream.pipeline)
 const unzip = promisify(crossZip.unzip)
-const rm = promisify(rimraf)
 
-const esFolder = join(__dirname, '..', 'elasticsearch')
-const zipFolder = join(esFolder, 'artifacts.zip')
-const specFolder = join(esFolder, 'rest-api-spec', 'api')
-const freeTestFolder = join(esFolder, 'rest-api-spec', 'test', 'free')
-const xPackTestFolder = join(esFolder, 'rest-api-spec', 'test', 'platinum')
-const artifactInfo = join(esFolder, 'info.json')
+const testYamlFolder = join(__dirname, '..', 'yaml-rest-tests')
+const zipFile = join(__dirname, '..', 'elasticsearch-clients-tests.zip')
 
-async function downloadArtifacts (opts) {
-  if (typeof opts.version !== 'string') {
-    throw new Error('Missing version')
-  }
+const schemaFolder = join(__dirname, '..', 'schema')
+const schemaJson = join(schemaFolder, 'schema.json')
 
+async function downloadArtifacts (localTests, version = 'main') {
   const log = ora('Checking out spec and test').start()
 
-  log.text = 'Resolving versions'
-  let resolved
-  try {
-    resolved = await resolve(opts.version, opts.hash)
-  } catch (err) {
-    log.fail(err.message)
-    process.exit(1)
+  const { GITHUB_TOKEN } = process.env
+
+  if (version !== 'main') {
+    version = version.split('.').slice(0, 2).join('.')
   }
 
-  opts.id = opts.id || resolved.id
-  opts.hash = opts.hash || resolved.hash
-  opts.version = resolved.version
+  log.text = 'Clean tests folder'
+  await rimraf(testYamlFolder)
+  await mkdir(testYamlFolder, { recursive: true })
 
-  const info = loadInfo()
+  log.text = `Fetch test YAML files for version ${version}`
 
-  if (info && info.version === opts.version) {
-    if (info.hash === opts.hash && info.id === opts.id) {
-      log.succeed('The artifact copy present locally is already up to date')
-      return
+  if (localTests) {
+    log.text = `Copying local tests from ${localTests}`
+    await cp(localTests, testYamlFolder, { recursive: true })
+  } else {
+    if (!GITHUB_TOKEN) {
+      log.fail("Missing required environment variable 'GITHUB_TOKEN'")
+      process.exit(1)
     }
+
+    const response = await fetch(`https://api.github.com/repos/elastic/elasticsearch-clients-tests/zipball/${version}`, {
+      headers: {
+        Authorization: `Bearer ${GITHUB_TOKEN}`,
+        Accept: 'application/vnd.github+json'
+      }
+    })
+
+    if (!response.ok) {
+      log.fail(`unexpected response ${response.statusText}`)
+      process.exit(1)
+    }
+
+    log.text = 'Downloading tests zipball'
+    await pipeline(response.body, createWriteStream(zipFile))
+
+    log.text = 'Unzipping tests'
+    await unzip(zipFile, testYamlFolder)
+
+    log.text = 'Cleanup'
+    await rimraf(zipFile)
   }
 
-  log.text = 'Cleanup checkouts/elasticsearch'
-  await rm(esFolder)
-  await mkdir(esFolder, { recursive: true })
+  log.text = 'Fetching Elasticsearch specification'
+  await rimraf(schemaFolder)
+  await mkdir(schemaFolder, { recursive: true })
 
-  log.text = 'Downloading artifacts'
-  const response = await fetch(resolved.url)
+  const response = await fetch(`https://raw.githubusercontent.com/elastic/elasticsearch-specification/${version}/output/schema/schema.json`)
   if (!response.ok) {
     log.fail(`unexpected response ${response.statusText}`)
     process.exit(1)
   }
-  await pipeline(response.body, createWriteStream(zipFolder))
 
-  log.text = 'Unzipping'
-  await unzip(zipFolder, esFolder)
-
-  log.text = 'Cleanup'
-  await rm(zipFolder)
-
-  log.text = 'Update info'
-  await writeFile(artifactInfo, JSON.stringify(opts), 'utf8')
+  log.text = 'Downloading schema.json'
+  await pipeline(response.body, createWriteStream(schemaJson))
 
   log.succeed('Done')
 }
 
-function loadInfo () {
-  try {
-    return require(artifactInfo)
-  } catch (err) {
-    return null
-  }
+async function main () {
+  await downloadArtifacts()
 }
 
-async function resolve (version, hash) {
-  const response = await fetch(`https://artifacts-api.elastic.co/v1/versions/${version}`)
-  if (!response.ok) {
-    throw new Error(`unexpected response ${response.statusText}`)
-  }
-
-  const data = await response.json()
-  const esBuilds = data.version.builds
-    .filter(build => build.projects.elasticsearch != null)
-    .map(build => {
-      return {
-        projects: build.projects.elasticsearch,
-        buildId: build.build_id,
-        date: build.start_time,
-        version: build.version
-      }
-    })
-    .sort((a, b) => {
-      const dA = new Date(a.date)
-      const dB = new Date(b.date)
-      if (dA > dB) return -1
-      if (dA < dB) return 1
-      return 0
-    })
-
-  if (hash != null) {
-    const build = esBuilds.find(build => build.projects.commit_hash === hash)
-    if (!build) {
-      throw new Error(`Can't find any build with hash '${hash}'`)
-    }
-    const zipKey = Object.keys(build.projects.packages).find(key => key.startsWith('rest-resources-zip-') && key.endsWith('.zip'))
-    return {
-      url: build.projects.packages[zipKey].url,
-      id: build.buildId,
-      hash: build.projects.commit_hash,
-      version: build.version
-    }
-  }
-
-  const lastBuild = esBuilds[0]
-  const zipKey = Object.keys(lastBuild.projects.packages).find(key => key.startsWith('rest-resources-zip-') && key.endsWith('.zip'))
-  return {
-    url: lastBuild.projects.packages[zipKey].url,
-    id: lastBuild.buildId,
-    hash: lastBuild.projects.commit_hash,
-    version: lastBuild.version
-  }
-}
-
-async function main (options) {
-  delete options._
-  await downloadArtifacts(options)
-}
 if (require.main === module) {
   process.on('unhandledRejection', function (err) {
     console.error(err)
     process.exit(1)
   })
 
-  const options = minimist(process.argv.slice(2), {
-    string: ['id', 'version', 'hash']
-  })
-  main(options).catch(t => {
+  main().catch(t => {
     console.log(t)
     process.exit(2)
   })
 }
 
 module.exports = downloadArtifacts
-module.exports.locations = {
-  specFolder,
-  freeTestFolder,
-  xPackTestFolder
-}
+module.exports.locations = { testYamlFolder, zipFile, schemaJson }

scripts/generate-docs-examples.js

@@ -20,7 +20,7 @@
 const { join } = require('path')
 const { writeFile } = require('fs/promises')
 const fetch = require('node-fetch')
-const rimraf = require('rimraf')
+const { rimraf } = require('rimraf')
 const ora = require('ora')
 const { convertRequests } = require('@elastic/request-converter')
 const minimist = require('minimist')