Improve integration test (#859)

* CI: Added junit plugin

* Updated .gitignore

* Added integration test reporter

* Updated integration testing suite

* Updated ci config

* Updated report file path

* Use refresh 'true' instead of 'wait_for'

* Disable junit reporting

* Refresh one single time

* Update security index name

* Updated skip test handling and use class syntax

* Updated test script

* Disable test timeout

* Added command to automatically remove an old snapshot

* Disable timeout in integration test script

* Updated logs and cleaned up git handling

* Fixed shouldSkip utility

* Updated cleanup code

* Updated cleanup code pt 2

* Rename Platinum to XPack

.ci/jobs/defaults.yml

@@ -65,3 +65,6 @@
     publishers:
     - email:
         recipients: infra-root+build@elastic.co
+    # - junit:
+    #     results: "*-junit.xml"
+    #     allow-empty-results: true

.gitignore

@@ -55,3 +55,5 @@ elasticsearch*
 api/generated.d.ts
 
 test/benchmarks/macro/fixtures/*
+
+*-junit.xml

package.json

@@ -19,7 +19,8 @@
     "test": "npm run lint && npm run test:unit && npm run test:behavior && npm run test:types",
     "test:unit": "tap test/unit/*.test.js -t 300 --no-coverage",
     "test:behavior": "tap test/behavior/*.test.js -t 300 --no-coverage",
-    "test:integration": "tap test/integration/index.js -T --harmony --no-esm --no-coverage",
+    "test:integration": "tap test/integration/index.js -T --no-coverage",
+    "test:integration:report": "npm run test:integration | tap-mocha-reporter xunit > $WORKSPACE/test-report-junit.xml",
     "test:types": "tsc --project ./test/types/tsconfig.json",
     "test:coverage": "nyc npm run test:unit && nyc report --reporter=text-lcov > coverage.lcov && codecov",
     "lint": "standard",
@@ -56,6 +57,7 @@
     "standard": "^12.0.1",
     "stoppable": "^1.1.0",
     "tap": "^13.0.1",
+    "tap-mocha-reporter": "^4.0.1",
     "typescript": "^3.4.5",
     "workq": "^2.1.0"
   },

scripts/es-docker-platinum.sh

@@ -5,6 +5,11 @@ testnodecrt="/.ci/certs/testnode.crt"
 testnodekey="/.ci/certs/testnode.key"
 cacrt="/.ci/certs/ca.crt"
 
+# pass `--clean` to reemove the old snapshot
+if [ "$1" != "" ]; then
+  docker rmi $(docker images --format '{{.Repository}}:{{.Tag}}' | grep '8.0.0-SNAPSHOT')
+fi
+
 exec docker run \
   --rm \
   -e "node.attr.testattr=test" \

test/integration/helper.js

@@ -56,11 +56,11 @@ const esDefaultUsers = [
   'remote_monitoring_user'
 ]
 
-function runInParallel (client, operation, options) {
+function runInParallel (client, operation, options, clientOptions) {
   if (options.length === 0) return Promise.resolve()
   const operations = options.map(opts => {
     const api = delve(client, operation).bind(client)
-    return api(opts)
+    return api(opts, clientOptions)
   })
 
   return Promise.all(operations)
@@ -80,4 +80,10 @@ function delve (obj, key, def, p) {
   return (obj === undefined || p < key.length) ? def : obj
 }
 
-module.exports = { runInParallel, esDefaultRoles, esDefaultUsers, delve }
+function to (promise) {
+  return promise.then(data => [null, data], err => [err, undefined])
+}
+
+const sleep = ms => new Promise(resolve => setTimeout(resolve, ms))
+
+module.exports = { runInParallel, esDefaultRoles, esDefaultUsers, delve, to, sleep }

test/integration/index.js

@@ -19,66 +19,70 @@
 
 'use strict'
 
-const assert = require('assert')
 const { readFileSync, accessSync, mkdirSync, readdirSync, statSync } = require('fs')
 const { join, sep } = require('path')
 const yaml = require('js-yaml')
 const Git = require('simple-git')
-const ora = require('ora')
 const tap = require('tap')
 const { Client } = require('../../index')
 const TestRunner = require('./test-runner')
+const { sleep } = require('./helper')
 
 const esRepo = 'https://github.com/elastic/elasticsearch.git'
 const esFolder = join(__dirname, '..', '..', 'elasticsearch')
 const yamlFolder = join(esFolder, 'rest-api-spec', 'src', 'main', 'resources', 'rest-api-spec', 'test')
 const xPackYamlFolder = join(esFolder, 'x-pack', 'plugin', 'src', 'test', 'resources', 'rest-api-spec', 'test')
-const customSkips = [
+
-  // fails with ES5 with x-pack enabled
+const ossSkips = {
-  'cat.allocation/10_basic.yaml',
+  // TODO: remove this once 'arbitrary_key' is implemented
-  'cat.indices/10_basic.yaml',
+  // https://github.com/elastic/elasticsearch/pull/41492
-  'cat.shards/10_basic.yaml',
+  'indices.split/30_copy_settings.yml': ['*'],
-  // fails with ES5,`repository` is a required field
-  'cat.snapshots/10_basic.yaml',
-  // fails with ES5 with x-pack enabled
-  'cat.templates/10_basic.yaml',
-  'cluster.health/10_basic.yaml',
-  'cluster.health/20_request_timeout.yaml',
-  'delete/11_shard_header.yaml',
-  'delete/45_parent_with_routing.yaml',
-  'delete/50_refresh.yaml',
-  'exists/40_routing.yaml',
-  'exists/55_parent_with_routing.yaml',
-  'get/55_parent_with_routing.yaml',
-  'get_source/55_parent_with_routing.yaml',
-  'indices.get_mapping/50_wildcard_expansion.yaml',
-  'indices.flush/10_basic.yaml',
-  'indices.open/10_basic.yaml',
-  'indices.open/20_multiple_indices.yaml',
-  'indices.stats/10_index.yaml',
-  'indices.shard_stores/10_basic.yaml',
-  'indices.segments/10_basic.yaml',
-  'mget/40_routing.yaml',
-  'mlt/20_docs.yaml',
-  'search/10_source_filtering.yaml',
-  'search/140_pre_filter_search_shards.yml',
-  'search.aggregation/10_histogram.yaml',
-  'search.aggregation/20_terms.yaml',
-  'search.aggregation/40_range.yaml',
-  'search.aggregation/50_filter.yaml',
-  'search.highlight/30_fvh.yml',
-  'snapshot.restore/10_basic.yaml',
-  'snapshot.status/10_basic.yaml',
-  'update/11_shard_header.yaml',
-  'update/55_parent_with_routing.yaml',
   // skipping because we are booting ES with `discovery.type=single-node`
   // and this test will fail because of this configuration
-  'nodes.stats/30_discovery.yml',
+  'nodes.stats/30_discovery.yml': ['*'],
   // the expected error is returning a 503,
   // which triggers a retry and the node to be marked as dead
-  'search.aggregation/240_max_buckets.yml'
+  'search.aggregation/240_max_buckets.yml': ['*'],
-]
+  // fails with ES5 with x-pack enabled
-const platinumBlackList = {
+  'cat.allocation/10_basic.yaml': ['*'],
+  'cat.indices/10_basic.yaml': ['*'],
+  'cat.shards/10_basic.yaml': ['*'],
+  // fails with ES5,`repository` is a required field
+  'cat.snapshots/10_basic.yaml': ['*'],
+  // fails with ES5 with x-pack enabled
+  'cat.templates/10_basic.yaml': ['*'],
+  'cluster.health/10_basic.yaml': ['*'],
+  'cluster.health/20_request_timeout.yaml': ['*'],
+  'delete/11_shard_header.yaml': ['*'],
+  'delete/45_parent_with_routing.yaml': ['*'],
+  'delete/50_refresh.yaml': ['*'],
+  'exists/40_routing.yaml': ['*'],
+  'exists/55_parent_with_routing.yaml': ['*'],
+  'get/55_parent_with_routing.yaml': ['*'],
+  'get_source/55_parent_with_routing.yaml': ['*'],
+  'indices.get_mapping/50_wildcard_expansion.yaml': ['*'],
+  'indices.flush/10_basic.yaml': ['*'],
+  'indices.open/10_basic.yaml': ['*'],
+  'indices.open/20_multiple_indices.yaml': ['*'],
+  'indices.stats/10_index.yaml': ['*'],
+  'indices.shard_stores/10_basic.yaml': ['*'],
+  'indices.segments/10_basic.yaml': ['*'],
+  'mget/40_routing.yaml': ['*'],
+  'mlt/20_docs.yaml': ['*'],
+  'search/10_source_filtering.yaml': ['*'],
+  'search/140_pre_filter_search_shards.yml': ['*'],
+  'search.aggregation/10_histogram.yaml': ['*'],
+  'search.aggregation/20_terms.yaml': ['*'],
+  'search.aggregation/40_range.yaml': ['*'],
+  'search.aggregation/50_filter.yaml': ['*'],
+  'search.highlight/30_fvh.yml': ['*'],
+  'snapshot.restore/10_basic.yaml': ['*'],
+  'snapshot.status/10_basic.yaml': ['*'],
+  'update/11_shard_header.yaml': ['*'],
+  'update/55_parent_with_routing.yaml': ['*']
+}
+
+const xPackBlackList = {
   // file path: test name
   'cat.aliases/10_basic.yml': ['Empty cluster'],
   'index/10_with_id.yml': ['Index with ID'],
@@ -104,279 +108,276 @@ const platinumBlackList = {
   'xpack/15_basic.yml': ['*']
 }
 
-function Runner (opts) {
+class Runner {
-  if (!(this instanceof Runner)) {
+  constructor (opts = {}) {
-    return new Runner(opts)
+    const options = { node: opts.node }
-  }
+    if (opts.isXPack) {
-  opts = opts || {}
+      options.ssl = {
-
+        ca: readFileSync(join(__dirname, '..', '..', '.ci', 'certs', 'ca.crt'), 'utf8'),
-  assert(opts.node, 'Missing base node')
+        rejectUnauthorized: false
-  this.bailout = opts.bailout
-  const options = { node: opts.node }
-  if (opts.isPlatinum) {
-    options.ssl = {
-      // NOTE: this path works only if we run
-      // the suite with npm scripts
-      ca: readFileSync('.ci/certs/ca.crt', 'utf8'),
-      rejectUnauthorized: false
-    }
-  }
-  this.client = new Client(options)
-  this.log = ora('Loading yaml suite').start()
-}
-
-Runner.prototype.waitCluster = function (callback, times = 0) {
-  this.log.text = 'Waiting for ElasticSearch'
-  this.client.cluster.health(
-    { waitForStatus: 'green', timeout: '50s' },
-    (err, res) => {
-      if (err && ++times < 10) {
-        setTimeout(() => {
-          this.waitCluster(callback, times)
-        }, 5000)
-      } else {
-        callback(err)
       }
     }
-  )
+    this.client = new Client(options)
-}
+    console.log('Loading yaml suite')
+  }
 
-/**
+  async waitCluster (client, times = 0) {
- * Runs the test suite
+    try {
- */
+      await client.cluster.health({ waitForStatus: 'green', timeout: '50s' })
-Runner.prototype.start = function (opts) {
+    } catch (err) {
-  const parse = this.parse.bind(this)
+      if (++times < 10) {
-  const client = this.client
+        await sleep(5000)
-
+        return this.waitCluster(client, times)
-  // client.on('response', (err, meta) => {
+      }
-  //   console.log('Request', meta.request)
+      console.error(err)
-  //   if (err) {
-  //     console.log('Error', err)
-  //   } else {
-  //     console.log('Response', JSON.stringify(meta.response, null, 2))
-  //   }
-  //   console.log()
-  // })
-
-  this.waitCluster(err => {
-    if (err) {
-      this.log.fail(err.message)
       process.exit(1)
     }
-    // Get the build hash of Elasticsearch
+  }
-    client.info((err, { body }) => {
-      if (err) {
-        this.log.fail(err.message)
-        process.exit(1)
-      }
-      const { number: version, build_hash: sha } = body.version
 
-      // Set the repository to the given sha and run the test suite
+  async start ({ isXPack }) {
-      this.withSHA(sha, () => {
+    const { client } = this
-        this.log.succeed(`Testing ${opts.isPlatinum ? 'platinum' : 'oss'} api...`)
+    const parse = this.parse.bind(this)
-        runTest.call(this, version)
-      })
-    })
-  })
 
-  function runTest (version) {
+    console.log('Waiting for Elasticsearch')
-    const files = []
+    await this.waitCluster(client)
+
+    const { body } = await client.info()
+    const { number: version, build_hash: sha } = body.version
+
+    console.log(`Checking out sha ${sha}...`)
+    await this.withSHA(sha)
+
+    console.log(`Testing ${isXPack ? 'XPack' : 'oss'} api...`)
+
+    const folders = []
       .concat(getAllFiles(yamlFolder))
-      .concat(opts.isPlatinum ? getAllFiles(xPackYamlFolder) : [])
+      .concat(isXPack ? getAllFiles(xPackYamlFolder) : [])
       .filter(t => !/(README|TODO)/g.test(t))
+      // we cluster the array based on the folder names,
+      // to provide a better test log output
+      .reduce((arr, file) => {
+        const path = file.slice(file.indexOf('/rest-api-spec/test'), file.lastIndexOf('/'))
+        var inserted = false
+        for (var i = 0; i < arr.length; i++) {
+          if (arr[i][0].includes(path)) {
+            inserted = true
+            arr[i].push(file)
+            break
+          }
+        }
+        if (!inserted) arr.push([file])
+        return arr
+      }, [])
 
-    files.forEach(runTestFile.bind(this))
+    for (const folder of folders) {
-    function runTestFile (file) {
+      // pretty name
-      // if (!file.endsWith('watcher/execute_watch/70_invalid.yml')) return
+      const apiName = folder[0].slice(
-      for (var i = 0; i < customSkips.length; i++) {
+        folder[0].indexOf(`${sep}rest-api-spec${sep}test`) + 19,
-        if (file.endsWith(customSkips[i])) return
+        folder[0].lastIndexOf(sep)
-      }
+      )
-      // create a subtest for the specific folder
-      tap.test(file.slice(file.indexOf(`${sep}elasticsearch${sep}`)), { jobs: 1 }, tap1 => {
-        // read the yaml file
-        const data = readFileSync(file, 'utf8')
-        // get the test yaml (as object), some file has multiple yaml documents inside,
-        // every document is separated by '---', so we split on the separator
-        // and then we remove the empty strings, finally we parse them
-        const tests = data
-          .split('\n---\n')
-          .map(s => s.trim())
-          .filter(Boolean)
-          .map(parse)
 
+      tap.test(`Testing ${apiName}`, { bail: true, timeout: 0 }, t => {
+        for (const file of folder) {
+          const data = readFileSync(file, 'utf8')
+          // get the test yaml (as object), some file has multiple yaml documents inside,
+          // every document is separated by '---', so we split on the separator
+          // and then we remove the empty strings, finally we parse them
+          const tests = data
+            .split('\n---\n')
+            .map(s => s.trim())
+            .filter(Boolean)
+            .map(parse)
+
+          t.test(
+            file.slice(file.lastIndexOf(apiName)),
+            testFile(file, tests)
+          )
+        }
+        t.end()
+      })
+    }
+
+    function testFile (file, tests) {
+      return t => {
         // get setup and teardown if present
         var setupTest = null
         var teardownTest = null
-        tests.forEach(test => {
+        for (const test of tests) {
           if (test.setup) setupTest = test.setup
           if (test.teardown) teardownTest = test.teardown
-        })
+        }
 
-        // run the tests
         tests.forEach(test => {
           const name = Object.keys(test)[0]
           if (name === 'setup' || name === 'teardown') return
-          // should skip the test inside `platinumBlackList`
+          if (shouldSkip(t, isXPack, file, name)) return
-          // if we are testing the platinum apis
+
-          if (opts.isPlatinum) {
-            const list = Object.keys(platinumBlackList)
-            for (i = 0; i < list.length; i++) {
-              const platTest = platinumBlackList[list[i]]
-              for (var j = 0; j < platTest.length; j++) {
-                if (file.endsWith(list[i]) && (name === platTest[j] || platTest[j] === '*')) {
-                  const testName = file.slice(file.indexOf(`${sep}elasticsearch${sep}`)) + ' / ' + name
-                  tap.skip(`Skipping test ${testName} because is blacklisted in the platinum test`)
-                  return
-                }
-              }
-            }
-          }
           // create a subtest for the specific folder + test file + test name
-          tap1.test(name, { jobs: 1, bail: this.bailout }, tap2 => {
+          t.test(name, async t => {
-            const testRunner = TestRunner({
+            const testRunner = new TestRunner({
               client,
               version,
-              tap: tap2,
+              tap: t,
-              isPlatinum: file.includes('x-pack')
+              isXPack: file.includes('x-pack')
             })
-            testRunner.run(setupTest, test[name], teardownTest, () => tap2.end())
+            await testRunner.run(setupTest, test[name], teardownTest)
           })
         })
-
+        t.end()
-        tap1.end()
+      }
-      })
     }
   }
-}
 
-/**
+  parse (data) {
- * Parses a given yaml document
+    try {
- * @param {string} yaml document
+      var doc = yaml.safeLoad(data)
- * @returns {object}
+    } catch (err) {
- */
+      console.error(err)
-Runner.prototype.parse = function (data) {
-  try {
-    var doc = yaml.safeLoad(data)
-  } catch (err) {
-    this.log.fail(err.message)
-    return
-  }
-  return doc
-}
-
-/**
- * Returns the filtered content of a given folder
- * @param {string} folder
- * @returns {Array} The content of the given folder
- */
-Runner.prototype.getTest = function (folder) {
-  const tests = readdirSync(folder)
-  return tests.filter(t => !/(README|TODO)/g.test(t))
-}
-
-/**
- * Sets the elasticsearch repository to the given sha.
- * If the repository is not present in `esFolder` it will
- * clone the repository and the checkout the sha.
- * If the repository is already present but it cannot checkout to
- * the given sha, it will perform a pull and then try again.
- * @param {string} sha
- * @param {function} callback
- */
-Runner.prototype.withSHA = function (sha, callback) {
-  var fresh = false
-  var retry = 0
-  var log = this.log
-
-  if (!this.pathExist(esFolder)) {
-    if (!this.createFolder(esFolder)) {
-      log.fail('Failed folder creation')
       return
     }
-    fresh = true
+    return doc
   }
 
-  const git = Git(esFolder)
+  getTest (folder) {
-
+    const tests = readdirSync(folder)
-  if (fresh) {
+    return tests.filter(t => !/(README|TODO)/g.test(t))
-    clone(checkout)
-  } else {
-    checkout()
   }
 
-  function checkout () {
+  /**
-    log.text = `Checking out sha '${sha}'`
+   * Sets the elasticsearch repository to the given sha.
-    git.checkout(sha, err => {
+   * If the repository is not present in `esFolder` it will
-      if (err) {
+   * clone the repository and the checkout the sha.
-        if (retry++ > 0) {
+   * If the repository is already present but it cannot checkout to
-          log.fail(`Cannot checkout sha '${sha}'`)
+   * the given sha, it will perform a pull and then try again.
-          return
+   * @param {string} sha
+   * @param {function} callback
+   */
+  withSHA (sha) {
+    return new Promise((resolve, reject) => {
+      _withSHA.call(this, err => err ? reject(err) : resolve())
+    })
+
+    function _withSHA (callback) {
+      var fresh = false
+      var retry = 0
+
+      if (!this.pathExist(esFolder)) {
+        if (!this.createFolder(esFolder)) {
+          return callback(new Error('Failed folder creation'))
         }
-        return pull(checkout)
+        fresh = true
       }
-      callback()
-    })
-  }
 
-  function pull (cb) {
+      const git = Git(esFolder)
-    log.text = 'Pulling elasticsearch repository...'
+
-    git.pull(err => {
+      if (fresh) {
-      if (err) {
+        clone(checkout)
-        log.fail(err.message)
+      } else {
-        return
+        checkout()
       }
-      cb()
-    })
-  }
 
-  function clone (cb) {
+      function checkout () {
-    log.text = 'Cloning elasticsearch repository...'
+        console.log(`Checking out sha '${sha}'`)
-    git.clone(esRepo, esFolder, err => {
+        git.checkout(sha, err => {
-      if (err) {
+          if (err) {
-        log.fail(err.message)
+            if (retry++ > 0) {
-        return
+              return callback(err)
+            }
+            return pull(checkout)
+          }
+          callback()
+        })
       }
-      cb()
-    })
-  }
-}
 
-/**
+      function pull (cb) {
- * Checks if the given path exists
+        console.log('Pulling elasticsearch repository...')
- * @param {string} path
+        git.pull(err => {
- * @returns {boolean} true if exists, false if not
+          if (err) {
- */
+            return callback(err)
-Runner.prototype.pathExist = function (path) {
+          }
-  try {
+          cb()
-    accessSync(path)
+        })
-    return true
+      }
-  } catch (err) {
-    return false
-  }
-}
 
-/**
+      function clone (cb) {
- * Creates the given folder
+        console.log('Cloning elasticsearch repository...')
- * @param {string} name
+        git.clone(esRepo, esFolder, err => {
- * @returns {boolean} true on success, false on failure
+          if (err) {
- */
+            return callback(err)
-Runner.prototype.createFolder = function (name) {
+          }
-  try {
+          cb()
-    mkdirSync(name)
+        })
-    return true
+      }
-  } catch (err) {
+    }
-    return false
+  }
+
+  /**
+   * Checks if the given path exists
+   * @param {string} path
+   * @returns {boolean} true if exists, false if not
+   */
+  pathExist (path) {
+    try {
+      accessSync(path)
+      return true
+    } catch (err) {
+      return false
+    }
+  }
+
+  /**
+   * Creates the given folder
+   * @param {string} name
+   * @returns {boolean} true on success, false on failure
+   */
+  createFolder (name) {
+    try {
+      mkdirSync(name)
+      return true
+    } catch (err) {
+      return false
+    }
   }
 }
 
 if (require.main === module) {
-  const url = process.env.TEST_ES_SERVER || 'http://localhost:9200'
+  const node = process.env.TEST_ES_SERVER || 'http://localhost:9200'
   const opts = {
-    node: url,
+    node,
-    isPlatinum: url.indexOf('@') > -1
+    isXPack: node.indexOf('@') > -1
   }
-  const runner = Runner(opts)
+  const runner = new Runner(opts)
-  runner.start(opts)
+  runner.start(opts).catch(console.log)
+}
+
+const shouldSkip = (t, isXPack, file, name) => {
+  var list = Object.keys(ossSkips)
+  for (var i = 0; i < list.length; i++) {
+    const ossTest = ossSkips[list[i]]
+    for (var j = 0; j < ossTest.length; j++) {
+      if (file.endsWith(list[i]) && (name === ossTest[j] || ossTest[j] === '*')) {
+        const testName = file.slice(file.indexOf(`${sep}elasticsearch${sep}`)) + ' / ' + name
+        t.comment(`Skipping test ${testName} because is blacklisted in the oss test`)
+        return true
+      }
+    }
+  }
+
+  if (file.includes('x-pack') || isXPack) {
+    list = Object.keys(xPackBlackList)
+    for (i = 0; i < list.length; i++) {
+      const platTest = xPackBlackList[list[i]]
+      for (j = 0; j < platTest.length; j++) {
+        if (file.endsWith(list[i]) && (name === platTest[j] || platTest[j] === '*')) {
+          const testName = file.slice(file.indexOf(`${sep}elasticsearch${sep}`)) + ' / ' + name
+          t.comment(`Skipping test ${testName} because is blacklisted in the XPack test`)
+          return true
+        }
+      }
+    }
+  }
+
+  return false
 }
 
 const getAllFiles = dir =>