Improve integration test (#859)

* CI: Added junit plugin

* Updated .gitignore

* Added integration test reporter

* Updated integration testing suite

* Updated ci config

* Updated report file path

* Use refresh 'true' instead of 'wait_for'

* Disable junit reporting

* Refresh one single time

* Update security index name

* Updated skip test handling and use class syntax

* Updated test script

* Disable test timeout

* Added command to automatically remove an old snapshot

* Disable timeout in integration test script

* Updated logs and cleaned up git handling

* Fixed shouldSkip utility

* Updated cleanup code

* Updated cleanup code pt 2

* Rename Platinum to XPack

.ci/jobs/defaults.yml

@@ -65,3 +65,6 @@
     publishers:
     - email:
         recipients: infra-root+build@elastic.co
+    # - junit:
+    #     results: "*-junit.xml"
+    #     allow-empty-results: true

.gitignore

@@ -55,3 +55,5 @@ elasticsearch*
 api/generated.d.ts
 
 test/benchmarks/macro/fixtures/*
+
+*-junit.xml

package.json

@@ -19,7 +19,8 @@
     "test": "npm run lint && npm run test:unit && npm run test:behavior && npm run test:types",
     "test:unit": "tap test/unit/*.test.js -t 300 --no-coverage",
     "test:behavior": "tap test/behavior/*.test.js -t 300 --no-coverage",
-    "test:integration": "tap test/integration/index.js -T --harmony --no-esm --no-coverage",
+    "test:integration": "tap test/integration/index.js -T --no-coverage",
+    "test:integration:report": "npm run test:integration | tap-mocha-reporter xunit > $WORKSPACE/test-report-junit.xml",
     "test:types": "tsc --project ./test/types/tsconfig.json",
     "test:coverage": "nyc npm run test:unit && nyc report --reporter=text-lcov > coverage.lcov && codecov",
     "lint": "standard",
@@ -56,6 +57,7 @@
     "standard": "^12.0.1",
     "stoppable": "^1.1.0",
     "tap": "^13.0.1",
+    "tap-mocha-reporter": "^4.0.1",
     "typescript": "^3.4.5",
     "workq": "^2.1.0"
   },

scripts/es-docker-platinum.sh

@@ -9,6 +9,11 @@ testnodecrt="/.ci/certs/testnode.crt"
 testnodekey="/.ci/certs/testnode.key"
 cacrt="/.ci/certs/ca.crt"
 
+# pass `--clean` to reemove the old snapshot
+if [ "$1" != "" ]; then
+  docker rmi $(docker images --format '{{.Repository}}:{{.Tag}}' | grep '8.0.0-SNAPSHOT')
+fi
+
 exec docker run \
   --rm \
   -e "node.attr.testattr=test" \

scripts/es-docker.sh

@@ -4,6 +4,11 @@
 # to delete an old image and download again
 # the latest snapshot.
 
+# pass `--clean` to reemove the old snapshot
+if [ "$1" != "" ]; then
+  docker rmi $(docker images --format '{{.Repository}}:{{.Tag}}' | grep '8.0.0-SNAPSHOT')
+fi
+
 exec docker run \
   --rm \
   -e "node.attr.testattr=test" \

test/integration/helper.js

@@ -58,11 +58,11 @@ const esDefaultUsers = [
   'remote_monitoring_user'
 ]
 
-function runInParallel (client, operation, options) {
+function runInParallel (client, operation, options, clientOptions) {
   if (options.length === 0) return Promise.resolve()
   const operations = options.map(opts => {
     const api = delve(client, operation).bind(client)
-    return api(opts)
+    return api(opts, clientOptions)
   })
 
   return Promise.all(operations)
@@ -82,4 +82,10 @@ function delve (obj, key, def, p) {
   return (obj === undefined || p < key.length) ? def : obj
 }
 
-module.exports = { runInParallel, esDefaultRoles, esDefaultUsers, delve }
+function to (promise) {
+  return promise.then(data => [null, data], err => [err, undefined])
+}
+
+const sleep = ms => new Promise(resolve => setTimeout(resolve, ms))
+
+module.exports = { runInParallel, esDefaultRoles, esDefaultUsers, delve, to, sleep }

test/integration/index.js

@@ -19,32 +19,32 @@
 
 'use strict'
 
-const assert = require('assert')
 const { readFileSync, accessSync, mkdirSync, readdirSync, statSync } = require('fs')
 const { join, sep } = require('path')
 const yaml = require('js-yaml')
 const Git = require('simple-git')
-const ora = require('ora')
 const tap = require('tap')
 const { Client } = require('../../index')
 const TestRunner = require('./test-runner')
+const { sleep } = require('./helper')
 
 const esRepo = 'https://github.com/elastic/elasticsearch.git'
 const esFolder = join(__dirname, '..', '..', 'elasticsearch')
 const yamlFolder = join(esFolder, 'rest-api-spec', 'src', 'main', 'resources', 'rest-api-spec', 'test')
 const xPackYamlFolder = join(esFolder, 'x-pack', 'plugin', 'src', 'test', 'resources', 'rest-api-spec', 'test')
-const customSkips = [
+
+const ossSkips = {
   // TODO: remove this once 'arbitrary_key' is implemented
   // https://github.com/elastic/elasticsearch/pull/41492
-  'indices.split/30_copy_settings.yml',
+  'indices.split/30_copy_settings.yml': ['*'],
   // skipping because we are booting ES with `discovery.type=single-node`
   // and this test will fail because of this configuration
-  'nodes.stats/30_discovery.yml',
+  'nodes.stats/30_discovery.yml': ['*'],
   // the expected error is returning a 503,
   // which triggers a retry and the node to be marked as dead
-  'search.aggregation/240_max_buckets.yml'
+  'search.aggregation/240_max_buckets.yml': ['*']
-]
+}
-const platinumBlackList = {
+const xPackBlackList = {
   // file path: test name
   'cat.aliases/10_basic.yml': ['Empty cluster'],
   'index/10_with_id.yml': ['Index with ID'],
@@ -81,95 +81,76 @@ const platinumBlackList = {
   'xpack/15_basic.yml': ['*']
 }
 
-function Runner (opts) {
+class Runner {
-  if (!(this instanceof Runner)) {
+  constructor (opts = {}) {
-    return new Runner(opts)
-  }
-  opts = opts || {}
-
-  assert(opts.node, 'Missing base node')
-  this.bailout = opts.bailout
     const options = { node: opts.node }
-  if (opts.isPlatinum) {
+    if (opts.isXPack) {
       options.ssl = {
-      // NOTE: this path works only if we run
+        ca: readFileSync(join(__dirname, '..', '..', '.ci', 'certs', 'ca.crt'), 'utf8'),
-      // the suite with npm scripts
-      ca: readFileSync('.ci/certs/ca.crt', 'utf8'),
         rejectUnauthorized: false
       }
     }
     this.client = new Client(options)
-  this.log = ora('Loading yaml suite').start()
+    console.log('Loading yaml suite')
-}
+  }
 
-Runner.prototype.waitCluster = function (callback, times = 0) {
+  async waitCluster (client, times = 0) {
-  this.log.text = 'Waiting for ElasticSearch'
+    try {
-  this.client.cluster.health(
+      await client.cluster.health({ waitForStatus: 'green', timeout: '50s' })
-    { waitForStatus: 'green', timeout: '50s' },
+    } catch (err) {
-    (err, res) => {
+      if (++times < 10) {
-      if (err && ++times < 10) {
+        await sleep(5000)
-        setTimeout(() => {
+        return this.waitCluster(client, times)
-          this.waitCluster(callback, times)
+      }
-        }, 5000)
+      console.error(err)
-      } else {
+      process.exit(1)
-        callback(err)
     }
   }
-  )
-}
 
-/**
+  async start ({ isXPack }) {
- * Runs the test suite
+    const { client } = this
- */
-Runner.prototype.start = function (opts) {
     const parse = this.parse.bind(this)
-  const client = this.client
 
-  // client.on('response', (err, meta) => {
+    console.log('Waiting for Elasticsearch')
-  //   console.log('Request', meta.request)
+    await this.waitCluster(client)
-  //   if (err) {
-  //     console.log('Error', err)
-  //   } else {
-  //     console.log('Response', JSON.stringify(meta.response, null, 2))
-  //   }
-  //   console.log()
-  // })
 
-  this.waitCluster(err => {
+    const { body } = await client.info()
-    if (err) {
-      this.log.fail(err.message)
-      process.exit(1)
-    }
-    // Get the build hash of Elasticsearch
-    client.info((err, { body }) => {
-      if (err) {
-        this.log.fail(err.message)
-        process.exit(1)
-      }
     const { number: version, build_hash: sha } = body.version
 
-      // Set the repository to the given sha and run the test suite
+    console.log(`Checking out sha ${sha}...`)
-      this.withSHA(sha, () => {
+    await this.withSHA(sha)
-        this.log.succeed(`Testing ${opts.isPlatinum ? 'platinum' : 'oss'} api...`)
-        runTest.call(this, version)
-      })
-    })
-  })
 
-  function runTest (version) {
+    console.log(`Testing ${isXPack ? 'XPack' : 'oss'} api...`)
-    const files = []
+
+    const folders = []
       .concat(getAllFiles(yamlFolder))
-      .concat(opts.isPlatinum ? getAllFiles(xPackYamlFolder) : [])
+      .concat(isXPack ? getAllFiles(xPackYamlFolder) : [])
       .filter(t => !/(README|TODO)/g.test(t))
-
+      // we cluster the array based on the folder names,
-    files.forEach(runTestFile.bind(this))
+      // to provide a better test log output
-    function runTestFile (file) {
+      .reduce((arr, file) => {
-      for (var i = 0; i < customSkips.length; i++) {
+        const path = file.slice(file.indexOf('/rest-api-spec/test'), file.lastIndexOf('/'))
-        if (file.endsWith(customSkips[i])) return
+        var inserted = false
+        for (var i = 0; i < arr.length; i++) {
+          if (arr[i][0].includes(path)) {
+            inserted = true
+            arr[i].push(file)
+            break
           }
-      // create a subtest for the specific folder
+        }
-      tap.test(file.slice(file.indexOf(`${sep}elasticsearch${sep}`)), { jobs: 1 }, tap1 => {
+        if (!inserted) arr.push([file])
-        // read the yaml file
+        return arr
+      }, [])
+
+    for (const folder of folders) {
+      // pretty name
+      const apiName = folder[0].slice(
+        folder[0].indexOf(`${sep}rest-api-spec${sep}test`) + 19,
+        folder[0].lastIndexOf(sep)
+      )
+
+      tap.test(`Testing ${apiName}`, { bail: true, timeout: 0 }, t => {
+        for (const file of folder) {
           const data = readFileSync(file, 'utf8')
           // get the test yaml (as object), some file has multiple yaml documents inside,
           // every document is separated by '---', so we split on the separator
@@ -180,77 +161,62 @@ Runner.prototype.start = function (opts) {
             .filter(Boolean)
             .map(parse)
 
+          t.test(
+            file.slice(file.lastIndexOf(apiName)),
+            testFile(file, tests)
+          )
+        }
+        t.end()
+      })
+    }
+
+    function testFile (file, tests) {
+      return t => {
         // get setup and teardown if present
         var setupTest = null
         var teardownTest = null
-        tests.forEach(test => {
+        for (const test of tests) {
           if (test.setup) setupTest = test.setup
           if (test.teardown) teardownTest = test.teardown
-        })
+        }
 
-        // run the tests
         tests.forEach(test => {
           const name = Object.keys(test)[0]
           if (name === 'setup' || name === 'teardown') return
-          // should skip the test inside `platinumBlackList`
+          if (shouldSkip(t, isXPack, file, name)) return
-          // if we are testing the platinum apis
+
-          if (opts.isPlatinum) {
-            const list = Object.keys(platinumBlackList)
-            for (i = 0; i < list.length; i++) {
-              const platTest = platinumBlackList[list[i]]
-              for (var j = 0; j < platTest.length; j++) {
-                if (file.endsWith(list[i]) && (name === platTest[j] || platTest[j] === '*')) {
-                  const testName = file.slice(file.indexOf(`${sep}elasticsearch${sep}`)) + ' / ' + name
-                  tap.skip(`Skipping test ${testName} because is blacklisted in the platinum test`)
-                  return
-                }
-              }
-            }
-          }
           // create a subtest for the specific folder + test file + test name
-          tap1.test(name, { jobs: 1, bail: this.bailout }, tap2 => {
+          t.test(name, async t => {
-            const testRunner = TestRunner({
+            const testRunner = new TestRunner({
               client,
               version,
-              tap: tap2,
+              tap: t,
-              isPlatinum: file.includes('x-pack')
+              isXPack: file.includes('x-pack')
             })
-            testRunner.run(setupTest, test[name], teardownTest, () => tap2.end())
+            await testRunner.run(setupTest, test[name], teardownTest)
           })
         })
-
+        t.end()
-        tap1.end()
+      }
-      })
     }
   }
-}
 
-/**
+  parse (data) {
- * Parses a given yaml document
- * @param {string} yaml document
- * @returns {object}
- */
-Runner.prototype.parse = function (data) {
     try {
       var doc = yaml.safeLoad(data)
     } catch (err) {
-    this.log.fail(err.message)
+      console.error(err)
       return
     }
     return doc
-}
+  }
 
-/**
+  getTest (folder) {
- * Returns the filtered content of a given folder
- * @param {string} folder
- * @returns {Array} The content of the given folder
- */
-Runner.prototype.getTest = function (folder) {
     const tests = readdirSync(folder)
     return tests.filter(t => !/(README|TODO)/g.test(t))
-}
+  }
 
-/**
+  /**
    * Sets the elasticsearch repository to the given sha.
    * If the repository is not present in `esFolder` it will
    * clone the repository and the checkout the sha.
@@ -259,15 +225,18 @@ Runner.prototype.getTest = function (folder) {
    * @param {string} sha
    * @param {function} callback
    */
-Runner.prototype.withSHA = function (sha, callback) {
+  withSHA (sha) {
+    return new Promise((resolve, reject) => {
+      _withSHA.call(this, err => err ? reject(err) : resolve())
+    })
+
+    function _withSHA (callback) {
       var fresh = false
       var retry = 0
-  var log = this.log
 
       if (!this.pathExist(esFolder)) {
         if (!this.createFolder(esFolder)) {
-      log.fail('Failed folder creation')
+          return callback(new Error('Failed folder creation'))
-      return
         }
         fresh = true
       }
@@ -281,12 +250,11 @@ Runner.prototype.withSHA = function (sha, callback) {
       }
 
       function checkout () {
-    log.text = `Checking out sha '${sha}'`
+        console.log(`Checking out sha '${sha}'`)
         git.checkout(sha, err => {
           if (err) {
             if (retry++ > 0) {
-          log.fail(`Cannot checkout sha '${sha}'`)
+              return callback(err)
-          return
             }
             return pull(checkout)
           }
@@ -295,64 +263,94 @@ Runner.prototype.withSHA = function (sha, callback) {
       }
 
       function pull (cb) {
-    log.text = 'Pulling elasticsearch repository...'
+        console.log('Pulling elasticsearch repository...')
         git.pull(err => {
           if (err) {
-        log.fail(err.message)
+            return callback(err)
-        return
           }
           cb()
         })
       }
 
       function clone (cb) {
-    log.text = 'Cloning elasticsearch repository...'
+        console.log('Cloning elasticsearch repository...')
         git.clone(esRepo, esFolder, err => {
           if (err) {
-        log.fail(err.message)
+            return callback(err)
-        return
           }
           cb()
         })
       }
-}
+    }
+  }
 
-/**
+  /**
    * Checks if the given path exists
    * @param {string} path
    * @returns {boolean} true if exists, false if not
    */
-Runner.prototype.pathExist = function (path) {
+  pathExist (path) {
     try {
       accessSync(path)
       return true
     } catch (err) {
       return false
     }
-}
+  }
 
-/**
+  /**
    * Creates the given folder
    * @param {string} name
    * @returns {boolean} true on success, false on failure
    */
-Runner.prototype.createFolder = function (name) {
+  createFolder (name) {
     try {
       mkdirSync(name)
       return true
     } catch (err) {
       return false
     }
+  }
 }
 
 if (require.main === module) {
-  const url = process.env.TEST_ES_SERVER || 'http://localhost:9200'
+  const node = process.env.TEST_ES_SERVER || 'http://localhost:9200'
   const opts = {
-    node: url,
+    node,
-    isPlatinum: url.indexOf('@') > -1
+    isXPack: node.indexOf('@') > -1
   }
-  const runner = Runner(opts)
+  const runner = new Runner(opts)
-  runner.start(opts)
+  runner.start(opts).catch(console.log)
+}
+
+const shouldSkip = (t, isXPack, file, name) => {
+  var list = Object.keys(ossSkips)
+  for (var i = 0; i < list.length; i++) {
+    const ossTest = ossSkips[list[i]]
+    for (var j = 0; j < ossTest.length; j++) {
+      if (file.endsWith(list[i]) && (name === ossTest[j] || ossTest[j] === '*')) {
+        const testName = file.slice(file.indexOf(`${sep}elasticsearch${sep}`)) + ' / ' + name
+        t.comment(`Skipping test ${testName} because is blacklisted in the oss test`)
+        return true
+      }
+    }
+  }
+
+  if (file.includes('x-pack') || isXPack) {
+    list = Object.keys(xPackBlackList)
+    for (i = 0; i < list.length; i++) {
+      const platTest = xPackBlackList[list[i]]
+      for (j = 0; j < platTest.length; j++) {
+        if (file.endsWith(list[i]) && (name === platTest[j] || platTest[j] === '*')) {
+          const testName = file.slice(file.indexOf(`${sep}elasticsearch${sep}`)) + ' / ' + name
+          t.comment(`Skipping test ${testName} because is blacklisted in the XPack test`)
+          return true
+        }
+      }
+    }
+  }
+
+  return false
 }
 
 const getAllFiles = dir =>

test/integration/test-runner.js

@@ -19,13 +19,14 @@
 
 'use strict'
 
+/* eslint camelcase: 0 */
+
 const t = require('tap')
 const semver = require('semver')
-const workq = require('workq')
 const helper = require('./helper')
 const { ConfigurationError } = require('../../lib/errors')
 
-const { delve } = helper
+const { delve, to } = helper
 
 const supportedFeatures = [
   'gtelte',
@@ -38,10 +39,8 @@ const supportedFeatures = [
   'catch_unauthorized'
 ]
 
-function TestRunner (opts) {
+class TestRunner {
-  if (!(this instanceof TestRunner)) {
+  constructor (opts = {}) {
-    return new TestRunner(opts)
-  }
     opts = opts || {}
 
     this.client = opts.client
@@ -49,158 +48,144 @@ function TestRunner (opts) {
     this.response = null
     this.stash = new Map()
     this.tap = opts.tap || t
-  this.isPlatinum = opts.isPlatinum
+    this.isXPack = opts.isXPack
-  this.q = opts.q || workq()
+  }
-}
 
-/**
+  /**
- * Runs a cleanup, removes all indices and templates
+   * Runs a cleanup, removes all indices, aliases, templates, and snapshots
- * @param {queue}
+   * @returns {Promise}
- * @param {function} done
    */
-TestRunner.prototype.cleanup = function (q, done) {
+  async cleanup () {
     this.tap.comment('Cleanup')
 
     this.response = null
     this.stash = new Map()
 
-  q.add((q, done) => {
+    try {
-    this.client.indices.delete({ index: '*' }, { ignore: 404 }, err => {
+      await this.client.indices.delete({ index: '_all' }, { ignore: 404 })
+    } catch (err) {
       this.tap.error(err, 'should not error: indices.delete')
-      done()
+    }
-    })
-  })
 
-  q.add((q, done) => {
+    try {
-    this.client.indices.deleteTemplate({ name: '*' }, { ignore: 404 }, err => {
+      await this.client.indices.deleteAlias({ index: '_all', name: '_all' }, { ignore: 404 })
+    } catch (err) {
+      this.tap.error(err, 'should not error: indices.deleteAlias')
+    }
+
+    try {
+      const { body: templates } = await this.client.indices.getTemplate()
+      await helper.runInParallel(
+        this.client, 'indices.deleteTemplate',
+        Object.keys(templates).map(t => ({ name: t }))
+      )
+    } catch (err) {
       this.tap.error(err, 'should not error: indices.deleteTemplate')
-      done()
+    }
-    })
-  })
 
-  q.add((q, done) => {
+    try {
-    this.client.snapshot.delete({ repository: '*', snapshot: '*' }, { ignore: 404 }, err => {
+      const { body: repositories } = await this.client.snapshot.getRepository()
-      this.tap.error(err, 'should not error: snapshot.delete')
+      for (const repository of Object.keys(repositories)) {
-      done()
+        const { body: snapshots } = await this.client.snapshot.get({ repository, snapshot: '_all' })
-    })
+        await helper.runInParallel(
-  })
+          this.client, 'snapshot.delete',
+          Object.keys(snapshots).map(snapshot => ({ snapshot, repository })),
+          { ignore: [404] }
+        )
+        await this.client.snapshot.deleteRepository({ repository }, { ignore: [404] })
+      }
+    } catch (err) {
+      this.tap.error(err, 'should not error: snapshot.delete / snapshot.deleteRepository')
+    }
+  }
 
-  q.add((q, done) => {
+  /**
-    this.client.snapshot.deleteRepository({ repository: '*' }, { ignore: 404 }, err => {
+   * Runs some additional API calls to prepare ES for the xpack test,
-      this.tap.error(err, 'should not error: snapshot.deleteRepository')
-      done()
-    })
-  })
-
-  done()
-}
-
-/**
- * Runs some additional API calls to prepare ES for the Platinum test,
    * This set of calls should be executed before the final clenup.
- * @param {queue}
+   * @returns {Promise}
- * @param {function} done
    */
-TestRunner.prototype.cleanupPlatinum = function (q, done) {
+  async cleanupXPack () {
-  this.tap.comment('Platinum Cleanup')
+    this.tap.comment('XPack Cleanup')
 
-  q.add((q, done) => {
+    try {
-    this.client.security.getRole((err, { body }) => {
+      const { body } = await this.client.security.getRole()
-      this.tap.error(err, 'should not error: security.getRole')
       const roles = Object.keys(body).filter(n => helper.esDefaultRoles.indexOf(n) === -1)
-      helper.runInParallel(
+      await helper.runInParallel(
         this.client, 'security.deleteRole',
-        roles.map(r => ({ name: r, refresh: 'wait_for' }))
+        roles.map(r => ({ name: r }))
       )
-        .then(() => done())
+    } catch (err) {
-        .catch(err => this.tap.error(err, 'should not error: security.deleteRole'))
+      this.tap.error(err, 'should not error: security role cleanup')
-    })
+    }
-  })
 
-  q.add((q, done) => {
+    try {
-    this.client.security.getUser((err, { body }) => {
+      const { body } = await this.client.security.getUser()
-      this.tap.error(err, 'should not error: security.getUser')
       const users = Object.keys(body).filter(n => helper.esDefaultUsers.indexOf(n) === -1)
-      helper.runInParallel(
+      await helper.runInParallel(
         this.client, 'security.deleteUser',
-        users.map(r => ({ username: r, refresh: 'wait_for' }))
+        users.map(r => ({ username: r }))
       )
-        .then(() => done())
+    } catch (err) {
-        .catch(err => this.tap.error(err, 'should not error: security.deleteUser'))
+      this.tap.error(err, 'should not error: security user cleanup')
-    })
+    }
-  })
 
-  q.add((q, done) => {
+    try {
-    this.client.security.getPrivileges((err, { body }) => {
+      const { body } = await this.client.security.getPrivileges()
-      this.tap.error(err, 'should not error: security.getPrivileges')
       const privileges = []
       Object.keys(body).forEach(app => {
         Object.keys(body[app]).forEach(priv => {
           privileges.push({
             name: body[app][priv].name,
-            application: body[app][priv].application,
+            application: body[app][priv].application
-            refresh: 'wait_for'
           })
         })
       })
-      helper.runInParallel(this.client, 'security.deletePrivileges', privileges)
+      await helper.runInParallel(this.client, 'security.deletePrivileges', privileges)
-        .then(() => done())
+    } catch (err) {
-        .catch(err => this.tap.error(err, 'should not error: security.deletePrivileges'))
+      this.tap.error(err, 'should not error: security privileges cleanup')
-    })
+    }
-  })
 
-  q.add((q, done) => {
+    try {
-    this.client.ml.stopDatafeed({ datafeedId: '*', force: true }, err => {
+      await this.client.ml.stopDatafeed({ datafeedId: '*', force: true })
-      this.tap.error(err, 'should not error: ml.stopDatafeed')
+      const { body } = await this.client.ml.getDatafeeds({ datafeedId: '*' })
-      this.client.ml.getDatafeeds({ datafeedId: '*' }, (err, { body }) => {
-        this.tap.error(err, 'should error: not ml.getDatafeeds')
       const feeds = body.datafeeds.map(f => f.datafeed_id)
-        helper.runInParallel(
+      await helper.runInParallel(
         this.client, 'ml.deleteDatafeed',
         feeds.map(f => ({ datafeedId: f }))
       )
-          .then(() => done())
+    } catch (err) {
-          .catch(err => this.tap.error(err, 'should not error: ml.deleteDatafeed'))
+      this.tap.error(err, 'should error: not ml datafeed cleanup')
-      })
+    }
-    })
-  })
 
-  q.add((q, done) => {
+    try {
-    this.client.ml.closeJob({ jobId: '*', force: true }, err => {
+      await this.client.ml.closeJob({ jobId: '*', force: true })
-      this.tap.error(err, 'should not error: ml.closeJob')
+      const { body } = await this.client.ml.getJobs({ jobId: '*' })
-      this.client.ml.getJobs({ jobId: '*' }, (err, { body }) => {
-        this.tap.error(err, 'should not error: ml.getJobs')
       const jobs = body.jobs.map(j => j.job_id)
-        helper.runInParallel(
+      await helper.runInParallel(
         this.client, 'ml.deleteJob',
         jobs.map(j => ({ jobId: j, waitForCompletion: true, force: true }))
       )
-          .then(() => done())
+    } catch (err) {
-          .catch(err => this.tap.error(err, 'should not error: ml.deleteJob'))
+      this.tap.error(err, 'should not error: ml job cleanup')
-      })
+    }
-    })
-  })
 
-  q.add((q, done) => {
+    try {
-    this.client.rollup.getJobs({ id: '_all' }, (err, { body }) => {
+      const { body } = await this.client.rollup.getJobs({ id: '_all' })
-      this.tap.error(err, 'should not error: rollup.getJobs')
       const jobs = body.jobs.map(j => j.config.id)
-      helper.runInParallel(
+      await helper.runInParallel(
         this.client, 'rollup.stopJob',
         jobs.map(j => ({ id: j, waitForCompletion: true }))
       )
-        .then(() => helper.runInParallel(
+      await helper.runInParallel(
         this.client, 'rollup.deleteJob',
         jobs.map(j => ({ id: j }))
-        ))
+      )
-        .then(() => done())
+    } catch (err) {
-        .catch(err => this.tap.error(err, 'should not error: rollup.stopJob/deleteJob'))
+      this.tap.error(err, 'should not error: rollup jobs cleanup')
-    })
+    }
-  })
 
-  q.add((q, done) => {
+    try {
-    this.client.tasks.list((err, { body }) => {
+      const { body } = await this.client.tasks.list()
-      this.tap.error(err, 'should not error: tasks.list')
       const tasks = Object.keys(body.nodes)
         .reduce((acc, node) => {
           const { tasks } = body.nodes[node]
@@ -210,112 +195,98 @@ TestRunner.prototype.cleanupPlatinum = function (q, done) {
           return acc
         }, [])
 
-      helper.runInParallel(
+      await helper.runInParallel(
         this.client, 'tasks.cancel',
         tasks.map(id => ({ taskId: id }))
       )
-        .then(() => done())
+    } catch (err) {
-        .catch(err => this.tap.error(err, 'should not error: tasks.cancel'))
+      this.tap.error(err, 'should not error: tasks cleanup')
-    })
+    }
-  })
 
-  q.add((q, done) => {
+    try {
-    this.client.indices.delete({ index: '.ml-*' }, { ignore: 404 }, err => {
+      await this.client.ilm.removePolicy({ index: '_all' })
-      this.tap.error(err, 'should not error: indices.delete (ml indices)')
+    } catch (err) {
-      done()
+      this.tap.error(err, 'should not error: ilm.removePolicy')
-    })
+    }
-  })
 
-  done()
+    // refresh the all indexes
-}
+    try {
+      await this.client.indices.refresh({ index: '_all' })
+    } catch (err) {
+      this.tap.error(err, 'should not error: indices.refresh')
+    }
+  }
 
-/**
+  /**
    * Runs the given test.
    * It runs the test components in the following order:
+   *    - skip check
+   *    - xpack user
    *    - setup
    *    - the actual test
    *    - teardown
+   *    - xpack cleanup
    *    - cleanup
-* Internally uses a queue to guarantee the order of the test sections.
    * @param {object} setup (null if not needed)
    * @param {object} test
    * @oaram {object} teardown (null if not needed)
- * @param {function} end
+   * @returns {Promise}
    */
-TestRunner.prototype.run = function (setup, test, teardown, end) {
+  async run (setup, test, teardown) {
     // if we should skip a feature in the setup/teardown section
     // we should skip the entire test file
     const skip = getSkip(setup) || getSkip(teardown)
     if (skip && this.shouldSkip(skip)) {
       this.skip(skip)
-    return end()
+      return
     }
 
-  if (this.isPlatinum) {
+    if (this.isXPack) {
+      // Some xpack test requires this user
       this.tap.comment('Creating x-pack user')
-    // Some platinum test requires this user
+      try {
-    this.q.add((q, done) => {
+        await this.client.security.putUser({
-      this.client.security.putUser({
           username: 'x_pack_rest_user',
           body: { password: 'x-pack-test-password', roles: ['superuser'] }
-      }, (err, { body }) => {
+        })
+      } catch (err) {
         this.tap.error(err, 'should not error: security.putUser')
-        done()
+      }
-      })
-    })
     }
 
-  if (setup) {
+    if (setup) await this.exec('Setup', setup)
-    this.q.add((q, done) => {
+
-      this.exec('Setup', setup, q, done)
+    await this.exec('Test', test)
-    })
+
+    if (teardown) await this.exec('Teardown', teardown)
+
+    if (this.isXPack) await this.cleanupXPack()
+
+    await this.cleanup()
   }
 
-  this.q.add((q, done) => {
+  /**
-    this.exec('Test', test, q, done)
-  })
-
-  if (teardown) {
-    this.q.add((q, done) => {
-      this.exec('Teardown', teardown, q, done)
-    })
-  }
-
-  if (this.isPlatinum) {
-    this.q.add((q, done) => {
-      this.cleanupPlatinum(q, done)
-    })
-  }
-
-  this.q.add((q, done) => {
-    this.cleanup(q, done)
-  })
-
-  this.q.add((q, done) => end() && done())
-}
-
-/**
    * Logs a skip
    * @param {object} the actions
    * @returns {TestRunner}
    */
-TestRunner.prototype.skip = function (action) {
+  skip (action) {
     if (action.reason && action.version) {
-    this.tap.skip(`Skip: ${action.reason} (${action.version})`)
+      this.tap.comment(`Skip: ${action.reason} (${action.version})`)
     } else if (action.features) {
-    this.tap.skip(`Skip: ${JSON.stringify(action.features)})`)
+      this.tap.comment(`Skip: ${JSON.stringify(action.features)})`)
     } else {
-    this.tap.skip('Skipped')
+      this.tap.comment('Skipped')
     }
     return this
-}
+  }
 
-/**
+  /**
    * Decides if a test should be skipped
    * @param {object} the actions
    * @returns {boolean}
    */
-TestRunner.prototype.shouldSkip = function (action) {
+  shouldSkip (action) {
     var shouldSkip = false
     // skip based on the version
     if (action.version) {
@@ -347,15 +318,15 @@ TestRunner.prototype.shouldSkip = function (action) {
     if (shouldSkip) return true
 
     return false
-}
+  }
 
-/**
+  /**
    * Updates the array syntax of keys and values
    * eg: 'hits.hits.1.stuff' to 'hits.hits[1].stuff'
    * @param {object} the action to update
    * @returns {obj} the updated action
    */
-TestRunner.prototype.updateArraySyntax = function (obj) {
+  updateArraySyntax (obj) {
     const newObj = {}
 
     for (const key in obj) {
@@ -372,9 +343,9 @@ TestRunner.prototype.updateArraySyntax = function (obj) {
     }
 
     return newObj
-}
+  }
 
-/**
+  /**
    * Fill the stashed values of a command
    * let's say the we have stashed the `master` value,
    *    is_true: nodes.$master.transport.profiles
@@ -383,7 +354,7 @@ TestRunner.prototype.updateArraySyntax = function (obj) {
    * @param {object|string} the action to update
    * @returns {object|string} the updated action
    */
-TestRunner.prototype.fillStashedValues = function (obj) {
+  fillStashedValues (obj) {
     if (typeof obj === 'string') {
       return getStashedValues.call(this, obj)
     }
@@ -449,26 +420,26 @@ TestRunner.prototype.fillStashedValues = function (obj) {
       // and call `.join` will coerce it to a string.
       return arr.length > 1 ? arr.join('.') : arr[0]
     }
-}
+  }
 
-/**
+  /**
    * Stashes a value
    * @param {string} the key to search in the previous response
    * @param {string} the name to identify the stashed value
    * @returns {TestRunner}
    */
-TestRunner.prototype.set = function (key, name) {
+  set (key, name) {
     this.stash.set(name, delve(this.response, key))
     return this
-}
+  }
 
-/**
+  /**
    * Applies a given transformation and stashes the result.
    * @param {string} the name to identify the stashed value
    * @param {string} the transformation function as string
    * @returns {TestRunner}
    */
-TestRunner.prototype.transform_and_set = function (name, transform) {
+  transform_and_set (name, transform) {
     if (/base64EncodeCredentials/.test(transform)) {
       const [user, password] = transform
         .slice(transform.indexOf('(') + 1, -1)
@@ -480,19 +451,24 @@ TestRunner.prototype.transform_and_set = function (name, transform) {
       throw new Error(`Unknown transform: '${transform}'`)
     }
     return this
-}
+  }
 
-/**
+  /**
    * Runs a client command
    * @param {object} the action to perform
- * @param {Queue}
+   * @returns {Promise}
    */
-TestRunner.prototype.do = function (action, done) {
+  async do (action) {
     const cmd = this.parseDo(action)
     const api = delve(this.client, cmd.method).bind(this.client)
+
     const options = { ignore: cmd.params.ignore, headers: action.headers }
     if (cmd.params.ignore) delete cmd.params.ignore
-  api(cmd.params, options, (err, { body, warnings }) => {
+
+    const [err, result] = await to(api(cmd.params, options))
+    var warnings = result ? result.warnings : null
+    var body = result ? result.body : null
+
     if (action.warnings && warnings === null) {
       this.tap.fail('We should get a warning header', action.warnings)
     } else if (!action.warnings && warnings !== null) {
@@ -542,22 +518,17 @@ TestRunner.prototype.do = function (action, done) {
       this.tap.error(err, `should not error: ${cmd.method}`, action)
       this.response = body
     }
+  }
 
-    done()
+  /**
-  })
-}
-
-/**
    * Runs an actual test
    * @param {string} the name of the test
    * @param {object} the actions to perform
- * @param {Queue}
+   * @returns {Promise}
    */
-TestRunner.prototype.exec = function (name, actions, q, done) {
+  async exec (name, actions) {
     this.tap.comment(name)
-  for (var i = 0; i < actions.length; i++) {
+    for (const action of actions) {
-    const action = actions[i]
-
       if (action.skip) {
         if (this.shouldSkip(action.skip)) {
           this.skip(this.fillStashedValues(action.skip))
@@ -566,29 +537,20 @@ TestRunner.prototype.exec = function (name, actions, q, done) {
       }
 
       if (action.do) {
-      q.add((q, done) => {
+        await this.do(this.fillStashedValues(action.do))
-        this.do(this.fillStashedValues(action.do), done)
-      })
       }
 
       if (action.set) {
-      q.add((q, done) => {
         const key = Object.keys(action.set)[0]
         this.set(this.fillStashedValues(key), action.set[key])
-        done()
-      })
       }
 
       if (action.transform_and_set) {
-      q.add((q, done) => {
         const key = Object.keys(action.transform_and_set)[0]
         this.transform_and_set(key, action.transform_and_set[key])
-        done()
-      })
       }
 
       if (action.match) {
-      q.add((q, done) => {
         const key = Object.keys(action.match)[0]
         this.match(
           // in some cases, the yaml refers to the body with an empty string
@@ -600,56 +562,41 @@ TestRunner.prototype.exec = function (name, actions, q, done) {
             : this.fillStashedValues(action.match)[key],
           action.match
         )
-        done()
-      })
       }
 
       if (action.lt) {
-      q.add((q, done) => {
         const key = Object.keys(action.lt)[0]
         this.lt(
           delve(this.response, this.fillStashedValues(key)),
           this.fillStashedValues(action.lt)[key]
         )
-        done()
-      })
       }
 
       if (action.gt) {
-      q.add((q, done) => {
         const key = Object.keys(action.gt)[0]
         this.gt(
           delve(this.response, this.fillStashedValues(key)),
           this.fillStashedValues(action.gt)[key]
         )
-        done()
-      })
       }
 
       if (action.lte) {
-      q.add((q, done) => {
         const key = Object.keys(action.lte)[0]
         this.lte(
           delve(this.response, this.fillStashedValues(key)),
           this.fillStashedValues(action.lte)[key]
         )
-        done()
-      })
       }
 
       if (action.gte) {
-      q.add((q, done) => {
         const key = Object.keys(action.gte)[0]
         this.gte(
           delve(this.response, this.fillStashedValues(key)),
           this.fillStashedValues(action.gte)[key]
         )
-        done()
-      })
       }
 
       if (action.length) {
-      q.add((q, done) => {
         const key = Object.keys(action.length)[0]
         this.length(
           key === '$body' || key === ''
@@ -659,62 +606,55 @@ TestRunner.prototype.exec = function (name, actions, q, done) {
             ? action.length[key]
             : this.fillStashedValues(action.length)[key]
         )
-        done()
-      })
       }
 
       if (action.is_true) {
-      q.add((q, done) => {
         const isTrue = this.fillStashedValues(action.is_true)
         this.is_true(
           delve(this.response, isTrue),
           isTrue
         )
-        done()
-      })
       }
 
       if (action.is_false) {
-      q.add((q, done) => {
         const isFalse = this.fillStashedValues(action.is_false)
         this.is_false(
           delve(this.response, isFalse),
           isFalse
         )
-        done()
-      })
       }
     }
-  done()
+  }
-}
 
-/**
+  /**
    * Asserts that the given value is truthy
    * @param {any} the value to check
    * @param {string} an optional message
+   * @returns {TestRunner}
    */
-TestRunner.prototype.is_true = function (val, msg) {
+  is_true (val, msg) {
     this.tap.true(val, `expect truthy value: ${msg} - value: ${JSON.stringify(val)}`)
     return this
-}
+  }
 
-/**
+  /**
    * Asserts that the given value is falsey
    * @param {any} the value to check
    * @param {string} an optional message
+   * @returns {TestRunner}
    */
-TestRunner.prototype.is_false = function (val, msg) {
+  is_false (val, msg) {
     this.tap.false(val, `expect falsey value: ${msg} - value: ${JSON.stringify(val)}`)
     return this
-}
+  }
 
-/**
+  /**
    * Asserts that two values are the same
    * @param {any} the first value
    * @param {any} the second value
    * @returns {TestRunner}
    */
-TestRunner.prototype.match = function (val1, val2, action) {
+  match (val1, val2, action) {
     // both values are objects
     if (typeof val1 === 'object' && typeof val2 === 'object') {
       this.tap.strictDeepEqual(val1, val2, action)
@@ -739,67 +679,67 @@ TestRunner.prototype.match = function (val1, val2, action) {
       this.tap.strictEqual(val1, val2, `should be equal: ${val1} - ${val2}, action: ${JSON.stringify(action)}`)
     }
     return this
-}
+  }
 
-/**
+  /**
    * Asserts that the first value is less than the second
    * It also verifies that the two values are numbers
    * @param {any} the first value
    * @param {any} the second value
    * @returns {TestRunner}
    */
-TestRunner.prototype.lt = function (val1, val2) {
+  lt (val1, val2) {
     ;[val1, val2] = getNumbers(val1, val2)
     this.tap.true(val1 < val2)
     return this
-}
+  }
 
-/**
+  /**
    * Asserts that the first value is greater than the second
    * It also verifies that the two values are numbers
    * @param {any} the first value
    * @param {any} the second value
    * @returns {TestRunner}
    */
-TestRunner.prototype.gt = function (val1, val2) {
+  gt (val1, val2) {
     ;[val1, val2] = getNumbers(val1, val2)
     this.tap.true(val1 > val2)
     return this
-}
+  }
 
-/**
+  /**
    * Asserts that the first value is less than or equal the second
    * It also verifies that the two values are numbers
    * @param {any} the first value
    * @param {any} the second value
    * @returns {TestRunner}
    */
-TestRunner.prototype.lte = function (val1, val2) {
+  lte (val1, val2) {
     ;[val1, val2] = getNumbers(val1, val2)
     this.tap.true(val1 <= val2)
     return this
-}
+  }
 
-/**
+  /**
    * Asserts that the first value is greater than or equal the second
    * It also verifies that the two values are numbers
    * @param {any} the first value
    * @param {any} the second value
    * @returns {TestRunner}
-*/
+  */
-TestRunner.prototype.gte = function (val1, val2) {
+  gte (val1, val2) {
     ;[val1, val2] = getNumbers(val1, val2)
     this.tap.true(val1 >= val2)
     return this
-}
+  }
 
-/**
+  /**
    * Asserts that the given value has the specified length
    * @param {string|object|array} the object to check
    * @param {number} the expected length
    * @returns {TestRunner}
    */
-TestRunner.prototype.length = function (val, len) {
+  length (val, len) {
     if (typeof val === 'string' || Array.isArray(val)) {
       this.tap.strictEqual(val.length, len)
     } else if (typeof val === 'object' && val !== null) {
@@ -808,9 +748,9 @@ TestRunner.prototype.length = function (val, len) {
       this.tap.fail(`length: the given value is invalid: ${val}`)
     }
     return this
-}
+  }
 
-/**
+  /**
    * Gets a `do` action object and returns a structured object,
    * where the action is the key and the parameter is the value.
    * Eg:
@@ -835,7 +775,7 @@ TestRunner.prototype.length = function (val, len) {
    * @param {object}
    * @returns {object}
    */
-TestRunner.prototype.parseDo = function (action) {
+  parseDo (action) {
     return Object.keys(action).reduce((acc, val) => {
       switch (val) {
         case 'catch':
@@ -887,6 +827,7 @@ TestRunner.prototype.parseDo = function (action) {
 
       return newObj
     }
+  }
 }
 
 function parseDoError (err, spec) {