elasticsearch-js/test/integration/index.js

// Licensed to Elasticsearch B.V under one or more agreements.
// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
// See the LICENSE file in the project root for more information

'use strict'

const { readFileSync, accessSync, mkdirSync, readdirSync, statSync } = require('fs')
const { join, sep } = require('path')
const yaml = require('js-yaml')
const Git = require('simple-git')
const tap = require('tap')
const { Client } = require('../../index')
const TestRunner = require('./test-runner')
const { sleep } = require('./helper')

const esRepo = 'https://github.com/elastic/elasticsearch.git'
const esFolder = join(__dirname, '..', '..', 'elasticsearch')
const yamlFolder = join(esFolder, 'rest-api-spec', 'src', 'main', 'resources', 'rest-api-spec', 'test')
const xPackYamlFolder = join(esFolder, 'x-pack', 'plugin', 'src', 'test', 'resources', 'rest-api-spec', 'test')

const ossSkips = {
  // TODO: remove this once 'arbitrary_key' is implemented
  // https://github.com/elastic/elasticsearch/pull/41492
  'indices.split/30_copy_settings.yml': ['*'],
  // skipping because we are booting ES with `discovery.type=single-node`
  // and this test will fail because of this configuration
  'nodes.stats/30_discovery.yml': ['*'],
  // the expected error is returning a 503,
  // which triggers a retry and the node to be marked as dead
  'search.aggregation/240_max_buckets.yml': ['*']
}
const xPackBlackList = {
  // file path: test name
  'cat.aliases/10_basic.yml': ['Empty cluster'],
  'index/10_with_id.yml': ['Index with ID'],
  'indices.get_alias/10_basic.yml': ['Get alias against closed indices'],
  'indices.get_alias/20_empty.yml': ['Check empty aliases when getting all aliases via /_alias'],
  // https://github.com/elastic/elasticsearch/pull/39400
  'ml/jobs_crud.yml': ['Test put job with id that is already taken'],
  // object keys must me strings, and `0.0.toString()` is `0`
  'ml/evaluate_data_frame.yml': [
    'Test binary_soft_classifition precision',
    'Test binary_soft_classifition recall',
    'Test binary_soft_classifition confusion_matrix'
  ],
  // it gets random failures on CI, must investigate
  'ml/set_upgrade_mode.yml': [
    'Attempt to open job when upgrade_mode is enabled',
    'Setting upgrade mode to disabled from enabled'
  ],
  // investigate why this is failing
  'monitoring/bulk/10_basic.yml': ['*'],
  'monitoring/bulk/20_privileges.yml': ['*'],
  'license/20_put_license.yml': ['*'],
  'snapshot/10_basic.yml': ['*'],
  // the body is correct, but the regex is failing
  'sql/sql.yml': ['Getting textual representation'],
  // we are setting two certificates in the docker config
  'ssl/10_basic.yml': ['*'],
  // the test passes, but the regex fails
  'transform/preview_transforms.yml': ['Test preview with non-existing source index'],
  'transforms_crud.yml': [
    'Test put transform with invalid source index',
    'Test alias scenarios',
    'Test invalid destination index name'
  ],
  'transforms_update.yml': [
    'Test put transform with invalid source index',
    'Test alias scenarios',
    'Test invalid destination index name'
  ],
  'ml/data_frame_analytics_crud.yml': [
    'Test put config given dest index contains uppercase chars',
    'Test put config with missing concrete source index',
    'Test put config with dest index same as source index',
    'Test put config with dest index matching multiple indices',
    'Test put config with dest index included in source via alias',
    'Test put config with remote source index'
  ],
  // docker issue?
  'watcher/execute_watch/60_http_input.yml': ['*'],
  // the checks are correct, but for some reason the test is failing on js side
  // I bet is because the backslashes in the rg
  'watcher/execute_watch/70_invalid.yml': ['*'],
  'watcher/put_watch/10_basic.yml': ['*'],
  'xpack/15_basic.yml': ['*']
}

class Runner {
  constructor (opts = {}) {
    const options = { node: opts.node }
    if (opts.isXPack) {
      options.ssl = {
        ca: readFileSync(join(__dirname, '..', '..', '.ci', 'certs', 'ca.crt'), 'utf8'),
        rejectUnauthorized: false
      }
    }
    this.client = new Client(options)
    console.log('Loading yaml suite')
  }

  async waitCluster (client, times = 0) {
    try {
      await client.cluster.health({ waitForStatus: 'green', timeout: '50s' })
    } catch (err) {
      if (++times < 10) {
        await sleep(5000)
        return this.waitCluster(client, times)
      }
      console.error(err)
      process.exit(1)
    }
  }

  async start ({ isXPack }) {
    const { client } = this
    const parse = this.parse.bind(this)

    console.log('Waiting for Elasticsearch')
    await this.waitCluster(client)

    const { body } = await client.info()
    const { number: version, build_hash: sha } = body.version

    console.log(`Checking out sha ${sha}...`)
    await this.withSHA(sha)

    console.log(`Testing ${isXPack ? 'XPack' : 'oss'} api...`)

    const folders = []
      .concat(getAllFiles(yamlFolder))
      .concat(isXPack ? getAllFiles(xPackYamlFolder) : [])
      .filter(t => !/(README|TODO)/g.test(t))
      // we cluster the array based on the folder names,
      // to provide a better test log output
      .reduce((arr, file) => {
        const path = file.slice(file.indexOf('/rest-api-spec/test'), file.lastIndexOf('/'))
        var inserted = false
        for (var i = 0; i < arr.length; i++) {
          if (arr[i][0].includes(path)) {
            inserted = true
            arr[i].push(file)
            break
          }
        }
        if (!inserted) arr.push([file])
        return arr
      }, [])

    for (const folder of folders) {
      // pretty name
      const apiName = folder[0].slice(
        folder[0].indexOf(`${sep}rest-api-spec${sep}test`) + 19,
        folder[0].lastIndexOf(sep)
      )

      tap.test(`Testing ${apiName}`, { bail: true, timeout: 0 }, t => {
        for (const file of folder) {
          const data = readFileSync(file, 'utf8')
          // get the test yaml (as object), some file has multiple yaml documents inside,
          // every document is separated by '---', so we split on the separator
          // and then we remove the empty strings, finally we parse them
          const tests = data
            .split('\n---\n')
            .map(s => s.trim())
            .filter(Boolean)
            .map(parse)

          t.test(
            file.slice(file.lastIndexOf(apiName)),
            testFile(file, tests)
          )
        }
        t.end()
      })
    }

    function testFile (file, tests) {
      return t => {
        // get setup and teardown if present
        var setupTest = null
        var teardownTest = null
        for (const test of tests) {
          if (test.setup) setupTest = test.setup
          if (test.teardown) teardownTest = test.teardown
        }

        tests.forEach(test => {
          const name = Object.keys(test)[0]
          if (name === 'setup' || name === 'teardown') return
          if (shouldSkip(t, isXPack, file, name)) return

          // create a subtest for the specific folder + test file + test name
          t.test(name, async t => {
            const testRunner = new TestRunner({
              client,
              version,
              tap: t,
              isXPack: file.includes('x-pack')
            })
            await testRunner.run(setupTest, test[name], teardownTest)
          })
        })
        t.end()
      }
    }
  }

  parse (data) {
    try {
      var doc = yaml.safeLoad(data)
    } catch (err) {
      console.error(err)
      return
    }
    return doc
  }

  getTest (folder) {
    const tests = readdirSync(folder)
    return tests.filter(t => !/(README|TODO)/g.test(t))
  }

  /**
   * Sets the elasticsearch repository to the given sha.
   * If the repository is not present in `esFolder` it will
   * clone the repository and the checkout the sha.
   * If the repository is already present but it cannot checkout to
   * the given sha, it will perform a pull and then try again.
   * @param {string} sha
   * @param {function} callback
   */
  withSHA (sha) {
    return new Promise((resolve, reject) => {
      _withSHA.call(this, err => err ? reject(err) : resolve())
    })

    function _withSHA (callback) {
      var fresh = false
      var retry = 0

      if (!this.pathExist(esFolder)) {
        if (!this.createFolder(esFolder)) {
          return callback(new Error('Failed folder creation'))
        }
        fresh = true
      }

      const git = Git(esFolder)

      if (fresh) {
        clone(checkout)
      } else {
        checkout()
      }

      function checkout () {
        console.log(`Checking out sha '${sha}'`)
        git.checkout(sha, err => {
          if (err) {
            if (retry++ > 0) {
              return callback(err)
            }
            return pull(checkout)
          }
          callback()
        })
      }

      function pull (cb) {
        console.log('Pulling elasticsearch repository...')
        git.pull(err => {
          if (err) {
            return callback(err)
          }
          cb()
        })
      }

      function clone (cb) {
        console.log('Cloning elasticsearch repository...')
        git.clone(esRepo, esFolder, err => {
          if (err) {
            return callback(err)
          }
          cb()
        })
      }
    }
  }

  /**
   * Checks if the given path exists
   * @param {string} path
   * @returns {boolean} true if exists, false if not
   */
  pathExist (path) {
    try {
      accessSync(path)
      return true
    } catch (err) {
      return false
    }
  }

  /**
   * Creates the given folder
   * @param {string} name
   * @returns {boolean} true on success, false on failure
   */
  createFolder (name) {
    try {
      mkdirSync(name)
      return true
    } catch (err) {
      return false
    }
  }
}

if (require.main === module) {
  const node = process.env.TEST_ES_SERVER || 'http://localhost:9200'
  const opts = {
    node,
    isXPack: node.indexOf('@') > -1
  }
  const runner = new Runner(opts)
  runner.start(opts).catch(console.log)
}

const shouldSkip = (t, isXPack, file, name) => {
  var list = Object.keys(ossSkips)
  for (var i = 0; i < list.length; i++) {
    const ossTest = ossSkips[list[i]]
    for (var j = 0; j < ossTest.length; j++) {
      if (file.endsWith(list[i]) && (name === ossTest[j] || ossTest[j] === '*')) {
        const testName = file.slice(file.indexOf(`${sep}elasticsearch${sep}`)) + ' / ' + name
        t.comment(`Skipping test ${testName} because is blacklisted in the oss test`)
        return true
      }
    }
  }

  if (file.includes('x-pack') || isXPack) {
    list = Object.keys(xPackBlackList)
    for (i = 0; i < list.length; i++) {
      const platTest = xPackBlackList[list[i]]
      for (j = 0; j < platTest.length; j++) {
        if (file.endsWith(list[i]) && (name === platTest[j] || platTest[j] === '*')) {
          const testName = file.slice(file.indexOf(`${sep}elasticsearch${sep}`)) + ' / ' + name
          t.comment(`Skipping test ${testName} because is blacklisted in the XPack test`)
          return true
        }
      }
    }
  }

  return false
}

const getAllFiles = dir =>
  readdirSync(dir).reduce((files, file) => {
    const name = join(dir, file)
    const isDirectory = statSync(name).isDirectory()
    return isDirectory ? [...files, ...getAllFiles(name)] : [...files, name]
  }, [])

module.exports = Runner