Added API generation scripts

package.json

@@ -18,6 +18,7 @@
     "test:integration": "tap test/integration/index.js -T --harmony",
     "lint": "standard",
     "lint:fix": "standard --fix",
+    "generate": "node scripts/run.js",
     "elasticsearch": "docker run --rm -e \"node.attr.testattr=test\" -e \"path.repo=/tmp\" -e \"repositories.url.allowed_urls=http://snapshot.*\" -p 9200:9200 docker.elastic.co/elasticsearch/elasticsearch:6.4.0"
   },
   "author": {
@@ -29,12 +30,14 @@
     "company": "Elasticsearch BV"
   },
   "devDependencies": {
+    "dedent": "^0.7.0",
     "js-yaml": "^3.12.0",
     "minimist": "^1.2.0",
     "ora": "^3.0.0",
     "readable-stream": "^3.0.1",
-    "semver": "^5.5.1",
+    "rimraf": "^2.6.2",
-    "simple-git": "^1.96.0",
+    "semver": "^5.6.0",
+    "simple-git": "^1.105.0",
     "sinon": "^6.1.5",
     "standard": "^12.0.0",
     "string-to-stream": "^1.1.1",

scripts/run.js

@@ -0,0 +1,72 @@
+'use strict'
+
+const { join } = require('path')
+const { readdirSync, writeFileSync } = require('fs')
+const minimist = require('minimist')
+const semver = require('semver')
+const ora = require('ora')
+const rimraf = require('rimraf')
+const standard = require('standard')
+const { generate, cloneAndCheckout, genFactory } = require('./utils')
+
+start(minimist(process.argv.slice(2), {
+  string: ['tag']
+}))
+
+function start (opts) {
+  const log = ora('Loading Elasticsearch Repository').start()
+  if (semver.valid(opts.tag) === null) {
+    log.fail(`Missing or invalid tag: ${opts.tag}`)
+    return
+  }
+  const packageFolder = join(__dirname, '..', 'api')
+  const apiOutputFolder = join(packageFolder, 'api')
+  const mainOutputFile = join(packageFolder, 'index.js')
+
+  log.text = 'Cleaning API folder...'
+  rimraf.sync(join(apiOutputFolder, '*.js'))
+
+  cloneAndCheckout({ log, tag: opts.tag }, (err, apiFolder) => {
+    if (err) {
+      log.fail(err.message)
+      return
+    }
+    const files = readdirSync(apiFolder)
+    files.forEach(generateApiFile(apiFolder, log))
+    writeFileSync(
+      mainOutputFile,
+      genFactory(apiOutputFolder),
+      { encoding: 'utf8' }
+    )
+    lintFiles(log)
+  })
+
+  function generateApiFile (apiFolder, log) {
+    var common = null
+    try {
+      common = require(join(apiFolder, '_common.json'))
+    } catch (e) {}
+
+    return function _generateApiFile (file) {
+      if (file === '_common.json') return
+      log.text = `Processing ${file}`
+
+      const spec = require(join(apiFolder, file))
+      const code = generate(spec, common)
+      const filePath = join(apiOutputFolder, `${file.slice(0, file.lastIndexOf('.'))}.js`)
+
+      writeFileSync(filePath, code, { encoding: 'utf8' })
+    }
+  }
+
+  function lintFiles (log) {
+    log.text = 'Linting...'
+    const files = [join(packageFolder, '*.js'), join(apiOutputFolder, '*.js')]
+    standard.lintFiles(files, { fix: true }, err => {
+      if (err) {
+        return log.fail(err.message)
+      }
+      log.succeed('Done!')
+    })
+  }
+}

scripts/utils/clone-es.js

@@ -0,0 +1,110 @@
+'use strict'
+
+const { accessSync, mkdirSync } = require('fs')
+const { join } = require('path')
+const Git = require('simple-git')
+
+const esRepo = 'https://github.com/elastic/elasticsearch.git'
+const esFolder = join(__dirname, '..', '..', 'elasticsearch')
+const apiFolder = join(esFolder, 'rest-api-spec', 'src', 'main', 'resources', 'rest-api-spec', 'api')
+
+function cloneAndCheckout (opts, callback) {
+  const { log, tag } = opts
+  withTag(tag, callback)
+
+  /**
+   * Sets the elasticsearch repository to the given tag.
+   * If the repository is not present in `esFolder` it will
+   * clone the repository and the checkout the tag.
+   * If the repository is already present but it cannot checkout to
+   * the given tag, it will perform a pull and then try again.
+   * @param {string} tag
+   * @param {function} callback
+   */
+  function withTag (tag, callback) {
+    var fresh = false
+    var retry = 0
+
+    if (!pathExist(esFolder)) {
+      if (!createFolder(esFolder)) {
+        log.fail('Failed folder creation')
+        return
+      }
+      fresh = true
+    }
+
+    const git = Git(esFolder)
+
+    if (fresh) {
+      clone(checkout)
+    } else {
+      checkout()
+    }
+
+    function checkout () {
+      log.text = `Checking out tag '${tag}'`
+      git.checkout(tag, err => {
+        if (err) {
+          if (retry++ > 0) {
+            callback(new Error(`Cannot checkout tag '${tag}'`))
+            return
+          }
+          return pull(checkout)
+        }
+        callback(null, apiFolder)
+      })
+    }
+
+    function pull (cb) {
+      log.text = 'Pulling elasticsearch repository...'
+      git.pull(err => {
+        if (err) {
+          callback(err)
+          return
+        }
+        cb()
+      })
+    }
+
+    function clone (cb) {
+      log.text = 'Cloning elasticsearch repository...'
+      git.clone(esRepo, esFolder, err => {
+        if (err) {
+          callback(err)
+          return
+        }
+        cb()
+      })
+    }
+  }
+
+  /**
+   * Checks if the given path exists
+   * @param {string} path
+   * @returns {boolean} true if exists, false if not
+   */
+  function pathExist (path) {
+    try {
+      accessSync(path)
+      return true
+    } catch (err) {
+      return false
+    }
+  }
+
+  /**
+   * Creates the given folder
+   * @param {string} name
+   * @returns {boolean} true on success, false on failure
+   */
+  function createFolder (name) {
+    try {
+      mkdirSync(name)
+      return true
+    } catch (err) {
+      return false
+    }
+  }
+}
+
+module.exports = cloneAndCheckout

scripts/utils/genMain.js

@@ -0,0 +1,106 @@
+'use strict'
+
+const { readdirSync } = require('fs')
+const dedent = require('dedent')
+
+function genFactory (folder) {
+  const apiToCamel = {}
+  // get all the API files
+  const apis = readdirSync(folder)
+    .map(file => {
+      const chunks = file.split('.')
+      // if the api has not a namespace
+      if (chunks.length === 2) {
+        return { name: chunks[0], group: null, file }
+      } else {
+        const [group, name] = chunks
+        return { name, group, file }
+      }
+    })
+    .reduce((acc, obj) => {
+      const { group, name, file } = obj
+      // create a namespace if present
+      if (group) {
+        acc[group] = acc[group] || {}
+        acc[group][name] = `require('./api/${file}')(opts)`
+      } else {
+        acc[name] = `require('./api/${file}')(opts)`
+      }
+      // save the snake_cased APIs for later use
+      if (isSnakeCased(name)) {
+        apiToCamel[group || '__root'] = apiToCamel[group || '__root'] || []
+        apiToCamel[group || '__root'].push(name)
+      }
+      return acc
+    }, {})
+
+  // serialize the API object
+  const apisStr = JSON.stringify(apis, null, 2)
+    // split & join to fix the indentation
+    .split('\n')
+    .join('\n    ')
+    // remove useless quotes
+    .replace(/"/g, '')
+
+  const fn = dedent`
+  'use strict'
+
+  const assert = require('assert')
+
+  function ESAPI (opts) {
+    assert(opts.makeRequest, 'Missing makeRequest function')
+    assert(opts.ConfigurationError, 'Missing ConfigurationError class')
+
+    const apis = ${apisStr}
+
+    ${generateDefinedProperties(apiToCamel).join('\n\n    ')}
+
+    return apis
+  }
+
+  module.exports = ESAPI
+  `
+
+  // new line at the end of file
+  return fn + '\n'
+}
+
+// generates an array of Object.defineProperties
+// to allow the use of camelCase APIs
+// instead of snake_cased
+function generateDefinedProperties (apiToCamel) {
+  const arr = []
+  for (const api in apiToCamel) {
+    const obj = api === '__root'
+      ? 'apis'
+      : `apis.${api}`
+    const code = `
+    Object.defineProperties(${obj}, {
+      ${apiToCamel[api].map(createGetter).join(',\n      ')}
+    })
+    `.trim()
+    arr.push(code)
+  }
+
+  return arr
+
+  function createGetter (api) {
+    return `
+      ${camelify(api)}: {
+        get: function () { return this.${api} },
+        enumerable: true
+      }
+    `.trim()
+  }
+}
+
+// from snake_case to camelCase
+function camelify (str) {
+  return str.replace(/_([a-z])/g, k => k[1].toUpperCase())
+}
+
+function isSnakeCased (str) {
+  return !!~str.indexOf('_')
+}
+
+module.exports = genFactory

scripts/utils/generate.js

@@ -0,0 +1,466 @@
+'use strict'
+
+const dedent = require('dedent')
+const allowedMethods = {
+  noBody: ['GET', 'HEAD', 'DELETE'],
+  body: ['POST', 'PUT', 'DELETE']
+}
+
+// list of apis that does not need any kind of validation
+// because of how the url is built
+const noPathValidation = [
+  'indices.get_alias',
+  'indices.exists_alias',
+  'indices.get_field_mapping',
+  'indices.get_mapping',
+  'indices.get_settings',
+  'indices.put_mapping',
+  'indices.stats',
+  'nodes.info',
+  'nodes.stats',
+  'nodes.usage',
+  'tasks.cancel'
+]
+
+// apis that uses bulkBody property
+const ndjsonApi = [
+  'bulk',
+  'msearch',
+  'msearch_template'
+]
+
+function generate (spec, common) {
+  const api = Object.keys(spec)[0]
+  const name = api
+    .replace(/\.([a-z])/g, k => k[1].toUpperCase())
+    .replace(/_([a-z])/g, k => k[1].toUpperCase())
+
+  const methods = spec[api].methods
+  const { path, paths, parts, params } = spec[api].url
+  const acceptedQuerystring = []
+  const required = []
+
+  for (const key in parts) {
+    if (parts[key].required) {
+      required.push(key)
+    }
+  }
+
+  for (const key in params) {
+    if (params[key].required) {
+      required.push(key)
+    }
+    acceptedQuerystring.push(key)
+  }
+
+  for (const key in spec[api]) {
+    const k = spec[api][key]
+    if (k && k.required) {
+      required.push(key)
+    }
+  }
+  if (common && common.params) {
+    for (const key in common.params) {
+      acceptedQuerystring.push(key)
+    }
+  }
+
+  const code = `
+  function ${safeWords(name)} (params, callback) {
+    if (typeof params === 'function' || params == null) {
+      callback = params
+      params = {}
+    }
+    // promises support
+    if (callback == null) {
+      return new Promise((resolve, reject) => {
+        ${safeWords(name)}(params, (err, body) => {
+          err ? reject(err) : resolve(body)
+        })
+      })
+    }
+
+    ${genRequiredChecks()}
+
+    ${genUrlValidation(paths, api)}
+
+    // build querystring object
+    const querystring = {}
+    const keys = Object.keys(params)
+    const acceptedQuerystring = [
+      ${genAcceptedQuerystring()}
+    ]
+    const acceptedQuerystringCamelCased = [
+      ${genAcceptedQuerystringCamelCased()}
+    ]
+
+    for (var i = 0, len = keys.length; i < len; i++) {
+      var key = keys[i]
+      if (acceptedQuerystring.indexOf(key) !== -1) {
+        querystring[key] = params[key]
+      } else {
+        var camelIndex = acceptedQuerystringCamelCased.indexOf(key)
+        if (camelIndex !== -1) {
+          querystring[acceptedQuerystring[camelIndex]] = params[key]
+        }
+      }
+    }
+
+    // configure http method
+    var method = params.method
+    if (method == null) {
+      ${generatePickMethod(methods)}
+    }
+
+    // validate headers object
+    if (params.headers != null && typeof params.headers !== 'object') {
+      return callback(
+        new ConfigurationError(\`Headers should be an object, instead got: \${typeof params.headers}\`),
+        { body: null, headers: null, statusCode: null }
+      )
+    }
+
+    var ignore = params.ignore || null
+    if (typeof ignore === 'number') {
+      ignore = [ignore]
+    }
+
+    // build request object
+    const parts = ${getUrlParts()}
+    const request = {
+      method,
+      ${buildPath(api)}
+      querystring,
+      ${genBody(api, methods, spec[api].body)}
+      headers: params.headers || null,
+      ignore,
+      requestTimeout: params.requestTimeout || null,
+      agent: null,
+      url: ''
+    }
+
+    return makeRequest(request, callback)
+  }
+  `.trim() // always call trim to avoid newlines
+
+  const fn = dedent`
+  'use strict'
+
+  function build${name[0].toUpperCase() + name.slice(1)} (opts) {
+    // eslint-disable-next-line no-unused-vars
+    const { makeRequest, ConfigurationError } = opts
+    ${generateDocumentation(spec[api], api)}
+    return ${code}
+  }
+
+  module.exports = build${name[0].toUpperCase() + name.slice(1)}
+`
+
+  // new line at the end of file
+  return fn + '\n'
+
+  function genRequiredChecks (param) {
+    const code = required
+      .map(_genRequiredCheck)
+      .concat(_noBody())
+      .filter(Boolean)
+
+    if (code.length) {
+      code.unshift('// check required parameters')
+    }
+
+    return code.join('\n        ')
+
+    function _genRequiredCheck (param) {
+      var camelCased = param[0] === '_'
+        ? '_' + param.slice(1).replace(/_([a-z])/g, k => k[1].toUpperCase())
+        : param.replace(/_([a-z])/g, k => k[1].toUpperCase())
+
+      if (param === camelCased) {
+        const check = `
+          if (params['${param}'] == null) {
+            return callback(
+              new ConfigurationError('Missing required parameter: ${param}'),
+              { body: null, headers: null, statusCode: null }
+            )
+          }
+        `
+        return check.trim()
+      } else {
+        const check = `
+          if (params['${param}'] == null && params['${camelCased}'] == null) {
+            return callback(
+              new ConfigurationError('Missing required parameter: ${param} or ${camelCased}'),
+              { body: null, headers: null, statusCode: null }
+            )
+          }
+        `
+        return check.trim()
+      }
+    }
+
+    function _noBody () {
+      const check = `
+        if (params.body != null) {
+          return callback(
+            new ConfigurationError('This API does not require a body'),
+            { body: null, headers: null, statusCode: null }
+          )
+        }
+      `
+      return spec[api].body === null ? check.trim() : ''
+    }
+  }
+
+  function genAcceptedQuerystring () {
+    return acceptedQuerystring
+      .map(q => `'${q}'`)
+      .join(',\n          ')
+  }
+
+  function genAcceptedQuerystringCamelCased () {
+    return acceptedQuerystring
+      .map(q => {
+        // if the key starts with `_` we should not camelify the first occurence
+        // eg: _source_include => _sourceInclude
+        return q[0] === '_'
+          ? '_' + q.slice(1).replace(/_([a-z])/g, k => k[1].toUpperCase())
+          : q.replace(/_([a-z])/g, k => k[1].toUpperCase())
+      })
+      .map(q => `'${q}'`)
+      .join(',\n          ')
+  }
+
+  function buildPath () {
+    // if the default path is static, we should add a dynamic check
+    // to figure out which path to use, see cluster.stats
+    // otherwise we can skip that check
+    const p1 = paths
+      .reduce((a, b) => a.split('/').length > b.split('/').length ? a : b)
+      .split('/')
+      .filter(chunk => !chunk.startsWith('{'))
+      .join('/')
+
+    const p2 = path
+      .split('/')
+      .filter(chunk => !chunk.startsWith('{'))
+      .join('/')
+
+    if (p1 === p2 || !needsPathValidation(api)) {
+      return `path: '/' + parts.filter(Boolean).map(encodeURIComponent).join('/'),`.trim()
+    }
+
+    const dynamicParts = checkDynamicParts()
+    if (dynamicParts.length) {
+      return `
+        path: ${dynamicParts}
+          ? '/' + parts.filter(Boolean).join('/')
+          : '${path}',
+      `.trim()
+    } else {
+      return `path: '/' + parts.filter(Boolean).join('/'),`.trim()
+    }
+  }
+
+  function checkDynamicParts () {
+    const chunks = paths
+      .reduce((a, b) => a.split('/').length > b.split('/').length ? a : b)
+      .split('/')
+      .filter(Boolean)
+
+    var str = ''
+    chunks.forEach((chunk, index) => {
+      if (chunk.startsWith('{') && chunk.endsWith('}')) {
+        chunk = chunk.slice(1, -1)
+        // url parts can be declared in camelCase fashion
+        var camelCased = chunk[0] === '_'
+          ? '_' + chunk.slice(1).replace(/_([a-z])/g, k => k[1].toUpperCase())
+          : chunk.replace(/_([a-z])/g, k => k[1].toUpperCase())
+
+        if (chunk === camelCased) {
+          str += `params['${chunk}'] != null && `
+        } else {
+          str += `(params['${chunk}'] || params['${camelCased}']) != null && `
+        }
+      }
+    })
+
+    // removes last ' && '
+    return str.slice(0, -4)
+  }
+
+  function getUrlParts () {
+    const chunks = paths
+      .reduce((a, b) => a.split('/').length > b.split('/').length ? a : b)
+      .split('/')
+      .filter(Boolean)
+    var str = '['
+    chunks.forEach((chunk, index) => {
+      if (chunk.startsWith('{') && chunk.endsWith('}')) {
+        chunk = chunk.slice(1, -1)
+        // url parts can be declared in camelCase fashion
+        var camelCased = chunk[0] === '_'
+          ? '_' + chunk.slice(1).replace(/_([a-z])/g, k => k[1].toUpperCase())
+          : chunk.replace(/_([a-z])/g, k => k[1].toUpperCase())
+
+        if (chunk === camelCased) {
+          str += `params['${chunk}']`
+        } else {
+          str += `params['${chunk}'] || params['${camelCased}']`
+        }
+      } else {
+        str += `'${chunk}'`
+      }
+      if (index !== chunks.length - 1) {
+        str += ', '
+      }
+    })
+    str += ']'
+    return str
+  }
+}
+
+function safeWords (str) {
+  if (str === 'delete') {
+    return '_delete'
+  }
+  return str
+}
+
+function generatePickMethod (methods) {
+  if (methods.length === 1) {
+    return `method = '${methods[0]}'`
+  }
+  const bodyMethod = getBodyMethod(methods)
+  const noBodyMethod = getNoBodyMethod(methods)
+  if (bodyMethod && noBodyMethod) {
+    return `method = params.body == null ? '${noBodyMethod}' : '${bodyMethod}'`
+  } else if (bodyMethod) {
+    return `
+        method = '${bodyMethod}'
+    `.trim()
+  } else {
+    return `
+        method = '${noBodyMethod}'
+    `.trim()
+  }
+}
+
+function genBody (api, methods, body) {
+  const bodyMethod = getBodyMethod(methods)
+  if (ndjsonApi.indexOf(api) > -1) {
+    return 'bulkBody: params.body,'
+  }
+  if (body === null && bodyMethod) {
+    return `body: '',`
+  } else if (bodyMethod) {
+    return `body: params.body || '',`
+  } else {
+    return 'body: null,'
+  }
+}
+
+function getBodyMethod (methods) {
+  const m = methods.filter(m => ~allowedMethods.body.indexOf(m))
+  if (m.length) return m[0]
+  return null
+}
+
+function getNoBodyMethod (methods) {
+  const m = methods.filter(m => ~allowedMethods.noBody.indexOf(m))
+  if (m.length) return m[0]
+  return null
+}
+
+function genUrlValidation (paths, api) {
+  // this api does not need url validation
+  if (!needsPathValidation(api)) return ''
+  // gets only the dynamic components of the url in an array
+  // then we reverse it. A parameters always require what is
+  // at its right in the array.
+  const chunks = paths
+    .reduce((a, b) => a.split('/').length > b.split('/').length ? a : b)
+    .split('/')
+    .filter(s => s.startsWith('{'))
+    .map(s => s.slice(1, -1))
+    .reverse()
+
+  var code = ''
+
+  const len = chunks.length
+  chunks.forEach((chunk, index) => {
+    if (index === len - 1) return
+    var params = []
+    var camelCased = chunk[0] === '_'
+      ? '_' + chunk.slice(1).replace(/_([a-z])/g, k => k[1].toUpperCase())
+      : chunk.replace(/_([a-z])/g, k => k[1].toUpperCase())
+
+    if (chunk === camelCased) {
+      code += `${index ? '} else ' : ''}if (params['${chunk}'] != null && (`
+    } else {
+      code += `${index ? '} else ' : ''}if ((params['${chunk}'] != null || params['${camelCased}'] != null) && (`
+    }
+    for (var i = index + 1; i < len; i++) {
+      params.push(chunks[i])
+      // url parts can be declared in camelCase fashion
+      camelCased = chunks[i][0] === '_'
+        ? '_' + chunks[i].slice(1).replace(/_([a-z])/g, k => k[1].toUpperCase())
+        : chunks[i].replace(/_([a-z])/g, k => k[1].toUpperCase())
+
+      if (chunks[i] === camelCased) {
+        code += `params['${chunks[i]}'] == null${i === len - 1 ? '' : ' || '}`
+      } else {
+        code += `(params['${chunks[i]}'] == null || params['${camelCased}')${i === len - 1 ? '' : ' || '}`
+      }
+    }
+    code += `)) {
+      return callback(
+        new ConfigurationError('Missing required parameter of the url: ${params.join(', ')}'),
+        { body: null, headers: null, statusCode: null }
+      )`
+  })
+
+  if (chunks.length > 1) {
+    code += '\n}'
+  }
+
+  if (code.length) {
+    code = '// check required url components\n' + code
+  }
+
+  return code.trim()
+}
+
+function generateDocumentation (api, op) {
+  const { parts = {}, params = {} } = api.url
+  const { body } = api
+
+  // we use `replace(/\u00A0/g, ' ')` to remove no breaking spaces
+  // because some parts of the description fields are using it
+
+  var doc = '/**\n'
+  doc += `     * Perform a [${op}](${api.documentation}) request\n     *\n`
+  Object.keys(parts).forEach(part => {
+    const obj = parts[part]
+    doc += `     * @param {${obj.type}} ${part} - ${obj.description.replace(/\u00A0/g, ' ')}\n`
+  })
+
+  Object.keys(params).forEach(param => {
+    const obj = params[param]
+    doc += `     * @param {${obj.type}} ${param} - ${obj.description.replace(/\u00A0/g, ' ')}\n`
+  })
+
+  if (body) {
+    doc += `     * @param {${body.type || 'object'}} body - ${body.description.replace(/\u00A0/g, ' ')}\n`
+  }
+
+  doc += '     */'
+
+  return doc
+}
+
+function needsPathValidation (api) {
+  return noPathValidation.indexOf(api) === -1
+}
+
+module.exports = generate

scripts/utils/index.js

@@ -0,0 +1,7 @@
+'use strict'
+
+const generate = require('./generate')
+const cloneAndCheckout = require('./clone-es')
+const genFactory = require('./genMain')
+
+module.exports = { generate, cloneAndCheckout, genFactory }