Major updates for testing and grunt, jenkins tests are now powered by the jenkins.sh script in the scripts directory.

scripts/export_all_clients.js

@@ -1,33 +0,0 @@
-var path = require('path');
-var fs = require('fs');
-
-var argv = require('optimist')
-  .default({
-    verbose: false
-  })
-  .alias({
-    v: 'verbose'
-  })
-  .argv;
-
-var steps = [];
-
-['browser', 'jquery', 'angular'].forEach(function (build) {
-
-  if (!fs.existsSync('../bower-elasticsearch-' + build) ||
-      !fs.existsSync('../bower-elasticsearch-' + build + '/.git')
-  ) {
-    throw new Error('Ensure that all of the bower repos are checked out next to this repo');
-  }
-
-  steps.push([
-    'run', {
-      cwd: '../bower-elasticsearch-' + build + '/',
-      cmd: 'npm',
-      args: ['run', 'generate']
-    }
-  ]);
-
-});
-
-require('./_steps')(argv, steps);

scripts/export_client.js

@@ -1,49 +0,0 @@
-var path = require('path');
-
-var argv = require('optimist')
-  .default({
-    buildName: '',
-    outputDir: '.',
-    verbose: false
-  })
-  .alias({
-    b: 'buildName',
-    o: 'outputDir',
-    v: 'verbose'
-  })
-  .argv;
-
-switch (argv.buildName) {
-case 'jquery':
-  var buildFile = './dist/elasticsearch.jquery.js';
-  var minifiedBuildFile = './dist/elasticsearch.jquery.min.js';
-  break;
-case 'angular':
-  var buildFile = './dist/elasticsearch.angular.js';
-  var minifiedBuildFile = './dist/elasticsearch.angular.min.js';
-  break;
-default:
-  var buildFile = './dist/elasticsearch.js';
-  var minifiedBuildFile = './dist/elasticsearch.min.js';
-  break;
-}
-
-var outputFile = path.join(argv.outputDir, 'elasticsearch.js');
-var minifiedOutputFile = path.join(argv.outputDir, 'elasticsearch.min.js');
-
-require('./_steps')(argv, [
-  ['run', {
-    cmd: 'npm',
-    args: ['run', 'build_clients']
-  }],
-  ['copy', {
-    from: buildFile,
-    to: outputFile
-  }],
-  ['copy', {
-    from: minifiedBuildFile,
-    to: minifiedOutputFile
-  }]
-]);
-
-

scripts/export_docs.js

@@ -14,7 +14,7 @@ var argv = require('optimist')
 require('./_steps')(argv, [
   ['runInModule', {
     cmd: 'node',
-    args: [path.join(__dirname, './generate/js_api'), '--force']
+    args: [path.join(__dirname, './generate'), '--force']
   }],
   ['copy', {
     from: path.join(__dirname, '../docs/_methods.jade'),

scripts/generate/_force.js

@@ -1,24 +0,0 @@
-var force = process.env.FORCE || process.env.FORCE_GEN;
-
-if (!force) {
-  var argv = require('optimist')
-    .options({
-      force: {
-        alias: 'f',
-        default: false,
-        boolean: true
-      }
-    });
-
-  if (process.env.npm_config_argv) {
-    // when called by NPM
-    argv = argv.parse(JSON.parse(process.env.npm_config_argv).original);
-  } else {
-    // when called directly
-    argv = argv.argv;
-  }
-
-  force = argv.force;
-}
-
-module.exports = force;

scripts/generate/index.js

@@ -0,0 +1,71 @@
+var cp = require('child_process');
+var async = require('async');
+var argv = require('optimist')
+  .options({
+    force: {
+      alias: 'f',
+      default: false,
+      boolean: true
+    },
+    verbose: {
+      alias: 'v',
+      default: false,
+      boolean: true
+    },
+    api: {
+      default: true,
+      boolean: true
+    },
+    tests: {
+      default: true,
+      boolean: true
+    }
+  });
+
+if (process.env.npm_config_argv) {
+  // when called by NPM
+  argv = argv.parse(JSON.parse(process.env.npm_config_argv).original);
+} else {
+  // when called directly
+  argv = argv.argv;
+}
+
+if (!argv.force && process.env.FORCE || process.env.FORCE_GEN) {
+  argv.force = argv.f = process.env.FORCE || process.env.FORCE_GEN;
+}
+
+function updateSubmodules(done) {
+  cp.exec('git submodule update --init --recursive', function (err, stdout, stderr) {
+    stdout = stdout.trim();
+    stderr = stderr.trim();
+
+    if (err) {
+      done(new Error('Unable to update submodules: ' + err.message));
+      return;
+    } else if (argv.verbose && stdout) {
+      console.log(stdout);
+    }
+
+    if (stderr) {
+      console.error(stderr);
+    }
+    done();
+  });
+}
+
+updateSubmodules(function (err) {
+  if (err) {
+    throw err;
+  }
+
+  var tasks = [];
+
+  if (argv.api) {
+    tasks.push(require('./js_api'));
+  }
+  if (argv.tests) {
+    tasks.push(require('./yaml_tests'));
+  }
+
+  async.parallel(tasks, function () {});
+});

scripts/generate/js_api.js

@@ -0,0 +1,344 @@
+var aliases; // defined at the bottom of this file.
+
+module.exports = function (done) {
+  /**
+   * Read the API actions form the rest-api-spec repo.
+   * @type {[type]}
+   */
+  var _ = require('../../src/lib/utils');
+  var fs = require('relative-fs').relativeTo(__dirname);
+  var async = require('async');
+  var templates = require('./templates');
+  var castExistsRE = /exists/;
+  var usesBulkBodyRE = /^(bulk|msearch)$/;
+  var urlParamRE = /\{(\w+)\}/g;
+
+  var files; // populated in readSpecFiles
+  var apiSpec; // populated by parseSpecFiles
+
+  // generate the API
+  async.series([
+    readSpecFiles,
+    parseSpecFiles,
+    writeApiFile,
+    ensureDocsDir,
+    writeMethodList,
+    writeMethodDocs
+  ], done);
+
+  function readSpecFiles(done) {
+    var apiDir = '../../src/rest-api-spec/api/';
+    files = fs.readdirSync(apiDir).map(function (filename) {
+      return require(apiDir + filename);
+    });
+    done();
+  }
+
+  function parseSpecFiles(done) {
+    var actions = [];
+
+    files.forEach(function (spec) {
+      __puke__transformSpec(spec).forEach(function (action) {
+        actions.push(action);
+      });
+    });
+
+    // collect the namespaces from the action locations
+    var namespaces = _.filter(_.map(actions, function (action) {
+      if (~action.location.indexOf('.')) {
+        var path = action.location.split('.').slice(0, -1);
+        _.pull(path, 'prototype');
+        return path.join('.');
+      }
+    }));
+
+    // seperate the proxy actions
+    var groups = _.groupBy(actions, function (action) {
+      return action.proxy ? 'proxies' : 'normal';
+    });
+
+    apiSpec = {
+      actions: groups.normal,
+      proxies: groups.proxies,
+      namespaces: _.unique(namespaces.sort(), true)
+    };
+
+    done();
+  }
+
+  function writeApiFile(done) {
+    var outputPath = require('path').join(__dirname, '../../src/lib/api.js');
+    console.log('writing', apiSpec.actions.length, 'api actions to', outputPath);
+    fs.writeFile(outputPath, templates.apiFile(apiSpec), done);
+  }
+
+  function ensureDocsDir(done) {
+    fs.stat('../../docs', function (err, stat) {
+      if (err) {
+        if (err.message.match(/enoent/i)) {
+          fs.mkdir('../../docs', done);
+        } else {
+          done(err);
+        }
+      } else if (stat.isDirectory()) {
+        done();
+      } else {
+        done(new Error('../../docs exists, but it is not a directory'));
+      }
+    });
+  }
+
+  function writeMethodList(done) {
+    fs.writeFile(
+      '../../docs/_method_list.jade',
+      templates.apiMethodList(apiSpec),
+      done
+    );
+  }
+
+  function writeMethodDocs(done) {
+    fs.writeFile(
+      '../../docs/_methods.jade',
+      templates.apiMethods(apiSpec),
+      done
+    );
+  }
+
+  function __puke__transformSpec(spec) {
+    var actions = [];
+
+    // itterate all of the specs within the file, should only be one
+    _.each(spec, function (def, name) {
+      //camelcase the name
+      name = _.map(name.split('.'), _.camelCase).join('.');
+
+      var steps = name.split('.');
+
+      function transformParamKeys(note, param, key) {
+        var cmlKey = _.camelCase(key);
+        if (cmlKey !== key) {
+          param.name = key;
+        }
+        note[cmlKey] = param;
+      }
+
+      def.url.params = _.transform(def.url.params, transformParamKeys, {});
+      def.url.parts = _.transform(def.url.parts, transformParamKeys, {});
+
+      var allParams = _.extend({}, def.url.params, def.url.parts);
+      var spec = {
+        name: name,
+        methods: _.map(def.methods, function (m) { return m.toUpperCase(); }),
+        params: def.url.params,
+        body: def.body || null,
+        path2lib: _.repeat('../', steps.length + 1) + 'lib/'
+      };
+
+      if (def.body && def.body.requires) {
+        spec.needBody = true;
+      }
+
+      if (usesBulkBodyRE.test(name)) {
+        spec.bulkBody = true;
+      }
+
+      if (castExistsRE.test(name)) {
+        spec.castExists = true;
+      }
+
+      var urls = _.difference(def.url.paths, aliases[name]);
+      urls = _.map(urls, function (url) {
+        var optionalVars = {};
+        var requiredVars = {};
+        var param;
+        var name;
+        var target;
+        var match;
+
+        if (url.charAt(0) !== '/') {
+          url = '/' + url;
+        }
+
+        while (match = urlParamRE.exec(url)) {
+          name = _.camelCase(match[1]);
+          param = def.url.parts[name] || {};
+          target = (param.required || !param.default) ? requiredVars : optionalVars;
+          target[name] = _.omit(param, 'required', 'description', 'name');
+        }
+
+        return _.omit({
+          fmt: url.replace(urlParamRE, function (full, match) {
+            return '<%=' + _.camelCase(match) + '%>';
+          }),
+          opt: _.size(optionalVars) ? optionalVars : null,
+          req: _.size(requiredVars) ? requiredVars : null,
+          sortOrder: _.size(requiredVars) * -1
+        }, function (v) {
+          return !v;
+        });
+      });
+
+      if (urls.length > 1) {
+        spec.urls = _.map(_.sortBy(urls, 'sortOrder'), function (url) {
+          return _.omit(url, 'sortOrder');
+        });
+      } else {
+        spec.url = urls[0];
+      }
+
+      spec.params = _.transform(spec.params, function (note, param, name) {
+        // param.name = name;
+        note[name] = _.pick(param, [
+          'type', 'default', 'options', 'required', 'name'
+        ]);
+      }, {});
+
+      if (_.size(spec.params) === 0) {
+        delete spec.params;
+      }
+
+      // escape method names with "special" keywords
+      var location = spec.name.split('.').join('.prototype.')
+        .replace(/(^|\.)(delete|default)(\.|$)/g, '[\'$2\']');
+
+      var action = {
+        _methods: spec.methods,
+        spec: _.pick(spec, [
+          'params',
+          'url',
+          'urls',
+          'needBody',
+          'bulkBody',
+          'castExists',
+          'castNotFound'
+        ]),
+        location: location,
+        docUrl: def.documentation,
+        name: spec.name,
+        allParams: allParams
+      };
+
+      function hasMethod(/* ...methods */) {
+        for (var i = 0; i < arguments.length; i++) {
+          if (~action._methods.indexOf(arguments[i])) {
+            continue;
+          } else {
+            return false;
+          }
+        }
+        return true;
+      }
+      function methodsAre(/* ...methods */) {
+        return hasMethod.apply(null, arguments) && arguments.length === action._methods.length;
+      }
+
+      var method;
+
+      if (action._methods.length === 1) {
+        method = action._methods[0];
+      } else {
+        // we need to define what the default method(s) will be
+        if (hasMethod('DELETE', 'POST')) {
+          method = 'POST';
+        }
+        else if (methodsAre('DELETE')) {
+          method = 'DELETE';
+        }
+        else if (methodsAre('POST', 'PUT')) {
+          method = action.name.match(/put/i) ? 'PUT' : 'POST';
+        }
+        else if (methodsAre('GET', 'POST')) {
+          method = 'POST';
+        }
+        else if (methodsAre('GET', 'HEAD')) {
+          if (action.spec.castExists) {
+            method = 'HEAD';
+          } else {
+            method = 'GET';
+          }
+        }
+      }
+
+      if (method) {
+        if (method !== 'GET') {
+          action.spec.method = method;
+        }
+      } else {
+        throw new Error('unable to pick a method for ' + JSON.stringify(action, null, '  '));
+      }
+
+      if (action.name === 'create') {
+        action.proxy = 'index';
+        action.transformBody = 'params.op_type = \'create\';';
+      }
+
+      actions.push(action);
+    });
+
+    return actions;
+  }
+};
+
+aliases = {
+  'cluster.nodeHotThreads': [
+    '/_cluster/nodes/hotthreads',
+    '/_cluster/nodes/hot_threads',
+    '/_nodes/hot_threads',
+    '/_cluster/nodes/{node_id}/hotthreads',
+    '/_cluster/nodes/{node_id}/hot_threads',
+    '/_nodes/{node_id}/hot_threads'
+  ],
+  'cluster.nodeInfo': [
+    '/_cluster/nodes',
+    '/_nodes/settings',
+    '/_nodes/os',
+    '/_nodes/process',
+    '/_nodes/jvm',
+    '/_nodes/thread_pool',
+    '/_nodes/network',
+    '/_nodes/transport',
+    '/_nodes/http',
+    '/_nodes/plugin',
+    '/_cluster/nodes/{node_id}',
+    '/_nodes/{node_id}/settings',
+    '/_nodes/{node_id}/os',
+    '/_nodes/{node_id}/process',
+    '/_nodes/{node_id}/jvm',
+    '/_nodes/{node_id}/thread_pool',
+    '/_nodes/{node_id}/network',
+    '/_nodes/{node_id}/transport',
+    '/_nodes/{node_id}/http',
+    '/_nodes/{node_id}/plugin'
+  ],
+  'cluster.nodeShutdown': [
+    '/_cluster/nodes/_shutdown'
+  ],
+  'cluster.nodeStats': [
+    '/_cluster/nodes/stats',
+    '/_nodes/stats/{metric_family}',
+    '/_nodes/stats/indices/{metric}/{fields}',
+    '/_cluster/nodes/{node_id}/stats',
+    '/_nodes/{node_id}/stats/{metric_family}',
+    '/_nodes/{node_id}/stats/indices/{metric}/{fields}'
+  ],
+  'get': [
+    '/{index}/{type}/{id}/_source'
+  ],
+  'indices.deleteMapping': [
+    '/{index}/{type}/_mapping'
+  ],
+  'indices.stats': [
+    '_stats/{metric_family}',
+    '/_stats/indexing',
+    '/_stats/indexing/{indexing_types}',
+    '/_stats/search/{search_groups}',
+    '/_stats/fielddata/{fields}',
+    '/{index}/_stats/{metric_family}',
+    '/{index}/_stats/indexing',
+    '/{index}/_stats/search/{search_groups}',
+    '/{index}/_stats/fielddata/{fields}'
+  ],
+  'search': [
+    '/_search'
+  ]
+};

scripts/generate/js_api/README.md

@@ -1,3 +0,0 @@
-Generate the API using the es [REST api spec](https://github.com/elasticsearch/elasticsearch-rest-api-spec).
-
-run by calling `npm run generate`. Force it to update, even if their has not been a new commit, but calling with `--force` or `-f`

scripts/generate/js_api/actions.js

@@ -1,240 +0,0 @@
-var _ = require('../../../src/lib/utils');
-
-var EventEmitter = require('events').EventEmitter;
-var aliases = require('./aliases');
-
-var castExistsRE = /exists/;
-var usesBulkBodyRE = /^(bulk|msearch)$/;
-var urlParamRE = /\{(\w+)\}/g;
-
-var specCount = 0;
-var actions = [];
-var doneParsing = false;
-
-require('../../get_spec')
-  .get('api/*.json')
-  .on('entry', transformFile)
-  .on('end', function () {
-    doneParsing = true;
-    if (actions.length === specCount) {
-      module.exports.emit('ready', actions);
-    }
-  });
-
-function transformFile(entry) {
-  specCount++;
-
-  // itterate all of the specs within the file, should only be one
-  _.each(JSON.parse(entry.data), function (def, name) {
-    //camelcase the name
-    name = _.map(name.split('.'), _.camelCase).join('.');
-
-    var steps = name.split('.');
-
-    function transformParamKeys(note, param, key) {
-      var cmlKey = _.camelCase(key);
-      if (cmlKey !== key) {
-        param.name = key;
-      }
-      note[cmlKey] = param;
-    }
-
-    def.url.params = _.transform(def.url.params, transformParamKeys, {});
-    def.url.parts = _.transform(def.url.parts, transformParamKeys, {});
-
-    var allParams = _.extend({}, def.url.params, def.url.parts);
-    var spec = {
-      name: name,
-      methods: _.map(def.methods, function (m) { return m.toUpperCase(); }),
-      params: def.url.params,
-      body: def.body || null,
-      path2lib: _.repeat('../', steps.length + 1) + 'lib/'
-    };
-
-    if (def.body && def.body.requires) {
-      spec.needBody = true;
-    }
-
-    if (usesBulkBodyRE.test(name)) {
-      spec.bulkBody = true;
-    }
-
-    if (castExistsRE.test(name)) {
-      spec.castExists = true;
-    }
-
-    var urls = _.difference(def.url.paths, aliases[name]);
-    urls = _.map(urls, function (url) {
-      var optionalVars = {};
-      var requiredVars = {};
-      var param;
-      var name;
-      var target;
-      var match;
-
-      if (url.charAt(0) !== '/') {
-        url = '/' + url;
-      }
-
-      while (match = urlParamRE.exec(url)) {
-        name = _.camelCase(match[1]);
-        param = def.url.parts[name] || {};
-        target = (param.required || !param.default) ? requiredVars : optionalVars;
-        target[name] = _.omit(param, 'required', 'description', 'name');
-      }
-
-      return _.omit({
-        fmt: url.replace(urlParamRE, function (full, match) {
-          return '<%=' + _.camelCase(match) + '%>';
-        }),
-        opt: _.size(optionalVars) ? optionalVars : null,
-        req: _.size(requiredVars) ? requiredVars : null,
-        sortOrder: _.size(requiredVars) * -1
-      }, function (v) {
-        return !v;
-      });
-    });
-
-    if (urls.length > 1) {
-      spec.urls = _.map(_.sortBy(urls, 'sortOrder'), function (url) {
-        return _.omit(url, 'sortOrder');
-      });
-    } else {
-      spec.url = urls[0];
-    }
-
-    spec.params = _.transform(spec.params, function (note, param, name) {
-      // param.name = name;
-      note[name] = _.pick(param, [
-        'type', 'default', 'options', 'required', 'name'
-      ]);
-    }, {});
-
-    if (_.size(spec.params) === 0) {
-      delete spec.params;
-    }
-
-    // escape method names with "special" keywords
-    var location = spec.name.split('.').join('.prototype.')
-      .replace(/(^|\.)(delete|default)(\.|$)/g, '[\'$2\']');
-
-    var action = {
-      _methods: spec.methods,
-      spec: _.pick(spec, [
-        'params',
-        'url',
-        'urls',
-        'needBody',
-        'bulkBody',
-        'castExists',
-        'castNotFound'
-      ]),
-      location: location,
-      docUrl: def.documentation,
-      name: spec.name,
-      allParams: allParams
-    };
-
-    function hasMethod(/* ...methods */) {
-      for (var i = 0; i < arguments.length; i++) {
-        if (~action._methods.indexOf(arguments[i])) {
-          continue;
-        } else {
-          return false;
-        }
-      }
-      return true;
-    }
-    function methodsAre(/* ...methods */) {
-      return hasMethod.apply(null, arguments) && arguments.length === action._methods.length;
-    }
-
-    var method;
-
-    if (action._methods.length === 1) {
-      method = action._methods[0];
-    } else {
-      // we need to define what the default method(s) will be
-      if (hasMethod('DELETE', 'POST')) {
-        method = 'POST';
-      }
-      else if (methodsAre('DELETE')) {
-        method = 'DELETE';
-      }
-      else if (methodsAre('POST', 'PUT')) {
-        method = action.name.match(/put/i) ? 'PUT' : 'POST';
-      }
-      else if (methodsAre('GET', 'POST')) {
-        method = 'POST';
-      }
-      else if (methodsAre('GET', 'HEAD')) {
-        if (action.spec.castExists) {
-          method = 'HEAD';
-        } else {
-          method = 'GET';
-        }
-      }
-    }
-
-    if (method) {
-      if (method !== 'GET') {
-        action.spec.method = method;
-      }
-    } else {
-      throw new Error('unable to pick a method for ' + JSON.stringify(action, null, '  '));
-    }
-
-    if (action.name === 'create') {
-      action.proxy = 'index';
-      action.transformBody = 'params.op_type = \'create\';';
-    }
-
-    if (actions.push(action) === specCount && doneParsing) {
-      module.exports.emit('ready', actions);
-    }
-  });
-}
-
-/**
- * un-comment to print out the default method for any action that has multiple options
- */
-module.exports = new EventEmitter();
-// module.exports.on('ready', function (actions) {
-//   var longestName = 0;
-//   var reports = {
-//     multi_methods: [],
-//     get_with_body: []
-//   };
-//   actions.forEach(function (action) {
-//     var name;
-
-//     // console.log(action);
-//     if (action._methods.length > 1) {
-//       name = action.name + ' (' + action._methods.join('/') + ')';
-//       longestName = Math.max(name.length, longestName);
-//       reports.multi_methods.push([name, action.spec.method || 'GET', action.docUrl]);
-//     }
-
-//     if (action._methods.length === 1 && action._methods[0] === 'GET' && action.body) {
-//       name = action.name + ' (' + action._methods.join('/') + ')';
-//       longestName = Math.max(name.length, longestName);
-//       reports.get_with_body.push([name, action.spec.method || 'GET', action.docUrl]);
-//     }
-//   });
-
-//   Object.keys(reports).forEach(function (key) {
-//     console.log('\n' + key);
-//     if (reports[key].length) {
-//       reports[key].forEach(function (line) {
-//         var name = line[0];
-//         var def = line[1];
-//         var docUrl = line[2];
-//         var spacing = (new Array(longestName - name.length + 1)).join(' ');
-//         console.log(name + spacing + ' [' + def + (def.length === 3 ? ' ' : '') + ']  ->  ' + docUrl);
-//       });
-//     } else {
-//       console.log('--nada--');
-//     }
-//     console.log('\n');
-//   });
-// });

scripts/generate/js_api/aliases.js

@@ -1,63 +0,0 @@
-module.exports = {
-  'cluster.nodeHotThreads': [
-    '/_cluster/nodes/hotthreads',
-    '/_cluster/nodes/hot_threads',
-    '/_nodes/hot_threads',
-    '/_cluster/nodes/{node_id}/hotthreads',
-    '/_cluster/nodes/{node_id}/hot_threads',
-    '/_nodes/{node_id}/hot_threads'
-  ],
-  'cluster.nodeInfo': [
-    '/_cluster/nodes',
-    '/_nodes/settings',
-    '/_nodes/os',
-    '/_nodes/process',
-    '/_nodes/jvm',
-    '/_nodes/thread_pool',
-    '/_nodes/network',
-    '/_nodes/transport',
-    '/_nodes/http',
-    '/_nodes/plugin',
-    '/_cluster/nodes/{node_id}',
-    '/_nodes/{node_id}/settings',
-    '/_nodes/{node_id}/os',
-    '/_nodes/{node_id}/process',
-    '/_nodes/{node_id}/jvm',
-    '/_nodes/{node_id}/thread_pool',
-    '/_nodes/{node_id}/network',
-    '/_nodes/{node_id}/transport',
-    '/_nodes/{node_id}/http',
-    '/_nodes/{node_id}/plugin'
-  ],
-  'cluster.nodeShutdown': [
-    '/_cluster/nodes/_shutdown'
-  ],
-  'cluster.nodeStats': [
-    '/_cluster/nodes/stats',
-    '/_nodes/stats/{metric_family}',
-    '/_nodes/stats/indices/{metric}/{fields}',
-    '/_cluster/nodes/{node_id}/stats',
-    '/_nodes/{node_id}/stats/{metric_family}',
-    '/_nodes/{node_id}/stats/indices/{metric}/{fields}'
-  ],
-  'get': [
-    '/{index}/{type}/{id}/_source'
-  ],
-  'indices.deleteMapping': [
-    '/{index}/{type}/_mapping'
-  ],
-  'indices.stats': [
-    '_stats/{metric_family}',
-    '/_stats/indexing',
-    '/_stats/indexing/{indexing_types}',
-    '/_stats/search/{search_groups}',
-    '/_stats/fielddata/{fields}',
-    '/{index}/_stats/{metric_family}',
-    '/{index}/_stats/indexing',
-    '/{index}/_stats/search/{search_groups}',
-    '/{index}/_stats/fielddata/{fields}'
-  ],
-  'search': [
-    '/_search'
-  ]
-};

scripts/generate/js_api/generate.js

@@ -1,59 +0,0 @@
-module.exports = function (force) {
-  var _ = require('../../../src/lib/utils');
-  var fs = require('fs');
-  var templates = require('./templates');
-  var restSpecUpdated = require('../../rest_spec_updated');
-
-  var outputPath = _.joinPath(__dirname, '../../../src/lib/api.js');
-  var docOutputDir = _.joinPath(__dirname, '../../../docs/');
-
-  function download() {
-    require('./actions').on('ready', function (actions) {
-      var namespaces = _.filter(_.map(actions, function (action) {
-        if (~action.location.indexOf('.')) {
-          var path = action.location.split('.').slice(0, -1);
-          _.pull(path, 'prototype');
-          return path.join('.');
-        }
-      }));
-
-      // seperate the proxy actions
-      var groups = _.groupBy(actions, function (action) {
-        return action.proxy ? 'proxies' : 'normal';
-      });
-
-      fs.unlink(outputPath, function () {
-        console.log('writing', actions.length, 'api actions to', outputPath);
-
-        fs.writeFileSync(outputPath, templates.apiFile({
-          actions: groups.normal,
-          proxies: groups.proxies,
-          namespaces: _.unique(namespaces.sort(), true)
-        }));
-
-        if (!fs.existsSync(docOutputDir)) {
-          fs.mkdirSync(docOutputDir);
-        }
-
-        fs.writeFileSync(docOutputDir + '_method_list.jade', templates.apiMethodList({
-          actions: actions
-        }));
-
-        fs.writeFileSync(docOutputDir + '_methods.jade', templates.apiMethods({
-          actions: actions
-        }));
-
-      });
-    });
-  }
-
-  if (force) {
-    download();
-  } else {
-    restSpecUpdated(function (err, updated) {
-      if (err || updated) {
-        download();
-      }
-    });
-  }
-};

scripts/generate/js_api/index.js

@@ -1 +0,0 @@
-require('./generate')(require('../_force'));

scripts/generate/logs/index.js

@@ -19,9 +19,6 @@ var argv = require('optimist')
   })
   .argv;
 
-// Error.stackTraceLimit = Infinity;
-// process.exit();
-
 var es = require('../../../src/elasticsearch');
 var _ = require('../../../src/lib/utils');
 var async = require('async');

scripts/generate/templates/api_file.tmpl

@@ -22,6 +22,7 @@ api.<%= namespace %> = function <%= className %>(transport) {
 });
 
 _.each(proxies, function (action) {%>
+
 <%= partials.client_action_proxy(action) %><%
 });
-%>
+%>

scripts/generate/templates/client_action.tmpl

@@ -9,4 +9,4 @@ _.each(allParams, function(param, paramName) { %>
   }
 %><% }) %>
  */
-api<%= (location[0] === '[' ? '' : '.') + location %> = ca(<%= stringify(spec, true) %>);
+api<%= (location[0] === '[' ? '' : '.') + location %> = ca(<%= stringify(spec, true) %>);

scripts/generate/templates/client_action_proxy.tmpl

@@ -16,4 +16,4 @@ if (typeof transformBody === 'string') { %>, {
   }
 }<%
 }
-%>);
+%>);

scripts/generate/templates/index.js

@@ -1,5 +1,5 @@
 
-var _ = require('../../../../src/lib/utils');
+var _ = require('../../../src/lib/utils');
 var fs = require('fs');
 var path = require('path');
 

scripts/generate/yaml_tests.js

@@ -0,0 +1,46 @@
+module.exports = function (done) {
+
+  /**
+   * Creates a JSON version of the YAML test suite that can be simply bundled for use in the browser.
+   */
+  var jsYaml = require('js-yaml');
+  var fs = require('relative-fs').relativeTo(__dirname);
+  var async = require('async');
+  var path = require('path');
+  var tests = {}; // populated in readYamlTests
+
+  // generate the yaml tests
+  async.series([
+    readYamlTests,
+    writeYamlTests
+  ], done);
+
+  function readYamlTests(done) {
+    var testDir = path.join(__dirname, '../../src/rest-api-spec/test/');
+
+    function readDirectories(dir) {
+      fs.readdirSync(dir).forEach(function (filename) {
+        var filePath = path.join(dir, filename);
+        var stat = fs.statSync(filePath);
+        if (stat.isDirectory()) {
+          readDirectories(filePath);
+        } else if (filename.match(/\.yaml$/)) {
+          var file = tests[path.relative(testDir, filePath)] = [];
+          jsYaml.loadAll(fs.readFileSync(filePath, 'utf8'), function (doc) {
+            file.push(doc);
+          });
+        }
+      });
+    }
+
+    readDirectories(testDir);
+    done();
+  }
+
+  function writeYamlTests(done) {
+    var testFile = require('path').resolve(__dirname, '../../test/integration/yaml_suite/yaml_tests.json');
+    fs.writeFileSync(testFile, JSON.stringify(tests, null, ' '), 'utf8');
+    console.log('wrote YAML tests as JSON to', testFile);
+    done();
+  }
+};

scripts/generate/yaml_tests/README.md

@@ -1,3 +0,0 @@
-Download the yaml suite's test specs, runs before each run of the suite to ensure they are up to date.
-
-run by calling `npm run generate`. Force it to update, even if their has not been a new commit, but calling with `--force` or `-f`

scripts/generate/yaml_tests/generate.js

@@ -1,44 +0,0 @@
-module.exports = function (force) {
-  /**
-   * Check that the test directory exists, and is less than a day old, otherwise wipe it out
-   * and rebuild
-   */
-  var fs = require('fs');
-  var path = require('path');
-  var jsYaml = require('js-yaml');
-  var spec = require('../../get_spec');
-  var restSpecUpdated = require('../../rest_spec_updated');
-
-  var testFile = path.resolve(__dirname, '../../../test/integration/yaml_suite/yaml_tests.json');
-
-  function download() {
-
-    var tests = {};
-
-    fs.unlink(testFile, function () {
-      spec.get('test/**/*.yaml')
-        .on('entry', function (entry) {
-          var filename = path.relative('test', entry.path);
-          var file = tests[filename] = [];
-          jsYaml.loadAll(entry.data, function (doc) {
-            file.push(doc);
-          });
-        })
-        .on('end', function () {
-          fs.writeFileSync(testFile, JSON.stringify(tests, null, ' '), 'utf8');
-          console.log('download yaml tests to', testFile);
-        });
-    });
-  }
-
-
-  if (force) {
-    download();
-  } else {
-    restSpecUpdated(function (err, updated) {
-      if (err || updated) {
-        download();
-      }
-    });
-  }
-};

scripts/generate/yaml_tests/index.js

@@ -1 +0,0 @@
-require('./generate')(require('../_force'));

scripts/jenkins.sh

@@ -0,0 +1,19 @@
+#!/bin/sh
+
+echo "generate the latest version of the yaml-tests"
+node scripts/generate/ --no-api 2>&1 > /dev/null
+
+echo "\n--- unit ---"
+./node_modules/.bin/mocha test/unit/test_*.js \
+  --require should \
+  --reporter ../../../test/utils/jenkins-reporter.js \
+  2> test-output-node-unit.xml
+
+echo "\n--- integration ---"
+# run the integration tests
+./node_modules/.bin/mocha test/integration/yaml_suite/index.js \
+  --require should \
+  --host localhost \
+  --port $es_port \
+  --reporter ../../../test/utils/jenkins-reporter.js \
+  2> test-output-node-integration.xml

scripts/make_j_unit_xml.js

@@ -1,81 +0,0 @@
-/**
- * The JUnit xml output desired by Jenkins essentially looks like this:
- *
- * testsuites:
- *   - testsuite: (name, timestamp, hostname, tests, failures, errors, time)
- *     - testcase: (error or failure, name, classname, time)
- *
- * Full XSD avaliable [here](http://windyroad.com.au/dl/Open%20Source/JUnit.xsd)
- *
- * from
- *
- * {
- *   stats: {
- *
- *   }
- *   suite: [
- *     {
- *       name:
- *       results: []
- *       suites: [] // optional
- *     }
- *   ]
- * }
- */
-module.exports = makeJUnitXml;
-
-var testXml = require('xmlbuilder');
-var suites = testXml.create('testsuites');
-var suiteCount = 0;
-var moment = require('moment');
-var _ = require('lodash');
-
-function makeJUnitXml(runnerName, testDetails) {
-  _.each(testDetails.suites, function serializeSuite(suiteInfo) {
-    var suite = suites.ele('testsuite', {
-      package: 'elasticsearch-js:yaml_tests',
-      id: suiteCount++,
-      name: suiteInfo.name,
-      timestamp: moment(suiteInfo.start).toJSON(),
-      hostname: 'localhost',
-      tests: (suiteInfo.results && suiteInfo.results.length) || 0,
-      failures: _.where(suiteInfo.results, {pass: false}).length,
-      errors: 0,
-      time: suiteInfo.time / 1000
-    });
-
-    _.each(suiteInfo.results, function (testInfo) {
-
-      var parts = suiteInfo.name.replace(/\.yaml$/, '').replace(/\./g, '_').split(/\//);
-      var section = parts.shift();
-      var behavior = parts.join('/');
-
-      var testcase = suite.ele('testcase', {
-        name: behavior + ' - ' + testInfo.name,
-        time: (testInfo.time || 0) / 1000,
-        classname: runnerName + '.' + section
-      });
-
-      if (testInfo.errMsg) {
-        testcase.ele('failure', {
-          message: testInfo.errMsg,
-          type: 'AssertError'
-        });
-      } else if (!testInfo.pass) {
-        testcase.ele('error', {
-          message: 'Unknown Error',
-          type: 'TestError'
-        });
-      }
-    });
-
-    if (suiteInfo.suites) {
-      _.each(suiteInfo.suites, serializeSuite);
-    }
-
-    suite.ele('system-out', {}).cdata(suiteInfo.stdout);
-    suite.ele('system-err', {}).cdata(suiteInfo.stderr);
-  });
-
-  return suites.toString({ pretty: true});
-}

scripts/run_browser_integration_suite/index.js

@@ -1,16 +1,9 @@
 var server = require('./server');
-var child_process = require('child_process');
 var _ = require('lodash');
 var open = require('open');
-var fs = require('fs');
-var path = require('path');
 var async = require('async');
 var chalk = require('chalk');
 
-var yamlTestSourceFile = path.join(__dirname, '../../test/integration/yaml_suite/index.js');
-var yamlTestBundleFile = path.join(__dirname, '../../test/integration/browser_yaml_suite/yaml_tests.js');
-var clientEntryFile = path.join(__dirname, '../../src/elasticsearch.js');
-
 var browserAppNames = _.transform({
   safari: {
     darwin: 'Safari'
@@ -79,102 +72,45 @@ if (badKeys.length) {
   console.log('opening browser suite in', server.browsers);
 }
 
+server.on('tests done', function (report) {
+  var reports = [];
+  var success = true;
+  _.each(report, function (testSucceeded, browser) {
+    var msg = browser + ':' + (success ? '✔︎' : '⚑');
+    if (testSucceeded) {
+      msg = chalk.green(msg);
+    } else {
+      msg = chalk.red(msg);
+      success = false;
+    }
+    reports.push(' - ' + msg);
+  });
+  console.log('test completed!\n', reports.join('\n'));
+  process.exit(success ? 0 : 1);
+});
+
 async.series([
   function (done) {
-    fs.exists('dist', function (yes) {
-      if (!argv.forceGen && yes) {
-        done();
-        return;
-      }
-
-      console.log('generating client with "grunt build"');
-      child_process.spawn('npm', ['run', 'build_clients'], {
-        stdio: 'inherit'
-      }).on('close', function (status) {
-        done(status && 'grunt closed with a status code of ' + status + '. aborting.');
-      });
+    server.listen(0, function () {
+      server.port = server.address().port;
+      console.log('server listening on port', server.port);
+      done();
     });
   },
   function (done) {
-    fs.exists(yamlTestBundleFile, function (yes) {
-      if (!argv.forceGen && yes) {
-        done();
-        return;
-      }
+    async.eachSeries(_.clone(server.browsers), function (browser, browserDone) {
+      open('http://localhost:' + server.port +
+        '?es_hostname=' + encodeURIComponent(argv.host) +
+        '&es_port=' + encodeURIComponent(argv.port) +
+        '&browser=' + encodeURIComponent(browser), browserAppNames[browser]);
 
-      console.log('generating browser\'s yaml_tests.js bundle');
-      var b = require('browserify')();
-
-      b.add(yamlTestSourceFile);
-      var bundle = b.bundle({
-        external: [
-          'optimist'
-        ],
-        ignore: [
-          'test/integration/yaml_suite/reporter',
-          clientEntryFile
-        ]
-      });
-      var file = fs.createWriteStream(yamlTestBundleFile, {
-        flags: 'w',
-        encoding: 'utf8',
-        mode: 0666
-      });
-
-      bundle.pipe(file);
-
-      file.once('error', function (err) {
-        done(err);
-      });
-
-      bundle.once('error', function (err) {
-        done(err);
-      });
-
-      bundle.once('end', function () {
-        done();
-      });
-
-    });
+      server.once('browser complete', _.bind(browserDone, null, null));
+    }, done);
   }
 ], function (err) {
   if (err) {
     console.error(err);
     process.exit(1);
-  } else {
-    server.listen(0, function () {
-      var port = server.address().port;
-      console.log('server listening on port', port);
-
-      async.eachSeries(_.clone(server.browsers), function (browser, done) {
-        open('http://localhost:' + port +
-          '?es_hostname=' + encodeURIComponent(argv.host) +
-          '&es_port=' + encodeURIComponent(argv.port) +
-          '&browser=' + encodeURIComponent(browser), browserAppNames[browser]);
-
-        server.once('browser complete', function () {
-          done();
-        });
-      });
-    });
-
-    server.on('tests done', function (report) {
-      var reports = [];
-      var success = true;
-      _.each(report, function (testSucceeded, browser) {
-        var msg = browser + ':' + (success ? '✔︎' : '⚑');
-        if (testSucceeded) {
-          msg = chalk.green(msg);
-        } else {
-          msg = chalk.red(msg);
-          success = false;
-        }
-        reports.push(' - ' + msg);
-      });
-      console.log('test completed!\n', reports.join('\n'));
-      process.exit(success ? 0 : 1);
-    });
   }
-
 });
 

scripts/run_browser_integration_suite/server.js

@@ -4,7 +4,7 @@ var path = require('path');
 var fs = require('fs');
 var _ = require('lodash');
 var chalk = require('chalk');
-var makeJUnitXml = require('../make_j_unit_xml');
+var makeJUnitXml = require('../../test/utils/make_j_unit_xml');
 var docRoot = path.join(__dirname, '../../test/integration/browser_yaml_suite');
 chalk.enabled = true;
 
@@ -21,8 +21,14 @@ var server = http.createServer(function (req, resp) {
   req.filename = path.join(docRoot, req.uri);
 
   var end = resp.end;
-  resp.end = function () {
-    console.log(chalk[this.statusCode < 300 ? 'green' : 'red'](this.statusCode), req.uri);
+  resp.end = function (data) {
+    console.log(
+      chalk[this.statusCode < 300 ? 'green' : 'red'](this.statusCode),
+      req.uri,
+      '-',
+      Buffer.byteLength(data || ''),
+      'bytes'
+    );
     end.apply(resp, arguments);
   };
 

scripts/run_tests.js

@@ -1,7 +1,6 @@
 var async = require('async');
 var cp = require('child_process');
 var chalk = require('chalk');
-var path = require('path');
 var argv = require('optimist')
   .usage([
     'Runner for the Elasticsearch.js unit and integration tests in both node and the browser.',
@@ -25,7 +24,7 @@ var argv = require('optimist')
       alias: 's'
     },
     unit: {
-      default: true,
+      default: false,
       alias: 'u'
     },
     integration: {
@@ -44,10 +43,6 @@ var argv = require('optimist')
       default: '*',
       alias: 'b'
     },
-    'xml-output': {
-      default: true,
-      alias: 'x'
-    },
     'check-upstream': {
       default: false,
       description: 'check for remote updates to the yaml test suite'
@@ -61,22 +56,17 @@ if (process.argv.indexOf('help') + process.argv.indexOf('--help') + process.argv
 
 if (process.env.npm_config_argv) {
   // when called by NPM
-  argv = argv.parse(JSON.parse(process.env.npm_config_argv).original);
+  argv = argv.parse([].concat(process.argv).concat(JSON.parse(process.env.npm_config_argv).original));
 } else {
-  // when called by NPM - via `npm test`
+  // when called directly
   argv = argv.argv;
 }
 
 var commands = [];
 var command;
 
-if (argv['just-browser']) {
-  argv.server = false;
-  argv.browsers = '*';
-}
-
 if (argv['check-upstream']) {
-  command = ['node', 'scripts/generate/yaml_tests/index.js'];
+  command = ['node', 'scripts/generate'];
   if (argv.force) {
     command.push('--force');
   }
@@ -85,21 +75,20 @@ if (argv['check-upstream']) {
 
 if (argv.unit) {
   if (argv.server) {
-    commands.push(['mocha', 'test/unit/test_*.js', '--require should']);
+    commands.push(['./node_modules/.bin/mocha', 'test/unit/test_*.js', '--require should']);
   }
   if (argv.browsers) {
-    commands.push(['testling', '.']);
+    commands.push(['./node_modules/.bin/testling', '.']);
   }
 }
 
 if (argv.integration) {
   if (argv.server) {
     commands.push([
-      'mocha',
+      './node_modules/.bin/mocha',
       'test/integration/yaml_suite/index.js',
-      '-b',
+      // '-b',
       '--require', 'should',
-      argv.x ? '--reporter' : '', argv.x ? path.join(__dirname, '../test/integration/yaml_suite/reporter.js') : '',
       '--host', argv.host,
       '--port', argv.port
     ].filter(Boolean));
@@ -125,6 +114,7 @@ if (commands.length) {
   async.forEachSeries(commands, function (args, done) {
     var command = args.shift();
     console.log(chalk.gray.bold('\n\n' + '# ' + command + ' ' + args.join(' ')));
+
     proc = cp.spawn(command, args, {
       stdio: 'inherit'
     });