Slight refactor to the api module, so it will simply extend the client like it did previously, and

also exposes itself on the module.exports so that is can be referenced externally.

Added "grunt run" task which currently has a single config, generate_js_api.

Removed the api spec submodule, generator just downloads master when it runs.

.gitignore

@@ -1,3 +1,3 @@
 node_modules
 scripts/scratch*
-test/integration-test.log
+test/integration/yaml-suite/log

package.json

@@ -25,7 +25,8 @@
     "expect.js": "~0.2.0",
     "async": "~0.2.9",
     "optimist": "~0.6.0",
-    "minimatch": "~0.2.12"
+    "minimatch": "~0.2.12",
+    "unzip": "~0.1.9"
   },
   "license": "Apache License",
   "dependencies": {

scripts/generate/js_api/index.js

@@ -7,7 +7,6 @@ var mkdirp = require('mkdirp');
 var outputPath = _.joinPath(__dirname, '../../../src/lib/api.js');
 
 var templates = require('./templates');
-var specs = require('./spec');
 
 // completely delete the output directory
 var clean = (function () {
@@ -41,7 +40,7 @@ var clean = (function () {
 
 })();
 
-exports.run = function () {
+require('./spec').on('ready', function (specs) {
   var defs = [];
   var namespaces = [];
 
@@ -123,6 +122,6 @@ exports.run = function () {
     actions: actions,
     namespaces: _.unique(namespaces.sort(), true)
   }));
-};
+});
+
 
-exports.run();

scripts/generate/js_api/spec.js

@@ -1,48 +1,85 @@
 var _ = require('../../../src/lib/utils')
 
-var docs = _.requireDir(module, '../../../es_api_spec/api');
+var EventEmitter = require('events').EventEmitter;
 var aliases = require('./aliases');
+var https = require('https');
+var unzip = require('unzip');
 
 var castNotFoundRE = /exists/;
 var usesBulkBodyRE = /^(bulk|msearch)$/;
 
-var defs = [];
+var specCount = 0;
+var doneParsing = false;
 
-// itterate all of the found docs
-Object.keys(docs).forEach(function (filename) {
-  Object.keys(docs[filename]).forEach(function (name) {
-    var def = docs[filename][name];
-    def.name = name;
-    defs.push(def);
+https.get('https://codeload.github.com/elasticsearch/elasticsearch-rest-api-spec/zip/master', function (incoming) {
+  incoming.pipe(unzip.Parse())
+  .on('entry', function (entry) {
+    if (entry.type === 'File' && entry.path.match(/(^|\/)api\/.*\.json$/)) {
+      specCount++;
+      return collectEntry(entry);
+    } else {
+      entry.autodrain();
+    }
   })
-});
+  .on('close', function () {
+    doneParsing = true;
+    if (specs.length === specCount) {
+      module.exports.emit('ready', specs);
+    }
+  });
+})
 
-module.exports = _.map(defs, function (def) {
-  var name = def.name;
-  var steps = name.split('.');
+var specs = [];
 
-  var spec = {
-    name: name,
-    methods: _.map(def.methods, function (m) { return m.toUpperCase(); }),
-    docUrl: def.documentation,
-    urlParts: def.url.parts,
-    params: def.url.params,
-    urls: _.difference(def.url.paths, aliases[name]),
-    body: def.body || null,
-    path2lib: _.repeat('../', steps.length + 1) + 'lib/'
-  };
+function collectEntry(entry) {
+  var file = '';
 
-  if (def.body && def.body.requires) {
-    spec.needBody = true;
+  function onData (chunk) {
+    file+= chunk;
   }
 
-  if (usesBulkBodyRE.test(name)) {
-    spec.bulkBody = true;
+  function onEnd () {
+    entry.removeListener('data', onData);
+    entry.removeListener('end', onEnd);
+    process.nextTick(function () {
+      transformFile(file);
+    });
   }
 
-  if (castNotFoundRE.test(name)) {
-    spec.castNotFound = true;
-  }
+  entry.on('data', onData)
+  entry.on('end', onEnd);
+}
 
-  return spec;
-});
+function transformFile(file) {
+  // itterate all of the specs within the file, should only be one
+  _.each(JSON.parse(file), function (def, name) {
+    var steps = name.split('.');
+    var spec = {
+      name: name,
+      methods: _.map(def.methods, function (m) { return m.toUpperCase(); }),
+      docUrl: def.documentation,
+      urlParts: def.url.parts,
+      params: def.url.params,
+      urls: _.difference(def.url.paths, aliases[name]),
+      body: def.body || null,
+      path2lib: _.repeat('../', steps.length + 1) + 'lib/'
+    };
+
+    if (def.body && def.body.requires) {
+      spec.needBody = true;
+    }
+
+    if (usesBulkBodyRE.test(name)) {
+      spec.bulkBody = true;
+    }
+
+    if (castNotFoundRE.test(name)) {
+      spec.castNotFound = true;
+    }
+    if (specs.push(spec) === specCount && doneParsing) {
+      module.exports.emit('ready', specs);
+    }
+  })
+}
+
+module.exports = new EventEmitter();

src/lib/api.js

@@ -1082,8 +1082,8 @@ api.get = ca({
  * Perform a [](http://elasticsearch.org/guide/reference/api/get/) request
  *
  * @param {Object} params - An object with parameters used to carry out this action
- * @param {String|ArrayOfStrings|Boolean} params._source_exclude
- * @param {String|ArrayOfStrings|Boolean} params._source_include
+ * @param {String|ArrayOfStrings|Boolean} params.exclude
+ * @param {String|ArrayOfStrings|Boolean} params.include
  * @param {String} params.parent
  * @param {String} params.preference
  * @param {Boolean} params.realtime
@@ -1095,10 +1095,10 @@ api.getSource = ca({
     'GET'
   ],
   params: {
-    _source_exclude: {
+    exclude: {
       type: 'list'
     },
-    _source_include: {
+    include: {
       type: 'list'
     },
     parent: {

src/lib/connections/http.js

@@ -106,7 +106,7 @@ HttpConnection.prototype.request = function (params, cb) {
     cleanUp(err);
   });
 
-  // timeout for the entire request, req.setTimeout only set the socket level timeout
+  // timeout for the entire request.
   timeoutId = setTimeout(function () {
     request.emit('error', new errors.RequestTimeout('Request timed out at ' + timeout + 'ms'));
   }, timeout);

test/integration/yaml-suite/server.js

@@ -17,9 +17,7 @@ exports.start = function (params, cb) {
       '-f',
       '-Des.cluster.name=' + params.clusterName,
       '-Des.path.data=' + params.dataPath,
-      '-Des.logger.level=DEBUG',
-      '-Des.gateway.type=none',
-      '-Des.index.store.type=memory',
+      // '-Des.logger.level=DEBUG',
       '-Des.discovery.zen.ping.multicast.enabled=false',
     ],
     {
@@ -33,7 +31,7 @@ exports.start = function (params, cb) {
     }
   );
 
-  server.stdout.on('data', function (line) {
+  server.stdout.on('data', function onProcessData(line) {
     line = _.trim(line.toString());
     var match;
     if (match = line.match(/bound_address \{inet\[\/?([^:]+):(\d+)\]\}/)) {
@@ -43,8 +41,8 @@ exports.start = function (params, cb) {
 
     if (line.match(/started\s*$/m)) {
       console.log('Personal ES Server started at', server.__hostname + ':' + server.__port);
-      server.stdout.removeAllListeners();
-      server.stderr.removeAllListeners();
+      server.stdout.removeListener('data', onProcessData);
+      server.stdout.on('data', _.noop);
       cb(null, server);
     }
   });
@@ -53,9 +51,11 @@ exports.start = function (params, cb) {
     console.error(_.trim(line.toString()));
   });
 
-  server.on('exit', function (code) {
+  server.on('close', function (code) {
+    server.stdout.removeAllListeners();
+    server.stderr.removeAllListeners();
     console.log('Personal ES Server Shutdown with exit code', code);
-  })
+  });
 
   process.on('exit', function () {
     server.kill();