Switched from downloading zips to downloading tarballs, as the unzip module was randomly sucking.

.gitignore

@@ -1,3 +1,4 @@
 node_modules
 scripts/scratch*
-test/integration/yaml-suite/log
+test/integration/yaml_suite/log
+test/integration/yaml_suite/tests

Gruntfile.js

@@ -17,8 +17,8 @@ module.exports = function (grunt) {
       unit: [
         'test/unit/**/*.test.js'
       ],
-      'yaml-suite': [
-        'test/integration/yaml-suite/index.js'
+      yaml_suite: [
+        'test/integration/yaml_suite/index.js'
       ],
       options: {
         colors: true,
@@ -32,6 +32,7 @@ module.exports = function (grunt) {
       source: {
         src: [
           'src/**/*.js',
+          'scripts/**/*.js',
           'Gruntfile.js'
         ],
         options: {
@@ -62,11 +63,17 @@ module.exports = function (grunt) {
         interupt: true
       }
     },
-    run: {
-      generate_js_api: {
+    generate: {
+      js_api: {
         cmd: 'node',
         args: [
-          'scripts/generate/js_api/index.js'
+          'scripts/generate/js_api'
+        ]
+      },
+      yaml_suite: {
+        cmd: 'node',
+        args: [
+          'scripts/generate/yaml_tests'
         ]
       }
     }//,
@@ -119,10 +126,14 @@ module.exports = function (grunt) {
 
   // Default task.
   // grunt.registerTask('docs', ['docular']);
-  grunt.registerTask('test', ['jshint', 'mochaTest']);
-  grunt.registerTask('default', ['jshint', 'mochaTest:unit']);
+  grunt.registerTask('default', [
+    'jshint',
+    'mochaTest:unit',
+    'generate:yaml_suite',
+    'mochaTest:yaml_suite'
+  ]);
 
-  grunt.task.registerMultiTask('run', 'used to run arbitrary commands', function () {
+  grunt.task.registerMultiTask('generate', 'used to generate things', function () {
     var done = this.async();
     var proc = require('child_process').spawn(
       this.data.cmd,

README.md

@@ -1,4 +1,5 @@
 elasticsearch-js
+[![Build Status](https://travis-ci.org/spenceralger/elasticsearch-js.png)](https://travis-ci.org/spenceralger/elasticsearch-js)
 =================
 Official *low-level* client for Elasticsearch.
 

package.json

@@ -25,15 +25,15 @@
     "expect.js": "~0.2.0",
     "async": "~0.2.9",
     "optimist": "~0.6.0",
-    "minimatch": "~0.2.12",
-    "unzip": "~0.1.9"
+    "minimatch": "~0.2.12"
   },
   "license": "Apache License",
   "dependencies": {
     "require-directory": "git://github.com/spenceralger/node-require-directory.git#master",
     "cli-color": "~0.2.3",
     "qs": "~0.6.5",
-    "lodash": "~2.2.1"
+    "lodash": "~2.2.1",
+    "tar": "~0.1.18"
   },
   "scripts": {
     "test": "grunt test"

scripts/clean.js

@@ -0,0 +1,35 @@
+
+/**
+ * Stupid simple recursive file/directory clearing. Nothing serious.
+ *
+ * @param  {String} path Location of the file/directory which should be wiped out
+ * @return {Boolean}     frue on success
+ */
+module.exports = function (path) {
+  try {
+    var stats = fs.statSync(path);
+    if (stats && stats.isDirectory()) {
+      console.log('removing', path, 'directory recursively');
+      rmDirRecursive(path);
+    } else {
+      console.log('removing', path);
+      fs.unlinkSync(path);
+    }
+    return true;
+  } catch (e) {
+    return false;
+  }
+};
+
+
+function rmDirRecursive(path) {
+  fs.readdirSync(path).forEach(function (file, index) {
+    var curPath = path + '/' + file;
+    if (fs.statSync(curPath).isDirectory()) { // recurse
+      rmDirRecursive(curPath);
+    } else { // delete file
+      fs.unlinkSync(curPath);
+    }
+  });
+  fs.rmdirSync(path);
+}

scripts/generate/js_api/README.md

@@ -1,3 +1,5 @@
-Generate the API using the es [REST api spec](https://github.com/elasticsearch/elasticsearch-rest-api-spec);
+Generate the API using the es [REST api spec](https://github.com/elasticsearch/elasticsearch-rest-api-spec).
 
-to run, call `grunt run:generate_js_api`
+to run, call `grunt`, or for just this task run `grunt generate:js_api`.
+
+To force a regen, delete the old `src/lib/api.js` file.

scripts/generate/js_api/index.js

@@ -1,44 +1,13 @@
+var outputPath = _.joinPath(__dirname, '../../../src/lib/api.js');
+
 var _ = require('../../../src/lib/utils');
 var asset = require('assert');
 var path = require('path');
 var fs = require('fs');
 var mkdirp = require('mkdirp');
-
-var outputPath = _.joinPath(__dirname, '../../../src/lib/api.js');
-
 var templates = require('./templates');
-
-// completely delete the output directory
-var clean = (function () {
-  function rmDirRecursive(path) {
-    fs.readdirSync(path).forEach(function (file, index) {
-      var curPath = path + '/' + file;
-      if (fs.statSync(curPath).isDirectory()) { // recurse
-        rmDirRecursive(curPath);
-      } else { // delete file
-        fs.unlinkSync(curPath);
-      }
-    });
-    fs.rmdirSync(path);
-  }
-
-  return function (path) {
-    try {
-      var stats = fs.statSync(path);
-      if (stats && stats.isDirectory()) {
-        console.log('removing', path, 'directory recursively');
-        rmDirRecursive(path);
-      } else {
-        console.log('removing', path);
-        fs.unlinkSync(path);
-      }
-      return true;
-    } catch (e) {
-      return false;
-    }
-  };
-
-})();
+var clean = require('../../clean');
+var urlParamRE = /\{(\w+)\}/g;
 
 require('./spec').on('ready', function (specs) {
   var defs = [];
@@ -53,7 +22,6 @@ require('./spec').on('ready', function (specs) {
           var requiredVars = {};
           var param;
           var target;
-          var urlParamRE = /\{(\w+)\}/g;
 
           if (url.charAt(0) !== '/') {
             url = '/' + url;

scripts/generate/js_api/spec.js

@@ -2,8 +2,6 @@ var _ = require('../../../src/lib/utils')
 
 var EventEmitter = require('events').EventEmitter;
 var aliases = require('./aliases');
-var https = require('https');
-var unzip = require('unzip');
 
 var castNotFoundRE = /exists/;
 var usesBulkBodyRE = /^(bulk|msearch)$/;
@@ -11,46 +9,22 @@ var usesBulkBodyRE = /^(bulk|msearch)$/;
 var specCount = 0;
 var doneParsing = false;
 
-https.get('https://codeload.github.com/elasticsearch/elasticsearch-rest-api-spec/zip/master', function (incoming) {
-  incoming.pipe(unzip.Parse())
-  .on('entry', function (entry) {
-    if (entry.type === 'File' && entry.path.match(/(^|\/)api\/.*\.json$/)) {
-      specCount++;
-      return collectEntry(entry);
-    } else {
-      entry.autodrain();
-    }
-  })
-  .on('close', function () {
+require('../../get_spec')
+  .get('api/*.json')
+  .on('entry', transformFile)
+  .on('end', function () {
     doneParsing = true;
     if (specs.length === specCount) {
       module.exports.emit('ready', specs);
     }
-  });
-})
+  })
 
 var specs = [];
 
-function collectEntry(entry) {
-  var file = '';
+function transformFile(entry) {
+  specCount++;
 
-  function onData (chunk) {
-    file+= chunk;
-  }
-
-  function onEnd () {
-    entry.removeListener('data', onData);
-    entry.removeListener('end', onEnd);
-    process.nextTick(function () {
-      transformFile(file);
-    });
-  }
-
-  entry.on('data', onData)
-  entry.on('end', onEnd);
-}
-
-function transformFile(file) {
+  var file = entry.data;
   // itterate all of the specs within the file, should only be one
   _.each(JSON.parse(file), function (def, name) {
     var steps = name.split('.');

scripts/generate/yaml_tests/README.md

@@ -0,0 +1,5 @@
+Download the yaml suite's test specs, runs before each run of the suite to ensure they are up to date, and will re-downloaded every day.
+
+to run, call `grunt` or for this specific task, `grunt generate:yaml_suite`.
+
+To force a regen, delete the old `test/integration/yaml_suite/tests` directory.

scripts/generate/yaml_tests/index.js

@@ -0,0 +1,34 @@
+/**
+ * Check that the test directory exists, and is less than a day old, otherwise wipe it out
+ * and rebuild
+ */
+var fs = require('fs');
+var path = require('path');
+var mkdirp = require('mkdirp');
+var spec = require('../../get_spec');
+var clean = require('../../clean');
+
+var testDir = path.resolve(__dirname, '../../../test/integration/yaml_suite/tests');
+
+function download() {
+
+  clean(testDir);
+  spec.get('test/**/*.yaml')
+    .on('entry', function (entry) {
+      entry.path = path.resolve(testDir, path.relative('test', entry.path));
+      mkdirp.sync(path.dirname(entry.path));
+      fs.writeFileSync(entry.path, entry.data, 'utf8');
+    })
+    .on('end', function () {
+      console.log('download yaml tests to', testDir);
+    });
+}
+
+try {
+  var stat = fs.statSync(testDir);
+  if (!stat.isDirectory() || stat.ctime < Date.now() - 86400000) {
+    download();
+  }
+} catch (e) {
+  download();
+}

scripts/get_spec.js

@@ -0,0 +1,63 @@
+var EventEmitter = require('events').EventEmitter;
+var Minimatch = require('minimatch').Minimatch;
+var https = require('https');
+var tar = require('tar');
+var zlib = require('zlib');
+var path = require('path');
+var _ = require('lodash');
+var url = require('url');
+var tarUrl = 'https://github.com/elasticsearch/elasticsearch-rest-api-spec/archive/master.tar.gz';
+
+exports.get = function (pattern) {
+  var stream = new EventEmitter();
+  var matcher = new Minimatch(pattern);
+
+  var req = https.get(tarUrl, function receiveZip (incoming) {
+    if (incoming.statusCode !== 200) {
+      req.abort();
+      if (incoming.headers.location) {
+        console.log('redirecting to', incoming.headers.location);
+        req = https.get(_.extend(
+            url.parse(tarUrl),
+            url.parse(incoming.headers.location)
+          ),
+          receiveZip
+        );
+      } else {
+        console.error('request failed', incoming.statusCode, incoming.headers)
+      }
+    } else {
+      incoming.pipe(zlib.createGunzip()).pipe(tar.Parse())
+        .on('entry', function (entry) {
+          entry.path = path.relative('elasticsearch-rest-api-spec-master', entry.path);
+          if (matcher.match(entry.path)) {
+            collectData(entry);
+          } else {
+            entry.resume();
+          }
+        })
+        .on('end', function () {
+          stream.emit('end');
+        });
+    }
+  });
+
+  function collectData(entry) {
+    entry.data = '';
+
+    entry.on('data', onData)
+    entry.on('end', onEnd);
+
+    function onData (chunk) {
+      entry.data += chunk;
+    }
+
+    function onEnd () {
+      entry.removeListener('data', onData);
+      entry.removeListener('end', onEnd);
+      stream.emit('entry', entry);
+    }
+  }
+
+  return stream;
+};

src/lib/Log.js

@@ -58,7 +58,7 @@ Log.prototype.close = function () {
     console.error('Something is still listening for log events, but the logger is closing.');
     this.clearAllListeners();
   }
-}
+};
 
 /**
  * Levels observed by the loggers, ordered by rank

src/lib/api.js

@@ -1960,6 +1960,9 @@ api.indices.prototype.getTemplate = ca({
           type: 'string'
         }
       }
+    },
+    {
+      fmt: '/_template'
     }
   ]
 });

src/lib/log.js

@@ -58,7 +58,7 @@ Log.prototype.close = function () {
     console.error('Something is still listening for log events, but the logger is closing.');
     this.clearAllListeners();
   }
-}
+};
 
 /**
  * Levels observed by the loggers, ordered by rank

src/lib/utils.js

@@ -385,13 +385,8 @@ utils.applyArgs = function (func, context, args, sliceIndex) {
  * @return {[type]} [description]
  */
 _.nextTick = function (cb) {
-  console.log('tick');
-  var args = Array.prototype.slice.call(arguments, 1);
   // bind the function and schedule it
-  process.nextTick(function () {
-    console.log('tock');
-    cb.apply(null, args);
-  });
+  process.nextTick(_.bindKey(_, 'applyArgs', cb, null, arguments, 1));
 };
 
 /**

test/integration/index.js

@@ -1,2 +0,0 @@
-require('./yaml-suite');
-require('./network-failures');

test/integration/yaml_suite/index.js

@@ -20,8 +20,7 @@ var argv = require('optimist')
 
 Error.stackTraceLimit = Infinity;
 
-// Where do the tests live?
-var TEST_DIR = path.resolve(__dirname, '../../../es_api_spec/test/');
+var testDir = path.resolve(__dirname, './tests');
 
 var doPattern = new Minimatch(argv.match);
 
@@ -84,7 +83,7 @@ before(function (done) {
       } else {
         done();
       }
-    })
+    });
   }
 });
 
@@ -105,27 +104,6 @@ before(function () {
 
 before(clearIndices);
 
-/**
- * recursively crawl the directory, looking for yaml files which will be passed to loadFile
- * @param  {String} dir - The directory to crawl
- * @return {undefined}
- */
-function loadDir(dir) {
-  fs.readdirSync(dir).forEach(function (fileName) {
-    describe(fileName, function () {
-      var location = path.join(dir, fileName),
-          stat = fs.statSync(location);
-
-      if (stat.isFile() && fileName.match(/\.yaml$/) && doPattern.match(path.relative(TEST_DIR, location))) {
-        loadFile(location);
-      }
-      else if (stat.isDirectory()) {
-        loadDir(location);
-      }
-    });
-  });
-}
-
 /**
  * The version that ES is running, in comparable string form XXX-XXX-XXX, fetched when needed
  * @type {String}
@@ -206,24 +184,6 @@ function rangeMatchesCurrentVersion(rangeString, done) {
   }
 }
 
-/**
- * read the file's contents, parse the yaml, pass to makeTestFromYamlDoc
- *
- * @param  {String} path - Full path to yaml file
- * @return {undefined}
- */
-function loadFile(location) {
-  jsYaml.loadAll(
-    fs.readFileSync(location, { encoding: 'utf8' }),
-    function (doc) {
-      makeTestFromYamlDoc(doc);
-    },
-    {
-      filename: location
-    }
-  );
-}
-
 /**
  * Read the test descriptions from a yaml document (usually only one test, per doc but
  * sometimes multiple docs per file, and because of the layout there COULD be
@@ -589,4 +549,46 @@ ActionRunner.prototype = {
   }
 };
 
-loadDir(TEST_DIR);
+/**
+ * recursively crawl the directory, looking for yaml files which will be passed to loadFile
+ * @param  {String} dir - The directory to crawl
+ * @return {undefined}
+ */
+(function () {
+
+  /**
+   * read the file's contents, parse the yaml, pass to makeTestFromYamlDoc
+   *
+   * @param  {String} path - Full path to yaml file
+   * @return {undefined}
+   */
+  function loadFile(location) {
+    jsYaml.loadAll(
+      fs.readFileSync(location, { encoding: 'utf8' }),
+      function (doc) {
+        makeTestFromYamlDoc(doc);
+      },
+      {
+        filename: location
+      }
+    );
+  }
+
+  function loadDir(dir) {
+    fs.readdirSync(dir).forEach(function (fileName) {
+      describe(fileName, function () {
+        var location = path.join(dir, fileName),
+            stat = fs.statSync(location);
+
+        if (stat.isFile() && fileName.match(/\.yaml$/) && doPattern.match(path.relative(testDir, location))) {
+          loadFile(location);
+        }
+        else if (stat.isDirectory()) {
+          loadDir(location);
+        }
+      });
+    });
+  }
+
+  loadDir(testDir);
+})();

test/integration/yaml_suite/server.js

@@ -42,7 +42,7 @@ exports.start = function (params, cb) {
     if (line.match(/started\s*$/m)) {
       console.log('Personal ES Server started at', server.__hostname + ':' + server.__port);
       server.stdout.removeListener('data', onProcessData);
-      server.stdout.on('data', _.noop);
+      server.stdout.resume();
       cb(null, server);
     }
   });