Switched from downloading zips to downloading tarballs, as the unzip module was randomly sucking.

scripts/generate/js_api/README.md

@@ -1,3 +1,5 @@
-Generate the API using the es [REST api spec](https://github.com/elasticsearch/elasticsearch-rest-api-spec);
+Generate the API using the es [REST api spec](https://github.com/elasticsearch/elasticsearch-rest-api-spec).
 
-to run, call `grunt run:generate_js_api`
+to run, call `grunt`, or for just this task run `grunt generate:js_api`.
+
+To force a regen, delete the old `src/lib/api.js` file.

scripts/generate/js_api/index.js

@@ -1,44 +1,13 @@
+var outputPath = _.joinPath(__dirname, '../../../src/lib/api.js');
+
 var _ = require('../../../src/lib/utils');
 var asset = require('assert');
 var path = require('path');
 var fs = require('fs');
 var mkdirp = require('mkdirp');
-
-var outputPath = _.joinPath(__dirname, '../../../src/lib/api.js');
-
 var templates = require('./templates');
-
-// completely delete the output directory
-var clean = (function () {
-  function rmDirRecursive(path) {
-    fs.readdirSync(path).forEach(function (file, index) {
-      var curPath = path + '/' + file;
-      if (fs.statSync(curPath).isDirectory()) { // recurse
-        rmDirRecursive(curPath);
-      } else { // delete file
-        fs.unlinkSync(curPath);
-      }
-    });
-    fs.rmdirSync(path);
-  }
-
-  return function (path) {
-    try {
-      var stats = fs.statSync(path);
-      if (stats && stats.isDirectory()) {
-        console.log('removing', path, 'directory recursively');
-        rmDirRecursive(path);
-      } else {
-        console.log('removing', path);
-        fs.unlinkSync(path);
-      }
-      return true;
-    } catch (e) {
-      return false;
-    }
-  };
-
-})();
+var clean = require('../../clean');
+var urlParamRE = /\{(\w+)\}/g;
 
 require('./spec').on('ready', function (specs) {
   var defs = [];
@@ -53,7 +22,6 @@ require('./spec').on('ready', function (specs) {
           var requiredVars = {};
           var param;
           var target;
-          var urlParamRE = /\{(\w+)\}/g;
 
           if (url.charAt(0) !== '/') {
             url = '/' + url;

scripts/generate/js_api/spec.js

@@ -2,8 +2,6 @@ var _ = require('../../../src/lib/utils')
 
 var EventEmitter = require('events').EventEmitter;
 var aliases = require('./aliases');
-var https = require('https');
-var unzip = require('unzip');
 
 var castNotFoundRE = /exists/;
 var usesBulkBodyRE = /^(bulk|msearch)$/;
@@ -11,46 +9,22 @@ var usesBulkBodyRE = /^(bulk|msearch)$/;
 var specCount = 0;
 var doneParsing = false;
 
-https.get('https://codeload.github.com/elasticsearch/elasticsearch-rest-api-spec/zip/master', function (incoming) {
-  incoming.pipe(unzip.Parse())
-  .on('entry', function (entry) {
-    if (entry.type === 'File' && entry.path.match(/(^|\/)api\/.*\.json$/)) {
-      specCount++;
-      return collectEntry(entry);
-    } else {
-      entry.autodrain();
-    }
-  })
-  .on('close', function () {
+require('../../get_spec')
+  .get('api/*.json')
+  .on('entry', transformFile)
+  .on('end', function () {
     doneParsing = true;
     if (specs.length === specCount) {
       module.exports.emit('ready', specs);
     }
-  });
-})
+  })
 
 var specs = [];
 
-function collectEntry(entry) {
-  var file = '';
+function transformFile(entry) {
+  specCount++;
 
-  function onData (chunk) {
-    file+= chunk;
-  }
-
-  function onEnd () {
-    entry.removeListener('data', onData);
-    entry.removeListener('end', onEnd);
-    process.nextTick(function () {
-      transformFile(file);
-    });
-  }
-
-  entry.on('data', onData)
-  entry.on('end', onEnd);
-}
-
-function transformFile(file) {
+  var file = entry.data;
   // itterate all of the specs within the file, should only be one
   _.each(JSON.parse(file), function (def, name) {
     var steps = name.split('.');

scripts/generate/yaml_tests/README.md

@@ -0,0 +1,5 @@
+Download the yaml suite's test specs, runs before each run of the suite to ensure they are up to date, and will re-downloaded every day.
+
+to run, call `grunt` or for this specific task, `grunt generate:yaml_suite`.
+
+To force a regen, delete the old `test/integration/yaml_suite/tests` directory.

scripts/generate/yaml_tests/index.js

@@ -0,0 +1,34 @@
+/**
+ * Check that the test directory exists, and is less than a day old, otherwise wipe it out
+ * and rebuild
+ */
+var fs = require('fs');
+var path = require('path');
+var mkdirp = require('mkdirp');
+var spec = require('../../get_spec');
+var clean = require('../../clean');
+
+var testDir = path.resolve(__dirname, '../../../test/integration/yaml_suite/tests');
+
+function download() {
+
+  clean(testDir);
+  spec.get('test/**/*.yaml')
+    .on('entry', function (entry) {
+      entry.path = path.resolve(testDir, path.relative('test', entry.path));
+      mkdirp.sync(path.dirname(entry.path));
+      fs.writeFileSync(entry.path, entry.data, 'utf8');
+    })
+    .on('end', function () {
+      console.log('download yaml tests to', testDir);
+    });
+}
+
+try {
+  var stat = fs.statSync(testDir);
+  if (!stat.isDirectory() || stat.ctime < Date.now() - 86400000) {
+    download();
+  }
+} catch (e) {
+  download();
+}