Modified the classnames for the test cases so that they work a bit better in Jenkins. Also adjusted the generate scripts to pull the latest commit sha from github.

scripts/generate/js_api/index.js

@@ -5,10 +5,12 @@ var fs = require('fs');
 var mkdirp = require('mkdirp');
 var templates = require('./templates');
 var clean = require('../../clean');
-var urlParamRE = /\{(\w+)\}/g;
+var restSpecUpdated = require('../../rest_spec_updated');
 
+var urlParamRE = /\{(\w+)\}/g;
 var outputPath = _.joinPath(__dirname, '../../../src/lib/api.js');
 var docOutputPath = _.joinPath(__dirname, '../../../docs/api.md');
+var lastFetchTmp = path.join(__dirname, './last_fetch.tmp');
 
 function download() {
   require('./actions').on('ready', function (actions) {
@@ -34,15 +36,8 @@ function download() {
   });
 }
 
-if (process.env.FORCE_GEN) {
-  download();
-} else {
-  try {
-    var stat = fs.statSync(outputPath);
-    if (!stat.isFile() || stat.ctime < Date.now() - 86400000) {
-      download();
-    }
-  } catch (e) {
+restSpecUpdated(function (err, updated) {
+  if (process.env.FORCE_GEN || err || updated) {
     download();
   }
-}
+});

scripts/generate/yaml_tests/index.js

@@ -8,6 +8,7 @@ var jsYaml = require('js-yaml');
 var spec = require('../../get_spec');
 var clean = require('../../clean');
 var _ = require('../../../src/lib/utils');
+var restSpecUpdated = require('../../rest_spec_updated');
 
 var testFile = path.resolve(__dirname, '../../../test/integration/yaml_suite/yaml_tests.json');
 
@@ -30,15 +31,9 @@ function download() {
     });
 }
 
-if (process.env.FORCE_GEN) {
-  download();
-} else {
-  try {
-    var stat = fs.statSync(testFile);
-    if (!stat.isFile() || stat.ctime < Date.now() - 86400000) {
-      download();
-    }
-  } catch (e) {
+
+restSpecUpdated(function (err, updated) {
+  if (process.env.FORCE_GEN || err || updated) {
     download();
   }
-}
+});

scripts/get_spec.js

@@ -17,7 +17,6 @@ exports.get = function (pattern) {
     if (incoming.statusCode !== 200) {
       req.abort();
       if (incoming.headers.location) {
-        console.log('redirecting to', incoming.headers.location);
         req = https.get(_.extend(
             url.parse(tarUrl),
             url.parse(incoming.headers.location)

scripts/rest_spec_updated.js

@@ -0,0 +1,64 @@
+var https = require('https');
+var lastCommitUrl = 'https://api.github.com/repos/elasticsearch/elasticsearch-rest-api-spec/commits/HEAD';
+var fs = require('fs');
+
+var lastRestSpecUpdateFile = __dirname + '/last_rest_spec_update.sha';
+var lastRestSpecUpdate;
+var updated;
+
+if (fs.existsSync(lastRestSpecUpdateFile)) {
+  lastRestSpecUpdate = fs.readFileSync(lastRestSpecUpdateFile, 'utf8');
+}
+
+var req = https.get(lastCommitUrl, function (incoming) {
+  if (incoming.statusCode !== 200) {
+    req.abort();
+    console.error('request for last commit failed', incoming.statusCode, incoming.headers);
+    return;
+  }
+
+  var body = '';
+
+  incoming.on('data', onData);
+  incoming.on('end', onEnd);
+
+  function onData(chunk) {
+    body += chunk;
+  }
+
+  function onEnd() {
+    incoming.removeListener('data', onData);
+    incoming.removeListener('end', onEnd);
+    var _req = req;
+    req = null;
+
+    var resp;
+    try {
+      resp = JSON.parse(body);
+    } catch (e) {
+      console.log('unable to parse response from github');
+      _req.emit('ready');
+      return;
+    }
+
+    if (lastRestSpecUpdate === resp.sha) {
+      updated = false;
+    } else {
+      updated = true;
+      fs.writeFileSync(lastRestSpecUpdateFile, resp.sha);
+    }
+    _req.emit('ready');
+  }
+});
+
+module.exports = function (cb) {
+  function done() {
+    cb(null, updated);
+  }
+
+  if (req) {
+    req.on('ready', done);
+  } else {
+    process.nextTick(done);
+  }
+};