[scripts/generate] clear out files from previous runs before generating

scripts/generate/index.js

@@ -28,8 +28,7 @@ var argv = require('optimist')
   });
 
 var path = require('path');
-var root = require('find-root')(__dirname);
-var fromRoot = path.join.bind(path, root);
+var fromRoot = path.join.bind(path, require('find-root')(__dirname));
 var utils = require(fromRoot('grunt/utils'));
 var _ = require(fromRoot('src/lib/utils'));
 var esUrl = 'https://github.com/elasticsearch/elasticsearch.git';
@@ -49,10 +48,19 @@ if (argv.branch) {
   branches = utils.branches;
 }
 
-var sourceDir = fromRoot('src/_elasticsearch_');
-function storeDir(branch) {
-  return fromRoot('src/_elasticsearch_' + _.snakeCase(branch));
-}
+var paths = {
+  root: fromRoot('.'),
+  src: fromRoot('src'),
+  esSrc: fromRoot('src/_elasticsearch_'),
+  docs: fromRoot('docs'),
+  apiSrc: 'src/lib/apis',
+  getArchiveDir: function (branch) {
+    return fromRoot('src/_elasticsearch_' + _.snakeCase(branch));
+  },
+  getArchiveTarball: function (branch) {
+    return fromRoot('src/_elasticsearch_' + _.snakeCase(branch) + '.tar');
+  }
+};
 
 function isDirectory(dir) {
   var stat;
@@ -60,6 +68,33 @@ function isDirectory(dir) {
   return (stat && stat.isDirectory());
 }
 
+function dirFilter(dir, fn) {
+  try {
+    return fs.readdirSync(dir)
+    .filter(function (name) {
+      return name !== '.' && name !== '..' && fn(name);
+    })
+    .map(function (filename) {
+      return path.join(dir, filename);
+    });
+  } catch (e) {
+    return [];
+  }
+}
+
+function dirRegex(dir, regexp) {
+  return dirFilter(dir, function (name) {
+    return name.match(regexp);
+  });
+}
+
+function dirOpts(dir, opts) {
+  opts = _.isArray(opts) ? opts : [opts];
+  return dirFilter(dir, function (name) {
+    return _.contains(opts, name);
+  });
+}
+
 function spawnStep(cmd, args, cwd) {
   return function (done) {
     spawn(cmd, args, {
@@ -71,48 +106,101 @@ function spawnStep(cmd, args, cwd) {
   };
 }
 
-function execStep(cmd, cwd) {
-  return function (done) {
-    spawn.exec(cmd, {
-      verbose: argv.verbose,
-      cwd: cwd
-    }, function (status) {
-      done(status ? new Error('Non-zero exit code: ' + status) : void 0);
-    });
-  };
-}
-
 function initStep() {
+  if (isDirectory(paths.esSrc)) return;
+
   return function (done) {
-    if (isDirectory(sourceDir)) return done();
     async.series([
-      spawnStep('git', ['init', '--bare', sourceDir], root),
-      spawnStep('git', ['remote', 'add', 'origin', esUrl], sourceDir)
+      spawnStep('git', ['init', '--bare', paths.esSrc], paths.root),
+      spawnStep('git', ['remote', 'add', 'origin', esUrl], paths.esSrc)
     ], done);
   };
 }
 
 function fetchBranchesStep() {
   var branchArgs = branches.map(function (b) { return b + ':' + b; });
-  return spawnStep('git', ['fetch', '--no-tags', '--force', 'origin'].concat(branchArgs), sourceDir);
+  return spawnStep('git', ['fetch', '--no-tags', '--force', 'origin'].concat(branchArgs), paths.esSrc);
+}
+
+
+function findGeneratedApiFiles() {
+  var anyApiMethodDocs = /^api_methods.*\.asciidoc$/;
+  var anyApiJsFiled = /^(master|[\d_]+)\.js$/;
+  var allBranches = _.isEqual(branches, utils.branches);
+
+  if (allBranches) {
+    return [
+      dirRegex(paths.docs, anyApiMethodDocs),
+      dirRegex(paths.apiSrc, anyApiJsFiled)
+    ];
+  }
+
+  return branches.reduce(function (files, branch) {
+    var b = _.snakeCase(branch);
+
+    files.push(dirOpts(paths.docs, 'api_methods_' + b + '.asciidoc'));
+
+    var isDefault = branch === utils.branches._default;
+    if (isDefault) {
+      files.push(dirOpts(paths.docs, 'api_methods.asciidoc'));
+    }
+
+    files.push(dirOpts(paths.apiSrc, b + '.js'));
+
+    return files;
+  }, []);
+}
+
+
+function clearGeneratedFiles() {
+  var esArchives = /^_elasticsearch_(master|[\d_]+|\.tar)$/;
+  var generatedFiles = [];
+
+  if (argv.api) {
+    generatedFiles.push(findGeneratedApiFiles());
+  }
+
+  generatedFiles.push(dirRegex(paths.src, esArchives));
+
+  var rmSteps = _.chain(generatedFiles)
+  .flatten()
+  .uniq()
+  .map(function (path) {
+    return spawnStep('rm', ['-rf', path]);
+  })
+  .value();
+
+  if (!rmSteps.length) return;
+
+  return function (done) {
+    return async.series(rmSteps, done);
+  };
 }
 
 function removePrevArchive(branch) {
-  return function (done) {
-    var dir = storeDir(branch);
+  if (!argv.update) return;
 
-    if (!isDirectory(dir)) return done();
-    else spawnStep('rm', ['-rf', dir], root)(done);
-  };
+  var dir = paths.getArchiveDir(branch);
+  if (!isDirectory(dir)) return;
+
+  return spawnStep('rm', ['-rf', dir], paths.root);
 }
 
 function createArchive(branch) {
   return function (done) {
-    var dir = storeDir(branch);
-    if (isDirectory(dir)) return done();
+    var dir = paths.getArchiveDir(branch);
+    var tarball = paths.getArchiveTarball(branch);
+
+    if (isDirectory(dir)) {
+      console.log(branch + ' archive already exists');
+      return process.nextTick(done);
+    }
+
     async.series([
-      spawnStep('mkdir', [dir], root),
-      execStep('git archive --format tar ' + branch + ' rest-api-spec | tar -x -C ' + dir, sourceDir)
+      spawnStep('mkdir', [dir], paths.root),
+      spawnStep('git', ['archive', '--format', 'tar', '--output', tarball, branch, 'rest-api-spec'], paths.esSrc),
+      spawnStep('tar', ['-x', '-f', tarball, '-C', dir]),
+      spawnStep('rm',  [tarball])
     ], done);
   };
 }
@@ -122,27 +210,36 @@ function generateStep(branch) {
     async.parallel([
       argv.api && async.apply(require('./js_api'), branch),
       argv.tests && async.apply(require('./yaml_tests'), branch)
-    ].filter(Boolean), function () {
-      console.log('----\n');
-      done();
-    });
+    ].filter(Boolean), done);
   };
 }
 
 var steps = [
   initStep(),
+  clearGeneratedFiles(),
   fetchBranchesStep()
-];
+].filter(Boolean);
+
 branches.forEach(function (branch) {
-  if (argv.update) steps.push(removePrevArchive(branch));
-  steps.push(
+  steps.push(_.partial(async.series, [
+    removePrevArchive(branch),
     createArchive(branch),
     generateStep(branch)
-  );
+  ].filter(Boolean)));
 });
 
-async.series(steps, function (err) {
-  if (err) {
-    throw err;
+async.series(
+  steps.map(function (step) {
+    return function (done) {
+      step(function (err) {
+        console.log('----\n');
+        done(err);
+      });
+    };
+  }),
+  function (err) {
+    if (err) {
+      throw err;
+    }
   }
-});
+);

scripts/generate/js_api.js

@@ -152,6 +152,7 @@ module.exports = function (branch, done) {
       'name'
     );
     docVars.branch = branch;
+    docVars.branchIsDefault = branch === utils.branches._default;
     docVars.branchSuffix = branchSuffix.replace(/_/g, '-');
     done();
   }

scripts/generate/templates/api_methods.tmpl

@@ -1,12 +1,11 @@
 [[api-reference<%= branchSuffix %>]]
 == <%= branch %> API
 
-<% if (branchSuffix) { %>
-NOTE: At this time, you must opt into the <%= branch %> API by setting the `apiVersion` config parameter.
-<% } else { %>
+<% if (branchIsDefault) { %>
 NOTE: This is currently the default API, but in upcomming versions that will change. We recommend setting the `apiVersion` config param when you instantiate your client to make sure that the API does not change unexpectedly.
-<%
-}
+<% } else { %>
+NOTE: At this time, you must opt into the <%= branch %> API by setting the `apiVersion` config parameter.
+<% }
 
 // method index%>
 [[js-api-method-index<%= branchSuffix %>]]<%