moved away from submodules

.gitignore

@@ -7,11 +7,10 @@ test/integration/yaml_suite/log
 .idea
 
 ## generated files
-scripts/last_rest_spec_update.sha
-test/integration/browser_yaml_suite/yaml_tests.js
 test/integration/yaml_suite/yaml_tests*.json
+src/elasticsearch/
+src/elasticsearch_0_90/
 junit-*.xml
-test.log
 elasticsearch*.log
 sauce_connect.lo*
 coverage.html

grunt/browser_clients.js

@@ -29,7 +29,7 @@ module.exports = function (grunt) {
     '_check_for_confirmation',
     '_upload_archive:release',
     'browser_clients:build',
-    'run:init_submodules',
+    'run:clone_bower_repo',
     'copy:dist_to_bower',
     'run:release_bower_subm_tag'
   ]);

grunt/config/run.js

@@ -34,13 +34,7 @@ module.exports = {
     exec: './.snapshots/master_nightly/bin/elasticsearch ' + esOpts,
     options: {
       wait: false,
-      quiet: true,
+      quiet: true
-      onClose: function () {
-
-      },
-      onReady: function () {
-
-      }
     }
   },
   'install_es_0.90': {
@@ -53,8 +47,11 @@ module.exports = {
       quiet: true
     }
   },
-  init_submodules: {
+  clone_bower_repo: {
-    exec: 'git submodule update --init',
+    exec: [
+      'test -d src/elasticsearch',
+      'git clone git@github.com:elasticsearch/bower-elasticsearch-js.git src/bower_es_js'
+    ].join(' || '),
     options: {
       quiet: true
     }

package.json

@@ -43,7 +43,6 @@
     "load-grunt-config": "~0.7.0",
     "grunt-s3": "~0.2.0-alpha.3",
     "grunt-run": "*",
-    "relative-fs": "0.0.1",
     "grunt-contrib-compress": "~0.5.3",
     "grunt-contrib-copy": "~0.4.1",
     "grunt-prompt": "~0.1.2",
@@ -53,7 +52,8 @@
     "expect.js": "~0.2.0",
     "aliasify": "~1.2.4",
     "express": "~3.4.7",
-    "grunt-saucelabs": "git://github.com/spenceralger/grunt-saucelabs.git"
+    "grunt-saucelabs": "git://github.com/spenceralger/grunt-saucelabs.git",
+    "find-root": "~0.1.1"
   },
   "license": "Apache 2.0",
   "dependencies": {
@@ -72,6 +72,6 @@
     "generate": "node scripts/generate"
   },
   "engines": {
-    "node" : ">=0.8 <0.11"
+    "node": ">=0.8 <0.11"
   }
 }

scripts/_spawn.js

@@ -33,7 +33,7 @@ function spawn(cmd, args, opts, cb) {
 
   out
     .pipe(estream.mapSync(function indent(line) {
-      return '    ' + line + '\n';
+      return line ? '    ' + line + '\n' : '';
     }))
     .pipe(process.stdout);
 

scripts/generate/index.js

@@ -1,12 +1,8 @@
 var async = require('async');
+var fs = require('fs');
 var spawn = require('../_spawn');
 var argv = require('optimist')
   .options({
-    force: {
-      alias: 'f',
-      default: false,
-      boolean: true
-    },
     verbose: {
       alias: 'v',
       default: false,
@@ -26,8 +22,11 @@ var argv = require('optimist')
     }
   });
 
-var root = require('path').join(__dirname, '../..');
+var path = require('path');
-var esSubModule = root + '/src/elasticsearch';
+var root = require('find-root')(__dirname);
+var fromRoot = path.join.bind(path, root);
+var _ = require(fromRoot('src/lib/utils'));
+var esUrl = 'https://github.com/elasticsearch/elasticsearch.git';
 
 if (process.env.npm_config_argv) {
   // when called by NPM
@@ -37,55 +36,75 @@ if (process.env.npm_config_argv) {
   argv = argv.argv;
 }
 
-if (!argv.force && process.env.FORCE || process.env.FORCE_GEN) {
+function isDirectory(dir) {
-  argv.force = argv.f = process.env.FORCE || process.env.FORCE_GEN;
+  var stat;
+  try { stat = fs.statSync(dir); } catch (e) {}
+  return (stat && stat.isDirectory());
 }
 
-function makeSpawn(cmd, args, cwd) {
+function storeDir(branch) {
+  var suffix = branch === 'master' ? '' : '_' + _.snakeCase(branch);
+  return fromRoot('src/elasticsearch' + suffix);
+}
+
+function spawnStep(cmd, args, cwd) {
   return function (done) {
     spawn(cmd, args, {
       verbose: argv.versbose,
-      cwd: cwd || esSubModule
+      cwd: cwd
     }, function (status) {
       done(status ? new Error('Non-zero exit code: %d', status) : void 0);
     });
   };
 }
 
-function generateBranch(branch, done) {
+function checkoutStep(branch) {
+  return function (done) {
+    var dir = storeDir(branch);
+
+    if (isDirectory(dir)) {
+      return done();
+    }
+
+    spawnStep('git', [
+      'clone', '--depth', '50', '--branch', branch, '--', esUrl, dir
+    ], root)(done);
+  };
+}
+
+function updateStep(branch) {
+  return function (done) {
+    if (!argv.update) {
+      return done();
+    }
+
+    var dir = storeDir(branch);
+
     async.series([
-    makeSpawn('git', ['reset', '--hard']),
+      spawnStep('git', ['fetch', 'origin', branch], dir),
-    makeSpawn('git', ['clean', '-fdx']),
+      spawnStep('git', ['reset', '--hard', 'origin/' + branch], dir),
-    makeSpawn('git', ['checkout', 'origin/' + branch]),
+      spawnStep('git', ['clean', '-fdx'], dir)
-    function (done) {
-      var tasks = [];
-
-      if (argv.api) {
-        tasks.push(
-          async.apply(require('./js_api'), branch)
-        );
-      }
-      if (argv.tests) {
-        tasks.push(
-          async.apply(require('./yaml_tests'), branch)
-        );
-      }
-
-      async.parallel(tasks, done);
-    }
     ], done);
+  };
 }
 
-var steps = [
+function generateStep(branch) {
-  makeSpawn('git', ['submodule', 'update', '--', esSubModule], root)
+  return function (done) {
-];
+    async.parallel([
-if (argv.update) {
+      argv.api && async.apply(require('./js_api'), branch),
-  steps.push(makeSpawn('git', ['fetch', 'origin'], esSubModule));
+      argv.tests && async.apply(require('./yaml_tests'), branch)
+    ].filter(Boolean), done);
+  };
 }
-steps.push(
+
-  async.apply(generateBranch, '0.90'),
+var steps = [];
-  async.apply(generateBranch, 'master')
+['master', '0.90'].forEach(function (branch) {
-);
+  steps.push(
+    checkoutStep(branch),
+    updateStep(branch),
+    generateStep(branch)
+  );
+});
 
 async.series(steps, function (err) {
   if (err) {

scripts/generate/js_api.js

@@ -4,10 +4,11 @@ module.exports = function (branch, done) {
    * @type {[type]}
    */
   var _ = require('../../src/lib/utils');
-  var fs = require('relative-fs').relativeTo(__dirname);
+  var fs = require('fs');
   var async = require('async');
   var chalk = require('chalk');
   var path = require('path');
+  var fromRoot = path.join.bind(path, require('find-root')(__dirname));
   var templates = require('./templates');
   var urlParamRE = /\{(\w+)\}/g;
 
@@ -16,6 +17,7 @@ module.exports = function (branch, done) {
   var docVars; // slightly modified clone of apiSpec for the docs
 
   var branchSuffix = branch === 'master' ? '' : '_' + _.snakeCase(branch);
+  var esDir = fromRoot('src/elasticsearch' + branchSuffix);
   var aliases = require('./aliases' + branchSuffix);
 
   // generate the API
@@ -32,7 +34,7 @@ module.exports = function (branch, done) {
   });
 
   function readSpecFiles(done) {
-    var apiDir = require('path').join(__dirname, '../../src/elasticsearch/rest-api-spec/api/');
+    var apiDir = path.join(esDir, 'rest-api-spec/api/');
     files = fs.readdirSync(apiDir).map(function (filename) {
       var module = require(apiDir + filename);
       delete require.cache[apiDir + filename];
@@ -87,14 +89,14 @@ module.exports = function (branch, done) {
   }
 
   function writeApiFile(done) {
-    var outputPath = require('path').join(__dirname, '../../src/lib/api' + branchSuffix + '.js');
+    var outputPath = fromRoot('src/lib/api' + branchSuffix + '.js');
     fs.writeFileSync(outputPath, templates.apiFile(apiSpec));
     console.log(chalk.white.bold('wrote'), apiSpec.actions.length, 'api actions to', outputPath);
     done();
   }
 
   function ensureDocsDir(done) {
-    fs.stat('../../docs', function (err, stat) {
+    fs.stat(fromRoot('docs'), function (err, stat) {
       if (err) {
         if (err.message.match(/enoent/i)) {
           fs.mkdir('../../docs', done);
@@ -124,7 +126,7 @@ module.exports = function (branch, done) {
   }
 
   function writeMethodDocs(done) {
-    var filename = path.resolve(__dirname, '../../docs/api_methods' + branchSuffix + '.asciidoc');
+    var filename = fromRoot('docs/api_methods' + branchSuffix + '.asciidoc');
     fs.writeFile(
       filename,
       templates.apiMethods(docVars),

scripts/generate/yaml_tests.js

@@ -3,14 +3,16 @@ module.exports = function (branch, done) {
    * Creates a JSON version of the YAML test suite that can be simply bundled for use in the browser.
    */
   var jsYaml = require('js-yaml');
-  var fs = require('relative-fs').relativeTo(__dirname);
+  var fs = require('fs');
   var async = require('async');
   var chalk = require('chalk');
-  var _ = require('../../src/lib/utils');
   var path = require('path');
+  var fromRoot = path.join.bind(path, require('find-root')(__dirname));
+  var _ = require(fromRoot('src/lib/utils'));
   var tests = {}; // populated in readYamlTests
 
   var branchSuffix = branch === 'master' ? '' : '_' + _.snakeCase(branch);
+  var esDir = fromRoot('src/elasticsearch' + branchSuffix);
 
   // generate the yaml tests
   async.series([
@@ -19,7 +21,7 @@ module.exports = function (branch, done) {
   ], done);
 
   function readYamlTests(done) {
-    var testDir = path.join(__dirname, '../../src/elasticsearch/rest-api-spec/test/');
+    var testDir = path.join(esDir, 'rest-api-spec/test/');
 
     function readDirectories(dir) {
       fs.readdirSync(dir).forEach(function (filename) {
@@ -41,7 +43,7 @@ module.exports = function (branch, done) {
   }
 
   function writeYamlTests(done) {
-    var testFile = require('path').resolve(__dirname, '../../test/integration/yaml_suite/yaml_tests' + branchSuffix + '.json');
+    var testFile = fromRoot('test/integration/yaml_suite/yaml_tests' + branchSuffix + '.json');
     fs.writeFileSync(testFile, JSON.stringify(tests, null, ' '), 'utf8');
     console.log(chalk.white.bold('wrote') + ' YAML tests as JSON to', testFile);
     done();

test/integration/yaml_suite/client_manager.js

@@ -8,9 +8,12 @@ if (BROWSER) {
 } else {
   var es = require('../../../src/elasticsearch');
 }
-var argv = require('./argv');
+
-var fs = require('relative-fs').relativeTo(require('path').join(__dirname, '../../../'));
 var _ = require('../../../src/lib/utils');
+var argv = require('./argv');
+var path = require('path');
+var fs = require('fs');
+var fromRoot = _.bindKey(path, 'join', require('find-root')(__dirname));
 
 // current client
 var client = null;
@@ -71,7 +74,7 @@ module.exports = {
 
       if (logConfig && logConfig.type === 'tracer') {
         try {
-          fs.unlinkSync('../../../elasticsearch-tracer.log');
+          fs.unlinkSync(fromRoot('elasticsearch-tracer.log'));
         } catch (e) {}
       }