save

grunt/config/esvm.js

@@ -0,0 +1,49 @@
+var utils = require('../utils');
+var _ = require('lodash-node');
+
+exports.options = {
+  nodes: 1,
+  config: {
+    'node.name': 'elasticsearch_js_test_runner',
+    'cluster.name': 'elasticsearch_js_test_runners',
+    'http.port': 9400,
+    'network.host': 'localhost',
+    'discovery.zen.ping_timeout': 1,
+    'discovery.zen.ping.multicast.enabled': false,
+    'logger.level': 'ERROR',
+  }
+};
+
+// targets for each branch
+utils.branches.forEach(function (branch) {
+  exports[branch] = {
+    options: {
+      branch: branch
+    }
+  };
+
+  switch (branch) {
+  case '0.90':
+  case '1.0':
+  case '1.1':
+    // no special treatment
+    break;
+  default:
+    exports[branch].options.config = _.merge({
+      'node.bench': true,
+      'script.disable_dynamic': false
+    }, exports.options.config);
+
+    break;
+  }
+});
+
+// ci target, based on env variables
+var ciVersion = process.env.ES_RELEASE;
+var ciBranch = process.env.TESTING_BRANCH;
+exports.ci_env = {
+  options: {
+    version: ciVersion,
+    branch: !ciVersion && ciBranch,
+  }
+};

grunt/config/run.js

@@ -1,13 +1,3 @@
-var esOpts = [
-  '-D es.http.port=9400',
-  '-D es.network.host=localhost',
-  '-D es.cluster.name=elasticsearch_js_test_runners',
-  '-D es.node.name=elasticsearch_js_test_runner',
-  '-D es.discovery.zen.ping.multicast.enabled=false',
-  '-D es.discovery.zen.ping_timeout=1',
-  '-D es.logger.level=ERROR',
-];
-
 var utils = require('../utils');
 
 var config = {
@@ -63,37 +53,9 @@ var config = {
 };
 
 utils.branches.forEach(function (branch) {
-
   config['generate_' + branch] = {
     exec: 'node ./scripts/generate/index.js --branch=' + branch
   };
-
-  config['install_es_' + branch] = {
-    exec: './scripts/es.sh install ' + branch,
-  };
-
-  var args = esOpts.slice(0);
-
-  switch (branch) {
-  case '0.90':
-    args.push('-f');
-    break;
-  case '1.0':
-  case '1.1':
-    // no special treatment
-    break;
-  default:
-    args.push('-Des.node.bench=true', '-Des.script.disable_dynamic=false');
-    break;
-  }
-
-  config['es_' + branch] = {
-    exec: './.snapshots/' + branch + '_nightly/bin/elasticsearch ' + args.join(' '),
-    options: {
-      wait: false,
-      quiet: true
-    }
-  };
 });
 
 module.exports = config;

grunt/tasks.js

@@ -21,10 +21,9 @@ module.exports = function (grunt) {
 
     branches.forEach(function (branch) {
       tasks.push(
-        'run:install_es_' + branch,
+        'esvm:' + branch,
-        'run:es_' + branch,
         'mochacov:integration_' + branch,
-        'stop:es_' + branch
+        'esvm_shutdown:' + branch
       );
     });
 
@@ -72,6 +71,7 @@ module.exports = function (grunt) {
         writeFile(browserBuildsPath, browserBuilds),
         writeFile(packagePath, JSON.stringify(pkg, null, '  '))
       ]);
-    }).nodeify(this.async());
+    })
+    .nodeify(this.async());
   });
 };

package.json

@@ -52,6 +52,7 @@
     "grunt-contrib-jshint": "spenceralger/grunt-contrib-jshint",
     "grunt-contrib-uglify": "~0.2.7",
     "grunt-contrib-watch": "~0.5.3",
+    "grunt-esvm": "~0.2.1",
     "grunt-mocha-cov": "~0.2.0",
     "grunt-open": "~0.2.2",
     "grunt-prompt": "~0.1.2",
@@ -91,4 +92,4 @@
   "engines": {
     "node": ">=0.8 <0.11"
   }
-}
+}

scripts/_utils.sh

@@ -1,171 +0,0 @@
-#!/usr/bin/env bash
-
-#####
-# Start a group of log output
-#####
-function group {
-  style='\x1b[1m\x1b[37m\x1b[4m'
-  reset='\x1b[24m\x1b[39m\x1b[22m'
-
-  echo -en "\n\n${style}${1}${reset}\n"
-}
-
-#####
-# Do, log, and check a call
-#####
-function call {
-  local DO="$*"
-  echo "\$ ${DO}"
-  echo "$DO" | bash
-  local RESULT=$?
-  if [ "$RESULT" -gt "0" ]; then
-    echo "non-zero exit code: $RESULT"
-    exit $RESULT
-  fi
-}
-
-function ensure_grunt {
-  if [[ ! -x "$(which grunt)" ]]; then
-    group "installing grunt"
-      call npm install --silent -g grunt-cli
-  fi
-}
-
-#####
-# call grunt, but make sure it's installed first
-#####
-function _grunt {
-  ensure_grunt
-  call grunt "$*"
-}
-
-#####
-# Download a version of ES and get it running
-# @arg ES_BRANCH - The branch to run off of
-# @arg ES_RELEASE - The specific release to run, overrides ES_BRANCH
-#####
-function manage_es {
-  local DO=$1
-  local ES_BRANCH=$2
-  local ES_RELEASE=$3
-
-  local ROOT="$PWD"
-  local SNAPSHOTS="$ROOT/.snapshots"
-  local PIDS="$ROOT/.snapshots/pids"
-
-  group "${DO}ing es"
-
-    if [ ! -d "$PIDS" ]; then
-      call mkdir -p "$PIDS"
-    fi
-
-    if [ -n "$ES_RELEASE" ]; then
-      local ES_VERSION="v${ES_RELEASE}"
-      local ES_URL="https://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-${ES_RELEASE}.zip"
-      local ES_DIR="${SNAPSHOTS}/${ES_VERSION}"
-    else
-      local BUCKET='s3-us-west-2.amazonaws.com/build.elasticsearch.org'
-
-      # TODO: TRASH THIS
-      if [[ $ES_BRANCH == "1.x" ]]; then local JDK='JDK7'
-      elif [[ $ES_BRANCH == "1.2" ]]; then local JDK='JDK7'
-      elif [[ $ES_BRANCH == "1.3" ]]; then local JDK='JDK7'
-      elif [[ $ES_BRANCH == "1.4" || $ES_BRANCH == "master" ]]; then
-        local JDK='JDK7'
-        local BUCKET='s3-eu-west-1.amazonaws.com/build-eu.elasticsearch.org'
-      else local JDK='JDK6'
-      fi
-
-      local ES_VERSION="${ES_BRANCH}_nightly"
-      local ES_URL="http://$BUCKET/origin/$ES_BRANCH/nightly/$JDK/elasticsearch-latest-SNAPSHOT.zip"
-      local DATE="$(date +%Y_%m_%d)"
-      local ES_DIR="${SNAPSHOTS}/${ES_VERSION}_${DATE}"
-    fi
-
-    local ES_BIN="$ES_DIR/bin/elasticsearch"
-    local PIDFILE="$ROOT/.snapshots/pids/$ES_VERSION"
-
-
-    case "$DO" in
-      reinstall)
-        if [ -x "$ES_BIN" ]; then
-          echo "removing $ES_VERSION"
-          rm -rf "${SNAPSHOTS}/${ES_VERSION}*"
-        fi
-        manage_es install "$ES_BRANCH" "$ES_RELEASE"
-      ;;
-      install)
-        if [ ! -x "$ES_BIN" ]; then
-          echo "Downloading Elasticsearch $ES_VERSION"
-          rm -rf "${SNAPSHOTS}/${ES_VERSION}*"
-          call curl --silent -O "$ES_URL"
-          unzip -q elasticsearch-*.zip
-          rm elasticsearch-*.zip
-          mv elasticsearch-*/ "$ES_DIR"
-          if [ -z "$ES_RELEASE" ]; then
-            ln -sf "$ES_DIR" "${SNAPSHOTS}/${ES_VERSION}"
-          fi
-        else
-          echo "$ES_VERSION installed"
-        fi
-      ;;
-      start)
-        # ensure that only one version is running at a time so that we can precisely kill them
-        if [ -f "$PIDFILE" ]; then
-          local PID="$(cat "$PIDFILE")"
-          kill -0 "$PID"
-          local RUNNING=$?
-
-          if [ $RUNNING -eq 0 ]; then
-            echo "Already running $ES_VERSION"
-            return 1
-          else
-            echo "PID file was left behind by ES"
-            rm "$PIDFILE"
-          fi
-        fi
-
-        manage_es install "$ES_BRANCH" "$ES_RELEASE"
-
-        if [ ! -x "$ES_BIN" ]; then
-          echo "Unable to find elasticsearch executable"
-          return 1
-        fi
-
-        local ES_OPTS="-p $PIDFILE -D es.http.port=9400 -D es.network.host=localhost -D es.cluster.name=elasticsearch_js_test_runners -D es.node.name=elasticsearch_js_test_runner -D es.discovery.zen.ping.multicast.enabled=false -D es.discovery.zen.ping_timeout=1 -D es.logger.level=ERROR"
-
-        if [ -n "$ES_NODE_NAME" ]; then
-          ES_OPTS="$ES_OPTS -Des.node.name=$ES_NODE_NAME"
-        fi
-
-        if [[ $ES_BRANCH != "0.90" ]]; then
-          # explicitly run as deamon
-          ES_OPTS="-d $ES_OPTS"
-        fi
-
-        if [[ $ES_BRANCH != "1.0" && $ES_BRANCH != "1.1" ]]; then
-          # enable scripting and benchmarks
-          ES_OPTS="$ES_OPTS -D es.node.bench=true -D es.script.disable_dynamic=false"
-        fi
-
-        call "$ES_BIN" "$ES_OPTS"
-      ;;
-      stop)
-        if [ -e "$PIDFILE" ]; then
-          local PID="$(cat "$PIDFILE")"
-          kill -0 "$PID"
-          local RUNNING=$?
-
-          if [ $RUNNING -eq 0 ]; then
-            kill "$PID"
-            echo "Elasticsearch $ES_VERSION stopped"
-            return 0
-          fi
-
-          rm "$PIDFILE"
-        fi
-        echo "Elasticsearch $ES_VERSION is not running."
-        return 1
-      ;;
-    esac
-}

scripts/ci.js

@@ -0,0 +1,121 @@
+#!/usr/bin/env node
+
+/**
+ * Run the tests, and setup es if needed
+ *
+ * ENV VARS:
+ *  ES_BRANCH - the ES branch we should use to generate the tests and download es
+ *  ES_RELEASE - a specific ES release to download in use for testing
+ *  NODE_UNIT=1 - 0/1 run the unit tests in node
+ *  NODE_INTEGRATION=1 - 0/1 run the integration tests in node
+ *  BROWSER_UNIT - the browser to test in using, sauce labs. One of 'ie', 'firefox', 'chrome'
+ *  COVERAGE - 0/1 check for coverage and ship it to coveralls
+ *******/
+
+var Promise = require('bluebird');
+var _ = require('lodash-node');
+var join = require('path').join;
+var fs = require('fs');
+var child_process = require('child_process');
+
+var ROOT = join(__dirname, '..');
+var GRUNT = join(ROOT, './node_modules/.bin/grunt');
+var MOCHA = join(ROOT, './node_modules/.bin/mocha');
+var BRANCH = process.env.ES_BRANCH || 'master';
+var MOCHA_REPORTER = 'test/utils/jenkins-reporter.js';
+
+var JENKINS = !!process.env.JENKINS;
+var NODE_UNIT = process.env.NODE_UNIT !== '0';
+var NODE_INTEGRATION = process.env.NODE_UNIT !== '0';
+var BROWSER_UNIT = process.env.NODE_UNIT === '1';
+var COVERAGE = process.env.NODE_UNIT === '1';
+
+function spawn(file, args, opts, block) {
+  return new Promise(function (resolve, reject) {
+    var cp = child_process.spawn(GRUNT, args, _.defaults(opts || {}, {
+      cwd: ROOT,
+      env: process.env,
+      stdio: 'inherit'
+    }));
+
+    block && block(cp);
+
+    cp.on('exit', function (code) {
+      if (code > 1) {
+        reject(new Error('non-zero exit code: ' + code));
+      } else {
+        resolve();
+      }
+    });
+  });
+}
+
+function node(/*args... */) {
+  return spawn('node', _.rest(arguments));
+}
+
+function grunt(/* args... */) {
+  return spawn(GRUNT, _.rest(arguments));
+}
+
+function mocha(report/*, args... */) {
+  return spawn(MOCHA, _.rest(arguments, 1), { stdio: [0, 1, 'pipe'] }, function (cp) {
+    cp.stderr.pipe(fs.createWriteStream(report));
+  });
+}
+
+var chain = Promise.resolve();
+
+if (NODE_UNIT && !JENKINS) {
+  chain = chain.then(function () {
+    return grunt('jshint', 'mochacov:unit');
+  });
+}
+
+
+if (NODE_UNIT && JENKINS) {
+  chain = chain.then(function () {
+    var report = join(ROOT, 'test/junit-node-unit.xml');
+    var tests = join(ROOT, 'test/unit/index.js');
+
+    return mocha(report, tests, '--reporter', join(ROOT, MOCHA_REPORTER));
+  });
+}
+
+if (NODE_INTEGRATION) {
+  chain = chain.then(function () {
+    return node('scripts/generate.js', '--no-api', '--branch', BRANCH);
+  });
+}
+
+if (NODE_INTEGRATION && !JENKINS) {
+  chain = chain.then(function () {
+    grunt('esvm:ci_env', 'mochacov:integration_' + BRANCH, 'esvm_shutdown:ci_env');
+  });
+}
+
+if (NODE_INTEGRATION && JENKINS) {
+  chain = chain.then(function () {
+    var branchSuffix = '_' + BRANCH.replace(/\./g, '_');
+    var tests = 'test/integration/yaml_suite/index' + branchSuffix + '.js';
+    var esPort = process.env.es_port || 9200;
+    var report = 'test/junit-node-integration.xml';
+
+    return mocha(report, tests, '--host', 'localhost', '--port', esPort, '--reporter', MOCHA_REPORTER);
+  });
+}
+
+if (BROWSER_UNIT) {
+  chain = chain.then(function () {
+    return grunt('browser_clients:build', 'run:browser_test_server', 'saucelabs-mocha');
+  });
+}
+
+if (COVERAGE) {
+  chain = chain.then(function () {
+    return grunt('mochacov:ship_coverage');
+  })
+  .catch(function () {
+    console.log('FAILED TO SHIP COVERAGE! but that\'s normal');
+  });
+}

scripts/ci.sh

@@ -13,13 +13,22 @@
 #
 ###########
 
-export ES_NODE_NAME="elasticsearch_js_test_runner"
-
-HERE="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 MOCHA="./node_modules/.bin/mocha"
 MOCHA_REPORTER="../../../test/utils/jenkins-reporter.js"
 
-source $HERE/_utils.sh
+# execute a command, and exit if it fails
+function crit {
+  $*
+  CODE=$?
+  if [[ $CODE -gt 0 ]]; then
+    echo "last command was critical, but it reported non-zero exit code $CODE";
+    exit;
+  fi
+}
+
+if [[ "$(which grunt)" == "" ]]; then
+  crit npm install -g grunt
+fi
 
 # normalize ES_BRANCH into TESTING_BRANCH
 if [[ -n "$ES_BRANCH" ]]; then
@@ -29,55 +38,48 @@ else
 fi
 
 if [[ "$NODE_UNIT" != "0" ]]; then
-  group "running unit tests"
+  if [[ -n "$JENKINS" ]]; then
-    if [[ -n "$JENKINS" ]]; then
+    $MOCHA test/unit/index.js --reporter $MOCHA_REPORTER 2> test/junit-node-unit.xml
-      $MOCHA test/unit/index.js --reporter $MOCHA_REPORTER 2> test/junit-node-unit.xml
+    if [ "$?" -gt "0" ]; then
-      if [ "$?" -gt "0" ]; then
+      echo "non-zero exit code: $RESULT"
-        echo "non-zero exit code: $RESULT"
+      cat test/junit-node-unit.xml
-        cat test/junit-node-unit.xml
-      fi
-    else
-      _grunt jshint mochacov:unit
     fi
+  else
+    crit grunt jshint mochacov:unit
+  fi
 fi
 
 if [[ "$NODE_INTEGRATION" != "0" ]]; then
-  group "generating tests"
+  crit node scripts/generate --no-api --branch $TESTING_BRANCH
-    call node scripts/generate --no-api --branch $TESTING_BRANCH
 
-  group "running integration tests"
+  if [[ -n "$JENKINS" ]]; then
-    if [[ -n "$JENKINS" ]]; then
+    # convert TESTING_BRANCH into BRANCH_SUFFIX
-      # convert TESTING_BRANCH into BRANCH_SUFFIX
+    BRANCH_SUFFIX="_${TESTING_BRANCH//./_}"
-      BRANCH_SUFFIX="_${TESTING_BRANCH//./_}"
 
-      # find value of ES_PORT
+    # find value of ES_PORT
-      if [[ -n "$es_port" ]]; then
+    if [[ -n "$es_port" ]]; then
-        # jenkins
+      # jenkins
-        ES_PORT=$es_port
+      ES_PORT=$es_port
-      else
-        ES_PORT=9200
-      fi
-
-      FILES=test/integration/yaml_suite/index${BRANCH_SUFFIX}.js
-      $MOCHA $FILES --host localhost --port $ES_PORT --reporter $MOCHA_REPORTER 2> test/junit-node-integration.xml
-      if [ "$?" -gt "0" ]; then
-        echo "non-zero exit code: $RESULT"
-        cat test/junit-node-unit.xml
-      fi
     else
-      manage_es start $TESTING_BRANCH $ES_RELEASE
+      ES_PORT=9200
-      _grunt mochacov:integration_$TESTING_BRANCH
-      manage_es stop $TESTING_BRANCH $ES_RELEASE
     fi
+
+    FILES=test/integration/yaml_suite/index${BRANCH_SUFFIX}.js
+    $MOCHA $FILES --host localhost --port $ES_PORT --reporter $MOCHA_REPORTER 2> test/junit-node-integration.xml
+    if [ "$?" -gt "0" ]; then
+      echo "non-zero exit code: $RESULT"
+      cat test/junit-node-unit.xml
+    fi
+  else
+    crit grunt esvm:ci_env "mochacov:integration_${TESTING_BRANCH}" esvm_shutdown:ci_env
+  fi
 fi
 
 if [[ "$BROWSER_UNIT" == "1" ]]; then
-  group "running browser tests"
+  crit grunt browser_clients:build run:browser_test_server saucelabs-mocha
-  _grunt browser_clients:build run:browser_test_server saucelabs-mocha
 fi
 
 if [[ "$COVERAGE" == "1" ]]; then
-  group "shipping coverage"
+  # don't fail even if this does
-    # don't fail even if this does
+  grunt --force mochacov:ship_coverage
-    _grunt --force mochacov:ship_coverage
 fi

scripts/es.sh

@@ -1,26 +0,0 @@
-#!/usr/bin/env bash
-
-if [ -z "$2" ]; then
-  echo "Usage:
-From the root of the elasticsearch-js project call:
-
-Start nightly:
-  ./scripts/es.sh start master
-
-Stop 0.90 branch:
-  ./scripts/es.sh stop 0.90
-
-Start relase version 0.90.7:
-  ./scripts/es.sh start 0.90 0.90.7
-"
-  exit 1
-fi
-
-source scripts/_utils.sh
-
-if [[ -z "$ES_NODE_NAME" ]]; then
-  export ES_NODE_NAME="elasticsearch_js_test_runner"
-fi
-
-manage_es $*
-exit $?

scripts/jenkins.sh

@@ -1,6 +1,5 @@
 #!/usr/bin/env bash
 
-export VERBOSE="true"
 export JENKINS="true"
 HERE="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"