From 6ac998d55074571fb601b623ec0ef3552feea905 Mon Sep 17 00:00:00 2001 From: Spencer Alger Date: Mon, 5 Jan 2015 16:08:22 -0700 Subject: [PATCH] Revert "save" This reverts commit 3e66652287593c84ae64b82db4e27d80823dcf9b. --- grunt/config/esvm.js | 49 ------------- grunt/config/run.js | 38 ++++++++++ grunt/tasks.js | 8 +- package.json | 3 +- scripts/_utils.sh | 171 +++++++++++++++++++++++++++++++++++++++++++ scripts/ci.js | 121 ------------------------------ scripts/ci.sh | 84 +++++++++++---------- scripts/es.sh | 26 +++++++ scripts/jenkins.sh | 1 + 9 files changed, 282 insertions(+), 219 deletions(-) delete mode 100644 grunt/config/esvm.js create mode 100644 scripts/_utils.sh delete mode 100644 scripts/ci.js create mode 100755 scripts/es.sh diff --git a/grunt/config/esvm.js b/grunt/config/esvm.js deleted file mode 100644 index 5e14e2be7..000000000 --- a/grunt/config/esvm.js +++ /dev/null @@ -1,49 +0,0 @@ -var utils = require('../utils'); -var _ = require('lodash-node'); - -exports.options = { - nodes: 1, - config: { - 'node.name': 'elasticsearch_js_test_runner', - 'cluster.name': 'elasticsearch_js_test_runners', - 'http.port': 9400, - 'network.host': 'localhost', - 'discovery.zen.ping_timeout': 1, - 'discovery.zen.ping.multicast.enabled': false, - 'logger.level': 'ERROR', - } -}; - -// targets for each branch -utils.branches.forEach(function (branch) { - exports[branch] = { - options: { - branch: branch - } - }; - - switch (branch) { - case '0.90': - case '1.0': - case '1.1': - // no special treatment - break; - default: - exports[branch].options.config = _.merge({ - 'node.bench': true, - 'script.disable_dynamic': false - }, exports.options.config); - - break; - } -}); - -// ci target, based on env variables -var ciVersion = process.env.ES_RELEASE; -var ciBranch = process.env.TESTING_BRANCH; -exports.ci_env = { - options: { - version: ciVersion, - branch: !ciVersion && ciBranch, - } -}; \ No newline at end of file diff --git a/grunt/config/run.js b/grunt/config/run.js index a9f521974..92dc3afa3 100644 --- a/grunt/config/run.js +++ b/grunt/config/run.js @@ -1,3 +1,13 @@ +var esOpts = [ + '-D es.http.port=9400', + '-D es.network.host=localhost', + '-D es.cluster.name=elasticsearch_js_test_runners', + '-D es.node.name=elasticsearch_js_test_runner', + '-D es.discovery.zen.ping.multicast.enabled=false', + '-D es.discovery.zen.ping_timeout=1', + '-D es.logger.level=ERROR', +]; + var utils = require('../utils'); var config = { @@ -53,9 +63,37 @@ var config = { }; utils.branches.forEach(function (branch) { + config['generate_' + branch] = { exec: 'node ./scripts/generate/index.js --branch=' + branch }; + + config['install_es_' + branch] = { + exec: './scripts/es.sh install ' + branch, + }; + + var args = esOpts.slice(0); + + switch (branch) { + case '0.90': + args.push('-f'); + break; + case '1.0': + case '1.1': + // no special treatment + break; + default: + args.push('-Des.node.bench=true', '-Des.script.disable_dynamic=false'); + break; + } + + config['es_' + branch] = { + exec: './.snapshots/' + branch + '_nightly/bin/elasticsearch ' + args.join(' '), + options: { + wait: false, + quiet: true + } + }; }); module.exports = config; \ No newline at end of file diff --git a/grunt/tasks.js b/grunt/tasks.js index d44fa612a..10ae320a8 100644 --- a/grunt/tasks.js +++ b/grunt/tasks.js @@ -21,9 +21,10 @@ module.exports = function (grunt) { branches.forEach(function (branch) { tasks.push( - 'esvm:' + branch, + 'run:install_es_' + branch, + 'run:es_' + branch, 'mochacov:integration_' + branch, - 'esvm_shutdown:' + branch + 'stop:es_' + branch ); }); @@ -71,7 +72,6 @@ module.exports = function (grunt) { writeFile(browserBuildsPath, browserBuilds), writeFile(packagePath, JSON.stringify(pkg, null, ' ')) ]); - }) - .nodeify(this.async()); + }).nodeify(this.async()); }); }; \ No newline at end of file diff --git a/package.json b/package.json index c364cec3a..b2c0a4f64 100644 --- a/package.json +++ b/package.json @@ -52,7 +52,6 @@ "grunt-contrib-jshint": "spenceralger/grunt-contrib-jshint", "grunt-contrib-uglify": "~0.2.7", "grunt-contrib-watch": "~0.5.3", - "grunt-esvm": "~0.2.1", "grunt-mocha-cov": "~0.2.0", "grunt-open": "~0.2.2", "grunt-prompt": "~0.1.2", @@ -92,4 +91,4 @@ "engines": { "node": ">=0.8 <0.11" } -} +} \ No newline at end of file diff --git a/scripts/_utils.sh b/scripts/_utils.sh new file mode 100644 index 000000000..0101223ba --- /dev/null +++ b/scripts/_utils.sh @@ -0,0 +1,171 @@ +#!/usr/bin/env bash + +##### +# Start a group of log output +##### +function group { + style='\x1b[1m\x1b[37m\x1b[4m' + reset='\x1b[24m\x1b[39m\x1b[22m' + + echo -en "\n\n${style}${1}${reset}\n" +} + +##### +# Do, log, and check a call +##### +function call { + local DO="$*" + echo "\$ ${DO}" + echo "$DO" | bash + local RESULT=$? + if [ "$RESULT" -gt "0" ]; then + echo "non-zero exit code: $RESULT" + exit $RESULT + fi +} + +function ensure_grunt { + if [[ ! -x "$(which grunt)" ]]; then + group "installing grunt" + call npm install --silent -g grunt-cli + fi +} + +##### +# call grunt, but make sure it's installed first +##### +function _grunt { + ensure_grunt + call grunt "$*" +} + +##### +# Download a version of ES and get it running +# @arg ES_BRANCH - The branch to run off of +# @arg ES_RELEASE - The specific release to run, overrides ES_BRANCH +##### +function manage_es { + local DO=$1 + local ES_BRANCH=$2 + local ES_RELEASE=$3 + + local ROOT="$PWD" + local SNAPSHOTS="$ROOT/.snapshots" + local PIDS="$ROOT/.snapshots/pids" + + group "${DO}ing es" + + if [ ! -d "$PIDS" ]; then + call mkdir -p "$PIDS" + fi + + if [ -n "$ES_RELEASE" ]; then + local ES_VERSION="v${ES_RELEASE}" + local ES_URL="https://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-${ES_RELEASE}.zip" + local ES_DIR="${SNAPSHOTS}/${ES_VERSION}" + else + local BUCKET='s3-us-west-2.amazonaws.com/build.elasticsearch.org' + + # TODO: TRASH THIS + if [[ $ES_BRANCH == "1.x" ]]; then local JDK='JDK7' + elif [[ $ES_BRANCH == "1.2" ]]; then local JDK='JDK7' + elif [[ $ES_BRANCH == "1.3" ]]; then local JDK='JDK7' + elif [[ $ES_BRANCH == "1.4" || $ES_BRANCH == "master" ]]; then + local JDK='JDK7' + local BUCKET='s3-eu-west-1.amazonaws.com/build-eu.elasticsearch.org' + else local JDK='JDK6' + fi + + local ES_VERSION="${ES_BRANCH}_nightly" + local ES_URL="http://$BUCKET/origin/$ES_BRANCH/nightly/$JDK/elasticsearch-latest-SNAPSHOT.zip" + local DATE="$(date +%Y_%m_%d)" + local ES_DIR="${SNAPSHOTS}/${ES_VERSION}_${DATE}" + fi + + local ES_BIN="$ES_DIR/bin/elasticsearch" + local PIDFILE="$ROOT/.snapshots/pids/$ES_VERSION" + + + case "$DO" in + reinstall) + if [ -x "$ES_BIN" ]; then + echo "removing $ES_VERSION" + rm -rf "${SNAPSHOTS}/${ES_VERSION}*" + fi + manage_es install "$ES_BRANCH" "$ES_RELEASE" + ;; + install) + if [ ! -x "$ES_BIN" ]; then + echo "Downloading Elasticsearch $ES_VERSION" + rm -rf "${SNAPSHOTS}/${ES_VERSION}*" + call curl --silent -O "$ES_URL" + unzip -q elasticsearch-*.zip + rm elasticsearch-*.zip + mv elasticsearch-*/ "$ES_DIR" + if [ -z "$ES_RELEASE" ]; then + ln -sf "$ES_DIR" "${SNAPSHOTS}/${ES_VERSION}" + fi + else + echo "$ES_VERSION installed" + fi + ;; + start) + # ensure that only one version is running at a time so that we can precisely kill them + if [ -f "$PIDFILE" ]; then + local PID="$(cat "$PIDFILE")" + kill -0 "$PID" + local RUNNING=$? + + if [ $RUNNING -eq 0 ]; then + echo "Already running $ES_VERSION" + return 1 + else + echo "PID file was left behind by ES" + rm "$PIDFILE" + fi + fi + + manage_es install "$ES_BRANCH" "$ES_RELEASE" + + if [ ! -x "$ES_BIN" ]; then + echo "Unable to find elasticsearch executable" + return 1 + fi + + local ES_OPTS="-p $PIDFILE -D es.http.port=9400 -D es.network.host=localhost -D es.cluster.name=elasticsearch_js_test_runners -D es.node.name=elasticsearch_js_test_runner -D es.discovery.zen.ping.multicast.enabled=false -D es.discovery.zen.ping_timeout=1 -D es.logger.level=ERROR" + + if [ -n "$ES_NODE_NAME" ]; then + ES_OPTS="$ES_OPTS -Des.node.name=$ES_NODE_NAME" + fi + + if [[ $ES_BRANCH != "0.90" ]]; then + # explicitly run as deamon + ES_OPTS="-d $ES_OPTS" + fi + + if [[ $ES_BRANCH != "1.0" && $ES_BRANCH != "1.1" ]]; then + # enable scripting and benchmarks + ES_OPTS="$ES_OPTS -D es.node.bench=true -D es.script.disable_dynamic=false" + fi + + call "$ES_BIN" "$ES_OPTS" + ;; + stop) + if [ -e "$PIDFILE" ]; then + local PID="$(cat "$PIDFILE")" + kill -0 "$PID" + local RUNNING=$? + + if [ $RUNNING -eq 0 ]; then + kill "$PID" + echo "Elasticsearch $ES_VERSION stopped" + return 0 + fi + + rm "$PIDFILE" + fi + echo "Elasticsearch $ES_VERSION is not running." + return 1 + ;; + esac +} \ No newline at end of file diff --git a/scripts/ci.js b/scripts/ci.js deleted file mode 100644 index 9b64180f9..000000000 --- a/scripts/ci.js +++ /dev/null @@ -1,121 +0,0 @@ -#!/usr/bin/env node - -/** - * Run the tests, and setup es if needed - * - * ENV VARS: - * ES_BRANCH - the ES branch we should use to generate the tests and download es - * ES_RELEASE - a specific ES release to download in use for testing - * NODE_UNIT=1 - 0/1 run the unit tests in node - * NODE_INTEGRATION=1 - 0/1 run the integration tests in node - * BROWSER_UNIT - the browser to test in using, sauce labs. One of 'ie', 'firefox', 'chrome' - * COVERAGE - 0/1 check for coverage and ship it to coveralls - *******/ - -var Promise = require('bluebird'); -var _ = require('lodash-node'); -var join = require('path').join; -var fs = require('fs'); -var child_process = require('child_process'); - -var ROOT = join(__dirname, '..'); -var GRUNT = join(ROOT, './node_modules/.bin/grunt'); -var MOCHA = join(ROOT, './node_modules/.bin/mocha'); -var BRANCH = process.env.ES_BRANCH || 'master'; -var MOCHA_REPORTER = 'test/utils/jenkins-reporter.js'; - -var JENKINS = !!process.env.JENKINS; -var NODE_UNIT = process.env.NODE_UNIT !== '0'; -var NODE_INTEGRATION = process.env.NODE_UNIT !== '0'; -var BROWSER_UNIT = process.env.NODE_UNIT === '1'; -var COVERAGE = process.env.NODE_UNIT === '1'; - -function spawn(file, args, opts, block) { - return new Promise(function (resolve, reject) { - var cp = child_process.spawn(GRUNT, args, _.defaults(opts || {}, { - cwd: ROOT, - env: process.env, - stdio: 'inherit' - })); - - block && block(cp); - - cp.on('exit', function (code) { - if (code > 1) { - reject(new Error('non-zero exit code: ' + code)); - } else { - resolve(); - } - }); - }); -} - -function node(/*args... */) { - return spawn('node', _.rest(arguments)); -} - -function grunt(/* args... */) { - return spawn(GRUNT, _.rest(arguments)); -} - -function mocha(report/*, args... */) { - return spawn(MOCHA, _.rest(arguments, 1), { stdio: [0, 1, 'pipe'] }, function (cp) { - cp.stderr.pipe(fs.createWriteStream(report)); - }); -} - -var chain = Promise.resolve(); - -if (NODE_UNIT && !JENKINS) { - chain = chain.then(function () { - return grunt('jshint', 'mochacov:unit'); - }); -} - - -if (NODE_UNIT && JENKINS) { - chain = chain.then(function () { - var report = join(ROOT, 'test/junit-node-unit.xml'); - var tests = join(ROOT, 'test/unit/index.js'); - - return mocha(report, tests, '--reporter', join(ROOT, MOCHA_REPORTER)); - }); -} - -if (NODE_INTEGRATION) { - chain = chain.then(function () { - return node('scripts/generate.js', '--no-api', '--branch', BRANCH); - }); -} - -if (NODE_INTEGRATION && !JENKINS) { - chain = chain.then(function () { - grunt('esvm:ci_env', 'mochacov:integration_' + BRANCH, 'esvm_shutdown:ci_env'); - }); -} - -if (NODE_INTEGRATION && JENKINS) { - chain = chain.then(function () { - var branchSuffix = '_' + BRANCH.replace(/\./g, '_'); - var tests = 'test/integration/yaml_suite/index' + branchSuffix + '.js'; - var esPort = process.env.es_port || 9200; - var report = 'test/junit-node-integration.xml'; - - return mocha(report, tests, '--host', 'localhost', '--port', esPort, '--reporter', MOCHA_REPORTER); - }); -} - -if (BROWSER_UNIT) { - chain = chain.then(function () { - return grunt('browser_clients:build', 'run:browser_test_server', 'saucelabs-mocha'); - }); -} - -if (COVERAGE) { - chain = chain.then(function () { - return grunt('mochacov:ship_coverage'); - }) - .catch(function () { - console.log('FAILED TO SHIP COVERAGE! but that\'s normal'); - }); -} \ No newline at end of file diff --git a/scripts/ci.sh b/scripts/ci.sh index 45dc341f6..b19e00faf 100755 --- a/scripts/ci.sh +++ b/scripts/ci.sh @@ -13,22 +13,13 @@ # ########### +export ES_NODE_NAME="elasticsearch_js_test_runner" + +HERE="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" MOCHA="./node_modules/.bin/mocha" MOCHA_REPORTER="../../../test/utils/jenkins-reporter.js" -# execute a command, and exit if it fails -function crit { - $* - CODE=$? - if [[ $CODE -gt 0 ]]; then - echo "last command was critical, but it reported non-zero exit code $CODE"; - exit; - fi -} - -if [[ "$(which grunt)" == "" ]]; then - crit npm install -g grunt -fi +source $HERE/_utils.sh # normalize ES_BRANCH into TESTING_BRANCH if [[ -n "$ES_BRANCH" ]]; then @@ -38,48 +29,55 @@ else fi if [[ "$NODE_UNIT" != "0" ]]; then - if [[ -n "$JENKINS" ]]; then - $MOCHA test/unit/index.js --reporter $MOCHA_REPORTER 2> test/junit-node-unit.xml - if [ "$?" -gt "0" ]; then - echo "non-zero exit code: $RESULT" - cat test/junit-node-unit.xml + group "running unit tests" + if [[ -n "$JENKINS" ]]; then + $MOCHA test/unit/index.js --reporter $MOCHA_REPORTER 2> test/junit-node-unit.xml + if [ "$?" -gt "0" ]; then + echo "non-zero exit code: $RESULT" + cat test/junit-node-unit.xml + fi + else + _grunt jshint mochacov:unit fi - else - crit grunt jshint mochacov:unit - fi fi if [[ "$NODE_INTEGRATION" != "0" ]]; then - crit node scripts/generate --no-api --branch $TESTING_BRANCH + group "generating tests" + call node scripts/generate --no-api --branch $TESTING_BRANCH - if [[ -n "$JENKINS" ]]; then - # convert TESTING_BRANCH into BRANCH_SUFFIX - BRANCH_SUFFIX="_${TESTING_BRANCH//./_}" + group "running integration tests" + if [[ -n "$JENKINS" ]]; then + # convert TESTING_BRANCH into BRANCH_SUFFIX + BRANCH_SUFFIX="_${TESTING_BRANCH//./_}" - # find value of ES_PORT - if [[ -n "$es_port" ]]; then - # jenkins - ES_PORT=$es_port + # find value of ES_PORT + if [[ -n "$es_port" ]]; then + # jenkins + ES_PORT=$es_port + else + ES_PORT=9200 + fi + + FILES=test/integration/yaml_suite/index${BRANCH_SUFFIX}.js + $MOCHA $FILES --host localhost --port $ES_PORT --reporter $MOCHA_REPORTER 2> test/junit-node-integration.xml + if [ "$?" -gt "0" ]; then + echo "non-zero exit code: $RESULT" + cat test/junit-node-unit.xml + fi else - ES_PORT=9200 + manage_es start $TESTING_BRANCH $ES_RELEASE + _grunt mochacov:integration_$TESTING_BRANCH + manage_es stop $TESTING_BRANCH $ES_RELEASE fi - - FILES=test/integration/yaml_suite/index${BRANCH_SUFFIX}.js - $MOCHA $FILES --host localhost --port $ES_PORT --reporter $MOCHA_REPORTER 2> test/junit-node-integration.xml - if [ "$?" -gt "0" ]; then - echo "non-zero exit code: $RESULT" - cat test/junit-node-unit.xml - fi - else - crit grunt esvm:ci_env "mochacov:integration_${TESTING_BRANCH}" esvm_shutdown:ci_env - fi fi if [[ "$BROWSER_UNIT" == "1" ]]; then - crit grunt browser_clients:build run:browser_test_server saucelabs-mocha + group "running browser tests" + _grunt browser_clients:build run:browser_test_server saucelabs-mocha fi if [[ "$COVERAGE" == "1" ]]; then - # don't fail even if this does - grunt --force mochacov:ship_coverage + group "shipping coverage" + # don't fail even if this does + _grunt --force mochacov:ship_coverage fi diff --git a/scripts/es.sh b/scripts/es.sh new file mode 100755 index 000000000..602cff05a --- /dev/null +++ b/scripts/es.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash + +if [ -z "$2" ]; then + echo "Usage: +From the root of the elasticsearch-js project call: + +Start nightly: + ./scripts/es.sh start master + +Stop 0.90 branch: + ./scripts/es.sh stop 0.90 + +Start relase version 0.90.7: + ./scripts/es.sh start 0.90 0.90.7 +" + exit 1 +fi + +source scripts/_utils.sh + +if [[ -z "$ES_NODE_NAME" ]]; then + export ES_NODE_NAME="elasticsearch_js_test_runner" +fi + +manage_es $* +exit $? \ No newline at end of file diff --git a/scripts/jenkins.sh b/scripts/jenkins.sh index 180c3e84f..1ec203b15 100755 --- a/scripts/jenkins.sh +++ b/scripts/jenkins.sh @@ -1,5 +1,6 @@ #!/usr/bin/env bash +export VERBOSE="true" export JENKINS="true" HERE="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"