remove packages to get npm audit to pass (#11)
* remove packages to get `npm audit` to pass * fix grunt.config.init() call lost in split up * remove integration tests, they don't work and nobody is running them * fix upload_to_s3 task after refactor
This commit is contained in:
Spencer
@ -1 +0,0 @@
|
||||
require('backport');
|
||||
352
scripts/ci.js
352
scripts/ci.js
@ -1,352 +0,0 @@
|
||||
/**
|
||||
* Run the tests, and setup es if needed
|
||||
*
|
||||
* ENV VARS:
|
||||
* RUN - a list of task names to run, specifying this turns of all other tasks
|
||||
* ES_REF - the ES branch/tag we should use to generate the tests and download es
|
||||
* ES_RELEASE - a specific ES release to download in use for testing
|
||||
* ES_PORT - the port number we should run elasticsearch on
|
||||
* ES_HOST - the hostname elasticsearch should bind to
|
||||
* ES_V - a version identifier used by jenkins. don't use this
|
||||
*
|
||||
* Tasks:
|
||||
* NODE_UNIT - run the unit tests in node (default: true)
|
||||
* NODE_INTEGRATION - run the integration tests in node (default: true)
|
||||
* SAUCE_LABS - run the browser tests (default: false)
|
||||
* CHECK_COVERAGE - check for coverage and ship it to coveralls (default: false)
|
||||
*
|
||||
*******/
|
||||
|
||||
var Promise = require('bluebird');
|
||||
var _ = require('lodash');
|
||||
var through2 = require('through2');
|
||||
var map = require('through2-map');
|
||||
var split = require('split');
|
||||
var join = require('path').join;
|
||||
var cp = require('child_process');
|
||||
var chalk = require('chalk');
|
||||
var format = require('util').format;
|
||||
|
||||
var NL_RE = /(\r?\n)/g;
|
||||
var ROOT = join(__dirname, '..');
|
||||
var GRUNT = join(ROOT, 'node_modules', '.bin', 'grunt');
|
||||
var ENV = _.clone(process.env);
|
||||
var JENKINS = !!ENV.JENKINS_HOME;
|
||||
var TASKS = [];
|
||||
|
||||
var output; // main output stream
|
||||
var taskOut; // task output stream
|
||||
|
||||
task('NODE_UNIT', true, function() {
|
||||
if (!JENKINS) {
|
||||
return grunt('mochacov:ci_unit');
|
||||
}
|
||||
|
||||
return grunt('mochacov:jenkins_unit');
|
||||
});
|
||||
|
||||
task('NODE_INTEGRATION', true, function() {
|
||||
var branch = ENV.ES_REF;
|
||||
|
||||
return node('scripts/generate', '--no-api', '--branch', branch).then(
|
||||
function() {
|
||||
var target = (JENKINS ? 'jenkins_' : '') + 'integration:' + branch;
|
||||
return grunt('esvm:ci_env', 'mocha_' + target, 'esvm_shutdown:ci_env');
|
||||
}
|
||||
);
|
||||
});
|
||||
|
||||
task('SAUCE_LABS', false, function() {
|
||||
return new Promise(function(resolve, reject) {
|
||||
// build the clients and start the server, once the server is ready call trySaucelabs()
|
||||
var serverTasks = [
|
||||
'browser_clients:build',
|
||||
'run:browser_test_server:keepalive',
|
||||
];
|
||||
spawn(GRUNT, serverTasks, function(proc) {
|
||||
var toLines = split();
|
||||
|
||||
proc.stdout.pipe(toLines).pipe(
|
||||
through2(function(line, enc, cb) {
|
||||
cb();
|
||||
|
||||
if (String(line).indexOf('listening on port 8000') === -1) return;
|
||||
|
||||
trySaucelabs()
|
||||
.finally(function() {
|
||||
if (proc) proc.kill();
|
||||
})
|
||||
.then(resolve, reject);
|
||||
|
||||
proc.on('exit', function() {
|
||||
proc = null;
|
||||
});
|
||||
proc.stdout.unpipe(toLines);
|
||||
toLines.end();
|
||||
})
|
||||
);
|
||||
})
|
||||
// ignore server errors
|
||||
.catch(_.noop);
|
||||
|
||||
// attempt to run tests on saucelabs and retry if it fails
|
||||
var saucelabsAttempts = 0;
|
||||
function trySaucelabs() {
|
||||
saucelabsAttempts++;
|
||||
return new Promise(function(resolve, reject) {
|
||||
log(chalk.green('saucelabs attempt #', saucelabsAttempts));
|
||||
spawn(GRUNT, ['saucelabs-mocha'], function(cp) {
|
||||
var failedTests = 0;
|
||||
cp.stdout.pipe(split()).pipe(
|
||||
map(function(line) {
|
||||
failedTests += String(line).trim() === 'Passed: false' ? 1 : 0;
|
||||
})
|
||||
);
|
||||
|
||||
cp.on('error', reject);
|
||||
cp.on('exit', function(code) {
|
||||
if (code > 0) {
|
||||
if (failedTests > 0) {
|
||||
return reject(new Error('Browser tests failed'));
|
||||
}
|
||||
|
||||
if (saucelabsAttempts >= 3) {
|
||||
return reject(
|
||||
new Error(
|
||||
'Saucelabs is like really really down. Tried 3 times'
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
log(chalk.blue('trying saucelabs again...'));
|
||||
return trySaucelabs().then(resolve, reject);
|
||||
}
|
||||
|
||||
return resolve();
|
||||
});
|
||||
})
|
||||
// swallow spawn() errors, custom error handler in place
|
||||
.catch(_.noop);
|
||||
});
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
task('CHECK_COVERAGE', false, function() {
|
||||
return grunt('mochacov:ship_coverage').catch(function() {
|
||||
log("FAILED TO SHIP COVERAGE! but that's okay");
|
||||
});
|
||||
});
|
||||
|
||||
execTask('SETUP', function() {
|
||||
return Promise.try(function readVersion() {
|
||||
if (!ENV.ES_V) {
|
||||
if (ENV.ES_RELEASE) {
|
||||
return ['v' + ENV.ES_RELEASE, ENV.ES_RELEASE];
|
||||
}
|
||||
|
||||
if (ENV.ES_REF) {
|
||||
return [ENV.ES_REF, null];
|
||||
}
|
||||
}
|
||||
|
||||
var match = ENV.ES_V.match(/^(.*)_nightly$/);
|
||||
if (match) {
|
||||
return [match[1], null];
|
||||
}
|
||||
|
||||
if (/^(?:1\.\d+|0\.90)\..*$/.test(ENV.ES_V)) {
|
||||
return ['v' + ENV.ES_V, ENV.ES_V];
|
||||
}
|
||||
|
||||
throw new Error('unable to parse ES_V ' + ENV.ES_V);
|
||||
})
|
||||
.then(function readOtherConf(ver) {
|
||||
if (!ver) {
|
||||
throw new Error(
|
||||
'Unable to run the ci script without at least an ES_REF or ES_RELEASE environment var.'
|
||||
);
|
||||
}
|
||||
|
||||
log('ES_PORT:', (ENV.ES_PORT = parseInt(ENV.ES_PORT || 9400, 10)));
|
||||
log('ES_HOST:', (ENV.ES_HOST = ENV.ES_HOST || 'localhost'));
|
||||
|
||||
if (ver[0]) log('ES_REF:', (ENV.ES_REF = ver[0]));
|
||||
else delete ENV.ES_REF;
|
||||
|
||||
if (ver[1]) log('ES_RELEASE:', (ENV.ES_RELEASE = ver[1]));
|
||||
else delete ENV.ES_RELEASE;
|
||||
})
|
||||
.then(function readTasks() {
|
||||
if (!ENV.RUN) {
|
||||
return _.filter(TASKS, { default: true });
|
||||
}
|
||||
|
||||
return ENV.RUN.split(',')
|
||||
.map(function(name) {
|
||||
return _.find(TASKS, { name: name.trim() });
|
||||
})
|
||||
.filter(Boolean);
|
||||
});
|
||||
})
|
||||
.then(function(queue) {
|
||||
if (!queue.length) {
|
||||
throw new Error('no tasks to run');
|
||||
}
|
||||
|
||||
// Recursively do tasks until the queue is empty
|
||||
return (function next() {
|
||||
if (!queue.length) return;
|
||||
return execTask(queue.shift()).then(next);
|
||||
})();
|
||||
})
|
||||
.then(function() {
|
||||
logImportant(chalk.bold.green('✔︎ SUCCESS'));
|
||||
})
|
||||
.catch(function(e) {
|
||||
logImportant(chalk.bold.red('✗ FAILURE\n\n' + e.stack));
|
||||
|
||||
// override process exit code once it is ready to close
|
||||
process.once('exit', function() {
|
||||
process.exit(1);
|
||||
});
|
||||
});
|
||||
|
||||
/** ****
|
||||
* utils
|
||||
******/
|
||||
function log() {
|
||||
var chunk = format.apply(null, arguments);
|
||||
(taskOut || output || process.stdout).write(chunk + '\n');
|
||||
}
|
||||
|
||||
function logImportant(text) {
|
||||
log('\n------------');
|
||||
log(text);
|
||||
log('------------');
|
||||
}
|
||||
|
||||
function push(m) {
|
||||
return function() {
|
||||
var args = _.toArray(arguments);
|
||||
var cb = args.pop();
|
||||
this.push(m.apply(this, args));
|
||||
cb();
|
||||
};
|
||||
}
|
||||
|
||||
function indent() {
|
||||
var str = through2(
|
||||
push(function(chunk) {
|
||||
return String(chunk).replace(NL_RE, '$1 ');
|
||||
}),
|
||||
push(function() {
|
||||
return '\n';
|
||||
})
|
||||
);
|
||||
str.push(' ');
|
||||
return str;
|
||||
}
|
||||
|
||||
function task(name, def, fn) {
|
||||
if (_.isFunction(def)) {
|
||||
fn = def;
|
||||
def = true;
|
||||
}
|
||||
|
||||
TASKS.push({
|
||||
name: name,
|
||||
default: def,
|
||||
fn: fn,
|
||||
});
|
||||
}
|
||||
|
||||
function execTask(name, task) {
|
||||
if (_.isObject(name)) {
|
||||
task = name.fn;
|
||||
name = name.name;
|
||||
}
|
||||
|
||||
output = through2();
|
||||
output.pipe(
|
||||
process.stdout,
|
||||
{ end: false }
|
||||
);
|
||||
|
||||
log(chalk.white.underline(name));
|
||||
|
||||
taskOut = through2();
|
||||
taskOut.pipe(indent()).pipe(output);
|
||||
|
||||
function flushTaskOut() {
|
||||
return new Promise(function(resolve) {
|
||||
// wait for the taskOut to finish writing before continuing
|
||||
output.once('finish', function() {
|
||||
log('');
|
||||
resolve();
|
||||
});
|
||||
taskOut.end(); // will end output as well
|
||||
taskOut = output = null;
|
||||
});
|
||||
}
|
||||
|
||||
return Promise.try(task).finally(flushTaskOut);
|
||||
}
|
||||
|
||||
function spawn(file, args, block) {
|
||||
return new Promise(function(resolve, reject) {
|
||||
var proc = cp.spawn(file, args, {
|
||||
cwd: ROOT,
|
||||
env: ENV,
|
||||
stdio: [0, 'pipe', 'pipe'],
|
||||
});
|
||||
|
||||
proc.stdout.pipe(
|
||||
taskOut,
|
||||
{ end: false }
|
||||
);
|
||||
proc.stderr.pipe(
|
||||
taskOut,
|
||||
{ end: false }
|
||||
);
|
||||
|
||||
var stdout = '';
|
||||
proc.stdout.pipe(
|
||||
through2(function(chunk, enc, cb) {
|
||||
stdout += chunk;
|
||||
cb();
|
||||
})
|
||||
);
|
||||
|
||||
if (block) block(proc);
|
||||
|
||||
proc.on('exit', function(code) {
|
||||
if (code > 0) {
|
||||
reject(new Error('non-zero exit code: ' + code));
|
||||
} else {
|
||||
resolve(stdout);
|
||||
}
|
||||
});
|
||||
|
||||
proc.on('error', function(origErr) {
|
||||
reject(
|
||||
new Error(
|
||||
'Unable to execute "' +
|
||||
file +
|
||||
' ' +
|
||||
args.join(' ') +
|
||||
'": ' +
|
||||
origErr.message
|
||||
)
|
||||
);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function node(/* args... */) {
|
||||
return spawn(process.execPath, _.toArray(arguments));
|
||||
}
|
||||
|
||||
function grunt(/* args... */) {
|
||||
return spawn(GRUNT, _.toArray(arguments));
|
||||
}
|
||||
@ -12,10 +12,6 @@ var argv = require('optimist').options({
|
||||
default: true,
|
||||
boolean: true,
|
||||
},
|
||||
tests: {
|
||||
default: true,
|
||||
boolean: true,
|
||||
},
|
||||
update: {
|
||||
default: true,
|
||||
boolean: true,
|
||||
@ -262,10 +258,7 @@ function createArchive(branch) {
|
||||
function generateStep(branch) {
|
||||
return function(done) {
|
||||
async.parallel(
|
||||
[
|
||||
argv.api && async.apply(require('./js_api'), branch),
|
||||
argv.tests && async.apply(require('./yaml_tests'), branch),
|
||||
].filter(Boolean),
|
||||
[argv.api && async.apply(require('./js_api'), branch)].filter(Boolean),
|
||||
done
|
||||
);
|
||||
};
|
||||
|
||||
@ -1,60 +0,0 @@
|
||||
module.exports = function(branch, done) {
|
||||
/**
|
||||
* Creates a JSON version of the YAML test suite that can be simply bundled for use in the browser.
|
||||
*/
|
||||
var jsYaml = require('js-yaml');
|
||||
var fs = require('fs');
|
||||
var async = require('async');
|
||||
var chalk = require('chalk');
|
||||
var path = require('path');
|
||||
var fromRoot = path.join.bind(path, require('find-root')(__dirname));
|
||||
var utils = require(fromRoot('src/lib/utils'));
|
||||
var tests = {}; // populated in readYamlTests
|
||||
|
||||
var esDir = fromRoot('src/_elasticsearch_' + utils.snakeCase(branch));
|
||||
|
||||
// generate the yaml tests
|
||||
async.series([readYamlTests, writeYamlTests, writeTestIndex], done);
|
||||
|
||||
function readYamlTests(done) {
|
||||
var testDir = path.join(esDir, 'rest-api-spec/test/');
|
||||
|
||||
function readDirectories(dir) {
|
||||
fs.readdirSync(dir).forEach(function(filename) {
|
||||
var filePath = path.join(dir, filename);
|
||||
var stat = fs.statSync(filePath);
|
||||
if (stat.isDirectory()) {
|
||||
readDirectories(filePath);
|
||||
} else if (filename.match(/\.yaml$/)) {
|
||||
var file = (tests[path.relative(testDir, filePath)] = []);
|
||||
jsYaml.loadAll(fs.readFileSync(filePath, 'utf8'), function(doc) {
|
||||
file.push(doc);
|
||||
});
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
readDirectories(testDir);
|
||||
done();
|
||||
}
|
||||
|
||||
function writeYamlTests(done) {
|
||||
var testFile = fromRoot(
|
||||
'test/integration/yaml_suite/yaml_tests_' +
|
||||
utils.snakeCase(branch) +
|
||||
'.json'
|
||||
);
|
||||
fs.writeFileSync(testFile, JSON.stringify(tests, null, ' '), 'utf8');
|
||||
console.log(chalk.white.bold('wrote') + ' YAML tests as JSON to', testFile);
|
||||
done();
|
||||
}
|
||||
|
||||
function writeTestIndex(done) {
|
||||
var file = fromRoot(
|
||||
'test/integration/yaml_suite/index_' + utils.snakeCase(branch) + '.js'
|
||||
);
|
||||
fs.writeFileSync(file, "require('./run')('" + branch + "');\n", 'utf8');
|
||||
console.log(chalk.white.bold('wrote') + ' YAML index to', file);
|
||||
done();
|
||||
}
|
||||
};
|
||||
@ -1,61 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
export JAVA_HOME="/usr/lib/jvm/jdk8"
|
||||
lockpath="/var/lock/setup_nodejs"
|
||||
|
||||
# pass a file name to aquire a lock
|
||||
function get_lock {
|
||||
echo "attempting to get a lock on $lockpath"
|
||||
lockfile -5 -r 120 "$lockpath"
|
||||
if [[ $? -gt 0 ]]; then
|
||||
echo "failed to get lock file $lockpath within 10 minutes"
|
||||
exit 1;
|
||||
else
|
||||
echo "acquired lock file $lockpath"
|
||||
fi
|
||||
}
|
||||
|
||||
# clear all aquired locks
|
||||
function release_lock {
|
||||
rm -f "$lockpath"
|
||||
echo "cleared lock file $lockpath"
|
||||
}
|
||||
|
||||
# execute a command, and exit if it fails
|
||||
function crit {
|
||||
$*
|
||||
CODE=$?
|
||||
if [[ $CODE -gt 0 ]]; then
|
||||
echo "last command was critical, but it reported non-zero exit code $CODE";
|
||||
release_lock # clear any locks
|
||||
exit $CODE;
|
||||
fi
|
||||
}
|
||||
|
||||
# install a specific version of Node and the latest version of NPM within that install
|
||||
function install_node {
|
||||
local version=$1
|
||||
|
||||
if [[ $(type -t nvm) != "function" ]]; then
|
||||
source /mnt/jenkins/nvm/nvm.sh
|
||||
ulimit -c unlimited
|
||||
fi
|
||||
|
||||
echo "installing node $version";
|
||||
crit nvm install "$version"
|
||||
|
||||
if [[ "$(which npm)" == "" ]]; then
|
||||
echo "npm is missing, reinstalling node version $version";
|
||||
crit nvm deactivate;
|
||||
crit nvm uninstall "$version";
|
||||
install_node "$version";
|
||||
return
|
||||
fi
|
||||
}
|
||||
|
||||
get_lock
|
||||
install_node "$(cat ./.node-version)"
|
||||
npm install
|
||||
release_lock
|
||||
|
||||
ES_PATH_REPO="./.es-snapshot-repos/$EXECUTOR_NUMBER/" ES_PORT=$((9400 + EXECUTOR_NUMBER)) RUN=NODE_UNIT,NODE_INTEGRATION VERBOSE=true node ./scripts/ci.js
|
||||
7
scripts/mocha.js
Normal file
7
scripts/mocha.js
Normal file
@ -0,0 +1,7 @@
|
||||
process.argv.splice(
|
||||
2,
|
||||
0,
|
||||
...[require.resolve('../test/unit/index.js'), '--reporter', 'nyan']
|
||||
);
|
||||
|
||||
require('mocha/bin/mocha');
|
||||
Reference in New Issue
Block a user