yaml tests passing using a personl instance of ES, working on specifying the host/port

This commit is contained in:
Spencer Alger
2013-10-14 12:34:46 -07:00
parent 9e608226e1
commit 46aeac57e4
129 changed files with 9754 additions and 9166 deletions

View File

@ -23,7 +23,8 @@
"validthis": true,
"sub": true,
"maxlen": 140,
"-W030":true,
"-W030": true,
"-W068": true,
"globals": {
"describe": true,
@ -32,4 +33,4 @@
"it": true,
"beforeEach": true
}
}
}

File diff suppressed because it is too large Load Diff

11
test/integration/args.js Normal file
View File

@ -0,0 +1,11 @@
var program = require('commander');
program
.version('0.0.1')
.option('-c, --clusterName [name]', 'Set the name of the "cluster" this node will start', 'yaml_test_runners')
.option('-n, --nodeName [name]', 'Set the node\'s name', 'yaml_runner')
.option('-p, --httpPort [port]', 'Set the port for this node', 9299)
.option('-d, --dataPath [path]', 'Do not set this', '/tmp/yaml-test-data')
.parse(process.argv);
module.exports = program;

View File

@ -1,48 +1,89 @@
var fs = require('fs')
, path = require('path')
, async = require('async')
, assert = require('assert')
, jsYaml = require('js-yaml')
, expect = require('expect.js')
, nodeunit = require('nodeunit')
, _ = require('../../src/lib/toolbelt')
, es = require('../../src/elasticsearch');
var path = require('path'),
fs = require('fs'),
async = require('async'),
jsYaml = require('js-yaml'),
expect = require('expect.js'),
server = require('./server'),
_ = require('../../src/lib/utils'),
es = require('../../src/elasticsearch');
require('mocha-as-promised')();
var argv = require('optimist')
.default('executable', path.join(process.env.ES_HOME, './bin/elasticsearch'))
.default('clusterName', 'yaml-test-runner')
.default('dataPath', '/tmp/yaml-test-runner')
.argv;
/**
* Where do our tests live?
* @type {[type]}
*/
if (argv.hostname || argv.port) {
console.log('working with ES instance at ' + argv.hostname + ':' + argv.port);
}
// Where do the tests live?
var TEST_DIR = path.resolve(__dirname, '../../es_api_spec/test/');
// var doRE = '.*';
// test names matching this will be run
var doRE = /(^|\/)(.*)\/.*/;
// var FILE_WHITELIST = ['indices.analyze/10_analyze.yaml'];
/**
* We'll use this client for all of our work
* @type {es.Client}
*/
var client = new es.Client({
hosts: ['localhost:9200'],
log: {
type: 'file',
level: 'trace',
path: path.resolve(__dirname, '../integration-test.log')
},
max_connections: 1
// a reference to a personal instance of ES Server
var esServer = null;
// the client
var client = null;
// location that the logger will write to
var logFile = path.resolve(__dirname, '../integration-test.log');
// empty all of the indices in ES please
function clearIndices (done) {
client.indices.delete({
index: '*',
ignore: 404
}, done);
}
// before running any tests...
before(function (done) {
// start our personal ES Server
this.timeout(null);
if (argv.hostname) {
done();
} else {
server.start(argv, function (err, server) {
esServer = server;
done(err);
});
}
});
function clearIndicies (done) {
client.indices.delete({
index: '*'
}, function () {
done();
before(function (done) {
// delete the integration log
fs.unlink(logFile, function (err) {
if (err && ~err.message.indexOf('ENOENT')) {
done();
} else {
done(err);
}
});
}
// before running any tests, clear the test* indicies
before(clearIndicies);
});
before(function () {
// create the client
client = new es.Client({
hosts: [
{
hostname: esServer ? esServer.__hostname : argv.hostname,
port: esServer ? esServer.__port : argv.port
}
],
log: {
type: 'file',
level: ['error', 'trace'],
path: logFile
}
});
});
before(clearIndices);
/**
* recursively crawl the directory, looking for yaml files which will be passed to loadFile
@ -52,8 +93,8 @@ before(clearIndicies);
function loadDir(dir) {
fs.readdirSync(dir).forEach(function (fileName) {
describe(fileName, function () {
var location = path.join(dir, fileName)
, stat = fs.statSync(location);
var location = path.join(dir, fileName),
stat = fs.statSync(location);
if (stat.isFile() && fileName.match(/\.yaml$/) && location.match(doRE)) {
loadFile(location);
@ -71,16 +112,30 @@ function loadDir(dir) {
*/
var ES_VERSION = null;
// core expression for finding a version
var versionExp = '([\\d\\.]*\\d)(?:\\.\\w+)?';
/**
* Regular Expression to extract version numbers from a version string
* Regular Expression to extract a version number from a string
* @type {RegExp}
*/
var versionExp = '([\\d\\.]*\\d)(?:\\.\\w+)?';
var versionRE = new RegExp(versionExp);
/**
* Regular Expression to extract a version range from a string
* @type {RegExp}
*/
var versionRangeRE = new RegExp(versionExp + '\\s*\\-\\s*' + versionExp);
/**
* Fetches the client.info, and parses out the version number to a comparable string
* @param done {Function} - callback
*/
function getVersionFromES(done) {
client.info().then(function (resp) {
client.info({}, function (err, resp) {
if (err) {
throw new Error('unable to get info about ES');
}
expect(resp.version.number).to.match(versionRE);
ES_VERSION = versionToComparableString(versionRE.exec(resp.version.number)[1]);
done();
@ -89,6 +144,7 @@ function getVersionFromES(done) {
/**
* Transform x.x.x into xxx.xxx.xxx, striping off any text at the end like beta or pre-alpha35
*
* @param {String} version - Version number represented as a string
* @return {String} - Version number represented as three numbers, seperated by -, all numbers are
* padded with 0 and will be three characters long so the strings can be compared.
@ -109,6 +165,7 @@ function versionToComparableString(version) {
/**
* Compare a version range to the ES_VERSION, determining if the current version
* falls within the range.
*
* @param {String} rangeString - a string representing two version numbers seperated by a "-"
* @return {Boolean} - is the current version within the range (inclusive)
*/
@ -131,15 +188,12 @@ function rangeMatchesCurrentVersion(rangeString, done) {
/**
* read the file's contents, parse the yaml, pass to makeTest
*
* @param {String} path - Full path to yaml file
* @return {undefined}
*/
function loadFile(location) {
var relativeName = path.relative(TEST_DIR, location)
, groupName = path.dirname(relativeName)
, fileName = path.basename(relativeName)
, docsInFile = []
, standardTestConfigs = [];
var docsInFile = [];
jsYaml.loadAll(
fs.readFileSync(location, { encoding:'utf8' }),
@ -158,6 +212,7 @@ function loadFile(location) {
* Read the test descriptions from a yaml document (usually only one test, per doc but
* sometimes multiple docs per file, and because of the layout there COULD be
* multiple test per test...)
*
* @param {Object} testConfigs - The yaml document
* @return {undefined}
*/
@ -165,7 +220,6 @@ function makeTest(testConfig, count) {
var setup;
if (_.has(testConfig, 'setup')) {
(new ActionRunner(testConfig.setup)).each(function (action, name) {
console.log('setup', name);
before(action);
});
delete testConfig.setup;
@ -180,9 +234,16 @@ function makeTest(testConfig, count) {
});
// after running the tests, remove all indices
after(clearIndicies);
after(clearIndices);
}
/**
* Class to wrap a single document from a yaml test file
*
* @constructor
* @class ActionRunner
* @param actions {Array} - The array of actions directly from the Yaml file
*/
function ActionRunner(actions) {
this._actions = [];
@ -319,8 +380,15 @@ ActionRunner.prototype = {
return from;
},
/**
* Do a skip operation, setting the skipping flag to true if the version matches
* the range defined in args.version
*
* @param args
* @param done
*/
do_skip: function (args, done) {
rangeMatchesCurrentVersion(args.version, function (match) {
rangeMatchesCurrentVersion(args.version, _.bind(function (match) {
if (match) {
this.skipping = true;
console.log('skipping the rest of these actions' + (args.reason ? ' because ' + args.reason : ''));
@ -328,18 +396,17 @@ ActionRunner.prototype = {
this.skipping = false;
}
done();
}.bind(this));
}, this));
},
/**
* Do a request, as outlined in the args
*
* @param {[type]} args [description]
* @param {Function} done [description]
* @return {[type]} [description]
*/
do_do: function (args, done) {
this._last_requests_response = null;
var catcher;
// resolve the catch arg to a value used for matching once the request is complete
@ -374,9 +441,7 @@ ActionRunner.prototype = {
var action = Object.keys(args).pop()
, clientActionName = _.map(action.split('.'), _.camelCase).join('.')
, clientAction = this.get(clientActionName, client)
, response
, error
, params = _.map(args[action], function (val) {
, params = _.map(args[action], function (val, name) {
if (typeof val === 'string' && val[0] === '$') {
return this.get(val);
}
@ -391,35 +456,27 @@ ActionRunner.prototype = {
catcher = null;
}
clientAction.call(client, params)
.then(function (resp) {
response = resp;
})
.fail(function (err, resp) {
error = err;
response = resp;
})
.finally(function () {
this._last_requests_response = response;
clientAction.call(client, params, _.bind(function (error, body, status) {
this._last_requests_response = body;
if (error){
if (catcher) {
if (catcher instanceof RegExp) {
// error message should match the regexp
expect(error.message).to.match(catcher);
} else if (typeof catcher === 'function') {
// error should be an instance of
expect(error).to.be.a(catcher);
} else {
throw new Error('Invalid catcher '+catcher);
}
if (error){
if (catcher) {
if (catcher instanceof RegExp) {
// error message should match the regexp
expect(error.message).to.match(catcher);
} else if (typeof catcher === 'function') {
// error should be an instance of
expect(error).to.be.a(catcher);
} else {
return done(error);
throw new Error('Invalid catcher ' + catcher);
}
} else {
throw error;
}
}
done();
}.bind(this));
done();
}, this));
} else {
throw new Error('stepped in do_do, did not find a function');
}
@ -456,6 +513,7 @@ ActionRunner.prototype = {
/**
* Test that the specified path exists in the response and has a
* false value (eg. 0, false, undefined, null or the empty string)
*
* @param {string} path - Path to the response value to test
* @return {undefined}
*/
@ -465,6 +523,7 @@ ActionRunner.prototype = {
/**
* Test that the response field (arg key) matches the value specified
*
* @param {Object} args - Hash of fields->values that need to be checked
* @return {undefined}
*/
@ -479,6 +538,7 @@ ActionRunner.prototype = {
/**
* Test that the response field (arg key) is less than the value specified
*
* @param {Object} args - Hash of fields->values that need to be checked
* @return {undefined}
*/
@ -490,6 +550,7 @@ ActionRunner.prototype = {
/**
* Test that the response field (arg key) is greater than the value specified
*
* @param {Object} args - Hash of fields->values that need to be checked
* @return {undefined}
*/
@ -502,6 +563,7 @@ ActionRunner.prototype = {
/**
* Test that the response field (arg key) has a length equal to that specified.
* For object values, checks the length of the keys.
*
* @param {Object} args - Hash of fields->values that need to be checked
* @return {undefined}
*/

View File

@ -0,0 +1,61 @@
var childProc = require('child_process'),
events = require('events'),
q = require('q'),
path = require('path'),
fs = require('fs'),
_ = require('../../src/lib/utils');
exports.start = function (params, cb) {
if (!fs.existsSync(params.executable)) {
return cb(new Error('unable to find elasticsearch executable'));
}
var server = childProc.spawn(
params.executable,
[
'-f',
'-D es.cluster.name=' + params.clusterName,
params.port ? '-D es.http.port=' + params.port : undefined,
'-D es.path.data=' + params.dataPath,
'-D es.gateway.type=none',
'-D es.index.store.type=memory',
'-D es.discovery.zen.ping.multicast.enabled=false',
],
{
cwd: undefined,
env: process.env,
stdio: [
'ignore',
'pipe',
'pipe'
]
}
);
server.stdout.on('data', function (line) {
line = line.toString();
var match;
// console.log(_.trim(line));
if (!params.port && (match = line.match(/bound_address \{inet\[\/?([^:]+):(\d+)\]\}/))) {
server.__hostname = match[1];
server.__port = match[2];
}
if (line.match(/started\s*$/)) {
console.log('Personal ES Server started at', server.__hostname + ':' + server.__port);
server.stdout.removeAllListeners();
server.stderr.removeAllListeners();
cb(null, server);
}
});
server.stderr.on('data', function (line) {
console.error(_.trim(line));
});
process.on('exit', function () {
server.kill();
});
};

2
test/mocha.opts Normal file
View File

@ -0,0 +1,2 @@
--require should
--reporter dot

View File

@ -1,22 +1,16 @@
/* node transport function tests */
'use strict';
var es = require('../../src/elasticsearch'),
api = require('../../src/lib/utils').requireDir(module, '../../src/api'),
expect = require('expect.js');
var Client = require('../../src/lib/client')
, _ = require('../../src/lib/toolbelt')
, api = _.requireDir(module, '../../src/api')
, c;
describe('Client instances creation', function () {
var client;
exports['Client instances creation'] = {
beforeEach(function () {
client = new es.Client();
});
'setUp': function (done) {
c = new Client();
done();
},
'api is inherited': function(test) {
test.equals(c.bulk, api.bulk);
test.equals(c.cluster.node_stats, api.cluster.node_stats);
test.done();
}
};
it('inherits the api', function () {
client.bulk.should.be.exactly(api.bulk);
client.cluster.nodeStats.should.be.exactly(api.cluster.node_stats);
});
});

View File

@ -0,0 +1,64 @@
var es = require('../../src/elasticsearch');
describe('Connection Pool', function () {
var client, pool;
beforeEach(function () {
client = new es.Client();
pool = client.connectionPool;
});
describe('default settings', function () {
it('by default has one living connection and no dead connections', function () {
pool.connections.alive.should.have.lengthOf(1);
pool.connections.dead.should.have.lengthOf(0);
});
});
describe('#setNodes', function () {
it('rejects anything accept an array of objects', function () {
(function () {
pool.setNodes([
'string'
]);
}).should.throw(TypeError);
(function () {
pool.setNodes('string');
}).should.throw(TypeError);
});
it('will clear out old nodes', function () {
// set an empty set of nodes
pool.setNodes([]);
pool.connections.alive.should.have.lengthOf(0);
pool.connections.dead.should.have.lengthOf(0);
});
it('will accept a new node list', function () {
var conns = pool.connections;
// set a list of 3 nodes
pool.setNodes([
{
hostname: 'es-cluster.us',
port: '9200'
},
{
hostname: 'es-cluster-1.us',
port: '9200'
},
{
hostname: 'es-cluster-2.us',
port: '9200'
}
]);
conns.alive.should.have.lengthOf(3);
conns.dead.should.have.lengthOf(0);
});
});
});

View File

@ -1,72 +0,0 @@
var Log = require('../../src/lib/log')
, log = new Log([])
, Stdio = require('../../src/lib/loggers/stdio')
, _ = require('../../src/lib/toolbelt')
, warningLogger;
exports['Stdio Logger'] = {
setUp: function (done) {
if (warningLogger) {
warningLogger.cleanUpListeners();
}
// new logger in warning mode
warningLogger = new Stdio({
levels: ['error', 'warning']
}, log);
done();
},
'logs error messages': function (test) {
test.expect(3);
warningLogger.write = function (to, label, colorize, what) {
test.equal(label, 'ERROR');
test.ok(_.isArray(what), 'Messages logged from calls to error should be an array');
test.equal(what[0], 'Test Error Message');
test.done();
};
log.error('Test Error Message');
},
'should log warnings': function (test) {
test.expect(2);
warningLogger.write = function (to, label, colorize, what) {
test.equal(label, 'WARNING');
test.equal(what, 'Test Warning Message');
test.done();
};
log.warning('Test Warning', 'Message');
},
'should NOT log info messages': function (test) {
if (log.info('Info')) {
test.ok(false, 'There shouldn\'t be listeners for info logs');
} else {
test.done();
}
},
'should NOT log debug messages': function (test) {
if (log.debug('Debug')) {
test.ok(false, 'There shouldn\'t be listeners for debug logs');
} else {
test.done();
}
},
'should NOT log trace messages': function (test) {
if (log.trace('curl "http://localhost:9200" -d "{ \"query\": ... }"')) {
test.ok(false, 'There shouldn\'t be listeners for trace logs');
} else {
test.done();
}
}
};

View File

@ -0,0 +1,66 @@
var es = require('../../src/elasticsearch'),
Stdio = require('../../src/lib/loggers/stdio'),
_ = require('../../src/lib/utils'),
expect = require('expect.js');
describe('Stdio Logger listening to levels warning and error', function () {
var client, logger;
before(function () {
client = new es.Client({
log: []
});
});
beforeEach(function () {
if (logger) {
logger.cleanUpListeners();
}
// new logger in warning mode
logger = new Stdio({
levels: ['error', 'warning']
}, client.log);
});
it('logs error messages', function (done) {
logger.write = function (to, label, colorize, what) {
label.should.eql('ERROR');
what.should.have.type('string');
what.should.match(/^Error: Test Error Message/);
done();
};
client.log.error('Test Error Message');
});
it('logs warning messages', function (done) {
logger.write = function (to, label, colorize, what) {
expect(label).to.be('WARNING');
expect(what).to.be('Test Warning Message');
done();
};
client.log.warning('Test Warning', 'Message');
});
it('does not log info messages', function () {
if (client.log.info('Info')) {
throw new Error('There shouldn\'t be listeners for info logs');
}
});
it('does not log debug messages', function () {
if (client.log.debug('Debug')) {
throw new Error('There shouldn\'t be listeners for debug logs');
}
});
it('does not log trace messages', function () {
if (client.log.trace('curl "http://localhost:9200" -d "{ \"query\": ... }"')) {
throw new Error('There shouldn\'t be listeners for trace logs');
}
});
});

View File

@ -1,4 +1,4 @@
var _ = require('../../src/lib/toolbelt')
var _ = require('../../src/lib/utils')
, expect = require('expect.js');
describe('Utils', function () {
@ -46,43 +46,43 @@ describe('Utils', function () {
describe('#isNumeric', function () {
it('likes Infinity', function () {
expect(_.isNumeric(Infinity)).to.eq(true);
expect(_.isNumeric(Infinity)).to.be.true;
});
it('likes strings', function () {
expect(_.isNumeric('100')).to.eq(true);
expect(_.isNumeric('100')).to.be.true;
});
it('likes integers', function () {
expect(_.isNumeric(100)).to.eq(true);
expect(_.isNumeric(100)).to.be.true;
});
it('likes floats', function () {
expect(_.isNumeric(100.1)).to.eq(true);
expect(_.isNumeric(100.1)).to.be.true;
});
it('likes exponentials', function () {
expect(_.isNumeric(100e1)).to.eq(true);
expect(_.isNumeric(100e1)).to.be.true;
});
it('likes hexidecimals', function () {
expect(_.isNumeric(0x100)).to.eq(true);
expect(_.isNumeric(0x100)).to.be.true;
});
it('likes imaginary numbers', function () {
expect(_.isNumeric('yeah right')).to.eq(false);
expect(_.isNumeric('yeah right')).to.be.false;
});
it('dislikes strings with words', function () {
expect(_.isNumeric('100heat')).to.eq(false);
expect(_.isNumeric('100heat')).to.be.false;
});
it('dislikes strings with words even if they are seperate', function () {
expect(_.isNumeric('100 pasta')).to.eq(false);
expect(_.isNumeric('100 pasta')).to.be.false;
});
it('dislikes null', function () {
expect(_.isNumeric(null)).to.eq(false);
expect(_.isNumeric(null)).to.be.false;
});
});
@ -119,14 +119,14 @@ describe('Utils', function () {
describe('#map', function () {
it('returns an object when passed an object', function () {
var out = _.map({a:1, b:2}, function (val) { return val * 2; });
expect(out).to.deep.eq({a:2, b:4});
expect(out).to.eql({a:2, b:4});
});
it('returns an array for anything else', function () {
var std = _.map([1, 2, 3], function (val) { return val * 2; });
expect(std)
.to.be.a('array')
.and.to.deep.eq(_.map('123', function (val) { return val * 2; }));
.and.to.eql(_.map('123', function (val) { return val * 2; }));
});
});
@ -137,41 +137,41 @@ describe('Utils', function () {
describe('#camelCase', function () {
it('find spaces, underscores, and other natural word breaks', function () {
expect(_.camelCase('Neil PatRICK hArris-is_a.dog')).to.eq('neilPatrickHarrisIsADog');
expect(_.camelCase('Neil PatRICK hArris-is_a.dog')).to.eql('neilPatrickHarrisIsADog');
});
it('ignores abreviations', function () {
expect(_.camelCase('JSON_parser')).to.eq('jsonParser');
expect(_.camelCase('JSON_parser')).to.eql('jsonParser');
});
});
describe('#studlyCase', function () {
it('find spaces, underscores, and other natural word breaks', function () {
expect(_.studlyCase('Neil PatRICK hArris-is_a.dog')).to.eq('NeilPatrickHarrisIsADog');
expect(_.studlyCase('Neil PatRICK hArris-is_a.dog')).to.eql('NeilPatrickHarrisIsADog');
});
it('ignores abreviations', function () {
expect(_.studlyCase('JSON_parser')).to.eq('JsonParser');
expect(_.studlyCase('JSON_parser')).to.eql('JsonParser');
});
});
describe('#toLowerString', function () {
it('transforms normal strings', function () {
expect(_.toLowerString('PASTA')).to.eq('pasta');
expect(_.toLowerString('PASTA')).to.eql('pasta');
});
it('ignores long form empty vals (null, false, undef)', function () {
expect(_.toLowerString(null)).to.eq('');
expect(_.toLowerString(false)).to.eq('');
expect(_.toLowerString(void 0)).to.eq('');
expect(_.toLowerString(null)).to.eql('');
expect(_.toLowerString(false)).to.eql('');
expect(_.toLowerString(void 0)).to.eql('');
});
it('uses the objects own toString', function () {
expect(_.toLowerString(['A', 'B'])).to.eq('a,b');
expect(_.toLowerString(['A', 'B'])).to.eql('a,b');
});
it('sorta kinda works on objects', function () {
expect(_.toLowerString({a: 'thing'})).to.eq('[object object]');
expect(_.toLowerString({a: 'thing'})).to.eql('[object object]');
});
});
});
@ -183,7 +183,7 @@ describe('Utils', function () {
var obj = {
foo:'bar'
};
expect(_.deepMerge(obj, { bar: 'baz' })).to.eq(obj);
expect(_.deepMerge(obj, { bar: 'baz' })).to.eql(obj);
});
it('concats arrays', function () {

View File

@ -0,0 +1,45 @@
var es = require('../../src/elasticsearch'),
sinon = require('sinon'),
expect = require('expect.js');
describe('transport', function () {
describe('#sniff', function () {
it('does a head request to "/"', function (done) {
var c = new es.Client();
// stub the tranports request method, arg 1 is a callback
sinon.stub(c.transport, 'request').callsArgAsync(1);
c.transport.sniff(function (err, resp) {
var spy = c.transport.request.getCall(0),
params = spy.args[0];
params.should.have.type('object');
params.should.have.property('path', '/_cluster/nodes');
if (params.method) {
params.should.be('GET');
}
done();
});
});
describe('when sniffOnStart === true', function () {
describe('and the cluster is down', function () {
before(function (done) {
var c = new es.Client({
sniffOnStart: true,
hosts: [
'intentionally-not-a-real-cluster.com:9200'
]
});
c.on('sniff complete', done);
});
it('should not have any connections in the connection pool', function () {
});
});
});
});
});