habsi/elasticsearch-js
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

test against the latest releases on travis

Browse Source
This commit is contained in:
Spencer Alger
2014-04-23 19:28:01 -07:00
parent cfd29b6111
commit 51e6b32123
4 changed files with 58 additions and 542 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
.travis.yml
View File

@ -2,9 +2,9 @@ language: node_js
node_js: "0.10"
secure: "UFAGQ6m/VnEahbj9vttY9YoA5h5rEBE6K7AvEEbWnt+VKppV+w3hu3HZxgKr8C9PWhCzqlGvsLh+kCqykZhISU1fBCK/Ttp3nSpMvvF5tI2u51Rj1qZ/7NUGRU0qVI9KFt0rJeXMJwq3fivb1H6aojfPD1gsIte7NHNjUfd0iUg="
env:
- ES_BRANCH=1.1 COVERAGE=1
- ES_BRANCH=1.0 NODE_UNIT=0
- ES_BRANCH=0.90 NODE_UNIT=0
- ES_BRANCH=1.1 ES_RELEASE=1.1.1 COVERAGE=1
- ES_BRANCH=1.0 ES_RELEASE=1.0.3 NODE_UNIT=0
- ES_BRANCH=0.90 ES_RELEASE=0.90.13 NODE_UNIT=0
- NODE_UNIT=0 NODE_INTEGRATION=0 BROWSER_UNIT=1
script: ./scripts/ci.sh
email:

99
scripts/generate/logs/index.js
View File

@ -1,4 +1,4 @@
/* jshint curly:false */
/* jshint curly:false, latedef:false */
// args
var argv = require('optimist')
.usage('node scripts/generate/logs [-h|--host localhost:9200] [-c|--count 14000] [-d|--days 7]')
@ -24,7 +24,7 @@ var es = require('../../../src/elasticsearch');
var async = require('async');
var moment = require('moment');
var makeSamples = require('./samples').make;
// var Promise = require('bluebird');
var Promise = require('bluebird');
var startingMoment = moment().utc().startOf('day').subtract('days', argv.days);
var endingMoment = moment().utc().endOf('day').add('days', argv.days);
@ -51,7 +51,7 @@ var total = argv.count;
console.log('Generating', total, 'events across ±', argv.days, 'days');
function createIndex(indexName, done) {
function createIndex(indexName) {
// console.log('ensuring index "%s" exists', indexName);
var indexBody = {
@ -103,20 +103,20 @@ function createIndex(indexName, done) {
}
};
client.indices.create({
return client.indices.create({
ignore: 400,
index: indexName,
body: indexBody
}, function (err) {
if (err) return done(err);
client.cluster.health({
})
.then(function () {
return client.cluster.health({
index: indexName,
waitForStatus: 'yellow'
}, done);
});
});
}
var queue = async.queue(function (events, done) {
var body = [];
@ -127,55 +127,66 @@ var queue = async.queue(function (events, done) {
event = event.body;
if (indices[event.index] !== true) {
deps.push(async.apply(createIndex, event.index));
deps.push(createIndex(event.index));
indices[event.index] = true;
}
body.push({ index: header }, event);
});
async.parallel(deps, function (err) {
if (err) return done(err);
Promise.all(deps)
.then(function () {
if (body.length) {
return client.bulk({
body: body
});
} else {
return {};
}
})
.then(function (resp) {
if (resp.errors) {
var errors = [];
client.bulk({
body: body
}, function (err, resp) {
if (err) return done(err);
resp.items.forEach(function (item, i) {
if (item.index.error) {
errors.push(item.index.error);
eventBuffer.push(events[i]);
}
});
if (resp.errors) {
console.log(resp);
console.log(JSON.stringify(body, null, ' '));
console.log(JSON.stringify(resp, null, ' '));
process.exit();
}
process.stdout.write('.');
done();
});
});
console.log('\n - errors - \n' + errors.join('\n') + '\n');
}
})
.finally(function () {
process.stdout.write('.');
})
.nodeify(done);
}, 1);
var eventBuffer = [];
eventBuffer.flush = function () {
if (eventBuffer.length === 3500 || doneCreatingEvents) {
queue.push([eventBuffer.splice(0)], function (err) {
if (err) {
console.error(err.resp);
console.error(err.stack);
process.exit();
}
});
}
};
queue.drain = function () {
if (doneCreatingEvents) {
if (doneCreatingEvents && eventBuffer.length === 0) {
client.close();
process.stdout.write('.\n\ncreated ' + total + ' events\n\n');
} else {
eventBuffer.flush();
}
};
var topLevelErrorHandler = function (err) {
if (err) {
console.error(err.resp);
console.error(err.stack);
process.exit();
}
};
var unqueuedEvents = [];
unqueuedEvents.queue = function () {
queue.push([unqueuedEvents.splice(0)], topLevelErrorHandler);
};
async.timesSeries(total, function (i, done) {
// random date, plus less random time
@ -225,7 +236,7 @@ async.timesSeries(total, function (i, done) {
event['@message'] = event.ip + ' - - [' + dateAsIso + '] "GET ' + event.request + ' HTTP/1.1" ' +
event.response + ' ' + event.bytes + ' "-" "' + event.agent + '"';
unqueuedEvents.push({
eventBuffer.push({
header: {
_index: event.index,
_type: samples.types(),
@ -234,9 +245,9 @@ async.timesSeries(total, function (i, done) {
body: event
});
unqueuedEvents.length === 3500 && unqueuedEvents.queue();
eventBuffer.flush();
setImmediate(done);
}, function () {
unqueuedEvents.length && unqueuedEvents.queue();
doneCreatingEvents = true;
eventBuffer.flush();
});

242
scripts/generate/logs/index_cbs.js
View File

@ -1,242 +0,0 @@
/* jshint curly:false */
// args
var argv = require('optimist')
.usage('node scripts/generate/logs [-h|--host localhost:9200] [-c|--count 14000] [-d|--days 7]')
.options({
count: {
alias: 'c',
type: 'number',
default: 14000
},
days: {
alias: 'd',
type: 'number',
required: true
},
host: {
alias: 'h',
default: 'localhost:9200'
}
})
.argv;
var es = require('../../../src/elasticsearch');
var async = require('async');
var moment = require('moment');
var makeSamples = require('./samples').make;
// var Promise = require('bluebird');
var startingMoment = moment().utc().startOf('day').subtract('days', argv.days);
var endingMoment = moment().utc().endOf('day').add('days', argv.days);
var clientConfig = {
// log: {
// level: 'trace',
// type: 'file',
// path: require('path').join(__dirname, '../../../log')
// }
};
if (argv.host) {
clientConfig.hosts = argv.host;
} else if (argv.hosts) {
clientConfig.hosts = JSON.parse(argv.hosts);
}
var client = new es.Client(clientConfig);
var samples = makeSamples(startingMoment, endingMoment);
var indices = {};
var doneCreatingEvents = false;
var total = argv.count;
console.log('Generating', total, 'events across ±', argv.days, 'days');
function createIndex(indexName, done) {
// console.log('ensuring index "%s" exists', indexName);
var indexBody = {
settings: {
index: {
number_of_shards: 1,
number_of_replicas: 0
}
},
mappings: {
_default_: {
properties: {
'@timestamp': {
type: 'date'
},
id: {
type: 'integer',
index: 'not_analyzed',
include_in_all: false
},
country: {
type: 'string',
index: 'not_analyzed'
},
agent: {
type: 'multi_field',
fields: {
agent: {
type: 'string',
index: 'analyzed'
},
raw: {
type: 'string',
index: 'not_analyzed'
}
}
},
clientip: {
type: 'ip'
},
ip: {
type: 'ip'
},
memory: {
type: 'double'
}
}
}
}
};
client.indices.create({
ignore: 400,
index: indexName,
body: indexBody
}, function (err) {
if (err) return done(err);
client.cluster.health({
index: indexName,
waitForStatus: 'yellow'
}, done);
});
}
var queue = async.queue(function (events, done) {
var body = [];
var deps = [];
events.forEach(function (event) {
var header = event.header;
event = event.body;
if (indices[event.index] !== true) {
deps.push(async.apply(createIndex, event.index));
indices[event.index] = true;
}
body.push({ index: header }, event);
});
async.parallel(deps, function (err) {
if (err) return done(err);
client.bulk({
body: body
}, function (err, resp) {
if (err) return done(err);
if (resp.errors) {
console.log(resp);
console.log(JSON.stringify(body, null, ' '));
console.log(JSON.stringify(resp, null, ' '));
process.exit();
}
process.stdout.write('.');
done();
});
});
}, 1);
queue.drain = function () {
if (doneCreatingEvents) {
client.close();
process.stdout.write('.\n\ncreated ' + total + ' events\n\n');
}
};
var topLevelErrorHandler = function (err) {
if (err) {
console.error(err.resp);
console.error(err.stack);
process.exit();
}
};
var unqueuedEvents = [];
unqueuedEvents.queue = function () {
queue.push([unqueuedEvents.splice(0)], topLevelErrorHandler);
};
async.timesSeries(total, function (i, done) {
// random date, plus less random time
var date = new Date(samples.randomMsInDayRange());
var ms = samples.lessRandomMsInDay();
// extract number of hours from the milliseconds
var hours = Math.floor(ms / 3600000);
ms = ms - hours * 3600000;
// extract number of minutes from the milliseconds
var minutes = Math.floor(ms / 60000);
ms = ms - minutes * 60000;
// extract number of seconds from the milliseconds
var seconds = Math.floor(ms / 1000);
ms = ms - seconds * 1000;
// apply the values found to the date
date.setUTCHours(hours, minutes, seconds, ms);
var dateAsIso = date.toISOString();
var indexName = 'logstash-' + dateAsIso.substr(0, 4) + '.' + dateAsIso.substr(5, 2) + '.' + dateAsIso.substr(8, 2);
var event = {};
event.index = indexName;
event['@timestamp'] = dateAsIso;
event.ip = samples.ips();
event.extension = samples.extensions();
event.response = samples.responseCodes();
event.country = samples.countries();
event.point = samples.airports();
event['@tags'] = [
samples.tags(),
samples.tags2()
];
event.utc_time = dateAsIso;
event.referer = 'http://' + samples.referrers() + '/' + samples.tags() + '/' + samples.astronauts();
event.agent = samples.userAgents();
event.clientip = event.ip;
event.bytes = event.response < 500 ? samples.lessRandomRespSize() : 0;
event.request = '/' + samples.astronauts() + '.' + event.extension;
if (event.extension === 'php') {
event.phpmemory = event.memory = event.bytes * 40;
}
event['@message'] = event.ip + ' - - [' + dateAsIso + '] "GET ' + event.request + ' HTTP/1.1" ' +
event.response + ' ' + event.bytes + ' "-" "' + event.agent + '"';
unqueuedEvents.push({
header: {
_index: event.index,
_type: samples.types(),
_id: i,
},
body: event
});
unqueuedEvents.length === 3500 && unqueuedEvents.queue();
setImmediate(done);
}, function () {
unqueuedEvents.length && unqueuedEvents.queue();
doneCreatingEvents = true;
});

253
scripts/generate/logs/index_promises.js
View File

@ -1,253 +0,0 @@
/* jshint curly:false, latedef:false */
// args
var argv = require('optimist')
.usage('node scripts/generate/logs [-h|--host localhost:9200] [-c|--count 14000] [-d|--days 7]')
.options({
count: {
alias: 'c',
type: 'number',
default: 14000
},
days: {
alias: 'd',
type: 'number',
required: true
},
host: {
alias: 'h',
default: 'localhost:9200'
}
})
.argv;
var es = require('../../../src/elasticsearch');
var async = require('async');
var moment = require('moment');
var makeSamples = require('./samples').make;
var Promise = require('bluebird');
var startingMoment = moment().utc().startOf('day').subtract('days', argv.days);
var endingMoment = moment().utc().endOf('day').add('days', argv.days);
var clientConfig = {
// log: {
// level: 'trace',
// type: 'file',
// path: require('path').join(__dirname, '../../../log')
// }
};
if (argv.host) {
clientConfig.hosts = argv.host;
} else if (argv.hosts) {
clientConfig.hosts = JSON.parse(argv.hosts);
}
var client = new es.Client(clientConfig);
var samples = makeSamples(startingMoment, endingMoment);
var indices = {};
var doneCreatingEvents = false;
var total = argv.count;
console.log('Generating', total, 'events across ±', argv.days, 'days');
function createIndex(indexName) {
// console.log('ensuring index "%s" exists', indexName);
var indexBody = {
settings: {
index: {
number_of_shards: 1,
number_of_replicas: 0
}
},
mappings: {
_default_: {
properties: {
'@timestamp': {
type: 'date'
},
id: {
type: 'integer',
index: 'not_analyzed',
include_in_all: false
},
country: {
type: 'string',
index: 'not_analyzed'
},
agent: {
type: 'multi_field',
fields: {
agent: {
type: 'string',
index: 'analyzed'
},
raw: {
type: 'string',
index: 'not_analyzed'
}
}
},
clientip: {
type: 'ip'
},
ip: {
type: 'ip'
},
memory: {
type: 'double'
}
}
}
}
};
return client.indices.create({
ignore: 400,
index: indexName,
body: indexBody
})
.then(function () {
return client.cluster.health({
index: indexName,
waitForStatus: 'yellow'
});
});
}
var queue = async.queue(function (events, done) {
var body = [];
var deps = [];
events.forEach(function (event) {
var header = event.header;
event = event.body;
if (indices[event.index] !== true) {
deps.push(createIndex(event.index));
indices[event.index] = true;
}
body.push({ index: header }, event);
});
Promise.all(deps)
.then(function () {
if (body.length) {
return client.bulk({
body: body
});
} else {
return {};
}
})
.then(function (resp) {
if (resp.errors) {
var errors = [];
resp.items.forEach(function (item, i) {
if (item.index.error) {
errors.push(item.index.error);
eventBuffer.push(events[i]);
}
});
console.log('\n - errors - \n' + errors.join('\n') + '\n');
}
})
.finally(function () {
process.stdout.write('.');
})
.nodeify(done);
}, 1);
var eventBuffer = [];
eventBuffer.flush = function () {
if (eventBuffer.length === 3500 || doneCreatingEvents) {
queue.push([eventBuffer.splice(0)], function (err) {
if (err) {
console.error(err.resp);
console.error(err.stack);
process.exit();
}
});
}
};
queue.drain = function () {
if (doneCreatingEvents && eventBuffer.length === 0) {
client.close();
process.stdout.write('.\n\ncreated ' + total + ' events\n\n');
} else {
eventBuffer.flush();
}
};
async.timesSeries(total, function (i, done) {
// random date, plus less random time
var date = new Date(samples.randomMsInDayRange());
var ms = samples.lessRandomMsInDay();
// extract number of hours from the milliseconds
var hours = Math.floor(ms / 3600000);
ms = ms - hours * 3600000;
// extract number of minutes from the milliseconds
var minutes = Math.floor(ms / 60000);
ms = ms - minutes * 60000;
// extract number of seconds from the milliseconds
var seconds = Math.floor(ms / 1000);
ms = ms - seconds * 1000;
// apply the values found to the date
date.setUTCHours(hours, minutes, seconds, ms);
var dateAsIso = date.toISOString();
var indexName = 'logstash-' + dateAsIso.substr(0, 4) + '.' + dateAsIso.substr(5, 2) + '.' + dateAsIso.substr(8, 2);
var event = {};
event.index = indexName;
event['@timestamp'] = dateAsIso;
event.ip = samples.ips();
event.extension = samples.extensions();
event.response = samples.responseCodes();
event.country = samples.countries();
event.point = samples.airports();
event['@tags'] = [
samples.tags(),
samples.tags2()
];
event.utc_time = dateAsIso;
event.referer = 'http://' + samples.referrers() + '/' + samples.tags() + '/' + samples.astronauts();
event.agent = samples.userAgents();
event.clientip = event.ip;
event.bytes = event.response < 500 ? samples.lessRandomRespSize() : 0;
event.request = '/' + samples.astronauts() + '.' + event.extension;
if (event.extension === 'php') {
event.phpmemory = event.memory = event.bytes * 40;
}
event['@message'] = event.ip + ' - - [' + dateAsIso + '] "GET ' + event.request + ' HTTP/1.1" ' +
event.response + ' ' + event.bytes + ' "-" "' + event.agent + '"';
eventBuffer.push({
header: {
_index: event.index,
_type: samples.types(),
_id: i,
},
body: event
});
eventBuffer.flush();
setImmediate(done);
}, function () {
doneCreatingEvents = true;
eventBuffer.flush();
});