From 51e6b32123474ebfba56bbea4f767e11a756fd57 Mon Sep 17 00:00:00 2001 From: Spencer Alger Date: Wed, 23 Apr 2014 19:28:01 -0700 Subject: [PATCH] test against the latest releases on travis --- .travis.yml | 6 +- scripts/generate/logs/index.js | 99 +++++----- scripts/generate/logs/index_cbs.js | 242 ----------------------- scripts/generate/logs/index_promises.js | 253 ------------------------ 4 files changed, 58 insertions(+), 542 deletions(-) delete mode 100644 scripts/generate/logs/index_cbs.js delete mode 100644 scripts/generate/logs/index_promises.js diff --git a/.travis.yml b/.travis.yml index efbd7c1fe..8bd766839 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,9 +2,9 @@ language: node_js node_js: "0.10" secure: "UFAGQ6m/VnEahbj9vttY9YoA5h5rEBE6K7AvEEbWnt+VKppV+w3hu3HZxgKr8C9PWhCzqlGvsLh+kCqykZhISU1fBCK/Ttp3nSpMvvF5tI2u51Rj1qZ/7NUGRU0qVI9KFt0rJeXMJwq3fivb1H6aojfPD1gsIte7NHNjUfd0iUg=" env: - - ES_BRANCH=1.1 COVERAGE=1 - - ES_BRANCH=1.0 NODE_UNIT=0 - - ES_BRANCH=0.90 NODE_UNIT=0 + - ES_BRANCH=1.1 ES_RELEASE=1.1.1 COVERAGE=1 + - ES_BRANCH=1.0 ES_RELEASE=1.0.3 NODE_UNIT=0 + - ES_BRANCH=0.90 ES_RELEASE=0.90.13 NODE_UNIT=0 - NODE_UNIT=0 NODE_INTEGRATION=0 BROWSER_UNIT=1 script: ./scripts/ci.sh email: diff --git a/scripts/generate/logs/index.js b/scripts/generate/logs/index.js index 1924d740f..a5713b122 100644 --- a/scripts/generate/logs/index.js +++ b/scripts/generate/logs/index.js @@ -1,4 +1,4 @@ -/* jshint curly:false */ +/* jshint curly:false, latedef:false */ // args var argv = require('optimist') .usage('node scripts/generate/logs [-h|--host localhost:9200] [-c|--count 14000] [-d|--days 7]') @@ -24,7 +24,7 @@ var es = require('../../../src/elasticsearch'); var async = require('async'); var moment = require('moment'); var makeSamples = require('./samples').make; -// var Promise = require('bluebird'); +var Promise = require('bluebird'); var startingMoment = moment().utc().startOf('day').subtract('days', argv.days); var endingMoment = moment().utc().endOf('day').add('days', argv.days); @@ -51,7 +51,7 @@ var total = argv.count; console.log('Generating', total, 'events across ±', argv.days, 'days'); -function createIndex(indexName, done) { +function createIndex(indexName) { // console.log('ensuring index "%s" exists', indexName); var indexBody = { @@ -103,20 +103,20 @@ function createIndex(indexName, done) { } }; - client.indices.create({ + return client.indices.create({ ignore: 400, index: indexName, body: indexBody - }, function (err) { - if (err) return done(err); - - client.cluster.health({ + }) + .then(function () { + return client.cluster.health({ index: indexName, waitForStatus: 'yellow' - }, done); + }); }); } + var queue = async.queue(function (events, done) { var body = []; @@ -127,55 +127,66 @@ var queue = async.queue(function (events, done) { event = event.body; if (indices[event.index] !== true) { - deps.push(async.apply(createIndex, event.index)); + deps.push(createIndex(event.index)); indices[event.index] = true; } body.push({ index: header }, event); }); - async.parallel(deps, function (err) { - if (err) return done(err); + Promise.all(deps) + .then(function () { + if (body.length) { + return client.bulk({ + body: body + }); + } else { + return {}; + } + }) + .then(function (resp) { + if (resp.errors) { + var errors = []; - client.bulk({ - body: body - }, function (err, resp) { - if (err) return done(err); + resp.items.forEach(function (item, i) { + if (item.index.error) { + errors.push(item.index.error); + eventBuffer.push(events[i]); + } + }); - if (resp.errors) { - console.log(resp); - console.log(JSON.stringify(body, null, ' ')); - console.log(JSON.stringify(resp, null, ' ')); - process.exit(); - } - - process.stdout.write('.'); - done(); - }); - }); + console.log('\n - errors - \n' + errors.join('\n') + '\n'); + } + }) + .finally(function () { + process.stdout.write('.'); + }) + .nodeify(done); }, 1); +var eventBuffer = []; +eventBuffer.flush = function () { + if (eventBuffer.length === 3500 || doneCreatingEvents) { + queue.push([eventBuffer.splice(0)], function (err) { + if (err) { + console.error(err.resp); + console.error(err.stack); + process.exit(); + } + }); + } +}; + queue.drain = function () { - if (doneCreatingEvents) { + if (doneCreatingEvents && eventBuffer.length === 0) { client.close(); process.stdout.write('.\n\ncreated ' + total + ' events\n\n'); + } else { + eventBuffer.flush(); } }; -var topLevelErrorHandler = function (err) { - if (err) { - console.error(err.resp); - console.error(err.stack); - process.exit(); - } -}; - -var unqueuedEvents = []; -unqueuedEvents.queue = function () { - queue.push([unqueuedEvents.splice(0)], topLevelErrorHandler); -}; - async.timesSeries(total, function (i, done) { // random date, plus less random time @@ -225,7 +236,7 @@ async.timesSeries(total, function (i, done) { event['@message'] = event.ip + ' - - [' + dateAsIso + '] "GET ' + event.request + ' HTTP/1.1" ' + event.response + ' ' + event.bytes + ' "-" "' + event.agent + '"'; - unqueuedEvents.push({ + eventBuffer.push({ header: { _index: event.index, _type: samples.types(), @@ -234,9 +245,9 @@ async.timesSeries(total, function (i, done) { body: event }); - unqueuedEvents.length === 3500 && unqueuedEvents.queue(); + eventBuffer.flush(); setImmediate(done); }, function () { - unqueuedEvents.length && unqueuedEvents.queue(); doneCreatingEvents = true; + eventBuffer.flush(); }); \ No newline at end of file diff --git a/scripts/generate/logs/index_cbs.js b/scripts/generate/logs/index_cbs.js deleted file mode 100644 index 1924d740f..000000000 --- a/scripts/generate/logs/index_cbs.js +++ /dev/null @@ -1,242 +0,0 @@ -/* jshint curly:false */ -// args -var argv = require('optimist') - .usage('node scripts/generate/logs [-h|--host localhost:9200] [-c|--count 14000] [-d|--days 7]') - .options({ - count: { - alias: 'c', - type: 'number', - default: 14000 - }, - days: { - alias: 'd', - type: 'number', - required: true - }, - host: { - alias: 'h', - default: 'localhost:9200' - } - }) - .argv; - -var es = require('../../../src/elasticsearch'); -var async = require('async'); -var moment = require('moment'); -var makeSamples = require('./samples').make; -// var Promise = require('bluebird'); - -var startingMoment = moment().utc().startOf('day').subtract('days', argv.days); -var endingMoment = moment().utc().endOf('day').add('days', argv.days); -var clientConfig = { - // log: { - // level: 'trace', - // type: 'file', - // path: require('path').join(__dirname, '../../../log') - // } -}; - -if (argv.host) { - clientConfig.hosts = argv.host; -} else if (argv.hosts) { - clientConfig.hosts = JSON.parse(argv.hosts); -} - -var client = new es.Client(clientConfig); -var samples = makeSamples(startingMoment, endingMoment); - -var indices = {}; -var doneCreatingEvents = false; -var total = argv.count; - -console.log('Generating', total, 'events across ±', argv.days, 'days'); - -function createIndex(indexName, done) { - // console.log('ensuring index "%s" exists', indexName); - - var indexBody = { - settings: { - index: { - number_of_shards: 1, - number_of_replicas: 0 - } - }, - mappings: { - _default_: { - properties: { - '@timestamp': { - type: 'date' - }, - id: { - type: 'integer', - index: 'not_analyzed', - include_in_all: false - }, - country: { - type: 'string', - index: 'not_analyzed' - }, - agent: { - type: 'multi_field', - fields: { - agent: { - type: 'string', - index: 'analyzed' - }, - raw: { - type: 'string', - index: 'not_analyzed' - } - } - }, - clientip: { - type: 'ip' - }, - ip: { - type: 'ip' - }, - memory: { - type: 'double' - } - } - } - } - }; - - client.indices.create({ - ignore: 400, - index: indexName, - body: indexBody - }, function (err) { - if (err) return done(err); - - client.cluster.health({ - index: indexName, - waitForStatus: 'yellow' - }, done); - }); -} - -var queue = async.queue(function (events, done) { - - var body = []; - var deps = []; - - events.forEach(function (event) { - var header = event.header; - event = event.body; - - if (indices[event.index] !== true) { - deps.push(async.apply(createIndex, event.index)); - indices[event.index] = true; - } - - body.push({ index: header }, event); - }); - - async.parallel(deps, function (err) { - if (err) return done(err); - - client.bulk({ - body: body - }, function (err, resp) { - if (err) return done(err); - - if (resp.errors) { - console.log(resp); - console.log(JSON.stringify(body, null, ' ')); - console.log(JSON.stringify(resp, null, ' ')); - process.exit(); - } - - process.stdout.write('.'); - done(); - }); - }); - -}, 1); - -queue.drain = function () { - if (doneCreatingEvents) { - client.close(); - process.stdout.write('.\n\ncreated ' + total + ' events\n\n'); - } -}; - -var topLevelErrorHandler = function (err) { - if (err) { - console.error(err.resp); - console.error(err.stack); - process.exit(); - } -}; - -var unqueuedEvents = []; -unqueuedEvents.queue = function () { - queue.push([unqueuedEvents.splice(0)], topLevelErrorHandler); -}; - -async.timesSeries(total, function (i, done) { - - // random date, plus less random time - var date = new Date(samples.randomMsInDayRange()); - - var ms = samples.lessRandomMsInDay(); - - // extract number of hours from the milliseconds - var hours = Math.floor(ms / 3600000); - ms = ms - hours * 3600000; - - // extract number of minutes from the milliseconds - var minutes = Math.floor(ms / 60000); - ms = ms - minutes * 60000; - - // extract number of seconds from the milliseconds - var seconds = Math.floor(ms / 1000); - ms = ms - seconds * 1000; - - // apply the values found to the date - date.setUTCHours(hours, minutes, seconds, ms); - - var dateAsIso = date.toISOString(); - var indexName = 'logstash-' + dateAsIso.substr(0, 4) + '.' + dateAsIso.substr(5, 2) + '.' + dateAsIso.substr(8, 2); - var event = {}; - - event.index = indexName; - event['@timestamp'] = dateAsIso; - event.ip = samples.ips(); - event.extension = samples.extensions(); - event.response = samples.responseCodes(); - event.country = samples.countries(); - event.point = samples.airports(); - event['@tags'] = [ - samples.tags(), - samples.tags2() - ]; - event.utc_time = dateAsIso; - event.referer = 'http://' + samples.referrers() + '/' + samples.tags() + '/' + samples.astronauts(); - event.agent = samples.userAgents(); - event.clientip = event.ip; - event.bytes = event.response < 500 ? samples.lessRandomRespSize() : 0; - event.request = '/' + samples.astronauts() + '.' + event.extension; - if (event.extension === 'php') { - event.phpmemory = event.memory = event.bytes * 40; - } - event['@message'] = event.ip + ' - - [' + dateAsIso + '] "GET ' + event.request + ' HTTP/1.1" ' + - event.response + ' ' + event.bytes + ' "-" "' + event.agent + '"'; - - unqueuedEvents.push({ - header: { - _index: event.index, - _type: samples.types(), - _id: i, - }, - body: event - }); - - unqueuedEvents.length === 3500 && unqueuedEvents.queue(); - setImmediate(done); -}, function () { - unqueuedEvents.length && unqueuedEvents.queue(); - doneCreatingEvents = true; -}); \ No newline at end of file diff --git a/scripts/generate/logs/index_promises.js b/scripts/generate/logs/index_promises.js deleted file mode 100644 index a5713b122..000000000 --- a/scripts/generate/logs/index_promises.js +++ /dev/null @@ -1,253 +0,0 @@ -/* jshint curly:false, latedef:false */ -// args -var argv = require('optimist') - .usage('node scripts/generate/logs [-h|--host localhost:9200] [-c|--count 14000] [-d|--days 7]') - .options({ - count: { - alias: 'c', - type: 'number', - default: 14000 - }, - days: { - alias: 'd', - type: 'number', - required: true - }, - host: { - alias: 'h', - default: 'localhost:9200' - } - }) - .argv; - -var es = require('../../../src/elasticsearch'); -var async = require('async'); -var moment = require('moment'); -var makeSamples = require('./samples').make; -var Promise = require('bluebird'); - -var startingMoment = moment().utc().startOf('day').subtract('days', argv.days); -var endingMoment = moment().utc().endOf('day').add('days', argv.days); -var clientConfig = { - // log: { - // level: 'trace', - // type: 'file', - // path: require('path').join(__dirname, '../../../log') - // } -}; - -if (argv.host) { - clientConfig.hosts = argv.host; -} else if (argv.hosts) { - clientConfig.hosts = JSON.parse(argv.hosts); -} - -var client = new es.Client(clientConfig); -var samples = makeSamples(startingMoment, endingMoment); - -var indices = {}; -var doneCreatingEvents = false; -var total = argv.count; - -console.log('Generating', total, 'events across ±', argv.days, 'days'); - -function createIndex(indexName) { - // console.log('ensuring index "%s" exists', indexName); - - var indexBody = { - settings: { - index: { - number_of_shards: 1, - number_of_replicas: 0 - } - }, - mappings: { - _default_: { - properties: { - '@timestamp': { - type: 'date' - }, - id: { - type: 'integer', - index: 'not_analyzed', - include_in_all: false - }, - country: { - type: 'string', - index: 'not_analyzed' - }, - agent: { - type: 'multi_field', - fields: { - agent: { - type: 'string', - index: 'analyzed' - }, - raw: { - type: 'string', - index: 'not_analyzed' - } - } - }, - clientip: { - type: 'ip' - }, - ip: { - type: 'ip' - }, - memory: { - type: 'double' - } - } - } - } - }; - - return client.indices.create({ - ignore: 400, - index: indexName, - body: indexBody - }) - .then(function () { - return client.cluster.health({ - index: indexName, - waitForStatus: 'yellow' - }); - }); -} - - -var queue = async.queue(function (events, done) { - - var body = []; - var deps = []; - - events.forEach(function (event) { - var header = event.header; - event = event.body; - - if (indices[event.index] !== true) { - deps.push(createIndex(event.index)); - indices[event.index] = true; - } - - body.push({ index: header }, event); - }); - - Promise.all(deps) - .then(function () { - if (body.length) { - return client.bulk({ - body: body - }); - } else { - return {}; - } - }) - .then(function (resp) { - if (resp.errors) { - var errors = []; - - resp.items.forEach(function (item, i) { - if (item.index.error) { - errors.push(item.index.error); - eventBuffer.push(events[i]); - } - }); - - console.log('\n - errors - \n' + errors.join('\n') + '\n'); - } - }) - .finally(function () { - process.stdout.write('.'); - }) - .nodeify(done); - -}, 1); - -var eventBuffer = []; -eventBuffer.flush = function () { - if (eventBuffer.length === 3500 || doneCreatingEvents) { - queue.push([eventBuffer.splice(0)], function (err) { - if (err) { - console.error(err.resp); - console.error(err.stack); - process.exit(); - } - }); - } -}; - -queue.drain = function () { - if (doneCreatingEvents && eventBuffer.length === 0) { - client.close(); - process.stdout.write('.\n\ncreated ' + total + ' events\n\n'); - } else { - eventBuffer.flush(); - } -}; - -async.timesSeries(total, function (i, done) { - - // random date, plus less random time - var date = new Date(samples.randomMsInDayRange()); - - var ms = samples.lessRandomMsInDay(); - - // extract number of hours from the milliseconds - var hours = Math.floor(ms / 3600000); - ms = ms - hours * 3600000; - - // extract number of minutes from the milliseconds - var minutes = Math.floor(ms / 60000); - ms = ms - minutes * 60000; - - // extract number of seconds from the milliseconds - var seconds = Math.floor(ms / 1000); - ms = ms - seconds * 1000; - - // apply the values found to the date - date.setUTCHours(hours, minutes, seconds, ms); - - var dateAsIso = date.toISOString(); - var indexName = 'logstash-' + dateAsIso.substr(0, 4) + '.' + dateAsIso.substr(5, 2) + '.' + dateAsIso.substr(8, 2); - var event = {}; - - event.index = indexName; - event['@timestamp'] = dateAsIso; - event.ip = samples.ips(); - event.extension = samples.extensions(); - event.response = samples.responseCodes(); - event.country = samples.countries(); - event.point = samples.airports(); - event['@tags'] = [ - samples.tags(), - samples.tags2() - ]; - event.utc_time = dateAsIso; - event.referer = 'http://' + samples.referrers() + '/' + samples.tags() + '/' + samples.astronauts(); - event.agent = samples.userAgents(); - event.clientip = event.ip; - event.bytes = event.response < 500 ? samples.lessRandomRespSize() : 0; - event.request = '/' + samples.astronauts() + '.' + event.extension; - if (event.extension === 'php') { - event.phpmemory = event.memory = event.bytes * 40; - } - event['@message'] = event.ip + ' - - [' + dateAsIso + '] "GET ' + event.request + ' HTTP/1.1" ' + - event.response + ' ' + event.bytes + ' "-" "' + event.agent + '"'; - - eventBuffer.push({ - header: { - _index: event.index, - _type: samples.types(), - _id: i, - }, - body: event - }); - - eventBuffer.flush(); - setImmediate(done); -}, function () { - doneCreatingEvents = true; - eventBuffer.flush(); -}); \ No newline at end of file