modified the config to own the connectionPool, transport, and a few other objects

This commit is contained in:
Spencer Alger
2013-10-18 09:40:55 -07:00
parent 0f9fa9c17c
commit 8cc87637e2
88 changed files with 1948 additions and 1338 deletions

View File

@ -1,5 +1,4 @@
var _ = require('<%= path2lib %>utils'),
paramHelper = require('<%= path2lib %>param_helper'),
errors = require('<%= path2lib %>errors'),
q = require('q');<%
@ -24,26 +23,24 @@ var <%= name %>Options = <%= stringify(options) %>;<%
%>
*/
function do<%= _.studlyCase(name) %>(params, cb) {
params = params || {};
if (typeof params === 'function') {
cb = params;
params = {};
} else {
params = params || {};
cb = typeof cb === 'function' ? cb : _.noop;
}
var request = {<%
if (~name.indexOf('exists')) {%>
ignore: _.union([404], params.ignore)<%
} else {%>
ignore: params.ignore<%
}
if (body) { if (_.contains(['bulk', 'msearch'], name)) {%>,
body: paramHelper.bulkBody(params.body, this.client.serializer) || null<%
} else { %>,
body: params.body || null<%
} }%>
}
, parts = {}
, query = {}
, responseOpts = {};
<%
var request = {
<%= writeRequestObjectBody(6, name, body, methods) %>
},
parts = {},
query = {},
responseOpts = {};
<%
if (methods.length > 1) { %>
// figure out the method
if (params.method = _.toUpperString(params.method)) {
if (<%= _.map(methods, function (method) { return 'params.method === ' + stringify(method) }).join(' || ') %>) {
request.method = params.method;
@ -58,8 +55,6 @@ if (methods.length > 1) { %>
request.method = <%= stringify(methods[0]) %>;<%
}%>
}<%
} else {%>
request.method = <%= stringify(methods[0]) %>;<%
}
%>

View File

@ -13,7 +13,10 @@ var _ = require('../../../../src/lib/utils')
function lines(i) {
function l(line) {
if (typeof line !== 'undefined') {
if (line === '') {
// no indent on empty lines
l.lines.push('');
} else if (typeof line !== 'undefined') {
l.lines.push(_.repeat(' ', l.indent) + line);
}
return l;
@ -158,6 +161,33 @@ var templateGlobals = {
return l.toString();
},
writeRequestObjectBody: function (indent, name, body, methods) {
var parts = [], l = lines(indent);
if (~name.indexOf('exists')) {
parts.push('ignore: _.union([404], params.ignore)');
} else {
parts.push('ignore: params.ignore');
}
if (body) {
if (_.contains(['bulk', 'msearch'], name)) {
parts.push('body: this.client.config.serializer.bulkBody(params.body || null)');
} else {
parts.push('body: params.body || null');
}
}
if (methods.length === 1) {
parts.push('method: ' + stringify(methods[0]));
}
_.each(parts, function (part, i) {
l(part + (i < parts.length - 1 ? ',' : ''));
});
return l.toString();
},
/**
* we want strings in code to use single-quotes, so this will JSON encode vars, but then
* modify them to follow our code standards.

View File

@ -1,6 +1,31 @@
// args
var count = parseInt(process.argv[2] || 14000, 10),
days = parseInt(process.argv[3] || 7, 10);
var argv = require('optimist')
.usage('node scripts/generate/logs [-h|--host localhost:9200] [-c|--count 14000] [-d|--days 7]')
.options({
count: {
alias: 'c',
type: 'number',
default: 14000
},
days: {
alias: 'c',
type: 'number',
required: true
},
host: {
alias: 'h',
default: 'localhost:9200'
}
})
.argv;
// Error.stackTraceLimit = Infinity;
// console.log(argv);
// process.exit();
var count = parseInt(argv._[0] || 14000, 10),
days = parseInt(argv._[1] || 7, 10);
var es = require('../../../src/elasticsearch'),
_ = require('../../../src/lib/utils'),
@ -10,11 +35,22 @@ var es = require('../../../src/elasticsearch'),
makeSamples = require('./samples').make,
startingMoment = moment().startOf('day').subtract('days', days),
endingMoment = moment().endOf('day').add('days', days),
client = new es.Client({
log: 'info'
});
clientConfig = {
log: {
level: ['info', 'error']
}
};
client.log.info('Generating', count, 'events across ±', days, 'days');
if (argv.host) {
clientConfig.hosts = argv.host;
} else if (argv.hosts) {
clientConfig.hosts = JSON.parse(argv.hosts);
}
var client = new es.Client(clientConfig);
var log = client.config.log;
log.info('Generating', count, 'events across ±', days, 'days');
fillIndecies(function () {
var actions = [],
@ -63,7 +99,7 @@ fillIndecies(function () {
actions.push(event);
if (actions.length === 3000 || i === count - 1) {
client.log.info('writing', actions.length / 2, 'documents');
client.config.log.info('writing', actions.length / 2, 'documents');
client.bulk({
body: actions
}, done);
@ -142,12 +178,12 @@ function fillIndecies(cb) {
movingDate.add('day', 1);
}
async.parallel(indexPushActions, function (err, responses) {
async.parralel(indexPushActions, function (err, responses) {
if (err) {
client.log.error(err);
client.config.log.error(err.message = 'Unable to create indicies: ' + err.message);
} else {
_.each(_.groupBy(responses), function (list, did) {
client.log.info(list.length, 'indicies', did);
client.config.log.info(list.length, 'indicies', did);
});
cb();
}