yaml tests passing using a personl instance of ES, working on specifying the host/port

scripts/generate/js_api/index.js

@@ -1,10 +1,10 @@
-var _ = require('../../src/lib/toolbelt')
+var _ = require('../../../src/lib/utils')
   , asset = require('assert')
   , path = require('path')
   , fs = require('fs')
   , mkdirp = require('mkdirp')
-  , docs = _.requireDir(module, '../../es_api_spec/api')
-  , outputDir = _.joinPath(__dirname, '../../src/api/')
+  , docs = _.requireDir(module, '../../../es_api_spec/api')
+  , outputDir = _.joinPath(__dirname, '../../../src/api/')
   , templates = require('./templates')
   , notes = require('./notes')
   , aliases = require('./aliases');
@@ -35,7 +35,7 @@ _.each(docs, function (doc) {
 
     var spec = {
       name: name,
-      methods: _.map(def.methods, function (m) { return m.toLowerCase(); }),
+      methods: _.map(def.methods, function (m) { return m.toUpperCase(); }),
       docUrl: def.documentation,
       urlParts: def.url.parts,
       params: def.url.params,

scripts/generate/js_api/notes.js

@@ -0,0 +1,2 @@
+// exports['cluster.node_info'] =
+//   '//TODO: this enpoint ***needs*** work, many of the possible urls are not supported';

scripts/generate/js_api/templates/action.tmpl

@@ -1,5 +1,7 @@
-var _ = require('<%= path2lib %>toolbelt')
-  , paramHelper = require('<%= path2lib %>param_helper');<%
+var _ = require('<%= path2lib %>utils'),
+  paramHelper = require('<%= path2lib %>param_helper'),
+  errors = require('<%= path2lib %>errors'),
+  q = require('q');<%
 
 
 if (_.keys(enumOptions).length) {
@@ -11,9 +13,6 @@ var <%= name %>Options = <%= stringify(options) %>;<%
 }
 %>
 
-<%= notes ? notes + '\n' : ''
-%>
-
 /**
  * Perform an elasticsearch [<%= name %>](<%= docUrl %>) request
  *
@@ -24,7 +23,7 @@ var <%= name %>Options = <%= stringify(options) %>;<%
 })
 %>
  */
-function do<%= _.studlyCase(name) %>(params, callback) {
+function do<%= _.studlyCase(name) %>(params, cb) {
   params = params || {};
 
   var request = {<%
@@ -39,22 +38,22 @@ function do<%= _.studlyCase(name) %>(params, callback) {
       body: params.body || null<%
       } }%>
     }
-    , url = {}
+    , parts = {}
     , query = {}
     , responseOpts = {};
     <%
 
 if (methods.length > 1) { %>
-  if (params.method = _.toLowerString(params.method)) {
+  if (params.method = _.toUpperString(params.method)) {
     if (<%= _.map(methods, function (method) { return 'params.method === ' + stringify(method) }).join(' || ') %>) {
       request.method = params.method;
     } else {
       throw new TypeError('Invalid method: should be one of <%= methods.join(', ') %>');
     }
   } else {<%
-    if (_.contains(methods, 'get')) {
-    var nonGet = _.find(methods, function (m) {return m !== 'get'; });%>
-    request.method = params.body ? <%= stringify(nonGet) %> : 'get';<%
+    if (_.contains(methods, 'GET')) {
+    var nonGet = _.find(methods, function (m) {return m !== 'GET'; });%>
+    request.method = params.body ? <%= stringify(nonGet) %> : 'GET';<%
     } else {%>
     request.method = <%= stringify(methods[0]) %>;<%
     }%>
@@ -64,21 +63,17 @@ if (methods.length > 1) { %>
 }
 %>
 
-  // find the url's params
-<%= writeParams(2, urlParts, 'url.') %>
+  // find the paths's params
+<%= writeParams(2, urlParts, 'parts.') %>
 
-  // build the url
+  // build the path
 <%= writeUrls(2, urls, urlParts, params) %>
 
   // build the query string
 <%= writeParams(2, params, 'query.') %>
-  request.url = request.url + _.makeQueryString(query);
+  request.path = request.path + _.makeQueryString(query);
 
-  var reqPromise = this.client.request(request);
-  if (callback) {
-    reqPromise.then(_.bind(callback, null, null), callback);
-  }
-  return reqPromise;
+<%= returnStatement(2, name) %>
 }
 
 module.exports = do<%= _.studlyCase(name) %>;

scripts/generate/js_api/templates/index.js

@@ -1,5 +1,5 @@
 
-var _ = require('../../../src/lib/toolbelt')
+var _ = require('../../../../src/lib/utils')
   , fs = require('fs')
   , path = require('path')
   , urlParamRE = /\{(\w+)\}/g;
@@ -13,7 +13,9 @@ var _ = require('../../../src/lib/toolbelt')
 function lines(i) {
 
   function l(line) {
-    l.lines.push(_.repeat(' ', l.indent) + line);
+    if (typeof line !== 'undefined') {
+      l.lines.push(_.repeat(' ', l.indent) + line);
+    }
     return l;
   }
 
@@ -25,14 +27,14 @@ function lines(i) {
     return l;
   };
 
-  l.in = function () {
+  l.in = function (line) {
     l.indent += 2;
-    return l;
+    return l(line);
   };
 
-  l.out = function () {
+  l.out = function (line) {
     l.indent -= 2;
-    return l;
+    return l(line);
   };
 
   l.toString = function () {
@@ -109,14 +111,14 @@ var templateGlobals = {
       // collect the vars from the url and replace them to form the js that will build the url
       var makeL = lines(), vars = [];
 
-      makeL('request.url = \'' + url.replace(urlParamRE, function (match, varName) {
+      makeL('request.path = \'' + url.replace(urlParamRE, function (match, varName) {
         var varDetails = urlParams[varName];
         varDetails.name = varName;
         vars.push(varDetails);
         if (urlVarIsRequired(varDetails)) {
-          return '\' + encodeURIComponent(url.' + varName + ') + \'';
+          return '\' + encodeURIComponent(parts.' + varName + ') + \'';
         } else {
-          return '\' + encodeURIComponent(url.' + varName + ' || ' + stringify(varDetails.default) + ') + \'';
+          return '\' + encodeURIComponent(parts.' + varName + ' || ' + stringify(varDetails.default) + ') + \'';
         }
       }) + '\';');
 
@@ -131,17 +133,20 @@ var templateGlobals = {
         var requiredVars = _.filter(vars, urlVarIsRequired);
 
         var condition = _.map(requiredVars, function (v) {
-          return 'url.hasOwnProperty(\'' + v.name + '\')';
+          return 'parts.hasOwnProperty(\'' + v.name + '\')';
         }).join(' && ');
 
-        l((urlIndex > 0 ? 'else ' : '') + (condition ? 'if (' + condition + ') ' : '') + '{').in();
-        l.split(makeL.toString()).out();
-        l('}');
+        l((urlIndex > 0 ? 'else ' : '') + (condition ? 'if (' + condition + ') ' : '') + '{')
+          .in()
+            .split(makeL.toString())
+          .out('}');
 
         if (urlIndex === urls.length - 1 && condition) {
-          l('else {').in();
-          l('throw new TypeError(\'Unable to build a url with those params. Supply at least ' + vars.join(', ') + '\');').out();
-          l('}');
+          l('else {')
+            .in('throw new TypeError(\'Unable to build a path with those params. Supply at least ' +
+              vars.join(', ') + '\');'
+            )
+          .out('}');
         }
 
       } else {
@@ -185,6 +190,22 @@ var templateGlobals = {
     } else {
       return name;
     }
+  },
+
+  returnStatement: function (indent, name) {
+    var l = lines(indent);
+    if (name.match(/(^|\.)exists/)) {
+      l('this.client.request(request, function (err, response) {')
+        .in('if (err instanceof errors.NotFound) {')
+          .in('cb(err, false);')
+        .out('} else {')
+          .in('cb(err, true);')
+        .out('}')
+      .out('});');
+    } else {
+      l('this.client.request(request, cb);');
+    }
+    return l.toString();
   }
 };
 

scripts/generate/logs/index.js

@@ -0,0 +1,155 @@
+// args
+var count = parseInt(process.argv[2] || 14000, 10),
+  days = parseInt(process.argv[3] || 7, 10);
+
+var es = require('../../../src/elasticsearch'),
+  _ = require('../../../src/lib/utils'),
+  q = require('q'),
+  async = require('async'),
+  moment = require('moment'),
+  makeSamples = require('./samples').make,
+  startingMoment = moment().startOf('day').subtract('days', days),
+  endingMoment = moment().endOf('day').add('days', days),
+  client = new es.Client({
+    log: 'info'
+  });
+
+client.log.info('Generating', count, 'events across ±', days, 'days');
+
+fillIndecies(function () {
+  var actions = [],
+    samples = makeSamples(startingMoment, endingMoment);
+
+  async.timesSeries(count, function (i, done) {
+    // random date, plus less random time
+    var date = moment(samples.randomMsInDayRange())
+      .utc()
+      .startOf('day')
+      .add('milliseconds', samples.lessRandomMsInDay());
+
+    var event = {
+      index: date.format('[logstash-]YYYY.MM.DD'),
+      '@timestamp': date.toISOString(),
+      ip: samples.ips(),
+      extension: samples.extensions(),
+      response: samples.responseCodes(),
+      country: samples.countries(),
+      point: samples.airports(),
+      '@tags': [samples.tags(), samples.tags2()],
+      utc_time: date.toISOString(),
+      referer: 'http://' + samples.referrers() + '/' + samples.tags() + '/' + samples.astronauts(),
+      agent: samples.userAgents(),
+    };
+
+    event.clientip = event.ip;
+    event.bytes = event.response < 500 ? samples.lessRandomRespSize() : 0;
+    event.request = '/' + samples.astronauts() + '.' + event.extension;
+    event.memory = event.extension === 'php' ? event.bytes * 40 : 0;
+    if (event.memory) {
+      event.phpmemory = event.memory;
+    }
+
+    event['@message'] = event.ip + ' - - [' + date.toISOString() + '] "GET ' + event.request + ' HTTP/1.1" ' +
+        event.respCode + ' ' + event.respSize + ' "-" "' + event.agent + '"';
+    event.src = JSON.stringify(event, null, '  ');
+
+    actions.push({
+      index: {
+        _index: event.index,
+        _type: samples.types(),
+        _id: i
+      }
+    });
+    actions.push(event);
+
+    if (actions.length === 3000 || i === count - 1) {
+      client.log.info('writing', actions.length / 2, 'documents');
+      client.bulk({
+        body: actions
+      }, done);
+      actions = [];
+    } else {
+      done();
+    }
+  }, function (err) {
+    if (err) {
+      throw err;
+    } else {
+      console.log('Done!');
+    }
+  });
+});
+
+function fillIndecies(cb) {
+  var movingDate = moment(startingMoment);
+  var indexBody = {
+    mappings: {
+      _default_: {
+        properties: {
+          '@timestamp': {
+            type: 'date',
+            index: 'not_analyzed'
+          },
+          id: {
+            type: 'integer',
+            index: 'not_analyzed',
+            include_in_all: false
+          },
+          country: {
+            type: 'string',
+            index: 'not_analyzed'
+          },
+          agent: {
+            type: 'multi_field',
+            fields: {
+              agent: {
+                type: 'string',
+                index: 'analyzed'
+              },
+              raw: {
+                type: 'string',
+                index: 'not_analyzed'
+              }
+            }
+          },
+          ip: {
+            type: 'ip'
+          }
+        }
+      }
+    }
+  },
+  indexPushActions = [];
+
+  function createDateIndex(indexName) {
+    return function (done) {
+      client.indices.create({
+        ignore: 400,
+        index: indexName,
+        body: indexBody
+      }, function (err, resp) {
+        if (err) {
+          done(err);
+        } else {
+          done(null, resp.error ? 'existed' : 'created');
+        }
+      });
+    };
+  }
+
+  while (movingDate.unix() < endingMoment.unix()) {
+    indexPushActions.push(createDateIndex(movingDate.format('[logstash-]YYYY.MM.DD')));
+    movingDate.add('day', 1);
+  }
+
+  async.parallel(indexPushActions, function (err, responses) {
+    if (err) {
+      client.log.error(err);
+    } else {
+      _.each(_.groupBy(responses), function (list, did) {
+        client.log.info(list.length, 'indicies', did);
+      });
+      cb();
+    }
+  });
+}

scripts/generate/logs/samples/astronauts.js

@@ -0,0 +1,559 @@
+module.exports = [
+  'Joseph M. Acaba',
+  'Loren Acton',
+  'Mike Adams',
+  'James Adamson',
+  'Viktor M. Afanasyev',
+  'Thomas Akers',
+  'Vladimir Aksyonov',
+  'Gemini 12',
+  'Aleksandar Panayotov Aleksandrov',
+  'Aleksandr Pavlovich Aleksandrov',
+  'Andrew M. Allen',
+  'Joseph P. Allen',
+  'Scott Altman',
+  'Apollo 8',
+  'Clayton Anderson',
+  'Michael P. Anderson',
+  'Claudie Haigneré',
+  'Dominic A. Antonelli',
+  'Jerome Apt',
+  'Lee Archambault',
+  'Gemini 8',
+  'Richard R. Arnold',
+  'Anatoly Artsebarsky',
+  'Yuri Artyukhin',
+  'Jeffrey Ashby',
+  'Oleg Atkov',
+  'Toktar Aubakirov',
+  'Sergei Avdeyev',
+  'James P. Bagian',
+  'Ellen S. Baker',
+  'Michael Baker',
+  'Aleksandr Balandin',
+  'Michael R. Barratt',
+  'Daniel Barry',
+  'John-David F. Bartoe',
+  'Charlie Bassett',
+  'Yuri Baturin',
+  'Patrick Baudry',
+  'Apollo 12',
+  'Robert L. Behnken',
+  'Ivan Bella',
+  'Pavel Belyayev',
+  'Georgi Beregovoi',
+  'Anatoli Berezovoy',
+  'John Blaha',
+  'Michael J. Bloomfield',
+  'Guion Bluford',
+  'Karol Bobko',
+  'Eric A. Boe',
+  'Charles Bolden',
+  'Roberta Bondar',
+  'Valentin Bondarenko',
+  'Andrei Borisenko',
+  'Gemini 7',
+  'Stephen G. Bowen',
+  'Kenneth Bowersox',
+  'Charles E. Brady, Jr.',
+  'Vance Brand',
+  'Daniel Brandenstein',
+  'Randolph Bresnik',
+  'Roy Bridges',
+  'Curtis Brown',
+  'David M. Brown',
+  'Mark Brown',
+  'James Buchli',
+  'Jay C. Buckey',
+  'Nikolai Budarin',
+  'John S. Bull',
+  'Daniel Burbank',
+  'Daniel Bursch',
+  'Valery Bykovsky',
+  'Robert D. Cabana',
+  'Yvonne Cagle',
+  'Fernando Caldeiro',
+  'Charles Camarda',
+  'Kenneth D. Cameron',
+  'Duane G. Carey',
+  'Scott Carpenter',
+  'Gerald P. Carr',
+  'Sonny Carter',
+  'John Casper',
+  'Christopher Cassidy',
+  'Robert J. Cenker',
+  'Gemini 9A',
+  'Roger B. Chaffee',
+  'Gregory Chamitoff',
+  'Franklin Chang-Diaz',
+  'Philip K. Chapman',
+  'Kalpana Chawla',
+  'Maurizio Cheli',
+  'Chen Quan',
+  'Leroy Chiao',
+  'Kevin P. Chilton',
+  'Jean-Loup Chrétien',
+  'Laurel B. Clark',
+  'Mary L. Cleave',
+  'Jean-François Clervoy',
+  'Michael R. Clifford',
+  'Michael Coats',
+  'Kenneth Cockrell',
+  'Catherine Coleman',
+  'Eileen Collins',
+  '',
+  'Gordon Cooper',
+  'Richard O. Covey',
+  'Timothy Creamer',
+  'John O. Creighton',
+  'Robert Crippen',
+  'Samantha Cristoforetti',
+  'Roger K. Crouch',
+  'Frank L. Culbertson, Jr.',
+  'Walter Cunningham',
+  'Robert Curbeam',
+  'Nancy J. Currie',
+  'Nancy Jan Davis',
+  'Lawrence J. DeLucas',
+  'Frank De Winne',
+  'Vladimir N. Dezhurov',
+  'Georgi Dobrovolski',
+  'Takao Doi',
+  'B. Alvin Drew',
+  'Brian Duffy',
+  'Apollo 16',
+  'Bonnie J. Dunbar',
+  'Pedro Duque',
+  'Samuel T. Durrance',
+  'James Dutton',
+  'Lev Dyomin',
+  'Tracy Caldwell Dyson',
+  'Vladimir Dzhanibekov',
+  'Joe Edwards',
+  'Donn F. Eisele',
+  'Anthony W. England',
+  'Joe H. Engle',
+  'Apollo 17',
+  'Reinhold Ewald',
+  'Léopold Eyharts',
+  'John Fabian',
+  'Muhammed Faris',
+  'Bertalan Farkas',
+  'Jean-Jacques Favier',
+  'Fèi Jùnlóng',
+  'Konstantin Feoktistov',
+  'Christopher Ferguson',
+  'Martin J. Fettman',
+  'Andrew J. Feustel',
+  'Anatoly Filipchenko',
+  'Michael Fincke',
+  'John L. Finley',
+  'Anna Lee Fisher',
+  'William Frederick Fisher',
+  'Klaus-Dietrich Flade',
+  'Michael Foale',
+  'Kevin A. Ford',
+  'Michael Foreman',
+  'Patrick Forrester',
+  'Michael Fossum',
+  'Ted Freeman',
+  'Stephen Frick',
+  'Dirk Frimout',
+  'Christer Fuglesang',
+  'Charles Fullerton',
+  'Reinhard Furrer',
+  'Satoshi Furukawa',
+  'F. Drew Gaffney',
+  'Yuri Gagarin',
+  'Ronald Garan',
+  'Dale Gardner',
+  'Guy Gardner',
+  'Marc Garneau',
+  'Owen Garriott',
+  'Charles Gemar',
+  'Michael Gernhardt',
+  'Alexander Gerst',
+  'Edward Gibson',
+  'Robert L. Gibson',
+  'Yuri Gidzenko',
+  'Ed Givens',
+  'Yuri Glazkov',
+  'John Glenn',
+  'Linda Godwin',
+  'Michael T. Good',
+  'Viktor Gorbatko',
+  'Gemini 11',
+  'Dominic Gorie',
+  'Ronald Grabe',
+  'Duane Graveline',
+  'Georgi Grechko',
+  'Frederick Gregory',
+  'William Gregory',
+  'David Griggs',
+  'Virgil I. "Gus" Grissom',
+  'John Grunsfeld',
+  'Aleksei Gubarev',
+  'Umberto Guidoni',
+  'Zhugderdemidiyn Gurragcha',
+  'Sidney Gutierrez',
+  'Chris Hadfield',
+  'Jean-Pierre Haigneré',
+  'Apollo 13',
+  'James Halsell',
+  'Kenneth Ham',
+  'Lloyd Hammond',
+  'Jeremy Hansen',
+  'Gregory Harbaugh',
+  'Bernard A. Harris, Jr.',
+  'Terry Hart',
+  'Henry Hartsfield',
+  'Frederick Hauck',
+  'Steven Hawley',
+  'Susan Helms',
+  'Karl Henize',
+  'Thomas Hennen',
+  'Terence Henricks',
+  'Miroslaw Hermaszewski',
+  'José Hernández',
+  'John Herrington',
+  'Richard Hieb',
+  'Joan Higginbotham',
+  'David Hilmers',
+  'Kathryn Hire',
+  'Charles Hobaugh',
+  'Jeffrey Hoffman',
+  'Donald Holmquest',
+  'Michael S. Hopkins',
+  'Scott Horowitz',
+  'Akihiko Hoshide',
+  'Millie Hughes-Fulford',
+  'Douglas G. Hurley',
+  'Rick Husband',
+  'Apollo 15',
+  'Aleksandr Ivanchenkov',
+  'Anatoli Ivanishin',
+  'Georgi Ivanov',
+  'Marsha Ivins',
+  'Sigmund Jähn',
+  'Mae Jemison',
+  'Tamara E. Jernigan',
+  'Brent W. Jett, Jr.',
+  'Jing Haipeng',
+  'Gregory C. Johnson',
+  'Gregory H. Johnson',
+  'Thomas D. Jones',
+  'Leonid Kadenyuk',
+  'Alexander Kaleri',
+  'Janet L. Kavandi',
+  'James M. Kelly',
+  'Mark Kelly',
+  'Scott Kelly',
+  'Joseph Kerwin',
+  'Yevgeny Khrunov',
+  'Robert S. Kimbrough',
+  'Leonid Kizim',
+  'Petr Klimuk',
+  'Pyotr Kolodin',
+  'Vladimir Komarov',
+  'Yelena Kondakova',
+  'Dmitri Kondratyev',
+  'Oleg Kononenko',
+  'Timothy L. Kopra',
+  'Mikhail Korniyenko',
+  'Valery Korzun',
+  'Oleg Kotov',
+  'Vladimir Kovalyonok',
+  'Konstantin Kozeyev',
+  'Kevin Kregel',
+  'Sergei Krikalev',
+  'Valeri Kubasov',
+  'André Kuipers',
+  'Aleksandr Laveykin',
+  'Robert Lawrence',
+  'Wendy Lawrence',
+  'Vasili Lazarev',
+  'Aleksandr Lazutkin',
+  'Valentin Lebedev',
+  'Mark C. Lee',
+  'David Leestma',
+  'William B. Lenoir',
+  'Aleksei Leonov',
+  'Frederick W. Leslie',
+  'Anatoli Levchenko',
+  'Byron Lichtenberg',
+  'Don Lind',
+  'Steven Lindsey',
+  'Jerry Linenger',
+  'Richard Linnehan',
+  'Gregory Linteris',
+  'Liu Boming',
+  'Liu Wang',
+  'Liu Yang',
+  'Yáng Lìwěi',
+  'Anthony Llewellyn',
+  'Paul Lockhart',
+  'Yuri Lonchakov',
+  'Michael Lopez-Alegria',
+  'Christopher Loria',
+  'John Lounge',
+  'Jack Lousma',
+  'Stanley G. Love',
+  'G. David Low',
+  'Edward Lu',
+  'Shannon Lucid',
+  'Vladimir Lyakhov',
+  'Steven MacLean',
+  'Sandra Magnus',
+  'Oleg Makarov',
+  'Yuri Malenchenko',
+  'Franco Malerba',
+  'Yuri Malyshev',
+  'Gennadi Manakov',
+  'Musa Manarov',
+  'Ravish Malhotra',
+  'Thomas Marshburn',
+  'Michael Massimino',
+  'Richard Mastracchio',
+  'K. Megan McArthur',
+  'William S. McArthur',
+  'Jon McBride',
+  'Bruce McCandless II',
+  'William C. McCool',
+  'Michael McCulley',
+  'James McDivitt',
+  'Donald McMonagle',
+  'Ronald McNair',
+  'Carl Meade',
+  'Bruce Melnick',
+  'Pamela Melroy',
+  'Leland D. Melvin',
+  'Ulf Merbold',
+  'Ernst Messerschmid',
+  'Dorothy M. Metcalf-Lindenburger',
+  'Curt Michel',
+  'Aleksandr Misurkin',
+  'Apollo 14',
+  'Andreas Mogensen',
+  'Abdul Ahad Mohmand',
+  'Mamoru Mohri',
+  'Barbara Morgan',
+  'Lee Morin',
+  'Boris Morukov',
+  'Chiaki Mukai',
+  'Richard Mullane',
+  'Talgat Musabayev',
+  'Story Musgrave',
+  'Steven R. Nagel',
+  'George Nelson',
+  'Grigori Nelyubov',
+  'Rodolfo Neri Vela',
+  'Paolo A. Nespoli',
+  'James H. Newman',
+  'Claude Nicollier',
+  'Niè Hǎishèng',
+  'Andriyan Nikolayev',
+  'Soichi Noguchi',
+  'Carlos I. Noriega',
+  'Oleg Novitskiy',
+  'Lisa Nowak',
+  'Karen Nyberg',
+  'Bryan O\'Connor',
+  'Ellen Ochoa',
+  'Wubbo Ockels',
+  'William Oefelein',
+  'Brian O\'Leary',
+  'John D. Olivas',
+  'Takuya Onishi',
+  'Ellison Onizuka',
+  'Yuri Onufrienko',
+  'Stephen Oswald',
+  'Robert Overmyer',
+  'Gennady Padalka',
+  'William Pailes',
+  'Scott Parazynski',
+  'Ronald A. Parise',
+  'Robert Parker',
+  'Luca Parmitano',
+  'Nicholas Patrick',
+  'Viktor Patsayev',
+  'James Pawelczyk',
+  'Julie Payette',
+  'Gary Payton',
+  'Timothy Peake',
+  'Philippe Perrin',
+  'Thomas Pesquet',
+  'Donald Peterson',
+  'Donald Pettit',
+  'Pham Tuan',
+  'John Phillips',
+  'William Pogue',
+  'Alan G. Poindexter',
+  'Mark Polansky',
+  'Alexander Poleshchuk',
+  'Valeri Polyakov',
+  'Marcos Pontes',
+  'Leonid Popov',
+  'Pavel Popovich',
+  'Charles Precourt',
+  'Dumitru Prunariu',
+  'Ilan Ramon',
+  'William Readdy',
+  'Kenneth Reightler',
+  'James F. Reilly',
+  'Garrett Reisman',
+  'Thomas Reiter',
+  'Vladimír Remek',
+  'Judith Resnik',
+  'Sergei Revin',
+  'Paul W. Richards',
+  'Richard N. Richards',
+  'Sally Ride',
+  'Patricia Robertson',
+  'Stephen Robinson',
+  'Russell L. Rogers',
+  'Roman Romanenko',
+  'Yuri Romanenko',
+  'Kent Rominger',
+  'Jerry L. Ross',
+  'Valery Rozhdestvensky',
+  'Nikolay Rukavishnikov',
+  'Mario Runco, Jr.',
+  'Sergei Ryazanski',
+  'Valery Ryumin',
+  'Albert Sacco',
+  'David Saint-Jacques',
+  'Aleksandr Samokutyayev',
+  'Gennadi Sarafanov',
+  'Robert Satcher',
+  'Sultan bin Salman bin Abdulaziz Al Saud',
+  'Viktor Savinykh',
+  'Svetlana Savitskaya',
+  'Wally Schirra',
+  'Hans Schlegel',
+  'Rusty Schweickart',
+  'Dick Scobee',
+  'Winston Scott',
+  'Paul Scully-Power',
+  'Richard Searfoss',
+  'Rhea Seddon',
+  'Elliot See',
+  'Ronald Sega',
+  'Piers Sellers',
+  'Aleksandr Serebrov',
+  'Vitali Sevastyanov',
+  'Yuri Shargin',
+  'Salizhan Sharipov',
+  'Rakesh Sharma',
+  'Vladimir Shatalov',
+  'Brewster Shaw',
+  'Mercury-Redstone 3',
+  'William Shepherd',
+  'Nancy Currie',
+  'Anton Shkaplerov',
+  'Georgi Shonin',
+  'Loren Shriver',
+  'Sheikh Muszaphar Shukor',
+  'Oleg Skripochka',
+  'Aleksandr Skvortsov',
+  'Donald "Deke" Slayton',
+  'Michael J. Smith',
+  'Steven Smith',
+  'Anatoly Solovyev',
+  'Vladimir Solovyov',
+  'Voskhod 1',
+  'Sherwood Spring',
+  'Robert Springer',
+  'Gemini 6A',
+  'Heidemarie Stefanyshyn-Piper',
+  'Robert Stewart',
+  'Susan Still Kilrain',
+  'Nicole Marie Passonno Stott',
+  'Krasimir Stoyanov',
+  'Gennady Strekalov',
+  'Frederick Sturckow',
+  'Kathryn Sullivan',
+  'Maksim Surayev',
+  'Steven Swanson',
+  'Arnaldo Tamayo Méndez',
+  'Daniel Tani',
+  'Joseph Tanner',
+  'Evgeny Tarelkin',
+  'MOL',
+  'Valentina Tereshkova',
+  'Norman Thagard',
+  'Gerhard Thiele',
+  'Robert Thirsk',
+  'Andrew Thomas',
+  'Donald Thomas',
+  'Stephen Thorne',
+  'Kathryn Thornton',
+  'William E. Thornton',
+  'Pierre Thuot',
+  'Gherman Titov',
+  'Vladimir Titov',
+  'Michel Tognini',
+  'Valery Tokarev',
+  'Sergei Treshchov',
+  'Eugene Trinh',
+  'Richard Truly',
+  'Bjarni Tryggvason',
+  'Vasili Tsibliyev',
+  'Mikhail Tyurin',
+  'Yury Usachov',
+  'Lodewijk van den Berg',
+  'James "Ox" van Hoften',
+  'Vladimir Vasyutin',
+  'Charles Veach',
+  'Franz Viehböck',
+  'Alexander Viktorenko',
+  'Pavel Vinogradov',
+  'Terry Virts',
+  'Roberto Vittori',
+  'Igor Volk',
+  'Alexander Volkov',
+  'Sergey Volkov',
+  'Vladislav Volkov',
+  'Boris Volynov',
+  'James Voss',
+  'Janice Voss',
+  'Koichi Wakata',
+  'Rex Walheim',
+  'Charles D. Walker',
+  'David M. Walker',
+  'Joseph A. Walker',
+  'Shannon Walker',
+  'Ulrich Walter',
+  'Carl Walz',
+  'Taylor Wang',
+  'Wang Yaping',
+  'Mary Weber',
+  'Paul Weitz',
+  'James Wetherbee',
+  'Douglas Wheelock',
+  'Edward White',
+  'Peggy Whitson',
+  'Terrence Wilcutt',
+  'Clifton Williams',
+  'Dafydd Williams',
+  'Donald Williams',
+  'Jeffrey Williams',
+  'Sunita "Suni" Williams',
+  'Barry Wilmore',
+  'Stephanie Wilson',
+  'Peter Wisoff',
+  'David Wolf',
+  'Neil Woodward',
+  'Naoko Yamazaki',
+  'Boris Yegorov',
+  'Aleksei Yeliseyev',
+  'Yi So-yeon',
+  'Gemini 3',
+  'Kimiya Yui',
+  'Fyodor Yurchikhin',
+  'Sergei Zalyotin',
+  'George D. Zamka',
+  'Zhai Zhigang',
+  'Zhang Xiaoguang',
+  'Vitaliy Zholobov',
+  'Vyacheslav Zudov'
+];

scripts/generate/logs/samples/countries.js

@@ -0,0 +1,250 @@
+module.exports = {
+  'CN': 1330044000,
+  'IN': 1173108018,
+  'US': 610232863,
+  'ID': 242968342,
+  'BR': 201103330,
+  'PK': 184404791,
+  'BD': 156118464,
+  'NG': 154000000,
+  'RU': 140702000,
+  'JP': 127288000,
+  'MX': 112468855,
+  'PH': 99900177,
+  'VN': 89571130,
+  'ET': 88013491,
+  'DE': 81802257,
+  'EG': 80471869,
+  'TR': 77804122,
+  'IR': 76923300,
+  'CD': 70916439,
+  'TH': 67089500,
+  'FR': 64768389,
+  'GB': 62348447,
+  'IT': 60340328,
+  'MM': 53414374,
+  'ZA': 49000000,
+  'KR': 48422644,
+  'ES': 46505963,
+  'UA': 45415596,
+  'CO': 44205293,
+  'TZ': 41892895,
+  'AR': 41343201,
+  'KE': 40046566,
+  'PL': 38500000,
+  'SD': 35000000,
+  'DZ': 34586184,
+  'CA': 33679000,
+  'UG': 33398682,
+  'MA': 31627428,
+  'PE': 29907003,
+  'IQ': 29671605,
+  'AF': 29121286,
+  'NP': 28951852,
+  'MY': 28274729,
+  'UZ': 27865738,
+  'VE': 27223228,
+  'SA': 25731776,
+  'GH': 24339838,
+  'YE': 23495361,
+  'KP': 22912177,
+  'TW': 22894384,
+  'SY': 22198110,
+  'MZ': 22061451,
+  'RO': 21959278,
+  'AU': 21515754,
+  'LK': 21513990,
+  'MG': 21281844,
+  'CI': 21058798,
+  'CM': 19294149,
+  'CL': 16746491,
+  'NL': 16645000,
+  'BF': 16241811,
+  'NE': 15878271,
+  'MW': 15447500,
+  'KZ': 15340000,
+  'EC': 14790608,
+  'KH': 14453680,
+  'ML': 13796354,
+  'GT': 13550440,
+  'ZM': 13460305,
+  'AO': 13068161,
+  'SN': 12323252,
+  'ZW': 11651858,
+  'CU': 11423000,
+  'RW': 11055976,
+  'GR': 11000000,
+  'CS': 10829175,
+  'PT': 10676000,
+  'TN': 10589025,
+  'TD': 10543464,
+  'CZ': 10476000,
+  'BE': 10403000,
+  'GN': 10324025,
+  'SO': 10112453,
+  'BO': 9947418,
+  'HU': 9930000,
+  'BI': 9863117,
+  'DO': 9823821,
+  'BY': 9685000,
+  'HT': 9648924,
+  'BJ': 9056010,
+  'SE': 9045000,
+  'AZ': 8303512,
+  'SS': 8260490,
+  'AT': 8205000,
+  'HN': 7989415,
+  'CH': 7581000,
+  'TJ': 7487489,
+  'IL': 7353985,
+  'RS': 7344847,
+  'BG': 7148785,
+  'HK': 6898686,
+  'TG': 6587239,
+  'LY': 6461454,
+  'JO': 6407085,
+  'PY': 6375830,
+  'LA': 6368162,
+  'PG': 6064515,
+  'SV': 6052064,
+  'NI': 5995928,
+  'ER': 5792984,
+  'KG': 5508626,
+  'DK': 5484000,
+  'SK': 5455000,
+  'SL': 5245695,
+  'FI': 5244000,
+  'NO': 5009150,
+  'AE': 4975593,
+  'TM': 4940916,
+  'CF': 4844927,
+  'SG': 4701069,
+  'GE': 4630000,
+  'IE': 4622917,
+  'BA': 4590000,
+  'CR': 4516220,
+  'HR': 4491000,
+  'MD': 4324000,
+  'NZ': 4252277,
+  'LB': 4125247,
+  'PR': 3916632,
+  'PS': 3800000,
+  'LR': 3685076,
+  'LT': 3565000,
+  'UY': 3477000,
+  'PA': 3410676,
+  'MR': 3205060,
+  'MN': 3086918,
+  'CG': 3039126,
+  'AL': 2986952,
+  'AM': 2968000,
+  'OM': 2967717,
+  'JM': 2847232,
+  'KW': 2789132,
+  'LV': 2217969,
+  'NA': 2128471,
+  'MK': 2061000,
+  'BW': 2029307,
+  'SI': 2007000,
+  'LS': 1919552,
+  'XK': 1800000,
+  'GM': 1593256,
+  'GW': 1565126,
+  'GA': 1545255,
+  'SZ': 1354051,
+  'MU': 1294104,
+  'EE': 1291170,
+  'TT': 1228691,
+  'TL': 1154625,
+  'CY': 1102677,
+  'GQ': 1014999,
+  'FJ': 875983,
+  'QA': 840926,
+  'RE': 776948,
+  'KM': 773407,
+  'GY': 748486,
+  'DJ': 740528,
+  'BH': 738004,
+  'BT': 699847,
+  'ME': 666730,
+  'SB': 559198,
+  'CV': 508659,
+  'LU': 497538,
+  'SR': 492829,
+  'MO': 449198,
+  'GP': 443000,
+  'MQ': 432900,
+  'MT': 403000,
+  'MV': 395650,
+  'BN': 395027,
+  'BZ': 314522,
+  'IS': 308910,
+  'BS': 301790,
+  'BB': 285653,
+  'EH': 273008,
+  'PF': 270485,
+  'VU': 221552,
+  'NC': 216494,
+  'GF': 195506,
+  'WS': 192001,
+  'ST': 175808,
+  'LC': 160922,
+  'GU': 159358,
+  'YT': 159042,
+  'CW': 141766,
+  'AN': 136197,
+  'TO': 122580,
+  'VI': 108708,
+  'GD': 107818,
+  'FM': 107708,
+  'VC': 104217,
+  'KI': 92533,
+  'JE': 90812,
+  'SC': 88340,
+  'AG': 86754,
+  'AD': 84000,
+  'IM': 75049,
+  'DM': 72813,
+  'AW': 71566,
+  'MH': 65859,
+  'BM': 65365,
+  'GG': 65228,
+  'AS': 57881,
+  'GL': 56375,
+  'MP': 53883,
+  'KN': 49898,
+  'FO': 48228,
+  'KY': 44270,
+  'SX': 37429,
+  'MF': 35925,
+  'LI': 35000,
+  'MC': 32965,
+  'SM': 31477,
+  'GI': 27884,
+  'AX': 26711,
+  'VG': 21730,
+  'CK': 21388,
+  'TC': 20556,
+  'PW': 19907,
+  'BQ': 18012,
+  'WF': 16025,
+  'AI': 13254,
+  'TV': 10472,
+  'NR': 10065,
+  'MS': 9341,
+  'BL': 8450,
+  'SH': 7460,
+  'PM': 7012,
+  'IO': 4000,
+  'FK': 2638,
+  'SJ': 2550,
+  'NU': 2166,
+  'NF': 1828,
+  'CX': 1500,
+  'TK': 1466,
+  'VA': 921,
+  'CC': 628,
+  'TF': 140,
+  'PN': 46,
+  'GS': 30
+};

scripts/generate/logs/samples/index.js

@@ -0,0 +1,93 @@
+var _ = require('../../../../src/lib/utils'),
+  WeightedList = require('./weighted_list'),
+  RandomList = require('./random_list'),
+  IpGenerator = require('./ip_generator'),
+  Stochator = require('./stochator'),
+  moment = require('moment'),
+  dayMs = 86400000;
+
+exports.make = function (startingMoment, endingMoment) {
+
+  var sets = {};
+
+  sets.randomMsInDayRange = new Stochator({
+    min: startingMoment.toDate().getTime(),
+    max: endingMoment.toDate().getTime()
+  }, 'get');
+
+  sets.lessRandomRespSize = new Stochator({
+    mean: 4500,
+    stddev: 4500 * 0.56,
+    min: 1500,
+    max: 10000
+  }, 'get');
+
+  sets.lessRandomMsInDay = new Stochator({
+    min: 0,
+    max: dayMs,
+    mean: dayMs / 2,
+    stdev: dayMs * 0.15,
+  }, 'get');
+
+  sets.astronauts = new RandomList(require('./astronauts').map(function (name) {
+    return name.replace(/\W+/g, '-').toLowerCase();
+  }));
+
+  sets.airports = new RandomList(require('./airports'));
+
+  sets.ips = new IpGenerator(100, 1000);
+
+  sets.countries = new WeightedList(require('./countries'));
+
+  sets.extensions = new WeightedList({
+    'html': 40,
+    'php': 30,
+    'png': 15,
+    'gif': 5,
+    'css': 10
+  });
+
+  sets.responseCodes = new WeightedList({
+    200: 92,
+    404: 5,
+    503: 3
+  });
+
+  sets.tags = new WeightedList({
+    'error': 6,
+    'warning': 10,
+    'success': 84
+  });
+
+  sets.tags2 = new WeightedList({
+    'security': 20,
+    'info': 75,
+    'login': 5
+  });
+
+  sets.timezones = new WeightedList({
+    '-07:00': 1
+  });
+
+  sets.referrers = new WeightedList({
+    'www.slate.com': 50,
+    'twitter.com': 35,
+    'facebook.com': 20,
+    'nytimes.com': 10
+  });
+
+  sets.userAgents = new WeightedList({
+    'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322)': 30,
+    'Mozilla/5.0 (X11; Linux i686) AppleWebKit/534.24 (KHTML, like Gecko) Chrome/11.0.696.50 Safari/534.24': 35,
+    'Mozilla/5.0 (X11; Linux x86_64; rv:6.0a1) Gecko/20110421 Firefox/6.0a1': 40
+  });
+
+  sets.types = new WeightedList({
+    'nginx': 1,
+    'apache': 4
+  });
+
+  return _.map(sets, function (set, name) {
+    return _.bindKey(set, 'get');
+  });
+};

scripts/generate/logs/samples/ip_generator.js

@@ -0,0 +1,36 @@
+module.exports = IpGenerator;
+
+var _ = require('../../../../src/lib/utils');
+
+function IpGenerator(maxCount, maxSessions) {
+  this.maxSessions = maxSessions;
+  this.maxCount = maxCount;
+
+  this.sessions = [];
+}
+
+IpGenerator.prototype.get = function () {
+  var session, index;
+  if (this.sessions.length < this.maxSessions) {
+    session = [this.makeRandom(), 0];
+    index = this.sessions.length;
+    this.sessions.push(session);
+  } else {
+    index = _.random(0, this.sessions.length - 1);
+    session = this.sessions[index];
+  }
+
+  if (session[1] > this.maxCount) {
+    this.sessions.splice(index, 1);
+    return this.getIp();
+  } else {
+    return session[0];
+  }
+};
+
+IpGenerator.prototype.makeRandom = function () {
+  return _.random(0, 255) + '.' +
+    _.random(0, 255) + '.' +
+    _.random(0, 255) + '.' +
+    _.random(0, 255);
+};

scripts/generate/logs/samples/random_list.js

@@ -0,0 +1,13 @@
+/**
+ * @class RandomList
+ */
+
+module.exports = RandomList;
+
+var _ = require('../../../../src/lib/utils');
+
+function RandomList(list) {
+  this.get = function () {
+    return list[Math.round(Math.random() * list.length)];
+  };
+}

scripts/generate/logs/samples/stochator.js

@@ -0,0 +1,464 @@
+(function() {
+  var Set, Stochator, callFunctions, floatGenerator, integerGenerator, inverseNormalCumulativeDistribution, isType, randomBoundedFloat, randomBoundedInteger, randomCharacter, randomColor, randomNormallyDistributedFloat, randomSetMember, randomSetMemberWithoutReplacement, randomWeightedSetMember, setGenerator, shuffleSet,
+    __indexOf = Array.prototype.indexOf || function(item) { for (var i = 0, l = this.length; i < l; i++) { if (i in this && this[i] === item) return i; } return -1; },
+    __slice = Array.prototype.slice;
+
+  Set = (function() {
+
+    function Set(values) {
+      this.values = values;
+      this.length = this.values.length;
+    }
+
+    Set.prototype.toString = function() {
+      return "[object Set]";
+    };
+
+    Set.prototype.copy = function() {
+      return this.values.slice(0, this.length);
+    };
+
+    Set.prototype.enumerate = function(depth) {
+      var d, digits, e, enumeration, enumerations, enumerationsLength, i;
+      if (depth == null) depth = this.length;
+      enumerationsLength = Math.pow(this.length, depth);
+      enumerations = [];
+      for (enumeration = 0; 0 <= enumerationsLength ? enumeration < enumerationsLength : enumeration > enumerationsLength; 0 <= enumerationsLength ? enumeration++ : enumeration--) {
+        e = enumeration;
+        digits = [];
+        for (i = 0; 0 <= depth ? i < depth : i > depth; 0 <= depth ? i++ : i--) {
+          d = e % this.length;
+          e -= d;
+          e /= this.length;
+          digits.push(this.values[d]);
+        }
+        enumerations.push(new Set(digits));
+      }
+      return new Set(enumerations);
+    };
+
+    Set.prototype.intersection = function(set) {
+      var value;
+      return new Set((function() {
+        var _i, _len, _ref, _results;
+        _ref = set.values;
+        _results = [];
+        for (_i = 0, _len = _ref.length; _i < _len; _i++) {
+          value = _ref[_i];
+          if (__indexOf.call(this.values, value) >= 0) _results.push(value);
+        }
+        return _results;
+      }).call(this));
+    };
+
+    Set.prototype.union = function(set) {
+      return new Set(this.values.concat(this.difference(set).values));
+    };
+
+    Set.prototype.difference = function(set) {
+      var value;
+      return new Set((function() {
+        var _i, _len, _ref, _results;
+        _ref = set.values;
+        _results = [];
+        for (_i = 0, _len = _ref.length; _i < _len; _i++) {
+          value = _ref[_i];
+          if (!(__indexOf.call(this.values, value) >= 0)) _results.push(value);
+        }
+        return _results;
+      }).call(this));
+    };
+
+    Set.prototype.symmetricDifference = function(set) {
+      return this.union(set).difference(this.intersection(set));
+    };
+
+    Set.prototype.reduce = function(iterator) {
+      return this.values.reduce(iterator);
+    };
+
+    Set.prototype.reverse = function() {
+      return new Set(this.copy().reverse());
+    };
+
+    Set.prototype.sort = function(compare) {
+      return this.copy().sort(compare);
+    };
+
+    Set.prototype.sum = function() {
+      var _ref;
+      return (_ref = this._sum) != null ? _ref : this._sum = this.reduce(function(a, b) {
+        return a + b;
+      });
+    };
+
+    Set.prototype.mean = function() {
+      var _ref;
+      return (_ref = this._mean) != null ? _ref : this._mean = this.sum() / this.length;
+    };
+
+    Set.prototype.stdev = function() {
+      var value, _ref;
+      return (_ref = this._stdev) != null ? _ref : this._stdev = Math.sqrt(new Set((function() {
+        var _i, _len, _ref2, _results;
+        _ref2 = this.values;
+        _results = [];
+        for (_i = 0, _len = _ref2.length; _i < _len; _i++) {
+          value = _ref2[_i];
+          _results.push(Math.pow(value - this.mean(), 2));
+        }
+        return _results;
+      }).call(this)).mean());
+    };
+
+    Set.prototype.get = function(index, dflt) {
+      if (this.values[index] != null) {
+        return this.values[index];
+      } else {
+        return dflt;
+      }
+    };
+
+    Set.prototype.each = function(iterator) {
+      var index, value, _len, _ref, _results;
+      _ref = this.values;
+      _results = [];
+      for (index = 0, _len = _ref.length; index < _len; index++) {
+        value = _ref[index];
+        _results.push(iterator(value, index));
+      }
+      return _results;
+    };
+
+    Set.prototype.map = function(iterator) {
+      var index, value;
+      return new Set((function() {
+        var _len, _ref, _results;
+        _ref = this.values;
+        _results = [];
+        for (index = 0, _len = _ref.length; index < _len; index++) {
+          value = _ref[index];
+          _results.push(iterator(value, index));
+        }
+        return _results;
+      }).call(this));
+    };
+
+    Set.prototype.pop = function(index) {
+      var value;
+      if (index == null) index = this.length - 1;
+      value = this.values[index];
+      this.values.splice(index, 1);
+      return value;
+    };
+
+    return Set;
+
+  })();
+
+  isType = function(type) {
+    return function(arg) {
+      return Object.prototype.toString.call(arg) === ("[object " + type + "]");
+    };
+  };
+
+  callFunctions = function(fns) {
+    var fn, _i, _len, _results;
+    _results = [];
+    for (_i = 0, _len = fns.length; _i < _len; _i++) {
+      fn = fns[_i];
+      _results.push(fn());
+    }
+    return _results;
+  };
+
+  randomBoundedFloat = function(min, max) {
+    var spread;
+    if (min == null) min = 0;
+    if (max == null) max = 1;
+    spread = max - min;
+    return Math.random() * spread + min;
+  };
+
+  randomBoundedInteger = function(min, max) {
+    var spread;
+    if (min == null) min = 0;
+    if (max == null) max = 1;
+    spread = 1 + max - min;
+    return Math.floor(Math.random() * spread) + min;
+  };
+
+  randomColor = function() {
+    var byte, mutator;
+    byte = {
+      kind: "integer",
+      min: 0,
+      max: 255
+    };
+    mutator = function(bytes) {
+      var blue, green, red;
+      red = bytes[0], green = bytes[1], blue = bytes[2];
+      return {
+        red: red,
+        green: green,
+        blue: blue
+      };
+    };
+    return new Stochator(byte, byte, byte, mutator).next;
+  };
+
+  randomNormallyDistributedFloat = function(mean, stdev, min, max) {
+    var float, seed;
+    seed = randomBoundedFloat();
+    float = inverseNormalCumulativeDistribution(seed) * stdev + mean;
+    if ((min != null) && (max != null)) {
+      return Math.min(max, Math.max(min, float));
+    } else {
+      return float;
+    }
+  };
+
+  randomCharacter = function(lowercase) {
+    var max, min, mutator, _ref;
+    _ref = lowercase ? [97, 122] : [65, 90], min = _ref[0], max = _ref[1];
+    mutator = function(charCode) {
+      return String.fromCharCode(charCode);
+    };
+    return new Stochator({
+      kind: "integer",
+      min: min,
+      max: max
+    }, mutator).next;
+  };
+
+  randomSetMember = function(set) {
+    var max;
+    max = set.length - 1;
+    return set.get(randomBoundedInteger(0, max));
+  };
+
+  randomSetMemberWithoutReplacement = function(set) {
+    if (!set.get(0)) return;
+    set.length -= 1;
+    return set.pop(randomBoundedInteger(0, set.length));
+  };
+
+  randomWeightedSetMember = function(set, weights) {
+    var float, member, weightSum, _ref;
+    _ref = [void 0, 0, randomBoundedFloat()], member = _ref[0], weightSum = _ref[1], float = _ref[2];
+    set.each(function(value, index) {
+      var weight;
+      if (member) return;
+      weight = weights.get(index);
+      if (float <= weightSum + weight && float >= weightSum) member = value;
+      return weightSum += weight;
+    });
+    return member;
+  };
+
+  inverseNormalCumulativeDistribution = function(probability) {
+    var base, coefficient, denomCoeffcients, denomMaxExponent, denominator, high, low, mapMaxExp, numCoefficients, numMaxExponent, numerator, _ref, _ref2;
+    high = probability > 0.97575;
+    low = probability < 0.02425;
+    if (low || high) {
+      numCoefficients = new Set([-7.784894002430293e-3, -3.223964580411365e-1, -2.400758277161838, -2.549732539343734, 4.374664141464968]);
+      denomCoeffcients = new Set([7.784695709041462e-3, 3.224671290700398e-1, 2.445134137142996, 3.754408661907416]);
+      _ref = [5, 4], numMaxExponent = _ref[0], denomMaxExponent = _ref[1];
+      coefficient = low ? 1 : -1;
+      base = Math.sqrt(-2 * Math.log(low ? probability : 1 - probability));
+    } else {
+      numCoefficients = new Set([-3.969683028665376e1, 2.209460984245205e2, -2.759285104469687e2, 1.383577518672690e2, -3.066479806614716e1, 2.506628277459239]);
+      denomCoeffcients = new Set([-5.447609879822406e1, 1.615858368580409e2, -1.556989798598866e2, 6.680131188771972e1, -1.328068155288572e1]);
+      _ref2 = [5, 5], numMaxExponent = _ref2[0], denomMaxExponent = _ref2[1];
+      coefficient = probability - 0.5;
+      base = Math.pow(coefficient, 2);
+    }
+    mapMaxExp = function(maxExp) {
+      return function(value, index) {
+        return value * Math.pow(base, maxExp - index);
+      };
+    };
+    numerator = numCoefficients.map(mapMaxExp(numMaxExponent)).sum();
+    denominator = denomCoeffcients.map(mapMaxExp(denomMaxExponent)).sum() + 1;
+    return coefficient * numerator / denominator;
+  };
+
+  shuffleSet = function(set) {
+    var index, randomIndex, tmp, values, _ref;
+    values = set.copy();
+    for (index = _ref = values.length - 1; _ref <= 0 ? index < 0 : index > 0; _ref <= 0 ? index++ : index--) {
+      randomIndex = randomBoundedInteger(0, index);
+      tmp = values[index];
+      values[index] = values[randomIndex];
+      values[randomIndex] = tmp;
+    }
+    return values;
+  };
+
+  floatGenerator = function(min, max, mean, stdev) {
+    if (mean && stdev) {
+      return function() {
+        return randomNormallyDistributedFloat(mean, stdev, min, max);
+      };
+    } else {
+      return function() {
+        return randomBoundedFloat(min, max);
+      };
+    }
+  };
+
+  integerGenerator = function(min, max) {
+    if (min == null) min = 0;
+    if (max == null) max = 1;
+    return function() {
+      return randomBoundedInteger(min, max);
+    };
+  };
+
+  setGenerator = function(values, replacement, shuffle, weights) {
+    var set, weightsSet;
+    if (replacement == null) replacement = true;
+    if (shuffle == null) shuffle = false;
+    if (weights == null) weights = null;
+    if (!values || !values.length) {
+      throw Error("Must provide a 'values' array for a set generator.");
+    }
+    set = new Set(values);
+    if (shuffle) {
+      return function() {
+        return shuffleSet(set);
+      };
+    } else if (replacement) {
+      if (weights) {
+        weightsSet = new Set(weights);
+        return function() {
+          return randomWeightedSetMember(set, weightsSet);
+        };
+      } else {
+        return function() {
+          return randomSetMember(set);
+        };
+      }
+    } else {
+      return function() {
+        return randomSetMemberWithoutReplacement(set);
+      };
+    }
+  };
+
+  Stochator = (function() {
+    var VERSION;
+
+    VERSION = "0.3.1";
+
+    function Stochator() {
+      var configs;
+      configs = 1 <= arguments.length ? __slice.call(arguments, 0) : [];
+      this.setGenerator(configs);
+    }
+
+    Stochator.prototype.createGenerator = function(config) {
+      var generator, max, mean, min, replacement, shuffle, stdev, values, weights;
+      if (config.kind == null) config.kind = "float";
+      generator = (function() {
+        switch (config.kind) {
+          case "float":
+            min = config.min, max = config.max, mean = config.mean, stdev = config.stdev;
+            return floatGenerator(min, max, mean, stdev);
+          case "integer":
+            return integerGenerator(config.min, config.max);
+          case "set":
+            values = config.values, replacement = config.replacement, shuffle = config.shuffle, weights = config.weights;
+            return setGenerator(values, replacement, shuffle, weights);
+          case "color":
+          case "rgb":
+            return randomColor(config.kind);
+          case "a-z":
+          case "A-Z":
+            return randomCharacter(config.kind === "a-z");
+        }
+      })();
+      if (!generator) {
+        throw Error("" + config.kind + " not a recognized kind.");
+      } else {
+        return generator;
+      }
+    };
+
+    Stochator.prototype.createGenerators = function(configs, mutator) {
+      var callGenerators, caller, config, generators,
+        _this = this;
+      if (configs[0] == null) configs[0] = {};
+      generators = (function() {
+        var _i, _len, _results;
+        _results = [];
+        for (_i = 0, _len = configs.length; _i < _len; _i++) {
+          config = configs[_i];
+          _results.push(this.createGenerator(config));
+        }
+        return _results;
+      }).call(this);
+      if (!mutator) {
+        callGenerators = generators.length === 1 ? function() {
+          return callFunctions(generators)[0];
+        } : function() {
+          return callFunctions(generators);
+        };
+      } else {
+        caller = generators.length === 1 ? function() {
+          return callFunctions(generators)[0];
+        } : function() {
+          return callFunctions(generators);
+        };
+        callGenerators = function() {
+          return _this.value = mutator.call(_this, caller(), _this.value);
+        };
+      }
+      return function(times) {
+        var time, _results;
+        if (times) {
+          _results = [];
+          for (time = 1; 1 <= times ? time <= times : time >= times; 1 <= times ? time++ : time--) {
+            _results.push(callGenerators());
+          }
+          return _results;
+        } else {
+          return callGenerators();
+        }
+      };
+    };
+
+    Stochator.prototype.setGenerator = function(configs) {
+      var config, generatorConfigs, mutator, name, _i, _len, _ref, _ref2;
+      generatorConfigs = [];
+      for (_i = 0, _len = configs.length; _i < _len; _i++) {
+        config = configs[_i];
+        if (isType("Object")(config)) {
+          generatorConfigs.push(config);
+        } else {
+          break;
+        }
+      }
+      _ref = configs.slice(generatorConfigs.length), name = _ref[0], mutator = _ref[1];
+      name || (name = "next");
+      if (isType("Function")(name)) {
+        _ref2 = ["next", name], name = _ref2[0], mutator = _ref2[1];
+      }
+      return this[name] = this.createGenerators(generatorConfigs, mutator);
+    };
+
+    Stochator.prototype.toString = function() {
+      return "[object Stochator]";
+    };
+
+    return Stochator;
+
+  })();
+
+  if (typeof module !== "undefined" && module !== null ? module.exports : void 0) {
+    module.exports = Stochator;
+  } else {
+    this.Stochator = Stochator;
+  }
+
+}).call(this);

scripts/generate/logs/samples/weighted_list.js

@@ -0,0 +1,121 @@
+/**
+ * @class WeightedList
+ * @constructor
+ */
+
+module.exports = WeightedList;
+
+var _ = require('../../../../src/lib/utils');
+
+function WeightedList(list) {
+  WeightedList.callSuper(this, arguments);
+  _.forEach(list, _.bindKey(this, 'push'));
+}
+_.inherits(WeightedList, Array);
+
+/**
+ * Add a value to the weighted list.
+ * @method add
+ * @param {Any} value
+ * @param {Number} [weight] Optional.  Defaults to 1.
+ * @return {Number} The index of the item that was added.
+ */
+WeightedList.prototype.push = function (weight, value) {
+  if (!weight && weight !== 0) {
+    weight = 1;
+  }
+
+  Array.prototype.push.call(this, {
+    value: value,
+    weight: weight
+  });
+
+  delete this._sum;
+  delete this._totals;
+
+  return this.length - 1;
+};
+
+WeightedList.prototype.get = function () {
+  return this[this._randomIndex()].value;
+};
+
+/**
+ * Returns an index by weighted random distribution.
+ * @method _randomIndex
+ * @protected
+ * @return {Number}
+ */
+WeightedList.prototype._randomIndex = function () {
+  var maximumIndex,
+    me = this,
+    middleIndex,
+    minimumIndex = 0,
+    random,
+    sum = me._sum,
+    total,
+    totals = me._totals;
+
+  if (!sum || !totals) {
+    me._update();
+
+    sum = me._sum;
+    totals = me._totals;
+
+    if (!sum || !totals || !totals.length) {
+      return null;
+    }
+  }
+
+  maximumIndex = totals.length - 1;
+  random = Math.random() * sum;
+
+  while (maximumIndex >= minimumIndex) {
+    middleIndex = (maximumIndex + minimumIndex) / 2;
+
+    if (middleIndex < 0) {
+      middleIndex = 0;
+    } else {
+      middleIndex = Math.floor(middleIndex);
+    }
+
+    total = totals[middleIndex];
+
+    if (random === total) {
+      middleIndex += 1;
+      break;
+    } else if (random < total) {
+      if (middleIndex && random > totals[middleIndex - 1]) {
+        break;
+      }
+
+      maximumIndex = middleIndex - 1;
+    } else if (random > total) {
+      minimumIndex = middleIndex + 1;
+    }
+  }
+
+  return middleIndex;
+};
+
+/**
+ * Updates chached data for achieving weighted random distribution.
+ * @method _update
+ * @chainable
+ * @protected
+ */
+WeightedList.prototype._update = function () {
+  var me = this,
+    sum = 0,
+    totals = [];
+
+  _.each(me, function (item) {
+    sum += item.weight;
+    totals.push(sum);
+  });
+
+  me._sum = sum;
+  me._totals = totals;
+
+  return me;
+};

scripts/generate_js_api/notes.js

@@ -1,2 +0,0 @@
-exports['cluster.node_info'] =
-  '//TODO: this enpoint ***needs*** work, many of the possible urls are not supported';

scripts/scratch.js

@@ -1,10 +0,0 @@
-var elasticsearch = require('../src/elasticsearch')
-  , EventEmitter = require('events').EventEmitter;
-
-var client = new elasticsearch.Client({
-  logger: 'trace'
-});
-
-console.log(EventEmitter.listenerCount(client.log, 'trace'));
-
-client.cluster.node_stats();

scripts/server.js

@@ -0,0 +1,29 @@
+var http = require('http'),
+    async = require('async');
+
+var server = http.createServer(function (req, resp) {
+  var closed, count = 0;
+
+  resp.on('close', function () {
+    closed = true;
+    console.log('response was closed');
+  });
+
+  process.removeAllListeners();
+
+  var interval = setInterval(function () {
+    if (count > 99 || resp.closed || closed) {
+      clearInterval(interval);
+      console.log('done writing', resp.socket.bytesWritten, 'bytes');
+      resp.end();
+    } else {
+      process.stdout.write('->');
+      resp.write('line of data, more to come... slowly!');
+      count++;
+    }
+  }, 100);
+});
+
+server.listen(7500, function () {
+  console.log('server listening at', server.address());
+});