lots of changes, should have committed earlier

2013-12-02 18:30:30 -07:00
parent 477f1aff84
commit 2b5f6cd80d
45 changed files with 2498 additions and 1935 deletions
--- a/scripts/export_client.js
+++ b/scripts/export_client.js
@ -14,7 +14,7 @@ var argv = require('optimist')
    o: 'outputDir',
    v: 'verbose'
  })
-  .parse(JSON.parse(process.env.npm_config_argv).original);
+  .argv;

 switch (argv.buildName) {
 case 'jquery':
--- a/scripts/export_docs.js
+++ b/scripts/export_docs.js
@ -0,0 +1,96 @@
+var path = require('path');
+var fs = require('fs');
+var format = require('util').format;
+var cp = require('child_process');
+
+var argv = require('optimist')
+  .default({
+    outputDir: '.',
+    verbose: false
+  })
+  .alias({
+    o: 'outputDir',
+    v: 'verbose'
+  })
+  .argv;
+
+var steps = [
+  [runInModule, 'node', [path.join(__dirname, './generate/js_api'), '--force']],
+  [
+    copy,
+    path.join(__dirname, '../docs/_methods.jade'),
+    path.join(argv.outputDir, '_methods.jade')
+  ],
+  [
+    copy,
+    path.join(__dirname, '../docs/_method_list.jade'),
+    path.join(argv.outputDir, '_method_list.jade')
+  ]
+];
+
+(function next() {
+  var step = steps.shift();
+  if (step) {
+    var fn = step.shift();
+    step.push(next);
+    fn.apply(null, step);
+  } else {
+    console.log('Done');
+    process.exit();
+  }
+})();
+
+function log() {
+  var out = format.apply(console, arguments);
+  if (argv.verbose) {
+    out = '\n' + out + '\n';
+  }
+  console.log(out);
+}
+
+function runInModule(cmd, args, exitCb) {
+  log('running', cmd, args.join(' '));
+
+  var proc = cp.spawn(cmd, args, {
+    stdio: argv.verbose ? 'inherit' : 'ignore'
+  });
+
+  proc.on('error', function (err) {
+    console.error('Error! --', err.message);
+    process.exit(1);
+  });
+
+  proc.on('exit', function (status) {
+    if (status) {
+      console.error('Error! --', cmd, 'exit status was', status);
+      process.exit(1);
+    } else {
+      exitCb();
+    }
+  });
+}
+
+function copy(from, to, done) {
+  log('copying', from, 'to', to);
+
+  var read = fs.createReadStream(from);
+  var write = fs.createWriteStream(to);
+
+  read.pipe(write);
+
+  read.on('error', function (err) {
+    console.error('unable to read: ' + from);
+    console.error(err.message);
+    process.exit(1);
+  });
+
+  write.on('error', function (err) {
+    console.error('unable to write to: ' + to);
+    console.error(err.message);
+    process.exit(1);
+  });
+
+  write.on('finish', function () {
+    done();
+  });
+}
--- a/scripts/generate/js_api/actions.js
+++ b/scripts/generate/js_api/actions.js
@ -198,24 +198,43 @@ function transformFile(entry) {
 /**
 * un-comment to print out the default method for any action that has multiple options
 */
-// module.exports = new EventEmitter();
+module.exports = new EventEmitter();
 // module.exports.on('ready', function (actions) {
 //   var longestName = 0;
-//   var lines = [];
+//   var reports = {
+//     multi_methods: [],
+//     get_with_body: []
+//   };
 //   actions.forEach(function (action) {
+//     var name;
+
 //     // console.log(action);
 //     if (action._methods.length > 1) {
-//       var name = action.name + ' (' + action._methods.join('/') + ')';
+//       name = action.name + ' (' + action._methods.join('/') + ')';
 //       longestName = Math.max(name.length, longestName);
-//       lines.push([name, action.spec.method || 'GET', action.docUrl]);
+//       reports.multi_methods.push([name, action.spec.method || 'GET', action.docUrl]);
+//     }
+
+//     if (action._methods.length === 1 && action._methods[0] === 'GET' && action.body) {
+//       name = action.name + ' (' + action._methods.join('/') + ')';
+//       longestName = Math.max(name.length, longestName);
+//       reports.get_with_body.push([name, action.spec.method || 'GET', action.docUrl]);
 //     }
 //   });

-//   lines.forEach(function (line) {
-//     var name = line[0];
-//     var def = line[1];
-//     var docUrl = line[2];
-//     var spacing = (new Array(longestName - name.length + 1)).join(' ');
-//     console.log(name + spacing + ' [' + def + (def.length === 3 ? ' ' : '') + ']  ->  ' + docUrl);
+//   Object.keys(reports).forEach(function (key) {
+//     console.log('\n' + key);
+//     if (reports[key].length) {
+//       reports[key].forEach(function (line) {
+//         var name = line[0];
+//         var def = line[1];
+//         var docUrl = line[2];
+//         var spacing = (new Array(longestName - name.length + 1)).join(' ');
+//         console.log(name + spacing + ' [' + def + (def.length === 3 ? ' ' : '') + ']  ->  ' + docUrl);
+//       });
+//     } else {
+//       console.log('--nada--');
+//     }
+//     console.log('\n');
 //   });
 // });
--- a/scripts/generate/js_api/index.js
+++ b/scripts/generate/js_api/index.js
@ -5,7 +5,7 @@ var clean = require('../../clean');
 var restSpecUpdated = require('../../rest_spec_updated');

 var outputPath = _.joinPath(__dirname, '../../../src/lib/api.js');
-var docOutputPath = _.joinPath(__dirname, '../../../docs/api.md');
+var docOutputDir = _.joinPath(__dirname, '../../../docs/');

 function download() {
  require('./actions').on('ready', function (actions) {
@ -32,14 +32,22 @@ function download() {
      namespaces: _.unique(namespaces.sort(), true)
    }));

-    fs.writeFileSync(docOutputPath, templates.apiDocs({
+    if (!fs.existsSync(docOutputDir)) {
+      fs.mkdirSync(docOutputDir);
+    }
+
+    fs.writeFileSync(docOutputDir + '_method_list.jade', templates.apiMethodList({
+      actions: actions
+    }));
+
+    fs.writeFileSync(docOutputDir + '_methods.jade', templates.apiMethods({
      actions: actions
    }));
  });
 }

 restSpecUpdated(function (err, updated) {
-  if (process.env.FORCE_GEN || process.env.npm_config_force || err || updated) {
+  if (err || updated) {
    download();
  }
 });
--- a/scripts/generate/js_api/templates/api_docs.tmpl
+++ b/scripts/generate/js_api/templates/api_docs.tmpl
@ -1,62 +0,0 @@
-# API
-
-## Table Of Contents
-
- - [Generic Params](#generic-params)
- - [Methods](#methods)<%
-
-function esc(str) {
-  return str.replace(/\|/g, '&#124;');
-}
-
-var _paramWithDefault = paramWithDefault;
-paramWithDefault = function (name, _default) {
-  return esc(_paramWithDefault(name, _default));
-};
-
-var _paramType = paramType;
-paramType = function (type) {
-  return esc(_paramType(type));
-};
-
-_.each(actions, function (action) {%>
-    - [<%= action.name %>](#<%= action.name.toLowerCase().replace(/[^\w ]/g, '').replace(/ /g, '-') %>)<%
-})
-
-%>
-
-## Generic Params
-
-Several parameters can be passed to any API method, and will control the way that those requests are carried out. These parameters are not listed in each method's param list.
-
-| Name | Type | Description |
-| ---- | ---- | ----------- |
-| `[timeout=10000]` | Number | The number of milliseconds this request has to complete. It defaults to the timeout specified at the client level, which defaults to 10 seconds. |
-| `ignore` | Number or Number[] | Don't treat these HTTP status codes as "errors". Example use cases could be `ignore: 404` or `ignore: [404]` |
-
-## Methods<%
-
-_.each(actions, function (action) {
-%>
-
-### <%= action.name %>()
-
-<%= action.docUrl %><%
-
-if (_.size(action.allParams)) { %>
-
-| Name | Type | Description |
-| ---- | ---- | ----------- |
-<%
-  _.each(action.allParams, function(param, paramName) {
-%>|`<%= paramWithDefault(paramName, param.default) %>` | <%= paramType(param.type) %> | <%= esc(param.description || '') %>|
-<% })
-
-} else {%>
-
-- none --
-<%
-
-}
-
-}); %>
--- a/scripts/generate/js_api/templates/api_method_list.tmpl
+++ b/scripts/generate/js_api/templates/api_method_list.tmpl
@ -0,0 +1,32 @@
+<%
+function esc(str) {
+  return str.replace(/\|/g, '&#124;');
+}
+
+var topActions = [];
+var names = {};
+
+_.each(actions, function (action) {
+  if (action.name.indexOf('.') > -1) {
+    var space = _.studlyCase(action.name.split('.').slice(0, -1).join('.'));
+    if (!names[space]) {
+      names[space] = [];
+    }
+    names[space].push(action);
+  } else {
+    topActions.push(action);
+  }
+}); %>
+
+ul<%
+_.each(topActions, function (action) {%>
+  li: a(href="api.html#<%= action.name.toLowerCase().replace(/[^\w]+/g, '-') %>") <%= action.name %><%
+});
+_.each(Object.keys(names).sort(), function (namespace) {%>
+  h3 <%= namespace %>
+  ul<%
+  _.each(names[namespace], function (action) {%>
+    li: a(href="api.html#<%= action.name.toLowerCase().replace(/[^\w]+/g, '-') %>") <%= action.name.replace(/^.*\./, '') %><%
+  })
+})
+%>
--- a/scripts/generate/js_api/templates/api_methods.tmpl
+++ b/scripts/generate/js_api/templates/api_methods.tmpl
@ -0,0 +1,40 @@
+<%
+_.each(actions, function (action) {
+var actionId = action.name.toLowerCase().replace(/[^\w]+/g, '-');
+
+%>
+
+h3#<%= actionId %>.fn
+  span.name <%= action.name %>
+  span.args (params, [callback])
+a.perma(href="api.html#<%= actionId %>", title="Permalink")
+a.esdoc(href="<%= action.docUrl %>", title="Endpoint Docs")
+//
+  h4 Spec:
+    pre
+      code <%= JSON.stringify(action, null, '  ').split('\n').map(function (line, i) {
+    return (i > 0 ? '      | ' : '') + line;
+  }).join('\n') %>
+
+h4 Params:
+ul.params.api
+<% _.each(action.allParams, function (param, paramName) { %>
+  li
+    code.name <%= paramWithDefault(paramName, param.default) %>
+    <%=
+      _.map(
+        ('<span class="types">' + paramType(param.type) + '</span> ' + (param.description || '')).split('\n'),
+        function (line) {
+          return '|  ' + line + '\n';
+        }
+      ).join('\n')
+    %>
+<% }); %>
+  li <a href="#api-conventions-params">the usual</a>
+
+h4 Method: <code><%= action.spec.method || 'GET' %></code>
+
+h4 Returns:
+p: a(href="#api-conventions-return") the usual<%
+});
+%>
--- a/scripts/generate/js_api/templates/index.js
+++ b/scripts/generate/js_api/templates/index.js
@ -52,13 +52,13 @@ var templateGlobals = {
  paramType: function (type) {
    switch (type && type.toLowerCase ? type.toLowerCase() : 'any') {
    case 'time':
-      return 'Date or Number';
+      return 'Date, Number';
    case 'any':
-      return '*';
+      return 'Anything';
    case 'enum':
      return 'String';
    case 'list':
-      return 'String or String[] or Boolean';
+      return 'String, String[], Boolean';
    default:
      return _.ucfirst(type);
    }
@ -91,5 +91,6 @@ templates.text = templates.string;

 module.exports = {
  apiFile: templates.api_file,
-  apiDocs: templates.api_docs
+  apiMethodList: templates.api_method_list,
+  apiMethods: templates.api_methods
 };
--- a/scripts/generate/logs/index.js
+++ b/scripts/generate/logs/index.js
@ -8,7 +8,7 @@ var argv = require('optimist')
      default: 14000
    },
    days: {
-      alias: 'c',
+      alias: 'd',
      type: 'number',
      required: true
    },
@ -22,22 +22,21 @@ var argv = require('optimist')
 // Error.stackTraceLimit = Infinity;
 // process.exit();

-var count = parseInt(argv._[0] || 14000, 10),
-  days = parseInt(argv._[1] || 7, 10);
-
-var es = require('../../../src/elasticsearch'),
-  _ = require('../../../src/lib/utils'),
-  async = require('async'),
-  moment = require('moment'),
-  makeSamples = require('./samples').make,
-  startingMoment = moment().startOf('day').subtract('days', days),
-  endingMoment = moment().endOf('day').add('days', days),
-  clientConfig = {
-    log: {
-      level: 'trace',
-      type: 'stdio'
-    }
-  };
+var es = require('../../../src/elasticsearch');
+var _ = require('../../../src/lib/utils');
+var async = require('async');
+var path = require('path');
+var moment = require('moment');
+var makeSamples = require('./samples').make;
+var startingMoment = moment().startOf('day').subtract('days', argv.days);
+var endingMoment = moment().endOf('day').add('days', argv.days);
+var clientConfig = {
+  log: {
+    level: 'trace',
+    type: 'file',
+    path: path.join(__dirname, '../../../log')
+  }
+};

 if (argv.host) {
  clientConfig.hosts = argv.host;
@ -47,13 +46,13 @@ if (argv.host) {

 var client = new es.Client(clientConfig);

-console.log('Generating', count, 'events across ±', days, 'days');
+console.log('Generating', argv.count, 'events across ±', argv.days, 'days');

 fillIndecies(function () {
-  var actions = [],
-    samples = makeSamples(startingMoment, endingMoment);
+  var actions = [];
+  var samples = makeSamples(startingMoment, endingMoment);

-  async.times(count, function (i, done) {
+  async.times(argv.count, function (i, done) {
    // random date, plus less random time
    var date = moment(samples.randomMsInDayRange())
      .utc()
@ -84,7 +83,6 @@ fillIndecies(function () {

    event['@message'] = event.ip + ' - - [' + date.toISOString() + '] "GET ' + event.request + ' HTTP/1.1" ' +
        event.response + ' ' + event.bytes + ' "-" "' + event.agent + '"';
-    event.src = JSON.stringify(event, null, '  ');

    actions.push({
      index: {
@ -95,7 +93,7 @@ fillIndecies(function () {
    });
    actions.push(event);

-    if (actions.length === 3000 || i === count - 1) {
+    if (actions.length === 3000 || i === argv.count - 1) {
      console.info('writing', actions.length / 2, 'documents');
      client.bulk({
        body: actions
@ -109,6 +107,7 @@ fillIndecies(function () {
      throw err;
    } else {
      console.log('Done!');
+      process.exit();
    }
  });
 });
--- a/scripts/generate/logs/samples/countries.js
+++ b/scripts/generate/logs/samples/countries.js
@ -1,250 +0,0 @@
-module.exports = {
-  'CN': 1330044000,
-  'IN': 1173108018,
-  'US': 610232863,
-  'ID': 242968342,
-  'BR': 201103330,
-  'PK': 184404791,
-  'BD': 156118464,
-  'NG': 154000000,
-  'RU': 140702000,
-  'JP': 127288000,
-  'MX': 112468855,
-  'PH': 99900177,
-  'VN': 89571130,
-  'ET': 88013491,
-  'DE': 81802257,
-  'EG': 80471869,
-  'TR': 77804122,
-  'IR': 76923300,
-  'CD': 70916439,
-  'TH': 67089500,
-  'FR': 64768389,
-  'GB': 62348447,
-  'IT': 60340328,
-  'MM': 53414374,
-  'ZA': 49000000,
-  'KR': 48422644,
-  'ES': 46505963,
-  'UA': 45415596,
-  'CO': 44205293,
-  'TZ': 41892895,
-  'AR': 41343201,
-  'KE': 40046566,
-  'PL': 38500000,
-  'SD': 35000000,
-  'DZ': 34586184,
-  'CA': 33679000,
-  'UG': 33398682,
-  'MA': 31627428,
-  'PE': 29907003,
-  'IQ': 29671605,
-  'AF': 29121286,
-  'NP': 28951852,
-  'MY': 28274729,
-  'UZ': 27865738,
-  'VE': 27223228,
-  'SA': 25731776,
-  'GH': 24339838,
-  'YE': 23495361,
-  'KP': 22912177,
-  'TW': 22894384,
-  'SY': 22198110,
-  'MZ': 22061451,
-  'RO': 21959278,
-  'AU': 21515754,
-  'LK': 21513990,
-  'MG': 21281844,
-  'CI': 21058798,
-  'CM': 19294149,
-  'CL': 16746491,
-  'NL': 16645000,
-  'BF': 16241811,
-  'NE': 15878271,
-  'MW': 15447500,
-  'KZ': 15340000,
-  'EC': 14790608,
-  'KH': 14453680,
-  'ML': 13796354,
-  'GT': 13550440,
-  'ZM': 13460305,
-  'AO': 13068161,
-  'SN': 12323252,
-  'ZW': 11651858,
-  'CU': 11423000,
-  'RW': 11055976,
-  'GR': 11000000,
-  'CS': 10829175,
-  'PT': 10676000,
-  'TN': 10589025,
-  'TD': 10543464,
-  'CZ': 10476000,
-  'BE': 10403000,
-  'GN': 10324025,
-  'SO': 10112453,
-  'BO': 9947418,
-  'HU': 9930000,
-  'BI': 9863117,
-  'DO': 9823821,
-  'BY': 9685000,
-  'HT': 9648924,
-  'BJ': 9056010,
-  'SE': 9045000,
-  'AZ': 8303512,
-  'SS': 8260490,
-  'AT': 8205000,
-  'HN': 7989415,
-  'CH': 7581000,
-  'TJ': 7487489,
-  'IL': 7353985,
-  'RS': 7344847,
-  'BG': 7148785,
-  'HK': 6898686,
-  'TG': 6587239,
-  'LY': 6461454,
-  'JO': 6407085,
-  'PY': 6375830,
-  'LA': 6368162,
-  'PG': 6064515,
-  'SV': 6052064,
-  'NI': 5995928,
-  'ER': 5792984,
-  'KG': 5508626,
-  'DK': 5484000,
-  'SK': 5455000,
-  'SL': 5245695,
-  'FI': 5244000,
-  'NO': 5009150,
-  'AE': 4975593,
-  'TM': 4940916,
-  'CF': 4844927,
-  'SG': 4701069,
-  'GE': 4630000,
-  'IE': 4622917,
-  'BA': 4590000,
-  'CR': 4516220,
-  'HR': 4491000,
-  'MD': 4324000,
-  'NZ': 4252277,
-  'LB': 4125247,
-  'PR': 3916632,
-  'PS': 3800000,
-  'LR': 3685076,
-  'LT': 3565000,
-  'UY': 3477000,
-  'PA': 3410676,
-  'MR': 3205060,
-  'MN': 3086918,
-  'CG': 3039126,
-  'AL': 2986952,
-  'AM': 2968000,
-  'OM': 2967717,
-  'JM': 2847232,
-  'KW': 2789132,
-  'LV': 2217969,
-  'NA': 2128471,
-  'MK': 2061000,
-  'BW': 2029307,
-  'SI': 2007000,
-  'LS': 1919552,
-  'XK': 1800000,
-  'GM': 1593256,
-  'GW': 1565126,
-  'GA': 1545255,
-  'SZ': 1354051,
-  'MU': 1294104,
-  'EE': 1291170,
-  'TT': 1228691,
-  'TL': 1154625,
-  'CY': 1102677,
-  'GQ': 1014999,
-  'FJ': 875983,
-  'QA': 840926,
-  'RE': 776948,
-  'KM': 773407,
-  'GY': 748486,
-  'DJ': 740528,
-  'BH': 738004,
-  'BT': 699847,
-  'ME': 666730,
-  'SB': 559198,
-  'CV': 508659,
-  'LU': 497538,
-  'SR': 492829,
-  'MO': 449198,
-  'GP': 443000,
-  'MQ': 432900,
-  'MT': 403000,
-  'MV': 395650,
-  'BN': 395027,
-  'BZ': 314522,
-  'IS': 308910,
-  'BS': 301790,
-  'BB': 285653,
-  'EH': 273008,
-  'PF': 270485,
-  'VU': 221552,
-  'NC': 216494,
-  'GF': 195506,
-  'WS': 192001,
-  'ST': 175808,
-  'LC': 160922,
-  'GU': 159358,
-  'YT': 159042,
-  'CW': 141766,
-  'AN': 136197,
-  'TO': 122580,
-  'VI': 108708,
-  'GD': 107818,
-  'FM': 107708,
-  'VC': 104217,
-  'KI': 92533,
-  'JE': 90812,
-  'SC': 88340,
-  'AG': 86754,
-  'AD': 84000,
-  'IM': 75049,
-  'DM': 72813,
-  'AW': 71566,
-  'MH': 65859,
-  'BM': 65365,
-  'GG': 65228,
-  'AS': 57881,
-  'GL': 56375,
-  'MP': 53883,
-  'KN': 49898,
-  'FO': 48228,
-  'KY': 44270,
-  'SX': 37429,
-  'MF': 35925,
-  'LI': 35000,
-  'MC': 32965,
-  'SM': 31477,
-  'GI': 27884,
-  'AX': 26711,
-  'VG': 21730,
-  'CK': 21388,
-  'TC': 20556,
-  'PW': 19907,
-  'BQ': 18012,
-  'WF': 16025,
-  'AI': 13254,
-  'TV': 10472,
-  'NR': 10065,
-  'MS': 9341,
-  'BL': 8450,
-  'SH': 7460,
-  'PM': 7012,
-  'IO': 4000,
-  'FK': 2638,
-  'SJ': 2550,
-  'NU': 2166,
-  'NF': 1828,
-  'CX': 1500,
-  'TK': 1466,
-  'VA': 921,
-  'CC': 628,
-  'TF': 140,
-  'PN': 46,
-  'GS': 30
-};
--- a/scripts/generate/logs/samples/countries.json
+++ b/scripts/generate/logs/samples/countries.json
@ -0,0 +1,250 @@
+{
+  "CN": 1330044000,
+  "IN": 1173108018,
+  "US": 610232863,
+  "ID": 242968342,
+  "BR": 201103330,
+  "PK": 184404791,
+  "BD": 156118464,
+  "NG": 154000000,
+  "RU": 140702000,
+  "JP": 127288000,
+  "MX": 112468855,
+  "PH": 99900177,
+  "VN": 89571130,
+  "ET": 88013491,
+  "DE": 81802257,
+  "EG": 80471869,
+  "TR": 77804122,
+  "IR": 76923300,
+  "CD": 70916439,
+  "TH": 67089500,
+  "FR": 64768389,
+  "GB": 62348447,
+  "IT": 60340328,
+  "MM": 53414374,
+  "ZA": 49000000,
+  "KR": 48422644,
+  "ES": 46505963,
+  "UA": 45415596,
+  "CO": 44205293,
+  "TZ": 41892895,
+  "AR": 41343201,
+  "KE": 40046566,
+  "PL": 38500000,
+  "SD": 35000000,
+  "DZ": 34586184,
+  "CA": 33679000,
+  "UG": 33398682,
+  "MA": 31627428,
+  "PE": 29907003,
+  "IQ": 29671605,
+  "AF": 29121286,
+  "NP": 28951852,
+  "MY": 28274729,
+  "UZ": 27865738,
+  "VE": 27223228,
+  "SA": 25731776,
+  "GH": 24339838,
+  "YE": 23495361,
+  "KP": 22912177,
+  "TW": 22894384,
+  "SY": 22198110,
+  "MZ": 22061451,
+  "RO": 21959278,
+  "AU": 21515754,
+  "LK": 21513990,
+  "MG": 21281844,
+  "CI": 21058798,
+  "CM": 19294149,
+  "CL": 16746491,
+  "NL": 16645000,
+  "BF": 16241811,
+  "NE": 15878271,
+  "MW": 15447500,
+  "KZ": 15340000,
+  "EC": 14790608,
+  "KH": 14453680,
+  "ML": 13796354,
+  "GT": 13550440,
+  "ZM": 13460305,
+  "AO": 13068161,
+  "SN": 12323252,
+  "ZW": 11651858,
+  "CU": 11423000,
+  "RW": 11055976,
+  "GR": 11000000,
+  "CS": 10829175,
+  "PT": 10676000,
+  "TN": 10589025,
+  "TD": 10543464,
+  "CZ": 10476000,
+  "BE": 10403000,
+  "GN": 10324025,
+  "SO": 10112453,
+  "BO": 9947418,
+  "HU": 9930000,
+  "BI": 9863117,
+  "DO": 9823821,
+  "BY": 9685000,
+  "HT": 9648924,
+  "BJ": 9056010,
+  "SE": 9045000,
+  "AZ": 8303512,
+  "SS": 8260490,
+  "AT": 8205000,
+  "HN": 7989415,
+  "CH": 7581000,
+  "TJ": 7487489,
+  "IL": 7353985,
+  "RS": 7344847,
+  "BG": 7148785,
+  "HK": 6898686,
+  "TG": 6587239,
+  "LY": 6461454,
+  "JO": 6407085,
+  "PY": 6375830,
+  "LA": 6368162,
+  "PG": 6064515,
+  "SV": 6052064,
+  "NI": 5995928,
+  "ER": 5792984,
+  "KG": 5508626,
+  "DK": 5484000,
+  "SK": 5455000,
+  "SL": 5245695,
+  "FI": 5244000,
+  "NO": 5009150,
+  "AE": 4975593,
+  "TM": 4940916,
+  "CF": 4844927,
+  "SG": 4701069,
+  "GE": 4630000,
+  "IE": 4622917,
+  "BA": 4590000,
+  "CR": 4516220,
+  "HR": 4491000,
+  "MD": 4324000,
+  "NZ": 4252277,
+  "LB": 4125247,
+  "PR": 3916632,
+  "PS": 3800000,
+  "LR": 3685076,
+  "LT": 3565000,
+  "UY": 3477000,
+  "PA": 3410676,
+  "MR": 3205060,
+  "MN": 3086918,
+  "CG": 3039126,
+  "AL": 2986952,
+  "AM": 2968000,
+  "OM": 2967717,
+  "JM": 2847232,
+  "KW": 2789132,
+  "LV": 2217969,
+  "NA": 2128471,
+  "MK": 2061000,
+  "BW": 2029307,
+  "SI": 2007000,
+  "LS": 1919552,
+  "XK": 1800000,
+  "GM": 1593256,
+  "GW": 1565126,
+  "GA": 1545255,
+  "SZ": 1354051,
+  "MU": 1294104,
+  "EE": 1291170,
+  "TT": 1228691,
+  "TL": 1154625,
+  "CY": 1102677,
+  "GQ": 1014999,
+  "FJ": 875983,
+  "QA": 840926,
+  "RE": 776948,
+  "KM": 773407,
+  "GY": 748486,
+  "DJ": 740528,
+  "BH": 738004,
+  "BT": 699847,
+  "ME": 666730,
+  "SB": 559198,
+  "CV": 508659,
+  "LU": 497538,
+  "SR": 492829,
+  "MO": 449198,
+  "GP": 443000,
+  "MQ": 432900,
+  "MT": 403000,
+  "MV": 395650,
+  "BN": 395027,
+  "BZ": 314522,
+  "IS": 308910,
+  "BS": 301790,
+  "BB": 285653,
+  "EH": 273008,
+  "PF": 270485,
+  "VU": 221552,
+  "NC": 216494,
+  "GF": 195506,
+  "WS": 192001,
+  "ST": 175808,
+  "LC": 160922,
+  "GU": 159358,
+  "YT": 159042,
+  "CW": 141766,
+  "AN": 136197,
+  "TO": 122580,
+  "VI": 108708,
+  "GD": 107818,
+  "FM": 107708,
+  "VC": 104217,
+  "KI": 92533,
+  "JE": 90812,
+  "SC": 88340,
+  "AG": 86754,
+  "AD": 84000,
+  "IM": 75049,
+  "DM": 72813,
+  "AW": 71566,
+  "MH": 65859,
+  "BM": 65365,
+  "GG": 65228,
+  "AS": 57881,
+  "GL": 56375,
+  "MP": 53883,
+  "KN": 49898,
+  "FO": 48228,
+  "KY": 44270,
+  "SX": 37429,
+  "MF": 35925,
+  "LI": 35000,
+  "MC": 32965,
+  "SM": 31477,
+  "GI": 27884,
+  "AX": 26711,
+  "VG": 21730,
+  "CK": 21388,
+  "TC": 20556,
+  "PW": 19907,
+  "BQ": 18012,
+  "WF": 16025,
+  "AI": 13254,
+  "TV": 10472,
+  "NR": 10065,
+  "MS": 9341,
+  "BL": 8450,
+  "SH": 7460,
+  "PM": 7012,
+  "IO": 4000,
+  "FK": 2638,
+  "SJ": 2550,
+  "NU": 2166,
+  "NF": 1828,
+  "CX": 1500,
+  "TK": 1466,
+  "VA": 921,
+  "CC": 628,
+  "TF": 140,
+  "PN": 46,
+  "GS": 30
+}
--- a/scripts/generate/logs/samples/weighted_list.js
+++ b/scripts/generate/logs/samples/weighted_list.js
@ -8,8 +8,11 @@ module.exports = WeightedList;
 var _ = require('../../../../src/lib/utils');

 function WeightedList(list) {
-  WeightedList.callSuper(this, arguments);
+  Array.call(this);
+
  _.forEach(list, _.bindKey(this, 'push'));
+
+  console.log(this);
 }
 _.inherits(WeightedList, Array);

--- a/scripts/generate/yaml_tests/index.js
+++ b/scripts/generate/yaml_tests/index.js
@ -32,7 +32,7 @@ function download() {


 restSpecUpdated(function (err, updated) {
-  if (process.env.FORCE_GEN || process.env.npm_config_force || err || updated) {
+  if (err || updated) {
    download();
  }
 });
--- a/scripts/rest_spec_updated.js
+++ b/scripts/rest_spec_updated.js
@ -7,6 +7,7 @@ var request = {
  }
 };
 var fs = require('fs');
+var _ = require('lodash');

 var lastRestSpecUpdateFile = __dirname + '/last_rest_spec_update.sha';
 var lastRestSpecUpdate;
@ -16,46 +17,61 @@ if (fs.existsSync(lastRestSpecUpdateFile)) {
  lastRestSpecUpdate = fs.readFileSync(lastRestSpecUpdateFile, 'utf8');
 }

-var req = https.get(request, function (incoming) {
-  if (incoming.statusCode !== 200) {
-    req.abort();
-    console.error('request for last commit failed', incoming.statusCode, incoming.headers);
-    return;
-  }
+var req = null;
+var force = false;

-  var body = '';
+if (process.env.npm_config_force ||
+    process.env.FORCE_GEN ||
+    _.contains(process.argv, '-f') ||
+    _.contains(process.argv, '--force')
+) {
+  force = true;
+}

-  incoming.on('data', onData);
-  incoming.on('end', onEnd);
-
-  function onData(chunk) {
-    body += chunk;
-  }
-
-  function onEnd() {
-    incoming.removeListener('data', onData);
-    incoming.removeListener('end', onEnd);
-    var _req = req;
-    req = null;
-
-    var resp;
-    try {
-      resp = JSON.parse(body);
-    } catch (e) {
-      console.log('unable to parse response from github');
-      _req.emit('ready');
+if (force) {
+  updated = true;
+} else {
+  req = https.get(request, function (incoming) {
+    if (incoming.statusCode !== 200) {
+      req.abort();
+      console.error('request for last commit failed', incoming.statusCode, incoming.headers);
      return;
    }

-    if (lastRestSpecUpdate === resp.sha) {
-      updated = false;
-    } else {
-      updated = true;
-      fs.writeFileSync(lastRestSpecUpdateFile, resp.sha);
+    var body = '';
+
+    incoming.on('data', onData);
+    incoming.on('end', onEnd);
+
+    function onData(chunk) {
+      body += chunk;
    }
-    _req.emit('ready');
-  }
-});
+
+    function onEnd() {
+      incoming.removeListener('data', onData);
+      incoming.removeListener('end', onEnd);
+      var _req = req;
+      req = null;
+
+      var resp;
+      try {
+        resp = JSON.parse(body);
+      } catch (e) {
+        console.log('unable to parse response from github');
+        _req.emit('ready');
+        return;
+      }
+
+      if (lastRestSpecUpdate === resp.sha) {
+        updated = false;
+      } else {
+        updated = true;
+        fs.writeFileSync(lastRestSpecUpdateFile, resp.sha);
+      }
+      _req.emit('ready');
+    }
+  });
+}

 module.exports = function (cb) {
  function done() {
--- a/scripts/run_browser_integration_suite/index.js
+++ b/scripts/run_browser_integration_suite/index.js
@ -5,69 +5,84 @@ var open = require('open');
 var fs = require('fs');
 var path = require('path');
 var async = require('async');
+var chalk = require('chalk');

 var yamlTestSourceFile = path.join(__dirname, '../../test/integration/yaml_suite/index.js');
-var yamlTestBundleFile = path.join(__dirname, '../../test/browser_integration/yaml_tests.js');
+var yamlTestBundleFile = path.join(__dirname, '../../test/integration/browser_yaml_suite/yaml_tests.js');
 var clientEntryFile = path.join(__dirname, '../../src/elasticsearch.js');

-var browsers = _.transform({
+var browserAppNames = _.transform({
  safari: {
    darwin: 'Safari'
  },
  chrome: {
    darwin: 'Google Chrome',
    win32: 'Google Chrome',
-    executable: 'google-chrome'
+    linux: 'google-chrome'
  },
  chromium: {
-
-    executable: 'chromium-browser',
+    linux: 'chromium-browser',
  },
  firefox: {
    darwin: 'Firefox',
    win32: 'Firefox',
-    executable: 'firefox'
+    linux: 'firefox'
  },
  opera: {
    darwin: 'Opera',
    win32: 'Opera',
-    executable: 'opera'
+    linux: 'opera'
  }
-}, function (browsers, config, name) {
+}, function (browserAppNames, config, name) {
  if (config[process.platform]) {
-    browsers[name] = config[process.platform];
+    browserAppNames[name] = config[process.platform];
    return;
  }

-  if (process.platform !== 'darwin' && process.platform !== 'win32' && config.executable) {
-    browsers[name] = config.executable;
+  if (process.platform !== 'darwin' && process.platform !== 'win32' && config.linux) {
+    browserAppNames[name] = config.executable;
    return;
  }
 }, {});

 var argv = require('optimist')
-  .default('browser', 'chrome')
-  .default('force_gen', false)
-  .boolean('force_gen')
-  .alias('f', 'force_gen')
-  .default('host', 'localhost')
-  .default('port', 9200)
+  .default({
+    browsers: '*',
+    forceGen: false,
+    host: 'localhost',
+    port: 9200
+  })
+  .boolean('forceGen')
+  .alias({
+    f: 'forceGen',
+    b: 'browsers',
+    h: 'host',
+    p: 'port'
+  })
  .argv;

-var browserAppName;
+server.browsers = [];
+
+if (argv.browsers === '*') {
+  server.browsers = _.keys(browserAppNames);
+} else {
+  argv.browsers.split(',').forEach(function (browser) {
+    server.browsers.push(browser);
+  });
+}
+
+var badKeys = _.difference(server.browsers, _.keys(browserAppNames));
+if (badKeys.length) {
+  console.error('Invalid keys: ' + badKeys.join(', '));
+  process.exit();
+} else {
+  console.log('opening browser suite in', server.browsers);
+}

 async.series([
-  function (done) {
-    if (browsers.hasOwnProperty(argv.browser)) {
-      browserAppName = browsers[argv.browser];
-      done();
-    } else {
-      done('--browser must be set to one of ' + _.keys(browsers).join(', ') + ' on this platform');
-    }
-  },
  function (done) {
    fs.exists('dist', function (yes) {
-      if (!argv.force_gen && yes) {
+      if (!argv.forceGen && yes) {
        done();
        return;
      }
@ -82,7 +97,7 @@ async.series([
  },
  function (done) {
    fs.exists(yamlTestBundleFile, function (yes) {
-      if (!argv.force_gen && yes) {
+      if (!argv.forceGen && yes) {
        done();
        return;
      }
@ -131,13 +146,32 @@ async.series([
      var port = server.address().port;
      console.log('server listening on port', port);

-      open('http://localhost:' + port + '?es_hostname=' + encodeURIComponent(argv.host) +
-           '&es_port=' + encodeURIComponent(argv.port) +
-           '&browser=' + encodeURIComponent(argv.browser), browserAppName);
+      async.eachSeries(_.clone(server.browsers), function (browser, done) {
+        open('http://localhost:' + port +
+          '?es_hostname=' + encodeURIComponent(argv.host) +
+          '&es_port=' + encodeURIComponent(argv.port) +
+          '&browser=' + encodeURIComponent(browser), browserAppNames[browser]);
+
+        server.once('browser complete', function () {
+          done();
+        });
+      });
    });

-    server.on('tests done', function (success) {
-      console.log('test completed', success ? 'successfully' : 'but failed');
+    server.on('tests done', function (report) {
+      var reports = [];
+      var success = true;
+      _.each(report, function (testSucceeded, browser) {
+        var msg = browser + ':' + (success ? '✔︎' : '⚑');
+        if (testSucceeded) {
+          msg = chalk.green(msg);
+        } else {
+          msg = chalk.red(msg);
+          success = false;
+        }
+        reports.push(' - ' + msg);
+      });
+      console.log('test completed!\n', reports.join('\n'));
      process.exit(success ? 0 : 1);
    });
  }
--- a/scripts/run_browser_integration_suite/server.js
+++ b/scripts/run_browser_integration_suite/server.js
@ -5,6 +5,7 @@ var fs = require('fs');
 var _ = require('lodash');
 var chalk = require('chalk');
 var makeJUnitXml = require('../make_j_unit_xml');
+var docRoot = path.join(__dirname, '../../test/integration/browser_yaml_suite');
 chalk.enabled = true;

 var middleware = [];
@ -17,7 +18,7 @@ var server = http.createServer(function (req, resp) {
  var parsedUrl = url.parse(req.url, true);
  req.uri = parsedUrl.pathname;
  req.query = parsedUrl.query;
-  req.filename = path.join(__dirname, '../../test/browser_integration/', req.uri);
+  req.filename = path.join(docRoot, req.uri);

  var end = resp.end;
  resp.end = function () {
@ -39,6 +40,11 @@ var server = http.createServer(function (req, resp) {
  next();
 });

+server.browsers = [
+  // browsers will go here, and will be removed once they call tests-complete
+  // once gone, we will emit "tests done"
+];
+
 function rand(length) {
  var str = '';
  while (str.length < length) {
@ -50,7 +56,7 @@ function rand(length) {
 function collectTestResults(req, resp) {
  var body = '';
  var browser = req.query.browser;
-  var logFilename = path.join(__dirname, '../../test-output-' + browser + '.xml');
+  var logFilename = 'test-output-' + browser + '.xml';

  req.on('data', function (chunk) {
    body += chunk;
@ -79,15 +85,33 @@ function collectTestResults(req, resp) {
      if (err) {
        console.log('unable to save test-output to', err.message);
        console.trace();
-        server.emit('tests done', false);
+        browserComplete(browser);
      } else {
        console.log('test output written to', logFilename);
-        server.emit('tests done', !testDetails.stats.failures);
+        browserComplete(browser, true);
      }
    });
  });
 }

+var report = {};
+
+function browserComplete(browser, success) {
+  var i = server.browsers.indexOf(browser);
+  report[browser] = success;
+  server.emit('browser complete', browser, success);
+  if (i >= 0) {
+    server.browsers.splice(i, 1);
+    if (server.browsers.length) {
+      console.log('waiting for', server.browsers.length, 'browsers');
+    } else {
+      server.emit('tests done', report);
+    }
+  } else {
+    console.error('invalid browser', browser);
+  }
+}
+

 middleware.push(function (req, resp, next) {
  // resolve filenames
@ -120,7 +144,7 @@ middleware.push(function (req, resp, next) {
      resp.end();
    } else {
      if (stats.isDirectory()) {
-        req.filename = path.join(req.filename, '../../test/browser_integration/index.html');
+        req.filename = path.join(docRoot, 'index.html');
      }
      next();
    }
@ -176,7 +200,7 @@ middleware.push(function (req, resp) {
        es_hostname: 'localhost',
        es_port: 9200,
        browser: 'unknown',
-        ts: 'no'//rand(5)
+        ts: rand(5)
      })));
    } else {
      resp.end(data);
--- a/scripts/run_tests.js
+++ b/scripts/run_tests.js
@ -9,13 +9,13 @@ var argv = require('optimist')
  ].join('\n'))
  .default({
    server: true,
-    browser: true,
    unit: true,
    integration: false,
    host: 'localhost',
    port: 9200,
    'xml-output': true,
-    'check-upstream': false
+    'check-upstream': false,
+    'browsers': '*'
  })
  .describe({
    host: 'hostname for elasticsearch instance used in integration tests',
@ -24,7 +24,7 @@ var argv = require('optimist')
  .alias({
    u: 'unit',
    i: 'integration',
-    b: 'browser',
+    b: 'browsers',
    s: 'server',
    x: 'xml-output'
  });
@ -46,7 +46,7 @@ var commands = [];

 if (argv['just-browser']) {
  argv.server = false;
-  argv.browser = true;
+  argv.browsers = '*';
 }

 if (argv['check-upstream']) {
@ -57,7 +57,7 @@ if (argv.unit) {
  if (argv.server) {
    commands.push(['mocha', 'test/unit/test_*.js', '--require=should']);
  }
-  if (argv.browser) {
+  if (argv.browsers) {
    commands.push(['testling', '.']);
  }
 }
@ -74,8 +74,13 @@ if (argv.integration) {
      '--port', argv.port
    ].filter(Boolean));
  }
-  if (argv.browser) {
-    commands.push(['node', 'scripts/run_browser_integration_suite/index.js']);
+  if (argv.browsers) {
+    commands.push([
+      'node',
+      'scripts/run_browser_integration_suite/index.js',
+      '--browsers',
+      argv.browsers
+    ]);
  }
 }