udpated the log generator to pause when there is a bulk queue overflow

package.json

@@ -32,7 +32,7 @@
   },
   "devDependencies": {
     "mocha": "^1.18.2",
-    "async": "~0.2.9",
+    "async": "~0.8.0",
     "moment": "~2.4.0",
     "js-yaml": "~2.1.3",
     "optimist": "~0.6.0",

scripts/generate/logs/index.js

@@ -105,10 +105,6 @@ function createIndex(indexName) {
             index: 'not_analyzed',
             include_in_all: false
           },
-          country: {
-            type: 'string',
-            index: 'not_analyzed'
-          },
           agent: {
             type: 'multi_field',
             fields: {
@@ -196,9 +192,9 @@ function createIndex(indexName) {
 
 
 var queue = async.queue(function (events, done) {
-
   var body = [];
   var deps = [];
+  var esBulkQueueOverflow = 0;
 
   events.forEach(function (event) {
     var header = event.header;
@@ -224,23 +220,31 @@ var queue = async.queue(function (events, done) {
   })
   .then(function (resp) {
     if (resp.errors) {
-      var errors = [];
-
       resp.items.forEach(function (item, i) {
         if (item.index.error) {
-          errors.push(item.index.error);
+          if (item.index.error.match(/^EsRejectedExecutionException/)) {
-          eventBuffer.push(events[i]);
+            esBulkQueueOverflow ++;
+            eventBuffer.push(events[i]);
+          }
         }
       });
-
-      console.log('\n - errors - \n' + errors.join('\n') + '\n');
     }
   })
   .finally(function () {
-    process.stdout.write('.');
+    if (esBulkQueueOverflow) {
+      process.stdout.write('w' + esBulkQueueOverflow + '-');
+
+      // pause for 10ms per queue overage
+      queue.pause();
+      setTimeout(function () {
+        queue.resume();
+      }, 10 * esBulkQueueOverflow);
+
+    } else {
+      process.stdout.write('.');
+    }
   })
   .nodeify(done);
-
 }, 1);
 
 var eventBuffer = [];
@@ -265,89 +269,107 @@ queue.drain = function () {
   }
 };
 
-async.timesSeries(total, function (i, done) {
+async.series([
+  function (done) {
+    client.cluster.putSettings({
+      body: {
+        transient: {
+          threadpool: {
+            bulk: {
+              queue_size: -1
+            }
+          }
+        }
+      }
+    }, done);
+  },
+  function (done) {
+    async.timesSeries(total, function (i, done) {
+      // random date, plus less random time
+      var date = new Date(samples.randomMsInDayRange());
 
-  // random date, plus less random time
+      var ms = samples.lessRandomMsInDay();
-  var date = new Date(samples.randomMsInDayRange());
 
-  var ms = samples.lessRandomMsInDay();
+      // extract number of hours from the milliseconds
+      var hours = Math.floor(ms / 3600000);
+      ms = ms - hours * 3600000;
 
-  // extract number of hours from the milliseconds
+      // extract number of minutes from the milliseconds
-  var hours = Math.floor(ms / 3600000);
+      var minutes = Math.floor(ms / 60000);
-  ms = ms - hours * 3600000;
+      ms = ms - minutes * 60000;
 
-  // extract number of minutes from the milliseconds
+      // extract number of seconds from the milliseconds
-  var minutes = Math.floor(ms / 60000);
+      var seconds = Math.floor(ms / 1000);
-  ms = ms - minutes * 60000;
+      ms = ms - seconds * 1000;
 
-  // extract number of seconds from the milliseconds
+      // apply the values found to the date
-  var seconds = Math.floor(ms / 1000);
+      date.setUTCHours(hours, minutes, seconds, ms);
-  ms = ms - seconds * 1000;
 
-  // apply the values found to the date
+      var dateAsIso = date.toISOString();
-  date.setUTCHours(hours, minutes, seconds, ms);
+      var indexName = 'logstash-' +
+        dateAsIso.substr(0, 4) + '.' + dateAsIso.substr(5, 2) + '.' + dateAsIso.substr(8, 2);
+      var event = {};
 
-  var dateAsIso = date.toISOString();
+      event.index = indexName;
-  var indexName = 'logstash-' + dateAsIso.substr(0, 4) + '.' + dateAsIso.substr(5, 2) + '.' + dateAsIso.substr(8, 2);
+      event['@timestamp'] = dateAsIso;
-  var event = {};
+      event.ip = samples.ips();
+      event.extension = samples.extensions();
+      event.response = samples.responseCodes();
 
-  event.index = indexName;
+      event.geo = {
-  event['@timestamp'] = dateAsIso;
+        coordinates: samples.airports(),
-  event.ip = samples.ips();
+        src: samples.countries(),
-  event.extension = samples.extensions();
+        dest: samples.countries()
-  event.response = samples.responseCodes();
+      };
+      event.geo.srcdest = event.geo.src + ':' + event.geo.dest;
 
-  event.geo = {
+      event['@tags'] = [
-    coordinates: samples.airports(),
+        samples.tags(),
-    src: samples.countries(),
+        samples.tags2()
-    dest: samples.countries()
+      ];
-  };
+      event.utc_time = dateAsIso;
-  event.geo.srcdest = event.geo.src + ':' + event.geo.dest;
+      event.referer = 'http://' + samples.referrers() + '/' + samples.tags() + '/' + samples.astronauts();
+      event.agent = samples.userAgents();
+      event.clientip = event.ip;
+      event.bytes = event.response < 500 ? samples.lessRandomRespSize() : 0;
+      event.request = '/' + samples.astronauts() + '.' + event.extension;
+      if (event.extension === 'php') {
+        event.phpmemory = event.memory = event.bytes * 40;
+      }
+      event['@message'] = event.ip + ' - - [' + dateAsIso + '] "GET ' + event.request + ' HTTP/1.1" ' +
+          event.response + ' ' + event.bytes + ' "-" "' + event.agent + '"';
+      event.spaces = 'this   is   a   thing    with lots of     spaces       wwwwoooooo';
+      event.xss = '<script>console.log("xss")</script>';
+      event.headings = [
+        '<h3>' + samples.astronauts() + '</h5>',
+        'http://' + samples.referrers() + '/' + samples.tags() + '/' + samples.astronauts()
+      ];
+      event.links = [
+        samples.astronauts() + '@' + samples.referrers(),
+        'http://' + samples.referrers() + '/' + samples.tags2() + '/' + samples.astronauts(),
+        'www.' + samples.referrers()
+      ];
 
-  event['@tags'] = [
+      event.machine = {
-    samples.tags(),
+        os: samples.randomOs(),
-    samples.tags2()
+        ram: samples.randomRam()
-  ];
+      };
-  event.utc_time = dateAsIso;
+
-  event.referer = 'http://' + samples.referrers() + '/' + samples.tags() + '/' + samples.astronauts();
+      eventBuffer.push({
-  event.agent = samples.userAgents();
+        header: {
-  event.clientip = event.ip;
+          _index: event.index,
-  event.bytes = event.response < 500 ? samples.lessRandomRespSize() : 0;
+          _type: samples.types(),
-  event.request = '/' + samples.astronauts() + '.' + event.extension;
+          _id: i,
-  if (event.extension === 'php') {
+        },
-    event.phpmemory = event.memory = event.bytes * 40;
+        body: event
+      });
+
+      eventBuffer.flush();
+      setImmediate(done);
+    }, done);
   }
-  event['@message'] = event.ip + ' - - [' + dateAsIso + '] "GET ' + event.request + ' HTTP/1.1" ' +
+], function (err) {
-      event.response + ' ' + event.bytes + ' "-" "' + event.agent + '"';
+  if (err) throw err;
-  event.spaces = 'this   is   a   thing    with lots of     spaces       wwwwoooooo';
-  event.xss = '<script>console.log("xss")</script>';
-  event.headings = [
-    '<h3>' + samples.astronauts() + '</h5>',
-    'http://' + samples.referrers() + '/' + samples.tags() + '/' + samples.astronauts()
-  ];
-  event.links = [
-    samples.astronauts() + '@' + samples.referrers(),
-    'http://' + samples.referrers() + '/' + samples.tags2() + '/' + samples.astronauts(),
-    'www.' + samples.referrers()
-  ];
-
-  event.machine = {
-    os: samples.randomOs(),
-    ram: samples.randomRam()
-  };
-
-  eventBuffer.push({
-    header: {
-      _index: event.index,
-      _type: samples.types(),
-      _id: i,
-    },
-    body: event
-  });
-
-  eventBuffer.flush();
-  setImmediate(done);
-}, function () {
   doneCreatingEvents = true;
   eventBuffer.flush();
-});
+});