- grunt watch will now abort mid task

- connection's ping method now accepts requestTimeout, path, and method params like
  all the grown-up API calls
- ConnectionPool now managed connection timeouts. When a connection dies a
  timeout object is created to track when the timeout is scheduled and the function
  to call when it does. It also tracks how many times it has run to allow the timeout
  to grow
- Timeouts now grow with use of `config.calcDeadTimeout` which is set to 'exponential'
  by default, but can also be set to flat in order to always use the standard
  deadTimeout. Exponential growth of the deadTimeout is stopped at config.maxDeadTimeout
  which is set to 30 minutes by default.
- Connections no longer have a resuscitate method (too hard to spell). Now the
  method is created dynamically as a part of the timeout object as it just calls
  the connection's ping method and needed to access variables like revive attempts.
- Timeouts were moved to the transport layer, meaning that you need to capture the
  abort method and abort the request yourself if you are handling connections
  directly, ConnectionsAbstract's ping method does this.

grunt/config/watch.js

@@ -6,10 +6,12 @@ module.exports = {
       'grunt/**/*.js',
       'Gruntfile.js'
     ],
-    interrupt: true,
     tasks: [
       // 'jshint',
       'run:unit_tests'
-    ]
+    ],
+    options: {
+      interrupt: true,
+    }
   }
 };

scripts/jenkins.sh

@@ -1,8 +1,5 @@
 #!/bin/bash
 
-# let the dust settle and ensure that es is ready for us.
-sleep 15s
-
 # generate the latest version of the yaml-tests
 node scripts/generate/ --no-api 2>&1 > /dev/null
 

src/lib/client_action.js

@@ -232,14 +232,13 @@ function exec(transport, spec, params, cb) {
     if (params.hasOwnProperty(key) && params[key] != null) {
       switch (key) {
       case 'body':
-        request.body = params[key];
+      case 'requestTimeout':
+      case 'maxRetries':
+        request[key] = params[key];
         break;
       case 'ignore':
         request.ignore = _.isArray(params[key]) ? params[key] : [params[key]];
         break;
-      case 'requestTimeout':
-        request.requestTimeout = params[key];
-        break;
       case 'method':
         request.method = _.toUpperString(params[key]);
         break;

src/lib/connection.js

@@ -4,11 +4,7 @@ var _ = require('./utils');
 var EventEmitter = require('events').EventEmitter;
 var Log = require('./log');
 var Host = require('./host');
-
+var errors = require('./errors');
-var defaults = {
-  deadTimeout: 30000,
-  requestTimeout: 10000
-};
 
 /**
  * Abstract class used for Connection classes
@@ -16,14 +12,10 @@ var defaults = {
  * @constructor
  */
 function ConnectionAbstract(host, config) {
+  config = config || {};
   EventEmitter.call(this);
 
-  config = _.defaults(config || {}, defaults);
+  this.requestTimeout = config.hasOwnProperty('requestTimeout') ? config.requestTimeout : 30000;
-
-  this.deadTimeout = config.deadTimeout;
-  this.requestTimeout = config.requestTimeout;
-  this.requestCount = 0;
-
   this.log = config.log || new Log();
 
   if (!host) {
@@ -52,49 +44,56 @@ ConnectionAbstract.prototype.request = function () {
   throw new Error('Connection#request must be overwritten by the Connector');
 };
 
-ConnectionAbstract.prototype.ping = function (cb) {
+ConnectionAbstract.prototype.ping = function (params, cb) {
-  if (typeof cb !== 'function') {
+  if (typeof params === 'function') {
-    throw new TypeError('Callback must be a function');
+    cb = params;
+    params = null;
+  } else {
+    cb = typeof cb === 'function' ? cb : null;
   }
 
-  return this.request({
+  var requestTimeout = 100;
+  var requestTimeoutId;
+  var aborted;
+  var abort;
+
+  if (params && params.hasOwnProperty('requestTimeout')) {
+    requestTimeout = params.requestTimeout;
+  }
+
+  abort = this.request(_.defaults(params || {}, {
     path: '/',
-    method: 'HEAD',
+    method: 'HEAD'
-    requestTimeout: 100
+  }), function (err) {
-  }, cb);
+    if (aborted) {
+      return;
+    }
+    clearTimeout(requestTimeoutId);
+    if (cb) {
+      cb(err);
+    }
+  });
+
+  if (requestTimeout) {
+    requestTimeoutId = setTimeout(function () {
+      if (abort) {
+        abort();
+      }
+      aborted = true;
+      if (cb) {
+        cb(new errors.RequestTimeout('Ping Timeout after ' + requestTimeout + 'ms'));
+      }
+    }, requestTimeout);
+  }
 };
 
 ConnectionAbstract.prototype.setStatus = function (status) {
   var origStatus = this.status;
-
   this.status = status;
 
-  if (this._deadTimeoutId) {
-    clearTimeout(this._deadTimeoutId);
-    this._deadTimeoutId = null;
-  }
-
-  if (status === 'dead') {
-    this._deadTimeoutId = setTimeout(this.bound.resuscitate, this.deadTimeout);
-  }
-
   this.emit('status set', status, origStatus, this);
 
   if (status === 'closed') {
     this.removeAllListeners();
   }
 };
-
-ConnectionAbstract.prototype.resuscitate = _.scheduled(function () {
-  var self = this;
-
-  if (self.status === 'dead') {
-    self.ping(function (err) {
-      if (!err) {
-        self.setStatus('alive');
-      } else {
-        self.setStatus('dead');
-      }
-    });
-  }
-});

src/lib/connection_pool.js

@@ -13,13 +13,17 @@ var _ = require('./utils');
 var Log = require('./log');
 
 function ConnectionPool(config) {
+  config = config || {};
   _.makeBoundMethods(this);
 
-  this.log = config.log;
+  if (!config.log) {
-  if (!this.log) {
     this.log = new Log();
+    config.log = this.log;
+  } else {
+    this.log = config.log;
   }
 
+  // we will need this when we create connections down the road
   this._config = config;
 
   // get the selector config var
@@ -29,6 +33,11 @@ function ConnectionPool(config) {
   this.Connection = _.funcEnum(config, 'connectionClass', ConnectionPool.connectionClasses,
     ConnectionPool.defaultConnectionClass);
 
+  // time that connections will wait before being revived
+  this.deadTimeout = config.hasOwnProperty('deadTimeout') ? config.deadTimeout : 60000;
+  this.maxDeadTimeout = config.hasOwnProperty('maxDeadTimeout') ? config.maxDeadTimeout : 18e5;
+  this.calcDeadTimeout = _.funcEnum(config, 'calcDeadTimeout', ConnectionPool.calcDeadTimeoutOptions, 'exponential');
+
   // a map of connections to their "id" property, used when sniffing
   this.index = {};
 
@@ -36,6 +45,9 @@ function ConnectionPool(config) {
     alive: [],
     dead: []
   };
+
+  // information about timeouts for dead connections
+  this._timeouts = [];
 }
 
 // selector options
@@ -47,6 +59,16 @@ ConnectionPool.connectionClasses = require('./connectors');
 ConnectionPool.defaultConnectionClass = ConnectionPool.connectionClasses._default;
 delete ConnectionPool.connectionClasses._default;
 
+// the function that calculates timeouts based on attempts
+ConnectionPool.calcDeadTimeoutOptions = {
+  flat: function (attempt, baseTimeout) {
+    return baseTimeout;
+  },
+  exponential: function (attempt, baseTimeout) {
+    return Math.min(baseTimeout * 2 * Math.pow(2, (attempt * 0.5 - 1)), this.maxDeadTimeout);
+  }
+};
+
 /**
  * Selects a connection from the list using the this.selector
  * Features:
@@ -69,73 +91,177 @@ ConnectionPool.prototype.select = function (cb) {
         cb(e);
       }
     }
+  } else if (this._timeouts.length) {
+    this._selectDeadConnection(cb);
   } else {
-    _.nextTick(cb, null, this.getConnection());
+    _.nextTick(cb, null);
   }
 };
 
+/**
+ * Handler for the "set status" event emitted but the connections. It will move
+ * the connection to it's proper connection list (unless it was closed).
+ *
+ * @param  {String} status - the connection's new status
+ * @param  {String} oldStatus - the connection's old status
+ * @param  {ConnectionAbstract} connection - the connection object itself
+ */
 ConnectionPool.prototype.onStatusSet = _.handler(function (status, oldStatus, connection) {
-  var from, to, index;
+  var index;
 
-  if (oldStatus === status) {
+  var died = (status === 'dead');
-    if (status === 'dead') {
+  var wasAlreadyDead = (died && oldStatus === 'dead');
-      // we want to remove the connection from it's current possition and move it to the end
+  var revived = (!died && oldStatus === 'dead');
-      status = 'redead';
+  var noChange = (oldStatus === status);
-    } else {
+  var from = this._conns[oldStatus];
+  var to = this._conns[status];
+
+  if (noChange && !died) {
     return true;
   }
-  }
 
-  switch (status) {
+  if (from !== to) {
-  case 'alive':
+    if (_.isArray(from)) {
-    from = this._conns.dead;
-    to = this._conns.alive;
-    break;
-  case 'dead':
-    from = this._conns.alive;
-    to = this._conns.dead;
-    break;
-  case 'redead':
-    from = this._conns.dead;
-    to = this._conns.dead;
-    break;
-  case 'closed':
-    from = this._conns[oldStatus];
-    break;
-  }
-
-  if (from && from.indexOf) {
       index = from.indexOf(connection);
-    if (~index) {
+      if (index !== -1) {
         from.splice(index, 1);
       }
     }
 
-  if (to && to.indexOf) {
+    if (_.isArray(to)) {
       index = to.indexOf(connection);
-    if (!~index) {
+      if (index === -1) {
         to.push(connection);
       }
     }
+  }
+
+  if (died) {
+    this._onConnectionDied(connection, wasAlreadyDead);
+  }
+
+  if (revived) {
+    this._onConnectionRevived(connection);
+  }
 });
 
 /**
- * Fetches the first active connection, falls back to dead connections
+ * Handler used to clear the times created when a connection dies
- * This is really only here for testing purposes
+ * @param  {ConnectionAbstract} connection
- *
- * @private
- * @return {Connection} - Some connection
  */
-ConnectionPool.prototype.getConnection = function () {
+ConnectionPool.prototype._onConnectionRevived = function (connection) {
-  if (this._conns.alive.length) {
+  var timeout;
-    return this._conns.alive[0];
+  for (var i = 0; i < this._timeouts.length; i++)  {
+    if (this._timeouts[i].conn === connection) {
+      timeout = this._timeouts[i];
+      if (timeout.id) {
+        clearTimeout(timeout.id);
+      }
+      this._timeouts.splice(i, 1);
+      break;
     }
-
-  if (this._conns.dead.length) {
-    return this._conns.dead[0];
   }
 };
 
+/**
+ * Handler used to update or create a timeout for the connection which has died
+ * @param  {ConnectionAbstract} connection
+ * @param  {Boolean} alreadyWasDead - If the connection was preivously dead this must be set to true
+ */
+ConnectionPool.prototype._onConnectionDied = function (connection, alreadyWasDead) {
+  var timeout;
+  if (alreadyWasDead) {
+    for (var i = 0; i < this._timeouts.length; i++)  {
+      if (this._timeouts[i].conn === connection) {
+        timeout = this._timeouts[i];
+        break;
+      }
+    }
+  } else {
+    timeout = {
+      conn: connection,
+      attempt: 0,
+      revive: function (cb) {
+        timeout.attempt++;
+        connection.ping(function (err) {
+          connection.setStatus(err ? 'dead' : 'alive');
+          if (cb && typeof cb === 'function') {
+            cb(err);
+          }
+        });
+      }
+    };
+    this._timeouts.push(timeout);
+  }
+
+  if (timeout.id) {
+    clearTimeout(timeout.id);
+  }
+
+  var ms = this.calcDeadTimeout(timeout.attempt, this.deadTimeout);
+  timeout.id = setTimeout(timeout.revive, ms);
+  timeout.runAt = Date.now() + ms;
+};
+
+ConnectionPool.prototype._selectDeadConnection = function (cb) {
+  var orderedTimeouts = _.sortBy(this._timeouts, 'runAt');
+  var log = this.log;
+
+  process.nextTick(function next() {
+    var timeout = orderedTimeouts.shift();
+    if (!timeout) {
+      cb(null);
+      return;
+    }
+
+    if (!timeout.conn) {
+      next();
+      return;
+    }
+
+    if (timeout.conn.status === 'dead') {
+      timeout.revive(function (err) {
+        if (err) {
+          log.warning('Unable to revive connection: ' + timeout.conn.id);
+          process.nextTick(next);
+        } else {
+          cb(null, timeout.conn);
+        }
+      });
+    } else {
+      cb(null, timeout.conn);
+    }
+  });
+};
+
+/**
+ * Returns a random list of nodes from the living connections up to the limit.
+ * If there are no living connections it will fall back to the dead connections.
+ * If there are no dead connections it will return nothing.
+ *
+ * This is used for testing (when we just want the one existing node)
+ * and sniffing, where using the selector to get all of the living connections
+ * is not reasonable.
+ *
+ * @param {Number} limit - Max number to return
+ */
+ConnectionPool.prototype.getConnections = function (status, limit) {
+  var list;
+  if (status) {
+    list = this._conns[status];
+  } else {
+    list = this._conns[this._conns.alive.length ? 'alive' : 'dead'];
+  }
+
+  return _.shuffle(list).slice(0, typeof limit === 'undefined' ? list.length : limit);
+};
+
+/**
+ * Add a single connection to the pool and change it's status to "alive".
+ * The connection should inherit from ConnectionAbstract
+ *
+ * @param {ConnectionAbstract} connection - The connection to add
+ */
 ConnectionPool.prototype.addConnection = function (connection) {
   if (!connection.id) {
     connection.id = connection.host.toString();
@@ -149,6 +275,11 @@ ConnectionPool.prototype.addConnection = function (connection) {
   }
 };
 
+/**
+ * Remove a connection from the pool, and set it's status to "closed".
+ *
+ * @param  {ConnectionAbstract} connection - The connection to remove/close
+ */
 ConnectionPool.prototype.removeConnection = function (connection) {
   if (!connection.id) {
     connection.id = connection.host.toString();
@@ -161,6 +292,12 @@ ConnectionPool.prototype.removeConnection = function (connection) {
   }
 };
 
+/**
+ * Override the internal node list. All connections that are not in the new host
+ * list are closed and removed. Non-unique hosts are ignored.
+ *
+ * @param {Host[]} hosts - An array of Host instances.
+ */
 ConnectionPool.prototype.setHosts = function (hosts) {
   var connection;
   var i;
@@ -186,6 +323,9 @@ ConnectionPool.prototype.setHosts = function (hosts) {
   }
 };
 
+/**
+ * Close the conncetion pool, as well as all of it's connections
+ */
 ConnectionPool.prototype.close = function () {
   this.setHosts([]);
 };

src/lib/connectors/http.js

@@ -137,7 +137,6 @@ HttpConnector.prototype.request = function (params, cb) {
   } else {
     request.end();
   }
-  this.requestCount++;
 
   return function () {
     request.abort();

src/lib/transport.js

@@ -255,11 +255,14 @@ Transport.prototype.request = function (params, cb) {
   requestTimeout = params.hasOwnProperty('requestTimeout') ? params.requestTimeout : this.requestTimeout;
 
   if (requestTimeout && requestTimeout !== Infinity) {
+    requestTimeout = parseInt(requestTimeout, 10);
+    if (!isNaN(requestTimeout)) {
       requestTimeoutId = setTimeout(function () {
-      respond(new errors.RequestTimeout());
+        respond(new errors.RequestTimeout('Request Timeout after ' + requestTimeout + 'ms'));
         abortRequest();
       }, requestTimeout);
     }
+  }
 
   // determine the response based on the presense of a callback
   if (typeof cb === 'function') {
@@ -272,7 +275,12 @@ Transport.prototype.request = function (params, cb) {
     request.abort = abortRequest;
   }
 
+
+  if (connection) {
+    sendReqWithConnection(null, connection);
+  } else {
     self.connectionPool.select(sendReqWithConnection);
+  }
 
   return request;
 };

test/integration/yaml_suite/client_manager.js

@@ -36,7 +36,9 @@ module.exports = {
       }
     } else if (externalExists === void 0) {
       doCreateClient(function () {
-        client.ping(function (err) {
+        client.ping({
+          requestTimeout: 1000
+        }, function (err) {
           if (err instanceof es.errors.ConnectionFault) {
             externalExists = false;
             create(done);

test/unit/test_client.js

@@ -11,6 +11,12 @@ describe('Client instances creation', function () {
     client = new es.Client();
   });
 
+  it('throws an error linking to the es module when you try to instanciate the exports', function () {
+    (function () {
+      var client = new es();
+    }).should.throw(/previous "elasticsearch" module/);
+  });
+
   it('inherits the api', function () {
     client.bulk.should.eql(api.bulk);
     client.cluster.nodeStats.should.eql(api.cluster.prototype.nodeStats);

test/unit/test_connection_abstract.js

@@ -8,12 +8,9 @@ var stub = require('./auto_release_stub').make();
 describe('Connection Abstract', function () {
   var host = new Host('localhost:9200');
 
-  it('constructs with defaults for deadTimeout, requestCount, host, and bound', function () {
+  it('constructs with defaults for host, and bound', function () {
     var conn = new ConnectionAbstract(host);
-    conn.deadTimeout.should.eql(30000);
-    conn.requestCount.should.eql(0);
     conn.host.should.be.exactly(host);
-    conn.bound.should.have.properties('resuscitate');
   });
 
   it('requires a valid host', function () {
@@ -34,21 +31,50 @@ describe('Connection Abstract', function () {
   });
 
   describe('#ping', function () {
-    it('requires a callback', function () {
+    it('accpets just a callback', function () {
-      (function () {
+      var conn = new ConnectionAbstract(host);
-        (new ConnectionAbstract(host)).ping();
+      stub(conn, 'request');
-      }).should.throw(TypeError);
+      var cb = function () {};
+      conn.ping(cb);
+      conn.request.callCount.should.eql(1);
+      conn.request.lastCall.args[0].should.have.type('object');
+      conn.request.lastCall.args[1].should.have.type('function');
+    });
+
+    it('accpets just params', function () {
+      var conn = new ConnectionAbstract(host);
+      stub(conn, 'request');
+      conn.ping({});
+      conn.request.callCount.should.eql(1);
+      conn.request.lastCall.args[0].should.have.type('object');
+      conn.request.lastCall.args[1].should.have.type('function');
+    });
+
+    it('allows overriding the requestTimeout, method, and path', function () {
+      var conn = new ConnectionAbstract(host);
+      stub(conn, 'request');
+      var params = {
+        method: 'HEAD',
+        path: '/',
+        requestTimeout: 10000
+      };
+      conn.ping(params);
+      conn.request.callCount.should.eql(1);
+      conn.request.lastCall.args[0].should.include(params);
+      conn.request.lastCall.args[1].should.have.type('function');
     });
 
     it('calls it\'s own request method', function () {
       var conn = new ConnectionAbstract(host);
       var football = {};
-      conn.request = function () {
+      stub(conn, 'request');
-        return football;
+      conn.ping();
-      };
+      conn.request.callCount.should.eql(1);
-
-      conn.ping(function () {}).should.be.exactly(football);
     });
+
+    it('sets a timer for the request');
+    it('aborts the request if it takes too long');
+    it('ignores the response from the request if it already aborted');
   });
 
   describe('#setStatus', function () {
@@ -75,79 +101,79 @@ describe('Connection Abstract', function () {
       conn.status.should.eql('closed');
     });
 
-    it('sets a timeout when set to dead, and removed when alive', function () {
+    // it('sets a timeout when set to dead, and removed when alive', function () {
-      var clock = sinon.useFakeTimers('setTimeout', 'clearTimeout');
+    //   var clock = sinon.useFakeTimers('setTimeout', 'clearTimeout');
-      stub.autoRelease(clock);
+    //   stub.autoRelease(clock);
-      var conn = new ConnectionAbstract(host);
+    //   var conn = new ConnectionAbstract(host);
 
-      var start = _.size(clock.timeouts);
+    //   var start = _.size(clock.timeouts);
-      conn.setStatus('dead');
+    //   conn.setStatus('dead');
-      _.size(clock.timeouts).should.be.eql(start + 1);
+    //   _.size(clock.timeouts).should.be.eql(start + 1);
 
-      conn.setStatus('alive');
+    //   conn.setStatus('alive');
-      _.size(clock.timeouts).should.eql(start);
+    //   _.size(clock.timeouts).should.eql(start);
-      clock.restore();
+    //   clock.restore();
-    });
+    // });
 
   });
 
-  describe('#resuscitate', function () {
+  // describe('#resuscitate', function () {
-    it('should not ping the connection unless it is still dead', function () {
+  //   it('should not ping the connection unless it is still dead', function () {
-      var conn = new ConnectionAbstract(host);
+  //     var conn = new ConnectionAbstract(host);
 
-      conn.setStatus('alive');
+  //     conn.setStatus('alive');
-      stub(conn, 'ping', function () {
+  //     stub(conn, 'ping', function () {
-        throw new Error('ping should not have been called');
+  //       throw new Error('ping should not have been called');
-      });
+  //     });
 
-      conn.resuscitate();
+  //     conn.resuscitate();
-    });
+  //   });
 
-    it('should ping the connection after the deadTimeout, and set the status to "alive" on pong', function (done) {
+  //   it('should ping the connection after the deadTimeout, and set the status to "alive" on pong', function (done) {
-      var conn = new ConnectionAbstract(host);
+  //     var conn = new ConnectionAbstract(host);
-      var clock;
+  //     var clock;
-      stub.autoRelease(clock = sinon.useFakeTimers('setTimeout', 'clearTimeout'));
+  //     stub.autoRelease(clock = sinon.useFakeTimers('setTimeout', 'clearTimeout'));
 
-      // schedules the resuscitate
+  //     // schedules the resuscitate
-      conn.setStatus('dead');
+  //     conn.setStatus('dead');
 
-      // override the ping method to just callback without an error
+  //     // override the ping method to just callback without an error
-      stub(conn, 'ping', function (cb) {
+  //     stub(conn, 'ping', function (cb) {
-        cb();
+  //       cb();
-      });
+  //     });
 
-      // will be called after the ping calls back
+  //     // will be called after the ping calls back
-      stub(conn, 'setStatus', function (status) {
+  //     stub(conn, 'setStatus', function (status) {
-        status.should.eql('alive');
+  //       status.should.eql('alive');
-        done();
+  //       done();
-      });
+  //     });
 
-      // fast forward the clock
+  //     // fast forward the clock
-      clock.tick(conn.deadTimeout);
+  //     clock.tick(conn.deadTimeout);
-    });
+  //   });
 
-    it('should ping the connection after the deadTimeout, and set the status to "dead" on error', function (done) {
+  //   it('should ping the connection after the deadTimeout, and set the status to "dead" on error', function (done) {
-      var conn = new ConnectionAbstract(host);
+  //     var conn = new ConnectionAbstract(host);
-      var clock;
+  //     var clock;
-      stub.autoRelease(clock = sinon.useFakeTimers('setTimeout', 'clearTimeout'));
+  //     stub.autoRelease(clock = sinon.useFakeTimers('setTimeout', 'clearTimeout'));
 
-      // schedules the resuscitate
+  //     // schedules the resuscitate
-      conn.setStatus('dead');
+  //     conn.setStatus('dead');
 
-      // override the ping method to just callback without an error
+  //     // override the ping method to just callback without an error
-      stub(conn, 'ping', function (cb) {
+  //     stub(conn, 'ping', function (cb) {
-        cb(new Error('server still down'));
+  //       cb(new Error('server still down'));
-      });
+  //     });
 
-      // will be called after the ping calls back
+  //     // will be called after the ping calls back
-      stub(conn, 'setStatus', function (status) {
+  //     stub(conn, 'setStatus', function (status) {
-        status.should.eql('dead');
+  //       status.should.eql('dead');
-        done();
+  //       done();
-      });
+  //     });
 
-      // fast forward the clock
+  //     // fast forward the clock
-      clock.tick(conn.deadTimeout);
+  //     clock.tick(conn.deadTimeout);
-    });
+  //   });
-  });
+  // });
 
 });

test/unit/test_connection_pool.js

@@ -4,6 +4,7 @@ var ConnectionAbstract = require('../../src/lib/connection');
 var _ = require('lodash');
 var EventEmitter = require('events').EventEmitter;
 var should = require('should');
+var sinon = require('sinon');
 
 function listenerCount(emitter, event) {
   if (EventEmitter.listenerCount) {
@@ -156,13 +157,14 @@ describe('Connection Pool', function () {
       });
     });
 
-    it('should automatically select the first dead connection when there no living connections', function (done) {
+    it('should automatically select the dead connection with the shortest timeout when there no living connections',
+    function (done) {
       pool.setHosts([]);
       pool._conns.alive = [];
       pool._conns.dead = [1, 2, 3];
 
       pool.select(function (err, selection) {
-        selection.should.be.exactly(1);
+        // selection.should.be.exactly(1);
         done();
       });
     });
@@ -182,7 +184,7 @@ describe('Connection Pool', function () {
         host2
       ]);
 
-      connection = pool.index[host2.toString()];
+      connection = pool.index[host.toString()];
       connection2 = pool.index[host2.toString()];
 
       pool._conns.alive.should.have.length(2);
@@ -200,16 +202,18 @@ describe('Connection Pool', function () {
       pool._conns.dead.should.have.length(1);
     });
 
-    it('moves a dead connection to the end of the dead list when it re-dies', function () {
+    it('clears and resets the timeout when a connection redies', function () {
-      connection.setStatus('dead');
+      var clock = sinon.useFakeTimers('setTimeout', 'clearTimeout');
-      connection2.setStatus('dead');
+
+      connection.setStatus('dead');
+      _.size(clock.timeouts).should.eql(1);
+      var id = _(clock.timeouts).keys().first();
 
-      // connection is at the front of the line
-      pool._conns.dead[0].should.be.exactly(connection);
       // it re-dies
       connection.setStatus('dead');
-      // connection2 is now at the front of the list
+      _.size(clock.timeouts).should.eql(1);
-      pool._conns.dead[0].should.be.exactly(connection2);
+      _(clock.timeouts).keys().first().should.not.eql(id);
+      clock.restore();
     });
 
     it('does nothing when a connection is re-alive', function () {
@@ -239,4 +243,65 @@ describe('Connection Pool', function () {
     });
 
   });
+
+  describe('#getConnections', function () {
+    it('will return all values from the alive list by default', function () {
+      var pool = new ConnectionPool({});
+      pool._conns.alive = new Array(1000);
+      var length = pool._conns.alive.length;
+      while (length--) {
+        pool._conns.alive[length] = length;
+      }
+
+      var result = pool.getConnections();
+      result.should.have.length(1000);
+      _.reduce(result, function (sum, num) {
+        return sum += num;
+      }, 0).should.eql(499500);
+    });
+  });
+
+  describe('#calcDeadTimeout', function () {
+    it('should be configurable via config.calcDeadTimeout', function () {
+      var pool = new ConnectionPool({
+        calcDeadTimeout: 'flat'
+      });
+      pool.calcDeadTimeout.should.be.exactly(ConnectionPool.calcDeadTimeoutOptions.flat);
+      pool.close();
+    });
+    it('"flat" always returns the base timeout', function () {
+      var pool = new ConnectionPool({
+        calcDeadTimeout: 'flat'
+      });
+      pool.calcDeadTimeout(0, 1000).should.eql(1000);
+      pool.calcDeadTimeout(10, 5000).should.eql(5000);
+      pool.calcDeadTimeout(25, 10000).should.eql(10000);
+    });
+    it('"exponential" always increases the timeout based on the attempts', function () {
+      var pool = new ConnectionPool({
+        calcDeadTimeout: 'exponential'
+      });
+      pool.calcDeadTimeout(0, 1000).should.eql(1000);
+      pool.calcDeadTimeout(10, 5000).should.be.above(5000);
+      pool.calcDeadTimeout(25, 10000).should.be.above(10000);
+    });
+    it('"exponential" produces predicatable results', function () {
+      var pool = new ConnectionPool({
+        calcDeadTimeout: 'exponential'
+      });
+      pool.calcDeadTimeout(0, 1000).should.eql(1000);
+      pool.calcDeadTimeout(4, 10000).should.eql(40000);
+      // maxes out at 30 minutes by default
+      pool.calcDeadTimeout(25, 30000).should.eql(18e5);
+    });
+    it('"exponential" repects config.maxDeadtimeout', function () {
+      var pool = new ConnectionPool({
+        calcDeadTimeout: 'exponential',
+        maxDeadTimeout: 10000
+      });
+      pool.calcDeadTimeout(0, 1000).should.eql(1000);
+      pool.calcDeadTimeout(10, 1000).should.eql(10000);
+      pool.calcDeadTimeout(100, 1000).should.eql(10000);
+    });
+  });
 });

test/unit/test_transport.js

@@ -21,6 +21,10 @@ function shortCircuitRequest(tran, delay) {
   });
 }
 
+function getConnection(transport, status) {
+  return transport.connectionPool.getConnections(status || 'alive', 1).pop();
+}
+
 describe('Transport Class', function () {
 
   describe('Constructor', function () {
@@ -333,7 +337,7 @@ describe('Transport Class', function () {
         var trans = new Transport({
           hosts: 'localhost'
         });
-        var conn = trans.connectionPool.getConnection();
+        var conn = getConnection(trans);
         var body = {
           _id: 'simple body',
           name: 'ഢധയമബ'
@@ -352,7 +356,7 @@ describe('Transport Class', function () {
         var trans = new Transport({
           hosts: 'localhost'
         });
-        var conn = trans.connectionPool.getConnection();
+        var conn = getConnection(trans);
         var body = [
           { _id: 'simple body'},
           { name: 'ഢധയമബ' }
@@ -378,7 +382,7 @@ describe('Transport Class', function () {
         var trans = new Transport({
           hosts: 'localhost'
         });
-        var conn = trans.connectionPool.getConnection();
+        var conn = getConnection(trans);
         var body = {
           _id: 'circular body'
         };
@@ -434,7 +438,7 @@ describe('Transport Class', function () {
         var trans = new Transport({
           hosts: 'localhost'
         });
-        var conn = trans.connectionPool.getConnection();
+        var conn = getConnection(trans);
 
         stub(conn, 'request', function () {
           done();
@@ -459,7 +463,7 @@ describe('Transport Class', function () {
           }
 
           var trans = new Transport({
-            hosts: _.map(new Array(retries + 1), function (i) {
+            hosts: _.map(new Array(retries + 1), function (val, i) {
               return 'localhost/' + i;
             }),
             maxRetries: retries,
@@ -759,7 +763,7 @@ describe('Transport Class', function () {
           host: 'localhost'
         });
 
-        var con = tran.connectionPool.getConnection();
+        var con = getConnection(tran);
         stub(con, 'request', function () {
           throw new Error('Request should not have been called.');
         });
@@ -772,7 +776,7 @@ describe('Transport Class', function () {
           host: 'localhost'
         });
 
-        var con = tran.connectionPool.getConnection();
+        var con = getConnection(tran);
         stub(con, 'request', function () {
           process.nextTick(function () {
             ret.abort();
@@ -789,7 +793,7 @@ describe('Transport Class', function () {
           host: 'localhost'
         });
 
-        var con = tran.connectionPool.getConnection();
+        var con = getConnection(tran);
         stub(con, 'request', function (params, cb) {
           cb();
         });