From 01d233a42871967557037437d827b38266a24d91 Mon Sep 17 00:00:00 2001 From: Spencer Alger Date: Fri, 22 Aug 2014 14:34:14 -0700 Subject: [PATCH 1/3] updated readme to point to new jenkins --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index c609f97f1..28858ce01 100644 --- a/README.md +++ b/README.md @@ -53,7 +53,7 @@ Check out the [Browser Builds](http://www.elasticsearch.org/guide/en/elasticsear ## Supported Elasticsearch Versions -[![Supporting Elasticsearch Version 0.90 to 1.3](http://img.shields.io/badge/elasticsearch-0.90--1.3-green.svg)](http://build.elasticsearch.com/job/es-js_nightly/) +[![Supporting Elasticsearch Version 0.90 to 1.3](http://img.shields.io/badge/elasticsearch-0.90--1.3-green.svg)](http://build-eu-1.elasticsearch.com/job/es-js_nightly/) Elasticsearch.js provides support for, and is regularly tested against, Elasticsearch releases 0.90.12 and greater. We also test against the latest changes in several branches in the Elasticsearch repository. To tell the client which version of Elastisearch you are using, and therefore the API it should provide, set the `apiVersion` config param. [More info](http://www.elasticsearch.org/guide/en/elasticsearch/client/javascript-api/current/configuration.html#_config_options) From aec0333a9198cb67aaab5144f6e98cc04db0a40a Mon Sep 17 00:00:00 2001 From: Marcelo Gornstein Date: Tue, 2 Sep 2014 13:05:14 -0300 Subject: [PATCH 2/3] This will add the capability for handling gzipped content to the http connector. On one hand, the response is handled with buffers instead of assuming it is a string, and then depending on the presence of the "content-encoding" header, will try to uncompress the response or return it as it is. Either way, it will try to handle the final result as an utf8 encoded string --- src/lib/connectors/http.js | 24 +++++++++++++---- test/unit/specs/http_connector.js | 44 +++++++++++++++++++++++++++++++ 2 files changed, 63 insertions(+), 5 deletions(-) diff --git a/src/lib/connectors/http.js b/src/lib/connectors/http.js index 3bb0d98ea..46badfd70 100644 --- a/src/lib/connectors/http.js +++ b/src/lib/connectors/http.js @@ -16,6 +16,7 @@ var _ = require('../utils'); var qs = require('querystring'); var ForeverAgent = require('./_custom_agent'); var ConnectionAbstract = require('../connection'); +var zlib = require('zlib'); /** * Connector used to talk to an elasticsearch node via HTTP @@ -123,8 +124,9 @@ HttpConnector.prototype.request = function (params, cb) { var request; var response; var status = 0; - var headers; + var headers = {}; var log = this.log; + var buffers = []; var reqParams = this.makeReqParams(params); @@ -144,7 +146,20 @@ HttpConnector.prototype.request = function (params, cb) { if (err) { cb(err); } else { - cb(err, response, status, headers); + response = Buffer.concat(buffers); + if (headers['content-encoding'] && headers['content-encoding'].match(/gzip/i)) { + zlib.unzip(response, function(gzErr, uncompressedResponse) { + if(gzErr) { + err = gzErr; + response = response.toString('binary'); + } else { + response = uncompressedResponse.toString('utf8'); + } + cb(err, response, status, headers); + }); + } else { + cb(err, response.toString('utf8'), status, headers); + } } }, this); @@ -152,11 +167,10 @@ HttpConnector.prototype.request = function (params, cb) { incoming = _incoming; status = incoming.statusCode; headers = incoming.headers; - incoming.setEncoding('utf8'); response = ''; - + buffers = []; incoming.on('data', function (d) { - response += d; + buffers.push(new Buffer(d)); }); incoming.on('error', cleanUp); diff --git a/test/unit/specs/http_connector.js b/test/unit/specs/http_connector.js index 63d7f8839..5c629c6b5 100644 --- a/test/unit/specs/http_connector.js +++ b/test/unit/specs/http_connector.js @@ -17,6 +17,8 @@ describe('Http Connector', function () { var expectSubObject = require('../../utils/expect_sub_object'); var MockRequest = require('../../mocks/request'); var MockIncommingMessage = require('../../mocks/incomming_message'); + var zlib = require('zlib'); + var estr = require('event-stream'); nock.disableNetConnect(); @@ -302,6 +304,48 @@ describe('Http Connector', function () { }); }); + it('collects the whole request body (compressed)', function (done) { + var server = nock('http://esjs.com:9200'); + var con = new HttpConnection(new Host('http://esjs.com:9200')); + var body = '{ "USER": "doc" }'; + zlib.deflate(body, function(err, compressedBody) { + server + .get('/users/1') + .reply(200, compressedBody, {'Content-Encoding': 'gzip'}); + + con.request({ + method: 'GET', + path: '/users/1' + }, function (err, resp, status) { + expect(err).to.be(undefined); + expect(resp).to.eql(body); + expect(status).to.eql(200); + server.done(); + done(); + }); + }); + }); + + it('Can handle uncompress errors', function (done) { + var server = nock('http://esjs.com:9200'); + var con = new HttpConnection(new Host('http://esjs.com:9200')); + var body = 'blah'; + server + .get('/users/1') + .reply(200, body, {'Content-Encoding': 'gzip'}); + + con.request({ + method: 'GET', + path: '/users/1' + }, function (err, resp, status) { + expect(err.errno).to.be(-3); + expect(resp).to.eql(body); + expect(status).to.eql(200); + server.done(); + done(); + }); + }); + it('Ignores serialization errors', function (done) { var server = nock('http://esjs.com:9200'); var con = new HttpConnection(new Host('http://esjs.com:9200')); From 83b6400000b7a9ed7895af7bec353df1f9bac64d Mon Sep 17 00:00:00 2001 From: Marcelo Gornstein Date: Thu, 4 Sep 2014 18:14:42 -0300 Subject: [PATCH 3/3] adding check and test for deflate algo --- src/lib/connectors/http.js | 3 ++- test/unit/specs/http_connector.js | 36 ++++++++++++++++++++++++++++--- 2 files changed, 35 insertions(+), 4 deletions(-) diff --git a/src/lib/connectors/http.js b/src/lib/connectors/http.js index 46badfd70..8014f3bcc 100644 --- a/src/lib/connectors/http.js +++ b/src/lib/connectors/http.js @@ -147,7 +147,8 @@ HttpConnector.prototype.request = function (params, cb) { cb(err); } else { response = Buffer.concat(buffers); - if (headers['content-encoding'] && headers['content-encoding'].match(/gzip/i)) { + var zipHdr = headers['content-encoding']; + if (zipHdr && (zipHdr.match(/gzip/i) || zipHdr.match(/deflate/i))) { zlib.unzip(response, function(gzErr, uncompressedResponse) { if(gzErr) { err = gzErr; diff --git a/test/unit/specs/http_connector.js b/test/unit/specs/http_connector.js index 5c629c6b5..84ac67cc8 100644 --- a/test/unit/specs/http_connector.js +++ b/test/unit/specs/http_connector.js @@ -304,11 +304,15 @@ describe('Http Connector', function () { }); }); - it('collects the whole request body (compressed)', function (done) { + it('collects the whole request body (gzip compressed)', function (done) { var server = nock('http://esjs.com:9200'); var con = new HttpConnection(new Host('http://esjs.com:9200')); - var body = '{ "USER": "doc" }'; - zlib.deflate(body, function(err, compressedBody) { + var elements = []; + for(var i = 0; i < 500; i++) { + elements.push({ "USER": "doc" }); + } + var body = JSON.stringify(elements); + zlib.gzip(body, function(err, compressedBody) { server .get('/users/1') .reply(200, compressedBody, {'Content-Encoding': 'gzip'}); @@ -326,6 +330,32 @@ describe('Http Connector', function () { }); }); + it('collects the whole request body (deflate compressed)', function (done) { + var server = nock('http://esjs.com:9200'); + var con = new HttpConnection(new Host('http://esjs.com:9200')); + var elements = []; + for(var i = 0; i < 500; i++) { + elements.push({ "USER": "doc" }); + } + var body = JSON.stringify(elements); + zlib.deflate(body, function(err, compressedBody) { + server + .get('/users/1') + .reply(200, compressedBody, {'Content-Encoding': 'deflate'}); + + con.request({ + method: 'GET', + path: '/users/1' + }, function (err, resp, status) { + expect(err).to.be(undefined); + expect(resp).to.eql(body); + expect(status).to.eql(200); + server.done(); + done(); + }); + }); + }); + it('Can handle uncompress errors', function (done) { var server = nock('http://esjs.com:9200'); var con = new HttpConnection(new Host('http://esjs.com:9200'));