[Backport 7.x] Added proxy support (#1276)

Co-authored-by: Tomas Della Vedova <delvedor@users.noreply.github.com>

docs/configuration.asciidoc

@@ -117,6 +117,24 @@ _Default:_ `false`
 |`http.SecureContextOptions` - ssl https://nodejs.org/api/tls.html[configuraton]. +
 _Default:_ `null`
 
+|`proxy`
+a|`string, URL` - If you are using an http(s) proxy, you can put its url here.
+The client will automatically handle the connection to it. +
+_Default:_ `null`
+[source,js]
+----
+const client = new Client({
+  node: 'http://localhost:9200',
+  proxy: 'http://localhost:8080'
+})
+
+// Proxy with basic authentication
+const client = new Client({
+  node: 'http://localhost:9200',
+  proxy: 'http://user:pwd@localhost:8080'
+})
+----
+
 |`agent`
 a|`http.AgentOptions, function` - http agent https://nodejs.org/api/http.html#http_new_agent_options[options], 
 or a function that returns an actual http agent instance. If you want to disable the http agent use entirely

docs/usage.asciidoc

@@ -210,6 +210,45 @@ _Default:_ `null`
 _Default:_ `null`
 |===
 
+=== Connecting through a proxy
+
+~Added~ ~in~ ~`v7.10.0`~
+
+If you need to pass through an http(s) proxy for connecting to Elasticsearch, the client offers
+out of the box a handy configuration for helping you with it. Under the hood it
+uses the https://github.com/delvedor/hpagent[`hpagent`] module.
+
+[source,js]
+----
+const client = new Client({
+  node: 'http://localhost:9200',
+  proxy: 'http://localhost:8080'
+})
+----
+
+Basic authentication is supported as well:
+
+[source,js]
+----
+const client = new Client({
+  node: 'http://localhost:9200',
+  proxy: 'http:user:pwd@//localhost:8080'
+})
+----
+
+If you are connecting through a not http(s) proxy, such as a `socks5` or `pac`,
+you can use the `agent` option to configure it.
+
+[source,js]
+----
+const SocksProxyAgent = require('socks-proxy-agent')
+const client = new Client({
+  node: 'http://localhost:9200',
+  agent () {
+    return new SocksProxyAgent('socks://127.0.0.1:1080')
+  }
+})
+----
 
 === Error handling
 

index.d.ts

@@ -107,6 +107,7 @@ interface ClientOptions {
   name?: string | symbol;
   auth?: BasicAuth | ApiKeyAuth;
   context?: Context;
+  proxy?: string | URL;
   cloud?: {
     id: string;
     // TODO: remove username and password here in 8

index.js

@@ -100,7 +100,8 @@ class Client {
       name: 'elasticsearch-js',
       auth: null,
       opaqueIdPrefix: null,
-      context: null
+      context: null,
+      proxy: null
     }, opts)
 
     this[kInitialOptions] = options
@@ -119,6 +120,7 @@ class Client {
         resurrectStrategy: options.resurrectStrategy,
         ssl: options.ssl,
         agent: options.agent,
+        proxy: options.proxy,
         Connection: options.Connection,
         auth: options.auth,
         emit: this[kEventEmitter].emit.bind(this[kEventEmitter]),

lib/Connection.d.ts

@@ -24,6 +24,8 @@ import { inspect, InspectOptions } from 'util'
 import { Readable as ReadableStream } from 'stream';
 import { ApiKeyAuth, BasicAuth } from './pool'
 import * as http from 'http'
+import * as https from 'https'
+import * as hpagent from 'hpagent'
 import { ConnectionOptions as TlsConnectionOptions } from 'tls'
 
 export declare type agentFn = () => any;
@@ -37,6 +39,7 @@ interface ConnectionOptions {
   status?: string;
   roles?: ConnectionRoles;
   auth?: BasicAuth | ApiKeyAuth;
+  proxy?: string | URL;
 }
 
 interface ConnectionRoles {
@@ -81,7 +84,7 @@ export default class Connection {
   makeRequest: any
   _openRequests: number
   _status: string
-  _agent: http.Agent
+  _agent: http.Agent | https.Agent | hpagent.HttpProxyAgent | hpagent.HttpsProxyAgent
   constructor(opts?: ConnectionOptions)
   request(params: RequestOptions, callback: (err: Error | null, response: http.IncomingMessage | null) => void): http.ClientRequest
   close(): Connection

lib/Connection.js

@@ -21,6 +21,7 @@
 
 const assert = require('assert')
 const { inspect } = require('util')
+const hpagent = require('hpagent')
 const http = require('http')
 const https = require('https')
 const debug = require('debug')('elasticsearch')
@@ -63,9 +64,16 @@ class Connection {
         maxFreeSockets: 256,
         scheduling: 'lifo'
       }, opts.agent)
-      this.agent = this.url.protocol === 'http:'
-        ? new http.Agent(agentOptions)
-        : new https.Agent(Object.assign({}, agentOptions, this.ssl))
+      if (opts.proxy) {
+        agentOptions.proxy = opts.proxy
+        this.agent = this.url.protocol === 'http:'
+          ? new hpagent.HttpProxyAgent(agentOptions)
+          : new hpagent.HttpsProxyAgent(Object.assign({}, agentOptions, this.ssl))
+      } else {
+        this.agent = this.url.protocol === 'http:'
+          ? new http.Agent(agentOptions)
+          : new https.Agent(Object.assign({}, agentOptions, this.ssl))
+      }
     }
 
     this.makeRequest = this.url.protocol === 'http:'

lib/pool/BaseConnectionPool.js

@@ -35,6 +35,7 @@ class BaseConnectionPool {
     this.auth = opts.auth || null
     this._ssl = opts.ssl
     this._agent = opts.agent
+    this._proxy = opts.proxy || null
   }
 
   getConnection () {
@@ -69,6 +70,8 @@ class BaseConnectionPool {
     if (opts.ssl == null) opts.ssl = this._ssl
     /* istanbul ignore else */
     if (opts.agent == null) opts.agent = this._agent
+    /* istanbul ignore else */
+    if (opts.proxy == null) opts.proxy = this._proxy
 
     const connection = new this.Connection(opts)
 

lib/pool/index.d.ts

@@ -27,6 +27,7 @@ import { nodeFilterFn, nodeSelectorFn } from '../Transport';
 interface BaseConnectionPoolOptions {
   ssl?: SecureContextOptions;
   agent?: AgentOptions;
+  proxy?: string | URL;
   auth?: BasicAuth | ApiKeyAuth;
   emit: (event: string | symbol, ...args: any[]) => boolean;
   Connection: typeof Connection;
@@ -83,6 +84,7 @@ declare class BaseConnectionPool {
   emit: (event: string | symbol, ...args: any[]) => boolean;
   _ssl: SecureContextOptions | null;
   _agent: AgentOptions | null;
+  _proxy: string | URL;
   auth: BasicAuth | ApiKeyAuth;
   Connection: typeof Connection;
   constructor(opts?: BaseConnectionPoolOptions);

package.json

@@ -60,6 +60,7 @@
     "minimist": "^1.2.0",
     "ora": "^3.4.0",
     "pretty-hrtime": "^1.0.3",
+    "proxy": "^1.0.2",
     "rimraf": "^2.6.3",
     "semver": "^6.0.0",
     "simple-git": "^1.110.0",
@@ -75,6 +76,7 @@
   "dependencies": {
     "debug": "^4.1.1",
     "decompress-response": "^4.2.0",
+    "hpagent": "^0.1.1",
     "ms": "^2.1.1",
     "pump": "^3.0.0",
     "secure-json-parse": "^2.1.0"

test/acceptance/proxy.test.js

@@ -0,0 +1,149 @@
+'use strict'
+
+// We are using self-signed certificates
+process.env.NODE_TLS_REJECT_UNAUTHORIZED = 0
+
+const { test } = require('tap')
+const { Client } = require('../../index')
+const {
+  buildProxy: {
+    createProxy,
+    createSecureProxy,
+    createServer,
+    createSecureServer
+  }
+} = require('../utils')
+
+test('http-http proxy support', async t => {
+  const server = await createServer()
+  const proxy = await createProxy()
+  server.on('request', (req, res) => {
+    t.strictEqual(req.url, '/_cluster/health')
+    res.setHeader('content-type', 'application/json')
+    res.end(JSON.stringify({ hello: 'world' }))
+  })
+
+  const client = new Client({
+    node: `http://${server.address().address}:${server.address().port}`,
+    proxy: `http://${proxy.address().address}:${proxy.address().port}`
+  })
+
+  const response = await client.cluster.health()
+  t.deepEqual(response.body, { hello: 'world' })
+
+  server.close()
+  proxy.close()
+})
+
+test('http-https proxy support', async t => {
+  const server = await createSecureServer()
+  const proxy = await createProxy()
+  server.on('request', (req, res) => {
+    t.strictEqual(req.url, '/_cluster/health')
+    res.setHeader('content-type', 'application/json')
+    res.end(JSON.stringify({ hello: 'world' }))
+  })
+
+  const client = new Client({
+    node: `https://${server.address().address}:${server.address().port}`,
+    proxy: `http://${proxy.address().address}:${proxy.address().port}`
+  })
+
+  const response = await client.cluster.health()
+  t.deepEqual(response.body, { hello: 'world' })
+
+  server.close()
+  proxy.close()
+})
+
+test('https-http proxy support', async t => {
+  const server = await createServer()
+  const proxy = await createSecureProxy()
+  server.on('request', (req, res) => {
+    t.strictEqual(req.url, '/_cluster/health')
+    res.setHeader('content-type', 'application/json')
+    res.end(JSON.stringify({ hello: 'world' }))
+  })
+
+  const client = new Client({
+    node: `http://${server.address().address}:${server.address().port}`,
+    proxy: `https://${proxy.address().address}:${proxy.address().port}`
+  })
+
+  const response = await client.cluster.health()
+  t.deepEqual(response.body, { hello: 'world' })
+
+  server.close()
+  proxy.close()
+})
+
+test('https-https proxy support', async t => {
+  const server = await createSecureServer()
+  const proxy = await createSecureProxy()
+  server.on('request', (req, res) => {
+    t.strictEqual(req.url, '/_cluster/health')
+    res.setHeader('content-type', 'application/json')
+    res.end(JSON.stringify({ hello: 'world' }))
+  })
+
+  const client = new Client({
+    node: `https://${server.address().address}:${server.address().port}`,
+    proxy: `https://${proxy.address().address}:${proxy.address().port}`
+  })
+
+  const response = await client.cluster.health()
+  t.deepEqual(response.body, { hello: 'world' })
+
+  server.close()
+  proxy.close()
+})
+
+test('http basic authentication', async t => {
+  const server = await createServer()
+  const proxy = await createProxy()
+  server.on('request', (req, res) => {
+    t.strictEqual(req.url, '/_cluster/health')
+    res.setHeader('content-type', 'application/json')
+    res.end(JSON.stringify({ hello: 'world' }))
+  })
+
+  proxy.authenticate = function (req, fn) {
+    fn(null, req.headers['proxy-authorization'] === `Basic ${Buffer.from('hello:world').toString('base64')}`)
+  }
+
+  const client = new Client({
+    node: `http://${server.address().address}:${server.address().port}`,
+    proxy: `http://hello:world@${proxy.address().address}:${proxy.address().port}`
+  })
+
+  const response = await client.cluster.health()
+  t.deepEqual(response.body, { hello: 'world' })
+
+  server.close()
+  proxy.close()
+})
+
+test('https basic authentication', async t => {
+  const server = await createSecureServer()
+  const proxy = await createProxy()
+  server.on('request', (req, res) => {
+    t.strictEqual(req.url, '/_cluster/health')
+    res.setHeader('content-type', 'application/json')
+    res.end(JSON.stringify({ hello: 'world' }))
+  })
+
+  proxy.authenticate = function (req, fn) {
+    fn(null, req.headers['proxy-authorization'] === `Basic ${Buffer.from('hello:world').toString('base64')}`)
+  }
+
+  const client = new Client({
+    node: `https://${server.address().address}:${server.address().port}`,
+    proxy: `http://hello:world@${proxy.address().address}:${proxy.address().port}`
+  })
+
+  const response = await client.cluster.health()
+  t.deepEqual(response.body, { hello: 'world' })
+
+  server.close()
+  proxy.close()
+})

test/acceptance/sniff.test.js

@@ -84,7 +84,8 @@ test('Should update the connection pool', t => {
               ml: false
             },
             ssl: null,
-            agent: null
+            agent: null,
+            proxy: null
           })
         }
       }

test/types/client-options.test-d.ts

@@ -692,3 +692,28 @@ expectError<errors.ConfigurationError>(
     context: 'hello world'
   })
 )
+
+/**
+ * `proxy` option
+ */
+expectType<Client>(
+  new Client({
+    node: 'http://localhost:9200',
+    proxy: 'http://localhost:8080'
+  })
+)
+
+expectType<Client>(
+  new Client({
+    node: 'http://localhost:9200',
+    proxy: new URL('http://localhost:8080')
+  })
+)
+
+expectError<errors.ConfigurationError>(
+  // @ts-expect-error
+  new Client({
+    node: 'http://localhost:9200',
+    proxy: 42
+  })
+)

test/unit/connection.test.js

@@ -24,6 +24,7 @@ const { inspect } = require('util')
 const { createGzip, createDeflate } = require('zlib')
 const { URL } = require('url')
 const { Agent } = require('http')
+const hpagent = require('hpagent')
 const intoStream = require('into-stream')
 const { buildServer } = require('../utils')
 const Connection = require('../../lib/Connection')
@@ -975,3 +976,25 @@ test('Should correctly resolve request pathname', t => {
     '/test/hello'
   )
 })
+
+test('Proxy agent (http)', t => {
+  t.plan(1)
+
+  const connection = new Connection({
+    url: new URL('http://localhost:9200'),
+    proxy: 'http://localhost:8080'
+  })
+
+  t.true(connection.agent instanceof hpagent.HttpProxyAgent)
+})
+
+test('Proxy agent (https)', t => {
+  t.plan(1)
+
+  const connection = new Connection({
+    url: new URL('https://localhost:9200'),
+    proxy: 'http://localhost:8080'
+  })
+
+  t.true(connection.agent instanceof hpagent.HttpsProxyAgent)
+})

test/utils/buildProxy.js

@@ -0,0 +1,60 @@
+// Licensed to Elasticsearch B.V under one or more agreements.
+// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
+// See the LICENSE file in the project root for more information
+
+'use strict'
+
+const proxy = require('proxy')
+const { readFileSync } = require('fs')
+const { join } = require('path')
+const http = require('http')
+const https = require('https')
+
+const ssl = {
+  key: readFileSync(join(__dirname, '..', 'fixtures', 'https.key')),
+  cert: readFileSync(join(__dirname, '..', 'fixtures', 'https.cert'))
+}
+
+function createProxy () {
+  return new Promise((resolve, reject) => {
+    const server = proxy(http.createServer())
+    server.listen(0, '127.0.0.1', () => {
+      resolve(server)
+    })
+  })
+}
+
+function createSecureProxy () {
+  return new Promise((resolve, reject) => {
+    const server = proxy(https.createServer(ssl))
+    server.listen(0, '127.0.0.1', () => {
+      resolve(server)
+    })
+  })
+}
+
+function createServer (handler, callback) {
+  return new Promise((resolve, reject) => {
+    const server = http.createServer()
+    server.listen(0, '127.0.0.1', () => {
+      resolve(server)
+    })
+  })
+}
+
+function createSecureServer (handler, callback) {
+  return new Promise((resolve, reject) => {
+    const server = https.createServer(ssl)
+    server.listen(0, '127.0.0.1', () => {
+      resolve(server)
+    })
+  })
+}
+
+module.exports = {
+  ssl,
+  createProxy,
+  createSecureProxy,
+  createServer,
+  createSecureServer
+}

test/utils/index.js

@@ -23,6 +23,7 @@ const { promisify } = require('util')
 const sleep = promisify(setTimeout)
 const buildServer = require('./buildServer')
 const buildCluster = require('./buildCluster')
+const buildProxy = require('./buildProxy')
 const connection = require('./MockConnection')
 
 async function waitCluster (client, waitForStatus = 'green', timeout = '50s', times = 0) {
@@ -43,6 +44,7 @@ async function waitCluster (client, waitForStatus = 'green', timeout = '50s', ti
 module.exports = {
   buildServer,
   buildCluster,
+  buildProxy,
   connection,
   waitCluster
 }