habsi/elasticsearch-js
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Client helpers (#1107)

Browse Source 
* Added client helpers

* Updated test

* The search helper should return only the documents

* Added code comments

* Fixed bug

* Updated test

* Removed bulkSize and added flushBytes

* Updated test

* Added concurrency

* Updated test

* Added support for 429 handling in the scroll search helper

* Updated test

* Updated stats count

* Updated test

* Fix test

* Use client maxRetries as default

* Updated type definitions

* Refactored bulk helper to be more consistent with the client api

* Updated test

* Improved error handling, added refreshOnCompletion option and forward additinal options to the bulk api

* Updated type definitions

* Updated test

* Fixed test on Node v8

* Updated test

* Added TODO

* Updated docs

* Added Node v8 note

* Updated scripts

* Removed useless files

* Added helpers to integration test

* Fix cli argument position

* Moar fixes

* Test run elasticsearch in github actions

* Use master action version

* Add vm.max_map_count step

* Test new action setup

* Added Configure sysctl limits step

* Updated action to latest version

* Don't run helpers integration test in jenkins

* Run helpers integratino test also with Node v10

* Updated docs

* Updated docs

* Updated helpers type definitions

* Added test for helpers type definitions

* Added license header
This commit is contained in:
Tomas Della Vedova
2020-03-23 17:43:10 +01:00
committed by GitHub
parent 6c82a4967e
commit d7836a16af
24 changed files with 7654 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
.ci/run-repository.sh
View File

@ -32,7 +32,7 @@ echo -e "\033[1m>>>>> NPM run test:integration >>>>>>>>>>>>>>>>>>>>>>>>>>>>>\033
repo=$(realpath $(dirname $(realpath -s $0))/../)
run_script_args=""
if [[ "$NODE_JS_VERSION" == "8" ]]; then
run_script_args="-- --node-arg=--harmony-async-iteration"
run_script_args="--harmony-async-iteration"
fi
docker run \
@ -43,4 +43,4 @@ docker run \
--name elasticsearch-js \
--rm \
elastic/elasticsearch-js \
npm run test:integration ${run_script_args}
node ${run_script_args} test/integration/index.js

36
.github/workflows/nodejs.yml vendored
View File

@ -65,6 +65,42 @@ jobs:
run: |
npm run test:unit -- --node-arg=--harmony-async-iteration
helpers-integration-test:
name: Helpers integration test
runs-on: ubuntu-latest
strategy:
matrix:
node-version: [10.x, 12.x]
steps:
- uses: actions/checkout@v2
- name: Configure sysctl limits
run: |
sudo swapoff -a
sudo sysctl -w vm.swappiness=1
sudo sysctl -w fs.file-max=262144
sudo sysctl -w vm.max_map_count=262144
- name: Runs Elasticsearch
uses: elastic/elastic-github-actions/elasticsearch@master
with:
stack-version: 8.0.0-SNAPSHOT
- name: Use Node.js ${{ matrix.node-version }}
uses: actions/setup-node@v1
with:
node-version: ${{ matrix.node-version }}
- name: Install
run: |
npm install
- name: Integration test
run: |
npm run test:integration:helpers
code-coverage:
name: Code coverage
runs-on: ubuntu-latest

1
README.md
View File

@ -60,6 +60,7 @@ We recommend that you write a lightweight proxy that uses this client instead.
- [Observability](https://www.elastic.co/guide/en/elasticsearch/client/javascript-api/current/observability.html)
- [Creating a child client](https://www.elastic.co/guide/en/elasticsearch/client/javascript-api/current/child-client.html)
- [Extend the client](https://www.elastic.co/guide/en/elasticsearch/client/javascript-api/current/extend-client.html)
- [Client helpers](https://www.elastic.co/guide/en/elasticsearch/client/javascript-api/current/client-helpers.html)
- [Typescript support](https://www.elastic.co/guide/en/elasticsearch/client/javascript-api/current/typescript.html)
- [Examples](https://www.elastic.co/guide/en/elasticsearch/client/javascript-api/current/examples.html)

2
docs/examples/bulk.asciidoc
View File

@ -4,6 +4,8 @@
The `bulk` API makes it possible to perform many index/delete operations in a
single API call. This can greatly increase the indexing speed.
NOTE: Did you know that we provide an helper for sending bulk request? You can find it {jsclient}/client-helpers.html[here].
[source,js]
----
'use strict'

2
docs/examples/scroll.asciidoc
View File

@ -19,6 +19,8 @@ In order to use scrolling, the initial search request should specify the scroll
parameter in the query string, which tells {es} how long it should keep the
“search context” alive.
NOTE: Did you know that we provide an helper for sending scroll requests? You can find it {jsclient}/client-helpers.html[here].
[source,js]
----
'use strict'

272
docs/helpers.asciidoc Normal file
View File

@ -0,0 +1,272 @@
[[client-helpers]]
== Client Helpers
The client comes with an handy collection of helpers to give you a more comfortable experience with some APIs.
CAUTION: The client helpers are experimental, and the API may change in the next minor releases.
If you are using the client with Node.js v8 you should run your code with the `--harmony-async-iteration` argument. +
eg: `node --harmony-async-iteration index.js`
=== Bulk Helper
Running Bulk requests can be complex due to the shape of the API, this helper aims to provide a nicer developer experience around the Bulk API.
==== Usage
[source,js]
----
const { createReadStream } = require('fs')
const split = require('split2')
const { Client } = require('@elastic/elasticsearch')
const client = new Client({ node: 'http://localhost:9200' })
const result = await client.helpers.bulk({
datasource: createReadStream('./dataset.ndjson').pipe(split()),
onDocument (doc) {
return {
index: { _index: 'my-index' }
}
}
})
console.log(result)
// {
// total: number,
// failed: number,
// retry: number,
// successful: number,
// time: number,
// bytes: number,
// aborted: boolean
// }
----
To create a new instance of the Bulk helper, you should access it as shown in the example above, the configuration options are:
[cols=2*]
|===
|`datasource`
a|An array or a readable stream with the data you need to index/create/update/delete.
It can be an array of strings or objects, but also a stream of json strings or JavaScript objects. +
If it is a stream, we recommend to use the https://www.npmjs.com/package/split2[`split2`] package, that will split the stream on new lines delimiters. +
This parameter is mandatory.
[source,js]
----
const { createReadStream } = require('fs')
const split = require('split2')
const b = client.helpers.bulk({
// if you just use split(), the data will be used as array of strings
datasource: createReadStream('./dataset.ndjson').pipe(split())
// if you need to manipulate the data, you can pass JSON.parse to split
datasource: createReadStream('./dataset.ndjson').pipe(split(JSON.parse))
})
----
|`onDocument`
a|A function that will be called for each document of the datasource. Inside this function you can manipulate the document and you must return the operation you want to execute with the document. Look at the link:{ref}/docs-bulk.html[Bulk API documentation] to see the supported operations. +
This parameter is mandatory.
[source,js]
----
const b = client.helpers.bulk({
onDocument (doc) {
return {
index: { _index: 'my-index' }
}
}
})
----
|`onDrop`
a|A function that will be called for everytime a document can't be indexed and it has reached the maximum amount of retries.
[source,js]
----
const b = client.helpers.bulk({
onDrop (doc) {
console.log(doc)
}
})
----
|`flushBytes`
a|The size of the bulk body in bytes to reach before to send it. Default of 5MB. +
_Default:_ `5000000`
[source,js]
----
const b = client.helpers.bulk({
flushBytes: 1000000
})
----
|`concurrency`
a|How many request will be executed at the same time. +
_Default:_ `5`
[source,js]
----
const b = client.helpers.bulk({
concurrency: 10
})
----
|`retries`
a|How many times a document will be retried before to call the `onDrop` callback. +
_Default:_ Client max retries.
[source,js]
----
const b = client.helpers.bulk({
retries: 3
})
----
|`wait`
a|How much time to wait before retries in milliseconds.+
_Default:_ 5000.
[source,js]
----
const b = client.helpers.bulk({
wait: 3000
})
----
|`refreshOnCompletion`
a|If `true`, at the end of the bulk operation it will run a refresh on all indices or on the specified indices. +
_Default:_ false.
[source,js]
----
const b = client.helpers.bulk({
refreshOnCompletion: true
// or
refreshOnCompletion: 'index-name'
})
----
|===
==== Abort a bulk operation
If needed, you can abort a bulk operation at any time. The bulk helper returns a https://promisesaplus.com/[thenable], which has an `abort` method.
NOTE: The abort method will stop the execution of the bulk operation, but if you are using a concurrency higher than one, the operations that are already running will not be stopped.
[source,js]
----
const { createReadStream } = require('fs')
const split = require('split2')
const { Client } = require('@elastic/elasticsearch')
const client = new Client({ node: 'http://localhost:9200' })
const b = client.helpers.bulk({
datasource: createReadStream('./dataset.ndjson').pipe(split()),
onDocument (doc) {
return {
index: { _index: 'my-index' }
}
},
onDrop (doc) {
b.abort()
}
})
console.log(await b)
----
==== Passing custom options to the Bulk API
You can pass any option supported by the link:{ref}/docs-bulk.html#docs-bulk-api-query-params[Bulk API] to the helper, and the helper will use those options in conjuction with the Bulk
API call.
[source,js]
----
const result = await client.helpers.bulk({
datasource: [...]
onDocument (doc) {
return {
index: { _index: 'my-index' }
}
},
pipeline: 'my-pipeline'
})
----
=== Search Helper
A simple wrapper around the search API. Instead of returning the entire `result` object it will return only the search documents result.
[source,js]
----
const documents = await client.helpers.search({
index: 'stackoverflow',
body: {
query: {
match: {
title: 'javascript'
}
}
}
})
for (const doc of documents) {
console.log(doc)
}
----
=== Scroll Search Helper
This helpers offers a simple and intuitive way to use the scroll search API. Once called, it returns an https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Statements/for-await...of[async iterator] which can be used in conjuction with a for-await...of. +
It handles automatically the `429` error and uses the client's `maxRetries` option.
[source,js]
----
const scrollSearch = await client.helpers.scrollSearch({
index: 'stackoverflow',
body: {
query: {
match: {
title: 'javascript'
}
}
}
})
for await (const result of scrollSearch) {
console.log(result)
}
----
==== Clear a scroll search
If needed, you can clear a scroll search by calling `result.clear()`:
[source,js]
----
for await (const result of scrollSearch) {
if (condition) {
await result.clear()
}
}
----
==== Quickly getting the documents
If you only need the documents from the result of a scroll search, you can access them via `result.documents`:
[source,js]
----
for await (const result of scrollSearch) {
console.log(result.documents)
}
----
=== Scroll Documents Helper
It works in the same way as the scroll search helper, but it returns only the documents instead. Note, every loop cycle will return you a single document, and you can't use the `clear` method.
[source,js]
----
const scrollSearch = await client.helpers.scrollDocuments({
index: 'stackoverflow',
body: {
query: {
match: {
title: 'javascript'
}
}
}
})
for await (const doc of scrollSearch) {
console.log(doc)
}
----

1
docs/index.asciidoc
View File

@ -12,5 +12,6 @@ include::authentication.asciidoc[]
include::observability.asciidoc[]
include::child.asciidoc[]
include::extend.asciidoc[]
include::helpers.asciidoc[]
include::typescript.asciidoc[]
include::examples/index.asciidoc[]

2
index.d.ts vendored
View File

@ -30,6 +30,7 @@ import {
ApiKeyAuth
} from './lib/pool';
import Serializer from './lib/Serializer';
import Helpers from './lib/Helpers';
import * as RequestParams from './api/requestParams';
import * as errors from './lib/errors';
@ -107,6 +108,7 @@ declare class Client extends EventEmitter {
transport: Transport;
serializer: Serializer;
extend: ClientExtends;
helpers: Helpers;
child(opts?: ClientOptions): Client;
close(callback?: Function): Promise<void> | void;
/* GENERATED */

3
index.js
View File

@ -10,6 +10,7 @@ const debug = require('debug')('elasticsearch')
const Transport = require('./lib/Transport')
const Connection = require('./lib/Connection')
const { ConnectionPool, CloudConnectionPool } = require('./lib/pool')
const Helpers = require('./lib/Helpers')
const Serializer = require('./lib/Serializer')
const errors = require('./lib/errors')
const { ConfigurationError } = errors
@ -126,6 +127,8 @@ class Client extends EventEmitter {
opaqueIdPrefix: options.opaqueIdPrefix
})
this.helpers = new Helpers({ client: this, maxRetries: options.maxRetries })
const apis = buildApi({
makeRequest: this.transport.request.bind(this.transport),
result: { body: null, statusCode: null, headers: null, warnings: null },

76
lib/Helpers.d.ts vendored Normal file
View File

@ -0,0 +1,76 @@
// Licensed to Elasticsearch B.V under one or more agreements.
// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
// See the LICENSE file in the project root for more information
import { Readable as ReadableStream } from 'stream'
import { TransportRequestOptions, ApiResponse, RequestBody, ResponseBody } from './Transport'
import { Search, Bulk } from '../api/requestParams'
export default class Helpers {
search<TRequestBody extends RequestBody, TDocument = unknown>(params: Search<TRequestBody>, options?: TransportRequestOptions): Promise<TDocument[]>
scrollSearch<TRequestBody extends RequestBody, TDocument = unknown, TResponse = ResponseBody, TContext = unknown>(params: Search<TRequestBody>, options?: TransportRequestOptions): AsyncIterable<ScrollSearchResponse<TDocument, TResponse, TContext>>
scrollDocuments<TRequestBody extends RequestBody, TDocument = unknown>(params: Search<TRequestBody>, options?: TransportRequestOptions): AsyncIterable<TDocument>
bulk(options: BulkHelperOptions): BulkHelper<BulkStats>
}
export interface ScrollSearchResponse<TDocument = unknown, TResponse = ResponseBody, TContext = unknown> extends ApiResponse<TResponse, TContext> {
clear: () => Promise<void>
documents: TDocument[]
}
export interface BulkHelper<T> extends Promise<T> {
abort: () => BulkHelper<T>
}
export interface BulkStats {
total: number
failed: number
retry: number
successful: number
time: number
bytes: number
aborted: boolean
}
interface IndexAction {
index: {
_index: string
[key: string]: any
}
}
interface CreateAction {
create: {
_index: string
[key: string]: any
}
}
interface UpdateActionOperation {
update: {
_index: string
[key: string]: any
}
}
interface DeleteAction {
delete: {
_index: string
[key: string]: any
}
}
type UpdateAction = [UpdateActionOperation, Record<string, any>]
type Action = IndexAction | CreateAction | UpdateAction | DeleteAction
type Omit<T, K extends keyof T> = Pick<T, Exclude<keyof T, K>>
export interface BulkHelperOptions extends Omit<Bulk, 'body'> {
datasource: any[] | Buffer | ReadableStream
onDocument: (doc: Record<string, any>) => Action
flushBytes?: number
concurrency?: number
retries?: number
wait?: number,
onDrop?: (doc: Record<string, any>) => void,
refreshOnCompletion?: boolean | string
}

428
lib/Helpers.js Normal file
View File

@ -0,0 +1,428 @@
// Licensed to Elasticsearch B.V under one or more agreements.
// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
// See the LICENSE file in the project root for more information
'use strict'
const { promisify } = require('util')
const { ResponseError, ConfigurationError } = require('./errors')
const pImmediate = promisify(setImmediate)
const sleep = promisify(setTimeout)
const kGetHits = Symbol('elasticsearch-get-hits')
const kClient = Symbol('elasticsearch-client')
const noop = () => {}
class Helpers {
constructor (opts) {
this[kClient] = opts.client
this.maxRetries = opts.maxRetries
}
[kGetHits] (body) {
if (body.hits && body.hits.hits) {
return body.hits.hits.map(d => d._source)
}
return []
}
/**
* Runs a search operation. The only difference between client.search and this utility,
* is that we are only returning the hits to the user and not the full ES response.
* @param {object} params - The Elasticsearch's search parameters.
* @param {object} options - The client optional configuration for this request.
* @return {array} The documents that matched the request.
*/
async search (params, options) {
const response = await this[kClient].search(params, options)
return this[kGetHits](response.body)
}
/**
* Runs a scroll search operation. This function returns an async iterator, allowing
* the user to use a for await loop to get all the results of a given search.
* ```js
* for await (const result of client.helpers.scrollSearch({ params })) {
* console.log(result)
* }
* ```
* Each result represents the entire body of a single scroll search request,
* if you just need to scroll the results, use scrollDocuments.
* This function handles automatically retries on 429 status code.
* @param {object} params - The Elasticsearch's search parameters.
* @param {object} options - The client optional configuration for this request.
* @return {iterator} the async iterator
*/
async * scrollSearch (params, options = {}) {
// TODO: study scroll search slices
const wait = options.wait || 5000
const maxRetries = options.maxRetries || this.maxRetries
if (Array.isArray(options.ignore)) {
options.ignore.push(429)
} else {
options.ignore = [429]
}
params.scroll = params.scroll || '1m'
let response = null
for (let i = 0; i < maxRetries; i++) {
response = await this[kClient].search(params, options)
if (response.statusCode !== 429) break
await sleep(wait)
}
if (response.statusCode === 429) {
throw new ResponseError(response)
}
let scrollId = response.body._scroll_id
let stop = false
const clear = async () => {
stop = true
await this[kClient].clearScroll(
{ body: { scroll_id: scrollId } },
{ ignore: [400] }
)
}
while (response.body.hits.hits.length > 0) {
scrollId = response.body._scroll_id
response.clear = clear
response.documents = this[kGetHits](response.body)
yield response
if (!scrollId || stop === true) {
break
}
for (let i = 0; i < maxRetries; i++) {
response = await this[kClient].scroll({
scroll: params.scroll,
body: {
scroll_id: scrollId
}
}, options)
if (response.statusCode !== 429) break
await sleep(wait)
}
if (response.statusCode === 429) {
throw new ResponseError(response)
}
}
}
/**
* Runs a scroll search operation. This function returns an async iterator, allowing
* the user to use a for await loop to get all the documents of a given search.
* ```js
* for await (const document of client.helpers.scrollSearch({ params })) {
* console.log(document)
* }
* ```
* Each document is what you will find by running a scrollSearch and iterating on the hits array.
* @param {object} params - The Elasticsearch's search parameters.
* @param {object} options - The client optional configuration for this request.
* @return {iterator} the async iterator
*/
async * scrollDocuments (params, options) {
for await (const { documents } of this.scrollSearch(params)) {
for (const document of documents) {
yield document
}
}
}
/**
* Creates a bulk helper instance. Once you configure it, you can pick which operation
* to execute with the given dataset, index, create, update, and delete.
* @param {object} options - The configuration of the bulk operation.
* @return {object} The possible orations to run with the datasource.
*/
bulk (options) {
// TODO: add an interval to force flush the body
// to handle the slow producer problem
const client = this[kClient]
const { serialize, deserialize } = client.serializer
const {
datasource,
onDocument,
flushBytes = 5000000,
concurrency = 5,
retries = this.maxRetries,
wait = 5000,
onDrop = noop,
refreshOnCompletion = false,
...bulkOptions
} = options
if (datasource === undefined) {
return Promise.reject(new ConfigurationError('bulk helper: the datasource is required'))
}
if (!(Array.isArray(datasource) || Buffer.isBuffer(datasource) || typeof datasource.pipe === 'function')) {
return Promise.reject(new ConfigurationError('bulk helper: the datasource must be an array or a buffer or a readable stream'))
}
if (onDocument === undefined) {
return Promise.reject(new ConfigurationError('bulk helper: the onDocument callback is required'))
}
let shouldAbort = false
const stats = {
total: 0,
failed: 0,
retry: 0,
successful: 0,
time: 0,
bytes: 0,
aborted: false
}
const p = iterate()
return {
then (onFulfilled, onRejected) {
return p.then(onFulfilled, onRejected)
},
catch (onRejected) {
return p.catch(onRejected)
},
abort () {
shouldAbort = true
stats.aborted = true
return this
}
}
/**
* Function that iterates over the given datasource and start a bulk operation as soon
* as it reaches the configured bulk size. It's designed to use the Node.js asynchronous
* model at this maximum capacity, as it will collect the next body to send while there is
* a running http call. In this way, the CPU time will be used carefully.
* The objects will be serialized right away, to approximate the byte length of the body.
* It creates an array of strings instead of a ndjson string because the bulkOperation
* will navigate the body for matching failed operations with the original document.
*/
async function iterate () {
const { semaphore, finish } = buildSemaphore()
const startTime = Date.now()
const bulkBody = []
let actionBody = ''
let payloadBody = ''
let chunkBytes = 0
for await (const chunk of datasource) {
if (shouldAbort === true) break
const action = onDocument(chunk)
const operation = Array.isArray(action)
? Object.keys(action[0])[0]
: Object.keys(action)[0]
if (operation === 'index' || operation === 'create') {
actionBody = serialize(action)
payloadBody = typeof chunk === 'string' ? chunk : serialize(chunk)
chunkBytes += Buffer.byteLength(actionBody) + Buffer.byteLength(payloadBody)
bulkBody.push(actionBody)
bulkBody.push(payloadBody)
} else if (operation === 'update') {
actionBody = serialize(action[0])
payloadBody = typeof chunk === 'string'
? `{doc:${chunk}}`
: serialize({ doc: chunk, ...action[1] })
chunkBytes += Buffer.byteLength(actionBody) + Buffer.byteLength(payloadBody)
bulkBody.push(actionBody)
bulkBody.push(payloadBody)
} else { // delete
actionBody = serialize(action)
chunkBytes += Buffer.byteLength(actionBody)
bulkBody.push(actionBody)
}
if (chunkBytes >= flushBytes) {
stats.bytes += chunkBytes
const send = await semaphore()
send(bulkBody.slice())
bulkBody.length = 0
chunkBytes = 0
}
}
// In some cases the previos http call does not have finished,
// or we didn't reach the flush bytes threshold, so we force one last operation.
if (shouldAbort === false && chunkBytes > 0) {
const send = await semaphore()
stats.bytes += chunkBytes
send(bulkBody)
}
await finish()
if (refreshOnCompletion) {
await client.indices.refresh({
index: typeof refreshOnCompletion === 'string'
? refreshOnCompletion
: '_all'
})
}
stats.time = Date.now() - startTime
stats.total = stats.successful + stats.failed
return stats
}
// This function builds a semaphore using the concurrency
// options of the bulk helper. It is used inside the iterator
// to guarantee that no more than the number of operations
// allowed to run at the same time are executed.
// It returns a semaphore function which resolves in the next tick
// if we didn't reach the maximim concurrency yet, otherwise it returns
// a promise that resolves as soon as one of the running request has finshed.
// The semaphore function resolves a send function, which will be used
// to send the actual bulk request.
// It also returns a finish function, which returns a promise that is resolved
// when there are no longer request running. It rejects an error if one
// of the request has failed for some reason.
function buildSemaphore () {
let resolveSemaphore = null
let resolveFinish = null
let rejectFinish = null
let error = null
let running = 0
return { semaphore, finish }
function finish () {
return new Promise((resolve, reject) => {
if (running === 0) {
if (error) {
reject(error)
} else {
resolve()
}
} else {
resolveFinish = resolve
rejectFinish = reject
}
})
}
function semaphore () {
if (running < concurrency) {
return pImmediate(send)
} else {
return new Promise((resolve, reject) => {
resolveSemaphore = resolve
})
}
}
function send (bulkBody) {
if (running >= concurrency) {
throw new Error('Max concurrency reached')
}
running += 1
bulkOperation(bulkBody, err => {
running -= 1
if (err) {
shouldAbort = true
error = err
}
if (resolveSemaphore) {
resolveSemaphore(send)
resolveSemaphore = null
} else if (resolveFinish && running === 0) {
if (error) {
rejectFinish(error)
} else {
resolveFinish()
}
}
})
}
}
function bulkOperation (bulkBody, callback) {
let retryCount = retries
let isRetrying = false
// Instead of going full on async-await, which would make the code easier to read,
// we have decided to use callback style instead.
// This because every time we use async await, V8 will create multiple promises
// behind the scenes, making the code slightly slower.
tryBulk(bulkBody, retryDocuments)
function retryDocuments (err, bulkBody) {
if (err) return callback(err)
if (shouldAbort === true) return callback()
isRetrying = true
if (bulkBody.length > 0) {
if (retryCount > 0) {
retryCount -= 1
setTimeout(tryBulk, wait, bulkBody, retryDocuments)
return
}
for (let i = 0, len = bulkBody.length; i < len; i = i + 2) {
const operation = Object.keys(deserialize(bulkBody[i]))[0]
onDrop({
status: 429,
error: null,
operation: deserialize(bulkBody[i]),
document: operation !== 'delete'
? deserialize(bulkBody[i + 1])
: null,
retried: isRetrying
})
stats.failed += 1
}
}
callback()
}
function tryBulk (bulkBody, callback) {
if (shouldAbort === true) return callback(null, [])
client.bulk(Object.assign({}, bulkOptions, { body: bulkBody }), (err, { body }) => {
if (err) return callback(err, null)
if (body.errors === false) {
stats.successful += body.items.length
return callback(null, [])
}
const retry = []
const { items } = body
for (let i = 0, len = items.length; i < len; i++) {
const action = items[i]
const operation = Object.keys(action)[0]
const { status } = action[operation]
const indexSlice = operation !== 'delete' ? i * 2 : i
if (status >= 400) {
// 429 is the only staus code where we might want to retry
// a document, because it was not an error in the document itself,
// but the ES node were handling too many operations.
if (status === 429) {
stats.retry += 1
retry.push(bulkBody[indexSlice])
if (operation !== 'delete') {
retry.push(bulkBody[indexSlice + 1])
}
} else {
onDrop({
status: status,
error: action[operation].error,
operation: deserialize(bulkBody[indexSlice]),
document: operation !== 'delete'
? deserialize(bulkBody[indexSlice + 1])
: null,
retried: isRetrying
})
stats.failed += 1
}
} else {
stats.successful += 1
}
}
callback(null, retry)
})
}
}
}
}
module.exports = Helpers

5
package.json
View File

@ -17,14 +17,15 @@
],
"scripts": {
"test": "npm run lint && npm run test:unit && npm run test:behavior && npm run test:types",
"test:unit": "tap test/unit/*.test.js -t 300 --no-coverage",
"test:unit": "tap test/unit/*.test.js test/unit/**/*.test.js -t 300 --no-coverage",
"test:behavior": "tap test/behavior/*.test.js -t 300 --no-coverage",
"test:integration": "node test/integration/index.js",
"test:integration:helpers": "tap test/integration/helpers/*.test.js --no-coverage -J",
"test:types": "tsd",
"test:coverage": "nyc tap test/unit/*.test.js test/behavior/*.test.js -t 300 && nyc report --reporter=text-lcov > coverage.lcov",
"lint": "standard",
"lint:fix": "standard --fix",
"ci": "npm run license-checker && npm test && npm run test:integration && npm run test:coverage",
"ci": "npm run license-checker && npm test && npm run test:integration:helpers && npm run test:integration && npm run test:coverage",
"license-checker": "license-checker --production --onlyAllow='MIT;Apache-2.0;Apache1.1;ISC;BSD-3-Clause;BSD-2-Clause'",
"elasticsearch": "./scripts/es-docker.sh",
"elasticsearch:xpack": "./scripts/es-docker-platinum.sh"

3
test/fixtures/small-dataset.ndjson vendored Normal file
View File

@ -0,0 +1,3 @@
{"user":"jon","age":23}
{"user":"arya","age":18}
{"user":"tyrion","age":39}

5000
test/fixtures/stackoverflow.ndjson vendored Normal file
View File

File diff suppressed because one or more lines are too long

189
test/integration/helpers/bulk.test.js Normal file
View File

@ -0,0 +1,189 @@
// Licensed to Elasticsearch B.V under one or more agreements.
// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
// See the LICENSE file in the project root for more information
'use strict'
const { createReadStream } = require('fs')
const { join } = require('path')
const split = require('split2')
const { test, beforeEach, afterEach } = require('tap')
const { waitCluster } = require('../../utils')
const { Client } = require('../../../')
const datasetPath = join(__dirname, '..', '..', 'fixtures', 'stackoverflow.ndjson')
const INDEX = `test-helpers-${process.pid}`
const client = new Client({
node: process.env.TEST_ES_SERVER || 'http://localhost:9200'
})
beforeEach(async () => {
await waitCluster(client)
await client.indices.create({ index: INDEX })
})
afterEach(async () => {
await client.indices.delete({ index: INDEX }, { ignore: 404 })
})
test('bulk index', async t => {
const stream = createReadStream(datasetPath)
const result = await client.helpers.bulk({
datasource: stream.pipe(split()),
refreshOnCompletion: INDEX,
onDrop (doc) {
t.fail('It should not drop any document')
},
onDocument (doc) {
return {
index: { _index: INDEX }
}
}
})
t.type(result.time, 'number')
t.type(result.bytes, 'number')
t.match(result, {
total: 5000,
successful: 5000,
retry: 0,
failed: 0,
aborted: false
})
const { body } = await client.count({ index: INDEX })
t.match(body, { count: 5000 })
})
test('bulk index with custom id', async t => {
const stream = createReadStream(datasetPath)
const result = await client.helpers.bulk({
datasource: stream.pipe(split(JSON.parse)),
onDrop (doc) {
t.fail('It should not drop any document')
},
onDocument (doc) {
return {
index: {
_index: INDEX,
_id: doc.id
}
}
}
})
t.type(result.time, 'number')
t.type(result.bytes, 'number')
t.match(result, {
total: 5000,
successful: 5000,
retry: 0,
failed: 0,
aborted: false
})
const { body } = await client.get({
index: INDEX,
id: '19273860' // id of document n° 4242
})
t.strictEqual(body._index, INDEX)
t.strictEqual(body._id, '19273860')
t.strictEqual(body._source.id, '19273860')
})
test('abort the operation on document drop', async t => {
const stream = createReadStream(datasetPath)
const b = client.helpers.bulk({
datasource: stream.pipe(split(JSON.parse)),
concurrency: 1,
onDrop (doc) {
t.strictEqual(doc.status, 400)
t.strictEqual(doc.error.type, 'mapper_parsing_exception')
t.strictEqual(doc.document.id, '45924372')
b.abort()
},
onDocument (doc) {
if (doc.id === '45924372') { // id of document n° 500
// this will break the mapping
doc.title = { foo: 'bar' }
}
return {
index: {
_index: INDEX,
_id: doc.id
}
}
}
})
const result = await b
t.type(result.time, 'number')
t.type(result.bytes, 'number')
t.strictEqual(result.total - 1, result.successful)
t.match(result, {
retry: 0,
failed: 1,
aborted: true
})
})
test('bulk delete', async t => {
const indexResult = await client.helpers.bulk({
datasource: createReadStream(datasetPath).pipe(split(JSON.parse)),
refreshOnCompletion: true,
onDrop (doc) {
t.fail('It should not drop any document')
},
onDocument (doc) {
return {
index: {
_index: INDEX,
_id: doc.id
}
}
}
})
t.type(indexResult.time, 'number')
t.type(indexResult.bytes, 'number')
t.match(indexResult, {
total: 5000,
successful: 5000,
retry: 0,
failed: 0,
aborted: false
})
const { body: afterIndex } = await client.count({ index: INDEX })
t.match(afterIndex, { count: 5000 })
const deleteResult = await client.helpers.bulk({
datasource: createReadStream(datasetPath).pipe(split(JSON.parse)),
refreshOnCompletion: true,
onDrop (doc) {
t.fail('It should not drop any document')
},
onDocument (doc) {
return {
delete: {
_index: INDEX,
_id: doc.id
}
}
}
})
t.type(deleteResult.time, 'number')
t.type(deleteResult.bytes, 'number')
t.match(deleteResult, {
total: 5000,
successful: 5000,
retry: 0,
failed: 0,
aborted: false
})
const { body: afterDelete } = await client.count({ index: INDEX })
t.match(afterDelete, { count: 0 })
})

103
test/integration/helpers/scroll.test.js Normal file
View File

@ -0,0 +1,103 @@
// Licensed to Elasticsearch B.V under one or more agreements.
// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
// See the LICENSE file in the project root for more information
'use strict'
const { createReadStream } = require('fs')
const { join } = require('path')
const split = require('split2')
const { test, beforeEach, afterEach } = require('tap')
const { waitCluster } = require('../../utils')
const { Client } = require('../../../')
const INDEX = `test-helpers-${process.pid}`
const client = new Client({
node: process.env.TEST_ES_SERVER || 'http://localhost:9200'
})
beforeEach(async () => {
await waitCluster(client)
await client.indices.create({ index: INDEX })
const stream = createReadStream(join(__dirname, '..', '..', 'fixtures', 'stackoverflow.ndjson'))
const result = await client.helpers.bulk({
datasource: stream.pipe(split()),
refreshOnCompletion: true,
onDocument (doc) {
return {
index: { _index: INDEX }
}
}
})
if (result.failed > 0) {
throw new Error('Failed bulk indexing docs')
}
})
afterEach(async () => {
await client.indices.delete({ index: INDEX }, { ignore: 404 })
})
test('search helper', async t => {
const scrollSearch = client.helpers.scrollSearch({
index: INDEX,
body: {
query: {
match: {
title: 'javascript'
}
}
}
})
var count = 0
for await (const search of scrollSearch) {
count += 1
for (const doc of search.documents) {
t.true(doc.title.toLowerCase().includes('javascript'))
}
}
t.strictEqual(count, 11)
})
test('clear a scroll search', async t => {
const scrollSearch = client.helpers.scrollSearch({
index: INDEX,
body: {
query: {
match: {
title: 'javascript'
}
}
}
})
var count = 0
for await (const search of scrollSearch) {
count += 1
if (count === 2) {
search.clear()
}
}
t.strictEqual(count, 2)
})
test('scroll documents', async t => {
const scrollSearch = client.helpers.scrollDocuments({
index: INDEX,
body: {
query: {
match: {
title: 'javascript'
}
}
}
})
var count = 0
for await (const doc of scrollSearch) {
count += 1
t.true(doc.title.toLowerCase().includes('javascript'))
}
t.strictEqual(count, 106)
})

56
test/integration/helpers/search.test.js Normal file
View File

@ -0,0 +1,56 @@
// Licensed to Elasticsearch B.V under one or more agreements.
// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
// See the LICENSE file in the project root for more information
'use strict'
const { createReadStream } = require('fs')
const { join } = require('path')
const split = require('split2')
const { test, beforeEach, afterEach } = require('tap')
const { waitCluster } = require('../../utils')
const { Client } = require('../../../')
const INDEX = `test-helpers-${process.pid}`
const client = new Client({
node: process.env.TEST_ES_SERVER || 'http://localhost:9200'
})
beforeEach(async () => {
await waitCluster(client)
await client.indices.create({ index: INDEX })
const stream = createReadStream(join(__dirname, '..', '..', 'fixtures', 'stackoverflow.ndjson'))
const result = await client.helpers.bulk({
datasource: stream.pipe(split()),
refreshOnCompletion: true,
onDocument (doc) {
return {
index: { _index: INDEX }
}
}
})
if (result.failed > 0) {
throw new Error('Failed bulk indexing docs')
}
})
afterEach(async () => {
await client.indices.delete({ index: INDEX }, { ignore: 404 })
})
test('search helper', async t => {
const results = await client.helpers.search({
index: INDEX,
body: {
query: {
match: {
title: 'javascript'
}
}
}
})
t.strictEqual(results.length, 10)
for (const result of results) {
t.true(result.title.toLowerCase().includes('javascript'))
}
})

330
test/types/helpers.test-d.ts Normal file
View File

@ -0,0 +1,330 @@
// Licensed to Elasticsearch B.V under one or more agreements.
// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
// See the LICENSE file in the project root for more information
import { expectType, expectError, expectAssignable } from 'tsd'
import { Client } from '../../'
import { RequestBody, ResponseBody } from '../../lib/Transport'
import {
BulkHelper,
BulkStats,
BulkHelperOptions,
ScrollSearchResponse
} from '../../lib/Helpers'
const client = new Client({
node: 'http://localhost:9200'
})
/// .helpers.bulk
const b = client.helpers.bulk({
datasource: [],
onDocument (doc) {
expectType<Record<string, any>>(doc)
return { index: { _index: 'test' } }
},
flushBytes: 5000000,
concurrency: 5,
retries: 3,
wait: 5000,
onDrop (doc) {
expectType<Record<string, any>>(doc)
},
refreshOnCompletion: true,
pipeline: 'my-pipeline'
})
expectType<BulkHelper<BulkStats>>(b)
expectType<BulkHelper<BulkStats>>(b.abort())
b.then(stats => expectType<BulkStats>(stats))
// body can't be provided
expectError(
client.helpers.bulk({
datasource: [],
onDocument (doc) {
return { index: { _index: 'test' } }
},
body: []
})
)
// test onDocument actions
// index
{
const options = {
datasource: [],
onDocument (doc: Record<string, any>) {
return { index: { _index: 'test' } }
}
}
expectAssignable<BulkHelperOptions>(options)
}
// create
{
const options = {
datasource: [],
onDocument (doc: Record<string, any>) {
return { create: { _index: 'test' } }
}
}
expectAssignable<BulkHelperOptions>(options)
}
// update
{
// without `:BulkHelperOptions` this test cannot pass
// but if we write these options inline inside
// a `.helper.bulk`, it works as expected
const options: BulkHelperOptions = {
datasource: [],
onDocument (doc: Record<string, any>) {
return [{ update: { _index: 'test' } }, doc]
}
}
expectAssignable<BulkHelperOptions>(options)
}
// delete
{
const options = {
datasource: [],
onDocument (doc: Record<string, any>) {
return { delete: { _index: 'test' } }
}
}
expectAssignable<BulkHelperOptions>(options)
}
/// .helpers.scrollSearch
// just search params
{
async function test () {
const scrollSearch = client.helpers.scrollSearch({
index: 'test',
body: {
query: {
match: { foo: 'bar' }
}
}
})
for await (const response of scrollSearch) {
expectAssignable<ScrollSearchResponse>(response)
}
}
}
// search params and options
{
async function test () {
const scrollSearch = client.helpers.scrollSearch({
index: 'test',
body: {
query: {
match: { foo: 'bar' }
}
}
}, { ignore: [404] })
for await (const response of scrollSearch) {
expectAssignable<ScrollSearchResponse>(response)
expectType<ResponseBody<Record<string, any>>>(response.body)
expectType<unknown[]>(response.documents)
expectType<unknown>(response.meta.context)
}
}
}
// with type defs
{
interface SearchBody {
query: {
match: { foo: string }
}
}
interface ShardsResponse {
total: number;
successful: number;
failed: number;
skipped: number;
}
interface Explanation {
value: number;
description: string;
details: Explanation[];
}
interface SearchResponse<T> {
took: number;
timed_out: boolean;
_scroll_id?: string;
_shards: ShardsResponse;
hits: {
total: number;
max_score: number;
hits: Array<{
_index: string;
_type: string;
_id: string;
_score: number;
_source: T;
_version?: number;
_explanation?: Explanation;
fields?: any;
highlight?: any;
inner_hits?: any;
matched_queries?: string[];
sort?: string[];
}>;
};
aggregations?: any;
}
interface Source {
foo: string
}
async function test () {
const scrollSearch = client.helpers.scrollSearch<SearchBody, Source, SearchResponse<Source>, string>({
index: 'test',
body: {
query: {
match: { foo: 'bar' }
}
}
})
for await (const response of scrollSearch) {
expectAssignable<ScrollSearchResponse>(response)
expectType<SearchResponse<Source>>(response.body)
expectType<Source[]>(response.documents)
expectType<string>(response.meta.context)
}
}
}
/// .helpers.scrollDocuments
// just search params
{
async function test () {
const scrollDocuments = client.helpers.scrollDocuments({
index: 'test',
body: {
query: {
match: { foo: 'bar' }
}
}
})
for await (const document of scrollDocuments) {
expectType<unknown>(document)
}
}
}
// search params and options
{
async function test () {
const scrollDocuments = client.helpers.scrollDocuments({
index: 'test',
body: {
query: {
match: { foo: 'bar' }
}
}
}, { ignore: [404] })
for await (const document of scrollDocuments) {
expectType<unknown>(document)
}
}
}
// with type defs
{
interface SearchBody {
query: {
match: { foo: string }
}
}
interface Source {
foo: string
}
async function test () {
const scrollDocuments = client.helpers.scrollDocuments<SearchBody, Source>({
index: 'test',
body: {
query: {
match: { foo: 'bar' }
}
}
})
for await (const document of scrollDocuments) {
expectType<Source>(document)
}
}
}
/// .helpers.search
// just search params
{
const p = client.helpers.search({
index: 'test',
body: {
query: {
match: { foo: 'bar' }
}
}
})
expectType<Promise<unknown[]>>(p)
expectType<unknown[]>(await p)
}
// search params and options
{
const p = client.helpers.search({
index: 'test',
body: {
query: {
match: { foo: 'bar' }
}
}
}, { ignore: [404] })
expectType<Promise<unknown[]>>(p)
expectType<unknown[]>(await p)
}
// with type defs
{
interface SearchBody {
query: {
match: { foo: string }
}
}
interface Source {
foo: string
}
const p = client.helpers.search<SearchBody, Source>({
index: 'test',
body: {
query: {
match: { foo: 'bar' }
}
}
})
expectType<Promise<Source[]>>(p)
expectType<Source[]>(await p)
}

813
test/unit/helpers/bulk.test.js Normal file
View File

@ -0,0 +1,813 @@
// Licensed to Elasticsearch B.V under one or more agreements.
// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
// See the LICENSE file in the project root for more information
'use strict'
const { createReadStream } = require('fs')
const { join } = require('path')
const split = require('split2')
const semver = require('semver')
const { test } = require('tap')
const { Client, errors } = require('../../../')
const { buildServer, connection } = require('../../utils')
const dataset = [
{ user: 'jon', age: 23 },
{ user: 'arya', age: 18 },
{ user: 'tyrion', age: 39 }
]
test('bulk index', t => {
t.test('datasource as array', t => {
t.test('Should perform a bulk request', async t => {
let count = 0
const MockConnection = connection.buildMockConnection({
onRequest (params) {
t.strictEqual(params.path, '/_bulk')
t.match(params.headers, { 'Content-Type': 'application/x-ndjson' })
const [action, payload] = params.body.split('\n')
t.deepEqual(JSON.parse(action), { index: { _index: 'test' } })
t.deepEqual(JSON.parse(payload), dataset[count++])
return { body: { errors: false, items: [{}] } }
}
})
const client = new Client({
node: 'http://localhost:9200',
Connection: MockConnection
})
const result = await client.helpers.bulk({
datasource: dataset.slice(),
flushBytes: 1,
concurrency: 1,
onDocument (doc) {
return {
index: { _index: 'test' }
}
},
onDrop (doc) {
t.fail('This should never be called')
}
})
t.type(result.time, 'number')
t.type(result.bytes, 'number')
t.match(result, {
total: 3,
successful: 3,
retry: 0,
failed: 0,
aborted: false
})
})
t.test('Should perform a bulk request (with concurrency)', async t => {
let count = 0
const MockConnection = connection.buildMockConnection({
onRequest (params) {
t.strictEqual(params.path, '/_bulk')
t.match(params.headers, { 'Content-Type': 'application/x-ndjson' })
const [action, payload] = params.body.split('\n')
t.deepEqual(JSON.parse(action), { index: { _index: 'test' } })
t.deepEqual(JSON.parse(payload), dataset[count++])
return { body: { errors: false, items: [{}] } }
}
})
const client = new Client({
node: 'http://localhost:9200',
Connection: MockConnection
})
const result = await client.helpers.bulk({
datasource: dataset.slice(),
flushBytes: 1,
concurrency: 3,
onDocument (doc) {
return {
index: { _index: 'test' }
}
},
onDrop (doc) {
t.fail('This should never be called')
}
})
t.type(result.time, 'number')
t.type(result.bytes, 'number')
t.match(result, {
total: 3,
successful: 3,
retry: 0,
failed: 0,
aborted: false
})
})
t.test('Should perform a bulk request (high flush size)', async t => {
const MockConnection = connection.buildMockConnection({
onRequest (params) {
t.strictEqual(params.path, '/_bulk')
t.match(params.headers, { 'Content-Type': 'application/x-ndjson' })
t.strictEqual(params.body.split('\n').filter(Boolean).length, 6)
return { body: { errors: false, items: new Array(3).fill({}) } }
}
})
const client = new Client({
node: 'http://localhost:9200',
Connection: MockConnection
})
const result = await client.helpers.bulk({
datasource: dataset.slice(),
flushBytes: 5000000,
concurrency: 1,
onDocument (doc) {
return {
index: { _index: 'test' }
}
},
onDrop (doc) {
t.fail('This should never be called')
}
})
t.type(result.time, 'number')
t.type(result.bytes, 'number')
t.match(result, {
total: 3,
successful: 3,
retry: 0,
failed: 0,
aborted: false
})
})
t.test('Should perform a bulk request (custom action)', async t => {
let count = 0
const MockConnection = connection.buildMockConnection({
onRequest (params) {
t.strictEqual(params.path, '/_bulk')
t.match(params.headers, { 'Content-Type': 'application/x-ndjson' })
const [action, payload] = params.body.split('\n')
t.deepEqual(JSON.parse(action), { index: { _index: 'test', _id: count } })
t.deepEqual(JSON.parse(payload), dataset[count++])
return { body: { errors: false, items: [{}] } }
}
})
const client = new Client({
node: 'http://localhost:9200',
Connection: MockConnection
})
let id = 0
const result = await client.helpers.bulk({
datasource: dataset.slice(),
flushBytes: 1,
concurrency: 1,
onDocument (doc) {
return {
index: {
_index: 'test',
_id: id++
}
}
},
onDrop (doc) {
t.fail('This should never be called')
}
})
t.type(result.time, 'number')
t.type(result.bytes, 'number')
t.match(result, {
total: 3,
successful: 3,
retry: 0,
failed: 0,
aborted: false
})
})
t.test('Should perform a bulk request (retry)', async t => {
if (semver.lt(process.versions.node, '10.0.0')) {
t.skip('This test will not pass on Node v8')
return
}
async function handler (req, res) {
t.strictEqual(req.url, '/_bulk')
t.match(req.headers, { 'content-type': 'application/x-ndjson' })
let body = ''
req.setEncoding('utf8')
for await (const chunk of req) {
body += chunk
}
const [, payload] = body.split('\n')
res.setHeader('content-type', 'application/json')
if (JSON.parse(payload).user === 'arya') {
res.end(JSON.stringify({
took: 0,
errors: true,
items: [{
index: {
status: 429
}
}]
}))
} else {
res.end(JSON.stringify({
took: 0,
errors: false,
items: [{}]
}))
}
}
const [{ port }, server] = await buildServer(handler)
const client = new Client({ node: `http://localhost:${port}` })
const result = await client.helpers.bulk({
datasource: dataset.slice(),
flushBytes: 1,
concurrency: 1,
wait: 10,
retries: 1,
onDocument (doc) {
return {
index: { _index: 'test' }
}
},
onDrop (doc) {
t.deepEqual(doc, {
status: 429,
error: null,
operation: { index: { _index: 'test' } },
document: { user: 'arya', age: 18 },
retried: true
})
}
})
t.type(result.time, 'number')
t.type(result.bytes, 'number')
t.match(result, {
total: 3,
successful: 2,
retry: 2,
failed: 1,
aborted: false
})
server.stop()
})
t.test('Should perform a bulk request (failure)', async t => {
if (semver.lt(process.versions.node, '10.0.0')) {
t.skip('This test will not pass on Node v8')
return
}
async function handler (req, res) {
t.strictEqual(req.url, '/_bulk')
t.match(req.headers, { 'content-type': 'application/x-ndjson' })
let body = ''
req.setEncoding('utf8')
for await (const chunk of req) {
body += chunk
}
const [, payload] = body.split('\n')
res.setHeader('content-type', 'application/json')
if (JSON.parse(payload).user === 'arya') {
res.end(JSON.stringify({
took: 0,
errors: true,
items: [{
index: {
status: 400,
error: { something: 'went wrong' }
}
}]
}))
} else {
res.end(JSON.stringify({
took: 0,
errors: false,
items: [{}]
}))
}
}
const [{ port }, server] = await buildServer(handler)
const client = new Client({ node: `http://localhost:${port}` })
const result = await client.helpers.bulk({
datasource: dataset.slice(),
flushBytes: 1,
concurrency: 1,
wait: 10,
onDocument (doc) {
return {
index: { _index: 'test' }
}
},
onDrop (doc) {
t.deepEqual(doc, {
status: 400,
error: { something: 'went wrong' },
operation: { index: { _index: 'test' } },
document: { user: 'arya', age: 18 },
retried: false
})
}
})
t.type(result.time, 'number')
t.type(result.bytes, 'number')
t.match(result, {
total: 3,
successful: 2,
retry: 0,
failed: 1,
aborted: false
})
server.stop()
})
t.test('Server error', async t => {
const MockConnection = connection.buildMockConnection({
onRequest (params) {
return {
statusCode: 500,
body: { somothing: 'went wrong' }
}
}
})
const client = new Client({
node: 'http://localhost:9200',
Connection: MockConnection
})
const b = client.helpers.bulk({
datasource: dataset.slice(),
flushBytes: 1,
concurrency: 1,
onDocument (doc) {
return {
index: { _index: 'test' }
}
},
onDrop (doc) {
t.fail('This should never be called')
}
})
try {
await b
t.fail('Should throw')
} catch (err) {
t.true(err instanceof errors.ResponseError)
}
})
t.test('Server error (high flush size, to trigger the finish error)', async t => {
const MockConnection = connection.buildMockConnection({
onRequest (params) {
return {
statusCode: 500,
body: { somothing: 'went wrong' }
}
}
})
const client = new Client({
node: 'http://localhost:9200',
Connection: MockConnection
})
const b = client.helpers.bulk({
datasource: dataset.slice(),
flushBytes: 5000000,
concurrency: 1,
onDocument (doc) {
return {
index: { _index: 'test' }
}
},
onDrop (doc) {
t.fail('This should never be called')
}
})
try {
await b
t.fail('Should throw')
} catch (err) {
t.true(err instanceof errors.ResponseError)
}
})
t.test('Should abort a bulk request', async t => {
if (semver.lt(process.versions.node, '10.0.0')) {
t.skip('This test will not pass on Node v8')
return
}
async function handler (req, res) {
t.strictEqual(req.url, '/_bulk')
t.match(req.headers, { 'content-type': 'application/x-ndjson' })
let body = ''
req.setEncoding('utf8')
for await (const chunk of req) {
body += chunk
}
const [, payload] = body.split('\n')
res.setHeader('content-type', 'application/json')
if (JSON.parse(payload).user === 'arya') {
res.end(JSON.stringify({
took: 0,
errors: true,
items: [{
index: {
status: 400,
error: { something: 'went wrong' }
}
}]
}))
} else {
res.end(JSON.stringify({
took: 0,
errors: false,
items: [{}]
}))
}
}
const [{ port }, server] = await buildServer(handler)
const client = new Client({ node: `http://localhost:${port}` })
const b = client.helpers.bulk({
datasource: dataset.slice(),
flushBytes: 1,
concurrency: 1,
wait: 10,
onDocument (doc) {
return {
index: { _index: 'test' }
}
},
onDrop (doc) {
b.abort()
}
})
const result = await b
t.type(result.time, 'number')
t.type(result.bytes, 'number')
t.match(result, {
total: 2,
successful: 1,
retry: 0,
failed: 1,
aborted: true
})
server.stop()
})
t.end()
})
t.test('datasource as stream', t => {
t.test('Should perform a bulk request', async t => {
let count = 0
const MockConnection = connection.buildMockConnection({
onRequest (params) {
t.strictEqual(params.path, '/_bulk')
t.match(params.headers, { 'Content-Type': 'application/x-ndjson' })
const [action, payload] = params.body.split('\n')
t.deepEqual(JSON.parse(action), { index: { _index: 'test', _id: count } })
t.deepEqual(JSON.parse(payload), dataset[count++])
return { body: { errors: false, items: [{}] } }
}
})
const client = new Client({
node: 'http://localhost:9200',
Connection: MockConnection
})
const stream = createReadStream(join(__dirname, '..', '..', 'fixtures', 'small-dataset.ndjson'), 'utf8')
let id = 0
const result = await client.helpers.bulk({
datasource: stream.pipe(split()),
flushBytes: 1,
concurrency: 1,
onDocument (doc) {
return {
index: {
_index: 'test',
_id: id++
}
}
},
onDrop (doc) {
t.fail('This should never be called')
}
})
t.type(result.time, 'number')
t.type(result.bytes, 'number')
t.match(result, {
total: 3,
successful: 3,
retry: 0,
failed: 0,
aborted: false
})
})
t.end()
})
t.end()
})
test('bulk create', t => {
t.test('Should perform a bulk request', async t => {
let count = 0
const MockConnection = connection.buildMockConnection({
onRequest (params) {
t.strictEqual(params.path, '/_bulk')
t.match(params.headers, { 'Content-Type': 'application/x-ndjson' })
const [action, payload] = params.body.split('\n')
t.deepEqual(JSON.parse(action), { create: { _index: 'test', _id: count } })
t.deepEqual(JSON.parse(payload), dataset[count++])
return { body: { errors: false, items: [{}] } }
}
})
const client = new Client({
node: 'http://localhost:9200',
Connection: MockConnection
})
let id = 0
const result = await client.helpers.bulk({
datasource: dataset.slice(),
flushBytes: 1,
concurrency: 1,
onDocument (doc) {
return {
create: {
_index: 'test',
_id: id++
}
}
},
onDrop (doc) {
t.fail('This should never be called')
}
})
t.type(result.time, 'number')
t.type(result.bytes, 'number')
t.match(result, {
total: 3,
successful: 3,
retry: 0,
failed: 0,
aborted: false
})
})
t.end()
})
test('bulk update', t => {
t.test('Should perform a bulk request', async t => {
let count = 0
const MockConnection = connection.buildMockConnection({
onRequest (params) {
t.strictEqual(params.path, '/_bulk')
t.match(params.headers, { 'Content-Type': 'application/x-ndjson' })
const [action, payload] = params.body.split('\n')
t.deepEqual(JSON.parse(action), { update: { _index: 'test', _id: count } })
t.deepEqual(JSON.parse(payload), { doc: dataset[count++], doc_as_upsert: true })
return { body: { errors: false, items: [{}] } }
}
})
const client = new Client({
node: 'http://localhost:9200',
Connection: MockConnection
})
let id = 0
const result = await client.helpers.bulk({
datasource: dataset.slice(),
flushBytes: 1,
concurrency: 1,
onDocument (doc) {
return [{
update: {
_index: 'test',
_id: id++
}
}, {
doc_as_upsert: true
}]
},
onDrop (doc) {
t.fail('This should never be called')
}
})
t.type(result.time, 'number')
t.type(result.bytes, 'number')
t.match(result, {
total: 3,
successful: 3,
retry: 0,
failed: 0,
aborted: false
})
})
t.end()
})
test('bulk delete', t => {
t.test('Should perform a bulk request', async t => {
let count = 0
const MockConnection = connection.buildMockConnection({
onRequest (params) {
t.strictEqual(params.path, '/_bulk')
t.match(params.headers, { 'Content-Type': 'application/x-ndjson' })
t.deepEqual(JSON.parse(params.body), { delete: { _index: 'test', _id: count++ } })
return { body: { errors: false, items: [{}] } }
}
})
const client = new Client({
node: 'http://localhost:9200',
Connection: MockConnection
})
let id = 0
const result = await client.helpers.bulk({
datasource: dataset.slice(),
flushBytes: 1,
concurrency: 1,
onDocument (doc) {
return {
delete: {
_index: 'test',
_id: id++
}
}
},
onDrop (doc) {
t.fail('This should never be called')
}
})
t.type(result.time, 'number')
t.type(result.bytes, 'number')
t.match(result, {
total: 3,
successful: 3,
retry: 0,
failed: 0,
aborted: false
})
})
t.test('Should perform a bulk request (failure)', async t => {
if (semver.lt(process.versions.node, '10.0.0')) {
t.skip('This test will not pass on Node v8')
return
}
async function handler (req, res) {
t.strictEqual(req.url, '/_bulk')
t.match(req.headers, { 'content-type': 'application/x-ndjson' })
let body = ''
req.setEncoding('utf8')
for await (const chunk of req) {
body += chunk
}
res.setHeader('content-type', 'application/json')
if (JSON.parse(body).delete._id === 1) {
res.end(JSON.stringify({
took: 0,
errors: true,
items: [{
delete: {
status: 400,
error: { something: 'went wrong' }
}
}]
}))
} else {
res.end(JSON.stringify({
took: 0,
errors: false,
items: [{}]
}))
}
}
const [{ port }, server] = await buildServer(handler)
const client = new Client({ node: `http://localhost:${port}` })
let id = 0
const result = await client.helpers.bulk({
datasource: dataset.slice(),
flushBytes: 1,
concurrency: 1,
wait: 10,
onDocument (doc) {
return {
delete: {
_index: 'test',
_id: id++
}
}
},
onDrop (doc) {
t.deepEqual(doc, {
status: 400,
error: { something: 'went wrong' },
operation: { delete: { _index: 'test', _id: 1 } },
document: null,
retried: false
})
}
})
t.type(result.time, 'number')
t.type(result.bytes, 'number')
t.match(result, {
total: 3,
successful: 2,
retry: 0,
failed: 1,
aborted: false
})
server.stop()
})
t.end()
})
test('errors', t => {
t.test('datasource type', async t => {
const client = new Client({
node: 'http://localhost:9200'
})
try {
await client.helpers.bulk({
datasource: 'hello',
onDocument (doc) {
return {
index: { _index: 'test' }
}
}
})
} catch (err) {
t.true(err instanceof errors.ConfigurationError)
t.is(err.message, 'bulk helper: the datasource must be an array or a buffer or a readable stream')
}
})
t.test('missing datasource', async t => {
const client = new Client({
node: 'http://localhost:9200'
})
try {
await client.helpers.bulk({
onDocument (doc) {
return {
index: { _index: 'test' }
}
}
})
} catch (err) {
t.true(err instanceof errors.ConfigurationError)
t.is(err.message, 'bulk helper: the datasource is required')
}
})
t.test('missing onDocument', async t => {
const client = new Client({
node: 'http://localhost:9200'
})
try {
await client.helpers.bulk({
datasource: dataset.slice()
})
} catch (err) {
t.true(err instanceof errors.ConfigurationError)
t.is(err.message, 'bulk helper: the onDocument callback is required')
}
})
t.end()
})

220
test/unit/helpers/scroll.test.js Normal file
View File

@ -0,0 +1,220 @@
// Licensed to Elasticsearch B.V under one or more agreements.
// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
// See the LICENSE file in the project root for more information
'use strict'
const { test } = require('tap')
const { Client, errors } = require('../../../')
const { connection } = require('../../utils')
test('Scroll search', async t => {
var count = 0
const MockConnection = connection.buildMockConnection({
onRequest (params) {
t.strictEqual(params.querystring, 'scroll=1m')
return {
body: {
_scroll_id: count === 3 ? undefined : 'id',
count,
hits: {
hits: [
{ _source: { one: 'one' } },
{ _source: { two: 'two' } },
{ _source: { three: 'three' } }
]
}
}
}
}
})
const client = new Client({
node: 'http://localhost:9200',
Connection: MockConnection
})
const scrollSearch = client.helpers.scrollSearch({
index: 'test',
body: { foo: 'bar' }
})
for await (const result of scrollSearch) {
t.strictEqual(result.body.count, count)
if (count < 3) {
t.strictEqual(result.body._scroll_id, 'id')
} else {
t.strictEqual(result.body._scroll_id, undefined)
}
count += 1
}
})
test('Clear a scroll search', async t => {
var count = 0
const MockConnection = connection.buildMockConnection({
onRequest (params) {
if (params.method === 'DELETE') {
const body = JSON.parse(params.body)
t.strictEqual(body.scroll_id, 'id')
}
return {
body: {
_scroll_id: count === 3 ? undefined : 'id',
count,
hits: {
hits: [
{ _source: { one: 'one' } },
{ _source: { two: 'two' } },
{ _source: { three: 'three' } }
]
}
}
}
}
})
const client = new Client({
node: 'http://localhost:9200',
Connection: MockConnection
})
const scrollSearch = client.helpers.scrollSearch({
index: 'test',
body: { foo: 'bar' }
})
for await (const result of scrollSearch) {
if (count === 2) {
t.fail('The scroll search should be cleared')
}
t.strictEqual(result.body.count, count)
if (count === 1) {
await result.clear()
}
count += 1
}
})
test('Scroll search (retry)', async t => {
var count = 0
const MockConnection = connection.buildMockConnection({
onRequest (params) {
if (count === 1) {
count += 1
return { body: {}, statusCode: 429 }
}
return {
statusCode: 200,
body: {
_scroll_id: count === 4 ? undefined : 'id',
count,
hits: {
hits: [
{ _source: { one: 'one' } },
{ _source: { two: 'two' } },
{ _source: { three: 'three' } }
]
}
}
}
}
})
const client = new Client({
node: 'http://localhost:9200',
Connection: MockConnection
})
const scrollSearch = client.helpers.scrollSearch({
index: 'test',
body: { foo: 'bar' }
}, {
wait: 10
})
for await (const result of scrollSearch) {
t.strictEqual(result.body.count, count)
t.notStrictEqual(result.body.count, 1)
if (count < 4) {
t.strictEqual(result.body._scroll_id, 'id')
} else {
t.strictEqual(result.body._scroll_id, undefined)
}
count += 1
}
})
test('Scroll search (retry throws and maxRetries)', async t => {
var count = 0
const MockConnection = connection.buildMockConnection({
onRequest (params) {
count += 1
return { body: {}, statusCode: 429 }
}
})
const client = new Client({
node: 'http://localhost:9200',
Connection: MockConnection,
maxRetries: 5
})
const scrollSearch = client.helpers.scrollSearch({
index: 'test',
body: { foo: 'bar' }
}, {
wait: 10
})
try {
for await (const result of scrollSearch) { // eslint-disable-line
t.fail('we should not be here')
}
} catch (err) {
t.true(err instanceof errors.ResponseError)
t.strictEqual(err.statusCode, 429)
t.strictEqual(count, 5)
}
})
test('Scroll search documents', async t => {
var count = 0
const MockConnection = connection.buildMockConnection({
onRequest (params) {
return {
body: {
_scroll_id: count === 3 ? undefined : 'id',
count,
hits: {
hits: [
{ _source: { val: 1 * count } },
{ _source: { val: 2 * count } },
{ _source: { val: 3 * count } }
]
}
}
}
}
})
const client = new Client({
node: 'http://localhost:9200',
Connection: MockConnection
})
const scrollSearch = client.helpers.scrollDocuments({
index: 'test',
body: { foo: 'bar' }
})
let n = 1
for await (const hit of scrollSearch) {
t.deepEqual(hit, { val: n * count })
n += 1
if (n === 4) {
count += 1
n = 1
}
}
})

42
test/unit/helpers/search.test.js Normal file
View File

@ -0,0 +1,42 @@
// Licensed to Elasticsearch B.V under one or more agreements.
// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
// See the LICENSE file in the project root for more information
'use strict'
const { test } = require('tap')
const { Client } = require('../../../')
const { connection } = require('../../utils')
test('Search should have an additional documents property', async t => {
const MockConnection = connection.buildMockConnection({
onRequest (params) {
return {
body: {
hits: {
hits: [
{ _source: { one: 'one' } },
{ _source: { two: 'two' } },
{ _source: { three: 'three' } }
]
}
}
}
}
})
const client = new Client({
node: 'http://localhost:9200',
Connection: MockConnection
})
const result = await client.helpers.search({
index: 'test',
body: { foo: 'bar' }
})
t.deepEqual(result, [
{ one: 'one' },
{ two: 'two' },
{ three: 'three' }
])
})

37
test/utils/MockConnection.js
View File

@ -4,6 +4,7 @@
'use strict'
const assert = require('assert')
const { Connection } = require('../../index')
const { TimeoutError } = require('../../lib/errors')
const intoStream = require('into-stream')
@ -99,6 +100,39 @@ class MockConnectionSniff extends Connection {
}
}
}
function buildMockConnection (opts) {
assert(opts.onRequest, 'Missing required onRequest option')
class MockConnection extends Connection {
request (params, callback) {
var { body, statusCode } = opts.onRequest(params)
if (typeof body !== 'string') {
body = JSON.stringify(body)
}
var aborted = false
const stream = intoStream(body)
stream.statusCode = statusCode || 200
stream.headers = {
'content-type': 'application/json;utf=8',
date: new Date().toISOString(),
connection: 'keep-alive',
'content-length': Buffer.byteLength(body)
}
process.nextTick(() => {
if (!aborted) {
callback(null, stream)
}
})
return {
abort: () => { aborted = true }
}
}
}
return MockConnection
}
function setStatusCode (path) {
const statusCode = Number(path.slice(1))
if (Number.isInteger(statusCode)) {
@ -111,5 +145,6 @@ module.exports = {
MockConnection,
MockConnectionTimeout,
MockConnectionError,
MockConnectionSniff
MockConnectionSniff,
buildMockConnection
}

20
test/utils/buildServer.js
View File

@ -38,11 +38,21 @@ function buildServer (handler, opts, cb) {
console.log('http server error', err)
process.exit(1)
})
server.listen(0, () => {
const port = server.address().port
debug(`Server '${serverId}' booted on port ${port}`)
cb(Object.assign({}, secureOpts, { port }), server)
})
if (cb === undefined) {
return new Promise((resolve, reject) => {
server.listen(0, () => {
const port = server.address().port
debug(`Server '${serverId}' booted on port ${port}`)
resolve([Object.assign({}, secureOpts, { port }), server])
})
})
} else {
server.listen(0, () => {
const port = server.address().port
debug(`Server '${serverId}' booted on port ${port}`)
cb(Object.assign({}, secureOpts, { port }), server)
})
}
}
module.exports = buildServer

20
test/utils/index.js
View File

@ -4,12 +4,30 @@
'use strict'
const { promisify } = require('util')
const sleep = promisify(setTimeout)
const buildServer = require('./buildServer')
const buildCluster = require('./buildCluster')
const connection = require('./MockConnection')
async function waitCluster (client, waitForStatus = 'green', timeout = '50s', times = 0) {
if (!client) {
throw new Error('waitCluster helper: missing client instance')
}
try {
await client.cluster.health({ waitForStatus, timeout })
} catch (err) {
if (++times < 10) {
await sleep(5000)
return waitCluster(client, waitForStatus, timeout, times)
}
throw err
}
}
module.exports = {
buildServer,
buildCluster,
connection
connection,
waitCluster
}