Client helpers (#1107)
* Added client helpers * Updated test * The search helper should return only the documents * Added code comments * Fixed bug * Updated test * Removed bulkSize and added flushBytes * Updated test * Added concurrency * Updated test * Added support for 429 handling in the scroll search helper * Updated test * Updated stats count * Updated test * Fix test * Use client maxRetries as default * Updated type definitions * Refactored bulk helper to be more consistent with the client api * Updated test * Improved error handling, added refreshOnCompletion option and forward additinal options to the bulk api * Updated type definitions * Updated test * Fixed test on Node v8 * Updated test * Added TODO * Updated docs * Added Node v8 note * Updated scripts * Removed useless files * Added helpers to integration test * Fix cli argument position * Moar fixes * Test run elasticsearch in github actions * Use master action version * Add vm.max_map_count step * Test new action setup * Added Configure sysctl limits step * Updated action to latest version * Don't run helpers integration test in jenkins * Run helpers integratino test also with Node v10 * Updated docs * Updated docs * Updated helpers type definitions * Added test for helpers type definitions * Added license header
This commit is contained in:
Tomas Della Vedova
committed by
GitHub
GitHub
parent
6c82a4967e
commit
d7836a16af
813
test/unit/helpers/bulk.test.js
Normal file
813
test/unit/helpers/bulk.test.js
Normal file
@ -0,0 +1,813 @@
|
||||
// Licensed to Elasticsearch B.V under one or more agreements.
|
||||
// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
|
||||
// See the LICENSE file in the project root for more information
|
||||
|
||||
'use strict'
|
||||
|
||||
const { createReadStream } = require('fs')
|
||||
const { join } = require('path')
|
||||
const split = require('split2')
|
||||
const semver = require('semver')
|
||||
const { test } = require('tap')
|
||||
const { Client, errors } = require('../../../')
|
||||
const { buildServer, connection } = require('../../utils')
|
||||
|
||||
const dataset = [
|
||||
{ user: 'jon', age: 23 },
|
||||
{ user: 'arya', age: 18 },
|
||||
{ user: 'tyrion', age: 39 }
|
||||
]
|
||||
|
||||
test('bulk index', t => {
|
||||
t.test('datasource as array', t => {
|
||||
t.test('Should perform a bulk request', async t => {
|
||||
let count = 0
|
||||
const MockConnection = connection.buildMockConnection({
|
||||
onRequest (params) {
|
||||
t.strictEqual(params.path, '/_bulk')
|
||||
t.match(params.headers, { 'Content-Type': 'application/x-ndjson' })
|
||||
const [action, payload] = params.body.split('\n')
|
||||
t.deepEqual(JSON.parse(action), { index: { _index: 'test' } })
|
||||
t.deepEqual(JSON.parse(payload), dataset[count++])
|
||||
return { body: { errors: false, items: [{}] } }
|
||||
}
|
||||
})
|
||||
|
||||
const client = new Client({
|
||||
node: 'http://localhost:9200',
|
||||
Connection: MockConnection
|
||||
})
|
||||
const result = await client.helpers.bulk({
|
||||
datasource: dataset.slice(),
|
||||
flushBytes: 1,
|
||||
concurrency: 1,
|
||||
onDocument (doc) {
|
||||
return {
|
||||
index: { _index: 'test' }
|
||||
}
|
||||
},
|
||||
onDrop (doc) {
|
||||
t.fail('This should never be called')
|
||||
}
|
||||
})
|
||||
|
||||
t.type(result.time, 'number')
|
||||
t.type(result.bytes, 'number')
|
||||
t.match(result, {
|
||||
total: 3,
|
||||
successful: 3,
|
||||
retry: 0,
|
||||
failed: 0,
|
||||
aborted: false
|
||||
})
|
||||
})
|
||||
|
||||
t.test('Should perform a bulk request (with concurrency)', async t => {
|
||||
let count = 0
|
||||
const MockConnection = connection.buildMockConnection({
|
||||
onRequest (params) {
|
||||
t.strictEqual(params.path, '/_bulk')
|
||||
t.match(params.headers, { 'Content-Type': 'application/x-ndjson' })
|
||||
const [action, payload] = params.body.split('\n')
|
||||
t.deepEqual(JSON.parse(action), { index: { _index: 'test' } })
|
||||
t.deepEqual(JSON.parse(payload), dataset[count++])
|
||||
return { body: { errors: false, items: [{}] } }
|
||||
}
|
||||
})
|
||||
|
||||
const client = new Client({
|
||||
node: 'http://localhost:9200',
|
||||
Connection: MockConnection
|
||||
})
|
||||
const result = await client.helpers.bulk({
|
||||
datasource: dataset.slice(),
|
||||
flushBytes: 1,
|
||||
concurrency: 3,
|
||||
onDocument (doc) {
|
||||
return {
|
||||
index: { _index: 'test' }
|
||||
}
|
||||
},
|
||||
onDrop (doc) {
|
||||
t.fail('This should never be called')
|
||||
}
|
||||
})
|
||||
|
||||
t.type(result.time, 'number')
|
||||
t.type(result.bytes, 'number')
|
||||
t.match(result, {
|
||||
total: 3,
|
||||
successful: 3,
|
||||
retry: 0,
|
||||
failed: 0,
|
||||
aborted: false
|
||||
})
|
||||
})
|
||||
|
||||
t.test('Should perform a bulk request (high flush size)', async t => {
|
||||
const MockConnection = connection.buildMockConnection({
|
||||
onRequest (params) {
|
||||
t.strictEqual(params.path, '/_bulk')
|
||||
t.match(params.headers, { 'Content-Type': 'application/x-ndjson' })
|
||||
t.strictEqual(params.body.split('\n').filter(Boolean).length, 6)
|
||||
return { body: { errors: false, items: new Array(3).fill({}) } }
|
||||
}
|
||||
})
|
||||
|
||||
const client = new Client({
|
||||
node: 'http://localhost:9200',
|
||||
Connection: MockConnection
|
||||
})
|
||||
const result = await client.helpers.bulk({
|
||||
datasource: dataset.slice(),
|
||||
flushBytes: 5000000,
|
||||
concurrency: 1,
|
||||
onDocument (doc) {
|
||||
return {
|
||||
index: { _index: 'test' }
|
||||
}
|
||||
},
|
||||
onDrop (doc) {
|
||||
t.fail('This should never be called')
|
||||
}
|
||||
})
|
||||
|
||||
t.type(result.time, 'number')
|
||||
t.type(result.bytes, 'number')
|
||||
t.match(result, {
|
||||
total: 3,
|
||||
successful: 3,
|
||||
retry: 0,
|
||||
failed: 0,
|
||||
aborted: false
|
||||
})
|
||||
})
|
||||
|
||||
t.test('Should perform a bulk request (custom action)', async t => {
|
||||
let count = 0
|
||||
const MockConnection = connection.buildMockConnection({
|
||||
onRequest (params) {
|
||||
t.strictEqual(params.path, '/_bulk')
|
||||
t.match(params.headers, { 'Content-Type': 'application/x-ndjson' })
|
||||
const [action, payload] = params.body.split('\n')
|
||||
t.deepEqual(JSON.parse(action), { index: { _index: 'test', _id: count } })
|
||||
t.deepEqual(JSON.parse(payload), dataset[count++])
|
||||
return { body: { errors: false, items: [{}] } }
|
||||
}
|
||||
})
|
||||
|
||||
const client = new Client({
|
||||
node: 'http://localhost:9200',
|
||||
Connection: MockConnection
|
||||
})
|
||||
let id = 0
|
||||
const result = await client.helpers.bulk({
|
||||
datasource: dataset.slice(),
|
||||
flushBytes: 1,
|
||||
concurrency: 1,
|
||||
onDocument (doc) {
|
||||
return {
|
||||
index: {
|
||||
_index: 'test',
|
||||
_id: id++
|
||||
}
|
||||
}
|
||||
},
|
||||
onDrop (doc) {
|
||||
t.fail('This should never be called')
|
||||
}
|
||||
})
|
||||
|
||||
t.type(result.time, 'number')
|
||||
t.type(result.bytes, 'number')
|
||||
t.match(result, {
|
||||
total: 3,
|
||||
successful: 3,
|
||||
retry: 0,
|
||||
failed: 0,
|
||||
aborted: false
|
||||
})
|
||||
})
|
||||
|
||||
t.test('Should perform a bulk request (retry)', async t => {
|
||||
if (semver.lt(process.versions.node, '10.0.0')) {
|
||||
t.skip('This test will not pass on Node v8')
|
||||
return
|
||||
}
|
||||
async function handler (req, res) {
|
||||
t.strictEqual(req.url, '/_bulk')
|
||||
t.match(req.headers, { 'content-type': 'application/x-ndjson' })
|
||||
|
||||
let body = ''
|
||||
req.setEncoding('utf8')
|
||||
for await (const chunk of req) {
|
||||
body += chunk
|
||||
}
|
||||
const [, payload] = body.split('\n')
|
||||
|
||||
res.setHeader('content-type', 'application/json')
|
||||
|
||||
if (JSON.parse(payload).user === 'arya') {
|
||||
res.end(JSON.stringify({
|
||||
took: 0,
|
||||
errors: true,
|
||||
items: [{
|
||||
index: {
|
||||
status: 429
|
||||
}
|
||||
}]
|
||||
}))
|
||||
} else {
|
||||
res.end(JSON.stringify({
|
||||
took: 0,
|
||||
errors: false,
|
||||
items: [{}]
|
||||
}))
|
||||
}
|
||||
}
|
||||
|
||||
const [{ port }, server] = await buildServer(handler)
|
||||
const client = new Client({ node: `http://localhost:${port}` })
|
||||
const result = await client.helpers.bulk({
|
||||
datasource: dataset.slice(),
|
||||
flushBytes: 1,
|
||||
concurrency: 1,
|
||||
wait: 10,
|
||||
retries: 1,
|
||||
onDocument (doc) {
|
||||
return {
|
||||
index: { _index: 'test' }
|
||||
}
|
||||
},
|
||||
onDrop (doc) {
|
||||
t.deepEqual(doc, {
|
||||
status: 429,
|
||||
error: null,
|
||||
operation: { index: { _index: 'test' } },
|
||||
document: { user: 'arya', age: 18 },
|
||||
retried: true
|
||||
})
|
||||
}
|
||||
})
|
||||
|
||||
t.type(result.time, 'number')
|
||||
t.type(result.bytes, 'number')
|
||||
t.match(result, {
|
||||
total: 3,
|
||||
successful: 2,
|
||||
retry: 2,
|
||||
failed: 1,
|
||||
aborted: false
|
||||
})
|
||||
server.stop()
|
||||
})
|
||||
|
||||
t.test('Should perform a bulk request (failure)', async t => {
|
||||
if (semver.lt(process.versions.node, '10.0.0')) {
|
||||
t.skip('This test will not pass on Node v8')
|
||||
return
|
||||
}
|
||||
async function handler (req, res) {
|
||||
t.strictEqual(req.url, '/_bulk')
|
||||
t.match(req.headers, { 'content-type': 'application/x-ndjson' })
|
||||
|
||||
let body = ''
|
||||
req.setEncoding('utf8')
|
||||
for await (const chunk of req) {
|
||||
body += chunk
|
||||
}
|
||||
const [, payload] = body.split('\n')
|
||||
|
||||
res.setHeader('content-type', 'application/json')
|
||||
|
||||
if (JSON.parse(payload).user === 'arya') {
|
||||
res.end(JSON.stringify({
|
||||
took: 0,
|
||||
errors: true,
|
||||
items: [{
|
||||
index: {
|
||||
status: 400,
|
||||
error: { something: 'went wrong' }
|
||||
}
|
||||
}]
|
||||
}))
|
||||
} else {
|
||||
res.end(JSON.stringify({
|
||||
took: 0,
|
||||
errors: false,
|
||||
items: [{}]
|
||||
}))
|
||||
}
|
||||
}
|
||||
|
||||
const [{ port }, server] = await buildServer(handler)
|
||||
const client = new Client({ node: `http://localhost:${port}` })
|
||||
const result = await client.helpers.bulk({
|
||||
datasource: dataset.slice(),
|
||||
flushBytes: 1,
|
||||
concurrency: 1,
|
||||
wait: 10,
|
||||
onDocument (doc) {
|
||||
return {
|
||||
index: { _index: 'test' }
|
||||
}
|
||||
},
|
||||
onDrop (doc) {
|
||||
t.deepEqual(doc, {
|
||||
status: 400,
|
||||
error: { something: 'went wrong' },
|
||||
operation: { index: { _index: 'test' } },
|
||||
document: { user: 'arya', age: 18 },
|
||||
retried: false
|
||||
})
|
||||
}
|
||||
})
|
||||
|
||||
t.type(result.time, 'number')
|
||||
t.type(result.bytes, 'number')
|
||||
t.match(result, {
|
||||
total: 3,
|
||||
successful: 2,
|
||||
retry: 0,
|
||||
failed: 1,
|
||||
aborted: false
|
||||
})
|
||||
server.stop()
|
||||
})
|
||||
|
||||
t.test('Server error', async t => {
|
||||
const MockConnection = connection.buildMockConnection({
|
||||
onRequest (params) {
|
||||
return {
|
||||
statusCode: 500,
|
||||
body: { somothing: 'went wrong' }
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
const client = new Client({
|
||||
node: 'http://localhost:9200',
|
||||
Connection: MockConnection
|
||||
})
|
||||
const b = client.helpers.bulk({
|
||||
datasource: dataset.slice(),
|
||||
flushBytes: 1,
|
||||
concurrency: 1,
|
||||
onDocument (doc) {
|
||||
return {
|
||||
index: { _index: 'test' }
|
||||
}
|
||||
},
|
||||
onDrop (doc) {
|
||||
t.fail('This should never be called')
|
||||
}
|
||||
})
|
||||
|
||||
try {
|
||||
await b
|
||||
t.fail('Should throw')
|
||||
} catch (err) {
|
||||
t.true(err instanceof errors.ResponseError)
|
||||
}
|
||||
})
|
||||
|
||||
t.test('Server error (high flush size, to trigger the finish error)', async t => {
|
||||
const MockConnection = connection.buildMockConnection({
|
||||
onRequest (params) {
|
||||
return {
|
||||
statusCode: 500,
|
||||
body: { somothing: 'went wrong' }
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
const client = new Client({
|
||||
node: 'http://localhost:9200',
|
||||
Connection: MockConnection
|
||||
})
|
||||
const b = client.helpers.bulk({
|
||||
datasource: dataset.slice(),
|
||||
flushBytes: 5000000,
|
||||
concurrency: 1,
|
||||
onDocument (doc) {
|
||||
return {
|
||||
index: { _index: 'test' }
|
||||
}
|
||||
},
|
||||
onDrop (doc) {
|
||||
t.fail('This should never be called')
|
||||
}
|
||||
})
|
||||
|
||||
try {
|
||||
await b
|
||||
t.fail('Should throw')
|
||||
} catch (err) {
|
||||
t.true(err instanceof errors.ResponseError)
|
||||
}
|
||||
})
|
||||
|
||||
t.test('Should abort a bulk request', async t => {
|
||||
if (semver.lt(process.versions.node, '10.0.0')) {
|
||||
t.skip('This test will not pass on Node v8')
|
||||
return
|
||||
}
|
||||
async function handler (req, res) {
|
||||
t.strictEqual(req.url, '/_bulk')
|
||||
t.match(req.headers, { 'content-type': 'application/x-ndjson' })
|
||||
|
||||
let body = ''
|
||||
req.setEncoding('utf8')
|
||||
for await (const chunk of req) {
|
||||
body += chunk
|
||||
}
|
||||
const [, payload] = body.split('\n')
|
||||
|
||||
res.setHeader('content-type', 'application/json')
|
||||
|
||||
if (JSON.parse(payload).user === 'arya') {
|
||||
res.end(JSON.stringify({
|
||||
took: 0,
|
||||
errors: true,
|
||||
items: [{
|
||||
index: {
|
||||
status: 400,
|
||||
error: { something: 'went wrong' }
|
||||
}
|
||||
}]
|
||||
}))
|
||||
} else {
|
||||
res.end(JSON.stringify({
|
||||
took: 0,
|
||||
errors: false,
|
||||
items: [{}]
|
||||
}))
|
||||
}
|
||||
}
|
||||
|
||||
const [{ port }, server] = await buildServer(handler)
|
||||
const client = new Client({ node: `http://localhost:${port}` })
|
||||
const b = client.helpers.bulk({
|
||||
datasource: dataset.slice(),
|
||||
flushBytes: 1,
|
||||
concurrency: 1,
|
||||
wait: 10,
|
||||
onDocument (doc) {
|
||||
return {
|
||||
index: { _index: 'test' }
|
||||
}
|
||||
},
|
||||
onDrop (doc) {
|
||||
b.abort()
|
||||
}
|
||||
})
|
||||
|
||||
const result = await b
|
||||
t.type(result.time, 'number')
|
||||
t.type(result.bytes, 'number')
|
||||
t.match(result, {
|
||||
total: 2,
|
||||
successful: 1,
|
||||
retry: 0,
|
||||
failed: 1,
|
||||
aborted: true
|
||||
})
|
||||
server.stop()
|
||||
})
|
||||
|
||||
t.end()
|
||||
})
|
||||
|
||||
t.test('datasource as stream', t => {
|
||||
t.test('Should perform a bulk request', async t => {
|
||||
let count = 0
|
||||
const MockConnection = connection.buildMockConnection({
|
||||
onRequest (params) {
|
||||
t.strictEqual(params.path, '/_bulk')
|
||||
t.match(params.headers, { 'Content-Type': 'application/x-ndjson' })
|
||||
const [action, payload] = params.body.split('\n')
|
||||
t.deepEqual(JSON.parse(action), { index: { _index: 'test', _id: count } })
|
||||
t.deepEqual(JSON.parse(payload), dataset[count++])
|
||||
return { body: { errors: false, items: [{}] } }
|
||||
}
|
||||
})
|
||||
|
||||
const client = new Client({
|
||||
node: 'http://localhost:9200',
|
||||
Connection: MockConnection
|
||||
})
|
||||
const stream = createReadStream(join(__dirname, '..', '..', 'fixtures', 'small-dataset.ndjson'), 'utf8')
|
||||
|
||||
let id = 0
|
||||
const result = await client.helpers.bulk({
|
||||
datasource: stream.pipe(split()),
|
||||
flushBytes: 1,
|
||||
concurrency: 1,
|
||||
onDocument (doc) {
|
||||
return {
|
||||
index: {
|
||||
_index: 'test',
|
||||
_id: id++
|
||||
}
|
||||
}
|
||||
},
|
||||
onDrop (doc) {
|
||||
t.fail('This should never be called')
|
||||
}
|
||||
})
|
||||
|
||||
t.type(result.time, 'number')
|
||||
t.type(result.bytes, 'number')
|
||||
t.match(result, {
|
||||
total: 3,
|
||||
successful: 3,
|
||||
retry: 0,
|
||||
failed: 0,
|
||||
aborted: false
|
||||
})
|
||||
})
|
||||
|
||||
t.end()
|
||||
})
|
||||
|
||||
t.end()
|
||||
})
|
||||
|
||||
test('bulk create', t => {
|
||||
t.test('Should perform a bulk request', async t => {
|
||||
let count = 0
|
||||
const MockConnection = connection.buildMockConnection({
|
||||
onRequest (params) {
|
||||
t.strictEqual(params.path, '/_bulk')
|
||||
t.match(params.headers, { 'Content-Type': 'application/x-ndjson' })
|
||||
const [action, payload] = params.body.split('\n')
|
||||
t.deepEqual(JSON.parse(action), { create: { _index: 'test', _id: count } })
|
||||
t.deepEqual(JSON.parse(payload), dataset[count++])
|
||||
return { body: { errors: false, items: [{}] } }
|
||||
}
|
||||
})
|
||||
|
||||
const client = new Client({
|
||||
node: 'http://localhost:9200',
|
||||
Connection: MockConnection
|
||||
})
|
||||
let id = 0
|
||||
const result = await client.helpers.bulk({
|
||||
datasource: dataset.slice(),
|
||||
flushBytes: 1,
|
||||
concurrency: 1,
|
||||
onDocument (doc) {
|
||||
return {
|
||||
create: {
|
||||
_index: 'test',
|
||||
_id: id++
|
||||
}
|
||||
}
|
||||
},
|
||||
onDrop (doc) {
|
||||
t.fail('This should never be called')
|
||||
}
|
||||
})
|
||||
|
||||
t.type(result.time, 'number')
|
||||
t.type(result.bytes, 'number')
|
||||
t.match(result, {
|
||||
total: 3,
|
||||
successful: 3,
|
||||
retry: 0,
|
||||
failed: 0,
|
||||
aborted: false
|
||||
})
|
||||
})
|
||||
t.end()
|
||||
})
|
||||
|
||||
test('bulk update', t => {
|
||||
t.test('Should perform a bulk request', async t => {
|
||||
let count = 0
|
||||
const MockConnection = connection.buildMockConnection({
|
||||
onRequest (params) {
|
||||
t.strictEqual(params.path, '/_bulk')
|
||||
t.match(params.headers, { 'Content-Type': 'application/x-ndjson' })
|
||||
const [action, payload] = params.body.split('\n')
|
||||
t.deepEqual(JSON.parse(action), { update: { _index: 'test', _id: count } })
|
||||
t.deepEqual(JSON.parse(payload), { doc: dataset[count++], doc_as_upsert: true })
|
||||
return { body: { errors: false, items: [{}] } }
|
||||
}
|
||||
})
|
||||
|
||||
const client = new Client({
|
||||
node: 'http://localhost:9200',
|
||||
Connection: MockConnection
|
||||
})
|
||||
let id = 0
|
||||
const result = await client.helpers.bulk({
|
||||
datasource: dataset.slice(),
|
||||
flushBytes: 1,
|
||||
concurrency: 1,
|
||||
onDocument (doc) {
|
||||
return [{
|
||||
update: {
|
||||
_index: 'test',
|
||||
_id: id++
|
||||
}
|
||||
}, {
|
||||
doc_as_upsert: true
|
||||
}]
|
||||
},
|
||||
onDrop (doc) {
|
||||
t.fail('This should never be called')
|
||||
}
|
||||
})
|
||||
|
||||
t.type(result.time, 'number')
|
||||
t.type(result.bytes, 'number')
|
||||
t.match(result, {
|
||||
total: 3,
|
||||
successful: 3,
|
||||
retry: 0,
|
||||
failed: 0,
|
||||
aborted: false
|
||||
})
|
||||
})
|
||||
t.end()
|
||||
})
|
||||
|
||||
test('bulk delete', t => {
|
||||
t.test('Should perform a bulk request', async t => {
|
||||
let count = 0
|
||||
const MockConnection = connection.buildMockConnection({
|
||||
onRequest (params) {
|
||||
t.strictEqual(params.path, '/_bulk')
|
||||
t.match(params.headers, { 'Content-Type': 'application/x-ndjson' })
|
||||
t.deepEqual(JSON.parse(params.body), { delete: { _index: 'test', _id: count++ } })
|
||||
return { body: { errors: false, items: [{}] } }
|
||||
}
|
||||
})
|
||||
|
||||
const client = new Client({
|
||||
node: 'http://localhost:9200',
|
||||
Connection: MockConnection
|
||||
})
|
||||
let id = 0
|
||||
const result = await client.helpers.bulk({
|
||||
datasource: dataset.slice(),
|
||||
flushBytes: 1,
|
||||
concurrency: 1,
|
||||
onDocument (doc) {
|
||||
return {
|
||||
delete: {
|
||||
_index: 'test',
|
||||
_id: id++
|
||||
}
|
||||
}
|
||||
},
|
||||
onDrop (doc) {
|
||||
t.fail('This should never be called')
|
||||
}
|
||||
})
|
||||
|
||||
t.type(result.time, 'number')
|
||||
t.type(result.bytes, 'number')
|
||||
t.match(result, {
|
||||
total: 3,
|
||||
successful: 3,
|
||||
retry: 0,
|
||||
failed: 0,
|
||||
aborted: false
|
||||
})
|
||||
})
|
||||
|
||||
t.test('Should perform a bulk request (failure)', async t => {
|
||||
if (semver.lt(process.versions.node, '10.0.0')) {
|
||||
t.skip('This test will not pass on Node v8')
|
||||
return
|
||||
}
|
||||
async function handler (req, res) {
|
||||
t.strictEqual(req.url, '/_bulk')
|
||||
t.match(req.headers, { 'content-type': 'application/x-ndjson' })
|
||||
|
||||
let body = ''
|
||||
req.setEncoding('utf8')
|
||||
for await (const chunk of req) {
|
||||
body += chunk
|
||||
}
|
||||
|
||||
res.setHeader('content-type', 'application/json')
|
||||
|
||||
if (JSON.parse(body).delete._id === 1) {
|
||||
res.end(JSON.stringify({
|
||||
took: 0,
|
||||
errors: true,
|
||||
items: [{
|
||||
delete: {
|
||||
status: 400,
|
||||
error: { something: 'went wrong' }
|
||||
}
|
||||
}]
|
||||
}))
|
||||
} else {
|
||||
res.end(JSON.stringify({
|
||||
took: 0,
|
||||
errors: false,
|
||||
items: [{}]
|
||||
}))
|
||||
}
|
||||
}
|
||||
|
||||
const [{ port }, server] = await buildServer(handler)
|
||||
const client = new Client({ node: `http://localhost:${port}` })
|
||||
let id = 0
|
||||
const result = await client.helpers.bulk({
|
||||
datasource: dataset.slice(),
|
||||
flushBytes: 1,
|
||||
concurrency: 1,
|
||||
wait: 10,
|
||||
onDocument (doc) {
|
||||
return {
|
||||
delete: {
|
||||
_index: 'test',
|
||||
_id: id++
|
||||
}
|
||||
}
|
||||
},
|
||||
onDrop (doc) {
|
||||
t.deepEqual(doc, {
|
||||
status: 400,
|
||||
error: { something: 'went wrong' },
|
||||
operation: { delete: { _index: 'test', _id: 1 } },
|
||||
document: null,
|
||||
retried: false
|
||||
})
|
||||
}
|
||||
})
|
||||
|
||||
t.type(result.time, 'number')
|
||||
t.type(result.bytes, 'number')
|
||||
t.match(result, {
|
||||
total: 3,
|
||||
successful: 2,
|
||||
retry: 0,
|
||||
failed: 1,
|
||||
aborted: false
|
||||
})
|
||||
server.stop()
|
||||
})
|
||||
|
||||
t.end()
|
||||
})
|
||||
|
||||
test('errors', t => {
|
||||
t.test('datasource type', async t => {
|
||||
const client = new Client({
|
||||
node: 'http://localhost:9200'
|
||||
})
|
||||
try {
|
||||
await client.helpers.bulk({
|
||||
datasource: 'hello',
|
||||
onDocument (doc) {
|
||||
return {
|
||||
index: { _index: 'test' }
|
||||
}
|
||||
}
|
||||
})
|
||||
} catch (err) {
|
||||
t.true(err instanceof errors.ConfigurationError)
|
||||
t.is(err.message, 'bulk helper: the datasource must be an array or a buffer or a readable stream')
|
||||
}
|
||||
})
|
||||
|
||||
t.test('missing datasource', async t => {
|
||||
const client = new Client({
|
||||
node: 'http://localhost:9200'
|
||||
})
|
||||
try {
|
||||
await client.helpers.bulk({
|
||||
onDocument (doc) {
|
||||
return {
|
||||
index: { _index: 'test' }
|
||||
}
|
||||
}
|
||||
})
|
||||
} catch (err) {
|
||||
t.true(err instanceof errors.ConfigurationError)
|
||||
t.is(err.message, 'bulk helper: the datasource is required')
|
||||
}
|
||||
})
|
||||
|
||||
t.test('missing onDocument', async t => {
|
||||
const client = new Client({
|
||||
node: 'http://localhost:9200'
|
||||
})
|
||||
try {
|
||||
await client.helpers.bulk({
|
||||
datasource: dataset.slice()
|
||||
})
|
||||
} catch (err) {
|
||||
t.true(err instanceof errors.ConfigurationError)
|
||||
t.is(err.message, 'bulk helper: the onDocument callback is required')
|
||||
}
|
||||
})
|
||||
|
||||
t.end()
|
||||
})
|
||||
220
test/unit/helpers/scroll.test.js
Normal file
220
test/unit/helpers/scroll.test.js
Normal file
@ -0,0 +1,220 @@
|
||||
// Licensed to Elasticsearch B.V under one or more agreements.
|
||||
// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
|
||||
// See the LICENSE file in the project root for more information
|
||||
|
||||
'use strict'
|
||||
|
||||
const { test } = require('tap')
|
||||
const { Client, errors } = require('../../../')
|
||||
const { connection } = require('../../utils')
|
||||
|
||||
test('Scroll search', async t => {
|
||||
var count = 0
|
||||
const MockConnection = connection.buildMockConnection({
|
||||
onRequest (params) {
|
||||
t.strictEqual(params.querystring, 'scroll=1m')
|
||||
return {
|
||||
body: {
|
||||
_scroll_id: count === 3 ? undefined : 'id',
|
||||
count,
|
||||
hits: {
|
||||
hits: [
|
||||
{ _source: { one: 'one' } },
|
||||
{ _source: { two: 'two' } },
|
||||
{ _source: { three: 'three' } }
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
const client = new Client({
|
||||
node: 'http://localhost:9200',
|
||||
Connection: MockConnection
|
||||
})
|
||||
|
||||
const scrollSearch = client.helpers.scrollSearch({
|
||||
index: 'test',
|
||||
body: { foo: 'bar' }
|
||||
})
|
||||
|
||||
for await (const result of scrollSearch) {
|
||||
t.strictEqual(result.body.count, count)
|
||||
if (count < 3) {
|
||||
t.strictEqual(result.body._scroll_id, 'id')
|
||||
} else {
|
||||
t.strictEqual(result.body._scroll_id, undefined)
|
||||
}
|
||||
count += 1
|
||||
}
|
||||
})
|
||||
|
||||
test('Clear a scroll search', async t => {
|
||||
var count = 0
|
||||
const MockConnection = connection.buildMockConnection({
|
||||
onRequest (params) {
|
||||
if (params.method === 'DELETE') {
|
||||
const body = JSON.parse(params.body)
|
||||
t.strictEqual(body.scroll_id, 'id')
|
||||
}
|
||||
return {
|
||||
body: {
|
||||
_scroll_id: count === 3 ? undefined : 'id',
|
||||
count,
|
||||
hits: {
|
||||
hits: [
|
||||
{ _source: { one: 'one' } },
|
||||
{ _source: { two: 'two' } },
|
||||
{ _source: { three: 'three' } }
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
const client = new Client({
|
||||
node: 'http://localhost:9200',
|
||||
Connection: MockConnection
|
||||
})
|
||||
|
||||
const scrollSearch = client.helpers.scrollSearch({
|
||||
index: 'test',
|
||||
body: { foo: 'bar' }
|
||||
})
|
||||
|
||||
for await (const result of scrollSearch) {
|
||||
if (count === 2) {
|
||||
t.fail('The scroll search should be cleared')
|
||||
}
|
||||
t.strictEqual(result.body.count, count)
|
||||
if (count === 1) {
|
||||
await result.clear()
|
||||
}
|
||||
count += 1
|
||||
}
|
||||
})
|
||||
|
||||
test('Scroll search (retry)', async t => {
|
||||
var count = 0
|
||||
const MockConnection = connection.buildMockConnection({
|
||||
onRequest (params) {
|
||||
if (count === 1) {
|
||||
count += 1
|
||||
return { body: {}, statusCode: 429 }
|
||||
}
|
||||
return {
|
||||
statusCode: 200,
|
||||
body: {
|
||||
_scroll_id: count === 4 ? undefined : 'id',
|
||||
count,
|
||||
hits: {
|
||||
hits: [
|
||||
{ _source: { one: 'one' } },
|
||||
{ _source: { two: 'two' } },
|
||||
{ _source: { three: 'three' } }
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
const client = new Client({
|
||||
node: 'http://localhost:9200',
|
||||
Connection: MockConnection
|
||||
})
|
||||
|
||||
const scrollSearch = client.helpers.scrollSearch({
|
||||
index: 'test',
|
||||
body: { foo: 'bar' }
|
||||
}, {
|
||||
wait: 10
|
||||
})
|
||||
|
||||
for await (const result of scrollSearch) {
|
||||
t.strictEqual(result.body.count, count)
|
||||
t.notStrictEqual(result.body.count, 1)
|
||||
if (count < 4) {
|
||||
t.strictEqual(result.body._scroll_id, 'id')
|
||||
} else {
|
||||
t.strictEqual(result.body._scroll_id, undefined)
|
||||
}
|
||||
count += 1
|
||||
}
|
||||
})
|
||||
|
||||
test('Scroll search (retry throws and maxRetries)', async t => {
|
||||
var count = 0
|
||||
const MockConnection = connection.buildMockConnection({
|
||||
onRequest (params) {
|
||||
count += 1
|
||||
return { body: {}, statusCode: 429 }
|
||||
}
|
||||
})
|
||||
|
||||
const client = new Client({
|
||||
node: 'http://localhost:9200',
|
||||
Connection: MockConnection,
|
||||
maxRetries: 5
|
||||
})
|
||||
|
||||
const scrollSearch = client.helpers.scrollSearch({
|
||||
index: 'test',
|
||||
body: { foo: 'bar' }
|
||||
}, {
|
||||
wait: 10
|
||||
})
|
||||
|
||||
try {
|
||||
for await (const result of scrollSearch) { // eslint-disable-line
|
||||
t.fail('we should not be here')
|
||||
}
|
||||
} catch (err) {
|
||||
t.true(err instanceof errors.ResponseError)
|
||||
t.strictEqual(err.statusCode, 429)
|
||||
t.strictEqual(count, 5)
|
||||
}
|
||||
})
|
||||
|
||||
test('Scroll search documents', async t => {
|
||||
var count = 0
|
||||
const MockConnection = connection.buildMockConnection({
|
||||
onRequest (params) {
|
||||
return {
|
||||
body: {
|
||||
_scroll_id: count === 3 ? undefined : 'id',
|
||||
count,
|
||||
hits: {
|
||||
hits: [
|
||||
{ _source: { val: 1 * count } },
|
||||
{ _source: { val: 2 * count } },
|
||||
{ _source: { val: 3 * count } }
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
const client = new Client({
|
||||
node: 'http://localhost:9200',
|
||||
Connection: MockConnection
|
||||
})
|
||||
|
||||
const scrollSearch = client.helpers.scrollDocuments({
|
||||
index: 'test',
|
||||
body: { foo: 'bar' }
|
||||
})
|
||||
|
||||
let n = 1
|
||||
for await (const hit of scrollSearch) {
|
||||
t.deepEqual(hit, { val: n * count })
|
||||
n += 1
|
||||
if (n === 4) {
|
||||
count += 1
|
||||
n = 1
|
||||
}
|
||||
}
|
||||
})
|
||||
42
test/unit/helpers/search.test.js
Normal file
42
test/unit/helpers/search.test.js
Normal file
@ -0,0 +1,42 @@
|
||||
// Licensed to Elasticsearch B.V under one or more agreements.
|
||||
// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
|
||||
// See the LICENSE file in the project root for more information
|
||||
|
||||
'use strict'
|
||||
|
||||
const { test } = require('tap')
|
||||
const { Client } = require('../../../')
|
||||
const { connection } = require('../../utils')
|
||||
|
||||
test('Search should have an additional documents property', async t => {
|
||||
const MockConnection = connection.buildMockConnection({
|
||||
onRequest (params) {
|
||||
return {
|
||||
body: {
|
||||
hits: {
|
||||
hits: [
|
||||
{ _source: { one: 'one' } },
|
||||
{ _source: { two: 'two' } },
|
||||
{ _source: { three: 'three' } }
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
const client = new Client({
|
||||
node: 'http://localhost:9200',
|
||||
Connection: MockConnection
|
||||
})
|
||||
|
||||
const result = await client.helpers.search({
|
||||
index: 'test',
|
||||
body: { foo: 'bar' }
|
||||
})
|
||||
t.deepEqual(result, [
|
||||
{ one: 'one' },
|
||||
{ two: 'two' },
|
||||
{ three: 'three' }
|
||||
])
|
||||
})
|
||||
Reference in New Issue
Block a user