358 lines
8.3 KiB
JavaScript
358 lines
8.3 KiB
JavaScript
// Licensed to Elasticsearch B.V under one or more agreements.
|
|
// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
|
|
// See the LICENSE file in the project root for more information
|
|
|
|
'use strict'
|
|
|
|
const { test } = require('tap')
|
|
const { Client, errors } = require('../../../')
|
|
const { connection } = require('../../utils')
|
|
|
|
test('Scroll search', async t => {
|
|
var count = 0
|
|
const MockConnection = connection.buildMockConnection({
|
|
onRequest (params) {
|
|
t.strictEqual(params.querystring, 'scroll=1m')
|
|
return {
|
|
body: {
|
|
_scroll_id: count === 3 ? undefined : 'id',
|
|
count,
|
|
hits: {
|
|
hits: [
|
|
{ _source: { one: 'one' } },
|
|
{ _source: { two: 'two' } },
|
|
{ _source: { three: 'three' } }
|
|
]
|
|
}
|
|
}
|
|
}
|
|
}
|
|
})
|
|
|
|
const client = new Client({
|
|
node: 'http://localhost:9200',
|
|
Connection: MockConnection
|
|
})
|
|
|
|
const scrollSearch = client.helpers.scrollSearch({
|
|
index: 'test',
|
|
body: { foo: 'bar' }
|
|
})
|
|
|
|
for await (const result of scrollSearch) {
|
|
t.strictEqual(result.body.count, count)
|
|
if (count < 3) {
|
|
t.strictEqual(result.body._scroll_id, 'id')
|
|
} else {
|
|
t.strictEqual(result.body._scroll_id, undefined)
|
|
}
|
|
count += 1
|
|
}
|
|
})
|
|
|
|
test('Clear a scroll search', async t => {
|
|
var count = 0
|
|
const MockConnection = connection.buildMockConnection({
|
|
onRequest (params) {
|
|
if (params.method === 'DELETE') {
|
|
const body = JSON.parse(params.body)
|
|
t.strictEqual(body.scroll_id, 'id')
|
|
}
|
|
return {
|
|
body: {
|
|
_scroll_id: count === 3 ? undefined : 'id',
|
|
count,
|
|
hits: {
|
|
hits: [
|
|
{ _source: { one: 'one' } },
|
|
{ _source: { two: 'two' } },
|
|
{ _source: { three: 'three' } }
|
|
]
|
|
}
|
|
}
|
|
}
|
|
}
|
|
})
|
|
|
|
const client = new Client({
|
|
node: 'http://localhost:9200',
|
|
Connection: MockConnection
|
|
})
|
|
|
|
const scrollSearch = client.helpers.scrollSearch({
|
|
index: 'test',
|
|
body: { foo: 'bar' }
|
|
})
|
|
|
|
for await (const result of scrollSearch) {
|
|
if (count === 2) {
|
|
t.fail('The scroll search should be cleared')
|
|
}
|
|
t.strictEqual(result.body.count, count)
|
|
if (count === 1) {
|
|
await result.clear()
|
|
}
|
|
count += 1
|
|
}
|
|
})
|
|
|
|
test('Scroll search (retry)', async t => {
|
|
var count = 0
|
|
const MockConnection = connection.buildMockConnection({
|
|
onRequest (params) {
|
|
if (count === 1) {
|
|
count += 1
|
|
return { body: {}, statusCode: 429 }
|
|
}
|
|
return {
|
|
statusCode: 200,
|
|
body: {
|
|
_scroll_id: count === 4 ? undefined : 'id',
|
|
count,
|
|
hits: {
|
|
hits: [
|
|
{ _source: { one: 'one' } },
|
|
{ _source: { two: 'two' } },
|
|
{ _source: { three: 'three' } }
|
|
]
|
|
}
|
|
}
|
|
}
|
|
}
|
|
})
|
|
|
|
const client = new Client({
|
|
node: 'http://localhost:9200',
|
|
Connection: MockConnection
|
|
})
|
|
|
|
const scrollSearch = client.helpers.scrollSearch({
|
|
index: 'test',
|
|
body: { foo: 'bar' }
|
|
}, {
|
|
wait: 10
|
|
})
|
|
|
|
for await (const result of scrollSearch) {
|
|
t.strictEqual(result.body.count, count)
|
|
t.notStrictEqual(result.body.count, 1)
|
|
if (count < 4) {
|
|
t.strictEqual(result.body._scroll_id, 'id')
|
|
} else {
|
|
t.strictEqual(result.body._scroll_id, undefined)
|
|
}
|
|
count += 1
|
|
}
|
|
})
|
|
|
|
test('Scroll search (retry throws and maxRetries)', async t => {
|
|
const maxRetries = 5
|
|
const expectedAttempts = maxRetries + 1
|
|
var count = 0
|
|
const MockConnection = connection.buildMockConnection({
|
|
onRequest (params) {
|
|
count += 1
|
|
return { body: {}, statusCode: 429 }
|
|
}
|
|
})
|
|
|
|
const client = new Client({
|
|
node: 'http://localhost:9200',
|
|
Connection: MockConnection,
|
|
maxRetries
|
|
})
|
|
|
|
const scrollSearch = client.helpers.scrollSearch({
|
|
index: 'test',
|
|
body: { foo: 'bar' }
|
|
}, {
|
|
wait: 10,
|
|
ignore: [404]
|
|
})
|
|
|
|
try {
|
|
for await (const result of scrollSearch) { // eslint-disable-line
|
|
t.fail('we should not be here')
|
|
}
|
|
} catch (err) {
|
|
t.true(err instanceof errors.ResponseError)
|
|
t.strictEqual(err.statusCode, 429)
|
|
t.strictEqual(count, expectedAttempts)
|
|
}
|
|
})
|
|
|
|
test('Scroll search (retry throws later)', async t => {
|
|
const maxRetries = 5
|
|
const expectedAttempts = maxRetries + 1
|
|
var count = 0
|
|
const MockConnection = connection.buildMockConnection({
|
|
onRequest (params) {
|
|
// filter_path should not be added if is not already present
|
|
t.strictEqual(params.querystring, 'scroll=1m')
|
|
if (count > 1) {
|
|
count += 1
|
|
return { body: {}, statusCode: 429 }
|
|
}
|
|
return {
|
|
statusCode: 200,
|
|
body: {
|
|
_scroll_id: count === 4 ? undefined : 'id',
|
|
count,
|
|
hits: {
|
|
hits: [
|
|
{ _source: { one: 'one' } },
|
|
{ _source: { two: 'two' } },
|
|
{ _source: { three: 'three' } }
|
|
]
|
|
}
|
|
}
|
|
}
|
|
}
|
|
})
|
|
|
|
const client = new Client({
|
|
node: 'http://localhost:9200',
|
|
Connection: MockConnection
|
|
})
|
|
|
|
const scrollSearch = client.helpers.scrollSearch({
|
|
index: 'test',
|
|
body: { foo: 'bar' }
|
|
}, {
|
|
wait: 10
|
|
})
|
|
|
|
try {
|
|
for await (const result of scrollSearch) { // eslint-disable-line
|
|
t.strictEqual(result.body.count, count)
|
|
count += 1
|
|
}
|
|
} catch (err) {
|
|
t.true(err instanceof errors.ResponseError)
|
|
t.strictEqual(err.statusCode, 429)
|
|
t.strictEqual(count, expectedAttempts)
|
|
}
|
|
})
|
|
|
|
test('Scroll search documents', async t => {
|
|
var count = 0
|
|
const MockConnection = connection.buildMockConnection({
|
|
onRequest (params) {
|
|
if (count === 0) {
|
|
t.strictEqual(params.querystring, 'filter_path=hits.hits._source%2C_scroll_id&scroll=1m')
|
|
} else {
|
|
t.strictEqual(params.querystring, 'scroll=1m')
|
|
t.strictEqual(params.body, '{"scroll_id":"id"}')
|
|
}
|
|
return {
|
|
body: {
|
|
_scroll_id: count === 3 ? undefined : 'id',
|
|
count,
|
|
hits: {
|
|
hits: [
|
|
{ _source: { val: 1 * count } },
|
|
{ _source: { val: 2 * count } },
|
|
{ _source: { val: 3 * count } }
|
|
]
|
|
}
|
|
}
|
|
}
|
|
}
|
|
})
|
|
|
|
const client = new Client({
|
|
node: 'http://localhost:9200',
|
|
Connection: MockConnection
|
|
})
|
|
|
|
const scrollSearch = client.helpers.scrollDocuments({
|
|
index: 'test',
|
|
body: { foo: 'bar' }
|
|
})
|
|
|
|
let n = 1
|
|
for await (const hit of scrollSearch) {
|
|
t.deepEqual(hit, { val: n * count })
|
|
n += 1
|
|
if (n === 4) {
|
|
count += 1
|
|
n = 1
|
|
}
|
|
}
|
|
})
|
|
|
|
test('Should not retry if maxRetries = 0', async t => {
|
|
const maxRetries = 0
|
|
const expectedAttempts = 1
|
|
var count = 0
|
|
const MockConnection = connection.buildMockConnection({
|
|
onRequest (params) {
|
|
count += 1
|
|
return { body: {}, statusCode: 429 }
|
|
}
|
|
})
|
|
|
|
const client = new Client({
|
|
node: 'http://localhost:9200',
|
|
Connection: MockConnection,
|
|
maxRetries
|
|
})
|
|
|
|
const scrollSearch = client.helpers.scrollSearch({
|
|
index: 'test',
|
|
body: { foo: 'bar' }
|
|
}, {
|
|
wait: 10,
|
|
ignore: [404]
|
|
})
|
|
|
|
try {
|
|
for await (const result of scrollSearch) { // eslint-disable-line
|
|
t.fail('we should not be here')
|
|
}
|
|
} catch (err) {
|
|
t.true(err instanceof errors.ResponseError)
|
|
t.strictEqual(err.statusCode, 429)
|
|
t.strictEqual(count, expectedAttempts)
|
|
}
|
|
})
|
|
|
|
test('Fix querystring for scroll search', async t => {
|
|
var count = 0
|
|
const MockConnection = connection.buildMockConnection({
|
|
onRequest (params) {
|
|
if (count === 0) {
|
|
t.strictEqual(params.querystring, 'size=1&scroll=1m')
|
|
} else {
|
|
t.strictEqual(params.querystring, 'scroll=1m')
|
|
}
|
|
return {
|
|
body: {
|
|
_scroll_id: count === 3 ? undefined : 'id',
|
|
hits: {
|
|
hits: [
|
|
{ _source: { val: count } }
|
|
]
|
|
}
|
|
}
|
|
}
|
|
}
|
|
})
|
|
|
|
const client = new Client({
|
|
node: 'http://localhost:9200',
|
|
Connection: MockConnection
|
|
})
|
|
|
|
const scrollSearch = client.helpers.scrollSearch({
|
|
index: 'test',
|
|
size: 1,
|
|
body: { foo: 'bar' }
|
|
})
|
|
|
|
for await (const response of scrollSearch) {
|
|
t.strictEqual(response.body.hits.hits.length, 1)
|
|
count += 1
|
|
}
|
|
})
|