Added async generator support in bulk helper (#1138)

* Added async generator support in bulk helper

* Updated test

* Updated docs

* Improved type definitions

* Updated onDrop callback type definition
This commit is contained in:
Tomas Della Vedova
2020-04-03 09:46:26 +02:00
committed by GitHub
parent bdd38597d8
commit df17fb99d0
5 changed files with 115 additions and 17 deletions

24
lib/Helpers.d.ts vendored
View File

@ -10,7 +10,7 @@ export default class Helpers {
search<TRequestBody extends RequestBody, TDocument = unknown>(params: Search<TRequestBody>, options?: TransportRequestOptions): Promise<TDocument[]>
scrollSearch<TRequestBody extends RequestBody, TDocument = unknown, TResponse = ResponseBody, TContext = unknown>(params: Search<TRequestBody>, options?: TransportRequestOptions): AsyncIterable<ScrollSearchResponse<TDocument, TResponse, TContext>>
scrollDocuments<TRequestBody extends RequestBody, TDocument = unknown>(params: Search<TRequestBody>, options?: TransportRequestOptions): AsyncIterable<TDocument>
bulk(options: BulkHelperOptions): BulkHelper<BulkStats>
bulk<TDocument = unknown>(options: BulkHelperOptions<TDocument>): BulkHelper<BulkStats>
}
export interface ScrollSearchResponse<TDocument = unknown, TResponse = ResponseBody, TContext = unknown> extends ApiResponse<TResponse, TContext> {
@ -64,13 +64,27 @@ type UpdateAction = [UpdateActionOperation, Record<string, any>]
type Action = IndexAction | CreateAction | UpdateAction | DeleteAction
type Omit<T, K extends keyof T> = Pick<T, Exclude<keyof T, K>>
export interface BulkHelperOptions extends Omit<Bulk, 'body'> {
datasource: any[] | Buffer | ReadableStream
onDocument: (doc: Record<string, any>) => Action
export interface BulkHelperOptions<TDocument = unknown> extends Omit<Bulk, 'body'> {
datasource: TDocument[] | Buffer | ReadableStream | AsyncIterator<TDocument>
onDocument: (doc: TDocument) => Action
flushBytes?: number
concurrency?: number
retries?: number
wait?: number,
onDrop?: (doc: Record<string, any>) => void,
onDrop?: (doc: OnDropDocument<TDocument>) => void,
refreshOnCompletion?: boolean | string
}
export interface OnDropDocument<TDocument = unknown> {
status: number
error: {
type: string,
reason: string,
caused_by: {
type: string,
reason: string
}
}
document: TDocument
retried: boolean
}