Bumped v6.8.4

X-Opaque-Id support (#997 )
* Added X-Opaque-Id support * Updated type definitions * Updated test * Updated docs
2019-12-03 14:38:58 +01:00 · 2019-12-03 14:33:33 +01:00 · 2019-10-28 15:41:42 +01:00 · 2019-10-28 14:46:40 +01:00 · 2019-10-28 14:46:32 +01:00 · 2019-10-28 14:46:03 +01:00
20 changed files with 498 additions and 355 deletions
--- a/.ci/Jenkinsfile
+++ b/.ci/Jenkinsfile
@ -1,224 +0,0 @@
-#!/usr/bin/env groovy
-
-@Library('apm@current') _
-
-def NODE_JS_VERSIONS = [8,10,12]
-def nodeJsVersion = NODE_JS_VERSIONS[randomNumber(min: 0, max:2)]
-
-pipeline {
-  agent {
-    label 'docker && immutable'
-  }
-
-  environment {
-    REPO = 'elasticsearch-js'
-    BASE_DIR = "src/github.com/elastic/${env.REPO}"
-    NODE_JS_DEFAULT_VERSION = "${nodeJsVersion}"
-    NODE_JS_VERSIONS = "${NODE_JS_VERSIONS.join(',')}"
-    HOME = "${env.WORKSPACE}"
-    npm_config_cache = 'npm-cache'
-  }
-
-  options {
-    timeout(time: 1, unit: 'HOURS')
-    buildDiscarder(logRotator(numToKeepStr: '20', artifactNumToKeepStr: '20', daysToKeepStr: '30'))
-    timestamps()
-    ansiColor('xterm')
-    disableResume()
-    durabilityHint('PERFORMANCE_OPTIMIZED')
-  }
-
-  triggers {
-    issueCommentTrigger('(?i).*(?:jenkins\\W+)?run\\W+(?:the\\W+)?tests(?:\\W+please)?.*')
-    // env.CHANGE_ID as a value in case of a commit or a pr, which means
-    // that we will have a daily cron job only for branches that don't have an active pr
-    cron(env.CHANGE_ID ? '' : '@daily')
-  }
-
-  stages {
-    stage('Checkout') {
-      options { skipDefaultCheckout() }
-      steps {
-        deleteDir()
-        gitCheckout(basedir: "${BASE_DIR}", githubNotifyFirstTimeContributor: false)
-        stash allowEmpty: true, name: 'source', useDefaultExcludes: false
-      }
-    }
-
-    stage('Install dependencies') {
-      options { skipDefaultCheckout() }
-      steps {
-        deleteDir()
-        unstash 'source'
-        script {
-          buildDockerImage(image: "node:${env.NODE_JS_DEFAULT_VERSION}-alpine").inside(){
-            dir("${BASE_DIR}"){
-              sh(label: 'System info', script: 'node --version; npm --version')
-              sh(label: 'Install dependencies', script: 'npm install')
-            }
-          }
-        }
-        stash allowEmpty: true, name: 'source-dependencies', useDefaultExcludes: false
-      }
-    }
-
-    stage('License check') {
-      options { skipDefaultCheckout() }
-      steps {
-        withGithubNotify(context: 'License check') {
-          deleteDir()
-          unstash 'source-dependencies'
-          script {
-            buildDockerImage(image: "node:${env.NODE_JS_DEFAULT_VERSION}-alpine").inside(){
-              dir("${BASE_DIR}"){
-                sh(label: 'Check production dependencies licenses', script: 'npm run license-checker')
-              }
-            }
-          }
-        }
-      }
-    }
-
-    stage('Linter') {
-      options { skipDefaultCheckout() }
-      steps {
-        withGithubNotify(context: 'Linter') {
-          deleteDir()
-          unstash 'source-dependencies'
-          script {
-            buildDockerImage(image: "node:${env.NODE_JS_DEFAULT_VERSION}-alpine").inside(){
-              dir("${BASE_DIR}"){
-                sh(label: 'Lint code with standardjs', script: 'npm run lint')
-              }
-            }
-          }
-        }
-      }
-    }
-
-    stage('Unit test') {
-      failFast true
-      options { skipDefaultCheckout() }
-      steps {
-        withGithubNotify(context: 'Unit test') {
-          script {
-            def versions = env.NODE_JS_VERSIONS.split(',')
-            def parallelTasks = [:]
-            versions.each{ version ->
-              parallelTasks["Node.js v${version}"] = buildUnitTest(version: version)
-            }
-            parallel(parallelTasks)
-          }
-        }
-      }
-    }
-
-    stage('Integration test') {
-      failFast true
-      options { skipDefaultCheckout() }
-      parallel {
-        stage('OSS') {
-          agent { label 'docker && immutable' }
-          options { skipDefaultCheckout() }
-          environment {
-            TEST_ES_SERVER = 'http://elasticsearch:9200'
-          }
-          steps {
-            withGithubNotify(context: 'Integration test OSS') {
-              deleteDir()
-              unstash 'source-dependencies'
-              dir("${BASE_DIR}"){
-                // Sometimes the docker registry fails and has random timeouts
-                // this block will retry a doker image 3 times before to fail.
-                retry(3) {
-                  sleep randomNumber(min: 5, max: 10)
-                  sh(label: 'Start Elasticsearch', script: './scripts/es-docker.sh --detach')
-                }
-              }
-              script {
-                buildDockerImage(fromDockerfile: true).inside('--network=elastic'){
-                  dir("${BASE_DIR}"){
-                    sh(label: 'Integration test', script: 'npm run test:integration | tee test-integration.tap')
-                    sh(label: 'Generating test reporting', script: './node_modules/.bin/tap-mocha-reporter xunit < test-integration.tap > junit-integration.xml')
-                  }
-                }
-              }
-              sh(label: 'Stop Elasticsearch', script: 'docker kill $(docker ps -q)')
-              junit(allowEmptyResults: true, keepLongStdio: true, testResults: "${BASE_DIR}/**/junit-*.xml")
-            }
-          }
-        }
-
-        stage('xPack') {
-          agent { label 'docker && immutable' }
-          options { skipDefaultCheckout() }
-          environment {
-            TEST_ES_SERVER = 'https://elastic:changeme@elasticsearch:9200'
-          }
-          steps {
-            withGithubNotify(context: 'Integration test xPack') {
-              deleteDir()
-              unstash 'source-dependencies'
-              dir("${BASE_DIR}"){
-                // Sometimes the docker registry fails and has random timeouts
-                // this block will retry a doker image 3 times before to fail.
-                retry(3) {
-                  sleep randomNumber(min: 5, max: 10)
-                  sh(label: 'Start Elasticsearch', script: './scripts/es-docker-platinum.sh --detach')
-                }
-              }
-              script {
-                buildDockerImage(fromDockerfile: true).inside('--network=elastic'){
-                  dir("${BASE_DIR}"){
-                    sh(label: 'Integration test', script: 'npm run test:integration | tee test-integration.tap')
-                    sh(label: 'Generating test reporting', script: './node_modules/.bin/tap-mocha-reporter xunit < test-integration.tap > junit-integration.xml')
-                  }
-                }
-              }
-              sh(label: 'Stop Elasticsearch', script: 'docker kill $(docker ps -q)')
-              junit(allowEmptyResults: true, keepLongStdio: true, testResults: "${BASE_DIR}/**/junit-*.xml")
-            }
-          }
-        }
-      }
-    }
-  }
-}
-
-// Sometimes the docker registry fails and has random timeouts
-// this function will retry a doker image 3 times before to fail.
-def buildDockerImage(args) {
-  def image
-  retry(3) {
-    sleep randomNumber(min: 5, max: 10)
-    if (args.fromDockerfile == true) {
-      image = docker.build('nodejs-image', "--build-arg NODE_JS_VERSION=${env.NODE_JS_DEFAULT_VERSION} ${BASE_DIR}/.ci/docker")
-    } else {
-      image = docker.image(args.image)
-      // make sure we have the latest available from Docker Hub
-      image.pull()
-    }
-  }
-  return image
-}
-
-def buildUnitTest(args) {
-  return {
-    node('docker && immutable') {
-      deleteDir()
-      unstash 'source'
-      script {
-        buildDockerImage(image: "node:${args.version}-alpine").inside(){
-          dir("${BASE_DIR}"){
-            sh(label: 'Install dependencies', script: 'npm install')
-            sh(label: 'Run unit test', script: 'npm run test:unit | tee test-unit.tap')
-            sh(label: 'Run behavior test', script: 'npm run test:behavior | tee test-behavior.tap')
-            sh(label: 'Run types test', script: 'npm run test:types')
-            sh(label: 'Generating test reporting', script: './node_modules/.bin/tap-mocha-reporter xunit < test-unit.tap > junit-unit.xml; ./node_modules/.bin/tap-mocha-reporter xunit < test-behavior.tap > junit-behavior.xml')
-          }
-        }
-      }
-      junit(allowEmptyResults: true, keepLongStdio: true, testResults: "${BASE_DIR}/**/junit-*.xml")
-    }
-  }
-}
--- a/.ci/jobs/defaults.yml
+++ b/.ci/jobs/defaults.yml
@ -0,0 +1,74 @@
+---
+
+##### GLOBAL METADATA
+
+- meta:
+    cluster: clients-ci
+
+##### JOB DEFAULTS
+
+- job:
+    project-type: matrix
+    logrotate:
+      daysToKeep: 30
+      numToKeep: 100
+    properties:
+    - github:
+        url: https://github.com/elastic/elasticsearch-js/
+    - inject:
+        properties-content: HOME=$JENKINS_HOME
+    concurrent: true
+    node: flyweight
+    scm:
+    - git:
+        name: origin
+        credentials-id: f6c7695a-671e-4f4f-a331-acdce44ff9ba
+        reference-repo: /var/lib/jenkins/.git-references/elasticsearch-js.git
+        branches:
+        - ${branch_specifier}
+        url: https://github.com/elastic/elasticsearch-js.git
+        wipe-workspace: 'True'
+    triggers:
+    - github
+    vault:
+      # vault read auth/approle/role/clients-ci/role-id
+      role_id: ddbd0d44-0e51-105b-177a-c8fdfd445126
+    axes:
+    - axis:
+        type: slave
+        name: label
+        values:
+        - linux
+    - axis:
+        type: yaml
+        filename: .ci/test-matrix.yml
+        name: ELASTICSEARCH_VERSION
+    - axis:
+        type: yaml
+        filename: .ci/test-matrix.yml
+        name: NODE_JS_VERSION
+    - axis:
+        type: yaml
+        filename: .ci/test-matrix.yml
+        name: TEST_SUITE
+    yaml-strategy:
+      exclude-key: exclude
+      filename: .ci/test-matrix.yml
+    wrappers:
+    - ansicolor
+    - timeout:
+        type: absolute
+        timeout: 120
+        fail: true
+    - timestamps
+    - workspace-cleanup
+    builders:
+    - shell: |-
+        #!/usr/local/bin/runbld
+        .ci/run-tests
+    publishers:
+    - email:
+        recipients: infra-root+build@elastic.co
+    # - junit:
+    #     results: "*-junit.xml"
+    #     allow-empty-results: true
--- a/.ci/jobs/elastic+elasticsearch-js+5.x.yml
+++ b/.ci/jobs/elastic+elasticsearch-js+5.x.yml
@ -0,0 +1,14 @@
+---
+- job:
+    name: elastic+elasticsearch-js+5.x
+    display-name: 'elastic / elasticsearch-js # 5.x'
+    description: Testing the elasticsearch-js 5.x branch.
+    parameters:
+    - string:
+        name: branch_specifier
+        default: refs/heads/5.x
+        description: the Git branch specifier to build (&lt;branchName&gt;, &lt;tagName&gt;,
+          &lt;commitId&gt;, etc.)
+    triggers:
+    - github
+    - timed: '@weekly'
--- a/.ci/jobs/elastic+elasticsearch-js+6.x.yml
+++ b/.ci/jobs/elastic+elasticsearch-js+6.x.yml
@ -0,0 +1,14 @@
+---
+- job:
+    name: elastic+elasticsearch-js+6.x
+    display-name: 'elastic / elasticsearch-js # 6.x'
+    description: Testing the elasticsearch-js 6.x branch.
+    parameters:
+    - string:
+        name: branch_specifier
+        default: refs/heads/6.x
+        description: the Git branch specifier to build (&lt;branchName&gt;, &lt;tagName&gt;,
+          &lt;commitId&gt;, etc.)
+    triggers:
+    - github
+    - timed: '@weekly'
--- a/.ci/jobs/elastic+elasticsearch-js+7.x.yml
+++ b/.ci/jobs/elastic+elasticsearch-js+7.x.yml
@ -0,0 +1,14 @@
+---
+- job:
+    name: elastic+elasticsearch-js+7.x
+    display-name: 'elastic / elasticsearch-js # 7.x'
+    description: Testing the elasticsearch-js 7.x branch.
+    parameters:
+    - string:
+        name: branch_specifier
+        default: refs/heads/7.x
+        description: the Git branch specifier to build (&lt;branchName&gt;, &lt;tagName&gt;,
+          &lt;commitId&gt;, etc.)
+    triggers:
+    - github
+    - timed: '@weekly'
--- a/.ci/jobs/elastic+elasticsearch-js+master.yml
+++ b/.ci/jobs/elastic+elasticsearch-js+master.yml
@ -0,0 +1,14 @@
+---
+- job:
+    name: elastic+elasticsearch-js+master
+    display-name: 'elastic / elasticsearch-js # master'
+    description: Testing the elasticsearch-js master branch.
+    parameters:
+    - string:
+        name: branch_specifier
+        default: refs/heads/master
+        description: the Git branch specifier to build (&lt;branchName&gt;, &lt;tagName&gt;,
+          &lt;commitId&gt;, etc.)
+    triggers:
+    - github
+    - timed: '@daily'
--- a/.ci/jobs/elastic+elasticsearch-js+mbp.yml
+++ b/.ci/jobs/elastic+elasticsearch-js+mbp.yml
@ -1,55 +0,0 @@
---
- job:
-    name: elastic+elasticsearch-js+mbp
-    display-name: 'elastic / elasticsearch-js'
-    description: Testing elasticsearch-js.
-    project-type: multibranch
-    properties: []
-    triggers: []
-    logrotate:
-      daysToKeep: 30
-    number-to-keep: '5'
-    days-to-keep: '1'
-    concurrent: true
-    node: linux
-    script-path: .ci/Jenkinsfile
-    scm:
-    - github:
-        branch-discovery: no-pr
-        discover-pr-forks-strategy: merge-current
-        discover-pr-forks-trust: permission
-        discover-pr-origin: merge-current
-        discover-tags: false
-        repo: elasticsearch-js
-        repo-owner: elastic
-        credentials-id: 2a9602aa-ab9f-4e52-baf3-b71ca88469c7-UserAndToken
-        ssh-checkout:
-          credentials: f6c7695a-671e-4f4f-a331-acdce44ff9ba
-        build-strategies:
-        - tags:
-            ignore-tags-older-than: -1
-            ignore-tags-newer-than: -1
-        - regular-branches: true
-        - change-request:
-            ignore-target-only-changes: false
-        clean:
-          after: true
-          before: true
-        prune: true
-        shallow-clone: true
-        depth: 3
-        do-not-fetch-tags: true
-        submodule:
-          disable: false
-          recursive: true
-          parent-credentials: true
-          timeout: 100
-        timeout: '15'
-        use-author: true
-        wipe-workspace: 'True'
-    periodic-folder-trigger: 1d
-    prune-dead-branches: true
-    publishers:
-    - email:
-        recipients: infra-root+build@elastic.co
-
--- a/.ci/jobs/elastic+elasticsearch-js+pull-request.yml
+++ b/.ci/jobs/elastic+elasticsearch-js+pull-request.yml
@ -0,0 +1,19 @@
+---
+- job:
+    name: elastic+elasticsearch-js+pull-request
+    display-name: 'elastic / elasticsearch-js # pull-request'
+    description: Testing of elasticsearch-js pull requests.
+    scm:
+    - git:
+        branches:
+        - ${ghprbActualCommit}
+        refspec: +refs/pull/*:refs/remotes/origin/pr/*
+    triggers:
+    - github-pull-request:
+        org-list:
+        - elastic
+        allow-whitelist-orgs-as-admins: true
+        github-hooks: true
+        status-context: clients-ci
+        cancel-builds-on-update: true
+    publishers: []
--- a/.ci/run-elasticsearch.sh
+++ b/.ci/run-elasticsearch.sh
@ -0,0 +1,175 @@
+#!/usr/bin/env bash
+#
+# Launch one or more Elasticsearch nodes via the Docker image,
+# to form a cluster suitable for running the REST API tests.
+#
+# Export the ELASTICSEARCH_VERSION variable, eg. 'elasticsearch:8.0.0-SNAPSHOT'.
+
+if [[ -z "$ELASTICSEARCH_VERSION" ]]; then
+  echo -e "\033[31;1mERROR:\033[0m Required environment variable [ELASTICSEARCH_VERSION] not set\033[0m"
+  exit 1
+fi
+
+set -euxo pipefail
+
+moniker=$(echo "$ELASTICSEARCH_VERSION" | tr -C "[:alnum:]" '-')
+suffix=rest-test
+
+NODE_NAME=${NODE_NAME-${moniker}node1}
+MASTER_NODE_NAME=${MASTER_NODE_NAME-${NODE_NAME}}
+CLUSTER_NAME=${CLUSTER_NAME-${moniker}${suffix}}
+HTTP_PORT=${HTTP_PORT-9200}
+
+ELASTIC_PASSWORD=${ELASTIC_PASSWORD-changeme}
+SSL_CERT=${SSL_CERT-"$PWD/certs/testnode.crt"}
+SSL_KEY=${SSL_KEY-"$PWD/certs/testnode.key"}
+SSL_CA=${SSL_CA-"$PWD/certs/ca.crt"}
+
+DETACH=${DETACH-false}
+CLEANUP=${CLEANUP-false}
+
+volume_name=${NODE_NAME}-${suffix}-data
+network_default=${moniker}${suffix}
+NETWORK_NAME=${NETWORK_NAME-"$network_default"}
+
+set +x
+
+function cleanup_volume {
+  if [[ "$(docker volume ls -q -f name=$1)" ]]; then
+    echo -e "\033[34;1mINFO:\033[0m Removing volume $1\033[0m"
+    (docker volume rm "$1") || true
+  fi
+}
+function cleanup_node {
+  if [[ "$(docker ps -q -f name=$1)" ]]; then
+    echo -e "\033[34;1mINFO:\033[0m Removing container $1\033[0m"
+    (docker container rm --force --volumes "$1") || true
+    cleanup_volume "$1-${suffix}-data"
+  fi
+}
+function cleanup_network {
+  if [[ "$(docker network ls -q -f name=$1)" ]]; then
+    echo -e "\033[34;1mINFO:\033[0m Removing network $1\033[0m"
+    (docker network rm "$1") || true
+  fi
+}
+
+function cleanup {
+  if [[ "$DETACH" != "true" ]] || [[ "$1" == "1" ]]; then
+    echo -e "\033[34;1mINFO:\033[0m clean the node and volume on startup (1) OR on exit if not detached\033[0m"
+    cleanup_node "$NODE_NAME"
+  fi
+  if [[ "$DETACH" != "true" ]]; then
+    echo -e "\033[34;1mINFO:\033[0m clean the network if not detached (start and exit)\033[0m"
+    cleanup_network "$NETWORK_NAME"
+  fi
+};
+trap "cleanup 0" EXIT
+
+if [[ "$CLEANUP" == "true" ]]; then
+  trap - EXIT
+  if [[ -z "$(docker network ls -q -f name=${NETWORK_NAME})" ]]; then
+    echo -e "\033[34;1mINFO:\033[0m $NETWORK_NAME is already deleted\033[0m"
+    exit 0
+  fi
+  containers=$(docker network inspect -f '{{ range $key, $value := .Containers }}{{ printf "%s\n" .Name}}{{ end }}' ${NETWORK_NAME})
+  while read -r container; do
+    cleanup_node "$container"
+  done <<< "$containers"
+  cleanup_network "$NETWORK_NAME"
+  echo -e "\033[32;1mSUCCESS:\033[0m Cleaned up and exiting\033[0m"
+  exit 0
+fi
+
+echo -e "\033[34;1mINFO:\033[0m Making sure previous run leftover infrastructure is removed \033[0m"
+cleanup 1
+
+echo -e "\033[34;1mINFO:\033[0m Creating network $NETWORK_NAME if it does not exist already \033[0m"
+docker network inspect "$NETWORK_NAME" > /dev/null 2>&1 || docker network create "$NETWORK_NAME"
+
+environment=($(cat <<-END
+  --env node.name=$NODE_NAME
+  --env cluster.name=$CLUSTER_NAME
+  --env cluster.routing.allocation.disk.threshold_enabled=false
+  --env bootstrap.memory_lock=true
+  --env node.attr.testattr=test
+  --env path.repo=/tmp
+  --env repositories.url.allowed_urls=http://snapshot.test*
+END
+))
+
+volumes=($(cat <<-END
+  --volume $volume_name:/usr/share/elasticsearch/data
+END
+))
+
+if [[ "$ELASTICSEARCH_VERSION" != *oss* ]]; then
+  environment+=($(cat <<-END
+    --env ELASTIC_PASSWORD=$ELASTIC_PASSWORD
+    --env xpack.license.self_generated.type=trial
+    --env xpack.security.enabled=true
+    --env xpack.security.http.ssl.enabled=true
+    --env xpack.security.http.ssl.verification_mode=certificate
+    --env xpack.security.http.ssl.key=certs/testnode.key
+    --env xpack.security.http.ssl.certificate=certs/testnode.crt
+    --env xpack.security.http.ssl.certificate_authorities=certs/ca.crt
+    --env xpack.security.transport.ssl.enabled=true
+    --env xpack.security.transport.ssl.key=certs/testnode.key
+    --env xpack.security.transport.ssl.certificate=certs/testnode.crt
+    --env xpack.security.transport.ssl.certificate_authorities=certs/ca.crt
+END
+))
+  volumes+=($(cat <<-END
+    --volume $SSL_CERT:/usr/share/elasticsearch/config/certs/testnode.crt
+    --volume $SSL_KEY:/usr/share/elasticsearch/config/certs/testnode.key
+    --volume $SSL_CA:/usr/share/elasticsearch/config/certs/ca.crt
+END
+))
+fi
+
+url="http://$NODE_NAME"
+if [[ "$ELASTICSEARCH_VERSION" != *oss* ]]; then
+  url="https://elastic:$ELASTIC_PASSWORD@$NODE_NAME"
+fi
+
+echo -e "\033[34;1mINFO:\033[0m Starting container $NODE_NAME \033[0m"
+set -x
+docker run \
+  --name "$NODE_NAME" \
+  --network "$NETWORK_NAME" \
+  --env ES_JAVA_OPTS=-"Xms1g -Xmx1g" \
+  "${environment[@]}" \
+  "${volumes[@]}" \
+  --publish "$HTTP_PORT":9200 \
+  --ulimit nofile=65536:65536 \
+  --ulimit memlock=-1:-1 \
+  --detach="$DETACH" \
+  --health-cmd="curl --silent --insecure --fail $url:9200/_cluster/health || exit 1" \
+  --health-interval=2s \
+  --health-retries=20 \
+  --health-timeout=2s \
+  --rm \
+  docker.elastic.co/elasticsearch/"$ELASTICSEARCH_VERSION";
+set +x
+
+if [[ "$DETACH" == "true" ]]; then
+  until [[ "$(docker inspect -f "{{.State.Health.Status}}" ${NODE_NAME})" != "starting" ]]; do
+    sleep 2;
+    echo ""
+    echo -e "\033[34;1mINFO:\033[0m waiting for node $NODE_NAME to be up\033[0m"
+  done;
+  # Always show the node getting started logs, this is very useful both on CI as well as while developing
+  docker logs "$NODE_NAME"
+  if [[ "$(docker inspect -f "{{.State.Health.Status}}" ${NODE_NAME})" != "healthy" ]]; then
+    cleanup 1
+    echo
+    echo -e "\033[31;1mERROR:\033[0m Failed to start ${ELASTICSEARCH_VERSION} in detached mode beyond health checks\033[0m"
+    echo -e "\033[31;1mERROR:\033[0m dumped the docker log before shutting the node down\033[0m"
+    exit 1
+  else
+    echo
+    echo -e "\033[32;1mSUCCESS:\033[0m Detached and healthy: ${NODE_NAME} on docker network: ${NETWORK_NAME}\033[0m"
+    echo -e "\033[32;1mSUCCESS:\033[0m Running on: ${url/$NODE_NAME/localhost}:${HTTP_PORT}\033[0m"
+    exit 0
+  fi
+fi
--- a/.ci/run-tests
+++ b/.ci/run-tests
@ -8,6 +8,7 @@
 #
 #   - $ELASTICSEARCH_VERSION
 #   - $NODE_JS_VERSION
+#   - $TEST_SUITE
 #

 set -eo pipefail
@ -18,93 +19,41 @@ export CODECOV_TOKEN=$(vault read -field=token secret/clients-ci/elasticsearch-j
 unset VAULT_ROLE_ID VAULT_SECRET_ID VAULT_TOKEN
 set -x

-function cleanup {
-  docker container rm --force --volumes elasticsearch-oss > /dev/null 2>&1 || true
-  docker container rm --force --volumes elasticsearch-platinum > /dev/null 2>&1 || true
-  docker container rm --force --volumes elasticsearch-js-oss > /dev/null 2>&1 || true
-  docker container rm --force --volumes elasticsearch-js-platinum > /dev/null 2>&1 || true
-  docker network rm esnet-oss > /dev/null
-  docker network rm esnet-platinum > /dev/null
-}
-
-trap cleanup EXIT
-
-# create network and volume
-docker network create esnet-oss
-docker network create esnet-platinum
-
-# create client image
 docker build \
  --file .ci/Dockerfile \
  --tag elastic/elasticsearch-js \
  --build-arg NODE_JS_VERSION=${NODE_JS_VERSION} \
  .

-# run elasticsearch oss
-docker run \
-  --rm \
-  --env "node.attr.testattr=test" \
-  --env "path.repo=/tmp" \
-  --env "repositories.url.allowed_urls=http://snapshot.*" \
-  --env "discovery.type=single-node" \
-  --network=esnet-oss \
-  --name=elasticsearch-oss \
-  --detach \
-  docker.elastic.co/elasticsearch/elasticsearch-oss:${ELASTICSEARCH_VERSION}
-
-# run elasticsearch platinum
+NODE_NAME="es1"
 repo=$(pwd)
 testnodecrt="/.ci/certs/testnode.crt"
 testnodekey="/.ci/certs/testnode.key"
 cacrt="/.ci/certs/ca.crt"

-docker run \
-  --rm \
-  --env "node.attr.testattr=test" \
-  --env "path.repo=/tmp" \
-  --env "repositories.url.allowed_urls=http://snapshot.*" \
-  --env "discovery.type=single-node" \
-  --env "ES_JAVA_OPTS=-Xms1g -Xmx1g" \
-  --env "ELASTIC_PASSWORD=changeme" \
-  --env "xpack.security.enabled=true" \
-  --env "xpack.license.self_generated.type=trial" \
-  --env "xpack.security.http.ssl.enabled=true" \
-  --env "xpack.security.http.ssl.verification_mode=certificate" \
-  --env "xpack.security.http.ssl.key=certs/testnode.key" \
-  --env "xpack.security.http.ssl.certificate=certs/testnode.crt" \
-  --env "xpack.security.http.ssl.certificate_authorities=certs/ca.crt" \
-  --env "xpack.security.transport.ssl.enabled=true" \
-  --env "xpack.security.transport.ssl.key=certs/testnode.key" \
-  --env "xpack.security.transport.ssl.certificate=certs/testnode.crt" \
-  --env "xpack.security.transport.ssl.certificate_authorities=certs/ca.crt" \
-  --volume "$repo$testnodecrt:/usr/share/elasticsearch/config/certs/testnode.crt" \
-  --volume "$repo$testnodekey:/usr/share/elasticsearch/config/certs/testnode.key" \
-  --volume "$repo$cacrt:/usr/share/elasticsearch/config/certs/ca.crt" \
-  --network=esnet-platinum \
-  --name=elasticsearch-platinum \
-  --detach \
-  docker.elastic.co/elasticsearch/elasticsearch:${ELASTICSEARCH_VERSION}
+elasticsearch_image="elasticsearch"
+elasticsearch_url="https://elastic:changeme@${NODE_NAME}:9200"
+if [[ $TEST_SUITE != "xpack" ]]; then
+  elasticsearch_image="elasticsearch-oss"
+  elasticsearch_url="http://${NODE_NAME}:9200"
+fi
+
+ELASTICSEARCH_VERSION="${elasticsearch_image}:${ELASTICSEARCH_VERSION}" \
+  NODE_NAME="${NODE_NAME}" \
+  NETWORK_NAME="esnet" \
+  DETACH=true \
+  SSL_CERT="${repo}${testnodecrt}" \
+  SSL_KEY="${repo}${testnodekey}" \
+  SSL_CA="${repo}${cacrt}" \
+  bash .ci/run-elasticsearch.sh

-# run the client unit and oss integration test
 docker run \
-  --network=esnet-oss \
-  --env "TEST_ES_SERVER=http://elasticsearch-oss:9200" \
+  --network=esnet \
+  --env "TEST_ES_SERVER=${elasticsearch_url}" \
  --env "CODECOV_TOKEN" \
  --volume $repo:/usr/src/app \
  --volume /usr/src/app/node_modules \
-  --name elasticsearch-js-oss \
+  --name elasticsearch-js \
  --rm \
  elastic/elasticsearch-js \
  npm run ci
-
-# run the client platinium integration test
-docker run \
-  --network=esnet-platinum \
-  --env "TEST_ES_SERVER=https://elastic:changeme@elasticsearch-platinum:9200" \
-  --volume $repo:/usr/src/app \
-  --volume /usr/src/app/node_modules \
-  --name elasticsearch-js-platinum \
-  --rm \
-  elastic/elasticsearch-js \
-  npm run test:integration
-
--- a/.ci/test-matrix.yml
+++ b/.ci/test-matrix.yml
@ -1,10 +1,14 @@
 ---
 ELASTICSEARCH_VERSION:
-  - 6.8.1
+  - 6.8.4

 NODE_JS_VERSION:
  - 12
  - 10
  - 8

+TEST_SUITE:
+  - oss
+  - xpack
+
 exclude: ~
--- a/README.md
+++ b/README.md
@ -2,7 +2,7 @@

 # Elasticsearch Node.js client

-[![js-standard-style](https://img.shields.io/badge/code%20style-standard-brightgreen.svg?style=flat)](http://standardjs.com/)  [![Build Status](https://clients-ci.elastic.co/job/elastic+elasticsearch-js+master/badge/icon)](https://clients-ci.elastic.co/job/elastic+elasticsearch-js+master/)  [![codecov](https://codecov.io/gh/elastic/elasticsearch-js/branch/master/graph/badge.svg)](https://codecov.io/gh/elastic/elasticsearch-js)  [![NPM downloads](https://img.shields.io/npm/dm/@elastic/elasticsearch.svg?style=flat)](https://www.npmjs.com/package/@elastic/elasticsearch)
+[![js-standard-style](https://img.shields.io/badge/code%20style-standard-brightgreen.svg?style=flat)](http://standardjs.com/)  [![Build Status](https://clients-ci.elastic.co/buildStatus/icon?job=elastic%2Belasticsearch-js%2Bmaster)](https://clients-ci.elastic.co/view/Javascript/job/elastic+elasticsearch-js+master/)  [![codecov](https://codecov.io/gh/elastic/elasticsearch-js/branch/master/graph/badge.svg)](https://codecov.io/gh/elastic/elasticsearch-js)  [![NPM downloads](https://img.shields.io/npm/dm/@elastic/elasticsearch.svg?style=flat)](https://www.npmjs.com/package/@elastic/elasticsearch)

 The official Node.js client for Elasticsearch.

--- a/docs/configuration.asciidoc
+++ b/docs/configuration.asciidoc
@ -173,6 +173,11 @@ function generateRequestId (params, options) {
 |`string` - The name to identify the client instance in the events. +
 _Default:_ `elasticsearch-js`

+|`opaqueIdPrefix`
+|`string` - A string that will be use to prefix any `X-Opaque-Id` header. +
+See https://www.elastic.co/guide/en/elasticsearch/client/javascript-api/current/observability.html#_x-opaque-id_support[`X-Opaque-Id` support] for more details. +
+_Default:_ `null`
+
 |`headers`
 |`object` - A set of custom headers to send in every request. +
 _Default:_ `{}`
--- a/docs/observability.asciidoc
+++ b/docs/observability.asciidoc
@ -248,3 +248,46 @@ child.search({
  if (err) console.log(err)
 })
 ----
+
+=== X-Opaque-Id support
+To improve the overall observability, the client offers an easy way to configure the `X-Opaque-Id` header. If you set the `X-Opaque-Id` in a specific request, this will allow you to discover this identifier in the https://www.elastic.co/guide/en/elasticsearch/reference/master/logging.html#deprecation-logging[deprecation logs], help you with https://www.elastic.co/guide/en/elasticsearch/reference/master/index-modules-slowlog.html#_identifying_search_slow_log_origin[identifying search slow log origin] as well as https://www.elastic.co/guide/en/elasticsearch/reference/master/tasks.html#_identifying_running_tasks[identifying running tasks].
+
+The `X-Opaque-Id` should be configured in each request, for doing that you can use the `opaqueId` option, as you can see in the following example. +
+The resulting header will be `{ 'X-Opaque-Id': 'my-search' }`.
+
+[source,js]
+----
+const { Client } = require('@elastic/elasticsearch')
+const client = new Client({
+  node: 'http://localhost:9200'
+})
+
+client.search({
+  index: 'my-index',
+  body: { foo: 'bar' }
+}, {
+  opaqueId: 'my-search'
+}, (err, result) => {
+  if (err) console.log(err)
+})
+----
+
+Sometimes it may be useful to prefix all the `X-Opaque-Id` headers with a specific string, in case you need to identify a specific client or server. For doing this, the client offers a top-level configuration option: `opaqueIdPrefix`. +
+In the following example, the resulting header will be `{ 'X-Opaque-Id': 'proxy-client::my-search' }`.
+[source,js]
+----
+const { Client } = require('@elastic/elasticsearch')
+const client = new Client({
+  node: 'http://localhost:9200',
+  opaqueIdPrefix: 'proxy-client::'
+})
+
+client.search({
+  index: 'my-index',
+  body: { foo: 'bar' }
+}, {
+  opaqueId: 'my-search'
+}, (err, result) => {
+  if (err) console.log(err)
+})
+----
--- a/index.d.ts
+++ b/index.d.ts
@ -94,6 +94,7 @@ interface ClientOptions {
  nodeFilter?: nodeFilterFn;
  nodeSelector?: nodeSelectorFn | string;
  headers?: anyObject;
+  opaqueIdPrefix?: string;
  generateRequestId?: generateRequestIdFn;
  name?: string;
  auth?: BasicAuth | ApiKeyAuth;
--- a/index.js
+++ b/index.js
@ -79,7 +79,8 @@ class Client extends EventEmitter {
      nodeSelector: 'round-robin',
      generateRequestId: null,
      name: 'elasticsearch-js',
-      auth: null
+      auth: null,
+      opaqueIdPrefix: null
    }, opts)

    this[kInitialOptions] = options
@ -121,7 +122,8 @@ class Client extends EventEmitter {
      nodeFilter: options.nodeFilter,
      nodeSelector: options.nodeSelector,
      generateRequestId: options.generateRequestId,
-      name: options.name
+      name: options.name,
+      opaqueIdPrefix: options.opaqueIdPrefix
    })

    const apis = buildApi({
--- a/lib/Transport.d.ts
+++ b/lib/Transport.d.ts
@ -38,6 +38,7 @@ interface TransportOptions {
  headers?: anyObject;
  generateRequestId?: generateRequestIdFn;
  name: string;
+  opaqueIdPrefix?: string;
 }

 export interface RequestEvent<T = any, C = any> {
@ -90,6 +91,7 @@ export interface TransportRequestOptions {
  id?: any;
  context?: any;
  warnings?: [string];
+  opaqueId?: string;
 }

 export interface TransportRequestCallback {
@ -121,6 +123,7 @@ export default class Transport {
  compression: 'gzip' | false;
  sniffInterval: number;
  sniffOnConnectionFault: boolean;
+  opaqueIdPrefix: string | null;
  sniffEndpoint: string;
  _sniffEnabled: boolean;
  _nextSniff: number;
--- a/lib/Transport.js
+++ b/lib/Transport.js
@ -41,6 +41,7 @@ class Transport {
    this.sniffEndpoint = opts.sniffEndpoint
    this.generateRequestId = opts.generateRequestId || generateRequestId()
    this.name = opts.name
+    this.opaqueIdPrefix = opts.opaqueIdPrefix

    this.nodeFilter = opts.nodeFilter || defaultNodeFilter
    if (typeof opts.nodeSelector === 'function') {
@ -114,6 +115,12 @@ class Transport {
      // TODO: make this assignment FAST
      const headers = Object.assign({}, this.headers, options.headers)

+      if (options.opaqueId !== undefined) {
+        headers['X-Opaque-Id'] = this.opaqueIdPrefix !== null
+          ? this.opaqueIdPrefix + options.opaqueId
+          : options.opaqueId
+      }
+
      // handle json body
      if (params.body != null) {
        if (shouldSerialize(params.body) === true) {
--- a/package.json
+++ b/package.json
@ -4,7 +4,7 @@
  "main": "index.js",
  "types": "index.d.ts",
  "homepage": "http://www.elastic.co/guide/en/elasticsearch/client/javascript-api/current/index.html",
-  "version": "6.8.3",
+  "version": "6.8.4",
  "keywords": [
    "elasticsearch",
    "elastic",
--- a/test/unit/client.test.js
+++ b/test/unit/client.test.js
@ -851,3 +851,87 @@ test('Elastic cloud config', t => {

  t.end()
 })
+
+test('Opaque Id support', t => {
+  t.test('No opaqueId', t => {
+    t.plan(3)
+
+    function handler (req, res) {
+      t.strictEqual(req.headers['x-opaque-id'], undefined)
+      res.setHeader('Content-Type', 'application/json;utf=8')
+      res.end(JSON.stringify({ hello: 'world' }))
+    }
+
+    buildServer(handler, ({ port }, server) => {
+      const client = new Client({
+        node: `http://localhost:${port}`
+      })
+
+      client.search({
+        index: 'test',
+        q: 'foo:bar'
+      }, (err, { body }) => {
+        t.error(err)
+        t.deepEqual(body, { hello: 'world' })
+        server.stop()
+      })
+    })
+  })
+
+  t.test('No prefix', t => {
+    t.plan(3)
+
+    function handler (req, res) {
+      t.strictEqual(req.headers['x-opaque-id'], 'bar')
+      res.setHeader('Content-Type', 'application/json;utf=8')
+      res.end(JSON.stringify({ hello: 'world' }))
+    }
+
+    buildServer(handler, ({ port }, server) => {
+      const client = new Client({
+        node: `http://localhost:${port}`
+      })
+
+      client.search({
+        index: 'test',
+        q: 'foo:bar'
+      }, {
+        opaqueId: 'bar'
+      }, (err, { body }) => {
+        t.error(err)
+        t.deepEqual(body, { hello: 'world' })
+        server.stop()
+      })
+    })
+  })
+
+  t.test('With prefix', t => {
+    t.plan(3)
+
+    function handler (req, res) {
+      t.strictEqual(req.headers['x-opaque-id'], 'foo-bar')
+      res.setHeader('Content-Type', 'application/json;utf=8')
+      res.end(JSON.stringify({ hello: 'world' }))
+    }
+
+    buildServer(handler, ({ port }, server) => {
+      const client = new Client({
+        node: `http://localhost:${port}`,
+        opaqueIdPrefix: 'foo-'
+      })
+
+      client.search({
+        index: 'test',
+        q: 'foo:bar'
+      }, {
+        opaqueId: 'bar'
+      }, (err, { body }) => {
+        t.error(err)
+        t.deepEqual(body, { hello: 'world' })
+        server.stop()
+      })
+    })
+  })
+
+  t.end()
+})
Author	SHA1	Message	Date
delvedor	b390193dcc	Bumped v6.8.4	2019-12-03 14:38:58 +01:00
Tomas Della Vedova	df76154388	X-Opaque-Id support (#997 ) * Added X-Opaque-Id support * Updated type definitions * Updated test * Updated docs	2019-12-03 14:33:33 +01:00
delvedor	3e2a823a72	Updated CI configuration	2019-10-28 15:41:42 +01:00
delvedor	8a1bb2665e	Updated jobs/defaults.yml to use the new job matrix	2019-10-28 14:46:40 +01:00
Tomas Della Vedova	3acee8b266	New CI configuration (#996 ) * Removed old files * Updated README * New CI configuration * Updated run-test script	2019-10-28 14:46:32 +01:00
Tomas Della Vedova	60a1f298d7	Move back to jjbb (#995 ) * Removed Jenkins pipelines configuration * Restored jjbb definitions	2019-10-28 14:46:03 +01:00