Use Jenkins pipelines (#937)

.ci/Jenkinsfile

@@ -0,0 +1,224 @@
+#!/usr/bin/env groovy
+
+@Library('apm@current') _
+
+def NODE_JS_VERSIONS = [8,10,12]
+def nodeJsVersion = NODE_JS_VERSIONS[randomNumber(min: 0, max:2)]
+
+pipeline {
+  agent {
+    label 'docker && immutable'
+  }
+
+  environment {
+    REPO = 'elasticsearch-js'
+    BASE_DIR = "src/github.com/elastic/${env.REPO}"
+    NODE_JS_DEFAULT_VERSION = "${nodeJsVersion}"
+    NODE_JS_VERSIONS = "${NODE_JS_VERSIONS.join(',')}"
+    HOME = "${env.WORKSPACE}"
+    npm_config_cache = 'npm-cache'
+  }
+
+  options {
+    timeout(time: 1, unit: 'HOURS')
+    buildDiscarder(logRotator(numToKeepStr: '20', artifactNumToKeepStr: '20', daysToKeepStr: '30'))
+    timestamps()
+    ansiColor('xterm')
+    disableResume()
+    durabilityHint('PERFORMANCE_OPTIMIZED')
+  }
+
+  triggers {
+    issueCommentTrigger('(?i).*(?:jenkins\\W+)?run\\W+(?:the\\W+)?tests(?:\\W+please)?.*')
+    // changeRequest() will return true in case of a commit or a pr
+    // we will have a daily cron job only ofr branches that don't have an active pr
+    cron(changeRequest() ? '' : '@daily')
+  }
+
+  stages {
+    stage('Checkout') {
+      options { skipDefaultCheckout() }
+      steps {
+        deleteDir()
+        gitCheckout(basedir: "${BASE_DIR}", githubNotifyFirstTimeContributor: false)
+        stash allowEmpty: true, name: 'source', useDefaultExcludes: false
+      }
+    }
+
+    stage('Install dependencies') {
+      options { skipDefaultCheckout() }
+      steps {
+        deleteDir()
+        unstash 'source'
+        script {
+          buildDockerImage(image: "node:${env.NODE_JS_DEFAULT_VERSION}-alpine").inside(){
+            dir("${BASE_DIR}"){
+              sh(label: 'System info', script: 'node --version; npm --version')
+              sh(label: 'Install dependencies', script: 'npm install')
+            }
+          }
+        }
+        stash allowEmpty: true, name: 'source-dependencies', useDefaultExcludes: false
+      }
+    }
+
+    stage('License check') {
+      options { skipDefaultCheckout() }
+      steps {
+        withGithubNotify(context: 'License check') {
+          deleteDir()
+          unstash 'source-dependencies'
+          script {
+            buildDockerImage(image: "node:${env.NODE_JS_DEFAULT_VERSION}-alpine").inside(){
+              dir("${BASE_DIR}"){
+                sh(label: 'Check production dependencies licenses', script: 'npm run license-checker')
+              }
+            }
+          }
+        }
+      }
+    }
+
+    stage('Linter') {
+      options { skipDefaultCheckout() }
+      steps {
+        withGithubNotify(context: 'Linter') {
+          deleteDir()
+          unstash 'source-dependencies'
+          script {
+            buildDockerImage(image: "node:${env.NODE_JS_DEFAULT_VERSION}-alpine").inside(){
+              dir("${BASE_DIR}"){
+                sh(label: 'Lint code with standardjs', script: 'npm run lint')
+              }
+            }
+          }
+        }
+      }
+    }
+
+    stage('Unit test') {
+      failFast true
+      options { skipDefaultCheckout() }
+      steps {
+        withGithubNotify(context: 'Unit test') {
+          script {
+            def versions = env.NODE_JS_VERSIONS.split(',')
+            def parallelTasks = [:]
+            versions.each{ version ->
+              parallelTasks["Node.js v${version}"] = buildUnitTest(version: version)
+            }
+            parallel(parallelTasks)
+          }
+        }
+      }
+    }
+
+    stage('Integration test') {
+      failFast true
+      options { skipDefaultCheckout() }
+      parallel {
+        stage('OSS') {
+          agent { label 'docker && immutable' }
+          options { skipDefaultCheckout() }
+          environment {
+            TEST_ES_SERVER = 'http://elasticsearch:9200'
+          }
+          steps {
+            withGithubNotify(context: 'Integration test OSS') {
+              deleteDir()
+              unstash 'source-dependencies'
+              dir("${BASE_DIR}"){
+                // Sometimes the docker registry fails and has random timeouts
+                // this block will retry a doker image 3 times before to fail.
+                retry(3) {
+                  sleep randomNumber(min: 5, max: 10)
+                  sh(label: 'Start Elasticsearch', script: './scripts/es-docker.sh --detach')
+                }
+              }
+              script {
+                buildDockerImage(fromDockerfile: true).inside('--network=elastic'){
+                  dir("${BASE_DIR}"){
+                    sh(label: 'Integration test', script: 'npm run test:integration | tee test-integration.tap')
+                    sh(label: 'Generating test reporting', script: './node_modules/.bin/tap-mocha-reporter xunit < test-integration.tap > junit-integration.xml')
+                  }
+                }
+              }
+              sh(label: 'Stop Elasticsearch', script: 'docker kill $(docker ps -q)')
+              junit(allowEmptyResults: true, keepLongStdio: true, testResults: "${BASE_DIR}/**/junit-*.xml")
+            }
+          }
+        }
+
+        stage('xPack') {
+          agent { label 'docker && immutable' }
+          options { skipDefaultCheckout() }
+          environment {
+            TEST_ES_SERVER = 'https://elastic:changeme@elasticsearch:9200'
+          }
+          steps {
+            withGithubNotify(context: 'Integration test xPack') {
+              deleteDir()
+              unstash 'source-dependencies'
+              dir("${BASE_DIR}"){
+                // Sometimes the docker registry fails and has random timeouts
+                // this block will retry a doker image 3 times before to fail.
+                retry(3) {
+                  sleep randomNumber(min: 5, max: 10)
+                  sh(label: 'Start Elasticsearch', script: './scripts/es-docker-platinum.sh --detach')
+                }
+              }
+              script {
+                buildDockerImage(fromDockerfile: true).inside('--network=elastic'){
+                  dir("${BASE_DIR}"){
+                    sh(label: 'Integration test', script: 'npm run test:integration | tee test-integration.tap')
+                    sh(label: 'Generating test reporting', script: './node_modules/.bin/tap-mocha-reporter xunit < test-integration.tap > junit-integration.xml')
+                  }
+                }
+              }
+              sh(label: 'Stop Elasticsearch', script: 'docker kill $(docker ps -q)')
+              junit(allowEmptyResults: true, keepLongStdio: true, testResults: "${BASE_DIR}/**/junit-*.xml")
+            }
+          }
+        }
+      }
+    }
+  }
+}
+
+// Sometimes the docker registry fails and has random timeouts
+// this function will retry a doker image 3 times before to fail.
+def buildDockerImage(args) {
+  def image
+  retry(3) {
+    sleep randomNumber(min: 5, max: 10)
+    if (args.fromDockerfile == true) {
+      image = docker.build('nodejs-image', "--build-arg NODE_JS_VERSION=${env.NODE_JS_DEFAULT_VERSION} ${BASE_DIR}/.ci/docker")
+    } else {
+      image = docker.image(args.image)
+      // make sure we have the latest available from Docker Hub
+      image.pull()
+    }
+  }
+  return image
+}
+
+def buildUnitTest(args) {
+  return {
+    node('docker && immutable') {
+      deleteDir()
+      unstash 'source'
+      script {
+        buildDockerImage(image: "node:${args.version}-alpine").inside(){
+          dir("${BASE_DIR}"){
+            sh(label: 'Install dependencies', script: 'npm install')
+            sh(label: 'Run unit test', script: 'npm run test:unit | tee test-unit.tap')
+            sh(label: 'Run behavior test', script: 'npm run test:behavior | tee test-behavior.tap')
+            sh(label: 'Run types test', script: 'npm run test:types')
+            sh(label: 'Generating test reporting', script: './node_modules/.bin/tap-mocha-reporter xunit < test-unit.tap > junit-unit.xml; ./node_modules/.bin/tap-mocha-reporter xunit < test-behavior.tap > junit-behavior.xml')
+          }
+        }
+      }
+      junit(allowEmptyResults: true, keepLongStdio: true, testResults: "${BASE_DIR}/**/junit-*.xml")
+    }
+  }
+}

.ci/docker/Dockerfile

@@ -0,0 +1,7 @@
+ARG NODE_JS_VERSION=10
+FROM node:${NODE_JS_VERSION}-alpine
+
+RUN apk --no-cache add git
+
+# Create app directory
+WORKDIR /usr/src/app

.ci/jobs/defaults.yml

@@ -1,70 +0,0 @@
----
-
-##### GLOBAL METADATA
-
-- meta:
-    cluster: clients-ci
-
-##### JOB DEFAULTS
-
-- job:
-    project-type: matrix
-    logrotate:
-      daysToKeep: 30
-      numToKeep: 100
-    properties:
-    - github:
-        url: https://github.com/elastic/elasticsearch-js/
-    - inject:
-        properties-content: HOME=$JENKINS_HOME
-    concurrent: true
-    node: flyweight
-    scm:
-    - git:
-        name: origin
-        credentials-id: f6c7695a-671e-4f4f-a331-acdce44ff9ba
-        reference-repo: /var/lib/jenkins/.git-references/elasticsearch-js.git
-        branches:
-        - ${branch_specifier}
-        url: https://github.com/elastic/elasticsearch-js.git
-        wipe-workspace: 'True'
-    triggers:
-    - github
-    vault:
-      # vault read auth/approle/role/clients-ci/role-id
-      role_id: ddbd0d44-0e51-105b-177a-c8fdfd445126
-    axes:
-    - axis:
-        type: slave
-        name: label
-        values:
-        - linux
-    - axis:
-        type: yaml
-        filename: .ci/test-matrix.yml
-        name: ELASTICSEARCH_VERSION
-    - axis:
-        type: yaml
-        filename: .ci/test-matrix.yml
-        name: NODE_JS_VERSION
-    yaml-strategy:
-      exclude-key: exclude
-      filename: .ci/test-matrix.yml
-    wrappers:
-    - ansicolor
-    - timeout:
-        type: absolute
-        timeout: 120
-        fail: true
-    - timestamps
-    - workspace-cleanup
-    builders:
-    - shell: |-
-        #!/usr/local/bin/runbld
-        .ci/run-tests
-    publishers:
-    - email:
-        recipients: infra-root+build@elastic.co
-    # - junit:
-    #     results: "*-junit.xml"
-    #     allow-empty-results: true

.ci/jobs/elastic+elasticsearch-js+5.x.yml

@@ -1,14 +0,0 @@
----
-- job:
-    name: elastic+elasticsearch-js+5.x
-    display-name: 'elastic / elasticsearch-js # 5.x'
-    description: Testing the elasticsearch-js 5.x branch.
-    parameters:
-    - string:
-        name: branch_specifier
-        default: refs/heads/5.x
-        description: the Git branch specifier to build (<branchName>, <tagName>,
-          <commitId>, etc.)
-    triggers:
-    - github
-    - timed: '@weekly'

.ci/jobs/elastic+elasticsearch-js+6.x.yml

@@ -1,14 +0,0 @@
----
-- job:
-    name: elastic+elasticsearch-js+6.x
-    display-name: 'elastic / elasticsearch-js # 6.x'
-    description: Testing the elasticsearch-js 6.x branch.
-    parameters:
-    - string:
-        name: branch_specifier
-        default: refs/heads/6.x
-        description: the Git branch specifier to build (<branchName>, <tagName>,
-          <commitId>, etc.)
-    triggers:
-    - github
-    - timed: '@weekly'

.ci/jobs/elastic+elasticsearch-js+7.x.yml

@@ -1,14 +0,0 @@
----
-- job:
-    name: elastic+elasticsearch-js+7.x
-    display-name: 'elastic / elasticsearch-js # 7.x'
-    description: Testing the elasticsearch-js 7.x branch.
-    parameters:
-    - string:
-        name: branch_specifier
-        default: refs/heads/7.x
-        description: the Git branch specifier to build (<branchName>, <tagName>,
-          <commitId>, etc.)
-    triggers:
-    - github
-    - timed: '@weekly'

.ci/jobs/elastic+elasticsearch-js+master.yml

@@ -1,14 +0,0 @@
----
-- job:
-    name: elastic+elasticsearch-js+master
-    display-name: 'elastic / elasticsearch-js # master'
-    description: Testing the elasticsearch-js master branch.
-    parameters:
-    - string:
-        name: branch_specifier
-        default: refs/heads/master
-        description: the Git branch specifier to build (<branchName>, <tagName>,
-          <commitId>, etc.)
-    triggers:
-    - github
-    - timed: '@daily'

.ci/jobs/elastic+elasticsearch-js+mbp.yml

@@ -0,0 +1,55 @@
+---
+- job:
+    name: elastic+elasticsearch-js+mbp
+    display-name: 'elastic / elasticsearch-js'
+    description: Testing elasticsearch-js.
+    project-type: multibranch
+    properties: []
+    triggers: []
+    logrotate:
+      daysToKeep: 30
+    number-to-keep: '5'
+    days-to-keep: '1'
+    concurrent: true
+    node: linux
+    script-path: .ci/Jenkinsfile
+    scm:
+    - github:
+        branch-discovery: no-pr
+        discover-pr-forks-strategy: merge-current
+        discover-pr-forks-trust: permission
+        discover-pr-origin: merge-current
+        discover-tags: false
+        repo: elasticsearch-js
+        repo-owner: elastic
+        credentials-id: 2a9602aa-ab9f-4e52-baf3-b71ca88469c7-UserAndToken
+        ssh-checkout:
+          credentials: f6c7695a-671e-4f4f-a331-acdce44ff9ba
+        build-strategies:
+        - tags:
+            ignore-tags-older-than: -1
+            ignore-tags-newer-than: -1
+        - regular-branches: true
+        - change-request:
+            ignore-target-only-changes: false
+        clean:
+          after: true
+          before: true
+        prune: true
+        shallow-clone: true
+        depth: 3
+        do-not-fetch-tags: true
+        submodule:
+          disable: false
+          recursive: true
+          parent-credentials: true
+          timeout: 100
+        timeout: '15'
+        use-author: true
+        wipe-workspace: 'True'
+    periodic-folder-trigger: 1d
+    prune-dead-branches: true
+    publishers:
+    - email:
+        recipients: infra-root+build@elastic.co
+

.ci/jobs/elastic+elasticsearch-js+pull-request.yml

@@ -1,19 +0,0 @@
----
-- job:
-    name: elastic+elasticsearch-js+pull-request
-    display-name: 'elastic / elasticsearch-js # pull-request'
-    description: Testing of elasticsearch-js pull requests.
-    scm:
-    - git:
-        branches:
-        - ${ghprbActualCommit}
-        refspec: +refs/pull/*:refs/remotes/origin/pr/*
-    triggers:
-    - github-pull-request:
-        org-list:
-        - elastic
-        allow-whitelist-orgs-as-admins: true
-        github-hooks: true
-        status-context: clients-ci
-        cancel-builds-on-update: true
-    publishers: []

.ci/packer_cache.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+
+source /usr/local/bin/bash_standard_lib.sh
+
+DOCKER_IMAGES="node:12-alpine
+node:10-alpine
+node:8-alpine
+"
+
+for di in ${DOCKER_IMAGES}
+do
+(retry 2 docker pull "${di}") || echo "Error pulling ${di} Docker image, we continue"
+done
+

package.json

@@ -20,13 +20,14 @@
     "test:unit": "tap test/unit/*.test.js -t 300 --no-coverage",
     "test:behavior": "tap test/behavior/*.test.js -t 300 --no-coverage",
     "test:integration": "tap test/integration/index.js -T --no-coverage",
-    "test:integration:report": "npm run test:integration | tap-mocha-reporter xunit > $WORKSPACE/test-report-junit.xml",
     "test:types": "tsc --project ./test/types/tsconfig.json",
     "test:coverage": "nyc tap test/unit/*.test.js test/behavior/*.test.js -t 300 && nyc report --reporter=text-lcov > coverage.lcov && codecov",
     "lint": "standard",
     "lint:fix": "standard --fix",
     "ci": "npm run license-checker && npm test && npm run test:integration && npm run test:coverage",
-    "license-checker": "license-checker --production --onlyAllow='MIT;Apache-2.0;Apache1.1;ISC;BSD-3-Clause;BSD-2-Clause'"
+    "license-checker": "license-checker --production --onlyAllow='MIT;Apache-2.0;Apache1.1;ISC;BSD-3-Clause;BSD-2-Clause'",
+    "elasticsearch": "./scripts/es-docker.sh",
+    "elasticsearch:xpack": "./scripts/es-docker-platinum.sh"
   },
   "author": {
     "name": "Tomas Della Vedova",

scripts/es-docker-platinum.sh

@@ -6,31 +6,66 @@ testnodekey="/.ci/certs/testnode.key"
 cacrt="/.ci/certs/ca.crt"
 
 # pass `--clean` to reemove the old snapshot
-if [ "$1" != "" ]; then
+if [ "$1" == "--clean" ]; then
   docker rmi $(docker images --format '{{.Repository}}:{{.Tag}}' | grep '8.0.0-SNAPSHOT')
 fi
 
-exec docker run \
-  --rm \
-  -e "node.attr.testattr=test" \
-  -e "path.repo=/tmp" \
-  -e "repositories.url.allowed_urls=http://snapshot.*" \
-  -e "discovery.type=single-node" \
-  -e "ES_JAVA_OPTS=-Xms1g -Xmx1g" \
-  -e "ELASTIC_PASSWORD=changeme" \
-  -e "xpack.security.enabled=true" \
-  -e "xpack.license.self_generated.type=trial" \
-  -e "xpack.security.http.ssl.enabled=true" \
-  -e "xpack.security.http.ssl.verification_mode=certificate" \
-  -e "xpack.security.http.ssl.key=certs/testnode.key" \
-  -e "xpack.security.http.ssl.certificate=certs/testnode.crt" \
-  -e "xpack.security.http.ssl.certificate_authorities=certs/ca.crt" \
-  -e "xpack.security.transport.ssl.enabled=true" \
-  -e "xpack.security.transport.ssl.key=certs/testnode.key" \
-  -e "xpack.security.transport.ssl.certificate=certs/testnode.crt" \
-  -e "xpack.security.transport.ssl.certificate_authorities=certs/ca.crt" \
-  -v "$repo$testnodecrt:/usr/share/elasticsearch/config/certs/testnode.crt" \
-  -v "$repo$testnodekey:/usr/share/elasticsearch/config/certs/testnode.key" \
-  -v "$repo$cacrt:/usr/share/elasticsearch/config/certs/ca.crt" \
-  -p 9200:9200 \
-  docker.elastic.co/elasticsearch/elasticsearch:6.7.1
+# Create the 'elastic' network if doesn't exist
+exec docker network ls | grep elastic > /dev/null || docker network create elastic > /dev/null
+
+if [ "$1" == "--detach" ]; then
+  exec docker run \
+    --rm \
+    -e "node.attr.testattr=test" \
+    -e "path.repo=/tmp" \
+    -e "repositories.url.allowed_urls=http://snapshot.*" \
+    -e "discovery.type=single-node" \
+    -e "ES_JAVA_OPTS=-Xms1g -Xmx1g" \
+    -e "ELASTIC_PASSWORD=changeme" \
+    -e "xpack.security.enabled=true" \
+    -e "xpack.license.self_generated.type=trial" \
+    -e "xpack.security.http.ssl.enabled=true" \
+    -e "xpack.security.http.ssl.verification_mode=certificate" \
+    -e "xpack.security.http.ssl.key=certs/testnode.key" \
+    -e "xpack.security.http.ssl.certificate=certs/testnode.crt" \
+    -e "xpack.security.http.ssl.certificate_authorities=certs/ca.crt" \
+    -e "xpack.security.transport.ssl.enabled=true" \
+    -e "xpack.security.transport.ssl.key=certs/testnode.key" \
+    -e "xpack.security.transport.ssl.certificate=certs/testnode.crt" \
+    -e "xpack.security.transport.ssl.certificate_authorities=certs/ca.crt" \
+    -v "$repo$testnodecrt:/usr/share/elasticsearch/config/certs/testnode.crt" \
+    -v "$repo$testnodekey:/usr/share/elasticsearch/config/certs/testnode.key" \
+    -v "$repo$cacrt:/usr/share/elasticsearch/config/certs/ca.crt" \
+    -p 9200:9200 \
+    --detach \
+    --network=elastic \
+    --name=elasticsearch \
+    docker.elastic.co/elasticsearch/elasticsearch:6.8.2
+else
+  exec docker run \
+    --rm \
+    -e "node.attr.testattr=test" \
+    -e "path.repo=/tmp" \
+    -e "repositories.url.allowed_urls=http://snapshot.*" \
+    -e "discovery.type=single-node" \
+    -e "ES_JAVA_OPTS=-Xms1g -Xmx1g" \
+    -e "ELASTIC_PASSWORD=changeme" \
+    -e "xpack.security.enabled=true" \
+    -e "xpack.license.self_generated.type=trial" \
+    -e "xpack.security.http.ssl.enabled=true" \
+    -e "xpack.security.http.ssl.verification_mode=certificate" \
+    -e "xpack.security.http.ssl.key=certs/testnode.key" \
+    -e "xpack.security.http.ssl.certificate=certs/testnode.crt" \
+    -e "xpack.security.http.ssl.certificate_authorities=certs/ca.crt" \
+    -e "xpack.security.transport.ssl.enabled=true" \
+    -e "xpack.security.transport.ssl.key=certs/testnode.key" \
+    -e "xpack.security.transport.ssl.certificate=certs/testnode.crt" \
+    -e "xpack.security.transport.ssl.certificate_authorities=certs/ca.crt" \
+    -v "$repo$testnodecrt:/usr/share/elasticsearch/config/certs/testnode.crt" \
+    -v "$repo$testnodekey:/usr/share/elasticsearch/config/certs/testnode.key" \
+    -v "$repo$cacrt:/usr/share/elasticsearch/config/certs/ca.crt" \
+    -p 9200:9200 \
+    --network=elastic \
+    --name=elasticsearch \
+    docker.elastic.co/elasticsearch/elasticsearch:6.8.2
+fi

scripts/es-docker.sh

@@ -1,12 +1,38 @@
 #!/bin/bash
 
-exec docker run \
-  --rm \
-  -e "node.attr.testattr=test" \
-  -e "path.repo=/tmp" \
-  -e "repositories.url.allowed_urls=http://snapshot.*" \
-  -e "discovery.type=single-node" \
-  -p 9200:9200 \
-  --network=elastic \
-  --name=elasticsearch \
-  docker.elastic.co/elasticsearch/elasticsearch:6.7.1
+# Images are cached locally, it may be needed
+# to delete an old image and download again
+# the latest snapshot.
+
+# pass `--clean` to reemove the old snapshot
+if [ "$1" == "--clean" ]; then
+  docker rmi $(docker images --format '{{.Repository}}:{{.Tag}}' | grep '8.0.0-SNAPSHOT')
+fi
+
+# Create the 'elastic' network if doesn't exist
+exec docker network ls | grep elastic > /dev/null || docker network create elastic > /dev/null
+
+if [ "$1" == "--detach" ]; then
+  exec docker run \
+    --rm \
+    -e "node.attr.testattr=test" \
+    -e "path.repo=/tmp" \
+    -e "repositories.url.allowed_urls=http://snapshot.*" \
+    -e "discovery.type=single-node" \
+    -p 9200:9200 \
+    --detach \
+    --network=elastic \
+    --name=elasticsearch \
+    docker.elastic.co/elasticsearch/elasticsearch:6.8.2
+else
+  exec docker run \
+    --rm \
+    -e "node.attr.testattr=test" \
+    -e "path.repo=/tmp" \
+    -e "repositories.url.allowed_urls=http://snapshot.*" \
+    -e "discovery.type=single-node" \
+    -p 9200:9200 \
+    --network=elastic \
+    --name=elasticsearch \
+    docker.elastic.co/elasticsearch/elasticsearch:6.8.2
+fi