New CI configuration (#996)

* Removed old files

* Updated README

* New CI configuration

* Updated run-test script

.ci/run-elasticsearch.sh

@@ -0,0 +1,177 @@
+#!/usr/bin/env bash
+#
+# Launch one or more Elasticsearch nodes via the Docker image,
+# to form a cluster suitable for running the REST API tests.
+#
+# Export the ELASTICSEARCH_VERSION variable, eg. 'elasticsearch:8.0.0-SNAPSHOT'.
+
+if [[ -z "$ELASTICSEARCH_VERSION" ]]; then
+  echo -e "\033[31;1mERROR:\033[0m Required environment variable [ELASTICSEARCH_VERSION] not set\033[0m"
+  exit 1
+fi
+
+set -euxo pipefail
+
+moniker=$(echo "$ELASTICSEARCH_VERSION" | tr -C "[:alnum:]" '-')
+suffix=rest-test
+
+NODE_NAME=${NODE_NAME-${moniker}node1}
+MASTER_NODE_NAME=${MASTER_NODE_NAME-${NODE_NAME}}
+CLUSTER_NAME=${CLUSTER_NAME-${moniker}${suffix}}
+HTTP_PORT=${HTTP_PORT-9200}
+
+ELASTIC_PASSWORD=${ELASTIC_PASSWORD-changeme}
+SSL_CERT=${SSL_CERT-"$PWD/certs/testnode.crt"}
+SSL_KEY=${SSL_KEY-"$PWD/certs/testnode.key"}
+SSL_CA=${SSL_CA-"$PWD/certs/ca.crt"}
+
+DETACH=${DETACH-false}
+CLEANUP=${CLEANUP-false}
+
+volume_name=${NODE_NAME}-${suffix}-data
+network_default=${moniker}${suffix}
+NETWORK_NAME=${NETWORK_NAME-"$network_default"}
+
+set +x
+
+function cleanup_volume {
+  if [[ "$(docker volume ls -q -f name=$1)" ]]; then
+    echo -e "\033[34;1mINFO:\033[0m Removing volume $1\033[0m"
+    (docker volume rm "$1") || true
+  fi
+}
+function cleanup_node {
+  if [[ "$(docker ps -q -f name=$1)" ]]; then
+    echo -e "\033[34;1mINFO:\033[0m Removing container $1\033[0m"
+    (docker container rm --force --volumes "$1") || true
+    cleanup_volume "$1-${suffix}-data"
+  fi
+}
+function cleanup_network {
+  if [[ "$(docker network ls -q -f name=$1)" ]]; then
+    echo -e "\033[34;1mINFO:\033[0m Removing network $1\033[0m"
+    (docker network rm "$1") || true
+  fi
+}
+
+function cleanup {
+  if [[ "$DETACH" != "true" ]] || [[ "$1" == "1" ]]; then
+    echo -e "\033[34;1mINFO:\033[0m clean the node and volume on startup (1) OR on exit if not detached\033[0m"
+    cleanup_node "$NODE_NAME"
+  fi
+  if [[ "$DETACH" != "true" ]]; then
+    echo -e "\033[34;1mINFO:\033[0m clean the network if not detached (start and exit)\033[0m"
+    cleanup_network "$NETWORK_NAME"
+  fi
+};
+trap "cleanup 0" EXIT
+
+if [[ "$CLEANUP" == "true" ]]; then
+  trap - EXIT
+  if [[ -z "$(docker network ls -q -f name=${NETWORK_NAME})" ]]; then
+    echo -e "\033[34;1mINFO:\033[0m $NETWORK_NAME is already deleted\033[0m"
+    exit 0
+  fi
+  containers=$(docker network inspect -f '{{ range $key, $value := .Containers }}{{ printf "%s\n" .Name}}{{ end }}' ${NETWORK_NAME})
+  while read -r container; do
+    cleanup_node "$container"
+  done <<< "$containers"
+  cleanup_network "$NETWORK_NAME"
+  echo -e "\033[32;1mSUCCESS:\033[0m Cleaned up and exiting\033[0m"
+  exit 0
+fi
+
+echo -e "\033[34;1mINFO:\033[0m Making sure previous run leftover infrastructure is removed \033[0m"
+cleanup 1
+
+echo -e "\033[34;1mINFO:\033[0m Creating network $NETWORK_NAME if it does not exist already \033[0m"
+docker network inspect "$NETWORK_NAME" > /dev/null 2>&1 || docker network create "$NETWORK_NAME"
+
+environment=($(cat <<-END
+  --env node.name=$NODE_NAME
+  --env cluster.name=$CLUSTER_NAME
+  --env cluster.initial_master_nodes=$MASTER_NODE_NAME
+  --env discovery.seed_hosts=$MASTER_NODE_NAME
+  --env cluster.routing.allocation.disk.threshold_enabled=false
+  --env bootstrap.memory_lock=true
+  --env node.attr.testattr=test
+  --env path.repo=/tmp
+  --env repositories.url.allowed_urls=http://snapshot.test*
+END
+))
+
+volumes=($(cat <<-END
+  --volume $volume_name:/usr/share/elasticsearch/data
+END
+))
+
+if [[ "$ELASTICSEARCH_VERSION" != *oss* ]]; then
+  environment+=($(cat <<-END
+    --env ELASTIC_PASSWORD=$ELASTIC_PASSWORD
+    --env xpack.license.self_generated.type=trial
+    --env xpack.security.enabled=true
+    --env xpack.security.http.ssl.enabled=true
+    --env xpack.security.http.ssl.verification_mode=certificate
+    --env xpack.security.http.ssl.key=certs/testnode.key
+    --env xpack.security.http.ssl.certificate=certs/testnode.crt
+    --env xpack.security.http.ssl.certificate_authorities=certs/ca.crt
+    --env xpack.security.transport.ssl.enabled=true
+    --env xpack.security.transport.ssl.key=certs/testnode.key
+    --env xpack.security.transport.ssl.certificate=certs/testnode.crt
+    --env xpack.security.transport.ssl.certificate_authorities=certs/ca.crt
+END
+))
+  volumes+=($(cat <<-END
+    --volume $SSL_CERT:/usr/share/elasticsearch/config/certs/testnode.crt
+    --volume $SSL_KEY:/usr/share/elasticsearch/config/certs/testnode.key
+    --volume $SSL_CA:/usr/share/elasticsearch/config/certs/ca.crt
+END
+))
+fi
+
+url="http://$NODE_NAME"
+if [[ "$ELASTICSEARCH_VERSION" != *oss* ]]; then
+  url="https://elastic:$ELASTIC_PASSWORD@$NODE_NAME"
+fi
+
+echo -e "\033[34;1mINFO:\033[0m Starting container $NODE_NAME \033[0m"
+set -x
+docker run \
+  --name "$NODE_NAME" \
+  --network "$NETWORK_NAME" \
+  --env ES_JAVA_OPTS=-"Xms1g -Xmx1g" \
+  "${environment[@]}" \
+  "${volumes[@]}" \
+  --publish "$HTTP_PORT":9200 \
+  --ulimit nofile=65536:65536 \
+  --ulimit memlock=-1:-1 \
+  --detach="$DETACH" \
+  --health-cmd="curl --silent --insecure --fail $url:9200/_cluster/health || exit 1" \
+  --health-interval=2s \
+  --health-retries=20 \
+  --health-timeout=2s \
+  --rm \
+  docker.elastic.co/elasticsearch/"$ELASTICSEARCH_VERSION";
+set +x
+
+if [[ "$DETACH" == "true" ]]; then
+  until [[ "$(docker inspect -f "{{.State.Health.Status}}" ${NODE_NAME})" != "starting" ]]; do
+    sleep 2;
+    echo ""
+    echo -e "\033[34;1mINFO:\033[0m waiting for node $NODE_NAME to be up\033[0m"
+  done;
+  # Always show the node getting started logs, this is very useful both on CI as well as while developing
+  docker logs "$NODE_NAME"
+  if [[ "$(docker inspect -f "{{.State.Health.Status}}" ${NODE_NAME})" != "healthy" ]]; then
+    cleanup 1
+    echo
+    echo -e "\033[31;1mERROR:\033[0m Failed to start ${ELASTICSEARCH_VERSION} in detached mode beyond health checks\033[0m"
+    echo -e "\033[31;1mERROR:\033[0m dumped the docker log before shutting the node down\033[0m"
+    exit 1
+  else
+    echo
+    echo -e "\033[32;1mSUCCESS:\033[0m Detached and healthy: ${NODE_NAME} on docker network: ${NETWORK_NAME}\033[0m"
+    echo -e "\033[32;1mSUCCESS:\033[0m Running on: ${url/$NODE_NAME/localhost}:${HTTP_PORT}\033[0m"
+    exit 0
+  fi
+fi

.ci/run-tests

@@ -8,6 +8,7 @@
 #
 #   - $ELASTICSEARCH_VERSION
 #   - $NODE_JS_VERSION
+#   - $TEST_SUITE
 #
 
 set -eo pipefail
@@ -18,48 +19,41 @@ export CODECOV_TOKEN=$(vault read -field=token secret/clients-ci/elasticsearch-j
 unset VAULT_ROLE_ID VAULT_SECRET_ID VAULT_TOKEN
 set -x
 
-function cleanup {
-  docker container rm --force --volumes elasticsearch-oss > /dev/null 2>&1 || true
-  docker container rm --force --volumes elasticsearch-js-oss > /dev/null 2>&1 || true
-  docker network rm esnet-oss > /dev/null
-}
-
-trap cleanup EXIT
-
-# create network and volume
-docker network create esnet-oss
-
-# create client image
 docker build \
   --file .ci/Dockerfile \
   --tag elastic/elasticsearch-js \
   --build-arg NODE_JS_VERSION=${NODE_JS_VERSION} \
   .
 
-# run elasticsearch oss
+NODE_NAME="es1"
-docker run \
+repo=$(pwd)
-  --rm \
+testnodecrt="/.ci/certs/testnode.crt"
-  --env "node.attr.testattr=test" \
+testnodekey="/.ci/certs/testnode.key"
-  --env "path.repo=/tmp" \
+cacrt="/.ci/certs/ca.crt"
-  --env "repositories.url.allowed_urls=http://snapshot.*" \
+
-  --env "discovery.zen.ping.unicast.hosts=elasticsearch"  \
+elasticsearch_image="elasticsearch"
-  --env "xpack.security.enabled=false" \
+elasticsearch_url="https://elastic:changeme@${NODE_NAME}:9200"
-  --env "xpack.monitoring.enabled=false" \
+if [[ $TEST_SUITE != "xpack" ]]; then
-  --env "xpack.ml.enabled=false" \
+  elasticsearch_image="elasticsearch-oss"
-  --env ES_JAVA_OPTS="-Xms1g -Xmx1g" \
+  elasticsearch_url="http://${NODE_NAME}:9200"
-  --network=esnet-oss \
+fi
-  --name=elasticsearch-oss \
+
-  --detach \
+ELASTICSEARCH_VERSION="${elasticsearch_image}:${ELASTICSEARCH_VERSION}" \
-  docker.elastic.co/elasticsearch/elasticsearch:${ELASTICSEARCH_VERSION}
+  NODE_NAME="${NODE_NAME}" \
+  NETWORK_NAME="esnet" \
+  DETACH=true \
+  SSL_CERT="${repo}${testnodecrt}" \
+  SSL_KEY="${repo}${testnodekey}" \
+  SSL_CA="${repo}${cacrt}" \
+  bash .ci/run-elasticsearch.sh
 
-# run the client unit and oss integration test
 docker run \
-  --network=esnet-oss \
+  --network=esnet \
-  --env "TEST_ES_SERVER=http://elasticsearch-oss:9200" \
+  --env "TEST_ES_SERVER=${elasticsearch_url}" \
   --env "CODECOV_TOKEN" \
   --volume $(pwd):/usr/src/app \
   --volume /usr/src/app/node_modules \
-  --name elasticsearch-js-oss \
+  --name elasticsearch-js \
   --rm \
   elastic/elasticsearch-js \
   npm run ci

.ci/test-matrix.yml

@@ -7,4 +7,7 @@ NODE_JS_VERSION:
   - 10
   - 8
 
+TEST_SUITE:
+  - oss
+
 exclude: ~

README.md

@@ -2,7 +2,7 @@
 
 # Elasticsearch Node.js client
 
-[![js-standard-style](https://img.shields.io/badge/code%20style-standard-brightgreen.svg?style=flat)](http://standardjs.com/)  [![Build Status](https://clients-ci.elastic.co/job/elastic+elasticsearch-js+master/badge/icon)](https://clients-ci.elastic.co/job/elastic+elasticsearch-js+master/)  [![codecov](https://codecov.io/gh/elastic/elasticsearch-js/branch/master/graph/badge.svg)](https://codecov.io/gh/elastic/elasticsearch-js)  [![NPM downloads](https://img.shields.io/npm/dm/@elastic/elasticsearch.svg?style=flat)](https://www.npmjs.com/package/@elastic/elasticsearch)
+[![js-standard-style](https://img.shields.io/badge/code%20style-standard-brightgreen.svg?style=flat)](http://standardjs.com/)  [![Build Status](https://clients-ci.elastic.co/buildStatus/icon?job=elastic%2Belasticsearch-js%2Bmaster)](https://clients-ci.elastic.co/view/Javascript/job/elastic+elasticsearch-js+master/)  [![codecov](https://codecov.io/gh/elastic/elasticsearch-js/branch/master/graph/badge.svg)](https://codecov.io/gh/elastic/elasticsearch-js)  [![NPM downloads](https://img.shields.io/npm/dm/@elastic/elasticsearch.svg?style=flat)](https://www.npmjs.com/package/@elastic/elasticsearch)
 
 The official Node.js client for Elasticsearch.