From 3acee8b2660e406394cf413f230f711cef47a9c4 Mon Sep 17 00:00:00 2001 From: Tomas Della Vedova Date: Mon, 28 Oct 2019 12:50:44 +0100 Subject: [PATCH] New CI configuration (#996) * Removed old files * Updated README * New CI configuration * Updated run-test script --- .ci/run-elasticsearch.sh | 177 +++++++++++++++++++++++++++++++++++++++ .ci/run-tests | 91 +++++--------------- .ci/test-matrix.yml | 4 + README.md | 2 +- 4 files changed, 202 insertions(+), 72 deletions(-) create mode 100644 .ci/run-elasticsearch.sh diff --git a/.ci/run-elasticsearch.sh b/.ci/run-elasticsearch.sh new file mode 100644 index 000000000..fabc7b549 --- /dev/null +++ b/.ci/run-elasticsearch.sh @@ -0,0 +1,177 @@ +#!/usr/bin/env bash +# +# Launch one or more Elasticsearch nodes via the Docker image, +# to form a cluster suitable for running the REST API tests. +# +# Export the ELASTICSEARCH_VERSION variable, eg. 'elasticsearch:8.0.0-SNAPSHOT'. + +if [[ -z "$ELASTICSEARCH_VERSION" ]]; then + echo -e "\033[31;1mERROR:\033[0m Required environment variable [ELASTICSEARCH_VERSION] not set\033[0m" + exit 1 +fi + +set -euxo pipefail + +moniker=$(echo "$ELASTICSEARCH_VERSION" | tr -C "[:alnum:]" '-') +suffix=rest-test + +NODE_NAME=${NODE_NAME-${moniker}node1} +MASTER_NODE_NAME=${MASTER_NODE_NAME-${NODE_NAME}} +CLUSTER_NAME=${CLUSTER_NAME-${moniker}${suffix}} +HTTP_PORT=${HTTP_PORT-9200} + +ELASTIC_PASSWORD=${ELASTIC_PASSWORD-changeme} +SSL_CERT=${SSL_CERT-"$PWD/certs/testnode.crt"} +SSL_KEY=${SSL_KEY-"$PWD/certs/testnode.key"} +SSL_CA=${SSL_CA-"$PWD/certs/ca.crt"} + +DETACH=${DETACH-false} +CLEANUP=${CLEANUP-false} + +volume_name=${NODE_NAME}-${suffix}-data +network_default=${moniker}${suffix} +NETWORK_NAME=${NETWORK_NAME-"$network_default"} + +set +x + +function cleanup_volume { + if [[ "$(docker volume ls -q -f name=$1)" ]]; then + echo -e "\033[34;1mINFO:\033[0m Removing volume $1\033[0m" + (docker volume rm "$1") || true + fi +} +function cleanup_node { + if [[ "$(docker ps -q -f name=$1)" ]]; then + echo -e "\033[34;1mINFO:\033[0m Removing container $1\033[0m" + (docker container rm --force --volumes "$1") || true + cleanup_volume "$1-${suffix}-data" + fi +} +function cleanup_network { + if [[ "$(docker network ls -q -f name=$1)" ]]; then + echo -e "\033[34;1mINFO:\033[0m Removing network $1\033[0m" + (docker network rm "$1") || true + fi +} + +function cleanup { + if [[ "$DETACH" != "true" ]] || [[ "$1" == "1" ]]; then + echo -e "\033[34;1mINFO:\033[0m clean the node and volume on startup (1) OR on exit if not detached\033[0m" + cleanup_node "$NODE_NAME" + fi + if [[ "$DETACH" != "true" ]]; then + echo -e "\033[34;1mINFO:\033[0m clean the network if not detached (start and exit)\033[0m" + cleanup_network "$NETWORK_NAME" + fi +}; +trap "cleanup 0" EXIT + +if [[ "$CLEANUP" == "true" ]]; then + trap - EXIT + if [[ -z "$(docker network ls -q -f name=${NETWORK_NAME})" ]]; then + echo -e "\033[34;1mINFO:\033[0m $NETWORK_NAME is already deleted\033[0m" + exit 0 + fi + containers=$(docker network inspect -f '{{ range $key, $value := .Containers }}{{ printf "%s\n" .Name}}{{ end }}' ${NETWORK_NAME}) + while read -r container; do + cleanup_node "$container" + done <<< "$containers" + cleanup_network "$NETWORK_NAME" + echo -e "\033[32;1mSUCCESS:\033[0m Cleaned up and exiting\033[0m" + exit 0 +fi + +echo -e "\033[34;1mINFO:\033[0m Making sure previous run leftover infrastructure is removed \033[0m" +cleanup 1 + +echo -e "\033[34;1mINFO:\033[0m Creating network $NETWORK_NAME if it does not exist already \033[0m" +docker network inspect "$NETWORK_NAME" > /dev/null 2>&1 || docker network create "$NETWORK_NAME" + +environment=($(cat <<-END + --env node.name=$NODE_NAME + --env cluster.name=$CLUSTER_NAME + --env cluster.initial_master_nodes=$MASTER_NODE_NAME + --env discovery.seed_hosts=$MASTER_NODE_NAME + --env cluster.routing.allocation.disk.threshold_enabled=false + --env bootstrap.memory_lock=true + --env node.attr.testattr=test + --env path.repo=/tmp + --env repositories.url.allowed_urls=http://snapshot.test* +END +)) + +volumes=($(cat <<-END + --volume $volume_name:/usr/share/elasticsearch/data +END +)) + +if [[ "$ELASTICSEARCH_VERSION" != *oss* ]]; then + environment+=($(cat <<-END + --env ELASTIC_PASSWORD=$ELASTIC_PASSWORD + --env xpack.license.self_generated.type=trial + --env xpack.security.enabled=true + --env xpack.security.http.ssl.enabled=true + --env xpack.security.http.ssl.verification_mode=certificate + --env xpack.security.http.ssl.key=certs/testnode.key + --env xpack.security.http.ssl.certificate=certs/testnode.crt + --env xpack.security.http.ssl.certificate_authorities=certs/ca.crt + --env xpack.security.transport.ssl.enabled=true + --env xpack.security.transport.ssl.key=certs/testnode.key + --env xpack.security.transport.ssl.certificate=certs/testnode.crt + --env xpack.security.transport.ssl.certificate_authorities=certs/ca.crt +END +)) + volumes+=($(cat <<-END + --volume $SSL_CERT:/usr/share/elasticsearch/config/certs/testnode.crt + --volume $SSL_KEY:/usr/share/elasticsearch/config/certs/testnode.key + --volume $SSL_CA:/usr/share/elasticsearch/config/certs/ca.crt +END +)) +fi + +url="http://$NODE_NAME" +if [[ "$ELASTICSEARCH_VERSION" != *oss* ]]; then + url="https://elastic:$ELASTIC_PASSWORD@$NODE_NAME" +fi + +echo -e "\033[34;1mINFO:\033[0m Starting container $NODE_NAME \033[0m" +set -x +docker run \ + --name "$NODE_NAME" \ + --network "$NETWORK_NAME" \ + --env ES_JAVA_OPTS=-"Xms1g -Xmx1g" \ + "${environment[@]}" \ + "${volumes[@]}" \ + --publish "$HTTP_PORT":9200 \ + --ulimit nofile=65536:65536 \ + --ulimit memlock=-1:-1 \ + --detach="$DETACH" \ + --health-cmd="curl --silent --insecure --fail $url:9200/_cluster/health || exit 1" \ + --health-interval=2s \ + --health-retries=20 \ + --health-timeout=2s \ + --rm \ + docker.elastic.co/elasticsearch/"$ELASTICSEARCH_VERSION"; +set +x + +if [[ "$DETACH" == "true" ]]; then + until [[ "$(docker inspect -f "{{.State.Health.Status}}" ${NODE_NAME})" != "starting" ]]; do + sleep 2; + echo "" + echo -e "\033[34;1mINFO:\033[0m waiting for node $NODE_NAME to be up\033[0m" + done; + # Always show the node getting started logs, this is very useful both on CI as well as while developing + docker logs "$NODE_NAME" + if [[ "$(docker inspect -f "{{.State.Health.Status}}" ${NODE_NAME})" != "healthy" ]]; then + cleanup 1 + echo + echo -e "\033[31;1mERROR:\033[0m Failed to start ${ELASTICSEARCH_VERSION} in detached mode beyond health checks\033[0m" + echo -e "\033[31;1mERROR:\033[0m dumped the docker log before shutting the node down\033[0m" + exit 1 + else + echo + echo -e "\033[32;1mSUCCESS:\033[0m Detached and healthy: ${NODE_NAME} on docker network: ${NETWORK_NAME}\033[0m" + echo -e "\033[32;1mSUCCESS:\033[0m Running on: ${url/$NODE_NAME/localhost}:${HTTP_PORT}\033[0m" + exit 0 + fi +fi diff --git a/.ci/run-tests b/.ci/run-tests index 3a0394a0c..62d2a56e9 100755 --- a/.ci/run-tests +++ b/.ci/run-tests @@ -8,6 +8,7 @@ # # - $ELASTICSEARCH_VERSION # - $NODE_JS_VERSION +# - $TEST_SUITE # set -eo pipefail @@ -18,93 +19,41 @@ export CODECOV_TOKEN=$(vault read -field=token secret/clients-ci/elasticsearch-j unset VAULT_ROLE_ID VAULT_SECRET_ID VAULT_TOKEN set -x -function cleanup { - docker container rm --force --volumes elasticsearch-oss > /dev/null 2>&1 || true - docker container rm --force --volumes elasticsearch-platinum > /dev/null 2>&1 || true - docker container rm --force --volumes elasticsearch-js-oss > /dev/null 2>&1 || true - docker container rm --force --volumes elasticsearch-js-platinum > /dev/null 2>&1 || true - docker network rm esnet-oss > /dev/null - docker network rm esnet-platinum > /dev/null -} - -trap cleanup EXIT - -# create network and volume -docker network create esnet-oss -docker network create esnet-platinum - -# create client image docker build \ --file .ci/Dockerfile \ --tag elastic/elasticsearch-js \ --build-arg NODE_JS_VERSION=${NODE_JS_VERSION} \ . -# run elasticsearch oss -docker run \ - --rm \ - --env "node.attr.testattr=test" \ - --env "path.repo=/tmp" \ - --env "repositories.url.allowed_urls=http://snapshot.*" \ - --env "discovery.type=single-node" \ - --network=esnet-oss \ - --name=elasticsearch-oss \ - --detach \ - docker.elastic.co/elasticsearch/elasticsearch-oss:${ELASTICSEARCH_VERSION} - -# run elasticsearch platinum +NODE_NAME="es1" repo=$(pwd) testnodecrt="/.ci/certs/testnode.crt" testnodekey="/.ci/certs/testnode.key" cacrt="/.ci/certs/ca.crt" -docker run \ - --rm \ - --env "node.attr.testattr=test" \ - --env "path.repo=/tmp" \ - --env "repositories.url.allowed_urls=http://snapshot.*" \ - --env "discovery.type=single-node" \ - --env "ES_JAVA_OPTS=-Xms1g -Xmx1g" \ - --env "ELASTIC_PASSWORD=changeme" \ - --env "xpack.security.enabled=true" \ - --env "xpack.license.self_generated.type=trial" \ - --env "xpack.security.http.ssl.enabled=true" \ - --env "xpack.security.http.ssl.verification_mode=certificate" \ - --env "xpack.security.http.ssl.key=certs/testnode.key" \ - --env "xpack.security.http.ssl.certificate=certs/testnode.crt" \ - --env "xpack.security.http.ssl.certificate_authorities=certs/ca.crt" \ - --env "xpack.security.transport.ssl.enabled=true" \ - --env "xpack.security.transport.ssl.key=certs/testnode.key" \ - --env "xpack.security.transport.ssl.certificate=certs/testnode.crt" \ - --env "xpack.security.transport.ssl.certificate_authorities=certs/ca.crt" \ - --volume "$repo$testnodecrt:/usr/share/elasticsearch/config/certs/testnode.crt" \ - --volume "$repo$testnodekey:/usr/share/elasticsearch/config/certs/testnode.key" \ - --volume "$repo$cacrt:/usr/share/elasticsearch/config/certs/ca.crt" \ - --network=esnet-platinum \ - --name=elasticsearch-platinum \ - --detach \ - docker.elastic.co/elasticsearch/elasticsearch:${ELASTICSEARCH_VERSION} +elasticsearch_image="elasticsearch" +elasticsearch_url="https://elastic:changeme@${NODE_NAME}:9200" +if [[ $TEST_SUITE != "xpack" ]]; then + elasticsearch_image="elasticsearch-oss" + elasticsearch_url="http://${NODE_NAME}:9200" +fi + +ELASTICSEARCH_VERSION="${elasticsearch_image}:${ELASTICSEARCH_VERSION}" \ + NODE_NAME="${NODE_NAME}" \ + NETWORK_NAME="esnet" \ + DETACH=true \ + SSL_CERT="${repo}${testnodecrt}" \ + SSL_KEY="${repo}${testnodekey}" \ + SSL_CA="${repo}${cacrt}" \ + bash .ci/run-elasticsearch.sh -# run the client unit and oss integration test docker run \ - --network=esnet-oss \ - --env "TEST_ES_SERVER=http://elasticsearch-oss:9200" \ + --network=esnet \ + --env "TEST_ES_SERVER=${elasticsearch_url}" \ --env "CODECOV_TOKEN" \ --volume $repo:/usr/src/app \ --volume /usr/src/app/node_modules \ - --name elasticsearch-js-oss \ + --name elasticsearch-js \ --rm \ elastic/elasticsearch-js \ npm run ci - -# run the client platinium integration test -docker run \ - --network=esnet-platinum \ - --env "TEST_ES_SERVER=https://elastic:changeme@elasticsearch-platinum:9200" \ - --volume $repo:/usr/src/app \ - --volume /usr/src/app/node_modules \ - --name elasticsearch-js-platinum \ - --rm \ - elastic/elasticsearch-js \ - npm run test:integration - diff --git a/.ci/test-matrix.yml b/.ci/test-matrix.yml index d93ac45e1..53521689d 100644 --- a/.ci/test-matrix.yml +++ b/.ci/test-matrix.yml @@ -7,4 +7,8 @@ NODE_JS_VERSION: - 10 - 8 +TEST_SUITE: + - oss + - xpack + exclude: ~ diff --git a/README.md b/README.md index b5143ac83..14875c020 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ # Elasticsearch Node.js client -[![js-standard-style](https://img.shields.io/badge/code%20style-standard-brightgreen.svg?style=flat)](http://standardjs.com/) [![Build Status](https://clients-ci.elastic.co/job/elastic+elasticsearch-js+master/badge/icon)](https://clients-ci.elastic.co/job/elastic+elasticsearch-js+master/) [![codecov](https://codecov.io/gh/elastic/elasticsearch-js/branch/master/graph/badge.svg)](https://codecov.io/gh/elastic/elasticsearch-js) [![NPM downloads](https://img.shields.io/npm/dm/@elastic/elasticsearch.svg?style=flat)](https://www.npmjs.com/package/@elastic/elasticsearch) +[![js-standard-style](https://img.shields.io/badge/code%20style-standard-brightgreen.svg?style=flat)](http://standardjs.com/) [![Build Status](https://clients-ci.elastic.co/buildStatus/icon?job=elastic%2Belasticsearch-js%2Bmaster)](https://clients-ci.elastic.co/view/Javascript/job/elastic+elasticsearch-js+master/) [![codecov](https://codecov.io/gh/elastic/elasticsearch-js/branch/master/graph/badge.svg)](https://codecov.io/gh/elastic/elasticsearch-js) [![NPM downloads](https://img.shields.io/npm/dm/@elastic/elasticsearch.svg?style=flat)](https://www.npmjs.com/package/@elastic/elasticsearch) The official Node.js client for Elasticsearch.