Buildkite pipeline for integration tests (#1863)

.buildkite/functions/cleanup.sh

@@ -0,0 +1,67 @@
+#!/usr/bin/env bash
+#
+# Shared cleanup routines between different steps
+#
+# Please source .ci/functions/imports.sh as a whole not just this file
+#
+# Version 1.0.0
+# - Initial version after refactor
+
+function cleanup_volume {
+  if [[ "$(docker volume ls -q -f name=$1)" ]]; then
+    echo -e "\033[34;1mINFO:\033[0m Removing volume $1\033[0m"
+    (docker volume rm "$1") || true
+  fi
+}
+function container_running {
+  if [[ "$(docker ps -q -f name=$1)" ]]; then
+    return 0;
+    else return 1;
+  fi
+}
+function cleanup_node {
+  if container_running "$1"; then
+    echo -e "\033[34;1mINFO:\033[0m Removing container $1\033[0m"
+    (docker container rm --force --volumes "$1") || true
+  fi
+  if [[ -n "$1" ]]; then
+    echo -e "\033[34;1mINFO:\033[0m Removing volume $1-${suffix}-data\033[0m"
+    cleanup_volume "$1-${suffix}-data"
+  fi
+}
+function cleanup_network {
+  if [[ "$(docker network ls -q -f name=$1)" ]]; then
+    echo -e "\033[34;1mINFO:\033[0m Removing network $1\033[0m"
+    (docker network rm "$1") || true
+  fi
+}
+
+function cleanup_trap {
+  status=$?
+  set +x
+  if [[ "$DETACH" != "true" ]]; then
+    echo -e "\033[34;1mINFO:\033[0m clean the network if not detached (start and exit)\033[0m"
+    cleanup_all_in_network "$1"
+  fi
+  # status is 0 or SIGINT
+  if [[ "$status" == "0" || "$status" == "130" ]]; then
+    echo -e "\n\033[32;1mSUCCESS run-tests\033[0m"
+    exit 0
+  else
+    echo -e "\n\033[31;1mFAILURE during run-tests\033[0m"
+    exit ${status}
+  fi
+};
+function cleanup_all_in_network {
+
+  if [[ -z "$(docker network ls -q -f name="^$1\$")" ]]; then
+    echo -e "\033[34;1mINFO:\033[0m $1 is already deleted\033[0m"
+    return 0
+  fi
+  containers=$(docker network inspect -f '{{ range $key, $value := .Containers }}{{ printf "%s\n" .Name}}{{ end }}' $1)
+  while read -r container; do
+    cleanup_node "$container"
+  done <<< "$containers"
+  cleanup_network $1
+  echo -e "\033[32;1mSUCCESS:\033[0m Cleaned up and exiting\033[0m"
+};

.buildkite/functions/imports.sh

@@ -0,0 +1,60 @@
+#!/usr/bin/env bash
+#
+# Sets up all the common variables and imports relevant functions
+#
+# Version 1.0.1
+# - Initial version after refactor
+# - Validate STACK_VERSION asap
+
+function require_stack_version() {
+  if [[ -z $STACK_VERSION ]]; then
+    echo -e "\033[31;1mERROR:\033[0m Required environment variable [STACK_VERSION] not set\033[0m"
+    exit 1
+  fi
+}
+
+require_stack_version
+
+if [[ -z $es_node_name ]]; then
+  # only set these once
+  set -euo pipefail
+  export TEST_SUITE=${TEST_SUITE-free}
+  export RUNSCRIPTS=${RUNSCRIPTS-}
+  export DETACH=${DETACH-false}
+  export CLEANUP=${CLEANUP-false}
+
+  export es_node_name=instance
+  export elastic_password=changeme
+  export elasticsearch_image=elasticsearch
+  export elasticsearch_scheme="https"
+  if [[ $TEST_SUITE != "platinum" ]]; then
+    export elasticsearch_scheme="http"
+  fi
+  export elasticsearch_url=${elasticsearch_scheme}://elastic:${elastic_password}@${es_node_name}:9200
+  export external_elasticsearch_url=${elasticsearch_url/$es_node_name/localhost}
+  export elasticsearch_container="${elasticsearch_image}:${STACK_VERSION}"
+
+  export suffix=rest-test
+  export moniker=$(echo "$elasticsearch_container" | tr -C "[:alnum:]" '-')
+  export network_name=${moniker}${suffix}
+
+  export ssl_cert="${script_path}/certs/testnode.crt"
+  export ssl_key="${script_path}/certs/testnode.key"
+  export ssl_ca="${script_path}/certs/ca.crt"
+
+fi
+
+  export script_path=$(dirname $(realpath -s $0))
+  source $script_path/functions/cleanup.sh
+  source $script_path/functions/wait-for-container.sh
+  trap "cleanup_trap ${network_name}" EXIT
+
+
+if [[ "$CLEANUP" == "true" ]]; then
+  cleanup_all_in_network $network_name
+  exit 0
+fi
+
+echo -e "\033[34;1mINFO:\033[0m Creating network $network_name if it does not exist already \033[0m"
+docker network inspect "$network_name" > /dev/null 2>&1 || docker network create "$network_name"
+

.buildkite/functions/wait-for-container.sh

@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+#
+# Exposes a routine scripts can call to wait for a container if that container set up a health command
+#
+# Please source .ci/functions/imports.sh as a whole not just this file
+#
+# Version 1.0.1
+# - Initial version after refactor
+# - Make sure wait_for_contiainer is silent
+
+function wait_for_container {
+  set +x
+  until ! container_running "$1" || (container_running "$1" && [[ "$(docker inspect -f "{{.State.Health.Status}}" ${1})" != "starting" ]]); do
+    echo ""
+    docker inspect -f "{{range .State.Health.Log}}{{.Output}}{{end}}" ${1}
+    echo -e "\033[34;1mINFO:\033[0m waiting for node $1 to be up\033[0m"
+    sleep 2;
+  done;
+
+  # Always show logs if the container is running, this is very useful both on CI as well as while developing
+  if container_running $1; then
+    docker logs $1
+  fi
+
+  if ! container_running $1 || [[ "$(docker inspect -f "{{.State.Health.Status}}" ${1})" != "healthy" ]]; then
+    cleanup_all_in_network $2
+    echo
+    echo -e "\033[31;1mERROR:\033[0m Failed to start $1 in detached mode beyond health checks\033[0m"
+    echo -e "\033[31;1mERROR:\033[0m dumped the docker log before shutting the node down\033[0m"
+    return 1
+  else
+    echo
+    echo -e "\033[32;1mSUCCESS:\033[0m Detached and healthy: ${1} on docker network: ${network_name}\033[0m"
+    return 0
+  fi
+}