diff --git a/.ci/run-elasticsearch.sh b/.ci/run-elasticsearch.sh
index 3f4e2f1da..2f360ab4f 100755
--- a/.ci/run-elasticsearch.sh
+++ b/.ci/run-elasticsearch.sh
@@ -22,12 +22,12 @@
# - Use https only when TEST_SUITE is "platinum", when "free" use http
# - Set xpack.security.enabled=false for "free" and xpack.security.enabled=true for "platinum"
-script_path=$(dirname $(realpath -s $0))
-source $script_path/functions/imports.sh
+script_path=$(dirname "$(realpath -s "$0")")
+source "$script_path/functions/imports.sh"
set -euo pipefail
-echo -e "\033[34;1mINFO:\033[0m Take down node if called twice with the same arguments (DETACH=true) or on seperate terminals \033[0m"
-cleanup_node $es_node_name
+echo -e "\033[34;1mINFO:\033[0m Take down node if called twice with the same arguments (DETACH=true) or on separate terminals \033[0m"
+cleanup_node "$es_node_name"
master_node_name=${es_node_name}
cluster_name=${moniker}${suffix}
diff --git a/Makefile b/Makefile
new file mode 100644
index 000000000..a374c22db
--- /dev/null
+++ b/Makefile
@@ -0,0 +1,12 @@
+.PHONY: integration-setup
+integration-setup: integration-cleanup
+ DETACH=true .ci/run-elasticsearch.sh
+
+.PHONY: integration-cleanup
+integration-cleanup:
+ docker stop instance || true
+ docker volume rm instance-rest-test-data || true
+
+.PHONY: integration
+integration: integration-setup
+ npm run test:integration
diff --git a/scripts/utils/generateApis.js b/scripts/utils/generateApis.js
index 53dc1abed..a1dddd063 100644
--- a/scripts/utils/generateApis.js
+++ b/scripts/utils/generateApis.js
@@ -228,7 +228,7 @@ function generateSingleApi (version, spec, common) {
${genUrlValidation(paths, api)}
- let { ${genQueryBlacklist(false)}, ...querystring } = params
+ let { ${genQueryDenylist(false)}, ...querystring } = params
querystring = snakeCaseKeys(acceptedQuerystring, snakeCase, querystring)
let path = ''
@@ -316,20 +316,20 @@ function generateSingleApi (version, spec, common) {
}, {})
}
- function genQueryBlacklist (addQuotes = true) {
+ function genQueryDenylist (addQuotes = true) {
const toCamelCase = str => {
return str[0] === '_'
? '_' + str.slice(1).replace(/_([a-z])/g, k => k[1].toUpperCase())
: str.replace(/_([a-z])/g, k => k[1].toUpperCase())
}
- const blacklist = ['method', 'body']
+ const denylist = ['method', 'body']
parts.forEach(p => {
const camelStr = toCamelCase(p)
- if (camelStr !== p) blacklist.push(`${camelStr}`)
- blacklist.push(`${p}`)
+ if (camelStr !== p) denylist.push(`${camelStr}`)
+ denylist.push(`${p}`)
})
- return addQuotes ? blacklist.map(q => `'${q}'`) : blacklist
+ return addQuotes ? denylist.map(q => `'${q}'`) : denylist
}
function buildPath () {
diff --git a/test/integration/README.md b/test/integration/README.md
index 0861dd8b9..a52ae2e54 100644
--- a/test/integration/README.md
+++ b/test/integration/README.md
@@ -5,8 +5,8 @@
Yes.
## Background
-Elasticsearch offers its entire API via HTTP REST endpoints. You can find the whole API specification for every version [here](https://github.com/elastic/elasticsearch/tree/master/rest-api-spec/src/main/resources/rest-api-spec/api).
-To support different languages at the same time, the Elasticsearch team decided to provide a [YAML specification](https://github.com/elastic/elasticsearch/tree/master/rest-api-spec/src/main/resources/rest-api-spec/test) to test every endpoint, body, headers, warning, error and so on.
+Elasticsearch offers its entire API via HTTP REST endpoints. You can find the whole API specification for every version [here](https://github.com/elastic/elasticsearch/tree/main/rest-api-spec/src/main/resources/rest-api-spec/api).
+To support different languages at the same time, the Elasticsearch team decided to provide a [YAML specification](https://github.com/elastic/elasticsearch/tree/main/rest-api-spec/src/main/resources/rest-api-spec/test) to test every endpoint, body, headers, warning, error and so on.
This testing suite uses that specification to generate the test for the specified version of Elasticsearch on the fly.
## Run
@@ -20,20 +20,45 @@ Once the Elasticsearch repository has been cloned, the testing suite will connec
The specification does not allow the test to be run in parallel, so it might take a while to run the entire testing suite; on my machine, `MacBookPro15,2 core i7 2.7GHz 16GB of RAM` it takes around four minutes.
+### Running locally
+
+If you want to run the integration tests on your development machine, you must have an Elasticsearch instance running first.
+A local instance can be spun up in a Docker container by running the [`.ci/run-elasticsearch.sh`](/.ci/run-elasticsearch.sh) script.
+This is the same script CI jobs use to run Elasticsearch for integration tests, so your results should be relatively consistent.
+
+To simplify the process of starting a container, testing, and cleaning up the container, you can run the `make integration` target:
+
+```sh
+# set some parameters
+export STACK_VERSION=8.7.0
+export TEST_SUITE=free # can be `free` or `platinum`
+make integration
+```
+
+If Elasticsearch doesn't come up, run `make integration-cleanup` and then `DETACH=false .ci/run-elasticsearch.sh` manually to read the startup logs.
+
+If you get an error about `vm.max_map_count` being too low, run `sudo sysctl -w vm.max_map_count=262144` to update the setting until the next reboot, or `sudo sysctl -w vm.max_map_count=262144 | sudo tee -a /etc/sysctl.conf` to update the setting permanently.
+
### Exit on the first failure
-Bu default the suite will run all the test, even if one assertion has failed. If you want to stop the test at the first failure, use the bailout option:
+
+By default the suite will run all the tests, even if one assertion has failed. If you want to stop the test at the first failure, use the bailout option:
+
```sh
npm run test:integration -- --bail
```
### Calculate the code coverage
+
If you want to calculate the code coverage just run the testing suite with the following parameters, once the test ends, it will open a browser window with the results.
+
```sh
npm run test:integration -- --cov --coverage-report=html
```
## How does this thing work?
+
At first sight, it might seem complicated, but once you understand what the moving parts are, it's quite easy.
+
1. Connects to the given Elasticsearch instance
1. Gets the ES version and build hash
1. Checkout to the given hash (and clone the repository if it is not present)
@@ -46,7 +71,4 @@ At first sight, it might seem complicated, but once you understand what the movi
Inside the `index.js` file, you will find the connection, cloning, reading and parsing part of the test, while inside the `test-runner.js` file you will find the function to handle the assertions. Inside `test-runner.js`, we use a [queue](https://github.com/delvedor/workq) to be sure that everything is run in the correct order.
-Checkout the [rest-api-spec readme](https://github.com/elastic/elasticsearch/blob/master/rest-api-spec/src/main/resources/rest-api-spec/test/README.asciidoc) if you want to know more about how the assertions work.
-
-#### Why are we running the test with the `--harmony` flag?
-Because on Node v6 the regex lookbehinds are not supported.
+Check out the [rest-api-spec readme](https://github.com/elastic/elasticsearch/blob/main/rest-api-spec/src/main/resources/rest-api-spec/test/README.asciidoc) if you want to know more about how the assertions work.
diff --git a/test/integration/index.js b/test/integration/index.js
index c794beb6c..5c4addcb7 100644
--- a/test/integration/index.js
+++ b/test/integration/index.js
@@ -27,6 +27,7 @@ process.on('unhandledRejection', function (err) {
const { writeFileSync, readFileSync, readdirSync, statSync } = require('fs')
const { join, sep } = require('path')
const yaml = require('js-yaml')
+const minimist = require('minimist')
const ms = require('ms')
const { Client } = require('../../index')
const { kProductCheck } = require('@elastic/transport/lib/symbols')
@@ -42,12 +43,24 @@ const MAX_API_TIME = 1000 * 90
const MAX_FILE_TIME = 1000 * 30
const MAX_TEST_TIME = 1000 * 3
+const options = minimist(process.argv.slice(2), {
+ boolean: ['bail']
+})
+
const freeSkips = {
// not supported yet
'/free/cluster.desired_nodes/10_basic.yml': ['*'],
+
+ // Cannot find methods on `Internal` object
+ '/free/cluster.desired_balance/10_basic.yml': ['*'],
+ '/free/cluster.desired_nodes/20_dry_run.yml': ['*'],
+ '/free/cluster.prevalidate_node_removal/10_basic.yml': ['*'],
+
'/free/health/30_feature.yml': ['*'],
'/free/health/40_useractions.yml': ['*'],
- // the v8 client never sends the scroll_id in querystgring,
+ '/free/health/40_diagnosis.yml': ['Diagnosis'],
+
+ // the v8 client never sends the scroll_id in querystring,
// the way the test is structured causes a security exception
'free/scroll/10_basic.yml': ['Body params override query string'],
'free/scroll/11_clear.yml': [
@@ -56,80 +69,99 @@ const freeSkips = {
],
'free/cat.allocation/10_basic.yml': ['*'],
'free/cat.snapshots/10_basic.yml': ['Test cat snapshots output'],
+
// TODO: remove this once 'arbitrary_key' is implemented
// https://github.com/elastic/elasticsearch/pull/41492
'indices.split/30_copy_settings.yml': ['*'],
'indices.stats/50_disk_usage.yml': ['Disk usage stats'],
'indices.stats/60_field_usage.yml': ['Field usage stats'],
+
// skipping because we are booting ES with `discovery.type=single-node`
// and this test will fail because of this configuration
'nodes.stats/30_discovery.yml': ['*'],
+
// the expected error is returning a 503,
// which triggers a retry and the node to be marked as dead
'search.aggregation/240_max_buckets.yml': ['*'],
+
// long values and json do not play nicely together
'search.aggregation/40_range.yml': ['Min and max long range bounds'],
+
// the yaml runner assumes that null means "does not exists",
// while null is a valid json value, so the check will fail
'search/320_disallow_queries.yml': ['Test disallow expensive queries'],
- 'free/tsdb/90_unsupported_operations.yml': ['noop update']
+ 'free/tsdb/90_unsupported_operations.yml': ['noop update'],
}
-const platinumBlackList = {
+
+const platinumDenyList = {
'api_key/10_basic.yml': ['Test get api key'],
'api_key/20_query.yml': ['*'],
'api_key/11_invalidation.yml': ['Test invalidate api key by realm name'],
'analytics/histogram.yml': ['Histogram requires values in increasing order'],
+
// this two test cases are broken, we should
// return on those in the future.
'analytics/top_metrics.yml': [
'sort by keyword field fails',
'sort by string script fails'
],
+
'cat.aliases/10_basic.yml': ['Empty cluster'],
'index/10_with_id.yml': ['Index with ID'],
'indices.get_alias/10_basic.yml': ['Get alias against closed indices'],
'indices.get_alias/20_empty.yml': ['Check empty aliases when getting all aliases via /_alias'],
'text_structure/find_structure.yml': ['*'],
+
// https://github.com/elastic/elasticsearch/pull/39400
'ml/jobs_crud.yml': ['Test put job with id that is already taken'],
+
// object keys must me strings, and `0.0.toString()` is `0`
'ml/evaluate_data_frame.yml': [
'Test binary_soft_classifition precision',
'Test binary_soft_classifition recall',
'Test binary_soft_classifition confusion_matrix'
],
+
// it gets random failures on CI, must investigate
'ml/set_upgrade_mode.yml': [
'Attempt to open job when upgrade_mode is enabled',
'Setting upgrade mode to disabled from enabled'
],
+
// The cleanup fails with a index not found when retrieving the jobs
'ml/get_datafeed_stats.yml': ['Test get datafeed stats when total_search_time_ms mapping is missing'],
'ml/bucket_correlation_agg.yml': ['Test correlation bucket agg simple'],
+
// start should be a string
'ml/jobs_get_result_overall_buckets.yml': ['Test overall buckets given epoch start and end params'],
+
// this can't happen with the client
'ml/start_data_frame_analytics.yml': ['Test start with inconsistent body/param ids'],
'ml/stop_data_frame_analytics.yml': ['Test stop with inconsistent body/param ids'],
'ml/preview_datafeed.yml': ['*'],
+
// Investigate why is failing
'ml/inference_crud.yml': ['*'],
'ml/categorization_agg.yml': ['Test categorization aggregation with poor settings'],
'ml/filter_crud.yml': ['*'],
+
// investigate why this is failing
'monitoring/bulk/10_basic.yml': ['*'],
'monitoring/bulk/20_privileges.yml': ['*'],
'license/20_put_license.yml': ['*'],
'snapshot/10_basic.yml': ['*'],
'snapshot/20_operator_privileges_disabled.yml': ['*'],
+
// the body is correct, but the regex is failing
'sql/sql.yml': ['Getting textual representation'],
'searchable_snapshots/10_usage.yml': ['*'],
'service_accounts/10_basic.yml': ['*'],
+
// we are setting two certificates in the docker config
'ssl/10_basic.yml': ['*'],
'token/10_basic.yml': ['*'],
'token/11_invalidation.yml': ['*'],
+
// very likely, the index template has not been loaded yet.
// we should run a indices.existsTemplate, but the name of the
// template may vary during time.
@@ -147,16 +179,20 @@ const platinumBlackList = {
'transforms_stats.yml': ['*'],
'transforms_stats_continuous.yml': ['*'],
'transforms_update.yml': ['*'],
+
// js does not support ulongs
'unsigned_long/10_basic.yml': ['*'],
'unsigned_long/20_null_value.yml': ['*'],
'unsigned_long/30_multi_fields.yml': ['*'],
'unsigned_long/40_different_numeric.yml': ['*'],
'unsigned_long/50_script_values.yml': ['*'],
+
// the v8 client flattens the body into the parent object
'platinum/users/10_basic.yml': ['Test put user with different username in body'],
+
// docker issue?
'watcher/execute_watch/60_http_input.yml': ['*'],
+
// the checks are correct, but for some reason the test is failing on js side
// I bet is because the backslashes in the rg
'watcher/execute_watch/70_invalid.yml': ['*'],
@@ -170,8 +206,16 @@ const platinumBlackList = {
'platinum/ml/delete_job_force.yml': ['Test force delete an open job that is referred by a started datafeed'],
'platinum/ml/evaluate_data_frame.yml': ['*'],
'platinum/ml/get_datafeed_stats.yml': ['*'],
+
// start should be a string in the yaml test
- 'platinum/ml/start_stop_datafeed.yml': ['*']
+ 'platinum/ml/start_stop_datafeed.yml': ['*'],
+
+ // health API not yet supported
+ '/platinum/health/10_usage.yml': ['*'],
+
+ // ML update_trained_model_deployment not supported yet
+ '/platinum/ml/3rd_party_deployment.yml': ['Test update deployment'],
+ '/platinum/ml/update_trained_model_deployment.yml': ['Test with unknown model id']
}
function runner (opts = {}) {
@@ -316,7 +360,12 @@ async function start ({ client, isXPack }) {
junitTestSuites.end()
generateJunitXmlReport(junit, isXPack ? 'platinum' : 'free')
console.error(err)
- process.exit(1)
+
+ if (options.bail) {
+ process.exit(1)
+ } else {
+ continue
+ }
}
const totalTestTime = now() - testTime
junitTestCase.end()
@@ -380,7 +429,8 @@ function generateJunitXmlReport (junit, suite) {
}
if (require.main === module) {
- const node = process.env.TEST_ES_SERVER || 'http://elastic:changeme@localhost:9200'
+ const scheme = process.env.TEST_SUITE === 'platinum' ? 'https' : 'http'
+ const node = process.env.TEST_ES_SERVER || `${scheme}://elastic:changeme@localhost:9200`
const opts = {
node,
isXPack: process.env.TEST_SUITE !== 'free'
@@ -395,20 +445,20 @@ const shouldSkip = (isXPack, file, name) => {
for (let j = 0; j < freeTest.length; j++) {
if (file.endsWith(list[i]) && (name === freeTest[j] || freeTest[j] === '*')) {
const testName = file.slice(file.indexOf(`${sep}elasticsearch${sep}`)) + ' / ' + name
- log(`Skipping test ${testName} because is blacklisted in the free test`)
+ log(`Skipping test ${testName} because it is denylisted in the free test suite`)
return true
}
}
}
if (file.includes('x-pack') || isXPack) {
- list = Object.keys(platinumBlackList)
+ list = Object.keys(platinumDenyList)
for (let i = 0; i < list.length; i++) {
- const platTest = platinumBlackList[list[i]]
+ const platTest = platinumDenyList[list[i]]
for (let j = 0; j < platTest.length; j++) {
if (file.endsWith(list[i]) && (name === platTest[j] || platTest[j] === '*')) {
const testName = file.slice(file.indexOf(`${sep}elasticsearch${sep}`)) + ' / ' + name
- log(`Skipping test ${testName} because is blacklisted in the platinum test`)
+ log(`Skipping test ${testName} because it is denylisted in the platinum test suite`)
return true
}
}