diff --git a/.ci/run-elasticsearch.sh b/.ci/run-elasticsearch.sh index 3f4e2f1da..2f360ab4f 100755 --- a/.ci/run-elasticsearch.sh +++ b/.ci/run-elasticsearch.sh @@ -22,12 +22,12 @@ # - Use https only when TEST_SUITE is "platinum", when "free" use http # - Set xpack.security.enabled=false for "free" and xpack.security.enabled=true for "platinum" -script_path=$(dirname $(realpath -s $0)) -source $script_path/functions/imports.sh +script_path=$(dirname "$(realpath -s "$0")") +source "$script_path/functions/imports.sh" set -euo pipefail -echo -e "\033[34;1mINFO:\033[0m Take down node if called twice with the same arguments (DETACH=true) or on seperate terminals \033[0m" -cleanup_node $es_node_name +echo -e "\033[34;1mINFO:\033[0m Take down node if called twice with the same arguments (DETACH=true) or on separate terminals \033[0m" +cleanup_node "$es_node_name" master_node_name=${es_node_name} cluster_name=${moniker}${suffix} diff --git a/Makefile b/Makefile new file mode 100644 index 000000000..a374c22db --- /dev/null +++ b/Makefile @@ -0,0 +1,12 @@ +.PHONY: integration-setup +integration-setup: integration-cleanup + DETACH=true .ci/run-elasticsearch.sh + +.PHONY: integration-cleanup +integration-cleanup: + docker stop instance || true + docker volume rm instance-rest-test-data || true + +.PHONY: integration +integration: integration-setup + npm run test:integration diff --git a/scripts/utils/generateApis.js b/scripts/utils/generateApis.js index 53dc1abed..a1dddd063 100644 --- a/scripts/utils/generateApis.js +++ b/scripts/utils/generateApis.js @@ -228,7 +228,7 @@ function generateSingleApi (version, spec, common) { ${genUrlValidation(paths, api)} - let { ${genQueryBlacklist(false)}, ...querystring } = params + let { ${genQueryDenylist(false)}, ...querystring } = params querystring = snakeCaseKeys(acceptedQuerystring, snakeCase, querystring) let path = '' @@ -316,20 +316,20 @@ function generateSingleApi (version, spec, common) { }, {}) } - function genQueryBlacklist (addQuotes = true) { + function genQueryDenylist (addQuotes = true) { const toCamelCase = str => { return str[0] === '_' ? '_' + str.slice(1).replace(/_([a-z])/g, k => k[1].toUpperCase()) : str.replace(/_([a-z])/g, k => k[1].toUpperCase()) } - const blacklist = ['method', 'body'] + const denylist = ['method', 'body'] parts.forEach(p => { const camelStr = toCamelCase(p) - if (camelStr !== p) blacklist.push(`${camelStr}`) - blacklist.push(`${p}`) + if (camelStr !== p) denylist.push(`${camelStr}`) + denylist.push(`${p}`) }) - return addQuotes ? blacklist.map(q => `'${q}'`) : blacklist + return addQuotes ? denylist.map(q => `'${q}'`) : denylist } function buildPath () { diff --git a/test/integration/README.md b/test/integration/README.md index 0861dd8b9..a52ae2e54 100644 --- a/test/integration/README.md +++ b/test/integration/README.md @@ -5,8 +5,8 @@ Yes. ## Background -Elasticsearch offers its entire API via HTTP REST endpoints. You can find the whole API specification for every version [here](https://github.com/elastic/elasticsearch/tree/master/rest-api-spec/src/main/resources/rest-api-spec/api).
-To support different languages at the same time, the Elasticsearch team decided to provide a [YAML specification](https://github.com/elastic/elasticsearch/tree/master/rest-api-spec/src/main/resources/rest-api-spec/test) to test every endpoint, body, headers, warning, error and so on.
+Elasticsearch offers its entire API via HTTP REST endpoints. You can find the whole API specification for every version [here](https://github.com/elastic/elasticsearch/tree/main/rest-api-spec/src/main/resources/rest-api-spec/api).
+To support different languages at the same time, the Elasticsearch team decided to provide a [YAML specification](https://github.com/elastic/elasticsearch/tree/main/rest-api-spec/src/main/resources/rest-api-spec/test) to test every endpoint, body, headers, warning, error and so on.
This testing suite uses that specification to generate the test for the specified version of Elasticsearch on the fly. ## Run @@ -20,20 +20,45 @@ Once the Elasticsearch repository has been cloned, the testing suite will connec The specification does not allow the test to be run in parallel, so it might take a while to run the entire testing suite; on my machine, `MacBookPro15,2 core i7 2.7GHz 16GB of RAM` it takes around four minutes. +### Running locally + +If you want to run the integration tests on your development machine, you must have an Elasticsearch instance running first. +A local instance can be spun up in a Docker container by running the [`.ci/run-elasticsearch.sh`](/.ci/run-elasticsearch.sh) script. +This is the same script CI jobs use to run Elasticsearch for integration tests, so your results should be relatively consistent. + +To simplify the process of starting a container, testing, and cleaning up the container, you can run the `make integration` target: + +```sh +# set some parameters +export STACK_VERSION=8.7.0 +export TEST_SUITE=free # can be `free` or `platinum` +make integration +``` + +If Elasticsearch doesn't come up, run `make integration-cleanup` and then `DETACH=false .ci/run-elasticsearch.sh` manually to read the startup logs. + +If you get an error about `vm.max_map_count` being too low, run `sudo sysctl -w vm.max_map_count=262144` to update the setting until the next reboot, or `sudo sysctl -w vm.max_map_count=262144 | sudo tee -a /etc/sysctl.conf` to update the setting permanently. + ### Exit on the first failure -Bu default the suite will run all the test, even if one assertion has failed. If you want to stop the test at the first failure, use the bailout option: + +By default the suite will run all the tests, even if one assertion has failed. If you want to stop the test at the first failure, use the bailout option: + ```sh npm run test:integration -- --bail ``` ### Calculate the code coverage + If you want to calculate the code coverage just run the testing suite with the following parameters, once the test ends, it will open a browser window with the results. + ```sh npm run test:integration -- --cov --coverage-report=html ``` ## How does this thing work? + At first sight, it might seem complicated, but once you understand what the moving parts are, it's quite easy. + 1. Connects to the given Elasticsearch instance 1. Gets the ES version and build hash 1. Checkout to the given hash (and clone the repository if it is not present) @@ -46,7 +71,4 @@ At first sight, it might seem complicated, but once you understand what the movi Inside the `index.js` file, you will find the connection, cloning, reading and parsing part of the test, while inside the `test-runner.js` file you will find the function to handle the assertions. Inside `test-runner.js`, we use a [queue](https://github.com/delvedor/workq) to be sure that everything is run in the correct order. -Checkout the [rest-api-spec readme](https://github.com/elastic/elasticsearch/blob/master/rest-api-spec/src/main/resources/rest-api-spec/test/README.asciidoc) if you want to know more about how the assertions work. - -#### Why are we running the test with the `--harmony` flag? -Because on Node v6 the regex lookbehinds are not supported. +Check out the [rest-api-spec readme](https://github.com/elastic/elasticsearch/blob/main/rest-api-spec/src/main/resources/rest-api-spec/test/README.asciidoc) if you want to know more about how the assertions work. diff --git a/test/integration/index.js b/test/integration/index.js index c794beb6c..5c4addcb7 100644 --- a/test/integration/index.js +++ b/test/integration/index.js @@ -27,6 +27,7 @@ process.on('unhandledRejection', function (err) { const { writeFileSync, readFileSync, readdirSync, statSync } = require('fs') const { join, sep } = require('path') const yaml = require('js-yaml') +const minimist = require('minimist') const ms = require('ms') const { Client } = require('../../index') const { kProductCheck } = require('@elastic/transport/lib/symbols') @@ -42,12 +43,24 @@ const MAX_API_TIME = 1000 * 90 const MAX_FILE_TIME = 1000 * 30 const MAX_TEST_TIME = 1000 * 3 +const options = minimist(process.argv.slice(2), { + boolean: ['bail'] +}) + const freeSkips = { // not supported yet '/free/cluster.desired_nodes/10_basic.yml': ['*'], + + // Cannot find methods on `Internal` object + '/free/cluster.desired_balance/10_basic.yml': ['*'], + '/free/cluster.desired_nodes/20_dry_run.yml': ['*'], + '/free/cluster.prevalidate_node_removal/10_basic.yml': ['*'], + '/free/health/30_feature.yml': ['*'], '/free/health/40_useractions.yml': ['*'], - // the v8 client never sends the scroll_id in querystgring, + '/free/health/40_diagnosis.yml': ['Diagnosis'], + + // the v8 client never sends the scroll_id in querystring, // the way the test is structured causes a security exception 'free/scroll/10_basic.yml': ['Body params override query string'], 'free/scroll/11_clear.yml': [ @@ -56,80 +69,99 @@ const freeSkips = { ], 'free/cat.allocation/10_basic.yml': ['*'], 'free/cat.snapshots/10_basic.yml': ['Test cat snapshots output'], + // TODO: remove this once 'arbitrary_key' is implemented // https://github.com/elastic/elasticsearch/pull/41492 'indices.split/30_copy_settings.yml': ['*'], 'indices.stats/50_disk_usage.yml': ['Disk usage stats'], 'indices.stats/60_field_usage.yml': ['Field usage stats'], + // skipping because we are booting ES with `discovery.type=single-node` // and this test will fail because of this configuration 'nodes.stats/30_discovery.yml': ['*'], + // the expected error is returning a 503, // which triggers a retry and the node to be marked as dead 'search.aggregation/240_max_buckets.yml': ['*'], + // long values and json do not play nicely together 'search.aggregation/40_range.yml': ['Min and max long range bounds'], + // the yaml runner assumes that null means "does not exists", // while null is a valid json value, so the check will fail 'search/320_disallow_queries.yml': ['Test disallow expensive queries'], - 'free/tsdb/90_unsupported_operations.yml': ['noop update'] + 'free/tsdb/90_unsupported_operations.yml': ['noop update'], } -const platinumBlackList = { + +const platinumDenyList = { 'api_key/10_basic.yml': ['Test get api key'], 'api_key/20_query.yml': ['*'], 'api_key/11_invalidation.yml': ['Test invalidate api key by realm name'], 'analytics/histogram.yml': ['Histogram requires values in increasing order'], + // this two test cases are broken, we should // return on those in the future. 'analytics/top_metrics.yml': [ 'sort by keyword field fails', 'sort by string script fails' ], + 'cat.aliases/10_basic.yml': ['Empty cluster'], 'index/10_with_id.yml': ['Index with ID'], 'indices.get_alias/10_basic.yml': ['Get alias against closed indices'], 'indices.get_alias/20_empty.yml': ['Check empty aliases when getting all aliases via /_alias'], 'text_structure/find_structure.yml': ['*'], + // https://github.com/elastic/elasticsearch/pull/39400 'ml/jobs_crud.yml': ['Test put job with id that is already taken'], + // object keys must me strings, and `0.0.toString()` is `0` 'ml/evaluate_data_frame.yml': [ 'Test binary_soft_classifition precision', 'Test binary_soft_classifition recall', 'Test binary_soft_classifition confusion_matrix' ], + // it gets random failures on CI, must investigate 'ml/set_upgrade_mode.yml': [ 'Attempt to open job when upgrade_mode is enabled', 'Setting upgrade mode to disabled from enabled' ], + // The cleanup fails with a index not found when retrieving the jobs 'ml/get_datafeed_stats.yml': ['Test get datafeed stats when total_search_time_ms mapping is missing'], 'ml/bucket_correlation_agg.yml': ['Test correlation bucket agg simple'], + // start should be a string 'ml/jobs_get_result_overall_buckets.yml': ['Test overall buckets given epoch start and end params'], + // this can't happen with the client 'ml/start_data_frame_analytics.yml': ['Test start with inconsistent body/param ids'], 'ml/stop_data_frame_analytics.yml': ['Test stop with inconsistent body/param ids'], 'ml/preview_datafeed.yml': ['*'], + // Investigate why is failing 'ml/inference_crud.yml': ['*'], 'ml/categorization_agg.yml': ['Test categorization aggregation with poor settings'], 'ml/filter_crud.yml': ['*'], + // investigate why this is failing 'monitoring/bulk/10_basic.yml': ['*'], 'monitoring/bulk/20_privileges.yml': ['*'], 'license/20_put_license.yml': ['*'], 'snapshot/10_basic.yml': ['*'], 'snapshot/20_operator_privileges_disabled.yml': ['*'], + // the body is correct, but the regex is failing 'sql/sql.yml': ['Getting textual representation'], 'searchable_snapshots/10_usage.yml': ['*'], 'service_accounts/10_basic.yml': ['*'], + // we are setting two certificates in the docker config 'ssl/10_basic.yml': ['*'], 'token/10_basic.yml': ['*'], 'token/11_invalidation.yml': ['*'], + // very likely, the index template has not been loaded yet. // we should run a indices.existsTemplate, but the name of the // template may vary during time. @@ -147,16 +179,20 @@ const platinumBlackList = { 'transforms_stats.yml': ['*'], 'transforms_stats_continuous.yml': ['*'], 'transforms_update.yml': ['*'], + // js does not support ulongs 'unsigned_long/10_basic.yml': ['*'], 'unsigned_long/20_null_value.yml': ['*'], 'unsigned_long/30_multi_fields.yml': ['*'], 'unsigned_long/40_different_numeric.yml': ['*'], 'unsigned_long/50_script_values.yml': ['*'], + // the v8 client flattens the body into the parent object 'platinum/users/10_basic.yml': ['Test put user with different username in body'], + // docker issue? 'watcher/execute_watch/60_http_input.yml': ['*'], + // the checks are correct, but for some reason the test is failing on js side // I bet is because the backslashes in the rg 'watcher/execute_watch/70_invalid.yml': ['*'], @@ -170,8 +206,16 @@ const platinumBlackList = { 'platinum/ml/delete_job_force.yml': ['Test force delete an open job that is referred by a started datafeed'], 'platinum/ml/evaluate_data_frame.yml': ['*'], 'platinum/ml/get_datafeed_stats.yml': ['*'], + // start should be a string in the yaml test - 'platinum/ml/start_stop_datafeed.yml': ['*'] + 'platinum/ml/start_stop_datafeed.yml': ['*'], + + // health API not yet supported + '/platinum/health/10_usage.yml': ['*'], + + // ML update_trained_model_deployment not supported yet + '/platinum/ml/3rd_party_deployment.yml': ['Test update deployment'], + '/platinum/ml/update_trained_model_deployment.yml': ['Test with unknown model id'] } function runner (opts = {}) { @@ -316,7 +360,12 @@ async function start ({ client, isXPack }) { junitTestSuites.end() generateJunitXmlReport(junit, isXPack ? 'platinum' : 'free') console.error(err) - process.exit(1) + + if (options.bail) { + process.exit(1) + } else { + continue + } } const totalTestTime = now() - testTime junitTestCase.end() @@ -380,7 +429,8 @@ function generateJunitXmlReport (junit, suite) { } if (require.main === module) { - const node = process.env.TEST_ES_SERVER || 'http://elastic:changeme@localhost:9200' + const scheme = process.env.TEST_SUITE === 'platinum' ? 'https' : 'http' + const node = process.env.TEST_ES_SERVER || `${scheme}://elastic:changeme@localhost:9200` const opts = { node, isXPack: process.env.TEST_SUITE !== 'free' @@ -395,20 +445,20 @@ const shouldSkip = (isXPack, file, name) => { for (let j = 0; j < freeTest.length; j++) { if (file.endsWith(list[i]) && (name === freeTest[j] || freeTest[j] === '*')) { const testName = file.slice(file.indexOf(`${sep}elasticsearch${sep}`)) + ' / ' + name - log(`Skipping test ${testName} because is blacklisted in the free test`) + log(`Skipping test ${testName} because it is denylisted in the free test suite`) return true } } } if (file.includes('x-pack') || isXPack) { - list = Object.keys(platinumBlackList) + list = Object.keys(platinumDenyList) for (let i = 0; i < list.length; i++) { - const platTest = platinumBlackList[list[i]] + const platTest = platinumDenyList[list[i]] for (let j = 0; j < platTest.length; j++) { if (file.endsWith(list[i]) && (name === platTest[j] || platTest[j] === '*')) { const testName = file.slice(file.indexOf(`${sep}elasticsearch${sep}`)) + ' / ' + name - log(`Skipping test ${testName} because is blacklisted in the platinum test`) + log(`Skipping test ${testName} because it is denylisted in the platinum test suite`) return true } }