Fixes to YAML REST integration test suite runner (#1837)
* Use more inclusive language * Don't bail on failing tests without --bail * Skip a few more free suite tests * Default to https when running platinum tests * Add make targets for local integration testing * Linter cleanup * Skip some platinum integration tests * Improvements to integration test README * Another free test to skip for now * Continue on non-bail test failure * Output cleanup
This commit is contained in:
Josh Mock
@ -22,12 +22,12 @@
|
||||
# - Use https only when TEST_SUITE is "platinum", when "free" use http
|
||||
# - Set xpack.security.enabled=false for "free" and xpack.security.enabled=true for "platinum"
|
||||
|
||||
script_path=$(dirname $(realpath -s $0))
|
||||
source $script_path/functions/imports.sh
|
||||
script_path=$(dirname "$(realpath -s "$0")")
|
||||
source "$script_path/functions/imports.sh"
|
||||
set -euo pipefail
|
||||
|
||||
echo -e "\033[34;1mINFO:\033[0m Take down node if called twice with the same arguments (DETACH=true) or on seperate terminals \033[0m"
|
||||
cleanup_node $es_node_name
|
||||
echo -e "\033[34;1mINFO:\033[0m Take down node if called twice with the same arguments (DETACH=true) or on separate terminals \033[0m"
|
||||
cleanup_node "$es_node_name"
|
||||
|
||||
master_node_name=${es_node_name}
|
||||
cluster_name=${moniker}${suffix}
|
||||
|
||||
12
Makefile
Normal file
12
Makefile
Normal file
@ -0,0 +1,12 @@
|
||||
.PHONY: integration-setup
|
||||
integration-setup: integration-cleanup
|
||||
DETACH=true .ci/run-elasticsearch.sh
|
||||
|
||||
.PHONY: integration-cleanup
|
||||
integration-cleanup:
|
||||
docker stop instance || true
|
||||
docker volume rm instance-rest-test-data || true
|
||||
|
||||
.PHONY: integration
|
||||
integration: integration-setup
|
||||
npm run test:integration
|
||||
@ -228,7 +228,7 @@ function generateSingleApi (version, spec, common) {
|
||||
|
||||
${genUrlValidation(paths, api)}
|
||||
|
||||
let { ${genQueryBlacklist(false)}, ...querystring } = params
|
||||
let { ${genQueryDenylist(false)}, ...querystring } = params
|
||||
querystring = snakeCaseKeys(acceptedQuerystring, snakeCase, querystring)
|
||||
|
||||
let path = ''
|
||||
@ -316,20 +316,20 @@ function generateSingleApi (version, spec, common) {
|
||||
}, {})
|
||||
}
|
||||
|
||||
function genQueryBlacklist (addQuotes = true) {
|
||||
function genQueryDenylist (addQuotes = true) {
|
||||
const toCamelCase = str => {
|
||||
return str[0] === '_'
|
||||
? '_' + str.slice(1).replace(/_([a-z])/g, k => k[1].toUpperCase())
|
||||
: str.replace(/_([a-z])/g, k => k[1].toUpperCase())
|
||||
}
|
||||
|
||||
const blacklist = ['method', 'body']
|
||||
const denylist = ['method', 'body']
|
||||
parts.forEach(p => {
|
||||
const camelStr = toCamelCase(p)
|
||||
if (camelStr !== p) blacklist.push(`${camelStr}`)
|
||||
blacklist.push(`${p}`)
|
||||
if (camelStr !== p) denylist.push(`${camelStr}`)
|
||||
denylist.push(`${p}`)
|
||||
})
|
||||
return addQuotes ? blacklist.map(q => `'${q}'`) : blacklist
|
||||
return addQuotes ? denylist.map(q => `'${q}'`) : denylist
|
||||
}
|
||||
|
||||
function buildPath () {
|
||||
|
||||
@ -5,8 +5,8 @@
|
||||
Yes.
|
||||
|
||||
## Background
|
||||
Elasticsearch offers its entire API via HTTP REST endpoints. You can find the whole API specification for every version [here](https://github.com/elastic/elasticsearch/tree/master/rest-api-spec/src/main/resources/rest-api-spec/api).<br/>
|
||||
To support different languages at the same time, the Elasticsearch team decided to provide a [YAML specification](https://github.com/elastic/elasticsearch/tree/master/rest-api-spec/src/main/resources/rest-api-spec/test) to test every endpoint, body, headers, warning, error and so on.<br/>
|
||||
Elasticsearch offers its entire API via HTTP REST endpoints. You can find the whole API specification for every version [here](https://github.com/elastic/elasticsearch/tree/main/rest-api-spec/src/main/resources/rest-api-spec/api).<br/>
|
||||
To support different languages at the same time, the Elasticsearch team decided to provide a [YAML specification](https://github.com/elastic/elasticsearch/tree/main/rest-api-spec/src/main/resources/rest-api-spec/test) to test every endpoint, body, headers, warning, error and so on.<br/>
|
||||
This testing suite uses that specification to generate the test for the specified version of Elasticsearch on the fly.
|
||||
|
||||
## Run
|
||||
@ -20,20 +20,45 @@ Once the Elasticsearch repository has been cloned, the testing suite will connec
|
||||
|
||||
The specification does not allow the test to be run in parallel, so it might take a while to run the entire testing suite; on my machine, `MacBookPro15,2 core i7 2.7GHz 16GB of RAM` it takes around four minutes.
|
||||
|
||||
### Running locally
|
||||
|
||||
If you want to run the integration tests on your development machine, you must have an Elasticsearch instance running first.
|
||||
A local instance can be spun up in a Docker container by running the [`.ci/run-elasticsearch.sh`](/.ci/run-elasticsearch.sh) script.
|
||||
This is the same script CI jobs use to run Elasticsearch for integration tests, so your results should be relatively consistent.
|
||||
|
||||
To simplify the process of starting a container, testing, and cleaning up the container, you can run the `make integration` target:
|
||||
|
||||
```sh
|
||||
# set some parameters
|
||||
export STACK_VERSION=8.7.0
|
||||
export TEST_SUITE=free # can be `free` or `platinum`
|
||||
make integration
|
||||
```
|
||||
|
||||
If Elasticsearch doesn't come up, run `make integration-cleanup` and then `DETACH=false .ci/run-elasticsearch.sh` manually to read the startup logs.
|
||||
|
||||
If you get an error about `vm.max_map_count` being too low, run `sudo sysctl -w vm.max_map_count=262144` to update the setting until the next reboot, or `sudo sysctl -w vm.max_map_count=262144 | sudo tee -a /etc/sysctl.conf` to update the setting permanently.
|
||||
|
||||
### Exit on the first failure
|
||||
Bu default the suite will run all the test, even if one assertion has failed. If you want to stop the test at the first failure, use the bailout option:
|
||||
|
||||
By default the suite will run all the tests, even if one assertion has failed. If you want to stop the test at the first failure, use the bailout option:
|
||||
|
||||
```sh
|
||||
npm run test:integration -- --bail
|
||||
```
|
||||
|
||||
### Calculate the code coverage
|
||||
|
||||
If you want to calculate the code coverage just run the testing suite with the following parameters, once the test ends, it will open a browser window with the results.
|
||||
|
||||
```sh
|
||||
npm run test:integration -- --cov --coverage-report=html
|
||||
```
|
||||
|
||||
## How does this thing work?
|
||||
|
||||
At first sight, it might seem complicated, but once you understand what the moving parts are, it's quite easy.
|
||||
|
||||
1. Connects to the given Elasticsearch instance
|
||||
1. Gets the ES version and build hash
|
||||
1. Checkout to the given hash (and clone the repository if it is not present)
|
||||
@ -46,7 +71,4 @@ At first sight, it might seem complicated, but once you understand what the movi
|
||||
|
||||
Inside the `index.js` file, you will find the connection, cloning, reading and parsing part of the test, while inside the `test-runner.js` file you will find the function to handle the assertions. Inside `test-runner.js`, we use a [queue](https://github.com/delvedor/workq) to be sure that everything is run in the correct order.
|
||||
|
||||
Checkout the [rest-api-spec readme](https://github.com/elastic/elasticsearch/blob/master/rest-api-spec/src/main/resources/rest-api-spec/test/README.asciidoc) if you want to know more about how the assertions work.
|
||||
|
||||
#### Why are we running the test with the `--harmony` flag?
|
||||
Because on Node v6 the regex lookbehinds are not supported.
|
||||
Check out the [rest-api-spec readme](https://github.com/elastic/elasticsearch/blob/main/rest-api-spec/src/main/resources/rest-api-spec/test/README.asciidoc) if you want to know more about how the assertions work.
|
||||
|
||||
@ -27,6 +27,7 @@ process.on('unhandledRejection', function (err) {
|
||||
const { writeFileSync, readFileSync, readdirSync, statSync } = require('fs')
|
||||
const { join, sep } = require('path')
|
||||
const yaml = require('js-yaml')
|
||||
const minimist = require('minimist')
|
||||
const ms = require('ms')
|
||||
const { Client } = require('../../index')
|
||||
const { kProductCheck } = require('@elastic/transport/lib/symbols')
|
||||
@ -42,12 +43,24 @@ const MAX_API_TIME = 1000 * 90
|
||||
const MAX_FILE_TIME = 1000 * 30
|
||||
const MAX_TEST_TIME = 1000 * 3
|
||||
|
||||
const options = minimist(process.argv.slice(2), {
|
||||
boolean: ['bail']
|
||||
})
|
||||
|
||||
const freeSkips = {
|
||||
// not supported yet
|
||||
'/free/cluster.desired_nodes/10_basic.yml': ['*'],
|
||||
|
||||
// Cannot find methods on `Internal` object
|
||||
'/free/cluster.desired_balance/10_basic.yml': ['*'],
|
||||
'/free/cluster.desired_nodes/20_dry_run.yml': ['*'],
|
||||
'/free/cluster.prevalidate_node_removal/10_basic.yml': ['*'],
|
||||
|
||||
'/free/health/30_feature.yml': ['*'],
|
||||
'/free/health/40_useractions.yml': ['*'],
|
||||
// the v8 client never sends the scroll_id in querystgring,
|
||||
'/free/health/40_diagnosis.yml': ['Diagnosis'],
|
||||
|
||||
// the v8 client never sends the scroll_id in querystring,
|
||||
// the way the test is structured causes a security exception
|
||||
'free/scroll/10_basic.yml': ['Body params override query string'],
|
||||
'free/scroll/11_clear.yml': [
|
||||
@ -56,80 +69,99 @@ const freeSkips = {
|
||||
],
|
||||
'free/cat.allocation/10_basic.yml': ['*'],
|
||||
'free/cat.snapshots/10_basic.yml': ['Test cat snapshots output'],
|
||||
|
||||
// TODO: remove this once 'arbitrary_key' is implemented
|
||||
// https://github.com/elastic/elasticsearch/pull/41492
|
||||
'indices.split/30_copy_settings.yml': ['*'],
|
||||
'indices.stats/50_disk_usage.yml': ['Disk usage stats'],
|
||||
'indices.stats/60_field_usage.yml': ['Field usage stats'],
|
||||
|
||||
// skipping because we are booting ES with `discovery.type=single-node`
|
||||
// and this test will fail because of this configuration
|
||||
'nodes.stats/30_discovery.yml': ['*'],
|
||||
|
||||
// the expected error is returning a 503,
|
||||
// which triggers a retry and the node to be marked as dead
|
||||
'search.aggregation/240_max_buckets.yml': ['*'],
|
||||
|
||||
// long values and json do not play nicely together
|
||||
'search.aggregation/40_range.yml': ['Min and max long range bounds'],
|
||||
|
||||
// the yaml runner assumes that null means "does not exists",
|
||||
// while null is a valid json value, so the check will fail
|
||||
'search/320_disallow_queries.yml': ['Test disallow expensive queries'],
|
||||
'free/tsdb/90_unsupported_operations.yml': ['noop update']
|
||||
'free/tsdb/90_unsupported_operations.yml': ['noop update'],
|
||||
}
|
||||
const platinumBlackList = {
|
||||
|
||||
const platinumDenyList = {
|
||||
'api_key/10_basic.yml': ['Test get api key'],
|
||||
'api_key/20_query.yml': ['*'],
|
||||
'api_key/11_invalidation.yml': ['Test invalidate api key by realm name'],
|
||||
'analytics/histogram.yml': ['Histogram requires values in increasing order'],
|
||||
|
||||
// this two test cases are broken, we should
|
||||
// return on those in the future.
|
||||
'analytics/top_metrics.yml': [
|
||||
'sort by keyword field fails',
|
||||
'sort by string script fails'
|
||||
],
|
||||
|
||||
'cat.aliases/10_basic.yml': ['Empty cluster'],
|
||||
'index/10_with_id.yml': ['Index with ID'],
|
||||
'indices.get_alias/10_basic.yml': ['Get alias against closed indices'],
|
||||
'indices.get_alias/20_empty.yml': ['Check empty aliases when getting all aliases via /_alias'],
|
||||
'text_structure/find_structure.yml': ['*'],
|
||||
|
||||
// https://github.com/elastic/elasticsearch/pull/39400
|
||||
'ml/jobs_crud.yml': ['Test put job with id that is already taken'],
|
||||
|
||||
// object keys must me strings, and `0.0.toString()` is `0`
|
||||
'ml/evaluate_data_frame.yml': [
|
||||
'Test binary_soft_classifition precision',
|
||||
'Test binary_soft_classifition recall',
|
||||
'Test binary_soft_classifition confusion_matrix'
|
||||
],
|
||||
|
||||
// it gets random failures on CI, must investigate
|
||||
'ml/set_upgrade_mode.yml': [
|
||||
'Attempt to open job when upgrade_mode is enabled',
|
||||
'Setting upgrade mode to disabled from enabled'
|
||||
],
|
||||
|
||||
// The cleanup fails with a index not found when retrieving the jobs
|
||||
'ml/get_datafeed_stats.yml': ['Test get datafeed stats when total_search_time_ms mapping is missing'],
|
||||
'ml/bucket_correlation_agg.yml': ['Test correlation bucket agg simple'],
|
||||
|
||||
// start should be a string
|
||||
'ml/jobs_get_result_overall_buckets.yml': ['Test overall buckets given epoch start and end params'],
|
||||
|
||||
// this can't happen with the client
|
||||
'ml/start_data_frame_analytics.yml': ['Test start with inconsistent body/param ids'],
|
||||
'ml/stop_data_frame_analytics.yml': ['Test stop with inconsistent body/param ids'],
|
||||
'ml/preview_datafeed.yml': ['*'],
|
||||
|
||||
// Investigate why is failing
|
||||
'ml/inference_crud.yml': ['*'],
|
||||
'ml/categorization_agg.yml': ['Test categorization aggregation with poor settings'],
|
||||
'ml/filter_crud.yml': ['*'],
|
||||
|
||||
// investigate why this is failing
|
||||
'monitoring/bulk/10_basic.yml': ['*'],
|
||||
'monitoring/bulk/20_privileges.yml': ['*'],
|
||||
'license/20_put_license.yml': ['*'],
|
||||
'snapshot/10_basic.yml': ['*'],
|
||||
'snapshot/20_operator_privileges_disabled.yml': ['*'],
|
||||
|
||||
// the body is correct, but the regex is failing
|
||||
'sql/sql.yml': ['Getting textual representation'],
|
||||
'searchable_snapshots/10_usage.yml': ['*'],
|
||||
'service_accounts/10_basic.yml': ['*'],
|
||||
|
||||
// we are setting two certificates in the docker config
|
||||
'ssl/10_basic.yml': ['*'],
|
||||
'token/10_basic.yml': ['*'],
|
||||
'token/11_invalidation.yml': ['*'],
|
||||
|
||||
// very likely, the index template has not been loaded yet.
|
||||
// we should run a indices.existsTemplate, but the name of the
|
||||
// template may vary during time.
|
||||
@ -147,16 +179,20 @@ const platinumBlackList = {
|
||||
'transforms_stats.yml': ['*'],
|
||||
'transforms_stats_continuous.yml': ['*'],
|
||||
'transforms_update.yml': ['*'],
|
||||
|
||||
// js does not support ulongs
|
||||
'unsigned_long/10_basic.yml': ['*'],
|
||||
'unsigned_long/20_null_value.yml': ['*'],
|
||||
'unsigned_long/30_multi_fields.yml': ['*'],
|
||||
'unsigned_long/40_different_numeric.yml': ['*'],
|
||||
'unsigned_long/50_script_values.yml': ['*'],
|
||||
|
||||
// the v8 client flattens the body into the parent object
|
||||
'platinum/users/10_basic.yml': ['Test put user with different username in body'],
|
||||
|
||||
// docker issue?
|
||||
'watcher/execute_watch/60_http_input.yml': ['*'],
|
||||
|
||||
// the checks are correct, but for some reason the test is failing on js side
|
||||
// I bet is because the backslashes in the rg
|
||||
'watcher/execute_watch/70_invalid.yml': ['*'],
|
||||
@ -170,8 +206,16 @@ const platinumBlackList = {
|
||||
'platinum/ml/delete_job_force.yml': ['Test force delete an open job that is referred by a started datafeed'],
|
||||
'platinum/ml/evaluate_data_frame.yml': ['*'],
|
||||
'platinum/ml/get_datafeed_stats.yml': ['*'],
|
||||
|
||||
// start should be a string in the yaml test
|
||||
'platinum/ml/start_stop_datafeed.yml': ['*']
|
||||
'platinum/ml/start_stop_datafeed.yml': ['*'],
|
||||
|
||||
// health API not yet supported
|
||||
'/platinum/health/10_usage.yml': ['*'],
|
||||
|
||||
// ML update_trained_model_deployment not supported yet
|
||||
'/platinum/ml/3rd_party_deployment.yml': ['Test update deployment'],
|
||||
'/platinum/ml/update_trained_model_deployment.yml': ['Test with unknown model id']
|
||||
}
|
||||
|
||||
function runner (opts = {}) {
|
||||
@ -316,7 +360,12 @@ async function start ({ client, isXPack }) {
|
||||
junitTestSuites.end()
|
||||
generateJunitXmlReport(junit, isXPack ? 'platinum' : 'free')
|
||||
console.error(err)
|
||||
process.exit(1)
|
||||
|
||||
if (options.bail) {
|
||||
process.exit(1)
|
||||
} else {
|
||||
continue
|
||||
}
|
||||
}
|
||||
const totalTestTime = now() - testTime
|
||||
junitTestCase.end()
|
||||
@ -380,7 +429,8 @@ function generateJunitXmlReport (junit, suite) {
|
||||
}
|
||||
|
||||
if (require.main === module) {
|
||||
const node = process.env.TEST_ES_SERVER || 'http://elastic:changeme@localhost:9200'
|
||||
const scheme = process.env.TEST_SUITE === 'platinum' ? 'https' : 'http'
|
||||
const node = process.env.TEST_ES_SERVER || `${scheme}://elastic:changeme@localhost:9200`
|
||||
const opts = {
|
||||
node,
|
||||
isXPack: process.env.TEST_SUITE !== 'free'
|
||||
@ -395,20 +445,20 @@ const shouldSkip = (isXPack, file, name) => {
|
||||
for (let j = 0; j < freeTest.length; j++) {
|
||||
if (file.endsWith(list[i]) && (name === freeTest[j] || freeTest[j] === '*')) {
|
||||
const testName = file.slice(file.indexOf(`${sep}elasticsearch${sep}`)) + ' / ' + name
|
||||
log(`Skipping test ${testName} because is blacklisted in the free test`)
|
||||
log(`Skipping test ${testName} because it is denylisted in the free test suite`)
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (file.includes('x-pack') || isXPack) {
|
||||
list = Object.keys(platinumBlackList)
|
||||
list = Object.keys(platinumDenyList)
|
||||
for (let i = 0; i < list.length; i++) {
|
||||
const platTest = platinumBlackList[list[i]]
|
||||
const platTest = platinumDenyList[list[i]]
|
||||
for (let j = 0; j < platTest.length; j++) {
|
||||
if (file.endsWith(list[i]) && (name === platTest[j] || platTest[j] === '*')) {
|
||||
const testName = file.slice(file.indexOf(`${sep}elasticsearch${sep}`)) + ' / ' + name
|
||||
log(`Skipping test ${testName} because is blacklisted in the platinum test`)
|
||||
log(`Skipping test ${testName} because it is denylisted in the platinum test suite`)
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user