[Backport 8.7] Integration test cleanup (#1841 (#1844)Co-authored-by: Josh Mock <josh@joshmock.com>
* Fix bad sysctl command in README * Add --suite and --test flags to integration tests So we can run a single suite or a single test without having to edit any code. * Drop several skipped integration tests Many of these skips are no longer necessary. Didn't do an exhaustive check of all skipped tests, so this is just a start. * Simplify cleanup make target Co-authored-by: Josh Mock <josh@joshmock.com>
This commit is contained in:
github-actions[bot]
committed by
GitHub
GitHub
parent
49eaea0f69
commit
ccf98d126b
3
Makefile
3
Makefile
@ -4,8 +4,7 @@ integration-setup: integration-cleanup
|
||||
|
||||
.PHONY: integration-cleanup
|
||||
integration-cleanup:
|
||||
docker stop instance || true
|
||||
docker volume rm instance-rest-test-data || true
|
||||
docker container rm --force --volumes instance || true
|
||||
|
||||
.PHONY: integration
|
||||
integration: integration-setup
|
||||
|
||||
@ -37,7 +37,7 @@ make integration
|
||||
|
||||
If Elasticsearch doesn't come up, run `make integration-cleanup` and then `DETACH=false .ci/run-elasticsearch.sh` manually to read the startup logs.
|
||||
|
||||
If you get an error about `vm.max_map_count` being too low, run `sudo sysctl -w vm.max_map_count=262144` to update the setting until the next reboot, or `sudo sysctl -w vm.max_map_count=262144 | sudo tee -a /etc/sysctl.conf` to update the setting permanently.
|
||||
If you get an error about `vm.max_map_count` being too low, run `sudo sysctl -w vm.max_map_count=262144` to update the setting until the next reboot, or `sudo sysctl -w vm.max_map_count=262144; echo 'vm.max_map_count=262144' | sudo tee -a /etc/sysctl.conf` to update the setting permanently.
|
||||
|
||||
### Exit on the first failure
|
||||
|
||||
|
||||
@ -30,7 +30,6 @@ const yaml = require('js-yaml')
|
||||
const minimist = require('minimist')
|
||||
const ms = require('ms')
|
||||
const { Client } = require('../../index')
|
||||
const { kProductCheck } = require('@elastic/transport/lib/symbols')
|
||||
const build = require('./test-runner')
|
||||
const { sleep } = require('./helper')
|
||||
const createJunitReporter = require('./reporter')
|
||||
@ -44,7 +43,8 @@ const MAX_FILE_TIME = 1000 * 30
|
||||
const MAX_TEST_TIME = 1000 * 3
|
||||
|
||||
const options = minimist(process.argv.slice(2), {
|
||||
boolean: ['bail']
|
||||
boolean: ['bail'],
|
||||
string: ['suite', 'test'],
|
||||
})
|
||||
|
||||
const freeSkips = {
|
||||
@ -56,10 +56,6 @@ const freeSkips = {
|
||||
'/free/cluster.desired_nodes/20_dry_run.yml': ['*'],
|
||||
'/free/cluster.prevalidate_node_removal/10_basic.yml': ['*'],
|
||||
|
||||
'/free/health/30_feature.yml': ['*'],
|
||||
'/free/health/40_useractions.yml': ['*'],
|
||||
'/free/health/40_diagnosis.yml': ['Diagnosis'],
|
||||
|
||||
// the v8 client never sends the scroll_id in querystring,
|
||||
// the way the test is structured causes a security exception
|
||||
'free/scroll/10_basic.yml': ['Body params override query string'],
|
||||
@ -70,9 +66,6 @@ const freeSkips = {
|
||||
'free/cat.allocation/10_basic.yml': ['*'],
|
||||
'free/cat.snapshots/10_basic.yml': ['Test cat snapshots output'],
|
||||
|
||||
// TODO: remove this once 'arbitrary_key' is implemented
|
||||
// https://github.com/elastic/elasticsearch/pull/41492
|
||||
'indices.split/30_copy_settings.yml': ['*'],
|
||||
'indices.stats/50_disk_usage.yml': ['Disk usage stats'],
|
||||
'indices.stats/60_field_usage.yml': ['Field usage stats'],
|
||||
|
||||
@ -99,22 +92,6 @@ const platinumDenyList = {
|
||||
'api_key/11_invalidation.yml': ['Test invalidate api key by realm name'],
|
||||
'analytics/histogram.yml': ['Histogram requires values in increasing order'],
|
||||
|
||||
// this two test cases are broken, we should
|
||||
// return on those in the future.
|
||||
'analytics/top_metrics.yml': [
|
||||
'sort by keyword field fails',
|
||||
'sort by string script fails'
|
||||
],
|
||||
|
||||
'cat.aliases/10_basic.yml': ['Empty cluster'],
|
||||
'index/10_with_id.yml': ['Index with ID'],
|
||||
'indices.get_alias/10_basic.yml': ['Get alias against closed indices'],
|
||||
'indices.get_alias/20_empty.yml': ['Check empty aliases when getting all aliases via /_alias'],
|
||||
'text_structure/find_structure.yml': ['*'],
|
||||
|
||||
// https://github.com/elastic/elasticsearch/pull/39400
|
||||
'ml/jobs_crud.yml': ['Test put job with id that is already taken'],
|
||||
|
||||
// object keys must me strings, and `0.0.toString()` is `0`
|
||||
'ml/evaluate_data_frame.yml': [
|
||||
'Test binary_soft_classifition precision',
|
||||
@ -122,12 +99,6 @@ const platinumDenyList = {
|
||||
'Test binary_soft_classifition confusion_matrix'
|
||||
],
|
||||
|
||||
// it gets random failures on CI, must investigate
|
||||
'ml/set_upgrade_mode.yml': [
|
||||
'Attempt to open job when upgrade_mode is enabled',
|
||||
'Setting upgrade mode to disabled from enabled'
|
||||
],
|
||||
|
||||
// The cleanup fails with a index not found when retrieving the jobs
|
||||
'ml/get_datafeed_stats.yml': ['Test get datafeed stats when total_search_time_ms mapping is missing'],
|
||||
'ml/bucket_correlation_agg.yml': ['Test correlation bucket agg simple'],
|
||||
@ -209,13 +180,6 @@ const platinumDenyList = {
|
||||
|
||||
// start should be a string in the yaml test
|
||||
'platinum/ml/start_stop_datafeed.yml': ['*'],
|
||||
|
||||
// health API not yet supported
|
||||
'/platinum/health/10_usage.yml': ['*'],
|
||||
|
||||
// ML update_trained_model_deployment not supported yet
|
||||
'/platinum/ml/3rd_party_deployment.yml': ['Test update deployment'],
|
||||
'/platinum/ml/update_trained_model_deployment.yml': ['Test with unknown model id']
|
||||
}
|
||||
|
||||
function runner (opts = {}) {
|
||||
@ -227,8 +191,6 @@ function runner (opts = {}) {
|
||||
}
|
||||
}
|
||||
const client = new Client(options)
|
||||
// TODO: remove the following line once https://github.com/elastic/elasticsearch/issues/82358 is fixed
|
||||
client.transport[kProductCheck] = null
|
||||
log('Loading yaml suite')
|
||||
start({ client, isXPack: opts.isXPack })
|
||||
.catch(err => {
|
||||
@ -333,13 +295,21 @@ async function start ({ client, isXPack }) {
|
||||
}
|
||||
|
||||
const cleanPath = file.slice(file.lastIndexOf(apiName))
|
||||
|
||||
// skip if --suite CLI arg doesn't match
|
||||
if (options.suite && !cleanPath.endsWith(options.suite)) continue
|
||||
|
||||
log(' ' + cleanPath)
|
||||
const junitTestSuite = junitTestSuites.testsuite(apiName.slice(1) + ' - ' + cleanPath)
|
||||
|
||||
for (const test of tests) {
|
||||
const testTime = now()
|
||||
const name = Object.keys(test)[0]
|
||||
|
||||
// skip setups, teardowns and anything that doesn't match --test flag when present
|
||||
if (name === 'setup' || name === 'teardown') continue
|
||||
if (options.test && !name.endsWith(options.test)) continue
|
||||
|
||||
const junitTestCase = junitTestSuite.testcase(name)
|
||||
|
||||
stats.total += 1
|
||||
@ -439,6 +409,8 @@ if (require.main === module) {
|
||||
}
|
||||
|
||||
const shouldSkip = (isXPack, file, name) => {
|
||||
if (options.suite || options.test) return false
|
||||
|
||||
let list = Object.keys(freeSkips)
|
||||
for (let i = 0; i < list.length; i++) {
|
||||
const freeTest = freeSkips[list[i]]
|
||||
|
||||
Reference in New Issue
Block a user