From b33da1975fe47ed8cacddb57a7c501b01c1fd28c Mon Sep 17 00:00:00 2001 From: Daniel Mitterdorfer Date: Thu, 26 Mar 2020 17:33:10 +0100 Subject: [PATCH] Run integration tests with pytest (#919) With this commit we add a proof-of-concept implementation of how integration tests could look like with py.test instead of shell scripts. For the proof of concept we migrate the integration tests for the list subcommands and source builds. It also spins up an Elasticsearch metrics store and checks that any preconditions (basically Docker being present and up and running) are met. Finally, it also integrates into CI by writing JUnit XML files for test results. Relates #736 --- Makefile | 2 +- integration-test.sh | 38 ------ it/__init__.py | 191 ++++++++++++++++++++++++++++ it/list_test.py | 45 +++++++ it/resources/rally-es-it.ini | 39 ++++++ it/resources/rally-in-memory-it.ini | 34 +++++ it/sources_test.py | 29 +++++ setup.cfg | 2 + setup.py | 2 +- tox.ini | 1 + 10 files changed, 343 insertions(+), 40 deletions(-) create mode 100644 it/__init__.py create mode 100644 it/list_test.py create mode 100644 it/resources/rally-es-it.ini create mode 100644 it/resources/rally-in-memory-it.ini create mode 100644 it/sources_test.py diff --git a/Makefile b/Makefile index ad7aef8f2..563f4b115 100644 --- a/Makefile +++ b/Makefile @@ -88,7 +88,7 @@ tox-env-clean: rm -rf .tox lint: check-venv - @find esrally benchmarks scripts tests -name "*.py" -exec $(VEPYLINT) -j0 -rn --load-plugins pylint_quotes --rcfile=$(CURDIR)/.pylintrc \{\} + + @find esrally benchmarks scripts tests it -name "*.py" -exec $(VEPYLINT) -j0 -rn --load-plugins pylint_quotes --rcfile=$(CURDIR)/.pylintrc \{\} + docs: check-venv @. $(VENV_ACTIVATE_FILE); cd docs && $(MAKE) html diff --git a/integration-test.sh b/integration-test.sh index bec4da6e5..00e4e852a 100755 --- a/integration-test.sh +++ b/integration-test.sh @@ -255,24 +255,6 @@ function test_configure { esrally configure --assume-defaults --configuration-name="config-integration-test" } -function test_list { - local cfg - random_configuration cfg - - info "test list races [${cfg}]" - esrally list races --configuration-name="${cfg}" - info "test list cars [${cfg}]" - esrally list cars --configuration-name="${cfg}" - info "test list Elasticsearch plugins [${cfg}]" - esrally list elasticsearch-plugins --configuration-name="${cfg}" - info "test list tracks [${cfg}]" - esrally list tracks --configuration-name="${cfg}" - info "test list can use track revision together with track repository" - esrally list tracks --configuration-name="${cfg}" --track-repository=default --track-revision=4080dc9850d07e23b6fc7cfcdc7cf57b14e5168d - info "test list telemetry [${cfg}]" - esrally list telemetry --configuration-name="${cfg}" -} - function test_info { local cfg random_configuration cfg @@ -298,22 +280,6 @@ function test_download { done } -function test_sources { - local cfg - random_configuration cfg - - # build Elasticsearch and a core plugin - info "test sources [--configuration-name=${cfg}], [--revision=latest], [--track=geonames], [--challenge=append-no-conflicts], [--car=4gheap] [--elasticsearch-plugins=analysis-icu]" - kill_rally_processes - wait_for_free_es_port - esrally --configuration-name="${cfg}" --on-error=abort --revision=latest --track=geonames --test-mode --challenge=append-no-conflicts --car=4gheap --elasticsearch-plugins=analysis-icu - - info "test sources [--configuration-name=${cfg}], [--pipeline=from-sources-skip-build], [--track=geonames], [--challenge=append-no-conflicts-index-only], [--car=4gheap,ea] " - kill_rally_processes - wait_for_free_es_port - esrally --configuration-name="${cfg}" --on-error=abort --pipeline=from-sources-skip-build --track=geonames --test-mode --challenge=append-no-conflicts-index-only --car="4gheap,ea" -} - function test_distributions { local cfg @@ -600,16 +566,12 @@ function run_test { fi echo "**************************************** TESTING CONFIGURATION OF RALLY ****************************************" test_configure - echo "**************************************** TESTING RALLY LIST COMMANDS *******************************************" - test_list echo "**************************************** TESTING RALLY INFO COMMAND ********************************************" test_info echo "**************************************** TESTING RALLY FAILS WITH UNUSED TRACK-PARAMS **************************" test_distribution_fails_with_wrong_track_params echo "**************************************** TESTING RALLY DOWNLOAD COMMAND ***********************************" test_download - echo "**************************************** TESTING RALLY WITH ES FROM SOURCES ************************************" - test_sources echo "**************************************** TESTING RALLY WITH ES DISTRIBUTIONS ***********************************" test_distributions echo "**************************************** TESTING RALLY WITH ES DOCKER IMAGE ***********************************" diff --git a/it/__init__.py b/it/__init__.py new file mode 100644 index 000000000..5dccf7afe --- /dev/null +++ b/it/__init__.py @@ -0,0 +1,191 @@ +# Licensed to Elasticsearch B.V. under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Elasticsearch B.V. licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import functools +import json +import os +import random +from string import Template + +import pytest + +from esrally import client +from esrally.utils import process, io + +CONFIG_NAMES = ["in-memory-it", "es-it"] + + +def all_rally_configs(t): + @functools.wraps(t) + @pytest.mark.parametrize("cfg", CONFIG_NAMES) + def wrapper(cfg, *args, **kwargs): + t(cfg, *args, **kwargs) + + return wrapper + + +def random_rally_config(t): + @functools.wraps(t) + @pytest.mark.parametrize("cfg", [random.choice(CONFIG_NAMES)]) + def wrapper(cfg, *args, **kwargs): + t(cfg, *args, **kwargs) + + return wrapper + + +def rally_in_mem(t): + @functools.wraps(t) + @pytest.mark.parametrize("cfg", ["in-memory-it"]) + def wrapper(cfg, *args, **kwargs): + t(cfg, *args, **kwargs) + + return wrapper + + +def rally_es(t): + @functools.wraps(t) + @pytest.mark.parametrize("cfg", ["es-it"]) + def wrapper(cfg, *args, **kwargs): + t(cfg, *args, **kwargs) + + return wrapper + + +def esrally(cfg, command_line): + return os.system("esrally {} --configuration-name=\"{}\"".format(command_line, cfg)) + + +def wait_until_port_is_free(port_number=39200, timeout=120): + import errno + import time + import socket + + start = time.perf_counter() + end = start + timeout + while time.perf_counter() < end: + c = socket.socket() + connect_result = c.connect_ex(("127.0.0.1", port_number)) + # noinspection PyBroadException + try: + if connect_result == errno.ECONNREFUSED: + c.close() + return + else: + c.close() + time.sleep(0.5) + except Exception: + pass + + raise TimeoutError(f"Port [{port_number}] is occupied after [{timeout}] seconds") + + +def check_prerequisites(): + if process.run_subprocess_with_logging("docker ps") != 0: + raise AssertionError("Docker must be installed and the daemon must be up and running to run integration tests.") + if process.run_subprocess_with_logging("docker-compose --help") != 0: + raise AssertionError("Docker Compose is required to run integration tests.") + + +class ConfigFile: + def __init__(self, config_name): + self.user_home = os.path.expanduser("~") + self.rally_home = os.path.join(self.user_home, ".rally") + self.config_file_name = "rally-{}.ini".format(config_name) + self.source_path = os.path.join(os.path.dirname(__file__), "resources", self.config_file_name) + self.target_path = os.path.join(self.rally_home, self.config_file_name) + + +class TestCluster: + def __init__(self, cfg): + self.cfg = cfg + self.installation_id = None + self.http_port = None + + def install(self, distribution_version, node_name, http_port): + self.http_port = http_port + transport_port = http_port + 100 + try: + output = process.run_subprocess_with_output( + "esrally install --configuration-name={cfg} --quiet --distribution-version={dist} --build-type=tar " + "--http-port={http_port} --node={node_name} --master-nodes={node_name} " + "--seed-hosts=\"127.0.0.1:{transport_port}\"".format(cfg=self.cfg, + dist=distribution_version, + http_port=http_port, + node_name=node_name, + transport_port=transport_port)) + + self.installation_id = json.loads("".join(output))["installation-id"] + except BaseException as e: + raise AssertionError("Failed to install Elasticsearch {}.".format(distribution_version), e) + + def start(self, race_id): + cmd = "start --runtime-jdk=\"bundled\" --installation-id={} --race-id={}".format(self.installation_id, race_id) + if esrally(self.cfg, cmd) != 0: + raise AssertionError("Failed to start Elasticsearch test cluster.") + es = client.EsClientFactory(hosts=[{"host": "127.0.0.1", "port": self.http_port}], client_options={}).create() + client.wait_for_rest_layer(es) + + def stop(self): + if self.installation_id: + if esrally(self.cfg, "stop --installation-id={}".format(self.installation_id)) != 0: + raise AssertionError("Failed to stop Elasticsearch test cluster.") + + +class EsMetricsStore: + VERSION = "7.6.0" + + def __init__(self): + self.cluster = TestCluster("in-memory-it") + + def start(self): + self.cluster.install(distribution_version=EsMetricsStore.VERSION, node_name="metrics-store", http_port=10200) + self.cluster.start(race_id="metrics-store") + + def stop(self): + self.cluster.stop() + + +def install_integration_test_config(): + def copy_config(name): + f = ConfigFile(name) + io.ensure_dir(f.rally_home) + with open(f.target_path, "w", encoding="UTF-8") as target: + with open(f.source_path, "r", encoding="UTF-8") as src: + contents = src.read() + target.write(Template(contents).substitute(USER_HOME=f.user_home)) + + for n in CONFIG_NAMES: + copy_config(n) + + +def remove_integration_test_config(): + for n in CONFIG_NAMES: + os.remove(ConfigFile(n).target_path) + + +ES_METRICS_STORE = EsMetricsStore() + + +def setup_module(): + check_prerequisites() + install_integration_test_config() + ES_METRICS_STORE.start() + + +def teardown_module(): + ES_METRICS_STORE.stop() + remove_integration_test_config() diff --git a/it/list_test.py b/it/list_test.py new file mode 100644 index 000000000..7e982f0ee --- /dev/null +++ b/it/list_test.py @@ -0,0 +1,45 @@ +# Licensed to Elasticsearch B.V. under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Elasticsearch B.V. licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import it + + +@it.all_rally_configs +def test_list_races(cfg): + assert it.esrally(cfg, "list races") == 0 + + +@it.rally_in_mem +def test_list_cars(cfg): + assert it.esrally(cfg, "list cars") == 0 + + +@it.rally_in_mem +def test_list_elasticsearch_plugins(cfg): + assert it.esrally(cfg, "list elasticsearch-plugins") == 0 + + +@it.rally_in_mem +def test_list_tracks(cfg): + assert it.esrally(cfg, "list tracks") == 0 + assert it.esrally(cfg, "list tracks --track-repository=default " + "--track-revision=4080dc9850d07e23b6fc7cfcdc7cf57b14e5168d") == 0 + + +@it.rally_in_mem +def test_list_telemetry(cfg): + assert it.esrally(cfg, "list telemetry") == 0 diff --git a/it/resources/rally-es-it.ini b/it/resources/rally-es-it.ini new file mode 100644 index 000000000..424bf1280 --- /dev/null +++ b/it/resources/rally-es-it.ini @@ -0,0 +1,39 @@ +[meta] +config.version = 17 + +[system] +env.name = local + +[node] +root.dir = ${USER_HOME}/.rally/benchmarks +src.root.dir = ${USER_HOME}/.rally/benchmarks/src + +[source] +remote.repo.url = https://github.com/elastic/elasticsearch.git +elasticsearch.src.subdir = elasticsearch + +[benchmarks] +local.dataset.cache = ${USER_HOME}/.rally/benchmarks/data + +[reporting] +datastore.type = elasticsearch +datastore.host = localhost +datastore.port = 10200 +datastore.secure = False +datastore.user = +datastore.password = + + +[tracks] +default.url = https://github.com/elastic/rally-tracks +eventdata.url = https://github.com/elastic/rally-eventdata-track + +[teams] +default.url = https://github.com/elastic/rally-teams + +[defaults] +preserve_benchmark_candidate = False + +[distributions] +release.cache = true + diff --git a/it/resources/rally-in-memory-it.ini b/it/resources/rally-in-memory-it.ini new file mode 100644 index 000000000..00ae4531e --- /dev/null +++ b/it/resources/rally-in-memory-it.ini @@ -0,0 +1,34 @@ +[meta] +config.version = 17 + +[system] +env.name = local + +[node] +root.dir = ${USER_HOME}/.rally/benchmarks +src.root.dir = ${USER_HOME}/.rally/benchmarks/src + +[source] +remote.repo.url = https://github.com/elastic/elasticsearch.git +elasticsearch.src.subdir = elasticsearch + +[benchmarks] +local.dataset.cache = ${USER_HOME}/.rally/benchmarks/data + +[reporting] +datastore.type = in-memory + + +[tracks] +default.url = https://github.com/elastic/rally-tracks +eventdata.url = https://github.com/elastic/rally-eventdata-track + +[teams] +default.url = https://github.com/elastic/rally-teams + +[defaults] +preserve_benchmark_candidate = False + +[distributions] +release.cache = true + diff --git a/it/sources_test.py b/it/sources_test.py new file mode 100644 index 000000000..ea253f121 --- /dev/null +++ b/it/sources_test.py @@ -0,0 +1,29 @@ +# Licensed to Elasticsearch B.V. under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Elasticsearch B.V. licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import it + + +@it.random_rally_config +def test_sources(cfg): + it.wait_until_port_is_free() + assert it.esrally(cfg, "--on-error=abort --revision=latest --track=geonames --test-mode " + "--challenge=append-no-conflicts --car=4gheap --elasticsearch-plugins=analysis-icu") == 0 + + it.wait_until_port_is_free() + assert it.esrally(cfg, "--on-error=abort --pipeline=from-sources-skip-build --track=geonames --test-mode " + "--challenge=append-no-conflicts-index-only --car=\"4gheap,ea\"") == 0 diff --git a/setup.cfg b/setup.cfg index 145de8d36..8270bbd0c 100644 --- a/setup.cfg +++ b/setup.cfg @@ -4,3 +4,5 @@ test=pytest [tool:pytest] addopts = --verbose --color=yes testpaths = tests +junit_family = xunit2 +junit_logging = all diff --git a/setup.py b/setup.py index 22643fd8a..1916dbcf2 100644 --- a/setup.py +++ b/setup.py @@ -112,7 +112,7 @@ def str_from_file(name): license="Apache License, Version 2.0", packages=find_packages( where=".", - exclude=("tests*", "benchmarks*") + exclude=("tests*", "benchmarks*", "it*") ), include_package_data=True, # supported Python versions. This will prohibit pip (> 9.0.0) from even installing Rally on an unsupported diff --git a/tox.ini b/tox.ini index 838392ca5..c33a34cec 100644 --- a/tox.ini +++ b/tox.ini @@ -32,6 +32,7 @@ setenv = LANG=C commands = py.test --junitxml=junit-{envname}.xml + py.test -s it --junitxml=junit-{envname}-it.xml ./integration-test.sh whitelist_externals =