From 9cf742722c762f1f60a35e2afea175773dbd37a2 Mon Sep 17 00:00:00 2001 From: Chinmay Gadgil Date: Tue, 14 Nov 2023 13:07:42 -0800 Subject: [PATCH] Rebasing Rename provision config instance (#400) Signed-off-by: Chinmay Gadgil --- docs/api/execute-test.md | 10 +- it/__init__.py | 8 +- it/distribution_test.py | 4 +- it/list_test.py | 6 +- it/sources_test.py | 4 +- it/tracker_test.py | 2 +- osbenchmark/benchmark.py | 108 ++++++++--------- osbenchmark/builder/builder.py | 54 ++++----- .../opensearch_distribution_downloader.py | 8 +- .../opensearch_source_downloader.py | 18 +-- ...search_distribution_repository_provider.py | 14 +-- .../builder/installers/bare_installer.py | 6 +- .../builder/installers/docker_installer.py | 30 ++--- .../preparers/opensearch_preparer.py | 18 +-- osbenchmark/builder/java_resolver.py | 6 +- osbenchmark/builder/launcher.py | 4 +- .../builder/launchers/docker_launcher.py | 2 +- .../launchers/local_process_launcher.py | 16 +-- ...riptor.py => cluster_config_descriptor.py} | 6 +- .../builder/models/cluster_config_types.py | 6 + .../models/provision_config_instance.py | 2 +- .../models/provision_config_instance_types.py | 6 - osbenchmark/builder/provision_config.py | 86 ++++++------- osbenchmark/builder/provisioner.py | 70 +++++------ osbenchmark/builder/supplier.py | 20 +-- .../builder/utils/java_home_resolver.py | 6 +- osbenchmark/metrics.py | 96 +++++++-------- .../cluster_configs}/v1/16gheap.ini | 2 +- .../cluster_configs}/v1/1gheap.ini | 2 +- .../v1/24gheap.ini | 2 +- .../cluster_configs}/v1/2gheap.ini | 2 +- .../v1/4gheap.ini | 2 +- .../cluster_configs}/v1/8gheap.ini | 2 +- .../v1/basic-license.ini | 0 .../templates/config/jvm.options | 0 .../v1/debug-non-safepoints.ini | 0 .../v1/default_distro/config.ini | 0 .../templates/config/opensearch.yml | 0 .../v1/defaults.ini | 2 +- .../v1/ea.ini | 0 .../v1/ea/templates/config/jvm.options | 0 .../v1/fp.ini | 0 .../v1/fp/templates/config/jvm.options | 0 .../v1/g1gc.ini | 0 .../v1/trial-license.ini | 0 .../v1/unpooled.ini | 0 .../v1/unpooled/templates/config/jvm.options | 0 .../v1/vanilla/README.md | 0 .../v1/vanilla/config.ini | 0 .../v1/vanilla/templates/config/jvm.options | 0 .../templates/config/log4j2.properties | 0 .../vanilla/templates/config/opensearch.yml | 0 .../cluster_configs}/v1/16gheap.ini | 2 +- .../cluster_configs}/v1/1gheap.ini | 2 +- .../v1/24gheap.ini | 2 +- .../cluster_configs}/v1/2gheap.ini | 2 +- .../v1/4gheap.ini | 2 +- .../cluster_configs}/v1/8gheap.ini | 2 +- .../v1/basic-license.ini | 0 .../templates/config/jvm.options | 0 .../v1/debug-non-safepoints.ini | 0 .../v1/defaults.ini | 2 +- .../v1/ea.ini | 0 .../v1/ea/templates/config/jvm.options | 0 .../v1/fp.ini | 0 .../v1/fp/templates/config/jvm.options | 0 .../v1/g1gc.ini | 0 .../v1/parallelgc.ini | 0 .../v1/trial-license.ini | 0 .../v1/unpooled.ini | 0 .../v1/unpooled/templates/config/jvm.options | 0 .../v1/vanilla/README.md | 0 .../v1/vanilla/config.ini | 0 .../v1/vanilla/templates/config/jvm.options | 0 .../templates/config/log4j2.properties | 0 .../vanilla/templates/config/opensearch.yml | 0 osbenchmark/results_publisher.py | 4 +- osbenchmark/test_execution_orchestrator.py | 14 +-- .../utils/config_path_resolver_test.py | 6 +- .../v1/32gheap.ini | 2 +- .../v1/another_with_hook.ini | 2 +- .../v1/default.ini | 2 +- .../v1/ea.ini | 0 .../cluster_configs/v1/empty_cfg_base.ini | 9 ++ .../v1/hook2/config.py | 0 .../cluster_configs/v1/missing_cfg_base.ini | 9 ++ .../data/cluster_configs/v1/multi_hook.ini | 6 + .../cluster_configs/v1/vanilla/config.ini | 5 + .../v1/verbose.ini | 0 .../v1/verbose_logging/config.ini | 0 .../data/cluster_configs/v1/with_hook.ini | 6 + .../v1/with_hook/config.py | 0 .../v1/empty_cfg_base.ini | 9 -- .../v1/missing_cfg_base.ini | 9 -- .../v1/multi_hook.ini | 6 - .../v1/vanilla/config.ini | 5 - .../v1/with_hook.ini | 6 - ...opensearch_distribution_downloader_test.py | 4 +- .../opensearch_source_downloader_test.py | 4 +- ...h_distribution_repository_provider_test.py | 12 +- .../builder/installers/bare_installer_test.py | 8 +- .../installers/docker_installer_test.py | 8 +- .../preparers/opensearch_preparer_test.py | 4 +- tests/builder/launcher_test.py | 14 +-- .../builder/launchers/docker_launcher_test.py | 4 +- .../launchers/local_process_launcher_test.py | 12 +- tests/builder/provision_config_test.py | 114 +++++++++--------- tests/builder/provisioner_test.py | 16 +-- tests/builder/supplier_test.py | 30 ++--- .../builder/utils/java_home_resolver_test.py | 10 +- tests/metrics_test.py | 46 +++---- .../worker_coordinator_test.py | 2 +- 112 files changed, 516 insertions(+), 516 deletions(-) rename osbenchmark/builder/models/{provision_config_instance_descriptor.py => cluster_config_descriptor.py} (87%) create mode 100644 osbenchmark/builder/models/cluster_config_types.py delete mode 100644 osbenchmark/builder/models/provision_config_instance_types.py rename osbenchmark/resources/provision_configs/{main/provision_config_instances => 1.0/cluster_configs}/v1/16gheap.ini (75%) rename osbenchmark/resources/provision_configs/{main/provision_config_instances => 1.0/cluster_configs}/v1/1gheap.ini (75%) rename osbenchmark/resources/provision_configs/1.0/{provision_config_instances => cluster_configs}/v1/24gheap.ini (75%) rename osbenchmark/resources/provision_configs/{main/provision_config_instances => 1.0/cluster_configs}/v1/2gheap.ini (75%) rename osbenchmark/resources/provision_configs/1.0/{provision_config_instances => cluster_configs}/v1/4gheap.ini (75%) rename osbenchmark/resources/provision_configs/{main/provision_config_instances => 1.0/cluster_configs}/v1/8gheap.ini (75%) rename osbenchmark/resources/provision_configs/1.0/{provision_config_instances => cluster_configs}/v1/basic-license.ini (100%) rename osbenchmark/resources/provision_configs/1.0/{provision_config_instances => cluster_configs}/v1/dbg_non_safepoints/templates/config/jvm.options (100%) rename osbenchmark/resources/provision_configs/1.0/{provision_config_instances => cluster_configs}/v1/debug-non-safepoints.ini (100%) rename osbenchmark/resources/provision_configs/1.0/{provision_config_instances => cluster_configs}/v1/default_distro/config.ini (100%) rename osbenchmark/resources/provision_configs/1.0/{provision_config_instances => cluster_configs}/v1/default_distro/templates/config/opensearch.yml (100%) rename osbenchmark/resources/provision_configs/1.0/{provision_config_instances => cluster_configs}/v1/defaults.ini (75%) rename osbenchmark/resources/provision_configs/1.0/{provision_config_instances => cluster_configs}/v1/ea.ini (100%) rename osbenchmark/resources/provision_configs/1.0/{provision_config_instances => cluster_configs}/v1/ea/templates/config/jvm.options (100%) rename osbenchmark/resources/provision_configs/1.0/{provision_config_instances => cluster_configs}/v1/fp.ini (100%) rename osbenchmark/resources/provision_configs/1.0/{provision_config_instances => cluster_configs}/v1/fp/templates/config/jvm.options (100%) rename osbenchmark/resources/provision_configs/1.0/{provision_config_instances => cluster_configs}/v1/g1gc.ini (100%) rename osbenchmark/resources/provision_configs/1.0/{provision_config_instances => cluster_configs}/v1/trial-license.ini (100%) rename osbenchmark/resources/provision_configs/1.0/{provision_config_instances => cluster_configs}/v1/unpooled.ini (100%) rename osbenchmark/resources/provision_configs/1.0/{provision_config_instances => cluster_configs}/v1/unpooled/templates/config/jvm.options (100%) rename osbenchmark/resources/provision_configs/1.0/{provision_config_instances => cluster_configs}/v1/vanilla/README.md (100%) rename osbenchmark/resources/provision_configs/1.0/{provision_config_instances => cluster_configs}/v1/vanilla/config.ini (100%) rename osbenchmark/resources/provision_configs/1.0/{provision_config_instances => cluster_configs}/v1/vanilla/templates/config/jvm.options (100%) rename osbenchmark/resources/provision_configs/1.0/{provision_config_instances => cluster_configs}/v1/vanilla/templates/config/log4j2.properties (100%) rename osbenchmark/resources/provision_configs/1.0/{provision_config_instances => cluster_configs}/v1/vanilla/templates/config/opensearch.yml (100%) rename osbenchmark/resources/provision_configs/{1.0/provision_config_instances => main/cluster_configs}/v1/16gheap.ini (75%) rename osbenchmark/resources/provision_configs/{1.0/provision_config_instances => main/cluster_configs}/v1/1gheap.ini (75%) rename osbenchmark/resources/provision_configs/main/{provision_config_instances => cluster_configs}/v1/24gheap.ini (75%) rename osbenchmark/resources/provision_configs/{1.0/provision_config_instances => main/cluster_configs}/v1/2gheap.ini (75%) rename osbenchmark/resources/provision_configs/main/{provision_config_instances => cluster_configs}/v1/4gheap.ini (75%) rename osbenchmark/resources/provision_configs/{1.0/provision_config_instances => main/cluster_configs}/v1/8gheap.ini (75%) rename osbenchmark/resources/provision_configs/main/{provision_config_instances => cluster_configs}/v1/basic-license.ini (100%) rename osbenchmark/resources/provision_configs/main/{provision_config_instances => cluster_configs}/v1/dbg_non_safepoints/templates/config/jvm.options (100%) rename osbenchmark/resources/provision_configs/main/{provision_config_instances => cluster_configs}/v1/debug-non-safepoints.ini (100%) rename osbenchmark/resources/provision_configs/main/{provision_config_instances => cluster_configs}/v1/defaults.ini (78%) rename osbenchmark/resources/provision_configs/main/{provision_config_instances => cluster_configs}/v1/ea.ini (100%) rename osbenchmark/resources/provision_configs/main/{provision_config_instances => cluster_configs}/v1/ea/templates/config/jvm.options (100%) rename osbenchmark/resources/provision_configs/main/{provision_config_instances => cluster_configs}/v1/fp.ini (100%) rename osbenchmark/resources/provision_configs/main/{provision_config_instances => cluster_configs}/v1/fp/templates/config/jvm.options (100%) rename osbenchmark/resources/provision_configs/main/{provision_config_instances => cluster_configs}/v1/g1gc.ini (100%) rename osbenchmark/resources/provision_configs/main/{provision_config_instances => cluster_configs}/v1/parallelgc.ini (100%) rename osbenchmark/resources/provision_configs/main/{provision_config_instances => cluster_configs}/v1/trial-license.ini (100%) rename osbenchmark/resources/provision_configs/main/{provision_config_instances => cluster_configs}/v1/unpooled.ini (100%) rename osbenchmark/resources/provision_configs/main/{provision_config_instances => cluster_configs}/v1/unpooled/templates/config/jvm.options (100%) rename osbenchmark/resources/provision_configs/main/{provision_config_instances => cluster_configs}/v1/vanilla/README.md (100%) rename osbenchmark/resources/provision_configs/main/{provision_config_instances => cluster_configs}/v1/vanilla/config.ini (100%) rename osbenchmark/resources/provision_configs/main/{provision_config_instances => cluster_configs}/v1/vanilla/templates/config/jvm.options (100%) rename osbenchmark/resources/provision_configs/main/{provision_config_instances => cluster_configs}/v1/vanilla/templates/config/log4j2.properties (100%) rename osbenchmark/resources/provision_configs/main/{provision_config_instances => cluster_configs}/v1/vanilla/templates/config/opensearch.yml (100%) rename tests/builder/data/{provision_config_instances => cluster_configs}/v1/32gheap.ini (76%) rename tests/builder/data/{provision_config_instances => cluster_configs}/v1/another_with_hook.ini (56%) rename tests/builder/data/{provision_config_instances => cluster_configs}/v1/default.ini (75%) rename tests/builder/data/{provision_config_instances => cluster_configs}/v1/ea.ini (100%) create mode 100644 tests/builder/data/cluster_configs/v1/empty_cfg_base.ini rename tests/builder/data/{provision_config_instances => cluster_configs}/v1/hook2/config.py (100%) create mode 100644 tests/builder/data/cluster_configs/v1/missing_cfg_base.ini create mode 100644 tests/builder/data/cluster_configs/v1/multi_hook.ini create mode 100644 tests/builder/data/cluster_configs/v1/vanilla/config.ini rename tests/builder/data/{provision_config_instances => cluster_configs}/v1/verbose.ini (100%) rename tests/builder/data/{provision_config_instances => cluster_configs}/v1/verbose_logging/config.ini (100%) create mode 100644 tests/builder/data/cluster_configs/v1/with_hook.ini rename tests/builder/data/{provision_config_instances => cluster_configs}/v1/with_hook/config.py (100%) delete mode 100644 tests/builder/data/provision_config_instances/v1/empty_cfg_base.ini delete mode 100644 tests/builder/data/provision_config_instances/v1/missing_cfg_base.ini delete mode 100644 tests/builder/data/provision_config_instances/v1/multi_hook.ini delete mode 100644 tests/builder/data/provision_config_instances/v1/vanilla/config.ini delete mode 100644 tests/builder/data/provision_config_instances/v1/with_hook.ini diff --git a/docs/api/execute-test.md b/docs/api/execute-test.md index 6398d5e48..b2372a215 100644 --- a/docs/api/execute-test.md +++ b/docs/api/execute-test.md @@ -61,9 +61,9 @@ Execute the `nyc_taxis` workload against an existing OpenSearch cluster with the Argument | Description | Required :--- | :--- |:--- `distribution-version` | Define the version of the OpenSearch distribution to download. Check https://opensearch.org/docs/version-history/ for released versions. | No -`provision-config-path` | Define the path to the provision_config_instance and plugin configurations to use. | No -`provision-config-repository` | Define repository from where Benchmark will load provision_configs and provision_config_instances (default: `default`). | No -`provision-config-revision` | Define a specific revision in the provision_config repository that Benchmark should use. | No +`cluster-config-path` | Define the path to the cluster-configs and plugin configurations to use. | No +`cluster-config-repository` | Define repository from where Benchmark will load cluster-configs (default: `default`). | No +`cluster-config-revision` | Define a specific revision in the cluster-config repository that Benchmark should use. | No `test-execution-id` | Define a unique id for this test_execution. | No `pipeline` | Select the pipeline to run. | No `revision` | Define the source code revision for building the benchmark candidate. 'current' uses the source tree as is, 'latest' fetches the latest version on main. It is also possible to specify a commit id or an ISO timestamp. The timestamp must be specified as: "@ts" where "ts" must be a valid ISO 8601 timestamp, e.g. "@2013-07-27T10:37:00Z" (default: `current`). | No @@ -73,8 +73,8 @@ Argument | Description | Required `workload` | Define the workload to use. List possible workloads with `opensearch-benchmark list workloads`. | No `workload-params` | Define a comma-separated list of key:value pairs that are injected verbatim to the workload as variables. | No `test-procedure` | Define the test_procedure to use. List possible test_procedures for workloads with `opensearch-benchmark list workloads`. | No -`provision-config-instance` | Define the provision_config_instance to use. List possible provision_config_instances with `opensearch-benchmark list provision_config_instances` (default: `defaults`). | No -`provision-config-instance-params` | Define a comma-separated list of key:value pairs that are injected verbatim as variables for the provision_config_instance. | No +`cluster-config` | Define the cluster-config to use. List possible cluster-configs with `opensearch-benchmark list cluster-configs` (default: `defaults`). | No +`provision-config-instance-params` | Define a comma-separated list of key:value pairs that are injected verbatim as variables for the cluster-config. | No `runtime-jdk` | The major version of the runtime JDK to use. | No `opensearch-plugins` | Define the OpenSearch plugins to install. (default: install no plugins). | No `plugin-params` | Define a comma-separated list of key:value pairs that are injected verbatim to all plugins as variables. | No diff --git a/it/__init__.py b/it/__init__.py index feeda06db..b85e05d45 100644 --- a/it/__init__.py +++ b/it/__init__.py @@ -165,19 +165,19 @@ def __init__(self, cfg): self.installation_id = None self.http_port = None - def install(self, distribution_version, node_name, provision_config_instance, http_port): + def install(self, distribution_version, node_name, cluster_config, http_port): self.http_port = http_port transport_port = http_port + 100 try: err, retcode = process.run_subprocess_with_stderr( "opensearch-benchmark install --configuration-name={cfg} --distribution-version={dist} --build-type=tar " "--http-port={http_port} --node={node_name} --master-nodes=" - "{node_name} --provision-config-instance={provision_config_instance} " + "{node_name} --cluster-config={cluster_config} " "--seed-hosts=\"127.0.0.1:{transport_port}\"".format(cfg=self.cfg, dist=distribution_version, http_port=http_port, node_name=node_name, - provision_config_instance=provision_config_instance, + cluster_config=cluster_config, transport_port=transport_port)) if retcode != 0: raise AssertionError("Failed to install OpenSearch {}.".format(distribution_version), err) @@ -210,7 +210,7 @@ def __init__(self): def start(self): self.cluster.install(distribution_version=OsMetricsStore.VERSION, node_name="metrics-store", - provision_config_instance="defaults", + cluster_config="defaults", http_port=10200) self.cluster.start(test_execution_id="metrics-store") diff --git a/it/distribution_test.py b/it/distribution_test.py index aee41ce49..8eeed9390 100644 --- a/it/distribution_test.py +++ b/it/distribution_test.py @@ -32,7 +32,7 @@ def test_tar_distributions(cfg): for workload in it.WORKLOADS: it.wait_until_port_is_free(port_number=port) assert it.execute_test(cfg, f"--distribution-version=\"{dist}\" --workload=\"{workload}\" " - f"--test-mode --provision-config-instance=4gheap --target-hosts=127.0.0.1:{port}") == 0 + f"--test-mode --cluster-config=4gheap --target-hosts=127.0.0.1:{port}") == 0 @it.random_benchmark_config @@ -43,4 +43,4 @@ def test_docker_distribution(cfg): it.wait_until_port_is_free(port_number=port) assert it.execute_test(cfg, f"--pipeline=\"docker\" --distribution-version=\"{dist}\" " f"--workload=\"geonames\" --test-procedure=\"append-no-conflicts-index-only\" --test-mode " - f"--provision-config-instance=4gheap --target-hosts=127.0.0.1:{port}") == 0 + f"--cluster-config=4gheap --target-hosts=127.0.0.1:{port}") == 0 diff --git a/it/list_test.py b/it/list_test.py index b99a8468d..bf890fd8b 100644 --- a/it/list_test.py +++ b/it/list_test.py @@ -31,9 +31,9 @@ def test_list_test_executions(cfg): @it.benchmark_in_mem -def test_list_provision_config_instances(cfg): - assert it.osbenchmark(cfg, "list provision_config_instances") == 0 - assert it.osbenchmark(cfg, "list provision_config_instances --provision-config-repository=default") == 0 +def test_list_cluster_configs(cfg): + assert it.osbenchmark(cfg, "list cluster-configs") == 0 + assert it.osbenchmark(cfg, "list cluster-configs --cluster-config-repository=default") == 0 @it.benchmark_in_mem diff --git a/it/sources_test.py b/it/sources_test.py index 478fe5b4c..96c45c83f 100644 --- a/it/sources_test.py +++ b/it/sources_test.py @@ -30,9 +30,9 @@ def test_sources(cfg): it.wait_until_port_is_free(port_number=port) assert it.execute_test(cfg, f"--pipeline=from-sources --revision=latest \ --workload=geonames --test-mode --target-hosts=127.0.0.1:{port} " - f"--test-procedure=append-no-conflicts --provision-config-instance=4gheap " + f"--test-procedure=append-no-conflicts --cluster-config=4gheap " f"--opensearch-plugins=analysis-icu") == 0 it.wait_until_port_is_free(port_number=port) assert it.execute_test(cfg, f"--pipeline=from-sources --workload=geonames --test-mode --target-hosts=127.0.0.1:{port} " - f"--test-procedure=append-no-conflicts-index-only --provision-config-instance=\"4gheap,ea\"") == 0 + f"--test-procedure=append-no-conflicts-index-only --cluster-config=\"4gheap,ea\"") == 0 diff --git a/it/tracker_test.py b/it/tracker_test.py index cb407a5e1..fc3d28912 100644 --- a/it/tracker_test.py +++ b/it/tracker_test.py @@ -38,7 +38,7 @@ def test_cluster(): test_execution_id = str(uuid.uuid4()) it.wait_until_port_is_free(port_number=port) - cluster.install(distribution_version=dist, node_name="benchmark-node", provision_config_instance="4gheap", http_port=port) + cluster.install(distribution_version=dist, node_name="benchmark-node", cluster_config="4gheap", http_port=port) cluster.start(test_execution_id=test_execution_id) yield cluster cluster.stop() diff --git a/osbenchmark/benchmark.py b/osbenchmark/benchmark.py index 25c069d85..8ae8f2d52 100644 --- a/osbenchmark/benchmark.py +++ b/osbenchmark/benchmark.py @@ -118,8 +118,8 @@ def add_workload_source(subparser): "configuration", metavar="configuration", help="The configuration for which Benchmark should show the available options. " - "Possible values are: telemetry, workloads, pipelines, test_executions, provision_config_instances, opensearch-plugins", - choices=["telemetry", "workloads", "pipelines", "test_executions", "provision_config_instances", "opensearch-plugins"]) + "Possible values are: telemetry, workloads, pipelines, test_executions, cluster-configs, opensearch-plugins", + choices=["telemetry", "workloads", "pipelines", "test_executions", "cluster-configs", "opensearch-plugins"]) list_parser.add_argument( "--limit", help="Limit the number of search results for recent test_executions (default: 10).", @@ -223,16 +223,16 @@ def add_workload_source(subparser): download_parser = subparsers.add_parser("download", help="Downloads an artifact") download_parser.add_argument( - "--provision-config-repository", - help="Define the repository from where Benchmark will load provision_configs and provision_config_instances (default: default).", + "--cluster-config-repository", + help="Define the repository from where Benchmark will load cluster-configs (default: default).", default="default") download_parser.add_argument( - "--provision-config-revision", - help="Define a specific revision in the provision_config repository that Benchmark should use.", + "--cluster-config-revision", + help="Define a specific revision in the cluster-config repository that Benchmark should use.", default=None) download_parser.add_argument( - "--provision-config-path", - help="Define the path to the provision_config_instance and plugin configurations to use.") + "--cluster-config-path", + help="Define the path to the cluster-config and plugin configurations to use.") download_parser.add_argument( "--distribution-version", type=supported_os_version, @@ -244,14 +244,14 @@ def add_workload_source(subparser): help="Define the repository from where the OpenSearch distribution should be downloaded (default: release).", default="release") download_parser.add_argument( - "--provision-config-instance", - help=f"Define the provision_config_instance to use. List possible " - f"provision_config_instances with `{PROGRAM_NAME} list " - f"provision_config_instances` (default: defaults).", + "--cluster-config", + help=f"Define the cluster-config to use. List possible " + f"cluster-configs with `{PROGRAM_NAME} list " + f"cluster-configs` (default: defaults).", default="defaults") # optimized for local usage download_parser.add_argument( - "--provision-config-instance-params", - help="Define a comma-separated list of key:value pairs that are injected verbatim as variables for the provision_config_instance.", + "--cluster-config-params", + help="Define a comma-separated list of key:value pairs that are injected verbatim as variables for the cluster-config.", default="" ) download_parser.add_argument( @@ -278,16 +278,16 @@ def add_workload_source(subparser): choices=["tar", "docker"], default="tar") install_parser.add_argument( - "--provision-config-repository", - help="Define the repository from where Benchmark will load provision_configs and provision_config_instances (default: default).", + "--cluster-config-repository", + help="Define the repository from where Benchmark will load cluster-configs (default: default).", default="default") install_parser.add_argument( - "--provision-config-revision", - help="Define a specific revision in the provision_config repository that Benchmark should use.", + "--cluster-config-revision", + help="Define a specific revision in the cluster-config repository that Benchmark should use.", default=None) install_parser.add_argument( - "--provision-config-path", - help="Define the path to the provision_config_instance and plugin configurations to use.") + "--cluster-config-path", + help="Define the path to the cluster-config and plugin configurations to use.") install_parser.add_argument( "--runtime-jdk", type=runtime_jdk, @@ -304,14 +304,14 @@ def add_workload_source(subparser): "Check https://opensearch.org/docs/version-history/ for released versions.", default="") install_parser.add_argument( - "--provision-config-instance", - help=f"Define the provision_config_instance to use. List possible " - f"provision_config_instances with `{PROGRAM_NAME} list " - f"provision_config_instances` (default: defaults).", + "--cluster-config", + help=f"Define the cluster-config to use. List possible " + f"cluster-configs with `{PROGRAM_NAME} list " + f"cluster-configs` (default: defaults).", default="defaults") # optimized for local usage install_parser.add_argument( - "--provision-config-instance-params", - help="Define a comma-separated list of key:value pairs that are injected verbatim as variables for the provision_config_instance.", + "--cluster-config-params", + help="Define a comma-separated list of key:value pairs that are injected verbatim as variables for the cluster-config.", default="" ) install_parser.add_argument( @@ -402,15 +402,15 @@ def add_workload_source(subparser): "Check https://opensearch.org/docs/version-history/ for released versions.", default="") p.add_argument( - "--provision-config-path", - help="Define the path to the provision_config_instance and plugin configurations to use.") + "--cluster-config-path", + help="Define the path to the cluster-config and plugin configurations to use.") p.add_argument( - "--provision-config-repository", - help="Define repository from where Benchmark will load provision_configs and provision_config_instances (default: default).", + "--cluster-config-repository", + help="Define repository from where Benchmark will load cluster-configs (default: default).", default="default") p.add_argument( - "--provision-config-revision", - help="Define a specific revision in the provision_config repository that Benchmark should use.", + "--cluster-config-revision", + help="Define a specific revision in the cluster-configs repository that Benchmark should use.", default=None) test_execution_parser.add_argument( @@ -445,14 +445,14 @@ def add_workload_source(subparser): "--test-procedure", help=f"Define the test_procedure to use. List possible test_procedures for workloads with `{PROGRAM_NAME} list workloads`.") test_execution_parser.add_argument( - "--provision-config-instance", - help=f"Define the provision_config_instance to use. List possible " - f"provision_config_instances with `{PROGRAM_NAME} list " - f"provision_config_instances` (default: defaults).", + "--cluster-config", + help=f"Define the cluster-config to use. List possible " + f"cluster-configs with `{PROGRAM_NAME} list " + f"cluster-configs` (default: defaults).", default="defaults") # optimized for local usage test_execution_parser.add_argument( - "--provision-config-instance-params", - help="Define a comma-separated list of key:value pairs that are injected verbatim as variables for the provision_config_instance.", + "--cluster-config-params", + help="Define a comma-separated list of key:value pairs that are injected verbatim as variables for the cluster-config.", default="" ) test_execution_parser.add_argument( @@ -644,8 +644,8 @@ def dispatch_list(cfg): test_execution_orchestrator.list_pipelines() elif what == "test_executions": metrics.list_test_executions(cfg) - elif what == "provision_config_instances": - provision_config.list_provision_config_instances(cfg) + elif what == "cluster-configs": + provision_config.list_cluster_configs(cfg) elif what == "opensearch-plugins": provision_config.list_plugins(cfg) else: @@ -790,28 +790,28 @@ def configure_workload_params(arg_parser, args, cfg, command_requires_workload=T cfg.add(config.Scope.applicationOverride, "workload", "exclude.tasks", opts.csv_to_list(args.exclude_tasks)) -def configure_builder_params(args, cfg, command_requires_provision_config_instance=True): - if args.provision_config_path: +def configure_builder_params(args, cfg, command_requires_cluster_config=True): + if args.cluster_config_path: cfg.add( config.Scope.applicationOverride, "builder", - "provision_config.path", os.path.abspath( - io.normalize_path(args.provision_config_path))) + "cluster_config.path", os.path.abspath( + io.normalize_path(args.cluster_config_path))) cfg.add(config.Scope.applicationOverride, "builder", "repository.name", None) cfg.add(config.Scope.applicationOverride, "builder", "repository.revision", None) else: - cfg.add(config.Scope.applicationOverride, "builder", "repository.name", args.provision_config_repository) - cfg.add(config.Scope.applicationOverride, "builder", "repository.revision", args.provision_config_revision) + cfg.add(config.Scope.applicationOverride, "builder", "repository.name", args.cluster_config_repository) + cfg.add(config.Scope.applicationOverride, "builder", "repository.revision", args.cluster_config_revision) - if command_requires_provision_config_instance: + if command_requires_cluster_config: if args.distribution_version: cfg.add(config.Scope.applicationOverride, "builder", "distribution.version", args.distribution_version) cfg.add(config.Scope.applicationOverride, "builder", "distribution.repository", args.distribution_repository) cfg.add(config.Scope.applicationOverride, "builder", - "provision_config_instance.names", opts.csv_to_list( - args.provision_config_instance)) + "cluster_config.names", opts.csv_to_list( + args.cluster_config)) cfg.add(config.Scope.applicationOverride, "builder", - "provision_config_instance.params", opts.to_dict( - args.provision_config_instance_params)) + "cluster_config.params", opts.to_dict( + args.cluster_config_params)) def configure_connection_params(arg_parser, args, cfg): @@ -850,7 +850,7 @@ def dispatch_sub_command(arg_parser, args, cfg): elif sub_command == "list": cfg.add(config.Scope.applicationOverride, "system", "list.config.option", args.configuration) cfg.add(config.Scope.applicationOverride, "system", "list.test_executions.max_results", args.limit) - configure_builder_params(args, cfg, command_requires_provision_config_instance=False) + configure_builder_params(args, cfg, command_requires_cluster_config=False) configure_workload_params(arg_parser, args, cfg, command_requires_workload=False) dispatch_list(cfg) elif sub_command == "download": @@ -869,7 +869,7 @@ def dispatch_sub_command(arg_parser, args, cfg): cfg.add(config.Scope.applicationOverride, "builder", "master.nodes", opts.csv_to_list(args.master_nodes)) cfg.add(config.Scope.applicationOverride, "builder", "seed.hosts", opts.csv_to_list(args.seed_hosts)) cfg.add(config.Scope.applicationOverride, "builder", - "provision_config_instance.plugins", opts.csv_to_list( + "cluster_config.plugins", opts.csv_to_list( args.opensearch_plugins)) cfg.add(config.Scope.applicationOverride, "builder", "plugin.params", opts.to_dict(args.plugin_params)) configure_builder_params(args, cfg) @@ -920,7 +920,7 @@ def dispatch_sub_command(arg_parser, args, cfg): cfg.add(config.Scope.applicationOverride, "builder", "runtime.jdk", args.runtime_jdk) cfg.add(config.Scope.applicationOverride, "builder", "source.revision", args.revision) cfg.add(config.Scope.applicationOverride, "builder", - "provision_config_instance.plugins", opts.csv_to_list( + "cluster_config.plugins", opts.csv_to_list( args.opensearch_plugins)) cfg.add(config.Scope.applicationOverride, "builder", "plugin.params", opts.to_dict(args.plugin_params)) cfg.add(config.Scope.applicationOverride, "builder", "preserve.install", convert.to_bool(args.preserve_install)) diff --git a/osbenchmark/builder/builder.py b/osbenchmark/builder/builder.py index 0ba6de5b8..e5ba959de 100644 --- a/osbenchmark/builder/builder.py +++ b/osbenchmark/builder/builder.py @@ -41,16 +41,16 @@ def download(cfg): - provision_config_instance, plugins = load_provision_config(cfg, external=False) + cluster_config, plugins = load_provision_config(cfg, external=False) - s = supplier.create(cfg, sources=False, distribution=True, provision_config_instance=provision_config_instance, plugins=plugins) + s = supplier.create(cfg, sources=False, distribution=True, cluster_config=cluster_config, plugins=plugins) binaries = s() console.println(json.dumps(binaries, indent=2), force=True) def install(cfg): root_path = paths.install_root(cfg) - provision_config_instance, plugins = load_provision_config(cfg, external=False) + cluster_config, plugins = load_provision_config(cfg, external=False) # A non-empty distribution-version is provided distribution = bool(cfg.opts("builder", "distribution.version", mandatory=False)) @@ -63,8 +63,8 @@ def install(cfg): seed_hosts = cfg.opts("builder", "seed.hosts") if build_type == "tar": - binary_supplier = supplier.create(cfg, sources, distribution, provision_config_instance, plugins) - p = provisioner.local(cfg=cfg, provision_config_instance=provision_config_instance, plugins=plugins, ip=ip, http_port=http_port, + binary_supplier = supplier.create(cfg, sources, distribution, cluster_config, plugins) + p = provisioner.local(cfg=cfg, cluster_config=cluster_config, plugins=plugins, ip=ip, http_port=http_port, all_node_ips=seed_hosts, all_node_names=master_nodes, target_root=root_path, node_name=node_name) node_config = p.prepare(binary=binary_supplier()) @@ -73,7 +73,7 @@ def install(cfg): raise exceptions.SystemSetupError("You cannot specify any plugins for Docker clusters. Please remove " "\"--opensearch-plugins\" and try again.") p = provisioner.docker( - cfg=cfg, provision_config_instance=provision_config_instance, + cfg=cfg, cluster_config=cluster_config, ip=ip, http_port=http_port, target_root=root_path, node_name=node_name) # there is no binary for Docker that can be downloaded / built upfront node_config = p.prepare(binary=None) @@ -202,8 +202,8 @@ def for_nodes(self, all_node_ips=None, all_node_ids=None, ip=None, port=None, no class EngineStarted: - def __init__(self, provision_config_revision): - self.provision_config_revision = provision_config_revision + def __init__(self, cluster_config_revision): + self.cluster_config_revision = cluster_config_revision class StopEngine: @@ -326,8 +326,8 @@ def __init__(self): self.test_execution_orchestrator = None self.cluster_launcher = None self.cluster = None - self.provision_config_instance = None - self.provision_config_revision = None + self.cluster_config = None + self.cluster_config_revision = None self.externally_provisioned = False def receiveUnrecognizedMessage(self, msg, sender): @@ -356,9 +356,9 @@ def receiveMsg_StartEngine(self, msg, sender): self.logger.info("Received signal from test execution orchestrator to start engine.") self.test_execution_orchestrator = sender self.cfg = msg.cfg - self.provision_config_instance, _ = load_provision_config(self.cfg, msg.external) + self.cluster_config, _ = load_provision_config(self.cfg, msg.external) # TODO: This is implicitly set by #load_provision_config() - can we gather this elsewhere? - self.provision_config_revision = self.cfg.opts("builder", "repository.revision") + self.cluster_config_revision = self.cfg.opts("builder", "repository.revision") # In our startup procedure we first create all builders. Only if this succeeds we'll continue. hosts = self.cfg.opts("client", "hosts").default @@ -425,7 +425,7 @@ def receiveMsg_NodesStopped(self, msg, sender): self.transition_when_all_children_responded(sender, msg, "cluster_stopping", "cluster_stopped", self.on_all_nodes_stopped) def on_all_nodes_started(self): - self.send(self.test_execution_orchestrator, EngineStarted(self.provision_config_revision)) + self.send(self.test_execution_orchestrator, EngineStarted(self.cluster_config_revision)) def reset_relative_time(self): for m in self.children: @@ -610,20 +610,20 @@ def receiveUnrecognizedMessage(self, msg, sender): ##################################################### def load_provision_config(cfg, external): - # externally provisioned clusters do not support provision_config_instances / plugins + # externally provisioned clusters do not support cluster_configs / plugins if external: - provision_config_instance = None + cluster_config = None plugins = [] else: - provision_config_path = provision_config.provision_config_path(cfg) - provision_config_instance = provision_config.load_provision_config_instance( - provision_config_path, - cfg.opts("builder", "provision_config_instance.names"), - cfg.opts("builder", "provision_config_instance.params")) - plugins = provision_config.load_plugins(provision_config_path, - cfg.opts("builder", "provision_config_instance.plugins", mandatory=False), + cluster_config_path = provision_config.cluster_config_path(cfg) + cluster_config = provision_config.load_cluster_config( + cluster_config_path, + cfg.opts("builder", "cluster_config.names"), + cfg.opts("builder", "cluster_config.params")) + plugins = provision_config.load_plugins(cluster_config_path, + cfg.opts("builder", "cluster_config.plugins", mandatory=False), cfg.opts("builder", "plugin.params", mandatory=False)) - return provision_config_instance, plugins + return cluster_config, plugins def create(cfg, metrics_store, node_ip, node_http_port, all_node_ips, all_node_ids, sources=False, distribution=False, @@ -631,16 +631,16 @@ def create(cfg, metrics_store, node_ip, node_http_port, all_node_ips, all_node_i test_execution_root_path = paths.test_execution_root(cfg) node_ids = cfg.opts("provisioning", "node.ids", mandatory=False) node_name_prefix = cfg.opts("provisioning", "node.name.prefix") - provision_config_instance, plugins = load_provision_config(cfg, external) + cluster_config, plugins = load_provision_config(cfg, external) if sources or distribution: - s = supplier.create(cfg, sources, distribution, provision_config_instance, plugins) + s = supplier.create(cfg, sources, distribution, cluster_config, plugins) p = [] all_node_names = ["%s-%s" % (node_name_prefix, n) for n in all_node_ids] for node_id in node_ids: node_name = "%s-%s" % (node_name_prefix, node_id) p.append( - provisioner.local(cfg, provision_config_instance, plugins, node_ip, node_http_port, all_node_ips, + provisioner.local(cfg, cluster_config, plugins, node_ip, node_http_port, all_node_ips, all_node_names, test_execution_root_path, node_name)) l = launcher.ProcessLauncher(cfg) elif external: @@ -653,7 +653,7 @@ def create(cfg, metrics_store, node_ip, node_http_port, all_node_ips, all_node_i p = [] for node_id in node_ids: node_name = "%s-%s" % (node_name_prefix, node_id) - p.append(provisioner.docker(cfg, provision_config_instance, node_ip, node_http_port, test_execution_root_path, node_name)) + p.append(provisioner.docker(cfg, cluster_config, node_ip, node_http_port, test_execution_root_path, node_name)) l = launcher.DockerLauncher(cfg) else: # It is a programmer error (and not a user error) if this function is called with wrong parameters diff --git a/osbenchmark/builder/downloaders/opensearch_distribution_downloader.py b/osbenchmark/builder/downloaders/opensearch_distribution_downloader.py index 3576bcdd1..4af29cdd7 100644 --- a/osbenchmark/builder/downloaders/opensearch_distribution_downloader.py +++ b/osbenchmark/builder/downloaders/opensearch_distribution_downloader.py @@ -7,10 +7,10 @@ class OpenSearchDistributionDownloader(Downloader): - def __init__(self, provision_config_instance, executor, path_manager, distribution_repository_provider): + def __init__(self, cluster_config, executor, path_manager, distribution_repository_provider): super().__init__(executor) self.logger = logging.getLogger(__name__) - self.provision_config_instance = provision_config_instance + self.cluster_config = cluster_config self.path_manager = path_manager self.distribution_repository_provider = distribution_repository_provider @@ -21,7 +21,7 @@ def download(self, host): def _fetch_binary(self, host): download_url = self.distribution_repository_provider.get_download_url(host) distribution_path = self._create_distribution_path(host, download_url) - opensearch_version = self.provision_config_instance.variables["distribution"]["version"] + opensearch_version = self.cluster_config.variables["distribution"]["version"] is_binary_present = self._is_binary_present(host, distribution_path) is_cache_enabled = self.distribution_repository_provider.is_cache_enabled() @@ -35,7 +35,7 @@ def _fetch_binary(self, host): return distribution_path def _create_distribution_path(self, host, download_url): - distribution_root_path = os.path.join(self.provision_config_instance.variables["node"]["root"]["dir"], "distributions") + distribution_root_path = os.path.join(self.cluster_config.variables["node"]["root"]["dir"], "distributions") self.path_manager.create_path(host, distribution_root_path) distribution_binary_name = self.distribution_repository_provider.get_file_name_from_download_url(download_url) diff --git a/osbenchmark/builder/downloaders/opensearch_source_downloader.py b/osbenchmark/builder/downloaders/opensearch_source_downloader.py index 34c05868d..4f36c4374 100644 --- a/osbenchmark/builder/downloaders/opensearch_source_downloader.py +++ b/osbenchmark/builder/downloaders/opensearch_source_downloader.py @@ -6,11 +6,11 @@ class OpenSearchSourceDownloader(Downloader): - def __init__(self, provision_config_instance, executor, source_repository_provider, binary_builder, template_renderer, + def __init__(self, cluster_config, executor, source_repository_provider, binary_builder, template_renderer, artifact_variables_provider): super().__init__(executor) self.logger = logging.getLogger(__name__) - self.provision_config_instance = provision_config_instance + self.cluster_config = cluster_config self.source_repository_provider = source_repository_provider self.binary_builder = binary_builder self.template_renderer = template_renderer @@ -26,19 +26,19 @@ def download(self, host): return {BinaryKeys.OPENSEARCH: self._get_zip_path(opensearch_source_path, artifact_variables)} def _get_opensearch_source_path(self): - node_root_dir = self.provision_config_instance.variables["source"]["root"]["dir"] - opensearch_source_subdir = self.provision_config_instance.variables["source"]["opensearch"]["subdir"] + node_root_dir = self.cluster_config.variables["source"]["root"]["dir"] + opensearch_source_subdir = self.cluster_config.variables["source"]["opensearch"]["subdir"] return os.path.join(node_root_dir, opensearch_source_subdir) def _fetch(self, host, opensearch_source_path): - plugin_remote_url = self.provision_config_instance.variables["source"]["remote"]["repo"]["url"] - plugin_revision = self.provision_config_instance.variables["source"]["revision"] + plugin_remote_url = self.cluster_config.variables["source"]["remote"]["repo"]["url"] + plugin_revision = self.cluster_config.variables["source"]["revision"] self.source_repository_provider.fetch_repository(host, plugin_remote_url, plugin_revision, opensearch_source_path) def _prepare(self, host, artifact_variables): - clean_command_template = self.provision_config_instance.variables["source"]["clean"]["command"] - build_command_template = self.provision_config_instance.variables["source"]["build"]["command"] + clean_command_template = self.cluster_config.variables["source"]["clean"]["command"] + build_command_template = self.cluster_config.variables["source"]["build"]["command"] if self.binary_builder: self.binary_builder.build(host, [ @@ -47,7 +47,7 @@ def _prepare(self, host, artifact_variables): ]) def _get_zip_path(self, opensearch_source_path, artifact_variables): - artifact_path_pattern_template = self.provision_config_instance.variables["source"]["artifact_path_pattern"] + artifact_path_pattern_template = self.cluster_config.variables["source"]["artifact_path_pattern"] artifact_path_pattern = self.template_renderer.render_template_string(artifact_path_pattern_template, artifact_variables) return os.path.join(opensearch_source_path, artifact_path_pattern) diff --git a/osbenchmark/builder/downloaders/repositories/opensearch_distribution_repository_provider.py b/osbenchmark/builder/downloaders/repositories/opensearch_distribution_repository_provider.py index 3ee7ec76b..f1375b63d 100644 --- a/osbenchmark/builder/downloaders/repositories/opensearch_distribution_repository_provider.py +++ b/osbenchmark/builder/downloaders/repositories/opensearch_distribution_repository_provider.py @@ -4,14 +4,14 @@ class OpenSearchDistributionRepositoryProvider: - def __init__(self, provision_config_instance, repository_url_provider): + def __init__(self, cluster_config, repository_url_provider): self.logger = logging.getLogger(__name__) - self.provision_config_instance = provision_config_instance + self.cluster_config = cluster_config self.repository_url_provider = repository_url_provider def get_download_url(self, host): - is_runtime_jdk_bundled = self.provision_config_instance.variables["system"]["runtime"]["jdk"]["bundled"] - distribution_repository = self.provision_config_instance.variables["distribution"]["repository"] + is_runtime_jdk_bundled = self.cluster_config.variables["system"]["runtime"]["jdk"]["bundled"] + distribution_repository = self.cluster_config.variables["distribution"]["repository"] self.logger.info("runtime_jdk_bundled? [%s]", is_runtime_jdk_bundled) if is_runtime_jdk_bundled: @@ -20,13 +20,13 @@ def get_download_url(self, host): url_key = f"distribution.jdk.unbundled.{distribution_repository}_url" self.logger.info("key: [%s]", url_key) - return self.repository_url_provider.render_url_for_key(host, self.provision_config_instance.variables, url_key) + return self.repository_url_provider.render_url_for_key(host, self.cluster_config.variables, url_key) def get_file_name_from_download_url(self, download_url): return download_url[download_url.rfind("/") + 1:] def is_cache_enabled(self): - distribution_repository = self.provision_config_instance.variables["distribution"]["repository"] - is_cache_enabled = self.provision_config_instance.variables["distribution"][distribution_repository]["cache"] + distribution_repository = self.cluster_config.variables["distribution"]["repository"] + is_cache_enabled = self.cluster_config.variables["distribution"][distribution_repository]["cache"] return convert.to_bool(is_cache_enabled) diff --git a/osbenchmark/builder/installers/bare_installer.py b/osbenchmark/builder/installers/bare_installer.py index d0961ff68..80c5cad93 100644 --- a/osbenchmark/builder/installers/bare_installer.py +++ b/osbenchmark/builder/installers/bare_installer.py @@ -8,9 +8,9 @@ class BareInstaller(Installer): - def __init__(self, provision_config_instance, executor, preparers): + def __init__(self, cluster_config, executor, preparers): super().__init__(executor) - self.provision_config_instance = provision_config_instance + self.cluster_config = cluster_config if isinstance(preparers, list): self.preparers = preparers else: @@ -54,7 +54,7 @@ def _apply_configs(self, host, preparer_to_node, config_vars): self.config_applier.apply_configs(host, node, preparer.get_config_paths(), config_vars) def _invoke_install_hooks(self, host, config_vars): - _, java_home = self.java_home_resolver.resolve_java_home(host, self.provision_config_instance) + _, java_home = self.java_home_resolver.resolve_java_home(host, self.cluster_config) env = {} if java_home: diff --git a/osbenchmark/builder/installers/docker_installer.py b/osbenchmark/builder/installers/docker_installer.py index 080c93784..95be57e12 100644 --- a/osbenchmark/builder/installers/docker_installer.py +++ b/osbenchmark/builder/installers/docker_installer.py @@ -13,10 +13,10 @@ class DockerInstaller(Installer): - def __init__(self, provision_config_instance, executor): + def __init__(self, cluster_config, executor): super().__init__(executor) self.logger = logging.getLogger(__name__) - self.provision_config_instance = provision_config_instance + self.cluster_config = cluster_config self.template_renderer = TemplateRenderer() self.path_manager = PathManager(executor) self.config_applier = ConfigApplier(executor, self.template_renderer, self.path_manager) @@ -30,8 +30,8 @@ def install(self, host, binaries, all_node_ips): def _create_node(self): node_name = str(uuid.uuid4()) - node_port = int(self.provision_config_instance.variables["node"]["port"]) - node_root_dir = os.path.join(self.provision_config_instance.variables["test_execution_root"], node_name) + node_port = int(self.cluster_config.variables["node"]["port"]) + node_root_dir = os.path.join(self.cluster_config.variables["test_execution_root"], node_name) node_data_paths = [os.path.join(node_root_dir, "data", str(uuid.uuid4()))] node_binary_path = os.path.join(node_root_dir, "install") node_log_dir = os.path.join(node_root_dir, "logs", "server") @@ -63,11 +63,11 @@ def _prepare_node(self, host, node): def _prepare_mounts(self, host, node): config_vars = self._get_config_vars(node) - return self.config_applier.apply_configs(host, node, self.provision_config_instance.config_paths, config_vars) + return self.config_applier.apply_configs(host, node, self.cluster_config.config_paths, config_vars) def _get_config_vars(self, node): provisioner_defaults = { - "cluster_name": self.provision_config_instance.variables["cluster_name"], + "cluster_name": self.cluster_config.variables["cluster_name"], "node_name": node.name, # we bind-mount the directories below on the host to these ones. "install_root_path": "/usr/share/opensearch", @@ -83,32 +83,32 @@ def _get_config_vars(self, node): } config_vars = {} - config_vars.update(self.provision_config_instance.variables["origin"]["docker"]) + config_vars.update(self.cluster_config.variables["origin"]["docker"]) config_vars.update(provisioner_defaults) return config_vars def _get_docker_vars(self, node, mounts): docker_vars = { - "os_version": self.provision_config_instance.variables["origin"]["distribution"]["version"], - "docker_image": self.provision_config_instance.variables["origin"]["docker"]["docker_image"], + "os_version": self.cluster_config.variables["origin"]["distribution"]["version"], + "docker_image": self.cluster_config.variables["origin"]["docker"]["docker_image"], "http_port": node.port, "os_data_dir": node.data_paths[0], "os_log_dir": node.log_path, "os_heap_dump_dir": node.heap_dump_path, "mounts": mounts } - self._add_if_defined_for_provision_config_instance(docker_vars, "docker_mem_limit") - self._add_if_defined_for_provision_config_instance(docker_vars, "docker_cpu_count") + self._add_if_defined_for_cluster_config(docker_vars, "docker_mem_limit") + self._add_if_defined_for_cluster_config(docker_vars, "docker_cpu_count") return docker_vars - def _add_if_defined_for_provision_config_instance(self, variables, key): - if key in self.provision_config_instance.variables["origin"]["docker"]: - variables[key] = self.provision_config_instance.variables["origin"]["docker"][key] + def _add_if_defined_for_cluster_config(self, variables, key): + if key in self.cluster_config.variables["origin"]["docker"]: + variables[key] = self.cluster_config.variables["origin"]["docker"][key] def _render_template_from_docker_file(self, variables): compose_file = os.path.join(paths.benchmark_root(), "resources", "docker-compose.yml.j2") return self.template_renderer.render_template_file(io.dirname(compose_file), variables, compose_file) def cleanup(self, host): - self.host_cleaner.cleanup(host, self.provision_config_instance.variables["preserve_install"]) + self.host_cleaner.cleanup(host, self.cluster_config.variables["preserve_install"]) diff --git a/osbenchmark/builder/installers/preparers/opensearch_preparer.py b/osbenchmark/builder/installers/preparers/opensearch_preparer.py index 677714f0e..01cafa628 100644 --- a/osbenchmark/builder/installers/preparers/opensearch_preparer.py +++ b/osbenchmark/builder/installers/preparers/opensearch_preparer.py @@ -10,11 +10,11 @@ class OpenSearchPreparer(Preparer): - def __init__(self, provision_config_instance, executor, hook_handler_class): + def __init__(self, cluster_config, executor, hook_handler_class): super().__init__(executor) self.logger = logging.getLogger(__name__) - self.provision_config_instance = provision_config_instance - self.hook_handler = hook_handler_class(self.provision_config_instance) + self.cluster_config = cluster_config + self.hook_handler = hook_handler_class(self.cluster_config) if self.hook_handler.can_load(): self.hook_handler.load() self.path_manager = PathManager(executor) @@ -28,8 +28,8 @@ def prepare(self, host, binaries): def _create_node(self): node_name = str(uuid.uuid4()) - node_port = int(self.provision_config_instance.variables["node"]["port"]) - node_root_dir = os.path.join(self.provision_config_instance.variables["test_execution_root"], node_name) + node_port = int(self.cluster_config.variables["node"]["port"]) + node_root_dir = os.path.join(self.cluster_config.variables["test_execution_root"], node_name) node_binary_path = os.path.join(node_root_dir, "install") node_log_dir = os.path.join(node_root_dir, "logs", "server") node_heap_dump_dir = os.path.join(node_root_dir, "heapdump") @@ -74,7 +74,7 @@ def _delete_prebundled_config_files(self, host, node): def get_config_vars(self, host, node, all_node_ips): installer_defaults = { - "cluster_name": self.provision_config_instance.variables["cluster_name"], + "cluster_name": self.cluster_config.variables["cluster_name"], "node_name": node.name, "data_paths": node.data_paths[0], "log_path": node.log_path, @@ -91,15 +91,15 @@ def get_config_vars(self, host, node, all_node_ips): "install_root_path": node.binary_path } config_vars = {} - config_vars.update(self.provision_config_instance.variables) + config_vars.update(self.cluster_config.variables) config_vars.update(installer_defaults) return config_vars def get_config_paths(self): - return self.provision_config_instance.config_paths + return self.cluster_config.config_paths def invoke_install_hook(self, host, phase, variables, env): self.hook_handler.invoke(phase.name, variables=variables, env=env) def cleanup(self, host): - self.host_cleaner.cleanup(host, self.provision_config_instance.variables["preserve_install"]) + self.host_cleaner.cleanup(host, self.cluster_config.variables["preserve_install"]) diff --git a/osbenchmark/builder/java_resolver.py b/osbenchmark/builder/java_resolver.py index 60cb06c7a..0b4f1dcdc 100644 --- a/osbenchmark/builder/java_resolver.py +++ b/osbenchmark/builder/java_resolver.py @@ -28,7 +28,7 @@ from osbenchmark.utils import jvm, sysstats -def java_home(provision_config_instance_runtime_jdks, specified_runtime_jdk=None, provides_bundled_jdk=False): +def java_home(cluster_config_runtime_jdks, specified_runtime_jdk=None, provides_bundled_jdk=False): def determine_runtime_jdks(): if specified_runtime_jdk: return [specified_runtime_jdk] @@ -43,12 +43,12 @@ def detect_jdk(jdks): logger = logging.getLogger(__name__) try: - allowed_runtime_jdks = [int(v) for v in provision_config_instance_runtime_jdks.split(",")] + allowed_runtime_jdks = [int(v) for v in cluster_config_runtime_jdks.split(",")] except ValueError: raise exceptions.SystemSetupError( "ProvisionConfigInstance config key \"runtime.jdk\" is invalid: \"{}\" (must be int)".format( - provision_config_instance_runtime_jdks)) + cluster_config_runtime_jdks)) runtime_jdk_versions = determine_runtime_jdks() diff --git a/osbenchmark/builder/launcher.py b/osbenchmark/builder/launcher.py index 8c29ed882..26b1945d4 100644 --- a/osbenchmark/builder/launcher.py +++ b/osbenchmark/builder/launcher.py @@ -146,9 +146,9 @@ def _start_node(self, node_configuration, node_count_on_host): data_paths = node_configuration.data_paths node_telemetry_dir = os.path.join(node_configuration.node_root_path, "telemetry") - java_major_version, java_home = java_resolver.java_home(node_configuration.provision_config_instance_runtime_jdks, + java_major_version, java_home = java_resolver.java_home(node_configuration.cluster_config_runtime_jdks, self.cfg.opts("builder", "runtime.jdk"), - node_configuration.provision_config_instance_provides_bundled_jdk) + node_configuration.cluster_config_provides_bundled_jdk) self.logger.info("Java major version: %s", java_major_version) self.logger.info("Java home: %s", java_home) diff --git a/osbenchmark/builder/launchers/docker_launcher.py b/osbenchmark/builder/launchers/docker_launcher.py index a191fbf21..b7c050f44 100644 --- a/osbenchmark/builder/launchers/docker_launcher.py +++ b/osbenchmark/builder/launchers/docker_launcher.py @@ -12,7 +12,7 @@ class DockerLauncher(Launcher): CONTAINER_WAIT_TIMEOUT_SECONDS = 10 * 60 CONTAINER_WAIT_INTERVAL_SECONDS = 0.5 - def __init__(self, provision_config_instance, shell_executor, metrics_store, clock=time.Clock): + def __init__(self, cluster_config, shell_executor, metrics_store, clock=time.Clock): super().__init__(shell_executor) self.logger = logging.getLogger(__name__) self.metrics_store = metrics_store diff --git a/osbenchmark/builder/launchers/local_process_launcher.py b/osbenchmark/builder/launchers/local_process_launcher.py index 9e143248d..0e0b9085a 100644 --- a/osbenchmark/builder/launchers/local_process_launcher.py +++ b/osbenchmark/builder/launchers/local_process_launcher.py @@ -16,10 +16,10 @@ class LocalProcessLauncher(Launcher): PROCESS_WAIT_TIMEOUT_SECONDS = 90 PROCESS_WAIT_INTERVAL_SECONDS = 0.5 - def __init__(self, provision_config_instance, shell_executor, metrics_store, clock=time.Clock): + def __init__(self, cluster_config, shell_executor, metrics_store, clock=time.Clock): super().__init__(shell_executor) self.logger = logging.getLogger(__name__) - self.provision_config_instance = provision_config_instance + self.cluster_config = cluster_config self.metrics_store = metrics_store self.waiter = PeriodicWaiter(LocalProcessLauncher.PROCESS_WAIT_INTERVAL_SECONDS, LocalProcessLauncher.PROCESS_WAIT_TIMEOUT_SECONDS, clock=clock) @@ -33,9 +33,9 @@ def _start_node(self, host, node_configuration, node_count_on_host): node_name = node_configuration.node_name binary_path = node_configuration.binary_path - java_major_version, java_home = java_resolver.java_home(node_configuration.provision_config_instance_runtime_jdks, - self.provision_config_instance.variables["system"]["runtime"]["jdk"], - node_configuration.provision_config_instance_provides_bundled_jdk) + java_major_version, java_home = java_resolver.java_home(node_configuration.cluster_config_runtime_jdks, + self.cluster_config.variables["system"]["runtime"]["jdk"], + node_configuration.cluster_config_provides_bundled_jdk) self.logger.info("Java major version: %s", java_major_version) self.logger.info("Java home: %s", java_home) self.logger.info("Starting node [%s].", node_name) @@ -57,8 +57,8 @@ def _prepare_telemetry(self, node_configuration, node_count_on_host, java_major_ data_paths = node_configuration.data_paths node_telemetry_dir = os.path.join(node_configuration.node_root_path, "telemetry") - enabled_devices = self.provision_config_instance.variables["telemetry"]["devices"] - telemetry_params = self.provision_config_instance.variables["telemetry"]["params"] + enabled_devices = self.cluster_config.variables["telemetry"]["devices"] + telemetry_params = self.cluster_config.variables["telemetry"]["params"] node_telemetry = [ telemetry.FlightRecorder(telemetry_params, node_telemetry_dir, java_major_version), @@ -74,7 +74,7 @@ def _prepare_telemetry(self, node_configuration, node_count_on_host, java_major_ def _prepare_env(self, node_name, java_home, telemetry): env = {k: v for k, v in os.environ.items() if k in - opts.csv_to_list(self.provision_config_instance.variables["system"]["env"]["passenv"])} + opts.csv_to_list(self.cluster_config.variables["system"]["env"]["passenv"])} if java_home: self._set_env(env, "PATH", os.path.join(java_home, "bin"), separator=os.pathsep, prepend=True) # This property is the higher priority starting in ES 7.12.0, and is the only supported java home in >=8.0 diff --git a/osbenchmark/builder/models/provision_config_instance_descriptor.py b/osbenchmark/builder/models/cluster_config_descriptor.py similarity index 87% rename from osbenchmark/builder/models/provision_config_instance_descriptor.py rename to osbenchmark/builder/models/cluster_config_descriptor.py index 504e24d8f..ce49bfe45 100644 --- a/osbenchmark/builder/models/provision_config_instance_descriptor.py +++ b/osbenchmark/builder/models/cluster_config_descriptor.py @@ -3,11 +3,11 @@ from osbenchmark.builder.models.cluster_flavors import ClusterFlavor from osbenchmark.builder.models.cluster_infra_providers import ClusterInfraProvider -from osbenchmark.builder.models.provision_config_instance_types import ProvisionConfigInstanceType +from osbenchmark.builder.models.cluster_config_types import ClusterConfigType @dataclass -class ProvisionConfigInstanceDescriptor: +class ClusterConfigDescriptor: """ A ProvisionConfigInstanceDescriptor represents a single source of provision config definition. These descriptors serve as an intermediary store of the cluster to be provisioned. Descriptors are created from each config source and played @@ -25,7 +25,7 @@ class ProvisionConfigInstanceDescriptor: name: str description: str = "" - type: ProvisionConfigInstanceType = ProvisionConfigInstanceType.PROVISION_CONFIG_INSTANCE + type: ClusterConfigType = ClusterConfigType.PROVISION_CONFIG_INSTANCE root_paths: List[str] = field(default_factory=list) provider: ClusterInfraProvider = None flavor: ClusterFlavor = None diff --git a/osbenchmark/builder/models/cluster_config_types.py b/osbenchmark/builder/models/cluster_config_types.py new file mode 100644 index 000000000..b36191f82 --- /dev/null +++ b/osbenchmark/builder/models/cluster_config_types.py @@ -0,0 +1,6 @@ +from enum import Enum + + +class ClusterConfigType(str, Enum): + PROVISION_CONFIG_INSTANCE = "cluster-config" + MIXIN = "mixin" diff --git a/osbenchmark/builder/models/provision_config_instance.py b/osbenchmark/builder/models/provision_config_instance.py index c417b9d38..00ed37332 100644 --- a/osbenchmark/builder/models/provision_config_instance.py +++ b/osbenchmark/builder/models/provision_config_instance.py @@ -12,7 +12,7 @@ class ProvisionConfigInstance: """ Creates new settings for a benchmark candidate. - :param names: Descriptive name(s) for this provision_config_instance. + :param names: Descriptive name(s) for this cluster-config. :param root_path: The root path from which bootstrap hooks should be loaded if any. May be ``None``. :param provider: The infrastructure provider for the cluster :param flavor: The flavor of cluster to be provisioned diff --git a/osbenchmark/builder/models/provision_config_instance_types.py b/osbenchmark/builder/models/provision_config_instance_types.py deleted file mode 100644 index 6badfdb7c..000000000 --- a/osbenchmark/builder/models/provision_config_instance_types.py +++ /dev/null @@ -1,6 +0,0 @@ -from enum import Enum - - -class ProvisionConfigInstanceType(str, Enum): - PROVISION_CONFIG_INSTANCE = "provision-config-instance" - MIXIN = "mixin" diff --git a/osbenchmark/builder/provision_config.py b/osbenchmark/builder/provision_config.py index 6bf10f82a..4c557587c 100644 --- a/osbenchmark/builder/provision_config.py +++ b/osbenchmark/builder/provision_config.py @@ -44,21 +44,21 @@ def _path_for(provision_config_root_path, provision_config_member_type): return root_path -def list_provision_config_instances(cfg): - loader = ProvisionConfigInstanceLoader(provision_config_path(cfg)) - provision_config_instances = [] - for name in loader.provision_config_instance_names(): - provision_config_instances.append(loader.load_provision_config_instance(name)) +def list_cluster_configs(cfg): + loader = ProvisionConfigInstanceLoader(cluster_config_path(cfg)) + cluster_configs = [] + for name in loader.cluster_config_names(): + cluster_configs.append(loader.load_cluster_config(name)) # first by type, then by name (we need to run the sort in reverse for that) # idiomatic way according to https://docs.python.org/3/howto/sorting.html#sort-stability-and-complex-sorts - provision_config_instances = sorted(sorted(provision_config_instances, key=lambda c: c.name), key=lambda c: c.type) - console.println("Available provision_config_instances:\n") + cluster_configs = sorted(sorted(cluster_configs, key=lambda c: c.name), key=lambda c: c.type) + console.println("Available cluster-configs:\n") console.println(tabulate.tabulate( - [[c.name, c.type, c.description] for c in provision_config_instances], + [[c.name, c.type, c.description] for c in cluster_configs], headers=["Name", "Type", "Description"])) -def load_provision_config_instance(repo, name, provision_config_instance_params=None): +def load_cluster_config(repo, name, cluster_config_params=None): class Component: def __init__(self, root_path, entry_point): self.root_path = root_path @@ -68,10 +68,10 @@ def __init__(self, root_path, entry_point): # preserve order as we append to existing config files later during provisioning. all_config_paths = [] all_config_base_vars = {} - all_provision_config_instance_vars = {} + all_cluster_config_vars = {} for n in name: - descriptor = ProvisionConfigInstanceLoader(repo).load_provision_config_instance(n, provision_config_instance_params) + descriptor = ProvisionConfigInstanceLoader(repo).load_cluster_config(n, cluster_config_params) for p in descriptor.config_paths: if p not in all_config_paths: all_config_paths.append(p) @@ -80,25 +80,25 @@ def __init__(self, root_path, entry_point): if BootstrapHookHandler(Component(root_path=p, entry_point=ProvisionConfigInstance.entry_point)).can_load(): if not root_path: root_path = p - # multiple provision_config_instances are based on the same hook + # multiple cluster_configs are based on the same hook elif root_path != p: raise exceptions.SystemSetupError( - "Invalid provision_config_instance: {}. Multiple bootstrap hooks are forbidden.".format(name)) + "Invalid cluster_config: {}. Multiple bootstrap hooks are forbidden.".format(name)) all_config_base_vars.update(descriptor.config_base_variables) - all_provision_config_instance_vars.update(descriptor.variables) + all_cluster_config_vars.update(descriptor.variables) if len(all_config_paths) == 0: - raise exceptions.SystemSetupError("At least one config base is required for provision_config_instance {}".format(name)) + raise exceptions.SystemSetupError("At least one config base is required for cluster_config {}".format(name)) variables = {} - # provision_config_instance variables *always* take precedence over config base variables + # cluster_config variables *always* take precedence over config base variables variables.update(all_config_base_vars) - variables.update(all_provision_config_instance_vars) + variables.update(all_cluster_config_vars) return ProvisionConfigInstance(name, root_path, all_config_paths, variables=variables) def list_plugins(cfg): - plugins = PluginLoader(provision_config_path(cfg)).plugins() + plugins = PluginLoader(cluster_config_path(cfg)).plugins() if plugins: console.println("Available OpenSearch plugins:\n") console.println(tabulate.tabulate([[p.name, p.config] for p in plugins], headers=["Name", "Configuration"])) @@ -128,7 +128,7 @@ def name_and_config(p): return plugins -def provision_config_path(cfg): +def cluster_config_path(cfg): root_path = cfg.opts("builder", "provision_config.path", mandatory=False) if root_path: return root_path @@ -152,40 +152,40 @@ def provision_config_path(cfg): class ProvisionConfigInstanceLoader: def __init__(self, provision_config_root_path): - self.provision_config_instances_dir = _path_for(provision_config_root_path, "provision_config_instances") + self.cluster_configs_dir = _path_for(provision_config_root_path, "cluster_configs") self.logger = logging.getLogger(__name__) - def provision_config_instance_names(self): - def __provision_config_instance_name(path): + def cluster_config_names(self): + def __cluster_config_name(path): p, _ = io.splitext(path) return io.basename(p) - def __is_provision_config_instance(path): + def __is_cluster_config(path): _, extension = io.splitext(path) return extension == ".ini" - return map(__provision_config_instance_name, filter( - __is_provision_config_instance, - os.listdir(self.provision_config_instances_dir))) + return map(__cluster_config_name, filter( + __is_cluster_config, + os.listdir(self.cluster_configs_dir))) - def _provision_config_instance_file(self, name): - return os.path.join(self.provision_config_instances_dir, "{}.ini".format(name)) + def _cluster_config_file(self, name): + return os.path.join(self.cluster_configs_dir, "{}.ini".format(name)) - def load_provision_config_instance(self, name, provision_config_instance_params=None): - provision_config_instance_config_file = self._provision_config_instance_file(name) - if not io.exists(provision_config_instance_config_file): + def load_cluster_config(self, name, cluster_config_params=None): + cluster_config_config_file = self._cluster_config_file(name) + if not io.exists(cluster_config_config_file): raise exceptions.SystemSetupError( - "Unknown provision_config_instance [{}]. List the available " - "provision_config_instances with {} list provision_config_instances.".format(name, PROGRAM_NAME)) - config = self._config_loader(provision_config_instance_config_file) + "Unknown cluster-config [{}]. List the available " + "cluster-configs with {} list cluster-configs.".format(name, PROGRAM_NAME)) + config = self._config_loader(cluster_config_config_file) root_paths = [] config_paths = [] config_base_vars = {} description = self._value(config, ["meta", "description"], default="") - provision_config_instance_type = self._value(config, ["meta", "type"], default="provision-config-instance") + cluster_config_type = self._value(config, ["meta", "type"], default="provision-config-instance") config_bases = self._value(config, ["config", "base"], default="").split(",") for base in config_bases: if base: - root_path = os.path.join(self.provision_config_instances_dir, base) + root_path = os.path.join(self.cluster_configs_dir, base) root_paths.append(root_path) config_paths.append(os.path.join(root_path, "templates")) config_file = os.path.join(root_path, "config.ini") @@ -193,16 +193,16 @@ def load_provision_config_instance(self, name, provision_config_instance_params= base_config = self._config_loader(config_file) self._copy_section(base_config, "variables", config_base_vars) - # it's possible that some provision_config_instances don't have a config base, e.g. mixins which only override variables + # it's possible that some cluster_configs don't have a config base, e.g. mixins which only override variables if len(config_paths) == 0: self.logger.info("ProvisionConfigInstance [%s] does not define any config paths. Assuming that it is used as a mixin.", name) variables = self._copy_section(config, "variables", {}) - # add all provision_config_instance params here to override any defaults - if provision_config_instance_params: - variables.update(provision_config_instance_params) + # add all cluster_config params here to override any defaults + if cluster_config_params: + variables.update(cluster_config_params) return ProvisionConfigInstanceDescriptor( - name, description, provision_config_instance_type, + name, description, cluster_config_type, root_paths, config_paths, config_base_vars, variables) def _config_loader(self, file_name): @@ -247,7 +247,7 @@ def __eq__(self, other): class ProvisionConfigInstance: - # name of the initial Python file to load for provision_config_instances. + # name of the initial Python file to load for cluster_configs. entry_point = "config" def __init__(self, names, root_path, config_paths, provider=ClusterInfraProvider.LOCAL, @@ -255,7 +255,7 @@ def __init__(self, names, root_path, config_paths, provider=ClusterInfraProvider """ Creates new settings for a benchmark candidate. - :param names: Descriptive name(s) for this provision_config_instance. + :param names: Descriptive name(s) for this cluster_config. :param root_path: The root path from which bootstrap hooks should be loaded if any. May be ``None``. :param config_paths: A non-empty list of paths where the raw config can be found. ;param provider: The infrastructure provider for the cluster diff --git a/osbenchmark/builder/provisioner.py b/osbenchmark/builder/provisioner.py index 16765efa9..452c2b259 100644 --- a/osbenchmark/builder/provisioner.py +++ b/osbenchmark/builder/provisioner.py @@ -37,40 +37,40 @@ from osbenchmark.utils import console, convert, io, process, versions -def local(cfg, provision_config_instance, plugins, ip, http_port, all_node_ips, all_node_names, target_root, node_name): +def local(cfg, cluster_config, plugins, ip, http_port, all_node_ips, all_node_names, target_root, node_name): distribution_version = cfg.opts("builder", "distribution.version", mandatory=False) node_root_dir = os.path.join(target_root, node_name) - runtime_jdk_bundled = convert.to_bool(provision_config_instance.mandatory_var("runtime.jdk.bundled")) - runtime_jdk = provision_config_instance.mandatory_var("runtime.jdk") + runtime_jdk_bundled = convert.to_bool(cluster_config.mandatory_var("runtime.jdk.bundled")) + runtime_jdk = cluster_config.mandatory_var("runtime.jdk") _, java_home = java_resolver.java_home(runtime_jdk, cfg.opts("builder", "runtime.jdk"), runtime_jdk_bundled) os_installer = OpenSearchInstaller( - provision_config_instance, java_home, node_name, + cluster_config, java_home, node_name, node_root_dir, all_node_ips, all_node_names, ip, http_port) plugin_installers = [PluginInstaller(plugin, java_home) for plugin in plugins] return BareProvisioner(os_installer, plugin_installers, distribution_version=distribution_version) -def docker(cfg, provision_config_instance, ip, http_port, target_root, node_name): +def docker(cfg, cluster_config, ip, http_port, target_root, node_name): distribution_version = cfg.opts("builder", "distribution.version", mandatory=False) benchmark_root = cfg.opts("node", "benchmark.root") node_root_dir = os.path.join(target_root, node_name) - return DockerProvisioner(provision_config_instance, node_name, ip, http_port, node_root_dir, distribution_version, benchmark_root) + return DockerProvisioner(cluster_config, node_name, ip, http_port, node_root_dir, distribution_version, benchmark_root) class NodeConfiguration: - def __init__(self, build_type, provision_config_instance_runtime_jdks, \ - provision_config_instance_provides_bundled_jdk, ip, node_name, \ + def __init__(self, build_type, cluster_config_runtime_jdks, \ + cluster_config_provides_bundled_jdk, ip, node_name, \ node_root_path, binary_path, data_paths): self.build_type = build_type - self.provision_config_instance_runtime_jdks = provision_config_instance_runtime_jdks - self.provision_config_instance_provides_bundled_jdk = provision_config_instance_provides_bundled_jdk + self.cluster_config_runtime_jdks = cluster_config_runtime_jdks + self.cluster_config_provides_bundled_jdk = cluster_config_provides_bundled_jdk self.ip = ip self.node_name = node_name self.node_root_path = node_root_path @@ -80,8 +80,8 @@ def __init__(self, build_type, provision_config_instance_runtime_jdks, \ def as_dict(self): return { "build-type": self.build_type, - "provision-config-instance-runtime-jdks": self.provision_config_instance_runtime_jdks, - "provision-config-instance-provides-bundled-jdk": self.provision_config_instance_provides_bundled_jdk, + "provision-config-instance-runtime-jdks": self.cluster_config_runtime_jdks, + "provision-config-instance-provides-bundled-jdk": self.cluster_config_provides_bundled_jdk, "ip": self.ip, "node-name": self.node_name, "node-root-path": self.node_root_path, @@ -206,8 +206,8 @@ def prepare(self, binary): for installer in self.plugin_installers: installer.invoke_install_hook(provision_config.BootstrapPhase.post_install, provisioner_vars.copy()) - return NodeConfiguration("tar", self.os_installer.provision_config_instance.mandatory_var("runtime.jdk"), - convert.to_bool(self.os_installer.provision_config_instance.mandatory_var("runtime.jdk.bundled")), + return NodeConfiguration("tar", self.os_installer.cluster_config.mandatory_var("runtime.jdk"), + convert.to_bool(self.os_installer.cluster_config.mandatory_var("runtime.jdk.bundled")), self.os_installer.node_ip, self.os_installer.node_name, self.os_installer.node_root_dir, self.os_installer.os_home_path, self.os_installer.data_paths) @@ -241,9 +241,9 @@ def _provisioner_variables(self): class OpenSearchInstaller: - def __init__(self, provision_config_instance, java_home, node_name, node_root_dir, all_node_ips, all_node_names, ip, http_port, + def __init__(self, cluster_config, java_home, node_name, node_root_dir, all_node_ips, all_node_names, ip, http_port, hook_handler_class=provision_config.BootstrapHookHandler): - self.provision_config_instance = provision_config_instance + self.cluster_config = cluster_config self.java_home = java_home self.node_name = node_name self.node_root_dir = node_root_dir @@ -254,7 +254,7 @@ def __init__(self, provision_config_instance, java_home, node_name, node_root_di self.all_node_names = all_node_names self.node_ip = ip self.http_port = http_port - self.hook_handler = hook_handler_class(self.provision_config_instance) + self.hook_handler = hook_handler_class(self.cluster_config) if self.hook_handler.can_load(): self.hook_handler.load() self.os_home_path = None @@ -307,17 +307,17 @@ def variables(self): "install_root_path": self.os_home_path } variables = {} - variables.update(self.provision_config_instance.variables) + variables.update(self.cluster_config.variables) variables.update(defaults) return variables @property def config_source_paths(self): - return self.provision_config_instance.config_paths + return self.cluster_config.config_paths def _data_paths(self): - if "data_paths" in self.provision_config_instance.variables: - data_paths = self.provision_config_instance.variables["data_paths"] + if "data_paths" in self.cluster_config.variables: + data_paths = self.cluster_config.variables["data_paths"] if isinstance(data_paths, str): return [data_paths] elif isinstance(data_paths, list): @@ -388,8 +388,8 @@ def sub_plugin_name(self): class DockerProvisioner: - def __init__(self, provision_config_instance, node_name, ip, http_port, node_root_dir, distribution_version, benchmark_root): - self.provision_config_instance = provision_config_instance + def __init__(self, cluster_config, node_name, ip, http_port, node_root_dir, distribution_version, benchmark_root): + self.cluster_config = cluster_config self.node_name = node_name self.node_ip = ip self.http_port = http_port @@ -420,7 +420,7 @@ def __init__(self, provision_config_instance, node_name, ip, http_port, node_roo } self.config_vars = {} - self.config_vars.update(self.provision_config_instance.variables) + self.config_vars.update(self.cluster_config.variables) self.config_vars.update(provisioner_defaults) def prepare(self, binary): @@ -439,11 +439,11 @@ def prepare(self, binary): mounts = {} - for provision_config_instance_config_path in self.provision_config_instance.config_paths: - for root, _, files in os.walk(provision_config_instance_config_path): + for cluster_config_config_path in self.cluster_config.config_paths: + for root, _, files in os.walk(cluster_config_config_path): env = jinja2.Environment(loader=jinja2.FileSystemLoader(root), autoescape=select_autoescape(['html', 'xml'])) - relative_root = root[len(provision_config_instance_config_path) + 1:] + relative_root = root[len(cluster_config_config_path) + 1:] absolute_target_root = os.path.join(self.binary_path, relative_root) io.ensure_dir(absolute_target_root) @@ -465,27 +465,27 @@ def prepare(self, binary): with open(os.path.join(self.binary_path, "docker-compose.yml"), mode="wt", encoding="utf-8") as f: f.write(docker_cfg) - return NodeConfiguration("docker", self.provision_config_instance.mandatory_var("runtime.jdk"), - convert.to_bool(self.provision_config_instance.mandatory_var("runtime.jdk.bundled")), self.node_ip, + return NodeConfiguration("docker", self.cluster_config.mandatory_var("runtime.jdk"), + convert.to_bool(self.cluster_config.mandatory_var("runtime.jdk.bundled")), self.node_ip, self.node_name, self.node_root_dir, self.binary_path, self.data_paths) def docker_vars(self, mounts): v = { "os_version": self.distribution_version, - "docker_image": self.provision_config_instance.mandatory_var("docker_image"), + "docker_image": self.cluster_config.mandatory_var("docker_image"), "http_port": self.http_port, "os_data_dir": self.data_paths[0], "os_log_dir": self.node_log_dir, "os_heap_dump_dir": self.heap_dump_dir, "mounts": mounts } - self._add_if_defined_for_provision_config_instance(v, "docker_mem_limit") - self._add_if_defined_for_provision_config_instance(v, "docker_cpu_count") + self._add_if_defined_for_cluster_config(v, "docker_mem_limit") + self._add_if_defined_for_cluster_config(v, "docker_cpu_count") return v - def _add_if_defined_for_provision_config_instance(self, variables, key): - if key in self.provision_config_instance.variables: - variables[key] = self.provision_config_instance.variables[key] + def _add_if_defined_for_cluster_config(self, variables, key): + if key in self.cluster_config.variables: + variables[key] = self.cluster_config.variables[key] def _render_template(self, loader, template_name, variables): try: diff --git a/osbenchmark/builder/supplier.py b/osbenchmark/builder/supplier.py index cd1920c4a..844899848 100644 --- a/osbenchmark/builder/supplier.py +++ b/osbenchmark/builder/supplier.py @@ -39,7 +39,7 @@ REVISION_PATTERN = r"(\w.*?):(.*)" -def create(cfg, sources, distribution, provision_config_instance, plugins=None): +def create(cfg, sources, distribution, cluster_config, plugins=None): logger = logging.getLogger(__name__) if plugins is None: plugins = [] @@ -58,7 +58,7 @@ def create(cfg, sources, distribution, provision_config_instance, plugins=None): template_renderer = TemplateRenderer(version=os_version, os_name=target_os, arch=target_arch) if build_needed: - raw_build_jdk = provision_config_instance.mandatory_var("build.jdk") + raw_build_jdk = cluster_config.mandatory_var("build.jdk") try: build_jdk = int(raw_build_jdk) except ValueError: @@ -71,8 +71,8 @@ def create(cfg, sources, distribution, provision_config_instance, plugins=None): distributions_root = os.path.join(cfg.opts("node", "root.dir"), cfg.opts("source", "distribution.dir")) dist_cfg = {} - # provision_config_instance / plugin defines defaults... - dist_cfg.update(provision_config_instance.variables) + # cluster_config / plugin defines defaults... + dist_cfg.update(cluster_config.variables) for plugin in plugins: for k, v in plugin.variables.items(): dist_cfg["plugin_{}_{}".format(plugin.name, k)] = v @@ -97,7 +97,7 @@ def create(cfg, sources, distribution, provision_config_instance, plugins=None): source_supplier = OpenSearchSourceSupplier(os_version, os_src_dir, remote_url=cfg.opts("source", "remote.repo.url"), - provision_config_instance=provision_config_instance, + cluster_config=cluster_config, builder=builder, template_renderer=template_renderer) @@ -387,11 +387,11 @@ def add(self, binaries): class OpenSearchSourceSupplier: - def __init__(self, revision, os_src_dir, remote_url, provision_config_instance, builder, template_renderer): + def __init__(self, revision, os_src_dir, remote_url, cluster_config, builder, template_renderer): self.revision = revision self.src_dir = os_src_dir self.remote_url = remote_url - self.provision_config_instance = provision_config_instance + self.cluster_config = cluster_config self.builder = builder self.template_renderer = template_renderer @@ -401,8 +401,8 @@ def fetch(self): def prepare(self): if self.builder: self.builder.build([ - self.template_renderer.render(self.provision_config_instance.mandatory_var("clean_command")), - self.template_renderer.render(self.provision_config_instance.mandatory_var("system.build_command")) + self.template_renderer.render(self.cluster_config.mandatory_var("clean_command")), + self.template_renderer.render(self.cluster_config.mandatory_var("system.build_command")) ]) def add(self, binaries): @@ -411,7 +411,7 @@ def add(self, binaries): def resolve_binary(self): try: path = os.path.join(self.src_dir, - self.template_renderer.render(self.provision_config_instance.mandatory_var("system.artifact_path_pattern"))) + self.template_renderer.render(self.cluster_config.mandatory_var("system.artifact_path_pattern"))) return glob.glob(path)[0] except IndexError: raise SystemSetupError("Couldn't find a tar.gz distribution. Please run Benchmark with the pipeline 'from-sources'.") diff --git a/osbenchmark/builder/utils/java_home_resolver.py b/osbenchmark/builder/utils/java_home_resolver.py index f48e5edef..ab39c2578 100644 --- a/osbenchmark/builder/utils/java_home_resolver.py +++ b/osbenchmark/builder/utils/java_home_resolver.py @@ -10,9 +10,9 @@ def __init__(self, executor): self.executor = executor self.jdk_resolver = JdkResolver(executor) - def resolve_java_home(self, host, provision_config_instance): - is_runtime_jdk_bundled = provision_config_instance.variables["system"]["runtime"]["jdk"]["bundled"] - runtime_jdks = provision_config_instance.variables["system"]["runtime"]["jdk"]["version"] + def resolve_java_home(self, host, cluster_config): + is_runtime_jdk_bundled = cluster_config.variables["system"]["runtime"]["jdk"]["bundled"] + runtime_jdks = cluster_config.variables["system"]["runtime"]["jdk"]["version"] try: allowed_runtime_jdks = [int(v) for v in runtime_jdks.split(",")] diff --git a/osbenchmark/metrics.py b/osbenchmark/metrics.py index d2b9743be..6157e0425 100644 --- a/osbenchmark/metrics.py +++ b/osbenchmark/metrics.py @@ -348,7 +348,7 @@ def calculate_system_results(store, node_name): return calc() -def metrics_store(cfg, read_only=True, workload=None, test_procedure=None, provision_config_instance=None, meta_info=None): +def metrics_store(cfg, read_only=True, workload=None, test_procedure=None, cluster_config=None, meta_info=None): """ Creates a proper metrics store based on the current configuration. @@ -362,12 +362,12 @@ def metrics_store(cfg, read_only=True, workload=None, test_procedure=None, provi test_execution_id = cfg.opts("system", "test_execution.id") test_execution_timestamp = cfg.opts("system", "time.start") - selected_provision_config_instance = cfg.opts("builder", "provision_config_instance.names") \ - if provision_config_instance is None else provision_config_instance + selected_cluster_config = cfg.opts("builder", "cluster_config.names") \ + if cluster_config is None else cluster_config store.open( test_execution_id, test_execution_timestamp, - workload, test_procedure, selected_provision_config_instance, + workload, test_procedure, selected_cluster_config, create=not read_only) return store @@ -434,8 +434,8 @@ def __init__(self, cfg, clock=time.Clock, meta_info=None): self._workload = None self._workload_params = cfg.opts("workload", "params", default_value={}, mandatory=False) self._test_procedure = None - self._provision_config_instance = None - self._provision_config_instance_name = None + self._cluster_config = None + self._cluster_config_name = None self._environment_name = cfg.opts("system", "env.name") self.opened = False if meta_info is None: @@ -452,16 +452,16 @@ def __init__(self, cfg, clock=time.Clock, meta_info=None): self.logger = logging.getLogger(__name__) def open(self, test_ex_id=None, test_ex_timestamp=None, workload_name=None,\ - test_procedure_name=None, provision_config_instance_name=None, ctx=None,\ + test_procedure_name=None, cluster_config_name=None, ctx=None,\ create=False): """ - Opens a metrics store for a specific test_execution, workload, test_procedure and provision_config_instance. + Opens a metrics store for a specific test_execution, workload, test_procedure and cluster_config. :param test_ex_id: The test execution id. This attribute is sufficient to uniquely identify a test_execution. :param test_ex_timestamp: The test execution timestamp as a datetime. :param workload_name: Workload name. :param test_procedure_name: TestProcedure name. - :param provision_config_instance_name: ProvisionConfigInstance name. + :param cluster_config_name: ProvisionConfigInstance name. :param ctx: An metrics store open context retrieved from another metrics store with ``#open_context``. :param create: True if an index should be created (if necessary). This is typically True, when attempting to write metrics and False when it is just opened for reading (as we can assume all necessary indices exist at this point). @@ -471,23 +471,23 @@ def open(self, test_ex_id=None, test_ex_timestamp=None, workload_name=None,\ self._test_execution_timestamp = ctx["test-execution-timestamp"] self._workload = ctx["workload"] self._test_procedure = ctx["test_procedure"] - self._provision_config_instance = ctx["provision-config-instance"] + self._cluster_config = ctx["provision-config-instance"] else: self._test_execution_id = test_ex_id self._test_execution_timestamp = time.to_iso8601(test_ex_timestamp) self._workload = workload_name self._test_procedure = test_procedure_name - self._provision_config_instance = provision_config_instance_name + self._cluster_config = cluster_config_name assert self._test_execution_id is not None, "Attempting to open metrics store without a test execution id" assert self._test_execution_timestamp is not None, "Attempting to open metrics store without a test execution timestamp" - self._provision_config_instance_name = "+".join(self._provision_config_instance) \ - if isinstance(self._provision_config_instance, list) \ - else self._provision_config_instance + self._cluster_config_name = "+".join(self._cluster_config) \ + if isinstance(self._cluster_config, list) \ + else self._cluster_config self.logger.info("Opening metrics store for test execution timestamp=[%s], workload=[%s]," - "test_procedure=[%s], provision_config_instance=[%s]", - self._test_execution_timestamp, self._workload, self._test_procedure, self._provision_config_instance) + "test_procedure=[%s], cluster_config=[%s]", + self._test_execution_timestamp, self._workload, self._test_procedure, self._cluster_config) user_tags = extract_user_tags_from_config(self._config) for k, v in user_tags.items(): @@ -556,7 +556,7 @@ def open_context(self): "test-execution-timestamp": self._test_execution_timestamp, "workload": self._workload, "test_procedure": self._test_procedure, - "provision-config-instance": self._provision_config_instance + "provision-config-instance": self._cluster_config } def put_value_cluster_level(self, name, value, unit=None, task=None, operation=None, operation_type=None, sample_type=SampleType.Normal, @@ -628,7 +628,7 @@ def _put_metric(self, level, level_key, name, value, unit, task, operation, oper "environment": self._environment_name, "workload": self._workload, "test_procedure": self._test_procedure, - "provision-config-instance": self._provision_config_instance_name, + "provision-config-instance": self._cluster_config_name, "name": name, "value": value, "unit": unit, @@ -685,7 +685,7 @@ def put_doc(self, doc, level=None, node_name=None, meta_data=None, absolute_time "environment": self._environment_name, "workload": self._workload, "test_procedure": self._test_procedure, - "provision-config-instance": self._provision_config_instance_name, + "provision-config-instance": self._cluster_config_name, }) if meta: @@ -873,13 +873,13 @@ def __init__(self, self._docs = None def open(self, test_ex_id=None, test_ex_timestamp=None, workload_name=None, \ - test_procedure_name=None, provision_config_instance_name=None, ctx=None, \ + test_procedure_name=None, cluster_config_name=None, ctx=None, \ create=False): self._docs = [] MetricsStore.open( self, test_ex_id, test_ex_timestamp, workload_name, test_procedure_name, - provision_config_instance_name, ctx, create) + cluster_config_name, ctx, create) self._index = self.index_name() # reduce a bit of noise in the metrics cluster log if create: @@ -915,9 +915,9 @@ def flush(self, refresh=True): self._client.bulk_index(index=self._index, doc_type=OsMetricsStore.METRICS_DOC_TYPE, items=self._docs) sw.stop() self.logger.info("Successfully added %d metrics documents for test execution timestamp=[%s], workload=[%s], " - "test_procedure=[%s], provision_config_instance=[%s] in [%f] seconds.", + "test_procedure=[%s], cluster_config=[%s] in [%f] seconds.", len(self._docs), self._test_execution_timestamp, - self._workload, self._test_procedure, self._provision_config_instance, sw.total_time()) + self._workload, self._test_procedure, self._cluster_config, sw.total_time()) self._docs = [] # ensure we can search immediately after flushing if refresh: @@ -1291,10 +1291,10 @@ def format_dict(d): test_execution.workload, format_dict(test_execution.workload_params), test_execution.test_procedure_name, - test_execution.provision_config_instance_name, + test_execution.cluster_config_name, format_dict(test_execution.user_tags), test_execution.workload_revision, - test_execution.provision_config_revision]) + test_execution.cluster_config_revision]) if len(test_executions) > 0: console.println("\nRecent test_executions:\n") @@ -1317,14 +1317,14 @@ def format_dict(d): def create_test_execution(cfg, workload, test_procedure, workload_revision=None): - provision_config_instance = cfg.opts("builder", "provision_config_instance.names") + cluster_config = cfg.opts("builder", "cluster_config.names") environment = cfg.opts("system", "env.name") test_execution_id = cfg.opts("system", "test_execution.id") test_execution_timestamp = cfg.opts("system", "time.start") user_tags = extract_user_tags_from_config(cfg) pipeline = cfg.opts("test_execution", "pipeline") workload_params = cfg.opts("workload", "params") - provision_config_instance_params = cfg.opts("builder", "provision_config_instance.params") + cluster_config_params = cfg.opts("builder", "cluster_config.params") plugin_params = cfg.opts("builder", "plugin.params") benchmark_version = version.version() benchmark_revision = version.revision() @@ -1338,7 +1338,7 @@ def create_test_execution(cfg, workload, test_procedure, workload_revision=None) return TestExecution(benchmark_version, benchmark_revision, environment, test_execution_id, test_execution_timestamp, pipeline, user_tags, workload, - workload_params, test_procedure, provision_config_instance, provision_config_instance_params, + workload_params, test_procedure, cluster_config, cluster_config_params, plugin_params, workload_revision, latency_percentiles=latency_percentiles, throughput_percentiles=throughput_percentiles) @@ -1346,9 +1346,9 @@ def create_test_execution(cfg, workload, test_procedure, workload_revision=None) class TestExecution: def __init__(self, benchmark_version, benchmark_revision, environment_name, test_execution_id, test_execution_timestamp, pipeline, user_tags, - workload, workload_params, test_procedure, provision_config_instance, - provision_config_instance_params, plugin_params, - workload_revision=None, provision_config_revision=None, + workload, workload_params, test_procedure, cluster_config, + cluster_config_params, plugin_params, + workload_revision=None, cluster_config_revision=None, distribution_version=None, distribution_flavor=None, revision=None, results=None, meta_data=None, latency_percentiles=None, throughput_percentiles=None): if results is None: @@ -1375,11 +1375,11 @@ def __init__(self, benchmark_version, benchmark_revision, environment_name, self.workload = workload self.workload_params = workload_params self.test_procedure = test_procedure - self.provision_config_instance = provision_config_instance - self.provision_config_instance_params = provision_config_instance_params + self.cluster_config = cluster_config + self.cluster_config_params = cluster_config_params self.plugin_params = plugin_params self.workload_revision = workload_revision - self.provision_config_revision = provision_config_revision + self.cluster_config_revision = cluster_config_revision self.distribution_version = distribution_version self.distribution_flavor = distribution_flavor self.revision = revision @@ -1398,10 +1398,10 @@ def test_procedure_name(self): return str(self.test_procedure) if self.test_procedure else None @property - def provision_config_instance_name(self): - return "+".join(self.provision_config_instance) \ - if isinstance(self.provision_config_instance, list) \ - else self.provision_config_instance + def cluster_config_name(self): + return "+".join(self.cluster_config) \ + if isinstance(self.cluster_config, list) \ + else self.cluster_config def add_results(self, results): self.results = results @@ -1419,12 +1419,12 @@ def as_dict(self): "pipeline": self.pipeline, "user-tags": self.user_tags, "workload": self.workload_name, - "provision-config-instance": self.provision_config_instance, + "provision-config-instance": self.cluster_config, "cluster": { "revision": self.revision, "distribution-version": self.distribution_version, "distribution-flavor": self.distribution_flavor, - "provision-config-revision": self.provision_config_revision, + "cluster-config-revision": self.cluster_config_revision, } } if self.results: @@ -1435,8 +1435,8 @@ def as_dict(self): d["test_procedure"] = self.test_procedure_name if self.workload_params: d["workload-params"] = self.workload_params - if self.provision_config_instance_params: - d["provision-config-instance-params"] = self.provision_config_instance_params + if self.cluster_config_params: + d["provision-config-instance-params"] = self.cluster_config_params if self.plugin_params: d["plugin-params"] = self.plugin_params return d @@ -1456,20 +1456,20 @@ def to_result_dicts(self): "user-tags": self.user_tags, "workload": self.workload_name, "test_procedure": self.test_procedure_name, - "provision-config-instance": self.provision_config_instance_name, + "provision-config-instance": self.cluster_config_name, # allow to logically delete records, e.g. for UI purposes when we only want to show the latest result "active": True } if self.distribution_version: result_template["distribution-major-version"] = versions.major_version(self.distribution_version) - if self.provision_config_revision: - result_template["provision-config-revision"] = self.provision_config_revision + if self.cluster_config_revision: + result_template["cluster-config-revision"] = self.cluster_config_revision if self.workload_revision: result_template["workload-revision"] = self.workload_revision if self.workload_params: result_template["workload-params"] = self.workload_params - if self.provision_config_instance_params: - result_template["provision-config-instance-params"] = self.provision_config_instance_params + if self.cluster_config_params: + result_template["provision-config-instance-params"] = self.cluster_config_params if self.plugin_params: result_template["plugin-params"] = self.plugin_params if self.meta_data: @@ -1495,7 +1495,7 @@ def from_dict(cls, d): d.get("test_procedure"), d["provision-config-instance"], d.get("provision-config-instance-params"), d.get("plugin-params"), workload_revision=d.get("workload-revision"), - provision_config_revision=cluster.get("provision-config-revision"), + cluster_config_revision=cluster.get("cluster-config-revision"), distribution_version=cluster.get("distribution-version"), distribution_flavor=cluster.get("distribution-flavor"), revision=cluster.get("revision"), results=d.get("results"), meta_data=d.get("meta", {})) diff --git a/osbenchmark/resources/provision_configs/main/provision_config_instances/v1/16gheap.ini b/osbenchmark/resources/provision_configs/1.0/cluster_configs/v1/16gheap.ini similarity index 75% rename from osbenchmark/resources/provision_configs/main/provision_config_instances/v1/16gheap.ini rename to osbenchmark/resources/provision_configs/1.0/cluster_configs/v1/16gheap.ini index 2e9bd2660..4a7a0b9f2 100644 --- a/osbenchmark/resources/provision_configs/main/provision_config_instances/v1/16gheap.ini +++ b/osbenchmark/resources/provision_configs/1.0/cluster_configs/v1/16gheap.ini @@ -1,6 +1,6 @@ [meta] description=Sets the Java heap to 16GB -type=provision_config_instance +type=cluster-config [config] base=vanilla diff --git a/osbenchmark/resources/provision_configs/main/provision_config_instances/v1/1gheap.ini b/osbenchmark/resources/provision_configs/1.0/cluster_configs/v1/1gheap.ini similarity index 75% rename from osbenchmark/resources/provision_configs/main/provision_config_instances/v1/1gheap.ini rename to osbenchmark/resources/provision_configs/1.0/cluster_configs/v1/1gheap.ini index 0d007e579..5ad606b3a 100644 --- a/osbenchmark/resources/provision_configs/main/provision_config_instances/v1/1gheap.ini +++ b/osbenchmark/resources/provision_configs/1.0/cluster_configs/v1/1gheap.ini @@ -1,6 +1,6 @@ [meta] description=Sets the Java heap to 1GB -type=provision_config_instance +type=cluster-config [config] base=vanilla diff --git a/osbenchmark/resources/provision_configs/1.0/provision_config_instances/v1/24gheap.ini b/osbenchmark/resources/provision_configs/1.0/cluster_configs/v1/24gheap.ini similarity index 75% rename from osbenchmark/resources/provision_configs/1.0/provision_config_instances/v1/24gheap.ini rename to osbenchmark/resources/provision_configs/1.0/cluster_configs/v1/24gheap.ini index 50e00a824..2fe01feff 100644 --- a/osbenchmark/resources/provision_configs/1.0/provision_config_instances/v1/24gheap.ini +++ b/osbenchmark/resources/provision_configs/1.0/cluster_configs/v1/24gheap.ini @@ -1,6 +1,6 @@ [meta] description=Sets the Java heap to 24GB -type=provision_config_instance +type=cluster-config [config] base=vanilla diff --git a/osbenchmark/resources/provision_configs/main/provision_config_instances/v1/2gheap.ini b/osbenchmark/resources/provision_configs/1.0/cluster_configs/v1/2gheap.ini similarity index 75% rename from osbenchmark/resources/provision_configs/main/provision_config_instances/v1/2gheap.ini rename to osbenchmark/resources/provision_configs/1.0/cluster_configs/v1/2gheap.ini index ba2a29911..afb00ed16 100644 --- a/osbenchmark/resources/provision_configs/main/provision_config_instances/v1/2gheap.ini +++ b/osbenchmark/resources/provision_configs/1.0/cluster_configs/v1/2gheap.ini @@ -1,6 +1,6 @@ [meta] description=Sets the Java heap to 2GB -type=provision_config_instance +type=cluster-config [config] base=vanilla diff --git a/osbenchmark/resources/provision_configs/1.0/provision_config_instances/v1/4gheap.ini b/osbenchmark/resources/provision_configs/1.0/cluster_configs/v1/4gheap.ini similarity index 75% rename from osbenchmark/resources/provision_configs/1.0/provision_config_instances/v1/4gheap.ini rename to osbenchmark/resources/provision_configs/1.0/cluster_configs/v1/4gheap.ini index d6e77baa1..cbc542d0b 100644 --- a/osbenchmark/resources/provision_configs/1.0/provision_config_instances/v1/4gheap.ini +++ b/osbenchmark/resources/provision_configs/1.0/cluster_configs/v1/4gheap.ini @@ -1,6 +1,6 @@ [meta] description=Sets the Java heap to 4GB -type=provision_config_instance +type=cluster-config [config] base=vanilla diff --git a/osbenchmark/resources/provision_configs/main/provision_config_instances/v1/8gheap.ini b/osbenchmark/resources/provision_configs/1.0/cluster_configs/v1/8gheap.ini similarity index 75% rename from osbenchmark/resources/provision_configs/main/provision_config_instances/v1/8gheap.ini rename to osbenchmark/resources/provision_configs/1.0/cluster_configs/v1/8gheap.ini index a8b4fd4b8..9d2bf6e6c 100644 --- a/osbenchmark/resources/provision_configs/main/provision_config_instances/v1/8gheap.ini +++ b/osbenchmark/resources/provision_configs/1.0/cluster_configs/v1/8gheap.ini @@ -1,6 +1,6 @@ [meta] description=Sets the Java heap to 8GB -type=provision_config_instance +type=cluster-config [config] base=vanilla diff --git a/osbenchmark/resources/provision_configs/1.0/provision_config_instances/v1/basic-license.ini b/osbenchmark/resources/provision_configs/1.0/cluster_configs/v1/basic-license.ini similarity index 100% rename from osbenchmark/resources/provision_configs/1.0/provision_config_instances/v1/basic-license.ini rename to osbenchmark/resources/provision_configs/1.0/cluster_configs/v1/basic-license.ini diff --git a/osbenchmark/resources/provision_configs/1.0/provision_config_instances/v1/dbg_non_safepoints/templates/config/jvm.options b/osbenchmark/resources/provision_configs/1.0/cluster_configs/v1/dbg_non_safepoints/templates/config/jvm.options similarity index 100% rename from osbenchmark/resources/provision_configs/1.0/provision_config_instances/v1/dbg_non_safepoints/templates/config/jvm.options rename to osbenchmark/resources/provision_configs/1.0/cluster_configs/v1/dbg_non_safepoints/templates/config/jvm.options diff --git a/osbenchmark/resources/provision_configs/1.0/provision_config_instances/v1/debug-non-safepoints.ini b/osbenchmark/resources/provision_configs/1.0/cluster_configs/v1/debug-non-safepoints.ini similarity index 100% rename from osbenchmark/resources/provision_configs/1.0/provision_config_instances/v1/debug-non-safepoints.ini rename to osbenchmark/resources/provision_configs/1.0/cluster_configs/v1/debug-non-safepoints.ini diff --git a/osbenchmark/resources/provision_configs/1.0/provision_config_instances/v1/default_distro/config.ini b/osbenchmark/resources/provision_configs/1.0/cluster_configs/v1/default_distro/config.ini similarity index 100% rename from osbenchmark/resources/provision_configs/1.0/provision_config_instances/v1/default_distro/config.ini rename to osbenchmark/resources/provision_configs/1.0/cluster_configs/v1/default_distro/config.ini diff --git a/osbenchmark/resources/provision_configs/1.0/provision_config_instances/v1/default_distro/templates/config/opensearch.yml b/osbenchmark/resources/provision_configs/1.0/cluster_configs/v1/default_distro/templates/config/opensearch.yml similarity index 100% rename from osbenchmark/resources/provision_configs/1.0/provision_config_instances/v1/default_distro/templates/config/opensearch.yml rename to osbenchmark/resources/provision_configs/1.0/cluster_configs/v1/default_distro/templates/config/opensearch.yml diff --git a/osbenchmark/resources/provision_configs/1.0/provision_config_instances/v1/defaults.ini b/osbenchmark/resources/provision_configs/1.0/cluster_configs/v1/defaults.ini similarity index 75% rename from osbenchmark/resources/provision_configs/1.0/provision_config_instances/v1/defaults.ini rename to osbenchmark/resources/provision_configs/1.0/cluster_configs/v1/defaults.ini index 0d007e579..5ad606b3a 100644 --- a/osbenchmark/resources/provision_configs/1.0/provision_config_instances/v1/defaults.ini +++ b/osbenchmark/resources/provision_configs/1.0/cluster_configs/v1/defaults.ini @@ -1,6 +1,6 @@ [meta] description=Sets the Java heap to 1GB -type=provision_config_instance +type=cluster-config [config] base=vanilla diff --git a/osbenchmark/resources/provision_configs/1.0/provision_config_instances/v1/ea.ini b/osbenchmark/resources/provision_configs/1.0/cluster_configs/v1/ea.ini similarity index 100% rename from osbenchmark/resources/provision_configs/1.0/provision_config_instances/v1/ea.ini rename to osbenchmark/resources/provision_configs/1.0/cluster_configs/v1/ea.ini diff --git a/osbenchmark/resources/provision_configs/1.0/provision_config_instances/v1/ea/templates/config/jvm.options b/osbenchmark/resources/provision_configs/1.0/cluster_configs/v1/ea/templates/config/jvm.options similarity index 100% rename from osbenchmark/resources/provision_configs/1.0/provision_config_instances/v1/ea/templates/config/jvm.options rename to osbenchmark/resources/provision_configs/1.0/cluster_configs/v1/ea/templates/config/jvm.options diff --git a/osbenchmark/resources/provision_configs/1.0/provision_config_instances/v1/fp.ini b/osbenchmark/resources/provision_configs/1.0/cluster_configs/v1/fp.ini similarity index 100% rename from osbenchmark/resources/provision_configs/1.0/provision_config_instances/v1/fp.ini rename to osbenchmark/resources/provision_configs/1.0/cluster_configs/v1/fp.ini diff --git a/osbenchmark/resources/provision_configs/1.0/provision_config_instances/v1/fp/templates/config/jvm.options b/osbenchmark/resources/provision_configs/1.0/cluster_configs/v1/fp/templates/config/jvm.options similarity index 100% rename from osbenchmark/resources/provision_configs/1.0/provision_config_instances/v1/fp/templates/config/jvm.options rename to osbenchmark/resources/provision_configs/1.0/cluster_configs/v1/fp/templates/config/jvm.options diff --git a/osbenchmark/resources/provision_configs/1.0/provision_config_instances/v1/g1gc.ini b/osbenchmark/resources/provision_configs/1.0/cluster_configs/v1/g1gc.ini similarity index 100% rename from osbenchmark/resources/provision_configs/1.0/provision_config_instances/v1/g1gc.ini rename to osbenchmark/resources/provision_configs/1.0/cluster_configs/v1/g1gc.ini diff --git a/osbenchmark/resources/provision_configs/1.0/provision_config_instances/v1/trial-license.ini b/osbenchmark/resources/provision_configs/1.0/cluster_configs/v1/trial-license.ini similarity index 100% rename from osbenchmark/resources/provision_configs/1.0/provision_config_instances/v1/trial-license.ini rename to osbenchmark/resources/provision_configs/1.0/cluster_configs/v1/trial-license.ini diff --git a/osbenchmark/resources/provision_configs/1.0/provision_config_instances/v1/unpooled.ini b/osbenchmark/resources/provision_configs/1.0/cluster_configs/v1/unpooled.ini similarity index 100% rename from osbenchmark/resources/provision_configs/1.0/provision_config_instances/v1/unpooled.ini rename to osbenchmark/resources/provision_configs/1.0/cluster_configs/v1/unpooled.ini diff --git a/osbenchmark/resources/provision_configs/1.0/provision_config_instances/v1/unpooled/templates/config/jvm.options b/osbenchmark/resources/provision_configs/1.0/cluster_configs/v1/unpooled/templates/config/jvm.options similarity index 100% rename from osbenchmark/resources/provision_configs/1.0/provision_config_instances/v1/unpooled/templates/config/jvm.options rename to osbenchmark/resources/provision_configs/1.0/cluster_configs/v1/unpooled/templates/config/jvm.options diff --git a/osbenchmark/resources/provision_configs/1.0/provision_config_instances/v1/vanilla/README.md b/osbenchmark/resources/provision_configs/1.0/cluster_configs/v1/vanilla/README.md similarity index 100% rename from osbenchmark/resources/provision_configs/1.0/provision_config_instances/v1/vanilla/README.md rename to osbenchmark/resources/provision_configs/1.0/cluster_configs/v1/vanilla/README.md diff --git a/osbenchmark/resources/provision_configs/1.0/provision_config_instances/v1/vanilla/config.ini b/osbenchmark/resources/provision_configs/1.0/cluster_configs/v1/vanilla/config.ini similarity index 100% rename from osbenchmark/resources/provision_configs/1.0/provision_config_instances/v1/vanilla/config.ini rename to osbenchmark/resources/provision_configs/1.0/cluster_configs/v1/vanilla/config.ini diff --git a/osbenchmark/resources/provision_configs/1.0/provision_config_instances/v1/vanilla/templates/config/jvm.options b/osbenchmark/resources/provision_configs/1.0/cluster_configs/v1/vanilla/templates/config/jvm.options similarity index 100% rename from osbenchmark/resources/provision_configs/1.0/provision_config_instances/v1/vanilla/templates/config/jvm.options rename to osbenchmark/resources/provision_configs/1.0/cluster_configs/v1/vanilla/templates/config/jvm.options diff --git a/osbenchmark/resources/provision_configs/1.0/provision_config_instances/v1/vanilla/templates/config/log4j2.properties b/osbenchmark/resources/provision_configs/1.0/cluster_configs/v1/vanilla/templates/config/log4j2.properties similarity index 100% rename from osbenchmark/resources/provision_configs/1.0/provision_config_instances/v1/vanilla/templates/config/log4j2.properties rename to osbenchmark/resources/provision_configs/1.0/cluster_configs/v1/vanilla/templates/config/log4j2.properties diff --git a/osbenchmark/resources/provision_configs/1.0/provision_config_instances/v1/vanilla/templates/config/opensearch.yml b/osbenchmark/resources/provision_configs/1.0/cluster_configs/v1/vanilla/templates/config/opensearch.yml similarity index 100% rename from osbenchmark/resources/provision_configs/1.0/provision_config_instances/v1/vanilla/templates/config/opensearch.yml rename to osbenchmark/resources/provision_configs/1.0/cluster_configs/v1/vanilla/templates/config/opensearch.yml diff --git a/osbenchmark/resources/provision_configs/1.0/provision_config_instances/v1/16gheap.ini b/osbenchmark/resources/provision_configs/main/cluster_configs/v1/16gheap.ini similarity index 75% rename from osbenchmark/resources/provision_configs/1.0/provision_config_instances/v1/16gheap.ini rename to osbenchmark/resources/provision_configs/main/cluster_configs/v1/16gheap.ini index 2e9bd2660..4a7a0b9f2 100644 --- a/osbenchmark/resources/provision_configs/1.0/provision_config_instances/v1/16gheap.ini +++ b/osbenchmark/resources/provision_configs/main/cluster_configs/v1/16gheap.ini @@ -1,6 +1,6 @@ [meta] description=Sets the Java heap to 16GB -type=provision_config_instance +type=cluster-config [config] base=vanilla diff --git a/osbenchmark/resources/provision_configs/1.0/provision_config_instances/v1/1gheap.ini b/osbenchmark/resources/provision_configs/main/cluster_configs/v1/1gheap.ini similarity index 75% rename from osbenchmark/resources/provision_configs/1.0/provision_config_instances/v1/1gheap.ini rename to osbenchmark/resources/provision_configs/main/cluster_configs/v1/1gheap.ini index 0d007e579..5ad606b3a 100644 --- a/osbenchmark/resources/provision_configs/1.0/provision_config_instances/v1/1gheap.ini +++ b/osbenchmark/resources/provision_configs/main/cluster_configs/v1/1gheap.ini @@ -1,6 +1,6 @@ [meta] description=Sets the Java heap to 1GB -type=provision_config_instance +type=cluster-config [config] base=vanilla diff --git a/osbenchmark/resources/provision_configs/main/provision_config_instances/v1/24gheap.ini b/osbenchmark/resources/provision_configs/main/cluster_configs/v1/24gheap.ini similarity index 75% rename from osbenchmark/resources/provision_configs/main/provision_config_instances/v1/24gheap.ini rename to osbenchmark/resources/provision_configs/main/cluster_configs/v1/24gheap.ini index 50e00a824..2fe01feff 100644 --- a/osbenchmark/resources/provision_configs/main/provision_config_instances/v1/24gheap.ini +++ b/osbenchmark/resources/provision_configs/main/cluster_configs/v1/24gheap.ini @@ -1,6 +1,6 @@ [meta] description=Sets the Java heap to 24GB -type=provision_config_instance +type=cluster-config [config] base=vanilla diff --git a/osbenchmark/resources/provision_configs/1.0/provision_config_instances/v1/2gheap.ini b/osbenchmark/resources/provision_configs/main/cluster_configs/v1/2gheap.ini similarity index 75% rename from osbenchmark/resources/provision_configs/1.0/provision_config_instances/v1/2gheap.ini rename to osbenchmark/resources/provision_configs/main/cluster_configs/v1/2gheap.ini index ba2a29911..afb00ed16 100644 --- a/osbenchmark/resources/provision_configs/1.0/provision_config_instances/v1/2gheap.ini +++ b/osbenchmark/resources/provision_configs/main/cluster_configs/v1/2gheap.ini @@ -1,6 +1,6 @@ [meta] description=Sets the Java heap to 2GB -type=provision_config_instance +type=cluster-config [config] base=vanilla diff --git a/osbenchmark/resources/provision_configs/main/provision_config_instances/v1/4gheap.ini b/osbenchmark/resources/provision_configs/main/cluster_configs/v1/4gheap.ini similarity index 75% rename from osbenchmark/resources/provision_configs/main/provision_config_instances/v1/4gheap.ini rename to osbenchmark/resources/provision_configs/main/cluster_configs/v1/4gheap.ini index d6e77baa1..cbc542d0b 100644 --- a/osbenchmark/resources/provision_configs/main/provision_config_instances/v1/4gheap.ini +++ b/osbenchmark/resources/provision_configs/main/cluster_configs/v1/4gheap.ini @@ -1,6 +1,6 @@ [meta] description=Sets the Java heap to 4GB -type=provision_config_instance +type=cluster-config [config] base=vanilla diff --git a/osbenchmark/resources/provision_configs/1.0/provision_config_instances/v1/8gheap.ini b/osbenchmark/resources/provision_configs/main/cluster_configs/v1/8gheap.ini similarity index 75% rename from osbenchmark/resources/provision_configs/1.0/provision_config_instances/v1/8gheap.ini rename to osbenchmark/resources/provision_configs/main/cluster_configs/v1/8gheap.ini index a8b4fd4b8..9d2bf6e6c 100644 --- a/osbenchmark/resources/provision_configs/1.0/provision_config_instances/v1/8gheap.ini +++ b/osbenchmark/resources/provision_configs/main/cluster_configs/v1/8gheap.ini @@ -1,6 +1,6 @@ [meta] description=Sets the Java heap to 8GB -type=provision_config_instance +type=cluster-config [config] base=vanilla diff --git a/osbenchmark/resources/provision_configs/main/provision_config_instances/v1/basic-license.ini b/osbenchmark/resources/provision_configs/main/cluster_configs/v1/basic-license.ini similarity index 100% rename from osbenchmark/resources/provision_configs/main/provision_config_instances/v1/basic-license.ini rename to osbenchmark/resources/provision_configs/main/cluster_configs/v1/basic-license.ini diff --git a/osbenchmark/resources/provision_configs/main/provision_config_instances/v1/dbg_non_safepoints/templates/config/jvm.options b/osbenchmark/resources/provision_configs/main/cluster_configs/v1/dbg_non_safepoints/templates/config/jvm.options similarity index 100% rename from osbenchmark/resources/provision_configs/main/provision_config_instances/v1/dbg_non_safepoints/templates/config/jvm.options rename to osbenchmark/resources/provision_configs/main/cluster_configs/v1/dbg_non_safepoints/templates/config/jvm.options diff --git a/osbenchmark/resources/provision_configs/main/provision_config_instances/v1/debug-non-safepoints.ini b/osbenchmark/resources/provision_configs/main/cluster_configs/v1/debug-non-safepoints.ini similarity index 100% rename from osbenchmark/resources/provision_configs/main/provision_config_instances/v1/debug-non-safepoints.ini rename to osbenchmark/resources/provision_configs/main/cluster_configs/v1/debug-non-safepoints.ini diff --git a/osbenchmark/resources/provision_configs/main/provision_config_instances/v1/defaults.ini b/osbenchmark/resources/provision_configs/main/cluster_configs/v1/defaults.ini similarity index 78% rename from osbenchmark/resources/provision_configs/main/provision_config_instances/v1/defaults.ini rename to osbenchmark/resources/provision_configs/main/cluster_configs/v1/defaults.ini index a24fadfff..c034ca2f6 100644 --- a/osbenchmark/resources/provision_configs/main/provision_config_instances/v1/defaults.ini +++ b/osbenchmark/resources/provision_configs/main/cluster_configs/v1/defaults.ini @@ -1,6 +1,6 @@ [meta] description=Sets the Java heap to 1GB -type=provision_config_instance +type=cluster-config [config] base=vanilla diff --git a/osbenchmark/resources/provision_configs/main/provision_config_instances/v1/ea.ini b/osbenchmark/resources/provision_configs/main/cluster_configs/v1/ea.ini similarity index 100% rename from osbenchmark/resources/provision_configs/main/provision_config_instances/v1/ea.ini rename to osbenchmark/resources/provision_configs/main/cluster_configs/v1/ea.ini diff --git a/osbenchmark/resources/provision_configs/main/provision_config_instances/v1/ea/templates/config/jvm.options b/osbenchmark/resources/provision_configs/main/cluster_configs/v1/ea/templates/config/jvm.options similarity index 100% rename from osbenchmark/resources/provision_configs/main/provision_config_instances/v1/ea/templates/config/jvm.options rename to osbenchmark/resources/provision_configs/main/cluster_configs/v1/ea/templates/config/jvm.options diff --git a/osbenchmark/resources/provision_configs/main/provision_config_instances/v1/fp.ini b/osbenchmark/resources/provision_configs/main/cluster_configs/v1/fp.ini similarity index 100% rename from osbenchmark/resources/provision_configs/main/provision_config_instances/v1/fp.ini rename to osbenchmark/resources/provision_configs/main/cluster_configs/v1/fp.ini diff --git a/osbenchmark/resources/provision_configs/main/provision_config_instances/v1/fp/templates/config/jvm.options b/osbenchmark/resources/provision_configs/main/cluster_configs/v1/fp/templates/config/jvm.options similarity index 100% rename from osbenchmark/resources/provision_configs/main/provision_config_instances/v1/fp/templates/config/jvm.options rename to osbenchmark/resources/provision_configs/main/cluster_configs/v1/fp/templates/config/jvm.options diff --git a/osbenchmark/resources/provision_configs/main/provision_config_instances/v1/g1gc.ini b/osbenchmark/resources/provision_configs/main/cluster_configs/v1/g1gc.ini similarity index 100% rename from osbenchmark/resources/provision_configs/main/provision_config_instances/v1/g1gc.ini rename to osbenchmark/resources/provision_configs/main/cluster_configs/v1/g1gc.ini diff --git a/osbenchmark/resources/provision_configs/main/provision_config_instances/v1/parallelgc.ini b/osbenchmark/resources/provision_configs/main/cluster_configs/v1/parallelgc.ini similarity index 100% rename from osbenchmark/resources/provision_configs/main/provision_config_instances/v1/parallelgc.ini rename to osbenchmark/resources/provision_configs/main/cluster_configs/v1/parallelgc.ini diff --git a/osbenchmark/resources/provision_configs/main/provision_config_instances/v1/trial-license.ini b/osbenchmark/resources/provision_configs/main/cluster_configs/v1/trial-license.ini similarity index 100% rename from osbenchmark/resources/provision_configs/main/provision_config_instances/v1/trial-license.ini rename to osbenchmark/resources/provision_configs/main/cluster_configs/v1/trial-license.ini diff --git a/osbenchmark/resources/provision_configs/main/provision_config_instances/v1/unpooled.ini b/osbenchmark/resources/provision_configs/main/cluster_configs/v1/unpooled.ini similarity index 100% rename from osbenchmark/resources/provision_configs/main/provision_config_instances/v1/unpooled.ini rename to osbenchmark/resources/provision_configs/main/cluster_configs/v1/unpooled.ini diff --git a/osbenchmark/resources/provision_configs/main/provision_config_instances/v1/unpooled/templates/config/jvm.options b/osbenchmark/resources/provision_configs/main/cluster_configs/v1/unpooled/templates/config/jvm.options similarity index 100% rename from osbenchmark/resources/provision_configs/main/provision_config_instances/v1/unpooled/templates/config/jvm.options rename to osbenchmark/resources/provision_configs/main/cluster_configs/v1/unpooled/templates/config/jvm.options diff --git a/osbenchmark/resources/provision_configs/main/provision_config_instances/v1/vanilla/README.md b/osbenchmark/resources/provision_configs/main/cluster_configs/v1/vanilla/README.md similarity index 100% rename from osbenchmark/resources/provision_configs/main/provision_config_instances/v1/vanilla/README.md rename to osbenchmark/resources/provision_configs/main/cluster_configs/v1/vanilla/README.md diff --git a/osbenchmark/resources/provision_configs/main/provision_config_instances/v1/vanilla/config.ini b/osbenchmark/resources/provision_configs/main/cluster_configs/v1/vanilla/config.ini similarity index 100% rename from osbenchmark/resources/provision_configs/main/provision_config_instances/v1/vanilla/config.ini rename to osbenchmark/resources/provision_configs/main/cluster_configs/v1/vanilla/config.ini diff --git a/osbenchmark/resources/provision_configs/main/provision_config_instances/v1/vanilla/templates/config/jvm.options b/osbenchmark/resources/provision_configs/main/cluster_configs/v1/vanilla/templates/config/jvm.options similarity index 100% rename from osbenchmark/resources/provision_configs/main/provision_config_instances/v1/vanilla/templates/config/jvm.options rename to osbenchmark/resources/provision_configs/main/cluster_configs/v1/vanilla/templates/config/jvm.options diff --git a/osbenchmark/resources/provision_configs/main/provision_config_instances/v1/vanilla/templates/config/log4j2.properties b/osbenchmark/resources/provision_configs/main/cluster_configs/v1/vanilla/templates/config/log4j2.properties similarity index 100% rename from osbenchmark/resources/provision_configs/main/provision_config_instances/v1/vanilla/templates/config/log4j2.properties rename to osbenchmark/resources/provision_configs/main/cluster_configs/v1/vanilla/templates/config/log4j2.properties diff --git a/osbenchmark/resources/provision_configs/main/provision_config_instances/v1/vanilla/templates/config/opensearch.yml b/osbenchmark/resources/provision_configs/main/cluster_configs/v1/vanilla/templates/config/opensearch.yml similarity index 100% rename from osbenchmark/resources/provision_configs/main/provision_config_instances/v1/vanilla/templates/config/opensearch.yml rename to osbenchmark/resources/provision_configs/main/cluster_configs/v1/vanilla/templates/config/opensearch.yml diff --git a/osbenchmark/results_publisher.py b/osbenchmark/results_publisher.py index ac9a631cf..caa9a2f1e 100644 --- a/osbenchmark/results_publisher.py +++ b/osbenchmark/results_publisher.py @@ -360,7 +360,7 @@ def publish(self, r1, r2): print_internal(" TestExecution timestamp: %s" % r1.test_execution_timestamp) if r1.test_procedure_name: print_internal(" TestProcedure: %s" % r1.test_procedure_name) - print_internal(" ProvisionConfigInstance: %s" % r1.provision_config_instance_name) + print_internal(" ProvisionConfigInstance: %s" % r1.cluster_config_name) if r1.user_tags: r1_user_tags = ", ".join(["%s=%s" % (k, v) for k, v in sorted(r1.user_tags.items())]) print_internal(" User tags: %s" % r1_user_tags) @@ -370,7 +370,7 @@ def publish(self, r1, r2): print_internal(" TestExecution timestamp: %s" % r2.test_execution_timestamp) if r2.test_procedure_name: print_internal(" TestProcedure: %s" % r2.test_procedure_name) - print_internal(" ProvisionConfigInstance: %s" % r2.provision_config_instance_name) + print_internal(" ProvisionConfigInstance: %s" % r2.cluster_config_name) if r2.user_tags: r2_user_tags = ", ".join(["%s=%s" % (k, v) for k, v in sorted(r2.user_tags.items())]) print_internal(" User tags: %s" % r2_user_tags) diff --git a/osbenchmark/test_execution_orchestrator.py b/osbenchmark/test_execution_orchestrator.py index 966de9a27..e686cf216 100644 --- a/osbenchmark/test_execution_orchestrator.py +++ b/osbenchmark/test_execution_orchestrator.py @@ -118,7 +118,7 @@ def receiveMsg_Setup(self, msg, sender): @actor.no_retry("test execution orchestrator") # pylint: disable=no-value-for-parameter def receiveMsg_EngineStarted(self, msg, sender): self.logger.info("Builder has started engine successfully.") - self.coordinator.test_execution.provision_config_revision = msg.provision_config_revision + self.coordinator.test_execution.cluster_config_revision = msg.cluster_config_revision self.main_worker_coordinator = self.createActor( worker_coordinator.WorkerCoordinatorActor, targetActorRequirements={"coordinator": True} @@ -222,16 +222,16 @@ def on_preparation_complete(self, distribution_flavor, distribution_version, rev # store test_execution initially (without any results) so other components can retrieve full metadata self.test_execution_store.store_test_execution(self.test_execution) if self.test_execution.test_procedure.auto_generated: - console.info("Executing test with workload [{}] and provision_config_instance {} with version [{}].\n" + console.info("Executing test with workload [{}] and cluster_config {} with version [{}].\n" .format(self.test_execution.workload_name, - self.test_execution.provision_config_instance, + self.test_execution.cluster_config, self.test_execution.distribution_version)) else: - console.info("Executing test with workload [{}], test_procedure [{}] and provision_config_instance {} with version [{}].\n" + console.info("Executing test with workload [{}], test_procedure [{}] and cluster_config {} with version [{}].\n" .format( self.test_execution.workload_name, self.test_execution.test_procedure_name, - self.test_execution.provision_config_instance, + self.test_execution.cluster_config, self.test_execution.distribution_version )) @@ -309,8 +309,8 @@ def from_distribution(cfg): def benchmark_only(cfg): set_default_hosts(cfg) - # We'll use a special provision_config_instance name for external benchmarks. - cfg.add(config.Scope.benchmark, "builder", "provision_config_instance.names", ["external"]) + # We'll use a special cluster_config name for external benchmarks. + cfg.add(config.Scope.benchmark, "builder", "cluster_config.names", ["external"]) return execute_test(cfg, external=True) diff --git a/tests/builder/configs/utils/config_path_resolver_test.py b/tests/builder/configs/utils/config_path_resolver_test.py index 9af5f503f..929a0ba6b 100644 --- a/tests/builder/configs/utils/config_path_resolver_test.py +++ b/tests/builder/configs/utils/config_path_resolver_test.py @@ -14,7 +14,7 @@ def setUp(self): self.config_path_resolver = ConfigPathResolver(self.cfg) @mock.patch('os.path.exists') - def test_provision_config_path_defined(self, path_exists): + def test_cluster_config_path_defined(self, path_exists): path_exists.return_value = True # opts("builder", "provision_config.path") self.cfg.opts.return_value = "/path/to/configs" @@ -26,7 +26,7 @@ def test_provision_config_path_defined(self, path_exists): @mock.patch('osbenchmark.utils.repo.BenchmarkRepository') @mock.patch('osbenchmark.utils.repo.BenchmarkRepository.set_provision_configs_dir') @mock.patch('os.path.exists') - def test_provision_config_path_not_defined(self, path_exists, set_repo, benchmark_repo, git_fetch): + def test_cluster_config_path_not_defined(self, path_exists, set_repo, benchmark_repo, git_fetch): path_exists.return_value = True # opts("builder", "provision_config.path"), opts("builder", "distribution.version"), opts("builder", "repository.name"), @@ -38,7 +38,7 @@ def test_provision_config_path_not_defined(self, path_exists, set_repo, benchmar self.assertEqual(config_path, "/root_dir/repo_dir/fake-repo/red/v36") @mock.patch('os.path.exists') - def test_provision_config_path_does_not_exist(self, path_exists): + def test_cluster_config_path_does_not_exist(self, path_exists): path_exists.return_value = False # opts("builder", "provision_config.path") self.cfg.opts.return_value = "/path/to/configs" diff --git a/tests/builder/data/provision_config_instances/v1/32gheap.ini b/tests/builder/data/cluster_configs/v1/32gheap.ini similarity index 76% rename from tests/builder/data/provision_config_instances/v1/32gheap.ini rename to tests/builder/data/cluster_configs/v1/32gheap.ini index 2ec3ace31..5c17db53f 100644 --- a/tests/builder/data/provision_config_instances/v1/32gheap.ini +++ b/tests/builder/data/cluster_configs/v1/32gheap.ini @@ -1,5 +1,5 @@ [meta] -type=provision_config_instance +type=cluster-config description=32GB heap size [config] diff --git a/tests/builder/data/provision_config_instances/v1/another_with_hook.ini b/tests/builder/data/cluster_configs/v1/another_with_hook.ini similarity index 56% rename from tests/builder/data/provision_config_instances/v1/another_with_hook.ini rename to tests/builder/data/cluster_configs/v1/another_with_hook.ini index 581e9c6d4..a1c68b7c9 100644 --- a/tests/builder/data/provision_config_instances/v1/another_with_hook.ini +++ b/tests/builder/data/cluster_configs/v1/another_with_hook.ini @@ -1,6 +1,6 @@ [meta] type = mixin -description = A verbose provision_config_instance with an install hook +description = A verbose cluster-config with an install hook [config] base = verbose_logging,with_hook diff --git a/tests/builder/data/provision_config_instances/v1/default.ini b/tests/builder/data/cluster_configs/v1/default.ini similarity index 75% rename from tests/builder/data/provision_config_instances/v1/default.ini rename to tests/builder/data/cluster_configs/v1/default.ini index 2b42f1753..6d31caf88 100644 --- a/tests/builder/data/provision_config_instances/v1/default.ini +++ b/tests/builder/data/cluster_configs/v1/default.ini @@ -1,5 +1,5 @@ [meta] -type=provision_config_instance +type=cluster-config description=Out of the box configuration [config] diff --git a/tests/builder/data/provision_config_instances/v1/ea.ini b/tests/builder/data/cluster_configs/v1/ea.ini similarity index 100% rename from tests/builder/data/provision_config_instances/v1/ea.ini rename to tests/builder/data/cluster_configs/v1/ea.ini diff --git a/tests/builder/data/cluster_configs/v1/empty_cfg_base.ini b/tests/builder/data/cluster_configs/v1/empty_cfg_base.ini new file mode 100644 index 000000000..cc8c26420 --- /dev/null +++ b/tests/builder/data/cluster_configs/v1/empty_cfg_base.ini @@ -0,0 +1,9 @@ +[meta] +type=cluster-config +description=A cluster-config with an empty config base + +[config] +base= + +[variables] +heap_size=1g \ No newline at end of file diff --git a/tests/builder/data/provision_config_instances/v1/hook2/config.py b/tests/builder/data/cluster_configs/v1/hook2/config.py similarity index 100% rename from tests/builder/data/provision_config_instances/v1/hook2/config.py rename to tests/builder/data/cluster_configs/v1/hook2/config.py diff --git a/tests/builder/data/cluster_configs/v1/missing_cfg_base.ini b/tests/builder/data/cluster_configs/v1/missing_cfg_base.ini new file mode 100644 index 000000000..a13b2d463 --- /dev/null +++ b/tests/builder/data/cluster_configs/v1/missing_cfg_base.ini @@ -0,0 +1,9 @@ +[meta] +type=cluster-config +description=A cluster-config with a missing config base + +[config] +# no base + +[variables] +heap_size=1g \ No newline at end of file diff --git a/tests/builder/data/cluster_configs/v1/multi_hook.ini b/tests/builder/data/cluster_configs/v1/multi_hook.ini new file mode 100644 index 000000000..c5f2f894f --- /dev/null +++ b/tests/builder/data/cluster_configs/v1/multi_hook.ini @@ -0,0 +1,6 @@ +[meta] +type=mixin +description=A cluster-config with multiple hooks + +[config] +base=vanilla,with_hook,hook2 diff --git a/tests/builder/data/cluster_configs/v1/vanilla/config.ini b/tests/builder/data/cluster_configs/v1/vanilla/config.ini new file mode 100644 index 000000000..23cdabdb0 --- /dev/null +++ b/tests/builder/data/cluster_configs/v1/vanilla/config.ini @@ -0,0 +1,5 @@ +[variables] +# this should *always* be overridden by the cluster-config definition (config base +# variables take least precedence) +heap_size = 0 +clean_command = ./gradlew clean diff --git a/tests/builder/data/provision_config_instances/v1/verbose.ini b/tests/builder/data/cluster_configs/v1/verbose.ini similarity index 100% rename from tests/builder/data/provision_config_instances/v1/verbose.ini rename to tests/builder/data/cluster_configs/v1/verbose.ini diff --git a/tests/builder/data/provision_config_instances/v1/verbose_logging/config.ini b/tests/builder/data/cluster_configs/v1/verbose_logging/config.ini similarity index 100% rename from tests/builder/data/provision_config_instances/v1/verbose_logging/config.ini rename to tests/builder/data/cluster_configs/v1/verbose_logging/config.ini diff --git a/tests/builder/data/cluster_configs/v1/with_hook.ini b/tests/builder/data/cluster_configs/v1/with_hook.ini new file mode 100644 index 000000000..3947f864b --- /dev/null +++ b/tests/builder/data/cluster_configs/v1/with_hook.ini @@ -0,0 +1,6 @@ +[meta] +type=mixin +description=A cluster-config with an install hook + +[config] +base=vanilla,with_hook diff --git a/tests/builder/data/provision_config_instances/v1/with_hook/config.py b/tests/builder/data/cluster_configs/v1/with_hook/config.py similarity index 100% rename from tests/builder/data/provision_config_instances/v1/with_hook/config.py rename to tests/builder/data/cluster_configs/v1/with_hook/config.py diff --git a/tests/builder/data/provision_config_instances/v1/empty_cfg_base.ini b/tests/builder/data/provision_config_instances/v1/empty_cfg_base.ini deleted file mode 100644 index fc700fabe..000000000 --- a/tests/builder/data/provision_config_instances/v1/empty_cfg_base.ini +++ /dev/null @@ -1,9 +0,0 @@ -[meta] -type=provision_config_instance -description=A provision_config_instance with an empty config base - -[config] -base= - -[variables] -heap_size=1g \ No newline at end of file diff --git a/tests/builder/data/provision_config_instances/v1/missing_cfg_base.ini b/tests/builder/data/provision_config_instances/v1/missing_cfg_base.ini deleted file mode 100644 index 8fdc13ccb..000000000 --- a/tests/builder/data/provision_config_instances/v1/missing_cfg_base.ini +++ /dev/null @@ -1,9 +0,0 @@ -[meta] -type=provision_config_instance -description=A provision_config_instance with a missing config base - -[config] -# no base - -[variables] -heap_size=1g \ No newline at end of file diff --git a/tests/builder/data/provision_config_instances/v1/multi_hook.ini b/tests/builder/data/provision_config_instances/v1/multi_hook.ini deleted file mode 100644 index 3c7683ab2..000000000 --- a/tests/builder/data/provision_config_instances/v1/multi_hook.ini +++ /dev/null @@ -1,6 +0,0 @@ -[meta] -type=mixin -description=A provision_config_instance with multiple hooks - -[config] -base=vanilla,with_hook,hook2 diff --git a/tests/builder/data/provision_config_instances/v1/vanilla/config.ini b/tests/builder/data/provision_config_instances/v1/vanilla/config.ini deleted file mode 100644 index 226e03301..000000000 --- a/tests/builder/data/provision_config_instances/v1/vanilla/config.ini +++ /dev/null @@ -1,5 +0,0 @@ -[variables] -# this should *always* be overridden by the provision_config_instance definition (config base -# variables take least precedence) -heap_size = 0 -clean_command = ./gradlew clean diff --git a/tests/builder/data/provision_config_instances/v1/with_hook.ini b/tests/builder/data/provision_config_instances/v1/with_hook.ini deleted file mode 100644 index fbeeadbab..000000000 --- a/tests/builder/data/provision_config_instances/v1/with_hook.ini +++ /dev/null @@ -1,6 +0,0 @@ -[meta] -type=mixin -description=A provision_config_instance with an install hook - -[config] -base=vanilla,with_hook diff --git a/tests/builder/downloaders/opensearch_distribution_downloader_test.py b/tests/builder/downloaders/opensearch_distribution_downloader_test.py index d59b9f614..d7abb040b 100644 --- a/tests/builder/downloaders/opensearch_distribution_downloader_test.py +++ b/tests/builder/downloaders/opensearch_distribution_downloader_test.py @@ -11,7 +11,7 @@ def setUp(self): self.host = None self.executor = Mock() - self.provision_config_instance = ProvisionConfigInstance(names="fake", root_path="also fake", config_paths="fake2", variables={ + self.cluster_config = ProvisionConfigInstance(names="fake", root_path="also fake", config_paths="fake2", variables={ "node": { "root": { "dir": "/fake/dir/for/download" @@ -24,7 +24,7 @@ def setUp(self): self.path_manager = Mock() self.distribution_repository_provider = Mock() - self.os_distro_downloader = OpenSearchDistributionDownloader(self.provision_config_instance, self.executor, self.path_manager, + self.os_distro_downloader = OpenSearchDistributionDownloader(self.cluster_config, self.executor, self.path_manager, self.distribution_repository_provider) diff --git a/tests/builder/downloaders/opensearch_source_downloader_test.py b/tests/builder/downloaders/opensearch_source_downloader_test.py index 7b3a72562..e75336af1 100644 --- a/tests/builder/downloaders/opensearch_source_downloader_test.py +++ b/tests/builder/downloaders/opensearch_source_downloader_test.py @@ -16,7 +16,7 @@ def setUp(self): self.template_renderer = Mock() self.artifact_variables_provider = Mock() - self.provision_config_instance = ProvisionConfigInstance(names="fake", root_path="also fake", config_paths="fake2", variables={ + self.cluster_config = ProvisionConfigInstance(names="fake", root_path="also fake", config_paths="fake2", variables={ "source": { "root": { "dir": "/fake/dir/for/source" @@ -40,7 +40,7 @@ def setUp(self): } }) - self.opensearch_source_downloader = OpenSearchSourceDownloader(self.provision_config_instance, self.executor, + self.opensearch_source_downloader = OpenSearchSourceDownloader(self.cluster_config, self.executor, self.source_repository_provider, self.binary_builder, self.template_renderer, self.artifact_variables_provider) diff --git a/tests/builder/downloaders/repositories/opensearch_distribution_repository_provider_test.py b/tests/builder/downloaders/repositories/opensearch_distribution_repository_provider_test.py index e6ad6392b..7b9accc51 100644 --- a/tests/builder/downloaders/repositories/opensearch_distribution_repository_provider_test.py +++ b/tests/builder/downloaders/repositories/opensearch_distribution_repository_provider_test.py @@ -9,7 +9,7 @@ class OpenSearchDistributionRepositoryProviderTest(TestCase): def setUp(self): self.host = None - self.provision_config_instance = ProvisionConfigInstance(names=None, config_paths=None, root_path=None, variables={ + self.cluster_config = ProvisionConfigInstance(names=None, config_paths=None, root_path=None, variables={ "system": { "runtime": { "jdk": { @@ -25,22 +25,22 @@ def setUp(self): } }) self.repository_url_provider = Mock() - self.os_distro_repo_provider = OpenSearchDistributionRepositoryProvider(self.provision_config_instance, + self.os_distro_repo_provider = OpenSearchDistributionRepositoryProvider(self.cluster_config, self.repository_url_provider) def test_get_url_bundled_jdk(self): self.os_distro_repo_provider.get_download_url(self.host) self.os_distro_repo_provider.repository_url_provider.render_url_for_key.assert_has_calls([ - mock.call(None, self.provision_config_instance.variables, "distribution.jdk.bundled.release_url") + mock.call(None, self.cluster_config.variables, "distribution.jdk.bundled.release_url") ]) def test_get_url_unbundled_jdk(self): - self.provision_config_instance.variables["system"]["runtime"]["jdk"]["bundled"] = False + self.cluster_config.variables["system"]["runtime"]["jdk"]["bundled"] = False self.os_distro_repo_provider.get_download_url(self.host) self.os_distro_repo_provider.repository_url_provider.render_url_for_key.assert_has_calls([ - mock.call(None, self.provision_config_instance.variables, "distribution.jdk.unbundled.release_url") + mock.call(None, self.cluster_config.variables, "distribution.jdk.unbundled.release_url") ]) def test_get_file_name(self): @@ -54,6 +54,6 @@ def test_is_cache_enabled_true(self): self.assertEqual(is_cache_enabled, True) def test_is_cache_enabled_false(self): - self.provision_config_instance.variables["distribution"]["release"]["cache"] = False + self.cluster_config.variables["distribution"]["release"]["cache"] = False is_cache_enabled = self.os_distro_repo_provider.is_cache_enabled() self.assertEqual(is_cache_enabled, False) diff --git a/tests/builder/installers/bare_installer_test.py b/tests/builder/installers/bare_installer_test.py index e14ee5d5f..eafeac9f7 100644 --- a/tests/builder/installers/bare_installer_test.py +++ b/tests/builder/installers/bare_installer_test.py @@ -21,7 +21,7 @@ def setUp(self): self.preparer = Mock() self.preparer2 = Mock() - self.provision_config_instance = ProvisionConfigInstance( + self.cluster_config = ProvisionConfigInstance( names="defaults", root_path="fake", config_paths=["/tmp"], @@ -34,7 +34,7 @@ def setUp(self): "preserve_install": False } ) - self.installer = BareInstaller(self.provision_config_instance, self.executor, self.preparer) + self.installer = BareInstaller(self.cluster_config, self.executor, self.preparer) self.installer.config_applier = Mock() self.installer.java_home_resolver = Mock() @@ -60,7 +60,7 @@ def test_install_only_opensearch(self): mock.call(self.host, "fake node", ["/tmp"], {"fake": "config"}) ]) self.installer.java_home_resolver.resolve_java_home.assert_has_calls([ - mock.call(self.host, self.provision_config_instance) + mock.call(self.host, self.cluster_config) ]) self.preparer.invoke_install_hook.assert_has_calls([ mock.call(self.host, BootstrapPhase.post_install, {"fake": "config"}, {"JAVA_HOME": "/path/to/java/home"}) @@ -115,7 +115,7 @@ def test_opensearch_and_plugin_installation(self): mock.call(self.host, None, ["/fake"], expected_config_vars) ]) self.installer.java_home_resolver.resolve_java_home.assert_has_calls([ - mock.call(self.host, self.provision_config_instance) + mock.call(self.host, self.cluster_config) ]) self.preparer.invoke_install_hook.assert_has_calls([ mock.call(self.host, BootstrapPhase.post_install, expected_config_vars, {"JAVA_HOME": "/path/to/java/home"}) diff --git a/tests/builder/installers/docker_installer_test.py b/tests/builder/installers/docker_installer_test.py index 4d2805020..e8f90fe01 100644 --- a/tests/builder/installers/docker_installer_test.py +++ b/tests/builder/installers/docker_installer_test.py @@ -23,7 +23,7 @@ def setUp(self): self.node_heap_dump_dir = os.path.join(self.node_root_dir, "heapdump") self.executor = Mock() - self.provision_config_instance = ProvisionConfigInstance( + self.cluster_config = ProvisionConfigInstance( names="fake", root_path=None, config_paths="/tmp", @@ -44,7 +44,7 @@ def setUp(self): } ) - self.installer = DockerInstaller(self.provision_config_instance, self.executor) + self.installer = DockerInstaller(self.cluster_config, self.executor) maxDiff = None @mock.patch("uuid.uuid4") @@ -136,8 +136,8 @@ def test_provisioning_with_variables(self, benchmark_root, uuid4): benchmark_root.return_value = os.path.normpath(os.path.join(os.path.dirname(os.path.realpath(__file__)), os.pardir, os.pardir, os.pardir, "osbenchmark")) - self.provision_config_instance.variables["origin"]["docker"]["docker_mem_limit"] = "256m" - self.provision_config_instance.variables["origin"]["docker"]["docker_cpu_count"] = 2 + self.cluster_config.variables["origin"]["docker"]["docker_mem_limit"] = "256m" + self.cluster_config.variables["origin"]["docker"]["docker_cpu_count"] = 2 node = self.installer._create_node() diff --git a/tests/builder/installers/preparers/opensearch_preparer_test.py b/tests/builder/installers/preparers/opensearch_preparer_test.py index 8a4a99137..d74f335af 100644 --- a/tests/builder/installers/preparers/opensearch_preparer_test.py +++ b/tests/builder/installers/preparers/opensearch_preparer_test.py @@ -25,7 +25,7 @@ def setUp(self): self.executor = Mock() self.hook_handler_class = Mock() - self.provision_config_instance = ProvisionConfigInstance( + self.cluster_config = ProvisionConfigInstance( names="defaults", root_path="fake", config_paths=["/tmp"], @@ -37,7 +37,7 @@ def setUp(self): } } ) - self.preparer = OpenSearchPreparer(self.provision_config_instance, self.executor, self.hook_handler_class) + self.preparer = OpenSearchPreparer(self.cluster_config, self.executor, self.hook_handler_class) self.preparer.path_manager = Mock() @mock.patch("uuid.uuid4") diff --git a/tests/builder/launcher_test.py b/tests/builder/launcher_test.py index 5e95ea1fd..f38ef00c3 100644 --- a/tests/builder/launcher_test.py +++ b/tests/builder/launcher_test.py @@ -158,7 +158,7 @@ def get_metrics_store(cfg): test_ex_timestamp=datetime.now(), workload_name="test", test_procedure_name="test", - provision_config_instance_name="test") + cluster_config_name="test") return ms @@ -187,8 +187,8 @@ def test_daemon_start_stop(self, wait_for_pidfile, chdir, get_size, supports, ja node_configs = [] for node in range(2): node_configs.append(NodeConfiguration(build_type="tar", - provision_config_instance_runtime_jdks="12,11", - provision_config_instance_provides_bundled_jdk=True, + cluster_config_runtime_jdks="12,11", + cluster_config_provides_bundled_jdk=True, ip="127.0.0.1", node_name="testnode-{}".format(node), node_root_path="/tmp", @@ -373,8 +373,8 @@ def test_starts_container_successfully(self, run_subprocess_with_output, run_sub docker = launcher.DockerLauncher(cfg) node_config = NodeConfiguration(build_type="docker", - provision_config_instance_runtime_jdks="12,11", - provision_config_instance_provides_bundled_jdk=True, + cluster_config_runtime_jdks="12,11", + cluster_config_provides_bundled_jdk=True, ip="127.0.0.1", node_name="testnode", node_root_path="/tmp", binary_path="/bin", data_paths="/tmp") @@ -408,8 +408,8 @@ def test_container_not_started(self, run_subprocess_with_output, run_subprocess_ docker = launcher.DockerLauncher(cfg, clock=TestClock(stop_watch=stop_watch)) node_config = NodeConfiguration( - build_type="docker", provision_config_instance_runtime_jdks="12,11", - provision_config_instance_provides_bundled_jdk=True, + build_type="docker", cluster_config_runtime_jdks="12,11", + cluster_config_provides_bundled_jdk=True, ip="127.0.0.1", node_name="testnode", node_root_path="/tmp", binary_path="/bin", data_paths="/tmp") diff --git a/tests/builder/launchers/docker_launcher_test.py b/tests/builder/launchers/docker_launcher_test.py index 2ac75855f..3acc64d05 100644 --- a/tests/builder/launchers/docker_launcher_test.py +++ b/tests/builder/launchers/docker_launcher_test.py @@ -19,8 +19,8 @@ def setUp(self): self.host = None self.node_config = NodeConfiguration(build_type="docker", - provision_config_instance_runtime_jdks="12,11", - provision_config_instance_provides_bundled_jdk=True, + cluster_config_runtime_jdks="12,11", + cluster_config_provides_bundled_jdk=True, ip="127.0.0.1", node_name="testnode", node_root_path="/tmp", binary_path="/bin", data_paths="/tmp") diff --git a/tests/builder/launchers/local_process_launcher_test.py b/tests/builder/launchers/local_process_launcher_test.py index 746be44ee..6d00da90e 100644 --- a/tests/builder/launchers/local_process_launcher_test.py +++ b/tests/builder/launchers/local_process_launcher_test.py @@ -32,10 +32,10 @@ def setUp(self): "params": None } } - self.provision_config_instance = ProvisionConfigInstance("fake_provision_config_instance", "/path/to/root", + self.cluster_config = ProvisionConfigInstance("fake_cluster_config", "/path/to/root", ["/path/to/config"], variables=self.variables) - self.launcher = LocalProcessLauncher(self.provision_config_instance, self.shell_executor, self.metrics_store) + self.launcher = LocalProcessLauncher(self.cluster_config, self.shell_executor, self.metrics_store) self.launcher.waiter = Mock() self.host = None self.path = "fake" @@ -51,8 +51,8 @@ def test_daemon_start_stop(self, process, telemetry, get_size, supports, java_ho node_configs = [] for node in range(2): node_configs.append(NodeConfiguration(build_type="tar", - provision_config_instance_runtime_jdks="12,11", - provision_config_instance_provides_bundled_jdk=True, + cluster_config_runtime_jdks="12,11", + cluster_config_provides_bundled_jdk=True, ip="127.0.0.1", node_name=f"testnode-{node}", node_root_path="/tmp", @@ -112,7 +112,7 @@ def test_bundled_jdk_not_in_path(self): self.assertIsNone(env.get("JAVA_HOME")) def test_pass_env_vars(self): - self.provision_config_instance.variables["system"]["env"]["passenv"] = "JAVA_HOME,FOO1" + self.cluster_config.variables["system"]["env"]["passenv"] = "JAVA_HOME,FOO1" os.environ["JAVA_HOME"] = "/path/to/java" os.environ["FOO1"] = "BAR1" @@ -127,7 +127,7 @@ def test_pass_env_vars(self): self.assertEqual(env["OPENSEARCH_JAVA_OPTS"], "-XX:+ExitOnOutOfMemoryError") def test_pass_java_opts(self): - self.provision_config_instance.variables["system"]["env"]["passenv"] = "OPENSEARCH_JAVA_OPTS" + self.cluster_config.variables["system"]["env"]["passenv"] = "OPENSEARCH_JAVA_OPTS" os.environ["OPENSEARCH_JAVA_OPTS"] = "-XX:-someJunk" diff --git a/tests/builder/provision_config_test.py b/tests/builder/provision_config_test.py index 868c4134c..dfc635e8c 100644 --- a/tests/builder/provision_config_test.py +++ b/tests/builder/provision_config_test.py @@ -41,118 +41,118 @@ def setUp(self): self.provision_config_dir = os.path.join(current_dir, "data") self.loader = provision_config.ProvisionConfigInstanceLoader(self.provision_config_dir) - def test_lists_provision_config_instance_names(self): + def test_lists_cluster_config_names(self): # contrary to the name this assertion compares contents but does not care about order. self.assertCountEqual( ["default", "with_hook", "32gheap", "missing_cfg_base", "empty_cfg_base", "ea", "verbose", "multi_hook", "another_with_hook"], - self.loader.provision_config_instance_names() + self.loader.cluster_config_names() ) - def test_load_known_provision_config_instance(self): - provision_config_instance = provision_config.load_provision_config_instance( + def test_load_known_cluster_config(self): + cluster_config = provision_config.load_cluster_config( self.provision_config_dir, ["default"], - provision_config_instance_params={"data_paths": ["/mnt/disk0", "/mnt/disk1"]}) - self.assertEqual("default", provision_config_instance.name) + cluster_config_params={"data_paths": ["/mnt/disk0", "/mnt/disk1"]}) + self.assertEqual("default", cluster_config.name) self.assertEqual( - [os.path.join(current_dir, "data", "provision_config_instances", "v1", "vanilla", "templates")], - provision_config_instance.config_paths) - self.assertIsNone(provision_config_instance.root_path) + [os.path.join(current_dir, "data", "cluster_configs", "v1", "vanilla", "templates")], + cluster_config.config_paths) + self.assertIsNone(cluster_config.root_path) self.assertDictEqual({ "heap_size": "1g", "clean_command": "./gradlew clean", "data_paths": ["/mnt/disk0", "/mnt/disk1"] - }, provision_config_instance.variables) - self.assertIsNone(provision_config_instance.root_path) + }, cluster_config.variables) + self.assertIsNone(cluster_config.root_path) - def test_load_provision_config_instance_with_mixin_single_config_base(self): - provision_config_instance = provision_config.load_provision_config_instance(self.provision_config_dir, ["32gheap", "ea"]) - self.assertEqual("32gheap+ea", provision_config_instance.name) + def test_load_cluster_config_with_mixin_single_config_base(self): + cluster_config = provision_config.load_cluster_config(self.provision_config_dir, ["32gheap", "ea"]) + self.assertEqual("32gheap+ea", cluster_config.name) self.assertEqual( - [os.path.join(current_dir, "data", "provision_config_instances", "v1", "vanilla", "templates")], - provision_config_instance.config_paths) - self.assertIsNone(provision_config_instance.root_path) + [os.path.join(current_dir, "data", "cluster_configs", "v1", "vanilla", "templates")], + cluster_config.config_paths) + self.assertIsNone(cluster_config.root_path) self.assertEqual({ "heap_size": "32g", "clean_command": "./gradlew clean", "assertions": "true" - }, provision_config_instance.variables) - self.assertIsNone(provision_config_instance.root_path) + }, cluster_config.variables) + self.assertIsNone(cluster_config.root_path) - def test_load_provision_config_instance_with_mixin_multiple_config_bases(self): - provision_config_instance = provision_config.load_provision_config_instance(self.provision_config_dir, ["32gheap", "ea", "verbose"]) - self.assertEqual("32gheap+ea+verbose", provision_config_instance.name) + def test_load_cluster_config_with_mixin_multiple_config_bases(self): + cluster_config = provision_config.load_cluster_config(self.provision_config_dir, ["32gheap", "ea", "verbose"]) + self.assertEqual("32gheap+ea+verbose", cluster_config.name) self.assertEqual([ - os.path.join(current_dir, "data", "provision_config_instances", "v1", "vanilla", "templates"), - os.path.join(current_dir, "data", "provision_config_instances", "v1", "verbose_logging", "templates"), - ], provision_config_instance.config_paths) - self.assertIsNone(provision_config_instance.root_path) + os.path.join(current_dir, "data", "cluster_configs", "v1", "vanilla", "templates"), + os.path.join(current_dir, "data", "cluster_configs", "v1", "verbose_logging", "templates"), + ], cluster_config.config_paths) + self.assertIsNone(cluster_config.root_path) self.assertEqual({ "heap_size": "32g", "clean_command": "./gradlew clean", "verbose_logging": "true", "assertions": "true" - }, provision_config_instance.variables) + }, cluster_config.variables) - def test_load_provision_config_instance_with_install_hook(self): - provision_config_instance = provision_config.load_provision_config_instance( + def test_load_cluster_config_with_install_hook(self): + cluster_config = provision_config.load_cluster_config( self.provision_config_dir, ["default", "with_hook"], - provision_config_instance_params={"data_paths": ["/mnt/disk0", "/mnt/disk1"]}) - self.assertEqual("default+with_hook", provision_config_instance.name) + cluster_config_params={"data_paths": ["/mnt/disk0", "/mnt/disk1"]}) + self.assertEqual("default+with_hook", cluster_config.name) self.assertEqual([ - os.path.join(current_dir, "data", "provision_config_instances", "v1", "vanilla", "templates"), - os.path.join(current_dir, "data", "provision_config_instances", "v1", "with_hook", "templates"), - ], provision_config_instance.config_paths) + os.path.join(current_dir, "data", "cluster_configs", "v1", "vanilla", "templates"), + os.path.join(current_dir, "data", "cluster_configs", "v1", "with_hook", "templates"), + ], cluster_config.config_paths) self.assertEqual( - os.path.join(current_dir, "data", "provision_config_instances", "v1", "with_hook"), - provision_config_instance.root_path) + os.path.join(current_dir, "data", "cluster_configs", "v1", "with_hook"), + cluster_config.root_path) self.assertDictEqual({ "heap_size": "1g", "clean_command": "./gradlew clean", "data_paths": ["/mnt/disk0", "/mnt/disk1"] - }, provision_config_instance.variables) + }, cluster_config.variables) - def test_load_provision_config_instance_with_multiple_bases_referring_same_install_hook(self): - provision_config_instance = provision_config.load_provision_config_instance( + def test_load_cluster_config_with_multiple_bases_referring_same_install_hook(self): + cluster_config = provision_config.load_cluster_config( self.provision_config_dir, ["with_hook", "another_with_hook"]) - self.assertEqual("with_hook+another_with_hook", provision_config_instance.name) + self.assertEqual("with_hook+another_with_hook", cluster_config.name) self.assertEqual([ - os.path.join(current_dir, "data", "provision_config_instances", "v1", "vanilla", "templates"), - os.path.join(current_dir, "data", "provision_config_instances", "v1", "with_hook", "templates"), - os.path.join(current_dir, "data", "provision_config_instances", "v1", "verbose_logging", "templates") - ], provision_config_instance.config_paths) + os.path.join(current_dir, "data", "cluster_configs", "v1", "vanilla", "templates"), + os.path.join(current_dir, "data", "cluster_configs", "v1", "with_hook", "templates"), + os.path.join(current_dir, "data", "cluster_configs", "v1", "verbose_logging", "templates") + ], cluster_config.config_paths) self.assertEqual( - os.path.join(current_dir, "data", "provision_config_instances", "v1", "with_hook"), - provision_config_instance.root_path) + os.path.join(current_dir, "data", "cluster_configs", "v1", "with_hook"), + cluster_config.root_path) self.assertDictEqual({ "heap_size": "16g", "clean_command": "./gradlew clean", "verbose_logging": "true" - }, provision_config_instance.variables) + }, cluster_config.variables) - def test_raises_error_on_unknown_provision_config_instance(self): + def test_raises_error_on_unknown_cluster_config(self): with self.assertRaises(exceptions.SystemSetupError) as ctx: - provision_config.load_provision_config_instance(self.provision_config_dir, ["don_t-know-you"]) + provision_config.load_cluster_config(self.provision_config_dir, ["don_t-know-you"]) self.assertRegex( ctx.exception.args[0], - r"Unknown provision_config_instance \[don_t-know-you\]. " - r"List the available provision_config_instances with [^\s]+ list provision_config_instances.") + r"Unknown cluster-config \[don_t-know-you\]. " + r"List the available cluster-configs with [^\s]+ list cluster-configs.") def test_raises_error_on_empty_config_base(self): with self.assertRaises(exceptions.SystemSetupError) as ctx: - provision_config.load_provision_config_instance(self.provision_config_dir, ["empty_cfg_base"]) - self.assertEqual("At least one config base is required for provision_config_instance ['empty_cfg_base']", ctx.exception.args[0]) + provision_config.load_cluster_config(self.provision_config_dir, ["empty_cfg_base"]) + self.assertEqual("At least one config base is required for cluster_config ['empty_cfg_base']", ctx.exception.args[0]) def test_raises_error_on_missing_config_base(self): with self.assertRaises(exceptions.SystemSetupError) as ctx: - provision_config.load_provision_config_instance(self.provision_config_dir, ["missing_cfg_base"]) - self.assertEqual("At least one config base is required for provision_config_instance ['missing_cfg_base']", ctx.exception.args[0]) + provision_config.load_cluster_config(self.provision_config_dir, ["missing_cfg_base"]) + self.assertEqual("At least one config base is required for cluster_config ['missing_cfg_base']", ctx.exception.args[0]) def test_raises_error_if_more_than_one_different_install_hook(self): with self.assertRaises(exceptions.SystemSetupError) as ctx: - provision_config.load_provision_config_instance(self.provision_config_dir, ["multi_hook"]) + provision_config.load_cluster_config(self.provision_config_dir, ["multi_hook"]) self.assertEqual( - "Invalid provision_config_instance: ['multi_hook']. Multiple bootstrap hooks are forbidden.", + "Invalid cluster_config: ['multi_hook']. Multiple bootstrap hooks are forbidden.", ctx.exception.args[0]) diff --git a/tests/builder/provisioner_test.py b/tests/builder/provisioner_test.py index 4b1539f59..66542fd63 100644 --- a/tests/builder/provisioner_test.py +++ b/tests/builder/provisioner_test.py @@ -45,7 +45,7 @@ def test_prepare_without_plugins(self, mock_rm, mock_ensure_dir, mock_decompress def null_apply_config(source_root_path, target_root_path, config_vars): apply_config_calls.append((source_root_path, target_root_path, config_vars)) - installer = provisioner.OpenSearchInstaller(provision_config_instance= + installer = provisioner.OpenSearchInstaller(cluster_config= provision_config.ProvisionConfigInstance( names="unit-test-provision-config-instance", root_path=None, @@ -64,7 +64,7 @@ def null_apply_config(source_root_path, target_root_path, config_vars): apply_config=null_apply_config) node_config = p.prepare({"opensearch": "/opt/opensearch-1.0.0.tar.gz"}) - self.assertEqual("8", node_config.provision_config_instance_runtime_jdks) + self.assertEqual("8", node_config.cluster_config_runtime_jdks) self.assertEqual("/opt/opensearch-1.0.0", node_config.binary_path) self.assertEqual(["/opt/opensearch-1.0.0/data"], node_config.data_paths) @@ -128,7 +128,7 @@ class OpenSearchInstallerTests(TestCase): @mock.patch("osbenchmark.utils.io.ensure_dir") @mock.patch("shutil.rmtree") def test_prepare_default_data_paths(self, mock_rm, mock_ensure_dir, mock_decompress): - installer = provisioner.OpenSearchInstaller(provision_config_instance=provision_config.ProvisionConfigInstance(names="defaults", + installer = provisioner.OpenSearchInstaller(cluster_config=provision_config.ProvisionConfigInstance(names="defaults", root_path=None, config_paths="/tmp"), java_home="/usr/local/javas/java8", @@ -165,7 +165,7 @@ def test_prepare_default_data_paths(self, mock_rm, mock_ensure_dir, mock_decompr @mock.patch("osbenchmark.utils.io.ensure_dir") @mock.patch("shutil.rmtree") def test_prepare_user_provided_data_path(self, mock_rm, mock_ensure_dir, mock_decompress): - installer = provisioner.OpenSearchInstaller(provision_config_instance=provision_config.ProvisionConfigInstance(names="defaults", + installer = provisioner.OpenSearchInstaller(cluster_config=provision_config.ProvisionConfigInstance(names="defaults", root_path=None, config_paths="/tmp", variables={"data_paths": "/tmp/some/data-path-dir"}), @@ -199,7 +199,7 @@ def test_prepare_user_provided_data_path(self, mock_rm, mock_ensure_dir, mock_de self.assertEqual(installer.data_paths, ["/tmp/some/data-path-dir"]) def test_invokes_hook_with_java_home(self): - installer = provisioner.OpenSearchInstaller(provision_config_instance=provision_config.ProvisionConfigInstance(names="defaults", + installer = provisioner.OpenSearchInstaller(cluster_config=provision_config.ProvisionConfigInstance(names="defaults", root_path="/tmp", config_paths="/tmp/templates", variables={"data_paths": "/tmp/some/data-path-dir"}), @@ -220,7 +220,7 @@ def test_invokes_hook_with_java_home(self): installer.hook_handler.hook_calls["post_install"]["kwargs"]) def test_invokes_hook_no_java_home(self): - installer = provisioner.OpenSearchInstaller(provision_config_instance=provision_config.ProvisionConfigInstance(names="defaults", + installer = provisioner.OpenSearchInstaller(cluster_config=provision_config.ProvisionConfigInstance(names="defaults", root_path="/tmp", config_paths="/tmp/templates", variables={"data_paths": "/tmp/some/data-path-dir"}), @@ -388,7 +388,7 @@ def test_provisioning_with_defaults(self, uuid4): "docker_image": "opensearchproject/opensearch" }) - docker = provisioner.DockerProvisioner(provision_config_instance=c, + docker = provisioner.DockerProvisioner(cluster_config=c, node_name="benchmark-node-0", ip="10.17.22.33", http_port=39200, @@ -483,7 +483,7 @@ def test_provisioning_with_variables(self, uuid4): "docker_cpu_count": 2 }) - docker = provisioner.DockerProvisioner(provision_config_instance=c, + docker = provisioner.DockerProvisioner(cluster_config=c, node_name="benchmark-node-0", ip="10.17.22.33", http_port=39200, diff --git a/tests/builder/supplier_test.py b/tests/builder/supplier_test.py index 25944705d..770f38ca2 100644 --- a/tests/builder/supplier_test.py +++ b/tests/builder/supplier_test.py @@ -471,7 +471,7 @@ def test_prunes_old_files(self, rm, lstat, isfile, listdir, exists): class OpenSearchSourceSupplierTests(TestCase): def test_no_build(self): - provision_config_instance = provision_config.ProvisionConfigInstance("default", root_path=None, config_paths=[], variables={ + cluster_config = provision_config.ProvisionConfigInstance("default", root_path=None, config_paths=[], variables={ "clean_command": "./gradlew clean", "system.build_command": "./gradlew assemble" }) @@ -479,14 +479,14 @@ def test_no_build(self): opensearch = supplier.OpenSearchSourceSupplier(revision="abc", os_src_dir="/src", remote_url="", - provision_config_instance=provision_config_instance, + cluster_config=cluster_config, builder=None, template_renderer=renderer) opensearch.prepare() # nothing has happened (intentionally) because there is no builder def test_build(self): - provision_config_instance = provision_config.ProvisionConfigInstance("default", root_path=None, config_paths=[], variables={ + cluster_config = provision_config.ProvisionConfigInstance("default", root_path=None, config_paths=[], variables={ "clean_command": "./gradlew clean", "system.build_command": "./gradlew assemble" }) @@ -495,15 +495,15 @@ def test_build(self): opensearch = supplier.OpenSearchSourceSupplier(revision="abc", os_src_dir="/src", remote_url="", - provision_config_instance=provision_config_instance, + cluster_config=cluster_config, builder=builder, template_renderer=renderer) opensearch.prepare() builder.build.assert_called_once_with(["./gradlew clean", "./gradlew assemble"]) - def test_raises_error_on_missing_provision_config_instance_variable(self): - provision_config_instance = provision_config.ProvisionConfigInstance("default", root_path=None, config_paths=[], variables={ + def test_raises_error_on_missing_cluster_config_variable(self): + cluster_config = provision_config.ProvisionConfigInstance("default", root_path=None, config_paths=[], variables={ "clean_command": "./gradlew clean", # system.build_command is not defined }) @@ -512,7 +512,7 @@ def test_raises_error_on_missing_provision_config_instance_variable(self): opensearch = supplier.OpenSearchSourceSupplier(revision="abc", os_src_dir="/src", remote_url="", - provision_config_instance=provision_config_instance, + cluster_config=cluster_config, builder=builder, template_renderer=renderer) with self.assertRaisesRegex(exceptions.SystemSetupError, @@ -523,7 +523,7 @@ def test_raises_error_on_missing_provision_config_instance_variable(self): @mock.patch("glob.glob", lambda p: ["opensearch.tar.gz"]) def test_add_opensearch_binary(self): - provision_config_instance = provision_config.ProvisionConfigInstance("default", root_path=None, config_paths=[], variables={ + cluster_config = provision_config.ProvisionConfigInstance("default", root_path=None, config_paths=[], variables={ "clean_command": "./gradlew clean", "system.build_command": "./gradlew assemble", "system.artifact_path_pattern": "distribution/archives/tar/build/distributions/*.tar.gz" @@ -532,7 +532,7 @@ def test_add_opensearch_binary(self): opensearch = supplier.OpenSearchSourceSupplier(revision="abc", os_src_dir="/src", remote_url="", - provision_config_instance=provision_config_instance, + cluster_config=cluster_config, builder=None, template_renderer=renderer) binaries = {} @@ -699,9 +699,9 @@ def test_create_suppliers_for_os_only_config(self): cfg.add(config.Scope.application, "distributions", "release.cache", True) cfg.add(config.Scope.application, "node", "root.dir", "/opt/benchmark") - provision_config_instance = provision_config.ProvisionConfigInstance("default", root_path=None, config_paths=[]) + cluster_config = provision_config.ProvisionConfigInstance("default", root_path=None, config_paths=[]) - composite_supplier = supplier.create(cfg, sources=False, distribution=True, provision_config_instance=provision_config_instance) + composite_supplier = supplier.create(cfg, sources=False, distribution=True, cluster_config=cluster_config) self.assertEqual(1, len(composite_supplier.suppliers)) self.assertIsInstance(composite_supplier.suppliers[0], supplier.OpenSearchDistributionSupplier) @@ -721,7 +721,7 @@ def test_create_suppliers_for_os_distribution_plugin_source_build(self): cfg.add(config.Scope.application, "source", "opensearch.src.subdir", "opensearch") cfg.add(config.Scope.application, "source", "plugin.community-plugin.src.dir", "/home/user/Projects/community-plugin") - provision_config_instance = provision_config.ProvisionConfigInstance( + cluster_config = provision_config.ProvisionConfigInstance( "default", root_path=None, config_paths=[], variables={"build.jdk": "10"}) core_plugin = provision_config.PluginDescriptor("analysis-icu", core_plugin=True) @@ -730,7 +730,7 @@ def test_create_suppliers_for_os_distribution_plugin_source_build(self): # --revision="community-plugin:effab" --distribution-version="1.0.0" composite_supplier = supplier.create( cfg, sources=False, distribution=True, - provision_config_instance=provision_config_instance, plugins=[ + cluster_config=cluster_config, plugins=[ core_plugin, external_plugin ]) @@ -757,7 +757,7 @@ def test_create_suppliers_for_os_and_plugin_source_build(self): cfg.add(config.Scope.application, "source", "remote.repo.url", "https://github.com/opensearch-project/OpenSearch.git") cfg.add(config.Scope.application, "source", "plugin.community-plugin.src.subdir", "opensearch-extra/community-plugin") - provision_config_instance = provision_config.ProvisionConfigInstance("default", root_path=None, config_paths=[], variables={ + cluster_config = provision_config.ProvisionConfigInstance("default", root_path=None, config_paths=[], variables={ "clean_command": "./gradlew clean", "build_command": "./gradlew assemble", "build.jdk": "11" @@ -768,7 +768,7 @@ def test_create_suppliers_for_os_and_plugin_source_build(self): # --revision="opensearch:abc,community-plugin:effab" composite_supplier = supplier.create( cfg, sources=True, distribution=False, - provision_config_instance=provision_config_instance, plugins=[ + cluster_config=cluster_config, plugins=[ core_plugin, external_plugin ]) diff --git a/tests/builder/utils/java_home_resolver_test.py b/tests/builder/utils/java_home_resolver_test.py index 992f52875..905b4f536 100644 --- a/tests/builder/utils/java_home_resolver_test.py +++ b/tests/builder/utils/java_home_resolver_test.py @@ -23,13 +23,13 @@ def setUp(self): } } } - self.provision_config_instance = ProvisionConfigInstance("fake_provision_config_instance", "/path/to/root", + self.cluster_config = ProvisionConfigInstance("fake_cluster_config", "/path/to/root", ["/path/to/config"], variables=self.variables) def test_resolves_java_home_for_default_runtime_jdk(self): self.executor.execute.return_value = ["Darwin"] self.java_home_resolver.jdk_resolver.resolve_jdk_path.return_value = (12, "/opt/jdk12") - major, java_home = self.java_home_resolver.resolve_java_home(self.host, self.provision_config_instance) + major, java_home = self.java_home_resolver.resolve_java_home(self.host, self.cluster_config) self.assertEqual(major, 12) self.assertEqual(java_home, "/opt/jdk12") @@ -38,7 +38,7 @@ def test_resolves_java_home_for_specific_runtime_jdk(self): self.variables["system"]["runtime"]["jdk"]["version"] = "8" self.executor.execute.return_value = ["Darwin"] self.java_home_resolver.jdk_resolver.resolve_jdk_path.return_value = (8, "/opt/jdk8") - major, java_home = self.java_home_resolver.resolve_java_home(self.host, self.provision_config_instance) + major, java_home = self.java_home_resolver.resolve_java_home(self.host, self.cluster_config) self.assertEqual(major, 8) self.assertEqual(java_home, "/opt/jdk8") @@ -46,7 +46,7 @@ def test_resolves_java_home_for_specific_runtime_jdk(self): def test_resolves_java_home_for_bundled_jdk_on_linux(self): self.executor.execute.return_value = ["Linux"] - major, java_home = self.java_home_resolver.resolve_java_home(self.host, self.provision_config_instance) + major, java_home = self.java_home_resolver.resolve_java_home(self.host, self.cluster_config) self.assertEqual(major, 12) self.assertEqual(java_home, None) @@ -54,5 +54,5 @@ def test_resolves_java_home_for_bundled_jdk_on_linux(self): def test_resolves_java_home_for_bundled_jdk_windows(self): self.executor.execute.return_value = ["Windows"] with self.assertRaises(SystemSetupError) as ctx: - self.java_home_resolver.resolve_java_home(self.host, self.provision_config_instance) + self.java_home_resolver.resolve_java_home(self.host, self.cluster_config) self.assertEqual("OpenSearch doesn't provide release artifacts for Windows currently.", ctx.exception.args[0]) diff --git a/tests/metrics_test.py b/tests/metrics_test.py index 0e223d40c..772e9171f 100644 --- a/tests/metrics_test.py +++ b/tests/metrics_test.py @@ -1295,10 +1295,10 @@ def test_store_test_execution(self): test_execution_timestamp=OsTestExecutionStoreTests.TEST_EXECUTION_TIMESTAMP, pipeline="from-sources", user_tags={"os": "Linux"}, workload=t, workload_params={"shard-count": 3}, test_procedure=t.default_test_procedure, - provision_config_instance="defaults", - provision_config_instance_params={"heap_size": "512mb"}, + cluster_config="defaults", + cluster_config_params={"heap_size": "512mb"}, plugin_params=None, - workload_revision="abc1", provision_config_revision="abc12333", distribution_version="5.0.0", + workload_revision="abc1", cluster_config_revision="abc12333", distribution_version="5.0.0", distribution_flavor="default", revision="aaaeeef", results=OsTestExecutionStoreTests.DictHolder( { @@ -1345,7 +1345,7 @@ def test_store_test_execution(self): "revision": "aaaeeef", "distribution-version": "5.0.0", "distribution-flavor": "default", - "provision-config-revision": "abc12333", + "cluster-config-revision": "abc12333", }, "results": { "young_gc_time": 100, @@ -1401,10 +1401,10 @@ def test_store_results(self): test_execution_timestamp=OsResultsStoreTests.TEST_EXECUTION_TIMESTAMP, pipeline="from-sources", user_tags={"os": "Linux"}, workload=t, workload_params=None, test_procedure=t.default_test_procedure, - provision_config_instance="4gheap", - provision_config_instance_params=None, + cluster_config="4gheap", + cluster_config_params=None, plugin_params={"some-param": True}, - workload_revision="abc1", provision_config_revision="123ab", distribution_version="5.0.0", + workload_revision="abc1", cluster_config_revision="123ab", distribution_version="5.0.0", distribution_flavor="oss", results=metrics.GlobalStats( { "young_gc_time": 100, @@ -1446,7 +1446,7 @@ def test_store_results(self): "os": "Linux" }, "workload": "unittest-workload", - "provision-config-revision": "123ab", + "cluster-config-revision": "123ab", "workload-revision": "abc1", "test_procedure": "index", "provision-config-instance": "4gheap", @@ -1476,7 +1476,7 @@ def test_store_results(self): "os": "Linux" }, "workload": "unittest-workload", - "provision-config-revision": "123ab", + "cluster-config-revision": "123ab", "workload-revision": "abc1", "test_procedure": "index", "provision-config-instance": "4gheap", @@ -1512,7 +1512,7 @@ def test_store_results(self): "os": "Linux" }, "workload": "unittest-workload", - "provision-config-revision": "123ab", + "cluster-config-revision": "123ab", "workload-revision": "abc1", "test_procedure": "index", "provision-config-instance": "4gheap", @@ -1551,11 +1551,11 @@ def test_store_results_with_missing_version(self): test_execution_timestamp=OsResultsStoreTests.TEST_EXECUTION_TIMESTAMP, pipeline="from-sources", user_tags={"os": "Linux"}, workload=t, workload_params=None, test_procedure=t.default_test_procedure, - provision_config_instance="4gheap", - provision_config_instance_params=None, + cluster_config="4gheap", + cluster_config_params=None, plugin_params=None, workload_revision="abc1", - provision_config_revision="123ab", + cluster_config_revision="123ab", distribution_version=None, distribution_flavor=None, results=metrics.GlobalStats( { @@ -1597,7 +1597,7 @@ def test_store_results_with_missing_version(self): "os": "Linux" }, "workload": "unittest-workload", - "provision-config-revision": "123ab", + "cluster-config-revision": "123ab", "workload-revision": "abc1", "test_procedure": "index", "provision-config-instance": "4gheap", @@ -1623,7 +1623,7 @@ def test_store_results_with_missing_version(self): "os": "Linux" }, "workload": "unittest-workload", - "provision-config-revision": "123ab", + "cluster-config-revision": "123ab", "workload-revision": "abc1", "test_procedure": "index", "provision-config-instance": "4gheap", @@ -1655,7 +1655,7 @@ def test_store_results_with_missing_version(self): "os": "Linux" }, "workload": "unittest-workload", - "provision-config-revision": "123ab", + "cluster-config-revision": "123ab", "workload-revision": "abc1", "test_procedure": "index", "provision-config-instance": "4gheap", @@ -1987,11 +1987,11 @@ def test_store_test_execution(self): test_execution_timestamp=FileTestExecutionStoreTests.TEST_EXECUTION_TIMESTAMP, pipeline="from-sources", user_tags={"os": "Linux"}, workload=t, workload_params={"clients": 12}, test_procedure=t.default_test_procedure, - provision_config_instance="4gheap", - provision_config_instance_params=None, + cluster_config="4gheap", + cluster_config_params=None, plugin_params=None, workload_revision="abc1", - provision_config_revision="abc12333", + cluster_config_revision="abc12333", distribution_version="5.0.0", distribution_flavor="default", revision="aaaeeef", results=FileTestExecutionStoreTests.DictHolder( @@ -2029,8 +2029,8 @@ def test_calculate_global_stats(self): cfg.add(config.Scope.application, "system", "time.start", datetime.datetime.now()) cfg.add(config.Scope.application, "system", "test_execution.id", "6ebc6e53-ee20-4b0c-99b4-09697987e9f4") cfg.add(config.Scope.application, "results_publishing", "datastore.type", "in-memory") - cfg.add(config.Scope.application, "builder", "provision_config_instance.names", ["unittest_provision_config_instance"]) - cfg.add(config.Scope.application, "builder", "provision_config_instance.params", {}) + cfg.add(config.Scope.application, "builder", "cluster_config.names", ["unittest_cluster_config"]) + cfg.add(config.Scope.application, "builder", "cluster_config.params", {}) cfg.add(config.Scope.application, "builder", "plugin.params", {}) cfg.add(config.Scope.application, "test_execution", "user.tag", "") cfg.add(config.Scope.application, "test_execution", "pipeline", "from-sources") @@ -2120,8 +2120,8 @@ def test_calculate_system_stats(self): cfg.add(config.Scope.application, "system", "time.start", datetime.datetime.now()) cfg.add(config.Scope.application, "system", "test_execution.id", "6ebc6e53-ee20-4b0c-99b4-09697987e9f4") cfg.add(config.Scope.application, "results_publishing", "datastore.type", "in-memory") - cfg.add(config.Scope.application, "builder", "provision_config_instance.names", ["unittest_provision_config_instance"]) - cfg.add(config.Scope.application, "builder", "provision_config_instance.params", {}) + cfg.add(config.Scope.application, "builder", "cluster_config.names", ["unittest_cluster_config"]) + cfg.add(config.Scope.application, "builder", "cluster_config.params", {}) cfg.add(config.Scope.application, "builder", "plugin.params", {}) cfg.add(config.Scope.application, "test_execution", "user.tag", "") cfg.add(config.Scope.application, "test_execution", "pipeline", "from-sources") diff --git a/tests/worker_coordinator/worker_coordinator_test.py b/tests/worker_coordinator/worker_coordinator_test.py index a6aba7507..bdd1c66ef 100644 --- a/tests/worker_coordinator/worker_coordinator_test.py +++ b/tests/worker_coordinator/worker_coordinator_test.py @@ -105,7 +105,7 @@ def setUp(self): self.cfg.add(config.Scope.application, "workload", "test.mode.enabled", True) self.cfg.add(config.Scope.application, "telemetry", "devices", []) self.cfg.add(config.Scope.application, "telemetry", "params", {"ccr-stats-indices": {"default": ["leader_index"]}}) - self.cfg.add(config.Scope.application, "builder", "provision_config_instance.names", ["default"]) + self.cfg.add(config.Scope.application, "builder", "cluster_config.names", ["default"]) self.cfg.add(config.Scope.application, "builder", "skip.rest.api.check", True) self.cfg.add(config.Scope.application, "client", "hosts", WorkerCoordinatorTests.Holder(all_hosts={"default": ["localhost:9200"]}))