set default pipeline to benchmark-only (#387)

Signed-off-by: Chinmay Gadgil <[email protected]>

it/sources_test.py

@@ -28,7 +28,8 @@
 def test_sources(cfg):
     port = 19200
     it.wait_until_port_is_free(port_number=port)
-    assert it.execute_test(cfg, f"--revision=latest --workload=geonames --test-mode  --target-hosts=127.0.0.1:{port} "
+    assert it.execute_test(cfg, f"--pipeline=from-sources --revision=latest \
+                           --workload=geonames --test-mode  --target-hosts=127.0.0.1:{port} "
                         f"--test-procedure=append-no-conflicts --provision-config-instance=4gheap "
                         f"--opensearch-plugins=analysis-icu") == 0
 

osbenchmark/test_execution_orchestrator.py

@@ -344,16 +344,18 @@ def list_pipelines():
 
 def run(cfg):
     logger = logging.getLogger(__name__)
-    name = cfg.opts("test_execution", "pipeline")
+    # pipeline is no more mandatory, will default to benchmark-only
+    name = cfg.opts("test_execution", "pipeline", mandatory=False)
     test_execution_id = cfg.opts("system", "test_execution.id")
     logger.info("Test Execution id [%s]", test_execution_id)
-    if len(name) == 0:
-        # assume from-distribution pipeline if distribution.version has been specified and --pipeline cli arg not set
+    if not name:
+        # assume from-distribution pipeline if distribution.version has been specified
         if cfg.exists("builder", "distribution.version"):
             name = "from-distribution"
         else:
-            name = "from-sources"
-        logger.info("User specified no pipeline. Automatically derived pipeline [%s].", name)
+            name = "benchmark-only"
+            logger.info("User did not specify distribution.version or pipeline. Using default pipeline [%s].", name)
+
         cfg.add(config.Scope.applicationOverride, "test_execution", "pipeline", name)
     else:
         logger.info("User specified pipeline [%s].", name)

tests/test_execution_orchestrator_test.py

@@ -114,3 +114,12 @@ def test_runs_a_known_pipeline(unittest_pipeline):
     test_execution_orchestrator.run(cfg)
 
     unittest_pipeline.target.assert_called_once_with(cfg)
+
+def test_runs_a_default_pipeline(benchmark_only_pipeline):
+    # with no pipeline specified, should default to benchmark-only
+    cfg = config.Config()
+    cfg.add(config.Scope.benchmark, "system", "test_execution.id", "28a032d1-0b03-4579-ad2a-c65316f126e9")
+
+    test_execution_orchestrator.run(cfg)
+
+    benchmark_only_pipeline.target.assert_called_once_with(cfg)