From fdb9389a07b703666d8166b0f3c8d59eba098d2b Mon Sep 17 00:00:00 2001 From: Jeremy Poulin Date: Thu, 25 Apr 2019 16:06:19 -0400 Subject: [PATCH 1/8] Added override for linchpinTarget and branch name (still defaults to arch). --- src/com/redhat/ci/Job.groovy | 6 +++- src/com/redhat/ci/hosts/Host.groovy | 3 ++ .../redhat/ci/hosts/ProvisionedHost.groovy | 3 ++ src/com/redhat/ci/hosts/TargetHost.groovy | 6 ++++ .../provisioners/LinchPinProvisioner.groovy | 13 +++++--- test/TestUtilsTest.groovy | 30 +++++++++++++++++++ test/resources/linchpin.latest | 2 +- vars/TestUtils.groovy | 6 ++-- 8 files changed, 60 insertions(+), 9 deletions(-) diff --git a/src/com/redhat/ci/Job.groovy b/src/com/redhat/ci/Job.groovy index f6cb097..eb6ca22 100644 --- a/src/com/redhat/ci/Job.groovy +++ b/src/com/redhat/ci/Job.groovy @@ -1,5 +1,7 @@ package com.redhat.ci +import java.util.logging.Logger +import java.util.logging.Level import com.redhat.ci.provisioner.ProvisioningConfig import com.redhat.ci.provisioner.ProvisioningService import com.redhat.ci.provisioner.ProvisioningException @@ -10,6 +12,7 @@ import com.redhat.ci.hosts.TargetHost * Represents a job that provisions the resources it needs, and runs @param body on them. */ class Job { + private static final Logger LOG = Logger.getLogger(Job.name) protected static final String SANDBOX_DIR = 'sandbox' protected Script script @@ -48,7 +51,7 @@ class Job { void run() { Map subJobs = [:] for (targetHost in targetHosts) { - subJobs[targetHost.arch] = jobWrapper(targetHost) + subJobs[targetHost.name ?: targetHost.id] = jobWrapper(targetHost) } // Run each single host job in parallel on each specified host @@ -113,6 +116,7 @@ class Job { try { host = provision(targetHost) } catch (e) { + LOG.log(Level.SEVERE, "Exception: ${e.message}", e) script.echo("Exception: ${e.message}") runInDirectory(SANDBOX_DIR) { onFailure(e, host) diff --git a/src/com/redhat/ci/hosts/Host.groovy b/src/com/redhat/ci/hosts/Host.groovy index 5cca11d..0d2b85c 100644 --- a/src/com/redhat/ci/hosts/Host.groovy +++ b/src/com/redhat/ci/hosts/Host.groovy @@ -4,6 +4,9 @@ package com.redhat.ci.hosts * Host primitives. */ class Host { + // Name for the host + String name = null + // ID for the host String id = UUID.randomUUID() diff --git a/src/com/redhat/ci/hosts/ProvisionedHost.groovy b/src/com/redhat/ci/hosts/ProvisionedHost.groovy index 112c47e..c6cd555 100644 --- a/src/com/redhat/ci/hosts/ProvisionedHost.groovy +++ b/src/com/redhat/ci/hosts/ProvisionedHost.groovy @@ -39,6 +39,7 @@ class ProvisionedHost extends TargetHost { ProvisionedHost(TargetHost target) { super() + this.name = target.name this.id = target.id this.arch = target.arch this.distro = target.distro @@ -50,6 +51,8 @@ class ProvisionedHost extends TargetHost { this.providerPriority = target.providerPriority this.provisioner = target.provisioner this.provisionerPriority = target.provisionerPriority + this.linchpinTargetEnabled = target.linchpinTargetEnabled + this.linchpinTarget = target.linchpinTarget this.bkrHostRequires = target.bkrHostRequires this.bkrJobGroup = target.bkrJobGroup this.bkrKsMeta = target.bkrKsMeta diff --git a/src/com/redhat/ci/hosts/TargetHost.groovy b/src/com/redhat/ci/hosts/TargetHost.groovy index d6847ff..2930e5a 100644 --- a/src/com/redhat/ci/hosts/TargetHost.groovy +++ b/src/com/redhat/ci/hosts/TargetHost.groovy @@ -19,6 +19,12 @@ class TargetHost extends Host { // Provisioner type priority list List provisionerPriority = null + // Flag that allows linchpinTarget to be disabled for custom PinFiles + Boolean linchpinTargetEnabled = true + + // Target of the linchpin PinFile (if linchpinTargetEnabled flag is set to true) + String linchpinTarget = null + // Beaker hostrequires // Overrides ProvisioningConfig's hostrequires List bkrHostRequires = null diff --git a/src/com/redhat/ci/provisioners/LinchPinProvisioner.groovy b/src/com/redhat/ci/provisioners/LinchPinProvisioner.groovy index a550040..a353742 100644 --- a/src/com/redhat/ci/provisioners/LinchPinProvisioner.groovy +++ b/src/com/redhat/ci/provisioners/LinchPinProvisioner.groovy @@ -4,6 +4,8 @@ import static com.redhat.ci.host.Type.UNKNOWN import static com.redhat.ci.host.Type.VM import static com.redhat.ci.host.Type.BAREMETAL +import java.util.logging.Logger +import java.util.logging.Level import com.redhat.ci.Utils import com.redhat.ci.hosts.TargetHost import com.redhat.ci.hosts.ProvisionedHost @@ -18,9 +20,10 @@ import groovy.json.JsonOutput */ class LinchPinProvisioner extends AbstractProvisioner { + private static final Logger LOG = Logger.getLogger(LinchPinProvisioner.name) private static final String HYPERVISOR = 'hypervisor' - private static final Map LINCHPIN_TARGETS = [ + private static final Map DEFAULT_TARGETS = [ (com.redhat.ci.provider.Type.BEAKER):'beaker-slave', ] @@ -52,8 +55,10 @@ class LinchPinProvisioner extends AbstractProvisioner { host.provider = com.redhat.ci.provider.Type.BEAKER host.typePriority = filterSupportedHostTypes(host.typePriority) host.type = host.typePriority.size() == 1 ? host.typePriority[0] : UNKNOWN + host.arch = host.arch ?: 'x86_64' host.distro = host.distro ?: 'RHEL-ALT-7.5' host.variant = host.variant ?: 'Server' + host.linchpinTarget = host.linchpinTarget ?: DEFAULT_TARGETS[host.provider] // Install keys we can connect via JNLP or SSH Utils.installCredentials(script, config) @@ -80,7 +85,7 @@ class LinchPinProvisioner extends AbstractProvisioner { "linchpin --workspace ${workspaceDir} " + "--config ${workspaceDir}/linchpin.conf " + "--template-data \'${getTemplateData(host, config)}\' " + - "--verbose up ${LINCHPIN_TARGETS[host.provider]}" + "--verbose up ${host.linchpinTargetEnabled ? host.linchpinTarget : ''}" ) } catch (e) { host.error = e.message @@ -126,6 +131,7 @@ class LinchPinProvisioner extends AbstractProvisioner { Utils.installRhpkg(script, config, host) } } catch (e) { + LOG.log(Level.SEVERE, e.message, e) host.error = host.error ? host.error + ", ${e.message}" : e.message script.echo("Error provisioning from LinchPin: ${host.error}") } @@ -247,8 +253,7 @@ class LinchPinProvisioner extends AbstractProvisioner { private String getLinchpinInventoryPath(Map linchpinLatest, ProvisionedHost host) { Map linchpinTargets = linchpinLatest["${host.linchpinTxId}"]['targets'][0] - String linchpinTarget = LINCHPIN_TARGETS[host.provider] - linchpinTargets[linchpinTarget]['outputs']['inventory_path'][0] + linchpinTargets[host.linchpinTarget]['outputs']['inventory_path'][0] } private String getHostname(ProvisionedHost host) { diff --git a/test/TestUtilsTest.groovy b/test/TestUtilsTest.groovy index 4f9ef89..3583a7e 100644 --- a/test/TestUtilsTest.groovy +++ b/test/TestUtilsTest.groovy @@ -93,6 +93,36 @@ class TestUtilsTest extends PipelineTestScript { assertNoExceptions() } + @Test + void shouldRunTestOnMultiHosts() { + ProvisioningConfig config = TestUtils.getProvisioningConfig(this) + List targets = [ + TestUtils.newTargetHost( + provisioner:com.redhat.ci.provisioner.Type.LINCHPIN, + provider:com.redhat.ci.provider.Type.BEAKER, + ), + TestUtils.newTargetHost( + provisioner:com.redhat.ci.provisioner.Type.LINCHPIN, + provider:com.redhat.ci.provider.Type.BEAKER, + linchpinTargetEnabled:false + ), + TestUtils.newTargetHost( + provisioner:com.redhat.ci.provisioner.Type.LINCHPIN, + provider:com.redhat.ci.provider.Type.BEAKER, + linchpinTarget:'custom-target' + ), + ] + TestUtils.runTest( + this, + targets, + config, + body, + onFailure, + onComplete) + + assertNoExceptions() + } + @Test void shouldRunTestOnBareMetalHost() { ProvisioningConfig config = TestUtils.getProvisioningConfig(this) diff --git a/test/resources/linchpin.latest b/test/resources/linchpin.latest index dd740a8..2bf6a17 100644 --- a/test/resources/linchpin.latest +++ b/test/resources/linchpin.latest @@ -1 +1 @@ -{"4": {"action": "up", "targets": [{"beaker-slave": {"1": {"uhash": "178e46", "rc": 0}, "outputs": {"inventory_path": ["/home/jpoulin/Projects/scratch/postup-hook-test/inventories/beaker-slave-178e46.inventory"], "resources": {"os_keypair_res": [], "rax_server_res": [], "aws_ec2_res": [], "os_server_res": [], "ovirt_vms_res": [], "aws_ec2_key_res": [], "gcloud_gce_res": [], "aws_s3_res": [], "duffy_res": [], "os_sg_res": [], "dummy_res": [{"failed": false, "changed": true, "hosts": ["dummy-node-178e46-0.example.net"], "dummy_file": "/tmp/dummy.hosts"}], "beaker_res": [], "aws_cfn_res": [], "os_heat_res": [], "os_obj_res": [], "libvirt_res": [], "os_volume_res": []}}, "inputs": {"hooks_data": {"postup": [{"context": true, "type": "ansible", "name": "test", "actions": [{"extra_vars": {"test": "Hello World"}, "playbook": "site.yml"}]}]}, "layout_data": {"inventory_layout": {"hosts": [{"count": 1, "name": "beaker-slave", "host_groups": ["rhel7", "certificate_authority", "repositories", "jenkins_slave", "master_node"]}]}}, "topology_data": {"topology_name": "beaker-slave", "resource_groups": [{"resource_group_name": "dummy-slaves", "resource_definitions": [{"count": 1, "role": "dummy_node", "name": "dummy-node"}], "resource_group_type": "dummy"}]}}}}]}} +{"4": {"action": "up", "targets": [{"beaker-slave": {"1": {"uhash": "178e46", "rc": 0}, "outputs": {"inventory_path": ["/home/jpoulin/Projects/scratch/postup-hook-test/inventories/beaker-slave-178e46.inventory"], "resources": {"os_keypair_res": [], "rax_server_res": [], "aws_ec2_res": [], "os_server_res": [], "ovirt_vms_res": [], "aws_ec2_key_res": [], "gcloud_gce_res": [], "aws_s3_res": [], "duffy_res": [], "os_sg_res": [], "dummy_res": [{"failed": false, "changed": true, "hosts": ["dummy-node-178e46-0.example.net"], "dummy_file": "/tmp/dummy.hosts"}], "beaker_res": [], "aws_cfn_res": [], "os_heat_res": [], "os_obj_res": [], "libvirt_res": [], "os_volume_res": []}}, "inputs": {"hooks_data": {"postup": [{"context": true, "type": "ansible", "name": "test", "actions": [{"extra_vars": {"test": "Hello World"}, "playbook": "site.yml"}]}]}, "layout_data": {"inventory_layout": {"hosts": [{"count": 1, "name": "beaker-slave", "host_groups": ["rhel7", "certificate_authority", "repositories", "jenkins_slave", "master_node"]}]}}, "topology_data": {"topology_name": "beaker-slave", "resource_groups": [{"resource_group_name": "dummy-slaves", "resource_definitions": [{"count": 1, "role": "dummy_node", "name": "dummy-node"}], "resource_group_type": "dummy"}]}}}, "custom-target": {"1": {"uhash": "178e46", "rc": 0}, "outputs": {"inventory_path": ["/home/jpoulin/Projects/scratch/postup-hook-test/inventories/beaker-slave-178e46.inventory"], "resources": {"os_keypair_res": [], "rax_server_res": [], "aws_ec2_res": [], "os_server_res": [], "ovirt_vms_res": [], "aws_ec2_key_res": [], "gcloud_gce_res": [], "aws_s3_res": [], "duffy_res": [], "os_sg_res": [], "dummy_res": [{"failed": false, "changed": true, "hosts": ["dummy-node-178e46-0.example.net"], "dummy_file": "/tmp/dummy.hosts"}], "beaker_res": [], "aws_cfn_res": [], "os_heat_res": [], "os_obj_res": [], "libvirt_res": [], "os_volume_res": []}}, "inputs": {"hooks_data": {"postup": [{"context": true, "type": "ansible", "name": "test", "actions": [{"extra_vars": {"test": "Hello World"}, "playbook": "site.yml"}]}]}, "layout_data": {"inventory_layout": {"hosts": [{"count": 1, "name": "beaker-slave", "host_groups": ["rhel7", "certificate_authority", "repositories", "jenkins_slave", "master_node"]}]}}, "topology_data": {"topology_name": "beaker-slave", "resource_groups": [{"resource_group_name": "dummy-slaves", "resource_definitions": [{"count": 1, "role": "dummy_node", "name": "dummy-node"}], "resource_group_type": "dummy"}]}}}}]}} diff --git a/vars/TestUtils.groovy b/vars/TestUtils.groovy index 093d1a6..5f8f047 100644 --- a/vars/TestUtils.groovy +++ b/vars/TestUtils.groovy @@ -44,8 +44,7 @@ class TestUtils { Closure test, Closure onFailure, Closure postRun = { }) { - TargetHost target = new TargetHost() - target.arch = arch + TargetHost target = new TargetHost(arch:arch, name:arch) runTest( script, target, @@ -77,7 +76,7 @@ class TestUtils { Closure postRun = { }) { List targets = [] for (arch in arches) { - targets.push(new TargetHost(arch:arch)) + targets.push(new TargetHost(arch:arch, name:arch)) } runTest( script, @@ -108,6 +107,7 @@ class TestUtils { Closure test, Closure onFailure, Closure postRun = { }) { + target.name ?: target.arch runTest( script, [ target ], From efb3cbcefa8676a26391d68578fad760418cb1c1 Mon Sep 17 00:00:00 2001 From: Jeremy Poulin Date: Wed, 8 May 2019 11:01:41 -0400 Subject: [PATCH 2/8] Replaced logging with echo step and logging stacktraces. --- src/com/redhat/ci/Job.groovy | 11 ++++------- .../redhat/ci/provisioners/LinchPinProvisioner.groovy | 5 +---- 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/src/com/redhat/ci/Job.groovy b/src/com/redhat/ci/Job.groovy index eb6ca22..cd1c648 100644 --- a/src/com/redhat/ci/Job.groovy +++ b/src/com/redhat/ci/Job.groovy @@ -1,7 +1,5 @@ package com.redhat.ci -import java.util.logging.Logger -import java.util.logging.Level import com.redhat.ci.provisioner.ProvisioningConfig import com.redhat.ci.provisioner.ProvisioningService import com.redhat.ci.provisioner.ProvisioningException @@ -12,7 +10,6 @@ import com.redhat.ci.hosts.TargetHost * Represents a job that provisions the resources it needs, and runs @param body on them. */ class Job { - private static final Logger LOG = Logger.getLogger(Job.name) protected static final String SANDBOX_DIR = 'sandbox' protected Script script @@ -116,8 +113,8 @@ class Job { try { host = provision(targetHost) } catch (e) { - LOG.log(Level.SEVERE, "Exception: ${e.message}", e) - script.echo("Exception: ${e.message}") + script.echo("Exception: ${e}") + script.echo("Stacktrace: $e.stackTrace") runInDirectory(SANDBOX_DIR) { onFailure(e, host) } @@ -132,7 +129,7 @@ class Job { body(host, config) } } catch (e) { - script.echo("Exception: ${e.message}") + script.echo("Exception: ${e}") runInDirectory(SANDBOX_DIR) { onFailure(e, host) } @@ -148,7 +145,7 @@ class Job { body(host, config) } } catch (e) { - script.echo("Exception: ${e.message}") + script.echo("Exception: ${e}") runInDirectory(SANDBOX_DIR) { onFailure(e, host) } diff --git a/src/com/redhat/ci/provisioners/LinchPinProvisioner.groovy b/src/com/redhat/ci/provisioners/LinchPinProvisioner.groovy index a353742..faf365b 100644 --- a/src/com/redhat/ci/provisioners/LinchPinProvisioner.groovy +++ b/src/com/redhat/ci/provisioners/LinchPinProvisioner.groovy @@ -4,8 +4,6 @@ import static com.redhat.ci.host.Type.UNKNOWN import static com.redhat.ci.host.Type.VM import static com.redhat.ci.host.Type.BAREMETAL -import java.util.logging.Logger -import java.util.logging.Level import com.redhat.ci.Utils import com.redhat.ci.hosts.TargetHost import com.redhat.ci.hosts.ProvisionedHost @@ -20,7 +18,6 @@ import groovy.json.JsonOutput */ class LinchPinProvisioner extends AbstractProvisioner { - private static final Logger LOG = Logger.getLogger(LinchPinProvisioner.name) private static final String HYPERVISOR = 'hypervisor' private static final Map DEFAULT_TARGETS = [ @@ -131,9 +128,9 @@ class LinchPinProvisioner extends AbstractProvisioner { Utils.installRhpkg(script, config, host) } } catch (e) { - LOG.log(Level.SEVERE, e.message, e) host.error = host.error ? host.error + ", ${e.message}" : e.message script.echo("Error provisioning from LinchPin: ${host.error}") + script.echo("Stacktrace: $e.stackTrace") } // An error occured, so we should ensure resources are cleaned up From 5b5578539f42ab1522fab82d6159242f88f7d698 Mon Sep 17 00:00:00 2001 From: Jeremy Poulin Date: Thu, 16 May 2019 21:17:10 -0400 Subject: [PATCH 3/8] Run tests should now use ansible 2.8 execution env --- vars/runTests.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vars/runTests.groovy b/vars/runTests.groovy index 5329501..5937c61 100644 --- a/vars/runTests.groovy +++ b/vars/runTests.groovy @@ -3,7 +3,7 @@ import com.redhat.ci.hosts.ProvisionedHost import com.redhat.ci.provisioner.Mode void call(ProvisioningConfig config, ProvisionedHost host) { - final String ACTIVATE_PROVISIONER = '. /home/jenkins/envs/provisioner/bin/activate;' + final String ACTIVATE_PROVISIONER = '. /home/jenkins/envs/ansible/bin/activate;' List exceptions = [] // JNLP Mode From c479e065ad47486e817268d9632faaf885662ac2 Mon Sep 17 00:00:00 2001 From: Jeremy Poulin Date: Wed, 22 May 2019 11:08:57 -0400 Subject: [PATCH 4/8] Added a teardown toggle to preserve build --- src/com/redhat/ci/provisioner/ProvisioningConfig.groovy | 3 +++ src/com/redhat/ci/provisioners/LinchPinProvisioner.groovy | 2 +- src/com/redhat/ci/provisioners/NoOpProvisioner.groovy | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/com/redhat/ci/provisioner/ProvisioningConfig.groovy b/src/com/redhat/ci/provisioner/ProvisioningConfig.groovy index 2c31ed2..8533eb3 100644 --- a/src/com/redhat/ci/provisioner/ProvisioningConfig.groovy +++ b/src/com/redhat/ci/provisioner/ProvisioningConfig.groovy @@ -109,6 +109,9 @@ class ProvisioningConfig { // This is only needed for tests that will use it to install from pkgs.devel.redhat.com Boolean installRhpkg = false + // Whether the job should teardown resources when complete + Boolean teardown = true + @SuppressWarnings('AbcMetric') ProvisioningConfig(Map params = [:], Map env = [:]) { params = params ?: [:] diff --git a/src/com/redhat/ci/provisioners/LinchPinProvisioner.groovy b/src/com/redhat/ci/provisioners/LinchPinProvisioner.groovy index faf365b..46311d4 100644 --- a/src/com/redhat/ci/provisioners/LinchPinProvisioner.groovy +++ b/src/com/redhat/ci/provisioners/LinchPinProvisioner.groovy @@ -148,7 +148,7 @@ class LinchPinProvisioner extends AbstractProvisioner { */ void teardown(ProvisionedHost host, ProvisioningConfig config) { // Check if the host was even created - if (!host) { + if (!host || !config.teardown) { script.echo(TEARDOWN_NOOP) return } diff --git a/src/com/redhat/ci/provisioners/NoOpProvisioner.groovy b/src/com/redhat/ci/provisioners/NoOpProvisioner.groovy index 483bb17..fb9628d 100644 --- a/src/com/redhat/ci/provisioners/NoOpProvisioner.groovy +++ b/src/com/redhat/ci/provisioners/NoOpProvisioner.groovy @@ -117,7 +117,7 @@ class NoOpProvisioner extends AbstractProvisioner { */ void teardown(ProvisionedHost host, ProvisioningConfig config) { // Check if the host was even created - if (!host) { + if (!host || !config.teardown) { return } From 4d5f7e5e024d83ed1834b0b250a384794649dc2e Mon Sep 17 00:00:00 2001 From: Jeremy Poulin Date: Wed, 29 May 2019 15:42:09 -0400 Subject: [PATCH 5/8] Support for additional beaker parameters (kernel options, host reqs, and keyvalue) --- .../redhat/ci/hosts/ProvisionedHost.groovy | 3 +++ src/com/redhat/ci/hosts/TargetHost.groovy | 9 ++++++++ .../provisioners/LinchPinProvisioner.groovy | 3 +++ workspace/PinFile | 22 ++++++++++++++++++- 4 files changed, 36 insertions(+), 1 deletion(-) diff --git a/src/com/redhat/ci/hosts/ProvisionedHost.groovy b/src/com/redhat/ci/hosts/ProvisionedHost.groovy index c6cd555..0558fd5 100644 --- a/src/com/redhat/ci/hosts/ProvisionedHost.groovy +++ b/src/com/redhat/ci/hosts/ProvisionedHost.groovy @@ -54,8 +54,11 @@ class ProvisionedHost extends TargetHost { this.linchpinTargetEnabled = target.linchpinTargetEnabled this.linchpinTarget = target.linchpinTarget this.bkrHostRequires = target.bkrHostRequires + this.bkrKeyValue = target.bkrKeyValue this.bkrJobGroup = target.bkrJobGroup this.bkrKsMeta = target.bkrKsMeta + this.bkrKernelOptions = target.bkrKernelOptions + this.bkrKernelOptionsPost = target.bkrKernelOptionsPost this.bkrMethod = target.bkrMethod this.reserveDuration = target.reserveDuration this.scriptParams = target.scriptParams diff --git a/src/com/redhat/ci/hosts/TargetHost.groovy b/src/com/redhat/ci/hosts/TargetHost.groovy index 2930e5a..34b6758 100644 --- a/src/com/redhat/ci/hosts/TargetHost.groovy +++ b/src/com/redhat/ci/hosts/TargetHost.groovy @@ -29,6 +29,9 @@ class TargetHost extends Host { // Overrides ProvisioningConfig's hostrequires List bkrHostRequires = null + // Beaker keyvalue + List bkrKeyValue = null + // Beaker jobgroup // Overrides ProvisioningConfig's jobgroup String bkrJobGroup = null @@ -36,6 +39,12 @@ class TargetHost extends Host { // Beaker ks_meta String bkrKsMeta = null + // Beaker kernel_options + String bkrKernelOptions = null + + // Beaker kernel_options_post + String bkrKernelOptionsPost = null + // Beaker installation method String bkrMethod = null diff --git a/src/com/redhat/ci/provisioners/LinchPinProvisioner.groovy b/src/com/redhat/ci/provisioners/LinchPinProvisioner.groovy index 46311d4..952baf2 100644 --- a/src/com/redhat/ci/provisioners/LinchPinProvisioner.groovy +++ b/src/com/redhat/ci/provisioners/LinchPinProvisioner.groovy @@ -196,10 +196,13 @@ class LinchPinProvisioner extends AbstractProvisioner { distro:host.distro, variant:host.variant, ks_meta:host.bkrKsMeta, + kernel_options:host.bkrKernelOptions, + kernel_options_post:host.bkrKernelOptionsPost, method:host.bkrMethod, reserve_duration:host.reserveDuration, job_group:host.bkrJobGroup ?: config.jobgroup, hostrequires:getHostRequires(host, config), + keyvalue:host.bkrKeyValue, inventory_vars:host.inventoryVars, ] diff --git a/workspace/PinFile b/workspace/PinFile index 3f24387..dad4b77 100644 --- a/workspace/PinFile +++ b/workspace/PinFile @@ -21,6 +21,12 @@ beaker-slave: {% if ks_meta %} ks_meta: {{ ks_meta | default('') }} {% endif %} + {% if kernel_options %} + kernel_options: {{ kernel_options | default('') }} + {% endif %} + {% if kernel_options_post %} + kernel_options_post: {{ kernel_options_post | default('') }} + {% endif %} {% if method %} method: {{ method | default('nfs') }} {% endif %} @@ -32,9 +38,23 @@ beaker-slave: {% if hostrequires %} hostrequires: {% for req in hostrequires %} + {% if req.tag %} - tag: "{{ req.tag }}" op: "{{ req.op }}" value: "{{ req.value }}" + {% if req.type %} + type: "{{ req.type }}" + {% endif %} + {% endif %} + {% if req.rawxml %} + - rawxml: '{{ req.rawxml }}' + {% endif %} + {% endfor %} + {% endif %} + {% if keyvalue %} + keyvalue: + {% for kv in keyvalue %} + - "{{ kv }}" {% endfor %} {% endif %} layout: @@ -42,7 +62,7 @@ beaker-slave: {% if inventory_vars %} vars: {% for key, value in inventory_vars.items() %} - {{ key }}: "{{ value }}" + {{ key }}: '{{ value }}' {% endfor %} {% endif %} hosts: From c969d648c20852978e7db8e02db3eb5b10a655a5 Mon Sep 17 00:00:00 2001 From: Jeremy Poulin Date: Fri, 31 May 2019 13:06:23 -0400 Subject: [PATCH 6/8] Using generic install to ensure ansible version is installed to target host in SSH mode --- vars/installBrewPkgs.groovy | 33 ++++++++++++++++++++++----------- 1 file changed, 22 insertions(+), 11 deletions(-) diff --git a/vars/installBrewPkgs.groovy b/vars/installBrewPkgs.groovy index 389fda0..051cf2b 100644 --- a/vars/installBrewPkgs.groovy +++ b/vars/installBrewPkgs.groovy @@ -1,5 +1,8 @@ -void call(Map params, Boolean privileged = false) { - String sudo = privileged ? 'sudo' : '' +import com.redhat.ci.Utils +import com.redhat.ci.hosts.ProvisionedHost +import com.redhat.ci.provisioner.ProvisioningConfig + +void call(Map params, Boolean privileged = false, ProvisioningConfig config = null, ProvisionedHost host = null) { Boolean taskRepoCreated = false if (params.CI_MESSAGE != '') { tid = getTaskId(params.CI_MESSAGE) @@ -11,14 +14,22 @@ void call(Map params, Boolean privileged = false) { } if (taskRepoCreated == true) { - sh """ - ${sudo} yum install -y yum-utils - URL=\$(cat task-repo.properties | grep TASK_REPO_URLS= | sed 's/TASK_REPO_URLS=//' | sed 's/;/\\n/g') - ${sudo} yum-config-manager --add-repo \${URL} - ${sudo} cat /etc/yum.repos.d/*download.eng.bos.redhat.com* - ${sudo} sed -i 's/gpgcheck=1/gpgcheck=0/g' /etc/yum.repos.d/*download.eng.bos.redhat.com* - echo "gpgcheck=0" | ${sudo} tee -a /etc/yum.repos.d/*download.eng.bos.redhat.com* - ${sudo} yum clean all - """ + Utils.genericInstall(this, config ?: new ProvisioningConfig(), host) { + privilegedOverride, sh -> + context = [ + env:[HOME:env.HOME], + files:['task-repo.properties'], + ] + String sudo = (privileged || privilegedOverride) ? 'sudo' : '' + sh(""" + ${sudo} yum install -y yum-utils + URL=\$(cat task-repo.properties | grep TASK_REPO_URLS= | sed 's/TASK_REPO_URLS=//' | sed 's/;/\\n/g') + ${sudo} yum-config-manager --add-repo \${URL} + ${sudo} cat /etc/yum.repos.d/*download.eng.bos.redhat.com* + ${sudo} sed -i 's/gpgcheck=1/gpgcheck=0/g' /etc/yum.repos.d/*download.eng.bos.redhat.com* + echo "gpgcheck=0" | ${sudo} tee -a /etc/yum.repos.d/*download.eng.bos.redhat.com* + ${sudo} yum clean all + """, context) + } } } From 88a11f1a8748961f85d8b4f380d9de0e7e985297 Mon Sep 17 00:00:00 2001 From: Jeremy Poulin Date: Wed, 12 Jun 2019 14:19:59 -0400 Subject: [PATCH 7/8] Runlog should now live in arch dir, logging inventory, and fixed script_params. --- resources/playbooks/run_scripts.yml | 15 ++++++++++++--- .../ci/provisioners/LinchPinProvisioner.groovy | 1 + vars/runTests.groovy | 5 +++-- 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/resources/playbooks/run_scripts.yml b/resources/playbooks/run_scripts.yml index 2c045ea..be8f9a3 100644 --- a/resources/playbooks/run_scripts.yml +++ b/resources/playbooks/run_scripts.yml @@ -1,6 +1,6 @@ --- - name: "Run Scripts on each Inventory Host" - hosts: all + hosts: master_node gather_facts: true vars_prompt: - name: test_dir @@ -8,6 +8,7 @@ tasks: - name: install rsync and the utilities to copy the tests to the remote host + become: yes package: name: "{{ item }}" state: latest @@ -41,14 +42,22 @@ - name: loop through script directories and create artifacts directory file: - path: "{{ script_dir }}/artifacts" + path: "{{ script_dir }}/artifacts/{{ ansible_architecture }}" state: directory loop: "{{ scripts_dir.files | map(attribute='path') | list }}" loop_control: loop_var: script_dir + - name: loop through script directories and create inventory + copy: + src: "{{ inventory }}" + dest: "{{ script_dir }}/inventory" + loop: "{{ scripts_dir.files | map(attribute='path') | list }}" + loop_control: + loop_var: script_dir + - name: loop through script directories, run test, and store artifacts - shell: "bash -x test.sh {{ script_args | default('') }} &> artifacts/{{ ansible_architecture }}-runlog.txt" + shell: "bash -x test.sh {{ script_params | default('') }} &> artifacts/{{ ansible_architecture }}/{{ ansible_architecture }}-runlog.txt" args: chdir: "{{ script_dir }}" loop: "{{ scripts_dir.files | map(attribute='path') | list }}" diff --git a/src/com/redhat/ci/provisioners/LinchPinProvisioner.groovy b/src/com/redhat/ci/provisioners/LinchPinProvisioner.groovy index 952baf2..9c2a289 100644 --- a/src/com/redhat/ci/provisioners/LinchPinProvisioner.groovy +++ b/src/com/redhat/ci/provisioners/LinchPinProvisioner.groovy @@ -97,6 +97,7 @@ class LinchPinProvisioner extends AbstractProvisioner { host.inventoryPath = getLinchpinInventoryPath(linchpinLatest, host) script.echo("inventoryPath:${host.inventoryPath}") + script.sh("cat ${host.inventoryPath}") host.hostname = getHostname(host) script.echo("hostname:${host.hostname}") diff --git a/vars/runTests.groovy b/vars/runTests.groovy index 5937c61..aaf8108 100644 --- a/vars/runTests.groovy +++ b/vars/runTests.groovy @@ -43,8 +43,9 @@ void call(ProvisioningConfig config, ProvisionedHost host) { writeFile(file:runScriptsPlaybook, text:runScripts) sh(""" ${ACTIVATE_PROVISIONER} - ansible-playbook -i '${host.inventoryPath}' --key-file "~/.ssh/id_rsa" \ - -e '{"test_dir":"${params.TEST_DIR}", "script_params":"${host.scriptParams ?: ''}"}' \ + ansible-playbook -i '${host.inventoryPath}' --limit master_node --key-file "~/.ssh/id_rsa" \ + -e '{"test_dir":"${params.TEST_DIR}", "inventory":"${host.inventoryPath}", \ + "script_params":"${host.scriptParams ?: ''}"}' \ ${runScriptsPlaybook} """) } catch (e) { From 4e80ecd9693431b0f235e32dc50549c332cae859 Mon Sep 17 00:00:00 2001 From: Jeremy Poulin Date: Thu, 16 May 2019 15:59:20 -0400 Subject: [PATCH 8/8] Bumped the provisioner version and updated format --- src/com/redhat/ci/provisioner/ProvisioningConfig.groovy | 2 +- vars/TestUtils.groovy | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/com/redhat/ci/provisioner/ProvisioningConfig.groovy b/src/com/redhat/ci/provisioner/ProvisioningConfig.groovy index 8533eb3..7ae897b 100644 --- a/src/com/redhat/ci/provisioner/ProvisioningConfig.groovy +++ b/src/com/redhat/ci/provisioner/ProvisioningConfig.groovy @@ -4,7 +4,7 @@ package com.redhat.ci.provisioner * Configuration needed to provision resources with a Provisioner. */ class ProvisioningConfig { - private static final String RELEASE_VERSION = 'v1.2.2' + private static final String RELEASE_VERSION = 'v1.3.0' private static final String KRB_PRINCIPAL_CREDENTIAL_ID_DEFAULT = 'redhat-multiarch-qe-krbprincipal' private static final String KEYTAB_CREDENTIAL_ID_DEFAULT = 'redhat-multiarch-qe-keytab' private static final String SSH_PRIV_KEY_CREDENTIAL_ID_DEFAULT = 'redhat-multiarch-qe-sshprivkey' diff --git a/vars/TestUtils.groovy b/vars/TestUtils.groovy index 5f8f047..534265c 100644 --- a/vars/TestUtils.groovy +++ b/vars/TestUtils.groovy @@ -168,7 +168,7 @@ class TestUtils { // This adds the custom provisioner slave container to the pod. Must be first with name 'jnlp' script.containerTemplate( name:'jnlp', - image:"${config.dockerUrl}/${config.tenant}/${config.provisioningImage}-${config.version}", + image:"${config.dockerUrl}/${config.tenant}/${config.provisioningImage}:${config.version}", ttyEnabled:false, args:'${computer.jnlpmac} ${computer.name}', command:'',