diff --git a/src/main/java/org/jenkinsci/plugins/vSphereCloud.java b/src/main/java/org/jenkinsci/plugins/vSphereCloud.java index 19213ce6..182aea6a 100644 --- a/src/main/java/org/jenkinsci/plugins/vSphereCloud.java +++ b/src/main/java/org/jenkinsci/plugins/vSphereCloud.java @@ -352,8 +352,7 @@ public Collection provision(final Label label, int excessWorkload) final List plannedNodes = new ArrayList(); synchronized (templateState) { templateState.pruneUnwantedRecords(); - Integer maxSlavesToProvisionBeforeCloudCapHit = calculateMaxAdditionalSlavesPermitted(); - if (maxSlavesToProvisionBeforeCloudCapHit != null && maxSlavesToProvisionBeforeCloudCapHit <= 0) { + if (!cloudHasCapacity()) { return Collections.emptySet(); // no capacity due to cloud instance cap } final List templates = getTemplates(label); @@ -361,13 +360,7 @@ public Collection provision(final Label label, int excessWorkload) VSLOG.log(Level.INFO, methodCallDescription + ": " + numberOfvSphereCloudSlaves + " existing slaves (=" + numberOfvSphereCloudSlaveExecutors + " executors), templates available are " + whatWeCouldUse); while (excessWorkloadSoFar > 0) { - if (maxSlavesToProvisionBeforeCloudCapHit != null) { - final int intValue = maxSlavesToProvisionBeforeCloudCapHit.intValue(); - if (intValue <= 0) { - break; // out of capacity due to cloud instance cap - } - maxSlavesToProvisionBeforeCloudCapHit = Integer.valueOf(intValue - 1); - } + if (!cloudHasCapacity()) break; final CloudProvisioningRecord whatWeShouldSpinUp = CloudProvisioningAlgorithm.findTemplateWithMostFreeCapacity(whatWeCouldUse); if (whatWeShouldSpinUp == null) { break; // out of capacity due to template instance cap @@ -387,6 +380,47 @@ public Collection provision(final Label label, int excessWorkload) } } + /** + * Pre-provisions nodes per template to save time on a VM boot. + * + * @param template + */ + public void preProvisionNodes(vSphereCloudSlaveTemplate template) { + final String methodCallDescription = "preProvisionNodesForTemplate(" + template.getLabelString() + ")"; + try { + synchronized (this) { + ensureLists(); + } + retryVMdeletionIfNecessary(template.getInstancesMin()); + synchronized (templateState) { + templateState.pruneUnwantedRecords(); + final CloudProvisioningRecord provisionable = templateState.getOrCreateRecord(template); + int nodesToProvision = CloudProvisioningAlgorithm.shouldPreProvisionNodes(provisionable); + VSLOG.log(Level.INFO, methodCallDescription + ": should pre-provision " + nodesToProvision + " nodes"); + while (nodesToProvision > 0) { + if (!cloudHasCapacity()) break; + final String nodeName = CloudProvisioningAlgorithm.findUnusedName(provisionable); + VSpherePlannedNode.createInstance(templateState, nodeName, provisionable); + nodesToProvision -= 1; + } + } + } catch (Exception ex) { + VSLOG.log(Level.WARNING, methodCallDescription + ": Failed.", ex); + } + } + + /** + * Check if at least one additional node can be provisioned. + */ + private boolean cloudHasCapacity(){ + Integer maxSlavesToProvisionBeforeCloudCapHit = calculateMaxAdditionalSlavesPermitted(); + if (maxSlavesToProvisionBeforeCloudCapHit != null && maxSlavesToProvisionBeforeCloudCapHit <= 0) { + VSLOG.info("The cloud is at max capacity. Can not provison more nodes."); + return false; + } + return true; + } + /** * Has another go at deleting VMs we failed to delete earlier. It's possible * that we were unable to talk to vSphere (or some other failure happened) diff --git a/src/main/java/org/jenkinsci/plugins/vSphereCloudSlaveTemplate.java b/src/main/java/org/jenkinsci/plugins/vSphereCloudSlaveTemplate.java index 781987fc..6364240b 100644 --- a/src/main/java/org/jenkinsci/plugins/vSphereCloudSlaveTemplate.java +++ b/src/main/java/org/jenkinsci/plugins/vSphereCloudSlaveTemplate.java @@ -49,7 +49,6 @@ import java.util.Set; import java.util.logging.Level; import java.util.logging.Logger; -import java.util.stream.Collectors; import javax.annotation.Nonnull; @@ -114,6 +113,7 @@ public class vSphereCloudSlaveTemplate implements Describable retentionStrategy, @@ -181,6 +182,7 @@ public vSphereCloudSlaveTemplate(final String cloneNamePrefix, this.saveFailure = saveFailure; this.targetResourcePool = targetResourcePool; this.targetHost = targetHost; + this.instancesMin = instancesMin; this.credentialsId = credentialsId; this.nodeProperties = Util.fixNull(nodeProperties); this.guestInfoProperties = Util.fixNull(guestInfoProperties); @@ -272,6 +274,10 @@ public int getLimitedRunCount() { return this.limitedRunCount; } + public int getInstancesMin() { + return this.instancesMin; + } + public boolean getSaveFailure() { return this.saveFailure; } @@ -550,6 +556,10 @@ public FormValidation doCheckLimitedRunCount(@QueryParameter String limitedRunCo return FormValidation.validateNonNegativeInteger(limitedRunCount); } + public FormValidation doCheckInstancesMin(@QueryParameter String instancesMin) { + return FormValidation.validateNonNegativeInteger(instancesMin); + } + public FormValidation doCheckTemplateInstanceCap(@QueryParameter String templateInstanceCap) { return FormValidation.validateNonNegativeInteger(templateInstanceCap); } diff --git a/src/main/java/org/jenkinsci/plugins/vsphere/VSpherePreProvisonWork.java b/src/main/java/org/jenkinsci/plugins/vsphere/VSpherePreProvisonWork.java new file mode 100644 index 00000000..e1cac35c --- /dev/null +++ b/src/main/java/org/jenkinsci/plugins/vsphere/VSpherePreProvisonWork.java @@ -0,0 +1,53 @@ +package org.jenkinsci.plugins.vsphere; + +import hudson.Extension; +import hudson.Functions; +import hudson.model.TaskListener; +import hudson.model.AsyncPeriodicWork; +import hudson.slaves.Cloud; +import jenkins.model.Jenkins; +import org.jenkinsci.plugins.vSphereCloud; +import org.jenkinsci.plugins.vSphereCloudSlaveTemplate; + +import java.util.logging.Level; +import java.util.logging.Logger; + +import org.kohsuke.accmod.Restricted; +import org.kohsuke.accmod.restrictions.NoExternalUse; + +/** + * A {@link AsyncPeriodicWork} that pre-provisions nodes to meet insntanceMin template value. + *

+ * The async work will check the number of active nodes + * and provision additional ones to meet template values. + * + * The check is happening every 2 minutes. + */ +@Extension +@Restricted(NoExternalUse.class) +public final class VSpherePreProvisonWork extends AsyncPeriodicWork { + private static final Logger LOGGER = Logger.getLogger(VSpherePreProvisonWork.class.getName()); + + public VSpherePreProvisonWork() { + super("Vsphere pre-provision check"); + } + + @Override + public long getRecurrencePeriod() { + return Functions.getIsUnitTest() ? Long.MAX_VALUE : MIN * 2; + } + + @Override + public void execute(TaskListener listener) { + for (Cloud cloud : Jenkins.getActiveInstance().clouds) { + if (!(cloud instanceof vSphereCloud)) continue; + vSphereCloud vsCloud = (vSphereCloud) cloud; + for (vSphereCloudSlaveTemplate template : vsCloud.getTemplates()) { + if (template.getInstancesMin() > 0) { + LOGGER.log(Level.INFO, "Check if template (label=" + template.getLabelString() + ") has enough active nodes to meet instances Min value"); + vsCloud.preProvisionNodes(template); + } + } + } + } +} diff --git a/src/main/java/org/jenkinsci/plugins/vsphere/tools/CloudProvisioningAlgorithm.java b/src/main/java/org/jenkinsci/plugins/vsphere/tools/CloudProvisioningAlgorithm.java index 9c4c3098..aab53a8d 100644 --- a/src/main/java/org/jenkinsci/plugins/vsphere/tools/CloudProvisioningAlgorithm.java +++ b/src/main/java/org/jenkinsci/plugins/vsphere/tools/CloudProvisioningAlgorithm.java @@ -81,6 +81,23 @@ public static String findUnusedName(CloudProvisioningRecord record) { + ", even after " + maxAttempts + " attempts."); } + /** + * Compares sum of provisioned and planned nodes for the template. + * + * If the sum is less than instanceMin template value we should provision more nodes, + * otherwise the value is satisfied and we should not add any more nodes yet. + * + * @param record + * Our record regarding the template the agent will be created + * from. + * @return A number of nodes to be provisioned. + */ + public static int shouldPreProvisionNodes(CloudProvisioningRecord record) { + int provisionedNodes = record.getCurrentlyProvisioned().size() + record.getCurrentlyPlanned().size(); + int requiredPreProvisionedNodes = record.getTemplate().getInstancesMin(); + return requiredPreProvisionedNodes - provisionedNodes; + } + private static String calcSequentialSuffix(final int attempt) { final int slaveNumber = attempt + 1; final String suffix = Integer.toString(slaveNumber); diff --git a/src/main/resources/org/jenkinsci/plugins/vSphereCloudSlaveTemplate/config.jelly b/src/main/resources/org/jenkinsci/plugins/vSphereCloudSlaveTemplate/config.jelly index 261a2173..43a39069 100644 --- a/src/main/resources/org/jenkinsci/plugins/vSphereCloudSlaveTemplate/config.jelly +++ b/src/main/resources/org/jenkinsci/plugins/vSphereCloudSlaveTemplate/config.jelly @@ -54,6 +54,10 @@ + + + + diff --git a/src/main/resources/org/jenkinsci/plugins/vSphereCloudSlaveTemplate/help-instancesMin.html b/src/main/resources/org/jenkinsci/plugins/vSphereCloudSlaveTemplate/help-instancesMin.html new file mode 100644 index 00000000..971bca52 --- /dev/null +++ b/src/main/resources/org/jenkinsci/plugins/vSphereCloudSlaveTemplate/help-instancesMin.html @@ -0,0 +1,12 @@ +

+ The number of VMs to be provisioned beforehand.
+ This allows to speed up CI runs by starting them immediately without waiting for a VM to get booted.
+
If the number is set to 0:
+
No VMs provisioned in advance.
+
If the number is bigger than 0:
+
The plugin provisions new VMs to meet the value.
+
If instances Min is bigger than instance Cap:
+
The plugin provisions max number of VMs specified in instance Cap (the smallest of cloud and template options).
+ The plugin checks the number of running VMs once in 2 minutes. +
Pre-provisoned VMs will be deleted based on the retention policy.
+
diff --git a/src/test/java/org/jenkinsci/plugins/NodePreProvisionTest.java b/src/test/java/org/jenkinsci/plugins/NodePreProvisionTest.java new file mode 100644 index 00000000..25a3bbd0 --- /dev/null +++ b/src/test/java/org/jenkinsci/plugins/NodePreProvisionTest.java @@ -0,0 +1,109 @@ +package org.jenkinsci.plugins; + +import static org.junit.Assert.*; +import static org.hamcrest.CoreMatchers.*; + + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.TimeUnit; +import java.util.logging.Handler; +import java.util.logging.Level; +import java.util.logging.LogRecord; +import java.util.logging.Logger; + +import org.jenkinsci.plugins.vSphereCloud; +import org.jenkinsci.plugins.vSphereCloudSlaveTemplate; +import org.jenkinsci.plugins.vSphereCloud.VSpherePlannedNode; +import org.jenkinsci.plugins.vsphere.VSphereConnectionConfig; +import org.jenkinsci.plugins.vsphere.tools.CloudProvisioningAlgorithm; +import org.jenkinsci.plugins.vsphere.tools.CloudProvisioningRecord; +import org.jenkinsci.plugins.vsphere.tools.CloudProvisioningState; + +import hudson.model.Label; +import hudson.slaves.JNLPLauncher; +import hudson.slaves.RetentionStrategy; +import hudson.slaves.NodeProvisioner.PlannedNode; + +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +public class NodePreProvisionTest { + /** Used when faking up test data */ + private static List stubVSphereCloudTemplates; + private static VSphereConnectionConfig vsConnectionConfig; + private static vSphereCloud stubVSphereCloud; + private static CloudProvisioningState stubVSphereTemplateState; + private Logger testLogger; + private List loggedMessages; + + @BeforeClass + public static void setupClass() { + stubVSphereCloudTemplates = new ArrayList(); + vsConnectionConfig = new VSphereConnectionConfig("vsHost", false, "credentialsId"); + stubVSphereCloud = new vSphereCloud(vsConnectionConfig, "vsDescription", 100, 100, stubVSphereCloudTemplates); + stubVSphereTemplateState = new CloudProvisioningState(stubVSphereCloud); + } + + @Before + public void setup() { + stubVSphereCloudTemplates.clear(); + loggedMessages = new ArrayList(); + // Get vSphereCloud logger + Logger logger = Logger.getLogger("vsphere-cloud"); + logger.setLevel(Level.ALL); + // final Handler[] handlers = logger.getHandlers(); + // for (final Handler handler : handlers) { + // logger.removeHandler(handler); + // } + final Handler testHandler = new Handler() { + @Override + public void publish(LogRecord record) { + loggedMessages.add(record); + } + + @Override + public void flush() { + } + + @Override + public void close() { + } + }; + logger.addHandler(testHandler); + testLogger = logger; + } + + @Test + public void shouldPreProvisionNodesWhenNotEnough() { + vSphereCloudSlaveTemplate template = createTemplate(10, 2); + provisionNode(template); + assertThat(stubVSphereTemplateState.countNodes(), equalTo(1)); + + // Here it says that there still should be provisioned 2 nodes, despite the fact there is 1 active already + stubVSphereCloud.preProvisionNodes(template); + + // Below is a draft line + //assertThat(loggedMessages.get(1).getMessage(), equalTo("should pre-provision 1 node")); + + } + + private vSphereCloudSlaveTemplate createTemplate(int templateCapacity, int instanceMin){ + return stubTemplate("templateCapacity" + templateCapacity + "instanceMin" + instanceMin, templateCapacity, instanceMin); + } + + private void provisionNode(vSphereCloudSlaveTemplate template) { + CloudProvisioningRecord provisionable = stubVSphereTemplateState.getOrCreateRecord(template); + final String nodeName = CloudProvisioningAlgorithm.findUnusedName(provisionable); + stubVSphereTemplateState.provisionedSlaveNowActive(provisionable, nodeName); + // Below doesn't work either + //VSpherePlannedNode.createInstance(stubVSphereTemplateState, nodeName, provisionable); + } + + private static vSphereCloudSlaveTemplate stubTemplate(String prefix, int templateInstanceCap, int instanceMin) { + return new vSphereCloudSlaveTemplate(prefix, "", null, null, false, null, null, null, null, null, null, templateInstanceCap, 1, + null, null, null, false, false, 0, 0, false, null, null, instanceMin, null, new JNLPLauncher(), + RetentionStrategy.NOOP, null, null); + } +} diff --git a/src/test/java/org/jenkinsci/plugins/vsphere/tools/CloudProvisioningAlgorithmTest.java b/src/test/java/org/jenkinsci/plugins/vsphere/tools/CloudProvisioningAlgorithmTest.java index b48ec3c7..9dde4d90 100644 --- a/src/test/java/org/jenkinsci/plugins/vsphere/tools/CloudProvisioningAlgorithmTest.java +++ b/src/test/java/org/jenkinsci/plugins/vsphere/tools/CloudProvisioningAlgorithmTest.java @@ -299,6 +299,21 @@ public void findUnusedNameGivenUncappedInstancesThenReturnsUniqueNames() { assertThat(actuals, everyItem(startsWith(prefix))); } + @Test + public void shouldPreProvisionNodesWhenNotEnough() { + // we don't care about cap here + // Given + int provisioned = 3; + int planned = 2; + final CloudProvisioningRecord record = createInstance(10, provisioned, planned); + + // When + int instanceMin = record.getTemplate().getInstancesMin(); + + // Then + assertThat(CloudProvisioningAlgorithm.shouldPreProvisionNodes(record), equalTo(instanceMin - (provisioned + planned))); + } + private CloudProvisioningRecord createInstance(int capacity, int provisioned, int planned) { final int iNum = ++instanceNumber; final vSphereCloudSlaveTemplate template = stubTemplate(iNum + "cap" + capacity, capacity); @@ -316,7 +331,7 @@ private CloudProvisioningRecord createInstance(int capacity, int provisioned, in private static vSphereCloudSlaveTemplate stubTemplate(String prefix, int templateInstanceCap) { return new vSphereCloudSlaveTemplate(prefix, "", null, null, false, null, null, null, null, null, null, templateInstanceCap, 1, - null, null, null, false, false, 0, 0, false, null, null, null, new JNLPLauncher(), + null, null, null, false, false, 0, 0, false, null, null, 2, null, new JNLPLauncher(), RetentionStrategy.NOOP, null, null); } diff --git a/src/test/java/org/jenkinsci/plugins/vsphere/tools/CloudProvisioningStateTest.java b/src/test/java/org/jenkinsci/plugins/vsphere/tools/CloudProvisioningStateTest.java index 3c055e0a..0f1cfb47 100644 --- a/src/test/java/org/jenkinsci/plugins/vsphere/tools/CloudProvisioningStateTest.java +++ b/src/test/java/org/jenkinsci/plugins/vsphere/tools/CloudProvisioningStateTest.java @@ -489,7 +489,7 @@ private CloudProvisioningRecord createRecord(CloudProvisioningState instance) { final String cloneNamePrefix = "prefix" + recordNumber; final vSphereCloudSlaveTemplate template = new vSphereCloudSlaveTemplate(cloneNamePrefix, "masterImageName", null, "snapshotName", false, "cluster", "resourcePool", "datastore", "folder", "customizationSpec", "templateDescription", 0, 1, "remoteFS", - "", Mode.NORMAL, false, false, 0, 0, false, "targetResourcePool", "targetHost", null, + "", Mode.NORMAL, false, false, 0, 0, false, "targetResourcePool", "targetHost", 0, null, new JNLPLauncher(), RetentionStrategy.NOOP, Collections.> emptyList(), Collections. emptyList()); stubVSphereCloudTemplates.add(template); diff --git a/src/test/java/org/jenkinsci/plugins/vsphere/tools/ConfigurationAsCodeTest.java b/src/test/java/org/jenkinsci/plugins/vsphere/tools/ConfigurationAsCodeTest.java index 15bf503c..d2fa6e0b 100644 --- a/src/test/java/org/jenkinsci/plugins/vsphere/tools/ConfigurationAsCodeTest.java +++ b/src/test/java/org/jenkinsci/plugins/vsphere/tools/ConfigurationAsCodeTest.java @@ -61,6 +61,7 @@ public void should_support_configuration_as_code() { assertThat(template.getTemplateInstanceCap(), is(5)); assertThat(template.getUseSnapshot(), is(true)); assertThat(template.getWaitForVMTools(), is(true)); + assertThat(template.getInstancesMin(), is(3)); List guestInfoProperties = template.getGuestInfoProperties(); assertThat(guestInfoProperties, hasSize(1)); VSphereGuestInfoProperty guestInfoProperty = guestInfoProperties.get(0); diff --git a/src/test/resources/org/jenkinsci/plugins/vsphere/tools/configuration-as-code.yml b/src/test/resources/org/jenkinsci/plugins/vsphere/tools/configuration-as-code.yml index 1e2ab34d..7a4a9bd6 100644 --- a/src/test/resources/org/jenkinsci/plugins/vsphere/tools/configuration-as-code.yml +++ b/src/test/resources/org/jenkinsci/plugins/vsphere/tools/configuration-as-code.yml @@ -16,6 +16,7 @@ jenkins: # ^ escapes the secret - name: "JENKINS_URL" value: "^${JENKINS_URL}" + instancesMin: 3 labelString: "windows vsphere" launchDelay: 60 launcher: diff --git a/src/test/resources/org/jenkinsci/plugins/vsphere/tools/expected_output.yml b/src/test/resources/org/jenkinsci/plugins/vsphere/tools/expected_output.yml index 25974603..90766846 100644 --- a/src/test/resources/org/jenkinsci/plugins/vsphere/tools/expected_output.yml +++ b/src/test/resources/org/jenkinsci/plugins/vsphere/tools/expected_output.yml @@ -10,6 +10,7 @@ guestInfoProperties: - name: "JENKINS_URL" value: "^${JENKINS_URL}" + instancesMin: 3 labelString: "windows vsphere" launchDelay: 60 launcher: