Add nodes pre-provisioning

src/main/java/org/jenkinsci/plugins/vSphereCloud.java

@@ -23,6 +23,8 @@
 import org.jenkinsci.plugins.folder.FolderVSphereCloudProperty;
 import org.jenkinsci.plugins.vsphere.VSphereConnectionConfig;
 import org.jenkinsci.plugins.vsphere.tools.*;
+import org.kohsuke.accmod.Restricted;
+import org.kohsuke.accmod.restrictions.NoExternalUse;
 import org.kohsuke.stapler.DataBoundConstructor;
 import org.kohsuke.stapler.QueryParameter;
 import org.kohsuke.stapler.Stapler;
@@ -215,7 +217,8 @@ public List<? extends vSphereCloudSlaveTemplate> getTemplates() {
         return this.templates;
     }
 
-    private vSphereCloudSlaveTemplate getTemplateForVM(final String vmName) {
+    @Restricted(NoExternalUse.class)
+    public vSphereCloudSlaveTemplate getTemplateForVM(final String vmName) {
         if (this.templates == null || vmName == null)
             return null;
         for (final vSphereCloudSlaveTemplate t : this.templates) {

src/main/java/org/jenkinsci/plugins/vSphereCloudSlave.java

@@ -133,6 +133,12 @@ public Integer getLimitedTestRunCount() {
         return LimitedTestRunCount;
     }
 
+    @Restricted(NoExternalUse.class)
+    public vSphereCloudSlaveTemplate getTemplate() {
+        vSphereCloud cloud = findOurVsInstance();
+        return cloud.getTemplateForVM(getVmName());
+    }
+
     public boolean isLaunchSupportForced() {
         return ((vSphereCloudLauncher) getLauncher()).getOverrideLaunchSupported() == Boolean.TRUE;
     }

src/main/java/org/jenkinsci/plugins/vSphereCloudSlaveComputer.java

@@ -2,8 +2,13 @@
 
 import java.io.PrintWriter;
 import java.io.StringWriter;
+import java.util.ArrayList;
+import java.util.List;
 
+import javax.annotation.Nonnull;
 import org.jenkinsci.plugins.vsphere.tools.VSphere;
+import org.kohsuke.accmod.Restricted;
+import org.kohsuke.accmod.restrictions.NoExternalUse;
 
 import com.vmware.vim25.VirtualHardware;
 import com.vmware.vim25.VirtualMachineConfigInfo;
@@ -14,8 +19,10 @@
 import com.vmware.vim25.mo.ManagedEntity;
 import com.vmware.vim25.mo.VirtualMachine;
 
+import hudson.model.Computer;
 import hudson.slaves.AbstractCloudComputer;
 import hudson.slaves.AbstractCloudSlave;
+import jenkins.model.Jenkins;
 
 public class vSphereCloudSlaveComputer extends AbstractCloudComputer {
     private final vSphereCloudSlave vSlave;
@@ -77,6 +84,19 @@ public String getVmInformationError() {
         return getVMInformation().errorEncounteredWhenDataWasRead;
     }
 
+    /**
+     * Get all vsphere computers.
+     */
+    @Restricted(NoExternalUse.class)
+    protected static @Nonnull List<vSphereCloudSlaveComputer> getAll() {
+        ArrayList<vSphereCloudSlaveComputer> out = new ArrayList<>();
+        for (final Computer c : Jenkins.get().getComputers()) {
+            if (!(c instanceof vSphereCloudSlaveComputer)) continue;
+            out.add((vSphereCloudSlaveComputer) c);
+        }
+        return out;
+    }
+
     /** 10 seconds */
     private static final long NANOSECONDS_TO_CACHE_VMINFORMATION = 10L * 1000000000L;
 

src/main/java/org/jenkinsci/plugins/vSphereCloudSlaveTemplate.java

@@ -49,7 +49,6 @@
 import java.util.Set;
 import java.util.logging.Level;
 import java.util.logging.Logger;
-import java.util.stream.Collectors;
 
 import javax.annotation.Nonnull;
 
@@ -114,6 +113,7 @@ public class vSphereCloudSlaveTemplate implements Describable<vSphereCloudSlaveT
     private final boolean saveFailure;
     private final String targetResourcePool;
     private final String targetHost;
+    private final int instancesMin;
     /**
      * Credentials from old configuration format. Credentials are now in the
      * {@link #launcher} configuration
@@ -152,6 +152,7 @@ public vSphereCloudSlaveTemplate(final String cloneNamePrefix,
                                      final boolean saveFailure,
                                      final String targetResourcePool,
                                      final String targetHost,
+                                     final int instancesMin,
                                      final String credentialsId /*deprecated*/,
                                      final ComputerLauncher launcher,
                                      final RetentionStrategy<?> retentionStrategy,
@@ -181,6 +182,7 @@ public vSphereCloudSlaveTemplate(final String cloneNamePrefix,
         this.saveFailure = saveFailure;
         this.targetResourcePool = targetResourcePool;
         this.targetHost = targetHost;
+        this.instancesMin = instancesMin;
         this.credentialsId = credentialsId;
         this.nodeProperties = Util.fixNull(nodeProperties);
         this.guestInfoProperties = Util.fixNull(guestInfoProperties);
@@ -272,6 +274,10 @@ public int getLimitedRunCount() {
         return this.limitedRunCount;
     }
 
+    public int getInstancesMin() {
+        return this.instancesMin;
+    }
+
     public boolean getSaveFailure() {
         return this.saveFailure;
     }
@@ -319,6 +325,37 @@ public RetentionStrategy<?> getRetentionStrategy() {
         return this.retentionStrategy;
     }
 
+    /**
+     * Return a list of running nodes provisioned using this template.
+     */
+    @Restricted(NoExternalUse.class)
+    public List<vSphereCloudSlaveComputer> getOnlineNodes() {
+        return getNodes(false);
+    }
+
+    /**
+     * Return a list of idle nodes provisioned using this template.
+     */
+    @Restricted(NoExternalUse.class)
+    public List<vSphereCloudSlaveComputer> getIdleNodes() {
+        return getNodes(true);
+    }
+
+    private List<vSphereCloudSlaveComputer> getNodes(boolean idle) {
+        List<vSphereCloudSlaveComputer> nodes = new ArrayList<>();
+        for (vSphereCloudSlaveComputer node : vSphereCloudSlaveComputer.getAll()) {
+            if (!node.isOnline()) continue;
+            if (idle && !node.isIdle()) continue;
+            String vmName = node.getName();
+            vSphereCloudSlaveTemplate nodeTemplate = getParent().getTemplateForVM(vmName);
+            // Filter out nodes from other clouds: nodeTemplate is null for these.
+            if (nodeTemplate == null) continue;
+            if (getLabelString() != nodeTemplate.getLabelString()) continue;
+            nodes.add(node);
+        }
+        return nodes;
+    }
+
     protected Object readResolve() {
         this.labelSet = Label.parse(labelString);
         if(this.templateInstanceCap == 0) {
@@ -550,6 +587,10 @@ public FormValidation doCheckLimitedRunCount(@QueryParameter String limitedRunCo
             return FormValidation.validateNonNegativeInteger(limitedRunCount);
         }
 
+        public FormValidation doCheckInstancesMin(@QueryParameter String instancesMin) {
+            return FormValidation.validateNonNegativeInteger(instancesMin);
+        }
+
         public FormValidation doCheckTemplateInstanceCap(@QueryParameter String templateInstanceCap) {
             return FormValidation.validateNonNegativeInteger(templateInstanceCap);
         }

src/main/java/org/jenkinsci/plugins/vsphere/VSphereNodeReconcileWork.java

@@ -0,0 +1,91 @@
+package org.jenkinsci.plugins.vsphere;
+
+import hudson.Extension;
+import hudson.Functions;
+import hudson.model.TaskListener;
+import hudson.model.AsyncPeriodicWork;
+import hudson.slaves.AbstractCloudSlave;
+import hudson.slaves.Cloud;
+import hudson.model.Label;
+import jenkins.model.Jenkins;
+import org.jenkinsci.plugins.vSphereCloud;
+import org.jenkinsci.plugins.vSphereCloudSlaveComputer;
+import org.jenkinsci.plugins.vSphereCloudSlaveTemplate;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import org.kohsuke.accmod.Restricted;
+import org.kohsuke.accmod.restrictions.NoExternalUse;
+
+/**
+ * A {@link AsyncPeriodicWork} that reconciles nodes to meet template values.
+ * <p>
+ * The async work will check the number of deployed nodes and provision (or
+ * delete) additional ones to meet template values. The check is happening every
+ * 2 minutes.
+ */
+@Extension
+@Restricted(NoExternalUse.class)
+public final class VSphereNodeReconcileWork extends AsyncPeriodicWork {
+    private static final Logger LOGGER = Logger.getLogger(VSphereNodeReconcileWork.class.getName());
+
+    public VSphereNodeReconcileWork() {
+        super("Vsphere nodes reconciliation");
+    }
+
+    @Override
+    public long getRecurrencePeriod() {
+        return Functions.getIsUnitTest() ? Long.MAX_VALUE : MIN * 2;
+    }
+
+    @Override
+    public void execute(TaskListener listener) {
+        for (Cloud cloud : Jenkins.getActiveInstance().clouds) {
+            if (!(cloud instanceof vSphereCloud)) continue;
+            for (vSphereCloudSlaveTemplate template : ((vSphereCloud) cloud).getTemplates()) {
+                String templateLabel = template.getLabelString();
+                Label label = Label.get(templateLabel);
+
+                int instancesMin = template.getInstancesMin();
+                List<vSphereCloudSlaveComputer> idleNodes = template.getIdleNodes();
+                List<vSphereCloudSlaveComputer> runningNodes = template.getOnlineNodes();
+                // Get max number of nodes that could be provisioned
+                int globalMaxNodes = ((vSphereCloud) cloud).getInstanceCap();
+                int templateMaxNodes = template.getTemplateInstanceCap();
+                int maxNodes = Math.min(globalMaxNodes, templateMaxNodes);
+
+                // if maxNumber is lower than instancesMin, we have to ignore instancesMin
+                int toProvision = Math.min(instancesMin - idleNodes.size(), maxNodes - runningNodes.size());
+                if (toProvision > 0) {
+                    // provision desired number of nodes for this label
+                    LOGGER.log(Level.INFO, "Pre-creating {0} instance(s) for template {1} in cloud {3}",
+                            new Object[] { toProvision, templateLabel, cloud.name });
+                    try {
+                        cloud.provision(label, toProvision);
+                    } catch (Throwable ex) {
+                        LOGGER.log(Level.SEVERE, "Failed to pre-create instance from template {0}. Exception: {1}",
+                                new Object[] { templateLabel, ex });
+                    }
+                } else if (toProvision < 0) {
+                    int toDelete = Math.min(idleNodes.size(), Math.abs(toProvision));
+                    for (int i = 0; i < toDelete; i++) {
+                        AbstractCloudSlave node = idleNodes.get(i).getNode();
+                        if (node == null) continue;
+                        LOGGER.log(Level.INFO, "Found excessive instance. Terminating {0} node {1}.",
+                                new Object[] { idleNodes.get(i).getName(), node });
+                        try {
+                            node.terminate();
+                        } catch (InterruptedException | IOException e) {
+                            LOGGER.log(Level.WARNING, e.getMessage());
+                            // try to delete it later
+                            continue;
+                        }
+                    }
+                }
+            }
+        }
+    }
+}

src/main/resources/org/jenkinsci/plugins/vSphereCloudSlaveTemplate/config.jelly

@@ -54,6 +54,10 @@
                 <f:textbox clazz="required number" default="0"/>
             </f:entry>
 
+            <f:entry title="${%Instances Min}" field="instancesMin">
+                <f:textbox clazz="required number" default="0"/>
+            </f:entry>
+
             <f:entry title="${%# of Executors}" field="numberOfExecutors">
                 <f:textbox clazz="required positive-number" default="1"/>
             </f:entry>

src/main/resources/org/jenkinsci/plugins/vSphereCloudSlaveTemplate/help-instancesMin.html

@@ -0,0 +1,12 @@
+<div>
+    The number of VMs to be provisioned beforehand.<br>
+    This allows to speed up CI runs by starting them immediately without waiting for a VM to get booted.<br>
+    <dt>If the number is set to 0:</dt>
+    <dd>No VMs provisioned in advance.</dd>
+    <dt>If the number is bigger than 0:</dt>
+    <dd>The plugin provisions new VMs to meet the value.</dd>
+    <dt>If instances Min is bigger than instance Cap:</dt>
+    <dd>The plugin provisions max number of VMs specified in instance Cap (the smallest of cloud and template options).</dd>
+    The plugin checks the number of running VMs once in 2 minutes.
+    <dt>The plugin respects retention policies.</dt>
+</div>

src/test/java/org/jenkinsci/plugins/vsphere/tools/CloudProvisioningAlgorithmTest.java

@@ -316,7 +316,7 @@ private CloudProvisioningRecord createInstance(int capacity, int provisioned, in
 
     private static vSphereCloudSlaveTemplate stubTemplate(String prefix, int templateInstanceCap) {
         return new vSphereCloudSlaveTemplate(prefix, "", null, null, false, null, null, null, null, null, null, templateInstanceCap, 1,
-                null, null, null, false, false, 0, 0, false, null, null, null, new JNLPLauncher(),
+                null, null, null, false, false, 0, 0, false, null, null, 0, null, new JNLPLauncher(),
                 RetentionStrategy.NOOP, null, null);
     }
 

src/test/java/org/jenkinsci/plugins/vsphere/tools/CloudProvisioningStateTest.java

@@ -489,7 +489,7 @@ private CloudProvisioningRecord createRecord(CloudProvisioningState instance) {
         final String cloneNamePrefix = "prefix" + recordNumber;
         final vSphereCloudSlaveTemplate template = new vSphereCloudSlaveTemplate(cloneNamePrefix, "masterImageName",
                 null, "snapshotName", false, "cluster", "resourcePool", "datastore", "folder", "customizationSpec", "templateDescription", 0, 1, "remoteFS",
-                "", Mode.NORMAL, false, false, 0, 0, false, "targetResourcePool", "targetHost", null,
+                "", Mode.NORMAL, false, false, 0, 0, false, "targetResourcePool", "targetHost", 0, null,
                 new JNLPLauncher(), RetentionStrategy.NOOP, Collections.<NodeProperty<?>> emptyList(),
                 Collections.<VSphereGuestInfoProperty> emptyList());
         stubVSphereCloudTemplates.add(template);

src/test/java/org/jenkinsci/plugins/vsphere/tools/ConfigurationAsCodeTest.java

@@ -61,6 +61,7 @@ public void should_support_configuration_as_code() {
         assertThat(template.getTemplateInstanceCap(), is(5));
         assertThat(template.getUseSnapshot(), is(true));
         assertThat(template.getWaitForVMTools(), is(true));
+        assertThat(template.getInstancesMin(), is(3));
         List<? extends VSphereGuestInfoProperty> guestInfoProperties = template.getGuestInfoProperties();
         assertThat(guestInfoProperties, hasSize(1));
         VSphereGuestInfoProperty guestInfoProperty = guestInfoProperties.get(0);

src/test/resources/org/jenkinsci/plugins/vsphere/tools/configuration-as-code.yml

@@ -16,6 +16,7 @@ jenkins:
               # ^ escapes the secret
               - name: "JENKINS_URL"
                 value: "^${JENKINS_URL}"
+            instancesMin: 3
             labelString: "windows vsphere"
             launchDelay: 60
             launcher:

src/test/resources/org/jenkinsci/plugins/vsphere/tools/expected_output.yml

@@ -10,6 +10,7 @@
       guestInfoProperties:
       - name: "JENKINS_URL"
         value: "^${JENKINS_URL}"
+      instancesMin: 3
       labelString: "windows vsphere"
       launchDelay: 60
       launcher: