Initial native checkout scm support (#380)

* Preliminary attempt at native `checkout scm` support Signed-off-by: Andrew Bayer <[email protected]> * Add test for `checkout scm` support Signed-off-by: Andrew Bayer <[email protected]> * Move to using a vanilla-package smoke test for `checkout scm` Signed-off-by: Andrew Bayer <[email protected]> * Switch from XML for creds/scm config to YAML Signed-off-by: Andrew Bayer <[email protected]> * Add credentials to test Signed-off-by: Andrew Bayer <[email protected]> * Switch to optional credentials in SCM YAML rather than standalone. Signed-off-by: Andrew Bayer <[email protected]> * Adding docs for `checkout scm` support. Signed-off-by: Andrew Bayer <[email protected]> * Fix exception handling, add link to SCM.adoc from README, document lack of Pipeline as YAML support Signed-off-by: Andrew Bayer <[email protected]>
jenkinsci · Oct 21, 2020 · eaa2af4 · eaa2af4
1 parent 38f6d0c
commit eaa2af4
Show file tree

Hide file tree

Showing 11 changed files with 310 additions and 19 deletions.
diff --git a/README.adoc b/README.adoc
@@ -160,6 +160,12 @@ The executable of Jenkinsfile Runner allows its invocation with these CLI option
  -c (--cause) VAL            : A string describing the cause of the run.
                                It will be attached to the build so that it appears in the
                                build log and becomes available to plug-ins and pipeline steps.
+--scm FILE                   : A YAML file defining the SCM and optional credentials to use
+                               with the SCM. If given, the SCM will be checked out into the
+                               workspace automatically in Declarative Pipelines, and will be
+                               available for use with `checkout scm` in Scripted Pipelines.
+                               Note that an SCM cannot currently be used with YAML pipelines.
+                               See link:./docs/using/SCM.adoc[this doc for more details].
  -jv (--jenkins-version) VAL : jenkins version to use (only in case 'warDir' is not
                                specified). Defaults to latest LTS.
  -w (--jenkins-war) FILE     : path to exploded jenkins war directory.

diff --git a/bootstrap/src/main/java/io/jenkins/jenkinsfile/runner/bootstrap/Bootstrap.java b/bootstrap/src/main/java/io/jenkins/jenkinsfile/runner/bootstrap/Bootstrap.java
@@ -1,13 +1,13 @@
 package io.jenkins.jenkinsfile.runner.bootstrap;
 
+import edu.umd.cs.findbugs.annotations.CheckForNull;
 import hudson.util.VersionNumber;
 import org.apache.commons.io.FileUtils;
 import org.apache.commons.io.FilenameUtils;
 import org.kohsuke.args4j.CmdLineException;
 import org.kohsuke.args4j.CmdLineParser;
 import org.kohsuke.args4j.Option;
 
-import edu.umd.cs.findbugs.annotations.CheckForNull;
 import javax.annotation.PostConstruct;
 import java.io.File;
 import java.io.IOException;
@@ -118,6 +118,9 @@ public class Bootstrap {
     @Option(name = "--cli", usage = "Launch interactive CLI.", forbids = { "-v", "--runWorkspace", "-a", "-ns" })
     public boolean cliOnly;
 
+    @Option(name = "--scm", usage = "YAML definition of the SCM, with optional credentials, to use for the project")
+    public File scm;
+
     public static void main(String[] args) throws Throwable {
         // break for attaching profiler
         if (Boolean.getBoolean("start.pause")) {

diff --git a/docs/using/SCM.adoc b/docs/using/SCM.adoc
@@ -0,0 +1,52 @@
+= Using `checkout scm` with Jenkinsfile Runner
+:toc:
+:toc-placement: preamble
+:toclevels: 3
+
+By default, Jenkinsfile Runner will use the directory containing the Jenkinsfile as
+the workspace for builds. However, this will not work correctly with external agents
+or plugins that expect to interact with the SCM. In order to enable these behaviors,
+you can specify a YAML file containing the definition of the SCM to checkout in the build.
+
+== Specifying an SCM definition on the CLI
+
+Add the `--scm (path to YAML file)` option to your `jenkinsfile-runner` invocation.
+Nothing else needs to be changed. Note that Pipeline as YAML is not currently supported
+with the `--scm` option.
+
+== SCM YAML format
+
+The SCM YAML format matches with Jenkins Configuration-as-Code syntax. For example:
+
+[source,yaml]
+----
+scm:
+  git:
+    userRemoteConfigs:
+    - url: https://github.com/jenkinsci/jenkinsfile-runner.git
+    branches:
+    - name: master
+----
+
+== Specifying and using credentials with `checkout scm`
+
+You can specify credentials to be used when checking out the configured SCM by
+adding the credentials definition to your YAML and referencing the ID in the SCM's
+configuration. For example:
+
+[source,yaml]
+----
+credential:
+  usernamePassword:
+    password: secret
+    scope: GLOBAL
+    id: user1
+    username: user1
+scm:
+  git:
+    userRemoteConfigs:
+    - url: https://github.com/jenkinsci/jenkinsfile-runner.git
+      credentialsId: user1
+    branches:
+    - name: master
+----
diff --git a/payload-dependencies/pom.xml b/payload-dependencies/pom.xml
@@ -62,6 +62,10 @@
       <groupId>org.jenkins-ci.plugins</groupId>
       <artifactId>scm-api</artifactId>
     </dependency>
+    <dependency>
+      <groupId>io.jenkins</groupId>
+      <artifactId>configuration-as-code</artifactId>
+    </dependency>
     <dependency>
       <groupId>io.jenkins.plugins</groupId>
       <artifactId>pipeline-as-yaml</artifactId>

diff --git a/payload/src/main/java/io/jenkins/jenkinsfile/runner/Runner.java b/payload/src/main/java/io/jenkins/jenkinsfile/runner/Runner.java
@@ -1,5 +1,11 @@
 package io.jenkins.jenkinsfile.runner;
 
+import com.cloudbees.plugins.credentials.Credentials;
+import com.cloudbees.plugins.credentials.CredentialsProvider;
+import com.cloudbees.plugins.credentials.CredentialsStore;
+import com.cloudbees.plugins.credentials.CredentialsUnavailableException;
+import com.cloudbees.plugins.credentials.SystemCredentialsProvider;
+import com.cloudbees.plugins.credentials.domains.Domain;
 import hudson.model.Action;
 import hudson.model.Cause;
 import hudson.model.CauseAction;
@@ -9,7 +15,6 @@
 import hudson.model.queue.QueueTaskFuture;
 import io.jenkins.jenkinsfile.runner.bootstrap.Bootstrap;
 import jenkins.model.Jenkins;
-
 import org.apache.commons.io.FileUtils;
 import org.jenkinsci.plugins.workflow.cps.CpsScmFlowDefinition;
 import org.jenkinsci.plugins.workflow.job.WorkflowJob;
@@ -37,11 +42,11 @@ public class Runner {
      */
     public int run(Bootstrap bootstrap) throws Exception {
         try {
-          Jenkins.checkGoodName(bootstrap.jobName); 
+          Jenkins.checkGoodName(bootstrap.jobName);
         } catch (Failure e) {
-          System.err.println(String.format("invalid job name: '%s': %s", bootstrap.jobName, e.getMessage())); 
+          System.err.println(String.format("invalid job name: '%s': %s", bootstrap.jobName, e.getMessage()));
           return -1;
-        } 
+        }
         Jenkins j = Jenkins.getInstance();
         WorkflowJob w = j.createProject(WorkflowJob.class, bootstrap.jobName);
         w.updateNextBuildNumber(bootstrap.buildNumber);
@@ -53,9 +58,23 @@ public int run(Bootstrap bootstrap) throws Exception {
           w.setDefinition(new PipelineAsYamlScriptFlowDefinition(
             FileUtils.readFileToString(bootstrap.jenkinsfile),!bootstrap.noSandBox));
         } else {
-          w.setDefinition(new CpsScmFlowDefinition(
-            new FileSystemSCM(bootstrap.jenkinsfile.getParent()), bootstrap.jenkinsfile.getName()));
-          workflowActionsList.add(new SetJenkinsfileLocation(bootstrap.jenkinsfile, !bootstrap.noSandBox));
+            if (bootstrap.scm != null) {
+                SCMContainer scm = SCMContainer.loadFromYAML(bootstrap.scm);
+                Credentials fromSCM = scm.getCredential();
+                if (fromSCM != null) {
+                    try {
+                        addCredentials(fromSCM);
+                    } catch (IOException | CredentialsUnavailableException e) {
+                        System.err.printf("could not create credentials: %s%n", e.getMessage());
+                        return -1;
+                    }
+                }
+                w.setDefinition(new CpsScmFlowDefinition(scm.getSCM(), bootstrap.jenkinsfile.getName()));
+            } else {
+                w.setDefinition(new CpsScmFlowDefinition(
+                        new FileSystemSCM(bootstrap.jenkinsfile.getParent()), bootstrap.jenkinsfile.getName()));
+            }
+            workflowActionsList.add(new SetJenkinsfileLocation(bootstrap.jenkinsfile, !bootstrap.noSandBox));
         }
 
         if (bootstrap.workflowParameters != null && bootstrap.workflowParameters.size() > 0) {
@@ -91,6 +110,17 @@ private CauseAction createCauseAction(String cause) {
       return new CauseAction(c);
     }
 
+    private CredentialsStore getStore() {
+        CredentialsStore store = null;
+        for (CredentialsStore s : CredentialsProvider.lookupStores(Jenkins.get())) {
+            if (s.getProvider() instanceof SystemCredentialsProvider.ProviderImpl) {
+                store = s;
+                break;
+            }
+        }
+        return store;
+    }
+
     private void writeLogTo(PrintStream out) throws IOException, InterruptedException {
         final int retryCnt = 10;
 
@@ -113,4 +143,13 @@ private void writeLogTo(PrintStream out) throws IOException, InterruptedExceptio
             }
         }
     }
+
+    private void addCredentials(Credentials creds) throws IOException, CredentialsUnavailableException {
+        CredentialsStore store = getStore();
+        if (store == null) {
+            throw new CredentialsUnavailableException("Credentials specified but could not find credentials store");
+        }
+        Domain globalDomain = Domain.global();
+        store.addCredentials(globalDomain, creds);
+    }
 }
diff --git a/payload/src/main/java/io/jenkins/jenkinsfile/runner/SCMContainer.java b/payload/src/main/java/io/jenkins/jenkinsfile/runner/SCMContainer.java
@@ -0,0 +1,47 @@
+package io.jenkins.jenkinsfile.runner;
+
+import com.cloudbees.plugins.credentials.Credentials;
+import edu.umd.cs.findbugs.annotations.CheckForNull;
+import hudson.scm.SCM;
+import io.jenkins.plugins.casc.ConfigurationContext;
+import io.jenkins.plugins.casc.ConfiguratorException;
+import io.jenkins.plugins.casc.ConfiguratorRegistry;
+import io.jenkins.plugins.casc.model.Mapping;
+import io.jenkins.plugins.casc.yaml.YamlSource;
+import io.jenkins.plugins.casc.yaml.YamlUtils;
+import org.kohsuke.stapler.DataBoundConstructor;
+
+import java.io.File;
+import java.nio.file.Path;
+import java.util.Collections;
+
+public class SCMContainer {
+    private final SCM scm;
+    private final Credentials credential;
+
+    @DataBoundConstructor
+    public SCMContainer(SCM scm, Credentials credential) {
+        this.scm = scm;
+        this.credential = credential;
+    }
+
+    public SCM getSCM() {
+        return scm;
+    }
+
+    @CheckForNull
+    public Credentials getCredential() {
+        return credential;
+    }
+
+    public static SCMContainer loadFromYAML(File input) throws ConfiguratorException {
+        ConfiguratorRegistry registry = ConfiguratorRegistry.get();
+        ConfigurationContext context = new ConfigurationContext(registry);
+
+        YamlSource<Path> ys = YamlSource.of(input.toPath());
+
+        Mapping config = YamlUtils.loadFrom(Collections.singletonList(ys), context);
+
+        return (SCMContainer) registry.lookupOrFail(SCMContainer.class).configure(config, context);
+    }
+}
diff --git a/tests/exclude-rsync.txt b/tests/exclude-rsync.txt
@@ -1,3 +1,4 @@
 demo
 tests/jenkinsfile-runner-test-framework
-**/.git*
+tests/.jenkinsfile-runner-test-framework
+**/.git*
diff --git a/tests/pom.xml b/tests/pom.xml
@@ -4,7 +4,7 @@
   <parent>
     <groupId>io.jenkins.jenkinsfile-runner</groupId>
     <artifactId>parent</artifactId>
-    <version>1.0-beta-12-SNAPSHOT</version>
+    <version>1.0-beta-18-SNAPSHOT</version>
   </parent>
 
   <artifactId>tests</artifactId>

diff --git a/vanilla-package/pom.xml b/vanilla-package/pom.xml
@@ -16,6 +16,31 @@
     <jfr.packaging.skip.assembly>true</jfr.packaging.skip.assembly>
   </properties>
 
+  <build>
+    <plugins>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-resources-plugin</artifactId>
+        <executions>
+          <execution>
+            <id>default-testResources</id>
+            <phase>process-test-resources</phase>
+            <goals>
+              <goal>testResources</goal>
+            </goals>
+            <configuration>
+              <skip>false</skip>
+              <resources>
+                <resource>
+                  <directory>src/test/resources</directory>
+                </resource>
+              </resources>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
+    </plugins>
+  </build>
   <dependencies>
     <!-- Extra Dependencies for Vanilla -->
     <dependency>