diff --git a/CHANGELOG.md b/CHANGELOG.md index 3cb1db45..9ca64150 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ listed in the changelog. ### Changed - Node.js 18 is now the default for `ods-build-npm` task ([#585](https://github.com/opendevstack/ods-pipeline/issues/585)) +- Images used in tasks are now pulled directly from the GitHub registry. "Wrapping" the images in the OpenShift/K8s cluster is not required anymore. If tasks need to trust a private certificate, it needs to be present as a K8s secret, which will then be mounted as a file in the pods. To add the secret to an existing installation, pass `--private-cert ` to `./install.sh`. For more details, see [#621](https://github.com/opendevstack/ods-pipeline/issues/621). ### Fixed diff --git a/Makefile b/Makefile index 01b7201f..fd241fb0 100644 --- a/Makefile +++ b/Makefile @@ -144,19 +144,3 @@ ifeq ($(strip $(namespace)),) endif cd scripts && ./install-inside-kind.sh -n $(namespace) .PHONY: deploy - -##@ OpenShift - -start-ods-builds: ## Start builds for each ODS BuildConfig - oc start-build ods-package-image - oc start-build ods-finish - oc start-build ods-go-toolset - oc start-build ods-gradle-toolset - oc start-build ods-helm - oc start-build ods-node16-npm-toolset - oc start-build ods-node18-npm-toolset - oc start-build ods-pipeline-manager - oc start-build ods-python-toolset - oc start-build ods-sonar - oc start-build ods-start -.PHONY: start-ods-builds diff --git a/build/package/Dockerfile.gradle-toolset b/build/package/Dockerfile.gradle-toolset index 8a5fa4aa..b9a670dc 100644 --- a/build/package/Dockerfile.gradle-toolset +++ b/build/package/Dockerfile.gradle-toolset @@ -37,13 +37,8 @@ COPY build/package/scripts/copy-build-if-cached.sh /usr/local/bin/copy-build-if- COPY build/package/scripts/copy-artifacts.sh /usr/local/bin/copy-artifacts COPY build/package/scripts/build-gradle.sh /usr/local/bin/build-gradle COPY build/package/scripts/supply-sonar-project-properties-default.sh /usr/local/bin/supply-sonar-project-properties-default -COPY build/package/scripts/set-gradle-proxy.sh /usr/local/bin/set-gradle-proxy -RUN chmod +x /usr/local/bin/build-gradle && \ - chmod +x /usr/local/bin/cache-build && \ - chmod +x /usr/local/bin/copy-build-if-cached && \ - chmod +x /usr/local/bin/copy-artifacts && \ - chmod +x /usr/local/bin/supply-sonar-project-properties-default && \ - chmod +x /usr/local/bin/set-gradle-proxy +COPY build/package/scripts/configure-gradle.sh /usr/local/bin/configure-gradle +COPY build/package/scripts/configure-truststore.sh /usr/local/bin/configure-truststore # Add sonar-project.properties COPY build/package/sonar-project.properties.d/gradle.properties /usr/local/default-sonar-project.properties diff --git a/build/package/Dockerfile.package-image b/build/package/Dockerfile.package-image index a17b3bc4..5272d4b1 100644 --- a/build/package/Dockerfile.package-image +++ b/build/package/Dockerfile.package-image @@ -51,6 +51,9 @@ RUN echo -e "build:1:1000\nbuild:1002:64535" > /etc/subuid \ # Install Trivy RUN curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sh -s -- -b /usr/local/bin "v${TRIVY_VERSION}" +# Add scripts +COPY build/package/scripts/download-aqua-scanner.sh /usr/local/bin/download-aqua-scanner + VOLUME /var/lib/containers VOLUME /home/build/.local/share/containers diff --git a/build/package/Dockerfile.sonar b/build/package/Dockerfile.sonar index b28d82b8..677f3723 100644 --- a/build/package/Dockerfile.sonar +++ b/build/package/Dockerfile.sonar @@ -32,13 +32,16 @@ RUN cd /tmp \ FROM registry.access.redhat.com/ubi8/ubi-minimal:8.4 ENV SONAR_EDITION="community" \ - OPENSSL_VERSION=1.1 + OPENSSL_VERSION=1.1 \ + JAVA_HOME=/usr/lib/jvm/jre-11 RUN microdnf install --nodocs openssl-${OPENSSL_VERSION}* java-11-openjdk-headless which && microdnf clean all COPY --from=builder /usr/local/bin/sonar /usr/local/bin/sonar COPY --from=builder /usr/local/sonar-scanner-cli /usr/local/sonar-scanner-cli COPY --from=builder /usr/local/cnes/cnesreport.jar /usr/local/cnes/cnesreport.jar +COPY build/package/scripts/configure-truststore.sh /usr/local/bin/configure-truststore + ENV PATH=/usr/local/sonar-scanner-cli/bin:$PATH USER 1001 diff --git a/build/package/scripts/build-gradle.sh b/build/package/scripts/build-gradle.sh index 6f372479..52e2e94b 100755 --- a/build/package/scripts/build-gradle.sh +++ b/build/package/scripts/build-gradle.sh @@ -43,8 +43,9 @@ echo "Using NEXUS_URL=$NEXUS_URL" echo "Using GRADLE_OPTS=$GRADLE_OPTS" echo "Using GRADLE_USER_HOME=$GRADLE_USER_HOME" echo "Using ARTIFACTS_DIR=$ARTIFACTS_DIR" +mkdir -p "${GRADLE_USER_HOME}" -set-gradle-proxy +configure-gradle echo cd "${WORKING_DIR}" diff --git a/build/package/scripts/build-npm.sh b/build/package/scripts/build-npm.sh index de17a932..ba3bbf3a 100755 --- a/build/package/scripts/build-npm.sh +++ b/build/package/scripts/build-npm.sh @@ -65,17 +65,17 @@ if [ "${WORKING_DIR}" != "." ]; then ARTIFACT_PREFIX="${WORKING_DIR/\//-}-" fi -echo "Configuring npm to use Nexus ..." -# Remove the protocol segment from NEXUS_URL -NEXUS_HOST=$(echo "${NEXUS_URL}" | sed -E 's/^\s*.*:\/\///g') -if [ -n "${NEXUS_HOST}" ] && [ -n "${NEXUS_USERNAME}" ] && [ -n "${NEXUS_PASSWORD}" ]; then +echo "Configuring npm to use Nexus (${NEXUS_URL}) ..." +if [ -n "${NEXUS_URL}" ] && [ -n "${NEXUS_USERNAME}" ] && [ -n "${NEXUS_PASSWORD}" ]; then NEXUS_AUTH="$(urlencode "${NEXUS_USERNAME}"):$(urlencode "${NEXUS_PASSWORD}")" npm config set registry="$NEXUS_URL"/repository/npmjs/ npm config set always-auth=true npm config set _auth="$(echo -n "$NEXUS_AUTH" | base64)" npm config set email=no-reply@opendevstack.org - npm config set ca=null - npm config set strict-ssl=false + if [ -f /etc/ssl/certs/private-cert.pem ]; then + echo "Configuring private cert ..." + npm config set cafile=/etc/ssl/certs/private-cert.pem + fi fi; echo "package-*.json checks ..." diff --git a/build/package/scripts/build-python.sh b/build/package/scripts/build-python.sh index 1efd8e0a..a06b0efb 100755 --- a/build/package/scripts/build-python.sh +++ b/build/package/scripts/build-python.sh @@ -55,7 +55,7 @@ if [ "${WORKING_DIR}" != "." ]; then ARTIFACT_PREFIX="${WORKING_DIR/\//-}-" fi -echo "Configuring pip to use Nexus ..." +echo "Configuring pip to use Nexus (${NEXUS_URL}) ..." # Remove the protocol segment from NEXUS_URL NEXUS_HOST=$(echo "${NEXUS_URL}" | sed -E 's/^\s*.*:\/\///g') if [ -n "${NEXUS_HOST}" ] && [ -n "${NEXUS_USERNAME}" ] && [ -n "${NEXUS_PASSWORD}" ]; then diff --git a/build/package/scripts/set-gradle-proxy.sh b/build/package/scripts/configure-gradle.sh similarity index 69% rename from build/package/scripts/set-gradle-proxy.sh rename to build/package/scripts/configure-gradle.sh index fddae0c8..81e1d17b 100755 --- a/build/package/scripts/set-gradle-proxy.sh +++ b/build/package/scripts/configure-gradle.sh @@ -3,8 +3,20 @@ # This script checks for env variable HTTP_PROXY and adds them to gradle.properties. CONTENT="" -if [[ $HTTP_PROXY != "" ]]; then +if [ -f /etc/ssl/certs/private-cert.pem ]; then + echo "Configuring Gradle to trust private cert ..." + configure-truststore --dest-store ".ods-cache/truststore/cacerts" + # shellcheck disable=SC2181 + if [ $? -ne 0 ]; then + exit 1 + fi + # Configure Gradle to use the modified trust store. + CONTENT+="systemProp.javax.net.ssl.trustStore=.ods-cache/keystore/cacerts\n" + CONTENT+="systemProp.javax.net.ssl.trustStorePassword=password\n" +fi +if [ "${HTTP_PROXY}" != "" ]; then + echo "Configuring Gradle to honor HTTP_PROXY ..." proxy=$(echo "$HTTP_PROXY" | sed -e "s|https://||g" | sed -e "s|http://||g") proxy_hostp=$(echo "$proxy" | cut -d "@" -f2) @@ -32,7 +44,8 @@ if [[ $HTTP_PROXY != "" ]]; then fi fi -if [[ $NO_PROXY != "" ]]; then +if [ "${NO_PROXY}" != "" ]; then + echo "Configuring Gradle to honor NO_PROXY ..." # shellcheck disable=SC2001 noproxy_host=$(echo "$NO_PROXY" | sed -e 's|\,\.|\,\*\.|g') # shellcheck disable=SC2001 @@ -41,6 +54,6 @@ if [[ $NO_PROXY != "" ]]; then CONTENT+="systemProp.https.nonProxyHosts=$noproxy_host\n" fi -if [[ $CONTENT != "" ]]; then +if [ "${CONTENT}" != "" ]; then echo -e "$CONTENT" > "${GRADLE_USER_HOME}/gradle.properties" fi diff --git a/build/package/scripts/configure-truststore.sh b/build/package/scripts/configure-truststore.sh new file mode 100755 index 00000000..22e74026 --- /dev/null +++ b/build/package/scripts/configure-truststore.sh @@ -0,0 +1,54 @@ +#!/bin/bash +set -u + +md5_bin="${MD5_BIN:-"md5sum --tag"}" +private_cert="/etc/ssl/certs/private-cert.pem" +src_truststore="${JAVA_HOME}/lib/security/cacerts" +src_pass="changeit" +dest_pass="changeit" + +while [[ "$#" -gt 0 ]]; do + case $1 in + + --src-store) src_truststore="$2"; shift;; + --src-store=*) src_truststore="${1#*=}";; + + --src-storepass) src_pass="$2"; shift;; + --src-storepass=*) src_pass="${1#*=}";; + + --dest-store) dest_truststore="$2"; shift;; + --dest-store=*) dest_truststore="${1#*=}";; + + --dest-storepass) dest_pass="$2"; shift;; + --dest-storepass=*) dest_pass="${1#*=}";; + + --debug) set -x; shift;; + + *) echo "Unknown parameter passed: $1"; exit 1;; +esac; shift; done + +dest_truststore_dir="${dest_truststore%/*}" +mkdir -p "${dest_truststore_dir}" +md5_private_cert_path="${dest_truststore_dir}/.md5-private-cert" +md5_private_cert=$(${md5_bin} "${private_cert}") + +if [ ! -f "${dest_truststore}" ] || [ "${md5_private_cert}" != "$(cat "${md5_private_cert_path}")" ]; then + echo "Creating truststore with private cert ..." + # Copy global keystone to location where we can write to (hide output containing warnings). + keytool -importkeystore \ + -srckeystore "${src_truststore}" -destkeystore "${dest_truststore}" \ + -deststorepass "${dest_pass}" -srcstorepass "${src_pass}" &> keytool-output.txt + # shellcheck disable=SC2181 + if [ $? -ne 0 ]; then + cat keytool-output.txt; exit 1 + fi + # Trust private cert (hide output containing warnings). + keytool -importcert -noprompt -trustcacerts \ + -alias private-cert -file "${private_cert}" \ + -keystore "${dest_truststore}" -storepass "${dest_pass}" &> keytool-output.txt + # shellcheck disable=SC2181 + if [ $? -ne 0 ]; then + cat keytool-output.txt; exit 1 + fi + echo "${md5_private_cert}" > "${md5_private_cert_path}" +fi diff --git a/build/package/scripts/download-aqua-scanner.sh b/build/package/scripts/download-aqua-scanner.sh new file mode 100755 index 00000000..12278b08 --- /dev/null +++ b/build/package/scripts/download-aqua-scanner.sh @@ -0,0 +1,39 @@ +#!/bin/bash +set -eu + +md5_bin="${MD5_BIN:-"md5sum --tag"}" +aqua_scanner_url="" +bin_dir=".ods-cache/bin" + +while [[ "$#" -gt 0 ]]; do + case $1 in + + --bin-dir) bin_dir="$2"; shift;; + --bin-dir=*) bin_dir="${1#*=}";; + + --aqua-scanner-url) aqua_scanner_url="$2"; shift;; + --aqua-scanner-url=*) aqua_scanner_url="${1#*=}";; + + --debug) set -x; shift;; + + *) echo "Unknown parameter passed: $1"; exit 1;; +esac; shift; done + +aqua_scanner_path="${bin_dir}/aquasec" +md5_aqua_scanner_url_path="${bin_dir}/.md5-aquasec" + +# Optionally install Aqua scanner. +# If the binary already exists and was downloaded from the +# URL given by aqua_scanner_url, skip download. +if [ -n "${aqua_scanner_url}" ] && [ "${aqua_scanner_url}" != "none" ]; then + md5_aqua_scanner_url=$(${md5_bin} -s "${aqua_scanner_url}") + if [ ! -f "${md5_aqua_scanner_url_path}" ] || [ "${md5_aqua_scanner_url}" != "$(cat "${md5_aqua_scanner_url_path}")" ]; then + echo 'Installing Aqua scanner...' + curl -v -sSf -L "${aqua_scanner_url}" -o aquasec + mv aquasec "${aqua_scanner_path}" + chmod +x "${aqua_scanner_path}" + echo "${md5_aqua_scanner_url}" > "${md5_aqua_scanner_url_path}" + echo 'Installed Aqua scanner version:' + "${aqua_scanner_path}" version + fi +fi diff --git a/cmd/artifact-download/main.go b/cmd/artifact-download/main.go index 2d667fb0..ca052b16 100644 --- a/cmd/artifact-download/main.go +++ b/cmd/artifact-download/main.go @@ -5,10 +5,14 @@ // // There are two main modes of the program: // (1) users supply (OpenShift) namespace, (Bitbucket) project, (Git) repository -// and a tag such as "v1.0.0". +// +// and a tag such as "v1.0.0". +// // (2) users run this program from the root of a Git repository and only supply -// (OpenShift) namespace and tag=WIP. In this case the latest artifacts are -// downloaded. +// +// (OpenShift) namespace and tag=WIP. In this case the latest artifacts are +// downloaded. +// // Mode (1) is the main use case, mode (2) is provided as a convenience feature // for developers. package main @@ -45,6 +49,7 @@ type options struct { version bool tag string outputDirectory string + privateCert string debug bool } @@ -72,6 +77,7 @@ func main() { flag.StringVar(&opts.repository, "repository", "", "Bitbucket repository key") flag.StringVar(&opts.tag, "tag", "", "Git tag to retrieve artifacts for, e.g. v1.0.0 (required)") flag.StringVar(&opts.outputDirectory, "output", "artifacts-out", "Directory to place outputs into") + flag.StringVar(&opts.privateCert, "private-cert", "", "Path to private certification (in PEM format)") flag.BoolVar(&opts.debug, "debug", (os.Getenv("DEBUG") == "true"), "Enable debug mode") flag.BoolVar(&opts.version, "version", false, "Display version of binary") flag.Parse() @@ -129,7 +135,7 @@ func main() { } // Bitbucket client - bcc, err := installation.NewBitbucketClientConfig(c, opts.namespace, logger) + bcc, err := installation.NewBitbucketClientConfig(c, opts.namespace, logger, opts.privateCert) if err != nil { log.Fatalf("Could not create Bitbucket client config: %s. Are you logged into the cluster?", err) } diff --git a/cmd/package-image/aqua.go b/cmd/package-image/aqua.go index 3e4ed05b..f6efc217 100644 --- a/cmd/package-image/aqua.go +++ b/cmd/package-image/aqua.go @@ -10,7 +10,7 @@ import ( ) const ( - aquasecBin = "aquasec" + aquasecBin = "./.ods-cache/bin/aquasec" scanComplianceFailureExitCode = 4 scanLicenseValidationFailureExitCode = 5 ) diff --git a/cmd/sonar/main.go b/cmd/sonar/main.go index 425e5476..8278263b 100644 --- a/cmd/sonar/main.go +++ b/cmd/sonar/main.go @@ -15,13 +15,26 @@ import ( ) type options struct { - sonarAuthToken string - sonarURL string - sonarEdition string - workingDir string - rootPath string - qualityGate bool - debug bool + sonarAuthToken string + sonarURL string + sonarEdition string + workingDir string + rootPath string + qualityGate bool + trustStore string + trustStorePassword string + debug bool +} + +var defaultOptions = options{ + sonarAuthToken: os.Getenv("SONAR_AUTH_TOKEN"), + sonarURL: os.Getenv("SONAR_URL"), + sonarEdition: os.Getenv("SONAR_EDITION"), + workingDir: ".", + qualityGate: false, + trustStore: "${JAVA_HOME}/lib/security/cacerts", + trustStorePassword: "changeit", + debug: (os.Getenv("DEBUG") == "true"), } func main() { @@ -31,12 +44,14 @@ func main() { } opts := options{rootPath: rootPath} - flag.StringVar(&opts.sonarAuthToken, "sonar-auth-token", os.Getenv("SONAR_AUTH_TOKEN"), "sonar-auth-token") - flag.StringVar(&opts.sonarURL, "sonar-url", os.Getenv("SONAR_URL"), "sonar-url") - flag.StringVar(&opts.sonarEdition, "sonar-edition", os.Getenv("SONAR_EDITION"), "sonar-edition") - flag.StringVar(&opts.workingDir, "working-dir", ".", "working directory") - flag.BoolVar(&opts.qualityGate, "quality-gate", false, "require quality gate pass") - flag.BoolVar(&opts.debug, "debug", (os.Getenv("DEBUG") == "true"), "debug mode") + flag.StringVar(&opts.sonarAuthToken, "sonar-auth-token", defaultOptions.sonarAuthToken, "sonar-auth-token") + flag.StringVar(&opts.sonarURL, "sonar-url", defaultOptions.sonarURL, "sonar-url") + flag.StringVar(&opts.sonarEdition, "sonar-edition", defaultOptions.sonarEdition, "sonar-edition") + flag.StringVar(&opts.workingDir, "working-dir", defaultOptions.workingDir, "working directory") + flag.BoolVar(&opts.qualityGate, "quality-gate", defaultOptions.qualityGate, "require quality gate pass") + flag.StringVar(&opts.trustStore, "truststore", defaultOptions.trustStore, "JKS truststore") + flag.StringVar(&opts.trustStorePassword, "truststore-pass", defaultOptions.trustStorePassword, "JKS truststore password") + flag.BoolVar(&opts.debug, "debug", defaultOptions.debug, "debug mode") flag.Parse() var logger logging.LeveledLoggerInterface @@ -58,11 +73,13 @@ func main() { } sonarClient, err := sonar.NewClient(&sonar.ClientConfig{ - APIToken: opts.sonarAuthToken, - BaseURL: opts.sonarURL, - ServerEdition: opts.sonarEdition, - Debug: opts.debug, - Logger: logger, + APIToken: opts.sonarAuthToken, + BaseURL: opts.sonarURL, + ServerEdition: opts.sonarEdition, + TrustStore: opts.trustStore, + TrustStorePassword: opts.trustStorePassword, + Debug: opts.debug, + Logger: logger, }) if err != nil { log.Fatal("sonar client:", err) @@ -96,17 +113,17 @@ func sonarScan( Base: ctxt.PullRequestBase, } } - scanStdout, err := sonarClient.Scan( + err := sonarClient.Scan( sonarProject, ctxt.GitRef, ctxt.GitCommitSHA, prInfo, + os.Stdout, + os.Stdin, ) if err != nil { - logger.Infof(scanStdout) return fmt.Errorf("scan failed: %w", err) } - logger.Infof(scanStdout) logger.Infof("Wait until compute engine task finishes ...") err = waitUntilComputeEngineTaskIsSuccessful(logger, sonarClient) diff --git a/cmd/sonar/main_test.go b/cmd/sonar/main_test.go index 0f1b5822..abac4fb9 100644 --- a/cmd/sonar/main_test.go +++ b/cmd/sonar/main_test.go @@ -1,6 +1,7 @@ package main import ( + "io" "os" "strings" "testing" @@ -17,9 +18,9 @@ type fakeClient struct { reportGenerated bool } -func (c *fakeClient) Scan(sonarProject, branch, commit string, pr *sonar.PullRequest) (string, error) { +func (c *fakeClient) Scan(sonarProject, branch, commit string, pr *sonar.PullRequest, outWriter, errWriter io.Writer) error { c.scanPerformed = true - return "", nil + return nil } func (c *fakeClient) QualityGateGet(p sonar.QualityGateGetParams) (*sonar.QualityGate, error) { diff --git a/cmd/start/cache.go b/cmd/start/cache.go index 07b3b64b..f50ab904 100644 --- a/cmd/start/cache.go +++ b/cmd/start/cache.go @@ -14,8 +14,7 @@ // // In addition to function deleteDirectoryContentsSpareCache a function // is provided to clean the cache. This is done in two functions to avoid -// making the code too complex as new cache cleaning functionality will -// likely be added in the future, for example to implement build skipping. +// making the code too complex. package main @@ -59,15 +58,12 @@ func deleteDirectoryContentsSpareCache(fsb FileSystemBase, fnRemove RemoveFunc) }, withBaseFileRemover(fsb.base, fnRemove)) } -// Cleans the cache -// At the moment only a cache for dependencies is supported and -// All other content is removed to ensure that tasks don't use -// cache areas accidentally. This effectively reserve other +// Cleans the cache. +// At the moment only a cache for dependencies is supported. The dependencies +// are assumed to live in a subdirectory of odsCacheDependenciesDirName. +// Files directly in odsCacheDependenciesDirName are removed to ensure that +// tasks don't use cache areas accidentally. This effectively reserves other // areas for future use. -// For example if in the future build skipping is supported -// there would likely be an area where the build output is kept -// per git-sha of the working-dir. In this case a suitable cleanup -// might delete such areas after a certain time (see PR #423). func cleanCache(fsb FileSystemBase, fnRemove RemoveFunc, expirationDays int) error { _, err := fsb.filesystem.Open(odsCacheDirName) if err != nil && os.IsNotExist(err) { @@ -79,24 +75,20 @@ func cleanCache(fsb FileSystemBase, fnRemove RemoveFunc, expirationDays int) err return err } cacheDependenciesPath := filepath.Join(".", odsCacheDependenciesDirName) - // To avoid spare files inside the cache which are not supported delete - // all other areas of the cache + // To avoid spare files inside the dependency cache which are not supported, + // delete all unknown folders in there. + // Files outside the dependency cache are left untouched. dirEntryFunc := func(path string, d fs.DirEntry) WalkAndRemovalFlags { - if !strings.HasPrefix(path, cacheDependenciesPath) { return 0 // allow files outside the dependency cache area for experimentation } // Dependencies must be inside a folder specific to a technology // such as for npm or go. - // Clean all files which are not directories + // Clean all files which are not directories. if path == cacheDependenciesPath { return enterDir } - // There should be no files below cacheDependenciesPath - // but all dirs are deemed valid. - // technology-folder names are not meant to be registered - // anywhere at this point. if d.IsDir() { return skipDir } else { @@ -104,17 +96,13 @@ func cleanCache(fsb FileSystemBase, fnRemove RemoveFunc, expirationDays int) err } } fnRemoveWithBase := withBaseFileRemover(filepath.Join(fsb.base, odsCacheDirName), fnRemove) - err = deleteDirRecursiveWithSkip( - fsCache, - dirEntryFunc, - fnRemoveWithBase) + err = deleteDirRecursiveWithSkip(fsCache, dirEntryFunc, fnRemoveWithBase) if err != nil { return err } keepTimestamp := time.Now().AddDate(0, 0, -1*expirationDays) // now delete build task cache - _, err = cleanupNotRecentlyUsed(fsCache, odsCacheBuildOutputDirName, keepTimestamp, - fnRemoveWithBase) + _, err = cleanupNotRecentlyUsed(fsCache, odsCacheBuildOutputDirName, keepTimestamp, fnRemoveWithBase) return err } diff --git a/cmd/start/main.go b/cmd/start/main.go index 0357935a..8e2deda8 100644 --- a/cmd/start/main.go +++ b/cmd/start/main.go @@ -222,7 +222,7 @@ func main() { } } - logger.Infof("Downloading any artifacts ...") + logger.Infof("Downloading any artifacts from %s ...", opts.nexusURL) // If there are subrepos, then all of them need to have a successful pipeline run. nexusClient, err := nexus.NewClient(&nexus.ClientConfig{ BaseURL: opts.nexusURL, @@ -380,15 +380,17 @@ func checkoutAndAssembleContext( log.Fatal(err) } logger.Infof("Checking out %s@%s into %s ...", url, gitFullRef, absCheckoutDir) - stdout, stderr, err := command.RunBuffered("/ko-app/git-init", []string{ - "-url", url, - "-revision", gitFullRef, - "-refspec", gitRefSpec, - "-path", absCheckoutDir, - "-sslVerify", sslVerify, - "-submodules", submodules, - "-depth", depth, - }) + gitInitArgs := []string{ + fmt.Sprintf("-url=%v", url), + fmt.Sprintf("-revision=%v", gitFullRef), + fmt.Sprintf("-refspec=%v", gitRefSpec), + fmt.Sprintf("-path=%v", absCheckoutDir), + fmt.Sprintf("-sslVerify=%v", sslVerify), + fmt.Sprintf("-submodules=%v", submodules), + fmt.Sprintf("-depth=%v", depth), + } + logger.Debugf("git-init %s", strings.Join(gitInitArgs, " ")) + stdout, stderr, err := command.RunBuffered("/ko-app/git-init", gitInitArgs) if err != nil { logger.Errorf(string(stderr)) log.Fatal(err) diff --git a/deploy/.gitignore b/deploy/.gitignore index 1b7b71a6..767ce472 100644 --- a/deploy/.gitignore +++ b/deploy/.gitignore @@ -1,2 +1,2 @@ *.dec -.kind-credentials/ +.kind-values/ diff --git a/deploy/README.md b/deploy/README.md index 30e9b66c..8b375419 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -6,18 +6,15 @@ Manifests in `ods-pipeline` are applied once per project by a project administra ## Subcharts -The `tasks`, `images` and `setup` subcharts are maintained in https://github.com/opendevstack/ods-pipeline, and may be used by project admins to control the deployment of ODS pipeline resources in the respective project namespace in OpenShift. +The `tasks` and `setup` subcharts are maintained in https://github.com/opendevstack/ods-pipeline, and may be used by project admins to control the deployment of ODS pipeline resources in the respective project namespace in OpenShift. ### Subcharts Contents The resources are defined using Helm: -* `BuildConfig` and `ImageStream` resources (in the `images` subchart) * `Task` resources (in `tasks` subchart) * `ConfigMap` and `Secret` resources used by ODS tasks (in `setup` subchart) * ODS pipeline manager (`Service`/`Deployment`) (in `setup` subchart) -The resources of the `images` subchart are only applicable for OpenShift clusters. The subcharts may individually be enabled or disabled via the umbrella chart's `values.yaml`. - ### Versioning In a KinD cluster there are no versions. Images use the implicit `latest` tag. That makes testing and local development easy. diff --git a/deploy/install.sh b/deploy/install.sh index f1c31cb7..28826f9b 100755 --- a/deploy/install.sh +++ b/deploy/install.sh @@ -14,10 +14,12 @@ CHART_DIR="./ods-pipeline" # Secrets AUTH_SEPARATOR=":" AQUA_AUTH="" +AQUA_SCANNER_URL="" BITBUCKET_AUTH="" BITBUCKET_WEBHOOK_SECRET="" NEXUS_AUTH="" SONAR_AUTH="" +PRIVATE_CERT="" # Check prerequisites. KUBECTL_BIN="" @@ -46,10 +48,12 @@ function usage { printf "\t--dry-run\t\t\tDo not apply any changes, instead just print what the script would do.\n" printf "\t--auth-separator\t\tCharacter to use as a separator for basic auth flags (defaults to '%s')\n" "$AUTH_SEPARATOR" printf "\t--aqua-auth\t\t\tUsername and password (separated by '%s') of an Aqua user (if not given, script will prompt for this).\n" "$AUTH_SEPARATOR" + printf "\t--aqua-scanner-url\t\t\tURL from which to download Aqua scanner (if not given, script will prompt for this).\n" printf "\t--bitbucket-auth\t\tAccess token of a Bitbucket user (if not given, script will prompt for this).\n" printf "\t--bitbucket-webhook-secret\tSecret to protect webhook endpoint with (if not given, script will generate this).\n" printf "\t--nexus-auth\t\t\tUsername and password (separated by '%s') of a Nexus user (if not given, script will prompt for this).\n" "$AUTH_SEPARATOR" printf "\t--sonar-auth\t\t\tAuth token of a SonarQube user (if not given, script will prompt for this).\n" + printf "\t--private-cert\t\t\tHost from which to download private certificate (if not given, script will skip this).\n" printf "\nExample:\n\n" printf "\t%s \ \ \n\t\t--namespace foo \ \ @@ -86,6 +90,9 @@ while [[ "$#" -gt 0 ]]; do --aqua-auth) AQUA_AUTH="$2"; shift;; --aqua-auth=*) AQUA_AUTH="${1#*=}";; + --aqua-scanner-url) AQUA_SCANNER_URL="$2"; shift;; + --aqua-scanner-url=*) AQUA_SCANNER_URL="${1#*=}";; + --bitbucket-auth) BITBUCKET_AUTH="$2"; shift;; --bitbucket-auth=*) BITBUCKET_AUTH="${1#*=}";; @@ -98,6 +105,9 @@ while [[ "$#" -gt 0 ]]; do --sonar-auth) SONAR_AUTH="$2"; shift;; --sonar-auth=*) SONAR_AUTH="${1#*=}";; + --private-cert) PRIVATE_CERT="$2"; shift;; + --private-cert=*) PRIVATE_CERT="${1#*=}";; + *) echo "Unknown parameter passed: $1"; exit 1;; esac; shift; done @@ -168,6 +178,35 @@ installSecret () { fi } +installTLSSecret () { + local secretName="$1" + local privateCert="$2" + local certFile="" + if [ -z "${privateCert}" ]; then + echo "No private cert given, skipping ..." + else + if [ "${privateCert:0:1}" == '/' ] || [ "${privateCert:0:2}" == './' ]; then + if [ ! -f "${privateCert}" ]; then + echo "No cert file exists at ${privateCert}"; exit 1 + fi + certFile="${privateCert}" + else + certFile="private-cert.pem.tmp" + openssl s_client -showcerts -connect "${privateCert}" "${certFile}" + fi + if "${KUBECTL_BIN}" -n "${NAMESPACE}" get "secret/${secretName}" &> /dev/null; then + echo "Re-creating secret ${secretName} ..." + "${KUBECTL_BIN}" -n "${NAMESPACE}" delete secret "${secretName}" + else + echo "Creating secret ${secretName} ..." + fi + "${KUBECTL_BIN}" -n "${NAMESPACE}" create secret generic "${secretName}" \ + --from-file=tls.crt="${certFile}" + rm private-cert.pem.tmp &>/dev/null || true + fi +} + # Manage serviceaccount ... if "${KUBECTL_BIN}" -n "${NAMESPACE}" get serviceaccount/"${SERVICEACCOUNT}" &> /dev/null; then echo "Serviceaccount exists already ..." @@ -192,8 +231,15 @@ else installSecret "ods-aqua-auth" \ "basic-auth-secret.yaml.tmpl" \ "${AQUA_AUTH}" \ - "Please enter the username of an Aqua user with scan permissions:" \ - "Please enter the password of this Aqua user (input will be hidden):" + "Please enter the username of an Aqua user with scan permissions. If you do not want to use Aqua, leave this empty:" \ + "Please enter the password of this Aqua user (input will be hidden). If you do not want to use Aqua, leave this empty:" + + # Aqua scanner URL is a single value. + installSecret "ods-aqua-scanner-url" \ + "opaque-secret.yaml.tmpl" \ + "${AQUA_SCANNER_URL}" \ + "" \ + "Please enter the URL from which to download the Aqua scanner binary. The URL may need to contain basic authentication - if so, ensure username/password are URL-encoded. Further, ensure that the version matches your Aqua server version. If you do not want to use Aqua, leave this empty:" # Bitbucket username is not required as PAT alone is enough. installSecret "ods-bitbucket-auth" \ @@ -221,6 +267,8 @@ else "${SONAR_AUTH}" \ "" \ "Please enter an auth token of a SonarQube user with scan permissions (input will be hidden):" + + installTLSSecret "ods-private-cert" "${PRIVATE_CERT}" fi echo "Installing Helm release ${RELEASE_NAME} ..." diff --git a/deploy/ods-pipeline/Chart.yaml b/deploy/ods-pipeline/Chart.yaml index af375267..e95b68e7 100644 --- a/deploy/ods-pipeline/Chart.yaml +++ b/deploy/ods-pipeline/Chart.yaml @@ -24,9 +24,6 @@ version: 0.8.0 appVersion: "0.8.0" dependencies: - - name: images - version: 0.8.0 - condition: images.enabled - name: setup version: 0.8.0 condition: setup.enabled diff --git a/deploy/ods-pipeline/charts/images/Chart.yaml b/deploy/ods-pipeline/charts/images/Chart.yaml deleted file mode 100644 index 59aa7433..00000000 --- a/deploy/ods-pipeline/charts/images/Chart.yaml +++ /dev/null @@ -1,23 +0,0 @@ -apiVersion: v2 -name: images -description: A Helm chart to setup ODS pipeline images - -# A chart can be either an 'application' or a 'library' chart. -# -# Application charts are a collection of templates that can be packaged into versioned archives -# to be deployed. -# -# Library charts provide useful utilities or functions for the chart developer. They're included as -# a dependency of application charts to inject those utilities and functions into the rendering -# pipeline. Library charts do not define any templates and therefore cannot be deployed. -type: application - -# This is the chart version. This version number should be incremented each time you make changes -# to the chart and its templates, including the app version. -# Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.8.0 - -# This is the version number of the application being deployed. This version number should be -# incremented each time you make changes to the application. Versions are not expected to -# follow Semantic Versioning. They should reflect the version the application is using. -appVersion: "0.8.0" diff --git a/deploy/ods-pipeline/charts/images/docker/Dockerfile.finish b/deploy/ods-pipeline/charts/images/docker/Dockerfile.finish deleted file mode 100644 index 064b53e2..00000000 --- a/deploy/ods-pipeline/charts/images/docker/Dockerfile.finish +++ /dev/null @@ -1,10 +0,0 @@ -ARG imageTag="latest" - -FROM ghcr.io/opendevstack/ods-pipeline/ods-finish:$imageTag - -ARG privateCertServer -USER root -RUN if [ -n "${privateCertServer}" ]; then openssl s_client -showcerts -connect "${privateCertServer}" "/etc/pki/ca-trust/source/anchors/${privateCertServer%:*}.pem" && \ - update-ca-trust; fi -USER 1001 diff --git a/deploy/ods-pipeline/charts/images/docker/Dockerfile.go-toolset b/deploy/ods-pipeline/charts/images/docker/Dockerfile.go-toolset deleted file mode 100644 index edfa0ce2..00000000 --- a/deploy/ods-pipeline/charts/images/docker/Dockerfile.go-toolset +++ /dev/null @@ -1,10 +0,0 @@ -ARG imageTag="latest" - -FROM ghcr.io/opendevstack/ods-pipeline/ods-go-toolset:$imageTag - -ARG privateCertServer -USER root -RUN if [ -n "${privateCertServer}" ]; then openssl s_client -showcerts -connect "${privateCertServer}" "/etc/pki/ca-trust/source/anchors/${privateCertServer%:*}.pem" && \ - update-ca-trust; fi -USER 1001 diff --git a/deploy/ods-pipeline/charts/images/docker/Dockerfile.gradle-toolset b/deploy/ods-pipeline/charts/images/docker/Dockerfile.gradle-toolset deleted file mode 100644 index 80d83308..00000000 --- a/deploy/ods-pipeline/charts/images/docker/Dockerfile.gradle-toolset +++ /dev/null @@ -1,10 +0,0 @@ -ARG imageTag="latest" - -FROM ghcr.io/opendevstack/ods-pipeline/ods-gradle-toolset:$imageTag - -ARG privateCertServer -USER root -RUN if [ -n "${privateCertServer}" ]; then openssl s_client -showcerts -connect "${privateCertServer}" "/etc/pki/ca-trust/source/anchors/${privateCertServer%:*}.pem" && \ - update-ca-trust; fi -USER 1001 diff --git a/deploy/ods-pipeline/charts/images/docker/Dockerfile.helm b/deploy/ods-pipeline/charts/images/docker/Dockerfile.helm deleted file mode 100644 index 7c9bbc56..00000000 --- a/deploy/ods-pipeline/charts/images/docker/Dockerfile.helm +++ /dev/null @@ -1,10 +0,0 @@ -ARG imageTag="latest" - -FROM ghcr.io/opendevstack/ods-pipeline/ods-helm:$imageTag - -ARG privateCertServer -USER root -RUN if [ -n "${privateCertServer}" ]; then openssl s_client -showcerts -connect "${privateCertServer}" "/etc/pki/ca-trust/source/anchors/${privateCertServer%:*}.pem" && \ - update-ca-trust; fi -USER 1001 diff --git a/deploy/ods-pipeline/charts/images/docker/Dockerfile.node16-npm-toolset b/deploy/ods-pipeline/charts/images/docker/Dockerfile.node16-npm-toolset deleted file mode 100644 index ca28169f..00000000 --- a/deploy/ods-pipeline/charts/images/docker/Dockerfile.node16-npm-toolset +++ /dev/null @@ -1,10 +0,0 @@ -ARG imageTag="latest" - -FROM ghcr.io/opendevstack/ods-pipeline/ods-node16-npm-toolset:$imageTag - -ARG privateCertServer -USER root -RUN if [ -n "${privateCertServer}" ]; then openssl s_client -showcerts -connect "${privateCertServer}" "/etc/pki/ca-trust/source/anchors/${privateCertServer%:*}.pem" && \ - update-ca-trust; fi -USER 1001 diff --git a/deploy/ods-pipeline/charts/images/docker/Dockerfile.node18-npm-toolset b/deploy/ods-pipeline/charts/images/docker/Dockerfile.node18-npm-toolset deleted file mode 100644 index 141bafb1..00000000 --- a/deploy/ods-pipeline/charts/images/docker/Dockerfile.node18-npm-toolset +++ /dev/null @@ -1,10 +0,0 @@ -ARG imageTag="latest" - -FROM ghcr.io/opendevstack/ods-pipeline/ods-node18-npm-toolset:$imageTag - -ARG privateCertServer -USER root -RUN if [ -n "${privateCertServer}" ]; then openssl s_client -showcerts -connect "${privateCertServer}" "/etc/pki/ca-trust/source/anchors/${privateCertServer%:*}.pem" && \ - update-ca-trust; fi -USER 1001 diff --git a/deploy/ods-pipeline/charts/images/docker/Dockerfile.package-image b/deploy/ods-pipeline/charts/images/docker/Dockerfile.package-image deleted file mode 100644 index f4e3dd8b..00000000 --- a/deploy/ods-pipeline/charts/images/docker/Dockerfile.package-image +++ /dev/null @@ -1,24 +0,0 @@ -ARG imageTag="latest" - -FROM ghcr.io/opendevstack/ods-pipeline/ods-package-image:$imageTag - -ARG aquasecScannerUrl -ARG privateCertServer - -USER root - -# Optionally install Aqua scanner. -RUN if [ -z $aquasecScannerUrl ] ; then echo 'Skipping Aqua scanner installation!' ; else echo 'Installing Aqua scanner... getting binary from' $aquasecScannerUrl \ - && curl -v -L $aquasecScannerUrl -o aquasec \ - && mv aquasec /usr/local/bin/ \ - && chmod +x /usr/local/bin/aquasec \ - && echo 'Aqua scanner version:' \ - && aquasec version \ - && echo 'Aqua scanner installation completed!'; \ - fi - -RUN if [ -n "${privateCertServer}" ]; then openssl s_client -showcerts -connect "${privateCertServer}" "/etc/pki/ca-trust/source/anchors/${privateCertServer%:*}.pem" && \ - update-ca-trust; fi - -USER 1001 diff --git a/deploy/ods-pipeline/charts/images/docker/Dockerfile.pipeline-manager b/deploy/ods-pipeline/charts/images/docker/Dockerfile.pipeline-manager deleted file mode 100644 index 2fbca7fd..00000000 --- a/deploy/ods-pipeline/charts/images/docker/Dockerfile.pipeline-manager +++ /dev/null @@ -1,10 +0,0 @@ -ARG imageTag="latest" - -FROM ghcr.io/opendevstack/ods-pipeline/ods-pipeline-manager:$imageTag - -ARG privateCertServer -USER root -RUN if [ -n "${privateCertServer}" ]; then openssl s_client -showcerts -connect "${privateCertServer}" "/etc/pki/ca-trust/source/anchors/${privateCertServer%:*}.pem" && \ - update-ca-trust; fi -USER 1001 diff --git a/deploy/ods-pipeline/charts/images/docker/Dockerfile.python-toolset b/deploy/ods-pipeline/charts/images/docker/Dockerfile.python-toolset deleted file mode 100644 index be3fc1c9..00000000 --- a/deploy/ods-pipeline/charts/images/docker/Dockerfile.python-toolset +++ /dev/null @@ -1,10 +0,0 @@ -ARG imageTag="latest" - -FROM ghcr.io/opendevstack/ods-pipeline/ods-python-toolset:$imageTag - -ARG privateCertServer -USER root -RUN if [ -n "${privateCertServer}" ]; then openssl s_client -showcerts -connect "${privateCertServer}" "/etc/pki/ca-trust/source/anchors/${privateCertServer%:*}.pem" && \ - update-ca-trust; fi -USER 1001 diff --git a/deploy/ods-pipeline/charts/images/docker/Dockerfile.sonar b/deploy/ods-pipeline/charts/images/docker/Dockerfile.sonar deleted file mode 100644 index 9f61a243..00000000 --- a/deploy/ods-pipeline/charts/images/docker/Dockerfile.sonar +++ /dev/null @@ -1,10 +0,0 @@ -ARG imageTag="latest" - -FROM ghcr.io/opendevstack/ods-pipeline/ods-sonar:$imageTag - -ARG privateCertServer -USER root -RUN if [ -n "${privateCertServer}" ]; then openssl s_client -showcerts -connect "${privateCertServer}" "/etc/pki/ca-trust/source/anchors/${privateCertServer%:*}.pem" && \ - update-ca-trust; fi -USER 1001 diff --git a/deploy/ods-pipeline/charts/images/docker/Dockerfile.start b/deploy/ods-pipeline/charts/images/docker/Dockerfile.start deleted file mode 100644 index 1d06d828..00000000 --- a/deploy/ods-pipeline/charts/images/docker/Dockerfile.start +++ /dev/null @@ -1,10 +0,0 @@ -ARG imageTag="latest" - -FROM ghcr.io/opendevstack/ods-pipeline/ods-start:$imageTag - -ARG privateCertServer -USER root -RUN if [ -n "${privateCertServer}" ]; then openssl s_client -showcerts -connect "${privateCertServer}" "/etc/pki/ca-trust/source/anchors/${privateCertServer%:*}.pem" && \ - update-ca-trust; fi -USER 1001 diff --git a/deploy/ods-pipeline/charts/images/templates/_helpers.tpl b/deploy/ods-pipeline/charts/images/templates/_helpers.tpl deleted file mode 100644 index 7ba5edc2..00000000 --- a/deploy/ods-pipeline/charts/images/templates/_helpers.tpl +++ /dev/null @@ -1,62 +0,0 @@ -{{/* -Expand the name of the chart. -*/}} -{{- define "chart.name" -}} -{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Create a default fully qualified app name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -If release name contains chart name it will be used as a full name. -*/}} -{{- define "chart.fullname" -}} -{{- if .Values.fullnameOverride }} -{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- $name := default .Chart.Name .Values.nameOverride }} -{{- if contains $name .Release.Name }} -{{- .Release.Name | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} -{{- end }} -{{- end }} -{{- end }} - -{{/* -Create chart name and version as used by the chart label. -*/}} -{{- define "chart.chart" -}} -{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Common labels -*/}} -{{- define "chart.labels" -}} -helm.sh/chart: {{ include "chart.chart" . }} -{{ include "chart.selectorLabels" . }} -{{- if .Chart.AppVersion }} -app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} -{{- end }} -app.kubernetes.io/managed-by: {{ .Release.Service }} -{{- end }} - -{{/* -Selector labels -*/}} -{{- define "chart.selectorLabels" -}} -app.kubernetes.io/name: {{ include "chart.name" . }} -app.kubernetes.io/instance: {{ .Release.Name }} -{{- end }} - -{{/* -Create the name of the service account to use -*/}} -{{- define "chart.serviceAccountName" -}} -{{- if .Values.serviceAccount.create }} -{{- default (include "chart.fullname" .) .Values.serviceAccount.name }} -{{- else }} -{{- default "default" .Values.serviceAccount.name }} -{{- end }} -{{- end }} diff --git a/deploy/ods-pipeline/charts/images/templates/bc-ods-finish.yaml b/deploy/ods-pipeline/charts/images/templates/bc-ods-finish.yaml deleted file mode 100644 index 3b45b1fa..00000000 --- a/deploy/ods-pipeline/charts/images/templates/bc-ods-finish.yaml +++ /dev/null @@ -1,28 +0,0 @@ -kind: BuildConfig -apiVersion: build.openshift.io/v1 -metadata: - name: ods-finish - labels: - {{- include "chart.labels" . | nindent 4}} -spec: - nodeSelector: null - output: - to: - kind: ImageStreamTag - name: 'ods-finish:{{.Values.global.imageTag | default .Chart.AppVersion}}' - resources: {} - successfulBuildsHistoryLimit: 5 - failedBuildsHistoryLimit: 5 - postCommit: {} - strategy: - type: Docker - dockerStrategy: - buildArgs: - - name: imageTag - value: '{{.Values.global.imageTag | default .Chart.AppVersion}}' - - name: privateCertServer - value: '{{.Values.privateCertServer}}' - source: - dockerfile: |- - {{- .Files.Get "docker/Dockerfile.finish" | nindent 6}} - runPolicy: Serial diff --git a/deploy/ods-pipeline/charts/images/templates/bc-ods-go-toolset.yaml b/deploy/ods-pipeline/charts/images/templates/bc-ods-go-toolset.yaml deleted file mode 100644 index ab0207cb..00000000 --- a/deploy/ods-pipeline/charts/images/templates/bc-ods-go-toolset.yaml +++ /dev/null @@ -1,30 +0,0 @@ -{{if or .Values.global.enabledTasks.buildGo .Values.goToolset}} -kind: BuildConfig -apiVersion: build.openshift.io/v1 -metadata: - name: ods-go-toolset - labels: - {{- include "chart.labels" . | nindent 4}} -spec: - nodeSelector: null - output: - to: - kind: ImageStreamTag - name: 'ods-go-toolset:{{.Values.global.imageTag | default .Chart.AppVersion}}' - resources: {} - successfulBuildsHistoryLimit: 5 - failedBuildsHistoryLimit: 5 - postCommit: {} - strategy: - type: Docker - dockerStrategy: - buildArgs: - - name: imageTag - value: '{{.Values.global.imageTag | default .Chart.AppVersion}}' - - name: privateCertServer - value: '{{.Values.privateCertServer}}' - source: - dockerfile: |- - {{- .Files.Get "docker/Dockerfile.go-toolset" | nindent 6}} - runPolicy: Serial -{{end}} diff --git a/deploy/ods-pipeline/charts/images/templates/bc-ods-gradle-toolset.yaml b/deploy/ods-pipeline/charts/images/templates/bc-ods-gradle-toolset.yaml deleted file mode 100644 index 5df23721..00000000 --- a/deploy/ods-pipeline/charts/images/templates/bc-ods-gradle-toolset.yaml +++ /dev/null @@ -1,30 +0,0 @@ -{{if or .Values.global.enabledTasks.buildGradle .Values.gradleToolset}} -kind: BuildConfig -apiVersion: build.openshift.io/v1 -metadata: - name: ods-gradle-toolset - labels: - {{- include "chart.labels" . | nindent 4}} -spec: - nodeSelector: null - output: - to: - kind: ImageStreamTag - name: 'ods-gradle-toolset:{{.Values.global.imageTag | default .Chart.AppVersion}}' - resources: {} - successfulBuildsHistoryLimit: 5 - failedBuildsHistoryLimit: 5 - postCommit: {} - strategy: - type: Docker - dockerStrategy: - buildArgs: - - name: imageTag - value: '{{.Values.global.imageTag | default .Chart.AppVersion}}' - - name: privateCertServer - value: '{{.Values.privateCertServer}}' - source: - dockerfile: |- - {{- .Files.Get "docker/Dockerfile.gradle-toolset" | nindent 6}} - runPolicy: Serial -{{end}} diff --git a/deploy/ods-pipeline/charts/images/templates/bc-ods-helm.yaml b/deploy/ods-pipeline/charts/images/templates/bc-ods-helm.yaml deleted file mode 100644 index dbccf014..00000000 --- a/deploy/ods-pipeline/charts/images/templates/bc-ods-helm.yaml +++ /dev/null @@ -1,30 +0,0 @@ -{{if or .Values.global.enabledTasks.deployHelm .Values.helm}} -kind: BuildConfig -apiVersion: build.openshift.io/v1 -metadata: - name: ods-helm - labels: - {{- include "chart.labels" . | nindent 4}} -spec: - nodeSelector: null - output: - to: - kind: ImageStreamTag - name: 'ods-helm:{{.Values.global.imageTag | default .Chart.AppVersion}}' - resources: {} - successfulBuildsHistoryLimit: 5 - failedBuildsHistoryLimit: 5 - postCommit: {} - strategy: - type: Docker - dockerStrategy: - buildArgs: - - name: imageTag - value: '{{.Values.global.imageTag | default .Chart.AppVersion}}' - - name: privateCertServer - value: '{{.Values.privateCertServer}}' - source: - dockerfile: |- - {{- .Files.Get "docker/Dockerfile.helm" | nindent 6}} - runPolicy: Serial -{{end}} diff --git a/deploy/ods-pipeline/charts/images/templates/bc-ods-node16-npm-toolset.yaml b/deploy/ods-pipeline/charts/images/templates/bc-ods-node16-npm-toolset.yaml deleted file mode 100644 index 92b9162d..00000000 --- a/deploy/ods-pipeline/charts/images/templates/bc-ods-node16-npm-toolset.yaml +++ /dev/null @@ -1,30 +0,0 @@ -{{if or .Values.global.enabledTasks.buildNPM .Values.node16NPMToolset}} -kind: BuildConfig -apiVersion: build.openshift.io/v1 -metadata: - name: ods-node16-npm-toolset - labels: - {{- include "chart.labels" . | nindent 4}} -spec: - nodeSelector: null - output: - to: - kind: ImageStreamTag - name: 'ods-node16-npm-toolset:{{.Values.global.imageTag | default .Chart.AppVersion}}' - resources: {} - successfulBuildsHistoryLimit: 5 - failedBuildsHistoryLimit: 5 - postCommit: {} - strategy: - type: Docker - dockerStrategy: - buildArgs: - - name: imageTag - value: '{{.Values.global.imageTag | default .Chart.AppVersion}}' - - name: privateCertServer - value: '{{.Values.privateCertServer}}' - source: - dockerfile: |- - {{- .Files.Get "docker/Dockerfile.node16-npm-toolset" | nindent 6}} - runPolicy: Serial -{{end}} diff --git a/deploy/ods-pipeline/charts/images/templates/bc-ods-node18-npm-toolset.yaml b/deploy/ods-pipeline/charts/images/templates/bc-ods-node18-npm-toolset.yaml deleted file mode 100644 index 0489756c..00000000 --- a/deploy/ods-pipeline/charts/images/templates/bc-ods-node18-npm-toolset.yaml +++ /dev/null @@ -1,30 +0,0 @@ -{{if or .Values.global.enabledTasks.buildNPM .Values.node18NPMToolset}} -kind: BuildConfig -apiVersion: build.openshift.io/v1 -metadata: - name: ods-node18-npm-toolset - labels: - {{- include "chart.labels" . | nindent 4}} -spec: - nodeSelector: null - output: - to: - kind: ImageStreamTag - name: 'ods-node18-npm-toolset:{{.Values.global.imageTag | default .Chart.AppVersion}}' - resources: {} - successfulBuildsHistoryLimit: 5 - failedBuildsHistoryLimit: 5 - postCommit: {} - strategy: - type: Docker - dockerStrategy: - buildArgs: - - name: imageTag - value: '{{.Values.global.imageTag | default .Chart.AppVersion}}' - - name: privateCertServer - value: '{{.Values.privateCertServer}}' - source: - dockerfile: |- - {{- .Files.Get "docker/Dockerfile.node18-npm-toolset" | nindent 6}} - runPolicy: Serial -{{end}} diff --git a/deploy/ods-pipeline/charts/images/templates/bc-ods-package-image.yaml b/deploy/ods-pipeline/charts/images/templates/bc-ods-package-image.yaml deleted file mode 100644 index 94716fc7..00000000 --- a/deploy/ods-pipeline/charts/images/templates/bc-ods-package-image.yaml +++ /dev/null @@ -1,32 +0,0 @@ -{{if or .Values.global.enabledTasks.packageImage .Values.packageImage}} -kind: BuildConfig -apiVersion: build.openshift.io/v1 -metadata: - name: ods-package-image - labels: - {{- include "chart.labels" . | nindent 4}} -spec: - nodeSelector: null - output: - to: - kind: ImageStreamTag - name: 'ods-package-image:{{.Values.global.imageTag | default .Chart.AppVersion}}' - resources: {} - successfulBuildsHistoryLimit: 5 - failedBuildsHistoryLimit: 5 - postCommit: {} - strategy: - type: Docker - dockerStrategy: - buildArgs: - - name: imageTag - value: '{{.Values.global.imageTag | default .Chart.AppVersion}}' - - name: privateCertServer - value: '{{.Values.privateCertServer}}' - - name: aquasecScannerUrl - value: '{{.Values.aquasecScannerUrl}}' - source: - dockerfile: |- - {{- .Files.Get "docker/Dockerfile.package-image" | nindent 6}} - runPolicy: Serial -{{end}} diff --git a/deploy/ods-pipeline/charts/images/templates/bc-ods-pipeline-manager.yaml b/deploy/ods-pipeline/charts/images/templates/bc-ods-pipeline-manager.yaml deleted file mode 100644 index 1736a639..00000000 --- a/deploy/ods-pipeline/charts/images/templates/bc-ods-pipeline-manager.yaml +++ /dev/null @@ -1,28 +0,0 @@ -kind: BuildConfig -apiVersion: build.openshift.io/v1 -metadata: - name: ods-pipeline-manager - labels: - {{- include "chart.labels" . | nindent 4}} -spec: - nodeSelector: null - output: - to: - kind: ImageStreamTag - name: 'ods-pipeline-manager:{{.Values.global.imageTag | default .Chart.AppVersion}}' - resources: {} - successfulBuildsHistoryLimit: 5 - failedBuildsHistoryLimit: 5 - postCommit: {} - strategy: - type: Docker - dockerStrategy: - buildArgs: - - name: imageTag - value: '{{.Values.global.imageTag | default .Chart.AppVersion}}' - - name: privateCertServer - value: '{{.Values.privateCertServer}}' - source: - dockerfile: |- - {{- .Files.Get "docker/Dockerfile.pipeline-manager" | nindent 6}} - runPolicy: Serial diff --git a/deploy/ods-pipeline/charts/images/templates/bc-ods-python-toolset.yaml b/deploy/ods-pipeline/charts/images/templates/bc-ods-python-toolset.yaml deleted file mode 100644 index a281cb04..00000000 --- a/deploy/ods-pipeline/charts/images/templates/bc-ods-python-toolset.yaml +++ /dev/null @@ -1,30 +0,0 @@ -{{if or .Values.global.enabledTasks.buildPython .Values.pythonToolset }} -kind: BuildConfig -apiVersion: build.openshift.io/v1 -metadata: - name: ods-python-toolset - labels: - {{- include "chart.labels" . | nindent 4}} -spec: - nodeSelector: null - output: - to: - kind: ImageStreamTag - name: 'ods-python-toolset:{{.Values.global.imageTag | default .Chart.AppVersion}}' - resources: {} - successfulBuildsHistoryLimit: 5 - failedBuildsHistoryLimit: 5 - postCommit: {} - strategy: - type: Docker - dockerStrategy: - buildArgs: - - name: imageTag - value: '{{.Values.global.imageTag | default .Chart.AppVersion}}' - - name: privateCertServer - value: '{{.Values.privateCertServer}}' - source: - dockerfile: |- - {{- .Files.Get "docker/Dockerfile.python-toolset" | nindent 6}} - runPolicy: Serial -{{end}} diff --git a/deploy/ods-pipeline/charts/images/templates/bc-ods-sonar.yaml b/deploy/ods-pipeline/charts/images/templates/bc-ods-sonar.yaml deleted file mode 100644 index c0e9ba65..00000000 --- a/deploy/ods-pipeline/charts/images/templates/bc-ods-sonar.yaml +++ /dev/null @@ -1,28 +0,0 @@ -kind: BuildConfig -apiVersion: build.openshift.io/v1 -metadata: - name: ods-sonar - labels: - {{- include "chart.labels" . | nindent 4}} -spec: - nodeSelector: null - output: - to: - kind: ImageStreamTag - name: 'ods-sonar:{{.Values.global.imageTag | default .Chart.AppVersion}}' - resources: {} - successfulBuildsHistoryLimit: 5 - failedBuildsHistoryLimit: 5 - postCommit: {} - strategy: - type: Docker - dockerStrategy: - buildArgs: - - name: imageTag - value: '{{.Values.global.imageTag | default .Chart.AppVersion}}' - - name: privateCertServer - value: '{{.Values.privateCertServer}}' - source: - dockerfile: |- - {{- .Files.Get "docker/Dockerfile.sonar" | nindent 6}} - runPolicy: Serial diff --git a/deploy/ods-pipeline/charts/images/templates/bc-ods-start.yaml b/deploy/ods-pipeline/charts/images/templates/bc-ods-start.yaml deleted file mode 100644 index 72668b51..00000000 --- a/deploy/ods-pipeline/charts/images/templates/bc-ods-start.yaml +++ /dev/null @@ -1,28 +0,0 @@ -kind: BuildConfig -apiVersion: build.openshift.io/v1 -metadata: - name: ods-start - labels: - {{- include "chart.labels" . | nindent 4}} -spec: - nodeSelector: null - output: - to: - kind: ImageStreamTag - name: 'ods-start:{{.Values.global.imageTag | default .Chart.AppVersion}}' - resources: {} - successfulBuildsHistoryLimit: 5 - failedBuildsHistoryLimit: 5 - postCommit: {} - strategy: - type: Docker - dockerStrategy: - buildArgs: - - name: imageTag - value: '{{.Values.global.imageTag | default .Chart.AppVersion}}' - - name: privateCertServer - value: '{{.Values.privateCertServer}}' - source: - dockerfile: |- - {{- .Files.Get "docker/Dockerfile.start" | nindent 6}} - runPolicy: Serial diff --git a/deploy/ods-pipeline/charts/images/templates/is-ods-finish.yaml b/deploy/ods-pipeline/charts/images/templates/is-ods-finish.yaml deleted file mode 100644 index c42ee665..00000000 --- a/deploy/ods-pipeline/charts/images/templates/is-ods-finish.yaml +++ /dev/null @@ -1,8 +0,0 @@ -apiVersion: image.openshift.io/v1 -kind: ImageStream -metadata: - name: ods-finish - labels: - {{- include "chart.labels" . | nindent 4}} - annotations: - "helm.sh/resource-policy": keep diff --git a/deploy/ods-pipeline/charts/images/templates/is-ods-go-toolset.yaml b/deploy/ods-pipeline/charts/images/templates/is-ods-go-toolset.yaml deleted file mode 100644 index e8a36b04..00000000 --- a/deploy/ods-pipeline/charts/images/templates/is-ods-go-toolset.yaml +++ /dev/null @@ -1,10 +0,0 @@ -{{if or .Values.global.enabledTasks.buildGo .Values.goToolset}} -apiVersion: image.openshift.io/v1 -kind: ImageStream -metadata: - name: ods-go-toolset - labels: - {{- include "chart.labels" . | nindent 4}} - annotations: - "helm.sh/resource-policy": keep -{{end}} diff --git a/deploy/ods-pipeline/charts/images/templates/is-ods-gradle-toolset.yaml b/deploy/ods-pipeline/charts/images/templates/is-ods-gradle-toolset.yaml deleted file mode 100644 index 6410ac7d..00000000 --- a/deploy/ods-pipeline/charts/images/templates/is-ods-gradle-toolset.yaml +++ /dev/null @@ -1,10 +0,0 @@ -{{if or .Values.global.enabledTasks.buildGradle .Values.gradleToolset}} -apiVersion: image.openshift.io/v1 -kind: ImageStream -metadata: - name: ods-gradle-toolset - labels: - {{- include "chart.labels" . | nindent 4}} - annotations: - "helm.sh/resource-policy": keep -{{end}} diff --git a/deploy/ods-pipeline/charts/images/templates/is-ods-helm.yaml b/deploy/ods-pipeline/charts/images/templates/is-ods-helm.yaml deleted file mode 100644 index 8360dd0e..00000000 --- a/deploy/ods-pipeline/charts/images/templates/is-ods-helm.yaml +++ /dev/null @@ -1,10 +0,0 @@ -{{if or .Values.global.enabledTasks.deployHelm .Values.helm}} -apiVersion: image.openshift.io/v1 -kind: ImageStream -metadata: - name: ods-helm - labels: - {{- include "chart.labels" . | nindent 4}} - annotations: - "helm.sh/resource-policy": keep -{{end}} diff --git a/deploy/ods-pipeline/charts/images/templates/is-ods-node16-npm-toolset.yaml b/deploy/ods-pipeline/charts/images/templates/is-ods-node16-npm-toolset.yaml deleted file mode 100644 index 5b0bbaeb..00000000 --- a/deploy/ods-pipeline/charts/images/templates/is-ods-node16-npm-toolset.yaml +++ /dev/null @@ -1,10 +0,0 @@ -{{if or .Values.global.enabledTasks.buildNPM .Values.node16NPMToolset}} -apiVersion: image.openshift.io/v1 -kind: ImageStream -metadata: - name: ods-node16-npm-toolset - labels: - {{- include "chart.labels" . | nindent 4}} - annotations: - "helm.sh/resource-policy": keep -{{end}} diff --git a/deploy/ods-pipeline/charts/images/templates/is-ods-node18-npm-toolset.yaml b/deploy/ods-pipeline/charts/images/templates/is-ods-node18-npm-toolset.yaml deleted file mode 100644 index 43be1863..00000000 --- a/deploy/ods-pipeline/charts/images/templates/is-ods-node18-npm-toolset.yaml +++ /dev/null @@ -1,10 +0,0 @@ -{{if or .Values.global.enabledTasks.buildNPM .Values.node18NPMToolset}} -apiVersion: image.openshift.io/v1 -kind: ImageStream -metadata: - name: ods-node18-npm-toolset - labels: - {{- include "chart.labels" . | nindent 4}} - annotations: - "helm.sh/resource-policy": keep -{{end}} diff --git a/deploy/ods-pipeline/charts/images/templates/is-ods-package-image.yaml b/deploy/ods-pipeline/charts/images/templates/is-ods-package-image.yaml deleted file mode 100644 index a88e5d8b..00000000 --- a/deploy/ods-pipeline/charts/images/templates/is-ods-package-image.yaml +++ /dev/null @@ -1,10 +0,0 @@ -{{if or .Values.global.enabledTasks.packageImage .Values.packageImage}} -apiVersion: image.openshift.io/v1 -kind: ImageStream -metadata: - name: ods-package-image - labels: - {{- include "chart.labels" . | nindent 4}} - annotations: - "helm.sh/resource-policy": keep -{{end}} diff --git a/deploy/ods-pipeline/charts/images/templates/is-ods-pipeline-manager.yaml b/deploy/ods-pipeline/charts/images/templates/is-ods-pipeline-manager.yaml deleted file mode 100644 index e5552010..00000000 --- a/deploy/ods-pipeline/charts/images/templates/is-ods-pipeline-manager.yaml +++ /dev/null @@ -1,8 +0,0 @@ -apiVersion: image.openshift.io/v1 -kind: ImageStream -metadata: - name: ods-pipeline-manager - labels: - {{- include "chart.labels" . | nindent 4}} - annotations: - "helm.sh/resource-policy": keep diff --git a/deploy/ods-pipeline/charts/images/templates/is-ods-python-toolset.yaml b/deploy/ods-pipeline/charts/images/templates/is-ods-python-toolset.yaml deleted file mode 100644 index 3a42d50e..00000000 --- a/deploy/ods-pipeline/charts/images/templates/is-ods-python-toolset.yaml +++ /dev/null @@ -1,10 +0,0 @@ -{{if or .Values.global.enabledTasks.buildPython .Values.pythonToolset }} -apiVersion: image.openshift.io/v1 -kind: ImageStream -metadata: - name: ods-python-toolset - labels: - {{- include "chart.labels" . | nindent 4}} - annotations: - "helm.sh/resource-policy": keep -{{end}} diff --git a/deploy/ods-pipeline/charts/images/templates/is-ods-sonar.yaml b/deploy/ods-pipeline/charts/images/templates/is-ods-sonar.yaml deleted file mode 100644 index cff36d38..00000000 --- a/deploy/ods-pipeline/charts/images/templates/is-ods-sonar.yaml +++ /dev/null @@ -1,8 +0,0 @@ -apiVersion: image.openshift.io/v1 -kind: ImageStream -metadata: - name: ods-sonar - labels: - {{- include "chart.labels" . | nindent 4}} - annotations: - "helm.sh/resource-policy": keep diff --git a/deploy/ods-pipeline/charts/images/templates/is-ods-start.yaml b/deploy/ods-pipeline/charts/images/templates/is-ods-start.yaml deleted file mode 100644 index 9d9b0445..00000000 --- a/deploy/ods-pipeline/charts/images/templates/is-ods-start.yaml +++ /dev/null @@ -1,8 +0,0 @@ -apiVersion: image.openshift.io/v1 -kind: ImageStream -metadata: - name: ods-start - labels: - {{- include "chart.labels" . | nindent 4}} - annotations: - "helm.sh/resource-policy": keep diff --git a/deploy/ods-pipeline/charts/images/templates/job-start-builds.yaml b/deploy/ods-pipeline/charts/images/templates/job-start-builds.yaml deleted file mode 100644 index 6a286bdb..00000000 --- a/deploy/ods-pipeline/charts/images/templates/job-start-builds.yaml +++ /dev/null @@ -1,20 +0,0 @@ -{{if default true .Values.autoBuild}} -apiVersion: batch/v1 -kind: Job -metadata: - name: ods-start-builds - annotations: - "helm.sh/hook": post-install,post-upgrade - "helm.sh/hook-weight": "1" - "helm.sh/hook-delete-policy": hook-succeeded,hook-failed -spec: - template: - spec: - serviceAccountName: pipeline - restartPolicy: Never - containers: - - name: post-upgrade-job - image: '{{.Values.autoBuildImage | default "quay.io/openshift/origin-cli:4.10"}}' - command: ["/bin/sh","-c"] - args: ["set -e; oc get bc -l=app.kubernetes.io/name=ods-pipeline -o=name | xargs -I % sh -c 'oc start-build %'"] -{{end}} diff --git a/deploy/ods-pipeline/charts/images/values.yaml b/deploy/ods-pipeline/charts/images/values.yaml deleted file mode 100644 index e8d0e03d..00000000 --- a/deploy/ods-pipeline/charts/images/values.yaml +++ /dev/null @@ -1,2 +0,0 @@ -# override name to be consistent with previous, separate chart naming convention(s) -nameOverride: ods-pipeline diff --git a/deploy/ods-pipeline/charts/setup/templates/deployment.yaml b/deploy/ods-pipeline/charts/setup/templates/deployment.yaml index ec1c8747..2f9d4488 100644 --- a/deploy/ods-pipeline/charts/setup/templates/deployment.yaml +++ b/deploy/ods-pipeline/charts/setup/templates/deployment.yaml @@ -17,8 +17,7 @@ spec: containers: - name: pipeline-manager securityContext: {} - image: "{{.Values.pipelineManager.image.registry}}/{{.Values.pipelineManager.image.namespace | default .Release.Namespace}}/{{.Values.pipelineManager.image.repository | default .Chart.Name}}:{{.Values.pipelineManager.image.tag | default .Chart.AppVersion}}" - imagePullPolicy: {{.Values.pipelineManager.image.pullPolicy}} + image: "{{.Values.pipelineManager.imageRepository}}/ods-pipeline-manager:{{.Values.pipelineManager.image.tag | default .Chart.AppVersion}}" ports: - name: http containerPort: 8080 diff --git a/deploy/ods-pipeline/charts/tasks/templates/_sonar-step.tpl b/deploy/ods-pipeline/charts/tasks/templates/_sonar-step.tpl index 5f70d0d7..773462a2 100644 --- a/deploy/ods-pipeline/charts/tasks/templates/_sonar-step.tpl +++ b/deploy/ods-pipeline/charts/tasks/templates/_sonar-step.tpl @@ -1,7 +1,7 @@ {{- define "sonar-step"}} - name: scan-with-sonar # Image is built from build/package/Dockerfile.sonar. - image: '{{.Values.registry}}/{{default .Release.Namespace .Values.namespace}}/ods-sonar:{{.Values.global.imageTag | default .Chart.AppVersion}}' + image: '{{.Values.imageRepository}}/ods-sonar:{{.Values.global.imageTag | default .Chart.AppVersion}}' env: - name: HOME value: '/tekton/home' @@ -31,10 +31,22 @@ echo "Skipping SonarQube analysis" else mkdir -p .ods/artifacts/sonarqube-analysis + + truststore="${JAVA_HOME}/lib/security/cacerts" + if [ -f /etc/ssl/certs/private-cert.pem ]; then + truststore=".ods-cache/truststore/cacerts" + fi + configure-truststore --dest-store "${truststore}" # sonar is built from cmd/sonar/main.go. sonar \ -working-dir=$(params.working-dir) \ - -quality-gate=$(params.sonar-quality-gate) + -quality-gate=$(params.sonar-quality-gate) \ + -truststore "${truststore}" fi + volumeMounts: + - mountPath: /etc/ssl/certs/private-cert.pem + name: private-cert + readOnly: true + subPath: tls.crt workingDir: $(workspaces.source.path) {{- end}} diff --git a/deploy/ods-pipeline/charts/tasks/templates/task-ods-build-go.yaml b/deploy/ods-pipeline/charts/tasks/templates/task-ods-build-go.yaml index 2e055bf7..a09302d5 100644 --- a/deploy/ods-pipeline/charts/tasks/templates/task-ods-build-go.yaml +++ b/deploy/ods-pipeline/charts/tasks/templates/task-ods-build-go.yaml @@ -117,7 +117,7 @@ spec: steps: - name: build-go-binary # Image is built from build/package/Dockerfile.go-toolset. - image: '{{.Values.registry}}/{{default .Release.Namespace .Values.namespace}}/ods-go-toolset:{{.Values.global.imageTag | default .Chart.AppVersion}}' + image: '{{.Values.imageRepository}}/ods-go-toolset:{{.Values.global.imageTag | default .Chart.AppVersion}}' env: - name: HOME value: '/tekton/home' @@ -167,8 +167,18 @@ spec: --output-dir=$(params.output-dir) \ --debug=${DEBUG} fi + volumeMounts: + - mountPath: /etc/ssl/certs/private-cert.pem + name: private-cert + readOnly: true + subPath: tls.crt workingDir: $(workspaces.source.path) {{- include "sonar-step" . | indent 4}} + volumes: + - name: private-cert + secret: + secretName: ods-private-cert + optional: true workspaces: - name: source {{end}} diff --git a/deploy/ods-pipeline/charts/tasks/templates/task-ods-build-gradle.yaml b/deploy/ods-pipeline/charts/tasks/templates/task-ods-build-gradle.yaml index 2ed8fe54..4443df1c 100644 --- a/deploy/ods-pipeline/charts/tasks/templates/task-ods-build-gradle.yaml +++ b/deploy/ods-pipeline/charts/tasks/templates/task-ods-build-gradle.yaml @@ -176,7 +176,7 @@ spec: steps: - name: build-gradle-binary # Image is built from build/package/Dockerfile.gradle-toolset. - image: '{{.Values.registry}}/{{default .Release.Namespace .Values.namespace}}/ods-gradle-toolset:{{.Values.global.imageTag | default .Chart.AppVersion}}' + image: '{{.Values.imageRepository}}/ods-gradle-toolset:{{.Values.global.imageTag | default .Chart.AppVersion}}' env: - name: DEBUG valueFrom: @@ -240,8 +240,18 @@ spec: --output-dir=$(params.output-dir) \ --debug=${DEBUG} fi + volumeMounts: + - mountPath: /etc/ssl/certs/private-cert.pem + name: private-cert + readOnly: true + subPath: tls.crt workingDir: $(workspaces.source.path) {{- include "sonar-step" . | indent 4}} + volumes: + - name: private-cert + secret: + secretName: ods-private-cert + optional: true workspaces: - name: source {{end}} diff --git a/deploy/ods-pipeline/charts/tasks/templates/task-ods-build-npm.yaml b/deploy/ods-pipeline/charts/tasks/templates/task-ods-build-npm.yaml index 1fd98538..6a3723a9 100644 --- a/deploy/ods-pipeline/charts/tasks/templates/task-ods-build-npm.yaml +++ b/deploy/ods-pipeline/charts/tasks/templates/task-ods-build-npm.yaml @@ -134,7 +134,7 @@ spec: steps: - name: build-npm # Image is built from build/package/Dockerfile.node-npm-toolset. - image: '{{.Values.registry}}/{{default .Release.Namespace .Values.namespace}}/ods-node$(params.node-version)-npm-toolset:{{.Values.global.imageTag | default .Chart.AppVersion}}' + image: '{{.Values.imageRepository}}/ods-node$(params.node-version)-npm-toolset:{{.Values.global.imageTag | default .Chart.AppVersion}}' env: - name: HOME value: '/tekton/home' @@ -197,8 +197,18 @@ spec: --output-dir=$(params.output-dir) \ --debug=${DEBUG} fi + volumeMounts: + - mountPath: /etc/ssl/certs/private-cert.pem + name: private-cert + readOnly: true + subPath: tls.crt workingDir: $(workspaces.source.path) {{- include "sonar-step" . | indent 4}} + volumes: + - name: private-cert + secret: + secretName: ods-private-cert + optional: true workspaces: - name: source {{end}} diff --git a/deploy/ods-pipeline/charts/tasks/templates/task-ods-build-python.yaml b/deploy/ods-pipeline/charts/tasks/templates/task-ods-build-python.yaml index 2fff994f..1f389c9c 100644 --- a/deploy/ods-pipeline/charts/tasks/templates/task-ods-build-python.yaml +++ b/deploy/ods-pipeline/charts/tasks/templates/task-ods-build-python.yaml @@ -96,7 +96,7 @@ spec: steps: - name: build-python # Image is built from build/package/Dockerfile.python-toolset. - image: '{{.Values.registry}}/{{default .Release.Namespace .Values.namespace}}/ods-python-toolset:{{.Values.global.imageTag | default .Chart.AppVersion}}' + image: '{{.Values.imageRepository}}/ods-python-toolset:{{.Values.global.imageTag | default .Chart.AppVersion}}' env: - name: HOME value: '/tekton/home' @@ -159,8 +159,18 @@ spec: --output-dir=$(params.output-dir) \ --debug=${DEBUG} fi + volumeMounts: + - mountPath: /etc/ssl/certs/private-cert.pem + name: private-cert + readOnly: true + subPath: tls.crt workingDir: $(workspaces.source.path) {{- include "sonar-step" . | indent 4}} + volumes: + - name: private-cert + secret: + secretName: ods-private-cert + optional: true workspaces: - name: source {{end}} diff --git a/deploy/ods-pipeline/charts/tasks/templates/task-ods-deploy-helm.yaml b/deploy/ods-pipeline/charts/tasks/templates/task-ods-deploy-helm.yaml index 42207d4a..36901184 100644 --- a/deploy/ods-pipeline/charts/tasks/templates/task-ods-deploy-helm.yaml +++ b/deploy/ods-pipeline/charts/tasks/templates/task-ods-deploy-helm.yaml @@ -101,7 +101,7 @@ spec: steps: - name: helm-upgrade-from-repo # Image is built from build/package/Dockerfile.helm. - image: '{{.Values.registry}}/{{default .Release.Namespace .Values.namespace}}/ods-helm:{{.Values.global.imageTag | default .Chart.AppVersion}}' + image: '{{.Values.imageRepository}}/ods-helm:{{.Values.global.imageTag | default .Chart.AppVersion}}' env: - name: DEBUG valueFrom: @@ -119,7 +119,17 @@ spec: -diff-flags="$(params.diff-flags)" \ -upgrade-flags="$(params.upgrade-flags)" \ -age-key-secret=$(params.age-key-secret) + volumeMounts: + - mountPath: /etc/ssl/certs/private-cert.pem + name: private-cert + readOnly: true + subPath: tls.crt workingDir: $(workspaces.source.path) + volumes: + - name: private-cert + secret: + secretName: ods-private-cert + optional: true workspaces: - name: source {{end}} diff --git a/deploy/ods-pipeline/charts/tasks/templates/task-ods-finish.yaml b/deploy/ods-pipeline/charts/tasks/templates/task-ods-finish.yaml index 7a93d691..af60ab55 100644 --- a/deploy/ods-pipeline/charts/tasks/templates/task-ods-finish.yaml +++ b/deploy/ods-pipeline/charts/tasks/templates/task-ods-finish.yaml @@ -34,7 +34,7 @@ spec: steps: - name: ods-finish # Image is built from build/package/Dockerfile.finish. - image: '{{.Values.registry}}/{{default .Release.Namespace .Values.namespace}}/ods-finish:{{.Values.global.imageTag | default .Chart.AppVersion}}' + image: '{{.Values.imageRepository}}/ods-finish:{{.Values.global.imageTag | default .Chart.AppVersion}}' env: - name: HOME value: '/tekton/home' @@ -84,14 +84,23 @@ spec: key: debug name: ods-pipeline resources: {} - workingDir: $(workspaces.source.path) script: | # ods-finish is built from cmd/finish/main.go. ods-finish \ -pipeline-run-name=$(params.pipeline-run-name) \ -aggregate-tasks-status=$(params.aggregate-tasks-status) - + volumeMounts: + - mountPath: /etc/ssl/certs/private-cert.pem + name: private-cert + readOnly: true + subPath: tls.crt + workingDir: $(workspaces.source.path) + volumes: + - name: private-cert + secret: + secretName: ods-private-cert + optional: true workspaces: - description: The git repo will be present onto the volume backing this workspace name: source diff --git a/deploy/ods-pipeline/charts/tasks/templates/task-ods-package-image.yaml b/deploy/ods-pipeline/charts/tasks/templates/task-ods-package-image.yaml index 0c7e59c8..f344d9fe 100644 --- a/deploy/ods-pipeline/charts/tasks/templates/task-ods-package-image.yaml +++ b/deploy/ods-pipeline/charts/tasks/templates/task-ods-package-image.yaml @@ -48,7 +48,7 @@ spec: - name: registry description: Image registry to push image to. type: string - default: '{{default .Values.registry .Values.pushRegistry}}' + default: '{{default .Values.pushRegistry}}' - name: image-stream description: Reference of the image stream buildah will produce. If not set, the value of `.ods/component` is used. type: string @@ -91,7 +91,7 @@ spec: steps: - name: build-and-push-image # Image is built from build/package/Dockerfile.package-image. - image: '{{.Values.registry}}/{{default .Release.Namespace .Values.namespace}}/ods-package-image:{{.Values.global.imageTag | default .Chart.AppVersion}}' + image: '{{.Values.imageRepository}}/ods-package-image:{{.Values.global.imageTag | default .Chart.AppVersion}}' env: - name: HOME value: '/tekton/home' @@ -125,6 +125,11 @@ spec: secretKeyRef: key: password name: ods-aqua-auth + - name: AQUA_SCANNER_URL + valueFrom: + secretKeyRef: + key: secret + name: ods-aqua-scanner-url - name: NEXUS_URL valueFrom: configMapKeyRef: @@ -148,6 +153,10 @@ spec: resources: {} script: | + download-aqua-scanner \ + --aqua-scanner-url=${AQUA_SCANNER_URL} \ + $(case ${DEBUG} in (true) printf -- '--debug'; esac) + # ods-package-image is built from cmd/package-image/main.go. ods-package-image \ -image-stream=$(params.image-stream) \ @@ -163,10 +172,18 @@ spec: volumeMounts: - mountPath: /var/lib/containers name: varlibcontainers + - mountPath: /etc/ssl/certs/private-cert.pem + name: private-cert + readOnly: true + subPath: tls.crt workingDir: $(workspaces.source.path) volumes: - emptyDir: {} name: varlibcontainers + - name: private-cert + secret: + secretName: ods-private-cert + optional: true workspaces: - name: source {{end}} diff --git a/deploy/ods-pipeline/charts/tasks/templates/task-ods-start.yaml b/deploy/ods-pipeline/charts/tasks/templates/task-ods-start.yaml index caed39de..57599932 100644 --- a/deploy/ods-pipeline/charts/tasks/templates/task-ods-start.yaml +++ b/deploy/ods-pipeline/charts/tasks/templates/task-ods-start.yaml @@ -124,7 +124,7 @@ spec: steps: - name: ods-start # Image is built from build/package/Dockerfile.start. - image: '{{.Values.registry}}/{{default .Release.Namespace .Values.namespace}}/ods-start:{{.Values.global.imageTag | default .Chart.AppVersion}}' + image: '{{.Values.imageRepository}}/ods-start:{{.Values.global.imageTag | default .Chart.AppVersion}}' env: - name: HOME value: '/tekton/home' @@ -174,9 +174,12 @@ spec: key: debug name: ods-pipeline resources: {} - workingDir: $(workspaces.source.path) script: | + if [ -f /etc/ssl/certs/private-cert.pem ]; then + git config --global http.sslCAInfo /etc/ssl/certs/private-cert.pem + fi + # ods-start is built from cmd/start/main.go. ods-start \ -project=$(params.project) \ @@ -198,7 +201,17 @@ spec: cp .ods/git-commit-sha $(results.commit.path) echo -n "$(params.url)" > $(results.url.path) - + volumeMounts: + - mountPath: /etc/ssl/certs/private-cert.pem + name: private-cert + readOnly: true + subPath: tls.crt + workingDir: $(workspaces.source.path) + volumes: + - name: private-cert + secret: + secretName: ods-private-cert + optional: true workspaces: - description: The git repo will be cloned onto the volume backing this workspace name: source diff --git a/deploy/ods-pipeline/charts/tasks/values.docs.yaml b/deploy/ods-pipeline/charts/tasks/values.docs.yaml index 585202c0..1640798d 100644 --- a/deploy/ods-pipeline/charts/tasks/values.docs.yaml +++ b/deploy/ods-pipeline/charts/tasks/values.docs.yaml @@ -9,7 +9,3 @@ global: buildNPM: true packageImage: true deployHelm: true - -registry: image-registry.openshift-image-registry.svc:5000 -namespace: ods -pushRegistry: image-registry.openshift-image-registry.svc:5000 diff --git a/deploy/ods-pipeline/charts/tasks/values.yaml b/deploy/ods-pipeline/charts/tasks/values.yaml index e8d0e03d..47b0c55b 100644 --- a/deploy/ods-pipeline/charts/tasks/values.yaml +++ b/deploy/ods-pipeline/charts/tasks/values.yaml @@ -1,2 +1,5 @@ # override name to be consistent with previous, separate chart naming convention(s) nameOverride: ods-pipeline + +imageRepository: ghcr.io/opendevstack/ods-pipeline +pushRegistry: image-registry.openshift-image-registry.svc:5000 diff --git a/deploy/ods-pipeline/values.kind.yaml b/deploy/ods-pipeline/values.kind.yaml index 3338ab51..e214118c 100644 --- a/deploy/ods-pipeline/values.kind.yaml +++ b/deploy/ods-pipeline/values.kind.yaml @@ -16,12 +16,9 @@ setup: tag: "latest" tasks: - # To test with the latest public ods-pipeline images, set global.imageTag to 'latest' and use: - # registry: ghcr.io - # namespace: opendevstack/ods-pipeline - registry: localhost:5000 - namespace: ods - pushRegistry: kind-registry.kind:5000 + # Image repository to pull task images from. + # To test with the latest public ods-pipeline images, set + # global.imageTag to 'latest' and use: 'ghcr.io/opendevstack/ods-pipeline'. + imageRepository: localhost:5000/ods -images: - enabled: false + pushRegistry: kind-registry.kind:5000 diff --git a/deploy/ods-pipeline/values.yaml b/deploy/ods-pipeline/values.yaml index 71e443db..e77a476c 100644 --- a/deploy/ods-pipeline/values.yaml +++ b/deploy/ods-pipeline/values.yaml @@ -20,22 +20,6 @@ global: deployHelm: true -# ####################################### # -# IMAGES CHART CONFIG # -# ####################################### # -images: - # enable chart containing Openshift image streams and build configs - enabled: true - # When using a local/private CA, specify the server (incl. port!) to pull the root CA cert from. - # privateCertServer: 'example.com:443' - # if needed, enable images even though related tasks are disabled, e.g. - # pythonToolset: true - # autoBuild controls whether builds for all BuildConfig resources are started - # automatically after a successful Helm upgrade. autoBuild is enabled by default. - # autoBuild: true - # autoBuildImage allows to override the image used for starting builds. - # autoBuildImage: 'quay.io/openshift/origin-cli:4.9' - # ####################################### # # SETUP CHART CONFIG # @@ -164,21 +148,8 @@ setup: storageSize: '5Gi' # Number of replicas to run for the pipeline manager. replicaCount: 1 - image: - # Image registry from which to pull the pipeline manager container image. - registry: 'image-registry.openshift-image-registry.svc:5000' - # Namespace from which to pull the pipeline manager container image. - # If not given, the image is pulled from the release namespace. - # namespace: 'ods' - # Repository (ImageStream) from which to pull the pipeline manager - # container image. - # If not given, the image name equals the chart name. - repository: 'ods-pipeline-manager' - # Pull policy. - pullPolicy: 'Always' - # Image tag to pull. - # If not given, defaults to the chart appVersion. - # tag: 'latest' + # Repository from which to pull the pipeline manager container image. + imageRepository: ghcr.io/opendevstack/ods-pipeline # Deployment pod resources. Typically these settings should not need to change. resources: limits: @@ -197,21 +168,14 @@ tasks: # enable task definition chart enabled: true - # Registry to pull task images from. - registry: image-registry.openshift-image-registry.svc:5000 - - # ImageStream to pull task images from. - # If images are not located within OpenShift image streams, this maps to - # the organisation under which the images are hosted. - # If not set, defaults to the Helm release namespace. - # namespace: ods + # Image repository to pull task images from. + imageRepository: ghcr.io/opendevstack/ods-pipeline # Custom task prefix (defaults to "ods") # taskPrefix: "foo" # Registry to push images to from ods-package-image task. - # If not set, defaults to the value of "registry". - # pushRegistry: image-registry.openshift-image-registry.svc:5000 + pushRegistry: image-registry.openshift-image-registry.svc:5000 # To define build task specific sidecars and quotas, add resources/sidecar section(s) per task, # e.g. diff --git a/deploy/values.yaml.tmpl b/deploy/values.yaml.tmpl index 1340ec55..bbc1e4b6 100644 --- a/deploy/values.yaml.tmpl +++ b/deploy/values.yaml.tmpl @@ -51,10 +51,3 @@ setup: storageProvisioner: 'kubernetes.io/aws-ebs' # Storage class. On AWS backed clusters, use 'gp2'. storageClassName: 'gp2' - -images: - # URL from which to download aqua-scanner binary. The URL may need to contain basic authentication. - # Make sure that username/password are URL-encoded and that the version matches - # your Aqua server version. - # If you do not want to use Aqua, leave this empty. - aquasecScannerUrl: '' diff --git a/docs/creating-an-ods-task.adoc b/docs/creating-an-ods-task.adoc index e0e9a5e6..ab60b68f 100644 --- a/docs/creating-an-ods-task.adoc +++ b/docs/creating-an-ods-task.adoc @@ -10,8 +10,6 @@ To create a technology-specific Task (e.g. python), the following files should b - [ ] build/package/Dockerfile.python-toolset - The Dockerfile with the dependencies and runtime. - [ ] build/package/scripts/build-python.sh - Bash script to carry out the build, linting, testing operations. -- [ ] deploy/ods-pipeline/charts/images/templates/bc-ods-build-python.yaml - BuildConfig to generate the ods-build-python image. -- [ ] deploy/ods-pipeline/charts/images/templates/is-ods-build-python.yaml - Create ImageStream resource in OpenShift. - [ ] deploy/ods-pipeline/charts/tasks/templates/task-ods-build-python.yaml - The Tekton Task. - [ ] docs/tasks/task-ods-build-python.adoc - To describe the task and its parameters. - [ ] test/tasks/ods-build-python_test.go - A test file to test the behavior of the Tekton Task. diff --git a/docs/design/software-design-specification.adoc b/docs/design/software-design-specification.adoc index bf9dc90b..51b3c72b 100644 --- a/docs/design/software-design-specification.adoc +++ b/docs/design/software-design-specification.adoc @@ -355,11 +355,11 @@ Supplies default SonarQube project properties file if required (SDS-SHARED-3). |=== | SDS-TASK-19 | `ods-package-image` Task resource -| Builds and scans a container image, then pushes it to a registry. References SDS-TASK-20 and executes SDS-TASK-21. +| Builds and scans a container image, then pushes it to a registry. References SDS-TASK-20 and executes SDS-TASK-25 and SDS-TASK-21. | SDS-TASK-20 | `ods-package-image` container image -| Container image to build, scan and push images. Based on `ubi8` (SDS-EXT-1), includes SDS-EXT-17, SDS-EXT-18, SDS-EXT-31 and SDS-TASK-21. If the build argument `aquasecScannerUrl` is set, the referenced Aqua Scanner binary is installed into the image as well. +| Container image to build, scan and push images. Based on `ubi8` (SDS-EXT-1), includes SDS-EXT-17, SDS-EXT-18, SDS-EXT-31 and SDS-TASK-21. | SDS-TASK-21 | `build-and-push` binary @@ -376,6 +376,10 @@ Pushes the image to the target registry (defaulting to an image stream in the na Generates the SBOM report of the image using SDS-EXT-31. The resulting report is placed in `.ods/artifacts/sboms/.spdx` in link:https://spdx.dev/[SPDX] format. If the Aqua scanner is installed in the base image, the pushed image shall be scanned. The resulting report is placed in `.ods/artifacts` and attached as a code insight to Bitbucket. + +| SDS-TASK-25 +| `download-aqua-scanner.sh` shell script +a| If a download URL is specified in the `ods-aqua-scanner-url` secret, the binary is downloaded from the given URL and placed onto the workspace PVC under `.ods-cache/bin`. Subsequent task runs check if the (same version of the) scanner is already present before downloading. |=== ==== `ods-deploy-helm` task diff --git a/docs/design/software-requirements-specification.adoc b/docs/design/software-requirements-specification.adoc index 215ca761..6f032cfd 100644 --- a/docs/design/software-requirements-specification.adoc +++ b/docs/design/software-requirements-specification.adoc @@ -284,7 +284,7 @@ a| The task shall build a Node.JS application using npm. | The task shall push the image to the target registry. | SRS-TASK-PACKAGE-IMAGE-3 -| If the Aqua scanner is installed in the base image, the pushed image shall be scanned. +| If an Aqua scanner download URL is provided, the pushed image shall be scanned with the Aqua scanner binary. | SRS-TASK-PACKAGE-IMAGE-4 | The task shall generate the SBOM report of the image. diff --git a/docs/development.adoc b/docs/development.adoc index 4e33542c..a7bcda13 100644 --- a/docs/development.adoc +++ b/docs/development.adoc @@ -43,6 +43,10 @@ go test -run ^TestTaskODSBuildNPM/build_backend_javascript_app$ github.com/opend Be aware that depending on the tested task, some local services (e.g. Bitbucket) need to run for the test to succeed. These are all started via `make prepare-local-env`, but more fine-grained control is possible too. These dependencies are explicitly set for each test suite and at the beginning of each test suite it will be checked if all required services are running. The tests will fail if at least one service is not running. +TIP: If you want to test build tasks without using SonarQube, pass `-skip-sonar` to `go test`. + +Tests can be run both without a private certificate (which is the default) or using a private certificate to test how the tasks perform when the services use a certificate which is not part of the OS default trusted certs. If you want to use a private certificate in the task tests, pass `-private-cert` to `go test`. + Particularly the task and e2e tests might consume some time and might run into a timeout. To modify the standard timeout (by default in sync with the timeout predefined for Github actions), set the environment variable `ODS_TESTTIMEOUT` (e.g. to `45m`). Also, if you make changes to the images backing the tasks (be it by changing the `Dockerfile` or by changing the scripts/commands installed there), make sure to rebuild and push the affected images to the KinD registry for your changes to take effect. You can do this e.g. through `./scripts/build-and-push-images.sh --image finish` (the name of the image flag is the suffix of the respective Dockerfile). diff --git a/docs/installation.adoc b/docs/installation.adoc index 44e79ff1..f6544a3d 100644 --- a/docs/installation.adoc +++ b/docs/installation.adoc @@ -5,7 +5,7 @@ This guide will show how to install ODS Pipeline in an existing ODS project. It An ODS Pipeline installation consists of the following resources: -* `BuildConfig`, `ImageStream` and `Task` resources +* `Task` resources * `ConfigMap` and `Secret` resources, e.g. holding credentials of centrally installed tools such as Nexus and SonarQube * A pipeline manager, which is creating pipeline runs in response to Bitbucket webhook requests @@ -57,6 +57,8 @@ If you have access to the OpenShift API from your local machine, you can simply The script will interactively ask for credentials (such as Bitbucket access token) and will create corresponding K8s secrets. If you prefer to pass these secrets via flags, see `./install.sh --help` for all options. +IMPORTANT: If tasks need to trust a private certificate, pass `--private-cert `. This will create a K8s secret containing the certificate from the specified host, which will then be mounted in pods during task runs. + TIP: You may pass `--dry-run` to review what `install.sh` will do before actually running the script. After you ran the install script, continue with the <> section. @@ -78,9 +80,7 @@ After you ran the install script, continue with the < "${HELM_GENERATED_VALUES_FILE}" +fi +if [ -f "${kind_values_dir}/bitbucket-${URL_SUFFIX}" ]; then + BITBUCKET_URL=$(cat "${kind_values_dir}/bitbucket-${URL_SUFFIX}") + echo " bitbucketUrl: '${BITBUCKET_URL}'" >> "${HELM_GENERATED_VALUES_FILE}" fi -if [ -f "${ODS_KIND_CREDENTIALS_DIR}/nexus-auth" ]; then - NEXUS_AUTH=$(cat "${ODS_KIND_CREDENTIALS_DIR}/nexus-auth") +if [ -f "${kind_values_dir}/nexus-${URL_SUFFIX}" ]; then + NEXUS_URL=$(cat "${kind_values_dir}/nexus-${URL_SUFFIX}") + echo " nexusUrl: '${NEXUS_URL}'" >> "${HELM_GENERATED_VALUES_FILE}" fi -if [ -f "${ODS_KIND_CREDENTIALS_DIR}/sonar-auth" ]; then - SONAR_AUTH=$(cat "${ODS_KIND_CREDENTIALS_DIR}/sonar-auth") +if [ -f "${kind_values_dir}/sonar-${URL_SUFFIX}" ]; then + SONAR_URL=$(cat "${kind_values_dir}/sonar-${URL_SUFFIX}") + echo " sonarUrl: '${SONAR_URL}'" >> "${HELM_GENERATED_VALUES_FILE}" fi "${ODS_PIPELINE_DIR}"/deploy/install.sh \ --aqua-auth "unavailable:unavailable" \ + --aqua-scanner-url "none" \ --bitbucket-auth "${BITBUCKET_AUTH}" \ --nexus-auth "${NEXUS_AUTH}" \ --sonar-auth "${SONAR_AUTH}" \ - -f ./ods-pipeline/values.kind.yaml,./ods-pipeline/values.generated.yaml "$@" + -f "./ods-pipeline/values.kind.yaml,${HELM_GENERATED_VALUES_FILE}" "$@" diff --git a/scripts/nexus/Dockerfile.arm64 b/scripts/nexus/Dockerfile.arm64 index 81d9d703..c1608bc7 100644 --- a/scripts/nexus/Dockerfile.arm64 +++ b/scripts/nexus/Dockerfile.arm64 @@ -1,6 +1,8 @@ FROM klo2k/nexus3:3.30.1-01 USER root + RUN echo "storage.diskCache.diskFreeSpaceLimit=2048" >> /opt/sonatype/nexus/etc/karaf/system.properties && \ echo "nexus.scripts.allowCreation=true" >> /opt/sonatype/nexus/etc/nexus-default.properties + USER nexus diff --git a/scripts/nexus/Dockerfile b/scripts/nexus/Dockerfile.x86_64 similarity index 99% rename from scripts/nexus/Dockerfile rename to scripts/nexus/Dockerfile.x86_64 index 6aa95940..636a7275 100644 --- a/scripts/nexus/Dockerfile +++ b/scripts/nexus/Dockerfile.x86_64 @@ -1,6 +1,8 @@ FROM sonatype/nexus3:3.30.1 USER root + RUN echo "storage.diskCache.diskFreeSpaceLimit=2048" >> /opt/sonatype/nexus/etc/karaf/system.properties && \ echo "nexus.scripts.allowCreation=true" >> /opt/sonatype/nexus/etc/nexus-default.properties + USER nexus diff --git a/scripts/nginx/nginx-bitbucket.conf b/scripts/nginx/nginx-bitbucket.conf new file mode 100644 index 00000000..723c247c --- /dev/null +++ b/scripts/nginx/nginx-bitbucket.conf @@ -0,0 +1,46 @@ +user nginx; +worker_processes auto; + +error_log /var/log/nginx/error.log notice; +pid /var/run/nginx.pid; + + +events { + worker_connections 1024; +} + + +http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + + access_log /var/log/nginx/access.log main; + + sendfile on; + #tcp_nopush on; + + keepalive_timeout 65; + + #gzip on; + + include /etc/nginx/conf.d/*.conf; + + server { + listen 7993 ssl; + server_name ods-test-bitbucket-tls.kind localhost; + ssl_certificate /etc/nginx/tls.crt; + ssl_certificate_key /etc/nginx/tls.key; + location / { + proxy_pass http://ods-test-bitbucket-server.kind:7990; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $remote_addr; + proxy_set_header X-Forwarded-Proto https; + } + } +} + + diff --git a/scripts/nginx/nginx-nexus.conf b/scripts/nginx/nginx-nexus.conf new file mode 100644 index 00000000..585408fe --- /dev/null +++ b/scripts/nginx/nginx-nexus.conf @@ -0,0 +1,46 @@ +user nginx; +worker_processes auto; + +error_log /var/log/nginx/error.log notice; +pid /var/run/nginx.pid; + + +events { + worker_connections 1024; +} + + +http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + + access_log /var/log/nginx/access.log main; + + sendfile on; + #tcp_nopush on; + + keepalive_timeout 65; + + #gzip on; + + include /etc/nginx/conf.d/*.conf; + + server { + listen 8443 ssl; + server_name ods-test-nexus-tls.kind localhost; + ssl_certificate /etc/nginx/tls.crt; + ssl_certificate_key /etc/nginx/tls.key; + location / { + proxy_pass http://ods-test-nexus.kind:8081; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $remote_addr; + proxy_set_header X-Forwarded-Proto https; + } + } +} + + diff --git a/scripts/nginx/nginx-sonarqube.conf b/scripts/nginx/nginx-sonarqube.conf new file mode 100644 index 00000000..cb4c8c51 --- /dev/null +++ b/scripts/nginx/nginx-sonarqube.conf @@ -0,0 +1,46 @@ +user nginx; +worker_processes auto; + +error_log /var/log/nginx/error.log notice; +pid /var/run/nginx.pid; + + +events { + worker_connections 1024; +} + + +http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + + access_log /var/log/nginx/access.log main; + + sendfile on; + #tcp_nopush on; + + keepalive_timeout 65; + + #gzip on; + + include /etc/nginx/conf.d/*.conf; + + server { + listen 9443 ssl; + server_name ods-test-sonarqube-tls.kind localhost; + ssl_certificate /etc/nginx/tls.crt; + ssl_certificate_key /etc/nginx/tls.key; + location / { + proxy_pass http://ods-test-sonarqube.kind:9000; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $remote_addr; + proxy_set_header X-Forwarded-Proto https; + } + } +} + + diff --git a/scripts/run-bitbucket.sh b/scripts/run-bitbucket.sh index 91bca6ff..eaaab1db 100755 --- a/scripts/run-bitbucket.sh +++ b/scripts/run-bitbucket.sh @@ -7,14 +7,14 @@ set -ue SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" ODS_PIPELINE_DIR=${SCRIPT_DIR%/*} -BITBUCKET_SERVER_HOST_PORT="7990" +HOST_HTTP_PORT="7990" +HOST_HTTPS_PORT="7993" BITBUCKET_SERVER_CONTAINER_NAME="ods-test-bitbucket-server" BITBUCKET_SERVER_IMAGE_NAME="atlassian/bitbucket" BITBUCKET_SERVER_IMAGE_TAG="7.6.5" BITBUCKET_POSTGRES_CONTAINER_NAME="ods-test-bitbucket-postgres" BITBUCKET_POSTGRES_IMAGE_TAG="12" -HELM_VALUES_FILE="${ODS_PIPELINE_DIR}/deploy/ods-pipeline/values.generated.yaml" -ODS_KIND_CREDENTIALS_DIR="${ODS_PIPELINE_DIR}/deploy/.kind-credentials" +kind_values_dir="${ODS_PIPELINE_DIR}/deploy/.kind-values" while [[ "$#" -gt 0 ]]; do case $1 in @@ -58,20 +58,24 @@ docker run --name ${BITBUCKET_SERVER_CONTAINER_NAME} \ -e JDBC_PASSWORD=jellyfish \ -e JDBC_URL=jdbc:postgresql://${BITBUCKET_POSTGRES_CONTAINER_NAME}.kind:5432/bitbucket \ -e ELASTICSEARCH_ENABLED=false \ - -d --net kind -p "${BITBUCKET_SERVER_HOST_PORT}:7990" -p 7999:7999 \ + -d --net kind -p "${HOST_HTTP_PORT}:7990" -p 7999:7999 \ "${BITBUCKET_SERVER_IMAGE_NAME}:${BITBUCKET_SERVER_IMAGE_TAG}" if ! "${SCRIPT_DIR}/waitfor-bitbucket.sh" ; then docker logs ${BITBUCKET_SERVER_CONTAINER_NAME} exit 1 fi -BITBUCKET_URL_FULL="http://${BITBUCKET_SERVER_CONTAINER_NAME}.kind:7990" + +echo "Launch TLS proxy" +TLS_CONTAINER_NAME="${BITBUCKET_SERVER_CONTAINER_NAME}-tls" +"${SCRIPT_DIR}/run-tls-proxy.sh" \ + --container-name "${TLS_CONTAINER_NAME}" \ + --https-port "${HOST_HTTPS_PORT}" \ + --nginx-conf "nginx-bitbucket.conf" # Write values / secrets so that it can be picked up by install.sh later. -if [ ! -e "${HELM_VALUES_FILE}" ]; then - echo "setup:" > "${HELM_VALUES_FILE}" -fi -echo " bitbucketUrl: '${BITBUCKET_URL_FULL}'" >> "${HELM_VALUES_FILE}" -mkdir -p "${ODS_KIND_CREDENTIALS_DIR}" -echo -n "admin:NzU0OTk1MjU0NjEzOpzj5hmFNAaawvupxPKpcJlsfNgP" > "${ODS_KIND_CREDENTIALS_DIR}/bitbucket-auth" -echo -n "webhook:s3cr3t" > "${ODS_KIND_CREDENTIALS_DIR}/bitbucket-webhook-secret" +mkdir -p "${kind_values_dir}" +echo -n "https://${TLS_CONTAINER_NAME}.kind:${HOST_HTTPS_PORT}" > "${kind_values_dir}/bitbucket-https" +echo -n "http://${BITBUCKET_SERVER_CONTAINER_NAME}.kind:${HOST_HTTP_PORT}" > "${kind_values_dir}/bitbucket-http" +echo -n "admin:NzU0OTk1MjU0NjEzOpzj5hmFNAaawvupxPKpcJlsfNgP" > "${kind_values_dir}/bitbucket-auth" +echo -n "webhook:s3cr3t" > "${kind_values_dir}/bitbucket-webhook-secret" diff --git a/scripts/run-nexus.sh b/scripts/run-nexus.sh index 4bba58e1..731798a8 100755 --- a/scripts/run-nexus.sh +++ b/scripts/run-nexus.sh @@ -5,7 +5,8 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" ODS_PIPELINE_DIR=${SCRIPT_DIR%/*} INSECURE="" -HOST_PORT="8081" +HOST_HTTP_PORT="8081" +HOST_HTTPS_PORT="8443" ADMIN_USER="admin" ADMIN_PASSWORD="" DEVELOPER_USERNAME="developer" @@ -14,8 +15,8 @@ NEXUS_URL= IMAGE_NAME="ods-test-nexus" CONTAINER_NAME="ods-test-nexus" NEXUS_IMAGE_TAG="3.30.1" -HELM_VALUES_FILE="${ODS_PIPELINE_DIR}/deploy/ods-pipeline/values.generated.yaml" -ODS_KIND_CREDENTIALS_DIR="${ODS_PIPELINE_DIR}/deploy/.kind-credentials" +kind_values_dir="${ODS_PIPELINE_DIR}/deploy/.kind-values" +DOCKER_CONTEXT_DIR="${ODS_PIPELINE_DIR}/test/testdata/private-cert" while [[ "$#" -gt 0 ]]; do case $1 in @@ -30,20 +31,16 @@ esac; shift; done echo "Run container using image tag ${NEXUS_IMAGE_TAG}" docker rm -f ${CONTAINER_NAME} || true cd "${SCRIPT_DIR}"/nexus -if [ "$(uname -m)" == "arm64" ]; then - docker build -t ${IMAGE_NAME} -f Dockerfile.arm64 . -else - docker build -t ${IMAGE_NAME} . -fi +docker build -t ${IMAGE_NAME} -f "Dockerfile.$(uname -m)" "${DOCKER_CONTEXT_DIR}" cd - &> /dev/null -docker run -d -p "${HOST_PORT}:8081" --net kind --name ${CONTAINER_NAME} ${IMAGE_NAME} +docker run -d -p "${HOST_HTTP_PORT}:8081" --net kind --name ${CONTAINER_NAME} ${IMAGE_NAME} if ! "${SCRIPT_DIR}/waitfor-nexus.sh" ; then docker logs ${CONTAINER_NAME} exit 1 -fi +fi -NEXUS_URL="http://localhost:${HOST_PORT}" +NEXUS_URL="http://localhost:${HOST_HTTP_PORT}" function runJsonScript { local jsonScriptName=$1 @@ -88,10 +85,15 @@ sed "s|@developer_password@|${DEVELOPER_PASSWORD}|g" "${SCRIPT_DIR}"/nexus/devel runJsonScript "createUser" "-d @${SCRIPT_DIR}/nexus/developer-user-with-password.json" rm "${SCRIPT_DIR}"/nexus/developer-user-with-password.json +echo "Launch TLS proxy" +TLS_CONTAINER_NAME="${CONTAINER_NAME}-tls" +"${SCRIPT_DIR}/run-tls-proxy.sh" \ + --container-name "${TLS_CONTAINER_NAME}" \ + --https-port "${HOST_HTTPS_PORT}" \ + --nginx-conf "nginx-nexus.conf" + # Write values / secrets so that it can be picked up by install.sh later. -if [ ! -e "${HELM_VALUES_FILE}" ]; then - echo "setup:" > "${HELM_VALUES_FILE}" -fi -echo " nexusUrl: 'http://${CONTAINER_NAME}.kind:8081'" >> "${HELM_VALUES_FILE}" -mkdir -p "${ODS_KIND_CREDENTIALS_DIR}" -echo -n "${DEVELOPER_USERNAME}:${DEVELOPER_PASSWORD}" > "${ODS_KIND_CREDENTIALS_DIR}/nexus-auth" +mkdir -p "${kind_values_dir}" +echo -n "https://${TLS_CONTAINER_NAME}.kind:${HOST_HTTPS_PORT}" > "${kind_values_dir}/nexus-https" +echo -n "http://${CONTAINER_NAME}.kind:${HOST_HTTP_PORT}" > "${kind_values_dir}/nexus-http" +echo -n "${DEVELOPER_USERNAME}:${DEVELOPER_PASSWORD}" > "${kind_values_dir}/nexus-auth" diff --git a/scripts/run-sonarqube.sh b/scripts/run-sonarqube.sh index e6b3568f..d477d720 100755 --- a/scripts/run-sonarqube.sh +++ b/scripts/run-sonarqube.sh @@ -5,7 +5,8 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" ODS_PIPELINE_DIR=${SCRIPT_DIR%/*} INSECURE="" -HOST_PORT="9000" +HOST_HTTP_PORT="9000" +HOST_HTTPS_PORT="9443" IMAGE_NAME="ods-test-sonarqube" CONTAINER_NAME="ods-test-sonarqube" SONAR_VERSION="8.4" @@ -13,8 +14,7 @@ SONAR_USERNAME="admin" SONAR_PASSWORD="admin" SONAR_EDITION="community" SONAR_IMAGE_TAG="${SONAR_VERSION}-${SONAR_EDITION}" -HELM_VALUES_FILE="${ODS_PIPELINE_DIR}/deploy/ods-pipeline/values.generated.yaml" -ODS_KIND_CREDENTIALS_DIR="${ODS_PIPELINE_DIR}/deploy/.kind-credentials" +kind_values_dir="${ODS_PIPELINE_DIR}/deploy/.kind-values" while [[ "$#" -gt 0 ]]; do case $1 in @@ -51,9 +51,9 @@ else docker build -t ${IMAGE_NAME}:${SONAR_IMAGE_TAG} --build-arg=from=sonarqube:${SONAR_IMAGE_TAG} . fi cd - &> /dev/null -docker run -d --net kind --name ${CONTAINER_NAME} -e SONAR_ES_BOOTSTRAP_CHECKS_DISABLE=true -p "${HOST_PORT}:9000" ${IMAGE_NAME}:${SONAR_IMAGE_TAG} +docker run -d --net kind --name ${CONTAINER_NAME} -e SONAR_ES_BOOTSTRAP_CHECKS_DISABLE=true -p "${HOST_HTTP_PORT}:9000" ${IMAGE_NAME}:${SONAR_IMAGE_TAG} -SONARQUBE_URL="http://localhost:${HOST_PORT}" +SONARQUBE_URL="http://localhost:${HOST_HTTP_PORT}" if ! "${SCRIPT_DIR}/waitfor-sonarqube.sh" ; then docker logs ${CONTAINER_NAME} exit 1 @@ -66,10 +66,15 @@ tokenResponse=$(curl ${INSECURE} -X POST -sSf --user "${SONAR_USERNAME}:${SONAR_ # {"login":"cd_user","name":"foo","token":"bar","createdAt":"2020-04-22T13:21:54+0000"} token=$(echo "${tokenResponse}" | jq -r .token) +echo "Launch TLS proxy" +TLS_CONTAINER_NAME="${CONTAINER_NAME}-tls" +"${SCRIPT_DIR}/run-tls-proxy.sh" \ + --container-name "${TLS_CONTAINER_NAME}" \ + --https-port "${HOST_HTTPS_PORT}" \ + --nginx-conf "nginx-sonarqube.conf" + # Write values / secrets so that it can be picked up by install.sh later. -if [ ! -e "${HELM_VALUES_FILE}" ]; then - echo "setup:" > "${HELM_VALUES_FILE}" -fi -echo " sonarUrl: 'http://${CONTAINER_NAME}.kind:9000'" >> "${HELM_VALUES_FILE}" -mkdir -p "${ODS_KIND_CREDENTIALS_DIR}" -echo -n ":${token}" > "${ODS_KIND_CREDENTIALS_DIR}/sonar-auth" +mkdir -p "${kind_values_dir}" +echo -n "https://${TLS_CONTAINER_NAME}.kind:${HOST_HTTPS_PORT}" > "${kind_values_dir}/sonar-https" +echo -n "http://${CONTAINER_NAME}.kind:${HOST_HTTP_PORT}" > "${kind_values_dir}/sonar-http" +echo -n ":${token}" > "${kind_values_dir}/sonar-auth" diff --git a/scripts/run-tls-proxy.sh b/scripts/run-tls-proxy.sh new file mode 100755 index 00000000..7f1e3e6e --- /dev/null +++ b/scripts/run-tls-proxy.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +set -ue + +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ods_pipeline_dir=${script_dir%/*} + +https_port="8443" +container_name="" +nginx_conf="" + +while [[ "$#" -gt 0 ]]; do + case $1 in + + -v|--verbose) set -x;; + + --container-name) container_name="$2"; shift;; + --container-name=*) container_name="${1#*=}";; + + --nginx-conf) nginx_conf="$2"; shift;; + --nginx-conf=*) nginx_conf="${1#*=}";; + + --https-port) https_port="$2"; shift;; + --https-port=*) https_port="${1#*=}";; + + *) echo "Unknown parameter passed: $1"; exit 1;; +esac; shift; done + +nginx_image="nginx:alpine-slim" +private_cert_dir="${ods_pipeline_dir}/test/testdata/private-cert" +if [ "$(uname -m)" == "arm64" ]; then + nginx_image="arm64v8/nginx:alpine-slim" +fi +docker rm -f "${container_name}" &> /dev/null || true +docker run --name "${container_name}" \ + -v "${script_dir}/nginx/${nginx_conf}:/etc/nginx/nginx.conf:ro" \ + -v "${private_cert_dir}/tls.crt:/etc/nginx/tls.crt:ro" \ + -v "${private_cert_dir}/tls.key:/etc/nginx/tls.key:ro" \ + -d --net kind -p "${https_port}:${https_port}" "${nginx_image}" diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index e6761dab..293d62f7 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -16,7 +16,6 @@ import ( "github.com/opendevstack/pipeline/internal/kubernetes" "github.com/opendevstack/pipeline/pkg/bitbucket" "github.com/opendevstack/pipeline/pkg/tasktesting" - "github.com/tektoncd/pipeline/pkg/apis/pipeline/v1beta1" tektonv1beta1 "github.com/tektoncd/pipeline/pkg/apis/pipeline/v1beta1" tekton "github.com/tektoncd/pipeline/pkg/client/clientset/versioned" corev1 "k8s.io/api/core/v1" @@ -26,6 +25,7 @@ import ( ) var outsideKindFlag = flag.Bool("outside-kind", false, "Whether to continue if not in KinD cluster") +var privateCertFlag = flag.Bool("private-cert", false, "Whether to run tests using a private cert") func TestE2E(t *testing.T) { tasktesting.CheckCluster(t, *outsideKindFlag) @@ -70,7 +70,7 @@ func TestE2E(t *testing.T) { } t.Logf("Workspace is in %s", wsDir) odsContext := tasktesting.SetupBitbucketRepo( - t, c.KubernetesClientSet, ns, wsDir, tasktesting.BitbucketProjectKey, + t, c.KubernetesClientSet, ns, wsDir, tasktesting.BitbucketProjectKey, *privateCertFlag, ) // The webhook URL needs to be the address of the KinD control plane on the node port. @@ -88,7 +88,7 @@ func TestE2E(t *testing.T) { if err != nil { t.Fatalf("could not get Bitbucket webhook secret: %s", err) } - bitbucketClient := tasktesting.BitbucketClientOrFatal(t, c.KubernetesClientSet, ns) + bitbucketClient := tasktesting.BitbucketClientOrFatal(t, c.KubernetesClientSet, ns, *privateCertFlag) _, err = bitbucketClient.WebhookCreate( odsContext.Project, odsContext.Repository, @@ -193,7 +193,7 @@ func waitForPipelineRunToBeTriggered(clientset *tekton.Clientset, ns string, tim ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() - var pipelineRunList *v1beta1.PipelineRunList + var pipelineRunList *tektonv1beta1.PipelineRunList for { time.Sleep(2 * time.Second) prs, err := clientset.TektonV1beta1().PipelineRuns(ns).List(ctx, metav1.ListOptions{}) diff --git a/test/scripts/download-aqua-scanner_test.go b/test/scripts/download-aqua-scanner_test.go new file mode 100644 index 00000000..4ba22cac --- /dev/null +++ b/test/scripts/download-aqua-scanner_test.go @@ -0,0 +1,126 @@ +package scripts_test + +import ( + "errors" + "flag" + "fmt" + "net/http" + "net/http/httptest" + "os" + "testing" + + "github.com/opendevstack/pipeline/internal/command" +) + +const ( + downloadAquaScannerScript = "../../build/package/scripts/download-aqua-scanner.sh" + fakeScannerBinary = `#!/bin/bash +echo 1.7.3` +) + +var md5Bin = flag.String("md5bin", "md5", "md5 binary to use") + +func TestCachedDownload(t *testing.T) { + hits := 0 + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + hits++ + fmt.Fprintln(w, fakeScannerBinary) + })) + defer ts.Close() + dir, cleanupDir := tmpDir(t) + defer cleanupDir() + + t.Log("Fresh run -> download") + runScript(t, dir, fmt.Sprintf("%s/foo", ts.URL)) + if hits != 1 { + t.Error("Wanted hit, got none") + } + fileExistsInDir(t, dir, "aquasec", ".md5-aquasec") + + t.Log("Second run for same URL -> no download") + runScript(t, dir, fmt.Sprintf("%s/foo", ts.URL)) + if hits > 1 { + t.Error("Wanted no further hit, got more") + } + fileExistsInDir(t, dir, "aquasec", ".md5-aquasec") + + t.Log("Third run for different URL -> download") + runScript(t, dir, fmt.Sprintf("%s/bar", ts.URL)) + if hits != 2 { + t.Error("Wanted further hit, got none") + } + fileExistsInDir(t, dir, "aquasec", ".md5-aquasec") +} + +func TestSkipDownload(t *testing.T) { + dir, cleanupDir := tmpDir(t) + defer cleanupDir() + + t.Log("No URL") + runScript(t, dir, "") + fileDoesNotExistInDir(t, dir, "aquasec", ".md5-aquasec") + + t.Log("URL set to 'none'") + runScript(t, dir, "none") + fileDoesNotExistInDir(t, dir, "aquasec", ".md5-aquasec") +} + +// runScript runs the download script against given url +// and places the downloaded file into dir. +func runScript(t *testing.T, dir, url string) { + err := command.Run( + downloadAquaScannerScript, + []string{ + fmt.Sprintf("--bin-dir=%s", dir), + fmt.Sprintf("--aqua-scanner-url=%s", url), + }, []string{fmt.Sprintf("MD5_BIN=%s", *md5Bin)}, + &testingLogWriter{t}, + &testingLogWriter{t}, + ) + if err != nil { + t.Fatal(err) + } +} + +// fileExistsInDir checks if file(s) exist in dir or errors. +func fileExistsInDir(t *testing.T, dir string, files ...string) { + for _, file := range files { + f := fmt.Sprintf("%s/%s", dir, file) + if _, err := os.Stat(f); errors.Is(err, os.ErrNotExist) { + t.Errorf("Want file %s, got %s", f, err) + } + } +} + +// fileDoesNotExistInDir checks if file(s) exist in dir or errors. +func fileDoesNotExistInDir(t *testing.T, dir string, files ...string) { + for _, file := range files { + f := fmt.Sprintf("%s/%s", dir, file) + if _, err := os.Stat(f); err == nil { + t.Errorf("Did not want file %s", f) + } + } +} + +// tmpDir creates a temp dir or fails. +func tmpDir(t *testing.T) (string, func()) { + t.Helper() + dir, err := os.MkdirTemp(".", "download-aqua-scanner-") + if err != nil { + t.Fatal(err) + } + return dir, func() { os.RemoveAll(dir) } +} + +// testingLogWriter implements io.Writer so that +// it can proxy to t.Log when an io.Writer is required. +type testingLogWriter struct { + t *testing.T +} + +// Write proxies to t.Logf. +func (f *testingLogWriter) Write(p []byte) (n int, err error) { + f.t.Helper() + f.t.Logf("%s", string(p)) + return len(p), nil +} diff --git a/test/tasks/common_test.go b/test/tasks/common_test.go index 937b62af..039e5e57 100644 --- a/test/tasks/common_test.go +++ b/test/tasks/common_test.go @@ -17,17 +17,41 @@ import ( "github.com/opendevstack/pipeline/pkg/pipelinectxt" "github.com/opendevstack/pipeline/pkg/sonar" "github.com/opendevstack/pipeline/pkg/tasktesting" + "golang.org/x/exp/slices" kclient "k8s.io/client-go/kubernetes" "sigs.k8s.io/yaml" ) var alwaysKeepTmpWorkspacesFlag = flag.Bool("always-keep-tmp-workspaces", false, "Whether to keep temporary workspaces from taskruns even when test is successful") var outsideKindFlag = flag.Bool("outside-kind", false, "Whether to continue if not in KinD cluster") +var skipSonarQubeFlag = flag.Bool("skip-sonar", false, "Whether to skip SonarQube steps") +var privateCertFlag = flag.Bool("private-cert", false, "Whether to run tests using a private cert") const ( taskKindRef = "Task" ) +// buildTaskParams forces all SonarQube params to be "falsy" +// if the skipSonarQubeFlag is set. +func buildTaskParams(p map[string]string) map[string]string { + if *skipSonarQubeFlag { + p["sonar-skip"] = "true" + p["sonar-quality-gate"] = "false" + } + return p +} + +// requiredServices takes a variable amount of services and removes +// SonarQube from the resulting slice if the skipSonarQubeFlag is set. +func requiredServices(s ...tasktesting.Service) []tasktesting.Service { + requiredServices := append([]tasktesting.Service{}, s...) + sqIndex := slices.Index(requiredServices, tasktesting.SonarQube) + if sqIndex != -1 && *skipSonarQubeFlag { + requiredServices = slices.Delete(requiredServices, sqIndex, sqIndex+1) + } + return requiredServices +} + func checkODSContext(t *testing.T, repoDir string, want *pipelinectxt.ODSContext) { checkODSFileContent(t, repoDir, "component", want.Component) checkODSFileContent(t, repoDir, "git-commit-sha", want.GitCommitSHA) @@ -139,6 +163,7 @@ func runTaskTestCases(t *testing.T, taskName string, requiredServices []tasktest SourceDir: tasktesting.StorageSourceDir, StorageCapacity: tasktesting.StorageCapacity, StorageClassName: tasktesting.StorageClassName, + PrivateCert: *privateCertFlag, }, ) diff --git a/test/tasks/ods-build-go_test.go b/test/tasks/ods-build-go_test.go index 8e0b48fe..c8d6cedb 100644 --- a/test/tasks/ods-build-go_test.go +++ b/test/tasks/ods-build-go_test.go @@ -19,21 +19,19 @@ func TestTaskODSBuildGo(t *testing.T) { goProverb := "Don't communicate by sharing memory, share memory by communicating." runTaskTestCases(t, "ods-build-go", - []tasktesting.Service{ - tasktesting.SonarQube, - }, + requiredServices(tasktesting.SonarQube), map[string]tasktesting.TestCase{ "build go app": { WorkspaceDirMapping: map[string]string{"source": "go-sample-app"}, PreRunFunc: func(t *testing.T, ctxt *tasktesting.TaskRunContext) { wsDir := ctxt.Workspaces["source"] ctxt.ODS = tasktesting.SetupGitRepo(t, ctxt.Namespace, wsDir) - ctxt.Params = map[string]string{ + ctxt.Params = buildTaskParams(map[string]string{ "go-os": runtime.GOOS, "go-arch": runtime.GOARCH, "sonar-quality-gate": "true", "cache-build": "false", - } + }) }, WantRunSuccess: true, PostRunFunc: func(t *testing.T, ctxt *tasktesting.TaskRunContext) { @@ -45,13 +43,17 @@ func TestTaskODSBuildGo(t *testing.T) { filepath.Join(pipelinectxt.LintReportsPath, "report.txt"), filepath.Join(pipelinectxt.XUnitReportsPath, "report.xml"), filepath.Join(pipelinectxt.CodeCoveragesPath, "coverage.out"), - filepath.Join(pipelinectxt.SonarAnalysisPath, "analysis-report.md"), - filepath.Join(pipelinectxt.SonarAnalysisPath, "issues-report.csv"), - filepath.Join(pipelinectxt.SonarAnalysisPath, "quality-gate.json"), ) - sonarProject := sonar.ProjectKey(ctxt.ODS, "") - checkSonarQualityGate(t, ctxt.Clients.KubernetesClientSet, ctxt.Namespace, sonarProject, true, "OK") + if !*skipSonarQubeFlag { + checkFilesExist(t, wsDir, + filepath.Join(pipelinectxt.SonarAnalysisPath, "analysis-report.md"), + filepath.Join(pipelinectxt.SonarAnalysisPath, "issues-report.csv"), + filepath.Join(pipelinectxt.SonarAnalysisPath, "quality-gate.json"), + ) + sonarProject := sonar.ProjectKey(ctxt.ODS, "") + checkSonarQualityGate(t, ctxt.Clients.KubernetesClientSet, ctxt.Namespace, sonarProject, true, "OK") + } wantLogMsg := "No sonar-project.properties present, using default:" if !strings.Contains(string(ctxt.CollectedLogs), wantLogMsg) { @@ -76,12 +78,11 @@ func TestTaskODSBuildGo(t *testing.T) { PreRunFunc: func(t *testing.T, ctxt *tasktesting.TaskRunContext) { wsDir := ctxt.Workspaces["source"] ctxt.ODS = tasktesting.SetupGitRepo(t, ctxt.Namespace, wsDir) - ctxt.Params = map[string]string{ + ctxt.Params = buildTaskParams(map[string]string{ "go-os": runtime.GOOS, "go-arch": runtime.GOARCH, "sonar-quality-gate": "true", - // "cache-build": "true", expected to be default - } + }) }, WantRunSuccess: true, PostRunFunc: func(t *testing.T, ctxt *tasktesting.TaskRunContext) { @@ -93,13 +94,18 @@ func TestTaskODSBuildGo(t *testing.T) { filepath.Join(pipelinectxt.LintReportsPath, "report.txt"), filepath.Join(pipelinectxt.XUnitReportsPath, "report.xml"), filepath.Join(pipelinectxt.CodeCoveragesPath, "coverage.out"), - filepath.Join(pipelinectxt.SonarAnalysisPath, "analysis-report.md"), - filepath.Join(pipelinectxt.SonarAnalysisPath, "issues-report.csv"), - filepath.Join(pipelinectxt.SonarAnalysisPath, "quality-gate.json"), ) - sonarProject := sonar.ProjectKey(ctxt.ODS, "") - checkSonarQualityGate(t, ctxt.Clients.KubernetesClientSet, ctxt.Namespace, sonarProject, true, "OK") + if !*skipSonarQubeFlag { + checkFilesExist(t, wsDir, + filepath.Join(pipelinectxt.SonarAnalysisPath, "analysis-report.md"), + filepath.Join(pipelinectxt.SonarAnalysisPath, "issues-report.csv"), + filepath.Join(pipelinectxt.SonarAnalysisPath, "quality-gate.json"), + ) + + sonarProject := sonar.ProjectKey(ctxt.ODS, "") + checkSonarQualityGate(t, ctxt.Clients.KubernetesClientSet, ctxt.Namespace, sonarProject, true, "OK") + } // This is not available when build skipping as the default is // supplied on the second repeat. @@ -134,12 +140,12 @@ func TestTaskODSBuildGo(t *testing.T) { createAppInSubDirectory(t, wsDir, subdir, "go-sample-app") ctxt.ODS = tasktesting.SetupGitRepo(t, ctxt.Namespace, wsDir) - ctxt.Params = map[string]string{ + ctxt.Params = buildTaskParams(map[string]string{ "go-os": runtime.GOOS, "go-arch": runtime.GOARCH, "sonar-quality-gate": "true", "working-dir": subdir, - } + }) }, WantRunSuccess: true, PostRunFunc: func(t *testing.T, ctxt *tasktesting.TaskRunContext) { @@ -154,12 +160,17 @@ func TestTaskODSBuildGo(t *testing.T) { filepath.Join(pipelinectxt.LintReportsPath, fmt.Sprintf("%s-report.txt", subdir)), filepath.Join(pipelinectxt.XUnitReportsPath, fmt.Sprintf("%s-report.xml", subdir)), filepath.Join(pipelinectxt.CodeCoveragesPath, fmt.Sprintf("%s-coverage.out", subdir)), - filepath.Join(pipelinectxt.SonarAnalysisPath, fmt.Sprintf("%s-analysis-report.md", subdir)), - filepath.Join(pipelinectxt.SonarAnalysisPath, fmt.Sprintf("%s-issues-report.csv", subdir)), - filepath.Join(pipelinectxt.SonarAnalysisPath, fmt.Sprintf("%s-quality-gate.json", subdir))) + ) - sonarProject := sonar.ProjectKey(ctxt.ODS, subdir+"-") - checkSonarQualityGate(t, ctxt.Clients.KubernetesClientSet, ctxt.Namespace, sonarProject, true, "OK") + if !*skipSonarQubeFlag { + checkFilesExist(t, wsDir, + filepath.Join(pipelinectxt.SonarAnalysisPath, fmt.Sprintf("%s-analysis-report.md", subdir)), + filepath.Join(pipelinectxt.SonarAnalysisPath, fmt.Sprintf("%s-issues-report.csv", subdir)), + filepath.Join(pipelinectxt.SonarAnalysisPath, fmt.Sprintf("%s-quality-gate.json", subdir)), + ) + sonarProject := sonar.ProjectKey(ctxt.ODS, subdir+"-") + checkSonarQualityGate(t, ctxt.Clients.KubernetesClientSet, ctxt.Namespace, sonarProject, true, "OK") + } b, _, err := command.RunBuffered(filepath.Join(wsDir, binary), []string{}) if err != nil { @@ -179,10 +190,10 @@ func TestTaskODSBuildGo(t *testing.T) { PreRunFunc: func(t *testing.T, ctxt *tasktesting.TaskRunContext) { wsDir := ctxt.Workspaces["source"] ctxt.ODS = tasktesting.SetupGitRepo(t, ctxt.Namespace, wsDir) - ctxt.Params = map[string]string{ + ctxt.Params = buildTaskParams(map[string]string{ "go-os": runtime.GOOS, "go-arch": runtime.GOARCH, - } + }) }, WantRunSuccess: false, PostRunFunc: func(t *testing.T, ctxt *tasktesting.TaskRunContext) { @@ -205,10 +216,10 @@ func TestTaskODSBuildGo(t *testing.T) { PreRunFunc: func(t *testing.T, ctxt *tasktesting.TaskRunContext) { wsDir := ctxt.Workspaces["source"] ctxt.ODS = tasktesting.SetupGitRepo(t, ctxt.Namespace, wsDir) - ctxt.Params = map[string]string{ + ctxt.Params = buildTaskParams(map[string]string{ "sonar-skip": "true", "pre-test-script": "pre-test-script.sh", - } + }) }, WantRunSuccess: true, PostRunFunc: func(t *testing.T, ctxt *tasktesting.TaskRunContext) { @@ -229,16 +240,18 @@ func TestTaskODSBuildGo(t *testing.T) { ctxt.ODS = tasktesting.SetupGitRepo(t, ctxt.Namespace, wsDir) writeContextFile(t, wsDir, "pr-key", "3") writeContextFile(t, wsDir, "pr-key", "master") - ctxt.Params = map[string]string{ + ctxt.Params = buildTaskParams(map[string]string{ "go-os": runtime.GOOS, "go-arch": runtime.GOARCH, "sonar-quality-gate": "true", - } + }) }, WantRunSuccess: true, PostRunFunc: func(t *testing.T, ctxt *tasktesting.TaskRunContext) { - sonarProject := sonar.ProjectKey(ctxt.ODS, "") - checkSonarQualityGate(t, ctxt.Clients.KubernetesClientSet, ctxt.Namespace, sonarProject, true, "OK") + if !*skipSonarQubeFlag { + sonarProject := sonar.ProjectKey(ctxt.ODS, "") + checkSonarQualityGate(t, ctxt.Clients.KubernetesClientSet, ctxt.Namespace, sonarProject, true, "OK") + } }, CleanupFunc: func(t *testing.T, ctxt *tasktesting.TaskRunContext) { wsDir := ctxt.Workspaces["source"] diff --git a/test/tasks/ods-build-gradle_test.go b/test/tasks/ods-build-gradle_test.go index 8290984b..f650d19a 100644 --- a/test/tasks/ods-build-gradle_test.go +++ b/test/tasks/ods-build-gradle_test.go @@ -13,20 +13,17 @@ import ( func TestTaskODSBuildGradle(t *testing.T) { runTaskTestCases(t, "ods-build-gradle", - []tasktesting.Service{ - tasktesting.Nexus, - tasktesting.SonarQube, - }, + requiredServices(tasktesting.Nexus, tasktesting.SonarQube), map[string]tasktesting.TestCase{ "task should build gradle app": { WorkspaceDirMapping: map[string]string{"source": "gradle-sample-app"}, PreRunFunc: func(t *testing.T, ctxt *tasktesting.TaskRunContext) { wsDir := ctxt.Workspaces["source"] ctxt.ODS = tasktesting.SetupGitRepo(t, ctxt.Namespace, wsDir) - ctxt.Params = map[string]string{ + ctxt.Params = buildTaskParams(map[string]string{ "sonar-quality-gate": "true", "cache-build": "false", - } + }) }, WantRunSuccess: true, PostRunFunc: func(t *testing.T, ctxt *tasktesting.TaskRunContext) { @@ -38,15 +35,20 @@ func TestTaskODSBuildGradle(t *testing.T) { filepath.Join(pipelinectxt.XUnitReportsPath, "TEST-ods.java.gradle.sample.app.AppTest.xml"), filepath.Join(pipelinectxt.XUnitReportsPath, "TEST-ods.java.gradle.sample.app.AppTest2.xml"), filepath.Join(pipelinectxt.CodeCoveragesPath, "coverage.xml"), - filepath.Join(pipelinectxt.SonarAnalysisPath, "analysis-report.md"), - filepath.Join(pipelinectxt.SonarAnalysisPath, "issues-report.csv"), - filepath.Join(pipelinectxt.SonarAnalysisPath, "quality-gate.json"), ) + if !*skipSonarQubeFlag { + checkFilesExist(t, wsDir, + filepath.Join(pipelinectxt.SonarAnalysisPath, "analysis-report.md"), + filepath.Join(pipelinectxt.SonarAnalysisPath, "issues-report.csv"), + filepath.Join(pipelinectxt.SonarAnalysisPath, "quality-gate.json"), + ) + } + logContains(ctxt.CollectedLogs, t, "--gradle-options=--no-daemon --stacktrace", "No sonar-project.properties present, using default:", - "Using NEXUS_URL=http://ods-test-nexus.kind:8081", + "ods-test-nexus", "Gradle 7.4.2", "Using GRADLE_OPTS=-Dorg.gradle.jvmargs=-Xmx512M", "Using GRADLE_USER_HOME=/workspace/source/.ods-cache/deps/gradle", @@ -60,10 +62,9 @@ func TestTaskODSBuildGradle(t *testing.T) { PreRunFunc: func(t *testing.T, ctxt *tasktesting.TaskRunContext) { wsDir := ctxt.Workspaces["source"] ctxt.ODS = tasktesting.SetupGitRepo(t, ctxt.Namespace, wsDir) - ctxt.Params = map[string]string{ + ctxt.Params = buildTaskParams(map[string]string{ "sonar-quality-gate": "true", - // "cache-build": "true", is expected to be default - } + }) }, WantRunSuccess: true, PostRunFunc: func(t *testing.T, ctxt *tasktesting.TaskRunContext) { @@ -75,15 +76,20 @@ func TestTaskODSBuildGradle(t *testing.T) { filepath.Join(pipelinectxt.XUnitReportsPath, "TEST-ods.java.gradle.sample.app.AppTest.xml"), filepath.Join(pipelinectxt.XUnitReportsPath, "TEST-ods.java.gradle.sample.app.AppTest2.xml"), filepath.Join(pipelinectxt.CodeCoveragesPath, "coverage.xml"), - filepath.Join(pipelinectxt.SonarAnalysisPath, "analysis-report.md"), - filepath.Join(pipelinectxt.SonarAnalysisPath, "issues-report.csv"), - filepath.Join(pipelinectxt.SonarAnalysisPath, "quality-gate.json"), ) + if !*skipSonarQubeFlag { + checkFilesExist(t, wsDir, + filepath.Join(pipelinectxt.SonarAnalysisPath, "analysis-report.md"), + filepath.Join(pipelinectxt.SonarAnalysisPath, "issues-report.csv"), + filepath.Join(pipelinectxt.SonarAnalysisPath, "quality-gate.json"), + ) + } + logContains(ctxt.CollectedLogs, t, "--gradle-options=--no-daemon --stacktrace", "No sonar-project.properties present, using default:", - "Using NEXUS_URL=http://ods-test-nexus.kind:8081", + "ods-test-nexus", "Gradle 7.4.2", "Using GRADLE_OPTS=-Dorg.gradle.jvmargs=-Xmx512M", "Using GRADLE_USER_HOME=/workspace/source/.ods-cache/deps/gradle", @@ -108,11 +114,16 @@ func TestTaskODSBuildGradle(t *testing.T) { filepath.Join(pipelinectxt.XUnitReportsPath, "TEST-ods.java.gradle.sample.app.AppTest.xml"), filepath.Join(pipelinectxt.XUnitReportsPath, "TEST-ods.java.gradle.sample.app.AppTest2.xml"), filepath.Join(pipelinectxt.CodeCoveragesPath, "coverage.xml"), - filepath.Join(pipelinectxt.SonarAnalysisPath, "analysis-report.md"), - filepath.Join(pipelinectxt.SonarAnalysisPath, "issues-report.csv"), - filepath.Join(pipelinectxt.SonarAnalysisPath, "quality-gate.json"), ) + if !*skipSonarQubeFlag { + checkFilesExist(t, wsDir, + filepath.Join(pipelinectxt.SonarAnalysisPath, "analysis-report.md"), + filepath.Join(pipelinectxt.SonarAnalysisPath, "issues-report.csv"), + filepath.Join(pipelinectxt.SonarAnalysisPath, "quality-gate.json"), + ) + } + logContains(ctxt.CollectedLogs, t, "Copying prior build artifacts from cache: /workspace/source/.ods-cache/build-task/gradle", "Copying prior build output from cache: /workspace/source/.ods-cache/build-task/gradle", diff --git a/test/tasks/ods-build-npm_test.go b/test/tasks/ods-build-npm_test.go index d5c9342a..69a519d4 100644 --- a/test/tasks/ods-build-npm_test.go +++ b/test/tasks/ods-build-npm_test.go @@ -15,20 +15,17 @@ import ( func TestTaskODSBuildNPM(t *testing.T) { runTaskTestCases(t, "ods-build-npm", - []tasktesting.Service{ - tasktesting.Nexus, - tasktesting.SonarQube, - }, + requiredServices(tasktesting.Nexus, tasktesting.SonarQube), map[string]tasktesting.TestCase{ "build typescript app with SQ scan": { WorkspaceDirMapping: map[string]string{"source": "typescript-sample-app"}, PreRunFunc: func(t *testing.T, ctxt *tasktesting.TaskRunContext) { wsDir := ctxt.Workspaces["source"] ctxt.ODS = tasktesting.SetupGitRepo(t, ctxt.Namespace, wsDir) - ctxt.Params = map[string]string{ + ctxt.Params = buildTaskParams(map[string]string{ "sonar-quality-gate": "true", "cache-build": "false", - } + }) }, WantRunSuccess: true, PostRunFunc: func(t *testing.T, ctxt *tasktesting.TaskRunContext) { @@ -39,22 +36,28 @@ func TestTaskODSBuildNPM(t *testing.T) { filepath.Join(pipelinectxt.CodeCoveragesPath, "clover.xml"), filepath.Join(pipelinectxt.CodeCoveragesPath, "coverage-final.json"), filepath.Join(pipelinectxt.CodeCoveragesPath, "lcov.info"), - filepath.Join(pipelinectxt.SonarAnalysisPath, "analysis-report.md"), - filepath.Join(pipelinectxt.SonarAnalysisPath, "issues-report.csv"), - filepath.Join(pipelinectxt.SonarAnalysisPath, "quality-gate.json"), filepath.Join(pipelinectxt.LintReportsPath, "report.txt"), "docker/dist/src/index.js", "docker/dist/package.json", "docker/dist/package-lock.json", ) + if !*skipSonarQubeFlag { + checkFilesExist(t, wsDir, + filepath.Join(pipelinectxt.SonarAnalysisPath, "analysis-report.md"), + filepath.Join(pipelinectxt.SonarAnalysisPath, "issues-report.csv"), + filepath.Join(pipelinectxt.SonarAnalysisPath, "quality-gate.json"), + ) + } wantLogMsg := "No sonar-project.properties present, using default:" if !strings.Contains(string(ctxt.CollectedLogs), wantLogMsg) { t.Fatalf("Want:\n%s\n\nGot:\n%s", wantLogMsg, string(ctxt.CollectedLogs)) } - sonarProject := sonar.ProjectKey(ctxt.ODS, "") - checkSonarQualityGate(t, ctxt.Clients.KubernetesClientSet, ctxt.Namespace, sonarProject, true, "OK") + if !*skipSonarQubeFlag { + sonarProject := sonar.ProjectKey(ctxt.ODS, "") + checkSonarQualityGate(t, ctxt.Clients.KubernetesClientSet, ctxt.Namespace, sonarProject, true, "OK") + } }, }, "build javascript app in subdirectory with build caching": { @@ -66,12 +69,10 @@ func TestTaskODSBuildNPM(t *testing.T) { createAppInSubDirectory(t, wsDir, subdir, "javascript-sample-app") ctxt.ODS = tasktesting.SetupGitRepo(t, ctxt.Namespace, wsDir) - ctxt.Params = map[string]string{ - "sonar-skip": "true", + ctxt.Params = buildTaskParams(map[string]string{ "working-dir": subdir, "output-dir": "../docker", - // "cache-build": "true", expected to be default now - } + }) }, WantRunSuccess: true, PostRunFunc: func(t *testing.T, ctxt *tasktesting.TaskRunContext) { @@ -133,10 +134,9 @@ func TestTaskODSBuildNPM(t *testing.T) { PreRunFunc: func(t *testing.T, ctxt *tasktesting.TaskRunContext) { wsDir := ctxt.Workspaces["source"] ctxt.ODS = tasktesting.SetupGitRepo(t, ctxt.Namespace, wsDir) - ctxt.Params = map[string]string{ - "sonar-skip": "true", + ctxt.Params = buildTaskParams(map[string]string{ "copy-node-modules": "true", - } + }) }, WantRunSuccess: true, PostRunFunc: func(t *testing.T, ctxt *tasktesting.TaskRunContext) { @@ -153,10 +153,9 @@ func TestTaskODSBuildNPM(t *testing.T) { PreRunFunc: func(t *testing.T, ctxt *tasktesting.TaskRunContext) { wsDir := ctxt.Workspaces["source"] ctxt.ODS = tasktesting.SetupGitRepo(t, ctxt.Namespace, wsDir) - ctxt.Params = map[string]string{ - "sonar-skip": "true", - "build-dir": "build", - } + ctxt.Params = buildTaskParams(map[string]string{ + "build-dir": "build", + }) }, WantRunSuccess: true, PostRunFunc: func(t *testing.T, ctxt *tasktesting.TaskRunContext) { diff --git a/test/tasks/ods-build-python_test.go b/test/tasks/ods-build-python_test.go index f6ce75eb..618928d1 100644 --- a/test/tasks/ods-build-python_test.go +++ b/test/tasks/ods-build-python_test.go @@ -15,20 +15,17 @@ import ( func TestTaskODSBuildPython(t *testing.T) { runTaskTestCases(t, "ods-build-python", - []tasktesting.Service{ - tasktesting.Nexus, - tasktesting.SonarQube, - }, + requiredServices(tasktesting.Nexus, tasktesting.SonarQube), map[string]tasktesting.TestCase{ "build python fastapi app": { WorkspaceDirMapping: map[string]string{"source": "python-fastapi-sample-app"}, PreRunFunc: func(t *testing.T, ctxt *tasktesting.TaskRunContext) { wsDir := ctxt.Workspaces["source"] ctxt.ODS = tasktesting.SetupGitRepo(t, ctxt.Namespace, wsDir) - ctxt.Params = map[string]string{ + ctxt.Params = buildTaskParams(map[string]string{ "sonar-quality-gate": "true", "cache-build": "false", - } + }) }, WantRunSuccess: true, PostRunFunc: func(t *testing.T, ctxt *tasktesting.TaskRunContext) { @@ -39,10 +36,14 @@ func TestTaskODSBuildPython(t *testing.T) { "docker/app/requirements.txt", filepath.Join(pipelinectxt.XUnitReportsPath, "report.xml"), filepath.Join(pipelinectxt.CodeCoveragesPath, "coverage.xml"), - filepath.Join(pipelinectxt.SonarAnalysisPath, "analysis-report.md"), - filepath.Join(pipelinectxt.SonarAnalysisPath, "issues-report.csv"), - filepath.Join(pipelinectxt.SonarAnalysisPath, "quality-gate.json"), ) + if !*skipSonarQubeFlag { + checkFilesExist(t, wsDir, + filepath.Join(pipelinectxt.SonarAnalysisPath, "analysis-report.md"), + filepath.Join(pipelinectxt.SonarAnalysisPath, "issues-report.csv"), + filepath.Join(pipelinectxt.SonarAnalysisPath, "quality-gate.json"), + ) + } wantContainsBytes, err := os.ReadFile("../../test/testdata/golden/ods-build-python/excerpt-from-coverage.xml") if err != nil { @@ -56,8 +57,12 @@ func TestTaskODSBuildPython(t *testing.T) { wantContains = strings.ReplaceAll(wantContains, " ", "") checkFileContentLeanContains(t, wsDir, filepath.Join(pipelinectxt.CodeCoveragesPath, "coverage.xml"), wantContains) - sonarProject := sonar.ProjectKey(ctxt.ODS, "") - checkSonarQualityGate(t, ctxt.Clients.KubernetesClientSet, ctxt.Namespace, sonarProject, true, "OK") + + if !*skipSonarQubeFlag { + sonarProject := sonar.ProjectKey(ctxt.ODS, "") + checkSonarQualityGate(t, ctxt.Clients.KubernetesClientSet, ctxt.Namespace, sonarProject, true, "OK") + } + wantLogMsg := "No sonar-project.properties present, using default:" if !strings.Contains(string(ctxt.CollectedLogs), wantLogMsg) { t.Fatalf("Want:\n%s\n\nGot:\n%s", wantLogMsg, string(ctxt.CollectedLogs)) @@ -69,10 +74,9 @@ func TestTaskODSBuildPython(t *testing.T) { PreRunFunc: func(t *testing.T, ctxt *tasktesting.TaskRunContext) { wsDir := ctxt.Workspaces["source"] ctxt.ODS = tasktesting.SetupGitRepo(t, ctxt.Namespace, wsDir) - ctxt.Params = map[string]string{ + ctxt.Params = buildTaskParams(map[string]string{ "sonar-quality-gate": "true", - // "cache-build": "true", expected to be default now - } + }) }, WantRunSuccess: true, PostRunFunc: func(t *testing.T, ctxt *tasktesting.TaskRunContext) { @@ -83,10 +87,14 @@ func TestTaskODSBuildPython(t *testing.T) { "docker/app/requirements.txt", filepath.Join(pipelinectxt.XUnitReportsPath, "report.xml"), filepath.Join(pipelinectxt.CodeCoveragesPath, "coverage.xml"), - filepath.Join(pipelinectxt.SonarAnalysisPath, "analysis-report.md"), - filepath.Join(pipelinectxt.SonarAnalysisPath, "issues-report.csv"), - filepath.Join(pipelinectxt.SonarAnalysisPath, "quality-gate.json"), ) + if !*skipSonarQubeFlag { + checkFilesExist(t, wsDir, + filepath.Join(pipelinectxt.SonarAnalysisPath, "analysis-report.md"), + filepath.Join(pipelinectxt.SonarAnalysisPath, "issues-report.csv"), + filepath.Join(pipelinectxt.SonarAnalysisPath, "quality-gate.json"), + ) + } wantContainsBytes, err := os.ReadFile("../../test/testdata/golden/ods-build-python/excerpt-from-coverage.xml") if err != nil { @@ -100,8 +108,11 @@ func TestTaskODSBuildPython(t *testing.T) { wantContains = strings.ReplaceAll(wantContains, " ", "") checkFileContentLeanContains(t, wsDir, filepath.Join(pipelinectxt.CodeCoveragesPath, "coverage.xml"), wantContains) - sonarProject := sonar.ProjectKey(ctxt.ODS, "") - checkSonarQualityGate(t, ctxt.Clients.KubernetesClientSet, ctxt.Namespace, sonarProject, true, "OK") + + if !*skipSonarQubeFlag { + sonarProject := sonar.ProjectKey(ctxt.ODS, "") + checkSonarQualityGate(t, ctxt.Clients.KubernetesClientSet, ctxt.Namespace, sonarProject, true, "OK") + } // This is not available when build skipping as the default is // supplied on the second repeat. @@ -128,11 +139,11 @@ func TestTaskODSBuildPython(t *testing.T) { createAppInSubDirectory(t, wsDir, subdir, "python-fastapi-sample-app") ctxt.ODS = tasktesting.SetupGitRepo(t, ctxt.Namespace, wsDir) - ctxt.Params = map[string]string{ + ctxt.Params = buildTaskParams(map[string]string{ "sonar-quality-gate": "true", "working-dir": subdir, "cache-build": "true", - } + }) }, WantRunSuccess: true, PostRunFunc: func(t *testing.T, ctxt *tasktesting.TaskRunContext) { @@ -144,13 +155,16 @@ func TestTaskODSBuildPython(t *testing.T) { fmt.Sprintf("%s/docker/app/requirements.txt", subdir), filepath.Join(pipelinectxt.XUnitReportsPath, fmt.Sprintf("%s-report.xml", subdir)), filepath.Join(pipelinectxt.CodeCoveragesPath, fmt.Sprintf("%s-coverage.xml", subdir)), - filepath.Join(pipelinectxt.SonarAnalysisPath, fmt.Sprintf("%s-analysis-report.md", subdir)), - filepath.Join(pipelinectxt.SonarAnalysisPath, fmt.Sprintf("%s-issues-report.csv", subdir)), - filepath.Join(pipelinectxt.SonarAnalysisPath, fmt.Sprintf("%s-quality-gate.json", subdir)), ) - - sonarProject := sonar.ProjectKey(ctxt.ODS, subdir+"-") - checkSonarQualityGate(t, ctxt.Clients.KubernetesClientSet, ctxt.Namespace, sonarProject, true, "OK") + if !*skipSonarQubeFlag { + checkFilesExist(t, wsDir, + filepath.Join(pipelinectxt.SonarAnalysisPath, fmt.Sprintf("%s-analysis-report.md", subdir)), + filepath.Join(pipelinectxt.SonarAnalysisPath, fmt.Sprintf("%s-issues-report.csv", subdir)), + filepath.Join(pipelinectxt.SonarAnalysisPath, fmt.Sprintf("%s-quality-gate.json", subdir)), + ) + sonarProject := sonar.ProjectKey(ctxt.ODS, subdir+"-") + checkSonarQualityGate(t, ctxt.Clients.KubernetesClientSet, ctxt.Namespace, sonarProject, true, "OK") + } }, }, "build python fastapi app with pre-test script": { @@ -158,10 +172,9 @@ func TestTaskODSBuildPython(t *testing.T) { PreRunFunc: func(t *testing.T, ctxt *tasktesting.TaskRunContext) { wsDir := ctxt.Workspaces["source"] ctxt.ODS = tasktesting.SetupGitRepo(t, ctxt.Namespace, wsDir) - ctxt.Params = map[string]string{ - "sonar-skip": "true", + ctxt.Params = buildTaskParams(map[string]string{ "pre-test-script": "pre-test-script.sh", - } + }) }, WantRunSuccess: true, PostRunFunc: func(t *testing.T, ctxt *tasktesting.TaskRunContext) { diff --git a/test/tasks/ods-finish_test.go b/test/tasks/ods-finish_test.go index 6276780b..2f07f84f 100644 --- a/test/tasks/ods-finish_test.go +++ b/test/tasks/ods-finish_test.go @@ -28,7 +28,7 @@ func TestTaskODSFinish(t *testing.T) { PreRunFunc: func(t *testing.T, ctxt *tasktesting.TaskRunContext) { wsDir := ctxt.Workspaces["source"] ctxt.ODS = tasktesting.SetupBitbucketRepo( - t, ctxt.Clients.KubernetesClientSet, ctxt.Namespace, wsDir, tasktesting.BitbucketProjectKey, + t, ctxt.Clients.KubernetesClientSet, ctxt.Namespace, wsDir, tasktesting.BitbucketProjectKey, *privateCertFlag, ) ctxt.Params = map[string]string{ "pipeline-run-name": "foo", @@ -37,7 +37,7 @@ func TestTaskODSFinish(t *testing.T) { }, WantRunSuccess: true, PostRunFunc: func(t *testing.T, ctxt *tasktesting.TaskRunContext) { - bitbucketClient := tasktesting.BitbucketClientOrFatal(t, ctxt.Clients.KubernetesClientSet, ctxt.Namespace) + bitbucketClient := tasktesting.BitbucketClientOrFatal(t, ctxt.Clients.KubernetesClientSet, ctxt.Namespace, *privateCertFlag) checkBuildStatus(t, bitbucketClient, ctxt.ODS.GitCommitSHA, bitbucket.BuildStatusFailed) }, }, @@ -46,13 +46,13 @@ func TestTaskODSFinish(t *testing.T) { PreRunFunc: func(t *testing.T, ctxt *tasktesting.TaskRunContext) { wsDir := ctxt.Workspaces["source"] ctxt.ODS = tasktesting.SetupBitbucketRepo( - t, ctxt.Clients.KubernetesClientSet, ctxt.Namespace, wsDir, tasktesting.BitbucketProjectKey, + t, ctxt.Clients.KubernetesClientSet, ctxt.Namespace, wsDir, tasktesting.BitbucketProjectKey, *privateCertFlag, ) // Pretend there is alredy a coverage report in Nexus // this assures the safeguard is working to avoid duplicate upload. // TODO: assure the safeguard is actually invoked by checking the logs. t.Log("Uploading coverage artifact to Nexus and writing manifest") - nexusClient := tasktesting.NexusClientOrFatal(t, ctxt.Clients.KubernetesClientSet, ctxt.Namespace) + nexusClient := tasktesting.NexusClientOrFatal(t, ctxt.Clients.KubernetesClientSet, ctxt.Namespace, *privateCertFlag) _, err := nexusClient.Upload( nexus.TemporaryRepositoryDefault, pipelinectxt.ArtifactGroup(ctxt.ODS, pipelinectxt.CodeCoveragesDir), @@ -86,7 +86,7 @@ func TestTaskODSFinish(t *testing.T) { }, WantRunSuccess: true, PostRunFunc: func(t *testing.T, ctxt *tasktesting.TaskRunContext) { - bitbucketClient := tasktesting.BitbucketClientOrFatal(t, ctxt.Clients.KubernetesClientSet, ctxt.Namespace) + bitbucketClient := tasktesting.BitbucketClientOrFatal(t, ctxt.Clients.KubernetesClientSet, ctxt.Namespace, *privateCertFlag) checkBuildStatus(t, bitbucketClient, ctxt.ODS.GitCommitSHA, bitbucket.BuildStatusSuccessful) checkArtifactsAreInNexus(t, ctxt, nexus.TemporaryRepositoryDefault) @@ -101,7 +101,7 @@ func TestTaskODSFinish(t *testing.T) { PreRunFunc: func(t *testing.T, ctxt *tasktesting.TaskRunContext) { wsDir := ctxt.Workspaces["source"] ctxt.ODS = tasktesting.SetupBitbucketRepo( - t, ctxt.Clients.KubernetesClientSet, ctxt.Namespace, wsDir, tasktesting.BitbucketProjectKey, + t, ctxt.Clients.KubernetesClientSet, ctxt.Namespace, wsDir, tasktesting.BitbucketProjectKey, *privateCertFlag, ) err := createFinishODSYML(wsDir) if err != nil { @@ -114,7 +114,7 @@ func TestTaskODSFinish(t *testing.T) { }, WantRunSuccess: true, PostRunFunc: func(t *testing.T, ctxt *tasktesting.TaskRunContext) { - bitbucketClient := tasktesting.BitbucketClientOrFatal(t, ctxt.Clients.KubernetesClientSet, ctxt.Namespace) + bitbucketClient := tasktesting.BitbucketClientOrFatal(t, ctxt.Clients.KubernetesClientSet, ctxt.Namespace, *privateCertFlag) checkBuildStatus(t, bitbucketClient, ctxt.ODS.GitCommitSHA, bitbucket.BuildStatusSuccessful) checkArtifactsAreInNexus(t, ctxt, nexus.PermanentRepositoryDefault) }, @@ -154,7 +154,7 @@ func createFinishODSYML(wsDir string) error { func checkArtifactsAreInNexus(t *testing.T, ctxt *tasktesting.TaskRunContext, targetRepository string) { - nexusClient := tasktesting.NexusClientOrFatal(t, ctxt.Clients.KubernetesClientSet, ctxt.Namespace) + nexusClient := tasktesting.NexusClientOrFatal(t, ctxt.Clients.KubernetesClientSet, ctxt.Namespace, *privateCertFlag) // List of expected artifacts to have been uploaded to Nexus artifactsMap := map[string][]string{ diff --git a/test/tasks/ods-start_test.go b/test/tasks/ods-start_test.go index c26105c2..00f0776c 100644 --- a/test/tasks/ods-start_test.go +++ b/test/tasks/ods-start_test.go @@ -32,10 +32,10 @@ func TestTaskODSStart(t *testing.T) { PreRunFunc: func(t *testing.T, ctxt *tasktesting.TaskRunContext) { wsDir := ctxt.Workspaces["source"] ctxt.ODS = tasktesting.SetupBitbucketRepo( - t, ctxt.Clients.KubernetesClientSet, ctxt.Namespace, wsDir, tasktesting.BitbucketProjectKey, + t, ctxt.Clients.KubernetesClientSet, ctxt.Namespace, wsDir, tasktesting.BitbucketProjectKey, *privateCertFlag, ) - nexusClient := tasktesting.NexusClientOrFatal(t, ctxt.Clients.KubernetesClientSet, ctxt.Namespace) + nexusClient := tasktesting.NexusClientOrFatal(t, ctxt.Clients.KubernetesClientSet, ctxt.Namespace, *privateCertFlag) artifactsBaseDir := filepath.Join(projectpath.Root, "test", tasktesting.TestdataWorkspacesPath, "hello-world-app-with-artifacts", pipelinectxt.ArtifactsPath) // Upload artifact to permanent storage. _, err := nexusClient.Upload( @@ -62,7 +62,7 @@ func TestTaskODSStart(t *testing.T) { checkODSContext(t, wsDir, ctxt.ODS) - bitbucketClient := tasktesting.BitbucketClientOrFatal(t, ctxt.Clients.KubernetesClientSet, ctxt.Namespace) + bitbucketClient := tasktesting.BitbucketClientOrFatal(t, ctxt.Clients.KubernetesClientSet, ctxt.Namespace, *privateCertFlag) checkBuildStatus(t, bitbucketClient, ctxt.ODS.GitCommitSHA, bitbucket.BuildStatusInProgress) checkFilesExist(t, wsDir, @@ -85,7 +85,7 @@ func TestTaskODSStart(t *testing.T) { t.Fatal(err) } subCtxt := tasktesting.SetupBitbucketRepo( - t, ctxt.Clients.KubernetesClientSet, ctxt.Namespace, tempDir, tasktesting.BitbucketProjectKey, + t, ctxt.Clients.KubernetesClientSet, ctxt.Namespace, tempDir, tasktesting.BitbucketProjectKey, *privateCertFlag, ) subrepoContext = subCtxt err = os.RemoveAll(tempDir) @@ -97,10 +97,10 @@ func TestTaskODSStart(t *testing.T) { t.Fatal(err) } ctxt.ODS = tasktesting.SetupBitbucketRepo( - t, ctxt.Clients.KubernetesClientSet, ctxt.Namespace, wsDir, tasktesting.BitbucketProjectKey, + t, ctxt.Clients.KubernetesClientSet, ctxt.Namespace, wsDir, tasktesting.BitbucketProjectKey, *privateCertFlag, ) - nexusClient := tasktesting.NexusClientOrFatal(t, ctxt.Clients.KubernetesClientSet, ctxt.Namespace) + nexusClient := tasktesting.NexusClientOrFatal(t, ctxt.Clients.KubernetesClientSet, ctxt.Namespace, *privateCertFlag) artifactsBaseDir := filepath.Join(projectpath.Root, "test", tasktesting.TestdataWorkspacesPath, "hello-world-app-with-artifacts", pipelinectxt.ArtifactsPath) _, err = nexusClient.Upload( nexus.TemporaryRepositoryDefault, @@ -148,7 +148,7 @@ func TestTaskODSStart(t *testing.T) { checkFileContent(t, destinationArtifactsBaseDir, xUnitFileSource, xUnitContent) checkFilesExist(t, destinationArtifactsBaseDir, pipelinectxt.ArtifactsManifestFilename) - bitbucketClient := tasktesting.BitbucketClientOrFatal(t, ctxt.Clients.KubernetesClientSet, ctxt.Namespace) + bitbucketClient := tasktesting.BitbucketClientOrFatal(t, ctxt.Clients.KubernetesClientSet, ctxt.Namespace, *privateCertFlag) checkBuildStatus(t, bitbucketClient, ctxt.ODS.GitCommitSHA, bitbucket.BuildStatusInProgress) }, @@ -167,7 +167,7 @@ func TestTaskODSStart(t *testing.T) { t.Fatal(err) } tasktesting.SetupBitbucketRepo( - t, ctxt.Clients.KubernetesClientSet, ctxt.Namespace, tempDir, tasktesting.BitbucketProjectKey, + t, ctxt.Clients.KubernetesClientSet, ctxt.Namespace, tempDir, tasktesting.BitbucketProjectKey, *privateCertFlag, ) err = os.RemoveAll(tempDir) if err != nil { @@ -178,7 +178,7 @@ func TestTaskODSStart(t *testing.T) { t.Fatal(err) } ctxt.ODS = tasktesting.SetupBitbucketRepo( - t, ctxt.Clients.KubernetesClientSet, ctxt.Namespace, wsDir, tasktesting.BitbucketProjectKey, + t, ctxt.Clients.KubernetesClientSet, ctxt.Namespace, wsDir, tasktesting.BitbucketProjectKey, *privateCertFlag, ) ctxt.Params = map[string]string{ "url": ctxt.ODS.GitURL, @@ -209,7 +209,7 @@ func TestTaskODSStart(t *testing.T) { t.Fatal(err) } ctxt.ODS = tasktesting.SetupBitbucketRepo( - t, ctxt.Clients.KubernetesClientSet, ctxt.Namespace, wsDir, tasktesting.BitbucketProjectKey, + t, ctxt.Clients.KubernetesClientSet, ctxt.Namespace, wsDir, tasktesting.BitbucketProjectKey, *privateCertFlag, ) ctxt.Params = map[string]string{ @@ -239,11 +239,11 @@ func TestTaskODSStart(t *testing.T) { t.Fatal(err) } ctxt.ODS = tasktesting.SetupBitbucketRepo( - t, ctxt.Clients.KubernetesClientSet, ctxt.Namespace, wsDir, tasktesting.BitbucketProjectKey, + t, ctxt.Clients.KubernetesClientSet, ctxt.Namespace, wsDir, tasktesting.BitbucketProjectKey, *privateCertFlag, ) // pretend there is already an RC tag for the current commit - bitbucketClient := tasktesting.BitbucketClientOrFatal(t, ctxt.Clients.KubernetesClientSet, ctxt.Namespace) + bitbucketClient := tasktesting.BitbucketClientOrFatal(t, ctxt.Clients.KubernetesClientSet, ctxt.Namespace, *privateCertFlag) _, err = bitbucketClient.TagCreate( ctxt.ODS.Project, ctxt.ODS.Repository, @@ -282,7 +282,7 @@ func TestTaskODSStart(t *testing.T) { PreRunFunc: func(t *testing.T, ctxt *tasktesting.TaskRunContext) { wsDir := ctxt.Workspaces["source"] ctxt.ODS = tasktesting.SetupBitbucketRepo( - t, ctxt.Clients.KubernetesClientSet, ctxt.Namespace, wsDir, tasktesting.BitbucketProjectKey, + t, ctxt.Clients.KubernetesClientSet, ctxt.Namespace, wsDir, tasktesting.BitbucketProjectKey, *privateCertFlag, ) tasktesting.EnableLfsOnBitbucketRepoOrFatal(t, filepath.Base(wsDir), tasktesting.BitbucketProjectKey) lfsFilename = "lfspicture.jpg" @@ -303,7 +303,7 @@ func TestTaskODSStart(t *testing.T) { checkODSContext(t, wsDir, ctxt.ODS) - bitbucketClient := tasktesting.BitbucketClientOrFatal(t, ctxt.Clients.KubernetesClientSet, ctxt.Namespace) + bitbucketClient := tasktesting.BitbucketClientOrFatal(t, ctxt.Clients.KubernetesClientSet, ctxt.Namespace, *privateCertFlag) checkBuildStatus(t, bitbucketClient, ctxt.ODS.GitCommitSHA, bitbucket.BuildStatusInProgress) checkFileHash(t, wsDir, lfsFilename, lfsFileHash) @@ -314,7 +314,7 @@ func TestTaskODSStart(t *testing.T) { } func checkForTag(t *testing.T, ctxt *tasktesting.TaskRunContext, wantTag string) *bitbucket.Tag { - bitbucketClient := tasktesting.BitbucketClientOrFatal(t, ctxt.Clients.KubernetesClientSet, ctxt.Namespace) + bitbucketClient := tasktesting.BitbucketClientOrFatal(t, ctxt.Clients.KubernetesClientSet, ctxt.Namespace, *privateCertFlag) var gotTag *bitbucket.Tag tagPage, err := bitbucketClient.TagList( ctxt.ODS.Project, diff --git a/test/testdata/private-cert/README.md b/test/testdata/private-cert/README.md new file mode 100644 index 00000000..b92584ac --- /dev/null +++ b/test/testdata/private-cert/README.md @@ -0,0 +1,9 @@ +This folder contains a self-signed certificate for testing purposes ONLY. + +The files were created like this: +``` +openssl req -config openssl.conf -new -newkey rsa:2048 -days 825 -nodes -x509 \ + -keyout tls.key -out tls.crt -extensions req_ext +``` + +Expiry of 825 days is due to an MacOS/Go issue, see https://myupbeat.wordpress.com/2022/09/09/self-signed-certificates-not-standards-compliant/. diff --git a/test/testdata/private-cert/openssl.conf b/test/testdata/private-cert/openssl.conf new file mode 100644 index 00000000..3945b16b --- /dev/null +++ b/test/testdata/private-cert/openssl.conf @@ -0,0 +1,23 @@ +[ req ] +default_bits = 2048 +prompt = no +distinguished_name = dn +req_extensions = req_ext +default_md = sha256 + +[ dn ] +C = DE +ST = RLP +L = Ingelheim +O = BI X Digital GmbH +CN = localhost +emailAddress = info@bix-digital.com + +[ req_ext ] +subjectAltName = @alt_names + +[ alt_names ] +DNS.1 = ods-test-nexus-tls.kind +DNS.2 = ods-test-sonarqube-tls.kind +DNS.3 = ods-test-bitbucket-server-tls.kind +DNS.4 = localhost diff --git a/test/testdata/private-cert/tls.crt b/test/testdata/private-cert/tls.crt new file mode 100644 index 00000000..43281cca --- /dev/null +++ b/test/testdata/private-cert/tls.crt @@ -0,0 +1,24 @@ +-----BEGIN CERTIFICATE----- +MIID/zCCAuegAwIBAgIJAJYtytgvWvyvMA0GCSqGSIb3DQEBCwUAMIGEMQswCQYD +VQQGEwJERTEMMAoGA1UECAwDUkxQMRIwEAYDVQQHDAlJbmdlbGhlaW0xGjAYBgNV +BAoMEUJJIFggRGlnaXRhbCBHbWJIMRIwEAYDVQQDDAlsb2NhbGhvc3QxIzAhBgkq +hkiG9w0BCQEWFGluZm9AYml4LWRpZ2l0YWwuY29tMB4XDTIzMDEyNjE0NDQwNVoX +DTI1MDQzMDE0NDQwNVowgYQxCzAJBgNVBAYTAkRFMQwwCgYDVQQIDANSTFAxEjAQ +BgNVBAcMCUluZ2VsaGVpbTEaMBgGA1UECgwRQkkgWCBEaWdpdGFsIEdtYkgxEjAQ +BgNVBAMMCWxvY2FsaG9zdDEjMCEGCSqGSIb3DQEJARYUaW5mb0BiaXgtZGlnaXRh +bC5jb20wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCyvBn9o26Q7Gp+ +4v4EZQZCBDJ4ylPxcIRA6U5bt/sKKsS7wgj5TMBxuElIZaSvU8F/RxXR1IhrECf6 +lsfaigrXK/rn83OgXVLRtmO5qOI8OgZvRvqv63uFS6xMp5uhbXyOVNeJS+CuEkiK +gWysgdFb39LbzS3pAp3pD/9Y/SrKnOn7Q9zIOUsEWVTz2eu4jO2GsaqCvu4LJBpa +1AXeiV58pTSfSLS/D6kJFWQ8FeVsmAK0B5X2gzXewMb2Z9aSHcLAkLR9qvWnvHdi +Iz/TgeX37xFF9QCIwgFHMJvTDfqQv573PmiGyXrMIslxI+VRFHeRjZJLBtLNQhD9 +Q4N4UF+RAgMBAAGjcjBwMG4GA1UdEQRnMGWCF29kcy10ZXN0LW5leHVzLXRscy5r +aW5kghtvZHMtdGVzdC1zb25hcnF1YmUtdGxzLmtpbmSCIm9kcy10ZXN0LWJpdGJ1 +Y2tldC1zZXJ2ZXItdGxzLmtpbmSCCWxvY2FsaG9zdDANBgkqhkiG9w0BAQsFAAOC +AQEAfRkgRRAfe1wPGz3405UcpLCD8fkktPoCOEidVQacs9X+0cZmkrrHnGSjDQgW +oioMyPxaFSBMngTZYWppFCIDVGSoM2ugyoizwgs+S/mxSnXr17EqT3QVIsHta9nL +QplJlLjqyfbFt2cbYHIUA/n3IapFEWjDuvxhM4u2UkcRRs4Y5IMz9ma9UlDGNK8s +pPk7AUZUBlVe00xHiGZbAW6CtWGgnlZfEGDhzerEJx0v2FzLPUKJfJw44eeG5Bbc +DK6qy4Or7piW75Qd710Qp6cptJECe67KuvpTe05GK5vju12h1UPJJD82SN4upwPU +6J5RBA7vupp39MnLAPL6h9BInA== +-----END CERTIFICATE----- diff --git a/test/testdata/private-cert/tls.key b/test/testdata/private-cert/tls.key new file mode 100644 index 00000000..7c01e536 --- /dev/null +++ b/test/testdata/private-cert/tls.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCyvBn9o26Q7Gp+ +4v4EZQZCBDJ4ylPxcIRA6U5bt/sKKsS7wgj5TMBxuElIZaSvU8F/RxXR1IhrECf6 +lsfaigrXK/rn83OgXVLRtmO5qOI8OgZvRvqv63uFS6xMp5uhbXyOVNeJS+CuEkiK +gWysgdFb39LbzS3pAp3pD/9Y/SrKnOn7Q9zIOUsEWVTz2eu4jO2GsaqCvu4LJBpa +1AXeiV58pTSfSLS/D6kJFWQ8FeVsmAK0B5X2gzXewMb2Z9aSHcLAkLR9qvWnvHdi +Iz/TgeX37xFF9QCIwgFHMJvTDfqQv573PmiGyXrMIslxI+VRFHeRjZJLBtLNQhD9 +Q4N4UF+RAgMBAAECggEAYa0IKyNwXgTFIjichvxKkjNG/dncS/qoW+zkJJ2DtIew +4cblLpByLvN+IPFjMWl9Ojg3xWb3ckLvBjP2XgjIadSgFrhlvQKjKvYYw7J2iGFR +eOh6QT9YTn0PWQjTtUFnK0VUULTFwXs+Z2rZX+DF8AZ+bh5P86G5eE+ExTX9d6Gl +G/CdW3DnFBkfyweI9oZkssRcMShsbJqpss9fJ2PFNxZ/+KLlBdzkSWiRji+LQDpV +hNld6GkAX779j5ZCEX8TmSGv0788/c53LLdk/DQGlk21k1HCjrgUSE4eYnHErJ0W +nYtcBVxao+KtN3cvi2qZMmUkX3HRV2aUbY1ZWWdE7QKBgQDcKWlgV3R7L9BJxr8l +iWpJqVCdvF0rygtbvDaXsbaLL5AB60Q1cPGJBK/1LVM5CUH9KSUNklZCXiR0xCa2 +uhcgotu3PR/EmMI4p1njeGla/pF04tKBb4wYOc7m+VDxxOAQR2n3jtsduofiu/3q +lB0SxxkENHSNqcAX8dBFvsW6qwKBgQDP1FfHTKnGZSJ+jGW0fib0ryzOffhGH+ia +CmsRIiGC/SwFTJMvWeI19XrOXman9T93AY3gHk5axrbNOInkU1USsW3uz3UMqyPs +m1T9JkJJq0X7IbcdhRTPBjPOIrWr1gDi2SFZGMBWrUhoPzhp5ah0qLq2eCNYHy8P +H5vR84yOswKBgEltvmmVgZH8dnnEwcfhNwPuZqbVRjXuoy8rbQHXEc0GRf992jgA +vW43lAAv58vkHJA1Atit7/serwYKzeav1jraVnGc/1VwpYaDumU1qd3hfd4AMLle +3NjPnwRGLwsT4K9NQjiA+u/FYMDsdlmgoLm8kclnPWBqN1lFG3PHlucVAoGBALAn +Du+B7zSVbBNO9LVzZ5llENlEtDbbvM8znWQ8RaM86x7itK4hxTZvBtP7947MtCou +Sw53IXqXi0GOdnHsn+LkOIV9mXxPYnedgP/AedbvMLWW8ho3PlfqreY+TTWEgePE +nhtie0P8Q674RtybJ5kZmogmc+6xH7X12KCviac5AoGBANny5b2iS/85Yy/ydb7J +WNHUQHDeN9oD0/ExD/EKpDAsK8q0gP305wWOexPYe1DZ3c4Ho6/qktRosb8QHe6K +QXf0lI01DAS/qMQuaIuehIOwHEDrI0+U1tIOPg36KVuF7RKrN68JN35FA18UK1WX +tiYrJ9MzhKZ3RIz6PDFx8ZS2 +-----END PRIVATE KEY-----