From d1c5c083b75c77c1e262ac2bcaa6c9ed4c56fb30 Mon Sep 17 00:00:00 2001 From: Scott K Logan Date: Mon, 10 Oct 2022 16:47:53 -0700 Subject: [PATCH] Disable Pulp operations in Jenkins RPM jobs This change leaves much of the Pulp infrastructure code present, but removes Pulp completely from the job execution workflows. --- .../release/rpm/binarypkg_job.xml.em | 45 ------------------- .../release/rpm/import_upstream_job.xml.em | 43 ------------------ .../release/rpm/sourcepkg_job.xml.em | 26 ----------- .../rpm/sync_packages_to_main_job.xml.em | 35 --------------- .../rpm/sync_packages_to_testing_job.xml.em | 28 ------------ 5 files changed, 177 deletions(-) diff --git a/ros_buildfarm/templates/release/rpm/binarypkg_job.xml.em b/ros_buildfarm/templates/release/rpm/binarypkg_job.xml.em index 7c3c8366e..f15b307d6 100644 --- a/ros_buildfarm/templates/release/rpm/binarypkg_job.xml.em +++ b/ros_buildfarm/templates/release/rpm/binarypkg_job.xml.em @@ -155,46 +155,6 @@ but disabled since the package is blacklisted (or not whitelisted) in the config 'echo "# END SECTION"', ]), ))@ -@(SNIPPET( - 'builder_shell', - script='\n'.join([ - 'echo "# BEGIN SECTION: Upload binaryrpm to Pulp"', - 'export PYTHONPATH=$WORKSPACE/ros_buildfarm:$PYTHONPATH', - "ls binarypkg/*.rpm | grep -v -e 'src\.rpm$' -e '-debug\(info\|source\)-' > binarypkg/upload_list.txt && " + - 'xargs -a binarypkg/upload_list.txt' + - ' python3 -u $WORKSPACE/ros_buildfarm/scripts/release/rpm/upload_package.py' + - ' --pulp-resource-record binarypkg/upload_record.txt', - 'echo "# END SECTION"', - 'echo "# BEGIN SECTION: Upload debug symbols"', - 'export PYTHONPATH=$WORKSPACE/ros_buildfarm:$PYTHONPATH', - "ls binarypkg/*.rpm | grep -e '-debug\(info\|source\)-' > binarypkg/upload_list_debug.txt && " + - 'xargs -a binarypkg/upload_list_debug.txt' + - ' python3 -u $WORKSPACE/ros_buildfarm/scripts/release/rpm/upload_package.py' + - ' --pulp-resource-record binarypkg/upload_record_debug.txt', - 'echo "# END SECTION"', - ]), -))@ -@(SNIPPET( - 'builder_parameterized-trigger', - project=import_package_job_name, - parameters='\n'.join([ - 'DISTRIBUTION_NAME=ros-building-%s-%s-%s' % ( - os_name, os_code_name, arch), - 'INVALIDATE_DOWNSTREAM=true']), - parameter_files=['binarypkg/upload_record.txt'], - continue_on_failure=False, -))@ -@(SNIPPET( - 'builder_parameterized-trigger', - project=import_package_job_name, - parameters='\n'.join([ - 'DISTRIBUTION_NAME=ros-building-%s-%s-%s-debug' % ( - os_name, os_code_name, arch), - 'INVALIDATE_DOWNSTREAM=false']), - parameter_files=['binarypkg/upload_record_debug.txt'], - continue_on_failure=False, - missing_parameter_files_skip=True, -))@ @(SNIPPET( 'builder_shell', script='\n'.join([ @@ -230,11 +190,6 @@ but disabled since the package is blacklisted (or not whitelisted) in the config ))@ -@(SNIPPET( - 'pulp_credentials', - credential_id=credential_id_pulp, - dest_credential_id=dest_credential_id, -))@ @[if timeout_minutes is not None]@ @(SNIPPET( 'build-wrapper_build-timeout', diff --git a/ros_buildfarm/templates/release/rpm/import_upstream_job.xml.em b/ros_buildfarm/templates/release/rpm/import_upstream_job.xml.em index 0046efb31..ffa9bf361 100644 --- a/ros_buildfarm/templates/release/rpm/import_upstream_job.xml.em +++ b/ros_buildfarm/templates/release/rpm/import_upstream_job.xml.em @@ -27,24 +27,6 @@ 'description': 'A specific text file or files containing RPM base URLs to import from. The default is the ros_bootstrap repository.', 'default_value': '/home/jenkins-agent/ros_bootstrap_rpm_urls.txt', }, - { - 'type': 'string', - 'name': 'REMOTE_SOURCE_EXPRESSION', - 'description': 'Expression to match pulp remote repositories', - 'default_value': '^ros-upstream-[^-]+-([^-]+-[^-]+-[^-]+(-debug)?)$', - }, - { - 'type': 'string', - 'name': 'DISTRIBUTION_DEST_EXPRESSION', - 'description': 'Expression to transform source remote to destination pulp distribution', - 'default_value': '^ros-(building|testing|main)-\\1$', - }, - { - 'type': 'boolean', - 'name': 'EXECUTE_IMPORT', - 'description': 'If this is not true, it will only do a dry run and print the expect import, but not execute.', - 'default_value': 'false', - }, ], ))@ @(SNIPPET( @@ -85,26 +67,6 @@ 'done', 'echo "# END SECTION"', ]), -))@ -@(SNIPPET( - 'builder_shell', - script='\n'.join([ - 'echo "# BEGIN SECTION: mirror repositories to pulp"', - 'export PYTHONPATH=$WORKSPACE/ros_buildfarm:$PYTHONPATH', - 'python3 -u $WORKSPACE/ros_buildfarm/scripts/release/rpm/mirror_repo.py' + - ' --remote-source-expression $REMOTE_SOURCE_EXPRESSION' + - ' --distribution-dest-expression "^\\g<0>$"', - 'echo "# END SECTION"', - '', - 'echo "# BEGIN SECTION: sync packages from mirrors to repos"', - 'if [ "$EXECUTE_IMPORT" != "true" ]; then DRY_RUN_ARG="--dry-run"; fi', - 'export PYTHONPATH=$WORKSPACE/ros_buildfarm:$PYTHONPATH', - 'python3 -u $WORKSPACE/ros_buildfarm/scripts/release/rpm/sync_repo.py' + - ' --distribution-source-expression $REMOTE_SOURCE_EXPRESSION' + - ' --distribution-dest-expression $DISTRIBUTION_DEST_EXPRESSION' + - ' $DRY_RUN_ARG', - 'echo "# END SECTION"', - ]), ))@ @@ -116,11 +78,6 @@ ))@ -@(SNIPPET( - 'pulp_credentials', - credential_id=credential_id_pulp, - dest_credential_id=dest_credential_id, -))@ @(SNIPPET( 'build-wrapper_timestamper', ))@ diff --git a/ros_buildfarm/templates/release/rpm/sourcepkg_job.xml.em b/ros_buildfarm/templates/release/rpm/sourcepkg_job.xml.em index af4ac8455..9cb85896b 100644 --- a/ros_buildfarm/templates/release/rpm/sourcepkg_job.xml.em +++ b/ros_buildfarm/templates/release/rpm/sourcepkg_job.xml.em @@ -134,27 +134,6 @@ but disabled since the package is blacklisted (or not whitelisted) in the config 'echo "# END SECTION"', ]), ))@ -@(SNIPPET( - 'builder_shell', - script='\n'.join([ - 'echo "# BEGIN SECTION: Upload sourcerpm to Pulp"', - 'export TZ="%s"' % timezone, - 'export PYTHONPATH=$WORKSPACE/ros_buildfarm:$PYTHONPATH', - 'python3 -u $WORKSPACE/ros_buildfarm/scripts/release/rpm/upload_package.py' + - ' --pulp-resource-record sourcepkg/upload_record.txt' + - ' sourcepkg/*.src.rpm', - 'echo "# END SECTION"', - ]), -))@ -@(SNIPPET( - 'builder_parameterized-trigger', - project=import_package_job_name, - parameters='\n'.join([ - 'DISTRIBUTION_NAME=ros-building-%s-%s-SRPMS' % ( - os_name, os_code_name)]), - parameter_files=['sourcepkg/upload_record.txt'], - continue_on_failure=True, -))@ @(SNIPPET( 'builder_shell', script='\n'.join([ @@ -193,11 +172,6 @@ but disabled since the package is blacklisted (or not whitelisted) in the config ))@ -@(SNIPPET( - 'pulp_credentials', - credential_id=credential_id_pulp, - dest_credential_id=dest_credential_id, -))@ @[if timeout_minutes is not None]@ @(SNIPPET( 'build-wrapper_build-timeout', diff --git a/ros_buildfarm/templates/release/rpm/sync_packages_to_main_job.xml.em b/ros_buildfarm/templates/release/rpm/sync_packages_to_main_job.xml.em index bad4b52fa..ad37973b3 100644 --- a/ros_buildfarm/templates/release/rpm/sync_packages_to_main_job.xml.em +++ b/ros_buildfarm/templates/release/rpm/sync_packages_to_main_job.xml.em @@ -53,36 +53,6 @@ ] + [ 'echo "# END SECTION"', ]), -))@ -@(SNIPPET( - 'builder_shell', - script='\n'.join([ - 'echo "# BEGIN SECTION: sync packages to main repos in Pulp"', - 'export PYTHONPATH=$WORKSPACE/ros_buildfarm:$PYTHONPATH', - 'python3 -u $WORKSPACE/ros_buildfarm/scripts/release/rpm/sync_repo.py' + - ' --distribution-source-expression "^ros-testing-([^-]*-[^-]*-[^-]*(-debug)?)$"' + - ' --distribution-dest-expression "^ros-main-\\1$"' + - ' --package-name-expression "^ros-%s-.*"' % rosdistro_name + - ' --invalidate-expression "^ros-%s-.*"' % rosdistro_name, - 'echo "# END SECTION"', - ]), -))@ -@(SNIPPET( - 'builder_shell', - script='\n'.join([ - 'echo "# BEGIN SECTION: mirror main Pulp repository content to disk"', - ] + [ - 'rsync --recursive --times --delete --itemize-changes rsync://127.0.0.1:1234/ros-main-%s-%s-SRPMS/ /var/repos/%s_pulp/main/%s/SRPMS/' % (os_name, os_code_name, os_name, os_code_name) - for os_name, os_versions in sync_targets.items() for os_code_name in os_versions - ] + [ - 'rsync --recursive --times --delete --exclude=debug --itemize-changes rsync://127.0.0.1:1234/ros-main-%s-%s-%s/ /var/repos/%s_pulp/main/%s/%s/' % (os_name, os_code_name, arch, os_name, os_code_name, arch) - for os_name, os_versions in sync_targets.items() for os_code_name, arches in os_versions.items() for arch in arches - ] + [ - 'rsync --recursive --times --delete --itemize-changes rsync://127.0.0.1:1234/ros-main-%s-%s-%s-debug/ /var/repos/%s_pulp/main/%s/%s/debug/' % (os_name, os_code_name, arch, os_name, os_code_name, arch) - for os_name, os_versions in sync_targets.items() for os_code_name, arches in os_versions.items() for arch in arches - ] + [ - 'echo "# END SECTION"', - ]), ))@ @@ -94,11 +64,6 @@ ))@ -@(SNIPPET( - 'pulp_credentials', - credential_id=credential_id_pulp, - dest_credential_id=dest_credential_id, -))@ @(SNIPPET( 'build-wrapper_timestamper', ))@ diff --git a/ros_buildfarm/templates/release/rpm/sync_packages_to_testing_job.xml.em b/ros_buildfarm/templates/release/rpm/sync_packages_to_testing_job.xml.em index c7c880fca..0035c5f94 100644 --- a/ros_buildfarm/templates/release/rpm/sync_packages_to_testing_job.xml.em +++ b/ros_buildfarm/templates/release/rpm/sync_packages_to_testing_job.xml.em @@ -98,29 +98,6 @@ 'createrepo-agent /var/repos/%s/testing/%s/ --sync=/var/repos/%s/building/%s/ --arch=SRPMS --arch=%s --sync-pattern="ros-%s-.*" --invalidate-family' % (os_name, os_code_name, os_name, os_code_name, arch, rosdistro_name), 'echo "# END SECTION"', ]), -))@ -@(SNIPPET( - 'builder_shell', - script='\n'.join([ - 'echo "# BEGIN SECTION: sync packages to testing repos"', - 'export PYTHONPATH=$WORKSPACE/ros_buildfarm:$PYTHONPATH', - 'python3 -u $WORKSPACE/ros_buildfarm/scripts/release/rpm/sync_repo.py' + - ' --distribution-source-expression "^ros-building-%s-%s-(SRPMS|%s(-debug)?)$"' % (os_name, os_code_name, arch) + - ' --distribution-dest-expression "^ros-testing-%s-%s-\\1$"' % (os_name, os_code_name) + - ' --package-name-expression "^ros-%s-.*"' % rosdistro_name + - ' --invalidate-expression "^ros-%s-.*"' % rosdistro_name, - 'echo "# END SECTION"', - ]), -))@ -@(SNIPPET( - 'builder_shell', - script='\n'.join([ - 'echo "# BEGIN SECTION: mirror testing repository content to disk"', - 'rsync --recursive --times --delete --itemize-changes rsync://127.0.0.1:1234/ros-testing-%s-%s-SRPMS/ /var/repos/%s_pulp/testing/%s/SRPMS/' % (os_name, os_code_name, os_name, os_code_name), - 'rsync --recursive --times --delete --exclude=debug --itemize-changes rsync://127.0.0.1:1234/ros-testing-%s-%s-%s/ /var/repos/%s_pulp/testing/%s/%s/' % (os_name, os_code_name, arch, os_name, os_code_name, arch), - 'rsync --recursive --times --delete --itemize-changes rsync://127.0.0.1:1234/ros-testing-%s-%s-%s-debug/ /var/repos/%s_pulp/testing/%s/%s/debug/' % (os_name, os_code_name, arch, os_name, os_code_name, arch), - 'echo "# END SECTION"', - ]), ))@ @@ -132,11 +109,6 @@ ))@ -@(SNIPPET( - 'pulp_credentials', - credential_id=credential_id_pulp, - dest_credential_id=dest_credential_id, -))@ @(SNIPPET( 'build-wrapper_timestamper', ))@