Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Switch from Pulp to createrepo-agent for RPMs #976

Merged
merged 15 commits into from
Sep 20, 2022
Merged

Switch from Pulp to createrepo-agent for RPMs #976

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
15 commits
Select commit Hold shift + click to select a range
3fd3fb3
Initial createrepo-agent support
cottsay Jun 14, 2022
10d2f77
Perform binary package invalidation
cottsay Jun 16, 2022
c481aed
Add support for createrepo-agent sync jobs
cottsay Aug 18, 2022
2e96c5f
Fix sync-to-main snippet for createrepo-agent
cottsay Aug 18, 2022
6947409
Don't need ssh when the job is already on the repo host
cottsay Aug 18, 2022
b0b7623
Disable pulp during package imports
cottsay Jun 22, 2022
340358c
Disable pulp during repository sync
cottsay Aug 18, 2022
146172b
Disable unused pulp-centric jobs
cottsay Aug 25, 2022
21aef18
Merge branch 'master' into cottsay/createrepo-agent-default
cottsay Aug 25, 2022
b9f2e08
Re-work credentials for uploading RPMs
cottsay Aug 25, 2022
3f34fce
Merge branch 'master' into cottsay/createrepo-agent-default
cottsay Aug 29, 2022
7caeabf
Re-add Pulp stuff for parallel use with createrepo-agent
cottsay Aug 29, 2022
bb2443d
Cleanups and add creatrepo-agent-based import_upstream
cottsay Aug 30, 2022
eca2625
Fix a trivial typo
cottsay Aug 30, 2022
c0281a9
Add doc for new config parameters
cottsay Aug 30, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions doc/configuration_options.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -237,6 +237,13 @@ The following options are valid in version ``2`` (beside the generic options):
* ``upload_credential_id``: the ID of the credential to upload the built
packages to the repository host.

* ``upload_credential_id_pulp``: the ID of the credential to upload the built
RPM packages to the repository host instance of Pulp.

* ``upload_host``: the hostname of the repository host where built packages
shoudl be uploaded to.
Only affects RPM builds at present.

* ``package_dependecy_behavior``: a dictionary with the following optional
keys:

Expand Down
8 changes: 8 additions & 0 deletions ros_buildfarm/config/release_build_file.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -109,10 +109,18 @@ def __init__(self, name, data): # noqa: D107
assert 'upload_credential_id' in data
self.upload_credential_id = data['upload_credential_id']

self.upload_credential_id_pulp = None
if 'upload_credential_id_pulp' in data:
self.upload_credential_id_pulp = data['upload_credential_id_pulp']

self.upload_destination_credential_id = None
if 'upload_destination_credential_id' in data:
self.upload_destination_credential_id = data['upload_destination_credential_id']

self.upload_host = None
if 'upload_host' in data:
self.upload_host = data['upload_host']

self.include_test_dependencies = True
self.run_package_tests = True
if data.get('package_dependency_behavior'):
Expand Down
18 changes: 8 additions & 10 deletions ros_buildfarm/release_job.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -655,7 +655,9 @@ def _get_sourcedeb_job_config(

'timeout_minutes': build_file.jenkins_source_job_timeout,

'upload_host': build_file.upload_host,
'credential_id': build_file.upload_credential_id,
'credential_id_pulp': build_file.upload_credential_id_pulp,
'dest_credential_id': build_file.upload_destination_credential_id,

'git_ssh_credential_id': config.git_ssh_credential_id,
Expand Down Expand Up @@ -742,7 +744,9 @@ def _get_binarydeb_job_config(

'timeout_minutes': build_file.jenkins_binary_job_timeout,

'upload_host': build_file.upload_host,
'credential_id': build_file.upload_credential_id,
'credential_id_pulp': build_file.upload_credential_id_pulp,
'dest_credential_id': build_file.upload_destination_credential_id,

'shared_ccache': build_file.shared_ccache,
Expand Down Expand Up @@ -791,7 +795,7 @@ def _get_import_package_job_config(build_file, package_format):
'abi_incompatibility_assumed': build_file.abi_incompatibility_assumed,
'notify_emails': build_file.notify_emails,
'ros_buildfarm_repository': get_repository(),
'credential_id': build_file.upload_credential_id,
'credential_id_pulp': build_file.upload_credential_id_pulp,
'dest_credential_id': build_file.upload_destination_credential_id,
}
job_config = expand_template(template_name, job_data)
Expand Down Expand Up @@ -857,7 +861,7 @@ def _get_sync_packages_to_testing_job_config(
rosdistro_name, package_format),

'notify_emails': build_file.notify_emails,
'credential_id': build_file.upload_credential_id,
'credential_id_pulp': build_file.upload_credential_id_pulp,
'dest_credential_id': build_file.upload_destination_credential_id,
}
job_config = expand_template(template_name, job_data)
Expand Down Expand Up @@ -900,22 +904,16 @@ def get_sync_packages_to_main_job_name(rosdistro_name, package_format):


def _get_sync_packages_to_main_job_config(rosdistro_name, build_file, package_format):
sync_targets = set()
for os_name, os_versions in build_file.targets.items():
for os_code_name, os_arches in os_versions.items():
for os_arch in os_arches.keys():
sync_targets.add((os_name, os_code_name, os_arch))

template_name = 'release/%s/sync_packages_to_main_job.xml.em' % package_format
job_data = {
'ros_buildfarm_repository': get_repository(),
'rosdistro_name': rosdistro_name,

'deb_sync_to_main_job_name': get_sync_packages_to_main_job_name(rosdistro_name, 'deb'),
'sync_targets': sync_targets,
'sync_targets': build_file.targets,

'notify_emails': build_file.notify_emails,
'credential_id': build_file.upload_credential_id,
'credential_id_pulp': build_file.upload_credential_id_pulp,
'dest_credential_id': build_file.upload_destination_credential_id,
}
job_config = expand_template(template_name, job_data)
Expand Down
2 changes: 2 additions & 0 deletions ros_buildfarm/scripts/release/generate_release_maintenance_jobs.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -122,7 +122,9 @@ def get_trigger_missed_jobs_job_config(args, config, build_file):
def get_import_upstream_job_config(args, config, build_file, package_format):
template_name = 'release/%s/import_upstream_job.xml.em' % package_format
data = {
'import_targets': build_file.targets,
'credential_id': build_file.upload_credential_id,
'credential_id_pulp': build_file.upload_credential_id_pulp,
'dest_credential_id': build_file.upload_destination_credential_id,
}
return _get_job_config(
Expand Down
18 changes: 17 additions & 1 deletion ros_buildfarm/templates/release/rpm/binarypkg_job.xml.em
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -147,6 +147,18 @@ but disabled since the package is blacklisted (or not whitelisted) in the config
'builder_shell',
script='\n'.join([
'echo "# BEGIN SECTION: Upload binaryrpm"',
'find binarypkg -mindepth 1 -maxdepth 1 -type f -name "*.rpm" -not -name "*.src.rpm" -fprint binarypkg/rpm_upload_args.txt -fprintf binarypkg/rpm_import_args.txt "--import=/tmp/upload-${BUILD_TAG}/%f\\n"',
'ssh %s -- mkdir -p /tmp/upload-${BUILD_TAG}/' % (upload_host,),
'xargs -a binarypkg/rpm_upload_args.txt -I @ scp @ %s:/tmp/upload-${BUILD_TAG}/' % (upload_host,),
'xargs -a binarypkg/rpm_import_args.txt ssh %s -- createrepo-agent /var/repos/%s/building/%s/ --arch %s --invalidate-family --invalidate-dependants' % (upload_host, os_name, os_code_name, arch),
'ssh %s -- rm -fr /tmp/upload-${BUILD_TAG}/' % (upload_host,),
cottsay marked this conversation as resolved.
Show resolved Hide resolved
'echo "# END SECTION"',
]),
))@
@(SNIPPET(
'builder_shell',
script='\n'.join([
'echo "# BEGIN SECTION: Upload binaryrpm to Pulp"',
'export PYTHONPATH=$WORKSPACE/ros_buildfarm:$PYTHONPATH',
"ls binarypkg/*.rpm | grep -v -e 'src\.rpm$' -e '-debug\(info\|source\)-' > binarypkg/upload_list.txt && " +
'xargs -a binarypkg/upload_list.txt' +
Expand Down Expand Up @@ -220,7 +232,7 @@ but disabled since the package is blacklisted (or not whitelisted) in the config
<buildWrappers>
@(SNIPPET(
'pulp_credentials',
credential_id=credential_id,
credential_id=credential_id_pulp,
dest_credential_id=dest_credential_id,
))@
@[if timeout_minutes is not None]@
Expand All @@ -231,6 +243,10 @@ but disabled since the package is blacklisted (or not whitelisted) in the config
@[end if]@
@(SNIPPET(
'build-wrapper_timestamper',
))@
@(SNIPPET(
'build-wrapper_ssh-agent',
credential_ids=[credential_id],
))@
</buildWrappers>
</project>
2 changes: 1 addition & 1 deletion ros_buildfarm/templates/release/rpm/import_package_job.xml.em
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,7 @@
<buildWrappers>
@(SNIPPET(
'pulp_credentials',
credential_id=credential_id,
credential_id=credential_id_pulp,
dest_credential_id=dest_credential_id,
))@
@(SNIPPET(
Expand Down
27 changes: 26 additions & 1 deletion ros_buildfarm/templates/release/rpm/import_upstream_job.xml.em
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,12 @@
@(SNIPPET(
'property_parameters-definition',
parameters=[
{
'type': 'string',
'name': 'repo_list_file',
'description': 'A specific text file or files containing RPM base URLs to import from. The default is the ros_bootstrap repository.',
'default_value': '/home/jenkins-agent/ros_bootstrap_rpm_urls.txt',
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Generally in favor of not adding more yaml where it isn't required but does the url format for importing repositories allow us to specify the expected signing key for the repository?

Even if the answer is no, I don't consider this a review blocker since we don't have an active RPM bootstrap repository or an immediate need for one.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Right now, createrepo-agent isn't even checking the upstream repository signatures. I have it on the TODO list in that repo. For local signature checking, the public key must be in the local keystore, but there isn't any association between the public key and the specific repository.

Reason number 19385619 I'd like to stop signing metadata and start signing packages like upstream Fedora and RHEL do.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'll probably ask this question offline as well but if we use a package signing approach does the repository utility do anything to validate the integrity of signed packages within it or is it meant to behave as a simple intermediary and expect that the downstream users are validating package integrity?

},
{
'type': 'string',
'name': 'REMOTE_SOURCE_EXPRESSION',
Expand Down Expand Up @@ -61,6 +67,25 @@
<triggers/>
<concurrentBuild>false</concurrentBuild>
<builders>
@(SNIPPET(
'builder_shell',
script='\n'.join([
'echo "# BEGIN SECTION: import upstream packages"',
'for f in $repo_list_file; do',
] + [
" sed 's/$distname/%s/g;s/$releasever/%s/g' $f | xargs -L1 createrepo-agent /var/repos/%s/building/%s/ --invalidate-family --arch=SRPMS --arch=%s --sync" % (os_name, os_code_name, os_name, os_code_name, ' --arch='.join(arches))
for os_name, os_versions in import_targets.items() for os_code_name, arches in os_versions.items()
] + [
" sed 's/$distname/%s/g;s/$releasever/%s/g' $f | xargs -L1 createrepo-agent /var/repos/%s/testing/%s/ --invalidate-family --arch=SRPMS --arch=%s --sync" % (os_name, os_code_name, os_name, os_code_name, ' --arch='.join(arches))
for os_name, os_versions in import_targets.items() for os_code_name, arches in os_versions.items()
] + [
" sed 's/$distname/%s/g;s/$releasever/%s/g' $f | xargs -L1 createrepo-agent /var/repos/%s/main/%s/ --invalidate-family --arch=SRPMS --arch=%s --sync" % (os_name, os_code_name, os_name, os_code_name, ' --arch='.join(arches))
for os_name, os_versions in import_targets.items() for os_code_name, arches in os_versions.items()
] + [
'done',
'echo "# END SECTION"',
]),
))@
@(SNIPPET(
'builder_shell',
script='\n'.join([
Expand Down Expand Up @@ -93,7 +118,7 @@
<buildWrappers>
@(SNIPPET(
'pulp_credentials',
credential_id=credential_id,
credential_id=credential_id_pulp,
dest_credential_id=dest_credential_id,
))@
@(SNIPPET(
Expand Down
18 changes: 17 additions & 1 deletion ros_buildfarm/templates/release/rpm/sourcepkg_job.xml.em
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -126,6 +126,18 @@ but disabled since the package is blacklisted (or not whitelisted) in the config
'builder_shell',
script='\n'.join([
'echo "# BEGIN SECTION: Upload sourcerpm"',
'find sourcepkg -mindepth 1 -maxdepth 1 -type f -name "*.src.rpm" -fprint sourcepkg/rpm_upload_args.txt -fprintf sourcepkg/rpm_import_args.txt "--import=/tmp/upload-${BUILD_TAG}/%f\\n"',
'ssh %s -- mkdir -p /tmp/upload-${BUILD_TAG}/' % (upload_host,),
'xargs -a sourcepkg/rpm_upload_args.txt -I @ scp @ %s:/tmp/upload-${BUILD_TAG}/' % (upload_host,),
'xargs -a sourcepkg/rpm_import_args.txt ssh %s -- createrepo-agent /var/repos/%s/building/%s/' % (upload_host, os_name, os_code_name),
'ssh %s -- rm -fr /tmp/upload-${BUILD_TAG}/' % (upload_host,),
'echo "# END SECTION"',
nuclearsandwich marked this conversation as resolved.
Show resolved Hide resolved
]),
))@
@(SNIPPET(
'builder_shell',
script='\n'.join([
'echo "# BEGIN SECTION: Upload sourcerpm to Pulp"',
'export TZ="%s"' % timezone,
'export PYTHONPATH=$WORKSPACE/ros_buildfarm:$PYTHONPATH',
'python3 -u $WORKSPACE/ros_buildfarm/scripts/release/rpm/upload_package.py' +
Expand Down Expand Up @@ -183,7 +195,7 @@ but disabled since the package is blacklisted (or not whitelisted) in the config
<buildWrappers>
@(SNIPPET(
'pulp_credentials',
credential_id=credential_id,
credential_id=credential_id_pulp,
dest_credential_id=dest_credential_id,
))@
@[if timeout_minutes is not None]@
Expand All @@ -194,6 +206,10 @@ but disabled since the package is blacklisted (or not whitelisted) in the config
@[end if]@
@(SNIPPET(
'build-wrapper_timestamper',
))@
@(SNIPPET(
'build-wrapper_ssh-agent',
credential_ids=[credential_id],
))@
</buildWrappers>
</project>
27 changes: 19 additions & 8 deletions ros_buildfarm/templates/release/rpm/sync_packages_to_main_job.xml.em
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,17 @@
'builder_shell',
script='\n'.join([
'echo "# BEGIN SECTION: sync packages to main repos"',
] + [
'createrepo-agent /var/repos/%s/main/%s/ --sync=/var/repos/%s/testing/%s/ --arch=SRPMS --arch=%s --sync-pattern="ros-%s-.*" --invalidate-family' % (os_name, os_code_name, os_name, os_code_name, ' --arch='.join(arches), rosdistro_name)
for rosdistro_name, sync_targets in ((rosdistro_name, sync_targets),) for os_name, os_versions in sync_targets.items() for os_code_name, arches in os_versions.items()
] + [
'echo "# END SECTION"',
]),
))@
@(SNIPPET(
'builder_shell',
script='\n'.join([
'echo "# BEGIN SECTION: sync packages to main repos in Pulp"',
'export PYTHONPATH=$WORKSPACE/ros_buildfarm:$PYTHONPATH',
'python3 -u $WORKSPACE/ros_buildfarm/scripts/release/rpm/sync_repo.py' +
' --distribution-source-expression "^ros-testing-([^-]*-[^-]*-[^-]*(-debug)?)$"' +
Expand All @@ -59,16 +70,16 @@
@(SNIPPET(
'builder_shell',
script='\n'.join([
'echo "# BEGIN SECTION: mirror main repository content to disk"',
'echo "# BEGIN SECTION: mirror main Pulp repository content to disk"',
] + [
'rsync --recursive --times --delete --itemize-changes rsync://127.0.0.1:1234/ros-main-%s-%s-SRPMS/ /var/repos/%s/main/%s/SRPMS/' % (os_name, os_code_name, os_name, os_code_name)
for os_name, os_code_name in set((os_name, os_code_name) for os_name, os_code_name, _ in sync_targets)
'rsync --recursive --times --delete --itemize-changes rsync://127.0.0.1:1234/ros-main-%s-%s-SRPMS/ /var/repos/%s_pulp/main/%s/SRPMS/' % (os_name, os_code_name, os_name, os_code_name)
for os_name, os_versions in sync_targets.items() for os_code_name in os_versions
] + [
'rsync --recursive --times --delete --exclude=debug --itemize-changes rsync://127.0.0.1:1234/ros-main-%s-%s-%s/ /var/repos/%s/main/%s/%s/' % (os_name, os_code_name, arch, os_name, os_code_name, arch)
for os_name, os_code_name, arch in sync_targets
'rsync --recursive --times --delete --exclude=debug --itemize-changes rsync://127.0.0.1:1234/ros-main-%s-%s-%s/ /var/repos/%s_pulp/main/%s/%s/' % (os_name, os_code_name, arch, os_name, os_code_name, arch)
for os_name, os_versions in sync_targets.items() for os_code_name, arches in os_versions.items() for arch in arches
] + [
'rsync --recursive --times --delete --itemize-changes rsync://127.0.0.1:1234/ros-main-%s-%s-%s-debug/ /var/repos/%s/main/%s/%s/debug/' % (os_name, os_code_name, arch, os_name, os_code_name, arch)
for os_name, os_code_name, arch in sync_targets
'rsync --recursive --times --delete --itemize-changes rsync://127.0.0.1:1234/ros-main-%s-%s-%s-debug/ /var/repos/%s_pulp/main/%s/%s/debug/' % (os_name, os_code_name, arch, os_name, os_code_name, arch)
for os_name, os_versions in sync_targets.items() for os_code_name, arches in os_versions.items() for arch in arches
] + [
'echo "# END SECTION"',
]),
Expand All @@ -85,7 +96,7 @@
<buildWrappers>
@(SNIPPET(
'pulp_credentials',
credential_id=credential_id,
credential_id=credential_id_pulp,
dest_credential_id=dest_credential_id,
))@
@(SNIPPET(
Expand Down
16 changes: 12 additions & 4 deletions ros_buildfarm/templates/release/rpm/sync_packages_to_testing_job.xml.em
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,14 @@
'echo "# END SECTION"',
]),
))@
@(SNIPPET(
'builder_shell',
script='\n'.join([
'echo "# BEGIN SECTION: sync packages to testing repos"',
'createrepo-agent /var/repos/%s/testing/%s/ --sync=/var/repos/%s/building/%s/ --arch=SRPMS --arch=%s --sync-pattern="ros-%s-.*" --invalidate-family' % (os_name, os_code_name, os_name, os_code_name, arch, rosdistro_name),
'echo "# END SECTION"',
]),
))@
@(SNIPPET(
'builder_shell',
script='\n'.join([
Expand All @@ -108,9 +116,9 @@
'builder_shell',
script='\n'.join([
'echo "# BEGIN SECTION: mirror testing repository content to disk"',
'rsync --recursive --times --delete --itemize-changes rsync://127.0.0.1:1234/ros-testing-%s-%s-SRPMS/ /var/repos/%s/testing/%s/SRPMS/' % (os_name, os_code_name, os_name, os_code_name),
'rsync --recursive --times --delete --exclude=debug --itemize-changes rsync://127.0.0.1:1234/ros-testing-%s-%s-%s/ /var/repos/%s/testing/%s/%s/' % (os_name, os_code_name, arch, os_name, os_code_name, arch),
'rsync --recursive --times --delete --itemize-changes rsync://127.0.0.1:1234/ros-testing-%s-%s-%s-debug/ /var/repos/%s/testing/%s/%s/debug/' % (os_name, os_code_name, arch, os_name, os_code_name, arch),
'rsync --recursive --times --delete --itemize-changes rsync://127.0.0.1:1234/ros-testing-%s-%s-SRPMS/ /var/repos/%s_pulp/testing/%s/SRPMS/' % (os_name, os_code_name, os_name, os_code_name),
'rsync --recursive --times --delete --exclude=debug --itemize-changes rsync://127.0.0.1:1234/ros-testing-%s-%s-%s/ /var/repos/%s_pulp/testing/%s/%s/' % (os_name, os_code_name, arch, os_name, os_code_name, arch),
'rsync --recursive --times --delete --itemize-changes rsync://127.0.0.1:1234/ros-testing-%s-%s-%s-debug/ /var/repos/%s_pulp/testing/%s/%s/debug/' % (os_name, os_code_name, arch, os_name, os_code_name, arch),
'echo "# END SECTION"',
]),
))@
Expand All @@ -126,7 +134,7 @@
<buildWrappers>
@(SNIPPET(
'pulp_credentials',
credential_id=credential_id,
credential_id=credential_id_pulp,
dest_credential_id=dest_credential_id,
))@
@(SNIPPET(
Expand Down