diff --git a/doc/configuration_options.rst b/doc/configuration_options.rst index aff876d97..acccc0489 100644 --- a/doc/configuration_options.rst +++ b/doc/configuration_options.rst @@ -173,13 +173,6 @@ Description of common options master which is commonly used to upload artifacts to another host. This credential id is set in the buildfarm_deployment. -* **Upload destination credential ID**: the ID of the credential entry managed - on the Jenkins master which contains the destination information used to - upload artifacts to another host. - This credential id is set in the buildfarm_deployment. - At present, this value is only used for RPM jobs. - - Specific options in release build files --------------------------------------- @@ -237,9 +230,6 @@ The following options are valid in version ``2`` (beside the generic options): * ``upload_credential_id``: the ID of the credential to upload the built packages to the repository host. -* ``upload_credential_id_pulp``: the ID of the credential to upload the built - RPM packages to the repository host instance of Pulp. - * ``upload_host``: the hostname of the repository host where built packages shoudl be uploaded to. Only affects RPM builds at present. diff --git a/ros_buildfarm/argument.py b/ros_buildfarm/argument.py index 20faefddf..ebfee8de3 100644 --- a/ros_buildfarm/argument.py +++ b/ros_buildfarm/argument.py @@ -395,49 +395,6 @@ def add_argument_build_tool_test_args(parser): help='Arbitrary arguments passed to the build tool during testing.') -def add_argument_pulp_base_url(parser): - from_env = os.environ.get('PULP_BASE_URL') - return parser.add_argument( - '--pulp-base-url', - default=from_env, required=not bool(from_env), - help='URL of the pulp API endpoint') - - -def add_argument_pulp_distribution_name(parser): - return parser.add_argument( - '--pulp-distribution-name', required=True, - help='Name of the pulp distribution to target with changes') - - -def add_argument_pulp_password(parser): - from_env = os.environ.get('PULP_PASSWORD') - return parser.add_argument( - '--pulp-password', - default=from_env, required=not bool(from_env), - help='Password used to access the pulp API endpoint') - - -def add_argument_pulp_resource_record(parser): - return parser.add_argument( - '--pulp-resource-record', default=None, metavar='FILE', - help='File in which to record the pulp HREFs of package resources') - - -def add_argument_pulp_task_timeout(parser): - return parser.add_argument( - '--pulp-task-timeout', - type=float, default=60.0, - help='Duration to wait (in seconds) for a pulp task to complete') - - -def add_argument_pulp_username(parser): - from_env = os.environ.get('PULP_USERNAME') - return parser.add_argument( - '--pulp-username', - default=from_env, required=not bool(from_env), - help='Username used to access the pulp API endpoint') - - def add_argument_repos_file_urls(parser): parser.add_argument( '--repos-file-urls', nargs='*', metavar='URL', diff --git a/ros_buildfarm/config/release_build_file.py b/ros_buildfarm/config/release_build_file.py index 2ed418163..76df34b6d 100644 --- a/ros_buildfarm/config/release_build_file.py +++ b/ros_buildfarm/config/release_build_file.py @@ -109,14 +109,6 @@ def __init__(self, name, data): # noqa: D107 assert 'upload_credential_id' in data self.upload_credential_id = data['upload_credential_id'] - self.upload_credential_id_pulp = None - if 'upload_credential_id_pulp' in data: - self.upload_credential_id_pulp = data['upload_credential_id_pulp'] - - self.upload_destination_credential_id = None - if 'upload_destination_credential_id' in data: - self.upload_destination_credential_id = data['upload_destination_credential_id'] - self.upload_host = None if 'upload_host' in data: self.upload_host = data['upload_host'] diff --git a/ros_buildfarm/pulp.py b/ros_buildfarm/pulp.py deleted file mode 100644 index ade00aa1c..000000000 --- a/ros_buildfarm/pulp.py +++ /dev/null @@ -1,373 +0,0 @@ -# Copyright 2020 Open Source Robotics Foundation, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import logging -import os -import re -import time - -from pulpcore.client import pulp_rpm -from pulpcore.client import pulpcore - -logger = logging.getLogger(__name__) - - -class PulpTaskError(RuntimeError): - - def __init__(self, task, state): # noqa: D107 - super().__init__("Pulp task '%s' did not complete (%s)" % (task.pulp_href, state)) - self.task = task - - -def format_pkg_ver(pkg): - return '%s%s-%s' % ( - (pkg.epoch + ':') if pkg.epoch != '0' else '', - pkg.version, - pkg.release) - - -def _enumerate_recursive_dependencies(packages, target_names): - new_names = set(target_names) - - while new_names: - target_names = new_names - new_names = set() - for pkg in packages: - if target_names.intersection(req[0] for req in pkg.requires): - yield (pkg.pulp_href, pkg) - new_names.add(pkg.name) - new_names.update(prov[0] for prov in pkg.provides) - - -class PulpPageIterator: - - def __init__(self, fetch_function, *args, **kwargs): # noqa: D107 - self._get_next = lambda offset: fetch_function(*args, **kwargs, offset=offset) - self._offset = 0 - self._next_page() - - def _next_page(self): - self._page = self._get_next(self._offset) - self._offset += len(self._page.results) - self._iter = iter(self._page.results) - if self._page.count: - logger.debug( - 'Fetched a page of %d results (%d%%)' % ( - len(self._page.results), 100.0 * self._offset / self._page.count)) - - def __iter__(self): - # Make sure we're at the beginning - if self._page.previous: - self._offset = 0 - self._next_page() - else: - self._iter = iter(self._page.results) - return self - - def __len__(self): - return self._page.count - - def __next__(self): - try: - return next(self._iter) - except StopIteration: - if not self._page.next: - raise - - self._next_page() - return next(self._iter) - - -class PulpRpmClient: - - def __init__( # noqa: D107 - self, base_url, username, password, - task_timeout=120.0, task_polling_interval=0.5): - self._task_timeout = task_timeout - self._task_polling_interval = task_polling_interval - - self._config = pulpcore.Configuration( - base_url, username=username, password=password) - - # https://pulp.plan.io/issues/5932 - self._config.safe_chars_for_path_param = '/' - - # Core APIs - self._core_client = pulpcore.ApiClient(self._config) - self._core_tasks_api = pulpcore.TasksApi(self._core_client) - self._core_orphans_api = pulpcore.OrphansApi(self._core_client) - - # RPM APIs - self._rpm_client = pulp_rpm.ApiClient(self._config) - self._rpm_distributions_api = pulp_rpm.DistributionsRpmApi(self._rpm_client) - self._rpm_packages_api = pulp_rpm.ContentPackagesApi(self._rpm_client) - self._rpm_publications_api = pulp_rpm.PublicationsRpmApi(self._rpm_client) - self._rpm_remotes_api = pulp_rpm.RemotesRpmApi(self._rpm_client) - self._rpm_repos_api = pulp_rpm.RepositoriesRpmApi(self._rpm_client) - self._rpm_repo_vers_api = pulp_rpm.RepositoriesRpmVersionsApi(self._rpm_client) - - def _wait_for_task(self, task_href): - task = self._core_tasks_api.read(task_href) - - timeout = self._task_timeout - while task.state != 'completed': - if task.state in ['failed', 'canceled']: - raise PulpTaskError(task, task.state) - logger.debug( - "Pulp task '%s' is '%s': checking again in %ds" % ( - task.pulp_href, task.state, self._task_polling_interval)) - time.sleep(self._task_polling_interval) - timeout -= self._task_polling_interval - if timeout <= 0: - task_cancel = pulpcore.PatchedTaskCancel('canceled') - task = self._core_tasks_api.tasks_cancel(task.pulp_href, task_cancel) - if task.state != 'completed': - raise PulpTaskError(task, 'timed out') - - task = self._core_tasks_api.read(task.pulp_href) - - return task - - def _publish_and_distribute(self, distribution, repo_version_href): - logger.debug('Publishing and distributing: ' + repo_version_href) - - # Publish the new version - publish_data = pulp_rpm.RpmRpmPublication(repository_version=repo_version_href) - publish_task_href = self._rpm_publications_api.create(publish_data).task - publish_task = self._wait_for_task(publish_task_href) - logger.debug('Created publication: ' + publish_task.created_resources[0]) - - # Distribute the publication at the original endpoint - distribution.publication = publish_task.created_resources[0] - distribute_task_href = self._rpm_distributions_api.partial_update( - distribution.pulp_href, distribution).task - self._wait_for_task(distribute_task_href) - logger.debug('Updated distribution: ' + distribution.pulp_href) - - def enumerate_distributions(self): - return PulpPageIterator( - self._rpm_distributions_api.list) - - def enumerate_pkgs_in_distribution_name(self, distribution_name): - distribution = self._rpm_distributions_api.list( - name=distribution_name).results[0] - publication = self._rpm_publications_api.read(distribution.publication) - return self.enumerate_pkgs_in_repo_ver(publication.repository_version) - - def enumerate_pkgs_in_repo_ver(self, repo_ver_href, names=None): - name__in = ','.join(names) if names is not None else None - return PulpPageIterator( - self._rpm_packages_api.list, repository_version=repo_ver_href, - fields='pulp_href,name,epoch,version,arch,release,provides,requires', - limit=250, name__in=name__in) - - def enumerate_remotes(self): - return PulpPageIterator(self._rpm_remotes_api.list) - - def enumerate_unused_repo_vers(self, distribution_name): - distribution = self._rpm_distributions_api.list( - name=distribution_name).results[0] - publication = self._rpm_publications_api.read(distribution.publication) - all_repo_vers = { - repo_ver.pulp_href: repo_ver for repo_ver in - PulpPageIterator( - self._rpm_repo_vers_api.list, rpm_rpm_repository_href=publication.repository, - limit=500)} - - logger.debug( - 'Fetched %d total repository versions for %s' % ( - len(all_repo_vers), distribution_name)) - - unused_repo_vers = [] - current_repo_ver = all_repo_vers.get( - all_repo_vers[publication.repository_version].base_version) - while current_repo_ver and current_repo_ver.number: - unused_repo_vers.append(current_repo_ver.pulp_href) - current_repo_ver = all_repo_vers.get(current_repo_ver.base_version) - - logger.debug( - 'Found %d ancestors of %s' % ( - len(unused_repo_vers), publication.repository_version)) - - return unused_repo_vers - - def import_and_invalidate( - self, distribution_name, packages_to_add, - invalidate_expression, invalidate_downstream, package_cache=None, dry_run=False): - logger.debug("Performing import and invalidation for '%s'" % (distribution_name,)) - distribution = self._rpm_distributions_api.list( - name=distribution_name).results[0] - logger.debug('Got distribution: ' + distribution.pulp_href) - old_publication = self._rpm_publications_api.read(distribution.publication) - logger.debug('Got old publication: ' + old_publication.pulp_href) - - if package_cache is None: - package_cache = {} - - # If we need to invalidate, fetch everything up front - if invalidate_expression or invalidate_downstream: - logger.debug('Getting package list for ' + old_publication.repository_version) - # Get the current packages - current_pkgs = { - pkg.pulp_href: pkg for pkg in - self.enumerate_pkgs_in_repo_ver(old_publication.repository_version)} - package_cache.update(current_pkgs) - - # Get the packages we're adding - logger.debug( - 'Getting information about %d packages being added' % (len(packages_to_add),)) - new_pkgs = { - pkg.pulp_href: pkg for pkg in - [package_cache.get(pkg_href) or self._rpm_packages_api.read(pkg_href) - for pkg_href in packages_to_add]} - - # If we didn't already fetch everything, enumerate only - # packages with the same name - if not invalidate_expression and not invalidate_downstream: - logger.debug('Getting information about packages being replaced') - # Get the current packages - names = set(p.name for p in new_pkgs.values()) - current_pkgs = { - pkg.pulp_href: pkg for pkg in - self.enumerate_pkgs_in_repo_ver( - old_publication.repository_version, names=names)} - package_cache.update(current_pkgs) - - # Invalidate packages - logger.debug('Determining packages to invalidate') - pkgs_to_remove = {} - new_pkg_names = set([pkg.name for pkg in new_pkgs.values()]) - # 1. Remove packages with the same name - pkgs_to_remove.update({ - pkg.pulp_href: pkg for pkg in current_pkgs.values() - if pkg.name in new_pkg_names}) - # 2. Remove downstream packages - if invalidate_downstream: - new_pkg_provides = new_pkg_names.union( - prov[0] for pkg in new_pkgs.values() for prov in pkg.provides) - pkgs_to_remove.update( - _enumerate_recursive_dependencies(current_pkgs.values(), new_pkg_provides)) - # 3. Remove packages matching the invalidation expression - if invalidate_expression: - compiled_expression = re.compile(invalidate_expression) - for pkg in current_pkgs.values(): - if compiled_expression.match(pkg.name): - pkgs_to_remove[pkg.pulp_href] = pkg - - # Prune the list of packages to add and remove - # 1. Packages being added *always* end up in the repo - for href_to_add in new_pkgs.keys(): - pkgs_to_remove.pop(href_to_add, None) - # 2. Packages being added don't need to get re-added - for href_in_current in current_pkgs.keys(): - new_pkgs.pop(href_in_current, None) - - if dry_run: - logger.debug('Finished (dry-run)') - return (new_pkgs.values(), pkgs_to_remove.values()) - - # Commit the changes - logger.debug('Committing changes') - mod_data = pulp_rpm.RepositoryAddRemoveContent( - add_content_units=list(new_pkgs.keys()), - remove_content_units=list(pkgs_to_remove.keys()), - base_version=old_publication.repository_version) - mod_task_href = self._rpm_repos_api.modify(old_publication.repository, mod_data).task - mod_task = self._wait_for_task(mod_task_href) - - # WORKAROUND FOR https://pulp.plan.io/issues/6811 - # - # This implementation doesn't care what the "latest" version of the repository - # looks like, but the implementation details of the add/remove operation in Pulp - # seem to. The expectation is that when `created_resources` is empty, the changes - # are unnecessary and already satisfied, but another case can lead to this - # behavior. See the referenced ticket for details. - # - # We can work around the behavior by specifically modifying the 'latest' version - # to look different from our intended state, and then ignoring that new version. - # This way we'll keep our revision history intact. - if not mod_task.created_resources and (new_pkgs or pkgs_to_remove): - # Create a new version with nothing in it. Hopefully that will be different - # from the 'latest'. - logger.warning('Working around pulp issue #6811') - workaround_mod_data = pulp_rpm.RepositoryAddRemoveContent( - add_content_units=[], - remove_content_units=['*'], - base_version=old_publication.repository_version) - workaround_mod_task_href = self._rpm_repos_api.modify( - old_publication.repository, - workaround_mod_data).task - workaround_mod_task = self._wait_for_task(workaround_mod_task_href) - assert workaround_mod_task.created_resources - - # Now that the 'latest' version has changed, re-run the original operation - mod_task_href = self._rpm_repos_api.modify( - old_publication.repository, - mod_data).task - mod_task = self._wait_for_task(mod_task_href) - # END WORKAROUND - - if mod_task.created_resources: - self._publish_and_distribute(distribution, mod_task.created_resources[0]) - else: - logger.warning('Modification operations resulted in no apparent changes') - - return (new_pkgs.values(), pkgs_to_remove.values()) - - def mirror_remote_to_distribution(self, remote_name, distribution_name, dry_run=False): - remote = self._rpm_remotes_api.list(name=remote_name).results[0] - distribution = self._rpm_distributions_api.list(name=distribution_name).results[0] - old_publication = self._rpm_publications_api.read(distribution.publication) - - sync_data = pulp_rpm.RpmRepositorySyncURL( - remote=remote.pulp_href, - mirror=True) - - if dry_run: - return - - sync_task_href = self._rpm_repos_api.sync(old_publication.repository, sync_data).task - sync_task = self._wait_for_task(sync_task_href) - - if sync_task.created_resources: - self._publish_and_distribute(distribution, sync_task.created_resources[0]) - - def remove_unused_content(self): - delete_task_href = self._core_orphans_api.delete().task - delete_task = self._wait_for_task(delete_task_href) - print('%s' % (delete_task,)) - - def remove_unused_repo_vers(self, distribution_name, dry_run=False): - unused_repo_vers = self.enumerate_unused_repo_vers(distribution_name) - - # Start removing the oldest ones first - unused_repo_vers.reverse() - - for repo_ver in unused_repo_vers: - if dry_run: - logger.debug('Removing %s (dry-run)' % (repo_ver,)) - continue - logger.debug('Removing %s' % (repo_ver,)) - delete_task_href = self._rpm_repo_vers_api.delete(repo_ver).task - self._wait_for_task(delete_task_href) - - def upload_pkg(self, file_path): - relative_path = os.path.basename(file_path) - upload_task_href = self._rpm_packages_api.create( - relative_path, file=file_path).task - logger.debug( - "Upload task for '%s': %s" % (os.path.basename(file_path), upload_task_href)) - upload_task = self._wait_for_task(upload_task_href) - - return self._rpm_packages_api.read(upload_task.created_resources[0]) diff --git a/ros_buildfarm/release_job.py b/ros_buildfarm/release_job.py index c34949e1a..69e54031d 100644 --- a/ros_buildfarm/release_job.py +++ b/ros_buildfarm/release_job.py @@ -139,11 +139,14 @@ def configure_release_jobs( all_view_configs = {} all_job_configs = OrderedDict() - job_name, job_config = configure_import_package_job( - config_url, rosdistro_name, release_build_name, - config=config, build_file=build_file, jenkins=jenkins, dry_run=dry_run) - if not jenkins: - all_job_configs[job_name] = job_config + for os_name, _ in platforms: + if package_format_mapping[os_name] not in ('rpm',): + job_name, job_config = configure_import_package_job( + config_url, rosdistro_name, release_build_name, + config=config, build_file=build_file, jenkins=jenkins, dry_run=dry_run) + if not jenkins: + all_job_configs[job_name] = job_config + break job_name, job_config = configure_sync_packages_to_main_job( config_url, rosdistro_name, release_build_name, @@ -390,7 +393,7 @@ def configure_release_job( config=None, build_file=None, index=None, dist_file=None, cached_pkgs=None, jenkins=None, views=None, - generate_import_package_job=True, + generate_import_package_job=None, generate_sync_packages_jobs=True, is_disabled=False, other_build_files_same_platform=None, groovy_script=None, @@ -408,6 +411,8 @@ def configure_release_job( if build_file is None: build_files = get_release_build_files(config, rosdistro_name) build_file = build_files[release_build_name] + if generate_import_package_job is None: + generate_import_package_job = package_format_mapping[os_name] not in ('rpm',) if index is None: index = get_index(config.rosdistro_index_url) @@ -664,8 +669,6 @@ def _get_sourcedeb_job_config( 'upload_host': build_file.upload_host, 'credential_id': build_file.upload_credential_id, - 'credential_id_pulp': build_file.upload_credential_id_pulp, - 'dest_credential_id': build_file.upload_destination_credential_id, 'git_ssh_credential_id': config.git_ssh_credential_id, } @@ -753,8 +756,6 @@ def _get_binarydeb_job_config( 'upload_host': build_file.upload_host, 'credential_id': build_file.upload_credential_id, - 'credential_id_pulp': build_file.upload_credential_id_pulp, - 'dest_credential_id': build_file.upload_destination_credential_id, 'shared_ccache': build_file.shared_ccache, } @@ -802,8 +803,6 @@ def _get_import_package_job_config(build_file, package_format): 'abi_incompatibility_assumed': build_file.abi_incompatibility_assumed, 'notify_emails': build_file.notify_emails, 'ros_buildfarm_repository': get_repository(), - 'credential_id_pulp': build_file.upload_credential_id_pulp, - 'dest_credential_id': build_file.upload_destination_credential_id, } job_config = expand_template(template_name, job_data) return job_config @@ -868,8 +867,6 @@ def _get_sync_packages_to_testing_job_config( rosdistro_name, package_format), 'notify_emails': build_file.notify_emails, - 'credential_id_pulp': build_file.upload_credential_id_pulp, - 'dest_credential_id': build_file.upload_destination_credential_id, } job_config = expand_template(template_name, job_data) return job_config @@ -920,8 +917,6 @@ def _get_sync_packages_to_main_job_config(rosdistro_name, build_file, package_fo 'sync_targets': build_file.targets, 'notify_emails': build_file.notify_emails, - 'credential_id_pulp': build_file.upload_credential_id_pulp, - 'dest_credential_id': build_file.upload_destination_credential_id, } job_config = expand_template(template_name, job_data) return job_config diff --git a/ros_buildfarm/scripts/release/generate_release_maintenance_jobs.py b/ros_buildfarm/scripts/release/generate_release_maintenance_jobs.py index e992a836b..3b552b2fe 100644 --- a/ros_buildfarm/scripts/release/generate_release_maintenance_jobs.py +++ b/ros_buildfarm/scripts/release/generate_release_maintenance_jobs.py @@ -124,8 +124,6 @@ def get_import_upstream_job_config(args, config, build_file, package_format): data = { 'import_targets': build_file.targets, 'credential_id': build_file.upload_credential_id, - 'credential_id_pulp': build_file.upload_credential_id_pulp, - 'dest_credential_id': build_file.upload_destination_credential_id, } return _get_job_config( args, config, config.notify_emails, template_name, diff --git a/ros_buildfarm/scripts/release/generate_release_script.py b/ros_buildfarm/scripts/release/generate_release_script.py index 4224ec03e..fb31278af 100644 --- a/ros_buildfarm/scripts/release/generate_release_script.py +++ b/ros_buildfarm/scripts/release/generate_release_script.py @@ -78,8 +78,7 @@ def beforeInclude(self, *args, **kwargs): ] script = '\n'.join(lines) elif 'Upload binary' in script or 'Upload source' in script: - # Skip scripts which are responsible for uploading resources - # to Pulp. + # Skip scripts which are responsible for uploading resources. return self.scripts.append(script) diff --git a/ros_buildfarm/scripts/release/rpm/cull_repo.py b/ros_buildfarm/scripts/release/rpm/cull_repo.py deleted file mode 100644 index feb2783c1..000000000 --- a/ros_buildfarm/scripts/release/rpm/cull_repo.py +++ /dev/null @@ -1,58 +0,0 @@ -# Copyright 2022 Open Source Robotics Foundation, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import argparse -import logging -import sys - -from ros_buildfarm.argument import add_argument_dry_run -from ros_buildfarm.argument import add_argument_pulp_base_url -from ros_buildfarm.argument import add_argument_pulp_distribution_name -from ros_buildfarm.argument import add_argument_pulp_password -from ros_buildfarm.argument import add_argument_pulp_task_timeout -from ros_buildfarm.argument import add_argument_pulp_username -from ros_buildfarm.common import Scope -from ros_buildfarm.pulp import PulpRpmClient - - -def main(argv=sys.argv[1:]): - logging.basicConfig( - level=logging.DEBUG, format='%(name)s %(levelname)s %(asctime)s: %(message)s') - - parser = argparse.ArgumentParser( - description='Cull unused repository snapshots and packages from Pulp') - add_argument_dry_run(parser) - add_argument_pulp_base_url(parser) - add_argument_pulp_distribution_name(parser) - add_argument_pulp_password(parser) - add_argument_pulp_task_timeout(parser) - add_argument_pulp_username(parser) - args = parser.parse_args(argv) - - pulp_client = PulpRpmClient( - args.pulp_base_url, args.pulp_username, args.pulp_password, - task_timeout=args.pulp_task_timeout) - - with Scope('SUBSECTION', 'removing unused snapshots'): - pulp_client.remove_unused_repo_vers(args.pulp_distribution_name, dry_run=args.dry_run) - - with Scope('SUBSECTION', 'removing unused content'): - if args.dry_run: - print('Skipped (dry-run)') - else: - pulp_client.remove_unused_content() - - -if __name__ == '__main__': - sys.exit(main()) diff --git a/ros_buildfarm/scripts/release/rpm/import_package.py b/ros_buildfarm/scripts/release/rpm/import_package.py deleted file mode 100644 index 4068f58f6..000000000 --- a/ros_buildfarm/scripts/release/rpm/import_package.py +++ /dev/null @@ -1,76 +0,0 @@ -# Copyright 2020 Open Source Robotics Foundation, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import argparse -import logging -import sys - -from ros_buildfarm.argument import add_argument_dry_run -from ros_buildfarm.argument import add_argument_invalidate -from ros_buildfarm.argument import add_argument_pulp_base_url -from ros_buildfarm.argument import add_argument_pulp_distribution_name -from ros_buildfarm.argument import add_argument_pulp_password -from ros_buildfarm.argument import add_argument_pulp_task_timeout -from ros_buildfarm.argument import add_argument_pulp_username -from ros_buildfarm.common import Scope -from ros_buildfarm.pulp import format_pkg_ver -from ros_buildfarm.pulp import PulpRpmClient - - -def main(argv=sys.argv[1:]): - logging.basicConfig( - level=logging.DEBUG, format='%(name)s %(levelname)s %(asctime)s: %(message)s') - - parser = argparse.ArgumentParser( - description='Import packages into a repository and publish it') - parser.add_argument( - 'package_resources', - nargs='*', metavar="PULP_HREF", - help='Identifiers for packages which should be imported') - add_argument_dry_run(parser) - add_argument_invalidate(parser) - parser.add_argument( - '--invalidate-expression', - default=None, - help='Any existing package names matching this expression will be removed') - add_argument_pulp_base_url(parser) - add_argument_pulp_distribution_name(parser) - add_argument_pulp_password(parser) - add_argument_pulp_task_timeout(parser) - add_argument_pulp_username(parser) - args = parser.parse_args(argv) - - pulp_client = PulpRpmClient( - args.pulp_base_url, args.pulp_username, args.pulp_password, - task_timeout=args.pulp_task_timeout) - - with Scope('SUBSECTION', 'performing repository transaction'): - pkgs_added, pkgs_removed = pulp_client.import_and_invalidate( - args.pulp_distribution_name, args.package_resources, - args.invalidate_expression, args.invalidate, dry_run=args.dry_run) - - with Scope('SUBSECTION', 'enumerating results'): - if not pkgs_added: - print('Not importing any new packages') - for pkg in pkgs_added: - print('Importing package: %s-%s.%s' % (pkg.name, format_pkg_ver(pkg), pkg.arch)) - - if not pkgs_removed: - print('Not removing any existing packages') - for pkg in pkgs_removed: - print('Removing package: %s-%s.%s' % (pkg.name, format_pkg_ver(pkg), pkg.arch)) - - -if __name__ == '__main__': - sys.exit(main()) diff --git a/ros_buildfarm/scripts/release/rpm/mirror_repo.py b/ros_buildfarm/scripts/release/rpm/mirror_repo.py deleted file mode 100644 index aa513a05f..000000000 --- a/ros_buildfarm/scripts/release/rpm/mirror_repo.py +++ /dev/null @@ -1,77 +0,0 @@ -# Copyright 2020 Open Source Robotics Foundation, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import argparse -import re -import sys - -from ros_buildfarm.argument import add_argument_dry_run -from ros_buildfarm.argument import add_argument_pulp_base_url -from ros_buildfarm.argument import add_argument_pulp_password -from ros_buildfarm.argument import add_argument_pulp_task_timeout -from ros_buildfarm.argument import add_argument_pulp_username -from ros_buildfarm.common import Scope -from ros_buildfarm.pulp import PulpRpmClient - - -def main(argv=sys.argv[1:]): - parser = argparse.ArgumentParser( - description='Mirror a remote RPM repository to a pulp distribution') - add_argument_dry_run(parser) - add_argument_pulp_base_url(parser) - add_argument_pulp_password(parser) - add_argument_pulp_task_timeout(parser) - add_argument_pulp_username(parser) - parser.add_argument( - '--remote-source-expression', required=True, - help='Expression to match for pulp remote names') - parser.add_argument( - '--distribution-dest-expression', required=True, - help='Expression to transform matching source remote names ' - 'into destination distribution names') - args = parser.parse_args(argv) - - pulp_client = PulpRpmClient( - args.pulp_base_url, args.pulp_username, args.pulp_password, - task_timeout=args.pulp_task_timeout) - - dists_to_sync = [] - with Scope('SUBSECTION', 'enumerating remotes and distributions to sync'): - remote_expression = re.compile(args.remote_source_expression) - distributions = {dist.name for dist in pulp_client.enumerate_distributions()} - for remote in pulp_client.enumerate_remotes(): - (dist_dest_pattern, matched_source) = remote_expression.subn( - args.distribution_dest_expression, remote.name) - if matched_source: - dist_dest_matches = [ - dist for dist in distributions if re.match(dist_dest_pattern, dist)] - if not dist_dest_matches: - print( - "No distributions match destination pattern '%s'" % dist_dest_pattern, - file=sys.stderr) - return 1 - dists_to_sync.extend((remote.name, dist_dest) for dist_dest in dist_dest_matches) - - dists_to_sync = sorted(set(dists_to_sync)) - print('Syncing %d distributions:' % len(dists_to_sync)) - for remote_name_and_dist_dest in dists_to_sync: - print('- %s => %s' % remote_name_and_dist_dest) - - with Scope('SUBSECTION', 'synchronizing remotes and publishing mirrors'): - for remote_name, dist_name in dists_to_sync: - pulp_client.mirror_remote_to_distribution(remote_name, dist_name, dry_run=args.dry_run) - - -if __name__ == '__main__': - sys.exit(main()) diff --git a/ros_buildfarm/scripts/release/rpm/sync_repo.py b/ros_buildfarm/scripts/release/rpm/sync_repo.py deleted file mode 100644 index 8dab92259..000000000 --- a/ros_buildfarm/scripts/release/rpm/sync_repo.py +++ /dev/null @@ -1,118 +0,0 @@ -# Copyright 2020 Open Source Robotics Foundation, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import argparse -import re -import sys - -from ros_buildfarm.argument import add_argument_dry_run -from ros_buildfarm.argument import add_argument_invalidate -from ros_buildfarm.argument import add_argument_pulp_base_url -from ros_buildfarm.argument import add_argument_pulp_password -from ros_buildfarm.argument import add_argument_pulp_task_timeout -from ros_buildfarm.argument import add_argument_pulp_username -from ros_buildfarm.common import Scope -from ros_buildfarm.pulp import format_pkg_ver -from ros_buildfarm.pulp import PulpRpmClient - - -def main(argv=sys.argv[1:]): - parser = argparse.ArgumentParser( - description='Sync packages between pulp distributions') - add_argument_dry_run(parser) - add_argument_invalidate(parser) - parser.add_argument( - '--invalidate-expression', - default=None, - help='Any existing package names matching this expression will be removed') - add_argument_pulp_base_url(parser) - add_argument_pulp_password(parser) - add_argument_pulp_task_timeout(parser) - add_argument_pulp_username(parser) - parser.add_argument( - '--package-name-expression', default='.*', - help='Expression to match against packages in the repositories') - parser.add_argument( - '--distribution-source-expression', required=True, - help='Expression to match for source distribution names') - parser.add_argument( - '--distribution-dest-expression', required=True, - help='Expression to transform matching source distribution names into destination names') - args = parser.parse_args(argv) - - pulp_client = PulpRpmClient( - args.pulp_base_url, args.pulp_username, args.pulp_password, - task_timeout=args.pulp_task_timeout) - - dists_to_sync = [] - with Scope('SUBSECTION', 'enumerating distributions to sync'): - dist_expression = re.compile(args.distribution_source_expression) - distributions = {dist.name for dist in pulp_client.enumerate_distributions()} - for dist_source in sorted(distributions): - (dist_dest_pattern, matched_source) = dist_expression.subn( - args.distribution_dest_expression, dist_source) - if matched_source: - dist_dest_matches = [ - dist for dist in distributions if re.match(dist_dest_pattern, dist)] - if not dist_dest_matches: - print( - "No distributions match destination pattern '%s'" % dist_dest_pattern, - file=sys.stderr) - return 1 - dists_to_sync.extend((dist_source, dist_dest) for dist_dest in dist_dest_matches) - - dists_to_sync = sorted(set(dists_to_sync)) - print('Syncing %d distributions:' % len(dists_to_sync)) - for dist_source_dest in dists_to_sync: - print('- %s => %s' % dist_source_dest) - - packages = {} - with Scope('SUBSECTION', 'enumerating packages to sync'): - package_expression = re.compile(args.package_name_expression) - for dist_source, _ in dists_to_sync: - packages[dist_source] = { - pkg.pulp_href: pkg - for pkg in pulp_client.enumerate_pkgs_in_distribution_name(dist_source) - if package_expression.match(pkg.name)} - - print('Matched %d packages from source distributions:' % ( - sum([len(pkgs) for pkgs in packages.values()]))) - for dist_source, _ in dists_to_sync: - print('- %s: %d matching packages' % (dist_source, len(packages[dist_source]))) - - with Scope('SUBSECTION', 'invalidation and committing changes'): - for dist_source, dist_dest in dists_to_sync: - packages_to_sync = packages[dist_source] - if not packages_to_sync: - print('Skipping sync from %s to %s' % (dist_source, dist_dest)) - continue - print('Syncing %d packages from %s to %s...%s' % ( - len(packages_to_sync), dist_source, dist_dest, - ' (dry run)' if args.dry_run else '')) - package_cache = dict(packages_to_sync) - new_pkgs, pkgs_removed = pulp_client.import_and_invalidate( - dist_dest, packages_to_sync, args.invalidate_expression, - args.invalidate, package_cache=package_cache, dry_run=args.dry_run) - print('- Added %d packages%s' % ( - len(new_pkgs), ' (dry run)' if args.dry_run else '')) - for pkg in sorted(new_pkgs, key=lambda pkg: pkg.name): - print(' - %s-%s' % (pkg.name, format_pkg_ver(pkg))) - print('- Removed %d packages%s' % ( - len(pkgs_removed), ' (dry run)' if args.dry_run else '')) - for pkg in sorted(pkgs_removed, key=lambda pkg: pkg.name): - print(' - %s-%s' % (pkg.name, format_pkg_ver(pkg))) - - -if __name__ == '__main__': - sys.exit(main()) diff --git a/ros_buildfarm/scripts/release/rpm/upload_package.py b/ros_buildfarm/scripts/release/rpm/upload_package.py deleted file mode 100644 index 76d00fa8f..000000000 --- a/ros_buildfarm/scripts/release/rpm/upload_package.py +++ /dev/null @@ -1,64 +0,0 @@ -# Copyright 2020 Open Source Robotics Foundation, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import argparse -import sys - -from ros_buildfarm.argument import add_argument_pulp_base_url -from ros_buildfarm.argument import add_argument_pulp_password -from ros_buildfarm.argument import add_argument_pulp_resource_record -from ros_buildfarm.argument import add_argument_pulp_task_timeout -from ros_buildfarm.argument import add_argument_pulp_username -from ros_buildfarm.common import Scope -from ros_buildfarm.pulp import format_pkg_ver -from ros_buildfarm.pulp import PulpRpmClient - - -def main(argv=sys.argv[1:]): - parser = argparse.ArgumentParser( - description='Upload package to pulp') - parser.add_argument( - 'package_file', - nargs='+', metavar='FILE', - help='Package file paths to upload') - add_argument_pulp_base_url(parser) - add_argument_pulp_password(parser) - add_argument_pulp_task_timeout(parser) - add_argument_pulp_username(parser) - add_argument_pulp_resource_record(parser) - args = parser.parse_args(argv) - - pulp_client = PulpRpmClient( - args.pulp_base_url, args.pulp_username, args.pulp_password, - task_timeout=args.pulp_task_timeout) - - with Scope('SUBSECTION', 'upload package(s) to pulp'): - created_resources = [] - - for file_path in args.package_file: - print("Uploading '%s'." % file_path) - created_rpm = pulp_client.upload_pkg(file_path) - created_resources.append(created_rpm.pulp_href) - - print('Created RPM resource: %s' % created_rpm.pulp_href) - print("Package '%s' version: %s" % (created_rpm.name, format_pkg_ver(created_rpm))) - - if args.pulp_resource_record: - print("Saving upload record to '%s'." % args.pulp_resource_record) - with open(args.pulp_resource_record, 'w') as resource_record: - resource_record.write('PULP_RESOURCES=%s\n' % ' '.join(created_resources)) - - -if __name__ == '__main__': - sys.exit(main()) diff --git a/ros_buildfarm/templates/release/rpm/import_package_job.xml.em b/ros_buildfarm/templates/release/rpm/import_package_job.xml.em deleted file mode 100644 index 9c20421a0..000000000 --- a/ros_buildfarm/templates/release/rpm/import_package_job.xml.em +++ /dev/null @@ -1,107 +0,0 @@ - - - Generated at @ESCAPE(now_str) from template '@ESCAPE(template_name)' - false - -@(SNIPPET( - 'property_log-rotator', - days_to_keep=30, - num_to_keep=10000, -))@ -@(SNIPPET( - 'property_job-priority', - priority=-1, -))@ -@(SNIPPET( - 'property_rebuild-settings', -))@ -@(SNIPPET( - 'property_requeue-job', -))@ -@(SNIPPET( - 'property_parameters-definition', - parameters=[ - { - 'type': 'string', - 'name': 'DISTRIBUTION_NAME', - }, - { - 'type': 'string', - 'name': 'PULP_RESOURCES', - }, - { - 'type': 'boolean', - 'name': 'INVALIDATE_DOWNSTREAM', - }, - { - 'type': 'string', - 'name': 'INVALIDATE_EXPRESSION', - }, - { - 'type': 'boolean', - 'name': 'DRY_RUN', - 'description': 'Skip the commit operation but show what would change', - }, - ], -))@ -@(SNIPPET( - 'property_job-weight', -))@ - -@(SNIPPET( - 'scm_git', - url=ros_buildfarm_repository.url, - branch_name=ros_buildfarm_repository.version or 'master', - relative_target_dir='ros_buildfarm', - refspec=None, -))@ - 2 - building_repository - false - false - false - false - - false - -@(SNIPPET( - 'builder_shell', - script='\n'.join([ - 'echo "# BEGIN SECTION: import RPM package"', - 'if [ "$INVALIDATE_DOWNSTREAM" = "true" ]; then INVALIDATE_ARG=--invalidate; fi', - 'if [ "$DRY_RUN" = "true" ]; then DRY_RUN_ARG="--dry-run"; fi', - 'if [ "$INVALIDATE_EXPRESSION" != "" ]; then INVALIDATE_EXPRESSION_ARG="--invalidate-expression $INVALIDATE_EXPRESSION"; fi', - 'export PYTHONPATH=$WORKSPACE/ros_buildfarm:$PYTHONPATH', - 'python3 -u $WORKSPACE/ros_buildfarm/scripts/release/rpm/import_package.py' + - ' --pulp-distribution-name $DISTRIBUTION_NAME' + - ' $PULP_RESOURCES' + - ' $INVALIDATE_ARG' + - ' $INVALIDATE_EXPRESSION_ARG' + - ' $DRY_RUN_ARG', - 'echo "# END SECTION"', - ]), -))@ - - -@(SNIPPET( - 'publisher_description-setter', - regexp='Importing package: (\S+)', - # to prevent overwriting the description of failed builds - regexp_for_failed='ThisRegExpShouldNeverMatch', -))@ -@(SNIPPET( - 'publisher_extended-email', - recipients=notify_emails, -))@ - - -@(SNIPPET( - 'pulp_credentials', - credential_id=credential_id_pulp, - dest_credential_id=dest_credential_id, -))@ -@(SNIPPET( - 'build-wrapper_timestamper', -))@ - - diff --git a/ros_buildfarm/templates/snippet/pulp_credentials.xml.em b/ros_buildfarm/templates/snippet/pulp_credentials.xml.em deleted file mode 100644 index 560f4a762..000000000 --- a/ros_buildfarm/templates/snippet/pulp_credentials.xml.em +++ /dev/null @@ -1,16 +0,0 @@ -@(SNIPPET( - 'credentials_binding', - bindings=[ - { - 'id': credential_id, - 'type': 'user-pass', - 'user_var': 'PULP_USERNAME', - 'pass_var': 'PULP_PASSWORD', - }, - { - 'id': dest_credential_id, - 'type': 'string', - 'var': 'PULP_BASE_URL', - }, - ], -))@