diff --git a/doc/configuration_options.rst b/doc/configuration_options.rst index a27e3148f..41eb17c24 100644 --- a/doc/configuration_options.rst +++ b/doc/configuration_options.rst @@ -213,6 +213,11 @@ The following options are valid in version ``2`` (beside the generic options): * ``jenkins_source_job_priority``: the job priority of *source* jobs. * ``jenkins_source_job_timeout``: the job timeout for *source* jobs. +* ``jenkins_binary_job_weight_override``: per-package override of the number of + executors on a worker which are required to execute a job. + All jobs default to ``1``. + Uses the Jenkins Heavy Job plugin. + * ``notifications``: a dictionary with the following keys: * ``emails``: a list of static email addresses. diff --git a/ros_buildfarm/ci_job.py b/ros_buildfarm/ci_job.py index fc419efcb..d6d8dd8ac 100644 --- a/ros_buildfarm/ci_job.py +++ b/ros_buildfarm/ci_job.py @@ -27,6 +27,7 @@ from ros_buildfarm.common import JobValidationError from ros_buildfarm.common import write_groovy_script_and_configs from ros_buildfarm.config import get_ci_build_files +from ros_buildfarm.config import get_global_ci_build_files from ros_buildfarm.config import get_index as get_config_index from ros_buildfarm.git import get_repository from ros_buildfarm.jenkins import JenkinsProxy @@ -39,7 +40,10 @@ def configure_ci_jobs( groovy_script=None, dry_run=False): """Configure all Jenkins CI jobs.""" config = get_config_index(config_url) - build_files = get_ci_build_files(config, rosdistro_name) + if not rosdistro_name: + build_files = get_global_ci_build_files(config) + else: + build_files = get_ci_build_files(config, rosdistro_name) if not ci_build_names: ci_build_names = build_files.keys() @@ -146,7 +150,10 @@ def configure_ci_job( if config is None: config = get_config_index(config_url) if build_file is None: - build_files = get_ci_build_files(config, rosdistro_name) + if not rosdistro_name: + build_files = get_global_ci_build_files(config) + else: + build_files = get_ci_build_files(config, rosdistro_name) build_file = build_files[ci_build_name] # Overwrite build_file.targets if build_targets is specified if build_targets is not None: diff --git a/ros_buildfarm/config/release_build_file.py b/ros_buildfarm/config/release_build_file.py index 76df34b6d..e50b8bd37 100644 --- a/ros_buildfarm/config/release_build_file.py +++ b/ros_buildfarm/config/release_build_file.py @@ -69,6 +69,11 @@ def __init__(self, name, data): # noqa: D107 self.jenkins_source_job_timeout = \ int(data['jenkins_source_job_timeout']) + self.jenkins_binary_job_weight_overrides = {} + if 'jenkins_binary_job_weight_overrides' in data: + self.jenkins_binary_job_weight_overrides = data['jenkins_binary_job_weight_overrides'] + assert isinstance(self.jenkins_binary_job_weight_overrides, dict) + self.package_whitelist = [] if 'package_whitelist' in data and data['package_whitelist']: self.package_whitelist = data['package_whitelist'] diff --git a/ros_buildfarm/release_job.py b/ros_buildfarm/release_job.py index c156153a0..caf4965ab 100644 --- a/ros_buildfarm/release_job.py +++ b/ros_buildfarm/release_job.py @@ -719,6 +719,7 @@ def _get_binarydeb_job_config( 'github_url': get_github_project_url(release_repository.url), 'job_priority': build_file.jenkins_binary_job_priority, + 'job_weight': build_file.jenkins_binary_job_weight_overrides.get(pkg_name), 'node_label': get_node_label( build_file.jenkins_binary_job_label, get_default_node_label('%s_%s%s_%s' % ( diff --git a/ros_buildfarm/scripts/doc/build_rosdoc2.py b/ros_buildfarm/scripts/doc/build_rosdoc2.py index 831b93ed4..60280a1b3 100644 --- a/ros_buildfarm/scripts/doc/build_rosdoc2.py +++ b/ros_buildfarm/scripts/doc/build_rosdoc2.py @@ -54,6 +54,7 @@ def main(argv=sys.argv[1:]): '-m', 'pip', 'install', + '--no-warn-script-location', '.'], env=env, cwd=args.rosdoc2_dir) diff --git a/ros_buildfarm/templates/release/deb/binarypkg_job.xml.em b/ros_buildfarm/templates/release/deb/binarypkg_job.xml.em index 38aefbfaa..c547ca8e3 100644 --- a/ros_buildfarm/templates/release/deb/binarypkg_job.xml.em +++ b/ros_buildfarm/templates/release/deb/binarypkg_job.xml.em @@ -40,6 +40,7 @@ but disabled since the package is blacklisted (or not whitelisted) in the config ))@ @(SNIPPET( 'property_job-weight', + weight=job_weight, ))@ @(SNIPPET( diff --git a/ros_buildfarm/templates/snippet/add_distribution_repositories.Dockerfile.em b/ros_buildfarm/templates/snippet/add_distribution_repositories.Dockerfile.em index 05959939a..13b88a610 100644 --- a/ros_buildfarm/templates/snippet/add_distribution_repositories.Dockerfile.em +++ b/ros_buildfarm/templates/snippet/add_distribution_repositories.Dockerfile.em @@ -9,7 +9,7 @@ ubuntu_before_bionic = ( @[if os_name == 'debian' and os_code_name not in debian_before_stretch or os_name == 'ubuntu' and os_code_name not in ubuntu_before_bionic]@ @# In Debian Stretch apt doesn't depend on gnupg anymore @# https://anonscm.debian.org/cgit/apt/apt.git/commit/?id=87d468fe355c87325c943c40043a0bb236b2407f -RUN for i in 1 2 3; do apt-get update && apt-get install -q -y gnupg && apt-get clean && break || if [ $i -lt 3 ]; then sleep 5; else false; fi; done +RUN for i in 1 2 3; do apt-get update && apt-get install -q -y gnupg ca-certificates && apt-get clean && break || if [ $i -lt 3 ]; then sleep 5; else false; fi; done @[end if]@ @[for i, key in enumerate(distribution_repository_keys)]@ RUN echo "@('\\n'.join(key.splitlines()))" > /tmp/keys/@(i).key && apt-key add /tmp/keys/@(i).key