Merge branch 'master' into cottsay/yaml-relative-url

doc/configuration_options.rst

@@ -213,6 +213,11 @@ The following options are valid in version ``2`` (beside the generic options):
 * ``jenkins_source_job_priority``: the job priority of *source* jobs.
 * ``jenkins_source_job_timeout``: the job timeout for *source* jobs.
 
+* ``jenkins_binary_job_weight_override``: per-package override of the number of
+  executors on a worker which are required to execute a job.
+  All jobs default to ``1``.
+  Uses the Jenkins Heavy Job plugin.
+
 * ``notifications``: a dictionary with the following keys:
 
   * ``emails``: a list of static email addresses.

ros_buildfarm/ci_job.py

@@ -27,6 +27,7 @@
 from ros_buildfarm.common import JobValidationError
 from ros_buildfarm.common import write_groovy_script_and_configs
 from ros_buildfarm.config import get_ci_build_files
+from ros_buildfarm.config import get_global_ci_build_files
 from ros_buildfarm.config import get_index as get_config_index
 from ros_buildfarm.git import get_repository
 from ros_buildfarm.jenkins import JenkinsProxy
@@ -39,7 +40,10 @@ def configure_ci_jobs(
         groovy_script=None, dry_run=False):
     """Configure all Jenkins CI jobs."""
     config = get_config_index(config_url)
-    build_files = get_ci_build_files(config, rosdistro_name)
+    if not rosdistro_name:
+        build_files = get_global_ci_build_files(config)
+    else:
+        build_files = get_ci_build_files(config, rosdistro_name)
 
     if not ci_build_names:
         ci_build_names = build_files.keys()
@@ -146,7 +150,10 @@ def configure_ci_job(
     if config is None:
         config = get_config_index(config_url)
     if build_file is None:
-        build_files = get_ci_build_files(config, rosdistro_name)
+        if not rosdistro_name:
+            build_files = get_global_ci_build_files(config)
+        else:
+            build_files = get_ci_build_files(config, rosdistro_name)
         build_file = build_files[ci_build_name]
     # Overwrite build_file.targets if build_targets is specified
     if build_targets is not None:

ros_buildfarm/config/release_build_file.py

@@ -69,6 +69,11 @@ def __init__(self, name, data):  # noqa: D107
             self.jenkins_source_job_timeout = \
                 int(data['jenkins_source_job_timeout'])
 
+        self.jenkins_binary_job_weight_overrides = {}
+        if 'jenkins_binary_job_weight_overrides' in data:
+            self.jenkins_binary_job_weight_overrides = data['jenkins_binary_job_weight_overrides']
+            assert isinstance(self.jenkins_binary_job_weight_overrides, dict)
+
         self.package_whitelist = []
         if 'package_whitelist' in data and data['package_whitelist']:
             self.package_whitelist = data['package_whitelist']

ros_buildfarm/release_job.py

@@ -719,6 +719,7 @@ def _get_binarydeb_job_config(
         'github_url': get_github_project_url(release_repository.url),
 
         'job_priority': build_file.jenkins_binary_job_priority,
+        'job_weight': build_file.jenkins_binary_job_weight_overrides.get(pkg_name),
         'node_label': get_node_label(
             build_file.jenkins_binary_job_label,
             get_default_node_label('%s_%s%s_%s' % (

ros_buildfarm/scripts/doc/build_rosdoc2.py

@@ -54,6 +54,7 @@ def main(argv=sys.argv[1:]):
                                   '-m',
                                   'pip',
                                   'install',
+                                  '--no-warn-script-location',
                                   '.'],
                                  env=env,
                                  cwd=args.rosdoc2_dir)

ros_buildfarm/templates/release/deb/binarypkg_job.xml.em

@@ -40,6 +40,7 @@ but disabled since the package is blacklisted (or not whitelisted) in the config
 ))@
 @(SNIPPET(
     'property_job-weight',
+    weight=job_weight,
 ))@
   </properties>
 @(SNIPPET(

ros_buildfarm/templates/snippet/add_distribution_repositories.Dockerfile.em

@@ -9,7 +9,7 @@ ubuntu_before_bionic = (
 @[if os_name == 'debian' and os_code_name not in debian_before_stretch or os_name == 'ubuntu' and os_code_name not in ubuntu_before_bionic]@
 @# In Debian Stretch apt doesn't depend on gnupg anymore
 @# https://anonscm.debian.org/cgit/apt/apt.git/commit/?id=87d468fe355c87325c943c40043a0bb236b2407f
-RUN for i in 1 2 3; do apt-get update && apt-get install -q -y gnupg && apt-get clean && break || if [ $i -lt 3 ]; then sleep 5; else false; fi; done
+RUN for i in 1 2 3; do apt-get update && apt-get install -q -y gnupg ca-certificates && apt-get clean && break || if [ $i -lt 3 ]; then sleep 5; else false; fi; done
 @[end if]@
 @[for i, key in enumerate(distribution_repository_keys)]@
 RUN echo "@('\\n'.join(key.splitlines()))" > /tmp/keys/@(i).key && apt-key add /tmp/keys/@(i).key