diff --git a/ros_buildfarm/jenkins.py b/ros_buildfarm/jenkins.py index edf0451a3..01b3f1b73 100644 --- a/ros_buildfarm/jenkins.py +++ b/ros_buildfarm/jenkins.py @@ -274,9 +274,8 @@ def _diff_configs(remote_config, new_config, context_lines): if ElementTree.tostring(remote_root) == ElementTree.tostring(new_root): return [] - encoding_type = 'utf-8' if sys.version_info[0] < 3 else 'unicode' - xml1 = ElementTree.tostring(remote_root, encoding=encoding_type) - xml2 = ElementTree.tostring(new_root, encoding=encoding_type) + xml1 = ElementTree.tostring(remote_root, encoding='unicode') + xml2 = ElementTree.tostring(new_root, encoding='unicode') lines1 = xml1.splitlines() lines2 = xml2.splitlines() diff --git a/ros_buildfarm/scripts/release/generate_release_script.py b/ros_buildfarm/scripts/release/generate_release_script.py index fb31278af..4dd019c89 100644 --- a/ros_buildfarm/scripts/release/generate_release_script.py +++ b/ros_buildfarm/scripts/release/generate_release_script.py @@ -42,10 +42,18 @@ def main(argv=sys.argv[1:]): add_argument_os_name(parser) add_argument_os_code_name(parser) add_argument_arch(parser) + parser.add_argument( + '--skip-binary', + action='store_true', + help='Skip the entire binary package build process') parser.add_argument( '--skip-install', action='store_true', help='Skip trying to install binarydeb') + parser.add_argument( + '--skip-source', + action='store_true', + help='Skip the entire source package build process') args = parser.parse_args(argv) package_format = package_format_mapping[args.os_name] @@ -103,28 +111,29 @@ def beforeInclude(self, *args, **kwargs): args.package_name, args.os_name, args.os_code_name, args.arch) separator_index = hook.scripts.index('--') - source_scripts = hook.scripts[:separator_index] - binary_scripts = hook.scripts[separator_index + 1:] - - # inject additional argument to skip fetching sourcedeb from repo - script_name = '/run_binary%s_job.py ' % deb_or_pkg - additional_argument = '--skip-download-sourcepkg ' - for i, script in enumerate(binary_scripts): - offset = script.find(script_name) - if offset != -1: - offset += len(script_name) - script = script[:offset] + additional_argument + script[offset:] - binary_scripts[i] = script - break - - # remove rm command for sourcedeb location - rm_command = 'rm -fr $WORKSPACE/binary%s' % deb_or_pkg - for i, script in enumerate(binary_scripts): - offset = script.find(rm_command) - if offset != -1: - script = script[:offset] + script[offset + len(rm_command):] - binary_scripts[i] = script - break + source_scripts = [] if args.skip_source else hook.scripts[:separator_index] + binary_scripts = [] if args.skip_binary else hook.scripts[separator_index + 1:] + + if source_scripts: + # inject additional argument to skip fetching sourcedeb from repo + script_name = '/run_binary%s_job.py ' % deb_or_pkg + additional_argument = '--skip-download-sourcepkg ' + for i, script in enumerate(binary_scripts): + offset = script.find(script_name) + if offset != -1: + offset += len(script_name) + script = script[:offset] + additional_argument + script[offset:] + binary_scripts[i] = script + break + + # remove rm command for sourcedeb location + rm_command = 'rm -fr $WORKSPACE/binary%s' % deb_or_pkg + for i, script in enumerate(binary_scripts): + offset = script.find(rm_command) + if offset != -1: + script = script[:offset] + script[offset + len(rm_command):] + binary_scripts[i] = script + break if args.skip_install: # remove install step diff --git a/ros_buildfarm/status_page.py b/ros_buildfarm/status_page.py index 702be7fa7..6bd20a585 100644 --- a/ros_buildfarm/status_page.py +++ b/ros_buildfarm/status_page.py @@ -582,16 +582,15 @@ def _version_is_gt_other(version, other_version): def _get_comparable_loose_versions(version_str1, version_str2): loose_version1 = LooseVersion(version_str1) loose_version2 = LooseVersion(version_str2) - if sys.version_info[0] > 2: - # might raise TypeError in Python 3: http://bugs.python.org/issue14894 - version_parts1 = loose_version1.version - version_parts2 = loose_version2.version - for i in range(min(len(version_parts1), len(version_parts2))): - try: - version_parts1[i] < version_parts2[i] - except TypeError: - version_parts1[i] = str(version_parts1[i]) - version_parts2[i] = str(version_parts2[i]) + # might raise TypeError in Python 3: http://bugs.python.org/issue14894 + version_parts1 = loose_version1.version + version_parts2 = loose_version2.version + for i in range(min(len(version_parts1), len(version_parts2))): + try: + version_parts1[i] < version_parts2[i] + except TypeError: + version_parts1[i] = str(version_parts1[i]) + version_parts2[i] = str(version_parts2[i]) return loose_version1, loose_version2 diff --git a/ros_buildfarm/templates/release/deb/binarypkg_job.xml.em b/ros_buildfarm/templates/release/deb/binarypkg_job.xml.em index c547ca8e3..18fa14b73 100644 --- a/ros_buildfarm/templates/release/deb/binarypkg_job.xml.em +++ b/ros_buildfarm/templates/release/deb/binarypkg_job.xml.em @@ -119,7 +119,7 @@ but disabled since the package is blacklisted (or not whitelisted) in the config 'echo "# BEGIN SECTION: Build Dockerfile - binarydeb task"', 'cd $WORKSPACE/docker_generating_docker', 'python3 -u $WORKSPACE/ros_buildfarm/scripts/misc/docker_pull_baseimage.py', - 'docker build --force-rm -t binarydeb_task_generation.%s_%s_%s_%s_%s .' % (rosdistro_name, os_name, os_code_name, arch, pkg_name), + 'docker build --network=host --force-rm -t binarydeb_task_generation.%s_%s_%s_%s_%s .' % (rosdistro_name, os_name, os_code_name, arch, pkg_name), 'echo "# END SECTION"', '', 'echo "# BEGIN SECTION: Run Dockerfile - binarydeb task"', @@ -138,6 +138,7 @@ but disabled since the package is blacklisted (or not whitelisted) in the config ' --rm ' + ' --cidfile=$WORKSPACE/docker_generating_docker/docker.cid' + ' -e=TRAVIS=$TRAVIS' + + ' --net=host' + ' -e=ROS_BUILDFARM_PULL_REQUEST_BRANCH=$ROS_BUILDFARM_PULL_REQUEST_BRANCH' + ' -v $WORKSPACE/ros_buildfarm:/tmp/ros_buildfarm:ro' + ' -v $WORKSPACE/binarydeb:/tmp/binarydeb' + @@ -159,7 +160,7 @@ but disabled since the package is blacklisted (or not whitelisted) in the config '# build and run build_binarydeb Dockerfile', 'cd $WORKSPACE/docker_build_binarydeb', 'python3 -u $WORKSPACE/ros_buildfarm/scripts/misc/docker_pull_baseimage.py', - 'docker build --force-rm -t binarydeb_build.%s_%s_%s_%s_%s .' % (rosdistro_name, os_name, os_code_name, arch, pkg_name), + 'docker build --network=host --force-rm -t binarydeb_build.%s_%s_%s_%s_%s .' % (rosdistro_name, os_name, os_code_name, arch, pkg_name), 'echo "# END SECTION"', '', 'echo "# BEGIN SECTION: Run Dockerfile - build binarydeb"', @@ -224,7 +225,7 @@ but disabled since the package is blacklisted (or not whitelisted) in the config @# 'echo "# BEGIN SECTION: Build Dockerfile - install"', @# 'cd $WORKSPACE/docker_install_binarydeb', @# 'python3 -u $WORKSPACE/ros_buildfarm/scripts/misc/docker_pull_baseimage.py', -@# 'docker build --force-rm -t binarydeb_install.%s_%s_%s_%s_%s .' % (rosdistro_name, os_name, os_code_name, arch, pkg_name), +@# 'docker build --network=host --force-rm -t binarydeb_install.%s_%s_%s_%s_%s .' % (rosdistro_name, os_name, os_code_name, arch, pkg_name), @# 'echo "# END SECTION"', @# '', @# 'echo "# BEGIN SECTION: Run Dockerfile - install"', diff --git a/ros_buildfarm/templates/release/deb/sourcepkg_job.xml.em b/ros_buildfarm/templates/release/deb/sourcepkg_job.xml.em index 2b3101195..50e712e57 100644 --- a/ros_buildfarm/templates/release/deb/sourcepkg_job.xml.em +++ b/ros_buildfarm/templates/release/deb/sourcepkg_job.xml.em @@ -104,7 +104,7 @@ but disabled since the package is blacklisted (or not whitelisted) in the config 'echo "# BEGIN SECTION: Build Dockerfile - generate sourcedeb"', 'cd $WORKSPACE/docker_sourcedeb', 'python3 -u $WORKSPACE/ros_buildfarm/scripts/misc/docker_pull_baseimage.py', - 'docker build --force-rm -t sourcedeb.%s_%s_%s_%s .' % (rosdistro_name, os_name, os_code_name, pkg_name), + 'docker build --network=host --force-rm -t sourcedeb.%s_%s_%s_%s .' % (rosdistro_name, os_name, os_code_name, pkg_name), 'echo "# END SECTION"', '', 'echo "# BEGIN SECTION: Run Dockerfile - generate sourcedeb"', diff --git a/ros_buildfarm/templates/release/release_script.sh.em b/ros_buildfarm/templates/release/release_script.sh.em index eda772dc7..9a415b936 100644 --- a/ros_buildfarm/templates/release/release_script.sh.em +++ b/ros_buildfarm/templates/release/release_script.sh.em @@ -45,6 +45,7 @@ echo "Using workspace for binary: $WORKSPACE" mkdir -p $WORKSPACE cd $WORKSPACE +@[if source_scripts]@ echo "" echo "Get artifacts from source job" @[if package_format == 'deb']@ @@ -56,6 +57,7 @@ mkdir -p $WORKSPACE/binarypkg/source @[else]@ @{assert False, "Unsupported packaging format '%s'" % package_format}@ @[end if]@ +@[end if]@ @(TEMPLATE( 'devel/devel_script_build.sh.em', diff --git a/ros_buildfarm/templates/release/rpm/binarypkg_job.xml.em b/ros_buildfarm/templates/release/rpm/binarypkg_job.xml.em index 573da5971..8fff5ac3c 100644 --- a/ros_buildfarm/templates/release/rpm/binarypkg_job.xml.em +++ b/ros_buildfarm/templates/release/rpm/binarypkg_job.xml.em @@ -130,6 +130,8 @@ but disabled since the package is blacklisted (or not whitelisted) in the config ] + ([ 'if [ ! -d "$HOME/.ccache" ]; then mkdir $HOME/.ccache; fi', ] if shared_ccache else []) + [ + '# If using Podman, change the user namespace to preserve UID. No effect if using Docker.', + 'export PODMAN_USERNS=keep-id', 'docker run' + ' --rm' + ' --privileged' + diff --git a/ros_buildfarm/templates/release/rpm/binarypkg_task.Dockerfile.em b/ros_buildfarm/templates/release/rpm/binarypkg_task.Dockerfile.em index a37b38d0b..dfcbef858 100644 --- a/ros_buildfarm/templates/release/rpm/binarypkg_task.Dockerfile.em +++ b/ros_buildfarm/templates/release/rpm/binarypkg_task.Dockerfile.em @@ -30,6 +30,7 @@ RUN dnf update --refresh -y RUN echo -e "@('\\n'.join(key.splitlines()))" > /etc/pki/mock/RPM-GPG-KEY-ros-buildfarm-@(i) @[end for]@ COPY mock_config.cfg /etc/mock/ros_buildfarm.cfg +RUN chmod 644 /etc/mock/ros_buildfarm.cfg USER buildfarm ENTRYPOINT ["sh", "-c"] diff --git a/ros_buildfarm/templates/release/rpm/mock_config.cfg.em b/ros_buildfarm/templates/release/rpm/mock_config.cfg.em index 4e577ad67..06ab70e18 100644 --- a/ros_buildfarm/templates/release/rpm/mock_config.cfg.em +++ b/ros_buildfarm/templates/release/rpm/mock_config.cfg.em @@ -1,5 +1,13 @@ include('/etc/mock/default.cfg') +def _expanded(k, opts): + """Get a config value and ensure it has been expanded.""" + exp, opts['__jinja_expand'] = opts.get('__jinja_expand', False), True + try: + return opts[k] + finally: + opts['__jinja_expand'] = exp + # Change the root name since we're modifying the chroot config_opts['root'] += '-ros-buildfarm' @@ -10,7 +18,13 @@ config_opts['use_bootstrap'] = False config_opts['chroot_setup_cmd'] += ' python3-rpm-macros' # Install weak dependencies to get group members -config_opts[f'dnf_builddep_opts'] = config_opts.get(f'dnf_builddep_opts', []) + ['--setopt=install_weak_deps=True'] +package_manager = _expanded('package_manager', config_opts) +additional_opts = ['--setopt=install_weak_deps=True'] +config_opts[f'{package_manager}_builddep_opts'] = config_opts.get(f'{package_manager}_builddep_opts', []) + additional_opts + +# Deal with dnf -> dnf4 transition - see rpm-software-management/mock#1496 +if package_manager == 'dnf': + config_opts[f'dnf4_builddep_opts'] = config_opts.get(f'dnf4_builddep_opts', []) + additional_opts @[if env_vars]@ # Set environment vars from the build config diff --git a/ros_buildfarm/templates/release/rpm/sourcepkg_job.xml.em b/ros_buildfarm/templates/release/rpm/sourcepkg_job.xml.em index 8f3e00acc..a91a89908 100644 --- a/ros_buildfarm/templates/release/rpm/sourcepkg_job.xml.em +++ b/ros_buildfarm/templates/release/rpm/sourcepkg_job.xml.em @@ -110,6 +110,8 @@ but disabled since the package is blacklisted (or not whitelisted) in the config 'echo "# BEGIN SECTION: Run Dockerfile - generate sourcerpm"', 'rm -fr $WORKSPACE/sourcepkg', 'mkdir -p $WORKSPACE/sourcepkg', + '# If using Podman, change the user namespace to preserve UID. No effect if using Docker.', + 'export PODMAN_USERNS=keep-id', 'docker run' + ' --rm' + ' --privileged' + diff --git a/ros_buildfarm/templates/release/rpm/sourcepkg_task.Dockerfile.em b/ros_buildfarm/templates/release/rpm/sourcepkg_task.Dockerfile.em index 8d61ba32b..d187e125e 100644 --- a/ros_buildfarm/templates/release/rpm/sourcepkg_task.Dockerfile.em +++ b/ros_buildfarm/templates/release/rpm/sourcepkg_task.Dockerfile.em @@ -25,6 +25,7 @@ RUN dnf update --refresh -y RUN echo -e "@('\\n'.join(key.splitlines()))" > /etc/pki/mock/RPM-GPG-KEY-ros-buildfarm-@(i) @[end for]@ COPY mock_config.cfg /etc/mock/ros_buildfarm.cfg +RUN chmod 644 /etc/mock/ros_buildfarm.cfg USER buildfarm ENTRYPOINT ["sh", "-c"] diff --git a/ros_buildfarm/templates/snippet/add_distribution_repositories.Dockerfile.em b/ros_buildfarm/templates/snippet/add_distribution_repositories.Dockerfile.em index 8d65d0c79..5a478535e 100644 --- a/ros_buildfarm/templates/snippet/add_distribution_repositories.Dockerfile.em +++ b/ros_buildfarm/templates/snippet/add_distribution_repositories.Dockerfile.em @@ -14,10 +14,10 @@ RUN for i in 1 2 3; do apt-get update && apt-get install -q -y gnupg ca-certific @[for i, key in enumerate(distribution_repository_keys)]@ RUN echo "@('\\n'.join(key.splitlines()))" > /tmp/keys/@(i).key@[if key] && apt-key add /tmp/keys/@(i).key@[end if] @[end for]@ -@[for url in distribution_repository_urls]@ -RUN echo deb @url @os_code_name main | tee -a /etc/apt/sources.list.d/buildfarm.list +@[for i, url in enumerate(distribution_repository_urls)]@ +RUN echo deb @[if not distribution_repository_keys[i]][trusted=yes] @[end if]@ @url @os_code_name main | tee -a /etc/apt/sources.list.d/buildfarm.list @[if add_source and url == target_repository]@ -RUN echo deb-src @url @os_code_name main | tee -a /etc/apt/sources.list.d/buildfarm.list +RUN echo deb-src @[if not distribution_repository_keys[i]][trusted=yes] @[end if]@ @url @os_code_name main | tee -a /etc/apt/sources.list.d/buildfarm.list @[end if]@ @[end for]@ @# On Ubuntu Trusty a newer version of dpkg is required to install Debian packages created by stdeb on newer distros diff --git a/setup.py b/setup.py index d40b5ca61..cb7ad7c79 100644 --- a/setup.py +++ b/setup.py @@ -25,6 +25,7 @@ # - ros_buildfarm/__init__.py # - stdeb.cfg 'version': '3.0.1-master', + 'python_requires': '>=3.6', 'packages': find_packages(exclude=['test']), 'package_data': { 'ros_buildfarm.templates': [ @@ -72,9 +73,6 @@ 'license': 'Apache 2.0', } -if os.sys.version_info[0] == 2: - kwargs['install_requires'].append('configparser') - if 'SKIP_PYTHON_MODULES' in os.environ: kwargs['packages'] = [] elif 'SKIP_PYTHON_SCRIPTS' in os.environ: diff --git a/stdeb.cfg b/stdeb.cfg index b0c71eaa1..5497fc842 100644 --- a/stdeb.cfg +++ b/stdeb.cfg @@ -3,26 +3,18 @@ Debian-Version: 100 ; ros-buildfarm-modules same version (without the branch suffix) as in: ; - ros_buildfarm/__init__.py ; - setup.py -Depends: python-argparse, python-catkin-pkg-modules, python-ros-buildfarm-modules (>= 3.0.0), python-rosdistro-modules (>= 1.0.0), python-yaml -; ros-buildfarm-modules same version (without the branch suffix) as in: -; - ros_buildfarm/__init__.py -; - setup.py Depends3: python3-catkin-pkg-modules (>= 0.2.6), python3-ros-buildfarm-modules (>= 3.0.0), python3-rosdistro-modules (>= 1.0.0), python3-yaml -Conflicts: python3-ros-buildfarm Conflicts3: python-ros-buildfarm -Suite: xenial yakkety zesty artful bionic cosmic disco eoan stretch buster -Suite3: xenial yakkety zesty artful bionic cosmic disco eoan focal stretch buster -Python2-Depends-Name: python +Suite3: focal jammy noble bookworm trixie +No-Python2: +X-Python3-Version: >= 3.6 Setup-Env-Vars: SKIP_PYTHON_MODULES=1 [ros_buildfarm_modules] -Depends: python-catkin-pkg-modules, python-configparser, python-empy, python-rosdistro-modules (>= 1.0.0), python-yaml, python3-empy, python3-vcstool (>= 0.1.37) Depends3: python3-catkin-pkg-modules (>= 0.2.6), python3-empy (< 4), python3-jenkinsapi, python3-rosdistro-modules (>= 1.0.0), python3-vcstool (>= 0.1.37), python3-yaml -Conflicts: python-ros-buildfarm (<< 1.3.0) Conflicts3: python3-ros-buildfarm (<< 1.3.0) -Replaces: python-ros-buildfarm (<< 1.3.0) Replaces3: python3-ros-buildfarm (<< 1.3.0) -Suite: xenial yakkety zesty artful bionic cosmic disco eoan stretch buster -Suite3: xenial yakkety zesty artful bionic cosmic disco eoan focal stretch buster -Python2-Depends-Name: python +Suite3: focal jammy noble bookworm trixie +No-Python2: +X-Python3-Version: >= 3.6 Setup-Env-Vars: SKIP_PYTHON_SCRIPTS=1