From bbbfb02782e649fee1374e0beed7a223e2f4028f Mon Sep 17 00:00:00 2001 From: Tobias Fischer Date: Fri, 19 Jan 2024 09:05:33 +1000 Subject: [PATCH 01/57] Initial modifications for rattler-build new recipe.yaml format --- vinca/main.py | 212 ++++++++++++++++++---------------------------- vinca/template.py | 11 ++- 2 files changed, 91 insertions(+), 132 deletions(-) diff --git a/vinca/main.py b/vinca/main.py index d123ef3..ed6cbf8 100644 --- a/vinca/main.py +++ b/vinca/main.py @@ -226,31 +226,29 @@ def generate_output(pkg_shortname, vinca_conf, distro, version, all_pkgs=None): if pkg_names[0] in vinca_conf["skip_built_packages"]: return None - # TODO: Remove hardcoded cmake version after building new versions of ament_cmake_export_target - # see: https://github.com/ament/ament_cmake/commit/796cef7d7df2ddb806f774a9889e608cc82285d3 output = { "package": {"name": pkg_names[0], "version": version}, "requirements": { "build": [ - "{{ compiler('cxx') }}", - "{{ compiler('c') }}", - {"sel(linux64)": "sysroot_linux-64 2.17"}, + "${{ compiler('cxx') }}", + "${{ compiler('c') }}", + {"if": "linux64", "then": ["sysroot_linux-64 2.17"]}, "ninja", "setuptools", - {"sel(unix)": "make"}, - {"sel(unix)": "coreutils"}, - {"sel(osx)": "tapi"}, - {"sel(build_platform != target_platform)": "pkg-config"}, + {"if": "unix", "then": ["make", "coreutils"]}, + {"if": "osx", "then": ["tapi"]}, + {"if": "build_platform != target_platform", "then": ["pkg-config"]}, "cmake", "cython", - {"sel(build_platform != target_platform)": "python"}, + {"if": "build_platform != target_platform", "then": ["python"]}, { - "sel(build_platform != target_platform)": "cross-python_{{ target_platform }}" + "if": "build_platform != target_platform", + "then": ["cross-python_{{ target_platform }}"] }, - {"sel(build_platform != target_platform)": "numpy"}, + {"if": "build_platform != target_platform", "then": ["numpy"]}, ], "host": [ - {"sel(build_platform == target_platform)": "pkg-config"}, + {"if": "build_platform == target_platform", "then": ["pkg-config"]}, "numpy", "pip", ], @@ -269,20 +267,20 @@ def generate_output(pkg_shortname, vinca_conf, distro, version, all_pkgs=None): output["requirements"]["run"].extend(resolved_python) output["requirements"]["host"].extend(resolved_python) if pkg.get_build_type() in ["cmake", "catkin"]: - output["build"]["script"] = { - "sel(win)": "bld_catkin.bat", - "sel(unix)": "build_catkin.sh", - } + output["build"]["script"] = [ + {"if": "win", "then": ["bld_catkin.bat"]}, + {"if": "unix", "then": ["build_catkin.sh"]}, + ] elif pkg.get_build_type() in ["ament_cmake"]: - output["build"]["script"] = { - "sel(win)": "bld_ament_cmake.bat", - "sel(unix)": "build_ament_cmake.sh", - } + output["build"]["script"] = [ + {"if": "win", "then": ["bld_ament_cmake.bat"]}, + {"if": "unix", "then": ["build_ament_cmake.sh"]}, + ] elif pkg.get_build_type() in ["ament_python"]: - output["build"]["script"] = { - "sel(win)": "bld_ament_python.bat", - "sel(unix)": "build_ament_python.sh", - } + output["build"]["script"] = [ + {"if": "win", "then": ["bld_ament_python.bat"]}, + {"if": "unix", "then": ["build_ament_python.sh"]}, + ] resolved_setuptools = resolve_pkgname("python-setuptools", vinca_conf, distro) output["requirements"]["host"].extend(resolved_setuptools) else: @@ -337,12 +335,11 @@ def generate_output(pkg_shortname, vinca_conf, distro, version, all_pkgs=None): build_deps.append(dep) # Hack to add cyclonedds into build for cross compilation - if pkg_shortname == "cyclonedds" or "cyclonedds" in ( - build_deps + build_tool_deps - ): + if pkg_shortname == "cyclonedds" or "cyclonedds" in (build_deps + build_tool_deps): output["requirements"]["build"].append( { - "sel(build_platform != target_platform)": f"ros-{config.ros_distro}-cyclonedds" + "if": "build_platform != target_platform", + "then": [f"ros-{config.ros_distro}-cyclonedds"] } ) @@ -391,45 +388,27 @@ def sortkey(k): output["requirements"]["host"] = sorted(output["requirements"]["host"], key=sortkey) output["requirements"]["run"] += [ - { - "sel(osx and x86_64)": "__osx >={{ MACOSX_DEPLOYMENT_TARGET|default('10.14') }}" - } + {"if": "osx and x86_64", "then": ["__osx >={{ MACOSX_DEPLOYMENT_TARGET|default('10.14') }}"]} ] if f"ros-{config.ros_distro}-pybind11-vendor" in output["requirements"]["host"]: output["requirements"]["host"] += ["pybind11"] if "pybind11" in output["requirements"]["host"]: output["requirements"]["build"] += [ - {"sel(build_platform != target_platform)": "pybind11"} - ] - # pkg-config + pyqt-builder + git + doxygen must be in build, not host for cross-compile - if "doxygen" in output["requirements"]["host"]: - output["requirements"]["build"] += [ - {"sel(build_platform != target_platform)": "doxygen"} - ] - while "doxygen" in output["requirements"]["host"]: - output["requirements"]["host"].remove("doxygen") - output["requirements"]["host"] += [ - {"sel(build_platform == target_platform)": "doxygen"} - ] - if "pyqt-builder" in output["requirements"]["host"]: - output["requirements"]["build"] += [ - {"sel(build_platform != target_platform)": "pyqt-builder"} - ] - while "pyqt-builder" in output["requirements"]["host"]: - output["requirements"]["host"].remove("pyqt-builder") - output["requirements"]["host"] += [ - {"sel(build_platform == target_platform)": "pyqt-builder"} - ] - if "git" in output["requirements"]["host"]: - output["requirements"]["build"] += [ - {"sel(build_platform != target_platform)": "git"} - ] - while "git" in output["requirements"]["host"]: - output["requirements"]["host"].remove("git") - output["requirements"]["host"] += [ - {"sel(build_platform == target_platform)": "git"} + {"if": "build_platform != target_platform", "then": ["pybind11"]} ] + # pyqt-builder + git + doxygen must be in build, not host for cross-compile + pkgs_move_to_build = ["pyqt-builder", "git", "doxygen"] + for pkg_move_to_build in pkgs_move_to_build: + if pkg_move_to_build in output["requirements"]["host"]: + output["requirements"]["build"] += [ + {"if": "build_platform != target_platform", "then": [pkg_move_to_build]} + ] + while pkg_move_to_build in output["requirements"]["host"]: + output["requirements"]["host"].remove(pkg_move_to_build) + output["requirements"]["host"] += [ + {"if": "build_platform == target_platform", "then": [pkg_move_to_build]} + ] # fix up OPENGL support for Unix if ( @@ -443,25 +422,14 @@ def sortkey(k): output["requirements"]["host"].remove("REQUIRE_OPENGL") output["requirements"]["build"] += [ - {"sel(linux)": "{{ cdt('mesa-libgl-devel') }}"}, - {"sel(linux)": "{{ cdt('mesa-dri-drivers') }}"}, - {"sel(linux)": "{{ cdt('libselinux') }}"}, - {"sel(linux)": "{{ cdt('libxdamage') }}"}, - {"sel(linux)": "{{ cdt('libxxf86vm') }}"}, - {"sel(linux)": "{{ cdt('libxfixes') }}"}, - {"sel(linux)": "{{ cdt('libxext') }}"}, - {"sel(linux)": "{{ cdt('libxau') }}"}, - {"sel(linux)": "{{ cdt('libxcb') }}"}, + {"if": "linux", "then": ["${{ cdt('mesa-libgl-devel') }}", "${{ cdt('mesa-dri-drivers') }}", "${{ cdt('libselinux') }}", "${{ cdt('libxdamage') }}", "${{ cdt('libxxf86vm') }}", "${{ cdt('libxfixes') }}", "${{ cdt('libxext') }}", "${{ cdt('libxau') }}", "${{ cdt('libxcb') }}"]} ] + output["requirements"]["host"] += [ - {"sel(unix)": "xorg-libx11"}, - {"sel(unix)": "xorg-libxext"}, - # 'xorg-libxfixes [unix]', + {"if": "unix", "then": ["xorg-libx11", "xorg-libxext"]}, ] output["requirements"]["run"] += [ - {"sel(unix)": "xorg-libx11"}, - {"sel(unix)": "xorg-libxext"}, - # 'xorg-libxfixes [unix]', + {"if": "unix", "then": ["xorg-libx11", "xorg-libxext"]}, ] # fix up GL support for Unix @@ -476,10 +444,7 @@ def sortkey(k): output["requirements"]["host"].remove("REQUIRE_GL") output["requirements"]["build"] += [ - {"sel(linux)": "{{ cdt('mesa-libgl-devel') }}"}, - {"sel(linux)": "{{ cdt('mesa-dri-drivers') }}"}, - {"sel(linux)": "{{ cdt('libselinux') }}"}, - {"sel(linux)": "{{ cdt('libxxf86vm') }}"}, + {"if": "linux", "then": ["${{ cdt('mesa-libgl-devel') }}", "${{ cdt('mesa-dri-drivers') }}", "${{ cdt('libselinux') }}", "${{ cdt('libxxf86vm') }}"]} ] # remove duplicates @@ -558,8 +523,8 @@ def generate_source(distro, vinca_conf): url, version = distro.get_released_repo(pkg_shortname) entry = {} - entry["git_url"] = url - entry["git_rev"] = version + entry["git"] = url + entry["rev"] = version pkg_names = resolve_pkgname(pkg_shortname, vinca_conf, distro) pkg_version = get_version(distro, vinca_conf, pkg_shortname) print("Checking ", pkg_shortname, pkg_version) @@ -572,7 +537,7 @@ def generate_source(distro, vinca_conf): if pkg_names[0] in vinca_conf["skip_built_packages"]: continue pkg_name = pkg_names[0] - entry["folder"] = "%s/src/work" % pkg_name + entry["target_directory"] = "%s/src/work" % pkg_name patches = [] pd = vinca_conf["_patches"].get(pkg_name) @@ -609,8 +574,8 @@ def generate_source_version(distro, vinca_conf): version = vinca_conf["package_version"][pkg_shortname]["version"] entry = {} - entry["git_url"] = url - entry["git_rev"] = version + entry["git"] = url + entry["rev"] = version pkg_names = resolve_pkgname(pkg_shortname, vinca_conf, distro) if vinca_conf.get("trigger_new_versions"): if ( @@ -622,7 +587,7 @@ def generate_source_version(distro, vinca_conf): if not pkg_names or pkg_names[0] in vinca_conf["skip_built_packages"]: continue pkg_name = pkg_names[0] - entry["folder"] = "%s/src/work" % pkg_name + entry["target_directory"] = "%s/src/work" % pkg_name patches = [] pd = vinca_conf["_patches"].get(pkg_name) @@ -649,13 +614,13 @@ def generate_fat_source(distro, vinca_conf): url, version = distro.get_released_repo(pkg_shortname) entry = {} - entry["git_url"] = url - entry["git_rev"] = version + entry["git"] = url + entry["rev"] = version pkg_names = resolve_pkgname(pkg_shortname, vinca_conf, distro) if not pkg_names: continue pkg_name = pkg_names[0] - entry["folder"] = "src/%s" % pkg_name + entry["target_directory"] = "src/%s" % pkg_name patch_path = os.path.join(vinca_conf["_patch_dir"], "%s.patch" % pkg_name) if os.path.exists(patch_path): entry["patches"] = [ @@ -706,7 +671,7 @@ def parse_package(pkg, distro, vinca_conf, path): recipe = { "package": {"name": final_name, "version": pkg["version"]}, "about": { - "home": "https://www.ros.org/", + "homepage": "https://www.ros.org/", "license": [str(lic) for lic in pkg["licenses"]], "summary": pkg["description"], "maintainers": [], @@ -714,25 +679,28 @@ def parse_package(pkg, distro, vinca_conf, path): "extra": {"recipe-maintainers": ["robostack"]}, "build": { "number": 0, - "script": {"sel(unix)": "build_catkin.sh", "sel(win)": "build_catkin.bat"}, + "script": [ + {"if": "unix", "then": ["build_catkin.sh"]}, + {"if": "win", "then": ["build_catkin.bat"]} + ], }, "source": {}, "requirements": { "build": [ - "{{ compiler('cxx') }}", - "{{ compiler('c') }}", - {"sel(linux64)": "sysroot_linux-64 2.17"}, + "${{ compiler('cxx') }}", + "${{ compiler('c') }}", + {"if": "linux64", "then": ["sysroot_linux-64 2.17"]}, "ninja", - {"sel(unix)": "make"}, - {"sel(unix)": "coreutils"}, + {"if": "unix", "then": ["make", "coreutils"]}, "cmake", - {"sel(build_platform != target_platform)": "python"}, + {"if": "build_platform != target_platform", "then": ["python"]}, { - "sel(build_platform != target_platform)": "cross-python_{{ target_platform }}" + "if": "build_platform != target_platform", + "then": ["cross-python_{{ target_platform }}"] }, - {"sel(build_platform != target_platform)": "cython"}, - {"sel(build_platform != target_platform)": "numpy"}, - {"sel(build_platform != target_platform)": "pybind11"}, + {"if": "build_platform != target_platform", "then": ["cython"]}, + {"if": "build_platform != target_platform", "then": ["numpy"]}, + {"if": "build_platform != target_platform", "then": ["pybind11"]}, ], "host": [], "run": [], @@ -743,25 +711,21 @@ def parse_package(pkg, distro, vinca_conf, path): name = p.name + " (" + p.email + ")" if p.email else p.name recipe["about"]["maintainers"].append(name) - # for p in pkg['maintainers'] : - # name = p.name + " (" + p.email + ")" if p.email else p.name - # recipe['about']['maintainers'].append(name) - for u in pkg["urls"]: # if u.type == 'repository' : - # recipe['source']['git_url'] = u.url - # recipe['source']['git_rev'] = recipe['package']['version'] + # recipe['source']['git'] = u.url + # recipe['source']['rev'] = recipe['package']['version'] if u.type == "website": - recipe["about"]["home"] = u.url + recipe["about"]["homepage"] = u.url # if u.type == 'bugtracker' : # recipe['about']['url_issues'] = u.url - if not recipe["source"].get("git_url", None): + if not recipe["source"].get("git", None): aux = path.split("/") print(aux[: len(aux) - 1]) recipe["source"]["path"] = "/".join(aux[: len(aux) - 1]) - recipe["source"]["folder"] = f"{final_name}/src/work" + recipe["source"]["target_directory"] = f"{final_name}/src/work" for d in pkg["buildtool_depends"]: recipe["requirements"]["host"].extend( @@ -800,10 +764,10 @@ def parse_package(pkg, distro, vinca_conf, path): ) if pkg.get_build_type() in ["cmake", "catkin"]: - recipe["build"]["script"] = { - "sel(win)": "bld_catkin.bat", - "sel(unix)": "build_catkin.sh", - } + recipe["build"]["script"] = [ + {"if": "win", "then": ["bld_catkin.bat"]}, + {"if": "unix", "then": ["build_catkin.sh"]}, + ] # fix up OPENGL support for Unix if ( @@ -817,22 +781,13 @@ def parse_package(pkg, distro, vinca_conf, path): recipe["requirements"]["host"].remove("REQUIRE_OPENGL") recipe["requirements"]["build"] += [ - {"sel(linux)": "{{ cdt('mesa-libgl-devel') }}"}, - {"sel(linux)": "{{ cdt('mesa-dri-drivers') }}"}, - {"sel(linux)": "{{ cdt('libselinux') }}"}, - {"sel(linux)": "{{ cdt('libxdamage') }}"}, - {"sel(linux)": "{{ cdt('libxxf86vm') }}"}, - {"sel(linux)": "{{ cdt('libxfixes') }}"}, - {"sel(linux)": "{{ cdt('libxext') }}"}, - {"sel(linux)": "{{ cdt('libxau') }}"}, + {"if": "linux", "then": ["${{ cdt('mesa-libgl-devel') }}", "${{ cdt('mesa-dri-drivers') }}", "${{ cdt('libselinux') }}", "${{ cdt('libxdamage') }}", "${{ cdt('libxxf86vm') }}", "${{ cdt('libxfixes') }}", "${{ cdt('libxext') }}", "${{ cdt('libxau') }}", "${{ cdt('libxcb') }}"]} ] recipe["requirements"]["host"] += [ - {"sel(unix)": "xorg-libx11"}, - {"sel(unix)": "xorg-libxext"}, + {"if": "unix", "then": ["xorg-libx11", "xorg-libxext"]}, ] recipe["requirements"]["run"] += [ - {"sel(unix)": "xorg-libx11"}, - {"sel(unix)": "xorg-libxext"}, + {"if": "unix", "then": ["xorg-libx11", "xorg-libxext"]}, ] # fix up GL support for Unix @@ -847,10 +802,7 @@ def parse_package(pkg, distro, vinca_conf, path): recipe["requirements"]["host"].remove("REQUIRE_GL") recipe["requirements"]["build"] += [ - {"sel(linux)": "{{ cdt('mesa-libgl-devel') }}"}, - {"sel(linux)": "{{ cdt('mesa-dri-drivers') }}"}, - {"sel(linux)": "{{ cdt('libselinux') }}"}, - {"sel(linux)": "{{ cdt('libxxf86vm') }}"}, + {"if": "linux", "then": ["${{ cdt('mesa-libgl-devel') }}", "${{ cdt('mesa-dri-drivers') }}", "${{ cdt('libselinux') }}", "${{ cdt('libxxf86vm') }}"]} ] return recipe diff --git a/vinca/template.py b/vinca/template.py index 574ac2e..02dc9f9 100644 --- a/vinca/template.py +++ b/vinca/template.py @@ -6,6 +6,8 @@ from pathlib import Path TEMPLATE = """\ +# yaml-language-server: $schema=https://raw.githubusercontent.com/prefix-dev/recipe-format/main/schema.json + package: name: ros version: 0.0.1 @@ -16,7 +18,7 @@ number: 0 about: - home: https://www.ros.org/ + homepage: https://www.ros.org/ license: BSD-3-Clause summary: | Robot Operating System @@ -80,7 +82,12 @@ def write_recipe(source, outputs, build_number=0, single_file=True): os.makedirs(recipe_dir / patch_dir, exist_ok=True) shutil.copyfile(p, recipe_dir / p) - for _, script in meta["build"]["script"].items(): + build_scripts = [] + for item in meta["build"]["script"]: + for filename in item['then']: + build_scripts.append(filename) + + for script in build_scripts: shutil.copyfile(script, recipe_dir / script) if "catkin" in o["package"]["name"] or "workspace" in o["package"]["name"]: shutil.copyfile("activate.sh", recipe_dir / "activate.sh") From 56152f8d4a035f3f721deabc2c546abad271fed6 Mon Sep 17 00:00:00 2001 From: Wolf Vollprecht Date: Fri, 19 Jan 2024 16:19:59 +0100 Subject: [PATCH 02/57] get rattler-build to run --- vinca/main.py | 26 +++++++------------------- vinca/template.py | 2 ++ 2 files changed, 9 insertions(+), 19 deletions(-) diff --git a/vinca/main.py b/vinca/main.py index ed6cbf8..c8cb934 100644 --- a/vinca/main.py +++ b/vinca/main.py @@ -243,7 +243,7 @@ def generate_output(pkg_shortname, vinca_conf, distro, version, all_pkgs=None): {"if": "build_platform != target_platform", "then": ["python"]}, { "if": "build_platform != target_platform", - "then": ["cross-python_{{ target_platform }}"] + "then": ["cross-python_${{ target_platform }}"] }, {"if": "build_platform != target_platform", "then": ["numpy"]}, ], @@ -267,20 +267,11 @@ def generate_output(pkg_shortname, vinca_conf, distro, version, all_pkgs=None): output["requirements"]["run"].extend(resolved_python) output["requirements"]["host"].extend(resolved_python) if pkg.get_build_type() in ["cmake", "catkin"]: - output["build"]["script"] = [ - {"if": "win", "then": ["bld_catkin.bat"]}, - {"if": "unix", "then": ["build_catkin.sh"]}, - ] + output["build"]["script"] = "${{ 'build_catkin.sh' if unix else 'build_catkin.bat' }}" elif pkg.get_build_type() in ["ament_cmake"]: - output["build"]["script"] = [ - {"if": "win", "then": ["bld_ament_cmake.bat"]}, - {"if": "unix", "then": ["build_ament_cmake.sh"]}, - ] + output["build"]["script"] = "${{ 'build_ament_cmake.sh' if unix else 'bld_ament_cmake.bat' }}" elif pkg.get_build_type() in ["ament_python"]: - output["build"]["script"] = [ - {"if": "win", "then": ["bld_ament_python.bat"]}, - {"if": "unix", "then": ["build_ament_python.sh"]}, - ] + output["build"]["script"] = "${{ 'build_ament_python.sh' if unix else 'bld_ament_python.bat' }}" resolved_setuptools = resolve_pkgname("python-setuptools", vinca_conf, distro) output["requirements"]["host"].extend(resolved_setuptools) else: @@ -388,7 +379,7 @@ def sortkey(k): output["requirements"]["host"] = sorted(output["requirements"]["host"], key=sortkey) output["requirements"]["run"] += [ - {"if": "osx and x86_64", "then": ["__osx >={{ MACOSX_DEPLOYMENT_TARGET|default('10.14') }}"]} + {"if": "osx and x86_64", "then": ["__osx >=${{ MACOSX_DEPLOYMENT_TARGET|default('10.14') }}"]} ] if f"ros-{config.ros_distro}-pybind11-vendor" in output["requirements"]["host"]: @@ -696,7 +687,7 @@ def parse_package(pkg, distro, vinca_conf, path): {"if": "build_platform != target_platform", "then": ["python"]}, { "if": "build_platform != target_platform", - "then": ["cross-python_{{ target_platform }}"] + "then": ["cross-python_${{ target_platform }}"] }, {"if": "build_platform != target_platform", "then": ["cython"]}, {"if": "build_platform != target_platform", "then": ["numpy"]}, @@ -764,10 +755,7 @@ def parse_package(pkg, distro, vinca_conf, path): ) if pkg.get_build_type() in ["cmake", "catkin"]: - recipe["build"]["script"] = [ - {"if": "win", "then": ["bld_catkin.bat"]}, - {"if": "unix", "then": ["build_catkin.sh"]}, - ] + recipe["build"]["script"] = "${{ 'build_catkin.sh' if unix else 'build_catkin.bat' }}" # fix up OPENGL support for Unix if ( diff --git a/vinca/template.py b/vinca/template.py index 02dc9f9..3962942 100644 --- a/vinca/template.py +++ b/vinca/template.py @@ -52,6 +52,8 @@ def write_recipe(source, outputs, build_number=0, single_file=True): meta["source"] = [source[k] for k in source] meta["outputs"] = outputs meta["package"]["version"] = f"{datetime.datetime.now():%Y.%m.%d}" + meta["recipe"] = meta["package"] + del meta["package"] meta["build"]["number"] = build_number with open("recipe.yaml", "w") as stream: file.dump(meta, stream) From 392c60ae3f9bbfc45e1043aab94cc833fc3d74df Mon Sep 17 00:00:00 2001 From: Tobias Fischer Date: Sat, 20 Jan 2024 12:47:52 +1000 Subject: [PATCH 03/57] Further fixes for actually working rattler-build --- vinca/main.py | 9 +++------ vinca/template.py | 7 ++----- 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/vinca/main.py b/vinca/main.py index c8cb934..093aa57 100644 --- a/vinca/main.py +++ b/vinca/main.py @@ -267,7 +267,7 @@ def generate_output(pkg_shortname, vinca_conf, distro, version, all_pkgs=None): output["requirements"]["run"].extend(resolved_python) output["requirements"]["host"].extend(resolved_python) if pkg.get_build_type() in ["cmake", "catkin"]: - output["build"]["script"] = "${{ 'build_catkin.sh' if unix else 'build_catkin.bat' }}" + output["build"]["script"] = "${{ 'build_catkin.sh' if unix else 'bld_catkin.bat' }}" elif pkg.get_build_type() in ["ament_cmake"]: output["build"]["script"] = "${{ 'build_ament_cmake.sh' if unix else 'bld_ament_cmake.bat' }}" elif pkg.get_build_type() in ["ament_python"]: @@ -670,10 +670,7 @@ def parse_package(pkg, distro, vinca_conf, path): "extra": {"recipe-maintainers": ["robostack"]}, "build": { "number": 0, - "script": [ - {"if": "unix", "then": ["build_catkin.sh"]}, - {"if": "win", "then": ["build_catkin.bat"]} - ], + "script": "${{ 'build_catkin.sh' if unix else 'bld_catkin.bat' }}", }, "source": {}, "requirements": { @@ -755,7 +752,7 @@ def parse_package(pkg, distro, vinca_conf, path): ) if pkg.get_build_type() in ["cmake", "catkin"]: - recipe["build"]["script"] = "${{ 'build_catkin.sh' if unix else 'build_catkin.bat' }}" + recipe["build"]["script"] = "${{ 'build_catkin.sh' if unix else 'bld_catkin.bat' }}" # fix up OPENGL support for Unix if ( diff --git a/vinca/template.py b/vinca/template.py index 3962942..04b4d4c 100644 --- a/vinca/template.py +++ b/vinca/template.py @@ -1,6 +1,7 @@ import datetime import shutil import os +import re from ruamel import yaml from pathlib import Path @@ -84,11 +85,7 @@ def write_recipe(source, outputs, build_number=0, single_file=True): os.makedirs(recipe_dir / patch_dir, exist_ok=True) shutil.copyfile(p, recipe_dir / p) - build_scripts = [] - for item in meta["build"]["script"]: - for filename in item['then']: - build_scripts.append(filename) - + build_scripts = re.findall(r"'(.*?)'", meta["build"]["script"]) for script in build_scripts: shutil.copyfile(script, recipe_dir / script) if "catkin" in o["package"]["name"] or "workspace" in o["package"]["name"]: From ecd18d5ce444e598677f27f6b0918229fc40979d Mon Sep 17 00:00:00 2001 From: Wolf Vollprecht Date: Fri, 12 Apr 2024 14:08:36 +0200 Subject: [PATCH 04/57] add post-process step to recipes --- .pre-commit-config.yaml | 30 +++++++-------- vinca/generate_azure.py | 4 +- vinca/generate_gha.py | 6 ++- vinca/main.py | 85 +++++++++++++++++++++++++++++++++-------- vinca/template.py | 33 +++++++++++++--- 5 files changed, 120 insertions(+), 38 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5b67027..a083285 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,16 +1,16 @@ repos: - repo: https://github.com/psf/black - rev: 19.10b0 + rev: 24.3.0 hooks: - id: black args: [--safe, --quiet] - repo: https://github.com/asottile/blacken-docs - rev: v1.7.0 + rev: 1.16.0 hooks: - id: blacken-docs - additional_dependencies: [black==19.10b0] + additional_dependencies: [black] - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v3.2.0 + rev: v4.6.0 hooks: - id: trailing-whitespace exclude: ^examples/ @@ -21,14 +21,14 @@ repos: # hooks: # - id: isort # exclude: tests/data -- repo: https://gitlab.com/pycqa/flake8 - rev: 3.8.3 - hooks: - - id: flake8 - exclude: tests/data - language_version: python3 - additional_dependencies: - - flake8-typing-imports==1.9.0 - - flake8-builtins==1.5.3 - - flake8-bugbear==20.1.4 - # - flake8-isort==3.0.1 +# - repo: https://github.com/PyCQA/flake8 +# rev: 7.0.0 +# hooks: +# - id: flake8 +# exclude: tests/data +# language_version: python3 +# additional_dependencies: +# - flake8-typing-imports==1.9.0 +# - flake8-builtins==1.5.3 +# - flake8-bugbear==20.1.4 +# # - flake8-isort==3.0.1 diff --git a/vinca/generate_azure.py b/vinca/generate_azure.py index b27c72e..c22bec3 100644 --- a/vinca/generate_azure.py +++ b/vinca/generate_azure.py @@ -526,7 +526,9 @@ def main(): if args.platform == "osx-64": build_osx_pipeline( - stages, args.trigger_branch, script=azure_osx_script, + stages, + args.trigger_branch, + script=azure_osx_script, ) if args.platform == "osx-arm64": diff --git a/vinca/generate_gha.py b/vinca/generate_gha.py index 0a51841..6c5ef0f 100644 --- a/vinca/generate_gha.py +++ b/vinca/generate_gha.py @@ -67,7 +67,7 @@ def parse_command_line(argv): action="store_true", help="search for additional_recipes folder?", ) - + parser.add_argument( "-b", "--batch_size", @@ -571,7 +571,9 @@ def main(): if args.platform == "osx-64": build_osx_pipeline( - stages, args.trigger_branch, script=azure_osx_script, + stages, + args.trigger_branch, + script=azure_osx_script, ) if args.platform == "osx-arm64": diff --git a/vinca/main.py b/vinca/main.py index 093aa57..a8c1b4d 100644 --- a/vinca/main.py +++ b/vinca/main.py @@ -243,7 +243,7 @@ def generate_output(pkg_shortname, vinca_conf, distro, version, all_pkgs=None): {"if": "build_platform != target_platform", "then": ["python"]}, { "if": "build_platform != target_platform", - "then": ["cross-python_${{ target_platform }}"] + "then": ["cross-python_${{ target_platform }}"], }, {"if": "build_platform != target_platform", "then": ["numpy"]}, ], @@ -267,11 +267,17 @@ def generate_output(pkg_shortname, vinca_conf, distro, version, all_pkgs=None): output["requirements"]["run"].extend(resolved_python) output["requirements"]["host"].extend(resolved_python) if pkg.get_build_type() in ["cmake", "catkin"]: - output["build"]["script"] = "${{ 'build_catkin.sh' if unix else 'bld_catkin.bat' }}" + output["build"][ + "script" + ] = "${{ 'build_catkin.sh' if unix else 'bld_catkin.bat' }}" elif pkg.get_build_type() in ["ament_cmake"]: - output["build"]["script"] = "${{ 'build_ament_cmake.sh' if unix else 'bld_ament_cmake.bat' }}" + output["build"][ + "script" + ] = "${{ 'build_ament_cmake.sh' if unix else 'bld_ament_cmake.bat' }}" elif pkg.get_build_type() in ["ament_python"]: - output["build"]["script"] = "${{ 'build_ament_python.sh' if unix else 'bld_ament_python.bat' }}" + output["build"][ + "script" + ] = "${{ 'build_ament_python.sh' if unix else 'bld_ament_python.bat' }}" resolved_setuptools = resolve_pkgname("python-setuptools", vinca_conf, distro) output["requirements"]["host"].extend(resolved_setuptools) else: @@ -326,11 +332,13 @@ def generate_output(pkg_shortname, vinca_conf, distro, version, all_pkgs=None): build_deps.append(dep) # Hack to add cyclonedds into build for cross compilation - if pkg_shortname == "cyclonedds" or "cyclonedds" in (build_deps + build_tool_deps): + if pkg_shortname == "cyclonedds" or "cyclonedds" in ( + build_deps + build_tool_deps + ): output["requirements"]["build"].append( { "if": "build_platform != target_platform", - "then": [f"ros-{config.ros_distro}-cyclonedds"] + "then": [f"ros-{config.ros_distro}-cyclonedds"], } ) @@ -379,7 +387,10 @@ def sortkey(k): output["requirements"]["host"] = sorted(output["requirements"]["host"], key=sortkey) output["requirements"]["run"] += [ - {"if": "osx and x86_64", "then": ["__osx >=${{ MACOSX_DEPLOYMENT_TARGET|default('10.14') }}"]} + { + "if": "osx and x86_64", + "then": ["__osx >=${{ MACOSX_DEPLOYMENT_TARGET|default('10.14') }}"], + } ] if f"ros-{config.ros_distro}-pybind11-vendor" in output["requirements"]["host"]: @@ -413,7 +424,20 @@ def sortkey(k): output["requirements"]["host"].remove("REQUIRE_OPENGL") output["requirements"]["build"] += [ - {"if": "linux", "then": ["${{ cdt('mesa-libgl-devel') }}", "${{ cdt('mesa-dri-drivers') }}", "${{ cdt('libselinux') }}", "${{ cdt('libxdamage') }}", "${{ cdt('libxxf86vm') }}", "${{ cdt('libxfixes') }}", "${{ cdt('libxext') }}", "${{ cdt('libxau') }}", "${{ cdt('libxcb') }}"]} + { + "if": "linux", + "then": [ + "${{ cdt('mesa-libgl-devel') }}", + "${{ cdt('mesa-dri-drivers') }}", + "${{ cdt('libselinux') }}", + "${{ cdt('libxdamage') }}", + "${{ cdt('libxxf86vm') }}", + "${{ cdt('libxfixes') }}", + "${{ cdt('libxext') }}", + "${{ cdt('libxau') }}", + "${{ cdt('libxcb') }}", + ], + } ] output["requirements"]["host"] += [ @@ -435,7 +459,15 @@ def sortkey(k): output["requirements"]["host"].remove("REQUIRE_GL") output["requirements"]["build"] += [ - {"if": "linux", "then": ["${{ cdt('mesa-libgl-devel') }}", "${{ cdt('mesa-dri-drivers') }}", "${{ cdt('libselinux') }}", "${{ cdt('libxxf86vm') }}"]} + { + "if": "linux", + "then": [ + "${{ cdt('mesa-libgl-devel') }}", + "${{ cdt('mesa-dri-drivers') }}", + "${{ cdt('libselinux') }}", + "${{ cdt('libxxf86vm') }}", + ], + } ] # remove duplicates @@ -576,7 +608,7 @@ def generate_source_version(distro, vinca_conf): continue else: if not pkg_names or pkg_names[0] in vinca_conf["skip_built_packages"]: - continue + continue pkg_name = pkg_names[0] entry["target_directory"] = "%s/src/work" % pkg_name @@ -684,7 +716,7 @@ def parse_package(pkg, distro, vinca_conf, path): {"if": "build_platform != target_platform", "then": ["python"]}, { "if": "build_platform != target_platform", - "then": ["cross-python_${{ target_platform }}"] + "then": ["cross-python_${{ target_platform }}"], }, {"if": "build_platform != target_platform", "then": ["cython"]}, {"if": "build_platform != target_platform", "then": ["numpy"]}, @@ -752,7 +784,9 @@ def parse_package(pkg, distro, vinca_conf, path): ) if pkg.get_build_type() in ["cmake", "catkin"]: - recipe["build"]["script"] = "${{ 'build_catkin.sh' if unix else 'bld_catkin.bat' }}" + recipe["build"][ + "script" + ] = "${{ 'build_catkin.sh' if unix else 'bld_catkin.bat' }}" # fix up OPENGL support for Unix if ( @@ -766,7 +800,20 @@ def parse_package(pkg, distro, vinca_conf, path): recipe["requirements"]["host"].remove("REQUIRE_OPENGL") recipe["requirements"]["build"] += [ - {"if": "linux", "then": ["${{ cdt('mesa-libgl-devel') }}", "${{ cdt('mesa-dri-drivers') }}", "${{ cdt('libselinux') }}", "${{ cdt('libxdamage') }}", "${{ cdt('libxxf86vm') }}", "${{ cdt('libxfixes') }}", "${{ cdt('libxext') }}", "${{ cdt('libxau') }}", "${{ cdt('libxcb') }}"]} + { + "if": "linux", + "then": [ + "${{ cdt('mesa-libgl-devel') }}", + "${{ cdt('mesa-dri-drivers') }}", + "${{ cdt('libselinux') }}", + "${{ cdt('libxdamage') }}", + "${{ cdt('libxxf86vm') }}", + "${{ cdt('libxfixes') }}", + "${{ cdt('libxext') }}", + "${{ cdt('libxau') }}", + "${{ cdt('libxcb') }}", + ], + } ] recipe["requirements"]["host"] += [ {"if": "unix", "then": ["xorg-libx11", "xorg-libxext"]}, @@ -787,7 +834,15 @@ def parse_package(pkg, distro, vinca_conf, path): recipe["requirements"]["host"].remove("REQUIRE_GL") recipe["requirements"]["build"] += [ - {"if": "linux", "then": ["${{ cdt('mesa-libgl-devel') }}", "${{ cdt('mesa-dri-drivers') }}", "${{ cdt('libselinux') }}", "${{ cdt('libxxf86vm') }}"]} + { + "if": "linux", + "then": [ + "${{ cdt('mesa-libgl-devel') }}", + "${{ cdt('mesa-dri-drivers') }}", + "${{ cdt('libselinux') }}", + "${{ cdt('libxxf86vm') }}", + ], + } ] return recipe @@ -815,7 +870,7 @@ def main(): generate_bld_colcon_merge() generate_bld_catkin_merge() generate_activate_hook() - + if arguments.trigger_new_versions: vinca_conf["trigger_new_versions"] = True else: diff --git a/vinca/template.py b/vinca/template.py index 04b4d4c..c2843f9 100644 --- a/vinca/template.py +++ b/vinca/template.py @@ -30,6 +30,24 @@ """ +post_process_items = [ + { + "files": ["*.pc"], + "regex": '(?:-L|-I)?"?([^;\\s]+/sysroot/)', + "replacement": "$(CONDA_BUILD_SYSROOT_S)", + }, + { + "files": ["*.cmake"], + "regex": '([^;\\s"]+/sysroot)', + "replacement": "$ENV{CONDA_BUILD_SYSROOT}", + }, + { + "files": ["*.cmake"], + "regex": '([^;\\s"]+/MacOSX\\d*\\.?\\d*\\.sdk)', + "replacement": "$ENV{CONDA_BUILD_SYSROOT}", + }, +] + def write_recipe_package(recipe): file = yaml.YAML() @@ -56,6 +74,7 @@ def write_recipe(source, outputs, build_number=0, single_file=True): meta["recipe"] = meta["package"] del meta["package"] meta["build"]["number"] = build_number + meta["build"]["post_process"] = post_process_items with open("recipe.yaml", "w") as stream: file.dump(meta, stream) else: @@ -73,7 +92,7 @@ def write_recipe(source, outputs, build_number=0, single_file=True): meta["package"]["version"] = o["package"]["version"] meta["build"]["number"] = build_number - + meta["build"]["post_process"] = post_process_items recipe_dir = (Path("recipes") / o["package"]["name"]).absolute() os.makedirs(recipe_dir, exist_ok=True) with open(recipe_dir / "recipe.yaml", "w") as stream: @@ -172,14 +191,18 @@ def generate_activate_hook(): template_in = pkg_resources.resource_filename("vinca", "templates/activate.bat.in") generate_template(template_in, open("activate.bat", "w")) - template_in = pkg_resources.resource_filename("vinca", "templates/deactivate.bat.in") + template_in = pkg_resources.resource_filename( + "vinca", "templates/deactivate.bat.in" + ) generate_template(template_in, open("deactivate.bat", "w")) - + template_in = pkg_resources.resource_filename("vinca", "templates/activate.ps1.in") generate_template(template_in, open("activate.ps1", "w")) - template_in = pkg_resources.resource_filename("vinca", "templates/deactivate.ps1.in") + template_in = pkg_resources.resource_filename( + "vinca", "templates/deactivate.ps1.in" + ) generate_template(template_in, open("deactivate.ps1", "w")) - + template_in = pkg_resources.resource_filename("vinca", "templates/activate.sh.in") generate_template(template_in, open("activate.sh", "w")) template_in = pkg_resources.resource_filename("vinca", "templates/deactivate.sh.in") From 55cda6efce971011d8ec0152aec2d4fe818ad85e Mon Sep 17 00:00:00 2001 From: Silvio Traversaro Date: Sat, 10 Feb 2024 18:40:02 +0100 Subject: [PATCH 05/57] Pass a relative path to PYTHON_INSTALL_DIR in ROS 2 --- vinca/templates/bld_ament_cmake.bat.in | 14 ++++++++++++-- vinca/templates/bld_colcon_merge.bat.in | 13 +++++++++++-- vinca/templates/build_ament_cmake.sh.in | 12 ++++++++---- 3 files changed, 31 insertions(+), 8 deletions(-) diff --git a/vinca/templates/bld_ament_cmake.bat.in b/vinca/templates/bld_ament_cmake.bat.in index 1ebd33a..34cd234 100644 --- a/vinca/templates/bld_ament_cmake.bat.in +++ b/vinca/templates/bld_ament_cmake.bat.in @@ -16,7 +16,17 @@ pushd build :: try to fix long paths issues by using default generator set "CMAKE_GENERATOR=Visual Studio %VS_MAJOR% %VS_YEAR%" -set "SP_DIR_FORWARDSLASHES=%SP_DIR:\=/%" + +:: PYTHON_INSTALL_DIR should be a relative path, see +:: https://github.com/ament/ament_cmake/blob/2.3.2/ament_cmake_python/README.md +:: So we compute the relative path of %SP_DIR% w.r.t. to LIBRARY_PREFIX, +:: but it is not trivial to do this in Command Prompt scripting, so let's do it via +:: python + +:: This line is scary, but it basically assigns the output of the command inside (` and `) +:: to the variable specified after DO SET +:: The equivalent in bash is PYTHON_INSTALL_DIR=`python -c ...` +FOR /F "tokens=* USEBACKQ" %%i IN (`python -c "import os;print(os.path.relpath(os.environ['SP_DIR'],os.environ['LIBRARY_PREFIX']))"`) DO SET PYTHON_INSTALL_DIR=%%i cmake ^ -G "%CMAKE_GENERATOR%" ^ @@ -30,8 +40,8 @@ cmake ^ -DBUILD_SHARED_LIBS=ON ^ -DBUILD_TESTING=OFF ^ -DCMAKE_OBJECT_PATH_MAX=255 ^ - -DPYTHON_INSTALL_DIR=%SP_DIR_FORWARDSLASHES% ^ --compile-no-warning-as-error ^ + -DPYTHON_INSTALL_DIR=%PYTHON_INSTALL_DIR% ^ %SRC_DIR%\%PKG_NAME%\src\work if errorlevel 1 exit 1 diff --git a/vinca/templates/bld_colcon_merge.bat.in b/vinca/templates/bld_colcon_merge.bat.in index c5189ae..a003a8e 100644 --- a/vinca/templates/bld_colcon_merge.bat.in +++ b/vinca/templates/bld_colcon_merge.bat.in @@ -6,7 +6,16 @@ setlocal set CC=cl.exe set CXX=cl.exe -set "SP_DIR_FORWARDSLASHES=%SP_DIR:\=/%" +:: PYTHON_INSTALL_DIR should be a relative path, see +:: https://github.com/ament/ament_cmake/blob/2.3.2/ament_cmake_python/README.md +:: So we compute the relative path of %SP_DIR% w.r.t. to LIBRARY_PREFIX, +:: but it is not trivial to do this in Command Prompt scripting, so let's do it via +:: python + +:: This line is scary, but it basically assigns the output of the command inside (` and `) +:: to the variable specified after DO SET +:: The equivalent in bash is PYTHON_INSTALL_DIR=`python -c ...` +FOR /F "tokens=* USEBACKQ" %%i IN (`python -c "import os;print(os.path.relpath(os.environ['SP_DIR'],os.environ['LIBRARY_PREFIX']))"`) DO SET PYTHON_INSTALL_DIR=%%i colcon build ^ --event-handlers console_cohesion+ ^ @@ -17,7 +26,7 @@ colcon build ^ -G Ninja ^ -DCMAKE_BUILD_TYPE=Release ^ -DBUILD_TESTING=OFF ^ - -DPYTHON_INSTALL_DIR=%SP_DIR_FORWARDSLASHES% ^ + -DPYTHON_INSTALL_DIR=%PYTHON_INSTALL_DIR% ^ -DPYTHON_EXECUTABLE=%PYTHON% if errorlevel 1 exit 1 diff --git a/vinca/templates/build_ament_cmake.sh.in b/vinca/templates/build_ament_cmake.sh.in index 52baa99..4bbafe9 100644 --- a/vinca/templates/build_ament_cmake.sh.in +++ b/vinca/templates/build_ament_cmake.sh.in @@ -23,9 +23,6 @@ echo "USING PKG_CONFIG_EXECUTABLE=${PKG_CONFIG_EXECUTABLE}" export ROS_PYTHON_VERSION=`$PYTHON_EXECUTABLE -c "import sys; print('%i.%i' % (sys.version_info[0:2]))"` echo "Using Python ${ROS_PYTHON_VERSION}" -# Fix up SP_DIR which for some reason might contain a path to a wrong Python version -FIXED_SP_DIR=$(echo $SP_DIR | sed -E "s/python[0-9]+\.[0-9]+/python$ROS_PYTHON_VERSION/") -echo "Using site-package dir ${FIXED_SP_DIR}" # see https://github.com/conda-forge/cross-python-feedstock/issues/24 if [[ "$CONDA_BUILD_CROSS_COMPILATION" == "1" ]]; then @@ -46,6 +43,13 @@ if [[ $target_platform =~ linux.* ]]; then ln -s $GXX ${BUILD_PREFIX}/bin/g++ fi; +# PYTHON_INSTALL_DIR should be a relative path, see +# https://github.com/ament/ament_cmake/blob/2.3.2/ament_cmake_python/README.md +# So we compute the relative path of $SP_DIR w.r.t. to $PREFIX, +# but it is not trivial to do this in bash scripting, so let's do it via python +export PYTHON_INSTALL_DIR=`python -c "import os;print(os.path.relpath(os.environ['SP_DIR'],os.environ['PREFIX']))"` +echo "Using site_packages: $PYTHON_INSTALL_DIR" + cmake \ -G "Ninja" \ -DCMAKE_INSTALL_PREFIX=$PREFIX \ @@ -58,7 +62,7 @@ cmake \ -DPython3_EXECUTABLE=$PYTHON_EXECUTABLE \ -DPython3_FIND_STRATEGY=LOCATION \ -DPKG_CONFIG_EXECUTABLE=$PKG_CONFIG_EXECUTABLE \ - -DPYTHON_INSTALL_DIR=$FIXED_SP_DIR \ + -DPYTHON_INSTALL_DIR=$PYTHON_INSTALL_DIR \ -DSETUPTOOLS_DEB_LAYOUT=OFF \ -DCATKIN_SKIP_TESTING=$SKIP_TESTING \ -DCMAKE_INSTALL_SYSTEM_RUNTIME_LIBS_SKIP=True \ From 4414b0ec607a92b4f95b85affc0d3352232ff9ca Mon Sep 17 00:00:00 2001 From: Wolf Vollprecht Date: Thu, 20 Jun 2024 11:33:11 +0200 Subject: [PATCH 06/57] fix run dependencies with jinja in them --- vinca/resolve.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vinca/resolve.py b/vinca/resolve.py index 44c9bd4..069bedb 100644 --- a/vinca/resolve.py +++ b/vinca/resolve.py @@ -76,7 +76,7 @@ def resolve_pkgname(pkg_shortname, vinca_conf, distro, is_rundep=False): pkg_names_pinned = [] for pkg_name in pkg_names: - if " " in pkg_name: + if " " in pkg_name and "${{" not in pkg_name: pkg_name_raw = pkg_name.split(" ")[0] else: pkg_name_raw = pkg_name From eb9616ceb8cc0df549c823d0ca61d8ba315d3a06 Mon Sep 17 00:00:00 2001 From: Wolf Vollprecht Date: Thu, 20 Jun 2024 21:19:24 +0200 Subject: [PATCH 07/57] add test functionality --- vinca/main.py | 20 ++++++++++++++++---- vinca/template.py | 10 +++++++++- 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/vinca/main.py b/vinca/main.py index a8c1b4d..1d0fff9 100644 --- a/vinca/main.py +++ b/vinca/main.py @@ -191,6 +191,12 @@ def read_vinca_yaml(filepath): vinca_conf["_patches"] = patches + tests = {} + test_dir = Path(filepath).parent / "tests" + for x in test_dir.glob("*.yaml"): + tests[os.path.basename(x).split(".")[0]] = x + vinca_conf["_tests"] = tests + if (patch_dir / "dependencies.yaml").exists(): vinca_conf["depmods"] = yaml.load(open(patch_dir / "dependencies.yaml")) if not vinca_conf.get("depmods"): @@ -688,9 +694,9 @@ def get_selected_packages(distro, vinca_conf): def parse_package(pkg, distro, vinca_conf, path): - name = pkg["name"].replace("_", "-") final_name = f"ros-{distro.name}-{name}" + recipe = { "package": {"name": final_name, "version": pkg["version"]}, "about": { @@ -727,6 +733,12 @@ def parse_package(pkg, distro, vinca_conf, path): }, } + if test := vinca_conf.get("_tests", {}).get(final_name): + # parse as yaml + text = test.read_text() + test_content = ruamel.yaml.safe_load(text) + recipe["test"] = test_content + for p in pkg["authors"]: name = p.name + " (" + p.email + ")" if p.email else p.name recipe["about"]["maintainers"].append(name) @@ -907,7 +919,7 @@ def main(): for o in outputs: sources[o["package"]["name"]] = o["source"] del o["source"] - write_recipe(sources, outputs) + write_recipe(sources, outputs, vinca_conf) else: if arguments.skip_already_built_repodata or vinca_conf.get("skip_existing"): @@ -999,9 +1011,9 @@ def main(): outputs = generate_outputs(distro, vinca_conf) if arguments.multiple_file: - write_recipe(source, outputs, vinca_conf.get("build_number", 0), False) + write_recipe(source, outputs, vinca_conf, False) else: - write_recipe(source, outputs, vinca_conf.get("build_number", 0)) + write_recipe(source, outputs, vinca_conf) if unsatisfied_deps: print("Unsatisfied dependencies:", unsatisfied_deps) diff --git a/vinca/template.py b/vinca/template.py index c2843f9..37527d1 100644 --- a/vinca/template.py +++ b/vinca/template.py @@ -60,7 +60,8 @@ def write_recipe_package(recipe): file.dump(recipe, stream) -def write_recipe(source, outputs, build_number=0, single_file=True): +def write_recipe(source, outputs, vinca_conf, single_file=True): + build_number = vinca_conf.get("build_number", 0) # single_file = False if single_file: file = yaml.YAML() @@ -93,6 +94,13 @@ def write_recipe(source, outputs, build_number=0, single_file=True): meta["build"]["number"] = build_number meta["build"]["post_process"] = post_process_items + + if test := vinca_conf["_tests"].get(o["package"]["name"]): + print("Using test: ", test) + text = test.read_text() + test_content = yaml.safe_load(text) + meta["tests"] = test_content["tests"] + recipe_dir = (Path("recipes") / o["package"]["name"]).absolute() os.makedirs(recipe_dir, exist_ok=True) with open(recipe_dir / "recipe.yaml", "w") as stream: From c261d1eae54d9952504355c9414f4b06cb926b0f Mon Sep 17 00:00:00 2001 From: Wolf Vollprecht Date: Fri, 21 Jun 2024 08:30:31 +0200 Subject: [PATCH 08/57] update skipping packages to work with .conda --- vinca/generate_gha.py | 4 ---- vinca/main.py | 3 ++- vinca/utils.py | 1 + 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/vinca/generate_gha.py b/vinca/generate_gha.py index 6c5ef0f..a42c82b 100644 --- a/vinca/generate_gha.py +++ b/vinca/generate_gha.py @@ -263,8 +263,6 @@ def build_linux_pipeline( if docker_image is None: docker_image = "condaforge/linux-anvil-cos7-x86_64" - jobs = [] - job_names = [] prev_batch_keys = [] for i, s in enumerate(stages): @@ -319,8 +317,6 @@ def build_osx_pipeline( if azure_template is None: azure_template = blurb - jobs = [] - job_names = [] prev_batch_keys = [] for i, s in enumerate(stages): stage_name = f"stage_{i}" diff --git a/vinca/main.py b/vinca/main.py index 1d0fff9..5fbe578 100644 --- a/vinca/main.py +++ b/vinca/main.py @@ -965,7 +965,8 @@ def main(): f"ros-{distro}-{pkg.replace('_', '-')}" for pkg in ensure_list(vinca_conf["packages_select_by_deps"]) ] - + all_pkgs = repodata.get("packages", {}) + all_pkgs.update(repodata.get("packages.conda", {})) for _, pkg in repodata.get("packages").items(): is_built = False if selected_bn is not None: diff --git a/vinca/utils.py b/vinca/utils.py index a516720..486ef88 100644 --- a/vinca/utils.py +++ b/vinca/utils.py @@ -46,6 +46,7 @@ def get_repodata(url_or_path, platform=None): if "://" not in url_or_path: with open(url_or_path) as fi: return json.load(fi) + print("Downloading repodata from ", url_or_path) m = hashlib.md5(url_or_path.encode("utf-8")).hexdigest()[:10] # print(tempfile.gettempdir()) From 6c9272d93840f379319220ce35ccf66d21079d16 Mon Sep 17 00:00:00 2001 From: Wolf Vollprecht Date: Fri, 21 Jun 2024 12:12:46 +0200 Subject: [PATCH 09/57] use stdlib --- vinca/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vinca/main.py b/vinca/main.py index 5fbe578..bbea0c5 100644 --- a/vinca/main.py +++ b/vinca/main.py @@ -238,7 +238,7 @@ def generate_output(pkg_shortname, vinca_conf, distro, version, all_pkgs=None): "build": [ "${{ compiler('cxx') }}", "${{ compiler('c') }}", - {"if": "linux64", "then": ["sysroot_linux-64 2.17"]}, + "${{ stdlib('c') }}", "ninja", "setuptools", {"if": "unix", "then": ["make", "coreutils"]}, @@ -715,7 +715,7 @@ def parse_package(pkg, distro, vinca_conf, path): "build": [ "${{ compiler('cxx') }}", "${{ compiler('c') }}", - {"if": "linux64", "then": ["sysroot_linux-64 2.17"]}, + "${{ stdlib('c') }}", "ninja", {"if": "unix", "then": ["make", "coreutils"]}, "cmake", From 64e05316008130829e3f7af98a9c4bf140d7f9d9 Mon Sep 17 00:00:00 2001 From: Jayden Grubb Date: Tue, 3 Sep 2024 02:45:17 +1000 Subject: [PATCH 10/57] Use snapshot file with -z flag --- vinca/distro.py | 12 +++++++++++- vinca/main.py | 27 ++++++++++++++++++++++----- 2 files changed, 33 insertions(+), 6 deletions(-) diff --git a/vinca/distro.py b/vinca/distro.py index 666e420..e545d9c 100644 --- a/vinca/distro.py +++ b/vinca/distro.py @@ -6,10 +6,11 @@ class Distro(object): - def __init__(self, distro_name, python_version=None): + def __init__(self, distro_name, python_version=None, snapshot=None): index = get_index(get_index_url()) self._distro = get_cached_distribution(index, distro_name) self.distro_name = distro_name + self.snapshot = snapshot # set up ROS environments if python_version is None: python_version = index.distributions[distro_name]["python_version"] @@ -60,6 +61,12 @@ def get_depends(self, pkg, ignore_pkgs=None): return dependencies def get_released_repo(self, pkg_name): + if pkg_name in self.snapshot: + return ( + self.snapshot[pkg_name].get("url", None), + self.snapshot[pkg_name].get("tag", None), + ) + pkg = self._distro.release_packages[pkg_name] repo = self._distro.repositories[pkg.repository_name].release_repository release_tag = get_release_tag(repo, pkg_name) @@ -74,6 +81,9 @@ def check_package(self, pkg_name): return False def get_version(self, pkg_name): + if pkg_name in self.snapshot: + return self.snapshot[pkg_name].get("version", None) + pkg = self._distro.release_packages[pkg_name] repo = self._distro.repositories[pkg.repository_name].release_repository return repo.version.split("-")[0] diff --git a/vinca/main.py b/vinca/main.py index 0be6391..1b5111e 100644 --- a/vinca/main.py +++ b/vinca/main.py @@ -126,6 +126,13 @@ def parse_command_line(argv): default=None, help="The conda platform to check existing recipes for.", ) + parser.add_argument( + "-z", + "--snapshot", + dest="snapshot", + default=None, + help="The version snapshot file (default: None)." + ) arguments = parser.parse_args(argv[1:]) global selected_platform config.parsed_args = arguments @@ -208,6 +215,15 @@ def read_vinca_yaml(filepath): return vinca_conf +def read_snapshot(filepath): + if not filepath: + return None + + yaml = ruamel.yaml.YAML() + snapshot = yaml.load(open(filepath, "r")) + return snapshot + + def generate_output(pkg_shortname, vinca_conf, distro, version, all_pkgs=None): if not all_pkgs: all_pkgs = [] @@ -534,7 +550,7 @@ def get_version(distro, vinca_conf, pkg_shortname): vinca_conf.get("package_version") and vinca_conf["package_version"][pkg_shortname] ): - version = vinca_conf["package_version"][pkg_shortname]["version"] + version = vinca_conf["package_version"][pkg_shortname]["version"] # TODO: What is this for? return version @@ -607,7 +623,7 @@ def generate_source_version(distro, vinca_conf): and vinca_conf["package_version"][pkg_shortname] ): url = vinca_conf["package_version"][pkg_shortname]["url"] - version = vinca_conf["package_version"][pkg_shortname]["version"] + version = vinca_conf["package_version"][pkg_shortname]["version"] # TODO: What is this for? entry = {} entry["git_url"] = url @@ -865,6 +881,7 @@ def main(): base_dir = os.path.abspath(arguments.dir) vinca_yaml = os.path.join(base_dir, "vinca.yaml") vinca_conf = read_vinca_yaml(vinca_yaml) + snapshot = read_snapshot(arguments.snapshot) from .template import generate_bld_ament_cmake from .template import generate_bld_ament_python @@ -879,7 +896,7 @@ def main(): generate_bld_colcon_merge() generate_bld_catkin_merge() generate_activate_hook() - + if arguments.trigger_new_versions: vinca_conf["trigger_new_versions"] = True else: @@ -892,7 +909,7 @@ def main(): if "python_version" in vinca_conf: python_version = vinca_conf["python_version"] - distro = Distro(vinca_conf["ros_distro"], python_version) + distro = Distro(vinca_conf["ros_distro"], python_version, snapshot) additional_pkgs, parsed_pkgs = [], [] for f in pkg_files: parsed_pkg = catkin_pkg.package.parse_package(f) @@ -994,7 +1011,7 @@ def main(): if "python_version" in vinca_conf: python_version = vinca_conf["python_version"] - distro = Distro(vinca_conf["ros_distro"], python_version) + distro = Distro(vinca_conf["ros_distro"], python_version, snapshot) selected_pkgs = get_selected_packages(distro, vinca_conf) From 686f05a8168d4e5dd2754f6ba62e90422736419e Mon Sep 17 00:00:00 2001 From: Jayden Grubb Date: Tue, 3 Sep 2024 12:26:36 +1000 Subject: [PATCH 11/57] Remove old package_version code --- vinca/main.py | 21 ++------------------- 1 file changed, 2 insertions(+), 19 deletions(-) diff --git a/vinca/main.py b/vinca/main.py index 1b5111e..7081251 100644 --- a/vinca/main.py +++ b/vinca/main.py @@ -544,21 +544,10 @@ def get_pkg(pkg_name): return outputs -def get_version(distro, vinca_conf, pkg_shortname): - version = distro.get_version(pkg_shortname) - if ( - vinca_conf.get("package_version") - and vinca_conf["package_version"][pkg_shortname] - ): - version = vinca_conf["package_version"][pkg_shortname]["version"] # TODO: What is this for? - - return version - - def generate_outputs_version(distro, vinca_conf): outputs = [] for pkg_shortname in vinca_conf["_selected_pkgs"]: - version = get_version(distro, vinca_conf, pkg_shortname) + version = distro.get_version(pkg_shortname) output = generate_output(pkg_shortname, vinca_conf, distro, version) if output is not None: outputs.append(output) @@ -578,7 +567,7 @@ def generate_source(distro, vinca_conf): entry["git_url"] = url entry["git_rev"] = version pkg_names = resolve_pkgname(pkg_shortname, vinca_conf, distro) - pkg_version = get_version(distro, vinca_conf, pkg_shortname) + pkg_version = distro.get_version(pkg_shortname) print("Checking ", pkg_shortname, pkg_version) if not pkg_names: continue @@ -618,12 +607,6 @@ def generate_source_version(distro, vinca_conf): continue url, version = distro.get_released_repo(pkg_shortname) - if ( - vinca_conf["package_version"] - and vinca_conf["package_version"][pkg_shortname] - ): - url = vinca_conf["package_version"][pkg_shortname]["url"] - version = vinca_conf["package_version"][pkg_shortname]["version"] # TODO: What is this for? entry = {} entry["git_url"] = url From e89a9e8662a56e7e11412e18a708c42a652239ff Mon Sep 17 00:00:00 2001 From: Jayden Grubb Date: Tue, 3 Sep 2024 14:44:40 +1000 Subject: [PATCH 12/57] Add snapshot system under vinca-snapshot --- setup.cfg | 1 + vinca/snapshot.py | 117 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 118 insertions(+) create mode 100644 vinca/snapshot.py diff --git a/setup.cfg b/setup.cfg index 237cb80..befac41 100644 --- a/setup.cfg +++ b/setup.cfg @@ -42,6 +42,7 @@ console_scripts = vinca-gha = vinca.generate_gha:main vinca-azure = vinca.generate_azure:main vinca-migrate = vinca.migrate:main + vinca-snapshot = vinca.snapshot:main [flake8] import-order-style = google diff --git a/vinca/snapshot.py b/vinca/snapshot.py new file mode 100644 index 0000000..164962d --- /dev/null +++ b/vinca/snapshot.py @@ -0,0 +1,117 @@ +import argparse +import os +import yaml + +from rosdistro import get_index, get_index_url, get_cached_distribution +from rosdistro.dependency_walker import DependencyWalker +from rosdistro.manifest_provider import get_release_tag + + +def main(): + parser = argparse.ArgumentParser( + description="Dependency snapshotting tool for ROS packages" + ) + parser.add_argument( + "-d", + "--distro", + type=str, + dest="distro", + default="humble", + help="ROS distribution to use (default: humble)", + required=False, + ) + parser.add_argument( + "-p", + "--package", + type=str, + dest="package", + default="ros_base", + help="ROS package to get dependencies for (default: ros_base)", + required=False, + ) + parser.add_argument( + "-t", + "--type", + type=str, + dest="types", + nargs="+", + default=[ + "buildtool", + "buildtool_export", + "build", + "build_export", + "run", + "test", + "exec", + ], + help="Dependency type to get (default: all)", + required=False, + ) + parser.add_argument( + "-s", + "--system", + type=str, + dest="system", + choices=["linux", "osx", "win64"], + default="linux", + help="System to get dependencies for (default: linux)", + required=False, + ) + parser.add_argument( + "-o", + "--output", + type=str, + dest="output", + default="snapshot.yaml", + help="Output file to write dependencies to", + required=False, + ) + parser.add_argument( + "-q", + "--quiet", + dest="quiet", + action="store_true", + help="Suppress output to stdout", + required=False, + ) + args = parser.parse_args() + + index = get_index(get_index_url()) + distro = get_cached_distribution(index, args.distro) + + python_version = index.distributions[args.distro]["python_version"] + os.environ["ROS_PYTHON_VERSION"] = "{0}".format(python_version) + os.environ["ROS_DISTRO"] = "{0}".format(args.distro) + if "ROS_ROOT" in os.environ: + os.environ.pop("ROS_ROOT") + if "ROS_PACKAGE_PATH" in os.environ: + os.environ.pop("ROS_PACKAGE_PATH") + + walker = DependencyWalker(distro, os.environ) + + deps = walker.get_recursive_depends( + pkg_name=args.package, + depend_types=args.types, + ros_packages_only=True, + ignore_pkgs=None, + ) + deps.add(args.package) + + if not args.quiet: + max_len = max([len(dep) for dep in deps]) + print("\033[1m{0:{2}} {1}\033[0m".format("Package", "Version", max_len + 2)) + + output = {} + + for dep in deps: + pkg = distro.release_packages[dep] + repo = distro.repositories[pkg.repository_name].release_repository + tag = get_release_tag(repo, dep) + version = repo.version.split("-")[0] + + if not args.quiet: + print("{0:{2}} {1}".format(dep, version, max_len + 2)) + output[dep] = {"url": repo.url, "version": version, "tag": tag} + + with open(args.output, "w") as f: + yaml.dump(output, f) From 0eb80d30110eaba4e7e227570626a2ddbdae7f79 Mon Sep 17 00:00:00 2001 From: Jayden Grubb Date: Sat, 7 Sep 2024 12:01:22 +1000 Subject: [PATCH 13/57] Fix Distro when snapshot is null --- vinca/distro.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vinca/distro.py b/vinca/distro.py index e545d9c..acf48c5 100644 --- a/vinca/distro.py +++ b/vinca/distro.py @@ -61,7 +61,7 @@ def get_depends(self, pkg, ignore_pkgs=None): return dependencies def get_released_repo(self, pkg_name): - if pkg_name in self.snapshot: + if self.snapshot and pkg_name in self.snapshot: return ( self.snapshot[pkg_name].get("url", None), self.snapshot[pkg_name].get("tag", None), @@ -81,7 +81,7 @@ def check_package(self, pkg_name): return False def get_version(self, pkg_name): - if pkg_name in self.snapshot: + if self.snapshot and pkg_name in self.snapshot: return self.snapshot[pkg_name].get("version", None) pkg = self._distro.release_packages[pkg_name] From 0c3783a3ba0d717bf3fb14640c42041ef8a95930 Mon Sep 17 00:00:00 2001 From: Jayden Grubb Date: Sat, 7 Sep 2024 12:02:26 +1000 Subject: [PATCH 14/57] Use Vinca Distro instead of disros --- vinca/snapshot.py | 63 +++++------------------------------------------ 1 file changed, 6 insertions(+), 57 deletions(-) diff --git a/vinca/snapshot.py b/vinca/snapshot.py index 164962d..7d20f62 100644 --- a/vinca/snapshot.py +++ b/vinca/snapshot.py @@ -1,10 +1,6 @@ import argparse -import os import yaml - -from rosdistro import get_index, get_index_url, get_cached_distribution -from rosdistro.dependency_walker import DependencyWalker -from rosdistro.manifest_provider import get_release_tag +from .distro import Distro def main(): @@ -29,34 +25,6 @@ def main(): help="ROS package to get dependencies for (default: ros_base)", required=False, ) - parser.add_argument( - "-t", - "--type", - type=str, - dest="types", - nargs="+", - default=[ - "buildtool", - "buildtool_export", - "build", - "build_export", - "run", - "test", - "exec", - ], - help="Dependency type to get (default: all)", - required=False, - ) - parser.add_argument( - "-s", - "--system", - type=str, - dest="system", - choices=["linux", "osx", "win64"], - default="linux", - help="System to get dependencies for (default: linux)", - required=False, - ) parser.add_argument( "-o", "--output", @@ -76,25 +44,8 @@ def main(): ) args = parser.parse_args() - index = get_index(get_index_url()) - distro = get_cached_distribution(index, args.distro) - - python_version = index.distributions[args.distro]["python_version"] - os.environ["ROS_PYTHON_VERSION"] = "{0}".format(python_version) - os.environ["ROS_DISTRO"] = "{0}".format(args.distro) - if "ROS_ROOT" in os.environ: - os.environ.pop("ROS_ROOT") - if "ROS_PACKAGE_PATH" in os.environ: - os.environ.pop("ROS_PACKAGE_PATH") - - walker = DependencyWalker(distro, os.environ) - - deps = walker.get_recursive_depends( - pkg_name=args.package, - depend_types=args.types, - ros_packages_only=True, - ignore_pkgs=None, - ) + distro = Distro(args.distro) + deps = distro.get_depends(args.package) deps.add(args.package) if not args.quiet: @@ -104,14 +55,12 @@ def main(): output = {} for dep in deps: - pkg = distro.release_packages[dep] - repo = distro.repositories[pkg.repository_name].release_repository - tag = get_release_tag(repo, dep) - version = repo.version.split("-")[0] + url, tag = distro.get_released_repo(dep) + version = distro.get_version(dep) + output[dep] = {"url": url, "version": version, "tag": tag} if not args.quiet: print("{0:{2}} {1}".format(dep, version, max_len + 2)) - output[dep] = {"url": repo.url, "version": version, "tag": tag} with open(args.output, "w") as f: yaml.dump(output, f) From a9024f3b1808be1e345900902b014c2a6f8b8660 Mon Sep 17 00:00:00 2001 From: Jacob Oursland Date: Tue, 10 Sep 2024 10:35:36 -0700 Subject: [PATCH 15/57] skip_existing: use packages.conda for packages to skip. --- vinca/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vinca/main.py b/vinca/main.py index bbea0c5..8bf6ee0 100644 --- a/vinca/main.py +++ b/vinca/main.py @@ -950,7 +950,7 @@ def main(): if "://" in fn: selected_bn = vinca_conf.get("build_number", 0) distro = vinca_conf["ros_distro"] - for pkg_name, pkg in repodata.get("packages").items(): + for pkg_name, pkg in repodata.get("packages.conda").items(): if pkg_name.startswith(f"ros-{distro}"): if pkg_name.rsplit("-", 2)[0] in additional_recipe_names: print( From 1f89b955d99946a18226bbb9399ec9def3bffc09 Mon Sep 17 00:00:00 2001 From: Jayden Grubb Date: Thu, 12 Sep 2024 21:43:08 +1000 Subject: [PATCH 16/57] Default vinca-snapshot package to ALL --- vinca/distro.py | 3 +++ vinca/snapshot.py | 21 +++++++++++++++------ 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/vinca/distro.py b/vinca/distro.py index acf48c5..de7d445 100644 --- a/vinca/distro.py +++ b/vinca/distro.py @@ -96,3 +96,6 @@ def check_ros1(self): def get_python_version(self): return self._python_version + + def get_package_names(self): + return self._distro.release_packages.keys() diff --git a/vinca/snapshot.py b/vinca/snapshot.py index 7d20f62..74ab2fd 100644 --- a/vinca/snapshot.py +++ b/vinca/snapshot.py @@ -21,8 +21,8 @@ def main(): "--package", type=str, dest="package", - default="ros_base", - help="ROS package to get dependencies for (default: ros_base)", + default=None, + help="ROS package to get dependencies for (default: ALL)", required=False, ) parser.add_argument( @@ -45,8 +45,12 @@ def main(): args = parser.parse_args() distro = Distro(args.distro) - deps = distro.get_depends(args.package) - deps.add(args.package) + + if args.package is None: + deps = distro.get_package_names() + else: + deps = distro.get_depends(args.package) + deps.add(args.package) if not args.quiet: max_len = max([len(dep) for dep in deps]) @@ -55,8 +59,13 @@ def main(): output = {} for dep in deps: - url, tag = distro.get_released_repo(dep) - version = distro.get_version(dep) + try: + url, tag = distro.get_released_repo(dep) + version = distro.get_version(dep) + except AttributeError: + print("\033[93mPackage '{}' has no version set, skipping...\033[0m".format(dep)) + continue + output[dep] = {"url": url, "version": version, "tag": tag} if not args.quiet: From fe5d9c0cc8ee089e2c882d2e385ef50f17517d31 Mon Sep 17 00:00:00 2001 From: Jacob Oursland Date: Thu, 12 Sep 2024 15:44:40 -0700 Subject: [PATCH 17/57] Add CMAKE_IGNORE_PREFIX_PATH to prevent use of Homebrew-installed packages. --- vinca/templates/build_ament_cmake.sh.in | 1 + 1 file changed, 1 insertion(+) diff --git a/vinca/templates/build_ament_cmake.sh.in b/vinca/templates/build_ament_cmake.sh.in index 4bbafe9..d26f218 100644 --- a/vinca/templates/build_ament_cmake.sh.in +++ b/vinca/templates/build_ament_cmake.sh.in @@ -68,6 +68,7 @@ cmake \ -DCMAKE_INSTALL_SYSTEM_RUNTIME_LIBS_SKIP=True \ -DBUILD_SHARED_LIBS=ON \ -DBUILD_TESTING=OFF \ + -DCMAKE_IGNORE_PREFIX_PATH="/opt/homebrew;/usr/local/homebrew" \ -DCMAKE_OSX_DEPLOYMENT_TARGET=$OSX_DEPLOYMENT_TARGET \ --compile-no-warning-as-error \ $SRC_DIR/$PKG_NAME/src/work From a17509cb1bb2f5a58888d5be084093b3b17140a0 Mon Sep 17 00:00:00 2001 From: Jacob Oursland Date: Sat, 14 Sep 2024 09:37:53 -0700 Subject: [PATCH 18/57] Add python to templates. --- vinca/main.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/vinca/main.py b/vinca/main.py index 8bf6ee0..046f6b9 100644 --- a/vinca/main.py +++ b/vinca/main.py @@ -240,6 +240,7 @@ def generate_output(pkg_shortname, vinca_conf, distro, version, all_pkgs=None): "${{ compiler('c') }}", "${{ stdlib('c') }}", "ninja", + "python", "setuptools", {"if": "unix", "then": ["make", "coreutils"]}, {"if": "osx", "then": ["tapi"]}, @@ -255,6 +256,7 @@ def generate_output(pkg_shortname, vinca_conf, distro, version, all_pkgs=None): ], "host": [ {"if": "build_platform == target_platform", "then": ["pkg-config"]}, + "python", "numpy", "pip", ], @@ -717,6 +719,7 @@ def parse_package(pkg, distro, vinca_conf, path): "${{ compiler('c') }}", "${{ stdlib('c') }}", "ninja", + "python", {"if": "unix", "then": ["make", "coreutils"]}, "cmake", {"if": "build_platform != target_platform", "then": ["python"]}, From 18f6e2ae80a02dadc96cc84100d1d4a3aa76d99e Mon Sep 17 00:00:00 2001 From: Jacob Oursland Date: Thu, 12 Sep 2024 15:44:40 -0700 Subject: [PATCH 19/57] Add CMAKE_IGNORE_PREFIX_PATH to prevent use of Homebrew-installed packages. --- vinca/templates/build_ament_cmake.sh.in | 1 + 1 file changed, 1 insertion(+) diff --git a/vinca/templates/build_ament_cmake.sh.in b/vinca/templates/build_ament_cmake.sh.in index 4bbafe9..d26f218 100644 --- a/vinca/templates/build_ament_cmake.sh.in +++ b/vinca/templates/build_ament_cmake.sh.in @@ -68,6 +68,7 @@ cmake \ -DCMAKE_INSTALL_SYSTEM_RUNTIME_LIBS_SKIP=True \ -DBUILD_SHARED_LIBS=ON \ -DBUILD_TESTING=OFF \ + -DCMAKE_IGNORE_PREFIX_PATH="/opt/homebrew;/usr/local/homebrew" \ -DCMAKE_OSX_DEPLOYMENT_TARGET=$OSX_DEPLOYMENT_TARGET \ --compile-no-warning-as-error \ $SRC_DIR/$PKG_NAME/src/work From 2f119ffc07e27d3a87b2d60f716a80ff08450102 Mon Sep 17 00:00:00 2001 From: Jacob Oursland Date: Sat, 14 Sep 2024 09:37:53 -0700 Subject: [PATCH 20/57] Add python to templates. --- vinca/main.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/vinca/main.py b/vinca/main.py index bbea0c5..7008021 100644 --- a/vinca/main.py +++ b/vinca/main.py @@ -240,6 +240,7 @@ def generate_output(pkg_shortname, vinca_conf, distro, version, all_pkgs=None): "${{ compiler('c') }}", "${{ stdlib('c') }}", "ninja", + "python", "setuptools", {"if": "unix", "then": ["make", "coreutils"]}, {"if": "osx", "then": ["tapi"]}, @@ -255,6 +256,7 @@ def generate_output(pkg_shortname, vinca_conf, distro, version, all_pkgs=None): ], "host": [ {"if": "build_platform == target_platform", "then": ["pkg-config"]}, + "python", "numpy", "pip", ], @@ -717,6 +719,7 @@ def parse_package(pkg, distro, vinca_conf, path): "${{ compiler('c') }}", "${{ stdlib('c') }}", "ninja", + "python", {"if": "unix", "then": ["make", "coreutils"]}, "cmake", {"if": "build_platform != target_platform", "then": ["python"]}, From 200ed5e26b08207eafc4961e92ce5b9ec3c1dd27 Mon Sep 17 00:00:00 2001 From: Jayden Grubb Date: Fri, 18 Oct 2024 19:38:44 +1000 Subject: [PATCH 21/57] Skip packages not in snapshot when snapshot specified --- vinca/distro.py | 2 +- vinca/main.py | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/vinca/distro.py b/vinca/distro.py index de7d445..aa6bf8f 100644 --- a/vinca/distro.py +++ b/vinca/distro.py @@ -74,7 +74,7 @@ def get_released_repo(self, pkg_name): def check_package(self, pkg_name): if pkg_name in self._distro.release_packages: - return True + return self.snapshot is None or pkg_name in self.snapshot elif pkg_name in self.build_packages: return True else: diff --git a/vinca/main.py b/vinca/main.py index 7081251..215e65b 100644 --- a/vinca/main.py +++ b/vinca/main.py @@ -547,6 +547,10 @@ def get_pkg(pkg_name): def generate_outputs_version(distro, vinca_conf): outputs = [] for pkg_shortname in vinca_conf["_selected_pkgs"]: + if not distro.check_package(pkg_shortname): + print(f"Could not generate output for {pkg_shortname}") + continue + version = distro.get_version(pkg_shortname) output = generate_output(pkg_shortname, vinca_conf, distro, version) if output is not None: From 0e23184afde4c45adeaee0fddd0c469dd55fed55 Mon Sep 17 00:00:00 2001 From: Silvio Traversaro Date: Wed, 18 Dec 2024 01:13:20 +0100 Subject: [PATCH 22/57] Use tag: instead of rev: in rattler-build recipe --- vinca/main.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/vinca/main.py b/vinca/main.py index 046f6b9..f4562de 100644 --- a/vinca/main.py +++ b/vinca/main.py @@ -555,7 +555,7 @@ def generate_source(distro, vinca_conf): url, version = distro.get_released_repo(pkg_shortname) entry = {} entry["git"] = url - entry["rev"] = version + entry["tag"] = version pkg_names = resolve_pkgname(pkg_shortname, vinca_conf, distro) pkg_version = get_version(distro, vinca_conf, pkg_shortname) print("Checking ", pkg_shortname, pkg_version) @@ -606,7 +606,7 @@ def generate_source_version(distro, vinca_conf): entry = {} entry["git"] = url - entry["rev"] = version + entry["tag"] = version pkg_names = resolve_pkgname(pkg_shortname, vinca_conf, distro) if vinca_conf.get("trigger_new_versions"): if ( @@ -646,7 +646,7 @@ def generate_fat_source(distro, vinca_conf): url, version = distro.get_released_repo(pkg_shortname) entry = {} entry["git"] = url - entry["rev"] = version + entry["tag"] = version pkg_names = resolve_pkgname(pkg_shortname, vinca_conf, distro) if not pkg_names: continue @@ -749,7 +749,7 @@ def parse_package(pkg, distro, vinca_conf, path): for u in pkg["urls"]: # if u.type == 'repository' : # recipe['source']['git'] = u.url - # recipe['source']['rev'] = recipe['package']['version'] + # recipe['source']['tag'] = recipe['package']['version'] if u.type == "website": recipe["about"]["homepage"] = u.url From f6ce41dd4e485f3230a305b5672d1bac00dbe4cf Mon Sep 17 00:00:00 2001 From: Silvio Date: Wed, 18 Dec 2024 12:24:02 +0100 Subject: [PATCH 23/57] Add compatibility with rattler-build 0.32 --- vinca/main.py | 10 +++++----- vinca/template.py | 21 ++++++++++++++++++++- 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/vinca/main.py b/vinca/main.py index f4562de..fdb776b 100644 --- a/vinca/main.py +++ b/vinca/main.py @@ -277,15 +277,15 @@ def generate_output(pkg_shortname, vinca_conf, distro, version, all_pkgs=None): if pkg.get_build_type() in ["cmake", "catkin"]: output["build"][ "script" - ] = "${{ 'build_catkin.sh' if unix else 'bld_catkin.bat' }}" + ] = "${{ '$RECIPE_DIR/build_catkin.sh' if unix else '%RECIPE_DIR%\\bld_catkin.bat' }}" elif pkg.get_build_type() in ["ament_cmake"]: output["build"][ "script" - ] = "${{ 'build_ament_cmake.sh' if unix else 'bld_ament_cmake.bat' }}" + ] = "${{ '$RECIPE_DIR/build_ament_cmake.sh' if unix else '%RECIPE_DIR%\\bld_ament_cmake.bat' }}" elif pkg.get_build_type() in ["ament_python"]: output["build"][ "script" - ] = "${{ 'build_ament_python.sh' if unix else 'bld_ament_python.bat' }}" + ] = "${{ '$RECIPE_DIR/build_ament_python.sh' if unix else '%RECIPE_DIR%\\bld_ament_python.bat' }}" resolved_setuptools = resolve_pkgname("python-setuptools", vinca_conf, distro) output["requirements"]["host"].extend(resolved_setuptools) else: @@ -710,7 +710,7 @@ def parse_package(pkg, distro, vinca_conf, path): "extra": {"recipe-maintainers": ["robostack"]}, "build": { "number": 0, - "script": "${{ 'build_catkin.sh' if unix else 'bld_catkin.bat' }}", + "script": "${{ '$RECIPE_DIR/build_catkin.sh' if unix else '%RECIPE_DIR%\\bld_catkin.bat' }}", }, "source": {}, "requirements": { @@ -801,7 +801,7 @@ def parse_package(pkg, distro, vinca_conf, path): if pkg.get_build_type() in ["cmake", "catkin"]: recipe["build"][ "script" - ] = "${{ 'build_catkin.sh' if unix else 'bld_catkin.bat' }}" + ] = "${{ '$RECIPE_DIR/build_catkin.sh' if unix else '%RECIPE_DIR%\\bld_catkin.bat' }}" # fix up OPENGL support for Unix if ( diff --git a/vinca/template.py b/vinca/template.py index 37527d1..81b2680 100644 --- a/vinca/template.py +++ b/vinca/template.py @@ -2,6 +2,7 @@ import shutil import os import re +import stat from ruamel import yaml from pathlib import Path @@ -59,6 +60,15 @@ def write_recipe_package(recipe): with open(recipe_path, "w") as stream: file.dump(recipe, stream) +def copyfile_with_exec_permissions(source_file, destination_file): + shutil.copyfile(source_file, destination_file) + + # It seems that rattler-build requires script to have executable permissions + if os.name == 'posix': + # Retrieve current permissions + current_permissions = os.stat(destination_file).st_mode + # Set executable permissions for user, group, and others + os.chmod(destination_file, current_permissions | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH) def write_recipe(source, outputs, vinca_conf, single_file=True): build_number = vinca_conf.get("build_number", 0) @@ -113,8 +123,10 @@ def write_recipe(source, outputs, vinca_conf, single_file=True): shutil.copyfile(p, recipe_dir / p) build_scripts = re.findall(r"'(.*?)'", meta["build"]["script"]) + baffer = meta["build"]["script"] for script in build_scripts: - shutil.copyfile(script, recipe_dir / script) + script_filename = script.replace("$RECIPE_DIR", "").replace("%RECIPE_DIR%", "").replace("/", "").replace("\\", "") + copyfile_with_exec_permissions(script_filename, recipe_dir / script_filename) if "catkin" in o["package"]["name"] or "workspace" in o["package"]["name"]: shutil.copyfile("activate.sh", recipe_dir / "activate.sh") shutil.copyfile("activate.bat", recipe_dir / "activate.bat") @@ -136,6 +148,13 @@ def generate_template(template_in, template_out): interpreter.file(open(template_in)) interpreter.shutdown() + # It seems that rattler-build requires script to have executable permissions + # See https://github.com/RoboStack/ros-humble/pull/229#issuecomment-2549988298 + if os.name == 'posix': + # Retrieve current permissions + current_permissions = os.stat(template_out.name).st_mode + # Set executable permissions for user, group, and others + os.chmod(template_out.name, current_permissions | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH) def generate_bld_ament_cmake(): import pkg_resources From 72363bafc116e5e15c4dc6b20eb9831237fbe6e0 Mon Sep 17 00:00:00 2001 From: Silvio Traversaro Date: Thu, 19 Dec 2024 20:34:05 +0000 Subject: [PATCH 24/57] Use libgl-devel for packages that require OpenGL --- vinca/main.py | 19 ++++--------------- 1 file changed, 4 insertions(+), 15 deletions(-) diff --git a/vinca/main.py b/vinca/main.py index fdb776b..da19b8f 100644 --- a/vinca/main.py +++ b/vinca/main.py @@ -431,19 +431,11 @@ def sortkey(k): while "REQUIRE_OPENGL" in output["requirements"]["host"]: output["requirements"]["host"].remove("REQUIRE_OPENGL") - output["requirements"]["build"] += [ + output["requirements"]["host"] += [ { "if": "linux", "then": [ - "${{ cdt('mesa-libgl-devel') }}", - "${{ cdt('mesa-dri-drivers') }}", - "${{ cdt('libselinux') }}", - "${{ cdt('libxdamage') }}", - "${{ cdt('libxxf86vm') }}", - "${{ cdt('libxfixes') }}", - "${{ cdt('libxext') }}", - "${{ cdt('libxau') }}", - "${{ cdt('libxcb') }}", + "libgl-devel" ], } ] @@ -466,14 +458,11 @@ def sortkey(k): while "REQUIRE_GL" in output["requirements"]["host"]: output["requirements"]["host"].remove("REQUIRE_GL") - output["requirements"]["build"] += [ + output["requirements"]["host"] += [ { "if": "linux", "then": [ - "${{ cdt('mesa-libgl-devel') }}", - "${{ cdt('mesa-dri-drivers') }}", - "${{ cdt('libselinux') }}", - "${{ cdt('libxxf86vm') }}", + "libgl-devel" ], } ] From 0424315c828527321a7893a49163018f7487a81e Mon Sep 17 00:00:00 2001 From: Tobias Fischer Date: Tue, 24 Dec 2024 11:40:45 +1000 Subject: [PATCH 25/57] Add req on patch which is needed for rviz-ogre-vendor on macos --- vinca/main.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/vinca/main.py b/vinca/main.py index da19b8f..6defb44 100644 --- a/vinca/main.py +++ b/vinca/main.py @@ -242,6 +242,7 @@ def generate_output(pkg_shortname, vinca_conf, distro, version, all_pkgs=None): "ninja", "python", "setuptools", + "patch", {"if": "unix", "then": ["make", "coreutils"]}, {"if": "osx", "then": ["tapi"]}, {"if": "build_platform != target_platform", "then": ["pkg-config"]}, @@ -709,6 +710,7 @@ def parse_package(pkg, distro, vinca_conf, path): "${{ stdlib('c') }}", "ninja", "python", + "patch", {"if": "unix", "then": ["make", "coreutils"]}, "cmake", {"if": "build_platform != target_platform", "then": ["python"]}, From df5494e91bf846c4fbb37c53f52d9cbbb3b4a371 Mon Sep 17 00:00:00 2001 From: Tobias Fischer Date: Wed, 25 Dec 2024 16:12:01 +1000 Subject: [PATCH 26/57] Add QT_HOST_PATH --- vinca/templates/build_ament_cmake.sh.in | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/vinca/templates/build_ament_cmake.sh.in b/vinca/templates/build_ament_cmake.sh.in index d26f218..1d36512 100644 --- a/vinca/templates/build_ament_cmake.sh.in +++ b/vinca/templates/build_ament_cmake.sh.in @@ -18,6 +18,12 @@ else OSX_DEPLOYMENT_TARGET="11.0" fi +if [[ "${CONDA_BUILD_CROSS_COMPILATION:-}" == "1" ]]; then + export QT_HOST_PATH="$BUILD_PREFIX" +else + export QT_HOST_PATH="$PREFIX" +fi + echo "USING PYTHON_EXECUTABLE=${PYTHON_EXECUTABLE}" echo "USING PKG_CONFIG_EXECUTABLE=${PKG_CONFIG_EXECUTABLE}" From 4215543e2eeace1b78a82ae5d414f8839ba4a9c4 Mon Sep 17 00:00:00 2001 From: Tobias Fischer Date: Thu, 26 Dec 2024 08:12:54 +1000 Subject: [PATCH 27/57] Fixup regex issues --- vinca/template.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/vinca/template.py b/vinca/template.py index 81b2680..5b8af6e 100644 --- a/vinca/template.py +++ b/vinca/template.py @@ -35,17 +35,17 @@ { "files": ["*.pc"], "regex": '(?:-L|-I)?"?([^;\\s]+/sysroot/)', - "replacement": "$(CONDA_BUILD_SYSROOT_S)", + "replacement": "$$(CONDA_BUILD_SYSROOT_S)", }, { "files": ["*.cmake"], "regex": '([^;\\s"]+/sysroot)', - "replacement": "$ENV{CONDA_BUILD_SYSROOT}", + "replacement": "$$ENV{CONDA_BUILD_SYSROOT}", }, { "files": ["*.cmake"], "regex": '([^;\\s"]+/MacOSX\\d*\\.?\\d*\\.sdk)', - "replacement": "$ENV{CONDA_BUILD_SYSROOT}", + "replacement": "$$ENV{CONDA_BUILD_SYSROOT}", }, ] From 61356d4aa61cee5a65dfc9f77fd2eb79111cbf26 Mon Sep 17 00:00:00 2001 From: Tobias Fischer Date: Fri, 27 Dec 2024 21:13:44 +1000 Subject: [PATCH 28/57] Update win_preconfig.bat --- vinca/azure_templates/win_preconfig.bat | 18 +----------------- 1 file changed, 1 insertion(+), 17 deletions(-) diff --git a/vinca/azure_templates/win_preconfig.bat b/vinca/azure_templates/win_preconfig.bat index 796cb9e..3bae13e 100644 --- a/vinca/azure_templates/win_preconfig.bat +++ b/vinca/azure_templates/win_preconfig.bat @@ -1,5 +1,4 @@ -set "CI=azure" -call activate base +set "CI=true" :: 4 cores available on GHA: https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners/about-github-hosted-runners :: CPU_COUNT is passed through conda build: https://github.com/conda/conda-build/pull/1149 @@ -7,10 +6,6 @@ set CPU_COUNT=4 set PYTHONUNBUFFERED=1 -conda config --set show_channel_urls true -conda config --set auto_update_conda false -conda config --set add_pip_as_python_dependency false - call setup_x64 :: Set the conda-build working directory to a smaller path @@ -28,14 +23,3 @@ if defined CI ( :: Make paths like C:\\hostedtoolcache\\windows\\Ruby\\2.5.7\\x64\\bin garbage set "PATH=%PATH:ostedtoolcache=%" - -mkdir "%CONDA%\\etc\\conda\\activate.d" - -echo set "CONDA_BLD_PATH=%CONDA_BLD_PATH%" > "%CONDA%\\etc\\conda\\activate.d\\conda-forge-ci-setup-activate.bat" -echo set "CPU_COUNT=%CPU_COUNT%" >> "%CONDA%\\etc\\conda\\activate.d\\conda-forge-ci-setup-activate.bat" -echo set "PYTHONUNBUFFERED=%PYTHONUNBUFFERED%" >> "%CONDA%\\etc\\conda\\activate.d\\conda-forge-ci-setup-activate.bat" -echo set "PATH=%PATH%" >> "%CONDA%\\etc\\conda\\activate.d\\conda-forge-ci-setup-activate.bat" - -conda info -conda config --show-sources -conda list --show-channel-urls From 3557e2548026017f25c7005bf79a1cd6db21b0a5 Mon Sep 17 00:00:00 2001 From: Tobias Fischer Date: Fri, 27 Dec 2024 21:17:53 +1000 Subject: [PATCH 29/57] Simplify win --- vinca/generate_gha.py | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/vinca/generate_gha.py b/vinca/generate_gha.py index a42c82b..18549d7 100644 --- a/vinca/generate_gha.py +++ b/vinca/generate_gha.py @@ -390,21 +390,13 @@ def build_win_pipeline(stages, trigger_branch, outfile="win.yml", azure_template "steps": [ {"name": "Checkout code", "uses": "actions/checkout@v4"}, { - "uses": "conda-incubator/setup-miniconda@v3", + "name": "Setup pixi", + "uses": "prefix-dev/setup-pixi@v0.8.1", "with": { - "channels": "conda-forge", - "miniforge-variant": "Mambaforge", - "miniforge-version": "latest", - "use-mamba": "true", - "channel-priority": "true", - "python-version": "3.11", - "activate-environment": "test", + "pixi-version": "conda-forge", + "cache": "true", }, }, - { - "run": "mamba install -c conda-forge -n base --yes --quiet conda-build=3.27 pip mamba ruamel.yaml anaconda-client boa", - "name": "Install conda-build, boa and activate environment", - }, { "uses": "egor-tensin/cleanup-path@v4", "with": { From 9cd1c85182083fa01bf726714da2273e17696f80 Mon Sep 17 00:00:00 2001 From: Silvio Traversaro Date: Fri, 27 Dec 2024 16:10:08 +0100 Subject: [PATCH 30/57] Windows fixes --- vinca/main.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/vinca/main.py b/vinca/main.py index 6defb44..6202d2e 100644 --- a/vinca/main.py +++ b/vinca/main.py @@ -242,8 +242,8 @@ def generate_output(pkg_shortname, vinca_conf, distro, version, all_pkgs=None): "ninja", "python", "setuptools", - "patch", - {"if": "unix", "then": ["make", "coreutils"]}, + {"if": "unix", "then": ["patch", "make", "coreutils"]}, + {"if": "win", "then": ["m2-patch"]}, {"if": "osx", "then": ["tapi"]}, {"if": "build_platform != target_platform", "then": ["pkg-config"]}, "cmake", @@ -278,15 +278,15 @@ def generate_output(pkg_shortname, vinca_conf, distro, version, all_pkgs=None): if pkg.get_build_type() in ["cmake", "catkin"]: output["build"][ "script" - ] = "${{ '$RECIPE_DIR/build_catkin.sh' if unix else '%RECIPE_DIR%\\bld_catkin.bat' }}" + ] = "${{ '$RECIPE_DIR/build_catkin.sh' if unix else '%RECIPE_DIR%\\\\bld_catkin.bat' }}" elif pkg.get_build_type() in ["ament_cmake"]: output["build"][ "script" - ] = "${{ '$RECIPE_DIR/build_ament_cmake.sh' if unix else '%RECIPE_DIR%\\bld_ament_cmake.bat' }}" + ] = "${{ '$RECIPE_DIR/build_ament_cmake.sh' if unix else '%RECIPE_DIR%\\\\bld_ament_cmake.bat' }}" elif pkg.get_build_type() in ["ament_python"]: output["build"][ "script" - ] = "${{ '$RECIPE_DIR/build_ament_python.sh' if unix else '%RECIPE_DIR%\\bld_ament_python.bat' }}" + ] = "${{ '$RECIPE_DIR/build_ament_python.sh' if unix else '%RECIPE_DIR%\\\\bld_ament_python.bat' }}" resolved_setuptools = resolve_pkgname("python-setuptools", vinca_conf, distro) output["requirements"]["host"].extend(resolved_setuptools) else: @@ -700,7 +700,7 @@ def parse_package(pkg, distro, vinca_conf, path): "extra": {"recipe-maintainers": ["robostack"]}, "build": { "number": 0, - "script": "${{ '$RECIPE_DIR/build_catkin.sh' if unix else '%RECIPE_DIR%\\bld_catkin.bat' }}", + "script": "${{ '$RECIPE_DIR/build_catkin.sh' if unix else '%RECIPE_DIR%\\\\bld_catkin.bat' }}", }, "source": {}, "requirements": { @@ -792,7 +792,7 @@ def parse_package(pkg, distro, vinca_conf, path): if pkg.get_build_type() in ["cmake", "catkin"]: recipe["build"][ "script" - ] = "${{ '$RECIPE_DIR/build_catkin.sh' if unix else '%RECIPE_DIR%\\bld_catkin.bat' }}" + ] = "${{ '$RECIPE_DIR/build_catkin.sh' if unix else '%RECIPE_DIR%\\\\bld_catkin.bat' }}" # fix up OPENGL support for Unix if ( From e7d4cf90b8c401ac0ab6be1b4af11f36f7fe4870 Mon Sep 17 00:00:00 2001 From: Silvio Traversaro Date: Fri, 27 Dec 2024 18:38:24 +0100 Subject: [PATCH 31/57] Avoid to pass \Lib in CMake variable --- vinca/templates/bld_ament_cmake.bat.in | 2 +- vinca/templates/bld_colcon_merge.bat.in | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/vinca/templates/bld_ament_cmake.bat.in b/vinca/templates/bld_ament_cmake.bat.in index 34cd234..2b3b7ab 100644 --- a/vinca/templates/bld_ament_cmake.bat.in +++ b/vinca/templates/bld_ament_cmake.bat.in @@ -26,7 +26,7 @@ set "CMAKE_GENERATOR=Visual Studio %VS_MAJOR% %VS_YEAR%" :: This line is scary, but it basically assigns the output of the command inside (` and `) :: to the variable specified after DO SET :: The equivalent in bash is PYTHON_INSTALL_DIR=`python -c ...` -FOR /F "tokens=* USEBACKQ" %%i IN (`python -c "import os;print(os.path.relpath(os.environ['SP_DIR'],os.environ['LIBRARY_PREFIX']))"`) DO SET PYTHON_INSTALL_DIR=%%i +FOR /F "tokens=* USEBACKQ" %%i IN (`python -c "import os;print(os.path.relpath(os.environ['SP_DIR'],os.environ['LIBRARY_PREFIX']).replace('\\','\\\\'))"`) DO SET PYTHON_INSTALL_DIR=%%i cmake ^ -G "%CMAKE_GENERATOR%" ^ diff --git a/vinca/templates/bld_colcon_merge.bat.in b/vinca/templates/bld_colcon_merge.bat.in index a003a8e..5624268 100644 --- a/vinca/templates/bld_colcon_merge.bat.in +++ b/vinca/templates/bld_colcon_merge.bat.in @@ -15,7 +15,7 @@ set CXX=cl.exe :: This line is scary, but it basically assigns the output of the command inside (` and `) :: to the variable specified after DO SET :: The equivalent in bash is PYTHON_INSTALL_DIR=`python -c ...` -FOR /F "tokens=* USEBACKQ" %%i IN (`python -c "import os;print(os.path.relpath(os.environ['SP_DIR'],os.environ['LIBRARY_PREFIX']))"`) DO SET PYTHON_INSTALL_DIR=%%i +FOR /F "tokens=* USEBACKQ" %%i IN (`python -c "import os;print(os.path.relpath(os.environ['SP_DIR'],os.environ['LIBRARY_PREFIX']).replace('\\','\\\\'))"`) DO SET PYTHON_INSTALL_DIR=%%i colcon build ^ --event-handlers console_cohesion+ ^ From 34cb8dc6264adbaa7b7685789c3ebb204cf939b3 Mon Sep 17 00:00:00 2001 From: Silvio Traversaro Date: Fri, 27 Dec 2024 18:46:47 +0100 Subject: [PATCH 32/57] Avoid to pass \Lib in a CMake option in Windows --- vinca/templates/bld_ament_cmake.bat.in | 2 +- vinca/templates/bld_colcon_merge.bat.in | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/vinca/templates/bld_ament_cmake.bat.in b/vinca/templates/bld_ament_cmake.bat.in index 2b3b7ab..fe65ed1 100644 --- a/vinca/templates/bld_ament_cmake.bat.in +++ b/vinca/templates/bld_ament_cmake.bat.in @@ -26,7 +26,7 @@ set "CMAKE_GENERATOR=Visual Studio %VS_MAJOR% %VS_YEAR%" :: This line is scary, but it basically assigns the output of the command inside (` and `) :: to the variable specified after DO SET :: The equivalent in bash is PYTHON_INSTALL_DIR=`python -c ...` -FOR /F "tokens=* USEBACKQ" %%i IN (`python -c "import os;print(os.path.relpath(os.environ['SP_DIR'],os.environ['LIBRARY_PREFIX']).replace('\\','\\\\'))"`) DO SET PYTHON_INSTALL_DIR=%%i +FOR /F "tokens=* USEBACKQ" %%i IN (`python -c "import os;print(os.path.relpath(os.environ['SP_DIR'],os.environ['LIBRARY_PREFIX']).replace('\\','/'))"`) DO SET PYTHON_INSTALL_DIR=%%i cmake ^ -G "%CMAKE_GENERATOR%" ^ diff --git a/vinca/templates/bld_colcon_merge.bat.in b/vinca/templates/bld_colcon_merge.bat.in index 5624268..b795d97 100644 --- a/vinca/templates/bld_colcon_merge.bat.in +++ b/vinca/templates/bld_colcon_merge.bat.in @@ -15,7 +15,7 @@ set CXX=cl.exe :: This line is scary, but it basically assigns the output of the command inside (` and `) :: to the variable specified after DO SET :: The equivalent in bash is PYTHON_INSTALL_DIR=`python -c ...` -FOR /F "tokens=* USEBACKQ" %%i IN (`python -c "import os;print(os.path.relpath(os.environ['SP_DIR'],os.environ['LIBRARY_PREFIX']).replace('\\','\\\\'))"`) DO SET PYTHON_INSTALL_DIR=%%i +FOR /F "tokens=* USEBACKQ" %%i IN (`python -c "import os;print(os.path.relpath(os.environ['SP_DIR'],os.environ['LIBRARY_PREFIX']).replace('\\','/'))"`) DO SET PYTHON_INSTALL_DIR=%%i colcon build ^ --event-handlers console_cohesion+ ^ From 7c72438e3e2cb1d148acbe4615fde770651a5c0b Mon Sep 17 00:00:00 2001 From: Silvio Traversaro Date: Sat, 28 Dec 2024 00:26:17 +0100 Subject: [PATCH 33/57] Use Ninja generator also on Windows --- vinca/templates/bld_ament_cmake.bat.in | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/vinca/templates/bld_ament_cmake.bat.in b/vinca/templates/bld_ament_cmake.bat.in index fe65ed1..d52dd7d 100644 --- a/vinca/templates/bld_ament_cmake.bat.in +++ b/vinca/templates/bld_ament_cmake.bat.in @@ -12,10 +12,7 @@ rd /s /q build mkdir build pushd build -:: set "CMAKE_GENERATOR=Ninja" - -:: try to fix long paths issues by using default generator -set "CMAKE_GENERATOR=Visual Studio %VS_MAJOR% %VS_YEAR%" +set "CMAKE_GENERATOR=Ninja" :: PYTHON_INSTALL_DIR should be a relative path, see :: https://github.com/ament/ament_cmake/blob/2.3.2/ament_cmake_python/README.md From 45a0f262ef5e9166896eec7759e890ad241abd16 Mon Sep 17 00:00:00 2001 From: Tobias Fischer Date: Sun, 29 Dec 2024 06:05:57 +1000 Subject: [PATCH 34/57] set -eo pipefail --- vinca/templates/build_ament_cmake.sh.in | 2 ++ 1 file changed, 2 insertions(+) diff --git a/vinca/templates/build_ament_cmake.sh.in b/vinca/templates/build_ament_cmake.sh.in index 1d36512..34df1a7 100644 --- a/vinca/templates/build_ament_cmake.sh.in +++ b/vinca/templates/build_ament_cmake.sh.in @@ -1,6 +1,8 @@ # Generated by vinca http://github.com/RoboStack/vinca. # DO NOT EDIT! +set -eo pipefail + rm -rf build mkdir build cd build From 1e95c2190f0e21d6d3575ab5dc9940f3cfd1071a Mon Sep 17 00:00:00 2001 From: Tobias Fischer Date: Sun, 29 Dec 2024 06:06:29 +1000 Subject: [PATCH 35/57] set -eo pipefail --- vinca/templates/build_catkin.sh.in | 2 ++ 1 file changed, 2 insertions(+) diff --git a/vinca/templates/build_catkin.sh.in b/vinca/templates/build_catkin.sh.in index 5ec8509..7829f73 100644 --- a/vinca/templates/build_catkin.sh.in +++ b/vinca/templates/build_catkin.sh.in @@ -1,6 +1,8 @@ # Generated by vinca http://github.com/RoboStack/vinca. # DO NOT EDIT! +set -eo pipefail + CATKIN_BUILD_BINARY_PACKAGE="ON" if [ "${PKG_NAME}" == "ros-@(ros_distro)-catkin" ]; then From 59b8bacfee22b1bb738a3741d35ecad23f3b16d7 Mon Sep 17 00:00:00 2001 From: Tobias Fischer Date: Sun, 29 Dec 2024 06:06:55 +1000 Subject: [PATCH 36/57] set -eo pipefail --- vinca/templates/build_ament_python.sh.in | 2 ++ 1 file changed, 2 insertions(+) diff --git a/vinca/templates/build_ament_python.sh.in b/vinca/templates/build_ament_python.sh.in index 8a20e59..7851a12 100644 --- a/vinca/templates/build_ament_python.sh.in +++ b/vinca/templates/build_ament_python.sh.in @@ -1,6 +1,8 @@ # Generated by vinca http://github.com/RoboStack/vinca. # DO NOT EDIT! +set -eo pipefail + pushd $SRC_DIR/$PKG_NAME/src/work # If there is a setup.cfg that contains install-scripts then we should not set it here From 0b1e16d1ade9716246c1618dc0a5c2a18ee01fd9 Mon Sep 17 00:00:00 2001 From: Tobias Fischer Date: Mon, 30 Dec 2024 14:28:59 +1000 Subject: [PATCH 37/57] Add git dep --- vinca/main.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/vinca/main.py b/vinca/main.py index 6202d2e..dcf751a 100644 --- a/vinca/main.py +++ b/vinca/main.py @@ -242,18 +242,14 @@ def generate_output(pkg_shortname, vinca_conf, distro, version, all_pkgs=None): "ninja", "python", "setuptools", + "git", {"if": "unix", "then": ["patch", "make", "coreutils"]}, {"if": "win", "then": ["m2-patch"]}, {"if": "osx", "then": ["tapi"]}, {"if": "build_platform != target_platform", "then": ["pkg-config"]}, "cmake", "cython", - {"if": "build_platform != target_platform", "then": ["python"]}, - { - "if": "build_platform != target_platform", - "then": ["cross-python_${{ target_platform }}"], - }, - {"if": "build_platform != target_platform", "then": ["numpy"]}, + {"if": "build_platform != target_platform", "then": ["python", "cross-python_${{ target_platform }}", "numpy"]}, ], "host": [ {"if": "build_platform == target_platform", "then": ["pkg-config"]}, From 7e7b6be5e1bed3f41cdc44c8e88bb3126162933f Mon Sep 17 00:00:00 2001 From: Silvio Traversaro Date: Tue, 31 Dec 2024 00:06:32 +0100 Subject: [PATCH 38/57] Do not use ninja on Windows to avoid long path issues but use parallel compilation with VS cmake generator --- vinca/templates/bld_ament_cmake.bat.in | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/vinca/templates/bld_ament_cmake.bat.in b/vinca/templates/bld_ament_cmake.bat.in index d52dd7d..823afff 100644 --- a/vinca/templates/bld_ament_cmake.bat.in +++ b/vinca/templates/bld_ament_cmake.bat.in @@ -12,7 +12,11 @@ rd /s /q build mkdir build pushd build -set "CMAKE_GENERATOR=Ninja" +:: set "CMAKE_GENERATOR=Ninja" +:: We use the Visual Studio generator as a workaround for +:: problems in Ninja when using long paths, see https://github.com/RoboStack/ros-humble/pull/229#issuecomment-2564856467 +:: Once those are solved, we can switch back to use Ninja +set "CMAKE_GENERATOR=Visual Studio %VS_MAJOR% %VS_YEAR%" :: PYTHON_INSTALL_DIR should be a relative path, see :: https://github.com/ament/ament_cmake/blob/2.3.2/ament_cmake_python/README.md @@ -42,5 +46,7 @@ cmake ^ %SRC_DIR%\%PKG_NAME%\src\work if errorlevel 1 exit 1 -cmake --build . --config Release --target install +:: We explicitly pass %CPU_COUNT% to cmake --build as we are not using Ninja, +:: see the comment before setting the CMAKE_GENERATOR env variable +cmake --build . --config Release --parallel %CPU_COUNT% --target install if errorlevel 1 exit 1 From 45af742aa5fa5119b884f4bb5845084e70206091 Mon Sep 17 00:00:00 2001 From: Tobias Fischer Date: Wed, 1 Jan 2025 06:33:10 +1000 Subject: [PATCH 39/57] fix qt --- vinca/templates/build_catkin.sh.in | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/vinca/templates/build_catkin.sh.in b/vinca/templates/build_catkin.sh.in index 7829f73..5dca507 100644 --- a/vinca/templates/build_catkin.sh.in +++ b/vinca/templates/build_catkin.sh.in @@ -29,6 +29,12 @@ else OSX_DEPLOYMENT_TARGET="11.0" fi +if [[ "${CONDA_BUILD_CROSS_COMPILATION:-}" == "1" ]]; then + export QT_HOST_PATH="$BUILD_PREFIX" +else + export QT_HOST_PATH="$PREFIX" +fi + echo "USING PYTHON_EXECUTABLE=${PYTHON_EXECUTABLE}" echo "USING PKG_CONFIG_EXECUTABLE=${PKG_CONFIG_EXECUTABLE}" From df6f4a6e1d37079e2e1c7acb0806fb22119dc822 Mon Sep 17 00:00:00 2001 From: Tobias Fischer Date: Fri, 3 Jan 2025 20:40:06 +1000 Subject: [PATCH 40/57] Fixup osx --- vinca/generate_gha.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vinca/generate_gha.py b/vinca/generate_gha.py index 18549d7..4e342b4 100644 --- a/vinca/generate_gha.py +++ b/vinca/generate_gha.py @@ -305,7 +305,7 @@ def build_linux_pipeline( def build_osx_pipeline( stages, trigger_branch, - vm_imagename="macos-11", + vm_imagename="macos-13", outfile="osx.yml", azure_template=None, script=azure_osx_script, From 6cf5306d3fa5e39322e491a423a6156152539ab6 Mon Sep 17 00:00:00 2001 From: Tobias Fischer Date: Sat, 4 Jan 2025 05:58:09 +1000 Subject: [PATCH 41/57] Update generate_gha.py --- vinca/generate_gha.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vinca/generate_gha.py b/vinca/generate_gha.py index 4e342b4..670fffe 100644 --- a/vinca/generate_gha.py +++ b/vinca/generate_gha.py @@ -305,7 +305,7 @@ def build_linux_pipeline( def build_osx_pipeline( stages, trigger_branch, - vm_imagename="macos-13", + vm_imagename="macos-latest", outfile="osx.yml", azure_template=None, script=azure_osx_script, From bf74a95e094198a8956201e8acf3e373a60bef24 Mon Sep 17 00:00:00 2001 From: Tobias Fischer Date: Sat, 4 Jan 2025 06:10:44 +1000 Subject: [PATCH 42/57] Fix additional recipe calcs for .conda packages --- vinca/generate_gha.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/vinca/generate_gha.py b/vinca/generate_gha.py index 670fffe..308aa75 100644 --- a/vinca/generate_gha.py +++ b/vinca/generate_gha.py @@ -198,7 +198,9 @@ def add_additional_recipes(args): print("Checking if ", name, version, bnumber, " exists") skip = False for repo in repodatas: - for _, pkg in repo.get("packages", {}).items(): + repo_pkgs = repo.get("packages", {}) + repo_pkgs.update(repo.get("packages.conda", {})) + for _, pkg in repo_pkgs.items(): if ( pkg["name"] == name and pkg["version"] == version From 0e71b45e78c448e582179d68753c813731ae1e83 Mon Sep 17 00:00:00 2001 From: Tobias Fischer Date: Sat, 4 Jan 2025 06:13:40 +1000 Subject: [PATCH 43/57] Fix repodata to work with both .tar.gz and .conda packages --- vinca/main.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/vinca/main.py b/vinca/main.py index dcf751a..044ff81 100644 --- a/vinca/main.py +++ b/vinca/main.py @@ -940,7 +940,9 @@ def main(): if "://" in fn: selected_bn = vinca_conf.get("build_number", 0) distro = vinca_conf["ros_distro"] - for pkg_name, pkg in repodata.get("packages.conda").items(): + all_pkgs = repodata.get("packages", {}) + all_pkgs.update(repodata.get("packages.conda", {})) + for pkg_name, pkg in all_pkgs.items(): if pkg_name.startswith(f"ros-{distro}"): if pkg_name.rsplit("-", 2)[0] in additional_recipe_names: print( @@ -957,7 +959,7 @@ def main(): ] all_pkgs = repodata.get("packages", {}) all_pkgs.update(repodata.get("packages.conda", {})) - for _, pkg in repodata.get("packages").items(): + for _, pkg in all_pkgs.items(): is_built = False if selected_bn is not None: if vinca_conf.get("full_rebuild", True): From 9a2a4fea22e33c7f100628592c45cdd32908aba6 Mon Sep 17 00:00:00 2001 From: Tobias Fischer Date: Sat, 4 Jan 2025 06:37:40 +1000 Subject: [PATCH 44/57] Add libopengl-devel --- vinca/main.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/vinca/main.py b/vinca/main.py index 044ff81..608a782 100644 --- a/vinca/main.py +++ b/vinca/main.py @@ -431,9 +431,7 @@ def sortkey(k): output["requirements"]["host"] += [ { "if": "linux", - "then": [ - "libgl-devel" - ], + "then": ["libgl-devel", "libopengl-devel"], } ] From 13aa5f77822c1df837a0534b5d3954ca5473fb4c Mon Sep 17 00:00:00 2001 From: Tobias Fischer Date: Sun, 5 Jan 2025 08:59:30 +1000 Subject: [PATCH 45/57] Go back to native osx-64 builds --- vinca/generate_gha.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vinca/generate_gha.py b/vinca/generate_gha.py index 308aa75..af38d4e 100644 --- a/vinca/generate_gha.py +++ b/vinca/generate_gha.py @@ -307,7 +307,7 @@ def build_linux_pipeline( def build_osx_pipeline( stages, trigger_branch, - vm_imagename="macos-latest", + vm_imagename="macos-13", outfile="osx.yml", azure_template=None, script=azure_osx_script, From 8f53656b3885e6ae10ae263460b755a66dcb4c5a Mon Sep 17 00:00:00 2001 From: Tobias Fischer Date: Sun, 5 Jan 2025 09:17:10 +1000 Subject: [PATCH 46/57] Add qt-main in build for x-compile --- vinca/main.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/vinca/main.py b/vinca/main.py index 608a782..ad459de 100644 --- a/vinca/main.py +++ b/vinca/main.py @@ -404,6 +404,10 @@ def sortkey(k): output["requirements"]["build"] += [ {"if": "build_platform != target_platform", "then": ["pybind11"]} ] + if "qt-main" in output["requirements"]["host"]: + output["requirements"]["build"] += [ + {"sel(build_platform != target_platform)": "qt-main"} + ] # pyqt-builder + git + doxygen must be in build, not host for cross-compile pkgs_move_to_build = ["pyqt-builder", "git", "doxygen"] for pkg_move_to_build in pkgs_move_to_build: From 8295f8f9b0fade22fff8683f491f203d145d639c Mon Sep 17 00:00:00 2001 From: Tobias Fischer Date: Sun, 5 Jan 2025 09:30:21 +1000 Subject: [PATCH 47/57] Fixup --- vinca/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vinca/main.py b/vinca/main.py index ad459de..1e9798c 100644 --- a/vinca/main.py +++ b/vinca/main.py @@ -406,7 +406,7 @@ def sortkey(k): ] if "qt-main" in output["requirements"]["host"]: output["requirements"]["build"] += [ - {"sel(build_platform != target_platform)": "qt-main"} + {"if": "build_platform != target_platform", "then": ["qt-main"]} ] # pyqt-builder + git + doxygen must be in build, not host for cross-compile pkgs_move_to_build = ["pyqt-builder", "git", "doxygen"] From 37b38745c4ef3792e17824ce54892ebdeecc17cb Mon Sep 17 00:00:00 2001 From: Tobias Fischer Date: Sun, 5 Jan 2025 10:53:29 +1000 Subject: [PATCH 48/57] try AUTOMOC_EXECUTABLE for x-compile --- vinca/templates/build_catkin.sh.in | 1 + 1 file changed, 1 insertion(+) diff --git a/vinca/templates/build_catkin.sh.in b/vinca/templates/build_catkin.sh.in index 5dca507..2515c9c 100644 --- a/vinca/templates/build_catkin.sh.in +++ b/vinca/templates/build_catkin.sh.in @@ -31,6 +31,7 @@ fi if [[ "${CONDA_BUILD_CROSS_COMPILATION:-}" == "1" ]]; then export QT_HOST_PATH="$BUILD_PREFIX" + export AUTOMOC_EXECUTABLE="$BUILD_PREFIX/bin/moc" else export QT_HOST_PATH="$PREFIX" fi From ed00d8107489a820642f6cda578b09df5870a465 Mon Sep 17 00:00:00 2001 From: Tobias Fischer Date: Sun, 5 Jan 2025 18:14:50 +1000 Subject: [PATCH 49/57] Use CMAKE_AUTOMOC_EXECUTABLE --- vinca/templates/build_catkin.sh.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vinca/templates/build_catkin.sh.in b/vinca/templates/build_catkin.sh.in index 2515c9c..a48e1f8 100644 --- a/vinca/templates/build_catkin.sh.in +++ b/vinca/templates/build_catkin.sh.in @@ -31,7 +31,7 @@ fi if [[ "${CONDA_BUILD_CROSS_COMPILATION:-}" == "1" ]]; then export QT_HOST_PATH="$BUILD_PREFIX" - export AUTOMOC_EXECUTABLE="$BUILD_PREFIX/bin/moc" + export CMAKE_AUTOMOC_EXECUTABLE="$BUILD_PREFIX/bin/moc" else export QT_HOST_PATH="$PREFIX" fi From 4f0e3abbe16e52304eb6cde6b5ab72882fc80486 Mon Sep 17 00:00:00 2001 From: Tobias Fischer Date: Sun, 5 Jan 2025 20:11:56 +1000 Subject: [PATCH 50/57] Set CMAKE_AUTOMOC_EXECUTABLE in cmake call --- vinca/templates/build_catkin.sh.in | 2 ++ 1 file changed, 2 insertions(+) diff --git a/vinca/templates/build_catkin.sh.in b/vinca/templates/build_catkin.sh.in index a48e1f8..b332597 100644 --- a/vinca/templates/build_catkin.sh.in +++ b/vinca/templates/build_catkin.sh.in @@ -34,6 +34,7 @@ if [[ "${CONDA_BUILD_CROSS_COMPILATION:-}" == "1" ]]; then export CMAKE_AUTOMOC_EXECUTABLE="$BUILD_PREFIX/bin/moc" else export QT_HOST_PATH="$PREFIX" + export CMAKE_AUTOMOC_EXECUTABLE="$PREFIX/bin/moc" fi echo "USING PYTHON_EXECUTABLE=${PYTHON_EXECUTABLE}" @@ -81,6 +82,7 @@ cmake ${CMAKE_ARGS} --compile-no-warning-as-error \ -DCMAKE_INSTALL_LIBDIR=lib \ -DCMAKE_NO_SYSTEM_FROM_IMPORTED=ON \ -DCMAKE_FIND_FRAMEWORK=LAST \ + -DCMAKE_AUTOMOC_EXECUTABLE=$CMAKE_AUTOMOC_EXECUTABLE \ -DBUILD_SHARED_LIBS=ON \ -DPYTHON_EXECUTABLE=$PYTHON_EXECUTABLE \ -DPython_EXECUTABLE=$PYTHON_EXECUTABLE \ From 6049843cc4ed6f07d0e7c75a81e3161f51b467fd Mon Sep 17 00:00:00 2001 From: Tobias Fischer Date: Sun, 5 Jan 2025 20:48:46 +1000 Subject: [PATCH 51/57] fix pixi-version --- vinca/generate_gha.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vinca/generate_gha.py b/vinca/generate_gha.py index af38d4e..76ba812 100644 --- a/vinca/generate_gha.py +++ b/vinca/generate_gha.py @@ -395,7 +395,7 @@ def build_win_pipeline(stages, trigger_branch, outfile="win.yml", azure_template "name": "Setup pixi", "uses": "prefix-dev/setup-pixi@v0.8.1", "with": { - "pixi-version": "conda-forge", + "pixi-version": "v0.39.4", "cache": "true", }, }, From d3b6b0d249eb204b254a94a174255aa860e49850 Mon Sep 17 00:00:00 2001 From: Tobias Fischer Date: Tue, 7 Jan 2025 08:59:10 +1000 Subject: [PATCH 52/57] Use build_linux.sh --- vinca/azure_templates/linux.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vinca/azure_templates/linux.sh b/vinca/azure_templates/linux.sh index e96a021..1c109a5 100644 --- a/vinca/azure_templates/linux.sh +++ b/vinca/azure_templates/linux.sh @@ -1,4 +1,4 @@ export CI=azure export GIT_BRANCH=$BUILD_SOURCEBRANCHNAME export FEEDSTOCK_NAME=$(basename ${BUILD_REPOSITORY_NAME}) -.scripts/run_docker_build.sh +.scripts/build_linux.sh From 931bc388e3b43412ca4977ce77deb9ada3542cc5 Mon Sep 17 00:00:00 2001 From: Tobias Fischer Date: Tue, 7 Jan 2025 21:25:01 +1000 Subject: [PATCH 53/57] Undo set AUTOMOC as it breaks qt5/qt6 --- vinca/templates/build_catkin.sh.in | 2 -- 1 file changed, 2 deletions(-) diff --git a/vinca/templates/build_catkin.sh.in b/vinca/templates/build_catkin.sh.in index b332597..103a78a 100644 --- a/vinca/templates/build_catkin.sh.in +++ b/vinca/templates/build_catkin.sh.in @@ -31,10 +31,8 @@ fi if [[ "${CONDA_BUILD_CROSS_COMPILATION:-}" == "1" ]]; then export QT_HOST_PATH="$BUILD_PREFIX" - export CMAKE_AUTOMOC_EXECUTABLE="$BUILD_PREFIX/bin/moc" else export QT_HOST_PATH="$PREFIX" - export CMAKE_AUTOMOC_EXECUTABLE="$PREFIX/bin/moc" fi echo "USING PYTHON_EXECUTABLE=${PYTHON_EXECUTABLE}" From 71fd33aafd53729f6140811fa2535b34f929e034 Mon Sep 17 00:00:00 2001 From: Tobias Fischer Date: Wed, 8 Jan 2025 10:02:27 +1000 Subject: [PATCH 54/57] Fix ros-noetic-euslisp which only works with make --- vinca/templates/build_catkin.sh.in | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/vinca/templates/build_catkin.sh.in b/vinca/templates/build_catkin.sh.in index 103a78a..099e604 100644 --- a/vinca/templates/build_catkin.sh.in +++ b/vinca/templates/build_catkin.sh.in @@ -73,6 +73,12 @@ fi export SKIP_TESTING=@(skip_testing) +if [ "${PKG_NAME}" == "ros-noetic-euslisp" ]; then + GENERATOR="Unix Makefiles" +else + GENERATOR="Ninja" +fi + cmake ${CMAKE_ARGS} --compile-no-warning-as-error \ -DCMAKE_INSTALL_PREFIX=$PREFIX \ -DCMAKE_PREFIX_PATH=$PREFIX \ @@ -92,7 +98,7 @@ cmake ${CMAKE_ARGS} --compile-no-warning-as-error \ -DCATKIN_SKIP_TESTING=$SKIP_TESTING \ -DCATKIN_BUILD_BINARY_PACKAGE=$CATKIN_BUILD_BINARY_PACKAGE \ -DCMAKE_OSX_DEPLOYMENT_TARGET=$OSX_DEPLOYMENT_TARGET \ - -G "Ninja" \ + -G "$GENERATOR" \ $SRC_DIR/$PKG_NAME/src/work cmake --build . --config Release --target all From c47948cdec7d20d480342161751a34e92a3b0eaf Mon Sep 17 00:00:00 2001 From: Tobias Fischer Date: Wed, 8 Jan 2025 15:34:37 +1000 Subject: [PATCH 55/57] Fix more packages --- vinca/templates/build_catkin.sh.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vinca/templates/build_catkin.sh.in b/vinca/templates/build_catkin.sh.in index 099e604..9832f8a 100644 --- a/vinca/templates/build_catkin.sh.in +++ b/vinca/templates/build_catkin.sh.in @@ -73,7 +73,7 @@ fi export SKIP_TESTING=@(skip_testing) -if [ "${PKG_NAME}" == "ros-noetic-euslisp" ]; then +if [ "${PKG_NAME}" == "ros-noetic-euslisp" ] || [ "${PKG_NAME}" = "ros-noetic-jskeus" ] || [ "${PKG_NAME}" = "ros-noetic-roseus" ]; then GENERATOR="Unix Makefiles" else GENERATOR="Ninja" From 1ecdea3a6bb7770a218a0b80a5e973383e1eb354 Mon Sep 17 00:00:00 2001 From: Tobias Fischer Date: Fri, 10 Jan 2025 19:48:55 +1000 Subject: [PATCH 56/57] Delete .drone.yml --- .drone.yml | 66 ------------------------------------------------------ 1 file changed, 66 deletions(-) delete mode 100644 .drone.yml diff --git a/.drone.yml b/.drone.yml deleted file mode 100644 index e872e28..0000000 --- a/.drone.yml +++ /dev/null @@ -1,66 +0,0 @@ ---- -kind: pipeline -type: docker -name: greeting - -steps: -- name: vinca install - image: condaforge/linux-anvil-cos7-x86_64 - environment: - ANACONDA_API_TOKEN: - from_secret: ANACONDA_UPLOAD - QUETZ_URL: - from_secret: QUETZ_URL - QUETZ_API_KEY: - from_secret: QUETZ_API_KEY - commands: - - export FEEDSTOCK_ROOT="$DRONE_WORKSPACE" - - export ANACONDA_API_TOKEN="$ANACONDA_API_TOKEN" - - export QUETZ_URL="$QUETZ_URL" - - export QUETZ_API_KEY="$QUETZ_API_KEY" - - export RECIPE_ROOT="$FEEDSTOCK_ROOT/recipe" - - export CI=drone - - export GIT_BRANCH="$DRONE_BRANCH" - - sed -i '$ichown -R conda:conda "$FEEDSTOCK_ROOT"' /opt/docker/bin/entrypoint - - /opt/docker/bin/entrypoint $FEEDSTOCK_ROOT/.scripts/build_linux.sh - trigger: - branch: - - boa - event: - exclude: - - pull_request ---- -kind: pipeline -type: exec -name: osxbuilder - -platform: - os: darwin - -steps: -- name: vinca install - environment: - ANACONDA_API_TOKEN: - from_secret: ANACONDA_UPLOAD - QUETZ_URL: - from_secret: QUETZ_URL - QUETZ_API_KEY: - from_secret: QUETZ_API_KEY - commands: - - export FEEDSTOCK_ROOT="$DRONE_WORKSPACE" - - export ANACONDA_API_TOKEN="$ANACONDA_API_TOKEN" - - export QUETZ_URL="$QUETZ_URL" - - export QUETZ_API_KEY="$QUETZ_API_KEY" - - export RECIPE_ROOT="$FEEDSTOCK_ROOT/recipe" - - export CI=drone - - export GIT_BRANCH="$DRONE_BRANCH" - - echo "Whats up party people?" - - $FEEDSTOCK_ROOT/.scripts/build_osx.sh - # - sed -i '$ichown -R conda:conda "$FEEDSTOCK_ROOT"' /opt/docker/bin/entrypoint - # - /opt/docker/bin/entrypoint $FEEDSTOCK_ROOT/.scripts/build_steps.sh - trigger: - branch: - - boa - event: - exclude: - - pull_request From 0a288818acc24b060c8fe05daa2fbdfe2cddcf8c Mon Sep 17 00:00:00 2001 From: Tobias Fischer Date: Fri, 10 Jan 2025 19:49:45 +1000 Subject: [PATCH 57/57] Bump version --- vinca/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vinca/__init__.py b/vinca/__init__.py index 3b93d0b..81f0fde 100644 --- a/vinca/__init__.py +++ b/vinca/__init__.py @@ -1 +1 @@ -__version__ = "0.0.2" +__version__ = "0.0.4"