Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Sbom command and plugin support #17203

Open
wants to merge 29 commits into
base: develop2
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
29 commits
Select commit Hold shift + click to select a range
c2bcece
initial
ErniGH Oct 23, 2024
7d10e59
use sbom in install
ErniGH Oct 23, 2024
005fb39
Revert "initial"
ErniGH Oct 24, 2024
94dd274
Revert "use sbom in install"
ErniGH Oct 24, 2024
e4f9591
wip
ErniGH Oct 24, 2024
bc25a61
wip
ErniGH Oct 24, 2024
2511245
wip
ErniGH Oct 24, 2024
036fb82
sbom generator
ErniGH Oct 25, 2024
7e4cf57
review suggestions
AbrilRBS Oct 25, 2024
00c5781
Cleanup
AbrilRBS Oct 25, 2024
b7f90ed
Update conans/client/graph/sbom.py
AbrilRBS Oct 25, 2024
51f1189
rename and move sbom method, move spdx script to different file and
ErniGH Oct 31, 2024
73ea04e
add subgraph implementation and test
ErniGH Nov 6, 2024
e1833ae
use new subgraph api, update generator, spdx and test
ErniGH Nov 6, 2024
cffeffb
sbom 0.1
ErniGH Nov 28, 2024
61cfe65
support cyclonedx
ErniGH Dec 4, 2024
2dc6c44
Merge branch 'develop2' into erni/sbom-implementation
ErniGH Dec 10, 2024
548478b
solve conflits
ErniGH Dec 10, 2024
a6d359b
Update conan/api/subapi/install.py
ErniGH Dec 10, 2024
89264b3
skip test sbom, calculate licenses, small fix
ErniGH Dec 10, 2024
f32ae94
fix f string
ErniGH Dec 10, 2024
aedc776
Add test for skipped binary
AbrilRBS Dec 10, 2024
56f4a8a
fix
ErniGH Dec 10, 2024
b060f80
"special root node" tests and fixes
ErniGH Dec 12, 2024
d1a27a4
fix breakpoints
ErniGH Dec 13, 2024
215a3e8
fix
ErniGH Dec 13, 2024
3b7125a
fix tests: Detect the new file in metadata when upload files
ErniGH Dec 13, 2024
dce962e
fix meradata logs and warn_tag test
ErniGH Dec 16, 2024
0629aa8
move sbom test from integrations to funcional
ErniGH Dec 16, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 22 additions & 0 deletions conan/internal/api/install/generators.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
import traceback
import importlib

from conan.api.output import ConanOutput
from conan.internal.cache.home_paths import HomePaths
from conans.client.subsystems import deduce_subsystem, subsystem_path
from conan.internal.errors import conanfile_exception_formatter
Expand Down Expand Up @@ -141,6 +142,8 @@ def write_generators(conanfile, hook_manager, home_folder, envs_generation=None)

_generate_aggregated_env(conanfile)

generate_graph_manifests(conanfile, home_folder)

hook_manager.execute("post_generate", conanfile=conanfile)


Expand All @@ -157,6 +160,25 @@ def _receive_conf(conanfile):
conanfile.conf.compose_conf(build_require.conf_info)


def generate_graph_manifests(conanfile, home_folder):
from conans.client.loader import load_python_file
mkdir(conanfile.package_metadata_folder)
sub_graph = conanfile.subgraph
sbom_plugin_path = HomePaths(home_folder).sbom_manifest_plugin_path
if os.path.exists(sbom_plugin_path):
mod, _ = load_python_file(sbom_plugin_path)

if not hasattr(mod, "generate_sbom"):
raise ConanException(
f"SBOM manifest plugin does not have a 'generate_sbom' method")
if not callable(mod.generate_sbom):
raise ConanException(
f"SBOM manifest plugin 'generate_sbom' is not a function")

conanfile.output.info(f"generating sbom")
# TODO think if this is conanfile or conanfile._conan_node
return mod.generate_sbom(sub_graph)

def _receive_generators(conanfile):
""" Collect generators_info from the immediate build_requires"""
for build_req in conanfile.dependencies.direct_build.values():
Expand Down
4 changes: 4 additions & 0 deletions conan/internal/cache/home_paths.py
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,10 @@ def auth_source_plugin_path(self):
def sign_plugin_path(self):
return os.path.join(self._home, _EXTENSIONS_FOLDER, _PLUGINS, "sign", "sign.py")

@property
def sbom_manifest_plugin_path(self):
return os.path.join(self._home, _EXTENSIONS_FOLDER, _PLUGINS, "sbom.py")

@property
def remotes_path(self):
return os.path.join(self._home, "remotes.json")
Expand Down
2 changes: 2 additions & 0 deletions conan/internal/methods.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
from conans.model.pkg_type import PackageType
from conans.model.requires import BuildRequirements, TestRequirements, ToolRequirements
from conans.util.files import mkdir, chdir, save
from conan.internal.api.install.generators import generate_graph_manifests


def run_source_method(conanfile, hook_manager):
Expand Down Expand Up @@ -63,6 +64,7 @@ def run_package_method(conanfile, package_id, hook_manager, ref):
with conanfile_remove_attr(conanfile, ['info'], "package"):
conanfile.package()
hook_manager.execute("post_package", conanfile=conanfile)
generate_graph_manifests(conanfile, conanfile._conan_helpers.home_folder)

save(os.path.join(conanfile.package_folder, CONANINFO), conanfile.info.dumps())
manifest = FileTreeManifest.create(conanfile.package_folder)
Expand Down
17 changes: 17 additions & 0 deletions conans/client/graph/graph.py
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,23 @@ def __init__(self, ref, conanfile, context, recipe=None, path=None, test=False):
self.replaced_requires = {} # To track the replaced requires for self.dependencies[old-ref]
self.skipped_build_requires = False

def subgraph(self):
nodes = [self]
opened = [self]
while opened:
new_opened = []
for o in opened:
for n in o.neighbors():
if n not in nodes:
nodes.append(n)
if n not in opened:
new_opened.append(n)
opened = new_opened

graph = DepsGraph()
graph.nodes = nodes
return graph

def __lt__(self, other):
"""
@type other: Node
Expand Down
7 changes: 7 additions & 0 deletions conans/client/graph/sbom.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
from conans.client.graph.spdx import spdx_json_generator
from conan.internal.cache.home_paths import HomePaths

def migrate_sbom_file(cache_folder):
from conans.client.migrations import update_file
sbom_path = HomePaths(cache_folder).sbom_manifest_plugin_path
update_file(sbom_path, spdx_json_generator)
239 changes: 239 additions & 0 deletions conans/client/graph/spdx.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,239 @@
# TODO RENAME THIS FILE

spdx_json_generator = """

def generate_sbom(graph, **kwargs):
cyclonedx_1_4(graph, **kwargs)
#spdx_sbom(graph, **kwargs)

def cyclonedx_1_4(graph, **kwargs):
import json
import os
import uuid
import time
from datetime import datetime, timezone
from conan import conan_version
from conan.errors import ConanException
from conan.api.subapi.graph import CONTEXT_BUILD
from conan.api.output import ConanOutput

has_special_root_node = not (getattr(graph.root.ref, "name", False) and getattr(graph.root.ref, "version", False) and getattr(graph.root.ref, "revision", False))
special_id = str(uuid.uuid4())
special_name = graph.root.conanfile.display_name.replace(".", "-").replace(" ", "_").replace("/", "-").upper()

components = [node for node in graph.nodes]
if has_special_root_node:
components = components[1:]

dependencies = []
if has_special_root_node:
deps = {"ref": special_id}
deps["dependsOn"] = [f"pkg:conan/{d.dst.name}@{d.dst.ref.version}?rref={d.dst.ref.revision}" for d in graph.root.dependencies]
dependencies.append(deps)
for c in components:
deps = {"ref": f"pkg:conan/{c.name}@{c.ref.version}?rref={c.ref.revision}"}
dependsOn = [f"pkg:conan/{d.dst.name}@{d.dst.ref.version}?rref={d.dst.ref.revision}" for d in c.dependencies]
if dependsOn:
deps["dependsOn"] = dependsOn
dependencies.append(deps)

def _calculate_licenses(component):
if isinstance(component.conanfile.license, str): # Just one license
return [{"license": {
"id": component.conanfile.license
}}]
return [{"license": {
"id": license
}} for license in c.conanfile.license]

sbom_cyclonedx_1_4 = {
**({"components": [{
"author": "Conan",
"bom-ref": special_id if has_special_root_node else f"pkg:conan/{c.name}@{c.ref.version}?rref={c.ref.revision}",
"description": c.conanfile.description,
**({"externalReferences": [{
"type": "website",
"url": c.conanfile.homepage
}]} if c.conanfile.homepage else {}),
**({"licenses": _calculate_licenses(c)} if c.conanfile.license else {}),
"name": c.name,
"fpurl": f"pkg:conan/{c.name}@{c.ref.version}?rref={c.ref.revision}",
"type": "library",
"version": str(c.ref.version),
} for c in components]} if components else {}),
**({"dependencies": dependencies} if dependencies else {}),
"metadata": {
"component": {
"author": "Conan",
"bom-ref": special_id if has_special_root_node else f"pkg:conan/{c.name}@{c.ref.version}?rref={c.ref.revision}",
"name": graph.root.conanfile.display_name,
"type": "library"
},
"timestamp": f"{datetime.fromtimestamp(time.time(), tz=timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ')}",
"tools": [{
"externalReferences": [{
"type": "website",
"url": "https://github.com/conan-io/conan"
}],
"name": "Conan-io"
}],
},
"serialNumber": f"urn:uuid:{uuid.uuid4()}",
"bomFormat": "CycloneDX",
"specVersion": "1.4",
"version": 1,
}
try:
metadata_folder = graph.root.conanfile.package_metadata_folder
file_name = f"{special_name}-cyclonedx.json" if has_special_root_node else f"{graph.root.name}-{graph.root.ref.version}-cyclonedx.json"
with open(os.path.join(metadata_folder, file_name), 'w') as f:
json.dump(sbom_cyclonedx_1_4, f, indent=4)
ConanOutput().success(f"CYCLONEDX CREATED - {graph.root.conanfile.package_metadata_folder}")
except Exception as e:
ConanException("error generating CYCLONEDX file")

def spdx_sbom(graph, **kwargs):
import os
import time
import json
import pathlib
from glob import glob
from datetime import datetime, timezone
from conan import conan_version
from conan.errors import ConanException
from conan.api.output import ConanOutput

name = graph.root.name if graph.root.name else "CLI"
version = "SPDX-2.2"
date = datetime.fromtimestamp(time.time(), tz=timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ')
ErniGH marked this conversation as resolved.
Show resolved Hide resolved
packages = []
files = []
relationships = []

# --- Root node ---
if graph.root.recipe != "Cli":
conan_data = graph.root.conanfile.conan_data
url_location = conan_data.get("sources", {}).get(graph.root.conanfile.version, {}).get("url", {}) if conan_data else None
checksum = conan_data.get("sources", {}).get(graph.root.conanfile.version, {}).get("sha256", {}) if conan_data else None
packages.extend([
{
"name": graph.root.ref.name,
"SPDXID": f"SPDXRef-{graph.root.ref}",
"version": str(graph.root.ref.version),
"downloadLocation": graph.root.conanfile.url or "NOASSERTION",
"homePage": graph.root.conanfile.homepage or "NOASSERTION",
"licenseConcluded": graph.root.conanfile.license or "NOASSERTION",
"licenseDeclared": "NOASSERTION",
"copyrightText": "NOASSERTION",
"description": graph.root.conanfile.description or "NOASSERTION",
"comment": f"This is the {graph.root.ref.name} package in the remote" # TODO It could be a local package

},
{
"name": f"{graph.root.pref} binary",
"SPDXID": f"SPDXRef-binary-{graph.root.ref}",
"downloadLocation": graph.root.remote.url if graph.root.remote else "NONE",
"licenseConcluded": graph.root.conanfile.license or "NOASSERTION",
"licenseDeclared": "NOASSERTION",
"copyrightText": "NOASSERTION",
"comment": f"This is the {graph.root.ref} binary generated by conan"
},
{
"name": f"{graph.root.ref.name} upstream",
"SPDXID": f"SPDXRef-resource-{graph.root.ref}",
"downloadLocation": url_location or "NONE",
**({"checksum": {
"algorithm": "SHA256",
"checksumValue": checksum
}} if checksum else {}),
"licenseConcluded": graph.root.conanfile.license or "NOASSERTION",
"licenseDeclared": "NOASSERTION",
"copyrightText": "NOASSERTION",
"comment": f"This is the {graph.root.ref.name} release file"
}])

relationships.extend([{
"spdxElementId": f"SPDXRef-binary-{graph.root.ref}",
"relationshipType": "DEPENDS_ON",
"relatedSpdxElement": f"SPDXRef-binary-{d.dst.ref}",
}for d in graph.root.dependencies])

relationships.append({
"spdxElementId": f"SPDXRef-{graph.root.ref}",
"relationshipType": "GENERATES",
"relatedSpdxElement": f"SPDXRef-binary-{graph.root.ref}",
})

exported_path = graph.root.conanfile.recipe_folder # /e folder
external_files = [f for f in glob(os.path.join(exported_path, "**", "*"), recursive=True) if not f.endswith('/')] if exported_path else []

try:
with open(os.path.join(graph.root.conanfile.recipe_folder, "conanmanifest.txt")) as conanmanifest:
external_files.extend([os.path.join(exported_path[:-1], *line.split(" ")[0][:-1].split("/")) for line in conanmanifest.readlines()[2:]])
except Exception:
pass

for i, file_name in enumerate(external_files):
checksum = None
files.append(
{
"fileName": file_name,
"SPDXID": f"SPDXRef-file-{graph.root.ref}-{i}",
**({"checksums":{
"algorithm": "SHA256",
"checksumValues": checksum,
}} if checksum else {}),
"licenseConcluded": "NOASSERTION",
"copyrightText": "NOASSERTION"
}
)
relationships.append({
"spdxElementId": f"SPDXRef-{graph.root.ref}",
"relationshipType": "CONTAINS",
"relatedSpdxElement": f"SPDXRef-file-{graph.root.ref}-{i}",
})

# --- Just the binaries for dependencies ---
for node in graph.nodes[1:]:
conan_data = node.conanfile.conan_data
url_location = conan_data.get("sources", {}).get(node.conanfile.version, {}).get("url", {}) if conan_data else None
checksum = conan_data.get("sources", {}).get(node.conanfile.version, {}).get("sha256", {}) if conan_data else None
packages.extend([
{
"name": f"{node.pref} binary",
"SPDXID": f"SPDXRef-binary-{node.ref}",
"downloadLocation": node.remote.url if node.remote else "NONE",
"licenseConcluded": node.conanfile.license or "NOASSERTION",
"licenseDeclared": "NOASSERTION",
"copyrightText": "NOASSERTION",
"comment": f"This is the {node.ref} binary generated by conan"
}])

relationships.extend([{
"spdxElementId": f"SPDXRef-binary-{node.ref}",
"relationshipType": "DEPENDS_ON",
"relatedSpdxElement": f"SPDXRef-binary-{d.dst.ref}",
}for d in node.dependencies])


# https://spdx.github.io/spdx-spec/v2.2.2/package-information/
data = {
"SPDXVersion": "SPDX-2.2",
"dataLicense": "CC0-1.0",
"SPDXID": "SPDXRef-DOCUMENT",
"documentName": f"{name}-{version}",
"documentNamespace": f"http://spdx.org/spdxdocs/{name}-{version}-{date}", # the date or hash to make it unique
"creator": f"Tool: Conan-{conan_version}",
"created": date, #YYYY-MM-DDThh:mm:ssZ
"packages": packages,
**({"files": files} if files else {}),
**({"relationships": relationships} if relationships else {}),
}
try:
metadata_folder = graph.root.conanfile.package_metadata_folder
with open(os.path.join(metadata_folder, f"{name}-{graph.root.ref.version if graph.root.ref else 'local'}.spdx.json"), 'w') as f:
json.dump(data, f, indent=4)
ConanOutput().success(f"SPDX CREATED - {graph.root.conanfile.package_metadata_folder}")
except Exception as e:
ConanException("error generating spdx file")
"""
3 changes: 3 additions & 0 deletions conans/client/migrations.py
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,9 @@ def _apply_migrations(self, old_version):
# Update profile plugin
from conan.internal.api.profile.profile_loader import migrate_profile_plugin
migrate_profile_plugin(self.cache_folder)
# Update sbom manifest plugins
from conans.client.graph.sbom import migrate_sbom_file
migrate_sbom_file(self.cache_folder)

if old_version and old_version < "2.0.14-":
_migrate_pkg_db_lru(self.cache_folder, old_version)
Expand Down
4 changes: 4 additions & 0 deletions conans/model/conan_file.py
Original file line number Diff line number Diff line change
Expand Up @@ -187,6 +187,10 @@ def output(self):
def context(self):
return self._conan_node.context

@property
def subgraph(self):
return self._conan_node.subgraph()

@property
def dependencies(self):
# Caching it, this object is requested many times
Expand Down
Empty file.
Loading
Loading