From f17604915afa15e0ddbe6a1769a91bb51aaea3e7 Mon Sep 17 00:00:00 2001 From: Maximilian Huber Date: Tue, 26 Oct 2021 16:30:30 +0200 Subject: [PATCH 1/4] wip Signed-off-by: Maximilian Huber --- tern/formats/opossumui/generator.py | 74 +++++++++++++++++++++++------ 1 file changed, 60 insertions(+), 14 deletions(-) diff --git a/tern/formats/opossumui/generator.py b/tern/formats/opossumui/generator.py index bca4f999..77259f87 100644 --- a/tern/formats/opossumui/generator.py +++ b/tern/formats/opossumui/generator.py @@ -8,6 +8,8 @@ import json import logging +from typing import Any, Dict, List + from tern.formats.spdx import spdx_common from tern.utils.general import get_git_rev_or_version from tern.utils import constants @@ -19,6 +21,50 @@ logger = logging.getLogger(constants.logger_name) +def _calculate_file_tree_from_paths(resources: List[str]) -> Dict[str, Any]: + filetree: Dict[str, Any] = {} + + for resource in resources: + if resource != "/": + path_elements: List[str] = list(filter(None, resource.split("/"))) + + _add_resource_to_tree( + target_filetree=filetree, + path_elements=path_elements, + resource_type=resource[-1] == "/" + ) + return filetree + + +def _add_resource_to_tree( + target_filetree: Dict[ + str, Any + ], + path_elements: List[str], + is_file: bool, +) -> None: + element_max_index = len(path_elements) - 1 + + for index, path_element in enumerate(path_elements): + if target_filetree.get(path_element) is None: + target_filetree[path_element] = ( + 1 + if _is_path_element_file_name( + index=index, + element_max_index=element_max_index, + is_file=is_file, + ) + else {} + ) + target_filetree = target_filetree[path_element] + + +def _is_path_element_file_name( + index: int, element_max_index: int, is_file: bool +) -> bool: + return (index == element_max_index) and is_file + + def get_resources(image_obj): '''Packages in each layer will be mapped to: /Layer_00x/Packages/pkg_source/pkg_name''' @@ -36,20 +82,21 @@ def get_external_attrs(image_obj): folder. The key is the uuid of the package and the value is a dictionary of metadata''' external_attrs = {} - image_uuids = {} + resources_to_attrs = {} + attr_breakpoints = {} for layer in image_obj.layers: for pkg in layer.packages: pkg_uuid = spdx_common.get_uuid() - if layer.layer_index in image_uuids.keys(): - try: - # See if source exists in layer - image_uuids[layer.layer_index][pkg.source].append(pkg_uuid) - except KeyError: - # If not, add the new source to existing layer - image_uuids[layer.layer_index][pkg.source] = [pkg_uuid] + + path = "/Layer_{}/Packages/{}/{}".format('%03d' % layer.layer_index, pkg.source, pkg.name) + attr_breakpoints.add("/Layer_{}/Packages/{}/".format('%03d' % layer.layer_index, pkg.source)) + attr_breakpoints.add("/Layer_{}/Packages/".format('%03d' % layer.layer_index)) + + if path in resources_to_attrs.keys(): + resources_to_attrs[path].append(pkg_uuid) else: - # add new layer and source - image_uuids[layer.layer_index] = {pkg.source: [pkg_uuid]} + resources_to_attrs[path] = [pkg_uuid] + pkg_comment = '' if pkg.origins.origins: for notice_origin in pkg.origins.origins: @@ -70,7 +117,7 @@ def get_external_attrs(image_obj): "licenseName": pkg_license if pkg_license else "NONE", "copyright": pkg.copyright if pkg.copyright else "NONE" } - return external_attrs, image_uuids + return external_attrs, resources_to_attrs def get_resource_attrs(uuids): @@ -113,10 +160,9 @@ def get_document_dict(image_obj): } } - docu_dict["resources"] = get_resources(image_obj) - docu_dict["externalAttributions"], uuids = get_external_attrs(image_obj) - docu_dict["resourcesToAttributions"] = get_resource_attrs(uuids) + docu_dict["externalAttributions"], docu_dict["resourcesToAttributions"] = get_external_attrs(image_obj) docu_dict["attributionBreakpoints"] = get_attr_breakpoints(image_obj) + docu_dict["resources"] = _calculate_file_tree_from_paths(list(docu_dict["resourcesToAttributions"].keys()).append(docu_dict["attributionBreakpoints"])) return docu_dict From baa98b0252cfd3e59be43257aa0d366d40b120aa Mon Sep 17 00:00:00 2001 From: Maximilian Huber Date: Tue, 26 Oct 2021 17:10:22 +0200 Subject: [PATCH 2/4] wip Signed-off-by: Maximilian Huber --- tern/classes/image_layer.py | 2 +- tern/formats/opossumui/generator.py | 83 ++++++++++++----------------- 2 files changed, 36 insertions(+), 49 deletions(-) diff --git a/tern/classes/image_layer.py b/tern/classes/image_layer.py index 3f5728a3..7b89d41a 100644 --- a/tern/classes/image_layer.py +++ b/tern/classes/image_layer.py @@ -29,9 +29,9 @@ class ImageLayer: Based on how container image layers are created, this is usually the last layer of the image that was imported import_str: The string from a build tool (like a Dockerfile) that + created this layer by importing it from another image layer_index: The index position of the layer in relationship to the other layers in the image. The base OS would be layer 1. - created this layer by importing it from another image files_analyzed: whether the files in this layer are analyzed or not analyzed_output: the result of the file analysis files: a list of files included in the image layer diff --git a/tern/formats/opossumui/generator.py b/tern/formats/opossumui/generator.py index 77259f87..874c4fbe 100644 --- a/tern/formats/opossumui/generator.py +++ b/tern/formats/opossumui/generator.py @@ -31,7 +31,7 @@ def _calculate_file_tree_from_paths(resources: List[str]) -> Dict[str, Any]: _add_resource_to_tree( target_filetree=filetree, path_elements=path_elements, - resource_type=resource[-1] == "/" + is_file=resource[-1] == "/" ) return filetree @@ -65,37 +65,49 @@ def _is_path_element_file_name( return (index == element_max_index) and is_file -def get_resources(image_obj): - '''Packages in each layer will be mapped to: - /Layer_00x/Packages/pkg_source/pkg_name''' - resources = {} - for layer in image_obj.layers: - resources["Layer_{}".format('%03d' % layer.layer_index)] = \ - {"Packages": {pkg.source: { - p.name: 1 for p in layer.packages if p.source == pkg.source} - for pkg in layer.packages}} - return resources - - def get_external_attrs(image_obj): '''Create a dict which contains attribution information about a file or folder. The key is the uuid of the package and the value is a dictionary of metadata''' external_attrs = {} resources_to_attrs = {} - attr_breakpoints = {} + attr_breakpoints = set() for layer in image_obj.layers: + layer_uuid = spdx_common.get_uuid() + + layer_path = f"/{'%03d' % layer.layer_index}" + + if f"{pkg_path}/" in resources_to_attrs.keys(): + resources_to_attrs[f"{pkg_path}/"].append(layer_uuid) + else: + resources_to_attrs[f"{pkg_path}/"] = [layer_uuid] + + external_attrs[layer_uuid] = { + "source": { + "name": "Tern:Layer", + "documentConfidence": int(70.0) + }, + "comment": str(layer) # TODO: should be the command that generated the layer + } + for pkg in layer.packages: pkg_uuid = spdx_common.get_uuid() - path = "/Layer_{}/Packages/{}/{}".format('%03d' % layer.layer_index, pkg.source, pkg.name) - attr_breakpoints.add("/Layer_{}/Packages/{}/".format('%03d' % layer.layer_index, pkg.source)) - attr_breakpoints.add("/Layer_{}/Packages/".format('%03d' % layer.layer_index)) + pkg_path = f"{layer_path}/Packages/{pkg.source}/{pkg.name}/" + attr_breakpoints.add(f"{layer_path}/Packages/{pkg.source}/") + attr_breakpoints.add(f"{layer_path}/Packages/") - if path in resources_to_attrs.keys(): - resources_to_attrs[path].append(pkg_uuid) + if pkg_path in resources_to_attrs.keys(): + resources_to_attrs[pkg_path].append(pkg_uuid) else: - resources_to_attrs[path] = [pkg_uuid] + resources_to_attrs[pkg_path] = [pkg_uuid] + + for file_in_pkg in pkg.files: + absolute_file_in_pkg = f"{layer_path}/{file_in_pkg}" + if absolute_file_in_pkg in resources_to_attrs.keys(): + resources_to_attrs[absolute_file_in_pkg].append(pkg_uuid) + else: + resources_to_attrs[absolute_file_in_pkg] = [pkg_uuid] pkg_comment = '' if pkg.origins.origins: @@ -107,7 +119,7 @@ def get_external_attrs(image_obj): ''.join(pkg.pkg_licenses) external_attrs[pkg_uuid] = { "source": { - "name": pkg.source, + "name": f"Tern:{pkg.source}", "documentConfidence": int(70.0) }, "comment": pkg_comment, @@ -117,31 +129,7 @@ def get_external_attrs(image_obj): "licenseName": pkg_license if pkg_license else "NONE", "copyright": pkg.copyright if pkg.copyright else "NONE" } - return external_attrs, resources_to_attrs - - -def get_resource_attrs(uuids): - '''Return the dictionary that maps a [package] path in the resource tree - to a list of externalAttribution uuid string(s): - {"/path/to/resource/": []}''' - resource_attrs = {} - for layer, layer_uuids in uuids.items(): - for source, pkg_uuids in layer_uuids.items(): - resource_attrs["/Layer_{}/Packages/{}/".format( - '%03d' % layer, source)] = pkg_uuids - return resource_attrs - - -def get_attr_breakpoints(image_obj): - '''We list pacakges in each layer under the Layer_00x/Packages/source - directory but that is not the literal path. Hence, we add - Layer_00x/Packages/ for each layer as an attribution breakpoint. - ''' - attr_breakpoints = [] - for layer in image_obj.layers: - attr_breakpoints.append("/Layer_{}/Packages/".format( - '%03d' % layer.layer_index)) - return attr_breakpoints + return external_attrs, resources_to_attrs, attr_breakpoints def get_document_dict(image_obj): @@ -160,8 +148,7 @@ def get_document_dict(image_obj): } } - docu_dict["externalAttributions"], docu_dict["resourcesToAttributions"] = get_external_attrs(image_obj) - docu_dict["attributionBreakpoints"] = get_attr_breakpoints(image_obj) + docu_dict["externalAttributions"], docu_dict["resourcesToAttributions"], docu_dict["attributionBreakpoints"] = get_external_attrs(image_obj) docu_dict["resources"] = _calculate_file_tree_from_paths(list(docu_dict["resourcesToAttributions"].keys()).append(docu_dict["attributionBreakpoints"])) return docu_dict From e3f89f9ce9fe91166fadfec46bacdc5286550f5c Mon Sep 17 00:00:00 2001 From: Maximilian Huber Date: Tue, 26 Oct 2021 17:19:44 +0200 Subject: [PATCH 3/4] wip Signed-off-by: Maximilian Huber --- tern/formats/opossumui/generator.py | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/tern/formats/opossumui/generator.py b/tern/formats/opossumui/generator.py index 874c4fbe..9b3bd18e 100644 --- a/tern/formats/opossumui/generator.py +++ b/tern/formats/opossumui/generator.py @@ -48,23 +48,11 @@ def _add_resource_to_tree( for index, path_element in enumerate(path_elements): if target_filetree.get(path_element) is None: target_filetree[path_element] = ( - 1 - if _is_path_element_file_name( - index=index, - element_max_index=element_max_index, - is_file=is_file, - ) - else {} + 1 if (index == element_max_index) and is_file else {} ) target_filetree = target_filetree[path_element] -def _is_path_element_file_name( - index: int, element_max_index: int, is_file: bool -) -> bool: - return (index == element_max_index) and is_file - - def get_external_attrs(image_obj): '''Create a dict which contains attribution information about a file or folder. The key is the uuid of the package and the value is a dictionary From 400bcbf2f0005fb7c42b4ee870a8fdd1d58d1d59 Mon Sep 17 00:00:00 2001 From: Maximilian Huber Date: Tue, 26 Oct 2021 17:27:15 +0200 Subject: [PATCH 4/4] wip Signed-off-by: Maximilian Huber --- tern/formats/opossumui/generator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tern/formats/opossumui/generator.py b/tern/formats/opossumui/generator.py index 9b3bd18e..e9fac2a6 100644 --- a/tern/formats/opossumui/generator.py +++ b/tern/formats/opossumui/generator.py @@ -75,7 +75,7 @@ def get_external_attrs(image_obj): "name": "Tern:Layer", "documentConfidence": int(70.0) }, - "comment": str(layer) # TODO: should be the command that generated the layer + "comment": layer.created_by } for pkg in layer.packages: