add SPDX versioning

This adds new options to the -f parameter like "spdxjson22", "spdxjson23", "spdxrdf22" etc.
Also extracts common code from the generators of all the different formats.

Signed-off-by: Armin Tänzer <armin.taenzer@tngtech.com>

Author: armintaenzertng

setup.cfg

@@ -52,11 +52,16 @@ tern.formats =
     yaml = tern.formats.yaml.generator:YAML
     html = tern.formats.html.generator:HTML
     cyclonedxjson = tern.formats.cyclonedx.cyclonedxjson.generator:CycloneDXJSON
-    spdxjson_new = tern.formats.spdx_new.spdxjson.generator:SpdxJSON
-    spdxyaml_new = tern.formats.spdx_new.spdxyaml.generator:SpdxYAML
-    spdxxml_new = tern.formats.spdx_new.spdxxml.generator:SpdxXML
-    spdxrdf_new = tern.formats.spdx_new.spdxrdf.generator:SpdxRDF
-    spdxtagvalue_new = tern.formats.spdx_new.spdxtagvalue.generator:SpdxTagValue
+    spdxjson22 = tern.formats.spdx_new.spdxjson22.generator:SpdxJSON22
+    spdxyaml22 = tern.formats.spdx_new.spdxyaml22.generator:SpdxYAML22
+    spdxxml22 = tern.formats.spdx_new.spdxxml22.generator:SpdxXML22
+    spdxrdf22 = tern.formats.spdx_new.spdxrdf22.generator:SpdxRDF22
+    spdxtagvalue22 = tern.formats.spdx_new.spdxtagvalue22.generator:SpdxTagValue22
+    spdxjson23 = tern.formats.spdx_new.spdxjson23.generator:SpdxJSON23
+    spdxyaml23 = tern.formats.spdx_new.spdxyaml23.generator:SpdxYAML23
+    spdxxml23 = tern.formats.spdx_new.spdxxml23.generator:SpdxXML23
+    spdxrdf23 = tern.formats.spdx_new.spdxrdf23.generator:SpdxRDF23
+    spdxtagvalue23 = tern.formats.spdx_new.spdxtagvalue23.generator:SpdxTagValue23
 tern.extensions =
     cve_bin_tool = tern.extensions.cve_bin_tool.executor:CveBinTool
     scancode = tern.extensions.scancode.executor:Scancode

tern/formats/spdx_new/general_helpers.py

@@ -7,14 +7,13 @@
 General helpers for SPDX document generator
 """
 import hashlib
-import io
 import re
 import uuid
 from datetime import datetime
-from typing import Union, Callable, IO, Tuple
+from typing import Union, Tuple
 
 from license_expression import get_spdx_licensing, LicenseExpression, Licensing
-from spdx_tools.spdx.model import SpdxNone, Document
+from spdx_tools.spdx.model import SpdxNone
 
 from tern.classes.file_data import FileData
 from tern.classes.image import Image
@@ -77,12 +76,6 @@ def get_package_license_declared(package_license_declared: str) -> Union[License
     return SpdxNone()
 
 
-def get_serialized_document_string(spdx_document: Document, writer_function: Callable[[Document, IO[str]], str]) -> str:
-    with io.StringIO() as stream:
-        writer_function(spdx_document, stream, validate=False)
-        return stream.getvalue()
-
-
 ###########################################################################################
 # central place for SPDXRef-generators to avoid circular imports as these are widely used #
 ###########################################################################################

tern/formats/spdx_new/make_spdx_model.py

@@ -4,17 +4,16 @@
 # SPDX-License-Identifier: BSD-2-Clause
 
 """
-Common functions that are useful for all SPDX serialization formats
+Functions to create an SPDX model instance from a list of Images or an ImageLayer
 """
 
-import logging
 from typing import List
 
 from spdx_tools.spdx.model import Document, CreationInfo, Actor, ActorType, Relationship, RelationshipType
 
 from tern.classes.image_layer import ImageLayer
 from tern.classes.template import Template
-from tern.formats.spdx_new.constants import DOCUMENT_ID, DOCUMENT_NAME, SPDX_VERSION, DATA_LICENSE, DOCUMENT_COMMENT, \
+from tern.formats.spdx_new.constants import DOCUMENT_ID, DOCUMENT_NAME, DATA_LICENSE, DOCUMENT_COMMENT, \
     LICENSE_LIST_VERSION, CREATOR_NAME, DOCUMENT_NAME_SNAPSHOT, DOCUMENT_NAMESPACE_SNAPSHOT
 from tern.formats.spdx_new.file_helpers import get_layer_files_list
 from tern.formats.spdx_new.general_helpers import get_current_timestamp, get_uuid
@@ -25,23 +24,19 @@
     get_image_dict, get_document_namespace
 from tern.formats.spdx_new.layer_helpers import get_layer_dict, get_image_layer_relationships, get_layer_extracted_licenses
 from tern.formats.spdx_new.package_helpers import get_packages_list, get_layer_packages_list
-from tern.utils import constants
 
 from tern.utils.general import get_git_rev_or_version
 
-# global logger
-logger = logging.getLogger(constants.logger_name)
 
-
-def make_spdx_model(image_obj_list: List[Image]) -> Document:
+def make_spdx_model(image_obj_list: List[Image], spdx_version: str) -> Document:
     template = SPDX()
     # we still don't know how SPDX documents could represent multiple
     # images. Hence, we will assume only one image is analyzed and the
     # input is a list of length 1
     image_obj = image_obj_list[0]
 
     creation_info = CreationInfo(
-        spdx_version=SPDX_VERSION,
+        spdx_version=spdx_version,
         spdx_id=DOCUMENT_ID,
         name=DOCUMENT_NAME.format(image_name=image_obj.name),
         document_namespace=get_document_namespace(image_obj),
@@ -73,13 +68,13 @@ def make_spdx_model(image_obj_list: List[Image]) -> Document:
     )
 
 
-def make_spdx_model_snapshot(layer_obj: ImageLayer, template: Template) -> Document:
+def make_spdx_model_snapshot(layer_obj: ImageLayer, template: Template, spdx_version: str) -> Document:
     """This is the SPDX document containing just the packages found at
     container build time"""
     timestamp = get_current_timestamp()
 
     creation_info = CreationInfo(
-        spdx_version=SPDX_VERSION,
+        spdx_version=spdx_version,
         spdx_id=DOCUMENT_ID,
         name=DOCUMENT_NAME_SNAPSHOT,
         document_namespace=DOCUMENT_NAMESPACE_SNAPSHOT.format(timestamp=timestamp, uuid=get_uuid()),

tern/formats/spdx_new/spdx_formats_helper.py

@@ -0,0 +1,78 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (c) 2021 VMware, Inc. All Rights Reserved.
+# SPDX-License-Identifier: BSD-2-Clause
+
+"""
+Handle imports and logging for different SPDX formats
+"""
+import io
+import logging
+from typing import Callable, IO, List
+
+from spdx_tools.spdx.model import Document
+
+from tern.classes.image import Image
+from tern.classes.image_layer import ImageLayer
+from tern.formats.spdx.spdx import SPDX
+from tern.formats.spdx_new.make_spdx_model import make_spdx_model, make_spdx_model_snapshot
+from tern.utils import constants
+
+logger = logging.getLogger(constants.logger_name)
+
+
+def get_spdx_from_image_list(image_obj_list: List[Image], spdx_format: str, spdx_version: str) -> str:
+    """Generate an SPDX document
+    WARNING: This assumes that the list consists of one image or the base
+    image and a stub image, in which case, the information in the stub
+    image is not applicable in the SPDX case as it is an empty image
+    object with no metadata as nothing got built.
+
+    For the sake of SPDX, an image is a 'Package' which 'CONTAINS' each
+    layer which is also a 'Package' which 'CONTAINS' the real Packages"""
+    logger.debug(f"Generating SPDX {spdx_format} document...")
+
+    spdx_document: Document = make_spdx_model(image_obj_list, spdx_version)
+
+    return convert_document_to_serialized_string(spdx_document, spdx_format)
+
+
+def get_spdx_from_layer(layer: ImageLayer, spdx_format: str, spdx_version: str) -> str:
+    """Generate an SPDX document containing package and file information
+    at container build time"""
+    logger.debug(f"Generating SPDX {spdx_format} snapshot document...")
+
+    template = SPDX()
+    spdx_document: Document = make_spdx_model_snapshot(layer, template, spdx_version)
+
+    return convert_document_to_serialized_string(spdx_document, spdx_format)
+
+
+def convert_document_to_serialized_string(spdx_document: Document, spdx_format: str) -> str:
+    if spdx_format == "JSON":
+        from spdx_tools.spdx.writer.json.json_writer import write_document_to_stream
+        return get_serialized_document_string(spdx_document, write_document_to_stream)
+    if spdx_format == "YAML":
+        from spdx_tools.spdx.writer.yaml.yaml_writer import write_document_to_stream
+        return get_serialized_document_string(spdx_document, write_document_to_stream)
+    if spdx_format == "XML":
+        from spdx_tools.spdx.writer.xml.xml_writer import write_document_to_stream
+        return get_serialized_document_string(spdx_document, write_document_to_stream)
+    if spdx_format == "RDF-XML":
+        return get_serialized_rdf_document_string(spdx_document)
+    if spdx_format == "Tag-Value":
+        from spdx_tools.spdx.writer.tagvalue.tagvalue_writer import write_document_to_stream
+        return get_serialized_document_string(spdx_document, write_document_to_stream)
+
+
+def get_serialized_document_string(spdx_document: Document, writer_function: Callable[[Document, IO[str]], str]) -> str:
+    with io.StringIO() as stream:
+        writer_function(spdx_document, stream, validate=False)
+        return stream.getvalue()
+
+
+def get_serialized_rdf_document_string(spdx_document: Document) -> str:
+    from spdx_tools.spdx.writer.rdf.rdf_writer import write_document_to_stream
+    with io.BytesIO() as stream:
+        write_document_to_stream(spdx_document, stream, validate=False)
+        return stream.getvalue().decode("UTF-8")

tern/formats/spdx_new/spdxjson/generator.py

@@ -0,0 +1,22 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (c) 2021 VMware, Inc. All Rights Reserved.
+# SPDX-License-Identifier: BSD-2-Clause
+
+"""
+SPDX-2.2 JSON document generator
+"""
+from typing import List
+
+from tern.classes.image import Image
+from tern.classes.image_layer import ImageLayer
+from tern.formats import generator
+from tern.formats.spdx_new.spdx_formats_helper import get_spdx_from_image_list, get_spdx_from_layer
+
+
+class SpdxJSON22(generator.Generate):
+    def generate(self, image_obj_list: List[Image], print_inclusive=False) -> str:
+        return get_spdx_from_image_list(image_obj_list, "JSON", "SPDX-2.2")
+
+    def generate_layer(self, layer: ImageLayer) -> str:
+        return get_spdx_from_layer(layer, "JSON", "SPDX-2.2")

tern/formats/spdx_new/spdxjson/__init__.py renamed to tern/formats/spdx_new/spdxjson22/__init__.py

@@ -0,0 +1,22 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (c) 2021 VMware, Inc. All Rights Reserved.
+# SPDX-License-Identifier: BSD-2-Clause
+
+"""
+SPDX-2.3 JSON document generator
+"""
+from typing import List
+
+from tern.classes.image import Image
+from tern.classes.image_layer import ImageLayer
+from tern.formats import generator
+from tern.formats.spdx_new.spdx_formats_helper import get_spdx_from_image_list, get_spdx_from_layer
+
+
+class SpdxJSON22(generator.Generate):
+    def generate(self, image_obj_list: List[Image], print_inclusive=False) -> str:
+        return get_spdx_from_image_list(image_obj_list, "JSON", "SPDX-2.3")
+
+    def generate_layer(self, layer: ImageLayer) -> str:
+        return get_spdx_from_layer(layer, "JSON", "SPDX-2.3")