Skip to content

Commit 1bad407

Browse files
AyanSinhaMahapatraAyanSinhaMahapatra
committed
Fix top level license detection bugs
Fixes a bug where top level license detections had license expressions and IDs which were not present anywhere in the scan. This was because we were dereferencing unknown license references after the unique license detection collections and so the modified identifiers and license expressions were not present in the top-level license detections. Fixes a bug where top level unique license detections were not properly collected from package license detections. Signed-off-by: Ayan Sinha Mahapatra <ayansmahapatra@gmail.com>
1 parent ca92534 commit 1bad407

34 files changed

+284
-132
lines changed

‎src/licensedcode/detection.py‎

Lines changed: 15 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -698,35 +698,32 @@ def collect_license_detections(codebase, include_license_clues=True):
698698
detection.file_region = detection.get_file_region(path=resource.path)
699699
resource_license_detections.append(detection)
700700

701-
all_license_detections.extend(
702-
list(process_detections(detections=resource_license_detections))
703-
)
701+
all_license_detections.extend(resource_license_detections)
704702

705-
if TRACE:
706-
logger_debug(
707-
f'before process_detections licenses:',
708-
f'resource_license_detections: {resource_license_detections}\n',
709-
f'all_license_detections: {all_license_detections}',
710-
)
703+
if TRACE:
704+
logger_debug(
705+
f'license detections collected at path {resource.path}:',
706+
f'resource_license_detections: {resource_license_detections}\n',
707+
f'all_license_detections: {all_license_detections}',
708+
)
711709

712-
if has_packages:
713-
package_data = getattr(resource, 'package_data', []) or []
710+
if has_packages:
711+
package_data = getattr(resource, 'package_data', []) or []
714712

715-
package_license_detection_mappings = []
716-
for package in package_data:
713+
package_license_detection_mappings = []
714+
for package in package_data:
717715

718-
if package["license_detections"]:
719-
package_license_detection_mappings.extend(package["license_detections"])
716+
if package["license_detections"]:
717+
package_license_detection_mappings.extend(package["license_detections"])
720718

721-
if package["other_license_detections"]:
722-
package_license_detection_mappings.extend(package["other_license_detections"])
719+
if package["other_license_detections"]:
720+
package_license_detection_mappings.extend(package["other_license_detections"])
723721

724722
if package_license_detection_mappings:
725723
package_license_detection_objects = detections_from_license_detection_mappings(
726724
license_detection_mappings=package_license_detection_mappings,
727725
file_path=resource.path,
728726
)
729-
730727
all_license_detections.extend(package_license_detection_objects)
731728

732729
return all_license_detections

‎src/licensedcode/plugin_license.py‎

Lines changed: 15 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -196,21 +196,6 @@ def process_codebase(self, codebase, license_diagnostics, **kwargs):
196196
if codebase.has_single_resource and not codebase.root.is_file:
197197
return
198198

199-
license_detections = collect_license_detections(
200-
codebase=codebase,
201-
include_license_clues=False
202-
)
203-
unique_license_detections = UniqueDetection.get_unique_detections(
204-
license_detections=license_detections,
205-
)
206-
207-
if TRACE:
208-
logger_debug(
209-
f'process_codebase: codebase license_detections',
210-
f'license_detections: {license_detections}\n',
211-
f'unique_license_detections: {unique_license_detections}',
212-
)
213-
214199
modified = False
215200
for resource in codebase.walk(topdown=False):
216201
# follow license references to other files
@@ -228,6 +213,21 @@ def process_codebase(self, codebase, license_diagnostics, **kwargs):
228213
f'after : {license_expressions_after}'
229214
)
230215

216+
license_detections = collect_license_detections(
217+
codebase=codebase,
218+
include_license_clues=False
219+
)
220+
unique_license_detections = UniqueDetection.get_unique_detections(
221+
license_detections=license_detections,
222+
)
223+
224+
if TRACE:
225+
logger_debug(
226+
f'process_codebase: codebase license_detections',
227+
f'license_detections: {license_detections}\n',
228+
f'unique_license_detections: {unique_license_detections}',
229+
)
230+
231231
codebase.attributes.license_detections.extend([
232232
unique_detection.to_dict(license_diagnostics=license_diagnostics)
233233
for unique_detection in unique_license_detections

‎tests/formattedcode/data/common/manifests-expected.json‎

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -475,6 +475,16 @@
475475
"license_expression": "cddl-1.0",
476476
"detection_count": 1
477477
},
478+
{
479+
"identifier": "cddl_1_0-9893b55c-3b2b-4ee8-a932-6c6c93a63fc5",
480+
"license_expression": "cddl-1.0",
481+
"detection_count": 1
482+
},
483+
{
484+
"identifier": "cddl_1_0-ef82fc8c-50cb-6f35-1814-d2eb0bc13e83",
485+
"license_expression": "cddl-1.0",
486+
"detection_count": 1
487+
},
478488
{
479489
"identifier": "apache_2_0-ec759ae0-ea5a-f138-793e-388520e080c0",
480490
"license_expression": "apache-2.0",
@@ -483,13 +493,18 @@
483493
{
484494
"identifier": "apache_2_0-d66ab77d-a5cc-7104-e702-dc7df61fe9e8",
485495
"license_expression": "apache-2.0",
486-
"detection_count": 1
496+
"detection_count": 3
487497
},
488498
{
489499
"identifier": "mit-3fce6ea2-8abd-6c6b-3ede-a37af7c6efee",
490500
"license_expression": "mit",
491501
"detection_count": 1
492502
},
503+
{
504+
"identifier": "mit-a822f434-d61f-f2b1-c792-8b8cb9e7b9bf",
505+
"license_expression": "mit",
506+
"detection_count": 1
507+
},
493508
{
494509
"identifier": "lgpl_3_0-121be3c2-9c80-df84-d3da-8f674e4125c0",
495510
"license_expression": "lgpl-3.0",

‎tests/formattedcode/data/common/manifests-expected.jsonlines‎

Lines changed: 19 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -20,9 +20,9 @@
2020
"system_environment": {
2121
"operating_system": "linux",
2222
"cpu_architecture": "64",
23-
"platform": "Linux-5.15.0-72-generic-x86_64-with-glibc2.29",
24-
"platform_version": "#79~20.04.1-Ubuntu SMP Thu Apr 20 22:12:07 UTC 2023",
25-
"python_version": "3.8.10 (default, Mar 13 2023, 10:26:41) \n[GCC 9.4.0]"
23+
"platform": "Linux-5.15.0-73-generic-x86_64-with-glibc2.29",
24+
"platform_version": "#80~20.04.1-Ubuntu SMP Wed May 17 14:58:14 UTC 2023",
25+
"python_version": "3.8.10 (default, May 26 2023, 14:05:08) \n[GCC 9.4.0]"
2626
},
2727
"spdx_license_list_version": "3.20",
2828
"files_count": 4
@@ -511,6 +511,16 @@
511511
"license_expression": "cddl-1.0",
512512
"detection_count": 1
513513
},
514+
{
515+
"identifier": "cddl_1_0-9893b55c-3b2b-4ee8-a932-6c6c93a63fc5",
516+
"license_expression": "cddl-1.0",
517+
"detection_count": 1
518+
},
519+
{
520+
"identifier": "cddl_1_0-ef82fc8c-50cb-6f35-1814-d2eb0bc13e83",
521+
"license_expression": "cddl-1.0",
522+
"detection_count": 1
523+
},
514524
{
515525
"identifier": "apache_2_0-ec759ae0-ea5a-f138-793e-388520e080c0",
516526
"license_expression": "apache-2.0",
@@ -519,13 +529,18 @@
519529
{
520530
"identifier": "apache_2_0-d66ab77d-a5cc-7104-e702-dc7df61fe9e8",
521531
"license_expression": "apache-2.0",
522-
"detection_count": 1
532+
"detection_count": 3
523533
},
524534
{
525535
"identifier": "mit-3fce6ea2-8abd-6c6b-3ede-a37af7c6efee",
526536
"license_expression": "mit",
527537
"detection_count": 1
528538
},
539+
{
540+
"identifier": "mit-a822f434-d61f-f2b1-c792-8b8cb9e7b9bf",
541+
"license_expression": "mit",
542+
"detection_count": 1
543+
},
529544
{
530545
"identifier": "lgpl_3_0-121be3c2-9c80-df84-d3da-8f674e4125c0",
531546
"license_expression": "lgpl-3.0",

‎tests/formattedcode/data/common/manifests-expected.yaml‎

Lines changed: 13 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -29,9 +29,9 @@ headers:
2929
system_environment:
3030
operating_system: linux
3131
cpu_architecture: 64
32-
platform: Linux-5.15.0-72-generic-x86_64-with-glibc2.29
33-
platform_version: '#79~20.04.1-Ubuntu SMP Thu Apr 20 22:12:07 UTC 2023'
34-
python_version: "3.8.10 (default, Mar 13 2023, 10:26:41) \n[GCC 9.4.0]"
32+
platform: Linux-5.15.0-73-generic-x86_64-with-glibc2.29
33+
platform_version: '#80~20.04.1-Ubuntu SMP Wed May 17 14:58:14 UTC 2023'
34+
python_version: "3.8.10 (default, May 26 2023, 14:05:08) \n[GCC 9.4.0]"
3535
spdx_license_list_version: '3.20'
3636
files_count: 4
3737
summary:
@@ -468,15 +468,24 @@ license_detections:
468468
- identifier: cddl_1_0-c6dbef4d-659c-289f-5ee9-1ca0278edad6
469469
license_expression: cddl-1.0
470470
detection_count: 1
471+
- identifier: cddl_1_0-9893b55c-3b2b-4ee8-a932-6c6c93a63fc5
472+
license_expression: cddl-1.0
473+
detection_count: 1
474+
- identifier: cddl_1_0-ef82fc8c-50cb-6f35-1814-d2eb0bc13e83
475+
license_expression: cddl-1.0
476+
detection_count: 1
471477
- identifier: apache_2_0-ec759ae0-ea5a-f138-793e-388520e080c0
472478
license_expression: apache-2.0
473479
detection_count: 1
474480
- identifier: apache_2_0-d66ab77d-a5cc-7104-e702-dc7df61fe9e8
475481
license_expression: apache-2.0
476-
detection_count: 1
482+
detection_count: 3
477483
- identifier: mit-3fce6ea2-8abd-6c6b-3ede-a37af7c6efee
478484
license_expression: mit
479485
detection_count: 1
486+
- identifier: mit-a822f434-d61f-f2b1-c792-8b8cb9e7b9bf
487+
license_expression: mit
488+
detection_count: 1
480489
- identifier: lgpl_3_0-121be3c2-9c80-df84-d3da-8f674e4125c0
481490
license_expression: lgpl-3.0
482491
detection_count: 1

‎tests/licensedcode/data/plugin_license/license_reference/license-ref-see-copying.expected.json‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7,10 +7,12 @@
77
"detection_log": []
88
},
99
{
10-
"identifier": "unknown_license_reference-561f1903-04c8-0d5a-2432-034a966157a3",
11-
"license_expression": "unknown-license-reference",
10+
"identifier": "apache-2.0-1c807a43-2040-70af-75aa-c343d5f2b90c",
11+
"license_expression": "apache-2.0",
1212
"detection_count": 1,
13-
"detection_log": []
13+
"detection_log": [
14+
"unknown-reference-to-local-file"
15+
]
1416
}
1517
],
1618
"files": [

‎tests/licensedcode/data/plugin_license/license_reference/scan-ref.expected.json‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7,10 +7,12 @@
77
"detection_log": []
88
},
99
{
10-
"identifier": "unknown_license_reference-8dac7670-e286-f6de-27a1-f2b5c87524ff",
11-
"license_expression": "unknown-license-reference",
10+
"identifier": "mit-20c01557-97bd-0022-052e-56c5ed8465ea",
11+
"license_expression": "mit",
1212
"detection_count": 1,
13-
"detection_log": []
13+
"detection_log": [
14+
"unknown-reference-to-local-file"
15+
]
1416
}
1517
],
1618
"files": [

‎tests/licensedcode/data/plugin_license/license_reference/scan-unknown-reference-copyright.expected.json‎

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,12 @@
11
{
22
"license_detections": [
33
{
4-
"identifier": "unknown_license_reference-87d47954-89f5-fcbf-a173-e0d899d7c5e2",
5-
"license_expression": "unknown-license-reference",
4+
"identifier": "x11-xconsortium-veillard-61f804f6-d484-92ca-09b5-26be51ac974e",
5+
"license_expression": "x11-xconsortium-veillard",
66
"detection_count": 2,
7-
"detection_log": []
7+
"detection_log": [
8+
"unknown-reference-to-local-file"
9+
]
810
},
911
{
1012
"identifier": "x11_xconsortium_veillard-b2601908-f03c-335c-5bbd-e72dc065c901",
@@ -13,10 +15,12 @@
1315
"detection_log": []
1416
},
1517
{
16-
"identifier": "unknown_license_reference-8715cef9-8691-1147-d5c0-154042d18db1",
17-
"license_expression": "unknown-license-reference",
18+
"identifier": "x11-xconsortium-veillard-50c015bd-e4e1-c6fe-eb82-9551473dd8e1",
19+
"license_expression": "x11-xconsortium-veillard",
1820
"detection_count": 1,
19-
"detection_log": []
21+
"detection_log": [
22+
"unknown-reference-to-local-file"
23+
]
2024
}
2125
],
2226
"files": [

‎tests/licensedcode/data/plugin_license/license_reference/unknown-ref-to-key-file-root.expected.json‎

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,12 @@
11
{
22
"license_detections": [
33
{
4-
"identifier": "unknown_license_reference-f177df3d-5d60-ab6b-92ac-9a3fb4cb783f",
5-
"license_expression": "unknown-license-reference",
4+
"identifier": "mit-e46a912c-b32a-30d5-dc27-c13824253230",
5+
"license_expression": "mit",
66
"detection_count": 4,
7-
"detection_log": []
7+
"detection_log": [
8+
"unknown-reference-to-local-file"
9+
]
810
},
911
{
1012
"identifier": "mit-d0a34f23-8c35-8874-a99f-6ed1e3b31f40",
@@ -31,10 +33,12 @@
3133
"detection_log": []
3234
},
3335
{
34-
"identifier": "mit-eea7f51e-637f-984c-1e86-a78bca648bf8",
36+
"identifier": "mit-ad99a349-2a14-9fe5-c6a6-366fd3b9067b",
3537
"license_expression": "mit",
3638
"detection_count": 1,
37-
"detection_log": []
39+
"detection_log": [
40+
"unknown-reference-to-local-file"
41+
]
3842
}
3943
],
4044
"files": [

‎tests/packagedcode/data/license_detection/reference-at-manifest/fizzler.expected.json‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -148,10 +148,12 @@
148148
"detection_log": []
149149
},
150150
{
151-
"identifier": "unknown_license_reference-45e84991-3b5b-bfa7-1b2f-7413856c448d",
152-
"license_expression": "unknown-license-reference",
151+
"identifier": "lgpl-2.0-plus AND gpl-1.0-plus-17115808-8fc8-9c98-b64d-c5013bcbde1b",
152+
"license_expression": "lgpl-2.0-plus AND gpl-1.0-plus",
153153
"detection_count": 1,
154-
"detection_log": []
154+
"detection_log": [
155+
"unknown-reference-to-local-file"
156+
]
155157
},
156158
{
157159
"identifier": "lgpl-2.0-plus AND gpl-1.0-plus-de66a0ae-df4d-ab6b-975e-8deb27c0f945",

0 commit comments

Comments
 (0)