Skip to content

Commit 6d439eb

Browse files
AyanSinhaMahapatraAyanSinhaMahapatra
committed
Fix top level license detection bugs
Fixes a bug where top level license detections had license expressions and IDs which were not present anywhere in the scan. This was because we were dereferencing unknown license references after the unique license detection collections and so the modified identifiers and license expressions were not present in the top-level license detections. Fixes a bug where top level unique license detections were not properly collected from package license detections. Signed-off-by: Ayan Sinha Mahapatra <ayansmahapatra@gmail.com>
1 parent ca92534 commit 6d439eb

21 files changed

+164
-116
lines changed

‎src/licensedcode/detection.py‎

Lines changed: 15 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -698,35 +698,32 @@ def collect_license_detections(codebase, include_license_clues=True):
698698
detection.file_region = detection.get_file_region(path=resource.path)
699699
resource_license_detections.append(detection)
700700

701-
all_license_detections.extend(
702-
list(process_detections(detections=resource_license_detections))
703-
)
701+
all_license_detections.extend(resource_license_detections)
704702

705-
if TRACE:
706-
logger_debug(
707-
f'before process_detections licenses:',
708-
f'resource_license_detections: {resource_license_detections}\n',
709-
f'all_license_detections: {all_license_detections}',
710-
)
703+
if TRACE:
704+
logger_debug(
705+
f'license detections collected at path {resource.path}:',
706+
f'resource_license_detections: {resource_license_detections}\n',
707+
f'all_license_detections: {all_license_detections}',
708+
)
711709

712-
if has_packages:
713-
package_data = getattr(resource, 'package_data', []) or []
710+
if has_packages:
711+
package_data = getattr(resource, 'package_data', []) or []
714712

715-
package_license_detection_mappings = []
716-
for package in package_data:
713+
package_license_detection_mappings = []
714+
for package in package_data:
717715

718-
if package["license_detections"]:
719-
package_license_detection_mappings.extend(package["license_detections"])
716+
if package["license_detections"]:
717+
package_license_detection_mappings.extend(package["license_detections"])
720718

721-
if package["other_license_detections"]:
722-
package_license_detection_mappings.extend(package["other_license_detections"])
719+
if package["other_license_detections"]:
720+
package_license_detection_mappings.extend(package["other_license_detections"])
723721

724722
if package_license_detection_mappings:
725723
package_license_detection_objects = detections_from_license_detection_mappings(
726724
license_detection_mappings=package_license_detection_mappings,
727725
file_path=resource.path,
728726
)
729-
730727
all_license_detections.extend(package_license_detection_objects)
731728

732729
return all_license_detections

‎src/licensedcode/plugin_license.py‎

Lines changed: 15 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -196,21 +196,6 @@ def process_codebase(self, codebase, license_diagnostics, **kwargs):
196196
if codebase.has_single_resource and not codebase.root.is_file:
197197
return
198198

199-
license_detections = collect_license_detections(
200-
codebase=codebase,
201-
include_license_clues=False
202-
)
203-
unique_license_detections = UniqueDetection.get_unique_detections(
204-
license_detections=license_detections,
205-
)
206-
207-
if TRACE:
208-
logger_debug(
209-
f'process_codebase: codebase license_detections',
210-
f'license_detections: {license_detections}\n',
211-
f'unique_license_detections: {unique_license_detections}',
212-
)
213-
214199
modified = False
215200
for resource in codebase.walk(topdown=False):
216201
# follow license references to other files
@@ -228,6 +213,21 @@ def process_codebase(self, codebase, license_diagnostics, **kwargs):
228213
f'after : {license_expressions_after}'
229214
)
230215

216+
license_detections = collect_license_detections(
217+
codebase=codebase,
218+
include_license_clues=False
219+
)
220+
unique_license_detections = UniqueDetection.get_unique_detections(
221+
license_detections=license_detections,
222+
)
223+
224+
if TRACE:
225+
logger_debug(
226+
f'process_codebase: codebase license_detections',
227+
f'license_detections: {license_detections}\n',
228+
f'unique_license_detections: {unique_license_detections}',
229+
)
230+
231231
codebase.attributes.license_detections.extend([
232232
unique_detection.to_dict(license_diagnostics=license_diagnostics)
233233
for unique_detection in unique_license_detections

‎tests/licensedcode/data/plugin_license/license_reference/license-ref-see-copying.expected.json‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7,10 +7,12 @@
77
"detection_log": []
88
},
99
{
10-
"identifier": "unknown_license_reference-561f1903-04c8-0d5a-2432-034a966157a3",
11-
"license_expression": "unknown-license-reference",
10+
"identifier": "apache-2.0-1c807a43-2040-70af-75aa-c343d5f2b90c",
11+
"license_expression": "apache-2.0",
1212
"detection_count": 1,
13-
"detection_log": []
13+
"detection_log": [
14+
"unknown-reference-to-local-file"
15+
]
1416
}
1517
],
1618
"files": [

‎tests/licensedcode/data/plugin_license/license_reference/scan-ref.expected.json‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7,10 +7,12 @@
77
"detection_log": []
88
},
99
{
10-
"identifier": "unknown_license_reference-8dac7670-e286-f6de-27a1-f2b5c87524ff",
11-
"license_expression": "unknown-license-reference",
10+
"identifier": "mit-20c01557-97bd-0022-052e-56c5ed8465ea",
11+
"license_expression": "mit",
1212
"detection_count": 1,
13-
"detection_log": []
13+
"detection_log": [
14+
"unknown-reference-to-local-file"
15+
]
1416
}
1517
],
1618
"files": [

‎tests/licensedcode/data/plugin_license/license_reference/scan-unknown-reference-copyright.expected.json‎

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,12 @@
11
{
22
"license_detections": [
33
{
4-
"identifier": "unknown_license_reference-87d47954-89f5-fcbf-a173-e0d899d7c5e2",
5-
"license_expression": "unknown-license-reference",
4+
"identifier": "x11-xconsortium-veillard-61f804f6-d484-92ca-09b5-26be51ac974e",
5+
"license_expression": "x11-xconsortium-veillard",
66
"detection_count": 2,
7-
"detection_log": []
7+
"detection_log": [
8+
"unknown-reference-to-local-file"
9+
]
810
},
911
{
1012
"identifier": "x11_xconsortium_veillard-b2601908-f03c-335c-5bbd-e72dc065c901",
@@ -13,10 +15,12 @@
1315
"detection_log": []
1416
},
1517
{
16-
"identifier": "unknown_license_reference-8715cef9-8691-1147-d5c0-154042d18db1",
17-
"license_expression": "unknown-license-reference",
18+
"identifier": "x11-xconsortium-veillard-50c015bd-e4e1-c6fe-eb82-9551473dd8e1",
19+
"license_expression": "x11-xconsortium-veillard",
1820
"detection_count": 1,
19-
"detection_log": []
21+
"detection_log": [
22+
"unknown-reference-to-local-file"
23+
]
2024
}
2125
],
2226
"files": [

‎tests/licensedcode/data/plugin_license/license_reference/unknown-ref-to-key-file-root.expected.json‎

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,12 @@
11
{
22
"license_detections": [
33
{
4-
"identifier": "unknown_license_reference-f177df3d-5d60-ab6b-92ac-9a3fb4cb783f",
5-
"license_expression": "unknown-license-reference",
4+
"identifier": "mit-e46a912c-b32a-30d5-dc27-c13824253230",
5+
"license_expression": "mit",
66
"detection_count": 4,
7-
"detection_log": []
7+
"detection_log": [
8+
"unknown-reference-to-local-file"
9+
]
810
},
911
{
1012
"identifier": "mit-d0a34f23-8c35-8874-a99f-6ed1e3b31f40",
@@ -31,10 +33,12 @@
3133
"detection_log": []
3234
},
3335
{
34-
"identifier": "mit-eea7f51e-637f-984c-1e86-a78bca648bf8",
36+
"identifier": "mit-ad99a349-2a14-9fe5-c6a6-366fd3b9067b",
3537
"license_expression": "mit",
3638
"detection_count": 1,
37-
"detection_log": []
39+
"detection_log": [
40+
"unknown-reference-to-local-file"
41+
]
3842
}
3943
],
4044
"files": [

‎tests/packagedcode/data/license_detection/reference-at-manifest/fizzler.expected.json‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -148,10 +148,12 @@
148148
"detection_log": []
149149
},
150150
{
151-
"identifier": "unknown_license_reference-45e84991-3b5b-bfa7-1b2f-7413856c448d",
152-
"license_expression": "unknown-license-reference",
151+
"identifier": "lgpl-2.0-plus AND gpl-1.0-plus-17115808-8fc8-9c98-b64d-c5013bcbde1b",
152+
"license_expression": "lgpl-2.0-plus AND gpl-1.0-plus",
153153
"detection_count": 1,
154-
"detection_log": []
154+
"detection_log": [
155+
"unknown-reference-to-local-file"
156+
]
155157
},
156158
{
157159
"identifier": "lgpl-2.0-plus AND gpl-1.0-plus-de66a0ae-df4d-ab6b-975e-8deb27c0f945",

‎tests/packagedcode/data/license_detection/reference-at-manifest/flutter_playtabs_bridge.expected.json‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -107,10 +107,12 @@
107107
"dependencies": [],
108108
"license_detections": [
109109
{
110-
"identifier": "unknown_license_reference-6b24f327-22b1-723e-10f9-37a8bc8b3c70",
111-
"license_expression": "unknown-license-reference",
112-
"detection_count": 1,
113-
"detection_log": []
110+
"identifier": "mit-a979a2a3-dfdb-02aa-2450-71641a61a264",
111+
"license_expression": "mit",
112+
"detection_count": 2,
113+
"detection_log": [
114+
"unknown-reference-to-local-file"
115+
]
114116
},
115117
{
116118
"identifier": "mit-ac40beba-5702-f54d-755f-333441314bb0",

‎tests/packagedcode/data/license_detection/reference-at-manifest/nanopb.expected.json‎

Lines changed: 2 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -99,18 +99,12 @@
9999
"detection_count": 1,
100100
"detection_log": []
101101
},
102-
{
103-
"identifier": "zlib-3777cd8f-b515-8132-88be-cec12676bbaf",
104-
"license_expression": "zlib",
105-
"detection_count": 1,
106-
"detection_log": []
107-
},
108102
{
109103
"identifier": "zlib-9531c668-be8d-7a25-49eb-c18c9dcd616b",
110104
"license_expression": "zlib",
111-
"detection_count": 1,
105+
"detection_count": 2,
112106
"detection_log": [
113-
"package-unknown-reference-to-local-file"
107+
"unknown-reference-to-local-file"
114108
]
115109
}
116110
],

‎tests/packagedcode/data/license_detection/reference-to-package/fusiondirectory.expected.json‎

Lines changed: 15 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -4758,37 +4758,37 @@
47584758
{
47594759
"identifier": "gpl_2_0_plus-53a26be2-1f88-01ec-d294-921768a513c9",
47604760
"license_expression": "gpl-2.0-plus",
4761-
"detection_count": 21,
4761+
"detection_count": 42,
47624762
"detection_log": []
47634763
},
47644764
{
47654765
"identifier": "gpl_2_0_plus_and_free_unknown-7df72521-0609-4896-2583-44c45dd3f8e6",
47664766
"license_expression": "gpl-2.0-plus AND free-unknown",
4767-
"detection_count": 1,
4767+
"detection_count": 2,
47684768
"detection_log": []
47694769
},
47704770
{
47714771
"identifier": "bsd_new-008266ae-6939-ad31-3617-228b0809863c",
47724772
"license_expression": "bsd-new",
4773-
"detection_count": 1,
4773+
"detection_count": 2,
47744774
"detection_log": []
47754775
},
47764776
{
47774777
"identifier": "gpl_2_0_plus_and_free_unknown-f9cd0597-d35d-ec72-0e6b-77970ad90317",
47784778
"license_expression": "gpl-2.0-plus AND free-unknown",
4779-
"detection_count": 1,
4779+
"detection_count": 2,
47804780
"detection_log": []
47814781
},
47824782
{
47834783
"identifier": "lgpl_3_0_plus-4c8d95b2-1acf-7a81-473c-d8e70962288c",
47844784
"license_expression": "lgpl-3.0-plus",
4785-
"detection_count": 1,
4785+
"detection_count": 2,
47864786
"detection_log": []
47874787
},
47884788
{
47894789
"identifier": "public_domain-1a6a4f2c-bd92-9942-920f-be3d0c2bbda6",
47904790
"license_expression": "public-domain",
4791-
"detection_count": 1,
4791+
"detection_count": 2,
47924792
"detection_log": [
47934793
"possible-false-positive",
47944794
"not-license-clues-as-more-detections-present"
@@ -4797,7 +4797,7 @@
47974797
{
47984798
"identifier": "gpl_2_0_plus-fed2dc38-09ac-103e-1b86-4a4f5c00614a",
47994799
"license_expression": "gpl-2.0-plus",
4800-
"detection_count": 1,
4800+
"detection_count": 2,
48014801
"detection_log": [
48024802
"possible-false-positive",
48034803
"not-license-clues-as-more-detections-present"
@@ -4806,7 +4806,7 @@
48064806
{
48074807
"identifier": "mit-1f9f2ae8-7020-0a13-7934-461c752929a4",
48084808
"license_expression": "mit",
4809-
"detection_count": 3,
4809+
"detection_count": 6,
48104810
"detection_log": [
48114811
"possible-false-positive",
48124812
"not-license-clues-as-more-detections-present"
@@ -4815,31 +4815,31 @@
48154815
{
48164816
"identifier": "bsd_original-9198bafe-47c0-f9dc-5ef1-bd276a69786e",
48174817
"license_expression": "bsd-original",
4818-
"detection_count": 1,
4818+
"detection_count": 2,
48194819
"detection_log": []
48204820
},
48214821
{
48224822
"identifier": "gpl_2_0_plus_and_gpl_3_0_plus_and_lgpl_2_1_plus_and_lgpl_3_0_plus_and_bsd_new_and_bsd_original_and_mit_and_public_domain_and_other_permissive-9e2a213a-3fc4-9ee3-8e3f-783829530b14",
48234823
"license_expression": "gpl-2.0-plus AND gpl-3.0-plus AND lgpl-2.1-plus AND lgpl-3.0-plus AND bsd-new AND bsd-original AND mit AND public-domain AND other-permissive",
4824-
"detection_count": 1,
4824+
"detection_count": 2,
48254825
"detection_log": []
48264826
},
48274827
{
48284828
"identifier": "gpl_2_0_plus-227f50b1-f05e-5b3b-b107-ae1e2f56448b",
48294829
"license_expression": "gpl-2.0-plus",
4830-
"detection_count": 22,
4830+
"detection_count": 44,
48314831
"detection_log": []
48324832
},
48334833
{
48344834
"identifier": "bsd_simplified-2383ae10-5494-e069-46c2-e2d6cb56951f",
48354835
"license_expression": "bsd-simplified",
4836-
"detection_count": 1,
4836+
"detection_count": 2,
48374837
"detection_log": []
48384838
},
48394839
{
48404840
"identifier": "lgpl_3_0-6b4a45da-85cc-8eb0-5613-be81d4efd6a5",
48414841
"license_expression": "lgpl-3.0",
4842-
"detection_count": 1,
4842+
"detection_count": 2,
48434843
"detection_log": [
48444844
"possible-false-positive",
48454845
"not-license-clues-as-more-detections-present"
@@ -4848,7 +4848,7 @@
48484848
{
48494849
"identifier": "mit_and_other_permissive-907b56d6-95b9-e85c-83c0-efd967c7e30a",
48504850
"license_expression": "mit AND other-permissive",
4851-
"detection_count": 1,
4851+
"detection_count": 2,
48524852
"detection_log": [
48534853
"possible-false-positive",
48544854
"not-license-clues-as-more-detections-present"
@@ -4857,7 +4857,7 @@
48574857
{
48584858
"identifier": "public_domain_and_bsd_original_and_gpl_1_0_plus-ccb73883-974d-42eb-d572-39fa7bf22f4c",
48594859
"license_expression": "public-domain AND bsd-original AND gpl-1.0-plus",
4860-
"detection_count": 1,
4860+
"detection_count": 2,
48614861
"detection_log": [
48624862
"possible-false-positive",
48634863
"not-license-clues-as-more-detections-present"

0 commit comments

Comments
 (0)