Skip to content

Commit 5558430

Browse files
JonoYangJonoYang
committed
Update ProjectCodebase
 * Remove root property and create root_resources * Update walk to walk from root resources * Update tests Signed-off-by: Jono Yang <jyang@nexb.com>
1 parent f7c91fa commit 5558430

File tree

9 files changed

+1079
-825
lines changed

9 files changed

+1079
-825
lines changed

‎scanpipe/pipes/codebase.py‎

Lines changed: 66 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,14 @@ def sort_by_lower_name(resource):
2929
return resource["name"].lower()
3030

3131

32-
def get_tree(resource, fields, codebase=None):
32+
def get_resource_fields(resource, fields):
33+
"""
34+
Return a mapping of fields from `fields` and values from `resource`
35+
"""
36+
return {field: getattr(resource, field) for field in fields}
37+
38+
39+
def get_resource_tree(resource, fields, codebase=None, seen_resources=set()):
3340
"""
3441
Returns a tree as a dictionary structure starting from the provided `resource`.
3542
@@ -38,23 +45,65 @@ def get_tree(resource, fields, codebase=None):
3845
- commoncode.resource.Resource
3946
4047
The data included for each child is controlled with the `fields` argument.
48+
4149
The `codebase` is only required in the context of a commoncode `Resource`
4250
input.
51+
52+
`seen_resources` is used when get_resource_tree() is used in the context of
53+
get_codebase_tree(). We keep track of child Resources we visit in
54+
`seen_resources`, so we don't visit them again in get_codebase_tree().
4355
"""
44-
resource_dict = {field: getattr(resource, field) for field in fields}
56+
resource_dict = get_resource_fields(resource, fields)
4557

4658
if resource.is_dir:
47-
children = [
48-
get_tree(child, fields, codebase) for child in resource.children(codebase)
49-
]
59+
children = []
60+
for child in resource.children(codebase):
61+
seen_resources.add(child.path)
62+
children.append(
63+
get_resource_tree(
64+
child,
65+
fields,
66+
codebase,
67+
seen_resources
68+
)
69+
)
5070
if children:
5171
resource_dict["children"] = sorted(children, key=sort_by_lower_name)
5272

5373
return resource_dict
5474

5575

56-
# TODO: Walking the ProjectCodebase is broken as we do not have a consistent way
57-
# to get the root of a codebase.
76+
def get_codebase_tree(codebase, fields):
77+
"""
78+
Returns a tree as a dictionary structure starting from the root resources of the provided `codebase`.
79+
80+
The following classes are supported for the input `codebase` object:
81+
- scanpipe.pipes.codebase.ProjectCodebase
82+
- commoncode.resource.Codebase
83+
- commoncode.resource.VirtualCodebase
84+
85+
The data included for each child is controlled with the `fields` argument.
86+
"""
87+
seen_resources = set()
88+
codebase_dict = dict(children=[])
89+
for resource in codebase.walk():
90+
path = resource.path
91+
if path in seen_resources:
92+
continue
93+
else:
94+
seen_resources.add(path)
95+
resource_dict = get_resource_fields(resource, fields)
96+
if resource.is_dir:
97+
children = []
98+
for child in resource.children(codebase):
99+
seen_resources.add(child.path)
100+
children.append(get_resource_tree(child, fields, codebase, seen_resources))
101+
if children:
102+
resource_dict["children"] = sorted(children, key=sort_by_lower_name)
103+
codebase_dict["children"].append(resource_dict)
104+
return codebase_dict
105+
106+
58107
class ProjectCodebase:
59108
"""
60109
Represents the codebase of a project stored in the database.
@@ -68,24 +117,21 @@ def __init__(self, project):
68117
self.project = project
69118

70119
@property
71-
def root(self):
72-
try:
73-
return self.project.codebaseresources.get(path="codebase")
74-
except ObjectDoesNotExist:
75-
raise AttributeError("Codebase root cannot be determined.")
120+
def root_resources(self):
121+
return self.project.codebaseresources.exclude(path__contains="/")
76122

77123
@property
78124
def resources(self):
79125
return self.project.codebaseresources.all()
80126

81127
def walk(self, topdown=True):
82-
root = self.root
83-
if topdown:
84-
yield root
85-
for resource in root.walk(topdown=topdown):
86-
yield resource
87-
if not topdown:
88-
yield root
128+
for root_resource in self.root_resources:
129+
if topdown:
130+
yield root_resource
131+
for resource in root_resource.walk(topdown=topdown):
132+
yield resource
133+
if not topdown:
134+
yield root_resource
89135

90136
def get_tree(self):
91-
return get_tree(self.root, fields=["name", "path"])
137+
return get_codebase_tree(self, fields=["name", "path"])

‎scanpipe/tests/data/asgiref-3.3.0.spdx.json‎

Lines changed: 21 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
"dataLicense": "CC0-1.0",
44
"SPDXID": "SPDXRef-DOCUMENT",
55
"name": "scancodeio_asgiref",
6-
"documentNamespace": "https://scancode.io/spdxdocs/694e35cd-7d28-40ba-b504-e9580b819427",
6+
"documentNamespace": "https://scancode.io/spdxdocs/4c7f2e2e-0c41-45d2-a6cc-7bbde9a9f440",
77
"creationInfo": {
88
"created": "2000-01-01T01:02:03Z",
99
"creators": [
@@ -14,7 +14,7 @@
1414
"packages": [
1515
{
1616
"name": "asgiref",
17-
"SPDXID": "SPDXRef-scancodeio-discoveredpackage-7f975593-4bdd-4217-8d04-3554d691f310",
17+
"SPDXID": "SPDXRef-scancodeio-discoveredpackage-d4227045-439f-41b8-98ff-067c55276b2f",
1818
"downloadLocation": "NOASSERTION",
1919
"licenseConcluded": "BSD-3-Clause AND BSD-3-Clause",
2020
"copyrightText": "NOASSERTION",
@@ -33,7 +33,7 @@
3333
},
3434
{
3535
"name": "asgiref",
36-
"SPDXID": "SPDXRef-scancodeio-discoveredpackage-edf75850-316d-4a46-9ca1-bde1f4fc3aab",
36+
"SPDXID": "SPDXRef-scancodeio-discoveredpackage-92213ebf-c2f0-4bd6-9d4a-38ea138d2f63",
3737
"downloadLocation": "NOASSERTION",
3838
"licenseConcluded": "BSD-3-Clause AND BSD-3-Clause",
3939
"copyrightText": "NOASSERTION",
@@ -52,7 +52,7 @@
5252
},
5353
{
5454
"name": "pytest",
55-
"SPDXID": "SPDXRef-scancodeio-discovereddependency-pkg:pypi/pytest?uuid=6ef94d27-ae57-408e-baa4-e20e3dc1f1aa",
55+
"SPDXID": "SPDXRef-scancodeio-discovereddependency-pkg:pypi/pytest?uuid=ebec93f1-d78e-49ee-a1f5-80c26a0dc32b",
5656
"downloadLocation": "NOASSERTION",
5757
"licenseConcluded": "NOASSERTION",
5858
"copyrightText": "NOASSERTION",
@@ -68,7 +68,7 @@
6868
},
6969
{
7070
"name": "pytest",
71-
"SPDXID": "SPDXRef-scancodeio-discovereddependency-pkg:pypi/pytest?uuid=4b195723-fe04-47c9-8b65-c08c60423c18",
71+
"SPDXID": "SPDXRef-scancodeio-discovereddependency-pkg:pypi/pytest?uuid=cd04cbe8-ba3e-412b-9c1d-7eb6a617f1d1",
7272
"downloadLocation": "NOASSERTION",
7373
"licenseConcluded": "NOASSERTION",
7474
"copyrightText": "NOASSERTION",
@@ -84,7 +84,7 @@
8484
},
8585
{
8686
"name": "pytest-asyncio",
87-
"SPDXID": "SPDXRef-scancodeio-discovereddependency-pkg:pypi/pytest-asyncio?uuid=87beebac-2c45-4adf-a700-a9a6ad1724e9",
87+
"SPDXID": "SPDXRef-scancodeio-discovereddependency-pkg:pypi/pytest-asyncio?uuid=1ea67643-91ab-49af-a05d-d9d3f64506d1",
8888
"downloadLocation": "NOASSERTION",
8989
"licenseConcluded": "NOASSERTION",
9090
"copyrightText": "NOASSERTION",
@@ -100,7 +100,7 @@
100100
},
101101
{
102102
"name": "pytest-asyncio",
103-
"SPDXID": "SPDXRef-scancodeio-discovereddependency-pkg:pypi/pytest-asyncio?uuid=26af7b66-1034-4e21-9a2a-6dc1616a34ec",
103+
"SPDXID": "SPDXRef-scancodeio-discovereddependency-pkg:pypi/pytest-asyncio?uuid=c8e504e9-3581-4b3d-bd9a-ac815d4bd63b",
104104
"downloadLocation": "NOASSERTION",
105105
"licenseConcluded": "NOASSERTION",
106106
"copyrightText": "NOASSERTION",
@@ -116,33 +116,33 @@
116116
}
117117
],
118118
"documentDescribes": [
119-
"SPDXRef-scancodeio-discoveredpackage-7f975593-4bdd-4217-8d04-3554d691f310",
120-
"SPDXRef-scancodeio-discoveredpackage-edf75850-316d-4a46-9ca1-bde1f4fc3aab",
121-
"SPDXRef-scancodeio-discovereddependency-pkg:pypi/pytest?uuid=6ef94d27-ae57-408e-baa4-e20e3dc1f1aa",
122-
"SPDXRef-scancodeio-discovereddependency-pkg:pypi/pytest?uuid=4b195723-fe04-47c9-8b65-c08c60423c18",
123-
"SPDXRef-scancodeio-discovereddependency-pkg:pypi/pytest-asyncio?uuid=87beebac-2c45-4adf-a700-a9a6ad1724e9",
124-
"SPDXRef-scancodeio-discovereddependency-pkg:pypi/pytest-asyncio?uuid=26af7b66-1034-4e21-9a2a-6dc1616a34ec"
119+
"SPDXRef-scancodeio-discoveredpackage-d4227045-439f-41b8-98ff-067c55276b2f",
120+
"SPDXRef-scancodeio-discoveredpackage-92213ebf-c2f0-4bd6-9d4a-38ea138d2f63",
121+
"SPDXRef-scancodeio-discovereddependency-pkg:pypi/pytest?uuid=ebec93f1-d78e-49ee-a1f5-80c26a0dc32b",
122+
"SPDXRef-scancodeio-discovereddependency-pkg:pypi/pytest?uuid=cd04cbe8-ba3e-412b-9c1d-7eb6a617f1d1",
123+
"SPDXRef-scancodeio-discovereddependency-pkg:pypi/pytest-asyncio?uuid=1ea67643-91ab-49af-a05d-d9d3f64506d1",
124+
"SPDXRef-scancodeio-discovereddependency-pkg:pypi/pytest-asyncio?uuid=c8e504e9-3581-4b3d-bd9a-ac815d4bd63b"
125125
],
126126
"files": [],
127127
"relationships": [
128128
{
129-
"spdxElementId": "SPDXRef-scancodeio-discovereddependency-pkg:pypi/pytest?uuid=6ef94d27-ae57-408e-baa4-e20e3dc1f1aa",
130-
"relatedSpdxElement": "SPDXRef-scancodeio-discoveredpackage-7f975593-4bdd-4217-8d04-3554d691f310",
129+
"spdxElementId": "SPDXRef-scancodeio-discovereddependency-pkg:pypi/pytest?uuid=ebec93f1-d78e-49ee-a1f5-80c26a0dc32b",
130+
"relatedSpdxElement": "SPDXRef-scancodeio-discoveredpackage-d4227045-439f-41b8-98ff-067c55276b2f",
131131
"relationshipType": "DEPENDENCY_OF"
132132
},
133133
{
134-
"spdxElementId": "SPDXRef-scancodeio-discovereddependency-pkg:pypi/pytest?uuid=4b195723-fe04-47c9-8b65-c08c60423c18",
135-
"relatedSpdxElement": "SPDXRef-scancodeio-discoveredpackage-edf75850-316d-4a46-9ca1-bde1f4fc3aab",
134+
"spdxElementId": "SPDXRef-scancodeio-discovereddependency-pkg:pypi/pytest?uuid=cd04cbe8-ba3e-412b-9c1d-7eb6a617f1d1",
135+
"relatedSpdxElement": "SPDXRef-scancodeio-discoveredpackage-92213ebf-c2f0-4bd6-9d4a-38ea138d2f63",
136136
"relationshipType": "DEPENDENCY_OF"
137137
},
138138
{
139-
"spdxElementId": "SPDXRef-scancodeio-discovereddependency-pkg:pypi/pytest-asyncio?uuid=87beebac-2c45-4adf-a700-a9a6ad1724e9",
140-
"relatedSpdxElement": "SPDXRef-scancodeio-discoveredpackage-7f975593-4bdd-4217-8d04-3554d691f310",
139+
"spdxElementId": "SPDXRef-scancodeio-discovereddependency-pkg:pypi/pytest-asyncio?uuid=1ea67643-91ab-49af-a05d-d9d3f64506d1",
140+
"relatedSpdxElement": "SPDXRef-scancodeio-discoveredpackage-d4227045-439f-41b8-98ff-067c55276b2f",
141141
"relationshipType": "DEPENDENCY_OF"
142142
},
143143
{
144-
"spdxElementId": "SPDXRef-scancodeio-discovereddependency-pkg:pypi/pytest-asyncio?uuid=26af7b66-1034-4e21-9a2a-6dc1616a34ec",
145-
"relatedSpdxElement": "SPDXRef-scancodeio-discoveredpackage-edf75850-316d-4a46-9ca1-bde1f4fc3aab",
144+
"spdxElementId": "SPDXRef-scancodeio-discovereddependency-pkg:pypi/pytest-asyncio?uuid=c8e504e9-3581-4b3d-bd9a-ac815d4bd63b",
145+
"relatedSpdxElement": "SPDXRef-scancodeio-discoveredpackage-92213ebf-c2f0-4bd6-9d4a-38ea138d2f63",
146146
"relationshipType": "DEPENDENCY_OF"
147147
}
148148
],

0 commit comments

Comments
 (0)