Skip to content

Commit 3105715

Browse files
JonoYangJonoYang
authored
Update ProjectCodebase and root_resources #624
* Remove root property and create root_resources * Update walk to walk from root resources * Update tests Signed-off-by: Jono Yang <jyang@nexb.com>
1 parent c2d8f0d commit 3105715

File tree

9 files changed

+1067
-826
lines changed

9 files changed

+1067
-826
lines changed

‎scanpipe/pipes/codebase.py‎

Lines changed: 62 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,14 @@ def sort_by_lower_name(resource):
2929
return resource["name"].lower()
3030

3131

32-
def get_tree(resource, fields, codebase=None):
32+
def get_resource_fields(resource, fields):
33+
"""
34+
Return a mapping of fields from `fields` and values from `resource`
35+
"""
36+
return {field: getattr(resource, field) for field in fields}
37+
38+
39+
def get_resource_tree(resource, fields, codebase=None, seen_resources=set()):
3340
"""
3441
Returns a tree as a dictionary structure starting from the provided `resource`.
3542
@@ -38,23 +45,61 @@ def get_tree(resource, fields, codebase=None):
3845
- commoncode.resource.Resource
3946
4047
The data included for each child is controlled with the `fields` argument.
48+
4149
The `codebase` is only required in the context of a commoncode `Resource`
4250
input.
51+
52+
`seen_resources` is used when get_resource_tree() is used in the context of
53+
get_codebase_tree(). We keep track of child Resources we visit in
54+
`seen_resources`, so we don't visit them again in get_codebase_tree().
4355
"""
44-
resource_dict = {field: getattr(resource, field) for field in fields}
56+
resource_dict = get_resource_fields(resource, fields)
4557

4658
if resource.is_dir:
47-
children = [
48-
get_tree(child, fields, codebase) for child in resource.children(codebase)
49-
]
59+
children = []
60+
for child in resource.children(codebase):
61+
seen_resources.add(child.path)
62+
children.append(get_resource_tree(child, fields, codebase, seen_resources))
5063
if children:
5164
resource_dict["children"] = sorted(children, key=sort_by_lower_name)
5265

5366
return resource_dict
5467

5568

56-
# TODO: Walking the ProjectCodebase is broken as we do not have a consistent way
57-
# to get the root of a codebase.
69+
def get_codebase_tree(codebase, fields):
70+
"""
71+
Returns a tree as a dictionary structure starting from the root resources of
72+
the provided `codebase`.
73+
74+
The following classes are supported for the input `codebase` object:
75+
- scanpipe.pipes.codebase.ProjectCodebase
76+
- commoncode.resource.Codebase
77+
- commoncode.resource.VirtualCodebase
78+
79+
The data included for each child is controlled with the `fields` argument.
80+
"""
81+
seen_resources = set()
82+
codebase_dict = dict(children=[])
83+
for resource in codebase.walk():
84+
path = resource.path
85+
if path in seen_resources:
86+
continue
87+
else:
88+
seen_resources.add(path)
89+
resource_dict = get_resource_fields(resource, fields)
90+
if resource.is_dir:
91+
children = []
92+
for child in resource.children(codebase):
93+
seen_resources.add(child.path)
94+
children.append(
95+
get_resource_tree(child, fields, codebase, seen_resources)
96+
)
97+
if children:
98+
resource_dict["children"] = sorted(children, key=sort_by_lower_name)
99+
codebase_dict["children"].append(resource_dict)
100+
return codebase_dict
101+
102+
58103
class ProjectCodebase:
59104
"""
60105
Represents the codebase of a project stored in the database.
@@ -68,24 +113,21 @@ def __init__(self, project):
68113
self.project = project
69114

70115
@property
71-
def root(self):
72-
try:
73-
return self.project.codebaseresources.get(path="codebase")
74-
except ObjectDoesNotExist:
75-
raise AttributeError("Codebase root cannot be determined.")
116+
def root_resources(self):
117+
return self.project.codebaseresources.exclude(path__contains="/")
76118

77119
@property
78120
def resources(self):
79121
return self.project.codebaseresources.all()
80122

81123
def walk(self, topdown=True):
82-
root = self.root
83-
if topdown:
84-
yield root
85-
for resource in root.walk(topdown=topdown):
86-
yield resource
87-
if not topdown:
88-
yield root
124+
for root_resource in self.root_resources:
125+
if topdown:
126+
yield root_resource
127+
for resource in root_resource.walk(topdown=topdown):
128+
yield resource
129+
if not topdown:
130+
yield root_resource
89131

90132
def get_tree(self):
91-
return get_tree(self.root, fields=["name", "path"])
133+
return get_codebase_tree(self, fields=["name", "path"])

‎scanpipe/tests/data/asgiref-3.3.0.spdx.json‎

Lines changed: 21 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
"dataLicense": "CC0-1.0",
44
"SPDXID": "SPDXRef-DOCUMENT",
55
"name": "scancodeio_asgiref",
6-
"documentNamespace": "https://scancode.io/spdxdocs/694e35cd-7d28-40ba-b504-e9580b819427",
6+
"documentNamespace": "https://scancode.io/spdxdocs/4c7f2e2e-0c41-45d2-a6cc-7bbde9a9f440",
77
"creationInfo": {
88
"created": "2000-01-01T01:02:03Z",
99
"creators": [
@@ -14,7 +14,7 @@
1414
"packages": [
1515
{
1616
"name": "asgiref",
17-
"SPDXID": "SPDXRef-scancodeio-discoveredpackage-7f975593-4bdd-4217-8d04-3554d691f310",
17+
"SPDXID": "SPDXRef-scancodeio-discoveredpackage-d4227045-439f-41b8-98ff-067c55276b2f",
1818
"downloadLocation": "NOASSERTION",
1919
"licenseConcluded": "BSD-3-Clause AND BSD-3-Clause",
2020
"copyrightText": "NOASSERTION",
@@ -33,7 +33,7 @@
3333
},
3434
{
3535
"name": "asgiref",
36-
"SPDXID": "SPDXRef-scancodeio-discoveredpackage-edf75850-316d-4a46-9ca1-bde1f4fc3aab",
36+
"SPDXID": "SPDXRef-scancodeio-discoveredpackage-92213ebf-c2f0-4bd6-9d4a-38ea138d2f63",
3737
"downloadLocation": "NOASSERTION",
3838
"licenseConcluded": "BSD-3-Clause AND BSD-3-Clause",
3939
"copyrightText": "NOASSERTION",
@@ -52,7 +52,7 @@
5252
},
5353
{
5454
"name": "pytest",
55-
"SPDXID": "SPDXRef-scancodeio-discovereddependency-pkg:pypi/pytest?uuid=6ef94d27-ae57-408e-baa4-e20e3dc1f1aa",
55+
"SPDXID": "SPDXRef-scancodeio-discovereddependency-pkg:pypi/pytest?uuid=ebec93f1-d78e-49ee-a1f5-80c26a0dc32b",
5656
"downloadLocation": "NOASSERTION",
5757
"licenseConcluded": "NOASSERTION",
5858
"copyrightText": "NOASSERTION",
@@ -68,7 +68,7 @@
6868
},
6969
{
7070
"name": "pytest",
71-
"SPDXID": "SPDXRef-scancodeio-discovereddependency-pkg:pypi/pytest?uuid=4b195723-fe04-47c9-8b65-c08c60423c18",
71+
"SPDXID": "SPDXRef-scancodeio-discovereddependency-pkg:pypi/pytest?uuid=cd04cbe8-ba3e-412b-9c1d-7eb6a617f1d1",
7272
"downloadLocation": "NOASSERTION",
7373
"licenseConcluded": "NOASSERTION",
7474
"copyrightText": "NOASSERTION",
@@ -84,7 +84,7 @@
8484
},
8585
{
8686
"name": "pytest-asyncio",
87-
"SPDXID": "SPDXRef-scancodeio-discovereddependency-pkg:pypi/pytest-asyncio?uuid=87beebac-2c45-4adf-a700-a9a6ad1724e9",
87+
"SPDXID": "SPDXRef-scancodeio-discovereddependency-pkg:pypi/pytest-asyncio?uuid=1ea67643-91ab-49af-a05d-d9d3f64506d1",
8888
"downloadLocation": "NOASSERTION",
8989
"licenseConcluded": "NOASSERTION",
9090
"copyrightText": "NOASSERTION",
@@ -100,7 +100,7 @@
100100
},
101101
{
102102
"name": "pytest-asyncio",
103-
"SPDXID": "SPDXRef-scancodeio-discovereddependency-pkg:pypi/pytest-asyncio?uuid=26af7b66-1034-4e21-9a2a-6dc1616a34ec",
103+
"SPDXID": "SPDXRef-scancodeio-discovereddependency-pkg:pypi/pytest-asyncio?uuid=c8e504e9-3581-4b3d-bd9a-ac815d4bd63b",
104104
"downloadLocation": "NOASSERTION",
105105
"licenseConcluded": "NOASSERTION",
106106
"copyrightText": "NOASSERTION",
@@ -116,33 +116,33 @@
116116
}
117117
],
118118
"documentDescribes": [
119-
"SPDXRef-scancodeio-discoveredpackage-7f975593-4bdd-4217-8d04-3554d691f310",
120-
"SPDXRef-scancodeio-discoveredpackage-edf75850-316d-4a46-9ca1-bde1f4fc3aab",
121-
"SPDXRef-scancodeio-discovereddependency-pkg:pypi/pytest?uuid=6ef94d27-ae57-408e-baa4-e20e3dc1f1aa",
122-
"SPDXRef-scancodeio-discovereddependency-pkg:pypi/pytest?uuid=4b195723-fe04-47c9-8b65-c08c60423c18",
123-
"SPDXRef-scancodeio-discovereddependency-pkg:pypi/pytest-asyncio?uuid=87beebac-2c45-4adf-a700-a9a6ad1724e9",
124-
"SPDXRef-scancodeio-discovereddependency-pkg:pypi/pytest-asyncio?uuid=26af7b66-1034-4e21-9a2a-6dc1616a34ec"
119+
"SPDXRef-scancodeio-discoveredpackage-d4227045-439f-41b8-98ff-067c55276b2f",
120+
"SPDXRef-scancodeio-discoveredpackage-92213ebf-c2f0-4bd6-9d4a-38ea138d2f63",
121+
"SPDXRef-scancodeio-discovereddependency-pkg:pypi/pytest?uuid=ebec93f1-d78e-49ee-a1f5-80c26a0dc32b",
122+
"SPDXRef-scancodeio-discovereddependency-pkg:pypi/pytest?uuid=cd04cbe8-ba3e-412b-9c1d-7eb6a617f1d1",
123+
"SPDXRef-scancodeio-discovereddependency-pkg:pypi/pytest-asyncio?uuid=1ea67643-91ab-49af-a05d-d9d3f64506d1",
124+
"SPDXRef-scancodeio-discovereddependency-pkg:pypi/pytest-asyncio?uuid=c8e504e9-3581-4b3d-bd9a-ac815d4bd63b"
125125
],
126126
"files": [],
127127
"relationships": [
128128
{
129-
"spdxElementId": "SPDXRef-scancodeio-discovereddependency-pkg:pypi/pytest?uuid=6ef94d27-ae57-408e-baa4-e20e3dc1f1aa",
130-
"relatedSpdxElement": "SPDXRef-scancodeio-discoveredpackage-7f975593-4bdd-4217-8d04-3554d691f310",
129+
"spdxElementId": "SPDXRef-scancodeio-discovereddependency-pkg:pypi/pytest?uuid=ebec93f1-d78e-49ee-a1f5-80c26a0dc32b",
130+
"relatedSpdxElement": "SPDXRef-scancodeio-discoveredpackage-d4227045-439f-41b8-98ff-067c55276b2f",
131131
"relationshipType": "DEPENDENCY_OF"
132132
},
133133
{
134-
"spdxElementId": "SPDXRef-scancodeio-discovereddependency-pkg:pypi/pytest?uuid=4b195723-fe04-47c9-8b65-c08c60423c18",
135-
"relatedSpdxElement": "SPDXRef-scancodeio-discoveredpackage-edf75850-316d-4a46-9ca1-bde1f4fc3aab",
134+
"spdxElementId": "SPDXRef-scancodeio-discovereddependency-pkg:pypi/pytest?uuid=cd04cbe8-ba3e-412b-9c1d-7eb6a617f1d1",
135+
"relatedSpdxElement": "SPDXRef-scancodeio-discoveredpackage-92213ebf-c2f0-4bd6-9d4a-38ea138d2f63",
136136
"relationshipType": "DEPENDENCY_OF"
137137
},
138138
{
139-
"spdxElementId": "SPDXRef-scancodeio-discovereddependency-pkg:pypi/pytest-asyncio?uuid=87beebac-2c45-4adf-a700-a9a6ad1724e9",
140-
"relatedSpdxElement": "SPDXRef-scancodeio-discoveredpackage-7f975593-4bdd-4217-8d04-3554d691f310",
139+
"spdxElementId": "SPDXRef-scancodeio-discovereddependency-pkg:pypi/pytest-asyncio?uuid=1ea67643-91ab-49af-a05d-d9d3f64506d1",
140+
"relatedSpdxElement": "SPDXRef-scancodeio-discoveredpackage-d4227045-439f-41b8-98ff-067c55276b2f",
141141
"relationshipType": "DEPENDENCY_OF"
142142
},
143143
{
144-
"spdxElementId": "SPDXRef-scancodeio-discovereddependency-pkg:pypi/pytest-asyncio?uuid=26af7b66-1034-4e21-9a2a-6dc1616a34ec",
145-
"relatedSpdxElement": "SPDXRef-scancodeio-discoveredpackage-edf75850-316d-4a46-9ca1-bde1f4fc3aab",
144+
"spdxElementId": "SPDXRef-scancodeio-discovereddependency-pkg:pypi/pytest-asyncio?uuid=c8e504e9-3581-4b3d-bd9a-ac815d4bd63b",
145+
"relatedSpdxElement": "SPDXRef-scancodeio-discoveredpackage-92213ebf-c2f0-4bd6-9d4a-38ea138d2f63",
146146
"relationshipType": "DEPENDENCY_OF"
147147
}
148148
],

0 commit comments

Comments
 (0)