Skip to content

Commit d62f75c

Browse files
JonoYangJonoYang
committed
Create DiscoveredDependency model #411
* Create functions to process dependencies from scans * Modify DiscoveredPackages so we can relate DiscoveredDependencies to it Signed-off-by: Jono Yang <jyang@nexb.com>
1 parent 58d4909 commit d62f75c

File tree

4 files changed

+176
-4
lines changed

4 files changed

+176
-4
lines changed

‎scanpipe/migrations/0016_remove_discoveredpackage_dependencies_and_more.py‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
# Generated by Django 4.0.4 on 2022-05-06 19:23
2+
3+
from django.db import migrations, models
4+
import django.db.models.deletion
5+
import scanpipe.models
6+
7+
8+
class Migration(migrations.Migration):
9+
10+
dependencies = [
11+
('scanpipe', '0015_alter_codebaseresource_project_and_more'),
12+
]
13+
14+
operations = [
15+
migrations.RemoveField(
16+
model_name='discoveredpackage',
17+
name='dependencies',
18+
),
19+
migrations.CreateModel(
20+
name='DiscoveredDependency',
21+
fields=[
22+
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
23+
('purl', models.CharField(help_text='The Package URL of this dependency.', max_length=256)),
24+
('extracted_requirement', models.CharField(help_text='The version requirements of this dependency.', max_length=32)),
25+
('scope', models.CharField(help_text='The scope of this dependency, how it is used in a project.', max_length=32)),
26+
('is_runtime', models.BooleanField(default=False)),
27+
('is_optional', models.BooleanField(default=False)),
28+
('is_resolved', models.BooleanField(default=False)),
29+
('dependency_uid', models.CharField(help_text='The unique identifier of this dependency.', max_length=256)),
30+
('for_package_uid', models.CharField(help_text='The unique identifier of the package this dependency is for.', max_length=256)),
31+
('datafile_path', models.CharField(blank=True, help_text='The relative path to the datafile where this dependency was detected from.', max_length=1024)),
32+
('datasource_id', models.CharField(help_text='The identifier for the datafile handler used to obtain this dependency.', max_length=64)),
33+
('project', models.ForeignKey(editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='%(class)ss', to='scanpipe.project')),
34+
],
35+
options={
36+
'abstract': False,
37+
},
38+
bases=(models.Model, scanpipe.models.SaveProjectErrorMixin),
39+
),
40+
migrations.AddField(
41+
model_name='discoveredpackage',
42+
name='dependencies',
43+
field=models.ManyToManyField(related_name='for_packages', to='scanpipe.discovereddependency'),
44+
),
45+
]

‎scanpipe/models.py‎

Lines changed: 99 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1721,10 +1721,8 @@ class DiscoveredPackage(
17211721
)
17221722
missing_resources = models.JSONField(default=list, blank=True)
17231723
modified_resources = models.JSONField(default=list, blank=True)
1724-
dependencies = models.JSONField(
1725-
default=list,
1726-
blank=True,
1727-
help_text=_("A list of dependencies for this package."),
1724+
dependencies = models.ManyToManyField(
1725+
"DiscoveredDependency", related_name="for_packages"
17281726
)
17291727

17301728
# `AbstractPackage` model overrides:
@@ -1832,6 +1830,103 @@ def update_from_data(self, package_data):
18321830
return updated_fields
18331831

18341832

1833+
class DiscoveredDependency(
1834+
ProjectRelatedModel,
1835+
SaveProjectErrorMixin,
1836+
):
1837+
"""
1838+
A project's Discovered Dependencies are records of the dependencies used by
1839+
system and application packages discovered in the code under analysis.
1840+
"""
1841+
purl = models.CharField(
1842+
max_length=256,
1843+
help_text=_(
1844+
"The Package URL of this dependency."
1845+
),
1846+
)
1847+
extracted_requirement = models.CharField(
1848+
max_length=32,
1849+
help_text=_(
1850+
"The version requirements of this dependency."
1851+
),
1852+
)
1853+
scope = models.CharField(
1854+
max_length=32,
1855+
help_text=_(
1856+
"The scope of this dependency, how it is used in a project."
1857+
),
1858+
)
1859+
1860+
is_runtime = models.BooleanField(default=False)
1861+
is_optional = models.BooleanField(default=False)
1862+
is_resolved = models.BooleanField(default=False)
1863+
1864+
dependency_uid = models.CharField(
1865+
max_length=256,
1866+
help_text=_(
1867+
"The unique identifier of this dependency."
1868+
),
1869+
)
1870+
for_package_uid = models.CharField(
1871+
max_length=256,
1872+
help_text=_(
1873+
"The unique identifier of the package this dependency is for."
1874+
),
1875+
)
1876+
datafile_path = models.CharField(
1877+
max_length=1024,
1878+
blank=True,
1879+
help_text=_(
1880+
"The relative path to the datafile where this dependency was detected from."
1881+
),
1882+
)
1883+
datasource_id = models.CharField(
1884+
max_length=64,
1885+
help_text=_(
1886+
"The identifier for the datafile handler used to obtain this dependency."
1887+
)
1888+
)
1889+
1890+
@classmethod
1891+
def create_from_data(cls, project, dependency_data):
1892+
"""
1893+
Creates and returns a DiscoveredPackage for a `project` from the `dependency_data`.
1894+
"""
1895+
if "resolved_package" in dependency_data:
1896+
dependency_data.pop("resolved_package")
1897+
discovered_dependency = cls(project=project, **dependency_data)
1898+
discovered_dependency.save()
1899+
return discovered_dependency
1900+
1901+
def update_from_data(self, dependency_data):
1902+
"""
1903+
Update this discovered dependency instance with the provided `dependency_data`.
1904+
The `save()` is called only if at least one field was modified.
1905+
"""
1906+
model_fields = DiscoveredPackage.model_fields()
1907+
updated_fields = []
1908+
1909+
for field_name, value in dependency_data.items():
1910+
skip_reasons = [
1911+
not value,
1912+
field_name not in model_fields,
1913+
]
1914+
if any(skip_reasons):
1915+
continue
1916+
1917+
current_value = getattr(self, field_name, None)
1918+
if not current_value:
1919+
setattr(self, field_name, value)
1920+
updated_fields.append(field_name)
1921+
elif current_value != value:
1922+
pass # TODO: handle this case
1923+
1924+
if updated_fields:
1925+
self.save()
1926+
1927+
return updated_fields
1928+
1929+
18351930
class WebhookSubscription(UUIDPKModel, ProjectRelatedModel):
18361931
target_url = models.URLField(_("Target URL"), max_length=1024)
18371932
sent = models.BooleanField(default=False)

‎scanpipe/pipes/__init__.py‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@
3232
from packageurl import normalize_qualifiers
3333

3434
from scanpipe.models import CodebaseResource
35+
from scanpipe.models import DiscoveredDependency
3536
from scanpipe.models import DiscoveredPackage
3637
from scanpipe.pipes import scancode
3738

@@ -69,6 +70,28 @@ def make_codebase_resource(project, location, rootfs_path=None):
6970
codebase_resource.save(save_error=False)
7071

7172

73+
def update_or_create_dependency(project, dependency_data):
74+
"""
75+
Gets, updates, or creates a DiscoveredDependency then returns it.
76+
Uses the `project` and `dependency_data` mapping to lookup and creates
77+
the DiscoveredDependency.
78+
"""
79+
if "resolved_package" in dependency_data:
80+
dependency_data.pop("resolved_package")
81+
82+
try:
83+
dependency = DiscoveredDependency.objects.get(project=project, **dependency_data)
84+
except DiscoveredDependency.DoesNotExist:
85+
dependency = None
86+
87+
if dependency:
88+
dependency.update_from_data(dependency_data)
89+
else:
90+
dependency = DiscoveredDependency.create_from_data(project, dependency_data)
91+
92+
return dependency
93+
94+
7295
def update_or_create_package(project, package_data, codebase_resource=None):
7396
"""
7497
Gets, updates or creates a DiscoveredPackage then returns it.
@@ -98,6 +121,11 @@ def update_or_create_package(project, package_data, codebase_resource=None):
98121
package_uids.append(package_uid)
99122
package.update_extra_data({"package_uids": package_uids})
100123

124+
# Associate all dependencies to this package
125+
dependencies = DiscoveredDependency.objects.filter(project=project, for_package_uid=package_uid)
126+
for dependency in dependencies:
127+
package.dependencies.add(dependency)
128+
101129
return package
102130

103131

‎scanpipe/pipes/scancode.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -404,6 +404,10 @@ def create_discovered_packages(project, scanned_codebase):
404404
Saves the packages of a ScanCode `scanned_codebase` scancode.resource.Codebase
405405
object to the database as a DiscoveredPackage of `project`.
406406
"""
407+
if hasattr(scanned_codebase.attributes, "dependencies"):
408+
for dependency_data in scanned_codebase.attributes.dependencies:
409+
pipes.update_or_create_dependency(project, dependency_data)
410+
407411
if hasattr(scanned_codebase.attributes, "packages"):
408412
for package_data in scanned_codebase.attributes.packages:
409413
pipes.update_or_create_package(project, package_data)

0 commit comments

Comments
 (0)