Skip to content

Commit 4665b2d

Browse files
AyanSinhaMahapatraAyanSinhaMahapatra
committed
Add function to merge package data
Adds functions to merge package data from multiple package manifests into a package instance. Adds tests for a simple python manifests case. Signed-off-by: Ayan Sinha Mahapatra <ayansmahapatra@gmail.com>
1 parent 33fecf8 commit 4665b2d

File tree

4 files changed

+455
-0
lines changed

4 files changed

+455
-0
lines changed

‎src/packagedcode/models.py‎

Lines changed: 151 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -792,12 +792,33 @@ class PackageInstance:
792792
help='List of files provided by this package.'
793793
)
794794

795+
def get_package_data(self):
796+
"""
797+
Returns a mapping of package data attributes and corresponding values.
798+
"""
799+
mapping = self.to_dict()
800+
801+
# Removes PackageInstance specific attributes
802+
for attribute in ('package_uuid', 'package_manifest_paths', 'files'):
803+
mapping.pop(attribute)
804+
805+
# Remove attributes which are BasePackage functions
806+
for attribute in ('repository_homepage_url', 'repository_download_url', 'api_data_url'):
807+
mapping.pop(attribute)
808+
809+
return mapping
810+
811+
795812
def populate_instance_from_manifests(self, package_manifests_by_path, uuid):
796813
"""
797814
Create a package instance object from one or multiple package manifests.
798815
"""
799816
for path, package_manifest in package_manifests_by_path.items():
817+
if TRACE:
818+
logger.debug('Merging package manifest data for: {}'.format(path))
819+
logger.debug('package manifest data: {}'.format(repr(package_manifest)))
800820
self.package_manifest_paths.append(path)
821+
self.merge_package_data_into_instance(package_manifest)
801822

802823
self.package_manifest_paths = tuple(self.package_manifest_paths)
803824

@@ -858,6 +879,136 @@ def get_file_patterns(self, manifests):
858879

859880
return manifest_file_patterns
860881

882+
def merge_package_data_into_instance(self, package_manifest, replace=False):
883+
"""
884+
Merge the `package_manifest` ScannedPackage object into the `package_instance`
885+
Package model object.
886+
When an `package_instance` field has no value one side and and the
887+
package_manifest field has a value, the package_instance field is always
888+
set to this value.
889+
If `replace` is True and a field has a value on both sides, then
890+
package_instance field value will be replaced by the package_manifest
891+
field value. Otherwise if `replace` is False, the package_instance
892+
field value is left unchanged in this case.
893+
"""
894+
895+
896+
existing_mapping = self.get_package_data()
897+
898+
# Remove PackageManifest specific attributes
899+
for attribute in ('md5', 'sha1', 'sha256', 'sha512'):
900+
package_manifest.pop(attribute)
901+
existing_mapping.pop(attribute)
902+
903+
for existing_field, existing_value in existing_mapping.items():
904+
new_value = package_manifest[existing_field]
905+
if TRACE:
906+
logger.debug(
907+
'\n'.join([
908+
'existing_field:', repr(existing_field),
909+
' existing_value:', repr(existing_value),
910+
' new_value:', repr(new_value)])
911+
)
912+
913+
# FIXME: handle Booleans???
914+
915+
# These fields has to be same across the package_manifests
916+
if existing_field in ('name', 'version', 'type', 'primary_language'):
917+
if existing_value and new_value and existing_value != new_value:
918+
raise Exception(
919+
'\n'.join([
920+
'Mismatched {} for {}:'.format(existing_field, self.uri),
921+
' existing_value: {}'.format(existing_value),
922+
' new_value: {}'.format(new_value)
923+
])
924+
)
925+
926+
if not new_value:
927+
if TRACE:
928+
logger.debug(' No new value for {}: skipping'.format(existing_field))
929+
continue
930+
931+
if not existing_value or replace:
932+
if TRACE and not existing_value:
933+
logger.debug(
934+
' No existing value: set to new: {}'.format(new_value))
935+
936+
if TRACE and replace:
937+
logger.debug(
938+
' Existing value and replace: set to new: {}'.format(new_value))
939+
940+
if existing_field == 'parties':
941+
# If `existing_field` is `parties`, then we update the `Party` table
942+
parties = new_value
943+
parties_new = []
944+
945+
for party in parties:
946+
party_new = Party(
947+
type=party['type'],
948+
role=party['role'],
949+
name=party['name'],
950+
email=party['email'],
951+
url=party['url'],
952+
)
953+
parties_new.append(party_new)
954+
955+
if replace:
956+
setattr(self, existing_field, parties_new)
957+
else:
958+
existing_value.extend(parties_new)
959+
960+
elif existing_field == 'dependencies':
961+
# If `existing_field` is `dependencies`, then we update the `DependentPackage` table
962+
dependencies = new_value
963+
deps_new = []
964+
965+
for dependency in dependencies:
966+
dep_new = DependentPackage(
967+
purl=dependency['purl'],
968+
requirement=dependency['requirement'],
969+
scope=dependency['scope'],
970+
is_runtime=dependency['is_runtime'],
971+
is_optional=dependency['is_optional'],
972+
is_resolved=dependency['is_resolved'],
973+
)
974+
deps_new.append(dep_new)
975+
976+
if replace:
977+
setattr(self, existing_field, deps_new)
978+
else:
979+
existing_value.extend(deps_new)
980+
981+
elif existing_field == 'purl':
982+
self.set_purl(package_url=new_value)
983+
984+
elif existing_field == 'root_path':
985+
continue
986+
987+
else:
988+
# If `existing_field` is not `parties` or `dependencies`, then the
989+
# `existing_field` is a regular field on the Package model and can
990+
# be updated normally.
991+
if TRACE:
992+
logger.debug("Set value to self: {} at {}".format(new_value, existing_field))
993+
logger.debug("Set value to self: types: {} at {}".format(type(new_value), type(existing_field)))
994+
setattr(self, existing_field, new_value)
995+
# package_instance.save()
996+
997+
if existing_value and new_value and existing_value != new_value:
998+
# ToDo: What to do when conflicting values are present
999+
# license_expression: do AND?
1000+
if TRACE:
1001+
logger.debug("Value mismatch between new and existing: ")
1002+
logger.debug(
1003+
'\n'.join([
1004+
'existing_field:', repr(existing_field),
1005+
' existing_value:', repr(existing_value),
1006+
' new_value:', repr(new_value)])
1007+
)
1008+
1009+
if TRACE:
1010+
logger.debug(' Nothing done')
1011+
8611012

8621013
# Package types
8631014
# NOTE: this is somewhat redundant with extractcode archive handlers

‎tests/packagedcode/data/instance/python-manifests-click-scanned-result.json‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
{
2+
"type": "pypi",
3+
"namespace": null,
4+
"name": "click",
5+
"version": "8.0.1",
6+
"qualifiers": {},
7+
"subpath": null,
8+
"primary_language": "Python",
9+
"description": "Composable command line interface toolkit\n\\$ click\\_\n==========\n\nClick is a Python package for creating beautiful command line interfaces\nin a composable way with as little code as necessary. It's the \"Command\nLine Interface Creation Kit\". It's highly configurable but comes with\nsensible defaults out of the box.\n\nIt aims to make the process of writing command line tools quick and fun\nwhile also preventing any frustration caused by the inability to\nimplement an intended CLI API.\n\nClick in three points:\n\n- Arbitrary nesting of commands\n- Automatic help page generation\n- Supports lazy loading of subcommands at runtime\n\n\nInstalling\n----------\n\nInstall and update using `pip`_:\n\n.. code-block:: text\n\n $ pip install -U click\n\n.. _pip: https://pip.pypa.io/en/stable/quickstart/\n\n\nA Simple Example\n----------------\n\n.. code-block:: python\n\n import click\n\n @click.command()\n @click.option(\"--count\", default=1, help=\"Number of greetings.\")\n @click.option(\"--name\", prompt=\"Your name\", help=\"The person to greet.\")\n def hello(count, name):\n \"\"\"Simple program that greets NAME for a total of COUNT times.\"\"\"\n for _ in range(count):\n click.echo(f\"Hello, {name}!\")\n\n if __name__ == '__main__':\n hello()\n\n.. code-block:: text\n\n $ python hello.py --count=3\n Your name: Click\n Hello, Click!\n Hello, Click!\n Hello, Click!\n\n\nDonate\n------\n\nThe Pallets organization develops and supports Click and other popular\npackages. In order to grow the community of contributors and users, and\nallow the maintainers to devote more time to the projects, `please\ndonate today`_.\n\n.. _please donate today: https://palletsprojects.com/donate\n\n\nLinks\n-----\n\n- Documentation: https://click.palletsprojects.com/\n- Changes: https://click.palletsprojects.com/changes/\n- PyPI Releases: https://pypi.org/project/click/\n- Source Code: https://github.com/pallets/click\n- Issue Tracker: https://github.com/pallets/click/issues\n- Website: https://palletsprojects.com/p/click\n- Twitter: https://twitter.com/PalletsTeam\n- Chat: https://discord.gg/pallets",
10+
"release_date": null,
11+
"parties": [],
12+
"keywords": [
13+
"Development Status :: 5 - Production/Stable",
14+
"Intended Audience :: Developers",
15+
"Operating System :: OS Independent",
16+
"Programming Language :: Python"
17+
],
18+
"homepage_url": "https://palletsprojects.com/p/click/",
19+
"download_url": null,
20+
"size": null,
21+
"sha1": null,
22+
"md5": null,
23+
"sha256": null,
24+
"sha512": null,
25+
"bug_tracking_url": "Issue Tracker, https://github.com/pallets/click/issues/",
26+
"code_view_url": "Source Code, https://github.com/pallets/click/",
27+
"vcs_url": null,
28+
"copyright": null,
29+
"license_expression": "bsd-new",
30+
"declared_license": {
31+
"license": "BSD-3-Clause",
32+
"classifiers": [
33+
"License :: OSI Approved :: BSD License"
34+
]
35+
},
36+
"notice_text": null,
37+
"root_path": null,
38+
"dependencies": [],
39+
"contains_source_code": null,
40+
"source_packages": [],
41+
"extra_data": {},
42+
"package_uuid": null,
43+
"package_manifest_paths": [],
44+
"files": [],
45+
"purl": "pkg:pypi/click@8.0.1",
46+
"repository_homepage_url": "https://pypi.org/project/https://pypi.org",
47+
"repository_download_url": "https://pypi.org/packages/source/c/click/click-8.0.1.tar.gz",
48+
"api_data_url": "https://pypi.org/pypi/click/8.0.1/json"
49+
}

0 commit comments

Comments
 (0)