Skip to content

Commit 7462158

Browse files
JonoYangJonoYang
authored
Merge pull request #2887 from nexB/fix-2886-packages
Do not raise exception on package data mismatch #2886
2 parents 86c7db5 + 65a0fc5 commit 7462158

File tree

12 files changed

+1092
-10
lines changed

12 files changed

+1092
-10
lines changed

‎src/packagedcode/models.py‎

Lines changed: 32 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -86,14 +86,14 @@
8686
SCANCODE_DEBUG_PACKAGE_API = os.environ.get('SCANCODE_DEBUG_PACKAGE_API', False)
8787

8888
TRACE = False or SCANCODE_DEBUG_PACKAGE_API
89-
TRACE_MERGING = True or SCANCODE_DEBUG_PACKAGE_API
89+
TRACE_MERGING = False or SCANCODE_DEBUG_PACKAGE_API
9090

9191
def logger_debug(*args):
9292
pass
9393

9494
logger = logging.getLogger(__name__)
9595

96-
if TRACE:
96+
if TRACE or TRACE_MERGING:
9797
import logging
9898

9999
logging.basicConfig(stream=sys.stdout)
@@ -838,12 +838,26 @@ def populate_package_data(self, package_data_by_path, uuid):
838838
"""
839839
Create a package instance object from one or multiple package data.
840840
"""
841+
paths_with_mismatch = []
842+
841843
for path, package_data in package_data_by_path.items():
844+
success = True
842845
if TRACE:
843846
logger.debug('Merging package manifest data for: {}'.format(path))
844847
logger.debug('package manifest data: {}'.format(repr(package_data)))
848+
849+
success = self.update(package_data.copy())
850+
if not success:
851+
paths_with_mismatch.append(path)
852+
853+
if TRACE:
854+
logger.debug('Failed to merge package manifest data for: {}'.format(path))
855+
continue
856+
845857
self.package_data_files.append(path)
846-
self.update(package_data.copy())
858+
859+
for path in paths_with_mismatch:
860+
package_data_by_path.pop(path)
847861

848862
self.package_data_files = tuple(self.package_data_files)
849863

@@ -931,6 +945,9 @@ def get_file_patterns(self, manifests):
931945

932946
def update(self, package_data, replace=False):
933947
"""
948+
Returns False if there's a type/name/version mismatch between the package
949+
and the package_data, otherwise returns True if update is successful.
950+
934951
Update the Package object with data from the `package_data`
935952
object.
936953
When an `package_instance` field has no value one side and and the
@@ -963,13 +980,15 @@ def update(self, package_data, replace=False):
963980
# These fields has to be same across the package_data
964981
if existing_field in ('name', 'version', 'type', 'primary_language'):
965982
if existing_value and new_value and existing_value != new_value:
966-
raise Exception(
967-
'\n'.join([
968-
'Mismatched {}:'.format(existing_field),
969-
' existing_value: {}'.format(existing_value),
970-
' new_value: {}'.format(new_value)
971-
])
972-
)
983+
if TRACE_MERGING:
984+
logger.debug(
985+
'\n'.join([
986+
'Mismatched {}:'.format(existing_field),
987+
' existing_value: {}'.format(existing_value),
988+
' new_value: {}'.format(new_value)
989+
])
990+
)
991+
return False
973992

974993
if not new_value:
975994
if TRACE_MERGING:
@@ -1059,6 +1078,9 @@ def update(self, package_data, replace=False):
10591078
if TRACE_MERGING:
10601079
logger.debug(' Nothing done')
10611080

1081+
# Return True if package data update is successful
1082+
return True
1083+
10621084

10631085
# Package types
10641086
# NOTE: this is somewhat redundant with extractcode archive handlers

‎tests/packagedcode/data/instance/pypi-with-test-manifests/LICENSE‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
Copyright Jason R. Coombs
2+
3+
Permission is hereby granted, free of charge, to any person obtaining a copy
4+
of this software and associated documentation files (the "Software"), to
5+
deal in the Software without restriction, including without limitation the
6+
rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
7+
sell copies of the Software, and to permit persons to whom the Software is
8+
furnished to do so, subject to the following conditions:
9+
10+
The above copyright notice and this permission notice shall be included in
11+
all copies or substantial portions of the Software.
12+
13+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
16+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
17+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
18+
FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
19+
IN THE SOFTWARE.

‎tests/packagedcode/data/instance/pypi-with-test-manifests/MANIFEST.in‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
recursive-include setuptools *.py *.exe *.xml *.tmpl
2+
recursive-include tests *.py
3+
recursive-include setuptools/tests *.html
4+
recursive-include docs *.py *.txt *.rst *.conf *.css *.css_t Makefile indexsidebar.html
5+
recursive-include setuptools/_vendor *.py *.txt
6+
recursive-include pkg_resources *.py *.txt
7+
recursive-include pkg_resources/tests/data *
8+
recursive-include tools *
9+
recursive-include changelog.d *
10+
include *.py
11+
include *.rst
12+
include MANIFEST.in
13+
include LICENSE
14+
include launcher.c
15+
include msvc-build-launcher.cmd
16+
include pytest.ini
17+
include tox.ini

‎tests/packagedcode/data/instance/pypi-with-test-manifests/PKG-INFO‎

Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,93 @@
1+
Metadata-Version: 2.1
2+
Name: setuptools
3+
Version: 58.2.0
4+
Summary: Easily download, build, install, upgrade, and uninstall Python packages
5+
Home-page: https://github.com/pypa/setuptools
6+
Author: Python Packaging Authority
7+
Author-email: distutils-sig@python.org
8+
License: UNKNOWN
9+
Project-URL: Documentation, https://setuptools.readthedocs.io/
10+
Keywords: CPAN PyPI distutils eggs package management
11+
Platform: UNKNOWN
12+
Classifier: Development Status :: 5 - Production/Stable
13+
Classifier: Intended Audience :: Developers
14+
Classifier: License :: OSI Approved :: MIT License
15+
Classifier: Programming Language :: Python :: 3
16+
Classifier: Programming Language :: Python :: 3 :: Only
17+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
18+
Classifier: Topic :: System :: Archiving :: Packaging
19+
Classifier: Topic :: System :: Systems Administration
20+
Classifier: Topic :: Utilities
21+
Requires-Python: >=3.6
22+
Provides-Extra: testing
23+
Provides-Extra: docs
24+
Provides-Extra: ssl
25+
Provides-Extra: certs
26+
License-File: LICENSE
27+
28+
.. image:: https://img.shields.io/pypi/v/setuptools.svg
29+
:target: `PyPI link`_
30+
31+
.. image:: https://img.shields.io/pypi/pyversions/setuptools.svg
32+
:target: `PyPI link`_
33+
34+
.. _PyPI link: https://pypi.org/project/setuptools
35+
36+
.. image:: https://github.com/pypa/setuptools/workflows/tests/badge.svg
37+
:target: https://github.com/pypa/setuptools/actions?query=workflow%3A%22tests%22
38+
:alt: tests
39+
40+
.. image:: https://img.shields.io/badge/code%20style-black-000000.svg
41+
:target: https://github.com/psf/black
42+
:alt: Code style: Black
43+
44+
.. image:: https://img.shields.io/readthedocs/setuptools/latest.svg
45+
:target: https://setuptools.readthedocs.io
46+
47+
.. image:: https://img.shields.io/badge/skeleton-2021-informational
48+
:target: https://blog.jaraco.com/skeleton
49+
50+
.. image:: https://img.shields.io/codecov/c/github/pypa/setuptools/master.svg?logo=codecov&logoColor=white
51+
:target: https://codecov.io/gh/pypa/setuptools
52+
53+
.. image:: https://tidelift.com/badges/github/pypa/setuptools?style=flat
54+
:target: https://tidelift.com/subscription/pkg/pypi-setuptools?utm_source=pypi-setuptools&utm_medium=readme
55+
56+
See the `Installation Instructions
57+
<https://packaging.python.org/installing/>`_ in the Python Packaging
58+
User's Guide for instructions on installing, upgrading, and uninstalling
59+
Setuptools.
60+
61+
Questions and comments should be directed to the `distutils-sig
62+
mailing list <http://mail.python.org/pipermail/distutils-sig/>`_.
63+
Bug reports and especially tested patches may be
64+
submitted directly to the `bug tracker
65+
<https://github.com/pypa/setuptools/issues>`_.
66+
67+
68+
Code of Conduct
69+
===============
70+
71+
Everyone interacting in the setuptools project's codebases, issue trackers,
72+
chat rooms, and mailing lists is expected to follow the
73+
`PSF Code of Conduct <https://github.com/pypa/.github/blob/main/CODE_OF_CONDUCT.md>`_.
74+
75+
76+
For Enterprise
77+
==============
78+
79+
Available as part of the Tidelift Subscription.
80+
81+
Setuptools and the maintainers of thousands of other packages are working with Tidelift to deliver one enterprise subscription that covers all of the open source you use.
82+
83+
`Learn more <https://tidelift.com/subscription/pkg/pypi-setuptools?utm_source=pypi-setuptools&utm_medium=referral&utm_campaign=github>`_.
84+
85+
86+
Security Contact
87+
================
88+
89+
To report a security vulnerability, please use the
90+
`Tidelift security contact <https://tidelift.com/security>`_.
91+
Tidelift will coordinate the fix and disclosure.
92+
93+

‎tests/packagedcode/data/instance/pypi-with-test-manifests/README.rst‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
.. image:: https://img.shields.io/pypi/v/setuptools.svg
2+
:target: `PyPI link`_
3+
4+
.. image:: https://img.shields.io/pypi/pyversions/setuptools.svg
5+
:target: `PyPI link`_
6+
7+
.. _PyPI link: https://pypi.org/project/setuptools
8+
9+
.. image:: https://github.com/pypa/setuptools/workflows/tests/badge.svg
10+
:target: https://github.com/pypa/setuptools/actions?query=workflow%3A%22tests%22
11+
:alt: tests
12+
13+
.. image:: https://img.shields.io/badge/code%20style-black-000000.svg
14+
:target: https://github.com/psf/black
15+
:alt: Code style: Black
16+
17+
.. image:: https://img.shields.io/readthedocs/setuptools/latest.svg
18+
:target: https://setuptools.readthedocs.io
19+
20+
.. image:: https://img.shields.io/badge/skeleton-2021-informational
21+
:target: https://blog.jaraco.com/skeleton
22+
23+
.. image:: https://img.shields.io/codecov/c/github/pypa/setuptools/master.svg?logo=codecov&logoColor=white
24+
:target: https://codecov.io/gh/pypa/setuptools
25+
26+
.. image:: https://tidelift.com/badges/github/pypa/setuptools?style=flat
27+
:target: https://tidelift.com/subscription/pkg/pypi-setuptools?utm_source=pypi-setuptools&utm_medium=readme
28+
29+
See the `Installation Instructions
30+
<https://packaging.python.org/installing/>`_ in the Python Packaging
31+
User's Guide for instructions on installing, upgrading, and uninstalling
32+
Setuptools.
33+
34+
Questions and comments should be directed to the `distutils-sig
35+
mailing list <http://mail.python.org/pipermail/distutils-sig/>`_.
36+
Bug reports and especially tested patches may be
37+
submitted directly to the `bug tracker
38+
<https://github.com/pypa/setuptools/issues>`_.
39+
40+
41+
Code of Conduct
42+
===============
43+
44+
Everyone interacting in the setuptools project's codebases, issue trackers,
45+
chat rooms, and mailing lists is expected to follow the
46+
`PSF Code of Conduct <https://github.com/pypa/.github/blob/main/CODE_OF_CONDUCT.md>`_.
47+
48+
49+
For Enterprise
50+
==============
51+
52+
Available as part of the Tidelift Subscription.
53+
54+
Setuptools and the maintainers of thousands of other packages are working with Tidelift to deliver one enterprise subscription that covers all of the open source you use.
55+
56+
`Learn more <https://tidelift.com/subscription/pkg/pypi-setuptools?utm_source=pypi-setuptools&utm_medium=referral&utm_campaign=github>`_.
57+
58+
59+
Security Contact
60+
================
61+
62+
To report a security vulnerability, please use the
63+
`Tidelift security contact <https://tidelift.com/security>`_.
64+
Tidelift will coordinate the fix and disclosure.

‎tests/packagedcode/data/instance/pypi-with-test-manifests/pkg_resources/tests/data/my-test-package-zip/my-test-package.zip‎

1.77 KB
Binary file not shown.

‎tests/packagedcode/data/instance/pypi-with-test-manifests/pyproject.toml‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
[build-system]
2+
requires = [
3+
"wheel",
4+
]
5+
build-backend = "setuptools.build_meta"
6+
backend-path = ["."]
7+
8+
[tool.black]
9+
skip-string-normalization = true
10+
11+
[tool.setuptools_scm]
12+
13+
[pytest.enabler.black]
14+
#addopts = "--black"
15+
16+
[pytest.enabler.mypy]
17+
#addopts = "--mypy"
18+
19+
[pytest.enabler.flake8]
20+
addopts = "--flake8"
21+
22+
[pytest.enabler.cov]
23+
addopts = "--cov"
24+
25+
[pytest.enabler.xdist]
26+
addopts = "-n auto"
27+
28+
[tool.towncrier]
29+
package = "setuptools"
30+
package_dir = "setuptools"
31+
filename = "CHANGES.rst"
32+
directory = "changelog.d"
33+
title_format = "v{version}"
34+
issue_format = "#{issue}"
35+
template = "towncrier_template.rst"
36+
underlines = ["-", "^"]
37+
38+
[[tool.towncrier.type]]
39+
directory = "deprecation"
40+
name = "Deprecations"
41+
showcontent = true
42+
43+
[[tool.towncrier.type]]
44+
directory = "breaking"
45+
name = "Breaking Changes"
46+
showcontent = true
47+
48+
[[tool.towncrier.type]]
49+
directory = "change"
50+
name = "Changes"
51+
showcontent = true
52+
53+
[[tool.towncrier.type]]
54+
directory = "doc"
55+
name = "Documentation changes"
56+
showcontent = true
57+
58+
[[tool.towncrier.type]]
59+
directory = "misc"
60+
name = "Misc"
61+
showcontent = true

0 commit comments

Comments
 (0)