Skip to content

Commit 986e75f

Browse files
TG1999TG1999
committed
Fix github improver
Signed-off-by: Tushar Goel <tushar.goel.dav@gmail.com>
1 parent a59b5e4 commit 986e75f

File tree

6 files changed

+200
-19
lines changed

6 files changed

+200
-19
lines changed

‎vulnerabilities/helpers.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -178,7 +178,8 @@ def nearest_patched_package(
178178

179179
affected_package_with_patched_package_objects.append(
180180
AffectedPackage(
181-
vulnerable_package=vulnerable_package.purl, patched_package=patched_package.purl
181+
vulnerable_package=vulnerable_package.purl,
182+
patched_package=patched_package.purl if patched_package else None,
182183
)
183184
)
184185

‎vulnerabilities/importers/github.py‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -527,11 +527,10 @@ def resolve_version_range(
527527
for package_version in package_versions or []:
528528
if package_version in ignorable_versions:
529529
continue
530-
# Remove leading 'v'
531-
if package_version.startswith("v") or package_version.startswith("V"):
532-
package_version = package_version.replace("V", "").replace("v", "")
533530
# Remove whitespace
534531
package_version = package_version.replace(" ", "")
532+
# Remove leading 'v'
533+
package_version = package_version.lstrip("v").lstrip("V")
535534
try:
536535
version = affected_version_range.version_class(package_version)
537536
except Exception:

‎vulnerabilities/package_managers_2.py‎

Lines changed: 32 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
from dateutil import parser as dateparser
1414
from django.utils.dateparse import parse_datetime
1515

16+
from vulnerabilities.helpers import get_item
1617
from vulnerabilities.package_managers import VersionResponse
1718

1819
LOGGER = logging.getLogger(__name__)
@@ -31,7 +32,11 @@ class VersionResponse:
3132

3233

3334
def get_response(url, type="json"):
34-
resp = requests.get(url=url)
35+
try:
36+
resp = requests.get(url=url)
37+
except:
38+
LOGGER.error(traceback.format_exc())
39+
return None
3540
if not resp.status_code == 200:
3641
LOGGER.error(f"Error while fetching {url}: {resp.status_code}")
3742
return None
@@ -86,20 +91,23 @@ def fetch(self, pkg):
8691
self.cache[pkg] = versions
8792
return
8893

89-
for version, download_items in response["releases"].items() or {}:
94+
releases = response.get("releases") or {}
95+
for version, download_items in releases.items():
9096
if download_items:
9197
latest_download_item = max(
9298
download_items,
9399
key=lambda download_item: dateparser.parse(
94100
download_item["upload_time_iso_8601"]
95-
if "upload_time_iso_8601" in download_item
96-
else LOGGER.error(f"{download_item} has no upload_time_iso_8601")
97-
),
101+
)
102+
if download_item.get("upload_time_iso_8601")
103+
else None,
98104
)
99105
versions.add(
100106
LegacyVersion(
101107
value=version,
102-
release_date=dateparser.parse(latest_download_item["upload_time_iso_8601"]),
108+
release_date=dateparser.parse(latest_download_item["upload_time_iso_8601"])
109+
if latest_download_item.get("upload_time_iso_8601")
110+
else None,
103111
)
104112
)
105113
self.cache[pkg] = versions
@@ -117,8 +125,13 @@ def fetch(self, pkg):
117125
self.cache[pkg] = versions
118126
return
119127
for release in response:
120-
if release["number"] and release["published_at"]:
128+
if release.get("published_at"):
121129
release_date = dateparser.parse(release["published_at"])
130+
elif release.get("created_at"):
131+
release_date = dateparser.parse(release["created_at"])
132+
else:
133+
release_date = None
134+
if release.get("number"):
122135
versions.add(LegacyVersion(value=release["number"], release_date=release_date))
123136
else:
124137
LOGGER.error(f"Failed to parse release {release}")
@@ -196,12 +209,16 @@ def nuget_url(pkg_name: str) -> str:
196209
@staticmethod
197210
def extract_versions(resp: dict) -> Set[LegacyVersion]:
198211
all_versions = set()
199-
for entry_group in resp["items"] or []:
200-
for entry in entry_group["items"] or []:
201-
catalog_entry = entry["catalogEntry"] or {}
212+
for entry_group in resp.get("items") or []:
213+
for entry in entry_group.get("items") or []:
214+
catalog_entry = entry.get("catalogEntry") or {}
202215
version = catalog_entry.get("version")
203-
release_date = dateparser.parse(catalog_entry.get("published"))
204-
if version and release_date:
216+
release_date = (
217+
dateparser.parse(catalog_entry["published"])
218+
if catalog_entry.get("published")
219+
else None
220+
)
221+
if version:
205222
all_versions.add(
206223
LegacyVersion(
207224
value=version,
@@ -345,17 +362,18 @@ def composer_url(pkg_name: str) -> Optional[str]:
345362
@staticmethod
346363
def extract_versions(resp: dict, pkg_name: str) -> Set[LegacyVersion]:
347364
all_versions = set()
348-
for version in resp["packages"][pkg_name]:
365+
for version in get_item(resp, "packages", pkg_name) or []:
349366
if "dev" in version:
350367
continue
351368

352369
# This if statement ensures, that all_versions contains only released versions
353370
# See https://github.com/composer/composer/blob/44a4429978d1b3c6223277b875762b2930e83e8c/doc/articles/versions.md#tags # nopep8
354371
# for explanation of removing 'v'
372+
time = get_item(resp, "packages", pkg_name, version, "time")
355373
all_versions.add(
356374
LegacyVersion(
357375
value=version.lstrip("v"),
358-
release_date=dateparser.parse(resp["packages"][pkg_name][version]["time"]),
376+
release_date=dateparser.parse(time) if time else None,
359377
)
360378
)
361379
return all_versions

‎vulnerabilities/tests/test_data/package_manager_data/gem.json‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
[
2+
{
3+
"authors": "David Heinemeier Hansson",
4+
"built_at": "2022-03-08T00:00:00.000Z",
5+
"published_at": "2022-03-08T17:50:52.496Z",
6+
"description": "Ruby on Rails is a full-stack web framework optimized for programmer happiness and sustainable productivity. It encourages beautiful code by favoring convention over configuration.",
7+
"downloads_count": 295102,
8+
"metadata": {
9+
"changelog_uri": "https://github.com/rails/rails/releases/tag/v7.0.2.3",
10+
"bug_tracker_uri": "https://github.com/rails/rails/issues",
11+
"source_code_uri": "https://github.com/rails/rails/tree/v7.0.2.3",
12+
"mailing_list_uri": "https://discuss.rubyonrails.org/c/rubyonrails-talk",
13+
"documentation_uri": "https://api.rubyonrails.org/v7.0.2.3/",
14+
"rubygems_mfa_required": true
15+
},
16+
"number": "7.0.2.3",
17+
"summary": "Full-stack web application framework.",
18+
"platform": "ruby",
19+
"rubygems_version": ">= 1.8.11",
20+
"ruby_version": ">= 2.7.0",
21+
"prerelease": false,
22+
"licenses": [
23+
"MIT"
24+
],
25+
"requirements": [],
26+
"sha": "ee4e24075c72dec6e02e3fcddec86399c2b4eb0466efe4ccb5f78f96d3daa283"
27+
},
28+
{
29+
"authors": "David Heinemeier Hansson",
30+
"built_at": "2022-02-11T00:00:00.000Z",
31+
"created_at": "2022-02-11T19:44:19.017Z",
32+
"description": "Ruby on Rails is a full-stack web framework optimized for programmer happiness and sustainable productivity. It encourages beautiful code by favoring convention over configuration.",
33+
"downloads_count": 347689,
34+
"metadata": {
35+
"changelog_uri": "https://github.com/rails/rails/releases/tag/v7.0.2.2",
36+
"bug_tracker_uri": "https://github.com/rails/rails/issues",
37+
"source_code_uri": "https://github.com/rails/rails/tree/v7.0.2.2",
38+
"mailing_list_uri": "https://discuss.rubyonrails.org/c/rubyonrails-talk",
39+
"documentation_uri": "https://api.rubyonrails.org/v7.0.2.2/",
40+
"rubygems_mfa_required": true
41+
},
42+
"number": "7.0.2.2",
43+
"summary": "Full-stack web application framework.",
44+
"platform": "ruby",
45+
"rubygems_version": ">= 1.8.11",
46+
"ruby_version": ">= 2.7.0",
47+
"prerelease": false,
48+
"licenses": [
49+
"MIT"
50+
],
51+
"requirements": [],
52+
"sha": "3393e21131e2120a42cf634416033e587b5dfdccdc84d1a2d2c176b847f6f17f"
53+
}
54+
]

‎vulnerabilities/tests/test_helpers.py‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,6 @@
1919
# for any legal advice.
2020
# VulnerableCode is a free software tool from nexB Inc. and others.
2121
# Visit https://github.com/nexB/vulnerablecode/ for support and download.
22-
from unittest import result
2322

2423
from packageurl import PackageURL
2524

@@ -31,6 +30,7 @@ def test_nearest_patched_package():
3130

3231
result = nearest_patched_package(
3332
vulnerable_packages=[
33+
PackageURL(type="npm", name="foo", version="2.0.4"),
3434
PackageURL(type="npm", name="foo", version="2.0.0"),
3535
PackageURL(type="npm", name="foo", version="2.0.1"),
3636
PackageURL(type="npm", name="foo", version="1.9.8"),
@@ -66,4 +66,10 @@ def test_nearest_patched_package():
6666
type="npm", namespace=None, name="foo", version="2.0.2", qualifiers={}, subpath=None
6767
),
6868
),
69+
LegacyAffectedPackage(
70+
vulnerable_package=PackageURL(
71+
type="npm", namespace=None, name="foo", version="2.0.4", qualifiers={}, subpath=None
72+
),
73+
patched_package=None,
74+
),
6975
] == result

‎vulnerabilities/tests/test_package_managers_2.py‎

Lines changed: 103 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,16 @@
11
import json
22
import os
33
from datetime import datetime
4+
from unittest import mock
45

56
import pytest
67
import pytz
78

89
from vulnerabilities.package_managers_2 import GoproxyVersionAPI
910
from vulnerabilities.package_managers_2 import LegacyVersion
1011
from vulnerabilities.package_managers_2 import NugetVersionAPI
12+
from vulnerabilities.package_managers_2 import PypiVersionAPI
13+
from vulnerabilities.package_managers_2 import RubyVersionAPI
1114

1215
BASE_DIR = os.path.dirname(os.path.abspath(__file__))
1316
TEST_DATA = os.path.join(BASE_DIR, "test_data", "package_manager_data")
@@ -69,3 +72,103 @@ def test_nuget_extract_version():
6972
value="3.5.1", release_date=datetime(2015, 1, 23, 1, 5, 44, 447000, tzinfo=pytz.UTC)
7073
),
7174
}
75+
76+
77+
def test_nuget_extract_version_with_illformed_data():
78+
assert NugetVersionAPI.extract_versions({"items": [{"items": [{"catalogEntry": {}}]}]}) == set()
79+
80+
81+
@mock.patch("vulnerabilities.package_managers_2.get_response")
82+
def test_pypi_fetch_data(mock_response):
83+
pypi_api = PypiVersionAPI()
84+
with open(os.path.join(TEST_DATA, "pypi.json"), "r") as f:
85+
mock_response.return_value = json.load(f)
86+
pypi_api.fetch("django")
87+
assert pypi_api.cache == {
88+
"django": {
89+
LegacyVersion(
90+
value="1.10.5",
91+
release_date=datetime(2017, 1, 4, 19, 23, 0, 596664, tzinfo=pytz.UTC),
92+
),
93+
LegacyVersion(
94+
value="1.10.8",
95+
release_date=datetime(2017, 9, 5, 15, 31, 58, 221021, tzinfo=pytz.UTC),
96+
),
97+
LegacyVersion(
98+
value="1.10rc1",
99+
release_date=datetime(2016, 7, 18, 18, 5, 5, 503584, tzinfo=pytz.UTC),
100+
),
101+
LegacyVersion(
102+
value="1.10.4",
103+
release_date=datetime(2016, 12, 1, 23, 46, 50, 215935, tzinfo=pytz.UTC),
104+
),
105+
LegacyVersion(
106+
value="1.10a1",
107+
release_date=datetime(2016, 5, 20, 12, 24, 59, 952686, tzinfo=pytz.UTC),
108+
),
109+
LegacyVersion(
110+
value="1.10.3",
111+
release_date=datetime(2016, 11, 1, 13, 57, 16, 55061, tzinfo=pytz.UTC),
112+
),
113+
LegacyVersion(
114+
value="1.10.1",
115+
release_date=datetime(2016, 9, 1, 23, 18, 18, 672706, tzinfo=pytz.UTC),
116+
),
117+
LegacyVersion(
118+
value="1.10.2",
119+
release_date=datetime(2016, 10, 1, 20, 5, 31, 330942, tzinfo=pytz.UTC),
120+
),
121+
LegacyVersion(
122+
value="1.10.7",
123+
release_date=datetime(2017, 4, 4, 14, 27, 54, 235551, tzinfo=pytz.UTC),
124+
),
125+
LegacyVersion(
126+
value="1.10.6",
127+
release_date=datetime(2017, 3, 1, 13, 37, 40, 243134, tzinfo=pytz.UTC),
128+
),
129+
LegacyVersion(
130+
value="1.1.4",
131+
release_date=datetime(2011, 2, 9, 4, 13, 7, 75, tzinfo=pytz.UTC),
132+
),
133+
LegacyVersion(
134+
value="1.10b1",
135+
release_date=datetime(2016, 6, 22, 1, 15, 17, 267637, tzinfo=pytz.UTC),
136+
),
137+
LegacyVersion(
138+
value="1.1.3",
139+
release_date=datetime(2010, 12, 23, 5, 14, 23, 509436, tzinfo=pytz.UTC),
140+
),
141+
LegacyVersion(
142+
value="1.10",
143+
release_date=datetime(2016, 8, 1, 18, 32, 16, 280614, tzinfo=pytz.UTC),
144+
),
145+
}
146+
}
147+
148+
149+
@mock.patch("vulnerabilities.package_managers_2.get_response")
150+
def test_pypi_fetch_with_no_release(mock_response):
151+
pypi_api = PypiVersionAPI()
152+
mock_response.return_value = {"info": {}}
153+
pypi_api.fetch("django")
154+
assert pypi_api.cache == {"django": set()}
155+
156+
157+
@mock.patch("vulnerabilities.package_managers_2.get_response")
158+
def test_pypi_fetch_with_no_release(mock_response):
159+
ruby_api = RubyVersionAPI()
160+
with open(os.path.join(TEST_DATA, "gem.json"), "r") as f:
161+
mock_response.return_value = json.load(f)
162+
ruby_api.fetch("rails")
163+
assert ruby_api.cache == {
164+
"rails": {
165+
LegacyVersion(
166+
value="7.0.2.3",
167+
release_date=datetime(2022, 3, 8, 17, 50, 52, 496000, tzinfo=pytz.UTC),
168+
),
169+
LegacyVersion(
170+
value="7.0.2.2",
171+
release_date=datetime(2022, 2, 11, 19, 44, 19, 17000, tzinfo=pytz.UTC),
172+
),
173+
}
174+
}

0 commit comments

Comments
 (0)