Merge pull request #281 from sbs2001/rust_new_format

Adapt rust importer to new advisory format

Author: sbs2001

requirements.txt

@@ -56,3 +56,4 @@ wcwidth==0.1.7
 whitenoise==5.0.1
 zipp==0.6.0
 requests==2.23.0
+toml==0.10.2

vulnerabilities/importers/rust.py

@@ -23,15 +23,16 @@
 import asyncio
 import json
 from itertools import chain
+import re
 from typing import Optional, Mapping
 from typing import Set
 from typing import Tuple
 from urllib.error import HTTPError
 from urllib.request import urlopen
 
-import pytoml as toml
 from dephell_specifier import RangeSpecifier
 from packageurl import PackageURL
+import toml
 
 from vulnerabilities.data_source import Advisory
 from vulnerabilities.data_source import GitDataSource
@@ -47,7 +48,7 @@ def __enter__(self):
             self._added_files, self._updated_files = self.file_changes(
                 subdir="crates",  # TODO Consider importing the advisories for cargo, etc as well.
                 recursive=True,
-                file_ext="toml",
+                file_ext="md",
             )
 
     @property
@@ -66,15 +67,15 @@ def updated_advisories(self) -> Set[Advisory]:
         return self._load_advisories(self._updated_files)
 
     def _load_advisories(self, files) -> Set[Advisory]:
-        files = [f for f in files if not f.endswith("-0000.toml")]  # skip temporary files
+        # per @tarcieri It will always be named RUSTSEC-0000-0000.md
+        # https://github.com/nexB/vulnerablecode/pull/281/files#r528899864
+        files = [f for f in files if not f.endswith("-0000.md")]  # skip temporary files
         packages = self.collect_packages(files)
         self.set_api(packages)
 
         while files:
             batch, files = files[: self.batch_size], files[self.batch_size:]
-
             advisories = set()
-
             for path in batch:
                 advisory = self._load_advisory(path)
                 if advisory:
@@ -84,13 +85,13 @@ def _load_advisories(self, files) -> Set[Advisory]:
     def collect_packages(self, paths):
         packages = set()
         for path in paths:
-            record = load_toml(path)
+            record = get_advisory_data(path)
             packages.add(record["advisory"]["package"])
 
         return packages
 
     def _load_advisory(self, path: str) -> Optional[Advisory]:
-        record = load_toml(path)
+        record = get_advisory_data(path)
         advisory = record.get("advisory", {})
         crate_name = advisory["package"]
         references = []
@@ -175,6 +176,63 @@ def categorize_versions(
     return unaffected, affected
 
 
-def load_toml(path):
-    with open(path) as f:
-        return toml.load(f)
+def get_toml_lines(lines):
+    """
+    Yield lines of TOML extracted from an iterable of text ``lines``.
+    The lines are expected to be from a RustSec Markdown advisory file with
+    embedded TOML metadata.
+
+    For example::
+
+    >>> text = '''
+    ... ```toml
+    ... [advisory]
+    ... id = "RUST-001"
+    ...
+    ... [versions]
+    ... patch = [">= 1.2.1"]
+    ... ```
+    ... # Use-after-free with objects returned by `Stream`'s `get_format_info`
+    ...
+    ... Affected versions contained a pair of use-after-free issues with the objects.
+    ... '''
+    >>> list(get_toml_lines(text.splitlines()))
+    ['', '[advisory]', 'id = "RUST-001"', '', '[versions]', 'patch = [">= 1.2.1"]']
+    """
+
+    for line in lines:
+        line = line.strip()
+        if line.startswith("```toml"):
+            continue
+        elif line.endswith("```"):
+            break
+        else:
+            yield line
+
+
+def data_from_toml_lines(lines):
+    """
+    Return a mapping of data from an iterable of TOML text ``lines``.
+
+    For example::
+
+    >>> lines = ['[advisory]', 'id = "RUST1"', '', '[versions]', 'patch = [">= 1"]']
+    >>> data_from_toml_lines(lines)
+    {'advisory': {'id': 'RUST1'}, 'versions': {'patch': ['>= 1']}}
+    """
+    return toml.loads("\n".join(lines))
+
+
+def get_advisory_data(location):
+    """
+    Return a mapping of vulnerability data from a RustSec advisory file at
+    ``location``.
+    RustSec advisories documents are .md files starting with a block of TOML
+    identified as the text inside a tripple-backtick TOML block. Per
+    https://github.com/RustSec/advisory-db#advisory-format:
+        Advisories are formatted in Markdown with TOML "front matter".
+    """
+
+    with open(location) as lines:
+        toml_lines = get_toml_lines(lines)
+        return data_from_toml_lines(toml_lines)

vulnerabilities/package_managers.py

@@ -319,5 +319,4 @@ async def fetch(self, owner_repo: str, session) -> None:
         endpoint = f"https://api.github.com/repos/{owner_repo}/git/refs/tags"
         resp = await session.request(method="GET", url=endpoint)
         resp = await resp.json()
-        print(resp)
         self.cache[owner_repo] = [release["ref"].split("/")[-1] for release in resp]

…ies/tests/test_data/rust-advisory-db.zip …bilities/tests/test_data/advisory-db.zipvulnerabilities/tests/test_data/rust-advisory-db.zip renamed to vulnerabilities/tests/test_data/advisory-db.zip vulnerabilities/tests/test_data/rust-advisory-db.zip renamed to vulnerabilities/tests/test_data/advisory-db.zip

@@ -0,0 +1,115 @@
+```toml
+[advisory]
+id = "CVE-2019-16760"
+package = "cargo"
+aliases = ["GHSA-phjm-8x66-qw4r"]
+date = "2019-09-30"
+url = "https://groups.google.com/forum/#!topic/rustlang-security-announcements/rVQ5e3TDnpQ"
+
+[versions]
+patched = [">= 1.26.0"]
+```
+
+# Cargo prior to Rust 1.26.0 may download the wrong dependency
+
+The Rust team was recently notified of a security concern when using older
+versions of Cargo to build crates which use the package rename feature added in
+newer versions of Cargo. If you're using Rust 1.26.0, released on 2018-05-10,
+or later you're not affected.
+
+The CVE for this vulnerability is [CVE-2019-16760][0].
+
+## Overview
+
+Cargo can be configured through `Cargo.toml` and the `[dependencies]` section
+to depend on different crates, such as those from crates.io. There are multiple
+ways to configure how you depend on crates as well, for example if you depend
+on `serde` and enable the `derive` feature it would look like:
+
+```toml
+serde = { version = "1.0", features = ['derive'] }
+```
+
+Rust 1.31.0 [introduced a new feature of Cargo][1] where one of the optional
+keys you can specify in this map is `package`, a way to [rename a crate
+locally][2]. For example if you preferred to use `serde1` locally instead of
+`serde`, you could write:
+
+```toml
+serde1 = { version = "1.0", features = ['derive'], package = "serde" }
+```
+
+It's the addition of the `package` key that causes Cargo to compile the crate
+differently. This feature was [first implemented][3] in Rust 1.26.0, but it was
+unstable at the time. For Rust 1.25.0 and prior, however, Cargo would ignore
+the `package` key and and interpret the dependency line as if it were:
+
+```toml
+serde1 = { version = "1.0", features = ['derive'] }
+```
+
+This means when compiled with Rust 1.25.0 and prior then it would attempt to
+download the `serde1` crate. A malicious user could squat the `serde1` name on
+crates.io to look like `serde 1.0.0` but instead act maliciously when built.
+
+In summary, usage of the `package` key to rename dependencies in `Cargo.toml`
+is ignored in Rust 1.25.0 and prior. When Rust 1.25.0 and prior is used Cargo
+will ignore `package` and download the wrong dependency, which could be
+squatted on crates.io to be a malicious package. This not only affects
+manifests that you write locally yourself, but also manifests published to
+crates.io. If you published a crate, for example, that depends on `serde1` to
+crates.io then users who depend on you may also be vulnerable if they use Rust
+1.25.0 and prior.
+
+## Affected Versions
+
+Rust 1.0.0 through Rust 1.25.0 is affected by this advisory because Cargo will
+ignore the `package` key in manifests. Rust 1.26.0 through Rust 1.30.0 are not
+affected and typically will emit an error because the `package` key is
+unstable. Rust 1.31.0 and after are not affected because Cargo understands the
+`package` key.
+
+In terms of Cargo versions, this affects Cargo up through Cargo 0.26.0. All
+future versions of Cargo are unaffected.
+
+## Mitigations
+
+We strongly recommend that users of the affected versions update their compiler
+to the latest available one. Preventing this issue from happening requires
+updating your compiler to either Rust 1.26.0 or newer.
+
+We will not be issuing a patch release for Rust versions prior to 1.26.0. Users
+of Rust 1.19.0 to Rust 1.25.0 can instead apply [the provided patches][4] to
+mitigate the issue.
+
+An audit of existing crates published to crates.io using the `package` key has
+been performed and there is no evidence that this vulnerability has been
+exploited in the wild. Our audit only covers the crates currently published on
+crates.io: if you notice crates exploiting this vulnerability in the future
+please don't hesitate to email secu...@rust-lang.org in accordance with [our
+security policy][5].
+
+## Timeline of events
+
+* Wed, Sep 18, 2019 at 13:54 UTC - Bug reported to secu...@rust-lang.org
+* Wed, Sep 18, 2019 at 15:35 UTC - Response confirming the report
+* Wed, Sep 18, 2019 - Cargo, Core, and crates.io teams confer on how best to
+handle this
+* Thu, Sep 19, 2019 - Confirmed with Elichai plan of action and continued to
+audit existing crates
+* Mon, Sep 23, 2019 - Advisory drafted, patches developed, audit completed
+* Mon, Sep 30, 2019 - Advisory published, security list informed of this issue
+
+## Acknowledgments
+
+Thanks to Elichai Turkel, who found this bug and reported it to us in accordance
+with our [security policy][5].
+
+## Links
+
+[0]: https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2019-16760
+[1]: https://blog.rust-lang.org/2018/12/06/Rust-1.31-and-rust-2018.html#cargo-features
+[2]: https://doc.rust-lang.org/cargo/reference/specifying-dependencies.html#renaming-dependencies-in-cargotoml
+[3]: https://github.com/rust-lang/cargo/pull/4953
+[4]: https://gist.github.com/pietroalbini/0d293b24a44babbeb6187e06eebd4992
+[5]: https://www.rust-lang.org/policies/security

vulnerabilities/tests/test_data/rust/CVE-2019-16760.md

@@ -268,7 +268,7 @@ class GitDataSourceTest(TestCase):
     @classmethod
     def setUpClass(cls) -> None:
         cls.tempdir = tempfile.mkdtemp()
-        zip_path = os.path.join(TEST_DATA, 'rust-advisory-db.zip')
+        zip_path = os.path.join(TEST_DATA, 'advisory-db.zip')
 
         with zipfile.ZipFile(zip_path, "r") as zip_ref:
             zip_ref.extractall(cls.tempdir)