initial open source string sketch

db/oss/.gitignore

@@ -0,0 +1,3 @@
+*.csv
+*.jsonl
+*.jsonl.gz

floss/qs/db/oss.py

@@ -0,0 +1,28 @@
+import gzip
+import pathlib
+from typing import Dict
+
+import msgspec
+
+
+class OpenSourceString(msgspec.Struct):
+    string: str
+    library_name: str
+    library_version: str
+    file_path: str
+    function_name: str
+    line_number: int | None = None
+
+
+class OpenSourceStringDatabase:
+    metadata_by_string: Dict[str, OpenSourceString]
+
+    @classmethod
+    def from_file(cls, path: pathlib.Path) -> "OpenSourceStringDatabase":
+        metadata_by_string: Dict[str, OpenSourceString] = {}
+        decoder = msgspec.json.Decoder(type=OpenSourceString)
+        for line in gzip.decompress(path.read_bytes()).split(b"\n"):
+            s = decoder.decode(line)
+            metadata_by_string[s.string] = s
+
+        return cls(metadata_by_string=metadata_by_string)

setup.py

@@ -15,6 +15,7 @@
     "networkx==2.5.1",
     "halo==0.0.31",
     "rich==13.3.4",
+    "msgspec==0.14.2",
 ]
 
 # this sets __version__