Merge pull request #38 from lsst-sqre/feature/redis

Feature/redis

.pre-commit-config.yaml

@@ -7,7 +7,7 @@ repos:
       - id: trailing-whitespace
 
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.6.0
+    rev: v0.6.5
     hooks:
       - id: ruff
         args: [--fix, --exit-non-zero-on-fix]

CHANGELOG.md

@@ -6,6 +6,13 @@ Find changes for the upcoming release in the project's [changelog.d directory](h
 
 <!-- scriv-insert-here -->
 
+<a id='changelog-0.2.2'></a>
+## 0.2.2 (2024-09-18)
+
+### New features
+
+- Avro schemas are now generated via dataclass rather than being strictly typed
+
 <a id='changelog-0.2.1'></a>
 ## 0.2.1 (2024-08-15)
 

requirements/main.in

@@ -11,4 +11,6 @@ requests
 pydantic
 pydantic-settings
 types-requests
-usgs-libcomcat
+usgs-libcomcat
+safir[redis]
+dataclasses_avroschema[pydantic]

requirements/tox.txt

@@ -1,8 +1,8 @@
 # This file was autogenerated by uv via the following command:
 #    uv pip compile --generate-hashes --output-file requirements/tox.txt requirements/tox.in
-cachetools==5.4.0 \
-    --hash=sha256:3ae3b49a3d5e28a77a0be2b37dbcb89005058959cb2323858c2657c4a8cab474 \
-    --hash=sha256:b8adc2e7c07f105ced7bc56dbb6dfbe7c4a00acce20e2227b3f355be89bc6827
+cachetools==5.5.0 \
+    --hash=sha256:02134e8439cdc2ffb62023ce1debca2944c3f289d66bb17ead3ab3dede74b292 \
+    --hash=sha256:2cc24fb4cbe39633fb7badd9db9ca6295d766d9c2995f245725a46715d050f2a
     # via tox
 chardet==5.2.0 \
     --hash=sha256:1b3b6ff479a8c414bc3fa2c0852995695c4a026dcd6d0633b2dd092ca39c1cf7 \
@@ -18,9 +18,9 @@ distlib==0.3.8 \
     # via
     #   -c requirements/dev.txt
     #   virtualenv
-filelock==3.15.4 \
-    --hash=sha256:2207938cbc1844345cb01a5a95524dae30f0ce089eba5b00378295a17e3e90cb \
-    --hash=sha256:6ca1fffae96225dab4c6eaf1c4f4f28cd2568d3ec2a44e15a08520504de468e7
+filelock==3.16.1 \
+    --hash=sha256:2082e5703d51fbf98ea75855d9d5527e33d8ff23099bec374a134febee6946b0 \
+    --hash=sha256:c249fbfcd5db47e5e2d6d62198e565475ee65e4831e2561c8e313fa7eb961435
     # via
     #   -c requirements/dev.txt
     #   tox
@@ -34,9 +34,9 @@ packaging==24.1 \
     #   pyproject-api
     #   tox
     #   tox-uv
-platformdirs==4.2.2 \
-    --hash=sha256:2d7a1657e36a80ea911db832a8a6ece5ee53d8de21edd5cc5879af6530b1bfee \
-    --hash=sha256:38b7b51f512eed9e84a22788b4bce1de17c0adb134d6becb09836e37d8654cd3
+platformdirs==4.3.6 \
+    --hash=sha256:357fb2acbc885b0419afd3ce3ed34564c13c9b95c89360cd9563f73aa5e2b907 \
+    --hash=sha256:73e575e1408ab8103900836b97580d5307456908a03e92031bab39e4554cc3fb
     # via
     #   -c requirements/dev.txt
     #   tox
@@ -47,43 +47,43 @@ pluggy==1.5.0 \
     # via
     #   -c requirements/dev.txt
     #   tox
-pyproject-api==1.7.1 \
-    --hash=sha256:2dc1654062c2b27733d8fd4cdda672b22fe8741ef1dde8e3a998a9547b071eeb \
-    --hash=sha256:7ebc6cd10710f89f4cf2a2731710a98abce37ebff19427116ff2174c9236a827
+pyproject-api==1.8.0 \
+    --hash=sha256:3d7d347a047afe796fd5d1885b1e391ba29be7169bd2f102fcd378f04273d228 \
+    --hash=sha256:77b8049f2feb5d33eefcc21b57f1e279636277a8ac8ad6b5871037b243778496
     # via tox
-tox==4.18.0 \
-    --hash=sha256:0a457400cf70615dc0627eb70d293e80cd95d8ce174bb40ac011011f0c03a249 \
-    --hash=sha256:5dfa1cab9f146becd6e351333a82f9e0ade374451630ba65ee54584624c27b58
+tox==4.19.0 \
+    --hash=sha256:66177d887f9d7ef8eaa9b58b187f7b865fa4c58650086c01336e82c9831e1867 \
+    --hash=sha256:6e20a520db7710f6980b8ec96bde189d6b8cf41b327ec703b03e1a2a447b1aaf
     # via
     #   -r requirements/tox.in
     #   tox-uv
-tox-uv==1.11.2 \
-    --hash=sha256:7f8f1737b3277e1cddcb5b89fcc5931d04923562c940ae60f29e140908566df2 \
-    --hash=sha256:a7aded5c3fb69f055b523357988c1055bb573e91bfd7ecfb9b5233ebcab5d10b
+tox-uv==1.11.4 \
+    --hash=sha256:10a6025d751108f17d8912bf177f1804ab7f9973ab39df6e599c47f7dc849c59 \
+    --hash=sha256:e4150968466c719aa1f61f0b37b82fa43dd61f17f2c6f9ac0877572885cc6275
     # via -r requirements/tox.in
-uv==0.2.36 \
-    --hash=sha256:083e56a18fc33395aeed4f56a47003e08f2ad9d5039af63ad5b107a241d0e7a3 \
-    --hash=sha256:139889680c2475afbab61c725df951c4dfa030c42c4eaa8f27d05286c96e8aab \
-    --hash=sha256:3f18322decfb0da577e40675620f6e6b9ffe1d8ee1de88a448bfe67fe7884626 \
-    --hash=sha256:463a45a34277b9334e500fce463f59408a6bab0c1b5cb3023f25185a4805a562 \
-    --hash=sha256:5a3800d2484b989041139ef96b395cec0e4e0a13132584b0147c739063a2494f \
-    --hash=sha256:5c8d624975f8355e00ad5f802ed27fcfc7b86d0bd50b57efe24bd665fd3f9a9b \
-    --hash=sha256:8753851cc10b0a67e5c5dd29a6f35a072341290cf27a7bb3193ddd92bda19f51 \
-    --hash=sha256:8820dd5b77ffcda07dde09712a43d969d39b0aace112d8074c540f19a4911cc2 \
-    --hash=sha256:89d3fb3d7a66fa4a4f7c938be0277457fe71179ec4e72758cfe16faec1daa362 \
-    --hash=sha256:8e5e2e8e218eb672a3bb57af0ab2d2d3de79119b5dc6b6edb03d349739e474ff \
-    --hash=sha256:a08d485cd8eae0352b4765005a4499ad5db073c3534866d68617bbb831ee219a \
-    --hash=sha256:a4fddaf0a6a995591042a57ac48557b9b2c1e2c7f09e0f7880f40c34e61f53f8 \
-    --hash=sha256:a7961f4d88100fc48129c918545cbb17b9a0d8e3d163c65985e1d1682e056321 \
-    --hash=sha256:a837b799e3af1535473b8ab14e414e50f595d547d295879db0d6b0943b7476df \
-    --hash=sha256:d093fd10aaf29e85128beaa514f8d37d7374cf1d1a95da966e15788a6fe7d55d \
-    --hash=sha256:e36cd4e9c1187d155139b98bcd2cfbfb275f9f601c550fcc38a283983c74f93d \
-    --hash=sha256:e79a4cdb3b89b011fafcaa853ebbb9254115f3f7cadbd9141492c48ceeac1b2d \
-    --hash=sha256:f1d711629dd8610933687ceea4ad82156ef7b2102c4e9da72afe6c01981f8a1a
+uv==0.4.12 \
+    --hash=sha256:0840d0141f54f64474c9dbd46787971859fac9deacc701091b44f1c47d066823 \
+    --hash=sha256:0d548c090bf38fb76b6493c90bbfbad30bfc4b41365019953bffbc54d32394ed \
+    --hash=sha256:0f00d15108af7b17f49d70714a31927eed27e192d5e5410822c098399d61196d \
+    --hash=sha256:31f7689c6f49b0489dc727b1e6f0f008f7db21388c3cf374577a445bd7d727b8 \
+    --hash=sha256:56901b53c9bcce81305826c89378058922b405d0fbfb5c2742dda7dc5fdf891c \
+    --hash=sha256:649d2974da5d867ca0230a15aa75d6e4625c2a71eddc0abaeebe7a167038f56b \
+    --hash=sha256:67327c5997a9c4531c0e13be8545aa6568a15c99a97770ac65f6dcc5600e8a9c \
+    --hash=sha256:6922ca516056069a6c835f0cf60053241bb3438e4ccc0356c223d4f5c0d92254 \
+    --hash=sha256:86635a9dd024d08499405c9e1c1087aa24ffbfe89eb6dde010e5a60855e661bc \
+    --hash=sha256:8a102ee30a41909634b28cb9d7d5a03af2953aa86ff941e24916093f4a74d44f \
+    --hash=sha256:8cbfa5ed4ea167291260416d71d54ffb949b0b98bcf945190adb8c65e30492be \
+    --hash=sha256:9aa768f4b94335a4145d74e73ff4721cb1a3e1fd1269f4bb95187a9f8d41f8e1 \
+    --hash=sha256:a1d2ada46563178cacfeb2ff8a3b2764381a953cee87002fad0b9181f4a35e0d \
+    --hash=sha256:a3c1b7b4a6e5258c0b20079beb1d22c3d306f7695eab8a3d3aea93b37db01b3a \
+    --hash=sha256:c081b13c7789b518a2077ed0c49d33c9d855e110a2f670e4f354696245089edc \
+    --hash=sha256:c6861b3c92da1cdc2cb18c76b0e05004413ce1cc95782a4b34b7ee002006efb8 \
+    --hash=sha256:dc638ff81e817a1c049c8bd51c623238dccf9bfbfb17e20878eaece6c74338bb \
+    --hash=sha256:e931a2add4dfec717184164a54608b99d37e0000b9c151bb020a0a2dcc6d5cc1
     # via tox-uv
-virtualenv==20.26.3 \
-    --hash=sha256:4c43a2a236279d9ea36a0d76f98d84bd6ca94ac4e0f4a3b9d46d05e10fea542a \
-    --hash=sha256:8cc4a31139e796e9a7de2cd5cf2489de1217193116a8fd42328f1bd65f434589
+virtualenv==20.26.5 \
+    --hash=sha256:4f3ac17b81fba3ce3bd6f4ead2749a72da5929c01774948e243db9ba41df4ff6 \
+    --hash=sha256:ce489cac131aa58f4b25e321d6d186171f78e6cb13fafbf32a840cee67733ff4
     # via
     #   -c requirements/dev.txt
     #   tox

src/sasquatchbackpack/schemas/__init__.py

@@ -0,0 +1 @@
+"""Avro schemas."""

src/sasquatchbackpack/schemas/usgs.py

@@ -0,0 +1,23 @@
+"""USGS Schemas."""
+
+from dataclasses import dataclass, field
+
+from dataclasses_avroschema.schema_generator import AvroModel
+
+
+@dataclass
+class EarthquakeSchema(AvroModel):
+    """Collection of earthquakes near the summit."""
+
+    timestamp: int
+    id: str = field(metadata={"description": "unique earthquake id"})
+    latitude: float = field(metadata={"units": "degree"})
+    longitude: float = field(metadata={"units": "degree"})
+    depth: float = field(metadata={"units": "km"})
+    magnitude: float = field(metadata={"units": "u.richter_magnitudes"})
+
+    class Meta:
+        """Schema metadata."""
+
+        namespace = "$namespace"
+        schema_name = "$topic_name"

src/sasquatchbackpack/scripts/usgs.py

@@ -2,11 +2,11 @@
 
 from dataclasses import dataclass
 from datetime import datetime, timedelta
-from pathlib import Path
 
 from libcomcat.search import search
 
 from sasquatchbackpack.sasquatch import DataSource
+from sasquatchbackpack.schemas.usgs import EarthquakeSchema
 
 __all__ = ["USGSSource", "USGSConfig"]
 
@@ -79,11 +79,6 @@ class USGSConfig:
     radius: int
     coords: tuple[float, float]
     magnitude_bounds: tuple[int, int]
-    schema_file: str = "src/sasquatchbackpack/schemas/usgs/earthquake.avsc"
-    cron_schema: str = (
-        "/opt/venv/lib/python3.12/site-packages/"
-        "sasquatchbackpack/schemas/usgs/earthquake.avsc"
-    )
     topic_name: str = "usgs_earthquake_data"
 
 
@@ -106,18 +101,14 @@ def __init__(
         super().__init__(config.topic_name)
         self.duration = config.duration
         self.config = config
+        self.schema = EarthquakeSchema.avro_schema().replace("double", "float")
         self.radius = config.radius
         self.coords = config.coords
         self.magnitude_bounds = config.magnitude_bounds
 
     def load_schema(self) -> str:
         """Load the relevant schema."""
-        try:
-            with Path(self.config.schema_file).open("r") as file:
-                return file.read()
-        except FileNotFoundError:
-            with Path(self.config.cron_schema).open("r") as file:
-                return file.read()
+        return self.schema
 
     def get_records(self) -> list[dict]:
         """Call the USGS Comcat API and assembles records.