factor crash-log s3 url

.github/workflows/daily.yml

@@ -103,7 +103,7 @@ jobs:
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           AWS_DEFAULT_REGION: us-east-1
-        run: njsp crash_logs -a s3://nj-crashes/data/njsp/crash-log.parquet -vi
+        run: njsp crash_logs --s3 -v
       - name: Compute plot_data.changed
         id: plot_data
         if: (github.event.schedule || inputs.update_plots) && (steps.update_slack_sha.outputs.sha || !inputs.short_circuit)

njsp/cli/crash_logs.py

@@ -8,7 +8,7 @@
 from nj_crashes.utils.log import err
 from njsp.cli.base import njsp
 from njsp.crash_log import get_crashes_df, DEFAULT_ROOT_SHA
-from njsp.paths import CRASHES_RELPATH
+from njsp.paths import CRASHES_RELPATH, S3_CRASH_LOG_PQT
 
 # Enforce column order, otherwise DFs built using 1 or more -a/--append-to chains can have different column orders (e.g.
 # STREET, HIGHWAY, and INJURIES may each appear for the first time, in a given FAUQStats XML file, in different orders).
@@ -46,8 +46,15 @@ def load_prefix(path: str) -> pd.DataFrame:
 @click.option("-p", "--load-parquet", is_flag=True, help=f"Load crashes from {CRASHES_RELPATH} (instead of FAUQStats XML files)")
 @click.option("-r", "--root", help=f"Ref to end at; if -a/--append-to is passed, defaults to the latest SHA in that DataFrame, {DEFAULT_ROOT_SHA} otherwise")
 @click.option("-s", "--since", help="Date to start from")
+@click.option('--s3', is_flag=True, help=f"Shorthand for CI use: `-a {S3_CRASH_LOG_PQT} -i`")
 @click.option("-v", "--verbose", is_flag=True, help="Print debug info")
-def crash_logs(append_to, write_dupes, head, in_place, out_path, load_parquet, root, since, verbose):
+def crash_logs(append_to, write_dupes, head, in_place, out_path, load_parquet, root, since, s3, verbose):
+    if s3:
+        if append_to:
+            raise ValueError("Cannot use --s3 with -a/--append-to")
+        append_to = S3_CRASH_LOG_PQT
+        in_place = True
+
     prefix = None
     if append_to:
         if not root:

njsp/paths.py

@@ -9,6 +9,7 @@
 CRASHES_RELPATH = relpath(CRASHES_PQT, ROOT_DIR)
 
 NJSP_DATA = join(NJSP_DIR, 'data')
+DATA_RELPATH = relpath(NJSP_DATA, ROOT_DIR)
 PROJECTED_CSV = join(NJSP_DATA, 'projected.csv')
 
 # Tabula template JSONs and NJSP summary PDFs live in this dir
@@ -25,7 +26,7 @@
 YTC_DB = path.join(WWW_NJSP, 'year-type-county.db')
 YTC_DB_URI = f'sqlite:///{YTC_DB}'
 
-S3_CRASH_LOG_PQT = 's3://nj-crashes/data/njsp/crash-log.pqt'
+S3_CRASH_LOG_PQT = f's3://nj-crashes/{DATA_RELPATH}/crash-log.parquet'
 
 
 def annual_ytc_url(year):