-
Notifications
You must be signed in to change notification settings - Fork 2
155 lines (155 loc) · 6.08 KB
/
daily.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
name: Daily refresh
on:
schedule:
- cron: "10 15 * * *"
push:
branches:
- daily
workflow_dispatch:
inputs:
short_circuit:
type: boolean
description: "Short-circuit unchanged stages"
default: true
refresh_data:
type: boolean
description: "Refresh data (from NJSP)"
default: true
update_pqts:
type: boolean
description: "Update parquets (in this repo)"
default: true
update_slack_sha:
description: "\"Update NJSP data\" commit hash (for posting updates to Slack)"
sync_aws:
type: boolean
description: "Sync to AWS"
default: true
update_crash_log:
type: boolean
description: "Update crash log in AWS"
default: true
update_plots:
type: boolean
description: "Update plots"
default: true
post_to_slack:
type: boolean
description: "Post to Slack"
default: true
build_www:
type: boolean
description: "Build www"
default: true
slack_channel_id:
description: "Slack channel override"
jobs:
refresh_data:
name: Refresh NJSP data, dispatch www rebuild (if necessary)
runs-on: ubuntu-latest
steps:
- name: Setup Python?
id: needs_python
run: echo "y=${{ github.event.schedule || inputs.refresh_data || inputs.update_pqts || inputs.update_slack_sha || inputs.update_plots || inputs.post_to_slack || '' }}" | tee -a $GITHUB_OUTPUT
- uses: actions/checkout@v4
if: steps.needs_python.outputs.needs_python || inputs.build_www
with:
ref: ${{ github.ref_name }}
- uses: webfactory/ssh-agent@v0.7.0
if: steps.needs_python.outputs.y
with:
ssh-private-key: ${{ secrets.GHA_DEPLOY_KEY }}
- uses: actions/setup-python@v4
if: steps.needs_python.outputs.y
with:
python-version: 3.11
cache: pip
- run: pip install -e .
if: steps.needs_python.outputs.y
- name: Configure Git author
if: steps.needs_python.outputs.y
run: |
git config --global user.name 'GitHub Actions'
git config --global user.email 'ryan-williams@users.noreply.github.com'
- name: Refresh data
id: refresh_data
if: github.event.schedule || inputs.refresh_data
run: njsp -cc refresh_data
- name: Update parquets
id: update_pqts
if: (github.event.schedule || inputs.update_pqts) && (steps.refresh_data.outputs.sha || !inputs.short_circuit)
run: njsp -cc update_pqts
- name: Compute Slack update SHA
# Use the relevant input SHA, if present, otherwise the output SHA from the previous step
id: update_slack_sha
if: inputs.update_slack_sha || steps.update_pqts.outputs.sha
run: |
echo "sha=${{ inputs.update_slack_sha || steps.update_pqts.outputs.sha }}" >> $GITHUB_OUTPUT
- name: AWS sync
if: inputs.sync_aws || steps.update_pqts.outputs.sha
env:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
AWS_DEFAULT_REGION: us-east-1
run: |
aws s3 sync --exclude '*' --include 'nj_crashes.db' ./ s3://nj-crashes/
aws s3api put-object-acl --bucket nj-crashes --key nj_crashes.db --acl public-read
aws s3 cp data/crashes.pqt s3://nj-crashes/data/crashes.parquet
aws s3api put-object-acl --bucket nj-crashes --key data/crashes.parquet --acl public-read
- name: Update crash log
if: inputs.update_crash_log || steps.refresh_data.outputs.sha
env:
GH_TOKEN: ${{ secrets.GH_TOKEN }}
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
AWS_DEFAULT_REGION: us-east-1
run: njsp crash_logs -a s3://nj-crashes/data/njsp/crash-log.parquet -vi
- name: Compute plot_data.changed
id: plot_data
if: (github.event.schedule || inputs.update_plots) && (steps.update_slack_sha.outputs.sha || !inputs.short_circuit)
run: echo "changed=1" >> $GITHUB_OUTPUT
- name: Refresh annual summaries
if: steps.plot_data.outputs.changed
run: njsp -cc refresh_summaries
- name: "Fetch ≈1yr of history"
if: steps.plot_data.outputs.changed
run: |
year=$(date +%Y)
let prv_year=year-1
since="$(date --date="$(date +%Y-%m-%d) -375 day" +%Y-%m-%d)"
echo "Fetching commits since $since"
git fetch --shallow-since "$since" origin ${{ github.ref_name }}
echo "Fetched $(git rev-list --count) revisions"
- name: Update YTD / ROY projections
if: steps.plot_data.outputs.changed
run: njsp -cc update_projections
- name: Update plot data
id: update_plots
if: steps.plot_data.outputs.changed
run: njsp -cc update_plots
- name: Post to Slack
id: post_to_slack
if: (github.event.schedule || inputs.post_to_slack) && (steps.update_slack_sha.outputs.sha || !inputs.short_circuit)
run: njsp slack sync -c ${{ steps.update_slack_sha.outputs.sha }}
env:
GH_TOKEN: ${{ secrets.GH_TOKEN }}
SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
SLACK_CHANNEL_ID: ${{ inputs.slack_channel_id || secrets.SLACK_CHANNEL_ID }}
- name: Decide whether to rebuild www
id: build_www
if: (github.event.schedule || inputs.build_www) && (steps.update_plots.outputs.sha || !inputs.short_circuit)
run: |
echo "run=1" >> $GITHUB_OUTPUT
echo "sha=$(git log -1 --format=%h)" >> $GITHUB_OUTPUT
outputs:
update_pqts: ${{ steps.update_pqts.outputs.sha }}
build_www: ${{ steps.build_www.outputs.run }}
sha: ${{ steps.build_www.outputs.sha }}
rebuild_www:
name: Rebuild www
needs: refresh_data
if: needs.refresh_data.outputs.build_www
uses: ./.github/workflows/www.yml
secrets: inherit
with:
ref: ${{ needs.refresh_data.outputs.sha }}