Skip to content

Commit af965e3

Browse files
MaineK00nMaineK00n
committed
feat(ci): add euvd
1 parent 9fa0a58 commit af965e3

File tree

10 files changed

+202
-51
lines changed

10 files changed

+202
-51
lines changed

‎.github/workflows/archive-raw.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,7 @@ jobs:
5151
- "endoflife-date-products"
5252
- "erlang-ghsa"
5353
- "erlang-osv"
54+
- "euvd"
5455
- "exploit-exploitdb"
5556
- "exploit-github"
5657
- "exploit-inthewild"

‎.github/workflows/archive.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,7 @@ on:
5353
- vuls-data-raw-endoflife-date-products
5454
- vuls-data-raw-erlang-ghsa
5555
- vuls-data-raw-erlang-osv
56+
- vuls-data-raw-euvd
5657
- vuls-data-raw-exploit-exploitdb
5758
- vuls-data-raw-exploit-github
5859
- vuls-data-raw-exploit-inthewild

‎.github/workflows/backup-daily.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,7 @@ jobs:
7676
- vuls-data-raw-epss
7777
- vuls-data-raw-erlang-ghsa
7878
- vuls-data-raw-erlang-osv
79+
- vuls-data-raw-euvd
7980
- vuls-data-raw-exploit-exploitdb
8081
- vuls-data-raw-exploit-github
8182
- vuls-data-raw-exploit-inthewild

‎.github/workflows/backup-monthly.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,7 @@ jobs:
7676
- vuls-data-raw-epss
7777
- vuls-data-raw-erlang-ghsa
7878
- vuls-data-raw-erlang-osv
79+
- vuls-data-raw-euvd
7980
- vuls-data-raw-exploit-exploitdb
8081
- vuls-data-raw-exploit-github
8182
- vuls-data-raw-exploit-inthewild

‎.github/workflows/backup-weekly.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,7 @@ jobs:
7676
- vuls-data-raw-epss
7777
- vuls-data-raw-erlang-ghsa
7878
- vuls-data-raw-erlang-osv
79+
- vuls-data-raw-euvd
7980
- vuls-data-raw-exploit-exploitdb
8081
- vuls-data-raw-exploit-github
8182
- vuls-data-raw-exploit-inthewild

‎.github/workflows/fetch-all.yml‎

Lines changed: 57 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -170,77 +170,62 @@ jobs:
170170
with:
171171
target: ${{ matrix.target }}
172172

173-
fetch-nvd-api:
173+
fetch-cisco-json:
174+
name: Fetch vuls-data-raw-cisco-json
175+
uses: ./.github/workflows/fetch-cisco-json.yml
176+
secrets:
177+
CISCO_CLIENT_KEY: ${{ secrets.CISCO_CLIENT_KEY }}
178+
CISCO_CLIENT_SECRET: ${{ secrets.CISCO_CLIENT_SECRET }}
179+
180+
fetch-cisco-cvrf-or-csaf:
174181
name: Fetch vuls-data-raw-${{ matrix.target }}
182+
if: ${{ success() || failure() }}
183+
needs: fetch-cisco-json
175184
strategy:
176185
fail-fast: false
177-
max-parallel: 1
178186
matrix:
179187
target:
180-
- "nvd-api-cve"
181-
- "nvd-api-cpe"
182-
- "nvd-api-cpematch"
183-
uses: ./.github/workflows/fetch-nvd-api.yml
188+
- "cisco-cvrf"
189+
- "cisco-csaf"
190+
uses: ./.github/workflows/fetch-cisco-cvrf-or-csaf.yml
184191
with:
185192
target: ${{ matrix.target }}
186-
secrets:
187-
NVD_API_KEY: ${{ secrets.NVD_API_KEY }}
188193

189-
fetch-redhat-package-manifest:
190-
name: Fetch vuls-data-raw-redhat-package-manifest
191-
uses: ./.github/workflows/fetch-redhat-package-manifest.yml
194+
fetch-epss:
195+
name: Fetch vuls-data-raw-epss
196+
uses: ./.github/workflows/fetch-epss.yml
192197

193-
fetch-msuc:
194-
name: Fetch vuls-data-raw-microsoft-msuc
195-
uses: ./.github/workflows/fetch-msuc.yml
198+
fetch-euvd:
199+
name: Fetch vuls-data-raw-euvd
200+
uses: ./.github/workflows/fetch-euvd.yml
196201

197202
fetch-fedora:
198203
name: Fetch vuls-data-raw-fedora
199204
uses: ./.github/workflows/fetch-fedora.yml
200205

201-
fetch-epss:
202-
name: Fetch vuls-data-raw-epss
203-
uses: ./.github/workflows/fetch-epss.yml
204-
205206
fetch-fortinet-cvrf:
206207
name: Fetch vuls-data-raw-fortinet-cvrf
207208
uses: ./.github/workflows/fetch-fortinet-cvrf.yml
208209

209-
fetch-vulncheck:
210-
name: Fetch vuls-data-raw-${{ matrix.target }}
211-
strategy:
212-
fail-fast: false
213-
matrix:
214-
target:
215-
- "vulncheck-kev"
216-
- "vulncheck-nist-nvd"
217-
- "vulncheck-nist-nvd2"
218-
uses: ./.github/workflows/fetch-vulncheck.yml
219-
with:
220-
target: ${{ matrix.target }}
221-
secrets:
222-
VULNCHECK_API_KEY: ${{ secrets.VULNCHECK_API_KEY }}
223-
224-
fetch-cisco-json:
225-
name: Fetch vuls-data-raw-cisco-json
226-
uses: ./.github/workflows/fetch-cisco-json.yml
227-
secrets:
228-
CISCO_CLIENT_KEY: ${{ secrets.CISCO_CLIENT_KEY }}
229-
CISCO_CLIENT_SECRET: ${{ secrets.CISCO_CLIENT_SECRET }}
210+
fetch-msuc:
211+
name: Fetch vuls-data-raw-microsoft-msuc
212+
uses: ./.github/workflows/fetch-msuc.yml
230213

231-
fetch-cisco-cvrf-or-csaf:
214+
fetch-nvd-api:
232215
name: Fetch vuls-data-raw-${{ matrix.target }}
233-
if: ${{ success() || failure() }}
234-
needs: fetch-cisco-json
235216
strategy:
236217
fail-fast: false
218+
max-parallel: 1
237219
matrix:
238220
target:
239-
- "cisco-cvrf"
240-
- "cisco-csaf"
241-
uses: ./.github/workflows/fetch-cisco-cvrf-or-csaf.yml
221+
- "nvd-api-cve"
222+
- "nvd-api-cpe"
223+
- "nvd-api-cpematch"
224+
uses: ./.github/workflows/fetch-nvd-api.yml
242225
with:
243226
target: ${{ matrix.target }}
227+
secrets:
228+
NVD_API_KEY: ${{ secrets.NVD_API_KEY }}
244229

245230
fetch-paloalto-json-or-csaf:
246231
name: Fetch vuls-data-raw-${{ matrix.target }}
@@ -256,6 +241,10 @@ jobs:
256241
with:
257242
target: ${{ matrix.target }}
258243

244+
fetch-redhat-package-manifest:
245+
name: Fetch vuls-data-raw-redhat-package-manifest
246+
uses: ./.github/workflows/fetch-redhat-package-manifest.yml
247+
259248
fetch-variot:
260249
name: Fetch vuls-data-raw-${{ matrix.target }}
261250
strategy:
@@ -270,22 +259,39 @@ jobs:
270259
secrets:
271260
VARIOT_API_KEY: ${{ secrets.VARIOT_API_KEY }}
272261

262+
fetch-vulncheck:
263+
name: Fetch vuls-data-raw-${{ matrix.target }}
264+
strategy:
265+
fail-fast: false
266+
matrix:
267+
target:
268+
- "vulncheck-kev"
269+
- "vulncheck-nist-nvd"
270+
- "vulncheck-nist-nvd2"
271+
uses: ./.github/workflows/fetch-vulncheck.yml
272+
with:
273+
target: ${{ matrix.target }}
274+
secrets:
275+
VULNCHECK_API_KEY: ${{ secrets.VULNCHECK_API_KEY }}
276+
273277
check:
274278
name: Decide whether to Git GC
275279
if: ${{ success() || failure() }}
276280
needs:
277281
[
278282
fetch-main,
279-
fetch-nvd-api,
280-
fetch-msuc,
281-
fetch-fedora,
282-
fetch-epss,
283-
fetch-fortinet-cvrf,
284-
fetch-vulncheck,
285283
fetch-cisco-json,
286284
fetch-cisco-cvrf-or-csaf,
285+
fetch-epss,
286+
fetch-euvd,
287+
fetch-fedora,
288+
fetch-fortinet-cvrf,
289+
fetch-nvd-api,
290+
fetch-msuc,
287291
fetch-paloalto-json-or-csaf,
292+
fetch-redhat-package-manifest,
288293
fetch-variot,
294+
fetch-vulncheck,
289295
]
290296
runs-on: ubuntu-latest
291297
outputs:

‎.github/workflows/fetch-euvd.yml‎

Lines changed: 135 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,135 @@
1+
name: Fetch EUVD
2+
3+
on:
4+
workflow_call:
5+
workflow_dispatch:
6+
7+
jobs:
8+
generate-year-month:
9+
name: Generate year-month
10+
runs-on: ubuntu-latest
11+
outputs:
12+
matrix: ${{ steps.generate.outputs.matrix }}
13+
steps:
14+
- name: Generate Year-Month
15+
id: generate
16+
run: |
17+
current_year=$(date +%Y)
18+
current_month=$(date +%m)
19+
20+
json="{\"include\":["
21+
22+
for year in $(seq 1999 $current_year); do
23+
for month in $(seq -w 1 12); do
24+
if [ "$year" -eq "$current_year" ] && [ "$month" -gt "$current_month" ]; then
25+
break
26+
fi
27+
json="$json{\"year\":\"$year\",\"month\":\"$month\"},"
28+
done
29+
done
30+
31+
json="${json%,}]}"
32+
echo "matrix=$json" >> $GITHUB_OUTPUT
33+
34+
fetch:
35+
name: Fetch vuls-data-raw-euvd ${{ matrix.year }}-${{ matrix.month }}
36+
runs-on: ubuntu-latest
37+
needs: generate-year-month
38+
strategy:
39+
fail-fast: false
40+
matrix: ${{ fromJson(needs.generate-year-month.outputs.matrix) }}
41+
steps:
42+
- name: Maximize build space
43+
uses: easimon/maximize-build-space@v10
44+
with:
45+
root-reserve-mb: 32768
46+
remove-dotnet: "true"
47+
remove-android: "true"
48+
remove-haskell: "true"
49+
remove-codeql: "true"
50+
remove-docker-images: "true"
51+
52+
- name: Check out code into the Go module directory
53+
uses: actions/checkout@v5
54+
with:
55+
repository: MaineK00n/vuls-data-update
56+
ref: main
57+
58+
- name: Set up Go 1.x
59+
uses: actions/setup-go@v6
60+
with:
61+
go-version-file: "go.mod"
62+
63+
- name: Install vuls-data-update
64+
run: go install ./cmd/vuls-data-update
65+
66+
- name: Fetch
67+
run: |
68+
mkdir vuls-data-raw-euvd
69+
vuls-data-update fetch euvd --dir vuls-data-raw-euvd ${{ matrix.year }}-${{ matrix.month }} --retry 10
70+
71+
- name: Create tarball
72+
run: tar --remove-files -acf vuls-data-raw-euvd.tar.zst vuls-data-raw-euvd || [[ $? == 1 ]]
73+
74+
- name: Push ghcr.io/vulsio/vuls-data-db:euvd-data-${{ matrix.year }}-${{ matrix.month }}
75+
run: vuls-data-update dotgit registry push --force --token ${{ secrets.GITHUB_TOKEN }} ghcr.io/vulsio/vuls-data-db:euvd-data-${{ matrix.year }}-${{ matrix.month }} vuls-data-raw-euvd.tar.zst
76+
77+
commit:
78+
name: Commit vuls-data-raw-euvd
79+
runs-on: ubuntu-latest
80+
if: ${{ success() || failure() }}
81+
needs: [generate-year-month, fetch]
82+
steps:
83+
- name: Maximize build space
84+
uses: easimon/maximize-build-space@v10
85+
with:
86+
root-reserve-mb: 32768
87+
remove-dotnet: "true"
88+
remove-android: "true"
89+
remove-haskell: "true"
90+
remove-codeql: "true"
91+
remove-docker-images: "true"
92+
93+
- name: Set up Go 1.x
94+
uses: actions/setup-go@v6
95+
with:
96+
go-version: "stable"
97+
98+
- name: Install vuls-data-update
99+
run: go install github.com/MaineK00n/vuls-data-update/cmd/vuls-data-update@main
100+
101+
- name: Pull ghcr.io/${{ github.repository }}:vuls-data-raw-euvd
102+
run: vuls-data-update dotgit pull --dir . --checkout main ghcr.io/${{ github.repository }}:vuls-data-raw-euvd
103+
104+
- name: Aggregate EUVD data
105+
run: |
106+
for ym in $(cat ${{ needs.generate-year-month.outputs.matrix }} | jq -r '.include[] | .year + "-" + .month'); do
107+
vuls-data-update dotgit pull --dir . --checkout "" ghcr.io/${{ github.repository }}:euvd-data-${ym}
108+
if ls ghcr.io/${{ github.repository }}/euvd-data-${ym}/* >/dev/null 2>&1; then
109+
mv ghcr.io/${{ github.repository }}/euvd-data-${ym}/* ghcr.io/${{ github.repository }}/vuls-data-raw-euvd/
110+
fi
111+
rmdir ghcr.io/${{ github.repository }}/euvd-data-${ym}
112+
done
113+
114+
- name: Set Git config
115+
run: |
116+
if git -C ghcr.io/${{ github.repository }}/vuls-data-raw-euvd remote | grep -q "^origin$"; then
117+
git -C ghcr.io/${{ github.repository }}/vuls-data-raw-euvd remote set-url origin ghcr.io/${{ github.repository }}:vuls-data-raw-euvd
118+
else
119+
git -C ghcr.io/${{ github.repository }}/vuls-data-raw-euvd remote add origin ghcr.io/${{ github.repository }}:vuls-data-raw-euvd
120+
fi
121+
git -C ghcr.io/${{ github.repository }}/vuls-data-raw-euvd config user.email "action@github.com"
122+
git -C ghcr.io/${{ github.repository }}/vuls-data-raw-euvd config user.name "GitHub Action"
123+
124+
- name: Commit
125+
run: |
126+
if [[ -n $(git -C ghcr.io/${{ github.repository }}/vuls-data-raw-euvd status --porcelain) ]]; then
127+
git -C ghcr.io/${{ github.repository }}/vuls-data-raw-euvd add .
128+
git -C ghcr.io/${{ github.repository }}/vuls-data-raw-euvd commit -m "update" -m "GitHub Actions: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}/job/${{ job.check_run_id }}"
129+
fi
130+
131+
- name: Create dotgit tarball
132+
run: vuls-data-update dotgit compress ghcr.io/${{ github.repository }}/vuls-data-raw-euvd
133+
134+
- name: Push ghcr.io/${{ github.repository }}:vuls-data-raw-euvd
135+
run: vuls-data-update dotgit registry push --force --token ${{ secrets.GITHUB_TOKEN }} ghcr.io/${{ github.repository }}:vuls-data-raw-euvd vuls-data-raw-euvd.tar.zst

‎.github/workflows/gc-raw.yml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -129,6 +129,9 @@ jobs:
129129
- tag: vuls-data-raw-erlang-osv
130130
pack-threads: 2
131131
pack-windowMemory: 4g
132+
- tag: vuls-data-raw-euvd
133+
pack-threads: 2
134+
pack-windowMemory: 4g
132135
- tag: vuls-data-raw-exploit-exploitdb
133136
pack-threads: 2
134137
pack-windowMemory: 4g

‎.github/workflows/gc.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -93,6 +93,7 @@ on:
9393
- vuls-data-raw-epss
9494
- vuls-data-raw-erlang-ghsa
9595
- vuls-data-raw-erlang-osv
96+
- vuls-data-raw-euvd
9697
- vuls-data-raw-exploit-exploitdb
9798
- vuls-data-raw-exploit-github
9899
- vuls-data-raw-exploit-inthewild

‎.github/workflows/restore-all.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -84,6 +84,7 @@ jobs:
8484
- vuls-data-raw-epss
8585
- vuls-data-raw-erlang-ghsa
8686
- vuls-data-raw-erlang-osv
87+
- vuls-data-raw-euvd
8788
- vuls-data-raw-exploit-exploitdb
8889
- vuls-data-raw-exploit-github
8990
- vuls-data-raw-exploit-inthewild

0 commit comments

Comments
 (0)