Verify Site Links #1725
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Verify Site Links | |
| on: | |
| # push: {} # uncommnet for testing | |
| schedule: # daily at 12 noon UTC | |
| - cron: '0 12 * * *' | |
| jobs: | |
| build: | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - name: content checkout | |
| uses: actions/checkout@v2.4.0 # https://github.com/marketplace/actions/checkout | |
| with: | |
| repository: demisto/content | |
| path: content | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| fetch-depth: 0 | |
| ref: 'master' | |
| - name: content-docs checkout | |
| uses: actions/checkout@v2.4.0 # https://github.com/marketplace/actions/checkout | |
| with: | |
| repository: demisto/content-docs | |
| path: content-docs | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| fetch-depth: 0 | |
| ref: 'master' | |
| - name: create ignore url files | |
| # The grep regex finds all URL's in integration README files in the section between ## Commands or <h2>Commands until the end of the file. | |
| # -R search recursively -P perl regex -o print only the match -z ignore new line -a process a binary file as if it were text | |
| run: | | |
| grep -RPoz "(?s)## Commands.*\Z|<h2>Commands.*\Z" --include "README.md" content/Packs/*/Integrations | grep -Poa "(http|https)://[a-zA-Z0-9./?=_%:-]*" | sort -u > all_urls_ignore.txt | |
| sed -i -e 's/^/-e "/' all_urls_ignore.txt | |
| sed -i -e 's/$/"/' all_urls_ignore.txt | |
| cat content-docs/urls_ignore.txt >> all_urls_ignore.txt | |
| cat all_urls_ignore.txt | |
| - name: Set up Go 1.13 | |
| uses: actions/setup-go@v2 | |
| with: | |
| go-version: 1.13.11 | |
| id: go | |
| - name: Setup muffet | |
| run: 'go get -u github.com/raviqqe/muffet' | |
| - name: Verify Links | |
| run: | | |
| muffet --version | |
| muffet -c 16 --ignore-fragments -b 8192 `< all_urls_ignore.txt` \ | |
| https://xsoar.pan.dev > muffet.txt 2>&1 && echo "muffet completed successfully" && exit 0 || echo "muffet failed but still need to verify output file..." | |
| grep -v -E '^\s+(401|403|405|429|x509|lookup|parse|timeout|dial)' muffet.txt > muffet-clean.out | |
| echo "Muffet output after 401, 403, 405, 429 and x509 remove:" | |
| cat muffet-clean.out | |
| if [ -n "$(grep -v '^https' muffet-clean.out)" ]; then | |
| echo "muffet failed verification!!!" | |
| exit 1 | |
| fi | |
| echo "All good! muffet passed!!!" | |
| - name: Save artifcats | |
| if: ${{ always() }} | |
| uses: actions/upload-artifact@v2 | |
| with: | |
| name: log-artifacts | |
| path: muffet.txt |