extend CI

.codespell.ignore.files

@@ -0,0 +1 @@
+venv,.git

.codespell.ignore.words

@@ -0,0 +1 @@
+

.github/workflows/formulaCI.yml

@@ -3,58 +3,183 @@ name: Formula CI
 # - run formula validation step - ci/validate-formula.sh
 # - deliver the package content to the configured repository
 # - submit the new package content to the upstream repository
-on: [push, pull_request]
+on: [push, pull_request]  # yamllint disable-line rule:truthy
 env:
-  PACKAGE_NAME: saphanabootstrap-formula 
+  PACKAGE_NAME: saphanabootstrap-formula
 jobs:
+  tab:
+    name: 'tabspace checking'
+    runs-on: ubuntu-20.04
+
+    # Use the Bash shell regardless whether the GitHub Actions runner is ubuntu-latest, macos-latest, or windows-latest
+    defaults:
+      run:
+        shell: bash
+
+    steps:
+      # Checkout the repository to the GitHub Actions runner
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: tab
+        run: make test-tab
+  codespell:
+    name: 'spell checking'
+    runs-on: ubuntu-20.04
+
+    # Use the Bash shell regardless whether the GitHub Actions runner is ubuntu-latest, macos-latest, or windows-latest
+    defaults:
+      run:
+        shell: bash
+
+    steps:
+      # Checkout the repository to the GitHub Actions runner
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Install linting tools
+        run: |
+          sudo apt-get install -y git python3 python3-pip
+          python3 -m pip install codespell
+
+      - name: codespell
+        run: make test-codespell
+
+  shellcheck:
+    name: 'script syntax check'
+    runs-on: ubuntu-20.04
+
+    # Use the Bash shell regardless whether the GitHub Actions runner is ubuntu-latest, macos-latest, or windows-latest
+    defaults:
+      run:
+        shell: bash
+
+    steps:
+      # Checkout the repository to the GitHub Actions runner
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Install linting tools
+        run: |
+          sudo apt-get install -y git python3 python3-pip shellcheck
+
+      - name: shellcheck
+        run: make test-shellcheck
+
+  yamllint:
+    name: 'yaml linting'
+    runs-on: ubuntu-20.04
+
+    # Use the Bash shell regardless whether the GitHub Actions runner is ubuntu-latest, macos-latest, or windows-latest
+    defaults:
+      run:
+        shell: bash
 
-  validation:
-    runs-on: ubuntu-18.04
     steps:
-    - uses: actions/checkout@v2
-      with:
-        fetch-depth: 0
-    - name: Add salt repository
-      uses: myci-actions/add-deb-repo@4
-      with:
-        repo: deb http://repo.saltstack.com/apt/ubuntu/18.04/amd64/3000/ bionic main
-        key-server: http://repo.saltstack.com/apt/ubuntu/18.04/amd64/3000/SALTSTACK-GPG-KEY.pub
-    - name: install salt
-      run: sudo apt-get install salt-common
-    - name: validation script
-      run: ci/validate-formula.sh
+      # Checkout the repository to the GitHub Actions runner
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Install linting tools
+        run: |
+          sudo apt-get install -y git python3 python3-pip
+          python3 -m pip install codespell
+
+      - name: yamllint
+        run: make test-yamllint
+
+  jsonlint:
+    name: 'json linting'
+    runs-on: ubuntu-20.04
+
+    # Use the Bash shell regardless whether the GitHub Actions runner is ubuntu-latest, macos-latest, or windows-latest
+    defaults:
+      run:
+        shell: bash
+
+    steps:
+      # Checkout the repository to the GitHub Actions runner
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Install linting tools
+        run: |
+          sudo apt-get install -y git python3 python3-pip
+          python3 -m pip install jsonlint
+
+      - name: jsonlint
+        run: make test-jsonlint
+
+  mlc:
+    name: 'markup link checker'
+    runs-on: ubuntu-20.04
+
+    # Use the Bash shell regardless whether the GitHub Actions runner is ubuntu-latest, macos-latest, or windows-latest
+    defaults:
+      run:
+        shell: bash
+
+    steps:
+      # Checkout the repository to the GitHub Actions runner
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Install linting tools
+        run: |
+          mkdir -p bin
+          curl -L https://github.com/becheran/mlc/releases/download/v0.14.3/mlc-x86_64-linux -o bin/mlc
+          chmod +x bin/mlc
+          echo "$PWD/bin" >> $GITHUB_PATH
+
+      - name: mlc
+        run: make test-mlc
+
+  salt:
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+      - name: install salt
+        run: |
+          sudo curl -fsSL -o /usr/share/keyrings/salt-archive-keyring.gpg https://repo.saltproject.io/py3/ubuntu/20.04/amd64/latest/salt-archive-keyring.gpg
+          echo "deb [signed-by=/usr/share/keyrings/salt-archive-keyring.gpg arch=amd64] https://repo.saltproject.io/py3/ubuntu/20.04/amd64/latest focal main" | sudo tee /etc/apt/sources.list.d/salt.list
+          sudo apt-get update
+          sudo apt-get install salt-common
+      - name: salt
+        run: make test-salt
 
   delivery:
-    needs: validation
-    runs-on: ubuntu-18.04
-    if: ${{ github.event_name != 'pull_request' }}  
-    container: 
+    needs: [tab, codespell, shellcheck, yamllint, jsonlint, mlc, salt]
+    runs-on: ubuntu-20.04
+    if: ${{ github.event_name != 'pull_request' }}
+    container:
       image: shap/continuous_deliver
       env:
         OBS_USER: ${{ secrets.OBS_USER }}
         OBS_PASS: ${{ secrets.OBS_PASS }}
         OBS_PROJECT: ${{ secrets.OBS_PROJECT }}
     steps:
-    - uses: actions/checkout@v2 
-      with:
-        fetch-depth: 0
-    - name: configure OSC  
-    # OSC credentials must be configured beforehand as the HOME variables cannot be changed from /github/home
-    # that is used to run osc commands 
-      run: | 
-        /scripts/init_osc_creds.sh
-        mkdir -p $HOME/.config/osc
-        cp /root/.config/osc/oscrc $HOME/.config/osc
-    - name: deliver package
-      run: |
-        sed -i 's~%%VERSION%%~${{ github.sha }}~' _service && \
-        sed -i 's~%%REPOSITORY%%~${{ github.repository }}~' _service && \
-        /scripts/upload.sh 
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+      - name: configure OSC
+        # OSC credentials must be configured beforehand as the HOME variables cannot be changed from /github/home
+        # that is used to run osc commands
+        run: |
+          /scripts/init_osc_creds.sh
+          mkdir -p $HOME/.config/osc
+          cp /root/.config/osc/oscrc $HOME/.config/osc
+      - name: deliver package
+        run: |
+          sed -i 's~%%VERSION%%~${{ github.sha }}~' _service && \
+          sed -i 's~%%REPOSITORY%%~${{ github.repository }}~' _service && \
+          /scripts/upload.sh
 
   submit:
-    needs: [validation, delivery]
-    runs-on: ubuntu-18.04
-    if: ${{ github.event_name != 'pull_request' && github.ref == 'refs/heads/master' }}    
+    needs: [tab, codespell, shellcheck, yamllint, jsonlint, mlc, salt, delivery]
+    runs-on: ubuntu-20.04
+    if: ${{ github.event_name != 'pull_request' && github.ref == 'refs/heads/master' }}
     container:
       image: shap/continuous_deliver
       env:
@@ -63,16 +188,16 @@ jobs:
         OBS_PROJECT: ${{ secrets.OBS_PROJECT}}
         TARGET_PROJECT: ${{ secrets.TARGET_PROJECT}}
     steps:
-    - uses: actions/checkout@v2
-      with:
-        fetch-depth: 0
-    - name: configure OSC
-      run: | 
-        /scripts/init_osc_creds.sh
-        mkdir -p $HOME/.config/osc
-        cp /root/.config/osc/oscrc $HOME/.config/osc
-    - name: submit package
-      run: |
-       sed -i 's~%%VERSION%%~${{ github.sha }}~' _service && \
-       sed -i 's~%%REPOSITORY%%~${{ github.repository }}~' _service && \
-       /scripts/submit.sh
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+      - name: configure OSC
+        run: |
+          /scripts/init_osc_creds.sh
+          mkdir -p $HOME/.config/osc
+          cp /root/.config/osc/oscrc $HOME/.config/osc
+      - name: submit package
+        run: |
+         sed -i 's~%%VERSION%%~${{ github.sha }}~' _service && \
+         sed -i 's~%%REPOSITORY%%~${{ github.repository }}~' _service && \
+         /scripts/submit.sh

.gitignore

@@ -3,3 +3,8 @@
 /minion
 /top.sls
 /var/
+venv
+.envrc
+.direnv
+shell.nix
+.ropeproject

.yamllint.yaml

@@ -0,0 +1,11 @@
+---
+extends: default
+
+ignore: |
+  venv
+
+rules:
+  # 80 chars should be enough, but don't fail if a line is longer
+  line-length:
+    max: 160
+    level: warning

Makefile

@@ -0,0 +1,98 @@
+# kudos:
+#   - https://medium.com/@exustash/three-good-practices-for-better-ci-cd-makefiles-5b93452e4cc3
+#   - https://le-gall.bzh/post/makefile-based-ci-chain-for-go/
+#   - https://makefiletutorial.com/
+#   - https://www.cl.cam.ac.uk/teaching/0910/UnixTools/make.pdf
+#
+#SHELL := /usr/bin/bash # set default shell
+#.SHELLFLAGS = -c # Run commands in a -c flag 
+
+.NOTPARALLEL: ;          # wait for this target to finish
+.EXPORT_ALL_VARIABLES: ; # send all vars to shell
+
+.PHONY: all # All targets are accessible for user
+.DEFAULT: help # Running Make will run the help target
+
+BRANCH := $(shell git rev-parse --abbrev-ref HEAD)
+ifeq ($(BRANCH), HEAD)
+	BRANCH := ${CI_BUILD_REF_NAME}
+endif
+
+# help: @ List available tasks of the project
+help:
+	@grep -E '[a-zA-Z\.\-]+:.*?@ .*$$' $(MAKEFILE_LIST)| tr -d '#'  | awk 'BEGIN {FS = ":.*?@ "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
+
+## test section
+# All tests are called on "." if possible.
+# If this is not possible a special loop is used
+# to sum up all error codes.
+
+# test: @ Run all defined tests
+test: test-tab test-codespell test-shellcheck test-yamllint test-jsonlint test-salt
+	@echo "All tests Done!"
+
+# test-tab: @ Run linting to find files containing tabspaces
+test-tab:
+	@for file in $(shell find . -regextype egrep -regex '.*\.(sls|yml|yaml)' ! -path "**/venv/*"); do\
+		grep -q -P '\t' $${file} ;\
+		if [ "$$?" -eq 0 ]; then\
+			err_add=1 ;\
+			echo "Tab found in $${file}" ;\
+			grep -H -n -P '\t' $${file} ;\
+		else \
+			err_add=0 ;\
+		fi;\
+		err=$$((err_add + err)) ;\
+	done; exit $$err
+
+# test-codespell: @ Run spell check
+test-codespell:
+	codespell -H -f -s -I .codespell.ignore.words -S $(shell cat .codespell.ignore.files) -C 4 -q 6
+
+# test-shellcheck: @ Run linting on all shell scripts
+test-shellcheck:
+	for file in $(shell find . -name '*.sh' ! -path "**/venv/*"); do\
+		echo $${file} ;\
+		shellcheck -s bash -x $${file};\
+		err=$$(($$? + err)) ;\
+	done; exit $$err
+
+# test-yamllint: @ Run linting on all yaml files
+test-yamllint:
+	# yamllint -c .yamllint.yaml -s .
+	yamllint -c .yamllint.yaml .
+
+# test-jsonlint: @ Run linting on all json files
+test-jsonlint:
+	for file in $(shell find . -name '*.json' ! -path "**/venv/*"); do\
+		echo $${file} ;\
+		jq << $${file} >/dev/null;\
+		err=$$(($$? + err)) ;\
+	done; exit $$err
+
+# test-mlc: @ Run markup link checker
+test-mlc:
+	mkdir -p aws/.terraform # make sure ingore-path exists
+	mlc --throttle 1000
+
+# test-salt: @ Run Salt Unit Tests
+test-salt:
+	cp pillar.example example/pillar/hana.sls
+	cp example/salt/top.sls .
+	echo "==========================================="
+	echo " Using primary host                      "
+	echo "==========================================="
+	cp ci/grains_hana01 grains
+	cp ci/minion minion
+	salt-call state.show_highstate --local --file-root=./ --config-dir=. --pillar-root=example/pillar --retcode-passthrough -l debug
+	echo
+	echo "==========================================="
+	echo " Using secondary host                      "
+	echo "==========================================="
+	cp ci/grains_hana02 grains
+	cp ci/minion minion
+	salt-call state.show_highstate --local --file-root=./ --config-dir=. --pillar-root=example/pillar --retcode-passthrough -l debug
+
+
+# all: @ Runs everything
+all: test

ci/grains_hana01

@@ -0,0 +1 @@
+host: hana01

ci/grains_hana02

@@ -0,0 +1,2 @@
+host: hana02
+hana_inst_folder: myfold

ci/minion

@@ -0,0 +1,2 @@
+root_dir: ./
+id: ci