feat: local fix mode (#5978)

Certain linters and formatters support fixing linting and formatting
issues (fix mode). Before this change, Super-linter runs linters and
formatters in a mode that doesn't modify the source code in any way
(check only mode).

With this change, Super-linter supports running linters and formatters
in fix mode if explicitly requested by the configuration. If the
configuration includes a variable named FIX_<language_name>,
Super-linters modifies the command to run the linter or formatter for
<language_name> to enable fix mode.

The modifications to the linter or formatter command that Super-linter
applies depend on what is the default for a particular linter: it either
removes or adds options to the command to run the linter or formatter.

.gitignore

@@ -89,7 +89,7 @@ test/reports
 # Test leftovers
 .lintr
 test/linters/rust_clippy/**/Cargo.lock
-test/linters/rust_clippy/**/target/**
+test/linters/rust_clippy/**/target
 
 # Super-linter ouputs
 super-linter-output/

Makefile

@@ -4,7 +4,7 @@
 all: info docker test ## Run all targets.
 
 .PHONY: test
-test: info validate-container-image-labels docker-build-check docker-dev-container-build-check test-lib inspec lint-codebase test-default-config-files test-actions-runner-debug test-actions-steps-debug test-runner-debug test-find lint-subset-files test-custom-ssl-cert test-non-default-workdir test-git-flags test-non-default-home-directory test-git-initial-commit test-git-merge-commit-push test-log-level test-use-find-and-ignore-gitignored-files test-linters-expect-failure-log-level-notice test-bash-exec-library-expect-success test-bash-exec-library-expect-failure test-save-super-linter-output test-save-super-linter-output-custom-path test-save-super-linter-custom-summary test-linters ## Run the test suite
+test: info validate-container-image-labels docker-build-check docker-dev-container-build-check test-lib inspec lint-codebase fix-codebase test-default-config-files test-actions-runner-debug test-actions-steps-debug test-runner-debug test-find lint-subset-files test-custom-ssl-cert test-non-default-workdir test-git-flags test-non-default-home-directory test-git-initial-commit test-git-merge-commit-push test-log-level test-use-find-and-ignore-gitignored-files test-linters-expect-failure-log-level-notice test-bash-exec-library-expect-success test-bash-exec-library-expect-failure test-save-super-linter-output test-save-super-linter-output-custom-path test-save-super-linter-custom-summary test-linters test-linters-fix-mode-expect-success ## Run the test suite
 
 # if this session isn't interactive, then we don't want to allocate a
 # TTY, which would fail, but if it is interactive, we do want to attach
@@ -231,18 +231,45 @@ test-git-flags: ## Run super-linter with different git-related flags
 .PHONY: lint-codebase
 lint-codebase: ## Lint the entire codebase
 	docker run \
+		-e CREATE_LOG_FILE=true \
 		-e RUN_LOCAL=true \
 		-e LOG_LEVEL=DEBUG \
 		-e DEFAULT_BRANCH=main \
 		-e ENABLE_GITHUB_ACTIONS_GROUP_TITLE=true \
 		-e FILTER_REGEX_EXCLUDE=".*(/test/linters/|CHANGELOG.md).*" \
 		-e GITLEAKS_CONFIG_FILE=".gitleaks-ignore-tests.toml" \
 		-e RENOVATE_SHAREABLE_CONFIG_PRESET_FILE_NAMES="default.json,hoge.json" \
+		-e SAVE_SUPER_LINTER_OUTPUT=true \
 		-e SAVE_SUPER_LINTER_SUMMARY=true \
 		-e VALIDATE_ALL_CODEBASE=true \
 		-v "$(CURDIR):/tmp/lint" \
 		$(SUPER_LINTER_TEST_CONTAINER_URL)
 
+# Return an error if there are changes to commit
+.PHONY: fix-codebase
+fix-codebase: ## Fix and format the entire codebase
+	docker run \
+		-e CREATE_LOG_FILE=true \
+		-e DEFAULT_BRANCH=main \
+		-e ENABLE_GITHUB_ACTIONS_GROUP_TITLE=true \
+		-e FILTER_REGEX_EXCLUDE=".*(/test/linters/|CHANGELOG.md).*" \
+		-e FIX_ENV=true \
+		-e FIX_JAVASCRIPT_ES=true \
+		-e FIX_JAVASCRIPT_PRETTIER=true \
+		-e FIX_JSON=true \
+		-e FIX_MARKDOWN=true \
+		-e FIX_SHELL_SHFMT=true \
+		-e GITLEAKS_CONFIG_FILE=".gitleaks-ignore-tests.toml" \
+		-e LOG_LEVEL=DEBUG \
+		-e RUN_LOCAL=true \
+		-e SAVE_SUPER_LINTER_OUTPUT=true \
+		-e SAVE_SUPER_LINTER_SUMMARY=true \
+		-e VALIDATE_ALL_CODEBASE=true \
+		-v "$(CURDIR):/tmp/lint" \
+		$(SUPER_LINTER_TEST_CONTAINER_URL) \
+	&& /bin/bash -c "source test/testUtils.sh; if ! CheckUnexpectedGitChanges ${CURDIR}; then exit 1; fi"
+
+
 # This is a smoke test to check how much time it takes to lint only a small
 # subset of files, compared to linting the whole codebase.
 .PHONY: lint-subset-files
@@ -389,6 +416,14 @@ test-non-default-home-directory: ## Test a non-default HOME directory
 		"run_test_cases_non_default_home" \
 		"$(IMAGE)"
 
+.PHONY: test-linters-fix-mode-expect-success
+test-linters-fix-mode-expect-success: ## Run the linters test suite (fix mode) expecting successes
+	$(CURDIR)/test/run-super-linter-tests.sh \
+		$(SUPER_LINTER_TEST_CONTAINER_URL) \
+		"run_test_case_fix_mode" \
+		"$(IMAGE)"
+
+
 .PHONY: test-linters
 test-linters: test-linters-expect-success test-linters-expect-failure ## Run the linters test suite
 

docs/add-new-linter.md

@@ -8,8 +8,10 @@ new tool, it should include:
 - Provide test cases:
 
   1. Create the `test/linters/<LANGUGAGE>` directory.
-  2. Provide at least one test case with a file that is supposed to pass validation: `test/linters/<LANGUAGE>/<name-of-tool>-good`
-  3. Provide at least one test case with a file that is supposed to fail validation: `test/linters/<LANGUAGE>/<name-of-tool>-bad`
+  2. Provide at least one test case with a file that is supposed to pass validation,
+    with the right file extension if needed: `test/linters/<LANGUAGE>/<name-of-tool>-good`
+  3. Provide at least one test case with a file that is supposed to fail validation,
+    with the right file extension if needed: `test/linters/<LANGUAGE>/<name-of-tool>-bad`
 
 - Update the test suite to check for installed packages, the commands that your new tool needs in the `PATH`, and the expected version command:
 
@@ -89,28 +91,73 @@ new tool, it should include:
 
 - Update the orchestration scripts to run the new tool:
 
-  - `globals/languages.sh`: add a new item to `LANGUAGES_ARRAY` array. Use the
-    "name" of the language, then a `_`, and finally the name of the linter. Example: `PYTHON_RUFF`
+  - `lib/globals/languages.sh`: add a new item to `LANGUAGES_ARRAY` array. Use the
+    "name" of the language, then a `_`, and finally the name of the linter. Example: `PYTHON_RUFF`.
+    In the context of this document, to avoid repetitions we reference this new
+    item as `<LANGUAGE_NAME>`.
+
   - Linter configuration:
-    - `globals/linterRules.sh`:
-      - If the new linter accepts a configuration files from the command line, add a new variable
-        with a default filename using the item that you added to the `LANGUAGES_ARRAY` as a prefix,
-        followed by the `CONFIG_FILE` suffix. Example:
-        `PYTHON_RUFF_FILE_NAME="${PYTHON_RUFF_CONFIG_FILE:-.ruff.toml}"`.
-      - If there are arguments that you can only pass using the command line, and you think users
-        might want to customize them, define a new variable using the item that
-        you added to the `LANGUAGES_ARRAY` as a prefix, followed by the
-        `COMMAND_ARGS` suffix. Example:
-        `GITHUB_ACTIONS_COMMAND_ARGS="${GITHUB_ACTIONS_COMMAND_ARGS:-""}"`
     - Create a new minimal configuration file in the `TEMPLATES` directory with the same name as the
       default configuration filename. Example: `TEMPLATES/.ruff.toml`.
-  - `lib/linter.sh`
-  - `lib/functions/linterCommands.sh`: define a new array to invoke the new linter.
-  - Provide the logic to populate the list of files or directories to examine: `lib/buildFileList.sh`
-  - If necessary, provide elaborate logic to detect if the tool should examine a file or a directory: `lib/detectFiles.sh`
-  - If the tool needs to take into account special cases:
-
-    - Provide new runtime validation checks in `lib/validation.sh`.
-    - Customize the logic to get the installed version of the tool: `scripts/linterVersions.sh`
-    - Provide custom logic to load configuration files: `lib/linterRules.sh`
-    - Provide custom logic for test cases and to run the tool: `lib/worker.sh`
+    - `lib/globals/linterRules.sh`:
+      - If the new linter accepts a configuration files from the command line,
+        define a new variable:
+        `<LANGUAGE_NAME>_FILE_NAME="${<LANGUAGE_NAME>_CONFIG_FILE:-"default-config-file-name.conf"}"`
+        where `default-config-file-name.conf` is the name of the new,
+        minimal configuration for the linter. Example:
+        `PYTHON_RUFF_FILE_NAME="${PYTHON_RUFF_CONFIG_FILE:-.ruff.toml}"`.
+      - If there are arguments that you can only pass using the command line, and you think users
+        might want to customize them, define a new variable using
+        `<LANGUAGE_NAME>_COMMAND_ARGS` and add it to the command if the
+        configuration provides it. Example:
+
+          ```bash
+          <LANGUAGE_NAME>_COMMAND_ARGS="${<LANGUAGE_NAME>_COMMAND_ARGS:-""}"
+          if [ -n "${<LANGUAGE_NAME>_COMMAND_ARGS:-}" ]; then
+            export <LANGUAGE_NAME>_COMMAND_ARGS
+            LINTER_COMMANDS_ARRAY_<LANGUAGE_NAME>+=("${<LANGUAGE_NAME>_COMMAND_ARGS}")
+          fi
+          ```
+
+  - Define the command to invoke the new linter:
+
+    - `lib/functions/linterCommands.sh`: add the command to invoke the linter.
+      Define a new variable: `LINTER_COMMANDS_ARRAY_<LANGUAGE_NAME>`.
+      Example:
+      `LINTER_COMMANDS_ARRAY_GO_MODULES=(golangci-lint run --allow-parallel-runners -c "${GO_LINTER_RULES}")`
+
+      If the linter needs to load a configuration file, add the relevant options
+      and paths to the command you just defined. The path to the configuration
+      file is automatically initialized by Super-linter using in the
+      `<LANGUAGE_NAME>_LINTER_RULES` variable, as in the `GO_LINTER_RULES`
+      example above for the `GO` language.
+
+    - `lib/globals/linterCommandsOptions.sh`: add "check only mode" and "fix
+      linting and formatting issues mode" options if the linter supports it.
+      Super-linter will automatically add them to the command to run the linter.
+
+      - If the linter runs in "fix linting and formatting issues mode" by
+        default, define a new variable with the options to add to the linter
+        command to enable "check only mode":
+        `<LANGUAGE_NAME>_CHECK_ONLY_MODE_OPTIONS=(....)`.
+        Example: `PYTHON_BLACK_CHECK_ONLY_MODE_OPTIONS=(--diff --check)`
+
+      - If the linter runs in "check only mode" by
+        default, define a new variable with the options to add to the linter
+        command to enable "fix linting and formatting issues mode":
+        `<LANGUAGE_NAME>_FIX_MODE_OPTIONS=(...)`.
+        Example: `ANSIBLE_FIX_MODE_OPTIONS=(--fix)`
+
+  - Provide the logic to populate the list of files or directories to examine: `lib/functions/buildFileList.sh`
+  - If necessary, provide elaborate logic to detect if the tool should examine a file or a directory: `lib/functions/detectFiles.sh`
+  - If the tool needs to take into account special cases, reach out to the
+    maintainers by creating a draft pull request and ask relevant questions
+    there. For example, you might need to provide new logic or customize
+    the existing one to:
+
+    - Validate the runtime environment: `lib/functions/validation.sh`.
+    - Get the installed version of the linter: `scripts/linterVersions.sh`
+    - Load configuration files: `lib/functions/linterRules.sh`
+    - Run the linter: `lib/functions/worker.sh`
+    - Compose the linter command: `lib/functions/linterCommands.sh`
+    - Modify the core Super-linter logic: `lib/linter.sh`

docs/run-linter-locally.md

@@ -139,3 +139,12 @@ To get the list of the available `Make` targets, run the following command:
 ```shell
 make help
 ```
+
+### Automatically fix formatting and linting issues
+
+To automatically fix linting and formatting issues when supported, run the
+following command:
+
+```shell
+make fix-codebase
+```

docs/upgrade-guide.md

@@ -39,17 +39,14 @@ This section helps you upgrade from super-linter `v6.7.0` to `v6.8.0`.
 
     - If you set `JAVASCRIPT_DEFAULT_STYLE=standard`, set
       `VALIDATE_JAVASCRIPT_PRETTIER=false`
-
     - If you set `TYPESCRIPT_DEFAULT_STYLE=standard`, set
       `VALIDATE_TYPESCRIPT_PRETTIER=false`
-
     - If you set `JAVASCRIPT_DEFAULT_STYLE=prettier`, set
       `VALIDATE_JAVASCRIPT_STANDARD=false`
-
     - If you set `TYPESCRIPT_DEFAULT_STYLE=prettier`, set
       `VALIDATE_TYPESCRIPT_STANDARD=false`
 
-  Finally, you remove both `JAVASCRIPT_DEFAULT_STYLE` and
+  Finally, remove both `JAVASCRIPT_DEFAULT_STYLE` and
   `TYPESCRIPT_DEFAULT_STYLE` from your Super-linter configuration.
 
 ## Upgrade from v5 to v6