Format fix tests (#2294)

* Add csharpier test files * Allow csharpier to fix files * Add format_fix test code * Revert files * Skip test for linters that does not format and cannot apply fixes * Add input test files * Add input test files * Add config value * Add input files * Add input files * Remove copy * Fix linter activation * [MegaLinter] Apply linters fixes * Add input files * Add input files * Fix xmllint autofix * xmllint format fix test * Add input files * Fix powershell formatter autofix * Add input files * Try to fix git error * Only use GitPython if repo exists * Remove unused import * [MegaLinter] Apply linters fixes * Add input files * Add input files * Fix git error * Add input files * Fix linter error * [MegaLinter] Apply linters fixes * Add input files * Add input files * [MegaLinter] Apply linters fixes * Fix npm-groovy-lint behaviour * Add missing linters to workflows * Add input files * [MegaLinter] Apply linters fixes * Generate all linters Dockerfile * Fixes * Add input files * Fix file filter logic * Remove unused imports * [MegaLinter] Apply linters fixes * Fix linter test generation * Fix bicep * Try to fix spell linter tests * [MegaLinter] Apply linters fixes * [MegaLinter] Apply linters fixes * Try to fix spell linter tests * Remove commented code * Minor changes * Update CHANGELOG.md * Update the CONTRIBUTING.md file Add the explanation of how to run the linter tests inside the container * Run build.sh --doc * [MegaLinter] Apply linters fixes * Try to fix test * [MegaLinter] Apply linters fixes * Decouple pre_test from core and move to linters * Fix linter errors * [MegaLinter] Apply linters fixes * Update Dockerfiles * Rename input test files * Fix tests --------- Co-authored-by: bdovaz <bdovaz@users.noreply.github.com>
oxsecurity · Feb 14, 2023 · 3c8b606 · 3c8b606
1 parent daf63d6
commit 3c8b606
Show file tree

Hide file tree

Showing 225 changed files with 12,438 additions and 4,805 deletions.
diff --git a/.automation/build.py b/.automation/build.py
@@ -7,6 +7,7 @@
 import logging
 import os
 import re
+import shutil
 import subprocess
 import sys
 from datetime import date, datetime
@@ -554,6 +555,9 @@ def match_flavor(item, flavor, flavor_info):
 
 # Automatically generate Dockerfile for standalone linters
 def generate_linter_dockerfiles():
+    # Remove all the contents of LINTERS_DIR beforehand so that the result is deterministic
+    shutil.rmtree(os.path.realpath(LINTERS_DIR))
+    os.makedirs(os.path.realpath(LINTERS_DIR))
     # Browse descriptors
     linters_md = "# Standalone linter docker images\n\n"
     linters_md += "| Linter key | Docker image | Size |\n"
@@ -571,9 +575,6 @@ def generate_linter_dockerfiles():
         )
         # Browse descriptor linters
         for linter in descriptor_linters:
-            # Do not build standalone linter if it does not manage SARIF
-            if linter.can_output_sarif is False:
-                continue
             # Unique linter dockerfile
             linter_lower_name = linter.name.lower()
             dockerfile = f"{LINTERS_DIR}/{linter_lower_name}/Dockerfile"
@@ -658,10 +659,21 @@ def generate_linter_dockerfiles():
 # Automatically generate a test class for each linter class
 # This could be done dynamically at runtime, but having a physical class is easier for developers in IDEs
 def generate_linter_test_classes():
+    test_linters_root = f"{REPO_HOME}/megalinter/tests/test_megalinter/linters"
+
+    # Remove all the contents of test_linters_root beforehand so that the result is deterministic
+    shutil.rmtree(os.path.realpath(test_linters_root))
+    os.makedirs(os.path.realpath(test_linters_root))
+
     linters = megalinter.linter_factory.list_all_linters()
     for linter in linters:
-        lang_lower = linter.descriptor_id.lower()
-        linter_name_lower = linter.linter_name.lower().replace("-", "_")
+        if linter.name is not None:
+            linter_name = linter.name
+        else:
+            lang_lower = linter.descriptor_id.lower()
+            linter_name = f"{lang_lower}_{linter.linter_name}"
+
+        linter_name_lower = linter_name.lower().replace("-", "_")
         test_class_code = f"""# !/usr/bin/env python3
 \"\"\"
 Unit tests for {linter.descriptor_id} linter {linter.linter_name}
@@ -673,14 +685,11 @@ def generate_linter_test_classes():
 from megalinter.tests.test_megalinter.LinterTestRoot import LinterTestRoot
 
 
-class {lang_lower}_{linter_name_lower}_test(TestCase, LinterTestRoot):
+class {linter_name_lower}_test(TestCase, LinterTestRoot):
     descriptor_id = "{linter.descriptor_id}"
     linter_name = "{linter.linter_name}"
 """
-        test_class_file_name = (
-            f"{REPO_HOME}/megalinter/tests/test_megalinter/"
-            + f"linters/{lang_lower}_{linter_name_lower}_test.py"
-        )
+        test_class_file_name = f"{test_linters_root}/{linter_name_lower}_test.py"
         if not os.path.isfile(test_class_file_name):
             file = open(
                 test_class_file_name,
@@ -3021,6 +3030,40 @@ def update_dependents_info():
     os.system(" ".join(command))
 
 
+def update_workflows_linters():
+    descriptors, _ = list_descriptors_for_build()
+
+    linters = ""
+
+    for descriptor in descriptors:
+        for linter in descriptor["linters"]:
+            if "name" in linter:
+                name = linter["name"].lower()
+            else:
+                lang_lower = descriptor["descriptor_id"].lower()
+                linter_name_lower = linter["linter_name"].lower().replace("-", "_")
+                name = f"{lang_lower}_{linter_name_lower}"
+
+            linters += f'            "{name}",\n'
+
+    update_workflow_linters(".github/workflows/deploy-DEV-linters.yml", linters)
+    update_workflow_linters(".github/workflows/deploy-BETA-linters.yml", linters)
+    update_workflow_linters(".github/workflows/deploy-RELEASE-linters.yml", linters)
+
+
+def update_workflow_linters(file_path, linters):
+    with open(file_path, "r", encoding="utf-8") as f:
+        file_content = f.read()
+        file_content = re.sub(
+            r"(linter:\s+\[\s*)([^\[\]]*?)(\s*\])",
+            rf"\1{re.escape(linters).replace(chr(92),'').strip()}\3",
+            file_content,
+        )
+
+    with open(file_path, "w") as f:
+        f.write(file_content)
+
+
 if __name__ == "__main__":
     try:
         logging.basicConfig(
@@ -3045,6 +3088,7 @@ def update_dependents_info():
     generate_all_flavors()
     generate_linter_dockerfiles()
     generate_linter_test_classes()
+    update_workflows_linters()
     if UPDATE_DOC is True:
         logging.info("Running documentation generators...")
         # refresh_users_info() # deprecated since now we use github-dependents-info