Add --verbose flag to reduce logging to warning+ level, Add Progress bars (rustfoundation#67)

* Add --verbose flag to reduce logging to warning+ level, Add spinner

* Remove spinner, Introduce tqdm, wrapping important loops in tqdm

* Add tqdm to pyproject.toml

* refactor: Add common.py to hold logger and tqdm wrapper, Disable tqdm progress on verbose mode, Remove Unused imports

* Remove extra return

* Apply suggestions from code review

Co-authored-by: Pete LeVasseur <plevasseur@gmail.com>

* Fix logger indentation

* merge --verbose into --debug, remove time.sleep()

---------

Co-authored-by: Pete LeVasseur <plevasseur@gmail.com>

Author: 2 people

builder/build_cli.py

@@ -28,7 +28,7 @@ def build_docs(
     serve: bool,
     debug: bool,
     offline: bool,
-    spec_lock_consistency_check: bool
+    spec_lock_consistency_check: bool,
 ) -> Path:
     """
     Builds the Sphinx documentation with the specified options.
@@ -70,10 +70,13 @@ def build_docs(
         conf_opt_values.append("enable_spec_lock_consistency=0")
     if offline:  
         conf_opt_values.append("offline=1")
+    if debug:  
+        conf_opt_values.append("debug=1")
+
     # Only add the --define argument if there are options to define
     if conf_opt_values:
-        args.append("--define")
         for opt in conf_opt_values:
+            args.append("--define") # each option needs its own --define
             args.append(opt)
 
     if serve:
@@ -172,6 +175,12 @@ def main(root):
     group.add_argument(
         "--xml", help="Generate Sphinx XML rather than HTML", action="store_true"
     )
+    group.add_argument(
+        "-v",
+        "--verbose",
+        help="Debug mode for the extensions, showing exceptions",
+        action="store_true",
+    )
     group.add_argument(
         "--debug",
         help="Debug mode for the extensions, showing exceptions",
@@ -183,6 +192,6 @@ def main(root):
         update_spec_lockfile(SPEC_CHECKSUM_URL, root / "src" / SPEC_LOCKFILE)
 
     rendered = build_docs(
-        root, "xml" if args.xml else "html", args.clear, args.serve, args.debug, args.offline, not args.ignore_spec_lock_diff
+        root, "xml" if args.xml else "html", args.clear, args.serve, args.debug, args.offline, not args.ignore_spec_lock_diff, 
     )
 

exts/coding_guidelines/__init__.py

@@ -1,21 +1,15 @@
 # SPDX-License-Identifier: MIT OR Apache-2.0
 # SPDX-FileCopyrightText: The Coding Guidelines Subcommittee Contributors
-
+  
 from . import fls_checks
 from . import write_guidelines_ids
 from . import std_role
 from . import fls_linking
 from . import guidelines_checks 
 
-from sphinx_needs.api import add_dynamic_function
-from sphinx.errors import SphinxError
+from .common import logger, get_tqdm, bar_format, logging 
 from sphinx.domains import Domain
 
-import logging
-
-# Get the Sphinx logger
-logger = logging.getLogger('sphinx')
-
 class CodingGuidelinesDomain(Domain):
     name = "coding-guidelines"
     label = "Rust Standard Library"
@@ -32,6 +26,19 @@ def get_objects(self):
     def merge_domaindata(self, docnames, other):
         pass  # No domain data to merge
 
+
+def on_build_finished(app, exception):
+    print("\nFinalizing build:")
+    for _ in get_tqdm(iterable=range(1), desc="Finalizing",bar_format=bar_format):
+        pass
+
+    outdir = app.outdir
+    if exception is not None:
+        print(" - Build failed")
+    else:
+        if not app.config.debug:
+            print(f" + Build complete -> {outdir}")
+
 def setup(app):
 
     app.add_domain(CodingGuidelinesDomain)
@@ -46,6 +53,10 @@ def setup(app):
         rebuild="env",  # Rebuild the environment when this changes
         types=[str],
     )
+    app.add_config_value(name='debug', 
+                         default=False, 
+                         rebuild='env'
+    )
     app.add_config_value(name='fls_paragraph_ids_url', 
                          default='https://rust-lang.github.io/fls/paragraph-ids.json', 
                          rebuild='env')
@@ -58,15 +69,16 @@ def setup(app):
         rebuild='env',
         types=[list],
     )
-
+    if app.config.debug:
+        logger.setLevel(logging.INFO)
+        common.disable_tqdm = True  
+    
     app.connect('env-check-consistency', guidelines_checks.validate_required_fields)
-
     app.connect('env-check-consistency', fls_checks.check_fls)
-
     app.connect('build-finished', write_guidelines_ids.build_finished)
-
     app.connect('build-finished', fls_linking.build_finished)
-
+    app.connect('build-finished', on_build_finished)
+    
     return {
         'version': '0.1',
         'parallel_read_safe': True,

exts/coding_guidelines/common.py

@@ -0,0 +1,16 @@
+
+from tqdm import tqdm
+import logging
+
+# This is a wrapper around tqdm that allows us to disable it with this global variable
+disable_tqdm = False 
+def get_tqdm(**kwargs):
+    kwargs['disable'] = disable_tqdm
+    return tqdm(**kwargs)
+# Get the Sphinx logger
+logger = logging.getLogger('sphinx')
+logger.setLevel(logging.WARNING)  
+
+# This is what controls the progress bar format
+bar_format = "{l_bar}{bar}| {n_fmt}/{total_fmt} {postfix}"
+

exts/coding_guidelines/fls_checks.py

@@ -1,16 +1,15 @@
 # SPDX-License-Identifier: MIT OR Apache-2.0
 # SPDX-FileCopyrightText: The Coding Guidelines Subcommittee Contributors
 
+
+from .common import logger, get_tqdm, bar_format, logging
+import time 
 import requests
-import logging
 import re
 import json
-from pathlib import Path
 from sphinx.errors import SphinxError
 from sphinx_needs.data import SphinxNeedsData
 
-# Get the Sphinx logger
-logger = logging.getLogger('sphinx')
 fls_paragraph_ids_url = "https://rust-lang.github.io/fls/paragraph-ids.json"
 
 class FLSValidationError(SphinxError):
@@ -88,9 +87,9 @@ def check_fls_exists_and_valid_format(app, env):
     # Regular expression for FLS ID validation
     # Format: fls_<12 alphanumeric chars including upper and lowercase>
     fls_pattern = re.compile(r'^fls_[a-zA-Z0-9]{9,12}$')
+
     for need_id, need in needs.items():
         logger.debug(f"ID: {need_id}, Need: {need}")
-
         if need.get('type') == 'guideline':
             fls_value = need.get("fls")
 
@@ -128,9 +127,15 @@ def check_fls_ids_correct(app, env, fls_ids):
     # Track any errors found
     invalid_ids = []
 
+    # prefiltering: this is mainly done for tqdm progress
+    guidelines = {k: v for k, v in needs.items() if v.get('type') == 'guideline'}
+    
+    pbar = get_tqdm(iterable=guidelines.items(), desc="Validating FLS IDs",bar_format=bar_format, unit="need")
+
     # Check each guideline's FLS reference
-    for need_id, need in needs.items():
+    for need_id, need in pbar:
         if need.get('type') == 'guideline':
+            pbar.set_postfix(fls_id=need_id)
             fls_value = need.get("fls")
 
             # Skip needs we already validated format for
@@ -141,17 +146,19 @@ def check_fls_ids_correct(app, env, fls_ids):
             if fls_value not in fls_ids:
                 invalid_ids.append((need_id, fls_value))
                 logger.warning(f"Need {need_id} references non-existent FLS ID: '{fls_value}'")
-    
-    # Raise error if any invalid IDs were found
-    if invalid_ids:
-        error_message = "The following needs reference non-existent FLS IDs:\n"
-        for need_id, fls_id in invalid_ids:
-            error_message += f"  - Need {need_id} references '{fls_id}'\n"
-        logger.error(error_message)
-        raise FLSValidationError(error_message)
-    
+        
+        # Raise error if any invalid IDs were found
+        if invalid_ids:
+            error_message = "The following needs reference non-existent FLS IDs:\n"
+            for need_id, fls_id in invalid_ids:
+                error_message += f"  - Need {need_id} references '{fls_id}'\n"
+            logger.error(error_message)
+            raise FLSValidationError(error_message)
+        
     logger.info("All FLS references in guidelines are valid")
 
+    pbar.close()  # Ensure cleanup
+
 
 def gather_fls_paragraph_ids(app, json_url):
     """
@@ -308,8 +315,14 @@ def check_fls_lock_consistency(app, env, fls_raw_data):
 
     # Map of FLS IDs to guidelines that reference them
     fls_to_guidelines = {}
-    for need_id, need in needs.items():
+
+    # prefiltering: this is mainly done for tqdm progress
+    guidelines = {k: v for k, v in needs.items() if v.get('type') == 'guideline'}
+    pbar = get_tqdm(iterable=guidelines.items(), desc="Checking fls lock consistency", bar_format=bar_format, unit="need")
+
+    for need_id, need in pbar:
         if need.get('type') == 'guideline':
+            pbar.set_postfix(fls_id=need_id)
             fls_value = need.get("fls")
             if fls_value:
                 if fls_value not in fls_to_guidelines:

exts/coding_guidelines/fls_linking.py

@@ -1,10 +1,8 @@
 import re
 import os
-import logging
 import sphinx
+from .common import logger
 
-# Get the Sphinx logger
-logger = logging.getLogger('sphinx')
 
 def build_finished(app, exception):
     """Hook to run at the end of the build process."""

exts/coding_guidelines/guidelines_checks.py

@@ -3,9 +3,8 @@
 
 from sphinx.errors import SphinxError
 from sphinx_needs.data import SphinxNeedsData
-import logging
+from .common import logger, get_tqdm, bar_format 
 
-logger = logging.getLogger('sphinx')
 
 class IntegrityCheckError(SphinxError):
     category = "Integrity Check Error"
@@ -20,20 +19,25 @@ def validate_required_fields(app, env):
 
     required_fields = app.config.required_guideline_fields  # Access the configured values
 
-    for key, value in needs.items():
+    # prefiltering: this is mainly done for tqdm progress
+    guidelines = {k: v for k, v in needs.items() if v.get('type') == 'guideline'}
+    pbar = get_tqdm(iterable=guidelines.items(), desc="Checking for required fields", bar_format=bar_format, unit="need")
+
+    for key, value in pbar:
         if value.get('type') == 'guideline':
             missing_fields = []
             for field in required_fields:
+                pbar.set_postfix(field=field if field is not None else "Missing")
                 if value.get(field) in  (None, '', []):
                     missing_fields.append(field)
 
+
             if missing_fields:
                 error_message = (
                     f"Guideline '{value.get('title')}' (ID: {value.get('id')}) "
                     f"in {value.get('docname')}:{value.get('lineno')} is missing the following required fields: "
                     f"{', '.join(missing_fields)}"
                 )
                 logger.error(error_message)
-                app.builder.statuscode = 1
+                app.builder.statuscode = 1 # mark the build as failed (0 means success)
                 raise IntegrityCheckError(error_message) 
-            logger.info("No missing required field")

exts/coding_guidelines/write_guidelines_ids.py

@@ -3,14 +3,12 @@
 """
 import hashlib
 import json
-import logging
 import os
 from collections import defaultdict
 import sphinx
 from sphinx_needs.data import SphinxNeedsData
+from .common import logger
 
-# Get the Sphinx logger
-logger = logging.getLogger('sphinx')
 
 def calculate_checksum(content, options):
     """Calculate a SHA-256 checksum of content and options"""

pyproject.toml

@@ -6,6 +6,7 @@ readme = "README.md"
 requires-python = ">=3.12"
 dependencies = [
     "builder",
+    "tqdm",
     "sphinx>=8.2.3",
     "sphinx-autobuild>=2024.10.3",
     "sphinx-needs>=5.1.0",