The output of commands of Breeze are only generated when they change

.pre-commit-config.yaml

@@ -187,7 +187,7 @@ repos:
         exclude: ^airflow/_vendor/
       - id: trailing-whitespace
         name: Remove trailing whitespace at end of line
-        exclude: ^airflow/_vendor/
+        exclude: ^airflow/_vendor/|^images/breeze/output.*$
       - id: fix-encoding-pragma
         name: Remove encoding header from python files
         exclude: ^airflow/_vendor/
@@ -324,30 +324,30 @@ repos:
         files: ^setup\.cfg$|^setup\.py$
         pass_filenames: false
         entry: ./scripts/ci/pre_commit/pre_commit_check_order_setup.py
-        additional_dependencies: ['rich==12.2.0']
+        additional_dependencies: ['rich']
       - id: check-extra-packages-references
         name: Checks setup extra packages
         description: Checks if all the libraries in setup.py are listed in extra-packages-ref.rst file
         language: python
         files: ^setup\.py$|^docs/apache-airflow/extra-packages-ref\.rst$
         pass_filenames: false
         entry: ./scripts/ci/pre_commit/pre_commit_check_setup_extra_packages_ref.py
-        additional_dependencies: ['rich==12.3.0']
+        additional_dependencies: ['rich']
         # This check might be removed when min-airflow-version in providers is 2.2
       - id: check-airflow-2-1-compatibility
         name: Check that providers are 2.1 compatible.
         entry: ./scripts/ci/pre_commit/pre_commit_check_2_1_compatibility.py
         language: python
         pass_filenames: true
         files: ^airflow/providers/.*\.py$
-        additional_dependencies: ['rich==12.3.0']
+        additional_dependencies: ['rich']
       - id: update-breeze-file
         name: Update output of breeze commands in BREEZE.rst
         entry: ./scripts/ci/pre_commit/pre_commit_breeze_cmd_line.py
         language: python
         files: ^BREEZE\.rst$|^dev/breeze/.*$
         pass_filenames: false
-        additional_dependencies: ['rich==12.3.0']
+        additional_dependencies: ['rich', 'rich-click']
       - id: update-local-yml-file
         name: Update mounts in the local yml file
         entry: ./scripts/ci/pre_commit/pre_commit_local_yml_mounts.sh
@@ -379,7 +379,7 @@ repos:
         language: python
         files: ^Dockerfile$
         pass_filenames: false
-        additional_dependencies: ['rich==12.3.0']
+        additional_dependencies: ['rich']
       - id: update-supported-versions
         name: Updates supported versions in documentation
         entry: ./scripts/ci/pre_commit/pre_commit_supported_versions.py
@@ -582,15 +582,15 @@ repos:
           - 'jsonschema>=3.2.0,<5.0.0'
           - 'tabulate==0.8.8'
           - 'jsonpath-ng==1.5.3'
-          - 'rich==12.3.0'
+          - 'rich'
       - id: check-pre-commit-information-consistent
         name: Update information about pre-commit hooks and verify ids and names
         entry: ./scripts/ci/pre_commit/pre_commit_check_pre_commit_hooks.py
         args:
           - --max-length=70
         language: python
         files: ^\.pre-commit-config\.yaml$|^scripts/ci/pre_commit/pre_commit_check_pre_commit_hook_names\.py$
-        additional_dependencies: ['pyyaml', 'jinja2', 'black==22.3.0', 'tabulate', 'rich==12.3.0']
+        additional_dependencies: ['pyyaml', 'jinja2', 'black==22.3.0', 'tabulate', 'rich']
         require_serial: true
         pass_filenames: false
       - id: check-airflow-providers-have-extras
@@ -600,7 +600,7 @@ repos:
         files: ^setup\.py$|^airflow/providers/.*\.py$
         pass_filenames: false
         require_serial: true
-        additional_dependencies: ['rich==12.3.0']
+        additional_dependencies: ['rich']
       - id: update-breeze-readme-config-hash
         name: Update Breeze README.md with config files hash
         language: python
@@ -615,15 +615,15 @@ repos:
         files: ^dev/breeze/.*$
         pass_filenames: false
         require_serial: true
-        additional_dependencies: ['click', 'rich==12.3.0']
+        additional_dependencies: ['click', 'rich']
       - id: check-system-tests-present
         name: Check if system tests have required segments of code
         entry: ./scripts/ci/pre_commit/pre_commit_check_system_tests.py
         language: python
         files: ^tests/system/.*/example_[^/]*.py$
         exclude: ^tests/system/providers/google/bigquery/example_bigquery_queries\.py$
         pass_filenames: true
-        additional_dependencies: ['rich==12.3.0']
+        additional_dependencies: ['rich']
       - id: lint-markdown
         name: Run markdownlint
         description: Checks the style of Markdown files.
@@ -714,15 +714,15 @@ repos:
         language: python
         pass_filenames: true
         files: ^\.github/workflows/.*\.yml$
-        additional_dependencies: ['PyYAML', 'rich==12.3.0']
+        additional_dependencies: ['PyYAML', 'rich']
       - id: check-docstring-param-types
         name: Check that docstrings do not specify param types
         entry: ./scripts/ci/pre_commit/pre_commit_docstring_param_type.py
         language: python
         pass_filenames: true
         files: \.py$
         exclude: ^airflow/_vendor/
-        additional_dependencies: ['rich==12.3.0']
+        additional_dependencies: ['rich']
       - id: lint-chart-schema
         name: Lint chart/values.schema.json file
         entry: ./scripts/ci/pre_commit/pre_commit_chart_schema.py
@@ -757,59 +757,59 @@ repos:
         entry: ./scripts/ci/pre_commit/pre_commit_mypy.py
         files: ^dev/.*\.py$
         require_serial: true
-        additional_dependencies: ['rich==12.3.0']
+        additional_dependencies: ['rich']
       - id: run-mypy
         name: Run mypy for core
         language: python
         entry: ./scripts/ci/pre_commit/pre_commit_mypy.py --namespace-packages
         files: \.py$
         exclude: ^provider_packages|^docs|^airflow/_vendor/|^airflow/providers|^airflow/migrations|^dev
         require_serial: true
-        additional_dependencies: ['rich==12.3.0']
+        additional_dependencies: ['rich']
       - id: run-mypy
         name: Run mypy for providers
         language: python
         entry: ./scripts/ci/pre_commit/pre_commit_mypy.py --namespace-packages
         files: ^airflow/providers/.*\.py$
         require_serial: true
-        additional_dependencies: ['rich==12.3.0']
+        additional_dependencies: ['rich']
       - id: run-mypy
         name: Run mypy for /docs/ folder
         language: python
         entry: ./scripts/ci/pre_commit/pre_commit_mypy.py
         files: ^docs/.*\.py$
         exclude: ^docs/rtd-deprecation
         require_serial: true
-        additional_dependencies: ['rich==12.3.0']
+        additional_dependencies: ['rich']
       - id: run-flake8
         name: Run flake8
         language: python
         entry: ./scripts/ci/pre_commit/pre_commit_flake8.py
         files: \.py$
         pass_filenames: true
         exclude: ^airflow/_vendor/
-        additional_dependencies: ['rich==12.3.0']
+        additional_dependencies: ['rich']
       - id: lint-javascript
         name: ESLint against airflow/ui
         language: python
         'types_or': [javascript, tsx, ts]
         files: ^airflow/ui/
         entry: ./scripts/ci/pre_commit/pre_commit_ui_lint.py
         pass_filenames: false
-        additional_dependencies: ['rich==12.3.0']
+        additional_dependencies: ['rich']
       - id: lint-javascript
         name: ESLint against current UI JavaScript files
         language: python
         'types_or': [javascript]
         files: ^airflow/www/static/js/
         entry: ./scripts/ci/pre_commit/pre_commit_www_lint.py
         pass_filenames: false
-        additional_dependencies: ['rich==12.3.0']
+        additional_dependencies: ['rich']
       - id: update-migration-references
         name: Update migration ref doc
         language: python
         entry: ./scripts/ci/pre_commit/pre_commit_migration_reference.py
         pass_filenames: false
         files: ^airflow/migrations/versions/.*\.py$|^docs/apache-airflow/migrations-ref\.rst$
-        additional_dependencies: ['rich==12.3.0']
+        additional_dependencies: ['rich']
         ## ONLY ADD PRE-COMMITS HERE THAT REQUIRE CI IMAGE

BREEZE.rst

@@ -417,6 +417,15 @@ Those are all available flags of ``config`` command:
   :width: 100%
   :alt: Breeze config
 
+
+You can also dump hash of the configuration options used - this is mostly use to generate the dump
+of help of the commands only when they change.
+
+.. image:: ./images/breeze/output-command-hash-export.svg
+  :width: 100%
+  :alt: Breeze command-hash-export
+
+
 Starting complete Airflow installation
 ======================================
 
@@ -539,6 +548,8 @@ Configuration and maintenance
 * Freeing space needed to run CI tests with ``breeze free-space`` command
 * Fixing ownership of files in your repository with ``breeze fix-ownership`` command
 * Print Breeze version with ``breeze version`` command
+* Outputs hash of commands defined by ``breeze`` with ``command-hash-export`` (useful to avoid needless
+  regeneration of Breeze images)
 
 Release tasks
 -------------

dev/breeze/README.md

@@ -52,6 +52,6 @@ PLEASE DO NOT MODIFY THE HASH BELOW! IT IS AUTOMATICALLY UPDATED BY PRE-COMMIT.
 
 ---------------------------------------------------------------------------------------------------------
 
-Package config hash: dfdac0e241f273bbe3ae4a3d5e9c9f39426d792c2468753d94fd0dfa6b5d46331703c5d419e5e8bf3c199129acde63cfa61e30c6a3b9a920047e0d115fd1d57c
+Package config hash: 7f2ce934ce9d90dcc7f97c10534416a8736cb37c1bdf10c0b9abaae77fef35875863188897e13b68899caac65072697341bd11f7cc39148b0c4774ef11a88df2
 
 ---------------------------------------------------------------------------------------------------------

dev/breeze/setup.cfg

@@ -63,7 +63,7 @@ install_requires =
     pytest-xdist
     pyyaml
     requests
-    rich==12.3.0
+    rich
     rich_click
 
 [options.packages.find]

dev/breeze/src/airflow_breeze/commands/configuration_and_maintenance_commands.py

@@ -14,15 +14,17 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
-
+import hashlib
+import json
 import os
 import shutil
 import subprocess
 import sys
 from pathlib import Path
-from typing import Optional
+from typing import IO, Any, Dict, Optional
 
 import click
+from click import Context
 
 from airflow_breeze import NAME, VERSION
 from airflow_breeze.commands.main_command import main
@@ -71,6 +73,7 @@
         "resource-check",
         "free-space",
         "fix-ownership",
+        "command-hash-export",
         "version",
     ],
 }
@@ -407,6 +410,29 @@ def resource_check(verbose: bool, dry_run: bool):
     check_docker_resources(shell_params.airflow_image_name, verbose=verbose, dry_run=dry_run)
 
 
+def dict_hash(dictionary: Dict[str, Any]) -> str:
+    """MD5 hash of a dictionary. Sorted and dumped via json to account for random sequence)"""
+    dhash = hashlib.md5()
+    encoded = json.dumps(dictionary, sort_keys=True, default=vars).encode()
+    dhash.update(encoded)
+    return dhash.hexdigest()
+
+
+@main.command(
+    name="command-hash-export",
+    help="Outputs hash of all click commands to file or stdout if `-` "
+    "is used (useful to see if images should be regenerated).",
+)
+@option_verbose
+@click.argument('output', type=click.File('wt'))
+def command_hash_export(verbose: bool, output: IO):
+    with Context(main) as ctx:
+        the_context_dict = ctx.to_info_dict()
+        if verbose:
+            get_console().print(the_context_dict)
+        output.write(dict_hash(the_context_dict) + "\n")
+
+
 @main.command(name="fix-ownership", help="Fix ownership of source files to be same as host user.")
 @option_verbose
 @option_dry_run

dev/breeze/src/airflow_breeze/commands/release_management_commands.py

@@ -80,7 +80,7 @@
             "name": "Provider verification flags",
             "options": [
                 "--use-airflow-version",
-                "--constraints-reference",
+                "--airflow-constraints-reference",
                 "--airflow-extras",
                 "--use-packages-from-dist",
                 "--package-format",

dev/breeze/src/airflow_breeze/utils/path_utils.py

@@ -211,6 +211,9 @@ def find_airflow_sources_root_to_operate_on() -> Path:
     :return: Path for the found sources.
 
     """
+    sources_root_from_env = os.getenv('AIRFLOW_SOURCES_ROOT', None)
+    if sources_root_from_env:
+        return Path(sources_root_from_env)
     installation_airflow_sources = get_installation_airflow_sources()
     if installation_airflow_sources is None and not skip_upgrade_check():
         get_console().print(