👷‍♂️Include only functional releases into compatibility matrix (#802)

lord-haffi · web-flow · commit 9a6529aa7c57 · 2024-05-17T11:36:51.000+02:00
* 👷‍♂️Include only functional releases into compatibility matrix

* 🚨mypy

* Make it work again

* Fetch all for doc test

* Default case current checkout commit

* 🚨pylint
diff --git a/.github/workflows/docs_latest.yml b/.github/workflows/docs_latest.yml
@@ -34,7 +34,7 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v4
         with:
-          fetch-depth: 0 # otherwise, you will failed to push refs to dest repo
+          fetch-depth: 0 # otherwise, you will fail to push refs to dest repo
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
diff --git a/.github/workflows/test_docs.yml b/.github/workflows/test_docs.yml
@@ -10,6 +10,8 @@ jobs:
         os: [ubuntu-latest]
     steps:
       - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v5
         with:
diff --git a/docs/compatibility/__main__.py b/docs/compatibility/__main__.py
@@ -6,171 +6,21 @@
 if you're testing locally.
 """
 
-import itertools
 import logging
 import re
-import shutil
 from pathlib import Path
-from typing import Any as _Any
-from typing import Iterable
 
 import bost.operations
-from bost import main as bost_main
-from bost.operations import update_references as bost_update_references
-from bost.pull import OWNER, REPO, SchemaMetadata, get_source_repo
+from bost.pull import OWNER, REPO
 
-from . import change_schemas, diff, loader, matrix
+from . import diff, loader, matrix, versioning
 
-BO4E_BASE_DIR = Path(__file__).parents[2] / "tmp/bo4e_json_schemas"
-LOCAL_JSON_SCHEMA_DIR = Path(__file__).parents[2] / "json_schemas"
 logger = logging.getLogger(__name__)
 
-
-def pull_bo4e_version(version: str, output: Path, gh_token: str | None = None) -> None:
-    """
-    Pull the BO4E version from the given version string.
-    """
-    bost_main(
-        output=output,
-        target_version=version,
-        update_refs=True,
-        set_default_version=False,
-        clear_output=True,
-        token=gh_token,
-    )
-
-
-def update_references(path: Path, version: str) -> None:
-    """
-    Update the references in the given path. This step is needed for the local build.
-    """
-    schema_namespace = {}
-    for schema_path in loader.get_namespace(path):
-        local_path = Path(path, *schema_path).with_suffix(".json")
-        schema_namespace[schema_path[-1]] = SchemaMetadata(
-            class_name=schema_path[-1],
-            download_url="",
-            module_path=schema_path,
-            file_path=local_path,
-            cached_path=local_path,
-            token=None,
-        )
-    for schema_metadata in schema_namespace.values():
-        bost_update_references(schema_metadata, schema_namespace, version)
-        schema_metadata.save()
-
-
-def pull_or_reuse_bo4e_version(version: str, gh_token: str | None = None, from_local: bool = False) -> Path:
-    """
-    Pull the BO4E version from the given version string or reuse the version if it was already pulled before.
-    If version is None use the BO4E version of the checkout working directory by assuming the compiled json
-    schemas in /json_schemas.
-    Returns the path of the bo4e directory.
-    """
-    bo4e_dir = BO4E_BASE_DIR / version
-
-    if from_local:
-        if not any(LOCAL_JSON_SCHEMA_DIR.rglob("*.json")):
-            raise ValueError(
-                "No local json schemas found in /json_schemas. "
-                "Please ensure that the json schemas are build on beforehand."
-            )
-        if bo4e_dir.exists():
-            shutil.rmtree(bo4e_dir)
-        shutil.copytree(LOCAL_JSON_SCHEMA_DIR, bo4e_dir)
-        update_references(bo4e_dir, version)
-    elif any(bo4e_dir.rglob("*.json")):
-        return bo4e_dir
-    else:
-        pull_bo4e_version(version, bo4e_dir, gh_token)
-    return bo4e_dir
-
-
-def compare_bo4e_versions(
-    version_old: str, version_new: str, gh_token: str | None = None, from_local: bool = False
-) -> Iterable[change_schemas.Change]:
-    """
-    Compare the old version with the new version.
-    If version_new is None use the BO4E version of the checkout working directory by assuming the compiled json
-    schemas in /json_schemas.
-    """
-    dir_old_schemas = pull_or_reuse_bo4e_version(version_old, gh_token)
-    dir_new_schemas = pull_or_reuse_bo4e_version(version_new, gh_token, from_local=from_local)
-    print(f"Comparing {version_old} with {version_new}")
-    yield from diff.diff_schemas(dir_old_schemas, dir_new_schemas)
-
-
-def compare_bo4e_versions_iteratively(
-    versions: Iterable[str], cur_version: str | None = None, gh_token: str | None = None
-) -> dict[tuple[str, str], Iterable[change_schemas.Change]]:
-    """
-    Compare the versions iteratively. Each version at index i will be compared to the version at index i+1.
-    Additionally, if cur_version is provided, the last version in the list will be compared to the version
-    in the checkout working directory. The value of cur_version will be used to set the key in the returned
-    dict.
-    Note:
-        - versions must contain at least one element.
-        - versions should be sorted in ascending order.
-        - if using cur_version, ensure that the json schemas of the checkout working directory
-          were build on beforehand. They should be located in /json_schemas.
-    """
-    print(f"Comparing versions {versions} with cur_version {cur_version}")
-    changes = {}
-    last_version: str = ""  # This value is never used but makes mypy and pylint happy
-    for version_old, version_new in itertools.pairwise(versions):
-        last_version = version_new
-        changes[version_old, version_new] = compare_bo4e_versions(version_old, version_new, gh_token)
-    if cur_version is not None:
-        changes[last_version, cur_version] = compare_bo4e_versions(last_version, cur_version, gh_token, from_local=True)
-    print("Comparisons finished.")
-    return changes
-
-
 REGEX_RELEASE_VERSION = re.compile(r"^v(\d{6}\.\d+\.\d+)$")
 REGEX_RELEASE_CANDIDATE_VERSION = re.compile(r"^v(\d{6}\.\d+\.\d+)-rc\d+$")
 
 
-def get_last_n_release_versions(n: int, include_rc: bool = False, gh_token: str | None = None) -> Iterable[str]:
-    """
-    Get the last n release versions from the BO4E repository.
-    """
-    repo = get_source_repo(gh_token)
-    releases = repo.get_releases()
-    counter = 0
-
-    for release in releases:
-        if not REGEX_RELEASE_VERSION.fullmatch(release.tag_name) and (
-            not include_rc or not REGEX_RELEASE_CANDIDATE_VERSION.fullmatch(release.tag_name)
-        ):
-            continue
-        counter += 1
-        yield release.tag_name
-        if counter >= n:
-            return
-
-    logger.warning("Only %d matching releases found. Returning all releases.", counter)
-
-
-def get_all_release_versions_since_20240100(include_rc: bool = False, gh_token: str | None = None) -> Iterable[str]:
-    """
-    Get all release versions since v202401.0.0 from the BO4E repository.
-    """
-    repo = get_source_repo(gh_token)
-    releases = repo.get_releases()
-    version_threshold = "v202401.0.0"
-
-    for release in releases:
-        if not REGEX_RELEASE_VERSION.fullmatch(release.tag_name) and (
-            not include_rc or not REGEX_RELEASE_CANDIDATE_VERSION.fullmatch(release.tag_name)
-        ):
-            continue
-        yield release.tag_name
-        if release.tag_name == version_threshold:
-            return
-
-    logger.warning("Threshold version %s not found. Returned all matching releases.", version_threshold)
-
-
 def _monkey_patch_bost_regex_if_local_testing(version: str) -> None:
     regex_expected_version = re.compile(r"^v\d+\.\d+\.\d+(?:-rc\d+)?$")
     if not regex_expected_version.fullmatch(version):
@@ -196,20 +46,20 @@ def create_tables_for_doc(
     The compatibility matrix will be built for last_n_versions + the current version in the checkout working directory.
     If you set last_n_versions = 0 all versions since v202401.0.0 will be compared.
     Note: The matrix will never contain the first version as column. Each column is a comparison to the version before.
-    Note: Release candidates are excluded.
+    Note: Only functional releases will be compared since technical releases are enforced to be fully compatible.
+    See https://github.com/bo4e/BO4E-python/issues/784
     """
     _monkey_patch_bost_regex_if_local_testing(gh_version)
     logger.info("Retrieving the last %d release versions", last_n_versions)
-    if last_n_versions > 0:
-        versions = list(reversed(list(get_last_n_release_versions(last_n_versions, gh_token=gh_token))))
-    else:
-        versions = list(reversed(list(get_all_release_versions_since_20240100(gh_token=gh_token))))
+    versions = list(
+        reversed(list(versioning.get_last_n_tags(last_n_versions, ref=gh_version, exclude_technical_bumps=True)))
+    )
     logger.info("Comparing versions iteratively: %s", " -> ".join([*versions, gh_version]))
-    changes_iterables = compare_bo4e_versions_iteratively(versions, gh_version, gh_token=gh_token)
+    changes_iterables = diff.compare_bo4e_versions_iteratively(versions, gh_version, gh_token=gh_token)
     logger.info("Building namespaces")
     changes = {key: list(value) for key, value in changes_iterables.items()}
-    namespaces = {version: list(loader.get_namespace(BO4E_BASE_DIR / version)) for version in versions}
-    namespaces[gh_version] = list(loader.get_namespace(BO4E_BASE_DIR / gh_version))
+    namespaces = {version: list(loader.get_namespace(loader.BO4E_BASE_DIR / version)) for version in versions}
+    namespaces[gh_version] = list(loader.get_namespace(loader.BO4E_BASE_DIR / gh_version))
     logger.info("Creating compatibility matrix")
     matrix.create_compatibility_matrix_csv(
         compatibility_matrix_output_file, [*versions, gh_version], namespaces, changes
@@ -224,5 +74,5 @@ def test_create_tables_for_doc() -> None:
     create_tables_for_doc(
         Path(__file__).parents[1] / "compatibility_matrix.csv",
         "local",
-        last_n_versions=3,
+        last_n_versions=0,
     )
diff --git a/docs/compatibility/diff.py b/docs/compatibility/diff.py
@@ -2,10 +2,10 @@
 Contains the logic to detect the different changes between two BO4E versions.
 """
 
+import itertools
 import re
 from pathlib import Path
-from typing import Any as _Any
-from typing import Iterable
+from typing import Iterable, Sequence
 
 from bost.schema import AllOf, AnyOf, Array, Object, Reference, SchemaRootType, SchemaType, StrEnum, String, TypeBase
 
@@ -320,3 +320,46 @@ def diff_schemas(schemas_old: Path, schemas_new: Path) -> Iterable[change_schema
             f"{'/'.join(schema_file.with_suffix('').parts)}#",
             f"{'/'.join(schema_file.with_suffix('').parts)}#",
         )
+
+
+def compare_bo4e_versions(
+    version_old: str, version_new: str, gh_token: str | None = None, from_local: bool = False
+) -> Iterable[change_schemas.Change]:
+    """
+    Compare the old version with the new version.
+    If version_new is None use the BO4E version of the checkout working directory by assuming the compiled json
+    schemas in /json_schemas.
+    """
+    dir_old_schemas = loader.pull_or_reuse_bo4e_version(version_old, gh_token)
+    dir_new_schemas = loader.pull_or_reuse_bo4e_version(version_new, gh_token, from_local=from_local)
+    print(f"Comparing {version_old} with {version_new}")
+    yield from diff_schemas(dir_old_schemas, dir_new_schemas)
+
+
+def compare_bo4e_versions_iteratively(
+    versions: Sequence[str], cur_version: str | None = None, gh_token: str | None = None
+) -> dict[tuple[str, str], Iterable[change_schemas.Change]]:
+    """
+    Compare the versions iteratively. Each version at index i will be compared to the version at index i+1.
+    Additionally, if cur_version is provided, the last version in the list will be compared to the version
+    in the checkout working directory. The value of cur_version will be used to set the key in the returned
+    dict.
+    Note:
+        - versions must contain at least one element.
+        - versions should be sorted in ascending order.
+        - if using cur_version, ensure that the json schemas of the checkout working directory
+          were build on beforehand. They should be located in /json_schemas.
+    """
+    print(f"Comparing versions {versions} with cur_version {cur_version}")
+    if len(versions) == 0:
+        print("No versions to compare.")
+        return {}
+    changes = {}
+    last_version: str = versions[0]  # This value is never used but makes mypy and pylint happy
+    for version_old, version_new in itertools.pairwise(versions):
+        last_version = version_new
+        changes[version_old, version_new] = compare_bo4e_versions(version_old, version_new, gh_token)
+    if cur_version is not None:
+        changes[last_version, cur_version] = compare_bo4e_versions(last_version, cur_version, gh_token, from_local=True)
+    print("Comparisons finished.")
+    return changes
diff --git a/docs/compatibility/loader.py b/docs/compatibility/loader.py
@@ -3,14 +3,21 @@
 """
 
 import json
+import shutil
 from pathlib import Path
 from typing import Iterable
 
+from bost import main as bost_main
+from bost.operations import update_references as bost_update_references
+from bost.pull import SchemaMetadata
 from bost.schema import SchemaRootType
 from pydantic import TypeAdapter
 
 from . import change_schemas
 
+BO4E_BASE_DIR = Path(__file__).parents[2] / "tmp/bo4e_json_schemas"
+LOCAL_JSON_SCHEMA_DIR = Path(__file__).parents[2] / "json_schemas"
+
 
 def load_schema_file(path: Path) -> SchemaRootType:
     """
@@ -45,3 +52,63 @@ def get_namespace(path: Path) -> Iterable[tuple[str, ...]]:
     for schema_file in path.rglob("*.json"):
         sub_path = schema_file.relative_to(path).parts[:-1]
         yield *sub_path, schema_file.stem
+
+
+def pull_bo4e_version(version: str, output: Path, gh_token: str | None = None) -> None:
+    """
+    Pull the BO4E version from the given version string.
+    """
+    bost_main(
+        output=output,
+        target_version=version,
+        update_refs=True,
+        set_default_version=False,
+        clear_output=True,
+        token=gh_token,
+    )
+
+
+def update_references(path: Path, version: str) -> None:
+    """
+    Update the references in the given path. This step is needed for the local build.
+    """
+    schema_namespace = {}
+    for schema_path in get_namespace(path):
+        local_path = Path(path, *schema_path).with_suffix(".json")
+        schema_namespace[schema_path[-1]] = SchemaMetadata(
+            class_name=schema_path[-1],
+            download_url="",
+            module_path=schema_path,
+            file_path=local_path,
+            cached_path=local_path,
+            token=None,
+        )
+    for schema_metadata in schema_namespace.values():
+        bost_update_references(schema_metadata, schema_namespace, version)
+        schema_metadata.save()
+
+
+def pull_or_reuse_bo4e_version(version: str, gh_token: str | None = None, from_local: bool = False) -> Path:
+    """
+    Pull the BO4E version from the given version string or reuse the version if it was already pulled before.
+    If version is None use the BO4E version of the checkout working directory by assuming the compiled json
+    schemas in /json_schemas.
+    Returns the path of the bo4e directory.
+    """
+    bo4e_dir = BO4E_BASE_DIR / version
+
+    if from_local:
+        if not any(LOCAL_JSON_SCHEMA_DIR.rglob("*.json")):
+            raise ValueError(
+                "No local json schemas found in /json_schemas. "
+                "Please ensure that the json schemas are build on beforehand."
+            )
+        if bo4e_dir.exists():
+            shutil.rmtree(bo4e_dir)
+        shutil.copytree(LOCAL_JSON_SCHEMA_DIR, bo4e_dir)
+        update_references(bo4e_dir, version)
+    elif any(bo4e_dir.rglob("*.json")):
+        return bo4e_dir
+    else:
+        pull_bo4e_version(version, bo4e_dir, gh_token)
+    return bo4e_dir
diff --git a/docs/compatibility/versioning.py b/docs/compatibility/versioning.py
