[v3-0-test] Fix redirects repository urls, use head repo when running PR workflows (#54301)

.github/workflows/ci-amd.yml

@@ -124,7 +124,8 @@ jobs:
         ${{ steps.selective-checks.outputs.selected-providers-list-as-string }}
       skip-pre-commits: ${{ steps.selective-checks.outputs.skip-pre-commits }}
       skip-providers-tests: ${{ steps.selective-checks.outputs.skip-providers-tests }}
-      source-head-repo: ${{ steps.source-run-info.outputs.source-head-repo }}
+      source-head-repo: ${{ steps.source-run-info.outputs.head-repo }}
+      source-head-ref: ${{ steps.source-run-info.outputs.head-ref }}
       sqlite-exclude: ${{ steps.selective-checks.outputs.sqlite-exclude }}
       testable-core-integrations: ${{ steps.selective-checks.outputs.testable-core-integrations }}
       testable-providers-integrations: ${{ steps.selective-checks.outputs.testable-providers-integrations }}
@@ -309,6 +310,8 @@ jobs:
       default-postgres-version: ${{ needs.build-info.outputs.default-postgres-version }}
       run-coverage: ${{ needs.build-info.outputs.run-coverage }}
       use-uv: ${{ needs.build-info.outputs.use-uv }}
+      source-head-repo: ${{ needs.build-info.outputs.source-head-repo }}
+      source-head-ref: ${{ needs.build-info.outputs.source-head-ref }}
     secrets:
       DOCS_AWS_ACCESS_KEY_ID: ${{ secrets.DOCS_AWS_ACCESS_KEY_ID }}
       DOCS_AWS_SECRET_ACCESS_KEY: ${{ secrets.DOCS_AWS_SECRET_ACCESS_KEY }}

.github/workflows/ci-image-checks.yml

@@ -104,6 +104,14 @@ on:  # yamllint disable-line rule:truthy
         description: "Whether to use uv to build the image (true/false)"
         required: true
         type: string
+      source-head-repo:
+        description: "The source head repository to use for back-references"
+        default: "apache/airflow"
+        type: string
+      source-head-ref:
+        description: "The source head ref to use for back-references"
+        default: "main"
+        type: string
     secrets:
       DOCS_AWS_ACCESS_KEY_ID:
         required: true
@@ -317,6 +325,8 @@ jobs:
       INCLUDE_SUCCESS_OUTPUTS: "${{ inputs.include-success-outputs }}"
       PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}"
       VERBOSE: "true"
+      HEAD_REPO: "${{ inputs.source-head-repo }}"
+      HEAD_REF: "${{ inputs.source-head-ref }}"
     steps:
       - name: "Cleanup repo"
         shell: bash

dev/breeze/doc/images/output_release-management_add-back-references.txt

@@ -1 +1 @@
-e9a2d362c309bde38992498e732bccfb
+046f2fbe2d8e77a2f4a8be3f6a27497e

dev/breeze/src/airflow_breeze/commands/ci_commands.py

@@ -285,6 +285,7 @@ class WorkflowInfo(NamedTuple):
     ref: str | None
     ref_name: str | None
     pr_number: int | None
+    head_ref: str | None = None
 
     def get_all_ga_outputs(self) -> Iterable[str]:
         from airflow_breeze.utils.github import get_ga_output
@@ -297,6 +298,7 @@ def get_all_ga_outputs(self) -> Iterable[str]:
         yield get_ga_output(name="runs-on", value=self.get_runs_on())
         yield get_ga_output(name="canary-run", value=self.is_canary_run())
         yield get_ga_output(name="run-coverage", value=self.run_coverage())
+        yield get_ga_output(name="head-ref", value=self.head_ref)
 
     def print_all_ga_outputs(self):
         for output in self.get_all_ga_outputs():
@@ -348,6 +350,7 @@ def workflow_info(context: str) -> WorkflowInfo:
     pr_number: int | None = None
     ref_name = ctx.get("ref_name")
     ref = ctx.get("ref")
+    head_ref = ctx.get("head_ref")
     if event_name == GithubEvents.PULL_REQUEST.value:
         event = ctx.get("event")
         if event:
@@ -387,6 +390,7 @@ def workflow_info(context: str) -> WorkflowInfo:
         pr_number=pr_number,
         ref=ref,
         ref_name=ref_name,
+        head_ref=head_ref,
     )
 
 

dev/breeze/src/airflow_breeze/commands/release_management_commands.py

@@ -44,6 +44,7 @@
 from rich.progress import Progress
 from rich.syntax import Syntax
 
+from airflow_breeze.branch_defaults import AIRFLOW_BRANCH
 from airflow_breeze.commands.ci_image_commands import rebuild_or_pull_ci_image_if_needed
 from airflow_breeze.commands.common_options import (
     argument_doc_packages,
@@ -1884,12 +1885,35 @@ def publish_docs(
 @argument_doc_packages
 @option_dry_run
 @option_verbose
+@click.option(
+    "--head-ref",
+    help="The branch of redirect files to use.",
+    default=AIRFLOW_BRANCH,
+    envvar="HEAD_REF",
+    show_default=True,
+)
+@click.option(
+    "--head-repo",
+    help="The repository of redirect files to use.",
+    default=APACHE_AIRFLOW_GITHUB_REPOSITORY,
+    envvar="HEAD_REPO",
+    show_default=True,
+)
 def add_back_references(
     airflow_site_directory: str,
     include_not_ready_providers: bool,
     include_removed_providers: bool,
     doc_packages: tuple[str, ...],
+    head_ref: str,
+    head_repo: str,
 ):
+    # head_ref and head_repo could be empty in canary runs
+
+    if head_ref == "":
+        head_ref = AIRFLOW_BRANCH
+    if head_repo == "":
+        head_repo = APACHE_AIRFLOW_GITHUB_REPOSITORY
+
     """Adds back references for documentation generated by build-docs and publish-docs"""
     site_path = Path(airflow_site_directory)
     if not site_path.is_dir():
@@ -1912,6 +1936,8 @@ def add_back_references(
                 include_not_ready=include_not_ready_providers,
             )
         ),
+        head_ref=head_ref,
+        head_repo=head_repo,
     )
 
 

dev/breeze/src/airflow_breeze/commands/release_management_commands_config.py

@@ -382,6 +382,8 @@
                 "--airflow-site-directory",
                 "--include-not-ready-providers",
                 "--include-removed-providers",
+                "--head-repo",
+                "--head-ref",
             ],
         },
     ],

dev/breeze/src/airflow_breeze/utils/add_back_references.py

@@ -26,9 +26,10 @@
 from airflow_breeze.utils.console import get_console
 
 airflow_redirects_link = (
-    "https://raw.githubusercontent.com/apache/airflow/main/airflow-core/docs/redirects.txt"
+    "https://raw.githubusercontent.com/{head_repo}/{head_ref}/airflow-core/docs/redirects.txt"
 )
-helm_redirects_link = "https://raw.githubusercontent.com/apache/airflow/main/docs/helm-chart/redirects.txt"
+
+helm_redirects_link = "https://raw.githubusercontent.com/{head_repo}/{head_ref}/docs/helm-chart/redirects.txt"
 
 
 def download_file(url):
@@ -64,10 +65,10 @@ def get_redirect_content(url: str):
     return f'<html><head><meta http-equiv="refresh" content="0; url={url}"/></head></html>'
 
 
-def get_github_provider_redirects_url(provider_name: str):
-    return (
-        f"https://raw.githubusercontent.com/apache/airflow/main/providers/{provider_name}/docs/redirects.txt"
-    )
+def get_github_provider_redirects_url(
+    provider_name: str, head_repo: str = "apache/airflow", head_ref: str = "main"
+) -> str:
+    return f"https://raw.githubusercontent.com/{head_repo}/{head_ref}/providers/{provider_name}/docs/redirects.txt"
 
 
 def crete_redirect_html_if_not_exist(path: Path, content: str):
@@ -133,15 +134,20 @@ def generate_back_references(link: str, base_path: Path):
 def start_generating_back_references(
     airflow_site_directory: Path,
     short_provider_ids: list[str],
+    head_repo: str = "apache/airflow",
+    head_ref: str = "main",
 ):
+    airflow_redirects_url = airflow_redirects_link.format(head_repo=head_repo, head_ref=head_ref)
+    helm_redirects_url = helm_redirects_link.format(head_repo=head_repo, head_ref=head_ref)
+
     docs_archive_path = airflow_site_directory / "docs-archive"
     airflow_docs_path = docs_archive_path / "apache-airflow"
     helm_docs_path = docs_archive_path / "helm-chart"
     if "apache-airflow" in short_provider_ids:
-        generate_back_references(airflow_redirects_link, airflow_docs_path)
+        generate_back_references(airflow_redirects_url, airflow_docs_path)
         short_provider_ids.remove("apache-airflow")
     if "helm-chart" in short_provider_ids:
-        generate_back_references(helm_redirects_link, helm_docs_path)
+        generate_back_references(helm_redirects_url, helm_docs_path)
         short_provider_ids.remove("helm-chart")
     if "docker-stack" in short_provider_ids:
         get_console().print("[info]Skipping docker-stack package. No back-reference needed.")
@@ -155,6 +161,6 @@ def start_generating_back_references(
             full_provider_name = f"apache-airflow-providers-{p.replace('.', '-')}"
             get_console().print(f"Processing airflow provider: {full_provider_name}")
             generate_back_references(
-                get_github_provider_redirects_url(slash_based_short_provider_id),
+                get_github_provider_redirects_url(slash_based_short_provider_id, head_repo, head_ref),
                 docs_archive_path / full_provider_name,
             )