Upgrade sphinx and related dependencies

docs/exts/airflow_intersphinx.py

@@ -126,7 +126,7 @@ def fetch_inventories(intersphinx_mapping) -> dict[str, Any]:
             cache: dict[Any, Any] = {}
             with concurrent.futures.ThreadPoolExecutor() as pool:
                 for name, (uri, invs) in intersphinx_mapping.values():
-                    pool.submit(fetch_inventory_group, name, uri, invs, cache, _MockApp(), now)
+                    pool.submit(fetch_inventory_group, name, uri, invs, cache, _MockApp(), now)  # type: ignore[arg-type]
 
             inv_dict = {}
             for uri, (name, now, invdata) in cache.items():

docs/sphinx_design/static/custom.css

@@ -40,3 +40,29 @@ div.admonition.warning {
 .rst-content .warning .admonition-title {
   background: #cc341d;
 }
+
+/* Patches as of moving to Sphinx 7 to get layout to previous state  */
+/* Needs to be cleaned in a follow-up to source this from the origin style in */
+/* https://github.com/apache/airflow-site/blob/main/landing-pages/site/assets/scss/_rst-content.scss */
+.base-layout {
+  padding-top: 123px !important;
+}
+
+section {
+  padding-top: 0rem !important;
+  padding-bottom: 0rem !important;
+}
+
+section ol li p:last-child, section ul li p:last-child {
+  margin-bottom: 0 !important;
+}
+
+a.headerlink {
+  content: "" !important;
+  font-size: 75% !important;
+}
+
+a.headerlink::after {
+  content: " [link]" !important;  /* Theme image not existing */
+  visibility: visible !important;
+}

hatch_build.py

@@ -148,25 +148,18 @@
 
 DOC_EXTRAS: dict[str, list[str]] = {
     "doc": [
-        "astroid>=2.12.3,<3.0",
+        "astroid>=3",
         "checksumdir>=1.2.0",
         "click>=8.1.8",
-        # Docutils 0.17.0 converts generated <div class="section"> into <section> and breaks our doc formatting
-        # By adding a lot of whitespace separation. This limit can be lifted when we update our doc to handle
-        # <section> tags for sections
-        "docutils<0.17,>=0.16",
-        # The new theme 0.1.0 is for Sphinx 7. We should change it to >=0.1.0 when we move to Sphinx 7
-        "sphinx-airflow-theme>=0.0.12,<0.1.0",
+        "docutils>=0.21",
+        "sphinx-airflow-theme>=0.1.0",
         "sphinx-argparse>=0.4.0",
-        # sphinx-autoapi fails with astroid 3.0, see: https://github.com/readthedocs/sphinx-autoapi/issues/407
-        # This was fixed in sphinx-autoapi 3.0, however it has requirement sphinx>=6.1, but we stuck on 5.x
-        "sphinx-autoapi>=2.1.1",
+        "sphinx-autoapi>=3",
         "sphinx-copybutton>=0.5.2",
         "sphinx-design>=0.5.0",
         "sphinx-jinja>=2.0.2",
         "sphinx-rtd-theme>=2.0.0",
-        # Currently we are using sphinx 5 but we need to migrate to Sphinx 7
-        "sphinx>=5.3.0,<6.0.0",
+        "sphinx>=7",
         "sphinxcontrib-applehelp>=1.0.4",
         "sphinxcontrib-devhelp>=1.0.2",
         "sphinxcontrib-htmlhelp>=2.0.1",
@@ -175,7 +168,7 @@
         "sphinxcontrib-jsmath>=1.0.1",
         "sphinxcontrib-qthelp>=1.0.3",
         "sphinxcontrib-redoc>=1.6.0",
-        "sphinxcontrib-serializinghtml==1.1.5",
+        "sphinxcontrib-serializinghtml>=1.1.5",
         "sphinxcontrib-spelling>=8.0.0",
     ],
     "doc-gen": [