Start working on the docs for the Task SDK

task-sdk/docs/.gitignore

@@ -0,0 +1 @@
+_build/

task-sdk/docs/api.rst

@@ -0,0 +1,77 @@
+ .. Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+ ..   http://www.apache.org/licenses/LICENSE-2.0
+
+ .. Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
+airflow.sdk API Reference
+=========================
+
+This page documents the full public API exposed in Airflow 3.0+ via the Task SDK python module.
+
+If something is not on this page it is best to assume that it is not part of the public API and use of it is entirely at your own risk
+-- we won't go out of our way break usage of them, but we make no promises either.
+
+.. :py:module: airflow.sdk
+
+Defining DAGs
+-------------
+
+.. autoapiclass:: airflow.sdk.DAG
+
+.. autoapifunction:: airflow.sdk.dag
+
+.. autoapiclass:: airflow.sdk.TaskGroup
+
+.. autoapifunction:: airflow.sdk.get_parsing_context
+
+.. autoapiclass:: airflow.sdk.definitions.context.AirflowParsingContext
+   :undoc-members:
+   :members:
+
+
+Tasks and Operators
+-------------------
+
+.. autoapiclass:: airflow.sdk.BaseOperator
+
+.. autoapiclass:: airflow.sdk.XComArg
+
+.. autoapifunction:: airflow.sdk.get_current_context
+
+Assets
+------
+
+.. autoapiclass:: airflow.sdk.Asset
+
+.. autoapiclass:: airflow.sdk.AssetAlias
+
+.. autoapiclass:: airflow.sdk.AssetAll
+
+.. autoapiclass:: airflow.sdk.AssetAny
+
+.. autoapiclass:: airflow.sdk.AssetWatcher
+
+
+.. Asset, AssetAlias, AssetAll, AssetAny, AssetWatcher
+
+Everything else
+---------------
+
+.. autoapimodule:: airflow.sdk
+  :members:
+  :exclude-members: BaseOperator, DAG, dag, asset, Asset, AssetAlias, AssetAll, AssetAny, AssetWatcher, TaskGroup, XComArg, get_current_context, get_parsing_context
+  :undoc-members:
+  :imported-members:
+  :no-index:

task-sdk/docs/conf.py

@@ -0,0 +1,88 @@
+# Disable Flake8 because of all the sphinx imports
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from __future__ import annotations
+
+from pathlib import Path
+
+CONF_DIR = Path(__file__).parent.absolute()
+
+project = "Apache Airflow Task SDK"
+
+language = "en"
+
+extensions = [
+    "sphinx.ext.autodoc",
+    "autoapi.extension",
+    "sphinx.ext.intersphinx",
+]
+
+autoapi_dirs = [CONF_DIR.joinpath("..", "src").resolve()]
+autoapi_root = "api"
+autoapi_ignore = [
+    "*/airflow/sdk/execution_time",
+    "*/airflow/sdk/api",
+    "*/_internal*",
+]
+autoapi_options = [
+    "undoc-members",
+    "members",
+]
+autoapi_add_toctree_entry = False
+autoapi_generate_api_docs = False
+
+autodoc_typehints = "description"
+
+# Prefer pyi over py files if both are found
+autoapi_file_patterns = ["*.pyi", "*.py"]
+# autoapi_generate_api_docs = False
+
+html_theme = "sphinx_airflow_theme"
+
+global_substitutions = {
+    "experimental": "This is an :ref:`experimental feature <experimental>`.",
+}
+
+rst_epilog = "\n".join(f".. |{key}| replace:: {replace}" for key, replace in global_substitutions.items())
+
+
+intersphinx_resolve_self = "airflow"
+intersphinx_mapping = {
+    "airflow": (
+        "https://airflow.apache.org/docs/apache-airflow/stable/",
+        (
+            "../../docs/_inventory_cache/apache-airflow/objects.inv",
+            "../../docs/_build/apache-airflow/objects.inv",
+            None,
+        ),
+    )
+}
+
+
+def skip_util_classes(app, objtype, name, obj, skip, options):
+    if "definitions" in name:
+        if name == "DAG":
+            obj.id = "airflow.sdk.DAG"
+            return skip
+        skip = True
+    return skip
+
+
+def setup(sphinx):
+    # sphinx.connect("autoapi-skip-member", skip_util_classes)
+    ...

task-sdk/docs/index.rst

@@ -0,0 +1,26 @@
+ .. Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+ ..   http://www.apache.org/licenses/LICENSE-2.0
+
+ .. Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
+Apache Airflow Task Execution SDK
+=================================
+
+:any:`DAG` is where to start. :any:`dag`
+
+.. toctree::
+  :hidden:
+
+  api

task-sdk/src/airflow/sdk/__init__.pyi

@@ -0,0 +1,93 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from airflow.sdk.bases.notifier import BaseNotifier as BaseNotifier
+from airflow.sdk.bases.operator import (
+    BaseOperator as BaseOperator,
+    chain as chain,
+    chain_linear as chain_linear,
+    cross_downstream as cross_downstream,
+)
+from airflow.sdk.bases.operatorlink import BaseOperatorLink as BaseOperatorLink
+from airflow.sdk.bases.sensor import (
+    BaseSensorOperator as BaseSensorOperator,
+    PokeReturnValue as PokeReturnValue,
+)
+from airflow.sdk.definitions.asset import (
+    Asset as Asset,
+    AssetAlias as AssetAlias,
+    AssetAll as AssetAll,
+    AssetAny as AssetAny,
+    AssetWatcher as AssetWatcher,
+)
+from airflow.sdk.definitions.asset.decorators import asset as asset
+from airflow.sdk.definitions.asset.metadata import Metadata as Metadata
+from airflow.sdk.definitions.connection import Connection as Connection
+from airflow.sdk.definitions.context import (
+    Context as Context,
+    get_current_context as get_current_context,
+    get_parsing_context as get_parsing_context,
+)
+from airflow.sdk.definitions.dag import DAG as DAG, dag as dag
+from airflow.sdk.definitions.decorators import setup as setup, task as task, teardown as teardown
+from airflow.sdk.definitions.decorators.task_group import task_group as task_group
+from airflow.sdk.definitions.edges import EdgeModifier as EdgeModifier, Label as Label
+from airflow.sdk.definitions.param import Param as Param
+from airflow.sdk.definitions.taskgroup import TaskGroup as TaskGroup
+from airflow.sdk.definitions.template import literal as literal
+from airflow.sdk.definitions.variable import Variable as Variable
+from airflow.sdk.definitions.xcom_arg import XComArg as XComArg
+from airflow.sdk.io.path import ObjectStoragePath as ObjectStoragePath
+
+__all__ = [
+    "__version__",
+    "Asset",
+    "AssetAlias",
+    "AssetAll",
+    "AssetAny",
+    "AssetWatcher",
+    "BaseNotifier",
+    "BaseOperator",
+    "BaseOperatorLink",
+    "BaseSensorOperator",
+    "Connection",
+    "Context",
+    "DAG",
+    "EdgeModifier",
+    "Label",
+    "Metadata",
+    "ObjectStoragePath",
+    "Param",
+    "PokeReturnValue",
+    "TaskGroup",
+    "Variable",
+    "XComArg",
+    "asset",
+    "chain",
+    "chain_linear",
+    "cross_downstream",
+    "dag",
+    "get_current_context",
+    "get_parsing_context",
+    "literal",
+    "setup",
+    "task",
+    "task_group",
+    "teardown",
+]
+
+__version__: str

task-sdk/src/airflow/sdk/bases/operator.py

@@ -78,9 +78,7 @@
     import jinja2
 
     from airflow.models.xcom_arg import XComArg
-    from airflow.sdk.definitions.context import Context
-    from airflow.sdk.definitions.dag import DAG
-    from airflow.sdk.definitions.taskgroup import TaskGroup
+    from airflow.sdk import DAG, Context, TaskGroup
     from airflow.serialization.enums import DagAttributeTypes
     from airflow.task.priority_strategy import PriorityWeightStrategy
     from airflow.triggers.base import BaseTrigger, StartTriggerArgs

task-sdk/src/airflow/sdk/definitions/context.py

@@ -118,6 +118,8 @@ class AirflowParsingContext(NamedTuple):
 
     If these values are not None, they will contain the specific DAG and Task ID that Airflow is requesting to
     execute. You can use these for optimizing dynamically generated DAG files.
+
+    You can obtain the current values via :py:func:`.get_parsing_context`.
     """
 
     dag_id: str | None

task-sdk/src/airflow/sdk/definitions/dag.py

@@ -277,7 +277,7 @@ class DAG:
     :param schedule: If provided, this defines the rules according to which DAG
         runs are scheduled. Possible values include a cron expression string,
         timedelta object, Timetable, or list of Asset objects.
-        See also :doc:`/howto/timetable`.
+        See also :external:doc:`howto/timetable`.
     :param start_date: The timestamp from which the scheduler will
         attempt to backfill. If this is not provided, backfilling must be done
         manually with an explicit time range.
@@ -351,7 +351,7 @@ class DAG:
     :param tags: List of tags to help filtering DAGs in the UI.
     :param owner_links: Dict of owners and their links, that will be clickable on the DAGs view UI.
         Can be used as an HTTP link (for example the link to your Slack channel), or a mailto link.
-        e.g: {"dag_owner": "https://airflow.apache.org/"}
+        e.g: ``{"dag_owner": "https://airflow.apache.org/"}``
     :param auto_register: Automatically register this DAG when it is used in a ``with`` block
     :param fail_fast: Fails currently running tasks when task in DAG fails.
         **Warning**: A fail stop dag can only have tasks with the default trigger rule ("all_success").