Feature/external sources in needsjson

docs/changelog.rst

@@ -4,6 +4,28 @@
 Changelog
 =========
 
+Unreleased
+----------
+
+External sources provenance in needs.json
+.........................................
+
+When external needs are included in the ``needs.json`` output (by relaxing
+:ref:`needs_builder_filter`), the JSON now contains an ``external_sources``
+section in each version block that records the full provenance chain.
+
+Each external need also receives an ``external_source`` field linking it to
+the ``base_url`` of its source configuration. When a downstream project
+consumes this ``needs.json`` and re-exports, the inherited sources are
+propagated with an ``origin`` field indicating the intermediate project.
+
+This enables full traceability across multi-project documentation chains
+(e.g. Project X |rarr| Project A |rarr| Project B).
+
+.. |rarr| unicode:: U+2192
+
+See :ref:`needs_external_needs` for configuration details.
+
 .. _`release:8.2.0`:
 
 8.2.0

docs/configuration.rst

@@ -1851,6 +1851,53 @@ Need objects imported via :ref:`needs_external_needs` get sorted out.
 
    needs_builder_filter = 'status=="open"'
 
+.. _`needs_builder_filter_provenance`:
+
+**External sources provenance in needs.json**
+
+.. versionadded:: 8.3.0
+
+When ``needs_builder_filter`` is set to include external needs (e.g. ``""`` or
+``"True"``), the generated ``needs.json`` will also include an
+``external_sources`` section in each version block. This records the
+:ref:`needs_external_needs` configuration entries that produced the external
+needs, enabling downstream consumers to reconstruct the full provenance chain.
+
+Each external need in the JSON output also receives an ``external_source``
+field containing the ``base_url`` of its source.
+
+**Example output:**
+
+.. code-block:: json
+
+   {
+     "versions": {
+       "1.0": {
+         "external_sources": [
+           {
+             "base_url": "https://upstream.io/en/latest",
+             "id_prefix": "UP_",
+             "json_path": "upstream_needs.json",
+             "origin": null
+           }
+         ],
+         "needs": {
+           "UP_REQ_01": {
+             "is_external": true,
+             "external_source": "https://upstream.io/en/latest",
+             "external_url": "https://upstream.io/en/latest/index.html#REQ_01"
+           }
+         }
+       }
+     }
+   }
+
+**Transitive provenance:** When a downstream project consumes a ``needs.json``
+that already contains ``external_sources``, those entries are inherited and
+re-exported with an ``origin`` field set to the intermediate project's
+``base_url``. This allows any consumer to trace the full chain of provenance
+back to the original source.
+
 .. _`needs_string_links`:
 
 needs_string_links

sphinx_needs/data.py

@@ -540,6 +540,41 @@ class NeedsInfoComputedType(TypedDict):
     """True if all constraints passed, False if any failed, None if not yet checked."""
 
 
+class ExternalSourceInfo(TypedDict, total=False):
+    """Metadata about an external source that contributed needs to this project.
+
+    Stored in the ``external_sources`` section of the needs.json output so that
+    downstream consumers can reconstruct the full provenance chain.
+    """
+
+    base_url: str
+    """The base URL of the external source (required)."""
+
+    target_url: str
+    """Jinja template for constructing per-need URLs (optional)."""
+
+    id_prefix: str
+    """Prefix applied to all IDs from this source (optional)."""
+
+    css_class: str
+    """CSS class applied to needs from this source (optional)."""
+
+    json_url: str
+    """Remote URL the JSON was fetched from (optional)."""
+
+    json_path: str
+    """Local path the JSON was loaded from (optional)."""
+
+    version: str
+    """Version used when loading from this source (optional)."""
+
+    origin: str | None
+    """base_url of the intermediate project that re-exported this source.
+
+    ``None`` means the source was directly consumed by this project.
+    """
+
+
 class NeedsBaseDataType(TypedDict):
     """A base type for data items collected from directives."""
 
@@ -989,6 +1024,56 @@ def get_need_node(self, need_id: str) -> Need | None:
             return self._needs_all_nodes[need_id].deepcopy()
         return None
 
+    # --- External source provenance tracking ---
+
+    @property
+    def external_source_map(self) -> dict[str, str]:
+        """Mapping of need_id to the base_url of the external source that provided it.
+
+        Populated during :func:`~sphinx_needs.external_needs.load_external_needs`.
+        """
+        try:
+            return self.env._needs_external_source_map
+        except AttributeError:
+            self.env._needs_external_source_map = {}
+        return self.env._needs_external_source_map
+
+    def register_external_source(self, need_id: str, base_url: str) -> None:
+        """Register which external source a need came from.
+
+        :param need_id: The ID of the external need (after prefix applied).
+        :param base_url: The ``base_url`` of the ExternalSource config entry.
+        """
+        self.external_source_map[need_id] = base_url
+
+    @property
+    def inherited_external_sources(self) -> list[ExternalSourceInfo]:
+        """External source entries inherited from consumed needs.json files.
+
+        These are propagated into the output ``needs.json`` so downstream
+        consumers can reconstruct the full provenance chain.
+        """
+        try:
+            return self.env._needs_inherited_external_sources
+        except AttributeError:
+            self.env._needs_inherited_external_sources = []
+        return self.env._needs_inherited_external_sources
+
+    def add_inherited_external_sources(self, sources: list[ExternalSourceInfo]) -> None:
+        """Add inherited external source entries (from a consumed needs.json).
+
+        Deduplicates by ``base_url``.
+
+        :param sources: The ``external_sources`` list from a consumed JSON file.
+        """
+        existing = {
+            s["base_url"] for s in self.inherited_external_sources if "base_url" in s
+        }
+        for source in sources:
+            if source.get("base_url") and source["base_url"] not in existing:
+                self.inherited_external_sources.append(source)
+                existing.add(source["base_url"])
+
 
 def merge_data(
     _app: Sphinx, env: BuildEnvironment, docnames: list[str], other: BuildEnvironment
@@ -1056,3 +1141,7 @@ def _merge(name: str, is_complex_dict: bool = False) -> None:
     _merge("_needs_all_nodes")
     _merge("_need_all_needextend")
     _merge("_needs_all_needumls")
+
+    # Merge external source provenance data
+    _merge("_needs_external_source_map")
+    this_data.add_inherited_external_sources(other_data.inherited_external_sources)

sphinx_needs/external_needs.py

@@ -2,6 +2,7 @@
 
 import json
 import os
+from typing import Any
 
 import requests
 from requests_file import FileAdapter
@@ -10,8 +11,8 @@
 
 from sphinx_needs._jinja import compile_template
 from sphinx_needs.api import InvalidNeedException, add_external_need, del_need
-from sphinx_needs.config import NeedsSphinxConfig
-from sphinx_needs.data import NeedsCoreFields, SphinxNeedsData
+from sphinx_needs.config import ExternalSource, NeedsSphinxConfig
+from sphinx_needs.data import ExternalSourceInfo, NeedsCoreFields, SphinxNeedsData
 from sphinx_needs.logging import get_logger, log_warning
 from sphinx_needs.need_item import NeedItemSourceExternal
 from sphinx_needs.utils import clean_log, import_prefix_link_edit
@@ -25,6 +26,7 @@ def load_external_needs(
     """Load needs from configured external sources."""
     needs_config = NeedsSphinxConfig(app.config)
     needs_schema = SphinxNeedsData(env).get_schema()
+    data_accessor = SphinxNeedsData(env)
 
     for idx, source in enumerate(needs_config.external_needs):
         if "base_url" not in source:
@@ -106,6 +108,10 @@ def load_external_needs(
                     )
                 )
 
+        # Inherit external_sources from the consumed JSON for provenance chain
+        if "external_sources" in data:
+            _inherit_external_sources(data_accessor, data["external_sources"], source)
+
         log.debug(f"Loading {len(needs)} needs.")
 
         defaults = (
@@ -124,6 +130,7 @@ def load_external_needs(
         # all known need fields in the project
         known_keys = {
             "full_title",  # legacy
+            "external_source",  # provenance metadata from needs.json
             *NeedsCoreFields,
             *(x for x in needs_schema.iter_link_field_names()),
             *(f"{x}_back" for x in needs_schema.iter_link_field_names()),
@@ -132,6 +139,7 @@ def load_external_needs(
         # all keys that should not be imported from external needs
         omitted_keys = {
             "full_title",  # legacy
+            "external_source",  # provenance metadata, not a need field
             *(k for k, v in NeedsCoreFields.items() if v.get("exclude_external")),
             *(f"{x}_back" for x in needs_schema.iter_link_field_names()),
         }
@@ -195,6 +203,8 @@ def load_external_needs(
                     allow_type_coercion=source.get("allow_type_coercion", True),
                     **need_params,
                 )
+                # Register provenance: which source produced this need
+                data_accessor.register_external_source(ext_need_id, source["base_url"])
             except InvalidNeedException as err:
                 location = source.get("json_url", "") or source.get("json_path", "")
                 log_warning(
@@ -220,3 +230,40 @@ def load_external_needs(
 
 class NeedsExternalException(BaseException):
     pass
+
+
+def _inherit_external_sources(
+    data_accessor: SphinxNeedsData,
+    sources_from_json: list[dict[str, Any]],
+    consuming_source: ExternalSource,
+) -> None:
+    """Inherit ``external_sources`` entries from a consumed needs.json.
+
+    For each entry in the consumed JSON's ``external_sources``, we re-record it
+    with ``origin`` set to the ``base_url`` of the source we are consuming from
+    (unless it already has an origin, meaning it was already a transitive entry).
+
+    :param data_accessor: The SphinxNeedsData instance.
+    :param sources_from_json: The ``external_sources`` list from the consumed JSON.
+    :param consuming_source: The ExternalSource config entry we are currently loading.
+    """
+    inherited: list[ExternalSourceInfo] = []
+    consumer_base_url = consuming_source["base_url"]
+    for entry in sources_from_json:
+        info = ExternalSourceInfo(
+            base_url=entry["base_url"],
+            origin=entry.get("origin") or consumer_base_url,
+        )
+        # Carry over optional fields if present
+        for key in (
+            "target_url",
+            "id_prefix",
+            "css_class",
+            "json_url",
+            "json_path",
+            "version",
+        ):
+            if key in entry:
+                info[key] = entry[key]
+        inherited.append(info)
+    data_accessor.add_inherited_external_sources(inherited)

sphinx_needs/needsfile.py

@@ -19,7 +19,7 @@
 from sphinx.environment import BuildEnvironment
 
 from sphinx_needs.config import NeedsSphinxConfig
-from sphinx_needs.data import NeedsCoreFields, SphinxNeedsData
+from sphinx_needs.data import ExternalSourceInfo, NeedsCoreFields, SphinxNeedsData
 from sphinx_needs.logging import get_logger, log_warning
 from sphinx_needs.need_item import NeedItem
 from sphinx_needs.needs_schema import FieldLiteralValue, FieldsSchema
@@ -111,6 +111,7 @@ def __init__(
             self.needs_list["created"] = ""
         self.log = log
 
+        self._data_accessor = SphinxNeedsData(env)
         self._exclude_need_keys = set(self.needs_config.json_exclude_fields)
 
         schema = SphinxNeedsData(env).get_schema()
@@ -174,6 +175,11 @@ def add_need(self, version: str, need_info: NeedItem) -> None:
                     key in self._need_defaults and value == self._need_defaults[key]
                 )
             }
+        # Inject external_source provenance for external needs
+        if need_info.get("is_external"):
+            source_map = self._data_accessor.external_source_map
+            if need_info["id"] in source_map:
+                writable_needs["external_source"] = source_map[need_info["id"]]
         self.needs_list["versions"][version]["needs"][need_info["id"]] = writable_needs
         self.needs_list["versions"][version]["needs_amount"] = len(
             self.needs_list["versions"][version]["needs"]
@@ -193,6 +199,54 @@ def _finalise(self) -> None:
         self.needs_list["current_version"] = self.current_version
         self.needs_list["project"] = self.project
 
+        # Build external_sources provenance metadata
+        external_sources = self._build_external_sources()
+        if external_sources:
+            self.needs_list["versions"][self.current_version]["external_sources"] = (
+                external_sources
+            )
+
+    def _build_external_sources(self) -> list[ExternalSourceInfo]:
+        """Build the ``external_sources`` list for the current version.
+
+        Combines:
+        - Direct sources from this project's ``needs_external_needs`` config
+        - Inherited sources from consumed needs.json files (transitive)
+        """
+        sources: list[ExternalSourceInfo] = []
+        seen_base_urls: set[str] = set()
+
+        # Direct sources from config
+        for ext_source in self.needs_config.external_needs:
+            base_url = ext_source.get("base_url", "")
+            if not base_url or base_url in seen_base_urls:
+                continue
+            info = ExternalSourceInfo(base_url=base_url, origin=None)
+            if "target_url" in ext_source:
+                info["target_url"] = ext_source["target_url"]
+            if "id_prefix" in ext_source:
+                info["id_prefix"] = ext_source["id_prefix"]
+            if "css_class" in ext_source:
+                info["css_class"] = ext_source["css_class"]
+            if "json_url" in ext_source:
+                info["json_url"] = ext_source["json_url"]
+            if "json_path" in ext_source:
+                info["json_path"] = ext_source["json_path"]
+            if "version" in ext_source:
+                info["version"] = ext_source["version"]
+            sources.append(info)
+            seen_base_urls.add(base_url)
+
+        # Inherited (transitive) sources
+        for inherited in self._data_accessor.inherited_external_sources:
+            base_url = inherited.get("base_url", "")
+            if not base_url or base_url in seen_base_urls:
+                continue
+            sources.append(inherited)
+            seen_base_urls.add(base_url)
+
+        return sources
+
     def write_json(self, needs_file: str = "needs.json", needs_path: str = "") -> None:
         self._finalise()
         needs_dir = needs_path if needs_path else self.outdir