✨ Add nullable key to needs_fields items

docs/conf.py

@@ -335,11 +335,12 @@ def custom_defined_func():
 
 from sphinx_needs.api import generate_need  # noqa: E402
 from sphinx_needs.config import NeedsSphinxConfig  # noqa: E402
-from sphinx_needs.data import SphinxNeedsData  # noqa: E402
+from sphinx_needs.data import NeedsCoreFields, SphinxNeedsData  # noqa: E402
 from sphinx_needs.logging import (  # noqa: E402
     WarningSubTypeDescription,
     WarningSubTypes,
 )
+from sphinx_needs.needs import _get_core_schema  # noqa: E402
 from sphinx_needs.needsfile import NeedsList  # noqa: E402
 
 
@@ -389,6 +390,54 @@ def run(self):
         return [root]
 
 
+class NeedCoreFieldsDirective(SphinxDirective):
+    """Directive to list all core need fields."""
+
+    def run(self):
+        table: list[list[str]] = []
+        head = ["Name", "Type", "Nullable", "Default"]
+        lengths = [len(h) for h in head]
+
+        for name, data in NeedsCoreFields.items():
+            if not data.get("add_to_field_schema", False):
+                continue
+            schema, nullable = _get_core_schema(data)
+            type_ = schema["type"]
+            item_type = (
+                schema.get("items", {}).get("type", "") if type_ == "array" else ""
+            )
+            default = data["schema"].get("default", None)
+            row = [
+                f"**{name}**",
+                type_ if not item_type else f"{type_}({item_type})",
+                str(nullable).lower(),
+                str(default) if default is not None else "",
+            ]
+            table.append(row)
+            lengths = [max(ln, len(col)) for col, ln in zip(row, lengths, strict=True)]
+
+        delimiter = " ".join(["=" * length for length in lengths]) + "\n"
+        content = delimiter
+        content += (
+            " ".join(f"{h:<{ln}}" for h, ln in zip(head, lengths, strict=True)) + "\n"
+        )
+        content += delimiter
+        for row in table:
+            content += (
+                " ".join(f"{col:<{ln}}" for col, ln in zip(row, lengths, strict=True))
+                + "\n"
+            )
+        content += delimiter
+
+        # print(content)
+
+        parsed = nodes.container(classes=["needs-core-fields"])
+        self.state.nested_parse(
+            StringList(content.splitlines()), self.content_offset, parsed
+        )
+        return [parsed]
+
+
 class NeedConfigDefaultRole(SphinxRole):
     """Role to add a default configuration value to the documentation."""
 
@@ -430,6 +479,7 @@ def suppress_linkcheck_warnings(app: Sphinx):
 def setup(app: Sphinx):
     app.add_directive("need-example", NeedExampleDirective)
     app.add_directive("need-warnings", NeedsWarningsDirective)
+    app.add_directive("need-core-fields", NeedCoreFieldsDirective)
     app.add_role("need_config_default", NeedConfigDefaultRole())
     app.connect("builder-inited", suppress_linkcheck_warnings)
     app.connect("env-before-read-docs", create_tutorial_needs, priority=600)

docs/configuration.rst

@@ -212,6 +212,8 @@ For new fields the following can be defined:
     This uses the same format as :ref:`needs_schema_definitions` and :ref:`needs_schema_definitions_from_json`.
     If specified, the field value will be validated against this schema when needs are parsed, see :ref:`schema_validation` for more details.
     By default the field schema will be ``{"type": "string"}``.
+- ``nullable``: If set to ``True``, the field can be set to ``None`` (optional), e.g. if no value is specifically given and no default applies,
+    If ``False``, the field must have a value (either explicitly set or via default/predicates), otherwise the need is invalid and will not be created.
 - ``predicates``: A list of ``(match expression, value)`` tuples (optional).
     If specified, these will be evaluated in order for any need that does not explicitly set the field, with the first match setting the field value.
 - ``default``: A default value for the field (optional).
@@ -228,6 +230,7 @@ For example:
                "type": "string",
                "format": "date",
            },
+           "nullable": False,
        },
        "cost": {
              "description": "Approximated cost in Euros",
@@ -264,14 +267,7 @@ Core field specialization
 
 The following core fields can be specialized:
 
-- ``title`` (string)
-- ``status`` (nullable string)
-- ``tags`` (array of strings)
-- ``collapse`` (boolean)
-- ``hide`` (boolean)
-- ``layout`` (nullable string)
-- ``style`` (nullable string)
-
+.. need-core-fields::
 
 Specialization allows you to redefine the description and tighten the schema of these fields:
 Schemas will inherit any constraints defined in the core schema that are not redefined, and redefinitions must be not weaken the constraints of the original schema (this is intended to obey the `Liskov substitution principle <https://en.wikipedia.org/wiki/Liskov_substitution_principle>`__).
@@ -286,6 +282,9 @@ For example, you could redefine the ``status`` and ``tags`` fields to only allow
                # adds tighter constraint on allowed values
                "enum": ["open", "in progress", "done", "closed"],
            },
+           # adds tighter constraint on nullability,
+           # i.e. the status must always be set
+           "nullable": False,
        },
          "tags": {
             "schema": {

sphinx_needs/api/configuration.py

@@ -92,6 +92,7 @@ def add_extra_option(
     *,
     description: str = "Added by add_extra_option API",
     schema: ExtraOptionSchemaTypes | None = None,
+    nullable: bool | None = None,
 ) -> None:
     """
     Adds an extra option to the configuration. This option can then later be used inside needs or ``add_need``.
@@ -108,9 +109,10 @@ def add_extra_option(
     :param name: Name of the extra option
     :param description: Description of the extra option
     :param schema: Schema definition for the extra option
+    :param nullable: Whether the field allows unset values.
     :return: None
     """
-    _NEEDS_CONFIG.add_extra_option(name, description, schema=schema)
+    _NEEDS_CONFIG.add_extra_option(name, description, schema=schema, nullable=nullable)
 
 
 # TODO(mh) add extra link api

sphinx_needs/api/need.py

@@ -397,15 +397,21 @@ def generate_need(
     _copy_links(links, needs_config)
 
     title, title_func = _convert_to_str_func("title", title_converted)
-    status, status_func = _convert_to_none_str_func("status", status_converted)
+    status, status_func = _convert_to_none_str_func(
+        needs_schema, "status", status_converted
+    )
     tags, tags_func = _convert_to_list_str_func("tags", tags_converted)
     constraints, constraints_func = _convert_to_list_str_func(
         "constraints", constraints_converted
     )
     collapse, collapse_func = _convert_to_bool_func("collapse", collapse_converted)
     hide, hide_func = _convert_to_bool_func("hide", hide_converted)
-    layout, layout_func = _convert_to_none_str_func("layout", layout_converted)
-    style, style_func = _convert_to_none_str_func("style", style_converted)
+    layout, layout_func = _convert_to_none_str_func(
+        needs_schema, "layout", layout_converted
+    )
+    style, style_func = _convert_to_none_str_func(
+        needs_schema, "style", style_converted
+    )
 
     dynamic_fields: dict[str, FieldFunctionArray | LinksFunctionArray] = {}
     if title_func:
@@ -483,7 +489,7 @@ def generate_need(
         if (extra_schema := needs_schema.get_extra_field(k)) is None:
             raise InvalidNeedException(
                 "invalid_extra_option",
-                f"Extra option {k!r} not in 'needs_extra_options'.",
+                f"Extra option {k!r} not in 'needs_fields'.",
             )
         if v is None:
             if not extra_schema.nullable:
@@ -649,17 +655,26 @@ def _convert_to_str_func(
 
 
 def _convert_to_none_str_func(
-    name: str, converted: FieldLiteralValue | FieldFunctionArray | None
+    schema: FieldsSchema,
+    name: str,
+    converted: FieldLiteralValue | FieldFunctionArray | None,
 ) -> tuple[None | str, None | FieldFunctionArray]:
+    field_schema = schema.get_core_field(name)
+    assert field_schema is not None, f"{name} field schema does not exist"
     if converted is None:
+        if not field_schema.nullable:
+            raise InvalidNeedException(
+                "invalid_value",
+                f"{name} is not nullable, but no value was given.",
+            )
         return None, None
     elif isinstance(converted, FieldLiteralValue) and isinstance(converted.value, str):
         return converted.value, None
     elif isinstance(converted, FieldFunctionArray) and all(
         isinstance(x, str | DynamicFunctionParsed | VariantFunctionParsed)
         for x in converted
     ):
-        return None, converted
+        return None if field_schema.nullable else "", converted
     else:
         raise InvalidNeedException(
             "invalid_value",

sphinx_needs/config.py

@@ -39,6 +39,8 @@ class ExtraOptionParams:
 
     description: str
     """A description of the option."""
+    nullable: bool | None = None
+    """Whether the field allows unset values."""
     schema: (
         ExtraOptionStringSchemaType
         | ExtraOptionBooleanSchemaType
@@ -112,6 +114,7 @@ def add_extra_option(
         | ExtraOptionNumberSchemaType
         | ExtraOptionMultiValueSchemaType
         | None = None,
+        nullable: None | bool = None,
         override: bool = False,
     ) -> None:
         """Adds an extra option to the configuration."""
@@ -142,6 +145,7 @@ def add_extra_option(
         self._extra_options[name] = ExtraOptionParams(
             description=description,
             schema=schema,
+            nullable=nullable,
         )
 
     @property
@@ -306,6 +310,8 @@ class NeedFields(TypedDict):
     If given, the schema will apply to all needs that use this option.
     For more granular control, use the `needs_schema_definitions` configuration.
     """
+    nullable: NotRequired[bool]
+    """Whether the field allows unset values."""
     default: NotRequired[Any]
     """Default value for the field."""
     predicates: NotRequired[list[tuple[str, Any]]]

sphinx_needs/needs.py

@@ -34,6 +34,7 @@
 )
 from sphinx_needs.data import (
     ENV_DATA_VERSION,
+    CoreFieldParameters,
     NeedsCoreFields,
     SphinxNeedsData,
     merge_data,
@@ -558,8 +559,9 @@ def load_config(app: Sphinx, *_args: Any) -> None:
             continue
         description = option_params.get("description", "Added by needs_fields config")
         schema = option_params.get("schema")
+        nullable = option_params.get("nullable")
         _NEEDS_CONFIG.add_extra_option(
-            option_name, description, schema=schema, override=True
+            option_name, description, schema=schema, nullable=nullable, override=True
         )
 
     # ensure options for `needgantt` functionality are added to the extra options
@@ -799,22 +801,27 @@ def check_configuration(app: Sphinx, config: Config) -> None:
     validate_schemas_config(app, needs_config)
 
 
+def _get_core_schema(data: CoreFieldParameters) -> tuple[dict[str, Any], bool]:
+    type_ = data["schema"]["type"]
+    nullable = False
+    if isinstance(type_, list):
+        assert type_[1] == "null", "Only nullable types supported as list"
+        type_ = type_[0]
+        nullable = True
+    schema = {"type": type_}
+    if type_ == "array":
+        schema["items"] = data["schema"].get("items", {"type": "string"})
+    return schema, nullable
+
+
 def create_schema(app: Sphinx, env: BuildEnvironment, _docnames: list[str]) -> None:
     needs_config = NeedsSphinxConfig(app.config)
     schema = FieldsSchema()
     for name, data in NeedsCoreFields.items():
         if not data.get("add_to_field_schema", False):
             continue
         description = data["description"]
-        type_ = data["schema"]["type"]
-        nullable = False
-        if isinstance(type_, list):
-            assert type_[1] == "null", "Only nullable types supported as list"
-            type_ = type_[0]
-            nullable = True
-        _schema = {"type": type_}
-        if type_ == "array":
-            _schema["items"] = data["schema"].get("items", {"type": "string"})
+        _schema, nullable = _get_core_schema(data)
 
         # merge in additional schema from needs_statuses and needs_tags config
         if name == "status" and needs_config.statuses:
@@ -855,9 +862,7 @@ def create_schema(app: Sphinx, env: BuildEnvironment, _docnames: list[str]) -> N
 
         if (core_override := needs_config._fields.get(name)) is not None:
             try:
-                field = create_inherited_field(
-                    field, cast(dict[str, Any], core_override)
-                )
+                field = create_inherited_field(field, core_override)
             except Exception as exc:
                 raise NeedsConfigException(
                     f"Invalid `needs_fields` core option override for {name!r}: {exc}"
@@ -875,14 +880,19 @@ def create_schema(app: Sphinx, env: BuildEnvironment, _docnames: list[str]) -> N
                 if extra.schema is not None
                 else {"type": "string"}
             )
+            if extra.nullable is not None:
+                nullable = extra.nullable
+            else:
+                # follows that of legacy (pre-schema) extra option,
+                # i.e. nullable if schema is defined
+                nullable = extra.schema is not None
             field = FieldSchema(
                 name=name,
                 description=extra.description,
                 schema=_schema,  # type: ignore[arg-type]
-                # TODO for nullable and default, currently if there is no schema,
-                # we configure so that the behaviour follows that of legacy (pre-schema) extra option,
-                # i.e. non-nullable and default of empty string (that can be overriden).
-                nullable=extra.schema is not None,
+                nullable=nullable,
+                # note, default follows that of legacy (pre-schema) extra option,
+                # i.e. default to "" only if no schema is defined
                 default=None if extra.schema is not None else FieldLiteralValue(""),
                 allow_defaults=True,
                 allow_extend=True,

sphinx_needs/needs_schema.py

@@ -4,10 +4,11 @@
 from collections.abc import Iterable, Iterator, Sequence
 from dataclasses import dataclass, replace
 from functools import lru_cache, partial
-from typing import TYPE_CHECKING, Any, Generic, Literal, TypeAlias, TypeVar
+from typing import TYPE_CHECKING, Any, Generic, Literal, TypeAlias, TypeVar, cast
 
 import jsonschema_rs
 
+from sphinx_needs.config import NeedFields
 from sphinx_needs.exceptions import VariantParsingException
 from sphinx_needs.schema.config import (
     ExtraOptionBooleanSchemaType,
@@ -996,7 +997,7 @@ def _split_string(
     )
 
 
-def create_inherited_field(parent: FieldSchema, child: dict[str, Any]) -> FieldSchema:
+def create_inherited_field(parent: FieldSchema, child: NeedFields) -> FieldSchema:
     """Create a new FieldSchema by inheriting from a parent FieldSchema and applying overrides from a child dictionary."""
     replacements: dict[str, Any] = {}
 
@@ -1007,10 +1008,15 @@ def create_inherited_field(parent: FieldSchema, child: dict[str, Any]) -> FieldS
 
     if "schema" in child:
         child_schema = child["schema"]
-        inherit_schema(parent.schema, child_schema)
+        inherit_schema(parent.schema, cast(dict[str, Any], child_schema))
         replacements["schema"] = child_schema
 
-    # TODO allow nullable to be inherited, only allow False -> True changes
+    if "nullable" in child:
+        if not isinstance(child["nullable"], bool):
+            raise ValueError("Child 'nullable' must be a boolean.")
+        if parent.nullable is False and child["nullable"] is True:
+            raise ValueError("Cannot change 'nullable' from False to True in child.")
+        replacements["nullable"] = child["nullable"]
 
     return replace(parent, **replacements)