feat: Support custom index generation by schema.Classifier (#37)

Date Index is supported too now :D

docs/_schemas/cat.py

@@ -3,10 +3,10 @@
 
 cat = Schema(
     'cat',
-    name=Field(referenceable=True, form=Field.Form.LINES),
+    name=Field(ref=True, form=Field.Forms.LINES),
     attrs={
-        'id': Field(unique=True, referenceable=True, required=True),
-        'color': Field(referenceable=True),
+        'id': Field(uniq=True, ref=True, required=True),
+        'color': Field(ref=True),
         'picture': Field(),
     },
     description_template=dedent("""

docs/_schemas/dog1.py

@@ -4,8 +4,8 @@
 dog = Schema(
     'dog',
     attrs={
-        'breed': Field(referenceable=True),
-        'color': Field(referenceable=True, form=Field.Form.WORDS),
+        'breed': Field(ref=True),
+        'color': Field(ref=True, form=Field.Forms.WORDS),
     },
     description_template=dedent("""
         :Breed: {{ breed }}

docs/_schemas/dog2.py

@@ -4,8 +4,8 @@
 dog = Schema(
     'dog',
     attrs={
-        'breed': Field(referenceable=True),
-        'color': Field(referenceable=True, form=Field.Form.WORDS),
+        'breed': Field(ref=True),
+        'color': Field(ref=True, form=Field.Forms.WORDS),
     },
     description_template=dedent("""
         :Breed: :any:dog.breed:`{{ breed }}`

docs/_schemas/tmplvar.py

@@ -3,7 +3,7 @@
 
 tmplvar = Schema(
     'tmplvar',
-    name=Field(unique=True, referenceable=True),
+    name=Field(uniq=True, ref=True),
     attrs={
         'type': Field(),
         'conf': Field(),

docs/conf.py

@@ -114,35 +114,37 @@
 
 # 
 # DOG FOOD CONFIGURATION START
-from any import Schema, Field as F
+from any import Schema, Field as F, DateClassifier
 sys.path.insert(0, os.path.abspath('.'))
 
 version_schema = Schema('version',
-                        name=F(unique=True, referenceable=True, required=True, form=F.Form.LINES),
-                        attrs={'date': F(referenceable=True)},
-                        content=F(form=F.Form.LINES),
+                        name=F(uniq=True, ref=True, required=True, form=F.Forms.LINES),
+                        attrs={
+                            'date': F(ref=True, classifiers=[DateClassifier(['%Y-%m-%d'])]),
+                        },
+                        content=F(form=F.Forms.LINES),
                         description_template=open('_templates/version.rst', 'r').read(),
                         reference_template='🏷️{{ title }}',
                         missing_reference_template='🏷️{{ title }}',
                         ambiguous_reference_template='🏷️{{ title }}')
 confval_schema = Schema('confval',
-                        name=F(unique=True, referenceable=True, required=True, form=F.Form.LINES),
+                        name=F(uniq=True, ref=True, required=True, form=F.Forms.LINES),
                         attrs={
                             'type': F(),
                             'default': F(),
-                            'choice': F(form=F.Form.WORDS),
+                            'choice': F(form=F.Forms.WORDS),
                             'versionadded': F(),
-                            'versionchanged': F(form=F.Form.LINES),
+                            'versionchanged': F(form=F.Forms.LINES),
                         },
                         content=F(),
                         description_template=open('_templates/confval.rst', 'r').read(),
                         reference_template='⚙️{{ title }}',
                         missing_reference_template='⚙️{{ title }}',
                         ambiguous_reference_template='⚙️{{ title }}')
 example_schema = Schema('example',
-                        name=F(referenceable=True),
+                        name=F(ref=True),
                         attrs={'style': F()},
-                        content=F(form=F.Form.LINES),
+                        content=F(form=F.Forms.LINES),
                         description_template=open('_templates/example.rst', 'r').read(),
                         reference_template='📝{{ title }}',
                         missing_reference_template='📝{{ title }}',

docs/usage.rst

@@ -30,11 +30,11 @@ The necessary python classes for writing schema are listed here:
 
    |
 
-.. autoclass:: any.Field.Form
+.. autoclass:: any.Field.Forms
 
-   .. autoattribute:: any.Field.Form.PLAIN
-   .. autoattribute:: any.Field.Form.WORDS
-   .. autoattribute:: any.Field.Form.LINES
+   .. autoattribute:: any.Field.Forms.PLAIN
+   .. autoattribute:: any.Field.Forms.WORDS
+   .. autoattribute:: any.Field.Forms.LINES
 
 Documenting Object
 ==================

src/sphinxnotes/any/__init__.py

@@ -14,7 +14,7 @@
 
 from .template import Environment as TemplateEnvironment
 from .domain import AnyDomain, warn_missing_reference
-from .schema import Schema, Field
+from .schema import Schema, Field, DateClassifier
 
 if TYPE_CHECKING:
     from sphinx.application import Sphinx
@@ -24,9 +24,10 @@
 
 logger = logging.getLogger(__name__)
 
-# Export
+# Re-Export
 Field = Field
 Schema = Schema
+DateClassifier = DateClassifier
 
 
 def _config_inited(app: Sphinx, config: Config) -> None:

src/sphinxnotes/any/domain.py

@@ -18,7 +18,7 @@
 from sphinx.util import logging
 from sphinx.util.nodes import make_refnode
 
-from .schema import Schema, Object
+from .schema import Schema, Object, PlainClassifier
 from .directives import AnyDirective
 from .roles import AnyRole
 from .indices import AnyIndex
@@ -176,29 +176,44 @@ def add_schema(cls, schema: Schema) -> None:
         # Add to schemas dict
         cls._schemas[schema.objtype] = schema
 
-        # Generates reftypes(role names) for all referenceable fields
         reftypes = [schema.objtype]
-        for name, field, _ in schema.fields_of(None):
-            if field.referenceable:
-                reftypes.append(objtype_and_objfield_to_reftype(schema.objtype, name))
-
-        # Roles is used for converting role name to corrsponding objtype
-        cls.object_types[schema.objtype] = ObjType(schema.objtype, *reftypes)
-        cls.directives[schema.objtype] = AnyDirective.derive(schema)
-        for r in reftypes:
-            # Create role for referencing object (by various fields)
-            _, field = reftype_to_objtype_and_objfield(r)
-            cls.roles[r] = AnyRole.derive(schema, field)(
+        for name, field in schema.fields(all=False):
+            if not field.ref:
+                continue
+
+            # Generates reftypes for all referenceable fields
+            # For later use when generating roles and indices.
+            reftype = objtype_and_objfield_to_reftype(schema.objtype, name)
+            reftypes.append(reftype)
+
+        for reftype in reftypes:
+            _, field = reftype_to_objtype_and_objfield(reftype)
+            # Create role for referencing object by field
+            cls.roles[reftype] = AnyRole.derive(schema, field)(
                 # Emit warning when missing reference (node['refwarn'] = True)
                 warn_dangling=True,
                 # Inner node (contnode) would be replaced in resolve_xref method,
                 # so fix its class.
                 innernodeclass=literal,
             )
 
-            index = AnyIndex.derive(schema, field)
-            cls.indices.append(index)
-            cls._indices_for_reftype[r] = index
+            # FIXME: name and content can not be index now
+            if field is not None:
+                classifiers = schema.attrs[field].classifiers
+            elif schema.name is not None:
+                classifiers = schema.name.classifiers
+            else:
+                classifiers = [PlainClassifier()]
+            # Generates index for indexing object by fields
+            for indexer in classifiers:
+                index = AnyIndex.derive(schema, field, indexer)
+                cls.indices.append(index)
+                cls._indices_for_reftype[reftype] = index  # TODO: mulitple catelogers.
+
+        # TODO: document
+        cls.object_types[schema.objtype] = ObjType(schema.objtype, *reftypes)
+        # Generates directive for creating object.
+        cls.directives[schema.objtype] = AnyDirective.derive(schema)
 
     def _get_index_anchor(self, reftype: str, refval: str) -> tuple[str, str]:
         """

src/sphinxnotes/any/indices.py

@@ -8,15 +8,15 @@
 :license: BSD, see LICENSE for details.
 """
 
-from typing import Iterable
+from typing import Iterable, TypeVar
 import re
 
-from sphinx.domains import Index, IndexEntry
+from sphinx.domains import Domain, Index, IndexEntry
 from sphinx.util import logging
 from docutils import core, nodes
 from docutils.parsers.rst import roles
 
-from .schema import Schema
+from .schema import Schema, Value, Classifier, Classif
 
 logger = logging.getLogger(__name__)
 
@@ -26,20 +26,20 @@ class AnyIndex(Index):
     Index subclass to provide the object reference index.
     """
 
+    domain: Domain  # for type hint
     schema: Schema
-    # TODO: document
     field: str | None = None
-
-    name: str
-    localname: str
-    shortname: str
+    classifier: Classifier
 
     @classmethod
-    def derive(cls, schema: Schema, field: str | None = None) -> type['AnyIndex']:
+    def derive(
+        cls, schema: Schema, field: str | None, classifier: Classifier
+    ) -> type['AnyIndex']:
         """Generate an AnyIndex child class for indexing object."""
+        # TODO: add Indexer.name
         if field:
             typ = f'Any{schema.objtype.title()}{field.title()}Index'
-            name = schema.objtype + '.' + field
+            name = schema.objtype + '.' + field  # TOOD: objtype_and_objfield_to_reftype
             localname = f'{schema.objtype.title()} {field.title()} Reference Index'
         else:
             typ = f'Any{schema.objtype.title()}Index'
@@ -49,65 +49,116 @@ def derive(cls, schema: Schema, field: str | None = None) -> type['AnyIndex']:
             typ,
             (cls,),
             {
-                'schema': schema,
-                'field': field,
                 'name': name,
                 'localname': localname,
                 'shortname': 'references',
+                'schema': schema,
+                'field': field,
+                'classifier': classifier,
             },
         )
 
     def generate(
         self, docnames: Iterable[str] | None = None
     ) -> tuple[list[tuple[str, list[IndexEntry]]], bool]:
         """Override parent method."""
-        content = {}  # type: dict[str, list[IndexEntry]]
-        # list of all references
-        objrefs = sorted(self.domain.data['references'].items())
 
-        # Reference value -> object IDs
-        objs_with_same_ref: dict[str, set[str]] = {}
+        # Single index for generating normal entries (subtype=0).
+        # Category (lv1) →  Category (for ordering objids) →  objids
+        singleidx: dict[Classif, dict[Classif, set[str]]] = {}
+        # Dual index for generating entrie (subtype=1) and its sub-entries (subtype=2).
+        # Category (lv1) →  Category (lv2) →  Category (for ordering objids) →  objids
+        dualidx: dict[Classif, dict[Classif, dict[Classif, set[str]]]] = {}
 
+        objrefs = sorted(self.domain.data['references'].items())
         for (objtype, objfield, objref), objids in objrefs:
             if objtype != self.schema.objtype:
                 continue
             if self.field and objfield != self.field:
                 continue
-            objs = objs_with_same_ref.setdefault(objref, set())
-            objs.update(objids)
-
-        for objref, objids in sorted(objs_with_same_ref.items()):
-            # Add a entry for objref
-            # 1: Entry with sub-entries.
-            entries = content.setdefault(objref, [])
-            for objid in sorted(objids):
-                docname, anchor, obj = self.domain.data['objects'][
-                    self.schema.objtype, objid
-                ]
-                if docnames and docname not in docnames:
-                    continue
-                name = self.schema.title_of(obj) or objid
-                extra = '' if name == objid else objid
-                objcont = self.schema.content_of(obj)
-                if isinstance(objcont, str):
-                    desc = objcont
-                elif isinstance(objcont, list):
-                    desc = '\n'.join(objcont)
+
+            # TODO: pass a real value
+            for catelog in self.classifier.classify(Value(objref)):
+                category = catelog.as_category()
+                entry = catelog.as_entry()
+                if entry is None:
+                    singleidx.setdefault(category, {}).setdefault(
+                        catelog, set()
+                    ).update(objids)
                 else:
-                    desc = ''
-                desc = strip_rst_markups(desc)  # strip rst markups
-                desc = ''.join(
-                    [ln for ln in desc.split('\n') if ln.strip()]
-                )  # strip NEWLINE
-                desc = desc[:50] + '…' if len(desc) > 50 else desc  # shorten
-                # 0: Normal entry
-                entries.append(IndexEntry(name, 0, docname, anchor, extra, '', desc))
-
-        # sort by first letter
-        sorted_content = sorted(content.items())
+                    dualidx.setdefault(category, {}).setdefault(entry, {}).setdefault(
+                        catelog, set()
+                    ).update(objids)
+
+        content: dict[Classif, list[IndexEntry]] = {}  # category →  entries
+        for category, entries in self._sort_by_catelog(singleidx):
+            index_entries = content.setdefault(category, [])
+            for category, objids in self._sort_by_catelog(entries):
+                for objid in objids:
+                    entry = self._generate_index_entry(objid, docnames, category)
+                    if entry is None:
+                        continue
+                    index_entries.append(entry)
+
+        for category, entries in self._sort_by_catelog(dualidx):
+            index_entries = content.setdefault(category, [])
+            for entry, subentries in self._sort_by_catelog(entries):
+                index_entries.append(self._generate_empty_index_entry(entry))
+                for subentry, objids in self._sort_by_catelog(subentries):
+                    for objid in objids:
+                        entry = self._generate_index_entry(objid, docnames, subentry)
+                        if entry is None:
+                            continue
+                        index_entries.append(entry)
+
+        # sort by category, and map classif -> str
+        sorted_content = [
+            (classif.leaf, entries)
+            for classif, entries in self._sort_by_catelog(content)
+        ]
 
         return sorted_content, False
 
+    def _generate_index_entry(
+        self, objid: str, ignore_docnames: Iterable[str] | None, category: Classif
+    ) -> IndexEntry | None:
+        docname, anchor, obj = self.domain.data['objects'][self.schema.objtype, objid]
+        if ignore_docnames and docname not in ignore_docnames:
+            return None
+        name = self.schema.title_of(obj) or objid
+        subtype = category.index_entry_subtype
+        extra = category.leaf
+        objcont = self.schema.content_of(obj)
+        if isinstance(objcont, str):
+            desc = objcont
+        elif isinstance(objcont, list):
+            desc = '\n'.join(objcont)  # FIXME: use schema.Form
+        else:
+            desc = ''
+        desc = strip_rst_markups(desc)  # strip rst markups
+        desc = ''.join([ln for ln in desc.split('\n') if ln.strip()])  # strip NEWLINE
+        desc = desc[:50] + '…' if len(desc) > 50 else desc  # shorten
+        return IndexEntry(
+            name,  # the name of the index entry to be displayed
+            subtype,  # the sub-entry related type
+            docname,  # docname where the entry is located
+            anchor,  # anchor for the entry within docname
+            extra,  # extra info for the entry
+            '',  # qualifier for the description
+            desc,  # description for the entry
+        )
+
+    def _generate_empty_index_entry(self, category: Classif) -> IndexEntry:
+        return IndexEntry(
+            category.leaf, category.index_entry_subtype, '', '', '', '', ''
+        )
+
+    _T = TypeVar('_T')
+
+    def _sort_by_catelog(self, d: dict[Classif, _T]) -> list[tuple[Classif, _T]]:
+        """Helper for sorting dict items by Category."""
+        return self.classifier.sort(d.items(), lambda x: x[0])
+
 
 def strip_rst_markups(rst: str) -> str:
     """Strip markups and newlines in rST.