Merge pull request #334 from hartym/develop

Docs update, new syntax.

Author: hartym

bin/update_apidoc.py

@@ -8,8 +8,8 @@
 
 
 class Module:
-    def __init__(self, name, title=None, *, automodule_options=None):
-
+    def __init__(self, name, title=None, *, automodule_options=None, append=None):
+        self.append = append
         self.name = name
         self.title = title or " ".join(map(str.title, self.name.split(".")[1:]))
         self.automodule_options = automodule_options or list()
@@ -18,20 +18,101 @@ def __repr__(self):
         return "<{} ({})>".format(self.title, self.name)
 
     def asdict(self):
-        return {"name": self.name, "title": self.title, "automodule_options": self.automodule_options}
+        return {
+            "append": self.append,
+            "automodule": True,
+            "automodule_options": self.automodule_options,
+            "name": self.name,
+            "title": self.title,
+        }
 
     def get_path(self):
         return os.path.join(__path__, apidoc_root, *self.name.split(".")) + ".rst"
 
 
+import inspect
+
+bonobo = __import__("bonobo")
+assert bonobo.__version__
+
+prefixes = {
+    "bonobo.nodes": None,
+    "bonobo._api": "bonobo",
+    "bonobo.structs.graphs": None,
+    "bonobo.execution.strategies": "bonobo",
+    "bonobo.registry": "bonobo",
+    "bonobo.util.environ": "bonobo",
+}
+api_objects = {}
+
+display_order = [("bonobo.structs.graphs", "Graphs"), ("bonobo.nodes", "Nodes"), ("bonobo", "Other top-level APIs")]
+
+for name in sorted(dir(bonobo)):
+    # ignore attributes starting by underscores
+    if name.startswith("_"):
+        continue
+    attr = getattr(bonobo, name)
+    if inspect.ismodule(attr):
+        continue
+
+    assert name in bonobo.__all__
+
+    o = getattr(bonobo, name)
+    modname = inspect.getmodule(o).__name__
+    family = None
+    family_override = None
+
+    for prefix, target in prefixes.items():
+        if modname == prefix or modname.startswith(prefix + "."):
+            family = target or prefix
+            display_name = ".".join([family, name])
+            break
+
+    if family is None:
+        raise Exception("Could not find family for {}".format(name))
+
+    api_objects.setdefault(family, [])
+    api_objects[family].append((name, o))
+
+api_content = []
+current_family = None
+for family, title in display_order:
+    if family != current_family:
+        if current_family is not None:
+            api_content.append("")
+            api_content.append("")
+        api_content.append(title)
+        api_content.append(":" * len(title))
+        api_content.append("")
+        current_family = family
+
+    for api_object in sorted(api_objects[family]):
+        object_type = "func" if inspect.isfunction(api_object[1]) else "class"
+        api_content.append("* :{}:`{}.{}` ".format(object_type, family, api_object[0]))
+
+    if family == "bonobo":
+        for api_object in sorted(api_objects[family]):
+            object_type = "function" if inspect.isfunction(api_object[1]) else "class"
+            api_content.append("")
+            api_content.append("")
+            api_content.append(api_object[0])
+            api_content.append("-" * len(api_object[0]))
+            api_content.append("")
+            api_content.append(".. auto{}:: {}.{}".format(object_type, family, api_object[0]))
+
+
+print("\n".join(api_content))
+
 modules = [
-    Module("bonobo", title="Bonobo"),
+    Module("bonobo", title="Bonobo", automodule_options=["no-members"], append="\n".join(api_content)),
     Module("bonobo.config"),
     Module("bonobo.constants", automodule_options=["no-members"]),
     Module("bonobo.execution"),
     Module("bonobo.execution.contexts"),
     Module("bonobo.execution.events"),
     Module("bonobo.execution.strategies"),
+    Module("bonobo.nodes"),
+    Module("bonobo.structs.graphs", title="Graphs"),
     Module("bonobo.util"),
 ]
 
@@ -50,8 +131,13 @@ def underlined_filter(txt, chr):
 
 :Module: :mod:`{{ name }}`
 
+{% if automodule %}
 .. automodule:: {{ name }}
 {% for opt in automodule_options %}   :{{ opt }}:{{ "\n" }}{% endfor %}
+{% endif %}
+{% if append %}
+{{ append }}
+{% endif %}
     """[
                 1:-1
             ]

bonobo/_api.py

@@ -1,6 +1,9 @@
 """
-Contains all the tools you need to get started with the framework, including (but not limited to) generic
-transformations, readers, writers, and tools for writing and executing graphs and jobs.
+The root :mod:`bonobo` package contains all the tools you need to get started with the framework, including (but not
+limited to) generic transformations, readers, writers, and tools for writing and executing graphs and jobs.
+
+Mostly, it exposes objects found in sub-packages, and although you can access them directly here, you should refer to
+the matching documentation pages when using them.
 
 All objects in this module are considered very safe to use, and backward compatibility when moving up from one version
 to another is maximal.

bonobo/nodes/__init__.py

@@ -1,3 +1,10 @@
+"""
+The :mod:`bonobo.nodes` module contains all builtin transformations that you can use out of the box in your ETL jobs.
+
+Please note that all objects from this package are also available directly through the root :mod:`bonobo` package.
+
+"""
+
 from bonobo.nodes.basics import *
 from bonobo.nodes.basics import __all__ as _all_basics
 from bonobo.nodes.filter import Filter

bonobo/structs/graphs.py

@@ -37,11 +37,20 @@ def __rshift__(self, other):
             )
 
         if len(nodes):
-            chain = self.graph.add_chain(*nodes, _input=self.last)
+            chain = self.graph.add_chain(*nodes, _input=self.last, use_existing_nodes=True)
             return GraphCursor(chain.graph, first=self.first, last=chain.output)
 
         return self
 
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        return None
+
+    def __eq__(self, other):
+        return self.graph == other.graph and self.first == other.first and self.last == other.last
+
 
 class PartialGraph:
     def __init__(self, *nodes):
@@ -73,6 +82,15 @@ def __len__(self):
     def __getitem__(self, key):
         return self.nodes[key]
 
+    def __enter__(self):
+        return self.get_cursor().__enter__()
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        return None
+
+    def __rshift__(self, other):
+        return self.get_cursor().__rshift__(other)
+
     def get_cursor(self, ref=BEGIN):
         return GraphCursor(self, last=self.index_of(ref))
 
@@ -96,6 +114,9 @@ def index_of(self, mixed):
 
         raise ValueError("Cannot find node matching {!r}.".format(mixed))
 
+    def indexes_of(self, *things):
+        return set(map(self.index_of, things))
+
     def outputs_of(self, idx_or_node, create=False):
         """Get a set of the outputs for a given node, node index or name.
         """
@@ -105,13 +126,13 @@ def outputs_of(self, idx_or_node, create=False):
             self.edges[idx_or_node] = set()
         return self.edges[idx_or_node]
 
-    def add_node(self, c, *, _name=None):
+    def add_node(self, new_node, *, _name=None):
         """Add a node without connections in this graph and returns its index.
         If _name is specified, name this node (string reference for  further usage).
         """
         idx = len(self.nodes)
         self.edges[idx] = set()
-        self.nodes.append(c)
+        self.nodes.append(new_node)
 
         if _name:
             if _name in self.named:
@@ -120,7 +141,14 @@ def add_node(self, c, *, _name=None):
 
         return idx
 
-    def add_chain(self, *nodes, _input=BEGIN, _output=None, _name=None):
+    def get_or_add_node(self, new_node, *, _name=None):
+        if new_node in self.nodes:
+            if _name is not None:
+                raise RuntimeError("Cannot name a node that is already present in the graph.")
+            return self.index_of(new_node)
+        return self.add_node(new_node, _name=_name)
+
+    def add_chain(self, *nodes, _input=BEGIN, _output=None, _name=None, use_existing_nodes=False):
         """Add `nodes` as a chain in this graph.
 
         **Input rules**
@@ -153,6 +181,8 @@ def add_chain(self, *nodes, _input=BEGIN, _output=None, _name=None):
         _first = None
         _last = None
 
+        get_node = self.get_or_add_node if use_existing_nodes else self.add_node
+
         # Sanity checks.
         if not len(nodes):
             if _input is None or _output is None:
@@ -164,7 +194,7 @@ def add_chain(self, *nodes, _input=BEGIN, _output=None, _name=None):
                 raise RuntimeError("Using add_chain(...) without nodes does not allow to use the _name parameter.")
 
         for i, node in enumerate(nodes):
-            _last = self.add_node(node, _name=_name if not i else None)
+            _last = get_node(node, _name=_name if not i else None)
 
             if _first is None:
                 _first = _last