Merge pull request #348 from moremoban/dev

chfw · web-flow · commit e456f2aea8fb · 2019-11-10T10:43:22.000Z
:eggs: 🎡 release 0.6.6
diff --git a/.moban.cd/changelog.yml b/.moban.cd/changelog.yml
@@ -2,7 +2,13 @@ name: moban
 organisation: moremoban
 releases:
   - changes:
-      - action: Updated
+      - action: Added
+        details:
+          - "support moban file inheritance. one base moban file and child repos can inherit and override"
+    date: 10.11.2019
+    version: 0.6.6
+  - changes:
+      - action: Added
         details:
           - "`#335`: support intermediate targets in moban file"
     date: 13.10.2019
diff --git a/.moban.cd/moban.yml b/.moban.cd/moban.yml
@@ -4,9 +4,9 @@ organisation: moremoban
 author: C. W.
 contact: wangc_2011@hotmail.com
 license: MIT
-version: 0.6.5
-current_version: 0.6.5
-release: 0.6.5
+version: 0.6.6
+current_version: 0.6.6
+release: 0.6.6
 branch: master
 master: index
 command_line_interface: "moban"
diff --git a/.moban.d/custom_conf.py.jj2 b/.moban.d/custom_conf.py.jj2
@@ -1,3 +0,0 @@
-{% include "conf.py.jj2" %}
-
-master_doc = "index"
diff --git a/CHANGELOG.rst b/CHANGELOG.rst
@@ -1,10 +1,18 @@
 Change log
 ================================================================================
 
+0.6.6 - 10.11.2019
+--------------------------------------------------------------------------------
+
+**Added**
+
+#. support moban file inheritance. one base moban file and child repos can
+   inherit and override
+
 0.6.5 - 13.10.2019
 --------------------------------------------------------------------------------
 
-**Updated**
+**Added**
 
 #. `#335 <https://github.com/moremoban/moban/issues/335>`_: support intermediate
    targets in moban file
diff --git a/README.rst b/README.rst
@@ -11,6 +11,12 @@ moban - 模板 Any template, any data in any location
 .. image:: https://codecov.io/gh/moremoban/moban/branch/master/graph/badge.svg
     :target: https://codecov.io/gh/moremoban/moban
 
+.. image:: https://badge.fury.io/py/moban.svg
+   :target: https://pypi.org/project/moban
+
+.. image:: https://pepy.tech/badge/moban
+   :target: https://pepy.tech/project/moban
+
 .. image:: https://readthedocs.org/projects/moban/badge/?version=latest
     :target: http://moban.readthedocs.org/en/latest/
 
@@ -39,6 +45,18 @@ road map.
 
 All use cases are documented `here <http://moban.readthedocs.org/en/latest/#tutorial>`_
 
+Support
+================================================================================
+
+If you like moban, please support me on `github <https://github.com/sponsors/chfw>`_,
+`patreon <https://www.patreon.com/bePatron?u=5537627>`_
+or `bounty source <https://salt.bountysource.com/teams/chfw-pyexcel>`_ to maintain
+the project and develop it further.
+
+With your financial support, I will be able to invest
+a little bit more time in coding, documentation and writing interesting extensions.
+
+
 Installation
 ================================================================================
 You can install it via pip:
diff --git a/docs/README.rst b/docs/README.rst
@@ -25,6 +25,7 @@ This section covers the use cases for moban. Please check them out individually.
 #. `Template files in a zip or tar`_
 #. `Template copying from a zip to a zip`_
 #. `Intermeidate targets`_
+#. `Mobanfile inheritance`_
    
 .. _Jinja2 command line: level-1-jinja2-cli
 .. _Template inheritance: level-2-template-inheritance
@@ -48,3 +49,5 @@ This section covers the use cases for moban. Please check them out individually.
 .. _Template files in a zip or tar: level-20-templates-configs-in-zip-or-tar
 .. _Template copying from a zip to a zip: level-21-copy-templates-into-an-alien-file-system
 .. _Intermeidate targets: level-22-intermediate-targets
+.. _Mobanfile inheritance: level-23-inherit-organisational-moban-file
+
diff --git a/docs/conf.py b/docs/conf.py
@@ -25,9 +25,9 @@
 copyright = '2017-2019 Onni Software Ltd.'
 author = 'C. W.'
 # The short X.Y version
-version = '0.6.5'
+version = '0.6.6'
 # The full version, including alpha/beta/rc tags
-release = '0.6.5'
+release = '0.6.6'
 
 # -- General configuration ---------------------------------------------------
 
@@ -82,5 +82,4 @@
 ]
 intersphinx_mapping.update({
 })
-
 master_doc = "index"
diff --git a/docs/index.rst b/docs/index.rst
@@ -37,6 +37,8 @@ examples folder.
    level-20-templates-configs-in-zip-or-tar/README.rst
    level-21-copy-templates-into-an-alien-file-system/README.rst
    level-22-intermediate-targets/README.rst
+   level-23-inherit-organisational-moban-file/README.rst
+
 
 For more complex use case, please look at `its usage in pyexcel project <http://pyexcel.readthedocs.io/en/latest/guide.html>`_
 
diff --git a/docs/level-22-intermediate-targets/README.rst b/docs/level-22-intermediate-targets/README.rst
diff --git a/docs/level-23-inherit-organisational-moban-file/.moban.yml b/docs/level-23-inherit-organisational-moban-file/.moban.yml
@@ -0,0 +1,3 @@
+overrides: parent.moban.yaml
+targets:
+  - output_a: template_a.jj2
diff --git a/docs/level-23-inherit-organisational-moban-file/README.rst b/docs/level-23-inherit-organisational-moban-file/README.rst
@@ -0,0 +1,10 @@
+Level 23: moban file inheritance
+================================================================================
+
+It is a bit tedious to repeat a few common configuration in moban file. Why not
+create a parent moban file? Then allow child project to deviate from.
+
+The answer is to use 'overrides' in `.moban.yaml`, so called moban file.
+
+`overrides` could over ride any data file format in any location in theory. And
+it support override a specific key set.
diff --git a/docs/level-23-inherit-organisational-moban-file/data.yaml b/docs/level-23-inherit-organisational-moban-file/data.yaml
@@ -0,0 +1,2 @@
+template_a: I am template a
+template_b: I am template b
diff --git a/docs/level-23-inherit-organisational-moban-file/parent.moban.yaml b/docs/level-23-inherit-organisational-moban-file/parent.moban.yaml
@@ -0,0 +1,4 @@
+configuration:
+  configuration: data.yaml
+targets:
+  - output_b: template_b.jj2
diff --git a/docs/level-23-inherit-organisational-moban-file/template_a.jj2 b/docs/level-23-inherit-organisational-moban-file/template_a.jj2
@@ -0,0 +1 @@
+{{template_a}}
diff --git a/docs/level-23-inherit-organisational-moban-file/template_b.jj2 b/docs/level-23-inherit-organisational-moban-file/template_b.jj2
@@ -0,0 +1 @@
+{{template_b}}
diff --git a/moban/_version.py b/moban/_version.py
@@ -1,2 +1,2 @@
-__version__ = "0.6.5"
+__version__ = "0.6.6"
 __author__ = "C. W."
diff --git a/moban/core/data_loader.py b/moban/core/data_loader.py
@@ -1,6 +1,7 @@
 from collections import OrderedDict
 
 from lml.plugin import PluginManager
+from ruamel.yaml.comments import CommentedSeq
 
 from moban import constants
 from moban.externals import file_system
@@ -27,17 +28,21 @@ def get_data(self, file_name):
 
 
 def load_data(base_dir, file_name):
+
     abs_file_path = search_file(base_dir, file_name)
     data = LOADER.get_data(abs_file_path)
     if data is not None:
         parent_data = OrderedDict()
-        if base_dir and constants.LABEL_OVERRIDES in data:
+        if constants.LABEL_OVERRIDES in data:
             overrides = data.pop(constants.LABEL_OVERRIDES)
             if not isinstance(overrides, list):
                 overrides = [overrides]
             for parent_file in overrides:
                 file_name, key = parent_file, None
-                if ":" in parent_file:
+                results = match_fs_url(parent_file)
+                if results:
+                    file_name, key = results
+                elif ":" in parent_file and "://" not in parent_file:
                     file_name, key = parent_file.split(":")
                 child_data = load_data(base_dir, file_name)
                 if data:
@@ -69,6 +74,12 @@ def merge(left, right):
                 left[key] = value
             else:
                 left[key] = merge(left[key], value)
+    else:
+        both_list_alike = (
+            isinstance(left, CommentedSeq) and isinstance(right, CommentedSeq)
+        ) or (isinstance(left, list) and isinstance(right, list))
+        if both_list_alike:
+            left.extend(right)
     return left
 
 
@@ -86,3 +97,11 @@ def search_file(base_dir, file_name):
         else:
             raise IOError(constants.ERROR_DATA_FILE_ABSENT % the_file)
     return the_file
+
+
+def match_fs_url(file_name):
+    import re
+
+    results = re.match("(.*://.*):(.*)", file_name)
+    if results:
+        return (results.group(1), results.group(2))
diff --git a/moban/main.py b/moban/main.py
@@ -14,12 +14,19 @@
 import logging.config
 from collections import defaultdict
 
+from ruamel.yaml import YAML
+
 from moban import constants, exceptions
 from moban.core import ENGINES, plugins, hashstore, mobanfile, data_loader
 from moban._version import __version__
 from moban.externals import reporter, file_system
 from moban.program_options import OPTIONS
 
+try:
+    from cStringIO import StringIO
+except ImportError:
+    from io import StringIO
+
 LOG = logging.getLogger()
 LOG_LEVEL = [logging.WARNING, logging.INFO, logging.DEBUG]
 
@@ -173,6 +180,10 @@ def handle_moban_file(moban_file, options):
     act upon default moban file
     """
     moban_file_configurations = data_loader.load_data(None, moban_file)
+    yaml = YAML(typ="rt")
+    dumped_yaml = StringIO()
+    yaml.dump(moban_file_configurations, dumped_yaml)
+    LOG.info(dumped_yaml.getvalue())
     if moban_file_configurations is None:
         raise exceptions.MobanfileGrammarException(
             constants.ERROR_INVALID_MOBAN_FILE % moban_file
diff --git a/mobanfile b/mobanfile
@@ -7,7 +7,7 @@ targets:
   - setup.py: moban_setup.py.jj2
   - moban/__init__.py: __init__.py.jj2
   - moban/_version.py: _version.py.jj2
-  - docs/conf.py: custom_conf.py.jj2
+  - docs/conf.py: conf.py.jj2
   - .travis.yml: moban_travis.yml.jj2
   - requirements.txt: requirements.txt.jj2
   - .gitignore: moban_gitignore.jj2
diff --git a/setup.py b/setup.py
@@ -38,7 +38,7 @@
 
 NAME = "moban"
 AUTHOR = "C. W."
-VERSION = "0.6.5"
+VERSION = "0.6.6"
 EMAIL = "wangc_2011@hotmail.com"
 LICENSE = "MIT"
 ENTRY_POINTS = {
@@ -50,7 +50,7 @@
     "Yet another jinja2 cli command for static text generation"
 )
 URL = "https://github.com/moremoban/moban"
-DOWNLOAD_URL = "%s/archive/0.6.5.tar.gz" % URL
+DOWNLOAD_URL = "%s/archive/0.6.6.tar.gz" % URL
 FILES = ["README.rst", "CONTRIBUTORS.rst", "CHANGELOG.rst"]
 KEYWORDS = [
     "python",
@@ -97,8 +97,8 @@
 }
 # You do not need to read beyond this line
 PUBLISH_COMMAND = "{0} setup.py sdist bdist_wheel upload -r pypi".format(sys.executable)
-GS_COMMAND = ("gs moban v0.6.5 " +
-              "Find 0.6.5 in changelog for more details")
+GS_COMMAND = ("gs moban v0.6.6 " +
+              "Find 0.6.6 in changelog for more details")
 NO_GS_MESSAGE = ("Automatic github release is disabled. " +
                  "Please install gease to enable it.")
 UPLOAD_FAILED_MSG = (
diff --git a/tests/data_loaders/__init__.py b/tests/data_loaders/__init__.py
diff --git a/tests/data_loaders/test_json_loader.py b/tests/data_loaders/test_json_loader.py
@@ -1,7 +1,7 @@
 import fs.path
 from nose.tools import eq_
 
-from moban.data_loaders.json_loader import open_json
+from moban.plugins.json_loader import open_json
 
 
 def test_open_json():
diff --git a/tests/data_loaders/test_merge_dict.py b/tests/data_loaders/test_merge_dict.py
@@ -1,4 +1,7 @@
-from moban.data_loaders.manager import merge
+from nose.tools import eq_
+from ruamel.yaml import YAML
+
+from moban.core.data_loader import merge
 
 
 def test_simple_union():
@@ -34,3 +37,30 @@ def test_three_level_conflict():
     default = {"L1": {"L2": {"L3": "Hi"}}}
     merged = merge(user, default)
     assert merged == {"L1": {"L2": {"L3": "World"}}}
+
+
+def test_merge_value_as_list():
+    user = {"L1": ["a", "b"]}
+    default = {"L1": ["c", "d"]}
+    merged = merge(user, default)
+    assert merged == {"L1": ["a", "b", "c", "d"]}
+
+
+def test_merge_value_as_list_in_yaml():
+    yaml = YAML(typ="rt")
+    user = yaml.load(
+        """
+L1:
+  - a
+  - b
+"""
+    )
+    default = yaml.load(
+        """
+L1:
+  - c
+  - d
+"""
+    )
+    merged = merge(user, default)
+    eq_(merged, {"L1": ["a", "b", "c", "d"]})
diff --git a/tests/data_loaders/test_overrides.py b/tests/data_loaders/test_overrides.py
@@ -2,7 +2,8 @@
 
 from nose.tools import eq_
 
-from moban.data_loaders.manager import load_data
+from moban.main import load_engine_factory_and_engines
+from moban.core.data_loader import load_data
 
 
 def test_overrides_a_list_of_config_files():
@@ -79,3 +80,10 @@ def test_overrides_nested_keys():
     }
 
     eq_(dict(actual), expected)
+
+
+def test_overrides_fs_url():
+    load_engine_factory_and_engines()
+    base_dir = os.path.join("tests", "fixtures")
+    actual = load_data(None, os.path.join(base_dir, "override_fs_url.yaml"))
+    assert "requires" in actual
diff --git a/tests/data_loaders/test_yaml_loader.py b/tests/data_loaders/test_yaml_loader.py
@@ -1,8 +1,8 @@
 import fs.path
 from nose.tools import eq_, raises
 
-from moban.data_loaders.yaml import open_yaml
-from moban.data_loaders.manager import load_data
+from moban.core.data_loader import load_data
+from moban.plugins.yaml_loader import open_yaml
 
 
 def test_simple_yaml():
diff --git a/tests/fixtures/override_fs_url.yaml b/tests/fixtures/override_fs_url.yaml
@@ -0,0 +1,3 @@
+overrides: "git://github.com/moremoban/moban!/tests/fixtures/.moban.yml"
+targets:
+  - my: test
diff --git a/tests/test_docs.py b/tests/test_docs.py
@@ -317,6 +317,14 @@ def test_level_22_intermediate_targets(self):
         folder = "level-22-intermediate-targets"
         self.run_moban(["moban"], folder, [("final", expected)])
 
+    def test_level_23_inherit_parent_moban_file(self):
+        folder = "level-23-inherit-organisational-moban-file"
+        self.run_moban(
+            ["moban"],
+            folder,
+            [("output_a", "I am template a"), ("output_b", "I am template b")],
+        )
+
     def test_misc_1(self):
         expected = "test file\n"
 

Original file line number	Diff line number	Diff line change
`@@ -1,3 +0,0 @@`
`1`		`-{% include "conf.py.jj2" %}`
`2`		`-`
`3`		`-master_doc = "index"`
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+overrides: parent.moban.yaml`
	`2`	`+targets:`
	`3`	`+ - output_a: template_a.jj2`