Introduce support for jinja2-based templates.

.gitignore

@@ -1,4 +1,5 @@
 # Byte-compiled / optimized / DLL files
+__jinja__/
 __pycache__/
 *.py[cod]
 *$py.class

local-requirements.txt

@@ -7,4 +7,5 @@ autopep8;python_version >= "3"
 # NOTE: paramiko-3.0.0 dropped python2 and python3.6 support
 paramiko;python_version >= "3.7"
 paramiko<3;python_version < "3.7"
+polib
 werkzeug

mig/lib/templates/__init__.py

@@ -0,0 +1,169 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+#
+# --- BEGIN_HEADER ---
+#
+# base - shared base helper functions
+# Copyright (C) 2003-2024  The MiG Project by the Science HPC Center at UCPH
+#
+# This file is part of MiG.
+#
+# MiG is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# MiG is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+#
+# -- END_HEADER ---
+#
+
+import errno
+from jinja2 import meta as jinja2_meta, select_autoescape, Environment, \
+    FileSystemLoader, FileSystemBytecodeCache
+import os
+import weakref
+
+from mig.shared.compat import PY2
+from mig.shared.defaults import MIG_BASE
+
+
+if PY2:
+    from chainmap import ChainMap
+else:
+    from collections import ChainMap
+
+TEMPLATES_DIR = os.path.join(MIG_BASE, 'mig/assets/templates')
+TEMPLATES_CACHE_DIR = os.path.join(TEMPLATES_DIR, '__jinja__')
+
+_all_template_dirs = [
+    TEMPLATES_DIR,
+]
+
+
+def _clear_global_store():
+    global _global_store
+    _global_store = None
+
+
+def cache_dir():
+    return TEMPLATES_CACHE_DIR
+
+
+def _global_template_dirs():
+    return _all_template_dirs
+
+
+class _BonundTemplate:
+    def __init__(self, template, template_args):
+        self.tmpl = template
+        self.args = template_args
+
+    def render(self):
+        return self.tmpl.render(**self.args)
+
+
+class _FormatContext:
+    def __init__(self, configuration):
+        self.output_format = None
+        self.configuration = configuration
+        self.script_map = {}
+        self.style_map = {}
+
+    def __getitem__(self, key):
+        return self.__dict__[key]
+
+    def __iter__(self):
+        return iter(self.__dict__)
+
+    def extend(self, template, template_args):
+        return _BonundTemplate(template, ChainMap(template_args, self))
+
+
+class TemplateStore:
+    def __init__(self, template_dirs, cache_dir=None, extra_globals=None):
+        assert cache_dir is not None
+
+        self._cache_dir = cache_dir
+        self._template_globals = extra_globals
+        self._template_environment = Environment(
+            extensions=['jinja2.ext.i18n'],
+            loader=FileSystemLoader(template_dirs),
+            bytecode_cache=FileSystemBytecodeCache(cache_dir, '%s'),
+            autoescape=select_autoescape()
+        )
+        self._template_environment.install_null_translations()
+
+    @property
+    def cache_dir(self):
+        return self._cache_dir
+
+    @property
+    def context(self):
+        return self._template_globals
+
+    def _get_template(self, template_fqname):
+        return self._template_environment.get_template(template_fqname)
+
+    def _get_template_source(self, template_fqname):
+        template = self._template_environment.get_template(template_fqname)
+        with open(template.filename) as f:
+            return f.read()
+
+    def grab_template(self, template_name, template_group, output_format, template_globals=None, **kwargs):
+        template_fqname = "%s_%s.%s.jinja" % (
+            template_group, template_name, output_format)
+        return self._template_environment.get_template(template_fqname, globals=template_globals)
+
+    def list_templates(self):
+        return [t for t in self._template_environment.list_templates() if t.endswith('.jinja')]
+
+    def extract_translations(self, template_fqname):
+        template_source = self._get_template_source(template_fqname)
+        return self._template_environment.extract_translations(template_source)
+
+    def extract_variables(self, template_fqname):
+        template_source = self._get_template_source(template_fqname)
+        ast = self._template_environment.parse(template_source)
+        return jinja2_meta.find_undeclared_variables(ast)
+
+    @staticmethod
+    def populated(template_dirs, cache_dir=None, context=None):
+        assert cache_dir is not None
+
+        try:
+            os.mkdir(cache_dir)
+        except OSError as direxc:
+            if direxc.errno != errno.EEXIST:  # FileExistsError
+                raise
+
+        store = TemplateStore(
+            template_dirs, cache_dir=cache_dir, extra_globals=context)
+
+        for template_fqname in store.list_templates():
+            store._get_template(template_fqname)
+
+        return store
+
+
+def init_global_templates(configuration, _templates_dirs=_global_template_dirs):
+    _context = configuration.context()
+
+    try:
+        return configuration.context(namespace='templates')
+    except KeyError as exc:
+        pass
+
+    store = TemplateStore.populated(
+        _templates_dirs(),
+        cache_dir=cache_dir(),
+        context=_FormatContext(configuration)
+    )
+    return configuration.context_set(store, namespace='templates')

mig/lib/templates/__main__.py

@@ -0,0 +1,120 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+#
+# --- BEGIN_HEADER ---
+#
+# base - shared base helper functions
+# Copyright (C) 2003-2024  The MiG Project by the Science HPC Center at UCPH
+#
+# This file is part of MiG.
+#
+# MiG is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# MiG is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+#
+# -- END_HEADER ---
+#
+
+from __future__ import print_function
+import os
+import sys
+
+from mig.lib.templates import init_global_templates, _global_template_dirs
+from mig.shared.compat import SimpleNamespace
+from mig.shared.conf import get_configuration_object
+
+
+def warn(message):
+    print(message, sys.stderr, True)
+
+
+def main(args, _print=print):
+    configuration = get_configuration_object(
+        config_file=args.config_file, skip_log=True, disable_auth_log=True)
+
+    if args.tmpldir:
+        def _template_dirs(): return [args.tmpldir]
+    else:
+        _template_dirs = _global_template_dirs
+
+    template_store = init_global_templates(
+        configuration, _templates_dirs=_template_dirs)
+
+    command = args.command
+    if command == 'show':
+        print(template_store.list_templates())
+    elif command == 'prime':
+        try:
+            os.mkdir(template_store.cache_dir)
+        except FileExistsError:
+            pass
+
+        for template_fqname in template_store.list_templates():
+            template_store._get_template(template_fqname)
+    elif command == 'translations':
+        try:
+            os.mkdir(template_store.cache_dir)
+        except FileExistsError:
+            pass
+
+        for template_fqname in template_store.list_templates():
+            extracted_tuples = list(
+                template_store.extract_translations(template_fqname))
+            if len(extracted_tuples) == 0:
+                continue
+            _print("<%s>" % (template_fqname,))
+            for _, __, string in template_store.extract_translations(template_fqname):
+                _print('  "%s"' % (string,))
+            _print("</%s>" % (template_fqname,))
+
+
+    elif command == 'translations-mo':
+        try:
+            os.mkdir(template_store.cache_dir)
+        except FileExistsError:
+            pass
+
+        import polib
+        pofile = polib.POFile()
+        for template_fqname in template_store.list_templates():
+            for _, __, string in template_store.extract_translations(template_fqname):
+                poentry = polib.POEntry(msgid=string, msgstr=string)
+                pofile.append(poentry)
+
+        _print(pofile)
+
+    elif command == 'vars':
+        for template_ref in template_store.list_templates():
+            _print("<%s>" % (template_ref,))
+            for var in template_store.extract_variables(template_ref):
+                _print("  {{%s}}" % (var,))
+            _print("</%s>" % (template_ref,))
+    else:
+        raise RuntimeError("unknown command: %s" % (command,))
+
+
+if __name__ == '__main__':
+    import argparse
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument('-c', dest='config_file', default=None)
+    parser.add_argument('--template-dir', dest='tmpldir')
+    parser.add_argument('command')
+    args = parser.parse_args()
+
+    try:
+        main(args)
+        sys.exit(0)
+    except Exception as exc:
+        warn(str(exc))
+        sys.exit(1)

mig/shared/configuration.py

@@ -1,3 +1,3 @@
 #!/usr/bin/python
 # -*- coding: utf-8 -*-
 #
@@ -20,7 +20,7 @@
 #
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 #
 # -- END_HEADER ---
 #
@@ -85,7 +85,7 @@
     if val.find('::') == -1:
         # logger.debug("nothing to expand in %r" % val)
         return val
     #logger.debug("expand any ENV and FILE content in %r" % val)
     env_pattern = "[a-zA-Z][a-zA-Z0-9_]+"
     cache_pattern = file_pattern = "[^ ]+"
     expanded = val
@@ -716,6 +716,7 @@
                  disable_auth_log=False):
         self.config_file = config_file
         self.mig_server_id = None
+        self._context = None
 
         configuration_options = copy.deepcopy(_CONFIGURATION_DEFAULTS)
 
@@ -727,6 +728,23 @@
                                disable_auth_log=disable_auth_log,
                                _config_file=config_file)
 
+    def context(self, namespace=None):
+        if self._context is None:
+            self._context = {}
+        if namespace is None:
+            return self._context
+        try:
+            return self._context[namespace]
+        except KeyError:
+            raise
+
+    def context_set(self, value, namespace=None):
+        assert namespace is not None
+
+        context = self.context()
+        context[namespace] = value
+        return value
+
     def reload_config(self, verbose, skip_log=False, disable_auth_log=False,
                       _config_file=None):
         """Re-read and parse configuration file. Optional skip_log arg

mig/shared/objecttypes.py

@@ -1,3 +1,3 @@
 #!/usr/bin/python
 # -*- coding: utf-8 -*-
 
@@ -28,9 +28,13 @@
 
 """ Defines valid objecttypes and provides a method to verify if an object is correct """
 
+from mig.lib.templates import init_global_templates
+
+
 start = {'object_type': 'start', 'required': [], 'optional': ['headers'
                                                               ]}
 end = {'object_type': 'end', 'required': [], 'optional': []}
+template = {'object_type': 'template'}
 timing_info = {'object_type': 'timing_info', 'required': [],
                'optional': []}
 title = {'object_type': 'title', 'required': ['text'],
@@ -396,6 +400,7 @@
 valid_types_list = [
     start,
     end,
+    template,
     timing_info,
     title,
     text,
@@ -499,6 +504,8 @@
     image_settings_list,
 ]
 
+base_template_required = set(('template_name', 'template_group', 'template_args,'))
+
 # valid_types_dict = {"title":title, "link":link, "header":header}
 
 # autogenerate dict based on list. Dictionary access is prefered to allow
@@ -510,8 +517,8 @@
 
 valid_types_dict = {}
 for valid_type in valid_types_list:
     valid_types_dict[valid_type['object_type']] = \
         eval(valid_type['object_type'])
 
 
 def get_object_type_info(object_type_list):
@@ -539,8 +546,8 @@
     return out
 
 
-def validate(input_object):
-    """ validate input_object """
+def validate(input_object, configuration=None):
+    """ validate presented objects against their definitions """
 
     if not type(input_object) == type([]):
         return (False, 'validate object must be a list' % ())
@@ -560,6 +567,19 @@
 
             this_object_type = obj['object_type']
             valid_object_type = valid_types_dict[this_object_type]
+
+            if this_object_type == 'template':
+                # the required keys stuff below is not applicable to templates
+                # because templates know what they need in terms of data thus
+                # are self-documenting - use this fact to perform validation
+                #template_ref = "%s_%s.html" % (obj['template_group'], )
+                store = init_global_templates(configuration)
+                template = store.grab_template(obj['template_name'], obj['template_group'], 'html')
+                valid_object_type = {
+                    'required': store.extract_variables(template)
+                }
+                obj = obj.get('template_args', None)
+
             if 'required' in valid_object_type:
                 for req in valid_object_type['required']:
                     if req not in obj: