diff --git a/documentation/Installation.md b/documentation/Installation.md index b7a3f36b9..8b657f1a1 100644 --- a/documentation/Installation.md +++ b/documentation/Installation.md @@ -5811,7 +5811,7 @@ Application of the list merge strategy is allowed in the following sections: There are settings in the configuration file that borrow their contents from the settings of the other sections. To avoid any duplication of the settings, the mechanism of dynamic variables is used. -This mechanism allows you to specify a link to one variable to another. +This mechanism allows you to specify a link from one variable to another. For example, the following parameters: @@ -5917,7 +5917,7 @@ Dynamic variables have some limitations that should be considered when working w kubemarine_variable: '{{ values.custom_variable }}' ``` * The start pointer of the Jinja2 template must be inside a pair of single or double quotes. The `{{` or `{%` out of quotes leads to a parsing error of the yaml file. -* The variable cannot refer to itself. It does not lead to any result, but it slows down the compilation process. +* The variable cannot refer to itself. * The variables cannot mutually refer to each other. For example, the following configuration: ```yaml @@ -5926,24 +5926,27 @@ Dynamic variables have some limitations that should be considered when working w variable_two: '{{ section.variable_one }}' ``` - This leads to the following result: - - ```yaml - section: - variable_one: '{{ section.variable_one }}' - variable_two: '{{ section.variable_one }}' - ``` - The variables copy each other, but since none of them lead to any result, there is a cyclic link to one of them. + This leads to the "cyclic reference" error. #### Jinja2 Expressions Escaping -Inventory strings can have strings containing characters that Jinja2 considers as their expressions. For example, if you specify a golang template. To avoid rendering errors for such expressions, it is possible to wrap them in exceptions `{% raw %}``{% endraw %}`. +Inventory strings can have strings containing characters that Jinja2 considers as their expressions. +For example, if you specify a golang template. +To avoid rendering errors for such expressions, it is possible to escape the special characters. For example: +```yaml +authority: '{{ "{{ .Name }}" }} 3600 IN SOA' +``` + +or + ```yaml authority: '{% raw %}{{ .Name }}{% endraw %} 3600 IN SOA' ``` +For more information, refer to https://jinja.palletsprojects.com/en/3.1.x/templates/#escaping + ### Environment Variables Kubemarine supports environment variables inside the following places: diff --git a/kubemarine/core/defaults.py b/kubemarine/core/defaults.py index 34875a7ed..c8232f128 100755 --- a/kubemarine/core/defaults.py +++ b/kubemarine/core/defaults.py @@ -11,8 +11,9 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import json import re -from typing import Optional, Dict, Any, Tuple, List, Callable, Union +from typing import Optional, Dict, Any, Tuple, List, Callable, Union, Sequence, cast, Type import yaml @@ -20,6 +21,7 @@ from kubemarine.core.errors import KME from kubemarine import jinja, keepalived, haproxy, controlplane, kubernetes, thirdparties from kubemarine.core import utils, static, log, os +from kubemarine.core.proxytypes import Primitive, Index, Node from kubemarine.core.yaml_merger import default_merger from kubemarine.cri.containerd import contains_old_format_properties @@ -53,8 +55,6 @@ invalid_node_name_regex = re.compile("[^a-z-.\\d]", re.M) -escaped_expression_regex = re.compile('({%[\\s*|]raw[\\s*|]%}.*?{%[\\s*|]endraw[\\s*|]%})', re.M) -jinja_query_regex = re.compile("{{ .* }}", re.M) @enrichment(EnrichmentStage.LIGHT, procedures=['add_node']) @@ -469,41 +469,33 @@ def _merge_inventory(cluster: KubernetesCluster, base: dict) -> dict: @enrichment(EnrichmentStage.LIGHT) def compile_connections(cluster: KubernetesCluster) -> None: - return _compile_inventory(cluster, inject_globals=False) + return _compile_inventory(cluster, light=True) @enrichment(EnrichmentStage.FULL) def compile_inventory(cluster: KubernetesCluster) -> None: - return _compile_inventory(cluster, inject_globals=True) + return _compile_inventory(cluster, light=False) -def _compile_inventory(cluster: KubernetesCluster, *, inject_globals: bool) -> None: +def _compile_inventory(cluster: KubernetesCluster, *, light: bool) -> None: inventory = cluster.inventory - # convert references in yaml to normal values - iterations = 100 - root = utils.deepcopy_yaml(inventory) - if inject_globals: - root['globals'] = static.GLOBALS - root['env'] = os.Environ() - while iterations > 0: + extra: Dict[str, Any] = {'env': os.Environ()} + if not light: + extra['globals'] = static.GLOBALS - cluster.log.verbose('Inventory is not rendered yet...') - inventory = compile_object(cluster.log, inventory, root) - - temp_dump = yaml.dump(inventory) + if light: + # Management of primitive values is currently not necessary for LIGHT stage. + env = Environment(cluster.log, inventory, recursive_compile=True, recursive_extra=extra) + jinja.compile_node(inventory, [], env) + else: + primitives_config = _get_primitive_values_registry() + env = Environment(cluster.log, inventory, recursive_compile=True, recursive_extra=extra, + primitives_config=primitives_config) + compile_node_with_primitives(inventory, [], env, primitives_config) - # remove golang specific - temp_dump = re.sub(escaped_expression_regex, '', temp_dump.replace('\n', '')) + remove_empty_items(inventory) - # it is necessary to carry out several iterations, - # in case we have dynamic variables that reference each other - if '{{' in temp_dump or '{%' in temp_dump: - iterations -= 1 - else: - iterations = 0 - - compile_object(cluster.log, inventory, root, ignore_jinja_escapes=False) dump_inventory(cluster, cluster.context, "cluster_precompiled.yaml") @@ -518,46 +510,59 @@ def dump_inventory(cluster: KubernetesCluster, context: dict, filename: str) -> utils.dump_file(context, data, filename) -def compile_object(logger: log.EnhancedLogger, struct: Any, root: dict, ignore_jinja_escapes: bool = True) -> Any: +PrimitivesConfig = List[Tuple[List[str], Callable[[Any], Any]]] + + +def compile_node_with_primitives(struct: Union[list, dict], + path: List[Union[str, int]], + env: jinja.Environment, + primitives_config: PrimitivesConfig) -> Union[list, dict]: if isinstance(struct, list): - new_struct = [] for i, v in enumerate(struct): - struct[i] = compile_object(logger, v, root, ignore_jinja_escapes=ignore_jinja_escapes) - # delete empty list entries, which can appear after jinja compilation - if struct[i] != '': - new_struct.append(struct[i]) - struct = new_struct - elif isinstance(struct, dict): + struct[i] = compile_object_with_primitives(v, path, i, env, primitives_config) + else: for k, v in struct.items(): - struct[k] = compile_object(logger, v, root, ignore_jinja_escapes=ignore_jinja_escapes) - elif isinstance(struct, str) and jinja.is_template(struct): - struct = compile_string(logger, struct, root, ignore_jinja_escapes=ignore_jinja_escapes) + struct[k] = compile_object_with_primitives(v, path, k, env, primitives_config) return struct -def compile_string(logger: log.EnhancedLogger, struct: str, root: dict, - ignore_jinja_escapes: bool = True) -> str: - logger.verbose("Rendering \"%s\"" % struct) +def compile_object_with_primitives(struct: Union[Primitive, list, dict], + path: List[Index], index: Index, + env: jinja.Environment, + primitives_config: PrimitivesConfig) -> Union[Primitive, list, dict]: + depth = len(path) + primitives_config = choose_nested_primitives_config(primitives_config, depth, index) - def precompile(struct: str) -> str: - return compile_string(logger, struct, root) + path.append(index) + if isinstance(struct, (list, dict)): + if primitives_config: + struct = compile_node_with_primitives(struct, path, env, primitives_config) + else: + struct = jinja.compile_node(struct, path, env) + else: + if isinstance(struct, str) and jinja.is_template(struct): + struct = env.compile_string(struct, jinja.Path(path)) - if ignore_jinja_escapes: - iterator = escaped_expression_regex.finditer(struct) - struct = re.sub(escaped_expression_regex, '', struct) - struct = jinja.new(logger, recursive_compiler=precompile, precompile_filters={ - 'kubernetes_version': kubernetes.verify_allowed_version - }).from_string(struct).render(**root) + struct = convert_primitive(struct, path, primitives_config) - # TODO this does not work for {raw}{jinja}{raw}{jinja} - for match in iterator: - span = match.span() - struct = struct[:span[0]] + match.group() + struct[span[0]:] - else: - struct = jinja.new(logger, recursive_compiler=precompile).from_string(struct).render(**root) + path.pop() + return struct + + +def remove_empty_items(struct: Any) -> Any: + if isinstance(struct, list): + new_struct = [] + for v in struct: + v = remove_empty_items(v) + # delete empty list entries, which can appear after jinja compilation + if v != '': + new_struct.append(v) + struct = new_struct + elif isinstance(struct, dict): + for k, v in struct.items(): + struct[k] = remove_empty_items(v) - logger.verbose("\tRendered as \"%s\"" % struct) return struct @@ -574,15 +579,13 @@ def escape_jinja_characters_for_inventory(cluster: KubernetesCluster, obj: Any) def _escape_jinja_character(value: str) -> str: - if '{{' in value and '}}' in value and re.search(jinja_query_regex, value): - matches = re.findall(jinja_query_regex, value) - for match in matches: - # TODO: rewrite to correct way of match replacement: now it can cause "{raw}{raw}xxx.." circular bug - value = value.replace(match, '{% raw %}'+match+'{% endraw %}') + if jinja.is_template(value): + value = '{{ %s }}' % (json.JSONEncoder().encode(value),) + return value -def _get_primitive_values_registry() -> List[Tuple[List[str], Callable[[Any], Any]]]: +def _get_primitive_values_registry() -> PrimitivesConfig: return [ (['services', 'cri', 'containerdConfig', 'plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options', @@ -601,42 +604,145 @@ def _get_primitive_values_registry() -> List[Tuple[List[str], Callable[[Any], An ] -@enrichment(EnrichmentStage.FULL) -def manage_primitive_values(cluster: KubernetesCluster) -> None: - paths_func_strip = _get_primitive_values_registry() - for search_path, func in paths_func_strip: - _convert_primitive_values(cluster.inventory, [], search_path, func) +def choose_nested_primitives_config(primitives_config: PrimitivesConfig, depth: int, index: Index) -> PrimitivesConfig: + nested_config = [] + for search in primitives_config: + search_path = search[0] + if depth == len(search_path): + continue + section = search_path[depth] + if section in ('*', index): + nested_config.append(search) -def _convert_primitive_values(struct: Union[Any], path: List[Union[str, int]], - search_path: List[str], func: Callable[[Any], Any]) -> None: - depth = len(path) - section = search_path[depth] - if section == '*': - if isinstance(struct, list): - for i in reversed(range(len(struct))): - _convert_primitive_value_section(struct, i, path, search_path, func) + return nested_config - elif isinstance(struct, dict): - for k in list(struct): - _convert_primitive_value_section(struct, k, path, search_path, func) - elif isinstance(struct, dict) and section in struct: - _convert_primitive_value_section(struct, section, path, search_path, func) +def convert_primitive(struct: Primitive, path: Sequence[Index], primitives_config: PrimitivesConfig) -> Primitive: + for search_path, func in primitives_config: + if len(search_path) == len(path): + try: + struct = func(struct) + except ValueError as e: + raise ValueError(f"{str(e)} in section {utils.pretty_path(path)}") from None + return struct -def _convert_primitive_value_section(struct: Union[dict, list], section: Union[str, int], - path: List[Union[str, int]], - search_path: List[str], func: Callable[[Any], Any]) -> None: - value = struct[section] # type: ignore[index] - path.append(section) - depth = len(path) - if depth < len(search_path): - _convert_primitive_values(value, path, search_path, func) - else: - try: - struct[section] = func(value) # type: ignore[index] - except ValueError as e: - raise ValueError(f"{str(e)} in section {utils.pretty_path(path)}") from None - path.pop() +class NodePrimitives(jinja.JinjaNode): + """ + A Node that both compiles template strings and converts primitive values in the underlying `dict` or `list`. + """ + + def __init__(self, delegate: Union[dict, list], *, + path: jinja.Path, env: jinja.Environment, + primitives_config: PrimitivesConfig): + super().__init__(delegate, path=path, env=env) + self.primitives_config = primitives_config + + def _child(self, index: Index, val: Union[list, dict]) -> jinja.Node: + primitives_config = self._nested_primitives_config(index) + return self._child_type(index)(val, path=self.path + (index,), env=self.env, + primitives_config=primitives_config) + + def _child_type(self, _: Index) -> Type['NodePrimitives']: + return NodePrimitives + + def _convert(self, index: Index, val: Primitive) -> Primitive: + val = super()._convert(index, val) + + primitives_config = self._nested_primitives_config(index) + val = convert_primitive(val, self.path + (index,), primitives_config) + + return val + + def _nested_primitives_config(self, index: Index) -> PrimitivesConfig: + depth = len(self.path) + return choose_nested_primitives_config(self.primitives_config, depth, index) + + +class NodesCustomization: + """ + Customize access to the particular sections of the inventory. + """ + + # pylint: disable=no-self-argument + + def __init__(nodes) -> None: + # The classes below should customize access to the sections of the inventory, + # while preserving the global behaviour of Node implementations: NodePrimitives, JinjaNode, Node. + + class Kubeadm(Node): + def descend(self, index: Index) -> Union[Primitive, Node]: + child: Union[Primitive, Node] = super().descend(index) + + if index == 'kubernetesVersion': + kubernetes.verify_allowed_version(cast(str, child)) + + return child + + class Services(Node): + def _child_type(self, index: Index) -> Type[Node]: + if index == 'kubeadm': + return nodes.Kubeadm + + return super()._child_type(index) + + class Root(Node): + def _child_type(self, index: Index) -> Type[Node]: + if index == 'services': + return nodes.Services + + return super()._child_type(index) + + nodes.Kubeadm: Type[Node] = Kubeadm + nodes.Services: Type[Node] = Services + nodes.Root: Type[Node] = Root + + def derive(nodes, Base: Type[Node], delegate: dict, **kwargs: Any) -> Node: + nodes.Kubeadm = cast(Type[Node], type("Kubeadm", (nodes.Kubeadm, Base), {})) + nodes.Services = cast(Type[Node], type("Services", (nodes.Services, Base), {})) + nodes.Root = cast(Type[Node], type("Root", (nodes.Root, Base), {})) + + return nodes.Root(delegate, **kwargs) + + +class Environment(jinja.Environment): + """ + Environment that supports recursive compilation and on-the-fly conversion of primitive values. + + It also customizes access to the particular sections of the inventory. + """ + + def __init__(self, logger: log.EnhancedLogger, recursive_values: dict, + *, + recursive_compile: bool = False, + recursive_extra: Dict[str, Any] = None, + primitives_config: PrimitivesConfig = None): + """ + Instantiate new environment and set default filters. + + :param logger: EnhancedLogger + :param recursive_values: The render values access to which should be customized. + They may also be automatically converted and compiled if necessary. + :param recursive_compile: Flag that enables recursive compilation. + :param recursive_extra: If recursive compilation occurs, these render values are supplied to the template. + :param primitives_config: List of sections and convertors of primitive values. + """ + self.recursive_compile = recursive_compile + self.primitives_config = primitives_config + super().__init__(logger, recursive_values, recursive_extra=recursive_extra) + + def create_root(self, delegate: dict) -> Node: + kwargs = {} + Base: Type[Node] + if not self.recursive_compile: + Base = Node + else: + Base = jinja.JinjaNode + kwargs = {"path": jinja.Path(), "env": self} + if self.primitives_config is not None: + Base = NodePrimitives + kwargs = {**kwargs, "primitives_config": self.primitives_config} + + return NodesCustomization().derive(Base, delegate, **kwargs) diff --git a/kubemarine/core/errors.py b/kubemarine/core/errors.py index 18fe71eff..7326b7dca 100644 --- a/kubemarine/core/errors.py +++ b/kubemarine/core/errors.py @@ -89,7 +89,7 @@ def get_kme_dictionary() -> dict: } -class _BaseKME(RuntimeError, ABC): +class BaseKME(RuntimeError, ABC): def __init__(self, code: str): self.code = code if self.code not in get_kme_dictionary(): @@ -105,7 +105,7 @@ def __str__(self) -> str: return self.code + ": " + self.message -class KME(_BaseKME): +class KME(BaseKME): def __init__(self, code: str, **kwargs: object): self.kwargs = kwargs super().__init__(code) @@ -118,7 +118,7 @@ def _format(self) -> str: return name.format_map(self.kwargs) -class KME0006(_BaseKME): +class KME0006(BaseKME): def __init__(self, offline: List[str], inaccessible: List[str]): self.offline = offline self.inaccessible = inaccessible @@ -179,7 +179,7 @@ def error_logger(msg: object) -> None: reason = wrap_kme_exception(reason) - if isinstance(reason, _BaseKME): + if isinstance(reason, BaseKME): error_logger(reason) return diff --git a/kubemarine/core/proxytypes.py b/kubemarine/core/proxytypes.py new file mode 100644 index 000000000..204276709 --- /dev/null +++ b/kubemarine/core/proxytypes.py @@ -0,0 +1,263 @@ +import json +from abc import ABC, abstractmethod +from typing import Any, Union, Sequence, List, Iterator, Mapping, Type, Dict + +import yaml + +Index = Union[str, int] +Primitive = Union[str, int, bool, float] + + +class Proxy(ABC): + """ + Abstract class that proxies getter methods to some other container. + """ + + def __repr__(self) -> str: + return repr(self._KM_materialize()) + + @abstractmethod + def _KM_materialize(self) -> Any: + """ + Create real container that can be serialized. + This method triggers access to all the items in the proxied container. + + :return: real container that can be serialized. + """ + pass + + +class DelegatingProxy(Proxy, ABC): + """ + Abstract class that proxies getter methods to some real container. + """ + + @abstractmethod + def _KM_unsafe(self) -> Union[dict, list]: + """ + Real proxied `dict` or `list`. It can be not the same as the proxied container. + Should be used only to obtain sequence of indexes (`str` keys of `dict`, or `int` indexes of `list`). + + :return: read proxied `dict` or `list`. + """ + pass + + @abstractmethod + def _KM__getitem__(self, index: Index) -> Union[Primitive, Proxy]: + """ + Get item from `dict` by string key, or from `list` by integer index. + If the item is also a container, it should be proxied. + + :param index: key of either `str`, or `int` type. + :return: primitive value or Proxy + """ + pass + + +class MappingProxy(DelegatingProxy, Mapping[str, Any], ABC): + """ + The main facade that proxies all mapping methods to some real mapping. + """ + + def __getitem__(self, k: str) -> Any: + return self._KM__getitem__(k) + + def __len__(self) -> int: + return len(self._KM_unsafe()) + + def __iter__(self) -> Iterator[str]: + return iter(self._KM_unsafe()) + + def _KM_materialize(self) -> Any: + return dict(self) + + +class SequenceProxy(DelegatingProxy, Sequence[Any], ABC): + """ + The main facade that proxies all sequence methods to some real sequence. + """ + + def __getitem__(self, index: Union[int, slice]) -> Any: + if isinstance(index, slice): + indexes = list(range(len(self)))[index] + return SliceProxy(self, indexes) + + return self._KM__getitem__(index) + + def __len__(self) -> int: + return len(self._KM_unsafe()) + + def _KM_materialize(self) -> Any: + return list(self) + + +class SliceProxy(Proxy, Sequence[Any]): + """ + Implementation of `slice` over SequenceProxy. + """ + + def __init__(self, sequence: SequenceProxy, indexes: List[int]): + self._KM_sequence = sequence + self._KM_indexes = indexes + + def __getitem__(self, index: Union[int, slice]) -> Any: + if isinstance(index, slice): + indexes = self._KM_indexes[index] + return SliceProxy(self._KM_sequence, indexes) + + return self._KM_sequence[self._KM_indexes[index]] + + def __len__(self) -> int: + return len(self._KM_indexes) + + def _KM_materialize(self) -> Any: + return list(self) + + +class Node: + """ + Highly extendable wrapper over `dict` or `list`. + """ + + def __init__(self, delegate: Union[dict, list]): + self.delegate = delegate + + def descend(self, index: Index) -> Union[Primitive, 'Node']: + """ + The main custom implementation to access the items of real `dict` or `list` from Proxies. + + Get a child item / value by the specified `index` from the underlying container. + + If the child item is also a `dict` or `list`, wrap it with the new instance of `Node` + with (probably) its own logic to resolve items. + + If the child item is a primitive value, it can be arbitrarily converted. + + :param index: key of either `str`, or `int` type. + :return: primitive value or new child Node instance. + """ + val: Union[Primitive, list, dict] = self.delegate[index] # type: ignore[index] + if isinstance(val, (list, dict)): + return self._child(index, val) + + return val + + def _child(self, index: Index, val: Union[list, dict]) -> 'Node': + """ + Instantiate new Node wrapper over the specified child item `val`. + It can be overridden if derived class has additional constructor parameters, + but should always instantiate an instance of `_child_type`. + + :param index: key of either `str`, or `int` type. + :param val: wrapped `dict` or `list`. + :return: new child Node. + """ + return self._child_type(index)(val) + + def _child_type(self, _: Index) -> Type['Node']: + """ + Return `Node` class that should wrap a child item by the specified `index`. + + To preserve behaviour of parent `Node`, the method should be overridden to return the derived `Node` class. + + :return: `Node` class to wrap child item. + """ + return Node + + +class MutableNode(Node, ABC): + """ + A Node that can change underlying `dict` or `list` on-the-fly during access to its items. + """ + + def descend(self, index: Index) -> Union[Primitive, Node]: + child = super().descend(index) + if isinstance(child, Node): + return child + + val = self._convert(index, child) + self.delegate[index] = val # type: ignore[index] + + return val + + @abstractmethod + def _convert(self, index: Index, val: Primitive) -> Primitive: + """ + Convert primitive value before putting to the underlying container. + + :param index: key of either `str`, or `int` type. + :param val: primitive value + :return: converted value + """ + pass + + def _child_type(self, _: Index) -> Type[Node]: + return MutableNode + + +class NodeProxy(DelegatingProxy, ABC): + """ + Abstract class that proxies getter methods to an instance of `Node`. + """ + + def __init__(self, node: Node): + self._KM_node = node + self._KM_cached: Dict[Index, Union[Primitive, Proxy]] = {} + + def _KM_unsafe(self) -> Union[dict, list]: + return self._KM_node.delegate + + def _KM__getitem__(self, index: Index) -> Union[Primitive, Proxy]: + if index not in self._KM_cached: + child = self._KM_node.descend(index) + val = ((NodeMapping(child) if isinstance(child.delegate, dict) else NodeSequence(child)) + if isinstance(child, Node) + else child) + + self._KM_cached[index] = val + return val + + return self._KM_cached[index] + + +class NodeMapping(NodeProxy, MappingProxy): + """ + Mapping that proxies all getter methods to an instance of `Node`. + """ + pass + + +class NodeSequence(NodeProxy, SequenceProxy): + """ + Sequence that proxies all getter methods to an instance of `Node`. + """ + pass + + +class ProxyJSONEncoder(json.JSONEncoder): + """ + Supports serialization of any Proxy to JSON. + """ + + def default(self, o: Any) -> Any: + if isinstance(o, Proxy): + return o._KM_materialize() # pylint: disable=protected-access + + return super().default(o) + + +def proxy_representer(dumper: yaml.Dumper, data: Proxy) -> Any: + val = data._KM_materialize() # pylint: disable=protected-access + return dumper.yaml_representers[type(val)](dumper, val) + + +class ProxyDumper(yaml.Dumper): # pylint: disable=too-many-ancestors + """ + Supports serialization of any `Node` proxy to YAML. + """ + pass + + +ProxyDumper.add_representer(NodeMapping, proxy_representer) +ProxyDumper.add_representer(NodeSequence, proxy_representer) +ProxyDumper.add_representer(SliceProxy, proxy_representer) diff --git a/kubemarine/core/resources.py b/kubemarine/core/resources.py index 8dcc06e56..95370353b 100644 --- a/kubemarine/core/resources.py +++ b/kubemarine/core/resources.py @@ -433,10 +433,6 @@ def enrichment_functions(self) -> List[c.EnrichmentFunction]: kubemarine.core.defaults.remove_service_roles, # Enrichment of inventory for LIGHT stage should be finished at this step. - # Should be just after compilation, but currently not necessary for LIGHT stage. - # Sections with primitive values may be potentially split in the future if necessary for LIGHT. - kubemarine.core.defaults.manage_primitive_values, - # Validation of procedure inventory enrichment after compilation kubemarine.kubernetes.verify_version, kubemarine.admission.verify_manage_enrichment, diff --git a/kubemarine/jinja.py b/kubemarine/jinja.py index c1f525285..9a641ca83 100644 --- a/kubemarine/jinja.py +++ b/kubemarine/jinja.py @@ -12,59 +12,207 @@ # See the License for the specific language governing permissions and # limitations under the License. import base64 -from typing import Callable, Dict, Any +import json +from typing import Callable, Dict, Any, List, Union, Set, Type from urllib.parse import quote_plus import yaml import jinja2 -from kubemarine.core import log, utils +from kubemarine.core import log, utils, errors +from kubemarine.core.proxytypes import ( + Index, Primitive, NodeMapping, ProxyDumper, ProxyJSONEncoder, MutableNode, Node +) +Path = tuple FILTER = Callable[[str], Any] -def new(_: log.EnhancedLogger, *, - recursive_compiler: Callable[[str], str] = None, - precompile_filters: Dict[str, FILTER] = None) -> jinja2.Environment: - def _precompile(filter_: str, struct: str, *args: Any, **kwargs: Any) -> str: +class JinjaNode(MutableNode): + """ + A Node that compiles template strings in the underlying `dict` or `list` on-the-fly during access to its items. + """ + + def __init__(self, delegate: Union[dict, list], + *, path: Path, env: 'Environment'): + super().__init__(delegate) + self.path = path + self.env = env + + def _child(self, index: Index, val: Union[list, dict]) -> Node: + return self._child_type(index)(val, path=self.path + (index,), env=self.env) + + def _child_type(self, _: Index) -> Type['JinjaNode']: + return JinjaNode + + def _convert(self, index: Index, val: Primitive) -> Primitive: + if isinstance(val, str) and is_template(val): + val = self.env.compile_string(val, self.path + (index,)) + + return val + + +class Context(jinja2.runtime.Context): + """ + An entry point from the jinja templates to the recursively compiled sections of inventory. + """ + + def __init__(self, *args: Any, **kwargs: Any): + super().__init__(*args, **kwargs) + self.environment: Environment = self.environment + + def resolve_or_missing(self, key: str) -> Any: + # pylint: disable=protected-access + + v = super().resolve_or_missing(key) + + if v is jinja2.runtime.missing and key in self.environment._recursive_values: + return self.environment._proxy_values[key] + + return v + + +class Environment(jinja2.Environment): + """ + Jinja environment that supports recursive compilation. + """ + + context_class = Context + + def __init__(self, logger: log.EnhancedLogger, recursive_values: dict, *, recursive_extra: Dict[str, Any] = None): + """ + Instantiate new environment and set default filters. + + :param logger: EnhancedLogger + :param recursive_values: If templates access to these values, they are automatically compiled if necessary. + :param recursive_extra: If recursive compilation occurs, these render values are supplied to the template. + """ + super().__init__() + self.logger = logger + + self._recursive_values = recursive_values + self._proxy_values = NodeMapping(self.create_root(self._recursive_values)) + + self._compiled: Set[Path] = set() + self._compiling: List[Path] = [] + + self._recursive_extra = {} + if recursive_extra is not None: + self._recursive_extra = recursive_extra + + self.policies['json.dumps_function'] = jinja_tojson + self.filters['toyaml'] = jinja_toyaml + + simple_string_filters: Dict[str, FILTER] = { + 'isipv4': lambda ip: utils.isipv(ip, [4]), + 'isipv6': lambda ip: utils.isipv(ip, [6]), + 'minorversion': utils.minor_version, + 'majorversion': utils.major_version, + 'versionkey': utils.version_key, + 'b64encode': lambda s: base64.b64encode(s.encode()).decode(), + 'b64decode': lambda s: base64.b64decode(s.encode()).decode(), + 'url_quote': quote_plus + } + + for name, filter_ in simple_string_filters.items(): + def make_filter(n: str, f: FILTER) -> FILTER: + return lambda s, *args, **kwargs: f(self._check_filter(n, s, *args, *kwargs)) + + self.filters[name] = make_filter(name, filter_) + + self.tests['has_role'] = lambda node, role: role in node['roles'] + self.tests['has_roles'] = lambda node, roles: bool(set(node['roles']) & set(roles)) + + # we need these filters because rendered cluster.yaml can contain variables like + # enable: 'true' + self.filters['is_true'] = utils.strtobool + self.filters['is_false'] = lambda v: not utils.strtobool(v) + + def create_root(self, delegate: dict) -> Node: + """ + Create the root wrapper over the recursively compiled container (inventory). + + :param delegate: the root container + :return: the root wrapper + """ + return JinjaNode(delegate, path=Path(), env=self) + + def compile_string(self, struct: str, path: Path) -> str: + """ + Compiles template string at the specified inventory `path`. + It is called both while going over the inventory, + and recursively if variables are accessed from templates. + + :param struct: template string + :param path: path of sections in the inventory + :return: compiled string + """ + if path in self._compiled: + return struct + + if path in self._compiling: + idx = self._compiling.index(path) + raise Exception( + f"Cyclic dynamic variables in inventory{' -> '.join(map(utils.pretty_path, self._compiling[idx:]))}") + + self.logger.verbose("Rendering \"%s\"" % struct) + + self._compiling.append(path) + + try: + struct = self.from_string(struct).render(self._recursive_extra) + except errors.BaseKME: + raise + except Exception as e: + raise ValueError(f"Failed to render {struct!r}\nin section {utils.pretty_path(path)}: {e}") from None + + self._compiling.pop() + + self._compiled.add(path) + + self.logger.verbose("\tRendered as \"%s\"" % struct) + return struct + + def _check_filter(self, filter_: str, struct: str, *args: Any, **kwargs: Any) -> str: if args or kwargs: raise ValueError(f"Filter {filter_!r} does not support extra arguments") if not isinstance(struct, str): raise ValueError(f"Filter {filter_!r} can be applied only on string") - # maybe we have non compiled string like templates/plugins/calico-{{ globals.compatibility_map }} ? - return recursive_compiler(struct) if recursive_compiler is not None and is_template(struct) else struct + return struct - env = jinja2.Environment() - if precompile_filters is None: - precompile_filters = {} - precompile_filters['isipv4'] = lambda ip: utils.isipv(ip, [4]) - precompile_filters['isipv6'] = lambda ip: utils.isipv(ip, [6]) - precompile_filters['minorversion'] = utils.minor_version - precompile_filters['majorversion'] = utils.major_version - precompile_filters['versionkey'] = utils.version_key - precompile_filters['b64encode'] = lambda s: base64.b64encode(s.encode()).decode() - precompile_filters['b64decode'] = lambda s: base64.b64decode(s.encode()).decode() - precompile_filters['url_quote'] = quote_plus +def jinja_tojson(obj: Any, **kwargs: Any) -> str: + return json.dumps(obj, cls=ProxyJSONEncoder, **kwargs) - for name, filter_ in precompile_filters.items(): - def make_filter(n: str, f: FILTER) -> FILTER: - return lambda s, *args, **kwargs: f(_precompile(n, s, *args, *kwargs)) - env.filters[name] = make_filter(name, filter_) - - env.filters['toyaml'] = lambda data: yaml.dump(data, default_flow_style=False) - env.tests['has_role'] = lambda node, role: role in node['roles'] - env.tests['has_roles'] = lambda node, roles: bool(set(node['roles']) & set(roles)) - - # we need these filters because rendered cluster.yaml can contain variables like - # enable: 'true' - env.filters['is_true'] = lambda v: v if isinstance(v, bool) else utils.strtobool(_precompile('is_true', v)) - env.filters['is_false'] = lambda v: not v if isinstance(v, bool) else not utils.strtobool(_precompile('is_false', v)) - return env +def jinja_toyaml(data: Any) -> str: + return yaml.dump(data, Dumper=ProxyDumper) def is_template(struct: str) -> bool: return '{{' in struct or '{%' in struct + + +def compile_node(struct: Union[list, dict], path: List[Union[str, int]], env: Environment) -> Union[list, dict]: + if isinstance(struct, list): + for i, v in enumerate(struct): + struct[i] = compile_object(v, path, i, env) + else: + for k, v in struct.items(): + struct[k] = compile_object(v, path, k, env) + + return struct + + +def compile_object(struct: Union[Primitive, list, dict], + path: List[Index], index: Index, env: Environment) -> Union[Primitive, list, dict]: + path.append(index) + if isinstance(struct, (list, dict)): + struct = compile_node(struct, path, env) + elif isinstance(struct, str) and is_template(struct): + struct = env.compile_string(struct, Path(path)) + + path.pop() + return struct diff --git a/kubemarine/plugins/__init__.py b/kubemarine/plugins/__init__.py index acae84bd4..9f33b8e68 100755 --- a/kubemarine/plugins/__init__.py +++ b/kubemarine/plugins/__init__.py @@ -997,6 +997,8 @@ def _apply_file(cluster: KubernetesCluster, config: dict, file_type: str) -> Non Apply yamls as is or renders and applies templates that match the config 'source' key. """ + from kubemarine.core import defaults # pylint: disable=cyclic-import + log = cluster.log do_render = config.get('do_render', True) @@ -1015,9 +1017,10 @@ def _apply_file(cluster: KubernetesCluster, config: dict, file_type: str) -> Non if split_extension[1] == ".j2": source_filename = split_extension[0] - render_vars = {**cluster.inventory, 'runtime_vars': cluster.context['runtime_vars'], 'env': kos.Environ()} + render_vars = {'runtime_vars': cluster.context['runtime_vars'], 'env': kos.Environ()} with utils.open_utf8(file, 'r') as template_stream: - generated_data = jinja.new(log).from_string(template_stream.read()).render(**render_vars) + env = defaults.Environment(log, cluster.inventory) + generated_data = env.from_string(template_stream.read()).render(**render_vars) utils.dump_file(cluster, generated_data, source_filename) source = io.StringIO(generated_data) diff --git a/kubemarine/procedures/check_iaas.py b/kubemarine/procedures/check_iaas.py index 120d1fef2..bac97d4e7 100755 --- a/kubemarine/procedures/check_iaas.py +++ b/kubemarine/procedures/check_iaas.py @@ -26,10 +26,11 @@ from typing import List, Dict, cast, Match, Iterator, Optional, Tuple, Set, Union import yaml +from jinja2 import Template from ordered_set import OrderedSet from kubemarine.core import flow, utils, static -from kubemarine import system, packages, jinja, thirdparties +from kubemarine import system, packages, thirdparties from kubemarine.core.cluster import KubernetesCluster from kubemarine.core.errors import KME0006 from kubemarine.testsuite import TestSuite, TestCase, TestFailure, TestWarn @@ -979,11 +980,11 @@ def get_stop_listener_cmd(port_listener: str) -> str: "&& if [ ! -z $pid ]; then sudo kill -9 $pid; echo \"killed pid $pid for port $port\"; fi" -def install_client(cluster: KubernetesCluster, group: DeferredGroup, proto: str, mtu: int, timeout: int) -> str: +def install_client(group: DeferredGroup, proto: str, mtu: int, timeout: int) -> str: check_script = utils.read_internal('resources/scripts/simple_port_client.py') udp_client = utils.get_remote_tmp_path(ext='py') for node in group.get_ordered_members_list(): - rendered_script = jinja.new(cluster.log).from_string(check_script).render({ + rendered_script = Template(check_script).render({ 'proto': proto, 'timeout': timeout, 'mtu': mtu, @@ -1006,7 +1007,7 @@ def check_connect_between_all_nodes(cluster: KubernetesCluster, group = get_python_group(cluster, True).get_accessible_nodes().new_defer() timeout = static.GLOBALS['connection']['defaults']['timeout'] - port_client = install_client(cluster, group, proto, mtu, timeout) + port_client = install_client(group, proto, mtu, timeout) connectivity_ports = get_ports_connectivity(cluster, proto).get(subnet_type, {}).get('output', {}) @@ -1136,7 +1137,7 @@ def install_listeners(cluster: KubernetesCluster, host = node.get_host() bind_address = host_to_ip[host] ip_version = ipaddress.ip_address(bind_address).version - rendered_script = jinja.new(logger).from_string(check_script).render({ + rendered_script = Template(check_script).render({ 'proto': proto, 'address': bind_address, 'ip_version': ip_version, diff --git a/kubemarine/resources/configurations/defaults.yaml b/kubemarine/resources/configurations/defaults.yaml index 6bdae2922..7c5496bc8 100644 --- a/kubemarine/resources/configurations/defaults.yaml +++ b/kubemarine/resources/configurations/defaults.yaml @@ -446,14 +446,14 @@ services: zone: '{{ cluster_name }}' data: match: '^(.*\.)?{{ cluster_name }}\.$' - answer: '{% raw %}{{ .Name }}{% endraw %} 3600 IN A {{ control_plain["internal"] }}' + answer: '{{ "{{ .Name }}" }} 3600 IN A {{ control_plain["internal"] }}' reject-aaaa: enabled: '{{ nodes[0]["internal_address"]|isipv4 }}' priority: 1 class: IN type: AAAA data: - authority: '{% raw %}{{ .Name }}{% endraw %} 3600 IN SOA coredns.kube-system.svc.cluster.local. hostmaster.coredns.kube-system.svc.cluster.local. (3600 3600 3600 3600 3600)' + authority: '{{ "{{ .Name }}" }} 3600 IN SOA coredns.kube-system.svc.cluster.local. hostmaster.coredns.kube-system.svc.cluster.local. (3600 3600 3600 3600 3600)' forward: - . - /etc/resolv.conf @@ -557,7 +557,7 @@ plugin_defaults: plugins: calico: - version: '{{ globals.compatibility_map.software["calico"][services.kubeadm.kubernetesVersion|kubernetes_version].version }}' + version: '{{ globals.compatibility_map.software["calico"][services.kubeadm.kubernetesVersion].version }}' install: true installation: priority: 0 @@ -573,12 +573,12 @@ plugins: - calico-node deployments: - calico-kube-controllers - - '{% if plugins.calico.typha.enabled | is_true %}calico-typha{% endif %}' + - '{% if plugins.calico.typha.enabled %}calico-typha{% endif %}' pods: - coredns - calico-kube-controllers - calico-node - - '{% if plugins.calico.typha.enabled | is_true %}calico-typha{% endif %}' + - '{% if plugins.calico.typha.enabled %}calico-typha{% endif %}' - thirdparty: /usr/bin/calicoctl - template: source: templates/plugins/calicoctl.cfg.j2 @@ -701,7 +701,7 @@ plugins: retries: 40 nginx-ingress-controller: - version: '{{ globals.compatibility_map.software["nginx-ingress-controller"][services.kubeadm.kubernetesVersion|kubernetes_version].version }}' + version: '{{ globals.compatibility_map.software["nginx-ingress-controller"][services.kubeadm.kubernetesVersion].version }}' install: true installation: registry: registry.k8s.io @@ -729,7 +729,7 @@ plugins: # redefine default value for controller >= v1.9.0 because we need to use snippet annotations for dashboard allow-snippet-annotations: "true" webhook: - image: 'ingress-nginx/kube-webhook-certgen:{{ globals.compatibility_map.software["nginx-ingress-controller"][services.kubeadm.kubernetesVersion|kubernetes_version]["webhook-version"] }}' + image: 'ingress-nginx/kube-webhook-certgen:{{ globals.compatibility_map.software["nginx-ingress-controller"][services.kubeadm.kubernetesVersion]["webhook-version"] }}' # resources values are based on https://github.com/kubernetes/ingress-nginx/blob/helm-chart-4.7.1/charts/ingress-nginx/values.yaml#L598 resources: requests: @@ -769,7 +769,7 @@ plugins: protocol: TCP kubernetes-dashboard: - version: '{{ globals.compatibility_map.software["kubernetes-dashboard"][services.kubeadm.kubernetesVersion|kubernetes_version].version }}' + version: '{{ globals.compatibility_map.software["kubernetes-dashboard"][services.kubeadm.kubernetesVersion].version }}' install: false installation: priority: 2 @@ -806,7 +806,7 @@ plugins: cpu: 1 memory: 200Mi metrics-scraper: - image: 'kubernetesui/metrics-scraper:{{ globals.compatibility_map.software["kubernetes-dashboard"][services.kubeadm.kubernetesVersion|kubernetes_version]["metrics-scraper-version"] }}' + image: 'kubernetesui/metrics-scraper:{{ globals.compatibility_map.software["kubernetes-dashboard"][services.kubeadm.kubernetesVersion]["metrics-scraper-version"] }}' nodeSelector: kubernetes.io/os: linux resources: @@ -846,7 +846,7 @@ plugins: number: 443 local-path-provisioner: - version: '{{ globals.compatibility_map.software["local-path-provisioner"][services.kubeadm.kubernetesVersion|kubernetes_version].version }}' + version: '{{ globals.compatibility_map.software["local-path-provisioner"][services.kubeadm.kubernetesVersion].version }}' install: false installation: priority: 2 @@ -864,7 +864,7 @@ plugins: is-default: "false" volume-dir: /opt/local-path-provisioner image: 'rancher/local-path-provisioner:{{ plugins["local-path-provisioner"].version }}' - helper-pod-image: 'library/busybox:{{ globals.compatibility_map.software["local-path-provisioner"][services.kubeadm.kubernetesVersion|kubernetes_version]["busybox-version"] }}' + helper-pod-image: 'library/busybox:{{ globals.compatibility_map.software["local-path-provisioner"][services.kubeadm.kubernetesVersion]["busybox-version"] }}' # resources values are based on https://github.com/rancher/local-path-provisioner/blob/v0.0.24/deploy/chart/local-path-provisioner/values.yaml#L69 resources: requests: diff --git a/test/unit/core/test_cluster.py b/test/unit/core/test_cluster.py index d65ae8bb9..a1aaf7cea 100644 --- a/test/unit/core/test_cluster.py +++ b/test/unit/core/test_cluster.py @@ -12,9 +12,13 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import os +import re import unittest +from unittest import mock +from test.unit import utils as test_utils -from kubemarine import demo +from kubemarine import demo, kubernetes from kubemarine.core.cluster import EnrichmentStage from kubemarine.demo import FakeKubernetesCluster @@ -253,6 +257,54 @@ def test_legacy_role(self): self.assertIn('control-plane', cluster.inventory['nodes'][0]['roles']) self.assertEqual('control-plane-1', cluster.nodes['control-plane'].get_node_name()) + def test_recursive_compile_inventory(self): + inventory = demo.generate_inventory(**demo.ALLINONE) + inventory['values'] = { + 'var1': '{{ values.var2 }}', + 'var2': '{{ "test-cluster" | upper }}', + } + inventory['cluster_name'] = '{{ values.var1 }}' + + cluster = demo.new_resources(inventory).cluster(EnrichmentStage.LIGHT) + self.assertEqual('TEST-CLUSTER', cluster.inventory['cluster_name']) + + def test_compile_env_variables(self): + inventory = demo.generate_inventory(**demo.ALLINONE) + inventory['values'] = { + 'variable': '{{ env.ENV_NAME }}', + } + + with mock.patch.dict(os.environ, {'ENV_NAME': 'value1'}): + cluster = demo.new_resources(inventory).cluster(EnrichmentStage.LIGHT) + + self.assertEqual('value1', cluster.inventory['values']['variable']) + + def test_recursive_compile_invalid_inventory_reference(self): + inventory = demo.generate_inventory(**demo.ALLINONE) + services_ref = '{{ services.kubeadm.kubernetesVersion }}' + inventory['values'] = { + 'var1': '{{ values.var2 }}', + 'var2': services_ref, + } + + with test_utils.assert_raises_regex(self, ValueError, re.escape( + f"Failed to render {services_ref!r}\n" + "in section ['values']['var2']: 'services' is undefined")): + demo.new_resources(inventory).cluster(EnrichmentStage.LIGHT) + + def test_compile_invalid_globals_reference(self): + inventory = demo.generate_inventory(**demo.ALLINONE) + kubernetes_version = kubernetes.get_kubernetes_version(inventory) + globals_ref = f'{{{{ globals.compatibility_map.software["calico"]["{kubernetes_version}"].version }}}}' + inventory['values'] = { + 'var1': globals_ref, + } + + with test_utils.assert_raises_regex(self, ValueError, re.escape( + f"Failed to render {globals_ref!r}\n" + "in section ['values']['var1']: 'globals' is undefined")): + demo.new_resources(inventory).cluster(EnrichmentStage.LIGHT) + if __name__ == '__main__': unittest.main() diff --git a/test/unit/core/test_jinja.py b/test/unit/core/test_jinja.py new file mode 100644 index 000000000..263b52284 --- /dev/null +++ b/test/unit/core/test_jinja.py @@ -0,0 +1,197 @@ +import json +import re +import unittest +from test.unit import utils as test_utils + +import yaml + +from kubemarine import demo + + +class TestCompilation(unittest.TestCase): + def test_escapes(self): + inventory = demo.generate_inventory(**demo.ALLINONE) + inventory['values'] = { + 'var1': "{{ '{{ .Name }}' }}", + 'var2': "{% raw %}{{ .Name }}{% endraw %}", + 'var3': '{% raw %}{{ raw1 }}{% endraw %}text1{% raw %}{{ raw2 }}{% endraw %}text2', + 'var4': 'A', + 'var5': '{% raw %}{{ raw1 }}{% endraw %}{{ values.var4 }}{% raw %}{{ raw2 }}{% endraw %}{{ values.var4 }}', + 'var6': '{% raw %}{{ raw1 }}{% endraw %}{{ values.var2 }}{% raw %}{{ raw2 }}{% endraw %}{{ values.var2 }}', + } + + def test(cluster_: demo.FakeKubernetesCluster): + values = cluster_.inventory['values'] + self.assertEqual('{{ .Name }}', values['var1']) + self.assertEqual('{{ .Name }}', values['var2']) + self.assertEqual('{{ raw1 }}text1{{ raw2 }}text2', values['var3']) + self.assertEqual('{{ raw1 }}A{{ raw2 }}A', values['var5']) + self.assertEqual('{{ raw1 }}{{ .Name }}{{ raw2 }}{{ .Name }}', values['var6']) + + cluster = demo.new_cluster(inventory) + test(cluster) + + cluster = demo.new_cluster(test_utils.make_finalized_inventory(cluster)) + test(cluster) + + def test_recursive_filters(self): + inventory = demo.generate_inventory(**demo.ALLINONE) + inventory['values'] = { + 'var1': "{{ values.template | b64encode | b64decode | upper }}", + 'var2': "{{ (values.int1 | int) + (values.int2 | int) }}", + 'var3': "{{ '{{ values.TEST }}' | b64encode | b64decode | lower }}", + 'var4': "{{ '{{ values.TEST }}' | lower }}", + 'template': '{{ "text" }}', + 'int1': '{{ 1 }}', + 'int2': '{{ 2 }}', + 'test': 'unexpected' + } + + cluster = demo.new_cluster(inventory) + values = cluster.inventory['values'] + + self.assertEqual('TEXT', values['var1']) + self.assertEqual('3', values['var2']) + self.assertEqual('{{ values.test }}', values['var3']) + self.assertEqual('{{ values.test }}', values['var4']) + + def test_not_existing_variables(self): + inventory = demo.generate_inventory(**demo.ALLINONE) + inventory['values'] = { + 'var1': '{{ values.notexists | default("not exists") }}', + 'var2': [], + 'var3': '{{ values.var2[0] | lower }}', + 'var4': '{{ values.var2[:10][0] | default("not exists") }}', + } + + cluster = demo.new_cluster(inventory) + values = cluster.inventory['values'] + + self.assertEqual('not exists', values['var1']) + self.assertEqual('', values['var3']) + self.assertEqual('not exists', values['var4']) + + def test_mappings(self): + inventory = demo.generate_inventory(**demo.ALLINONE) + template_map = { + 'foo': 'bar', 'array': ['{{ values.ref }}', 2], + } + expected_map = { + 'foo': 'bar', 'array': ['text', 2], + } + inventory['values'] = { + 'var1': '{{ values.map }}', + 'var2': '{{ values.map | tojson }}', + 'map': template_map, + 'var3': '{{ values.map | toyaml }}', + 'var4': '{{ values is mapping }}', + 'var5': '{{ "ref" in values }}', + 'var6': '{{ "missed" in values }}', + 'ref': 'text', + } + + cluster = demo.new_cluster(inventory) + values = cluster.inventory['values'] + + self.assertEqual("{'foo': 'bar', 'array': ['text', 2]}", values['var1']) + self.assertEqual(expected_map, json.loads(values['var2'])) + self.assertEqual(expected_map, yaml.safe_load(values['var3'])) + self.assertEqual('True', values['var4']) + self.assertEqual('True', values['var5']) + self.assertEqual('False', values['var6']) + + def test_sequences(self): + inventory = demo.generate_inventory(**demo.ALLINONE) + template_list = [1, 2, {'3': '{{ values.ref }}'}, 4, 5, 6, 7, 8, 9] + expected_list = [1, 2, {'3': 'text'}, 4, 5, 6, 7, 8, 9] + inventory['values'] = { + 'list': template_list, + 'var1': '{{ values.list }}', + 'var2': '{{ values.list | tojson }}', + 'var3': '{{ values.list | toyaml }}', + 'var4': '{{ values.list is sequence }}', + 'var5': '{{ 5 in values.list }}', + 'var6': '{{ 0 in values.list }}', + 'ref': 'text', + } + + cluster = demo.new_cluster(inventory) + values = cluster.inventory['values'] + + self.assertEqual("[1, 2, {'3': 'text'}, 4, 5, 6, 7, 8, 9]", values['var1']) + self.assertEqual(expected_list, json.loads(values['var2'])) + self.assertEqual(expected_list, yaml.safe_load(values['var3'])) + self.assertEqual('True', values['var4']) + self.assertEqual('True', values['var5']) + self.assertEqual('False', values['var6']) + + def test_slices(self): + inventory = demo.generate_inventory(**demo.ALLINONE) + template_list = [1, 2, {'3': '{{ values.ref }}'}, 4, 5, 6, 7, 8, 9] + expected_slice = [1, {'3': 'text'}] + inventory['values'] = { + 'list': template_list, + 'var1': '{{ values.list[:4:2] }}', + 'var2': '{{ values.list[:4:2] | tojson }}', + 'var3': '{{ values.list[:4:2] | toyaml }}', + 'var4': '{{ values.list[:4:2] is sequence }}', + 'var5': '{{ 1 in values.list[:4:2] }}', + 'var6': '{{ 2 in values.list[:4:2] }}', + 'var7': '{{ values.list[:6:2][1:][1] }}', + 'ref': 'text', + } + + cluster = demo.new_cluster(inventory) + values = cluster.inventory['values'] + + self.assertEqual("[1, {'3': 'text'}]", values['var1']) + self.assertEqual(expected_slice, json.loads(values['var2'])) + self.assertEqual(expected_slice, yaml.safe_load(values['var3'])) + self.assertEqual('True', values['var4']) + self.assertEqual('True', values['var5']) + self.assertEqual('False', values['var6']) + self.assertEqual('5', values['var7']) + + def test_redefine_inventory_section(self): + inventory = demo.generate_inventory(**demo.ALLINONE) + inventory['cluster_name'] = 'test-cluster' + inventory['values'] = { + 'var1': '{% set cluster_name = "redefined" %}{{ cluster_name }}', + 'var2': '{{ cluster_name }}', + } + + cluster = demo.new_cluster(inventory) + values = cluster.inventory['values'] + + self.assertEqual("redefined", values['var1']) + self.assertEqual("test-cluster", values['var2']) + + def test_cyclic_references(self): + inventory = demo.generate_inventory(**demo.ALLINONE) + inventory['values'] = { + 'var0': '{{ values.var1 }}', + 'var1': '{{ values.var3 }}', + 'var2': '{{ values.var1 }}', + 'var3': '{{ values.var2 }}', + } + + with test_utils.assert_raises_regex(self, ValueError, re.escape( + "Cyclic dynamic variables in inventory['values']['var1'] -> ['values']['var3'] -> ['values']['var2']")): + demo.new_cluster(inventory) + + def test_cyclic_references_proxy_types(self): + inventory = demo.generate_inventory(**demo.ALLINONE) + inventory['values'] = { + 'var1': [1, '{{ values.var3 }}', '{{ values.var2 }}', 4], + 'var2': '{{ values.var1[1] }}', + 'var3': '{{ values.var1[1:][:2][1] }}', + } + + with test_utils.assert_raises_regex(self, ValueError, re.escape( + "Cyclic dynamic variables in inventory" + "['values']['var1'][1] -> ['values']['var3'] -> ['values']['var1'][2] -> ['values']['var2']")): + demo.new_cluster(inventory) + + +if __name__ == '__main__': + unittest.main() diff --git a/test/unit/core/test_run_actions.py b/test/unit/core/test_run_actions.py index 9160eaf57..d6b74ae37 100644 --- a/test/unit/core/test_run_actions.py +++ b/test/unit/core/test_run_actions.py @@ -25,7 +25,7 @@ import yaml from kubemarine import demo, kubernetes, testsuite, procedures -from kubemarine.core import flow, errors, action, utils, schema, resources as res, summary, defaults +from kubemarine.core import flow, action, utils, schema, resources as res, summary, defaults from kubemarine.core.cluster import KubernetesCluster, EnrichmentStage from kubemarine.core.yaml_merger import default_merger from kubemarine.procedures import upgrade, install, check_iaas, check_paas, migrate_kubemarine @@ -116,11 +116,8 @@ def _run_actions(self, actions: List[action.Action], if exception_message is None: flow.run_actions(resources, actions) else: - with self.assertRaisesRegex(Exception, exception_message): - try: - flow.run_actions(resources, actions) - except errors.FailException as e: - raise e.reason + with test_utils.assert_raises_regex(self, Exception, exception_message): + flow.run_actions(resources, actions) return resources.cluster(EnrichmentStage.LIGHT).uploaded_archives diff --git a/test/unit/plugins/test_template.py b/test/unit/plugins/test_template.py index 71b66b900..2e8ea51bf 100644 --- a/test/unit/plugins/test_template.py +++ b/test/unit/plugins/test_template.py @@ -11,15 +11,16 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - +import os import unittest +from test.unit import utils as test_utils -from kubemarine import demo -from kubemarine.core import errors +from kubemarine import demo, plugins +from kubemarine.core import errors, utils from kubemarine.plugins import verify_template, apply_template -class TestTemplate(unittest.TestCase): +class TestTemplate(test_utils.CommonTest): def test_verify_missed_source(self): for procedure_type in ('template', 'config'): @@ -121,6 +122,32 @@ def test_apply_template(self): cnt += 1 self.assertEqual(1, cnt) + @test_utils.temporary_directory + def test_compile_template(self): + template_file = os.path.join(self.tmpdir, 'template.yaml.j2') + with utils.open_external(template_file, 'w') as t: + t.write('Compiled: {{ plugins.my_plugin.compiled }}') + + inventory = demo.generate_inventory(**demo.ALLINONE) + + inventory['plugins'] = {'my_plugin': { + 'compiled': '{{ "{{ Yes }}" }}', + 'install': True, + 'installation': {'procedures': [{'template': template_file}]} + }} + + cluster = demo.new_cluster(inventory) + + result = demo.create_nodegroup_result(cluster.nodes["master"], hide=False) + cluster.fake_shell.add(result, "sudo", [f"kubectl apply -f /etc/kubernetes/template.yaml"], usage_limit=1) + + plugins.install(cluster, {'my_plugin': cluster.inventory['plugins']['my_plugin']}) + + host = cluster.nodes['all'].get_host() + destination_data = cluster.fake_fs.read(host, '/etc/kubernetes/template.yaml') + + self.assertEqual('Compiled: {{ Yes }}', destination_data, "Inventory variable should be expanded.") + if __name__ == '__main__': unittest.main() diff --git a/test/unit/test_coredns.py b/test/unit/test_coredns.py index 677171646..61f85e79d 100755 --- a/test/unit/test_coredns.py +++ b/test/unit/test_coredns.py @@ -15,6 +15,7 @@ import unittest +from test.unit import utils as test_utils from kubemarine import coredns, system, demo @@ -155,14 +156,23 @@ def test_configmap_generation_with_hosts(self): def test_configmap_generation_with_corefile_defaults(self): inventory = demo.generate_inventory(**demo.MINIHA_KEEPALIVED) + + def test(cluster_: demo.FakeKubernetesCluster): + config = coredns.generate_configmap(cluster_.inventory) + self.assertIn('prometheus :9153', config) + self.assertIn('cache 30', config) + self.assertIn('loadbalance', config) + self.assertIn('hosts /etc/coredns/Hosts', config) + self.assertIn('template IN A k8s.fake.local', config) + self.assertIn('forward . /etc/resolv.conf', config) + self.assertIn('answer "{{ .Name }} 3600 IN A %s"' % (cluster_.inventory['control_plain']['internal'],), config) + self.assertIn('authority "{{ .Name }} 3600 IN SOA', config) + cluster = demo.new_cluster(inventory) - config = coredns.generate_configmap(cluster.inventory) - self.assertIn('prometheus :9153', config) - self.assertIn('cache 30', config) - self.assertIn('loadbalance', config) - self.assertIn('hosts /etc/coredns/Hosts', config) - self.assertIn('template IN A k8s.fake.local', config) - self.assertIn('forward . /etc/resolv.conf', config) + test(cluster) + + cluster = demo.new_cluster(test_utils.make_finalized_inventory(cluster)) + test(cluster) def test_configmap_generation_with_corefile_defaults_disabled(self): inventory = demo.generate_inventory(**demo.MINIHA_KEEPALIVED) @@ -186,6 +196,8 @@ def test_configmap_generation_with_corefile_defaults_disabled(self): self.assertNotIn('hosts /etc/coredns/Hosts', config) self.assertIn('template IN A k8s.fake.local', config) self.assertNotIn('forward . /etc/resolv.conf', config) + self.assertIn('answer "{{ .Name }} 3600 IN A %s"' % (cluster.inventory['control_plain']['internal'],), config) + self.assertIn('authority "{{ .Name }} 3600 IN SOA', config) if __name__ == '__main__': diff --git a/test/unit/test_defaults.py b/test/unit/test_defaults.py index 3979f65cb..a9b2513d6 100755 --- a/test/unit/test_defaults.py +++ b/test/unit/test_defaults.py @@ -438,6 +438,18 @@ def test_custom_jinja_enrichment(self): self.assertEqual(True, inventory['plugins']['kubernetes-dashboard']['install']) + def test_recursive_reference_primitive_template(self): + inventory = demo.generate_inventory(**demo.ALLINONE) + inventory['plugins'] = {'my_plugin': { + 'var1': '{% if plugins.my_plugin.install %}unexpected{% else %}ok{% endif %}', + 'install': '{{ "false" }}', + }} + + cluster = demo.new_cluster(inventory) + inventory = cluster.inventory + self.assertEqual(False, inventory['plugins']['my_plugin']['install']) + self.assertEqual('ok', inventory['plugins']['my_plugin']['var1']) + def _actual_sysctl_params(self, cluster: demo.FakeKubernetesCluster, node: NodeGroup) -> Set[str]: return { record.split(' = ')[0] diff --git a/test/unit/test_k8s_cert.py b/test/unit/test_k8s_cert.py index be8a60cd5..20998584b 100644 --- a/test/unit/test_k8s_cert.py +++ b/test/unit/test_k8s_cert.py @@ -18,7 +18,7 @@ from test.unit import utils as test_utils from kubemarine import demo, plugins -from kubemarine.core import flow, errors +from kubemarine.core import flow from kubemarine.plugins import nginx_ingress from kubemarine.procedures import cert_renew @@ -94,15 +94,12 @@ def _procedure_cert(self) -> dict: return self.cert_renew.setdefault('nginx-ingress-controller', {}) def _run_and_check(self, called: bool) -> demo.FakeResources: - with mock.patch.object(plugins, plugins.install_plugin.__name__) as run: + with mock.patch.object(plugins, plugins.install_plugin.__name__) as run, test_utils.unwrap_fail(): resources = test_utils.FakeResources(self.context, self.inventory, procedure_inventory=self.cert_renew, nodes_context=demo.generate_nodes_context(self.inventory)) try: flow.run_actions(resources, [cert_renew.CertRenewAction()]) - except errors.FailException as e: - if e.reason is not None: - raise e.reason finally: self.assertEqual(called, run.called) diff --git a/test/unit/test_migrate_kubemarine.py b/test/unit/test_migrate_kubemarine.py index f95182c17..34ddabf58 100644 --- a/test/unit/test_migrate_kubemarine.py +++ b/test/unit/test_migrate_kubemarine.py @@ -1012,11 +1012,8 @@ def _services_upgraded(self, type_: str, ctx, expected_services: Dict[str, bool] @contextmanager def _assert_raises_kme(self): - with self.assertRaisesRegex(errors._BaseKME, "KME"): - try: - yield - except errors.FailException as e: - raise e.reason + with self.assertRaisesRegex(errors.BaseKME, "KME"), test_utils.unwrap_fail(): + yield def test_reinstall_dashboard_change_inventory_run_other_patch_collect_summary(self): for change_hostname in (False, True): diff --git a/test/unit/utils.py b/test/unit/utils.py index 28260397a..f945f561a 100644 --- a/test/unit/utils.py +++ b/test/unit/utils.py @@ -21,7 +21,7 @@ from contextlib import contextmanager from copy import deepcopy from types import FunctionType -from typing import Dict, Iterator, Callable, cast, Any, List, Optional, Union +from typing import Dict, Iterator, Callable, cast, Any, List, Optional, Union, Type from unittest import mock import yaml @@ -157,12 +157,27 @@ def assert_raises_kme(test: unittest.TestCase, code: str, *, escape: bool = Fals msg_pattern = str(exception) if escape: msg_pattern = re.escape(msg_pattern) - with test.assertRaisesRegex(type(exception), msg_pattern): - try: - yield - except errors.FailException as e: + + with assert_raises_regex(test, type(exception), msg_pattern): + yield + + +@contextmanager +def assert_raises_regex(test: unittest.TestCase, expected_exception: Type[Exception], expected_regex: str): + with test.assertRaisesRegex(expected_exception, expected_regex), unwrap_fail(): + yield + + +@contextmanager +def unwrap_fail(): + try: + yield + except errors.FailException as e: + if e.reason is not None: raise e.reason + raise + @contextmanager def backup_globals():