From 7e642b83810e8635705e4784351903e3fabac5d4 Mon Sep 17 00:00:00 2001 From: rallytime Date: Thu, 31 Mar 2016 13:50:51 -0600 Subject: [PATCH 1/4] Audit config.py default types and values - first sweep - Updated standard types in VALID_OPTS dict where necessary - Updated default values to match VALID_OPTS in DEFAULT_*_OPTS dicts - Updated master/minion configuration reference docs to match changes - Removed enumerate_proxy_minions value as it's not referenced anywhere in code --- doc/ref/cli/salt-key.rst | 10 ++-- doc/ref/configuration/master.rst | 52 +++++++++++--------- doc/ref/configuration/minion.rst | 71 ++++++++++++++-------------- doc/topics/tutorials/intro_scale.rst | 6 ++- salt/config.py | 34 +++++++------ 5 files changed, 90 insertions(+), 83 deletions(-) diff --git a/doc/ref/cli/salt-key.rst b/doc/ref/cli/salt-key.rst index ca2d31b171a6..f7a5c8ae0195 100644 --- a/doc/ref/cli/salt-key.rst +++ b/doc/ref/cli/salt-key.rst @@ -137,7 +137,7 @@ Actions Key Generation Options ------------------------ +---------------------- .. option:: --gen-keys=GEN_KEYS @@ -157,10 +157,10 @@ Key Generation Options .. option:: --gen-signature - Create a signature file of the masters public-key named - master_pubkey_signature. The signature can be send to a minion in the - masters auth-reply and enables the minion to verify the masters public-key - cryptographically. This requires a new signing-key- pair which can be + Create a signature file of the master's public-key named + master_pubkey_signature. The signature can be sent to a minion in the + master's auth-reply and enables the minion to verify the master's public-key + cryptographically. This requires a new signing-key-pair which can be auto-created with the --auto-create parameter. .. option:: --priv=PRIV diff --git a/doc/ref/configuration/master.rst b/doc/ref/configuration/master.rst index 6ea27a8ce109..db44fa5e9021 100644 --- a/doc/ref/configuration/master.rst +++ b/doc/ref/configuration/master.rst @@ -64,7 +64,7 @@ The network port to set up the publication interface. .. conf_master:: master_id ``master_id`` ----------------- +------------- Default: ``None`` @@ -182,7 +182,7 @@ Specify the location of the master pidfile. ``root_dir`` ------------ -Default: :file:`/` +Default: ``/`` The system root directory to operate from, change this to make Salt run from an alternative root. @@ -203,13 +203,13 @@ an alternative root. ``pki_dir`` ----------- -Default: :file:`/etc/salt/pki` +Default: ``/etc/salt/pki/master`` The directory to store the pki authentication keys. .. code-block:: yaml - pki_dir: /etc/salt/pki + pki_dir: /etc/salt/pki/master .. conf_master:: extension_modules @@ -244,14 +244,14 @@ for Salt modules. ``cachedir`` ------------ -Default: :file:`/var/cache/salt` +Default: ``/var/cache/salt/master`` The location used to store cache information, particularly the job information for executed salt commands. .. code-block:: yaml - cachedir: /var/cache/salt + cachedir: /var/cache/salt/master .. conf_master:: verify_env @@ -275,6 +275,10 @@ Default: ``24`` Set the number of hours to keep old job information. +.. code-block:: yaml + + keep_jobs: 24 + .. conf_master:: timeout ``timeout`` @@ -350,7 +354,7 @@ grains for the master. Default: ``True`` -The master maintains a job cache, while this is a great addition it can be +The master maintains a job cache. While this is a great addition, it can be a burden on the master for larger deployments (over 5000 minions). Disabling the job cache will make previously executed jobs unavailable to the jobs system and is not generally recommended. Normally it is wise to make @@ -380,7 +384,7 @@ predetermine what minions are expected to reply from executions. Default: ``''`` -Used to specify a default returner for all minions, when this option is set +Used to specify a default returner for all minions. When this option is set, the specified returner needs to be properly configured and the minions will always default to sending returns to this returner. This will also disable the local job cache on the master. @@ -392,7 +396,7 @@ local job cache on the master. .. conf_master:: event_return ``event_return`` ------------------ +---------------- .. versionadded:: 2015.5.0 @@ -417,7 +421,7 @@ configuration requirements. Read the returner's documentation. .. versionadded:: 2014.7.0 -Default: 'local_cache' +Default: ``local_cache`` Specify the returner to use for the job cache. The job cache will only be interacted with from the salt master and therefore does not need to be @@ -451,7 +455,7 @@ Default: 0 The number of minions the master should allow to connect. Use this to accommodate the number of minions per master if you have different types of hardware serving -your minions. The default of ``0`` means unlimited connections. Please note, that +your minions. The default of ``0`` means unlimited connections. Please note that this can slow down the authentication process a bit in large setups. .. code-block:: yaml @@ -476,7 +480,7 @@ performance of max_minions. .. conf_master:: presence_events ``presence_events`` ----------------------- +------------------- Default: False @@ -699,7 +703,7 @@ security purposes. Default: ``False`` -Sign the master auth-replies with a cryptographic signature of the masters +Sign the master auth-replies with a cryptographic signature of the master's public key. Please see the tutorial how to use these settings in the `Multimaster-PKI with Failover Tutorial `_ @@ -727,8 +731,8 @@ The customizable name of the signing-key-pair without suffix. Default: ``master_pubkey_signature`` -The name of the file in the masters pki-directory that holds the pre-calculated -signature of the masters public-key. +The name of the file in the master's pki-directory that holds the pre-calculated +signature of the master's public-key. .. code-block:: yaml @@ -778,6 +782,11 @@ Default: ``[]`` Set additional directories to search for runner modules. +.. code-block:: yaml + + runner_dirs: + - /var/lib/salt/runners + .. conf_master:: cython_enable ``cython_enable`` @@ -866,7 +875,7 @@ The renderer to use on the minions to render the state data. Default: ``False`` -Set the global failhard flag, this informs all states to stop running states +Set the global failhard flag. This informs all states to stop running states at the moment a single state fails. .. code-block:: yaml @@ -882,8 +891,7 @@ Default: ``True`` Controls the verbosity of state runs. By default, the results of all states are returned, but setting this value to ``False`` will cause salt to only display -output for states which either failed, or succeeded without making any changes -to the minion. +output for states that failed or states that have changes. .. code-block:: yaml @@ -1909,7 +1917,7 @@ functions. The available ext_pillar functions can be found herein: By default, the ext_pillar interface is not configured to run. -Default: ``None`` +Default: ``[]`` .. code-block:: yaml @@ -2080,7 +2088,7 @@ value must be set to True ``syndic_master`` ----------------- -Default: ``None`` +Default: ``''`` If this master will be running a salt-syndic to connect to a higher level master, specify the higher level master with this configuration value. @@ -2381,7 +2389,7 @@ Range Cluster Settings ``range_server`` ---------------- -Default: ``''`` +Default: ``'range:80'`` The range server (and optional port) that serves your cluster information https://github.com/ytoolshed/range/wiki/%22yamlfile%22-module-file-spec @@ -2470,7 +2478,7 @@ Default: ``/srv/salt/win/repo/winrepo.p`` ``win_gitrepos`` ---------------- -Default: ``''`` +Default: ``['https://github.com/saltstack/salt-winrepo.git']`` List of git repositories to include with the local repo. diff --git a/doc/ref/configuration/minion.rst b/doc/ref/configuration/minion.rst index eaec99aec60f..7b194cd7bf00 100644 --- a/doc/ref/configuration/minion.rst +++ b/doc/ref/configuration/minion.rst @@ -110,14 +110,16 @@ the minion and it will connect to the new master. ``master_alive_interval`` ------------------------- -.. code-block:: yaml - - master_alive_interval: 30 +Default: ``0`` Configures how often, in seconds, the minion will verify that the current master is alive and responding. The minion will try to establish a connection to the next master in the list if it finds the existing one is dead. +.. code-block:: yaml + + master_alive_interval: 30 + ``master_shuffle`` ------------------ @@ -134,7 +136,7 @@ Python's :func:`random.shuffle ` method. master_shuffle: True ``random_master`` ------------------- +----------------- Default: ``False`` @@ -149,7 +151,7 @@ Python's :func:`random.randint ` method. .. conf_minion:: retry_dns ``retry_dns`` ---------------- +------------- Default: ``30`` @@ -191,11 +193,11 @@ The user to run the Salt processes .. conf_minion:: sudo_user ``sudo_user`` --------- +------------- Default: ``''`` -Setting ``sudo_user`` will cause salt to run all execution modules under an +Setting ``sudo_user`` will cause salt to run all execution modules under a sudo to the user given in ``sudo_user``. The user under which the salt minion process itself runs will still be that provided in :conf_minion:`user` above, but all execution modules run by the minion will be rerouted through sudo. @@ -237,13 +239,13 @@ This directory is prepended to the following options: :conf_minion:`pki_dir`, ``pki_dir`` ----------- -Default: ``/etc/salt/pki`` +Default: ``/etc/salt/pki/minion`` The directory used to store the minion's public and private keys. .. code-block:: yaml - pki_dir: /etc/salt/pki + pki_dir: /etc/salt/pki/minion .. conf_minion:: id @@ -285,13 +287,13 @@ FQDN (for instance, Solaris). ``cachedir`` ------------ -Default: ``/var/cache/salt`` +Default: ``/var/cache/salt/minion`` The location for minion cache data. .. code-block:: yaml - cachedir: /var/cache/salt + cachedir: /var/cache/salt/minion .. conf_minion:: verify_env @@ -363,7 +365,7 @@ The directory where Unix sockets will be kept. ``backup_mode`` --------------- -Default: ``[]`` +Default: ``''`` Backup files replaced by file.managed and file.recurse under cachedir. @@ -390,6 +392,8 @@ master. ``random_reauth_delay`` ----------------------- +Default: ``10`` + When the master key changes, the minion will try to re-auth itself to receive the new master key. In larger environments this can cause a syn-flood on the master because all minions try to re-auth immediately. To prevent this @@ -406,7 +410,7 @@ parameter. The wait-time will be a random number of seconds between ``acceptance_wait_time_max`` ---------------------------- -Default: ``None`` +Default: ``0`` The maximum number of seconds to wait until attempting to re\-authenticate with the master. If set, the wait will increase by acceptance_wait_time @@ -414,7 +418,7 @@ seconds each iteration. .. code-block:: yaml - acceptance_wait_time_max: None + acceptance_wait_time_max: 0 .. conf_minion:: recon_default @@ -461,7 +465,7 @@ Short example: Default: ``True`` Generate a random wait time on minion start. The wait time will be a random value -between recon_default and recon_default and recon_max. Having all minions reconnect +between recon_default and recon_default + recon_max. Having all minions reconnect with the same recon_default and recon_max value kind of defeats the purpose of being able to change these settings. If all minions have the same values and the setup is quite large (several thousand minions), they will still flood the master. The desired @@ -474,7 +478,7 @@ behavior is to have time-frame within all minions try to reconnect. .. conf_minion:: return_retry_timer ``return_retry_timer`` -------------------- +---------------------- Default: ``5`` @@ -488,12 +492,12 @@ The default timeout for a minion return attempt. .. conf_minion:: return_retry_timer_max ``return_retry_timer_max`` -------------------- +-------------------------- Default: ``10`` The maximum timeout for a minion return attempt. If non-zero the minion return -retry timeout will be a random int beween ``return_retry_timer`` and +retry timeout will be a random int between ``return_retry_timer`` and ``return_retry_timer_max`` .. code-block:: yaml @@ -509,7 +513,6 @@ Default: ``True`` The connection to the master ret_port is kept open. When set to False, the minion creates a new connection for every return to the master. -environment, set this value to ``False``. .. code-block:: yaml @@ -619,7 +622,7 @@ A list of extra directories to search for Salt returners .. code-block:: yaml - returners_dirs: + returner_dirs: - /var/lib/salt/returners .. conf_minion:: states_dirs @@ -674,7 +677,7 @@ A list of extra directories to search for Salt renderers Default: ``False`` Set this value to true to enable auto-loading and compiling of ``.pyx`` modules, -This setting requires that ``gcc`` and ``cython`` are installed on the minion +This setting requires that ``gcc`` and ``cython`` are installed on the minion. .. code-block:: yaml @@ -719,12 +722,11 @@ The default renderer used for local state executions ``state_verbose`` ----------------- -Default: ``False`` +Default: ``True`` -state_verbose allows for the data returned from the minion to be more -verbose. Normally only states that fail or states that have changes are -returned, but setting state_verbose to ``True`` will return all states that -were checked +Controls the verbosity of state runs. By default, the results of all states are +returned, but setting this value to ``False`` will cause salt to only display +output for states that failed or states that have changes. .. code-block:: yaml @@ -752,9 +754,9 @@ the output will be shortened to a single line. Default: ``True`` -autoload_dynamic_modules Turns on automatic loading of modules found in the -environments on the master. This is turned on by default, to turn of -auto-loading modules when states run set this value to ``False`` +autoload_dynamic_modules turns on automatic loading of modules found in the +environments on the master. This is turned on by default. To turn off +auto-loading modules when states run, set this value to ``False``. .. code-block:: yaml @@ -765,9 +767,9 @@ auto-loading modules when states run set this value to ``False`` Default: ``True`` clean_dynamic_modules keeps the dynamic modules on the minion in sync with -the dynamic modules on the master, this means that if a dynamic module is +the dynamic modules on the master. This means that if a dynamic module is not on the master it will be deleted from the minion. By default this is -enabled and can be disabled by changing this value to ``False`` +enabled and can be disabled by changing this value to ``False``. .. code-block:: yaml @@ -927,7 +929,6 @@ before the initial key exchange. The master fingerprint can be found by running .. conf_minion:: verify_master_pubkey_sign - ``verify_master_pubkey_sign`` ----------------------------- @@ -971,7 +972,7 @@ minion's pki directory. Default: ``False`` -If :conf_minion:`verify_master_pubkey_sign` is enabled, the signature is only verified, +If :conf_minion:`verify_master_pubkey_sign` is enabled, the signature is only verified if the public-key of the master changes. If the signature should always be verified, this can be set to ``True``. @@ -1145,11 +1146,9 @@ This can be used to control logging levels more specifically. See also Default: ``False`` -Set the global failhard flag, this informs all states to stop running states +Set the global failhard flag. This informs all states to stop running states at the moment a single state fails - - .. code-block:: yaml failhard: False diff --git a/doc/topics/tutorials/intro_scale.rst b/doc/topics/tutorials/intro_scale.rst index efe39f4970b1..5114708ef2e6 100644 --- a/doc/topics/tutorials/intro_scale.rst +++ b/doc/topics/tutorials/intro_scale.rst @@ -106,13 +106,15 @@ the sample configuration file (default values) .. code-block:: yaml - recon_default: 100ms + recon_default: 1000 recon_max: 5000 recon_randomize: True -- recon_default: the default value the socket should use, i.e. 100ms +- recon_default: the default value the socket should use, i.e. 1000. This value is in + milliseconds. (1000ms = 1 second) - recon_max: the max value that the socket should use as a delay before trying to reconnect + This value is in milliseconds. (5000ms = 5 seconds) - recon_randomize: enables randomization between recon_default and recon_max To tune this values to an existing environment, a few decision have to be made. diff --git a/salt/config.py b/salt/config.py index 28657ec8461e..eba83f34d1b5 100644 --- a/salt/config.py +++ b/salt/config.py @@ -144,12 +144,12 @@ 'grains': dict, 'permissive_pki_access': bool, 'default_include': str, - 'update_url': bool, + 'update_url': (bool, str), 'update_restart_services': list, 'retry_dns': float, 'recon_max': float, 'recon_default': float, - 'recon_randomize': float, + 'recon_randomize': bool, 'return_retry_timer': int, 'return_retry_timer_max': int, 'event_return': str, @@ -210,7 +210,7 @@ 'ping_on_rotate': bool, 'peer': dict, 'preserve_minion_cache': bool, - 'syndic_master': str, + 'syndic_master': (str, list), 'runner_dirs': list, 'client_acl': dict, 'client_acl_blacklist': dict, @@ -219,8 +219,8 @@ 'token_expire': int, 'file_recv': bool, 'file_recv_max_size': int, - 'file_ignore_regex': bool, - 'file_ignore_glob': bool, + 'file_ignore_regex': (list, str), + 'file_ignore_glob': (list, str), 'fileserver_backend': list, 'fileserver_followsymlinks': bool, 'fileserver_ignoresymlinks': bool, @@ -257,7 +257,6 @@ 'sign_pub_messages': bool, 'keysize': int, 'transport': str, - 'enumerate_proxy_minions': bool, 'gather_job_timeout': int, 'auth_timeout': int, 'auth_tries': int, @@ -303,7 +302,7 @@ 'interface': '0.0.0.0', 'master': 'salt', 'master_type': 'str', - 'master_port': '4506', + 'master_port': 4506, 'master_finger': '', 'master_shuffle': False, 'master_alive_interval': 0, @@ -314,7 +313,7 @@ 'user': 'root', 'root_dir': salt.syspaths.ROOT_DIR, 'pki_dir': os.path.join(salt.syspaths.CONFIG_DIR, 'pki', 'minion'), - 'id': None, + 'id': '', 'cachedir': os.path.join(salt.syspaths.CACHE_DIR, 'minion'), 'cache_jobs': False, 'grains_cache': False, @@ -342,8 +341,8 @@ 'fileserver_limit_traversal': False, 'file_recv': False, 'file_recv_max_size': 100, - 'file_ignore_regex': None, - 'file_ignore_glob': None, + 'file_ignore_regex': [], + 'file_ignore_glob': [], 'fileserver_backend': ['roots'], 'fileserver_followsymlinks': True, 'fileserver_ignoresymlinks': False, @@ -460,13 +459,13 @@ DEFAULT_MASTER_OPTS = { 'interface': '0.0.0.0', - 'publish_port': '4505', + 'publish_port': 4505, 'pub_hwm': 1000, 'auth_mode': 1, 'user': 'root', 'worker_threads': 5, 'sock_dir': os.path.join(salt.syspaths.SOCK_DIR, 'master'), - 'ret_port': '4506', + 'ret_port': 4506, 'timeout': 5, 'keep_jobs': 24, 'root_dir': salt.syspaths.ROOT_DIR, @@ -536,8 +535,8 @@ 'file_recv': False, 'file_recv_max_size': 100, 'file_buffer_size': 1048576, - 'file_ignore_regex': None, - 'file_ignore_glob': None, + 'file_ignore_regex': [], + 'file_ignore_glob': [], 'fileserver_backend': ['roots'], 'fileserver_followsymlinks': True, 'fileserver_ignoresymlinks': False, @@ -607,7 +606,6 @@ 'sign_pub_messages': False, 'keysize': 2048, 'transport': 'zeromq', - 'enumerate_proxy_minions': False, 'gather_job_timeout': 10, 'syndic_event_forward_timeout': 0.5, 'syndic_max_event_process_time': 0.5, @@ -779,7 +777,7 @@ def format_multi_opt(valid_type): ) for error in errors: - log.debug(error) + log.warning(error) if errors: return False return True @@ -2016,7 +2014,7 @@ def apply_minion_config(overrides=None, # No ID provided. Will getfqdn save us? using_ip_for_id = False - if opts['id'] is None: + if not opts['id']: opts['id'], using_ip_for_id = get_id( opts, cache_minion_id=cache_minion_id) @@ -2129,7 +2127,7 @@ def apply_master_config(overrides=None, defaults=None): using_ip_for_id = False append_master = False - if opts.get('id') is None: + if not opts.get('id'): opts['id'], using_ip_for_id = get_id( opts, cache_minion_id=None) From 413c371ccd0c861d5c475798813ae66989c67d9b Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Fri, 1 Apr 2016 15:49:20 -0500 Subject: [PATCH 2/4] Properly support packages with blank "Release" param in pkg.latest_version --- salt/modules/yumpkg.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/salt/modules/yumpkg.py b/salt/modules/yumpkg.py index aa5c5381a77f..78924987e9e6 100644 --- a/salt/modules/yumpkg.py +++ b/salt/modules/yumpkg.py @@ -240,7 +240,10 @@ def _yum_pkginfo(output): cur['arch'], osarch) else: - if key == 'repoid': + if key == 'version': + # Suppport packages with no 'Release' parameter + value = value.rstrip('-') + elif key == 'repoid': # Installed packages show a '@' at the beginning value = value.lstrip('@') cur[key] = value From 083c477fd31a1431945a24669b156d5a2a1177ef Mon Sep 17 00:00:00 2001 From: rallytime Date: Fri, 1 Apr 2016 16:09:57 -0600 Subject: [PATCH 3/4] Use six.string_types in config default tuples --- salt/config.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/salt/config.py b/salt/config.py index eba83f34d1b5..defca4204082 100644 --- a/salt/config.py +++ b/salt/config.py @@ -3,9 +3,8 @@ All salt configuration loading and defaults should be in this module ''' -from __future__ import absolute_import - # Import python libs +from __future__ import absolute_import from __future__ import generators import glob import os @@ -62,7 +61,7 @@ 'daemons', 'flo') VALID_OPTS = { - 'master': (str, list), + 'master': (list, salt.ext.six.string_types), 'master_port': int, 'master_type': str, 'master_finger': str, @@ -144,7 +143,7 @@ 'grains': dict, 'permissive_pki_access': bool, 'default_include': str, - 'update_url': (bool, str), + 'update_url': (bool, salt.ext.six.string_types), 'update_restart_services': list, 'retry_dns': float, 'recon_max': float, @@ -210,7 +209,7 @@ 'ping_on_rotate': bool, 'peer': dict, 'preserve_minion_cache': bool, - 'syndic_master': (str, list), + 'syndic_master': (salt.ext.six.string_types, list), 'runner_dirs': list, 'client_acl': dict, 'client_acl_blacklist': dict, @@ -219,8 +218,8 @@ 'token_expire': int, 'file_recv': bool, 'file_recv_max_size': int, - 'file_ignore_regex': (list, str), - 'file_ignore_glob': (list, str), + 'file_ignore_regex': (list, salt.ext.six.string_types), + 'file_ignore_glob': (list, salt.ext.six.string_types), 'fileserver_backend': list, 'fileserver_followsymlinks': bool, 'fileserver_ignoresymlinks': bool, From 0491513204af4e4fb272099e502cf9913d90d548 Mon Sep 17 00:00:00 2001 From: rallytime Date: Fri, 1 Apr 2016 17:12:39 -0600 Subject: [PATCH 4/4] Don't be so explicit. Just use string_types. --- salt/config.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/salt/config.py b/salt/config.py index defca4204082..193839251da4 100644 --- a/salt/config.py +++ b/salt/config.py @@ -61,7 +61,7 @@ 'daemons', 'flo') VALID_OPTS = { - 'master': (list, salt.ext.six.string_types), + 'master': (list, string_types), 'master_port': int, 'master_type': str, 'master_finger': str, @@ -143,7 +143,7 @@ 'grains': dict, 'permissive_pki_access': bool, 'default_include': str, - 'update_url': (bool, salt.ext.six.string_types), + 'update_url': (bool, string_types), 'update_restart_services': list, 'retry_dns': float, 'recon_max': float, @@ -209,7 +209,7 @@ 'ping_on_rotate': bool, 'peer': dict, 'preserve_minion_cache': bool, - 'syndic_master': (salt.ext.six.string_types, list), + 'syndic_master': (string_types, list), 'runner_dirs': list, 'client_acl': dict, 'client_acl_blacklist': dict, @@ -218,8 +218,8 @@ 'token_expire': int, 'file_recv': bool, 'file_recv_max_size': int, - 'file_ignore_regex': (list, salt.ext.six.string_types), - 'file_ignore_glob': (list, salt.ext.six.string_types), + 'file_ignore_regex': (list, string_types), + 'file_ignore_glob': (list, string_types), 'fileserver_backend': list, 'fileserver_followsymlinks': bool, 'fileserver_ignoresymlinks': bool,