Use XDG Base directories for settings, cache and runtime data

Jenkinsfile

@@ -57,7 +57,7 @@ pipeline {
                     sh 'mkdir -p $HOME/core/$BRANCH_ALIAS/allure'
                     sh 'mkdir -p $HOME/core/$BRANCH_ALIAS/mycroft-logs'
                     sh 'docker run \
-                        -v "$HOME/voight-kampff/identity:/root/.mycroft/identity" \
+                        -v "$HOME/voight-kampff/identity:/root/.config/mycroft/identity" \
                         -v "$HOME/core/$BRANCH_ALIAS/allure:/root/allure" \
                         -v "$HOME/core/$BRANCH_ALIAS/mycroft-logs:/var/log/mycroft" \
                         --label build=${JOB_NAME} \

README.md

@@ -71,20 +71,20 @@ Mycroft is nothing without skills.  There are a handful of default skills that a
 
 ## Pairing Information
 Pairing information generated by registering with Home is stored in:
-`~/.mycroft/identity/identity2.json` <b><-- DO NOT SHARE THIS WITH OTHERS!</b>
+`~/.config/mycroft/identity/identity2.json` <b><-- DO NOT SHARE THIS WITH OTHERS!</b>
 
 ## Configuration
 Mycroft configuration consists of 4 possible locations:
-- `mycroft-core/mycroft/configuration/mycroft.conf`(Defaults)
+- `mycroft-core/mycroft/configuration/mycroft.conf` (Defaults)
 - [Mycroft Home](https://home.mycroft.ai) (Remote)
-- `/etc/mycroft/mycroft.conf`(Machine)
-- `$HOME/.mycroft/mycroft.conf`(User)
+- `/etc/mycroft/mycroft.conf` (Machine)
+- `$XDG_CONFIG_DIR/mycroft/mycroft.conf` (which is by default `$HOME/.config/mycroft/mycroft.conf`) (USER)
 
 When the configuration loader starts, it looks in these locations in this order, and loads ALL configurations. Keys that exist in multiple configuration files will be overridden by the last file to contain the value. This process results in a minimal amount being written for a specific device and user, without modifying default distribution files.
 
 ## Using Mycroft Without Home
 
-If you do not wish to use the Mycroft Home service, before starting Mycroft for the first time, create `$HOME/.mycroft/mycroft.conf` with the following contents:
+If you do not wish to use the Mycroft Home service, before starting Mycroft for the first time, create `$HOME/.config/mycroft/mycroft.conf` with the following contents:
 
 ```
 {

bin/mycroft-config

@@ -95,13 +95,13 @@ function validate_config_file() {
     return $result
 }
 
-_conf_file="~/.mycroft/mycroft.conf"
+_conf_file="~/.config/mycroft/mycroft.conf"
 function name_to_path() {
     case ${1} in
         "system")     _conf_file="/etc/mycroft/mycroft.conf" ;;
-        "user")       _conf_file=$(readlink -f ~/.mycroft/mycroft.conf) ;;
+        "user")       _conf_file=$(readlink -f ~/.config/mycroft/mycroft.conf) ;;
         "default")    _conf_file="$DIR/../mycroft/configuration/mycroft.conf" ;;
-        "remote")     _conf_file="/var/tmp/mycroft_web_cache.json" ;;
+        "remote")     _conf_file="$HOME/.cache/mycroft/web_cache.json" ;;
 
         *)
             echo "ERROR: Unknown name '${1}'."

mycroft/api/__init__.py

@@ -20,8 +20,6 @@
 from requests import HTTPError, RequestException
 
 from mycroft.configuration import Configuration
-from mycroft.configuration.config import DEFAULT_CONFIG, SYSTEM_CONFIG, \
-    USER_CONFIG
 from mycroft.identity import IdentityManager, identity_lock
 from mycroft.version import VersionManager
 from mycroft.util import get_arch, connected, LOG
@@ -49,12 +47,9 @@ class Api:
     def __init__(self, path):
         self.path = path
 
-        # Load the config, skipping the REMOTE_CONFIG since we are
+        # Load the config, skipping the remote config since we are
         # getting the info needed to get to it!
-        config = Configuration.get([DEFAULT_CONFIG,
-                                    SYSTEM_CONFIG,
-                                    USER_CONFIG],
-                                   cache=False)
+        config = Configuration.get(cache=False, remote=False)
         config_server = config.get("server")
         self.url = config_server.get("url")
         self.version = config_server.get("version")
@@ -238,9 +233,7 @@ def activate(self, state, token):
         platform_build = ""
 
         # load just the local configs to get platform info
-        config = Configuration.get([SYSTEM_CONFIG,
-                                    USER_CONFIG],
-                                   cache=False)
+        config = Configuration.get(cache=False, remote=False)
         if "enclosure" in config:
             platform = config.get("enclosure").get("platform", "unknown")
             platform_build = config.get("enclosure").get("platform_build", "")
@@ -262,9 +255,7 @@ def update_version(self):
         platform_build = ""
 
         # load just the local configs to get platform info
-        config = Configuration.get([SYSTEM_CONFIG,
-                                    USER_CONFIG],
-                                   cache=False)
+        config = Configuration.get(cache=False, remote=False)
         if "enclosure" in config:
             platform = config.get("enclosure").get("platform", "unknown")
             platform_build = config.get("enclosure").get("platform_build", "")

mycroft/client/enclosure/__main__.py

@@ -17,7 +17,7 @@
 This provides any "enclosure" specific functionality, for example GUI or
 control over the Mark-1 Faceplate.
 """
-from mycroft.configuration import LocalConf, SYSTEM_CONFIG
+from mycroft.configuration import Configuration
 from mycroft.util.log import LOG
 from mycroft.util import wait_for_exit_signal, reset_sigint_handler
 
@@ -70,8 +70,8 @@ def main(ready_hook=on_ready, error_hook=on_error, stopping_hook=on_stopping):
     only the GUI bus will be started.
     """
     # Read the system configuration
-    system_config = LocalConf(SYSTEM_CONFIG)
-    platform = system_config.get("enclosure", {}).get("platform")
+    config = Configuration.get(remote=False)
+    platform = config.get("enclosure", {}).get("platform")
 
     enclosure = create_enclosure(platform)
     if enclosure:

mycroft/client/enclosure/mark1/__init__.py

@@ -15,8 +15,10 @@
 import subprocess
 import time
 import sys
+import os
 from alsaaudio import Mixer
 from threading import Thread, Timer
+from xdg import BaseDirectory
 
 import serial
 
@@ -163,6 +165,9 @@ def process(self, data):
         if "unit.factory-reset" in data:
             self.bus.emit(Message("speak", {
                 'utterance': mycroft.dialog.get("reset to factory defaults")}))
+            subprocess.call(
+                'rm ~/.config/mycroft/identity/identity2.json',
+                shell=True)
             subprocess.call(
                 'rm ~/.mycroft/identity/identity2.json',
                 shell=True)

mycroft/client/speech/hotword_factory.py

@@ -28,10 +28,12 @@
 from shutil import rmtree
 from threading import Timer, Thread
 from urllib.error import HTTPError
+from xdg import BaseDirectory
 
 from petact import install_package
 
-from mycroft.configuration import Configuration, LocalConf, USER_CONFIG
+from mycroft.configuration import Configuration, LocalConf
+from mycroft.configuration.locations import OLD_USER_CONFIG
 from mycroft.util.log import LOG
 from mycroft.util.monotonic_event import MonotonicEvent
 from mycroft.util.plugins import load_plugin
@@ -191,7 +193,27 @@ def __init__(self, key_phrase="hey mycroft", config=None, lang="en-us"):
         from precise_runner import (
             PreciseRunner, PreciseEngine, ReadWriteStream
         )
-        local_conf = LocalConf(USER_CONFIG)
+
+        # We need to save to a writeable location, but the key we need
+        # might be stored in a different, unwriteable, location
+        # Make sure we pick the key we need from wherever it's located,
+        # but save to a writeable location only
+        local_conf = LocalConf(join(BaseDirectory.save_config_path('mycroft'),
+                                    'mycroft.conf'))
+
+        for dir in BaseDirectory.load_config_paths('mycroft'):
+            conf = LocalConf(join(dir, 'mycroft.conf'))
+            # If the current config contains the precise key use it,
+            # otherwise continue to the next file
+            if conf.get('precise', None) is not None:
+                local_conf['precise'] = conf.get('precise', None)
+                break
+
+        # If the key is not found yet, it might still exist on the old
+        # (deprecated) location
+        if local_conf.get('precise', None) is None:
+            local_conf = LocalConf(OLD_USER_CONFIG)
+
         if (local_conf.get('precise', {}).get('dist_url') ==
                 'http://bootstrap.mycroft.ai/artifacts/static/daily/'):
             del local_conf['precise']['dist_url']
@@ -249,7 +271,10 @@ def update_precise(self, precise_config):
 
     @property
     def folder(self):
-        return join(expanduser('~'), '.mycroft', 'precise')
+        old_path = join(expanduser('~'), '.mycroft', 'precise')
+        if os.path.isdir(old_path):
+            return old_path
+        return join(BaseDirectory.save_data_path('mycroft', 'precise'))
 
     @property
     def install_destination(self):
@@ -359,8 +384,23 @@ def found_wake_word(self, frame_data):
 class PorcupineHotWord(HotWordEngine):
     def __init__(self, key_phrase="hey mycroft", config=None, lang="en-us"):
         super(PorcupineHotWord, self).__init__(key_phrase, config, lang)
+
+        xdg_data_path = None
+
+        # Use the old path if available
+        old_path = join(expanduser('~'), '.mycroft', 'Porcupine')
+        if os.path.isdir(old_path):
+            xdg_data_path = old_path
+
+        # Otherwise use the new XDG config dirs
+        if xdg_data_path is None:
+            for dir in BaseDirectory.load_data_paths('mycroft', 'Porcupine'):
+                if os.path.isdir(dir):
+                    xdg_data_path = dir
+                    break
+
         porcupine_path = expanduser(self.config.get(
-            "porcupine_path", join('~', '.mycroft', 'Porcupine')))
+            "porcupine_path", xdg_data_path))
         keyword_file_paths = [expanduser(x.strip()) for x in self.config.get(
             "keyword_file_path", "hey_mycroft.ppn").split(',')]
         sensitivities = self.config.get("sensitivities", 0.5)

mycroft/client/text/text_client.py

@@ -15,6 +15,7 @@
 import sys
 import io
 from math import ceil
+from xdg import BaseDirectory
 
 from .gui_server import start_qml_gui
 
@@ -142,7 +143,7 @@ def handleNonAscii(text):
 ##############################################################################
 # Settings
 
-config_file = os.path.join(os.path.expanduser("~"), ".mycroft_cli.conf")
+filename = "mycroft_cli.conf"
 
 
 def load_mycroft_config(bus):
@@ -171,6 +172,25 @@ def load_settings():
     global max_log_lines
     global show_meter
 
+    config_file = None
+
+    # Old location
+    path = os.path.join(os.path.expanduser("~"), ".mycroft_cli.conf")
+    if os.path.isfile(path):
+        config_file = path
+
+    # Check XDG_CONFIG_DIR
+    if config_file is None:
+        for dir in BaseDirectory.load_config_paths('mycroft'):
+            file = os.path.join(dir, filename)
+            if os.path.isfile(file):
+                config_file = file
+                break
+
+    # Check /etc/mycroft
+    if config_file is None:
+        config_file = os.path.join("/etc/mycroft", filename)
+
     try:
         with io.open(config_file, 'r') as f:
             config = json.load(f)
@@ -196,6 +216,10 @@ def save_settings():
     config["show_last_key"] = show_last_key
     config["max_log_lines"] = max_log_lines
     config["show_meter"] = show_meter
+
+    config_file = os.path.join(
+        BaseDirectory.save_config_path("mycroft"), filename)
+
     with io.open(config_file, 'w') as f:
         f.write(str(json.dumps(config, ensure_ascii=False)))
 

mycroft/configuration/__init__.py

@@ -13,4 +13,3 @@
 # limitations under the License.
 #
 from .config import Configuration, LocalConf, RemoteConf
-from .locations import SYSTEM_CONFIG, USER_CONFIG

mycroft/configuration/config.py

@@ -17,15 +17,15 @@
 import re
 import json
 import inflection
-from os.path import exists, isfile
+from os.path import exists, isfile, join, expanduser, dirname
 from requests import RequestException
+from xdg import BaseDirectory
 
+from .locations import DEFAULT_CONFIG, OLD_USER_CONFIG, USER_CONFIG
+from .locations import SYSTEM_CONFIG
 from mycroft.util.json_helper import load_commented_json, merge_dict
 from mycroft.util.log import LOG
 
-from .locations import (DEFAULT_CONFIG, SYSTEM_CONFIG, USER_CONFIG,
-                        WEB_CONFIG_CACHE)
-
 
 def is_remote_list(values):
     """Check if list corresponds to a backend formatted collection of dicts
@@ -127,7 +127,8 @@ class RemoteConf(LocalConf):
     def __init__(self, cache=None):
         super(RemoteConf, self).__init__(None)
 
-        cache = cache or WEB_CONFIG_CACHE
+        cache = cache or join(BaseDirectory.save_cache_path('mycroft'),
+                              'web_cache.json')
         from mycroft.api import is_paired
         if not is_paired():
             self.load_local(cache)
@@ -173,7 +174,7 @@ class Configuration:
     __patch = {}  # Patch config that skills can update to override config
 
     @staticmethod
-    def get(configs=None, cache=True):
+    def get(configs=None, cache=True, remote=True):
         """Get configuration
 
         Returns cached instance if available otherwise builds a new
@@ -182,30 +183,72 @@ def get(configs=None, cache=True):
         Arguments:
             configs (list): List of configuration dicts
             cache (boolean): True if the result should be cached
+            remote (boolean): False if the Mycroft Home settings shouldn't
+                              be loaded
+
 
         Returns:
             (dict) configuration dictionary.
         """
         if Configuration.__config:
             return Configuration.__config
         else:
-            return Configuration.load_config_stack(configs, cache)
+            return Configuration.load_config_stack(configs, cache, remote)
 
     @staticmethod
-    def load_config_stack(configs=None, cache=False):
+    def load_config_stack(configs=None, cache=False, remote=True):
         """Load a stack of config dicts into a single dict
 
         Arguments:
             configs (list): list of dicts to load
             cache (boolean): True if result should be cached
-
+            remote (boolean): False if the Mycroft Home settings shouldn't
+                              be loaded
         Returns:
             (dict) merged dict of all configuration files
         """
         if not configs:
-            configs = [LocalConf(DEFAULT_CONFIG), RemoteConf(),
-                       LocalConf(SYSTEM_CONFIG), LocalConf(USER_CONFIG),
-                       Configuration.__patch]
+            configs = configs or []
+
+            # First use the patched config
+            configs.append(Configuration.__patch)
+
+            # Then use XDG config
+            # This includes both the user config and
+            # /etc/xdg/mycroft/mycroft.conf
+            for dir in BaseDirectory.load_config_paths('mycroft'):
+                configs.append(LocalConf(join(dir, 'mycroft.conf')))
+
+            # Then check the old user config
+            if isfile(OLD_USER_CONFIG):
+                LOG.warning(" ===============================================")
+                LOG.warning(" ==             DEPRECATION WARNING           ==")
+                LOG.warning(" ===============================================")
+                LOG.warning(" You still have a config file at " +
+                            OLD_USER_CONFIG)
+                LOG.warning(" Note that this location is deprecated and will" +
+                            " not be used in the future")
+                LOG.warning(" Please move it to " +
+                            BaseDirectory.save_config_path('mycroft'))
+                configs.append(LocalConf(OLD_USER_CONFIG))
+
+            # Then check the XDG user config
+            if isfile(USER_CONFIG):
+                configs.append(LocalConf(USER_CONFIG))
+
+            # Then use remote config
+            if remote:
+                configs.append(RemoteConf())
+
+            # Then use the system config (/etc/mycroft/mycroft.conf)
+            configs.append(LocalConf(SYSTEM_CONFIG))
+
+            # Then use the config that comes with the package
+            configs.append(LocalConf(DEFAULT_CONFIG))
+
+            # Make sure we reverse the array, as merge_dict will put every new
+            # file on top of the previous one
+            configs = reversed(configs)
         else:
             # Handle strings in stack
             for index, item in enumerate(configs):