appmenus: support per-vm appmenus templates

Add support for multiple appmenus templates dir. Specifically,
TemplateBasedVM can have own appmenus templates (for example extracted
from ~/.local/share/applications), which extend/override those from
the template. Technically, each VM have now a list of appmenus template
directories (not a single dir), which are searched in relevance order
(up in 'template' hierarhy).

This is especially useful if one install and application in
TemplateBasedVM for example in user home or /usr/local. This makes it
easier to add such application to the menu.

For this change to work properly, there is a need for change in
qubes.GetAppmenus service on the VM side, to not report applications
installed on /, if changes in / are not persistent there (i.e.
TemplateBasedVM). Otherwise applications installed in templates will be
retrieved multiple times, wasting time and disk space. But updating VM
later should clean this up.

Fixes QubesOS/qubes-issues#4152

qubesappmenus/__init__.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python2
+#!/usr/bin/python3
 #
 # The Qubes OS Project, http://www.qubes-os.org
 #
@@ -27,6 +27,8 @@
 import shutil
 import dbus
 import logging
+
+import itertools
 import pkg_resources
 import xdg.BaseDirectory
 
@@ -64,28 +66,35 @@ class AppmenusPaths:
 
 
 class Appmenus(object):
-    def templates_dir(self, vm):
+    def templates_dirs(self, vm):
         """
 
         :type vm: qubes.vm.qubesvm.QubesVM
         """
-        if vm.updateable:
-            return os.path.join(basedir, vm.name,
-                AppmenusSubdirs.templates_subdir)
-        elif hasattr(vm, 'template'):
-            return self.templates_dir(vm.template)
-        else:
-            return None
-
-    def template_icons_dir(self, vm):
+        dirs = []
+        my_dir = os.path.join(basedir, vm.name,
+            AppmenusSubdirs.templates_subdir)
+        dirs.append(my_dir)
+        if hasattr(vm, 'template'):
+            dirs.extend(self.templates_dirs(vm.template))
+        return dirs
+
+    def template_icons_dirs(self, vm):
         '''Directory for not yet colore icons'''
-        if vm.updateable:
-            return os.path.join(basedir, vm.name,
-                AppmenusSubdirs.template_icons_subdir)
-        elif hasattr(vm, 'template'):
-            return self.template_icons_dir(vm.template)
-        else:
-            return None
+        dirs = []
+        my_dir = os.path.join(basedir, vm.name,
+            AppmenusSubdirs.template_icons_subdir)
+        dirs.append(my_dir)
+        if hasattr(vm, 'template'):
+            dirs.extend(self.template_icons_dirs(vm.template))
+        return dirs
+
+    def template_for_file(self, template_dirs, name):
+        '''Find first template named *name* in *template_dirs*'''
+        for d in template_dirs:
+            path = os.path.join(d, name)
+            if os.path.exists(path):
+                return path
 
     def appmenus_dir(self, vm):
         '''Desktop files generated for particular VM'''
@@ -136,28 +145,41 @@ def write_desktop_file(self, vm, source, destination_path, dispvm=False):
             replace("%VMDIR%", os.path.join(basedir, vm.name)).\
             replace("%XDGICON%", icon)
         if os.path.exists(destination_path):
-            current_dest = open(destination_path).read()
-            if current_dest == data:
-                return False
+            with open(destination_path) as dest_f:
+                current_dest = dest_f.read()
+                if current_dest == data:
+                    return False
         with open(destination_path, "w") as f:
             f.write(data)
         return True
 
+    def get_available_filenames(self, vm):
+        '''Yield filenames of available .desktop files'''
+        templates_dirs = self.templates_dirs(vm)
+        templates_dirs = (x for x in templates_dirs if os.path.isdir(x))
+        if not templates_dirs:
+            return
+
+        listed = set()
+        for template_dir in templates_dirs:
+            for filename in os.listdir(template_dir):
+                if filename in listed:
+                    continue
+                listed.add(filename)
+                yield os.path.join(template_dir, filename)
+
     def get_available(self, vm):
         # TODO icon path (#2885)
-        templates_dir = self.templates_dir(vm)
-        if templates_dir is None or not os.path.isdir(templates_dir):
-            return
-        for filename in os.listdir(templates_dir):
-            with open(os.path.join(templates_dir, filename)) as file:
+        for filename in self.get_available_filenames(vm):
+            with open(filename) as file:
                 name = None
                 for line in file:
                     if line.startswith('Name=%VMNAME%: '):
                         name = line.partition('Name=%VMNAME%: ')[2].strip()
                         break
             assert name is not None, \
                 'template {!r} does not contain name'.format(filename)
-            yield (filename, name)
+            yield (os.path.basename(filename), name)
 
     def appmenus_create(self, vm, force=False, refresh_cache=True):
         """Create/update .desktop files
@@ -174,7 +196,8 @@ def appmenus_create(self, vm, force=False, refresh_cache=True):
         if vm.klass == 'DispVM':
             return
 
-        vm.log.info("Creating appmenus")
+        if hasattr(vm, 'log'):
+            vm.log.info("Creating appmenus")
         appmenus_dir = self.appmenus_dir(vm)
         if not os.path.exists(appmenus_dir):
             os.makedirs(appmenus_dir)
@@ -191,23 +214,20 @@ def appmenus_create(self, vm, force=False, refresh_cache=True):
                 dispvm):
             anything_changed = True
             directory_changed = True
-        templates_dir = self.templates_dir(vm)
-        if os.path.exists(templates_dir):
-            appmenus = os.listdir(templates_dir)
-        else:
-            appmenus = []
+        appmenus = list(self.get_available_filenames(vm))
         changed_appmenus = []
         if os.path.exists(self.whitelist_path(vm)):
             whitelist = [x.rstrip() for x in open(self.whitelist_path(vm))]
-            appmenus = [x for x in appmenus if x in whitelist]
+            appmenus = [x for x in appmenus if os.path.basename(x) in whitelist]
 
         for appmenu in appmenus:
+            appmenu_basename = os.path.basename(appmenu)
             if self.write_desktop_file(vm,
-                    os.path.join(templates_dir, appmenu),
+                    appmenu,
                     os.path.join(appmenus_dir,
-                        '-'.join((vm.name, appmenu))),
+                        '-'.join((vm.name, appmenu_basename))),
                     dispvm):
-                changed_appmenus.append(appmenu)
+                changed_appmenus.append(appmenu_basename)
         if self.write_desktop_file(vm,
                 pkg_resources.resource_string(
                     __name__, 'qubes-vm-settings.desktop.template'
@@ -219,7 +239,7 @@ def appmenus_create(self, vm, force=False, refresh_cache=True):
         if changed_appmenus:
             anything_changed = True
 
-        target_appmenus = ['-'.join((vm.name, x))
+        target_appmenus = ['-'.join((vm.name, os.path.basename(x)))
             for x in appmenus + ['qubes-vm-settings.desktop']]
 
         # remove old entries
@@ -322,13 +342,11 @@ def appmenus_remove(self, vm, refresh_cache=True):
                 subprocess.call(['kbuildsycoca' +
                                  os.environ.get('KDE_SESSION_VERSION', '4')])
 
-    def appicons_create(self, vm, srcdir=None, force=False):
+    def appicons_create(self, vm, srcdirs=(), force=False):
         """Create/update applications icons"""
-        if srcdir is None:
-            srcdir = self.template_icons_dir(vm)
-        if srcdir is None:
-            return
-        if not os.path.exists(srcdir):
+        if not srcdirs:
+            srcdirs = self.template_icons_dirs(vm)
+        if not srcdirs:
             return
 
         if vm.features.get('internal', False):
@@ -353,13 +371,16 @@ def appicons_create(self, vm, srcdir=None, force=False):
             expected_icons = [os.path.splitext(x)[0] + '.png'
                 for x in whitelist]
         else:
-            expected_icons = os.listdir(srcdir)
-
-        for icon in os.listdir(srcdir):
-            if icon not in expected_icons:
+            expected_icons = list(itertools.chain.from_iterable(
+                os.listdir(srcdir)
+                for srcdir in srcdirs
+                if os.path.exists(srcdir)))
+
+        for icon in expected_icons:
+            src_icon = self.template_for_file(srcdirs, icon)
+            if not src_icon:
                 continue
 
-            src_icon = os.path.join(srcdir, icon)
             dst_icon = os.path.join(dstdir, icon)
             if not os.path.exists(dst_icon) or force or \
                     os.path.getmtime(src_icon) > os.path.getmtime(dst_icon):
@@ -391,15 +412,20 @@ def appmenus_init(self, vm, src=None):
                 src = vm.template
             except AttributeError:
                 pass
-        if vm.updateable and src is None:
-            os.makedirs(self.templates_dir(vm))
-            os.makedirs(self.template_icons_dir(vm))
+        own_templates_dir = os.path.join(basedir, vm.name,
+                AppmenusSubdirs.templates_subdir)
+        own_template_icons_dir = os.path.join(basedir, vm.name,
+                AppmenusSubdirs.template_icons_subdir)
+        if src is None:
+            os.makedirs(own_templates_dir)
+            os.makedirs(os.path.join(basedir, vm.name,
+                AppmenusSubdirs.template_icons_subdir))
 
         if vm.virt_mode == 'hvm' and src is None:
             vm.log.info("Creating appmenus directory: {0}".format(
-                self.templates_dir(vm)))
+                own_templates_dir))
             shutil.copy(AppmenusPaths.appmenu_start_hvm_template,
-                        self.templates_dir(vm))
+                        own_templates_dir)
 
         source_whitelist_filename = 'vm-' + AppmenusSubdirs.whitelist
         if src and os.path.exists(
@@ -423,12 +449,14 @@ def appmenus_init(self, vm, src=None):
                         os.path.join(basedir, vm.name, whitelist))
 
             vm.log.info("Creating/copying appmenus templates")
-            if os.path.isdir(self.templates_dir(src)):
-                shutil.copytree(self.templates_dir(src),
-                                self.templates_dir(vm))
-            if os.path.isdir(self.template_icons_dir(src)):
-                shutil.copytree(self.template_icons_dir(src),
-                                self.template_icons_dir(vm))
+            for src_dir in self.templates_dirs(src):
+                if os.path.isdir(src_dir):
+                    shutil.copytree(src_dir,
+                                    own_templates_dir)
+            for src_dir in self.template_icons_dirs(src):
+                if os.path.isdir(src_dir):
+                    shutil.copytree(src_dir,
+                                    own_template_icons_dir)
 
     def set_default_whitelist(self, vm, applications_list):
         '''Update default applications list for VMs created on this template

qubesappmenus/receive.py

@@ -280,24 +280,26 @@ def process_appmenus_templates(appmenusext, vm, appmenus):
     legacy_appmenus = vm.features.check_with_template(
         'appmenus-legacy', False)
 
-    if not os.path.exists(appmenusext.templates_dir(vm)):
-        os.makedirs(appmenusext.templates_dir(vm))
+    templates_dir = appmenusext.templates_dirs(vm)[0]
+    if not os.path.exists(templates_dir):
+        os.makedirs(templates_dir)
 
-    if not os.path.exists(appmenusext.template_icons_dir(vm)):
-        os.makedirs(appmenusext.template_icons_dir(vm))
+    template_icons_dir = appmenusext.template_icons_dirs(vm)[0]
+    if not os.path.exists(template_icons_dir):
+        os.makedirs(template_icons_dir)
 
     if vm.virt_mode == 'hvm':
         if not os.path.exists(os.path.join(
-                appmenusext.templates_dir(vm),
+                templates_dir,
                 os.path.basename(
                     qubesappmenus.AppmenusPaths.appmenu_start_hvm_template))):
             shutil.copy(qubesappmenus.AppmenusPaths.appmenu_start_hvm_template,
-                appmenusext.templates_dir(vm))
+                templates_dir)
 
 
     for appmenu_name in appmenus.keys():
         appmenu_path = os.path.join(
-            appmenusext.templates_dir(vm),
+            templates_dir,
             appmenu_name) + '.desktop'
         if os.path.exists(appmenu_path):
             vm.log.info("Updating {0}".format(appmenu_name))
@@ -309,7 +311,7 @@ def process_appmenus_templates(appmenusext, vm, appmenus):
         # TODO     new_appmenus[appmenu_name].pop('Icon', None)
         if 'Icon' in appmenus[appmenu_name]:
             # the following line is used for time comparison
-            icondest = os.path.join(appmenusext.template_icons_dir(vm),
+            icondest = os.path.join(template_icons_dir,
                                     appmenu_name + '.png')
 
             try:
@@ -334,13 +336,13 @@ def process_appmenus_templates(appmenusext, vm, appmenus):
             appmenus[appmenu_name], legacy_appmenus)
 
     # Delete appmenus of removed applications
-    for appmenu_file in os.listdir(appmenusext.templates_dir(vm)):
+    for appmenu_file in os.listdir(templates_dir):
         if not appmenu_file.endswith('.desktop'):
             continue
 
         if appmenu_file[:-len('.desktop')] not in appmenus:
             vm.log.info("Removing {0}".format(appmenu_file))
-            os.unlink(os.path.join(appmenusext.templates_dir(vm),
+            os.unlink(os.path.join(templates_dir,
                 appmenu_file))
 
     os.umask(old_umask)

qubesappmenus/tests.py

@@ -87,38 +87,42 @@ def setUp(self):
         self.ext = qubesappmenus.Appmenus()
         self.basedir = os.path.expanduser('~/.local/share/qubes-appmenus')
 
-    def test_000_templates_dir(self):
+    def test_000_templates_dirs(self):
         self.assertEqual(
-            self.ext.templates_dir(self.standalone),
-            os.path.join(self.basedir,
-                self.standalone.name, 'apps.templates')
+            self.ext.templates_dirs(self.standalone),
+            [os.path.join(self.basedir,
+                self.standalone.name, 'apps.templates')]
         )
         self.assertEqual(
-            self.ext.templates_dir(self.template),
-            os.path.join(self.basedir,
-                self.template.name, 'apps.templates')
+            self.ext.templates_dirs(self.template),
+            [os.path.join(self.basedir,
+                self.template.name, 'apps.templates')]
         )
         self.assertEqual(
-            self.ext.templates_dir(self.appvm),
-            os.path.join(self.basedir,
-                self.template.name, 'apps.templates')
+            self.ext.templates_dirs(self.appvm),
+            [os.path.join(self.basedir,
+                self.appvm.name, 'apps.templates'),
+             os.path.join(self.basedir,
+                self.template.name, 'apps.templates')]
         )
 
     def test_001_template_icons_dir(self):
         self.assertEqual(
-            self.ext.template_icons_dir(self.standalone),
-            os.path.join(self.basedir,
-                self.standalone.name, 'apps.tempicons')
+            self.ext.template_icons_dirs(self.standalone),
+            [os.path.join(self.basedir,
+                self.standalone.name, 'apps.tempicons')]
         )
         self.assertEqual(
-            self.ext.template_icons_dir(self.template),
-            os.path.join(self.basedir,
-                self.template.name, 'apps.tempicons')
+            self.ext.template_icons_dirs(self.template),
+            [os.path.join(self.basedir,
+                self.template.name, 'apps.tempicons')]
         )
         self.assertEqual(
-            self.ext.template_icons_dir(self.appvm),
-            os.path.join(self.basedir,
-                self.template.name, 'apps.tempicons')
+            self.ext.template_icons_dirs(self.appvm),
+            [os.path.join(self.basedir,
+                self.appvm.name, 'apps.tempicons'),
+             os.path.join(self.basedir,
+                self.template.name, 'apps.tempicons')]
         )
 
     def test_002_appmenus_dir(self):