Wait on network to be available before trying to run backups against remote repos

src/vorta/network_status/abc.py

@@ -2,8 +2,10 @@
 from datetime import datetime
 from typing import List, NamedTuple, Optional
 
+from PyQt6.QtCore import QObject, pyqtSignal
 
-class NetworkStatusMonitor:
+
+class NetworkStatusMonitor(QObject):
     @classmethod
     def get_network_status_monitor(cls) -> 'NetworkStatusMonitor':
         if sys.platform == 'darwin':
@@ -22,10 +24,22 @@ def get_network_status_monitor(cls) -> 'NetworkStatusMonitor':
             except (UnsupportedException, DBusException):
                 return NullNetworkStatusMonitor()
 
+    network_status_changed = pyqtSignal(bool, name="networkStatusChanged")
+
+    def __init__(self, parent=None):
+        super().__init__(parent)
+
     def is_network_status_available(self):
         """Is the network status really available, and not just a dummy implementation?"""
         return type(self) is not NetworkStatusMonitor
 
+    def is_network_active(self) -> bool:
+        """Is there an active network connection.
+
+        True signals that the network is up. The internet may still not be reachable though.
+        """
+        raise NotImplementedError()
+
     def is_network_metered(self) -> bool:
         """Is the currently connected network a metered connection?"""
         raise NotImplementedError()
@@ -47,6 +61,12 @@ class SystemWifiInfo(NamedTuple):
 class NullNetworkStatusMonitor(NetworkStatusMonitor):
     """Dummy implementation, in case we don't have one for current platform."""
 
+    def __init__(self):
+        super().__init__()
+
+    def is_network_active(self):
+        return True
+
     def is_network_status_available(self):
         return False
 

src/vorta/network_status/darwin.py

@@ -9,6 +9,9 @@
 
 
 class DarwinNetworkStatus(NetworkStatusMonitor):
+    def __init__(self):
+        super().__init__()
+
     def is_network_metered(self) -> bool:
         interface: CWInterface = self._get_wifi_interface()
 
@@ -25,6 +28,10 @@ def is_network_metered(self) -> bool:
 
         return is_ios_hotspot or any(is_network_metered_with_android(d) for d in get_network_devices())
 
+    def is_network_active(self):
+        # Not yet implemented
+        return True
+
     def get_current_wifi(self) -> Optional[str]:
         """
         Get current SSID or None if Wi-Fi is off.

src/vorta/network_status/network_manager.py

@@ -4,7 +4,7 @@
 from typing import Any, List, Mapping, NamedTuple, Optional
 
 from PyQt6 import QtDBus
-from PyQt6.QtCore import QObject, QVersionNumber
+from PyQt6.QtCore import QObject, QVersionNumber, pyqtSignal, pyqtSlot
 
 from vorta.network_status.abc import NetworkStatusMonitor, SystemWifiInfo
 
@@ -13,7 +13,9 @@
 
 class NetworkManagerMonitor(NetworkStatusMonitor):
     def __init__(self, nm_adapter: 'NetworkManagerDBusAdapter' = None):
+        super().__init__()
         self._nm = nm_adapter or NetworkManagerDBusAdapter.get_system_nm_adapter()
+        self._nm.network_status_changed.connect(self.network_status_changed)
 
     def is_network_metered(self) -> bool:
         try:
@@ -25,6 +27,13 @@ def is_network_metered(self) -> bool:
             logger.exception("Failed to check if network is metered, assuming it isn't")
             return False
 
+    def is_network_active(self):
+        try:
+            return self._nm.is_network_connected()
+        except DBusException:
+            logger.exception("Failed to check connectivity state. Assuming connected")
+            return True
+
     def get_current_wifi(self) -> Optional[str]:
         # Only check the primary connection. VPN over WiFi will still show the WiFi as Primary Connection.
         # We don't check all active connections, as NM won't disable WiFi when connecting a cable.
@@ -98,10 +107,20 @@ class NetworkManagerDBusAdapter(QObject):
 
     BUS_NAME = 'org.freedesktop.NetworkManager'
     NM_PATH = '/org/freedesktop/NetworkManager'
+    INTERFACE_NAME = 'org.freedesktop.NetworkManager'
+    # Use the NMState everywhere in lieu of Connected. There is no change signal for
+    # Connected and it appears that the connected state changes after the state change.
+    # i.e. immediately asking for current connectivity can return the old value
+    SIGNAL_NAME = 'StateChanged'
+
+    network_status_changed = pyqtSignal(bool, name="networkStatusChanged")
 
     def __init__(self, parent, bus):
         super().__init__(parent)
         self._bus = bus
+        self._bus.connect(
+            self.BUS_NAME, self.NM_PATH, self.INTERFACE_NAME, self.SIGNAL_NAME, 'u', self.networkStateChanged
+        )
         self._nm = self._get_iface(self.NM_PATH, 'org.freedesktop.NetworkManager')
 
     @classmethod
@@ -114,6 +133,12 @@ def get_system_nm_adapter(cls) -> 'NetworkManagerDBusAdapter':
             raise UnsupportedException("Can't connect to NetworkManager")
         return nm_adapter
 
+    @pyqtSlot("unsigned int")
+    def networkStateChanged(self, state):
+        logger.debug(f'network state changed: {state}')
+        # https://www.networkmanager.dev/docs/api/latest/nm-dbus-types.html#NMState
+        self.network_status_changed.emit(_is_network_connected(NMState(state)))
+
     def isValid(self):
         if not self._nm.isValid():
             return False
@@ -126,6 +151,12 @@ def isValid(self):
             return False
         return True
 
+    def is_network_connected(self) -> bool:
+        return _is_network_connected(self.get_network_state())
+
+    def get_network_state(self) -> 'NMState':
+        return NMState(read_dbus_property(self._nm, 'State'))
+
     def get_primary_connection_path(self) -> Optional[str]:
         return read_dbus_property(self._nm, 'PrimaryConnection')
 
@@ -155,6 +186,16 @@ def _get_iface(self, path, interface) -> QtDBus.QDBusInterface:
         return QtDBus.QDBusInterface(self.BUS_NAME, path, interface, self._bus)
 
 
+def _is_network_connected(state: 'NMState') -> bool:
+    # We treat site and global as connected because having a default route means you
+    # can reach something. This might need to include LOCAL eventually depending on use
+    # cases
+    return state in (
+        NMState.NM_STATE_CONNECTED_SITE,
+        NMState.NM_STATE_CONNECTED_GLOBAL,
+    )
+
+
 def read_dbus_property(obj, property):
     # QDBusInterface.property() didn't work for some reason
     props = QtDBus.QDBusInterface(obj.service(), obj.path(), 'org.freedesktop.DBus.Properties', obj.connection())
@@ -186,3 +227,16 @@ class NMDeviceType(Enum):
     # Only the types we care about
     UNKNOWN = 0
     WIFI = 2
+
+
+class NMState(Enum):
+    """https://www.networkmanager.dev/docs/api/latest/nm-dbus-types.html#NMState"""
+
+    NM_STATE_UNKNOWN = 0
+    NM_STATE_DISABLED = 10
+    NM_STATE_DISCONNECTED = 20
+    NM_STATE_DISCONNECTING = 30
+    NM_STATE_CONNECTING = 40
+    NM_STATE_CONNECTED_LOCAL = 50
+    NM_STATE_CONNECTED_SITE = 60
+    NM_STATE_CONNECTED_GLOBAL = 70

src/vorta/scheduler.py

@@ -19,7 +19,7 @@
 from vorta.i18n import translate
 from vorta.notifications import VortaNotifications
 from vorta.store.models import BackupProfileModel, EventLogModel
-from vorta.utils import borg_compat
+from vorta.utils import borg_compat, get_network_status_monitor
 
 logger = logging.getLogger(__name__)
 
@@ -62,6 +62,11 @@ def __init__(self):
         # connect signals
         self.app.backup_finished_event.connect(lambda res: self.set_timer_for_profile(res['params']['profile_id']))
 
+        # Connect to network manager to monitor net status
+        self.net_status = get_network_status_monitor()
+        self.net_status.network_status_changed.connect(self.networkStatusChanged)
+        self._net_up = self.net_status.is_network_active()
+
         # connect to `systemd-logind` to receive sleep/resume events
         # The signal `PrepareForSleep` will be emitted before and after hibernation.
         service = "org.freedesktop.login1"
@@ -79,6 +84,17 @@ def __init__(self):
     def loginSuspendNotify(self, suspend: bool):
         if not suspend:
             logger.debug("Got login suspend/resume notification")
+            # Defensively refetch in case the network status didn't arrive
+            self._net_up = self.net_status.is_network_active()
+            self.reload_all_timers()
+
+    @QtCore.pyqtSlot(bool)
+    def networkStatusChanged(self, up: bool):
+        reload = self._net_up != up
+        self._net_up = up
+        logger.debug(f"network status up={up}")
+        if reload:
+            logger.info("updating schedule due to network status change")
             self.reload_all_timers()
 
     def tr(self, *args, **kwargs):
@@ -294,9 +310,10 @@ def set_timer_for_profile(self, profile_id: int):
 
             logger.debug('Last run time: %s', last_time)
 
+            needs_network = profile.repo is not None and profile.repo.is_remote_repo()
             # handle missing of a scheduled time
             if next_time <= dt.now():
-                if profile.schedule_make_up_missed:
+                if profile.schedule_make_up_missed and (self._net_up or not needs_network):
                     self.lock.release()
                     try:
                         logger.debug(
@@ -309,6 +326,8 @@ def set_timer_for_profile(self, profile_id: int):
                         self.lock.acquire()  # with-statement will try to release
 
                     return  # create_backup will lead to a call to this method
+                elif profile.schedule_make_up_missed and not self._net_up and needs_network:
+                    logger.debug('Skipping catchup %s (%s), the network is not available', profile.name, profile.id)
 
                 # calculate next time from now
                 if profile.schedule_mode == 'interval':
@@ -364,7 +383,15 @@ def set_timer_for_profile(self, profile_id: int):
     def reload_all_timers(self):
         logger.debug('Refreshing all scheduler timers')
         for profile in BackupProfileModel.select():
-            self.set_timer_for_profile(profile.id)
+            # Only set a timer for the profile if the network is actually up
+            if profile.repo is None:
+                logger.debug("nothing scheduled for %s because of unset repo", profile.id)
+            elif not profile.repo.is_remote_repo() or self._net_up:
+                logger.debug("scheduling %s", profile.id)
+                self.set_timer_for_profile(profile.id)
+            else:
+                logger.debug("Network is down, not scheduling %s", profile.id)
+                self.remove_job(profile.id)
 
     def next_job(self):
         now = dt.now()