Merge pull request #137 from ManiMatter/dev

Skip cleanup when qbit disconnected + small improvements

config/definitions.py

@@ -62,12 +62,21 @@
     exit()
 
 ########### Enrich setting variables
-if RADARR_URL:      RADARR_URL      += '/api/v3'
-if SONARR_URL:      SONARR_URL      += '/api/v3'
-if LIDARR_URL:      LIDARR_URL      += '/api/v1'
-if READARR_URL:     READARR_URL     += '/api/v1'
-if WHISPARR_URL:    WHISPARR_URL    += '/api/v3'
-if QBITTORRENT_URL: QBITTORRENT_URL += '/api/v2'
+if RADARR_URL:      RADARR_URL =        RADARR_URL.rstrip('/')      + '/api/v3'
+if SONARR_URL:      SONARR_URL =        SONARR_URL.rstrip('/')      + '/api/v3'
+if LIDARR_URL:      LIDARR_URL =        LIDARR_URL.rstrip('/')      + '/api/v1'
+if READARR_URL:     READARR_URL =       READARR_URL.rstrip('/')     + '/api/v1'
+if WHISPARR_URL:    WHISPARR_URL =      WHISPARR_URL.rstrip('/')    + '/api/v3'
+if QBITTORRENT_URL: QBITTORRENT_URL =   QBITTORRENT_URL.rstrip('/') + '/api/v2'
+
+RADARR_MIN_VERSION          = '5.3.6.8608'
+SONARR_MIN_VERSION          = '4.0.1.1131'
+LIDARR_MIN_VERSION          = None
+READARR_MIN_VERSION         = None
+WHISPARR_MIN_VERSION        = '2.0.0.548'
+QBITTORRENT_MIN_VERSION     = '4.3.0'
+
+SUPPORTED_ARR_APPS  = ['RADARR', 'SONARR', 'LIDARR', 'READARR', 'WHISPARR']
 
 ########### Add Variables to Dictionary
 settingsDict = {}

main.py

@@ -26,40 +26,12 @@ class Download_Sizes_Tracker:
     # Keeps track of the file sizes of the downloads
     def __init__(self, dict):
         self.dict = dict
-
-#
-
-async def getProtectedAndPrivateFromQbit(settingsDict):
-    # Returns two lists containing the hashes of Qbit that are either protected by tag, or are private trackers (if IGNORE_PRIVATE_TRACKERS is true)
-    protectedDownloadIDs = []
-    privateDowloadIDs = []
-    if settingsDict['QBITTORRENT_URL']:
-        # Fetch all torrents
-        qbitItems = await rest_get(settingsDict['QBITTORRENT_URL']+'/torrents/info',params={}, cookies=settingsDict['QBIT_COOKIE'])
-        # Fetch protected torrents (by tag)
-        for qbitItem in qbitItems:
-            if settingsDict['NO_STALLED_REMOVAL_QBIT_TAG'] in qbitItem.get('tags'):
-                protectedDownloadIDs.append(str.upper(qbitItem['hash']))
-        # Fetch private torrents
-        if settingsDict['IGNORE_PRIVATE_TRACKERS']:
-            for qbitItem in qbitItems:           
-                qbitItemProperties = await rest_get(settingsDict['QBITTORRENT_URL']+'/torrents/properties',params={'hash': qbitItem['hash']}, cookies=settingsDict['QBIT_COOKIE'])
-                qbitItem['is_private'] = qbitItemProperties.get('is_private', None) # Adds the is_private flag to qbitItem info for simplified logging
-                if qbitItemProperties.get('is_private', False):
-                    privateDowloadIDs.append(str.upper(qbitItem['hash']))
-        logger.debug('main/getProtectedAndPrivateFromQbit/qbitItems: %s', str([{"hash": str.upper(item["hash"]), "name": item["name"], "category": item["category"], "tags": item["tags"], "is_private": item.get("is_private", None)} for item in qbitItems]))
 
-    logger.debug('main/getProtectedAndPrivateFromQbit/protectedDownloadIDs: %s', str(protectedDownloadIDs))
-    logger.debug('main/getProtectedAndPrivateFromQbit/privateDowloadIDs: %s', str(privateDowloadIDs))   
-
-    return protectedDownloadIDs, privateDowloadIDs
-
 # Main function
 async def main(settingsDict):
 # Adds to settings Dict the instances that are actually configures
-    arrApplications  = ['RADARR', 'SONARR', 'LIDARR', 'READARR', 'WHISPARR']
     settingsDict['INSTANCES'] = []
-    for arrApplication in arrApplications:
+    for arrApplication in settingsDict['SUPPORTED_ARR_APPS']:
         if settingsDict[arrApplication + '_URL']:
             settingsDict['INSTANCES'].append(arrApplication)
 
@@ -81,12 +53,6 @@ async def main(settingsDict):
     showSettings(settingsDict)
 
     # Check Minimum Version and if instances are reachable and retrieve qbit cookie
-    settingsDict['RADARR_MIN_VERSION']   = '5.3.6.8608'
-    settingsDict['SONARR_MIN_VERSION']   = '4.0.1.1131'
-    settingsDict['LIDARR_MIN_VERSION']   = None
-    settingsDict['READARR_MIN_VERSION']  = None
-    settingsDict['WHISPARR_MIN_VERSION'] = '2.0.0.548'
-    settingsDict['QBITTORRENT_MIN_VERSION']  = '4.3.0'
     settingsDict = await instanceChecks(settingsDict)
 
     # Create qBit protection tag if not existing

src/jobs/remove_failed.py

@@ -1,4 +1,4 @@
-from src.utils.shared import (errorDetails, formattedQueueInfo, get_queue, privateTrackerCheck, protectedDownloadCheck, execute_checks, permittedAttemptsCheck, remove_download)
+from src.utils.shared import (errorDetails, formattedQueueInfo, get_queue, privateTrackerCheck, protectedDownloadCheck, execute_checks, permittedAttemptsCheck, remove_download, qBitOffline)
 import sys, os, traceback
 import logging, verboselogs
 logger = verboselogs.VerboseLogger(__name__)
@@ -10,6 +10,7 @@ async def remove_failed(settingsDict, BASE_URL, API_KEY, NAME, deleted_downloads
         queue = await get_queue(BASE_URL, API_KEY)
         logger.debug('remove_failed/queue IN: %s', formattedQueueInfo(queue))
         if not queue: return 0
+        if await qBitOffline(settingsDict, failType, NAME): return 0
         # Find items affected
         affectedItems = []
         for queueItem in queue['records']: 

src/jobs/remove_failed_imports.py

@@ -20,19 +20,31 @@ async def remove_failed_imports(settingsDict, BASE_URL, API_KEY, NAME, deleted_d
 
                 if queueItem['status'] == 'completed' \
                     and queueItem['trackedDownloadStatus'] == 'warning' \
-                    and (queueItem['trackedDownloadState'] == 'importPending' or queueItem['trackedDownloadState'] == 'importFailed' or  queueItem['trackedDownloadState'] == 'importBlocked'):
-
+                    and queueItem['trackedDownloadState'] in {'importPending', 'importFailed', 'importBlocked'}:
+
+                    # Find messages that find specified pattern and put them into a "removal_message" that will be displayed in the logger when removing the affected item
+                    removal_messages = ['Tracked Download State: ' + queueItem['trackedDownloadState']]
                     for statusMessage in queueItem['statusMessages']:
-                        if not settingsDict['FAILED_IMPORT_MESSAGE_PATTERNS'] or any(any(pattern in message for pattern in settingsDict['FAILED_IMPORT_MESSAGE_PATTERNS']) for message in statusMessage.get('messages', [])):
-                            affectedItems.append(queueItem)
+                        if not settingsDict['FAILED_IMPORT_MESSAGE_PATTERNS']: # No patterns defined - including all status messages in the removal_messages
+                            removal_messages.append ('Status Messages (All):')
+                            removal_messages.extend(f"- {msg}" for msg in statusMessage.get('messages', []))
                             break
 
+                        removal_messages.append ('Status Messages (matching specified patterns):')
+                        messages = statusMessage.get('messages', [])
+                        for message in messages:
+                            if any(pattern in message for pattern in settingsDict['FAILED_IMPORT_MESSAGE_PATTERNS']):
+                                removal_messages.append(f"- {message}")
+
+                queueItem['removal_messages'] = removal_messages
+                affectedItems.append(queueItem)
+
         affectedItems = await execute_checks(settingsDict, affectedItems, failType, BASE_URL, API_KEY, NAME, deleted_downloads, defective_tracker, privateDowloadIDs, protectedDownloadIDs, 
                                             addToBlocklist = True, 
                                             doPrivateTrackerCheck = False, 
                                             doProtectedDownloadCheck = True, 
                                             doPermittedAttemptsCheck = False,
-                                            extraParameters = ['keepTorrentForPrivateTrackers']
+                                            extraParameters = {'keepTorrentForPrivateTrackers': True}
                                             )
         return len(affectedItems)
     except Exception as error:

src/jobs/remove_metadata_missing.py

@@ -1,4 +1,4 @@
-from src.utils.shared import (errorDetails, formattedQueueInfo, get_queue, privateTrackerCheck, protectedDownloadCheck, execute_checks, permittedAttemptsCheck, remove_download)
+from src.utils.shared import (errorDetails, formattedQueueInfo, get_queue, privateTrackerCheck, protectedDownloadCheck, execute_checks, permittedAttemptsCheck, remove_download, qBitOffline)
 import sys, os, traceback
 import logging, verboselogs
 logger = verboselogs.VerboseLogger(__name__)
@@ -10,8 +10,9 @@ async def remove_metadata_missing(settingsDict, BASE_URL, API_KEY, NAME, deleted
         queue = await get_queue(BASE_URL, API_KEY)
         logger.debug('remove_metadata_missing/queue IN: %s', formattedQueueInfo(queue))
         if not queue: return 0
+        if await qBitOffline(settingsDict, failType, NAME): return 0
         # Find items affected
-        affectedItems = []
+        affectedItems = []       
         for queueItem in queue['records']: 
             if 'errorMessage' in queueItem and 'status' in queueItem:
                 if  queueItem['status'] == 'queued' and queueItem['errorMessage'] == 'qBittorrent is downloading metadata':

src/jobs/remove_missing_files.py

@@ -1,4 +1,4 @@
-from src.utils.shared import (errorDetails, formattedQueueInfo, get_queue, privateTrackerCheck, protectedDownloadCheck, execute_checks, permittedAttemptsCheck, remove_download)
+from src.utils.shared import (errorDetails, formattedQueueInfo, get_queue, privateTrackerCheck, protectedDownloadCheck, execute_checks, permittedAttemptsCheck, remove_download, qBitOffline)
 import sys, os, traceback
 import logging, verboselogs
 logger = verboselogs.VerboseLogger(__name__)
@@ -10,6 +10,7 @@ async def remove_missing_files(settingsDict, BASE_URL, API_KEY, NAME, deleted_do
         queue = await get_queue(BASE_URL, API_KEY)
         logger.debug('remove_missing_files/queue IN: %s', formattedQueueInfo(queue))
         if not queue: return 0
+        if await qBitOffline(settingsDict, failType, NAME): return 0
         # Find items affected
         affectedItems = []
         for queueItem in queue['records']: 

src/jobs/remove_slow.py

@@ -1,4 +1,4 @@
-from src.utils.shared import (errorDetails, formattedQueueInfo, get_queue, privateTrackerCheck, protectedDownloadCheck, execute_checks, permittedAttemptsCheck, remove_download)
+from src.utils.shared import (errorDetails, formattedQueueInfo, get_queue, privateTrackerCheck, protectedDownloadCheck, execute_checks, permittedAttemptsCheck, remove_download, qBitOffline)
 import sys, os, traceback
 import logging, verboselogs
 from src.utils.rest import (rest_get)
@@ -11,16 +11,10 @@ async def remove_slow(settingsDict, BASE_URL, API_KEY, NAME, deleted_downloads,
         queue = await get_queue(BASE_URL, API_KEY)
         logger.debug('remove_slow/queue IN: %s', formattedQueueInfo(queue))
         if not queue: return 0
+        if await qBitOffline(settingsDict, failType, NAME): return 0
         # Find items affected
         affectedItems = []
         alreadyCheckedDownloadIDs = []
-
-        if settingsDict['QBITTORRENT_URL']:
-            qBitConnectionStatus = (await rest_get(settingsDict['QBITTORRENT_URL']+'/sync/maindata', cookies=settingsDict['QBIT_COOKIE']))['server_state']['connection_status']
-            if qBitConnectionStatus == 'disconnected':
-                logger.warning('>>> qBittorrent is disconnected. Skipping %s queue cleaning failed on %s.',failType, NAME)
-                return 0
-
         for queueItem in queue['records']:
             if 'downloadId' in queueItem and 'size' in queueItem and 'sizeleft' in queueItem and 'status' in queueItem:
                 if queueItem['downloadId'] not in alreadyCheckedDownloadIDs:

src/jobs/remove_stalled.py

@@ -1,4 +1,4 @@
-from src.utils.shared import (errorDetails, formattedQueueInfo, get_queue, privateTrackerCheck, protectedDownloadCheck, execute_checks, permittedAttemptsCheck, remove_download)
+from src.utils.shared import (errorDetails, formattedQueueInfo, get_queue, privateTrackerCheck, protectedDownloadCheck, execute_checks, permittedAttemptsCheck, remove_download, qBitOffline)
 import sys, os, traceback
 import logging, verboselogs
 logger = verboselogs.VerboseLogger(__name__)
@@ -10,6 +10,7 @@ async def remove_stalled(settingsDict, BASE_URL, API_KEY, NAME, deleted_download
         queue = await get_queue(BASE_URL, API_KEY)
         logger.debug('remove_stalled/queue IN: %s', formattedQueueInfo(queue))
         if not queue: return 0
+        if await qBitOffline(settingsDict, failType, NAME): return 0
         # Find items affected
         affectedItems = []
         for queueItem in queue['records']: 

src/utils/loadScripts.py

@@ -18,15 +18,39 @@ def setLoggingFormat(settingsDict):
 
 
 async def getArrInstanceName(settingsDict, arrApp):
-    # Retrieves the names of the arr instances, and if not defined, sets a default
+    # Retrieves the names of the arr instances, and if not defined, sets a default (should in theory not be requried, since UI already enforces a value)
     try:
         if settingsDict[arrApp + '_URL']:
             settingsDict[arrApp + '_NAME'] = (await rest_get(settingsDict[arrApp + '_URL']+'/system/status', settingsDict[arrApp + '_KEY']))['instanceName']
     except:
-            settingsDict[arrApp + '_NAME'] = arrApp.capitalize()
+            settingsDict[arrApp + '_NAME'] = arrApp.title()
     return settingsDict
 
+async def getProtectedAndPrivateFromQbit(settingsDict):
+    # Returns two lists containing the hashes of Qbit that are either protected by tag, or are private trackers (if IGNORE_PRIVATE_TRACKERS is true)
+    protectedDownloadIDs = []
+    privateDowloadIDs = []
+    if settingsDict['QBITTORRENT_URL']:
+        # Fetch all torrents
+        qbitItems = await rest_get(settingsDict['QBITTORRENT_URL']+'/torrents/info',params={}, cookies=settingsDict['QBIT_COOKIE'])
+        # Fetch protected torrents (by tag)
+        for qbitItem in qbitItems:
+            if settingsDict['NO_STALLED_REMOVAL_QBIT_TAG'] in qbitItem.get('tags'):
+                protectedDownloadIDs.append(str.upper(qbitItem['hash']))
+        # Fetch private torrents
+        if settingsDict['IGNORE_PRIVATE_TRACKERS']:
+            for qbitItem in qbitItems:           
+                qbitItemProperties = await rest_get(settingsDict['QBITTORRENT_URL']+'/torrents/properties',params={'hash': qbitItem['hash']}, cookies=settingsDict['QBIT_COOKIE'])
+                qbitItem['is_private'] = qbitItemProperties.get('is_private', None) # Adds the is_private flag to qbitItem info for simplified logging
+                if qbitItemProperties.get('is_private', False):
+                    privateDowloadIDs.append(str.upper(qbitItem['hash']))
+        logger.debug('main/getProtectedAndPrivateFromQbit/qbitItems: %s', str([{"hash": str.upper(item["hash"]), "name": item["name"], "category": item["category"], "tags": item["tags"], "is_private": item.get("is_private", None)} for item in qbitItems]))
+
+    logger.debug('main/getProtectedAndPrivateFromQbit/protectedDownloadIDs: %s', str(protectedDownloadIDs))
+    logger.debug('main/getProtectedAndPrivateFromQbit/privateDowloadIDs: %s', str(privateDowloadIDs))   
 
+    return protectedDownloadIDs, privateDowloadIDs
+
 
 def showSettings(settingsDict):
     # Prints out the settings
@@ -69,9 +93,18 @@ def showSettings(settingsDict):
 
     for instance in settingsDict['INSTANCES']:
         if settingsDict[instance + '_URL']: 
-            logger.info('%s: %s', settingsDict[instance + '_NAME'], settingsDict[instance + '_URL'])   
+            logger.info(
+                    '%s%s: %s',
+                    instance.title(),
+                    f" ({settingsDict.get(instance + '_NAME')})" if settingsDict.get(instance + '_NAME') != instance.title() else "",
+                    (settingsDict[instance + '_URL']).split('/api')[0]
+                )
 
-    if settingsDict['QBITTORRENT_URL']: logger.info('qBittorrent: %s', settingsDict['QBITTORRENT_URL'])    
+    if settingsDict['QBITTORRENT_URL']: 
+        logger.info(
+            'qBittorrent: %s', 
+            (settingsDict['QBITTORRENT_URL']).split('/api')[0]
+        )    
 
     logger.info('') 
     return   
@@ -95,21 +128,35 @@ async def instanceChecks(settingsDict):
     # Check ARR-apps
     for instance in settingsDict['INSTANCES']:
         if settingsDict[instance + '_URL']:    
+            # Check instance is reachable
             try: 
-                await asyncio.get_event_loop().run_in_executor(None, lambda: requests.get(settingsDict[instance + '_URL']+'/system/status', params=None, headers={'X-Api-Key': settingsDict[instance + '_KEY']}, verify=settingsDict['SSL_VERIFICATION']))
+                response = await asyncio.get_event_loop().run_in_executor(None, lambda: requests.get(settingsDict[instance + '_URL']+'/system/status', params=None, headers={'X-Api-Key': settingsDict[instance + '_KEY']}, verify=settingsDict['SSL_VERIFICATION']))
+                response.raise_for_status()
             except Exception as error:
                 error_occured = True
-                logger.error('!! %s Error: !!', settingsDict[instance + '_NAME'])
-                logger.error(error)
-            if not error_occured:                
+                logger.error('!! %s Error: !!', instance.title())
+                logger.error('> %s', error)
+                if isinstance(error, requests.exceptions.HTTPError) and error.response.status_code == 401:
+                    logger.error ('> Have you configured %s correctly?', instance + '_KEY')
+
+            if not error_occured:  
+                # Check if network settings are pointing to the right Arr-apps
+                current_app = (await rest_get(settingsDict[instance + '_URL']+'/system/status', settingsDict[instance + '_KEY']))['appName']
+                if current_app.upper() != instance:
+                    error_occured = True
+                    logger.error('!! %s Error: !!', instance.title())                    
+                    logger.error('> Your %s points to a %s instance, rather than %s. Did you specify the wrong IP?', instance + '_URL', current_app, instance.title())
+
+            if not error_occured:
+                # Check minimum version requirements are met
                 current_version = (await rest_get(settingsDict[instance + '_URL']+'/system/status', settingsDict[instance + '_KEY']))['version']
                 if settingsDict[instance + '_MIN_VERSION']:
                     if version.parse(current_version) < version.parse(settingsDict[instance + '_MIN_VERSION']):
                         error_occured = True
-                        logger.error('!! %s Error: !!', settingsDict[instance + '_NAME'])
-                        logger.error('Please update %s to at least version %s. Current version: %s',  settingsDict[instance + '_MIN_VERSION'],current_version)
+                        logger.error('!! %s Error: !!', instance.title())
+                        logger.error('> Please update %s to at least version %s. Current version: %s', instance.title(), settingsDict[instance + '_MIN_VERSION'], current_version)
             if not error_occured:
-                logger.info('OK | %s', settingsDict[instance + '_NAME'])     
+                logger.info('OK | %s', instance.title())     
                 logger.debug('Current version of %s: %s', instance, current_version)  
 
     # Check Bittorrent
@@ -124,8 +171,8 @@ async def instanceChecks(settingsDict):
         except Exception as error:
             error_occured = True
             logger.error('!! %s Error: !!', 'qBittorrent')
-            logger.error(error)
-            logger.error('Details:')
+            logger.error('> %s', error)
+            logger.error('> Details:')
             logger.error(response.text)
 
         if not error_occured:
@@ -141,7 +188,7 @@ async def instanceChecks(settingsDict):
 
 
     if error_occured:
-        logger.warning('At least one instance was not reachable. Waiting for 60 seconds, then exiting Decluttarr.')      
+        logger.warning('At least one instance had a problem. Waiting for 60 seconds, then exiting Decluttarr.')      
         await asyncio.sleep(60)
         exit()