Cleaning up and renaming CallDeferred (#513)

* Cleaning up and renaming CallDeferred

- Cleans the @CallDeferred decorator's internals to behave equally with or without a task queue
- Added more debug to find bugs
- Renamed callDeferred to CallDeferred
- Added wrapper callDeferred with deprecation warning

* Some renamings

- Renamed _deferedTasks into _deferred_tasks (typo fix & PEP8)
- Renamed internal mkDefered into make_deferred (typo fix & PEP8)

* Removed "logger" and use "logging" again

IMHO it is better to log everything as usual. We can later add specific loggers.
I will explain this in next VIUR meetup.

* Use wrapped function instead of lambda to keep task-func name/attrs

* Fix typo DEFERED_TASK_CALLED --> DEFERRED_TASK_CALLED

Co-authored-by: Sven Eberth <se@mausbrand.de>

core/bones/file.py

@@ -1,15 +1,15 @@
 from viur.core.bones.treeleaf import TreeLeafBone
 from viur.core import request, conf, db
 from viur.core.utils import downloadUrlFor
-from viur.core.tasks import callDeferred
+from viur.core.tasks import CallDeferred
 from hashlib import sha256
 import logging
 from typing import Union, Dict, Any, List
 from itertools import chain
 from time import time
 
 
-@callDeferred
+@CallDeferred
 def ensureDerived(key: db.Key, srcKey, deriveMap: Dict[str, Any], refreshKey: db.Key = None):
     """
     Ensure that pending thumbnails or other derived Files are build

core/cache.py

@@ -201,7 +201,7 @@ def enableCache(urls: List[str], userSensitive: int = 0, languageSensitive: bool
     return lambda f: wrapCallable(f, urls, userSensitive, languageSensitive, evaluatedArgs, maxCacheTime)
 
 
-@tasks.callDeferred
+@tasks.CallDeferred
 def flushCache(prefix: str = None, key: Union[db.Key, None] = None, kind: Union[str, None] = None):
     """
         Flushes the cache. Its possible the flush only a part of the cache by specifying

core/email.py

@@ -6,7 +6,7 @@
 from viur.core.config import conf
 from viur.core import utils, db
 from viur.core.utils import projectID
-from viur.core.tasks import callDeferred, QueryIter, PeriodicTask, DeleteEntitiesIter
+from viur.core.tasks import CallDeferred, QueryIter, PeriodicTask, DeleteEntitiesIter
 
 """
     This module implements an email delivery system for ViUR. Emails will be queued so that we don't overwhelm
@@ -81,7 +81,7 @@ def transportSuccessfulCallback(entity: db.Entity):
         pass
 
 
-@callDeferred
+@CallDeferred
 def sendEmailDeferred(emailKey: db.Key):
     """
         Callback from the Taskqueue to send the given Email

core/modules/file.py

@@ -19,7 +19,7 @@
 from viur.core.bones import BaseBone, BooleanBone, KeyBone, NumericBone, StringBone
 from viur.core.prototypes.tree import SkelType, Tree, TreeSkel
 from viur.core.skeleton import skeletonByKind
-from viur.core.tasks import PeriodicTask, callDeferred
+from viur.core.tasks import PeriodicTask, CallDeferred
 from viur.core.utils import projectID, sanitizeFileName
 
 credentials, project = google.auth.default()
@@ -298,7 +298,7 @@ def write(self, filename: str, content: Any, mimetype: str = "text/plain", width
 
         return skel.toDB()
 
-    @callDeferred
+    @CallDeferred
     def deleteRecursive(self, parentKey):
         files = db.Query(self.leafSkelCls().kindName).filter("parentdir =", parentKey).iter()
         for fileEntry in files:
@@ -578,7 +578,7 @@ def startCheckForUnreferencedBlobs():
     doCheckForUnreferencedBlobs()
 
 
-@callDeferred
+@CallDeferred
 def doCheckForUnreferencedBlobs(cursor=None):
     def getOldBlobKeysTxn(dbKey):
         obj = db.Get(dbKey)
@@ -623,7 +623,7 @@ def startCleanupDeletedFiles():
     doCleanupDeletedFiles()
 
 
-@callDeferred
+@CallDeferred
 def doCleanupDeletedFiles(cursor=None):
     maxIterCount = 2  # How often a file will be checked for deletion
     query = db.Query("viur-deleted-files")

core/modules/user.py

@@ -5,7 +5,7 @@
 from viur.core.bones.base import ReadFromClientErrorSeverity, UniqueValue, UniqueLockMethod
 from viur.core.bones.password import pbkdf2
 from viur.core import errors, conf, securitykey
-from viur.core.tasks import StartupTask, callDeferred
+from viur.core.tasks import StartupTask, CallDeferred
 from viur.core.securityheaders import extendCsp
 from viur.core.ratelimit import RateLimit
 from time import time
@@ -317,7 +317,7 @@ def pwrecover(self, *args, **kwargs):
             session["user.auth_userpassword.pwrecover"] = None
             return self.userModule.render.view(None, self.passwordRecoverySuccessTemplate)
 
-    @callDeferred
+    @CallDeferred
     def sendUserPasswordRecoveryCode(self, userName: str, recoveryKey: str) -> None:
         """
             Sends the given recovery code to the user given in userName. This function runs deferred

core/prototypes/tree.py

@@ -6,7 +6,7 @@
 from viur.core.cache import flushCache
 from viur.core.prototypes import BasicApplication
 from viur.core.skeleton import Skeleton, SkeletonInstance
-from viur.core.tasks import callDeferred
+from viur.core.tasks import CallDeferred
 from viur.core.utils import currentRequest
 
 SkelType = Literal["node", "leaf"]
@@ -189,7 +189,7 @@ def getRootNode(self, entryKey: str) -> SkeletonInstance:
         rootNodeSkel.fromDB(repo.key)
         return rootNodeSkel
 
-    @callDeferred
+    @CallDeferred
     def updateParentRepo(self, parentNode: str, newRepoKey: str, depth: int = 0):
         """
         Recursively fixes the parentrepo key after a move operation.
@@ -473,7 +473,7 @@ def delete(self, skelType: SkelType, key: str, *args, **kwargs) -> Any:
         self.onDeleted(skelType, skel)
         return self.render.deleteSuccess(skel, skelType=skelType)
 
-    @callDeferred
+    @CallDeferred
     def deleteRecursive(self, parentKey: str):
         """
         Recursively processes a delete request.

core/securitykey.py

@@ -24,7 +24,7 @@
 from viur.core.utils import generateRandomString
 from viur.core.utils import currentSession, currentRequest
 from viur.core import db
-from viur.core.tasks import PeriodicTask, callDeferred
+from viur.core.tasks import PeriodicTask, CallDeferred
 from typing import Union
 from viur.core.utils import utcNow
 
@@ -96,7 +96,7 @@ def startClearSKeys() -> None:
     doClearSKeys((datetime.now() - timedelta(seconds=300)).strftime("%d.%m.%Y %H:%M:%S"))
 
 
-@callDeferred
+@CallDeferred
 def doClearSKeys(timeStamp: str) -> None:
     query = db.Query(securityKeyKindName).filter("until <", datetime.strptime(timeStamp, "%d.%m.%Y %H:%M:%S"))
     for oldKey in query.run(100):

core/session.py

@@ -5,7 +5,7 @@
 from time import time
 
 from viur.core.request import BrowseHandler
-from viur.core.tasks import PeriodicTask, callDeferred
+from viur.core.tasks import PeriodicTask, CallDeferred
 from viur.core import utils, db
 from viur.core.config import conf
 
@@ -246,7 +246,7 @@ def validateStaticSecurityKey(self, key: str) -> bool:
         return compare_digest(self.staticSecurityKey, key)
 
 
-@callDeferred
+@CallDeferred
 def killSessionByUser(user: Optional[str] = None):
     """
         Invalidates all active sessions for the given *user*.
@@ -274,7 +274,7 @@ def startClearSessions():
     doClearSessions(time() - (conf["viur.session.lifeTime"] + 300))
 
 
-@callDeferred
+@CallDeferred
 def doClearSessions(timeStamp: str) -> None:
     query = db.Query(GaeSession.kindName).filter("lastseen <", timeStamp)
     for oldKey in query.run(100):

core/skeleton.py

@@ -13,7 +13,7 @@
 from viur.core import conf, db, email, errors, utils
 from viur.core.bones import BaseBone, DateBone, KeyBone, RelationalBone, SelectBone, StringBone
 from viur.core.bones.base import ReadFromClientError, ReadFromClientErrorSeverity, getSystemInitialized
-from viur.core.tasks import CallableTask, CallableTaskBase, QueryIter, callDeferred
+from viur.core.tasks import CallableTask, CallableTaskBase, QueryIter, CallDeferred
 
 try:
     import pytz
@@ -1244,7 +1244,7 @@ def __init__(self, baseSkel=None):
 
 ### Tasks ###
 
-@callDeferred
+@CallDeferred
 def processRemovedRelations(removedKey, cursor=None):
     updateListQuery = db.Query("viur-relations").filter("dest.__key__ =", removedKey) \
         .filter("viur_relational_consistency >", 2)
@@ -1274,7 +1274,7 @@ def processRemovedRelations(removedKey, cursor=None):
         processRemovedRelations(removedKey, updateListQuery.getCursor())
 
 
-@callDeferred
+@CallDeferred
 def updateRelations(destKey: db.Key, minChangeTime: int, changedBone: Optional[str], cursor: Optional[str] = None):
     """
         This function updates Entities, which may have a copy of values from another entity which has been recently
@@ -1424,7 +1424,7 @@ def execute(self, module, *args, **kwargs):
         processVacuumRelationsChunk(module.strip(), None, notify=notify)
 
 
-@callDeferred
+@CallDeferred
 def processVacuumRelationsChunk(module, cursor, allCount=0, removedCount=0, notify=None):
     """
         Processes 100 Entries and calls the next batch