performance optimizations

.github/workflows/build.yml

@@ -12,12 +12,10 @@ on:
 jobs:
 
   django_42:
-
     runs-on: ubuntu-latest
     strategy:
       matrix:
         python-version: ["3.8", "3.11", "3.12", "3.13"]
-
     steps:
     - uses: actions/checkout@v2
     - uses: kamiazya/setup-graphviz@v1
@@ -50,13 +48,11 @@ jobs:
         parallel: true
         flag-name: Unit Test
 
-  django_5:
-
+  django_52:
     runs-on: ubuntu-latest
     strategy:
       matrix:
         python-version: ["3.10", "3.11", "3.12", "3.13"]
-
     steps:
     - uses: actions/checkout@v2
     - uses: kamiazya/setup-graphviz@v1
@@ -88,9 +84,8 @@ jobs:
       with:
         parallel: true
         flag-name: Unit Test
-
-  mysql:
 
+  mysql:
     runs-on: ubuntu-latest
     strategy:
       matrix:
@@ -139,7 +134,6 @@ jobs:
         flag-name: Unit Test
 
   postgres:
-
     runs-on: ubuntu-latest
     strategy:
       matrix:
@@ -187,7 +181,6 @@ jobs:
         flag-name: Unit Test
 
   postgres-psycopg3:
-
     runs-on: ubuntu-latest
     strategy:
       matrix:
@@ -235,7 +228,6 @@ jobs:
         flag-name: Unit Test
 
   mariadb:
-
     runs-on: ubuntu-latest
     strategy:
       matrix:
@@ -283,8 +275,50 @@ jobs:
         parallel: true
         flag-name: Unit Test
 
+  backend:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        updatemode: ['FLAT', 'SAVE', 'BULK']
+        python-version: ["3.13"]
+    env:
+      UPDATEMODE: ${{ matrix.updatemode }}
+    steps:
+    - uses: actions/checkout@v2
+    - uses: kamiazya/setup-graphviz@v1
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install "Django~=5.2"
+        pip install -r example/requirements-ci.txt
+    - name: Setup DB
+      run: |
+        ./example/manage.py makemigrations
+        ./example/manage.py migrate
+    - name: Run tests
+      run: |
+        echo $UPDATEMODE
+        coverage run --parallel-mode --branch --source='computedfields' ./example/manage.py test exampleapp
+        coverage run --parallel-mode --branch --source='computedfields' ./example/manage.py test test_full
+        coverage combine
+        coverage report
+    - name: Build docs
+      run: |
+        cd docs && make html
+        cd ../
+    - name: Coveralls
+      uses: AndreMiras/coveralls-python-action@develop
+      with:
+        parallel: true
+        flag-name: Unit Test
+
+
   coveralls_finish:
-    needs: [django_42, django_5, mysql, postgres, postgres-psycopg3, mariadb]
+    needs: [django_42, django_52, mysql, postgres, postgres-psycopg3, mariadb, backend]
     runs-on: ubuntu-latest
     steps:
     - name: Coveralls Finished

computedfields/backends.py

@@ -0,0 +1,29 @@
+from fast_update.fast import fast_update
+from fast_update.update import flat_update
+from django.db.models import QuerySet
+from typing import Sequence, Any, Iterable
+
+
+def fast(queryset: QuerySet, objs: Sequence[Any], fields: Iterable[str]) -> int:
+    return fast_update(queryset, objs, tuple(fields), None, True)
+
+def flat(queryset: QuerySet, objs: Sequence[Any], fields: Iterable[str]) -> int:
+    return flat_update(queryset, objs, tuple(fields), True)
+
+def save(queryset: QuerySet, objs: Sequence[Any], fields: Iterable[str]) -> int:
+    from .resolver import NotComputed
+    with NotComputed():
+        for inst in objs:
+            inst.save(update_fields=fields)
+        return len(objs)
+
+def bulk(queryset: QuerySet, objs: Sequence[Any], fields: Iterable[str]) -> int:
+    return queryset.model._base_manager.bulk_update(objs, fields)
+
+
+UPDATE_IMPLEMENTATIONS = {
+    'FAST': fast,
+    'FLAT': flat,
+    'SAVE': save,
+    'BULK': bulk
+}

computedfields/helpers.py

@@ -1,6 +1,7 @@
 from itertools import tee, zip_longest
 from django.db.models import Model, QuerySet
-from typing import Any, Iterator, List, Sequence, Type, TypeVar, Tuple, Union, Generator, Iterable
+from typing import (Any, Iterator, List, Sequence, Type, TypeVar, Tuple, Union,
+                    Generator, Iterable, Optional, FrozenSet)
 
 T = TypeVar('T', covariant=True)
 
@@ -88,3 +89,9 @@ def proxy_to_base_model(proxymodel: Type[Model]) -> Union[Type[Model], None]:
 
 def are_same(*args) -> bool:
     return len(set(args)) == 1
+
+
+def frozenset_none(data: Optional[Iterable[Any]]) -> Optional[FrozenSet[Any]]:
+    if data is None:
+        return
+    return frozenset(data)

computedfields/management/commands/checkdata.py

@@ -80,7 +80,7 @@ def action_check(self, models, progress, size, json_out):
         for model in models:
             qs = model._base_manager.all()
             amount = qs.count()
-            fields = set(active_resolver.computed_models[model].keys())
+            fields = frozenset(active_resolver.computed_models[model].keys())
             qsize = active_resolver.get_querysize(model, fields, size)
             self.eprint(f'- {self.style.MIGRATE_LABEL(modelname(model))}')
             self.eprint(f'  Fields: {", ".join(fields)}')

computedfields/management/commands/updatedata.py

@@ -10,6 +10,7 @@
 from computedfields.models import active_resolver
 from computedfields.helpers import modelname, slice_iterator
 from computedfields.settings import settings
+from computedfields.backends import UPDATE_IMPLEMENTATIONS
 from ._helpers import retrieve_computed_models, HAS_TQDM, tqdm
 
 from typing import Type, cast
@@ -48,8 +49,8 @@ def add_arguments(self, parser):
             '-m', '--mode',
             default='default',
             type=str,
-            choices=('loop', 'bulk', 'fast'),
-            help='Set explicit update mode, default: bulk/fast from settings.py.'
+            choices=('FAST', 'FLAT', 'SAVE', 'BULK', 'LOOP'),
+            help='Set explicit update mode, default is taken from settings.py.'
         )
         parser.add_argument(
             '-q', '--querysize',
@@ -85,7 +86,7 @@ def action_fileinput(self, file, size, progress):
             model_name, desync = data.get('model'), data.get('desync')
             model: Type[Model] = cast(Type[Model], apps.get_model(model_name))
             amount = len(desync)
-            fields = set(active_resolver.computed_models[model].keys())
+            fields = frozenset(active_resolver.computed_models[model].keys())
             self.stdout.write(f'- {self.style.MIGRATE_LABEL(modelname(model))}')
             self.stdout.write(f'  Fields: {", ".join(fields)}')
             self.stdout.write(f'  Desync Records: {amount}')
@@ -112,32 +113,20 @@ def action_default(self, models, size, show_progress, mode=''):
         Runs either in fast or bulk mode, whatever was set in settings.
         """
         if not mode:
-            mode = 'fast' if settings.COMPUTEDFIELDS_FASTUPDATE else 'bulk'
+            mode = settings.COMPUTEDFIELDS_UPDATEMODE
             self.stdout.write(f'Update mode: settings.py --> {mode}')
 
         self.stdout.write(f'Default querysize: {size}')
         self.stdout.write('Models:')
         for model in models:
             qs = model._base_manager.all()
             amount = qs.count()
-            fields = set(active_resolver.computed_models[model].keys())
+            fields = frozenset(active_resolver.computed_models[model].keys())
             self.stdout.write(f'- {self.style.MIGRATE_LABEL(modelname(model))}')
             self.stdout.write(f'  Fields: {", ".join(fields)}')
             self.stdout.write(f'  Records: {amount}')
             self.stdout.write(f'  Querysize: {active_resolver.get_querysize(model, fields, size)}')
 
-            # TODO: dummy test code to get some idea about long taking tasks in the update tree
-            # this is linked to bad perf from slicing and distinct() calls in bulk_updater (#101)
-            ##qs = qs.filter(pk__in=range(1, 1001))
-            #counted = count_dependent(qs)
-            #explained = explain_dependent(qs, query_pks=False)
-            #self.stdout.write('records to check:', counted)
-            #for ex in explained:
-            #    self.stdout.write(ex)
-            #timer(lambda: explain_dependent(qs), 1)
-            #timer(lambda: count_dependent(qs), 1)
-            #return
-
             if not amount:
                 continue
             if show_progress:
@@ -157,20 +146,33 @@ def action_default(self, models, size, show_progress, mode=''):
             else:
                 active_resolver.update_dependent(qs, querysize=size)
 
-    def action_bulk(self, models, size, show_progress):
-        active_resolver.use_fastupdate = False
-        self.stdout.write('Update mode: bulk')
-        self.action_default(models, size, show_progress, 'bulk')
-
-    def action_fast(self, models, size, show_progress):
-        active_resolver.use_fastupdate = True
-        active_resolver._batchsize = settings.COMPUTEDFIELDS_BATCHSIZE_FAST
+    def action_FAST(self, models, size, show_progress):
+        active_resolver._update_mode = 'FAST'
+        active_resolver._update = UPDATE_IMPLEMENTATIONS['FAST']
         self.stdout.write('Update mode: fast')
-        self.action_default(models, size, show_progress, 'fast')
+        self.action_default(models, size, show_progress, 'FAST')
+
+    def action_FLAT(self, models, size, show_progress):
+        active_resolver._update_mode = 'FLAT'
+        active_resolver._update = UPDATE_IMPLEMENTATIONS['FLAT']
+        self.stdout.write('Update mode: flat')
+        self.action_default(models, size, show_progress, 'FLAT')
+
+    def action_SAVE(self, models, size, show_progress):
+        active_resolver._update_mode = 'SAVE'
+        active_resolver._update = UPDATE_IMPLEMENTATIONS['SAVE']
+        self.stdout.write('Update mode: save')
+        self.action_default(models, size, show_progress, 'SAVE')
+
+    def action_BULK(self, models, size, show_progress):
+        active_resolver._update_mode = 'BULK'
+        active_resolver._update = UPDATE_IMPLEMENTATIONS['BULK']
+        self.stdout.write('Update mode: bulk')
+        self.action_default(models, size, show_progress, 'BULK')
 
     @transaction.atomic
-    def action_loop(self, models, size, show_progress):
-        self.stdout.write('Update mode: loop')
+    def action_LOOP(self, models, size, show_progress):
+        self.stdout.write('Update mode: LOOP')
         self.stdout.write(f'Global querysize: {size}')
         self.stdout.write('Models:')
         if size != settings.COMPUTEDFIELDS_QUERYSIZE:
@@ -181,7 +183,7 @@ def action_loop(self, models, size, show_progress):
         for model in models:
             qs = model._base_manager.all()
             amount = qs.count()
-            fields = list(active_resolver.computed_models[model].keys())
+            fields = frozenset(active_resolver.computed_models[model].keys())
             qsize = active_resolver.get_querysize(model, fields, size)
             self.stdout.write(f'- {self.style.MIGRATE_LABEL(modelname(model))}')
             self.stdout.write(f'  Fields: {", ".join(fields)}')
@@ -204,33 +206,3 @@ def action_loop(self, models, size, show_progress):
             else:
                 for obj in slice_iterator(qs, qsize):
                     obj.save()
-
-
-# get some explaining on update_dependent
-#def count_dependent(queryset, fields=None):
-#    #counted = queryset.count()
-#    counted = len(set(queryset.values_list('pk', flat=True).iterator()))
-#    if counted:
-#        updates = active_resolver._querysets_for_update(queryset.model, queryset, fields).values()
-#        for qs, f in updates:
-#            counted += count_dependent(qs, f)
-#    return counted
-#
-#def explain_dependent(queryset, fields=None, level=0, query_pks=False):
-#    s = time()
-#    #counted = queryset.count()
-#    counted = len(set(queryset.values_list('pk', flat=True).iterator()))
-#    d = time() - s
-#    res = [(level, queryset.model, fields, counted, d, queryset.distinct().values_list('pk', flat=True) if query_pks else [])]
-#    if counted:
-#        updates = active_resolver._querysets_for_update(queryset.model, queryset, fields).values()
-#        for qs, f in updates:
-#            res += explain_dependent(qs, f, level+1, query_pks)
-#    return res
-#
-#
-#def timer(f, n):
-#    start = time()
-#    for _ in range(n):
-#        f()
-#    print(time()-start)