Ref #369 Use bulk_create to save number of queries

silk/collector.py

@@ -154,10 +154,20 @@ def finalise(self):
                 self.request.prof_file = f.name
                 self.request.save()
 
-        for _, query in self.queries.items():
-            query_model = models.SQLQuery.objects.create(**query)
-            query['model'] = query_model
-        for _, profile in self.profiles.items():
+        sql_queries = []
+        for identifier, query in self.queries.items():
+            query['identifier'] = identifier
+            sql_query = models.SQLQuery(**query)
+            sql_queries += [sql_query]
+
+        models.SQLQuery.objects.bulk_create(sql_queries)
+        sql_queries = models.SQLQuery.objects.filter(request=self.request)
+        for sql_query in sql_queries.all():
+            query = self.queries.get(sql_query.identifier)
+            if query:
+                query['model'] = sql_query
+
+        for profile in self.profiles.values():
             profile_query_models = []
             if TYP_QUERIES in profile:
                 profile_queries = profile[TYP_QUERIES]

silk/migrations/0007_sqlquery_identifier.py

@@ -0,0 +1,18 @@
+# Generated by Django 2.2.6 on 2019-10-26 12:57
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('silk', '0006_fix_request_prof_file_blank'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='sqlquery',
+            name='identifier',
+            field=models.IntegerField(default=-1),
+        ),
+    ]

silk/models.py

@@ -216,32 +216,26 @@ def raw_body_decoded(self):
 
 # TODO rewrite docstring
 class SQLQueryManager(models.Manager):
+    @transaction.atomic
     def bulk_create(self, *args, **kwargs):
         """ensure that num_sql_queries remains consistent. Bulk create does not call
         the model save() method and hence we must add this logic here too"""
         if len(args):
             objs = args[0]
         else:
             objs = kwargs.get('objs')
+        for obj in objs:
+            obj.prepare_save()
 
-        with transaction.atomic():
-            request_counter = Counter([x.request_id for x in objs])
-            requests = Request.objects.filter(pk__in=request_counter.keys())
-            # TODO: Not that there is ever more than one request (but there could be eventually)
-            # but perhaps there is a cleaner way of apply the increment from the counter without iterating
-            # and saving individually? e.g. bulk update but with diff. increments. Couldn't come up with this
-            # off hand.
-            for r in requests:
-                r.num_sql_queries = F('num_sql_queries') + request_counter[r.pk]
-                r.save()
-            return super(SQLQueryManager, self).bulk_create(*args, **kwargs)
+        return super(SQLQueryManager, self).bulk_create(*args, **kwargs)
 
 
 class SQLQuery(models.Model):
     query = TextField()
     start_time = DateTimeField(null=True, blank=True, default=timezone.now)
     end_time = DateTimeField(null=True, blank=True)
     time_taken = FloatField(blank=True, null=True)
+    identifier = IntegerField(default=-1)
     request = ForeignKey(
         Request, related_name='queries', null=True,
         blank=True, db_index=True, on_delete=models.CASCADE,
@@ -289,9 +283,7 @@ def tables_involved(self):
                     pass
         return tables
 
-    @transaction.atomic()
-    def save(self, *args, **kwargs):
-
+    def prepare_save(self):
         if self.end_time and self.start_time:
             interval = self.end_time - self.start_time
             self.time_taken = interval.total_seconds() * 1000
@@ -301,6 +293,9 @@ def save(self, *args, **kwargs):
                 self.request.num_sql_queries += 1
                 self.request.save(update_fields=['num_sql_queries'])
 
+    @transaction.atomic()
+    def save(self, *args, **kwargs):
+        self.prepare_save()
         super(SQLQuery, self).save(*args, **kwargs)
 
     @transaction.atomic()