Don't load full model during data migrations

indigo_api/data_migrations/__init__.py

@@ -18,6 +18,17 @@ def migrate_document_version(self, version):
                 version.serialized_data = json.dumps([data])
                 return True
 
+    def migrate_document_xml(self, document):
+        """ Migrate the raw document XML. This is useful in migrations where
+        we don't have access to the full document model's support methods.
+        """
+        xml = etree.fromstring(document.document_xml)
+        self.ns = xml.nsmap[None]
+        changed, xml = self.migrate_xml(xml)
+        if changed:
+            document.document_xml = etree.tostring(xml, encoding='unicode')
+        return changed
+
     def migrate_xml(self, xml):
         """ Migrates an XML document, returning tuple (changed, xml) where
         changed is a boolean indicating if the document has changed, and xml

indigo_api/migrations/0009_migrate_attachment_eids.py

@@ -8,8 +8,9 @@
 def forwards(apps, schema_editor):
     """ Migrate attachment eIds, and change annotations to no longer use the attachment scope prefix.
     """
-    from indigo_api.models import Document
-    from indigo_api.models import Annotation
+    Document = apps.get_model('indigo_api', 'Document')
+    Annotation = apps.get_model('indigo_api', 'Annotation')
+
     from indigo_api.models.documents import post_save_document
     from reversion.models import Version
 
@@ -19,9 +20,9 @@ def forwards(apps, schema_editor):
     db_alias = schema_editor.connection.alias
     migration = CorrectAttachmentEids()
 
-    for doc in Document.objects.using(db_alias).order_by('-pk'):
+    for doc in Document.objects.using(db_alias).order_by('-pk').iterator(100):
         print(f"Migrating {doc}")
-        if migration.migrate_document(doc):
+        if migration.migrate_document_xml(doc):
             print("  Changed")
             doc.save()
 
@@ -46,7 +47,7 @@ def forwards(apps, schema_editor):
 
     # migrate historical document versions
     print("Migrating versions")
-    for version in Version.objects.get_for_model(Document).order_by('-pk').iterator():
+    for version in Version.objects.get_for_model(Document).order_by('-pk').iterator(100):
         print(f"Migrating version {version.pk}")
         if migration.migrate_document_version(version):
             print("  Changed")

indigo_api/migrations/0010_migrate_crossheadings.py

@@ -6,20 +6,21 @@
 
 
 def forwards(apps, schema_editor):
-    from indigo_api.models import Document
-    from indigo_api.models import Annotation
     from indigo_api.models.documents import post_save_document
     from reversion.models import Version
 
+    Document = apps.get_model('indigo_api', 'Document')
+    Annotation = apps.get_model('indigo_api', 'Annotation')
+
     # disconnect signals
     signals.post_save.disconnect(post_save_document, Document)
 
     db_alias = schema_editor.connection.alias
     migration = RealCrossHeadings()
 
-    for doc in Document.objects.using(db_alias).order_by('-pk'):
+    for doc in Document.objects.using(db_alias).order_by('-pk').iterator(100):
         print(f"Migrating {doc}")
-        if migration.migrate_document(doc):
+        if migration.migrate_document_xml(doc):
             print("  Changed")
             doc.save()
 
@@ -33,7 +34,7 @@ def forwards(apps, schema_editor):
 
     # migrate historical document versions
     print("Migrating versions")
-    for version in Version.objects.get_for_model(Document).order_by('-pk').iterator():
+    for version in Version.objects.get_for_model(Document).order_by('-pk').iterator(100):
         print(f"Migrating version {version.pk}")
         if migration.migrate_document_version(version):
             print("  Changed")