Skip to content

Commit d812fda

Browse files
NiedielnitsevIvanUsamaSadiq
authored andcommitted
feat: Models for import_from_modulestore (#36515)
A new application has been created, described in this ADR: #36545 have been created, as well as related models for mapping original content and new content created during the import process. Python and Django APIs, as well as a Django admin interface, will soon follow.
1 parent d107751 commit d812fda

File tree

14 files changed

+246
-5
lines changed

14 files changed

+246
-5
lines changed

.github/workflows/unit-test-shards.json

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -238,6 +238,7 @@
238238
"cms/djangoapps/cms_user_tasks/",
239239
"cms/djangoapps/course_creators/",
240240
"cms/djangoapps/export_course_metadata/",
241+
"cms/djangoapps/import_from_modulestore/",
241242
"cms/djangoapps/maintenance/",
242243
"cms/djangoapps/models/",
243244
"cms/djangoapps/pipeline_js/",
Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
========================
2+
Import from Modulestore
3+
========================
4+
5+
The new Django application `import_from_modulestore` is designed to
6+
automate the process of importing course legacy OLX content from Modulestore
7+
to Content Libraries. The application allows users to easily and quickly
8+
migrate existing course content, minimizing the manual work and potential
9+
errors associated with manual migration.
10+
The new app makes the import process automated and easy to manage.
11+
12+
The main problems solved by the application:
13+
14+
* Reducing the time to import course content.
15+
* Ensuring data integrity during the transfer.
16+
* Ability to choose which content to import before the final import.
17+
18+
------------------------------
19+
Import from Modulestore Usage
20+
------------------------------
21+
22+
* Import course elements at the level of sections, subsections, units,
23+
and xblocks into the Content Libraries.
24+
* Choose the structure of this import, whether it will be only xblocks
25+
from a particular course or full sections/subsections/units.
26+
* Store the history of imports.
27+
* Synchronize the course content with the library content (when re-importing,
28+
the blocks can be updated according to changes in the original course).
29+
* The new import mechanism ensures data integrity at the time of importing
30+
by saving the course in StagedContent.
31+
* Importing the legacy library content into the new Content Libraries.

cms/djangoapps/import_from_modulestore/__init__.py

Whitespace-only changes.
Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
"""
2+
This module contains the admin configuration for the Import model.
3+
"""
4+
from django.contrib import admin
5+
6+
from .models import Import, PublishableEntityImport, PublishableEntityMapping
7+
8+
9+
class ImportAdmin(admin.ModelAdmin):
10+
"""
11+
Admin configuration for the Import model.
12+
"""
13+
14+
list_display = (
15+
'uuid',
16+
'created',
17+
'status',
18+
'source_key',
19+
'target_change',
20+
)
21+
list_filter = (
22+
'status',
23+
)
24+
search_fields = (
25+
'source_key',
26+
'target_change',
27+
)
28+
29+
raw_id_fields = ('user',)
30+
readonly_fields = ('status',)
31+
32+
33+
admin.site.register(Import, ImportAdmin)
34+
admin.site.register(PublishableEntityImport)
35+
admin.site.register(PublishableEntityMapping)
Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
"""
2+
App for importing from the modulestore tools.
3+
"""
4+
5+
from django.apps import AppConfig
6+
7+
8+
class ImportFromModulestoreConfig(AppConfig):
9+
"""
10+
App for importing legacy content from the modulestore.
11+
"""
12+
13+
name = 'cms.djangoapps.import_from_modulestore'
Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
"""
2+
This module contains the data models for the import_from_modulestore app.
3+
"""
4+
from django.db.models import TextChoices
5+
from django.utils.translation import gettext_lazy as _
6+
7+
8+
class ImportStatus(TextChoices):
9+
"""
10+
The status of this modulestore-to-learning-core import.
11+
"""
12+
13+
NOT_STARTED = 'not_started', _('Waiting to stage content')
14+
STAGING = 'staging', _('Staging content for import')
15+
STAGING_FAILED = _('Failed to stage content')
16+
STAGED = 'staged', _('Content is staged and ready for import')
17+
IMPORTING = 'importing', _('Importing staged content')
18+
IMPORTING_FAILED = 'importing_failed', _('Failed to import staged content')
19+
IMPORTED = 'imported', _('Successfully imported content')
20+
CANCELED = 'canceled', _('Canceled')

cms/djangoapps/import_from_modulestore/migrations/__init__.py

Whitespace-only changes.
Lines changed: 140 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,140 @@
1+
"""
2+
Models for the course to library import app.
3+
"""
4+
5+
import uuid as uuid_tools
6+
7+
from django.contrib.auth import get_user_model
8+
from django.db import models
9+
from django.utils.translation import gettext_lazy as _
10+
11+
from model_utils.models import TimeStampedModel
12+
from opaque_keys.edx.django.models import (
13+
LearningContextKeyField,
14+
UsageKeyField,
15+
)
16+
from openedx_learning.api.authoring_models import LearningPackage, PublishableEntity
17+
18+
from .data import ImportStatus
19+
20+
User = get_user_model()
21+
22+
23+
class Import(TimeStampedModel):
24+
"""
25+
Represents the action of a user importing a modulestore-based course or legacy
26+
library into a learning-core based learning package (today, that is always a content library).
27+
"""
28+
29+
uuid = models.UUIDField(default=uuid_tools.uuid4, editable=False, unique=True)
30+
status = models.CharField(
31+
max_length=100,
32+
choices=ImportStatus.choices,
33+
default=ImportStatus.NOT_STARTED,
34+
db_index=True
35+
)
36+
user = models.ForeignKey(User, on_delete=models.CASCADE)
37+
38+
# Note: For now, this will always be a course key. In the future, it may be a legacy library key.
39+
source_key = LearningContextKeyField(help_text=_('The modulestore course'), max_length=255, db_index=True)
40+
target_change = models.ForeignKey(to='oel_publishing.DraftChangeLog', on_delete=models.SET_NULL, null=True)
41+
42+
class Meta:
43+
verbose_name = _('Import from modulestore')
44+
verbose_name_plural = _('Imports from modulestore')
45+
46+
def __str__(self):
47+
return f'{self.source_key}{self.target_change}'
48+
49+
def set_status(self, status: ImportStatus):
50+
"""
51+
Set import status.
52+
"""
53+
self.status = status
54+
self.save()
55+
if status in [ImportStatus.IMPORTED, ImportStatus.CANCELED]:
56+
self.clean_related_staged_content()
57+
58+
def clean_related_staged_content(self) -> None:
59+
"""
60+
Clean related staged content.
61+
"""
62+
for staged_content_for_import in self.staged_content_for_import.all():
63+
staged_content_for_import.staged_content.delete()
64+
65+
66+
class PublishableEntityMapping(TimeStampedModel):
67+
"""
68+
Represents a mapping between a source usage key and a target publishable entity.
69+
"""
70+
71+
source_usage_key = UsageKeyField(
72+
max_length=255,
73+
help_text=_('Original usage key/ID of the thing that has been imported.'),
74+
)
75+
target_package = models.ForeignKey(LearningPackage, on_delete=models.CASCADE)
76+
target_entity = models.ForeignKey(PublishableEntity, on_delete=models.CASCADE)
77+
78+
class Meta:
79+
unique_together = ('source_usage_key', 'target_package')
80+
81+
def __str__(self):
82+
return f'{self.source_usage_key}{self.target_entity}'
83+
84+
85+
class PublishableEntityImport(TimeStampedModel):
86+
"""
87+
Represents a publishableentity version that has been imported into a learning package (e.g. content library)
88+
89+
This is a many-to-many relationship between a container version and a course to library import.
90+
"""
91+
92+
import_event = models.ForeignKey(Import, on_delete=models.CASCADE)
93+
resulting_mapping = models.ForeignKey(PublishableEntityMapping, on_delete=models.SET_NULL, null=True, blank=True)
94+
resulting_change = models.OneToOneField(
95+
to='oel_publishing.DraftChangeLogRecord',
96+
# a changelog record can be pruned, which would set this to NULL, but not delete the
97+
# entire import record
98+
null=True,
99+
on_delete=models.SET_NULL,
100+
)
101+
102+
class Meta:
103+
unique_together = (
104+
('import_event', 'resulting_mapping'),
105+
)
106+
107+
def __str__(self):
108+
return f'{self.import_event}{self.resulting_mapping}'
109+
110+
111+
class StagedContentForImport(TimeStampedModel):
112+
"""
113+
Represents m2m relationship between an import and staged content created for that import.
114+
"""
115+
116+
import_event = models.ForeignKey(
117+
Import,
118+
on_delete=models.CASCADE,
119+
related_name='staged_content_for_import',
120+
)
121+
staged_content = models.OneToOneField(
122+
to='content_staging.StagedContent',
123+
on_delete=models.CASCADE,
124+
related_name='staged_content_for_import',
125+
)
126+
# Since StagedContent stores all the keys of the saved blocks, this field was added to optimize search.
127+
source_usage_key = UsageKeyField(
128+
max_length=255,
129+
help_text=_(
130+
'The original Usage key of the highest-level component that was saved in StagedContent.'
131+
),
132+
)
133+
134+
class Meta:
135+
unique_together = (
136+
('import_event', 'staged_content'),
137+
)
138+
139+
def __str__(self):
140+
return f'{self.import_event}{self.staged_content}'

cms/envs/common.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1667,6 +1667,7 @@
16671667
'openedx.core.djangoapps.course_groups', # not used in cms (yet), but tests run
16681668
'cms.djangoapps.xblock_config.apps.XBlockConfig',
16691669
'cms.djangoapps.export_course_metadata.apps.ExportCourseMetadataConfig',
1670+
'cms.djangoapps.import_from_modulestore.apps.ImportFromModulestoreConfig',
16701671

16711672
# New (Learning-Core-based) XBlock runtime
16721673
'openedx.core.djangoapps.xblock.apps.StudioXBlockAppConfig',

requirements/constraints.txt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -112,7 +112,7 @@ numpy<2.0.0
112112
# Date: 2023-09-18
113113
# pinning this version to avoid updates while the library is being developed
114114
# Issue for unpinning: https://github.com/openedx/edx-platform/issues/35269
115-
openedx-learning==0.23.0
115+
openedx-learning==0.23.1
116116

117117
# Date: 2023-11-29
118118
# Open AI version 1.0.0 dropped support for openai.ChatCompletion which is currently in use in enterprise.

0 commit comments

Comments
 (0)