Master release for hotfixes completed before 2022-07-07

.github/workflows/pythontest.yml

@@ -15,16 +15,18 @@ jobs:
         with:
           github_token: ${{ github.token }}
           paths: '["**.py", "requirements.txt", "requirements-dev.txt", ".github/workflows/pythontest.yml"]'
+
   unit_test:
     name: Python unit tests
     needs: pre_job
     if: ${{ needs.pre_job.outputs.should_skip != 'true' }}
     runs-on: ubuntu-latest
+
     services:
       # Label used to access the service container
       postgres:
         # Docker Hub image
-        image: postgres
+        image: postgres:12
         # Provide the password for postgres
         env:
           POSTGRES_USER: learningequality
@@ -42,7 +44,7 @@ jobs:
       # Label used to access the service container
       redis:
         # Docker Hub image
-        image: redis
+        image: redis:4.0.9
         # Set health checks to wait until redis has started
         options: >-
           --health-cmd "redis-cli ping"
@@ -52,16 +54,16 @@ jobs:
         ports:
           # Maps port 6379 on service container to the host
           - 6379:6379
-      minio:
-        image: bitnami/minio
-        env:
-          MINIO_API_CORS_ALLOW_ORIGIN: http://localhost:8080
-          MINIO_ACCESS_KEY: development
-          MINIO_SECRET_KEY: development
-        ports:
-          - 9000:9000
     steps:
     - uses: actions/checkout@v2
+    - name: Set up minio
+      run: |
+        docker run -d -p 9000:9000 --name minio \
+                    -e "MINIO_ACCESS_KEY=development" \
+                    -e "MINIO_SECRET_KEY=development" \
+                    -v /tmp/minio_data:/data \
+                    -v /tmp/minio_config:/root/.minio \
+                    minio/minio server /data
     - name: Set up Python 3.6
       uses: actions/setup-python@v2
       with:

.pre-commit-config.yaml

@@ -1,5 +1,5 @@
 repos:
--   repo: git://github.com/pre-commit/pre-commit-hooks
+-   repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v2.2.1
     hooks:
     -   id: trailing-whitespace

Makefile

@@ -11,7 +11,7 @@ dummyusers:
 	cd contentcuration/ && python manage.py loaddata contentcuration/fixtures/admin_user_token.json
 
 prodceleryworkers:
-	cd contentcuration/ && celery -A contentcuration worker -l info --concurrency=3 --task-events --without-mingle --without-gossip
+	cd contentcuration/ && celery -A contentcuration worker -l info --concurrency=3 --task-events
 
 prodcelerydashboard:
 	# connect to the celery dashboard by visiting http://localhost:5555
@@ -135,6 +135,12 @@ docs: clean-docs
 setup:
 	python contentcuration/manage.py setup
 
+filedurations:
+	python contentcuration/manage.py set_file_duration
+
+learningactivities:
+	python contentcuration/manage.py set_default_learning_activities
+
 export COMPOSE_PROJECT_NAME=studio_$(shell git rev-parse --abbrev-ref HEAD)
 
 purge-postgres:

README.md

@@ -55,7 +55,7 @@ To confirm that the services are running, run `docker ps`, and you should see th
 CONTAINER ID        IMAGE                             COMMAND                  CREATED             STATUS              PORTS                    NAMES
 e09c5c203b93        redis:4.0.9                       "docker-entrypoint.s…"   51 seconds ago      Up 49 seconds       0.0.0.0:6379->6379/tcp   studio_vue-refactor_redis_1
 6164371efb6b        minio/minio                       "minio server /data"     51 seconds ago      Up 49 seconds       0.0.0.0:9000->9000/tcp   studio_vue-refactor_minio_1
-c86bbfa3a59e        postgres:9.6                      "docker-entrypoint.s…"   51 seconds ago      Up 49 seconds       0.0.0.0:5432->5432/tcp   studio_vue-refactor_postgres_1
+c86bbfa3a59e        postgres:12.10                      "docker-entrypoint.s…"   51 seconds ago      Up 49 seconds       0.0.0.0:5432->5432/tcp   studio_vue-refactor_postgres_1
 ```
 
 

contentcuration/contentcuration/admin.py

@@ -1,8 +1,9 @@
 from django.contrib import admin
 
-from contentcuration.models import Exercise, AssessmentItem, License, User
+from contentcuration.models import AssessmentItem
+from contentcuration.models import License
+from contentcuration.models import User
 
-admin.site.register(Exercise)
 admin.site.register(AssessmentItem)
 admin.site.register(License)
 

contentcuration/contentcuration/apps.py

@@ -1,7 +1,7 @@
 from django.apps import AppConfig
 from django.conf import settings
 
-from contentcuration.utils.minio_utils import ensure_storage_bucket_public
+from contentcuration.utils.storage_common import is_gcs_backend
 
 
 class ContentConfig(AppConfig):
@@ -10,5 +10,7 @@ class ContentConfig(AppConfig):
     def ready(self):
         # see note in the celery_signals.py file for why we import here.
         import contentcuration.utils.celery.signals  # noqa
-        if settings.AWS_AUTO_CREATE_BUCKET:
+
+        if settings.AWS_AUTO_CREATE_BUCKET and not is_gcs_backend():
+            from contentcuration.utils.minio_utils import ensure_storage_bucket_public
             ensure_storage_bucket_public()

contentcuration/contentcuration/constants/contentnode.py

@@ -0,0 +1,13 @@
+from le_utils.constants import content_kinds
+from le_utils.constants.labels import learning_activities
+
+
+kind_activity_map = {
+    content_kinds.EXERCISE: learning_activities.PRACTICE,
+    content_kinds.VIDEO: learning_activities.WATCH,
+    content_kinds.AUDIO: learning_activities.LISTEN,
+    content_kinds.DOCUMENT: learning_activities.READ,
+    content_kinds.HTML5: learning_activities.EXPLORE,
+    content_kinds.H5P: learning_activities.EXPLORE,
+    content_kinds.SLIDESHOW: learning_activities.READ,
+}

contentcuration/contentcuration/dev_urls.py

@@ -8,7 +8,6 @@
 from drf_yasg.views import get_schema_view
 from rest_framework import permissions
 
-import contentcuration.views.files as file_views
 from .urls import urlpatterns
 
 
@@ -48,21 +47,6 @@ def webpack_redirect_view(request):
         r"^redoc/$", schema_view.with_ui("redoc", cache_timeout=0), name="schema-redoc"
     ),
     re_path(r"^api-auth/", include("rest_framework.urls", namespace="rest_framework")),
-    re_path(
-        r"^" + settings.STORAGE_URL[1:] + "(?P<path>.*)$",
-        file_views.debug_serve_file,
-        name="debug_serve_file",
-    ),
-    re_path(
-        r"^" + settings.CONTENT_DATABASE_URL[1:] + "(?P<path>.*)$",
-        file_views.debug_serve_content_database_file,
-        name="content_database_debug_serve_file",
-    ),
-    re_path(
-        r"^" + settings.CSV_URL[1:] + "(?P<path>.*)$",
-        file_views.debug_serve_file,
-        name="csv_debug_serve_file",
-    ),
 ]
 
 if getattr(settings, "DEBUG_PANEL_ACTIVE", False):

contentcuration/contentcuration/frontend/channelEdit/views/files/FileUpload.vue

@@ -108,7 +108,10 @@
         return this.getContentNodeFiles(this.nodeId);
       },
       presets() {
-        return FormatPresetsList.filter(p => p.kind_id === this.node.kind);
+        // Explicitly exclude any 'dependency' presets for now
+        return FormatPresetsList.filter(
+          p => p.kind_id === this.node.kind && !p.id.includes('dependency')
+        );
       },
       fileCount() {
         return this.primaryFileMapping.filter(item => item.file && !item.file.error).length;

contentcuration/contentcuration/management/commands/loadconstants.py

@@ -1,9 +1,6 @@
-from future import standard_library
-
-standard_library.install_aliases()
-from builtins import str
-from builtins import object
 import logging as logmodule
+from builtins import object
+from builtins import str
 
 from django.conf import settings
 from django.contrib.sites.models import Site

contentcuration/contentcuration/management/commands/set_default_learning_activities.py

@@ -0,0 +1,37 @@
+import logging as logmodule
+import time
+
+from django.core.management.base import BaseCommand
+
+from contentcuration.constants.contentnode import kind_activity_map
+from contentcuration.models import ContentNode
+
+logmodule.basicConfig(level=logmodule.INFO)
+logging = logmodule.getLogger('command')
+
+
+CHUNKSIZE = 10000
+
+
+class Command(BaseCommand):
+
+    def handle(self, *args, **options):
+        start = time.time()
+
+        for kind, activity in kind_activity_map.items():
+            kind_start = time.time()
+            map_to_set = {
+                activity: True
+            }
+
+            null_learning_activities = ContentNode.objects.filter(kind=kind, learning_activities__isnull=True).values_list("id", flat=True)
+
+            logging.info("Setting default learning activities for kind: {}".format(kind))
+
+            while null_learning_activities.exists():
+                updated_count = ContentNode.objects.filter(id__in=null_learning_activities[0:CHUNKSIZE]).update(learning_activities=map_to_set)
+                logging.info("Updated {} content nodes of kind {} with learning activity {}".format(updated_count, kind, activity))
+
+            logging.info("Finished setting default learning activities for kind: {} in {} seconds".format(kind, time.time() - kind_start))
+
+        logging.info('Finished setting all null learning activities in {} seconds'.format(time.time() - start))

contentcuration/contentcuration/management/commands/set_file_duration.py

@@ -0,0 +1,112 @@
+import logging as logmodule
+import subprocess
+import time
+
+from django.core.management.base import BaseCommand
+
+from contentcuration.models import File
+from contentcuration.models import MEDIA_PRESETS
+
+logmodule.basicConfig(level=logmodule.INFO)
+logging = logmodule.getLogger('command')
+
+
+CHUNKSIZE = 10000
+
+
+def extract_duration_of_media(f_in, extension):
+    """
+    For more details on these commands, refer to the ffmpeg Wiki:
+    https://trac.ffmpeg.org/wiki/FFprobeTips#Formatcontainerduration
+    """
+    result = subprocess.check_output(
+        [
+            "ffprobe",
+            "-v",
+            "error",
+            "-show_entries",
+            "format=duration",
+            "-of",
+            "default=noprint_wrappers=1:nokey=1",
+            "-loglevel",
+            "panic",
+            "-f",
+            extension,
+            "-"
+        ],
+        stdin=f_in,
+    )
+    result = result.decode("utf-8").strip()
+    try:
+        return int(float(result))
+    except ValueError:
+        # This can happen if ffprobe returns N/A for the duration
+        # So instead we try to stream the entire file to get the value
+        f_in.seek(0)
+        result = subprocess.run(
+            [
+                "ffmpeg",
+                "-i",
+                "pipe:",
+                "-f",
+                "null",
+                "-",
+            ],
+            stdin=f_in,
+            stderr=subprocess.PIPE
+        )
+        second_last_line = result.stderr.decode("utf-8").strip().splitlines()[-2]
+        time_code = second_last_line.split(" time=")[1].split(" ")[0]
+        hours, minutes, seconds = time_code.split(":")
+        try:
+            hours = int(hours)
+        except ValueError:
+            hours = 0
+        try:
+            minutes = int(minutes)
+        except ValueError:
+            minutes = 0
+        try:
+            seconds = int(float(seconds))
+        except ValueError:
+            seconds = 0
+        return (hours * 60 + minutes) * 60 + seconds
+
+
+class Command(BaseCommand):
+
+    def handle(self, *args, **options):
+        start = time.time()
+
+        logging.info("Setting default duration for media presets: {}".format(MEDIA_PRESETS))
+
+        excluded_files = set()
+
+        null_duration = File.objects.filter(preset_id__in=MEDIA_PRESETS, duration__isnull=True)
+        null_duration_count = null_duration.count()
+        updated_count = 0
+
+        i = 0
+
+        while i < null_duration_count:
+            for file in null_duration[i:i + CHUNKSIZE]:
+                if file.file_on_disk.name in excluded_files:
+                    continue
+                file.refresh_from_db()
+                if file.duration is not None:
+                    continue
+                try:
+                    with file.file_on_disk.open() as f:
+                        duration = extract_duration_of_media(f, file.file_format.extension)
+                    if duration:
+                        updated_count += File.objects.filter(checksum=file.checksum, preset_id__in=MEDIA_PRESETS).update(duration=duration)
+                except FileNotFoundError:
+                    logging.warning("File {} not found".format(file))
+                    excluded_files.add(file.file_on_disk.name)
+                except subprocess.CalledProcessError:
+                    logging.warning("File {} could not be read for duration".format(file))
+                    excluded_files.add(file.file_on_disk.name)
+
+            i += CHUNKSIZE
+
+        logging.info('Finished setting all null duration for {} files in {} seconds'.format(updated_count, time.time() - start))

contentcuration/contentcuration/management/commands/setup.py

@@ -23,7 +23,7 @@
 from contentcuration.utils.db_tools import create_topic
 from contentcuration.utils.db_tools import create_user
 from contentcuration.utils.files import duplicate_file
-from contentcuration.utils.minio_utils import ensure_storage_bucket_public
+from contentcuration.utils.storage_common import is_gcs_backend
 
 logmodule.basicConfig()
 logging = logmodule.getLogger(__name__)
@@ -55,7 +55,9 @@ def handle(self, *args, **options):
             sys.exit()
 
         # create the minio bucket
-        ensure_storage_bucket_public()
+        if not is_gcs_backend():
+            from contentcuration.utils.minio_utils import ensure_storage_bucket_public
+            ensure_storage_bucket_public()
 
         # create the cache table
         try: