Fake video detection

Dockerfile

@@ -1,4 +1,4 @@
-FROM ubuntu:20.04
+FROM python:3.9.9-bullseye
 
 EXPOSE 80
 
@@ -10,45 +10,25 @@ ADD entrypoint*.sh /app/
 
 WORKDIR /app
 
+ENV DEBIAN_FRONTEND=noninteractive
+
 # install app dependencies, build app and remove dev dependencies
 RUN apt-get update && apt-get install -y --no-install-recommends \
-    build-essential \
-    python3.8 \
-    python3.8-venv \
-    python3.8-dev \
-    python3.8-gdbm \
-    libpq5 \
-    libpq-dev \
-    libffi-dev \
-    libssl-dev \
-    virtualenv \
-    gnupg \
-    curl \
-    git \
     authbind \
-    && curl -sL https://deb.nodesource.com/setup_12.x | bash - \
+    libatlas-base-dev libhdf5-dev libavutil-dev libswresample-dev libavcodec-dev libavformat-dev libswscale-dev \
+    && curl -sL https://deb.nodesource.com/setup_16.x | bash - \
     && apt-get install nodejs -y \
     && npm --prefix frontend install \
     && npm --prefix frontend run build-prod \
     && mkdir -p staticassets \
     && mkdir -p /nefarious-db \
-    && python3.8 -m venv /env \
+    && python -m venv /env \
     && /env/bin/pip install -U pip \
-    && /env/bin/pip install --no-cache-dir -r requirements.txt \
+    && /env/bin/pip install --no-cache-dir --only-binary :all: --extra-index-url https://www.piwheels.org/simple -r requirements.txt \
     && /env/bin/python manage.py collectstatic --no-input \
     && rm -rf frontend/node_modules \
     && apt-get remove -y \
-        build-essential \
         nodejs \
-        python3.8-venv \
-        python3.8-dev \
-        libpq-dev \
-        libffi-dev \
-        libssl-dev \
-        virtualenv \
-        curl \
-        gnupg \
-        git \
     && apt-get autoremove -y \
     && rm -rf /var/lib/apt/lists/* \
     && true

README.md

@@ -27,6 +27,7 @@ Features:
 - [x] Imports existing libraries
 - [x] VPN integration (optional)
 - [x] Auto download subtitles via [opensubtitles](https://www.opensubtitles.com/) [api](https://opensubtitles.stoplight.io/)
+- [x] Auto detect & blacklist [spam/fake](https://github.com/lardbit/nefarious/pull/180) video content
 
 ### Contents
 

src/nefarious/api/mixins.py

@@ -1,12 +1,10 @@
 from rest_framework.decorators import action
 from rest_framework.response import Response
 
-from nefarious.models import NefariousSettings, TorrentBlacklist, WatchMediaBase
+from nefarious.models import WatchMediaBase
 from nefarious.tasks import send_websocket_message_task
-from nefarious.transmission import get_transmission_client
-from nefarious.utils import destroy_transmission_result
+from nefarious.utils import destroy_transmission_result, blacklist_media_and_retry
 from nefarious import websocket
-from nefarious.utils import logger_foreground
 
 
 class UserReferenceViewSetMixin:
@@ -25,49 +23,10 @@ class BlacklistAndRetryMixin:
     ViewSet Mixin which adds a torrent result to a "black list" and retries the media (i.e movie/tv) instance
     """
 
-    def _watch_media_task(self, watch_media_id: int):
-        """
-        Child classes need to define how to queue the new task
-        """
-        raise NotImplementedError
-
     @action(['post'], detail=True, url_path='blacklist-auto-retry')
     def blacklist_auto_retry(self, request, pk):
-
         watch_media = self.get_object()
-        nefarious_settings = NefariousSettings.get()
-
-        # if media still has a torrent reference
-        if watch_media.transmission_torrent_hash:
-
-            # add to blacklist
-            logger_foreground.info('Blacklisting {}'.format(watch_media.transmission_torrent_hash))
-            TorrentBlacklist.objects.get_or_create(
-                hash=watch_media.transmission_torrent_hash,
-                defaults=dict(
-                    name=str(watch_media),
-                )
-            )
-
-            # remove torrent and delete data
-            logger_foreground.info('Removing blacklisted torrent hash: {}'.format(watch_media.transmission_torrent_hash))
-            transmission_client = get_transmission_client(nefarious_settings=nefarious_settings)
-            transmission_client.remove_torrent([watch_media.transmission_torrent_hash], delete_data=True)
-
-            # unset torrent details
-            watch_media.transmission_torrent_hash = None
-            watch_media.transmission_torrent_name = None
-
-        # unset previous details
-        watch_media.collected = False
-        watch_media.collected_date = None
-        watch_media.download_path = None
-        watch_media.last_attempt_date = None
-        watch_media.save()
-
-        # re-queue search
-        self._watch_media_task(watch_media_id=watch_media.id)
-
+        blacklist_media_and_retry(watch_media)
         return Response(self.serializer_class(watch_media).data)
 
 

src/nefarious/api/viewsets.py

@@ -44,9 +44,6 @@ def perform_update(self, serializer):
         # create a task to download the movie
         watch_movie_task.delay(serializer.instance.id)
 
-    def _watch_media_task(self, watch_media_id: int):
-        watch_movie_task.delay(watch_media_id)
-
 
 @method_decorator(gzip_page, name='dispatch')
 class WatchTVShowViewSet(WebSocketMediaMessageUpdatedMixin, UserReferenceViewSetMixin, viewsets.ModelViewSet):
@@ -90,12 +87,6 @@ class WatchTVSeasonViewSet(WebSocketMediaMessageUpdatedMixin, DestroyTransmissio
     filter_backends = (DjangoFilterBackend, SearchFilter, OrderingFilter,)
     filter_fields = ('collected',)
 
-    def _watch_media_task(self, watch_media_id: int):
-        """
-        blacklist & retry function to queue the new task
-        """
-        watch_tv_show_season_task.delay(watch_media_id)
-
 
 @method_decorator(gzip_page, name='dispatch')
 class WatchTVSeasonRequestViewSet(WebSocketMediaMessageUpdatedMixin, UserReferenceViewSetMixin, viewsets.ModelViewSet):
@@ -164,9 +155,6 @@ class WatchTVEpisodeViewSet(WebSocketMediaMessageUpdatedMixin, DestroyTransmissi
     filter_backends = (DjangoFilterBackend, SearchFilter, OrderingFilter,)
     filter_fields = ('collected',)
 
-    def _watch_media_task(self, watch_media_id: int):
-        watch_tv_episode_task.delay(watch_media_id)
-
     def perform_create(self, serializer):
         super().perform_create(serializer)
         # create a task to download the episode

src/nefarious/importer/base.py

@@ -43,9 +43,6 @@ def _handle_missing_title(self, parser, path) -> tuple:
     def _get_parser(self, file_name):
         raise NotImplementedError
 
-    def _is_parser_exact_match(self, parser) -> bool:
-        return True
-
     def _get_tmdb_search_results(self, title):
         raise NotImplementedError
 
@@ -73,31 +70,28 @@ def ingest_path(self, file_path):
                         return False
                 file_extension = file_extension_match.group()
                 if file_extension in video_extensions():
-                    if self._is_parser_exact_match(parser):
-                        if self.media_class.objects.filter(download_path=file_path).exists():
-                            logger_background.info('[SKIP] skipping already-processed file "{}"'.format(file_path))
+                    if self.media_class.objects.filter(download_path=file_path).exists():
+                        logger_background.info('[SKIP] skipping already-processed file "{}"'.format(file_path))
+                        return False
+                    # get or set tmdb search results for this title in the cache
+                    tmdb_results = cache.get(title)
+                    if not tmdb_results:
+                        try:
+                            tmdb_results = self._get_tmdb_search_results(title)
+                        except HTTPError:
+                            logger_background.error('[ERROR_TMDB] tmdb search exception for title {} on file "{}"'.format(title, file_path))
                             return False
-                        # get or set tmdb search results for this title in the cache
-                        tmdb_results = cache.get(title)
-                        if not tmdb_results:
-                            try:
-                                tmdb_results = self._get_tmdb_search_results(title)
-                            except HTTPError:
-                                logger_background.error('[ERROR_TMDB] tmdb search exception for title {} on file "{}"'.format(title, file_path))
-                                return False
-                            cache.set(title, tmdb_results, 60 * 60)
-                        # loop over results for the exact match
-                        for tmdb_result in tmdb_results['results']:
-                            # normalize titles and see if they match
-                            if self._is_result_match_title(parser, tmdb_result, title):
-                                watch_media = self._handle_match(parser, tmdb_result, title, file_path)
-                                if watch_media:
-                                    logger_background.info('[MATCH] Saved media "{}" from file "{}"'.format(watch_media, file_path))
-                                    return watch_media
-                        else:  # for/else
-                            logger_background.warning('[NO_MATCH_MEDIA] No media match for title "{}" and file "{}"'.format(title, file_path))
-                    else:
-                        logger_background.warning('[NO_MATCH_EXACT] No exact title match for title "{}" and file "{}"'.format(title, file_path))
+                        cache.set(title, tmdb_results, 60 * 60)
+                    # loop over results for the exact match
+                    for tmdb_result in tmdb_results['results']:
+                        # normalize titles and see if they match
+                        if self._is_result_match_title(parser, tmdb_result, title):
+                            watch_media = self._handle_match(parser, tmdb_result, title, file_path)
+                            if watch_media:
+                                logger_background.info('[MATCH] Saved media "{}" from file "{}"'.format(watch_media, file_path))
+                                return watch_media
+                    else:  # for/else
+                        logger_background.warning('[NO_MATCH_MEDIA] No media match for title "{}" and file "{}"'.format(title, file_path))
                 else:
                     logger_background.warning('[NO_MATCH_VIDEO] No valid video file extension for file "{}"'.format(file_path))
             else:

src/nefarious/management/commands/video-detection.py

@@ -0,0 +1,20 @@
+from nefarious.video_detection import VideoDetect
+from django.core.management.base import BaseCommand
+
+
+class Command(BaseCommand):
+
+    help = 'Inspects a video file to determine how accurate it is'
+
+    def add_arguments(self, parser):
+        parser.add_argument('video_path', type=str)
+
+    def handle(self, *args, **options):
+        self.stdout.write(self.style.WARNING('Testing videos in path: {}'.format(options['video_path'])))
+        if not VideoDetect.has_valid_video_in_path(options['video_path']):
+            self.stdout.write(self.style.ERROR('Video frames too similar'))
+        else:
+            self.stdout.write(self.style.SUCCESS('Found accurate looking videos'))
+
+
+

src/nefarious/tasks.py

@@ -22,10 +22,10 @@
 from nefarious.processors import WatchMovieProcessor, WatchTVEpisodeProcessor, WatchTVSeasonProcessor
 from nefarious.tmdb import get_tmdb_client
 from nefarious.transmission import get_transmission_client
-from nefarious.utils import get_media_new_path_and_name, update_media_release_date
+from nefarious.utils import get_media_new_path_and_name, update_media_release_date, blacklist_media_and_retry
 from nefarious import websocket, notification
 from nefarious.utils import logger_background
-
+from nefarious.video_detection import VideoDetect
 
 app.conf.beat_schedule = {
     'Completed Media Task': {
@@ -157,9 +157,25 @@ def completed_media_task():
             # download is complete
             if torrent.progress == 100:
 
-                # flag media as completed
                 logger_background.info('Media completed: {}'.format(media))
 
+                # run video detection on the relevant video files for movies
+                if isinstance(media, WatchMovie):
+                    staging_path = os.path.join(
+                        settings.INTERNAL_DOWNLOAD_PATH,
+                        settings.UNPROCESSED_PATH,
+                        nefarious_settings.transmission_movie_download_dir.lstrip('/'),
+                        torrent.name,
+                    )
+                    try:
+                        if not VideoDetect.has_valid_video_in_path(staging_path):
+                            blacklist_media_and_retry(media)
+                            logger_background.error("Blacklisting video '{}' because no valid video was found: {}".format(media, staging_path))
+                            continue
+                    except Exception as e:
+                        logger_background.exception(e)
+                        logger_background.error('error during video detection for {} with path {}'.format(media, staging_path))
+
                 # special handling for tv seasons
                 if isinstance(media, WatchTVSeason):
 
@@ -203,7 +219,7 @@ def completed_media_task():
                 media_type, data = websocket.get_media_type_and_serialized_watch_media(media)
                 websocket.send_message(websocket.ACTION_UPDATED, media_type, data)
 
-                # send notification
+                # send user notification
                 notification.send_message(message='{} was downloaded'.format(media))
 
                 # define the import path

src/nefarious/utils.py

@@ -8,7 +8,7 @@
 from urllib.parse import urlparse
 from transmissionrpc import TransmissionError
 
-from nefarious.models import NefariousSettings, WatchMovie, WatchTVSeason, WatchTVEpisode, WatchMediaBase
+from nefarious.models import NefariousSettings, WatchMovie, WatchTVSeason, WatchTVEpisode, WatchMediaBase, TorrentBlacklist
 from nefarious.tmdb import get_tmdb_client
 from nefarious.transmission import get_transmission_client
 
@@ -231,3 +231,45 @@ def update_media_release_date(media, release_date):
         media.save()
     else:
         logger_background.warning('Skipping empty release date for {}'.format(media))
+
+
+def blacklist_media_and_retry(watch_media):
+    # import here to avoid circular dependencies
+    from nefarious.tasks import watch_tv_episode_task, watch_tv_show_season_task, watch_movie_task
+
+    nefarious_settings = NefariousSettings.get()
+
+    # if media still has a torrent reference
+    if watch_media.transmission_torrent_hash:
+        # add to blacklist
+        logger_foreground.info('Blacklisting {}'.format(watch_media.transmission_torrent_hash))
+        TorrentBlacklist.objects.get_or_create(
+            hash=watch_media.transmission_torrent_hash,
+            defaults=dict(
+                name=str(watch_media),
+            )
+        )
+
+        # remove torrent and delete data
+        logger_foreground.info('Removing blacklisted torrent hash: {}'.format(watch_media.transmission_torrent_hash))
+        transmission_client = get_transmission_client(nefarious_settings=nefarious_settings)
+        transmission_client.remove_torrent([watch_media.transmission_torrent_hash], delete_data=True)
+
+        # unset torrent details
+        watch_media.transmission_torrent_hash = None
+        watch_media.transmission_torrent_name = None
+
+    # unset previous details
+    watch_media.collected = False
+    watch_media.collected_date = None
+    watch_media.download_path = None
+    watch_media.last_attempt_date = None
+    watch_media.save()
+
+    # re-queue search
+    if isinstance(watch_media, WatchMovie):
+        watch_movie_task.delay(watch_media.id)
+    elif isinstance(watch_media, WatchTVSeason):
+        watch_tv_show_season_task.delay(watch_media.id)
+    elif isinstance(watch_media, WatchTVEpisode):
+        watch_tv_episode_task.delay(watch_media.id)