Merge branch 'dev'

README.md

@@ -26,6 +26,8 @@ This project is writen under the **python 3.6**, I haven't test the codes in
 any other python versions. So, I hightly recommend you to use python 3.6 to
 avoid any troubles.
 
+Also, we need **[ffmpeg](https://www.ffmpeg.org/download.html)** for post
+processing. Lack of ffmpeg may case some funtions not working.
 
 # Install
 

youtube_dl_webui/core.py

@@ -186,6 +186,7 @@ def event_batch(cls, svr, event, data, arg):
 class WorkMsgDispatcher(object):
 
     _task_mgr = None
+    logger = logging.getLogger('ydl_webui')
 
     @classmethod
     def init(cls, task_mgr):
@@ -212,10 +213,18 @@ def event_fatal(cls, svr, event, data, arg):
     @classmethod
     def event_progress(cls, svr, event, data, arg):
         tid, data = data['tid'], data['data']
-        cls._task_mgr.progress_update(tid, data)
+        try:
+            cls._task_mgr.progress_update(tid, data)
+        except TaskInexistenceError:
+            cls.logger.error('Cannot update progress, task does not exist')
 
-        if data['status'] == 'finished':
+    @classmethod
+    def event_worker_done(cls, svr, event, data, arg):
+        tid, data = data['tid'], data['data']
+        try:
             cls._task_mgr.finish_task(tid)
+        except TaskInexistenceError:
+            cls.logger.error('Cannot finish, task does not exist')
 
 
 def load_conf_from_file(cmd_args):
@@ -271,6 +280,7 @@ def __init__(self, cmd_args=None):
         self.msg_mgr.reg_event('log',        WorkMsgDispatcher.event_log)
         self.msg_mgr.reg_event('progress',   WorkMsgDispatcher.event_progress)
         self.msg_mgr.reg_event('fatal',      WorkMsgDispatcher.event_fatal)
+        self.msg_mgr.reg_event('worker_done',WorkMsgDispatcher.event_worker_done)
 
         self.server = Server(web_cli, self.conf['server']['host'], self.conf['server']['port'])
 

youtube_dl_webui/db.py

@@ -186,11 +186,7 @@ def delete_task(self, tid):
         if row is None:
             raise TaskInexistenceError('')
 
-        dl_file = None
-        if row['tmpfilename'] != '':
-            dl_file = row['tmpfilename']
-        elif row['filename'] != '':
-            dl_file = row['filename']
+        dl_file = row['filename']
 
         self.db.execute('DELETE FROM task_status WHERE tid=(?)', (tid, ))
         self.db.execute('DELETE FROM task_info WHERE tid=(?)', (tid, ))

youtube_dl_webui/msg.py

@@ -50,9 +50,11 @@ def __init__(self):
     def new_cli(self, cli_name=None):
         uuid = None
         if cli_name is not None:
+            # For named client, we create unique queue for communicating with server
             uuid = cli_name
             cli = CliMsg(cli_name, Queue(), self._svrQ)
         else:
+            # Anonymous client is a client who needn't to talk to the server.
             uuid = new_uuid()
             cli = CliMsg(uuid, None, self._svrQ)
 
@@ -61,6 +63,10 @@ def new_cli(self, cli_name=None):
         return cli
 
     def reg_event(self, event, cb_func, arg=None):
+        # callback functions should have the signature of callback(svr, event, data, arg)
+        #
+        # svr is an instance of SrvMsg class, so the callback can directly send
+        # mssages via svr to its corresponding client.
         self._evnt_cb_dict[event] = (cb_func, arg)
 
     def run(self):

youtube_dl_webui/task.py

@@ -4,6 +4,7 @@
 import logging
 import os
 import json
+import glob
 
 from time import time
 from collections import deque
@@ -149,7 +150,7 @@ def start_task(self, tid, ignore_state=False):
             if status['state'] == state_index['finished']:
                 raise TaskError('Task is finished')
 
-            task = Task(tid, self._msg_cli, ydl_opts=ydl_opts, info=info, 
+            task = Task(tid, self._msg_cli, ydl_opts=ydl_opts, info=info,
                         status=status, log_size=self._conf['general']['log_size'])
             self._tasks_dict[tid] = task
 
@@ -211,9 +212,14 @@ def delete_task(self, tid, del_file=False):
             raise TaskInexistenceError(e.msg)
 
         if del_file and dl_file is not None:
-            abs_dl_file = os.path.join(os.getcwd(), dl_file)
-            self.logger.debug('delete file: %s' %(abs_dl_file))
-            os.remove(abs_dl_file)
+            file_wo_ext, ext = dl_file, None
+            while ext != '':
+                file_wo_ext, ext = os.path.splitext(file_wo_ext)
+
+            for fname in os.listdir(os.getcwd()):
+                if fname.startswith(file_wo_ext):
+                    self.logger.debug('delete file: %s' %(fname))
+                    os.remove(os.path.join(os.getcwd(), fname))
 
     def query(self, tid, exerpt=True):
         db_ret = self._db.query_task(tid)
@@ -279,5 +285,10 @@ def launch_unfinished(self):
         tid_list = self._db.launch_unfinished()
 
         for t in tid_list:
-            self.start_task(t)
+            try:
+                self.start_task(t)
+            except TaskError as e:
+                self.logger.warn("Task %s is in downloading or finished state", tid)
+            except TaskInexistenceError:
+                self.logger.error('Task does not exist')
 

youtube_dl_webui/test/post_processor/bestvideo_bestaudio_merging.py

@@ -0,0 +1,35 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+from __future__ import unicode_literals
+from os import chdir
+import youtube_dl
+
+
+class MyLogger(object):
+    def debug(self, msg):
+        print('dbg: ', msg)
+
+    def warning(self, msg):
+        print('warn: ', msg)
+
+    def error(self, msg):
+        print('err: ', msg)
+
+
+def my_hook(d):
+    print(d)
+    if d['status'] == 'finished':
+        print('--------------- finish -----------------')
+
+if __name__ == '__main__':
+    chdir('/tmp')
+
+    ydl_opts = {
+        'format': 'bestvideo+bestaudio',
+        'logger': MyLogger(),
+        'progress_hooks': [my_hook],
+    }
+
+    with youtube_dl.YoutubeDL(ydl_opts) as ydl:
+        ret = ydl.extract_info('https://www.youtube.com/watch?v=jZvC7NWkeA0')

youtube_dl_webui/worker.py

@@ -126,6 +126,8 @@ def run(self):
                 event_handler = FatalEvent(self.tid, self.msg_cli)
                 event_handler.invalid_url(self.url);
 
+        self.msg_cli.put('worker_done', {'tid': self.tid, 'data': {}})
+
     def stop(self):
         self.logger.info('Terminating Process ...')
         self.terminate()