From 86e62e9faa2bbef421971de83b8b29a75c869379 Mon Sep 17 00:00:00 2001 From: d0u9 Date: Wed, 11 Apr 2018 15:58:00 +0800 Subject: [PATCH 1/8] correct version info --- setup.py | 2 +- youtube_dl_webui/templates/index.html | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index e64ebd7..b17e683 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ setup ( name='youtube_dl_webui', - version='0.2.1', + version='rolling', packages=['youtube_dl_webui'], license='GPL-2.0', author='d0u9, yuanyingfeiyu', diff --git a/youtube_dl_webui/templates/index.html b/youtube_dl_webui/templates/index.html index f1f4967..e63b3c7 100644 --- a/youtube_dl_webui/templates/index.html +++ b/youtube_dl_webui/templates/index.html @@ -141,7 +141,7 @@
About
- Version:web_dev + Version:rolling
Github:https://github.com/d0u9/youtube-dl-webui From 1cde9badcb6bb4af5843e4dfa2f20b547be2bd8d Mon Sep 17 00:00:00 2001 From: d0u9 Date: Thu, 28 Sep 2017 15:03:42 +0800 Subject: [PATCH 2/8] some bug fix --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index b17e683..64f99b9 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ setup ( name='youtube_dl_webui', - version='rolling', + version='dev', packages=['youtube_dl_webui'], license='GPL-2.0', author='d0u9, yuanyingfeiyu', From 3d47bccd020dd6906a6caf447f5f188043018df7 Mon Sep 17 00:00:00 2001 From: d0u9 Date: Wed, 11 Apr 2018 15:55:56 +0800 Subject: [PATCH 3/8] correct version --- youtube_dl_webui/templates/index.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl_webui/templates/index.html b/youtube_dl_webui/templates/index.html index e63b3c7..b1ac8c3 100644 --- a/youtube_dl_webui/templates/index.html +++ b/youtube_dl_webui/templates/index.html @@ -141,7 +141,7 @@
About
- Version:rolling + Version:dev
Github:https://github.com/d0u9/youtube-dl-webui From c7879bcf7ebac9df24d2435ca5b08bc735e2f425 Mon Sep 17 00:00:00 2001 From: d0u9 Date: Wed, 11 Apr 2018 16:28:16 +0800 Subject: [PATCH 4/8] Add test for post porcessor --- .../test/post_processor/post_processor.py | 40 +++++++++++++++++++ youtube_dl_webui/test/postprocessor.py | 40 +++++++++++++++++++ 2 files changed, 80 insertions(+) create mode 100644 youtube_dl_webui/test/post_processor/post_processor.py create mode 100644 youtube_dl_webui/test/postprocessor.py diff --git a/youtube_dl_webui/test/post_processor/post_processor.py b/youtube_dl_webui/test/post_processor/post_processor.py new file mode 100644 index 0000000..a6846e8 --- /dev/null +++ b/youtube_dl_webui/test/post_processor/post_processor.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +from __future__ import unicode_literals +from os import chdir +import youtube_dl + + +class MyLogger(object): + def debug(self, msg): + pass + + def warning(self, msg): + pass + + def error(self, msg): + print(msg) + + +def my_hook(d): + if d['status'] == 'finished': + print('Done downloading, now converting ...') + +if __name__ == '__main__': + chdir('/tmp') + + ydl_opts = { + # 'format': 'bestaudio+best', + 'format': 'bestvideo+bestaudio', + 'postprocessors': [{ + 'key': 'FFmpegExtractAudio', + 'preferredcodec': 'mp3', + 'preferredquality': '192', + }], + 'logger': MyLogger(), + 'progress_hooks': [my_hook], + } + + with youtube_dl.YoutubeDL(ydl_opts) as ydl: + ydl.download(['https://www.youtube.com/watch?v=BaW_jenozKc']) diff --git a/youtube_dl_webui/test/postprocessor.py b/youtube_dl_webui/test/postprocessor.py new file mode 100644 index 0000000..a6846e8 --- /dev/null +++ b/youtube_dl_webui/test/postprocessor.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +from __future__ import unicode_literals +from os import chdir +import youtube_dl + + +class MyLogger(object): + def debug(self, msg): + pass + + def warning(self, msg): + pass + + def error(self, msg): + print(msg) + + +def my_hook(d): + if d['status'] == 'finished': + print('Done downloading, now converting ...') + +if __name__ == '__main__': + chdir('/tmp') + + ydl_opts = { + # 'format': 'bestaudio+best', + 'format': 'bestvideo+bestaudio', + 'postprocessors': [{ + 'key': 'FFmpegExtractAudio', + 'preferredcodec': 'mp3', + 'preferredquality': '192', + }], + 'logger': MyLogger(), + 'progress_hooks': [my_hook], + } + + with youtube_dl.YoutubeDL(ydl_opts) as ydl: + ydl.download(['https://www.youtube.com/watch?v=BaW_jenozKc']) From 96062d9a2e96821a48e48cdf552e09a9f12f5525 Mon Sep 17 00:00:00 2001 From: d0u9 Date: Wed, 11 Apr 2018 17:53:07 +0800 Subject: [PATCH 5/8] Fix a potential error which may case server halt. Some exceptions are ignored. In some critical situation, race condition may cause server halt. --- youtube_dl_webui/core.py | 12 ++++++++---- youtube_dl_webui/msg.py | 6 ++++++ youtube_dl_webui/task.py | 9 +++++++-- 3 files changed, 21 insertions(+), 6 deletions(-) diff --git a/youtube_dl_webui/core.py b/youtube_dl_webui/core.py index f544b1a..5544947 100644 --- a/youtube_dl_webui/core.py +++ b/youtube_dl_webui/core.py @@ -186,6 +186,7 @@ def event_batch(cls, svr, event, data, arg): class WorkMsgDispatcher(object): _task_mgr = None + logger = logging.getLogger('ydl_webui') @classmethod def init(cls, task_mgr): @@ -212,10 +213,13 @@ def event_fatal(cls, svr, event, data, arg): @classmethod def event_progress(cls, svr, event, data, arg): tid, data = data['tid'], data['data'] - cls._task_mgr.progress_update(tid, data) - - if data['status'] == 'finished': - cls._task_mgr.finish_task(tid) + try: + cls._task_mgr.progress_update(tid, data) + except TaskInexistenceError: + cls.logger.error('Cannot update progress, task does not exist') + else: + if data['status'] == 'finished': + cls._task_mgr.finish_task(tid) def load_conf_from_file(cmd_args): diff --git a/youtube_dl_webui/msg.py b/youtube_dl_webui/msg.py index caba0ba..2ca3650 100644 --- a/youtube_dl_webui/msg.py +++ b/youtube_dl_webui/msg.py @@ -50,9 +50,11 @@ def __init__(self): def new_cli(self, cli_name=None): uuid = None if cli_name is not None: + # For named client, we create unique queue for communicating with server uuid = cli_name cli = CliMsg(cli_name, Queue(), self._svrQ) else: + # Anonymous client is a client who needn't to talk to the server. uuid = new_uuid() cli = CliMsg(uuid, None, self._svrQ) @@ -61,6 +63,10 @@ def new_cli(self, cli_name=None): return cli def reg_event(self, event, cb_func, arg=None): + # callback functions should have the signature of callback(svr, event, data, arg) + # + # svr is an instance of SrvMsg class, so the callback can directly send + # mssages via svr to its corresponding client. self._evnt_cb_dict[event] = (cb_func, arg) def run(self): diff --git a/youtube_dl_webui/task.py b/youtube_dl_webui/task.py index b9e0a3c..218bb97 100644 --- a/youtube_dl_webui/task.py +++ b/youtube_dl_webui/task.py @@ -149,7 +149,7 @@ def start_task(self, tid, ignore_state=False): if status['state'] == state_index['finished']: raise TaskError('Task is finished') - task = Task(tid, self._msg_cli, ydl_opts=ydl_opts, info=info, + task = Task(tid, self._msg_cli, ydl_opts=ydl_opts, info=info, status=status, log_size=self._conf['general']['log_size']) self._tasks_dict[tid] = task @@ -279,5 +279,10 @@ def launch_unfinished(self): tid_list = self._db.launch_unfinished() for t in tid_list: - self.start_task(t) + try: + self.start_task(t) + except TaskError as e: + self.logger.warn("Task %s is in downloading or finished state", tid) + except TaskInexistenceError: + self.logger.error('Task does not exist') From 083769de66086bdb31b80075b2485d893e7db244 Mon Sep 17 00:00:00 2001 From: d0u9 Date: Tue, 17 Apr 2018 15:29:28 +0800 Subject: [PATCH 6/8] add test for testing format merging --- ...ssor.py => bestvideo_bestaudio_merging.py} | 17 +++----- youtube_dl_webui/test/postprocessor.py | 40 ------------------- 2 files changed, 6 insertions(+), 51 deletions(-) rename youtube_dl_webui/test/post_processor/{post_processor.py => bestvideo_bestaudio_merging.py} (58%) delete mode 100644 youtube_dl_webui/test/postprocessor.py diff --git a/youtube_dl_webui/test/post_processor/post_processor.py b/youtube_dl_webui/test/post_processor/bestvideo_bestaudio_merging.py similarity index 58% rename from youtube_dl_webui/test/post_processor/post_processor.py rename to youtube_dl_webui/test/post_processor/bestvideo_bestaudio_merging.py index a6846e8..f91007a 100644 --- a/youtube_dl_webui/test/post_processor/post_processor.py +++ b/youtube_dl_webui/test/post_processor/bestvideo_bestaudio_merging.py @@ -8,33 +8,28 @@ class MyLogger(object): def debug(self, msg): - pass + print('dbg: ', msg) def warning(self, msg): - pass + print('warn: ', msg) def error(self, msg): - print(msg) + print('err: ', msg) def my_hook(d): + print(d) if d['status'] == 'finished': - print('Done downloading, now converting ...') + print('--------------- finish -----------------') if __name__ == '__main__': chdir('/tmp') ydl_opts = { - # 'format': 'bestaudio+best', 'format': 'bestvideo+bestaudio', - 'postprocessors': [{ - 'key': 'FFmpegExtractAudio', - 'preferredcodec': 'mp3', - 'preferredquality': '192', - }], 'logger': MyLogger(), 'progress_hooks': [my_hook], } with youtube_dl.YoutubeDL(ydl_opts) as ydl: - ydl.download(['https://www.youtube.com/watch?v=BaW_jenozKc']) + ret = ydl.extract_info('https://www.youtube.com/watch?v=jZvC7NWkeA0') diff --git a/youtube_dl_webui/test/postprocessor.py b/youtube_dl_webui/test/postprocessor.py deleted file mode 100644 index a6846e8..0000000 --- a/youtube_dl_webui/test/postprocessor.py +++ /dev/null @@ -1,40 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -from __future__ import unicode_literals -from os import chdir -import youtube_dl - - -class MyLogger(object): - def debug(self, msg): - pass - - def warning(self, msg): - pass - - def error(self, msg): - print(msg) - - -def my_hook(d): - if d['status'] == 'finished': - print('Done downloading, now converting ...') - -if __name__ == '__main__': - chdir('/tmp') - - ydl_opts = { - # 'format': 'bestaudio+best', - 'format': 'bestvideo+bestaudio', - 'postprocessors': [{ - 'key': 'FFmpegExtractAudio', - 'preferredcodec': 'mp3', - 'preferredquality': '192', - }], - 'logger': MyLogger(), - 'progress_hooks': [my_hook], - } - - with youtube_dl.YoutubeDL(ydl_opts) as ydl: - ydl.download(['https://www.youtube.com/watch?v=BaW_jenozKc']) From 9f534bf2fbd7455f0c326856295a8b74cd9aa166 Mon Sep 17 00:00:00 2001 From: d0u9 Date: Wed, 18 Apr 2018 15:37:56 +0800 Subject: [PATCH 7/8] Fullly support bestvideo + bestaudio format. For some high quality videos, video and audio files will be downloaded separately if using `bestvideo+bestaudio` format. --- README.md | 2 ++ youtube_dl_webui/core.py | 9 ++++++--- youtube_dl_webui/db.py | 6 +----- youtube_dl_webui/task.py | 12 +++++++++--- youtube_dl_webui/worker.py | 2 ++ 5 files changed, 20 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 0bd9baa..16cd091 100644 --- a/README.md +++ b/README.md @@ -26,6 +26,8 @@ This project is writen under the **python 3.6**, I haven't test the codes in any other python versions. So, I hightly recommend you to use python 3.6 to avoid any troubles. +Also, we need **[ffmpeg](https://www.ffmpeg.org/download.html)** for post +processing. Lack of ffmpeg may case some funtions not working. # Install diff --git a/youtube_dl_webui/core.py b/youtube_dl_webui/core.py index 5544947..0eabbc4 100644 --- a/youtube_dl_webui/core.py +++ b/youtube_dl_webui/core.py @@ -217,9 +217,11 @@ def event_progress(cls, svr, event, data, arg): cls._task_mgr.progress_update(tid, data) except TaskInexistenceError: cls.logger.error('Cannot update progress, task does not exist') - else: - if data['status'] == 'finished': - cls._task_mgr.finish_task(tid) + + @classmethod + def event_worker_done(cls, svr, event, data, arg): + tid, data = data['tid'], data['data'] + cls._task_mgr.finish_task(tid) def load_conf_from_file(cmd_args): @@ -275,6 +277,7 @@ def __init__(self, cmd_args=None): self.msg_mgr.reg_event('log', WorkMsgDispatcher.event_log) self.msg_mgr.reg_event('progress', WorkMsgDispatcher.event_progress) self.msg_mgr.reg_event('fatal', WorkMsgDispatcher.event_fatal) + self.msg_mgr.reg_event('worker_done',WorkMsgDispatcher.event_worker_done) self.server = Server(web_cli, self.conf['server']['host'], self.conf['server']['port']) diff --git a/youtube_dl_webui/db.py b/youtube_dl_webui/db.py index a43831c..7edd610 100644 --- a/youtube_dl_webui/db.py +++ b/youtube_dl_webui/db.py @@ -186,11 +186,7 @@ def delete_task(self, tid): if row is None: raise TaskInexistenceError('') - dl_file = None - if row['tmpfilename'] != '': - dl_file = row['tmpfilename'] - elif row['filename'] != '': - dl_file = row['filename'] + dl_file = row['filename'] self.db.execute('DELETE FROM task_status WHERE tid=(?)', (tid, )) self.db.execute('DELETE FROM task_info WHERE tid=(?)', (tid, )) diff --git a/youtube_dl_webui/task.py b/youtube_dl_webui/task.py index 218bb97..6329054 100644 --- a/youtube_dl_webui/task.py +++ b/youtube_dl_webui/task.py @@ -4,6 +4,7 @@ import logging import os import json +import glob from time import time from collections import deque @@ -211,9 +212,14 @@ def delete_task(self, tid, del_file=False): raise TaskInexistenceError(e.msg) if del_file and dl_file is not None: - abs_dl_file = os.path.join(os.getcwd(), dl_file) - self.logger.debug('delete file: %s' %(abs_dl_file)) - os.remove(abs_dl_file) + file_wo_ext, ext = dl_file, None + while ext != '': + file_wo_ext, ext = os.path.splitext(file_wo_ext) + + for fname in os.listdir(os.getcwd()): + if fname.startswith(file_wo_ext): + self.logger.debug('delete file: %s' %(fname)) + os.remove(os.path.join(os.getcwd(), fname)) def query(self, tid, exerpt=True): db_ret = self._db.query_task(tid) diff --git a/youtube_dl_webui/worker.py b/youtube_dl_webui/worker.py index 9249389..f736e17 100644 --- a/youtube_dl_webui/worker.py +++ b/youtube_dl_webui/worker.py @@ -126,6 +126,8 @@ def run(self): event_handler = FatalEvent(self.tid, self.msg_cli) event_handler.invalid_url(self.url); + self.msg_cli.put('worker_done', {'tid': self.tid, 'data': {}}) + def stop(self): self.logger.info('Terminating Process ...') self.terminate() From 0548e71d04b1ce3d944a2914ffa42b944038d58d Mon Sep 17 00:00:00 2001 From: d0u9 Date: Wed, 25 Apr 2018 17:13:35 +0800 Subject: [PATCH 8/8] bug fix --- youtube_dl_webui/core.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/youtube_dl_webui/core.py b/youtube_dl_webui/core.py index 0eabbc4..1a26d38 100644 --- a/youtube_dl_webui/core.py +++ b/youtube_dl_webui/core.py @@ -221,7 +221,10 @@ def event_progress(cls, svr, event, data, arg): @classmethod def event_worker_done(cls, svr, event, data, arg): tid, data = data['tid'], data['data'] - cls._task_mgr.finish_task(tid) + try: + cls._task_mgr.finish_task(tid) + except TaskInexistenceError: + cls.logger.error('Cannot finish, task does not exist') def load_conf_from_file(cmd_args):