From de0e0ba729cac6b3a9407cc6a1629aa3072f0b93 Mon Sep 17 00:00:00 2001 From: hect0x7 <93357912+hect0x7@users.noreply.github.com> Date: Sun, 29 Oct 2023 21:44:27 +0800 Subject: [PATCH] =?UTF-8?q?v2.3.14:=20=E6=8F=92=E4=BB=B6=E6=94=AF=E6=8C=81?= =?UTF-8?q?=E8=B0=83=E7=94=A8=E7=8E=AF=E5=A2=83=E5=8F=98=E9=87=8F=EF=BC=8C?= =?UTF-8?q?=E6=96=B0=E5=A2=9E=E6=8F=92=E4=BB=B6=E3=80=90=E5=8F=91=E9=80=81?= =?UTF-8?q?QQ=E9=82=AE=E4=BB=B6=E3=80=91=EF=BC=8C=E4=BC=98=E5=8C=96?= =?UTF-8?q?=E7=A7=BB=E5=8A=A8=E7=AB=AF=E7=BC=93=E5=AD=98scramble=5Fid?= =?UTF-8?q?=E5=87=8F=E5=B0=91=E8=AF=B7=E6=B1=82=E3=80=82=20(#157)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/download.yml | 10 ++++ .github/workflows/download_dispatch.yml | 11 +++- README.md | 4 +- assets/docs/sources/index.md | 37 ++++++------ assets/docs/sources/option_file_syntax.md | 9 +++ assets/option/option_workflow_download.yml | 20 ++++++- src/jmcomic/__init__.py | 2 +- src/jmcomic/jm_client_impl.py | 13 ++++- src/jmcomic/jm_option.py | 6 +- src/jmcomic/jm_plugin.py | 68 ++++++++++++++++------ src/jmcomic/jm_toolkit.py | 9 ++- usage/workflow_download.py | 22 +------ 12 files changed, 140 insertions(+), 71 deletions(-) diff --git a/.github/workflows/download.yml b/.github/workflows/download.yml index 704e3337..5abd7b38 100644 --- a/.github/workflows/download.yml +++ b/.github/workflows/download.yml @@ -12,8 +12,18 @@ jobs: crawler: runs-on: ubuntu-latest env: + # 登录相关secrets JM_USERNAME: ${{ secrets.JM_USERNAME }} JM_PASSWORD: ${{ secrets.JM_PASSWORD }} + + # 邮件相关secrets + EMAIL_FROM: ${{ secrets.EMAIL_FROM }} + EMAIL_TO: ${{ secrets.EMAIL_TO }} + EMAIL_PASS: ${{ secrets.EMAIL_PASS }} + EMAIL_TITLE: ${{ secrets.EMAIL_TITLE }} + EMAIL_CONTENT: ${{ secrets.EMAIL_CONTENT }} + + # 固定值 JM_DOWNLOAD_DIR: /home/runner/work/jmcomic/download/ ZIP_NAME: '本子.tar.gz' UPLOAD_NAME: '下载完成的本子' diff --git a/.github/workflows/download_dispatch.yml b/.github/workflows/download_dispatch.yml index 1277af34..9e37c01b 100644 --- a/.github/workflows/download_dispatch.yml +++ b/.github/workflows/download_dispatch.yml @@ -68,9 +68,18 @@ jobs: ZIP_NAME: ${{ github.event.inputs.ZIP_NAME }} UPLOAD_NAME: ${{ github.event.inputs.UPLOAD_NAME }} IMAGE_SUFFIX: ${{ github.event.inputs.IMAGE_SUFFIX }} - # secrets + + # 登录相关secrets JM_USERNAME: ${{ secrets.JM_USERNAME }} JM_PASSWORD: ${{ secrets.JM_PASSWORD }} + + # 邮件相关secrets + EMAIL_FROM: ${{ secrets.EMAIL_FROM }} + EMAIL_TO: ${{ secrets.EMAIL_TO }} + EMAIL_PASS: ${{ secrets.EMAIL_PASS }} + EMAIL_TITLE: ${{ secrets.EMAIL_TITLE }} + EMAIL_CONTENT: ${{ secrets.EMAIL_CONTENT }} + # 固定值 JM_DOWNLOAD_DIR: /home/runner/work/jmcomic/download/ diff --git a/README.md b/README.md index ec80e8e7..ef899599 100644 --- a/README.md +++ b/README.md @@ -54,7 +54,7 @@ $ jmcomic 422866 - **可扩展性强** - **支持Plugin插件，可以方便地扩展功能，以及使用别人的插件** - - 目前内置支持的插件有：`登录插件` `硬件占用监控插件` `只下载新章插件` `压缩文件插件` `下载特定后缀图片插件` + - 目前内置支持的插件有：`登录插件` `硬件占用监控插件` `只下载新章插件` `压缩文件插件` `下载特定后缀图片插件` `发送QQ邮件插件` - 支持自定义本子/章节/图片下载前后的回调函数 - 支持自定义debug日志 - 支持自定义类：`Downloader（负责调度）` `Option（负责配置）` `Client（负责请求）` `实体类`等 @@ -65,7 +65,7 @@ $ jmcomic 422866 下面列出一些常用的文档链接： -* [option配置文件语法](./assets/docs/sources/option_file_syntax.md) +* [option配置文件语法（包含插件配置）](./assets/docs/sources/option_file_syntax.md) * [常用类和方法演示（下载本子、获取实体类、搜索本子）](assets/docs/sources/tutorial/3_demo.md) * [命令行使用教程](assets/docs/sources/tutorial/2_command_line.md) * [GitHub Actions使用教程](./assets/docs/sources/tutorial/1_github_actions.md) diff --git a/assets/docs/sources/index.md b/assets/docs/sources/index.md index 6604d90c..2e777c69 100644 --- a/assets/docs/sources/index.md +++ b/assets/docs/sources/index.md @@ -7,34 +7,31 @@ Python API for JMComic（禁漫天堂） - Bypasses Cloudflare anti-bot measures. - Multiple usage ways: - - GitHub Actions: Requires only a GitHub account. (See - tutorial → [Tutorial - Download Album via GitHub Actions](./tutorial/1_github_actions.md)) - - - Command line: No need to write Python code, simple and easy to use. (See tutorial → [Tutorial - Download Album via Command Line](./tutorial/2_command_line.md)) - - Python code: The most flexible and powerful way, requires some basic knowledge of Python programming. - + - GitHub Actions: Requires only a GitHub account. (See + tutorial → [Tutorial - Download Album via GitHub Actions](./tutorial/1_github_actions.md)) + - Command line: No need to write Python code, simple and easy to use. (See tutorial → [Tutorial - Download Album via Command Line](./tutorial/2_command_line.md)) + - Python code: The most flexible and powerful way, requires some basic knowledge of Python programming. - Supports two client implementations: web-based and mobile-based. Switchable through configuration (mobile-based has better IP compatibility, web-based has higher efficiency). - Supports automatic request retry and domain switching mechanism. - Multi-threaded downloading (can be fine-tuned to one thread per image, highly efficient). - Highly configurable: - - Can be used without configuration, very convenient. - - Configuration can be generated from a configuration file, supports multiple file formats. - - Configuration options - include: `request domain`, `client implementation`, `number of chapters/images downloaded simultaneously`, `image format conversion`, `download path rules`, `request metadata (headers, cookies, proxies)`, - and more. - + - Can be used without configuration, very convenient. + - Configuration can be generated from a configuration file, supports multiple file formats. + - Configuration options + include: `request domain`, `client implementation`, `number of chapters/images downloaded simultaneously`, `image format conversion`, `download path rules`, `request metadata (headers, cookies, proxies)`, + and more. - Highly extensible: - - Supports Plugin plugins for easy functionality extension and use of other plugins. - - Currently built-in - plugins: `login plugin`, `hardware usage monitoring plugin`, `only download new chapters plugin`, `zip compression plugin`. - - Supports custom callback functions before and after downloading album/chapter/images. - - Supports custom debug logging. - - Supports custom core - classes: `Downloader (responsible for scheduling)`, `Option (responsible for configuration)`, `Client (responsible for requests)`, `entity classes`, - and more. + - Supports Plugin plugins for easy functionality extension and use of other plugins. + - Currently built-in + plugins: `login plugin`, `hardware usage monitoring plugin`, `only download new chapters plugin`, `zip compression plugin`, `image suffix filter plugin` `send qq email plugin`. + - Supports custom callback functions before and after downloading album/chapter/images. + - Supports custom debug logging. + - Supports custom core + classes: `Downloader (responsible for scheduling)`, `Option (responsible for configuration)`, `Client (responsible for requests)`, `entity classes`, + and more. ## Install diff --git a/assets/docs/sources/option_file_syntax.md b/assets/docs/sources/option_file_syntax.md index 6a6a95b8..8fcc8a6f 100644 --- a/assets/docs/sources/option_file_syntax.md +++ b/assets/docs/sources/option_file_syntax.md @@ -93,6 +93,7 @@ dir_rule: ## 3. option插件配置项 ```yml # 插件的配置示例 +# 当kwargs的值为字符串类型时，支持使用环境变量，语法为 ${环境变量名} plugins: after_init: - plugin: usage_log # 实时打印硬件占用率的插件 @@ -127,4 +128,12 @@ plugins: zip_dir: D:/jmcomic/zip/ # 压缩文件存放的文件夹 delete_original_file: true # 压缩成功后，删除所有原文件和文件夹 + - plugin: send_qq_email # 发送qq邮件插件 + kwargs: + msg_from: ${EMAIL} # 发件人 + msg_to: aaa@qq.com # 收件人 + password: dkjlakdjlkas # 发件人的授权码 + title: jmcomic # 标题 + content: jmcomic finished !!! # 内容 + ``` \ No newline at end of file diff --git a/assets/option/option_workflow_download.yml b/assets/option/option_workflow_download.yml index 4bdbc023..d34ed8db 100644 --- a/assets/option/option_workflow_download.yml +++ b/assets/option/option_workflow_download.yml @@ -5,7 +5,7 @@ dir_rule: client: domain: - html: [jmcomic1.me, jmcomic.me] + html: [ jmcomic1.me, jmcomic.me ] # 插件配置 plugins: @@ -15,7 +15,21 @@ plugins: interval: 0.5 # 间隔时间 enable_warning: false # 不告警 - - plugin: client_proxy + - plugin: client_proxy # 提高移动端的请求效率的插件 kwargs: proxy_client_key: cl_proxy_future - whitelist: [ api, ] \ No newline at end of file + whitelist: [ api, ] + + - plugin: login # 登录插件 + kwargs: + username: ${JM_USERNAME} + password: ${JM_PASSWORD} + + after_download: # 全部下载完成以后 + - plugin: send_qq_email # 发送邮件，如果未配置下面的前3个环境变量则不会发送。 + kwargs: + msg_from: ${EMAIL_FROM} + msg_to: ${EMAIL_TO} + password: ${EMAIL_PASS} + title: ${EMAIL_TITLE} + content: ${EMAIL_CONTENT} \ No newline at end of file diff --git a/src/jmcomic/__init__.py b/src/jmcomic/__init__.py index 8dc8bc74..dc437561 100644 --- a/src/jmcomic/__init__.py +++ b/src/jmcomic/__init__.py @@ -2,7 +2,7 @@ # 被依赖方 <--- 使用方 # config <--- entity <--- toolkit <--- client <--- option <--- downloader -__version__ = '2.3.12' +__version__ = '2.3.14' from .api import * from .jm_plugin import * diff --git a/src/jmcomic/jm_client_impl.py b/src/jmcomic/jm_client_impl.py index 1275c68a..365927a8 100644 --- a/src/jmcomic/jm_client_impl.py +++ b/src/jmcomic/jm_client_impl.py @@ -443,7 +443,7 @@ def get_photo_detail(self, return photo - def get_scramble_id(self, photo_id): + def get_scramble_id(self, photo_id, album_id=None): """ 带有缓存的fetch_scramble_id，缓存位于JmModuleConfig.SCRAMBLE_CACHE """ @@ -451,8 +451,14 @@ def get_scramble_id(self, photo_id): if photo_id in cache: return cache[photo_id] + if album_id is not None and album_id in cache: + return cache[album_id] + scramble_id = self.fetch_scramble_id(photo_id) cache[photo_id] = scramble_id + if album_id is not None: + cache[album_id] = scramble_id + return scramble_id def fetch_detail_entity(self, apid, clazz): @@ -511,10 +517,11 @@ def fetch_photo_additional_field(self, photo: JmPhotoDetail, fetch_album: bool, 23做法要改不止一处地方 """ if fetch_album: - photo.from_album = self.get_album_detail(photo.photo_id) + photo.from_album = self.get_album_detail(photo.album_id) if fetch_scramble_id: - photo.scramble_id = self.get_scramble_id(photo.album_id) + # 同album的scramble_id相同 + photo.scramble_id = self.get_scramble_id(photo.photo_id, photo.album_id) def setting(self) -> JmApiResp: """ diff --git a/src/jmcomic/jm_option.py b/src/jmcomic/jm_option.py index 5f61e0dd..2426676d 100644 --- a/src/jmcomic/jm_option.py +++ b/src/jmcomic/jm_option.py @@ -381,7 +381,7 @@ def decide_postman_headers(self, client_key, domain): if is_client_type(JmApiClient): # 移动端 # 不配置headers，由client每次请求前创建headers - return {} + return None if is_client_type(JmHtmlClient): # 网页端 @@ -487,6 +487,10 @@ def fix_kwargs(self, kwargs) -> Dict[str, Any]: new_kwargs: Dict[str, Any] = {} for k, v in kwargs.items(): + if isinstance(v, str): + newv = JmcomicText.parse_dsl_text(v) + v = newv + if isinstance(k, str): new_kwargs[k] = v continue diff --git a/src/jmcomic/jm_plugin.py b/src/jmcomic/jm_plugin.py index 4a071fb9..dd192504 100644 --- a/src/jmcomic/jm_plugin.py +++ b/src/jmcomic/jm_plugin.py @@ -26,6 +26,13 @@ def build(cls, option: JmOption) -> 'JmOptionPlugin': """ return cls(option) + @classmethod + def debug(cls, msg, topic=None): + jm_debug( + topic=f'plugin.{cls.plugin_key}' + (f'.{topic}' if topic is not None else ''), + msg=msg + ) + class JmLoginPlugin(JmOptionPlugin): """ @@ -33,9 +40,12 @@ class JmLoginPlugin(JmOptionPlugin): """ plugin_key = 'login' - def invoke(self, username, password) -> None: - assert isinstance(username, str), '用户名必须是str' - assert isinstance(password, str), '密码必须是str' + def invoke(self, + username: str, + password: str, + ) -> None: + if not (username and password): + return client = self.option.new_jm_client() client.login(username, password) @@ -45,7 +55,7 @@ def invoke(self, username, password) -> None: meta_data = postman.get('meta_data', {}) meta_data['cookies'] = cookies postman['meta_data'] = meta_data - jm_debug('plugin.login', '登录成功') + self.debug('登录成功') class UsageLogPlugin(JmOptionPlugin): @@ -119,7 +129,7 @@ def warning(): if len(warning_msg_list) != 0: warning_msg_list.insert(0, '硬件占用告警，占用过高可能导致系统卡死！') warning_msg_list.append('') - jm_debug('plugin.psutil.warning', '\n'.join(warning_msg_list)) + self.debug('\n'.join(warning_msg_list), topic='warning') while True: # 获取CPU占用率（0~100） @@ -140,7 +150,7 @@ def warning(): # f"发送的字节数: {network_bytes_sent}", # f"接收的字节数: {network_bytes_received}", ]) - jm_debug('plugin.psutil.log', msg) + self.debug(msg, topic='log') if enable_warning is True: # 警告 @@ -254,12 +264,13 @@ def zip_photo(self, photo, image_list: list, zip_path: str): all_filepath = set(map(lambda t: t[0], image_list)) if len(all_filepath) == 0: - jm_debug('plugin.zip.skip', '无下载文件，无需压缩') + self.debug('无下载文件，无需压缩', 'skip') return from common import backup_dir_to_zip backup_dir_to_zip(photo_dir, zip_path, acceptor=lambda f: f in all_filepath) - jm_debug('plugin.zip.finish', f'压缩章节[{photo.photo_id}]成功 → {zip_path}') + self.debug(f'压缩章节[{photo.photo_id}]成功 → {zip_path}', 'finish') + return photo_dir def zip_album(self, album, photo_dict: dict, zip_path): @@ -276,7 +287,7 @@ def zip_album(self, album, photo_dict: dict, zip_path): all_filepath.add(path) if len(all_filepath) == 0: - jm_debug('plugin.zip.skip', '无下载文件，无需压缩') + self.debug('无下载文件，无需压缩', 'skip') return from common import backup_dir_to_zip @@ -286,7 +297,7 @@ def zip_album(self, album, photo_dict: dict, zip_path): acceptor=lambda f: f in all_filepath ) - jm_debug('plugin.zip.finish', f'压缩本子[{album.album_id}]成功 → {zip_path}') + self.debug(f'压缩本子[{album.album_id}]成功 → {zip_path}', 'finish') return album_dir def after_zip(self, dir_zip_dict: Dict[str, str]): @@ -319,12 +330,12 @@ def delete_all_files_and_empty_dir(self, all_downloaded: dict, dir_list: List[st for photo, image_list in photo_dict.items(): for f, image in image_list: os.remove(f) - jm_debug('plugin.zip.remove', f'删除原文件: {f}') + self.debug(f'删除原文件: {f}', 'remove') for d in dir_list: if len(os.listdir(d)) == 0: os.removedirs(d) - jm_debug('plugin.zip.remove', f'删除文件夹: {d}') + self.debug(f'删除文件夹: {d}', 'remove') class ClientProxyPlugin(JmOptionPlugin): @@ -338,7 +349,7 @@ def invoke(self, if whitelist is not None: whitelist = set(whitelist) - clazz = JmModuleConfig.client_impl_class(proxy_client_key) + proxy_clazz = JmModuleConfig.client_impl_class(proxy_client_key) clazz_init_kwargs = kwargs new_jm_client = self.option.new_jm_client @@ -347,8 +358,8 @@ def hook_new_jm_client(*args, **kwargs): if whitelist is not None and client.client_key not in whitelist: return client - jm_debug('plugin.client_proxy', f'proxy client {client} with {proxy_client_key}') - return clazz(client, **clazz_init_kwargs) + self.debug(f'proxy client {client} with {proxy_clazz}') + return proxy_clazz(client, **clazz_init_kwargs) self.option.new_jm_client = hook_new_jm_client @@ -368,9 +379,8 @@ def invoke(self, def apply_filter_then_decide_cache(image: JmImageDetail): if image.img_file_suffix not in allowed_suffix_set: - jm_debug('image.filter.skip', - f'跳过下载图片: {image.tag}，' - f'因为其后缀\'{image.img_file_suffix}\'不在允许的后缀集合{allowed_suffix_set}内') + self.debug(f'跳过下载图片: {image.tag}，' + f'因为其后缀\'{image.img_file_suffix}\'不在允许的后缀集合{allowed_suffix_set}内') # hook is_exists True to skip download image.is_exists = True return True @@ -379,3 +389,25 @@ def apply_filter_then_decide_cache(image: JmImageDetail): return option_decide_cache(image) self.option.decide_download_cache = apply_filter_then_decide_cache + + +class SendQQEmailPlugin(JmOptionPlugin): + plugin_key = 'send_qq_email' + + def invoke(self, + msg_from, + msg_to, + password, + title, + content, + album=None, + downloader=None, + ) -> None: + if not (msg_from and msg_to and password): + self.debug('发送邮件的相关参数为空，不处理') + return + from common import EmailConfig + econfig = EmailConfig(msg_from, msg_to, password) + epostman = econfig.create_email_postman() + epostman.send(content, title) + self.debug('Email sent successfully') diff --git a/src/jmcomic/jm_toolkit.py b/src/jmcomic/jm_toolkit.py index aeb520a8..a6e6321a 100644 --- a/src/jmcomic/jm_toolkit.py +++ b/src/jmcomic/jm_toolkit.py @@ -203,14 +203,17 @@ def match_os_env(cls, match: Match) -> str: name = match[1] value = os.getenv(name, None) assert value is not None, f"未配置环境变量: {name}" - return os.path.abspath(value) + return value dsl_replacer = DSLReplacer() @classmethod def parse_to_abspath(cls, dsl_text: str) -> str: - path = cls.dsl_replacer.parse_dsl_text(dsl_text) - return os.path.abspath(path) + return os.path.abspath(cls.parse_dsl_text(dsl_text)) + + @classmethod + def parse_dsl_text(cls, dsl_text: str) -> str: + return cls.dsl_replacer.parse_dsl_text(dsl_text) # 支持dsl: #{???} -> os.getenv(???) diff --git a/usage/workflow_download.py b/usage/workflow_download.py index 044aec1f..4baa98fd 100644 --- a/usage/workflow_download.py +++ b/usage/workflow_download.py @@ -35,8 +35,9 @@ def main(): helper.album_id_list = list(album_id_set) helper.photo_id_list = list(photo_id_set) - helper.run(get_option()) - + option = get_option() + helper.run(option) + option.call_all_plugin('after_download') def get_option(): # 读取 option 配置文件 @@ -48,9 +49,6 @@ def get_option(): # 把请求错误的html下载到文件，方便GitHub Actions下载查看日志 log_before_raise() - # 登录，如果有配置的话 - login_if_configured(option) - return option @@ -70,20 +68,6 @@ def cover_option_config(option: JmOption): option.download.image.suffix = fix_suffix(suffix) -def login_if_configured(option): - # 检查环境变量中是否有禁漫的用户名和密码，如果有则登录 - # 禁漫的大部分本子，下载是不需要登录的，少部分敏感题材需要登录 - # 如果你希望以登录状态下载本子，你需要自己配置一下GitHub Actions的 `secrets` - # 配置的方式很简单，网页上点一点就可以了 - # 具体做法请去看官方教程：https://docs.github.com/en/actions/security-guides/encrypted-secrets - # 萌新注意！！！如果你想 `开源` 你的禁漫帐号，你也可以直接把账号密码写到下面的代码😅 - username = get_env('JM_USERNAME', None) - password = get_env('JM_PASSWORD', None) - if username is not None and password is not None: - # 调用login插件 - JmLoginPlugin(option).invoke(username, password) - - def log_before_raise(): jm_download_dir = get_env('JM_DOWNLOAD_DIR', workspace()) mkdir_if_not_exists(jm_download_dir)