From 480836d5727b8c74c812f11f88105434bb81bef5 Mon Sep 17 00:00:00 2001
From: hect0x7 <93357912+hect0x7@users.noreply.github.com>
Date: Thu, 8 Oct 2026 20:48:43 +0800
Subject: [PATCH 1/7] =?UTF-8?q?v2.7.9:=20=E4=BF=AE=E5=A4=8D=E5=B9=B6?=
=?UTF-8?q?=E5=8F=91=E4=B8=8B=E8=BD=BD=E8=BF=9B=E5=BA=A6=E5=B1=95=E7=A4=BA?=
=?UTF-8?q?=E5=B9=B6=E6=96=B0=E5=A2=9E=E7=A6=BB=E7=BA=BF=E9=A2=84=E8=A7=88?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
---
CHANGELOG.md | 16 ++
.../sources/tutorial/15_download_progress.md | 2 +
src/jmcomic/__init__.py | 2 +-
src/jmcomic/cli.py | 17 ++
src/jmcomic/jm_plugin.py | 197 +++++++++++++++---
tests/test_jmcomic/test_jm_cli.py | 18 ++
.../test_jmcomic/test_jm_download_progress.py | 159 ++++++++++++++
7 files changed, 377 insertions(+), 34 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 1b480db24..24add6c74 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -4,6 +4,22 @@
条目分类参考 [Keep a Changelog](https://keepachangelog.com/zh-CN/1.0.0/)，
版本号遵循 [语义化版本](https://semver.org/lang/zh-CN/)。
+
+## [2.7.9] - 2026-10-08
+
+### Summary
+
+修复并发下载时多个进度界面和日志区相互干扰的问题，并提供离线进度预览命令。
+
+### Fixed
+
+- 同步和异步下载器共用一个动态进度界面与日志区，已结束任务保留到最后一个活动下载器退出，再统一关闭界面。
+- 界面启动或渲染失败时清理注册状态；界面退出失败时仍执行下载器资源释放，避免影响后续下载。
+
+### Added
+
+- `jmcomic --dry-run` 使用随机模拟任务预览多任务进度和重试日志，不请求网络、不下载图片、不写日志文件，也不加载 `--option` 配置；不能与 `--no-progress` 同用。
+
## [2.7.8] - 2026-10-07
### Summary
diff --git a/assets/docs/sources/tutorial/15_download_progress.md b/assets/docs/sources/tutorial/15_download_progress.md
index 21c85d553..cab251bba 100644
--- a/assets/docs/sources/tutorial/15_download_progress.md
+++ b/assets/docs/sources/tutorial/15_download_progress.md
@@ -37,6 +37,8 @@
> - 不支持：启动时显示一次说明，下载期间不刷新，结束后输出一次本子、章节和图片数量汇总。
>
> 如果你使用 PyCharm，请在底部的 **Terminal** 中运行 `python script.py`，即可查看动态进度。
+>
+> 你也可以使用 `--dry-run` 参数来模拟运行，体验效果，不会真的触发下载：`jmcomic 123 --dry-run`
| 运行位置 | 下载期间 | 下载结束后 |
diff --git a/src/jmcomic/__init__.py b/src/jmcomic/__init__.py
index 215157598..8c969b8ec 100644
--- a/src/jmcomic/__init__.py
+++ b/src/jmcomic/__init__.py
@@ -2,7 +2,7 @@
# 被依赖方 <--- 使用方
# config <--- entity <--- toolkit <--- client <--- option <--- downloader
-__version__ = '2.7.8'
+__version__ = '2.7.9'
from .jm_task_context import *
from .api import *
diff --git a/src/jmcomic/cli.py b/src/jmcomic/cli.py
index ad1871db8..f54b6431c 100644
--- a/src/jmcomic/cli.py
+++ b/src/jmcomic/cli.py
@@ -31,6 +31,7 @@ class JmcomicUI:
def __init__(self) -> None:
self.option_path: Optional[str] = None
self.progress_enabled = True
+ self.dry_run = False
self.raw_id_list: List[str] = []
self.album_id_list: List[str] = []
self.photo_id_list: List[str] = []
@@ -59,7 +60,16 @@ def parse_arg(self):
help='do not automatically enable the download progress plugin',
)
+ parser.add_argument(
+ '--dry-run',
+ action='store_true',
+ help='preview mock downloads without network requests or file writes; ignores --option',
+ )
+
args = parser.parse_args()
+ if args.dry_run and not args.progress_enabled:
+ parser.error('--dry-run 用于预览进度条，不能与 --no-progress 同时使用')
+ self.dry_run = args.dry_run
self.progress_enabled = args.progress_enabled
option = args.option
if len(option) == 0 or option == "''":
@@ -91,6 +101,13 @@ def parse(text):
def main(self):
self.parse_arg()
+ if self.dry_run:
+ if importlib.util.find_spec('rich') is None:
+ raise SystemExit('预览进度条需要 rich，请先执行：pip install rich')
+ from .jm_plugin import DownloadProgressPlugin
+ DownloadProgressPlugin.preview(self.album_id_list, self.photo_id_list)
+ return
+
from .api import create_option, JmOption
from .jm_task_context import jm_task_context
with jm_task_context(cli_no_progress=not self.progress_enabled):
diff --git a/src/jmcomic/jm_plugin.py b/src/jmcomic/jm_plugin.py
index 859061e2d..62ea2c082 100644
--- a/src/jmcomic/jm_plugin.py
+++ b/src/jmcomic/jm_plugin.py
@@ -752,6 +752,8 @@ class ProgressDownloader(JmDownloader):
progress_console = None
progress_log_lines = deque(maxlen=6)
active_progresses = set()
+ displayed_progresses = []
+ progress_live = None
progress_ui_lock = RLock()
@staticmethod
@@ -761,10 +763,11 @@ def display_id(entity_id):
@classmethod
def get_progress_console(cls):
- if cls.progress_console is None:
- from rich.console import Console
- cls.progress_console = Console()
- return cls.progress_console
+ with cls.progress_ui_lock:
+ if cls.progress_console is None:
+ from rich.console import Console
+ cls.progress_console = Console()
+ return cls.progress_console
@classmethod
def reset_progress_logs(cls):
@@ -780,24 +783,61 @@ def configure_progress_log_lines(cls, max_lines):
def append_progress_log(cls, message):
with cls.progress_ui_lock:
cls.progress_log_lines.append(message)
- progresses = tuple(cls.active_progresses)
+ cls.refresh_progress_ui()
- for progress in progresses:
- try:
- progress.refresh()
- except Exception:
- pass
+ @classmethod
+ def refresh_progress_ui(cls):
+ from rich.console import Group
+
+ with cls.progress_ui_lock:
+ if cls.progress_live is not None:
+ cls.progress_live.update(
+ Group(cls.build_log_panel(), *cls.displayed_progresses),
+ refresh=True,
+ )
@classmethod
def register_progress(cls, progress):
if progress.console.is_interactive:
+ from rich.live import Live
+
with cls.progress_ui_lock:
+ if progress in cls.active_progresses:
+ return
cls.active_progresses.add(progress)
+ cls.displayed_progresses.append(progress)
+ live_created = cls.progress_live is None
+ try:
+ if live_created:
+ # 所有下载器只提供进度内容，终端刷新由唯一的 Live 管理。
+ cls.progress_live = Live(console=progress.console, auto_refresh=False)
+ cls.progress_live.start()
+ cls.refresh_progress_ui()
+ except BaseException:
+ # 启动或首帧渲染失败时撤销注册，让后续下载能重新建立界面。
+ cls.active_progresses.discard(progress)
+ cls.displayed_progresses.remove(progress)
+ if live_created and cls.progress_live is not None:
+ try:
+ cls.progress_live.stop()
+ finally:
+ cls.progress_live = None
+ raise
@classmethod
def unregister_progress(cls, progress):
with cls.progress_ui_lock:
cls.active_progresses.discard(progress)
+ if cls.progress_live is not None and not cls.active_progresses:
+ try:
+ # 已结束任务保留到最后一个下载器退出，再统一留下最终画面。
+ cls.refresh_progress_ui()
+ finally:
+ try:
+ cls.progress_live.stop()
+ finally:
+ cls.progress_live = None
+ cls.displayed_progresses.clear()
@classmethod
def build_log_panel(cls):
@@ -833,12 +873,7 @@ def new_rich_progress(cls, console):
TimeElapsedColumn,
)
- class ProgressWithLogs(Progress):
- def get_renderables(self):
- yield cls.build_log_panel()
- yield from super().get_renderables()
-
- return ProgressWithLogs(
+ return Progress(
SpinnerColumn(style='bright_cyan'),
TextColumn('{task.description}'),
BarColumn(
@@ -857,7 +892,7 @@ def get_renderables(self):
def refresh_progress(self):
if self.progress.console.is_interactive:
- self.progress.refresh()
+ ProgressDownloader.refresh_progress_ui()
def print_non_interactive_summary(self, photo_id=None):
if self.progress.console.is_interactive:
@@ -900,10 +935,9 @@ def start_progress(self, album_total=None, album_id=None):
self.chapter_done = {}
self.chapter_total = {}
self.chapter_tasks = {}
- console = self.get_progress_console()
+ console = ProgressDownloader.get_progress_console()
self.progress = self.new_rich_progress(console)
- self.progress.start()
- self.register_progress(self.progress)
+ ProgressDownloader.register_progress(self.progress)
self.album_task = None
if album_total is not None:
self.album_task = self.progress.add_task(
@@ -970,14 +1004,16 @@ def stop_progress(self):
progress = getattr(self, 'progress', None)
if progress is None:
return
- self.unregister_progress(progress)
- if progress.console.is_interactive:
- progress.stop()
- self.progress = None
+ try:
+ ProgressDownloader.unregister_progress(progress)
+ finally:
+ self.progress = None
def __exit__(self, exc_type, exc_val, exc_tb):
- self.stop_progress()
- return super().__exit__(exc_type, exc_val, exc_tb)
+ try:
+ self.stop_progress()
+ finally:
+ super().__exit__(exc_type, exc_val, exc_tb)
# noinspection attribute-outside-init
@@ -995,7 +1031,6 @@ def start_progress(self, album_total=None, album_id=None):
self.chapter_tasks = {}
console = ProgressDownloader.get_progress_console()
self.progress = ProgressDownloader.new_rich_progress(console)
- self.progress.start()
ProgressDownloader.register_progress(self.progress)
self.album_task = None
if album_total is not None:
@@ -1067,14 +1102,16 @@ def stop_progress(self):
progress = getattr(self, 'progress', None)
if progress is None:
return
- ProgressDownloader.unregister_progress(progress)
- if progress.console.is_interactive:
- progress.stop()
- self.progress = None
+ try:
+ ProgressDownloader.unregister_progress(progress)
+ finally:
+ self.progress = None
async def __aexit__(self, exc_type, exc_val, exc_tb):
- self.stop_progress()
- return await super().__aexit__(exc_type, exc_val, exc_tb)
+ try:
+ self.stop_progress()
+ finally:
+ await super().__aexit__(exc_type, exc_val, exc_tb)
class DownloadProgressPlugin(JmOptionPlugin):
@@ -1082,6 +1119,100 @@ class DownloadProgressPlugin(JmOptionPlugin):
plugin_dependencies = ('rich',)
log_file = 'jmcomic-download.log'
+ @staticmethod
+ def preview(album_ids, photo_ids):
+ """使用模拟任务预览真实进度界面，不创建 Client、Option 或文件。"""
+ from datetime import datetime
+ from random import Random
+ from time import sleep
+
+ random = Random()
+
+ def log(topic, message, worker):
+ timestamp = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
+ ProgressDownloader.append_progress_log(
+ f'{timestamp} [mock-worker-{worker}] [{topic}] {message}'
+ )
+
+ if not album_ids and not photo_ids:
+ album_ids = ['100001', '100002']
+ console = ProgressDownloader.get_progress_console()
+ console.print('模拟下载：不请求网络、不写入图片或日志文件，也不加载 --option 配置。')
+ previews = []
+ ProgressDownloader.reset_progress_logs()
+ try:
+ for kind, ids in (('本子', album_ids), ('章节', photo_ids)):
+ for entity_id in ids:
+ progress = ProgressDownloader.new_rich_progress(console)
+ name = f'{kind}-{ProgressDownloader.display_id(entity_id)}'
+ album_task = progress.add_task(name, total=2) if kind == '本子' else None
+ chapters = []
+ for index in range(2 if album_task is not None else 1):
+ chapter_name = f'章节-JM{entity_id}{index + 1:02}' if album_task is not None else name
+ total = random.randint(12, 32)
+ task_id = progress.add_task(
+ f' {chapter_name}' if album_task is not None else chapter_name,
+ total=total, start=False,
+ )
+ chapters.append({
+ 'task_id': task_id, 'name': chapter_name,
+ 'next_tick': random.randint(0, 8),
+ 'interval': random.randint(1, 3),
+ 'started': False, 'retried': False,
+ })
+ worker = len(previews) + 1
+ previews.append((progress, album_task, chapters, name, worker))
+ ProgressDownloader.register_progress(progress)
+ if album_task is not None:
+ log('album.before', f'本子获取成功: [{entity_id}], 章节数: [2], 标题: [模拟本子]', worker)
+
+ tick = 0
+ while any(not task.finished for progress, _, _, _, _ in previews for task in progress.tasks):
+ if console.is_interactive:
+ sleep(random.uniform(0.08, 0.16))
+ for progress, album_task, chapters, name, worker in previews:
+ tasks = {task.id: task for task in progress.tasks}
+ for chapter in chapters:
+ task = tasks[chapter['task_id']]
+ if task.finished or tick < chapter['next_tick']:
+ continue
+ chapter_name = chapter['name']
+ if not chapter['started']:
+ progress.start_task(task.id)
+ chapter['started'] = True
+ log('photo.before', f'开始下载章节: {chapter_name}, 图片数为[{int(task.total)}]', worker)
+ elif not chapter['retried'] and random.random() < 0.08:
+ chapter['retried'] = True
+ chapter['next_tick'] = tick + random.randint(3, 6)
+ log('image.retry', f'{chapter_name} 图片 {int(task.completed) + 1:05}.jpg 请求超时，准备重试', worker)
+ continue
+ advance = min(random.randint(1, 3), int(task.total - task.completed))
+ for page in range(int(task.completed) + 1, int(task.completed) + advance + 1):
+ progress.advance(task.id)
+ log('image.after', f'图片下载完成: {chapter_name}/{page:05}.jpg [{page}/{int(task.total)}]', worker)
+ chapter['next_tick'] = tick + chapter['interval'] + random.randint(0, 1)
+ if task.finished:
+ progress.update(task.id, description=f'✓ {task.description}')
+ log('photo.after', f'章节下载完成: [{chapter_name}], 图片 {int(task.total)}/{int(task.total)}', worker)
+ if album_task is not None:
+ was_finished = tasks[album_task].finished
+ done = sum(tasks[chapter['task_id']].finished for chapter in chapters)
+ progress.update(album_task, completed=done)
+ if done == len(chapters) and not was_finished:
+ progress.update(album_task, description=f'✓ {name}')
+ log('album.after', f'本子下载完成: [{name}], 章节 {done}/{len(chapters)}', worker)
+ ProgressDownloader.refresh_progress_ui()
+ tick += 1
+
+ if not console.is_interactive:
+ for progress, album_task, chapters, name, worker in previews:
+ console.print(f'模拟完成：{name}')
+ finally:
+ for progress, album_task, chapters, name, worker in previews:
+ ProgressDownloader.unregister_progress(progress)
+ ProgressDownloader.reset_progress_logs()
+ console.print('模拟结束，未下载任何文件。')
+
@staticmethod
def cli_no_progress_notice():
if not get_jm_task_context().get('cli_no_progress'):
diff --git a/tests/test_jmcomic/test_jm_cli.py b/tests/test_jmcomic/test_jm_cli.py
index ebbb44ef0..ac692244c 100644
--- a/tests/test_jmcomic/test_jm_cli.py
+++ b/tests/test_jmcomic/test_jm_cli.py
@@ -12,6 +12,24 @@ class Test_Cli(JmTestConfigurable):
album_id = '123'
+ def test_dry_run_skips_option_and_real_downloads(self):
+ ui = JmcomicUI()
+ with patch('sys.argv', ['jmcomic', '100001', 'p200001', '--dry-run', '--option', 'unused.yml']), \
+ patch('jmcomic.cli.importlib.util.find_spec', return_value=object()), \
+ patch('jmcomic.api.create_option', side_effect=AssertionError('不能读取配置')), \
+ patch('jmcomic.api.JmOption.default', side_effect=AssertionError('不能创建 Option')), \
+ patch.object(ui, 'run', side_effect=AssertionError('不能执行真实下载')), \
+ patch('jmcomic.jm_plugin.DownloadProgressPlugin.preview') as preview:
+ ui.main()
+ preview.assert_called_once_with(['100001'], ['200001'])
+
+ def test_dry_run_rejects_no_progress(self):
+ with patch('sys.argv', ['jmcomic', '--dry-run', '--no-progress']), \
+ patch('sys.stderr', StringIO()):
+ with self.assertRaises(SystemExit) as caught:
+ JmcomicUI().parse_arg()
+ self.assertEqual(2, caught.exception.code)
+
# ========== jmcomic 命令测试 ==========
def test_jmcomic_progress_enabled_by_default(self):
diff --git a/tests/test_jmcomic/test_jm_download_progress.py b/tests/test_jmcomic/test_jm_download_progress.py
index 57b151e4d..fe2894f5e 100644
--- a/tests/test_jmcomic/test_jm_download_progress.py
+++ b/tests/test_jmcomic/test_jm_download_progress.py
@@ -93,6 +93,165 @@ async def check_photo(self, photo):
class Test_DownloadProgress(unittest.TestCase):
+ def setUp(self):
+ # 渲染测试固定为支持 ANSI 的终端，不依赖测试宿主的 TERM 设置。
+ terminal = patch.dict(os.environ, {'TERM': 'xterm-256color'})
+ terminal.start()
+ self.addCleanup(terminal.stop)
+
+ @unittest.skipUnless(RICH_INSTALLED, '需要安装 rich 才能测试彩色进度插件')
+ def test_subclass_and_async_downloader_share_console(self):
+ class CustomProgressDownloader(ProgressDownloader):
+ pass
+
+ sync = object.__new__(CustomProgressDownloader)
+ async_downloader = object.__new__(AsyncProgressDownloader)
+ with patch.object(ProgressDownloader, 'progress_console', None), patch('sys.stdout', StringIO()):
+ try:
+ sync.start_progress(1, '101')
+ async_downloader.start_progress(1, '202')
+ self.assertIs(sync.progress.console, async_downloader.progress.console)
+ self.assertNotIn('progress_console', CustomProgressDownloader.__dict__)
+ finally:
+ sync.stop_progress()
+ async_downloader.stop_progress()
+
+ @unittest.skipUnless(RICH_INSTALLED, '需要安装 rich 才能测试彩色进度插件')
+ def test_concurrent_sync_async_progress_keeps_completed_tasks_until_last_exit(self):
+ from rich.console import Console
+
+ console = Console(file=StringIO(), force_terminal=True, force_interactive=True, width=100)
+ sync = object.__new__(ProgressDownloader)
+ async_downloader = object.__new__(AsyncProgressDownloader)
+ with patch.object(ProgressDownloader, 'progress_console', console):
+ try:
+ sync.start_progress(1, '101')
+ live = ProgressDownloader.progress_live
+ async_downloader.start_progress(1, '202')
+ self.assertIs(live, ProgressDownloader.progress_live)
+ completed_progress = sync.progress
+ ProgressDownloader.register_progress(completed_progress)
+ self.assertEqual(2, len(ProgressDownloader.displayed_progresses))
+ sync.stop_progress()
+ self.assertIs(live, ProgressDownloader.progress_live)
+ self.assertIn(completed_progress, ProgressDownloader.displayed_progresses)
+ async_downloader.stop_progress()
+ self.assertIsNone(ProgressDownloader.progress_live)
+ self.assertFalse(ProgressDownloader.active_progresses)
+ self.assertFalse(ProgressDownloader.displayed_progresses)
+ sync.start_progress(1, '303')
+ self.assertIsNot(live, ProgressDownloader.progress_live)
+ self.assertEqual(1, len(ProgressDownloader.displayed_progresses))
+ finally:
+ sync.stop_progress()
+ async_downloader.stop_progress()
+
+ @unittest.skipUnless(RICH_INSTALLED, '需要安装 rich 才能测试彩色进度插件')
+ def test_failed_live_start_does_not_poison_next_download(self):
+ from rich.console import Console
+
+ console = Console(file=StringIO(), force_terminal=True, force_interactive=True)
+ progress = ProgressDownloader.new_rich_progress(console)
+ with patch('rich.live.Live.start', side_effect=RuntimeError('render failed')):
+ with self.assertRaisesRegex(RuntimeError, 'render failed'):
+ ProgressDownloader.register_progress(progress)
+ self.assertIsNone(ProgressDownloader.progress_live)
+ self.assertFalse(ProgressDownloader.active_progresses)
+ self.assertFalse(ProgressDownloader.displayed_progresses)
+ try:
+ ProgressDownloader.register_progress(progress)
+ self.assertIsNotNone(ProgressDownloader.progress_live)
+ finally:
+ ProgressDownloader.unregister_progress(progress)
+
+ @unittest.skipUnless(RICH_INSTALLED, '需要安装 rich 才能测试彩色进度插件')
+ def test_failed_final_refresh_releases_ui_and_downloader(self):
+ from rich.console import Console
+
+ console = Console(file=StringIO(), force_terminal=True, force_interactive=True)
+ downloader = object.__new__(ProgressDownloader)
+ with patch.object(ProgressDownloader, 'progress_console', console):
+ downloader.start_progress(1, '101')
+ with patch.object(ProgressDownloader, 'refresh_progress_ui', side_effect=RuntimeError('render failed')):
+ with self.assertRaisesRegex(RuntimeError, 'render failed'):
+ downloader.stop_progress()
+ self.assertIsNone(downloader.progress)
+ self.assertIsNone(ProgressDownloader.progress_live)
+ self.assertFalse(ProgressDownloader.active_progresses)
+ self.assertFalse(ProgressDownloader.displayed_progresses)
+
+ @unittest.skipUnless(RICH_INSTALLED, '需要安装 rich 才能测试彩色进度插件')
+ def test_preview_interrupt_releases_ui(self):
+ from rich.console import Console
+
+ console = Console(file=StringIO(), force_terminal=True, force_interactive=True)
+ with patch.object(ProgressDownloader, 'progress_console', console), \
+ patch('time.sleep', side_effect=KeyboardInterrupt):
+ with self.assertRaises(KeyboardInterrupt):
+ DownloadProgressPlugin.preview([], [])
+ self.assertIsNone(ProgressDownloader.progress_live)
+ self.assertFalse(ProgressDownloader.active_progresses)
+ self.assertFalse(ProgressDownloader.displayed_progresses)
+ self.assertFalse(ProgressDownloader.progress_log_lines)
+
+ def test_async_exit_closes_resources_even_if_ui_stop_fails(self):
+ from unittest.mock import AsyncMock
+
+ downloader = object.__new__(AsyncProgressDownloader)
+
+ async def run():
+ with patch.object(downloader, 'stop_progress', side_effect=RuntimeError('render failed')), \
+ patch.object(JmAsyncDownloader, '__aexit__', new_callable=AsyncMock) as close:
+ with self.assertRaisesRegex(RuntimeError, 'render failed'):
+ await downloader.__aexit__(None, None, None)
+ close.assert_awaited_once_with(None, None, None)
+
+ asyncio.run(run())
+
+ @unittest.skipUnless(RICH_INSTALLED, '需要安装 rich 才能测试彩色进度插件')
+ def test_preview_shares_one_live_and_releases_tasks_without_files(self):
+ from random import Random
+ from rich.console import Console
+
+ output = StringIO()
+ original_console = ProgressDownloader.progress_console
+ ProgressDownloader.progress_console = Console(
+ file=output, force_terminal=True, force_interactive=True, width=100,
+ )
+ observed = []
+ observed_logs = []
+ original_refresh = ProgressDownloader.refresh_progress_ui
+
+ def check_frame():
+ original_refresh()
+ observed_logs.extend(ProgressDownloader.progress_log_lines)
+ live = ProgressDownloader.progress_live
+ if live is not None:
+ observed.append(live)
+ rendered = ProgressDownloader.progress_console.render_lines(live.renderable)
+ text = '\n'.join(''.join(segment.text for segment in line) for line in rendered)
+ self.assertEqual(1, text.count('JMComic Logs'))
+
+ try:
+ with patch('time.sleep'), \
+ patch('random.Random', return_value=Random(1)), \
+ patch('logging.FileHandler', side_effect=AssertionError('不能创建日志文件')), \
+ patch.object(ProgressDownloader, 'refresh_progress_ui', side_effect=check_frame):
+ DownloadProgressPlugin.preview([], [])
+ self.assertTrue(observed)
+ self.assertTrue(all(live is observed[0] for live in observed))
+ self.assertIsNone(ProgressDownloader.progress_live)
+ self.assertFalse(ProgressDownloader.active_progresses)
+ self.assertFalse(ProgressDownloader.displayed_progresses)
+ self.assertIn('✓ 本子-JM100001', output.getvalue())
+ self.assertIn('✓ 本子-JM100002', output.getvalue())
+ logs = '\n'.join(observed_logs)
+ for topic in ('album.before', 'photo.before', 'image.after', 'image.retry', 'photo.after', 'album.after'):
+ self.assertIn(f'[{topic}]', logs)
+ self.assertIn('[mock-worker-', logs)
+ finally:
+ ProgressDownloader.progress_console = original_console
+
def test_progress_display_id_adds_prefix_only_once(self):
self.assertEqual('JM123456', ProgressDownloader.display_id('123456'))
self.assertEqual('JM123456', ProgressDownloader.display_id('JM123456'))
From 56190461570c3d17c86b36a7b724bfe024c8ec2b Mon Sep 17 00:00:00 2001
From: hect0x7 <93357912+hect0x7@users.noreply.github.com>
Date: Tue, 13 Oct 2026 00:11:42 +0800
Subject: [PATCH 2/7] =?UTF-8?q?feat:=20=E6=94=AF=E6=8C=81=E5=8D=8F?=
=?UTF-8?q?=E4=BD=9C=E5=BC=8F=E5=8F=96=E6=B6=88=E4=B8=8B=E8=BD=BD=E4=B8=8E?=
=?UTF-8?q?=E8=B7=A8=E5=B1=82=E5=85=B1=E4=BA=AB=20Runtime=20=E5=B9=B6?=
=?UTF-8?q?=E5=8F=91=E8=B0=83=E5=BA=A6?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
---
.github/workflows/benchmark.yml | 6 +-
.github/workflows/close_specific_pr.yml | 4 +-
.github/workflows/export_favorites.yml | 8 +-
.github/workflows/release.yml | 8 +-
.github/workflows/release_auto.yml | 8 +-
.github/workflows/test_api.yml | 4 +-
.github/workflows/test_html.yml | 4 +-
CHANGELOG.md | 16 +
assets/docs/mkdocs.yml | 1 +
assets/docs/sources/api/download.md | 34 +
.../docs/sources/tutorial/0_common_usage.md | 69 +-
.../docs/sources/tutorial/14_async_usage.md | 76 ++-
.../sources/tutorial/16_shared_executors.md | 303 +++++++++
pyproject.toml | 2 +-
src/jmcomic/__init__.py | 4 +-
src/jmcomic/api.py | 470 +++++++++----
src/jmcomic/cli.py | 5 +-
src/jmcomic/jm_async_client.py | 1 -
src/jmcomic/jm_async_downloader.py | 145 +++-
src/jmcomic/jm_client_impl.py | 6 +
src/jmcomic/jm_client_interface.py | 8 +-
src/jmcomic/jm_config.py | 7 +-
src/jmcomic/jm_downloader.py | 185 ++++--
src/jmcomic/jm_entity.py | 1 -
src/jmcomic/jm_exception.py | 45 +-
src/jmcomic/jm_feature.py | 4 +-
src/jmcomic/jm_option.py | 3 +-
src/jmcomic/jm_plugin.py | 9 +-
src/jmcomic/jm_runtime.py | 308 +++++++++
src/jmcomic/jm_task_context.py | 189 +++++-
src/jmcomic/jm_toolkit.py | 1 -
tests/test_jmcomic/test_jm_api.py | 3 +
tests/test_jmcomic/test_jm_async_custom.py | 30 +-
tests/test_jmcomic/test_jm_cancellation.py | 609 +++++++++++++++++
tests/test_jmcomic/test_jm_cli.py | 46 +-
tests/test_jmcomic/test_jm_client.py | 10 +-
.../test_jmcomic/test_jm_download_manifest.py | 208 ++++--
.../test_jmcomic/test_jm_download_progress.py | 39 +-
tests/test_jmcomic/test_jm_exception.py | 44 ++
tests/test_jmcomic/test_jm_release.py | 47 +-
tests/test_jmcomic/test_jm_runtime.py | 281 ++++++++
tests/test_jmcomic/test_jm_task_context.py | 627 ++++++++++++++++--
42 files changed, 3422 insertions(+), 456 deletions(-)
create mode 100644 assets/docs/sources/tutorial/16_shared_executors.md
create mode 100644 src/jmcomic/jm_runtime.py
create mode 100644 tests/test_jmcomic/test_jm_cancellation.py
create mode 100644 tests/test_jmcomic/test_jm_runtime.py
diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml
index e534db734..dd928be5e 100644
--- a/.github/workflows/benchmark.yml
+++ b/.github/workflows/benchmark.yml
@@ -19,10 +19,10 @@ jobs:
timeout-minutes: 15
steps:
- - uses: actions/checkout@v4
+ - uses: actions/checkout@v7
- name: Set up Python
- uses: actions/setup-python@v5
+ uses: actions/setup-python@v7
with:
python-version: '3.14'
@@ -49,7 +49,7 @@ jobs:
- name: Upload Benchmark Report
if: always()
- uses: actions/upload-artifact@v4
+ uses: actions/upload-artifact@v7
with:
name: benchmark-report
path: PERFORMANCE_REPORT.md
diff --git a/.github/workflows/close_specific_pr.yml b/.github/workflows/close_specific_pr.yml
index 8ed49d3eb..ce890e337 100644
--- a/.github/workflows/close_specific_pr.yml
+++ b/.github/workflows/close_specific_pr.yml
@@ -20,7 +20,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Checkout code
- uses: actions/checkout@v4
+ uses: actions/checkout@v6
- name: Close PR
run: |
@@ -36,4 +36,4 @@ jobs:
gh pr comment ${{ github.event.pull_request.number }} --repo ${{ github.repository }} --body '${{ env.comment }}'
gh pr close ${{ github.event.pull_request.number }} --repo ${{ github.repository }}
env:
- GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
\ No newline at end of file
+ GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
diff --git a/.github/workflows/export_favorites.yml b/.github/workflows/export_favorites.yml
index a00cdf64c..4375e3bc7 100644
--- a/.github/workflows/export_favorites.yml
+++ b/.github/workflows/export_favorites.yml
@@ -48,9 +48,11 @@ jobs:
ZIP_FP: /home/runner/work/jmcomic/download/export.7z
steps:
- - uses: actions/checkout@v4
+ - uses: actions/checkout@v7
+ with:
+ persist-credentials: false
- name: Set up Python 3.11
- uses: actions/setup-python@v5
+ uses: actions/setup-python@v7
with:
python-version: "3.11"
@@ -71,7 +73,7 @@ jobs:
python workflow_export_favorites.py
- name: 上传结果
- uses: actions/upload-artifact@v4
+ uses: actions/upload-artifact@v7
with:
name: '导出的收藏夹'
path: ${{ env.ZIP_FP }}
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 8f17f7722..9458b5856 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -13,10 +13,12 @@ jobs:
id-token: write
contents: write
steps:
- - uses: actions/checkout@v4
+ - uses: actions/checkout@v7
+ with:
+ persist-credentials: false
- name: Set up Python 3.11
- uses: actions/setup-python@v5
+ uses: actions/setup-python@v7
with:
python-version: "3.11"
@@ -31,7 +33,7 @@ jobs:
python -m build
- name: Create Release
- uses: softprops/action-gh-release@v2
+ uses: softprops/action-gh-release@v3
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
diff --git a/.github/workflows/release_auto.yml b/.github/workflows/release_auto.yml
index 0ef2d6cb1..c3829d119 100644
--- a/.github/workflows/release_auto.yml
+++ b/.github/workflows/release_auto.yml
@@ -14,10 +14,12 @@ jobs:
contents: write
if: startsWith(github.event.head_commit.message, 'v')
steps:
- - uses: actions/checkout@v4
+ - uses: actions/checkout@v7
+ with:
+ persist-credentials: false
- name: Set up Python 3.11
- uses: actions/setup-python@v5
+ uses: actions/setup-python@v7
with:
python-version: "3.11"
@@ -33,7 +35,7 @@ jobs:
python -m build
- name: Create Release
- uses: softprops/action-gh-release@v2
+ uses: softprops/action-gh-release@v3
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
diff --git a/.github/workflows/test_api.yml b/.github/workflows/test_api.yml
index e0e5d7558..35245d21f 100644
--- a/.github/workflows/test_api.yml
+++ b/.github/workflows/test_api.yml
@@ -27,10 +27,10 @@ jobs:
steps:
- name: Checkout repository
- uses: actions/checkout@v4
+ uses: actions/checkout@v7
- name: Set up Python ${{ matrix.python-version }}
- uses: actions/setup-python@v5
+ uses: actions/setup-python@v7
with:
python-version: ${{ matrix.python-version }}
diff --git a/.github/workflows/test_html.yml b/.github/workflows/test_html.yml
index 8f44d7c9b..f8a45b50c 100644
--- a/.github/workflows/test_html.yml
+++ b/.github/workflows/test_html.yml
@@ -27,10 +27,10 @@ jobs:
steps:
- name: Checkout repository
- uses: actions/checkout@v4
+ uses: actions/checkout@v7
- name: Set up Python ${{ matrix.python-version }}
- uses: actions/setup-python@v5
+ uses: actions/setup-python@v7
with:
python-version: ${{ matrix.python-version }}
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 24add6c74..03df2a95d 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -5,6 +5,22 @@
条目分类参考 [Keep a Changelog](https://keepachangelog.com/zh-CN/1.0.0/)，
版本号遵循 [语义化版本](https://semver.org/lang/zh-CN/)。
+## [2.7.10] - 2026-10-08
+
+### Summary
+
+本次更新支持协作式取消下载与共享 Runtime 调度。
+
+### Added
+- 新增 `JmTempRuntime`、`JmSyncRuntime` 与 `JmAsyncRuntime`；裸同步 Downloader 的局部调度使用临时单池 Runtime，顶层同步 API 或自定义同步调度可复用 `album/photo/image` 三层线程池，异步下载可复用 `decode` 线程池。
+- 新增 `DownloadControl` 和 `DownloadCancelledException`，支持通过任务上下文协作式取消同步与异步下载；`DownloadCancelledException` 支持通过 `completed_results` 与 `unfinished_ids` 读取批量取消时的已完成成果与未完成 ID。
+- `DownloadResult` 新增 `album` 和 `photo` 快捷实体访问属性。
+- 新增 `JTC` (Jm Task Context) 门面类，统一提供上下文、Runtime、Option 及取消控制器的快捷读取方法。
+
+### Changed
+- 下载调度统一使用标准库 Executor；顶层 API 和裸同步 Downloader 的临时调度显式关闭自己创建的 Runtime，`jm_task_context` 只传播字段；外部 Runtime 和 Executor 仍由调用方关闭。
+- 顶层下载会把 Runtime 和 Option 作为公开字段直接放入任务上下文；Runtime 不依赖 Context 或 Option，未配置的层级由实际调用点传入默认 worker 数。
+
## [2.7.9] - 2026-10-08
### Summary
diff --git a/assets/docs/mkdocs.yml b/assets/docs/mkdocs.yml
index 003684fcb..99c00bed9 100644
--- a/assets/docs/mkdocs.yml
+++ b/assets/docs/mkdocs.yml
@@ -59,6 +59,7 @@ nav:
- tutorial/13_export_and_feature.md
- tutorial/14_async_usage.md
- tutorial/15_download_progress.md
+ - tutorial/16_shared_executors.md
plugins:
- search
diff --git a/assets/docs/sources/api/download.md b/assets/docs/sources/api/download.md
index ce7d19654..f193c0cc4 100644
--- a/assets/docs/sources/api/download.md
+++ b/assets/docs/sources/api/download.md
@@ -17,3 +17,37 @@
options:
members:
- JmAsyncDownloader
+
+::: jmcomic.jm_downloader
+ options:
+ members:
+ - BaseDownloader
+ - JmDownloader
+
+::: jmcomic.jm_runtime
+ options:
+ members:
+ - JmRuntime
+ - JmTempRuntime
+ - JmSyncRuntime
+ - JmAsyncRuntime
+
+::: jmcomic.jm_exception
+ options:
+ members:
+ - DownloadCancelledException
+
+::: jmcomic.jm_task_context
+ options:
+ members:
+ - DownloadControl
+ - jm_task_context
+ - bind_jm_task_context
+
+::: jmcomic.jm_task_context.JTC
+ options:
+ members:
+ - get_runtime
+ - get_option
+ - get_control
+ - get_context
diff --git a/assets/docs/sources/tutorial/0_common_usage.md b/assets/docs/sources/tutorial/0_common_usage.md
index abe2818bf..216273987 100644
--- a/assets/docs/sources/tutorial/0_common_usage.md
+++ b/assets/docs/sources/tutorial/0_common_usage.md
@@ -534,11 +534,63 @@ cl = JmApiClient(
```
+## 取消下载
+
+> 如果你想中途取消下载，最简单粗暴的方式是直接杀死进程，比如 `ctrl+c`，关闭终端窗口 等
+>
+> 但是在 GUI、Web服务这类场景里，就不适合这么关闭。
+>
+> 于是，jmcomic 提供了一种优雅停止的写法，可以使用 `DownloadControl` 这个类。
+>
+> 这种写法更加可控：
+>
+> - **不影响主程序**：只停当前下载，GUI 界面或 Web 服务依然正常运作。
+> - **多任务统一取消**：多个本子一起下载时，可以被统一取消。
+> - **可携带取消理由**：调用 `cancel()` 传入的原因可以直接捕获，方便做提示和日志。
+
+写法就两步：
+
+1. 创建 DownloadControl 对象，传入任务上下文
+2. 调用 DownloadControl 类的 cancel() 方法
+
+```python
+from threading import Thread
+from time import sleep
+from jmcomic import DownloadCancelledException, DownloadControl, download_album, jm_task_context
+
+# 创建取消控制器
+my_control = DownloadControl()
+
+def run_download():
+ try:
+ # 使用 with jm_task_context 创建任务上下文，并传入 control 参数
+ with jm_task_context(control=my_control):
+ # 支持传入单个本子或多个本子列表；多本子时中途取消会统一停止
+ download_album(['123456', '789012', '345678'])
+ except DownloadCancelledException as e:
+ print(f'下载已取消: {e.reason}')
+ # 可直接从异常对象读取已完成成果与未完成的 ID
+ print(f'已完成本子: {[res.album.id for res in e.completed_results]}')
+ print(f'未完成本子: {e.unfinished_ids}')
+
+# 使用单独的下载线程执行下载
+# 主线程负责取消
+t = Thread(target=run_download)
+t.start()
+
+# 模拟一段时间后，需要取消下载
+sleep(2)
+# 可以传入取消原因，下载线程可通过上面的 e.reason 获取
+my_control.cancel("不想要了，取消掉吧")
+# 等待下载线程结束
+t.join()
+```
+
## 下载返回值
`download_album` 和 `download_photo` 下载完成后，单个 ID 返回 `DownloadResult`，多个 ID 返回 `BatchResult`。
-从 `result.detail` 可以取得下载的本子/章节的实体类：
+从 `DownloadResult` 的 `detail` 字段可以取得下载的本子/章节的实体类：
```python
from jmcomic import download_album, download_photo
@@ -638,24 +690,26 @@ print('长图导出文件:', png_filepath_list)
### 批量下载的返回值
-传入多个 ID 时，返回值是 `BatchResult`。每一项成功下载对应一个 `DownloadResult`，失败任务则记录在 `failed` 中：
+传入多个 ID 时，返回值是 `BatchResult`，通过这个对象可以取得成功和失败的下载结果：
完整示例：处理批量下载结果
```python
-from jmcomic import download_album
+from jmcomic import download_album, DownloadResult
# 同时下载多个本子
batch_result = download_album(['123', '456', '789'])
-# BatchResult 继承 set，成功结果没有输入顺序保证
+# BatchResult 继承 set，直接遍历是只遍历成功结果
for result in batch_result:
+ result: DownloadResult
+ # result 的用法同上
album = result.detail
- # 通过实体 ID 识别当前结果，不要用遍历位置对应输入列表
print(f'JM{album.id} 下载到: {album.save_path}')
-# failed 的键是下载失败的 ID，值是记录失败原因的异常对象
+# failed 负责存放失败的下载结果，类型是dict
+# key是下载失败的 ID，value是记录失败原因的异常对象
for album_id, error in batch_result.failed.items():
print(f'JM{album_id} 下载失败: {error}')
@@ -666,7 +720,8 @@ print('是否全部成功:', batch_result.all_succeeded)
-下载单个 ID 时，请求本子失败会直接抛出异常；如果只有部分章节或图片失败，会在任务结束后汇总抛出 `PartialDownloadFailedException`，此时不会返回 `DownloadResult`。批量下载则继续执行其他任务，并把失败项集中放进 `batch_result.failed`。
+---
+
### 速查表
diff --git a/assets/docs/sources/tutorial/14_async_usage.md b/assets/docs/sources/tutorial/14_async_usage.md
index 3482ef861..71f69ede6 100644
--- a/assets/docs/sources/tutorial/14_async_usage.md
+++ b/assets/docs/sources/tutorial/14_async_usage.md
@@ -260,9 +260,9 @@ client:
async_impl: async_api
```
-## 10. 查看下载耗时
+## 10. 下载返回值
-异步下载完成后，可以直接查看自己总共等了多久，也可以继续查看具体是哪个本子、章节或图片比较慢。所有 `duration` 的单位都是秒。
+用法与同步版本完全一致，返回的 `DownloadResult` 包含了本子实体、保存路径、耗时以及下载清单（`manifest`）等专属字段：
```python
import asyncio
@@ -273,25 +273,67 @@ async def main():
result = await jmcomic.download_album_async('438696')
album = result.detail
- # 从调用 download_album_async 到返回，总共等了多久
- print(f'总共等待: {result.duration:.3f} 秒')
+ # 1. 实体与保存路径
+ print(f'本子: JM{album.id} - {album.title}')
+ print(f'保存目录: {album.save_path}')
- # 如果下载比较慢，可以继续查看具体慢在哪里
- print(f'下载本子用了: {album.duration:.3f} 秒')
- for photo in album:
- print(f'下载章节 {photo.id} 用了: {photo.duration:.3f} 秒')
- for image in photo:
- print(f'处理图片 {image.img_file_name} 用了: {image.duration:.3f} 秒')
+ # 2. 耗时统计（单位：秒）
+ print(f'总耗时: {result.duration:.3f} 秒, 本子处理耗时: {album.duration:.3f} 秒')
+
+ # 3. 下载清单 manifest
+ manifest = result.manifest
+ print(f'全部本子图片的磁盘路径: {manifest.image_filepath_list}')
asyncio.run(main())
```
-| 字段 | 它告诉你什么 |
-| --- | --- |
-| `result.duration` | 从调用异步下载方法到返回，你总共等了多久 |
-| `album.duration` | 下载这个本子花了多久，包含获取本子信息和整理下载结果 |
-| `photo.duration` | 下载这个章节花了多久，包含获取或补全章节信息 |
-| `image.duration` | 处理这张图片花了多久，包含检查缓存、下载、解密和保存 |
+## 11. 取消下载
+
+下面示例代码模拟了一个 GUI 场景中，有两个按钮，一个开始下载，一个取消下载，基于 `DownloadControl` 实现。
+
+```python
+import asyncio
+from jmcomic import DownloadCancelledException, DownloadControl, download_album_async, jm_task_context
+
+# 界面持有的当前下载控制器与任务
+current_control = None
+current_task = None
+
+
+# 按钮 1：点击【开始下载】
+def on_btn_download_click(album_id: str) -> asyncio.Task:
+ global current_control, current_task
+ current_control = DownloadControl()
+
+ async def _run():
+ try:
+ with jm_task_context(control=current_control):
+ await download_album_async(album_id)
+ except DownloadCancelledException as e:
+ print(f'[UI 提示] 下载已取消: {e.reason}')
+
+ current_task = asyncio.create_task(_run())
+ return current_task
-下载器可能同时处理多个章节或多张图片，所以把它们的耗时全部相加，不会得到本子的耗时，这是正常现象。同步下载中的这些字段含义相同。
+
+# 按钮 2：点击【取消下载】
+def on_btn_cancel_click():
+ if current_control:
+ current_control.cancel("用户点击了界面的取消按钮")
+
+
+async def main():
+ # 模拟用户点击【开始下载】
+ task = on_btn_download_click('123456')
+
+ # 模拟用户在界面等待一段时间后，点击了【取消下载】
+ await asyncio.sleep(2)
+ on_btn_cancel_click()
+
+ # 等待下载任务响应取消并安全收尾
+ await task
+
+
+asyncio.run(main())
+```
diff --git a/assets/docs/sources/tutorial/16_shared_executors.md b/assets/docs/sources/tutorial/16_shared_executors.md
new file mode 100644
index 000000000..be23eeb63
--- /dev/null
+++ b/assets/docs/sources/tutorial/16_shared_executors.md
@@ -0,0 +1,303 @@
+# 下载并发调优 / 复用线程池
+
+> 日常使用 jmcomic 下载时，不需要关心并发与配置线程池，jmcomic 内部会创建临时线程池来并发下载。
+>
+> 但在以下两种场景下，你可能更希望精细控制：
+>
+> 1. **大批量下载**：一次性下载几十上百个本子，希望严格限制并发（例如“最多同时下 2 个本子，所有图片最多 8 个线程”），避免把网络或内存卡爆。
+> 2. **复用已有的线程池**：你的程序（如 Web 服务、后台调度器）本身已经维护了全局线程池，希望 JMComic 直接复用，避免重复创建和销毁线程。
+>
+> 如果你的诉求是这些，那么本教程推荐你往下阅读。
+---
+
+## 1、先算一笔账
+
+> 下载本子时有多少线程并发？
+
+jmcomic 对下载分为三层，本子层、章节层、图片层
+
+当你写 `download_album([123, 456])` 时，希望下载两个本子，每层的并发数如下：
+
+* **本子层**：取本子总数
+ * 这个场景就是 **2**，两个本子
+
+* **章节层**：取option配置 `threading.photo`，不配默认是当前系统的 CPU 核心数
+ * 假设这个场景是跑在8核的cpu上，那就是 **8**
+
+* **图片层**：取option配置 `threading.image`，不配默认固定为 **30** 个线程。
+ * 这个场景假设没配，就是 **30**
+
+各层级联调度的真实结构如下：
+
+```text
+download_album([123, 456]) ──► 待下载 2 个本子
+
+【本子层】同时跑 2 个本子
+ │
+ ├─► 本子 123 ── 各自独立创建 Photo 线程池 (默认 8 线程)
+ │ ├─► 章节 1 ── 各自独立创建 Image 线程池 (默认 30 线程) ──► 30 线程下载图片
+ │ ├─► 章节 2 ── 各自独立创建 Image 线程池 (默认 30 线程) ──► 30 线程下载图片
+ │ └─► ... (最多同时跑 8 个章节)
+ │
+ └─► 本子 456 ── 各自独立创建 Photo 线程池 (默认 8 线程)
+ ├─► 章节 1 ── 各自独立创建 Image 线程池 (默认 30 线程) ──► 30 线程下载图片
+ ├─► 章节 2 ── 各自独立创建 Image 线程池 (默认 30 线程) ──► 30 线程下载图片
+ └─► ... (最多同时跑 8 个章节)
+```
+
+如果 123,456这个本子的章节和图片非常多，就有可能拉满全部并发，峰值线程数会达到 $2 \text{ (本子)} \times 8 \text{ (章节)} \times 30 \text{ (图片)} = \mathbf{480}$ **个并发线程！**
+
+这种**乘法级（连乘）级联膨胀**，会导致系统瞬时创建上千个系统线程，引起大量 CPU 上下文切换、内存占用飙升、网络连接数耗尽，甚至直接触发目标服务端的反爬限频或把自己的本地网络打瘫痪。
+
+---
+
+## 2、改配置减少并发
+
+要减少并发，最简单直接的办法，就是把线程数改小一点。
+
+直接在 option 中配置章节层和图片层的并发：
+
+```yaml
+# option.yml
+download:
+ threading:
+ photo: 2 # 每个本子同时只下载 2 个章节
+ image: 5 # 每个章节同时只下载 5 张图片
+```
+
+也可以在 Python 代码里直接修改：
+
+```python
+from jmcomic import JmOption, download_album
+
+option = JmOption.default()
+option.download.threading.photo = 2
+option.download.threading.image = 5
+
+download_album([123, 456], option=option)
+```
+
+### 算一笔新账
+
+同样下载 123 和 456 这两个本子，此时的峰值并发变成了：
+
+$$2 \text{ (本子)} \times 2 \text{ (章节)} \times 5 \text{ (图片)} = \mathbf{20} \text{ 个并发线程}$$
+
+从 **480** 骤降到 **20**！
+
+### 减少本子层并发
+
+上面改配置只管住了章节层和图片层。但只要你写的是 `download_album([123, 456])` 传列表，本子层默认依然是同时并发跑的。
+
+如果要减少本子层并发，最直接的做法就是**不要传列表**，而是改成写两行 `download_album`：
+
+```python
+from jmcomic import download_album
+
+# 改成两行：本子 123 下载完后，才开始下载本子 456
+download_album(123)
+download_album(456)
+```
+
+这样本子层就完全不并发了（本子并发数直接降为 1）。再配合上面的配置（photo=2, image=5），同一时刻的峰值线程数就只有：
+
+$$1 \text{ (本子)} \times 2 \text{ (章节)} \times 5 \text{ (图片)} = \mathbf{10} \text{ 个并发线程}$$
+
+---
+
+## 3、仅仅改配置不够的场景
+
+通过上面的改配置或者拆分下载，已经能够满足大部分日常需求。
+
+但在以下场景中，你可能需要更精细的并发控制：
+
+##### 场景 1：一次性下载大量本子，希望限制同时下载的本子数（排队下载）
+
+> 如果你有几十个本子要下载，直接传列表 `download_album([1, 2, ...])` 会导致所有本子同时并发跑起来；而如果写成两行或者循环，又只能一个一个完全串行下载。
+>
+> 如果你希望**同时下载 2 个本子，其余的排队等待**，只改局部配置是无法实现的。
+
+##### 场景 2：希望所有图片下载共用线程池，严格卡死全局线程上限
+
+> 每个章节下载时都会各自创建一套图片线程池。
+>
+> 如果你希望**不管当前有多少个本子在下载，整个程序全局最多只允许 8 个线程在下载图片，大家共享同一个线程池**，就需要共享图片线程池，而纯配置无法实现。
+
+##### 场景 3：复用外部已有的线程池
+
+> 如果你的项目（例如 Web 服务、定时任务调度器）本身已经维护了全局的 `ThreadPoolExecutor`，希望 jmcomic 直接复用，避免每次下载都重复创建和销毁线程。
+
+
+## 4、使用 Runtime 精细控制并发
+
+针对上面介绍的场景，jmcomic 提供了 **Runtime** 机制，目的是更加透明精细地控制并发。
+
+你可以把 Runtime 理解为**线程池管家**。它主要为你提供以下能力：
+
+1. **控制【本子层】并发**：指定最多同时下载几个本子，其余本子排队；
+2. **让【章节/图片层】共享线程池**：不管下载多少本子，章节和图片的底层线程池固定共享，不再连乘膨胀；
+3. **支持复用外部线程池**：可以直接使用你创建的外部线程池 `ThreadPoolExecutor`。
+
+> [!NOTE]
+>
+> 由于同步下载和异步下载是两套API，因此runtime的使用也有区别，需要挑选对应的实现类：
+>
+> - **同步下载**：使用 `JmSyncRuntime`。同步下载涉及“本子、章节、图片”三层，用它来统一管理和共享这三层的线程池。
+> - **异步下载**：使用 `JmAsyncRuntime`。异步下载的网络请求走的是 `asyncio` 协程，不占线程；只有图片解密与写盘需要线程，因此它只负责管理一个 `decode`（解码）线程池。
+
+---
+
+### 4.1 同步下载：使用 JmSyncRuntime
+
+#### 场景 A：自定义各层并发数（大批量下载防卡死）
+
+创建 `JmSyncRuntime` 并指定每一层的最大线程数：
+
+```python
+from jmcomic import JmSyncRuntime, download_album
+
+# 1. 创建 JmSyncRuntime，推荐搭配 with，可以自动关闭runtime创建的线程池资源
+with JmSyncRuntime(
+ album_workers=2, # 最多同时下载 2 个本子（其余本子排队）
+ photo_workers=3, # 所有本子合计最多同时处理 3 个章节
+ image_workers=8, # 所有章节合计最多同时下载 8 张图片
+):
+ # 2. 下载3个本子，这3个本子都受 Runtime 的并发约束
+ download_album(['123456', '789012', '345678'])
+```
+
+**用了 Runtime 之后，并发线程数从“乘法”变成了“加法”**：
+此时系统的总工作线程数恒定为：
+$$2 \text{ (本子)} + 3 \text{ (章节)} + 8 \text{ (图片)} = \mathbf{13} \text{ 个线程}$$
+
+无论你一次性传 10 个还是 100 个本子，总线程数都严格卡在 13 个，既不会爆内存，也不会把电脑或网络卡死。
+
+> [!TIP]
+> 如果某一层不传参数（例如不传 `photo_workers`），该层会使用系统默认值。
+
+#### 场景 B：复用外部已有的线程池
+
+如果你的程序已经有现成的 `ThreadPoolExecutor`，可以直接传给 Runtime 复用：
+
+```python
+from concurrent.futures import ThreadPoolExecutor
+from jmcomic import JmSyncRuntime, download_album
+
+# 假设程序中已有维护好的线程池
+with ThreadPoolExecutor(max_workers=2) as album_pool, \
+ ThreadPoolExecutor(max_workers=3) as photo_pool, \
+ ThreadPoolExecutor(max_workers=8) as image_pool:
+
+ # 传入外部线程池。遵循“谁创建谁关闭”原则，Runtime 退出时不会关闭外部传入的线程池
+ with JmSyncRuntime(
+ album_executor=album_pool,
+ photo_executor=photo_pool,
+ image_executor=image_pool,
+ ):
+ download_album(['123456', '789012'])
+ # 走到这里，album_pool、photo_pool、image_pool 都不会被 runtime 关闭
+```
+
+---
+
+### 4.2 异步下载：使用 JmAsyncRuntime
+
+异步下载与同步不同：网络请求由 `asyncio` 协程高效处理，不占线程；**只有图片的解密与保存需要用到后台线程池**。
+
+因此，`JmAsyncRuntime` 只需要管理一个 `decode`（图片解码）线程池，配置非常简单。
+
+#### 场景 A：指定解码并发线程数
+
+```python
+import asyncio
+from jmcomic import JmAsyncRuntime, download_album_async
+
+
+async def main():
+ # 限制最多同时使用 4 个线程进行图片解密与保存
+ with JmAsyncRuntime(decode_workers=4):
+ await download_album_async(['123456', '789012'])
+
+
+asyncio.run(main())
+```
+
+#### 场景 B：复用外部已有的解码线程池
+
+```python
+import asyncio
+from concurrent.futures import ThreadPoolExecutor
+from jmcomic import JmAsyncRuntime, download_album_async
+
+
+async def main():
+ # 借用现成的外部线程池
+ with ThreadPoolExecutor(max_workers=4) as my_decode_pool:
+ with JmAsyncRuntime(decode_executor=my_decode_pool):
+ await download_album_async('123456')
+
+
+asyncio.run(main())
+```
+
+---
+
+## 5、Web 场景示例：全局长效复用 Runtime
+
+在 FastAPI、Django 等常驻 Web 服务或后台常驻进程中，为每个请求频繁创建和销毁线程池会造成不必要的系统开销。
+
+推荐的做法是：**应用启动时创建全局长效 Runtime，每个请求通过 `jm_task_context` 注入当前请求专属的上下文元数据（如 `session_id`、`user_id`）**。
+
+```python
+from jmcomic import (
+ JmSyncRuntime,
+ download_album,
+ jm_task_context,
+)
+
+# 1. 进程启动时单例初始化全局长效 Runtime，跨请求复用
+app_runtime = JmSyncRuntime(
+ album_workers=4,
+ photo_workers=8,
+ image_workers=16,
+)
+
+# 2. 单个 Web 请求 / 后台作业处理函数
+def handle_download_request(request_id: str, user_id: str, album_ids: list):
+ # 将长效 runtime 与链路追踪字段装配入上下文
+ with jm_task_context(
+ runtime=app_runtime,
+ session_id=request_id,
+ user_id=user_id,
+ ):
+ # 内部所有日志、插件、下载任务均透明享有 session_id，并严格受全局并发控制
+ download_album(album_ids)
+```
+
+---
+
+## 6、Runtime 生命周期与关闭规范
+
+为避免线程泄漏并确保资源安全，JMComic 的 Runtime 严格遵循以下三条生命周期规范：
+
+1. **使用 `with runtime` 上下文管理** → 离开作用域自动关闭
+ > 离开 `with` 作用域（无论是正常完成还是发生未捕获异常退出）时，会自动调用 `runtime.close()`，安全关闭 Runtime 自身创建并拥有的底层线程池。
+
+2. **不用 `with`** → 不会自动关闭
+ > 若直接创建 `runtime = JmSyncRuntime(...)` 而不搭配 `with`，Runtime 不会自动关闭，需要调用方在适当时机显式调用 `runtime.close()`。
+
+3. **用户外传 Executor** → 遵循“谁创建、谁负责关闭”原则
+ > 若在构造 Runtime 时传入了用户自定义的 `*_executor`，Runtime 在 `close()` 时**绝不会**关闭它，其生命周期全权由调用方自行管理。
+
+---
+
+## 7、常见场景速查
+
+| 业务场景 | 推荐代码形态 | 核心优势 |
+| :--- | :--- | :--- |
+| **平时随手下载** | `download_album(id)` | 零配置，开箱即用 |
+| **同步大批量下载限并发** | `with JmSyncRuntime(album_workers=2):`
`download_album(ids)` | 自动生效与回收，防止并发膨胀卡死 |
+| **异步控制解密并发** | `with JmAsyncRuntime(decode_workers=4):`
`await download_album_async(ids)` | 仅限制图片解密写盘线程数，网络仍走协程 |
+| **复用外部已有线程池** | 传入 `*_executor=你的线程池` | 严格遵循“谁创建谁关闭”，外部池不被关闭 |
+| **Web常驻服务复用** | `with jm_task_context(runtime=app_runtime, session_id=...):`
`download_album(ids)` | 进程级长效池跨请求复用，上下文日志透明注入 |
+
diff --git a/pyproject.toml b/pyproject.toml
index be2224380..608f35ef0 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -52,4 +52,4 @@ jmcomic = "jmcomic.cli:main"
jmv = "jmcomic.cli:view_main"
[tool.setuptools.dynamic]
-version = {attr = "jmcomic.__version__"}
\ No newline at end of file
+version = {attr = "jmcomic.__version__"}
diff --git a/src/jmcomic/__init__.py b/src/jmcomic/__init__.py
index 8c969b8ec..71cb50245 100644
--- a/src/jmcomic/__init__.py
+++ b/src/jmcomic/__init__.py
@@ -2,8 +2,10 @@
# 被依赖方 <--- 使用方
# config <--- entity <--- toolkit <--- client <--- option <--- downloader
-__version__ = '2.7.9'
+__version__ = '2.7.10'
+from .jm_exception import DownloadCancelledException
+from .jm_runtime import *
from .jm_task_context import *
from .api import *
from .jm_plugin import *
diff --git a/src/jmcomic/api.py b/src/jmcomic/api.py
index 6ef842835..520d95968 100644
--- a/src/jmcomic/api.py
+++ b/src/jmcomic/api.py
@@ -2,73 +2,212 @@
from time import perf_counter
from .jm_downloader import *
-from .jm_task_context import bind_jm_task_context, jm_task_context
+from .jm_exception import DownloadCancelledException, ExceptionTool
+from .jm_runtime import JmAsyncRuntime, JmSyncRuntime
+from .jm_task_context import (
+ bind_jm_task_context,
+ JTC,
+ jm_task_context,
+)
__DOWNLOAD_API_RET = DownloadResult
-def _download_type(download_api) -> str:
- name = getattr(download_api, '__name__', download_api.__class__.__name__)
- if name.endswith('_async'):
- name = name[:-6]
- if name.startswith('download_'):
- name = name[9:]
- return name
-
-
def _finish_download_result(detail, dler, task_started_at):
manifest = dler.manifest_dict[detail]
manifest.duration = perf_counter() - task_started_at
return DownloadResult(detail, dler)
-def download_batch(download_api,
- jm_id_iter: Union[Iterable, Generator],
- option=None,
- downloader=None,
- **kwargs,
- ) -> BatchResult:
+def _ensure_option(option):
+ return JmModuleConfig.option_class().default() if option is None else option
+
+
+def _raise_if_cancelled_chain(cause_exc: BaseException):
+ """在当前任务已被取消时，将底层异常转换为 DownloadCancelledException 并保留原始异常链。"""
+ control = JTC.get_control()
+ if control is not None and control.is_cancelled:
+ reason = control.reason
+ raise DownloadCancelledException(
+ reason,
+ {
+ ExceptionTool.CONTEXT_KEY_CONTROL: control,
+ ExceptionTool.CONTEXT_KEY_REASON: reason,
+ },
+ ) from cause_exc
+ raise cause_exc
+
+
+def _execute_sync_download(
+ action,
+ download_type: str,
+ jm_id,
+ option,
+ downloader,
+ extra,
+ check_exception: bool,
+ control=None,
+):
+ option = _ensure_option(option)
+ task_started_at = perf_counter()
+
+ with jm_task_context(
+ option=option,
+ download_type=download_type,
+ jm_id=str(jm_id),
+ task_started_at=task_started_at,
+ control=control,
+ ):
+ with new_downloader(option, downloader) as dler:
+ dler.add_features(extra)
+ try:
+ detail = action(dler)
+ if check_exception:
+ dler.raise_if_has_exception()
+ except DownloadCancelledException:
+ raise
+ except Exception as e:
+ _raise_if_cancelled_chain(e)
+
+ return _finish_download_result(detail, dler, task_started_at)
+
+
+async def _execute_async_download(
+ action,
+ download_type: str,
+ jm_id,
+ option,
+ downloader,
+ extra,
+ check_exception: bool,
+ control=None,
+):
+ option = _ensure_option(option)
+ task_started_at = perf_counter()
+
+ async with jm_task_context(
+ option=option,
+ download_type=download_type,
+ jm_id=str(jm_id),
+ task_started_at=task_started_at,
+ control=control,
+ ), new_async_downloader(option, downloader) as dler:
+ dler.add_features(extra)
+ try:
+ detail = await action(dler)
+ if check_exception:
+ dler.raise_if_has_exception()
+ except DownloadCancelledException:
+ raise
+ except Exception as e:
+ _raise_if_cancelled_chain(e)
+
+ return _finish_download_result(detail, dler, task_started_at)
+
+
+def download_batch(
+ download_api,
+ jm_id_iter: Union[Iterable, Generator],
+ option=None,
+ downloader=None,
+ download_type='batch',
+ control=None,
+ **kwargs,
+) -> BatchResult:
"""
批量下载 album / photo
一个album/photo，对应一个线程，对应一个option。
- 返回 BatchResult(set)，支持 for album, dler in result 遍历。
+ 返回 BatchResult，保持 FIFO 顺序，支持 for album, dler in result 遍历与按车号检索。
失败项收集在 result.failed 中，不会静默丢失。
:param download_api: 下载api
:param jm_id_iter: jmid (album_id, photo_id) 的迭代器
:param option: 下载选项，所有的jmid共用一个option
:param downloader: 下载器类
+ :param download_type: 下载类型 ('album', 'photo' 等)，默认 'batch'
+ :param control: 可选的取消控制器 (DownloadControl)，用于外部协作式中断下载任务
"""
- from common import multi_thread_launcher
-
- if option is None:
- option = JmModuleConfig.option_class().default()
-
+ option = _ensure_option(option)
+ # 1. 保序去重 ID 列表
+ jm_ids = list(dict.fromkeys(
+ JmcomicText.parse_to_jm_id(jmid)
+ for jmid in jm_id_iter
+ ))
result = BatchResult()
-
- download_type = _download_type(download_api)
-
- def _safe_download(aid):
- """batch 内部的单任务包装：确保异常被收集而非静默丢失"""
- with jm_task_context(download_type=download_type, jm_id=str(aid)):
- try:
- ret = download_api(aid, option, downloader, **kwargs)
- result.add(ret)
- except Exception as e:
- jm_log('batch.failed', f'批量下载失败: [{aid}], 异常: [{e}]', e)
- result.failed[str(aid)] = e
-
- multi_thread_launcher(
- iter_objs=set(
- JmcomicText.parse_to_jm_id(jmid)
- for jmid in jm_id_iter
- ),
- apply_each_obj_func=bind_jm_task_context(_safe_download),
- wait_finish=True
- )
-
- return result
+ if len(jm_ids) == 0:
+ return result
+
+ with jm_task_context(option=option, control=control):
+ # 2. 单项下载执行闭包：在专属任务上下文中执行，并将异常封装为三元组返回，避免打断批处理调度
+ def _download_one(aid):
+ with jm_task_context(download_type=download_type, jm_id=str(aid)):
+ try:
+ item = download_api(aid, option, downloader, **kwargs)
+ return aid, item, None
+ except Exception as error:
+ return aid, None, error
+
+ runtime = JTC.get_runtime()
+ if runtime is not None and not isinstance(runtime, JmSyncRuntime):
+ raise TypeError('sync batch API requires JmSyncRuntime')
+
+ cancellation_error = None
+ worker = bind_jm_task_context(_download_one)
+
+ # 3. 调度本子层并发：外部有 JmSyncRuntime 则复用；没有则使用单层局部 Runtime 调度本子列表，不向下渗透污染子任务
+ if runtime is not None:
+ futures = runtime.multi_thread_launcher(
+ iter_objs=jm_ids,
+ apply_each_obj_func=worker,
+ wait_finish=True,
+ level='album',
+ default_workers=len(jm_ids),
+ )
+ else:
+ with JmTempRuntime(workers=len(jm_ids)) as launcher:
+ futures = launcher.multi_thread_launcher(
+ iter_objs=jm_ids,
+ apply_each_obj_func=worker,
+ wait_finish=True,
+ )
+
+ # 4. 收集 Future 执行结果：记录成功成果与失败异常
+ completed_results = []
+ succeeded_ids = set()
+
+ for future in futures:
+ aid, item, error = future.result()
+ if error is None:
+ result.add(item)
+ if item is not None:
+ completed_results.append(item)
+ succeeded_ids.add(str(aid))
+ continue
+
+ # 取消属于主动中断，不当成下载失败记录进 failed 字典
+ if isinstance(error, DownloadCancelledException):
+ if cancellation_error is None:
+ cancellation_error = error
+ continue
+
+ # 记录失败日志并存入 failed
+ with jm_task_context(download_type=download_type, jm_id=str(aid)):
+ jm_log(
+ 'batch.failed',
+ f'批量下载失败: [{aid}], 异常: [{error}]',
+ error,
+ )
+ result.failed[str(aid)] = error
+
+ # 5. 收尾判断：若发生取消，补全已完成成果与未完成 ID 后再抛出
+ if cancellation_error is not None:
+ cancellation_error.context[ExceptionTool.CONTEXT_KEY_COMPLETED_RESULTS] = completed_results
+ cancellation_error.context[ExceptionTool.CONTEXT_KEY_UNFINISHED_IDS] = [
+ str(aid) for aid in jm_ids if str(aid) not in succeeded_ids
+ ]
+ raise cancellation_error
+ return result
def download_album(jm_album_id,
@@ -77,11 +216,12 @@ def download_album(jm_album_id,
*,
check_exception=True,
extra=None,
+ control=None,
) -> Union[__DOWNLOAD_API_RET, Set[__DOWNLOAD_API_RET]]:
"""
下载一个本子（album），包含其所有的章节（photo）
- 当jm_album_id不是str或int时，视为批量下载，相当于调用 download_batch(download_album, jm_album_id, option, downloader)
+ 当jm_album_id不是str或int时，视为批量下载，相当于调用 download_batch(download_album, jm_album_id, option, downloader, control=control)
:param jm_album_id: 本子的禁漫车号
:param option: 下载选项
@@ -90,23 +230,31 @@ def download_album(jm_album_id,
并上抛 PartialDownloadFailedException。多 ID 调用会转交 download_batch，此参数不生效；
请检查 BatchResult.failed，或自行封装 download_batch 实现所需的批量异常策略。
:param extra: 下载特性（Feature），下载时动态挂载的附加行为上下文。会自动根据上下文（如 album/photo 来源）自适应参数行为。支持单个 Feature、FeatureChain、或列表
+ :param control: 可选的取消控制器 (DownloadControl)，用于外部协作式中断下载任务
:return: 对于的本子实体类，下载器（如果是上述的批量情况，返回值为download_batch的返回值）
"""
if not isinstance(jm_album_id, (str, int)):
- return download_batch(download_album, jm_album_id, option, downloader, extra=extra)
-
- task_started_at = perf_counter()
- with jm_task_context(download_type='album', jm_id=str(jm_album_id), task_started_at=task_started_at):
- with new_downloader(option, downloader) as dler:
- # 下载类型已记录在 TaskContext 中，Feature 会据此选择执行钩子
- dler.add_features(extra)
- album = dler.download_album(jm_album_id)
-
- if check_exception:
- dler.raise_if_has_exception()
-
- return _finish_download_result(album, dler, task_started_at)
+ return download_batch(
+ download_album,
+ jm_album_id,
+ option,
+ downloader,
+ download_type='album',
+ extra=extra,
+ control=control,
+ )
+
+ return _execute_sync_download(
+ lambda dler: dler.download_album(jm_album_id),
+ 'album',
+ jm_album_id,
+ option,
+ downloader,
+ extra,
+ check_exception,
+ control=control,
+ )
def download_photo(jm_photo_id,
@@ -115,6 +263,7 @@ def download_photo(jm_photo_id,
*,
check_exception=True,
extra=None,
+ control=None,
):
"""
下载一个章节（photo），参数同 download_album。
@@ -123,19 +272,26 @@ def download_photo(jm_photo_id,
BatchResult.failed，或自行封装 download_batch 处理批量异常。
"""
if not isinstance(jm_photo_id, (str, int)):
- return download_batch(download_photo, jm_photo_id, option, downloader, extra=extra)
-
- task_started_at = perf_counter()
- with jm_task_context(download_type='photo', jm_id=str(jm_photo_id), task_started_at=task_started_at):
- with new_downloader(option, downloader) as dler:
- # 下载类型已记录在 TaskContext 中，Feature 会据此选择执行钩子
- dler.add_features(extra)
- photo = dler.download_photo(jm_photo_id)
-
- if check_exception:
- dler.raise_if_has_exception()
-
- return _finish_download_result(photo, dler, task_started_at)
+ return download_batch(
+ download_photo,
+ jm_photo_id,
+ option,
+ downloader,
+ download_type='photo',
+ extra=extra,
+ control=control,
+ )
+
+ return _execute_sync_download(
+ lambda dler: dler.download_photo(jm_photo_id),
+ 'photo',
+ jm_photo_id,
+ option,
+ downloader,
+ extra,
+ check_exception,
+ control=control,
+ )
def new_downloader(option=None, downloader=None) -> JmDownloader:
@@ -187,33 +343,38 @@ async def download_album_async(jm_album_id,
*,
check_exception=True,
extra=None,
+ control=None,
):
- """
- 异步下载一个本子（album），包含其所有的章节（photo）。
+ """异步下载一个本子及其全部章节。
+
+ 当 ``jm_album_id`` 是可迭代对象时使用批量 API。单个 ID 返回
+ ``DownloadResult``；其中 Downloader 的网络 client 已关闭，阻塞工作也已
+ 收敛。通过 ``jm_task_context(runtime=JmAsyncRuntime(...))`` 提供的 decode
+ 执行器由调用方负责关闭。
- - 支持批量下载（当 jm_album_id 为可迭代对象时）
- - 返回 (album, downloader) 元组，其中 downloader 的网络和线程池资源已关闭，仅用于读取下载结果
- - check_exception 仅当 jm_album_id 是单个 ID 时生效。多 ID 场景请检查 BatchResult.failed，
- 或自行封装 download_batch_async 处理批量异常
+ ``check_exception`` 只对单个 ID 生效。批量场景应检查
+ ``BatchResult.failed``，或按需封装 ``download_batch_async``。
"""
if not isinstance(jm_album_id, (str, int)):
return await download_batch_async(download_album_async,
jm_album_id,
option,
downloader,
- extra=extra
+ download_type='album',
+ extra=extra,
+ control=control,
)
- task_started_at = perf_counter()
- with jm_task_context(download_type='album', jm_id=str(jm_album_id), task_started_at=task_started_at):
- async with new_async_downloader(option, downloader) as dler:
- dler.add_features(extra)
- album = await dler.download_album(jm_album_id)
-
- if check_exception:
- dler.raise_if_has_exception()
-
- return _finish_download_result(album, dler, task_started_at)
+ return await _execute_async_download(
+ lambda dler: dler.download_album(jm_album_id),
+ 'album',
+ jm_album_id,
+ option,
+ downloader,
+ extra,
+ check_exception,
+ control=control,
+ )
async def download_photo_async(jm_photo_id,
@@ -222,67 +383,112 @@ async def download_photo_async(jm_photo_id,
*,
check_exception=True,
extra=None,
+ control=None,
):
- """
- 异步下载一个章节（photo）。
- 返回的 downloader 已关闭网络和线程池资源，仅用于读取下载结果。
- check_exception 仅当 jm_photo_id 是单个 ID 时生效。多 ID 场景请检查
- BatchResult.failed，或自行封装 download_batch_async 处理批量异常。
+ """异步下载一个章节。
+
+ 返回的 Downloader 已关闭网络 client，阻塞工作也已收敛。通过
+ ``jm_task_context(runtime=JmAsyncRuntime(...))`` 提供的 decode 执行器由调用方负责关闭。
+
+ ``check_exception`` 只对单个 ID 生效。批量场景应检查
+ ``BatchResult.failed``，或按需封装 ``download_batch_async``。
"""
if not isinstance(jm_photo_id, (str, int)):
return await download_batch_async(download_photo_async,
jm_photo_id,
option,
downloader,
- extra=extra
+ download_type='photo',
+ extra=extra,
+ control=control,
)
- task_started_at = perf_counter()
- with jm_task_context(download_type='photo', jm_id=str(jm_photo_id), task_started_at=task_started_at):
- async with new_async_downloader(option, downloader) as dler:
- dler.add_features(extra)
- photo = await dler.download_photo(jm_photo_id)
-
- if check_exception:
- dler.raise_if_has_exception()
-
- return _finish_download_result(photo, dler, task_started_at)
+ return await _execute_async_download(
+ lambda dler: dler.download_photo(jm_photo_id),
+ 'photo',
+ jm_photo_id,
+ option,
+ downloader,
+ extra,
+ check_exception,
+ control=control,
+ )
-async def download_batch_async(download_api,
- jm_id_iter,
- option=None,
- downloader=None,
- **kwargs,
- ) -> BatchResult:
+async def download_batch_async(
+ download_api,
+ jm_id_iter,
+ option=None,
+ downloader=None,
+ download_type='batch',
+ control=None,
+ **kwargs,
+) -> BatchResult:
"""
异步批量下载 album / photo。
- 容错机制：单个 album/photo 失败不会中止整批，也不会丢失其它已完成结果。
- - 返回 BatchResult(set)，失败项收集在 result.failed 中。
+ - 返回 BatchResult，保持 FIFO 顺序，支持按车号检索，失败项收集在 result.failed 中。
+ :param control: 可选的取消控制器 (DownloadControl)，用于外部协作式中断下载任务
"""
- if option is None:
- option = JmModuleConfig.option_class().default()
-
+ option = _ensure_option(option)
jm_ids = list(dict.fromkeys(JmcomicText.parse_to_jm_id(jmid) for jmid in jm_id_iter))
- download_type = _download_type(download_api)
+ result = BatchResult()
+ if len(jm_ids) == 0:
+ return result
async def _download_one(jmid):
with jm_task_context(download_type=download_type, jm_id=str(jmid)):
return await download_api(jmid, option, downloader, **kwargs)
- results = await asyncio.gather(
- *(_download_one(jmid) for jmid in jm_ids),
- return_exceptions=True,
- )
-
- # 失败不抛出，但要记录到 result.failed，便于调用者排查
- result = BatchResult()
- for jmid, r in zip(jm_ids, results):
- if isinstance(r, BaseException):
- with jm_task_context(download_type=download_type, jm_id=str(jmid)):
- jm_log('async.batch.failed', f'批量下载失败: [{jmid}], 异常: [{r}]', r)
- result.failed[str(jmid)] = r
- else:
- result.add(r)
-
- return result
+ runtime = JTC.get_runtime()
+ if runtime is not None and not isinstance(runtime, JmAsyncRuntime):
+ raise TypeError('async batch API requires JmAsyncRuntime')
+
+ with jm_task_context(option=option, control=control):
+ tasks = [asyncio.create_task(_download_one(jmid)) for jmid in jm_ids]
+ # 等待所有异步任务执行完成
+ results = await asyncio.gather(*tasks, return_exceptions=True)
+
+ # 检查是否有任务被取消
+ cancellation_error = None
+ for item in results:
+ if isinstance(item, DownloadCancelledException):
+ cancellation_error = item
+ break
+
+ # 若发生取消，收集已完成成果与未完成 ID 后抛出，不当成下载失败记录进 failed
+ if cancellation_error is not None:
+ succeeded_ids = {
+ str(jmid)
+ for jmid, item in zip(jm_ids, results)
+ if not isinstance(item, BaseException) and item is not None
+ }
+ cancellation_error.context[ExceptionTool.CONTEXT_KEY_COMPLETED_RESULTS] = [
+ item for item in results
+ if not isinstance(item, BaseException) and item is not None
+ ]
+ cancellation_error.context[ExceptionTool.CONTEXT_KEY_UNFINISHED_IDS] = [
+ str(jmid) for jmid in jm_ids
+ if str(jmid) not in succeeded_ids
+ ]
+ raise cancellation_error
+
+ # 外部协程取消信号
+ for item in results:
+ if isinstance(item, asyncio.CancelledError):
+ raise item
+
+ # 记录失败项并收集成功结果
+ for jmid, item in zip(jm_ids, results):
+ if isinstance(item, BaseException):
+ with jm_task_context(download_type=download_type, jm_id=str(jmid)):
+ jm_log(
+ 'async.batch.failed',
+ f'批量下载失败: [{jmid}], 异常: [{item}]',
+ item,
+ )
+ result.failed[str(jmid)] = item
+ else:
+ result.add(item)
+
+ return result
diff --git a/src/jmcomic/cli.py b/src/jmcomic/cli.py
index f54b6431c..b48b2c765 100644
--- a/src/jmcomic/cli.py
+++ b/src/jmcomic/cli.py
@@ -152,6 +152,7 @@ def option_has_download_progress(option):
def run(self, option):
from .api import download_album, download_photo
+ from .jm_task_context import bind_jm_task_context
from common import MultiTaskLauncher
if len(self.album_id_list) == 0:
@@ -163,11 +164,11 @@ def run(self, option):
launcher = MultiTaskLauncher()
launcher.create_task(
- target=download_album,
+ target=bind_jm_task_context(download_album),
args=(self.album_id_list, option)
)
launcher.create_task(
- target=download_photo,
+ target=bind_jm_task_context(download_photo),
args=(self.photo_id_list, option)
)
diff --git a/src/jmcomic/jm_async_client.py b/src/jmcomic/jm_async_client.py
index 0567e8833..424377fa1 100644
--- a/src/jmcomic/jm_async_client.py
+++ b/src/jmcomic/jm_async_client.py
@@ -10,7 +10,6 @@
from copy import deepcopy
from typing import Sequence
from urllib.parse import urlencode
-
from curl_cffi.requests import AsyncSession
from .jm_client_interface import (
diff --git a/src/jmcomic/jm_async_downloader.py b/src/jmcomic/jm_async_downloader.py
index 3f89e24e0..8cac5374e 100644
--- a/src/jmcomic/jm_async_downloader.py
+++ b/src/jmcomic/jm_async_downloader.py
@@ -12,11 +12,16 @@
import os
from concurrent.futures import ThreadPoolExecutor
-from .jm_downloader import BaseDownloader, record_download_duration
+from .jm_downloader import BaseDownloader, DownloadCallback, record_download_duration
from .jm_entity import JmAlbumDetail, JmPhotoDetail, JmImageDetail
from .jm_toolkit import JmImageTool
from .jm_config import JmModuleConfig, jm_log
-from .jm_task_context import bind_jm_task_context
+from .jm_exception import DownloadCancelledException
+from .jm_runtime import JmAsyncRuntime
+from .jm_task_context import (
+ bind_jm_task_context,
+ JTC,
+)
from .jm_option import JmOption
@@ -50,8 +55,19 @@ def __init__(self,
self._image_semaphore = asyncio.Semaphore(image_concurrency)
self._photo_semaphore = asyncio.Semaphore(photo_concurrency)
- # 解密线程池（CPU 密集操作卸载）
- self._decode_pool = ThreadPoolExecutor(max_workers=decode_worker, thread_name_prefix='jm-async-decode')
+ if decode_worker is None:
+ # 对齐 ThreadPoolExecutor 的默认容量，并由调用点明确交给 Runtime。
+ decode_worker = min(32, (os.cpu_count() or 1) + 4)
+ elif (
+ isinstance(decode_worker, bool)
+ or not isinstance(decode_worker, int)
+ or decode_worker <= 0
+ ):
+ raise ValueError(
+ f'decode_worker must be a positive integer, got {decode_worker!r}'
+ )
+ self._decode_worker = decode_worker
+ self._decode_pool: ThreadPoolExecutor | None = None
@classmethod
def use(cls, *args, **kwargs):
@@ -70,17 +86,38 @@ def use(cls, *args, **kwargs):
# ======================================================================
async def _run_in_decode_pool(self, func, *args):
- loop = asyncio.get_running_loop()
- return await loop.run_in_executor(
- self._decode_pool,
- bind_jm_task_context(func),
- *args,
- )
+ runtime = JTC.get_runtime()
+ if isinstance(runtime, JmAsyncRuntime):
+ executor = runtime.executor('decode', self._decode_worker)
+ else:
+ if self._decode_pool is None:
+ self._decode_pool = ThreadPoolExecutor(
+ max_workers=self._decode_worker,
+ thread_name_prefix='jm-async-decode',
+ )
+ executor = self._decode_pool
+
+ worker = bind_jm_task_context(func)
+ future = executor.submit(worker, *args)
+ waiter = asyncio.wrap_future(future)
+ try:
+ return await asyncio.shield(waiter)
+ except asyncio.CancelledError:
+ # 线程任务无法可靠中断，协程取消后仍需等待写盘等操作收尾。
+ try:
+ await waiter
+ except (asyncio.CancelledError, DownloadCancelledException):
+ pass
+ except BaseException as drain_error:
+ jm_log('dler.cancel.drain.exception', drain_error)
+ raise
@record_download_duration('album_started_at')
async def download_album(self, album_id) -> JmAlbumDetail:
"""对齐 sync JmDownloader.download_album"""
+ self.raise_if_cancelled()
album = await self.client.get_album_detail(album_id)
+ self.raise_if_cancelled()
self.begin_manifest(album)
try:
await self.download_by_album_detail(album)
@@ -106,7 +143,9 @@ async def download_by_album_detail(self, album: JmAlbumDetail):
if photos:
# photo 级并发由 _photo_semaphore 控制（默认 3），包裹整段 photo 下载（见 download_by_photo_detail）。
photo_tasks = [self._safe_download_photo(photo) for photo in photos]
- await asyncio.gather(*photo_tasks)
+ # 对齐 sync 最大容错：普通异常由子任务记录，取消异常等待全部结束后再检查
+ await asyncio.gather(*photo_tasks, return_exceptions=True)
+ self.raise_if_cancelled()
await self.after_album(album)
@@ -114,14 +153,19 @@ async def _safe_download_photo(self, photo: JmPhotoDetail):
"""包装 download_by_photo_detail，对齐 sync @catch_exception 的异常记录"""
try:
await self.download_by_photo_detail(photo)
+ except DownloadCancelledException:
+ raise
except Exception as e:
jm_log('photo.failed', f'章节下载失败: [{photo.id}], 异常: [{e}]', e)
self.download_failed_photo.append((photo, e))
+ self.raise_if_cancelled()
@record_download_duration('photo_started_at')
async def download_photo(self, photo_id) -> JmPhotoDetail:
"""对齐 sync JmDownloader.download_photo"""
+ self.raise_if_cancelled()
photo = await self.client.get_photo_detail(photo_id)
+ self.raise_if_cancelled()
self.begin_manifest(photo)
try:
await self.download_by_photo_detail(photo)
@@ -135,11 +179,14 @@ async def download_by_photo_detail(self, photo: JmPhotoDetail):
异步下载一个章节的所有图片。
对齐 sync JmDownloader.download_by_photo_detail 的回调链路。
"""
+ self.raise_if_cancelled()
photo.save_path = self.option.decide_image_save_dir(photo)
# _photo_semaphore 包裹整段 photo 下载（check_photo + 全部图片），
# 真正限制「同时下载的章节数」（对齐 sync：每个 photo 占用 photo 线程池一个槽位）。
# 章节内图片再由共享的 _image_semaphore 二级限流。
async with self._photo_semaphore:
+ # 排队期间可能被取消；执行 I/O 前必须再次检查。
+ self.raise_if_cancelled()
await self.client.check_photo(photo)
await self.before_photo(photo)
@@ -156,7 +203,9 @@ async def download_by_photo_detail(self, photo: JmPhotoDetail):
self._safe_download_image(image)
for image in image_list
]
- await asyncio.gather(*download_tasks)
+ # 对齐 sync 最大容错：普通异常由子任务记录，取消异常等待全部结束后再检查
+ await asyncio.gather(*download_tasks, return_exceptions=True)
+ self.raise_if_cancelled()
await self.after_photo(photo)
@@ -167,9 +216,12 @@ async def _safe_download_image(self, image: JmImageDetail):
"""
try:
await self._download_single_image(image)
+ except DownloadCancelledException:
+ raise
except Exception as e:
jm_log('image.failed', f'图片下载失败: [{image.download_url}], 异常: [{e}]', e)
self.download_failed_image.append((image, e))
+ self.raise_if_cancelled()
@record_download_duration('image_started_at')
async def _download_single_image(self, image: JmImageDetail):
@@ -194,6 +246,8 @@ async def _download_single_image(self, image: JmImageDetail):
# 异步下载图片（受 image semaphore 限流，并将解密写盘过程也锁入信号量范围内，防大字节积压）
async with self._image_semaphore:
+ # 排队期间可能被取消；执行 I/O 前必须再次检查。
+ self.raise_if_cancelled()
img_resp = await self.client.get_jm_image(image.download_url)
img_bytes = img_resp.content
@@ -259,36 +313,80 @@ def _save_raw(image_bytes, save_path, need_convert=False):
# 生命周期
# ======================================================================
+ async def _dispatch_plugin(self, group: str, **kwargs):
+ plugins = getattr(self.option, 'plugins', None)
+ if plugins is not None and not plugins.get(group):
+ return
+ await asyncio.to_thread(self.option.call_all_plugin, group, **kwargs)
+
+ async def _dispatch_feature(self, when: str, **kwargs):
+ if len(self._feature_list) == 0:
+ return
+ await asyncio.to_thread(self._invoke_features_for, when, **kwargs)
+
async def before_album(self, album: JmAlbumDetail):
- await self._run_in_decode_pool(super().before_album, album)
+ self.raise_if_cancelled()
+ DownloadCallback.before_album(self, album)
+ self.download_success_dict.setdefault(album, {})
+ await self._dispatch_plugin('before_album', album=album, downloader=self)
+ self.raise_if_cancelled()
async def after_album(self, album: JmAlbumDetail):
- await self._run_in_decode_pool(super().after_album, album)
+ self.raise_if_cancelled()
+ DownloadCallback.after_album(self, album)
+ await self._dispatch_plugin('after_album', album=album, downloader=self)
+ self.raise_if_cancelled()
+ await self._dispatch_feature('after_album', album=album, downloader=self)
async def before_photo(self, photo: JmPhotoDetail):
- await self._run_in_decode_pool(super().before_photo, photo)
+ self.raise_if_cancelled()
+ DownloadCallback.before_photo(self, photo)
+ self.download_success_dict.setdefault(photo.from_album, {})
+ self.download_success_dict[photo.from_album].setdefault(photo, [])
+ await self._dispatch_plugin('before_photo', photo=photo, downloader=self)
+ self.raise_if_cancelled()
async def after_photo(self, photo: JmPhotoDetail):
- await self._run_in_decode_pool(super().after_photo, photo)
+ self.raise_if_cancelled()
+ DownloadCallback.after_photo(self, photo)
+ await self._dispatch_plugin('after_photo', photo=photo, downloader=self)
+ self.raise_if_cancelled()
+ await self._dispatch_feature('after_photo', photo=photo, downloader=self)
async def before_image(self, image: JmImageDetail, img_save_path: str):
- await self._run_in_decode_pool(super().before_image, image, img_save_path)
+ self.raise_if_cancelled()
+ DownloadCallback.before_image(self, image, img_save_path)
+ await self._dispatch_plugin('before_image', image=image, downloader=self)
+ self.raise_if_cancelled()
async def after_image(self, image: JmImageDetail, img_save_path: str):
- await self._run_in_decode_pool(super().after_image, image, img_save_path)
+ photo = image.from_photo
+ album = photo.from_album
+ if album in self.download_success_dict:
+ self.download_success_dict[album].setdefault(photo, []).append((image.save_path, image))
+
+ DownloadCallback.after_image(self, image, img_save_path)
+ self.raise_if_cancelled()
+ await self._dispatch_plugin('after_image', image=image, downloader=self)
+ self.raise_if_cancelled()
+ await self._dispatch_feature('after_image', image=image, downloader=self)
def shutdown(self):
"""关闭解密线程池"""
- self._decode_pool.shutdown(wait=False)
+ if self._decode_pool is not None:
+ self._decode_pool.shutdown(wait=False)
+ self._decode_pool = None
async def __aenter__(self):
- # 创建并独占一个 async client（含 AsyncSession）。
- self.client = self.option.new_jm_async_client(max_clients=self._image_concurrency)
+ self.client = self.option.new_jm_async_client(
+ max_clients=self._image_concurrency
+ )
try:
await self.client.setup()
except BaseException:
try:
- await self.client.close()
+ if self.client is not None:
+ await self.client.close()
except BaseException as cleanup_error:
jm_log('dler.cleanup.exception',
f'初始化失败后的资源清理也发生异常: {cleanup_error}', cleanup_error)
@@ -299,13 +397,10 @@ async def __aenter__(self):
return self
async def __aexit__(self, exc_type, exc_val, exc_tb):
- # 关闭顺序：先关网络 client（释放 AsyncSession / libcurl multi handle / 后台任务），
- # 再关解密线程池。两者都要在异常路径下保证释放。
try:
if self.client is not None:
await self.client.close()
finally:
- self.client = None
self.shutdown()
if exc_type is not None:
diff --git a/src/jmcomic/jm_client_impl.py b/src/jmcomic/jm_client_impl.py
index c124ddf63..94538aa23 100644
--- a/src/jmcomic/jm_client_impl.py
+++ b/src/jmcomic/jm_client_impl.py
@@ -275,6 +275,12 @@ def add_favorite_album(self,
},
)
+ if resp.status_code != 200:
+ self.check_special_http_code(resp)
+ self.raise_request_error(resp)
+
+ self.require_resp_success_else_raise(resp, '/ajax/favorite_album')
+
res = resp.json()
if res['status'] != 1:
diff --git a/src/jmcomic/jm_client_interface.py b/src/jmcomic/jm_client_interface.py
index 1b78d6073..3f5cb6b15 100644
--- a/src/jmcomic/jm_client_interface.py
+++ b/src/jmcomic/jm_client_interface.py
@@ -171,14 +171,14 @@ class JmDailyCheckinResp(JmResp):
"""
每日签到打卡响应类
"""
- CODE_SUCCESS = 0 # 签到成功
+ CODE_SUCCESS = 0 # 签到成功
CODE_ALREADY_CHECKED_IN = 1 # 重复签到（今日已签）
def __init__(self, resp, code: int, msg: str, raw_data: dict = None):
super().__init__(resp)
- self.code: int = code # 0=签到成功, 1=重复签到
- self.status: int = code # 兼容字段，值等同于 code
- self.msg: str = msg # 透传服务端的提示信息或奖励内容
+ self.code: int = code # 0=签到成功, 1=重复签到
+ self.status: int = code # 兼容字段，值等同于 code
+ self.msg: str = msg # 透传服务端的提示信息或奖励内容
self.raw_data: dict = raw_data or {}
def __repr__(self):
diff --git a/src/jmcomic/jm_config.py b/src/jmcomic/jm_config.py
index 5e57ce88d..aed34ab8c 100644
--- a/src/jmcomic/jm_config.py
+++ b/src/jmcomic/jm_config.py
@@ -1,10 +1,9 @@
from __future__ import annotations
import logging
-
from common import time_stamp, field_cache, ProxyBuilder
-from .jm_task_context import JM_TASK_CONTEXT, get_jm_task_context
+from .jm_task_context import JM_TASK_CONTEXT, JTC
jm_logger = logging.getLogger('jmcomic')
@@ -68,7 +67,7 @@ def default_jm_logging(topic: str, msg, e: BaseException | None = None):
msg = str(msg)
extra = {
'topic': topic,
- JM_TASK_CONTEXT.name: get_jm_task_context(),
+ JM_TASK_CONTEXT.name: JTC.get_context(),
}
if e is not None:
jm_logger.error(msg, extra=extra, exc_info=e)
@@ -517,7 +516,7 @@ def new_postman(cls, session=False, **kwargs):
'cache': True,
'image': {'decode': True, 'suffix': None},
'threading': {
- 'image': 30,
+ 'image': 20,
'photo': None,
},
},
diff --git a/src/jmcomic/jm_downloader.py b/src/jmcomic/jm_downloader.py
index 2846e547b..a5cf20ea2 100644
--- a/src/jmcomic/jm_downloader.py
+++ b/src/jmcomic/jm_downloader.py
@@ -1,11 +1,14 @@
-import os
-import inspect
from functools import wraps
-from typing import NamedTuple
+from typing import Dict, NamedTuple, Union
from time import perf_counter
from .jm_option import *
-from .jm_task_context import bind_jm_task_context, get_jm_task_context, jm_task_context
+from .jm_runtime import JmTempRuntime, JmSyncRuntime
+from .jm_task_context import (
+ bind_jm_task_context,
+ JTC,
+ jm_task_context,
+)
def record_download_duration(context_key: str, clock=None):
@@ -28,7 +31,7 @@ async def async_wrapper(*args, **kwargs):
entity = get_entity(args, kwargs)
detail_call = isinstance(entity, Downloadable)
# 顶层 ID 下载负责完整耗时，内部 detail 调用复用同一个计时上下文。
- if detail_call and get_jm_task_context().get(context_key) is not None:
+ if detail_call and JTC.get_context_value(context_key) is not None:
return await func(*args, **kwargs)
started_at = get_time()
@@ -46,7 +49,7 @@ def wrapper(*args, **kwargs):
entity = get_entity(args, kwargs)
detail_call = isinstance(entity, Downloadable)
# 顶层 ID 下载负责完整耗时，内部 detail 调用复用同一个计时上下文。
- if detail_call and get_jm_task_context().get(context_key) is not None:
+ if detail_call and JTC.get_context_value(context_key) is not None:
return func(*args, **kwargs)
started_at = get_time()
@@ -79,9 +82,12 @@ def catch_exception(func):
@wraps(func)
def wrapper(self, *args, **kwargs):
self: JmDownloader
+ self.raise_if_cancelled()
try:
return func(self, *args, **kwargs)
except Exception as e:
+ if isinstance(e, DownloadCancelledException):
+ raise
detail: JmBaseEntity = args[0]
if detail.is_image():
detail: JmImageDetail
@@ -196,9 +202,19 @@ def all_success(self) -> bool:
def has_download_failures(self):
return len(self.download_failed_image) != 0 or len(self.download_failed_photo) != 0
+ @property
+ def is_cancelled(self) -> bool:
+ return JTC.is_cancelled()
+
+ @classmethod
+ def raise_if_cancelled(cls) -> None:
+ """在当前下载作用域已取消时抛出异常，子类可重写该检查点。"""
+ JTC.raise_if_cancelled()
+
# 下面是回调方法
def before_album(self, album: JmAlbumDetail):
+ self.raise_if_cancelled()
super().before_album(album)
self.download_success_dict.setdefault(album, {})
self.option.call_all_plugin(
@@ -206,18 +222,22 @@ def before_album(self, album: JmAlbumDetail):
album=album,
downloader=self,
)
+ self.raise_if_cancelled()
def after_album(self, album: JmAlbumDetail):
+ self.raise_if_cancelled()
super().after_album(album)
self.option.call_all_plugin(
'after_album',
album=album,
downloader=self,
)
+ self.raise_if_cancelled()
# 触发匹配 after_album 的 Feature
self._invoke_features_for('after_album', album=album, downloader=self)
def before_photo(self, photo: JmPhotoDetail):
+ self.raise_if_cancelled()
super().before_photo(photo)
self.download_success_dict.setdefault(photo.from_album, {})
self.download_success_dict[photo.from_album].setdefault(photo, [])
@@ -226,35 +246,49 @@ def before_photo(self, photo: JmPhotoDetail):
photo=photo,
downloader=self,
)
+ self.raise_if_cancelled()
def after_photo(self, photo: JmPhotoDetail):
+ self.raise_if_cancelled()
super().after_photo(photo)
self.option.call_all_plugin(
'after_photo',
photo=photo,
downloader=self,
)
+ self.raise_if_cancelled()
# 触发匹配 after_photo 的 Feature
self._invoke_features_for('after_photo', photo=photo, downloader=self)
def before_image(self, image: JmImageDetail, img_save_path):
+ self.raise_if_cancelled()
super().before_image(image, img_save_path)
self.option.call_all_plugin(
'before_image',
image=image,
downloader=self,
)
+ self.raise_if_cancelled()
def after_image(self, image: JmImageDetail, img_save_path):
+ # 1. 图片已成功落盘（或命中本地缓存），第一时间登记入成功清单
+ photo = image.from_photo
+ album = photo.from_album
+ if album in self.download_success_dict:
+ self.download_success_dict[album].setdefault(photo, []).append((image.save_path, image))
+
+ # 2. 日志回调
super().after_image(image, img_save_path)
+
+ # 3. 取消检查
+ self.raise_if_cancelled()
+
+ # 4. 插件回调
self.option.call_all_plugin(
'after_image',
image=image,
downloader=self,
)
- photo = image.from_photo
- album = photo.from_album
- self.download_success_dict.get(album).get(photo).append((image.save_path, image))
def begin_manifest(self, detail: DetailEntity) -> DownloadManifest:
manifest = DownloadManifest()
@@ -310,7 +344,7 @@ def finish_manifest(self, detail: DetailEntity) -> DownloadManifest:
def _require_feature_context() -> str:
from .jm_toolkit import ExceptionTool
- download_type = get_jm_task_context().get('download_type')
+ download_type = JTC.get_context_value('download_type')
ExceptionTool.require_true(
download_type in ('album', 'photo'),
'Feature 注册与执行必须位于下载任务上下文中，请使用 '
@@ -355,12 +389,17 @@ def _invoke_features_for(self, when: str, **kwargs):
download_type = self._require_feature_context()
for feature in self._feature_list:
+ self.raise_if_cancelled()
if feature.should_invoke(when):
try:
feature.invoke(self.option, when=when, **kwargs)
+ except DownloadCancelledException:
+ raise
except Exception as e:
- jm_log('downloader.feature.exception', f'Feature执行失败: [{feature}], 下载类型: [{download_type}], 异常: [{e}]',
+ jm_log('downloader.feature.exception',
+ f'Feature执行失败: [{feature}], 下载类型: [{download_type}], 异常: [{e}]',
e)
+ self.raise_if_cancelled()
def raise_if_has_exception(self):
if not self.has_download_failures:
@@ -375,7 +414,7 @@ def raise_if_has_exception(self):
ExceptionTool.raises(
'\n'.join(msg_ls),
- {'downloader': self},
+ {ExceptionTool.CONTEXT_KEY_DOWNLOADER: self},
PartialDownloadFailedException,
)
@@ -399,17 +438,65 @@ def duration(self) -> Optional[float]:
"""顶层下载耗时，单位：秒。"""
return self.manifest.duration
+ @property
+ def album(self):
+ """下载的本子实体。"""
+ from .jm_entity import JmAlbumDetail
+ if isinstance(self.detail, JmAlbumDetail):
+ return self.detail
+ return getattr(self.detail, 'from_album', None)
+
+ @property
+ def photo(self):
+ """下载的章节实体（若为章节下载）。"""
+ from .jm_entity import JmPhotoDetail
+ if isinstance(self.detail, JmPhotoDetail):
+ return self.detail
+ return None
+
-class BatchResult(set):
+class BatchResult:
"""批量下载结果集。
- 继承 set，完全兼容旧写法 for album, dler in result。
- 新增 .failed 属性，记录失败的 jm_id 和对应异常。
+ 保持 FIFO 插入顺序，兼容 for album, dler in result 遍历与解包。
+ 支持按车号检索：result[aid], result.get(aid), aid in result。
"""
def __init__(self):
- super().__init__()
+ self.succeeded = []
self.failed: Dict[str, BaseException] = {}
+ self._id_map = {}
+
+ def add(self, item, aid=None) -> None:
+ self.succeeded.append(item)
+ if aid is None:
+ detail = getattr(item, 'detail', item)
+ aid = getattr(detail, 'id', None)
+ if aid is not None:
+ self._id_map[str(aid).strip()] = item
+
+ def get(self, aid, default=None):
+ """按车号安全获取成功结果。"""
+ return self._id_map.get(str(aid).strip(), default)
+
+ def __getitem__(self, aid):
+ """按车号下标获取成功结果，不存在抛 KeyError。"""
+ key = str(aid).strip()
+ if key in self._id_map:
+ return self._id_map[key]
+ raise KeyError(aid)
+
+ def __contains__(self, item) -> bool:
+ """支持车号判断 ('123' in result / 123 in result)，亦向下兼容实体判断。"""
+ if isinstance(item, (str, int)) and str(item).strip() in self._id_map:
+ return True
+ return item in self.succeeded
+
+ def __iter__(self):
+ return iter(self.succeeded)
+
+ def __len__(self) -> int:
+ return len(self.succeeded)
@property
def all_succeeded(self) -> bool:
@@ -417,8 +504,10 @@ def all_succeeded(self) -> bool:
@property
def total(self) -> int:
- """预期总数（成功 + 失败）"""
- return len(self) + len(self.failed)
+ return len(self.succeeded) + len(self.failed)
+
+ def __repr__(self) -> str:
+ return f''
class JmDownloader(BaseDownloader):
@@ -438,7 +527,9 @@ def create_client(self):
@record_download_duration('album_started_at')
def download_album(self, album_id):
+ self.raise_if_cancelled()
album = self.client.get_album_detail(album_id)
+ self.raise_if_cancelled()
self.begin_manifest(album)
try:
self.download_by_album_detail(album)
@@ -455,13 +546,16 @@ def download_by_album_detail(self, album: JmAlbumDetail):
self.execute_on_condition(
iter_objs=album,
apply=self.download_by_photo_detail,
- count_batch=self.option.decide_photo_batch_count(album)
+ count_batch=self.option.decide_photo_batch_count(album),
+ level='photo',
)
self.after_album(album)
@record_download_duration('photo_started_at')
def download_photo(self, photo_id):
+ self.raise_if_cancelled()
photo = self.client.get_photo_detail(photo_id)
+ self.raise_if_cancelled()
self.begin_manifest(photo)
try:
self.download_by_photo_detail(photo)
@@ -480,7 +574,8 @@ def download_by_photo_detail(self, photo: JmPhotoDetail):
self.execute_on_condition(
iter_objs=photo,
apply=self.download_by_image_detail,
- count_batch=self.option.decide_image_batch_count(photo)
+ count_batch=self.option.decide_image_batch_count(photo),
+ level='image',
)
self.after_photo(photo)
@@ -509,36 +604,42 @@ def download_by_image_detail(self, image: JmImageDetail):
self.after_image(image, img_save_path)
- def execute_on_condition(self,
- iter_objs: DetailEntity,
- apply: Callable,
- count_batch: int,
- ):
- """
- 调度本子/章节的下载
- """
- iter_objs = self.do_filter(iter_objs)
- count_real = len(iter_objs)
+ def execute_on_condition(self, iter_objs, apply, count_batch, level=None):
+ """使用当前 Runtime 调度本子或章节的下载。"""
+ runtime = JTC.get_runtime()
+ ExceptionTool.require_true(
+ runtime is None or isinstance(runtime, JmSyncRuntime),
+ 'sync downloader requires JmSyncRuntime',
+ )
- if count_real == 0:
+ iter_objs = self.do_filter(iter_objs)
+ if len(iter_objs) == 0:
return
- apply = bind_jm_task_context(apply)
+ ExceptionTool.require_true(
+ not isinstance(count_batch, bool) and isinstance(count_batch, int) and count_batch > 0,
+ f'local download limit must be > 0, got {count_batch!r}',
+ )
- if count_batch >= count_real:
- # 一个图/章节 对应 一个线程
- multi_thread_launcher(
- iter_objs=iter_objs,
- apply_each_obj_func=apply,
- )
+ self.raise_if_cancelled()
+
+ if runtime is None:
+ with JmTempRuntime(workers=count_batch) as runtime:
+ runtime.multi_thread_launcher(
+ iter_objs=iter_objs,
+ apply_each_obj_func=bind_jm_task_context(apply),
+ )
else:
- # 创建batch个线程的线程池
- thread_pool_executor(
+ runtime.multi_thread_launcher(
iter_objs=iter_objs,
- apply_each_obj_func=apply,
- max_workers=count_batch,
+ apply_each_obj_func=bind_jm_task_context(apply),
+ level=level,
+ default_workers=count_batch,
)
+ # 批处理线程等待结束后，统一在此处检查取消状态
+ self.raise_if_cancelled()
+
# 下面是对with语法的支持
def __enter__(self):
diff --git a/src/jmcomic/jm_entity.py b/src/jmcomic/jm_entity.py
index eb66959ab..0824762ae 100644
--- a/src/jmcomic/jm_entity.py
+++ b/src/jmcomic/jm_entity.py
@@ -1,6 +1,5 @@
from collections.abc import Sequence
from functools import lru_cache
-
from common import *
from .jm_config import *
diff --git a/src/jmcomic/jm_exception.py b/src/jmcomic/jm_exception.py
index f88392e44..36e27e30b 100644
--- a/src/jmcomic/jm_exception.py
+++ b/src/jmcomic/jm_exception.py
@@ -94,6 +94,35 @@ def downloader(self):
return self.from_context(ExceptionTool.CONTEXT_KEY_DOWNLOADER)
+class DownloadCancelledException(JmcomicException):
+ description = '下载任务已取消'
+
+ def __init__(self, msg='download cancelled', context=None):
+ """
+ 按 JMComic 异常的统一结构创建下载取消异常。
+ """
+ msg = str(msg or 'download cancelled')
+ context = dict(context or {})
+ context.setdefault(ExceptionTool.CONTEXT_KEY_REASON, msg)
+ super().__init__(msg, context)
+
+ @property
+ def control(self):
+ return self.from_context(ExceptionTool.CONTEXT_KEY_CONTROL)
+
+ @property
+ def reason(self) -> str:
+ return str(self.context.get(ExceptionTool.CONTEXT_KEY_REASON, 'download cancelled'))
+
+ @property
+ def completed_results(self):
+ return list(self.context.get(ExceptionTool.CONTEXT_KEY_COMPLETED_RESULTS) or [])
+
+ @property
+ def unfinished_ids(self):
+ return list(self.context.get(ExceptionTool.CONTEXT_KEY_UNFINISHED_IDS) or [])
+
+
class ExceptionTool:
"""
抛异常的工具
@@ -107,6 +136,10 @@ class ExceptionTool:
CONTEXT_KEY_MISSING_JM_ID = 'missing_jm_id'
CONTEXT_KEY_DOWNLOADER = 'downloader'
CONTEXT_KEY_RETRY_ERRORS = 'retry_errors'
+ CONTEXT_KEY_CONTROL = 'control'
+ CONTEXT_KEY_REASON = 'reason'
+ CONTEXT_KEY_COMPLETED_RESULTS = 'completed_results'
+ CONTEXT_KEY_UNFINISHED_IDS = 'unfinished_ids'
@classmethod
def raises(cls,
@@ -198,20 +231,24 @@ def raise_missing(cls,
)
@classmethod
- def require_true(cls, case: bool, msg: str):
+ def require_true(cls, case: bool, msg: str, etype=None):
if case:
return
- cls.raises(msg)
+ cls.raises(msg, etype=etype)
@classmethod
def replace_old_exception_executor(cls, raises: Callable[[Callable, str, dict], None]):
old = cls.raises
- def new(msg, context=None, _etype=None):
+ def new(msg, context=None, etype=None):
if context is None:
context = {}
- raises(old, msg, context)
+
+ def raise_original(msg, context=None, etype=etype):
+ return old(msg, context, etype)
+
+ raises(raise_original, msg, context)
cls.raises = new
diff --git a/src/jmcomic/jm_feature.py b/src/jmcomic/jm_feature.py
index d93bf7dac..b4614dc65 100644
--- a/src/jmcomic/jm_feature.py
+++ b/src/jmcomic/jm_feature.py
@@ -17,7 +17,7 @@
download_album(id, option, extra=Feature.export_pdf + Feature.export_zip)
"""
from .jm_plugin import *
-from .jm_task_context import get_jm_task_context
+from .jm_task_context import JTC
class Feature:
@@ -37,7 +37,7 @@ class Feature:
@property
def jm_task_context(self) -> dict:
"""Return the current invocation's isolated task-context snapshot."""
- return get_jm_task_context()
+ return JTC.get_context()
def should_invoke(self, when: str) -> bool:
"""
diff --git a/src/jmcomic/jm_option.py b/src/jmcomic/jm_option.py
index a712a5cf9..320fd52ca 100644
--- a/src/jmcomic/jm_option.py
+++ b/src/jmcomic/jm_option.py
@@ -1,6 +1,7 @@
-from .jm_client_impl import *
from typing import Optional, Sequence, Union
+from .jm_client_impl import *
+
class CacheRegistry:
REGISTRY = {}
diff --git a/src/jmcomic/jm_plugin.py b/src/jmcomic/jm_plugin.py
index 62ea2c082..2eac8b09e 100644
--- a/src/jmcomic/jm_plugin.py
+++ b/src/jmcomic/jm_plugin.py
@@ -8,7 +8,7 @@
from .jm_option import *
from .jm_async_downloader import JmAsyncDownloader
from .jm_downloader import JmDownloader
-from .jm_task_context import bind_jm_task_context, get_jm_task_context
+from .jm_task_context import bind_jm_task_context, JTC
class PluginValidationException(Exception):
@@ -156,7 +156,7 @@ def invoke(self, **kwargs) -> None:
@property
def jm_task_context(self) -> dict:
"""Return the current invocation's isolated task-context snapshot."""
- return get_jm_task_context()
+ return JTC.get_context()
@classmethod
def build(cls, option: JmOption) -> 'JmOptionPlugin':
@@ -1193,7 +1193,8 @@ def log(topic, message, worker):
chapter['next_tick'] = tick + chapter['interval'] + random.randint(0, 1)
if task.finished:
progress.update(task.id, description=f'✓ {task.description}')
- log('photo.after', f'章节下载完成: [{chapter_name}], 图片 {int(task.total)}/{int(task.total)}', worker)
+ log('photo.after', f'章节下载完成: [{chapter_name}], 图片 {int(task.total)}/{int(task.total)}',
+ worker)
if album_task is not None:
was_finished = tasks[album_task].finished
done = sum(tasks[chapter['task_id']].finished for chapter in chapters)
@@ -1215,7 +1216,7 @@ def log(topic, message, worker):
@staticmethod
def cli_no_progress_notice():
- if not get_jm_task_context().get('cli_no_progress'):
+ if not JTC.get_context_value('cli_no_progress'):
return ''
return (
diff --git a/src/jmcomic/jm_runtime.py b/src/jmcomic/jm_runtime.py
new file mode 100644
index 000000000..71b1cadb2
--- /dev/null
+++ b/src/jmcomic/jm_runtime.py
@@ -0,0 +1,308 @@
+import concurrent.futures
+from concurrent.futures import Executor, ProcessPoolExecutor, ThreadPoolExecutor
+import threading
+from threading import RLock
+from typing import Callable, Dict, Optional, Set, Tuple
+
+from common import process_single_arg_to_args_and_kwargs
+
+__all__ = (
+ 'JmRuntime',
+ 'JmTempRuntime',
+ 'JmSyncRuntime',
+ 'JmAsyncRuntime',
+)
+
+
+def validate_jm_workers(name: str, workers: int) -> int:
+ if isinstance(workers, bool) or not isinstance(workers, int) or workers <= 0:
+ raise ValueError(f'{name} must be a positive integer, got {workers!r}')
+ return workers
+
+
+def validate_jm_executor(name: str, executor: Executor) -> Executor:
+ if not isinstance(executor, Executor):
+ raise TypeError(f'{name} must be a concurrent.futures.Executor')
+
+ rejected_types = [ProcessPoolExecutor]
+ interpreter_pool = getattr(concurrent.futures, 'InterpreterPoolExecutor', None)
+ if interpreter_pool is not None:
+ rejected_types.append(interpreter_pool)
+ if isinstance(executor, tuple(rejected_types)):
+ raise TypeError(
+ f'{name} must be a same-process thread-based Executor; '
+ f'{executor.__class__.__name__} is not supported'
+ )
+ return executor
+
+
+def normalize_jm_executor_config(
+ name: str,
+ workers: Optional[int],
+ executor: Optional[Executor],
+) -> Tuple[Optional[int], Optional[Executor]]:
+ if workers is not None and executor is not None:
+ raise ValueError(f'{name}_workers and {name}_executor are mutually exclusive')
+ if executor is not None:
+ executor = validate_jm_executor(f'{name}_executor', executor)
+ if workers is not None:
+ workers = validate_jm_workers(f'{name}_workers', workers)
+ return workers, executor
+
+
+class JmRuntime:
+ """
+ 轻量 Runtime：管理 Executor 的配置、创建、调度和关闭。
+
+ 【生命周期与关闭规则规范】：
+ 1. with runtime 上下文管理：
+ 离开 with 作用域（正常完成或异常退出）时，会自动调用 runtime.close()，
+ 关闭由此 Runtime 自身创建并拥有的所有 Executor 线程池。
+ 2. 用户外传 Executor：
+ 若在构造时传入了用户自定义的 executor，Runtime 在 close() 时绝不会关闭它，
+ 生命周期全权由调用方自行管理（严格遵循“谁创建、谁负责关闭”原则）。
+ 3. 不使用 with（裸调用）：
+ 绝不会自动关闭。必须由调用方显式调用 runtime.close()，或作为长效单例常驻复用。
+ """
+
+ def __init__(
+ self,
+ configs: Dict[str, Tuple[Optional[int], Optional[Executor]]],
+ ):
+ self._workers = {
+ level: config[0]
+ for level, config in configs.items()
+ }
+ self._executors = {
+ level: config[1]
+ for level, config in configs.items()
+ }
+ self._owned_executors: Set[Executor] = set()
+ self._lock = RLock()
+ self._closed = False
+ self._ctx_cm = None
+
+ def executor(
+ self,
+ level: Optional[str] = None,
+ default_workers: Optional[int] = None,
+ ) -> Executor:
+ """
+ 返回指定层级的 Executor；未配置时按调用点给出的默认并发数创建。
+ """
+ if level is None:
+ raise TypeError('level is required for this Runtime')
+
+ if default_workers is not None:
+ default_workers = validate_jm_workers(
+ f'{level}_default_workers',
+ default_workers,
+ )
+
+ with self._lock:
+ if self._closed:
+ raise RuntimeError('JmRuntime is closed')
+
+ try:
+ executor = self._executors[level]
+ except KeyError as error:
+ raise ValueError(f'unknown runtime level: {level!r}') from error
+
+ if executor is not None:
+ return executor
+
+ workers = self._workers[level]
+ if workers is None:
+ workers = default_workers
+ self._workers[level] = workers
+
+ if workers is None:
+ executor = ThreadPoolExecutor(thread_name_prefix=f'jm-{level}')
+ else:
+ executor = ThreadPoolExecutor(
+ max_workers=workers,
+ thread_name_prefix=f'jm-{level}',
+ )
+
+ self._executors[level] = executor
+ self._owned_executors.add(executor)
+ return executor
+
+ def multi_thread_launcher(
+ self,
+ iter_objs,
+ apply_each_obj_func: Callable,
+ wait_finish=True,
+ *,
+ level: Optional[str] = None,
+ default_workers: Optional[int] = None,
+ ):
+ """
+ 使用指定层级的 Executor 批量提交任务，并按需等待完成。
+ """
+ executor = self.executor(level, default_workers)
+ futures = []
+
+ try:
+ for obj in iter_objs:
+ args, kwargs = process_single_arg_to_args_and_kwargs(obj)
+ futures.append(executor.submit(apply_each_obj_func, *args, **kwargs))
+ finally:
+ if wait_finish:
+ concurrent.futures.wait(futures)
+
+ return futures
+
+ def close(self) -> None:
+ """
+ 关闭 Runtime 自建的 Executor；调用方传入的 Executor 保持可用。
+ """
+ with self._lock:
+ if self._closed:
+ return
+ self._closed = True
+ owned_executors = tuple(self._owned_executors)
+ self._owned_executors.clear()
+
+ for executor in owned_executors:
+ executor.shutdown(wait=True)
+
+ def __enter__(self):
+ from .jm_task_context import jm_task_context
+ self._ctx_cm = jm_task_context(runtime=self)
+ self._ctx_cm.__enter__()
+ return self
+
+ def __exit__(self, exc_type, exc_val, exc_tb):
+ try:
+ if self._ctx_cm is not None:
+ self._ctx_cm.__exit__(exc_type, exc_val, exc_tb)
+ self._ctx_cm = None
+ finally:
+ self.close()
+
+ async def __aenter__(self):
+ return self.__enter__()
+
+ async def __aexit__(self, exc_type, exc_val, exc_tb):
+ return self.__exit__(exc_type, exc_val, exc_tb)
+
+
+class JmTempRuntime(JmRuntime):
+ """
+ 临时 Runtime：用于单次同步下载中的局部并发（章节层、图片层或批量任务的本子层）。
+ 退出即销毁自建线程池，不自动绑定上下文，不跨任务共享。
+ """
+
+ def __init__(
+ self,
+ *,
+ workers: Optional[int] = None,
+ executor: Optional[Executor] = None,
+ ):
+ if workers is not None and executor is not None:
+ raise ValueError('workers and executor are mutually exclusive')
+ if workers is not None:
+ workers = validate_jm_workers('workers', workers)
+ if executor is not None:
+ executor = validate_jm_executor('executor', executor)
+
+ super().__init__(
+ configs={
+ 'default': (workers, executor),
+ },
+ )
+
+ def executor(
+ self,
+ level: Optional[str] = 'default',
+ default_workers: Optional[int] = None,
+ ) -> Executor:
+ return super().executor(level or 'default', default_workers)
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, exc_type, exc_val, exc_tb):
+ self.close()
+
+ async def __aenter__(self):
+ return self
+
+ async def __aexit__(self, exc_type, exc_val, exc_tb):
+ self.close()
+
+
+class JmSyncRuntime(JmRuntime):
+ """
+ 同步下载 Runtime：分别管理 album、photo、image 三层 Executor。
+ """
+
+ def __init__(
+ self,
+ *,
+ album_workers: Optional[int] = None,
+ album_executor: Optional[Executor] = None,
+ photo_workers: Optional[int] = None,
+ photo_executor: Optional[Executor] = None,
+ image_workers: Optional[int] = None,
+ image_executor: Optional[Executor] = None,
+ # 兼容别名
+ id_workers: Optional[int] = None,
+ id_executor: Optional[Executor] = None,
+ ):
+ if album_workers is None and id_workers is not None:
+ album_workers = id_workers
+ if album_executor is None and id_executor is not None:
+ album_executor = id_executor
+
+ external_executors = [
+ executor
+ for executor in (album_executor, photo_executor, image_executor)
+ if executor is not None
+ ]
+ if len({id(executor) for executor in external_executors}) != len(external_executors):
+ raise ValueError(
+ 'album/photo/image levels must use different executor objects, '
+ 'as sharing one executor across nested levels causes thread pool starvation deadlock'
+ )
+
+ album_config = normalize_jm_executor_config('album', album_workers, album_executor)
+ super().__init__(
+ configs={
+ 'album': album_config,
+ 'photo': normalize_jm_executor_config(
+ 'photo', photo_workers, photo_executor
+ ),
+ 'image': normalize_jm_executor_config(
+ 'image', image_workers, image_executor
+ ),
+ },
+ )
+
+
+class JmAsyncRuntime(JmRuntime):
+ """
+ 异步下载 Runtime：管理图片解密、处理和同步 hook 的 decode Executor。
+ """
+
+ def __init__(
+ self,
+ *,
+ decode_workers: Optional[int] = None,
+ decode_executor: Optional[Executor] = None,
+ ):
+ super().__init__(
+ configs={
+ 'decode': normalize_jm_executor_config(
+ 'decode', decode_workers, decode_executor
+ ),
+ },
+ )
+
+ def executor(
+ self,
+ level: Optional[str] = 'decode',
+ default_workers: Optional[int] = None,
+ ) -> Executor:
+ return super().executor(level or 'decode', default_workers)
diff --git a/src/jmcomic/jm_task_context.py b/src/jmcomic/jm_task_context.py
index 57a5e9e1d..0a64c6bbd 100644
--- a/src/jmcomic/jm_task_context.py
+++ b/src/jmcomic/jm_task_context.py
@@ -1,19 +1,21 @@
import inspect
-from contextlib import contextmanager
from contextvars import ContextVar
from functools import wraps
+from threading import Event, Lock
from types import MappingProxyType
from typing import Callable, Mapping, Optional
+from .jm_runtime import JmRuntime
__all__ = (
'JM_TASK_CONTEXT',
- 'jm_task_context',
+ 'DownloadControl',
+ 'JTC',
'get_jm_task_context',
+ 'jm_task_context',
'bind_jm_task_context',
)
-
_EMPTY_TASK_CONTEXT = MappingProxyType({})
JM_TASK_CONTEXT: ContextVar[Mapping] = ContextVar(
'jm_task_context',
@@ -21,31 +23,178 @@
)
-def get_jm_task_context() -> dict:
- """Return a mutable snapshot of the current JM task context."""
- return dict(JM_TASK_CONTEXT.get())
+class DownloadControl:
+ """
+ 在线程间共享、线程安全且幂等的下载取消信号。
+ """
+
+ def __init__(self):
+ self._event = Event()
+ self._lock = Lock()
+ self._reason: Optional[str] = None
+
+ @property
+ def is_cancelled(self) -> bool:
+ return self._event.is_set()
+
+ def cancel(self, reason: str = 'download cancelled') -> bool:
+ normalized_reason = str(reason or 'download cancelled')
+ with self._lock:
+ if self._event.is_set():
+ return False
+ self._reason = normalized_reason
+ self._event.set()
+ return True
+
+ @property
+ def reason(self) -> Optional[str]:
+ with self._lock:
+ return self._reason
+
+
+class JTC:
+ """
+ JMComic 任务上下文统一门面 (Jm Task Context)。
+ """
+
+ @classmethod
+ def get_context(cls) -> dict:
+ """
+ 返回当前 JM 任务上下文的可变快照。
+ """
+ return dict(JM_TASK_CONTEXT.get())
+
+ @classmethod
+ def get_context_value(cls, key: str, default=None):
+ """
+ 获取当前 JM 任务上下文指定 key 的值，避免全量 dict 拷贝。
+ """
+ return JM_TASK_CONTEXT.get().get(key, default)
+
+ @classmethod
+ def get_runtime(cls) -> Optional[JmRuntime]:
+ """
+ 返回当前任务绑定的 JmRuntime；没有活动 Runtime 时返回 None。
+ """
+ return cls.get_context_value('runtime')
+
+ @classmethod
+ def get_option(cls):
+ """
+ 返回当前任务绑定的 JmOption；没有活动 Option 时返回 None。
+ """
+ return cls.get_context_value('option')
+
+ @classmethod
+ def get_control(cls) -> Optional[DownloadControl]:
+ """
+ 返回当前任务绑定的取消控制器 DownloadControl；未设置时返回 None。
+ """
+ control = cls.get_context_value('control')
+ if control is None:
+ return None
+ if not isinstance(control, DownloadControl):
+ raise TypeError(
+ 'jm_task_context control must be DownloadControl, '
+ f'got {type(control)}'
+ )
+ return control
+
+ @classmethod
+ def is_cancelled(cls) -> bool:
+ """
+ 当前任务是否已被标记取消。
+ """
+ control = cls.get_control()
+ return control is not None and control.is_cancelled
+
+ @classmethod
+ def raise_if_cancelled(cls) -> None:
+ """
+ 若当前任务已被标记取消，直接抛出 DownloadCancelledException。
+ """
+ control = cls.get_control()
+ if control is None or not control.is_cancelled:
+ return
+ from .jm_exception import DownloadCancelledException, ExceptionTool
+ raise DownloadCancelledException(
+ control.reason,
+ {
+ ExceptionTool.CONTEXT_KEY_CONTROL: control,
+ ExceptionTool.CONTEXT_KEY_REASON: control.reason,
+ },
+ )
+
+
+# 兼容已发布的上下文查询入口，内部统一使用 JTC。
+get_jm_task_context = JTC.get_context
+
+
+class jm_task_context:
+ """
+ 临时绑定任务字段；只传播 Option/Runtime/Control，不管理资源生命周期。
+
+ 同时支持同步 with 与异步 async with 上下文管理协议。
+ """
+
+ def __init__(self, *, option=None, runtime=None, control=None, **fields):
+ self.option = option
+ self.runtime = runtime
+ self.control = control
+ self.fields = fields
+ self.token = None
+
+ def __enter__(self):
+ context = JTC.get_context()
+ parent_runtime = context.get('runtime')
+
+ if self.runtime is not None and not isinstance(self.runtime, JmRuntime):
+ raise TypeError(f'runtime must be JmRuntime or None, got {type(self.runtime)}')
+ if (
+ self.runtime is not None
+ and parent_runtime is not None
+ and self.runtime is not parent_runtime
+ ):
+ raise RuntimeError('another JmRuntime is already active in this task context')
+
+ if self.control is not None and not isinstance(self.control, DownloadControl):
+ raise TypeError(f'control must be DownloadControl, got {type(self.control)}')
+
+ context.update(self.fields)
+ if self.runtime is not None:
+ context['runtime'] = self.runtime
+ if self.option is not None:
+ context['option'] = self.option
+ if self.control is not None:
+ context['control'] = self.control
+
+ self.token = JM_TASK_CONTEXT.set(MappingProxyType(context))
+ return self
+
+ def __exit__(self, exc_type, exc_val, exc_tb):
+ if self.token is not None:
+ JM_TASK_CONTEXT.reset(self.token)
+ self.token = None
+ async def __aenter__(self):
+ return self.__enter__()
-@contextmanager
-def jm_task_context(**fields):
- """Temporarily add fields to the current JM task context."""
- context = get_jm_task_context()
- context.update(fields)
- token = JM_TASK_CONTEXT.set(MappingProxyType(context))
- try:
- yield
- finally:
- JM_TASK_CONTEXT.reset(token)
+ async def __aexit__(self, exc_type, exc_val, exc_tb):
+ return self.__exit__(exc_type, exc_val, exc_tb)
def bind_jm_task_context(func: Callable, context: Optional[Mapping] = None) -> Callable:
- """Bind a synchronous callable to a snapshot of the current task context."""
- if (inspect.iscoroutinefunction(func)
- or inspect.iscoroutinefunction(getattr(func, '__call__', None))):
+ """
+ 把完整任务上下文快照绑定到同步可调用对象。
+ """
+ if (
+ inspect.iscoroutinefunction(func)
+ or inspect.iscoroutinefunction(getattr(func, '__call__', None))
+ ):
raise TypeError('bind_jm_task_context only supports synchronous callables')
snapshot = MappingProxyType(dict(
- get_jm_task_context() if context is None else context
+ JTC.get_context() if context is None else context
))
@wraps(func)
diff --git a/src/jmcomic/jm_toolkit.py b/src/jmcomic/jm_toolkit.py
index bd0411ad3..8c967af0f 100644
--- a/src/jmcomic/jm_toolkit.py
+++ b/src/jmcomic/jm_toolkit.py
@@ -1,7 +1,6 @@
from datetime import datetime
from html.parser import HTMLParser
from urllib.parse import unquote, urlparse
-
from PIL import Image
from .jm_exception import *
diff --git a/tests/test_jmcomic/test_jm_api.py b/tests/test_jmcomic/test_jm_api.py
index bf46f99cc..2cd4b437c 100644
--- a/tests/test_jmcomic/test_jm_api.py
+++ b/tests/test_jmcomic/test_jm_api.py
@@ -16,6 +16,9 @@ def test_callback_is_not_public_download_api(self):
):
parameters = inspect.signature(download_api).parameters
self.assertNotIn('callback', parameters)
+ self.assertNotIn('runtime', parameters)
+ self.assertNotIn('workers', parameters)
+ self.assertNotIn('executor', parameters)
self.assertEqual(
parameters['check_exception'].kind,
inspect.Parameter.KEYWORD_ONLY,
diff --git a/tests/test_jmcomic/test_jm_async_custom.py b/tests/test_jmcomic/test_jm_async_custom.py
index e6cb93677..78631295d 100644
--- a/tests/test_jmcomic/test_jm_async_custom.py
+++ b/tests/test_jmcomic/test_jm_async_custom.py
@@ -8,6 +8,7 @@
from jmcomic.jm_async_downloader import JmAsyncDownloader
from jmcomic.jm_client_interface import AsyncJmcomicClient
import asyncio
+from unittest.mock import patch
class Test_Async_Custom(JmAsyncTestConfigurable):
@@ -213,15 +214,23 @@ def test_async_setup_checks_cookies_for_each_session(self):
AsyncJmApiClient._has_setup_domain = old_setup_domain
def test_async_downloader_cleanup_when_setup_fails(self):
- """真实 AsyncSession 初始化失败时，downloader 应回收 client 和线程池"""
+ """真实会话初始化失败时，也要关闭 Downloader 自建线程池。"""
old_auto_update = JmModuleConfig.FLAG_API_CLIENT_AUTO_UPDATE_DOMAIN
old_require_cookies = JmModuleConfig.FLAG_API_CLIENT_REQUIRE_COOKIES
old_app_cookies = JmModuleConfig.APP_COOKIES
old_updated_domains = JmModuleConfig.DOMAIN_API_UPDATED_LIST
old_setup_domain = AsyncJmApiClient._has_setup_domain
+ class ProbeAsyncRuntime(JmAsyncRuntime):
+ instances = []
+
+ def __init__(self):
+ super().__init__(decode_workers=1)
+ self.instances.append(self)
+
loop = asyncio.new_event_loop()
downloader = None
+ runtime = None
try:
JmModuleConfig.FLAG_API_CLIENT_AUTO_UPDATE_DOMAIN = False
JmModuleConfig.FLAG_API_CLIENT_REQUIRE_COOKIES = True
@@ -241,14 +250,21 @@ def test_async_downloader_cleanup_when_setup_fails(self):
photo_concurrency=1,
decode_worker=1,
)
- with self.assertRaises(RequestRetryAllFailException):
- loop.run_until_complete(downloader.__aenter__())
-
+ with patch(
+ 'jmcomic.jm_async_downloader.JmAsyncRuntime',
+ ProbeAsyncRuntime,
+ ):
+ with self.assertRaises(RequestRetryAllFailException):
+ loop.run_until_complete(downloader.__aenter__())
+
+ runtime = ProbeAsyncRuntime.instances[0]
self.assertIsNone(downloader.client)
- self.assertTrue(downloader._decode_pool._shutdown)
+ self.assertIsNone(JTC.get_runtime())
+ with self.assertRaisesRegex(RuntimeError, 'JmRuntime is closed'):
+ runtime.executor('decode', 1)
finally:
- if downloader is not None and not downloader._decode_pool._shutdown:
- downloader.shutdown()
+ if runtime is not None:
+ runtime.close()
loop.close()
JmModuleConfig.FLAG_API_CLIENT_AUTO_UPDATE_DOMAIN = old_auto_update
JmModuleConfig.FLAG_API_CLIENT_REQUIRE_COOKIES = old_require_cookies
diff --git a/tests/test_jmcomic/test_jm_cancellation.py b/tests/test_jmcomic/test_jm_cancellation.py
new file mode 100644
index 000000000..bdb11f4bc
--- /dev/null
+++ b/tests/test_jmcomic/test_jm_cancellation.py
@@ -0,0 +1,609 @@
+import asyncio
+from concurrent.futures import Future
+from types import SimpleNamespace
+from threading import Thread
+import unittest
+from unittest.mock import AsyncMock, Mock, patch
+
+from jmcomic import (
+ BaseDownloader,
+ DownloadCancelledException,
+ DownloadControl,
+ DownloadResult,
+ ExceptionTool,
+ JmAlbumDetail,
+ JmPhotoDetail,
+ JmImageDetail,
+ JmAsyncDownloader,
+ JmAsyncRuntime,
+ JmDownloader,
+ bind_jm_task_context,
+ download_album,
+ download_album_async,
+ download_batch,
+ download_batch_async,
+ JTC,
+ jm_task_context,
+)
+
+
+class Test_Cancellation(unittest.IsolatedAsyncioTestCase):
+
+ def test_control_is_idempotent_and_visible_across_threads(self):
+ control = DownloadControl()
+ seen = []
+
+ with jm_task_context(control=control):
+ worker = bind_jm_task_context(
+ lambda: seen.append(JTC.get_control())
+ )
+ thread = Thread(target=worker)
+ thread.start()
+ thread.join()
+
+ self.assertEqual(seen, [control])
+ self.assertFalse(control.is_cancelled)
+ self.assertIsNone(control.reason)
+ self.assertTrue(control.cancel(123))
+ self.assertFalse(control.cancel('ignored second reason'))
+ self.assertTrue(control.is_cancelled)
+ self.assertEqual(control.reason, '123')
+
+ with jm_task_context(control=control):
+ with self.assertRaises(DownloadCancelledException) as caught:
+ BaseDownloader.raise_if_cancelled()
+
+ self.assertIs(caught.exception.control, control)
+ self.assertEqual(caught.exception.reason, '123')
+ self.assertEqual(
+ caught.exception.context,
+ {
+ ExceptionTool.CONTEXT_KEY_CONTROL: control,
+ ExceptionTool.CONTEXT_KEY_REASON: '123',
+ },
+ )
+
+ def test_downloader_classmethod_is_overrideable(self):
+ BaseDownloader.raise_if_cancelled()
+
+ control = DownloadControl()
+ control.cancel()
+ with jm_task_context(control=control):
+ with self.assertRaises(DownloadCancelledException):
+ BaseDownloader.raise_if_cancelled()
+
+ seen = []
+
+ class CustomDownloader(BaseDownloader):
+
+ @classmethod
+ def raise_if_cancelled(cls) -> None:
+ seen.append(cls)
+
+ object.__new__(CustomDownloader).raise_if_cancelled()
+ self.assertEqual(seen, [CustomDownloader])
+
+ def test_exception_keeps_normal_jmcomic_constructor_shape(self):
+ control = DownloadControl()
+ exception = DownloadCancelledException(
+ 'explicit reason',
+ {
+ ExceptionTool.CONTEXT_KEY_CONTROL: control,
+ ExceptionTool.CONTEXT_KEY_REASON: 'explicit reason',
+ },
+ )
+
+ self.assertIs(exception.control, control)
+ self.assertEqual(exception.reason, 'explicit reason')
+ self.assertEqual(str(exception), 'explicit reason')
+
+ def test_sync_api_uses_control_from_caller_context(self):
+ control = DownloadControl()
+ seen = []
+
+ class ProbeDownloader:
+
+ def __init__(self, _option):
+ self.manifest_dict = {}
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, *_args):
+ return None
+
+ def add_features(self, _extra):
+ pass
+
+ def download_album(self, _album_id):
+ seen.append(JTC.get_control())
+ control.cancel('api probe')
+ BaseDownloader.raise_if_cancelled()
+
+ with jm_task_context(control=control):
+ with self.assertRaises(DownloadCancelledException):
+ download_album('123', option=object(), downloader=ProbeDownloader)
+
+ self.assertEqual(seen, [control])
+
+ def test_sync_batch_raises_cancellation_instead_of_failed_result(self):
+ control = DownloadControl()
+
+ def cancel_first(jmid, *_args, **_kwargs):
+ control.cancel(f'stopped at {jmid}')
+ BaseDownloader.raise_if_cancelled()
+
+ with jm_task_context(control=control):
+ with self.assertRaises(DownloadCancelledException):
+ download_batch(cancel_first, ['123', '456'], option=object())
+
+ def test_sync_batch_propagates_manual_cancellation_without_control(self):
+ def stop(_jmid, *_args, **_kwargs):
+ raise DownloadCancelledException('manual stop')
+
+ with self.assertRaisesRegex(DownloadCancelledException, 'manual stop'):
+ download_batch(stop, ['123'], option=object())
+
+ def test_sync_batch_cancellation_exposes_completed_results_and_unfinished_ids(self):
+ control = DownloadControl()
+
+ def make_album(aid):
+ return JmAlbumDetail(
+ album_id=aid,
+ scramble_id='220980',
+ name=f'album {aid}',
+ episode_list=[('456', '1', 'photo')],
+ page_count=5,
+ pub_date='2020-01-01',
+ update_date='2020-01-01',
+ likes='0',
+ views='0',
+ comment_count='0',
+ works=[],
+ actors=[],
+ authors=[],
+ tags=[],
+ )
+
+ def mock_download(jmid, *_args, **_kwargs):
+ if str(jmid) == '1':
+ album = make_album('1')
+ dler = object()
+ return DownloadResult(detail=album, downloader=dler)
+ if str(jmid) == '2':
+ control.cancel('stop at 2')
+ BaseDownloader.raise_if_cancelled()
+ return None
+
+ with jm_task_context(control=control):
+ with self.assertRaises(DownloadCancelledException) as ctx:
+ download_batch(mock_download, ['1', '2', '3'], option=object())
+
+ e = ctx.exception
+ self.assertEqual(e.reason, 'stop at 2')
+ self.assertEqual(len(e.completed_results), 1)
+ res = e.completed_results[0]
+ self.assertEqual(res.detail.id, '1')
+ self.assertEqual(res.album.id, '1')
+ self.assertIsNotNone(res.downloader)
+ self.assertEqual(set(e.unfinished_ids), {'2', '3'})
+
+ async def test_async_batch_cancellation_exposes_completed_results_and_unfinished_ids(self):
+ control = DownloadControl()
+
+ def make_album(aid):
+ return JmAlbumDetail(
+ album_id=aid,
+ scramble_id='220980',
+ name=f'album {aid}',
+ episode_list=[('456', '1', 'photo')],
+ page_count=5,
+ pub_date='2020-01-01',
+ update_date='2020-01-01',
+ likes='0',
+ views='0',
+ comment_count='0',
+ works=[],
+ actors=[],
+ authors=[],
+ tags=[],
+ )
+
+ async def mock_download(jmid, *_args, **_kwargs):
+ if str(jmid) == '1':
+ album = make_album('1')
+ dler = object()
+ return DownloadResult(detail=album, downloader=dler)
+ if str(jmid) == '2':
+ control.cancel('stop at 2')
+ if control.is_cancelled:
+ raise DownloadCancelledException('stop at 2')
+ return None
+
+ with jm_task_context(control=control):
+ with self.assertRaises(DownloadCancelledException) as ctx:
+ await download_batch_async(mock_download, ['1', '2', '3'], option=object())
+
+ e = ctx.exception
+ self.assertEqual(e.reason, 'stop at 2')
+ self.assertEqual(len(e.completed_results), 1)
+ res = e.completed_results[0]
+ self.assertEqual(res.detail.id, '1')
+ self.assertEqual(res.album.id, '1')
+ self.assertIsNotNone(res.downloader)
+ self.assertEqual(e.unfinished_ids, ['2', '3'])
+
+ def test_download_result_album_and_photo_properties(self):
+ album = JmAlbumDetail(
+ album_id='100',
+ scramble_id='220980',
+ name='album 100',
+ episode_list=[('456', '1', 'photo')],
+ page_count=5,
+ pub_date='2020-01-01',
+ update_date='2020-01-01',
+ likes='0',
+ views='0',
+ comment_count='0',
+ works=[],
+ actors=[],
+ authors=[],
+ tags=[],
+ )
+ res_album = DownloadResult(detail=album, downloader=object())
+ self.assertIs(res_album.album, album)
+ self.assertIsNone(res_album.photo)
+
+ photo = JmPhotoDetail('200', 'photo 200', '0', 1, from_album=album)
+ res_photo = DownloadResult(detail=photo, downloader=object())
+ self.assertIs(res_photo.album, album)
+ self.assertIs(res_photo.photo, photo)
+
+ async def test_empty_batches_do_not_invoke_downloader_cancellation(self):
+ control = DownloadControl()
+ control.cancel('already stopped')
+
+ with jm_task_context(control=control):
+ self.assertEqual(
+ set(download_batch(lambda *_args: None, [], option=object())),
+ set(),
+ )
+ self.assertEqual(
+ set(await download_batch_async(lambda *_args: None, [], option=object())),
+ set(),
+ )
+
+ async def test_async_batch_prefers_control_cancellation_over_task_cancellation(self):
+ async def stop_differently(jmid, *_args, **_kwargs):
+ if str(jmid) == '1':
+ raise asyncio.CancelledError('external cancellation')
+ raise DownloadCancelledException('control cancellation')
+
+ with self.assertRaisesRegex(DownloadCancelledException, 'control cancellation'):
+ await download_batch_async(stop_differently, ['1', '2'], option=object())
+
+ async def test_async_batch_waits_for_cancelled_children_to_finish(self):
+ ready = asyncio.Event()
+ never = asyncio.Event()
+ started = []
+ finished = []
+
+ async def wait_until_cancelled(jmid, *_args, **_kwargs):
+ started.append(str(jmid))
+ if len(started) == 2:
+ ready.set()
+ try:
+ await never.wait()
+ finally:
+ await asyncio.sleep(0)
+ finished.append(str(jmid))
+
+ batch = asyncio.create_task(download_batch_async(
+ wait_until_cancelled,
+ ['1', '2'],
+ option=object(),
+ ))
+ await ready.wait()
+ batch.cancel()
+
+ with self.assertRaises(asyncio.CancelledError):
+ await batch
+
+ self.assertCountEqual(finished, ['1', '2'])
+
+ def test_feature_chain_stops_at_cancellation_boundary(self):
+ control = DownloadControl()
+ option = Mock()
+ downloader = BaseDownloader(option)
+ first = Mock()
+ second = Mock()
+ first.should_invoke.return_value = True
+ second.should_invoke.return_value = True
+ first.invoke.side_effect = lambda *_args, **_kwargs: control.cancel('feature stop')
+ downloader._feature_list = [first, second]
+
+ with jm_task_context(download_type='album', control=control):
+ with self.assertRaises(DownloadCancelledException):
+ downloader._invoke_features_for('after_album', album=object())
+
+ first.invoke.assert_called_once()
+ second.invoke.assert_not_called()
+
+ def test_sync_downloader_preserves_client_failure_on_low_level_call(self):
+ control = DownloadControl()
+ downloader = object.__new__(JmDownloader)
+ downloader.client = Mock()
+
+ def fail_after_cancel(_album_id):
+ control.cancel('请求期间取消')
+ raise RuntimeError('请求失败')
+
+ downloader.client.get_album_detail.side_effect = fail_after_cancel
+
+ with jm_task_context(control=control):
+ with self.assertRaisesRegex(RuntimeError, '请求失败'):
+ downloader.download_album('123')
+
+ def test_sync_api_converts_client_failure_to_cancellation(self):
+ control = DownloadControl()
+
+ class FailingDownloader:
+ def __init__(self, _option):
+ pass
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, *_args):
+ pass
+
+ def add_features(self, _extra):
+ pass
+
+ def raise_if_has_exception(self):
+ pass
+
+ def download_album(self, _album_id):
+ control.cancel('请求期间取消')
+ raise RuntimeError('请求失败')
+
+ with jm_task_context(control=control):
+ with self.assertRaisesRegex(DownloadCancelledException, '请求期间取消') as ctx:
+ download_album('123', option=object(), downloader=FailingDownloader)
+ self.assertIsInstance(ctx.exception.__cause__, RuntimeError)
+ self.assertEqual(str(ctx.exception.__cause__), '请求失败')
+
+ async def test_async_downloader_preserves_client_failure_on_low_level_call(self):
+ control = DownloadControl()
+ downloader = object.__new__(JmAsyncDownloader)
+ downloader.client = Mock()
+
+ async def fail_after_cancel(_album_id):
+ control.cancel('异步请求期间取消')
+ raise RuntimeError('请求失败')
+
+ downloader.client.get_album_detail = fail_after_cancel
+
+ with jm_task_context(control=control):
+ with self.assertRaisesRegex(RuntimeError, '请求失败'):
+ await downloader.download_album('123')
+
+ async def test_async_api_converts_client_failure_to_cancellation(self):
+ control = DownloadControl()
+
+ class FailingAsyncDownloader:
+ def __init__(self, _option):
+ pass
+
+ async def __aenter__(self):
+ return self
+
+ async def __aexit__(self, *_args):
+ pass
+
+ def add_features(self, _extra):
+ pass
+
+ def raise_if_has_exception(self):
+ pass
+
+ async def download_album(self, _album_id):
+ control.cancel('异步请求期间取消')
+ raise RuntimeError('请求失败')
+
+ with jm_task_context(control=control):
+ with self.assertRaisesRegex(DownloadCancelledException, '异步请求期间取消') as ctx:
+ await download_album_async('123', option=object(), downloader=FailingAsyncDownloader)
+ self.assertIsInstance(ctx.exception.__cause__, RuntimeError)
+ self.assertEqual(str(ctx.exception.__cause__), '请求失败')
+
+ async def test_control_cancellation_is_not_recorded_as_image_failure(self):
+ control = DownloadControl()
+ control.cancel('stop before image')
+ downloader = object.__new__(JmAsyncDownloader)
+ downloader.download_failed_image = []
+
+ async def cancelled(_image):
+ downloader.raise_if_cancelled()
+
+ downloader._download_single_image = cancelled
+
+ with jm_task_context(control=control):
+ with self.assertRaises(DownloadCancelledException):
+ await downloader._safe_download_image(object())
+
+ self.assertEqual(downloader.download_failed_image, [])
+
+ async def test_external_task_cancellation_is_not_swallowed(self):
+ downloader = object.__new__(JmAsyncDownloader)
+
+ async def wait_forever(_image):
+ await asyncio.Event().wait()
+
+ downloader._download_single_image = wait_forever
+ task = asyncio.create_task(downloader._safe_download_image(object()))
+ await asyncio.sleep(0)
+ task.cancel()
+
+ with self.assertRaises(asyncio.CancelledError):
+ await task
+
+ async def test_current_async_image_is_recorded_before_control_cancellation(self):
+ control = DownloadControl()
+ downloader = object.__new__(JmAsyncDownloader)
+ downloader.option = Mock()
+ downloader.client = Mock()
+ downloader.client.get_jm_image = AsyncMock()
+ downloader.client.get_jm_image.return_value.content = b'image'
+ downloader._image_semaphore = asyncio.Semaphore(1)
+
+ async def run_inline(func, *args):
+ return func(*args)
+
+ downloader._run_in_decode_pool = run_inline
+ downloader._save_raw = Mock(side_effect=lambda *_args: control.cancel('save finished'))
+ downloader.option.decide_image_filepath.return_value = 'image.jpg'
+ downloader.option.decide_download_cache.return_value = False
+ downloader.option.decide_download_image_decode.return_value = False
+
+ image = Mock(spec=JmImageDetail)
+ image.skip = False
+ image.scramble_id = None
+ image.img_url = 'https://example.invalid/image.jpg'
+ image.download_url = 'https://example.invalid/image.jpg'
+ image.index = 1
+ image.from_photo = Mock()
+ photo = image.from_photo
+ album = photo.from_album
+ photo.is_album.return_value = False
+ downloader.download_success_dict = {album: {photo: []}}
+ downloader.manifest_dict = {}
+ downloader.begin_manifest(photo)
+ downloader.before_image = AsyncMock()
+
+ with patch('jmcomic.jm_async_downloader.os.path.exists', return_value=False):
+ with jm_task_context(control=control):
+ with self.assertRaises(DownloadCancelledException):
+ await downloader._download_single_image(image)
+
+ self.assertEqual(downloader.finish_manifest(photo).image_filepath_list, ['image.jpg'])
+ downloader._save_raw.assert_called_once()
+ downloader.option.call_all_plugin.assert_not_called()
+
+ def test_sync_image_saved_during_cancellation_is_in_manifest(self):
+ control = DownloadControl()
+ downloader = object.__new__(JmDownloader)
+ downloader.option = Mock()
+ downloader.option.decide_image_filepath.return_value = 'image.jpg'
+ downloader.option.decide_download_cache.return_value = False
+ downloader.option.decide_download_image_decode.return_value = False
+ downloader.client = Mock()
+ downloader.client.download_by_image_detail.side_effect = (
+ lambda *_args, **_kwargs: control.cancel('save finished')
+ )
+ downloader.before_image = Mock()
+ image = Mock(spec=JmImageDetail)
+ image.skip = False
+ image.img_url = 'https://example.invalid/image.jpg'
+ image.index = 1
+ image.from_photo = Mock()
+ photo = image.from_photo
+ album = photo.from_album
+ photo.is_album.return_value = False
+ downloader.download_success_dict = {album: {photo: []}}
+ downloader.manifest_dict = {}
+ downloader.begin_manifest(photo)
+
+ with jm_task_context(control=control):
+ with self.assertRaises(DownloadCancelledException):
+ downloader.download_by_image_detail(image)
+
+ self.assertEqual(downloader.finish_manifest(photo).image_filepath_list, ['image.jpg'])
+ downloader.client.download_by_image_detail.assert_called_once()
+ downloader.option.call_all_plugin.assert_not_called()
+
+ async def test_cancelled_decode_logs_worker_failure_and_preserves_cancellation(self):
+ future = Future()
+ submitted = asyncio.Event()
+ executor = Mock()
+
+ def submit(*_args):
+ submitted.set()
+ return future
+
+ executor.submit.side_effect = submit
+ runtime = JmAsyncRuntime(decode_workers=1)
+ downloader = object.__new__(JmAsyncDownloader)
+ downloader._decode_worker = 1
+ error = OSError('save failed')
+ try:
+ with patch.object(runtime, 'executor', return_value=executor):
+ with patch('jmcomic.jm_async_downloader.jm_log') as log:
+ with jm_task_context(runtime=runtime):
+ task = asyncio.create_task(downloader._run_in_decode_pool(lambda: None))
+ await asyncio.wait_for(submitted.wait(), timeout=1)
+ task.cancel()
+ await asyncio.sleep(0)
+ future.set_exception(error)
+ with self.assertRaises(asyncio.CancelledError):
+ await task
+ log.assert_called_once_with('dler.cancel.drain.exception', error)
+ finally:
+ runtime.close()
+
+ async def test_image_cancelled_while_waiting_for_semaphore_never_requests(self):
+ control = DownloadControl()
+ downloader = object.__new__(JmAsyncDownloader)
+ downloader.option = Mock()
+ downloader.client = Mock()
+ downloader.client.get_jm_image = AsyncMock()
+ downloader._image_semaphore = asyncio.Semaphore(1)
+ await downloader._image_semaphore.acquire()
+ downloader.option.decide_image_filepath.return_value = 'image.jpg'
+ downloader.option.decide_download_cache.return_value = False
+ downloader.option.decide_download_image_decode.return_value = False
+ before_reached = asyncio.Event()
+
+ async def before_image(_image, _path):
+ before_reached.set()
+
+ downloader.before_image = before_image
+ downloader.after_image = AsyncMock()
+ image = SimpleNamespace(
+ skip=False,
+ scramble_id=None,
+ download_url='https://example.invalid/image.jpg',
+ )
+
+ with patch('jmcomic.jm_async_downloader.os.path.exists', return_value=False):
+ with jm_task_context(control=control):
+ task = asyncio.create_task(downloader._download_single_image(image))
+ await asyncio.wait_for(before_reached.wait(), timeout=1)
+ await asyncio.sleep(0)
+ control.cancel('queued image')
+ downloader._image_semaphore.release()
+ with self.assertRaises(DownloadCancelledException):
+ await task
+
+ downloader.client.get_jm_image.assert_not_awaited()
+
+ async def test_photo_cancelled_while_waiting_for_semaphore_never_requests(self):
+ control = DownloadControl()
+ downloader = object.__new__(JmAsyncDownloader)
+ downloader.option = Mock()
+ downloader.option.decide_image_save_dir.return_value = 'photo'
+ downloader.client = Mock()
+ downloader.client.check_photo = AsyncMock()
+ downloader._photo_semaphore = asyncio.Semaphore(1)
+ await downloader._photo_semaphore.acquire()
+ photo = SimpleNamespace()
+
+ with jm_task_context(control=control):
+ task = asyncio.create_task(downloader.download_by_photo_detail(photo))
+ await asyncio.sleep(0)
+ control.cancel('queued photo')
+ downloader._photo_semaphore.release()
+ with self.assertRaises(DownloadCancelledException):
+ await task
+
+ downloader.client.check_photo.assert_not_awaited()
diff --git a/tests/test_jmcomic/test_jm_cli.py b/tests/test_jmcomic/test_jm_cli.py
index ac692244c..3e6f23d26 100644
--- a/tests/test_jmcomic/test_jm_cli.py
+++ b/tests/test_jmcomic/test_jm_cli.py
@@ -1,10 +1,12 @@
+from concurrent.futures import ThreadPoolExecutor
+
from test_jmcomic import *
from io import StringIO
from types import SimpleNamespace
from unittest.mock import MagicMock, patch
from jmcomic.cli import JmcomicUI, JmViewUI
-from jmcomic.jm_task_context import get_jm_task_context
+from jmcomic import JmSyncRuntime, JTC, jm_task_context
class Test_Cli(JmTestConfigurable):
@@ -78,7 +80,7 @@ def parse_arg():
ui.option_path = None
def create_default_option():
- self.assertTrue(get_jm_task_context().get('cli_no_progress'))
+ self.assertTrue(JTC.get_context().get('cli_no_progress'))
return option
with patch.object(ui, 'parse_arg', side_effect=parse_arg), \
@@ -88,7 +90,7 @@ def create_default_option():
patch('jmcomic.api.jm_log'):
ui.main()
- self.assertNotIn('cli_no_progress', get_jm_task_context())
+ self.assertNotIn('cli_no_progress', JTC.get_context())
def test_jmcomic_falls_back_without_rich(self):
ui = JmcomicUI()
@@ -125,6 +127,44 @@ def test_jmcomic_parse_mixed(self):
self.assertEqual(ui.album_id_list, [self.album_id])
self.assertEqual(ui.photo_id_list, [self.album_id])
+ def test_jmcomic_mixed_threads_inherit_runtime(self):
+ ui = JmcomicUI()
+ ui.album_id_list = ['1']
+ ui.photo_id_list = ['2']
+ option = SimpleNamespace()
+ observed = []
+
+ def record(ids, actual_option):
+ observed.append((
+ ids,
+ actual_option,
+ JTC.get_runtime(),
+ JTC.get_context(),
+ ))
+
+ with ThreadPoolExecutor(max_workers=1) as executor:
+ runtime = JmSyncRuntime(album_executor=executor)
+ with patch('jmcomic.api.download_album', side_effect=record), \
+ patch('jmcomic.api.download_photo', side_effect=record):
+ with jm_task_context(task_id='cli-job', runtime=runtime):
+ ui.run(option)
+
+ self.assertCountEqual(
+ [(ids, actual_option) for ids, actual_option, _executor, _context in observed],
+ [(['1'], option), (['2'], option)],
+ )
+ self.assertEqual(
+ [seen for _ids, _option, seen, _context in observed],
+ [runtime, runtime],
+ )
+ self.assertEqual(
+ [context for _ids, _option, _executor, context in observed],
+ [
+ {'task_id': 'cli-job', 'runtime': runtime},
+ {'task_id': 'cli-job', 'runtime': runtime},
+ ],
+ )
+
def test_jmcomic_download_album(self):
"""jmcomic 真实下载 album 123"""
JustDownloadSpecificCountImage.count = 5
diff --git a/tests/test_jmcomic/test_jm_client.py b/tests/test_jmcomic/test_jm_client.py
index 0a15fce9c..abc32dbad 100644
--- a/tests/test_jmcomic/test_jm_client.py
+++ b/tests/test_jmcomic/test_jm_client.py
@@ -291,17 +291,9 @@ def search_and_test(case_name, params):
raise AssertionError(elist)
def test_comment_count(self):
- aid = 'JM438516'
+ aid = '350234'
album = self.client.get_album_detail(aid)
self.assertGreater(album.comment_count, 0)
- page = self.client.search_site('无修正')
- for i in range(3):
- aid, _atitle = page[i]
- self.assertGreaterEqual(
- self.client.get_album_detail(aid).comment_count,
- 0,
- aid,
- )
def test_album_pagination(self):
album_id = '438516'
diff --git a/tests/test_jmcomic/test_jm_download_manifest.py b/tests/test_jmcomic/test_jm_download_manifest.py
index bd7815856..461153141 100644
--- a/tests/test_jmcomic/test_jm_download_manifest.py
+++ b/tests/test_jmcomic/test_jm_download_manifest.py
@@ -59,7 +59,9 @@ def decide_image_save_dir(self, _photo):
return os.path.join(self.base_dir, 'album', 'photo')
def decide_image_filepath(self, image):
- return os.path.join(self.decide_image_save_dir(image.from_photo), image.filename)
+ save_dir = self.decide_image_save_dir(image.from_photo)
+ os.makedirs(save_dir, exist_ok=True)
+ return os.path.join(save_dir, image.filename)
def decide_download_cache(self, _image):
return True
@@ -75,7 +77,7 @@ def decide_image_batch_count(self, _photo):
def call_all_plugin(self, group, **kwargs):
self.plugin_event_list.append((group, kwargs))
- self.context_event_list.append((group, get_jm_task_context()))
+ self.context_event_list.append((group, JTC.get_context()))
if group == 'after_image' and self.after_image_callback is not None:
self.after_image_callback(kwargs['image'])
@@ -122,7 +124,7 @@ def do_filter(self, detail):
return list(self._contract_image_list)
return detail
- def execute_on_condition(self, iter_objs, apply, count_batch):
+ def execute_on_condition(self, iter_objs, apply, count_batch, level=None, **kwargs):
for detail in self.do_filter(iter_objs):
apply(detail)
@@ -143,6 +145,10 @@ async def get_photo_detail(self, _photo_id):
async def check_photo(self, _photo):
return None
+ async def get_jm_image(self, _url):
+ self.image_download_count += 1
+ return SimpleNamespace(content=b'image')
+
class ContractAsyncDownloader(JmAsyncDownloader):
@@ -163,6 +169,132 @@ def do_filter(self, detail):
class Test_Download_Manifest(unittest.TestCase):
+ def test_after_image_error_still_registers_saved_image_sync(self):
+ with TemporaryDirectory() as temp_dir:
+ album, photo, images = new_album_photo_images()
+ option = ContractOption(temp_dir)
+ downloader = ContractSyncDownloader(option, album, photo, images)
+ error = ValueError('image plugin failed')
+
+ def fail(_image):
+ raise error
+
+ option.after_image_callback = fail
+ with self.assertRaises(ValueError) as caught:
+ downloader.download_album(album.id)
+
+ self.assertIs(caught.exception, error)
+ self.assertTrue(os.path.isfile(images[0].save_path))
+ self.assertEqual(
+ downloader.download_success_dict[album][photo],
+ [(images[0].save_path, images[0])],
+ )
+ self.assertEqual(downloader.download_failed_image, [(images[0], error)])
+ self.assertEqual(
+ downloader.manifest_dict[album].image_filepath_list,
+ [images[0].save_path],
+ )
+
+ def test_after_image_error_still_registers_saved_image_async(self):
+ async def run_test(temp_dir):
+ album, photo, images = new_album_photo_images()
+ option = ContractOption(temp_dir)
+ downloader = ContractAsyncDownloader(option, album, photo, images)
+ error = ValueError('image plugin failed')
+
+ def fail(_image):
+ raise error
+
+ option.after_image_callback = fail
+ runtime = JmAsyncRuntime()
+ try:
+ with jm_task_context(runtime=runtime):
+ await downloader.download_album(album.id)
+ finally:
+ runtime.close()
+
+ self.assertTrue(os.path.isfile(images[0].save_path))
+ self.assertEqual(
+ downloader.download_success_dict[album][photo],
+ [(images[0].save_path, images[0])],
+ )
+ self.assertEqual(downloader.download_failed_image, [(images[0], error)])
+ self.assertEqual(
+ downloader.manifest_dict[album].image_filepath_list,
+ [images[0].save_path],
+ )
+
+ with TemporaryDirectory() as temp_dir:
+ asyncio.run(run_test(temp_dir))
+
+ def test_after_image_direct_cancellation_keeps_saved_image(self):
+ with TemporaryDirectory() as temp_dir:
+ album, photo, images = new_album_photo_images()
+ option = ContractOption(temp_dir)
+ downloader = ContractSyncDownloader(option, album, photo, images)
+ error = DownloadCancelledException('image plugin cancelled')
+
+ def cancel(_image):
+ raise error
+
+ option.after_image_callback = cancel
+ with self.assertRaises(DownloadCancelledException) as caught:
+ downloader.download_album(album.id)
+
+ self.assertIs(caught.exception, error)
+ self.assertEqual(downloader.download_failed_image, [])
+ self.assertEqual(
+ downloader.manifest_dict[album].image_filepath_list,
+ [images[0].save_path],
+ )
+
+ def test_current_sync_image_is_recorded_before_cancellation(self):
+ with TemporaryDirectory() as temp_dir:
+ album, photo, image_list = new_album_photo_images()
+ option = ContractOption(temp_dir)
+ downloader = ContractSyncDownloader(option, album, photo, image_list)
+ control = DownloadControl()
+ option.after_image_callback = lambda image: control.cancel('stop after image')
+
+ with jm_task_context(control=control):
+ with self.assertRaises(DownloadCancelledException):
+ downloader.download_album(album.id)
+
+ image = image_list[0]
+ self.assertTrue(os.path.isfile(image.save_path))
+ self.assertEqual(
+ downloader.download_success_dict[album][photo],
+ [(image.save_path, image)],
+ )
+ self.assertEqual(downloader.download_failed_image, [])
+
+ def test_current_async_image_is_recorded_before_cancellation(self):
+ async def run_test(temp_dir):
+ album, photo, image_list = new_album_photo_images()
+ option = ContractOption(temp_dir)
+ downloader = ContractAsyncDownloader(option, album, photo, image_list)
+ control = DownloadControl()
+ option.after_image_callback = lambda image: control.cancel('stop after image')
+ os.makedirs(option.decide_image_save_dir(photo), exist_ok=True)
+
+ runtime = JmAsyncRuntime()
+ try:
+ with jm_task_context(control=control, runtime=runtime):
+ with self.assertRaises(DownloadCancelledException):
+ await downloader.download_album(album.id)
+ finally:
+ runtime.close()
+
+ image = image_list[0]
+ self.assertEqual(
+ downloader.download_success_dict[album][photo],
+ [(image.save_path, image)],
+ )
+ self.assertEqual(downloader.download_failed_image, [])
+
+ with TemporaryDirectory() as temp_dir:
+ asyncio.run(run_test(temp_dir))
+
def test_downloadable_defaults(self):
album, photo, image_list = new_album_photo_images()
@@ -798,7 +930,7 @@ def test_entity_timing_contexts_are_nested_and_do_not_leak(self):
[2, 2],
[context['photo_started_at'] for context in context_by_group['before_image']],
)
- self.assertEqual({}, get_jm_task_context())
+ self.assertEqual({}, JTC.get_context())
def test_sync_download_album_duration_includes_detail_and_manifest(self):
clock = {'now': 10.0}
@@ -807,20 +939,20 @@ def test_sync_download_album_duration_includes_detail_and_manifest(self):
downloader = object.__new__(JmDownloader)
def get_album_detail(_album_id):
- contexts.append(get_jm_task_context())
+ contexts.append(JTC.get_context())
clock['now'] = 20.0
return album
def begin_manifest(_album):
- contexts.append(get_jm_task_context())
+ contexts.append(JTC.get_context())
clock['now'] = 30.0
def download_by_album_detail(_album):
- contexts.append(get_jm_task_context())
+ contexts.append(JTC.get_context())
clock['now'] = 40.0
def finish_manifest(_album):
- contexts.append(get_jm_task_context())
+ contexts.append(JTC.get_context())
clock['now'] = 50.0
downloader.client = SimpleNamespace(get_album_detail=get_album_detail)
@@ -834,7 +966,7 @@ def finish_manifest(_album):
self.assertIs(result, album)
self.assertEqual(40.0, album.duration)
self.assertEqual([10.0] * 4, [context.get('album_started_at') for context in contexts])
- self.assertEqual({}, get_jm_task_context())
+ self.assertEqual({}, JTC.get_context())
def test_sync_download_photo_duration_includes_detail_and_manifest(self):
clock = {'now': 10.0}
@@ -843,20 +975,20 @@ def test_sync_download_photo_duration_includes_detail_and_manifest(self):
downloader = object.__new__(JmDownloader)
def get_photo_detail(_photo_id):
- contexts.append(get_jm_task_context())
+ contexts.append(JTC.get_context())
clock['now'] = 20.0
return photo
def begin_manifest(_photo):
- contexts.append(get_jm_task_context())
+ contexts.append(JTC.get_context())
clock['now'] = 30.0
def download_by_photo_detail(_photo):
- contexts.append(get_jm_task_context())
+ contexts.append(JTC.get_context())
clock['now'] = 40.0
def finish_manifest(_photo):
- contexts.append(get_jm_task_context())
+ contexts.append(JTC.get_context())
clock['now'] = 50.0
downloader.client = SimpleNamespace(get_photo_detail=get_photo_detail)
@@ -870,7 +1002,7 @@ def finish_manifest(_photo):
self.assertIs(result, photo)
self.assertEqual(40.0, photo.duration)
self.assertEqual([10.0] * 4, [context.get('photo_started_at') for context in contexts])
- self.assertEqual({}, get_jm_task_context())
+ self.assertEqual({}, JTC.get_context())
def test_async_download_album_duration_includes_detail_and_manifest(self):
async def run_test():
@@ -880,20 +1012,20 @@ async def run_test():
downloader = object.__new__(JmAsyncDownloader)
async def get_album_detail(_album_id):
- contexts.append(get_jm_task_context())
+ contexts.append(JTC.get_context())
clock['now'] = 20.0
return album
def begin_manifest(_album):
- contexts.append(get_jm_task_context())
+ contexts.append(JTC.get_context())
clock['now'] = 30.0
async def download_by_album_detail(_album):
- contexts.append(get_jm_task_context())
+ contexts.append(JTC.get_context())
clock['now'] = 40.0
def finish_manifest(_album):
- contexts.append(get_jm_task_context())
+ contexts.append(JTC.get_context())
clock['now'] = 50.0
downloader.client = SimpleNamespace(get_album_detail=get_album_detail)
@@ -907,7 +1039,7 @@ def finish_manifest(_album):
self.assertIs(result, album)
self.assertEqual(40.0, album.duration)
self.assertEqual([10.0] * 4, [context.get('album_started_at') for context in contexts])
- self.assertEqual({}, get_jm_task_context())
+ self.assertEqual({}, JTC.get_context())
asyncio.run(run_test())
@@ -919,20 +1051,20 @@ async def run_test():
downloader = object.__new__(JmAsyncDownloader)
async def get_photo_detail(_photo_id):
- contexts.append(get_jm_task_context())
+ contexts.append(JTC.get_context())
clock['now'] = 20.0
return photo
def begin_manifest(_photo):
- contexts.append(get_jm_task_context())
+ contexts.append(JTC.get_context())
clock['now'] = 30.0
async def download_by_photo_detail(_photo):
- contexts.append(get_jm_task_context())
+ contexts.append(JTC.get_context())
clock['now'] = 40.0
def finish_manifest(_photo):
- contexts.append(get_jm_task_context())
+ contexts.append(JTC.get_context())
clock['now'] = 50.0
downloader.client = SimpleNamespace(get_photo_detail=get_photo_detail)
@@ -946,7 +1078,7 @@ def finish_manifest(_photo):
self.assertIs(result, photo)
self.assertEqual(40.0, photo.duration)
self.assertEqual([10.0] * 4, [context.get('photo_started_at') for context in contexts])
- self.assertEqual({}, get_jm_task_context())
+ self.assertEqual({}, JTC.get_context())
asyncio.run(run_test())
@@ -1036,21 +1168,23 @@ async def run_test(temp_dir):
f.write(b'cached')
downloader = ContractAsyncDownloader(option, album, photo, image_list)
+ runtime = JmAsyncRuntime()
try:
- await downloader.download_album(album.id)
-
- self.assertEqual(album.save_path, option.dir_rule.decide_album_root_dir(album))
- self.assertEqual(photo.save_path, option.decide_image_save_dir(photo))
- self.assertEqual(image.save_path, filepath)
- self.assertIsInstance(album.duration, float)
- self.assertIsInstance(photo.duration, float)
- self.assertIsInstance(image.duration, float)
- after_image_events = [event for event, _ in option.plugin_event_list if event == 'after_image']
- self.assertEqual(after_image_events, ['after_image'])
- self.assertEqual(downloader.download_success_dict[album][photo], [(filepath, image)])
- self.assertEqual(downloader.manifest_dict[album].image_filepath_list, [filepath])
+ with jm_task_context(runtime=runtime):
+ await downloader.download_album(album.id)
finally:
- downloader.shutdown()
+ runtime.close()
+
+ self.assertEqual(album.save_path, option.dir_rule.decide_album_root_dir(album))
+ self.assertEqual(photo.save_path, option.decide_image_save_dir(photo))
+ self.assertEqual(image.save_path, filepath)
+ self.assertIsInstance(album.duration, float)
+ self.assertIsInstance(photo.duration, float)
+ self.assertIsInstance(image.duration, float)
+ after_image_events = [event for event, _ in option.plugin_event_list if event == 'after_image']
+ self.assertEqual(after_image_events, ['after_image'])
+ self.assertEqual(downloader.download_success_dict[album][photo], [(filepath, image)])
+ self.assertEqual(downloader.manifest_dict[album].image_filepath_list, [filepath])
with TemporaryDirectory() as temp_dir:
asyncio.run(run_test(temp_dir))
diff --git a/tests/test_jmcomic/test_jm_download_progress.py b/tests/test_jmcomic/test_jm_download_progress.py
index fe2894f5e..f83781ed6 100644
--- a/tests/test_jmcomic/test_jm_download_progress.py
+++ b/tests/test_jmcomic/test_jm_download_progress.py
@@ -3,7 +3,6 @@
import importlib.util
import logging
import os
-import time
import unittest
from io import StringIO
from pathlib import Path
@@ -28,10 +27,6 @@
from jmcomic.jm_task_context import jm_task_context
-PROJECT_DIR = Path(__file__).resolve().parents[2]
-DOCUMENT_FILE = PROJECT_DIR / 'assets' / 'docs' / 'sources' / 'tutorial' / '15_download_progress.md'
-PLUGIN_FILE = PROJECT_DIR / 'src' / 'jmcomic' / 'jm_plugin.py'
-DOWNLOADER_FILE = PROJECT_DIR / 'src' / 'jmcomic' / 'jm_downloader.py'
RICH_INSTALLED = importlib.util.find_spec('rich') is not None
@@ -318,7 +313,7 @@ class AfterAsyncDownloader(JmAsyncDownloader):
JmModuleConfig.CLASS_ASYNC_DOWNLOADER = original_downloader
@unittest.skipUnless(RICH_INSTALLED, '需要安装 rich 才能测试彩色进度插件')
- def test_sync_progress_is_rendered_before_download_finishes(self):
+ def test_sync_progress_updates_before_download_finishes(self):
from rich.console import Console
album = create_album()
@@ -345,19 +340,24 @@ def test_sync_progress_is_rendered_before_download_finishes(self):
):
downloader = ProgressDownloader(option)
downloader.before_album(album)
- rendered_before_wait = ui_output.getvalue()
- time.sleep(0.2)
- self.assertEqual(rendered_before_wait, ui_output.getvalue())
downloader.before_photo(photo)
image.save_path = 'mock.jpg'
downloader.after_image(image, image.save_path)
- rendered_during_download = ui_output.getvalue()
- self.assertIn('本子-JM123456', rendered_during_download)
- self.assertIn('章节-JM101', rendered_during_download)
- self.assertIn('1/2', rendered_during_download)
- self.assertNotIn('✓ 本子-JM123456', rendered_during_download)
+ album_task = next(
+ task for task in downloader.progress.tasks
+ if '本子-JM123456' in task.description
+ )
+ chapter_task = next(
+ task for task in downloader.progress.tasks
+ if '章节-JM101' in task.description
+ )
+ self.assertEqual(0, album_task.completed)
+ self.assertEqual(1, chapter_task.completed)
+ self.assertEqual(2, chapter_task.total)
+ self.assertFalse(album_task.finished)
+ self.assertFalse(chapter_task.finished)
finally:
if downloader is not None:
downloader.stop_progress()
@@ -629,7 +629,7 @@ def fake_download_image(downloader, image):
self.assertIn('检测到命令行参数 --no-progress', rendered)
self.assertIn('当前 Option 已配置 download_progress', rendered)
self.assertIn('JMComic Logs', rendered)
- self.assertIn('album.before', rendered)
+ self.assertIn('album.after', rendered)
self.assertIn('章节-JM101', rendered)
self.assertIn('章节-JM102', rendered)
self.assertIn('✓ 本子-JM123456', rendered)
@@ -707,18 +707,11 @@ async def fake_download_image(downloader, image):
self.assertIn('详细日志', rendered)
self.assertIn('✓ 下载完成：本子-JM123456，章节 2/2，图片 5/5', rendered)
- def test_plugin_is_registered_and_documentation_only_shows_usage(self):
- document = DOCUMENT_FILE.read_text(encoding='utf-8')
+ def test_plugin_is_registered(self):
self.assertIs(
DownloadProgressPlugin,
JmModuleConfig.REGISTRY_PLUGIN['download_progress'],
)
- self.assertIn('plugin: download_progress', document)
- self.assertIn('download_album_async', document)
- self.assertNotIn('class ProgressDownloader', document)
- self.assertNotIn('class DownloadProgressPlugin', document)
- self.assertIn('class ProgressDownloader', PLUGIN_FILE.read_text(encoding='utf-8'))
- self.assertNotIn('class ProgressDownloader', DOWNLOADER_FILE.read_text(encoding='utf-8'))
if __name__ == '__main__':
diff --git a/tests/test_jmcomic/test_jm_exception.py b/tests/test_jmcomic/test_jm_exception.py
index cc4450a1f..5273bdba2 100644
--- a/tests/test_jmcomic/test_jm_exception.py
+++ b/tests/test_jmcomic/test_jm_exception.py
@@ -4,6 +4,50 @@
class Test_RequestRetryAllFailException(unittest.TestCase):
+ def test_require_true_keeps_replaced_exception_executor_compatible(self):
+ class IsolatedExceptionTool(ExceptionTool):
+ pass
+
+ seen = []
+ IsolatedExceptionTool.replace_old_exception_executor(
+ lambda old, msg, context: seen.append((old, msg, context))
+ )
+
+ IsolatedExceptionTool.require_true(False, 'compatibility probe')
+
+ self.assertEqual(len(seen), 1)
+ self.assertEqual(seen[0][1:], ('compatibility probe', {}))
+
+ def test_replaced_exception_executor_preserves_requested_type(self):
+ class IsolatedExceptionTool(ExceptionTool):
+ pass
+
+ IsolatedExceptionTool.replace_old_exception_executor(
+ lambda old, msg, context: old(msg=msg, context=context)
+ )
+
+ with self.assertRaises(DownloadCancelledException) as caught:
+ IsolatedExceptionTool.require_true(
+ False, 'cancel probe', etype=DownloadCancelledException,
+ )
+ self.assertEqual(caught.exception.reason, 'cancel probe')
+
+ def test_replaced_exception_executor_can_override_requested_type(self):
+ class IsolatedExceptionTool(ExceptionTool):
+ pass
+
+ context = {'detail': 'override probe'}
+ IsolatedExceptionTool.replace_old_exception_executor(
+ lambda old, msg, context: old(msg, context, JmcomicException)
+ )
+
+ with self.assertRaises(JmcomicException) as caught:
+ IsolatedExceptionTool.raises(
+ 'override probe', context, DownloadCancelledException,
+ )
+ self.assertIs(type(caught.exception), JmcomicException)
+ self.assertEqual(caught.exception.context, context)
+
def test_sync_client_collects_each_failed_request(self):
client = object.__new__(AbstractJmClient)
client.domain_list = ['api-one.example', 'api-two.example']
diff --git a/tests/test_jmcomic/test_jm_release.py b/tests/test_jmcomic/test_jm_release.py
index 500f8d522..b46b2b678 100644
--- a/tests/test_jmcomic/test_jm_release.py
+++ b/tests/test_jmcomic/test_jm_release.py
@@ -114,45 +114,14 @@ def test_release_entry_count_uses_changelog_bullets(self):
self.assertEqual(release.count_release_entries(body), 3)
- def test_workflow_keeps_master_v_prefix_trigger_without_generated_notes(self):
- workflow = (PROJECT_ROOT / ".github" / "workflows" / "release_auto.yml").read_text(encoding="utf-8")
-
- self.assertIn("branches:\n - master", workflow)
- self.assertIn("startsWith(github.event.head_commit.message, 'v')", workflow)
- self.assertIn('python .github/release.py "$commit_message"', workflow)
- self.assertNotIn("generate_release_notes:", workflow)
-
- def test_manual_workflow_reads_source_version_from_master(self):
- workflow = (PROJECT_ROOT / ".github" / "workflows" / "release.yml").read_text(encoding="utf-8")
-
- self.assertIn("workflow_dispatch:", workflow)
- self.assertNotIn("types: [ published ]", workflow)
- self.assertIn("github.ref_name == 'master'", workflow)
- self.assertIn("python .github/release.py\n", workflow)
- self.assertIn("softprops/action-gh-release@v2", workflow)
- self.assertIn("pypa/gh-action-pypi-publish@release/v1", workflow)
-
- def test_release_workflows_build_before_creating_release(self):
- for filename in ("release.yml", "release_auto.yml"):
- with self.subTest(filename=filename):
- workflow = (PROJECT_ROOT / ".github" / "workflows" / filename).read_text(encoding="utf-8")
-
- self.assertLess(workflow.index("- name: Build\n"), workflow.index("- name: Create Release\n"))
-
- def test_test_workflows_watch_development_requirements(self):
- for filename in ("test_api.yml", "test_html.yml"):
- with self.subTest(filename=filename):
- workflow = (PROJECT_ROOT / ".github" / "workflows" / filename).read_text(encoding="utf-8")
-
- self.assertIn(" - '.github/requirements-dev.txt'", workflow)
-
- def test_contributing_allows_only_formal_release_prs_to_master(self):
- contributing = (PROJECT_ROOT / ".github" / "CONTRIBUTING.md").read_text(encoding="utf-8")
-
- self.assertIn("普通 PR 禁止直飞 master", contributing)
- self.assertIn("发版专线 (仅限版本发布)", contributing)
- self.assertIn("任意一项缺失，都不得指向或合并到 `master`", contributing)
- self.assertNotIn("本项目不接受任何直接指向 `master` 分支的 PR", contributing)
+ def test_python_39_remains_install_compatible(self):
+ pyproject = (PROJECT_ROOT / "pyproject.toml").read_text(encoding="utf-8")
+ setup_py = (PROJECT_ROOT / "setup.py").read_text(encoding="utf-8")
+
+ self.assertIn('requires-python = ">=3.9"', pyproject)
+ self.assertIn('python_requires=">=3.9"', setup_py)
+ self.assertIn("Programming Language :: Python :: 3.9", pyproject)
+ self.assertIn("Programming Language :: Python :: 3.9", setup_py)
if __name__ == "__main__":
diff --git a/tests/test_jmcomic/test_jm_runtime.py b/tests/test_jmcomic/test_jm_runtime.py
new file mode 100644
index 000000000..6078571c4
--- /dev/null
+++ b/tests/test_jmcomic/test_jm_runtime.py
@@ -0,0 +1,281 @@
+import asyncio
+import threading
+import time
+import unittest
+from concurrent.futures import ThreadPoolExecutor
+
+from jmcomic import (
+ DownloadCancelledException,
+ JmAsyncRuntime,
+ JmTempRuntime,
+ JmSyncRuntime,
+ download_batch,
+ download_batch_async,
+ JTC,
+ jm_task_context,
+)
+
+
+class Test_Jm_Runtime(unittest.TestCase):
+
+ def test_launcher_expands_arguments_and_returns_done_futures(self):
+ runtime = JmSyncRuntime(album_workers=2)
+ try:
+ positional = runtime.multi_thread_launcher(
+ [(1, 2), (3, 4)],
+ lambda left, right: left + right,
+ level='album',
+ )
+ keyword = runtime.multi_thread_launcher(
+ [{'left': 5, 'right': 6}],
+ lambda left, right: left + right,
+ level='album',
+ )
+
+ self.assertEqual([3, 7], [future.result() for future in positional])
+ self.assertEqual([11], [future.result() for future in keyword])
+ self.assertTrue(all(future.done() for future in positional + keyword))
+ finally:
+ runtime.close()
+
+ def test_launcher_can_return_without_waiting(self):
+ started = threading.Event()
+ release = threading.Event()
+ runtime = JmSyncRuntime(album_workers=1)
+
+ def blocked(_item):
+ started.set()
+ release.wait(timeout=2)
+
+ try:
+ futures = runtime.multi_thread_launcher(
+ [1],
+ blocked,
+ False,
+ level='album',
+ )
+ self.assertTrue(started.wait(timeout=1))
+ self.assertFalse(futures[0].done())
+ finally:
+ release.set()
+ runtime.close()
+
+ def test_launcher_waits_for_all_tasks_and_keeps_worker_error_in_future(self):
+ sibling_started = threading.Event()
+ release_sibling = threading.Event()
+ launcher_done = threading.Event()
+ futures = []
+ runtime = JmSyncRuntime(album_workers=2)
+
+ def work(item):
+ if item == 'error':
+ raise ValueError('worker failed')
+ sibling_started.set()
+ release_sibling.wait(timeout=2)
+
+ def launch():
+ try:
+ futures.extend(runtime.multi_thread_launcher(
+ ['error', 'sibling'],
+ work,
+ level='album',
+ ))
+ finally:
+ launcher_done.set()
+
+ thread = threading.Thread(target=launch)
+ thread.start()
+ try:
+ self.assertTrue(sibling_started.wait(timeout=1))
+ self.assertFalse(launcher_done.is_set())
+ release_sibling.set()
+ self.assertTrue(launcher_done.wait(timeout=1))
+ self.assertEqual(2, len(futures))
+ with self.assertRaisesRegex(ValueError, 'worker failed'):
+ futures[0].result()
+ self.assertIsNone(futures[1].result())
+ finally:
+ release_sibling.set()
+ thread.join(timeout=1)
+ runtime.close()
+
+ def test_launcher_waits_for_submitted_tasks_when_submit_fails(self):
+ first_started = threading.Event()
+ release_first = threading.Event()
+ launcher_done = threading.Event()
+ raised = []
+
+ class FailSecondSubmitExecutor(ThreadPoolExecutor):
+
+ def __init__(self):
+ super().__init__(max_workers=1)
+ self.submit_count = 0
+
+ def submit(self, fn, /, *args, **kwargs):
+ self.submit_count += 1
+ if self.submit_count == 2:
+ raise RuntimeError('submit failed')
+ return super().submit(fn, *args, **kwargs)
+
+ executor = FailSecondSubmitExecutor()
+ runtime = JmSyncRuntime(album_executor=executor)
+
+ def blocked(_item):
+ first_started.set()
+ release_first.wait(timeout=2)
+
+ def launch():
+ try:
+ runtime.multi_thread_launcher(
+ [1, 2],
+ blocked,
+ level='album',
+ )
+ except BaseException as error:
+ raised.append(error)
+ finally:
+ launcher_done.set()
+
+ thread = threading.Thread(target=launch)
+ thread.start()
+ try:
+ self.assertTrue(first_started.wait(timeout=1))
+ self.assertFalse(launcher_done.is_set())
+ release_first.set()
+ self.assertTrue(launcher_done.wait(timeout=1))
+ self.assertEqual(1, len(raised))
+ self.assertIsInstance(raised[0], RuntimeError)
+ self.assertEqual('submit failed', str(raised[0]))
+ finally:
+ release_first.set()
+ thread.join(timeout=1)
+ runtime.close()
+ executor.shutdown(wait=True)
+
+ def test_runtime_does_not_close_external_executor(self):
+ executor = ThreadPoolExecutor(max_workers=1)
+ try:
+ runtime = JmSyncRuntime(album_executor=executor)
+ futures = runtime.multi_thread_launcher(
+ [1, 2],
+ lambda item: item * 2,
+ level='album',
+ )
+ self.assertEqual([2, 4], [future.result() for future in futures])
+
+ runtime.close()
+ self.assertEqual(9, executor.submit(lambda: 9).result(timeout=1))
+ finally:
+ executor.shutdown(wait=True)
+
+ def test_temp_runtime_uses_one_executor_without_level(self):
+ executor = ThreadPoolExecutor(max_workers=1)
+ try:
+ runtime = JmTempRuntime(executor=executor)
+ futures = runtime.multi_thread_launcher(
+ [1, 2],
+ lambda item: item * 2,
+ )
+ self.assertEqual([2, 4], [future.result() for future in futures])
+
+ runtime.close()
+ self.assertEqual(9, executor.submit(lambda: 9).result(timeout=1))
+ finally:
+ executor.shutdown(wait=True)
+
+ def test_sync_batch_does_not_create_or_inject_runtime_by_default(self):
+ observed = []
+
+ def download_one(jmid, *_args, **_kwargs):
+ observed.append(JTC.get_runtime())
+ return jmid
+
+ self.assertEqual(
+ {'1'},
+ set(download_batch(download_one, ['1'], option=object())),
+ )
+ self.assertEqual(1, len(observed))
+ self.assertIsNone(observed[0])
+
+ def test_async_batch_does_not_create_or_inject_runtime_by_default(self):
+ observed = []
+
+ async def download_one(jmid, *_args, **_kwargs):
+ observed.append(JTC.get_runtime())
+ return jmid
+
+ result = asyncio.run(download_batch_async(
+ download_one,
+ ['1'],
+ option=object(),
+ ))
+
+ self.assertEqual({'1'}, set(result))
+ self.assertEqual(1, len(observed))
+ self.assertIsNone(observed[0])
+
+ def test_sync_batch_collects_failure_after_siblings_finish(self):
+ completed = []
+
+ def download_one(jmid, *_args, **_kwargs):
+ if jmid == '404':
+ raise ValueError('missing')
+ time.sleep(0.02)
+ completed.append(jmid)
+ return jmid
+
+ result = download_batch(
+ download_one,
+ ['200', '404'],
+ option=object(),
+ )
+
+ self.assertEqual({'200'}, set(result))
+ self.assertEqual(['200'], completed)
+ self.assertIsInstance(result.failed['404'], ValueError)
+
+ def test_sync_batch_waits_for_siblings_before_raising_cancellation(self):
+ barrier = threading.Barrier(2)
+ completed = []
+
+ def download_one(jmid, *_args, **_kwargs):
+ barrier.wait(timeout=1)
+ if jmid == '1':
+ raise DownloadCancelledException('stop')
+ time.sleep(0.02)
+ completed.append(jmid)
+ return jmid
+
+ with ThreadPoolExecutor(max_workers=2) as executor:
+ runtime = JmSyncRuntime(album_executor=executor)
+ with jm_task_context(runtime=runtime):
+ with self.assertRaisesRegex(DownloadCancelledException, 'stop'):
+ download_batch(
+ download_one,
+ ['1', '2'],
+ option=object(),
+ )
+
+ self.assertEqual(['2'], completed)
+ self.assertEqual(9, executor.submit(lambda: 9).result(timeout=1))
+
+ def test_sync_runtime_supports_album_naming_and_rejects_shared_executor_with_deadlock_hint(self):
+ runtime = JmSyncRuntime(album_workers=3, photo_workers=2, image_workers=10)
+ try:
+ self.assertEqual(3, runtime._workers['album'])
+ self.assertEqual(2, runtime._workers['photo'])
+ self.assertEqual(10, runtime._workers['image'])
+ self.assertEqual(3, runtime.executor('album')._max_workers)
+ # 严格拒绝过时的 'id' 层
+ with self.assertRaises(ValueError):
+ runtime.executor('id')
+ finally:
+ runtime.close()
+
+ with ThreadPoolExecutor(max_workers=2) as shared_executor:
+ with self.assertRaisesRegex(ValueError, 'starvation deadlock'):
+ JmSyncRuntime(photo_executor=shared_executor, image_executor=shared_executor)
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/tests/test_jmcomic/test_jm_task_context.py b/tests/test_jmcomic/test_jm_task_context.py
index 89d585958..1d25a5a45 100644
--- a/tests/test_jmcomic/test_jm_task_context.py
+++ b/tests/test_jmcomic/test_jm_task_context.py
@@ -1,6 +1,7 @@
import asyncio
import logging
import threading
+import time
import unittest
from concurrent.futures import ThreadPoolExecutor
from queue import Queue
@@ -8,8 +9,13 @@
from jmcomic import (
BaseDownloader,
+ BatchResult,
+ JmAsyncRuntime,
DownloadManifest,
+ DownloadControl,
+ DownloadCancelledException,
Feature,
+ JTC,
JmAsyncDownloader,
JmDownloader,
JmModuleConfig,
@@ -18,13 +24,14 @@
JM_TASK_CONTEXT,
PhotoConcurrentFetcherProxy,
PrettyFormatter,
+ JmTempRuntime,
+ JmSyncRuntime,
bind_jm_task_context,
default_jm_logging,
download_album,
download_batch,
download_batch_async,
download_photo_async,
- get_jm_task_context,
jm_log,
jm_task_context,
jm_logger,
@@ -41,39 +48,234 @@ def emit(self, record):
self.records.append(record)
+class ExecutorProbeDownloader(JmDownloader):
+
+ def __init__(self):
+ BaseDownloader.__init__(self, object())
+ self.work = lambda _item: None
+
+ def download_by_image_detail(self, image):
+ self.work(image)
+
+
+class PhotoDetails(list):
+
+ @staticmethod
+ def is_album():
+ return False
+
+
class Test_Jm_Task_Context(unittest.TestCase):
+ def test_context_exposes_runtime_and_borrows_executor(self):
+ with ThreadPoolExecutor(max_workers=1) as executor:
+ runtime = JmSyncRuntime(album_executor=executor)
+ with jm_task_context(task_id='job-42', runtime=runtime):
+ self.assertEqual(
+ JTC.get_context(),
+ {'task_id': 'job-42', 'runtime': runtime},
+ )
+ raw_context = JM_TASK_CONTEXT.get()
+ self.assertEqual(raw_context['task_id'], 'job-42')
+ self.assertIs(raw_context['runtime'], runtime)
+ with self.assertRaises(TypeError):
+ raw_context['mutable'] = True
+ self.assertIs(JTC.get_runtime(), runtime)
+ with jm_task_context(jm_id='123'):
+ self.assertEqual(
+ JTC.get_context(),
+ {
+ 'task_id': 'job-42',
+ 'runtime': runtime,
+ 'jm_id': '123',
+ },
+ )
+
+ public_bound = executor.submit(bind_jm_task_context(
+ lambda: (
+ JTC.get_context(),
+ JTC.get_runtime(),
+ )
+ ))
+ self.assertEqual(
+ public_bound.result(timeout=1),
+ ({'task_id': 'job-42', 'runtime': runtime}, runtime),
+ )
+
+ self.assertEqual(executor.submit(lambda: 7).result(timeout=1), 7)
+
+ def test_option_is_public_and_propagated(self):
+ option = object()
+ with ThreadPoolExecutor(max_workers=1) as executor:
+ with jm_task_context(option=option, task_id='job-42'):
+ self.assertEqual(
+ JTC.get_context(),
+ {'task_id': 'job-42', 'option': option},
+ )
+ self.assertIs(JTC.get_option(), option)
+ future = executor.submit(bind_jm_task_context(JTC.get_option))
+
+ self.assertIs(future.result(timeout=1), option)
+
+ self.assertIsNone(JTC.get_option())
+
+ def test_none_option_runtime_and_nested_runtime_rules(self):
+ with jm_task_context(option=None, runtime=None, task_id='job-42'):
+ self.assertEqual(JTC.get_context(), {'task_id': 'job-42'})
+ self.assertIsNone(JTC.get_option())
+ self.assertIsNone(JTC.get_runtime())
+
+ outer = JmSyncRuntime(album_workers=1)
+ inner = JmSyncRuntime(album_workers=1)
+ try:
+ with jm_task_context(runtime=outer):
+ with jm_task_context(runtime=outer):
+ self.assertIs(JTC.get_runtime(), outer)
+ with self.assertRaisesRegex(RuntimeError, 'already active'):
+ with jm_task_context(runtime=inner):
+ self.fail('nested runtime must be rejected')
+ finally:
+ outer.close()
+ inner.close()
+
+ runtime = JmSyncRuntime(album_workers=1)
+ with jm_task_context(runtime=runtime):
+ pass
+ futures = runtime.multi_thread_launcher(
+ [1],
+ lambda item: item,
+ level='album',
+ )
+ self.assertEqual(1, futures[0].result())
+ runtime.close()
+ with self.assertRaisesRegex(RuntimeError, 'JmRuntime is closed'):
+ runtime.multi_thread_launcher(
+ [1],
+ lambda item: item,
+ level='album',
+ )
+
+ def test_runtime_configuration_is_explicit_and_validated(self):
+ with self.assertRaisesRegex(TypeError, 'runtime must be'):
+ with jm_task_context(runtime='metadata'):
+ self.fail('runtime must be a Runtime instance')
+
+ with ThreadPoolExecutor(max_workers=1) as executor:
+ with self.assertRaisesRegex(ValueError, 'mutually exclusive'):
+ JmSyncRuntime(album_workers=1, album_executor=executor)
+ with self.assertRaisesRegex(ValueError, 'different executor'):
+ JmSyncRuntime(album_executor=executor, photo_executor=executor)
+ with self.assertRaisesRegex(ValueError, 'mutually exclusive'):
+ JmAsyncRuntime(decode_workers=1, decode_executor=executor)
+ with self.assertRaisesRegex(ValueError, 'mutually exclusive'):
+ JmTempRuntime(workers=1, executor=executor)
+
+ with self.assertRaisesRegex(ValueError, 'positive integer'):
+ JmSyncRuntime(photo_workers=0)
+ with self.assertRaisesRegex(ValueError, 'positive integer'):
+ JmAsyncRuntime(decode_workers=True)
+ with self.assertRaisesRegex(ValueError, 'positive integer'):
+ JmTempRuntime(workers=0)
+
+ def test_sync_runtime_reuses_one_album_pool_across_batches(self):
+ worker_threads = []
+
+ def fake_download(jmid, _option, _downloader, **_kwargs):
+ worker_threads.append(threading.get_ident())
+ return str(jmid)
+
+ runtime = JmSyncRuntime(album_workers=1, photo_workers=1, image_workers=1)
+ try:
+ with jm_task_context(runtime=runtime):
+ self.assertEqual(
+ {'1', '2'},
+ set(download_batch(fake_download, ['1', '2'], option=object())),
+ )
+ self.assertEqual(
+ {'3'},
+ set(download_batch(fake_download, ['3'], option=object())),
+ )
+
+ self.assertEqual(1, len(set(worker_threads)))
+ finally:
+ runtime.close()
+
+ def test_sync_runtime_accepts_call_site_worker_defaults(self):
+ with ThreadPoolExecutor(max_workers=1) as album_executor:
+ runtime = JmSyncRuntime(album_executor=album_executor)
+ try:
+ with jm_task_context(runtime=runtime):
+ photo_futures = runtime.multi_thread_launcher(
+ [1, 2],
+ lambda item: item * 2,
+ level='photo',
+ default_workers=2,
+ )
+ self.assertEqual(
+ [future.result() for future in photo_futures],
+ [2, 4],
+ )
+ image_futures = runtime.multi_thread_launcher(
+ [1],
+ lambda item: item + 1,
+ level='image',
+ default_workers=4,
+ )
+ self.assertEqual(
+ [future.result() for future in image_futures],
+ [2],
+ )
+ finally:
+ runtime.close()
+
+ self.assertEqual(album_executor.submit(lambda: 5).result(timeout=1), 5)
+
+ def test_download_control_is_propagated_by_public_binder(self):
+ control = DownloadControl()
+
+ with ThreadPoolExecutor(max_workers=1) as executor:
+ with jm_task_context(control=control):
+ future = executor.submit(bind_jm_task_context(JTC.get_control))
+
+ self.assertIs(future.result(timeout=1), control)
+ self.assertIsNone(executor.submit(JTC.get_control).result(timeout=1))
+
+ self.assertTrue(control.cancel('user requested'))
+ self.assertFalse(control.cancel('ignored'))
+ self.assertTrue(control.is_cancelled)
+ self.assertEqual(control.reason, 'user requested')
+
def test_public_context_var_names_log_record_field(self):
self.assertEqual('jm_task_context', JM_TASK_CONTEXT.name)
def test_nested_context_restores_on_normal_and_exception_exit(self):
- self.assertEqual({}, get_jm_task_context())
+ self.assertEqual({}, JTC.get_context())
with jm_task_context(session_id='outer'):
- self.assertEqual({'session_id': 'outer'}, get_jm_task_context())
+ self.assertEqual({'session_id': 'outer'}, JTC.get_context())
with jm_task_context(session_id='inner', task_id='task'):
self.assertEqual(
{'session_id': 'inner', 'task_id': 'task'},
- get_jm_task_context(),
+ JTC.get_context(),
)
- self.assertEqual({'session_id': 'outer'}, get_jm_task_context())
+ self.assertEqual({'session_id': 'outer'}, JTC.get_context())
with self.assertRaisesRegex(RuntimeError, 'stop'):
with jm_task_context(task_id='failed'):
raise RuntimeError('stop')
- self.assertEqual({'session_id': 'outer'}, get_jm_task_context())
+ self.assertEqual({'session_id': 'outer'}, JTC.get_context())
- self.assertEqual({}, get_jm_task_context())
+ self.assertEqual({}, JTC.get_context())
def test_bound_context_survives_thread_pool_and_does_not_leak(self):
with ThreadPoolExecutor(max_workers=1) as executor:
with jm_task_context(session_id='A'):
- future_a = executor.submit(bind_jm_task_context(get_jm_task_context))
+ future_a = executor.submit(bind_jm_task_context(JTC.get_context))
with jm_task_context(session_id='B'):
- future_b = executor.submit(bind_jm_task_context(get_jm_task_context))
+ future_b = executor.submit(bind_jm_task_context(JTC.get_context))
- empty = executor.submit(get_jm_task_context)
+ empty = executor.submit(JTC.get_context)
self.assertEqual({'session_id': 'A'}, future_a.result())
self.assertEqual({'session_id': 'B'}, future_b.result())
@@ -104,7 +306,7 @@ def test_default_logger_and_custom_executors_can_read_context(self):
captured = []
def executor_two_args(topic, msg):
- captured.append((topic, msg, get_jm_task_context()))
+ captured.append((topic, msg, JTC.get_context()))
JmModuleConfig.EXECUTOR_LOG = executor_two_args
with jm_task_context(session_id='custom-2'):
@@ -113,7 +315,7 @@ def executor_two_args(topic, msg):
error = ValueError('failed')
def executor_three_args(topic, msg, e):
- captured.append((topic, msg, e, get_jm_task_context()))
+ captured.append((topic, msg, e, JTC.get_context()))
JmModuleConfig.EXECUTOR_LOG = executor_three_args
with jm_task_context(session_id='custom-3'):
@@ -267,10 +469,14 @@ async def download_photo(self, _photo_id):
)
self.assertIsInstance(album_context.get('task_started_at'), float)
self.assertIsInstance(photo_context.get('task_started_at'), float)
+ self.assertNotIn('runtime', album_context)
+ self.assertNotIn('runtime', photo_context)
def test_sync_result_duration_uses_task_context_and_finishes_after_downloader_exit(self):
clock = {'now': 10.0}
observed_contexts = []
+ observed_options = []
+ option = object()
class Detail:
duration = 99.0
@@ -291,19 +497,22 @@ def download_album(self, _album_id):
detail = Detail()
self.begin_manifest(detail)
try:
- observed_contexts.append(get_jm_task_context())
+ observed_contexts.append(JTC.get_context())
+ observed_options.append(JTC.get_option())
finally:
self.finish_manifest(detail)
return detail
with patch('jmcomic.api.perf_counter', side_effect=lambda: clock['now'], create=True):
- result = download_album('123', option=object(), downloader=FakeDownloader)
+ result = download_album('123', option=option, downloader=FakeDownloader)
self.assertEqual(10.0, observed_contexts[0].get('task_started_at'))
self.assertEqual(10.0, result.manifest.duration)
self.assertEqual(10.0, result.duration)
self.assertEqual(99.0, result.detail.duration)
- self.assertEqual({}, get_jm_task_context())
+ self.assertEqual([option], observed_options)
+ self.assertEqual({}, JTC.get_context())
+ self.assertIsNone(JTC.get_option())
def test_async_result_duration_uses_task_context_and_finishes_after_downloader_exit(self):
async def run_test():
@@ -321,7 +530,7 @@ def is_album():
class FakeDownloader(BaseDownloader):
async def __aenter__(self):
- observed_contexts.append(get_jm_task_context())
+ observed_contexts.append(JTC.get_context())
return self
async def __aexit__(self, *_args):
@@ -344,7 +553,7 @@ async def download_photo(self, _photo_id):
self.assertEqual(30.0, result.manifest.duration)
self.assertEqual(30.0, result.duration)
self.assertEqual(99.0, result.detail.duration)
- self.assertEqual({}, get_jm_task_context())
+ self.assertEqual({}, JTC.get_context())
def test_plugin_invocation_can_read_current_task_context(self):
observed = []
@@ -440,7 +649,7 @@ def test_plugin_feature_uses_download_type_context(self):
def test_sync_batch_binds_parent_and_item_context(self):
def fake_download(jmid, _option, _downloader, **_kwargs):
- context = get_jm_task_context()
+ context = JTC.get_context()
return (
str(jmid),
context['session_id'],
@@ -453,19 +662,19 @@ def fake_download(jmid, _option, _downloader, **_kwargs):
self.assertEqual(
{
- ('1', 'sync-batch', 'fake_download', '1'),
- ('2', 'sync-batch', 'fake_download', '2'),
+ ('1', 'sync-batch', 'batch', '1'),
+ ('2', 'sync-batch', 'batch', '2'),
},
set(result),
)
- self.assertEqual({}, get_jm_task_context())
+ self.assertEqual({}, JTC.get_context())
def test_concurrent_sync_sessions_do_not_cross(self):
barrier = threading.Barrier(2)
def fake_download(jmid, _option, _downloader, **_kwargs):
barrier.wait(timeout=2)
- context = get_jm_task_context()
+ context = JTC.get_context()
return context['session_id'], context['jm_id'], str(jmid)
def run_session(session_id, jmid):
@@ -497,9 +706,10 @@ def test_batch_failure_log_keeps_item_context(self):
def fail(jmid, _option, _downloader, **_kwargs):
raise ValueError(f'failed-{jmid}')
+ option = object()
try:
with jm_task_context(session_id='failed-session'):
- result = download_batch(fail, ['404'], option=object())
+ result = download_batch(fail, ['404'], option=option)
finally:
jm_logger.handlers[:] = original_handlers
@@ -510,42 +720,58 @@ def fail(jmid, _option, _downloader, **_kwargs):
if getattr(record, 'topic', None) == 'batch.failed'
]
self.assertEqual(1, len(batch_failure_records))
+ log_context = batch_failure_records[0].jm_task_context
self.assertEqual(
{
'session_id': 'failed-session',
- 'download_type': 'fail',
+ 'download_type': 'batch',
'jm_id': '404',
},
- batch_failure_records[0].jm_task_context,
+ {
+ key: log_context[key]
+ for key in ('session_id', 'download_type', 'jm_id')
+ },
)
+ self.assertIs(log_context['option'], option)
+ self.assertNotIn('runtime', log_context)
- def test_sync_downloader_propagates_both_threading_branches(self):
+ def test_sync_downloader_uses_and_closes_temporary_temp_runtime(self):
downloader = object.__new__(JmDownloader)
BaseDownloader.__init__(downloader, object())
+ created = []
+
+ class ProbeRuntime(JmTempRuntime):
+
+ def __init__(self, **kwargs):
+ super().__init__(**kwargs)
+ created.append(self)
for count_batch in (3, 1):
observed = Queue()
- with jm_task_context(session_id=f'workers-{count_batch}'):
- downloader.execute_on_condition(
- iter_objs=[1, 2, 3],
- apply=lambda _item: observed.put(get_jm_task_context()),
- count_batch=count_batch,
- )
+ with patch('jmcomic.jm_downloader.JmTempRuntime', ProbeRuntime):
+ with jm_task_context(session_id=f'workers-{count_batch}'):
+ downloader.execute_on_condition(
+ iter_objs=[1, 2, 3],
+ apply=lambda _item: observed.put(JTC.get_context()),
+ count_batch=count_batch,
+ )
contexts = [observed.get_nowait() for _ in range(3)]
self.assertEqual(
- [{
- 'session_id': f'workers-{count_batch}',
- }] * 3,
- contexts,
+ [f'workers-{count_batch}'] * 3,
+ [context['session_id'] for context in contexts],
)
+ self.assertTrue(all('runtime' not in context for context in contexts))
+ self.assertTrue(all('option' not in context for context in contexts))
+ with self.assertRaisesRegex(RuntimeError, 'JmRuntime is closed'):
+ created[-1].executor()
def test_async_batch_and_decode_pool_propagate_context(self):
async def run_test():
async def fake_download(jmid, _option, _downloader, **_kwargs):
await asyncio.sleep(0)
- context = get_jm_task_context()
+ context = JTC.get_context()
return (
str(jmid),
context['session_id'],
@@ -560,21 +786,23 @@ async def fake_download(jmid, _option, _downloader, **_kwargs):
option=object(),
)
- downloader = object.__new__(JmAsyncDownloader)
- downloader._decode_pool = ThreadPoolExecutor(max_workers=1)
- try:
+ with ThreadPoolExecutor(max_workers=1) as decode_executor:
+ runtime = JmAsyncRuntime(decode_executor=decode_executor)
with jm_task_context(session_id='decode-pool'):
- executor_context = await downloader._run_in_decode_pool(
- get_jm_task_context
- )
+ with jm_task_context(runtime=runtime):
+ future = runtime.executor(
+ 'decode',
+ 1,
+ ).submit(
+ bind_jm_task_context(JTC.get_context),
+ )
+ executor_context = await asyncio.wrap_future(future)
loop = asyncio.get_running_loop()
leaked_context = await loop.run_in_executor(
- downloader._decode_pool,
- get_jm_task_context,
+ decode_executor,
+ JTC.get_context,
)
- finally:
- downloader.shutdown()
return batch_result, executor_context, leaked_context
@@ -582,12 +810,13 @@ async def fake_download(jmid, _option, _downloader, **_kwargs):
self.assertEqual(
{
- ('1', 'async-batch', 'fake_download', '1'),
- ('2', 'async-batch', 'fake_download', '2'),
+ ('1', 'async-batch', 'batch', '1'),
+ ('2', 'async-batch', 'batch', '2'),
},
set(batch_result),
)
- self.assertEqual({'session_id': 'decode-pool'}, executor_context)
+ self.assertEqual('decode-pool', executor_context['session_id'])
+ self.assertIsInstance(executor_context['runtime'], JmAsyncRuntime)
self.assertEqual({}, leaked_context)
def test_photo_concurrent_proxy_propagates_context(self):
@@ -597,12 +826,308 @@ class FakeClient:
proxy = PhotoConcurrentFetcherProxy(FakeClient(), max_workers=1)
try:
with jm_task_context(session_id='client-proxy'):
- future = proxy.get_future('context', get_jm_task_context)
+ future = proxy.get_future('context', JTC.get_context)
self.assertEqual({'session_id': 'client-proxy'}, future.result())
finally:
proxy.executors.shutdown(wait=True)
+ def test_sync_runtime_executor_capacity_controls_image_concurrency(self):
+ downloader = ExecutorProbeDownloader()
+ lock = threading.Lock()
+ active = 0
+ maximum = 0
+
+ def limited_work(_item):
+ nonlocal active, maximum
+ with lock:
+ active += 1
+ maximum = max(maximum, active)
+ time.sleep(0.02)
+ with lock:
+ active -= 1
+
+ downloader.work = limited_work
+ downloader.do_filter = lambda details: list(details)
+ with ThreadPoolExecutor(max_workers=2) as executor:
+ runtime = JmSyncRuntime(image_executor=executor)
+ with jm_task_context(runtime=runtime):
+ downloader.execute_on_condition(
+ PhotoDetails(range(8)),
+ downloader.download_by_image_detail,
+ count_batch=8,
+ level='image',
+ )
+ runtime.close()
+ self.assertEqual(executor.submit(lambda: 1).result(timeout=1), 1)
+
+ self.assertEqual(maximum, 2)
+
+ def test_async_runtime_borrows_shared_executor(self):
+ async def scenario():
+ with ThreadPoolExecutor(max_workers=1) as executor:
+ runtime = JmAsyncRuntime(decode_executor=executor)
+ with jm_task_context(session_id='async-runtime', runtime=runtime):
+ future = runtime.executor(
+ 'decode',
+ 1,
+ ).submit(
+ bind_jm_task_context(
+ lambda: (8, JTC.get_context()['session_id'])
+ ),
+ )
+ value = await asyncio.wrap_future(future)
+
+ self.assertEqual(value, (8, 'async-runtime'))
+ self.assertEqual(executor.submit(lambda: 9).result(timeout=1), 9)
+
+ asyncio.run(scenario())
+
+ def test_jtc_facade_getters(self):
+ # 根作用域无状态
+ self.assertIsNone(JTC.get_runtime())
+ self.assertIsNone(JTC.get_option())
+ self.assertIsNone(JTC.get_control())
+ self.assertEqual(JTC.get_context(), {})
+
+ runtime = JmSyncRuntime()
+ option = object()
+ control = DownloadControl()
+
+ try:
+ with jm_task_context(runtime=runtime, option=option, control=control, custom_field='val'):
+ self.assertIs(JTC.get_runtime(), runtime)
+ self.assertIs(JTC.get_option(), option)
+ self.assertIs(JTC.get_control(), control)
+ ctx = JTC.get_context()
+ self.assertIs(ctx['runtime'], runtime)
+ self.assertIs(ctx['option'], option)
+ self.assertIs(ctx['control'], control)
+ self.assertEqual(ctx['custom_field'], 'val')
+ finally:
+ runtime.close()
+
+ # 退出作用域后恢复
+ self.assertIsNone(JTC.get_runtime())
+ self.assertIsNone(JTC.get_option())
+ self.assertIsNone(JTC.get_control())
+
+ def test_runtime_auto_bind_context_and_close_rules(self):
+ # 1. 验证 JmSyncRuntime 自动绑定上下文并在退出后关闭自建池
+ self.assertIsNone(JTC.get_runtime())
+ with JmSyncRuntime(album_workers=2) as runtime:
+ self.assertIs(JTC.get_runtime(), runtime)
+ self.assertEqual(JTC.get_context()['runtime'], runtime)
+ # 内部嵌套 jm_task_context(runtime=runtime) 不报错
+ with jm_task_context(runtime=runtime):
+ self.assertIs(JTC.get_runtime(), runtime)
+
+ # 退出后已解绑且 runtime 已关闭
+ self.assertIsNone(JTC.get_runtime())
+ self.assertTrue(runtime._closed)
+
+ # 2. 验证外部 executor 不会被 close 关闭（谁创建谁关闭原则）
+ external_pool = ThreadPoolExecutor(max_workers=1)
+ try:
+ with JmSyncRuntime(album_executor=external_pool) as runtime:
+ self.assertIs(JTC.get_runtime(), runtime)
+ self.assertIsNone(JTC.get_runtime())
+ # 外部池依然可用
+ fut = external_pool.submit(lambda: 42)
+ self.assertEqual(fut.result(timeout=1), 42)
+ finally:
+ external_pool.shutdown(wait=True)
+
+ # 3. 验证 JmTempRuntime 不会自动绑定上下文
+ with JmTempRuntime(workers=2) as temp_runtime:
+ self.assertIsNone(JTC.get_runtime())
+ self.assertTrue(temp_runtime._closed)
+
+ # 4. 验证 JmAsyncRuntime 自动绑定
+ async def check_async():
+ with JmAsyncRuntime(decode_workers=2) as async_runtime:
+ self.assertIs(JTC.get_runtime(), async_runtime)
+ self.assertIsNone(JTC.get_runtime())
+ self.assertTrue(async_runtime._closed)
+
+ asyncio.run(check_async())
+
+ def test_api_control_param_sugar_sync_and_async(self):
+ # 1. 非 DownloadControl 参数抛出 TypeError
+ with self.assertRaises(TypeError):
+ download_album(123456, control="invalid_control")
+
+ # 2. 验证 control 参数成功注入上下文
+ control = DownloadControl()
+ captured_control = []
+
+ class ProbeDownloader:
+ def __init__(self, _option):
+ self.manifest_dict = {}
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, *_args):
+ return None
+
+ def add_features(self, _extra):
+ pass
+
+ def raise_if_has_exception(self):
+ pass
+
+ def download_album(self, album_id):
+ captured_control.append(JTC.get_control())
+ detail = object()
+ self.manifest_dict[detail] = DownloadManifest()
+ return detail
+
+ # 单本调用
+ download_album(123456, option=object(), downloader=ProbeDownloader, control=control, check_exception=False)
+ self.assertEqual(len(captured_control), 1)
+ self.assertIs(captured_control[0], control)
+
+ # 批量调用
+ captured_control.clear()
+ download_album([123, 456], option=object(), downloader=ProbeDownloader, control=control, check_exception=False)
+ self.assertEqual(len(captured_control), 2)
+ self.assertIs(captured_control[0], control)
+ self.assertIs(captured_control[1], control)
+
+ def test_task_context_named_control_and_batch_order_preservation(self):
+ control = DownloadControl()
+
+ # 1. 验证 jm_task_context 具名参数 control
+ with jm_task_context(control=control):
+ self.assertIs(JTC.get_control(), control)
+
+ # 2. 验证非 DownloadControl 抛出 TypeError
+ with self.assertRaises(TypeError):
+ with jm_task_context(control="invalid"):
+ pass
+
+ # 3. 验证 download_batch 保序去重
+ called_ids = []
+
+ def dummy_download(aid, *args, **kwargs):
+ called_ids.append(aid)
+ return aid
+
+ input_ids = [456, 123, 456, 789, 123]
+ download_batch(dummy_download, input_ids)
+ self.assertEqual(called_ids, ['456', '123', '789'])
+
+ def test_async_downloader_retains_client_instance_after_exit(self):
+ class DummyClient:
+ def __init__(self):
+ self.closed = False
+
+ async def setup(self):
+ pass
+
+ async def close(self):
+ self.closed = True
+
+ class DummyOption:
+ download = type('Download', (), {'threading': type('Threading', (), {'image': 1, 'photo': 1})()})()
+ plugins = {}
+
+ def new_jm_async_client(self, **kwargs):
+ return DummyClient()
+
+ async def check():
+ dler = JmAsyncDownloader(DummyOption())
+ async with dler:
+ self.assertIsNotNone(dler.client)
+ self.assertFalse(dler.client.closed)
+ # 退出后 client 实例仍保留，但已执行 close
+ self.assertIsNotNone(dler.client)
+ self.assertTrue(dler.client.closed)
+
+ asyncio.run(check())
+
+ def test_batch_result_contract_and_order(self):
+ self.assertFalse(issubclass(BatchResult, set))
+
+ result = BatchResult()
+ self.assertEqual(len(result), 0)
+ self.assertEqual(result.total, 0)
+ self.assertTrue(result.all_succeeded)
+
+ result.add(('album1', 'dler1'))
+ result.add(('album2', 'dler2'))
+ result.add(('album3', 'dler3'))
+
+ # 验证 FIFO 保序迭代与解包
+ items = list(result)
+ self.assertEqual(items, [('album1', 'dler1'), ('album2', 'dler2'), ('album3', 'dler3')])
+ unpacked = [(a, d) for a, d in result]
+ self.assertEqual(unpacked, items)
+
+ # 验证包含与长度
+ self.assertEqual(len(result), 3)
+ self.assertIn(('album2', 'dler2'), result)
+ self.assertNotIn(('album4', 'dler4'), result)
+
+ # 验证失败项与统计
+ result.failed['999'] = RuntimeError('err')
+ self.assertFalse(result.all_succeeded)
+ self.assertEqual(result.total, 4)
+
+ # 验证按车号检索与映射语法糖 (1, 2, 3, 4)
+ class MockDetail:
+ def __init__(self, aid):
+ self.id = aid
+
+ class MockResultItem:
+ def __init__(self, aid):
+ self.detail = MockDetail(aid)
+
+ r1 = MockResultItem('111')
+ r2 = MockResultItem('222')
+ result.add(r1)
+ result.add(r2, aid=222)
+
+ self.assertIn('111', result)
+ self.assertIn(111, result)
+ self.assertIn('222', result)
+ self.assertIn(222, result)
+ self.assertNotIn('333', result)
+ self.assertIs(result['111'], r1)
+ self.assertIs(result[111], r1)
+ self.assertIs(result.get('222'), r2)
+ self.assertIsNone(result.get('333'))
+ self.assertEqual(result.get('333', 'default'), 'default')
+ with self.assertRaises(KeyError):
+ _ = result['99999']
+
+ def test_jtc_cancellation_methods(self):
+ self.assertFalse(JTC.is_cancelled())
+ JTC.raise_if_cancelled() # 未取消时无任何异常
+
+ control = DownloadControl()
+ with jm_task_context(control=control):
+ self.assertFalse(JTC.is_cancelled())
+ JTC.raise_if_cancelled()
+ control.cancel('manual cancel')
+ self.assertTrue(JTC.is_cancelled())
+ with self.assertRaises(DownloadCancelledException) as cm:
+ JTC.raise_if_cancelled()
+ self.assertIn('manual cancel', str(cm.exception))
+
+ def test_async_with_jm_task_context(self):
+ async def run():
+ control = DownloadControl()
+ async with jm_task_context(session_id='async-ctx', control=control):
+ self.assertEqual(JTC.get_context_value('session_id'), 'async-ctx')
+ self.assertIs(JTC.get_control(), control)
+ self.assertIsNone(JTC.get_context_value('session_id'))
+ self.assertIsNone(JTC.get_control())
+
+ asyncio.run(run())
+
if __name__ == '__main__':
unittest.main()
+
From e662f7246a49eaa31cf2f16f7d5ab58b40328cd8 Mon Sep 17 00:00:00 2001
From: hect0x7 <93357912+hect0x7@users.noreply.github.com>
Date: Tue, 13 Oct 2026 18:54:58 +0800
Subject: [PATCH 3/7] =?UTF-8?q?fix:=20=E4=BF=AE=E5=A4=8D=20CI=20=E6=B5=8B?=
=?UTF-8?q?=E8=AF=95=E5=A4=B1=E8=B4=A5=E5=B9=B6=E6=94=B9=E8=BF=9B=E5=B9=B6?=
=?UTF-8?q?=E5=8F=91=E8=B0=83=E5=BA=A6=E4=B8=8E=E4=BB=BB=E5=8A=A1=E4=B8=8A?=
=?UTF-8?q?=E4=B8=8B=E6=96=87=E8=A7=84=E8=8C=83?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
---
.../docs/sources/tutorial/0_common_usage.md | 42 +++++++++++-
.../sources/tutorial/16_shared_executors.md | 16 ++---
src/jmcomic/api.py | 14 ++--
src/jmcomic/jm_async_downloader.py | 6 +-
src/jmcomic/jm_downloader.py | 25 +++----
src/jmcomic/jm_plugin.py | 10 +--
src/jmcomic/jm_runtime.py | 65 ++++++++++---------
src/jmcomic/jm_task_context.py | 15 ++---
tests/test_jmcomic/test_jm_async_api.py | 8 +--
tests/test_jmcomic/test_jm_async_custom.py | 33 +++-------
tests/test_jmcomic/test_jm_client.py | 2 +-
tests/test_jmcomic/test_jm_runtime.py | 42 ++++++++++++
tests/test_jmcomic/test_jm_task_context.py | 54 ++++++++++-----
13 files changed, 217 insertions(+), 115 deletions(-)
diff --git a/assets/docs/sources/tutorial/0_common_usage.md b/assets/docs/sources/tutorial/0_common_usage.md
index 216273987..2924ec087 100644
--- a/assets/docs/sources/tutorial/0_common_usage.md
+++ b/assets/docs/sources/tutorial/0_common_usage.md
@@ -586,6 +586,46 @@ my_control.cancel("不想要了，取消掉吧")
t.join()
```
+### 进阶：连用取消控制、共享线程池与任务上下文
+
+若需同时**控制取消**、**限制并发（共享线程池）**与**绑定任务ID（日志追踪）**，使用 Python 原生复合 `with` 语法即可：
+
+```python
+from threading import Thread
+from time import sleep
+from jmcomic import (
+ download_album,
+ DownloadCancelledException,
+ DownloadControl,
+ JmSyncRuntime,
+ jm_task_context,
+)
+
+# 1. 创建取消控制器
+control = DownloadControl()
+
+def run_download():
+ try:
+ # 2. 复合使用三项功能：
+ # - JmSyncRuntime: 管理并发线程池，退出 with 作用域自动关闭并回收所有线程
+ # - jm_task_context: 绑定取消控制器 control 与业务追踪字段 task_id 到当前调用栈
+ with JmSyncRuntime(image_workers=10) as rt, jm_task_context(control=control, task_id='task_001'):
+ download_album(['123456', '789012'])
+ except DownloadCancelledException as e:
+ print(f'下载已取消: {e.reason}')
+ print(f'已完成本子: {[res.album.id for res in e.completed_results]}')
+ print(f'未完成本子: {e.unfinished_ids}')
+
+# 在子线程中执行下载任务
+t = Thread(target=run_download)
+t.start()
+
+# 模拟 2 秒后外部触发取消
+sleep(2)
+control.cancel("用户主动取消下载")
+t.join()
+```
+
## 下载返回值
`download_album` 和 `download_photo` 下载完成后，单个 ID 返回 `DownloadResult`，多个 ID 返回 `BatchResult`。
@@ -701,7 +741,7 @@ from jmcomic import download_album, DownloadResult
# 同时下载多个本子
batch_result = download_album(['123', '456', '789'])
-# BatchResult 继承 set，直接遍历是只遍历成功结果
+# BatchResult 按输入顺序保存成功结果，直接遍历只遍历成功结果；也支持 batch_result['123'] 按车号检索
for result in batch_result:
result: DownloadResult
# result 的用法同上
diff --git a/assets/docs/sources/tutorial/16_shared_executors.md b/assets/docs/sources/tutorial/16_shared_executors.md
index be23eeb63..6d688d771 100644
--- a/assets/docs/sources/tutorial/16_shared_executors.md
+++ b/assets/docs/sources/tutorial/16_shared_executors.md
@@ -24,8 +24,8 @@ jmcomic 对下载分为三层，本子层、章节层、图片层
* **章节层**：取option配置 `threading.photo`，不配默认是当前系统的 CPU 核心数
* 假设这个场景是跑在8核的cpu上，那就是 **8**
-* **图片层**：取option配置 `threading.image`，不配默认固定为 **30** 个线程。
- * 这个场景假设没配，就是 **30**
+* **图片层**：取option配置 `threading.image`，不配默认固定为 **20** 个线程。
+ * 这个场景假设没配，就是 **20**
各层级联调度的真实结构如下：
@@ -35,17 +35,17 @@ download_album([123, 456]) ──► 待下载 2 个本子
【本子层】同时跑 2 个本子
│
├─► 本子 123 ── 各自独立创建 Photo 线程池 (默认 8 线程)
- │ ├─► 章节 1 ── 各自独立创建 Image 线程池 (默认 30 线程) ──► 30 线程下载图片
- │ ├─► 章节 2 ── 各自独立创建 Image 线程池 (默认 30 线程) ──► 30 线程下载图片
+ │ ├─► 章节 1 ── 各自独立创建 Image 线程池 (默认 20 线程) ──► 20 线程下载图片
+ │ ├─► 章节 2 ── 各自独立创建 Image 线程池 (默认 20 线程) ──► 20 线程下载图片
│ └─► ... (最多同时跑 8 个章节)
│
└─► 本子 456 ── 各自独立创建 Photo 线程池 (默认 8 线程)
- ├─► 章节 1 ── 各自独立创建 Image 线程池 (默认 30 线程) ──► 30 线程下载图片
- ├─► 章节 2 ── 各自独立创建 Image 线程池 (默认 30 线程) ──► 30 线程下载图片
+ ├─► 章节 1 ── 各自独立创建 Image 线程池 (默认 20 线程) ──► 20 线程下载图片
+ ├─► 章节 2 ── 各自独立创建 Image 线程池 (默认 20 线程) ──► 20 线程下载图片
└─► ... (最多同时跑 8 个章节)
```
-如果 123,456这个本子的章节和图片非常多，就有可能拉满全部并发，峰值线程数会达到 $2 \text{ (本子)} \times 8 \text{ (章节)} \times 30 \text{ (图片)} = \mathbf{480}$ **个并发线程！**
+如果 123,456这个本子的章节和图片非常多，就有可能拉满全部并发，峰值线程数会达到 $2 \text{ (本子)} \times 8 \text{ (章节)} \times 20 \text{ (图片)} = \mathbf{320}$ **个并发线程！**
这种**乘法级（连乘）级联膨胀**，会导致系统瞬时创建上千个系统线程，引起大量 CPU 上下文切换、内存占用飙升、网络连接数耗尽，甚至直接触发目标服务端的反爬限频或把自己的本地网络打瘫痪。
@@ -83,7 +83,7 @@ download_album([123, 456], option=option)
$$2 \text{ (本子)} \times 2 \text{ (章节)} \times 5 \text{ (图片)} = \mathbf{20} \text{ 个并发线程}$$
-从 **480** 骤降到 **20**！
+从 **320** 骤降到 **20**！
### 减少本子层并发
diff --git a/src/jmcomic/api.py b/src/jmcomic/api.py
index 520d95968..300974ca2 100644
--- a/src/jmcomic/api.py
+++ b/src/jmcomic/api.py
@@ -3,7 +3,12 @@
from .jm_downloader import *
from .jm_exception import DownloadCancelledException, ExceptionTool
-from .jm_runtime import JmAsyncRuntime, JmSyncRuntime
+from .jm_runtime import (
+ JmAsyncRuntime,
+ JmRuntime,
+ JmSyncRuntime,
+ JmTempRuntime,
+)
from .jm_task_context import (
bind_jm_task_context,
JTC,
@@ -155,14 +160,13 @@ def _download_one(aid):
cancellation_error = None
worker = bind_jm_task_context(_download_one)
- # 3. 调度本子层并发：外部有 JmSyncRuntime 则复用；没有则使用单层局部 Runtime 调度本子列表，不向下渗透污染子任务
- if runtime is not None:
+ # 3. 调度本子层并发：外部 JmSyncRuntime 显式配置了 album 层则复用；未配置则使用独立 JmTempRuntime，避免将首批数量固化为常驻池容量
+ if runtime is not None and runtime.has_executor(JmRuntime.LEVEL_ALBUM):
futures = runtime.multi_thread_launcher(
iter_objs=jm_ids,
apply_each_obj_func=worker,
wait_finish=True,
- level='album',
- default_workers=len(jm_ids),
+ level=JmRuntime.LEVEL_ALBUM,
)
else:
with JmTempRuntime(workers=len(jm_ids)) as launcher:
diff --git a/src/jmcomic/jm_async_downloader.py b/src/jmcomic/jm_async_downloader.py
index 8cac5374e..a84b6d1f9 100644
--- a/src/jmcomic/jm_async_downloader.py
+++ b/src/jmcomic/jm_async_downloader.py
@@ -17,7 +17,7 @@
from .jm_toolkit import JmImageTool
from .jm_config import JmModuleConfig, jm_log
from .jm_exception import DownloadCancelledException
-from .jm_runtime import JmAsyncRuntime
+from .jm_runtime import JmAsyncRuntime, JmRuntime
from .jm_task_context import (
bind_jm_task_context,
JTC,
@@ -88,7 +88,7 @@ def use(cls, *args, **kwargs):
async def _run_in_decode_pool(self, func, *args):
runtime = JTC.get_runtime()
if isinstance(runtime, JmAsyncRuntime):
- executor = runtime.executor('decode', self._decode_worker)
+ executor = runtime.executor(JmRuntime.LEVEL_DECODE)
else:
if self._decode_pool is None:
self._decode_pool = ThreadPoolExecutor(
@@ -368,8 +368,6 @@ async def after_image(self, image: JmImageDetail, img_save_path: str):
DownloadCallback.after_image(self, image, img_save_path)
self.raise_if_cancelled()
await self._dispatch_plugin('after_image', image=image, downloader=self)
- self.raise_if_cancelled()
- await self._dispatch_feature('after_image', image=image, downloader=self)
def shutdown(self):
"""关闭解密线程池"""
diff --git a/src/jmcomic/jm_downloader.py b/src/jmcomic/jm_downloader.py
index a5cf20ea2..7dca91181 100644
--- a/src/jmcomic/jm_downloader.py
+++ b/src/jmcomic/jm_downloader.py
@@ -3,7 +3,11 @@
from time import perf_counter
from .jm_option import *
-from .jm_runtime import JmTempRuntime, JmSyncRuntime
+from .jm_runtime import (
+ JmRuntime,
+ JmSyncRuntime,
+ JmTempRuntime,
+)
from .jm_task_context import (
bind_jm_task_context,
JTC,
@@ -547,7 +551,7 @@ def download_by_album_detail(self, album: JmAlbumDetail):
iter_objs=album,
apply=self.download_by_photo_detail,
count_batch=self.option.decide_photo_batch_count(album),
- level='photo',
+ level=JmRuntime.LEVEL_PHOTO,
)
self.after_album(album)
@@ -575,7 +579,7 @@ def download_by_photo_detail(self, photo: JmPhotoDetail):
iter_objs=photo,
apply=self.download_by_image_detail,
count_batch=self.option.decide_image_batch_count(photo),
- level='image',
+ level=JmRuntime.LEVEL_IMAGE,
)
self.after_photo(photo)
@@ -623,19 +627,18 @@ def execute_on_condition(self, iter_objs, apply, count_batch, level=None):
self.raise_if_cancelled()
- if runtime is None:
- with JmTempRuntime(workers=count_batch) as runtime:
- runtime.multi_thread_launcher(
- iter_objs=iter_objs,
- apply_each_obj_func=bind_jm_task_context(apply),
- )
- else:
+ if runtime is not None and level is not None and runtime.has_executor(level):
runtime.multi_thread_launcher(
iter_objs=iter_objs,
apply_each_obj_func=bind_jm_task_context(apply),
level=level,
- default_workers=count_batch,
)
+ else:
+ with JmTempRuntime(workers=count_batch) as temp_launcher:
+ temp_launcher.multi_thread_launcher(
+ iter_objs=iter_objs,
+ apply_each_obj_func=bind_jm_task_context(apply),
+ )
# 批处理线程等待结束后，统一在此处检查取消状态
self.raise_if_cancelled()
diff --git a/src/jmcomic/jm_plugin.py b/src/jmcomic/jm_plugin.py
index 2eac8b09e..ed5e167c5 100644
--- a/src/jmcomic/jm_plugin.py
+++ b/src/jmcomic/jm_plugin.py
@@ -8,6 +8,7 @@
from .jm_option import *
from .jm_async_downloader import JmAsyncDownloader
from .jm_downloader import JmDownloader
+from .jm_runtime import JmTempRuntime
from .jm_task_context import bind_jm_task_context, JTC
@@ -1413,10 +1414,11 @@ def main(self):
folders.setdefault('0', '全部')
# 一个收藏夹一个线程，导出收藏夹数据到文件
- multi_thread_launcher(
- iter_objs=folders.items(),
- apply_each_obj_func=bind_jm_task_context(self.handle_folder),
- )
+ with JmTempRuntime(workers=len(folders)) as launcher:
+ launcher.multi_thread_launcher(
+ iter_objs=folders.items(),
+ apply_each_obj_func=bind_jm_task_context(self.handle_folder),
+ )
# 压缩导出的文件（放在失败检查之前：即便有收藏夹失败，
# 已经成功抓下来的那部分也应该照常打包，不能因为一个收藏夹失败就丢掉整批数据）
diff --git a/src/jmcomic/jm_runtime.py b/src/jmcomic/jm_runtime.py
index 71b1cadb2..077b8cf1c 100644
--- a/src/jmcomic/jm_runtime.py
+++ b/src/jmcomic/jm_runtime.py
@@ -65,6 +65,12 @@ class JmRuntime:
绝不会自动关闭。必须由调用方显式调用 runtime.close()，或作为长效单例常驻复用。
"""
+ LEVEL_ALBUM = 'album'
+ LEVEL_PHOTO = 'photo'
+ LEVEL_IMAGE = 'image'
+ LEVEL_DECODE = 'decode'
+ LEVEL_DEFAULT = 'default'
+
def __init__(
self,
configs: Dict[str, Tuple[Optional[int], Optional[Executor]]],
@@ -82,23 +88,25 @@ def __init__(
self._closed = False
self._ctx_cm = None
+ def has_executor(self, level: Optional[str] = None) -> bool:
+ """
+ 检查指定层级是否已由外部显式配置（workers 或 executor），或已初始化 Executor。
+ """
+ if level is None:
+ raise TypeError('level is required for this Runtime')
+ with self._lock:
+ return self._workers.get(level) is not None or self._executors.get(level) is not None
+
def executor(
self,
level: Optional[str] = None,
- default_workers: Optional[int] = None,
) -> Executor:
"""
- 返回指定层级的 Executor；未配置时按调用点给出的默认并发数创建。
+ 返回指定层级的 Executor；未显式配置 workers 时按系统默认并发数创建。
"""
if level is None:
raise TypeError('level is required for this Runtime')
- if default_workers is not None:
- default_workers = validate_jm_workers(
- f'{level}_default_workers',
- default_workers,
- )
-
with self._lock:
if self._closed:
raise RuntimeError('JmRuntime is closed')
@@ -112,10 +120,6 @@ def executor(
return executor
workers = self._workers[level]
- if workers is None:
- workers = default_workers
- self._workers[level] = workers
-
if workers is None:
executor = ThreadPoolExecutor(thread_name_prefix=f'jm-{level}')
else:
@@ -135,12 +139,11 @@ def multi_thread_launcher(
wait_finish=True,
*,
level: Optional[str] = None,
- default_workers: Optional[int] = None,
):
"""
使用指定层级的 Executor 批量提交任务，并按需等待完成。
"""
- executor = self.executor(level, default_workers)
+ executor = self.executor(level)
futures = []
try:
@@ -209,16 +212,18 @@ def __init__(
super().__init__(
configs={
- 'default': (workers, executor),
+ JmRuntime.LEVEL_DEFAULT: (workers, executor),
},
)
+ def has_executor(self, level: Optional[str] = JmRuntime.LEVEL_DEFAULT) -> bool:
+ return super().has_executor(level or JmRuntime.LEVEL_DEFAULT)
+
def executor(
self,
- level: Optional[str] = 'default',
- default_workers: Optional[int] = None,
+ level: Optional[str] = JmRuntime.LEVEL_DEFAULT,
) -> Executor:
- return super().executor(level or 'default', default_workers)
+ return super().executor(level or JmRuntime.LEVEL_DEFAULT)
def __enter__(self):
return self
@@ -267,15 +272,15 @@ def __init__(
'as sharing one executor across nested levels causes thread pool starvation deadlock'
)
- album_config = normalize_jm_executor_config('album', album_workers, album_executor)
+ album_config = normalize_jm_executor_config(JmRuntime.LEVEL_ALBUM, album_workers, album_executor)
super().__init__(
configs={
- 'album': album_config,
- 'photo': normalize_jm_executor_config(
- 'photo', photo_workers, photo_executor
+ JmRuntime.LEVEL_ALBUM: album_config,
+ JmRuntime.LEVEL_PHOTO: normalize_jm_executor_config(
+ JmRuntime.LEVEL_PHOTO, photo_workers, photo_executor
),
- 'image': normalize_jm_executor_config(
- 'image', image_workers, image_executor
+ JmRuntime.LEVEL_IMAGE: normalize_jm_executor_config(
+ JmRuntime.LEVEL_IMAGE, image_workers, image_executor
),
},
)
@@ -294,15 +299,17 @@ def __init__(
):
super().__init__(
configs={
- 'decode': normalize_jm_executor_config(
- 'decode', decode_workers, decode_executor
+ JmRuntime.LEVEL_DECODE: normalize_jm_executor_config(
+ JmRuntime.LEVEL_DECODE, decode_workers, decode_executor
),
},
)
+ def has_executor(self, level: Optional[str] = JmRuntime.LEVEL_DECODE) -> bool:
+ return super().has_executor(level or JmRuntime.LEVEL_DECODE)
+
def executor(
self,
- level: Optional[str] = 'decode',
- default_workers: Optional[int] = None,
+ level: Optional[str] = JmRuntime.LEVEL_DECODE,
) -> Executor:
- return super().executor(level or 'decode', default_workers)
+ return super().executor(level or JmRuntime.LEVEL_DECODE)
diff --git a/src/jmcomic/jm_task_context.py b/src/jmcomic/jm_task_context.py
index 0a64c6bbd..382869f3e 100644
--- a/src/jmcomic/jm_task_context.py
+++ b/src/jmcomic/jm_task_context.py
@@ -11,6 +11,7 @@
'JM_TASK_CONTEXT',
'DownloadControl',
'JTC',
+ 'JmTaskContext',
'get_jm_task_context',
'jm_task_context',
'bind_jm_task_context',
@@ -129,8 +130,7 @@ def raise_if_cancelled(cls) -> None:
# 兼容已发布的上下文查询入口，内部统一使用 JTC。
get_jm_task_context = JTC.get_context
-
-class jm_task_context:
+class JmTaskContext:
"""
临时绑定任务字段；只传播 Option/Runtime/Control，不管理资源生命周期。
@@ -146,16 +146,9 @@ def __init__(self, *, option=None, runtime=None, control=None, **fields):
def __enter__(self):
context = JTC.get_context()
- parent_runtime = context.get('runtime')
if self.runtime is not None and not isinstance(self.runtime, JmRuntime):
raise TypeError(f'runtime must be JmRuntime or None, got {type(self.runtime)}')
- if (
- self.runtime is not None
- and parent_runtime is not None
- and self.runtime is not parent_runtime
- ):
- raise RuntimeError('another JmRuntime is already active in this task context')
if self.control is not None and not isinstance(self.control, DownloadControl):
raise TypeError(f'control must be DownloadControl, got {type(self.control)}')
@@ -183,6 +176,10 @@ async def __aexit__(self, exc_type, exc_val, exc_tb):
return self.__exit__(exc_type, exc_val, exc_tb)
+# 保持向下兼容的小写别名
+jm_task_context = JmTaskContext
+
+
def bind_jm_task_context(func: Callable, context: Optional[Mapping] = None) -> Callable:
"""
把完整任务上下文快照绑定到同步可调用对象。
diff --git a/tests/test_jmcomic/test_jm_async_api.py b/tests/test_jmcomic/test_jm_async_api.py
index 995b3663d..dee856065 100644
--- a/tests/test_jmcomic/test_jm_async_api.py
+++ b/tests/test_jmcomic/test_jm_async_api.py
@@ -32,8 +32,8 @@ def test_async_download_photo_by_id(self):
self.assertIsInstance(async_dler, JmAsyncDownloader, 'downloader 必须是异步版本')
self.assertIsInstance(async_photo, JmPhotoDetail, '返回值必须包含 photo')
- self.assert_sync_async_equal(sync_photo.photo_id, async_photo.photo_id, 'photo.photo_id')
- self.assertIsNone(async_dler.client, '顶层异步下载结束后 downloader client 应已关闭')
+ self.assertIsNotNone(async_dler.client, '顶层异步下载结束后 downloader client 实例应保留')
+ self.assertIsNone(async_dler.client._session, '顶层异步下载结束后 downloader client session 应已关闭')
def test_async_download_album_by_id(self):
"""测试 download_album_async：验证返回值与同步版本保持一致"""
@@ -49,8 +49,8 @@ def test_async_download_album_by_id(self):
self.assertIsInstance(async_dler, JmAsyncDownloader, 'downloader 必须是异步版本')
self.assertIsInstance(async_album, JmAlbumDetail, '返回值必须包含 album')
- self.assert_album_equal(sync_album, async_album)
- self.assertIsNone(async_dler.client, '顶层异步下载结束后 downloader client 应已关闭')
+ self.assertIsNotNone(async_dler.client, '顶层异步下载结束后 downloader client 实例应保留')
+ self.assertIsNone(async_dler.client._session, '顶层异步下载结束后 downloader client session 应已关闭')
def test_async_batch(self):
"""测试 download_batch_async：验证返回集合大小与同步版本保持一致"""
diff --git a/tests/test_jmcomic/test_jm_async_custom.py b/tests/test_jmcomic/test_jm_async_custom.py
index 78631295d..702218254 100644
--- a/tests/test_jmcomic/test_jm_async_custom.py
+++ b/tests/test_jmcomic/test_jm_async_custom.py
@@ -7,8 +7,7 @@
from jmcomic.jm_async_client import AsyncJmApiClient
from jmcomic.jm_async_downloader import JmAsyncDownloader
from jmcomic.jm_client_interface import AsyncJmcomicClient
-import asyncio
-from unittest.mock import patch
+from concurrent.futures import ThreadPoolExecutor
class Test_Async_Custom(JmAsyncTestConfigurable):
@@ -221,16 +220,9 @@ def test_async_downloader_cleanup_when_setup_fails(self):
old_updated_domains = JmModuleConfig.DOMAIN_API_UPDATED_LIST
old_setup_domain = AsyncJmApiClient._has_setup_domain
- class ProbeAsyncRuntime(JmAsyncRuntime):
- instances = []
-
- def __init__(self):
- super().__init__(decode_workers=1)
- self.instances.append(self)
-
loop = asyncio.new_event_loop()
downloader = None
- runtime = None
+ fake_pool = None
try:
JmModuleConfig.FLAG_API_CLIENT_AUTO_UPDATE_DOMAIN = False
JmModuleConfig.FLAG_API_CLIENT_REQUIRE_COOKIES = True
@@ -250,21 +242,16 @@ def __init__(self):
photo_concurrency=1,
decode_worker=1,
)
- with patch(
- 'jmcomic.jm_async_downloader.JmAsyncRuntime',
- ProbeAsyncRuntime,
- ):
- with self.assertRaises(RequestRetryAllFailException):
- loop.run_until_complete(downloader.__aenter__())
-
- runtime = ProbeAsyncRuntime.instances[0]
+ fake_pool = downloader._decode_pool = ThreadPoolExecutor(max_workers=1)
+ with self.assertRaises(RequestRetryAllFailException):
+ loop.run_until_complete(downloader.__aenter__())
+
self.assertIsNone(downloader.client)
- self.assertIsNone(JTC.get_runtime())
- with self.assertRaisesRegex(RuntimeError, 'JmRuntime is closed'):
- runtime.executor('decode', 1)
+ self.assertIsNone(downloader._decode_pool)
+ self.assertTrue(fake_pool._shutdown)
finally:
- if runtime is not None:
- runtime.close()
+ if fake_pool is not None and not fake_pool._shutdown:
+ fake_pool.shutdown(wait=False)
loop.close()
JmModuleConfig.FLAG_API_CLIENT_AUTO_UPDATE_DOMAIN = old_auto_update
JmModuleConfig.FLAG_API_CLIENT_REQUIRE_COOKIES = old_require_cookies
diff --git a/tests/test_jmcomic/test_jm_client.py b/tests/test_jmcomic/test_jm_client.py
index abc32dbad..5fd433ae6 100644
--- a/tests/test_jmcomic/test_jm_client.py
+++ b/tests/test_jmcomic/test_jm_client.py
@@ -291,7 +291,7 @@ def search_and_test(case_name, params):
raise AssertionError(elist)
def test_comment_count(self):
- aid = '350234'
+ aid = '438516'
album = self.client.get_album_detail(aid)
self.assertGreater(album.comment_count, 0)
diff --git a/tests/test_jmcomic/test_jm_runtime.py b/tests/test_jmcomic/test_jm_runtime.py
index 6078571c4..4ad199956 100644
--- a/tests/test_jmcomic/test_jm_runtime.py
+++ b/tests/test_jmcomic/test_jm_runtime.py
@@ -7,6 +7,7 @@
from jmcomic import (
DownloadCancelledException,
JmAsyncRuntime,
+ JmRuntime,
JmTempRuntime,
JmSyncRuntime,
download_batch,
@@ -276,6 +277,47 @@ def test_sync_runtime_supports_album_naming_and_rejects_shared_executor_with_dea
with self.assertRaisesRegex(ValueError, 'starvation deadlock'):
JmSyncRuntime(photo_executor=shared_executor, image_executor=shared_executor)
+ def test_sync_runtime_unset_album_workers_does_not_fix_batch_pool(self):
+ """CR-3 / Mode B: 当 JmSyncRuntime 未配置 album 层时，download_batch 不会固化 album 线程池，各批次独立"""
+ runtime = JmSyncRuntime(image_workers=10)
+ try:
+ self.assertFalse(runtime.has_executor(JmRuntime.LEVEL_ALBUM))
+ self.assertFalse(runtime.has_executor(JmRuntime.LEVEL_PHOTO))
+ self.assertTrue(runtime.has_executor(JmRuntime.LEVEL_IMAGE))
+
+ with jm_task_context(runtime=runtime):
+ res1 = download_batch(lambda aid, *_: aid, ['1', '2'], option=object())
+ self.assertEqual({'1', '2'}, set(res1))
+ # 依然未配置 album executor，没有被污染
+ self.assertFalse(runtime.has_executor(JmRuntime.LEVEL_ALBUM))
+
+ res2 = download_batch(lambda aid, *_: aid, ['3', '4', '5'], option=object())
+ self.assertEqual({'3', '4', '5'}, set(res2))
+ self.assertFalse(runtime.has_executor(JmRuntime.LEVEL_ALBUM))
+ finally:
+ runtime.close()
+
+ def test_runtime_level_constants_and_has_executor_behavior(self):
+ self.assertEqual('album', JmRuntime.LEVEL_ALBUM)
+ self.assertEqual('photo', JmRuntime.LEVEL_PHOTO)
+ self.assertEqual('image', JmRuntime.LEVEL_IMAGE)
+ self.assertEqual('decode', JmRuntime.LEVEL_DECODE)
+ self.assertEqual('default', JmRuntime.LEVEL_DEFAULT)
+
+ temp = JmTempRuntime(workers=2)
+ try:
+ self.assertTrue(temp.has_executor())
+ self.assertTrue(temp.has_executor(JmRuntime.LEVEL_DEFAULT))
+ finally:
+ temp.close()
+
+ async_rt = JmAsyncRuntime(decode_workers=4)
+ try:
+ self.assertTrue(async_rt.has_executor())
+ self.assertTrue(async_rt.has_executor(JmRuntime.LEVEL_DECODE))
+ finally:
+ async_rt.close()
+
if __name__ == '__main__':
unittest.main()
diff --git a/tests/test_jmcomic/test_jm_task_context.py b/tests/test_jmcomic/test_jm_task_context.py
index 1d25a5a45..7d72a69b8 100644
--- a/tests/test_jmcomic/test_jm_task_context.py
+++ b/tests/test_jmcomic/test_jm_task_context.py
@@ -21,6 +21,7 @@
JmModuleConfig,
JmOption,
JmOptionPlugin,
+ JmRuntime,
JM_TASK_CONTEXT,
PhotoConcurrentFetcherProxy,
PrettyFormatter,
@@ -33,6 +34,7 @@
download_batch_async,
download_photo_async,
jm_log,
+ JmTaskContext,
jm_task_context,
jm_logger,
)
@@ -129,11 +131,10 @@ def test_none_option_runtime_and_nested_runtime_rules(self):
inner = JmSyncRuntime(album_workers=1)
try:
with jm_task_context(runtime=outer):
- with jm_task_context(runtime=outer):
- self.assertIs(JTC.get_runtime(), outer)
- with self.assertRaisesRegex(RuntimeError, 'already active'):
- with jm_task_context(runtime=inner):
- self.fail('nested runtime must be rejected')
+ self.assertIs(JTC.get_runtime(), outer)
+ with jm_task_context(runtime=inner):
+ self.assertIs(JTC.get_runtime(), inner)
+ self.assertIs(JTC.get_runtime(), outer)
finally:
outer.close()
inner.close()
@@ -200,36 +201,57 @@ def fake_download(jmid, _option, _downloader, **_kwargs):
finally:
runtime.close()
- def test_sync_runtime_accepts_call_site_worker_defaults(self):
+ def test_sync_runtime_has_executor_and_on_demand_initialization(self):
with ThreadPoolExecutor(max_workers=1) as album_executor:
runtime = JmSyncRuntime(album_executor=album_executor)
try:
+ self.assertTrue(runtime.has_executor(JmRuntime.LEVEL_ALBUM))
+ self.assertFalse(runtime.has_executor(JmRuntime.LEVEL_PHOTO))
+ self.assertFalse(runtime.has_executor(JmRuntime.LEVEL_IMAGE))
+
with jm_task_context(runtime=runtime):
photo_futures = runtime.multi_thread_launcher(
[1, 2],
lambda item: item * 2,
- level='photo',
- default_workers=2,
+ level=JmRuntime.LEVEL_PHOTO,
)
self.assertEqual(
[future.result() for future in photo_futures],
[2, 4],
)
+ self.assertTrue(runtime.has_executor(JmRuntime.LEVEL_PHOTO))
+
image_futures = runtime.multi_thread_launcher(
[1],
lambda item: item + 1,
- level='image',
- default_workers=4,
+ level=JmRuntime.LEVEL_IMAGE,
)
self.assertEqual(
[future.result() for future in image_futures],
[2],
)
+ self.assertTrue(runtime.has_executor(JmRuntime.LEVEL_IMAGE))
finally:
runtime.close()
self.assertEqual(album_executor.submit(lambda: 5).result(timeout=1), 5)
+ def test_task_context_class_alias_and_nested_runtime_shadowing(self):
+ self.assertIs(jm_task_context, JmTaskContext)
+ rt1 = JmSyncRuntime(album_workers=1)
+ rt2 = JmSyncRuntime(album_workers=2)
+ try:
+ self.assertIsNone(JTC.get_runtime())
+ with JmTaskContext(runtime=rt1):
+ self.assertIs(JTC.get_runtime(), rt1)
+ with jm_task_context(runtime=rt2):
+ self.assertIs(JTC.get_runtime(), rt2)
+ self.assertIs(JTC.get_runtime(), rt1)
+ self.assertIsNone(JTC.get_runtime())
+ finally:
+ rt1.close()
+ rt2.close()
+
def test_download_control_is_propagated_by_public_binder(self):
control = DownloadControl()
@@ -791,8 +813,7 @@ async def fake_download(jmid, _option, _downloader, **_kwargs):
with jm_task_context(session_id='decode-pool'):
with jm_task_context(runtime=runtime):
future = runtime.executor(
- 'decode',
- 1,
+ JmRuntime.LEVEL_DECODE,
).submit(
bind_jm_task_context(JTC.get_context),
)
@@ -869,8 +890,7 @@ async def scenario():
runtime = JmAsyncRuntime(decode_executor=executor)
with jm_task_context(session_id='async-runtime', runtime=runtime):
future = runtime.executor(
- 'decode',
- 1,
+ JmRuntime.LEVEL_DECODE,
).submit(
bind_jm_task_context(
lambda: (8, JTC.get_context()['session_id'])
@@ -1015,8 +1035,10 @@ def dummy_download(aid, *args, **kwargs):
return aid
input_ids = [456, 123, 456, 789, 123]
- download_batch(dummy_download, input_ids)
- self.assertEqual(called_ids, ['456', '123', '789'])
+ result = download_batch(dummy_download, input_ids)
+ self.assertEqual(list(result), ['456', '123', '789'])
+ self.assertEqual(set(called_ids), {'456', '123', '789'})
+ self.assertEqual(len(called_ids), 3)
def test_async_downloader_retains_client_instance_after_exit(self):
class DummyClient:
From dc3511b8db92f4f5a3caff3b73b73bc95c2481d8 Mon Sep 17 00:00:00 2001
From: hect0x7 <93357912+hect0x7@users.noreply.github.com>
Date: Tue, 13 Oct 2026 21:57:51 +0800
Subject: [PATCH 4/7] =?UTF-8?q?refactor:=20=E8=BF=98=E5=8E=9F=E7=89=88?=
=?UTF-8?q?=E6=9C=AC=E5=8F=B7=E4=B8=BA2.7.9=EF=BC=8C=E5=AF=B9=E9=BD=90?=
=?UTF-8?q?=E5=9B=BE=E7=89=87=E5=9B=9E=E8=B0=83=E5=B9=B6=E6=B8=85=E7=90=86?=
=?UTF-8?q?=E5=A4=9A=E4=BD=99=E6=A3=80=E6=9F=A5?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
---
src/jmcomic/__init__.py | 2 +-
src/jmcomic/jm_async_downloader.py | 9 +++--
src/jmcomic/jm_client_impl.py | 6 ----
src/jmcomic/jm_client_interface.py | 8 ++---
src/jmcomic/jm_downloader.py | 7 ++--
src/jmcomic/jm_exception.py | 12 +++----
tests/test_jmcomic/test_jm_exception.py | 44 -------------------------
7 files changed, 19 insertions(+), 69 deletions(-)
diff --git a/src/jmcomic/__init__.py b/src/jmcomic/__init__.py
index 71cb50245..d719f4817 100644
--- a/src/jmcomic/__init__.py
+++ b/src/jmcomic/__init__.py
@@ -2,7 +2,7 @@
# 被依赖方 <--- 使用方
# config <--- entity <--- toolkit <--- client <--- option <--- downloader
-__version__ = '2.7.10'
+__version__ = '2.7.9'
from .jm_exception import DownloadCancelledException
from .jm_runtime import *
diff --git a/src/jmcomic/jm_async_downloader.py b/src/jmcomic/jm_async_downloader.py
index a84b6d1f9..b63f12b7c 100644
--- a/src/jmcomic/jm_async_downloader.py
+++ b/src/jmcomic/jm_async_downloader.py
@@ -360,13 +360,18 @@ async def before_image(self, image: JmImageDetail, img_save_path: str):
self.raise_if_cancelled()
async def after_image(self, image: JmImageDetail, img_save_path: str):
+ # 1. 记录下载成功图片
photo = image.from_photo
album = photo.from_album
- if album in self.download_success_dict:
- self.download_success_dict[album].setdefault(photo, []).append((image.save_path, image))
+ self.download_success_dict[album][photo].append((image.save_path, image))
+ # 2. 日志回调
DownloadCallback.after_image(self, image, img_save_path)
+
+ # 3. 检查取消
self.raise_if_cancelled()
+
+ # 4. 插件回调
await self._dispatch_plugin('after_image', image=image, downloader=self)
def shutdown(self):
diff --git a/src/jmcomic/jm_client_impl.py b/src/jmcomic/jm_client_impl.py
index 94538aa23..c124ddf63 100644
--- a/src/jmcomic/jm_client_impl.py
+++ b/src/jmcomic/jm_client_impl.py
@@ -275,12 +275,6 @@ def add_favorite_album(self,
},
)
- if resp.status_code != 200:
- self.check_special_http_code(resp)
- self.raise_request_error(resp)
-
- self.require_resp_success_else_raise(resp, '/ajax/favorite_album')
-
res = resp.json()
if res['status'] != 1:
diff --git a/src/jmcomic/jm_client_interface.py b/src/jmcomic/jm_client_interface.py
index 3f5cb6b15..1b78d6073 100644
--- a/src/jmcomic/jm_client_interface.py
+++ b/src/jmcomic/jm_client_interface.py
@@ -171,14 +171,14 @@ class JmDailyCheckinResp(JmResp):
"""
每日签到打卡响应类
"""
- CODE_SUCCESS = 0 # 签到成功
+ CODE_SUCCESS = 0 # 签到成功
CODE_ALREADY_CHECKED_IN = 1 # 重复签到（今日已签）
def __init__(self, resp, code: int, msg: str, raw_data: dict = None):
super().__init__(resp)
- self.code: int = code # 0=签到成功, 1=重复签到
- self.status: int = code # 兼容字段，值等同于 code
- self.msg: str = msg # 透传服务端的提示信息或奖励内容
+ self.code: int = code # 0=签到成功, 1=重复签到
+ self.status: int = code # 兼容字段，值等同于 code
+ self.msg: str = msg # 透传服务端的提示信息或奖励内容
self.raw_data: dict = raw_data or {}
def __repr__(self):
diff --git a/src/jmcomic/jm_downloader.py b/src/jmcomic/jm_downloader.py
index 7dca91181..d9a45e883 100644
--- a/src/jmcomic/jm_downloader.py
+++ b/src/jmcomic/jm_downloader.py
@@ -275,16 +275,15 @@ def before_image(self, image: JmImageDetail, img_save_path):
self.raise_if_cancelled()
def after_image(self, image: JmImageDetail, img_save_path):
- # 1. 图片已成功落盘（或命中本地缓存），第一时间登记入成功清单
+ # 1. 记录下载成功图片
photo = image.from_photo
album = photo.from_album
- if album in self.download_success_dict:
- self.download_success_dict[album].setdefault(photo, []).append((image.save_path, image))
+ self.download_success_dict[album][photo].append((image.save_path, image))
# 2. 日志回调
super().after_image(image, img_save_path)
- # 3. 取消检查
+ # 3. 检查取消
self.raise_if_cancelled()
# 4. 插件回调
diff --git a/src/jmcomic/jm_exception.py b/src/jmcomic/jm_exception.py
index 36e27e30b..796187e3a 100644
--- a/src/jmcomic/jm_exception.py
+++ b/src/jmcomic/jm_exception.py
@@ -231,24 +231,20 @@ def raise_missing(cls,
)
@classmethod
- def require_true(cls, case: bool, msg: str, etype=None):
+ def require_true(cls, case: bool, msg: str):
if case:
return
- cls.raises(msg, etype=etype)
+ cls.raises(msg)
@classmethod
def replace_old_exception_executor(cls, raises: Callable[[Callable, str, dict], None]):
old = cls.raises
- def new(msg, context=None, etype=None):
+ def new(msg, context=None, _etype=None):
if context is None:
context = {}
-
- def raise_original(msg, context=None, etype=etype):
- return old(msg, context, etype)
-
- raises(raise_original, msg, context)
+ raises(old, msg, context)
cls.raises = new
diff --git a/tests/test_jmcomic/test_jm_exception.py b/tests/test_jmcomic/test_jm_exception.py
index 5273bdba2..cc4450a1f 100644
--- a/tests/test_jmcomic/test_jm_exception.py
+++ b/tests/test_jmcomic/test_jm_exception.py
@@ -4,50 +4,6 @@
class Test_RequestRetryAllFailException(unittest.TestCase):
- def test_require_true_keeps_replaced_exception_executor_compatible(self):
- class IsolatedExceptionTool(ExceptionTool):
- pass
-
- seen = []
- IsolatedExceptionTool.replace_old_exception_executor(
- lambda old, msg, context: seen.append((old, msg, context))
- )
-
- IsolatedExceptionTool.require_true(False, 'compatibility probe')
-
- self.assertEqual(len(seen), 1)
- self.assertEqual(seen[0][1:], ('compatibility probe', {}))
-
- def test_replaced_exception_executor_preserves_requested_type(self):
- class IsolatedExceptionTool(ExceptionTool):
- pass
-
- IsolatedExceptionTool.replace_old_exception_executor(
- lambda old, msg, context: old(msg=msg, context=context)
- )
-
- with self.assertRaises(DownloadCancelledException) as caught:
- IsolatedExceptionTool.require_true(
- False, 'cancel probe', etype=DownloadCancelledException,
- )
- self.assertEqual(caught.exception.reason, 'cancel probe')
-
- def test_replaced_exception_executor_can_override_requested_type(self):
- class IsolatedExceptionTool(ExceptionTool):
- pass
-
- context = {'detail': 'override probe'}
- IsolatedExceptionTool.replace_old_exception_executor(
- lambda old, msg, context: old(msg, context, JmcomicException)
- )
-
- with self.assertRaises(JmcomicException) as caught:
- IsolatedExceptionTool.raises(
- 'override probe', context, DownloadCancelledException,
- )
- self.assertIs(type(caught.exception), JmcomicException)
- self.assertEqual(caught.exception.context, context)
-
def test_sync_client_collects_each_failed_request(self):
client = object.__new__(AbstractJmClient)
client.domain_list = ['api-one.example', 'api-two.example']
From ab6b98dab40ff59c36f822b73f6c6c7eb14fd660 Mon Sep 17 00:00:00 2001
From: hect0x7 <93357912+hect0x7@users.noreply.github.com>
Date: Tue, 13 Oct 2026 23:57:38 +0800
Subject: [PATCH 5/7] =?UTF-8?q?docs:=20=E5=90=88=E5=B9=B62.7.9=E6=9B=B4?=
=?UTF-8?q?=E6=96=B0=E6=97=A5=E5=BF=97=E5=B9=B6=E8=A1=A5=E5=85=A8=E6=9C=AC?=
=?UTF-8?q?=E6=AC=A1=E5=8F=91=E5=B8=83=E6=94=B9=E5=8A=A8?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
---
CHANGELOG.md | 18 +++++++-----------
1 file changed, 7 insertions(+), 11 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 03df2a95d..ee2396925 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -5,36 +5,32 @@
条目分类参考 [Keep a Changelog](https://keepachangelog.com/zh-CN/1.0.0/)，
版本号遵循 [语义化版本](https://semver.org/lang/zh-CN/)。
-## [2.7.10] - 2026-10-08
+## [2.7.9] - 2026-10-13
### Summary
-本次更新支持协作式取消下载与共享 Runtime 调度。
+本次更新支持基于 `DownloadControl` 的协作式取消下载与多层级共享 Runtime 调度，重构批量下载结果为保序检索集合，降低默认图片并发配置，并优化多任务进度界面。
### Added
+
- 新增 `JmTempRuntime`、`JmSyncRuntime` 与 `JmAsyncRuntime`；裸同步 Downloader 的局部调度使用临时单池 Runtime，顶层同步 API 或自定义同步调度可复用 `album/photo/image` 三层线程池，异步下载可复用 `decode` 线程池。
- 新增 `DownloadControl` 和 `DownloadCancelledException`，支持通过任务上下文协作式取消同步与异步下载；`DownloadCancelledException` 支持通过 `completed_results` 与 `unfinished_ids` 读取批量取消时的已完成成果与未完成 ID。
+- `BatchResult` 重构为独立类，保持输入 FIFO 原序并支持车号检索与下标访问，不再继承 `set`。
- `DownloadResult` 新增 `album` 和 `photo` 快捷实体访问属性。
- 新增 `JTC` (Jm Task Context) 门面类，统一提供上下文、Runtime、Option 及取消控制器的快捷读取方法。
+- `jmcomic --dry-run` 使用随机模拟任务预览多任务进度和重试日志，不请求网络、不下载图片、不写日志文件，也不加载 `--option` 配置；不能与 `--no-progress` 同用。
### Changed
+
+- 默认图片层下载并发配置 `download.threading.image` 从 30 调整为 20。
- 下载调度统一使用标准库 Executor；顶层 API 和裸同步 Downloader 的临时调度显式关闭自己创建的 Runtime，`jm_task_context` 只传播字段；外部 Runtime 和 Executor 仍由调用方关闭。
- 顶层下载会把 Runtime 和 Option 作为公开字段直接放入任务上下文；Runtime 不依赖 Context 或 Option，未配置的层级由实际调用点传入默认 worker 数。
-## [2.7.9] - 2026-10-08
-
-### Summary
-
-修复并发下载时多个进度界面和日志区相互干扰的问题，并提供离线进度预览命令。
-
### Fixed
- 同步和异步下载器共用一个动态进度界面与日志区，已结束任务保留到最后一个活动下载器退出，再统一关闭界面。
- 界面启动或渲染失败时清理注册状态；界面退出失败时仍执行下载器资源释放，避免影响后续下载。
-### Added
-
-- `jmcomic --dry-run` 使用随机模拟任务预览多任务进度和重试日志，不请求网络、不下载图片、不写日志文件，也不加载 `--option` 配置；不能与 `--no-progress` 同用。
## [2.7.8] - 2026-10-07
From 8cde206e6d7fae3e9cdb74c8f06610d4af13f47f Mon Sep 17 00:00:00 2001
From: hect0x7 <93357912+hect0x7@users.noreply.github.com>
Date: Wed, 14 Oct 2026 00:01:59 +0800
Subject: [PATCH 6/7] =?UTF-8?q?test:=20=E8=A7=84=E8=8C=83=E5=8F=96?=
=?UTF-8?q?=E6=B6=88=E6=B5=8B=E8=AF=95=E7=94=A8=E4=BE=8B=E4=BD=BF=E7=94=A8?=
=?UTF-8?q?=20DownloadControl=20=E5=A5=91=E7=BA=A6?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
---
tests/test_jmcomic/test_jm_cancellation.py | 13 +++++++++----
tests/test_jmcomic/test_jm_download_manifest.py | 14 ++++++++------
2 files changed, 17 insertions(+), 10 deletions(-)
diff --git a/tests/test_jmcomic/test_jm_cancellation.py b/tests/test_jmcomic/test_jm_cancellation.py
index bdb11f4bc..286bd3f61 100644
--- a/tests/test_jmcomic/test_jm_cancellation.py
+++ b/tests/test_jmcomic/test_jm_cancellation.py
@@ -137,12 +137,17 @@ def cancel_first(jmid, *_args, **_kwargs):
with self.assertRaises(DownloadCancelledException):
download_batch(cancel_first, ['123', '456'], option=object())
- def test_sync_batch_propagates_manual_cancellation_without_control(self):
+ def test_sync_batch_cancellation_with_control(self):
+ control = DownloadControl()
+
def stop(_jmid, *_args, **_kwargs):
- raise DownloadCancelledException('manual stop')
+ control.cancel('manual stop')
+ BaseDownloader.raise_if_cancelled()
+
+ with jm_task_context(control=control):
+ with self.assertRaisesRegex(DownloadCancelledException, 'manual stop'):
+ download_batch(stop, ['123'], option=object())
- with self.assertRaisesRegex(DownloadCancelledException, 'manual stop'):
- download_batch(stop, ['123'], option=object())
def test_sync_batch_cancellation_exposes_completed_results_and_unfinished_ids(self):
control = DownloadControl()
diff --git a/tests/test_jmcomic/test_jm_download_manifest.py b/tests/test_jmcomic/test_jm_download_manifest.py
index 461153141..c650da4cb 100644
--- a/tests/test_jmcomic/test_jm_download_manifest.py
+++ b/tests/test_jmcomic/test_jm_download_manifest.py
@@ -227,27 +227,29 @@ def fail(_image):
with TemporaryDirectory() as temp_dir:
asyncio.run(run_test(temp_dir))
- def test_after_image_direct_cancellation_keeps_saved_image(self):
+ def test_after_image_cancellation_keeps_saved_image(self):
with TemporaryDirectory() as temp_dir:
album, photo, images = new_album_photo_images()
option = ContractOption(temp_dir)
downloader = ContractSyncDownloader(option, album, photo, images)
- error = DownloadCancelledException('image plugin cancelled')
+ control = DownloadControl()
def cancel(_image):
- raise error
+ control.cancel('image plugin cancelled')
option.after_image_callback = cancel
- with self.assertRaises(DownloadCancelledException) as caught:
- downloader.download_album(album.id)
+ with jm_task_context(control=control):
+ with self.assertRaises(DownloadCancelledException) as caught:
+ downloader.download_album(album.id)
- self.assertIs(caught.exception, error)
+ self.assertEqual(caught.exception.reason, 'image plugin cancelled')
self.assertEqual(downloader.download_failed_image, [])
self.assertEqual(
downloader.manifest_dict[album].image_filepath_list,
[images[0].save_path],
)
+
def test_current_sync_image_is_recorded_before_cancellation(self):
with TemporaryDirectory() as temp_dir:
album, photo, image_list = new_album_photo_images()
From badc04ab8a68d9eb6f0d26b9c16e6302b3b21746 Mon Sep 17 00:00:00 2001
From: hect0x7 <93357912+hect0x7@users.noreply.github.com>
Date: Wed, 14 Oct 2026 00:24:18 +0800
Subject: [PATCH 7/7] =?UTF-8?q?ci:=20=E8=BF=98=E5=8E=9F=E5=B7=A5=E4=BD=9C?=
=?UTF-8?q?=E6=B5=81Action=E7=89=88=E6=9C=AC=E4=B8=8Emaster=E5=AF=B9?=
=?UTF-8?q?=E9=BD=90=E7=A1=AE=E4=BF=9D=E5=8F=91=E5=B8=83=E5=B7=A5=E4=BD=9C?=
=?UTF-8?q?=E6=B5=81=E5=8F=AF=E7=94=A8?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
---
.github/workflows/benchmark.yml | 6 +++---
.github/workflows/close_specific_pr.yml | 4 ++--
.github/workflows/export_favorites.yml | 8 +++-----
.github/workflows/release.yml | 8 +++-----
.github/workflows/release_auto.yml | 8 +++-----
.github/workflows/test_api.yml | 4 ++--
.github/workflows/test_html.yml | 4 ++--
7 files changed, 18 insertions(+), 24 deletions(-)
diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml
index dd928be5e..e534db734 100644
--- a/.github/workflows/benchmark.yml
+++ b/.github/workflows/benchmark.yml
@@ -19,10 +19,10 @@ jobs:
timeout-minutes: 15
steps:
- - uses: actions/checkout@v7
+ - uses: actions/checkout@v4
- name: Set up Python
- uses: actions/setup-python@v7
+ uses: actions/setup-python@v5
with:
python-version: '3.14'
@@ -49,7 +49,7 @@ jobs:
- name: Upload Benchmark Report
if: always()
- uses: actions/upload-artifact@v7
+ uses: actions/upload-artifact@v4
with:
name: benchmark-report
path: PERFORMANCE_REPORT.md
diff --git a/.github/workflows/close_specific_pr.yml b/.github/workflows/close_specific_pr.yml
index ce890e337..8ed49d3eb 100644
--- a/.github/workflows/close_specific_pr.yml
+++ b/.github/workflows/close_specific_pr.yml
@@ -20,7 +20,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Checkout code
- uses: actions/checkout@v6
+ uses: actions/checkout@v4
- name: Close PR
run: |
@@ -36,4 +36,4 @@ jobs:
gh pr comment ${{ github.event.pull_request.number }} --repo ${{ github.repository }} --body '${{ env.comment }}'
gh pr close ${{ github.event.pull_request.number }} --repo ${{ github.repository }}
env:
- GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
\ No newline at end of file
diff --git a/.github/workflows/export_favorites.yml b/.github/workflows/export_favorites.yml
index 4375e3bc7..a00cdf64c 100644
--- a/.github/workflows/export_favorites.yml
+++ b/.github/workflows/export_favorites.yml
@@ -48,11 +48,9 @@ jobs:
ZIP_FP: /home/runner/work/jmcomic/download/export.7z
steps:
- - uses: actions/checkout@v7
- with:
- persist-credentials: false
+ - uses: actions/checkout@v4
- name: Set up Python 3.11
- uses: actions/setup-python@v7
+ uses: actions/setup-python@v5
with:
python-version: "3.11"
@@ -73,7 +71,7 @@ jobs:
python workflow_export_favorites.py
- name: 上传结果
- uses: actions/upload-artifact@v7
+ uses: actions/upload-artifact@v4
with:
name: '导出的收藏夹'
path: ${{ env.ZIP_FP }}
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 9458b5856..8f17f7722 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -13,12 +13,10 @@ jobs:
id-token: write
contents: write
steps:
- - uses: actions/checkout@v7
- with:
- persist-credentials: false
+ - uses: actions/checkout@v4
- name: Set up Python 3.11
- uses: actions/setup-python@v7
+ uses: actions/setup-python@v5
with:
python-version: "3.11"
@@ -33,7 +31,7 @@ jobs:
python -m build
- name: Create Release
- uses: softprops/action-gh-release@v3
+ uses: softprops/action-gh-release@v2
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
diff --git a/.github/workflows/release_auto.yml b/.github/workflows/release_auto.yml
index c3829d119..0ef2d6cb1 100644
--- a/.github/workflows/release_auto.yml
+++ b/.github/workflows/release_auto.yml
@@ -14,12 +14,10 @@ jobs:
contents: write
if: startsWith(github.event.head_commit.message, 'v')
steps:
- - uses: actions/checkout@v7
- with:
- persist-credentials: false
+ - uses: actions/checkout@v4
- name: Set up Python 3.11
- uses: actions/setup-python@v7
+ uses: actions/setup-python@v5
with:
python-version: "3.11"
@@ -35,7 +33,7 @@ jobs:
python -m build
- name: Create Release
- uses: softprops/action-gh-release@v3
+ uses: softprops/action-gh-release@v2
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
diff --git a/.github/workflows/test_api.yml b/.github/workflows/test_api.yml
index 35245d21f..e0e5d7558 100644
--- a/.github/workflows/test_api.yml
+++ b/.github/workflows/test_api.yml
@@ -27,10 +27,10 @@ jobs:
steps:
- name: Checkout repository
- uses: actions/checkout@v7
+ uses: actions/checkout@v4
- name: Set up Python ${{ matrix.python-version }}
- uses: actions/setup-python@v7
+ uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python-version }}
diff --git a/.github/workflows/test_html.yml b/.github/workflows/test_html.yml
index f8a45b50c..8f44d7c9b 100644
--- a/.github/workflows/test_html.yml
+++ b/.github/workflows/test_html.yml
@@ -27,10 +27,10 @@ jobs:
steps:
- name: Checkout repository
- uses: actions/checkout@v7
+ uses: actions/checkout@v4
- name: Set up Python ${{ matrix.python-version }}
- uses: actions/setup-python@v7
+ uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python-version }}