mirror of
https://github.com/NanmiCoder/MediaCrawler.git
synced 2026-02-25 17:40:51 +08:00
feat: 添加并发爬虫数量控制参数
- 新增 --max_concurrency 命令行参数 - 用于控制并发爬虫数量 - 默认值为 1
This commit is contained in:
@@ -258,6 +258,14 @@ async def parse_cmd(argv: Optional[Sequence[str]] = None):
|
||||
rich_help_panel="Comment Configuration",
|
||||
),
|
||||
] = config.CRAWLER_MAX_COMMENTS_COUNT_SINGLENOTES,
|
||||
max_concurrency: Annotated[
|
||||
int,
|
||||
typer.Option(
|
||||
"--max_concurrency",
|
||||
help="Maximum number of concurrent crawlers",
|
||||
rich_help_panel="Performance Configuration",
|
||||
),
|
||||
] = config.MAX_CONCURRENCY_NUM,
|
||||
) -> SimpleNamespace:
|
||||
"""MediaCrawler 命令行入口"""
|
||||
|
||||
@@ -283,6 +291,7 @@ async def parse_cmd(argv: Optional[Sequence[str]] = None):
|
||||
config.SAVE_DATA_OPTION = save_data_option.value
|
||||
config.COOKIES = cookies
|
||||
config.CRAWLER_MAX_COMMENTS_COUNT_SINGLENOTES = max_comments_count_singlenotes
|
||||
config.MAX_CONCURRENCY_NUM = max_concurrency
|
||||
|
||||
# Set platform-specific ID lists for detail/creator mode
|
||||
if specified_id_list:
|
||||
|
||||
Reference in New Issue
Block a user