mirror of
https://github.com/NanmiCoder/MediaCrawler.git
synced 2026-02-16 20:11:12 +08:00
Merge pull request #821 from wanzirong/feature/max-concurrency-param
feat: 添加并发爬虫数量控制参数 --max_concurrency_num
This commit is contained in:
@@ -258,6 +258,14 @@ async def parse_cmd(argv: Optional[Sequence[str]] = None):
|
||||
rich_help_panel="Comment Configuration",
|
||||
),
|
||||
] = config.CRAWLER_MAX_COMMENTS_COUNT_SINGLENOTES,
|
||||
max_concurrency_num: Annotated[
|
||||
int,
|
||||
typer.Option(
|
||||
"--max_concurrency_num",
|
||||
help="Maximum number of concurrent crawlers",
|
||||
rich_help_panel="Performance Configuration",
|
||||
),
|
||||
] = config.MAX_CONCURRENCY_NUM,
|
||||
) -> SimpleNamespace:
|
||||
"""MediaCrawler 命令行入口"""
|
||||
|
||||
@@ -283,6 +291,7 @@ async def parse_cmd(argv: Optional[Sequence[str]] = None):
|
||||
config.SAVE_DATA_OPTION = save_data_option.value
|
||||
config.COOKIES = cookies
|
||||
config.CRAWLER_MAX_COMMENTS_COUNT_SINGLENOTES = max_comments_count_singlenotes
|
||||
config.MAX_CONCURRENCY_NUM = max_concurrency_num
|
||||
|
||||
# Set platform-specific ID lists for detail/creator mode
|
||||
if specified_id_list:
|
||||
|
||||
Reference in New Issue
Block a user