Merge pull request #900 from zanmeipaul/main

feat: 启动任务接口添加帖子/视频数量与评论数量覆盖支持

api/schemas/crawler.py

@@ -18,7 +18,10 @@
 
 from enum import Enum
 from typing import Optional, Literal
-from pydantic import BaseModel
+from pydantic import BaseModel, Field
+
+
+MAX_API_LIMIT_COUNT = 10000
 
 
 class PlatformEnum(str, Enum):
@@ -71,6 +74,8 @@ class CrawlerStartRequest(BaseModel):
     save_option: SaveDataOptionEnum = SaveDataOptionEnum.JSONL
     cookies: str = ""
     headless: bool = False
+    max_notes_count: Optional[int] = Field(default=None, ge=1, le=MAX_API_LIMIT_COUNT)
+    max_comments_count: Optional[int] = Field(default=None, ge=1, le=MAX_API_LIMIT_COUNT)
 
 
 class CrawlerStatusResponse(BaseModel):

api/services/crawler_manager.py

@@ -225,6 +225,12 @@ class CrawlerManager:
         cmd.extend(["--get_comment", "true" if config.enable_comments else "false"])
         cmd.extend(["--get_sub_comment", "true" if config.enable_sub_comments else "false"])
 
+        if config.max_notes_count is not None:
+            cmd.extend(["--crawler_max_notes_count", str(config.max_notes_count)])
+
+        if config.max_comments_count is not None:
+            cmd.extend(["--max_comments_count_singlenotes", str(config.max_comments_count)])
+
         if config.cookies:
             cmd.extend(["--cookies", config.cookies])