mirror of
https://github.com/NanmiCoder/MediaCrawler.git
synced 2026-06-08 02:47:26 +08:00
i18n: translate all Chinese comments, docstrings, and logger messages to English
Comprehensive translation of Chinese text to English across the entire codebase: - api/: FastAPI server documentation and logger messages - cache/: Cache abstraction layer comments and docstrings - database/: Database models and MongoDB store documentation - media_platform/: All platform crawlers (Bilibili, Douyin, Kuaishou, Tieba, Weibo, Xiaohongshu, Zhihu) - model/: Data model documentation - proxy/: Proxy pool and provider documentation - store/: Data storage layer comments - tools/: Utility functions and browser automation - test/: Test file documentation Preserved: Chinese disclaimer header (lines 10-18) for legal compliance 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
程序员阿江(Relakkes)
@@ -48,7 +48,7 @@ class BaiduTieBaClient(AbstractApiClient):
|
||||
):
|
||||
self.ip_pool: Optional[ProxyIpPool] = ip_pool
|
||||
self.timeout = timeout
|
||||
# 使用传入的headers(包含真实浏览器UA)或默认headers
|
||||
# Use provided headers (including real browser UA) or default headers
|
||||
self.headers = headers or {
|
||||
"User-Agent": utils.get_user_agent(),
|
||||
"Cookie": "",
|
||||
@@ -56,21 +56,21 @@ class BaiduTieBaClient(AbstractApiClient):
|
||||
self._host = "https://tieba.baidu.com"
|
||||
self._page_extractor = TieBaExtractor()
|
||||
self.default_ip_proxy = default_ip_proxy
|
||||
self.playwright_page = playwright_page # Playwright页面对象
|
||||
self.playwright_page = playwright_page # Playwright page object
|
||||
|
||||
def _sync_request(self, method, url, proxy=None, **kwargs):
|
||||
"""
|
||||
同步的requests请求方法
|
||||
Synchronous requests method
|
||||
Args:
|
||||
method: 请求方法
|
||||
url: 请求的URL
|
||||
proxy: 代理IP
|
||||
**kwargs: 其他请求参数
|
||||
method: Request method
|
||||
url: Request URL
|
||||
proxy: Proxy IP
|
||||
**kwargs: Other request parameters
|
||||
|
||||
Returns:
|
||||
response对象
|
||||
Response object
|
||||
"""
|
||||
# 构造代理字典
|
||||
# Construct proxy dictionary
|
||||
proxies = None
|
||||
if proxy:
|
||||
proxies = {
|
||||
@@ -78,7 +78,7 @@ class BaiduTieBaClient(AbstractApiClient):
|
||||
"https": proxy,
|
||||
}
|
||||
|
||||
# 发送请求
|
||||
# Send request
|
||||
response = requests.request(
|
||||
method=method,
|
||||
url=url,
|
||||
@@ -91,7 +91,7 @@ class BaiduTieBaClient(AbstractApiClient):
|
||||
|
||||
async def _refresh_proxy_if_expired(self) -> None:
|
||||
"""
|
||||
检测代理是否过期，如果过期则自动刷新
|
||||
Check if proxy is expired and automatically refresh if necessary
|
||||
"""
|
||||
if self.ip_pool is None:
|
||||
return
|
||||
@@ -101,7 +101,7 @@ class BaiduTieBaClient(AbstractApiClient):
|
||||
"[BaiduTieBaClient._refresh_proxy_if_expired] Proxy expired, refreshing..."
|
||||
)
|
||||
new_proxy = await self.ip_pool.get_or_refresh_proxy()
|
||||
# 更新代理URL
|
||||
# Update proxy URL
|
||||
_, self.default_ip_proxy = utils.format_proxy_info(new_proxy)
|
||||
utils.logger.info(
|
||||
f"[BaiduTieBaClient._refresh_proxy_if_expired] New proxy: {new_proxy.ip}:{new_proxy.port}"
|
||||
@@ -110,23 +110,23 @@ class BaiduTieBaClient(AbstractApiClient):
|
||||
@retry(stop=stop_after_attempt(3), wait=wait_fixed(1))
|
||||
async def request(self, method, url, return_ori_content=False, proxy=None, **kwargs) -> Union[str, Any]:
|
||||
"""
|
||||
封装requests的公共请求方法，对请求响应做一些处理
|
||||
Common request method wrapper for requests, handles request responses
|
||||
Args:
|
||||
method: 请求方法
|
||||
url: 请求的URL
|
||||
return_ori_content: 是否返回原始内容
|
||||
proxy: 代理IP
|
||||
**kwargs: 其他请求参数，例如请求头、请求体等
|
||||
method: Request method
|
||||
url: Request URL
|
||||
return_ori_content: Whether to return original content
|
||||
proxy: Proxy IP
|
||||
**kwargs: Other request parameters, such as headers, request body, etc.
|
||||
|
||||
Returns:
|
||||
|
||||
"""
|
||||
# 每次请求前检测代理是否过期
|
||||
# Check if proxy is expired before each request
|
||||
await self._refresh_proxy_if_expired()
|
||||
|
||||
actual_proxy = proxy if proxy else self.default_ip_proxy
|
||||
|
||||
# 在线程池中执行同步的requests请求
|
||||
# Execute synchronous requests in thread pool
|
||||
response = await asyncio.to_thread(
|
||||
self._sync_request,
|
||||
method,
|
||||
@@ -151,11 +151,11 @@ class BaiduTieBaClient(AbstractApiClient):
|
||||
|
||||
async def get(self, uri: str, params=None, return_ori_content=False, **kwargs) -> Any:
|
||||
"""
|
||||
GET请求，对请求头签名
|
||||
GET request with header signing
|
||||
Args:
|
||||
uri: 请求路由
|
||||
params: 请求参数
|
||||
return_ori_content: 是否返回原始内容
|
||||
uri: Request route
|
||||
params: Request parameters
|
||||
return_ori_content: Whether to return original content
|
||||
|
||||
Returns:
|
||||
|
||||
@@ -175,15 +175,15 @@ class BaiduTieBaClient(AbstractApiClient):
|
||||
self.default_ip_proxy = proxy
|
||||
return res
|
||||
|
||||
utils.logger.error(f"[BaiduTieBaClient.get] 达到了最大重试次数，IP已经被Block，请尝试更换新的IP代理: {e}")
|
||||
raise Exception(f"[BaiduTieBaClient.get] 达到了最大重试次数，IP已经被Block，请尝试更换新的IP代理: {e}")
|
||||
utils.logger.error(f"[BaiduTieBaClient.get] Reached maximum retry attempts, IP is blocked, please try a new IP proxy: {e}")
|
||||
raise Exception(f"[BaiduTieBaClient.get] Reached maximum retry attempts, IP is blocked, please try a new IP proxy: {e}")
|
||||
|
||||
async def post(self, uri: str, data: dict, **kwargs) -> Dict:
|
||||
"""
|
||||
POST请求，对请求头签名
|
||||
POST request with header signing
|
||||
Args:
|
||||
uri: 请求路由
|
||||
data: 请求体参数
|
||||
uri: Request route
|
||||
data: Request body parameters
|
||||
|
||||
Returns:
|
||||
|
||||
@@ -193,13 +193,13 @@ class BaiduTieBaClient(AbstractApiClient):
|
||||
|
||||
async def pong(self, browser_context: BrowserContext = None) -> bool:
|
||||
"""
|
||||
用于检查登录态是否失效了
|
||||
使用Cookie检测而非API调用,避免被检测
|
||||
Check if login state is still valid
|
||||
Uses Cookie detection instead of API calls to avoid detection
|
||||
Args:
|
||||
browser_context: 浏览器上下文对象
|
||||
browser_context: Browser context object
|
||||
|
||||
Returns:
|
||||
bool: True表示已登录,False表示未登录
|
||||
bool: True if logged in, False if not logged in
|
||||
"""
|
||||
utils.logger.info("[BaiduTieBaClient.pong] Begin to check tieba login state by cookies...")
|
||||
|
||||
@@ -208,13 +208,13 @@ class BaiduTieBaClient(AbstractApiClient):
|
||||
return False
|
||||
|
||||
try:
|
||||
# 从浏览器获取cookies并检查关键登录cookie
|
||||
# Get cookies from browser and check key login cookies
|
||||
_, cookie_dict = utils.convert_cookies(await browser_context.cookies())
|
||||
|
||||
# 百度贴吧的登录标识: STOKEN 或 PTOKEN
|
||||
# Baidu Tieba login identifiers: STOKEN or PTOKEN
|
||||
stoken = cookie_dict.get("STOKEN")
|
||||
ptoken = cookie_dict.get("PTOKEN")
|
||||
bduss = cookie_dict.get("BDUSS") # 百度通用登录cookie
|
||||
bduss = cookie_dict.get("BDUSS") # Baidu universal login cookie
|
||||
|
||||
if stoken or ptoken or bduss:
|
||||
utils.logger.info(f"[BaiduTieBaClient.pong] Login state verified by cookies (STOKEN: {bool(stoken)}, PTOKEN: {bool(ptoken)}, BDUSS: {bool(bduss)})")
|
||||
@@ -229,9 +229,9 @@ class BaiduTieBaClient(AbstractApiClient):
|
||||
|
||||
async def update_cookies(self, browser_context: BrowserContext):
|
||||
"""
|
||||
API客户端提供的更新cookies方法，一般情况下登录成功后会调用此方法
|
||||
Update cookies method provided by API client, usually called after successful login
|
||||
Args:
|
||||
browser_context: 浏览器上下文对象
|
||||
browser_context: Browser context object
|
||||
|
||||
Returns:
|
||||
|
||||
@@ -249,13 +249,13 @@ class BaiduTieBaClient(AbstractApiClient):
|
||||
note_type: SearchNoteType = SearchNoteType.FIXED_THREAD,
|
||||
) -> List[TiebaNote]:
|
||||
"""
|
||||
根据关键词搜索贴吧帖子 (使用Playwright访问页面,避免API检测)
|
||||
Search Tieba posts by keyword (uses Playwright to access page, avoiding API detection)
|
||||
Args:
|
||||
keyword: 关键词
|
||||
page: 分页第几页
|
||||
page_size: 每页大小
|
||||
sort: 结果排序方式
|
||||
note_type: 帖子类型（主题贴｜主题+回复混合模式）
|
||||
keyword: Keyword
|
||||
page: Page number
|
||||
page_size: Page size
|
||||
sort: Result sort method
|
||||
note_type: Post type (main thread | main thread + reply mixed mode)
|
||||
Returns:
|
||||
|
||||
"""
|
||||
@@ -263,8 +263,8 @@ class BaiduTieBaClient(AbstractApiClient):
|
||||
utils.logger.error("[BaiduTieBaClient.get_notes_by_keyword] playwright_page is None, cannot use browser mode")
|
||||
raise Exception("playwright_page is required for browser-based search")
|
||||
|
||||
# 构造搜索URL
|
||||
# 示例: https://tieba.baidu.com/f/search/res?ie=utf-8&qw=编程
|
||||
# Construct search URL
|
||||
# Example: https://tieba.baidu.com/f/search/res?ie=utf-8&qw=keyword
|
||||
search_url = f"{self._host}/f/search/res"
|
||||
params = {
|
||||
"ie": "utf-8",
|
||||
@@ -275,64 +275,64 @@ class BaiduTieBaClient(AbstractApiClient):
|
||||
"only_thread": note_type.value,
|
||||
}
|
||||
|
||||
# 拼接完整URL
|
||||
# Concatenate full URL
|
||||
full_url = f"{search_url}?{urlencode(params)}"
|
||||
utils.logger.info(f"[BaiduTieBaClient.get_notes_by_keyword] 访问搜索页面: {full_url}")
|
||||
utils.logger.info(f"[BaiduTieBaClient.get_notes_by_keyword] Accessing search page: {full_url}")
|
||||
|
||||
try:
|
||||
# 使用Playwright访问搜索页面
|
||||
# Use Playwright to access search page
|
||||
await self.playwright_page.goto(full_url, wait_until="domcontentloaded")
|
||||
|
||||
# 等待页面加载,使用配置文件中的延时设置
|
||||
# Wait for page loading, using delay setting from config file
|
||||
await asyncio.sleep(config.CRAWLER_MAX_SLEEP_SEC)
|
||||
|
||||
# 获取页面HTML内容
|
||||
# Get page HTML content
|
||||
page_content = await self.playwright_page.content()
|
||||
utils.logger.info(f"[BaiduTieBaClient.get_notes_by_keyword] 成功获取搜索页面HTML,长度: {len(page_content)}")
|
||||
utils.logger.info(f"[BaiduTieBaClient.get_notes_by_keyword] Successfully retrieved search page HTML, length: {len(page_content)}")
|
||||
|
||||
# 提取搜索结果
|
||||
# Extract search results
|
||||
notes = self._page_extractor.extract_search_note_list(page_content)
|
||||
utils.logger.info(f"[BaiduTieBaClient.get_notes_by_keyword] 提取到 {len(notes)} 条帖子")
|
||||
utils.logger.info(f"[BaiduTieBaClient.get_notes_by_keyword] Extracted {len(notes)} posts")
|
||||
return notes
|
||||
|
||||
except Exception as e:
|
||||
utils.logger.error(f"[BaiduTieBaClient.get_notes_by_keyword] 搜索失败: {e}")
|
||||
utils.logger.error(f"[BaiduTieBaClient.get_notes_by_keyword] Search failed: {e}")
|
||||
raise
|
||||
|
||||
async def get_note_by_id(self, note_id: str) -> TiebaNote:
|
||||
"""
|
||||
根据帖子ID获取帖子详情 (使用Playwright访问页面,避免API检测)
|
||||
Get post details by post ID (uses Playwright to access page, avoiding API detection)
|
||||
Args:
|
||||
note_id: 帖子ID
|
||||
note_id: Post ID
|
||||
|
||||
Returns:
|
||||
TiebaNote: 帖子详情对象
|
||||
TiebaNote: Post detail object
|
||||
"""
|
||||
if not self.playwright_page:
|
||||
utils.logger.error("[BaiduTieBaClient.get_note_by_id] playwright_page is None, cannot use browser mode")
|
||||
raise Exception("playwright_page is required for browser-based note detail fetching")
|
||||
|
||||
# 构造帖子详情URL
|
||||
# Construct post detail URL
|
||||
note_url = f"{self._host}/p/{note_id}"
|
||||
utils.logger.info(f"[BaiduTieBaClient.get_note_by_id] 访问帖子详情页面: {note_url}")
|
||||
utils.logger.info(f"[BaiduTieBaClient.get_note_by_id] Accessing post detail page: {note_url}")
|
||||
|
||||
try:
|
||||
# 使用Playwright访问帖子详情页面
|
||||
# Use Playwright to access post detail page
|
||||
await self.playwright_page.goto(note_url, wait_until="domcontentloaded")
|
||||
|
||||
# 等待页面加载,使用配置文件中的延时设置
|
||||
# Wait for page loading, using delay setting from config file
|
||||
await asyncio.sleep(config.CRAWLER_MAX_SLEEP_SEC)
|
||||
|
||||
# 获取页面HTML内容
|
||||
# Get page HTML content
|
||||
page_content = await self.playwright_page.content()
|
||||
utils.logger.info(f"[BaiduTieBaClient.get_note_by_id] 成功获取帖子详情HTML,长度: {len(page_content)}")
|
||||
utils.logger.info(f"[BaiduTieBaClient.get_note_by_id] Successfully retrieved post detail HTML, length: {len(page_content)}")
|
||||
|
||||
# 提取帖子详情
|
||||
# Extract post details
|
||||
note_detail = self._page_extractor.extract_note_detail(page_content)
|
||||
return note_detail
|
||||
|
||||
except Exception as e:
|
||||
utils.logger.error(f"[BaiduTieBaClient.get_note_by_id] 获取帖子详情失败: {e}")
|
||||
utils.logger.error(f"[BaiduTieBaClient.get_note_by_id] Failed to get post details: {e}")
|
||||
raise
|
||||
|
||||
async def get_note_all_comments(
|
||||
@@ -343,14 +343,14 @@ class BaiduTieBaClient(AbstractApiClient):
|
||||
max_count: int = 10,
|
||||
) -> List[TiebaComment]:
|
||||
"""
|
||||
获取指定帖子下的所有一级评论 (使用Playwright访问页面,避免API检测)
|
||||
Get all first-level comments for specified post (uses Playwright to access page, avoiding API detection)
|
||||
Args:
|
||||
note_detail: 帖子详情对象
|
||||
crawl_interval: 爬取一次笔记的延迟单位（秒）
|
||||
callback: 一次笔记爬取结束后的回调函数
|
||||
max_count: 一次帖子爬取的最大评论数量
|
||||
note_detail: Post detail object
|
||||
crawl_interval: Crawl delay interval in seconds
|
||||
callback: Callback function after one post crawl completes
|
||||
max_count: Maximum number of comments to crawl per post
|
||||
Returns:
|
||||
List[TiebaComment]: 评论列表
|
||||
List[TiebaComment]: Comment list
|
||||
"""
|
||||
if not self.playwright_page:
|
||||
utils.logger.error("[BaiduTieBaClient.get_note_all_comments] playwright_page is None, cannot use browser mode")
|
||||
@@ -360,30 +360,30 @@ class BaiduTieBaClient(AbstractApiClient):
|
||||
current_page = 1
|
||||
|
||||
while note_detail.total_replay_page >= current_page and len(result) < max_count:
|
||||
# 构造评论页URL
|
||||
# Construct comment page URL
|
||||
comment_url = f"{self._host}/p/{note_detail.note_id}?pn={current_page}"
|
||||
utils.logger.info(f"[BaiduTieBaClient.get_note_all_comments] 访问评论页面: {comment_url}")
|
||||
utils.logger.info(f"[BaiduTieBaClient.get_note_all_comments] Accessing comment page: {comment_url}")
|
||||
|
||||
try:
|
||||
# 使用Playwright访问评论页面
|
||||
# Use Playwright to access comment page
|
||||
await self.playwright_page.goto(comment_url, wait_until="domcontentloaded")
|
||||
|
||||
# 等待页面加载,使用配置文件中的延时设置
|
||||
# Wait for page loading, using delay setting from config file
|
||||
await asyncio.sleep(config.CRAWLER_MAX_SLEEP_SEC)
|
||||
|
||||
# 获取页面HTML内容
|
||||
# Get page HTML content
|
||||
page_content = await self.playwright_page.content()
|
||||
|
||||
# 提取评论
|
||||
# Extract comments
|
||||
comments = self._page_extractor.extract_tieba_note_parment_comments(
|
||||
page_content, note_id=note_detail.note_id
|
||||
)
|
||||
|
||||
if not comments:
|
||||
utils.logger.info(f"[BaiduTieBaClient.get_note_all_comments] 第{current_page}页没有评论,停止爬取")
|
||||
utils.logger.info(f"[BaiduTieBaClient.get_note_all_comments] Page {current_page} has no comments, stopping crawl")
|
||||
break
|
||||
|
||||
# 限制评论数量
|
||||
# Limit comment count
|
||||
if len(result) + len(comments) > max_count:
|
||||
comments = comments[:max_count - len(result)]
|
||||
|
||||
@@ -392,7 +392,7 @@ class BaiduTieBaClient(AbstractApiClient):
|
||||
|
||||
result.extend(comments)
|
||||
|
||||
# 获取所有子评论
|
||||
# Get all sub-comments
|
||||
await self.get_comments_all_sub_comments(
|
||||
comments, crawl_interval=crawl_interval, callback=callback
|
||||
)
|
||||
@@ -401,10 +401,10 @@ class BaiduTieBaClient(AbstractApiClient):
|
||||
current_page += 1
|
||||
|
||||
except Exception as e:
|
||||
utils.logger.error(f"[BaiduTieBaClient.get_note_all_comments] 获取第{current_page}页评论失败: {e}")
|
||||
utils.logger.error(f"[BaiduTieBaClient.get_note_all_comments] Failed to get page {current_page} comments: {e}")
|
||||
break
|
||||
|
||||
utils.logger.info(f"[BaiduTieBaClient.get_note_all_comments] 共获取 {len(result)} 条一级评论")
|
||||
utils.logger.info(f"[BaiduTieBaClient.get_note_all_comments] Total retrieved {len(result)} first-level comments")
|
||||
return result
|
||||
|
||||
async def get_comments_all_sub_comments(
|
||||
@@ -414,14 +414,14 @@ class BaiduTieBaClient(AbstractApiClient):
|
||||
callback: Optional[Callable] = None,
|
||||
) -> List[TiebaComment]:
|
||||
"""
|
||||
获取指定评论下的所有子评论 (使用Playwright访问页面,避免API检测)
|
||||
Get all sub-comments for specified comments (uses Playwright to access page, avoiding API detection)
|
||||
Args:
|
||||
comments: 评论列表
|
||||
crawl_interval: 爬取一次笔记的延迟单位（秒）
|
||||
callback: 一次笔记爬取结束后的回调函数
|
||||
comments: Comment list
|
||||
crawl_interval: Crawl delay interval in seconds
|
||||
callback: Callback function after one post crawl completes
|
||||
|
||||
Returns:
|
||||
List[TiebaComment]: 子评论列表
|
||||
List[TiebaComment]: Sub-comment list
|
||||
"""
|
||||
if not config.ENABLE_GET_SUB_COMMENTS:
|
||||
return []
|
||||
@@ -440,7 +440,7 @@ class BaiduTieBaClient(AbstractApiClient):
|
||||
max_sub_page_num = parment_comment.sub_comment_count // 10 + 1
|
||||
|
||||
while max_sub_page_num >= current_page:
|
||||
# 构造子评论URL
|
||||
# Construct sub-comment URL
|
||||
sub_comment_url = (
|
||||
f"{self._host}/p/comment?"
|
||||
f"tid={parment_comment.note_id}&"
|
||||
@@ -448,19 +448,19 @@ class BaiduTieBaClient(AbstractApiClient):
|
||||
f"fid={parment_comment.tieba_id}&"
|
||||
f"pn={current_page}"
|
||||
)
|
||||
utils.logger.info(f"[BaiduTieBaClient.get_comments_all_sub_comments] 访问子评论页面: {sub_comment_url}")
|
||||
utils.logger.info(f"[BaiduTieBaClient.get_comments_all_sub_comments] Accessing sub-comment page: {sub_comment_url}")
|
||||
|
||||
try:
|
||||
# 使用Playwright访问子评论页面
|
||||
# Use Playwright to access sub-comment page
|
||||
await self.playwright_page.goto(sub_comment_url, wait_until="domcontentloaded")
|
||||
|
||||
# 等待页面加载,使用配置文件中的延时设置
|
||||
# Wait for page loading, using delay setting from config file
|
||||
await asyncio.sleep(config.CRAWLER_MAX_SLEEP_SEC)
|
||||
|
||||
# 获取页面HTML内容
|
||||
# Get page HTML content
|
||||
page_content = await self.playwright_page.content()
|
||||
|
||||
# 提取子评论
|
||||
# Extract sub-comments
|
||||
sub_comments = self._page_extractor.extract_tieba_note_sub_comments(
|
||||
page_content, parent_comment=parment_comment
|
||||
)
|
||||
@@ -468,7 +468,7 @@ class BaiduTieBaClient(AbstractApiClient):
|
||||
if not sub_comments:
|
||||
utils.logger.info(
|
||||
f"[BaiduTieBaClient.get_comments_all_sub_comments] "
|
||||
f"评论{parment_comment.comment_id}第{current_page}页没有子评论,停止爬取"
|
||||
f"Comment {parment_comment.comment_id} page {current_page} has no sub-comments, stopping crawl"
|
||||
)
|
||||
break
|
||||
|
||||
@@ -482,125 +482,125 @@ class BaiduTieBaClient(AbstractApiClient):
|
||||
except Exception as e:
|
||||
utils.logger.error(
|
||||
f"[BaiduTieBaClient.get_comments_all_sub_comments] "
|
||||
f"获取评论{parment_comment.comment_id}第{current_page}页子评论失败: {e}"
|
||||
f"Failed to get comment {parment_comment.comment_id} page {current_page} sub-comments: {e}"
|
||||
)
|
||||
break
|
||||
|
||||
utils.logger.info(f"[BaiduTieBaClient.get_comments_all_sub_comments] 共获取 {len(all_sub_comments)} 条子评论")
|
||||
utils.logger.info(f"[BaiduTieBaClient.get_comments_all_sub_comments] Total retrieved {len(all_sub_comments)} sub-comments")
|
||||
return all_sub_comments
|
||||
|
||||
async def get_notes_by_tieba_name(self, tieba_name: str, page_num: int) -> List[TiebaNote]:
|
||||
"""
|
||||
根据贴吧名称获取帖子列表 (使用Playwright访问页面,避免API检测)
|
||||
Get post list by Tieba name (uses Playwright to access page, avoiding API detection)
|
||||
Args:
|
||||
tieba_name: 贴吧名称
|
||||
page_num: 分页页码
|
||||
tieba_name: Tieba name
|
||||
page_num: Page number
|
||||
|
||||
Returns:
|
||||
List[TiebaNote]: 帖子列表
|
||||
List[TiebaNote]: Post list
|
||||
"""
|
||||
if not self.playwright_page:
|
||||
utils.logger.error("[BaiduTieBaClient.get_notes_by_tieba_name] playwright_page is None, cannot use browser mode")
|
||||
raise Exception("playwright_page is required for browser-based tieba note fetching")
|
||||
|
||||
# 构造贴吧帖子列表URL
|
||||
# Construct Tieba post list URL
|
||||
tieba_url = f"{self._host}/f?kw={quote(tieba_name)}&pn={page_num}"
|
||||
utils.logger.info(f"[BaiduTieBaClient.get_notes_by_tieba_name] 访问贴吧页面: {tieba_url}")
|
||||
utils.logger.info(f"[BaiduTieBaClient.get_notes_by_tieba_name] Accessing Tieba page: {tieba_url}")
|
||||
|
||||
try:
|
||||
# 使用Playwright访问贴吧页面
|
||||
# Use Playwright to access Tieba page
|
||||
await self.playwright_page.goto(tieba_url, wait_until="domcontentloaded")
|
||||
|
||||
# 等待页面加载,使用配置文件中的延时设置
|
||||
# Wait for page loading, using delay setting from config file
|
||||
await asyncio.sleep(config.CRAWLER_MAX_SLEEP_SEC)
|
||||
|
||||
# 获取页面HTML内容
|
||||
# Get page HTML content
|
||||
page_content = await self.playwright_page.content()
|
||||
utils.logger.info(f"[BaiduTieBaClient.get_notes_by_tieba_name] 成功获取贴吧页面HTML,长度: {len(page_content)}")
|
||||
utils.logger.info(f"[BaiduTieBaClient.get_notes_by_tieba_name] Successfully retrieved Tieba page HTML, length: {len(page_content)}")
|
||||
|
||||
# 提取帖子列表
|
||||
# Extract post list
|
||||
notes = self._page_extractor.extract_tieba_note_list(page_content)
|
||||
utils.logger.info(f"[BaiduTieBaClient.get_notes_by_tieba_name] 提取到 {len(notes)} 条帖子")
|
||||
utils.logger.info(f"[BaiduTieBaClient.get_notes_by_tieba_name] Extracted {len(notes)} posts")
|
||||
return notes
|
||||
|
||||
except Exception as e:
|
||||
utils.logger.error(f"[BaiduTieBaClient.get_notes_by_tieba_name] 获取贴吧帖子列表失败: {e}")
|
||||
utils.logger.error(f"[BaiduTieBaClient.get_notes_by_tieba_name] Failed to get Tieba post list: {e}")
|
||||
raise
|
||||
|
||||
async def get_creator_info_by_url(self, creator_url: str) -> str:
|
||||
"""
|
||||
根据创作者URL获取创作者信息 (使用Playwright访问页面,避免API检测)
|
||||
Get creator information by creator URL (uses Playwright to access page, avoiding API detection)
|
||||
Args:
|
||||
creator_url: 创作者主页URL
|
||||
creator_url: Creator homepage URL
|
||||
|
||||
Returns:
|
||||
str: 页面HTML内容
|
||||
str: Page HTML content
|
||||
"""
|
||||
if not self.playwright_page:
|
||||
utils.logger.error("[BaiduTieBaClient.get_creator_info_by_url] playwright_page is None, cannot use browser mode")
|
||||
raise Exception("playwright_page is required for browser-based creator info fetching")
|
||||
|
||||
utils.logger.info(f"[BaiduTieBaClient.get_creator_info_by_url] 访问创作者主页: {creator_url}")
|
||||
utils.logger.info(f"[BaiduTieBaClient.get_creator_info_by_url] Accessing creator homepage: {creator_url}")
|
||||
|
||||
try:
|
||||
# 使用Playwright访问创作者主页
|
||||
# Use Playwright to access creator homepage
|
||||
await self.playwright_page.goto(creator_url, wait_until="domcontentloaded")
|
||||
|
||||
# 等待页面加载,使用配置文件中的延时设置
|
||||
# Wait for page loading, using delay setting from config file
|
||||
await asyncio.sleep(config.CRAWLER_MAX_SLEEP_SEC)
|
||||
|
||||
# 获取页面HTML内容
|
||||
# Get page HTML content
|
||||
page_content = await self.playwright_page.content()
|
||||
utils.logger.info(f"[BaiduTieBaClient.get_creator_info_by_url] 成功获取创作者主页HTML,长度: {len(page_content)}")
|
||||
utils.logger.info(f"[BaiduTieBaClient.get_creator_info_by_url] Successfully retrieved creator homepage HTML, length: {len(page_content)}")
|
||||
|
||||
return page_content
|
||||
|
||||
except Exception as e:
|
||||
utils.logger.error(f"[BaiduTieBaClient.get_creator_info_by_url] 获取创作者主页失败: {e}")
|
||||
utils.logger.error(f"[BaiduTieBaClient.get_creator_info_by_url] Failed to get creator homepage: {e}")
|
||||
raise
|
||||
|
||||
async def get_notes_by_creator(self, user_name: str, page_number: int) -> Dict:
|
||||
"""
|
||||
根据创作者获取创作者的帖子 (使用Playwright访问页面,避免API检测)
|
||||
Get creator's posts by creator (uses Playwright to access page, avoiding API detection)
|
||||
Args:
|
||||
user_name: 创作者用户名
|
||||
page_number: 页码
|
||||
user_name: Creator username
|
||||
page_number: Page number
|
||||
|
||||
Returns:
|
||||
Dict: 包含帖子数据的字典
|
||||
Dict: Dictionary containing post data
|
||||
"""
|
||||
if not self.playwright_page:
|
||||
utils.logger.error("[BaiduTieBaClient.get_notes_by_creator] playwright_page is None, cannot use browser mode")
|
||||
raise Exception("playwright_page is required for browser-based creator notes fetching")
|
||||
|
||||
# 构造创作者帖子列表URL
|
||||
# Construct creator post list URL
|
||||
creator_url = f"{self._host}/home/get/getthread?un={quote(user_name)}&pn={page_number}&id=utf-8&_={utils.get_current_timestamp()}"
|
||||
utils.logger.info(f"[BaiduTieBaClient.get_notes_by_creator] 访问创作者帖子列表: {creator_url}")
|
||||
utils.logger.info(f"[BaiduTieBaClient.get_notes_by_creator] Accessing creator post list: {creator_url}")
|
||||
|
||||
try:
|
||||
# 使用Playwright访问创作者帖子列表页面
|
||||
# Use Playwright to access creator post list page
|
||||
await self.playwright_page.goto(creator_url, wait_until="domcontentloaded")
|
||||
|
||||
# 等待页面加载,使用配置文件中的延时设置
|
||||
# Wait for page loading, using delay setting from config file
|
||||
await asyncio.sleep(config.CRAWLER_MAX_SLEEP_SEC)
|
||||
|
||||
# 获取页面内容(这个接口返回JSON)
|
||||
# Get page content (this API returns JSON)
|
||||
page_content = await self.playwright_page.content()
|
||||
|
||||
# 提取JSON数据(页面会包含<pre>标签或直接是JSON)
|
||||
# Extract JSON data (page will contain <pre> tag or is directly JSON)
|
||||
try:
|
||||
# 尝试从页面中提取JSON
|
||||
# Try to extract JSON from page
|
||||
json_text = await self.playwright_page.evaluate("() => document.body.innerText")
|
||||
result = json.loads(json_text)
|
||||
utils.logger.info(f"[BaiduTieBaClient.get_notes_by_creator] 成功获取创作者帖子数据")
|
||||
utils.logger.info(f"[BaiduTieBaClient.get_notes_by_creator] Successfully retrieved creator post data")
|
||||
return result
|
||||
except json.JSONDecodeError as e:
|
||||
utils.logger.error(f"[BaiduTieBaClient.get_notes_by_creator] JSON解析失败: {e}")
|
||||
utils.logger.error(f"[BaiduTieBaClient.get_notes_by_creator] 页面内容: {page_content[:500]}")
|
||||
utils.logger.error(f"[BaiduTieBaClient.get_notes_by_creator] JSON parsing failed: {e}")
|
||||
utils.logger.error(f"[BaiduTieBaClient.get_notes_by_creator] Page content: {page_content[:500]}")
|
||||
raise Exception(f"Failed to parse JSON from creator notes page: {e}")
|
||||
|
||||
except Exception as e:
|
||||
utils.logger.error(f"[BaiduTieBaClient.get_notes_by_creator] 获取创作者帖子列表失败: {e}")
|
||||
utils.logger.error(f"[BaiduTieBaClient.get_notes_by_creator] Failed to get creator post list: {e}")
|
||||
raise
|
||||
|
||||
async def get_all_notes_by_creator_user_name(
|
||||
@@ -612,18 +612,18 @@ class BaiduTieBaClient(AbstractApiClient):
|
||||
creator_page_html_content: str = None,
|
||||
) -> List[TiebaNote]:
|
||||
"""
|
||||
根据创作者用户名获取创作者所有帖子
|
||||
Get all creator posts by creator username
|
||||
Args:
|
||||
user_name: 创作者用户名
|
||||
crawl_interval: 爬取一次笔记的延迟单位（秒）
|
||||
callback: 一次笔记爬取结束后的回调函数，是一个awaitable类型的函数
|
||||
max_note_count: 帖子最大获取数量，如果为0则获取所有
|
||||
creator_page_html_content: 创作者主页HTML内容
|
||||
user_name: Creator username
|
||||
crawl_interval: Crawl delay interval in seconds
|
||||
callback: Callback function after one post crawl completes, an awaitable function
|
||||
max_note_count: Maximum number of posts to retrieve, if 0 then get all
|
||||
creator_page_html_content: Creator homepage HTML content
|
||||
|
||||
Returns:
|
||||
|
||||
"""
|
||||
# 百度贴吧比较特殊一些，前10个帖子是直接展示在主页上的，要单独处理，通过API获取不到
|
||||
# Baidu Tieba is special, the first 10 posts are directly displayed on the homepage and need special handling, cannot be obtained through API
|
||||
result: List[TiebaNote] = []
|
||||
if creator_page_html_content:
|
||||
thread_id_list = (self._page_extractor.extract_tieba_thread_id_list_from_creator_page(creator_page_html_content))
|
||||
|
||||
Reference in New Issue
Block a user