mirror of
https://github.com/NanmiCoder/MediaCrawler.git
synced 2026-06-09 03:17:25 +08:00
i18n: translate all Chinese comments, docstrings, and logger messages to English
Comprehensive translation of Chinese text to English across the entire codebase: - api/: FastAPI server documentation and logger messages - cache/: Cache abstraction layer comments and docstrings - database/: Database models and MongoDB store documentation - media_platform/: All platform crawlers (Bilibili, Douyin, Kuaishou, Tieba, Weibo, Xiaohongshu, Zhihu) - model/: Data model documentation - proxy/: Proxy pool and provider documentation - store/: Data storage layer comments - tools/: Utility functions and browser automation - test/: Test file documentation Preserved: Chinese disclaimer header (lines 10-18) for legal compliance 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
@@ -48,7 +48,7 @@ class BaiduTieBaClient(AbstractApiClient):
|
||||
):
|
||||
self.ip_pool: Optional[ProxyIpPool] = ip_pool
|
||||
self.timeout = timeout
|
||||
# 使用传入的headers(包含真实浏览器UA)或默认headers
|
||||
# Use provided headers (including real browser UA) or default headers
|
||||
self.headers = headers or {
|
||||
"User-Agent": utils.get_user_agent(),
|
||||
"Cookie": "",
|
||||
@@ -56,21 +56,21 @@ class BaiduTieBaClient(AbstractApiClient):
|
||||
self._host = "https://tieba.baidu.com"
|
||||
self._page_extractor = TieBaExtractor()
|
||||
self.default_ip_proxy = default_ip_proxy
|
||||
self.playwright_page = playwright_page # Playwright页面对象
|
||||
self.playwright_page = playwright_page # Playwright page object
|
||||
|
||||
def _sync_request(self, method, url, proxy=None, **kwargs):
|
||||
"""
|
||||
同步的requests请求方法
|
||||
Synchronous requests method
|
||||
Args:
|
||||
method: 请求方法
|
||||
url: 请求的URL
|
||||
proxy: 代理IP
|
||||
**kwargs: 其他请求参数
|
||||
method: Request method
|
||||
url: Request URL
|
||||
proxy: Proxy IP
|
||||
**kwargs: Other request parameters
|
||||
|
||||
Returns:
|
||||
response对象
|
||||
Response object
|
||||
"""
|
||||
# 构造代理字典
|
||||
# Construct proxy dictionary
|
||||
proxies = None
|
||||
if proxy:
|
||||
proxies = {
|
||||
@@ -78,7 +78,7 @@ class BaiduTieBaClient(AbstractApiClient):
|
||||
"https": proxy,
|
||||
}
|
||||
|
||||
# 发送请求
|
||||
# Send request
|
||||
response = requests.request(
|
||||
method=method,
|
||||
url=url,
|
||||
@@ -91,7 +91,7 @@ class BaiduTieBaClient(AbstractApiClient):
|
||||
|
||||
async def _refresh_proxy_if_expired(self) -> None:
|
||||
"""
|
||||
检测代理是否过期,如果过期则自动刷新
|
||||
Check if proxy is expired and automatically refresh if necessary
|
||||
"""
|
||||
if self.ip_pool is None:
|
||||
return
|
||||
@@ -101,7 +101,7 @@ class BaiduTieBaClient(AbstractApiClient):
|
||||
"[BaiduTieBaClient._refresh_proxy_if_expired] Proxy expired, refreshing..."
|
||||
)
|
||||
new_proxy = await self.ip_pool.get_or_refresh_proxy()
|
||||
# 更新代理URL
|
||||
# Update proxy URL
|
||||
_, self.default_ip_proxy = utils.format_proxy_info(new_proxy)
|
||||
utils.logger.info(
|
||||
f"[BaiduTieBaClient._refresh_proxy_if_expired] New proxy: {new_proxy.ip}:{new_proxy.port}"
|
||||
@@ -110,23 +110,23 @@ class BaiduTieBaClient(AbstractApiClient):
|
||||
@retry(stop=stop_after_attempt(3), wait=wait_fixed(1))
|
||||
async def request(self, method, url, return_ori_content=False, proxy=None, **kwargs) -> Union[str, Any]:
|
||||
"""
|
||||
封装requests的公共请求方法,对请求响应做一些处理
|
||||
Common request method wrapper for requests, handles request responses
|
||||
Args:
|
||||
method: 请求方法
|
||||
url: 请求的URL
|
||||
return_ori_content: 是否返回原始内容
|
||||
proxy: 代理IP
|
||||
**kwargs: 其他请求参数,例如请求头、请求体等
|
||||
method: Request method
|
||||
url: Request URL
|
||||
return_ori_content: Whether to return original content
|
||||
proxy: Proxy IP
|
||||
**kwargs: Other request parameters, such as headers, request body, etc.
|
||||
|
||||
Returns:
|
||||
|
||||
"""
|
||||
# 每次请求前检测代理是否过期
|
||||
# Check if proxy is expired before each request
|
||||
await self._refresh_proxy_if_expired()
|
||||
|
||||
actual_proxy = proxy if proxy else self.default_ip_proxy
|
||||
|
||||
# 在线程池中执行同步的requests请求
|
||||
# Execute synchronous requests in thread pool
|
||||
response = await asyncio.to_thread(
|
||||
self._sync_request,
|
||||
method,
|
||||
@@ -151,11 +151,11 @@ class BaiduTieBaClient(AbstractApiClient):
|
||||
|
||||
async def get(self, uri: str, params=None, return_ori_content=False, **kwargs) -> Any:
|
||||
"""
|
||||
GET请求,对请求头签名
|
||||
GET request with header signing
|
||||
Args:
|
||||
uri: 请求路由
|
||||
params: 请求参数
|
||||
return_ori_content: 是否返回原始内容
|
||||
uri: Request route
|
||||
params: Request parameters
|
||||
return_ori_content: Whether to return original content
|
||||
|
||||
Returns:
|
||||
|
||||
@@ -175,15 +175,15 @@ class BaiduTieBaClient(AbstractApiClient):
|
||||
self.default_ip_proxy = proxy
|
||||
return res
|
||||
|
||||
utils.logger.error(f"[BaiduTieBaClient.get] 达到了最大重试次数,IP已经被Block,请尝试更换新的IP代理: {e}")
|
||||
raise Exception(f"[BaiduTieBaClient.get] 达到了最大重试次数,IP已经被Block,请尝试更换新的IP代理: {e}")
|
||||
utils.logger.error(f"[BaiduTieBaClient.get] Reached maximum retry attempts, IP is blocked, please try a new IP proxy: {e}")
|
||||
raise Exception(f"[BaiduTieBaClient.get] Reached maximum retry attempts, IP is blocked, please try a new IP proxy: {e}")
|
||||
|
||||
async def post(self, uri: str, data: dict, **kwargs) -> Dict:
|
||||
"""
|
||||
POST请求,对请求头签名
|
||||
POST request with header signing
|
||||
Args:
|
||||
uri: 请求路由
|
||||
data: 请求体参数
|
||||
uri: Request route
|
||||
data: Request body parameters
|
||||
|
||||
Returns:
|
||||
|
||||
@@ -193,13 +193,13 @@ class BaiduTieBaClient(AbstractApiClient):
|
||||
|
||||
async def pong(self, browser_context: BrowserContext = None) -> bool:
|
||||
"""
|
||||
用于检查登录态是否失效了
|
||||
使用Cookie检测而非API调用,避免被检测
|
||||
Check if login state is still valid
|
||||
Uses Cookie detection instead of API calls to avoid detection
|
||||
Args:
|
||||
browser_context: 浏览器上下文对象
|
||||
browser_context: Browser context object
|
||||
|
||||
Returns:
|
||||
bool: True表示已登录,False表示未登录
|
||||
bool: True if logged in, False if not logged in
|
||||
"""
|
||||
utils.logger.info("[BaiduTieBaClient.pong] Begin to check tieba login state by cookies...")
|
||||
|
||||
@@ -208,13 +208,13 @@ class BaiduTieBaClient(AbstractApiClient):
|
||||
return False
|
||||
|
||||
try:
|
||||
# 从浏览器获取cookies并检查关键登录cookie
|
||||
# Get cookies from browser and check key login cookies
|
||||
_, cookie_dict = utils.convert_cookies(await browser_context.cookies())
|
||||
|
||||
# 百度贴吧的登录标识: STOKEN 或 PTOKEN
|
||||
# Baidu Tieba login identifiers: STOKEN or PTOKEN
|
||||
stoken = cookie_dict.get("STOKEN")
|
||||
ptoken = cookie_dict.get("PTOKEN")
|
||||
bduss = cookie_dict.get("BDUSS") # 百度通用登录cookie
|
||||
bduss = cookie_dict.get("BDUSS") # Baidu universal login cookie
|
||||
|
||||
if stoken or ptoken or bduss:
|
||||
utils.logger.info(f"[BaiduTieBaClient.pong] Login state verified by cookies (STOKEN: {bool(stoken)}, PTOKEN: {bool(ptoken)}, BDUSS: {bool(bduss)})")
|
||||
@@ -229,9 +229,9 @@ class BaiduTieBaClient(AbstractApiClient):
|
||||
|
||||
async def update_cookies(self, browser_context: BrowserContext):
|
||||
"""
|
||||
API客户端提供的更新cookies方法,一般情况下登录成功后会调用此方法
|
||||
Update cookies method provided by API client, usually called after successful login
|
||||
Args:
|
||||
browser_context: 浏览器上下文对象
|
||||
browser_context: Browser context object
|
||||
|
||||
Returns:
|
||||
|
||||
@@ -249,13 +249,13 @@ class BaiduTieBaClient(AbstractApiClient):
|
||||
note_type: SearchNoteType = SearchNoteType.FIXED_THREAD,
|
||||
) -> List[TiebaNote]:
|
||||
"""
|
||||
根据关键词搜索贴吧帖子 (使用Playwright访问页面,避免API检测)
|
||||
Search Tieba posts by keyword (uses Playwright to access page, avoiding API detection)
|
||||
Args:
|
||||
keyword: 关键词
|
||||
page: 分页第几页
|
||||
page_size: 每页大小
|
||||
sort: 结果排序方式
|
||||
note_type: 帖子类型(主题贴|主题+回复混合模式)
|
||||
keyword: Keyword
|
||||
page: Page number
|
||||
page_size: Page size
|
||||
sort: Result sort method
|
||||
note_type: Post type (main thread | main thread + reply mixed mode)
|
||||
Returns:
|
||||
|
||||
"""
|
||||
@@ -263,8 +263,8 @@ class BaiduTieBaClient(AbstractApiClient):
|
||||
utils.logger.error("[BaiduTieBaClient.get_notes_by_keyword] playwright_page is None, cannot use browser mode")
|
||||
raise Exception("playwright_page is required for browser-based search")
|
||||
|
||||
# 构造搜索URL
|
||||
# 示例: https://tieba.baidu.com/f/search/res?ie=utf-8&qw=编程
|
||||
# Construct search URL
|
||||
# Example: https://tieba.baidu.com/f/search/res?ie=utf-8&qw=keyword
|
||||
search_url = f"{self._host}/f/search/res"
|
||||
params = {
|
||||
"ie": "utf-8",
|
||||
@@ -275,64 +275,64 @@ class BaiduTieBaClient(AbstractApiClient):
|
||||
"only_thread": note_type.value,
|
||||
}
|
||||
|
||||
# 拼接完整URL
|
||||
# Concatenate full URL
|
||||
full_url = f"{search_url}?{urlencode(params)}"
|
||||
utils.logger.info(f"[BaiduTieBaClient.get_notes_by_keyword] 访问搜索页面: {full_url}")
|
||||
utils.logger.info(f"[BaiduTieBaClient.get_notes_by_keyword] Accessing search page: {full_url}")
|
||||
|
||||
try:
|
||||
# 使用Playwright访问搜索页面
|
||||
# Use Playwright to access search page
|
||||
await self.playwright_page.goto(full_url, wait_until="domcontentloaded")
|
||||
|
||||
# 等待页面加载,使用配置文件中的延时设置
|
||||
# Wait for page loading, using delay setting from config file
|
||||
await asyncio.sleep(config.CRAWLER_MAX_SLEEP_SEC)
|
||||
|
||||
# 获取页面HTML内容
|
||||
# Get page HTML content
|
||||
page_content = await self.playwright_page.content()
|
||||
utils.logger.info(f"[BaiduTieBaClient.get_notes_by_keyword] 成功获取搜索页面HTML,长度: {len(page_content)}")
|
||||
utils.logger.info(f"[BaiduTieBaClient.get_notes_by_keyword] Successfully retrieved search page HTML, length: {len(page_content)}")
|
||||
|
||||
# 提取搜索结果
|
||||
# Extract search results
|
||||
notes = self._page_extractor.extract_search_note_list(page_content)
|
||||
utils.logger.info(f"[BaiduTieBaClient.get_notes_by_keyword] 提取到 {len(notes)} 条帖子")
|
||||
utils.logger.info(f"[BaiduTieBaClient.get_notes_by_keyword] Extracted {len(notes)} posts")
|
||||
return notes
|
||||
|
||||
except Exception as e:
|
||||
utils.logger.error(f"[BaiduTieBaClient.get_notes_by_keyword] 搜索失败: {e}")
|
||||
utils.logger.error(f"[BaiduTieBaClient.get_notes_by_keyword] Search failed: {e}")
|
||||
raise
|
||||
|
||||
async def get_note_by_id(self, note_id: str) -> TiebaNote:
|
||||
"""
|
||||
根据帖子ID获取帖子详情 (使用Playwright访问页面,避免API检测)
|
||||
Get post details by post ID (uses Playwright to access page, avoiding API detection)
|
||||
Args:
|
||||
note_id: 帖子ID
|
||||
note_id: Post ID
|
||||
|
||||
Returns:
|
||||
TiebaNote: 帖子详情对象
|
||||
TiebaNote: Post detail object
|
||||
"""
|
||||
if not self.playwright_page:
|
||||
utils.logger.error("[BaiduTieBaClient.get_note_by_id] playwright_page is None, cannot use browser mode")
|
||||
raise Exception("playwright_page is required for browser-based note detail fetching")
|
||||
|
||||
# 构造帖子详情URL
|
||||
# Construct post detail URL
|
||||
note_url = f"{self._host}/p/{note_id}"
|
||||
utils.logger.info(f"[BaiduTieBaClient.get_note_by_id] 访问帖子详情页面: {note_url}")
|
||||
utils.logger.info(f"[BaiduTieBaClient.get_note_by_id] Accessing post detail page: {note_url}")
|
||||
|
||||
try:
|
||||
# 使用Playwright访问帖子详情页面
|
||||
# Use Playwright to access post detail page
|
||||
await self.playwright_page.goto(note_url, wait_until="domcontentloaded")
|
||||
|
||||
# 等待页面加载,使用配置文件中的延时设置
|
||||
# Wait for page loading, using delay setting from config file
|
||||
await asyncio.sleep(config.CRAWLER_MAX_SLEEP_SEC)
|
||||
|
||||
# 获取页面HTML内容
|
||||
# Get page HTML content
|
||||
page_content = await self.playwright_page.content()
|
||||
utils.logger.info(f"[BaiduTieBaClient.get_note_by_id] 成功获取帖子详情HTML,长度: {len(page_content)}")
|
||||
utils.logger.info(f"[BaiduTieBaClient.get_note_by_id] Successfully retrieved post detail HTML, length: {len(page_content)}")
|
||||
|
||||
# 提取帖子详情
|
||||
# Extract post details
|
||||
note_detail = self._page_extractor.extract_note_detail(page_content)
|
||||
return note_detail
|
||||
|
||||
except Exception as e:
|
||||
utils.logger.error(f"[BaiduTieBaClient.get_note_by_id] 获取帖子详情失败: {e}")
|
||||
utils.logger.error(f"[BaiduTieBaClient.get_note_by_id] Failed to get post details: {e}")
|
||||
raise
|
||||
|
||||
async def get_note_all_comments(
|
||||
@@ -343,14 +343,14 @@ class BaiduTieBaClient(AbstractApiClient):
|
||||
max_count: int = 10,
|
||||
) -> List[TiebaComment]:
|
||||
"""
|
||||
获取指定帖子下的所有一级评论 (使用Playwright访问页面,避免API检测)
|
||||
Get all first-level comments for specified post (uses Playwright to access page, avoiding API detection)
|
||||
Args:
|
||||
note_detail: 帖子详情对象
|
||||
crawl_interval: 爬取一次笔记的延迟单位(秒)
|
||||
callback: 一次笔记爬取结束后的回调函数
|
||||
max_count: 一次帖子爬取的最大评论数量
|
||||
note_detail: Post detail object
|
||||
crawl_interval: Crawl delay interval in seconds
|
||||
callback: Callback function after one post crawl completes
|
||||
max_count: Maximum number of comments to crawl per post
|
||||
Returns:
|
||||
List[TiebaComment]: 评论列表
|
||||
List[TiebaComment]: Comment list
|
||||
"""
|
||||
if not self.playwright_page:
|
||||
utils.logger.error("[BaiduTieBaClient.get_note_all_comments] playwright_page is None, cannot use browser mode")
|
||||
@@ -360,30 +360,30 @@ class BaiduTieBaClient(AbstractApiClient):
|
||||
current_page = 1
|
||||
|
||||
while note_detail.total_replay_page >= current_page and len(result) < max_count:
|
||||
# 构造评论页URL
|
||||
# Construct comment page URL
|
||||
comment_url = f"{self._host}/p/{note_detail.note_id}?pn={current_page}"
|
||||
utils.logger.info(f"[BaiduTieBaClient.get_note_all_comments] 访问评论页面: {comment_url}")
|
||||
utils.logger.info(f"[BaiduTieBaClient.get_note_all_comments] Accessing comment page: {comment_url}")
|
||||
|
||||
try:
|
||||
# 使用Playwright访问评论页面
|
||||
# Use Playwright to access comment page
|
||||
await self.playwright_page.goto(comment_url, wait_until="domcontentloaded")
|
||||
|
||||
# 等待页面加载,使用配置文件中的延时设置
|
||||
# Wait for page loading, using delay setting from config file
|
||||
await asyncio.sleep(config.CRAWLER_MAX_SLEEP_SEC)
|
||||
|
||||
# 获取页面HTML内容
|
||||
# Get page HTML content
|
||||
page_content = await self.playwright_page.content()
|
||||
|
||||
# 提取评论
|
||||
# Extract comments
|
||||
comments = self._page_extractor.extract_tieba_note_parment_comments(
|
||||
page_content, note_id=note_detail.note_id
|
||||
)
|
||||
|
||||
if not comments:
|
||||
utils.logger.info(f"[BaiduTieBaClient.get_note_all_comments] 第{current_page}页没有评论,停止爬取")
|
||||
utils.logger.info(f"[BaiduTieBaClient.get_note_all_comments] Page {current_page} has no comments, stopping crawl")
|
||||
break
|
||||
|
||||
# 限制评论数量
|
||||
# Limit comment count
|
||||
if len(result) + len(comments) > max_count:
|
||||
comments = comments[:max_count - len(result)]
|
||||
|
||||
@@ -392,7 +392,7 @@ class BaiduTieBaClient(AbstractApiClient):
|
||||
|
||||
result.extend(comments)
|
||||
|
||||
# 获取所有子评论
|
||||
# Get all sub-comments
|
||||
await self.get_comments_all_sub_comments(
|
||||
comments, crawl_interval=crawl_interval, callback=callback
|
||||
)
|
||||
@@ -401,10 +401,10 @@ class BaiduTieBaClient(AbstractApiClient):
|
||||
current_page += 1
|
||||
|
||||
except Exception as e:
|
||||
utils.logger.error(f"[BaiduTieBaClient.get_note_all_comments] 获取第{current_page}页评论失败: {e}")
|
||||
utils.logger.error(f"[BaiduTieBaClient.get_note_all_comments] Failed to get page {current_page} comments: {e}")
|
||||
break
|
||||
|
||||
utils.logger.info(f"[BaiduTieBaClient.get_note_all_comments] 共获取 {len(result)} 条一级评论")
|
||||
utils.logger.info(f"[BaiduTieBaClient.get_note_all_comments] Total retrieved {len(result)} first-level comments")
|
||||
return result
|
||||
|
||||
async def get_comments_all_sub_comments(
|
||||
@@ -414,14 +414,14 @@ class BaiduTieBaClient(AbstractApiClient):
|
||||
callback: Optional[Callable] = None,
|
||||
) -> List[TiebaComment]:
|
||||
"""
|
||||
获取指定评论下的所有子评论 (使用Playwright访问页面,避免API检测)
|
||||
Get all sub-comments for specified comments (uses Playwright to access page, avoiding API detection)
|
||||
Args:
|
||||
comments: 评论列表
|
||||
crawl_interval: 爬取一次笔记的延迟单位(秒)
|
||||
callback: 一次笔记爬取结束后的回调函数
|
||||
comments: Comment list
|
||||
crawl_interval: Crawl delay interval in seconds
|
||||
callback: Callback function after one post crawl completes
|
||||
|
||||
Returns:
|
||||
List[TiebaComment]: 子评论列表
|
||||
List[TiebaComment]: Sub-comment list
|
||||
"""
|
||||
if not config.ENABLE_GET_SUB_COMMENTS:
|
||||
return []
|
||||
@@ -440,7 +440,7 @@ class BaiduTieBaClient(AbstractApiClient):
|
||||
max_sub_page_num = parment_comment.sub_comment_count // 10 + 1
|
||||
|
||||
while max_sub_page_num >= current_page:
|
||||
# 构造子评论URL
|
||||
# Construct sub-comment URL
|
||||
sub_comment_url = (
|
||||
f"{self._host}/p/comment?"
|
||||
f"tid={parment_comment.note_id}&"
|
||||
@@ -448,19 +448,19 @@ class BaiduTieBaClient(AbstractApiClient):
|
||||
f"fid={parment_comment.tieba_id}&"
|
||||
f"pn={current_page}"
|
||||
)
|
||||
utils.logger.info(f"[BaiduTieBaClient.get_comments_all_sub_comments] 访问子评论页面: {sub_comment_url}")
|
||||
utils.logger.info(f"[BaiduTieBaClient.get_comments_all_sub_comments] Accessing sub-comment page: {sub_comment_url}")
|
||||
|
||||
try:
|
||||
# 使用Playwright访问子评论页面
|
||||
# Use Playwright to access sub-comment page
|
||||
await self.playwright_page.goto(sub_comment_url, wait_until="domcontentloaded")
|
||||
|
||||
# 等待页面加载,使用配置文件中的延时设置
|
||||
# Wait for page loading, using delay setting from config file
|
||||
await asyncio.sleep(config.CRAWLER_MAX_SLEEP_SEC)
|
||||
|
||||
# 获取页面HTML内容
|
||||
# Get page HTML content
|
||||
page_content = await self.playwright_page.content()
|
||||
|
||||
# 提取子评论
|
||||
# Extract sub-comments
|
||||
sub_comments = self._page_extractor.extract_tieba_note_sub_comments(
|
||||
page_content, parent_comment=parment_comment
|
||||
)
|
||||
@@ -468,7 +468,7 @@ class BaiduTieBaClient(AbstractApiClient):
|
||||
if not sub_comments:
|
||||
utils.logger.info(
|
||||
f"[BaiduTieBaClient.get_comments_all_sub_comments] "
|
||||
f"评论{parment_comment.comment_id}第{current_page}页没有子评论,停止爬取"
|
||||
f"Comment {parment_comment.comment_id} page {current_page} has no sub-comments, stopping crawl"
|
||||
)
|
||||
break
|
||||
|
||||
@@ -482,125 +482,125 @@ class BaiduTieBaClient(AbstractApiClient):
|
||||
except Exception as e:
|
||||
utils.logger.error(
|
||||
f"[BaiduTieBaClient.get_comments_all_sub_comments] "
|
||||
f"获取评论{parment_comment.comment_id}第{current_page}页子评论失败: {e}"
|
||||
f"Failed to get comment {parment_comment.comment_id} page {current_page} sub-comments: {e}"
|
||||
)
|
||||
break
|
||||
|
||||
utils.logger.info(f"[BaiduTieBaClient.get_comments_all_sub_comments] 共获取 {len(all_sub_comments)} 条子评论")
|
||||
utils.logger.info(f"[BaiduTieBaClient.get_comments_all_sub_comments] Total retrieved {len(all_sub_comments)} sub-comments")
|
||||
return all_sub_comments
|
||||
|
||||
async def get_notes_by_tieba_name(self, tieba_name: str, page_num: int) -> List[TiebaNote]:
|
||||
"""
|
||||
根据贴吧名称获取帖子列表 (使用Playwright访问页面,避免API检测)
|
||||
Get post list by Tieba name (uses Playwright to access page, avoiding API detection)
|
||||
Args:
|
||||
tieba_name: 贴吧名称
|
||||
page_num: 分页页码
|
||||
tieba_name: Tieba name
|
||||
page_num: Page number
|
||||
|
||||
Returns:
|
||||
List[TiebaNote]: 帖子列表
|
||||
List[TiebaNote]: Post list
|
||||
"""
|
||||
if not self.playwright_page:
|
||||
utils.logger.error("[BaiduTieBaClient.get_notes_by_tieba_name] playwright_page is None, cannot use browser mode")
|
||||
raise Exception("playwright_page is required for browser-based tieba note fetching")
|
||||
|
||||
# 构造贴吧帖子列表URL
|
||||
# Construct Tieba post list URL
|
||||
tieba_url = f"{self._host}/f?kw={quote(tieba_name)}&pn={page_num}"
|
||||
utils.logger.info(f"[BaiduTieBaClient.get_notes_by_tieba_name] 访问贴吧页面: {tieba_url}")
|
||||
utils.logger.info(f"[BaiduTieBaClient.get_notes_by_tieba_name] Accessing Tieba page: {tieba_url}")
|
||||
|
||||
try:
|
||||
# 使用Playwright访问贴吧页面
|
||||
# Use Playwright to access Tieba page
|
||||
await self.playwright_page.goto(tieba_url, wait_until="domcontentloaded")
|
||||
|
||||
# 等待页面加载,使用配置文件中的延时设置
|
||||
# Wait for page loading, using delay setting from config file
|
||||
await asyncio.sleep(config.CRAWLER_MAX_SLEEP_SEC)
|
||||
|
||||
# 获取页面HTML内容
|
||||
# Get page HTML content
|
||||
page_content = await self.playwright_page.content()
|
||||
utils.logger.info(f"[BaiduTieBaClient.get_notes_by_tieba_name] 成功获取贴吧页面HTML,长度: {len(page_content)}")
|
||||
utils.logger.info(f"[BaiduTieBaClient.get_notes_by_tieba_name] Successfully retrieved Tieba page HTML, length: {len(page_content)}")
|
||||
|
||||
# 提取帖子列表
|
||||
# Extract post list
|
||||
notes = self._page_extractor.extract_tieba_note_list(page_content)
|
||||
utils.logger.info(f"[BaiduTieBaClient.get_notes_by_tieba_name] 提取到 {len(notes)} 条帖子")
|
||||
utils.logger.info(f"[BaiduTieBaClient.get_notes_by_tieba_name] Extracted {len(notes)} posts")
|
||||
return notes
|
||||
|
||||
except Exception as e:
|
||||
utils.logger.error(f"[BaiduTieBaClient.get_notes_by_tieba_name] 获取贴吧帖子列表失败: {e}")
|
||||
utils.logger.error(f"[BaiduTieBaClient.get_notes_by_tieba_name] Failed to get Tieba post list: {e}")
|
||||
raise
|
||||
|
||||
async def get_creator_info_by_url(self, creator_url: str) -> str:
|
||||
"""
|
||||
根据创作者URL获取创作者信息 (使用Playwright访问页面,避免API检测)
|
||||
Get creator information by creator URL (uses Playwright to access page, avoiding API detection)
|
||||
Args:
|
||||
creator_url: 创作者主页URL
|
||||
creator_url: Creator homepage URL
|
||||
|
||||
Returns:
|
||||
str: 页面HTML内容
|
||||
str: Page HTML content
|
||||
"""
|
||||
if not self.playwright_page:
|
||||
utils.logger.error("[BaiduTieBaClient.get_creator_info_by_url] playwright_page is None, cannot use browser mode")
|
||||
raise Exception("playwright_page is required for browser-based creator info fetching")
|
||||
|
||||
utils.logger.info(f"[BaiduTieBaClient.get_creator_info_by_url] 访问创作者主页: {creator_url}")
|
||||
utils.logger.info(f"[BaiduTieBaClient.get_creator_info_by_url] Accessing creator homepage: {creator_url}")
|
||||
|
||||
try:
|
||||
# 使用Playwright访问创作者主页
|
||||
# Use Playwright to access creator homepage
|
||||
await self.playwright_page.goto(creator_url, wait_until="domcontentloaded")
|
||||
|
||||
# 等待页面加载,使用配置文件中的延时设置
|
||||
# Wait for page loading, using delay setting from config file
|
||||
await asyncio.sleep(config.CRAWLER_MAX_SLEEP_SEC)
|
||||
|
||||
# 获取页面HTML内容
|
||||
# Get page HTML content
|
||||
page_content = await self.playwright_page.content()
|
||||
utils.logger.info(f"[BaiduTieBaClient.get_creator_info_by_url] 成功获取创作者主页HTML,长度: {len(page_content)}")
|
||||
utils.logger.info(f"[BaiduTieBaClient.get_creator_info_by_url] Successfully retrieved creator homepage HTML, length: {len(page_content)}")
|
||||
|
||||
return page_content
|
||||
|
||||
except Exception as e:
|
||||
utils.logger.error(f"[BaiduTieBaClient.get_creator_info_by_url] 获取创作者主页失败: {e}")
|
||||
utils.logger.error(f"[BaiduTieBaClient.get_creator_info_by_url] Failed to get creator homepage: {e}")
|
||||
raise
|
||||
|
||||
async def get_notes_by_creator(self, user_name: str, page_number: int) -> Dict:
|
||||
"""
|
||||
根据创作者获取创作者的帖子 (使用Playwright访问页面,避免API检测)
|
||||
Get creator's posts by creator (uses Playwright to access page, avoiding API detection)
|
||||
Args:
|
||||
user_name: 创作者用户名
|
||||
page_number: 页码
|
||||
user_name: Creator username
|
||||
page_number: Page number
|
||||
|
||||
Returns:
|
||||
Dict: 包含帖子数据的字典
|
||||
Dict: Dictionary containing post data
|
||||
"""
|
||||
if not self.playwright_page:
|
||||
utils.logger.error("[BaiduTieBaClient.get_notes_by_creator] playwright_page is None, cannot use browser mode")
|
||||
raise Exception("playwright_page is required for browser-based creator notes fetching")
|
||||
|
||||
# 构造创作者帖子列表URL
|
||||
# Construct creator post list URL
|
||||
creator_url = f"{self._host}/home/get/getthread?un={quote(user_name)}&pn={page_number}&id=utf-8&_={utils.get_current_timestamp()}"
|
||||
utils.logger.info(f"[BaiduTieBaClient.get_notes_by_creator] 访问创作者帖子列表: {creator_url}")
|
||||
utils.logger.info(f"[BaiduTieBaClient.get_notes_by_creator] Accessing creator post list: {creator_url}")
|
||||
|
||||
try:
|
||||
# 使用Playwright访问创作者帖子列表页面
|
||||
# Use Playwright to access creator post list page
|
||||
await self.playwright_page.goto(creator_url, wait_until="domcontentloaded")
|
||||
|
||||
# 等待页面加载,使用配置文件中的延时设置
|
||||
# Wait for page loading, using delay setting from config file
|
||||
await asyncio.sleep(config.CRAWLER_MAX_SLEEP_SEC)
|
||||
|
||||
# 获取页面内容(这个接口返回JSON)
|
||||
# Get page content (this API returns JSON)
|
||||
page_content = await self.playwright_page.content()
|
||||
|
||||
# 提取JSON数据(页面会包含<pre>标签或直接是JSON)
|
||||
# Extract JSON data (page will contain <pre> tag or is directly JSON)
|
||||
try:
|
||||
# 尝试从页面中提取JSON
|
||||
# Try to extract JSON from page
|
||||
json_text = await self.playwright_page.evaluate("() => document.body.innerText")
|
||||
result = json.loads(json_text)
|
||||
utils.logger.info(f"[BaiduTieBaClient.get_notes_by_creator] 成功获取创作者帖子数据")
|
||||
utils.logger.info(f"[BaiduTieBaClient.get_notes_by_creator] Successfully retrieved creator post data")
|
||||
return result
|
||||
except json.JSONDecodeError as e:
|
||||
utils.logger.error(f"[BaiduTieBaClient.get_notes_by_creator] JSON解析失败: {e}")
|
||||
utils.logger.error(f"[BaiduTieBaClient.get_notes_by_creator] 页面内容: {page_content[:500]}")
|
||||
utils.logger.error(f"[BaiduTieBaClient.get_notes_by_creator] JSON parsing failed: {e}")
|
||||
utils.logger.error(f"[BaiduTieBaClient.get_notes_by_creator] Page content: {page_content[:500]}")
|
||||
raise Exception(f"Failed to parse JSON from creator notes page: {e}")
|
||||
|
||||
except Exception as e:
|
||||
utils.logger.error(f"[BaiduTieBaClient.get_notes_by_creator] 获取创作者帖子列表失败: {e}")
|
||||
utils.logger.error(f"[BaiduTieBaClient.get_notes_by_creator] Failed to get creator post list: {e}")
|
||||
raise
|
||||
|
||||
async def get_all_notes_by_creator_user_name(
|
||||
@@ -612,18 +612,18 @@ class BaiduTieBaClient(AbstractApiClient):
|
||||
creator_page_html_content: str = None,
|
||||
) -> List[TiebaNote]:
|
||||
"""
|
||||
根据创作者用户名获取创作者所有帖子
|
||||
Get all creator posts by creator username
|
||||
Args:
|
||||
user_name: 创作者用户名
|
||||
crawl_interval: 爬取一次笔记的延迟单位(秒)
|
||||
callback: 一次笔记爬取结束后的回调函数,是一个awaitable类型的函数
|
||||
max_note_count: 帖子最大获取数量,如果为0则获取所有
|
||||
creator_page_html_content: 创作者主页HTML内容
|
||||
user_name: Creator username
|
||||
crawl_interval: Crawl delay interval in seconds
|
||||
callback: Callback function after one post crawl completes, an awaitable function
|
||||
max_note_count: Maximum number of posts to retrieve, if 0 then get all
|
||||
creator_page_html_content: Creator homepage HTML content
|
||||
|
||||
Returns:
|
||||
|
||||
"""
|
||||
# 百度贴吧比较特殊一些,前10个帖子是直接展示在主页上的,要单独处理,通过API获取不到
|
||||
# Baidu Tieba is special, the first 10 posts are directly displayed on the homepage and need special handling, cannot be obtained through API
|
||||
result: List[TiebaNote] = []
|
||||
if creator_page_html_content:
|
||||
thread_id_list = (self._page_extractor.extract_tieba_thread_id_list_from_creator_page(creator_page_html_content))
|
||||
|
||||
@@ -79,9 +79,9 @@ class TieBaCrawler(AbstractCrawler):
|
||||
)
|
||||
|
||||
async with async_playwright() as playwright:
|
||||
# 根据配置选择启动模式
|
||||
# Choose startup mode based on configuration
|
||||
if config.ENABLE_CDP_MODE:
|
||||
utils.logger.info("[BaiduTieBaCrawler] 使用CDP模式启动浏览器")
|
||||
utils.logger.info("[BaiduTieBaCrawler] Launching browser in CDP mode")
|
||||
self.browser_context = await self.launch_browser_with_cdp(
|
||||
playwright,
|
||||
playwright_proxy_format,
|
||||
@@ -89,7 +89,7 @@ class TieBaCrawler(AbstractCrawler):
|
||||
headless=config.CDP_HEADLESS,
|
||||
)
|
||||
else:
|
||||
utils.logger.info("[BaiduTieBaCrawler] 使用标准模式启动浏览器")
|
||||
utils.logger.info("[BaiduTieBaCrawler] Launching browser in standard mode")
|
||||
# Launch a browser context.
|
||||
chromium = playwright.chromium
|
||||
self.browser_context = await self.launch_browser(
|
||||
@@ -99,12 +99,12 @@ class TieBaCrawler(AbstractCrawler):
|
||||
headless=config.HEADLESS,
|
||||
)
|
||||
|
||||
# 注入反检测脚本 - 针对百度的特殊检测
|
||||
# Inject anti-detection scripts - for Baidu's special detection
|
||||
await self._inject_anti_detection_scripts()
|
||||
|
||||
self.context_page = await self.browser_context.new_page()
|
||||
|
||||
# 先访问百度首页,再点击贴吧链接,避免触发安全验证
|
||||
# First visit Baidu homepage, then click Tieba link to avoid triggering security verification
|
||||
await self._navigate_to_tieba_via_baidu()
|
||||
|
||||
# Create a client to interact with the baidutieba website.
|
||||
@@ -399,29 +399,29 @@ class TieBaCrawler(AbstractCrawler):
|
||||
|
||||
async def _navigate_to_tieba_via_baidu(self):
|
||||
"""
|
||||
模拟真实用户访问路径:
|
||||
1. 先访问百度首页 (https://www.baidu.com/)
|
||||
2. 等待页面加载
|
||||
3. 点击顶部导航栏的"贴吧"链接
|
||||
4. 跳转到贴吧首页
|
||||
Simulate real user access path:
|
||||
1. First visit Baidu homepage (https://www.baidu.com/)
|
||||
2. Wait for page to load
|
||||
3. Click "Tieba" link in top navigation bar
|
||||
4. Jump to Tieba homepage
|
||||
|
||||
这样做可以避免触发百度的安全验证
|
||||
This avoids triggering Baidu's security verification
|
||||
"""
|
||||
utils.logger.info("[TieBaCrawler] 模拟真实用户访问路径...")
|
||||
utils.logger.info("[TieBaCrawler] Simulating real user access path...")
|
||||
|
||||
try:
|
||||
# Step 1: 访问百度首页
|
||||
utils.logger.info("[TieBaCrawler] Step 1: 访问百度首页 https://www.baidu.com/")
|
||||
# Step 1: Visit Baidu homepage
|
||||
utils.logger.info("[TieBaCrawler] Step 1: Visiting Baidu homepage https://www.baidu.com/")
|
||||
await self.context_page.goto("https://www.baidu.com/", wait_until="domcontentloaded")
|
||||
|
||||
# Step 2: 等待页面加载,使用配置文件中的延时设置
|
||||
utils.logger.info(f"[TieBaCrawler] Step 2: 等待 {config.CRAWLER_MAX_SLEEP_SEC}秒 模拟用户浏览...")
|
||||
# Step 2: Wait for page loading, using delay setting from config file
|
||||
utils.logger.info(f"[TieBaCrawler] Step 2: Waiting {config.CRAWLER_MAX_SLEEP_SEC} seconds to simulate user browsing...")
|
||||
await asyncio.sleep(config.CRAWLER_MAX_SLEEP_SEC)
|
||||
|
||||
# Step 3: 查找并点击"贴吧"链接
|
||||
utils.logger.info("[TieBaCrawler] Step 3: 查找并点击'贴吧'链接...")
|
||||
# Step 3: Find and click "Tieba" link
|
||||
utils.logger.info("[TieBaCrawler] Step 3: Finding and clicking 'Tieba' link...")
|
||||
|
||||
# 尝试多种选择器,确保能找到贴吧链接
|
||||
# Try multiple selectors to ensure finding the Tieba link
|
||||
tieba_selectors = [
|
||||
'a[href="http://tieba.baidu.com/"]',
|
||||
'a[href="https://tieba.baidu.com/"]',
|
||||
@@ -434,74 +434,74 @@ class TieBaCrawler(AbstractCrawler):
|
||||
try:
|
||||
tieba_link = await self.context_page.wait_for_selector(selector, timeout=5000)
|
||||
if tieba_link:
|
||||
utils.logger.info(f"[TieBaCrawler] 找到贴吧链接 (selector: {selector})")
|
||||
utils.logger.info(f"[TieBaCrawler] Found Tieba link (selector: {selector})")
|
||||
break
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
if not tieba_link:
|
||||
utils.logger.warning("[TieBaCrawler] 未找到贴吧链接,直接访问贴吧首页")
|
||||
utils.logger.warning("[TieBaCrawler] Tieba link not found, directly accessing Tieba homepage")
|
||||
await self.context_page.goto(self.index_url, wait_until="domcontentloaded")
|
||||
return
|
||||
|
||||
# Step 4: 点击贴吧链接 (检查是否会打开新标签页)
|
||||
utils.logger.info("[TieBaCrawler] Step 4: 点击贴吧链接...")
|
||||
# Step 4: Click Tieba link (check if it will open in a new tab)
|
||||
utils.logger.info("[TieBaCrawler] Step 4: Clicking Tieba link...")
|
||||
|
||||
# 检查链接的target属性
|
||||
# Check link's target attribute
|
||||
target_attr = await tieba_link.get_attribute("target")
|
||||
utils.logger.info(f"[TieBaCrawler] 链接target属性: {target_attr}")
|
||||
utils.logger.info(f"[TieBaCrawler] Link target attribute: {target_attr}")
|
||||
|
||||
if target_attr == "_blank":
|
||||
# 如果是新标签页,需要等待新页面并切换
|
||||
utils.logger.info("[TieBaCrawler] 链接会在新标签页打开,等待新页面...")
|
||||
# If it's a new tab, need to wait for new page and switch
|
||||
utils.logger.info("[TieBaCrawler] Link will open in new tab, waiting for new page...")
|
||||
|
||||
async with self.browser_context.expect_page() as new_page_info:
|
||||
await tieba_link.click()
|
||||
|
||||
# 获取新打开的页面
|
||||
# Get newly opened page
|
||||
new_page = await new_page_info.value
|
||||
await new_page.wait_for_load_state("domcontentloaded")
|
||||
|
||||
# 关闭旧的百度首页
|
||||
# Close old Baidu homepage
|
||||
await self.context_page.close()
|
||||
|
||||
# 切换到新的贴吧页面
|
||||
# Switch to new Tieba page
|
||||
self.context_page = new_page
|
||||
utils.logger.info("[TieBaCrawler] ✅ 已切换到新标签页 (贴吧页面)")
|
||||
utils.logger.info("[TieBaCrawler] Successfully switched to new tab (Tieba page)")
|
||||
else:
|
||||
# 如果是同一标签页跳转,正常等待导航
|
||||
utils.logger.info("[TieBaCrawler] 链接在当前标签页跳转...")
|
||||
# If it's same tab navigation, wait for navigation normally
|
||||
utils.logger.info("[TieBaCrawler] Link navigates in current tab...")
|
||||
async with self.context_page.expect_navigation(wait_until="domcontentloaded"):
|
||||
await tieba_link.click()
|
||||
|
||||
# Step 5: 等待页面稳定,使用配置文件中的延时设置
|
||||
utils.logger.info(f"[TieBaCrawler] Step 5: 页面加载完成,等待 {config.CRAWLER_MAX_SLEEP_SEC}秒...")
|
||||
# Step 5: Wait for page to stabilize, using delay setting from config file
|
||||
utils.logger.info(f"[TieBaCrawler] Step 5: Page loaded, waiting {config.CRAWLER_MAX_SLEEP_SEC} seconds...")
|
||||
await asyncio.sleep(config.CRAWLER_MAX_SLEEP_SEC)
|
||||
|
||||
current_url = self.context_page.url
|
||||
utils.logger.info(f"[TieBaCrawler] ✅ 成功通过百度首页进入贴吧! 当前URL: {current_url}")
|
||||
utils.logger.info(f"[TieBaCrawler] Successfully entered Tieba via Baidu homepage! Current URL: {current_url}")
|
||||
|
||||
except Exception as e:
|
||||
utils.logger.error(f"[TieBaCrawler] 通过百度首页访问贴吧失败: {e}")
|
||||
utils.logger.info("[TieBaCrawler] 回退:直接访问贴吧首页")
|
||||
utils.logger.error(f"[TieBaCrawler] Failed to access Tieba via Baidu homepage: {e}")
|
||||
utils.logger.info("[TieBaCrawler] Fallback: directly accessing Tieba homepage")
|
||||
await self.context_page.goto(self.index_url, wait_until="domcontentloaded")
|
||||
|
||||
async def _inject_anti_detection_scripts(self):
|
||||
"""
|
||||
注入反检测JavaScript脚本
|
||||
针对百度贴吧的特殊检测机制
|
||||
Inject anti-detection JavaScript scripts
|
||||
For Baidu Tieba's special detection mechanism
|
||||
"""
|
||||
utils.logger.info("[TieBaCrawler] Injecting anti-detection scripts...")
|
||||
|
||||
# 轻量级反检测脚本,只覆盖关键检测点
|
||||
# Lightweight anti-detection script, only covering key detection points
|
||||
anti_detection_js = """
|
||||
// 覆盖 navigator.webdriver
|
||||
// Override navigator.webdriver
|
||||
Object.defineProperty(navigator, 'webdriver', {
|
||||
get: () => undefined,
|
||||
configurable: true
|
||||
});
|
||||
|
||||
// 覆盖 window.navigator.chrome
|
||||
// Override window.navigator.chrome
|
||||
if (!window.navigator.chrome) {
|
||||
window.navigator.chrome = {
|
||||
runtime: {},
|
||||
@@ -511,7 +511,7 @@ class TieBaCrawler(AbstractCrawler):
|
||||
};
|
||||
}
|
||||
|
||||
// 覆盖 Permissions API
|
||||
// Override Permissions API
|
||||
const originalQuery = window.navigator.permissions.query;
|
||||
window.navigator.permissions.query = (parameters) => (
|
||||
parameters.name === 'notifications' ?
|
||||
@@ -519,19 +519,19 @@ class TieBaCrawler(AbstractCrawler):
|
||||
originalQuery(parameters)
|
||||
);
|
||||
|
||||
// 覆盖 plugins 长度(让它看起来有插件)
|
||||
// Override plugins length (make it look like there are plugins)
|
||||
Object.defineProperty(navigator, 'plugins', {
|
||||
get: () => [1, 2, 3, 4, 5],
|
||||
configurable: true
|
||||
});
|
||||
|
||||
// 覆盖 languages
|
||||
// Override languages
|
||||
Object.defineProperty(navigator, 'languages', {
|
||||
get: () => ['zh-CN', 'zh', 'en'],
|
||||
configurable: true
|
||||
});
|
||||
|
||||
// 移除 window.cdc_ 等 ChromeDriver 残留
|
||||
// Remove window.cdc_ and other ChromeDriver remnants
|
||||
delete window.cdc_adoQpoasnfa76pfcZLmcfl_Array;
|
||||
delete window.cdc_adoQpoasnfa76pfcZLmcfl_Promise;
|
||||
delete window.cdc_adoQpoasnfa76pfcZLmcfl_Symbol;
|
||||
@@ -548,21 +548,21 @@ class TieBaCrawler(AbstractCrawler):
|
||||
"""
|
||||
Create tieba client with real browser User-Agent and complete headers
|
||||
Args:
|
||||
httpx_proxy: HTTP代理
|
||||
ip_pool: IP代理池
|
||||
httpx_proxy: HTTP proxy
|
||||
ip_pool: IP proxy pool
|
||||
|
||||
Returns:
|
||||
BaiduTieBaClient实例
|
||||
BaiduTieBaClient instance
|
||||
"""
|
||||
utils.logger.info("[TieBaCrawler.create_tieba_client] Begin create tieba API client...")
|
||||
|
||||
# 从真实浏览器提取User-Agent,避免被检测
|
||||
# Extract User-Agent from real browser to avoid detection
|
||||
user_agent = await self.context_page.evaluate("() => navigator.userAgent")
|
||||
utils.logger.info(f"[TieBaCrawler.create_tieba_client] Extracted User-Agent from browser: {user_agent}")
|
||||
|
||||
cookie_str, cookie_dict = utils.convert_cookies(await self.browser_context.cookies())
|
||||
|
||||
# 构建完整的浏览器请求头,模拟真实浏览器行为
|
||||
# Build complete browser request headers, simulating real browser behavior
|
||||
tieba_client = BaiduTieBaClient(
|
||||
timeout=10,
|
||||
ip_pool=ip_pool,
|
||||
@@ -572,7 +572,7 @@ class TieBaCrawler(AbstractCrawler):
|
||||
"Accept-Language": "zh-CN,zh;q=0.9",
|
||||
"Accept-Encoding": "gzip, deflate, br",
|
||||
"Connection": "keep-alive",
|
||||
"User-Agent": user_agent, # 使用真实浏览器的UA
|
||||
"User-Agent": user_agent, # Use real browser UA
|
||||
"Cookie": cookie_str,
|
||||
"Host": "tieba.baidu.com",
|
||||
"Referer": "https://tieba.baidu.com/",
|
||||
@@ -585,7 +585,7 @@ class TieBaCrawler(AbstractCrawler):
|
||||
"sec-ch-ua-mobile": "?0",
|
||||
"sec-ch-ua-platform": '"macOS"',
|
||||
},
|
||||
playwright_page=self.context_page, # 传入playwright页面对象
|
||||
playwright_page=self.context_page, # Pass in playwright page object
|
||||
)
|
||||
return tieba_client
|
||||
|
||||
@@ -623,7 +623,7 @@ class TieBaCrawler(AbstractCrawler):
|
||||
proxy=playwright_proxy, # type: ignore
|
||||
viewport={"width": 1920, "height": 1080},
|
||||
user_agent=user_agent,
|
||||
channel="chrome", # 使用系统的Chrome稳定版
|
||||
channel="chrome", # Use system's stable Chrome version
|
||||
)
|
||||
return browser_context
|
||||
else:
|
||||
@@ -641,7 +641,7 @@ class TieBaCrawler(AbstractCrawler):
|
||||
headless: bool = True,
|
||||
) -> BrowserContext:
|
||||
"""
|
||||
使用CDP模式启动浏览器
|
||||
Launch browser using CDP mode
|
||||
"""
|
||||
try:
|
||||
self.cdp_manager = CDPBrowserManager()
|
||||
@@ -652,15 +652,15 @@ class TieBaCrawler(AbstractCrawler):
|
||||
headless=headless,
|
||||
)
|
||||
|
||||
# 显示浏览器信息
|
||||
# Display browser information
|
||||
browser_info = await self.cdp_manager.get_browser_info()
|
||||
utils.logger.info(f"[TieBaCrawler] CDP浏览器信息: {browser_info}")
|
||||
utils.logger.info(f"[TieBaCrawler] CDP browser info: {browser_info}")
|
||||
|
||||
return browser_context
|
||||
|
||||
except Exception as e:
|
||||
utils.logger.error(f"[TieBaCrawler] CDP模式启动失败,回退到标准模式: {e}")
|
||||
# 回退到标准模式
|
||||
utils.logger.error(f"[TieBaCrawler] CDP mode launch failed, falling back to standard mode: {e}")
|
||||
# Fall back to standard mode
|
||||
chromium = playwright.chromium
|
||||
return await self.launch_browser(
|
||||
chromium, playwright_proxy, user_agent, headless
|
||||
@@ -672,7 +672,7 @@ class TieBaCrawler(AbstractCrawler):
|
||||
Returns:
|
||||
|
||||
"""
|
||||
# 如果使用CDP模式,需要特殊处理
|
||||
# If using CDP mode, need special handling
|
||||
if self.cdp_manager:
|
||||
await self.cdp_manager.cleanup()
|
||||
self.cdp_manager = None
|
||||
|
||||
@@ -23,16 +23,16 @@ from enum import Enum
|
||||
|
||||
class SearchSortType(Enum):
|
||||
"""search sort type"""
|
||||
# 按时间倒序
|
||||
# Sort by time in descending order
|
||||
TIME_DESC = "1"
|
||||
# 按时间顺序
|
||||
# Sort by time in ascending order
|
||||
TIME_ASC = "0"
|
||||
# 按相关性顺序
|
||||
# Sort by relevance
|
||||
RELEVANCE_ORDER = "2"
|
||||
|
||||
|
||||
class SearchNoteType(Enum):
|
||||
# 只看主题贴
|
||||
# Only view main posts
|
||||
MAIN_THREAD = "1"
|
||||
# 混合模式(帖子+回复)
|
||||
# Mixed mode (posts + replies)
|
||||
FIXED_THREAD = "0"
|
||||
|
||||
@@ -42,12 +42,12 @@ class TieBaExtractor:
|
||||
@staticmethod
|
||||
def extract_search_note_list(page_content: str) -> List[TiebaNote]:
|
||||
"""
|
||||
提取贴吧帖子列表,这里提取的关键词搜索结果页的数据,还缺少帖子的回复数和回复页等数据
|
||||
Extract Tieba post list from keyword search result pages, still missing reply count and reply page data
|
||||
Args:
|
||||
page_content: 页面内容的HTML字符串
|
||||
page_content: HTML string of page content
|
||||
|
||||
Returns:
|
||||
包含帖子信息的字典列表
|
||||
List of Tieba post objects
|
||||
"""
|
||||
xpath_selector = "//div[@class='s_post']"
|
||||
post_list = Selector(text=page_content).xpath(xpath_selector)
|
||||
@@ -71,12 +71,12 @@ class TieBaExtractor:
|
||||
|
||||
def extract_tieba_note_list(self, page_content: str) -> List[TiebaNote]:
|
||||
"""
|
||||
提取贴吧帖子列表
|
||||
Extract Tieba post list from Tieba page
|
||||
Args:
|
||||
page_content:
|
||||
page_content: HTML string of page content
|
||||
|
||||
Returns:
|
||||
|
||||
List of Tieba post objects
|
||||
"""
|
||||
page_content = page_content.replace('<!--', "")
|
||||
content_selector = Selector(text=page_content)
|
||||
@@ -106,21 +106,21 @@ class TieBaExtractor:
|
||||
|
||||
def extract_note_detail(self, page_content: str) -> TiebaNote:
|
||||
"""
|
||||
提取贴吧帖子详情
|
||||
Extract Tieba post details from post detail page
|
||||
Args:
|
||||
page_content:
|
||||
page_content: HTML string of page content
|
||||
|
||||
Returns:
|
||||
|
||||
Tieba post detail object
|
||||
"""
|
||||
content_selector = Selector(text=page_content)
|
||||
first_floor_selector = content_selector.xpath("//div[@class='p_postlist'][1]")
|
||||
only_view_author_link = content_selector.xpath("//*[@id='lzonly_cntn']/@href").get(default='').strip()
|
||||
note_id = only_view_author_link.split("?")[0].split("/")[-1]
|
||||
# 帖子回复数、回复页数
|
||||
# Post reply count and reply page count
|
||||
thread_num_infos = content_selector.xpath(
|
||||
"//div[@id='thread_theme_5']//li[@class='l_reply_num']//span[@class='red']")
|
||||
# IP地理位置、发表时间
|
||||
# IP location and publish time
|
||||
other_info_content = content_selector.xpath(".//div[@class='post-tail-wrap']").get(default="").strip()
|
||||
ip_location, publish_time = self.extract_ip_and_pub_time(other_info_content)
|
||||
note = TiebaNote(note_id=note_id, title=content_selector.xpath("//title/text()").get(default='').strip(),
|
||||
@@ -138,18 +138,18 @@ class TieBaExtractor:
|
||||
publish_time=publish_time,
|
||||
total_replay_num=thread_num_infos[0].xpath("./text()").get(default='').strip(),
|
||||
total_replay_page=thread_num_infos[1].xpath("./text()").get(default='').strip(), )
|
||||
note.title = note.title.replace(f"【{note.tieba_name}】_百度贴吧", "")
|
||||
note.title = note.title.replace(f"【{note.tieba_name}】_Baidu Tieba", "")
|
||||
return note
|
||||
|
||||
def extract_tieba_note_parment_comments(self, page_content: str, note_id: str) -> List[TiebaComment]:
|
||||
"""
|
||||
提取贴吧帖子一级评论
|
||||
Extract Tieba post first-level comments from comment page
|
||||
Args:
|
||||
page_content:
|
||||
note_id:
|
||||
page_content: HTML string of page content
|
||||
note_id: Post ID
|
||||
|
||||
Returns:
|
||||
|
||||
List of first-level comment objects
|
||||
"""
|
||||
xpath_selector = "//div[@class='l_post l_post_bright j_l_post clearfix ']"
|
||||
comment_list = Selector(text=page_content).xpath(xpath_selector)
|
||||
@@ -180,13 +180,13 @@ class TieBaExtractor:
|
||||
|
||||
def extract_tieba_note_sub_comments(self, page_content: str, parent_comment: TiebaComment) -> List[TiebaComment]:
|
||||
"""
|
||||
提取贴吧帖子二级评论
|
||||
Extract Tieba post second-level comments from sub-comment page
|
||||
Args:
|
||||
page_content:
|
||||
parent_comment:
|
||||
page_content: HTML string of page content
|
||||
parent_comment: Parent comment object
|
||||
|
||||
Returns:
|
||||
|
||||
List of second-level comment objects
|
||||
"""
|
||||
selector = Selector(page_content)
|
||||
comments = []
|
||||
@@ -215,12 +215,12 @@ class TieBaExtractor:
|
||||
|
||||
def extract_creator_info(self, html_content: str) -> TiebaCreator:
|
||||
"""
|
||||
提取贴吧创作者信息
|
||||
Extract Tieba creator information from creator homepage
|
||||
Args:
|
||||
html_content:
|
||||
html_content: HTML string of creator homepage
|
||||
|
||||
Returns:
|
||||
|
||||
Tieba creator object
|
||||
"""
|
||||
selector = Selector(text=html_content)
|
||||
user_link_selector = selector.xpath("//p[@class='space']/a")
|
||||
@@ -251,12 +251,12 @@ class TieBaExtractor:
|
||||
html_content: str
|
||||
) -> List[str]:
|
||||
"""
|
||||
提取贴吧创作者主页的帖子列表
|
||||
Extract post ID list from Tieba creator's homepage
|
||||
Args:
|
||||
html_content:
|
||||
html_content: HTML string of creator homepage
|
||||
|
||||
Returns:
|
||||
|
||||
List of post IDs
|
||||
"""
|
||||
selector = Selector(text=html_content)
|
||||
thread_id_list = []
|
||||
@@ -271,12 +271,12 @@ class TieBaExtractor:
|
||||
|
||||
def extract_ip_and_pub_time(self, html_content: str) -> Tuple[str, str]:
|
||||
"""
|
||||
提取IP位置和发布时间
|
||||
Extract IP location and publish time from HTML content
|
||||
Args:
|
||||
html_content:
|
||||
html_content: HTML string
|
||||
|
||||
Returns:
|
||||
|
||||
Tuple of (IP location, publish time)
|
||||
"""
|
||||
pattern_pub_time = re.compile(r'<span class="tail-info">(\d{4}-\d{2}-\d{2} \d{2}:\d{2})</span>')
|
||||
time_match = pattern_pub_time.search(html_content)
|
||||
@@ -286,12 +286,12 @@ class TieBaExtractor:
|
||||
@staticmethod
|
||||
def extract_ip(html_content: str) -> str:
|
||||
"""
|
||||
提取IP
|
||||
Extract IP location from HTML content
|
||||
Args:
|
||||
html_content:
|
||||
html_content: HTML string
|
||||
|
||||
Returns:
|
||||
|
||||
IP location string
|
||||
"""
|
||||
pattern_ip = re.compile(r'IP属地:(\S+)</span>')
|
||||
ip_match = pattern_ip.search(html_content)
|
||||
@@ -301,28 +301,28 @@ class TieBaExtractor:
|
||||
@staticmethod
|
||||
def extract_gender(html_content: str) -> str:
|
||||
"""
|
||||
提取性别
|
||||
Extract gender from HTML content
|
||||
Args:
|
||||
html_content:
|
||||
html_content: HTML string
|
||||
|
||||
Returns:
|
||||
|
||||
Gender string ('Male', 'Female', or 'Unknown')
|
||||
"""
|
||||
if GENDER_MALE in html_content:
|
||||
return '男'
|
||||
return 'Male'
|
||||
elif GENDER_FEMALE in html_content:
|
||||
return '女'
|
||||
return '未知'
|
||||
return 'Female'
|
||||
return 'Unknown'
|
||||
|
||||
@staticmethod
|
||||
def extract_follow_and_fans(selectors: List[Selector]) -> Tuple[str, str]:
|
||||
"""
|
||||
提取关注数和粉丝数
|
||||
Extract follow count and fan count from selectors
|
||||
Args:
|
||||
selectors:
|
||||
selectors: List of selector objects
|
||||
|
||||
Returns:
|
||||
|
||||
Tuple of (follow count, fan count)
|
||||
"""
|
||||
pattern = re.compile(r'<span class="concern_num">\(<a[^>]*>(\d+)</a>\)</span>')
|
||||
follow_match = pattern.findall(selectors[0].get())
|
||||
@@ -334,9 +334,15 @@ class TieBaExtractor:
|
||||
@staticmethod
|
||||
def extract_registration_duration(html_content: str) -> str:
|
||||
"""
|
||||
"<span>吧龄:1.9年</span>"
|
||||
Returns: 1.9年
|
||||
Extract Tieba age from HTML content
|
||||
Example: "<span>吧龄:1.9年</span>"
|
||||
Returns: "1.9年"
|
||||
|
||||
Args:
|
||||
html_content: HTML string
|
||||
|
||||
Returns:
|
||||
Tieba age string
|
||||
"""
|
||||
pattern = re.compile(r'<span>吧龄:(\S+)</span>')
|
||||
match = pattern.search(html_content)
|
||||
@@ -345,22 +351,22 @@ class TieBaExtractor:
|
||||
@staticmethod
|
||||
def extract_data_field_value(selector: Selector) -> Dict:
|
||||
"""
|
||||
提取data-field的值
|
||||
Extract data-field value from selector
|
||||
Args:
|
||||
selector:
|
||||
selector: Selector object
|
||||
|
||||
Returns:
|
||||
|
||||
Dictionary containing data-field value
|
||||
"""
|
||||
data_field_value = selector.xpath("./@data-field").get(default='').strip()
|
||||
if not data_field_value or data_field_value == "{}":
|
||||
return {}
|
||||
try:
|
||||
# 先使用 html.unescape 处理转义字符 再json.loads 将 JSON 字符串转换为 Python 字典
|
||||
# First use html.unescape to handle escape characters, then json.loads to convert JSON string to Python dictionary
|
||||
unescaped_json_str = html.unescape(data_field_value)
|
||||
data_field_dict_value = json.loads(unescaped_json_str)
|
||||
except Exception as ex:
|
||||
print(f"extract_data_field_value,错误信息:{ex}, 尝试使用其他方式解析")
|
||||
print(f"extract_data_field_value, error: {ex}, trying alternative parsing method")
|
||||
data_field_dict_value = {}
|
||||
return data_field_dict_value
|
||||
|
||||
|
||||
@@ -50,7 +50,7 @@ class BaiduTieBaLogin(AbstractLogin):
|
||||
@retry(stop=stop_after_attempt(600), wait=wait_fixed(1), retry=retry_if_result(lambda value: value is False))
|
||||
async def check_login_state(self) -> bool:
|
||||
"""
|
||||
轮训检查登录状态是否成功,成功返回True否则返回False
|
||||
Poll to check if login status is successful, return True if successful, otherwise return False
|
||||
|
||||
Returns:
|
||||
|
||||
|
||||
Reference in New Issue
Block a user