jiazhizhong/MediaCrawler
1
0
Fork 0
mirror of https://github.com/NanmiCoder/MediaCrawler.git synced 2026-06-09 11:27:26 +08:00
Code Issues Packages Projects Releases Wiki Activity

处理子评论获取失败导致整个流程中断问题

Browse Source 
Co-Authored-By: ouzhuowei <190020754@qq.com>
This commit is contained in:
ouzhuowei
2026-02-10 17:53:30 +08:00
parent 4ad065ce9a
commit e54463ac78
1 changed files with 53 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files

89
media_platform/xhs/client.py
View File

@@ -460,44 +460,61 @@ class XiaoHongShuClient(AbstractApiClient, ProxyRefreshMixin):
result = [] result = []
for comment in comments: for comment in comments:
note_id = comment.get("note_id") try:
sub_comments = comment.get("sub_comments") note_id = comment.get("note_id")
if sub_comments and callback: sub_comments = comment.get("sub_comments")
await callback(note_id, sub_comments) if sub_comments and callback:
await callback(note_id, sub_comments)
sub_comment_has_more = comment.get("sub_comment_has_more") sub_comment_has_more = comment.get("sub_comment_has_more")
if not sub_comment_has_more: if not sub_comment_has_more:
continue
root_comment_id = comment.get("id")
sub_comment_cursor = comment.get("sub_comment_cursor")
while sub_comment_has_more:
comments_res = await self.get_note_sub_comments(
note_id=note_id,
root_comment_id=root_comment_id,
xsec_token=xsec_token,
num=10,
cursor=sub_comment_cursor,
)
if comments_res is None:
utils.logger.info(
f"[XiaoHongShuClient.get_comments_all_sub_comments] No response found for note_id: {note_id}"
)
continue continue
sub_comment_has_more = comments_res.get("has_more", False)
sub_comment_cursor = comments_res.get("cursor", "") root_comment_id = comment.get("id")
if "comments" not in comments_res: sub_comment_cursor = comment.get("sub_comment_cursor")
utils.logger.info(
f"[XiaoHongShuClient.get_comments_all_sub_comments] No 'comments' key found in response: {comments_res}" while sub_comment_has_more:
) try:
break comments_res = await self.get_note_sub_comments(
comments = comments_res["comments"] note_id=note_id,
if callback: root_comment_id=root_comment_id,
await callback(note_id, comments) xsec_token=xsec_token,
await asyncio.sleep(crawl_interval) num=10,
result.extend(comments) cursor=sub_comment_cursor,
)
if comments_res is None:
utils.logger.info(
f"[XiaoHongShuClient.get_comments_all_sub_comments] No response found for note_id: {note_id}"
)
break
sub_comment_has_more = comments_res.get("has_more", False)
sub_comment_cursor = comments_res.get("cursor", "")
if "comments" not in comments_res:
utils.logger.info(
f"[XiaoHongShuClient.get_comments_all_sub_comments] No 'comments' key found in response: {comments_res}"
)
break
comments = comments_res["comments"]
if callback:
await callback(note_id, comments)
await asyncio.sleep(crawl_interval)
result.extend(comments)
except DataFetchError as e:
utils.logger.warning(
f"[XiaoHongShuClient.get_comments_all_sub_comments] Failed to get sub-comments for note_id: {note_id}, root_comment_id: {root_comment_id}, error: {e}. Skipping this comment's sub-comments."
)
break # 跳出当前评论的子评论获取循环，继续处理下一个评论
except Exception as e:
utils.logger.error(
f"[XiaoHongShuClient.get_comments_all_sub_comments] Unexpected error when getting sub-comments for note_id: {note_id}, root_comment_id: {root_comment_id}, error: {e}"
)
break
except Exception as e:
utils.logger.error(
f"[XiaoHongShuClient.get_comments_all_sub_comments] Error processing comment: {comment.get('id', 'unknown')}, error: {e}. Continuing with next comment."
)
continue # 继续处理下一个评论
return result return result
async def get_creator_info( async def get_creator_info(