jiazhizhong/MediaCrawler
1
0
Fork 0
mirror of https://github.com/NanmiCoder/MediaCrawler.git synced 2026-02-06 15:11:12 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: xhs comments add xsec_token

Browse Source
This commit is contained in:
leantli
2024-12-03 18:25:21 +08:00
parent 6001fc8a52
commit e830ada574
2 changed files with 61 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

42
media_platform/xhs/core.py
View File

@@ -135,7 +135,8 @@ class XiaoHongShuCrawler(AbstractCrawler):
utils.logger.info(
f"[XiaoHongShuCrawler.search] search xhs keyword: {keyword}, page: {page}"
)
note_id_list: List[str] = []
note_ids: List[str] = []
xsec_tokens: List[str] = []
notes_res = await self.xhs_client.get_note_by_keyword(
keyword=keyword,
search_id=search_id,
@@ -168,12 +169,13 @@ class XiaoHongShuCrawler(AbstractCrawler):
if note_detail:
await xhs_store.update_xhs_note(note_detail)
await self.get_notice_media(note_detail)
note_id_list.append(note_detail.get("note_id"))
note_ids.append(note_detail.get("note_id"))
xsec_tokens.append(note_detail.get("xsec_token"))
page += 1
utils.logger.info(
f"[XiaoHongShuCrawler.search] Note details: {note_details}"
)
await self.batch_get_note_comments(note_id_list)
await self.batch_get_note_comments(note_ids, xsec_tokens)
except DataFetchError:
utils.logger.error(
"[XiaoHongShuCrawler.search] Get note detail error"
@@ -200,8 +202,12 @@ class XiaoHongShuCrawler(AbstractCrawler):
callback=self.fetch_creator_notes_detail,
)
note_ids = [note_item.get("note_id") for note_item in all_notes_list]
await self.batch_get_note_comments(note_ids)
note_ids = []
xsec_tokens = []
for note_item in all_notes_list:
note_ids.append(note_item.get("note_id"))
xsec_tokens.append(note_item.get("xsec_token"))
await self.batch_get_note_comments(note_ids, xsec_tokens)
async def fetch_creator_notes_detail(self, note_list: List[Dict]):
"""
@@ -245,12 +251,14 @@ class XiaoHongShuCrawler(AbstractCrawler):
get_note_detail_task_list.append(crawler_task)
need_get_comment_note_ids = []
xsec_tokens = []
note_details = await asyncio.gather(*get_note_detail_task_list)
for note_detail in note_details:
if note_detail:
need_get_comment_note_ids.append(note_detail.get("note_id", ""))
xsec_tokens.append(note_detail.get("xsec_token", ""))
await xhs_store.update_xhs_note(note_detail)
await self.batch_get_note_comments(need_get_comment_note_ids)
await self.batch_get_note_comments(need_get_comment_note_ids, xsec_tokens)
async def get_note_detail_async_task(
self,
@@ -291,8 +299,10 @@ class XiaoHongShuCrawler(AbstractCrawler):
)
if not note_detail_from_html:
# 如果网页版笔记详情获取失败，则尝试API获取
note_detail_from_api: Optional[Dict] = await self.xhs_client.get_note_by_id(
note_id, xsec_source, xsec_token
note_detail_from_api: Optional[Dict] = (
await self.xhs_client.get_note_by_id(
note_id, xsec_source, xsec_token
)
)
note_detail = note_detail_from_html or note_detail_from_api
if note_detail:
@@ -311,7 +321,9 @@ class XiaoHongShuCrawler(AbstractCrawler):
)
return None
async def batch_get_note_comments(self, note_list: List[str]):
async def batch_get_note_comments(
self, note_list: List[str], xsec_tokens: List[str]
):
"""Batch get note comments"""
if not config.ENABLE_GET_COMMENTS:
utils.logger.info(
@@ -324,14 +336,19 @@ class XiaoHongShuCrawler(AbstractCrawler):
)
semaphore = asyncio.Semaphore(config.MAX_CONCURRENCY_NUM)
task_list: List[Task] = []
for note_id in note_list:
for index, note_id in enumerate(note_list):
task = asyncio.create_task(
self.get_comments(note_id, semaphore), name=note_id
self.get_comments(
note_id=note_id, xsec_token=xsec_tokens[index], semaphore=semaphore
),
name=note_id,
)
task_list.append(task)
await asyncio.gather(*task_list)
async def get_comments(self, note_id: str, semaphore: asyncio.Semaphore):
async def get_comments(
self, note_id: str, xsec_token: str, semaphore: asyncio.Semaphore
):
"""Get note comments with keyword filtering and quantity limitation"""
async with semaphore:
utils.logger.info(
@@ -339,6 +356,7 @@ class XiaoHongShuCrawler(AbstractCrawler):
)
await self.xhs_client.get_note_all_comments(
note_id=note_id,
xsec_token=xsec_token,
crawl_interval=random.random(),
callback=xhs_store.batch_update_xhs_note_comments,
max_count=CRAWLER_MAX_COMMENTS_COUNT_SINGLENOTES,