jiazhizhong/MediaCrawler
1
0
Fork 0
mirror of https://github.com/NanmiCoder/MediaCrawler.git synced 2026-06-07 10:27:25 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix:翻页时search id不变

Browse Source
This commit is contained in:
liugangdao
2024-10-24 13:57:27 +08:00
parent fa2bcc4181
commit 78c09c4ae1
2 changed files with 5 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
media_platform/xhs/core.py
View File

@@ -31,7 +31,7 @@ from var import crawler_type_var, source_keyword_var
from .client import XiaoHongShuClient
from .exception import DataFetchError
from .field import SearchSortType
from .help import parse_note_info_from_note_url
from .help import parse_note_info_from_note_url, get_search_id
from .login import XiaoHongShuLogin
@@ -112,6 +112,7 @@ class XiaoHongShuCrawler(AbstractCrawler):
source_keyword_var.set(keyword)
utils.logger.info(f"[XiaoHongShuCrawler.search] Current search keyword: {keyword}")
page = 1
search_id = get_search_id()
while (page - start_page + 1) * xhs_limit_count <= config.CRAWLER_MAX_NOTES_COUNT:
if page < start_page:
utils.logger.info(f"[XiaoHongShuCrawler.search] Skip page {page}")
@@ -123,6 +124,7 @@ class XiaoHongShuCrawler(AbstractCrawler):
note_id_list: List[str] = []
notes_res = await self.xhs_client.get_note_by_keyword(
keyword=keyword,
search_id=search_id,
page=page,
sort=SearchSortType(config.SORT_TYPE) if config.SORT_TYPE != '' else SearchSortType.GENERAL,
)