jiazhizhong/MediaCrawler
1
0
Fork 0
mirror of https://github.com/NanmiCoder/MediaCrawler.git synced 2026-06-06 18:07:26 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix: 修复抖音关键词搜索bug

Browse Source
This commit is contained in:
Relakkes
2024-08-20 03:09:42 +08:00
parent 2ab14920a3
commit 04cbe549af
2 changed files with 8 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
media_platform/douyin/core.py
View File

@@ -83,6 +83,7 @@ class DouYinCrawler(AbstractCrawler):
utils.logger.info(f"[DouYinCrawler.search] Current keyword: {keyword}")
aweme_list: List[str] = []
page = 0
dy_search_id = ""
while (page - start_page + 1) * dy_limit_count <= config.CRAWLER_MAX_NOTES_COUNT:
if page < start_page:
utils.logger.info(f"[DouYinCrawler.search] Skip {page}")
@@ -92,7 +93,8 @@ class DouYinCrawler(AbstractCrawler):
utils.logger.info(f"[DouYinCrawler.search] search douyin keyword: {keyword}, page: {page}")
posts_res = await self.dy_client.search_info_by_keyword(keyword=keyword,
offset=page * dy_limit_count - dy_limit_count,
publish_time=PublishTimeType(config.PUBLISH_TIME_TYPE)
publish_time=PublishTimeType(config.PUBLISH_TIME_TYPE),
search_id=dy_search_id
)
except DataFetchError:
utils.logger.error(f"[DouYinCrawler.search] search douyin keyword: {keyword} failed")
@@ -103,7 +105,7 @@ class DouYinCrawler(AbstractCrawler):
utils.logger.error(
f"[DouYinCrawler.search] search douyin keyword: {keyword} failed，账号也许被风控了。")
break
dy_search_id = posts_res.get("extra", {}).get("logid", "")
for post_item in posts_res.get("data"):
try:
aweme_info: Dict = post_item.get("aweme_info") or \