jiazhizhong/MediaCrawler
1
0
Fork 0
mirror of https://github.com/NanmiCoder/MediaCrawler.git synced 2026-04-21 11:17:38 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: logger record current search page

Browse Source
This commit is contained in:
Relakkes
2024-06-24 22:24:51 +08:00
parent a0e5a29af8
commit d3eeccbaac
8 changed files with 6 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
media_platform/bilibili/core.py
View File

@@ -106,6 +106,7 @@ class BilibiliCrawler(AbstractCrawler):
page += 1
continue
utils.logger.info(f"[BilibiliCrawler.search] search bilibili keyword: {keyword}, page: {page}")
video_id_list: List[str] = []
videos_res = await self.bili_client.search_video_by_keyword(
keyword=keyword,
@@ -126,7 +127,6 @@ class BilibiliCrawler(AbstractCrawler):
if video_item:
video_id_list.append(video_item.get("View").get("aid"))
await bilibili_store.update_bilibili_video(video_item)
page += 1
await self.batch_get_video_comments(video_id_list)

1
media_platform/douyin/core.py
View File

@@ -90,6 +90,7 @@ class DouYinCrawler(AbstractCrawler):
page += 1
continue
try:
utils.logger.info(f"[DouYinCrawler.search] search douyin keyword: {keyword}, page: {page}")
posts_res = await self.dy_client.search_info_by_keyword(keyword=keyword,
offset=page * dy_limit_count - dy_limit_count,
publish_time=PublishTimeType(config.PUBLISH_TIME_TYPE)

2
media_platform/kuaishou/core.py
View File

@@ -92,7 +92,7 @@ class KuaishouCrawler(AbstractCrawler):
utils.logger.info(f"[KuaishouCrawler.search] Skip page: {page}")
page += 1
continue
utils.logger.info(f"[KuaishouCrawler.search] search kuaishou keyword: {keyword}, page: {page}")
video_id_list: List[str] = []
videos_res = await self.ks_client.search_info_by_keyword(
keyword=keyword,

2
media_platform/weibo/core.py
View File

@@ -108,7 +108,7 @@ class WeiboCrawler(AbstractCrawler):
utils.logger.info(f"[WeiboCrawler.search] Skip page: {page}")
page += 1
continue
utils.logger.info(f"[WeiboCrawler.search] search weibo keyword: {keyword}, page: {page}")
search_res = await self.wb_client.get_note_by_keyword(
keyword=keyword,
page=page,

1
media_platform/xhs/core.py
View File

@@ -102,6 +102,7 @@ class XiaoHongShuCrawler(AbstractCrawler):
continue
try:
utils.logger.info(f"[XiaoHongShuCrawler.search] search xhs keyword: {keyword}, page: {page}")
note_id_list: List[str] = []
notes_res = await self.xhs_client.get_note_by_keyword(
keyword=keyword,