jiazhizhong/MediaCrawler
1
0
Fork 0
mirror of https://github.com/NanmiCoder/MediaCrawler.git synced 2026-06-09 19:37:25 +08:00
Code Issues Packages Projects Releases Wiki Activity

添加功能:(哔哩哔哩,快手,小红书)每个视频/帖子抓取评论最大条数限制,评论关键词筛选

Browse Source
This commit is contained in:
peanutsplash
2023-12-13 23:53:12 +08:00
parent 5c42076ff8
commit f17a85305e
5 changed files with 68 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
media_platform/douyin/core.py
View File

@@ -132,7 +132,7 @@ class DouYinCrawler(AbstractCrawler):
semaphore = asyncio.Semaphore(config.MAX_CONCURRENCY_NUM)
for aweme_id in aweme_list:
task = asyncio.create_task(
self.get_comments(aweme_id, semaphore, max_comments=config.DY_MAX_COMMENTS_PER_POST), name=aweme_id)
self.get_comments(aweme_id, semaphore, max_comments=config.MAX_COMMENTS_PER_POST), name=aweme_id)
task_list.append(task)
await asyncio.wait(task_list)
@@ -143,7 +143,7 @@ class DouYinCrawler(AbstractCrawler):
comments = await self.dy_client.get_aweme_all_comments(
aweme_id=aweme_id,
max_comments=max_comments, # 最大数量
keywords=config.DY_COMMENT_KEYWORDS # 关键词列表
keywords=config.COMMENT_KEYWORDS # 关键词列表
)
# 现在返回的 comments 已经是经过关键词筛选的
await douyin.batch_update_dy_aweme_comments(aweme_id, comments)