jiazhizhong/MediaCrawler
1
0
Fork 0
mirror of https://github.com/NanmiCoder/MediaCrawler.git synced 2026-06-09 11:27:26 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: Bilibili comment done

Browse Source
This commit is contained in:
Relakkes
2023-12-09 21:10:01 +08:00
parent a136ea2739
commit 97d7a0c38b
10 changed files with 199 additions and 58 deletions
Download Patch File Download Diff File Expand all files Collapse all files

32
media_platform/bilibili/client.py
View File

@@ -77,9 +77,10 @@ class BilibiliClient:
sub_key = sub_url.rsplit('/', 1)[1].split('.')[0]
return img_key, sub_key
async def get(self, uri: str, params=None) -> Dict:
async def get(self, uri: str, params=None, enable_params_sign: bool = True) -> Dict:
final_uri = uri
params = await self.pre_request_data(params)
if enable_params_sign:
params = await self.pre_request_data(params)
if isinstance(params, dict):
final_uri = (f"{uri}?"
f"{urlencode(params)}")
@@ -102,7 +103,7 @@ class BilibiliClient:
utils.logger.info("use cache login state get web interface successfull!")
ping_flag = True
except Exception as e:
utils.logger.error(f"Pong kuaishou failed: {e}, and try to login again...")
utils.logger.error(f"Pong bilibili failed: {e}, and try to login again...")
ping_flag = False
return ping_flag
@@ -133,23 +134,25 @@ class BilibiliClient:
async def get_video_info(self, video_id: str) -> Dict:
"""
Kuaishou web video detail api
Bilibli web video detail api
:param video_id:
:return:
"""
post_data = {
uri = "/x/web-interface/view/detail"
params = {
"aid": video_id
}
return await self.post("", post_data)
return await self.get(uri, params, enable_params_sign=False)
async def get_video_comments(self,
video_id: str,
order_mode: CommentOrderType = CommentOrderType.DEFAULT,
pagination_str: str = '{"offset":""}'
next: int = 0
) -> Dict:
"""get video comments
:param video_id: 视频 ID
:param order_mode: 排序方式
:param pagination_str: 分页字符串，由 api 返回下一个评论请求的分页信息
:param next: 评论页选择
:return:
"""
uri = "/x/v2/reply/wbi/main"
@@ -157,9 +160,10 @@ class BilibiliClient:
"oid": video_id,
"mode": order_mode.value,
"type": 1,
"pagination_str": pagination_str
"ps": 20,
"next": next
}
return await self.post(uri, post_data)
return await self.get(uri, post_data)
async def get_video_all_comments(self, video_id: str, crawl_interval: float = 1.0, is_fetch_sub_comments=False,
callback: Optional[Callable] = None, ):
@@ -174,13 +178,13 @@ class BilibiliClient:
result = []
is_end = False
pagination_str = '{"offset":""}'
next_page =0
while not is_end:
comments_res = await self.get_video_comments(video_id, CommentOrderType.DEFAULT, pagination_str)
curson_info: Dict = comments_res.get("data").get("cursor")
comments_res = await self.get_video_comments(video_id, CommentOrderType.DEFAULT, next_page)
curson_info: Dict = comments_res.get("cursor")
comment_list: List[Dict] = comments_res.get("replies", [])
is_end = curson_info.get("is_end")
pagination_str = curson_info.get("pagination_reply").get("next_offset")
next_page = curson_info.get("next")
if callback: # 如果有回调函数，就执行回调函数
await callback(video_id, comment_list)
await asyncio.sleep(crawl_interval)

82
media_platform/bilibili/core.py
View File

@@ -16,11 +16,12 @@ from playwright.async_api import (BrowserContext, BrowserType, Page,
import config
from base.base_crawler import AbstractCrawler
from models import bilibili
from proxy.proxy_ip_pool import create_ip_pool, IpInfoModel
from proxy.proxy_ip_pool import IpInfoModel, create_ip_pool
from tools import utils
from var import comment_tasks_var, crawler_type_var
from .client import BilibiliClient
from .exception import DataFetchError
from .field import SearchOrderType
from .login import BilibiliLogin
@@ -107,17 +108,88 @@ class BilibiliCrawler(AbstractCrawler):
order=SearchOrderType.DEFAULT,
)
video_list: List[Dict] = videos_res.get("result")
for video_item in video_list:
video_id_list.append(video_item.get("id"))
await bilibili.update_bilibili_video(video_item)
semaphore = asyncio.Semaphore(config.MAX_CONCURRENCY_NUM)
task_list = [
self.get_video_info_task(video_item.get("aid"), semaphore)
for video_item in video_list
]
video_items = await asyncio.gather(*task_list)
for video_item in video_items:
if video_item:
video_id_list.append(video_item.get("View").get("aid"))
await bilibili.update_bilibili_video(video_item)
page += 1
await self.batch_get_video_comments(video_id_list)
async def batch_get_video_comments(self, video_id_list: List[str]):
"""
batch get video comments
:param video_id_list:
:return:
"""
utils.logger.info(f"[batch_get_video_comments] video ids:{video_id_list}")
semaphore = asyncio.Semaphore(config.MAX_CONCURRENCY_NUM)
task_list: List[Task] = []
for video_id in video_id_list:
task = asyncio.create_task(self.get_comments(video_id, semaphore), name=video_id)
task_list.append(task)
await asyncio.gather(*task_list)
async def get_comments(self, video_id: str, semaphore: asyncio.Semaphore):
"""
get comment for video id
:param video_id:
:param semaphore:
:return:
"""
async with semaphore:
try:
utils.logger.info(f"[get_comments] bengin get video_id: {video_id} comments ...")
await self.bili_client.get_video_all_comments(
video_id=video_id,
crawl_interval=random.random(),
callback=bilibili.batch_update_bilibili_video_comments
)
except DataFetchError as ex:
utils.logger.error(f"[get_comments] get video_id: {video_id} comment error: {ex}")
except Exception as e:
utils.logger.error(f"[get_comments] may be been blocked, err:", e)
async def get_specified_videos(self):
"""
get specified videos info
:return:
"""
pass
semaphore = asyncio.Semaphore(config.MAX_CONCURRENCY_NUM)
task_list = [
self.get_video_info_task(video_id=video_id, semaphore=semaphore) for video_id in config.BILI_SPECIFIED_ID_LIST
]
video_details = await asyncio.gather(*task_list)
for video_detail in video_details:
if video_detail is not None:
await bilibili.update_bilibili_video(video_detail)
await self.batch_get_video_comments(config.BILI_SPECIFIED_ID_LIST)
async def get_video_info_task(self, video_id: str, semaphore: asyncio.Semaphore) -> Optional[Dict]:
"""
Get video detail task
:param video_id:
:param semaphore:
:return:
"""
async with semaphore:
try:
result = await self.bili_client.get_video_info(video_id)
return result
except DataFetchError as ex:
utils.logger.error(f"Get video detail error: {ex}")
return None
except KeyError as ex:
utils.logger.error(f"have not fund note detail video_id:{video_id}, err: {ex}")
return None
async def create_bilibili_client(self, httpx_proxy: Optional[str]) -> BilibiliClient:
"""Create xhs client"""

5
media_platform/bilibili/help.py
View File

@@ -1,4 +1,4 @@
# -*- coding: utf-8 -*-
# -*- coding: utf-8 -*-
# @Author : relakkes@gmail.com
# @Time : 2023/12/2 23:26
# @Desc : bilibili 请求参数签名
@@ -52,7 +52,6 @@ class BilibiliSign:
salt = self.get_salt()
wbi_sign = md5((query + salt).encode()).hexdigest() # 计算 w_rid
req_data['w_rid'] = wbi_sign
print(urllib.parse.urlencode(req_data))
return req_data
@@ -67,5 +66,5 @@ if __name__ == '__main__':
value = kvalues[1]
_req_data[key] = value
print("pre req_data", _req_data)
_req_data = BilibiliSign(img_key=_img_key, sub_key=_sub_key).sign(req_data=_req_data)
_req_data = BilibiliSign(img_key=_img_key, sub_key=_sub_key).sign(req_data={"aid":170001})
print(_req_data)

2
media_platform/douyin/core.py
View File

@@ -9,7 +9,7 @@ from playwright.async_api import (BrowserContext, BrowserType, Page,
import config
from base.base_crawler import AbstractCrawler
from models import douyin
from proxy.proxy_ip_pool import create_ip_pool, IpInfoModel
from proxy.proxy_ip_pool import IpInfoModel, create_ip_pool
from tools import utils
from var import crawler_type_var

2
media_platform/kuaishou/core.py
View File

@@ -11,7 +11,7 @@ from playwright.async_api import (BrowserContext, BrowserType, Page,
import config
from base.base_crawler import AbstractCrawler
from models import kuaishou
from proxy.proxy_ip_pool import create_ip_pool, IpInfoModel
from proxy.proxy_ip_pool import IpInfoModel, create_ip_pool
from tools import utils
from var import comment_tasks_var, crawler_type_var

2
media_platform/xhs/core.py
View File

@@ -10,7 +10,7 @@ from playwright.async_api import (BrowserContext, BrowserType, Page,
import config
from base.base_crawler import AbstractCrawler
from models import xiaohongshu as xhs_model
from proxy.proxy_ip_pool import create_ip_pool, IpInfoModel
from proxy.proxy_ip_pool import IpInfoModel, create_ip_pool
from tools import utils
from var import crawler_type_var