mirror of
https://github.com/NanmiCoder/MediaCrawler.git
synced 2026-06-06 09:57:25 +08:00
feat: 小红书支持通过博主ID采集笔记和评论,小红书type=search时支持配置按哪种排序方式获取笔记数据,小红书笔记增加视频地址和标签字段
This commit is contained in:
@@ -31,12 +31,20 @@ async def update_xhs_note(note_item: Dict):
|
||||
user_info = note_item.get("user", {})
|
||||
interact_info = note_item.get("interact_info", {})
|
||||
image_list: List[Dict] = note_item.get("image_list", [])
|
||||
tag_list: List[Dict] = note_item.get("tag_list", [])
|
||||
|
||||
video_url = ''
|
||||
if note_item.get('type') == 'video':
|
||||
videos = note_item.get('video').get('media').get('stream').get('h264')
|
||||
if type(videos).__name__ == 'list':
|
||||
video_url = ','.join([ v.get('master_url') for v in videos])
|
||||
|
||||
local_db_item = {
|
||||
"note_id": note_item.get("note_id"),
|
||||
"type": note_item.get("type"),
|
||||
"title": note_item.get("title") or note_item.get("desc", "")[:255],
|
||||
"desc": note_item.get("desc", ""),
|
||||
"video_url": video_url,
|
||||
"time": note_item.get("time"),
|
||||
"last_update_time": note_item.get("last_update_time", 0),
|
||||
"user_id": user_info.get("user_id"),
|
||||
@@ -48,6 +56,7 @@ async def update_xhs_note(note_item: Dict):
|
||||
"share_count": interact_info.get("share_count"),
|
||||
"ip_location": note_item.get("ip_location", ""),
|
||||
"image_list": ','.join([img.get('url', '') for img in image_list]),
|
||||
"tag_list": ','.join([tag.get('name', '') for tag in tag_list if tag.get('type')=='topic']),
|
||||
"last_modify_ts": utils.get_current_timestamp(),
|
||||
"note_url": f"https://www.xiaohongshu.com/explore/{note_id}"
|
||||
}
|
||||
@@ -77,3 +86,32 @@ async def update_xhs_note_comment(note_id: str, comment_item: Dict):
|
||||
}
|
||||
utils.logger.info(f"[store.xhs.update_xhs_note_comment] xhs note comment:{local_db_item}")
|
||||
await XhsStoreFactory.create_store().store_comment(local_db_item)
|
||||
|
||||
async def save_creator(user_id: str, creator: Dict):
|
||||
user_info = creator.get('basicInfo', {})
|
||||
|
||||
follows = 0
|
||||
fans = 0
|
||||
interaction = 0
|
||||
for i in creator.get('interactions'):
|
||||
if i.get('type') == 'follows':
|
||||
follows = i.get('count')
|
||||
elif i.get('type') == 'fans':
|
||||
fans = i.get('count')
|
||||
elif i.get('type') == 'interaction':
|
||||
interaction = i.get('count')
|
||||
|
||||
local_db_item = {
|
||||
'user_id': user_id,
|
||||
'nickname': user_info.get('nickname'),
|
||||
'gender': '女' if user_info.get('gender') == 1 else '男' ,
|
||||
'avatar': user_info.get('images'),
|
||||
'desc': user_info.get('desc'),
|
||||
'ip_location': user_info.get('ip_location'),
|
||||
'follows': follows,
|
||||
'fans': fans,
|
||||
'interaction': interaction,
|
||||
'tag_list': json.dumps({tag.get('tagType'):tag.get('name') for tag in creator.get('tags')}),
|
||||
}
|
||||
utils.logger.info(f"[store.xhs.save_creator] creator:{local_db_item}")
|
||||
await XhsStoreFactory.create_store().store_creator(local_db_item)
|
||||
Reference in New Issue
Block a user